1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Local APIC virtualization 5 * 6 * Copyright (C) 2006 Qumranet, Inc. 7 * Copyright (C) 2007 Novell 8 * Copyright (C) 2007 Intel 9 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Dor Laor <dor.laor@qumranet.com> 13 * Gregory Haskins <ghaskins@novell.com> 14 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 15 * 16 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 17 */ 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/kvm_host.h> 21 #include <linux/kvm.h> 22 #include <linux/mm.h> 23 #include <linux/highmem.h> 24 #include <linux/smp.h> 25 #include <linux/hrtimer.h> 26 #include <linux/io.h> 27 #include <linux/export.h> 28 #include <linux/math64.h> 29 #include <linux/slab.h> 30 #include <asm/processor.h> 31 #include <asm/mce.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "ioapic.h" 42 #include "trace.h" 43 #include "x86.h" 44 #include "cpuid.h" 45 #include "hyperv.h" 46 #include "smm.h" 47 48 #ifndef CONFIG_X86_64 49 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 50 #else 51 #define mod_64(x, y) ((x) % (y)) 52 #endif 53 54 /* 14 is the version for Xeon and Pentium 8.4.8*/ 55 #define APIC_VERSION 0x14UL 56 #define LAPIC_MMIO_LENGTH (1 << 12) 57 /* followed define is not in apicdef.h */ 58 #define MAX_APIC_VECTOR 256 59 #define APIC_VECTORS_PER_REG 32 60 61 static bool lapic_timer_advance_dynamic __read_mostly; 62 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN 100 /* clock cycles */ 63 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX 10000 /* clock cycles */ 64 #define LAPIC_TIMER_ADVANCE_NS_INIT 1000 65 #define LAPIC_TIMER_ADVANCE_NS_MAX 5000 66 /* step-by-step approximation to mitigate fluctuation */ 67 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8 68 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data); 69 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data); 70 71 static inline void __kvm_lapic_set_reg(char *regs, int reg_off, u32 val) 72 { 73 *((u32 *) (regs + reg_off)) = val; 74 } 75 76 static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) 77 { 78 __kvm_lapic_set_reg(apic->regs, reg_off, val); 79 } 80 81 static __always_inline u64 __kvm_lapic_get_reg64(char *regs, int reg) 82 { 83 BUILD_BUG_ON(reg != APIC_ICR); 84 return *((u64 *) (regs + reg)); 85 } 86 87 static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg) 88 { 89 return __kvm_lapic_get_reg64(apic->regs, reg); 90 } 91 92 static __always_inline void __kvm_lapic_set_reg64(char *regs, int reg, u64 val) 93 { 94 BUILD_BUG_ON(reg != APIC_ICR); 95 *((u64 *) (regs + reg)) = val; 96 } 97 98 static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, 99 int reg, u64 val) 100 { 101 __kvm_lapic_set_reg64(apic->regs, reg, val); 102 } 103 104 static inline int apic_test_vector(int vec, void *bitmap) 105 { 106 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 107 } 108 109 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 110 { 111 struct kvm_lapic *apic = vcpu->arch.apic; 112 113 return apic_test_vector(vector, apic->regs + APIC_ISR) || 114 apic_test_vector(vector, apic->regs + APIC_IRR); 115 } 116 117 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 118 { 119 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 120 } 121 122 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 123 { 124 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 125 } 126 127 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ); 128 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ); 129 130 static inline int apic_enabled(struct kvm_lapic *apic) 131 { 132 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 133 } 134 135 #define LVT_MASK \ 136 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 137 138 #define LINT_MASK \ 139 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 140 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 141 142 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) 143 { 144 return apic->vcpu->vcpu_id; 145 } 146 147 static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu) 148 { 149 return pi_inject_timer && kvm_vcpu_apicv_active(vcpu) && 150 (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm)); 151 } 152 153 bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu) 154 { 155 return kvm_x86_ops.set_hv_timer 156 && !(kvm_mwait_in_guest(vcpu->kvm) || 157 kvm_can_post_timer_interrupt(vcpu)); 158 } 159 160 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu) 161 { 162 return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE; 163 } 164 165 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id) 166 { 167 return ((id >> 4) << 16) | (1 << (id & 0xf)); 168 } 169 170 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, 171 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { 172 switch (map->logical_mode) { 173 case KVM_APIC_MODE_SW_DISABLED: 174 /* Arbitrarily use the flat map so that @cluster isn't NULL. */ 175 *cluster = map->xapic_flat_map; 176 *mask = 0; 177 return true; 178 case KVM_APIC_MODE_X2APIC: { 179 u32 offset = (dest_id >> 16) * 16; 180 u32 max_apic_id = map->max_apic_id; 181 182 if (offset <= max_apic_id) { 183 u8 cluster_size = min(max_apic_id - offset + 1, 16U); 184 185 offset = array_index_nospec(offset, map->max_apic_id + 1); 186 *cluster = &map->phys_map[offset]; 187 *mask = dest_id & (0xffff >> (16 - cluster_size)); 188 } else { 189 *mask = 0; 190 } 191 192 return true; 193 } 194 case KVM_APIC_MODE_XAPIC_FLAT: 195 *cluster = map->xapic_flat_map; 196 *mask = dest_id & 0xff; 197 return true; 198 case KVM_APIC_MODE_XAPIC_CLUSTER: 199 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf]; 200 *mask = dest_id & 0xf; 201 return true; 202 case KVM_APIC_MODE_MAP_DISABLED: 203 return false; 204 default: 205 WARN_ON_ONCE(1); 206 return false; 207 } 208 } 209 210 static void kvm_apic_map_free(struct rcu_head *rcu) 211 { 212 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu); 213 214 kvfree(map); 215 } 216 217 static int kvm_recalculate_phys_map(struct kvm_apic_map *new, 218 struct kvm_vcpu *vcpu, 219 bool *xapic_id_mismatch) 220 { 221 struct kvm_lapic *apic = vcpu->arch.apic; 222 u32 x2apic_id = kvm_x2apic_id(apic); 223 u32 xapic_id = kvm_xapic_id(apic); 224 u32 physical_id; 225 226 /* 227 * For simplicity, KVM always allocates enough space for all possible 228 * xAPIC IDs. Yell, but don't kill the VM, as KVM can continue on 229 * without the optimized map. 230 */ 231 if (WARN_ON_ONCE(xapic_id > new->max_apic_id)) 232 return -EINVAL; 233 234 /* 235 * Bail if a vCPU was added and/or enabled its APIC between allocating 236 * the map and doing the actual calculations for the map. Note, KVM 237 * hardcodes the x2APIC ID to vcpu_id, i.e. there's no TOCTOU bug if 238 * the compiler decides to reload x2apic_id after this check. 239 */ 240 if (x2apic_id > new->max_apic_id) 241 return -E2BIG; 242 243 /* 244 * Deliberately truncate the vCPU ID when detecting a mismatched APIC 245 * ID to avoid false positives if the vCPU ID, i.e. x2APIC ID, is a 246 * 32-bit value. Any unwanted aliasing due to truncation results will 247 * be detected below. 248 */ 249 if (!apic_x2apic_mode(apic) && xapic_id != (u8)vcpu->vcpu_id) 250 *xapic_id_mismatch = true; 251 252 /* 253 * Apply KVM's hotplug hack if userspace has enable 32-bit APIC IDs. 254 * Allow sending events to vCPUs by their x2APIC ID even if the target 255 * vCPU is in legacy xAPIC mode, and silently ignore aliased xAPIC IDs 256 * (the x2APIC ID is truncated to 8 bits, causing IDs > 0xff to wrap 257 * and collide). 258 * 259 * Honor the architectural (and KVM's non-optimized) behavior if 260 * userspace has not enabled 32-bit x2APIC IDs. Each APIC is supposed 261 * to process messages independently. If multiple vCPUs have the same 262 * effective APIC ID, e.g. due to the x2APIC wrap or because the guest 263 * manually modified its xAPIC IDs, events targeting that ID are 264 * supposed to be recognized by all vCPUs with said ID. 265 */ 266 if (vcpu->kvm->arch.x2apic_format) { 267 /* See also kvm_apic_match_physical_addr(). */ 268 if (apic_x2apic_mode(apic) || x2apic_id > 0xff) 269 new->phys_map[x2apic_id] = apic; 270 271 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) 272 new->phys_map[xapic_id] = apic; 273 } else { 274 /* 275 * Disable the optimized map if the physical APIC ID is already 276 * mapped, i.e. is aliased to multiple vCPUs. The optimized 277 * map requires a strict 1:1 mapping between IDs and vCPUs. 278 */ 279 if (apic_x2apic_mode(apic)) 280 physical_id = x2apic_id; 281 else 282 physical_id = xapic_id; 283 284 if (new->phys_map[physical_id]) 285 return -EINVAL; 286 287 new->phys_map[physical_id] = apic; 288 } 289 290 return 0; 291 } 292 293 static void kvm_recalculate_logical_map(struct kvm_apic_map *new, 294 struct kvm_vcpu *vcpu) 295 { 296 struct kvm_lapic *apic = vcpu->arch.apic; 297 enum kvm_apic_logical_mode logical_mode; 298 struct kvm_lapic **cluster; 299 u16 mask; 300 u32 ldr; 301 302 if (new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) 303 return; 304 305 if (!kvm_apic_sw_enabled(apic)) 306 return; 307 308 ldr = kvm_lapic_get_reg(apic, APIC_LDR); 309 if (!ldr) 310 return; 311 312 if (apic_x2apic_mode(apic)) { 313 logical_mode = KVM_APIC_MODE_X2APIC; 314 } else { 315 ldr = GET_APIC_LOGICAL_ID(ldr); 316 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) 317 logical_mode = KVM_APIC_MODE_XAPIC_FLAT; 318 else 319 logical_mode = KVM_APIC_MODE_XAPIC_CLUSTER; 320 } 321 322 /* 323 * To optimize logical mode delivery, all software-enabled APICs must 324 * be configured for the same mode. 325 */ 326 if (new->logical_mode == KVM_APIC_MODE_SW_DISABLED) { 327 new->logical_mode = logical_mode; 328 } else if (new->logical_mode != logical_mode) { 329 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 330 return; 331 } 332 333 /* 334 * In x2APIC mode, the LDR is read-only and derived directly from the 335 * x2APIC ID, thus is guaranteed to be addressable. KVM reuses 336 * kvm_apic_map.phys_map to optimize logical mode x2APIC interrupts by 337 * reversing the LDR calculation to get cluster of APICs, i.e. no 338 * additional work is required. 339 */ 340 if (apic_x2apic_mode(apic)) { 341 WARN_ON_ONCE(ldr != kvm_apic_calc_x2apic_ldr(kvm_x2apic_id(apic))); 342 return; 343 } 344 345 if (WARN_ON_ONCE(!kvm_apic_map_get_logical_dest(new, ldr, 346 &cluster, &mask))) { 347 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 348 return; 349 } 350 351 if (!mask) 352 return; 353 354 ldr = ffs(mask) - 1; 355 if (!is_power_of_2(mask) || cluster[ldr]) 356 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED; 357 else 358 cluster[ldr] = apic; 359 } 360 361 /* 362 * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock. 363 * 364 * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with 365 * apic_map_lock_held. 366 */ 367 enum { 368 CLEAN, 369 UPDATE_IN_PROGRESS, 370 DIRTY 371 }; 372 373 void kvm_recalculate_apic_map(struct kvm *kvm) 374 { 375 struct kvm_apic_map *new, *old = NULL; 376 struct kvm_vcpu *vcpu; 377 unsigned long i; 378 u32 max_id = 255; /* enough space for any xAPIC ID */ 379 bool xapic_id_mismatch = false; 380 381 /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map. */ 382 if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN) 383 return; 384 385 WARN_ONCE(!irqchip_in_kernel(kvm), 386 "Dirty APIC map without an in-kernel local APIC"); 387 388 mutex_lock(&kvm->arch.apic_map_lock); 389 /* 390 * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map 391 * (if clean) or the APIC registers (if dirty). 392 */ 393 if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty, 394 DIRTY, UPDATE_IN_PROGRESS) == CLEAN) { 395 /* Someone else has updated the map. */ 396 mutex_unlock(&kvm->arch.apic_map_lock); 397 return; 398 } 399 400 kvm_for_each_vcpu(i, vcpu, kvm) 401 if (kvm_apic_present(vcpu)) 402 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); 403 404 new = kvzalloc(sizeof(struct kvm_apic_map) + 405 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), 406 GFP_KERNEL_ACCOUNT); 407 408 if (!new) 409 goto out; 410 411 new->max_apic_id = max_id; 412 new->logical_mode = KVM_APIC_MODE_SW_DISABLED; 413 414 kvm_for_each_vcpu(i, vcpu, kvm) { 415 if (!kvm_apic_present(vcpu)) 416 continue; 417 418 if (kvm_recalculate_phys_map(new, vcpu, &xapic_id_mismatch)) { 419 kvfree(new); 420 new = NULL; 421 goto out; 422 } 423 424 kvm_recalculate_logical_map(new, vcpu); 425 } 426 out: 427 /* 428 * The optimized map is effectively KVM's internal version of APICv, 429 * and all unwanted aliasing that results in disabling the optimized 430 * map also applies to APICv. 431 */ 432 if (!new) 433 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED); 434 else 435 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED); 436 437 if (!new || new->logical_mode == KVM_APIC_MODE_MAP_DISABLED) 438 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED); 439 else 440 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED); 441 442 if (xapic_id_mismatch) 443 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED); 444 else 445 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED); 446 447 old = rcu_dereference_protected(kvm->arch.apic_map, 448 lockdep_is_held(&kvm->arch.apic_map_lock)); 449 rcu_assign_pointer(kvm->arch.apic_map, new); 450 /* 451 * Write kvm->arch.apic_map before clearing apic->apic_map_dirty. 452 * If another update has come in, leave it DIRTY. 453 */ 454 atomic_cmpxchg_release(&kvm->arch.apic_map_dirty, 455 UPDATE_IN_PROGRESS, CLEAN); 456 mutex_unlock(&kvm->arch.apic_map_lock); 457 458 if (old) 459 call_rcu(&old->rcu, kvm_apic_map_free); 460 461 kvm_make_scan_ioapic_request(kvm); 462 } 463 464 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 465 { 466 bool enabled = val & APIC_SPIV_APIC_ENABLED; 467 468 kvm_lapic_set_reg(apic, APIC_SPIV, val); 469 470 if (enabled != apic->sw_enabled) { 471 apic->sw_enabled = enabled; 472 if (enabled) 473 static_branch_slow_dec_deferred(&apic_sw_disabled); 474 else 475 static_branch_inc(&apic_sw_disabled.key); 476 477 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 478 } 479 480 /* Check if there are APF page ready requests pending */ 481 if (enabled) 482 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu); 483 } 484 485 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) 486 { 487 kvm_lapic_set_reg(apic, APIC_ID, id << 24); 488 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 489 } 490 491 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 492 { 493 kvm_lapic_set_reg(apic, APIC_LDR, id); 494 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 495 } 496 497 static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val) 498 { 499 kvm_lapic_set_reg(apic, APIC_DFR, val); 500 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 501 } 502 503 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) 504 { 505 u32 ldr = kvm_apic_calc_x2apic_ldr(id); 506 507 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); 508 509 kvm_lapic_set_reg(apic, APIC_ID, id); 510 kvm_lapic_set_reg(apic, APIC_LDR, ldr); 511 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 512 } 513 514 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 515 { 516 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 517 } 518 519 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 520 { 521 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 522 } 523 524 static inline int apic_lvtt_period(struct kvm_lapic *apic) 525 { 526 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 527 } 528 529 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 530 { 531 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 532 } 533 534 static inline int apic_lvt_nmi_mode(u32 lvt_val) 535 { 536 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 537 } 538 539 static inline bool kvm_lapic_lvt_supported(struct kvm_lapic *apic, int lvt_index) 540 { 541 return apic->nr_lvt_entries > lvt_index; 542 } 543 544 static inline int kvm_apic_calc_nr_lvt_entries(struct kvm_vcpu *vcpu) 545 { 546 return KVM_APIC_MAX_NR_LVT_ENTRIES - !(vcpu->arch.mcg_cap & MCG_CMCI_P); 547 } 548 549 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 550 { 551 struct kvm_lapic *apic = vcpu->arch.apic; 552 u32 v = 0; 553 554 if (!lapic_in_kernel(vcpu)) 555 return; 556 557 v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); 558 559 /* 560 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) 561 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with 562 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC 563 * version first and level-triggered interrupts never get EOIed in 564 * IOAPIC. 565 */ 566 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) && 567 !ioapic_in_kernel(vcpu->kvm)) 568 v |= APIC_LVR_DIRECTED_EOI; 569 kvm_lapic_set_reg(apic, APIC_LVR, v); 570 } 571 572 void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu) 573 { 574 int nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); 575 struct kvm_lapic *apic = vcpu->arch.apic; 576 int i; 577 578 if (!lapic_in_kernel(vcpu) || nr_lvt_entries == apic->nr_lvt_entries) 579 return; 580 581 /* Initialize/mask any "new" LVT entries. */ 582 for (i = apic->nr_lvt_entries; i < nr_lvt_entries; i++) 583 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); 584 585 apic->nr_lvt_entries = nr_lvt_entries; 586 587 /* The number of LVT entries is reflected in the version register. */ 588 kvm_apic_set_version(vcpu); 589 } 590 591 static const unsigned int apic_lvt_mask[KVM_APIC_MAX_NR_LVT_ENTRIES] = { 592 [LVT_TIMER] = LVT_MASK, /* timer mode mask added at runtime */ 593 [LVT_THERMAL_MONITOR] = LVT_MASK | APIC_MODE_MASK, 594 [LVT_PERFORMANCE_COUNTER] = LVT_MASK | APIC_MODE_MASK, 595 [LVT_LINT0] = LINT_MASK, 596 [LVT_LINT1] = LINT_MASK, 597 [LVT_ERROR] = LVT_MASK, 598 [LVT_CMCI] = LVT_MASK | APIC_MODE_MASK 599 }; 600 601 static int find_highest_vector(void *bitmap) 602 { 603 int vec; 604 u32 *reg; 605 606 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 607 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 608 reg = bitmap + REG_POS(vec); 609 if (*reg) 610 return __fls(*reg) + vec; 611 } 612 613 return -1; 614 } 615 616 static u8 count_vectors(void *bitmap) 617 { 618 int vec; 619 u32 *reg; 620 u8 count = 0; 621 622 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 623 reg = bitmap + REG_POS(vec); 624 count += hweight32(*reg); 625 } 626 627 return count; 628 } 629 630 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr) 631 { 632 u32 i, vec; 633 u32 pir_val, irr_val, prev_irr_val; 634 int max_updated_irr; 635 636 max_updated_irr = -1; 637 *max_irr = -1; 638 639 for (i = vec = 0; i <= 7; i++, vec += 32) { 640 pir_val = READ_ONCE(pir[i]); 641 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10)); 642 if (pir_val) { 643 prev_irr_val = irr_val; 644 irr_val |= xchg(&pir[i], 0); 645 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val; 646 if (prev_irr_val != irr_val) { 647 max_updated_irr = 648 __fls(irr_val ^ prev_irr_val) + vec; 649 } 650 } 651 if (irr_val) 652 *max_irr = __fls(irr_val) + vec; 653 } 654 655 return ((max_updated_irr != -1) && 656 (max_updated_irr == *max_irr)); 657 } 658 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 659 660 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr) 661 { 662 struct kvm_lapic *apic = vcpu->arch.apic; 663 664 return __kvm_apic_update_irr(pir, apic->regs, max_irr); 665 } 666 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 667 668 static inline int apic_search_irr(struct kvm_lapic *apic) 669 { 670 return find_highest_vector(apic->regs + APIC_IRR); 671 } 672 673 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 674 { 675 int result; 676 677 /* 678 * Note that irr_pending is just a hint. It will be always 679 * true with virtual interrupt delivery enabled. 680 */ 681 if (!apic->irr_pending) 682 return -1; 683 684 result = apic_search_irr(apic); 685 ASSERT(result == -1 || result >= 16); 686 687 return result; 688 } 689 690 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 691 { 692 if (unlikely(apic->apicv_active)) { 693 /* need to update RVI */ 694 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 695 static_call_cond(kvm_x86_hwapic_irr_update)(apic->vcpu, 696 apic_find_highest_irr(apic)); 697 } else { 698 apic->irr_pending = false; 699 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); 700 if (apic_search_irr(apic) != -1) 701 apic->irr_pending = true; 702 } 703 } 704 705 void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec) 706 { 707 apic_clear_irr(vec, vcpu->arch.apic); 708 } 709 EXPORT_SYMBOL_GPL(kvm_apic_clear_irr); 710 711 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 712 { 713 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 714 return; 715 716 /* 717 * With APIC virtualization enabled, all caching is disabled 718 * because the processor can modify ISR under the hood. Instead 719 * just set SVI. 720 */ 721 if (unlikely(apic->apicv_active)) 722 static_call_cond(kvm_x86_hwapic_isr_update)(vec); 723 else { 724 ++apic->isr_count; 725 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 726 /* 727 * ISR (in service register) bit is set when injecting an interrupt. 728 * The highest vector is injected. Thus the latest bit set matches 729 * the highest bit in ISR. 730 */ 731 apic->highest_isr_cache = vec; 732 } 733 } 734 735 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 736 { 737 int result; 738 739 /* 740 * Note that isr_count is always 1, and highest_isr_cache 741 * is always -1, with APIC virtualization enabled. 742 */ 743 if (!apic->isr_count) 744 return -1; 745 if (likely(apic->highest_isr_cache != -1)) 746 return apic->highest_isr_cache; 747 748 result = find_highest_vector(apic->regs + APIC_ISR); 749 ASSERT(result == -1 || result >= 16); 750 751 return result; 752 } 753 754 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 755 { 756 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 757 return; 758 759 /* 760 * We do get here for APIC virtualization enabled if the guest 761 * uses the Hyper-V APIC enlightenment. In this case we may need 762 * to trigger a new interrupt delivery by writing the SVI field; 763 * on the other hand isr_count and highest_isr_cache are unused 764 * and must be left alone. 765 */ 766 if (unlikely(apic->apicv_active)) 767 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); 768 else { 769 --apic->isr_count; 770 BUG_ON(apic->isr_count < 0); 771 apic->highest_isr_cache = -1; 772 } 773 } 774 775 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 776 { 777 /* This may race with setting of irr in __apic_accept_irq() and 778 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 779 * will cause vmexit immediately and the value will be recalculated 780 * on the next vmentry. 781 */ 782 return apic_find_highest_irr(vcpu->arch.apic); 783 } 784 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); 785 786 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 787 int vector, int level, int trig_mode, 788 struct dest_map *dest_map); 789 790 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 791 struct dest_map *dest_map) 792 { 793 struct kvm_lapic *apic = vcpu->arch.apic; 794 795 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 796 irq->level, irq->trig_mode, dest_map); 797 } 798 799 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map, 800 struct kvm_lapic_irq *irq, u32 min) 801 { 802 int i, count = 0; 803 struct kvm_vcpu *vcpu; 804 805 if (min > map->max_apic_id) 806 return 0; 807 808 for_each_set_bit(i, ipi_bitmap, 809 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) { 810 if (map->phys_map[min + i]) { 811 vcpu = map->phys_map[min + i]->vcpu; 812 count += kvm_apic_set_irq(vcpu, irq, NULL); 813 } 814 } 815 816 return count; 817 } 818 819 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, 820 unsigned long ipi_bitmap_high, u32 min, 821 unsigned long icr, int op_64_bit) 822 { 823 struct kvm_apic_map *map; 824 struct kvm_lapic_irq irq = {0}; 825 int cluster_size = op_64_bit ? 64 : 32; 826 int count; 827 828 if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK)) 829 return -KVM_EINVAL; 830 831 irq.vector = icr & APIC_VECTOR_MASK; 832 irq.delivery_mode = icr & APIC_MODE_MASK; 833 irq.level = (icr & APIC_INT_ASSERT) != 0; 834 irq.trig_mode = icr & APIC_INT_LEVELTRIG; 835 836 rcu_read_lock(); 837 map = rcu_dereference(kvm->arch.apic_map); 838 839 count = -EOPNOTSUPP; 840 if (likely(map)) { 841 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min); 842 min += cluster_size; 843 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min); 844 } 845 846 rcu_read_unlock(); 847 return count; 848 } 849 850 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 851 { 852 853 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 854 sizeof(val)); 855 } 856 857 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 858 { 859 860 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 861 sizeof(*val)); 862 } 863 864 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 865 { 866 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 867 } 868 869 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 870 { 871 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) 872 return; 873 874 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 875 } 876 877 static bool pv_eoi_test_and_clr_pending(struct kvm_vcpu *vcpu) 878 { 879 u8 val; 880 881 if (pv_eoi_get_user(vcpu, &val) < 0) 882 return false; 883 884 val &= KVM_PV_EOI_ENABLED; 885 886 if (val && pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) 887 return false; 888 889 /* 890 * Clear pending bit in any case: it will be set again on vmentry. 891 * While this might not be ideal from performance point of view, 892 * this makes sure pv eoi is only enabled when we know it's safe. 893 */ 894 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 895 896 return val; 897 } 898 899 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) 900 { 901 int highest_irr; 902 if (kvm_x86_ops.sync_pir_to_irr) 903 highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu); 904 else 905 highest_irr = apic_find_highest_irr(apic); 906 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr) 907 return -1; 908 return highest_irr; 909 } 910 911 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) 912 { 913 u32 tpr, isrv, ppr, old_ppr; 914 int isr; 915 916 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); 917 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); 918 isr = apic_find_highest_isr(apic); 919 isrv = (isr != -1) ? isr : 0; 920 921 if ((tpr & 0xf0) >= (isrv & 0xf0)) 922 ppr = tpr & 0xff; 923 else 924 ppr = isrv & 0xf0; 925 926 *new_ppr = ppr; 927 if (old_ppr != ppr) 928 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); 929 930 return ppr < old_ppr; 931 } 932 933 static void apic_update_ppr(struct kvm_lapic *apic) 934 { 935 u32 ppr; 936 937 if (__apic_update_ppr(apic, &ppr) && 938 apic_has_interrupt_for_ppr(apic, ppr) != -1) 939 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 940 } 941 942 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu) 943 { 944 apic_update_ppr(vcpu->arch.apic); 945 } 946 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr); 947 948 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 949 { 950 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); 951 apic_update_ppr(apic); 952 } 953 954 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) 955 { 956 return mda == (apic_x2apic_mode(apic) ? 957 X2APIC_BROADCAST : APIC_BROADCAST); 958 } 959 960 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) 961 { 962 if (kvm_apic_broadcast(apic, mda)) 963 return true; 964 965 /* 966 * Hotplug hack: Accept interrupts for vCPUs in xAPIC mode as if they 967 * were in x2APIC mode if the target APIC ID can't be encoded as an 968 * xAPIC ID. This allows unique addressing of hotplugged vCPUs (which 969 * start in xAPIC mode) with an APIC ID that is unaddressable in xAPIC 970 * mode. Match the x2APIC ID if and only if the target APIC ID can't 971 * be encoded in xAPIC to avoid spurious matches against a vCPU that 972 * changed its (addressable) xAPIC ID (which is writable). 973 */ 974 if (apic_x2apic_mode(apic) || mda > 0xff) 975 return mda == kvm_x2apic_id(apic); 976 977 return mda == kvm_xapic_id(apic); 978 } 979 980 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 981 { 982 u32 logical_id; 983 984 if (kvm_apic_broadcast(apic, mda)) 985 return true; 986 987 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); 988 989 if (apic_x2apic_mode(apic)) 990 return ((logical_id >> 16) == (mda >> 16)) 991 && (logical_id & mda & 0xffff) != 0; 992 993 logical_id = GET_APIC_LOGICAL_ID(logical_id); 994 995 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { 996 case APIC_DFR_FLAT: 997 return (logical_id & mda) != 0; 998 case APIC_DFR_CLUSTER: 999 return ((logical_id >> 4) == (mda >> 4)) 1000 && (logical_id & mda & 0xf) != 0; 1001 default: 1002 return false; 1003 } 1004 } 1005 1006 /* The KVM local APIC implementation has two quirks: 1007 * 1008 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs 1009 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID. 1010 * KVM doesn't do that aliasing. 1011 * 1012 * - in-kernel IOAPIC messages have to be delivered directly to 1013 * x2APIC, because the kernel does not support interrupt remapping. 1014 * In order to support broadcast without interrupt remapping, x2APIC 1015 * rewrites the destination of non-IPI messages from APIC_BROADCAST 1016 * to X2APIC_BROADCAST. 1017 * 1018 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is 1019 * important when userspace wants to use x2APIC-format MSIs, because 1020 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7". 1021 */ 1022 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id, 1023 struct kvm_lapic *source, struct kvm_lapic *target) 1024 { 1025 bool ipi = source != NULL; 1026 1027 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled && 1028 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target)) 1029 return X2APIC_BROADCAST; 1030 1031 return dest_id; 1032 } 1033 1034 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 1035 int shorthand, unsigned int dest, int dest_mode) 1036 { 1037 struct kvm_lapic *target = vcpu->arch.apic; 1038 u32 mda = kvm_apic_mda(vcpu, dest, source, target); 1039 1040 ASSERT(target); 1041 switch (shorthand) { 1042 case APIC_DEST_NOSHORT: 1043 if (dest_mode == APIC_DEST_PHYSICAL) 1044 return kvm_apic_match_physical_addr(target, mda); 1045 else 1046 return kvm_apic_match_logical_addr(target, mda); 1047 case APIC_DEST_SELF: 1048 return target == source; 1049 case APIC_DEST_ALLINC: 1050 return true; 1051 case APIC_DEST_ALLBUT: 1052 return target != source; 1053 default: 1054 return false; 1055 } 1056 } 1057 EXPORT_SYMBOL_GPL(kvm_apic_match_dest); 1058 1059 int kvm_vector_to_index(u32 vector, u32 dest_vcpus, 1060 const unsigned long *bitmap, u32 bitmap_size) 1061 { 1062 u32 mod; 1063 int i, idx = -1; 1064 1065 mod = vector % dest_vcpus; 1066 1067 for (i = 0; i <= mod; i++) { 1068 idx = find_next_bit(bitmap, bitmap_size, idx + 1); 1069 BUG_ON(idx == bitmap_size); 1070 } 1071 1072 return idx; 1073 } 1074 1075 static void kvm_apic_disabled_lapic_found(struct kvm *kvm) 1076 { 1077 if (!kvm->arch.disabled_lapic_found) { 1078 kvm->arch.disabled_lapic_found = true; 1079 pr_info("Disabled LAPIC found during irq injection\n"); 1080 } 1081 } 1082 1083 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src, 1084 struct kvm_lapic_irq *irq, struct kvm_apic_map *map) 1085 { 1086 if (kvm->arch.x2apic_broadcast_quirk_disabled) { 1087 if ((irq->dest_id == APIC_BROADCAST && 1088 map->logical_mode != KVM_APIC_MODE_X2APIC)) 1089 return true; 1090 if (irq->dest_id == X2APIC_BROADCAST) 1091 return true; 1092 } else { 1093 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src); 1094 if (irq->dest_id == (x2apic_ipi ? 1095 X2APIC_BROADCAST : APIC_BROADCAST)) 1096 return true; 1097 } 1098 1099 return false; 1100 } 1101 1102 /* Return true if the interrupt can be handled by using *bitmap as index mask 1103 * for valid destinations in *dst array. 1104 * Return false if kvm_apic_map_get_dest_lapic did nothing useful. 1105 * Note: we may have zero kvm_lapic destinations when we return true, which 1106 * means that the interrupt should be dropped. In this case, *bitmap would be 1107 * zero and *dst undefined. 1108 */ 1109 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm, 1110 struct kvm_lapic **src, struct kvm_lapic_irq *irq, 1111 struct kvm_apic_map *map, struct kvm_lapic ***dst, 1112 unsigned long *bitmap) 1113 { 1114 int i, lowest; 1115 1116 if (irq->shorthand == APIC_DEST_SELF && src) { 1117 *dst = src; 1118 *bitmap = 1; 1119 return true; 1120 } else if (irq->shorthand) 1121 return false; 1122 1123 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map)) 1124 return false; 1125 1126 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 1127 if (irq->dest_id > map->max_apic_id) { 1128 *bitmap = 0; 1129 } else { 1130 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1); 1131 *dst = &map->phys_map[dest_id]; 1132 *bitmap = 1; 1133 } 1134 return true; 1135 } 1136 1137 *bitmap = 0; 1138 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst, 1139 (u16 *)bitmap)) 1140 return false; 1141 1142 if (!kvm_lowest_prio_delivery(irq)) 1143 return true; 1144 1145 if (!kvm_vector_hashing_enabled()) { 1146 lowest = -1; 1147 for_each_set_bit(i, bitmap, 16) { 1148 if (!(*dst)[i]) 1149 continue; 1150 if (lowest < 0) 1151 lowest = i; 1152 else if (kvm_apic_compare_prio((*dst)[i]->vcpu, 1153 (*dst)[lowest]->vcpu) < 0) 1154 lowest = i; 1155 } 1156 } else { 1157 if (!*bitmap) 1158 return true; 1159 1160 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap), 1161 bitmap, 16); 1162 1163 if (!(*dst)[lowest]) { 1164 kvm_apic_disabled_lapic_found(kvm); 1165 *bitmap = 0; 1166 return true; 1167 } 1168 } 1169 1170 *bitmap = (lowest >= 0) ? 1 << lowest : 0; 1171 1172 return true; 1173 } 1174 1175 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 1176 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map) 1177 { 1178 struct kvm_apic_map *map; 1179 unsigned long bitmap; 1180 struct kvm_lapic **dst = NULL; 1181 int i; 1182 bool ret; 1183 1184 *r = -1; 1185 1186 if (irq->shorthand == APIC_DEST_SELF) { 1187 if (KVM_BUG_ON(!src, kvm)) { 1188 *r = 0; 1189 return true; 1190 } 1191 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 1192 return true; 1193 } 1194 1195 rcu_read_lock(); 1196 map = rcu_dereference(kvm->arch.apic_map); 1197 1198 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap); 1199 if (ret) { 1200 *r = 0; 1201 for_each_set_bit(i, &bitmap, 16) { 1202 if (!dst[i]) 1203 continue; 1204 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 1205 } 1206 } 1207 1208 rcu_read_unlock(); 1209 return ret; 1210 } 1211 1212 /* 1213 * This routine tries to handle interrupts in posted mode, here is how 1214 * it deals with different cases: 1215 * - For single-destination interrupts, handle it in posted mode 1216 * - Else if vector hashing is enabled and it is a lowest-priority 1217 * interrupt, handle it in posted mode and use the following mechanism 1218 * to find the destination vCPU. 1219 * 1. For lowest-priority interrupts, store all the possible 1220 * destination vCPUs in an array. 1221 * 2. Use "guest vector % max number of destination vCPUs" to find 1222 * the right destination vCPU in the array for the lowest-priority 1223 * interrupt. 1224 * - Otherwise, use remapped mode to inject the interrupt. 1225 */ 1226 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq, 1227 struct kvm_vcpu **dest_vcpu) 1228 { 1229 struct kvm_apic_map *map; 1230 unsigned long bitmap; 1231 struct kvm_lapic **dst = NULL; 1232 bool ret = false; 1233 1234 if (irq->shorthand) 1235 return false; 1236 1237 rcu_read_lock(); 1238 map = rcu_dereference(kvm->arch.apic_map); 1239 1240 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) && 1241 hweight16(bitmap) == 1) { 1242 unsigned long i = find_first_bit(&bitmap, 16); 1243 1244 if (dst[i]) { 1245 *dest_vcpu = dst[i]->vcpu; 1246 ret = true; 1247 } 1248 } 1249 1250 rcu_read_unlock(); 1251 return ret; 1252 } 1253 1254 /* 1255 * Add a pending IRQ into lapic. 1256 * Return 1 if successfully added and 0 if discarded. 1257 */ 1258 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 1259 int vector, int level, int trig_mode, 1260 struct dest_map *dest_map) 1261 { 1262 int result = 0; 1263 struct kvm_vcpu *vcpu = apic->vcpu; 1264 1265 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 1266 trig_mode, vector); 1267 switch (delivery_mode) { 1268 case APIC_DM_LOWEST: 1269 vcpu->arch.apic_arb_prio++; 1270 fallthrough; 1271 case APIC_DM_FIXED: 1272 if (unlikely(trig_mode && !level)) 1273 break; 1274 1275 /* FIXME add logic for vcpu on reset */ 1276 if (unlikely(!apic_enabled(apic))) 1277 break; 1278 1279 result = 1; 1280 1281 if (dest_map) { 1282 __set_bit(vcpu->vcpu_id, dest_map->map); 1283 dest_map->vectors[vcpu->vcpu_id] = vector; 1284 } 1285 1286 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { 1287 if (trig_mode) 1288 kvm_lapic_set_vector(vector, 1289 apic->regs + APIC_TMR); 1290 else 1291 kvm_lapic_clear_vector(vector, 1292 apic->regs + APIC_TMR); 1293 } 1294 1295 static_call(kvm_x86_deliver_interrupt)(apic, delivery_mode, 1296 trig_mode, vector); 1297 break; 1298 1299 case APIC_DM_REMRD: 1300 result = 1; 1301 vcpu->arch.pv.pv_unhalted = 1; 1302 kvm_make_request(KVM_REQ_EVENT, vcpu); 1303 kvm_vcpu_kick(vcpu); 1304 break; 1305 1306 case APIC_DM_SMI: 1307 if (!kvm_inject_smi(vcpu)) { 1308 kvm_vcpu_kick(vcpu); 1309 result = 1; 1310 } 1311 break; 1312 1313 case APIC_DM_NMI: 1314 result = 1; 1315 kvm_inject_nmi(vcpu); 1316 kvm_vcpu_kick(vcpu); 1317 break; 1318 1319 case APIC_DM_INIT: 1320 if (!trig_mode || level) { 1321 result = 1; 1322 /* assumes that there are only KVM_APIC_INIT/SIPI */ 1323 apic->pending_events = (1UL << KVM_APIC_INIT); 1324 kvm_make_request(KVM_REQ_EVENT, vcpu); 1325 kvm_vcpu_kick(vcpu); 1326 } 1327 break; 1328 1329 case APIC_DM_STARTUP: 1330 result = 1; 1331 apic->sipi_vector = vector; 1332 /* make sure sipi_vector is visible for the receiver */ 1333 smp_wmb(); 1334 set_bit(KVM_APIC_SIPI, &apic->pending_events); 1335 kvm_make_request(KVM_REQ_EVENT, vcpu); 1336 kvm_vcpu_kick(vcpu); 1337 break; 1338 1339 case APIC_DM_EXTINT: 1340 /* 1341 * Should only be called by kvm_apic_local_deliver() with LVT0, 1342 * before NMI watchdog was enabled. Already handled by 1343 * kvm_apic_accept_pic_intr(). 1344 */ 1345 break; 1346 1347 default: 1348 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 1349 delivery_mode); 1350 break; 1351 } 1352 return result; 1353 } 1354 1355 /* 1356 * This routine identifies the destination vcpus mask meant to receive the 1357 * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find 1358 * out the destination vcpus array and set the bitmap or it traverses to 1359 * each available vcpu to identify the same. 1360 */ 1361 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, 1362 unsigned long *vcpu_bitmap) 1363 { 1364 struct kvm_lapic **dest_vcpu = NULL; 1365 struct kvm_lapic *src = NULL; 1366 struct kvm_apic_map *map; 1367 struct kvm_vcpu *vcpu; 1368 unsigned long bitmap, i; 1369 int vcpu_idx; 1370 bool ret; 1371 1372 rcu_read_lock(); 1373 map = rcu_dereference(kvm->arch.apic_map); 1374 1375 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu, 1376 &bitmap); 1377 if (ret) { 1378 for_each_set_bit(i, &bitmap, 16) { 1379 if (!dest_vcpu[i]) 1380 continue; 1381 vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx; 1382 __set_bit(vcpu_idx, vcpu_bitmap); 1383 } 1384 } else { 1385 kvm_for_each_vcpu(i, vcpu, kvm) { 1386 if (!kvm_apic_present(vcpu)) 1387 continue; 1388 if (!kvm_apic_match_dest(vcpu, NULL, 1389 irq->shorthand, 1390 irq->dest_id, 1391 irq->dest_mode)) 1392 continue; 1393 __set_bit(i, vcpu_bitmap); 1394 } 1395 } 1396 rcu_read_unlock(); 1397 } 1398 1399 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 1400 { 1401 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 1402 } 1403 1404 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) 1405 { 1406 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); 1407 } 1408 1409 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 1410 { 1411 int trigger_mode; 1412 1413 /* Eoi the ioapic only if the ioapic doesn't own the vector. */ 1414 if (!kvm_ioapic_handles_vector(apic, vector)) 1415 return; 1416 1417 /* Request a KVM exit to inform the userspace IOAPIC. */ 1418 if (irqchip_split(apic->vcpu->kvm)) { 1419 apic->vcpu->arch.pending_ioapic_eoi = vector; 1420 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); 1421 return; 1422 } 1423 1424 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 1425 trigger_mode = IOAPIC_LEVEL_TRIG; 1426 else 1427 trigger_mode = IOAPIC_EDGE_TRIG; 1428 1429 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 1430 } 1431 1432 static int apic_set_eoi(struct kvm_lapic *apic) 1433 { 1434 int vector = apic_find_highest_isr(apic); 1435 1436 trace_kvm_eoi(apic, vector); 1437 1438 /* 1439 * Not every write EOI will has corresponding ISR, 1440 * one example is when Kernel check timer on setup_IO_APIC 1441 */ 1442 if (vector == -1) 1443 return vector; 1444 1445 apic_clear_isr(vector, apic); 1446 apic_update_ppr(apic); 1447 1448 if (to_hv_vcpu(apic->vcpu) && 1449 test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap)) 1450 kvm_hv_synic_send_eoi(apic->vcpu, vector); 1451 1452 kvm_ioapic_send_eoi(apic, vector); 1453 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1454 return vector; 1455 } 1456 1457 /* 1458 * this interface assumes a trap-like exit, which has already finished 1459 * desired side effect including vISR and vPPR update. 1460 */ 1461 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 1462 { 1463 struct kvm_lapic *apic = vcpu->arch.apic; 1464 1465 trace_kvm_eoi(apic, vector); 1466 1467 kvm_ioapic_send_eoi(apic, vector); 1468 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 1469 } 1470 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 1471 1472 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high) 1473 { 1474 struct kvm_lapic_irq irq; 1475 1476 /* KVM has no delay and should always clear the BUSY/PENDING flag. */ 1477 WARN_ON_ONCE(icr_low & APIC_ICR_BUSY); 1478 1479 irq.vector = icr_low & APIC_VECTOR_MASK; 1480 irq.delivery_mode = icr_low & APIC_MODE_MASK; 1481 irq.dest_mode = icr_low & APIC_DEST_MASK; 1482 irq.level = (icr_low & APIC_INT_ASSERT) != 0; 1483 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 1484 irq.shorthand = icr_low & APIC_SHORT_MASK; 1485 irq.msi_redir_hint = false; 1486 if (apic_x2apic_mode(apic)) 1487 irq.dest_id = icr_high; 1488 else 1489 irq.dest_id = GET_XAPIC_DEST_FIELD(icr_high); 1490 1491 trace_kvm_apic_ipi(icr_low, irq.dest_id); 1492 1493 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 1494 } 1495 EXPORT_SYMBOL_GPL(kvm_apic_send_ipi); 1496 1497 static u32 apic_get_tmcct(struct kvm_lapic *apic) 1498 { 1499 ktime_t remaining, now; 1500 s64 ns; 1501 1502 ASSERT(apic != NULL); 1503 1504 /* if initial count is 0, current count should also be 0 */ 1505 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || 1506 apic->lapic_timer.period == 0) 1507 return 0; 1508 1509 now = ktime_get(); 1510 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1511 if (ktime_to_ns(remaining) < 0) 1512 remaining = 0; 1513 1514 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 1515 return div64_u64(ns, (APIC_BUS_CYCLE_NS * apic->divide_count)); 1516 } 1517 1518 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 1519 { 1520 struct kvm_vcpu *vcpu = apic->vcpu; 1521 struct kvm_run *run = vcpu->run; 1522 1523 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 1524 run->tpr_access.rip = kvm_rip_read(vcpu); 1525 run->tpr_access.is_write = write; 1526 } 1527 1528 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 1529 { 1530 if (apic->vcpu->arch.tpr_access_reporting) 1531 __report_tpr_access(apic, write); 1532 } 1533 1534 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 1535 { 1536 u32 val = 0; 1537 1538 if (offset >= LAPIC_MMIO_LENGTH) 1539 return 0; 1540 1541 switch (offset) { 1542 case APIC_ARBPRI: 1543 break; 1544 1545 case APIC_TMCCT: /* Timer CCR */ 1546 if (apic_lvtt_tscdeadline(apic)) 1547 return 0; 1548 1549 val = apic_get_tmcct(apic); 1550 break; 1551 case APIC_PROCPRI: 1552 apic_update_ppr(apic); 1553 val = kvm_lapic_get_reg(apic, offset); 1554 break; 1555 case APIC_TASKPRI: 1556 report_tpr_access(apic, false); 1557 fallthrough; 1558 default: 1559 val = kvm_lapic_get_reg(apic, offset); 1560 break; 1561 } 1562 1563 return val; 1564 } 1565 1566 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 1567 { 1568 return container_of(dev, struct kvm_lapic, dev); 1569 } 1570 1571 #define APIC_REG_MASK(reg) (1ull << ((reg) >> 4)) 1572 #define APIC_REGS_MASK(first, count) \ 1573 (APIC_REG_MASK(first) * ((1ull << (count)) - 1)) 1574 1575 u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic) 1576 { 1577 /* Leave bits '0' for reserved and write-only registers. */ 1578 u64 valid_reg_mask = 1579 APIC_REG_MASK(APIC_ID) | 1580 APIC_REG_MASK(APIC_LVR) | 1581 APIC_REG_MASK(APIC_TASKPRI) | 1582 APIC_REG_MASK(APIC_PROCPRI) | 1583 APIC_REG_MASK(APIC_LDR) | 1584 APIC_REG_MASK(APIC_SPIV) | 1585 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) | 1586 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) | 1587 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) | 1588 APIC_REG_MASK(APIC_ESR) | 1589 APIC_REG_MASK(APIC_ICR) | 1590 APIC_REG_MASK(APIC_LVTT) | 1591 APIC_REG_MASK(APIC_LVTTHMR) | 1592 APIC_REG_MASK(APIC_LVTPC) | 1593 APIC_REG_MASK(APIC_LVT0) | 1594 APIC_REG_MASK(APIC_LVT1) | 1595 APIC_REG_MASK(APIC_LVTERR) | 1596 APIC_REG_MASK(APIC_TMICT) | 1597 APIC_REG_MASK(APIC_TMCCT) | 1598 APIC_REG_MASK(APIC_TDCR); 1599 1600 if (kvm_lapic_lvt_supported(apic, LVT_CMCI)) 1601 valid_reg_mask |= APIC_REG_MASK(APIC_LVTCMCI); 1602 1603 /* ARBPRI, DFR, and ICR2 are not valid in x2APIC mode. */ 1604 if (!apic_x2apic_mode(apic)) 1605 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI) | 1606 APIC_REG_MASK(APIC_DFR) | 1607 APIC_REG_MASK(APIC_ICR2); 1608 1609 return valid_reg_mask; 1610 } 1611 EXPORT_SYMBOL_GPL(kvm_lapic_readable_reg_mask); 1612 1613 static int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 1614 void *data) 1615 { 1616 unsigned char alignment = offset & 0xf; 1617 u32 result; 1618 1619 /* 1620 * WARN if KVM reads ICR in x2APIC mode, as it's an 8-byte register in 1621 * x2APIC and needs to be manually handled by the caller. 1622 */ 1623 WARN_ON_ONCE(apic_x2apic_mode(apic) && offset == APIC_ICR); 1624 1625 if (alignment + len > 4) 1626 return 1; 1627 1628 if (offset > 0x3f0 || 1629 !(kvm_lapic_readable_reg_mask(apic) & APIC_REG_MASK(offset))) 1630 return 1; 1631 1632 result = __apic_read(apic, offset & ~0xf); 1633 1634 trace_kvm_apic_read(offset, result); 1635 1636 switch (len) { 1637 case 1: 1638 case 2: 1639 case 4: 1640 memcpy(data, (char *)&result + alignment, len); 1641 break; 1642 default: 1643 printk(KERN_ERR "Local APIC read with len = %x, " 1644 "should be 1,2, or 4 instead\n", len); 1645 break; 1646 } 1647 return 0; 1648 } 1649 1650 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1651 { 1652 return addr >= apic->base_address && 1653 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1654 } 1655 1656 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1657 gpa_t address, int len, void *data) 1658 { 1659 struct kvm_lapic *apic = to_lapic(this); 1660 u32 offset = address - apic->base_address; 1661 1662 if (!apic_mmio_in_range(apic, address)) 1663 return -EOPNOTSUPP; 1664 1665 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 1666 if (!kvm_check_has_quirk(vcpu->kvm, 1667 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 1668 return -EOPNOTSUPP; 1669 1670 memset(data, 0xff, len); 1671 return 0; 1672 } 1673 1674 kvm_lapic_reg_read(apic, offset, len, data); 1675 1676 return 0; 1677 } 1678 1679 static void update_divide_count(struct kvm_lapic *apic) 1680 { 1681 u32 tmp1, tmp2, tdcr; 1682 1683 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); 1684 tmp1 = tdcr & 0xf; 1685 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1686 apic->divide_count = 0x1 << (tmp2 & 0x7); 1687 } 1688 1689 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) 1690 { 1691 /* 1692 * Do not allow the guest to program periodic timers with small 1693 * interval, since the hrtimers are not throttled by the host 1694 * scheduler. 1695 */ 1696 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 1697 s64 min_period = min_timer_period_us * 1000LL; 1698 1699 if (apic->lapic_timer.period < min_period) { 1700 pr_info_ratelimited( 1701 "vcpu %i: requested %lld ns " 1702 "lapic timer period limited to %lld ns\n", 1703 apic->vcpu->vcpu_id, 1704 apic->lapic_timer.period, min_period); 1705 apic->lapic_timer.period = min_period; 1706 } 1707 } 1708 } 1709 1710 static void cancel_hv_timer(struct kvm_lapic *apic); 1711 1712 static void cancel_apic_timer(struct kvm_lapic *apic) 1713 { 1714 hrtimer_cancel(&apic->lapic_timer.timer); 1715 preempt_disable(); 1716 if (apic->lapic_timer.hv_timer_in_use) 1717 cancel_hv_timer(apic); 1718 preempt_enable(); 1719 atomic_set(&apic->lapic_timer.pending, 0); 1720 } 1721 1722 static void apic_update_lvtt(struct kvm_lapic *apic) 1723 { 1724 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & 1725 apic->lapic_timer.timer_mode_mask; 1726 1727 if (apic->lapic_timer.timer_mode != timer_mode) { 1728 if (apic_lvtt_tscdeadline(apic) != (timer_mode == 1729 APIC_LVT_TIMER_TSCDEADLINE)) { 1730 cancel_apic_timer(apic); 1731 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 1732 apic->lapic_timer.period = 0; 1733 apic->lapic_timer.tscdeadline = 0; 1734 } 1735 apic->lapic_timer.timer_mode = timer_mode; 1736 limit_periodic_timer_frequency(apic); 1737 } 1738 } 1739 1740 /* 1741 * On APICv, this test will cause a busy wait 1742 * during a higher-priority task. 1743 */ 1744 1745 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1746 { 1747 struct kvm_lapic *apic = vcpu->arch.apic; 1748 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); 1749 1750 if (kvm_apic_hw_enabled(apic)) { 1751 int vec = reg & APIC_VECTOR_MASK; 1752 void *bitmap = apic->regs + APIC_ISR; 1753 1754 if (apic->apicv_active) 1755 bitmap = apic->regs + APIC_IRR; 1756 1757 if (apic_test_vector(vec, bitmap)) 1758 return true; 1759 } 1760 return false; 1761 } 1762 1763 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles) 1764 { 1765 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns; 1766 1767 /* 1768 * If the guest TSC is running at a different ratio than the host, then 1769 * convert the delay to nanoseconds to achieve an accurate delay. Note 1770 * that __delay() uses delay_tsc whenever the hardware has TSC, thus 1771 * always for VMX enabled hardware. 1772 */ 1773 if (vcpu->arch.tsc_scaling_ratio == kvm_caps.default_tsc_scaling_ratio) { 1774 __delay(min(guest_cycles, 1775 nsec_to_cycles(vcpu, timer_advance_ns))); 1776 } else { 1777 u64 delay_ns = guest_cycles * 1000000ULL; 1778 do_div(delay_ns, vcpu->arch.virtual_tsc_khz); 1779 ndelay(min_t(u32, delay_ns, timer_advance_ns)); 1780 } 1781 } 1782 1783 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu, 1784 s64 advance_expire_delta) 1785 { 1786 struct kvm_lapic *apic = vcpu->arch.apic; 1787 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns; 1788 u64 ns; 1789 1790 /* Do not adjust for tiny fluctuations or large random spikes. */ 1791 if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX || 1792 abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN) 1793 return; 1794 1795 /* too early */ 1796 if (advance_expire_delta < 0) { 1797 ns = -advance_expire_delta * 1000000ULL; 1798 do_div(ns, vcpu->arch.virtual_tsc_khz); 1799 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP; 1800 } else { 1801 /* too late */ 1802 ns = advance_expire_delta * 1000000ULL; 1803 do_div(ns, vcpu->arch.virtual_tsc_khz); 1804 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP; 1805 } 1806 1807 if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX)) 1808 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; 1809 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 1810 } 1811 1812 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1813 { 1814 struct kvm_lapic *apic = vcpu->arch.apic; 1815 u64 guest_tsc, tsc_deadline; 1816 1817 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1818 apic->lapic_timer.expired_tscdeadline = 0; 1819 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1820 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1821 1822 if (lapic_timer_advance_dynamic) { 1823 adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline); 1824 /* 1825 * If the timer fired early, reread the TSC to account for the 1826 * overhead of the above adjustment to avoid waiting longer 1827 * than is necessary. 1828 */ 1829 if (guest_tsc < tsc_deadline) 1830 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1831 } 1832 1833 if (guest_tsc < tsc_deadline) 1834 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc); 1835 } 1836 1837 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) 1838 { 1839 if (lapic_in_kernel(vcpu) && 1840 vcpu->arch.apic->lapic_timer.expired_tscdeadline && 1841 vcpu->arch.apic->lapic_timer.timer_advance_ns && 1842 lapic_timer_int_injected(vcpu)) 1843 __kvm_wait_lapic_expire(vcpu); 1844 } 1845 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire); 1846 1847 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic) 1848 { 1849 struct kvm_timer *ktimer = &apic->lapic_timer; 1850 1851 kvm_apic_local_deliver(apic, APIC_LVTT); 1852 if (apic_lvtt_tscdeadline(apic)) { 1853 ktimer->tscdeadline = 0; 1854 } else if (apic_lvtt_oneshot(apic)) { 1855 ktimer->tscdeadline = 0; 1856 ktimer->target_expiration = 0; 1857 } 1858 } 1859 1860 static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) 1861 { 1862 struct kvm_vcpu *vcpu = apic->vcpu; 1863 struct kvm_timer *ktimer = &apic->lapic_timer; 1864 1865 if (atomic_read(&apic->lapic_timer.pending)) 1866 return; 1867 1868 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) 1869 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1870 1871 if (!from_timer_fn && apic->apicv_active) { 1872 WARN_ON(kvm_get_running_vcpu() != vcpu); 1873 kvm_apic_inject_pending_timer_irqs(apic); 1874 return; 1875 } 1876 1877 if (kvm_use_posted_timer_interrupt(apic->vcpu)) { 1878 /* 1879 * Ensure the guest's timer has truly expired before posting an 1880 * interrupt. Open code the relevant checks to avoid querying 1881 * lapic_timer_int_injected(), which will be false since the 1882 * interrupt isn't yet injected. Waiting until after injecting 1883 * is not an option since that won't help a posted interrupt. 1884 */ 1885 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline && 1886 vcpu->arch.apic->lapic_timer.timer_advance_ns) 1887 __kvm_wait_lapic_expire(vcpu); 1888 kvm_apic_inject_pending_timer_irqs(apic); 1889 return; 1890 } 1891 1892 atomic_inc(&apic->lapic_timer.pending); 1893 kvm_make_request(KVM_REQ_UNBLOCK, vcpu); 1894 if (from_timer_fn) 1895 kvm_vcpu_kick(vcpu); 1896 } 1897 1898 static void start_sw_tscdeadline(struct kvm_lapic *apic) 1899 { 1900 struct kvm_timer *ktimer = &apic->lapic_timer; 1901 u64 guest_tsc, tscdeadline = ktimer->tscdeadline; 1902 u64 ns = 0; 1903 ktime_t expire; 1904 struct kvm_vcpu *vcpu = apic->vcpu; 1905 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1906 unsigned long flags; 1907 ktime_t now; 1908 1909 if (unlikely(!tscdeadline || !this_tsc_khz)) 1910 return; 1911 1912 local_irq_save(flags); 1913 1914 now = ktime_get(); 1915 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 1916 1917 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1918 do_div(ns, this_tsc_khz); 1919 1920 if (likely(tscdeadline > guest_tsc) && 1921 likely(ns > apic->lapic_timer.timer_advance_ns)) { 1922 expire = ktime_add_ns(now, ns); 1923 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns); 1924 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD); 1925 } else 1926 apic_timer_expired(apic, false); 1927 1928 local_irq_restore(flags); 1929 } 1930 1931 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict) 1932 { 1933 return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count; 1934 } 1935 1936 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) 1937 { 1938 ktime_t now, remaining; 1939 u64 ns_remaining_old, ns_remaining_new; 1940 1941 apic->lapic_timer.period = 1942 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); 1943 limit_periodic_timer_frequency(apic); 1944 1945 now = ktime_get(); 1946 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); 1947 if (ktime_to_ns(remaining) < 0) 1948 remaining = 0; 1949 1950 ns_remaining_old = ktime_to_ns(remaining); 1951 ns_remaining_new = mul_u64_u32_div(ns_remaining_old, 1952 apic->divide_count, old_divisor); 1953 1954 apic->lapic_timer.tscdeadline += 1955 nsec_to_cycles(apic->vcpu, ns_remaining_new) - 1956 nsec_to_cycles(apic->vcpu, ns_remaining_old); 1957 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); 1958 } 1959 1960 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg) 1961 { 1962 ktime_t now; 1963 u64 tscl = rdtsc(); 1964 s64 deadline; 1965 1966 now = ktime_get(); 1967 apic->lapic_timer.period = 1968 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); 1969 1970 if (!apic->lapic_timer.period) { 1971 apic->lapic_timer.tscdeadline = 0; 1972 return false; 1973 } 1974 1975 limit_periodic_timer_frequency(apic); 1976 deadline = apic->lapic_timer.period; 1977 1978 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { 1979 if (unlikely(count_reg != APIC_TMICT)) { 1980 deadline = tmict_to_ns(apic, 1981 kvm_lapic_get_reg(apic, count_reg)); 1982 if (unlikely(deadline <= 0)) { 1983 if (apic_lvtt_period(apic)) 1984 deadline = apic->lapic_timer.period; 1985 else 1986 deadline = 0; 1987 } 1988 else if (unlikely(deadline > apic->lapic_timer.period)) { 1989 pr_info_ratelimited( 1990 "vcpu %i: requested lapic timer restore with " 1991 "starting count register %#x=%u (%lld ns) > initial count (%lld ns). " 1992 "Using initial count to start timer.\n", 1993 apic->vcpu->vcpu_id, 1994 count_reg, 1995 kvm_lapic_get_reg(apic, count_reg), 1996 deadline, apic->lapic_timer.period); 1997 kvm_lapic_set_reg(apic, count_reg, 0); 1998 deadline = apic->lapic_timer.period; 1999 } 2000 } 2001 } 2002 2003 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 2004 nsec_to_cycles(apic->vcpu, deadline); 2005 apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline); 2006 2007 return true; 2008 } 2009 2010 static void advance_periodic_target_expiration(struct kvm_lapic *apic) 2011 { 2012 ktime_t now = ktime_get(); 2013 u64 tscl = rdtsc(); 2014 ktime_t delta; 2015 2016 /* 2017 * Synchronize both deadlines to the same time source or 2018 * differences in the periods (caused by differences in the 2019 * underlying clocks or numerical approximation errors) will 2020 * cause the two to drift apart over time as the errors 2021 * accumulate. 2022 */ 2023 apic->lapic_timer.target_expiration = 2024 ktime_add_ns(apic->lapic_timer.target_expiration, 2025 apic->lapic_timer.period); 2026 delta = ktime_sub(apic->lapic_timer.target_expiration, now); 2027 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + 2028 nsec_to_cycles(apic->vcpu, delta); 2029 } 2030 2031 static void start_sw_period(struct kvm_lapic *apic) 2032 { 2033 if (!apic->lapic_timer.period) 2034 return; 2035 2036 if (ktime_after(ktime_get(), 2037 apic->lapic_timer.target_expiration)) { 2038 apic_timer_expired(apic, false); 2039 2040 if (apic_lvtt_oneshot(apic)) 2041 return; 2042 2043 advance_periodic_target_expiration(apic); 2044 } 2045 2046 hrtimer_start(&apic->lapic_timer.timer, 2047 apic->lapic_timer.target_expiration, 2048 HRTIMER_MODE_ABS_HARD); 2049 } 2050 2051 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu) 2052 { 2053 if (!lapic_in_kernel(vcpu)) 2054 return false; 2055 2056 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; 2057 } 2058 2059 static void cancel_hv_timer(struct kvm_lapic *apic) 2060 { 2061 WARN_ON(preemptible()); 2062 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 2063 static_call(kvm_x86_cancel_hv_timer)(apic->vcpu); 2064 apic->lapic_timer.hv_timer_in_use = false; 2065 } 2066 2067 static bool start_hv_timer(struct kvm_lapic *apic) 2068 { 2069 struct kvm_timer *ktimer = &apic->lapic_timer; 2070 struct kvm_vcpu *vcpu = apic->vcpu; 2071 bool expired; 2072 2073 WARN_ON(preemptible()); 2074 if (!kvm_can_use_hv_timer(vcpu)) 2075 return false; 2076 2077 if (!ktimer->tscdeadline) 2078 return false; 2079 2080 if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired)) 2081 return false; 2082 2083 ktimer->hv_timer_in_use = true; 2084 hrtimer_cancel(&ktimer->timer); 2085 2086 /* 2087 * To simplify handling the periodic timer, leave the hv timer running 2088 * even if the deadline timer has expired, i.e. rely on the resulting 2089 * VM-Exit to recompute the periodic timer's target expiration. 2090 */ 2091 if (!apic_lvtt_period(apic)) { 2092 /* 2093 * Cancel the hv timer if the sw timer fired while the hv timer 2094 * was being programmed, or if the hv timer itself expired. 2095 */ 2096 if (atomic_read(&ktimer->pending)) { 2097 cancel_hv_timer(apic); 2098 } else if (expired) { 2099 apic_timer_expired(apic, false); 2100 cancel_hv_timer(apic); 2101 } 2102 } 2103 2104 trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use); 2105 2106 return true; 2107 } 2108 2109 static void start_sw_timer(struct kvm_lapic *apic) 2110 { 2111 struct kvm_timer *ktimer = &apic->lapic_timer; 2112 2113 WARN_ON(preemptible()); 2114 if (apic->lapic_timer.hv_timer_in_use) 2115 cancel_hv_timer(apic); 2116 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) 2117 return; 2118 2119 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 2120 start_sw_period(apic); 2121 else if (apic_lvtt_tscdeadline(apic)) 2122 start_sw_tscdeadline(apic); 2123 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); 2124 } 2125 2126 static void restart_apic_timer(struct kvm_lapic *apic) 2127 { 2128 preempt_disable(); 2129 2130 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending)) 2131 goto out; 2132 2133 if (!start_hv_timer(apic)) 2134 start_sw_timer(apic); 2135 out: 2136 preempt_enable(); 2137 } 2138 2139 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) 2140 { 2141 struct kvm_lapic *apic = vcpu->arch.apic; 2142 2143 preempt_disable(); 2144 /* If the preempt notifier has already run, it also called apic_timer_expired */ 2145 if (!apic->lapic_timer.hv_timer_in_use) 2146 goto out; 2147 WARN_ON(kvm_vcpu_is_blocking(vcpu)); 2148 apic_timer_expired(apic, false); 2149 cancel_hv_timer(apic); 2150 2151 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { 2152 advance_periodic_target_expiration(apic); 2153 restart_apic_timer(apic); 2154 } 2155 out: 2156 preempt_enable(); 2157 } 2158 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer); 2159 2160 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) 2161 { 2162 restart_apic_timer(vcpu->arch.apic); 2163 } 2164 2165 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) 2166 { 2167 struct kvm_lapic *apic = vcpu->arch.apic; 2168 2169 preempt_disable(); 2170 /* Possibly the TSC deadline timer is not enabled yet */ 2171 if (apic->lapic_timer.hv_timer_in_use) 2172 start_sw_timer(apic); 2173 preempt_enable(); 2174 } 2175 2176 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) 2177 { 2178 struct kvm_lapic *apic = vcpu->arch.apic; 2179 2180 WARN_ON(!apic->lapic_timer.hv_timer_in_use); 2181 restart_apic_timer(apic); 2182 } 2183 2184 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg) 2185 { 2186 atomic_set(&apic->lapic_timer.pending, 0); 2187 2188 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) 2189 && !set_target_expiration(apic, count_reg)) 2190 return; 2191 2192 restart_apic_timer(apic); 2193 } 2194 2195 static void start_apic_timer(struct kvm_lapic *apic) 2196 { 2197 __start_apic_timer(apic, APIC_TMICT); 2198 } 2199 2200 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 2201 { 2202 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); 2203 2204 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { 2205 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; 2206 if (lvt0_in_nmi_mode) { 2207 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 2208 } else 2209 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); 2210 } 2211 } 2212 2213 static int get_lvt_index(u32 reg) 2214 { 2215 if (reg == APIC_LVTCMCI) 2216 return LVT_CMCI; 2217 if (reg < APIC_LVTT || reg > APIC_LVTERR) 2218 return -1; 2219 return array_index_nospec( 2220 (reg - APIC_LVTT) >> 4, KVM_APIC_MAX_NR_LVT_ENTRIES); 2221 } 2222 2223 static int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 2224 { 2225 int ret = 0; 2226 2227 trace_kvm_apic_write(reg, val); 2228 2229 switch (reg) { 2230 case APIC_ID: /* Local APIC ID */ 2231 if (!apic_x2apic_mode(apic)) { 2232 kvm_apic_set_xapic_id(apic, val >> 24); 2233 } else { 2234 ret = 1; 2235 } 2236 break; 2237 2238 case APIC_TASKPRI: 2239 report_tpr_access(apic, true); 2240 apic_set_tpr(apic, val & 0xff); 2241 break; 2242 2243 case APIC_EOI: 2244 apic_set_eoi(apic); 2245 break; 2246 2247 case APIC_LDR: 2248 if (!apic_x2apic_mode(apic)) 2249 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 2250 else 2251 ret = 1; 2252 break; 2253 2254 case APIC_DFR: 2255 if (!apic_x2apic_mode(apic)) 2256 kvm_apic_set_dfr(apic, val | 0x0FFFFFFF); 2257 else 2258 ret = 1; 2259 break; 2260 2261 case APIC_SPIV: { 2262 u32 mask = 0x3ff; 2263 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 2264 mask |= APIC_SPIV_DIRECTED_EOI; 2265 apic_set_spiv(apic, val & mask); 2266 if (!(val & APIC_SPIV_APIC_ENABLED)) { 2267 int i; 2268 2269 for (i = 0; i < apic->nr_lvt_entries; i++) { 2270 kvm_lapic_set_reg(apic, APIC_LVTx(i), 2271 kvm_lapic_get_reg(apic, APIC_LVTx(i)) | APIC_LVT_MASKED); 2272 } 2273 apic_update_lvtt(apic); 2274 atomic_set(&apic->lapic_timer.pending, 0); 2275 2276 } 2277 break; 2278 } 2279 case APIC_ICR: 2280 WARN_ON_ONCE(apic_x2apic_mode(apic)); 2281 2282 /* No delay here, so we always clear the pending bit */ 2283 val &= ~APIC_ICR_BUSY; 2284 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2)); 2285 kvm_lapic_set_reg(apic, APIC_ICR, val); 2286 break; 2287 case APIC_ICR2: 2288 if (apic_x2apic_mode(apic)) 2289 ret = 1; 2290 else 2291 kvm_lapic_set_reg(apic, APIC_ICR2, val & 0xff000000); 2292 break; 2293 2294 case APIC_LVT0: 2295 apic_manage_nmi_watchdog(apic, val); 2296 fallthrough; 2297 case APIC_LVTTHMR: 2298 case APIC_LVTPC: 2299 case APIC_LVT1: 2300 case APIC_LVTERR: 2301 case APIC_LVTCMCI: { 2302 u32 index = get_lvt_index(reg); 2303 if (!kvm_lapic_lvt_supported(apic, index)) { 2304 ret = 1; 2305 break; 2306 } 2307 if (!kvm_apic_sw_enabled(apic)) 2308 val |= APIC_LVT_MASKED; 2309 val &= apic_lvt_mask[index]; 2310 kvm_lapic_set_reg(apic, reg, val); 2311 break; 2312 } 2313 2314 case APIC_LVTT: 2315 if (!kvm_apic_sw_enabled(apic)) 2316 val |= APIC_LVT_MASKED; 2317 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 2318 kvm_lapic_set_reg(apic, APIC_LVTT, val); 2319 apic_update_lvtt(apic); 2320 break; 2321 2322 case APIC_TMICT: 2323 if (apic_lvtt_tscdeadline(apic)) 2324 break; 2325 2326 cancel_apic_timer(apic); 2327 kvm_lapic_set_reg(apic, APIC_TMICT, val); 2328 start_apic_timer(apic); 2329 break; 2330 2331 case APIC_TDCR: { 2332 uint32_t old_divisor = apic->divide_count; 2333 2334 kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb); 2335 update_divide_count(apic); 2336 if (apic->divide_count != old_divisor && 2337 apic->lapic_timer.period) { 2338 hrtimer_cancel(&apic->lapic_timer.timer); 2339 update_target_expiration(apic, old_divisor); 2340 restart_apic_timer(apic); 2341 } 2342 break; 2343 } 2344 case APIC_ESR: 2345 if (apic_x2apic_mode(apic) && val != 0) 2346 ret = 1; 2347 break; 2348 2349 case APIC_SELF_IPI: 2350 /* 2351 * Self-IPI exists only when x2APIC is enabled. Bits 7:0 hold 2352 * the vector, everything else is reserved. 2353 */ 2354 if (!apic_x2apic_mode(apic) || (val & ~APIC_VECTOR_MASK)) 2355 ret = 1; 2356 else 2357 kvm_apic_send_ipi(apic, APIC_DEST_SELF | val, 0); 2358 break; 2359 default: 2360 ret = 1; 2361 break; 2362 } 2363 2364 /* 2365 * Recalculate APIC maps if necessary, e.g. if the software enable bit 2366 * was toggled, the APIC ID changed, etc... The maps are marked dirty 2367 * on relevant changes, i.e. this is a nop for most writes. 2368 */ 2369 kvm_recalculate_apic_map(apic->vcpu->kvm); 2370 2371 return ret; 2372 } 2373 2374 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 2375 gpa_t address, int len, const void *data) 2376 { 2377 struct kvm_lapic *apic = to_lapic(this); 2378 unsigned int offset = address - apic->base_address; 2379 u32 val; 2380 2381 if (!apic_mmio_in_range(apic, address)) 2382 return -EOPNOTSUPP; 2383 2384 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { 2385 if (!kvm_check_has_quirk(vcpu->kvm, 2386 KVM_X86_QUIRK_LAPIC_MMIO_HOLE)) 2387 return -EOPNOTSUPP; 2388 2389 return 0; 2390 } 2391 2392 /* 2393 * APIC register must be aligned on 128-bits boundary. 2394 * 32/64/128 bits registers must be accessed thru 32 bits. 2395 * Refer SDM 8.4.1 2396 */ 2397 if (len != 4 || (offset & 0xf)) 2398 return 0; 2399 2400 val = *(u32*)data; 2401 2402 kvm_lapic_reg_write(apic, offset & 0xff0, val); 2403 2404 return 0; 2405 } 2406 2407 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 2408 { 2409 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 2410 } 2411 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 2412 2413 /* emulate APIC access in a trap manner */ 2414 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 2415 { 2416 struct kvm_lapic *apic = vcpu->arch.apic; 2417 u64 val; 2418 2419 /* 2420 * ICR is a single 64-bit register when x2APIC is enabled. For legacy 2421 * xAPIC, ICR writes need to go down the common (slightly slower) path 2422 * to get the upper half from ICR2. 2423 */ 2424 if (apic_x2apic_mode(apic) && offset == APIC_ICR) { 2425 val = kvm_lapic_get_reg64(apic, APIC_ICR); 2426 kvm_apic_send_ipi(apic, (u32)val, (u32)(val >> 32)); 2427 trace_kvm_apic_write(APIC_ICR, val); 2428 } else { 2429 /* TODO: optimize to just emulate side effect w/o one more write */ 2430 val = kvm_lapic_get_reg(apic, offset); 2431 kvm_lapic_reg_write(apic, offset, (u32)val); 2432 } 2433 } 2434 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 2435 2436 void kvm_free_lapic(struct kvm_vcpu *vcpu) 2437 { 2438 struct kvm_lapic *apic = vcpu->arch.apic; 2439 2440 if (!vcpu->arch.apic) 2441 return; 2442 2443 hrtimer_cancel(&apic->lapic_timer.timer); 2444 2445 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 2446 static_branch_slow_dec_deferred(&apic_hw_disabled); 2447 2448 if (!apic->sw_enabled) 2449 static_branch_slow_dec_deferred(&apic_sw_disabled); 2450 2451 if (apic->regs) 2452 free_page((unsigned long)apic->regs); 2453 2454 kfree(apic); 2455 } 2456 2457 /* 2458 *---------------------------------------------------------------------- 2459 * LAPIC interface 2460 *---------------------------------------------------------------------- 2461 */ 2462 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 2463 { 2464 struct kvm_lapic *apic = vcpu->arch.apic; 2465 2466 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) 2467 return 0; 2468 2469 return apic->lapic_timer.tscdeadline; 2470 } 2471 2472 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 2473 { 2474 struct kvm_lapic *apic = vcpu->arch.apic; 2475 2476 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) 2477 return; 2478 2479 hrtimer_cancel(&apic->lapic_timer.timer); 2480 apic->lapic_timer.tscdeadline = data; 2481 start_apic_timer(apic); 2482 } 2483 2484 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 2485 { 2486 apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4); 2487 } 2488 2489 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 2490 { 2491 u64 tpr; 2492 2493 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 2494 2495 return (tpr & 0xf0) >> 4; 2496 } 2497 2498 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 2499 { 2500 u64 old_value = vcpu->arch.apic_base; 2501 struct kvm_lapic *apic = vcpu->arch.apic; 2502 2503 vcpu->arch.apic_base = value; 2504 2505 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) 2506 kvm_update_cpuid_runtime(vcpu); 2507 2508 if (!apic) 2509 return; 2510 2511 /* update jump label if enable bit changes */ 2512 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 2513 if (value & MSR_IA32_APICBASE_ENABLE) { 2514 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2515 static_branch_slow_dec_deferred(&apic_hw_disabled); 2516 /* Check if there are APF page ready requests pending */ 2517 kvm_make_request(KVM_REQ_APF_READY, vcpu); 2518 } else { 2519 static_branch_inc(&apic_hw_disabled.key); 2520 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 2521 } 2522 } 2523 2524 if ((old_value ^ value) & X2APIC_ENABLE) { 2525 if (value & X2APIC_ENABLE) 2526 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); 2527 else if (value & MSR_IA32_APICBASE_ENABLE) 2528 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2529 } 2530 2531 if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) { 2532 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 2533 static_call_cond(kvm_x86_set_virtual_apic_mode)(vcpu); 2534 } 2535 2536 apic->base_address = apic->vcpu->arch.apic_base & 2537 MSR_IA32_APICBASE_BASE; 2538 2539 if ((value & MSR_IA32_APICBASE_ENABLE) && 2540 apic->base_address != APIC_DEFAULT_PHYS_BASE) { 2541 kvm_set_apicv_inhibit(apic->vcpu->kvm, 2542 APICV_INHIBIT_REASON_APIC_BASE_MODIFIED); 2543 } 2544 } 2545 2546 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu) 2547 { 2548 struct kvm_lapic *apic = vcpu->arch.apic; 2549 2550 if (apic->apicv_active) { 2551 /* irr_pending is always true when apicv is activated. */ 2552 apic->irr_pending = true; 2553 apic->isr_count = 1; 2554 } else { 2555 /* 2556 * Don't clear irr_pending, searching the IRR can race with 2557 * updates from the CPU as APICv is still active from hardware's 2558 * perspective. The flag will be cleared as appropriate when 2559 * KVM injects the interrupt. 2560 */ 2561 apic->isr_count = count_vectors(apic->regs + APIC_ISR); 2562 } 2563 apic->highest_isr_cache = -1; 2564 } 2565 2566 int kvm_alloc_apic_access_page(struct kvm *kvm) 2567 { 2568 struct page *page; 2569 void __user *hva; 2570 int ret = 0; 2571 2572 mutex_lock(&kvm->slots_lock); 2573 if (kvm->arch.apic_access_memslot_enabled || 2574 kvm->arch.apic_access_memslot_inhibited) 2575 goto out; 2576 2577 hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 2578 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE); 2579 if (IS_ERR(hva)) { 2580 ret = PTR_ERR(hva); 2581 goto out; 2582 } 2583 2584 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 2585 if (is_error_page(page)) { 2586 ret = -EFAULT; 2587 goto out; 2588 } 2589 2590 /* 2591 * Do not pin the page in memory, so that memory hot-unplug 2592 * is able to migrate it. 2593 */ 2594 put_page(page); 2595 kvm->arch.apic_access_memslot_enabled = true; 2596 out: 2597 mutex_unlock(&kvm->slots_lock); 2598 return ret; 2599 } 2600 EXPORT_SYMBOL_GPL(kvm_alloc_apic_access_page); 2601 2602 void kvm_inhibit_apic_access_page(struct kvm_vcpu *vcpu) 2603 { 2604 struct kvm *kvm = vcpu->kvm; 2605 2606 if (!kvm->arch.apic_access_memslot_enabled) 2607 return; 2608 2609 kvm_vcpu_srcu_read_unlock(vcpu); 2610 2611 mutex_lock(&kvm->slots_lock); 2612 2613 if (kvm->arch.apic_access_memslot_enabled) { 2614 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0); 2615 /* 2616 * Clear "enabled" after the memslot is deleted so that a 2617 * different vCPU doesn't get a false negative when checking 2618 * the flag out of slots_lock. No additional memory barrier is 2619 * needed as modifying memslots requires waiting other vCPUs to 2620 * drop SRCU (see above), and false positives are ok as the 2621 * flag is rechecked after acquiring slots_lock. 2622 */ 2623 kvm->arch.apic_access_memslot_enabled = false; 2624 2625 /* 2626 * Mark the memslot as inhibited to prevent reallocating the 2627 * memslot during vCPU creation, e.g. if a vCPU is hotplugged. 2628 */ 2629 kvm->arch.apic_access_memslot_inhibited = true; 2630 } 2631 2632 mutex_unlock(&kvm->slots_lock); 2633 2634 kvm_vcpu_srcu_read_lock(vcpu); 2635 } 2636 2637 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) 2638 { 2639 struct kvm_lapic *apic = vcpu->arch.apic; 2640 u64 msr_val; 2641 int i; 2642 2643 if (!init_event) { 2644 msr_val = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE; 2645 if (kvm_vcpu_is_reset_bsp(vcpu)) 2646 msr_val |= MSR_IA32_APICBASE_BSP; 2647 kvm_lapic_set_base(vcpu, msr_val); 2648 } 2649 2650 if (!apic) 2651 return; 2652 2653 /* Stop the timer in case it's a reset to an active apic */ 2654 hrtimer_cancel(&apic->lapic_timer.timer); 2655 2656 /* The xAPIC ID is set at RESET even if the APIC was already enabled. */ 2657 if (!init_event) 2658 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); 2659 kvm_apic_set_version(apic->vcpu); 2660 2661 for (i = 0; i < apic->nr_lvt_entries; i++) 2662 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); 2663 apic_update_lvtt(apic); 2664 if (kvm_vcpu_is_reset_bsp(vcpu) && 2665 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED)) 2666 kvm_lapic_set_reg(apic, APIC_LVT0, 2667 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 2668 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2669 2670 kvm_apic_set_dfr(apic, 0xffffffffU); 2671 apic_set_spiv(apic, 0xff); 2672 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); 2673 if (!apic_x2apic_mode(apic)) 2674 kvm_apic_set_ldr(apic, 0); 2675 kvm_lapic_set_reg(apic, APIC_ESR, 0); 2676 if (!apic_x2apic_mode(apic)) { 2677 kvm_lapic_set_reg(apic, APIC_ICR, 0); 2678 kvm_lapic_set_reg(apic, APIC_ICR2, 0); 2679 } else { 2680 kvm_lapic_set_reg64(apic, APIC_ICR, 0); 2681 } 2682 kvm_lapic_set_reg(apic, APIC_TDCR, 0); 2683 kvm_lapic_set_reg(apic, APIC_TMICT, 0); 2684 for (i = 0; i < 8; i++) { 2685 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 2686 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 2687 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 2688 } 2689 kvm_apic_update_apicv(vcpu); 2690 update_divide_count(apic); 2691 atomic_set(&apic->lapic_timer.pending, 0); 2692 2693 vcpu->arch.pv_eoi.msr_val = 0; 2694 apic_update_ppr(apic); 2695 if (apic->apicv_active) { 2696 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); 2697 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, -1); 2698 static_call_cond(kvm_x86_hwapic_isr_update)(-1); 2699 } 2700 2701 vcpu->arch.apic_arb_prio = 0; 2702 vcpu->arch.apic_attention = 0; 2703 2704 kvm_recalculate_apic_map(vcpu->kvm); 2705 } 2706 2707 /* 2708 *---------------------------------------------------------------------- 2709 * timer interface 2710 *---------------------------------------------------------------------- 2711 */ 2712 2713 static bool lapic_is_periodic(struct kvm_lapic *apic) 2714 { 2715 return apic_lvtt_period(apic); 2716 } 2717 2718 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 2719 { 2720 struct kvm_lapic *apic = vcpu->arch.apic; 2721 2722 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) 2723 return atomic_read(&apic->lapic_timer.pending); 2724 2725 return 0; 2726 } 2727 2728 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 2729 { 2730 u32 reg = kvm_lapic_get_reg(apic, lvt_type); 2731 int vector, mode, trig_mode; 2732 2733 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 2734 vector = reg & APIC_VECTOR_MASK; 2735 mode = reg & APIC_MODE_MASK; 2736 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 2737 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 2738 NULL); 2739 } 2740 return 0; 2741 } 2742 2743 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 2744 { 2745 struct kvm_lapic *apic = vcpu->arch.apic; 2746 2747 if (apic) 2748 kvm_apic_local_deliver(apic, APIC_LVT0); 2749 } 2750 2751 static const struct kvm_io_device_ops apic_mmio_ops = { 2752 .read = apic_mmio_read, 2753 .write = apic_mmio_write, 2754 }; 2755 2756 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 2757 { 2758 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 2759 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 2760 2761 apic_timer_expired(apic, true); 2762 2763 if (lapic_is_periodic(apic)) { 2764 advance_periodic_target_expiration(apic); 2765 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 2766 return HRTIMER_RESTART; 2767 } else 2768 return HRTIMER_NORESTART; 2769 } 2770 2771 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns) 2772 { 2773 struct kvm_lapic *apic; 2774 2775 ASSERT(vcpu != NULL); 2776 2777 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT); 2778 if (!apic) 2779 goto nomem; 2780 2781 vcpu->arch.apic = apic; 2782 2783 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); 2784 if (!apic->regs) { 2785 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 2786 vcpu->vcpu_id); 2787 goto nomem_free_apic; 2788 } 2789 apic->vcpu = vcpu; 2790 2791 apic->nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); 2792 2793 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 2794 HRTIMER_MODE_ABS_HARD); 2795 apic->lapic_timer.timer.function = apic_timer_fn; 2796 if (timer_advance_ns == -1) { 2797 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; 2798 lapic_timer_advance_dynamic = true; 2799 } else { 2800 apic->lapic_timer.timer_advance_ns = timer_advance_ns; 2801 lapic_timer_advance_dynamic = false; 2802 } 2803 2804 /* 2805 * Stuff the APIC ENABLE bit in lieu of temporarily incrementing 2806 * apic_hw_disabled; the full RESET value is set by kvm_lapic_reset(). 2807 */ 2808 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 2809 static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 2810 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 2811 2812 return 0; 2813 nomem_free_apic: 2814 kfree(apic); 2815 vcpu->arch.apic = NULL; 2816 nomem: 2817 return -ENOMEM; 2818 } 2819 2820 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 2821 { 2822 struct kvm_lapic *apic = vcpu->arch.apic; 2823 u32 ppr; 2824 2825 if (!kvm_apic_present(vcpu)) 2826 return -1; 2827 2828 __apic_update_ppr(apic, &ppr); 2829 return apic_has_interrupt_for_ppr(apic, ppr); 2830 } 2831 EXPORT_SYMBOL_GPL(kvm_apic_has_interrupt); 2832 2833 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 2834 { 2835 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); 2836 2837 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 2838 return 1; 2839 if ((lvt0 & APIC_LVT_MASKED) == 0 && 2840 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 2841 return 1; 2842 return 0; 2843 } 2844 2845 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 2846 { 2847 struct kvm_lapic *apic = vcpu->arch.apic; 2848 2849 if (atomic_read(&apic->lapic_timer.pending) > 0) { 2850 kvm_apic_inject_pending_timer_irqs(apic); 2851 atomic_set(&apic->lapic_timer.pending, 0); 2852 } 2853 } 2854 2855 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 2856 { 2857 int vector = kvm_apic_has_interrupt(vcpu); 2858 struct kvm_lapic *apic = vcpu->arch.apic; 2859 u32 ppr; 2860 2861 if (vector == -1) 2862 return -1; 2863 2864 /* 2865 * We get here even with APIC virtualization enabled, if doing 2866 * nested virtualization and L1 runs with the "acknowledge interrupt 2867 * on exit" mode. Then we cannot inject the interrupt via RVI, 2868 * because the process would deliver it through the IDT. 2869 */ 2870 2871 apic_clear_irr(vector, apic); 2872 if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) { 2873 /* 2874 * For auto-EOI interrupts, there might be another pending 2875 * interrupt above PPR, so check whether to raise another 2876 * KVM_REQ_EVENT. 2877 */ 2878 apic_update_ppr(apic); 2879 } else { 2880 /* 2881 * For normal interrupts, PPR has been raised and there cannot 2882 * be a higher-priority pending interrupt---except if there was 2883 * a concurrent interrupt injection, but that would have 2884 * triggered KVM_REQ_EVENT already. 2885 */ 2886 apic_set_isr(vector, apic); 2887 __apic_update_ppr(apic, &ppr); 2888 } 2889 2890 return vector; 2891 } 2892 2893 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, 2894 struct kvm_lapic_state *s, bool set) 2895 { 2896 if (apic_x2apic_mode(vcpu->arch.apic)) { 2897 u32 *id = (u32 *)(s->regs + APIC_ID); 2898 u32 *ldr = (u32 *)(s->regs + APIC_LDR); 2899 u64 icr; 2900 2901 if (vcpu->kvm->arch.x2apic_format) { 2902 if (*id != vcpu->vcpu_id) 2903 return -EINVAL; 2904 } else { 2905 if (set) 2906 *id >>= 24; 2907 else 2908 *id <<= 24; 2909 } 2910 2911 /* 2912 * In x2APIC mode, the LDR is fixed and based on the id. And 2913 * ICR is internally a single 64-bit register, but needs to be 2914 * split to ICR+ICR2 in userspace for backwards compatibility. 2915 */ 2916 if (set) { 2917 *ldr = kvm_apic_calc_x2apic_ldr(*id); 2918 2919 icr = __kvm_lapic_get_reg(s->regs, APIC_ICR) | 2920 (u64)__kvm_lapic_get_reg(s->regs, APIC_ICR2) << 32; 2921 __kvm_lapic_set_reg64(s->regs, APIC_ICR, icr); 2922 } else { 2923 icr = __kvm_lapic_get_reg64(s->regs, APIC_ICR); 2924 __kvm_lapic_set_reg(s->regs, APIC_ICR2, icr >> 32); 2925 } 2926 } 2927 2928 return 0; 2929 } 2930 2931 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2932 { 2933 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); 2934 2935 /* 2936 * Get calculated timer current count for remaining timer period (if 2937 * any) and store it in the returned register set. 2938 */ 2939 __kvm_lapic_set_reg(s->regs, APIC_TMCCT, 2940 __apic_read(vcpu->arch.apic, APIC_TMCCT)); 2941 2942 return kvm_apic_state_fixup(vcpu, s, false); 2943 } 2944 2945 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) 2946 { 2947 struct kvm_lapic *apic = vcpu->arch.apic; 2948 int r; 2949 2950 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 2951 /* set SPIV separately to get count of SW disabled APICs right */ 2952 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 2953 2954 r = kvm_apic_state_fixup(vcpu, s, true); 2955 if (r) { 2956 kvm_recalculate_apic_map(vcpu->kvm); 2957 return r; 2958 } 2959 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); 2960 2961 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); 2962 kvm_recalculate_apic_map(vcpu->kvm); 2963 kvm_apic_set_version(vcpu); 2964 2965 apic_update_ppr(apic); 2966 cancel_apic_timer(apic); 2967 apic->lapic_timer.expired_tscdeadline = 0; 2968 apic_update_lvtt(apic); 2969 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); 2970 update_divide_count(apic); 2971 __start_apic_timer(apic, APIC_TMCCT); 2972 kvm_lapic_set_reg(apic, APIC_TMCCT, 0); 2973 kvm_apic_update_apicv(vcpu); 2974 if (apic->apicv_active) { 2975 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu); 2976 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, apic_find_highest_irr(apic)); 2977 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic)); 2978 } 2979 kvm_make_request(KVM_REQ_EVENT, vcpu); 2980 if (ioapic_in_kernel(vcpu->kvm)) 2981 kvm_rtc_eoi_tracking_restore_one(vcpu); 2982 2983 vcpu->arch.apic_arb_prio = 0; 2984 2985 return 0; 2986 } 2987 2988 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 2989 { 2990 struct hrtimer *timer; 2991 2992 if (!lapic_in_kernel(vcpu) || 2993 kvm_can_post_timer_interrupt(vcpu)) 2994 return; 2995 2996 timer = &vcpu->arch.apic->lapic_timer.timer; 2997 if (hrtimer_cancel(timer)) 2998 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD); 2999 } 3000 3001 /* 3002 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 3003 * 3004 * Detect whether guest triggered PV EOI since the 3005 * last entry. If yes, set EOI on guests's behalf. 3006 * Clear PV EOI in guest memory in any case. 3007 */ 3008 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 3009 struct kvm_lapic *apic) 3010 { 3011 int vector; 3012 /* 3013 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 3014 * and KVM_PV_EOI_ENABLED in guest memory as follows: 3015 * 3016 * KVM_APIC_PV_EOI_PENDING is unset: 3017 * -> host disabled PV EOI. 3018 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 3019 * -> host enabled PV EOI, guest did not execute EOI yet. 3020 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 3021 * -> host enabled PV EOI, guest executed EOI. 3022 */ 3023 BUG_ON(!pv_eoi_enabled(vcpu)); 3024 3025 if (pv_eoi_test_and_clr_pending(vcpu)) 3026 return; 3027 vector = apic_set_eoi(apic); 3028 trace_kvm_pv_eoi(apic, vector); 3029 } 3030 3031 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 3032 { 3033 u32 data; 3034 3035 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 3036 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 3037 3038 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 3039 return; 3040 3041 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 3042 sizeof(u32))) 3043 return; 3044 3045 apic_set_tpr(vcpu->arch.apic, data & 0xff); 3046 } 3047 3048 /* 3049 * apic_sync_pv_eoi_to_guest - called before vmentry 3050 * 3051 * Detect whether it's safe to enable PV EOI and 3052 * if yes do so. 3053 */ 3054 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 3055 struct kvm_lapic *apic) 3056 { 3057 if (!pv_eoi_enabled(vcpu) || 3058 /* IRR set or many bits in ISR: could be nested. */ 3059 apic->irr_pending || 3060 /* Cache not set: could be safe but we don't bother. */ 3061 apic->highest_isr_cache == -1 || 3062 /* Need EOI to update ioapic. */ 3063 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { 3064 /* 3065 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 3066 * so we need not do anything here. 3067 */ 3068 return; 3069 } 3070 3071 pv_eoi_set_pending(apic->vcpu); 3072 } 3073 3074 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 3075 { 3076 u32 data, tpr; 3077 int max_irr, max_isr; 3078 struct kvm_lapic *apic = vcpu->arch.apic; 3079 3080 apic_sync_pv_eoi_to_guest(vcpu, apic); 3081 3082 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 3083 return; 3084 3085 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; 3086 max_irr = apic_find_highest_irr(apic); 3087 if (max_irr < 0) 3088 max_irr = 0; 3089 max_isr = apic_find_highest_isr(apic); 3090 if (max_isr < 0) 3091 max_isr = 0; 3092 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 3093 3094 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 3095 sizeof(u32)); 3096 } 3097 3098 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 3099 { 3100 if (vapic_addr) { 3101 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 3102 &vcpu->arch.apic->vapic_cache, 3103 vapic_addr, sizeof(u32))) 3104 return -EINVAL; 3105 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 3106 } else { 3107 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 3108 } 3109 3110 vcpu->arch.apic->vapic_addr = vapic_addr; 3111 return 0; 3112 } 3113 3114 int kvm_x2apic_icr_write(struct kvm_lapic *apic, u64 data) 3115 { 3116 data &= ~APIC_ICR_BUSY; 3117 3118 kvm_apic_send_ipi(apic, (u32)data, (u32)(data >> 32)); 3119 kvm_lapic_set_reg64(apic, APIC_ICR, data); 3120 trace_kvm_apic_write(APIC_ICR, data); 3121 return 0; 3122 } 3123 3124 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data) 3125 { 3126 u32 low; 3127 3128 if (reg == APIC_ICR) { 3129 *data = kvm_lapic_get_reg64(apic, APIC_ICR); 3130 return 0; 3131 } 3132 3133 if (kvm_lapic_reg_read(apic, reg, 4, &low)) 3134 return 1; 3135 3136 *data = low; 3137 3138 return 0; 3139 } 3140 3141 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data) 3142 { 3143 /* 3144 * ICR is a 64-bit register in x2APIC mode (and Hyper-V PV vAPIC) and 3145 * can be written as such, all other registers remain accessible only 3146 * through 32-bit reads/writes. 3147 */ 3148 if (reg == APIC_ICR) 3149 return kvm_x2apic_icr_write(apic, data); 3150 3151 /* Bits 63:32 are reserved in all other registers. */ 3152 if (data >> 32) 3153 return 1; 3154 3155 return kvm_lapic_reg_write(apic, reg, (u32)data); 3156 } 3157 3158 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 3159 { 3160 struct kvm_lapic *apic = vcpu->arch.apic; 3161 u32 reg = (msr - APIC_BASE_MSR) << 4; 3162 3163 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 3164 return 1; 3165 3166 return kvm_lapic_msr_write(apic, reg, data); 3167 } 3168 3169 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 3170 { 3171 struct kvm_lapic *apic = vcpu->arch.apic; 3172 u32 reg = (msr - APIC_BASE_MSR) << 4; 3173 3174 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) 3175 return 1; 3176 3177 return kvm_lapic_msr_read(apic, reg, data); 3178 } 3179 3180 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 3181 { 3182 if (!lapic_in_kernel(vcpu)) 3183 return 1; 3184 3185 return kvm_lapic_msr_write(vcpu->arch.apic, reg, data); 3186 } 3187 3188 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 3189 { 3190 if (!lapic_in_kernel(vcpu)) 3191 return 1; 3192 3193 return kvm_lapic_msr_read(vcpu->arch.apic, reg, data); 3194 } 3195 3196 int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) 3197 { 3198 u64 addr = data & ~KVM_MSR_ENABLED; 3199 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; 3200 unsigned long new_len; 3201 int ret; 3202 3203 if (!IS_ALIGNED(addr, 4)) 3204 return 1; 3205 3206 if (data & KVM_MSR_ENABLED) { 3207 if (addr == ghc->gpa && len <= ghc->len) 3208 new_len = ghc->len; 3209 else 3210 new_len = len; 3211 3212 ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); 3213 if (ret) 3214 return ret; 3215 } 3216 3217 vcpu->arch.pv_eoi.msr_val = data; 3218 3219 return 0; 3220 } 3221 3222 int kvm_apic_accept_events(struct kvm_vcpu *vcpu) 3223 { 3224 struct kvm_lapic *apic = vcpu->arch.apic; 3225 u8 sipi_vector; 3226 int r; 3227 3228 if (!kvm_apic_has_pending_init_or_sipi(vcpu)) 3229 return 0; 3230 3231 if (is_guest_mode(vcpu)) { 3232 r = kvm_check_nested_events(vcpu); 3233 if (r < 0) 3234 return r == -EBUSY ? 0 : r; 3235 /* 3236 * Continue processing INIT/SIPI even if a nested VM-Exit 3237 * occurred, e.g. pending SIPIs should be dropped if INIT+SIPI 3238 * are blocked as a result of transitioning to VMX root mode. 3239 */ 3240 } 3241 3242 /* 3243 * INITs are blocked while CPU is in specific states (SMM, VMX root 3244 * mode, SVM with GIF=0), while SIPIs are dropped if the CPU isn't in 3245 * wait-for-SIPI (WFS). 3246 */ 3247 if (!kvm_apic_init_sipi_allowed(vcpu)) { 3248 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); 3249 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 3250 return 0; 3251 } 3252 3253 if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) { 3254 kvm_vcpu_reset(vcpu, true); 3255 if (kvm_vcpu_is_bsp(apic->vcpu)) 3256 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3257 else 3258 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 3259 } 3260 if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) { 3261 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 3262 /* evaluate pending_events before reading the vector */ 3263 smp_rmb(); 3264 sipi_vector = apic->sipi_vector; 3265 static_call(kvm_x86_vcpu_deliver_sipi_vector)(vcpu, sipi_vector); 3266 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3267 } 3268 } 3269 return 0; 3270 } 3271 3272 void kvm_lapic_exit(void) 3273 { 3274 static_key_deferred_flush(&apic_hw_disabled); 3275 WARN_ON(static_branch_unlikely(&apic_hw_disabled.key)); 3276 static_key_deferred_flush(&apic_sw_disabled); 3277 WARN_ON(static_branch_unlikely(&apic_sw_disabled.key)); 3278 } 3279