1 2 /* 3 * Local APIC virtualization 4 * 5 * Copyright (C) 2006 Qumranet, Inc. 6 * Copyright (C) 2007 Novell 7 * Copyright (C) 2007 Intel 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Dor Laor <dor.laor@qumranet.com> 12 * Gregory Haskins <ghaskins@novell.com> 13 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 14 * 15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/mm.h> 24 #include <linux/highmem.h> 25 #include <linux/smp.h> 26 #include <linux/hrtimer.h> 27 #include <linux/io.h> 28 #include <linux/module.h> 29 #include <linux/math64.h> 30 #include <linux/slab.h> 31 #include <asm/processor.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <asm/delay.h> 37 #include <linux/atomic.h> 38 #include <linux/jump_label.h> 39 #include "kvm_cache_regs.h" 40 #include "irq.h" 41 #include "trace.h" 42 #include "x86.h" 43 #include "cpuid.h" 44 45 #ifndef CONFIG_X86_64 46 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 47 #else 48 #define mod_64(x, y) ((x) % (y)) 49 #endif 50 51 #define PRId64 "d" 52 #define PRIx64 "llx" 53 #define PRIu64 "u" 54 #define PRIo64 "o" 55 56 #define APIC_BUS_CYCLE_NS 1 57 58 /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ 59 #define apic_debug(fmt, arg...) 60 61 #define APIC_LVT_NUM 6 62 /* 14 is the version for Xeon and Pentium 8.4.8*/ 63 #define APIC_VERSION (0x14UL | ((APIC_LVT_NUM - 1) << 16)) 64 #define LAPIC_MMIO_LENGTH (1 << 12) 65 /* followed define is not in apicdef.h */ 66 #define APIC_SHORT_MASK 0xc0000 67 #define APIC_DEST_NOSHORT 0x0 68 #define APIC_DEST_MASK 0x800 69 #define MAX_APIC_VECTOR 256 70 #define APIC_VECTORS_PER_REG 32 71 72 #define APIC_BROADCAST 0xFF 73 #define X2APIC_BROADCAST 0xFFFFFFFFul 74 75 #define VEC_POS(v) ((v) & (32 - 1)) 76 #define REG_POS(v) (((v) >> 5) << 4) 77 78 static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) 79 { 80 *((u32 *) (apic->regs + reg_off)) = val; 81 } 82 83 static inline int apic_test_vector(int vec, void *bitmap) 84 { 85 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 86 } 87 88 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 89 { 90 struct kvm_lapic *apic = vcpu->arch.apic; 91 92 return apic_test_vector(vector, apic->regs + APIC_ISR) || 93 apic_test_vector(vector, apic->regs + APIC_IRR); 94 } 95 96 static inline void apic_set_vector(int vec, void *bitmap) 97 { 98 set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 99 } 100 101 static inline void apic_clear_vector(int vec, void *bitmap) 102 { 103 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 104 } 105 106 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 107 { 108 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 109 } 110 111 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 112 { 113 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 114 } 115 116 struct static_key_deferred apic_hw_disabled __read_mostly; 117 struct static_key_deferred apic_sw_disabled __read_mostly; 118 119 static inline int apic_enabled(struct kvm_lapic *apic) 120 { 121 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 122 } 123 124 #define LVT_MASK \ 125 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 126 127 #define LINT_MASK \ 128 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 129 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 130 131 static inline int kvm_apic_id(struct kvm_lapic *apic) 132 { 133 return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff; 134 } 135 136 static void recalculate_apic_map(struct kvm *kvm) 137 { 138 struct kvm_apic_map *new, *old = NULL; 139 struct kvm_vcpu *vcpu; 140 int i; 141 142 new = kzalloc(sizeof(struct kvm_apic_map), GFP_KERNEL); 143 144 mutex_lock(&kvm->arch.apic_map_lock); 145 146 if (!new) 147 goto out; 148 149 new->ldr_bits = 8; 150 /* flat mode is default */ 151 new->cid_shift = 8; 152 new->cid_mask = 0; 153 new->lid_mask = 0xff; 154 new->broadcast = APIC_BROADCAST; 155 156 kvm_for_each_vcpu(i, vcpu, kvm) { 157 struct kvm_lapic *apic = vcpu->arch.apic; 158 159 if (!kvm_apic_present(vcpu)) 160 continue; 161 162 if (apic_x2apic_mode(apic)) { 163 new->ldr_bits = 32; 164 new->cid_shift = 16; 165 new->cid_mask = new->lid_mask = 0xffff; 166 new->broadcast = X2APIC_BROADCAST; 167 } else if (kvm_apic_get_reg(apic, APIC_LDR)) { 168 if (kvm_apic_get_reg(apic, APIC_DFR) == 169 APIC_DFR_CLUSTER) { 170 new->cid_shift = 4; 171 new->cid_mask = 0xf; 172 new->lid_mask = 0xf; 173 } else { 174 new->cid_shift = 8; 175 new->cid_mask = 0; 176 new->lid_mask = 0xff; 177 } 178 } 179 180 /* 181 * All APICs have to be configured in the same mode by an OS. 182 * We take advatage of this while building logical id loockup 183 * table. After reset APICs are in software disabled mode, so if 184 * we find apic with different setting we assume this is the mode 185 * OS wants all apics to be in; build lookup table accordingly. 186 */ 187 if (kvm_apic_sw_enabled(apic)) 188 break; 189 } 190 191 kvm_for_each_vcpu(i, vcpu, kvm) { 192 struct kvm_lapic *apic = vcpu->arch.apic; 193 u16 cid, lid; 194 u32 ldr, aid; 195 196 if (!kvm_apic_present(vcpu)) 197 continue; 198 199 aid = kvm_apic_id(apic); 200 ldr = kvm_apic_get_reg(apic, APIC_LDR); 201 cid = apic_cluster_id(new, ldr); 202 lid = apic_logical_id(new, ldr); 203 204 if (aid < ARRAY_SIZE(new->phys_map)) 205 new->phys_map[aid] = apic; 206 if (lid && cid < ARRAY_SIZE(new->logical_map)) 207 new->logical_map[cid][ffs(lid) - 1] = apic; 208 } 209 out: 210 old = rcu_dereference_protected(kvm->arch.apic_map, 211 lockdep_is_held(&kvm->arch.apic_map_lock)); 212 rcu_assign_pointer(kvm->arch.apic_map, new); 213 mutex_unlock(&kvm->arch.apic_map_lock); 214 215 if (old) 216 kfree_rcu(old, rcu); 217 218 kvm_vcpu_request_scan_ioapic(kvm); 219 } 220 221 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 222 { 223 bool enabled = val & APIC_SPIV_APIC_ENABLED; 224 225 apic_set_reg(apic, APIC_SPIV, val); 226 227 if (enabled != apic->sw_enabled) { 228 apic->sw_enabled = enabled; 229 if (enabled) { 230 static_key_slow_dec_deferred(&apic_sw_disabled); 231 recalculate_apic_map(apic->vcpu->kvm); 232 } else 233 static_key_slow_inc(&apic_sw_disabled.key); 234 } 235 } 236 237 static inline void kvm_apic_set_id(struct kvm_lapic *apic, u8 id) 238 { 239 apic_set_reg(apic, APIC_ID, id << 24); 240 recalculate_apic_map(apic->vcpu->kvm); 241 } 242 243 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 244 { 245 apic_set_reg(apic, APIC_LDR, id); 246 recalculate_apic_map(apic->vcpu->kvm); 247 } 248 249 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 250 { 251 return !(kvm_apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 252 } 253 254 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 255 { 256 return kvm_apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 257 } 258 259 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 260 { 261 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; 262 } 263 264 static inline int apic_lvtt_period(struct kvm_lapic *apic) 265 { 266 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; 267 } 268 269 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 270 { 271 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; 272 } 273 274 static inline int apic_lvt_nmi_mode(u32 lvt_val) 275 { 276 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 277 } 278 279 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 280 { 281 struct kvm_lapic *apic = vcpu->arch.apic; 282 struct kvm_cpuid_entry2 *feat; 283 u32 v = APIC_VERSION; 284 285 if (!kvm_vcpu_has_lapic(vcpu)) 286 return; 287 288 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 289 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31)))) 290 v |= APIC_LVR_DIRECTED_EOI; 291 apic_set_reg(apic, APIC_LVR, v); 292 } 293 294 static const unsigned int apic_lvt_mask[APIC_LVT_NUM] = { 295 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 296 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 297 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 298 LINT_MASK, LINT_MASK, /* LVT0-1 */ 299 LVT_MASK /* LVTERR */ 300 }; 301 302 static int find_highest_vector(void *bitmap) 303 { 304 int vec; 305 u32 *reg; 306 307 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 308 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 309 reg = bitmap + REG_POS(vec); 310 if (*reg) 311 return fls(*reg) - 1 + vec; 312 } 313 314 return -1; 315 } 316 317 static u8 count_vectors(void *bitmap) 318 { 319 int vec; 320 u32 *reg; 321 u8 count = 0; 322 323 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 324 reg = bitmap + REG_POS(vec); 325 count += hweight32(*reg); 326 } 327 328 return count; 329 } 330 331 void __kvm_apic_update_irr(u32 *pir, void *regs) 332 { 333 u32 i, pir_val; 334 335 for (i = 0; i <= 7; i++) { 336 pir_val = xchg(&pir[i], 0); 337 if (pir_val) 338 *((u32 *)(regs + APIC_IRR + i * 0x10)) |= pir_val; 339 } 340 } 341 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr); 342 343 void kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir) 344 { 345 struct kvm_lapic *apic = vcpu->arch.apic; 346 347 __kvm_apic_update_irr(pir, apic->regs); 348 } 349 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 350 351 static inline void apic_set_irr(int vec, struct kvm_lapic *apic) 352 { 353 apic_set_vector(vec, apic->regs + APIC_IRR); 354 /* 355 * irr_pending must be true if any interrupt is pending; set it after 356 * APIC_IRR to avoid race with apic_clear_irr 357 */ 358 apic->irr_pending = true; 359 } 360 361 static inline int apic_search_irr(struct kvm_lapic *apic) 362 { 363 return find_highest_vector(apic->regs + APIC_IRR); 364 } 365 366 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 367 { 368 int result; 369 370 /* 371 * Note that irr_pending is just a hint. It will be always 372 * true with virtual interrupt delivery enabled. 373 */ 374 if (!apic->irr_pending) 375 return -1; 376 377 kvm_x86_ops->sync_pir_to_irr(apic->vcpu); 378 result = apic_search_irr(apic); 379 ASSERT(result == -1 || result >= 16); 380 381 return result; 382 } 383 384 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 385 { 386 struct kvm_vcpu *vcpu; 387 388 vcpu = apic->vcpu; 389 390 if (unlikely(kvm_apic_vid_enabled(vcpu->kvm))) { 391 /* try to update RVI */ 392 apic_clear_vector(vec, apic->regs + APIC_IRR); 393 kvm_make_request(KVM_REQ_EVENT, vcpu); 394 } else { 395 apic->irr_pending = false; 396 apic_clear_vector(vec, apic->regs + APIC_IRR); 397 if (apic_search_irr(apic) != -1) 398 apic->irr_pending = true; 399 } 400 } 401 402 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 403 { 404 struct kvm_vcpu *vcpu; 405 406 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 407 return; 408 409 vcpu = apic->vcpu; 410 411 /* 412 * With APIC virtualization enabled, all caching is disabled 413 * because the processor can modify ISR under the hood. Instead 414 * just set SVI. 415 */ 416 if (unlikely(kvm_x86_ops->hwapic_isr_update)) 417 kvm_x86_ops->hwapic_isr_update(vcpu->kvm, vec); 418 else { 419 ++apic->isr_count; 420 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 421 /* 422 * ISR (in service register) bit is set when injecting an interrupt. 423 * The highest vector is injected. Thus the latest bit set matches 424 * the highest bit in ISR. 425 */ 426 apic->highest_isr_cache = vec; 427 } 428 } 429 430 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 431 { 432 int result; 433 434 /* 435 * Note that isr_count is always 1, and highest_isr_cache 436 * is always -1, with APIC virtualization enabled. 437 */ 438 if (!apic->isr_count) 439 return -1; 440 if (likely(apic->highest_isr_cache != -1)) 441 return apic->highest_isr_cache; 442 443 result = find_highest_vector(apic->regs + APIC_ISR); 444 ASSERT(result == -1 || result >= 16); 445 446 return result; 447 } 448 449 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 450 { 451 struct kvm_vcpu *vcpu; 452 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 453 return; 454 455 vcpu = apic->vcpu; 456 457 /* 458 * We do get here for APIC virtualization enabled if the guest 459 * uses the Hyper-V APIC enlightenment. In this case we may need 460 * to trigger a new interrupt delivery by writing the SVI field; 461 * on the other hand isr_count and highest_isr_cache are unused 462 * and must be left alone. 463 */ 464 if (unlikely(kvm_x86_ops->hwapic_isr_update)) 465 kvm_x86_ops->hwapic_isr_update(vcpu->kvm, 466 apic_find_highest_isr(apic)); 467 else { 468 --apic->isr_count; 469 BUG_ON(apic->isr_count < 0); 470 apic->highest_isr_cache = -1; 471 } 472 } 473 474 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 475 { 476 int highest_irr; 477 478 /* This may race with setting of irr in __apic_accept_irq() and 479 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 480 * will cause vmexit immediately and the value will be recalculated 481 * on the next vmentry. 482 */ 483 if (!kvm_vcpu_has_lapic(vcpu)) 484 return 0; 485 highest_irr = apic_find_highest_irr(vcpu->arch.apic); 486 487 return highest_irr; 488 } 489 490 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 491 int vector, int level, int trig_mode, 492 unsigned long *dest_map); 493 494 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 495 unsigned long *dest_map) 496 { 497 struct kvm_lapic *apic = vcpu->arch.apic; 498 499 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 500 irq->level, irq->trig_mode, dest_map); 501 } 502 503 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 504 { 505 506 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 507 sizeof(val)); 508 } 509 510 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 511 { 512 513 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 514 sizeof(*val)); 515 } 516 517 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 518 { 519 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 520 } 521 522 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) 523 { 524 u8 val; 525 if (pv_eoi_get_user(vcpu, &val) < 0) 526 apic_debug("Can't read EOI MSR value: 0x%llx\n", 527 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 528 return val & 0x1; 529 } 530 531 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 532 { 533 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) { 534 apic_debug("Can't set EOI MSR value: 0x%llx\n", 535 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 536 return; 537 } 538 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 539 } 540 541 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) 542 { 543 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) { 544 apic_debug("Can't clear EOI MSR value: 0x%llx\n", 545 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 546 return; 547 } 548 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 549 } 550 551 void kvm_apic_update_tmr(struct kvm_vcpu *vcpu, u32 *tmr) 552 { 553 struct kvm_lapic *apic = vcpu->arch.apic; 554 int i; 555 556 for (i = 0; i < 8; i++) 557 apic_set_reg(apic, APIC_TMR + 0x10 * i, tmr[i]); 558 } 559 560 static void apic_update_ppr(struct kvm_lapic *apic) 561 { 562 u32 tpr, isrv, ppr, old_ppr; 563 int isr; 564 565 old_ppr = kvm_apic_get_reg(apic, APIC_PROCPRI); 566 tpr = kvm_apic_get_reg(apic, APIC_TASKPRI); 567 isr = apic_find_highest_isr(apic); 568 isrv = (isr != -1) ? isr : 0; 569 570 if ((tpr & 0xf0) >= (isrv & 0xf0)) 571 ppr = tpr & 0xff; 572 else 573 ppr = isrv & 0xf0; 574 575 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", 576 apic, ppr, isr, isrv); 577 578 if (old_ppr != ppr) { 579 apic_set_reg(apic, APIC_PROCPRI, ppr); 580 if (ppr < old_ppr) 581 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 582 } 583 } 584 585 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 586 { 587 apic_set_reg(apic, APIC_TASKPRI, tpr); 588 apic_update_ppr(apic); 589 } 590 591 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 dest) 592 { 593 return dest == (apic_x2apic_mode(apic) ? 594 X2APIC_BROADCAST : APIC_BROADCAST); 595 } 596 597 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 dest) 598 { 599 return kvm_apic_id(apic) == dest || kvm_apic_broadcast(apic, dest); 600 } 601 602 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) 603 { 604 u32 logical_id; 605 606 if (kvm_apic_broadcast(apic, mda)) 607 return true; 608 609 logical_id = kvm_apic_get_reg(apic, APIC_LDR); 610 611 if (apic_x2apic_mode(apic)) 612 return ((logical_id >> 16) == (mda >> 16)) 613 && (logical_id & mda & 0xffff) != 0; 614 615 logical_id = GET_APIC_LOGICAL_ID(logical_id); 616 617 switch (kvm_apic_get_reg(apic, APIC_DFR)) { 618 case APIC_DFR_FLAT: 619 return (logical_id & mda) != 0; 620 case APIC_DFR_CLUSTER: 621 return ((logical_id >> 4) == (mda >> 4)) 622 && (logical_id & mda & 0xf) != 0; 623 default: 624 apic_debug("Bad DFR vcpu %d: %08x\n", 625 apic->vcpu->vcpu_id, kvm_apic_get_reg(apic, APIC_DFR)); 626 return false; 627 } 628 } 629 630 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 631 int short_hand, unsigned int dest, int dest_mode) 632 { 633 struct kvm_lapic *target = vcpu->arch.apic; 634 635 apic_debug("target %p, source %p, dest 0x%x, " 636 "dest_mode 0x%x, short_hand 0x%x\n", 637 target, source, dest, dest_mode, short_hand); 638 639 ASSERT(target); 640 switch (short_hand) { 641 case APIC_DEST_NOSHORT: 642 if (dest_mode == APIC_DEST_PHYSICAL) 643 return kvm_apic_match_physical_addr(target, dest); 644 else 645 return kvm_apic_match_logical_addr(target, dest); 646 case APIC_DEST_SELF: 647 return target == source; 648 case APIC_DEST_ALLINC: 649 return true; 650 case APIC_DEST_ALLBUT: 651 return target != source; 652 default: 653 apic_debug("kvm: apic: Bad dest shorthand value %x\n", 654 short_hand); 655 return false; 656 } 657 } 658 659 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 660 struct kvm_lapic_irq *irq, int *r, unsigned long *dest_map) 661 { 662 struct kvm_apic_map *map; 663 unsigned long bitmap = 1; 664 struct kvm_lapic **dst; 665 int i; 666 bool ret = false; 667 668 *r = -1; 669 670 if (irq->shorthand == APIC_DEST_SELF) { 671 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 672 return true; 673 } 674 675 if (irq->shorthand) 676 return false; 677 678 rcu_read_lock(); 679 map = rcu_dereference(kvm->arch.apic_map); 680 681 if (!map) 682 goto out; 683 684 if (irq->dest_id == map->broadcast) 685 goto out; 686 687 ret = true; 688 689 if (irq->dest_mode == APIC_DEST_PHYSICAL) { 690 if (irq->dest_id >= ARRAY_SIZE(map->phys_map)) 691 goto out; 692 693 dst = &map->phys_map[irq->dest_id]; 694 } else { 695 u32 mda = irq->dest_id << (32 - map->ldr_bits); 696 u16 cid = apic_cluster_id(map, mda); 697 698 if (cid >= ARRAY_SIZE(map->logical_map)) 699 goto out; 700 701 dst = map->logical_map[cid]; 702 703 bitmap = apic_logical_id(map, mda); 704 705 if (irq->delivery_mode == APIC_DM_LOWEST) { 706 int l = -1; 707 for_each_set_bit(i, &bitmap, 16) { 708 if (!dst[i]) 709 continue; 710 if (l < 0) 711 l = i; 712 else if (kvm_apic_compare_prio(dst[i]->vcpu, dst[l]->vcpu) < 0) 713 l = i; 714 } 715 716 bitmap = (l >= 0) ? 1 << l : 0; 717 } 718 } 719 720 for_each_set_bit(i, &bitmap, 16) { 721 if (!dst[i]) 722 continue; 723 if (*r < 0) 724 *r = 0; 725 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 726 } 727 out: 728 rcu_read_unlock(); 729 return ret; 730 } 731 732 /* 733 * Add a pending IRQ into lapic. 734 * Return 1 if successfully added and 0 if discarded. 735 */ 736 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 737 int vector, int level, int trig_mode, 738 unsigned long *dest_map) 739 { 740 int result = 0; 741 struct kvm_vcpu *vcpu = apic->vcpu; 742 743 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 744 trig_mode, vector); 745 switch (delivery_mode) { 746 case APIC_DM_LOWEST: 747 vcpu->arch.apic_arb_prio++; 748 case APIC_DM_FIXED: 749 /* FIXME add logic for vcpu on reset */ 750 if (unlikely(!apic_enabled(apic))) 751 break; 752 753 result = 1; 754 755 if (dest_map) 756 __set_bit(vcpu->vcpu_id, dest_map); 757 758 if (kvm_x86_ops->deliver_posted_interrupt) 759 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector); 760 else { 761 apic_set_irr(vector, apic); 762 763 kvm_make_request(KVM_REQ_EVENT, vcpu); 764 kvm_vcpu_kick(vcpu); 765 } 766 break; 767 768 case APIC_DM_REMRD: 769 result = 1; 770 vcpu->arch.pv.pv_unhalted = 1; 771 kvm_make_request(KVM_REQ_EVENT, vcpu); 772 kvm_vcpu_kick(vcpu); 773 break; 774 775 case APIC_DM_SMI: 776 apic_debug("Ignoring guest SMI\n"); 777 break; 778 779 case APIC_DM_NMI: 780 result = 1; 781 kvm_inject_nmi(vcpu); 782 kvm_vcpu_kick(vcpu); 783 break; 784 785 case APIC_DM_INIT: 786 if (!trig_mode || level) { 787 result = 1; 788 /* assumes that there are only KVM_APIC_INIT/SIPI */ 789 apic->pending_events = (1UL << KVM_APIC_INIT); 790 /* make sure pending_events is visible before sending 791 * the request */ 792 smp_wmb(); 793 kvm_make_request(KVM_REQ_EVENT, vcpu); 794 kvm_vcpu_kick(vcpu); 795 } else { 796 apic_debug("Ignoring de-assert INIT to vcpu %d\n", 797 vcpu->vcpu_id); 798 } 799 break; 800 801 case APIC_DM_STARTUP: 802 apic_debug("SIPI to vcpu %d vector 0x%02x\n", 803 vcpu->vcpu_id, vector); 804 result = 1; 805 apic->sipi_vector = vector; 806 /* make sure sipi_vector is visible for the receiver */ 807 smp_wmb(); 808 set_bit(KVM_APIC_SIPI, &apic->pending_events); 809 kvm_make_request(KVM_REQ_EVENT, vcpu); 810 kvm_vcpu_kick(vcpu); 811 break; 812 813 case APIC_DM_EXTINT: 814 /* 815 * Should only be called by kvm_apic_local_deliver() with LVT0, 816 * before NMI watchdog was enabled. Already handled by 817 * kvm_apic_accept_pic_intr(). 818 */ 819 break; 820 821 default: 822 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 823 delivery_mode); 824 break; 825 } 826 return result; 827 } 828 829 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 830 { 831 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 832 } 833 834 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 835 { 836 if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) && 837 kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) { 838 int trigger_mode; 839 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 840 trigger_mode = IOAPIC_LEVEL_TRIG; 841 else 842 trigger_mode = IOAPIC_EDGE_TRIG; 843 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 844 } 845 } 846 847 static int apic_set_eoi(struct kvm_lapic *apic) 848 { 849 int vector = apic_find_highest_isr(apic); 850 851 trace_kvm_eoi(apic, vector); 852 853 /* 854 * Not every write EOI will has corresponding ISR, 855 * one example is when Kernel check timer on setup_IO_APIC 856 */ 857 if (vector == -1) 858 return vector; 859 860 apic_clear_isr(vector, apic); 861 apic_update_ppr(apic); 862 863 kvm_ioapic_send_eoi(apic, vector); 864 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 865 return vector; 866 } 867 868 /* 869 * this interface assumes a trap-like exit, which has already finished 870 * desired side effect including vISR and vPPR update. 871 */ 872 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 873 { 874 struct kvm_lapic *apic = vcpu->arch.apic; 875 876 trace_kvm_eoi(apic, vector); 877 878 kvm_ioapic_send_eoi(apic, vector); 879 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 880 } 881 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 882 883 static void apic_send_ipi(struct kvm_lapic *apic) 884 { 885 u32 icr_low = kvm_apic_get_reg(apic, APIC_ICR); 886 u32 icr_high = kvm_apic_get_reg(apic, APIC_ICR2); 887 struct kvm_lapic_irq irq; 888 889 irq.vector = icr_low & APIC_VECTOR_MASK; 890 irq.delivery_mode = icr_low & APIC_MODE_MASK; 891 irq.dest_mode = icr_low & APIC_DEST_MASK; 892 irq.level = icr_low & APIC_INT_ASSERT; 893 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 894 irq.shorthand = icr_low & APIC_SHORT_MASK; 895 if (apic_x2apic_mode(apic)) 896 irq.dest_id = icr_high; 897 else 898 irq.dest_id = GET_APIC_DEST_FIELD(icr_high); 899 900 trace_kvm_apic_ipi(icr_low, irq.dest_id); 901 902 apic_debug("icr_high 0x%x, icr_low 0x%x, " 903 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " 904 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n", 905 icr_high, icr_low, irq.shorthand, irq.dest_id, 906 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, 907 irq.vector); 908 909 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 910 } 911 912 static u32 apic_get_tmcct(struct kvm_lapic *apic) 913 { 914 ktime_t remaining; 915 s64 ns; 916 u32 tmcct; 917 918 ASSERT(apic != NULL); 919 920 /* if initial count is 0, current count should also be 0 */ 921 if (kvm_apic_get_reg(apic, APIC_TMICT) == 0 || 922 apic->lapic_timer.period == 0) 923 return 0; 924 925 remaining = hrtimer_get_remaining(&apic->lapic_timer.timer); 926 if (ktime_to_ns(remaining) < 0) 927 remaining = ktime_set(0, 0); 928 929 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 930 tmcct = div64_u64(ns, 931 (APIC_BUS_CYCLE_NS * apic->divide_count)); 932 933 return tmcct; 934 } 935 936 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 937 { 938 struct kvm_vcpu *vcpu = apic->vcpu; 939 struct kvm_run *run = vcpu->run; 940 941 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 942 run->tpr_access.rip = kvm_rip_read(vcpu); 943 run->tpr_access.is_write = write; 944 } 945 946 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 947 { 948 if (apic->vcpu->arch.tpr_access_reporting) 949 __report_tpr_access(apic, write); 950 } 951 952 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 953 { 954 u32 val = 0; 955 956 if (offset >= LAPIC_MMIO_LENGTH) 957 return 0; 958 959 switch (offset) { 960 case APIC_ID: 961 if (apic_x2apic_mode(apic)) 962 val = kvm_apic_id(apic); 963 else 964 val = kvm_apic_id(apic) << 24; 965 break; 966 case APIC_ARBPRI: 967 apic_debug("Access APIC ARBPRI register which is for P6\n"); 968 break; 969 970 case APIC_TMCCT: /* Timer CCR */ 971 if (apic_lvtt_tscdeadline(apic)) 972 return 0; 973 974 val = apic_get_tmcct(apic); 975 break; 976 case APIC_PROCPRI: 977 apic_update_ppr(apic); 978 val = kvm_apic_get_reg(apic, offset); 979 break; 980 case APIC_TASKPRI: 981 report_tpr_access(apic, false); 982 /* fall thru */ 983 default: 984 val = kvm_apic_get_reg(apic, offset); 985 break; 986 } 987 988 return val; 989 } 990 991 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 992 { 993 return container_of(dev, struct kvm_lapic, dev); 994 } 995 996 static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 997 void *data) 998 { 999 unsigned char alignment = offset & 0xf; 1000 u32 result; 1001 /* this bitmask has a bit cleared for each reserved register */ 1002 static const u64 rmask = 0x43ff01ffffffe70cULL; 1003 1004 if ((alignment + len) > 4) { 1005 apic_debug("KVM_APIC_READ: alignment error %x %d\n", 1006 offset, len); 1007 return 1; 1008 } 1009 1010 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) { 1011 apic_debug("KVM_APIC_READ: read reserved register %x\n", 1012 offset); 1013 return 1; 1014 } 1015 1016 result = __apic_read(apic, offset & ~0xf); 1017 1018 trace_kvm_apic_read(offset, result); 1019 1020 switch (len) { 1021 case 1: 1022 case 2: 1023 case 4: 1024 memcpy(data, (char *)&result + alignment, len); 1025 break; 1026 default: 1027 printk(KERN_ERR "Local APIC read with len = %x, " 1028 "should be 1,2, or 4 instead\n", len); 1029 break; 1030 } 1031 return 0; 1032 } 1033 1034 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1035 { 1036 return kvm_apic_hw_enabled(apic) && 1037 addr >= apic->base_address && 1038 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1039 } 1040 1041 static int apic_mmio_read(struct kvm_io_device *this, 1042 gpa_t address, int len, void *data) 1043 { 1044 struct kvm_lapic *apic = to_lapic(this); 1045 u32 offset = address - apic->base_address; 1046 1047 if (!apic_mmio_in_range(apic, address)) 1048 return -EOPNOTSUPP; 1049 1050 apic_reg_read(apic, offset, len, data); 1051 1052 return 0; 1053 } 1054 1055 static void update_divide_count(struct kvm_lapic *apic) 1056 { 1057 u32 tmp1, tmp2, tdcr; 1058 1059 tdcr = kvm_apic_get_reg(apic, APIC_TDCR); 1060 tmp1 = tdcr & 0xf; 1061 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1062 apic->divide_count = 0x1 << (tmp2 & 0x7); 1063 1064 apic_debug("timer divide count is 0x%x\n", 1065 apic->divide_count); 1066 } 1067 1068 static void apic_timer_expired(struct kvm_lapic *apic) 1069 { 1070 struct kvm_vcpu *vcpu = apic->vcpu; 1071 wait_queue_head_t *q = &vcpu->wq; 1072 struct kvm_timer *ktimer = &apic->lapic_timer; 1073 1074 if (atomic_read(&apic->lapic_timer.pending)) 1075 return; 1076 1077 atomic_inc(&apic->lapic_timer.pending); 1078 kvm_set_pending_timer(vcpu); 1079 1080 if (waitqueue_active(q)) 1081 wake_up_interruptible(q); 1082 1083 if (apic_lvtt_tscdeadline(apic)) 1084 ktimer->expired_tscdeadline = ktimer->tscdeadline; 1085 } 1086 1087 /* 1088 * On APICv, this test will cause a busy wait 1089 * during a higher-priority task. 1090 */ 1091 1092 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu) 1093 { 1094 struct kvm_lapic *apic = vcpu->arch.apic; 1095 u32 reg = kvm_apic_get_reg(apic, APIC_LVTT); 1096 1097 if (kvm_apic_hw_enabled(apic)) { 1098 int vec = reg & APIC_VECTOR_MASK; 1099 void *bitmap = apic->regs + APIC_ISR; 1100 1101 if (kvm_x86_ops->deliver_posted_interrupt) 1102 bitmap = apic->regs + APIC_IRR; 1103 1104 if (apic_test_vector(vec, bitmap)) 1105 return true; 1106 } 1107 return false; 1108 } 1109 1110 void wait_lapic_expire(struct kvm_vcpu *vcpu) 1111 { 1112 struct kvm_lapic *apic = vcpu->arch.apic; 1113 u64 guest_tsc, tsc_deadline; 1114 1115 if (!kvm_vcpu_has_lapic(vcpu)) 1116 return; 1117 1118 if (apic->lapic_timer.expired_tscdeadline == 0) 1119 return; 1120 1121 if (!lapic_timer_int_injected(vcpu)) 1122 return; 1123 1124 tsc_deadline = apic->lapic_timer.expired_tscdeadline; 1125 apic->lapic_timer.expired_tscdeadline = 0; 1126 guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu, native_read_tsc()); 1127 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); 1128 1129 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */ 1130 if (guest_tsc < tsc_deadline) 1131 __delay(tsc_deadline - guest_tsc); 1132 } 1133 1134 static void start_apic_timer(struct kvm_lapic *apic) 1135 { 1136 ktime_t now; 1137 1138 atomic_set(&apic->lapic_timer.pending, 0); 1139 1140 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { 1141 /* lapic timer in oneshot or periodic mode */ 1142 now = apic->lapic_timer.timer.base->get_time(); 1143 apic->lapic_timer.period = (u64)kvm_apic_get_reg(apic, APIC_TMICT) 1144 * APIC_BUS_CYCLE_NS * apic->divide_count; 1145 1146 if (!apic->lapic_timer.period) 1147 return; 1148 /* 1149 * Do not allow the guest to program periodic timers with small 1150 * interval, since the hrtimers are not throttled by the host 1151 * scheduler. 1152 */ 1153 if (apic_lvtt_period(apic)) { 1154 s64 min_period = min_timer_period_us * 1000LL; 1155 1156 if (apic->lapic_timer.period < min_period) { 1157 pr_info_ratelimited( 1158 "kvm: vcpu %i: requested %lld ns " 1159 "lapic timer period limited to %lld ns\n", 1160 apic->vcpu->vcpu_id, 1161 apic->lapic_timer.period, min_period); 1162 apic->lapic_timer.period = min_period; 1163 } 1164 } 1165 1166 hrtimer_start(&apic->lapic_timer.timer, 1167 ktime_add_ns(now, apic->lapic_timer.period), 1168 HRTIMER_MODE_ABS); 1169 1170 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" 1171 PRIx64 ", " 1172 "timer initial count 0x%x, period %lldns, " 1173 "expire @ 0x%016" PRIx64 ".\n", __func__, 1174 APIC_BUS_CYCLE_NS, ktime_to_ns(now), 1175 kvm_apic_get_reg(apic, APIC_TMICT), 1176 apic->lapic_timer.period, 1177 ktime_to_ns(ktime_add_ns(now, 1178 apic->lapic_timer.period))); 1179 } else if (apic_lvtt_tscdeadline(apic)) { 1180 /* lapic timer in tsc deadline mode */ 1181 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline; 1182 u64 ns = 0; 1183 ktime_t expire; 1184 struct kvm_vcpu *vcpu = apic->vcpu; 1185 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1186 unsigned long flags; 1187 1188 if (unlikely(!tscdeadline || !this_tsc_khz)) 1189 return; 1190 1191 local_irq_save(flags); 1192 1193 now = apic->lapic_timer.timer.base->get_time(); 1194 guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu, native_read_tsc()); 1195 if (likely(tscdeadline > guest_tsc)) { 1196 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1197 do_div(ns, this_tsc_khz); 1198 expire = ktime_add_ns(now, ns); 1199 expire = ktime_sub_ns(expire, lapic_timer_advance_ns); 1200 hrtimer_start(&apic->lapic_timer.timer, 1201 expire, HRTIMER_MODE_ABS); 1202 } else 1203 apic_timer_expired(apic); 1204 1205 local_irq_restore(flags); 1206 } 1207 } 1208 1209 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1210 { 1211 int nmi_wd_enabled = apic_lvt_nmi_mode(kvm_apic_get_reg(apic, APIC_LVT0)); 1212 1213 if (apic_lvt_nmi_mode(lvt0_val)) { 1214 if (!nmi_wd_enabled) { 1215 apic_debug("Receive NMI setting on APIC_LVT0 " 1216 "for cpu %d\n", apic->vcpu->vcpu_id); 1217 apic->vcpu->kvm->arch.vapics_in_nmi_mode++; 1218 } 1219 } else if (nmi_wd_enabled) 1220 apic->vcpu->kvm->arch.vapics_in_nmi_mode--; 1221 } 1222 1223 static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 1224 { 1225 int ret = 0; 1226 1227 trace_kvm_apic_write(reg, val); 1228 1229 switch (reg) { 1230 case APIC_ID: /* Local APIC ID */ 1231 if (!apic_x2apic_mode(apic)) 1232 kvm_apic_set_id(apic, val >> 24); 1233 else 1234 ret = 1; 1235 break; 1236 1237 case APIC_TASKPRI: 1238 report_tpr_access(apic, true); 1239 apic_set_tpr(apic, val & 0xff); 1240 break; 1241 1242 case APIC_EOI: 1243 apic_set_eoi(apic); 1244 break; 1245 1246 case APIC_LDR: 1247 if (!apic_x2apic_mode(apic)) 1248 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 1249 else 1250 ret = 1; 1251 break; 1252 1253 case APIC_DFR: 1254 if (!apic_x2apic_mode(apic)) { 1255 apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1256 recalculate_apic_map(apic->vcpu->kvm); 1257 } else 1258 ret = 1; 1259 break; 1260 1261 case APIC_SPIV: { 1262 u32 mask = 0x3ff; 1263 if (kvm_apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1264 mask |= APIC_SPIV_DIRECTED_EOI; 1265 apic_set_spiv(apic, val & mask); 1266 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1267 int i; 1268 u32 lvt_val; 1269 1270 for (i = 0; i < APIC_LVT_NUM; i++) { 1271 lvt_val = kvm_apic_get_reg(apic, 1272 APIC_LVTT + 0x10 * i); 1273 apic_set_reg(apic, APIC_LVTT + 0x10 * i, 1274 lvt_val | APIC_LVT_MASKED); 1275 } 1276 atomic_set(&apic->lapic_timer.pending, 0); 1277 1278 } 1279 break; 1280 } 1281 case APIC_ICR: 1282 /* No delay here, so we always clear the pending bit */ 1283 apic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); 1284 apic_send_ipi(apic); 1285 break; 1286 1287 case APIC_ICR2: 1288 if (!apic_x2apic_mode(apic)) 1289 val &= 0xff000000; 1290 apic_set_reg(apic, APIC_ICR2, val); 1291 break; 1292 1293 case APIC_LVT0: 1294 apic_manage_nmi_watchdog(apic, val); 1295 case APIC_LVTTHMR: 1296 case APIC_LVTPC: 1297 case APIC_LVT1: 1298 case APIC_LVTERR: 1299 /* TODO: Check vector */ 1300 if (!kvm_apic_sw_enabled(apic)) 1301 val |= APIC_LVT_MASKED; 1302 1303 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1304 apic_set_reg(apic, reg, val); 1305 1306 break; 1307 1308 case APIC_LVTT: { 1309 u32 timer_mode = val & apic->lapic_timer.timer_mode_mask; 1310 1311 if (apic->lapic_timer.timer_mode != timer_mode) { 1312 apic->lapic_timer.timer_mode = timer_mode; 1313 hrtimer_cancel(&apic->lapic_timer.timer); 1314 } 1315 1316 if (!kvm_apic_sw_enabled(apic)) 1317 val |= APIC_LVT_MASKED; 1318 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1319 apic_set_reg(apic, APIC_LVTT, val); 1320 break; 1321 } 1322 1323 case APIC_TMICT: 1324 if (apic_lvtt_tscdeadline(apic)) 1325 break; 1326 1327 hrtimer_cancel(&apic->lapic_timer.timer); 1328 apic_set_reg(apic, APIC_TMICT, val); 1329 start_apic_timer(apic); 1330 break; 1331 1332 case APIC_TDCR: 1333 if (val & 4) 1334 apic_debug("KVM_WRITE:TDCR %x\n", val); 1335 apic_set_reg(apic, APIC_TDCR, val); 1336 update_divide_count(apic); 1337 break; 1338 1339 case APIC_ESR: 1340 if (apic_x2apic_mode(apic) && val != 0) { 1341 apic_debug("KVM_WRITE:ESR not zero %x\n", val); 1342 ret = 1; 1343 } 1344 break; 1345 1346 case APIC_SELF_IPI: 1347 if (apic_x2apic_mode(apic)) { 1348 apic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff)); 1349 } else 1350 ret = 1; 1351 break; 1352 default: 1353 ret = 1; 1354 break; 1355 } 1356 if (ret) 1357 apic_debug("Local APIC Write to read-only register %x\n", reg); 1358 return ret; 1359 } 1360 1361 static int apic_mmio_write(struct kvm_io_device *this, 1362 gpa_t address, int len, const void *data) 1363 { 1364 struct kvm_lapic *apic = to_lapic(this); 1365 unsigned int offset = address - apic->base_address; 1366 u32 val; 1367 1368 if (!apic_mmio_in_range(apic, address)) 1369 return -EOPNOTSUPP; 1370 1371 /* 1372 * APIC register must be aligned on 128-bits boundary. 1373 * 32/64/128 bits registers must be accessed thru 32 bits. 1374 * Refer SDM 8.4.1 1375 */ 1376 if (len != 4 || (offset & 0xf)) { 1377 /* Don't shout loud, $infamous_os would cause only noise. */ 1378 apic_debug("apic write: bad size=%d %lx\n", len, (long)address); 1379 return 0; 1380 } 1381 1382 val = *(u32*)data; 1383 1384 /* too common printing */ 1385 if (offset != APIC_EOI) 1386 apic_debug("%s: offset 0x%x with length 0x%x, and value is " 1387 "0x%x\n", __func__, offset, len, val); 1388 1389 apic_reg_write(apic, offset & 0xff0, val); 1390 1391 return 0; 1392 } 1393 1394 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 1395 { 1396 if (kvm_vcpu_has_lapic(vcpu)) 1397 apic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 1398 } 1399 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 1400 1401 /* emulate APIC access in a trap manner */ 1402 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 1403 { 1404 u32 val = 0; 1405 1406 /* hw has done the conditional check and inst decode */ 1407 offset &= 0xff0; 1408 1409 apic_reg_read(vcpu->arch.apic, offset, 4, &val); 1410 1411 /* TODO: optimize to just emulate side effect w/o one more write */ 1412 apic_reg_write(vcpu->arch.apic, offset, val); 1413 } 1414 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 1415 1416 void kvm_free_lapic(struct kvm_vcpu *vcpu) 1417 { 1418 struct kvm_lapic *apic = vcpu->arch.apic; 1419 1420 if (!vcpu->arch.apic) 1421 return; 1422 1423 hrtimer_cancel(&apic->lapic_timer.timer); 1424 1425 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 1426 static_key_slow_dec_deferred(&apic_hw_disabled); 1427 1428 if (!apic->sw_enabled) 1429 static_key_slow_dec_deferred(&apic_sw_disabled); 1430 1431 if (apic->regs) 1432 free_page((unsigned long)apic->regs); 1433 1434 kfree(apic); 1435 } 1436 1437 /* 1438 *---------------------------------------------------------------------- 1439 * LAPIC interface 1440 *---------------------------------------------------------------------- 1441 */ 1442 1443 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 1444 { 1445 struct kvm_lapic *apic = vcpu->arch.apic; 1446 1447 if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) || 1448 apic_lvtt_period(apic)) 1449 return 0; 1450 1451 return apic->lapic_timer.tscdeadline; 1452 } 1453 1454 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 1455 { 1456 struct kvm_lapic *apic = vcpu->arch.apic; 1457 1458 if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) || 1459 apic_lvtt_period(apic)) 1460 return; 1461 1462 hrtimer_cancel(&apic->lapic_timer.timer); 1463 apic->lapic_timer.tscdeadline = data; 1464 start_apic_timer(apic); 1465 } 1466 1467 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 1468 { 1469 struct kvm_lapic *apic = vcpu->arch.apic; 1470 1471 if (!kvm_vcpu_has_lapic(vcpu)) 1472 return; 1473 1474 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 1475 | (kvm_apic_get_reg(apic, APIC_TASKPRI) & 4)); 1476 } 1477 1478 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 1479 { 1480 u64 tpr; 1481 1482 if (!kvm_vcpu_has_lapic(vcpu)) 1483 return 0; 1484 1485 tpr = (u64) kvm_apic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 1486 1487 return (tpr & 0xf0) >> 4; 1488 } 1489 1490 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 1491 { 1492 u64 old_value = vcpu->arch.apic_base; 1493 struct kvm_lapic *apic = vcpu->arch.apic; 1494 1495 if (!apic) { 1496 value |= MSR_IA32_APICBASE_BSP; 1497 vcpu->arch.apic_base = value; 1498 return; 1499 } 1500 1501 if (!kvm_vcpu_is_bsp(apic->vcpu)) 1502 value &= ~MSR_IA32_APICBASE_BSP; 1503 vcpu->arch.apic_base = value; 1504 1505 /* update jump label if enable bit changes */ 1506 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 1507 if (value & MSR_IA32_APICBASE_ENABLE) 1508 static_key_slow_dec_deferred(&apic_hw_disabled); 1509 else 1510 static_key_slow_inc(&apic_hw_disabled.key); 1511 recalculate_apic_map(vcpu->kvm); 1512 } 1513 1514 if ((old_value ^ value) & X2APIC_ENABLE) { 1515 if (value & X2APIC_ENABLE) { 1516 u32 id = kvm_apic_id(apic); 1517 u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf)); 1518 kvm_apic_set_ldr(apic, ldr); 1519 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true); 1520 } else 1521 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false); 1522 } 1523 1524 apic->base_address = apic->vcpu->arch.apic_base & 1525 MSR_IA32_APICBASE_BASE; 1526 1527 if ((value & MSR_IA32_APICBASE_ENABLE) && 1528 apic->base_address != APIC_DEFAULT_PHYS_BASE) 1529 pr_warn_once("APIC base relocation is unsupported by KVM"); 1530 1531 /* with FSB delivery interrupt, we can restart APIC functionality */ 1532 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " 1533 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); 1534 1535 } 1536 1537 void kvm_lapic_reset(struct kvm_vcpu *vcpu) 1538 { 1539 struct kvm_lapic *apic; 1540 int i; 1541 1542 apic_debug("%s\n", __func__); 1543 1544 ASSERT(vcpu); 1545 apic = vcpu->arch.apic; 1546 ASSERT(apic != NULL); 1547 1548 /* Stop the timer in case it's a reset to an active apic */ 1549 hrtimer_cancel(&apic->lapic_timer.timer); 1550 1551 kvm_apic_set_id(apic, vcpu->vcpu_id); 1552 kvm_apic_set_version(apic->vcpu); 1553 1554 for (i = 0; i < APIC_LVT_NUM; i++) 1555 apic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); 1556 apic->lapic_timer.timer_mode = 0; 1557 apic_set_reg(apic, APIC_LVT0, 1558 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 1559 1560 apic_set_reg(apic, APIC_DFR, 0xffffffffU); 1561 apic_set_spiv(apic, 0xff); 1562 apic_set_reg(apic, APIC_TASKPRI, 0); 1563 kvm_apic_set_ldr(apic, 0); 1564 apic_set_reg(apic, APIC_ESR, 0); 1565 apic_set_reg(apic, APIC_ICR, 0); 1566 apic_set_reg(apic, APIC_ICR2, 0); 1567 apic_set_reg(apic, APIC_TDCR, 0); 1568 apic_set_reg(apic, APIC_TMICT, 0); 1569 for (i = 0; i < 8; i++) { 1570 apic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 1571 apic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 1572 apic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 1573 } 1574 apic->irr_pending = kvm_apic_vid_enabled(vcpu->kvm); 1575 apic->isr_count = kvm_apic_vid_enabled(vcpu->kvm); 1576 apic->highest_isr_cache = -1; 1577 update_divide_count(apic); 1578 atomic_set(&apic->lapic_timer.pending, 0); 1579 if (kvm_vcpu_is_bsp(vcpu)) 1580 kvm_lapic_set_base(vcpu, 1581 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); 1582 vcpu->arch.pv_eoi.msr_val = 0; 1583 apic_update_ppr(apic); 1584 1585 vcpu->arch.apic_arb_prio = 0; 1586 vcpu->arch.apic_attention = 0; 1587 1588 apic_debug("%s: vcpu=%p, id=%d, base_msr=" 1589 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__, 1590 vcpu, kvm_apic_id(apic), 1591 vcpu->arch.apic_base, apic->base_address); 1592 } 1593 1594 /* 1595 *---------------------------------------------------------------------- 1596 * timer interface 1597 *---------------------------------------------------------------------- 1598 */ 1599 1600 static bool lapic_is_periodic(struct kvm_lapic *apic) 1601 { 1602 return apic_lvtt_period(apic); 1603 } 1604 1605 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 1606 { 1607 struct kvm_lapic *apic = vcpu->arch.apic; 1608 1609 if (kvm_vcpu_has_lapic(vcpu) && apic_enabled(apic) && 1610 apic_lvt_enabled(apic, APIC_LVTT)) 1611 return atomic_read(&apic->lapic_timer.pending); 1612 1613 return 0; 1614 } 1615 1616 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 1617 { 1618 u32 reg = kvm_apic_get_reg(apic, lvt_type); 1619 int vector, mode, trig_mode; 1620 1621 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 1622 vector = reg & APIC_VECTOR_MASK; 1623 mode = reg & APIC_MODE_MASK; 1624 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 1625 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 1626 NULL); 1627 } 1628 return 0; 1629 } 1630 1631 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 1632 { 1633 struct kvm_lapic *apic = vcpu->arch.apic; 1634 1635 if (apic) 1636 kvm_apic_local_deliver(apic, APIC_LVT0); 1637 } 1638 1639 static const struct kvm_io_device_ops apic_mmio_ops = { 1640 .read = apic_mmio_read, 1641 .write = apic_mmio_write, 1642 }; 1643 1644 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 1645 { 1646 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 1647 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 1648 1649 apic_timer_expired(apic); 1650 1651 if (lapic_is_periodic(apic)) { 1652 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 1653 return HRTIMER_RESTART; 1654 } else 1655 return HRTIMER_NORESTART; 1656 } 1657 1658 int kvm_create_lapic(struct kvm_vcpu *vcpu) 1659 { 1660 struct kvm_lapic *apic; 1661 1662 ASSERT(vcpu != NULL); 1663 apic_debug("apic_init %d\n", vcpu->vcpu_id); 1664 1665 apic = kzalloc(sizeof(*apic), GFP_KERNEL); 1666 if (!apic) 1667 goto nomem; 1668 1669 vcpu->arch.apic = apic; 1670 1671 apic->regs = (void *)get_zeroed_page(GFP_KERNEL); 1672 if (!apic->regs) { 1673 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 1674 vcpu->vcpu_id); 1675 goto nomem_free_apic; 1676 } 1677 apic->vcpu = vcpu; 1678 1679 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 1680 HRTIMER_MODE_ABS); 1681 apic->lapic_timer.timer.function = apic_timer_fn; 1682 1683 /* 1684 * APIC is created enabled. This will prevent kvm_lapic_set_base from 1685 * thinking that APIC satet has changed. 1686 */ 1687 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 1688 kvm_lapic_set_base(vcpu, 1689 APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE); 1690 1691 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 1692 kvm_lapic_reset(vcpu); 1693 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 1694 1695 return 0; 1696 nomem_free_apic: 1697 kfree(apic); 1698 nomem: 1699 return -ENOMEM; 1700 } 1701 1702 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 1703 { 1704 struct kvm_lapic *apic = vcpu->arch.apic; 1705 int highest_irr; 1706 1707 if (!kvm_vcpu_has_lapic(vcpu) || !apic_enabled(apic)) 1708 return -1; 1709 1710 apic_update_ppr(apic); 1711 highest_irr = apic_find_highest_irr(apic); 1712 if ((highest_irr == -1) || 1713 ((highest_irr & 0xF0) <= kvm_apic_get_reg(apic, APIC_PROCPRI))) 1714 return -1; 1715 return highest_irr; 1716 } 1717 1718 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 1719 { 1720 u32 lvt0 = kvm_apic_get_reg(vcpu->arch.apic, APIC_LVT0); 1721 int r = 0; 1722 1723 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 1724 r = 1; 1725 if ((lvt0 & APIC_LVT_MASKED) == 0 && 1726 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 1727 r = 1; 1728 return r; 1729 } 1730 1731 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 1732 { 1733 struct kvm_lapic *apic = vcpu->arch.apic; 1734 1735 if (!kvm_vcpu_has_lapic(vcpu)) 1736 return; 1737 1738 if (atomic_read(&apic->lapic_timer.pending) > 0) { 1739 kvm_apic_local_deliver(apic, APIC_LVTT); 1740 if (apic_lvtt_tscdeadline(apic)) 1741 apic->lapic_timer.tscdeadline = 0; 1742 atomic_set(&apic->lapic_timer.pending, 0); 1743 } 1744 } 1745 1746 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 1747 { 1748 int vector = kvm_apic_has_interrupt(vcpu); 1749 struct kvm_lapic *apic = vcpu->arch.apic; 1750 1751 if (vector == -1) 1752 return -1; 1753 1754 /* 1755 * We get here even with APIC virtualization enabled, if doing 1756 * nested virtualization and L1 runs with the "acknowledge interrupt 1757 * on exit" mode. Then we cannot inject the interrupt via RVI, 1758 * because the process would deliver it through the IDT. 1759 */ 1760 1761 apic_set_isr(vector, apic); 1762 apic_update_ppr(apic); 1763 apic_clear_irr(vector, apic); 1764 return vector; 1765 } 1766 1767 void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu, 1768 struct kvm_lapic_state *s) 1769 { 1770 struct kvm_lapic *apic = vcpu->arch.apic; 1771 1772 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 1773 /* set SPIV separately to get count of SW disabled APICs right */ 1774 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 1775 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); 1776 /* call kvm_apic_set_id() to put apic into apic_map */ 1777 kvm_apic_set_id(apic, kvm_apic_id(apic)); 1778 kvm_apic_set_version(vcpu); 1779 1780 apic_update_ppr(apic); 1781 hrtimer_cancel(&apic->lapic_timer.timer); 1782 update_divide_count(apic); 1783 start_apic_timer(apic); 1784 apic->irr_pending = true; 1785 apic->isr_count = kvm_apic_vid_enabled(vcpu->kvm) ? 1786 1 : count_vectors(apic->regs + APIC_ISR); 1787 apic->highest_isr_cache = -1; 1788 if (kvm_x86_ops->hwapic_irr_update) 1789 kvm_x86_ops->hwapic_irr_update(vcpu, 1790 apic_find_highest_irr(apic)); 1791 if (unlikely(kvm_x86_ops->hwapic_isr_update)) 1792 kvm_x86_ops->hwapic_isr_update(vcpu->kvm, 1793 apic_find_highest_isr(apic)); 1794 kvm_make_request(KVM_REQ_EVENT, vcpu); 1795 kvm_rtc_eoi_tracking_restore_one(vcpu); 1796 } 1797 1798 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 1799 { 1800 struct hrtimer *timer; 1801 1802 if (!kvm_vcpu_has_lapic(vcpu)) 1803 return; 1804 1805 timer = &vcpu->arch.apic->lapic_timer.timer; 1806 if (hrtimer_cancel(timer)) 1807 hrtimer_start_expires(timer, HRTIMER_MODE_ABS); 1808 } 1809 1810 /* 1811 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 1812 * 1813 * Detect whether guest triggered PV EOI since the 1814 * last entry. If yes, set EOI on guests's behalf. 1815 * Clear PV EOI in guest memory in any case. 1816 */ 1817 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 1818 struct kvm_lapic *apic) 1819 { 1820 bool pending; 1821 int vector; 1822 /* 1823 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 1824 * and KVM_PV_EOI_ENABLED in guest memory as follows: 1825 * 1826 * KVM_APIC_PV_EOI_PENDING is unset: 1827 * -> host disabled PV EOI. 1828 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 1829 * -> host enabled PV EOI, guest did not execute EOI yet. 1830 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 1831 * -> host enabled PV EOI, guest executed EOI. 1832 */ 1833 BUG_ON(!pv_eoi_enabled(vcpu)); 1834 pending = pv_eoi_get_pending(vcpu); 1835 /* 1836 * Clear pending bit in any case: it will be set again on vmentry. 1837 * While this might not be ideal from performance point of view, 1838 * this makes sure pv eoi is only enabled when we know it's safe. 1839 */ 1840 pv_eoi_clr_pending(vcpu); 1841 if (pending) 1842 return; 1843 vector = apic_set_eoi(apic); 1844 trace_kvm_pv_eoi(apic, vector); 1845 } 1846 1847 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 1848 { 1849 u32 data; 1850 1851 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 1852 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 1853 1854 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 1855 return; 1856 1857 kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 1858 sizeof(u32)); 1859 1860 apic_set_tpr(vcpu->arch.apic, data & 0xff); 1861 } 1862 1863 /* 1864 * apic_sync_pv_eoi_to_guest - called before vmentry 1865 * 1866 * Detect whether it's safe to enable PV EOI and 1867 * if yes do so. 1868 */ 1869 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 1870 struct kvm_lapic *apic) 1871 { 1872 if (!pv_eoi_enabled(vcpu) || 1873 /* IRR set or many bits in ISR: could be nested. */ 1874 apic->irr_pending || 1875 /* Cache not set: could be safe but we don't bother. */ 1876 apic->highest_isr_cache == -1 || 1877 /* Need EOI to update ioapic. */ 1878 kvm_ioapic_handles_vector(vcpu->kvm, apic->highest_isr_cache)) { 1879 /* 1880 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 1881 * so we need not do anything here. 1882 */ 1883 return; 1884 } 1885 1886 pv_eoi_set_pending(apic->vcpu); 1887 } 1888 1889 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 1890 { 1891 u32 data, tpr; 1892 int max_irr, max_isr; 1893 struct kvm_lapic *apic = vcpu->arch.apic; 1894 1895 apic_sync_pv_eoi_to_guest(vcpu, apic); 1896 1897 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 1898 return; 1899 1900 tpr = kvm_apic_get_reg(apic, APIC_TASKPRI) & 0xff; 1901 max_irr = apic_find_highest_irr(apic); 1902 if (max_irr < 0) 1903 max_irr = 0; 1904 max_isr = apic_find_highest_isr(apic); 1905 if (max_isr < 0) 1906 max_isr = 0; 1907 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 1908 1909 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 1910 sizeof(u32)); 1911 } 1912 1913 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 1914 { 1915 if (vapic_addr) { 1916 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 1917 &vcpu->arch.apic->vapic_cache, 1918 vapic_addr, sizeof(u32))) 1919 return -EINVAL; 1920 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 1921 } else { 1922 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 1923 } 1924 1925 vcpu->arch.apic->vapic_addr = vapic_addr; 1926 return 0; 1927 } 1928 1929 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1930 { 1931 struct kvm_lapic *apic = vcpu->arch.apic; 1932 u32 reg = (msr - APIC_BASE_MSR) << 4; 1933 1934 if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic)) 1935 return 1; 1936 1937 if (reg == APIC_ICR2) 1938 return 1; 1939 1940 /* if this is ICR write vector before command */ 1941 if (reg == APIC_ICR) 1942 apic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 1943 return apic_reg_write(apic, reg, (u32)data); 1944 } 1945 1946 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 1947 { 1948 struct kvm_lapic *apic = vcpu->arch.apic; 1949 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; 1950 1951 if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic)) 1952 return 1; 1953 1954 if (reg == APIC_DFR || reg == APIC_ICR2) { 1955 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n", 1956 reg); 1957 return 1; 1958 } 1959 1960 if (apic_reg_read(apic, reg, 4, &low)) 1961 return 1; 1962 if (reg == APIC_ICR) 1963 apic_reg_read(apic, APIC_ICR2, 4, &high); 1964 1965 *data = (((u64)high) << 32) | low; 1966 1967 return 0; 1968 } 1969 1970 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 1971 { 1972 struct kvm_lapic *apic = vcpu->arch.apic; 1973 1974 if (!kvm_vcpu_has_lapic(vcpu)) 1975 return 1; 1976 1977 /* if this is ICR write vector before command */ 1978 if (reg == APIC_ICR) 1979 apic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 1980 return apic_reg_write(apic, reg, (u32)data); 1981 } 1982 1983 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 1984 { 1985 struct kvm_lapic *apic = vcpu->arch.apic; 1986 u32 low, high = 0; 1987 1988 if (!kvm_vcpu_has_lapic(vcpu)) 1989 return 1; 1990 1991 if (apic_reg_read(apic, reg, 4, &low)) 1992 return 1; 1993 if (reg == APIC_ICR) 1994 apic_reg_read(apic, APIC_ICR2, 4, &high); 1995 1996 *data = (((u64)high) << 32) | low; 1997 1998 return 0; 1999 } 2000 2001 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data) 2002 { 2003 u64 addr = data & ~KVM_MSR_ENABLED; 2004 if (!IS_ALIGNED(addr, 4)) 2005 return 1; 2006 2007 vcpu->arch.pv_eoi.msr_val = data; 2008 if (!pv_eoi_enabled(vcpu)) 2009 return 0; 2010 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, 2011 addr, sizeof(u8)); 2012 } 2013 2014 void kvm_apic_accept_events(struct kvm_vcpu *vcpu) 2015 { 2016 struct kvm_lapic *apic = vcpu->arch.apic; 2017 u8 sipi_vector; 2018 unsigned long pe; 2019 2020 if (!kvm_vcpu_has_lapic(vcpu) || !apic->pending_events) 2021 return; 2022 2023 pe = xchg(&apic->pending_events, 0); 2024 2025 if (test_bit(KVM_APIC_INIT, &pe)) { 2026 kvm_lapic_reset(vcpu); 2027 kvm_vcpu_reset(vcpu); 2028 if (kvm_vcpu_is_bsp(apic->vcpu)) 2029 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2030 else 2031 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 2032 } 2033 if (test_bit(KVM_APIC_SIPI, &pe) && 2034 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 2035 /* evaluate pending_events before reading the vector */ 2036 smp_rmb(); 2037 sipi_vector = apic->sipi_vector; 2038 apic_debug("vcpu %d received sipi with vector # %x\n", 2039 vcpu->vcpu_id, sipi_vector); 2040 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector); 2041 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 2042 } 2043 } 2044 2045 void kvm_lapic_init(void) 2046 { 2047 /* do not patch jump label more than once per second */ 2048 jump_label_rate_limit(&apic_hw_disabled, HZ); 2049 jump_label_rate_limit(&apic_sw_disabled, HZ); 2050 } 2051