1 2 /* 3 * Local APIC virtualization 4 * 5 * Copyright (C) 2006 Qumranet, Inc. 6 * Copyright (C) 2007 Novell 7 * Copyright (C) 2007 Intel 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Dor Laor <dor.laor@qumranet.com> 12 * Gregory Haskins <ghaskins@novell.com> 13 * Yaozu (Eddie) Dong <eddie.dong@intel.com> 14 * 15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/kvm.h> 23 #include <linux/mm.h> 24 #include <linux/highmem.h> 25 #include <linux/smp.h> 26 #include <linux/hrtimer.h> 27 #include <linux/io.h> 28 #include <linux/module.h> 29 #include <linux/math64.h> 30 #include <linux/slab.h> 31 #include <asm/processor.h> 32 #include <asm/msr.h> 33 #include <asm/page.h> 34 #include <asm/current.h> 35 #include <asm/apicdef.h> 36 #include <linux/atomic.h> 37 #include <linux/jump_label.h> 38 #include "kvm_cache_regs.h" 39 #include "irq.h" 40 #include "trace.h" 41 #include "x86.h" 42 #include "cpuid.h" 43 44 #ifndef CONFIG_X86_64 45 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y)) 46 #else 47 #define mod_64(x, y) ((x) % (y)) 48 #endif 49 50 #define PRId64 "d" 51 #define PRIx64 "llx" 52 #define PRIu64 "u" 53 #define PRIo64 "o" 54 55 #define APIC_BUS_CYCLE_NS 1 56 57 /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ 58 #define apic_debug(fmt, arg...) 59 60 #define APIC_LVT_NUM 6 61 /* 14 is the version for Xeon and Pentium 8.4.8*/ 62 #define APIC_VERSION (0x14UL | ((APIC_LVT_NUM - 1) << 16)) 63 #define LAPIC_MMIO_LENGTH (1 << 12) 64 /* followed define is not in apicdef.h */ 65 #define APIC_SHORT_MASK 0xc0000 66 #define APIC_DEST_NOSHORT 0x0 67 #define APIC_DEST_MASK 0x800 68 #define MAX_APIC_VECTOR 256 69 #define APIC_VECTORS_PER_REG 32 70 71 #define VEC_POS(v) ((v) & (32 - 1)) 72 #define REG_POS(v) (((v) >> 5) << 4) 73 74 static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) 75 { 76 *((u32 *) (apic->regs + reg_off)) = val; 77 } 78 79 static inline int apic_test_vector(int vec, void *bitmap) 80 { 81 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 82 } 83 84 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) 85 { 86 struct kvm_lapic *apic = vcpu->arch.apic; 87 88 return apic_test_vector(vector, apic->regs + APIC_ISR) || 89 apic_test_vector(vector, apic->regs + APIC_IRR); 90 } 91 92 static inline void apic_set_vector(int vec, void *bitmap) 93 { 94 set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 95 } 96 97 static inline void apic_clear_vector(int vec, void *bitmap) 98 { 99 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 100 } 101 102 static inline int __apic_test_and_set_vector(int vec, void *bitmap) 103 { 104 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 105 } 106 107 static inline int __apic_test_and_clear_vector(int vec, void *bitmap) 108 { 109 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 110 } 111 112 struct static_key_deferred apic_hw_disabled __read_mostly; 113 struct static_key_deferred apic_sw_disabled __read_mostly; 114 115 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) 116 { 117 if ((kvm_apic_get_reg(apic, APIC_SPIV) ^ val) & APIC_SPIV_APIC_ENABLED) { 118 if (val & APIC_SPIV_APIC_ENABLED) 119 static_key_slow_dec_deferred(&apic_sw_disabled); 120 else 121 static_key_slow_inc(&apic_sw_disabled.key); 122 } 123 apic_set_reg(apic, APIC_SPIV, val); 124 } 125 126 static inline int apic_enabled(struct kvm_lapic *apic) 127 { 128 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); 129 } 130 131 #define LVT_MASK \ 132 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) 133 134 #define LINT_MASK \ 135 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 136 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 137 138 static inline int kvm_apic_id(struct kvm_lapic *apic) 139 { 140 return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff; 141 } 142 143 #define KVM_X2APIC_CID_BITS 0 144 145 static void recalculate_apic_map(struct kvm *kvm) 146 { 147 struct kvm_apic_map *new, *old = NULL; 148 struct kvm_vcpu *vcpu; 149 int i; 150 151 new = kzalloc(sizeof(struct kvm_apic_map), GFP_KERNEL); 152 153 mutex_lock(&kvm->arch.apic_map_lock); 154 155 if (!new) 156 goto out; 157 158 new->ldr_bits = 8; 159 /* flat mode is default */ 160 new->cid_shift = 8; 161 new->cid_mask = 0; 162 new->lid_mask = 0xff; 163 164 kvm_for_each_vcpu(i, vcpu, kvm) { 165 struct kvm_lapic *apic = vcpu->arch.apic; 166 u16 cid, lid; 167 u32 ldr; 168 169 if (!kvm_apic_present(vcpu)) 170 continue; 171 172 /* 173 * All APICs have to be configured in the same mode by an OS. 174 * We take advatage of this while building logical id loockup 175 * table. After reset APICs are in xapic/flat mode, so if we 176 * find apic with different setting we assume this is the mode 177 * OS wants all apics to be in; build lookup table accordingly. 178 */ 179 if (apic_x2apic_mode(apic)) { 180 new->ldr_bits = 32; 181 new->cid_shift = 16; 182 new->cid_mask = (1 << KVM_X2APIC_CID_BITS) - 1; 183 new->lid_mask = 0xffff; 184 } else if (kvm_apic_sw_enabled(apic) && 185 !new->cid_mask /* flat mode */ && 186 kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_CLUSTER) { 187 new->cid_shift = 4; 188 new->cid_mask = 0xf; 189 new->lid_mask = 0xf; 190 } 191 192 new->phys_map[kvm_apic_id(apic)] = apic; 193 194 ldr = kvm_apic_get_reg(apic, APIC_LDR); 195 cid = apic_cluster_id(new, ldr); 196 lid = apic_logical_id(new, ldr); 197 198 if (lid) 199 new->logical_map[cid][ffs(lid) - 1] = apic; 200 } 201 out: 202 old = rcu_dereference_protected(kvm->arch.apic_map, 203 lockdep_is_held(&kvm->arch.apic_map_lock)); 204 rcu_assign_pointer(kvm->arch.apic_map, new); 205 mutex_unlock(&kvm->arch.apic_map_lock); 206 207 if (old) 208 kfree_rcu(old, rcu); 209 210 kvm_vcpu_request_scan_ioapic(kvm); 211 } 212 213 static inline void kvm_apic_set_id(struct kvm_lapic *apic, u8 id) 214 { 215 apic_set_reg(apic, APIC_ID, id << 24); 216 recalculate_apic_map(apic->vcpu->kvm); 217 } 218 219 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) 220 { 221 apic_set_reg(apic, APIC_LDR, id); 222 recalculate_apic_map(apic->vcpu->kvm); 223 } 224 225 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 226 { 227 return !(kvm_apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 228 } 229 230 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 231 { 232 return kvm_apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 233 } 234 235 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 236 { 237 return ((kvm_apic_get_reg(apic, APIC_LVTT) & 238 apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT); 239 } 240 241 static inline int apic_lvtt_period(struct kvm_lapic *apic) 242 { 243 return ((kvm_apic_get_reg(apic, APIC_LVTT) & 244 apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC); 245 } 246 247 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 248 { 249 return ((kvm_apic_get_reg(apic, APIC_LVTT) & 250 apic->lapic_timer.timer_mode_mask) == 251 APIC_LVT_TIMER_TSCDEADLINE); 252 } 253 254 static inline int apic_lvt_nmi_mode(u32 lvt_val) 255 { 256 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI; 257 } 258 259 void kvm_apic_set_version(struct kvm_vcpu *vcpu) 260 { 261 struct kvm_lapic *apic = vcpu->arch.apic; 262 struct kvm_cpuid_entry2 *feat; 263 u32 v = APIC_VERSION; 264 265 if (!kvm_vcpu_has_lapic(vcpu)) 266 return; 267 268 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 269 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31)))) 270 v |= APIC_LVR_DIRECTED_EOI; 271 apic_set_reg(apic, APIC_LVR, v); 272 } 273 274 static const unsigned int apic_lvt_mask[APIC_LVT_NUM] = { 275 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 276 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 277 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 278 LINT_MASK, LINT_MASK, /* LVT0-1 */ 279 LVT_MASK /* LVTERR */ 280 }; 281 282 static int find_highest_vector(void *bitmap) 283 { 284 int vec; 285 u32 *reg; 286 287 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; 288 vec >= 0; vec -= APIC_VECTORS_PER_REG) { 289 reg = bitmap + REG_POS(vec); 290 if (*reg) 291 return fls(*reg) - 1 + vec; 292 } 293 294 return -1; 295 } 296 297 static u8 count_vectors(void *bitmap) 298 { 299 int vec; 300 u32 *reg; 301 u8 count = 0; 302 303 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { 304 reg = bitmap + REG_POS(vec); 305 count += hweight32(*reg); 306 } 307 308 return count; 309 } 310 311 void kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir) 312 { 313 u32 i, pir_val; 314 struct kvm_lapic *apic = vcpu->arch.apic; 315 316 for (i = 0; i <= 7; i++) { 317 pir_val = xchg(&pir[i], 0); 318 if (pir_val) 319 *((u32 *)(apic->regs + APIC_IRR + i * 0x10)) |= pir_val; 320 } 321 } 322 EXPORT_SYMBOL_GPL(kvm_apic_update_irr); 323 324 static inline void apic_set_irr(int vec, struct kvm_lapic *apic) 325 { 326 apic->irr_pending = true; 327 apic_set_vector(vec, apic->regs + APIC_IRR); 328 } 329 330 static inline int apic_search_irr(struct kvm_lapic *apic) 331 { 332 return find_highest_vector(apic->regs + APIC_IRR); 333 } 334 335 static inline int apic_find_highest_irr(struct kvm_lapic *apic) 336 { 337 int result; 338 339 /* 340 * Note that irr_pending is just a hint. It will be always 341 * true with virtual interrupt delivery enabled. 342 */ 343 if (!apic->irr_pending) 344 return -1; 345 346 kvm_x86_ops->sync_pir_to_irr(apic->vcpu); 347 result = apic_search_irr(apic); 348 ASSERT(result == -1 || result >= 16); 349 350 return result; 351 } 352 353 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) 354 { 355 struct kvm_vcpu *vcpu; 356 357 vcpu = apic->vcpu; 358 359 apic_clear_vector(vec, apic->regs + APIC_IRR); 360 if (unlikely(kvm_apic_vid_enabled(vcpu->kvm))) 361 /* try to update RVI */ 362 kvm_make_request(KVM_REQ_EVENT, vcpu); 363 else { 364 vec = apic_search_irr(apic); 365 apic->irr_pending = (vec != -1); 366 } 367 } 368 369 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) 370 { 371 struct kvm_vcpu *vcpu; 372 373 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) 374 return; 375 376 vcpu = apic->vcpu; 377 378 /* 379 * With APIC virtualization enabled, all caching is disabled 380 * because the processor can modify ISR under the hood. Instead 381 * just set SVI. 382 */ 383 if (unlikely(kvm_apic_vid_enabled(vcpu->kvm))) 384 kvm_x86_ops->hwapic_isr_update(vcpu->kvm, vec); 385 else { 386 ++apic->isr_count; 387 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); 388 /* 389 * ISR (in service register) bit is set when injecting an interrupt. 390 * The highest vector is injected. Thus the latest bit set matches 391 * the highest bit in ISR. 392 */ 393 apic->highest_isr_cache = vec; 394 } 395 } 396 397 static inline int apic_find_highest_isr(struct kvm_lapic *apic) 398 { 399 int result; 400 401 /* 402 * Note that isr_count is always 1, and highest_isr_cache 403 * is always -1, with APIC virtualization enabled. 404 */ 405 if (!apic->isr_count) 406 return -1; 407 if (likely(apic->highest_isr_cache != -1)) 408 return apic->highest_isr_cache; 409 410 result = find_highest_vector(apic->regs + APIC_ISR); 411 ASSERT(result == -1 || result >= 16); 412 413 return result; 414 } 415 416 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) 417 { 418 struct kvm_vcpu *vcpu; 419 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) 420 return; 421 422 vcpu = apic->vcpu; 423 424 /* 425 * We do get here for APIC virtualization enabled if the guest 426 * uses the Hyper-V APIC enlightenment. In this case we may need 427 * to trigger a new interrupt delivery by writing the SVI field; 428 * on the other hand isr_count and highest_isr_cache are unused 429 * and must be left alone. 430 */ 431 if (unlikely(kvm_apic_vid_enabled(vcpu->kvm))) 432 kvm_x86_ops->hwapic_isr_update(vcpu->kvm, 433 apic_find_highest_isr(apic)); 434 else { 435 --apic->isr_count; 436 BUG_ON(apic->isr_count < 0); 437 apic->highest_isr_cache = -1; 438 } 439 } 440 441 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 442 { 443 int highest_irr; 444 445 /* This may race with setting of irr in __apic_accept_irq() and 446 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq 447 * will cause vmexit immediately and the value will be recalculated 448 * on the next vmentry. 449 */ 450 if (!kvm_vcpu_has_lapic(vcpu)) 451 return 0; 452 highest_irr = apic_find_highest_irr(vcpu->arch.apic); 453 454 return highest_irr; 455 } 456 457 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 458 int vector, int level, int trig_mode, 459 unsigned long *dest_map); 460 461 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, 462 unsigned long *dest_map) 463 { 464 struct kvm_lapic *apic = vcpu->arch.apic; 465 466 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, 467 irq->level, irq->trig_mode, dest_map); 468 } 469 470 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val) 471 { 472 473 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val, 474 sizeof(val)); 475 } 476 477 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val) 478 { 479 480 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val, 481 sizeof(*val)); 482 } 483 484 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) 485 { 486 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; 487 } 488 489 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) 490 { 491 u8 val; 492 if (pv_eoi_get_user(vcpu, &val) < 0) 493 apic_debug("Can't read EOI MSR value: 0x%llx\n", 494 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 495 return val & 0x1; 496 } 497 498 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) 499 { 500 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) { 501 apic_debug("Can't set EOI MSR value: 0x%llx\n", 502 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 503 return; 504 } 505 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 506 } 507 508 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) 509 { 510 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) { 511 apic_debug("Can't clear EOI MSR value: 0x%llx\n", 512 (unsigned long long)vcpu->arch.pv_eoi.msr_val); 513 return; 514 } 515 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); 516 } 517 518 void kvm_apic_update_tmr(struct kvm_vcpu *vcpu, u32 *tmr) 519 { 520 struct kvm_lapic *apic = vcpu->arch.apic; 521 int i; 522 523 for (i = 0; i < 8; i++) 524 apic_set_reg(apic, APIC_TMR + 0x10 * i, tmr[i]); 525 } 526 527 static void apic_update_ppr(struct kvm_lapic *apic) 528 { 529 u32 tpr, isrv, ppr, old_ppr; 530 int isr; 531 532 old_ppr = kvm_apic_get_reg(apic, APIC_PROCPRI); 533 tpr = kvm_apic_get_reg(apic, APIC_TASKPRI); 534 isr = apic_find_highest_isr(apic); 535 isrv = (isr != -1) ? isr : 0; 536 537 if ((tpr & 0xf0) >= (isrv & 0xf0)) 538 ppr = tpr & 0xff; 539 else 540 ppr = isrv & 0xf0; 541 542 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", 543 apic, ppr, isr, isrv); 544 545 if (old_ppr != ppr) { 546 apic_set_reg(apic, APIC_PROCPRI, ppr); 547 if (ppr < old_ppr) 548 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 549 } 550 } 551 552 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) 553 { 554 apic_set_reg(apic, APIC_TASKPRI, tpr); 555 apic_update_ppr(apic); 556 } 557 558 int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest) 559 { 560 return dest == 0xff || kvm_apic_id(apic) == dest; 561 } 562 563 int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda) 564 { 565 int result = 0; 566 u32 logical_id; 567 568 if (apic_x2apic_mode(apic)) { 569 logical_id = kvm_apic_get_reg(apic, APIC_LDR); 570 return logical_id & mda; 571 } 572 573 logical_id = GET_APIC_LOGICAL_ID(kvm_apic_get_reg(apic, APIC_LDR)); 574 575 switch (kvm_apic_get_reg(apic, APIC_DFR)) { 576 case APIC_DFR_FLAT: 577 if (logical_id & mda) 578 result = 1; 579 break; 580 case APIC_DFR_CLUSTER: 581 if (((logical_id >> 4) == (mda >> 0x4)) 582 && (logical_id & mda & 0xf)) 583 result = 1; 584 break; 585 default: 586 apic_debug("Bad DFR vcpu %d: %08x\n", 587 apic->vcpu->vcpu_id, kvm_apic_get_reg(apic, APIC_DFR)); 588 break; 589 } 590 591 return result; 592 } 593 594 int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, 595 int short_hand, int dest, int dest_mode) 596 { 597 int result = 0; 598 struct kvm_lapic *target = vcpu->arch.apic; 599 600 apic_debug("target %p, source %p, dest 0x%x, " 601 "dest_mode 0x%x, short_hand 0x%x\n", 602 target, source, dest, dest_mode, short_hand); 603 604 ASSERT(target); 605 switch (short_hand) { 606 case APIC_DEST_NOSHORT: 607 if (dest_mode == 0) 608 /* Physical mode. */ 609 result = kvm_apic_match_physical_addr(target, dest); 610 else 611 /* Logical mode. */ 612 result = kvm_apic_match_logical_addr(target, dest); 613 break; 614 case APIC_DEST_SELF: 615 result = (target == source); 616 break; 617 case APIC_DEST_ALLINC: 618 result = 1; 619 break; 620 case APIC_DEST_ALLBUT: 621 result = (target != source); 622 break; 623 default: 624 apic_debug("kvm: apic: Bad dest shorthand value %x\n", 625 short_hand); 626 break; 627 } 628 629 return result; 630 } 631 632 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, 633 struct kvm_lapic_irq *irq, int *r, unsigned long *dest_map) 634 { 635 struct kvm_apic_map *map; 636 unsigned long bitmap = 1; 637 struct kvm_lapic **dst; 638 int i; 639 bool ret = false; 640 641 *r = -1; 642 643 if (irq->shorthand == APIC_DEST_SELF) { 644 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); 645 return true; 646 } 647 648 if (irq->shorthand) 649 return false; 650 651 rcu_read_lock(); 652 map = rcu_dereference(kvm->arch.apic_map); 653 654 if (!map) 655 goto out; 656 657 if (irq->dest_mode == 0) { /* physical mode */ 658 if (irq->delivery_mode == APIC_DM_LOWEST || 659 irq->dest_id == 0xff) 660 goto out; 661 dst = &map->phys_map[irq->dest_id & 0xff]; 662 } else { 663 u32 mda = irq->dest_id << (32 - map->ldr_bits); 664 665 dst = map->logical_map[apic_cluster_id(map, mda)]; 666 667 bitmap = apic_logical_id(map, mda); 668 669 if (irq->delivery_mode == APIC_DM_LOWEST) { 670 int l = -1; 671 for_each_set_bit(i, &bitmap, 16) { 672 if (!dst[i]) 673 continue; 674 if (l < 0) 675 l = i; 676 else if (kvm_apic_compare_prio(dst[i]->vcpu, dst[l]->vcpu) < 0) 677 l = i; 678 } 679 680 bitmap = (l >= 0) ? 1 << l : 0; 681 } 682 } 683 684 for_each_set_bit(i, &bitmap, 16) { 685 if (!dst[i]) 686 continue; 687 if (*r < 0) 688 *r = 0; 689 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map); 690 } 691 692 ret = true; 693 out: 694 rcu_read_unlock(); 695 return ret; 696 } 697 698 /* 699 * Add a pending IRQ into lapic. 700 * Return 1 if successfully added and 0 if discarded. 701 */ 702 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, 703 int vector, int level, int trig_mode, 704 unsigned long *dest_map) 705 { 706 int result = 0; 707 struct kvm_vcpu *vcpu = apic->vcpu; 708 709 switch (delivery_mode) { 710 case APIC_DM_LOWEST: 711 vcpu->arch.apic_arb_prio++; 712 case APIC_DM_FIXED: 713 /* FIXME add logic for vcpu on reset */ 714 if (unlikely(!apic_enabled(apic))) 715 break; 716 717 result = 1; 718 719 if (dest_map) 720 __set_bit(vcpu->vcpu_id, dest_map); 721 722 if (kvm_x86_ops->deliver_posted_interrupt) 723 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector); 724 else { 725 apic_set_irr(vector, apic); 726 727 kvm_make_request(KVM_REQ_EVENT, vcpu); 728 kvm_vcpu_kick(vcpu); 729 } 730 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode, 731 trig_mode, vector, false); 732 break; 733 734 case APIC_DM_REMRD: 735 result = 1; 736 vcpu->arch.pv.pv_unhalted = 1; 737 kvm_make_request(KVM_REQ_EVENT, vcpu); 738 kvm_vcpu_kick(vcpu); 739 break; 740 741 case APIC_DM_SMI: 742 apic_debug("Ignoring guest SMI\n"); 743 break; 744 745 case APIC_DM_NMI: 746 result = 1; 747 kvm_inject_nmi(vcpu); 748 kvm_vcpu_kick(vcpu); 749 break; 750 751 case APIC_DM_INIT: 752 if (!trig_mode || level) { 753 result = 1; 754 /* assumes that there are only KVM_APIC_INIT/SIPI */ 755 apic->pending_events = (1UL << KVM_APIC_INIT); 756 /* make sure pending_events is visible before sending 757 * the request */ 758 smp_wmb(); 759 kvm_make_request(KVM_REQ_EVENT, vcpu); 760 kvm_vcpu_kick(vcpu); 761 } else { 762 apic_debug("Ignoring de-assert INIT to vcpu %d\n", 763 vcpu->vcpu_id); 764 } 765 break; 766 767 case APIC_DM_STARTUP: 768 apic_debug("SIPI to vcpu %d vector 0x%02x\n", 769 vcpu->vcpu_id, vector); 770 result = 1; 771 apic->sipi_vector = vector; 772 /* make sure sipi_vector is visible for the receiver */ 773 smp_wmb(); 774 set_bit(KVM_APIC_SIPI, &apic->pending_events); 775 kvm_make_request(KVM_REQ_EVENT, vcpu); 776 kvm_vcpu_kick(vcpu); 777 break; 778 779 case APIC_DM_EXTINT: 780 /* 781 * Should only be called by kvm_apic_local_deliver() with LVT0, 782 * before NMI watchdog was enabled. Already handled by 783 * kvm_apic_accept_pic_intr(). 784 */ 785 break; 786 787 default: 788 printk(KERN_ERR "TODO: unsupported delivery mode %x\n", 789 delivery_mode); 790 break; 791 } 792 return result; 793 } 794 795 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2) 796 { 797 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio; 798 } 799 800 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) 801 { 802 if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) && 803 kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) { 804 int trigger_mode; 805 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 806 trigger_mode = IOAPIC_LEVEL_TRIG; 807 else 808 trigger_mode = IOAPIC_EDGE_TRIG; 809 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); 810 } 811 } 812 813 static int apic_set_eoi(struct kvm_lapic *apic) 814 { 815 int vector = apic_find_highest_isr(apic); 816 817 trace_kvm_eoi(apic, vector); 818 819 /* 820 * Not every write EOI will has corresponding ISR, 821 * one example is when Kernel check timer on setup_IO_APIC 822 */ 823 if (vector == -1) 824 return vector; 825 826 apic_clear_isr(vector, apic); 827 apic_update_ppr(apic); 828 829 kvm_ioapic_send_eoi(apic, vector); 830 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 831 return vector; 832 } 833 834 /* 835 * this interface assumes a trap-like exit, which has already finished 836 * desired side effect including vISR and vPPR update. 837 */ 838 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) 839 { 840 struct kvm_lapic *apic = vcpu->arch.apic; 841 842 trace_kvm_eoi(apic, vector); 843 844 kvm_ioapic_send_eoi(apic, vector); 845 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); 846 } 847 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated); 848 849 static void apic_send_ipi(struct kvm_lapic *apic) 850 { 851 u32 icr_low = kvm_apic_get_reg(apic, APIC_ICR); 852 u32 icr_high = kvm_apic_get_reg(apic, APIC_ICR2); 853 struct kvm_lapic_irq irq; 854 855 irq.vector = icr_low & APIC_VECTOR_MASK; 856 irq.delivery_mode = icr_low & APIC_MODE_MASK; 857 irq.dest_mode = icr_low & APIC_DEST_MASK; 858 irq.level = icr_low & APIC_INT_ASSERT; 859 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG; 860 irq.shorthand = icr_low & APIC_SHORT_MASK; 861 if (apic_x2apic_mode(apic)) 862 irq.dest_id = icr_high; 863 else 864 irq.dest_id = GET_APIC_DEST_FIELD(icr_high); 865 866 trace_kvm_apic_ipi(icr_low, irq.dest_id); 867 868 apic_debug("icr_high 0x%x, icr_low 0x%x, " 869 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " 870 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n", 871 icr_high, icr_low, irq.shorthand, irq.dest_id, 872 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode, 873 irq.vector); 874 875 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); 876 } 877 878 static u32 apic_get_tmcct(struct kvm_lapic *apic) 879 { 880 ktime_t remaining; 881 s64 ns; 882 u32 tmcct; 883 884 ASSERT(apic != NULL); 885 886 /* if initial count is 0, current count should also be 0 */ 887 if (kvm_apic_get_reg(apic, APIC_TMICT) == 0 || 888 apic->lapic_timer.period == 0) 889 return 0; 890 891 remaining = hrtimer_get_remaining(&apic->lapic_timer.timer); 892 if (ktime_to_ns(remaining) < 0) 893 remaining = ktime_set(0, 0); 894 895 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); 896 tmcct = div64_u64(ns, 897 (APIC_BUS_CYCLE_NS * apic->divide_count)); 898 899 return tmcct; 900 } 901 902 static void __report_tpr_access(struct kvm_lapic *apic, bool write) 903 { 904 struct kvm_vcpu *vcpu = apic->vcpu; 905 struct kvm_run *run = vcpu->run; 906 907 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu); 908 run->tpr_access.rip = kvm_rip_read(vcpu); 909 run->tpr_access.is_write = write; 910 } 911 912 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) 913 { 914 if (apic->vcpu->arch.tpr_access_reporting) 915 __report_tpr_access(apic, write); 916 } 917 918 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) 919 { 920 u32 val = 0; 921 922 if (offset >= LAPIC_MMIO_LENGTH) 923 return 0; 924 925 switch (offset) { 926 case APIC_ID: 927 if (apic_x2apic_mode(apic)) 928 val = kvm_apic_id(apic); 929 else 930 val = kvm_apic_id(apic) << 24; 931 break; 932 case APIC_ARBPRI: 933 apic_debug("Access APIC ARBPRI register which is for P6\n"); 934 break; 935 936 case APIC_TMCCT: /* Timer CCR */ 937 if (apic_lvtt_tscdeadline(apic)) 938 return 0; 939 940 val = apic_get_tmcct(apic); 941 break; 942 case APIC_PROCPRI: 943 apic_update_ppr(apic); 944 val = kvm_apic_get_reg(apic, offset); 945 break; 946 case APIC_TASKPRI: 947 report_tpr_access(apic, false); 948 /* fall thru */ 949 default: 950 val = kvm_apic_get_reg(apic, offset); 951 break; 952 } 953 954 return val; 955 } 956 957 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev) 958 { 959 return container_of(dev, struct kvm_lapic, dev); 960 } 961 962 static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len, 963 void *data) 964 { 965 unsigned char alignment = offset & 0xf; 966 u32 result; 967 /* this bitmask has a bit cleared for each reserved register */ 968 static const u64 rmask = 0x43ff01ffffffe70cULL; 969 970 if ((alignment + len) > 4) { 971 apic_debug("KVM_APIC_READ: alignment error %x %d\n", 972 offset, len); 973 return 1; 974 } 975 976 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) { 977 apic_debug("KVM_APIC_READ: read reserved register %x\n", 978 offset); 979 return 1; 980 } 981 982 result = __apic_read(apic, offset & ~0xf); 983 984 trace_kvm_apic_read(offset, result); 985 986 switch (len) { 987 case 1: 988 case 2: 989 case 4: 990 memcpy(data, (char *)&result + alignment, len); 991 break; 992 default: 993 printk(KERN_ERR "Local APIC read with len = %x, " 994 "should be 1,2, or 4 instead\n", len); 995 break; 996 } 997 return 0; 998 } 999 1000 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1001 { 1002 return kvm_apic_hw_enabled(apic) && 1003 addr >= apic->base_address && 1004 addr < apic->base_address + LAPIC_MMIO_LENGTH; 1005 } 1006 1007 static int apic_mmio_read(struct kvm_io_device *this, 1008 gpa_t address, int len, void *data) 1009 { 1010 struct kvm_lapic *apic = to_lapic(this); 1011 u32 offset = address - apic->base_address; 1012 1013 if (!apic_mmio_in_range(apic, address)) 1014 return -EOPNOTSUPP; 1015 1016 apic_reg_read(apic, offset, len, data); 1017 1018 return 0; 1019 } 1020 1021 static void update_divide_count(struct kvm_lapic *apic) 1022 { 1023 u32 tmp1, tmp2, tdcr; 1024 1025 tdcr = kvm_apic_get_reg(apic, APIC_TDCR); 1026 tmp1 = tdcr & 0xf; 1027 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 1028 apic->divide_count = 0x1 << (tmp2 & 0x7); 1029 1030 apic_debug("timer divide count is 0x%x\n", 1031 apic->divide_count); 1032 } 1033 1034 static void start_apic_timer(struct kvm_lapic *apic) 1035 { 1036 ktime_t now; 1037 atomic_set(&apic->lapic_timer.pending, 0); 1038 1039 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { 1040 /* lapic timer in oneshot or periodic mode */ 1041 now = apic->lapic_timer.timer.base->get_time(); 1042 apic->lapic_timer.period = (u64)kvm_apic_get_reg(apic, APIC_TMICT) 1043 * APIC_BUS_CYCLE_NS * apic->divide_count; 1044 1045 if (!apic->lapic_timer.period) 1046 return; 1047 /* 1048 * Do not allow the guest to program periodic timers with small 1049 * interval, since the hrtimers are not throttled by the host 1050 * scheduler. 1051 */ 1052 if (apic_lvtt_period(apic)) { 1053 s64 min_period = min_timer_period_us * 1000LL; 1054 1055 if (apic->lapic_timer.period < min_period) { 1056 pr_info_ratelimited( 1057 "kvm: vcpu %i: requested %lld ns " 1058 "lapic timer period limited to %lld ns\n", 1059 apic->vcpu->vcpu_id, 1060 apic->lapic_timer.period, min_period); 1061 apic->lapic_timer.period = min_period; 1062 } 1063 } 1064 1065 hrtimer_start(&apic->lapic_timer.timer, 1066 ktime_add_ns(now, apic->lapic_timer.period), 1067 HRTIMER_MODE_ABS); 1068 1069 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" 1070 PRIx64 ", " 1071 "timer initial count 0x%x, period %lldns, " 1072 "expire @ 0x%016" PRIx64 ".\n", __func__, 1073 APIC_BUS_CYCLE_NS, ktime_to_ns(now), 1074 kvm_apic_get_reg(apic, APIC_TMICT), 1075 apic->lapic_timer.period, 1076 ktime_to_ns(ktime_add_ns(now, 1077 apic->lapic_timer.period))); 1078 } else if (apic_lvtt_tscdeadline(apic)) { 1079 /* lapic timer in tsc deadline mode */ 1080 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline; 1081 u64 ns = 0; 1082 struct kvm_vcpu *vcpu = apic->vcpu; 1083 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz; 1084 unsigned long flags; 1085 1086 if (unlikely(!tscdeadline || !this_tsc_khz)) 1087 return; 1088 1089 local_irq_save(flags); 1090 1091 now = apic->lapic_timer.timer.base->get_time(); 1092 guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu, native_read_tsc()); 1093 if (likely(tscdeadline > guest_tsc)) { 1094 ns = (tscdeadline - guest_tsc) * 1000000ULL; 1095 do_div(ns, this_tsc_khz); 1096 } 1097 hrtimer_start(&apic->lapic_timer.timer, 1098 ktime_add_ns(now, ns), HRTIMER_MODE_ABS); 1099 1100 local_irq_restore(flags); 1101 } 1102 } 1103 1104 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1105 { 1106 int nmi_wd_enabled = apic_lvt_nmi_mode(kvm_apic_get_reg(apic, APIC_LVT0)); 1107 1108 if (apic_lvt_nmi_mode(lvt0_val)) { 1109 if (!nmi_wd_enabled) { 1110 apic_debug("Receive NMI setting on APIC_LVT0 " 1111 "for cpu %d\n", apic->vcpu->vcpu_id); 1112 apic->vcpu->kvm->arch.vapics_in_nmi_mode++; 1113 } 1114 } else if (nmi_wd_enabled) 1115 apic->vcpu->kvm->arch.vapics_in_nmi_mode--; 1116 } 1117 1118 static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) 1119 { 1120 int ret = 0; 1121 1122 trace_kvm_apic_write(reg, val); 1123 1124 switch (reg) { 1125 case APIC_ID: /* Local APIC ID */ 1126 if (!apic_x2apic_mode(apic)) 1127 kvm_apic_set_id(apic, val >> 24); 1128 else 1129 ret = 1; 1130 break; 1131 1132 case APIC_TASKPRI: 1133 report_tpr_access(apic, true); 1134 apic_set_tpr(apic, val & 0xff); 1135 break; 1136 1137 case APIC_EOI: 1138 apic_set_eoi(apic); 1139 break; 1140 1141 case APIC_LDR: 1142 if (!apic_x2apic_mode(apic)) 1143 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); 1144 else 1145 ret = 1; 1146 break; 1147 1148 case APIC_DFR: 1149 if (!apic_x2apic_mode(apic)) { 1150 apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1151 recalculate_apic_map(apic->vcpu->kvm); 1152 } else 1153 ret = 1; 1154 break; 1155 1156 case APIC_SPIV: { 1157 u32 mask = 0x3ff; 1158 if (kvm_apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1159 mask |= APIC_SPIV_DIRECTED_EOI; 1160 apic_set_spiv(apic, val & mask); 1161 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1162 int i; 1163 u32 lvt_val; 1164 1165 for (i = 0; i < APIC_LVT_NUM; i++) { 1166 lvt_val = kvm_apic_get_reg(apic, 1167 APIC_LVTT + 0x10 * i); 1168 apic_set_reg(apic, APIC_LVTT + 0x10 * i, 1169 lvt_val | APIC_LVT_MASKED); 1170 } 1171 atomic_set(&apic->lapic_timer.pending, 0); 1172 1173 } 1174 break; 1175 } 1176 case APIC_ICR: 1177 /* No delay here, so we always clear the pending bit */ 1178 apic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); 1179 apic_send_ipi(apic); 1180 break; 1181 1182 case APIC_ICR2: 1183 if (!apic_x2apic_mode(apic)) 1184 val &= 0xff000000; 1185 apic_set_reg(apic, APIC_ICR2, val); 1186 break; 1187 1188 case APIC_LVT0: 1189 apic_manage_nmi_watchdog(apic, val); 1190 case APIC_LVTTHMR: 1191 case APIC_LVTPC: 1192 case APIC_LVT1: 1193 case APIC_LVTERR: 1194 /* TODO: Check vector */ 1195 if (!kvm_apic_sw_enabled(apic)) 1196 val |= APIC_LVT_MASKED; 1197 1198 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1199 apic_set_reg(apic, reg, val); 1200 1201 break; 1202 1203 case APIC_LVTT: 1204 if ((kvm_apic_get_reg(apic, APIC_LVTT) & 1205 apic->lapic_timer.timer_mode_mask) != 1206 (val & apic->lapic_timer.timer_mode_mask)) 1207 hrtimer_cancel(&apic->lapic_timer.timer); 1208 1209 if (!kvm_apic_sw_enabled(apic)) 1210 val |= APIC_LVT_MASKED; 1211 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1212 apic_set_reg(apic, APIC_LVTT, val); 1213 break; 1214 1215 case APIC_TMICT: 1216 if (apic_lvtt_tscdeadline(apic)) 1217 break; 1218 1219 hrtimer_cancel(&apic->lapic_timer.timer); 1220 apic_set_reg(apic, APIC_TMICT, val); 1221 start_apic_timer(apic); 1222 break; 1223 1224 case APIC_TDCR: 1225 if (val & 4) 1226 apic_debug("KVM_WRITE:TDCR %x\n", val); 1227 apic_set_reg(apic, APIC_TDCR, val); 1228 update_divide_count(apic); 1229 break; 1230 1231 case APIC_ESR: 1232 if (apic_x2apic_mode(apic) && val != 0) { 1233 apic_debug("KVM_WRITE:ESR not zero %x\n", val); 1234 ret = 1; 1235 } 1236 break; 1237 1238 case APIC_SELF_IPI: 1239 if (apic_x2apic_mode(apic)) { 1240 apic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff)); 1241 } else 1242 ret = 1; 1243 break; 1244 default: 1245 ret = 1; 1246 break; 1247 } 1248 if (ret) 1249 apic_debug("Local APIC Write to read-only register %x\n", reg); 1250 return ret; 1251 } 1252 1253 static int apic_mmio_write(struct kvm_io_device *this, 1254 gpa_t address, int len, const void *data) 1255 { 1256 struct kvm_lapic *apic = to_lapic(this); 1257 unsigned int offset = address - apic->base_address; 1258 u32 val; 1259 1260 if (!apic_mmio_in_range(apic, address)) 1261 return -EOPNOTSUPP; 1262 1263 /* 1264 * APIC register must be aligned on 128-bits boundary. 1265 * 32/64/128 bits registers must be accessed thru 32 bits. 1266 * Refer SDM 8.4.1 1267 */ 1268 if (len != 4 || (offset & 0xf)) { 1269 /* Don't shout loud, $infamous_os would cause only noise. */ 1270 apic_debug("apic write: bad size=%d %lx\n", len, (long)address); 1271 return 0; 1272 } 1273 1274 val = *(u32*)data; 1275 1276 /* too common printing */ 1277 if (offset != APIC_EOI) 1278 apic_debug("%s: offset 0x%x with length 0x%x, and value is " 1279 "0x%x\n", __func__, offset, len, val); 1280 1281 apic_reg_write(apic, offset & 0xff0, val); 1282 1283 return 0; 1284 } 1285 1286 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 1287 { 1288 if (kvm_vcpu_has_lapic(vcpu)) 1289 apic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 1290 } 1291 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 1292 1293 /* emulate APIC access in a trap manner */ 1294 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset) 1295 { 1296 u32 val = 0; 1297 1298 /* hw has done the conditional check and inst decode */ 1299 offset &= 0xff0; 1300 1301 apic_reg_read(vcpu->arch.apic, offset, 4, &val); 1302 1303 /* TODO: optimize to just emulate side effect w/o one more write */ 1304 apic_reg_write(vcpu->arch.apic, offset, val); 1305 } 1306 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode); 1307 1308 void kvm_free_lapic(struct kvm_vcpu *vcpu) 1309 { 1310 struct kvm_lapic *apic = vcpu->arch.apic; 1311 1312 if (!vcpu->arch.apic) 1313 return; 1314 1315 hrtimer_cancel(&apic->lapic_timer.timer); 1316 1317 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)) 1318 static_key_slow_dec_deferred(&apic_hw_disabled); 1319 1320 if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED)) 1321 static_key_slow_dec_deferred(&apic_sw_disabled); 1322 1323 if (apic->regs) 1324 free_page((unsigned long)apic->regs); 1325 1326 kfree(apic); 1327 } 1328 1329 /* 1330 *---------------------------------------------------------------------- 1331 * LAPIC interface 1332 *---------------------------------------------------------------------- 1333 */ 1334 1335 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 1336 { 1337 struct kvm_lapic *apic = vcpu->arch.apic; 1338 1339 if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) || 1340 apic_lvtt_period(apic)) 1341 return 0; 1342 1343 return apic->lapic_timer.tscdeadline; 1344 } 1345 1346 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 1347 { 1348 struct kvm_lapic *apic = vcpu->arch.apic; 1349 1350 if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) || 1351 apic_lvtt_period(apic)) 1352 return; 1353 1354 hrtimer_cancel(&apic->lapic_timer.timer); 1355 apic->lapic_timer.tscdeadline = data; 1356 start_apic_timer(apic); 1357 } 1358 1359 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) 1360 { 1361 struct kvm_lapic *apic = vcpu->arch.apic; 1362 1363 if (!kvm_vcpu_has_lapic(vcpu)) 1364 return; 1365 1366 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 1367 | (kvm_apic_get_reg(apic, APIC_TASKPRI) & 4)); 1368 } 1369 1370 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 1371 { 1372 u64 tpr; 1373 1374 if (!kvm_vcpu_has_lapic(vcpu)) 1375 return 0; 1376 1377 tpr = (u64) kvm_apic_get_reg(vcpu->arch.apic, APIC_TASKPRI); 1378 1379 return (tpr & 0xf0) >> 4; 1380 } 1381 1382 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) 1383 { 1384 u64 old_value = vcpu->arch.apic_base; 1385 struct kvm_lapic *apic = vcpu->arch.apic; 1386 1387 if (!apic) { 1388 value |= MSR_IA32_APICBASE_BSP; 1389 vcpu->arch.apic_base = value; 1390 return; 1391 } 1392 1393 if (!kvm_vcpu_is_bsp(apic->vcpu)) 1394 value &= ~MSR_IA32_APICBASE_BSP; 1395 vcpu->arch.apic_base = value; 1396 1397 /* update jump label if enable bit changes */ 1398 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) { 1399 if (value & MSR_IA32_APICBASE_ENABLE) 1400 static_key_slow_dec_deferred(&apic_hw_disabled); 1401 else 1402 static_key_slow_inc(&apic_hw_disabled.key); 1403 recalculate_apic_map(vcpu->kvm); 1404 } 1405 1406 if ((old_value ^ value) & X2APIC_ENABLE) { 1407 if (value & X2APIC_ENABLE) { 1408 u32 id = kvm_apic_id(apic); 1409 u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf)); 1410 kvm_apic_set_ldr(apic, ldr); 1411 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true); 1412 } else 1413 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false); 1414 } 1415 1416 apic->base_address = apic->vcpu->arch.apic_base & 1417 MSR_IA32_APICBASE_BASE; 1418 1419 /* with FSB delivery interrupt, we can restart APIC functionality */ 1420 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " 1421 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); 1422 1423 } 1424 1425 void kvm_lapic_reset(struct kvm_vcpu *vcpu) 1426 { 1427 struct kvm_lapic *apic; 1428 int i; 1429 1430 apic_debug("%s\n", __func__); 1431 1432 ASSERT(vcpu); 1433 apic = vcpu->arch.apic; 1434 ASSERT(apic != NULL); 1435 1436 /* Stop the timer in case it's a reset to an active apic */ 1437 hrtimer_cancel(&apic->lapic_timer.timer); 1438 1439 kvm_apic_set_id(apic, vcpu->vcpu_id); 1440 kvm_apic_set_version(apic->vcpu); 1441 1442 for (i = 0; i < APIC_LVT_NUM; i++) 1443 apic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); 1444 apic_set_reg(apic, APIC_LVT0, 1445 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 1446 1447 apic_set_reg(apic, APIC_DFR, 0xffffffffU); 1448 apic_set_spiv(apic, 0xff); 1449 apic_set_reg(apic, APIC_TASKPRI, 0); 1450 kvm_apic_set_ldr(apic, 0); 1451 apic_set_reg(apic, APIC_ESR, 0); 1452 apic_set_reg(apic, APIC_ICR, 0); 1453 apic_set_reg(apic, APIC_ICR2, 0); 1454 apic_set_reg(apic, APIC_TDCR, 0); 1455 apic_set_reg(apic, APIC_TMICT, 0); 1456 for (i = 0; i < 8; i++) { 1457 apic_set_reg(apic, APIC_IRR + 0x10 * i, 0); 1458 apic_set_reg(apic, APIC_ISR + 0x10 * i, 0); 1459 apic_set_reg(apic, APIC_TMR + 0x10 * i, 0); 1460 } 1461 apic->irr_pending = kvm_apic_vid_enabled(vcpu->kvm); 1462 apic->isr_count = kvm_apic_vid_enabled(vcpu->kvm); 1463 apic->highest_isr_cache = -1; 1464 update_divide_count(apic); 1465 atomic_set(&apic->lapic_timer.pending, 0); 1466 if (kvm_vcpu_is_bsp(vcpu)) 1467 kvm_lapic_set_base(vcpu, 1468 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP); 1469 vcpu->arch.pv_eoi.msr_val = 0; 1470 apic_update_ppr(apic); 1471 1472 vcpu->arch.apic_arb_prio = 0; 1473 vcpu->arch.apic_attention = 0; 1474 1475 apic_debug("%s: vcpu=%p, id=%d, base_msr=" 1476 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__, 1477 vcpu, kvm_apic_id(apic), 1478 vcpu->arch.apic_base, apic->base_address); 1479 } 1480 1481 /* 1482 *---------------------------------------------------------------------- 1483 * timer interface 1484 *---------------------------------------------------------------------- 1485 */ 1486 1487 static bool lapic_is_periodic(struct kvm_lapic *apic) 1488 { 1489 return apic_lvtt_period(apic); 1490 } 1491 1492 int apic_has_pending_timer(struct kvm_vcpu *vcpu) 1493 { 1494 struct kvm_lapic *apic = vcpu->arch.apic; 1495 1496 if (kvm_vcpu_has_lapic(vcpu) && apic_enabled(apic) && 1497 apic_lvt_enabled(apic, APIC_LVTT)) 1498 return atomic_read(&apic->lapic_timer.pending); 1499 1500 return 0; 1501 } 1502 1503 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 1504 { 1505 u32 reg = kvm_apic_get_reg(apic, lvt_type); 1506 int vector, mode, trig_mode; 1507 1508 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 1509 vector = reg & APIC_VECTOR_MASK; 1510 mode = reg & APIC_MODE_MASK; 1511 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 1512 return __apic_accept_irq(apic, mode, vector, 1, trig_mode, 1513 NULL); 1514 } 1515 return 0; 1516 } 1517 1518 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu) 1519 { 1520 struct kvm_lapic *apic = vcpu->arch.apic; 1521 1522 if (apic) 1523 kvm_apic_local_deliver(apic, APIC_LVT0); 1524 } 1525 1526 static const struct kvm_io_device_ops apic_mmio_ops = { 1527 .read = apic_mmio_read, 1528 .write = apic_mmio_write, 1529 }; 1530 1531 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) 1532 { 1533 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 1534 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); 1535 struct kvm_vcpu *vcpu = apic->vcpu; 1536 wait_queue_head_t *q = &vcpu->wq; 1537 1538 /* 1539 * There is a race window between reading and incrementing, but we do 1540 * not care about potentially losing timer events in the !reinject 1541 * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked 1542 * in vcpu_enter_guest. 1543 */ 1544 if (!atomic_read(&ktimer->pending)) { 1545 atomic_inc(&ktimer->pending); 1546 /* FIXME: this code should not know anything about vcpus */ 1547 kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu); 1548 } 1549 1550 if (waitqueue_active(q)) 1551 wake_up_interruptible(q); 1552 1553 if (lapic_is_periodic(apic)) { 1554 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 1555 return HRTIMER_RESTART; 1556 } else 1557 return HRTIMER_NORESTART; 1558 } 1559 1560 int kvm_create_lapic(struct kvm_vcpu *vcpu) 1561 { 1562 struct kvm_lapic *apic; 1563 1564 ASSERT(vcpu != NULL); 1565 apic_debug("apic_init %d\n", vcpu->vcpu_id); 1566 1567 apic = kzalloc(sizeof(*apic), GFP_KERNEL); 1568 if (!apic) 1569 goto nomem; 1570 1571 vcpu->arch.apic = apic; 1572 1573 apic->regs = (void *)get_zeroed_page(GFP_KERNEL); 1574 if (!apic->regs) { 1575 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", 1576 vcpu->vcpu_id); 1577 goto nomem_free_apic; 1578 } 1579 apic->vcpu = vcpu; 1580 1581 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 1582 HRTIMER_MODE_ABS); 1583 apic->lapic_timer.timer.function = apic_timer_fn; 1584 1585 /* 1586 * APIC is created enabled. This will prevent kvm_lapic_set_base from 1587 * thinking that APIC satet has changed. 1588 */ 1589 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE; 1590 kvm_lapic_set_base(vcpu, 1591 APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE); 1592 1593 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */ 1594 kvm_lapic_reset(vcpu); 1595 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 1596 1597 return 0; 1598 nomem_free_apic: 1599 kfree(apic); 1600 nomem: 1601 return -ENOMEM; 1602 } 1603 1604 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) 1605 { 1606 struct kvm_lapic *apic = vcpu->arch.apic; 1607 int highest_irr; 1608 1609 if (!kvm_vcpu_has_lapic(vcpu) || !apic_enabled(apic)) 1610 return -1; 1611 1612 apic_update_ppr(apic); 1613 highest_irr = apic_find_highest_irr(apic); 1614 if ((highest_irr == -1) || 1615 ((highest_irr & 0xF0) <= kvm_apic_get_reg(apic, APIC_PROCPRI))) 1616 return -1; 1617 return highest_irr; 1618 } 1619 1620 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 1621 { 1622 u32 lvt0 = kvm_apic_get_reg(vcpu->arch.apic, APIC_LVT0); 1623 int r = 0; 1624 1625 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 1626 r = 1; 1627 if ((lvt0 & APIC_LVT_MASKED) == 0 && 1628 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 1629 r = 1; 1630 return r; 1631 } 1632 1633 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) 1634 { 1635 struct kvm_lapic *apic = vcpu->arch.apic; 1636 1637 if (!kvm_vcpu_has_lapic(vcpu)) 1638 return; 1639 1640 if (atomic_read(&apic->lapic_timer.pending) > 0) { 1641 kvm_apic_local_deliver(apic, APIC_LVTT); 1642 atomic_set(&apic->lapic_timer.pending, 0); 1643 } 1644 } 1645 1646 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) 1647 { 1648 int vector = kvm_apic_has_interrupt(vcpu); 1649 struct kvm_lapic *apic = vcpu->arch.apic; 1650 1651 if (vector == -1) 1652 return -1; 1653 1654 /* 1655 * We get here even with APIC virtualization enabled, if doing 1656 * nested virtualization and L1 runs with the "acknowledge interrupt 1657 * on exit" mode. Then we cannot inject the interrupt via RVI, 1658 * because the process would deliver it through the IDT. 1659 */ 1660 1661 apic_set_isr(vector, apic); 1662 apic_update_ppr(apic); 1663 apic_clear_irr(vector, apic); 1664 return vector; 1665 } 1666 1667 void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu, 1668 struct kvm_lapic_state *s) 1669 { 1670 struct kvm_lapic *apic = vcpu->arch.apic; 1671 1672 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base); 1673 /* set SPIV separately to get count of SW disabled APICs right */ 1674 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); 1675 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); 1676 /* call kvm_apic_set_id() to put apic into apic_map */ 1677 kvm_apic_set_id(apic, kvm_apic_id(apic)); 1678 kvm_apic_set_version(vcpu); 1679 1680 apic_update_ppr(apic); 1681 hrtimer_cancel(&apic->lapic_timer.timer); 1682 update_divide_count(apic); 1683 start_apic_timer(apic); 1684 apic->irr_pending = true; 1685 apic->isr_count = kvm_apic_vid_enabled(vcpu->kvm) ? 1686 1 : count_vectors(apic->regs + APIC_ISR); 1687 apic->highest_isr_cache = -1; 1688 kvm_x86_ops->hwapic_isr_update(vcpu->kvm, apic_find_highest_isr(apic)); 1689 kvm_make_request(KVM_REQ_EVENT, vcpu); 1690 kvm_rtc_eoi_tracking_restore_one(vcpu); 1691 } 1692 1693 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 1694 { 1695 struct hrtimer *timer; 1696 1697 if (!kvm_vcpu_has_lapic(vcpu)) 1698 return; 1699 1700 timer = &vcpu->arch.apic->lapic_timer.timer; 1701 if (hrtimer_cancel(timer)) 1702 hrtimer_start_expires(timer, HRTIMER_MODE_ABS); 1703 } 1704 1705 /* 1706 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt 1707 * 1708 * Detect whether guest triggered PV EOI since the 1709 * last entry. If yes, set EOI on guests's behalf. 1710 * Clear PV EOI in guest memory in any case. 1711 */ 1712 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, 1713 struct kvm_lapic *apic) 1714 { 1715 bool pending; 1716 int vector; 1717 /* 1718 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host 1719 * and KVM_PV_EOI_ENABLED in guest memory as follows: 1720 * 1721 * KVM_APIC_PV_EOI_PENDING is unset: 1722 * -> host disabled PV EOI. 1723 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set: 1724 * -> host enabled PV EOI, guest did not execute EOI yet. 1725 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: 1726 * -> host enabled PV EOI, guest executed EOI. 1727 */ 1728 BUG_ON(!pv_eoi_enabled(vcpu)); 1729 pending = pv_eoi_get_pending(vcpu); 1730 /* 1731 * Clear pending bit in any case: it will be set again on vmentry. 1732 * While this might not be ideal from performance point of view, 1733 * this makes sure pv eoi is only enabled when we know it's safe. 1734 */ 1735 pv_eoi_clr_pending(vcpu); 1736 if (pending) 1737 return; 1738 vector = apic_set_eoi(apic); 1739 trace_kvm_pv_eoi(apic, vector); 1740 } 1741 1742 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu) 1743 { 1744 u32 data; 1745 1746 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention)) 1747 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); 1748 1749 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 1750 return; 1751 1752 kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 1753 sizeof(u32)); 1754 1755 apic_set_tpr(vcpu->arch.apic, data & 0xff); 1756 } 1757 1758 /* 1759 * apic_sync_pv_eoi_to_guest - called before vmentry 1760 * 1761 * Detect whether it's safe to enable PV EOI and 1762 * if yes do so. 1763 */ 1764 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu, 1765 struct kvm_lapic *apic) 1766 { 1767 if (!pv_eoi_enabled(vcpu) || 1768 /* IRR set or many bits in ISR: could be nested. */ 1769 apic->irr_pending || 1770 /* Cache not set: could be safe but we don't bother. */ 1771 apic->highest_isr_cache == -1 || 1772 /* Need EOI to update ioapic. */ 1773 kvm_ioapic_handles_vector(vcpu->kvm, apic->highest_isr_cache)) { 1774 /* 1775 * PV EOI was disabled by apic_sync_pv_eoi_from_guest 1776 * so we need not do anything here. 1777 */ 1778 return; 1779 } 1780 1781 pv_eoi_set_pending(apic->vcpu); 1782 } 1783 1784 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu) 1785 { 1786 u32 data, tpr; 1787 int max_irr, max_isr; 1788 struct kvm_lapic *apic = vcpu->arch.apic; 1789 1790 apic_sync_pv_eoi_to_guest(vcpu, apic); 1791 1792 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 1793 return; 1794 1795 tpr = kvm_apic_get_reg(apic, APIC_TASKPRI) & 0xff; 1796 max_irr = apic_find_highest_irr(apic); 1797 if (max_irr < 0) 1798 max_irr = 0; 1799 max_isr = apic_find_highest_isr(apic); 1800 if (max_isr < 0) 1801 max_isr = 0; 1802 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24); 1803 1804 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, 1805 sizeof(u32)); 1806 } 1807 1808 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr) 1809 { 1810 if (vapic_addr) { 1811 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 1812 &vcpu->arch.apic->vapic_cache, 1813 vapic_addr, sizeof(u32))) 1814 return -EINVAL; 1815 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 1816 } else { 1817 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention); 1818 } 1819 1820 vcpu->arch.apic->vapic_addr = vapic_addr; 1821 return 0; 1822 } 1823 1824 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1825 { 1826 struct kvm_lapic *apic = vcpu->arch.apic; 1827 u32 reg = (msr - APIC_BASE_MSR) << 4; 1828 1829 if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic)) 1830 return 1; 1831 1832 /* if this is ICR write vector before command */ 1833 if (msr == 0x830) 1834 apic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 1835 return apic_reg_write(apic, reg, (u32)data); 1836 } 1837 1838 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data) 1839 { 1840 struct kvm_lapic *apic = vcpu->arch.apic; 1841 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0; 1842 1843 if (!irqchip_in_kernel(vcpu->kvm) || !apic_x2apic_mode(apic)) 1844 return 1; 1845 1846 if (apic_reg_read(apic, reg, 4, &low)) 1847 return 1; 1848 if (msr == 0x830) 1849 apic_reg_read(apic, APIC_ICR2, 4, &high); 1850 1851 *data = (((u64)high) << 32) | low; 1852 1853 return 0; 1854 } 1855 1856 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) 1857 { 1858 struct kvm_lapic *apic = vcpu->arch.apic; 1859 1860 if (!kvm_vcpu_has_lapic(vcpu)) 1861 return 1; 1862 1863 /* if this is ICR write vector before command */ 1864 if (reg == APIC_ICR) 1865 apic_reg_write(apic, APIC_ICR2, (u32)(data >> 32)); 1866 return apic_reg_write(apic, reg, (u32)data); 1867 } 1868 1869 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) 1870 { 1871 struct kvm_lapic *apic = vcpu->arch.apic; 1872 u32 low, high = 0; 1873 1874 if (!kvm_vcpu_has_lapic(vcpu)) 1875 return 1; 1876 1877 if (apic_reg_read(apic, reg, 4, &low)) 1878 return 1; 1879 if (reg == APIC_ICR) 1880 apic_reg_read(apic, APIC_ICR2, 4, &high); 1881 1882 *data = (((u64)high) << 32) | low; 1883 1884 return 0; 1885 } 1886 1887 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data) 1888 { 1889 u64 addr = data & ~KVM_MSR_ENABLED; 1890 if (!IS_ALIGNED(addr, 4)) 1891 return 1; 1892 1893 vcpu->arch.pv_eoi.msr_val = data; 1894 if (!pv_eoi_enabled(vcpu)) 1895 return 0; 1896 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, 1897 addr, sizeof(u8)); 1898 } 1899 1900 void kvm_apic_accept_events(struct kvm_vcpu *vcpu) 1901 { 1902 struct kvm_lapic *apic = vcpu->arch.apic; 1903 unsigned int sipi_vector; 1904 unsigned long pe; 1905 1906 if (!kvm_vcpu_has_lapic(vcpu) || !apic->pending_events) 1907 return; 1908 1909 pe = xchg(&apic->pending_events, 0); 1910 1911 if (test_bit(KVM_APIC_INIT, &pe)) { 1912 kvm_lapic_reset(vcpu); 1913 kvm_vcpu_reset(vcpu); 1914 if (kvm_vcpu_is_bsp(apic->vcpu)) 1915 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 1916 else 1917 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 1918 } 1919 if (test_bit(KVM_APIC_SIPI, &pe) && 1920 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 1921 /* evaluate pending_events before reading the vector */ 1922 smp_rmb(); 1923 sipi_vector = apic->sipi_vector; 1924 apic_debug("vcpu %d received sipi with vector # %x\n", 1925 vcpu->vcpu_id, sipi_vector); 1926 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector); 1927 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 1928 } 1929 } 1930 1931 void kvm_lapic_init(void) 1932 { 1933 /* do not patch jump label more than once per second */ 1934 jump_label_rate_limit(&apic_hw_disabled, HZ); 1935 jump_label_rate_limit(&apic_sw_disabled, HZ); 1936 } 1937