1 // SPDX-License-Identifier: GPL-2.0-only
2
3 /*
4 * Local APIC virtualization
5 *
6 * Copyright (C) 2006 Qumranet, Inc.
7 * Copyright (C) 2007 Novell
8 * Copyright (C) 2007 Intel
9 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
10 *
11 * Authors:
12 * Dor Laor <dor.laor@qumranet.com>
13 * Gregory Haskins <ghaskins@novell.com>
14 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
15 *
16 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
17 */
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/kvm_host.h>
21 #include <linux/kvm.h>
22 #include <linux/mm.h>
23 #include <linux/highmem.h>
24 #include <linux/smp.h>
25 #include <linux/hrtimer.h>
26 #include <linux/io.h>
27 #include <linux/export.h>
28 #include <linux/math64.h>
29 #include <linux/slab.h>
30 #include <asm/processor.h>
31 #include <asm/mce.h>
32 #include <asm/msr.h>
33 #include <asm/page.h>
34 #include <asm/current.h>
35 #include <asm/apicdef.h>
36 #include <asm/delay.h>
37 #include <linux/atomic.h>
38 #include <linux/jump_label.h>
39 #include "kvm_cache_regs.h"
40 #include "irq.h"
41 #include "ioapic.h"
42 #include "trace.h"
43 #include "x86.h"
44 #include "xen.h"
45 #include "cpuid.h"
46 #include "hyperv.h"
47 #include "smm.h"
48
49 #ifndef CONFIG_X86_64
50 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
51 #else
52 #define mod_64(x, y) ((x) % (y))
53 #endif
54
55 /* 14 is the version for Xeon and Pentium 8.4.8*/
56 #define APIC_VERSION 0x14UL
57 #define LAPIC_MMIO_LENGTH (1 << 12)
58 /* followed define is not in apicdef.h */
59 #define MAX_APIC_VECTOR 256
60 #define APIC_VECTORS_PER_REG 32
61
62 static bool lapic_timer_advance_dynamic __read_mostly;
63 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN 100 /* clock cycles */
64 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX 10000 /* clock cycles */
65 #define LAPIC_TIMER_ADVANCE_NS_INIT 1000
66 #define LAPIC_TIMER_ADVANCE_NS_MAX 5000
67 /* step-by-step approximation to mitigate fluctuation */
68 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
69 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data);
70 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data);
71
__kvm_lapic_set_reg(char * regs,int reg_off,u32 val)72 static inline void __kvm_lapic_set_reg(char *regs, int reg_off, u32 val)
73 {
74 *((u32 *) (regs + reg_off)) = val;
75 }
76
kvm_lapic_set_reg(struct kvm_lapic * apic,int reg_off,u32 val)77 static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val)
78 {
79 __kvm_lapic_set_reg(apic->regs, reg_off, val);
80 }
81
__kvm_lapic_get_reg64(char * regs,int reg)82 static __always_inline u64 __kvm_lapic_get_reg64(char *regs, int reg)
83 {
84 BUILD_BUG_ON(reg != APIC_ICR);
85 return *((u64 *) (regs + reg));
86 }
87
kvm_lapic_get_reg64(struct kvm_lapic * apic,int reg)88 static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg)
89 {
90 return __kvm_lapic_get_reg64(apic->regs, reg);
91 }
92
__kvm_lapic_set_reg64(char * regs,int reg,u64 val)93 static __always_inline void __kvm_lapic_set_reg64(char *regs, int reg, u64 val)
94 {
95 BUILD_BUG_ON(reg != APIC_ICR);
96 *((u64 *) (regs + reg)) = val;
97 }
98
kvm_lapic_set_reg64(struct kvm_lapic * apic,int reg,u64 val)99 static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic,
100 int reg, u64 val)
101 {
102 __kvm_lapic_set_reg64(apic->regs, reg, val);
103 }
104
apic_test_vector(int vec,void * bitmap)105 static inline int apic_test_vector(int vec, void *bitmap)
106 {
107 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
108 }
109
kvm_apic_pending_eoi(struct kvm_vcpu * vcpu,int vector)110 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
111 {
112 struct kvm_lapic *apic = vcpu->arch.apic;
113
114 return apic_test_vector(vector, apic->regs + APIC_ISR) ||
115 apic_test_vector(vector, apic->regs + APIC_IRR);
116 }
117
__apic_test_and_set_vector(int vec,void * bitmap)118 static inline int __apic_test_and_set_vector(int vec, void *bitmap)
119 {
120 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
121 }
122
__apic_test_and_clear_vector(int vec,void * bitmap)123 static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
124 {
125 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
126 }
127
128 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
129 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
130
apic_enabled(struct kvm_lapic * apic)131 static inline int apic_enabled(struct kvm_lapic *apic)
132 {
133 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
134 }
135
136 #define LVT_MASK \
137 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
138
139 #define LINT_MASK \
140 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
141 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
142
kvm_x2apic_id(struct kvm_lapic * apic)143 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
144 {
145 return apic->vcpu->vcpu_id;
146 }
147
kvm_can_post_timer_interrupt(struct kvm_vcpu * vcpu)148 static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
149 {
150 return pi_inject_timer && kvm_vcpu_apicv_active(vcpu) &&
151 (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm));
152 }
153
kvm_can_use_hv_timer(struct kvm_vcpu * vcpu)154 bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu)
155 {
156 return kvm_x86_ops.set_hv_timer
157 && !(kvm_mwait_in_guest(vcpu->kvm) ||
158 kvm_can_post_timer_interrupt(vcpu));
159 }
160
kvm_use_posted_timer_interrupt(struct kvm_vcpu * vcpu)161 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
162 {
163 return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
164 }
165
kvm_apic_calc_x2apic_ldr(u32 id)166 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
167 {
168 return ((id >> 4) << 16) | (1 << (id & 0xf));
169 }
170
kvm_apic_map_get_logical_dest(struct kvm_apic_map * map,u32 dest_id,struct kvm_lapic *** cluster,u16 * mask)171 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
172 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
173 switch (map->logical_mode) {
174 case KVM_APIC_MODE_SW_DISABLED:
175 /* Arbitrarily use the flat map so that @cluster isn't NULL. */
176 *cluster = map->xapic_flat_map;
177 *mask = 0;
178 return true;
179 case KVM_APIC_MODE_X2APIC: {
180 u32 offset = (dest_id >> 16) * 16;
181 u32 max_apic_id = map->max_apic_id;
182
183 if (offset <= max_apic_id) {
184 u8 cluster_size = min(max_apic_id - offset + 1, 16U);
185
186 offset = array_index_nospec(offset, map->max_apic_id + 1);
187 *cluster = &map->phys_map[offset];
188 *mask = dest_id & (0xffff >> (16 - cluster_size));
189 } else {
190 *mask = 0;
191 }
192
193 return true;
194 }
195 case KVM_APIC_MODE_XAPIC_FLAT:
196 *cluster = map->xapic_flat_map;
197 *mask = dest_id & 0xff;
198 return true;
199 case KVM_APIC_MODE_XAPIC_CLUSTER:
200 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
201 *mask = dest_id & 0xf;
202 return true;
203 case KVM_APIC_MODE_MAP_DISABLED:
204 return false;
205 default:
206 WARN_ON_ONCE(1);
207 return false;
208 }
209 }
210
kvm_apic_map_free(struct rcu_head * rcu)211 static void kvm_apic_map_free(struct rcu_head *rcu)
212 {
213 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
214
215 kvfree(map);
216 }
217
kvm_recalculate_phys_map(struct kvm_apic_map * new,struct kvm_vcpu * vcpu,bool * xapic_id_mismatch)218 static int kvm_recalculate_phys_map(struct kvm_apic_map *new,
219 struct kvm_vcpu *vcpu,
220 bool *xapic_id_mismatch)
221 {
222 struct kvm_lapic *apic = vcpu->arch.apic;
223 u32 x2apic_id = kvm_x2apic_id(apic);
224 u32 xapic_id = kvm_xapic_id(apic);
225 u32 physical_id;
226
227 /*
228 * For simplicity, KVM always allocates enough space for all possible
229 * xAPIC IDs. Yell, but don't kill the VM, as KVM can continue on
230 * without the optimized map.
231 */
232 if (WARN_ON_ONCE(xapic_id > new->max_apic_id))
233 return -EINVAL;
234
235 /*
236 * Bail if a vCPU was added and/or enabled its APIC between allocating
237 * the map and doing the actual calculations for the map. Note, KVM
238 * hardcodes the x2APIC ID to vcpu_id, i.e. there's no TOCTOU bug if
239 * the compiler decides to reload x2apic_id after this check.
240 */
241 if (x2apic_id > new->max_apic_id)
242 return -E2BIG;
243
244 /*
245 * Deliberately truncate the vCPU ID when detecting a mismatched APIC
246 * ID to avoid false positives if the vCPU ID, i.e. x2APIC ID, is a
247 * 32-bit value. Any unwanted aliasing due to truncation results will
248 * be detected below.
249 */
250 if (!apic_x2apic_mode(apic) && xapic_id != (u8)vcpu->vcpu_id)
251 *xapic_id_mismatch = true;
252
253 /*
254 * Apply KVM's hotplug hack if userspace has enable 32-bit APIC IDs.
255 * Allow sending events to vCPUs by their x2APIC ID even if the target
256 * vCPU is in legacy xAPIC mode, and silently ignore aliased xAPIC IDs
257 * (the x2APIC ID is truncated to 8 bits, causing IDs > 0xff to wrap
258 * and collide).
259 *
260 * Honor the architectural (and KVM's non-optimized) behavior if
261 * userspace has not enabled 32-bit x2APIC IDs. Each APIC is supposed
262 * to process messages independently. If multiple vCPUs have the same
263 * effective APIC ID, e.g. due to the x2APIC wrap or because the guest
264 * manually modified its xAPIC IDs, events targeting that ID are
265 * supposed to be recognized by all vCPUs with said ID.
266 */
267 if (vcpu->kvm->arch.x2apic_format) {
268 /* See also kvm_apic_match_physical_addr(). */
269 if (apic_x2apic_mode(apic) || x2apic_id > 0xff)
270 new->phys_map[x2apic_id] = apic;
271
272 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
273 new->phys_map[xapic_id] = apic;
274 } else {
275 /*
276 * Disable the optimized map if the physical APIC ID is already
277 * mapped, i.e. is aliased to multiple vCPUs. The optimized
278 * map requires a strict 1:1 mapping between IDs and vCPUs.
279 */
280 if (apic_x2apic_mode(apic))
281 physical_id = x2apic_id;
282 else
283 physical_id = xapic_id;
284
285 if (new->phys_map[physical_id])
286 return -EINVAL;
287
288 new->phys_map[physical_id] = apic;
289 }
290
291 return 0;
292 }
293
kvm_recalculate_logical_map(struct kvm_apic_map * new,struct kvm_vcpu * vcpu)294 static void kvm_recalculate_logical_map(struct kvm_apic_map *new,
295 struct kvm_vcpu *vcpu)
296 {
297 struct kvm_lapic *apic = vcpu->arch.apic;
298 enum kvm_apic_logical_mode logical_mode;
299 struct kvm_lapic **cluster;
300 u16 mask;
301 u32 ldr;
302
303 if (new->logical_mode == KVM_APIC_MODE_MAP_DISABLED)
304 return;
305
306 if (!kvm_apic_sw_enabled(apic))
307 return;
308
309 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
310 if (!ldr)
311 return;
312
313 if (apic_x2apic_mode(apic)) {
314 logical_mode = KVM_APIC_MODE_X2APIC;
315 } else {
316 ldr = GET_APIC_LOGICAL_ID(ldr);
317 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
318 logical_mode = KVM_APIC_MODE_XAPIC_FLAT;
319 else
320 logical_mode = KVM_APIC_MODE_XAPIC_CLUSTER;
321 }
322
323 /*
324 * To optimize logical mode delivery, all software-enabled APICs must
325 * be configured for the same mode.
326 */
327 if (new->logical_mode == KVM_APIC_MODE_SW_DISABLED) {
328 new->logical_mode = logical_mode;
329 } else if (new->logical_mode != logical_mode) {
330 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED;
331 return;
332 }
333
334 /*
335 * In x2APIC mode, the LDR is read-only and derived directly from the
336 * x2APIC ID, thus is guaranteed to be addressable. KVM reuses
337 * kvm_apic_map.phys_map to optimize logical mode x2APIC interrupts by
338 * reversing the LDR calculation to get cluster of APICs, i.e. no
339 * additional work is required.
340 */
341 if (apic_x2apic_mode(apic))
342 return;
343
344 if (WARN_ON_ONCE(!kvm_apic_map_get_logical_dest(new, ldr,
345 &cluster, &mask))) {
346 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED;
347 return;
348 }
349
350 if (!mask)
351 return;
352
353 ldr = ffs(mask) - 1;
354 if (!is_power_of_2(mask) || cluster[ldr])
355 new->logical_mode = KVM_APIC_MODE_MAP_DISABLED;
356 else
357 cluster[ldr] = apic;
358 }
359
360 /*
361 * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock.
362 *
363 * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with
364 * apic_map_lock_held.
365 */
366 enum {
367 CLEAN,
368 UPDATE_IN_PROGRESS,
369 DIRTY
370 };
371
kvm_recalculate_apic_map(struct kvm * kvm)372 void kvm_recalculate_apic_map(struct kvm *kvm)
373 {
374 struct kvm_apic_map *new, *old = NULL;
375 struct kvm_vcpu *vcpu;
376 unsigned long i;
377 u32 max_id = 255; /* enough space for any xAPIC ID */
378 bool xapic_id_mismatch;
379 int r;
380
381 /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map. */
382 if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN)
383 return;
384
385 WARN_ONCE(!irqchip_in_kernel(kvm),
386 "Dirty APIC map without an in-kernel local APIC");
387
388 mutex_lock(&kvm->arch.apic_map_lock);
389
390 retry:
391 /*
392 * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map (if clean)
393 * or the APIC registers (if dirty). Note, on retry the map may have
394 * not yet been marked dirty by whatever task changed a vCPU's x2APIC
395 * ID, i.e. the map may still show up as in-progress. In that case
396 * this task still needs to retry and complete its calculation.
397 */
398 if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty,
399 DIRTY, UPDATE_IN_PROGRESS) == CLEAN) {
400 /* Someone else has updated the map. */
401 mutex_unlock(&kvm->arch.apic_map_lock);
402 return;
403 }
404
405 /*
406 * Reset the mismatch flag between attempts so that KVM does the right
407 * thing if a vCPU changes its xAPIC ID, but do NOT reset max_id, i.e.
408 * keep max_id strictly increasing. Disallowing max_id from shrinking
409 * ensures KVM won't get stuck in an infinite loop, e.g. if the vCPU
410 * with the highest x2APIC ID is toggling its APIC on and off.
411 */
412 xapic_id_mismatch = false;
413
414 kvm_for_each_vcpu(i, vcpu, kvm)
415 if (kvm_apic_present(vcpu))
416 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
417
418 new = kvzalloc(sizeof(struct kvm_apic_map) +
419 sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
420 GFP_KERNEL_ACCOUNT);
421
422 if (!new)
423 goto out;
424
425 new->max_apic_id = max_id;
426 new->logical_mode = KVM_APIC_MODE_SW_DISABLED;
427
428 kvm_for_each_vcpu(i, vcpu, kvm) {
429 if (!kvm_apic_present(vcpu))
430 continue;
431
432 r = kvm_recalculate_phys_map(new, vcpu, &xapic_id_mismatch);
433 if (r) {
434 kvfree(new);
435 new = NULL;
436 if (r == -E2BIG) {
437 cond_resched();
438 goto retry;
439 }
440
441 goto out;
442 }
443
444 kvm_recalculate_logical_map(new, vcpu);
445 }
446 out:
447 /*
448 * The optimized map is effectively KVM's internal version of APICv,
449 * and all unwanted aliasing that results in disabling the optimized
450 * map also applies to APICv.
451 */
452 if (!new)
453 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED);
454 else
455 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED);
456
457 if (!new || new->logical_mode == KVM_APIC_MODE_MAP_DISABLED)
458 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED);
459 else
460 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED);
461
462 if (xapic_id_mismatch)
463 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED);
464 else
465 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_APIC_ID_MODIFIED);
466
467 old = rcu_dereference_protected(kvm->arch.apic_map,
468 lockdep_is_held(&kvm->arch.apic_map_lock));
469 rcu_assign_pointer(kvm->arch.apic_map, new);
470 /*
471 * Write kvm->arch.apic_map before clearing apic->apic_map_dirty.
472 * If another update has come in, leave it DIRTY.
473 */
474 atomic_cmpxchg_release(&kvm->arch.apic_map_dirty,
475 UPDATE_IN_PROGRESS, CLEAN);
476 mutex_unlock(&kvm->arch.apic_map_lock);
477
478 if (old)
479 call_rcu(&old->rcu, kvm_apic_map_free);
480
481 kvm_make_scan_ioapic_request(kvm);
482 }
483
apic_set_spiv(struct kvm_lapic * apic,u32 val)484 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
485 {
486 bool enabled = val & APIC_SPIV_APIC_ENABLED;
487
488 kvm_lapic_set_reg(apic, APIC_SPIV, val);
489
490 if (enabled != apic->sw_enabled) {
491 apic->sw_enabled = enabled;
492 if (enabled)
493 static_branch_slow_dec_deferred(&apic_sw_disabled);
494 else
495 static_branch_inc(&apic_sw_disabled.key);
496
497 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
498 }
499
500 /* Check if there are APF page ready requests pending */
501 if (enabled) {
502 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu);
503 kvm_xen_sw_enable_lapic(apic->vcpu);
504 }
505 }
506
kvm_apic_set_xapic_id(struct kvm_lapic * apic,u8 id)507 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
508 {
509 kvm_lapic_set_reg(apic, APIC_ID, id << 24);
510 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
511 }
512
kvm_apic_set_ldr(struct kvm_lapic * apic,u32 id)513 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
514 {
515 kvm_lapic_set_reg(apic, APIC_LDR, id);
516 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
517 }
518
kvm_apic_set_dfr(struct kvm_lapic * apic,u32 val)519 static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val)
520 {
521 kvm_lapic_set_reg(apic, APIC_DFR, val);
522 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
523 }
524
kvm_apic_set_x2apic_id(struct kvm_lapic * apic,u32 id)525 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
526 {
527 u32 ldr = kvm_apic_calc_x2apic_ldr(id);
528
529 WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
530
531 kvm_lapic_set_reg(apic, APIC_ID, id);
532 kvm_lapic_set_reg(apic, APIC_LDR, ldr);
533 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
534 }
535
apic_lvt_enabled(struct kvm_lapic * apic,int lvt_type)536 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
537 {
538 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
539 }
540
apic_lvtt_oneshot(struct kvm_lapic * apic)541 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
542 {
543 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
544 }
545
apic_lvtt_period(struct kvm_lapic * apic)546 static inline int apic_lvtt_period(struct kvm_lapic *apic)
547 {
548 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
549 }
550
apic_lvtt_tscdeadline(struct kvm_lapic * apic)551 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
552 {
553 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
554 }
555
apic_lvt_nmi_mode(u32 lvt_val)556 static inline int apic_lvt_nmi_mode(u32 lvt_val)
557 {
558 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
559 }
560
kvm_lapic_lvt_supported(struct kvm_lapic * apic,int lvt_index)561 static inline bool kvm_lapic_lvt_supported(struct kvm_lapic *apic, int lvt_index)
562 {
563 return apic->nr_lvt_entries > lvt_index;
564 }
565
kvm_apic_calc_nr_lvt_entries(struct kvm_vcpu * vcpu)566 static inline int kvm_apic_calc_nr_lvt_entries(struct kvm_vcpu *vcpu)
567 {
568 return KVM_APIC_MAX_NR_LVT_ENTRIES - !(vcpu->arch.mcg_cap & MCG_CMCI_P);
569 }
570
kvm_apic_set_version(struct kvm_vcpu * vcpu)571 void kvm_apic_set_version(struct kvm_vcpu *vcpu)
572 {
573 struct kvm_lapic *apic = vcpu->arch.apic;
574 u32 v = 0;
575
576 if (!lapic_in_kernel(vcpu))
577 return;
578
579 v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16);
580
581 /*
582 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
583 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
584 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
585 * version first and level-triggered interrupts never get EOIed in
586 * IOAPIC.
587 */
588 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) &&
589 !ioapic_in_kernel(vcpu->kvm))
590 v |= APIC_LVR_DIRECTED_EOI;
591 kvm_lapic_set_reg(apic, APIC_LVR, v);
592 }
593
kvm_apic_after_set_mcg_cap(struct kvm_vcpu * vcpu)594 void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu)
595 {
596 int nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu);
597 struct kvm_lapic *apic = vcpu->arch.apic;
598 int i;
599
600 if (!lapic_in_kernel(vcpu) || nr_lvt_entries == apic->nr_lvt_entries)
601 return;
602
603 /* Initialize/mask any "new" LVT entries. */
604 for (i = apic->nr_lvt_entries; i < nr_lvt_entries; i++)
605 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED);
606
607 apic->nr_lvt_entries = nr_lvt_entries;
608
609 /* The number of LVT entries is reflected in the version register. */
610 kvm_apic_set_version(vcpu);
611 }
612
613 static const unsigned int apic_lvt_mask[KVM_APIC_MAX_NR_LVT_ENTRIES] = {
614 [LVT_TIMER] = LVT_MASK, /* timer mode mask added at runtime */
615 [LVT_THERMAL_MONITOR] = LVT_MASK | APIC_MODE_MASK,
616 [LVT_PERFORMANCE_COUNTER] = LVT_MASK | APIC_MODE_MASK,
617 [LVT_LINT0] = LINT_MASK,
618 [LVT_LINT1] = LINT_MASK,
619 [LVT_ERROR] = LVT_MASK,
620 [LVT_CMCI] = LVT_MASK | APIC_MODE_MASK
621 };
622
find_highest_vector(void * bitmap)623 static int find_highest_vector(void *bitmap)
624 {
625 int vec;
626 u32 *reg;
627
628 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
629 vec >= 0; vec -= APIC_VECTORS_PER_REG) {
630 reg = bitmap + REG_POS(vec);
631 if (*reg)
632 return __fls(*reg) + vec;
633 }
634
635 return -1;
636 }
637
count_vectors(void * bitmap)638 static u8 count_vectors(void *bitmap)
639 {
640 int vec;
641 u32 *reg;
642 u8 count = 0;
643
644 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
645 reg = bitmap + REG_POS(vec);
646 count += hweight32(*reg);
647 }
648
649 return count;
650 }
651
__kvm_apic_update_irr(u32 * pir,void * regs,int * max_irr)652 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
653 {
654 u32 i, vec;
655 u32 pir_val, irr_val, prev_irr_val;
656 int max_updated_irr;
657
658 max_updated_irr = -1;
659 *max_irr = -1;
660
661 for (i = vec = 0; i <= 7; i++, vec += 32) {
662 u32 *p_irr = (u32 *)(regs + APIC_IRR + i * 0x10);
663
664 irr_val = *p_irr;
665 pir_val = READ_ONCE(pir[i]);
666
667 if (pir_val) {
668 pir_val = xchg(&pir[i], 0);
669
670 prev_irr_val = irr_val;
671 do {
672 irr_val = prev_irr_val | pir_val;
673 } while (prev_irr_val != irr_val &&
674 !try_cmpxchg(p_irr, &prev_irr_val, irr_val));
675
676 if (prev_irr_val != irr_val)
677 max_updated_irr = __fls(irr_val ^ prev_irr_val) + vec;
678 }
679 if (irr_val)
680 *max_irr = __fls(irr_val) + vec;
681 }
682
683 return ((max_updated_irr != -1) &&
684 (max_updated_irr == *max_irr));
685 }
686 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
687
kvm_apic_update_irr(struct kvm_vcpu * vcpu,u32 * pir,int * max_irr)688 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
689 {
690 struct kvm_lapic *apic = vcpu->arch.apic;
691 bool irr_updated = __kvm_apic_update_irr(pir, apic->regs, max_irr);
692
693 if (unlikely(!apic->apicv_active && irr_updated))
694 apic->irr_pending = true;
695 return irr_updated;
696 }
697 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
698
apic_search_irr(struct kvm_lapic * apic)699 static inline int apic_search_irr(struct kvm_lapic *apic)
700 {
701 return find_highest_vector(apic->regs + APIC_IRR);
702 }
703
apic_find_highest_irr(struct kvm_lapic * apic)704 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
705 {
706 int result;
707
708 /*
709 * Note that irr_pending is just a hint. It will be always
710 * true with virtual interrupt delivery enabled.
711 */
712 if (!apic->irr_pending)
713 return -1;
714
715 result = apic_search_irr(apic);
716 ASSERT(result == -1 || result >= 16);
717
718 return result;
719 }
720
apic_clear_irr(int vec,struct kvm_lapic * apic)721 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
722 {
723 if (unlikely(apic->apicv_active)) {
724 /* need to update RVI */
725 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
726 static_call_cond(kvm_x86_hwapic_irr_update)(apic->vcpu,
727 apic_find_highest_irr(apic));
728 } else {
729 apic->irr_pending = false;
730 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
731 if (apic_search_irr(apic) != -1)
732 apic->irr_pending = true;
733 }
734 }
735
kvm_apic_clear_irr(struct kvm_vcpu * vcpu,int vec)736 void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec)
737 {
738 apic_clear_irr(vec, vcpu->arch.apic);
739 }
740 EXPORT_SYMBOL_GPL(kvm_apic_clear_irr);
741
apic_set_isr(int vec,struct kvm_lapic * apic)742 static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
743 {
744 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
745 return;
746
747 /*
748 * With APIC virtualization enabled, all caching is disabled
749 * because the processor can modify ISR under the hood. Instead
750 * just set SVI.
751 */
752 if (unlikely(apic->apicv_active))
753 static_call_cond(kvm_x86_hwapic_isr_update)(vec);
754 else {
755 ++apic->isr_count;
756 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
757 /*
758 * ISR (in service register) bit is set when injecting an interrupt.
759 * The highest vector is injected. Thus the latest bit set matches
760 * the highest bit in ISR.
761 */
762 apic->highest_isr_cache = vec;
763 }
764 }
765
apic_find_highest_isr(struct kvm_lapic * apic)766 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
767 {
768 int result;
769
770 /*
771 * Note that isr_count is always 1, and highest_isr_cache
772 * is always -1, with APIC virtualization enabled.
773 */
774 if (!apic->isr_count)
775 return -1;
776 if (likely(apic->highest_isr_cache != -1))
777 return apic->highest_isr_cache;
778
779 result = find_highest_vector(apic->regs + APIC_ISR);
780 ASSERT(result == -1 || result >= 16);
781
782 return result;
783 }
784
apic_clear_isr(int vec,struct kvm_lapic * apic)785 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
786 {
787 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
788 return;
789
790 /*
791 * We do get here for APIC virtualization enabled if the guest
792 * uses the Hyper-V APIC enlightenment. In this case we may need
793 * to trigger a new interrupt delivery by writing the SVI field;
794 * on the other hand isr_count and highest_isr_cache are unused
795 * and must be left alone.
796 */
797 if (unlikely(apic->apicv_active))
798 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic));
799 else {
800 --apic->isr_count;
801 BUG_ON(apic->isr_count < 0);
802 apic->highest_isr_cache = -1;
803 }
804 }
805
kvm_lapic_find_highest_irr(struct kvm_vcpu * vcpu)806 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
807 {
808 /* This may race with setting of irr in __apic_accept_irq() and
809 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
810 * will cause vmexit immediately and the value will be recalculated
811 * on the next vmentry.
812 */
813 return apic_find_highest_irr(vcpu->arch.apic);
814 }
815 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
816
817 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
818 int vector, int level, int trig_mode,
819 struct dest_map *dest_map);
820
kvm_apic_set_irq(struct kvm_vcpu * vcpu,struct kvm_lapic_irq * irq,struct dest_map * dest_map)821 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
822 struct dest_map *dest_map)
823 {
824 struct kvm_lapic *apic = vcpu->arch.apic;
825
826 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
827 irq->level, irq->trig_mode, dest_map);
828 }
829
__pv_send_ipi(unsigned long * ipi_bitmap,struct kvm_apic_map * map,struct kvm_lapic_irq * irq,u32 min)830 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map,
831 struct kvm_lapic_irq *irq, u32 min)
832 {
833 int i, count = 0;
834 struct kvm_vcpu *vcpu;
835
836 if (min > map->max_apic_id)
837 return 0;
838
839 for_each_set_bit(i, ipi_bitmap,
840 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
841 if (map->phys_map[min + i]) {
842 vcpu = map->phys_map[min + i]->vcpu;
843 count += kvm_apic_set_irq(vcpu, irq, NULL);
844 }
845 }
846
847 return count;
848 }
849
kvm_pv_send_ipi(struct kvm * kvm,unsigned long ipi_bitmap_low,unsigned long ipi_bitmap_high,u32 min,unsigned long icr,int op_64_bit)850 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
851 unsigned long ipi_bitmap_high, u32 min,
852 unsigned long icr, int op_64_bit)
853 {
854 struct kvm_apic_map *map;
855 struct kvm_lapic_irq irq = {0};
856 int cluster_size = op_64_bit ? 64 : 32;
857 int count;
858
859 if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK))
860 return -KVM_EINVAL;
861
862 irq.vector = icr & APIC_VECTOR_MASK;
863 irq.delivery_mode = icr & APIC_MODE_MASK;
864 irq.level = (icr & APIC_INT_ASSERT) != 0;
865 irq.trig_mode = icr & APIC_INT_LEVELTRIG;
866
867 rcu_read_lock();
868 map = rcu_dereference(kvm->arch.apic_map);
869
870 count = -EOPNOTSUPP;
871 if (likely(map)) {
872 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min);
873 min += cluster_size;
874 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min);
875 }
876
877 rcu_read_unlock();
878 return count;
879 }
880
pv_eoi_put_user(struct kvm_vcpu * vcpu,u8 val)881 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
882 {
883
884 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
885 sizeof(val));
886 }
887
pv_eoi_get_user(struct kvm_vcpu * vcpu,u8 * val)888 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
889 {
890
891 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
892 sizeof(*val));
893 }
894
pv_eoi_enabled(struct kvm_vcpu * vcpu)895 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
896 {
897 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
898 }
899
pv_eoi_set_pending(struct kvm_vcpu * vcpu)900 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
901 {
902 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0)
903 return;
904
905 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
906 }
907
pv_eoi_test_and_clr_pending(struct kvm_vcpu * vcpu)908 static bool pv_eoi_test_and_clr_pending(struct kvm_vcpu *vcpu)
909 {
910 u8 val;
911
912 if (pv_eoi_get_user(vcpu, &val) < 0)
913 return false;
914
915 val &= KVM_PV_EOI_ENABLED;
916
917 if (val && pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0)
918 return false;
919
920 /*
921 * Clear pending bit in any case: it will be set again on vmentry.
922 * While this might not be ideal from performance point of view,
923 * this makes sure pv eoi is only enabled when we know it's safe.
924 */
925 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
926
927 return val;
928 }
929
apic_has_interrupt_for_ppr(struct kvm_lapic * apic,u32 ppr)930 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
931 {
932 int highest_irr;
933 if (kvm_x86_ops.sync_pir_to_irr)
934 highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
935 else
936 highest_irr = apic_find_highest_irr(apic);
937 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
938 return -1;
939 return highest_irr;
940 }
941
__apic_update_ppr(struct kvm_lapic * apic,u32 * new_ppr)942 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
943 {
944 u32 tpr, isrv, ppr, old_ppr;
945 int isr;
946
947 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
948 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
949 isr = apic_find_highest_isr(apic);
950 isrv = (isr != -1) ? isr : 0;
951
952 if ((tpr & 0xf0) >= (isrv & 0xf0))
953 ppr = tpr & 0xff;
954 else
955 ppr = isrv & 0xf0;
956
957 *new_ppr = ppr;
958 if (old_ppr != ppr)
959 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
960
961 return ppr < old_ppr;
962 }
963
apic_update_ppr(struct kvm_lapic * apic)964 static void apic_update_ppr(struct kvm_lapic *apic)
965 {
966 u32 ppr;
967
968 if (__apic_update_ppr(apic, &ppr) &&
969 apic_has_interrupt_for_ppr(apic, ppr) != -1)
970 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
971 }
972
kvm_apic_update_ppr(struct kvm_vcpu * vcpu)973 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
974 {
975 apic_update_ppr(vcpu->arch.apic);
976 }
977 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
978
apic_set_tpr(struct kvm_lapic * apic,u32 tpr)979 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
980 {
981 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
982 apic_update_ppr(apic);
983 }
984
kvm_apic_broadcast(struct kvm_lapic * apic,u32 mda)985 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
986 {
987 return mda == (apic_x2apic_mode(apic) ?
988 X2APIC_BROADCAST : APIC_BROADCAST);
989 }
990
kvm_apic_match_physical_addr(struct kvm_lapic * apic,u32 mda)991 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
992 {
993 if (kvm_apic_broadcast(apic, mda))
994 return true;
995
996 /*
997 * Hotplug hack: Accept interrupts for vCPUs in xAPIC mode as if they
998 * were in x2APIC mode if the target APIC ID can't be encoded as an
999 * xAPIC ID. This allows unique addressing of hotplugged vCPUs (which
1000 * start in xAPIC mode) with an APIC ID that is unaddressable in xAPIC
1001 * mode. Match the x2APIC ID if and only if the target APIC ID can't
1002 * be encoded in xAPIC to avoid spurious matches against a vCPU that
1003 * changed its (addressable) xAPIC ID (which is writable).
1004 */
1005 if (apic_x2apic_mode(apic) || mda > 0xff)
1006 return mda == kvm_x2apic_id(apic);
1007
1008 return mda == kvm_xapic_id(apic);
1009 }
1010
kvm_apic_match_logical_addr(struct kvm_lapic * apic,u32 mda)1011 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
1012 {
1013 u32 logical_id;
1014
1015 if (kvm_apic_broadcast(apic, mda))
1016 return true;
1017
1018 logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
1019
1020 if (apic_x2apic_mode(apic))
1021 return ((logical_id >> 16) == (mda >> 16))
1022 && (logical_id & mda & 0xffff) != 0;
1023
1024 logical_id = GET_APIC_LOGICAL_ID(logical_id);
1025
1026 switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
1027 case APIC_DFR_FLAT:
1028 return (logical_id & mda) != 0;
1029 case APIC_DFR_CLUSTER:
1030 return ((logical_id >> 4) == (mda >> 4))
1031 && (logical_id & mda & 0xf) != 0;
1032 default:
1033 return false;
1034 }
1035 }
1036
1037 /* The KVM local APIC implementation has two quirks:
1038 *
1039 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
1040 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
1041 * KVM doesn't do that aliasing.
1042 *
1043 * - in-kernel IOAPIC messages have to be delivered directly to
1044 * x2APIC, because the kernel does not support interrupt remapping.
1045 * In order to support broadcast without interrupt remapping, x2APIC
1046 * rewrites the destination of non-IPI messages from APIC_BROADCAST
1047 * to X2APIC_BROADCAST.
1048 *
1049 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is
1050 * important when userspace wants to use x2APIC-format MSIs, because
1051 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
1052 */
kvm_apic_mda(struct kvm_vcpu * vcpu,unsigned int dest_id,struct kvm_lapic * source,struct kvm_lapic * target)1053 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
1054 struct kvm_lapic *source, struct kvm_lapic *target)
1055 {
1056 bool ipi = source != NULL;
1057
1058 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
1059 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
1060 return X2APIC_BROADCAST;
1061
1062 return dest_id;
1063 }
1064
kvm_apic_match_dest(struct kvm_vcpu * vcpu,struct kvm_lapic * source,int shorthand,unsigned int dest,int dest_mode)1065 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
1066 int shorthand, unsigned int dest, int dest_mode)
1067 {
1068 struct kvm_lapic *target = vcpu->arch.apic;
1069 u32 mda = kvm_apic_mda(vcpu, dest, source, target);
1070
1071 ASSERT(target);
1072 switch (shorthand) {
1073 case APIC_DEST_NOSHORT:
1074 if (dest_mode == APIC_DEST_PHYSICAL)
1075 return kvm_apic_match_physical_addr(target, mda);
1076 else
1077 return kvm_apic_match_logical_addr(target, mda);
1078 case APIC_DEST_SELF:
1079 return target == source;
1080 case APIC_DEST_ALLINC:
1081 return true;
1082 case APIC_DEST_ALLBUT:
1083 return target != source;
1084 default:
1085 return false;
1086 }
1087 }
1088 EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
1089
kvm_vector_to_index(u32 vector,u32 dest_vcpus,const unsigned long * bitmap,u32 bitmap_size)1090 int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
1091 const unsigned long *bitmap, u32 bitmap_size)
1092 {
1093 u32 mod;
1094 int i, idx = -1;
1095
1096 mod = vector % dest_vcpus;
1097
1098 for (i = 0; i <= mod; i++) {
1099 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
1100 BUG_ON(idx == bitmap_size);
1101 }
1102
1103 return idx;
1104 }
1105
kvm_apic_disabled_lapic_found(struct kvm * kvm)1106 static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
1107 {
1108 if (!kvm->arch.disabled_lapic_found) {
1109 kvm->arch.disabled_lapic_found = true;
1110 pr_info("Disabled LAPIC found during irq injection\n");
1111 }
1112 }
1113
kvm_apic_is_broadcast_dest(struct kvm * kvm,struct kvm_lapic ** src,struct kvm_lapic_irq * irq,struct kvm_apic_map * map)1114 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
1115 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
1116 {
1117 if (kvm->arch.x2apic_broadcast_quirk_disabled) {
1118 if ((irq->dest_id == APIC_BROADCAST &&
1119 map->logical_mode != KVM_APIC_MODE_X2APIC))
1120 return true;
1121 if (irq->dest_id == X2APIC_BROADCAST)
1122 return true;
1123 } else {
1124 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
1125 if (irq->dest_id == (x2apic_ipi ?
1126 X2APIC_BROADCAST : APIC_BROADCAST))
1127 return true;
1128 }
1129
1130 return false;
1131 }
1132
1133 /* Return true if the interrupt can be handled by using *bitmap as index mask
1134 * for valid destinations in *dst array.
1135 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
1136 * Note: we may have zero kvm_lapic destinations when we return true, which
1137 * means that the interrupt should be dropped. In this case, *bitmap would be
1138 * zero and *dst undefined.
1139 */
kvm_apic_map_get_dest_lapic(struct kvm * kvm,struct kvm_lapic ** src,struct kvm_lapic_irq * irq,struct kvm_apic_map * map,struct kvm_lapic *** dst,unsigned long * bitmap)1140 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
1141 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
1142 struct kvm_apic_map *map, struct kvm_lapic ***dst,
1143 unsigned long *bitmap)
1144 {
1145 int i, lowest;
1146
1147 if (irq->shorthand == APIC_DEST_SELF && src) {
1148 *dst = src;
1149 *bitmap = 1;
1150 return true;
1151 } else if (irq->shorthand)
1152 return false;
1153
1154 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
1155 return false;
1156
1157 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
1158 if (irq->dest_id > map->max_apic_id) {
1159 *bitmap = 0;
1160 } else {
1161 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
1162 *dst = &map->phys_map[dest_id];
1163 *bitmap = 1;
1164 }
1165 return true;
1166 }
1167
1168 *bitmap = 0;
1169 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
1170 (u16 *)bitmap))
1171 return false;
1172
1173 if (!kvm_lowest_prio_delivery(irq))
1174 return true;
1175
1176 if (!kvm_vector_hashing_enabled()) {
1177 lowest = -1;
1178 for_each_set_bit(i, bitmap, 16) {
1179 if (!(*dst)[i])
1180 continue;
1181 if (lowest < 0)
1182 lowest = i;
1183 else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
1184 (*dst)[lowest]->vcpu) < 0)
1185 lowest = i;
1186 }
1187 } else {
1188 if (!*bitmap)
1189 return true;
1190
1191 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
1192 bitmap, 16);
1193
1194 if (!(*dst)[lowest]) {
1195 kvm_apic_disabled_lapic_found(kvm);
1196 *bitmap = 0;
1197 return true;
1198 }
1199 }
1200
1201 *bitmap = (lowest >= 0) ? 1 << lowest : 0;
1202
1203 return true;
1204 }
1205
kvm_irq_delivery_to_apic_fast(struct kvm * kvm,struct kvm_lapic * src,struct kvm_lapic_irq * irq,int * r,struct dest_map * dest_map)1206 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
1207 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
1208 {
1209 struct kvm_apic_map *map;
1210 unsigned long bitmap;
1211 struct kvm_lapic **dst = NULL;
1212 int i;
1213 bool ret;
1214
1215 *r = -1;
1216
1217 if (irq->shorthand == APIC_DEST_SELF) {
1218 if (KVM_BUG_ON(!src, kvm)) {
1219 *r = 0;
1220 return true;
1221 }
1222 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
1223 return true;
1224 }
1225
1226 rcu_read_lock();
1227 map = rcu_dereference(kvm->arch.apic_map);
1228
1229 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
1230 if (ret) {
1231 *r = 0;
1232 for_each_set_bit(i, &bitmap, 16) {
1233 if (!dst[i])
1234 continue;
1235 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1236 }
1237 }
1238
1239 rcu_read_unlock();
1240 return ret;
1241 }
1242
1243 /*
1244 * This routine tries to handle interrupts in posted mode, here is how
1245 * it deals with different cases:
1246 * - For single-destination interrupts, handle it in posted mode
1247 * - Else if vector hashing is enabled and it is a lowest-priority
1248 * interrupt, handle it in posted mode and use the following mechanism
1249 * to find the destination vCPU.
1250 * 1. For lowest-priority interrupts, store all the possible
1251 * destination vCPUs in an array.
1252 * 2. Use "guest vector % max number of destination vCPUs" to find
1253 * the right destination vCPU in the array for the lowest-priority
1254 * interrupt.
1255 * - Otherwise, use remapped mode to inject the interrupt.
1256 */
kvm_intr_is_single_vcpu_fast(struct kvm * kvm,struct kvm_lapic_irq * irq,struct kvm_vcpu ** dest_vcpu)1257 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
1258 struct kvm_vcpu **dest_vcpu)
1259 {
1260 struct kvm_apic_map *map;
1261 unsigned long bitmap;
1262 struct kvm_lapic **dst = NULL;
1263 bool ret = false;
1264
1265 if (irq->shorthand)
1266 return false;
1267
1268 rcu_read_lock();
1269 map = rcu_dereference(kvm->arch.apic_map);
1270
1271 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
1272 hweight16(bitmap) == 1) {
1273 unsigned long i = find_first_bit(&bitmap, 16);
1274
1275 if (dst[i]) {
1276 *dest_vcpu = dst[i]->vcpu;
1277 ret = true;
1278 }
1279 }
1280
1281 rcu_read_unlock();
1282 return ret;
1283 }
1284
1285 /*
1286 * Add a pending IRQ into lapic.
1287 * Return 1 if successfully added and 0 if discarded.
1288 */
__apic_accept_irq(struct kvm_lapic * apic,int delivery_mode,int vector,int level,int trig_mode,struct dest_map * dest_map)1289 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1290 int vector, int level, int trig_mode,
1291 struct dest_map *dest_map)
1292 {
1293 int result = 0;
1294 struct kvm_vcpu *vcpu = apic->vcpu;
1295
1296 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1297 trig_mode, vector);
1298 switch (delivery_mode) {
1299 case APIC_DM_LOWEST:
1300 vcpu->arch.apic_arb_prio++;
1301 fallthrough;
1302 case APIC_DM_FIXED:
1303 if (unlikely(trig_mode && !level))
1304 break;
1305
1306 /* FIXME add logic for vcpu on reset */
1307 if (unlikely(!apic_enabled(apic)))
1308 break;
1309
1310 result = 1;
1311
1312 if (dest_map) {
1313 __set_bit(vcpu->vcpu_id, dest_map->map);
1314 dest_map->vectors[vcpu->vcpu_id] = vector;
1315 }
1316
1317 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1318 if (trig_mode)
1319 kvm_lapic_set_vector(vector,
1320 apic->regs + APIC_TMR);
1321 else
1322 kvm_lapic_clear_vector(vector,
1323 apic->regs + APIC_TMR);
1324 }
1325
1326 static_call(kvm_x86_deliver_interrupt)(apic, delivery_mode,
1327 trig_mode, vector);
1328 break;
1329
1330 case APIC_DM_REMRD:
1331 result = 1;
1332 vcpu->arch.pv.pv_unhalted = 1;
1333 kvm_make_request(KVM_REQ_EVENT, vcpu);
1334 kvm_vcpu_kick(vcpu);
1335 break;
1336
1337 case APIC_DM_SMI:
1338 if (!kvm_inject_smi(vcpu)) {
1339 kvm_vcpu_kick(vcpu);
1340 result = 1;
1341 }
1342 break;
1343
1344 case APIC_DM_NMI:
1345 result = 1;
1346 kvm_inject_nmi(vcpu);
1347 kvm_vcpu_kick(vcpu);
1348 break;
1349
1350 case APIC_DM_INIT:
1351 if (!trig_mode || level) {
1352 result = 1;
1353 /* assumes that there are only KVM_APIC_INIT/SIPI */
1354 apic->pending_events = (1UL << KVM_APIC_INIT);
1355 kvm_make_request(KVM_REQ_EVENT, vcpu);
1356 kvm_vcpu_kick(vcpu);
1357 }
1358 break;
1359
1360 case APIC_DM_STARTUP:
1361 result = 1;
1362 apic->sipi_vector = vector;
1363 /* make sure sipi_vector is visible for the receiver */
1364 smp_wmb();
1365 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1366 kvm_make_request(KVM_REQ_EVENT, vcpu);
1367 kvm_vcpu_kick(vcpu);
1368 break;
1369
1370 case APIC_DM_EXTINT:
1371 /*
1372 * Should only be called by kvm_apic_local_deliver() with LVT0,
1373 * before NMI watchdog was enabled. Already handled by
1374 * kvm_apic_accept_pic_intr().
1375 */
1376 break;
1377
1378 default:
1379 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1380 delivery_mode);
1381 break;
1382 }
1383 return result;
1384 }
1385
1386 /*
1387 * This routine identifies the destination vcpus mask meant to receive the
1388 * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find
1389 * out the destination vcpus array and set the bitmap or it traverses to
1390 * each available vcpu to identify the same.
1391 */
kvm_bitmap_or_dest_vcpus(struct kvm * kvm,struct kvm_lapic_irq * irq,unsigned long * vcpu_bitmap)1392 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
1393 unsigned long *vcpu_bitmap)
1394 {
1395 struct kvm_lapic **dest_vcpu = NULL;
1396 struct kvm_lapic *src = NULL;
1397 struct kvm_apic_map *map;
1398 struct kvm_vcpu *vcpu;
1399 unsigned long bitmap, i;
1400 int vcpu_idx;
1401 bool ret;
1402
1403 rcu_read_lock();
1404 map = rcu_dereference(kvm->arch.apic_map);
1405
1406 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu,
1407 &bitmap);
1408 if (ret) {
1409 for_each_set_bit(i, &bitmap, 16) {
1410 if (!dest_vcpu[i])
1411 continue;
1412 vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx;
1413 __set_bit(vcpu_idx, vcpu_bitmap);
1414 }
1415 } else {
1416 kvm_for_each_vcpu(i, vcpu, kvm) {
1417 if (!kvm_apic_present(vcpu))
1418 continue;
1419 if (!kvm_apic_match_dest(vcpu, NULL,
1420 irq->shorthand,
1421 irq->dest_id,
1422 irq->dest_mode))
1423 continue;
1424 __set_bit(i, vcpu_bitmap);
1425 }
1426 }
1427 rcu_read_unlock();
1428 }
1429
kvm_apic_compare_prio(struct kvm_vcpu * vcpu1,struct kvm_vcpu * vcpu2)1430 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1431 {
1432 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1433 }
1434
kvm_ioapic_handles_vector(struct kvm_lapic * apic,int vector)1435 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1436 {
1437 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1438 }
1439
kvm_ioapic_send_eoi(struct kvm_lapic * apic,int vector)1440 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1441 {
1442 int trigger_mode;
1443
1444 /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1445 if (!kvm_ioapic_handles_vector(apic, vector))
1446 return;
1447
1448 /* Request a KVM exit to inform the userspace IOAPIC. */
1449 if (irqchip_split(apic->vcpu->kvm)) {
1450 apic->vcpu->arch.pending_ioapic_eoi = vector;
1451 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1452 return;
1453 }
1454
1455 if (apic_test_vector(vector, apic->regs + APIC_TMR))
1456 trigger_mode = IOAPIC_LEVEL_TRIG;
1457 else
1458 trigger_mode = IOAPIC_EDGE_TRIG;
1459
1460 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1461 }
1462
apic_set_eoi(struct kvm_lapic * apic)1463 static int apic_set_eoi(struct kvm_lapic *apic)
1464 {
1465 int vector = apic_find_highest_isr(apic);
1466
1467 trace_kvm_eoi(apic, vector);
1468
1469 /*
1470 * Not every write EOI will has corresponding ISR,
1471 * one example is when Kernel check timer on setup_IO_APIC
1472 */
1473 if (vector == -1)
1474 return vector;
1475
1476 apic_clear_isr(vector, apic);
1477 apic_update_ppr(apic);
1478
1479 if (to_hv_vcpu(apic->vcpu) &&
1480 test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap))
1481 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1482
1483 kvm_ioapic_send_eoi(apic, vector);
1484 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1485 return vector;
1486 }
1487
1488 /*
1489 * this interface assumes a trap-like exit, which has already finished
1490 * desired side effect including vISR and vPPR update.
1491 */
kvm_apic_set_eoi_accelerated(struct kvm_vcpu * vcpu,int vector)1492 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1493 {
1494 struct kvm_lapic *apic = vcpu->arch.apic;
1495
1496 trace_kvm_eoi(apic, vector);
1497
1498 kvm_ioapic_send_eoi(apic, vector);
1499 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1500 }
1501 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1502
kvm_apic_send_ipi(struct kvm_lapic * apic,u32 icr_low,u32 icr_high)1503 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
1504 {
1505 struct kvm_lapic_irq irq;
1506
1507 /* KVM has no delay and should always clear the BUSY/PENDING flag. */
1508 WARN_ON_ONCE(icr_low & APIC_ICR_BUSY);
1509
1510 irq.vector = icr_low & APIC_VECTOR_MASK;
1511 irq.delivery_mode = icr_low & APIC_MODE_MASK;
1512 irq.dest_mode = icr_low & APIC_DEST_MASK;
1513 irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1514 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1515 irq.shorthand = icr_low & APIC_SHORT_MASK;
1516 irq.msi_redir_hint = false;
1517 if (apic_x2apic_mode(apic))
1518 irq.dest_id = icr_high;
1519 else
1520 irq.dest_id = GET_XAPIC_DEST_FIELD(icr_high);
1521
1522 trace_kvm_apic_ipi(icr_low, irq.dest_id);
1523
1524 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
1525 }
1526 EXPORT_SYMBOL_GPL(kvm_apic_send_ipi);
1527
apic_get_tmcct(struct kvm_lapic * apic)1528 static u32 apic_get_tmcct(struct kvm_lapic *apic)
1529 {
1530 ktime_t remaining, now;
1531 s64 ns;
1532
1533 ASSERT(apic != NULL);
1534
1535 /* if initial count is 0, current count should also be 0 */
1536 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1537 apic->lapic_timer.period == 0)
1538 return 0;
1539
1540 now = ktime_get();
1541 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1542 if (ktime_to_ns(remaining) < 0)
1543 remaining = 0;
1544
1545 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1546 return div64_u64(ns, (APIC_BUS_CYCLE_NS * apic->divide_count));
1547 }
1548
__report_tpr_access(struct kvm_lapic * apic,bool write)1549 static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1550 {
1551 struct kvm_vcpu *vcpu = apic->vcpu;
1552 struct kvm_run *run = vcpu->run;
1553
1554 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1555 run->tpr_access.rip = kvm_rip_read(vcpu);
1556 run->tpr_access.is_write = write;
1557 }
1558
report_tpr_access(struct kvm_lapic * apic,bool write)1559 static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1560 {
1561 if (apic->vcpu->arch.tpr_access_reporting)
1562 __report_tpr_access(apic, write);
1563 }
1564
__apic_read(struct kvm_lapic * apic,unsigned int offset)1565 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1566 {
1567 u32 val = 0;
1568
1569 if (offset >= LAPIC_MMIO_LENGTH)
1570 return 0;
1571
1572 switch (offset) {
1573 case APIC_ARBPRI:
1574 break;
1575
1576 case APIC_TMCCT: /* Timer CCR */
1577 if (apic_lvtt_tscdeadline(apic))
1578 return 0;
1579
1580 val = apic_get_tmcct(apic);
1581 break;
1582 case APIC_PROCPRI:
1583 apic_update_ppr(apic);
1584 val = kvm_lapic_get_reg(apic, offset);
1585 break;
1586 case APIC_TASKPRI:
1587 report_tpr_access(apic, false);
1588 fallthrough;
1589 default:
1590 val = kvm_lapic_get_reg(apic, offset);
1591 break;
1592 }
1593
1594 return val;
1595 }
1596
to_lapic(struct kvm_io_device * dev)1597 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1598 {
1599 return container_of(dev, struct kvm_lapic, dev);
1600 }
1601
1602 #define APIC_REG_MASK(reg) (1ull << ((reg) >> 4))
1603 #define APIC_REGS_MASK(first, count) \
1604 (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1605
kvm_lapic_readable_reg_mask(struct kvm_lapic * apic)1606 u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic)
1607 {
1608 /* Leave bits '0' for reserved and write-only registers. */
1609 u64 valid_reg_mask =
1610 APIC_REG_MASK(APIC_ID) |
1611 APIC_REG_MASK(APIC_LVR) |
1612 APIC_REG_MASK(APIC_TASKPRI) |
1613 APIC_REG_MASK(APIC_PROCPRI) |
1614 APIC_REG_MASK(APIC_LDR) |
1615 APIC_REG_MASK(APIC_SPIV) |
1616 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1617 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1618 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1619 APIC_REG_MASK(APIC_ESR) |
1620 APIC_REG_MASK(APIC_ICR) |
1621 APIC_REG_MASK(APIC_LVTT) |
1622 APIC_REG_MASK(APIC_LVTTHMR) |
1623 APIC_REG_MASK(APIC_LVTPC) |
1624 APIC_REG_MASK(APIC_LVT0) |
1625 APIC_REG_MASK(APIC_LVT1) |
1626 APIC_REG_MASK(APIC_LVTERR) |
1627 APIC_REG_MASK(APIC_TMICT) |
1628 APIC_REG_MASK(APIC_TMCCT) |
1629 APIC_REG_MASK(APIC_TDCR);
1630
1631 if (kvm_lapic_lvt_supported(apic, LVT_CMCI))
1632 valid_reg_mask |= APIC_REG_MASK(APIC_LVTCMCI);
1633
1634 /* ARBPRI, DFR, and ICR2 are not valid in x2APIC mode. */
1635 if (!apic_x2apic_mode(apic))
1636 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI) |
1637 APIC_REG_MASK(APIC_DFR) |
1638 APIC_REG_MASK(APIC_ICR2);
1639
1640 return valid_reg_mask;
1641 }
1642 EXPORT_SYMBOL_GPL(kvm_lapic_readable_reg_mask);
1643
kvm_lapic_reg_read(struct kvm_lapic * apic,u32 offset,int len,void * data)1644 static int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
1645 void *data)
1646 {
1647 unsigned char alignment = offset & 0xf;
1648 u32 result;
1649
1650 /*
1651 * WARN if KVM reads ICR in x2APIC mode, as it's an 8-byte register in
1652 * x2APIC and needs to be manually handled by the caller.
1653 */
1654 WARN_ON_ONCE(apic_x2apic_mode(apic) && offset == APIC_ICR);
1655
1656 if (alignment + len > 4)
1657 return 1;
1658
1659 if (offset > 0x3f0 ||
1660 !(kvm_lapic_readable_reg_mask(apic) & APIC_REG_MASK(offset)))
1661 return 1;
1662
1663 result = __apic_read(apic, offset & ~0xf);
1664
1665 trace_kvm_apic_read(offset, result);
1666
1667 switch (len) {
1668 case 1:
1669 case 2:
1670 case 4:
1671 memcpy(data, (char *)&result + alignment, len);
1672 break;
1673 default:
1674 printk(KERN_ERR "Local APIC read with len = %x, "
1675 "should be 1,2, or 4 instead\n", len);
1676 break;
1677 }
1678 return 0;
1679 }
1680
apic_mmio_in_range(struct kvm_lapic * apic,gpa_t addr)1681 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1682 {
1683 return addr >= apic->base_address &&
1684 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1685 }
1686
apic_mmio_read(struct kvm_vcpu * vcpu,struct kvm_io_device * this,gpa_t address,int len,void * data)1687 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1688 gpa_t address, int len, void *data)
1689 {
1690 struct kvm_lapic *apic = to_lapic(this);
1691 u32 offset = address - apic->base_address;
1692
1693 if (!apic_mmio_in_range(apic, address))
1694 return -EOPNOTSUPP;
1695
1696 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1697 if (!kvm_check_has_quirk(vcpu->kvm,
1698 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1699 return -EOPNOTSUPP;
1700
1701 memset(data, 0xff, len);
1702 return 0;
1703 }
1704
1705 kvm_lapic_reg_read(apic, offset, len, data);
1706
1707 return 0;
1708 }
1709
update_divide_count(struct kvm_lapic * apic)1710 static void update_divide_count(struct kvm_lapic *apic)
1711 {
1712 u32 tmp1, tmp2, tdcr;
1713
1714 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
1715 tmp1 = tdcr & 0xf;
1716 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1717 apic->divide_count = 0x1 << (tmp2 & 0x7);
1718 }
1719
limit_periodic_timer_frequency(struct kvm_lapic * apic)1720 static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1721 {
1722 /*
1723 * Do not allow the guest to program periodic timers with small
1724 * interval, since the hrtimers are not throttled by the host
1725 * scheduler.
1726 */
1727 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1728 s64 min_period = min_timer_period_us * 1000LL;
1729
1730 if (apic->lapic_timer.period < min_period) {
1731 pr_info_ratelimited(
1732 "vcpu %i: requested %lld ns "
1733 "lapic timer period limited to %lld ns\n",
1734 apic->vcpu->vcpu_id,
1735 apic->lapic_timer.period, min_period);
1736 apic->lapic_timer.period = min_period;
1737 }
1738 }
1739 }
1740
1741 static void cancel_hv_timer(struct kvm_lapic *apic);
1742
cancel_apic_timer(struct kvm_lapic * apic)1743 static void cancel_apic_timer(struct kvm_lapic *apic)
1744 {
1745 hrtimer_cancel(&apic->lapic_timer.timer);
1746 preempt_disable();
1747 if (apic->lapic_timer.hv_timer_in_use)
1748 cancel_hv_timer(apic);
1749 preempt_enable();
1750 atomic_set(&apic->lapic_timer.pending, 0);
1751 }
1752
apic_update_lvtt(struct kvm_lapic * apic)1753 static void apic_update_lvtt(struct kvm_lapic *apic)
1754 {
1755 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1756 apic->lapic_timer.timer_mode_mask;
1757
1758 if (apic->lapic_timer.timer_mode != timer_mode) {
1759 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1760 APIC_LVT_TIMER_TSCDEADLINE)) {
1761 cancel_apic_timer(apic);
1762 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1763 apic->lapic_timer.period = 0;
1764 apic->lapic_timer.tscdeadline = 0;
1765 }
1766 apic->lapic_timer.timer_mode = timer_mode;
1767 limit_periodic_timer_frequency(apic);
1768 }
1769 }
1770
1771 /*
1772 * On APICv, this test will cause a busy wait
1773 * during a higher-priority task.
1774 */
1775
lapic_timer_int_injected(struct kvm_vcpu * vcpu)1776 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1777 {
1778 struct kvm_lapic *apic = vcpu->arch.apic;
1779 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1780
1781 if (kvm_apic_hw_enabled(apic)) {
1782 int vec = reg & APIC_VECTOR_MASK;
1783 void *bitmap = apic->regs + APIC_ISR;
1784
1785 if (apic->apicv_active)
1786 bitmap = apic->regs + APIC_IRR;
1787
1788 if (apic_test_vector(vec, bitmap))
1789 return true;
1790 }
1791 return false;
1792 }
1793
__wait_lapic_expire(struct kvm_vcpu * vcpu,u64 guest_cycles)1794 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1795 {
1796 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1797
1798 /*
1799 * If the guest TSC is running at a different ratio than the host, then
1800 * convert the delay to nanoseconds to achieve an accurate delay. Note
1801 * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1802 * always for VMX enabled hardware.
1803 */
1804 if (vcpu->arch.tsc_scaling_ratio == kvm_caps.default_tsc_scaling_ratio) {
1805 __delay(min(guest_cycles,
1806 nsec_to_cycles(vcpu, timer_advance_ns)));
1807 } else {
1808 u64 delay_ns = guest_cycles * 1000000ULL;
1809 do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1810 ndelay(min_t(u32, delay_ns, timer_advance_ns));
1811 }
1812 }
1813
adjust_lapic_timer_advance(struct kvm_vcpu * vcpu,s64 advance_expire_delta)1814 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
1815 s64 advance_expire_delta)
1816 {
1817 struct kvm_lapic *apic = vcpu->arch.apic;
1818 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
1819 u64 ns;
1820
1821 /* Do not adjust for tiny fluctuations or large random spikes. */
1822 if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX ||
1823 abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN)
1824 return;
1825
1826 /* too early */
1827 if (advance_expire_delta < 0) {
1828 ns = -advance_expire_delta * 1000000ULL;
1829 do_div(ns, vcpu->arch.virtual_tsc_khz);
1830 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1831 } else {
1832 /* too late */
1833 ns = advance_expire_delta * 1000000ULL;
1834 do_div(ns, vcpu->arch.virtual_tsc_khz);
1835 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1836 }
1837
1838 if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX))
1839 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
1840 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1841 }
1842
__kvm_wait_lapic_expire(struct kvm_vcpu * vcpu)1843 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1844 {
1845 struct kvm_lapic *apic = vcpu->arch.apic;
1846 u64 guest_tsc, tsc_deadline;
1847
1848 tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1849 apic->lapic_timer.expired_tscdeadline = 0;
1850 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1851 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
1852
1853 if (lapic_timer_advance_dynamic) {
1854 adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline);
1855 /*
1856 * If the timer fired early, reread the TSC to account for the
1857 * overhead of the above adjustment to avoid waiting longer
1858 * than is necessary.
1859 */
1860 if (guest_tsc < tsc_deadline)
1861 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1862 }
1863
1864 if (guest_tsc < tsc_deadline)
1865 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
1866 }
1867
kvm_wait_lapic_expire(struct kvm_vcpu * vcpu)1868 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1869 {
1870 if (lapic_in_kernel(vcpu) &&
1871 vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
1872 vcpu->arch.apic->lapic_timer.timer_advance_ns &&
1873 lapic_timer_int_injected(vcpu))
1874 __kvm_wait_lapic_expire(vcpu);
1875 }
1876 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
1877
kvm_apic_inject_pending_timer_irqs(struct kvm_lapic * apic)1878 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1879 {
1880 struct kvm_timer *ktimer = &apic->lapic_timer;
1881
1882 kvm_apic_local_deliver(apic, APIC_LVTT);
1883 if (apic_lvtt_tscdeadline(apic)) {
1884 ktimer->tscdeadline = 0;
1885 } else if (apic_lvtt_oneshot(apic)) {
1886 ktimer->tscdeadline = 0;
1887 ktimer->target_expiration = 0;
1888 }
1889 }
1890
apic_timer_expired(struct kvm_lapic * apic,bool from_timer_fn)1891 static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn)
1892 {
1893 struct kvm_vcpu *vcpu = apic->vcpu;
1894 struct kvm_timer *ktimer = &apic->lapic_timer;
1895
1896 if (atomic_read(&apic->lapic_timer.pending))
1897 return;
1898
1899 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1900 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1901
1902 if (!from_timer_fn && apic->apicv_active) {
1903 WARN_ON(kvm_get_running_vcpu() != vcpu);
1904 kvm_apic_inject_pending_timer_irqs(apic);
1905 return;
1906 }
1907
1908 if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1909 /*
1910 * Ensure the guest's timer has truly expired before posting an
1911 * interrupt. Open code the relevant checks to avoid querying
1912 * lapic_timer_int_injected(), which will be false since the
1913 * interrupt isn't yet injected. Waiting until after injecting
1914 * is not an option since that won't help a posted interrupt.
1915 */
1916 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
1917 vcpu->arch.apic->lapic_timer.timer_advance_ns)
1918 __kvm_wait_lapic_expire(vcpu);
1919 kvm_apic_inject_pending_timer_irqs(apic);
1920 return;
1921 }
1922
1923 atomic_inc(&apic->lapic_timer.pending);
1924 kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1925 if (from_timer_fn)
1926 kvm_vcpu_kick(vcpu);
1927 }
1928
start_sw_tscdeadline(struct kvm_lapic * apic)1929 static void start_sw_tscdeadline(struct kvm_lapic *apic)
1930 {
1931 struct kvm_timer *ktimer = &apic->lapic_timer;
1932 u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
1933 u64 ns = 0;
1934 ktime_t expire;
1935 struct kvm_vcpu *vcpu = apic->vcpu;
1936 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1937 unsigned long flags;
1938 ktime_t now;
1939
1940 if (unlikely(!tscdeadline || !this_tsc_khz))
1941 return;
1942
1943 local_irq_save(flags);
1944
1945 now = ktime_get();
1946 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1947
1948 ns = (tscdeadline - guest_tsc) * 1000000ULL;
1949 do_div(ns, this_tsc_khz);
1950
1951 if (likely(tscdeadline > guest_tsc) &&
1952 likely(ns > apic->lapic_timer.timer_advance_ns)) {
1953 expire = ktime_add_ns(now, ns);
1954 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1955 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
1956 } else
1957 apic_timer_expired(apic, false);
1958
1959 local_irq_restore(flags);
1960 }
1961
tmict_to_ns(struct kvm_lapic * apic,u32 tmict)1962 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict)
1963 {
1964 return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count;
1965 }
1966
update_target_expiration(struct kvm_lapic * apic,uint32_t old_divisor)1967 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1968 {
1969 ktime_t now, remaining;
1970 u64 ns_remaining_old, ns_remaining_new;
1971
1972 apic->lapic_timer.period =
1973 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
1974 limit_periodic_timer_frequency(apic);
1975
1976 now = ktime_get();
1977 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1978 if (ktime_to_ns(remaining) < 0)
1979 remaining = 0;
1980
1981 ns_remaining_old = ktime_to_ns(remaining);
1982 ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1983 apic->divide_count, old_divisor);
1984
1985 apic->lapic_timer.tscdeadline +=
1986 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1987 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1988 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1989 }
1990
set_target_expiration(struct kvm_lapic * apic,u32 count_reg)1991 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg)
1992 {
1993 ktime_t now;
1994 u64 tscl = rdtsc();
1995 s64 deadline;
1996
1997 now = ktime_get();
1998 apic->lapic_timer.period =
1999 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
2000
2001 if (!apic->lapic_timer.period) {
2002 apic->lapic_timer.tscdeadline = 0;
2003 return false;
2004 }
2005
2006 limit_periodic_timer_frequency(apic);
2007 deadline = apic->lapic_timer.period;
2008
2009 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
2010 if (unlikely(count_reg != APIC_TMICT)) {
2011 deadline = tmict_to_ns(apic,
2012 kvm_lapic_get_reg(apic, count_reg));
2013 if (unlikely(deadline <= 0)) {
2014 if (apic_lvtt_period(apic))
2015 deadline = apic->lapic_timer.period;
2016 else
2017 deadline = 0;
2018 }
2019 else if (unlikely(deadline > apic->lapic_timer.period)) {
2020 pr_info_ratelimited(
2021 "vcpu %i: requested lapic timer restore with "
2022 "starting count register %#x=%u (%lld ns) > initial count (%lld ns). "
2023 "Using initial count to start timer.\n",
2024 apic->vcpu->vcpu_id,
2025 count_reg,
2026 kvm_lapic_get_reg(apic, count_reg),
2027 deadline, apic->lapic_timer.period);
2028 kvm_lapic_set_reg(apic, count_reg, 0);
2029 deadline = apic->lapic_timer.period;
2030 }
2031 }
2032 }
2033
2034 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
2035 nsec_to_cycles(apic->vcpu, deadline);
2036 apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline);
2037
2038 return true;
2039 }
2040
advance_periodic_target_expiration(struct kvm_lapic * apic)2041 static void advance_periodic_target_expiration(struct kvm_lapic *apic)
2042 {
2043 ktime_t now = ktime_get();
2044 u64 tscl = rdtsc();
2045 ktime_t delta;
2046
2047 /*
2048 * Synchronize both deadlines to the same time source or
2049 * differences in the periods (caused by differences in the
2050 * underlying clocks or numerical approximation errors) will
2051 * cause the two to drift apart over time as the errors
2052 * accumulate.
2053 */
2054 apic->lapic_timer.target_expiration =
2055 ktime_add_ns(apic->lapic_timer.target_expiration,
2056 apic->lapic_timer.period);
2057 delta = ktime_sub(apic->lapic_timer.target_expiration, now);
2058 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
2059 nsec_to_cycles(apic->vcpu, delta);
2060 }
2061
start_sw_period(struct kvm_lapic * apic)2062 static void start_sw_period(struct kvm_lapic *apic)
2063 {
2064 if (!apic->lapic_timer.period)
2065 return;
2066
2067 if (ktime_after(ktime_get(),
2068 apic->lapic_timer.target_expiration)) {
2069 apic_timer_expired(apic, false);
2070
2071 if (apic_lvtt_oneshot(apic))
2072 return;
2073
2074 advance_periodic_target_expiration(apic);
2075 }
2076
2077 hrtimer_start(&apic->lapic_timer.timer,
2078 apic->lapic_timer.target_expiration,
2079 HRTIMER_MODE_ABS_HARD);
2080 }
2081
kvm_lapic_hv_timer_in_use(struct kvm_vcpu * vcpu)2082 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
2083 {
2084 if (!lapic_in_kernel(vcpu))
2085 return false;
2086
2087 return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
2088 }
2089
cancel_hv_timer(struct kvm_lapic * apic)2090 static void cancel_hv_timer(struct kvm_lapic *apic)
2091 {
2092 WARN_ON(preemptible());
2093 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
2094 static_call(kvm_x86_cancel_hv_timer)(apic->vcpu);
2095 apic->lapic_timer.hv_timer_in_use = false;
2096 }
2097
start_hv_timer(struct kvm_lapic * apic)2098 static bool start_hv_timer(struct kvm_lapic *apic)
2099 {
2100 struct kvm_timer *ktimer = &apic->lapic_timer;
2101 struct kvm_vcpu *vcpu = apic->vcpu;
2102 bool expired;
2103
2104 WARN_ON(preemptible());
2105 if (!kvm_can_use_hv_timer(vcpu))
2106 return false;
2107
2108 if (!ktimer->tscdeadline)
2109 return false;
2110
2111 if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
2112 return false;
2113
2114 ktimer->hv_timer_in_use = true;
2115 hrtimer_cancel(&ktimer->timer);
2116
2117 /*
2118 * To simplify handling the periodic timer, leave the hv timer running
2119 * even if the deadline timer has expired, i.e. rely on the resulting
2120 * VM-Exit to recompute the periodic timer's target expiration.
2121 */
2122 if (!apic_lvtt_period(apic)) {
2123 /*
2124 * Cancel the hv timer if the sw timer fired while the hv timer
2125 * was being programmed, or if the hv timer itself expired.
2126 */
2127 if (atomic_read(&ktimer->pending)) {
2128 cancel_hv_timer(apic);
2129 } else if (expired) {
2130 apic_timer_expired(apic, false);
2131 cancel_hv_timer(apic);
2132 }
2133 }
2134
2135 trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
2136
2137 return true;
2138 }
2139
start_sw_timer(struct kvm_lapic * apic)2140 static void start_sw_timer(struct kvm_lapic *apic)
2141 {
2142 struct kvm_timer *ktimer = &apic->lapic_timer;
2143
2144 WARN_ON(preemptible());
2145 if (apic->lapic_timer.hv_timer_in_use)
2146 cancel_hv_timer(apic);
2147 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
2148 return;
2149
2150 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
2151 start_sw_period(apic);
2152 else if (apic_lvtt_tscdeadline(apic))
2153 start_sw_tscdeadline(apic);
2154 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
2155 }
2156
restart_apic_timer(struct kvm_lapic * apic)2157 static void restart_apic_timer(struct kvm_lapic *apic)
2158 {
2159 preempt_disable();
2160
2161 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
2162 goto out;
2163
2164 if (!start_hv_timer(apic))
2165 start_sw_timer(apic);
2166 out:
2167 preempt_enable();
2168 }
2169
kvm_lapic_expired_hv_timer(struct kvm_vcpu * vcpu)2170 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
2171 {
2172 struct kvm_lapic *apic = vcpu->arch.apic;
2173
2174 preempt_disable();
2175 /* If the preempt notifier has already run, it also called apic_timer_expired */
2176 if (!apic->lapic_timer.hv_timer_in_use)
2177 goto out;
2178 WARN_ON(kvm_vcpu_is_blocking(vcpu));
2179 apic_timer_expired(apic, false);
2180 cancel_hv_timer(apic);
2181
2182 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
2183 advance_periodic_target_expiration(apic);
2184 restart_apic_timer(apic);
2185 }
2186 out:
2187 preempt_enable();
2188 }
2189 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
2190
kvm_lapic_switch_to_hv_timer(struct kvm_vcpu * vcpu)2191 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
2192 {
2193 restart_apic_timer(vcpu->arch.apic);
2194 }
2195
kvm_lapic_switch_to_sw_timer(struct kvm_vcpu * vcpu)2196 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
2197 {
2198 struct kvm_lapic *apic = vcpu->arch.apic;
2199
2200 preempt_disable();
2201 /* Possibly the TSC deadline timer is not enabled yet */
2202 if (apic->lapic_timer.hv_timer_in_use)
2203 start_sw_timer(apic);
2204 preempt_enable();
2205 }
2206
kvm_lapic_restart_hv_timer(struct kvm_vcpu * vcpu)2207 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
2208 {
2209 struct kvm_lapic *apic = vcpu->arch.apic;
2210
2211 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
2212 restart_apic_timer(apic);
2213 }
2214
__start_apic_timer(struct kvm_lapic * apic,u32 count_reg)2215 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg)
2216 {
2217 atomic_set(&apic->lapic_timer.pending, 0);
2218
2219 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
2220 && !set_target_expiration(apic, count_reg))
2221 return;
2222
2223 restart_apic_timer(apic);
2224 }
2225
start_apic_timer(struct kvm_lapic * apic)2226 static void start_apic_timer(struct kvm_lapic *apic)
2227 {
2228 __start_apic_timer(apic, APIC_TMICT);
2229 }
2230
apic_manage_nmi_watchdog(struct kvm_lapic * apic,u32 lvt0_val)2231 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
2232 {
2233 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
2234
2235 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
2236 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
2237 if (lvt0_in_nmi_mode) {
2238 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
2239 } else
2240 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
2241 }
2242 }
2243
get_lvt_index(u32 reg)2244 static int get_lvt_index(u32 reg)
2245 {
2246 if (reg == APIC_LVTCMCI)
2247 return LVT_CMCI;
2248 if (reg < APIC_LVTT || reg > APIC_LVTERR)
2249 return -1;
2250 return array_index_nospec(
2251 (reg - APIC_LVTT) >> 4, KVM_APIC_MAX_NR_LVT_ENTRIES);
2252 }
2253
kvm_lapic_reg_write(struct kvm_lapic * apic,u32 reg,u32 val)2254 static int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
2255 {
2256 int ret = 0;
2257
2258 trace_kvm_apic_write(reg, val);
2259
2260 switch (reg) {
2261 case APIC_ID: /* Local APIC ID */
2262 if (!apic_x2apic_mode(apic)) {
2263 kvm_apic_set_xapic_id(apic, val >> 24);
2264 } else {
2265 ret = 1;
2266 }
2267 break;
2268
2269 case APIC_TASKPRI:
2270 report_tpr_access(apic, true);
2271 apic_set_tpr(apic, val & 0xff);
2272 break;
2273
2274 case APIC_EOI:
2275 apic_set_eoi(apic);
2276 break;
2277
2278 case APIC_LDR:
2279 if (!apic_x2apic_mode(apic))
2280 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
2281 else
2282 ret = 1;
2283 break;
2284
2285 case APIC_DFR:
2286 if (!apic_x2apic_mode(apic))
2287 kvm_apic_set_dfr(apic, val | 0x0FFFFFFF);
2288 else
2289 ret = 1;
2290 break;
2291
2292 case APIC_SPIV: {
2293 u32 mask = 0x3ff;
2294 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
2295 mask |= APIC_SPIV_DIRECTED_EOI;
2296 apic_set_spiv(apic, val & mask);
2297 if (!(val & APIC_SPIV_APIC_ENABLED)) {
2298 int i;
2299
2300 for (i = 0; i < apic->nr_lvt_entries; i++) {
2301 kvm_lapic_set_reg(apic, APIC_LVTx(i),
2302 kvm_lapic_get_reg(apic, APIC_LVTx(i)) | APIC_LVT_MASKED);
2303 }
2304 apic_update_lvtt(apic);
2305 atomic_set(&apic->lapic_timer.pending, 0);
2306
2307 }
2308 break;
2309 }
2310 case APIC_ICR:
2311 WARN_ON_ONCE(apic_x2apic_mode(apic));
2312
2313 /* No delay here, so we always clear the pending bit */
2314 val &= ~APIC_ICR_BUSY;
2315 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
2316 kvm_lapic_set_reg(apic, APIC_ICR, val);
2317 break;
2318 case APIC_ICR2:
2319 if (apic_x2apic_mode(apic))
2320 ret = 1;
2321 else
2322 kvm_lapic_set_reg(apic, APIC_ICR2, val & 0xff000000);
2323 break;
2324
2325 case APIC_LVT0:
2326 apic_manage_nmi_watchdog(apic, val);
2327 fallthrough;
2328 case APIC_LVTTHMR:
2329 case APIC_LVTPC:
2330 case APIC_LVT1:
2331 case APIC_LVTERR:
2332 case APIC_LVTCMCI: {
2333 u32 index = get_lvt_index(reg);
2334 if (!kvm_lapic_lvt_supported(apic, index)) {
2335 ret = 1;
2336 break;
2337 }
2338 if (!kvm_apic_sw_enabled(apic))
2339 val |= APIC_LVT_MASKED;
2340 val &= apic_lvt_mask[index];
2341 kvm_lapic_set_reg(apic, reg, val);
2342 break;
2343 }
2344
2345 case APIC_LVTT:
2346 if (!kvm_apic_sw_enabled(apic))
2347 val |= APIC_LVT_MASKED;
2348 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
2349 kvm_lapic_set_reg(apic, APIC_LVTT, val);
2350 apic_update_lvtt(apic);
2351 break;
2352
2353 case APIC_TMICT:
2354 if (apic_lvtt_tscdeadline(apic))
2355 break;
2356
2357 cancel_apic_timer(apic);
2358 kvm_lapic_set_reg(apic, APIC_TMICT, val);
2359 start_apic_timer(apic);
2360 break;
2361
2362 case APIC_TDCR: {
2363 uint32_t old_divisor = apic->divide_count;
2364
2365 kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb);
2366 update_divide_count(apic);
2367 if (apic->divide_count != old_divisor &&
2368 apic->lapic_timer.period) {
2369 hrtimer_cancel(&apic->lapic_timer.timer);
2370 update_target_expiration(apic, old_divisor);
2371 restart_apic_timer(apic);
2372 }
2373 break;
2374 }
2375 case APIC_ESR:
2376 if (apic_x2apic_mode(apic) && val != 0)
2377 ret = 1;
2378 break;
2379
2380 case APIC_SELF_IPI:
2381 /*
2382 * Self-IPI exists only when x2APIC is enabled. Bits 7:0 hold
2383 * the vector, everything else is reserved.
2384 */
2385 if (!apic_x2apic_mode(apic) || (val & ~APIC_VECTOR_MASK))
2386 ret = 1;
2387 else
2388 kvm_apic_send_ipi(apic, APIC_DEST_SELF | val, 0);
2389 break;
2390 default:
2391 ret = 1;
2392 break;
2393 }
2394
2395 /*
2396 * Recalculate APIC maps if necessary, e.g. if the software enable bit
2397 * was toggled, the APIC ID changed, etc... The maps are marked dirty
2398 * on relevant changes, i.e. this is a nop for most writes.
2399 */
2400 kvm_recalculate_apic_map(apic->vcpu->kvm);
2401
2402 return ret;
2403 }
2404
apic_mmio_write(struct kvm_vcpu * vcpu,struct kvm_io_device * this,gpa_t address,int len,const void * data)2405 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
2406 gpa_t address, int len, const void *data)
2407 {
2408 struct kvm_lapic *apic = to_lapic(this);
2409 unsigned int offset = address - apic->base_address;
2410 u32 val;
2411
2412 if (!apic_mmio_in_range(apic, address))
2413 return -EOPNOTSUPP;
2414
2415 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
2416 if (!kvm_check_has_quirk(vcpu->kvm,
2417 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2418 return -EOPNOTSUPP;
2419
2420 return 0;
2421 }
2422
2423 /*
2424 * APIC register must be aligned on 128-bits boundary.
2425 * 32/64/128 bits registers must be accessed thru 32 bits.
2426 * Refer SDM 8.4.1
2427 */
2428 if (len != 4 || (offset & 0xf))
2429 return 0;
2430
2431 val = *(u32*)data;
2432
2433 kvm_lapic_reg_write(apic, offset & 0xff0, val);
2434
2435 return 0;
2436 }
2437
kvm_lapic_set_eoi(struct kvm_vcpu * vcpu)2438 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2439 {
2440 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
2441 }
2442 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2443
2444 #define X2APIC_ICR_RESERVED_BITS (GENMASK_ULL(31, 20) | GENMASK_ULL(17, 16) | BIT(13))
2445
kvm_x2apic_icr_write(struct kvm_lapic * apic,u64 data)2446 int kvm_x2apic_icr_write(struct kvm_lapic *apic, u64 data)
2447 {
2448 if (data & X2APIC_ICR_RESERVED_BITS)
2449 return 1;
2450
2451 /*
2452 * The BUSY bit is reserved on both Intel and AMD in x2APIC mode, but
2453 * only AMD requires it to be zero, Intel essentially just ignores the
2454 * bit. And if IPI virtualization (Intel) or x2AVIC (AMD) is enabled,
2455 * the CPU performs the reserved bits checks, i.e. the underlying CPU
2456 * behavior will "win". Arbitrarily clear the BUSY bit, as there is no
2457 * sane way to provide consistent behavior with respect to hardware.
2458 */
2459 data &= ~APIC_ICR_BUSY;
2460
2461 kvm_apic_send_ipi(apic, (u32)data, (u32)(data >> 32));
2462 if (kvm_x86_ops.x2apic_icr_is_split) {
2463 kvm_lapic_set_reg(apic, APIC_ICR, data);
2464 kvm_lapic_set_reg(apic, APIC_ICR2, data >> 32);
2465 } else {
2466 kvm_lapic_set_reg64(apic, APIC_ICR, data);
2467 }
2468 trace_kvm_apic_write(APIC_ICR, data);
2469 return 0;
2470 }
2471
kvm_x2apic_icr_read(struct kvm_lapic * apic)2472 static u64 kvm_x2apic_icr_read(struct kvm_lapic *apic)
2473 {
2474 if (kvm_x86_ops.x2apic_icr_is_split)
2475 return (u64)kvm_lapic_get_reg(apic, APIC_ICR) |
2476 (u64)kvm_lapic_get_reg(apic, APIC_ICR2) << 32;
2477
2478 return kvm_lapic_get_reg64(apic, APIC_ICR);
2479 }
2480
2481 /* emulate APIC access in a trap manner */
kvm_apic_write_nodecode(struct kvm_vcpu * vcpu,u32 offset)2482 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2483 {
2484 struct kvm_lapic *apic = vcpu->arch.apic;
2485
2486 /*
2487 * ICR is a single 64-bit register when x2APIC is enabled, all others
2488 * registers hold 32-bit values. For legacy xAPIC, ICR writes need to
2489 * go down the common path to get the upper half from ICR2.
2490 *
2491 * Note, using the write helpers may incur an unnecessary write to the
2492 * virtual APIC state, but KVM needs to conditionally modify the value
2493 * in certain cases, e.g. to clear the ICR busy bit. The cost of extra
2494 * conditional branches is likely a wash relative to the cost of the
2495 * maybe-unecessary write, and both are in the noise anyways.
2496 */
2497 if (apic_x2apic_mode(apic) && offset == APIC_ICR)
2498 WARN_ON_ONCE(kvm_x2apic_icr_write(apic, kvm_x2apic_icr_read(apic)));
2499 else
2500 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
2501 }
2502 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2503
kvm_free_lapic(struct kvm_vcpu * vcpu)2504 void kvm_free_lapic(struct kvm_vcpu *vcpu)
2505 {
2506 struct kvm_lapic *apic = vcpu->arch.apic;
2507
2508 if (!vcpu->arch.apic)
2509 return;
2510
2511 hrtimer_cancel(&apic->lapic_timer.timer);
2512
2513 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2514 static_branch_slow_dec_deferred(&apic_hw_disabled);
2515
2516 if (!apic->sw_enabled)
2517 static_branch_slow_dec_deferred(&apic_sw_disabled);
2518
2519 if (apic->regs)
2520 free_page((unsigned long)apic->regs);
2521
2522 kfree(apic);
2523 }
2524
2525 /*
2526 *----------------------------------------------------------------------
2527 * LAPIC interface
2528 *----------------------------------------------------------------------
2529 */
kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu * vcpu)2530 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2531 {
2532 struct kvm_lapic *apic = vcpu->arch.apic;
2533
2534 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
2535 return 0;
2536
2537 return apic->lapic_timer.tscdeadline;
2538 }
2539
kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu * vcpu,u64 data)2540 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2541 {
2542 struct kvm_lapic *apic = vcpu->arch.apic;
2543
2544 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
2545 return;
2546
2547 hrtimer_cancel(&apic->lapic_timer.timer);
2548 apic->lapic_timer.tscdeadline = data;
2549 start_apic_timer(apic);
2550 }
2551
kvm_lapic_set_tpr(struct kvm_vcpu * vcpu,unsigned long cr8)2552 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2553 {
2554 apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4);
2555 }
2556
kvm_lapic_get_cr8(struct kvm_vcpu * vcpu)2557 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2558 {
2559 u64 tpr;
2560
2561 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
2562
2563 return (tpr & 0xf0) >> 4;
2564 }
2565
kvm_lapic_set_base(struct kvm_vcpu * vcpu,u64 value)2566 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2567 {
2568 u64 old_value = vcpu->arch.apic_base;
2569 struct kvm_lapic *apic = vcpu->arch.apic;
2570
2571 vcpu->arch.apic_base = value;
2572
2573 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2574 kvm_update_cpuid_runtime(vcpu);
2575
2576 if (!apic)
2577 return;
2578
2579 /* update jump label if enable bit changes */
2580 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2581 if (value & MSR_IA32_APICBASE_ENABLE) {
2582 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2583 static_branch_slow_dec_deferred(&apic_hw_disabled);
2584 /* Check if there are APF page ready requests pending */
2585 kvm_make_request(KVM_REQ_APF_READY, vcpu);
2586 } else {
2587 static_branch_inc(&apic_hw_disabled.key);
2588 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
2589 }
2590 }
2591
2592 if ((old_value ^ value) & X2APIC_ENABLE) {
2593 if (value & X2APIC_ENABLE)
2594 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2595 else if (value & MSR_IA32_APICBASE_ENABLE)
2596 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2597 }
2598
2599 if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) {
2600 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
2601 static_call_cond(kvm_x86_set_virtual_apic_mode)(vcpu);
2602 }
2603
2604 apic->base_address = apic->vcpu->arch.apic_base &
2605 MSR_IA32_APICBASE_BASE;
2606
2607 if ((value & MSR_IA32_APICBASE_ENABLE) &&
2608 apic->base_address != APIC_DEFAULT_PHYS_BASE) {
2609 kvm_set_apicv_inhibit(apic->vcpu->kvm,
2610 APICV_INHIBIT_REASON_APIC_BASE_MODIFIED);
2611 }
2612 }
2613
kvm_apic_update_apicv(struct kvm_vcpu * vcpu)2614 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu)
2615 {
2616 struct kvm_lapic *apic = vcpu->arch.apic;
2617
2618 /*
2619 * When APICv is enabled, KVM must always search the IRR for a pending
2620 * IRQ, as other vCPUs and devices can set IRR bits even if the vCPU
2621 * isn't running. If APICv is disabled, KVM _should_ search the IRR
2622 * for a pending IRQ. But KVM currently doesn't ensure *all* hardware,
2623 * e.g. CPUs and IOMMUs, has seen the change in state, i.e. searching
2624 * the IRR at this time could race with IRQ delivery from hardware that
2625 * still sees APICv as being enabled.
2626 *
2627 * FIXME: Ensure other vCPUs and devices observe the change in APICv
2628 * state prior to updating KVM's metadata caches, so that KVM
2629 * can safely search the IRR and set irr_pending accordingly.
2630 */
2631 apic->irr_pending = true;
2632
2633 if (apic->apicv_active)
2634 apic->isr_count = 1;
2635 else
2636 apic->isr_count = count_vectors(apic->regs + APIC_ISR);
2637
2638 apic->highest_isr_cache = -1;
2639 }
2640
kvm_alloc_apic_access_page(struct kvm * kvm)2641 int kvm_alloc_apic_access_page(struct kvm *kvm)
2642 {
2643 struct page *page;
2644 void __user *hva;
2645 int ret = 0;
2646
2647 mutex_lock(&kvm->slots_lock);
2648 if (kvm->arch.apic_access_memslot_enabled ||
2649 kvm->arch.apic_access_memslot_inhibited)
2650 goto out;
2651
2652 hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
2653 APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
2654 if (IS_ERR(hva)) {
2655 ret = PTR_ERR(hva);
2656 goto out;
2657 }
2658
2659 page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
2660 if (is_error_page(page)) {
2661 ret = -EFAULT;
2662 goto out;
2663 }
2664
2665 /*
2666 * Do not pin the page in memory, so that memory hot-unplug
2667 * is able to migrate it.
2668 */
2669 put_page(page);
2670 kvm->arch.apic_access_memslot_enabled = true;
2671 out:
2672 mutex_unlock(&kvm->slots_lock);
2673 return ret;
2674 }
2675 EXPORT_SYMBOL_GPL(kvm_alloc_apic_access_page);
2676
kvm_inhibit_apic_access_page(struct kvm_vcpu * vcpu)2677 void kvm_inhibit_apic_access_page(struct kvm_vcpu *vcpu)
2678 {
2679 struct kvm *kvm = vcpu->kvm;
2680
2681 if (!kvm->arch.apic_access_memslot_enabled)
2682 return;
2683
2684 kvm_vcpu_srcu_read_unlock(vcpu);
2685
2686 mutex_lock(&kvm->slots_lock);
2687
2688 if (kvm->arch.apic_access_memslot_enabled) {
2689 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0);
2690 /*
2691 * Clear "enabled" after the memslot is deleted so that a
2692 * different vCPU doesn't get a false negative when checking
2693 * the flag out of slots_lock. No additional memory barrier is
2694 * needed as modifying memslots requires waiting other vCPUs to
2695 * drop SRCU (see above), and false positives are ok as the
2696 * flag is rechecked after acquiring slots_lock.
2697 */
2698 kvm->arch.apic_access_memslot_enabled = false;
2699
2700 /*
2701 * Mark the memslot as inhibited to prevent reallocating the
2702 * memslot during vCPU creation, e.g. if a vCPU is hotplugged.
2703 */
2704 kvm->arch.apic_access_memslot_inhibited = true;
2705 }
2706
2707 mutex_unlock(&kvm->slots_lock);
2708
2709 kvm_vcpu_srcu_read_lock(vcpu);
2710 }
2711
kvm_lapic_reset(struct kvm_vcpu * vcpu,bool init_event)2712 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
2713 {
2714 struct kvm_lapic *apic = vcpu->arch.apic;
2715 u64 msr_val;
2716 int i;
2717
2718 static_call_cond(kvm_x86_apicv_pre_state_restore)(vcpu);
2719
2720 if (!init_event) {
2721 msr_val = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE;
2722 if (kvm_vcpu_is_reset_bsp(vcpu))
2723 msr_val |= MSR_IA32_APICBASE_BSP;
2724 kvm_lapic_set_base(vcpu, msr_val);
2725 }
2726
2727 if (!apic)
2728 return;
2729
2730 /* Stop the timer in case it's a reset to an active apic */
2731 hrtimer_cancel(&apic->lapic_timer.timer);
2732
2733 /* The xAPIC ID is set at RESET even if the APIC was already enabled. */
2734 if (!init_event)
2735 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2736 kvm_apic_set_version(apic->vcpu);
2737
2738 for (i = 0; i < apic->nr_lvt_entries; i++)
2739 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED);
2740 apic_update_lvtt(apic);
2741 if (kvm_vcpu_is_reset_bsp(vcpu) &&
2742 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2743 kvm_lapic_set_reg(apic, APIC_LVT0,
2744 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2745 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2746
2747 kvm_apic_set_dfr(apic, 0xffffffffU);
2748 apic_set_spiv(apic, 0xff);
2749 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2750 if (!apic_x2apic_mode(apic))
2751 kvm_apic_set_ldr(apic, 0);
2752 kvm_lapic_set_reg(apic, APIC_ESR, 0);
2753 if (!apic_x2apic_mode(apic)) {
2754 kvm_lapic_set_reg(apic, APIC_ICR, 0);
2755 kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2756 } else {
2757 kvm_lapic_set_reg64(apic, APIC_ICR, 0);
2758 }
2759 kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2760 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
2761 for (i = 0; i < 8; i++) {
2762 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2763 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2764 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
2765 }
2766 kvm_apic_update_apicv(vcpu);
2767 update_divide_count(apic);
2768 atomic_set(&apic->lapic_timer.pending, 0);
2769
2770 vcpu->arch.pv_eoi.msr_val = 0;
2771 apic_update_ppr(apic);
2772 if (apic->apicv_active) {
2773 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu);
2774 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, -1);
2775 static_call_cond(kvm_x86_hwapic_isr_update)(-1);
2776 }
2777
2778 vcpu->arch.apic_arb_prio = 0;
2779 vcpu->arch.apic_attention = 0;
2780
2781 kvm_recalculate_apic_map(vcpu->kvm);
2782 }
2783
2784 /*
2785 *----------------------------------------------------------------------
2786 * timer interface
2787 *----------------------------------------------------------------------
2788 */
2789
lapic_is_periodic(struct kvm_lapic * apic)2790 static bool lapic_is_periodic(struct kvm_lapic *apic)
2791 {
2792 return apic_lvtt_period(apic);
2793 }
2794
apic_has_pending_timer(struct kvm_vcpu * vcpu)2795 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2796 {
2797 struct kvm_lapic *apic = vcpu->arch.apic;
2798
2799 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2800 return atomic_read(&apic->lapic_timer.pending);
2801
2802 return 0;
2803 }
2804
kvm_apic_local_deliver(struct kvm_lapic * apic,int lvt_type)2805 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2806 {
2807 u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2808 int vector, mode, trig_mode;
2809 int r;
2810
2811 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2812 vector = reg & APIC_VECTOR_MASK;
2813 mode = reg & APIC_MODE_MASK;
2814 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2815
2816 r = __apic_accept_irq(apic, mode, vector, 1, trig_mode, NULL);
2817 if (r && lvt_type == APIC_LVTPC &&
2818 guest_cpuid_is_intel_compatible(apic->vcpu))
2819 kvm_lapic_set_reg(apic, APIC_LVTPC, reg | APIC_LVT_MASKED);
2820 return r;
2821 }
2822 return 0;
2823 }
2824
kvm_apic_nmi_wd_deliver(struct kvm_vcpu * vcpu)2825 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2826 {
2827 struct kvm_lapic *apic = vcpu->arch.apic;
2828
2829 if (apic)
2830 kvm_apic_local_deliver(apic, APIC_LVT0);
2831 }
2832
2833 static const struct kvm_io_device_ops apic_mmio_ops = {
2834 .read = apic_mmio_read,
2835 .write = apic_mmio_write,
2836 };
2837
apic_timer_fn(struct hrtimer * data)2838 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2839 {
2840 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2841 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2842
2843 apic_timer_expired(apic, true);
2844
2845 if (lapic_is_periodic(apic)) {
2846 advance_periodic_target_expiration(apic);
2847 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2848 return HRTIMER_RESTART;
2849 } else
2850 return HRTIMER_NORESTART;
2851 }
2852
kvm_create_lapic(struct kvm_vcpu * vcpu,int timer_advance_ns)2853 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2854 {
2855 struct kvm_lapic *apic;
2856
2857 ASSERT(vcpu != NULL);
2858
2859 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
2860 if (!apic)
2861 goto nomem;
2862
2863 vcpu->arch.apic = apic;
2864
2865 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
2866 if (!apic->regs) {
2867 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2868 vcpu->vcpu_id);
2869 goto nomem_free_apic;
2870 }
2871 apic->vcpu = vcpu;
2872
2873 apic->nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu);
2874
2875 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2876 HRTIMER_MODE_ABS_HARD);
2877 apic->lapic_timer.timer.function = apic_timer_fn;
2878 if (timer_advance_ns == -1) {
2879 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
2880 lapic_timer_advance_dynamic = true;
2881 } else {
2882 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2883 lapic_timer_advance_dynamic = false;
2884 }
2885
2886 /*
2887 * Stuff the APIC ENABLE bit in lieu of temporarily incrementing
2888 * apic_hw_disabled; the full RESET value is set by kvm_lapic_reset().
2889 */
2890 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2891 static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2892 kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2893
2894 return 0;
2895 nomem_free_apic:
2896 kfree(apic);
2897 vcpu->arch.apic = NULL;
2898 nomem:
2899 return -ENOMEM;
2900 }
2901
kvm_apic_has_interrupt(struct kvm_vcpu * vcpu)2902 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2903 {
2904 struct kvm_lapic *apic = vcpu->arch.apic;
2905 u32 ppr;
2906
2907 if (!kvm_apic_present(vcpu))
2908 return -1;
2909
2910 __apic_update_ppr(apic, &ppr);
2911 return apic_has_interrupt_for_ppr(apic, ppr);
2912 }
2913 EXPORT_SYMBOL_GPL(kvm_apic_has_interrupt);
2914
kvm_apic_accept_pic_intr(struct kvm_vcpu * vcpu)2915 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2916 {
2917 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
2918
2919 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2920 return 1;
2921 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2922 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2923 return 1;
2924 return 0;
2925 }
2926
kvm_inject_apic_timer_irqs(struct kvm_vcpu * vcpu)2927 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2928 {
2929 struct kvm_lapic *apic = vcpu->arch.apic;
2930
2931 if (atomic_read(&apic->lapic_timer.pending) > 0) {
2932 kvm_apic_inject_pending_timer_irqs(apic);
2933 atomic_set(&apic->lapic_timer.pending, 0);
2934 }
2935 }
2936
kvm_get_apic_interrupt(struct kvm_vcpu * vcpu)2937 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2938 {
2939 int vector = kvm_apic_has_interrupt(vcpu);
2940 struct kvm_lapic *apic = vcpu->arch.apic;
2941 u32 ppr;
2942
2943 if (vector == -1)
2944 return -1;
2945
2946 /*
2947 * We get here even with APIC virtualization enabled, if doing
2948 * nested virtualization and L1 runs with the "acknowledge interrupt
2949 * on exit" mode. Then we cannot inject the interrupt via RVI,
2950 * because the process would deliver it through the IDT.
2951 */
2952
2953 apic_clear_irr(vector, apic);
2954 if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) {
2955 /*
2956 * For auto-EOI interrupts, there might be another pending
2957 * interrupt above PPR, so check whether to raise another
2958 * KVM_REQ_EVENT.
2959 */
2960 apic_update_ppr(apic);
2961 } else {
2962 /*
2963 * For normal interrupts, PPR has been raised and there cannot
2964 * be a higher-priority pending interrupt---except if there was
2965 * a concurrent interrupt injection, but that would have
2966 * triggered KVM_REQ_EVENT already.
2967 */
2968 apic_set_isr(vector, apic);
2969 __apic_update_ppr(apic, &ppr);
2970 }
2971
2972 return vector;
2973 }
2974
kvm_apic_state_fixup(struct kvm_vcpu * vcpu,struct kvm_lapic_state * s,bool set)2975 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2976 struct kvm_lapic_state *s, bool set)
2977 {
2978 if (apic_x2apic_mode(vcpu->arch.apic)) {
2979 u32 x2apic_id = kvm_x2apic_id(vcpu->arch.apic);
2980 u32 *id = (u32 *)(s->regs + APIC_ID);
2981 u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2982 u64 icr;
2983
2984 if (vcpu->kvm->arch.x2apic_format) {
2985 if (*id != x2apic_id)
2986 return -EINVAL;
2987 } else {
2988 /*
2989 * Ignore the userspace value when setting APIC state.
2990 * KVM's model is that the x2APIC ID is readonly, e.g.
2991 * KVM only supports delivering interrupts to KVM's
2992 * version of the x2APIC ID. However, for backwards
2993 * compatibility, don't reject attempts to set a
2994 * mismatched ID for userspace that hasn't opted into
2995 * x2apic_format.
2996 */
2997 if (set)
2998 *id = x2apic_id;
2999 else
3000 *id = x2apic_id << 24;
3001 }
3002
3003 /*
3004 * In x2APIC mode, the LDR is fixed and based on the id. And
3005 * if the ICR is _not_ split, ICR is internally a single 64-bit
3006 * register, but needs to be split to ICR+ICR2 in userspace for
3007 * backwards compatibility.
3008 */
3009 if (set)
3010 *ldr = kvm_apic_calc_x2apic_ldr(x2apic_id);
3011
3012 if (!kvm_x86_ops.x2apic_icr_is_split) {
3013 if (set) {
3014 icr = __kvm_lapic_get_reg(s->regs, APIC_ICR) |
3015 (u64)__kvm_lapic_get_reg(s->regs, APIC_ICR2) << 32;
3016 __kvm_lapic_set_reg64(s->regs, APIC_ICR, icr);
3017 } else {
3018 icr = __kvm_lapic_get_reg64(s->regs, APIC_ICR);
3019 __kvm_lapic_set_reg(s->regs, APIC_ICR2, icr >> 32);
3020 }
3021 }
3022 }
3023
3024 return 0;
3025 }
3026
kvm_apic_get_state(struct kvm_vcpu * vcpu,struct kvm_lapic_state * s)3027 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
3028 {
3029 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
3030
3031 /*
3032 * Get calculated timer current count for remaining timer period (if
3033 * any) and store it in the returned register set.
3034 */
3035 __kvm_lapic_set_reg(s->regs, APIC_TMCCT,
3036 __apic_read(vcpu->arch.apic, APIC_TMCCT));
3037
3038 return kvm_apic_state_fixup(vcpu, s, false);
3039 }
3040
kvm_apic_set_state(struct kvm_vcpu * vcpu,struct kvm_lapic_state * s)3041 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
3042 {
3043 struct kvm_lapic *apic = vcpu->arch.apic;
3044 int r;
3045
3046 static_call_cond(kvm_x86_apicv_pre_state_restore)(vcpu);
3047
3048 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
3049 /* set SPIV separately to get count of SW disabled APICs right */
3050 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
3051
3052 r = kvm_apic_state_fixup(vcpu, s, true);
3053 if (r) {
3054 kvm_recalculate_apic_map(vcpu->kvm);
3055 return r;
3056 }
3057 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
3058
3059 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
3060 kvm_recalculate_apic_map(vcpu->kvm);
3061 kvm_apic_set_version(vcpu);
3062
3063 apic_update_ppr(apic);
3064 cancel_apic_timer(apic);
3065 apic->lapic_timer.expired_tscdeadline = 0;
3066 apic_update_lvtt(apic);
3067 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
3068 update_divide_count(apic);
3069 __start_apic_timer(apic, APIC_TMCCT);
3070 kvm_lapic_set_reg(apic, APIC_TMCCT, 0);
3071 kvm_apic_update_apicv(vcpu);
3072 if (apic->apicv_active) {
3073 static_call_cond(kvm_x86_apicv_post_state_restore)(vcpu);
3074 static_call_cond(kvm_x86_hwapic_irr_update)(vcpu, apic_find_highest_irr(apic));
3075 static_call_cond(kvm_x86_hwapic_isr_update)(apic_find_highest_isr(apic));
3076 }
3077 kvm_make_request(KVM_REQ_EVENT, vcpu);
3078 if (ioapic_in_kernel(vcpu->kvm))
3079 kvm_rtc_eoi_tracking_restore_one(vcpu);
3080
3081 vcpu->arch.apic_arb_prio = 0;
3082
3083 return 0;
3084 }
3085
__kvm_migrate_apic_timer(struct kvm_vcpu * vcpu)3086 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
3087 {
3088 struct hrtimer *timer;
3089
3090 if (!lapic_in_kernel(vcpu) ||
3091 kvm_can_post_timer_interrupt(vcpu))
3092 return;
3093
3094 timer = &vcpu->arch.apic->lapic_timer.timer;
3095 if (hrtimer_cancel(timer))
3096 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
3097 }
3098
3099 /*
3100 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
3101 *
3102 * Detect whether guest triggered PV EOI since the
3103 * last entry. If yes, set EOI on guests's behalf.
3104 * Clear PV EOI in guest memory in any case.
3105 */
apic_sync_pv_eoi_from_guest(struct kvm_vcpu * vcpu,struct kvm_lapic * apic)3106 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
3107 struct kvm_lapic *apic)
3108 {
3109 int vector;
3110 /*
3111 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
3112 * and KVM_PV_EOI_ENABLED in guest memory as follows:
3113 *
3114 * KVM_APIC_PV_EOI_PENDING is unset:
3115 * -> host disabled PV EOI.
3116 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
3117 * -> host enabled PV EOI, guest did not execute EOI yet.
3118 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
3119 * -> host enabled PV EOI, guest executed EOI.
3120 */
3121 BUG_ON(!pv_eoi_enabled(vcpu));
3122
3123 if (pv_eoi_test_and_clr_pending(vcpu))
3124 return;
3125 vector = apic_set_eoi(apic);
3126 trace_kvm_pv_eoi(apic, vector);
3127 }
3128
kvm_lapic_sync_from_vapic(struct kvm_vcpu * vcpu)3129 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
3130 {
3131 u32 data;
3132
3133 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
3134 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
3135
3136 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
3137 return;
3138
3139 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
3140 sizeof(u32)))
3141 return;
3142
3143 apic_set_tpr(vcpu->arch.apic, data & 0xff);
3144 }
3145
3146 /*
3147 * apic_sync_pv_eoi_to_guest - called before vmentry
3148 *
3149 * Detect whether it's safe to enable PV EOI and
3150 * if yes do so.
3151 */
apic_sync_pv_eoi_to_guest(struct kvm_vcpu * vcpu,struct kvm_lapic * apic)3152 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
3153 struct kvm_lapic *apic)
3154 {
3155 if (!pv_eoi_enabled(vcpu) ||
3156 /* IRR set or many bits in ISR: could be nested. */
3157 apic->irr_pending ||
3158 /* Cache not set: could be safe but we don't bother. */
3159 apic->highest_isr_cache == -1 ||
3160 /* Need EOI to update ioapic. */
3161 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
3162 /*
3163 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
3164 * so we need not do anything here.
3165 */
3166 return;
3167 }
3168
3169 pv_eoi_set_pending(apic->vcpu);
3170 }
3171
kvm_lapic_sync_to_vapic(struct kvm_vcpu * vcpu)3172 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
3173 {
3174 u32 data, tpr;
3175 int max_irr, max_isr;
3176 struct kvm_lapic *apic = vcpu->arch.apic;
3177
3178 apic_sync_pv_eoi_to_guest(vcpu, apic);
3179
3180 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
3181 return;
3182
3183 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
3184 max_irr = apic_find_highest_irr(apic);
3185 if (max_irr < 0)
3186 max_irr = 0;
3187 max_isr = apic_find_highest_isr(apic);
3188 if (max_isr < 0)
3189 max_isr = 0;
3190 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
3191
3192 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
3193 sizeof(u32));
3194 }
3195
kvm_lapic_set_vapic_addr(struct kvm_vcpu * vcpu,gpa_t vapic_addr)3196 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
3197 {
3198 if (vapic_addr) {
3199 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
3200 &vcpu->arch.apic->vapic_cache,
3201 vapic_addr, sizeof(u32)))
3202 return -EINVAL;
3203 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
3204 } else {
3205 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
3206 }
3207
3208 vcpu->arch.apic->vapic_addr = vapic_addr;
3209 return 0;
3210 }
3211
kvm_lapic_msr_read(struct kvm_lapic * apic,u32 reg,u64 * data)3212 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data)
3213 {
3214 u32 low;
3215
3216 if (reg == APIC_ICR) {
3217 *data = kvm_x2apic_icr_read(apic);
3218 return 0;
3219 }
3220
3221 if (kvm_lapic_reg_read(apic, reg, 4, &low))
3222 return 1;
3223
3224 *data = low;
3225
3226 return 0;
3227 }
3228
kvm_lapic_msr_write(struct kvm_lapic * apic,u32 reg,u64 data)3229 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data)
3230 {
3231 /*
3232 * ICR is a 64-bit register in x2APIC mode (and Hyper-V PV vAPIC) and
3233 * can be written as such, all other registers remain accessible only
3234 * through 32-bit reads/writes.
3235 */
3236 if (reg == APIC_ICR)
3237 return kvm_x2apic_icr_write(apic, data);
3238
3239 /* Bits 63:32 are reserved in all other registers. */
3240 if (data >> 32)
3241 return 1;
3242
3243 return kvm_lapic_reg_write(apic, reg, (u32)data);
3244 }
3245
kvm_x2apic_msr_write(struct kvm_vcpu * vcpu,u32 msr,u64 data)3246 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
3247 {
3248 struct kvm_lapic *apic = vcpu->arch.apic;
3249 u32 reg = (msr - APIC_BASE_MSR) << 4;
3250
3251 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
3252 return 1;
3253
3254 return kvm_lapic_msr_write(apic, reg, data);
3255 }
3256
kvm_x2apic_msr_read(struct kvm_vcpu * vcpu,u32 msr,u64 * data)3257 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
3258 {
3259 struct kvm_lapic *apic = vcpu->arch.apic;
3260 u32 reg = (msr - APIC_BASE_MSR) << 4;
3261
3262 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
3263 return 1;
3264
3265 return kvm_lapic_msr_read(apic, reg, data);
3266 }
3267
kvm_hv_vapic_msr_write(struct kvm_vcpu * vcpu,u32 reg,u64 data)3268 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
3269 {
3270 if (!lapic_in_kernel(vcpu))
3271 return 1;
3272
3273 return kvm_lapic_msr_write(vcpu->arch.apic, reg, data);
3274 }
3275
kvm_hv_vapic_msr_read(struct kvm_vcpu * vcpu,u32 reg,u64 * data)3276 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
3277 {
3278 if (!lapic_in_kernel(vcpu))
3279 return 1;
3280
3281 return kvm_lapic_msr_read(vcpu->arch.apic, reg, data);
3282 }
3283
kvm_lapic_set_pv_eoi(struct kvm_vcpu * vcpu,u64 data,unsigned long len)3284 int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
3285 {
3286 u64 addr = data & ~KVM_MSR_ENABLED;
3287 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
3288 unsigned long new_len;
3289 int ret;
3290
3291 if (!IS_ALIGNED(addr, 4))
3292 return 1;
3293
3294 if (data & KVM_MSR_ENABLED) {
3295 if (addr == ghc->gpa && len <= ghc->len)
3296 new_len = ghc->len;
3297 else
3298 new_len = len;
3299
3300 ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
3301 if (ret)
3302 return ret;
3303 }
3304
3305 vcpu->arch.pv_eoi.msr_val = data;
3306
3307 return 0;
3308 }
3309
kvm_apic_accept_events(struct kvm_vcpu * vcpu)3310 int kvm_apic_accept_events(struct kvm_vcpu *vcpu)
3311 {
3312 struct kvm_lapic *apic = vcpu->arch.apic;
3313 u8 sipi_vector;
3314 int r;
3315
3316 if (!kvm_apic_has_pending_init_or_sipi(vcpu))
3317 return 0;
3318
3319 if (is_guest_mode(vcpu)) {
3320 r = kvm_check_nested_events(vcpu);
3321 if (r < 0)
3322 return r == -EBUSY ? 0 : r;
3323 /*
3324 * Continue processing INIT/SIPI even if a nested VM-Exit
3325 * occurred, e.g. pending SIPIs should be dropped if INIT+SIPI
3326 * are blocked as a result of transitioning to VMX root mode.
3327 */
3328 }
3329
3330 /*
3331 * INITs are blocked while CPU is in specific states (SMM, VMX root
3332 * mode, SVM with GIF=0), while SIPIs are dropped if the CPU isn't in
3333 * wait-for-SIPI (WFS).
3334 */
3335 if (!kvm_apic_init_sipi_allowed(vcpu)) {
3336 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
3337 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
3338 return 0;
3339 }
3340
3341 if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) {
3342 kvm_vcpu_reset(vcpu, true);
3343 if (kvm_vcpu_is_bsp(apic->vcpu))
3344 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
3345 else
3346 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
3347 }
3348 if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) {
3349 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
3350 /* evaluate pending_events before reading the vector */
3351 smp_rmb();
3352 sipi_vector = apic->sipi_vector;
3353 static_call(kvm_x86_vcpu_deliver_sipi_vector)(vcpu, sipi_vector);
3354 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
3355 }
3356 }
3357 return 0;
3358 }
3359
kvm_lapic_exit(void)3360 void kvm_lapic_exit(void)
3361 {
3362 static_key_deferred_flush(&apic_hw_disabled);
3363 WARN_ON(static_branch_unlikely(&apic_hw_disabled.key));
3364 static_key_deferred_flush(&apic_sw_disabled);
3365 WARN_ON(static_branch_unlikely(&apic_sw_disabled.key));
3366 }
3367