x86.c (34f4335c16a5f4bb7da6c8d2d5e780b6a163846a) x86.c (b9181c8ef35636152facc72f801f27b4264df8c0)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * derived from drivers/kvm/kvm_main.c
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright (C) 2008 Qumranet, Inc.

--- 73 unchanged lines hidden (view full) ---

82#include <asm/sgx.h>
83#include <clocksource/hyperv_timer.h>
84
85#define CREATE_TRACE_POINTS
86#include "trace.h"
87
88#define MAX_IO_MSRS 256
89#define KVM_MAX_MCE_BANKS 32
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * derived from drivers/kvm/kvm_main.c
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright (C) 2008 Qumranet, Inc.

--- 73 unchanged lines hidden (view full) ---

82#include <asm/sgx.h>
83#include <clocksource/hyperv_timer.h>
84
85#define CREATE_TRACE_POINTS
86#include "trace.h"
87
88#define MAX_IO_MSRS 256
89#define KVM_MAX_MCE_BANKS 32
90u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
91EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
92
90
91struct kvm_caps kvm_caps __read_mostly = {
92 .supported_mce_cap = MCG_CTL_P | MCG_SER_P,
93};
94EXPORT_SYMBOL_GPL(kvm_caps);
95
93#define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e))
94
95#define emul_to_vcpu(ctxt) \
96 ((struct kvm_vcpu *)(ctxt)->vcpu)
97
98/* EFER defaults:
99 * - enable syscall per default because its emulated by KVM
100 * - enable LME and LMA per default on 64 bit KVM

--- 45 unchanged lines hidden (view full) ---

146EXPORT_SYMBOL_GPL(report_ignored_msrs);
147
148unsigned int min_timer_period_us = 200;
149module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
150
151static bool __read_mostly kvmclock_periodic_sync = true;
152module_param(kvmclock_periodic_sync, bool, S_IRUGO);
153
96#define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e))
97
98#define emul_to_vcpu(ctxt) \
99 ((struct kvm_vcpu *)(ctxt)->vcpu)
100
101/* EFER defaults:
102 * - enable syscall per default because its emulated by KVM
103 * - enable LME and LMA per default on 64 bit KVM

--- 45 unchanged lines hidden (view full) ---

149EXPORT_SYMBOL_GPL(report_ignored_msrs);
150
151unsigned int min_timer_period_us = 200;
152module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
153
154static bool __read_mostly kvmclock_periodic_sync = true;
155module_param(kvmclock_periodic_sync, bool, S_IRUGO);
156
154bool __read_mostly kvm_has_tsc_control;
155EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
156u32 __read_mostly kvm_max_guest_tsc_khz;
157EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
158u8 __read_mostly kvm_tsc_scaling_ratio_frac_bits;
159EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
160u64 __read_mostly kvm_max_tsc_scaling_ratio;
161EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
162u64 __read_mostly kvm_default_tsc_scaling_ratio;
163EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
164bool __read_mostly kvm_has_bus_lock_exit;
165EXPORT_SYMBOL_GPL(kvm_has_bus_lock_exit);
166
167/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
168static u32 __read_mostly tsc_tolerance_ppm = 250;
169module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
170
171/*
172 * lapic timer advance (tscdeadline mode only) in nanoseconds. '-1' enables
173 * adaptive tuning starting from default advancement of 1000ns. '0' disables
174 * advancement entirely. Any other value is used as-is and disables adaptive

--- 55 unchanged lines hidden (view full) ---

230bool __read_mostly allow_smaller_maxphyaddr = 0;
231EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);
232
233bool __read_mostly enable_apicv = true;
234EXPORT_SYMBOL_GPL(enable_apicv);
235
236u64 __read_mostly host_xss;
237EXPORT_SYMBOL_GPL(host_xss);
157/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
158static u32 __read_mostly tsc_tolerance_ppm = 250;
159module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
160
161/*
162 * lapic timer advance (tscdeadline mode only) in nanoseconds. '-1' enables
163 * adaptive tuning starting from default advancement of 1000ns. '0' disables
164 * advancement entirely. Any other value is used as-is and disables adaptive

--- 55 unchanged lines hidden (view full) ---

220bool __read_mostly allow_smaller_maxphyaddr = 0;
221EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);
222
223bool __read_mostly enable_apicv = true;
224EXPORT_SYMBOL_GPL(enable_apicv);
225
226u64 __read_mostly host_xss;
227EXPORT_SYMBOL_GPL(host_xss);
238u64 __read_mostly supported_xss;
239EXPORT_SYMBOL_GPL(supported_xss);
240
241const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
242 KVM_GENERIC_VM_STATS(),
243 STATS_DESC_COUNTER(VM, mmu_shadow_zapped),
244 STATS_DESC_COUNTER(VM, mmu_pte_write),
245 STATS_DESC_COUNTER(VM, mmu_pde_zapped),
246 STATS_DESC_COUNTER(VM, mmu_flooded),
247 STATS_DESC_COUNTER(VM, mmu_recycled),

--- 43 unchanged lines hidden (view full) ---

291 STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
292 STATS_DESC_COUNTER(VCPU, hypercalls),
293 STATS_DESC_COUNTER(VCPU, irq_injections),
294 STATS_DESC_COUNTER(VCPU, nmi_injections),
295 STATS_DESC_COUNTER(VCPU, req_event),
296 STATS_DESC_COUNTER(VCPU, nested_run),
297 STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
298 STATS_DESC_COUNTER(VCPU, directed_yield_successful),
228
229const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
230 KVM_GENERIC_VM_STATS(),
231 STATS_DESC_COUNTER(VM, mmu_shadow_zapped),
232 STATS_DESC_COUNTER(VM, mmu_pte_write),
233 STATS_DESC_COUNTER(VM, mmu_pde_zapped),
234 STATS_DESC_COUNTER(VM, mmu_flooded),
235 STATS_DESC_COUNTER(VM, mmu_recycled),

--- 43 unchanged lines hidden (view full) ---

279 STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
280 STATS_DESC_COUNTER(VCPU, hypercalls),
281 STATS_DESC_COUNTER(VCPU, irq_injections),
282 STATS_DESC_COUNTER(VCPU, nmi_injections),
283 STATS_DESC_COUNTER(VCPU, req_event),
284 STATS_DESC_COUNTER(VCPU, nested_run),
285 STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
286 STATS_DESC_COUNTER(VCPU, directed_yield_successful),
299 STATS_DESC_COUNTER(VCPU, preemption_reported),
300 STATS_DESC_COUNTER(VCPU, preemption_other),
301 STATS_DESC_ICOUNTER(VCPU, guest_mode)
287 STATS_DESC_ICOUNTER(VCPU, guest_mode),
288 STATS_DESC_COUNTER(VCPU, notify_window_exits),
302};
303
304const struct kvm_stats_header kvm_vcpu_stats_header = {
305 .name_size = KVM_STATS_NAME_SIZE,
306 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
307 .id_offset = sizeof(struct kvm_stats_header),
308 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
309 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
310 sizeof(kvm_vcpu_stats_desc),
311};
312
313u64 __read_mostly host_xcr0;
289};
290
291const struct kvm_stats_header kvm_vcpu_stats_header = {
292 .name_size = KVM_STATS_NAME_SIZE,
293 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
294 .id_offset = sizeof(struct kvm_stats_header),
295 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
296 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
297 sizeof(kvm_vcpu_stats_desc),
298};
299
300u64 __read_mostly host_xcr0;
314u64 __read_mostly supported_xcr0;
315EXPORT_SYMBOL_GPL(supported_xcr0);
316
317static struct kmem_cache *x86_emulator_cache;
318
319/*
320 * When called, it means the previous get/set msr reached an invalid msr.
321 * Return true if we want to ignore/silent this failed msr access.
322 */
323static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write)

--- 1121 unchanged lines hidden (view full) ---

1445 MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
1446 MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
1447 MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
1448 MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
1449 MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
1450 MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
1451 MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
1452 MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
301
302static struct kmem_cache *x86_emulator_cache;
303
304/*
305 * When called, it means the previous get/set msr reached an invalid msr.
306 * Return true if we want to ignore/silent this failed msr access.
307 */
308static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write)

--- 1121 unchanged lines hidden (view full) ---

1430 MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
1431 MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
1432 MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
1433 MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
1434 MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
1435 MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
1436 MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
1437 MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
1438 MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG,
1453
1454 MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
1455 MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
1456 MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
1457 MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
1458 MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
1459 MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
1460 MSR_IA32_XFD, MSR_IA32_XFD_ERR,

--- 880 unchanged lines hidden (view full) ---

2341static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2342
2343static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2344{
2345 u64 ratio;
2346
2347 /* Guest TSC same frequency as host TSC? */
2348 if (!scale) {
1439
1440 MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
1441 MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
1442 MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
1443 MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
1444 MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
1445 MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
1446 MSR_IA32_XFD, MSR_IA32_XFD_ERR,

--- 880 unchanged lines hidden (view full) ---

2327static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2328
2329static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2330{
2331 u64 ratio;
2332
2333 /* Guest TSC same frequency as host TSC? */
2334 if (!scale) {
2349 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_default_tsc_scaling_ratio);
2335 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2350 return 0;
2351 }
2352
2353 /* TSC scaling supported? */
2336 return 0;
2337 }
2338
2339 /* TSC scaling supported? */
2354 if (!kvm_has_tsc_control) {
2340 if (!kvm_caps.has_tsc_control) {
2355 if (user_tsc_khz > tsc_khz) {
2356 vcpu->arch.tsc_catchup = 1;
2357 vcpu->arch.tsc_always_catchup = 1;
2358 return 0;
2359 } else {
2360 pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
2361 return -1;
2362 }
2363 }
2364
2365 /* TSC scaling required - calculate ratio */
2341 if (user_tsc_khz > tsc_khz) {
2342 vcpu->arch.tsc_catchup = 1;
2343 vcpu->arch.tsc_always_catchup = 1;
2344 return 0;
2345 } else {
2346 pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
2347 return -1;
2348 }
2349 }
2350
2351 /* TSC scaling required - calculate ratio */
2366 ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
2352 ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits,
2367 user_tsc_khz, tsc_khz);
2368
2353 user_tsc_khz, tsc_khz);
2354
2369 if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
2355 if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) {
2370 pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
2371 user_tsc_khz);
2372 return -1;
2373 }
2374
2375 kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
2376 return 0;
2377}
2378
2379static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
2380{
2381 u32 thresh_lo, thresh_hi;
2382 int use_scaling = 0;
2383
2384 /* tsc_khz can be zero if TSC calibration fails */
2385 if (user_tsc_khz == 0) {
2386 /* set tsc_scaling_ratio to a safe value */
2356 pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
2357 user_tsc_khz);
2358 return -1;
2359 }
2360
2361 kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
2362 return 0;
2363}
2364
2365static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
2366{
2367 u32 thresh_lo, thresh_hi;
2368 int use_scaling = 0;
2369
2370 /* tsc_khz can be zero if TSC calibration fails */
2371 if (user_tsc_khz == 0) {
2372 /* set tsc_scaling_ratio to a safe value */
2387 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_default_tsc_scaling_ratio);
2373 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2388 return -1;
2389 }
2390
2391 /* Compute a scale to convert nanoseconds in TSC cycles */
2392 kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
2393 &vcpu->arch.virtual_tsc_shift,
2394 &vcpu->arch.virtual_tsc_mult);
2395 vcpu->arch.virtual_tsc_khz = user_tsc_khz;

--- 60 unchanged lines hidden (view full) ---

2456/*
2457 * Multiply tsc by a fixed point number represented by ratio.
2458 *
2459 * The most significant 64-N bits (mult) of ratio represent the
2460 * integral part of the fixed point number; the remaining N bits
2461 * (frac) represent the fractional part, ie. ratio represents a fixed
2462 * point number (mult + frac * 2^(-N)).
2463 *
2374 return -1;
2375 }
2376
2377 /* Compute a scale to convert nanoseconds in TSC cycles */
2378 kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
2379 &vcpu->arch.virtual_tsc_shift,
2380 &vcpu->arch.virtual_tsc_mult);
2381 vcpu->arch.virtual_tsc_khz = user_tsc_khz;

--- 60 unchanged lines hidden (view full) ---

2442/*
2443 * Multiply tsc by a fixed point number represented by ratio.
2444 *
2445 * The most significant 64-N bits (mult) of ratio represent the
2446 * integral part of the fixed point number; the remaining N bits
2447 * (frac) represent the fractional part, ie. ratio represents a fixed
2448 * point number (mult + frac * 2^(-N)).
2449 *
2464 * N equals to kvm_tsc_scaling_ratio_frac_bits.
2450 * N equals to kvm_caps.tsc_scaling_ratio_frac_bits.
2465 */
2466static inline u64 __scale_tsc(u64 ratio, u64 tsc)
2467{
2451 */
2452static inline u64 __scale_tsc(u64 ratio, u64 tsc)
2453{
2468 return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
2454 return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits);
2469}
2470
2471u64 kvm_scale_tsc(u64 tsc, u64 ratio)
2472{
2473 u64 _tsc = tsc;
2474
2455}
2456
2457u64 kvm_scale_tsc(u64 tsc, u64 ratio)
2458{
2459 u64 _tsc = tsc;
2460
2475 if (ratio != kvm_default_tsc_scaling_ratio)
2461 if (ratio != kvm_caps.default_tsc_scaling_ratio)
2476 _tsc = __scale_tsc(ratio, tsc);
2477
2478 return _tsc;
2479}
2480EXPORT_SYMBOL_GPL(kvm_scale_tsc);
2481
2482static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2483{

--- 10 unchanged lines hidden (view full) ---

2494 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
2495}
2496EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
2497
2498u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
2499{
2500 u64 nested_offset;
2501
2462 _tsc = __scale_tsc(ratio, tsc);
2463
2464 return _tsc;
2465}
2466EXPORT_SYMBOL_GPL(kvm_scale_tsc);
2467
2468static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2469{

--- 10 unchanged lines hidden (view full) ---

2480 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
2481}
2482EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
2483
2484u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
2485{
2486 u64 nested_offset;
2487
2502 if (l2_multiplier == kvm_default_tsc_scaling_ratio)
2488 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio)
2503 nested_offset = l1_offset;
2504 else
2505 nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
2489 nested_offset = l1_offset;
2490 else
2491 nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
2506 kvm_tsc_scaling_ratio_frac_bits);
2492 kvm_caps.tsc_scaling_ratio_frac_bits);
2507
2508 nested_offset += l2_offset;
2509 return nested_offset;
2510}
2511EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset);
2512
2513u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
2514{
2493
2494 nested_offset += l2_offset;
2495 return nested_offset;
2496}
2497EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset);
2498
2499u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
2500{
2515 if (l2_multiplier != kvm_default_tsc_scaling_ratio)
2501 if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio)
2516 return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
2502 return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
2517 kvm_tsc_scaling_ratio_frac_bits);
2503 kvm_caps.tsc_scaling_ratio_frac_bits);
2518
2519 return l1_multiplier;
2520}
2521EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier);
2522
2523static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
2524{
2525 trace_kvm_write_tsc_offset(vcpu->vcpu_id,

--- 25 unchanged lines hidden (view full) ---

2551 /* Userspace is changing the multiplier while L2 is active */
2552 if (is_guest_mode(vcpu))
2553 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2554 l1_multiplier,
2555 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2556 else
2557 vcpu->arch.tsc_scaling_ratio = l1_multiplier;
2558
2504
2505 return l1_multiplier;
2506}
2507EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier);
2508
2509static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
2510{
2511 trace_kvm_write_tsc_offset(vcpu->vcpu_id,

--- 25 unchanged lines hidden (view full) ---

2537 /* Userspace is changing the multiplier while L2 is active */
2538 if (is_guest_mode(vcpu))
2539 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2540 l1_multiplier,
2541 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2542 else
2543 vcpu->arch.tsc_scaling_ratio = l1_multiplier;
2544
2559 if (kvm_has_tsc_control)
2545 if (kvm_caps.has_tsc_control)
2560 static_call(kvm_x86_write_tsc_multiplier)(
2561 vcpu, vcpu->arch.tsc_scaling_ratio);
2562}
2563
2564static inline bool kvm_check_tsc_unstable(void)
2565{
2566#ifdef CONFIG_X86_64
2567 /*

--- 119 unchanged lines hidden (view full) ---

2687 s64 adjustment)
2688{
2689 u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2690 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2691}
2692
2693static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2694{
2546 static_call(kvm_x86_write_tsc_multiplier)(
2547 vcpu, vcpu->arch.tsc_scaling_ratio);
2548}
2549
2550static inline bool kvm_check_tsc_unstable(void)
2551{
2552#ifdef CONFIG_X86_64
2553 /*

--- 119 unchanged lines hidden (view full) ---

2673 s64 adjustment)
2674{
2675 u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2676 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2677}
2678
2679static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2680{
2695 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
2681 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)
2696 WARN_ON(adjustment < 0);
2697 adjustment = kvm_scale_tsc((u64) adjustment,
2698 vcpu->arch.l1_tsc_scaling_ratio);
2699 adjust_tsc_offset_guest(vcpu, adjustment);
2700}
2701
2702#ifdef CONFIG_X86_64
2703

--- 396 unchanged lines hidden (view full) ---

3100 tsc_timestamp = tsc;
3101 }
3102 }
3103
3104 local_irq_restore(flags);
3105
3106 /* With all the info we got, fill in the values */
3107
2682 WARN_ON(adjustment < 0);
2683 adjustment = kvm_scale_tsc((u64) adjustment,
2684 vcpu->arch.l1_tsc_scaling_ratio);
2685 adjust_tsc_offset_guest(vcpu, adjustment);
2686}
2687
2688#ifdef CONFIG_X86_64
2689

--- 396 unchanged lines hidden (view full) ---

3086 tsc_timestamp = tsc;
3087 }
3088 }
3089
3090 local_irq_restore(flags);
3091
3092 /* With all the info we got, fill in the values */
3093
3108 if (kvm_has_tsc_control)
3094 if (kvm_caps.has_tsc_control)
3109 tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
3110 v->arch.l1_tsc_scaling_ratio);
3111
3112 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
3113 kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
3114 &vcpu->hv_clock.tsc_shift,
3115 &vcpu->hv_clock.tsc_to_system_mul);
3116 vcpu->hw_tsc_khz = tgt_tsc_khz;

--- 114 unchanged lines hidden (view full) ---

3231 msr < MSR_IA32_MCx_CTL(bank_num)) {
3232 u32 offset = array_index_nospec(
3233 msr - MSR_IA32_MC0_CTL,
3234 MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);
3235
3236 /* only 0 or all 1s can be written to IA32_MCi_CTL
3237 * some Linux kernels though clear bit 10 in bank 4 to
3238 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
3095 tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
3096 v->arch.l1_tsc_scaling_ratio);
3097
3098 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
3099 kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
3100 &vcpu->hv_clock.tsc_shift,
3101 &vcpu->hv_clock.tsc_to_system_mul);
3102 vcpu->hw_tsc_khz = tgt_tsc_khz;

--- 114 unchanged lines hidden (view full) ---

3217 msr < MSR_IA32_MCx_CTL(bank_num)) {
3218 u32 offset = array_index_nospec(
3219 msr - MSR_IA32_MC0_CTL,
3220 MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);
3221
3222 /* only 0 or all 1s can be written to IA32_MCi_CTL
3223 * some Linux kernels though clear bit 10 in bank 4 to
3224 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
3239 * this to avoid an uncatched #GP in the guest
3225 * this to avoid an uncatched #GP in the guest.
3226 *
3227 * UNIXWARE clears bit 0 of MC1_CTL to ignore
3228 * correctable, single-bit ECC data errors.
3240 */
3241 if ((offset & 0x3) == 0 &&
3229 */
3230 if ((offset & 0x3) == 0 &&
3242 data != 0 && (data | (1 << 10)) != ~(u64)0)
3231 data != 0 && (data | (1 << 10) | 1) != ~(u64)0)
3243 return -1;
3244
3245 /* MCi_STATUS */
3246 if (!msr_info->host_initiated &&
3247 (offset & 0x3) == 1 && data != 0) {
3248 if (!can_set_mci_status(vcpu))
3249 return -1;
3250 }

--- 301 unchanged lines hidden (view full) ---

3552 /* Before back to guest, tsc_timestamp must be adjusted
3553 * as well, otherwise guest's percpu pvclock time could jump.
3554 */
3555 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3556 }
3557 vcpu->arch.ia32_tsc_adjust_msr = data;
3558 }
3559 break;
3232 return -1;
3233
3234 /* MCi_STATUS */
3235 if (!msr_info->host_initiated &&
3236 (offset & 0x3) == 1 && data != 0) {
3237 if (!can_set_mci_status(vcpu))
3238 return -1;
3239 }

--- 301 unchanged lines hidden (view full) ---

3541 /* Before back to guest, tsc_timestamp must be adjusted
3542 * as well, otherwise guest's percpu pvclock time could jump.
3543 */
3544 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3545 }
3546 vcpu->arch.ia32_tsc_adjust_msr = data;
3547 }
3548 break;
3560 case MSR_IA32_MISC_ENABLE:
3549 case MSR_IA32_MISC_ENABLE: {
3550 u64 old_val = vcpu->arch.ia32_misc_enable_msr;
3551 u64 pmu_mask = MSR_IA32_MISC_ENABLE_PMU_RO_MASK |
3552 MSR_IA32_MISC_ENABLE_EMON;
3553
3554 /* RO bits */
3555 if (!msr_info->host_initiated &&
3556 ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK))
3557 return 1;
3558
3559 /*
3560 * For a dummy user space, the order of setting vPMU capabilities and
3561 * initialising MSR_IA32_MISC_ENABLE is not strictly guaranteed, so to
3562 * avoid inconsistent functionality we keep the vPMU bits unchanged here.
3563 */
3564 data &= ~pmu_mask;
3565 data |= old_val & pmu_mask;
3561 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
3566 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
3562 ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
3567 ((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
3563 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3564 return 1;
3565 vcpu->arch.ia32_misc_enable_msr = data;
3566 kvm_update_cpuid_runtime(vcpu);
3567 } else {
3568 vcpu->arch.ia32_misc_enable_msr = data;
3569 }
3570 break;
3568 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3569 return 1;
3570 vcpu->arch.ia32_misc_enable_msr = data;
3571 kvm_update_cpuid_runtime(vcpu);
3572 } else {
3573 vcpu->arch.ia32_misc_enable_msr = data;
3574 }
3575 break;
3576 }
3571 case MSR_IA32_SMBASE:
3572 if (!msr_info->host_initiated)
3573 return 1;
3574 vcpu->arch.smbase = data;
3575 break;
3576 case MSR_IA32_POWER_CTL:
3577 vcpu->arch.msr_ia32_power_ctl = data;
3578 break;

--- 10 unchanged lines hidden (view full) ---

3589 if (!msr_info->host_initiated &&
3590 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3591 return 1;
3592 /*
3593 * KVM supports exposing PT to the guest, but does not support
3594 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
3595 * XSAVES/XRSTORS to save/restore PT MSRs.
3596 */
3577 case MSR_IA32_SMBASE:
3578 if (!msr_info->host_initiated)
3579 return 1;
3580 vcpu->arch.smbase = data;
3581 break;
3582 case MSR_IA32_POWER_CTL:
3583 vcpu->arch.msr_ia32_power_ctl = data;
3584 break;

--- 10 unchanged lines hidden (view full) ---

3595 if (!msr_info->host_initiated &&
3596 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3597 return 1;
3598 /*
3599 * KVM supports exposing PT to the guest, but does not support
3600 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
3601 * XSAVES/XRSTORS to save/restore PT MSRs.
3602 */
3597 if (data & ~supported_xss)
3603 if (data & ~kvm_caps.supported_xss)
3598 return 1;
3599 vcpu->arch.ia32_xss = data;
3600 kvm_update_cpuid_runtime(vcpu);
3601 break;
3602 case MSR_SMI_COUNT:
3603 if (!msr_info->host_initiated)
3604 return 1;
3605 vcpu->arch.smi_count = data;

--- 89 unchanged lines hidden (view full) ---

3695 return set_msr_mce(vcpu, msr_info);
3696
3697 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3698 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3699 pr = true;
3700 fallthrough;
3701 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3702 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3604 return 1;
3605 vcpu->arch.ia32_xss = data;
3606 kvm_update_cpuid_runtime(vcpu);
3607 break;
3608 case MSR_SMI_COUNT:
3609 if (!msr_info->host_initiated)
3610 return 1;
3611 vcpu->arch.smi_count = data;

--- 89 unchanged lines hidden (view full) ---

3701 return set_msr_mce(vcpu, msr_info);
3702
3703 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3704 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3705 pr = true;
3706 fallthrough;
3707 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3708 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3703 if (kvm_pmu_is_valid_msr(vcpu, msr))
3709 if (kvm_pmu_is_valid_msr(vcpu, msr, msr_info->host_initiated))
3704 return kvm_pmu_set_msr(vcpu, msr_info);
3705
3706 if (pr || data != 0)
3707 vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
3708 "0x%x data 0x%llx\n", msr, data);
3709 break;
3710 case MSR_K7_CLK_CTL:
3711 /*

--- 66 unchanged lines hidden (view full) ---

3778
3779 if (data & ~kvm_guest_supported_xfd(vcpu))
3780 return 1;
3781
3782 vcpu->arch.guest_fpu.xfd_err = data;
3783 break;
3784#endif
3785 default:
3710 return kvm_pmu_set_msr(vcpu, msr_info);
3711
3712 if (pr || data != 0)
3713 vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
3714 "0x%x data 0x%llx\n", msr, data);
3715 break;
3716 case MSR_K7_CLK_CTL:
3717 /*

--- 66 unchanged lines hidden (view full) ---

3784
3785 if (data & ~kvm_guest_supported_xfd(vcpu))
3786 return 1;
3787
3788 vcpu->arch.guest_fpu.xfd_err = data;
3789 break;
3790#endif
3791 default:
3786 if (kvm_pmu_is_valid_msr(vcpu, msr))
3792 if (kvm_pmu_is_valid_msr(vcpu, msr, msr_info->host_initiated))
3787 return kvm_pmu_set_msr(vcpu, msr_info);
3788 return KVM_MSR_RET_INVALID;
3789 }
3790 return 0;
3791}
3792EXPORT_SYMBOL_GPL(kvm_set_msr_common);
3793
3794static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)

--- 63 unchanged lines hidden (view full) ---

3858 case MSR_RAPL_POWER_UNIT:
3859 case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */
3860 case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */
3861 case MSR_PKG_ENERGY_STATUS: /* Total package */
3862 case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */
3863 msr_info->data = 0;
3864 break;
3865 case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3793 return kvm_pmu_set_msr(vcpu, msr_info);
3794 return KVM_MSR_RET_INVALID;
3795 }
3796 return 0;
3797}
3798EXPORT_SYMBOL_GPL(kvm_set_msr_common);
3799
3800static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)

--- 63 unchanged lines hidden (view full) ---

3864 case MSR_RAPL_POWER_UNIT:
3865 case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */
3866 case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */
3867 case MSR_PKG_ENERGY_STATUS: /* Total package */
3868 case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */
3869 msr_info->data = 0;
3870 break;
3871 case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3866 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3872 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index, msr_info->host_initiated))
3867 return kvm_pmu_get_msr(vcpu, msr_info);
3868 if (!msr_info->host_initiated)
3869 return 1;
3870 msr_info->data = 0;
3871 break;
3872 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3873 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3874 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3875 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3873 return kvm_pmu_get_msr(vcpu, msr_info);
3874 if (!msr_info->host_initiated)
3875 return 1;
3876 msr_info->data = 0;
3877 break;
3878 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3879 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3880 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3881 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3876 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3882 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index, msr_info->host_initiated))
3877 return kvm_pmu_get_msr(vcpu, msr_info);
3878 msr_info->data = 0;
3879 break;
3880 case MSR_IA32_UCODE_REV:
3881 msr_info->data = vcpu->arch.microcode_version;
3882 break;
3883 case MSR_IA32_ARCH_CAPABILITIES:
3884 if (!msr_info->host_initiated &&

--- 229 unchanged lines hidden (view full) ---

4114 if (!msr_info->host_initiated &&
4115 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4116 return 1;
4117
4118 msr_info->data = vcpu->arch.guest_fpu.xfd_err;
4119 break;
4120#endif
4121 default:
3883 return kvm_pmu_get_msr(vcpu, msr_info);
3884 msr_info->data = 0;
3885 break;
3886 case MSR_IA32_UCODE_REV:
3887 msr_info->data = vcpu->arch.microcode_version;
3888 break;
3889 case MSR_IA32_ARCH_CAPABILITIES:
3890 if (!msr_info->host_initiated &&

--- 229 unchanged lines hidden (view full) ---

4120 if (!msr_info->host_initiated &&
4121 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4122 return 1;
4123
4124 msr_info->data = vcpu->arch.guest_fpu.xfd_err;
4125 break;
4126#endif
4127 default:
4122 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
4128 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index, msr_info->host_initiated))
4123 return kvm_pmu_get_msr(vcpu, msr_info);
4124 return KVM_MSR_RET_INVALID;
4125 }
4126 return 0;
4127}
4128EXPORT_SYMBOL_GPL(kvm_get_msr_common);
4129
4130/*

--- 141 unchanged lines hidden (view full) ---

4272 case KVM_CAP_DISABLE_QUIRKS:
4273 case KVM_CAP_SET_BOOT_CPU_ID:
4274 case KVM_CAP_SPLIT_IRQCHIP:
4275 case KVM_CAP_IMMEDIATE_EXIT:
4276 case KVM_CAP_PMU_EVENT_FILTER:
4277 case KVM_CAP_GET_MSR_FEATURES:
4278 case KVM_CAP_MSR_PLATFORM_INFO:
4279 case KVM_CAP_EXCEPTION_PAYLOAD:
4129 return kvm_pmu_get_msr(vcpu, msr_info);
4130 return KVM_MSR_RET_INVALID;
4131 }
4132 return 0;
4133}
4134EXPORT_SYMBOL_GPL(kvm_get_msr_common);
4135
4136/*

--- 141 unchanged lines hidden (view full) ---

4278 case KVM_CAP_DISABLE_QUIRKS:
4279 case KVM_CAP_SET_BOOT_CPU_ID:
4280 case KVM_CAP_SPLIT_IRQCHIP:
4281 case KVM_CAP_IMMEDIATE_EXIT:
4282 case KVM_CAP_PMU_EVENT_FILTER:
4283 case KVM_CAP_GET_MSR_FEATURES:
4284 case KVM_CAP_MSR_PLATFORM_INFO:
4285 case KVM_CAP_EXCEPTION_PAYLOAD:
4286 case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
4280 case KVM_CAP_SET_GUEST_DEBUG:
4281 case KVM_CAP_LAST_CPU:
4282 case KVM_CAP_X86_USER_SPACE_MSR:
4283 case KVM_CAP_X86_MSR_FILTER:
4284 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
4285#ifdef CONFIG_X86_SGX_KVM
4286 case KVM_CAP_SGX_ATTRIBUTE:
4287#endif

--- 61 unchanged lines hidden (view full) ---

4349 case KVM_CAP_MCE:
4350 r = KVM_MAX_MCE_BANKS;
4351 break;
4352 case KVM_CAP_XCRS:
4353 r = boot_cpu_has(X86_FEATURE_XSAVE);
4354 break;
4355 case KVM_CAP_TSC_CONTROL:
4356 case KVM_CAP_VM_TSC_CONTROL:
4287 case KVM_CAP_SET_GUEST_DEBUG:
4288 case KVM_CAP_LAST_CPU:
4289 case KVM_CAP_X86_USER_SPACE_MSR:
4290 case KVM_CAP_X86_MSR_FILTER:
4291 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
4292#ifdef CONFIG_X86_SGX_KVM
4293 case KVM_CAP_SGX_ATTRIBUTE:
4294#endif

--- 61 unchanged lines hidden (view full) ---

4356 case KVM_CAP_MCE:
4357 r = KVM_MAX_MCE_BANKS;
4358 break;
4359 case KVM_CAP_XCRS:
4360 r = boot_cpu_has(X86_FEATURE_XSAVE);
4361 break;
4362 case KVM_CAP_TSC_CONTROL:
4363 case KVM_CAP_VM_TSC_CONTROL:
4357 r = kvm_has_tsc_control;
4364 r = kvm_caps.has_tsc_control;
4358 break;
4359 case KVM_CAP_X2APIC_API:
4360 r = KVM_X2APIC_API_VALID_FLAGS;
4361 break;
4362 case KVM_CAP_NESTED_STATE:
4363 r = kvm_x86_ops.nested_ops->get_state ?
4364 kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
4365 break;

--- 5 unchanged lines hidden (view full) ---

4371 break;
4372 case KVM_CAP_SMALLER_MAXPHYADDR:
4373 r = (int) allow_smaller_maxphyaddr;
4374 break;
4375 case KVM_CAP_STEAL_TIME:
4376 r = sched_info_on();
4377 break;
4378 case KVM_CAP_X86_BUS_LOCK_EXIT:
4365 break;
4366 case KVM_CAP_X2APIC_API:
4367 r = KVM_X2APIC_API_VALID_FLAGS;
4368 break;
4369 case KVM_CAP_NESTED_STATE:
4370 r = kvm_x86_ops.nested_ops->get_state ?
4371 kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
4372 break;

--- 5 unchanged lines hidden (view full) ---

4378 break;
4379 case KVM_CAP_SMALLER_MAXPHYADDR:
4380 r = (int) allow_smaller_maxphyaddr;
4381 break;
4382 case KVM_CAP_STEAL_TIME:
4383 r = sched_info_on();
4384 break;
4385 case KVM_CAP_X86_BUS_LOCK_EXIT:
4379 if (kvm_has_bus_lock_exit)
4386 if (kvm_caps.has_bus_lock_exit)
4380 r = KVM_BUS_LOCK_DETECTION_OFF |
4381 KVM_BUS_LOCK_DETECTION_EXIT;
4382 else
4383 r = 0;
4384 break;
4385 case KVM_CAP_XSAVE2: {
4386 u64 guest_perm = xstate_get_guest_group_perm();
4387
4387 r = KVM_BUS_LOCK_DETECTION_OFF |
4388 KVM_BUS_LOCK_DETECTION_EXIT;
4389 else
4390 r = 0;
4391 break;
4392 case KVM_CAP_XSAVE2: {
4393 u64 guest_perm = xstate_get_guest_group_perm();
4394
4388 r = xstate_required_size(supported_xcr0 & guest_perm, false);
4395 r = xstate_required_size(kvm_caps.supported_xcr0 & guest_perm, false);
4389 if (r < sizeof(struct kvm_xsave))
4390 r = sizeof(struct kvm_xsave);
4391 break;
4392 case KVM_CAP_PMU_CAPABILITY:
4393 r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0;
4394 break;
4395 }
4396 case KVM_CAP_DISABLE_QUIRKS2:
4397 r = KVM_X86_VALID_QUIRKS;
4398 break;
4396 if (r < sizeof(struct kvm_xsave))
4397 r = sizeof(struct kvm_xsave);
4398 break;
4399 case KVM_CAP_PMU_CAPABILITY:
4400 r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0;
4401 break;
4402 }
4403 case KVM_CAP_DISABLE_QUIRKS2:
4404 r = KVM_X86_VALID_QUIRKS;
4405 break;
4406 case KVM_CAP_X86_NOTIFY_VMEXIT:
4407 r = kvm_caps.has_notify_vmexit;
4408 break;
4399 default:
4400 break;
4401 }
4402 return r;
4403}
4404
4405static inline void __user *kvm_get_attr_addr(struct kvm_device_attr *attr)
4406{

--- 11 unchanged lines hidden (view full) ---

4418 if (attr->group)
4419 return -ENXIO;
4420
4421 if (IS_ERR(uaddr))
4422 return PTR_ERR(uaddr);
4423
4424 switch (attr->attr) {
4425 case KVM_X86_XCOMP_GUEST_SUPP:
4409 default:
4410 break;
4411 }
4412 return r;
4413}
4414
4415static inline void __user *kvm_get_attr_addr(struct kvm_device_attr *attr)
4416{

--- 11 unchanged lines hidden (view full) ---

4428 if (attr->group)
4429 return -ENXIO;
4430
4431 if (IS_ERR(uaddr))
4432 return PTR_ERR(uaddr);
4433
4434 switch (attr->attr) {
4435 case KVM_X86_XCOMP_GUEST_SUPP:
4426 if (put_user(supported_xcr0, uaddr))
4436 if (put_user(kvm_caps.supported_xcr0, uaddr))
4427 return -EFAULT;
4428 return 0;
4429 default:
4430 return -ENXIO;
4431 break;
4432 }
4433}
4434

--- 60 unchanged lines hidden (view full) ---

4495 r = -EFAULT;
4496 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4497 goto out;
4498 r = 0;
4499 break;
4500 }
4501 case KVM_X86_GET_MCE_CAP_SUPPORTED:
4502 r = -EFAULT;
4437 return -EFAULT;
4438 return 0;
4439 default:
4440 return -ENXIO;
4441 break;
4442 }
4443}
4444

--- 60 unchanged lines hidden (view full) ---

4505 r = -EFAULT;
4506 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4507 goto out;
4508 r = 0;
4509 break;
4510 }
4511 case KVM_X86_GET_MCE_CAP_SUPPORTED:
4512 r = -EFAULT;
4503 if (copy_to_user(argp, &kvm_mce_cap_supported,
4504 sizeof(kvm_mce_cap_supported)))
4513 if (copy_to_user(argp, &kvm_caps.supported_mce_cap,
4514 sizeof(kvm_caps.supported_mce_cap)))
4505 goto out;
4506 r = 0;
4507 break;
4508 case KVM_GET_MSR_FEATURE_INDEX_LIST: {
4509 struct kvm_msr_list __user *user_msr_list = argp;
4510 struct kvm_msr_list msr_list;
4511 unsigned int n;
4512

--- 109 unchanged lines hidden (view full) ---

4622
4623static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
4624{
4625 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4626 struct kvm_steal_time __user *st;
4627 struct kvm_memslots *slots;
4628 static const u8 preempted = KVM_VCPU_PREEMPTED;
4629
4515 goto out;
4516 r = 0;
4517 break;
4518 case KVM_GET_MSR_FEATURE_INDEX_LIST: {
4519 struct kvm_msr_list __user *user_msr_list = argp;
4520 struct kvm_msr_list msr_list;
4521 unsigned int n;
4522

--- 109 unchanged lines hidden (view full) ---

4632
4633static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
4634{
4635 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4636 struct kvm_steal_time __user *st;
4637 struct kvm_memslots *slots;
4638 static const u8 preempted = KVM_VCPU_PREEMPTED;
4639
4630 /*
4631 * The vCPU can be marked preempted if and only if the VM-Exit was on
4632 * an instruction boundary and will not trigger guest emulation of any
4633 * kind (see vcpu_run). Vendor specific code controls (conservatively)
4634 * when this is true, for example allowing the vCPU to be marked
4635 * preempted if and only if the VM-Exit was due to a host interrupt.
4636 */
4637 if (!vcpu->arch.at_instruction_boundary) {
4638 vcpu->stat.preemption_other++;
4639 return;
4640 }
4641
4642 vcpu->stat.preemption_reported++;
4643 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4644 return;
4645
4646 if (vcpu->arch.st.preempted)
4647 return;
4648
4649 /* This happens on process exit */
4650 if (unlikely(current->mm != vcpu->kvm->mm))

--- 13 unchanged lines hidden (view full) ---

4664
4665 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
4666}
4667
4668void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4669{
4670 int idx;
4671
4640 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4641 return;
4642
4643 if (vcpu->arch.st.preempted)
4644 return;
4645
4646 /* This happens on process exit */
4647 if (unlikely(current->mm != vcpu->kvm->mm))

--- 13 unchanged lines hidden (view full) ---

4661
4662 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
4663}
4664
4665void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4666{
4667 int idx;
4668
4672 if (vcpu->preempted) {
4673 if (!vcpu->arch.guest_state_protected)
4674 vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
4669 if (vcpu->preempted && !vcpu->arch.guest_state_protected)
4670 vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
4675
4671
4676 /*
4677 * Take the srcu lock as memslots will be accessed to check the gfn
4678 * cache generation against the memslots generation.
4679 */
4680 idx = srcu_read_lock(&vcpu->kvm->srcu);
4681 if (kvm_xen_msr_enabled(vcpu->kvm))
4682 kvm_xen_runstate_set_preempted(vcpu);
4683 else
4684 kvm_steal_time_set_preempted(vcpu);
4685 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4686 }
4672 /*
4673 * Take the srcu lock as memslots will be accessed to check the gfn
4674 * cache generation against the memslots generation.
4675 */
4676 idx = srcu_read_lock(&vcpu->kvm->srcu);
4677 if (kvm_xen_msr_enabled(vcpu->kvm))
4678 kvm_xen_runstate_set_preempted(vcpu);
4679 else
4680 kvm_steal_time_set_preempted(vcpu);
4681 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4687
4688 static_call(kvm_x86_vcpu_put)(vcpu);
4689 vcpu->arch.last_host_tsc = rdtsc();
4690}
4691
4692static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
4693 struct kvm_lapic_state *s)
4694{

--- 100 unchanged lines hidden (view full) ---

4795 u64 mcg_cap)
4796{
4797 int r;
4798 unsigned bank_num = mcg_cap & 0xff, bank;
4799
4800 r = -EINVAL;
4801 if (!bank_num || bank_num > KVM_MAX_MCE_BANKS)
4802 goto out;
4682
4683 static_call(kvm_x86_vcpu_put)(vcpu);
4684 vcpu->arch.last_host_tsc = rdtsc();
4685}
4686
4687static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
4688 struct kvm_lapic_state *s)
4689{

--- 100 unchanged lines hidden (view full) ---

4790 u64 mcg_cap)
4791{
4792 int r;
4793 unsigned bank_num = mcg_cap & 0xff, bank;
4794
4795 r = -EINVAL;
4796 if (!bank_num || bank_num > KVM_MAX_MCE_BANKS)
4797 goto out;
4803 if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
4798 if (mcg_cap & ~(kvm_caps.supported_mce_cap | 0xff | 0xff0000))
4804 goto out;
4805 r = 0;
4806 vcpu->arch.mcg_cap = mcg_cap;
4807 /* Init IA32_MCG_CTL to all 1s */
4808 if (mcg_cap & MCG_CTL_P)
4809 vcpu->arch.mcg_ctl = ~(u64)0;
4810 /* Init IA32_MCi_CTL to all 1s */
4811 for (bank = 0; bank < bank_num; bank++)

--- 121 unchanged lines hidden (view full) ---

4933 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
4934 events->smi.latched_init = kvm_lapic_latched_init(vcpu);
4935
4936 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
4937 | KVM_VCPUEVENT_VALID_SHADOW
4938 | KVM_VCPUEVENT_VALID_SMM);
4939 if (vcpu->kvm->arch.exception_payload_enabled)
4940 events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
4799 goto out;
4800 r = 0;
4801 vcpu->arch.mcg_cap = mcg_cap;
4802 /* Init IA32_MCG_CTL to all 1s */
4803 if (mcg_cap & MCG_CTL_P)
4804 vcpu->arch.mcg_ctl = ~(u64)0;
4805 /* Init IA32_MCi_CTL to all 1s */
4806 for (bank = 0; bank < bank_num; bank++)

--- 121 unchanged lines hidden (view full) ---

4928 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
4929 events->smi.latched_init = kvm_lapic_latched_init(vcpu);
4930
4931 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
4932 | KVM_VCPUEVENT_VALID_SHADOW
4933 | KVM_VCPUEVENT_VALID_SMM);
4934 if (vcpu->kvm->arch.exception_payload_enabled)
4935 events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
4936 if (vcpu->kvm->arch.triple_fault_event) {
4937 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4938 events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT;
4939 }
4941
4942 memset(&events->reserved, 0, sizeof(events->reserved));
4943}
4944
4945static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm);
4946
4947static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
4948 struct kvm_vcpu_events *events)
4949{
4950 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
4951 | KVM_VCPUEVENT_VALID_SIPI_VECTOR
4952 | KVM_VCPUEVENT_VALID_SHADOW
4953 | KVM_VCPUEVENT_VALID_SMM
4940
4941 memset(&events->reserved, 0, sizeof(events->reserved));
4942}
4943
4944static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm);
4945
4946static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
4947 struct kvm_vcpu_events *events)
4948{
4949 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
4950 | KVM_VCPUEVENT_VALID_SIPI_VECTOR
4951 | KVM_VCPUEVENT_VALID_SHADOW
4952 | KVM_VCPUEVENT_VALID_SMM
4954 | KVM_VCPUEVENT_VALID_PAYLOAD))
4953 | KVM_VCPUEVENT_VALID_PAYLOAD
4954 | KVM_VCPUEVENT_VALID_TRIPLE_FAULT))
4955 return -EINVAL;
4956
4957 if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
4958 if (!vcpu->kvm->arch.exception_payload_enabled)
4959 return -EINVAL;
4960 if (events->exception.pending)
4961 events->exception.injected = 0;
4962 else

--- 56 unchanged lines hidden (view full) ---

5019 if (lapic_in_kernel(vcpu)) {
5020 if (events->smi.latched_init)
5021 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5022 else
5023 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5024 }
5025 }
5026
4955 return -EINVAL;
4956
4957 if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
4958 if (!vcpu->kvm->arch.exception_payload_enabled)
4959 return -EINVAL;
4960 if (events->exception.pending)
4961 events->exception.injected = 0;
4962 else

--- 56 unchanged lines hidden (view full) ---

5019 if (lapic_in_kernel(vcpu)) {
5020 if (events->smi.latched_init)
5021 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5022 else
5023 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5024 }
5025 }
5026
5027 if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) {
5028 if (!vcpu->kvm->arch.triple_fault_event)
5029 return -EINVAL;
5030 if (events->triple_fault.pending)
5031 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5032 else
5033 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5034 }
5035
5027 kvm_make_request(KVM_REQ_EVENT, vcpu);
5028
5029 return 0;
5030}
5031
5032static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
5033 struct kvm_debugregs *dbgregs)
5034{

--- 52 unchanged lines hidden (view full) ---

5087static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
5088 struct kvm_xsave *guest_xsave)
5089{
5090 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5091 return 0;
5092
5093 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
5094 guest_xsave->region,
5036 kvm_make_request(KVM_REQ_EVENT, vcpu);
5037
5038 return 0;
5039}
5040
5041static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
5042 struct kvm_debugregs *dbgregs)
5043{

--- 52 unchanged lines hidden (view full) ---

5096static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
5097 struct kvm_xsave *guest_xsave)
5098{
5099 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5100 return 0;
5101
5102 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
5103 guest_xsave->region,
5095 supported_xcr0, &vcpu->arch.pkru);
5104 kvm_caps.supported_xcr0,
5105 &vcpu->arch.pkru);
5096}
5097
5098static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
5099 struct kvm_xcrs *guest_xcrs)
5100{
5101 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
5102 guest_xcrs->nr_xcrs = 0;
5103 return;

--- 488 unchanged lines hidden (view full) ---

5592 break;
5593 }
5594 case KVM_SET_TSC_KHZ: {
5595 u32 user_tsc_khz;
5596
5597 r = -EINVAL;
5598 user_tsc_khz = (u32)arg;
5599
5106}
5107
5108static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
5109 struct kvm_xcrs *guest_xcrs)
5110{
5111 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
5112 guest_xcrs->nr_xcrs = 0;
5113 return;

--- 488 unchanged lines hidden (view full) ---

5602 break;
5603 }
5604 case KVM_SET_TSC_KHZ: {
5605 u32 user_tsc_khz;
5606
5607 r = -EINVAL;
5608 user_tsc_khz = (u32)arg;
5609
5600 if (kvm_has_tsc_control &&
5601 user_tsc_khz >= kvm_max_guest_tsc_khz)
5610 if (kvm_caps.has_tsc_control &&
5611 user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
5602 goto out;
5603
5604 if (user_tsc_khz == 0)
5605 user_tsc_khz = tsc_khz;
5606
5607 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
5608 r = 0;
5609

--- 410 unchanged lines hidden (view full) ---

6020 case KVM_CAP_MSR_PLATFORM_INFO:
6021 kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
6022 r = 0;
6023 break;
6024 case KVM_CAP_EXCEPTION_PAYLOAD:
6025 kvm->arch.exception_payload_enabled = cap->args[0];
6026 r = 0;
6027 break;
5612 goto out;
5613
5614 if (user_tsc_khz == 0)
5615 user_tsc_khz = tsc_khz;
5616
5617 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
5618 r = 0;
5619

--- 410 unchanged lines hidden (view full) ---

6030 case KVM_CAP_MSR_PLATFORM_INFO:
6031 kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
6032 r = 0;
6033 break;
6034 case KVM_CAP_EXCEPTION_PAYLOAD:
6035 kvm->arch.exception_payload_enabled = cap->args[0];
6036 r = 0;
6037 break;
6038 case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
6039 kvm->arch.triple_fault_event = cap->args[0];
6040 r = 0;
6041 break;
6028 case KVM_CAP_X86_USER_SPACE_MSR:
6029 kvm->arch.user_space_msr_mask = cap->args[0];
6030 r = 0;
6031 break;
6032 case KVM_CAP_X86_BUS_LOCK_EXIT:
6033 r = -EINVAL;
6034 if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
6035 break;
6036
6037 if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
6038 (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
6039 break;
6040
6042 case KVM_CAP_X86_USER_SPACE_MSR:
6043 kvm->arch.user_space_msr_mask = cap->args[0];
6044 r = 0;
6045 break;
6046 case KVM_CAP_X86_BUS_LOCK_EXIT:
6047 r = -EINVAL;
6048 if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
6049 break;
6050
6051 if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
6052 (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
6053 break;
6054
6041 if (kvm_has_bus_lock_exit &&
6055 if (kvm_caps.has_bus_lock_exit &&
6042 cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
6043 kvm->arch.bus_lock_detection_enabled = true;
6044 r = 0;
6045 break;
6046#ifdef CONFIG_X86_SGX_KVM
6047 case KVM_CAP_SGX_ATTRIBUTE: {
6048 unsigned long allowed_attributes = 0;
6049

--- 46 unchanged lines hidden (view full) ---

6096
6097 mutex_lock(&kvm->lock);
6098 if (!kvm->created_vcpus) {
6099 kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
6100 r = 0;
6101 }
6102 mutex_unlock(&kvm->lock);
6103 break;
6056 cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
6057 kvm->arch.bus_lock_detection_enabled = true;
6058 r = 0;
6059 break;
6060#ifdef CONFIG_X86_SGX_KVM
6061 case KVM_CAP_SGX_ATTRIBUTE: {
6062 unsigned long allowed_attributes = 0;
6063

--- 46 unchanged lines hidden (view full) ---

6110
6111 mutex_lock(&kvm->lock);
6112 if (!kvm->created_vcpus) {
6113 kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
6114 r = 0;
6115 }
6116 mutex_unlock(&kvm->lock);
6117 break;
6118 case KVM_CAP_MAX_VCPU_ID:
6119 r = -EINVAL;
6120 if (cap->args[0] > KVM_MAX_VCPU_IDS)
6121 break;
6122
6123 mutex_lock(&kvm->lock);
6124 if (kvm->arch.max_vcpu_ids == cap->args[0]) {
6125 r = 0;
6126 } else if (!kvm->arch.max_vcpu_ids) {
6127 kvm->arch.max_vcpu_ids = cap->args[0];
6128 r = 0;
6129 }
6130 mutex_unlock(&kvm->lock);
6131 break;
6132 case KVM_CAP_X86_NOTIFY_VMEXIT:
6133 r = -EINVAL;
6134 if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS)
6135 break;
6136 if (!kvm_caps.has_notify_vmexit)
6137 break;
6138 if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED))
6139 break;
6140 mutex_lock(&kvm->lock);
6141 if (!kvm->created_vcpus) {
6142 kvm->arch.notify_window = cap->args[0] >> 32;
6143 kvm->arch.notify_vmexit_flags = (u32)cap->args[0];
6144 r = 0;
6145 }
6146 mutex_unlock(&kvm->lock);
6147 break;
6104 default:
6105 r = -EINVAL;
6106 break;
6107 }
6108 return r;
6109}
6110
6111static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)

--- 456 unchanged lines hidden (view full) ---

6568 r = kvm_vm_ioctl_get_clock(kvm, argp);
6569 break;
6570 case KVM_SET_TSC_KHZ: {
6571 u32 user_tsc_khz;
6572
6573 r = -EINVAL;
6574 user_tsc_khz = (u32)arg;
6575
6148 default:
6149 r = -EINVAL;
6150 break;
6151 }
6152 return r;
6153}
6154
6155static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)

--- 456 unchanged lines hidden (view full) ---

6612 r = kvm_vm_ioctl_get_clock(kvm, argp);
6613 break;
6614 case KVM_SET_TSC_KHZ: {
6615 u32 user_tsc_khz;
6616
6617 r = -EINVAL;
6618 user_tsc_khz = (u32)arg;
6619
6576 if (kvm_has_tsc_control &&
6577 user_tsc_khz >= kvm_max_guest_tsc_khz)
6620 if (kvm_caps.has_tsc_control &&
6621 user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
6578 goto out;
6579
6580 if (user_tsc_khz == 0)
6581 user_tsc_khz = tsc_khz;
6582
6583 WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz);
6584 r = 0;
6585

--- 58 unchanged lines hidden (view full) ---

6644 r = -ENOTTY;
6645 }
6646out:
6647 return r;
6648}
6649
6650static void kvm_init_msr_list(void)
6651{
6622 goto out;
6623
6624 if (user_tsc_khz == 0)
6625 user_tsc_khz = tsc_khz;
6626
6627 WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz);
6628 r = 0;
6629

--- 58 unchanged lines hidden (view full) ---

6688 r = -ENOTTY;
6689 }
6690out:
6691 return r;
6692}
6693
6694static void kvm_init_msr_list(void)
6695{
6652 struct x86_pmu_capability x86_pmu;
6653 u32 dummy[2];
6654 unsigned i;
6655
6656 BUILD_BUG_ON_MSG(KVM_PMC_MAX_FIXED != 3,
6657 "Please update the fixed PMCs in msrs_to_saved_all[]");
6658
6696 u32 dummy[2];
6697 unsigned i;
6698
6699 BUILD_BUG_ON_MSG(KVM_PMC_MAX_FIXED != 3,
6700 "Please update the fixed PMCs in msrs_to_saved_all[]");
6701
6659 perf_get_x86_pmu_capability(&x86_pmu);
6660
6661 num_msrs_to_save = 0;
6662 num_emulated_msrs = 0;
6663 num_msr_based_features = 0;
6664
6665 for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
6666 if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
6667 continue;
6668

--- 35 unchanged lines hidden (view full) ---

6704 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
6705 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
6706 msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
6707 intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
6708 continue;
6709 break;
6710 case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
6711 if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
6702 num_msrs_to_save = 0;
6703 num_emulated_msrs = 0;
6704 num_msr_based_features = 0;
6705
6706 for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
6707 if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
6708 continue;
6709

--- 35 unchanged lines hidden (view full) ---

6745 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
6746 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
6747 msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
6748 intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
6749 continue;
6750 break;
6751 case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
6752 if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
6712 min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
6753 min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp))
6713 continue;
6714 break;
6715 case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
6716 if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
6754 continue;
6755 break;
6756 case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
6757 if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
6717 min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
6758 min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp))
6718 continue;
6719 break;
6720 case MSR_IA32_XFD:
6721 case MSR_IA32_XFD_ERR:
6722 if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
6723 continue;
6724 break;
6725 default:

--- 2009 unchanged lines hidden (view full) ---

8735 kvm_make_mclock_inprogress_request(kvm);
8736
8737 /* no guest entries from this point */
8738 hyperv_stop_tsc_emulation();
8739
8740 /* TSC frequency always matches when on Hyper-V */
8741 for_each_present_cpu(cpu)
8742 per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
6759 continue;
6760 break;
6761 case MSR_IA32_XFD:
6762 case MSR_IA32_XFD_ERR:
6763 if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
6764 continue;
6765 break;
6766 default:

--- 2009 unchanged lines hidden (view full) ---

8776 kvm_make_mclock_inprogress_request(kvm);
8777
8778 /* no guest entries from this point */
8779 hyperv_stop_tsc_emulation();
8780
8781 /* TSC frequency always matches when on Hyper-V */
8782 for_each_present_cpu(cpu)
8783 per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
8743 kvm_max_guest_tsc_khz = tsc_khz;
8784 kvm_caps.max_guest_tsc_khz = tsc_khz;
8744
8745 list_for_each_entry(kvm, &vm_list, vm_list) {
8746 __kvm_start_pvclock_update(kvm);
8747 pvclock_update_vm_gtod_copy(kvm);
8748 kvm_end_pvclock_update(kvm);
8749 }
8750
8751 mutex_unlock(&kvm_lock);

--- 245 unchanged lines hidden (view full) ---

8997 r = kvm_mmu_vendor_module_init();
8998 if (r)
8999 goto out_free_percpu;
9000
9001 kvm_timer_init();
9002
9003 if (boot_cpu_has(X86_FEATURE_XSAVE)) {
9004 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
8785
8786 list_for_each_entry(kvm, &vm_list, vm_list) {
8787 __kvm_start_pvclock_update(kvm);
8788 pvclock_update_vm_gtod_copy(kvm);
8789 kvm_end_pvclock_update(kvm);
8790 }
8791
8792 mutex_unlock(&kvm_lock);

--- 245 unchanged lines hidden (view full) ---

9038 r = kvm_mmu_vendor_module_init();
9039 if (r)
9040 goto out_free_percpu;
9041
9042 kvm_timer_init();
9043
9044 if (boot_cpu_has(X86_FEATURE_XSAVE)) {
9045 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
9005 supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
9046 kvm_caps.supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
9006 }
9007
9008 if (pi_inject_timer == -1)
9009 pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
9010#ifdef CONFIG_X86_64
9011 pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
9012
9013 if (hypervisor_is_type(X86_HYPER_MS_HYPERV))

--- 403 unchanged lines hidden (view full) ---

9417 return 1;
9418 }
9419
9420 return kvm_x86_ops.nested_ops->check_events(vcpu);
9421}
9422
9423static void kvm_inject_exception(struct kvm_vcpu *vcpu)
9424{
9047 }
9048
9049 if (pi_inject_timer == -1)
9050 pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
9051#ifdef CONFIG_X86_64
9052 pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
9053
9054 if (hypervisor_is_type(X86_HYPER_MS_HYPERV))

--- 403 unchanged lines hidden (view full) ---

9458 return 1;
9459 }
9460
9461 return kvm_x86_ops.nested_ops->check_events(vcpu);
9462}
9463
9464static void kvm_inject_exception(struct kvm_vcpu *vcpu)
9465{
9466 trace_kvm_inj_exception(vcpu->arch.exception.nr,
9467 vcpu->arch.exception.has_error_code,
9468 vcpu->arch.exception.error_code,
9469 vcpu->arch.exception.injected);
9470
9425 if (vcpu->arch.exception.error_code && !is_protmode(vcpu))
9426 vcpu->arch.exception.error_code = false;
9427 static_call(kvm_x86_queue_exception)(vcpu);
9428}
9429
9430static int inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit)
9431{
9432 int r;

--- 19 unchanged lines hidden (view full) ---

9452 * serviced prior to recognizing any new events in order to
9453 * fully complete the previous instruction.
9454 */
9455 else if (!vcpu->arch.exception.pending) {
9456 if (vcpu->arch.nmi_injected) {
9457 static_call(kvm_x86_inject_nmi)(vcpu);
9458 can_inject = false;
9459 } else if (vcpu->arch.interrupt.injected) {
9471 if (vcpu->arch.exception.error_code && !is_protmode(vcpu))
9472 vcpu->arch.exception.error_code = false;
9473 static_call(kvm_x86_queue_exception)(vcpu);
9474}
9475
9476static int inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit)
9477{
9478 int r;

--- 19 unchanged lines hidden (view full) ---

9498 * serviced prior to recognizing any new events in order to
9499 * fully complete the previous instruction.
9500 */
9501 else if (!vcpu->arch.exception.pending) {
9502 if (vcpu->arch.nmi_injected) {
9503 static_call(kvm_x86_inject_nmi)(vcpu);
9504 can_inject = false;
9505 } else if (vcpu->arch.interrupt.injected) {
9460 static_call(kvm_x86_inject_irq)(vcpu);
9506 static_call(kvm_x86_inject_irq)(vcpu, true);
9461 can_inject = false;
9462 }
9463 }
9464
9465 WARN_ON_ONCE(vcpu->arch.exception.injected &&
9466 vcpu->arch.exception.pending);
9467
9468 /*

--- 5 unchanged lines hidden (view full) ---

9474 if (is_guest_mode(vcpu)) {
9475 r = kvm_check_nested_events(vcpu);
9476 if (r < 0)
9477 goto out;
9478 }
9479
9480 /* try to inject new event if pending */
9481 if (vcpu->arch.exception.pending) {
9507 can_inject = false;
9508 }
9509 }
9510
9511 WARN_ON_ONCE(vcpu->arch.exception.injected &&
9512 vcpu->arch.exception.pending);
9513
9514 /*

--- 5 unchanged lines hidden (view full) ---

9520 if (is_guest_mode(vcpu)) {
9521 r = kvm_check_nested_events(vcpu);
9522 if (r < 0)
9523 goto out;
9524 }
9525
9526 /* try to inject new event if pending */
9527 if (vcpu->arch.exception.pending) {
9482 trace_kvm_inj_exception(vcpu->arch.exception.nr,
9483 vcpu->arch.exception.has_error_code,
9484 vcpu->arch.exception.error_code);
9485
9486 vcpu->arch.exception.pending = false;
9487 vcpu->arch.exception.injected = true;
9488
9489 if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
9490 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
9491 X86_EFLAGS_RF);
9492
9493 if (vcpu->arch.exception.nr == DB_VECTOR) {
9494 kvm_deliver_exception_payload(vcpu);
9495 if (vcpu->arch.dr7 & DR7_GD) {
9496 vcpu->arch.dr7 &= ~DR7_GD;
9497 kvm_update_dr7(vcpu);
9498 }
9499 }
9500
9501 kvm_inject_exception(vcpu);
9528 if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
9529 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
9530 X86_EFLAGS_RF);
9531
9532 if (vcpu->arch.exception.nr == DB_VECTOR) {
9533 kvm_deliver_exception_payload(vcpu);
9534 if (vcpu->arch.dr7 & DR7_GD) {
9535 vcpu->arch.dr7 &= ~DR7_GD;
9536 kvm_update_dr7(vcpu);
9537 }
9538 }
9539
9540 kvm_inject_exception(vcpu);
9541
9542 vcpu->arch.exception.pending = false;
9543 vcpu->arch.exception.injected = true;
9544
9502 can_inject = false;
9503 }
9504
9505 /* Don't inject interrupts if the user asked to avoid doing so */
9506 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)
9507 return 0;
9508
9509 /*

--- 36 unchanged lines hidden (view full) ---

9546 }
9547
9548 if (kvm_cpu_has_injectable_intr(vcpu)) {
9549 r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
9550 if (r < 0)
9551 goto out;
9552 if (r) {
9553 kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
9545 can_inject = false;
9546 }
9547
9548 /* Don't inject interrupts if the user asked to avoid doing so */
9549 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)
9550 return 0;
9551
9552 /*

--- 36 unchanged lines hidden (view full) ---

9589 }
9590
9591 if (kvm_cpu_has_injectable_intr(vcpu)) {
9592 r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
9593 if (r < 0)
9594 goto out;
9595 if (r) {
9596 kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
9554 static_call(kvm_x86_inject_irq)(vcpu);
9597 static_call(kvm_x86_inject_irq)(vcpu, false);
9555 WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
9556 }
9557 if (kvm_cpu_has_injectable_intr(vcpu))
9558 static_call(kvm_x86_enable_irq_window)(vcpu);
9559 }
9560
9561 if (is_guest_mode(vcpu) &&
9562 kvm_x86_ops.nested_ops->hv_timer_pending &&

--- 871 unchanged lines hidden (view full) ---

10434/* Called within kvm->srcu read side. */
10435static int vcpu_run(struct kvm_vcpu *vcpu)
10436{
10437 int r;
10438
10439 vcpu->arch.l1tf_flush_l1d = true;
10440
10441 for (;;) {
9598 WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
9599 }
9600 if (kvm_cpu_has_injectable_intr(vcpu))
9601 static_call(kvm_x86_enable_irq_window)(vcpu);
9602 }
9603
9604 if (is_guest_mode(vcpu) &&
9605 kvm_x86_ops.nested_ops->hv_timer_pending &&

--- 871 unchanged lines hidden (view full) ---

10477/* Called within kvm->srcu read side. */
10478static int vcpu_run(struct kvm_vcpu *vcpu)
10479{
10480 int r;
10481
10482 vcpu->arch.l1tf_flush_l1d = true;
10483
10484 for (;;) {
10442 /*
10443 * If another guest vCPU requests a PV TLB flush in the middle
10444 * of instruction emulation, the rest of the emulation could
10445 * use a stale page translation. Assume that any code after
10446 * this point can start executing an instruction.
10447 */
10448 vcpu->arch.at_instruction_boundary = false;
10449 if (kvm_vcpu_running(vcpu)) {
10450 r = vcpu_enter_guest(vcpu);
10451 } else {
10452 r = vcpu_block(vcpu);
10453 }
10454
10455 if (r <= 0)
10456 break;

--- 799 unchanged lines hidden (view full) ---

11256 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
11257 }
11258
11259 return 0;
11260}
11261
11262int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
11263{
10485 if (kvm_vcpu_running(vcpu)) {
10486 r = vcpu_enter_guest(vcpu);
10487 } else {
10488 r = vcpu_block(vcpu);
10489 }
10490
10491 if (r <= 0)
10492 break;

--- 799 unchanged lines hidden (view full) ---

11292 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
11293 }
11294
11295 return 0;
11296}
11297
11298int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
11299{
11264 if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
11300 if (kvm_check_tsc_unstable() && kvm->created_vcpus)
11265 pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
11266 "guest TSC will not be reliable\n");
11267
11301 pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
11302 "guest TSC will not be reliable\n");
11303
11268 return 0;
11304 if (!kvm->arch.max_vcpu_ids)
11305 kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS;
11306
11307 if (id >= kvm->arch.max_vcpu_ids)
11308 return -EINVAL;
11309
11310 return static_call(kvm_x86_vcpu_precreate)(kvm);
11269}
11270
11271int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
11272{
11273 struct page *page;
11274 int r;
11275
11276 vcpu->arch.last_vmentry_cpu = -1;

--- 420 unchanged lines hidden (view full) ---

11697 struct kvm_x86_init_ops *ops = opaque;
11698 int r;
11699
11700 rdmsrl_safe(MSR_EFER, &host_efer);
11701
11702 if (boot_cpu_has(X86_FEATURE_XSAVES))
11703 rdmsrl(MSR_IA32_XSS, host_xss);
11704
11311}
11312
11313int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
11314{
11315 struct page *page;
11316 int r;
11317
11318 vcpu->arch.last_vmentry_cpu = -1;

--- 420 unchanged lines hidden (view full) ---

11739 struct kvm_x86_init_ops *ops = opaque;
11740 int r;
11741
11742 rdmsrl_safe(MSR_EFER, &host_efer);
11743
11744 if (boot_cpu_has(X86_FEATURE_XSAVES))
11745 rdmsrl(MSR_IA32_XSS, host_xss);
11746
11747 kvm_init_pmu_capability();
11748
11705 r = ops->hardware_setup();
11706 if (r != 0)
11707 return r;
11708
11709 kvm_ops_update(ops);
11710
11711 kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
11712
11713 if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
11749 r = ops->hardware_setup();
11750 if (r != 0)
11751 return r;
11752
11753 kvm_ops_update(ops);
11754
11755 kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
11756
11757 if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
11714 supported_xss = 0;
11758 kvm_caps.supported_xss = 0;
11715
11716#define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
11717 cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
11718#undef __kvm_cpu_cap_has
11719
11759
11760#define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
11761 cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
11762#undef __kvm_cpu_cap_has
11763
11720 if (kvm_has_tsc_control) {
11764 if (kvm_caps.has_tsc_control) {
11721 /*
11722 * Make sure the user can only configure tsc_khz values that
11723 * fit into a signed integer.
11724 * A min value is not calculated because it will always
11725 * be 1 on all machines.
11726 */
11727 u64 max = min(0x7fffffffULL,
11765 /*
11766 * Make sure the user can only configure tsc_khz values that
11767 * fit into a signed integer.
11768 * A min value is not calculated because it will always
11769 * be 1 on all machines.
11770 */
11771 u64 max = min(0x7fffffffULL,
11728 __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
11729 kvm_max_guest_tsc_khz = max;
11772 __scale_tsc(kvm_caps.max_tsc_scaling_ratio, tsc_khz));
11773 kvm_caps.max_guest_tsc_khz = max;
11730 }
11774 }
11731 kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
11775 kvm_caps.default_tsc_scaling_ratio = 1ULL << kvm_caps.tsc_scaling_ratio_frac_bits;
11732 kvm_init_msr_list();
11733 return 0;
11734}
11735
11736void kvm_arch_hardware_unsetup(void)
11737{
11738 kvm_unregister_perf_callbacks();
11739

--- 1379 unchanged lines hidden --- 		11776 kvm_init_msr_list();
11777 return 0;
11778}
11779
11780void kvm_arch_hardware_unsetup(void)
11781{
11782 kvm_unregister_perf_callbacks();
11783

--- 1379 unchanged lines hidden ---