hyperv.c (bd38b32053eb1c53ddb7030cf0fc6d700f7f1d82) hyperv.c (5974565bc26d6a599189db7c0b1f79eaa9af8eb9)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * KVM Microsoft Hyper-V emulation
4 *
5 * derived from arch/x86/kvm/x86.c
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright (C) 2008 Qumranet, Inc.

--- 22 unchanged lines hidden (view full) ---

31#include <linux/sched/cputime.h>
32#include <linux/eventfd.h>
33
34#include <asm/apicdef.h>
35#include <trace/events/kvm.h>
36
37#include "trace.h"
38#include "irq.h"
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * KVM Microsoft Hyper-V emulation
4 *
5 * derived from arch/x86/kvm/x86.c
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright (C) 2008 Qumranet, Inc.

--- 22 unchanged lines hidden (view full) ---

31#include <linux/sched/cputime.h>
32#include <linux/eventfd.h>
33
34#include <asm/apicdef.h>
35#include <trace/events/kvm.h>
36
37#include "trace.h"
38#include "irq.h"
39#include "fpu.h"
39
40/* "Hv#1" signature */
41#define HYPERV_CPUID_SIGNATURE_EAX 0x31237648
42
43#define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, 64)
44
45static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
46 bool vcpu_kick);

--- 1588 unchanged lines hidden (view full) ---

1635 u64 param;
1636 u64 ingpa;
1637 u64 outgpa;
1638 u16 code;
1639 u16 rep_cnt;
1640 u16 rep_idx;
1641 bool fast;
1642 bool rep;
40
41/* "Hv#1" signature */
42#define HYPERV_CPUID_SIGNATURE_EAX 0x31237648
43
44#define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, 64)
45
46static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
47 bool vcpu_kick);

--- 1588 unchanged lines hidden (view full) ---

1636 u64 param;
1637 u64 ingpa;
1638 u64 outgpa;
1639 u16 code;
1640 u16 rep_cnt;
1641 u16 rep_idx;
1642 bool fast;
1643 bool rep;
1644 sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS];
1643};
1644
1645static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool ex)
1646{
1645};
1646
1647static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool ex)
1648{
1649 int i;
1650 gpa_t gpa;
1647 struct kvm *kvm = vcpu->kvm;
1648 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1649 struct hv_tlb_flush_ex flush_ex;
1650 struct hv_tlb_flush flush;
1651 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
1652 DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS);
1653 unsigned long *vcpu_mask;
1654 u64 valid_bank_mask;
1655 u64 sparse_banks[64];
1656 int sparse_banks_len;
1657 bool all_cpus;
1658
1659 if (!ex) {
1651 struct kvm *kvm = vcpu->kvm;
1652 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1653 struct hv_tlb_flush_ex flush_ex;
1654 struct hv_tlb_flush flush;
1655 u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
1656 DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS);
1657 unsigned long *vcpu_mask;
1658 u64 valid_bank_mask;
1659 u64 sparse_banks[64];
1660 int sparse_banks_len;
1661 bool all_cpus;
1662
1663 if (!ex) {
1660 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush, sizeof(flush))))
1661 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1664 if (hc->fast) {
1665 flush.address_space = hc->ingpa;
1666 flush.flags = hc->outgpa;
1667 flush.processor_mask = sse128_lo(hc->xmm[0]);
1668 } else {
1669 if (unlikely(kvm_read_guest(kvm, hc->ingpa,
1670 &flush, sizeof(flush))))
1671 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1672 }
1662
1663 trace_kvm_hv_flush_tlb(flush.processor_mask,
1664 flush.address_space, flush.flags);
1665
1666 valid_bank_mask = BIT_ULL(0);
1667 sparse_banks[0] = flush.processor_mask;
1668
1669 /*
1670 * Work around possible WS2012 bug: it sends hypercalls
1671 * with processor_mask = 0x0 and HV_FLUSH_ALL_PROCESSORS clear,
1672 * while also expecting us to flush something and crashing if
1673 * we don't. Let's treat processor_mask == 0 same as
1674 * HV_FLUSH_ALL_PROCESSORS.
1675 */
1676 all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) ||
1677 flush.processor_mask == 0;
1678 } else {
1673
1674 trace_kvm_hv_flush_tlb(flush.processor_mask,
1675 flush.address_space, flush.flags);
1676
1677 valid_bank_mask = BIT_ULL(0);
1678 sparse_banks[0] = flush.processor_mask;
1679
1680 /*
1681 * Work around possible WS2012 bug: it sends hypercalls
1682 * with processor_mask = 0x0 and HV_FLUSH_ALL_PROCESSORS clear,
1683 * while also expecting us to flush something and crashing if
1684 * we don't. Let's treat processor_mask == 0 same as
1685 * HV_FLUSH_ALL_PROCESSORS.
1686 */
1687 all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) ||
1688 flush.processor_mask == 0;
1689 } else {
1679 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex,
1680 sizeof(flush_ex))))
1681 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1690 if (hc->fast) {
1691 flush_ex.address_space = hc->ingpa;
1692 flush_ex.flags = hc->outgpa;
1693 memcpy(&flush_ex.hv_vp_set,
1694 &hc->xmm[0], sizeof(hc->xmm[0]));
1695 } else {
1696 if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex,
1697 sizeof(flush_ex))))
1698 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1699 }
1682
1683 trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask,
1684 flush_ex.hv_vp_set.format,
1685 flush_ex.address_space,
1686 flush_ex.flags);
1687
1688 valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask;
1689 all_cpus = flush_ex.hv_vp_set.format !=
1690 HV_GENERIC_SET_SPARSE_4K;
1691
1700
1701 trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask,
1702 flush_ex.hv_vp_set.format,
1703 flush_ex.address_space,
1704 flush_ex.flags);
1705
1706 valid_bank_mask = flush_ex.hv_vp_set.valid_bank_mask;
1707 all_cpus = flush_ex.hv_vp_set.format !=
1708 HV_GENERIC_SET_SPARSE_4K;
1709
1692 sparse_banks_len =
1693 bitmap_weight((unsigned long *)&valid_bank_mask, 64) *
1694 sizeof(sparse_banks[0]);
1710 sparse_banks_len = bitmap_weight((unsigned long *)&valid_bank_mask, 64);
1695
1696 if (!sparse_banks_len && !all_cpus)
1697 goto ret_success;
1698
1711
1712 if (!sparse_banks_len && !all_cpus)
1713 goto ret_success;
1714
1699 if (!all_cpus &&
1700 kvm_read_guest(kvm,
1701 hc->ingpa + offsetof(struct hv_tlb_flush_ex,
1702 hv_vp_set.bank_contents),
1703 sparse_banks,
1704 sparse_banks_len))
1705 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1715 if (!all_cpus) {
1716 if (hc->fast) {
1717 if (sparse_banks_len > HV_HYPERCALL_MAX_XMM_REGISTERS - 1)
1718 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1719 for (i = 0; i < sparse_banks_len; i += 2) {
1720 sparse_banks[i] = sse128_lo(hc->xmm[i / 2 + 1]);
1721 sparse_banks[i + 1] = sse128_hi(hc->xmm[i / 2 + 1]);
1722 }
1723 } else {
1724 gpa = hc->ingpa + offsetof(struct hv_tlb_flush_ex,
1725 hv_vp_set.bank_contents);
1726 if (unlikely(kvm_read_guest(kvm, gpa, sparse_banks,
1727 sparse_banks_len *
1728 sizeof(sparse_banks[0]))))
1729 return HV_STATUS_INVALID_HYPERCALL_INPUT;
1730 }
1731 }
1706 }
1707
1708 cpumask_clear(&hv_vcpu->tlb_flush);
1709
1710 vcpu_mask = all_cpus ? NULL :
1711 sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask,
1712 vp_bitmap, vcpu_bitmap);
1713

--- 179 unchanged lines hidden (view full) ---

1893 rcu_read_unlock();
1894 if (!eventfd)
1895 return HV_STATUS_INVALID_PORT_ID;
1896
1897 eventfd_signal(eventfd, 1);
1898 return HV_STATUS_SUCCESS;
1899}
1900
1732 }
1733
1734 cpumask_clear(&hv_vcpu->tlb_flush);
1735
1736 vcpu_mask = all_cpus ? NULL :
1737 sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask,
1738 vp_bitmap, vcpu_bitmap);
1739

--- 179 unchanged lines hidden (view full) ---

1919 rcu_read_unlock();
1920 if (!eventfd)
1921 return HV_STATUS_INVALID_PORT_ID;
1922
1923 eventfd_signal(eventfd, 1);
1924 return HV_STATUS_SUCCESS;
1925}
1926
1927static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc)
1928{
1929 switch (hc->code) {
1930 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
1931 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
1932 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
1933 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
1934 return true;
1935 }
1936
1937 return false;
1938}
1939
1940static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc)
1941{
1942 int reg;
1943
1944 kvm_fpu_get();
1945 for (reg = 0; reg < HV_HYPERCALL_MAX_XMM_REGISTERS; reg++)
1946 _kvm_read_sse_reg(reg, &hc->xmm[reg]);
1947 kvm_fpu_put();
1948}
1949
1901int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
1902{
1903 struct kvm_hv_hcall hc;
1904 u64 ret = HV_STATUS_SUCCESS;
1905
1906 /*
1907 * hypercall generates UD from non zero cpl and real mode
1908 * per HYPER-V spec

--- 20 unchanged lines hidden (view full) ---

1929 }
1930
1931 hc.code = hc.param & 0xffff;
1932 hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT);
1933 hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff;
1934 hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff;
1935 hc.rep = !!(hc.rep_cnt || hc.rep_idx);
1936
1950int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
1951{
1952 struct kvm_hv_hcall hc;
1953 u64 ret = HV_STATUS_SUCCESS;
1954
1955 /*
1956 * hypercall generates UD from non zero cpl and real mode
1957 * per HYPER-V spec

--- 20 unchanged lines hidden (view full) ---

1978 }
1979
1980 hc.code = hc.param & 0xffff;
1981 hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT);
1982 hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff;
1983 hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff;
1984 hc.rep = !!(hc.rep_cnt || hc.rep_idx);
1985
1986 if (hc.fast && is_xmm_fast_hypercall(&hc))
1987 kvm_hv_hypercall_read_xmm(&hc);
1988
1937 trace_kvm_hv_hypercall(hc.code, hc.fast, hc.rep_cnt, hc.rep_idx,
1938 hc.ingpa, hc.outgpa);
1939
1940 switch (hc.code) {
1941 case HVCALL_NOTIFY_LONG_SPIN_WAIT:
1942 if (unlikely(hc.rep)) {
1943 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1944 break;

--- 19 unchanged lines hidden (view full) ---

1964 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL;
1965 vcpu->run->hyperv.u.hcall.input = hc.param;
1966 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa;
1967 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa;
1968 vcpu->arch.complete_userspace_io =
1969 kvm_hv_hypercall_complete_userspace;
1970 return 0;
1971 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
1989 trace_kvm_hv_hypercall(hc.code, hc.fast, hc.rep_cnt, hc.rep_idx,
1990 hc.ingpa, hc.outgpa);
1991
1992 switch (hc.code) {
1993 case HVCALL_NOTIFY_LONG_SPIN_WAIT:
1994 if (unlikely(hc.rep)) {
1995 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1996 break;

--- 19 unchanged lines hidden (view full) ---

2016 vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL;
2017 vcpu->run->hyperv.u.hcall.input = hc.param;
2018 vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa;
2019 vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa;
2020 vcpu->arch.complete_userspace_io =
2021 kvm_hv_hypercall_complete_userspace;
2022 return 0;
2023 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST:
1972 if (unlikely(hc.fast || !hc.rep_cnt || hc.rep_idx)) {
2024 if (unlikely(!hc.rep_cnt || hc.rep_idx)) {
1973 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1974 break;
1975 }
1976 ret = kvm_hv_flush_tlb(vcpu, &hc, false);
1977 break;
1978 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
2025 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2026 break;
2027 }
2028 ret = kvm_hv_flush_tlb(vcpu, &hc, false);
2029 break;
2030 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE:
1979 if (unlikely(hc.fast || hc.rep)) {
2031 if (unlikely(hc.rep)) {
1980 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1981 break;
1982 }
1983 ret = kvm_hv_flush_tlb(vcpu, &hc, false);
1984 break;
1985 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
2032 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2033 break;
2034 }
2035 ret = kvm_hv_flush_tlb(vcpu, &hc, false);
2036 break;
2037 case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX:
1986 if (unlikely(hc.fast || !hc.rep_cnt || hc.rep_idx)) {
2038 if (unlikely(!hc.rep_cnt || hc.rep_idx)) {
1987 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1988 break;
1989 }
1990 ret = kvm_hv_flush_tlb(vcpu, &hc, true);
1991 break;
1992 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
2039 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2040 break;
2041 }
2042 ret = kvm_hv_flush_tlb(vcpu, &hc, true);
2043 break;
2044 case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX:
1993 if (unlikely(hc.fast || hc.rep)) {
2045 if (unlikely(hc.rep)) {
1994 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1995 break;
1996 }
1997 ret = kvm_hv_flush_tlb(vcpu, &hc, true);
1998 break;
1999 case HVCALL_SEND_IPI:
2000 if (unlikely(hc.rep)) {
2001 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;

--- 271 unchanged lines hidden --- 		2046 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
2047 break;
2048 }
2049 ret = kvm_hv_flush_tlb(vcpu, &hc, true);
2050 break;
2051 case HVCALL_SEND_IPI:
2052 if (unlikely(hc.rep)) {
2053 ret = HV_STATUS_INVALID_HYPERCALL_INPUT;

--- 271 unchanged lines hidden ---