1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * amx tests 4 * 5 * Copyright (C) 2021, Intel, Inc. 6 * 7 * Tests for amx #NM exception and save/restore. 8 */ 9 10 #define _GNU_SOURCE /* for program_invocation_short_name */ 11 #include <fcntl.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #include <sys/ioctl.h> 16 #include <sys/syscall.h> 17 18 #include "test_util.h" 19 20 #include "kvm_util.h" 21 #include "processor.h" 22 #include "vmx.h" 23 24 #ifndef __x86_64__ 25 # error This test is 64-bit only 26 #endif 27 28 #define NUM_TILES 8 29 #define TILE_SIZE 1024 30 #define XSAVE_SIZE ((NUM_TILES * TILE_SIZE) + PAGE_SIZE) 31 32 /* Tile configuration associated: */ 33 #define MAX_TILES 16 34 #define RESERVED_BYTES 14 35 36 #define XFEATURE_XTILECFG 17 37 #define XFEATURE_XTILEDATA 18 38 #define XFEATURE_MASK_XTILECFG (1 << XFEATURE_XTILECFG) 39 #define XFEATURE_MASK_XTILEDATA (1 << XFEATURE_XTILEDATA) 40 #define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILECFG | XFEATURE_MASK_XTILEDATA) 41 42 #define XSAVE_HDR_OFFSET 512 43 44 struct xsave_data { 45 u8 area[XSAVE_SIZE]; 46 } __aligned(64); 47 48 struct tile_config { 49 u8 palette_id; 50 u8 start_row; 51 u8 reserved[RESERVED_BYTES]; 52 u16 colsb[MAX_TILES]; 53 u8 rows[MAX_TILES]; 54 }; 55 56 struct tile_data { 57 u8 data[NUM_TILES * TILE_SIZE]; 58 }; 59 60 struct xtile_info { 61 u16 bytes_per_tile; 62 u16 bytes_per_row; 63 u16 max_names; 64 u16 max_rows; 65 u32 xsave_offset; 66 u32 xsave_size; 67 }; 68 69 static struct xtile_info xtile; 70 71 static inline u64 __xgetbv(u32 index) 72 { 73 u32 eax, edx; 74 75 asm volatile("xgetbv;" 76 : "=a" (eax), "=d" (edx) 77 : "c" (index)); 78 return eax + ((u64)edx << 32); 79 } 80 81 static inline void __xsetbv(u32 index, u64 value) 82 { 83 u32 eax = value; 84 u32 edx = value >> 32; 85 86 asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); 87 } 88 89 static inline void __ldtilecfg(void *cfg) 90 { 91 asm volatile(".byte 0xc4,0xe2,0x78,0x49,0x00" 92 : : "a"(cfg)); 93 } 94 95 static inline void __tileloadd(void *tile) 96 { 97 asm volatile(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10" 98 : : "a"(tile), "d"(0)); 99 } 100 101 static inline void __tilerelease(void) 102 { 103 asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::); 104 } 105 106 static inline void __xsavec(struct xsave_data *data, uint64_t rfbm) 107 { 108 uint32_t rfbm_lo = rfbm; 109 uint32_t rfbm_hi = rfbm >> 32; 110 111 asm volatile("xsavec (%%rdi)" 112 : : "D" (data), "a" (rfbm_lo), "d" (rfbm_hi) 113 : "memory"); 114 } 115 116 static inline void check_cpuid_xsave(void) 117 { 118 GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE)); 119 GUEST_ASSERT(this_cpu_has(X86_FEATURE_OSXSAVE)); 120 } 121 122 static bool check_xsave_supports_xtile(void) 123 { 124 return __xgetbv(0) & XFEATURE_MASK_XTILE; 125 } 126 127 static void check_xtile_info(void) 128 { 129 GUEST_ASSERT(this_cpu_has_p(X86_PROPERTY_XSTATE_MAX_SIZE_XCR0)); 130 GUEST_ASSERT(this_cpu_property(X86_PROPERTY_XSTATE_MAX_SIZE_XCR0) <= XSAVE_SIZE); 131 132 xtile.xsave_offset = this_cpu_property(X86_PROPERTY_XSTATE_TILE_OFFSET); 133 GUEST_ASSERT(xtile.xsave_offset == 2816); 134 xtile.xsave_size = this_cpu_property(X86_PROPERTY_XSTATE_TILE_SIZE); 135 GUEST_ASSERT(xtile.xsave_size == 8192); 136 GUEST_ASSERT(sizeof(struct tile_data) >= xtile.xsave_size); 137 138 GUEST_ASSERT(this_cpu_has_p(X86_PROPERTY_AMX_NR_TILE_REGS)); 139 xtile.max_names = this_cpu_property(X86_PROPERTY_AMX_NR_TILE_REGS); 140 GUEST_ASSERT(xtile.max_names == 8); 141 xtile.bytes_per_tile = this_cpu_property(X86_PROPERTY_AMX_BYTES_PER_TILE); 142 GUEST_ASSERT(xtile.bytes_per_tile == 1024); 143 xtile.bytes_per_row = this_cpu_property(X86_PROPERTY_AMX_BYTES_PER_ROW); 144 GUEST_ASSERT(xtile.bytes_per_row == 64); 145 xtile.max_rows = this_cpu_property(X86_PROPERTY_AMX_MAX_ROWS); 146 GUEST_ASSERT(xtile.max_rows == 16); 147 } 148 149 static void set_tilecfg(struct tile_config *cfg) 150 { 151 int i; 152 153 /* Only palette id 1 */ 154 cfg->palette_id = 1; 155 for (i = 0; i < xtile.max_names; i++) { 156 cfg->colsb[i] = xtile.bytes_per_row; 157 cfg->rows[i] = xtile.max_rows; 158 } 159 } 160 161 static void set_xstatebv(void *data, uint64_t bv) 162 { 163 *(uint64_t *)(data + XSAVE_HDR_OFFSET) = bv; 164 } 165 166 static u64 get_xstatebv(void *data) 167 { 168 return *(u64 *)(data + XSAVE_HDR_OFFSET); 169 } 170 171 static void init_regs(void) 172 { 173 uint64_t cr4, xcr0; 174 175 /* turn on CR4.OSXSAVE */ 176 cr4 = get_cr4(); 177 cr4 |= X86_CR4_OSXSAVE; 178 set_cr4(cr4); 179 180 xcr0 = __xgetbv(0); 181 xcr0 |= XFEATURE_MASK_XTILE; 182 __xsetbv(0x0, xcr0); 183 } 184 185 static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, 186 struct tile_data *tiledata, 187 struct xsave_data *xsave_data) 188 { 189 init_regs(); 190 check_cpuid_xsave(); 191 check_xsave_supports_xtile(); 192 check_xtile_info(); 193 GUEST_SYNC(1); 194 195 /* xfd=0, enable amx */ 196 wrmsr(MSR_IA32_XFD, 0); 197 GUEST_SYNC(2); 198 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0); 199 set_tilecfg(amx_cfg); 200 __ldtilecfg(amx_cfg); 201 GUEST_SYNC(3); 202 /* Check save/restore when trap to userspace */ 203 __tileloadd(tiledata); 204 GUEST_SYNC(4); 205 __tilerelease(); 206 GUEST_SYNC(5); 207 /* bit 18 not in the XCOMP_BV after xsavec() */ 208 set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA); 209 __xsavec(xsave_data, XFEATURE_MASK_XTILEDATA); 210 GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0); 211 212 /* xfd=0x40000, disable amx tiledata */ 213 wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA); 214 GUEST_SYNC(6); 215 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA); 216 set_tilecfg(amx_cfg); 217 __ldtilecfg(amx_cfg); 218 /* Trigger #NM exception */ 219 __tileloadd(tiledata); 220 GUEST_SYNC(10); 221 222 GUEST_DONE(); 223 } 224 225 void guest_nm_handler(struct ex_regs *regs) 226 { 227 /* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */ 228 GUEST_SYNC(7); 229 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA); 230 GUEST_SYNC(8); 231 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA); 232 /* Clear xfd_err */ 233 wrmsr(MSR_IA32_XFD_ERR, 0); 234 /* xfd=0, enable amx */ 235 wrmsr(MSR_IA32_XFD, 0); 236 GUEST_SYNC(9); 237 } 238 239 int main(int argc, char *argv[]) 240 { 241 struct kvm_regs regs1, regs2; 242 struct kvm_vcpu *vcpu; 243 struct kvm_vm *vm; 244 struct kvm_run *run; 245 struct kvm_x86_state *state; 246 int xsave_restore_size; 247 vm_vaddr_t amx_cfg, tiledata, xsavedata; 248 struct ucall uc; 249 u32 amx_offset; 250 int stage, ret; 251 252 /* 253 * Note, all off-by-default features must be enabled before anything 254 * caches KVM_GET_SUPPORTED_CPUID, e.g. before using kvm_cpu_has(). 255 */ 256 vm_xsave_require_permission(XSTATE_XTILE_DATA_BIT); 257 258 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XFD)); 259 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XSAVE)); 260 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_AMX_TILE)); 261 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILECFG)); 262 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILEDATA)); 263 264 /* Create VM */ 265 vm = vm_create_with_one_vcpu(&vcpu, guest_code); 266 267 TEST_ASSERT(kvm_cpu_has_p(X86_PROPERTY_XSTATE_MAX_SIZE), 268 "KVM should enumerate max XSAVE size when XSAVE is supported"); 269 xsave_restore_size = kvm_cpu_property(X86_PROPERTY_XSTATE_MAX_SIZE); 270 271 run = vcpu->run; 272 vcpu_regs_get(vcpu, ®s1); 273 274 /* Register #NM handler */ 275 vm_init_descriptor_tables(vm); 276 vcpu_init_descriptor_tables(vcpu); 277 vm_install_exception_handler(vm, NM_VECTOR, guest_nm_handler); 278 279 /* amx cfg for guest_code */ 280 amx_cfg = vm_vaddr_alloc_page(vm); 281 memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize()); 282 283 /* amx tiledata for guest_code */ 284 tiledata = vm_vaddr_alloc_pages(vm, 2); 285 memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize()); 286 287 /* xsave data for guest_code */ 288 xsavedata = vm_vaddr_alloc_pages(vm, 3); 289 memset(addr_gva2hva(vm, xsavedata), 0, 3 * getpagesize()); 290 vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xsavedata); 291 292 for (stage = 1; ; stage++) { 293 vcpu_run(vcpu); 294 TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, 295 "Stage %d: unexpected exit reason: %u (%s),\n", 296 stage, run->exit_reason, 297 exit_reason_str(run->exit_reason)); 298 299 switch (get_ucall(vcpu, &uc)) { 300 case UCALL_ABORT: 301 REPORT_GUEST_ASSERT(uc); 302 /* NOT REACHED */ 303 case UCALL_SYNC: 304 switch (uc.args[1]) { 305 case 1: 306 case 2: 307 case 3: 308 case 5: 309 case 6: 310 case 7: 311 case 8: 312 fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]); 313 break; 314 case 4: 315 case 10: 316 fprintf(stderr, 317 "GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]); 318 319 /* Compacted mode, get amx offset by xsave area 320 * size subtract 8K amx size. 321 */ 322 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE; 323 state = vcpu_save_state(vcpu); 324 void *amx_start = (void *)state->xsave + amx_offset; 325 void *tiles_data = (void *)addr_gva2hva(vm, tiledata); 326 /* Only check TMM0 register, 1 tile */ 327 ret = memcmp(amx_start, tiles_data, TILE_SIZE); 328 TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret); 329 kvm_x86_state_cleanup(state); 330 break; 331 case 9: 332 fprintf(stderr, 333 "GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]); 334 break; 335 } 336 break; 337 case UCALL_DONE: 338 fprintf(stderr, "UCALL_DONE\n"); 339 goto done; 340 default: 341 TEST_FAIL("Unknown ucall %lu", uc.cmd); 342 } 343 344 state = vcpu_save_state(vcpu); 345 memset(®s1, 0, sizeof(regs1)); 346 vcpu_regs_get(vcpu, ®s1); 347 348 kvm_vm_release(vm); 349 350 /* Restore state in a new VM. */ 351 vcpu = vm_recreate_with_one_vcpu(vm); 352 vcpu_load_state(vcpu, state); 353 run = vcpu->run; 354 kvm_x86_state_cleanup(state); 355 356 memset(®s2, 0, sizeof(regs2)); 357 vcpu_regs_get(vcpu, ®s2); 358 TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), 359 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", 360 (ulong) regs2.rdi, (ulong) regs2.rsi); 361 } 362 done: 363 kvm_vm_free(vm); 364 } 365