1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * amx tests 4 * 5 * Copyright (C) 2021, Intel, Inc. 6 * 7 * Tests for amx #NM exception and save/restore. 8 */ 9 10 #define _GNU_SOURCE /* for program_invocation_short_name */ 11 #include <fcntl.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #include <sys/ioctl.h> 16 #include <sys/syscall.h> 17 18 #include "test_util.h" 19 20 #include "kvm_util.h" 21 #include "processor.h" 22 #include "vmx.h" 23 24 #ifndef __x86_64__ 25 # error This test is 64-bit only 26 #endif 27 28 #define VCPU_ID 0 29 #define X86_FEATURE_XSAVE (1 << 26) 30 #define X86_FEATURE_OSXSAVE (1 << 27) 31 32 #define PAGE_SIZE (1 << 12) 33 #define NUM_TILES 8 34 #define TILE_SIZE 1024 35 #define XSAVE_SIZE ((NUM_TILES * TILE_SIZE) + PAGE_SIZE) 36 37 /* Tile configuration associated: */ 38 #define MAX_TILES 16 39 #define RESERVED_BYTES 14 40 41 #define XFEATURE_XTILECFG 17 42 #define XFEATURE_XTILEDATA 18 43 #define XFEATURE_MASK_XTILECFG (1 << XFEATURE_XTILECFG) 44 #define XFEATURE_MASK_XTILEDATA (1 << XFEATURE_XTILEDATA) 45 #define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILECFG | XFEATURE_MASK_XTILEDATA) 46 47 #define TILE_CPUID 0x1d 48 #define XSTATE_CPUID 0xd 49 #define TILE_PALETTE_CPUID_SUBLEAVE 0x1 50 #define XSTATE_USER_STATE_SUBLEAVE 0x0 51 52 #define XSAVE_HDR_OFFSET 512 53 54 struct xsave_data { 55 u8 area[XSAVE_SIZE]; 56 } __aligned(64); 57 58 struct tile_config { 59 u8 palette_id; 60 u8 start_row; 61 u8 reserved[RESERVED_BYTES]; 62 u16 colsb[MAX_TILES]; 63 u8 rows[MAX_TILES]; 64 }; 65 66 struct tile_data { 67 u8 data[NUM_TILES * TILE_SIZE]; 68 }; 69 70 struct xtile_info { 71 u16 bytes_per_tile; 72 u16 bytes_per_row; 73 u16 max_names; 74 u16 max_rows; 75 u32 xsave_offset; 76 u32 xsave_size; 77 }; 78 79 static struct xtile_info xtile; 80 81 static inline u64 __xgetbv(u32 index) 82 { 83 u32 eax, edx; 84 85 asm volatile("xgetbv;" 86 : "=a" (eax), "=d" (edx) 87 : "c" (index)); 88 return eax + ((u64)edx << 32); 89 } 90 91 static inline void __xsetbv(u32 index, u64 value) 92 { 93 u32 eax = value; 94 u32 edx = value >> 32; 95 96 asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); 97 } 98 99 static inline void __ldtilecfg(void *cfg) 100 { 101 asm volatile(".byte 0xc4,0xe2,0x78,0x49,0x00" 102 : : "a"(cfg)); 103 } 104 105 static inline void __tileloadd(void *tile) 106 { 107 asm volatile(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10" 108 : : "a"(tile), "d"(0)); 109 } 110 111 static inline void __tilerelease(void) 112 { 113 asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::); 114 } 115 116 static inline void __xsavec(struct xsave_data *data, uint64_t rfbm) 117 { 118 uint32_t rfbm_lo = rfbm; 119 uint32_t rfbm_hi = rfbm >> 32; 120 121 asm volatile("xsavec (%%rdi)" 122 : : "D" (data), "a" (rfbm_lo), "d" (rfbm_hi) 123 : "memory"); 124 } 125 126 static inline void check_cpuid_xsave(void) 127 { 128 uint32_t eax, ebx, ecx, edx; 129 130 eax = 1; 131 ecx = 0; 132 cpuid(&eax, &ebx, &ecx, &edx); 133 if (!(ecx & X86_FEATURE_XSAVE)) 134 GUEST_ASSERT(!"cpuid: no CPU xsave support!"); 135 if (!(ecx & X86_FEATURE_OSXSAVE)) 136 GUEST_ASSERT(!"cpuid: no OS xsave support!"); 137 } 138 139 static bool check_xsave_supports_xtile(void) 140 { 141 return __xgetbv(0) & XFEATURE_MASK_XTILE; 142 } 143 144 static bool enum_xtile_config(void) 145 { 146 u32 eax, ebx, ecx, edx; 147 148 eax = TILE_CPUID; 149 ecx = TILE_PALETTE_CPUID_SUBLEAVE; 150 151 cpuid(&eax, &ebx, &ecx, &edx); 152 if (!eax || !ebx || !ecx) 153 return false; 154 155 xtile.max_names = ebx >> 16; 156 if (xtile.max_names < NUM_TILES) 157 return false; 158 159 xtile.bytes_per_tile = eax >> 16; 160 if (xtile.bytes_per_tile < TILE_SIZE) 161 return false; 162 163 xtile.bytes_per_row = ebx; 164 xtile.max_rows = ecx; 165 166 return true; 167 } 168 169 static bool enum_xsave_tile(void) 170 { 171 u32 eax, ebx, ecx, edx; 172 173 eax = XSTATE_CPUID; 174 ecx = XFEATURE_XTILEDATA; 175 176 cpuid(&eax, &ebx, &ecx, &edx); 177 if (!eax || !ebx) 178 return false; 179 180 xtile.xsave_offset = ebx; 181 xtile.xsave_size = eax; 182 183 return true; 184 } 185 186 static bool check_xsave_size(void) 187 { 188 u32 eax, ebx, ecx, edx; 189 bool valid = false; 190 191 eax = XSTATE_CPUID; 192 ecx = XSTATE_USER_STATE_SUBLEAVE; 193 194 cpuid(&eax, &ebx, &ecx, &edx); 195 if (ebx && ebx <= XSAVE_SIZE) 196 valid = true; 197 198 return valid; 199 } 200 201 static bool check_xtile_info(void) 202 { 203 bool ret = false; 204 205 if (!check_xsave_size()) 206 return ret; 207 208 if (!enum_xsave_tile()) 209 return ret; 210 211 if (!enum_xtile_config()) 212 return ret; 213 214 if (sizeof(struct tile_data) >= xtile.xsave_size) 215 ret = true; 216 217 return ret; 218 } 219 220 static void set_tilecfg(struct tile_config *cfg) 221 { 222 int i; 223 224 /* Only palette id 1 */ 225 cfg->palette_id = 1; 226 for (i = 0; i < xtile.max_names; i++) { 227 cfg->colsb[i] = xtile.bytes_per_row; 228 cfg->rows[i] = xtile.max_rows; 229 } 230 } 231 232 static void set_xstatebv(void *data, uint64_t bv) 233 { 234 *(uint64_t *)(data + XSAVE_HDR_OFFSET) = bv; 235 } 236 237 static u64 get_xstatebv(void *data) 238 { 239 return *(u64 *)(data + XSAVE_HDR_OFFSET); 240 } 241 242 static void init_regs(void) 243 { 244 uint64_t cr4, xcr0; 245 246 /* turn on CR4.OSXSAVE */ 247 cr4 = get_cr4(); 248 cr4 |= X86_CR4_OSXSAVE; 249 set_cr4(cr4); 250 251 xcr0 = __xgetbv(0); 252 xcr0 |= XFEATURE_MASK_XTILE; 253 __xsetbv(0x0, xcr0); 254 } 255 256 static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, 257 struct tile_data *tiledata, 258 struct xsave_data *xsave_data) 259 { 260 init_regs(); 261 check_cpuid_xsave(); 262 GUEST_ASSERT(check_xsave_supports_xtile()); 263 GUEST_ASSERT(check_xtile_info()); 264 265 /* check xtile configs */ 266 GUEST_ASSERT(xtile.xsave_offset == 2816); 267 GUEST_ASSERT(xtile.xsave_size == 8192); 268 GUEST_ASSERT(xtile.max_names == 8); 269 GUEST_ASSERT(xtile.bytes_per_tile == 1024); 270 GUEST_ASSERT(xtile.bytes_per_row == 64); 271 GUEST_ASSERT(xtile.max_rows == 16); 272 GUEST_SYNC(1); 273 274 /* xfd=0, enable amx */ 275 wrmsr(MSR_IA32_XFD, 0); 276 GUEST_SYNC(2); 277 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0); 278 set_tilecfg(amx_cfg); 279 __ldtilecfg(amx_cfg); 280 GUEST_SYNC(3); 281 /* Check save/restore when trap to userspace */ 282 __tileloadd(tiledata); 283 GUEST_SYNC(4); 284 __tilerelease(); 285 GUEST_SYNC(5); 286 /* bit 18 not in the XCOMP_BV after xsavec() */ 287 set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA); 288 __xsavec(xsave_data, XFEATURE_MASK_XTILEDATA); 289 GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0); 290 291 /* xfd=0x40000, disable amx tiledata */ 292 wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA); 293 GUEST_SYNC(6); 294 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA); 295 set_tilecfg(amx_cfg); 296 __ldtilecfg(amx_cfg); 297 /* Trigger #NM exception */ 298 __tileloadd(tiledata); 299 GUEST_SYNC(10); 300 301 GUEST_DONE(); 302 } 303 304 void guest_nm_handler(struct ex_regs *regs) 305 { 306 /* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */ 307 GUEST_SYNC(7); 308 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA); 309 GUEST_SYNC(8); 310 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA); 311 /* Clear xfd_err */ 312 wrmsr(MSR_IA32_XFD_ERR, 0); 313 /* xfd=0, enable amx */ 314 wrmsr(MSR_IA32_XFD, 0); 315 GUEST_SYNC(9); 316 } 317 318 int main(int argc, char *argv[]) 319 { 320 struct kvm_cpuid_entry2 *entry; 321 struct kvm_regs regs1, regs2; 322 bool amx_supported = false; 323 struct kvm_vm *vm; 324 struct kvm_run *run; 325 struct kvm_x86_state *state; 326 int xsave_restore_size = 0; 327 vm_vaddr_t amx_cfg, tiledata, xsavedata; 328 struct ucall uc; 329 u32 amx_offset; 330 int stage, ret; 331 332 vm_xsave_req_perm(XSTATE_XTILE_DATA_BIT); 333 334 /* Create VM */ 335 vm = vm_create_default(VCPU_ID, 0, guest_code); 336 337 entry = kvm_get_supported_cpuid_entry(1); 338 if (!(entry->ecx & X86_FEATURE_XSAVE)) { 339 print_skip("XSAVE feature not supported"); 340 exit(KSFT_SKIP); 341 } 342 343 if (kvm_get_cpuid_max_basic() >= 0xd) { 344 entry = kvm_get_supported_cpuid_index(0xd, 0); 345 amx_supported = entry && !!(entry->eax & XFEATURE_MASK_XTILE); 346 if (!amx_supported) { 347 print_skip("AMX is not supported by the vCPU (eax=0x%x)", entry->eax); 348 exit(KSFT_SKIP); 349 } 350 /* Get xsave/restore max size */ 351 xsave_restore_size = entry->ecx; 352 } 353 354 run = vcpu_state(vm, VCPU_ID); 355 vcpu_regs_get(vm, VCPU_ID, ®s1); 356 357 /* Register #NM handler */ 358 vm_init_descriptor_tables(vm); 359 vcpu_init_descriptor_tables(vm, VCPU_ID); 360 vm_install_exception_handler(vm, NM_VECTOR, guest_nm_handler); 361 362 /* amx cfg for guest_code */ 363 amx_cfg = vm_vaddr_alloc_page(vm); 364 memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize()); 365 366 /* amx tiledata for guest_code */ 367 tiledata = vm_vaddr_alloc_pages(vm, 2); 368 memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize()); 369 370 /* xsave data for guest_code */ 371 xsavedata = vm_vaddr_alloc_pages(vm, 3); 372 memset(addr_gva2hva(vm, xsavedata), 0, 3 * getpagesize()); 373 vcpu_args_set(vm, VCPU_ID, 3, amx_cfg, tiledata, xsavedata); 374 375 for (stage = 1; ; stage++) { 376 _vcpu_run(vm, VCPU_ID); 377 TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, 378 "Stage %d: unexpected exit reason: %u (%s),\n", 379 stage, run->exit_reason, 380 exit_reason_str(run->exit_reason)); 381 382 switch (get_ucall(vm, VCPU_ID, &uc)) { 383 case UCALL_ABORT: 384 TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], 385 __FILE__, uc.args[1]); 386 /* NOT REACHED */ 387 case UCALL_SYNC: 388 switch (uc.args[1]) { 389 case 1: 390 case 2: 391 case 3: 392 case 5: 393 case 6: 394 case 7: 395 case 8: 396 fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]); 397 break; 398 case 4: 399 case 10: 400 fprintf(stderr, 401 "GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]); 402 403 /* Compacted mode, get amx offset by xsave area 404 * size subtract 8K amx size. 405 */ 406 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE; 407 state = vcpu_save_state(vm, VCPU_ID); 408 void *amx_start = (void *)state->xsave + amx_offset; 409 void *tiles_data = (void *)addr_gva2hva(vm, tiledata); 410 /* Only check TMM0 register, 1 tile */ 411 ret = memcmp(amx_start, tiles_data, TILE_SIZE); 412 TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret); 413 kvm_x86_state_cleanup(state); 414 break; 415 case 9: 416 fprintf(stderr, 417 "GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]); 418 break; 419 } 420 break; 421 case UCALL_DONE: 422 fprintf(stderr, "UCALL_DONE\n"); 423 goto done; 424 default: 425 TEST_FAIL("Unknown ucall %lu", uc.cmd); 426 } 427 428 state = vcpu_save_state(vm, VCPU_ID); 429 memset(®s1, 0, sizeof(regs1)); 430 vcpu_regs_get(vm, VCPU_ID, ®s1); 431 432 kvm_vm_release(vm); 433 434 /* Restore state in a new VM. */ 435 kvm_vm_restart(vm, O_RDWR); 436 vm_vcpu_add(vm, VCPU_ID); 437 vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); 438 vcpu_load_state(vm, VCPU_ID, state); 439 run = vcpu_state(vm, VCPU_ID); 440 kvm_x86_state_cleanup(state); 441 442 memset(®s2, 0, sizeof(regs2)); 443 vcpu_regs_get(vm, VCPU_ID, ®s2); 444 TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), 445 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", 446 (ulong) regs2.rdi, (ulong) regs2.rsi); 447 } 448 done: 449 kvm_vm_free(vm); 450 } 451