1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * amx tests 4 * 5 * Copyright (C) 2021, Intel, Inc. 6 * 7 * Tests for amx #NM exception and save/restore. 8 */ 9 10 #define _GNU_SOURCE /* for program_invocation_short_name */ 11 #include <fcntl.h> 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <string.h> 15 #include <sys/ioctl.h> 16 #include <sys/syscall.h> 17 18 #include "test_util.h" 19 20 #include "kvm_util.h" 21 #include "processor.h" 22 #include "vmx.h" 23 24 #ifndef __x86_64__ 25 # error This test is 64-bit only 26 #endif 27 28 #define NUM_TILES 8 29 #define TILE_SIZE 1024 30 #define XSAVE_SIZE ((NUM_TILES * TILE_SIZE) + PAGE_SIZE) 31 32 /* Tile configuration associated: */ 33 #define MAX_TILES 16 34 #define RESERVED_BYTES 14 35 36 #define XFEATURE_XTILECFG 17 37 #define XFEATURE_XTILEDATA 18 38 #define XFEATURE_MASK_XTILECFG (1 << XFEATURE_XTILECFG) 39 #define XFEATURE_MASK_XTILEDATA (1 << XFEATURE_XTILEDATA) 40 #define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILECFG | XFEATURE_MASK_XTILEDATA) 41 42 #define TILE_CPUID 0x1d 43 #define XSTATE_CPUID 0xd 44 #define TILE_PALETTE_CPUID_SUBLEAVE 0x1 45 #define XSTATE_USER_STATE_SUBLEAVE 0x0 46 47 #define XSAVE_HDR_OFFSET 512 48 49 struct xsave_data { 50 u8 area[XSAVE_SIZE]; 51 } __aligned(64); 52 53 struct tile_config { 54 u8 palette_id; 55 u8 start_row; 56 u8 reserved[RESERVED_BYTES]; 57 u16 colsb[MAX_TILES]; 58 u8 rows[MAX_TILES]; 59 }; 60 61 struct tile_data { 62 u8 data[NUM_TILES * TILE_SIZE]; 63 }; 64 65 struct xtile_info { 66 u16 bytes_per_tile; 67 u16 bytes_per_row; 68 u16 max_names; 69 u16 max_rows; 70 u32 xsave_offset; 71 u32 xsave_size; 72 }; 73 74 static struct xtile_info xtile; 75 76 static inline u64 __xgetbv(u32 index) 77 { 78 u32 eax, edx; 79 80 asm volatile("xgetbv;" 81 : "=a" (eax), "=d" (edx) 82 : "c" (index)); 83 return eax + ((u64)edx << 32); 84 } 85 86 static inline void __xsetbv(u32 index, u64 value) 87 { 88 u32 eax = value; 89 u32 edx = value >> 32; 90 91 asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); 92 } 93 94 static inline void __ldtilecfg(void *cfg) 95 { 96 asm volatile(".byte 0xc4,0xe2,0x78,0x49,0x00" 97 : : "a"(cfg)); 98 } 99 100 static inline void __tileloadd(void *tile) 101 { 102 asm volatile(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10" 103 : : "a"(tile), "d"(0)); 104 } 105 106 static inline void __tilerelease(void) 107 { 108 asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::); 109 } 110 111 static inline void __xsavec(struct xsave_data *data, uint64_t rfbm) 112 { 113 uint32_t rfbm_lo = rfbm; 114 uint32_t rfbm_hi = rfbm >> 32; 115 116 asm volatile("xsavec (%%rdi)" 117 : : "D" (data), "a" (rfbm_lo), "d" (rfbm_hi) 118 : "memory"); 119 } 120 121 static inline void check_cpuid_xsave(void) 122 { 123 GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE)); 124 GUEST_ASSERT(this_cpu_has(X86_FEATURE_OSXSAVE)); 125 } 126 127 static bool check_xsave_supports_xtile(void) 128 { 129 return __xgetbv(0) & XFEATURE_MASK_XTILE; 130 } 131 132 static bool enum_xtile_config(void) 133 { 134 u32 eax, ebx, ecx, edx; 135 136 __cpuid(TILE_CPUID, TILE_PALETTE_CPUID_SUBLEAVE, &eax, &ebx, &ecx, &edx); 137 if (!eax || !ebx || !ecx) 138 return false; 139 140 xtile.max_names = ebx >> 16; 141 if (xtile.max_names < NUM_TILES) 142 return false; 143 144 xtile.bytes_per_tile = eax >> 16; 145 if (xtile.bytes_per_tile < TILE_SIZE) 146 return false; 147 148 xtile.bytes_per_row = ebx; 149 xtile.max_rows = ecx; 150 151 return true; 152 } 153 154 static bool enum_xsave_tile(void) 155 { 156 u32 eax, ebx, ecx, edx; 157 158 __cpuid(XSTATE_CPUID, XFEATURE_XTILEDATA, &eax, &ebx, &ecx, &edx); 159 if (!eax || !ebx) 160 return false; 161 162 xtile.xsave_offset = ebx; 163 xtile.xsave_size = eax; 164 165 return true; 166 } 167 168 static bool check_xsave_size(void) 169 { 170 u32 eax, ebx, ecx, edx; 171 bool valid = false; 172 173 __cpuid(XSTATE_CPUID, XSTATE_USER_STATE_SUBLEAVE, &eax, &ebx, &ecx, &edx); 174 if (ebx && ebx <= XSAVE_SIZE) 175 valid = true; 176 177 return valid; 178 } 179 180 static bool check_xtile_info(void) 181 { 182 bool ret = false; 183 184 if (!check_xsave_size()) 185 return ret; 186 187 if (!enum_xsave_tile()) 188 return ret; 189 190 if (!enum_xtile_config()) 191 return ret; 192 193 if (sizeof(struct tile_data) >= xtile.xsave_size) 194 ret = true; 195 196 return ret; 197 } 198 199 static void set_tilecfg(struct tile_config *cfg) 200 { 201 int i; 202 203 /* Only palette id 1 */ 204 cfg->palette_id = 1; 205 for (i = 0; i < xtile.max_names; i++) { 206 cfg->colsb[i] = xtile.bytes_per_row; 207 cfg->rows[i] = xtile.max_rows; 208 } 209 } 210 211 static void set_xstatebv(void *data, uint64_t bv) 212 { 213 *(uint64_t *)(data + XSAVE_HDR_OFFSET) = bv; 214 } 215 216 static u64 get_xstatebv(void *data) 217 { 218 return *(u64 *)(data + XSAVE_HDR_OFFSET); 219 } 220 221 static void init_regs(void) 222 { 223 uint64_t cr4, xcr0; 224 225 /* turn on CR4.OSXSAVE */ 226 cr4 = get_cr4(); 227 cr4 |= X86_CR4_OSXSAVE; 228 set_cr4(cr4); 229 230 xcr0 = __xgetbv(0); 231 xcr0 |= XFEATURE_MASK_XTILE; 232 __xsetbv(0x0, xcr0); 233 } 234 235 static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg, 236 struct tile_data *tiledata, 237 struct xsave_data *xsave_data) 238 { 239 init_regs(); 240 check_cpuid_xsave(); 241 GUEST_ASSERT(check_xsave_supports_xtile()); 242 GUEST_ASSERT(check_xtile_info()); 243 244 /* check xtile configs */ 245 GUEST_ASSERT(xtile.xsave_offset == 2816); 246 GUEST_ASSERT(xtile.xsave_size == 8192); 247 GUEST_ASSERT(xtile.max_names == 8); 248 GUEST_ASSERT(xtile.bytes_per_tile == 1024); 249 GUEST_ASSERT(xtile.bytes_per_row == 64); 250 GUEST_ASSERT(xtile.max_rows == 16); 251 GUEST_SYNC(1); 252 253 /* xfd=0, enable amx */ 254 wrmsr(MSR_IA32_XFD, 0); 255 GUEST_SYNC(2); 256 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0); 257 set_tilecfg(amx_cfg); 258 __ldtilecfg(amx_cfg); 259 GUEST_SYNC(3); 260 /* Check save/restore when trap to userspace */ 261 __tileloadd(tiledata); 262 GUEST_SYNC(4); 263 __tilerelease(); 264 GUEST_SYNC(5); 265 /* bit 18 not in the XCOMP_BV after xsavec() */ 266 set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA); 267 __xsavec(xsave_data, XFEATURE_MASK_XTILEDATA); 268 GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0); 269 270 /* xfd=0x40000, disable amx tiledata */ 271 wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA); 272 GUEST_SYNC(6); 273 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA); 274 set_tilecfg(amx_cfg); 275 __ldtilecfg(amx_cfg); 276 /* Trigger #NM exception */ 277 __tileloadd(tiledata); 278 GUEST_SYNC(10); 279 280 GUEST_DONE(); 281 } 282 283 void guest_nm_handler(struct ex_regs *regs) 284 { 285 /* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */ 286 GUEST_SYNC(7); 287 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA); 288 GUEST_SYNC(8); 289 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA); 290 /* Clear xfd_err */ 291 wrmsr(MSR_IA32_XFD_ERR, 0); 292 /* xfd=0, enable amx */ 293 wrmsr(MSR_IA32_XFD, 0); 294 GUEST_SYNC(9); 295 } 296 297 int main(int argc, char *argv[]) 298 { 299 struct kvm_regs regs1, regs2; 300 struct kvm_vcpu *vcpu; 301 struct kvm_vm *vm; 302 struct kvm_run *run; 303 struct kvm_x86_state *state; 304 int xsave_restore_size; 305 vm_vaddr_t amx_cfg, tiledata, xsavedata; 306 struct ucall uc; 307 u32 amx_offset; 308 int stage, ret; 309 310 vm_xsave_require_permission(XSTATE_XTILE_DATA_BIT); 311 312 /* Create VM */ 313 vm = vm_create_with_one_vcpu(&vcpu, guest_code); 314 315 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XSAVE)); 316 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_AMX_TILE)); 317 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILECFG)); 318 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILEDATA)); 319 320 /* Get xsave/restore max size */ 321 xsave_restore_size = kvm_get_supported_cpuid_entry(0xd)->ecx; 322 323 run = vcpu->run; 324 vcpu_regs_get(vcpu, ®s1); 325 326 /* Register #NM handler */ 327 vm_init_descriptor_tables(vm); 328 vcpu_init_descriptor_tables(vcpu); 329 vm_install_exception_handler(vm, NM_VECTOR, guest_nm_handler); 330 331 /* amx cfg for guest_code */ 332 amx_cfg = vm_vaddr_alloc_page(vm); 333 memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize()); 334 335 /* amx tiledata for guest_code */ 336 tiledata = vm_vaddr_alloc_pages(vm, 2); 337 memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize()); 338 339 /* xsave data for guest_code */ 340 xsavedata = vm_vaddr_alloc_pages(vm, 3); 341 memset(addr_gva2hva(vm, xsavedata), 0, 3 * getpagesize()); 342 vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xsavedata); 343 344 for (stage = 1; ; stage++) { 345 vcpu_run(vcpu); 346 TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, 347 "Stage %d: unexpected exit reason: %u (%s),\n", 348 stage, run->exit_reason, 349 exit_reason_str(run->exit_reason)); 350 351 switch (get_ucall(vcpu, &uc)) { 352 case UCALL_ABORT: 353 REPORT_GUEST_ASSERT(uc); 354 /* NOT REACHED */ 355 case UCALL_SYNC: 356 switch (uc.args[1]) { 357 case 1: 358 case 2: 359 case 3: 360 case 5: 361 case 6: 362 case 7: 363 case 8: 364 fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]); 365 break; 366 case 4: 367 case 10: 368 fprintf(stderr, 369 "GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]); 370 371 /* Compacted mode, get amx offset by xsave area 372 * size subtract 8K amx size. 373 */ 374 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE; 375 state = vcpu_save_state(vcpu); 376 void *amx_start = (void *)state->xsave + amx_offset; 377 void *tiles_data = (void *)addr_gva2hva(vm, tiledata); 378 /* Only check TMM0 register, 1 tile */ 379 ret = memcmp(amx_start, tiles_data, TILE_SIZE); 380 TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret); 381 kvm_x86_state_cleanup(state); 382 break; 383 case 9: 384 fprintf(stderr, 385 "GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]); 386 break; 387 } 388 break; 389 case UCALL_DONE: 390 fprintf(stderr, "UCALL_DONE\n"); 391 goto done; 392 default: 393 TEST_FAIL("Unknown ucall %lu", uc.cmd); 394 } 395 396 state = vcpu_save_state(vcpu); 397 memset(®s1, 0, sizeof(regs1)); 398 vcpu_regs_get(vcpu, ®s1); 399 400 kvm_vm_release(vm); 401 402 /* Restore state in a new VM. */ 403 vcpu = vm_recreate_with_one_vcpu(vm); 404 vcpu_load_state(vcpu, state); 405 run = vcpu->run; 406 kvm_x86_state_cleanup(state); 407 408 memset(®s2, 0, sizeof(regs2)); 409 vcpu_regs_get(vcpu, ®s2); 410 TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), 411 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", 412 (ulong) regs2.rdi, (ulong) regs2.rsi); 413 } 414 done: 415 kvm_vm_free(vm); 416 } 417