1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 ARM Limited. 4 */ 5 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <stddef.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <sys/auxv.h> 14 #include <sys/prctl.h> 15 #include <asm/hwcap.h> 16 #include <asm/sigcontext.h> 17 #include <asm/unistd.h> 18 19 #include "../../kselftest.h" 20 21 #include "syscall-abi.h" 22 23 /* 24 * The kernel defines a much larger SVE_VQ_MAX than is expressable in 25 * the architecture, this creates a *lot* of overhead filling the 26 * buffers (especially ZA) on emulated platforms so use the actual 27 * architectural maximum instead. 28 */ 29 #define ARCH_SVE_VQ_MAX 16 30 31 static int default_sme_vl; 32 33 static int sve_vl_count; 34 static unsigned int sve_vls[ARCH_SVE_VQ_MAX]; 35 static int sme_vl_count; 36 static unsigned int sme_vls[ARCH_SVE_VQ_MAX]; 37 38 extern void do_syscall(int sve_vl, int sme_vl); 39 fill_random(void * buf,size_t size)40 static void fill_random(void *buf, size_t size) 41 { 42 int i; 43 uint32_t *lbuf = buf; 44 45 /* random() returns a 32 bit number regardless of the size of long */ 46 for (i = 0; i < size / sizeof(uint32_t); i++) 47 lbuf[i] = random(); 48 } 49 50 /* 51 * We also repeat the test for several syscalls to try to expose different 52 * behaviour. 53 */ 54 static struct syscall_cfg { 55 int syscall_nr; 56 const char *name; 57 } syscalls[] = { 58 { __NR_getpid, "getpid()" }, 59 { __NR_sched_yield, "sched_yield()" }, 60 }; 61 62 #define NUM_GPR 31 63 uint64_t gpr_in[NUM_GPR]; 64 uint64_t gpr_out[NUM_GPR]; 65 setup_gpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)66 static void setup_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 67 uint64_t svcr) 68 { 69 fill_random(gpr_in, sizeof(gpr_in)); 70 gpr_in[8] = cfg->syscall_nr; 71 memset(gpr_out, 0, sizeof(gpr_out)); 72 } 73 check_gpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)74 static int check_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, uint64_t svcr) 75 { 76 int errors = 0; 77 int i; 78 79 /* 80 * GPR x0-x7 may be clobbered, and all others should be preserved. 81 */ 82 for (i = 9; i < ARRAY_SIZE(gpr_in); i++) { 83 if (gpr_in[i] != gpr_out[i]) { 84 ksft_print_msg("%s SVE VL %d mismatch in GPR %d: %llx != %llx\n", 85 cfg->name, sve_vl, i, 86 gpr_in[i], gpr_out[i]); 87 errors++; 88 } 89 } 90 91 return errors; 92 } 93 94 #define NUM_FPR 32 95 uint64_t fpr_in[NUM_FPR * 2]; 96 uint64_t fpr_out[NUM_FPR * 2]; 97 uint64_t fpr_zero[NUM_FPR * 2]; 98 setup_fpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)99 static void setup_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 100 uint64_t svcr) 101 { 102 fill_random(fpr_in, sizeof(fpr_in)); 103 memset(fpr_out, 0, sizeof(fpr_out)); 104 } 105 check_fpr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)106 static int check_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 107 uint64_t svcr) 108 { 109 int errors = 0; 110 int i; 111 112 if (!sve_vl && !(svcr & SVCR_SM_MASK)) { 113 for (i = 0; i < ARRAY_SIZE(fpr_in); i++) { 114 if (fpr_in[i] != fpr_out[i]) { 115 ksft_print_msg("%s Q%d/%d mismatch %llx != %llx\n", 116 cfg->name, 117 i / 2, i % 2, 118 fpr_in[i], fpr_out[i]); 119 errors++; 120 } 121 } 122 } 123 124 /* 125 * In streaming mode the whole register set should be cleared 126 * by the transition out of streaming mode. 127 */ 128 if (svcr & SVCR_SM_MASK) { 129 if (memcmp(fpr_zero, fpr_out, sizeof(fpr_out)) != 0) { 130 ksft_print_msg("%s FPSIMD registers non-zero exiting SM\n", 131 cfg->name); 132 errors++; 133 } 134 } 135 136 return errors; 137 } 138 139 #define SVE_Z_SHARED_BYTES (128 / 8) 140 141 static uint8_t z_zero[__SVE_ZREG_SIZE(ARCH_SVE_VQ_MAX)]; 142 uint8_t z_in[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(ARCH_SVE_VQ_MAX)]; 143 uint8_t z_out[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(ARCH_SVE_VQ_MAX)]; 144 setup_z(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)145 static void setup_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 146 uint64_t svcr) 147 { 148 fill_random(z_in, sizeof(z_in)); 149 fill_random(z_out, sizeof(z_out)); 150 } 151 check_z(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)152 static int check_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 153 uint64_t svcr) 154 { 155 size_t reg_size = sve_vl; 156 int errors = 0; 157 int i; 158 159 if (!sve_vl) 160 return 0; 161 162 for (i = 0; i < SVE_NUM_ZREGS; i++) { 163 uint8_t *in = &z_in[reg_size * i]; 164 uint8_t *out = &z_out[reg_size * i]; 165 166 if (svcr & SVCR_SM_MASK) { 167 /* 168 * In streaming mode the whole register should 169 * be cleared by the transition out of 170 * streaming mode. 171 */ 172 if (memcmp(z_zero, out, reg_size) != 0) { 173 ksft_print_msg("%s SVE VL %d Z%d non-zero\n", 174 cfg->name, sve_vl, i); 175 errors++; 176 } 177 } else { 178 /* 179 * For standard SVE the low 128 bits should be 180 * preserved and any additional bits cleared. 181 */ 182 if (memcmp(in, out, SVE_Z_SHARED_BYTES) != 0) { 183 ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n", 184 cfg->name, sve_vl, i); 185 errors++; 186 } 187 188 if (reg_size > SVE_Z_SHARED_BYTES && 189 (memcmp(z_zero, out + SVE_Z_SHARED_BYTES, 190 reg_size - SVE_Z_SHARED_BYTES) != 0)) { 191 ksft_print_msg("%s SVE VL %d Z%d high bits non-zero\n", 192 cfg->name, sve_vl, i); 193 errors++; 194 } 195 } 196 } 197 198 return errors; 199 } 200 201 uint8_t p_in[SVE_NUM_PREGS * __SVE_PREG_SIZE(ARCH_SVE_VQ_MAX)]; 202 uint8_t p_out[SVE_NUM_PREGS * __SVE_PREG_SIZE(ARCH_SVE_VQ_MAX)]; 203 setup_p(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)204 static void setup_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 205 uint64_t svcr) 206 { 207 fill_random(p_in, sizeof(p_in)); 208 fill_random(p_out, sizeof(p_out)); 209 } 210 check_p(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)211 static int check_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 212 uint64_t svcr) 213 { 214 size_t reg_size = sve_vq_from_vl(sve_vl) * 2; /* 1 bit per VL byte */ 215 216 int errors = 0; 217 int i; 218 219 if (!sve_vl) 220 return 0; 221 222 /* After a syscall the P registers should be zeroed */ 223 for (i = 0; i < SVE_NUM_PREGS * reg_size; i++) 224 if (p_out[i]) 225 errors++; 226 if (errors) 227 ksft_print_msg("%s SVE VL %d predicate registers non-zero\n", 228 cfg->name, sve_vl); 229 230 return errors; 231 } 232 233 uint8_t ffr_in[__SVE_PREG_SIZE(ARCH_SVE_VQ_MAX)]; 234 uint8_t ffr_out[__SVE_PREG_SIZE(ARCH_SVE_VQ_MAX)]; 235 setup_ffr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)236 static void setup_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 237 uint64_t svcr) 238 { 239 /* 240 * If we are in streaming mode and do not have FA64 then FFR 241 * is unavailable. 242 */ 243 if ((svcr & SVCR_SM_MASK) && 244 !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)) { 245 memset(&ffr_in, 0, sizeof(ffr_in)); 246 return; 247 } 248 249 /* 250 * It is only valid to set a contiguous set of bits starting 251 * at 0. For now since we're expecting this to be cleared by 252 * a syscall just set all bits. 253 */ 254 memset(ffr_in, 0xff, sizeof(ffr_in)); 255 fill_random(ffr_out, sizeof(ffr_out)); 256 } 257 check_ffr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)258 static int check_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 259 uint64_t svcr) 260 { 261 size_t reg_size = sve_vq_from_vl(sve_vl) * 2; /* 1 bit per VL byte */ 262 int errors = 0; 263 int i; 264 265 if (!sve_vl) 266 return 0; 267 268 if ((svcr & SVCR_SM_MASK) && 269 !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)) 270 return 0; 271 272 /* After a syscall FFR should be zeroed */ 273 for (i = 0; i < reg_size; i++) 274 if (ffr_out[i]) 275 errors++; 276 if (errors) 277 ksft_print_msg("%s SVE VL %d FFR non-zero\n", 278 cfg->name, sve_vl); 279 280 return errors; 281 } 282 283 uint64_t svcr_in, svcr_out; 284 setup_svcr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)285 static void setup_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 286 uint64_t svcr) 287 { 288 svcr_in = svcr; 289 } 290 check_svcr(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)291 static int check_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 292 uint64_t svcr) 293 { 294 int errors = 0; 295 296 if (svcr_out & SVCR_SM_MASK) { 297 ksft_print_msg("%s Still in SM, SVCR %llx\n", 298 cfg->name, svcr_out); 299 errors++; 300 } 301 302 if ((svcr_in & SVCR_ZA_MASK) != (svcr_out & SVCR_ZA_MASK)) { 303 ksft_print_msg("%s PSTATE.ZA changed, SVCR %llx != %llx\n", 304 cfg->name, svcr_in, svcr_out); 305 errors++; 306 } 307 308 return errors; 309 } 310 311 uint8_t za_in[ZA_SIG_REGS_SIZE(ARCH_SVE_VQ_MAX)]; 312 uint8_t za_out[ZA_SIG_REGS_SIZE(ARCH_SVE_VQ_MAX)]; 313 setup_za(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)314 static void setup_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 315 uint64_t svcr) 316 { 317 fill_random(za_in, sizeof(za_in)); 318 memset(za_out, 0, sizeof(za_out)); 319 } 320 check_za(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)321 static int check_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 322 uint64_t svcr) 323 { 324 size_t reg_size = sme_vl * sme_vl; 325 int errors = 0; 326 327 if (!(svcr & SVCR_ZA_MASK)) 328 return 0; 329 330 if (memcmp(za_in, za_out, reg_size) != 0) { 331 ksft_print_msg("SME VL %d ZA does not match\n", sme_vl); 332 errors++; 333 } 334 335 return errors; 336 } 337 338 uint8_t zt_in[ZT_SIG_REG_BYTES] __attribute__((aligned(16))); 339 uint8_t zt_out[ZT_SIG_REG_BYTES] __attribute__((aligned(16))); 340 setup_zt(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)341 static void setup_zt(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 342 uint64_t svcr) 343 { 344 fill_random(zt_in, sizeof(zt_in)); 345 memset(zt_out, 0, sizeof(zt_out)); 346 } 347 check_zt(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)348 static int check_zt(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 349 uint64_t svcr) 350 { 351 int errors = 0; 352 353 if (!(getauxval(AT_HWCAP2) & HWCAP2_SME2)) 354 return 0; 355 356 if (!(svcr & SVCR_ZA_MASK)) 357 return 0; 358 359 if (memcmp(zt_in, zt_out, sizeof(zt_in)) != 0) { 360 ksft_print_msg("SME VL %d ZT does not match\n", sme_vl); 361 errors++; 362 } 363 364 return errors; 365 } 366 367 typedef void (*setup_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 368 uint64_t svcr); 369 typedef int (*check_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 370 uint64_t svcr); 371 372 /* 373 * Each set of registers has a setup function which is called before 374 * the syscall to fill values in a global variable for loading by the 375 * test code and a check function which validates that the results are 376 * as expected. Vector lengths are passed everywhere, a vector length 377 * of 0 should be treated as do not test. 378 */ 379 static struct { 380 setup_fn setup; 381 check_fn check; 382 } regset[] = { 383 { setup_gpr, check_gpr }, 384 { setup_fpr, check_fpr }, 385 { setup_z, check_z }, 386 { setup_p, check_p }, 387 { setup_ffr, check_ffr }, 388 { setup_svcr, check_svcr }, 389 { setup_za, check_za }, 390 { setup_zt, check_zt }, 391 }; 392 do_test(struct syscall_cfg * cfg,int sve_vl,int sme_vl,uint64_t svcr)393 static bool do_test(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 394 uint64_t svcr) 395 { 396 int errors = 0; 397 int i; 398 399 for (i = 0; i < ARRAY_SIZE(regset); i++) 400 regset[i].setup(cfg, sve_vl, sme_vl, svcr); 401 402 do_syscall(sve_vl, sme_vl); 403 404 for (i = 0; i < ARRAY_SIZE(regset); i++) 405 errors += regset[i].check(cfg, sve_vl, sme_vl, svcr); 406 407 return errors == 0; 408 } 409 test_one_syscall(struct syscall_cfg * cfg)410 static void test_one_syscall(struct syscall_cfg *cfg) 411 { 412 int sve, sme; 413 int ret; 414 415 /* FPSIMD only case */ 416 ksft_test_result(do_test(cfg, 0, default_sme_vl, 0), 417 "%s FPSIMD\n", cfg->name); 418 419 for (sve = 0; sve < sve_vl_count; sve++) { 420 ret = prctl(PR_SVE_SET_VL, sve_vls[sve]); 421 if (ret == -1) 422 ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n", 423 strerror(errno), errno); 424 425 ksft_test_result(do_test(cfg, sve_vls[sve], default_sme_vl, 0), 426 "%s SVE VL %d\n", cfg->name, sve_vls[sve]); 427 428 for (sme = 0; sme < sme_vl_count; sme++) { 429 ret = prctl(PR_SME_SET_VL, sme_vls[sme]); 430 if (ret == -1) 431 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n", 432 strerror(errno), errno); 433 434 ksft_test_result(do_test(cfg, sve_vls[sve], 435 sme_vls[sme], 436 SVCR_ZA_MASK | SVCR_SM_MASK), 437 "%s SVE VL %d/SME VL %d SM+ZA\n", 438 cfg->name, sve_vls[sve], 439 sme_vls[sme]); 440 ksft_test_result(do_test(cfg, sve_vls[sve], 441 sme_vls[sme], SVCR_SM_MASK), 442 "%s SVE VL %d/SME VL %d SM\n", 443 cfg->name, sve_vls[sve], 444 sme_vls[sme]); 445 ksft_test_result(do_test(cfg, sve_vls[sve], 446 sme_vls[sme], SVCR_ZA_MASK), 447 "%s SVE VL %d/SME VL %d ZA\n", 448 cfg->name, sve_vls[sve], 449 sme_vls[sme]); 450 } 451 } 452 453 for (sme = 0; sme < sme_vl_count; sme++) { 454 ret = prctl(PR_SME_SET_VL, sme_vls[sme]); 455 if (ret == -1) 456 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n", 457 strerror(errno), errno); 458 459 ksft_test_result(do_test(cfg, 0, sme_vls[sme], 460 SVCR_ZA_MASK | SVCR_SM_MASK), 461 "%s SME VL %d SM+ZA\n", 462 cfg->name, sme_vls[sme]); 463 ksft_test_result(do_test(cfg, 0, sme_vls[sme], SVCR_SM_MASK), 464 "%s SME VL %d SM\n", 465 cfg->name, sme_vls[sme]); 466 ksft_test_result(do_test(cfg, 0, sme_vls[sme], SVCR_ZA_MASK), 467 "%s SME VL %d ZA\n", 468 cfg->name, sme_vls[sme]); 469 } 470 } 471 sve_count_vls(void)472 void sve_count_vls(void) 473 { 474 unsigned int vq; 475 int vl; 476 477 if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) 478 return; 479 480 /* 481 * Enumerate up to ARCH_SVE_VQ_MAX vector lengths 482 */ 483 for (vq = ARCH_SVE_VQ_MAX; vq > 0; vq /= 2) { 484 vl = prctl(PR_SVE_SET_VL, vq * 16); 485 if (vl == -1) 486 ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n", 487 strerror(errno), errno); 488 489 vl &= PR_SVE_VL_LEN_MASK; 490 491 if (vq != sve_vq_from_vl(vl)) 492 vq = sve_vq_from_vl(vl); 493 494 sve_vls[sve_vl_count++] = vl; 495 } 496 } 497 sme_count_vls(void)498 void sme_count_vls(void) 499 { 500 unsigned int vq; 501 int vl; 502 503 if (!(getauxval(AT_HWCAP2) & HWCAP2_SME)) 504 return; 505 506 /* 507 * Enumerate up to ARCH_SVE_VQ_MAX vector lengths 508 */ 509 for (vq = ARCH_SVE_VQ_MAX; vq > 0; vq /= 2) { 510 vl = prctl(PR_SME_SET_VL, vq * 16); 511 if (vl == -1) 512 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n", 513 strerror(errno), errno); 514 515 vl &= PR_SME_VL_LEN_MASK; 516 517 /* Found lowest VL */ 518 if (sve_vq_from_vl(vl) > vq) 519 break; 520 521 if (vq != sve_vq_from_vl(vl)) 522 vq = sve_vq_from_vl(vl); 523 524 sme_vls[sme_vl_count++] = vl; 525 } 526 527 /* Ensure we configure a SME VL, used to flag if SVCR is set */ 528 default_sme_vl = sme_vls[0]; 529 } 530 main(void)531 int main(void) 532 { 533 int i; 534 int tests = 1; /* FPSIMD */ 535 int sme_ver; 536 537 srandom(getpid()); 538 539 ksft_print_header(); 540 541 sve_count_vls(); 542 sme_count_vls(); 543 544 tests += sve_vl_count; 545 tests += sme_vl_count * 3; 546 tests += (sve_vl_count * sme_vl_count) * 3; 547 ksft_set_plan(ARRAY_SIZE(syscalls) * tests); 548 549 if (getauxval(AT_HWCAP2) & HWCAP2_SME2) 550 sme_ver = 2; 551 else 552 sme_ver = 1; 553 554 if (getauxval(AT_HWCAP2) & HWCAP2_SME_FA64) 555 ksft_print_msg("SME%d with FA64\n", sme_ver); 556 else if (getauxval(AT_HWCAP2) & HWCAP2_SME) 557 ksft_print_msg("SME%d without FA64\n", sme_ver); 558 559 for (i = 0; i < ARRAY_SIZE(syscalls); i++) 560 test_one_syscall(&syscalls[i]); 561 562 ksft_print_cnts(); 563 564 return 0; 565 } 566