1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 ARM Limited. 4 */ 5 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <stddef.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <sys/auxv.h> 14 #include <sys/prctl.h> 15 #include <asm/hwcap.h> 16 #include <asm/sigcontext.h> 17 #include <asm/unistd.h> 18 19 #include "../../kselftest.h" 20 21 #include "syscall-abi.h" 22 23 #define NUM_VL ((SVE_VQ_MAX - SVE_VQ_MIN) + 1) 24 25 static int default_sme_vl; 26 27 extern void do_syscall(int sve_vl, int sme_vl); 28 29 static void fill_random(void *buf, size_t size) 30 { 31 int i; 32 uint32_t *lbuf = buf; 33 34 /* random() returns a 32 bit number regardless of the size of long */ 35 for (i = 0; i < size / sizeof(uint32_t); i++) 36 lbuf[i] = random(); 37 } 38 39 /* 40 * We also repeat the test for several syscalls to try to expose different 41 * behaviour. 42 */ 43 static struct syscall_cfg { 44 int syscall_nr; 45 const char *name; 46 } syscalls[] = { 47 { __NR_getpid, "getpid()" }, 48 { __NR_sched_yield, "sched_yield()" }, 49 }; 50 51 #define NUM_GPR 31 52 uint64_t gpr_in[NUM_GPR]; 53 uint64_t gpr_out[NUM_GPR]; 54 55 static void setup_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 56 uint64_t svcr) 57 { 58 fill_random(gpr_in, sizeof(gpr_in)); 59 gpr_in[8] = cfg->syscall_nr; 60 memset(gpr_out, 0, sizeof(gpr_out)); 61 } 62 63 static int check_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, uint64_t svcr) 64 { 65 int errors = 0; 66 int i; 67 68 /* 69 * GPR x0-x7 may be clobbered, and all others should be preserved. 70 */ 71 for (i = 9; i < ARRAY_SIZE(gpr_in); i++) { 72 if (gpr_in[i] != gpr_out[i]) { 73 ksft_print_msg("%s SVE VL %d mismatch in GPR %d: %llx != %llx\n", 74 cfg->name, sve_vl, i, 75 gpr_in[i], gpr_out[i]); 76 errors++; 77 } 78 } 79 80 return errors; 81 } 82 83 #define NUM_FPR 32 84 uint64_t fpr_in[NUM_FPR * 2]; 85 uint64_t fpr_out[NUM_FPR * 2]; 86 87 static void setup_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 88 uint64_t svcr) 89 { 90 fill_random(fpr_in, sizeof(fpr_in)); 91 memset(fpr_out, 0, sizeof(fpr_out)); 92 } 93 94 static int check_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 95 uint64_t svcr) 96 { 97 int errors = 0; 98 int i; 99 100 if (!sve_vl) { 101 for (i = 0; i < ARRAY_SIZE(fpr_in); i++) { 102 if (fpr_in[i] != fpr_out[i]) { 103 ksft_print_msg("%s Q%d/%d mismatch %llx != %llx\n", 104 cfg->name, 105 i / 2, i % 2, 106 fpr_in[i], fpr_out[i]); 107 errors++; 108 } 109 } 110 } 111 112 return errors; 113 } 114 115 #define SVE_Z_SHARED_BYTES (128 / 8) 116 117 static uint8_t z_zero[__SVE_ZREG_SIZE(SVE_VQ_MAX)]; 118 uint8_t z_in[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; 119 uint8_t z_out[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; 120 121 static void setup_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 122 uint64_t svcr) 123 { 124 fill_random(z_in, sizeof(z_in)); 125 fill_random(z_out, sizeof(z_out)); 126 } 127 128 static int check_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 129 uint64_t svcr) 130 { 131 size_t reg_size = sve_vl; 132 int errors = 0; 133 int i; 134 135 if (!sve_vl) 136 return 0; 137 138 for (i = 0; i < SVE_NUM_ZREGS; i++) { 139 uint8_t *in = &z_in[reg_size * i]; 140 uint8_t *out = &z_out[reg_size * i]; 141 142 if (svcr & SVCR_SM_MASK) { 143 /* 144 * In streaming mode the whole register should 145 * be cleared by the transition out of 146 * streaming mode. 147 */ 148 if (memcmp(z_zero, out, reg_size) != 0) { 149 ksft_print_msg("%s SVE VL %d Z%d non-zero\n", 150 cfg->name, sve_vl, i); 151 errors++; 152 } 153 } else { 154 /* 155 * For standard SVE the low 128 bits should be 156 * preserved and any additional bits cleared. 157 */ 158 if (memcmp(in, out, SVE_Z_SHARED_BYTES) != 0) { 159 ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n", 160 cfg->name, sve_vl, i); 161 errors++; 162 } 163 164 if (reg_size > SVE_Z_SHARED_BYTES && 165 (memcmp(z_zero, out + SVE_Z_SHARED_BYTES, 166 reg_size - SVE_Z_SHARED_BYTES) != 0)) { 167 ksft_print_msg("%s SVE VL %d Z%d high bits non-zero\n", 168 cfg->name, sve_vl, i); 169 errors++; 170 } 171 } 172 } 173 174 return errors; 175 } 176 177 uint8_t p_in[SVE_NUM_PREGS * __SVE_PREG_SIZE(SVE_VQ_MAX)]; 178 uint8_t p_out[SVE_NUM_PREGS * __SVE_PREG_SIZE(SVE_VQ_MAX)]; 179 180 static void setup_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 181 uint64_t svcr) 182 { 183 fill_random(p_in, sizeof(p_in)); 184 fill_random(p_out, sizeof(p_out)); 185 } 186 187 static int check_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 188 uint64_t svcr) 189 { 190 size_t reg_size = sve_vq_from_vl(sve_vl) * 2; /* 1 bit per VL byte */ 191 192 int errors = 0; 193 int i; 194 195 if (!sve_vl) 196 return 0; 197 198 /* After a syscall the P registers should be zeroed */ 199 for (i = 0; i < SVE_NUM_PREGS * reg_size; i++) 200 if (p_out[i]) 201 errors++; 202 if (errors) 203 ksft_print_msg("%s SVE VL %d predicate registers non-zero\n", 204 cfg->name, sve_vl); 205 206 return errors; 207 } 208 209 uint8_t ffr_in[__SVE_PREG_SIZE(SVE_VQ_MAX)]; 210 uint8_t ffr_out[__SVE_PREG_SIZE(SVE_VQ_MAX)]; 211 212 static void setup_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 213 uint64_t svcr) 214 { 215 /* 216 * If we are in streaming mode and do not have FA64 then FFR 217 * is unavailable. 218 */ 219 if ((svcr & SVCR_SM_MASK) && 220 !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)) { 221 memset(&ffr_in, 0, sizeof(ffr_in)); 222 return; 223 } 224 225 /* 226 * It is only valid to set a contiguous set of bits starting 227 * at 0. For now since we're expecting this to be cleared by 228 * a syscall just set all bits. 229 */ 230 memset(ffr_in, 0xff, sizeof(ffr_in)); 231 fill_random(ffr_out, sizeof(ffr_out)); 232 } 233 234 static int check_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 235 uint64_t svcr) 236 { 237 size_t reg_size = sve_vq_from_vl(sve_vl) * 2; /* 1 bit per VL byte */ 238 int errors = 0; 239 int i; 240 241 if (!sve_vl) 242 return 0; 243 244 if ((svcr & SVCR_SM_MASK) && 245 !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)) 246 return 0; 247 248 /* After a syscall FFR should be zeroed */ 249 for (i = 0; i < reg_size; i++) 250 if (ffr_out[i]) 251 errors++; 252 if (errors) 253 ksft_print_msg("%s SVE VL %d FFR non-zero\n", 254 cfg->name, sve_vl); 255 256 return errors; 257 } 258 259 uint64_t svcr_in, svcr_out; 260 261 static void setup_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 262 uint64_t svcr) 263 { 264 svcr_in = svcr; 265 } 266 267 static int check_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 268 uint64_t svcr) 269 { 270 int errors = 0; 271 272 if (svcr_out & SVCR_SM_MASK) { 273 ksft_print_msg("%s Still in SM, SVCR %llx\n", 274 cfg->name, svcr_out); 275 errors++; 276 } 277 278 if ((svcr_in & SVCR_ZA_MASK) != (svcr_out & SVCR_ZA_MASK)) { 279 ksft_print_msg("%s PSTATE.ZA changed, SVCR %llx != %llx\n", 280 cfg->name, svcr_in, svcr_out); 281 errors++; 282 } 283 284 return errors; 285 } 286 287 uint8_t za_in[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; 288 uint8_t za_out[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; 289 290 static void setup_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 291 uint64_t svcr) 292 { 293 fill_random(za_in, sizeof(za_in)); 294 memset(za_out, 0, sizeof(za_out)); 295 } 296 297 static int check_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 298 uint64_t svcr) 299 { 300 size_t reg_size = sme_vl * sme_vl; 301 int errors = 0; 302 303 if (!(svcr & SVCR_ZA_MASK)) 304 return 0; 305 306 if (memcmp(za_in, za_out, reg_size) != 0) { 307 ksft_print_msg("SME VL %d ZA does not match\n", sme_vl); 308 errors++; 309 } 310 311 return errors; 312 } 313 314 uint8_t zt_in[ZT_SIG_REG_BYTES] __attribute__((aligned(16))); 315 uint8_t zt_out[ZT_SIG_REG_BYTES] __attribute__((aligned(16))); 316 317 static void setup_zt(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 318 uint64_t svcr) 319 { 320 fill_random(zt_in, sizeof(zt_in)); 321 memset(zt_out, 0, sizeof(zt_out)); 322 } 323 324 static int check_zt(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 325 uint64_t svcr) 326 { 327 int errors = 0; 328 329 if (!(getauxval(AT_HWCAP2) & HWCAP2_SME2)) 330 return 0; 331 332 if (!(svcr & SVCR_ZA_MASK)) 333 return 0; 334 335 if (memcmp(zt_in, zt_out, sizeof(zt_in)) != 0) { 336 ksft_print_msg("SME VL %d ZT does not match\n", sme_vl); 337 errors++; 338 } 339 340 return errors; 341 } 342 343 typedef void (*setup_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 344 uint64_t svcr); 345 typedef int (*check_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 346 uint64_t svcr); 347 348 /* 349 * Each set of registers has a setup function which is called before 350 * the syscall to fill values in a global variable for loading by the 351 * test code and a check function which validates that the results are 352 * as expected. Vector lengths are passed everywhere, a vector length 353 * of 0 should be treated as do not test. 354 */ 355 static struct { 356 setup_fn setup; 357 check_fn check; 358 } regset[] = { 359 { setup_gpr, check_gpr }, 360 { setup_fpr, check_fpr }, 361 { setup_z, check_z }, 362 { setup_p, check_p }, 363 { setup_ffr, check_ffr }, 364 { setup_svcr, check_svcr }, 365 { setup_za, check_za }, 366 { setup_zt, check_zt }, 367 }; 368 369 static bool do_test(struct syscall_cfg *cfg, int sve_vl, int sme_vl, 370 uint64_t svcr) 371 { 372 int errors = 0; 373 int i; 374 375 for (i = 0; i < ARRAY_SIZE(regset); i++) 376 regset[i].setup(cfg, sve_vl, sme_vl, svcr); 377 378 do_syscall(sve_vl, sme_vl); 379 380 for (i = 0; i < ARRAY_SIZE(regset); i++) 381 errors += regset[i].check(cfg, sve_vl, sme_vl, svcr); 382 383 return errors == 0; 384 } 385 386 static void test_one_syscall(struct syscall_cfg *cfg) 387 { 388 int sve_vq, sve_vl; 389 int sme_vq, sme_vl; 390 391 /* FPSIMD only case */ 392 ksft_test_result(do_test(cfg, 0, default_sme_vl, 0), 393 "%s FPSIMD\n", cfg->name); 394 395 if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) 396 return; 397 398 for (sve_vq = SVE_VQ_MAX; sve_vq > 0; --sve_vq) { 399 sve_vl = prctl(PR_SVE_SET_VL, sve_vq * 16); 400 if (sve_vl == -1) 401 ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n", 402 strerror(errno), errno); 403 404 sve_vl &= PR_SVE_VL_LEN_MASK; 405 406 if (sve_vq != sve_vq_from_vl(sve_vl)) 407 sve_vq = sve_vq_from_vl(sve_vl); 408 409 ksft_test_result(do_test(cfg, sve_vl, default_sme_vl, 0), 410 "%s SVE VL %d\n", cfg->name, sve_vl); 411 412 if (!(getauxval(AT_HWCAP2) & HWCAP2_SME)) 413 continue; 414 415 for (sme_vq = SVE_VQ_MAX; sme_vq > 0; --sme_vq) { 416 sme_vl = prctl(PR_SME_SET_VL, sme_vq * 16); 417 if (sme_vl == -1) 418 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n", 419 strerror(errno), errno); 420 421 sme_vl &= PR_SME_VL_LEN_MASK; 422 423 if (sme_vq != sve_vq_from_vl(sme_vl)) 424 sme_vq = sve_vq_from_vl(sme_vl); 425 426 ksft_test_result(do_test(cfg, sve_vl, sme_vl, 427 SVCR_ZA_MASK | SVCR_SM_MASK), 428 "%s SVE VL %d/SME VL %d SM+ZA\n", 429 cfg->name, sve_vl, sme_vl); 430 ksft_test_result(do_test(cfg, sve_vl, sme_vl, 431 SVCR_SM_MASK), 432 "%s SVE VL %d/SME VL %d SM\n", 433 cfg->name, sve_vl, sme_vl); 434 ksft_test_result(do_test(cfg, sve_vl, sme_vl, 435 SVCR_ZA_MASK), 436 "%s SVE VL %d/SME VL %d ZA\n", 437 cfg->name, sve_vl, sme_vl); 438 } 439 } 440 } 441 442 int sve_count_vls(void) 443 { 444 unsigned int vq; 445 int vl_count = 0; 446 int vl; 447 448 if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) 449 return 0; 450 451 /* 452 * Enumerate up to SVE_VQ_MAX vector lengths 453 */ 454 for (vq = SVE_VQ_MAX; vq > 0; --vq) { 455 vl = prctl(PR_SVE_SET_VL, vq * 16); 456 if (vl == -1) 457 ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n", 458 strerror(errno), errno); 459 460 vl &= PR_SVE_VL_LEN_MASK; 461 462 if (vq != sve_vq_from_vl(vl)) 463 vq = sve_vq_from_vl(vl); 464 465 vl_count++; 466 } 467 468 return vl_count; 469 } 470 471 int sme_count_vls(void) 472 { 473 unsigned int vq; 474 int vl_count = 0; 475 int vl; 476 477 if (!(getauxval(AT_HWCAP2) & HWCAP2_SME)) 478 return 0; 479 480 /* Ensure we configure a SME VL, used to flag if SVCR is set */ 481 default_sme_vl = 16; 482 483 /* 484 * Enumerate up to SVE_VQ_MAX vector lengths 485 */ 486 for (vq = SVE_VQ_MAX; vq > 0; --vq) { 487 vl = prctl(PR_SME_SET_VL, vq * 16); 488 if (vl == -1) 489 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n", 490 strerror(errno), errno); 491 492 vl &= PR_SME_VL_LEN_MASK; 493 494 if (vq != sve_vq_from_vl(vl)) 495 vq = sve_vq_from_vl(vl); 496 497 vl_count++; 498 } 499 500 return vl_count; 501 } 502 503 int main(void) 504 { 505 int i; 506 int tests = 1; /* FPSIMD */ 507 int sme_ver; 508 509 srandom(getpid()); 510 511 ksft_print_header(); 512 tests += sve_count_vls(); 513 tests += (sve_count_vls() * sme_count_vls()) * 3; 514 ksft_set_plan(ARRAY_SIZE(syscalls) * tests); 515 516 if (getauxval(AT_HWCAP2) & HWCAP2_SME2) 517 sme_ver = 2; 518 else 519 sme_ver = 1; 520 521 if (getauxval(AT_HWCAP2) & HWCAP2_SME_FA64) 522 ksft_print_msg("SME%d with FA64\n", sme_ver); 523 else if (getauxval(AT_HWCAP2) & HWCAP2_SME) 524 ksft_print_msg("SME%d without FA64\n", sme_ver); 525 526 for (i = 0; i < ARRAY_SIZE(syscalls); i++) 527 test_one_syscall(&syscalls[i]); 528 529 ksft_print_cnts(); 530 531 return 0; 532 } 533