1// SPDX-License-Identifier: GPL-2.0-only 2// Copyright (C) 2015-2019 ARM Limited. 3// Original author: Dave Martin <Dave.Martin@arm.com> 4// 5// Simple Scalable Vector Extension context switch test 6// Repeatedly writes unique test patterns into each SVE register 7// and reads them back to verify integrity. 8// 9// for x in `seq 1 NR_CPUS`; do sve-test & pids=$pids\ $! ; done 10// (leave it running for as long as you want...) 11// kill $pids 12 13#include <asm/unistd.h> 14#include "assembler.h" 15#include "asm-offsets.h" 16 17#define NZR 32 18#define NPR 16 19#define MAXVL_B (2048 / 8) 20 21.arch_extension sve 22 23.macro _sve_ldr_v zt, xn 24 ldr z\zt, [x\xn] 25.endm 26 27.macro _sve_str_v zt, xn 28 str z\zt, [x\xn] 29.endm 30 31.macro _sve_ldr_p pt, xn 32 ldr p\pt, [x\xn] 33.endm 34 35.macro _sve_str_p pt, xn 36 str p\pt, [x\xn] 37.endm 38 39// Generate accessor functions to read/write programmatically selected 40// SVE registers. 41// x0 is the register index to access 42// x1 is the memory address to read from (getz,setp) or store to (setz,setp) 43// All clobber x0-x2 44define_accessor setz, NZR, _sve_ldr_v 45define_accessor getz, NZR, _sve_str_v 46define_accessor setp, NPR, _sve_ldr_p 47define_accessor getp, NPR, _sve_str_p 48 49// Print a single character x0 to stdout 50// Clobbers x0-x2,x8 51function putc 52 str x0, [sp, #-16]! 53 54 mov x0, #1 // STDOUT_FILENO 55 mov x1, sp 56 mov x2, #1 57 mov x8, #__NR_write 58 svc #0 59 60 add sp, sp, #16 61 ret 62endfunction 63 64// Print a NUL-terminated string starting at address x0 to stdout 65// Clobbers x0-x3,x8 66function puts 67 mov x1, x0 68 69 mov x2, #0 700: ldrb w3, [x0], #1 71 cbz w3, 1f 72 add x2, x2, #1 73 b 0b 74 751: mov w0, #1 // STDOUT_FILENO 76 mov x8, #__NR_write 77 svc #0 78 79 ret 80endfunction 81 82// Utility macro to print a literal string 83// Clobbers x0-x4,x8 84.macro puts string 85 .pushsection .rodata.str1.1, "aMS", 1 86.L__puts_literal\@: .string "\string" 87 .popsection 88 89 ldr x0, =.L__puts_literal\@ 90 bl puts 91.endm 92 93// Print an unsigned decimal number x0 to stdout 94// Clobbers x0-x4,x8 95function putdec 96 mov x1, sp 97 str x30, [sp, #-32]! // Result can't be > 20 digits 98 99 mov x2, #0 100 strb w2, [x1, #-1]! // Write the NUL terminator 101 102 mov x2, #10 1030: udiv x3, x0, x2 // div-mod loop to generate the digits 104 msub x0, x3, x2, x0 105 add w0, w0, #'0' 106 strb w0, [x1, #-1]! 107 mov x0, x3 108 cbnz x3, 0b 109 110 ldrb w0, [x1] 111 cbnz w0, 1f 112 mov w0, #'0' // Print "0" for 0, not "" 113 strb w0, [x1, #-1]! 114 1151: mov x0, x1 116 bl puts 117 118 ldr x30, [sp], #32 119 ret 120endfunction 121 122// Print an unsigned decimal number x0 to stdout, followed by a newline 123// Clobbers x0-x5,x8 124function putdecn 125 mov x5, x30 126 127 bl putdec 128 mov x0, #'\n' 129 bl putc 130 131 ret x5 132endfunction 133 134// Clobbers x0-x3,x8 135function puthexb 136 str x30, [sp, #-0x10]! 137 138 mov w3, w0 139 lsr w0, w0, #4 140 bl puthexnibble 141 mov w0, w3 142 143 ldr x30, [sp], #0x10 144 // fall through to puthexnibble 145endfunction 146// Clobbers x0-x2,x8 147function puthexnibble 148 and w0, w0, #0xf 149 cmp w0, #10 150 blo 1f 151 add w0, w0, #'a' - ('9' + 1) 1521: add w0, w0, #'0' 153 b putc 154endfunction 155 156// x0=data in, x1=size in, clobbers x0-x5,x8 157function dumphex 158 str x30, [sp, #-0x10]! 159 160 mov x4, x0 161 mov x5, x1 162 1630: subs x5, x5, #1 164 b.lo 1f 165 ldrb w0, [x4], #1 166 bl puthexb 167 b 0b 168 1691: ldr x30, [sp], #0x10 170 ret 171endfunction 172 173// Declare some storate space to shadow the SVE register contents: 174.pushsection .text 175.data 176.align 4 177zref: 178 .space MAXVL_B * NZR 179pref: 180 .space MAXVL_B / 8 * NPR 181ffrref: 182 .space MAXVL_B / 8 183scratch: 184 .space MAXVL_B 185.popsection 186 187// Trivial memory copy: copy x2 bytes, starting at address x1, to address x0. 188// Clobbers x0-x3 189function memcpy 190 cmp x2, #0 191 b.eq 1f 1920: ldrb w3, [x1], #1 193 strb w3, [x0], #1 194 subs x2, x2, #1 195 b.ne 0b 1961: ret 197endfunction 198 199// Generate a test pattern for storage in SVE registers 200// x0: pid (16 bits) 201// x1: register number (6 bits) 202// x2: generation (4 bits) 203 204// These values are used to constuct a 32-bit pattern that is repeated in the 205// scratch buffer as many times as will fit: 206// bits 31:28 generation number (increments once per test_loop) 207// bits 27:22 32-bit lane index 208// bits 21:16 register number 209// bits 15: 0 pid 210 211function pattern 212 orr w1, w0, w1, lsl #16 213 orr w2, w1, w2, lsl #28 214 215 ldr x0, =scratch 216 mov w1, #MAXVL_B / 4 217 2180: str w2, [x0], #4 219 add w2, w2, #(1 << 22) 220 subs w1, w1, #1 221 bne 0b 222 223 ret 224endfunction 225 226// Get the address of shadow data for SVE Z-register Z<xn> 227.macro _adrz xd, xn, nrtmp 228 ldr \xd, =zref 229 rdvl x\nrtmp, #1 230 madd \xd, x\nrtmp, \xn, \xd 231.endm 232 233// Get the address of shadow data for SVE P-register P<xn - NZR> 234.macro _adrp xd, xn, nrtmp 235 ldr \xd, =pref 236 rdvl x\nrtmp, #1 237 lsr x\nrtmp, x\nrtmp, #3 238 sub \xn, \xn, #NZR 239 madd \xd, x\nrtmp, \xn, \xd 240.endm 241 242// Set up test pattern in a SVE Z-register 243// x0: pid 244// x1: register number 245// x2: generation 246function setup_zreg 247 mov x4, x30 248 249 mov x6, x1 250 bl pattern 251 _adrz x0, x6, 2 252 mov x5, x0 253 ldr x1, =scratch 254 bl memcpy 255 256 mov x0, x6 257 mov x1, x5 258 bl setz 259 260 ret x4 261endfunction 262 263// Set up test pattern in a SVE P-register 264// x0: pid 265// x1: register number 266// x2: generation 267function setup_preg 268 mov x4, x30 269 270 mov x6, x1 271 bl pattern 272 _adrp x0, x6, 2 273 mov x5, x0 274 ldr x1, =scratch 275 bl memcpy 276 277 mov x0, x6 278 mov x1, x5 279 bl setp 280 281 ret x4 282endfunction 283 284// Set up test pattern in the FFR 285// x0: pid 286// x2: generation 287// Beware: corrupts P0. 288function setup_ffr 289 mov x4, x30 290 291 bl pattern 292 ldr x0, =ffrref 293 ldr x1, =scratch 294 rdvl x2, #1 295 lsr x2, x2, #3 296 bl memcpy 297 298 mov x0, #0 299 ldr x1, =ffrref 300 bl setp 301 302 wrffr p0.b 303 304 ret x4 305endfunction 306 307// Fill x1 bytes starting at x0 with 0xae (for canary purposes) 308// Clobbers x1, x2. 309function memfill_ae 310 mov w2, #0xae 311 b memfill 312endfunction 313 314// Fill x1 bytes starting at x0 with 0. 315// Clobbers x1, x2. 316function memclr 317 mov w2, #0 318endfunction 319 // fall through to memfill 320 321// Trivial memory fill: fill x1 bytes starting at address x0 with byte w2 322// Clobbers x1 323function memfill 324 cmp x1, #0 325 b.eq 1f 326 3270: strb w2, [x0], #1 328 subs x1, x1, #1 329 b.ne 0b 330 3311: ret 332endfunction 333 334// Trivial memory compare: compare x2 bytes starting at address x0 with 335// bytes starting at address x1. 336// Returns only if all bytes match; otherwise, the program is aborted. 337// Clobbers x0-x5. 338function memcmp 339 cbz x2, 2f 340 341 stp x0, x1, [sp, #-0x20]! 342 str x2, [sp, #0x10] 343 344 mov x5, #0 3450: ldrb w3, [x0, x5] 346 ldrb w4, [x1, x5] 347 add x5, x5, #1 348 cmp w3, w4 349 b.ne 1f 350 subs x2, x2, #1 351 b.ne 0b 352 3531: ldr x2, [sp, #0x10] 354 ldp x0, x1, [sp], #0x20 355 b.ne barf 356 3572: ret 358endfunction 359 360// Verify that a SVE Z-register matches its shadow in memory, else abort 361// x0: reg number 362// Clobbers x0-x7. 363function check_zreg 364 mov x3, x30 365 366 _adrz x5, x0, 6 367 mov x4, x0 368 ldr x7, =scratch 369 370 mov x0, x7 371 mov x1, x6 372 bl memfill_ae 373 374 mov x0, x4 375 mov x1, x7 376 bl getz 377 378 mov x0, x5 379 mov x1, x7 380 mov x2, x6 381 mov x30, x3 382 b memcmp 383endfunction 384 385// Verify that a SVE P-register matches its shadow in memory, else abort 386// x0: reg number 387// Clobbers x0-x7. 388function check_preg 389 mov x3, x30 390 391 _adrp x5, x0, 6 392 mov x4, x0 393 ldr x7, =scratch 394 395 mov x0, x7 396 mov x1, x6 397 bl memfill_ae 398 399 mov x0, x4 400 mov x1, x7 401 bl getp 402 403 mov x0, x5 404 mov x1, x7 405 mov x2, x6 406 mov x30, x3 407 b memcmp 408endfunction 409 410// Verify that the FFR matches its shadow in memory, else abort 411// Beware -- corrupts P0. 412// Clobbers x0-x5. 413function check_ffr 414 mov x3, x30 415 416 ldr x4, =scratch 417 rdvl x5, #1 418 lsr x5, x5, #3 419 420 mov x0, x4 421 mov x1, x5 422 bl memfill_ae 423 424 rdffr p0.b 425 mov x0, #0 426 mov x1, x4 427 bl getp 428 429 ldr x0, =ffrref 430 mov x1, x4 431 mov x2, x5 432 mov x30, x3 433 b memcmp 434endfunction 435 436// Any SVE register modified here can cause corruption in the main 437// thread -- but *only* the registers modified here. 438function irritator_handler 439 // Increment the irritation signal count (x23): 440 ldr x0, [x2, #ucontext_regs + 8 * 23] 441 add x0, x0, #1 442 str x0, [x2, #ucontext_regs + 8 * 23] 443 444 // Corrupt some random Z-regs 445 adr x0, .text + (irritator_handler - .text) / 16 * 16 446 movi v0.8b, #1 447 movi v9.16b, #2 448 movi v31.8b, #3 449 // And P0 450 rdffr p0.b 451 // And FFR 452 wrffr p15.b 453 454 ret 455endfunction 456 457function terminate_handler 458 mov w21, w0 459 mov x20, x2 460 461 puts "Terminated by signal " 462 mov w0, w21 463 bl putdec 464 puts ", no error, iterations=" 465 ldr x0, [x20, #ucontext_regs + 8 * 22] 466 bl putdec 467 puts ", signals=" 468 ldr x0, [x20, #ucontext_regs + 8 * 23] 469 bl putdecn 470 471 mov x0, #0 472 mov x8, #__NR_exit 473 svc #0 474endfunction 475 476// w0: signal number 477// x1: sa_action 478// w2: sa_flags 479// Clobbers x0-x6,x8 480function setsignal 481 str x30, [sp, #-((sa_sz + 15) / 16 * 16 + 16)]! 482 483 mov w4, w0 484 mov x5, x1 485 mov w6, w2 486 487 add x0, sp, #16 488 mov x1, #sa_sz 489 bl memclr 490 491 mov w0, w4 492 add x1, sp, #16 493 str w6, [x1, #sa_flags] 494 str x5, [x1, #sa_handler] 495 mov x2, #0 496 mov x3, #sa_mask_sz 497 mov x8, #__NR_rt_sigaction 498 svc #0 499 500 cbz w0, 1f 501 502 puts "sigaction failure\n" 503 b .Labort 504 5051: ldr x30, [sp], #((sa_sz + 15) / 16 * 16 + 16) 506 ret 507endfunction 508 509// Main program entry point 510.globl _start 511function _start 512_start: 513 // Sanity-check and report the vector length 514 515 rdvl x19, #8 516 cmp x19, #128 517 b.lo 1f 518 cmp x19, #2048 519 b.hi 1f 520 tst x19, #(8 - 1) 521 b.eq 2f 522 5231: puts "Bad vector length: " 524 mov x0, x19 525 bl putdecn 526 b .Labort 527 5282: puts "Vector length:\t" 529 mov x0, x19 530 bl putdec 531 puts " bits\n" 532 533 // Obtain our PID, to ensure test pattern uniqueness between processes 534 535 mov x8, #__NR_getpid 536 svc #0 537 mov x20, x0 538 539 puts "PID:\t" 540 mov x0, x20 541 bl putdecn 542 543 mov x23, #0 // Irritation signal count 544 545 mov w0, #SIGINT 546 adr x1, terminate_handler 547 mov w2, #SA_SIGINFO 548 bl setsignal 549 550 mov w0, #SIGTERM 551 adr x1, terminate_handler 552 mov w2, #SA_SIGINFO 553 bl setsignal 554 555 mov w0, #SIGUSR1 556 adr x1, irritator_handler 557 mov w2, #SA_SIGINFO 558 orr w2, w2, #SA_NODEFER 559 bl setsignal 560 561 mov x22, #0 // generation number, increments per iteration 562.Ltest_loop: 563 rdvl x0, #8 564 cmp x0, x19 565 b.ne vl_barf 566 567 mov x21, #0 // Set up Z-regs & shadow with test pattern 5680: mov x0, x20 569 mov x1, x21 570 and x2, x22, #0xf 571 bl setup_zreg 572 add x21, x21, #1 573 cmp x21, #NZR 574 b.lo 0b 575 576 mov x0, x20 // Set up FFR & shadow with test pattern 577 mov x1, #NZR + NPR 578 and x2, x22, #0xf 579 bl setup_ffr 580 5810: mov x0, x20 // Set up P-regs & shadow with test pattern 582 mov x1, x21 583 and x2, x22, #0xf 584 bl setup_preg 585 add x21, x21, #1 586 cmp x21, #NZR + NPR 587 b.lo 0b 588 589// Can't do this when SVE state is volatile across SVC: 590// mov x8, #__NR_sched_yield // Encourage preemption 591// svc #0 592 593 mov x21, #0 5940: mov x0, x21 595 bl check_zreg 596 add x21, x21, #1 597 cmp x21, #NZR 598 b.lo 0b 599 6000: mov x0, x21 601 bl check_preg 602 add x21, x21, #1 603 cmp x21, #NZR + NPR 604 b.lo 0b 605 606 bl check_ffr 607 608 add x22, x22, #1 609 b .Ltest_loop 610 611.Labort: 612 mov x0, #0 613 mov x1, #SIGABRT 614 mov x8, #__NR_kill 615 svc #0 616endfunction 617 618function barf 619// fpsimd.c acitivty log dump hack 620// ldr w0, =0xdeadc0de 621// mov w8, #__NR_exit 622// svc #0 623// end hack 624 mov x10, x0 // expected data 625 mov x11, x1 // actual data 626 mov x12, x2 // data size 627 628 puts "Mismatch: PID=" 629 mov x0, x20 630 bl putdec 631 puts ", iteration=" 632 mov x0, x22 633 bl putdec 634 puts ", reg=" 635 mov x0, x21 636 bl putdecn 637 puts "\tExpected [" 638 mov x0, x10 639 mov x1, x12 640 bl dumphex 641 puts "]\n\tGot [" 642 mov x0, x11 643 mov x1, x12 644 bl dumphex 645 puts "]\n" 646 647 mov x8, #__NR_getpid 648 svc #0 649// fpsimd.c acitivty log dump hack 650// ldr w0, =0xdeadc0de 651// mov w8, #__NR_exit 652// svc #0 653// ^ end of hack 654 mov x1, #SIGABRT 655 mov x8, #__NR_kill 656 svc #0 657// mov x8, #__NR_exit 658// mov x1, #1 659// svc #0 660endfunction 661 662function vl_barf 663 mov x10, x0 664 665 puts "Bad active VL: " 666 mov x0, x10 667 bl putdecn 668 669 mov x8, #__NR_exit 670 mov x1, #1 671 svc #0 672endfunction 673