1 /* 2 * AArch64 specific helpers 3 * 4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de> 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/units.h" 22 #include "cpu.h" 23 #include "gdbstub/helpers.h" 24 #include "exec/helper-proto.h" 25 #include "qemu/host-utils.h" 26 #include "qemu/log.h" 27 #include "qemu/main-loop.h" 28 #include "qemu/bitops.h" 29 #include "internals.h" 30 #include "qemu/crc32c.h" 31 #include "exec/cpu-common.h" 32 #include "accel/tcg/cpu-ldst.h" 33 #include "accel/tcg/helper-retaddr.h" 34 #include "accel/tcg/probe.h" 35 #include "exec/target_page.h" 36 #include "exec/tlb-flags.h" 37 #include "qemu/int128.h" 38 #include "qemu/atomic128.h" 39 #include "fpu/softfloat.h" 40 #include <zlib.h> /* for crc32 */ 41 #ifdef CONFIG_USER_ONLY 42 #include "user/page-protection.h" 43 #endif 44 #include "vec_internal.h" 45 46 /* C2.4.7 Multiply and divide */ 47 /* special cases for 0 and LLONG_MIN are mandated by the standard */ 48 uint64_t HELPER(udiv64)(uint64_t num, uint64_t den) 49 { 50 if (den == 0) { 51 return 0; 52 } 53 return num / den; 54 } 55 56 int64_t HELPER(sdiv64)(int64_t num, int64_t den) 57 { 58 if (den == 0) { 59 return 0; 60 } 61 if (num == LLONG_MIN && den == -1) { 62 return LLONG_MIN; 63 } 64 return num / den; 65 } 66 67 uint64_t HELPER(rbit64)(uint64_t x) 68 { 69 return revbit64(x); 70 } 71 72 void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm) 73 { 74 update_spsel(env, imm); 75 } 76 77 void HELPER(msr_set_allint_el1)(CPUARMState *env) 78 { 79 /* ALLINT update to PSTATE. */ 80 if (arm_hcrx_el2_eff(env) & HCRX_TALLINT) { 81 raise_exception_ra(env, EXCP_UDEF, 82 syn_aa64_sysregtrap(0, 1, 0, 4, 1, 0x1f, 0), 2, 83 GETPC()); 84 } 85 86 env->pstate |= PSTATE_ALLINT; 87 } 88 89 static void daif_check(CPUARMState *env, uint32_t op, 90 uint32_t imm, uintptr_t ra) 91 { 92 /* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */ 93 if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { 94 raise_exception_ra(env, EXCP_UDEF, 95 syn_aa64_sysregtrap(0, extract32(op, 0, 3), 96 extract32(op, 3, 3), 4, 97 imm, 0x1f, 0), 98 exception_target_el(env), ra); 99 } 100 } 101 102 void HELPER(msr_i_daifset)(CPUARMState *env, uint32_t imm) 103 { 104 daif_check(env, 0x1e, imm, GETPC()); 105 env->daif |= (imm << 6) & PSTATE_DAIF; 106 arm_rebuild_hflags(env); 107 } 108 109 void HELPER(msr_i_daifclear)(CPUARMState *env, uint32_t imm) 110 { 111 daif_check(env, 0x1f, imm, GETPC()); 112 env->daif &= ~((imm << 6) & PSTATE_DAIF); 113 arm_rebuild_hflags(env); 114 } 115 116 /* Convert a softfloat float_relation_ (as returned by 117 * the float*_compare functions) to the correct ARM 118 * NZCV flag state. 119 */ 120 static inline uint32_t float_rel_to_flags(int res) 121 { 122 uint64_t flags; 123 switch (res) { 124 case float_relation_equal: 125 flags = PSTATE_Z | PSTATE_C; 126 break; 127 case float_relation_less: 128 flags = PSTATE_N; 129 break; 130 case float_relation_greater: 131 flags = PSTATE_C; 132 break; 133 case float_relation_unordered: 134 default: 135 flags = PSTATE_C | PSTATE_V; 136 break; 137 } 138 return flags; 139 } 140 141 uint64_t HELPER(vfp_cmph_a64)(uint32_t x, uint32_t y, float_status *fp_status) 142 { 143 return float_rel_to_flags(float16_compare_quiet(x, y, fp_status)); 144 } 145 146 uint64_t HELPER(vfp_cmpeh_a64)(uint32_t x, uint32_t y, float_status *fp_status) 147 { 148 return float_rel_to_flags(float16_compare(x, y, fp_status)); 149 } 150 151 uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, float_status *fp_status) 152 { 153 return float_rel_to_flags(float32_compare_quiet(x, y, fp_status)); 154 } 155 156 uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, float_status *fp_status) 157 { 158 return float_rel_to_flags(float32_compare(x, y, fp_status)); 159 } 160 161 uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, float_status *fp_status) 162 { 163 return float_rel_to_flags(float64_compare_quiet(x, y, fp_status)); 164 } 165 166 uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, float_status *fp_status) 167 { 168 return float_rel_to_flags(float64_compare(x, y, fp_status)); 169 } 170 171 float32 HELPER(vfp_mulxs)(float32 a, float32 b, float_status *fpst) 172 { 173 a = float32_squash_input_denormal(a, fpst); 174 b = float32_squash_input_denormal(b, fpst); 175 176 if ((float32_is_zero(a) && float32_is_infinity(b)) || 177 (float32_is_infinity(a) && float32_is_zero(b))) { 178 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ 179 return make_float32((1U << 30) | 180 ((float32_val(a) ^ float32_val(b)) & (1U << 31))); 181 } 182 return float32_mul(a, b, fpst); 183 } 184 185 float64 HELPER(vfp_mulxd)(float64 a, float64 b, float_status *fpst) 186 { 187 a = float64_squash_input_denormal(a, fpst); 188 b = float64_squash_input_denormal(b, fpst); 189 190 if ((float64_is_zero(a) && float64_is_infinity(b)) || 191 (float64_is_infinity(a) && float64_is_zero(b))) { 192 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ 193 return make_float64((1ULL << 62) | 194 ((float64_val(a) ^ float64_val(b)) & (1ULL << 63))); 195 } 196 return float64_mul(a, b, fpst); 197 } 198 199 /* 64bit/double versions of the neon float compare functions */ 200 uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, float_status *fpst) 201 { 202 return -float64_eq_quiet(a, b, fpst); 203 } 204 205 uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, float_status *fpst) 206 { 207 return -float64_le(b, a, fpst); 208 } 209 210 uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, float_status *fpst) 211 { 212 return -float64_lt(b, a, fpst); 213 } 214 215 /* 216 * Reciprocal step and sqrt step. Note that unlike the A32/T32 217 * versions, these do a fully fused multiply-add or 218 * multiply-add-and-halve. 219 * The FPCR.AH == 1 versions need to avoid flipping the sign of NaN. 220 */ 221 #define DO_RECPS(NAME, CTYPE, FLOATTYPE, CHSFN) \ 222 CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ 223 { \ 224 a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ 225 b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ 226 a = FLOATTYPE ## _ ## CHSFN(a); \ 227 if ((FLOATTYPE ## _is_infinity(a) && FLOATTYPE ## _is_zero(b)) || \ 228 (FLOATTYPE ## _is_infinity(b) && FLOATTYPE ## _is_zero(a))) { \ 229 return FLOATTYPE ## _two; \ 230 } \ 231 return FLOATTYPE ## _muladd(a, b, FLOATTYPE ## _two, 0, fpst); \ 232 } 233 234 DO_RECPS(recpsf_f16, uint32_t, float16, chs) 235 DO_RECPS(recpsf_f32, float32, float32, chs) 236 DO_RECPS(recpsf_f64, float64, float64, chs) 237 DO_RECPS(recpsf_ah_f16, uint32_t, float16, ah_chs) 238 DO_RECPS(recpsf_ah_f32, float32, float32, ah_chs) 239 DO_RECPS(recpsf_ah_f64, float64, float64, ah_chs) 240 241 #define DO_RSQRTSF(NAME, CTYPE, FLOATTYPE, CHSFN) \ 242 CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ 243 { \ 244 a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ 245 b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ 246 a = FLOATTYPE ## _ ## CHSFN(a); \ 247 if ((FLOATTYPE ## _is_infinity(a) && FLOATTYPE ## _is_zero(b)) || \ 248 (FLOATTYPE ## _is_infinity(b) && FLOATTYPE ## _is_zero(a))) { \ 249 return FLOATTYPE ## _one_point_five; \ 250 } \ 251 return FLOATTYPE ## _muladd_scalbn(a, b, FLOATTYPE ## _three, \ 252 -1, 0, fpst); \ 253 } \ 254 255 DO_RSQRTSF(rsqrtsf_f16, uint32_t, float16, chs) 256 DO_RSQRTSF(rsqrtsf_f32, float32, float32, chs) 257 DO_RSQRTSF(rsqrtsf_f64, float64, float64, chs) 258 DO_RSQRTSF(rsqrtsf_ah_f16, uint32_t, float16, ah_chs) 259 DO_RSQRTSF(rsqrtsf_ah_f32, float32, float32, ah_chs) 260 DO_RSQRTSF(rsqrtsf_ah_f64, float64, float64, ah_chs) 261 262 /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */ 263 uint32_t HELPER(frecpx_f16)(uint32_t a, float_status *fpst) 264 { 265 uint16_t val16, sbit; 266 int16_t exp; 267 268 if (float16_is_any_nan(a)) { 269 float16 nan = a; 270 if (float16_is_signaling_nan(a, fpst)) { 271 float_raise(float_flag_invalid, fpst); 272 if (!fpst->default_nan_mode) { 273 nan = float16_silence_nan(a, fpst); 274 } 275 } 276 if (fpst->default_nan_mode) { 277 nan = float16_default_nan(fpst); 278 } 279 return nan; 280 } 281 282 a = float16_squash_input_denormal(a, fpst); 283 284 val16 = float16_val(a); 285 sbit = 0x8000 & val16; 286 exp = extract32(val16, 10, 5); 287 288 if (exp == 0) { 289 return make_float16(deposit32(sbit, 10, 5, 0x1e)); 290 } else { 291 return make_float16(deposit32(sbit, 10, 5, ~exp)); 292 } 293 } 294 295 float32 HELPER(frecpx_f32)(float32 a, float_status *fpst) 296 { 297 uint32_t val32, sbit; 298 int32_t exp; 299 300 if (float32_is_any_nan(a)) { 301 float32 nan = a; 302 if (float32_is_signaling_nan(a, fpst)) { 303 float_raise(float_flag_invalid, fpst); 304 if (!fpst->default_nan_mode) { 305 nan = float32_silence_nan(a, fpst); 306 } 307 } 308 if (fpst->default_nan_mode) { 309 nan = float32_default_nan(fpst); 310 } 311 return nan; 312 } 313 314 a = float32_squash_input_denormal(a, fpst); 315 316 val32 = float32_val(a); 317 sbit = 0x80000000ULL & val32; 318 exp = extract32(val32, 23, 8); 319 320 if (exp == 0) { 321 return make_float32(sbit | (0xfe << 23)); 322 } else { 323 return make_float32(sbit | (~exp & 0xff) << 23); 324 } 325 } 326 327 float64 HELPER(frecpx_f64)(float64 a, float_status *fpst) 328 { 329 uint64_t val64, sbit; 330 int64_t exp; 331 332 if (float64_is_any_nan(a)) { 333 float64 nan = a; 334 if (float64_is_signaling_nan(a, fpst)) { 335 float_raise(float_flag_invalid, fpst); 336 if (!fpst->default_nan_mode) { 337 nan = float64_silence_nan(a, fpst); 338 } 339 } 340 if (fpst->default_nan_mode) { 341 nan = float64_default_nan(fpst); 342 } 343 return nan; 344 } 345 346 a = float64_squash_input_denormal(a, fpst); 347 348 val64 = float64_val(a); 349 sbit = 0x8000000000000000ULL & val64; 350 exp = extract64(float64_val(a), 52, 11); 351 352 if (exp == 0) { 353 return make_float64(sbit | (0x7feULL << 52)); 354 } else { 355 return make_float64(sbit | (~exp & 0x7ffULL) << 52); 356 } 357 } 358 359 float32 HELPER(fcvtx_f64_to_f32)(float64 a, float_status *fpst) 360 { 361 float32 r; 362 int old = get_float_rounding_mode(fpst); 363 364 set_float_rounding_mode(float_round_to_odd, fpst); 365 r = float64_to_float32(a, fpst); 366 set_float_rounding_mode(old, fpst); 367 return r; 368 } 369 370 /* 371 * AH=1 min/max have some odd special cases: 372 * comparing two zeroes (regardless of sign), (NaN, anything), 373 * or (anything, NaN) should return the second argument (possibly 374 * squashed to zero). 375 * Also, denormal outputs are not squashed to zero regardless of FZ or FZ16. 376 */ 377 #define AH_MINMAX_HELPER(NAME, CTYPE, FLOATTYPE, MINMAX) \ 378 CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ 379 { \ 380 bool save; \ 381 CTYPE r; \ 382 a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ 383 b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ 384 if (FLOATTYPE ## _is_zero(a) && FLOATTYPE ## _is_zero(b)) { \ 385 return b; \ 386 } \ 387 if (FLOATTYPE ## _is_any_nan(a) || \ 388 FLOATTYPE ## _is_any_nan(b)) { \ 389 float_raise(float_flag_invalid, fpst); \ 390 return b; \ 391 } \ 392 save = get_flush_to_zero(fpst); \ 393 set_flush_to_zero(false, fpst); \ 394 r = FLOATTYPE ## _ ## MINMAX(a, b, fpst); \ 395 set_flush_to_zero(save, fpst); \ 396 return r; \ 397 } 398 399 AH_MINMAX_HELPER(vfp_ah_minh, dh_ctype_f16, float16, min) 400 AH_MINMAX_HELPER(vfp_ah_mins, float32, float32, min) 401 AH_MINMAX_HELPER(vfp_ah_mind, float64, float64, min) 402 AH_MINMAX_HELPER(vfp_ah_maxh, dh_ctype_f16, float16, max) 403 AH_MINMAX_HELPER(vfp_ah_maxs, float32, float32, max) 404 AH_MINMAX_HELPER(vfp_ah_maxd, float64, float64, max) 405 AH_MINMAX_HELPER(sme2_ah_fmax_b16, bfloat16, bfloat16, max) 406 AH_MINMAX_HELPER(sme2_ah_fmin_b16, bfloat16, bfloat16, min) 407 408 /* 64-bit versions of the CRC helpers. Note that although the operation 409 * (and the prototypes of crc32c() and crc32() mean that only the bottom 410 * 32 bits of the accumulator and result are used, we pass and return 411 * uint64_t for convenience of the generated code. Unlike the 32-bit 412 * instruction set versions, val may genuinely have 64 bits of data in it. 413 * The upper bytes of val (above the number specified by 'bytes') must have 414 * been zeroed out by the caller. 415 */ 416 uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes) 417 { 418 uint8_t buf[8]; 419 420 stq_le_p(buf, val); 421 422 /* zlib crc32 converts the accumulator and output to one's complement. */ 423 return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff; 424 } 425 426 uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes) 427 { 428 uint8_t buf[8]; 429 430 stq_le_p(buf, val); 431 432 /* Linux crc32c converts the output to one's complement. */ 433 return crc32c(acc, buf, bytes) ^ 0xffffffff; 434 } 435 436 /* 437 * AdvSIMD half-precision 438 */ 439 440 #define ADVSIMD_HELPER(name, suffix) HELPER(glue(glue(advsimd_, name), suffix)) 441 442 #define ADVSIMD_HALFOP(name) \ 443 uint32_t ADVSIMD_HELPER(name, h)(uint32_t a, uint32_t b, float_status *fpst) \ 444 { \ 445 return float16_ ## name(a, b, fpst); \ 446 } 447 448 #define ADVSIMD_TWOHALFOP(name) \ 449 uint32_t ADVSIMD_HELPER(name, 2h)(uint32_t two_a, uint32_t two_b, \ 450 float_status *fpst) \ 451 { \ 452 float16 a1, a2, b1, b2; \ 453 uint32_t r1, r2; \ 454 a1 = extract32(two_a, 0, 16); \ 455 a2 = extract32(two_a, 16, 16); \ 456 b1 = extract32(two_b, 0, 16); \ 457 b2 = extract32(two_b, 16, 16); \ 458 r1 = float16_ ## name(a1, b1, fpst); \ 459 r2 = float16_ ## name(a2, b2, fpst); \ 460 return deposit32(r1, 16, 16, r2); \ 461 } 462 463 ADVSIMD_TWOHALFOP(add) 464 ADVSIMD_TWOHALFOP(sub) 465 ADVSIMD_TWOHALFOP(mul) 466 ADVSIMD_TWOHALFOP(div) 467 ADVSIMD_TWOHALFOP(min) 468 ADVSIMD_TWOHALFOP(max) 469 ADVSIMD_TWOHALFOP(minnum) 470 ADVSIMD_TWOHALFOP(maxnum) 471 472 /* Data processing - scalar floating-point and advanced SIMD */ 473 static float16 float16_mulx(float16 a, float16 b, float_status *fpst) 474 { 475 a = float16_squash_input_denormal(a, fpst); 476 b = float16_squash_input_denormal(b, fpst); 477 478 if ((float16_is_zero(a) && float16_is_infinity(b)) || 479 (float16_is_infinity(a) && float16_is_zero(b))) { 480 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ 481 return make_float16((1U << 14) | 482 ((float16_val(a) ^ float16_val(b)) & (1U << 15))); 483 } 484 return float16_mul(a, b, fpst); 485 } 486 487 ADVSIMD_HALFOP(mulx) 488 ADVSIMD_TWOHALFOP(mulx) 489 490 /* fused multiply-accumulate */ 491 uint32_t HELPER(advsimd_muladdh)(uint32_t a, uint32_t b, uint32_t c, 492 float_status *fpst) 493 { 494 return float16_muladd(a, b, c, 0, fpst); 495 } 496 497 uint32_t HELPER(advsimd_muladd2h)(uint32_t two_a, uint32_t two_b, 498 uint32_t two_c, float_status *fpst) 499 { 500 float16 a1, a2, b1, b2, c1, c2; 501 uint32_t r1, r2; 502 a1 = extract32(two_a, 0, 16); 503 a2 = extract32(two_a, 16, 16); 504 b1 = extract32(two_b, 0, 16); 505 b2 = extract32(two_b, 16, 16); 506 c1 = extract32(two_c, 0, 16); 507 c2 = extract32(two_c, 16, 16); 508 r1 = float16_muladd(a1, b1, c1, 0, fpst); 509 r2 = float16_muladd(a2, b2, c2, 0, fpst); 510 return deposit32(r1, 16, 16, r2); 511 } 512 513 /* 514 * Floating point comparisons produce an integer result. Softfloat 515 * routines return float_relation types which we convert to the 0/-1 516 * Neon requires. 517 */ 518 519 #define ADVSIMD_CMPRES(test) (test) ? 0xffff : 0 520 521 uint32_t HELPER(advsimd_ceq_f16)(uint32_t a, uint32_t b, float_status *fpst) 522 { 523 int compare = float16_compare_quiet(a, b, fpst); 524 return ADVSIMD_CMPRES(compare == float_relation_equal); 525 } 526 527 uint32_t HELPER(advsimd_cge_f16)(uint32_t a, uint32_t b, float_status *fpst) 528 { 529 int compare = float16_compare(a, b, fpst); 530 return ADVSIMD_CMPRES(compare == float_relation_greater || 531 compare == float_relation_equal); 532 } 533 534 uint32_t HELPER(advsimd_cgt_f16)(uint32_t a, uint32_t b, float_status *fpst) 535 { 536 int compare = float16_compare(a, b, fpst); 537 return ADVSIMD_CMPRES(compare == float_relation_greater); 538 } 539 540 uint32_t HELPER(advsimd_acge_f16)(uint32_t a, uint32_t b, float_status *fpst) 541 { 542 float16 f0 = float16_abs(a); 543 float16 f1 = float16_abs(b); 544 int compare = float16_compare(f0, f1, fpst); 545 return ADVSIMD_CMPRES(compare == float_relation_greater || 546 compare == float_relation_equal); 547 } 548 549 uint32_t HELPER(advsimd_acgt_f16)(uint32_t a, uint32_t b, float_status *fpst) 550 { 551 float16 f0 = float16_abs(a); 552 float16 f1 = float16_abs(b); 553 int compare = float16_compare(f0, f1, fpst); 554 return ADVSIMD_CMPRES(compare == float_relation_greater); 555 } 556 557 /* round to integral */ 558 uint32_t HELPER(advsimd_rinth_exact)(uint32_t x, float_status *fp_status) 559 { 560 return float16_round_to_int(x, fp_status); 561 } 562 563 uint32_t HELPER(advsimd_rinth)(uint32_t x, float_status *fp_status) 564 { 565 int old_flags = get_float_exception_flags(fp_status), new_flags; 566 float16 ret; 567 568 ret = float16_round_to_int(x, fp_status); 569 570 /* Suppress any inexact exceptions the conversion produced */ 571 if (!(old_flags & float_flag_inexact)) { 572 new_flags = get_float_exception_flags(fp_status); 573 set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status); 574 } 575 576 return ret; 577 } 578 579 static int el_from_spsr(uint32_t spsr) 580 { 581 /* Return the exception level that this SPSR is requesting a return to, 582 * or -1 if it is invalid (an illegal return) 583 */ 584 if (spsr & PSTATE_nRW) { 585 switch (spsr & CPSR_M) { 586 case ARM_CPU_MODE_USR: 587 return 0; 588 case ARM_CPU_MODE_HYP: 589 return 2; 590 case ARM_CPU_MODE_FIQ: 591 case ARM_CPU_MODE_IRQ: 592 case ARM_CPU_MODE_SVC: 593 case ARM_CPU_MODE_ABT: 594 case ARM_CPU_MODE_UND: 595 case ARM_CPU_MODE_SYS: 596 return 1; 597 case ARM_CPU_MODE_MON: 598 /* Returning to Mon from AArch64 is never possible, 599 * so this is an illegal return. 600 */ 601 default: 602 return -1; 603 } 604 } else { 605 if (extract32(spsr, 1, 1)) { 606 /* Return with reserved M[1] bit set */ 607 return -1; 608 } 609 if (extract32(spsr, 0, 4) == 1) { 610 /* return to EL0 with M[0] bit set */ 611 return -1; 612 } 613 return extract32(spsr, 2, 2); 614 } 615 } 616 617 static void cpsr_write_from_spsr_elx(CPUARMState *env, 618 uint32_t val) 619 { 620 uint32_t mask; 621 622 /* Save SPSR_ELx.SS into PSTATE. */ 623 env->pstate = (env->pstate & ~PSTATE_SS) | (val & PSTATE_SS); 624 val &= ~PSTATE_SS; 625 626 /* Move DIT to the correct location for CPSR */ 627 if (val & PSTATE_DIT) { 628 val &= ~PSTATE_DIT; 629 val |= CPSR_DIT; 630 } 631 632 mask = aarch32_cpsr_valid_mask(env->features, \ 633 &env_archcpu(env)->isar); 634 cpsr_write(env, val, mask, CPSRWriteRaw); 635 } 636 637 void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc) 638 { 639 ARMCPU *cpu = env_archcpu(env); 640 int cur_el = arm_current_el(env); 641 unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el); 642 uint32_t spsr = env->banked_spsr[spsr_idx]; 643 int new_el; 644 bool return_to_aa64 = (spsr & PSTATE_nRW) == 0; 645 646 aarch64_save_sp(env, cur_el); 647 648 arm_clear_exclusive(env); 649 650 /* We must squash the PSTATE.SS bit to zero unless both of the 651 * following hold: 652 * 1. debug exceptions are currently disabled 653 * 2. singlestep will be active in the EL we return to 654 * We check 1 here and 2 after we've done the pstate/cpsr write() to 655 * transition to the EL we're going to. 656 */ 657 if (arm_generate_debug_exceptions(env)) { 658 spsr &= ~PSTATE_SS; 659 } 660 661 new_el = el_from_spsr(spsr); 662 if (new_el == -1) { 663 goto illegal_return; 664 } 665 if (new_el > cur_el || (new_el == 2 && !arm_is_el2_enabled(env))) { 666 /* Disallow return to an EL which is unimplemented or higher 667 * than the current one. 668 */ 669 goto illegal_return; 670 } 671 672 /* 673 * FEAT_RME forbids return from EL3 to a lower exception level 674 * with an invalid security state. 675 * We don't need an explicit check for FEAT_RME here because we enforce 676 * in scr_write() that you can't set the NSE bit without it. 677 */ 678 if (cur_el == 3 && new_el < 3 && 679 (env->cp15.scr_el3 & (SCR_NS | SCR_NSE)) == SCR_NSE) { 680 goto illegal_return; 681 } 682 683 if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) { 684 /* Return to an EL which is configured for a different register width */ 685 goto illegal_return; 686 } 687 688 if (!return_to_aa64 && !cpu_isar_feature(aa64_aa32, cpu)) { 689 /* Return to AArch32 when CPU is AArch64-only */ 690 goto illegal_return; 691 } 692 693 if (new_el == 1 && (arm_hcr_el2_eff(env) & HCR_TGE)) { 694 goto illegal_return; 695 } 696 697 bql_lock(); 698 arm_call_pre_el_change_hook(cpu); 699 bql_unlock(); 700 701 if (!return_to_aa64) { 702 env->aarch64 = false; 703 /* We do a raw CPSR write because aarch64_sync_64_to_32() 704 * will sort the register banks out for us, and we've already 705 * caught all the bad-mode cases in el_from_spsr(). 706 */ 707 cpsr_write_from_spsr_elx(env, spsr); 708 if (!arm_singlestep_active(env)) { 709 env->pstate &= ~PSTATE_SS; 710 } 711 aarch64_sync_64_to_32(env); 712 713 if (spsr & CPSR_T) { 714 env->regs[15] = new_pc & ~0x1; 715 } else { 716 env->regs[15] = new_pc & ~0x3; 717 } 718 helper_rebuild_hflags_a32(env, new_el); 719 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to " 720 "AArch32 EL%d PC 0x%" PRIx32 "\n", 721 cur_el, new_el, env->regs[15]); 722 } else { 723 int tbii; 724 725 env->aarch64 = true; 726 spsr &= aarch64_pstate_valid_mask(&cpu->isar); 727 pstate_write(env, spsr); 728 if (!arm_singlestep_active(env)) { 729 env->pstate &= ~PSTATE_SS; 730 } 731 aarch64_restore_sp(env, new_el); 732 helper_rebuild_hflags_a64(env, new_el); 733 734 /* 735 * Apply TBI to the exception return address. We had to delay this 736 * until after we selected the new EL, so that we could select the 737 * correct TBI+TBID bits. This is made easier by waiting until after 738 * the hflags rebuild, since we can pull the composite TBII field 739 * from there. 740 */ 741 tbii = EX_TBFLAG_A64(env->hflags, TBII); 742 if ((tbii >> extract64(new_pc, 55, 1)) & 1) { 743 /* TBI is enabled. */ 744 int core_mmu_idx = arm_env_mmu_index(env); 745 if (regime_has_2_ranges(core_to_aa64_mmu_idx(core_mmu_idx))) { 746 new_pc = sextract64(new_pc, 0, 56); 747 } else { 748 new_pc = extract64(new_pc, 0, 56); 749 } 750 } 751 env->pc = new_pc; 752 753 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to " 754 "AArch64 EL%d PC 0x%" PRIx64 "\n", 755 cur_el, new_el, env->pc); 756 } 757 758 /* 759 * Note that cur_el can never be 0. If new_el is 0, then 760 * el0_a64 is return_to_aa64, else el0_a64 is ignored. 761 */ 762 aarch64_sve_change_el(env, cur_el, new_el, return_to_aa64); 763 764 bql_lock(); 765 arm_call_el_change_hook(cpu); 766 bql_unlock(); 767 768 return; 769 770 illegal_return: 771 /* Illegal return events of various kinds have architecturally 772 * mandated behaviour: 773 * restore NZCV and DAIF from SPSR_ELx 774 * set PSTATE.IL 775 * restore PC from ELR_ELx 776 * no change to exception level, execution state or stack pointer 777 */ 778 env->pstate |= PSTATE_IL; 779 env->pc = new_pc; 780 spsr &= PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT; 781 spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT); 782 pstate_write(env, spsr); 783 if (!arm_singlestep_active(env)) { 784 env->pstate &= ~PSTATE_SS; 785 } 786 helper_rebuild_hflags_a64(env, cur_el); 787 qemu_log_mask(LOG_GUEST_ERROR, "Illegal exception return at EL%d: " 788 "resuming execution at 0x%" PRIx64 "\n", cur_el, env->pc); 789 } 790 791 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in) 792 { 793 uintptr_t ra = GETPC(); 794 795 /* 796 * Implement DC ZVA, which zeroes a fixed-length block of memory. 797 * Note that we do not implement the (architecturally mandated) 798 * alignment fault for attempts to use this on Device memory 799 * (which matches the usual QEMU behaviour of not implementing either 800 * alignment faults or any memory attribute handling). 801 */ 802 int blocklen = 4 << env_archcpu(env)->dcz_blocksize; 803 uint64_t vaddr = vaddr_in & ~(blocklen - 1); 804 int mmu_idx = arm_env_mmu_index(env); 805 void *mem; 806 807 /* 808 * Trapless lookup. In addition to actual invalid page, may 809 * return NULL for I/O, watchpoints, clean pages, etc. 810 */ 811 mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx); 812 813 #ifndef CONFIG_USER_ONLY 814 if (unlikely(!mem)) { 815 /* 816 * Trap if accessing an invalid page. DC_ZVA requires that we supply 817 * the original pointer for an invalid page. But watchpoints require 818 * that we probe the actual space. So do both. 819 */ 820 (void) probe_write(env, vaddr_in, 1, mmu_idx, ra); 821 mem = probe_write(env, vaddr, blocklen, mmu_idx, ra); 822 823 if (unlikely(!mem)) { 824 /* 825 * The only remaining reason for mem == NULL is I/O. 826 * Just do a series of byte writes as the architecture demands. 827 */ 828 for (int i = 0; i < blocklen; i++) { 829 cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra); 830 } 831 return; 832 } 833 } 834 #endif 835 836 set_helper_retaddr(ra); 837 memset(mem, 0, blocklen); 838 clear_helper_retaddr(); 839 } 840 841 void HELPER(unaligned_access)(CPUARMState *env, uint64_t addr, 842 uint32_t access_type, uint32_t mmu_idx) 843 { 844 arm_cpu_do_unaligned_access(env_cpu(env), addr, access_type, 845 mmu_idx, GETPC()); 846 } 847 848 /* Memory operations (memset, memmove, memcpy) */ 849 850 /* 851 * Return true if the CPY* and SET* insns can execute; compare 852 * pseudocode CheckMOPSEnabled(), though we refactor it a little. 853 */ 854 static bool mops_enabled(CPUARMState *env) 855 { 856 int el = arm_current_el(env); 857 858 if (el < 2 && 859 (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE) && 860 !(arm_hcrx_el2_eff(env) & HCRX_MSCEN)) { 861 return false; 862 } 863 864 if (el == 0) { 865 if (!el_is_in_host(env, 0)) { 866 return env->cp15.sctlr_el[1] & SCTLR_MSCEN; 867 } else { 868 return env->cp15.sctlr_el[2] & SCTLR_MSCEN; 869 } 870 } 871 return true; 872 } 873 874 static void check_mops_enabled(CPUARMState *env, uintptr_t ra) 875 { 876 if (!mops_enabled(env)) { 877 raise_exception_ra(env, EXCP_UDEF, syn_uncategorized(), 878 exception_target_el(env), ra); 879 } 880 } 881 882 /* 883 * Return the target exception level for an exception due 884 * to mismatched arguments in a FEAT_MOPS copy or set. 885 * Compare pseudocode MismatchedCpySetTargetEL() 886 */ 887 static int mops_mismatch_exception_target_el(CPUARMState *env) 888 { 889 int el = arm_current_el(env); 890 891 if (el > 1) { 892 return el; 893 } 894 if (el == 0 && (arm_hcr_el2_eff(env) & HCR_TGE)) { 895 return 2; 896 } 897 if (el == 1 && (arm_hcrx_el2_eff(env) & HCRX_MCE2)) { 898 return 2; 899 } 900 return 1; 901 } 902 903 /* 904 * Check whether an M or E instruction was executed with a CF value 905 * indicating the wrong option for this implementation. 906 * Assumes we are always Option A. 907 */ 908 static void check_mops_wrong_option(CPUARMState *env, uint32_t syndrome, 909 uintptr_t ra) 910 { 911 if (env->CF != 0) { 912 syndrome |= 1 << 17; /* Set the wrong-option bit */ 913 raise_exception_ra(env, EXCP_UDEF, syndrome, 914 mops_mismatch_exception_target_el(env), ra); 915 } 916 } 917 918 /* 919 * Return the maximum number of bytes we can transfer starting at addr 920 * without crossing a page boundary. 921 */ 922 static uint64_t page_limit(uint64_t addr) 923 { 924 return TARGET_PAGE_ALIGN(addr + 1) - addr; 925 } 926 927 /* 928 * Return the number of bytes we can copy starting from addr and working 929 * backwards without crossing a page boundary. 930 */ 931 static uint64_t page_limit_rev(uint64_t addr) 932 { 933 return (addr & ~TARGET_PAGE_MASK) + 1; 934 } 935 936 /* 937 * Perform part of a memory set on an area of guest memory starting at 938 * toaddr (a dirty address) and extending for setsize bytes. 939 * 940 * Returns the number of bytes actually set, which might be less than 941 * setsize; the caller should loop until the whole set has been done. 942 * The caller should ensure that the guest registers are correct 943 * for the possibility that the first byte of the set encounters 944 * an exception or watchpoint. We guarantee not to take any faults 945 * for bytes other than the first. 946 */ 947 static uint64_t set_step(CPUARMState *env, uint64_t toaddr, 948 uint64_t setsize, uint32_t data, int memidx, 949 uint32_t *mtedesc, uintptr_t ra) 950 { 951 void *mem; 952 953 setsize = MIN(setsize, page_limit(toaddr)); 954 if (*mtedesc) { 955 uint64_t mtesize = mte_mops_probe(env, toaddr, setsize, *mtedesc); 956 if (mtesize == 0) { 957 /* Trap, or not. All CPU state is up to date */ 958 mte_check_fail(env, *mtedesc, toaddr, ra); 959 /* Continue, with no further MTE checks required */ 960 *mtedesc = 0; 961 } else { 962 /* Advance to the end, or to the tag mismatch */ 963 setsize = MIN(setsize, mtesize); 964 } 965 } 966 967 toaddr = useronly_clean_ptr(toaddr); 968 /* 969 * Trapless lookup: returns NULL for invalid page, I/O, 970 * watchpoints, clean pages, etc. 971 */ 972 mem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, memidx); 973 974 #ifndef CONFIG_USER_ONLY 975 if (unlikely(!mem)) { 976 /* 977 * Slow-path: just do one byte write. This will handle the 978 * watchpoint, invalid page, etc handling correctly. 979 * For clean code pages, the next iteration will see 980 * the page dirty and will use the fast path. 981 */ 982 cpu_stb_mmuidx_ra(env, toaddr, data, memidx, ra); 983 return 1; 984 } 985 #endif 986 /* Easy case: just memset the host memory */ 987 set_helper_retaddr(ra); 988 memset(mem, data, setsize); 989 clear_helper_retaddr(); 990 return setsize; 991 } 992 993 /* 994 * Similar, but setting tags. The architecture requires us to do this 995 * in 16-byte chunks. SETP accesses are not tag checked; they set 996 * the tags. 997 */ 998 static uint64_t set_step_tags(CPUARMState *env, uint64_t toaddr, 999 uint64_t setsize, uint32_t data, int memidx, 1000 uint32_t *mtedesc, uintptr_t ra) 1001 { 1002 void *mem; 1003 uint64_t cleanaddr; 1004 1005 setsize = MIN(setsize, page_limit(toaddr)); 1006 1007 cleanaddr = useronly_clean_ptr(toaddr); 1008 /* 1009 * Trapless lookup: returns NULL for invalid page, I/O, 1010 * watchpoints, clean pages, etc. 1011 */ 1012 mem = tlb_vaddr_to_host(env, cleanaddr, MMU_DATA_STORE, memidx); 1013 1014 #ifndef CONFIG_USER_ONLY 1015 if (unlikely(!mem)) { 1016 /* 1017 * Slow-path: just do one write. This will handle the 1018 * watchpoint, invalid page, etc handling correctly. 1019 * The architecture requires that we do 16 bytes at a time, 1020 * and we know both ptr and size are 16 byte aligned. 1021 * For clean code pages, the next iteration will see 1022 * the page dirty and will use the fast path. 1023 */ 1024 uint64_t repldata = data * 0x0101010101010101ULL; 1025 MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, memidx); 1026 cpu_st16_mmu(env, toaddr, int128_make128(repldata, repldata), oi16, ra); 1027 mte_mops_set_tags(env, toaddr, 16, *mtedesc); 1028 return 16; 1029 } 1030 #endif 1031 /* Easy case: just memset the host memory */ 1032 set_helper_retaddr(ra); 1033 memset(mem, data, setsize); 1034 clear_helper_retaddr(); 1035 mte_mops_set_tags(env, toaddr, setsize, *mtedesc); 1036 return setsize; 1037 } 1038 1039 typedef uint64_t StepFn(CPUARMState *env, uint64_t toaddr, 1040 uint64_t setsize, uint32_t data, 1041 int memidx, uint32_t *mtedesc, uintptr_t ra); 1042 1043 /* Extract register numbers from a MOPS exception syndrome value */ 1044 static int mops_destreg(uint32_t syndrome) 1045 { 1046 return extract32(syndrome, 10, 5); 1047 } 1048 1049 static int mops_srcreg(uint32_t syndrome) 1050 { 1051 return extract32(syndrome, 5, 5); 1052 } 1053 1054 static int mops_sizereg(uint32_t syndrome) 1055 { 1056 return extract32(syndrome, 0, 5); 1057 } 1058 1059 /* 1060 * Return true if TCMA and TBI bits mean we need to do MTE checks. 1061 * We only need to do this once per MOPS insn, not for every page. 1062 */ 1063 static bool mte_checks_needed(uint64_t ptr, uint32_t desc) 1064 { 1065 int bit55 = extract64(ptr, 55, 1); 1066 1067 /* 1068 * Note that tbi_check() returns true for "access checked" but 1069 * tcma_check() returns true for "access unchecked". 1070 */ 1071 if (!tbi_check(desc, bit55)) { 1072 return false; 1073 } 1074 return !tcma_check(desc, bit55, allocation_tag_from_addr(ptr)); 1075 } 1076 1077 /* Take an exception if the SETG addr/size are not granule aligned */ 1078 static void check_setg_alignment(CPUARMState *env, uint64_t ptr, uint64_t size, 1079 uint32_t memidx, uintptr_t ra) 1080 { 1081 if ((size != 0 && !QEMU_IS_ALIGNED(ptr, TAG_GRANULE)) || 1082 !QEMU_IS_ALIGNED(size, TAG_GRANULE)) { 1083 arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE, 1084 memidx, ra); 1085 1086 } 1087 } 1088 1089 static uint64_t arm_reg_or_xzr(CPUARMState *env, int reg) 1090 { 1091 /* 1092 * Runtime equivalent of cpu_reg() -- return the CPU register value, 1093 * for contexts when index 31 means XZR (not SP). 1094 */ 1095 return reg == 31 ? 0 : env->xregs[reg]; 1096 } 1097 1098 /* 1099 * For the Memory Set operation, our implementation chooses 1100 * always to use "option A", where we update Xd to the final 1101 * address in the SETP insn, and set Xn to be -(bytes remaining). 1102 * On SETM and SETE insns we only need update Xn. 1103 * 1104 * @env: CPU 1105 * @syndrome: syndrome value for mismatch exceptions 1106 * (also contains the register numbers we need to use) 1107 * @mtedesc: MTE descriptor word 1108 * @stepfn: function which does a single part of the set operation 1109 * @is_setg: true if this is the tag-setting SETG variant 1110 */ 1111 static void do_setp(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, 1112 StepFn *stepfn, bool is_setg, uintptr_t ra) 1113 { 1114 /* Prologue: we choose to do up to the next page boundary */ 1115 int rd = mops_destreg(syndrome); 1116 int rs = mops_srcreg(syndrome); 1117 int rn = mops_sizereg(syndrome); 1118 uint8_t data = arm_reg_or_xzr(env, rs); 1119 uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); 1120 uint64_t toaddr = env->xregs[rd]; 1121 uint64_t setsize = env->xregs[rn]; 1122 uint64_t stagesetsize, step; 1123 1124 check_mops_enabled(env, ra); 1125 1126 if (setsize > INT64_MAX) { 1127 setsize = INT64_MAX; 1128 if (is_setg) { 1129 setsize &= ~0xf; 1130 } 1131 } 1132 1133 if (unlikely(is_setg)) { 1134 check_setg_alignment(env, toaddr, setsize, memidx, ra); 1135 } else if (!mte_checks_needed(toaddr, mtedesc)) { 1136 mtedesc = 0; 1137 } 1138 1139 stagesetsize = MIN(setsize, page_limit(toaddr)); 1140 while (stagesetsize) { 1141 env->xregs[rd] = toaddr; 1142 env->xregs[rn] = setsize; 1143 step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra); 1144 toaddr += step; 1145 setsize -= step; 1146 stagesetsize -= step; 1147 } 1148 /* Insn completed, so update registers to the Option A format */ 1149 env->xregs[rd] = toaddr + setsize; 1150 env->xregs[rn] = -setsize; 1151 1152 /* Set NZCV = 0000 to indicate we are an Option A implementation */ 1153 env->NF = 0; 1154 env->ZF = 1; /* our env->ZF encoding is inverted */ 1155 env->CF = 0; 1156 env->VF = 0; 1157 } 1158 1159 void HELPER(setp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1160 { 1161 do_setp(env, syndrome, mtedesc, set_step, false, GETPC()); 1162 } 1163 1164 void HELPER(setgp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1165 { 1166 do_setp(env, syndrome, mtedesc, set_step_tags, true, GETPC()); 1167 } 1168 1169 static void do_setm(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, 1170 StepFn *stepfn, bool is_setg, uintptr_t ra) 1171 { 1172 /* Main: we choose to do all the full-page chunks */ 1173 CPUState *cs = env_cpu(env); 1174 int rd = mops_destreg(syndrome); 1175 int rs = mops_srcreg(syndrome); 1176 int rn = mops_sizereg(syndrome); 1177 uint8_t data = arm_reg_or_xzr(env, rs); 1178 uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; 1179 uint64_t setsize = -env->xregs[rn]; 1180 uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); 1181 uint64_t step, stagesetsize; 1182 1183 check_mops_enabled(env, ra); 1184 1185 /* 1186 * We're allowed to NOP out "no data to copy" before the consistency 1187 * checks; we choose to do so. 1188 */ 1189 if (env->xregs[rn] == 0) { 1190 return; 1191 } 1192 1193 check_mops_wrong_option(env, syndrome, ra); 1194 1195 /* 1196 * Our implementation will work fine even if we have an unaligned 1197 * destination address, and because we update Xn every time around 1198 * the loop below and the return value from stepfn() may be less 1199 * than requested, we might find toaddr is unaligned. So we don't 1200 * have an IMPDEF check for alignment here. 1201 */ 1202 1203 if (unlikely(is_setg)) { 1204 check_setg_alignment(env, toaddr, setsize, memidx, ra); 1205 } else if (!mte_checks_needed(toaddr, mtedesc)) { 1206 mtedesc = 0; 1207 } 1208 1209 /* Do the actual memset: we leave the last partial page to SETE */ 1210 stagesetsize = setsize & TARGET_PAGE_MASK; 1211 while (stagesetsize > 0) { 1212 step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra); 1213 toaddr += step; 1214 setsize -= step; 1215 stagesetsize -= step; 1216 env->xregs[rn] = -setsize; 1217 if (stagesetsize > 0 && unlikely(cpu_loop_exit_requested(cs))) { 1218 cpu_loop_exit_restore(cs, ra); 1219 } 1220 } 1221 } 1222 1223 void HELPER(setm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1224 { 1225 do_setm(env, syndrome, mtedesc, set_step, false, GETPC()); 1226 } 1227 1228 void HELPER(setgm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1229 { 1230 do_setm(env, syndrome, mtedesc, set_step_tags, true, GETPC()); 1231 } 1232 1233 static void do_sete(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, 1234 StepFn *stepfn, bool is_setg, uintptr_t ra) 1235 { 1236 /* Epilogue: do the last partial page */ 1237 int rd = mops_destreg(syndrome); 1238 int rs = mops_srcreg(syndrome); 1239 int rn = mops_sizereg(syndrome); 1240 uint8_t data = arm_reg_or_xzr(env, rs); 1241 uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; 1242 uint64_t setsize = -env->xregs[rn]; 1243 uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); 1244 uint64_t step; 1245 1246 check_mops_enabled(env, ra); 1247 1248 /* 1249 * We're allowed to NOP out "no data to copy" before the consistency 1250 * checks; we choose to do so. 1251 */ 1252 if (setsize == 0) { 1253 return; 1254 } 1255 1256 check_mops_wrong_option(env, syndrome, ra); 1257 1258 /* 1259 * Our implementation has no address alignment requirements, but 1260 * we do want to enforce the "less than a page" size requirement, 1261 * so we don't need to have the "check for interrupts" here. 1262 */ 1263 if (setsize >= TARGET_PAGE_SIZE) { 1264 raise_exception_ra(env, EXCP_UDEF, syndrome, 1265 mops_mismatch_exception_target_el(env), ra); 1266 } 1267 1268 if (unlikely(is_setg)) { 1269 check_setg_alignment(env, toaddr, setsize, memidx, ra); 1270 } else if (!mte_checks_needed(toaddr, mtedesc)) { 1271 mtedesc = 0; 1272 } 1273 1274 /* Do the actual memset */ 1275 while (setsize > 0) { 1276 step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra); 1277 toaddr += step; 1278 setsize -= step; 1279 env->xregs[rn] = -setsize; 1280 } 1281 } 1282 1283 void HELPER(sete)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1284 { 1285 do_sete(env, syndrome, mtedesc, set_step, false, GETPC()); 1286 } 1287 1288 void HELPER(setge)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1289 { 1290 do_sete(env, syndrome, mtedesc, set_step_tags, true, GETPC()); 1291 } 1292 1293 /* 1294 * Perform part of a memory copy from the guest memory at fromaddr 1295 * and extending for copysize bytes, to the guest memory at 1296 * toaddr. Both addresses are dirty. 1297 * 1298 * Returns the number of bytes actually set, which might be less than 1299 * copysize; the caller should loop until the whole copy has been done. 1300 * The caller should ensure that the guest registers are correct 1301 * for the possibility that the first byte of the copy encounters 1302 * an exception or watchpoint. We guarantee not to take any faults 1303 * for bytes other than the first. 1304 */ 1305 static uint64_t copy_step(CPUARMState *env, uint64_t toaddr, uint64_t fromaddr, 1306 uint64_t copysize, int wmemidx, int rmemidx, 1307 uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) 1308 { 1309 void *rmem; 1310 void *wmem; 1311 1312 /* Don't cross a page boundary on either source or destination */ 1313 copysize = MIN(copysize, page_limit(toaddr)); 1314 copysize = MIN(copysize, page_limit(fromaddr)); 1315 /* 1316 * Handle MTE tag checks: either handle the tag mismatch for byte 0, 1317 * or else copy up to but not including the byte with the mismatch. 1318 */ 1319 if (*rdesc) { 1320 uint64_t mtesize = mte_mops_probe(env, fromaddr, copysize, *rdesc); 1321 if (mtesize == 0) { 1322 mte_check_fail(env, *rdesc, fromaddr, ra); 1323 *rdesc = 0; 1324 } else { 1325 copysize = MIN(copysize, mtesize); 1326 } 1327 } 1328 if (*wdesc) { 1329 uint64_t mtesize = mte_mops_probe(env, toaddr, copysize, *wdesc); 1330 if (mtesize == 0) { 1331 mte_check_fail(env, *wdesc, toaddr, ra); 1332 *wdesc = 0; 1333 } else { 1334 copysize = MIN(copysize, mtesize); 1335 } 1336 } 1337 1338 toaddr = useronly_clean_ptr(toaddr); 1339 fromaddr = useronly_clean_ptr(fromaddr); 1340 /* Trapless lookup of whether we can get a host memory pointer */ 1341 wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); 1342 rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); 1343 1344 #ifndef CONFIG_USER_ONLY 1345 /* 1346 * If we don't have host memory for both source and dest then just 1347 * do a single byte copy. This will handle watchpoints, invalid pages, 1348 * etc correctly. For clean code pages, the next iteration will see 1349 * the page dirty and will use the fast path. 1350 */ 1351 if (unlikely(!rmem || !wmem)) { 1352 uint8_t byte; 1353 if (rmem) { 1354 byte = *(uint8_t *)rmem; 1355 } else { 1356 byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); 1357 } 1358 if (wmem) { 1359 *(uint8_t *)wmem = byte; 1360 } else { 1361 cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); 1362 } 1363 return 1; 1364 } 1365 #endif 1366 /* Easy case: just memmove the host memory */ 1367 set_helper_retaddr(ra); 1368 memmove(wmem, rmem, copysize); 1369 clear_helper_retaddr(); 1370 return copysize; 1371 } 1372 1373 /* 1374 * Do part of a backwards memory copy. Here toaddr and fromaddr point 1375 * to the *last* byte to be copied. 1376 */ 1377 static uint64_t copy_step_rev(CPUARMState *env, uint64_t toaddr, 1378 uint64_t fromaddr, 1379 uint64_t copysize, int wmemidx, int rmemidx, 1380 uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) 1381 { 1382 void *rmem; 1383 void *wmem; 1384 1385 /* Don't cross a page boundary on either source or destination */ 1386 copysize = MIN(copysize, page_limit_rev(toaddr)); 1387 copysize = MIN(copysize, page_limit_rev(fromaddr)); 1388 1389 /* 1390 * Handle MTE tag checks: either handle the tag mismatch for byte 0, 1391 * or else copy up to but not including the byte with the mismatch. 1392 */ 1393 if (*rdesc) { 1394 uint64_t mtesize = mte_mops_probe_rev(env, fromaddr, copysize, *rdesc); 1395 if (mtesize == 0) { 1396 mte_check_fail(env, *rdesc, fromaddr, ra); 1397 *rdesc = 0; 1398 } else { 1399 copysize = MIN(copysize, mtesize); 1400 } 1401 } 1402 if (*wdesc) { 1403 uint64_t mtesize = mte_mops_probe_rev(env, toaddr, copysize, *wdesc); 1404 if (mtesize == 0) { 1405 mte_check_fail(env, *wdesc, toaddr, ra); 1406 *wdesc = 0; 1407 } else { 1408 copysize = MIN(copysize, mtesize); 1409 } 1410 } 1411 1412 toaddr = useronly_clean_ptr(toaddr); 1413 fromaddr = useronly_clean_ptr(fromaddr); 1414 /* Trapless lookup of whether we can get a host memory pointer */ 1415 wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); 1416 rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); 1417 1418 #ifndef CONFIG_USER_ONLY 1419 /* 1420 * If we don't have host memory for both source and dest then just 1421 * do a single byte copy. This will handle watchpoints, invalid pages, 1422 * etc correctly. For clean code pages, the next iteration will see 1423 * the page dirty and will use the fast path. 1424 */ 1425 if (unlikely(!rmem || !wmem)) { 1426 uint8_t byte; 1427 if (rmem) { 1428 byte = *(uint8_t *)rmem; 1429 } else { 1430 byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); 1431 } 1432 if (wmem) { 1433 *(uint8_t *)wmem = byte; 1434 } else { 1435 cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); 1436 } 1437 return 1; 1438 } 1439 #endif 1440 /* 1441 * Easy case: just memmove the host memory. Note that wmem and 1442 * rmem here point to the *last* byte to copy. 1443 */ 1444 set_helper_retaddr(ra); 1445 memmove(wmem - (copysize - 1), rmem - (copysize - 1), copysize); 1446 clear_helper_retaddr(); 1447 return copysize; 1448 } 1449 1450 /* 1451 * for the Memory Copy operation, our implementation chooses always 1452 * to use "option A", where we update Xd and Xs to the final addresses 1453 * in the CPYP insn, and then in CPYM and CPYE only need to update Xn. 1454 * 1455 * @env: CPU 1456 * @syndrome: syndrome value for mismatch exceptions 1457 * (also contains the register numbers we need to use) 1458 * @wdesc: MTE descriptor for the writes (destination) 1459 * @rdesc: MTE descriptor for the reads (source) 1460 * @move: true if this is CPY (memmove), false for CPYF (memcpy forwards) 1461 */ 1462 static void do_cpyp(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1463 uint32_t rdesc, uint32_t move, uintptr_t ra) 1464 { 1465 int rd = mops_destreg(syndrome); 1466 int rs = mops_srcreg(syndrome); 1467 int rn = mops_sizereg(syndrome); 1468 uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); 1469 uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); 1470 bool forwards = true; 1471 uint64_t toaddr = env->xregs[rd]; 1472 uint64_t fromaddr = env->xregs[rs]; 1473 uint64_t copysize = env->xregs[rn]; 1474 uint64_t stagecopysize, step; 1475 1476 check_mops_enabled(env, ra); 1477 1478 1479 if (move) { 1480 /* 1481 * Copy backwards if necessary. The direction for a non-overlapping 1482 * copy is IMPDEF; we choose forwards. 1483 */ 1484 if (copysize > 0x007FFFFFFFFFFFFFULL) { 1485 copysize = 0x007FFFFFFFFFFFFFULL; 1486 } 1487 uint64_t fs = extract64(fromaddr, 0, 56); 1488 uint64_t ts = extract64(toaddr, 0, 56); 1489 uint64_t fe = extract64(fromaddr + copysize, 0, 56); 1490 1491 if (fs < ts && fe > ts) { 1492 forwards = false; 1493 } 1494 } else { 1495 if (copysize > INT64_MAX) { 1496 copysize = INT64_MAX; 1497 } 1498 } 1499 1500 if (!mte_checks_needed(fromaddr, rdesc)) { 1501 rdesc = 0; 1502 } 1503 if (!mte_checks_needed(toaddr, wdesc)) { 1504 wdesc = 0; 1505 } 1506 1507 if (forwards) { 1508 stagecopysize = MIN(copysize, page_limit(toaddr)); 1509 stagecopysize = MIN(stagecopysize, page_limit(fromaddr)); 1510 while (stagecopysize) { 1511 env->xregs[rd] = toaddr; 1512 env->xregs[rs] = fromaddr; 1513 env->xregs[rn] = copysize; 1514 step = copy_step(env, toaddr, fromaddr, stagecopysize, 1515 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1516 toaddr += step; 1517 fromaddr += step; 1518 copysize -= step; 1519 stagecopysize -= step; 1520 } 1521 /* Insn completed, so update registers to the Option A format */ 1522 env->xregs[rd] = toaddr + copysize; 1523 env->xregs[rs] = fromaddr + copysize; 1524 env->xregs[rn] = -copysize; 1525 } else { 1526 /* 1527 * In a reverse copy the to and from addrs in Xs and Xd are the start 1528 * of the range, but it's more convenient for us to work with pointers 1529 * to the last byte being copied. 1530 */ 1531 toaddr += copysize - 1; 1532 fromaddr += copysize - 1; 1533 stagecopysize = MIN(copysize, page_limit_rev(toaddr)); 1534 stagecopysize = MIN(stagecopysize, page_limit_rev(fromaddr)); 1535 while (stagecopysize) { 1536 env->xregs[rn] = copysize; 1537 step = copy_step_rev(env, toaddr, fromaddr, stagecopysize, 1538 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1539 copysize -= step; 1540 stagecopysize -= step; 1541 toaddr -= step; 1542 fromaddr -= step; 1543 } 1544 /* 1545 * Insn completed, so update registers to the Option A format. 1546 * For a reverse copy this is no different to the CPYP input format. 1547 */ 1548 env->xregs[rn] = copysize; 1549 } 1550 1551 /* Set NZCV = 0000 to indicate we are an Option A implementation */ 1552 env->NF = 0; 1553 env->ZF = 1; /* our env->ZF encoding is inverted */ 1554 env->CF = 0; 1555 env->VF = 0; 1556 } 1557 1558 void HELPER(cpyp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1559 uint32_t rdesc) 1560 { 1561 do_cpyp(env, syndrome, wdesc, rdesc, true, GETPC()); 1562 } 1563 1564 void HELPER(cpyfp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1565 uint32_t rdesc) 1566 { 1567 do_cpyp(env, syndrome, wdesc, rdesc, false, GETPC()); 1568 } 1569 1570 static void do_cpym(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1571 uint32_t rdesc, uint32_t move, uintptr_t ra) 1572 { 1573 /* Main: we choose to copy until less than a page remaining */ 1574 CPUState *cs = env_cpu(env); 1575 int rd = mops_destreg(syndrome); 1576 int rs = mops_srcreg(syndrome); 1577 int rn = mops_sizereg(syndrome); 1578 uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); 1579 uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); 1580 bool forwards = true; 1581 uint64_t toaddr, fromaddr, copysize, step; 1582 1583 check_mops_enabled(env, ra); 1584 1585 /* We choose to NOP out "no data to copy" before consistency checks */ 1586 if (env->xregs[rn] == 0) { 1587 return; 1588 } 1589 1590 check_mops_wrong_option(env, syndrome, ra); 1591 1592 if (move) { 1593 forwards = (int64_t)env->xregs[rn] < 0; 1594 } 1595 1596 if (forwards) { 1597 toaddr = env->xregs[rd] + env->xregs[rn]; 1598 fromaddr = env->xregs[rs] + env->xregs[rn]; 1599 copysize = -env->xregs[rn]; 1600 } else { 1601 copysize = env->xregs[rn]; 1602 /* This toaddr and fromaddr point to the *last* byte to copy */ 1603 toaddr = env->xregs[rd] + copysize - 1; 1604 fromaddr = env->xregs[rs] + copysize - 1; 1605 } 1606 1607 if (!mte_checks_needed(fromaddr, rdesc)) { 1608 rdesc = 0; 1609 } 1610 if (!mte_checks_needed(toaddr, wdesc)) { 1611 wdesc = 0; 1612 } 1613 1614 /* Our implementation has no particular parameter requirements for CPYM */ 1615 1616 /* Do the actual memmove */ 1617 if (forwards) { 1618 while (copysize >= TARGET_PAGE_SIZE) { 1619 step = copy_step(env, toaddr, fromaddr, copysize, 1620 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1621 toaddr += step; 1622 fromaddr += step; 1623 copysize -= step; 1624 env->xregs[rn] = -copysize; 1625 if (copysize >= TARGET_PAGE_SIZE && 1626 unlikely(cpu_loop_exit_requested(cs))) { 1627 cpu_loop_exit_restore(cs, ra); 1628 } 1629 } 1630 } else { 1631 while (copysize >= TARGET_PAGE_SIZE) { 1632 step = copy_step_rev(env, toaddr, fromaddr, copysize, 1633 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1634 toaddr -= step; 1635 fromaddr -= step; 1636 copysize -= step; 1637 env->xregs[rn] = copysize; 1638 if (copysize >= TARGET_PAGE_SIZE && 1639 unlikely(cpu_loop_exit_requested(cs))) { 1640 cpu_loop_exit_restore(cs, ra); 1641 } 1642 } 1643 } 1644 } 1645 1646 void HELPER(cpym)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1647 uint32_t rdesc) 1648 { 1649 do_cpym(env, syndrome, wdesc, rdesc, true, GETPC()); 1650 } 1651 1652 void HELPER(cpyfm)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1653 uint32_t rdesc) 1654 { 1655 do_cpym(env, syndrome, wdesc, rdesc, false, GETPC()); 1656 } 1657 1658 static void do_cpye(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1659 uint32_t rdesc, uint32_t move, uintptr_t ra) 1660 { 1661 /* Epilogue: do the last partial page */ 1662 int rd = mops_destreg(syndrome); 1663 int rs = mops_srcreg(syndrome); 1664 int rn = mops_sizereg(syndrome); 1665 uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); 1666 uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); 1667 bool forwards = true; 1668 uint64_t toaddr, fromaddr, copysize, step; 1669 1670 check_mops_enabled(env, ra); 1671 1672 /* We choose to NOP out "no data to copy" before consistency checks */ 1673 if (env->xregs[rn] == 0) { 1674 return; 1675 } 1676 1677 check_mops_wrong_option(env, syndrome, ra); 1678 1679 if (move) { 1680 forwards = (int64_t)env->xregs[rn] < 0; 1681 } 1682 1683 if (forwards) { 1684 toaddr = env->xregs[rd] + env->xregs[rn]; 1685 fromaddr = env->xregs[rs] + env->xregs[rn]; 1686 copysize = -env->xregs[rn]; 1687 } else { 1688 copysize = env->xregs[rn]; 1689 /* This toaddr and fromaddr point to the *last* byte to copy */ 1690 toaddr = env->xregs[rd] + copysize - 1; 1691 fromaddr = env->xregs[rs] + copysize - 1; 1692 } 1693 1694 if (!mte_checks_needed(fromaddr, rdesc)) { 1695 rdesc = 0; 1696 } 1697 if (!mte_checks_needed(toaddr, wdesc)) { 1698 wdesc = 0; 1699 } 1700 1701 /* Check the size; we don't want to have do a check-for-interrupts */ 1702 if (copysize >= TARGET_PAGE_SIZE) { 1703 raise_exception_ra(env, EXCP_UDEF, syndrome, 1704 mops_mismatch_exception_target_el(env), ra); 1705 } 1706 1707 /* Do the actual memmove */ 1708 if (forwards) { 1709 while (copysize > 0) { 1710 step = copy_step(env, toaddr, fromaddr, copysize, 1711 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1712 toaddr += step; 1713 fromaddr += step; 1714 copysize -= step; 1715 env->xregs[rn] = -copysize; 1716 } 1717 } else { 1718 while (copysize > 0) { 1719 step = copy_step_rev(env, toaddr, fromaddr, copysize, 1720 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1721 toaddr -= step; 1722 fromaddr -= step; 1723 copysize -= step; 1724 env->xregs[rn] = copysize; 1725 } 1726 } 1727 } 1728 1729 void HELPER(cpye)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1730 uint32_t rdesc) 1731 { 1732 do_cpye(env, syndrome, wdesc, rdesc, true, GETPC()); 1733 } 1734 1735 void HELPER(cpyfe)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1736 uint32_t rdesc) 1737 { 1738 do_cpye(env, syndrome, wdesc, rdesc, false, GETPC()); 1739 } 1740 1741 static bool is_guarded_page(CPUARMState *env, target_ulong addr, uintptr_t ra) 1742 { 1743 #ifdef CONFIG_USER_ONLY 1744 return page_get_flags(addr) & PAGE_BTI; 1745 #else 1746 CPUTLBEntryFull *full; 1747 void *host; 1748 int mmu_idx = cpu_mmu_index(env_cpu(env), true); 1749 int flags = probe_access_full(env, addr, 0, MMU_INST_FETCH, mmu_idx, 1750 false, &host, &full, ra); 1751 1752 assert(!(flags & TLB_INVALID_MASK)); 1753 return full->extra.arm.guarded; 1754 #endif 1755 } 1756 1757 void HELPER(guarded_page_check)(CPUARMState *env) 1758 { 1759 /* 1760 * We have already verified that bti is enabled, and that the 1761 * instruction at PC is not ok for BTYPE. This is always at 1762 * the beginning of a block, so PC is always up-to-date and 1763 * no unwind is required. 1764 */ 1765 if (is_guarded_page(env, env->pc, 0)) { 1766 raise_exception(env, EXCP_UDEF, syn_btitrap(env->btype), 1767 exception_target_el(env)); 1768 } 1769 } 1770 1771 void HELPER(guarded_page_br)(CPUARMState *env, target_ulong pc) 1772 { 1773 /* 1774 * We have already checked for branch via x16 and x17. 1775 * What remains for choosing BTYPE is checking for a guarded page. 1776 */ 1777 env->btype = is_guarded_page(env, pc, GETPC()) ? 3 : 1; 1778 } 1779