1 /* 2 * ARM v8.3-PAuth Operations 3 * 4 * Copyright (c) 2019 Linaro, Ltd. 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "cpu.h" 22 #include "internals.h" 23 #include "exec/exec-all.h" 24 #include "exec/cpu_ldst.h" 25 #include "exec/helper-proto.h" 26 #include "tcg/tcg-gvec-desc.h" 27 #include "qemu/xxhash.h" 28 29 30 static uint64_t pac_cell_shuffle(uint64_t i) 31 { 32 uint64_t o = 0; 33 34 o |= extract64(i, 52, 4); 35 o |= extract64(i, 24, 4) << 4; 36 o |= extract64(i, 44, 4) << 8; 37 o |= extract64(i, 0, 4) << 12; 38 39 o |= extract64(i, 28, 4) << 16; 40 o |= extract64(i, 48, 4) << 20; 41 o |= extract64(i, 4, 4) << 24; 42 o |= extract64(i, 40, 4) << 28; 43 44 o |= extract64(i, 32, 4) << 32; 45 o |= extract64(i, 12, 4) << 36; 46 o |= extract64(i, 56, 4) << 40; 47 o |= extract64(i, 20, 4) << 44; 48 49 o |= extract64(i, 8, 4) << 48; 50 o |= extract64(i, 36, 4) << 52; 51 o |= extract64(i, 16, 4) << 56; 52 o |= extract64(i, 60, 4) << 60; 53 54 return o; 55 } 56 57 static uint64_t pac_cell_inv_shuffle(uint64_t i) 58 { 59 uint64_t o = 0; 60 61 o |= extract64(i, 12, 4); 62 o |= extract64(i, 24, 4) << 4; 63 o |= extract64(i, 48, 4) << 8; 64 o |= extract64(i, 36, 4) << 12; 65 66 o |= extract64(i, 56, 4) << 16; 67 o |= extract64(i, 44, 4) << 20; 68 o |= extract64(i, 4, 4) << 24; 69 o |= extract64(i, 16, 4) << 28; 70 71 o |= i & MAKE_64BIT_MASK(32, 4); 72 o |= extract64(i, 52, 4) << 36; 73 o |= extract64(i, 28, 4) << 40; 74 o |= extract64(i, 8, 4) << 44; 75 76 o |= extract64(i, 20, 4) << 48; 77 o |= extract64(i, 0, 4) << 52; 78 o |= extract64(i, 40, 4) << 56; 79 o |= i & MAKE_64BIT_MASK(60, 4); 80 81 return o; 82 } 83 84 static uint64_t pac_sub(uint64_t i) 85 { 86 static const uint8_t sub[16] = { 87 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe, 88 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa, 89 }; 90 uint64_t o = 0; 91 int b; 92 93 for (b = 0; b < 64; b += 4) { 94 o |= (uint64_t)sub[(i >> b) & 0xf] << b; 95 } 96 return o; 97 } 98 99 static uint64_t pac_sub1(uint64_t i) 100 { 101 static const uint8_t sub1[16] = { 102 0xa, 0xd, 0xe, 0x6, 0xf, 0x7, 0x3, 0x5, 103 0x9, 0x8, 0x0, 0xc, 0xb, 0x1, 0x2, 0x4, 104 }; 105 uint64_t o = 0; 106 int b; 107 108 for (b = 0; b < 64; b += 4) { 109 o |= (uint64_t)sub1[(i >> b) & 0xf] << b; 110 } 111 return o; 112 } 113 114 static uint64_t pac_inv_sub(uint64_t i) 115 { 116 static const uint8_t inv_sub[16] = { 117 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9, 118 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3, 119 }; 120 uint64_t o = 0; 121 int b; 122 123 for (b = 0; b < 64; b += 4) { 124 o |= (uint64_t)inv_sub[(i >> b) & 0xf] << b; 125 } 126 return o; 127 } 128 129 static int rot_cell(int cell, int n) 130 { 131 /* 4-bit rotate left by n. */ 132 cell |= cell << 4; 133 return extract32(cell, 4 - n, 4); 134 } 135 136 static uint64_t pac_mult(uint64_t i) 137 { 138 uint64_t o = 0; 139 int b; 140 141 for (b = 0; b < 4 * 4; b += 4) { 142 int i0, i4, i8, ic, t0, t1, t2, t3; 143 144 i0 = extract64(i, b, 4); 145 i4 = extract64(i, b + 4 * 4, 4); 146 i8 = extract64(i, b + 8 * 4, 4); 147 ic = extract64(i, b + 12 * 4, 4); 148 149 t0 = rot_cell(i8, 1) ^ rot_cell(i4, 2) ^ rot_cell(i0, 1); 150 t1 = rot_cell(ic, 1) ^ rot_cell(i4, 1) ^ rot_cell(i0, 2); 151 t2 = rot_cell(ic, 2) ^ rot_cell(i8, 1) ^ rot_cell(i0, 1); 152 t3 = rot_cell(ic, 1) ^ rot_cell(i8, 2) ^ rot_cell(i4, 1); 153 154 o |= (uint64_t)t3 << b; 155 o |= (uint64_t)t2 << (b + 4 * 4); 156 o |= (uint64_t)t1 << (b + 8 * 4); 157 o |= (uint64_t)t0 << (b + 12 * 4); 158 } 159 return o; 160 } 161 162 static uint64_t tweak_cell_rot(uint64_t cell) 163 { 164 return (cell >> 1) | (((cell ^ (cell >> 1)) & 1) << 3); 165 } 166 167 static uint64_t tweak_shuffle(uint64_t i) 168 { 169 uint64_t o = 0; 170 171 o |= extract64(i, 16, 4) << 0; 172 o |= extract64(i, 20, 4) << 4; 173 o |= tweak_cell_rot(extract64(i, 24, 4)) << 8; 174 o |= extract64(i, 28, 4) << 12; 175 176 o |= tweak_cell_rot(extract64(i, 44, 4)) << 16; 177 o |= extract64(i, 8, 4) << 20; 178 o |= extract64(i, 12, 4) << 24; 179 o |= tweak_cell_rot(extract64(i, 32, 4)) << 28; 180 181 o |= extract64(i, 48, 4) << 32; 182 o |= extract64(i, 52, 4) << 36; 183 o |= extract64(i, 56, 4) << 40; 184 o |= tweak_cell_rot(extract64(i, 60, 4)) << 44; 185 186 o |= tweak_cell_rot(extract64(i, 0, 4)) << 48; 187 o |= extract64(i, 4, 4) << 52; 188 o |= tweak_cell_rot(extract64(i, 40, 4)) << 56; 189 o |= tweak_cell_rot(extract64(i, 36, 4)) << 60; 190 191 return o; 192 } 193 194 static uint64_t tweak_cell_inv_rot(uint64_t cell) 195 { 196 return ((cell << 1) & 0xf) | ((cell & 1) ^ (cell >> 3)); 197 } 198 199 static uint64_t tweak_inv_shuffle(uint64_t i) 200 { 201 uint64_t o = 0; 202 203 o |= tweak_cell_inv_rot(extract64(i, 48, 4)); 204 o |= extract64(i, 52, 4) << 4; 205 o |= extract64(i, 20, 4) << 8; 206 o |= extract64(i, 24, 4) << 12; 207 208 o |= extract64(i, 0, 4) << 16; 209 o |= extract64(i, 4, 4) << 20; 210 o |= tweak_cell_inv_rot(extract64(i, 8, 4)) << 24; 211 o |= extract64(i, 12, 4) << 28; 212 213 o |= tweak_cell_inv_rot(extract64(i, 28, 4)) << 32; 214 o |= tweak_cell_inv_rot(extract64(i, 60, 4)) << 36; 215 o |= tweak_cell_inv_rot(extract64(i, 56, 4)) << 40; 216 o |= tweak_cell_inv_rot(extract64(i, 16, 4)) << 44; 217 218 o |= extract64(i, 32, 4) << 48; 219 o |= extract64(i, 36, 4) << 52; 220 o |= extract64(i, 40, 4) << 56; 221 o |= tweak_cell_inv_rot(extract64(i, 44, 4)) << 60; 222 223 return o; 224 } 225 226 static uint64_t pauth_computepac_architected(uint64_t data, uint64_t modifier, 227 ARMPACKey key, bool isqarma3) 228 { 229 static const uint64_t RC[5] = { 230 0x0000000000000000ull, 231 0x13198A2E03707344ull, 232 0xA4093822299F31D0ull, 233 0x082EFA98EC4E6C89ull, 234 0x452821E638D01377ull, 235 }; 236 const uint64_t alpha = 0xC0AC29B7C97C50DDull; 237 int iterations = isqarma3 ? 2 : 4; 238 /* 239 * Note that in the ARM pseudocode, key0 contains bits <127:64> 240 * and key1 contains bits <63:0> of the 128-bit key. 241 */ 242 uint64_t key0 = key.hi, key1 = key.lo; 243 uint64_t workingval, runningmod, roundkey, modk0; 244 int i; 245 246 modk0 = (key0 << 63) | ((key0 >> 1) ^ (key0 >> 63)); 247 runningmod = modifier; 248 workingval = data ^ key0; 249 250 for (i = 0; i <= iterations; ++i) { 251 roundkey = key1 ^ runningmod; 252 workingval ^= roundkey; 253 workingval ^= RC[i]; 254 if (i > 0) { 255 workingval = pac_cell_shuffle(workingval); 256 workingval = pac_mult(workingval); 257 } 258 if (isqarma3) { 259 workingval = pac_sub1(workingval); 260 } else { 261 workingval = pac_sub(workingval); 262 } 263 runningmod = tweak_shuffle(runningmod); 264 } 265 roundkey = modk0 ^ runningmod; 266 workingval ^= roundkey; 267 workingval = pac_cell_shuffle(workingval); 268 workingval = pac_mult(workingval); 269 if (isqarma3) { 270 workingval = pac_sub1(workingval); 271 } else { 272 workingval = pac_sub(workingval); 273 } 274 workingval = pac_cell_shuffle(workingval); 275 workingval = pac_mult(workingval); 276 workingval ^= key1; 277 workingval = pac_cell_inv_shuffle(workingval); 278 if (isqarma3) { 279 workingval = pac_sub1(workingval); 280 } else { 281 workingval = pac_inv_sub(workingval); 282 } 283 workingval = pac_mult(workingval); 284 workingval = pac_cell_inv_shuffle(workingval); 285 workingval ^= key0; 286 workingval ^= runningmod; 287 for (i = 0; i <= iterations; ++i) { 288 if (isqarma3) { 289 workingval = pac_sub1(workingval); 290 } else { 291 workingval = pac_inv_sub(workingval); 292 } 293 if (i < iterations) { 294 workingval = pac_mult(workingval); 295 workingval = pac_cell_inv_shuffle(workingval); 296 } 297 runningmod = tweak_inv_shuffle(runningmod); 298 roundkey = key1 ^ runningmod; 299 workingval ^= RC[iterations - i]; 300 workingval ^= roundkey; 301 workingval ^= alpha; 302 } 303 workingval ^= modk0; 304 305 return workingval; 306 } 307 308 static uint64_t pauth_computepac_impdef(uint64_t data, uint64_t modifier, 309 ARMPACKey key) 310 { 311 return qemu_xxhash64_4(data, modifier, key.lo, key.hi); 312 } 313 314 static uint64_t pauth_computepac(CPUARMState *env, uint64_t data, 315 uint64_t modifier, ARMPACKey key) 316 { 317 if (cpu_isar_feature(aa64_pauth_qarma5, env_archcpu(env))) { 318 return pauth_computepac_architected(data, modifier, key, false); 319 } else if (cpu_isar_feature(aa64_pauth_qarma3, env_archcpu(env))) { 320 return pauth_computepac_architected(data, modifier, key, true); 321 } else { 322 return pauth_computepac_impdef(data, modifier, key); 323 } 324 } 325 326 static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier, 327 ARMPACKey *key, bool data) 328 { 329 ARMCPU *cpu = env_archcpu(env); 330 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); 331 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false); 332 ARMPauthFeature pauth_feature = cpu_isar_feature(pauth_feature, cpu); 333 uint64_t pac, ext_ptr, ext, test; 334 int bot_bit, top_bit; 335 336 /* If tagged pointers are in use, use ptr<55>, otherwise ptr<63>. */ 337 if (param.tbi) { 338 ext = sextract64(ptr, 55, 1); 339 } else { 340 ext = sextract64(ptr, 63, 1); 341 } 342 343 /* Build a pointer with known good extension bits. */ 344 top_bit = 64 - 8 * param.tbi; 345 bot_bit = 64 - param.tsz; 346 ext_ptr = deposit64(ptr, bot_bit, top_bit - bot_bit, ext); 347 348 pac = pauth_computepac(env, ext_ptr, modifier, *key); 349 350 /* 351 * Check if the ptr has good extension bits and corrupt the 352 * pointer authentication code if not. 353 */ 354 test = sextract64(ptr, bot_bit, top_bit - bot_bit); 355 if (test != 0 && test != -1) { 356 if (pauth_feature >= PauthFeat_2) { 357 /* No action required */ 358 } else if (pauth_feature == PauthFeat_EPAC) { 359 pac = 0; 360 } else { 361 /* 362 * Note that our top_bit is one greater than the pseudocode's 363 * version, hence "- 2" here. 364 */ 365 pac ^= MAKE_64BIT_MASK(top_bit - 2, 1); 366 } 367 } 368 369 /* 370 * Preserve the determination between upper and lower at bit 55, 371 * and insert pointer authentication code. 372 */ 373 if (pauth_feature >= PauthFeat_2) { 374 pac ^= ptr; 375 } 376 if (param.tbi) { 377 ptr &= ~MAKE_64BIT_MASK(bot_bit, 55 - bot_bit + 1); 378 pac &= MAKE_64BIT_MASK(bot_bit, 54 - bot_bit + 1); 379 } else { 380 ptr &= MAKE_64BIT_MASK(0, bot_bit); 381 pac &= ~(MAKE_64BIT_MASK(55, 1) | MAKE_64BIT_MASK(0, bot_bit)); 382 } 383 ext &= MAKE_64BIT_MASK(55, 1); 384 return pac | ext | ptr; 385 } 386 387 static uint64_t pauth_original_ptr(uint64_t ptr, ARMVAParameters param) 388 { 389 uint64_t mask = pauth_ptr_mask(param); 390 391 /* Note that bit 55 is used whether or not the regime has 2 ranges. */ 392 if (extract64(ptr, 55, 1)) { 393 return ptr | mask; 394 } else { 395 return ptr & ~mask; 396 } 397 } 398 399 static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier, 400 ARMPACKey *key, bool data, int keynumber, 401 uintptr_t ra, bool is_combined) 402 { 403 ARMCPU *cpu = env_archcpu(env); 404 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); 405 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false); 406 ARMPauthFeature pauth_feature = cpu_isar_feature(pauth_feature, cpu); 407 int bot_bit, top_bit; 408 uint64_t pac, orig_ptr, cmp_mask; 409 410 orig_ptr = pauth_original_ptr(ptr, param); 411 pac = pauth_computepac(env, orig_ptr, modifier, *key); 412 bot_bit = 64 - param.tsz; 413 top_bit = 64 - 8 * param.tbi; 414 415 cmp_mask = MAKE_64BIT_MASK(bot_bit, top_bit - bot_bit); 416 cmp_mask &= ~MAKE_64BIT_MASK(55, 1); 417 418 if (pauth_feature >= PauthFeat_2) { 419 return ptr ^ (pac & cmp_mask); 420 } 421 422 if ((pac ^ ptr) & cmp_mask) { 423 int error_code = (keynumber << 1) | (keynumber ^ 1); 424 if (param.tbi) { 425 return deposit64(orig_ptr, 53, 2, error_code); 426 } else { 427 return deposit64(orig_ptr, 61, 2, error_code); 428 } 429 } 430 return orig_ptr; 431 } 432 433 static uint64_t pauth_strip(CPUARMState *env, uint64_t ptr, bool data) 434 { 435 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); 436 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false); 437 438 return pauth_original_ptr(ptr, param); 439 } 440 441 static G_NORETURN 442 void pauth_trap(CPUARMState *env, int target_el, uintptr_t ra) 443 { 444 raise_exception_ra(env, EXCP_UDEF, syn_pactrap(), target_el, ra); 445 } 446 447 static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra) 448 { 449 if (el < 2 && arm_is_el2_enabled(env)) { 450 uint64_t hcr = arm_hcr_el2_eff(env); 451 bool trap = !(hcr & HCR_API); 452 if (el == 0) { 453 /* Trap only applies to EL1&0 regime. */ 454 trap &= (hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE); 455 } 456 /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */ 457 if (trap) { 458 pauth_trap(env, 2, ra); 459 } 460 } 461 if (el < 3 && arm_feature(env, ARM_FEATURE_EL3)) { 462 if (!(env->cp15.scr_el3 & SCR_API)) { 463 pauth_trap(env, 3, ra); 464 } 465 } 466 } 467 468 static bool pauth_key_enabled(CPUARMState *env, int el, uint32_t bit) 469 { 470 return (arm_sctlr(env, el) & bit) != 0; 471 } 472 473 uint64_t HELPER(pacia)(CPUARMState *env, uint64_t x, uint64_t y) 474 { 475 int el = arm_current_el(env); 476 if (!pauth_key_enabled(env, el, SCTLR_EnIA)) { 477 return x; 478 } 479 pauth_check_trap(env, el, GETPC()); 480 return pauth_addpac(env, x, y, &env->keys.apia, false); 481 } 482 483 uint64_t HELPER(pacib)(CPUARMState *env, uint64_t x, uint64_t y) 484 { 485 int el = arm_current_el(env); 486 if (!pauth_key_enabled(env, el, SCTLR_EnIB)) { 487 return x; 488 } 489 pauth_check_trap(env, el, GETPC()); 490 return pauth_addpac(env, x, y, &env->keys.apib, false); 491 } 492 493 uint64_t HELPER(pacda)(CPUARMState *env, uint64_t x, uint64_t y) 494 { 495 int el = arm_current_el(env); 496 if (!pauth_key_enabled(env, el, SCTLR_EnDA)) { 497 return x; 498 } 499 pauth_check_trap(env, el, GETPC()); 500 return pauth_addpac(env, x, y, &env->keys.apda, true); 501 } 502 503 uint64_t HELPER(pacdb)(CPUARMState *env, uint64_t x, uint64_t y) 504 { 505 int el = arm_current_el(env); 506 if (!pauth_key_enabled(env, el, SCTLR_EnDB)) { 507 return x; 508 } 509 pauth_check_trap(env, el, GETPC()); 510 return pauth_addpac(env, x, y, &env->keys.apdb, true); 511 } 512 513 uint64_t HELPER(pacga)(CPUARMState *env, uint64_t x, uint64_t y) 514 { 515 uint64_t pac; 516 517 pauth_check_trap(env, arm_current_el(env), GETPC()); 518 pac = pauth_computepac(env, x, y, env->keys.apga); 519 520 return pac & 0xffffffff00000000ull; 521 } 522 523 static uint64_t pauth_autia(CPUARMState *env, uint64_t x, uint64_t y, 524 uintptr_t ra, bool is_combined) 525 { 526 int el = arm_current_el(env); 527 if (!pauth_key_enabled(env, el, SCTLR_EnIA)) { 528 return x; 529 } 530 pauth_check_trap(env, el, ra); 531 return pauth_auth(env, x, y, &env->keys.apia, false, 0, ra, is_combined); 532 } 533 534 uint64_t HELPER(autia)(CPUARMState *env, uint64_t x, uint64_t y) 535 { 536 return pauth_autia(env, x, y, GETPC(), false); 537 } 538 539 uint64_t HELPER(autia_combined)(CPUARMState *env, uint64_t x, uint64_t y) 540 { 541 return pauth_autia(env, x, y, GETPC(), true); 542 } 543 544 static uint64_t pauth_autib(CPUARMState *env, uint64_t x, uint64_t y, 545 uintptr_t ra, bool is_combined) 546 { 547 int el = arm_current_el(env); 548 if (!pauth_key_enabled(env, el, SCTLR_EnIB)) { 549 return x; 550 } 551 pauth_check_trap(env, el, ra); 552 return pauth_auth(env, x, y, &env->keys.apib, false, 1, ra, is_combined); 553 } 554 555 uint64_t HELPER(autib)(CPUARMState *env, uint64_t x, uint64_t y) 556 { 557 return pauth_autib(env, x, y, GETPC(), false); 558 } 559 560 uint64_t HELPER(autib_combined)(CPUARMState *env, uint64_t x, uint64_t y) 561 { 562 return pauth_autib(env, x, y, GETPC(), true); 563 } 564 565 static uint64_t pauth_autda(CPUARMState *env, uint64_t x, uint64_t y, 566 uintptr_t ra, bool is_combined) 567 { 568 int el = arm_current_el(env); 569 if (!pauth_key_enabled(env, el, SCTLR_EnDA)) { 570 return x; 571 } 572 pauth_check_trap(env, el, ra); 573 return pauth_auth(env, x, y, &env->keys.apda, true, 0, ra, is_combined); 574 } 575 576 uint64_t HELPER(autda)(CPUARMState *env, uint64_t x, uint64_t y) 577 { 578 return pauth_autda(env, x, y, GETPC(), false); 579 } 580 581 uint64_t HELPER(autda_combined)(CPUARMState *env, uint64_t x, uint64_t y) 582 { 583 return pauth_autda(env, x, y, GETPC(), true); 584 } 585 586 static uint64_t pauth_autdb(CPUARMState *env, uint64_t x, uint64_t y, 587 uintptr_t ra, bool is_combined) 588 { 589 int el = arm_current_el(env); 590 if (!pauth_key_enabled(env, el, SCTLR_EnDB)) { 591 return x; 592 } 593 pauth_check_trap(env, el, ra); 594 return pauth_auth(env, x, y, &env->keys.apdb, true, 1, ra, is_combined); 595 } 596 597 uint64_t HELPER(autdb)(CPUARMState *env, uint64_t x, uint64_t y) 598 { 599 return pauth_autdb(env, x, y, GETPC(), false); 600 } 601 602 uint64_t HELPER(autdb_combined)(CPUARMState *env, uint64_t x, uint64_t y) 603 { 604 return pauth_autdb(env, x, y, GETPC(), true); 605 } 606 607 uint64_t HELPER(xpaci)(CPUARMState *env, uint64_t a) 608 { 609 return pauth_strip(env, a, false); 610 } 611 612 uint64_t HELPER(xpacd)(CPUARMState *env, uint64_t a) 613 { 614 return pauth_strip(env, a, true); 615 } 616