1 /* 2 * ARM v8.3-PAuth Operations 3 * 4 * Copyright (c) 2019 Linaro, Ltd. 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "cpu.h" 22 #include "internals.h" 23 #include "exec/exec-all.h" 24 #include "exec/cpu_ldst.h" 25 #include "exec/helper-proto.h" 26 #include "tcg/tcg-gvec-desc.h" 27 #include "qemu/xxhash.h" 28 29 30 static uint64_t pac_cell_shuffle(uint64_t i) 31 { 32 uint64_t o = 0; 33 34 o |= extract64(i, 52, 4); 35 o |= extract64(i, 24, 4) << 4; 36 o |= extract64(i, 44, 4) << 8; 37 o |= extract64(i, 0, 4) << 12; 38 39 o |= extract64(i, 28, 4) << 16; 40 o |= extract64(i, 48, 4) << 20; 41 o |= extract64(i, 4, 4) << 24; 42 o |= extract64(i, 40, 4) << 28; 43 44 o |= extract64(i, 32, 4) << 32; 45 o |= extract64(i, 12, 4) << 36; 46 o |= extract64(i, 56, 4) << 40; 47 o |= extract64(i, 20, 4) << 44; 48 49 o |= extract64(i, 8, 4) << 48; 50 o |= extract64(i, 36, 4) << 52; 51 o |= extract64(i, 16, 4) << 56; 52 o |= extract64(i, 60, 4) << 60; 53 54 return o; 55 } 56 57 static uint64_t pac_cell_inv_shuffle(uint64_t i) 58 { 59 uint64_t o = 0; 60 61 o |= extract64(i, 12, 4); 62 o |= extract64(i, 24, 4) << 4; 63 o |= extract64(i, 48, 4) << 8; 64 o |= extract64(i, 36, 4) << 12; 65 66 o |= extract64(i, 56, 4) << 16; 67 o |= extract64(i, 44, 4) << 20; 68 o |= extract64(i, 4, 4) << 24; 69 o |= extract64(i, 16, 4) << 28; 70 71 o |= i & MAKE_64BIT_MASK(32, 4); 72 o |= extract64(i, 52, 4) << 36; 73 o |= extract64(i, 28, 4) << 40; 74 o |= extract64(i, 8, 4) << 44; 75 76 o |= extract64(i, 20, 4) << 48; 77 o |= extract64(i, 0, 4) << 52; 78 o |= extract64(i, 40, 4) << 56; 79 o |= i & MAKE_64BIT_MASK(60, 4); 80 81 return o; 82 } 83 84 static uint64_t pac_sub(uint64_t i) 85 { 86 static const uint8_t sub[16] = { 87 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe, 88 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa, 89 }; 90 uint64_t o = 0; 91 int b; 92 93 for (b = 0; b < 64; b += 4) { 94 o |= (uint64_t)sub[(i >> b) & 0xf] << b; 95 } 96 return o; 97 } 98 99 static uint64_t pac_inv_sub(uint64_t i) 100 { 101 static const uint8_t inv_sub[16] = { 102 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9, 103 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3, 104 }; 105 uint64_t o = 0; 106 int b; 107 108 for (b = 0; b < 64; b += 4) { 109 o |= (uint64_t)inv_sub[(i >> b) & 0xf] << b; 110 } 111 return o; 112 } 113 114 static int rot_cell(int cell, int n) 115 { 116 /* 4-bit rotate left by n. */ 117 cell |= cell << 4; 118 return extract32(cell, 4 - n, 4); 119 } 120 121 static uint64_t pac_mult(uint64_t i) 122 { 123 uint64_t o = 0; 124 int b; 125 126 for (b = 0; b < 4 * 4; b += 4) { 127 int i0, i4, i8, ic, t0, t1, t2, t3; 128 129 i0 = extract64(i, b, 4); 130 i4 = extract64(i, b + 4 * 4, 4); 131 i8 = extract64(i, b + 8 * 4, 4); 132 ic = extract64(i, b + 12 * 4, 4); 133 134 t0 = rot_cell(i8, 1) ^ rot_cell(i4, 2) ^ rot_cell(i0, 1); 135 t1 = rot_cell(ic, 1) ^ rot_cell(i4, 1) ^ rot_cell(i0, 2); 136 t2 = rot_cell(ic, 2) ^ rot_cell(i8, 1) ^ rot_cell(i0, 1); 137 t3 = rot_cell(ic, 1) ^ rot_cell(i8, 2) ^ rot_cell(i4, 1); 138 139 o |= (uint64_t)t3 << b; 140 o |= (uint64_t)t2 << (b + 4 * 4); 141 o |= (uint64_t)t1 << (b + 8 * 4); 142 o |= (uint64_t)t0 << (b + 12 * 4); 143 } 144 return o; 145 } 146 147 static uint64_t tweak_cell_rot(uint64_t cell) 148 { 149 return (cell >> 1) | (((cell ^ (cell >> 1)) & 1) << 3); 150 } 151 152 static uint64_t tweak_shuffle(uint64_t i) 153 { 154 uint64_t o = 0; 155 156 o |= extract64(i, 16, 4) << 0; 157 o |= extract64(i, 20, 4) << 4; 158 o |= tweak_cell_rot(extract64(i, 24, 4)) << 8; 159 o |= extract64(i, 28, 4) << 12; 160 161 o |= tweak_cell_rot(extract64(i, 44, 4)) << 16; 162 o |= extract64(i, 8, 4) << 20; 163 o |= extract64(i, 12, 4) << 24; 164 o |= tweak_cell_rot(extract64(i, 32, 4)) << 28; 165 166 o |= extract64(i, 48, 4) << 32; 167 o |= extract64(i, 52, 4) << 36; 168 o |= extract64(i, 56, 4) << 40; 169 o |= tweak_cell_rot(extract64(i, 60, 4)) << 44; 170 171 o |= tweak_cell_rot(extract64(i, 0, 4)) << 48; 172 o |= extract64(i, 4, 4) << 52; 173 o |= tweak_cell_rot(extract64(i, 40, 4)) << 56; 174 o |= tweak_cell_rot(extract64(i, 36, 4)) << 60; 175 176 return o; 177 } 178 179 static uint64_t tweak_cell_inv_rot(uint64_t cell) 180 { 181 return ((cell << 1) & 0xf) | ((cell & 1) ^ (cell >> 3)); 182 } 183 184 static uint64_t tweak_inv_shuffle(uint64_t i) 185 { 186 uint64_t o = 0; 187 188 o |= tweak_cell_inv_rot(extract64(i, 48, 4)); 189 o |= extract64(i, 52, 4) << 4; 190 o |= extract64(i, 20, 4) << 8; 191 o |= extract64(i, 24, 4) << 12; 192 193 o |= extract64(i, 0, 4) << 16; 194 o |= extract64(i, 4, 4) << 20; 195 o |= tweak_cell_inv_rot(extract64(i, 8, 4)) << 24; 196 o |= extract64(i, 12, 4) << 28; 197 198 o |= tweak_cell_inv_rot(extract64(i, 28, 4)) << 32; 199 o |= tweak_cell_inv_rot(extract64(i, 60, 4)) << 36; 200 o |= tweak_cell_inv_rot(extract64(i, 56, 4)) << 40; 201 o |= tweak_cell_inv_rot(extract64(i, 16, 4)) << 44; 202 203 o |= extract64(i, 32, 4) << 48; 204 o |= extract64(i, 36, 4) << 52; 205 o |= extract64(i, 40, 4) << 56; 206 o |= tweak_cell_inv_rot(extract64(i, 44, 4)) << 60; 207 208 return o; 209 } 210 211 static uint64_t pauth_computepac_architected(uint64_t data, uint64_t modifier, 212 ARMPACKey key) 213 { 214 static const uint64_t RC[5] = { 215 0x0000000000000000ull, 216 0x13198A2E03707344ull, 217 0xA4093822299F31D0ull, 218 0x082EFA98EC4E6C89ull, 219 0x452821E638D01377ull, 220 }; 221 const uint64_t alpha = 0xC0AC29B7C97C50DDull; 222 /* 223 * Note that in the ARM pseudocode, key0 contains bits <127:64> 224 * and key1 contains bits <63:0> of the 128-bit key. 225 */ 226 uint64_t key0 = key.hi, key1 = key.lo; 227 uint64_t workingval, runningmod, roundkey, modk0; 228 int i; 229 230 modk0 = (key0 << 63) | ((key0 >> 1) ^ (key0 >> 63)); 231 runningmod = modifier; 232 workingval = data ^ key0; 233 234 for (i = 0; i <= 4; ++i) { 235 roundkey = key1 ^ runningmod; 236 workingval ^= roundkey; 237 workingval ^= RC[i]; 238 if (i > 0) { 239 workingval = pac_cell_shuffle(workingval); 240 workingval = pac_mult(workingval); 241 } 242 workingval = pac_sub(workingval); 243 runningmod = tweak_shuffle(runningmod); 244 } 245 roundkey = modk0 ^ runningmod; 246 workingval ^= roundkey; 247 workingval = pac_cell_shuffle(workingval); 248 workingval = pac_mult(workingval); 249 workingval = pac_sub(workingval); 250 workingval = pac_cell_shuffle(workingval); 251 workingval = pac_mult(workingval); 252 workingval ^= key1; 253 workingval = pac_cell_inv_shuffle(workingval); 254 workingval = pac_inv_sub(workingval); 255 workingval = pac_mult(workingval); 256 workingval = pac_cell_inv_shuffle(workingval); 257 workingval ^= key0; 258 workingval ^= runningmod; 259 for (i = 0; i <= 4; ++i) { 260 workingval = pac_inv_sub(workingval); 261 if (i < 4) { 262 workingval = pac_mult(workingval); 263 workingval = pac_cell_inv_shuffle(workingval); 264 } 265 runningmod = tweak_inv_shuffle(runningmod); 266 roundkey = key1 ^ runningmod; 267 workingval ^= RC[4 - i]; 268 workingval ^= roundkey; 269 workingval ^= alpha; 270 } 271 workingval ^= modk0; 272 273 return workingval; 274 } 275 276 static uint64_t pauth_computepac_impdef(uint64_t data, uint64_t modifier, 277 ARMPACKey key) 278 { 279 return qemu_xxhash64_4(data, modifier, key.lo, key.hi); 280 } 281 282 static uint64_t pauth_computepac(CPUARMState *env, uint64_t data, 283 uint64_t modifier, ARMPACKey key) 284 { 285 if (cpu_isar_feature(aa64_pauth_arch, env_archcpu(env))) { 286 return pauth_computepac_architected(data, modifier, key); 287 } else { 288 return pauth_computepac_impdef(data, modifier, key); 289 } 290 } 291 292 static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier, 293 ARMPACKey *key, bool data) 294 { 295 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); 296 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); 297 uint64_t pac, ext_ptr, ext, test; 298 int bot_bit, top_bit; 299 300 /* If tagged pointers are in use, use ptr<55>, otherwise ptr<63>. */ 301 if (param.tbi) { 302 ext = sextract64(ptr, 55, 1); 303 } else { 304 ext = sextract64(ptr, 63, 1); 305 } 306 307 /* Build a pointer with known good extension bits. */ 308 top_bit = 64 - 8 * param.tbi; 309 bot_bit = 64 - param.tsz; 310 ext_ptr = deposit64(ptr, bot_bit, top_bit - bot_bit, ext); 311 312 pac = pauth_computepac(env, ext_ptr, modifier, *key); 313 314 /* 315 * Check if the ptr has good extension bits and corrupt the 316 * pointer authentication code if not. 317 */ 318 test = sextract64(ptr, bot_bit, top_bit - bot_bit); 319 if (test != 0 && test != -1) { 320 /* 321 * Note that our top_bit is one greater than the pseudocode's 322 * version, hence "- 2" here. 323 */ 324 pac ^= MAKE_64BIT_MASK(top_bit - 2, 1); 325 } 326 327 /* 328 * Preserve the determination between upper and lower at bit 55, 329 * and insert pointer authentication code. 330 */ 331 if (param.tbi) { 332 ptr &= ~MAKE_64BIT_MASK(bot_bit, 55 - bot_bit + 1); 333 pac &= MAKE_64BIT_MASK(bot_bit, 54 - bot_bit + 1); 334 } else { 335 ptr &= MAKE_64BIT_MASK(0, bot_bit); 336 pac &= ~(MAKE_64BIT_MASK(55, 1) | MAKE_64BIT_MASK(0, bot_bit)); 337 } 338 ext &= MAKE_64BIT_MASK(55, 1); 339 return pac | ext | ptr; 340 } 341 342 static uint64_t pauth_ptr_mask_internal(ARMVAParameters param) 343 { 344 int bot_pac_bit = 64 - param.tsz; 345 int top_pac_bit = 64 - 8 * param.tbi; 346 347 return MAKE_64BIT_MASK(bot_pac_bit, top_pac_bit - bot_pac_bit); 348 } 349 350 static uint64_t pauth_original_ptr(uint64_t ptr, ARMVAParameters param) 351 { 352 uint64_t mask = pauth_ptr_mask_internal(param); 353 354 /* Note that bit 55 is used whether or not the regime has 2 ranges. */ 355 if (extract64(ptr, 55, 1)) { 356 return ptr | mask; 357 } else { 358 return ptr & ~mask; 359 } 360 } 361 362 uint64_t pauth_ptr_mask(CPUARMState *env, uint64_t ptr, bool data) 363 { 364 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); 365 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); 366 367 return pauth_ptr_mask_internal(param); 368 } 369 370 static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier, 371 ARMPACKey *key, bool data, int keynumber) 372 { 373 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); 374 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); 375 int bot_bit, top_bit; 376 uint64_t pac, orig_ptr, test; 377 378 orig_ptr = pauth_original_ptr(ptr, param); 379 pac = pauth_computepac(env, orig_ptr, modifier, *key); 380 bot_bit = 64 - param.tsz; 381 top_bit = 64 - 8 * param.tbi; 382 383 test = (pac ^ ptr) & ~MAKE_64BIT_MASK(55, 1); 384 if (unlikely(extract64(test, bot_bit, top_bit - bot_bit))) { 385 int error_code = (keynumber << 1) | (keynumber ^ 1); 386 if (param.tbi) { 387 return deposit64(orig_ptr, 53, 2, error_code); 388 } else { 389 return deposit64(orig_ptr, 61, 2, error_code); 390 } 391 } 392 return orig_ptr; 393 } 394 395 static uint64_t pauth_strip(CPUARMState *env, uint64_t ptr, bool data) 396 { 397 ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); 398 ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); 399 400 return pauth_original_ptr(ptr, param); 401 } 402 403 static G_NORETURN 404 void pauth_trap(CPUARMState *env, int target_el, uintptr_t ra) 405 { 406 raise_exception_ra(env, EXCP_UDEF, syn_pactrap(), target_el, ra); 407 } 408 409 static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra) 410 { 411 if (el < 2 && arm_is_el2_enabled(env)) { 412 uint64_t hcr = arm_hcr_el2_eff(env); 413 bool trap = !(hcr & HCR_API); 414 if (el == 0) { 415 /* Trap only applies to EL1&0 regime. */ 416 trap &= (hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE); 417 } 418 /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */ 419 if (trap) { 420 pauth_trap(env, 2, ra); 421 } 422 } 423 if (el < 3 && arm_feature(env, ARM_FEATURE_EL3)) { 424 if (!(env->cp15.scr_el3 & SCR_API)) { 425 pauth_trap(env, 3, ra); 426 } 427 } 428 } 429 430 static bool pauth_key_enabled(CPUARMState *env, int el, uint32_t bit) 431 { 432 return (arm_sctlr(env, el) & bit) != 0; 433 } 434 435 uint64_t HELPER(pacia)(CPUARMState *env, uint64_t x, uint64_t y) 436 { 437 int el = arm_current_el(env); 438 if (!pauth_key_enabled(env, el, SCTLR_EnIA)) { 439 return x; 440 } 441 pauth_check_trap(env, el, GETPC()); 442 return pauth_addpac(env, x, y, &env->keys.apia, false); 443 } 444 445 uint64_t HELPER(pacib)(CPUARMState *env, uint64_t x, uint64_t y) 446 { 447 int el = arm_current_el(env); 448 if (!pauth_key_enabled(env, el, SCTLR_EnIB)) { 449 return x; 450 } 451 pauth_check_trap(env, el, GETPC()); 452 return pauth_addpac(env, x, y, &env->keys.apib, false); 453 } 454 455 uint64_t HELPER(pacda)(CPUARMState *env, uint64_t x, uint64_t y) 456 { 457 int el = arm_current_el(env); 458 if (!pauth_key_enabled(env, el, SCTLR_EnDA)) { 459 return x; 460 } 461 pauth_check_trap(env, el, GETPC()); 462 return pauth_addpac(env, x, y, &env->keys.apda, true); 463 } 464 465 uint64_t HELPER(pacdb)(CPUARMState *env, uint64_t x, uint64_t y) 466 { 467 int el = arm_current_el(env); 468 if (!pauth_key_enabled(env, el, SCTLR_EnDB)) { 469 return x; 470 } 471 pauth_check_trap(env, el, GETPC()); 472 return pauth_addpac(env, x, y, &env->keys.apdb, true); 473 } 474 475 uint64_t HELPER(pacga)(CPUARMState *env, uint64_t x, uint64_t y) 476 { 477 uint64_t pac; 478 479 pauth_check_trap(env, arm_current_el(env), GETPC()); 480 pac = pauth_computepac(env, x, y, env->keys.apga); 481 482 return pac & 0xffffffff00000000ull; 483 } 484 485 uint64_t HELPER(autia)(CPUARMState *env, uint64_t x, uint64_t y) 486 { 487 int el = arm_current_el(env); 488 if (!pauth_key_enabled(env, el, SCTLR_EnIA)) { 489 return x; 490 } 491 pauth_check_trap(env, el, GETPC()); 492 return pauth_auth(env, x, y, &env->keys.apia, false, 0); 493 } 494 495 uint64_t HELPER(autib)(CPUARMState *env, uint64_t x, uint64_t y) 496 { 497 int el = arm_current_el(env); 498 if (!pauth_key_enabled(env, el, SCTLR_EnIB)) { 499 return x; 500 } 501 pauth_check_trap(env, el, GETPC()); 502 return pauth_auth(env, x, y, &env->keys.apib, false, 1); 503 } 504 505 uint64_t HELPER(autda)(CPUARMState *env, uint64_t x, uint64_t y) 506 { 507 int el = arm_current_el(env); 508 if (!pauth_key_enabled(env, el, SCTLR_EnDA)) { 509 return x; 510 } 511 pauth_check_trap(env, el, GETPC()); 512 return pauth_auth(env, x, y, &env->keys.apda, true, 0); 513 } 514 515 uint64_t HELPER(autdb)(CPUARMState *env, uint64_t x, uint64_t y) 516 { 517 int el = arm_current_el(env); 518 if (!pauth_key_enabled(env, el, SCTLR_EnDB)) { 519 return x; 520 } 521 pauth_check_trap(env, el, GETPC()); 522 return pauth_auth(env, x, y, &env->keys.apdb, true, 1); 523 } 524 525 uint64_t HELPER(xpaci)(CPUARMState *env, uint64_t a) 526 { 527 return pauth_strip(env, a, false); 528 } 529 530 uint64_t HELPER(xpacd)(CPUARMState *env, uint64_t a) 531 { 532 return pauth_strip(env, a, true); 533 } 534