1/* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Andrzej Zaborowski 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25#include "elf.h" 26 27int arm_arch = __ARM_ARCH; 28 29#ifndef use_idiv_instructions 30bool use_idiv_instructions; 31#endif 32#ifndef use_neon_instructions 33bool use_neon_instructions; 34#endif 35 36/* Used for function call generation. */ 37#define TCG_TARGET_STACK_ALIGN 8 38#define TCG_TARGET_CALL_STACK_OFFSET 0 39#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL 40#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN 41#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_EVEN 42#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_BY_REF 43 44#ifdef CONFIG_DEBUG_TCG 45static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { 46 "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", 47 "%r8", "%r9", "%r10", "%r11", "%r12", "%sp", "%r14", "%pc", 48 "%q0", "%q1", "%q2", "%q3", "%q4", "%q5", "%q6", "%q7", 49 "%q8", "%q9", "%q10", "%q11", "%q12", "%q13", "%q14", "%q15", 50}; 51#endif 52 53static const int tcg_target_reg_alloc_order[] = { 54 TCG_REG_R4, 55 TCG_REG_R5, 56 TCG_REG_R6, 57 TCG_REG_R7, 58 TCG_REG_R8, 59 TCG_REG_R9, 60 TCG_REG_R10, 61 TCG_REG_R11, 62 TCG_REG_R13, 63 TCG_REG_R0, 64 TCG_REG_R1, 65 TCG_REG_R2, 66 TCG_REG_R3, 67 TCG_REG_R12, 68 TCG_REG_R14, 69 70 TCG_REG_Q0, 71 TCG_REG_Q1, 72 TCG_REG_Q2, 73 TCG_REG_Q3, 74 /* Q4 - Q7 are call-saved, and skipped. */ 75 TCG_REG_Q8, 76 TCG_REG_Q9, 77 TCG_REG_Q10, 78 TCG_REG_Q11, 79 TCG_REG_Q12, 80 TCG_REG_Q13, 81 TCG_REG_Q14, 82 TCG_REG_Q15, 83}; 84 85static const int tcg_target_call_iarg_regs[4] = { 86 TCG_REG_R0, TCG_REG_R1, TCG_REG_R2, TCG_REG_R3 87}; 88 89static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) 90{ 91 tcg_debug_assert(kind == TCG_CALL_RET_NORMAL); 92 tcg_debug_assert(slot >= 0 && slot <= 3); 93 return TCG_REG_R0 + slot; 94} 95 96#define TCG_REG_TMP TCG_REG_R12 97#define TCG_VEC_TMP TCG_REG_Q15 98#define TCG_REG_GUEST_BASE TCG_REG_R11 99 100typedef enum { 101 COND_EQ = 0x0, 102 COND_NE = 0x1, 103 COND_CS = 0x2, /* Unsigned greater or equal */ 104 COND_CC = 0x3, /* Unsigned less than */ 105 COND_MI = 0x4, /* Negative */ 106 COND_PL = 0x5, /* Zero or greater */ 107 COND_VS = 0x6, /* Overflow */ 108 COND_VC = 0x7, /* No overflow */ 109 COND_HI = 0x8, /* Unsigned greater than */ 110 COND_LS = 0x9, /* Unsigned less or equal */ 111 COND_GE = 0xa, 112 COND_LT = 0xb, 113 COND_GT = 0xc, 114 COND_LE = 0xd, 115 COND_AL = 0xe, 116} ARMCond; 117 118#define TO_CPSR (1 << 20) 119 120#define SHIFT_IMM_LSL(im) (((im) << 7) | 0x00) 121#define SHIFT_IMM_LSR(im) (((im) << 7) | 0x20) 122#define SHIFT_IMM_ASR(im) (((im) << 7) | 0x40) 123#define SHIFT_IMM_ROR(im) (((im) << 7) | 0x60) 124#define SHIFT_REG_LSL(rs) (((rs) << 8) | 0x10) 125#define SHIFT_REG_LSR(rs) (((rs) << 8) | 0x30) 126#define SHIFT_REG_ASR(rs) (((rs) << 8) | 0x50) 127#define SHIFT_REG_ROR(rs) (((rs) << 8) | 0x70) 128 129typedef enum { 130 ARITH_AND = 0x0 << 21, 131 ARITH_EOR = 0x1 << 21, 132 ARITH_SUB = 0x2 << 21, 133 ARITH_RSB = 0x3 << 21, 134 ARITH_ADD = 0x4 << 21, 135 ARITH_ADC = 0x5 << 21, 136 ARITH_SBC = 0x6 << 21, 137 ARITH_RSC = 0x7 << 21, 138 ARITH_TST = 0x8 << 21 | TO_CPSR, 139 ARITH_CMP = 0xa << 21 | TO_CPSR, 140 ARITH_CMN = 0xb << 21 | TO_CPSR, 141 ARITH_ORR = 0xc << 21, 142 ARITH_MOV = 0xd << 21, 143 ARITH_BIC = 0xe << 21, 144 ARITH_MVN = 0xf << 21, 145 146 INSN_B = 0x0a000000, 147 148 INSN_CLZ = 0x016f0f10, 149 INSN_RBIT = 0x06ff0f30, 150 151 INSN_LDMIA = 0x08b00000, 152 INSN_STMDB = 0x09200000, 153 154 INSN_LDR_IMM = 0x04100000, 155 INSN_LDR_REG = 0x06100000, 156 INSN_STR_IMM = 0x04000000, 157 INSN_STR_REG = 0x06000000, 158 159 INSN_LDRH_IMM = 0x005000b0, 160 INSN_LDRH_REG = 0x001000b0, 161 INSN_LDRSH_IMM = 0x005000f0, 162 INSN_LDRSH_REG = 0x001000f0, 163 INSN_STRH_IMM = 0x004000b0, 164 INSN_STRH_REG = 0x000000b0, 165 166 INSN_LDRB_IMM = 0x04500000, 167 INSN_LDRB_REG = 0x06500000, 168 INSN_LDRSB_IMM = 0x005000d0, 169 INSN_LDRSB_REG = 0x001000d0, 170 INSN_STRB_IMM = 0x04400000, 171 INSN_STRB_REG = 0x06400000, 172 173 INSN_LDRD_IMM = 0x004000d0, 174 INSN_LDRD_REG = 0x000000d0, 175 INSN_STRD_IMM = 0x004000f0, 176 INSN_STRD_REG = 0x000000f0, 177 178 INSN_DMB_ISH = 0xf57ff05b, 179 INSN_DMB_MCR = 0xee070fba, 180 181 INSN_MSRI_CPSR = 0x0360f000, 182 183 /* Architected nop introduced in v6k. */ 184 /* ??? This is an MSR (imm) 0,0,0 insn. Anyone know if this 185 also Just So Happened to do nothing on pre-v6k so that we 186 don't need to conditionalize it? */ 187 INSN_NOP_v6k = 0xe320f000, 188 /* Otherwise the assembler uses mov r0,r0 */ 189 INSN_NOP_v4 = (COND_AL << 28) | ARITH_MOV, 190 191 INSN_VADD = 0xf2000800, 192 INSN_VAND = 0xf2000110, 193 INSN_VBIC = 0xf2100110, 194 INSN_VEOR = 0xf3000110, 195 INSN_VORN = 0xf2300110, 196 INSN_VORR = 0xf2200110, 197 INSN_VSUB = 0xf3000800, 198 INSN_VMUL = 0xf2000910, 199 INSN_VQADD = 0xf2000010, 200 INSN_VQADD_U = 0xf3000010, 201 INSN_VQSUB = 0xf2000210, 202 INSN_VQSUB_U = 0xf3000210, 203 INSN_VMAX = 0xf2000600, 204 INSN_VMAX_U = 0xf3000600, 205 INSN_VMIN = 0xf2000610, 206 INSN_VMIN_U = 0xf3000610, 207 208 INSN_VABS = 0xf3b10300, 209 INSN_VMVN = 0xf3b00580, 210 INSN_VNEG = 0xf3b10380, 211 212 INSN_VCEQ0 = 0xf3b10100, 213 INSN_VCGT0 = 0xf3b10000, 214 INSN_VCGE0 = 0xf3b10080, 215 INSN_VCLE0 = 0xf3b10180, 216 INSN_VCLT0 = 0xf3b10200, 217 218 INSN_VCEQ = 0xf3000810, 219 INSN_VCGE = 0xf2000310, 220 INSN_VCGT = 0xf2000300, 221 INSN_VCGE_U = 0xf3000310, 222 INSN_VCGT_U = 0xf3000300, 223 224 INSN_VSHLI = 0xf2800510, /* VSHL (immediate) */ 225 INSN_VSARI = 0xf2800010, /* VSHR.S */ 226 INSN_VSHRI = 0xf3800010, /* VSHR.U */ 227 INSN_VSLI = 0xf3800510, 228 INSN_VSHL_S = 0xf2000400, /* VSHL.S (register) */ 229 INSN_VSHL_U = 0xf3000400, /* VSHL.U (register) */ 230 231 INSN_VBSL = 0xf3100110, 232 INSN_VBIT = 0xf3200110, 233 INSN_VBIF = 0xf3300110, 234 235 INSN_VTST = 0xf2000810, 236 237 INSN_VDUP_G = 0xee800b10, /* VDUP (ARM core register) */ 238 INSN_VDUP_S = 0xf3b00c00, /* VDUP (scalar) */ 239 INSN_VLDR_D = 0xed100b00, /* VLDR.64 */ 240 INSN_VLD1 = 0xf4200000, /* VLD1 (multiple single elements) */ 241 INSN_VLD1R = 0xf4a00c00, /* VLD1 (single element to all lanes) */ 242 INSN_VST1 = 0xf4000000, /* VST1 (multiple single elements) */ 243 INSN_VMOVI = 0xf2800010, /* VMOV (immediate) */ 244} ARMInsn; 245 246#define INSN_NOP (use_armv7_instructions ? INSN_NOP_v6k : INSN_NOP_v4) 247 248static const uint8_t tcg_cond_to_arm_cond[] = { 249 [TCG_COND_EQ] = COND_EQ, 250 [TCG_COND_NE] = COND_NE, 251 [TCG_COND_LT] = COND_LT, 252 [TCG_COND_GE] = COND_GE, 253 [TCG_COND_LE] = COND_LE, 254 [TCG_COND_GT] = COND_GT, 255 /* unsigned */ 256 [TCG_COND_LTU] = COND_CC, 257 [TCG_COND_GEU] = COND_CS, 258 [TCG_COND_LEU] = COND_LS, 259 [TCG_COND_GTU] = COND_HI, 260}; 261 262static int encode_imm(uint32_t imm); 263 264/* TCG private relocation type: add with pc+imm8 */ 265#define R_ARM_PC8 11 266 267/* TCG private relocation type: vldr with imm8 << 2 */ 268#define R_ARM_PC11 12 269 270static bool reloc_pc24(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 271{ 272 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 273 ptrdiff_t offset = (tcg_ptr_byte_diff(target, src_rx) - 8) >> 2; 274 275 if (offset == sextract32(offset, 0, 24)) { 276 *src_rw = deposit32(*src_rw, 0, 24, offset); 277 return true; 278 } 279 return false; 280} 281 282static bool reloc_pc13(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 283{ 284 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 285 ptrdiff_t offset = tcg_ptr_byte_diff(target, src_rx) - 8; 286 287 if (offset >= -0xfff && offset <= 0xfff) { 288 tcg_insn_unit insn = *src_rw; 289 bool u = (offset >= 0); 290 if (!u) { 291 offset = -offset; 292 } 293 insn = deposit32(insn, 23, 1, u); 294 insn = deposit32(insn, 0, 12, offset); 295 *src_rw = insn; 296 return true; 297 } 298 return false; 299} 300 301static bool reloc_pc11(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 302{ 303 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 304 ptrdiff_t offset = (tcg_ptr_byte_diff(target, src_rx) - 8) / 4; 305 306 if (offset >= -0xff && offset <= 0xff) { 307 tcg_insn_unit insn = *src_rw; 308 bool u = (offset >= 0); 309 if (!u) { 310 offset = -offset; 311 } 312 insn = deposit32(insn, 23, 1, u); 313 insn = deposit32(insn, 0, 8, offset); 314 *src_rw = insn; 315 return true; 316 } 317 return false; 318} 319 320static bool reloc_pc8(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 321{ 322 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 323 ptrdiff_t offset = tcg_ptr_byte_diff(target, src_rx) - 8; 324 int imm12 = encode_imm(offset); 325 326 if (imm12 >= 0) { 327 *src_rw = deposit32(*src_rw, 0, 12, imm12); 328 return true; 329 } 330 return false; 331} 332 333static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 334 intptr_t value, intptr_t addend) 335{ 336 tcg_debug_assert(addend == 0); 337 switch (type) { 338 case R_ARM_PC24: 339 return reloc_pc24(code_ptr, (const tcg_insn_unit *)value); 340 case R_ARM_PC13: 341 return reloc_pc13(code_ptr, (const tcg_insn_unit *)value); 342 case R_ARM_PC11: 343 return reloc_pc11(code_ptr, (const tcg_insn_unit *)value); 344 case R_ARM_PC8: 345 return reloc_pc8(code_ptr, (const tcg_insn_unit *)value); 346 default: 347 g_assert_not_reached(); 348 } 349} 350 351#define TCG_CT_CONST_ARM 0x100 352#define TCG_CT_CONST_INV 0x200 353#define TCG_CT_CONST_NEG 0x400 354#define TCG_CT_CONST_ZERO 0x800 355#define TCG_CT_CONST_ORRI 0x1000 356#define TCG_CT_CONST_ANDI 0x2000 357 358#define ALL_GENERAL_REGS 0xffffu 359#define ALL_VECTOR_REGS 0xffff0000u 360 361/* 362 * r0-r3 will be overwritten when reading the tlb entry (system-mode only); 363 * r14 will be overwritten by the BLNE branching to the slow path. 364 */ 365#define ALL_QLDST_REGS \ 366 (ALL_GENERAL_REGS & ~((tcg_use_softmmu ? 0xf : 0) | (1 << TCG_REG_R14))) 367 368/* 369 * ARM immediates for ALU instructions are made of an unsigned 8-bit 370 * right-rotated by an even amount between 0 and 30. 371 * 372 * Return < 0 if @imm cannot be encoded, else the entire imm12 field. 373 */ 374static int encode_imm(uint32_t imm) 375{ 376 uint32_t rot, imm8; 377 378 /* Simple case, no rotation required. */ 379 if ((imm & ~0xff) == 0) { 380 return imm; 381 } 382 383 /* Next, try a simple even shift. */ 384 rot = ctz32(imm) & ~1; 385 imm8 = imm >> rot; 386 rot = 32 - rot; 387 if ((imm8 & ~0xff) == 0) { 388 goto found; 389 } 390 391 /* 392 * Finally, try harder with rotations. 393 * The ctz test above will have taken care of rotates >= 8. 394 */ 395 for (rot = 2; rot < 8; rot += 2) { 396 imm8 = rol32(imm, rot); 397 if ((imm8 & ~0xff) == 0) { 398 goto found; 399 } 400 } 401 /* Fail: imm cannot be encoded. */ 402 return -1; 403 404 found: 405 /* Note that rot is even, and we discard bit 0 by shifting by 7. */ 406 return rot << 7 | imm8; 407} 408 409static int encode_imm_nofail(uint32_t imm) 410{ 411 int ret = encode_imm(imm); 412 tcg_debug_assert(ret >= 0); 413 return ret; 414} 415 416static bool check_fit_imm(uint32_t imm) 417{ 418 return encode_imm(imm) >= 0; 419} 420 421/* Return true if v16 is a valid 16-bit shifted immediate. */ 422static bool is_shimm16(uint16_t v16, int *cmode, int *imm8) 423{ 424 if (v16 == (v16 & 0xff)) { 425 *cmode = 0x8; 426 *imm8 = v16 & 0xff; 427 return true; 428 } else if (v16 == (v16 & 0xff00)) { 429 *cmode = 0xa; 430 *imm8 = v16 >> 8; 431 return true; 432 } 433 return false; 434} 435 436/* Return true if v32 is a valid 32-bit shifted immediate. */ 437static bool is_shimm32(uint32_t v32, int *cmode, int *imm8) 438{ 439 if (v32 == (v32 & 0xff)) { 440 *cmode = 0x0; 441 *imm8 = v32 & 0xff; 442 return true; 443 } else if (v32 == (v32 & 0xff00)) { 444 *cmode = 0x2; 445 *imm8 = (v32 >> 8) & 0xff; 446 return true; 447 } else if (v32 == (v32 & 0xff0000)) { 448 *cmode = 0x4; 449 *imm8 = (v32 >> 16) & 0xff; 450 return true; 451 } else if (v32 == (v32 & 0xff000000)) { 452 *cmode = 0x6; 453 *imm8 = v32 >> 24; 454 return true; 455 } 456 return false; 457} 458 459/* Return true if v32 is a valid 32-bit shifting ones immediate. */ 460static bool is_soimm32(uint32_t v32, int *cmode, int *imm8) 461{ 462 if ((v32 & 0xffff00ff) == 0xff) { 463 *cmode = 0xc; 464 *imm8 = (v32 >> 8) & 0xff; 465 return true; 466 } else if ((v32 & 0xff00ffff) == 0xffff) { 467 *cmode = 0xd; 468 *imm8 = (v32 >> 16) & 0xff; 469 return true; 470 } 471 return false; 472} 473 474/* 475 * Return non-zero if v32 can be formed by MOVI+ORR. 476 * Place the parameters for MOVI in (cmode, imm8). 477 * Return the cmode for ORR; the imm8 can be had via extraction from v32. 478 */ 479static int is_shimm32_pair(uint32_t v32, int *cmode, int *imm8) 480{ 481 int i; 482 483 for (i = 6; i > 0; i -= 2) { 484 /* Mask out one byte we can add with ORR. */ 485 uint32_t tmp = v32 & ~(0xffu << (i * 4)); 486 if (is_shimm32(tmp, cmode, imm8) || 487 is_soimm32(tmp, cmode, imm8)) { 488 break; 489 } 490 } 491 return i; 492} 493 494/* Return true if V is a valid 16-bit or 32-bit shifted immediate. */ 495static bool is_shimm1632(uint32_t v32, int *cmode, int *imm8) 496{ 497 if (v32 == deposit32(v32, 16, 16, v32)) { 498 return is_shimm16(v32, cmode, imm8); 499 } else { 500 return is_shimm32(v32, cmode, imm8); 501 } 502} 503 504/* Test if a constant matches the constraint. 505 * TODO: define constraints for: 506 * 507 * ldr/str offset: between -0xfff and 0xfff 508 * ldrh/strh offset: between -0xff and 0xff 509 * mov operand2: values represented with x << (2 * y), x < 0x100 510 * add, sub, eor...: ditto 511 */ 512static bool tcg_target_const_match(int64_t val, int ct, 513 TCGType type, TCGCond cond, int vece) 514{ 515 if (ct & TCG_CT_CONST) { 516 return 1; 517 } else if ((ct & TCG_CT_CONST_ARM) && check_fit_imm(val)) { 518 return 1; 519 } else if ((ct & TCG_CT_CONST_INV) && check_fit_imm(~val)) { 520 return 1; 521 } else if ((ct & TCG_CT_CONST_NEG) && check_fit_imm(-val)) { 522 return 1; 523 } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) { 524 return 1; 525 } 526 527 switch (ct & (TCG_CT_CONST_ORRI | TCG_CT_CONST_ANDI)) { 528 case 0: 529 break; 530 case TCG_CT_CONST_ANDI: 531 val = ~val; 532 /* fallthru */ 533 case TCG_CT_CONST_ORRI: 534 if (val == deposit64(val, 32, 32, val)) { 535 int cmode, imm8; 536 return is_shimm1632(val, &cmode, &imm8); 537 } 538 break; 539 default: 540 /* Both bits should not be set for the same insn. */ 541 g_assert_not_reached(); 542 } 543 544 return 0; 545} 546 547static void tcg_out_b_imm(TCGContext *s, ARMCond cond, int32_t offset) 548{ 549 tcg_out32(s, (cond << 28) | INSN_B | 550 (((offset - 8) >> 2) & 0x00ffffff)); 551} 552 553static void tcg_out_bl_imm(TCGContext *s, ARMCond cond, int32_t offset) 554{ 555 tcg_out32(s, (cond << 28) | 0x0b000000 | 556 (((offset - 8) >> 2) & 0x00ffffff)); 557} 558 559static void tcg_out_blx_reg(TCGContext *s, ARMCond cond, TCGReg rn) 560{ 561 tcg_out32(s, (cond << 28) | 0x012fff30 | rn); 562} 563 564static void tcg_out_blx_imm(TCGContext *s, int32_t offset) 565{ 566 tcg_out32(s, 0xfa000000 | ((offset & 2) << 23) | 567 (((offset - 8) >> 2) & 0x00ffffff)); 568} 569 570static void tcg_out_dat_reg(TCGContext *s, ARMCond cond, ARMInsn opc, 571 TCGReg rd, TCGReg rn, TCGReg rm, int shift) 572{ 573 tcg_out32(s, (cond << 28) | (0 << 25) | opc | 574 (rn << 16) | (rd << 12) | shift | rm); 575} 576 577static void tcg_out_mov_reg(TCGContext *s, ARMCond cond, TCGReg rd, TCGReg rm) 578{ 579 /* Simple reg-reg move, optimising out the 'do nothing' case */ 580 if (rd != rm) { 581 tcg_out_dat_reg(s, cond, ARITH_MOV, rd, 0, rm, SHIFT_IMM_LSL(0)); 582 } 583} 584 585static void tcg_out_bx_reg(TCGContext *s, ARMCond cond, TCGReg rn) 586{ 587 tcg_out32(s, (cond << 28) | 0x012fff10 | rn); 588} 589 590static void tcg_out_b_reg(TCGContext *s, ARMCond cond, TCGReg rn) 591{ 592 /* 593 * Unless the C portion of QEMU is compiled as thumb, we don't need 594 * true BX semantics; merely a branch to an address held in a register. 595 */ 596 tcg_out_bx_reg(s, cond, rn); 597} 598 599static void tcg_out_dat_imm(TCGContext *s, ARMCond cond, ARMInsn opc, 600 TCGReg rd, TCGReg rn, int im) 601{ 602 tcg_out32(s, (cond << 28) | (1 << 25) | opc | 603 (rn << 16) | (rd << 12) | im); 604} 605 606static void tcg_out_ldstm(TCGContext *s, ARMCond cond, ARMInsn opc, 607 TCGReg rn, uint16_t mask) 608{ 609 tcg_out32(s, (cond << 28) | opc | (rn << 16) | mask); 610} 611 612/* Note that this routine is used for both LDR and LDRH formats, so we do 613 not wish to include an immediate shift at this point. */ 614static void tcg_out_memop_r(TCGContext *s, ARMCond cond, ARMInsn opc, TCGReg rt, 615 TCGReg rn, TCGReg rm, bool u, bool p, bool w) 616{ 617 tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24) 618 | (w << 21) | (rn << 16) | (rt << 12) | rm); 619} 620 621static void tcg_out_memop_8(TCGContext *s, ARMCond cond, ARMInsn opc, TCGReg rt, 622 TCGReg rn, int imm8, bool p, bool w) 623{ 624 bool u = 1; 625 if (imm8 < 0) { 626 imm8 = -imm8; 627 u = 0; 628 } 629 tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24) | (w << 21) | 630 (rn << 16) | (rt << 12) | ((imm8 & 0xf0) << 4) | (imm8 & 0xf)); 631} 632 633static void tcg_out_memop_12(TCGContext *s, ARMCond cond, ARMInsn opc, 634 TCGReg rt, TCGReg rn, int imm12, bool p, bool w) 635{ 636 bool u = 1; 637 if (imm12 < 0) { 638 imm12 = -imm12; 639 u = 0; 640 } 641 tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24) | (w << 21) | 642 (rn << 16) | (rt << 12) | imm12); 643} 644 645static void tcg_out_ld32_12(TCGContext *s, ARMCond cond, TCGReg rt, 646 TCGReg rn, int imm12) 647{ 648 tcg_out_memop_12(s, cond, INSN_LDR_IMM, rt, rn, imm12, 1, 0); 649} 650 651static void tcg_out_st32_12(TCGContext *s, ARMCond cond, TCGReg rt, 652 TCGReg rn, int imm12) 653{ 654 tcg_out_memop_12(s, cond, INSN_STR_IMM, rt, rn, imm12, 1, 0); 655} 656 657static void tcg_out_ld32_r(TCGContext *s, ARMCond cond, TCGReg rt, 658 TCGReg rn, TCGReg rm) 659{ 660 tcg_out_memop_r(s, cond, INSN_LDR_REG, rt, rn, rm, 1, 1, 0); 661} 662 663static void tcg_out_st32_r(TCGContext *s, ARMCond cond, TCGReg rt, 664 TCGReg rn, TCGReg rm) 665{ 666 tcg_out_memop_r(s, cond, INSN_STR_REG, rt, rn, rm, 1, 1, 0); 667} 668 669static void tcg_out_ldrd_8(TCGContext *s, ARMCond cond, TCGReg rt, 670 TCGReg rn, int imm8) 671{ 672 tcg_out_memop_8(s, cond, INSN_LDRD_IMM, rt, rn, imm8, 1, 0); 673} 674 675static void tcg_out_ldrd_r(TCGContext *s, ARMCond cond, TCGReg rt, 676 TCGReg rn, TCGReg rm) 677{ 678 tcg_out_memop_r(s, cond, INSN_LDRD_REG, rt, rn, rm, 1, 1, 0); 679} 680 681static void tcg_out_strd_8(TCGContext *s, ARMCond cond, TCGReg rt, 682 TCGReg rn, int imm8) 683{ 684 tcg_out_memop_8(s, cond, INSN_STRD_IMM, rt, rn, imm8, 1, 0); 685} 686 687static void tcg_out_strd_r(TCGContext *s, ARMCond cond, TCGReg rt, 688 TCGReg rn, TCGReg rm) 689{ 690 tcg_out_memop_r(s, cond, INSN_STRD_REG, rt, rn, rm, 1, 1, 0); 691} 692 693/* Register pre-increment with base writeback. */ 694static void tcg_out_ld32_rwb(TCGContext *s, ARMCond cond, TCGReg rt, 695 TCGReg rn, TCGReg rm) 696{ 697 tcg_out_memop_r(s, cond, INSN_LDR_REG, rt, rn, rm, 1, 1, 1); 698} 699 700static void tcg_out_st32_rwb(TCGContext *s, ARMCond cond, TCGReg rt, 701 TCGReg rn, TCGReg rm) 702{ 703 tcg_out_memop_r(s, cond, INSN_STR_REG, rt, rn, rm, 1, 1, 1); 704} 705 706static void tcg_out_ld16u_8(TCGContext *s, ARMCond cond, TCGReg rt, 707 TCGReg rn, int imm8) 708{ 709 tcg_out_memop_8(s, cond, INSN_LDRH_IMM, rt, rn, imm8, 1, 0); 710} 711 712static void tcg_out_st16_8(TCGContext *s, ARMCond cond, TCGReg rt, 713 TCGReg rn, int imm8) 714{ 715 tcg_out_memop_8(s, cond, INSN_STRH_IMM, rt, rn, imm8, 1, 0); 716} 717 718static void tcg_out_ld16u_r(TCGContext *s, ARMCond cond, TCGReg rt, 719 TCGReg rn, TCGReg rm) 720{ 721 tcg_out_memop_r(s, cond, INSN_LDRH_REG, rt, rn, rm, 1, 1, 0); 722} 723 724static void tcg_out_st16_r(TCGContext *s, ARMCond cond, TCGReg rt, 725 TCGReg rn, TCGReg rm) 726{ 727 tcg_out_memop_r(s, cond, INSN_STRH_REG, rt, rn, rm, 1, 1, 0); 728} 729 730static void tcg_out_ld16s_8(TCGContext *s, ARMCond cond, TCGReg rt, 731 TCGReg rn, int imm8) 732{ 733 tcg_out_memop_8(s, cond, INSN_LDRSH_IMM, rt, rn, imm8, 1, 0); 734} 735 736static void tcg_out_ld16s_r(TCGContext *s, ARMCond cond, TCGReg rt, 737 TCGReg rn, TCGReg rm) 738{ 739 tcg_out_memop_r(s, cond, INSN_LDRSH_REG, rt, rn, rm, 1, 1, 0); 740} 741 742static void tcg_out_ld8_12(TCGContext *s, ARMCond cond, TCGReg rt, 743 TCGReg rn, int imm12) 744{ 745 tcg_out_memop_12(s, cond, INSN_LDRB_IMM, rt, rn, imm12, 1, 0); 746} 747 748static void tcg_out_st8_12(TCGContext *s, ARMCond cond, TCGReg rt, 749 TCGReg rn, int imm12) 750{ 751 tcg_out_memop_12(s, cond, INSN_STRB_IMM, rt, rn, imm12, 1, 0); 752} 753 754static void tcg_out_ld8_r(TCGContext *s, ARMCond cond, TCGReg rt, 755 TCGReg rn, TCGReg rm) 756{ 757 tcg_out_memop_r(s, cond, INSN_LDRB_REG, rt, rn, rm, 1, 1, 0); 758} 759 760static void tcg_out_st8_r(TCGContext *s, ARMCond cond, TCGReg rt, 761 TCGReg rn, TCGReg rm) 762{ 763 tcg_out_memop_r(s, cond, INSN_STRB_REG, rt, rn, rm, 1, 1, 0); 764} 765 766static void tcg_out_ld8s_8(TCGContext *s, ARMCond cond, TCGReg rt, 767 TCGReg rn, int imm8) 768{ 769 tcg_out_memop_8(s, cond, INSN_LDRSB_IMM, rt, rn, imm8, 1, 0); 770} 771 772static void tcg_out_ld8s_r(TCGContext *s, ARMCond cond, TCGReg rt, 773 TCGReg rn, TCGReg rm) 774{ 775 tcg_out_memop_r(s, cond, INSN_LDRSB_REG, rt, rn, rm, 1, 1, 0); 776} 777 778static void tcg_out_movi_pool(TCGContext *s, ARMCond cond, 779 TCGReg rd, uint32_t arg) 780{ 781 new_pool_label(s, arg, R_ARM_PC13, s->code_ptr, 0); 782 tcg_out_ld32_12(s, cond, rd, TCG_REG_PC, 0); 783} 784 785static void tcg_out_movi32(TCGContext *s, ARMCond cond, 786 TCGReg rd, uint32_t arg) 787{ 788 int imm12, diff, opc, sh1, sh2; 789 uint32_t tt0, tt1, tt2; 790 791 /* Check a single MOV/MVN before anything else. */ 792 imm12 = encode_imm(arg); 793 if (imm12 >= 0) { 794 tcg_out_dat_imm(s, cond, ARITH_MOV, rd, 0, imm12); 795 return; 796 } 797 imm12 = encode_imm(~arg); 798 if (imm12 >= 0) { 799 tcg_out_dat_imm(s, cond, ARITH_MVN, rd, 0, imm12); 800 return; 801 } 802 803 /* Check for a pc-relative address. This will usually be the TB, 804 or within the TB, which is immediately before the code block. */ 805 diff = tcg_pcrel_diff(s, (void *)arg) - 8; 806 if (diff >= 0) { 807 imm12 = encode_imm(diff); 808 if (imm12 >= 0) { 809 tcg_out_dat_imm(s, cond, ARITH_ADD, rd, TCG_REG_PC, imm12); 810 return; 811 } 812 } else { 813 imm12 = encode_imm(-diff); 814 if (imm12 >= 0) { 815 tcg_out_dat_imm(s, cond, ARITH_SUB, rd, TCG_REG_PC, imm12); 816 return; 817 } 818 } 819 820 /* Use movw + movt. */ 821 if (use_armv7_instructions) { 822 /* movw */ 823 tcg_out32(s, (cond << 28) | 0x03000000 | (rd << 12) 824 | ((arg << 4) & 0x000f0000) | (arg & 0xfff)); 825 if (arg & 0xffff0000) { 826 /* movt */ 827 tcg_out32(s, (cond << 28) | 0x03400000 | (rd << 12) 828 | ((arg >> 12) & 0x000f0000) | ((arg >> 16) & 0xfff)); 829 } 830 return; 831 } 832 833 /* Look for sequences of two insns. If we have lots of 1's, we can 834 shorten the sequence by beginning with mvn and then clearing 835 higher bits with eor. */ 836 tt0 = arg; 837 opc = ARITH_MOV; 838 if (ctpop32(arg) > 16) { 839 tt0 = ~arg; 840 opc = ARITH_MVN; 841 } 842 sh1 = ctz32(tt0) & ~1; 843 tt1 = tt0 & ~(0xff << sh1); 844 sh2 = ctz32(tt1) & ~1; 845 tt2 = tt1 & ~(0xff << sh2); 846 if (tt2 == 0) { 847 int rot; 848 849 rot = ((32 - sh1) << 7) & 0xf00; 850 tcg_out_dat_imm(s, cond, opc, rd, 0, ((tt0 >> sh1) & 0xff) | rot); 851 rot = ((32 - sh2) << 7) & 0xf00; 852 tcg_out_dat_imm(s, cond, ARITH_EOR, rd, rd, 853 ((tt0 >> sh2) & 0xff) | rot); 854 return; 855 } 856 857 /* Otherwise, drop it into the constant pool. */ 858 tcg_out_movi_pool(s, cond, rd, arg); 859} 860 861/* 862 * Emit either the reg,imm or reg,reg form of a data-processing insn. 863 * rhs must satisfy the "rI" constraint. 864 */ 865static void tcg_out_dat_rI(TCGContext *s, ARMCond cond, ARMInsn opc, 866 TCGReg dst, TCGReg lhs, TCGArg rhs, int rhs_is_const) 867{ 868 if (rhs_is_const) { 869 tcg_out_dat_imm(s, cond, opc, dst, lhs, encode_imm_nofail(rhs)); 870 } else { 871 tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0)); 872 } 873} 874 875/* 876 * Emit either the reg,imm or reg,reg form of a data-processing insn. 877 * rhs must satisfy the "rIK" constraint. 878 */ 879static void tcg_out_dat_IK(TCGContext *s, ARMCond cond, ARMInsn opc, 880 ARMInsn opinv, TCGReg dst, TCGReg lhs, TCGArg rhs) 881{ 882 int imm12 = encode_imm(rhs); 883 if (imm12 < 0) { 884 imm12 = encode_imm_nofail(~rhs); 885 opc = opinv; 886 } 887 tcg_out_dat_imm(s, cond, opc, dst, lhs, imm12); 888} 889 890static void tcg_out_dat_rIK(TCGContext *s, ARMCond cond, ARMInsn opc, 891 ARMInsn opinv, TCGReg dst, TCGReg lhs, TCGArg rhs, 892 bool rhs_is_const) 893{ 894 if (rhs_is_const) { 895 tcg_out_dat_IK(s, cond, opc, opinv, dst, lhs, rhs); 896 } else { 897 tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0)); 898 } 899} 900 901static void tcg_out_dat_IN(TCGContext *s, ARMCond cond, ARMInsn opc, 902 ARMInsn opneg, TCGReg dst, TCGReg lhs, TCGArg rhs) 903{ 904 int imm12 = encode_imm(rhs); 905 if (imm12 < 0) { 906 imm12 = encode_imm_nofail(-rhs); 907 opc = opneg; 908 } 909 tcg_out_dat_imm(s, cond, opc, dst, lhs, imm12); 910} 911 912static void tcg_out_dat_rIN(TCGContext *s, ARMCond cond, ARMInsn opc, 913 ARMInsn opneg, TCGReg dst, TCGReg lhs, TCGArg rhs, 914 bool rhs_is_const) 915{ 916 /* Emit either the reg,imm or reg,reg form of a data-processing insn. 917 * rhs must satisfy the "rIN" constraint. 918 */ 919 if (rhs_is_const) { 920 tcg_out_dat_IN(s, cond, opc, opneg, dst, lhs, rhs); 921 } else { 922 tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0)); 923 } 924} 925 926static void tcg_out_ext8s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn) 927{ 928 /* sxtb */ 929 tcg_out32(s, 0x06af0070 | (COND_AL << 28) | (rd << 12) | rn); 930} 931 932static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn) 933{ 934 tcg_out_dat_imm(s, COND_AL, ARITH_AND, rd, rn, 0xff); 935} 936 937static void tcg_out_ext16s(TCGContext *s, TCGType t, TCGReg rd, TCGReg rn) 938{ 939 /* sxth */ 940 tcg_out32(s, 0x06bf0070 | (COND_AL << 28) | (rd << 12) | rn); 941} 942 943static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn) 944{ 945 /* uxth */ 946 tcg_out32(s, 0x06ff0070 | (COND_AL << 28) | (rd << 12) | rn); 947} 948 949static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn) 950{ 951 g_assert_not_reached(); 952} 953 954static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn) 955{ 956 g_assert_not_reached(); 957} 958 959static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn) 960{ 961 g_assert_not_reached(); 962} 963 964static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn) 965{ 966 g_assert_not_reached(); 967} 968 969static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn) 970{ 971 g_assert_not_reached(); 972} 973 974static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 975 TCGReg a2, unsigned ofs, unsigned len) 976{ 977 /* bfi/bfc */ 978 tcg_debug_assert(a0 == a1); 979 tcg_out32(s, 0x07c00010 | (COND_AL << 28) | (a0 << 12) | a2 980 | (ofs << 7) | ((ofs + len - 1) << 16)); 981} 982 983static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 984 tcg_target_long a2, unsigned ofs, unsigned len) 985{ 986 /* bfi becomes bfc with rn == 15. */ 987 tgen_deposit(s, type, a0, a1, 15, ofs, len); 988} 989 990static const TCGOutOpDeposit outop_deposit = { 991 .base.static_constraint = C_O1_I2(r, 0, rZ), 992 .out_rrr = tgen_deposit, 993 .out_rri = tgen_depositi, 994}; 995 996static void tgen_extract(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn, 997 unsigned ofs, unsigned len) 998{ 999 /* According to gcc, AND can be faster. */ 1000 if (ofs == 0 && len <= 8) { 1001 tcg_out_dat_imm(s, COND_AL, ARITH_AND, rd, rn, 1002 encode_imm_nofail((1 << len) - 1)); 1003 return; 1004 } 1005 1006 if (use_armv7_instructions) { 1007 /* ubfx */ 1008 tcg_out32(s, 0x07e00050 | (COND_AL << 28) | (rd << 12) | rn 1009 | (ofs << 7) | ((len - 1) << 16)); 1010 return; 1011 } 1012 1013 assert(ofs % 8 == 0); 1014 switch (len) { 1015 case 8: 1016 /* uxtb */ 1017 tcg_out32(s, 0x06ef0070 | (COND_AL << 28) | 1018 (rd << 12) | (ofs << 7) | rn); 1019 break; 1020 case 16: 1021 /* uxth */ 1022 tcg_out32(s, 0x06ff0070 | (COND_AL << 28) | 1023 (rd << 12) | (ofs << 7) | rn); 1024 break; 1025 default: 1026 g_assert_not_reached(); 1027 } 1028} 1029 1030static const TCGOutOpExtract outop_extract = { 1031 .base.static_constraint = C_O1_I1(r, r), 1032 .out_rr = tgen_extract, 1033}; 1034 1035static void tgen_sextract(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn, 1036 unsigned ofs, unsigned len) 1037{ 1038 if (use_armv7_instructions) { 1039 /* sbfx */ 1040 tcg_out32(s, 0x07a00050 | (COND_AL << 28) | (rd << 12) | rn 1041 | (ofs << 7) | ((len - 1) << 16)); 1042 return; 1043 } 1044 1045 assert(ofs % 8 == 0); 1046 switch (len) { 1047 case 8: 1048 /* sxtb */ 1049 tcg_out32(s, 0x06af0070 | (COND_AL << 28) | 1050 (rd << 12) | (ofs << 7) | rn); 1051 break; 1052 case 16: 1053 /* sxth */ 1054 tcg_out32(s, 0x06bf0070 | (COND_AL << 28) | 1055 (rd << 12) | (ofs << 7) | rn); 1056 break; 1057 default: 1058 g_assert_not_reached(); 1059 } 1060} 1061 1062static const TCGOutOpExtract outop_sextract = { 1063 .base.static_constraint = C_O1_I1(r, r), 1064 .out_rr = tgen_sextract, 1065}; 1066 1067 1068static void tcg_out_ld32u(TCGContext *s, ARMCond cond, 1069 TCGReg rd, TCGReg rn, int32_t offset) 1070{ 1071 if (offset > 0xfff || offset < -0xfff) { 1072 tcg_out_movi32(s, cond, TCG_REG_TMP, offset); 1073 tcg_out_ld32_r(s, cond, rd, rn, TCG_REG_TMP); 1074 } else 1075 tcg_out_ld32_12(s, cond, rd, rn, offset); 1076} 1077 1078static void tcg_out_st32(TCGContext *s, ARMCond cond, 1079 TCGReg rd, TCGReg rn, int32_t offset) 1080{ 1081 if (offset > 0xfff || offset < -0xfff) { 1082 tcg_out_movi32(s, cond, TCG_REG_TMP, offset); 1083 tcg_out_st32_r(s, cond, rd, rn, TCG_REG_TMP); 1084 } else 1085 tcg_out_st32_12(s, cond, rd, rn, offset); 1086} 1087 1088/* 1089 * The _goto case is normally between TBs within the same code buffer, and 1090 * with the code buffer limited to 16MB we wouldn't need the long case. 1091 * But we also use it for the tail-call to the qemu_ld/st helpers, which does. 1092 */ 1093static void tcg_out_goto(TCGContext *s, ARMCond cond, const tcg_insn_unit *addr) 1094{ 1095 intptr_t addri = (intptr_t)addr; 1096 ptrdiff_t disp = tcg_pcrel_diff(s, addr); 1097 bool arm_mode = !(addri & 1); 1098 1099 if (arm_mode && disp - 8 < 0x01fffffd && disp - 8 > -0x01fffffd) { 1100 tcg_out_b_imm(s, cond, disp); 1101 return; 1102 } 1103 1104 /* LDR is interworking from v5t. */ 1105 tcg_out_movi_pool(s, cond, TCG_REG_PC, addri); 1106} 1107 1108/* 1109 * The call case is mostly used for helpers - so it's not unreasonable 1110 * for them to be beyond branch range. 1111 */ 1112static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *addr) 1113{ 1114 intptr_t addri = (intptr_t)addr; 1115 ptrdiff_t disp = tcg_pcrel_diff(s, addr); 1116 bool arm_mode = !(addri & 1); 1117 1118 if (disp - 8 < 0x02000000 && disp - 8 >= -0x02000000) { 1119 if (arm_mode) { 1120 tcg_out_bl_imm(s, COND_AL, disp); 1121 } else { 1122 tcg_out_blx_imm(s, disp); 1123 } 1124 return; 1125 } 1126 1127 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, addri); 1128 tcg_out_blx_reg(s, COND_AL, TCG_REG_TMP); 1129} 1130 1131static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr, 1132 const TCGHelperInfo *info) 1133{ 1134 tcg_out_call_int(s, addr); 1135} 1136 1137static void tcg_out_goto_label(TCGContext *s, ARMCond cond, TCGLabel *l) 1138{ 1139 if (l->has_value) { 1140 tcg_out_goto(s, cond, l->u.value_ptr); 1141 } else { 1142 tcg_out_reloc(s, s->code_ptr, R_ARM_PC24, l, 0); 1143 tcg_out_b_imm(s, cond, 0); 1144 } 1145} 1146 1147static void tcg_out_br(TCGContext *s, TCGLabel *l) 1148{ 1149 tcg_out_goto_label(s, COND_AL, l); 1150} 1151 1152static void tcg_out_mb(TCGContext *s, unsigned a0) 1153{ 1154 if (use_armv7_instructions) { 1155 tcg_out32(s, INSN_DMB_ISH); 1156 } else { 1157 tcg_out32(s, INSN_DMB_MCR); 1158 } 1159} 1160 1161static TCGCond tgen_cmp(TCGContext *s, TCGCond cond, TCGReg a, TCGReg b) 1162{ 1163 if (is_tst_cond(cond)) { 1164 tcg_out_dat_reg(s, COND_AL, ARITH_TST, 0, a, b, SHIFT_IMM_LSL(0)); 1165 return tcg_tst_eqne_cond(cond); 1166 } 1167 tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, a, b, SHIFT_IMM_LSL(0)); 1168 return cond; 1169} 1170 1171static TCGCond tgen_cmpi(TCGContext *s, TCGCond cond, TCGReg a, TCGArg b) 1172{ 1173 int imm12; 1174 1175 if (!is_tst_cond(cond)) { 1176 tcg_out_dat_IN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0, a, b); 1177 return cond; 1178 } 1179 1180 /* 1181 * The compare constraints allow rIN, but TST does not support N. 1182 * Be prepared to load the constant into a scratch register. 1183 */ 1184 imm12 = encode_imm(b); 1185 if (imm12 >= 0) { 1186 tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, a, imm12); 1187 } else { 1188 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, b); 1189 tcg_out_dat_reg(s, COND_AL, ARITH_TST, 0, 1190 a, TCG_REG_TMP, SHIFT_IMM_LSL(0)); 1191 } 1192 return tcg_tst_eqne_cond(cond); 1193} 1194 1195static TCGCond tcg_out_cmp(TCGContext *s, TCGCond cond, TCGReg a, 1196 TCGArg b, int b_const) 1197{ 1198 if (b_const) { 1199 return tgen_cmpi(s, cond, a, b); 1200 } else { 1201 return tgen_cmp(s, cond, a, b); 1202 } 1203} 1204 1205static TCGCond tcg_out_cmp2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, 1206 TCGArg bl, bool const_bl, TCGArg bh, bool const_bh) 1207{ 1208 switch (cond) { 1209 case TCG_COND_EQ: 1210 case TCG_COND_NE: 1211 case TCG_COND_LTU: 1212 case TCG_COND_LEU: 1213 case TCG_COND_GTU: 1214 case TCG_COND_GEU: 1215 /* 1216 * We perform a conditional comparison. If the high half is 1217 * equal, then overwrite the flags with the comparison of the 1218 * low half. The resulting flags cover the whole. 1219 */ 1220 tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0, ah, bh, const_bh); 1221 tcg_out_dat_rI(s, COND_EQ, ARITH_CMP, 0, al, bl, const_bl); 1222 return cond; 1223 1224 case TCG_COND_TSTEQ: 1225 case TCG_COND_TSTNE: 1226 /* Similar, but with TST instead of CMP. */ 1227 tcg_out_dat_rI(s, COND_AL, ARITH_TST, 0, ah, bh, const_bh); 1228 tcg_out_dat_rI(s, COND_EQ, ARITH_TST, 0, al, bl, const_bl); 1229 return tcg_tst_eqne_cond(cond); 1230 1231 case TCG_COND_LT: 1232 case TCG_COND_GE: 1233 /* We perform a double-word subtraction and examine the result. 1234 We do not actually need the result of the subtract, so the 1235 low part "subtract" is a compare. For the high half we have 1236 no choice but to compute into a temporary. */ 1237 tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0, al, bl, const_bl); 1238 tcg_out_dat_rI(s, COND_AL, ARITH_SBC | TO_CPSR, 1239 TCG_REG_TMP, ah, bh, const_bh); 1240 return cond; 1241 1242 case TCG_COND_LE: 1243 case TCG_COND_GT: 1244 /* Similar, but with swapped arguments, via reversed subtract. */ 1245 tcg_out_dat_rI(s, COND_AL, ARITH_RSB | TO_CPSR, 1246 TCG_REG_TMP, al, bl, const_bl); 1247 tcg_out_dat_rI(s, COND_AL, ARITH_RSC | TO_CPSR, 1248 TCG_REG_TMP, ah, bh, const_bh); 1249 return tcg_swap_cond(cond); 1250 1251 default: 1252 g_assert_not_reached(); 1253 } 1254} 1255 1256/* 1257 * Note that TCGReg references Q-registers. 1258 * Q-regno = 2 * D-regno, so shift left by 1 while inserting. 1259 */ 1260static uint32_t encode_vd(TCGReg rd) 1261{ 1262 tcg_debug_assert(rd >= TCG_REG_Q0); 1263 return (extract32(rd, 3, 1) << 22) | (extract32(rd, 0, 3) << 13); 1264} 1265 1266static uint32_t encode_vn(TCGReg rn) 1267{ 1268 tcg_debug_assert(rn >= TCG_REG_Q0); 1269 return (extract32(rn, 3, 1) << 7) | (extract32(rn, 0, 3) << 17); 1270} 1271 1272static uint32_t encode_vm(TCGReg rm) 1273{ 1274 tcg_debug_assert(rm >= TCG_REG_Q0); 1275 return (extract32(rm, 3, 1) << 5) | (extract32(rm, 0, 3) << 1); 1276} 1277 1278static void tcg_out_vreg2(TCGContext *s, ARMInsn insn, int q, int vece, 1279 TCGReg d, TCGReg m) 1280{ 1281 tcg_out32(s, insn | (vece << 18) | (q << 6) | 1282 encode_vd(d) | encode_vm(m)); 1283} 1284 1285static void tcg_out_vreg3(TCGContext *s, ARMInsn insn, int q, int vece, 1286 TCGReg d, TCGReg n, TCGReg m) 1287{ 1288 tcg_out32(s, insn | (vece << 20) | (q << 6) | 1289 encode_vd(d) | encode_vn(n) | encode_vm(m)); 1290} 1291 1292static void tcg_out_vmovi(TCGContext *s, TCGReg rd, 1293 int q, int op, int cmode, uint8_t imm8) 1294{ 1295 tcg_out32(s, INSN_VMOVI | encode_vd(rd) | (q << 6) | (op << 5) 1296 | (cmode << 8) | extract32(imm8, 0, 4) 1297 | (extract32(imm8, 4, 3) << 16) 1298 | (extract32(imm8, 7, 1) << 24)); 1299} 1300 1301static void tcg_out_vshifti(TCGContext *s, ARMInsn insn, int q, 1302 TCGReg rd, TCGReg rm, int l_imm6) 1303{ 1304 tcg_out32(s, insn | (q << 6) | encode_vd(rd) | encode_vm(rm) | 1305 (extract32(l_imm6, 6, 1) << 7) | 1306 (extract32(l_imm6, 0, 6) << 16)); 1307} 1308 1309static void tcg_out_vldst(TCGContext *s, ARMInsn insn, 1310 TCGReg rd, TCGReg rn, int offset) 1311{ 1312 if (offset != 0) { 1313 if (check_fit_imm(offset) || check_fit_imm(-offset)) { 1314 tcg_out_dat_rIN(s, COND_AL, ARITH_ADD, ARITH_SUB, 1315 TCG_REG_TMP, rn, offset, true); 1316 } else { 1317 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, offset); 1318 tcg_out_dat_reg(s, COND_AL, ARITH_ADD, 1319 TCG_REG_TMP, TCG_REG_TMP, rn, 0); 1320 } 1321 rn = TCG_REG_TMP; 1322 } 1323 tcg_out32(s, insn | (rn << 16) | encode_vd(rd) | 0xf); 1324} 1325 1326typedef struct { 1327 ARMCond cond; 1328 TCGReg base; 1329 int index; 1330 bool index_scratch; 1331 TCGAtomAlign aa; 1332} HostAddress; 1333 1334bool tcg_target_has_memory_bswap(MemOp memop) 1335{ 1336 return false; 1337} 1338 1339static TCGReg ldst_ra_gen(TCGContext *s, const TCGLabelQemuLdst *l, int arg) 1340{ 1341 /* We arrive at the slow path via "BLNE", so R14 contains l->raddr. */ 1342 return TCG_REG_R14; 1343} 1344 1345static const TCGLdstHelperParam ldst_helper_param = { 1346 .ra_gen = ldst_ra_gen, 1347 .ntmp = 1, 1348 .tmp = { TCG_REG_TMP }, 1349}; 1350 1351static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) 1352{ 1353 MemOp opc = get_memop(lb->oi); 1354 1355 if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 1356 return false; 1357 } 1358 1359 tcg_out_ld_helper_args(s, lb, &ldst_helper_param); 1360 tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]); 1361 tcg_out_ld_helper_ret(s, lb, false, &ldst_helper_param); 1362 1363 tcg_out_goto(s, COND_AL, lb->raddr); 1364 return true; 1365} 1366 1367static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) 1368{ 1369 MemOp opc = get_memop(lb->oi); 1370 1371 if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 1372 return false; 1373 } 1374 1375 tcg_out_st_helper_args(s, lb, &ldst_helper_param); 1376 1377 /* Tail-call to the helper, which will return to the fast path. */ 1378 tcg_out_goto(s, COND_AL, qemu_st_helpers[opc & MO_SIZE]); 1379 return true; 1380} 1381 1382/* We expect to use an 9-bit sign-magnitude negative offset from ENV. */ 1383#define MIN_TLB_MASK_TABLE_OFS -256 1384 1385static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, 1386 TCGReg addr, MemOpIdx oi, bool is_ld) 1387{ 1388 TCGLabelQemuLdst *ldst = NULL; 1389 MemOp opc = get_memop(oi); 1390 unsigned a_mask; 1391 1392 if (tcg_use_softmmu) { 1393 *h = (HostAddress){ 1394 .cond = COND_AL, 1395 .base = addr, 1396 .index = TCG_REG_R1, 1397 .index_scratch = true, 1398 }; 1399 } else { 1400 *h = (HostAddress){ 1401 .cond = COND_AL, 1402 .base = addr, 1403 .index = guest_base ? TCG_REG_GUEST_BASE : -1, 1404 .index_scratch = false, 1405 }; 1406 } 1407 1408 h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); 1409 a_mask = (1 << h->aa.align) - 1; 1410 1411 if (tcg_use_softmmu) { 1412 int mem_index = get_mmuidx(oi); 1413 int cmp_off = is_ld ? offsetof(CPUTLBEntry, addr_read) 1414 : offsetof(CPUTLBEntry, addr_write); 1415 int fast_off = tlb_mask_table_ofs(s, mem_index); 1416 unsigned s_mask = (1 << (opc & MO_SIZE)) - 1; 1417 TCGReg t_addr; 1418 1419 ldst = new_ldst_label(s); 1420 ldst->is_ld = is_ld; 1421 ldst->oi = oi; 1422 ldst->addr_reg = addr; 1423 1424 /* Load cpu->neg.tlb.f[mmu_idx].{mask,table} into {r0,r1}. */ 1425 QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0); 1426 QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 4); 1427 tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_AREG0, fast_off); 1428 1429 /* Extract the tlb index from the address into R0. */ 1430 tcg_out_dat_reg(s, COND_AL, ARITH_AND, TCG_REG_R0, TCG_REG_R0, addr, 1431 SHIFT_IMM_LSR(TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS)); 1432 1433 /* 1434 * Add the tlb_table pointer, creating the CPUTLBEntry address in R1. 1435 * Load the tlb comparator into R2 and the fast path addend into R1. 1436 */ 1437 if (cmp_off == 0) { 1438 tcg_out_ld32_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0); 1439 } else { 1440 tcg_out_dat_reg(s, COND_AL, ARITH_ADD, 1441 TCG_REG_R1, TCG_REG_R1, TCG_REG_R0, 0); 1442 tcg_out_ld32_12(s, COND_AL, TCG_REG_R2, TCG_REG_R1, cmp_off); 1443 } 1444 1445 /* Load the tlb addend. */ 1446 tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R1, 1447 offsetof(CPUTLBEntry, addend)); 1448 1449 /* 1450 * Check alignment, check comparators. 1451 * Do this in 2-4 insns. Use MOVW for v7, if possible, 1452 * to reduce the number of sequential conditional instructions. 1453 * Almost all guests have at least 4k pages, which means that we need 1454 * to clear at least 9 bits even for an 8-byte memory, which means it 1455 * isn't worth checking for an immediate operand for BIC. 1456 * 1457 * For unaligned accesses, test the page of the last unit of alignment. 1458 * This leaves the least significant alignment bits unchanged, and of 1459 * course must be zero. 1460 */ 1461 t_addr = addr; 1462 if (a_mask < s_mask) { 1463 t_addr = TCG_REG_R0; 1464 tcg_out_dat_imm(s, COND_AL, ARITH_ADD, t_addr, 1465 addr, s_mask - a_mask); 1466 } 1467 if (use_armv7_instructions && TARGET_PAGE_BITS <= 16) { 1468 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, ~(TARGET_PAGE_MASK | a_mask)); 1469 tcg_out_dat_reg(s, COND_AL, ARITH_BIC, TCG_REG_TMP, 1470 t_addr, TCG_REG_TMP, 0); 1471 tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, 1472 TCG_REG_R2, TCG_REG_TMP, 0); 1473 } else { 1474 if (a_mask) { 1475 tcg_debug_assert(a_mask <= 0xff); 1476 tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addr, a_mask); 1477 } 1478 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP, 0, t_addr, 1479 SHIFT_IMM_LSR(TARGET_PAGE_BITS)); 1480 tcg_out_dat_reg(s, (a_mask ? COND_EQ : COND_AL), ARITH_CMP, 1481 0, TCG_REG_R2, TCG_REG_TMP, 1482 SHIFT_IMM_LSL(TARGET_PAGE_BITS)); 1483 } 1484 } else if (a_mask) { 1485 ldst = new_ldst_label(s); 1486 ldst->is_ld = is_ld; 1487 ldst->oi = oi; 1488 ldst->addr_reg = addr; 1489 1490 /* We are expecting alignment to max out at 7 */ 1491 tcg_debug_assert(a_mask <= 0xff); 1492 /* tst addr, #mask */ 1493 tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addr, a_mask); 1494 } 1495 1496 return ldst; 1497} 1498 1499static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg datalo, 1500 TCGReg datahi, HostAddress h) 1501{ 1502 TCGReg base; 1503 1504 /* Byte swapping is left to middle-end expansion. */ 1505 tcg_debug_assert((opc & MO_BSWAP) == 0); 1506 1507 switch (opc & MO_SSIZE) { 1508 case MO_UB: 1509 if (h.index < 0) { 1510 tcg_out_ld8_12(s, h.cond, datalo, h.base, 0); 1511 } else { 1512 tcg_out_ld8_r(s, h.cond, datalo, h.base, h.index); 1513 } 1514 break; 1515 case MO_SB: 1516 if (h.index < 0) { 1517 tcg_out_ld8s_8(s, h.cond, datalo, h.base, 0); 1518 } else { 1519 tcg_out_ld8s_r(s, h.cond, datalo, h.base, h.index); 1520 } 1521 break; 1522 case MO_UW: 1523 if (h.index < 0) { 1524 tcg_out_ld16u_8(s, h.cond, datalo, h.base, 0); 1525 } else { 1526 tcg_out_ld16u_r(s, h.cond, datalo, h.base, h.index); 1527 } 1528 break; 1529 case MO_SW: 1530 if (h.index < 0) { 1531 tcg_out_ld16s_8(s, h.cond, datalo, h.base, 0); 1532 } else { 1533 tcg_out_ld16s_r(s, h.cond, datalo, h.base, h.index); 1534 } 1535 break; 1536 case MO_UL: 1537 if (h.index < 0) { 1538 tcg_out_ld32_12(s, h.cond, datalo, h.base, 0); 1539 } else { 1540 tcg_out_ld32_r(s, h.cond, datalo, h.base, h.index); 1541 } 1542 break; 1543 case MO_UQ: 1544 /* We used pair allocation for datalo, so already should be aligned. */ 1545 tcg_debug_assert((datalo & 1) == 0); 1546 tcg_debug_assert(datahi == datalo + 1); 1547 /* LDRD requires alignment; double-check that. */ 1548 if (memop_alignment_bits(opc) >= MO_64) { 1549 if (h.index < 0) { 1550 tcg_out_ldrd_8(s, h.cond, datalo, h.base, 0); 1551 break; 1552 } 1553 /* 1554 * Rm (the second address op) must not overlap Rt or Rt + 1. 1555 * Since datalo is aligned, we can simplify the test via alignment. 1556 * Flip the two address arguments if that works. 1557 */ 1558 if ((h.index & ~1) != datalo) { 1559 tcg_out_ldrd_r(s, h.cond, datalo, h.base, h.index); 1560 break; 1561 } 1562 if ((h.base & ~1) != datalo) { 1563 tcg_out_ldrd_r(s, h.cond, datalo, h.index, h.base); 1564 break; 1565 } 1566 } 1567 if (h.index < 0) { 1568 base = h.base; 1569 if (datalo == h.base) { 1570 tcg_out_mov_reg(s, h.cond, TCG_REG_TMP, base); 1571 base = TCG_REG_TMP; 1572 } 1573 } else if (h.index_scratch) { 1574 tcg_out_ld32_rwb(s, h.cond, datalo, h.index, h.base); 1575 tcg_out_ld32_12(s, h.cond, datahi, h.index, 4); 1576 break; 1577 } else { 1578 tcg_out_dat_reg(s, h.cond, ARITH_ADD, TCG_REG_TMP, 1579 h.base, h.index, SHIFT_IMM_LSL(0)); 1580 base = TCG_REG_TMP; 1581 } 1582 tcg_out_ld32_12(s, h.cond, datalo, base, 0); 1583 tcg_out_ld32_12(s, h.cond, datahi, base, 4); 1584 break; 1585 default: 1586 g_assert_not_reached(); 1587 } 1588} 1589 1590static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data, 1591 TCGReg addr, MemOpIdx oi) 1592{ 1593 MemOp opc = get_memop(oi); 1594 TCGLabelQemuLdst *ldst; 1595 HostAddress h; 1596 1597 ldst = prepare_host_addr(s, &h, addr, oi, true); 1598 if (ldst) { 1599 ldst->type = type; 1600 ldst->datalo_reg = data; 1601 ldst->datahi_reg = -1; 1602 1603 /* 1604 * This a conditional BL only to load a pointer within this 1605 * opcode into LR for the slow path. We will not be using 1606 * the value for a tail call. 1607 */ 1608 ldst->label_ptr[0] = s->code_ptr; 1609 tcg_out_bl_imm(s, COND_NE, 0); 1610 } 1611 1612 tcg_out_qemu_ld_direct(s, opc, data, -1, h); 1613 1614 if (ldst) { 1615 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 1616 } 1617} 1618 1619static const TCGOutOpQemuLdSt outop_qemu_ld = { 1620 .base.static_constraint = C_O1_I1(r, q), 1621 .out = tgen_qemu_ld, 1622}; 1623 1624static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo, 1625 TCGReg datahi, TCGReg addr, MemOpIdx oi) 1626{ 1627 MemOp opc = get_memop(oi); 1628 TCGLabelQemuLdst *ldst; 1629 HostAddress h; 1630 1631 ldst = prepare_host_addr(s, &h, addr, oi, true); 1632 if (ldst) { 1633 ldst->type = type; 1634 ldst->datalo_reg = datalo; 1635 ldst->datahi_reg = datahi; 1636 1637 /* 1638 * This a conditional BL only to load a pointer within this 1639 * opcode into LR for the slow path. We will not be using 1640 * the value for a tail call. 1641 */ 1642 ldst->label_ptr[0] = s->code_ptr; 1643 tcg_out_bl_imm(s, COND_NE, 0); 1644 } 1645 1646 tcg_out_qemu_ld_direct(s, opc, datalo, datahi, h); 1647 1648 if (ldst) { 1649 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 1650 } 1651} 1652 1653static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = { 1654 .base.static_constraint = C_O2_I1(e, p, q), 1655 .out = tgen_qemu_ld2, 1656}; 1657 1658static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg datalo, 1659 TCGReg datahi, HostAddress h) 1660{ 1661 /* Byte swapping is left to middle-end expansion. */ 1662 tcg_debug_assert((opc & MO_BSWAP) == 0); 1663 1664 switch (opc & MO_SIZE) { 1665 case MO_8: 1666 if (h.index < 0) { 1667 tcg_out_st8_12(s, h.cond, datalo, h.base, 0); 1668 } else { 1669 tcg_out_st8_r(s, h.cond, datalo, h.base, h.index); 1670 } 1671 break; 1672 case MO_16: 1673 if (h.index < 0) { 1674 tcg_out_st16_8(s, h.cond, datalo, h.base, 0); 1675 } else { 1676 tcg_out_st16_r(s, h.cond, datalo, h.base, h.index); 1677 } 1678 break; 1679 case MO_32: 1680 if (h.index < 0) { 1681 tcg_out_st32_12(s, h.cond, datalo, h.base, 0); 1682 } else { 1683 tcg_out_st32_r(s, h.cond, datalo, h.base, h.index); 1684 } 1685 break; 1686 case MO_64: 1687 /* We used pair allocation for datalo, so already should be aligned. */ 1688 tcg_debug_assert((datalo & 1) == 0); 1689 tcg_debug_assert(datahi == datalo + 1); 1690 /* STRD requires alignment; double-check that. */ 1691 if (memop_alignment_bits(opc) >= MO_64) { 1692 if (h.index < 0) { 1693 tcg_out_strd_8(s, h.cond, datalo, h.base, 0); 1694 } else { 1695 tcg_out_strd_r(s, h.cond, datalo, h.base, h.index); 1696 } 1697 } else if (h.index < 0) { 1698 tcg_out_st32_12(s, h.cond, datalo, h.base, 0); 1699 tcg_out_st32_12(s, h.cond, datahi, h.base, 4); 1700 } else if (h.index_scratch) { 1701 tcg_out_st32_rwb(s, h.cond, datalo, h.index, h.base); 1702 tcg_out_st32_12(s, h.cond, datahi, h.index, 4); 1703 } else { 1704 tcg_out_dat_reg(s, h.cond, ARITH_ADD, TCG_REG_TMP, 1705 h.base, h.index, SHIFT_IMM_LSL(0)); 1706 tcg_out_st32_12(s, h.cond, datalo, TCG_REG_TMP, 0); 1707 tcg_out_st32_12(s, h.cond, datahi, TCG_REG_TMP, 4); 1708 } 1709 break; 1710 default: 1711 g_assert_not_reached(); 1712 } 1713} 1714 1715static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data, 1716 TCGReg addr, MemOpIdx oi) 1717{ 1718 MemOp opc = get_memop(oi); 1719 TCGLabelQemuLdst *ldst; 1720 HostAddress h; 1721 1722 ldst = prepare_host_addr(s, &h, addr, oi, false); 1723 if (ldst) { 1724 ldst->type = type; 1725 ldst->datalo_reg = data; 1726 ldst->datahi_reg = -1; 1727 1728 h.cond = COND_EQ; 1729 tcg_out_qemu_st_direct(s, opc, data, -1, h); 1730 1731 /* The conditional call is last, as we're going to return here. */ 1732 ldst->label_ptr[0] = s->code_ptr; 1733 tcg_out_bl_imm(s, COND_NE, 0); 1734 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 1735 } else { 1736 tcg_out_qemu_st_direct(s, opc, data, -1, h); 1737 } 1738} 1739 1740static const TCGOutOpQemuLdSt outop_qemu_st = { 1741 .base.static_constraint = C_O0_I2(q, q), 1742 .out = tgen_qemu_st, 1743}; 1744 1745static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo, 1746 TCGReg datahi, TCGReg addr, MemOpIdx oi) 1747{ 1748 MemOp opc = get_memop(oi); 1749 TCGLabelQemuLdst *ldst; 1750 HostAddress h; 1751 1752 ldst = prepare_host_addr(s, &h, addr, oi, false); 1753 if (ldst) { 1754 ldst->type = type; 1755 ldst->datalo_reg = datalo; 1756 ldst->datahi_reg = datahi; 1757 1758 h.cond = COND_EQ; 1759 tcg_out_qemu_st_direct(s, opc, datalo, datahi, h); 1760 1761 /* The conditional call is last, as we're going to return here. */ 1762 ldst->label_ptr[0] = s->code_ptr; 1763 tcg_out_bl_imm(s, COND_NE, 0); 1764 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 1765 } else { 1766 tcg_out_qemu_st_direct(s, opc, datalo, datahi, h); 1767 } 1768} 1769 1770static const TCGOutOpQemuLdSt2 outop_qemu_st2 = { 1771 .base.static_constraint = C_O0_I3(Q, p, q), 1772 .out = tgen_qemu_st2, 1773}; 1774 1775static void tcg_out_epilogue(TCGContext *s); 1776 1777static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) 1778{ 1779 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, arg); 1780 tcg_out_epilogue(s); 1781} 1782 1783static void tcg_out_goto_tb(TCGContext *s, int which) 1784{ 1785 uintptr_t i_addr; 1786 intptr_t i_disp; 1787 1788 /* Direct branch will be patched by tb_target_set_jmp_target. */ 1789 set_jmp_insn_offset(s, which); 1790 tcg_out32(s, INSN_NOP); 1791 1792 /* When branch is out of range, fall through to indirect. */ 1793 i_addr = get_jmp_target_addr(s, which); 1794 i_disp = tcg_pcrel_diff(s, (void *)i_addr) - 8; 1795 tcg_debug_assert(i_disp < 0); 1796 if (i_disp >= -0xfff) { 1797 tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, i_disp); 1798 } else { 1799 /* 1800 * The TB is close, but outside the 12 bits addressable by 1801 * the load. We can extend this to 20 bits with a sub of a 1802 * shifted immediate from pc. 1803 */ 1804 int h = -i_disp; 1805 int l = -(h & 0xfff); 1806 1807 h = encode_imm_nofail(h + l); 1808 tcg_out_dat_imm(s, COND_AL, ARITH_SUB, TCG_REG_R0, TCG_REG_PC, h); 1809 tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, l); 1810 } 1811 set_jmp_reset_offset(s, which); 1812} 1813 1814static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0) 1815{ 1816 tcg_out_b_reg(s, COND_AL, a0); 1817} 1818 1819void tb_target_set_jmp_target(const TranslationBlock *tb, int n, 1820 uintptr_t jmp_rx, uintptr_t jmp_rw) 1821{ 1822 uintptr_t addr = tb->jmp_target_addr[n]; 1823 ptrdiff_t offset = addr - (jmp_rx + 8); 1824 tcg_insn_unit insn; 1825 1826 /* Either directly branch, or fall through to indirect branch. */ 1827 if (offset == sextract64(offset, 0, 26)) { 1828 /* B <addr> */ 1829 insn = deposit32((COND_AL << 28) | INSN_B, 0, 24, offset >> 2); 1830 } else { 1831 insn = INSN_NOP; 1832 } 1833 1834 qatomic_set((uint32_t *)jmp_rw, insn); 1835 flush_idcache_range(jmp_rx, jmp_rw, 4); 1836} 1837 1838 1839static void tgen_add(TCGContext *s, TCGType type, 1840 TCGReg a0, TCGReg a1, TCGReg a2) 1841{ 1842 tcg_out_dat_reg(s, COND_AL, ARITH_ADD, a0, a1, a2, SHIFT_IMM_LSL(0)); 1843} 1844 1845static void tgen_addi(TCGContext *s, TCGType type, 1846 TCGReg a0, TCGReg a1, tcg_target_long a2) 1847{ 1848 tcg_out_dat_IN(s, COND_AL, ARITH_ADD, ARITH_SUB, a0, a1, a2); 1849} 1850 1851static const TCGOutOpBinary outop_add = { 1852 .base.static_constraint = C_O1_I2(r, r, rIN), 1853 .out_rrr = tgen_add, 1854 .out_rri = tgen_addi, 1855}; 1856 1857static void tgen_addco(TCGContext *s, TCGType type, 1858 TCGReg a0, TCGReg a1, TCGReg a2) 1859{ 1860 tcg_out_dat_reg(s, COND_AL, ARITH_ADD | TO_CPSR, 1861 a0, a1, a2, SHIFT_IMM_LSL(0)); 1862} 1863 1864static void tgen_addco_imm(TCGContext *s, TCGType type, 1865 TCGReg a0, TCGReg a1, tcg_target_long a2) 1866{ 1867 tcg_out_dat_IN(s, COND_AL, ARITH_ADD | TO_CPSR, ARITH_SUB | TO_CPSR, 1868 a0, a1, a2); 1869} 1870 1871static const TCGOutOpBinary outop_addco = { 1872 .base.static_constraint = C_O1_I2(r, r, rIN), 1873 .out_rrr = tgen_addco, 1874 .out_rri = tgen_addco_imm, 1875}; 1876 1877static void tgen_addci(TCGContext *s, TCGType type, 1878 TCGReg a0, TCGReg a1, TCGReg a2) 1879{ 1880 tcg_out_dat_reg(s, COND_AL, ARITH_ADC, a0, a1, a2, SHIFT_IMM_LSL(0)); 1881} 1882 1883static void tgen_addci_imm(TCGContext *s, TCGType type, 1884 TCGReg a0, TCGReg a1, tcg_target_long a2) 1885{ 1886 tcg_out_dat_IK(s, COND_AL, ARITH_ADC, ARITH_SBC, a0, a1, a2); 1887} 1888 1889static const TCGOutOpAddSubCarry outop_addci = { 1890 .base.static_constraint = C_O1_I2(r, r, rIK), 1891 .out_rrr = tgen_addci, 1892 .out_rri = tgen_addci_imm, 1893}; 1894 1895static void tgen_addcio(TCGContext *s, TCGType type, 1896 TCGReg a0, TCGReg a1, TCGReg a2) 1897{ 1898 tcg_out_dat_reg(s, COND_AL, ARITH_ADC | TO_CPSR, 1899 a0, a1, a2, SHIFT_IMM_LSL(0)); 1900} 1901 1902static void tgen_addcio_imm(TCGContext *s, TCGType type, 1903 TCGReg a0, TCGReg a1, tcg_target_long a2) 1904{ 1905 tcg_out_dat_IK(s, COND_AL, ARITH_ADC | TO_CPSR, ARITH_SBC | TO_CPSR, 1906 a0, a1, a2); 1907} 1908 1909static const TCGOutOpBinary outop_addcio = { 1910 .base.static_constraint = C_O1_I2(r, r, rIK), 1911 .out_rrr = tgen_addcio, 1912 .out_rri = tgen_addcio_imm, 1913}; 1914 1915/* Set C to @c; NZVQ all set to 0. */ 1916static void tcg_out_movi_apsr_c(TCGContext *s, bool c) 1917{ 1918 int imm12 = encode_imm_nofail(c << 29); 1919 tcg_out32(s, (COND_AL << 28) | INSN_MSRI_CPSR | 0x80000 | imm12); 1920} 1921 1922static void tcg_out_set_carry(TCGContext *s) 1923{ 1924 tcg_out_movi_apsr_c(s, 1); 1925} 1926 1927static void tgen_and(TCGContext *s, TCGType type, 1928 TCGReg a0, TCGReg a1, TCGReg a2) 1929{ 1930 tcg_out_dat_reg(s, COND_AL, ARITH_AND, a0, a1, a2, SHIFT_IMM_LSL(0)); 1931} 1932 1933static void tgen_andi(TCGContext *s, TCGType type, 1934 TCGReg a0, TCGReg a1, tcg_target_long a2) 1935{ 1936 tcg_out_dat_IK(s, COND_AL, ARITH_AND, ARITH_BIC, a0, a1, a2); 1937} 1938 1939static const TCGOutOpBinary outop_and = { 1940 .base.static_constraint = C_O1_I2(r, r, rIK), 1941 .out_rrr = tgen_and, 1942 .out_rri = tgen_andi, 1943}; 1944 1945static void tgen_andc(TCGContext *s, TCGType type, 1946 TCGReg a0, TCGReg a1, TCGReg a2) 1947{ 1948 tcg_out_dat_reg(s, COND_AL, ARITH_BIC, a0, a1, a2, SHIFT_IMM_LSL(0)); 1949} 1950 1951static const TCGOutOpBinary outop_andc = { 1952 .base.static_constraint = C_O1_I2(r, r, r), 1953 .out_rrr = tgen_andc, 1954}; 1955 1956static void tgen_clz(TCGContext *s, TCGType type, 1957 TCGReg a0, TCGReg a1, TCGReg a2) 1958{ 1959 tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0, a1, 0); 1960 tcg_out_dat_reg(s, COND_NE, INSN_CLZ, a0, 0, a1, 0); 1961 tcg_out_mov_reg(s, COND_EQ, a0, a2); 1962} 1963 1964static void tgen_clzi(TCGContext *s, TCGType type, 1965 TCGReg a0, TCGReg a1, tcg_target_long a2) 1966{ 1967 if (a2 == 32) { 1968 tcg_out_dat_reg(s, COND_AL, INSN_CLZ, a0, 0, a1, 0); 1969 } else { 1970 tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0, a1, 0); 1971 tcg_out_dat_reg(s, COND_NE, INSN_CLZ, a0, 0, a1, 0); 1972 tcg_out_movi32(s, COND_EQ, a0, a2); 1973 } 1974} 1975 1976static const TCGOutOpBinary outop_clz = { 1977 .base.static_constraint = C_O1_I2(r, r, rIK), 1978 .out_rrr = tgen_clz, 1979 .out_rri = tgen_clzi, 1980}; 1981 1982static const TCGOutOpUnary outop_ctpop = { 1983 .base.static_constraint = C_NotImplemented, 1984}; 1985 1986static void tgen_ctz(TCGContext *s, TCGType type, 1987 TCGReg a0, TCGReg a1, TCGReg a2) 1988{ 1989 tcg_out_dat_reg(s, COND_AL, INSN_RBIT, TCG_REG_TMP, 0, a1, 0); 1990 tgen_clz(s, TCG_TYPE_I32, a0, TCG_REG_TMP, a2); 1991} 1992 1993static void tgen_ctzi(TCGContext *s, TCGType type, 1994 TCGReg a0, TCGReg a1, tcg_target_long a2) 1995{ 1996 tcg_out_dat_reg(s, COND_AL, INSN_RBIT, TCG_REG_TMP, 0, a1, 0); 1997 tgen_clzi(s, TCG_TYPE_I32, a0, TCG_REG_TMP, a2); 1998} 1999 2000static TCGConstraintSetIndex cset_ctz(TCGType type, unsigned flags) 2001{ 2002 return use_armv7_instructions ? C_O1_I2(r, r, rIK) : C_NotImplemented; 2003} 2004 2005static const TCGOutOpBinary outop_ctz = { 2006 .base.static_constraint = C_Dynamic, 2007 .base.dynamic_constraint = cset_ctz, 2008 .out_rrr = tgen_ctz, 2009 .out_rri = tgen_ctzi, 2010}; 2011 2012static TCGConstraintSetIndex cset_idiv(TCGType type, unsigned flags) 2013{ 2014 return use_idiv_instructions ? C_O1_I2(r, r, r) : C_NotImplemented; 2015} 2016 2017static void tgen_divs(TCGContext *s, TCGType type, 2018 TCGReg a0, TCGReg a1, TCGReg a2) 2019{ 2020 /* sdiv */ 2021 tcg_out32(s, 0x0710f010 | (COND_AL << 28) | (a0 << 16) | a1 | (a2 << 8)); 2022} 2023 2024static const TCGOutOpBinary outop_divs = { 2025 .base.static_constraint = C_Dynamic, 2026 .base.dynamic_constraint = cset_idiv, 2027 .out_rrr = tgen_divs, 2028}; 2029 2030static const TCGOutOpDivRem outop_divs2 = { 2031 .base.static_constraint = C_NotImplemented, 2032}; 2033 2034static void tgen_divu(TCGContext *s, TCGType type, 2035 TCGReg a0, TCGReg a1, TCGReg a2) 2036{ 2037 /* udiv */ 2038 tcg_out32(s, 0x0730f010 | (COND_AL << 28) | (a0 << 16) | a1 | (a2 << 8)); 2039} 2040 2041static const TCGOutOpBinary outop_divu = { 2042 .base.static_constraint = C_Dynamic, 2043 .base.dynamic_constraint = cset_idiv, 2044 .out_rrr = tgen_divu, 2045}; 2046 2047static const TCGOutOpDivRem outop_divu2 = { 2048 .base.static_constraint = C_NotImplemented, 2049}; 2050 2051static const TCGOutOpBinary outop_eqv = { 2052 .base.static_constraint = C_NotImplemented, 2053}; 2054 2055static void tgen_mul(TCGContext *s, TCGType type, 2056 TCGReg a0, TCGReg a1, TCGReg a2) 2057{ 2058 /* mul */ 2059 tcg_out32(s, (COND_AL << 28) | 0x90 | (a0 << 16) | (a1 << 8) | a2); 2060} 2061 2062static const TCGOutOpBinary outop_mul = { 2063 .base.static_constraint = C_O1_I2(r, r, r), 2064 .out_rrr = tgen_mul, 2065}; 2066 2067static void tgen_muls2(TCGContext *s, TCGType type, 2068 TCGReg rd0, TCGReg rd1, TCGReg rn, TCGReg rm) 2069{ 2070 /* smull */ 2071 tcg_out32(s, (COND_AL << 28) | 0x00c00090 | 2072 (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); 2073} 2074 2075static const TCGOutOpMul2 outop_muls2 = { 2076 .base.static_constraint = C_O2_I2(r, r, r, r), 2077 .out_rrrr = tgen_muls2, 2078}; 2079 2080static const TCGOutOpBinary outop_mulsh = { 2081 .base.static_constraint = C_NotImplemented, 2082}; 2083 2084static void tgen_mulu2(TCGContext *s, TCGType type, 2085 TCGReg rd0, TCGReg rd1, TCGReg rn, TCGReg rm) 2086{ 2087 /* umull */ 2088 tcg_out32(s, (COND_AL << 28) | 0x00800090 | 2089 (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn); 2090} 2091 2092static const TCGOutOpMul2 outop_mulu2 = { 2093 .base.static_constraint = C_O2_I2(r, r, r, r), 2094 .out_rrrr = tgen_mulu2, 2095}; 2096 2097static const TCGOutOpBinary outop_muluh = { 2098 .base.static_constraint = C_NotImplemented, 2099}; 2100 2101static const TCGOutOpBinary outop_nand = { 2102 .base.static_constraint = C_NotImplemented, 2103}; 2104 2105static const TCGOutOpBinary outop_nor = { 2106 .base.static_constraint = C_NotImplemented, 2107}; 2108 2109static void tgen_or(TCGContext *s, TCGType type, 2110 TCGReg a0, TCGReg a1, TCGReg a2) 2111{ 2112 tcg_out_dat_reg(s, COND_AL, ARITH_ORR, a0, a1, a2, SHIFT_IMM_LSL(0)); 2113} 2114 2115static void tgen_ori(TCGContext *s, TCGType type, 2116 TCGReg a0, TCGReg a1, tcg_target_long a2) 2117{ 2118 tcg_out_dat_imm(s, COND_AL, ARITH_ORR, a0, a1, encode_imm_nofail(a2)); 2119} 2120 2121static const TCGOutOpBinary outop_or = { 2122 .base.static_constraint = C_O1_I2(r, r, rI), 2123 .out_rrr = tgen_or, 2124 .out_rri = tgen_ori, 2125}; 2126 2127static const TCGOutOpBinary outop_orc = { 2128 .base.static_constraint = C_NotImplemented, 2129}; 2130 2131static const TCGOutOpBinary outop_rems = { 2132 .base.static_constraint = C_NotImplemented, 2133}; 2134 2135static const TCGOutOpBinary outop_remu = { 2136 .base.static_constraint = C_NotImplemented, 2137}; 2138 2139static const TCGOutOpBinary outop_rotl = { 2140 .base.static_constraint = C_NotImplemented, 2141}; 2142 2143static void tgen_rotr(TCGContext *s, TCGType type, 2144 TCGReg a0, TCGReg a1, TCGReg a2) 2145{ 2146 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_ROR(a2)); 2147} 2148 2149static void tgen_rotri(TCGContext *s, TCGType type, 2150 TCGReg a0, TCGReg a1, tcg_target_long a2) 2151{ 2152 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_IMM_ROR(a2 & 0x1f)); 2153} 2154 2155static const TCGOutOpBinary outop_rotr = { 2156 .base.static_constraint = C_O1_I2(r, r, ri), 2157 .out_rrr = tgen_rotr, 2158 .out_rri = tgen_rotri, 2159}; 2160 2161static void tgen_sar(TCGContext *s, TCGType type, 2162 TCGReg a0, TCGReg a1, TCGReg a2) 2163{ 2164 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_ASR(a2)); 2165} 2166 2167static void tgen_sari(TCGContext *s, TCGType type, 2168 TCGReg a0, TCGReg a1, tcg_target_long a2) 2169{ 2170 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, 2171 SHIFT_IMM_ASR(a2 & 0x1f)); 2172} 2173 2174static const TCGOutOpBinary outop_sar = { 2175 .base.static_constraint = C_O1_I2(r, r, ri), 2176 .out_rrr = tgen_sar, 2177 .out_rri = tgen_sari, 2178}; 2179 2180static void tgen_shl(TCGContext *s, TCGType type, 2181 TCGReg a0, TCGReg a1, TCGReg a2) 2182{ 2183 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_LSL(a2)); 2184} 2185 2186static void tgen_shli(TCGContext *s, TCGType type, 2187 TCGReg a0, TCGReg a1, tcg_target_long a2) 2188{ 2189 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, 2190 SHIFT_IMM_LSL(a2 & 0x1f)); 2191} 2192 2193static const TCGOutOpBinary outop_shl = { 2194 .base.static_constraint = C_O1_I2(r, r, ri), 2195 .out_rrr = tgen_shl, 2196 .out_rri = tgen_shli, 2197}; 2198 2199static void tgen_shr(TCGContext *s, TCGType type, 2200 TCGReg a0, TCGReg a1, TCGReg a2) 2201{ 2202 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, SHIFT_REG_LSR(a2)); 2203} 2204 2205static void tgen_shri(TCGContext *s, TCGType type, 2206 TCGReg a0, TCGReg a1, tcg_target_long a2) 2207{ 2208 tcg_out_dat_reg(s, COND_AL, ARITH_MOV, a0, 0, a1, 2209 SHIFT_IMM_LSR(a2 & 0x1f)); 2210} 2211 2212static const TCGOutOpBinary outop_shr = { 2213 .base.static_constraint = C_O1_I2(r, r, ri), 2214 .out_rrr = tgen_shr, 2215 .out_rri = tgen_shri, 2216}; 2217 2218static void tgen_sub(TCGContext *s, TCGType type, 2219 TCGReg a0, TCGReg a1, TCGReg a2) 2220{ 2221 tcg_out_dat_reg(s, COND_AL, ARITH_SUB, a0, a1, a2, SHIFT_IMM_LSL(0)); 2222} 2223 2224static void tgen_subfi(TCGContext *s, TCGType type, 2225 TCGReg a0, tcg_target_long a1, TCGReg a2) 2226{ 2227 tcg_out_dat_imm(s, COND_AL, ARITH_RSB, a0, a2, encode_imm_nofail(a1)); 2228} 2229 2230static const TCGOutOpSubtract outop_sub = { 2231 .base.static_constraint = C_O1_I2(r, rI, r), 2232 .out_rrr = tgen_sub, 2233 .out_rir = tgen_subfi, 2234}; 2235 2236static void tgen_subbo_rrr(TCGContext *s, TCGType type, 2237 TCGReg a0, TCGReg a1, TCGReg a2) 2238{ 2239 tcg_out_dat_reg(s, COND_AL, ARITH_SUB | TO_CPSR, 2240 a0, a1, a2, SHIFT_IMM_LSL(0)); 2241} 2242 2243static void tgen_subbo_rri(TCGContext *s, TCGType type, 2244 TCGReg a0, TCGReg a1, tcg_target_long a2) 2245{ 2246 tcg_out_dat_IN(s, COND_AL, ARITH_SUB | TO_CPSR, ARITH_ADD | TO_CPSR, 2247 a0, a1, a2); 2248} 2249 2250static void tgen_subbo_rir(TCGContext *s, TCGType type, 2251 TCGReg a0, tcg_target_long a1, TCGReg a2) 2252{ 2253 tcg_out_dat_imm(s, COND_AL, ARITH_RSB | TO_CPSR, 2254 a0, a2, encode_imm_nofail(a1)); 2255} 2256 2257static void tgen_subbo_rii(TCGContext *s, TCGType type, 2258 TCGReg a0, tcg_target_long a1, tcg_target_long a2) 2259{ 2260 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, a2); 2261 tgen_subbo_rir(s, TCG_TYPE_I32, a0, a1, TCG_REG_TMP); 2262} 2263 2264static const TCGOutOpAddSubCarry outop_subbo = { 2265 .base.static_constraint = C_O1_I2(r, rI, rIN), 2266 .out_rrr = tgen_subbo_rrr, 2267 .out_rri = tgen_subbo_rri, 2268 .out_rir = tgen_subbo_rir, 2269 .out_rii = tgen_subbo_rii, 2270}; 2271 2272static void tgen_subbi_rrr(TCGContext *s, TCGType type, 2273 TCGReg a0, TCGReg a1, TCGReg a2) 2274{ 2275 tcg_out_dat_reg(s, COND_AL, ARITH_SBC, 2276 a0, a1, a2, SHIFT_IMM_LSL(0)); 2277} 2278 2279static void tgen_subbi_rri(TCGContext *s, TCGType type, 2280 TCGReg a0, TCGReg a1, tcg_target_long a2) 2281{ 2282 tcg_out_dat_IK(s, COND_AL, ARITH_SBC, ARITH_ADC, a0, a1, a2); 2283} 2284 2285static void tgen_subbi_rir(TCGContext *s, TCGType type, 2286 TCGReg a0, tcg_target_long a1, TCGReg a2) 2287{ 2288 tcg_out_dat_imm(s, COND_AL, ARITH_RSC, a0, a2, encode_imm_nofail(a1)); 2289} 2290 2291static void tgen_subbi_rii(TCGContext *s, TCGType type, 2292 TCGReg a0, tcg_target_long a1, tcg_target_long a2) 2293{ 2294 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, a2); 2295 tgen_subbi_rir(s, TCG_TYPE_I32, a0, a1, TCG_REG_TMP); 2296} 2297 2298static const TCGOutOpAddSubCarry outop_subbi = { 2299 .base.static_constraint = C_O1_I2(r, rI, rIK), 2300 .out_rrr = tgen_subbi_rrr, 2301 .out_rri = tgen_subbi_rri, 2302 .out_rir = tgen_subbi_rir, 2303 .out_rii = tgen_subbi_rii, 2304}; 2305 2306static void tgen_subbio_rrr(TCGContext *s, TCGType type, 2307 TCGReg a0, TCGReg a1, TCGReg a2) 2308{ 2309 tcg_out_dat_reg(s, COND_AL, ARITH_SBC | TO_CPSR, 2310 a0, a1, a2, SHIFT_IMM_LSL(0)); 2311} 2312 2313static void tgen_subbio_rri(TCGContext *s, TCGType type, 2314 TCGReg a0, TCGReg a1, tcg_target_long a2) 2315{ 2316 tcg_out_dat_IK(s, COND_AL, ARITH_SBC | TO_CPSR, ARITH_ADC | TO_CPSR, 2317 a0, a1, a2); 2318} 2319 2320static void tgen_subbio_rir(TCGContext *s, TCGType type, 2321 TCGReg a0, tcg_target_long a1, TCGReg a2) 2322{ 2323 tcg_out_dat_imm(s, COND_AL, ARITH_RSC | TO_CPSR, 2324 a0, a2, encode_imm_nofail(a1)); 2325} 2326 2327static void tgen_subbio_rii(TCGContext *s, TCGType type, 2328 TCGReg a0, tcg_target_long a1, tcg_target_long a2) 2329{ 2330 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP, a2); 2331 tgen_subbio_rir(s, TCG_TYPE_I32, a0, a1, TCG_REG_TMP); 2332} 2333 2334static const TCGOutOpAddSubCarry outop_subbio = { 2335 .base.static_constraint = C_O1_I2(r, rI, rIK), 2336 .out_rrr = tgen_subbio_rrr, 2337 .out_rri = tgen_subbio_rri, 2338 .out_rir = tgen_subbio_rir, 2339 .out_rii = tgen_subbio_rii, 2340}; 2341 2342static void tcg_out_set_borrow(TCGContext *s) 2343{ 2344 tcg_out_movi_apsr_c(s, 0); /* borrow = !carry */ 2345} 2346 2347static void tgen_xor(TCGContext *s, TCGType type, 2348 TCGReg a0, TCGReg a1, TCGReg a2) 2349{ 2350 tcg_out_dat_reg(s, COND_AL, ARITH_EOR, a0, a1, a2, SHIFT_IMM_LSL(0)); 2351} 2352 2353static void tgen_xori(TCGContext *s, TCGType type, 2354 TCGReg a0, TCGReg a1, tcg_target_long a2) 2355{ 2356 tcg_out_dat_imm(s, COND_AL, ARITH_EOR, a0, a1, encode_imm_nofail(a2)); 2357} 2358 2359static const TCGOutOpBinary outop_xor = { 2360 .base.static_constraint = C_O1_I2(r, r, rI), 2361 .out_rrr = tgen_xor, 2362 .out_rri = tgen_xori, 2363}; 2364 2365static void tgen_bswap16(TCGContext *s, TCGType type, 2366 TCGReg rd, TCGReg rn, unsigned flags) 2367{ 2368 if (flags & TCG_BSWAP_OS) { 2369 /* revsh */ 2370 tcg_out32(s, 0x06ff0fb0 | (COND_AL << 28) | (rd << 12) | rn); 2371 return; 2372 } 2373 2374 /* rev16 */ 2375 tcg_out32(s, 0x06bf0fb0 | (COND_AL << 28) | (rd << 12) | rn); 2376 if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { 2377 tcg_out_ext16u(s, rd, rd); 2378 } 2379} 2380 2381static const TCGOutOpBswap outop_bswap16 = { 2382 .base.static_constraint = C_O1_I1(r, r), 2383 .out_rr = tgen_bswap16, 2384}; 2385 2386static void tgen_bswap32(TCGContext *s, TCGType type, 2387 TCGReg rd, TCGReg rn, unsigned flags) 2388{ 2389 /* rev */ 2390 tcg_out32(s, 0x06bf0f30 | (COND_AL << 28) | (rd << 12) | rn); 2391} 2392 2393static const TCGOutOpBswap outop_bswap32 = { 2394 .base.static_constraint = C_O1_I1(r, r), 2395 .out_rr = tgen_bswap32, 2396}; 2397 2398static const TCGOutOpUnary outop_bswap64 = { 2399 .base.static_constraint = C_NotImplemented, 2400}; 2401 2402static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) 2403{ 2404 tgen_subfi(s, type, a0, 0, a1); 2405} 2406 2407static const TCGOutOpUnary outop_neg = { 2408 .base.static_constraint = C_O1_I1(r, r), 2409 .out_rr = tgen_neg, 2410}; 2411 2412static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1) 2413{ 2414 tcg_out_dat_reg(s, COND_AL, ARITH_MVN, a0, 0, a1, SHIFT_IMM_LSL(0)); 2415} 2416 2417static const TCGOutOpUnary outop_not = { 2418 .base.static_constraint = C_O1_I1(r, r), 2419 .out_rr = tgen_not, 2420}; 2421 2422static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond, 2423 TCGReg a0, TCGReg a1, TCGLabel *l) 2424{ 2425 cond = tgen_cmp(s, cond, a0, a1); 2426 tcg_out_goto_label(s, tcg_cond_to_arm_cond[cond], l); 2427} 2428 2429static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond, 2430 TCGReg a0, tcg_target_long a1, TCGLabel *l) 2431{ 2432 cond = tgen_cmpi(s, cond, a0, a1); 2433 tcg_out_goto_label(s, tcg_cond_to_arm_cond[cond], l); 2434} 2435 2436static const TCGOutOpBrcond outop_brcond = { 2437 .base.static_constraint = C_O0_I2(r, rIN), 2438 .out_rr = tgen_brcond, 2439 .out_ri = tgen_brcondi, 2440}; 2441 2442static void finish_setcond(TCGContext *s, TCGCond cond, TCGReg ret, bool neg) 2443{ 2444 tcg_out_movi32(s, tcg_cond_to_arm_cond[tcg_invert_cond(cond)], ret, 0); 2445 tcg_out_movi32(s, tcg_cond_to_arm_cond[cond], ret, neg ? -1 : 1); 2446} 2447 2448static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond, 2449 TCGReg a0, TCGReg a1, TCGReg a2) 2450{ 2451 cond = tgen_cmp(s, cond, a1, a2); 2452 finish_setcond(s, cond, a0, false); 2453} 2454 2455static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond, 2456 TCGReg a0, TCGReg a1, tcg_target_long a2) 2457{ 2458 cond = tgen_cmpi(s, cond, a1, a2); 2459 finish_setcond(s, cond, a0, false); 2460} 2461 2462static const TCGOutOpSetcond outop_setcond = { 2463 .base.static_constraint = C_O1_I2(r, r, rIN), 2464 .out_rrr = tgen_setcond, 2465 .out_rri = tgen_setcondi, 2466}; 2467 2468static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond, 2469 TCGReg a0, TCGReg a1, TCGReg a2) 2470{ 2471 cond = tgen_cmp(s, cond, a1, a2); 2472 finish_setcond(s, cond, a0, true); 2473} 2474 2475static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond, 2476 TCGReg a0, TCGReg a1, tcg_target_long a2) 2477{ 2478 cond = tgen_cmpi(s, cond, a1, a2); 2479 finish_setcond(s, cond, a0, true); 2480} 2481 2482static const TCGOutOpSetcond outop_negsetcond = { 2483 .base.static_constraint = C_O1_I2(r, r, rIN), 2484 .out_rrr = tgen_negsetcond, 2485 .out_rri = tgen_negsetcondi, 2486}; 2487 2488static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond, 2489 TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, 2490 TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf) 2491{ 2492 cond = tcg_out_cmp(s, cond, c1, c2, const_c2); 2493 tcg_out_dat_rIK(s, tcg_cond_to_arm_cond[cond], ARITH_MOV, ARITH_MVN, 2494 ret, 0, vt, const_vt); 2495} 2496 2497static const TCGOutOpMovcond outop_movcond = { 2498 .base.static_constraint = C_O1_I4(r, r, rIN, rIK, 0), 2499 .out = tgen_movcond, 2500}; 2501 2502static void tgen_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, 2503 TCGArg bl, bool const_bl, TCGArg bh, bool const_bh, 2504 TCGLabel *l) 2505{ 2506 cond = tcg_out_cmp2(s, cond, al, ah, bl, const_bl, bh, const_bh); 2507 tcg_out_goto_label(s, tcg_cond_to_arm_cond[cond], l); 2508} 2509 2510static const TCGOutOpBrcond2 outop_brcond2 = { 2511 .base.static_constraint = C_O0_I4(r, r, rI, rI), 2512 .out = tgen_brcond2, 2513}; 2514 2515static void tgen_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, 2516 TCGReg al, TCGReg ah, 2517 TCGArg bl, bool const_bl, 2518 TCGArg bh, bool const_bh) 2519{ 2520 cond = tcg_out_cmp2(s, cond, al, ah, bl, const_bl, bh, const_bh); 2521 finish_setcond(s, cond, ret, false); 2522} 2523 2524static const TCGOutOpSetcond2 outop_setcond2 = { 2525 .base.static_constraint = C_O1_I4(r, r, r, rI, rI), 2526 .out = tgen_setcond2, 2527}; 2528 2529static void tgen_extract2(TCGContext *s, TCGType type, TCGReg a0, 2530 TCGReg a1, TCGReg a2, unsigned shr) 2531{ 2532 /* We can do extract2 in 2 insns, vs the 3 required otherwise. */ 2533 tgen_shli(s, TCG_TYPE_I32, TCG_REG_TMP, a2, 32 - shr); 2534 tcg_out_dat_reg(s, COND_AL, ARITH_ORR, a0, TCG_REG_TMP, 2535 a1, SHIFT_IMM_LSR(shr)); 2536} 2537 2538static const TCGOutOpExtract2 outop_extract2 = { 2539 .base.static_constraint = C_O1_I2(r, r, r), 2540 .out_rrr = tgen_extract2, 2541}; 2542 2543static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg rd, 2544 TCGReg rn, ptrdiff_t offset) 2545{ 2546 if (offset > 0xfff || offset < -0xfff) { 2547 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); 2548 tcg_out_ld8_r(s, COND_AL, rd, rn, TCG_REG_TMP); 2549 } else { 2550 tcg_out_ld8_12(s, COND_AL, rd, rn, offset); 2551 } 2552} 2553 2554static const TCGOutOpLoad outop_ld8u = { 2555 .base.static_constraint = C_O1_I1(r, r), 2556 .out = tgen_ld8u, 2557}; 2558 2559static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg rd, 2560 TCGReg rn, ptrdiff_t offset) 2561{ 2562 if (offset > 0xff || offset < -0xff) { 2563 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); 2564 tcg_out_ld8s_r(s, COND_AL, rd, rn, TCG_REG_TMP); 2565 } else { 2566 tcg_out_ld8s_8(s, COND_AL, rd, rn, offset); 2567 } 2568} 2569 2570static const TCGOutOpLoad outop_ld8s = { 2571 .base.static_constraint = C_O1_I1(r, r), 2572 .out = tgen_ld8s, 2573}; 2574 2575static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg rd, 2576 TCGReg rn, ptrdiff_t offset) 2577{ 2578 if (offset > 0xff || offset < -0xff) { 2579 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); 2580 tcg_out_ld16u_r(s, COND_AL, rd, rn, TCG_REG_TMP); 2581 } else { 2582 tcg_out_ld16u_8(s, COND_AL, rd, rn, offset); 2583 } 2584} 2585 2586static const TCGOutOpLoad outop_ld16u = { 2587 .base.static_constraint = C_O1_I1(r, r), 2588 .out = tgen_ld16u, 2589}; 2590 2591static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg rd, 2592 TCGReg rn, ptrdiff_t offset) 2593{ 2594 if (offset > 0xff || offset < -0xff) { 2595 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); 2596 tcg_out_ld16s_r(s, COND_AL, rd, rn, TCG_REG_TMP); 2597 } else { 2598 tcg_out_ld16s_8(s, COND_AL, rd, rn, offset); 2599 } 2600} 2601 2602static const TCGOutOpLoad outop_ld16s = { 2603 .base.static_constraint = C_O1_I1(r, r), 2604 .out = tgen_ld16s, 2605}; 2606 2607static void tgen_st8(TCGContext *s, TCGType type, TCGReg rd, 2608 TCGReg rn, ptrdiff_t offset) 2609{ 2610 if (offset > 0xfff || offset < -0xfff) { 2611 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); 2612 tcg_out_st8_r(s, COND_AL, rd, rn, TCG_REG_TMP); 2613 } else { 2614 tcg_out_st8_12(s, COND_AL, rd, rn, offset); 2615 } 2616} 2617 2618static const TCGOutOpStore outop_st8 = { 2619 .base.static_constraint = C_O0_I2(r, r), 2620 .out_r = tgen_st8, 2621}; 2622 2623static void tgen_st16(TCGContext *s, TCGType type, TCGReg rd, 2624 TCGReg rn, ptrdiff_t offset) 2625{ 2626 if (offset > 0xff || offset < -0xff) { 2627 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, offset); 2628 tcg_out_st16_r(s, COND_AL, rd, rn, TCG_REG_TMP); 2629 } else { 2630 tcg_out_st16_8(s, COND_AL, rd, rn, offset); 2631 } 2632} 2633 2634static const TCGOutOpStore outop_st16 = { 2635 .base.static_constraint = C_O0_I2(r, r), 2636 .out_r = tgen_st16, 2637}; 2638 2639static const TCGOutOpStore outop_st = { 2640 .base.static_constraint = C_O0_I2(r, r), 2641 .out_r = tcg_out_st, 2642}; 2643 2644static TCGConstraintSetIndex 2645tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags) 2646{ 2647 switch (op) { 2648 case INDEX_op_st_vec: 2649 return C_O0_I2(w, r); 2650 case INDEX_op_ld_vec: 2651 case INDEX_op_dupm_vec: 2652 return C_O1_I1(w, r); 2653 case INDEX_op_dup_vec: 2654 return C_O1_I1(w, wr); 2655 case INDEX_op_abs_vec: 2656 case INDEX_op_neg_vec: 2657 case INDEX_op_not_vec: 2658 case INDEX_op_shli_vec: 2659 case INDEX_op_shri_vec: 2660 case INDEX_op_sari_vec: 2661 return C_O1_I1(w, w); 2662 case INDEX_op_dup2_vec: 2663 case INDEX_op_add_vec: 2664 case INDEX_op_mul_vec: 2665 case INDEX_op_smax_vec: 2666 case INDEX_op_smin_vec: 2667 case INDEX_op_ssadd_vec: 2668 case INDEX_op_sssub_vec: 2669 case INDEX_op_sub_vec: 2670 case INDEX_op_umax_vec: 2671 case INDEX_op_umin_vec: 2672 case INDEX_op_usadd_vec: 2673 case INDEX_op_ussub_vec: 2674 case INDEX_op_xor_vec: 2675 case INDEX_op_arm_sshl_vec: 2676 case INDEX_op_arm_ushl_vec: 2677 return C_O1_I2(w, w, w); 2678 case INDEX_op_arm_sli_vec: 2679 return C_O1_I2(w, 0, w); 2680 case INDEX_op_or_vec: 2681 case INDEX_op_andc_vec: 2682 return C_O1_I2(w, w, wO); 2683 case INDEX_op_and_vec: 2684 case INDEX_op_orc_vec: 2685 return C_O1_I2(w, w, wV); 2686 case INDEX_op_cmp_vec: 2687 return C_O1_I2(w, w, wZ); 2688 case INDEX_op_bitsel_vec: 2689 return C_O1_I3(w, w, w, w); 2690 default: 2691 return C_NotImplemented; 2692 } 2693} 2694 2695static void tcg_target_init(TCGContext *s) 2696{ 2697 /* 2698 * Only probe for the platform and capabilities if we haven't already 2699 * determined maximum values at compile time. 2700 */ 2701#if !defined(use_idiv_instructions) || !defined(use_neon_instructions) 2702 { 2703 unsigned long hwcap = qemu_getauxval(AT_HWCAP); 2704#ifndef use_idiv_instructions 2705 use_idiv_instructions = (hwcap & HWCAP_ARM_IDIVA) != 0; 2706#endif 2707#ifndef use_neon_instructions 2708 use_neon_instructions = (hwcap & HWCAP_ARM_NEON) != 0; 2709#endif 2710 } 2711#endif 2712 2713 if (__ARM_ARCH < 7) { 2714 const char *pl = (const char *)qemu_getauxval(AT_PLATFORM); 2715 if (pl != NULL && pl[0] == 'v' && pl[1] >= '4' && pl[1] <= '9') { 2716 arm_arch = pl[1] - '0'; 2717 } 2718 2719 if (arm_arch < 6) { 2720 error_report("TCG: ARMv%d is unsupported; exiting", arm_arch); 2721 exit(EXIT_FAILURE); 2722 } 2723 } 2724 2725 tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; 2726 2727 tcg_target_call_clobber_regs = 0; 2728 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0); 2729 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1); 2730 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2); 2731 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3); 2732 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R12); 2733 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14); 2734 2735 if (use_neon_instructions) { 2736 tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS; 2737 tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS; 2738 2739 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q0); 2740 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q1); 2741 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q2); 2742 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q3); 2743 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q8); 2744 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q9); 2745 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q10); 2746 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q11); 2747 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q12); 2748 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q13); 2749 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q14); 2750 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_Q15); 2751 } 2752 2753 s->reserved_regs = 0; 2754 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK); 2755 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP); 2756 tcg_regset_set_reg(s->reserved_regs, TCG_REG_PC); 2757 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP); 2758} 2759 2760static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, 2761 TCGReg arg1, intptr_t arg2) 2762{ 2763 switch (type) { 2764 case TCG_TYPE_I32: 2765 tcg_out_ld32u(s, COND_AL, arg, arg1, arg2); 2766 return; 2767 case TCG_TYPE_V64: 2768 /* regs 1; size 8; align 8 */ 2769 tcg_out_vldst(s, INSN_VLD1 | 0x7d0, arg, arg1, arg2); 2770 return; 2771 case TCG_TYPE_V128: 2772 /* 2773 * We have only 8-byte alignment for the stack per the ABI. 2774 * Rather than dynamically re-align the stack, it's easier 2775 * to simply not request alignment beyond that. So: 2776 * regs 2; size 8; align 8 2777 */ 2778 tcg_out_vldst(s, INSN_VLD1 | 0xad0, arg, arg1, arg2); 2779 return; 2780 default: 2781 g_assert_not_reached(); 2782 } 2783} 2784 2785static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, 2786 TCGReg arg1, intptr_t arg2) 2787{ 2788 switch (type) { 2789 case TCG_TYPE_I32: 2790 tcg_out_st32(s, COND_AL, arg, arg1, arg2); 2791 return; 2792 case TCG_TYPE_V64: 2793 /* regs 1; size 8; align 8 */ 2794 tcg_out_vldst(s, INSN_VST1 | 0x7d0, arg, arg1, arg2); 2795 return; 2796 case TCG_TYPE_V128: 2797 /* See tcg_out_ld re alignment: regs 2; size 8; align 8 */ 2798 tcg_out_vldst(s, INSN_VST1 | 0xad0, arg, arg1, arg2); 2799 return; 2800 default: 2801 g_assert_not_reached(); 2802 } 2803} 2804 2805static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 2806 TCGReg base, intptr_t ofs) 2807{ 2808 return false; 2809} 2810 2811static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 2812{ 2813 if (ret == arg) { 2814 return true; 2815 } 2816 switch (type) { 2817 case TCG_TYPE_I32: 2818 if (ret < TCG_REG_Q0 && arg < TCG_REG_Q0) { 2819 tcg_out_mov_reg(s, COND_AL, ret, arg); 2820 return true; 2821 } 2822 return false; 2823 2824 case TCG_TYPE_V64: 2825 case TCG_TYPE_V128: 2826 /* "VMOV D,N" is an alias for "VORR D,N,N". */ 2827 tcg_out_vreg3(s, INSN_VORR, type - TCG_TYPE_V64, 0, ret, arg, arg); 2828 return true; 2829 2830 default: 2831 g_assert_not_reached(); 2832 } 2833} 2834 2835static void tcg_out_movi(TCGContext *s, TCGType type, 2836 TCGReg ret, tcg_target_long arg) 2837{ 2838 tcg_debug_assert(type == TCG_TYPE_I32); 2839 tcg_debug_assert(ret < TCG_REG_Q0); 2840 tcg_out_movi32(s, COND_AL, ret, arg); 2841} 2842 2843static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) 2844{ 2845 return false; 2846} 2847 2848static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, 2849 tcg_target_long imm) 2850{ 2851 int enc, opc = ARITH_ADD; 2852 2853 /* All of the easiest immediates to encode are positive. */ 2854 if (imm < 0) { 2855 imm = -imm; 2856 opc = ARITH_SUB; 2857 } 2858 enc = encode_imm(imm); 2859 if (enc >= 0) { 2860 tcg_out_dat_imm(s, COND_AL, opc, rd, rs, enc); 2861 } else { 2862 tcg_out_movi32(s, COND_AL, TCG_REG_TMP, imm); 2863 tcg_out_dat_reg(s, COND_AL, opc, rd, rs, 2864 TCG_REG_TMP, SHIFT_IMM_LSL(0)); 2865 } 2866} 2867 2868/* Type is always V128, with I64 elements. */ 2869static void tcg_out_dup2_vec(TCGContext *s, TCGReg rd, TCGReg rl, TCGReg rh) 2870{ 2871 /* Move high element into place first. */ 2872 /* VMOV Dd+1, Ds */ 2873 tcg_out_vreg3(s, INSN_VORR | (1 << 12), 0, 0, rd, rh, rh); 2874 /* Move low element into place; tcg_out_mov will check for nop. */ 2875 tcg_out_mov(s, TCG_TYPE_V64, rd, rl); 2876} 2877 2878static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 2879 TCGReg rd, TCGReg rs) 2880{ 2881 int q = type - TCG_TYPE_V64; 2882 2883 if (vece == MO_64) { 2884 if (type == TCG_TYPE_V128) { 2885 tcg_out_dup2_vec(s, rd, rs, rs); 2886 } else { 2887 tcg_out_mov(s, TCG_TYPE_V64, rd, rs); 2888 } 2889 } else if (rs < TCG_REG_Q0) { 2890 int b = (vece == MO_8); 2891 int e = (vece == MO_16); 2892 tcg_out32(s, INSN_VDUP_G | (b << 22) | (q << 21) | (e << 5) | 2893 encode_vn(rd) | (rs << 12)); 2894 } else { 2895 int imm4 = 1 << vece; 2896 tcg_out32(s, INSN_VDUP_S | (imm4 << 16) | (q << 6) | 2897 encode_vd(rd) | encode_vm(rs)); 2898 } 2899 return true; 2900} 2901 2902static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 2903 TCGReg rd, TCGReg base, intptr_t offset) 2904{ 2905 if (vece == MO_64) { 2906 tcg_out_ld(s, TCG_TYPE_V64, rd, base, offset); 2907 if (type == TCG_TYPE_V128) { 2908 tcg_out_dup2_vec(s, rd, rd, rd); 2909 } 2910 } else { 2911 int q = type - TCG_TYPE_V64; 2912 tcg_out_vldst(s, INSN_VLD1R | (vece << 6) | (q << 5), 2913 rd, base, offset); 2914 } 2915 return true; 2916} 2917 2918static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 2919 TCGReg rd, int64_t v64) 2920{ 2921 int q = type - TCG_TYPE_V64; 2922 int cmode, imm8, i; 2923 2924 /* Test all bytes equal first. */ 2925 if (vece == MO_8) { 2926 tcg_out_vmovi(s, rd, q, 0, 0xe, v64); 2927 return; 2928 } 2929 2930 /* 2931 * Test all bytes 0x00 or 0xff second. This can match cases that 2932 * might otherwise take 2 or 3 insns for MO_16 or MO_32 below. 2933 */ 2934 for (i = imm8 = 0; i < 8; i++) { 2935 uint8_t byte = v64 >> (i * 8); 2936 if (byte == 0xff) { 2937 imm8 |= 1 << i; 2938 } else if (byte != 0) { 2939 goto fail_bytes; 2940 } 2941 } 2942 tcg_out_vmovi(s, rd, q, 1, 0xe, imm8); 2943 return; 2944 fail_bytes: 2945 2946 /* 2947 * Tests for various replications. For each element width, if we 2948 * cannot find an expansion there's no point checking a larger 2949 * width because we already know by replication it cannot match. 2950 */ 2951 if (vece == MO_16) { 2952 uint16_t v16 = v64; 2953 2954 if (is_shimm16(v16, &cmode, &imm8)) { 2955 tcg_out_vmovi(s, rd, q, 0, cmode, imm8); 2956 return; 2957 } 2958 if (is_shimm16(~v16, &cmode, &imm8)) { 2959 tcg_out_vmovi(s, rd, q, 1, cmode, imm8); 2960 return; 2961 } 2962 2963 /* 2964 * Otherwise, all remaining constants can be loaded in two insns: 2965 * rd = v16 & 0xff, rd |= v16 & 0xff00. 2966 */ 2967 tcg_out_vmovi(s, rd, q, 0, 0x8, v16 & 0xff); 2968 tcg_out_vmovi(s, rd, q, 0, 0xb, v16 >> 8); /* VORRI */ 2969 return; 2970 } 2971 2972 if (vece == MO_32) { 2973 uint32_t v32 = v64; 2974 2975 if (is_shimm32(v32, &cmode, &imm8) || 2976 is_soimm32(v32, &cmode, &imm8)) { 2977 tcg_out_vmovi(s, rd, q, 0, cmode, imm8); 2978 return; 2979 } 2980 if (is_shimm32(~v32, &cmode, &imm8) || 2981 is_soimm32(~v32, &cmode, &imm8)) { 2982 tcg_out_vmovi(s, rd, q, 1, cmode, imm8); 2983 return; 2984 } 2985 2986 /* 2987 * Restrict the set of constants to those we can load with 2988 * two instructions. Others we load from the pool. 2989 */ 2990 i = is_shimm32_pair(v32, &cmode, &imm8); 2991 if (i) { 2992 tcg_out_vmovi(s, rd, q, 0, cmode, imm8); 2993 tcg_out_vmovi(s, rd, q, 0, i | 1, extract32(v32, i * 4, 8)); 2994 return; 2995 } 2996 i = is_shimm32_pair(~v32, &cmode, &imm8); 2997 if (i) { 2998 tcg_out_vmovi(s, rd, q, 1, cmode, imm8); 2999 tcg_out_vmovi(s, rd, q, 1, i | 1, extract32(~v32, i * 4, 8)); 3000 return; 3001 } 3002 } 3003 3004 /* 3005 * As a last resort, load from the constant pool. 3006 */ 3007 if (!q || vece == MO_64) { 3008 new_pool_l2(s, R_ARM_PC11, s->code_ptr, 0, v64, v64 >> 32); 3009 /* VLDR Dd, [pc + offset] */ 3010 tcg_out32(s, INSN_VLDR_D | encode_vd(rd) | (0xf << 16)); 3011 if (q) { 3012 tcg_out_dup2_vec(s, rd, rd, rd); 3013 } 3014 } else { 3015 new_pool_label(s, (uint32_t)v64, R_ARM_PC8, s->code_ptr, 0); 3016 /* add tmp, pc, offset */ 3017 tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_TMP, TCG_REG_PC, 0); 3018 tcg_out_dupm_vec(s, type, MO_32, rd, TCG_REG_TMP, 0); 3019 } 3020} 3021 3022static const ARMInsn vec_cmp_insn[16] = { 3023 [TCG_COND_EQ] = INSN_VCEQ, 3024 [TCG_COND_GT] = INSN_VCGT, 3025 [TCG_COND_GE] = INSN_VCGE, 3026 [TCG_COND_GTU] = INSN_VCGT_U, 3027 [TCG_COND_GEU] = INSN_VCGE_U, 3028}; 3029 3030static const ARMInsn vec_cmp0_insn[16] = { 3031 [TCG_COND_EQ] = INSN_VCEQ0, 3032 [TCG_COND_GT] = INSN_VCGT0, 3033 [TCG_COND_GE] = INSN_VCGE0, 3034 [TCG_COND_LT] = INSN_VCLT0, 3035 [TCG_COND_LE] = INSN_VCLE0, 3036}; 3037 3038static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 3039 unsigned vecl, unsigned vece, 3040 const TCGArg args[TCG_MAX_OP_ARGS], 3041 const int const_args[TCG_MAX_OP_ARGS]) 3042{ 3043 TCGType type = vecl + TCG_TYPE_V64; 3044 unsigned q = vecl; 3045 TCGArg a0, a1, a2, a3; 3046 int cmode, imm8; 3047 3048 a0 = args[0]; 3049 a1 = args[1]; 3050 a2 = args[2]; 3051 3052 switch (opc) { 3053 case INDEX_op_ld_vec: 3054 tcg_out_ld(s, type, a0, a1, a2); 3055 return; 3056 case INDEX_op_st_vec: 3057 tcg_out_st(s, type, a0, a1, a2); 3058 return; 3059 case INDEX_op_dupm_vec: 3060 tcg_out_dupm_vec(s, type, vece, a0, a1, a2); 3061 return; 3062 case INDEX_op_dup2_vec: 3063 tcg_out_dup2_vec(s, a0, a1, a2); 3064 return; 3065 case INDEX_op_abs_vec: 3066 tcg_out_vreg2(s, INSN_VABS, q, vece, a0, a1); 3067 return; 3068 case INDEX_op_neg_vec: 3069 tcg_out_vreg2(s, INSN_VNEG, q, vece, a0, a1); 3070 return; 3071 case INDEX_op_not_vec: 3072 tcg_out_vreg2(s, INSN_VMVN, q, 0, a0, a1); 3073 return; 3074 case INDEX_op_add_vec: 3075 tcg_out_vreg3(s, INSN_VADD, q, vece, a0, a1, a2); 3076 return; 3077 case INDEX_op_mul_vec: 3078 tcg_out_vreg3(s, INSN_VMUL, q, vece, a0, a1, a2); 3079 return; 3080 case INDEX_op_smax_vec: 3081 tcg_out_vreg3(s, INSN_VMAX, q, vece, a0, a1, a2); 3082 return; 3083 case INDEX_op_smin_vec: 3084 tcg_out_vreg3(s, INSN_VMIN, q, vece, a0, a1, a2); 3085 return; 3086 case INDEX_op_sub_vec: 3087 tcg_out_vreg3(s, INSN_VSUB, q, vece, a0, a1, a2); 3088 return; 3089 case INDEX_op_ssadd_vec: 3090 tcg_out_vreg3(s, INSN_VQADD, q, vece, a0, a1, a2); 3091 return; 3092 case INDEX_op_sssub_vec: 3093 tcg_out_vreg3(s, INSN_VQSUB, q, vece, a0, a1, a2); 3094 return; 3095 case INDEX_op_umax_vec: 3096 tcg_out_vreg3(s, INSN_VMAX_U, q, vece, a0, a1, a2); 3097 return; 3098 case INDEX_op_umin_vec: 3099 tcg_out_vreg3(s, INSN_VMIN_U, q, vece, a0, a1, a2); 3100 return; 3101 case INDEX_op_usadd_vec: 3102 tcg_out_vreg3(s, INSN_VQADD_U, q, vece, a0, a1, a2); 3103 return; 3104 case INDEX_op_ussub_vec: 3105 tcg_out_vreg3(s, INSN_VQSUB_U, q, vece, a0, a1, a2); 3106 return; 3107 case INDEX_op_xor_vec: 3108 tcg_out_vreg3(s, INSN_VEOR, q, 0, a0, a1, a2); 3109 return; 3110 case INDEX_op_arm_sshl_vec: 3111 /* 3112 * Note that Vm is the data and Vn is the shift count, 3113 * therefore the arguments appear reversed. 3114 */ 3115 tcg_out_vreg3(s, INSN_VSHL_S, q, vece, a0, a2, a1); 3116 return; 3117 case INDEX_op_arm_ushl_vec: 3118 /* See above. */ 3119 tcg_out_vreg3(s, INSN_VSHL_U, q, vece, a0, a2, a1); 3120 return; 3121 case INDEX_op_shli_vec: 3122 tcg_out_vshifti(s, INSN_VSHLI, q, a0, a1, a2 + (8 << vece)); 3123 return; 3124 case INDEX_op_shri_vec: 3125 tcg_out_vshifti(s, INSN_VSHRI, q, a0, a1, (16 << vece) - a2); 3126 return; 3127 case INDEX_op_sari_vec: 3128 tcg_out_vshifti(s, INSN_VSARI, q, a0, a1, (16 << vece) - a2); 3129 return; 3130 case INDEX_op_arm_sli_vec: 3131 tcg_out_vshifti(s, INSN_VSLI, q, a0, a2, args[3] + (8 << vece)); 3132 return; 3133 3134 case INDEX_op_andc_vec: 3135 if (!const_args[2]) { 3136 tcg_out_vreg3(s, INSN_VBIC, q, 0, a0, a1, a2); 3137 return; 3138 } 3139 a2 = ~a2; 3140 /* fall through */ 3141 case INDEX_op_and_vec: 3142 if (const_args[2]) { 3143 is_shimm1632(~a2, &cmode, &imm8); 3144 if (a0 == a1) { 3145 tcg_out_vmovi(s, a0, q, 1, cmode | 1, imm8); /* VBICI */ 3146 return; 3147 } 3148 tcg_out_vmovi(s, a0, q, 1, cmode, imm8); /* VMVNI */ 3149 a2 = a0; 3150 } 3151 tcg_out_vreg3(s, INSN_VAND, q, 0, a0, a1, a2); 3152 return; 3153 3154 case INDEX_op_orc_vec: 3155 if (!const_args[2]) { 3156 tcg_out_vreg3(s, INSN_VORN, q, 0, a0, a1, a2); 3157 return; 3158 } 3159 a2 = ~a2; 3160 /* fall through */ 3161 case INDEX_op_or_vec: 3162 if (const_args[2]) { 3163 is_shimm1632(a2, &cmode, &imm8); 3164 if (a0 == a1) { 3165 tcg_out_vmovi(s, a0, q, 0, cmode | 1, imm8); /* VORRI */ 3166 return; 3167 } 3168 tcg_out_vmovi(s, a0, q, 0, cmode, imm8); /* VMOVI */ 3169 a2 = a0; 3170 } 3171 tcg_out_vreg3(s, INSN_VORR, q, 0, a0, a1, a2); 3172 return; 3173 3174 case INDEX_op_cmp_vec: 3175 { 3176 TCGCond cond = args[3]; 3177 ARMInsn insn; 3178 3179 switch (cond) { 3180 case TCG_COND_NE: 3181 if (const_args[2]) { 3182 tcg_out_vreg3(s, INSN_VTST, q, vece, a0, a1, a1); 3183 } else { 3184 tcg_out_vreg3(s, INSN_VCEQ, q, vece, a0, a1, a2); 3185 tcg_out_vreg2(s, INSN_VMVN, q, 0, a0, a0); 3186 } 3187 break; 3188 3189 case TCG_COND_TSTNE: 3190 case TCG_COND_TSTEQ: 3191 if (const_args[2]) { 3192 /* (x & 0) == 0 */ 3193 tcg_out_dupi_vec(s, type, MO_8, a0, 3194 -(cond == TCG_COND_TSTEQ)); 3195 break; 3196 } 3197 tcg_out_vreg3(s, INSN_VTST, q, vece, a0, a1, a2); 3198 if (cond == TCG_COND_TSTEQ) { 3199 tcg_out_vreg2(s, INSN_VMVN, q, 0, a0, a0); 3200 } 3201 break; 3202 3203 default: 3204 if (const_args[2]) { 3205 insn = vec_cmp0_insn[cond]; 3206 if (insn) { 3207 tcg_out_vreg2(s, insn, q, vece, a0, a1); 3208 return; 3209 } 3210 tcg_out_dupi_vec(s, type, MO_8, TCG_VEC_TMP, 0); 3211 a2 = TCG_VEC_TMP; 3212 } 3213 insn = vec_cmp_insn[cond]; 3214 if (insn == 0) { 3215 TCGArg t; 3216 t = a1, a1 = a2, a2 = t; 3217 cond = tcg_swap_cond(cond); 3218 insn = vec_cmp_insn[cond]; 3219 tcg_debug_assert(insn != 0); 3220 } 3221 tcg_out_vreg3(s, insn, q, vece, a0, a1, a2); 3222 break; 3223 } 3224 } 3225 return; 3226 3227 case INDEX_op_bitsel_vec: 3228 a3 = args[3]; 3229 if (a0 == a3) { 3230 tcg_out_vreg3(s, INSN_VBIT, q, 0, a0, a2, a1); 3231 } else if (a0 == a2) { 3232 tcg_out_vreg3(s, INSN_VBIF, q, 0, a0, a3, a1); 3233 } else { 3234 tcg_out_mov(s, type, a0, a1); 3235 tcg_out_vreg3(s, INSN_VBSL, q, 0, a0, a2, a3); 3236 } 3237 return; 3238 3239 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ 3240 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ 3241 default: 3242 g_assert_not_reached(); 3243 } 3244} 3245 3246int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) 3247{ 3248 switch (opc) { 3249 case INDEX_op_add_vec: 3250 case INDEX_op_sub_vec: 3251 case INDEX_op_and_vec: 3252 case INDEX_op_andc_vec: 3253 case INDEX_op_or_vec: 3254 case INDEX_op_orc_vec: 3255 case INDEX_op_xor_vec: 3256 case INDEX_op_not_vec: 3257 case INDEX_op_shli_vec: 3258 case INDEX_op_shri_vec: 3259 case INDEX_op_sari_vec: 3260 case INDEX_op_ssadd_vec: 3261 case INDEX_op_sssub_vec: 3262 case INDEX_op_usadd_vec: 3263 case INDEX_op_ussub_vec: 3264 case INDEX_op_bitsel_vec: 3265 return 1; 3266 case INDEX_op_abs_vec: 3267 case INDEX_op_cmp_vec: 3268 case INDEX_op_mul_vec: 3269 case INDEX_op_neg_vec: 3270 case INDEX_op_smax_vec: 3271 case INDEX_op_smin_vec: 3272 case INDEX_op_umax_vec: 3273 case INDEX_op_umin_vec: 3274 return vece < MO_64; 3275 case INDEX_op_shlv_vec: 3276 case INDEX_op_shrv_vec: 3277 case INDEX_op_sarv_vec: 3278 case INDEX_op_rotli_vec: 3279 case INDEX_op_rotlv_vec: 3280 case INDEX_op_rotrv_vec: 3281 return -1; 3282 default: 3283 return 0; 3284 } 3285} 3286 3287void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, 3288 TCGArg a0, ...) 3289{ 3290 va_list va; 3291 TCGv_vec v0, v1, v2, t1, t2, c1; 3292 TCGArg a2; 3293 3294 va_start(va, a0); 3295 v0 = temp_tcgv_vec(arg_temp(a0)); 3296 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 3297 a2 = va_arg(va, TCGArg); 3298 va_end(va); 3299 3300 switch (opc) { 3301 case INDEX_op_shlv_vec: 3302 /* 3303 * Merely propagate shlv_vec to arm_ushl_vec. 3304 * In this way we don't set TCG_TARGET_HAS_shv_vec 3305 * because everything is done via expansion. 3306 */ 3307 v2 = temp_tcgv_vec(arg_temp(a2)); 3308 vec_gen_3(INDEX_op_arm_ushl_vec, type, vece, tcgv_vec_arg(v0), 3309 tcgv_vec_arg(v1), tcgv_vec_arg(v2)); 3310 break; 3311 3312 case INDEX_op_shrv_vec: 3313 case INDEX_op_sarv_vec: 3314 /* Right shifts are negative left shifts for NEON. */ 3315 v2 = temp_tcgv_vec(arg_temp(a2)); 3316 t1 = tcg_temp_new_vec(type); 3317 tcg_gen_neg_vec(vece, t1, v2); 3318 if (opc == INDEX_op_shrv_vec) { 3319 opc = INDEX_op_arm_ushl_vec; 3320 } else { 3321 opc = INDEX_op_arm_sshl_vec; 3322 } 3323 vec_gen_3(opc, type, vece, tcgv_vec_arg(v0), 3324 tcgv_vec_arg(v1), tcgv_vec_arg(t1)); 3325 tcg_temp_free_vec(t1); 3326 break; 3327 3328 case INDEX_op_rotli_vec: 3329 t1 = tcg_temp_new_vec(type); 3330 tcg_gen_shri_vec(vece, t1, v1, -a2 & ((8 << vece) - 1)); 3331 vec_gen_4(INDEX_op_arm_sli_vec, type, vece, 3332 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(v1), a2); 3333 tcg_temp_free_vec(t1); 3334 break; 3335 3336 case INDEX_op_rotlv_vec: 3337 v2 = temp_tcgv_vec(arg_temp(a2)); 3338 t1 = tcg_temp_new_vec(type); 3339 c1 = tcg_constant_vec(type, vece, 8 << vece); 3340 tcg_gen_sub_vec(vece, t1, v2, c1); 3341 /* Right shifts are negative left shifts for NEON. */ 3342 vec_gen_3(INDEX_op_arm_ushl_vec, type, vece, tcgv_vec_arg(t1), 3343 tcgv_vec_arg(v1), tcgv_vec_arg(t1)); 3344 vec_gen_3(INDEX_op_arm_ushl_vec, type, vece, tcgv_vec_arg(v0), 3345 tcgv_vec_arg(v1), tcgv_vec_arg(v2)); 3346 tcg_gen_or_vec(vece, v0, v0, t1); 3347 tcg_temp_free_vec(t1); 3348 break; 3349 3350 case INDEX_op_rotrv_vec: 3351 v2 = temp_tcgv_vec(arg_temp(a2)); 3352 t1 = tcg_temp_new_vec(type); 3353 t2 = tcg_temp_new_vec(type); 3354 c1 = tcg_constant_vec(type, vece, 8 << vece); 3355 tcg_gen_neg_vec(vece, t1, v2); 3356 tcg_gen_sub_vec(vece, t2, c1, v2); 3357 /* Right shifts are negative left shifts for NEON. */ 3358 vec_gen_3(INDEX_op_arm_ushl_vec, type, vece, tcgv_vec_arg(t1), 3359 tcgv_vec_arg(v1), tcgv_vec_arg(t1)); 3360 vec_gen_3(INDEX_op_arm_ushl_vec, type, vece, tcgv_vec_arg(t2), 3361 tcgv_vec_arg(v1), tcgv_vec_arg(t2)); 3362 tcg_gen_or_vec(vece, v0, t1, t2); 3363 tcg_temp_free_vec(t1); 3364 tcg_temp_free_vec(t2); 3365 break; 3366 3367 default: 3368 g_assert_not_reached(); 3369 } 3370} 3371 3372static void tcg_out_nop_fill(tcg_insn_unit *p, int count) 3373{ 3374 int i; 3375 for (i = 0; i < count; ++i) { 3376 p[i] = INSN_NOP; 3377 } 3378} 3379 3380/* Compute frame size via macros, to share between tcg_target_qemu_prologue 3381 and tcg_register_jit. */ 3382 3383#define PUSH_SIZE ((11 - 4 + 1 + 1) * sizeof(tcg_target_long)) 3384 3385#define FRAME_SIZE \ 3386 ((PUSH_SIZE \ 3387 + TCG_STATIC_CALL_ARGS_SIZE \ 3388 + CPU_TEMP_BUF_NLONGS * sizeof(long) \ 3389 + TCG_TARGET_STACK_ALIGN - 1) \ 3390 & -TCG_TARGET_STACK_ALIGN) 3391 3392#define STACK_ADDEND (FRAME_SIZE - PUSH_SIZE) 3393 3394static void tcg_target_qemu_prologue(TCGContext *s) 3395{ 3396 /* Calling convention requires us to save r4-r11 and lr. */ 3397 /* stmdb sp!, { r4 - r11, lr } */ 3398 tcg_out_ldstm(s, COND_AL, INSN_STMDB, TCG_REG_CALL_STACK, 3399 (1 << TCG_REG_R4) | (1 << TCG_REG_R5) | (1 << TCG_REG_R6) | 3400 (1 << TCG_REG_R7) | (1 << TCG_REG_R8) | (1 << TCG_REG_R9) | 3401 (1 << TCG_REG_R10) | (1 << TCG_REG_R11) | (1 << TCG_REG_R14)); 3402 3403 /* Reserve callee argument and tcg temp space. */ 3404 tcg_out_dat_rI(s, COND_AL, ARITH_SUB, TCG_REG_CALL_STACK, 3405 TCG_REG_CALL_STACK, STACK_ADDEND, 1); 3406 tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE, 3407 CPU_TEMP_BUF_NLONGS * sizeof(long)); 3408 3409 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); 3410 3411 if (!tcg_use_softmmu && guest_base) { 3412 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base); 3413 tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE); 3414 } 3415 3416 tcg_out_b_reg(s, COND_AL, tcg_target_call_iarg_regs[1]); 3417 3418 /* 3419 * Return path for goto_ptr. Set return value to 0, a-la exit_tb, 3420 * and fall through to the rest of the epilogue. 3421 */ 3422 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); 3423 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, 0); 3424 tcg_out_epilogue(s); 3425} 3426 3427static void tcg_out_epilogue(TCGContext *s) 3428{ 3429 /* Release local stack frame. */ 3430 tcg_out_dat_rI(s, COND_AL, ARITH_ADD, TCG_REG_CALL_STACK, 3431 TCG_REG_CALL_STACK, STACK_ADDEND, 1); 3432 3433 /* ldmia sp!, { r4 - r11, pc } */ 3434 tcg_out_ldstm(s, COND_AL, INSN_LDMIA, TCG_REG_CALL_STACK, 3435 (1 << TCG_REG_R4) | (1 << TCG_REG_R5) | (1 << TCG_REG_R6) | 3436 (1 << TCG_REG_R7) | (1 << TCG_REG_R8) | (1 << TCG_REG_R9) | 3437 (1 << TCG_REG_R10) | (1 << TCG_REG_R11) | (1 << TCG_REG_PC)); 3438} 3439 3440static void tcg_out_tb_start(TCGContext *s) 3441{ 3442 /* nothing to do */ 3443} 3444 3445typedef struct { 3446 DebugFrameHeader h; 3447 uint8_t fde_def_cfa[4]; 3448 uint8_t fde_reg_ofs[18]; 3449} DebugFrame; 3450 3451#define ELF_HOST_MACHINE EM_ARM 3452 3453/* We're expecting a 2 byte uleb128 encoded value. */ 3454QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14)); 3455 3456static const DebugFrame debug_frame = { 3457 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */ 3458 .h.cie.id = -1, 3459 .h.cie.version = 1, 3460 .h.cie.code_align = 1, 3461 .h.cie.data_align = 0x7c, /* sleb128 -4 */ 3462 .h.cie.return_column = 14, 3463 3464 /* Total FDE size does not include the "len" member. */ 3465 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), 3466 3467 .fde_def_cfa = { 3468 12, 13, /* DW_CFA_def_cfa sp, ... */ 3469 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 3470 (FRAME_SIZE >> 7) 3471 }, 3472 .fde_reg_ofs = { 3473 /* The following must match the stmdb in the prologue. */ 3474 0x8e, 1, /* DW_CFA_offset, lr, -4 */ 3475 0x8b, 2, /* DW_CFA_offset, r11, -8 */ 3476 0x8a, 3, /* DW_CFA_offset, r10, -12 */ 3477 0x89, 4, /* DW_CFA_offset, r9, -16 */ 3478 0x88, 5, /* DW_CFA_offset, r8, -20 */ 3479 0x87, 6, /* DW_CFA_offset, r7, -24 */ 3480 0x86, 7, /* DW_CFA_offset, r6, -28 */ 3481 0x85, 8, /* DW_CFA_offset, r5, -32 */ 3482 0x84, 9, /* DW_CFA_offset, r4, -36 */ 3483 } 3484}; 3485 3486void tcg_register_jit(const void *buf, size_t buf_size) 3487{ 3488 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); 3489} 3490