1/* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25#include "elf.h" 26#include "../tcg-pool.c.inc" 27 28/* 29 * Standardize on the _CALL_FOO symbols used by GCC: 30 * Apple XCode does not define _CALL_DARWIN. 31 * Clang defines _CALL_ELF (64-bit) but not _CALL_SYSV (32-bit). 32 */ 33#if !defined(_CALL_SYSV) && \ 34 !defined(_CALL_DARWIN) && \ 35 !defined(_CALL_AIX) && \ 36 !defined(_CALL_ELF) 37# if defined(__APPLE__) 38# define _CALL_DARWIN 39# elif defined(__ELF__) && TCG_TARGET_REG_BITS == 32 40# define _CALL_SYSV 41# else 42# error "Unknown ABI" 43# endif 44#endif 45 46#ifdef _CALL_SYSV 47# define TCG_TARGET_CALL_ALIGN_ARGS 1 48#endif 49 50/* For some memory operations, we need a scratch that isn't R0. For the AIX 51 calling convention, we can re-use the TOC register since we'll be reloading 52 it at every call. Otherwise R12 will do nicely as neither a call-saved 53 register nor a parameter register. */ 54#ifdef _CALL_AIX 55# define TCG_REG_TMP1 TCG_REG_R2 56#else 57# define TCG_REG_TMP1 TCG_REG_R12 58#endif 59 60#define TCG_VEC_TMP1 TCG_REG_V0 61#define TCG_VEC_TMP2 TCG_REG_V1 62 63#define TCG_REG_TB TCG_REG_R31 64#define USE_REG_TB (TCG_TARGET_REG_BITS == 64) 65 66/* Shorthand for size of a pointer. Avoid promotion to unsigned. */ 67#define SZP ((int)sizeof(void *)) 68 69/* Shorthand for size of a register. */ 70#define SZR (TCG_TARGET_REG_BITS / 8) 71 72#define TCG_CT_CONST_S16 0x100 73#define TCG_CT_CONST_U16 0x200 74#define TCG_CT_CONST_S32 0x400 75#define TCG_CT_CONST_U32 0x800 76#define TCG_CT_CONST_ZERO 0x1000 77#define TCG_CT_CONST_MONE 0x2000 78#define TCG_CT_CONST_WSZ 0x4000 79 80#define ALL_GENERAL_REGS 0xffffffffu 81#define ALL_VECTOR_REGS 0xffffffff00000000ull 82 83#ifdef CONFIG_SOFTMMU 84#define ALL_QLOAD_REGS \ 85 (ALL_GENERAL_REGS & \ 86 ~((1 << TCG_REG_R3) | (1 << TCG_REG_R4) | (1 << TCG_REG_R5))) 87#define ALL_QSTORE_REGS \ 88 (ALL_GENERAL_REGS & ~((1 << TCG_REG_R3) | (1 << TCG_REG_R4) | \ 89 (1 << TCG_REG_R5) | (1 << TCG_REG_R6))) 90#else 91#define ALL_QLOAD_REGS (ALL_GENERAL_REGS & ~(1 << TCG_REG_R3)) 92#define ALL_QSTORE_REGS ALL_QLOAD_REGS 93#endif 94 95TCGPowerISA have_isa; 96static bool have_isel; 97bool have_altivec; 98bool have_vsx; 99 100#ifndef CONFIG_SOFTMMU 101#define TCG_GUEST_BASE_REG 30 102#endif 103 104#ifdef CONFIG_DEBUG_TCG 105static const char tcg_target_reg_names[TCG_TARGET_NB_REGS][4] = { 106 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", 107 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", 108 "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", 109 "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31", 110 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", 111 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", 112 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", 113 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", 114}; 115#endif 116 117static const int tcg_target_reg_alloc_order[] = { 118 TCG_REG_R14, /* call saved registers */ 119 TCG_REG_R15, 120 TCG_REG_R16, 121 TCG_REG_R17, 122 TCG_REG_R18, 123 TCG_REG_R19, 124 TCG_REG_R20, 125 TCG_REG_R21, 126 TCG_REG_R22, 127 TCG_REG_R23, 128 TCG_REG_R24, 129 TCG_REG_R25, 130 TCG_REG_R26, 131 TCG_REG_R27, 132 TCG_REG_R28, 133 TCG_REG_R29, 134 TCG_REG_R30, 135 TCG_REG_R31, 136 TCG_REG_R12, /* call clobbered, non-arguments */ 137 TCG_REG_R11, 138 TCG_REG_R2, 139 TCG_REG_R13, 140 TCG_REG_R10, /* call clobbered, arguments */ 141 TCG_REG_R9, 142 TCG_REG_R8, 143 TCG_REG_R7, 144 TCG_REG_R6, 145 TCG_REG_R5, 146 TCG_REG_R4, 147 TCG_REG_R3, 148 149 /* V0 and V1 reserved as temporaries; V20 - V31 are call-saved */ 150 TCG_REG_V2, /* call clobbered, vectors */ 151 TCG_REG_V3, 152 TCG_REG_V4, 153 TCG_REG_V5, 154 TCG_REG_V6, 155 TCG_REG_V7, 156 TCG_REG_V8, 157 TCG_REG_V9, 158 TCG_REG_V10, 159 TCG_REG_V11, 160 TCG_REG_V12, 161 TCG_REG_V13, 162 TCG_REG_V14, 163 TCG_REG_V15, 164 TCG_REG_V16, 165 TCG_REG_V17, 166 TCG_REG_V18, 167 TCG_REG_V19, 168}; 169 170static const int tcg_target_call_iarg_regs[] = { 171 TCG_REG_R3, 172 TCG_REG_R4, 173 TCG_REG_R5, 174 TCG_REG_R6, 175 TCG_REG_R7, 176 TCG_REG_R8, 177 TCG_REG_R9, 178 TCG_REG_R10 179}; 180 181static const int tcg_target_call_oarg_regs[] = { 182 TCG_REG_R3, 183 TCG_REG_R4 184}; 185 186static const int tcg_target_callee_save_regs[] = { 187#ifdef _CALL_DARWIN 188 TCG_REG_R11, 189#endif 190 TCG_REG_R14, 191 TCG_REG_R15, 192 TCG_REG_R16, 193 TCG_REG_R17, 194 TCG_REG_R18, 195 TCG_REG_R19, 196 TCG_REG_R20, 197 TCG_REG_R21, 198 TCG_REG_R22, 199 TCG_REG_R23, 200 TCG_REG_R24, 201 TCG_REG_R25, 202 TCG_REG_R26, 203 TCG_REG_R27, /* currently used for the global env */ 204 TCG_REG_R28, 205 TCG_REG_R29, 206 TCG_REG_R30, 207 TCG_REG_R31 208}; 209 210static inline bool in_range_b(tcg_target_long target) 211{ 212 return target == sextract64(target, 0, 26); 213} 214 215static uint32_t reloc_pc24_val(const tcg_insn_unit *pc, 216 const tcg_insn_unit *target) 217{ 218 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); 219 tcg_debug_assert(in_range_b(disp)); 220 return disp & 0x3fffffc; 221} 222 223static bool reloc_pc24(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 224{ 225 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 226 ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx); 227 228 if (in_range_b(disp)) { 229 *src_rw = (*src_rw & ~0x3fffffc) | (disp & 0x3fffffc); 230 return true; 231 } 232 return false; 233} 234 235static uint16_t reloc_pc14_val(const tcg_insn_unit *pc, 236 const tcg_insn_unit *target) 237{ 238 ptrdiff_t disp = tcg_ptr_byte_diff(target, pc); 239 tcg_debug_assert(disp == (int16_t) disp); 240 return disp & 0xfffc; 241} 242 243static bool reloc_pc14(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 244{ 245 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 246 ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx); 247 248 if (disp == (int16_t) disp) { 249 *src_rw = (*src_rw & ~0xfffc) | (disp & 0xfffc); 250 return true; 251 } 252 return false; 253} 254 255/* test if a constant matches the constraint */ 256static bool tcg_target_const_match(int64_t val, TCGType type, int ct) 257{ 258 if (ct & TCG_CT_CONST) { 259 return 1; 260 } 261 262 /* The only 32-bit constraint we use aside from 263 TCG_CT_CONST is TCG_CT_CONST_S16. */ 264 if (type == TCG_TYPE_I32) { 265 val = (int32_t)val; 266 } 267 268 if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) { 269 return 1; 270 } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) { 271 return 1; 272 } else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { 273 return 1; 274 } else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { 275 return 1; 276 } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) { 277 return 1; 278 } else if ((ct & TCG_CT_CONST_MONE) && val == -1) { 279 return 1; 280 } else if ((ct & TCG_CT_CONST_WSZ) 281 && val == (type == TCG_TYPE_I32 ? 32 : 64)) { 282 return 1; 283 } 284 return 0; 285} 286 287#define OPCD(opc) ((opc)<<26) 288#define XO19(opc) (OPCD(19)|((opc)<<1)) 289#define MD30(opc) (OPCD(30)|((opc)<<2)) 290#define MDS30(opc) (OPCD(30)|((opc)<<1)) 291#define XO31(opc) (OPCD(31)|((opc)<<1)) 292#define XO58(opc) (OPCD(58)|(opc)) 293#define XO62(opc) (OPCD(62)|(opc)) 294#define VX4(opc) (OPCD(4)|(opc)) 295 296#define B OPCD( 18) 297#define BC OPCD( 16) 298#define LBZ OPCD( 34) 299#define LHZ OPCD( 40) 300#define LHA OPCD( 42) 301#define LWZ OPCD( 32) 302#define LWZUX XO31( 55) 303#define STB OPCD( 38) 304#define STH OPCD( 44) 305#define STW OPCD( 36) 306 307#define STD XO62( 0) 308#define STDU XO62( 1) 309#define STDX XO31(149) 310 311#define LD XO58( 0) 312#define LDX XO31( 21) 313#define LDU XO58( 1) 314#define LDUX XO31( 53) 315#define LWA XO58( 2) 316#define LWAX XO31(341) 317 318#define ADDIC OPCD( 12) 319#define ADDI OPCD( 14) 320#define ADDIS OPCD( 15) 321#define ORI OPCD( 24) 322#define ORIS OPCD( 25) 323#define XORI OPCD( 26) 324#define XORIS OPCD( 27) 325#define ANDI OPCD( 28) 326#define ANDIS OPCD( 29) 327#define MULLI OPCD( 7) 328#define CMPLI OPCD( 10) 329#define CMPI OPCD( 11) 330#define SUBFIC OPCD( 8) 331 332#define LWZU OPCD( 33) 333#define STWU OPCD( 37) 334 335#define RLWIMI OPCD( 20) 336#define RLWINM OPCD( 21) 337#define RLWNM OPCD( 23) 338 339#define RLDICL MD30( 0) 340#define RLDICR MD30( 1) 341#define RLDIMI MD30( 3) 342#define RLDCL MDS30( 8) 343 344#define BCLR XO19( 16) 345#define BCCTR XO19(528) 346#define CRAND XO19(257) 347#define CRANDC XO19(129) 348#define CRNAND XO19(225) 349#define CROR XO19(449) 350#define CRNOR XO19( 33) 351 352#define EXTSB XO31(954) 353#define EXTSH XO31(922) 354#define EXTSW XO31(986) 355#define ADD XO31(266) 356#define ADDE XO31(138) 357#define ADDME XO31(234) 358#define ADDZE XO31(202) 359#define ADDC XO31( 10) 360#define AND XO31( 28) 361#define SUBF XO31( 40) 362#define SUBFC XO31( 8) 363#define SUBFE XO31(136) 364#define SUBFME XO31(232) 365#define SUBFZE XO31(200) 366#define OR XO31(444) 367#define XOR XO31(316) 368#define MULLW XO31(235) 369#define MULHW XO31( 75) 370#define MULHWU XO31( 11) 371#define DIVW XO31(491) 372#define DIVWU XO31(459) 373#define CMP XO31( 0) 374#define CMPL XO31( 32) 375#define LHBRX XO31(790) 376#define LWBRX XO31(534) 377#define LDBRX XO31(532) 378#define STHBRX XO31(918) 379#define STWBRX XO31(662) 380#define STDBRX XO31(660) 381#define MFSPR XO31(339) 382#define MTSPR XO31(467) 383#define SRAWI XO31(824) 384#define NEG XO31(104) 385#define MFCR XO31( 19) 386#define MFOCRF (MFCR | (1u << 20)) 387#define NOR XO31(124) 388#define CNTLZW XO31( 26) 389#define CNTLZD XO31( 58) 390#define CNTTZW XO31(538) 391#define CNTTZD XO31(570) 392#define CNTPOPW XO31(378) 393#define CNTPOPD XO31(506) 394#define ANDC XO31( 60) 395#define ORC XO31(412) 396#define EQV XO31(284) 397#define NAND XO31(476) 398#define ISEL XO31( 15) 399 400#define MULLD XO31(233) 401#define MULHD XO31( 73) 402#define MULHDU XO31( 9) 403#define DIVD XO31(489) 404#define DIVDU XO31(457) 405 406#define LBZX XO31( 87) 407#define LHZX XO31(279) 408#define LHAX XO31(343) 409#define LWZX XO31( 23) 410#define STBX XO31(215) 411#define STHX XO31(407) 412#define STWX XO31(151) 413 414#define EIEIO XO31(854) 415#define HWSYNC XO31(598) 416#define LWSYNC (HWSYNC | (1u << 21)) 417 418#define SPR(a, b) ((((a)<<5)|(b))<<11) 419#define LR SPR(8, 0) 420#define CTR SPR(9, 0) 421 422#define SLW XO31( 24) 423#define SRW XO31(536) 424#define SRAW XO31(792) 425 426#define SLD XO31( 27) 427#define SRD XO31(539) 428#define SRAD XO31(794) 429#define SRADI XO31(413<<1) 430 431#define BRH XO31(219) 432#define BRW XO31(155) 433#define BRD XO31(187) 434 435#define TW XO31( 4) 436#define TRAP (TW | TO(31)) 437 438#define NOP ORI /* ori 0,0,0 */ 439 440#define LVX XO31(103) 441#define LVEBX XO31(7) 442#define LVEHX XO31(39) 443#define LVEWX XO31(71) 444#define LXSDX (XO31(588) | 1) /* v2.06, force tx=1 */ 445#define LXVDSX (XO31(332) | 1) /* v2.06, force tx=1 */ 446#define LXSIWZX (XO31(12) | 1) /* v2.07, force tx=1 */ 447#define LXV (OPCD(61) | 8 | 1) /* v3.00, force tx=1 */ 448#define LXSD (OPCD(57) | 2) /* v3.00 */ 449#define LXVWSX (XO31(364) | 1) /* v3.00, force tx=1 */ 450 451#define STVX XO31(231) 452#define STVEWX XO31(199) 453#define STXSDX (XO31(716) | 1) /* v2.06, force sx=1 */ 454#define STXSIWX (XO31(140) | 1) /* v2.07, force sx=1 */ 455#define STXV (OPCD(61) | 8 | 5) /* v3.00, force sx=1 */ 456#define STXSD (OPCD(61) | 2) /* v3.00 */ 457 458#define VADDSBS VX4(768) 459#define VADDUBS VX4(512) 460#define VADDUBM VX4(0) 461#define VADDSHS VX4(832) 462#define VADDUHS VX4(576) 463#define VADDUHM VX4(64) 464#define VADDSWS VX4(896) 465#define VADDUWS VX4(640) 466#define VADDUWM VX4(128) 467#define VADDUDM VX4(192) /* v2.07 */ 468 469#define VSUBSBS VX4(1792) 470#define VSUBUBS VX4(1536) 471#define VSUBUBM VX4(1024) 472#define VSUBSHS VX4(1856) 473#define VSUBUHS VX4(1600) 474#define VSUBUHM VX4(1088) 475#define VSUBSWS VX4(1920) 476#define VSUBUWS VX4(1664) 477#define VSUBUWM VX4(1152) 478#define VSUBUDM VX4(1216) /* v2.07 */ 479 480#define VNEGW (VX4(1538) | (6 << 16)) /* v3.00 */ 481#define VNEGD (VX4(1538) | (7 << 16)) /* v3.00 */ 482 483#define VMAXSB VX4(258) 484#define VMAXSH VX4(322) 485#define VMAXSW VX4(386) 486#define VMAXSD VX4(450) /* v2.07 */ 487#define VMAXUB VX4(2) 488#define VMAXUH VX4(66) 489#define VMAXUW VX4(130) 490#define VMAXUD VX4(194) /* v2.07 */ 491#define VMINSB VX4(770) 492#define VMINSH VX4(834) 493#define VMINSW VX4(898) 494#define VMINSD VX4(962) /* v2.07 */ 495#define VMINUB VX4(514) 496#define VMINUH VX4(578) 497#define VMINUW VX4(642) 498#define VMINUD VX4(706) /* v2.07 */ 499 500#define VCMPEQUB VX4(6) 501#define VCMPEQUH VX4(70) 502#define VCMPEQUW VX4(134) 503#define VCMPEQUD VX4(199) /* v2.07 */ 504#define VCMPGTSB VX4(774) 505#define VCMPGTSH VX4(838) 506#define VCMPGTSW VX4(902) 507#define VCMPGTSD VX4(967) /* v2.07 */ 508#define VCMPGTUB VX4(518) 509#define VCMPGTUH VX4(582) 510#define VCMPGTUW VX4(646) 511#define VCMPGTUD VX4(711) /* v2.07 */ 512#define VCMPNEB VX4(7) /* v3.00 */ 513#define VCMPNEH VX4(71) /* v3.00 */ 514#define VCMPNEW VX4(135) /* v3.00 */ 515 516#define VSLB VX4(260) 517#define VSLH VX4(324) 518#define VSLW VX4(388) 519#define VSLD VX4(1476) /* v2.07 */ 520#define VSRB VX4(516) 521#define VSRH VX4(580) 522#define VSRW VX4(644) 523#define VSRD VX4(1732) /* v2.07 */ 524#define VSRAB VX4(772) 525#define VSRAH VX4(836) 526#define VSRAW VX4(900) 527#define VSRAD VX4(964) /* v2.07 */ 528#define VRLB VX4(4) 529#define VRLH VX4(68) 530#define VRLW VX4(132) 531#define VRLD VX4(196) /* v2.07 */ 532 533#define VMULEUB VX4(520) 534#define VMULEUH VX4(584) 535#define VMULEUW VX4(648) /* v2.07 */ 536#define VMULOUB VX4(8) 537#define VMULOUH VX4(72) 538#define VMULOUW VX4(136) /* v2.07 */ 539#define VMULUWM VX4(137) /* v2.07 */ 540#define VMULLD VX4(457) /* v3.10 */ 541#define VMSUMUHM VX4(38) 542 543#define VMRGHB VX4(12) 544#define VMRGHH VX4(76) 545#define VMRGHW VX4(140) 546#define VMRGLB VX4(268) 547#define VMRGLH VX4(332) 548#define VMRGLW VX4(396) 549 550#define VPKUHUM VX4(14) 551#define VPKUWUM VX4(78) 552 553#define VAND VX4(1028) 554#define VANDC VX4(1092) 555#define VNOR VX4(1284) 556#define VOR VX4(1156) 557#define VXOR VX4(1220) 558#define VEQV VX4(1668) /* v2.07 */ 559#define VNAND VX4(1412) /* v2.07 */ 560#define VORC VX4(1348) /* v2.07 */ 561 562#define VSPLTB VX4(524) 563#define VSPLTH VX4(588) 564#define VSPLTW VX4(652) 565#define VSPLTISB VX4(780) 566#define VSPLTISH VX4(844) 567#define VSPLTISW VX4(908) 568 569#define VSLDOI VX4(44) 570 571#define XXPERMDI (OPCD(60) | (10 << 3) | 7) /* v2.06, force ax=bx=tx=1 */ 572#define XXSEL (OPCD(60) | (3 << 4) | 0xf) /* v2.06, force ax=bx=cx=tx=1 */ 573#define XXSPLTIB (OPCD(60) | (360 << 1) | 1) /* v3.00, force tx=1 */ 574 575#define MFVSRD (XO31(51) | 1) /* v2.07, force sx=1 */ 576#define MFVSRWZ (XO31(115) | 1) /* v2.07, force sx=1 */ 577#define MTVSRD (XO31(179) | 1) /* v2.07, force tx=1 */ 578#define MTVSRWZ (XO31(243) | 1) /* v2.07, force tx=1 */ 579#define MTVSRDD (XO31(435) | 1) /* v3.00, force tx=1 */ 580#define MTVSRWS (XO31(403) | 1) /* v3.00, force tx=1 */ 581 582#define RT(r) ((r)<<21) 583#define RS(r) ((r)<<21) 584#define RA(r) ((r)<<16) 585#define RB(r) ((r)<<11) 586#define TO(t) ((t)<<21) 587#define SH(s) ((s)<<11) 588#define MB(b) ((b)<<6) 589#define ME(e) ((e)<<1) 590#define BO(o) ((o)<<21) 591#define MB64(b) ((b)<<5) 592#define FXM(b) (1 << (19 - (b))) 593 594#define VRT(r) (((r) & 31) << 21) 595#define VRA(r) (((r) & 31) << 16) 596#define VRB(r) (((r) & 31) << 11) 597#define VRC(r) (((r) & 31) << 6) 598 599#define LK 1 600 601#define TAB(t, a, b) (RT(t) | RA(a) | RB(b)) 602#define SAB(s, a, b) (RS(s) | RA(a) | RB(b)) 603#define TAI(s, a, i) (RT(s) | RA(a) | ((i) & 0xffff)) 604#define SAI(s, a, i) (RS(s) | RA(a) | ((i) & 0xffff)) 605 606#define BF(n) ((n)<<23) 607#define BI(n, c) (((c)+((n)*4))<<16) 608#define BT(n, c) (((c)+((n)*4))<<21) 609#define BA(n, c) (((c)+((n)*4))<<16) 610#define BB(n, c) (((c)+((n)*4))<<11) 611#define BC_(n, c) (((c)+((n)*4))<<6) 612 613#define BO_COND_TRUE BO(12) 614#define BO_COND_FALSE BO( 4) 615#define BO_ALWAYS BO(20) 616 617enum { 618 CR_LT, 619 CR_GT, 620 CR_EQ, 621 CR_SO 622}; 623 624static const uint32_t tcg_to_bc[] = { 625 [TCG_COND_EQ] = BC | BI(7, CR_EQ) | BO_COND_TRUE, 626 [TCG_COND_NE] = BC | BI(7, CR_EQ) | BO_COND_FALSE, 627 [TCG_COND_LT] = BC | BI(7, CR_LT) | BO_COND_TRUE, 628 [TCG_COND_GE] = BC | BI(7, CR_LT) | BO_COND_FALSE, 629 [TCG_COND_LE] = BC | BI(7, CR_GT) | BO_COND_FALSE, 630 [TCG_COND_GT] = BC | BI(7, CR_GT) | BO_COND_TRUE, 631 [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE, 632 [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE, 633 [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE, 634 [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE, 635}; 636 637/* The low bit here is set if the RA and RB fields must be inverted. */ 638static const uint32_t tcg_to_isel[] = { 639 [TCG_COND_EQ] = ISEL | BC_(7, CR_EQ), 640 [TCG_COND_NE] = ISEL | BC_(7, CR_EQ) | 1, 641 [TCG_COND_LT] = ISEL | BC_(7, CR_LT), 642 [TCG_COND_GE] = ISEL | BC_(7, CR_LT) | 1, 643 [TCG_COND_LE] = ISEL | BC_(7, CR_GT) | 1, 644 [TCG_COND_GT] = ISEL | BC_(7, CR_GT), 645 [TCG_COND_LTU] = ISEL | BC_(7, CR_LT), 646 [TCG_COND_GEU] = ISEL | BC_(7, CR_LT) | 1, 647 [TCG_COND_LEU] = ISEL | BC_(7, CR_GT) | 1, 648 [TCG_COND_GTU] = ISEL | BC_(7, CR_GT), 649}; 650 651static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 652 intptr_t value, intptr_t addend) 653{ 654 const tcg_insn_unit *target; 655 int16_t lo; 656 int32_t hi; 657 658 value += addend; 659 target = (const tcg_insn_unit *)value; 660 661 switch (type) { 662 case R_PPC_REL14: 663 return reloc_pc14(code_ptr, target); 664 case R_PPC_REL24: 665 return reloc_pc24(code_ptr, target); 666 case R_PPC_ADDR16: 667 /* 668 * We are (slightly) abusing this relocation type. In particular, 669 * assert that the low 2 bits are zero, and do not modify them. 670 * That way we can use this with LD et al that have opcode bits 671 * in the low 2 bits of the insn. 672 */ 673 if ((value & 3) || value != (int16_t)value) { 674 return false; 675 } 676 *code_ptr = (*code_ptr & ~0xfffc) | (value & 0xfffc); 677 break; 678 case R_PPC_ADDR32: 679 /* 680 * We are abusing this relocation type. Again, this points to 681 * a pair of insns, lis + load. This is an absolute address 682 * relocation for PPC32 so the lis cannot be removed. 683 */ 684 lo = value; 685 hi = value - lo; 686 if (hi + lo != value) { 687 return false; 688 } 689 code_ptr[0] = deposit32(code_ptr[0], 0, 16, hi >> 16); 690 code_ptr[1] = deposit32(code_ptr[1], 0, 16, lo); 691 break; 692 default: 693 g_assert_not_reached(); 694 } 695 return true; 696} 697 698static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt, 699 TCGReg base, tcg_target_long offset); 700 701static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 702{ 703 if (ret == arg) { 704 return true; 705 } 706 switch (type) { 707 case TCG_TYPE_I64: 708 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 709 /* fallthru */ 710 case TCG_TYPE_I32: 711 if (ret < TCG_REG_V0) { 712 if (arg < TCG_REG_V0) { 713 tcg_out32(s, OR | SAB(arg, ret, arg)); 714 break; 715 } else if (have_isa_2_07) { 716 tcg_out32(s, (type == TCG_TYPE_I32 ? MFVSRWZ : MFVSRD) 717 | VRT(arg) | RA(ret)); 718 break; 719 } else { 720 /* Altivec does not support vector->integer moves. */ 721 return false; 722 } 723 } else if (arg < TCG_REG_V0) { 724 if (have_isa_2_07) { 725 tcg_out32(s, (type == TCG_TYPE_I32 ? MTVSRWZ : MTVSRD) 726 | VRT(ret) | RA(arg)); 727 break; 728 } else { 729 /* Altivec does not support integer->vector moves. */ 730 return false; 731 } 732 } 733 /* fallthru */ 734 case TCG_TYPE_V64: 735 case TCG_TYPE_V128: 736 tcg_debug_assert(ret >= TCG_REG_V0 && arg >= TCG_REG_V0); 737 tcg_out32(s, VOR | VRT(ret) | VRA(arg) | VRB(arg)); 738 break; 739 default: 740 g_assert_not_reached(); 741 } 742 return true; 743} 744 745static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs, 746 int sh, int mb) 747{ 748 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 749 sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1); 750 mb = MB64((mb >> 5) | ((mb << 1) & 0x3f)); 751 tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb); 752} 753 754static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs, 755 int sh, int mb, int me) 756{ 757 tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me)); 758} 759 760static inline void tcg_out_ext8s(TCGContext *s, TCGReg dst, TCGReg src) 761{ 762 tcg_out32(s, EXTSB | RA(dst) | RS(src)); 763} 764 765static inline void tcg_out_ext16s(TCGContext *s, TCGReg dst, TCGReg src) 766{ 767 tcg_out32(s, EXTSH | RA(dst) | RS(src)); 768} 769 770static inline void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src) 771{ 772 tcg_out32(s, ANDI | SAI(src, dst, 0xffff)); 773} 774 775static inline void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src) 776{ 777 tcg_out32(s, EXTSW | RA(dst) | RS(src)); 778} 779 780static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src) 781{ 782 tcg_out_rld(s, RLDICL, dst, src, 0, 32); 783} 784 785static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c) 786{ 787 tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c); 788} 789 790static inline void tcg_out_shli64(TCGContext *s, TCGReg dst, TCGReg src, int c) 791{ 792 tcg_out_rld(s, RLDICR, dst, src, c, 63 - c); 793} 794 795static inline void tcg_out_sari32(TCGContext *s, TCGReg dst, TCGReg src, int c) 796{ 797 /* Limit immediate shift count lest we create an illegal insn. */ 798 tcg_out32(s, SRAWI | RA(dst) | RS(src) | SH(c & 31)); 799} 800 801static inline void tcg_out_shri32(TCGContext *s, TCGReg dst, TCGReg src, int c) 802{ 803 tcg_out_rlw(s, RLWINM, dst, src, 32 - c, c, 31); 804} 805 806static inline void tcg_out_shri64(TCGContext *s, TCGReg dst, TCGReg src, int c) 807{ 808 tcg_out_rld(s, RLDICL, dst, src, 64 - c, c); 809} 810 811static inline void tcg_out_sari64(TCGContext *s, TCGReg dst, TCGReg src, int c) 812{ 813 tcg_out32(s, SRADI | RA(dst) | RS(src) | SH(c & 0x1f) | ((c >> 4) & 2)); 814} 815 816static void tcg_out_bswap16(TCGContext *s, TCGReg dst, TCGReg src, int flags) 817{ 818 TCGReg tmp = dst == src ? TCG_REG_R0 : dst; 819 820 if (have_isa_3_10) { 821 tcg_out32(s, BRH | RA(dst) | RS(src)); 822 if (flags & TCG_BSWAP_OS) { 823 tcg_out_ext16s(s, dst, dst); 824 } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { 825 tcg_out_ext16u(s, dst, dst); 826 } 827 return; 828 } 829 830 /* 831 * In the following, 832 * dep(a, b, m) -> (a & ~m) | (b & m) 833 * 834 * Begin with: src = xxxxabcd 835 */ 836 /* tmp = rol32(src, 24) & 0x000000ff = 0000000c */ 837 tcg_out_rlw(s, RLWINM, tmp, src, 24, 24, 31); 838 /* tmp = dep(tmp, rol32(src, 8), 0x0000ff00) = 000000dc */ 839 tcg_out_rlw(s, RLWIMI, tmp, src, 8, 16, 23); 840 841 if (flags & TCG_BSWAP_OS) { 842 tcg_out_ext16s(s, dst, tmp); 843 } else { 844 tcg_out_mov(s, TCG_TYPE_REG, dst, tmp); 845 } 846} 847 848static void tcg_out_bswap32(TCGContext *s, TCGReg dst, TCGReg src, int flags) 849{ 850 TCGReg tmp = dst == src ? TCG_REG_R0 : dst; 851 852 if (have_isa_3_10) { 853 tcg_out32(s, BRW | RA(dst) | RS(src)); 854 if (flags & TCG_BSWAP_OS) { 855 tcg_out_ext32s(s, dst, dst); 856 } else if ((flags & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { 857 tcg_out_ext32u(s, dst, dst); 858 } 859 return; 860 } 861 862 /* 863 * Stolen from gcc's builtin_bswap32. 864 * In the following, 865 * dep(a, b, m) -> (a & ~m) | (b & m) 866 * 867 * Begin with: src = xxxxabcd 868 */ 869 /* tmp = rol32(src, 8) & 0xffffffff = 0000bcda */ 870 tcg_out_rlw(s, RLWINM, tmp, src, 8, 0, 31); 871 /* tmp = dep(tmp, rol32(src, 24), 0xff000000) = 0000dcda */ 872 tcg_out_rlw(s, RLWIMI, tmp, src, 24, 0, 7); 873 /* tmp = dep(tmp, rol32(src, 24), 0x0000ff00) = 0000dcba */ 874 tcg_out_rlw(s, RLWIMI, tmp, src, 24, 16, 23); 875 876 if (flags & TCG_BSWAP_OS) { 877 tcg_out_ext32s(s, dst, tmp); 878 } else { 879 tcg_out_mov(s, TCG_TYPE_REG, dst, tmp); 880 } 881} 882 883static void tcg_out_bswap64(TCGContext *s, TCGReg dst, TCGReg src) 884{ 885 TCGReg t0 = dst == src ? TCG_REG_R0 : dst; 886 TCGReg t1 = dst == src ? dst : TCG_REG_R0; 887 888 if (have_isa_3_10) { 889 tcg_out32(s, BRD | RA(dst) | RS(src)); 890 return; 891 } 892 893 /* 894 * In the following, 895 * dep(a, b, m) -> (a & ~m) | (b & m) 896 * 897 * Begin with: src = abcdefgh 898 */ 899 /* t0 = rol32(src, 8) & 0xffffffff = 0000fghe */ 900 tcg_out_rlw(s, RLWINM, t0, src, 8, 0, 31); 901 /* t0 = dep(t0, rol32(src, 24), 0xff000000) = 0000hghe */ 902 tcg_out_rlw(s, RLWIMI, t0, src, 24, 0, 7); 903 /* t0 = dep(t0, rol32(src, 24), 0x0000ff00) = 0000hgfe */ 904 tcg_out_rlw(s, RLWIMI, t0, src, 24, 16, 23); 905 906 /* t0 = rol64(t0, 32) = hgfe0000 */ 907 tcg_out_rld(s, RLDICL, t0, t0, 32, 0); 908 /* t1 = rol64(src, 32) = efghabcd */ 909 tcg_out_rld(s, RLDICL, t1, src, 32, 0); 910 911 /* t0 = dep(t0, rol32(t1, 24), 0xffffffff) = hgfebcda */ 912 tcg_out_rlw(s, RLWIMI, t0, t1, 8, 0, 31); 913 /* t0 = dep(t0, rol32(t1, 24), 0xff000000) = hgfedcda */ 914 tcg_out_rlw(s, RLWIMI, t0, t1, 24, 0, 7); 915 /* t0 = dep(t0, rol32(t1, 24), 0x0000ff00) = hgfedcba */ 916 tcg_out_rlw(s, RLWIMI, t0, t1, 24, 16, 23); 917 918 tcg_out_mov(s, TCG_TYPE_REG, dst, t0); 919} 920 921/* Emit a move into ret of arg, if it can be done in one insn. */ 922static bool tcg_out_movi_one(TCGContext *s, TCGReg ret, tcg_target_long arg) 923{ 924 if (arg == (int16_t)arg) { 925 tcg_out32(s, ADDI | TAI(ret, 0, arg)); 926 return true; 927 } 928 if (arg == (int32_t)arg && (arg & 0xffff) == 0) { 929 tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16)); 930 return true; 931 } 932 return false; 933} 934 935static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, 936 tcg_target_long arg, bool in_prologue) 937{ 938 intptr_t tb_diff; 939 tcg_target_long tmp; 940 int shift; 941 942 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); 943 944 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { 945 arg = (int32_t)arg; 946 } 947 948 /* Load 16-bit immediates with one insn. */ 949 if (tcg_out_movi_one(s, ret, arg)) { 950 return; 951 } 952 953 /* Load addresses within the TB with one insn. */ 954 tb_diff = tcg_tbrel_diff(s, (void *)arg); 955 if (!in_prologue && USE_REG_TB && tb_diff == (int16_t)tb_diff) { 956 tcg_out32(s, ADDI | TAI(ret, TCG_REG_TB, tb_diff)); 957 return; 958 } 959 960 /* Load 32-bit immediates with two insns. Note that we've already 961 eliminated bare ADDIS, so we know both insns are required. */ 962 if (TCG_TARGET_REG_BITS == 32 || arg == (int32_t)arg) { 963 tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16)); 964 tcg_out32(s, ORI | SAI(ret, ret, arg)); 965 return; 966 } 967 if (arg == (uint32_t)arg && !(arg & 0x8000)) { 968 tcg_out32(s, ADDI | TAI(ret, 0, arg)); 969 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16)); 970 return; 971 } 972 973 /* Load masked 16-bit value. */ 974 if (arg > 0 && (arg & 0x8000)) { 975 tmp = arg | 0x7fff; 976 if ((tmp & (tmp + 1)) == 0) { 977 int mb = clz64(tmp + 1) + 1; 978 tcg_out32(s, ADDI | TAI(ret, 0, arg)); 979 tcg_out_rld(s, RLDICL, ret, ret, 0, mb); 980 return; 981 } 982 } 983 984 /* Load common masks with 2 insns. */ 985 shift = ctz64(arg); 986 tmp = arg >> shift; 987 if (tmp == (int16_t)tmp) { 988 tcg_out32(s, ADDI | TAI(ret, 0, tmp)); 989 tcg_out_shli64(s, ret, ret, shift); 990 return; 991 } 992 shift = clz64(arg); 993 if (tcg_out_movi_one(s, ret, arg << shift)) { 994 tcg_out_shri64(s, ret, ret, shift); 995 return; 996 } 997 998 /* Load addresses within 2GB of TB with 2 (or rarely 3) insns. */ 999 if (!in_prologue && USE_REG_TB && tb_diff == (int32_t)tb_diff) { 1000 tcg_out_mem_long(s, ADDI, ADD, ret, TCG_REG_TB, tb_diff); 1001 return; 1002 } 1003 1004 /* Use the constant pool, if possible. */ 1005 if (!in_prologue && USE_REG_TB) { 1006 new_pool_label(s, arg, R_PPC_ADDR16, s->code_ptr, 1007 tcg_tbrel_diff(s, NULL)); 1008 tcg_out32(s, LD | TAI(ret, TCG_REG_TB, 0)); 1009 return; 1010 } 1011 1012 tmp = arg >> 31 >> 1; 1013 tcg_out_movi(s, TCG_TYPE_I32, ret, tmp); 1014 if (tmp) { 1015 tcg_out_shli64(s, ret, ret, 32); 1016 } 1017 if (arg & 0xffff0000) { 1018 tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16)); 1019 } 1020 if (arg & 0xffff) { 1021 tcg_out32(s, ORI | SAI(ret, ret, arg)); 1022 } 1023} 1024 1025static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 1026 TCGReg ret, int64_t val) 1027{ 1028 uint32_t load_insn; 1029 int rel, low; 1030 intptr_t add; 1031 1032 switch (vece) { 1033 case MO_8: 1034 low = (int8_t)val; 1035 if (low >= -16 && low < 16) { 1036 tcg_out32(s, VSPLTISB | VRT(ret) | ((val & 31) << 16)); 1037 return; 1038 } 1039 if (have_isa_3_00) { 1040 tcg_out32(s, XXSPLTIB | VRT(ret) | ((val & 0xff) << 11)); 1041 return; 1042 } 1043 break; 1044 1045 case MO_16: 1046 low = (int16_t)val; 1047 if (low >= -16 && low < 16) { 1048 tcg_out32(s, VSPLTISH | VRT(ret) | ((val & 31) << 16)); 1049 return; 1050 } 1051 break; 1052 1053 case MO_32: 1054 low = (int32_t)val; 1055 if (low >= -16 && low < 16) { 1056 tcg_out32(s, VSPLTISW | VRT(ret) | ((val & 31) << 16)); 1057 return; 1058 } 1059 break; 1060 } 1061 1062 /* 1063 * Otherwise we must load the value from the constant pool. 1064 */ 1065 if (USE_REG_TB) { 1066 rel = R_PPC_ADDR16; 1067 add = tcg_tbrel_diff(s, NULL); 1068 } else { 1069 rel = R_PPC_ADDR32; 1070 add = 0; 1071 } 1072 1073 if (have_vsx) { 1074 load_insn = type == TCG_TYPE_V64 ? LXSDX : LXVDSX; 1075 load_insn |= VRT(ret) | RB(TCG_REG_TMP1); 1076 if (TCG_TARGET_REG_BITS == 64) { 1077 new_pool_label(s, val, rel, s->code_ptr, add); 1078 } else { 1079 new_pool_l2(s, rel, s->code_ptr, add, val >> 32, val); 1080 } 1081 } else { 1082 load_insn = LVX | VRT(ret) | RB(TCG_REG_TMP1); 1083 if (TCG_TARGET_REG_BITS == 64) { 1084 new_pool_l2(s, rel, s->code_ptr, add, val, val); 1085 } else { 1086 new_pool_l4(s, rel, s->code_ptr, add, 1087 val >> 32, val, val >> 32, val); 1088 } 1089 } 1090 1091 if (USE_REG_TB) { 1092 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, 0, 0)); 1093 load_insn |= RA(TCG_REG_TB); 1094 } else { 1095 tcg_out32(s, ADDIS | TAI(TCG_REG_TMP1, 0, 0)); 1096 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, TCG_REG_TMP1, 0)); 1097 } 1098 tcg_out32(s, load_insn); 1099} 1100 1101static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret, 1102 tcg_target_long arg) 1103{ 1104 switch (type) { 1105 case TCG_TYPE_I32: 1106 case TCG_TYPE_I64: 1107 tcg_debug_assert(ret < TCG_REG_V0); 1108 tcg_out_movi_int(s, type, ret, arg, false); 1109 break; 1110 1111 default: 1112 g_assert_not_reached(); 1113 } 1114} 1115 1116static bool mask_operand(uint32_t c, int *mb, int *me) 1117{ 1118 uint32_t lsb, test; 1119 1120 /* Accept a bit pattern like: 1121 0....01....1 1122 1....10....0 1123 0..01..10..0 1124 Keep track of the transitions. */ 1125 if (c == 0 || c == -1) { 1126 return false; 1127 } 1128 test = c; 1129 lsb = test & -test; 1130 test += lsb; 1131 if (test & (test - 1)) { 1132 return false; 1133 } 1134 1135 *me = clz32(lsb); 1136 *mb = test ? clz32(test & -test) + 1 : 0; 1137 return true; 1138} 1139 1140static bool mask64_operand(uint64_t c, int *mb, int *me) 1141{ 1142 uint64_t lsb; 1143 1144 if (c == 0) { 1145 return false; 1146 } 1147 1148 lsb = c & -c; 1149 /* Accept 1..10..0. */ 1150 if (c == -lsb) { 1151 *mb = 0; 1152 *me = clz64(lsb); 1153 return true; 1154 } 1155 /* Accept 0..01..1. */ 1156 if (lsb == 1 && (c & (c + 1)) == 0) { 1157 *mb = clz64(c + 1) + 1; 1158 *me = 63; 1159 return true; 1160 } 1161 return false; 1162} 1163 1164static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c) 1165{ 1166 int mb, me; 1167 1168 if (mask_operand(c, &mb, &me)) { 1169 tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me); 1170 } else if ((c & 0xffff) == c) { 1171 tcg_out32(s, ANDI | SAI(src, dst, c)); 1172 return; 1173 } else if ((c & 0xffff0000) == c) { 1174 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16)); 1175 return; 1176 } else { 1177 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c); 1178 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0)); 1179 } 1180} 1181 1182static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c) 1183{ 1184 int mb, me; 1185 1186 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 1187 if (mask64_operand(c, &mb, &me)) { 1188 if (mb == 0) { 1189 tcg_out_rld(s, RLDICR, dst, src, 0, me); 1190 } else { 1191 tcg_out_rld(s, RLDICL, dst, src, 0, mb); 1192 } 1193 } else if ((c & 0xffff) == c) { 1194 tcg_out32(s, ANDI | SAI(src, dst, c)); 1195 return; 1196 } else if ((c & 0xffff0000) == c) { 1197 tcg_out32(s, ANDIS | SAI(src, dst, c >> 16)); 1198 return; 1199 } else { 1200 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c); 1201 tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0)); 1202 } 1203} 1204 1205static void tcg_out_zori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c, 1206 int op_lo, int op_hi) 1207{ 1208 if (c >> 16) { 1209 tcg_out32(s, op_hi | SAI(src, dst, c >> 16)); 1210 src = dst; 1211 } 1212 if (c & 0xffff) { 1213 tcg_out32(s, op_lo | SAI(src, dst, c)); 1214 src = dst; 1215 } 1216} 1217 1218static void tcg_out_ori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c) 1219{ 1220 tcg_out_zori32(s, dst, src, c, ORI, ORIS); 1221} 1222 1223static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c) 1224{ 1225 tcg_out_zori32(s, dst, src, c, XORI, XORIS); 1226} 1227 1228static void tcg_out_b(TCGContext *s, int mask, const tcg_insn_unit *target) 1229{ 1230 ptrdiff_t disp = tcg_pcrel_diff(s, target); 1231 if (in_range_b(disp)) { 1232 tcg_out32(s, B | (disp & 0x3fffffc) | mask); 1233 } else { 1234 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, (uintptr_t)target); 1235 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | CTR); 1236 tcg_out32(s, BCCTR | BO_ALWAYS | mask); 1237 } 1238} 1239 1240static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt, 1241 TCGReg base, tcg_target_long offset) 1242{ 1243 tcg_target_long orig = offset, l0, l1, extra = 0, align = 0; 1244 bool is_int_store = false; 1245 TCGReg rs = TCG_REG_TMP1; 1246 1247 switch (opi) { 1248 case LD: case LWA: 1249 align = 3; 1250 /* FALLTHRU */ 1251 default: 1252 if (rt > TCG_REG_R0 && rt < TCG_REG_V0) { 1253 rs = rt; 1254 break; 1255 } 1256 break; 1257 case LXSD: 1258 case STXSD: 1259 align = 3; 1260 break; 1261 case LXV: 1262 case STXV: 1263 align = 15; 1264 break; 1265 case STD: 1266 align = 3; 1267 /* FALLTHRU */ 1268 case STB: case STH: case STW: 1269 is_int_store = true; 1270 break; 1271 } 1272 1273 /* For unaligned, or very large offsets, use the indexed form. */ 1274 if (offset & align || offset != (int32_t)offset || opi == 0) { 1275 if (rs == base) { 1276 rs = TCG_REG_R0; 1277 } 1278 tcg_debug_assert(!is_int_store || rs != rt); 1279 tcg_out_movi(s, TCG_TYPE_PTR, rs, orig); 1280 tcg_out32(s, opx | TAB(rt & 31, base, rs)); 1281 return; 1282 } 1283 1284 l0 = (int16_t)offset; 1285 offset = (offset - l0) >> 16; 1286 l1 = (int16_t)offset; 1287 1288 if (l1 < 0 && orig >= 0) { 1289 extra = 0x4000; 1290 l1 = (int16_t)(offset - 0x4000); 1291 } 1292 if (l1) { 1293 tcg_out32(s, ADDIS | TAI(rs, base, l1)); 1294 base = rs; 1295 } 1296 if (extra) { 1297 tcg_out32(s, ADDIS | TAI(rs, base, extra)); 1298 base = rs; 1299 } 1300 if (opi != ADDI || base != rt || l0 != 0) { 1301 tcg_out32(s, opi | TAI(rt & 31, base, l0)); 1302 } 1303} 1304 1305static void tcg_out_vsldoi(TCGContext *s, TCGReg ret, 1306 TCGReg va, TCGReg vb, int shb) 1307{ 1308 tcg_out32(s, VSLDOI | VRT(ret) | VRA(va) | VRB(vb) | (shb << 6)); 1309} 1310 1311static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, 1312 TCGReg base, intptr_t offset) 1313{ 1314 int shift; 1315 1316 switch (type) { 1317 case TCG_TYPE_I32: 1318 if (ret < TCG_REG_V0) { 1319 tcg_out_mem_long(s, LWZ, LWZX, ret, base, offset); 1320 break; 1321 } 1322 if (have_isa_2_07 && have_vsx) { 1323 tcg_out_mem_long(s, 0, LXSIWZX, ret, base, offset); 1324 break; 1325 } 1326 tcg_debug_assert((offset & 3) == 0); 1327 tcg_out_mem_long(s, 0, LVEWX, ret, base, offset); 1328 shift = (offset - 4) & 0xc; 1329 if (shift) { 1330 tcg_out_vsldoi(s, ret, ret, ret, shift); 1331 } 1332 break; 1333 case TCG_TYPE_I64: 1334 if (ret < TCG_REG_V0) { 1335 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 1336 tcg_out_mem_long(s, LD, LDX, ret, base, offset); 1337 break; 1338 } 1339 /* fallthru */ 1340 case TCG_TYPE_V64: 1341 tcg_debug_assert(ret >= TCG_REG_V0); 1342 if (have_vsx) { 1343 tcg_out_mem_long(s, have_isa_3_00 ? LXSD : 0, LXSDX, 1344 ret, base, offset); 1345 break; 1346 } 1347 tcg_debug_assert((offset & 7) == 0); 1348 tcg_out_mem_long(s, 0, LVX, ret, base, offset & -16); 1349 if (offset & 8) { 1350 tcg_out_vsldoi(s, ret, ret, ret, 8); 1351 } 1352 break; 1353 case TCG_TYPE_V128: 1354 tcg_debug_assert(ret >= TCG_REG_V0); 1355 tcg_debug_assert((offset & 15) == 0); 1356 tcg_out_mem_long(s, have_isa_3_00 ? LXV : 0, 1357 LVX, ret, base, offset); 1358 break; 1359 default: 1360 g_assert_not_reached(); 1361 } 1362} 1363 1364static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, 1365 TCGReg base, intptr_t offset) 1366{ 1367 int shift; 1368 1369 switch (type) { 1370 case TCG_TYPE_I32: 1371 if (arg < TCG_REG_V0) { 1372 tcg_out_mem_long(s, STW, STWX, arg, base, offset); 1373 break; 1374 } 1375 if (have_isa_2_07 && have_vsx) { 1376 tcg_out_mem_long(s, 0, STXSIWX, arg, base, offset); 1377 break; 1378 } 1379 assert((offset & 3) == 0); 1380 tcg_debug_assert((offset & 3) == 0); 1381 shift = (offset - 4) & 0xc; 1382 if (shift) { 1383 tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, shift); 1384 arg = TCG_VEC_TMP1; 1385 } 1386 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset); 1387 break; 1388 case TCG_TYPE_I64: 1389 if (arg < TCG_REG_V0) { 1390 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 1391 tcg_out_mem_long(s, STD, STDX, arg, base, offset); 1392 break; 1393 } 1394 /* fallthru */ 1395 case TCG_TYPE_V64: 1396 tcg_debug_assert(arg >= TCG_REG_V0); 1397 if (have_vsx) { 1398 tcg_out_mem_long(s, have_isa_3_00 ? STXSD : 0, 1399 STXSDX, arg, base, offset); 1400 break; 1401 } 1402 tcg_debug_assert((offset & 7) == 0); 1403 if (offset & 8) { 1404 tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, 8); 1405 arg = TCG_VEC_TMP1; 1406 } 1407 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset); 1408 tcg_out_mem_long(s, 0, STVEWX, arg, base, offset + 4); 1409 break; 1410 case TCG_TYPE_V128: 1411 tcg_debug_assert(arg >= TCG_REG_V0); 1412 tcg_out_mem_long(s, have_isa_3_00 ? STXV : 0, 1413 STVX, arg, base, offset); 1414 break; 1415 default: 1416 g_assert_not_reached(); 1417 } 1418} 1419 1420static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 1421 TCGReg base, intptr_t ofs) 1422{ 1423 return false; 1424} 1425 1426static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2, 1427 int const_arg2, int cr, TCGType type) 1428{ 1429 int imm; 1430 uint32_t op; 1431 1432 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); 1433 1434 /* Simplify the comparisons below wrt CMPI. */ 1435 if (type == TCG_TYPE_I32) { 1436 arg2 = (int32_t)arg2; 1437 } 1438 1439 switch (cond) { 1440 case TCG_COND_EQ: 1441 case TCG_COND_NE: 1442 if (const_arg2) { 1443 if ((int16_t) arg2 == arg2) { 1444 op = CMPI; 1445 imm = 1; 1446 break; 1447 } else if ((uint16_t) arg2 == arg2) { 1448 op = CMPLI; 1449 imm = 1; 1450 break; 1451 } 1452 } 1453 op = CMPL; 1454 imm = 0; 1455 break; 1456 1457 case TCG_COND_LT: 1458 case TCG_COND_GE: 1459 case TCG_COND_LE: 1460 case TCG_COND_GT: 1461 if (const_arg2) { 1462 if ((int16_t) arg2 == arg2) { 1463 op = CMPI; 1464 imm = 1; 1465 break; 1466 } 1467 } 1468 op = CMP; 1469 imm = 0; 1470 break; 1471 1472 case TCG_COND_LTU: 1473 case TCG_COND_GEU: 1474 case TCG_COND_LEU: 1475 case TCG_COND_GTU: 1476 if (const_arg2) { 1477 if ((uint16_t) arg2 == arg2) { 1478 op = CMPLI; 1479 imm = 1; 1480 break; 1481 } 1482 } 1483 op = CMPL; 1484 imm = 0; 1485 break; 1486 1487 default: 1488 tcg_abort(); 1489 } 1490 op |= BF(cr) | ((type == TCG_TYPE_I64) << 21); 1491 1492 if (imm) { 1493 tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff)); 1494 } else { 1495 if (const_arg2) { 1496 tcg_out_movi(s, type, TCG_REG_R0, arg2); 1497 arg2 = TCG_REG_R0; 1498 } 1499 tcg_out32(s, op | RA(arg1) | RB(arg2)); 1500 } 1501} 1502 1503static void tcg_out_setcond_eq0(TCGContext *s, TCGType type, 1504 TCGReg dst, TCGReg src) 1505{ 1506 if (type == TCG_TYPE_I32) { 1507 tcg_out32(s, CNTLZW | RS(src) | RA(dst)); 1508 tcg_out_shri32(s, dst, dst, 5); 1509 } else { 1510 tcg_out32(s, CNTLZD | RS(src) | RA(dst)); 1511 tcg_out_shri64(s, dst, dst, 6); 1512 } 1513} 1514 1515static void tcg_out_setcond_ne0(TCGContext *s, TCGReg dst, TCGReg src) 1516{ 1517 /* X != 0 implies X + -1 generates a carry. Extra addition 1518 trickery means: R = X-1 + ~X + C = X-1 + (-X+1) + C = C. */ 1519 if (dst != src) { 1520 tcg_out32(s, ADDIC | TAI(dst, src, -1)); 1521 tcg_out32(s, SUBFE | TAB(dst, dst, src)); 1522 } else { 1523 tcg_out32(s, ADDIC | TAI(TCG_REG_R0, src, -1)); 1524 tcg_out32(s, SUBFE | TAB(dst, TCG_REG_R0, src)); 1525 } 1526} 1527 1528static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2, 1529 bool const_arg2) 1530{ 1531 if (const_arg2) { 1532 if ((uint32_t)arg2 == arg2) { 1533 tcg_out_xori32(s, TCG_REG_R0, arg1, arg2); 1534 } else { 1535 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, arg2); 1536 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, TCG_REG_R0)); 1537 } 1538 } else { 1539 tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, arg2)); 1540 } 1541 return TCG_REG_R0; 1542} 1543 1544static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond, 1545 TCGArg arg0, TCGArg arg1, TCGArg arg2, 1546 int const_arg2) 1547{ 1548 int crop, sh; 1549 1550 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); 1551 1552 /* Ignore high bits of a potential constant arg2. */ 1553 if (type == TCG_TYPE_I32) { 1554 arg2 = (uint32_t)arg2; 1555 } 1556 1557 /* Handle common and trivial cases before handling anything else. */ 1558 if (arg2 == 0) { 1559 switch (cond) { 1560 case TCG_COND_EQ: 1561 tcg_out_setcond_eq0(s, type, arg0, arg1); 1562 return; 1563 case TCG_COND_NE: 1564 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { 1565 tcg_out_ext32u(s, TCG_REG_R0, arg1); 1566 arg1 = TCG_REG_R0; 1567 } 1568 tcg_out_setcond_ne0(s, arg0, arg1); 1569 return; 1570 case TCG_COND_GE: 1571 tcg_out32(s, NOR | SAB(arg1, arg0, arg1)); 1572 arg1 = arg0; 1573 /* FALLTHRU */ 1574 case TCG_COND_LT: 1575 /* Extract the sign bit. */ 1576 if (type == TCG_TYPE_I32) { 1577 tcg_out_shri32(s, arg0, arg1, 31); 1578 } else { 1579 tcg_out_shri64(s, arg0, arg1, 63); 1580 } 1581 return; 1582 default: 1583 break; 1584 } 1585 } 1586 1587 /* If we have ISEL, we can implement everything with 3 or 4 insns. 1588 All other cases below are also at least 3 insns, so speed up the 1589 code generator by not considering them and always using ISEL. */ 1590 if (have_isel) { 1591 int isel, tab; 1592 1593 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type); 1594 1595 isel = tcg_to_isel[cond]; 1596 1597 tcg_out_movi(s, type, arg0, 1); 1598 if (isel & 1) { 1599 /* arg0 = (bc ? 0 : 1) */ 1600 tab = TAB(arg0, 0, arg0); 1601 isel &= ~1; 1602 } else { 1603 /* arg0 = (bc ? 1 : 0) */ 1604 tcg_out_movi(s, type, TCG_REG_R0, 0); 1605 tab = TAB(arg0, arg0, TCG_REG_R0); 1606 } 1607 tcg_out32(s, isel | tab); 1608 return; 1609 } 1610 1611 switch (cond) { 1612 case TCG_COND_EQ: 1613 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2); 1614 tcg_out_setcond_eq0(s, type, arg0, arg1); 1615 return; 1616 1617 case TCG_COND_NE: 1618 arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2); 1619 /* Discard the high bits only once, rather than both inputs. */ 1620 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { 1621 tcg_out_ext32u(s, TCG_REG_R0, arg1); 1622 arg1 = TCG_REG_R0; 1623 } 1624 tcg_out_setcond_ne0(s, arg0, arg1); 1625 return; 1626 1627 case TCG_COND_GT: 1628 case TCG_COND_GTU: 1629 sh = 30; 1630 crop = 0; 1631 goto crtest; 1632 1633 case TCG_COND_LT: 1634 case TCG_COND_LTU: 1635 sh = 29; 1636 crop = 0; 1637 goto crtest; 1638 1639 case TCG_COND_GE: 1640 case TCG_COND_GEU: 1641 sh = 31; 1642 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_LT) | BB(7, CR_LT); 1643 goto crtest; 1644 1645 case TCG_COND_LE: 1646 case TCG_COND_LEU: 1647 sh = 31; 1648 crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_GT) | BB(7, CR_GT); 1649 crtest: 1650 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type); 1651 if (crop) { 1652 tcg_out32(s, crop); 1653 } 1654 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7)); 1655 tcg_out_rlw(s, RLWINM, arg0, TCG_REG_R0, sh, 31, 31); 1656 break; 1657 1658 default: 1659 tcg_abort(); 1660 } 1661} 1662 1663static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l) 1664{ 1665 if (l->has_value) { 1666 bc |= reloc_pc14_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr); 1667 } else { 1668 tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0); 1669 } 1670 tcg_out32(s, bc); 1671} 1672 1673static void tcg_out_brcond(TCGContext *s, TCGCond cond, 1674 TCGArg arg1, TCGArg arg2, int const_arg2, 1675 TCGLabel *l, TCGType type) 1676{ 1677 tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type); 1678 tcg_out_bc(s, tcg_to_bc[cond], l); 1679} 1680 1681static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond, 1682 TCGArg dest, TCGArg c1, TCGArg c2, TCGArg v1, 1683 TCGArg v2, bool const_c2) 1684{ 1685 /* If for some reason both inputs are zero, don't produce bad code. */ 1686 if (v1 == 0 && v2 == 0) { 1687 tcg_out_movi(s, type, dest, 0); 1688 return; 1689 } 1690 1691 tcg_out_cmp(s, cond, c1, c2, const_c2, 7, type); 1692 1693 if (have_isel) { 1694 int isel = tcg_to_isel[cond]; 1695 1696 /* Swap the V operands if the operation indicates inversion. */ 1697 if (isel & 1) { 1698 int t = v1; 1699 v1 = v2; 1700 v2 = t; 1701 isel &= ~1; 1702 } 1703 /* V1 == 0 is handled by isel; V2 == 0 must be handled by hand. */ 1704 if (v2 == 0) { 1705 tcg_out_movi(s, type, TCG_REG_R0, 0); 1706 } 1707 tcg_out32(s, isel | TAB(dest, v1, v2)); 1708 } else { 1709 if (dest == v2) { 1710 cond = tcg_invert_cond(cond); 1711 v2 = v1; 1712 } else if (dest != v1) { 1713 if (v1 == 0) { 1714 tcg_out_movi(s, type, dest, 0); 1715 } else { 1716 tcg_out_mov(s, type, dest, v1); 1717 } 1718 } 1719 /* Branch forward over one insn */ 1720 tcg_out32(s, tcg_to_bc[cond] | 8); 1721 if (v2 == 0) { 1722 tcg_out_movi(s, type, dest, 0); 1723 } else { 1724 tcg_out_mov(s, type, dest, v2); 1725 } 1726 } 1727} 1728 1729static void tcg_out_cntxz(TCGContext *s, TCGType type, uint32_t opc, 1730 TCGArg a0, TCGArg a1, TCGArg a2, bool const_a2) 1731{ 1732 if (const_a2 && a2 == (type == TCG_TYPE_I32 ? 32 : 64)) { 1733 tcg_out32(s, opc | RA(a0) | RS(a1)); 1734 } else { 1735 tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, 7, type); 1736 /* Note that the only other valid constant for a2 is 0. */ 1737 if (have_isel) { 1738 tcg_out32(s, opc | RA(TCG_REG_R0) | RS(a1)); 1739 tcg_out32(s, tcg_to_isel[TCG_COND_EQ] | TAB(a0, a2, TCG_REG_R0)); 1740 } else if (!const_a2 && a0 == a2) { 1741 tcg_out32(s, tcg_to_bc[TCG_COND_EQ] | 8); 1742 tcg_out32(s, opc | RA(a0) | RS(a1)); 1743 } else { 1744 tcg_out32(s, opc | RA(a0) | RS(a1)); 1745 tcg_out32(s, tcg_to_bc[TCG_COND_NE] | 8); 1746 if (const_a2) { 1747 tcg_out_movi(s, type, a0, 0); 1748 } else { 1749 tcg_out_mov(s, type, a0, a2); 1750 } 1751 } 1752 } 1753} 1754 1755static void tcg_out_cmp2(TCGContext *s, const TCGArg *args, 1756 const int *const_args) 1757{ 1758 static const struct { uint8_t bit1, bit2; } bits[] = { 1759 [TCG_COND_LT ] = { CR_LT, CR_LT }, 1760 [TCG_COND_LE ] = { CR_LT, CR_GT }, 1761 [TCG_COND_GT ] = { CR_GT, CR_GT }, 1762 [TCG_COND_GE ] = { CR_GT, CR_LT }, 1763 [TCG_COND_LTU] = { CR_LT, CR_LT }, 1764 [TCG_COND_LEU] = { CR_LT, CR_GT }, 1765 [TCG_COND_GTU] = { CR_GT, CR_GT }, 1766 [TCG_COND_GEU] = { CR_GT, CR_LT }, 1767 }; 1768 1769 TCGCond cond = args[4], cond2; 1770 TCGArg al, ah, bl, bh; 1771 int blconst, bhconst; 1772 int op, bit1, bit2; 1773 1774 al = args[0]; 1775 ah = args[1]; 1776 bl = args[2]; 1777 bh = args[3]; 1778 blconst = const_args[2]; 1779 bhconst = const_args[3]; 1780 1781 switch (cond) { 1782 case TCG_COND_EQ: 1783 op = CRAND; 1784 goto do_equality; 1785 case TCG_COND_NE: 1786 op = CRNAND; 1787 do_equality: 1788 tcg_out_cmp(s, cond, al, bl, blconst, 6, TCG_TYPE_I32); 1789 tcg_out_cmp(s, cond, ah, bh, bhconst, 7, TCG_TYPE_I32); 1790 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ)); 1791 break; 1792 1793 case TCG_COND_LT: 1794 case TCG_COND_LE: 1795 case TCG_COND_GT: 1796 case TCG_COND_GE: 1797 case TCG_COND_LTU: 1798 case TCG_COND_LEU: 1799 case TCG_COND_GTU: 1800 case TCG_COND_GEU: 1801 bit1 = bits[cond].bit1; 1802 bit2 = bits[cond].bit2; 1803 op = (bit1 != bit2 ? CRANDC : CRAND); 1804 cond2 = tcg_unsigned_cond(cond); 1805 1806 tcg_out_cmp(s, cond, ah, bh, bhconst, 6, TCG_TYPE_I32); 1807 tcg_out_cmp(s, cond2, al, bl, blconst, 7, TCG_TYPE_I32); 1808 tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2)); 1809 tcg_out32(s, CROR | BT(7, CR_EQ) | BA(6, bit1) | BB(7, CR_EQ)); 1810 break; 1811 1812 default: 1813 tcg_abort(); 1814 } 1815} 1816 1817static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, 1818 const int *const_args) 1819{ 1820 tcg_out_cmp2(s, args + 1, const_args + 1); 1821 tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7)); 1822 tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, 31, 31, 31); 1823} 1824 1825static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args, 1826 const int *const_args) 1827{ 1828 tcg_out_cmp2(s, args, const_args); 1829 tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5])); 1830} 1831 1832static void tcg_out_mb(TCGContext *s, TCGArg a0) 1833{ 1834 uint32_t insn = HWSYNC; 1835 a0 &= TCG_MO_ALL; 1836 if (a0 == TCG_MO_LD_LD) { 1837 insn = LWSYNC; 1838 } else if (a0 == TCG_MO_ST_ST) { 1839 insn = EIEIO; 1840 } 1841 tcg_out32(s, insn); 1842} 1843 1844void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, 1845 uintptr_t jmp_rw, uintptr_t addr) 1846{ 1847 if (TCG_TARGET_REG_BITS == 64) { 1848 tcg_insn_unit i1, i2; 1849 intptr_t tb_diff = addr - tc_ptr; 1850 intptr_t br_diff = addr - (jmp_rx + 4); 1851 uint64_t pair; 1852 1853 /* This does not exercise the range of the branch, but we do 1854 still need to be able to load the new value of TCG_REG_TB. 1855 But this does still happen quite often. */ 1856 if (tb_diff == (int16_t)tb_diff) { 1857 i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff); 1858 i2 = B | (br_diff & 0x3fffffc); 1859 } else { 1860 intptr_t lo = (int16_t)tb_diff; 1861 intptr_t hi = (int32_t)(tb_diff - lo); 1862 assert(tb_diff == hi + lo); 1863 i1 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16); 1864 i2 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo); 1865 } 1866#ifdef HOST_WORDS_BIGENDIAN 1867 pair = (uint64_t)i1 << 32 | i2; 1868#else 1869 pair = (uint64_t)i2 << 32 | i1; 1870#endif 1871 1872 /* As per the enclosing if, this is ppc64. Avoid the _Static_assert 1873 within qatomic_set that would fail to build a ppc32 host. */ 1874 qatomic_set__nocheck((uint64_t *)jmp_rw, pair); 1875 flush_idcache_range(jmp_rx, jmp_rw, 8); 1876 } else { 1877 intptr_t diff = addr - jmp_rx; 1878 tcg_debug_assert(in_range_b(diff)); 1879 qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc)); 1880 flush_idcache_range(jmp_rx, jmp_rw, 4); 1881 } 1882} 1883 1884static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) 1885{ 1886#ifdef _CALL_AIX 1887 /* Look through the descriptor. If the branch is in range, and we 1888 don't have to spend too much effort on building the toc. */ 1889 const void *tgt = ((const void * const *)target)[0]; 1890 uintptr_t toc = ((const uintptr_t *)target)[1]; 1891 intptr_t diff = tcg_pcrel_diff(s, tgt); 1892 1893 if (in_range_b(diff) && toc == (uint32_t)toc) { 1894 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc); 1895 tcg_out_b(s, LK, tgt); 1896 } else { 1897 /* Fold the low bits of the constant into the addresses below. */ 1898 intptr_t arg = (intptr_t)target; 1899 int ofs = (int16_t)arg; 1900 1901 if (ofs + 8 < 0x8000) { 1902 arg -= ofs; 1903 } else { 1904 ofs = 0; 1905 } 1906 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, arg); 1907 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs); 1908 tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR); 1909 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP); 1910 tcg_out32(s, BCCTR | BO_ALWAYS | LK); 1911 } 1912#elif defined(_CALL_ELF) && _CALL_ELF == 2 1913 intptr_t diff; 1914 1915 /* In the ELFv2 ABI, we have to set up r12 to contain the destination 1916 address, which the callee uses to compute its TOC address. */ 1917 /* FIXME: when the branch is in range, we could avoid r12 load if we 1918 knew that the destination uses the same TOC, and what its local 1919 entry point offset is. */ 1920 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R12, (intptr_t)target); 1921 1922 diff = tcg_pcrel_diff(s, target); 1923 if (in_range_b(diff)) { 1924 tcg_out_b(s, LK, target); 1925 } else { 1926 tcg_out32(s, MTSPR | RS(TCG_REG_R12) | CTR); 1927 tcg_out32(s, BCCTR | BO_ALWAYS | LK); 1928 } 1929#else 1930 tcg_out_b(s, LK, target); 1931#endif 1932} 1933 1934static const uint32_t qemu_ldx_opc[16] = { 1935 [MO_UB] = LBZX, 1936 [MO_UW] = LHZX, 1937 [MO_UL] = LWZX, 1938 [MO_Q] = LDX, 1939 [MO_SW] = LHAX, 1940 [MO_SL] = LWAX, 1941 [MO_BSWAP | MO_UB] = LBZX, 1942 [MO_BSWAP | MO_UW] = LHBRX, 1943 [MO_BSWAP | MO_UL] = LWBRX, 1944 [MO_BSWAP | MO_Q] = LDBRX, 1945}; 1946 1947static const uint32_t qemu_stx_opc[16] = { 1948 [MO_UB] = STBX, 1949 [MO_UW] = STHX, 1950 [MO_UL] = STWX, 1951 [MO_Q] = STDX, 1952 [MO_BSWAP | MO_UB] = STBX, 1953 [MO_BSWAP | MO_UW] = STHBRX, 1954 [MO_BSWAP | MO_UL] = STWBRX, 1955 [MO_BSWAP | MO_Q] = STDBRX, 1956}; 1957 1958static const uint32_t qemu_exts_opc[4] = { 1959 EXTSB, EXTSH, EXTSW, 0 1960}; 1961 1962#if defined (CONFIG_SOFTMMU) 1963#include "../tcg-ldst.c.inc" 1964 1965/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr, 1966 * int mmu_idx, uintptr_t ra) 1967 */ 1968static void * const qemu_ld_helpers[16] = { 1969 [MO_UB] = helper_ret_ldub_mmu, 1970 [MO_LEUW] = helper_le_lduw_mmu, 1971 [MO_LEUL] = helper_le_ldul_mmu, 1972 [MO_LEQ] = helper_le_ldq_mmu, 1973 [MO_BEUW] = helper_be_lduw_mmu, 1974 [MO_BEUL] = helper_be_ldul_mmu, 1975 [MO_BEQ] = helper_be_ldq_mmu, 1976}; 1977 1978/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr, 1979 * uintxx_t val, int mmu_idx, uintptr_t ra) 1980 */ 1981static void * const qemu_st_helpers[16] = { 1982 [MO_UB] = helper_ret_stb_mmu, 1983 [MO_LEUW] = helper_le_stw_mmu, 1984 [MO_LEUL] = helper_le_stl_mmu, 1985 [MO_LEQ] = helper_le_stq_mmu, 1986 [MO_BEUW] = helper_be_stw_mmu, 1987 [MO_BEUL] = helper_be_stl_mmu, 1988 [MO_BEQ] = helper_be_stq_mmu, 1989}; 1990 1991/* We expect to use a 16-bit negative offset from ENV. */ 1992QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); 1993QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -32768); 1994 1995/* Perform the TLB load and compare. Places the result of the comparison 1996 in CR7, loads the addend of the TLB into R3, and returns the register 1997 containing the guest address (zero-extended into R4). Clobbers R0 and R2. */ 1998 1999static TCGReg tcg_out_tlb_read(TCGContext *s, MemOp opc, 2000 TCGReg addrlo, TCGReg addrhi, 2001 int mem_index, bool is_read) 2002{ 2003 int cmp_off 2004 = (is_read 2005 ? offsetof(CPUTLBEntry, addr_read) 2006 : offsetof(CPUTLBEntry, addr_write)); 2007 int fast_off = TLB_MASK_TABLE_OFS(mem_index); 2008 int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); 2009 int table_off = fast_off + offsetof(CPUTLBDescFast, table); 2010 unsigned s_bits = opc & MO_SIZE; 2011 unsigned a_bits = get_alignment_bits(opc); 2012 2013 /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */ 2014 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_AREG0, mask_off); 2015 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R4, TCG_AREG0, table_off); 2016 2017 /* Extract the page index, shifted into place for tlb index. */ 2018 if (TCG_TARGET_REG_BITS == 32) { 2019 tcg_out_shri32(s, TCG_REG_TMP1, addrlo, 2020 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); 2021 } else { 2022 tcg_out_shri64(s, TCG_REG_TMP1, addrlo, 2023 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); 2024 } 2025 tcg_out32(s, AND | SAB(TCG_REG_R3, TCG_REG_R3, TCG_REG_TMP1)); 2026 2027 /* Load the TLB comparator. */ 2028 if (cmp_off == 0 && TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { 2029 uint32_t lxu = (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32 2030 ? LWZUX : LDUX); 2031 tcg_out32(s, lxu | TAB(TCG_REG_TMP1, TCG_REG_R3, TCG_REG_R4)); 2032 } else { 2033 tcg_out32(s, ADD | TAB(TCG_REG_R3, TCG_REG_R3, TCG_REG_R4)); 2034 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { 2035 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP1, TCG_REG_R3, cmp_off + 4); 2036 tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R4, TCG_REG_R3, cmp_off); 2037 } else { 2038 tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP1, TCG_REG_R3, cmp_off); 2039 } 2040 } 2041 2042 /* Load the TLB addend for use on the fast path. Do this asap 2043 to minimize any load use delay. */ 2044 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3, 2045 offsetof(CPUTLBEntry, addend)); 2046 2047 /* Clear the non-page, non-alignment bits from the address */ 2048 if (TCG_TARGET_REG_BITS == 32) { 2049 /* We don't support unaligned accesses on 32-bits. 2050 * Preserve the bottom bits and thus trigger a comparison 2051 * failure on unaligned accesses. 2052 */ 2053 if (a_bits < s_bits) { 2054 a_bits = s_bits; 2055 } 2056 tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0, 2057 (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS); 2058 } else { 2059 TCGReg t = addrlo; 2060 2061 /* If the access is unaligned, we need to make sure we fail if we 2062 * cross a page boundary. The trick is to add the access size-1 2063 * to the address before masking the low bits. That will make the 2064 * address overflow to the next page if we cross a page boundary, 2065 * which will then force a mismatch of the TLB compare. 2066 */ 2067 if (a_bits < s_bits) { 2068 unsigned a_mask = (1 << a_bits) - 1; 2069 unsigned s_mask = (1 << s_bits) - 1; 2070 tcg_out32(s, ADDI | TAI(TCG_REG_R0, t, s_mask - a_mask)); 2071 t = TCG_REG_R0; 2072 } 2073 2074 /* Mask the address for the requested alignment. */ 2075 if (TARGET_LONG_BITS == 32) { 2076 tcg_out_rlw(s, RLWINM, TCG_REG_R0, t, 0, 2077 (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS); 2078 /* Zero-extend the address for use in the final address. */ 2079 tcg_out_ext32u(s, TCG_REG_R4, addrlo); 2080 addrlo = TCG_REG_R4; 2081 } else if (a_bits == 0) { 2082 tcg_out_rld(s, RLDICR, TCG_REG_R0, t, 0, 63 - TARGET_PAGE_BITS); 2083 } else { 2084 tcg_out_rld(s, RLDICL, TCG_REG_R0, t, 2085 64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits); 2086 tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0); 2087 } 2088 } 2089 2090 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { 2091 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1, 2092 0, 7, TCG_TYPE_I32); 2093 tcg_out_cmp(s, TCG_COND_EQ, addrhi, TCG_REG_R4, 0, 6, TCG_TYPE_I32); 2094 tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ)); 2095 } else { 2096 tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1, 2097 0, 7, TCG_TYPE_TL); 2098 } 2099 2100 return addrlo; 2101} 2102 2103/* Record the context of a call to the out of line helper code for the slow 2104 path for a load or store, so that we can later generate the correct 2105 helper code. */ 2106static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi, 2107 TCGReg datalo_reg, TCGReg datahi_reg, 2108 TCGReg addrlo_reg, TCGReg addrhi_reg, 2109 tcg_insn_unit *raddr, tcg_insn_unit *lptr) 2110{ 2111 TCGLabelQemuLdst *label = new_ldst_label(s); 2112 2113 label->is_ld = is_ld; 2114 label->oi = oi; 2115 label->datalo_reg = datalo_reg; 2116 label->datahi_reg = datahi_reg; 2117 label->addrlo_reg = addrlo_reg; 2118 label->addrhi_reg = addrhi_reg; 2119 label->raddr = tcg_splitwx_to_rx(raddr); 2120 label->label_ptr[0] = lptr; 2121} 2122 2123static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) 2124{ 2125 TCGMemOpIdx oi = lb->oi; 2126 MemOp opc = get_memop(oi); 2127 TCGReg hi, lo, arg = TCG_REG_R3; 2128 2129 if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 2130 return false; 2131 } 2132 2133 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0); 2134 2135 lo = lb->addrlo_reg; 2136 hi = lb->addrhi_reg; 2137 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { 2138#ifdef TCG_TARGET_CALL_ALIGN_ARGS 2139 arg |= 1; 2140#endif 2141 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi); 2142 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo); 2143 } else { 2144 /* If the address needed to be zero-extended, we'll have already 2145 placed it in R4. The only remaining case is 64-bit guest. */ 2146 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo); 2147 } 2148 2149 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi); 2150 tcg_out32(s, MFSPR | RT(arg) | LR); 2151 2152 tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); 2153 2154 lo = lb->datalo_reg; 2155 hi = lb->datahi_reg; 2156 if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) { 2157 tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4); 2158 tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3); 2159 } else if (opc & MO_SIGN) { 2160 uint32_t insn = qemu_exts_opc[opc & MO_SIZE]; 2161 tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3)); 2162 } else { 2163 tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3); 2164 } 2165 2166 tcg_out_b(s, 0, lb->raddr); 2167 return true; 2168} 2169 2170static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) 2171{ 2172 TCGMemOpIdx oi = lb->oi; 2173 MemOp opc = get_memop(oi); 2174 MemOp s_bits = opc & MO_SIZE; 2175 TCGReg hi, lo, arg = TCG_REG_R3; 2176 2177 if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 2178 return false; 2179 } 2180 2181 tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0); 2182 2183 lo = lb->addrlo_reg; 2184 hi = lb->addrhi_reg; 2185 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { 2186#ifdef TCG_TARGET_CALL_ALIGN_ARGS 2187 arg |= 1; 2188#endif 2189 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi); 2190 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo); 2191 } else { 2192 /* If the address needed to be zero-extended, we'll have already 2193 placed it in R4. The only remaining case is 64-bit guest. */ 2194 tcg_out_mov(s, TCG_TYPE_TL, arg++, lo); 2195 } 2196 2197 lo = lb->datalo_reg; 2198 hi = lb->datahi_reg; 2199 if (TCG_TARGET_REG_BITS == 32) { 2200 switch (s_bits) { 2201 case MO_64: 2202#ifdef TCG_TARGET_CALL_ALIGN_ARGS 2203 arg |= 1; 2204#endif 2205 tcg_out_mov(s, TCG_TYPE_I32, arg++, hi); 2206 /* FALLTHRU */ 2207 case MO_32: 2208 tcg_out_mov(s, TCG_TYPE_I32, arg++, lo); 2209 break; 2210 default: 2211 tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31); 2212 break; 2213 } 2214 } else { 2215 if (s_bits == MO_64) { 2216 tcg_out_mov(s, TCG_TYPE_I64, arg++, lo); 2217 } else { 2218 tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits)); 2219 } 2220 } 2221 2222 tcg_out_movi(s, TCG_TYPE_I32, arg++, oi); 2223 tcg_out32(s, MFSPR | RT(arg) | LR); 2224 2225 tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); 2226 2227 tcg_out_b(s, 0, lb->raddr); 2228 return true; 2229} 2230#endif /* SOFTMMU */ 2231 2232static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) 2233{ 2234 TCGReg datalo, datahi, addrlo, rbase; 2235 TCGReg addrhi __attribute__((unused)); 2236 TCGMemOpIdx oi; 2237 MemOp opc, s_bits; 2238#ifdef CONFIG_SOFTMMU 2239 int mem_index; 2240 tcg_insn_unit *label_ptr; 2241#endif 2242 2243 datalo = *args++; 2244 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); 2245 addrlo = *args++; 2246 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); 2247 oi = *args++; 2248 opc = get_memop(oi); 2249 s_bits = opc & MO_SIZE; 2250 2251#ifdef CONFIG_SOFTMMU 2252 mem_index = get_mmuidx(oi); 2253 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true); 2254 2255 /* Load a pointer into the current opcode w/conditional branch-link. */ 2256 label_ptr = s->code_ptr; 2257 tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK); 2258 2259 rbase = TCG_REG_R3; 2260#else /* !CONFIG_SOFTMMU */ 2261 rbase = guest_base ? TCG_GUEST_BASE_REG : 0; 2262 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { 2263 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); 2264 addrlo = TCG_REG_TMP1; 2265 } 2266#endif 2267 2268 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) { 2269 if (opc & MO_BSWAP) { 2270 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); 2271 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo)); 2272 tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0)); 2273 } else if (rbase != 0) { 2274 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); 2275 tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo)); 2276 tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0)); 2277 } else if (addrlo == datahi) { 2278 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4)); 2279 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0)); 2280 } else { 2281 tcg_out32(s, LWZ | TAI(datahi, addrlo, 0)); 2282 tcg_out32(s, LWZ | TAI(datalo, addrlo, 4)); 2283 } 2284 } else { 2285 uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)]; 2286 if (!have_isa_2_06 && insn == LDBRX) { 2287 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); 2288 tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo)); 2289 tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0)); 2290 tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0); 2291 } else if (insn) { 2292 tcg_out32(s, insn | TAB(datalo, rbase, addrlo)); 2293 } else { 2294 insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)]; 2295 tcg_out32(s, insn | TAB(datalo, rbase, addrlo)); 2296 insn = qemu_exts_opc[s_bits]; 2297 tcg_out32(s, insn | RA(datalo) | RS(datalo)); 2298 } 2299 } 2300 2301#ifdef CONFIG_SOFTMMU 2302 add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi, 2303 s->code_ptr, label_ptr); 2304#endif 2305} 2306 2307static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) 2308{ 2309 TCGReg datalo, datahi, addrlo, rbase; 2310 TCGReg addrhi __attribute__((unused)); 2311 TCGMemOpIdx oi; 2312 MemOp opc, s_bits; 2313#ifdef CONFIG_SOFTMMU 2314 int mem_index; 2315 tcg_insn_unit *label_ptr; 2316#endif 2317 2318 datalo = *args++; 2319 datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); 2320 addrlo = *args++; 2321 addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); 2322 oi = *args++; 2323 opc = get_memop(oi); 2324 s_bits = opc & MO_SIZE; 2325 2326#ifdef CONFIG_SOFTMMU 2327 mem_index = get_mmuidx(oi); 2328 addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false); 2329 2330 /* Load a pointer into the current opcode w/conditional branch-link. */ 2331 label_ptr = s->code_ptr; 2332 tcg_out32(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK); 2333 2334 rbase = TCG_REG_R3; 2335#else /* !CONFIG_SOFTMMU */ 2336 rbase = guest_base ? TCG_GUEST_BASE_REG : 0; 2337 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { 2338 tcg_out_ext32u(s, TCG_REG_TMP1, addrlo); 2339 addrlo = TCG_REG_TMP1; 2340 } 2341#endif 2342 2343 if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) { 2344 if (opc & MO_BSWAP) { 2345 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); 2346 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo)); 2347 tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0)); 2348 } else if (rbase != 0) { 2349 tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4)); 2350 tcg_out32(s, STWX | SAB(datahi, rbase, addrlo)); 2351 tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0)); 2352 } else { 2353 tcg_out32(s, STW | TAI(datahi, addrlo, 0)); 2354 tcg_out32(s, STW | TAI(datalo, addrlo, 4)); 2355 } 2356 } else { 2357 uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)]; 2358 if (!have_isa_2_06 && insn == STDBRX) { 2359 tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo)); 2360 tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4)); 2361 tcg_out_shri64(s, TCG_REG_R0, datalo, 32); 2362 tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1)); 2363 } else { 2364 tcg_out32(s, insn | SAB(datalo, rbase, addrlo)); 2365 } 2366 } 2367 2368#ifdef CONFIG_SOFTMMU 2369 add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi, 2370 s->code_ptr, label_ptr); 2371#endif 2372} 2373 2374static void tcg_out_nop_fill(tcg_insn_unit *p, int count) 2375{ 2376 int i; 2377 for (i = 0; i < count; ++i) { 2378 p[i] = NOP; 2379 } 2380} 2381 2382/* Parameters for function call generation, used in tcg.c. */ 2383#define TCG_TARGET_STACK_ALIGN 16 2384#define TCG_TARGET_EXTEND_ARGS 1 2385 2386#ifdef _CALL_AIX 2387# define LINK_AREA_SIZE (6 * SZR) 2388# define LR_OFFSET (1 * SZR) 2389# define TCG_TARGET_CALL_STACK_OFFSET (LINK_AREA_SIZE + 8 * SZR) 2390#elif defined(_CALL_DARWIN) 2391# define LINK_AREA_SIZE (6 * SZR) 2392# define LR_OFFSET (2 * SZR) 2393#elif TCG_TARGET_REG_BITS == 64 2394# if defined(_CALL_ELF) && _CALL_ELF == 2 2395# define LINK_AREA_SIZE (4 * SZR) 2396# define LR_OFFSET (1 * SZR) 2397# endif 2398#else /* TCG_TARGET_REG_BITS == 32 */ 2399# if defined(_CALL_SYSV) 2400# define LINK_AREA_SIZE (2 * SZR) 2401# define LR_OFFSET (1 * SZR) 2402# endif 2403#endif 2404#ifndef LR_OFFSET 2405# error "Unhandled abi" 2406#endif 2407#ifndef TCG_TARGET_CALL_STACK_OFFSET 2408# define TCG_TARGET_CALL_STACK_OFFSET LINK_AREA_SIZE 2409#endif 2410 2411#define CPU_TEMP_BUF_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long)) 2412#define REG_SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * SZR) 2413 2414#define FRAME_SIZE ((TCG_TARGET_CALL_STACK_OFFSET \ 2415 + TCG_STATIC_CALL_ARGS_SIZE \ 2416 + CPU_TEMP_BUF_SIZE \ 2417 + REG_SAVE_SIZE \ 2418 + TCG_TARGET_STACK_ALIGN - 1) \ 2419 & -TCG_TARGET_STACK_ALIGN) 2420 2421#define REG_SAVE_BOT (FRAME_SIZE - REG_SAVE_SIZE) 2422 2423static void tcg_target_qemu_prologue(TCGContext *s) 2424{ 2425 int i; 2426 2427#ifdef _CALL_AIX 2428 const void **desc = (const void **)s->code_ptr; 2429 desc[0] = tcg_splitwx_to_rx(desc + 2); /* entry point */ 2430 desc[1] = 0; /* environment pointer */ 2431 s->code_ptr = (void *)(desc + 2); /* skip over descriptor */ 2432#endif 2433 2434 tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE, 2435 CPU_TEMP_BUF_SIZE); 2436 2437 /* Prologue */ 2438 tcg_out32(s, MFSPR | RT(TCG_REG_R0) | LR); 2439 tcg_out32(s, (SZR == 8 ? STDU : STWU) 2440 | SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE)); 2441 2442 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) { 2443 tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], 2444 TCG_REG_R1, REG_SAVE_BOT + i * SZR); 2445 } 2446 tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET); 2447 2448#ifndef CONFIG_SOFTMMU 2449 if (guest_base) { 2450 tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base, true); 2451 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); 2452 } 2453#endif 2454 2455 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); 2456 tcg_out32(s, MTSPR | RS(tcg_target_call_iarg_regs[1]) | CTR); 2457 if (USE_REG_TB) { 2458 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, tcg_target_call_iarg_regs[1]); 2459 } 2460 tcg_out32(s, BCCTR | BO_ALWAYS); 2461 2462 /* Epilogue */ 2463 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); 2464 2465 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET); 2466 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) { 2467 tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], 2468 TCG_REG_R1, REG_SAVE_BOT + i * SZR); 2469 } 2470 tcg_out32(s, MTSPR | RS(TCG_REG_R0) | LR); 2471 tcg_out32(s, ADDI | TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE)); 2472 tcg_out32(s, BCLR | BO_ALWAYS); 2473} 2474 2475static void tcg_out_op(TCGContext *s, TCGOpcode opc, 2476 const TCGArg args[TCG_MAX_OP_ARGS], 2477 const int const_args[TCG_MAX_OP_ARGS]) 2478{ 2479 TCGArg a0, a1, a2; 2480 2481 switch (opc) { 2482 case INDEX_op_exit_tb: 2483 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]); 2484 tcg_out_b(s, 0, tcg_code_gen_epilogue); 2485 break; 2486 case INDEX_op_goto_tb: 2487 if (s->tb_jmp_insn_offset) { 2488 /* Direct jump. */ 2489 if (TCG_TARGET_REG_BITS == 64) { 2490 /* Ensure the next insns are 8-byte aligned. */ 2491 if ((uintptr_t)s->code_ptr & 7) { 2492 tcg_out32(s, NOP); 2493 } 2494 s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); 2495 tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); 2496 tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); 2497 } else { 2498 s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); 2499 tcg_out32(s, B); 2500 s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); 2501 break; 2502 } 2503 } else { 2504 /* Indirect jump. */ 2505 tcg_debug_assert(s->tb_jmp_insn_offset == NULL); 2506 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0, 2507 (intptr_t)(s->tb_jmp_insn_offset + args[0])); 2508 } 2509 tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); 2510 tcg_out32(s, BCCTR | BO_ALWAYS); 2511 set_jmp_reset_offset(s, args[0]); 2512 if (USE_REG_TB) { 2513 /* For the unlinked case, need to reset TCG_REG_TB. */ 2514 tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, 2515 -tcg_current_code_size(s)); 2516 } 2517 break; 2518 case INDEX_op_goto_ptr: 2519 tcg_out32(s, MTSPR | RS(args[0]) | CTR); 2520 if (USE_REG_TB) { 2521 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_TB, args[0]); 2522 } 2523 tcg_out32(s, ADDI | TAI(TCG_REG_R3, 0, 0)); 2524 tcg_out32(s, BCCTR | BO_ALWAYS); 2525 break; 2526 case INDEX_op_br: 2527 { 2528 TCGLabel *l = arg_label(args[0]); 2529 uint32_t insn = B; 2530 2531 if (l->has_value) { 2532 insn |= reloc_pc24_val(tcg_splitwx_to_rx(s->code_ptr), 2533 l->u.value_ptr); 2534 } else { 2535 tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0); 2536 } 2537 tcg_out32(s, insn); 2538 } 2539 break; 2540 case INDEX_op_ld8u_i32: 2541 case INDEX_op_ld8u_i64: 2542 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]); 2543 break; 2544 case INDEX_op_ld8s_i32: 2545 case INDEX_op_ld8s_i64: 2546 tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]); 2547 tcg_out_ext8s(s, args[0], args[0]); 2548 break; 2549 case INDEX_op_ld16u_i32: 2550 case INDEX_op_ld16u_i64: 2551 tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]); 2552 break; 2553 case INDEX_op_ld16s_i32: 2554 case INDEX_op_ld16s_i64: 2555 tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]); 2556 break; 2557 case INDEX_op_ld_i32: 2558 case INDEX_op_ld32u_i64: 2559 tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]); 2560 break; 2561 case INDEX_op_ld32s_i64: 2562 tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]); 2563 break; 2564 case INDEX_op_ld_i64: 2565 tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]); 2566 break; 2567 case INDEX_op_st8_i32: 2568 case INDEX_op_st8_i64: 2569 tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]); 2570 break; 2571 case INDEX_op_st16_i32: 2572 case INDEX_op_st16_i64: 2573 tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]); 2574 break; 2575 case INDEX_op_st_i32: 2576 case INDEX_op_st32_i64: 2577 tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]); 2578 break; 2579 case INDEX_op_st_i64: 2580 tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]); 2581 break; 2582 2583 case INDEX_op_add_i32: 2584 a0 = args[0], a1 = args[1], a2 = args[2]; 2585 if (const_args[2]) { 2586 do_addi_32: 2587 tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2); 2588 } else { 2589 tcg_out32(s, ADD | TAB(a0, a1, a2)); 2590 } 2591 break; 2592 case INDEX_op_sub_i32: 2593 a0 = args[0], a1 = args[1], a2 = args[2]; 2594 if (const_args[1]) { 2595 if (const_args[2]) { 2596 tcg_out_movi(s, TCG_TYPE_I32, a0, a1 - a2); 2597 } else { 2598 tcg_out32(s, SUBFIC | TAI(a0, a2, a1)); 2599 } 2600 } else if (const_args[2]) { 2601 a2 = -a2; 2602 goto do_addi_32; 2603 } else { 2604 tcg_out32(s, SUBF | TAB(a0, a2, a1)); 2605 } 2606 break; 2607 2608 case INDEX_op_and_i32: 2609 a0 = args[0], a1 = args[1], a2 = args[2]; 2610 if (const_args[2]) { 2611 tcg_out_andi32(s, a0, a1, a2); 2612 } else { 2613 tcg_out32(s, AND | SAB(a1, a0, a2)); 2614 } 2615 break; 2616 case INDEX_op_and_i64: 2617 a0 = args[0], a1 = args[1], a2 = args[2]; 2618 if (const_args[2]) { 2619 tcg_out_andi64(s, a0, a1, a2); 2620 } else { 2621 tcg_out32(s, AND | SAB(a1, a0, a2)); 2622 } 2623 break; 2624 case INDEX_op_or_i64: 2625 case INDEX_op_or_i32: 2626 a0 = args[0], a1 = args[1], a2 = args[2]; 2627 if (const_args[2]) { 2628 tcg_out_ori32(s, a0, a1, a2); 2629 } else { 2630 tcg_out32(s, OR | SAB(a1, a0, a2)); 2631 } 2632 break; 2633 case INDEX_op_xor_i64: 2634 case INDEX_op_xor_i32: 2635 a0 = args[0], a1 = args[1], a2 = args[2]; 2636 if (const_args[2]) { 2637 tcg_out_xori32(s, a0, a1, a2); 2638 } else { 2639 tcg_out32(s, XOR | SAB(a1, a0, a2)); 2640 } 2641 break; 2642 case INDEX_op_andc_i32: 2643 a0 = args[0], a1 = args[1], a2 = args[2]; 2644 if (const_args[2]) { 2645 tcg_out_andi32(s, a0, a1, ~a2); 2646 } else { 2647 tcg_out32(s, ANDC | SAB(a1, a0, a2)); 2648 } 2649 break; 2650 case INDEX_op_andc_i64: 2651 a0 = args[0], a1 = args[1], a2 = args[2]; 2652 if (const_args[2]) { 2653 tcg_out_andi64(s, a0, a1, ~a2); 2654 } else { 2655 tcg_out32(s, ANDC | SAB(a1, a0, a2)); 2656 } 2657 break; 2658 case INDEX_op_orc_i32: 2659 if (const_args[2]) { 2660 tcg_out_ori32(s, args[0], args[1], ~args[2]); 2661 break; 2662 } 2663 /* FALLTHRU */ 2664 case INDEX_op_orc_i64: 2665 tcg_out32(s, ORC | SAB(args[1], args[0], args[2])); 2666 break; 2667 case INDEX_op_eqv_i32: 2668 if (const_args[2]) { 2669 tcg_out_xori32(s, args[0], args[1], ~args[2]); 2670 break; 2671 } 2672 /* FALLTHRU */ 2673 case INDEX_op_eqv_i64: 2674 tcg_out32(s, EQV | SAB(args[1], args[0], args[2])); 2675 break; 2676 case INDEX_op_nand_i32: 2677 case INDEX_op_nand_i64: 2678 tcg_out32(s, NAND | SAB(args[1], args[0], args[2])); 2679 break; 2680 case INDEX_op_nor_i32: 2681 case INDEX_op_nor_i64: 2682 tcg_out32(s, NOR | SAB(args[1], args[0], args[2])); 2683 break; 2684 2685 case INDEX_op_clz_i32: 2686 tcg_out_cntxz(s, TCG_TYPE_I32, CNTLZW, args[0], args[1], 2687 args[2], const_args[2]); 2688 break; 2689 case INDEX_op_ctz_i32: 2690 tcg_out_cntxz(s, TCG_TYPE_I32, CNTTZW, args[0], args[1], 2691 args[2], const_args[2]); 2692 break; 2693 case INDEX_op_ctpop_i32: 2694 tcg_out32(s, CNTPOPW | SAB(args[1], args[0], 0)); 2695 break; 2696 2697 case INDEX_op_clz_i64: 2698 tcg_out_cntxz(s, TCG_TYPE_I64, CNTLZD, args[0], args[1], 2699 args[2], const_args[2]); 2700 break; 2701 case INDEX_op_ctz_i64: 2702 tcg_out_cntxz(s, TCG_TYPE_I64, CNTTZD, args[0], args[1], 2703 args[2], const_args[2]); 2704 break; 2705 case INDEX_op_ctpop_i64: 2706 tcg_out32(s, CNTPOPD | SAB(args[1], args[0], 0)); 2707 break; 2708 2709 case INDEX_op_mul_i32: 2710 a0 = args[0], a1 = args[1], a2 = args[2]; 2711 if (const_args[2]) { 2712 tcg_out32(s, MULLI | TAI(a0, a1, a2)); 2713 } else { 2714 tcg_out32(s, MULLW | TAB(a0, a1, a2)); 2715 } 2716 break; 2717 2718 case INDEX_op_div_i32: 2719 tcg_out32(s, DIVW | TAB(args[0], args[1], args[2])); 2720 break; 2721 2722 case INDEX_op_divu_i32: 2723 tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2])); 2724 break; 2725 2726 case INDEX_op_shl_i32: 2727 if (const_args[2]) { 2728 /* Limit immediate shift count lest we create an illegal insn. */ 2729 tcg_out_shli32(s, args[0], args[1], args[2] & 31); 2730 } else { 2731 tcg_out32(s, SLW | SAB(args[1], args[0], args[2])); 2732 } 2733 break; 2734 case INDEX_op_shr_i32: 2735 if (const_args[2]) { 2736 /* Limit immediate shift count lest we create an illegal insn. */ 2737 tcg_out_shri32(s, args[0], args[1], args[2] & 31); 2738 } else { 2739 tcg_out32(s, SRW | SAB(args[1], args[0], args[2])); 2740 } 2741 break; 2742 case INDEX_op_sar_i32: 2743 if (const_args[2]) { 2744 tcg_out_sari32(s, args[0], args[1], args[2]); 2745 } else { 2746 tcg_out32(s, SRAW | SAB(args[1], args[0], args[2])); 2747 } 2748 break; 2749 case INDEX_op_rotl_i32: 2750 if (const_args[2]) { 2751 tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31); 2752 } else { 2753 tcg_out32(s, RLWNM | SAB(args[1], args[0], args[2]) 2754 | MB(0) | ME(31)); 2755 } 2756 break; 2757 case INDEX_op_rotr_i32: 2758 if (const_args[2]) { 2759 tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31); 2760 } else { 2761 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32)); 2762 tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0) 2763 | MB(0) | ME(31)); 2764 } 2765 break; 2766 2767 case INDEX_op_brcond_i32: 2768 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], 2769 arg_label(args[3]), TCG_TYPE_I32); 2770 break; 2771 case INDEX_op_brcond_i64: 2772 tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], 2773 arg_label(args[3]), TCG_TYPE_I64); 2774 break; 2775 case INDEX_op_brcond2_i32: 2776 tcg_out_brcond2(s, args, const_args); 2777 break; 2778 2779 case INDEX_op_neg_i32: 2780 case INDEX_op_neg_i64: 2781 tcg_out32(s, NEG | RT(args[0]) | RA(args[1])); 2782 break; 2783 2784 case INDEX_op_not_i32: 2785 case INDEX_op_not_i64: 2786 tcg_out32(s, NOR | SAB(args[1], args[0], args[1])); 2787 break; 2788 2789 case INDEX_op_add_i64: 2790 a0 = args[0], a1 = args[1], a2 = args[2]; 2791 if (const_args[2]) { 2792 do_addi_64: 2793 tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2); 2794 } else { 2795 tcg_out32(s, ADD | TAB(a0, a1, a2)); 2796 } 2797 break; 2798 case INDEX_op_sub_i64: 2799 a0 = args[0], a1 = args[1], a2 = args[2]; 2800 if (const_args[1]) { 2801 if (const_args[2]) { 2802 tcg_out_movi(s, TCG_TYPE_I64, a0, a1 - a2); 2803 } else { 2804 tcg_out32(s, SUBFIC | TAI(a0, a2, a1)); 2805 } 2806 } else if (const_args[2]) { 2807 a2 = -a2; 2808 goto do_addi_64; 2809 } else { 2810 tcg_out32(s, SUBF | TAB(a0, a2, a1)); 2811 } 2812 break; 2813 2814 case INDEX_op_shl_i64: 2815 if (const_args[2]) { 2816 /* Limit immediate shift count lest we create an illegal insn. */ 2817 tcg_out_shli64(s, args[0], args[1], args[2] & 63); 2818 } else { 2819 tcg_out32(s, SLD | SAB(args[1], args[0], args[2])); 2820 } 2821 break; 2822 case INDEX_op_shr_i64: 2823 if (const_args[2]) { 2824 /* Limit immediate shift count lest we create an illegal insn. */ 2825 tcg_out_shri64(s, args[0], args[1], args[2] & 63); 2826 } else { 2827 tcg_out32(s, SRD | SAB(args[1], args[0], args[2])); 2828 } 2829 break; 2830 case INDEX_op_sar_i64: 2831 if (const_args[2]) { 2832 tcg_out_sari64(s, args[0], args[1], args[2]); 2833 } else { 2834 tcg_out32(s, SRAD | SAB(args[1], args[0], args[2])); 2835 } 2836 break; 2837 case INDEX_op_rotl_i64: 2838 if (const_args[2]) { 2839 tcg_out_rld(s, RLDICL, args[0], args[1], args[2], 0); 2840 } else { 2841 tcg_out32(s, RLDCL | SAB(args[1], args[0], args[2]) | MB64(0)); 2842 } 2843 break; 2844 case INDEX_op_rotr_i64: 2845 if (const_args[2]) { 2846 tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0); 2847 } else { 2848 tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64)); 2849 tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0)); 2850 } 2851 break; 2852 2853 case INDEX_op_mul_i64: 2854 a0 = args[0], a1 = args[1], a2 = args[2]; 2855 if (const_args[2]) { 2856 tcg_out32(s, MULLI | TAI(a0, a1, a2)); 2857 } else { 2858 tcg_out32(s, MULLD | TAB(a0, a1, a2)); 2859 } 2860 break; 2861 case INDEX_op_div_i64: 2862 tcg_out32(s, DIVD | TAB(args[0], args[1], args[2])); 2863 break; 2864 case INDEX_op_divu_i64: 2865 tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2])); 2866 break; 2867 2868 case INDEX_op_qemu_ld_i32: 2869 tcg_out_qemu_ld(s, args, false); 2870 break; 2871 case INDEX_op_qemu_ld_i64: 2872 tcg_out_qemu_ld(s, args, true); 2873 break; 2874 case INDEX_op_qemu_st_i32: 2875 tcg_out_qemu_st(s, args, false); 2876 break; 2877 case INDEX_op_qemu_st_i64: 2878 tcg_out_qemu_st(s, args, true); 2879 break; 2880 2881 case INDEX_op_ext8s_i32: 2882 case INDEX_op_ext8s_i64: 2883 tcg_out_ext8s(s, args[0], args[1]); 2884 break; 2885 case INDEX_op_ext16s_i32: 2886 case INDEX_op_ext16s_i64: 2887 tcg_out_ext16s(s, args[0], args[1]); 2888 break; 2889 case INDEX_op_ext_i32_i64: 2890 case INDEX_op_ext32s_i64: 2891 tcg_out_ext32s(s, args[0], args[1]); 2892 break; 2893 case INDEX_op_extu_i32_i64: 2894 tcg_out_ext32u(s, args[0], args[1]); 2895 break; 2896 2897 case INDEX_op_setcond_i32: 2898 tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2], 2899 const_args[2]); 2900 break; 2901 case INDEX_op_setcond_i64: 2902 tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2], 2903 const_args[2]); 2904 break; 2905 case INDEX_op_setcond2_i32: 2906 tcg_out_setcond2(s, args, const_args); 2907 break; 2908 2909 case INDEX_op_bswap16_i32: 2910 case INDEX_op_bswap16_i64: 2911 tcg_out_bswap16(s, args[0], args[1], args[2]); 2912 break; 2913 case INDEX_op_bswap32_i32: 2914 tcg_out_bswap32(s, args[0], args[1], 0); 2915 break; 2916 case INDEX_op_bswap32_i64: 2917 tcg_out_bswap32(s, args[0], args[1], args[2]); 2918 break; 2919 case INDEX_op_bswap64_i64: 2920 tcg_out_bswap64(s, args[0], args[1]); 2921 break; 2922 2923 case INDEX_op_deposit_i32: 2924 if (const_args[2]) { 2925 uint32_t mask = ((2u << (args[4] - 1)) - 1) << args[3]; 2926 tcg_out_andi32(s, args[0], args[0], ~mask); 2927 } else { 2928 tcg_out_rlw(s, RLWIMI, args[0], args[2], args[3], 2929 32 - args[3] - args[4], 31 - args[3]); 2930 } 2931 break; 2932 case INDEX_op_deposit_i64: 2933 if (const_args[2]) { 2934 uint64_t mask = ((2ull << (args[4] - 1)) - 1) << args[3]; 2935 tcg_out_andi64(s, args[0], args[0], ~mask); 2936 } else { 2937 tcg_out_rld(s, RLDIMI, args[0], args[2], args[3], 2938 64 - args[3] - args[4]); 2939 } 2940 break; 2941 2942 case INDEX_op_extract_i32: 2943 tcg_out_rlw(s, RLWINM, args[0], args[1], 2944 32 - args[2], 32 - args[3], 31); 2945 break; 2946 case INDEX_op_extract_i64: 2947 tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 64 - args[3]); 2948 break; 2949 2950 case INDEX_op_movcond_i32: 2951 tcg_out_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], args[2], 2952 args[3], args[4], const_args[2]); 2953 break; 2954 case INDEX_op_movcond_i64: 2955 tcg_out_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], args[2], 2956 args[3], args[4], const_args[2]); 2957 break; 2958 2959#if TCG_TARGET_REG_BITS == 64 2960 case INDEX_op_add2_i64: 2961#else 2962 case INDEX_op_add2_i32: 2963#endif 2964 /* Note that the CA bit is defined based on the word size of the 2965 environment. So in 64-bit mode it's always carry-out of bit 63. 2966 The fallback code using deposit works just as well for 32-bit. */ 2967 a0 = args[0], a1 = args[1]; 2968 if (a0 == args[3] || (!const_args[5] && a0 == args[5])) { 2969 a0 = TCG_REG_R0; 2970 } 2971 if (const_args[4]) { 2972 tcg_out32(s, ADDIC | TAI(a0, args[2], args[4])); 2973 } else { 2974 tcg_out32(s, ADDC | TAB(a0, args[2], args[4])); 2975 } 2976 if (const_args[5]) { 2977 tcg_out32(s, (args[5] ? ADDME : ADDZE) | RT(a1) | RA(args[3])); 2978 } else { 2979 tcg_out32(s, ADDE | TAB(a1, args[3], args[5])); 2980 } 2981 if (a0 != args[0]) { 2982 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0); 2983 } 2984 break; 2985 2986#if TCG_TARGET_REG_BITS == 64 2987 case INDEX_op_sub2_i64: 2988#else 2989 case INDEX_op_sub2_i32: 2990#endif 2991 a0 = args[0], a1 = args[1]; 2992 if (a0 == args[5] || (!const_args[3] && a0 == args[3])) { 2993 a0 = TCG_REG_R0; 2994 } 2995 if (const_args[2]) { 2996 tcg_out32(s, SUBFIC | TAI(a0, args[4], args[2])); 2997 } else { 2998 tcg_out32(s, SUBFC | TAB(a0, args[4], args[2])); 2999 } 3000 if (const_args[3]) { 3001 tcg_out32(s, (args[3] ? SUBFME : SUBFZE) | RT(a1) | RA(args[5])); 3002 } else { 3003 tcg_out32(s, SUBFE | TAB(a1, args[5], args[3])); 3004 } 3005 if (a0 != args[0]) { 3006 tcg_out_mov(s, TCG_TYPE_REG, args[0], a0); 3007 } 3008 break; 3009 3010 case INDEX_op_muluh_i32: 3011 tcg_out32(s, MULHWU | TAB(args[0], args[1], args[2])); 3012 break; 3013 case INDEX_op_mulsh_i32: 3014 tcg_out32(s, MULHW | TAB(args[0], args[1], args[2])); 3015 break; 3016 case INDEX_op_muluh_i64: 3017 tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2])); 3018 break; 3019 case INDEX_op_mulsh_i64: 3020 tcg_out32(s, MULHD | TAB(args[0], args[1], args[2])); 3021 break; 3022 3023 case INDEX_op_mb: 3024 tcg_out_mb(s, args[0]); 3025 break; 3026 3027 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ 3028 case INDEX_op_mov_i64: 3029 case INDEX_op_call: /* Always emitted via tcg_out_call. */ 3030 default: 3031 tcg_abort(); 3032 } 3033} 3034 3035int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) 3036{ 3037 switch (opc) { 3038 case INDEX_op_and_vec: 3039 case INDEX_op_or_vec: 3040 case INDEX_op_xor_vec: 3041 case INDEX_op_andc_vec: 3042 case INDEX_op_not_vec: 3043 return 1; 3044 case INDEX_op_orc_vec: 3045 return have_isa_2_07; 3046 case INDEX_op_add_vec: 3047 case INDEX_op_sub_vec: 3048 case INDEX_op_smax_vec: 3049 case INDEX_op_smin_vec: 3050 case INDEX_op_umax_vec: 3051 case INDEX_op_umin_vec: 3052 case INDEX_op_shlv_vec: 3053 case INDEX_op_shrv_vec: 3054 case INDEX_op_sarv_vec: 3055 case INDEX_op_rotlv_vec: 3056 return vece <= MO_32 || have_isa_2_07; 3057 case INDEX_op_ssadd_vec: 3058 case INDEX_op_sssub_vec: 3059 case INDEX_op_usadd_vec: 3060 case INDEX_op_ussub_vec: 3061 return vece <= MO_32; 3062 case INDEX_op_cmp_vec: 3063 case INDEX_op_shli_vec: 3064 case INDEX_op_shri_vec: 3065 case INDEX_op_sari_vec: 3066 case INDEX_op_rotli_vec: 3067 return vece <= MO_32 || have_isa_2_07 ? -1 : 0; 3068 case INDEX_op_neg_vec: 3069 return vece >= MO_32 && have_isa_3_00; 3070 case INDEX_op_mul_vec: 3071 switch (vece) { 3072 case MO_8: 3073 case MO_16: 3074 return -1; 3075 case MO_32: 3076 return have_isa_2_07 ? 1 : -1; 3077 case MO_64: 3078 return have_isa_3_10; 3079 } 3080 return 0; 3081 case INDEX_op_bitsel_vec: 3082 return have_vsx; 3083 case INDEX_op_rotrv_vec: 3084 return -1; 3085 default: 3086 return 0; 3087 } 3088} 3089 3090static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 3091 TCGReg dst, TCGReg src) 3092{ 3093 tcg_debug_assert(dst >= TCG_REG_V0); 3094 3095 /* Splat from integer reg allowed via constraints for v3.00. */ 3096 if (src < TCG_REG_V0) { 3097 tcg_debug_assert(have_isa_3_00); 3098 switch (vece) { 3099 case MO_64: 3100 tcg_out32(s, MTVSRDD | VRT(dst) | RA(src) | RB(src)); 3101 return true; 3102 case MO_32: 3103 tcg_out32(s, MTVSRWS | VRT(dst) | RA(src)); 3104 return true; 3105 default: 3106 /* Fail, so that we fall back on either dupm or mov+dup. */ 3107 return false; 3108 } 3109 } 3110 3111 /* 3112 * Recall we use (or emulate) VSX integer loads, so the integer is 3113 * right justified within the left (zero-index) double-word. 3114 */ 3115 switch (vece) { 3116 case MO_8: 3117 tcg_out32(s, VSPLTB | VRT(dst) | VRB(src) | (7 << 16)); 3118 break; 3119 case MO_16: 3120 tcg_out32(s, VSPLTH | VRT(dst) | VRB(src) | (3 << 16)); 3121 break; 3122 case MO_32: 3123 tcg_out32(s, VSPLTW | VRT(dst) | VRB(src) | (1 << 16)); 3124 break; 3125 case MO_64: 3126 if (have_vsx) { 3127 tcg_out32(s, XXPERMDI | VRT(dst) | VRA(src) | VRB(src)); 3128 break; 3129 } 3130 tcg_out_vsldoi(s, TCG_VEC_TMP1, src, src, 8); 3131 tcg_out_vsldoi(s, dst, TCG_VEC_TMP1, src, 8); 3132 break; 3133 default: 3134 g_assert_not_reached(); 3135 } 3136 return true; 3137} 3138 3139static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 3140 TCGReg out, TCGReg base, intptr_t offset) 3141{ 3142 int elt; 3143 3144 tcg_debug_assert(out >= TCG_REG_V0); 3145 switch (vece) { 3146 case MO_8: 3147 if (have_isa_3_00) { 3148 tcg_out_mem_long(s, LXV, LVX, out, base, offset & -16); 3149 } else { 3150 tcg_out_mem_long(s, 0, LVEBX, out, base, offset); 3151 } 3152 elt = extract32(offset, 0, 4); 3153#ifndef HOST_WORDS_BIGENDIAN 3154 elt ^= 15; 3155#endif 3156 tcg_out32(s, VSPLTB | VRT(out) | VRB(out) | (elt << 16)); 3157 break; 3158 case MO_16: 3159 tcg_debug_assert((offset & 1) == 0); 3160 if (have_isa_3_00) { 3161 tcg_out_mem_long(s, LXV | 8, LVX, out, base, offset & -16); 3162 } else { 3163 tcg_out_mem_long(s, 0, LVEHX, out, base, offset); 3164 } 3165 elt = extract32(offset, 1, 3); 3166#ifndef HOST_WORDS_BIGENDIAN 3167 elt ^= 7; 3168#endif 3169 tcg_out32(s, VSPLTH | VRT(out) | VRB(out) | (elt << 16)); 3170 break; 3171 case MO_32: 3172 if (have_isa_3_00) { 3173 tcg_out_mem_long(s, 0, LXVWSX, out, base, offset); 3174 break; 3175 } 3176 tcg_debug_assert((offset & 3) == 0); 3177 tcg_out_mem_long(s, 0, LVEWX, out, base, offset); 3178 elt = extract32(offset, 2, 2); 3179#ifndef HOST_WORDS_BIGENDIAN 3180 elt ^= 3; 3181#endif 3182 tcg_out32(s, VSPLTW | VRT(out) | VRB(out) | (elt << 16)); 3183 break; 3184 case MO_64: 3185 if (have_vsx) { 3186 tcg_out_mem_long(s, 0, LXVDSX, out, base, offset); 3187 break; 3188 } 3189 tcg_debug_assert((offset & 7) == 0); 3190 tcg_out_mem_long(s, 0, LVX, out, base, offset & -16); 3191 tcg_out_vsldoi(s, TCG_VEC_TMP1, out, out, 8); 3192 elt = extract32(offset, 3, 1); 3193#ifndef HOST_WORDS_BIGENDIAN 3194 elt = !elt; 3195#endif 3196 if (elt) { 3197 tcg_out_vsldoi(s, out, out, TCG_VEC_TMP1, 8); 3198 } else { 3199 tcg_out_vsldoi(s, out, TCG_VEC_TMP1, out, 8); 3200 } 3201 break; 3202 default: 3203 g_assert_not_reached(); 3204 } 3205 return true; 3206} 3207 3208static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 3209 unsigned vecl, unsigned vece, 3210 const TCGArg args[TCG_MAX_OP_ARGS], 3211 const int const_args[TCG_MAX_OP_ARGS]) 3212{ 3213 static const uint32_t 3214 add_op[4] = { VADDUBM, VADDUHM, VADDUWM, VADDUDM }, 3215 sub_op[4] = { VSUBUBM, VSUBUHM, VSUBUWM, VSUBUDM }, 3216 mul_op[4] = { 0, 0, VMULUWM, VMULLD }, 3217 neg_op[4] = { 0, 0, VNEGW, VNEGD }, 3218 eq_op[4] = { VCMPEQUB, VCMPEQUH, VCMPEQUW, VCMPEQUD }, 3219 ne_op[4] = { VCMPNEB, VCMPNEH, VCMPNEW, 0 }, 3220 gts_op[4] = { VCMPGTSB, VCMPGTSH, VCMPGTSW, VCMPGTSD }, 3221 gtu_op[4] = { VCMPGTUB, VCMPGTUH, VCMPGTUW, VCMPGTUD }, 3222 ssadd_op[4] = { VADDSBS, VADDSHS, VADDSWS, 0 }, 3223 usadd_op[4] = { VADDUBS, VADDUHS, VADDUWS, 0 }, 3224 sssub_op[4] = { VSUBSBS, VSUBSHS, VSUBSWS, 0 }, 3225 ussub_op[4] = { VSUBUBS, VSUBUHS, VSUBUWS, 0 }, 3226 umin_op[4] = { VMINUB, VMINUH, VMINUW, VMINUD }, 3227 smin_op[4] = { VMINSB, VMINSH, VMINSW, VMINSD }, 3228 umax_op[4] = { VMAXUB, VMAXUH, VMAXUW, VMAXUD }, 3229 smax_op[4] = { VMAXSB, VMAXSH, VMAXSW, VMAXSD }, 3230 shlv_op[4] = { VSLB, VSLH, VSLW, VSLD }, 3231 shrv_op[4] = { VSRB, VSRH, VSRW, VSRD }, 3232 sarv_op[4] = { VSRAB, VSRAH, VSRAW, VSRAD }, 3233 mrgh_op[4] = { VMRGHB, VMRGHH, VMRGHW, 0 }, 3234 mrgl_op[4] = { VMRGLB, VMRGLH, VMRGLW, 0 }, 3235 muleu_op[4] = { VMULEUB, VMULEUH, VMULEUW, 0 }, 3236 mulou_op[4] = { VMULOUB, VMULOUH, VMULOUW, 0 }, 3237 pkum_op[4] = { VPKUHUM, VPKUWUM, 0, 0 }, 3238 rotl_op[4] = { VRLB, VRLH, VRLW, VRLD }; 3239 3240 TCGType type = vecl + TCG_TYPE_V64; 3241 TCGArg a0 = args[0], a1 = args[1], a2 = args[2]; 3242 uint32_t insn; 3243 3244 switch (opc) { 3245 case INDEX_op_ld_vec: 3246 tcg_out_ld(s, type, a0, a1, a2); 3247 return; 3248 case INDEX_op_st_vec: 3249 tcg_out_st(s, type, a0, a1, a2); 3250 return; 3251 case INDEX_op_dupm_vec: 3252 tcg_out_dupm_vec(s, type, vece, a0, a1, a2); 3253 return; 3254 3255 case INDEX_op_add_vec: 3256 insn = add_op[vece]; 3257 break; 3258 case INDEX_op_sub_vec: 3259 insn = sub_op[vece]; 3260 break; 3261 case INDEX_op_neg_vec: 3262 insn = neg_op[vece]; 3263 a2 = a1; 3264 a1 = 0; 3265 break; 3266 case INDEX_op_mul_vec: 3267 insn = mul_op[vece]; 3268 break; 3269 case INDEX_op_ssadd_vec: 3270 insn = ssadd_op[vece]; 3271 break; 3272 case INDEX_op_sssub_vec: 3273 insn = sssub_op[vece]; 3274 break; 3275 case INDEX_op_usadd_vec: 3276 insn = usadd_op[vece]; 3277 break; 3278 case INDEX_op_ussub_vec: 3279 insn = ussub_op[vece]; 3280 break; 3281 case INDEX_op_smin_vec: 3282 insn = smin_op[vece]; 3283 break; 3284 case INDEX_op_umin_vec: 3285 insn = umin_op[vece]; 3286 break; 3287 case INDEX_op_smax_vec: 3288 insn = smax_op[vece]; 3289 break; 3290 case INDEX_op_umax_vec: 3291 insn = umax_op[vece]; 3292 break; 3293 case INDEX_op_shlv_vec: 3294 insn = shlv_op[vece]; 3295 break; 3296 case INDEX_op_shrv_vec: 3297 insn = shrv_op[vece]; 3298 break; 3299 case INDEX_op_sarv_vec: 3300 insn = sarv_op[vece]; 3301 break; 3302 case INDEX_op_and_vec: 3303 insn = VAND; 3304 break; 3305 case INDEX_op_or_vec: 3306 insn = VOR; 3307 break; 3308 case INDEX_op_xor_vec: 3309 insn = VXOR; 3310 break; 3311 case INDEX_op_andc_vec: 3312 insn = VANDC; 3313 break; 3314 case INDEX_op_not_vec: 3315 insn = VNOR; 3316 a2 = a1; 3317 break; 3318 case INDEX_op_orc_vec: 3319 insn = VORC; 3320 break; 3321 3322 case INDEX_op_cmp_vec: 3323 switch (args[3]) { 3324 case TCG_COND_EQ: 3325 insn = eq_op[vece]; 3326 break; 3327 case TCG_COND_NE: 3328 insn = ne_op[vece]; 3329 break; 3330 case TCG_COND_GT: 3331 insn = gts_op[vece]; 3332 break; 3333 case TCG_COND_GTU: 3334 insn = gtu_op[vece]; 3335 break; 3336 default: 3337 g_assert_not_reached(); 3338 } 3339 break; 3340 3341 case INDEX_op_bitsel_vec: 3342 tcg_out32(s, XXSEL | VRT(a0) | VRC(a1) | VRB(a2) | VRA(args[3])); 3343 return; 3344 3345 case INDEX_op_dup2_vec: 3346 assert(TCG_TARGET_REG_BITS == 32); 3347 /* With inputs a1 = xLxx, a2 = xHxx */ 3348 tcg_out32(s, VMRGHW | VRT(a0) | VRA(a2) | VRB(a1)); /* a0 = xxHL */ 3349 tcg_out_vsldoi(s, TCG_VEC_TMP1, a0, a0, 8); /* tmp = HLxx */ 3350 tcg_out_vsldoi(s, a0, a0, TCG_VEC_TMP1, 8); /* a0 = HLHL */ 3351 return; 3352 3353 case INDEX_op_ppc_mrgh_vec: 3354 insn = mrgh_op[vece]; 3355 break; 3356 case INDEX_op_ppc_mrgl_vec: 3357 insn = mrgl_op[vece]; 3358 break; 3359 case INDEX_op_ppc_muleu_vec: 3360 insn = muleu_op[vece]; 3361 break; 3362 case INDEX_op_ppc_mulou_vec: 3363 insn = mulou_op[vece]; 3364 break; 3365 case INDEX_op_ppc_pkum_vec: 3366 insn = pkum_op[vece]; 3367 break; 3368 case INDEX_op_rotlv_vec: 3369 insn = rotl_op[vece]; 3370 break; 3371 case INDEX_op_ppc_msum_vec: 3372 tcg_debug_assert(vece == MO_16); 3373 tcg_out32(s, VMSUMUHM | VRT(a0) | VRA(a1) | VRB(a2) | VRC(args[3])); 3374 return; 3375 3376 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ 3377 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ 3378 default: 3379 g_assert_not_reached(); 3380 } 3381 3382 tcg_debug_assert(insn != 0); 3383 tcg_out32(s, insn | VRT(a0) | VRA(a1) | VRB(a2)); 3384} 3385 3386static void expand_vec_shi(TCGType type, unsigned vece, TCGv_vec v0, 3387 TCGv_vec v1, TCGArg imm, TCGOpcode opci) 3388{ 3389 TCGv_vec t1; 3390 3391 if (vece == MO_32) { 3392 /* 3393 * Only 5 bits are significant, and VSPLTISB can represent -16..15. 3394 * So using negative numbers gets us the 4th bit easily. 3395 */ 3396 imm = sextract32(imm, 0, 5); 3397 } else { 3398 imm &= (8 << vece) - 1; 3399 } 3400 3401 /* Splat w/bytes for xxspltib when 2.07 allows MO_64. */ 3402 t1 = tcg_constant_vec(type, MO_8, imm); 3403 vec_gen_3(opci, type, vece, tcgv_vec_arg(v0), 3404 tcgv_vec_arg(v1), tcgv_vec_arg(t1)); 3405} 3406 3407static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, 3408 TCGv_vec v1, TCGv_vec v2, TCGCond cond) 3409{ 3410 bool need_swap = false, need_inv = false; 3411 3412 tcg_debug_assert(vece <= MO_32 || have_isa_2_07); 3413 3414 switch (cond) { 3415 case TCG_COND_EQ: 3416 case TCG_COND_GT: 3417 case TCG_COND_GTU: 3418 break; 3419 case TCG_COND_NE: 3420 if (have_isa_3_00 && vece <= MO_32) { 3421 break; 3422 } 3423 /* fall through */ 3424 case TCG_COND_LE: 3425 case TCG_COND_LEU: 3426 need_inv = true; 3427 break; 3428 case TCG_COND_LT: 3429 case TCG_COND_LTU: 3430 need_swap = true; 3431 break; 3432 case TCG_COND_GE: 3433 case TCG_COND_GEU: 3434 need_swap = need_inv = true; 3435 break; 3436 default: 3437 g_assert_not_reached(); 3438 } 3439 3440 if (need_inv) { 3441 cond = tcg_invert_cond(cond); 3442 } 3443 if (need_swap) { 3444 TCGv_vec t1; 3445 t1 = v1, v1 = v2, v2 = t1; 3446 cond = tcg_swap_cond(cond); 3447 } 3448 3449 vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0), 3450 tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); 3451 3452 if (need_inv) { 3453 tcg_gen_not_vec(vece, v0, v0); 3454 } 3455} 3456 3457static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0, 3458 TCGv_vec v1, TCGv_vec v2) 3459{ 3460 TCGv_vec t1 = tcg_temp_new_vec(type); 3461 TCGv_vec t2 = tcg_temp_new_vec(type); 3462 TCGv_vec c0, c16; 3463 3464 switch (vece) { 3465 case MO_8: 3466 case MO_16: 3467 vec_gen_3(INDEX_op_ppc_muleu_vec, type, vece, tcgv_vec_arg(t1), 3468 tcgv_vec_arg(v1), tcgv_vec_arg(v2)); 3469 vec_gen_3(INDEX_op_ppc_mulou_vec, type, vece, tcgv_vec_arg(t2), 3470 tcgv_vec_arg(v1), tcgv_vec_arg(v2)); 3471 vec_gen_3(INDEX_op_ppc_mrgh_vec, type, vece + 1, tcgv_vec_arg(v0), 3472 tcgv_vec_arg(t1), tcgv_vec_arg(t2)); 3473 vec_gen_3(INDEX_op_ppc_mrgl_vec, type, vece + 1, tcgv_vec_arg(t1), 3474 tcgv_vec_arg(t1), tcgv_vec_arg(t2)); 3475 vec_gen_3(INDEX_op_ppc_pkum_vec, type, vece, tcgv_vec_arg(v0), 3476 tcgv_vec_arg(v0), tcgv_vec_arg(t1)); 3477 break; 3478 3479 case MO_32: 3480 tcg_debug_assert(!have_isa_2_07); 3481 /* 3482 * Only 5 bits are significant, and VSPLTISB can represent -16..15. 3483 * So using -16 is a quick way to represent 16. 3484 */ 3485 c16 = tcg_constant_vec(type, MO_8, -16); 3486 c0 = tcg_constant_vec(type, MO_8, 0); 3487 3488 vec_gen_3(INDEX_op_rotlv_vec, type, MO_32, tcgv_vec_arg(t1), 3489 tcgv_vec_arg(v2), tcgv_vec_arg(c16)); 3490 vec_gen_3(INDEX_op_ppc_mulou_vec, type, MO_16, tcgv_vec_arg(t2), 3491 tcgv_vec_arg(v1), tcgv_vec_arg(v2)); 3492 vec_gen_4(INDEX_op_ppc_msum_vec, type, MO_16, tcgv_vec_arg(t1), 3493 tcgv_vec_arg(v1), tcgv_vec_arg(t1), tcgv_vec_arg(c0)); 3494 vec_gen_3(INDEX_op_shlv_vec, type, MO_32, tcgv_vec_arg(t1), 3495 tcgv_vec_arg(t1), tcgv_vec_arg(c16)); 3496 tcg_gen_add_vec(MO_32, v0, t1, t2); 3497 break; 3498 3499 default: 3500 g_assert_not_reached(); 3501 } 3502 tcg_temp_free_vec(t1); 3503 tcg_temp_free_vec(t2); 3504} 3505 3506void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, 3507 TCGArg a0, ...) 3508{ 3509 va_list va; 3510 TCGv_vec v0, v1, v2, t0; 3511 TCGArg a2; 3512 3513 va_start(va, a0); 3514 v0 = temp_tcgv_vec(arg_temp(a0)); 3515 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 3516 a2 = va_arg(va, TCGArg); 3517 3518 switch (opc) { 3519 case INDEX_op_shli_vec: 3520 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_shlv_vec); 3521 break; 3522 case INDEX_op_shri_vec: 3523 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_shrv_vec); 3524 break; 3525 case INDEX_op_sari_vec: 3526 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_sarv_vec); 3527 break; 3528 case INDEX_op_rotli_vec: 3529 expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_rotlv_vec); 3530 break; 3531 case INDEX_op_cmp_vec: 3532 v2 = temp_tcgv_vec(arg_temp(a2)); 3533 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); 3534 break; 3535 case INDEX_op_mul_vec: 3536 v2 = temp_tcgv_vec(arg_temp(a2)); 3537 expand_vec_mul(type, vece, v0, v1, v2); 3538 break; 3539 case INDEX_op_rotlv_vec: 3540 v2 = temp_tcgv_vec(arg_temp(a2)); 3541 t0 = tcg_temp_new_vec(type); 3542 tcg_gen_neg_vec(vece, t0, v2); 3543 tcg_gen_rotlv_vec(vece, v0, v1, t0); 3544 tcg_temp_free_vec(t0); 3545 break; 3546 default: 3547 g_assert_not_reached(); 3548 } 3549 va_end(va); 3550} 3551 3552static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) 3553{ 3554 switch (op) { 3555 case INDEX_op_goto_ptr: 3556 return C_O0_I1(r); 3557 3558 case INDEX_op_ld8u_i32: 3559 case INDEX_op_ld8s_i32: 3560 case INDEX_op_ld16u_i32: 3561 case INDEX_op_ld16s_i32: 3562 case INDEX_op_ld_i32: 3563 case INDEX_op_ctpop_i32: 3564 case INDEX_op_neg_i32: 3565 case INDEX_op_not_i32: 3566 case INDEX_op_ext8s_i32: 3567 case INDEX_op_ext16s_i32: 3568 case INDEX_op_bswap16_i32: 3569 case INDEX_op_bswap32_i32: 3570 case INDEX_op_extract_i32: 3571 case INDEX_op_ld8u_i64: 3572 case INDEX_op_ld8s_i64: 3573 case INDEX_op_ld16u_i64: 3574 case INDEX_op_ld16s_i64: 3575 case INDEX_op_ld32u_i64: 3576 case INDEX_op_ld32s_i64: 3577 case INDEX_op_ld_i64: 3578 case INDEX_op_ctpop_i64: 3579 case INDEX_op_neg_i64: 3580 case INDEX_op_not_i64: 3581 case INDEX_op_ext8s_i64: 3582 case INDEX_op_ext16s_i64: 3583 case INDEX_op_ext32s_i64: 3584 case INDEX_op_ext_i32_i64: 3585 case INDEX_op_extu_i32_i64: 3586 case INDEX_op_bswap16_i64: 3587 case INDEX_op_bswap32_i64: 3588 case INDEX_op_bswap64_i64: 3589 case INDEX_op_extract_i64: 3590 return C_O1_I1(r, r); 3591 3592 case INDEX_op_st8_i32: 3593 case INDEX_op_st16_i32: 3594 case INDEX_op_st_i32: 3595 case INDEX_op_st8_i64: 3596 case INDEX_op_st16_i64: 3597 case INDEX_op_st32_i64: 3598 case INDEX_op_st_i64: 3599 return C_O0_I2(r, r); 3600 3601 case INDEX_op_add_i32: 3602 case INDEX_op_and_i32: 3603 case INDEX_op_or_i32: 3604 case INDEX_op_xor_i32: 3605 case INDEX_op_andc_i32: 3606 case INDEX_op_orc_i32: 3607 case INDEX_op_eqv_i32: 3608 case INDEX_op_shl_i32: 3609 case INDEX_op_shr_i32: 3610 case INDEX_op_sar_i32: 3611 case INDEX_op_rotl_i32: 3612 case INDEX_op_rotr_i32: 3613 case INDEX_op_setcond_i32: 3614 case INDEX_op_and_i64: 3615 case INDEX_op_andc_i64: 3616 case INDEX_op_shl_i64: 3617 case INDEX_op_shr_i64: 3618 case INDEX_op_sar_i64: 3619 case INDEX_op_rotl_i64: 3620 case INDEX_op_rotr_i64: 3621 case INDEX_op_setcond_i64: 3622 return C_O1_I2(r, r, ri); 3623 3624 case INDEX_op_mul_i32: 3625 case INDEX_op_mul_i64: 3626 return C_O1_I2(r, r, rI); 3627 3628 case INDEX_op_div_i32: 3629 case INDEX_op_divu_i32: 3630 case INDEX_op_nand_i32: 3631 case INDEX_op_nor_i32: 3632 case INDEX_op_muluh_i32: 3633 case INDEX_op_mulsh_i32: 3634 case INDEX_op_orc_i64: 3635 case INDEX_op_eqv_i64: 3636 case INDEX_op_nand_i64: 3637 case INDEX_op_nor_i64: 3638 case INDEX_op_div_i64: 3639 case INDEX_op_divu_i64: 3640 case INDEX_op_mulsh_i64: 3641 case INDEX_op_muluh_i64: 3642 return C_O1_I2(r, r, r); 3643 3644 case INDEX_op_sub_i32: 3645 return C_O1_I2(r, rI, ri); 3646 case INDEX_op_add_i64: 3647 return C_O1_I2(r, r, rT); 3648 case INDEX_op_or_i64: 3649 case INDEX_op_xor_i64: 3650 return C_O1_I2(r, r, rU); 3651 case INDEX_op_sub_i64: 3652 return C_O1_I2(r, rI, rT); 3653 case INDEX_op_clz_i32: 3654 case INDEX_op_ctz_i32: 3655 case INDEX_op_clz_i64: 3656 case INDEX_op_ctz_i64: 3657 return C_O1_I2(r, r, rZW); 3658 3659 case INDEX_op_brcond_i32: 3660 case INDEX_op_brcond_i64: 3661 return C_O0_I2(r, ri); 3662 3663 case INDEX_op_movcond_i32: 3664 case INDEX_op_movcond_i64: 3665 return C_O1_I4(r, r, ri, rZ, rZ); 3666 case INDEX_op_deposit_i32: 3667 case INDEX_op_deposit_i64: 3668 return C_O1_I2(r, 0, rZ); 3669 case INDEX_op_brcond2_i32: 3670 return C_O0_I4(r, r, ri, ri); 3671 case INDEX_op_setcond2_i32: 3672 return C_O1_I4(r, r, r, ri, ri); 3673 case INDEX_op_add2_i64: 3674 case INDEX_op_add2_i32: 3675 return C_O2_I4(r, r, r, r, rI, rZM); 3676 case INDEX_op_sub2_i64: 3677 case INDEX_op_sub2_i32: 3678 return C_O2_I4(r, r, rI, rZM, r, r); 3679 3680 case INDEX_op_qemu_ld_i32: 3681 return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 3682 ? C_O1_I1(r, L) 3683 : C_O1_I2(r, L, L)); 3684 3685 case INDEX_op_qemu_st_i32: 3686 return (TCG_TARGET_REG_BITS == 64 || TARGET_LONG_BITS == 32 3687 ? C_O0_I2(S, S) 3688 : C_O0_I3(S, S, S)); 3689 3690 case INDEX_op_qemu_ld_i64: 3691 return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) 3692 : TARGET_LONG_BITS == 32 ? C_O2_I1(L, L, L) 3693 : C_O2_I2(L, L, L, L)); 3694 3695 case INDEX_op_qemu_st_i64: 3696 return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(S, S) 3697 : TARGET_LONG_BITS == 32 ? C_O0_I3(S, S, S) 3698 : C_O0_I4(S, S, S, S)); 3699 3700 case INDEX_op_add_vec: 3701 case INDEX_op_sub_vec: 3702 case INDEX_op_mul_vec: 3703 case INDEX_op_and_vec: 3704 case INDEX_op_or_vec: 3705 case INDEX_op_xor_vec: 3706 case INDEX_op_andc_vec: 3707 case INDEX_op_orc_vec: 3708 case INDEX_op_cmp_vec: 3709 case INDEX_op_ssadd_vec: 3710 case INDEX_op_sssub_vec: 3711 case INDEX_op_usadd_vec: 3712 case INDEX_op_ussub_vec: 3713 case INDEX_op_smax_vec: 3714 case INDEX_op_smin_vec: 3715 case INDEX_op_umax_vec: 3716 case INDEX_op_umin_vec: 3717 case INDEX_op_shlv_vec: 3718 case INDEX_op_shrv_vec: 3719 case INDEX_op_sarv_vec: 3720 case INDEX_op_rotlv_vec: 3721 case INDEX_op_rotrv_vec: 3722 case INDEX_op_ppc_mrgh_vec: 3723 case INDEX_op_ppc_mrgl_vec: 3724 case INDEX_op_ppc_muleu_vec: 3725 case INDEX_op_ppc_mulou_vec: 3726 case INDEX_op_ppc_pkum_vec: 3727 case INDEX_op_dup2_vec: 3728 return C_O1_I2(v, v, v); 3729 3730 case INDEX_op_not_vec: 3731 case INDEX_op_neg_vec: 3732 return C_O1_I1(v, v); 3733 3734 case INDEX_op_dup_vec: 3735 return have_isa_3_00 ? C_O1_I1(v, vr) : C_O1_I1(v, v); 3736 3737 case INDEX_op_ld_vec: 3738 case INDEX_op_dupm_vec: 3739 return C_O1_I1(v, r); 3740 3741 case INDEX_op_st_vec: 3742 return C_O0_I2(v, r); 3743 3744 case INDEX_op_bitsel_vec: 3745 case INDEX_op_ppc_msum_vec: 3746 return C_O1_I3(v, v, v, v); 3747 3748 default: 3749 g_assert_not_reached(); 3750 } 3751} 3752 3753static void tcg_target_init(TCGContext *s) 3754{ 3755 unsigned long hwcap = qemu_getauxval(AT_HWCAP); 3756 unsigned long hwcap2 = qemu_getauxval(AT_HWCAP2); 3757 3758 have_isa = tcg_isa_base; 3759 if (hwcap & PPC_FEATURE_ARCH_2_06) { 3760 have_isa = tcg_isa_2_06; 3761 } 3762#ifdef PPC_FEATURE2_ARCH_2_07 3763 if (hwcap2 & PPC_FEATURE2_ARCH_2_07) { 3764 have_isa = tcg_isa_2_07; 3765 } 3766#endif 3767#ifdef PPC_FEATURE2_ARCH_3_00 3768 if (hwcap2 & PPC_FEATURE2_ARCH_3_00) { 3769 have_isa = tcg_isa_3_00; 3770 } 3771#endif 3772#ifdef PPC_FEATURE2_ARCH_3_10 3773 if (hwcap2 & PPC_FEATURE2_ARCH_3_10) { 3774 have_isa = tcg_isa_3_10; 3775 } 3776#endif 3777 3778#ifdef PPC_FEATURE2_HAS_ISEL 3779 /* Prefer explicit instruction from the kernel. */ 3780 have_isel = (hwcap2 & PPC_FEATURE2_HAS_ISEL) != 0; 3781#else 3782 /* Fall back to knowing Power7 (2.06) has ISEL. */ 3783 have_isel = have_isa_2_06; 3784#endif 3785 3786 if (hwcap & PPC_FEATURE_HAS_ALTIVEC) { 3787 have_altivec = true; 3788 /* We only care about the portion of VSX that overlaps Altivec. */ 3789 if (hwcap & PPC_FEATURE_HAS_VSX) { 3790 have_vsx = true; 3791 } 3792 } 3793 3794 tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; 3795 tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; 3796 if (have_altivec) { 3797 tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull; 3798 tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull; 3799 } 3800 3801 tcg_target_call_clobber_regs = 0; 3802 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0); 3803 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2); 3804 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3); 3805 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4); 3806 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5); 3807 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6); 3808 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R7); 3809 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8); 3810 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9); 3811 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10); 3812 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11); 3813 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R12); 3814 3815 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0); 3816 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1); 3817 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2); 3818 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3); 3819 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4); 3820 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5); 3821 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6); 3822 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7); 3823 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V8); 3824 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V9); 3825 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V10); 3826 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V11); 3827 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V12); 3828 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V13); 3829 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V14); 3830 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V15); 3831 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16); 3832 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17); 3833 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18); 3834 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19); 3835 3836 s->reserved_regs = 0; 3837 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */ 3838 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */ 3839#if defined(_CALL_SYSV) 3840 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* toc pointer */ 3841#endif 3842#if defined(_CALL_SYSV) || TCG_TARGET_REG_BITS == 64 3843 tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */ 3844#endif 3845 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); /* mem temp */ 3846 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP1); 3847 tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP2); 3848 if (USE_REG_TB) { 3849 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); /* tb->tc_ptr */ 3850 } 3851} 3852 3853#ifdef __ELF__ 3854typedef struct { 3855 DebugFrameCIE cie; 3856 DebugFrameFDEHeader fde; 3857 uint8_t fde_def_cfa[4]; 3858 uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3]; 3859} DebugFrame; 3860 3861/* We're expecting a 2 byte uleb128 encoded value. */ 3862QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14)); 3863 3864#if TCG_TARGET_REG_BITS == 64 3865# define ELF_HOST_MACHINE EM_PPC64 3866#else 3867# define ELF_HOST_MACHINE EM_PPC 3868#endif 3869 3870static DebugFrame debug_frame = { 3871 .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */ 3872 .cie.id = -1, 3873 .cie.version = 1, 3874 .cie.code_align = 1, 3875 .cie.data_align = (-SZR & 0x7f), /* sleb128 -SZR */ 3876 .cie.return_column = 65, 3877 3878 /* Total FDE size does not include the "len" member. */ 3879 .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset), 3880 3881 .fde_def_cfa = { 3882 12, TCG_REG_R1, /* DW_CFA_def_cfa r1, ... */ 3883 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 3884 (FRAME_SIZE >> 7) 3885 }, 3886 .fde_reg_ofs = { 3887 /* DW_CFA_offset_extended_sf, lr, LR_OFFSET */ 3888 0x11, 65, (LR_OFFSET / -SZR) & 0x7f, 3889 } 3890}; 3891 3892void tcg_register_jit(const void *buf, size_t buf_size) 3893{ 3894 uint8_t *p = &debug_frame.fde_reg_ofs[3]; 3895 int i; 3896 3897 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) { 3898 p[0] = 0x80 + tcg_target_callee_save_regs[i]; 3899 p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * SZR)) / SZR; 3900 } 3901 3902 debug_frame.fde.func_start = (uintptr_t)buf; 3903 debug_frame.fde.func_len = buf_size; 3904 3905 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); 3906} 3907#endif /* __ELF__ */ 3908