1 /* 2 * ARM SVE Load/Store Helpers 3 * 4 * Copyright (c) 2018-2022 Linaro 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #ifndef TARGET_ARM_SVE_LDST_INTERNAL_H 21 #define TARGET_ARM_SVE_LDST_INTERNAL_H 22 23 #include "accel/tcg/cpu-ldst.h" 24 25 /* 26 * Load one element into @vd + @reg_off from @host. 27 * The controlling predicate is known to be true. 28 */ 29 typedef void sve_ldst1_host_fn(void *vd, intptr_t reg_off, void *host); 30 31 /* 32 * Load one element into @vd + @reg_off from (@env, @vaddr, @ra). 33 * The controlling predicate is known to be true. 34 */ 35 typedef void sve_ldst1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off, 36 target_ulong vaddr, uintptr_t retaddr); 37 38 /* 39 * Generate the above primitives. 40 */ 41 42 #define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \ 43 static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ 44 { TYPEM val = HOST(host); *(TYPEE *)(vd + H(reg_off)) = val; } 45 46 #define DO_ST_HOST(NAME, H, TYPEE, TYPEM, HOST) \ 47 static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ 48 { TYPEM val = *(TYPEE *)(vd + H(reg_off)); HOST(host, val); } 49 50 #define DO_LD_TLB(NAME, H, TYPEE, TYPEM, TLB) \ 51 static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ 52 intptr_t reg_off, target_ulong addr, uintptr_t ra) \ 53 { \ 54 TYPEM val = TLB(env, useronly_clean_ptr(addr), ra); \ 55 *(TYPEE *)(vd + H(reg_off)) = val; \ 56 } 57 58 #define DO_ST_TLB(NAME, H, TYPEE, TYPEM, TLB) \ 59 static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ 60 intptr_t reg_off, target_ulong addr, uintptr_t ra) \ 61 { \ 62 TYPEM val = *(TYPEE *)(vd + H(reg_off)); \ 63 TLB(env, useronly_clean_ptr(addr), val, ra); \ 64 } 65 66 #define DO_LD_PRIM_1(NAME, H, TE, TM) \ 67 DO_LD_HOST(NAME, H, TE, TM, ldub_p) \ 68 DO_LD_TLB(NAME, H, TE, TM, cpu_ldub_data_ra) 69 70 DO_LD_PRIM_1(ld1bb, H1, uint8_t, uint8_t) 71 DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t) 72 DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t, int8_t) 73 DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t) 74 DO_LD_PRIM_1(ld1bss, H1_4, uint32_t, int8_t) 75 DO_LD_PRIM_1(ld1bdu, H1_8, uint64_t, uint8_t) 76 DO_LD_PRIM_1(ld1bds, H1_8, uint64_t, int8_t) 77 78 #define DO_ST_PRIM_1(NAME, H, TE, TM) \ 79 DO_ST_HOST(st1##NAME, H, TE, TM, stb_p) \ 80 DO_ST_TLB(st1##NAME, H, TE, TM, cpu_stb_data_ra) 81 82 DO_ST_PRIM_1(bb, H1, uint8_t, uint8_t) 83 DO_ST_PRIM_1(bh, H1_2, uint16_t, uint8_t) 84 DO_ST_PRIM_1(bs, H1_4, uint32_t, uint8_t) 85 DO_ST_PRIM_1(bd, H1_8, uint64_t, uint8_t) 86 87 #define DO_LD_PRIM_2(NAME, H, TE, TM, LD) \ 88 DO_LD_HOST(ld1##NAME##_be, H, TE, TM, LD##_be_p) \ 89 DO_LD_HOST(ld1##NAME##_le, H, TE, TM, LD##_le_p) \ 90 DO_LD_TLB(ld1##NAME##_be, H, TE, TM, cpu_##LD##_be_data_ra) \ 91 DO_LD_TLB(ld1##NAME##_le, H, TE, TM, cpu_##LD##_le_data_ra) 92 93 #define DO_ST_PRIM_2(NAME, H, TE, TM, ST) \ 94 DO_ST_HOST(st1##NAME##_be, H, TE, TM, ST##_be_p) \ 95 DO_ST_HOST(st1##NAME##_le, H, TE, TM, ST##_le_p) \ 96 DO_ST_TLB(st1##NAME##_be, H, TE, TM, cpu_##ST##_be_data_ra) \ 97 DO_ST_TLB(st1##NAME##_le, H, TE, TM, cpu_##ST##_le_data_ra) 98 99 DO_LD_PRIM_2(hh, H1_2, uint16_t, uint16_t, lduw) 100 DO_LD_PRIM_2(hsu, H1_4, uint32_t, uint16_t, lduw) 101 DO_LD_PRIM_2(hss, H1_4, uint32_t, int16_t, lduw) 102 DO_LD_PRIM_2(hdu, H1_8, uint64_t, uint16_t, lduw) 103 DO_LD_PRIM_2(hds, H1_8, uint64_t, int16_t, lduw) 104 105 DO_ST_PRIM_2(hh, H1_2, uint16_t, uint16_t, stw) 106 DO_ST_PRIM_2(hs, H1_4, uint32_t, uint16_t, stw) 107 DO_ST_PRIM_2(hd, H1_8, uint64_t, uint16_t, stw) 108 109 DO_LD_PRIM_2(ss, H1_4, uint32_t, uint32_t, ldl) 110 DO_LD_PRIM_2(sdu, H1_8, uint64_t, uint32_t, ldl) 111 DO_LD_PRIM_2(sds, H1_8, uint64_t, int32_t, ldl) 112 113 DO_ST_PRIM_2(ss, H1_4, uint32_t, uint32_t, stl) 114 DO_ST_PRIM_2(sd, H1_8, uint64_t, uint32_t, stl) 115 116 DO_LD_PRIM_2(dd, H1_8, uint64_t, uint64_t, ldq) 117 DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) 118 119 #define DO_LD_PRIM_3(NAME, FUNC) \ 120 static inline void sve_##NAME##_host(void *vd, \ 121 intptr_t reg_off, void *host) \ 122 { sve_##FUNC##_host(vd, reg_off, host); \ 123 *(uint64_t *)(vd + reg_off + 8) = 0; } \ 124 static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ 125 intptr_t reg_off, target_ulong addr, uintptr_t ra) \ 126 { sve_##FUNC##_tlb(env, vd, reg_off, addr, ra); \ 127 *(uint64_t *)(vd + reg_off + 8) = 0; } 128 129 DO_LD_PRIM_3(ld1squ_be, ld1sdu_be) 130 DO_LD_PRIM_3(ld1squ_le, ld1sdu_le) 131 DO_LD_PRIM_3(ld1dqu_be, ld1dd_be) 132 DO_LD_PRIM_3(ld1dqu_le, ld1dd_le) 133 134 #define sve_st1sq_be_host sve_st1sd_be_host 135 #define sve_st1sq_le_host sve_st1sd_le_host 136 #define sve_st1sq_be_tlb sve_st1sd_be_tlb 137 #define sve_st1sq_le_tlb sve_st1sd_le_tlb 138 139 #define sve_st1dq_be_host sve_st1dd_be_host 140 #define sve_st1dq_le_host sve_st1dd_le_host 141 #define sve_st1dq_be_tlb sve_st1dd_be_tlb 142 #define sve_st1dq_le_tlb sve_st1dd_le_tlb 143 144 /* 145 * The ARMVectorReg elements are stored in host-endian 64-bit units. 146 * For 128-bit quantities, the sequence defined by the Elem[] pseudocode 147 * corresponds to storing the two 64-bit pieces in little-endian order. 148 */ 149 /* FIXME: Nothing in this file makes any effort at atomicity. */ 150 151 static inline void sve_ld1qq_be_host(void *vd, intptr_t reg_off, void *host) 152 { 153 sve_ld1dd_be_host(vd, reg_off + 8, host); 154 sve_ld1dd_be_host(vd, reg_off, host + 8); 155 } 156 157 static inline void sve_ld1qq_le_host(void *vd, intptr_t reg_off, void *host) 158 { 159 sve_ld1dd_le_host(vd, reg_off, host); 160 sve_ld1dd_le_host(vd, reg_off + 8, host + 8); 161 } 162 163 static inline void 164 sve_ld1qq_be_tlb(CPUARMState *env, void *vd, intptr_t reg_off, 165 target_ulong addr, uintptr_t ra) 166 { 167 sve_ld1dd_be_tlb(env, vd, reg_off + 8, addr, ra); 168 sve_ld1dd_be_tlb(env, vd, reg_off, addr + 8, ra); 169 } 170 171 static inline void 172 sve_ld1qq_le_tlb(CPUARMState *env, void *vd, intptr_t reg_off, 173 target_ulong addr, uintptr_t ra) 174 { 175 sve_ld1dd_le_tlb(env, vd, reg_off, addr, ra); 176 sve_ld1dd_le_tlb(env, vd, reg_off + 8, addr + 8, ra); 177 } 178 179 static inline void sve_st1qq_be_host(void *vd, intptr_t reg_off, void *host) 180 { 181 sve_st1dd_be_host(vd, reg_off + 8, host); 182 sve_st1dd_be_host(vd, reg_off, host + 8); 183 } 184 185 static inline void sve_st1qq_le_host(void *vd, intptr_t reg_off, void *host) 186 { 187 sve_st1dd_le_host(vd, reg_off, host); 188 sve_st1dd_le_host(vd, reg_off + 8, host + 8); 189 } 190 191 static inline void 192 sve_st1qq_be_tlb(CPUARMState *env, void *vd, intptr_t reg_off, 193 target_ulong addr, uintptr_t ra) 194 { 195 sve_st1dd_be_tlb(env, vd, reg_off + 8, addr, ra); 196 sve_st1dd_be_tlb(env, vd, reg_off, addr + 8, ra); 197 } 198 199 static inline void 200 sve_st1qq_le_tlb(CPUARMState *env, void *vd, intptr_t reg_off, 201 target_ulong addr, uintptr_t ra) 202 { 203 sve_st1dd_le_tlb(env, vd, reg_off, addr, ra); 204 sve_st1dd_le_tlb(env, vd, reg_off + 8, addr + 8, ra); 205 } 206 207 #undef DO_LD_TLB 208 #undef DO_ST_TLB 209 #undef DO_LD_HOST 210 #undef DO_LD_PRIM_1 211 #undef DO_ST_PRIM_1 212 #undef DO_LD_PRIM_2 213 #undef DO_ST_PRIM_2 214 #undef DO_LD_PRIM_3 215 216 /* 217 * Resolve the guest virtual address to info->host and info->flags. 218 * If @nofault, return false if the page is invalid, otherwise 219 * exit via page fault exception. 220 */ 221 222 typedef struct { 223 void *host; 224 int flags; 225 MemTxAttrs attrs; 226 bool tagged; 227 } SVEHostPage; 228 229 bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env, 230 target_ulong addr, int mem_off, MMUAccessType access_type, 231 int mmu_idx, uintptr_t retaddr); 232 233 /* 234 * Analyse contiguous data, protected by a governing predicate. 235 */ 236 237 typedef enum { 238 FAULT_NO, 239 FAULT_FIRST, 240 FAULT_ALL, 241 } SVEContFault; 242 243 typedef struct { 244 /* 245 * First and last element wholly contained within the two pages. 246 * mem_off_first[0] and reg_off_first[0] are always set >= 0. 247 * reg_off_last[0] may be < 0 if the first element crosses pages. 248 * All of mem_off_first[1], reg_off_first[1] and reg_off_last[1] 249 * are set >= 0 only if there are complete elements on a second page. 250 * 251 * The reg_off_* offsets are relative to the internal vector register. 252 * The mem_off_first offset is relative to the memory address; the 253 * two offsets are different when a load operation extends, a store 254 * operation truncates, or for multi-register operations. 255 */ 256 int16_t mem_off_first[2]; 257 int16_t reg_off_first[2]; 258 int16_t reg_off_last[2]; 259 260 /* 261 * One element that is misaligned and spans both pages, 262 * or -1 if there is no such active element. 263 */ 264 int16_t mem_off_split; 265 int16_t reg_off_split; 266 267 /* 268 * The byte offset at which the entire operation crosses a page boundary. 269 * Set >= 0 if and only if the entire operation spans two pages. 270 */ 271 int16_t page_split; 272 273 /* TLB data for the two pages. */ 274 SVEHostPage page[2]; 275 } SVEContLdSt; 276 277 /* 278 * Find first active element on each page, and a loose bound for the 279 * final element on each page. Identify any single element that spans 280 * the page boundary. Return true if there are any active elements. 281 */ 282 bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg, 283 intptr_t reg_max, int esz, int msize); 284 285 /* 286 * Resolve the guest virtual addresses to info->page[]. 287 * Control the generation of page faults with @fault. Return false if 288 * there is no work to do, which can only happen with @fault == FAULT_NO. 289 */ 290 bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, 291 CPUARMState *env, target_ulong addr, 292 MMUAccessType access_type, uintptr_t retaddr); 293 294 #ifdef CONFIG_USER_ONLY 295 static inline void 296 sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, 297 target_ulong addr, int esize, int msize, 298 int wp_access, uintptr_t retaddr) 299 { } 300 #else 301 void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, 302 uint64_t *vg, target_ulong addr, 303 int esize, int msize, int wp_access, 304 uintptr_t retaddr); 305 #endif 306 307 void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, 308 target_ulong addr, int esize, int msize, 309 uint32_t mtedesc, uintptr_t ra); 310 311 #endif /* TARGET_ARM_SVE_LDST_INTERNAL_H */ 312