1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #if DATA_SIZE == 16 22 # define SUFFIX o 23 # define DATA_TYPE Int128 24 # define BSWAP bswap128 25 #elif DATA_SIZE == 8 26 # define SUFFIX q 27 # define DATA_TYPE uint64_t 28 # define BSWAP bswap64 29 #elif DATA_SIZE == 4 30 # define SUFFIX l 31 # define DATA_TYPE uint32_t 32 # define BSWAP bswap32 33 #elif DATA_SIZE == 2 34 # define SUFFIX w 35 # define DATA_TYPE uint16_t 36 # define BSWAP bswap16 37 #elif DATA_SIZE == 1 38 # define SUFFIX b 39 # define DATA_TYPE uint8_t 40 # define BSWAP 41 #else 42 # error unsupported data size 43 #endif 44 45 #if DATA_SIZE >= 4 46 # define ABI_TYPE DATA_TYPE 47 #else 48 # define ABI_TYPE uint32_t 49 #endif 50 51 /* Define host-endian atomic operations. Note that END is used within 52 the ATOMIC_NAME macro, and redefined below. */ 53 #if DATA_SIZE == 1 54 # define END 55 #elif defined(HOST_WORDS_BIGENDIAN) 56 # define END _be 57 #else 58 # define END _le 59 #endif 60 61 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 62 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 63 { 64 ATOMIC_MMU_DECLS; 65 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 66 DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); 67 ATOMIC_MMU_CLEANUP; 68 return ret; 69 } 70 71 #if DATA_SIZE >= 16 72 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 73 { 74 ATOMIC_MMU_DECLS; 75 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 76 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 77 ATOMIC_MMU_CLEANUP; 78 return val; 79 } 80 81 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 82 ABI_TYPE val EXTRA_ARGS) 83 { 84 ATOMIC_MMU_DECLS; 85 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 86 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 87 ATOMIC_MMU_CLEANUP; 88 } 89 #else 90 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 91 ABI_TYPE val EXTRA_ARGS) 92 { 93 ATOMIC_MMU_DECLS; 94 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 95 DATA_TYPE ret = atomic_xchg__nocheck(haddr, val); 96 ATOMIC_MMU_CLEANUP; 97 return ret; 98 } 99 100 #define GEN_ATOMIC_HELPER(X) \ 101 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 102 ABI_TYPE val EXTRA_ARGS) \ 103 { \ 104 ATOMIC_MMU_DECLS; \ 105 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 106 DATA_TYPE ret = atomic_##X(haddr, val); \ 107 ATOMIC_MMU_CLEANUP; \ 108 return ret; \ 109 } 110 111 GEN_ATOMIC_HELPER(fetch_add) 112 GEN_ATOMIC_HELPER(fetch_and) 113 GEN_ATOMIC_HELPER(fetch_or) 114 GEN_ATOMIC_HELPER(fetch_xor) 115 GEN_ATOMIC_HELPER(add_fetch) 116 GEN_ATOMIC_HELPER(and_fetch) 117 GEN_ATOMIC_HELPER(or_fetch) 118 GEN_ATOMIC_HELPER(xor_fetch) 119 120 #undef GEN_ATOMIC_HELPER 121 #endif /* DATA SIZE >= 16 */ 122 123 #undef END 124 125 #if DATA_SIZE > 1 126 127 /* Define reverse-host-endian atomic operations. Note that END is used 128 within the ATOMIC_NAME macro. */ 129 #ifdef HOST_WORDS_BIGENDIAN 130 # define END _le 131 #else 132 # define END _be 133 #endif 134 135 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 136 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 137 { 138 ATOMIC_MMU_DECLS; 139 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 140 DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 141 ATOMIC_MMU_CLEANUP; 142 return BSWAP(ret); 143 } 144 145 #if DATA_SIZE >= 16 146 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 147 { 148 ATOMIC_MMU_DECLS; 149 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 150 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 151 ATOMIC_MMU_CLEANUP; 152 return BSWAP(val); 153 } 154 155 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 156 ABI_TYPE val EXTRA_ARGS) 157 { 158 ATOMIC_MMU_DECLS; 159 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 160 val = BSWAP(val); 161 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 162 ATOMIC_MMU_CLEANUP; 163 } 164 #else 165 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 166 ABI_TYPE val EXTRA_ARGS) 167 { 168 ATOMIC_MMU_DECLS; 169 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 170 ABI_TYPE ret = atomic_xchg__nocheck(haddr, BSWAP(val)); 171 ATOMIC_MMU_CLEANUP; 172 return BSWAP(ret); 173 } 174 175 #define GEN_ATOMIC_HELPER(X) \ 176 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 177 ABI_TYPE val EXTRA_ARGS) \ 178 { \ 179 ATOMIC_MMU_DECLS; \ 180 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 181 DATA_TYPE ret = atomic_##X(haddr, BSWAP(val)); \ 182 ATOMIC_MMU_CLEANUP; \ 183 return BSWAP(ret); \ 184 } 185 186 GEN_ATOMIC_HELPER(fetch_and) 187 GEN_ATOMIC_HELPER(fetch_or) 188 GEN_ATOMIC_HELPER(fetch_xor) 189 GEN_ATOMIC_HELPER(and_fetch) 190 GEN_ATOMIC_HELPER(or_fetch) 191 GEN_ATOMIC_HELPER(xor_fetch) 192 193 #undef GEN_ATOMIC_HELPER 194 195 /* Note that for addition, we need to use a separate cmpxchg loop instead 196 of bswaps for the reverse-host-endian helpers. */ 197 ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr, 198 ABI_TYPE val EXTRA_ARGS) 199 { 200 ATOMIC_MMU_DECLS; 201 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 202 DATA_TYPE ldo, ldn, ret, sto; 203 204 ldo = atomic_read__nocheck(haddr); 205 while (1) { 206 ret = BSWAP(ldo); 207 sto = BSWAP(ret + val); 208 ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto); 209 if (ldn == ldo) { 210 ATOMIC_MMU_CLEANUP; 211 return ret; 212 } 213 ldo = ldn; 214 } 215 } 216 217 ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr, 218 ABI_TYPE val EXTRA_ARGS) 219 { 220 ATOMIC_MMU_DECLS; 221 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 222 DATA_TYPE ldo, ldn, ret, sto; 223 224 ldo = atomic_read__nocheck(haddr); 225 while (1) { 226 ret = BSWAP(ldo) + val; 227 sto = BSWAP(ret); 228 ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto); 229 if (ldn == ldo) { 230 ATOMIC_MMU_CLEANUP; 231 return ret; 232 } 233 ldo = ldn; 234 } 235 } 236 #endif /* DATA_SIZE >= 16 */ 237 238 #undef END 239 #endif /* DATA_SIZE > 1 */ 240 241 #undef BSWAP 242 #undef ABI_TYPE 243 #undef DATA_TYPE 244 #undef SUFFIX 245 #undef DATA_SIZE 246