1 /* 2 * Atomic helper templates 3 * Included from tcg-runtime.c and cputlb.c. 4 * 5 * Copyright (c) 2016 Red Hat, Inc 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #if DATA_SIZE == 16 22 # define SUFFIX o 23 # define DATA_TYPE Int128 24 # define BSWAP bswap128 25 #elif DATA_SIZE == 8 26 # define SUFFIX q 27 # define DATA_TYPE uint64_t 28 # define BSWAP bswap64 29 #elif DATA_SIZE == 4 30 # define SUFFIX l 31 # define DATA_TYPE uint32_t 32 # define BSWAP bswap32 33 #elif DATA_SIZE == 2 34 # define SUFFIX w 35 # define DATA_TYPE uint16_t 36 # define BSWAP bswap16 37 #elif DATA_SIZE == 1 38 # define SUFFIX b 39 # define DATA_TYPE uint8_t 40 # define BSWAP 41 #else 42 # error unsupported data size 43 #endif 44 45 #if DATA_SIZE >= 4 46 # define ABI_TYPE DATA_TYPE 47 #else 48 # define ABI_TYPE uint32_t 49 #endif 50 51 /* Define host-endian atomic operations. Note that END is used within 52 the ATOMIC_NAME macro, and redefined below. */ 53 #if DATA_SIZE == 1 54 # define END 55 #elif defined(HOST_WORDS_BIGENDIAN) 56 # define END _be 57 #else 58 # define END _le 59 #endif 60 61 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 62 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 63 { 64 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 65 DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv); 66 ATOMIC_MMU_CLEANUP; 67 return ret; 68 } 69 70 #if DATA_SIZE >= 16 71 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 72 { 73 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 74 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 75 ATOMIC_MMU_CLEANUP; 76 return val; 77 } 78 79 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 80 ABI_TYPE val EXTRA_ARGS) 81 { 82 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 83 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 84 ATOMIC_MMU_CLEANUP; 85 } 86 #else 87 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 88 ABI_TYPE val EXTRA_ARGS) 89 { 90 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 91 DATA_TYPE ret = atomic_xchg__nocheck(haddr, val); 92 ATOMIC_MMU_CLEANUP; 93 return ret; 94 } 95 96 #define GEN_ATOMIC_HELPER(X) \ 97 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 98 ABI_TYPE val EXTRA_ARGS) \ 99 { \ 100 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 101 DATA_TYPE ret = atomic_##X(haddr, val); \ 102 ATOMIC_MMU_CLEANUP; \ 103 return ret; \ 104 } 105 106 GEN_ATOMIC_HELPER(fetch_add) 107 GEN_ATOMIC_HELPER(fetch_and) 108 GEN_ATOMIC_HELPER(fetch_or) 109 GEN_ATOMIC_HELPER(fetch_xor) 110 GEN_ATOMIC_HELPER(add_fetch) 111 GEN_ATOMIC_HELPER(and_fetch) 112 GEN_ATOMIC_HELPER(or_fetch) 113 GEN_ATOMIC_HELPER(xor_fetch) 114 115 #undef GEN_ATOMIC_HELPER 116 #endif /* DATA SIZE >= 16 */ 117 118 #undef END 119 120 #if DATA_SIZE > 1 121 122 /* Define reverse-host-endian atomic operations. Note that END is used 123 within the ATOMIC_NAME macro. */ 124 #ifdef HOST_WORDS_BIGENDIAN 125 # define END _le 126 #else 127 # define END _be 128 #endif 129 130 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr, 131 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS) 132 { 133 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 134 DATA_TYPE ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)); 135 ATOMIC_MMU_CLEANUP; 136 return BSWAP(ret); 137 } 138 139 #if DATA_SIZE >= 16 140 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS) 141 { 142 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP; 143 __atomic_load(haddr, &val, __ATOMIC_RELAXED); 144 ATOMIC_MMU_CLEANUP; 145 return BSWAP(val); 146 } 147 148 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr, 149 ABI_TYPE val EXTRA_ARGS) 150 { 151 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 152 val = BSWAP(val); 153 __atomic_store(haddr, &val, __ATOMIC_RELAXED); 154 ATOMIC_MMU_CLEANUP; 155 } 156 #else 157 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr, 158 ABI_TYPE val EXTRA_ARGS) 159 { 160 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 161 ABI_TYPE ret = atomic_xchg__nocheck(haddr, BSWAP(val)); 162 ATOMIC_MMU_CLEANUP; 163 return BSWAP(ret); 164 } 165 166 #define GEN_ATOMIC_HELPER(X) \ 167 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \ 168 ABI_TYPE val EXTRA_ARGS) \ 169 { \ 170 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \ 171 DATA_TYPE ret = atomic_##X(haddr, BSWAP(val)); \ 172 ATOMIC_MMU_CLEANUP; \ 173 return BSWAP(ret); \ 174 } 175 176 GEN_ATOMIC_HELPER(fetch_and) 177 GEN_ATOMIC_HELPER(fetch_or) 178 GEN_ATOMIC_HELPER(fetch_xor) 179 GEN_ATOMIC_HELPER(and_fetch) 180 GEN_ATOMIC_HELPER(or_fetch) 181 GEN_ATOMIC_HELPER(xor_fetch) 182 183 #undef GEN_ATOMIC_HELPER 184 185 /* Note that for addition, we need to use a separate cmpxchg loop instead 186 of bswaps for the reverse-host-endian helpers. */ 187 ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr, 188 ABI_TYPE val EXTRA_ARGS) 189 { 190 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 191 DATA_TYPE ldo, ldn, ret, sto; 192 193 ldo = atomic_read__nocheck(haddr); 194 while (1) { 195 ret = BSWAP(ldo); 196 sto = BSWAP(ret + val); 197 ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto); 198 if (ldn == ldo) { 199 ATOMIC_MMU_CLEANUP; 200 return ret; 201 } 202 ldo = ldn; 203 } 204 } 205 206 ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr, 207 ABI_TYPE val EXTRA_ARGS) 208 { 209 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; 210 DATA_TYPE ldo, ldn, ret, sto; 211 212 ldo = atomic_read__nocheck(haddr); 213 while (1) { 214 ret = BSWAP(ldo) + val; 215 sto = BSWAP(ret); 216 ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto); 217 if (ldn == ldo) { 218 ATOMIC_MMU_CLEANUP; 219 return ret; 220 } 221 ldo = ldn; 222 } 223 } 224 #endif /* DATA_SIZE >= 16 */ 225 226 #undef END 227 #endif /* DATA_SIZE > 1 */ 228 229 #undef BSWAP 230 #undef ABI_TYPE 231 #undef DATA_TYPE 232 #undef SUFFIX 233 #undef DATA_SIZE 234