1/* 2 * SPDX-License-Identifier: GPL-2.0-or-later 3 * Load/store for 128-bit atomic operations, AArch64 version. 4 * 5 * Copyright (C) 2018, 2023 Linaro, Ltd. 6 * 7 * See docs/devel/atomics.rst for discussion about the guarantees each 8 * atomic primitive is meant to provide. 9 */ 10 11#ifndef AARCH64_ATOMIC128_LDST_H 12#define AARCH64_ATOMIC128_LDST_H 13 14#include "host/cpuinfo.h" 15#include "tcg/debug-assert.h" 16 17/* 18 * Through gcc 10, aarch64 has no support for 128-bit atomics. 19 * Through clang 16, without -march=armv8.4-a, __atomic_load_16 20 * is incorrectly expanded to a read-write operation. 21 * 22 * Anyway, this method allows runtime detection of FEAT_LSE2. 23 */ 24 25#define HAVE_ATOMIC128_RO (cpuinfo & CPUINFO_LSE2) 26#define HAVE_ATOMIC128_RW 1 27 28static inline Int128 atomic16_read_ro(const Int128 *ptr) 29{ 30 uint64_t l, h; 31 32 tcg_debug_assert(HAVE_ATOMIC128_RO); 33 /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */ 34 asm("ldp %[l], %[h], %[mem]" 35 : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr)); 36 37 return int128_make128(l, h); 38} 39 40static inline Int128 atomic16_read_rw(Int128 *ptr) 41{ 42 uint64_t l, h; 43 uint32_t tmp; 44 45 if (cpuinfo & CPUINFO_LSE2) { 46 /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */ 47 asm("ldp %[l], %[h], %[mem]" 48 : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr)); 49 } else { 50 /* The load must be paired with the store to guarantee not tearing. */ 51 asm("0: ldxp %[l], %[h], %[mem]\n\t" 52 "stxp %w[tmp], %[l], %[h], %[mem]\n\t" 53 "cbnz %w[tmp], 0b" 54 : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), [l] "=&r"(l), [h] "=&r"(h)); 55 } 56 57 return int128_make128(l, h); 58} 59 60static inline void atomic16_set(Int128 *ptr, Int128 val) 61{ 62 uint64_t l = int128_getlo(val), h = int128_gethi(val); 63 uint64_t t1, t2; 64 65 if (cpuinfo & CPUINFO_LSE2) { 66 /* With FEAT_LSE2, 16-byte aligned STP is atomic. */ 67 asm("stp %[l], %[h], %[mem]" 68 : [mem] "=m"(*ptr) : [l] "r"(l), [h] "r"(h)); 69 } else { 70 /* Load into temporaries to acquire the exclusive access lock. */ 71 asm("0: ldxp %[t1], %[t2], %[mem]\n\t" 72 "stxp %w[t1], %[l], %[h], %[mem]\n\t" 73 "cbnz %w[t1], 0b" 74 : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2) 75 : [l] "r"(l), [h] "r"(h)); 76 } 77} 78 79#endif /* AARCH64_ATOMIC128_LDST_H */ 80