1*883cc6c5SPhilippe Mathieu-Daudé/* 2*883cc6c5SPhilippe Mathieu-Daudé * SPDX-License-Identifier: GPL-2.0-or-later 3*883cc6c5SPhilippe Mathieu-Daudé * Load/store for 128-bit atomic operations, generic version. 4*883cc6c5SPhilippe Mathieu-Daudé * 5*883cc6c5SPhilippe Mathieu-Daudé * Copyright (C) 2018, 2023 Linaro, Ltd. 6*883cc6c5SPhilippe Mathieu-Daudé * 7*883cc6c5SPhilippe Mathieu-Daudé * See docs/devel/atomics.rst for discussion about the guarantees each 8*883cc6c5SPhilippe Mathieu-Daudé * atomic primitive is meant to provide. 9*883cc6c5SPhilippe Mathieu-Daudé */ 10*883cc6c5SPhilippe Mathieu-Daudé 11*883cc6c5SPhilippe Mathieu-Daudé#ifndef HOST_ATOMIC128_LDST_H 12*883cc6c5SPhilippe Mathieu-Daudé#define HOST_ATOMIC128_LDST_H 13*883cc6c5SPhilippe Mathieu-Daudé 14*883cc6c5SPhilippe Mathieu-Daudé#if defined(CONFIG_ATOMIC128) 15*883cc6c5SPhilippe Mathieu-Daudé# define HAVE_ATOMIC128_RO 1 16*883cc6c5SPhilippe Mathieu-Daudé# define HAVE_ATOMIC128_RW 1 17*883cc6c5SPhilippe Mathieu-Daudé 18*883cc6c5SPhilippe Mathieu-Daudéstatic inline Int128 ATTRIBUTE_ATOMIC128_OPT 19*883cc6c5SPhilippe Mathieu-Daudéatomic16_read_ro(const Int128 *ptr) 20*883cc6c5SPhilippe Mathieu-Daudé{ 21*883cc6c5SPhilippe Mathieu-Daudé const __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16); 22*883cc6c5SPhilippe Mathieu-Daudé Int128Alias r; 23*883cc6c5SPhilippe Mathieu-Daudé 24*883cc6c5SPhilippe Mathieu-Daudé r.i = qatomic_read__nocheck(ptr_align); 25*883cc6c5SPhilippe Mathieu-Daudé return r.s; 26*883cc6c5SPhilippe Mathieu-Daudé} 27*883cc6c5SPhilippe Mathieu-Daudé 28*883cc6c5SPhilippe Mathieu-Daudéstatic inline Int128 ATTRIBUTE_ATOMIC128_OPT 29*883cc6c5SPhilippe Mathieu-Daudéatomic16_read_rw(Int128 *ptr) 30*883cc6c5SPhilippe Mathieu-Daudé{ 31*883cc6c5SPhilippe Mathieu-Daudé return atomic16_read_ro(ptr); 32*883cc6c5SPhilippe Mathieu-Daudé} 33*883cc6c5SPhilippe Mathieu-Daudé 34*883cc6c5SPhilippe Mathieu-Daudéstatic inline void ATTRIBUTE_ATOMIC128_OPT 35*883cc6c5SPhilippe Mathieu-Daudéatomic16_set(Int128 *ptr, Int128 val) 36*883cc6c5SPhilippe Mathieu-Daudé{ 37*883cc6c5SPhilippe Mathieu-Daudé __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16); 38*883cc6c5SPhilippe Mathieu-Daudé Int128Alias v; 39*883cc6c5SPhilippe Mathieu-Daudé 40*883cc6c5SPhilippe Mathieu-Daudé v.s = val; 41*883cc6c5SPhilippe Mathieu-Daudé qatomic_set__nocheck(ptr_align, v.i); 42*883cc6c5SPhilippe Mathieu-Daudé} 43*883cc6c5SPhilippe Mathieu-Daudé 44*883cc6c5SPhilippe Mathieu-Daudé#elif defined(CONFIG_CMPXCHG128) 45*883cc6c5SPhilippe Mathieu-Daudé# define HAVE_ATOMIC128_RO 0 46*883cc6c5SPhilippe Mathieu-Daudé# define HAVE_ATOMIC128_RW 1 47*883cc6c5SPhilippe Mathieu-Daudé 48*883cc6c5SPhilippe Mathieu-DaudéInt128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr); 49*883cc6c5SPhilippe Mathieu-Daudé 50*883cc6c5SPhilippe Mathieu-Daudéstatic inline Int128 ATTRIBUTE_ATOMIC128_OPT 51*883cc6c5SPhilippe Mathieu-Daudéatomic16_read_rw(Int128 *ptr) 52*883cc6c5SPhilippe Mathieu-Daudé{ 53*883cc6c5SPhilippe Mathieu-Daudé /* Maybe replace 0 with 0, returning the old value. */ 54*883cc6c5SPhilippe Mathieu-Daudé Int128 z = int128_make64(0); 55*883cc6c5SPhilippe Mathieu-Daudé return atomic16_cmpxchg(ptr, z, z); 56*883cc6c5SPhilippe Mathieu-Daudé} 57*883cc6c5SPhilippe Mathieu-Daudé 58*883cc6c5SPhilippe Mathieu-Daudéstatic inline void ATTRIBUTE_ATOMIC128_OPT 59*883cc6c5SPhilippe Mathieu-Daudéatomic16_set(Int128 *ptr, Int128 val) 60*883cc6c5SPhilippe Mathieu-Daudé{ 61*883cc6c5SPhilippe Mathieu-Daudé Int128Aligned *ptr_align = __builtin_assume_aligned(ptr, 16); 62*883cc6c5SPhilippe Mathieu-Daudé __int128_t old; 63*883cc6c5SPhilippe Mathieu-Daudé Int128Alias new; 64*883cc6c5SPhilippe Mathieu-Daudé 65*883cc6c5SPhilippe Mathieu-Daudé new.s = val; 66*883cc6c5SPhilippe Mathieu-Daudé do { 67*883cc6c5SPhilippe Mathieu-Daudé old = *ptr_align; 68*883cc6c5SPhilippe Mathieu-Daudé } while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i)); 69*883cc6c5SPhilippe Mathieu-Daudé} 70*883cc6c5SPhilippe Mathieu-Daudé 71*883cc6c5SPhilippe Mathieu-Daudé#else 72*883cc6c5SPhilippe Mathieu-Daudé# define HAVE_ATOMIC128_RO 0 73*883cc6c5SPhilippe Mathieu-Daudé# define HAVE_ATOMIC128_RW 0 74*883cc6c5SPhilippe Mathieu-Daudé 75*883cc6c5SPhilippe Mathieu-Daudé/* Fallback definitions that must be optimized away, or error. */ 76*883cc6c5SPhilippe Mathieu-DaudéInt128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr); 77*883cc6c5SPhilippe Mathieu-DaudéInt128 QEMU_ERROR("unsupported atomic") atomic16_read_rw(Int128 *ptr); 78*883cc6c5SPhilippe Mathieu-Daudévoid QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val); 79*883cc6c5SPhilippe Mathieu-Daudé#endif 80*883cc6c5SPhilippe Mathieu-Daudé 81*883cc6c5SPhilippe Mathieu-Daudé#endif /* HOST_ATOMIC128_LDST_H */ 82