accel/tcg/ldst_atomicity.c.inc

6  * SPDX-License-Identifier: GPL-2.0-or-later
9  * See the COPYING file in the top-level directory.
12 #include "host/load-extract-al16-al8.h.inc"
13 #include "host/store-insert-al16.h.inc"
27  * examined separately for atomicity, return -lg2.
33     MemOp half = size ? size - 1 : 0;
47         tmp = (1 << size) - 1;
63              * Both halves are naturally aligned and atomic.
68              * One of the pair crosses the boundary, and is non-atomic.
71             atmax = -half;
78          * that must be aligned.  Note that we only really need ctz4() --
104  * @pv: host address
106  * Atomically load 2 aligned bytes from @pv.
116  * @pv: host address
118  * Atomically load 4 aligned bytes from @pv.
128  * @pv: host address
130  * Atomically load 8 aligned bytes from @pv.
143  * @ra: host unwind address
144  * @pv: host address
146  * Atomically load 8 aligned bytes from @pv.
170     /* Ultimate fallback: re-execute in serial context. */
178  * @ra: host unwind address
179  * @pv: host address
181  * Atomically load 16 aligned bytes from @pv.
201      * the write is complete -- tests/tcg/multiarch/munmap-pthread.c
215     /* Ultimate fallback: re-execute in serial context. */
222  * @pv: host address
224  * Load 4 bytes from @p, from two sequential atomic 4-byte loads.
237         return (a << sh) | (b >> (-sh & 31));
239         return (a >> sh) | (b << (-sh & 31));
245  * @pv: host address
247  * Load 8 bytes from @p, from two sequential atomic 8-byte loads.
260         return (a << sh) | (b >> (-sh & 63));
262         return (a >> sh) | (b << (-sh & 63));
269  * @ra: host unwind address
270  * @pv: host address
273  * Atomically load @s bytes from @p, when p % s != 0, and [p, p+s-1] does
274  * not cross an 8-byte boundary.  This means that we can perform an atomic
275  * 8-byte load and extract.
283     int shr = (HOST_BIG_ENDIAN ? 8 - s - o : o) * 8;
292  * @ra: host unwind address
293  * @p: host address
297  * and p % 16 + s > 8.  I.e. does not cross a 16-byte
298  * boundary, but *does* cross an 8-byte boundary.
309     int shr = (HOST_BIG_ENDIAN ? 16 - s - o : o) * 8;
325  * @pv: host address
327  * Load 4 bytes from @pv, with two 2-byte atomic loads.
343  * @pv: host address
345  * Load 8 bytes from @pv, with four 2-byte atomic loads.
361  * @pv: host address
363  * Load 8 bytes from @pv, with two 4-byte atomic loads.
379  * @pv: host address
381  * Load 8 bytes from aligned @pv, with at least 4-byte atomicity.
394  * @p: host address
409         intptr_t left_in_page = -(pi | TARGET_PAGE_MASK);
423             return load_atomic4(pv - 1) >> 8;
436  * @p: host address
451         intptr_t left_in_page = -(pi | TARGET_PAGE_MASK);
461     case -MO_16:
481  * @p: host address
493      * If the host does not support 8-byte atomics, wait until we have
520     case -MO_32:
533  * @p: host address
547      * If the host does not support 16-byte atomics, wait until we have
575     case -MO_64:
593  * @pv: host address
596  * Atomically store 2 aligned bytes to @pv.
606  * @pv: host address
609  * Atomically store 4 aligned bytes to @pv.
619  * @pv: host address
622  * Atomically store 8 aligned bytes to @pv.
661  * @p: host address
681  * @p: host address
702  * @pv: host address
706  * Store @size bytes at @p.  The bytes to store are extracted in little-endian order
720  * @pv: host address
724  * As store_bytes_leN, but atomically on each aligned part.
754         size -= n;
762  * @pv: host address
767  * Four aligned bytes are guaranteed to cover the store.
784     store_atom_insert_al4(pv - o, v, m);
790  * @pv: host address
795  * Eight aligned bytes are guaranteed to cover the store.
813     store_atom_insert_al8(pv - o, v, m);
819  * @pv: host address
824  * 16 aligned bytes are guaranteed to cover the store.
839         m = int128_make128(-1, MAKE_64BIT_MASK(0, sz - 64));
849     store_atom_insert_al16(pv - o, v, m);
854     return int128_gethi(val_le) >> (sz - 64);
859  * @p: host address
887         store_atom_insert_al4(pv - 1, (uint32_t)val << 8, MAKE_64BIT_MASK(8, 16));
891             store_atom_insert_al8(pv - 3, (uint64_t)val << 24, MAKE_64BIT_MASK(24, 16));
898             store_atom_insert_al16(pv - 7, v, m);
911  * @p: host address
936     case -MO_16:
940             int s1 = 4 - s2;
951             case 0: /* aligned */
979  * @p: host address
1007     case -MO_32:
1011             int s1 = 8 - s2;
1022             case 0: /* aligned */
1045  * @p: host address
1086     case -MO_64:
1090             int s1 = 16 - s2;
1105             case 0: /* aligned */