1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020 Google LLC. 4 */ 5 #ifndef __ASM_RWONCE_H 6 #define __ASM_RWONCE_H 7 8 #if defined(CONFIG_LTO) && !defined(__ASSEMBLY__) 9 10 #include <linux/compiler_types.h> 11 #include <asm/alternative-macros.h> 12 13 #ifndef BUILD_VDSO 14 15 #ifdef CONFIG_AS_HAS_LDAPR 16 #define __LOAD_RCPC(sfx, regs...) \ 17 ALTERNATIVE( \ 18 "ldar" #sfx "\t" #regs, \ 19 ".arch_extension rcpc\n" \ 20 "ldapr" #sfx "\t" #regs, \ 21 ARM64_HAS_LDAPR) 22 #else 23 #define __LOAD_RCPC(sfx, regs...) "ldar" #sfx "\t" #regs 24 #endif /* CONFIG_AS_HAS_LDAPR */ 25 26 /* 27 * When building with LTO, there is an increased risk of the compiler 28 * converting an address dependency headed by a READ_ONCE() invocation 29 * into a control dependency and consequently allowing for harmful 30 * reordering by the CPU. 31 * 32 * Ensure that such transformations are harmless by overriding the generic 33 * READ_ONCE() definition with one that provides RCpc acquire semantics 34 * when building with LTO. 35 */ 36 #define __READ_ONCE(x) \ 37 ({ \ 38 typeof(&(x)) __x = &(x); \ 39 int atomic = 1; \ 40 union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \ 41 switch (sizeof(x)) { \ 42 case 1: \ 43 asm volatile(__LOAD_RCPC(b, %w0, %1) \ 44 : "=r" (*(__u8 *)__u.__c) \ 45 : "Q" (*__x) : "memory"); \ 46 break; \ 47 case 2: \ 48 asm volatile(__LOAD_RCPC(h, %w0, %1) \ 49 : "=r" (*(__u16 *)__u.__c) \ 50 : "Q" (*__x) : "memory"); \ 51 break; \ 52 case 4: \ 53 asm volatile(__LOAD_RCPC(, %w0, %1) \ 54 : "=r" (*(__u32 *)__u.__c) \ 55 : "Q" (*__x) : "memory"); \ 56 break; \ 57 case 8: \ 58 asm volatile(__LOAD_RCPC(, %0, %1) \ 59 : "=r" (*(__u64 *)__u.__c) \ 60 : "Q" (*__x) : "memory"); \ 61 break; \ 62 default: \ 63 atomic = 0; \ 64 } \ 65 atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ 66 }) 67 68 #endif /* !BUILD_VDSO */ 69 #endif /* CONFIG_LTO && !__ASSEMBLY__ */ 70 71 #include <asm-generic/rwonce.h> 72 73 #endif /* __ASM_RWONCE_H */ 74