1 #ifndef _ASM_X86_BARRIER_H 2 #define _ASM_X86_BARRIER_H 3 4 #include <asm/alternative.h> 5 #include <asm/nops.h> 6 7 /* 8 * Force strict CPU ordering. 9 * And yes, this might be required on UP too when we're talking 10 * to devices. 11 */ 12 13 #ifdef CONFIG_X86_32 14 #define mb() asm volatile(ALTERNATIVE("lock; addl $0,0(%%esp)", "mfence", \ 15 X86_FEATURE_XMM2) ::: "memory", "cc") 16 #define rmb() asm volatile(ALTERNATIVE("lock; addl $0,0(%%esp)", "lfence", \ 17 X86_FEATURE_XMM2) ::: "memory", "cc") 18 #define wmb() asm volatile(ALTERNATIVE("lock; addl $0,0(%%esp)", "sfence", \ 19 X86_FEATURE_XMM2) ::: "memory", "cc") 20 #else 21 #define mb() asm volatile("mfence":::"memory") 22 #define rmb() asm volatile("lfence":::"memory") 23 #define wmb() asm volatile("sfence" ::: "memory") 24 #endif 25 26 #ifdef CONFIG_X86_PPRO_FENCE 27 #define dma_rmb() rmb() 28 #else 29 #define dma_rmb() barrier() 30 #endif 31 #define dma_wmb() barrier() 32 33 #define __smp_mb() mb() 34 #define __smp_rmb() dma_rmb() 35 #define __smp_wmb() barrier() 36 #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) 37 38 #if defined(CONFIG_X86_PPRO_FENCE) 39 40 /* 41 * For this option x86 doesn't have a strong TSO memory 42 * model and we should fall back to full barriers. 43 */ 44 45 #define __smp_store_release(p, v) \ 46 do { \ 47 compiletime_assert_atomic_type(*p); \ 48 __smp_mb(); \ 49 WRITE_ONCE(*p, v); \ 50 } while (0) 51 52 #define __smp_load_acquire(p) \ 53 ({ \ 54 typeof(*p) ___p1 = READ_ONCE(*p); \ 55 compiletime_assert_atomic_type(*p); \ 56 __smp_mb(); \ 57 ___p1; \ 58 }) 59 60 #else /* regular x86 TSO memory ordering */ 61 62 #define __smp_store_release(p, v) \ 63 do { \ 64 compiletime_assert_atomic_type(*p); \ 65 barrier(); \ 66 WRITE_ONCE(*p, v); \ 67 } while (0) 68 69 #define __smp_load_acquire(p) \ 70 ({ \ 71 typeof(*p) ___p1 = READ_ONCE(*p); \ 72 compiletime_assert_atomic_type(*p); \ 73 barrier(); \ 74 ___p1; \ 75 }) 76 77 #endif 78 79 /* Atomic operations are already serializing on x86 */ 80 #define __smp_mb__before_atomic() barrier() 81 #define __smp_mb__after_atomic() barrier() 82 83 #include <asm-generic/barrier.h> 84 85 #endif /* _ASM_X86_BARRIER_H */ 86