1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_COOKIE_H 3 #define __LINUX_COOKIE_H 4 5 #include <linux/atomic.h> 6 #include <linux/percpu.h> 7 #include <asm/local.h> 8 9 struct pcpu_gen_cookie { 10 local_t nesting; 11 u64 last; 12 } __aligned(16); 13 14 struct gen_cookie { 15 struct pcpu_gen_cookie __percpu *local; 16 atomic64_t forward_last ____cacheline_aligned_in_smp; 17 atomic64_t reverse_last; 18 }; 19 20 #define COOKIE_LOCAL_BATCH 4096 21 22 #define DEFINE_COOKIE(name) \ 23 static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name); \ 24 static struct gen_cookie name = { \ 25 .local = &__##name, \ 26 .forward_last = ATOMIC64_INIT(0), \ 27 .reverse_last = ATOMIC64_INIT(0), \ 28 } 29 30 static __always_inline u64 gen_cookie_next(struct gen_cookie *gc) 31 { 32 struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local); 33 u64 val; 34 35 if (likely(local_inc_return(&local->nesting) == 1)) { 36 val = local->last; 37 if (__is_defined(CONFIG_SMP) && 38 unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) { 39 s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH, 40 &gc->forward_last); 41 val = next - COOKIE_LOCAL_BATCH; 42 } 43 local->last = ++val; 44 } else { 45 val = atomic64_dec_return(&gc->reverse_last); 46 } 47 local_dec(&local->nesting); 48 return val; 49 } 50 51 #endif /* __LINUX_COOKIE_H */ 52