192acdc58SDaniel Borkmann /* SPDX-License-Identifier: GPL-2.0 */ 292acdc58SDaniel Borkmann #ifndef __LINUX_COOKIE_H 392acdc58SDaniel Borkmann #define __LINUX_COOKIE_H 492acdc58SDaniel Borkmann 592acdc58SDaniel Borkmann #include <linux/atomic.h> 692acdc58SDaniel Borkmann #include <linux/percpu.h> 792acdc58SDaniel Borkmann #include <asm/local.h> 892acdc58SDaniel Borkmann 992acdc58SDaniel Borkmann struct pcpu_gen_cookie { 1092acdc58SDaniel Borkmann local_t nesting; 1192acdc58SDaniel Borkmann u64 last; 1292acdc58SDaniel Borkmann } __aligned(16); 1392acdc58SDaniel Borkmann 1492acdc58SDaniel Borkmann struct gen_cookie { 1592acdc58SDaniel Borkmann struct pcpu_gen_cookie __percpu *local; 1692acdc58SDaniel Borkmann atomic64_t forward_last ____cacheline_aligned_in_smp; 1792acdc58SDaniel Borkmann atomic64_t reverse_last; 1892acdc58SDaniel Borkmann }; 1992acdc58SDaniel Borkmann 2092acdc58SDaniel Borkmann #define COOKIE_LOCAL_BATCH 4096 2192acdc58SDaniel Borkmann 2292acdc58SDaniel Borkmann #define DEFINE_COOKIE(name) \ 2392acdc58SDaniel Borkmann static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name); \ 2492acdc58SDaniel Borkmann static struct gen_cookie name = { \ 2592acdc58SDaniel Borkmann .local = &__##name, \ 2692acdc58SDaniel Borkmann .forward_last = ATOMIC64_INIT(0), \ 2792acdc58SDaniel Borkmann .reverse_last = ATOMIC64_INIT(0), \ 2892acdc58SDaniel Borkmann } 2992acdc58SDaniel Borkmann gen_cookie_next(struct gen_cookie * gc)3092acdc58SDaniel Borkmannstatic __always_inline u64 gen_cookie_next(struct gen_cookie *gc) 3192acdc58SDaniel Borkmann { 3292acdc58SDaniel Borkmann struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local); 3392acdc58SDaniel Borkmann u64 val; 3492acdc58SDaniel Borkmann 3592acdc58SDaniel Borkmann if (likely(local_inc_return(&local->nesting) == 1)) { 3692acdc58SDaniel Borkmann val = local->last; 3792acdc58SDaniel Borkmann if (__is_defined(CONFIG_SMP) && 3892acdc58SDaniel Borkmann unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) { 3992acdc58SDaniel Borkmann s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH, 4092acdc58SDaniel Borkmann &gc->forward_last); 4192acdc58SDaniel Borkmann val = next - COOKIE_LOCAL_BATCH; 4292acdc58SDaniel Borkmann } 4392acdc58SDaniel Borkmann local->last = ++val; 4492acdc58SDaniel Borkmann } else { 4592acdc58SDaniel Borkmann val = atomic64_dec_return(&gc->reverse_last); 4692acdc58SDaniel Borkmann } 4792acdc58SDaniel Borkmann local_dec(&local->nesting); 4892acdc58SDaniel Borkmann return val; 4992acdc58SDaniel Borkmann } 5092acdc58SDaniel Borkmann 5192acdc58SDaniel Borkmann #endif /* __LINUX_COOKIE_H */ 52