2428 static inline unsigned long next_tid(unsigned long tid)  in next_tid()  argument
2430 	return tid + TID_STEP;  in next_tid()
2434 static inline unsigned int tid_to_cpu(unsigned long tid)  in tid_to_cpu()  argument
2436 	return tid % TID_STEP;  in tid_to_cpu()
2439 static inline unsigned long tid_to_event(unsigned long tid)  in tid_to_event()  argument
2441 	return tid / TID_STEP;  in tid_to_event()
2451 		const struct kmem_cache *s, unsigned long tid)  in note_cmpxchg_failure()  argument
2454 	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);  in note_cmpxchg_failure()
2459 	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))  in note_cmpxchg_failure()
2461 			tid_to_cpu(tid), tid_to_cpu(actual_tid));  in note_cmpxchg_failure()
2464 	if (tid_to_event(tid) != tid_to_event(actual_tid))  in note_cmpxchg_failure()
2466 			tid_to_event(tid), tid_to_event(actual_tid));  in note_cmpxchg_failure()
2469 			actual_tid, tid, next_tid(tid));  in note_cmpxchg_failure()
2482 		c->tid = init_tid(cpu);  in init_kmem_cache_cpus()
2757 	c->tid = next_tid(c->tid);  in flush_slab()
2775 	c->tid = next_tid(c->tid);  in __flush_cpu_slab()
3033 			   unsigned long tid)  in __update_cpu_freelist_fast()  argument
3035 	freelist_aba_t old = { .freelist = freelist_old, .counter = tid };  in __update_cpu_freelist_fast()
3036 	freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };  in __update_cpu_freelist_fast()
3155 		c->tid = next_tid(c->tid);  in ___slab_alloc()
3174 	c->tid = next_tid(c->tid);  in ___slab_alloc()
3188 	c->tid = next_tid(c->tid);  in ___slab_alloc()
3288 		c->tid = next_tid(c->tid);  in ___slab_alloc()
3334 	unsigned long tid;  in __slab_alloc_node()  local
3351 	tid = READ_ONCE(c->tid);  in __slab_alloc_node()
3393 		if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) {  in __slab_alloc_node()
3394 			note_cmpxchg_failure("slab_alloc", s, tid);  in __slab_alloc_node()
3740 	unsigned long tid;  in do_slab_free()  local
3751 	tid = READ_ONCE(c->tid);  in do_slab_free()
3766 		if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {  in do_slab_free()
3767 			note_cmpxchg_failure("slab_free", s, tid);  in do_slab_free()
3778 		tid = c->tid;  in do_slab_free()
3783 		c->tid = next_tid(tid);  in do_slab_free()
3966 			c->tid = next_tid(c->tid);  in __kmem_cache_alloc_bulk()
3990 	c->tid = next_tid(c->tid);  in __kmem_cache_alloc_bulk()