Lines Matching refs:cpu_slab
340 raw_cpu_inc(s->cpu_slab->stat[si]); in stat()
2454 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); in note_cmpxchg_failure()
2480 c = per_cpu_ptr(s->cpu_slab, cpu); in init_kmem_cache_cpus()
2668 local_lock_irqsave(&s->cpu_slab->lock, flags); in unfreeze_partials()
2669 partial_slab = this_cpu_read(s->cpu_slab->partial); in unfreeze_partials()
2670 this_cpu_write(s->cpu_slab->partial, NULL); in unfreeze_partials()
2671 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in unfreeze_partials()
2703 local_lock_irqsave(&s->cpu_slab->lock, flags); in put_cpu_partial()
2705 oldslab = this_cpu_read(s->cpu_slab->partial); in put_cpu_partial()
2726 this_cpu_write(s->cpu_slab->partial, slab); in put_cpu_partial()
2728 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in put_cpu_partial()
2750 local_lock_irqsave(&s->cpu_slab->lock, flags); in flush_slab()
2759 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in flush_slab()
2769 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in __flush_cpu_slab()
2805 c = this_cpu_ptr(s->cpu_slab); in flush_cpu_slab()
2815 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in has_cpu_slab()
3038 return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full, in __update_cpu_freelist_fast()
3056 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in get_freelist()
3142 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
3144 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3156 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3165 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in ___slab_alloc()
3175 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3180 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
3182 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3189 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3195 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
3197 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3201 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3208 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3222 slub_put_cpu_ptr(s->cpu_slab); in ___slab_alloc()
3224 c = slub_get_cpu_ptr(s->cpu_slab); in ___slab_alloc()
3281 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
3290 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3319 c = slub_get_cpu_ptr(s->cpu_slab); in __slab_alloc()
3324 slub_put_cpu_ptr(s->cpu_slab); in __slab_alloc()
3350 c = raw_cpu_ptr(s->cpu_slab); in __slab_alloc_node()
3750 c = raw_cpu_ptr(s->cpu_slab); in do_slab_free()
3772 local_lock(&s->cpu_slab->lock); in do_slab_free()
3773 c = this_cpu_ptr(s->cpu_slab); in do_slab_free()
3775 local_unlock(&s->cpu_slab->lock); in do_slab_free()
3785 local_unlock(&s->cpu_slab->lock); in do_slab_free()
3946 c = slub_get_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
3947 local_lock_irqsave(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
3968 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
3979 c = this_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
3982 local_lock_irqsave(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
3991 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
3992 slub_put_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
3997 slub_put_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
4226 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), in alloc_kmem_cache_cpus()
4229 if (!s->cpu_slab) in alloc_kmem_cache_cpus()
4305 free_percpu(s->cpu_slab); in __kmem_cache_release()
5423 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, in show_slab_objects()
5654 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5669 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5866 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; in show_stat()
5892 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; in clear_stat()