212 static inline bool kmem_cache_debug(struct kmem_cache *s)  in kmem_cache_debug()
217 static inline bool slub_debug_orig_size(struct kmem_cache *s)  in slub_debug_orig_size()
223 void *fixup_red_left(struct kmem_cache *s, void *p)  in fixup_red_left()
231 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)  in kmem_cache_has_cpu_partial()
322 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }  in sysfs_slab_add()
323 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)  in sysfs_slab_alias()
330 static inline void debugfs_slab_add(struct kmem_cache *s) { }  in debugfs_slab_add()
333 static inline void stat(const struct kmem_cache *s, enum stat_item si)  in stat()
374 static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,  in freelist_ptr_encode()
387 static inline void *freelist_ptr_decode(const struct kmem_cache *s,  in freelist_ptr_decode()
400 static inline void *get_freepointer(struct kmem_cache *s, void *object)  in get_freepointer()
412 static void prefetch_freepointer(const struct kmem_cache *s, void *object)  in prefetch_freepointer()
429 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)  in get_freepointer_safe()
443 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)  in set_freepointer()
487 static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)  in slub_set_cpu_partial()
504 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)  in slub_set_cpu_partial()
569 static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,  in __slab_update_freelist()
599 static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,  in slab_update_freelist()
634 static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,  in __fill_map()
666 static inline unsigned int size_from_object(struct kmem_cache *s)  in size_from_object()
674 static inline void *restore_red_left(struct kmem_cache *s, void *p)  in restore_red_left()
715 static inline int check_valid_pointer(struct kmem_cache *s,  in check_valid_pointer()
746 static inline bool freeptr_outside_object(struct kmem_cache *s)  in freeptr_outside_object()
755 static inline unsigned int get_info_end(struct kmem_cache *s)  in get_info_end()
763 static struct track *get_track(struct kmem_cache *s, void *object,  in get_track()
792 static void set_track_update(struct kmem_cache *s, void *object,  in set_track_update()
807 static __always_inline void set_track(struct kmem_cache *s, void *object,  in set_track()
815 static void init_tracking(struct kmem_cache *s, void *object)  in init_tracking()
826 static void print_track(const char *s, struct track *t, unsigned long pr_time)  in print_track()
844 void print_tracking(struct kmem_cache *s, void *object)  in print_tracking()
869 static inline void set_orig_size(struct kmem_cache *s,  in set_orig_size()
894 static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)  in get_orig_size()
907 void skip_orig_size_check(struct kmem_cache *s, const void *object)  in skip_orig_size_check()
912 static void slab_bug(struct kmem_cache *s, char *fmt, ...)  in slab_bug()
927 static void slab_fix(struct kmem_cache *s, char *fmt, ...)  in slab_fix()
942 static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)  in print_trailer()
984 static void object_err(struct kmem_cache *s, struct slab *slab,  in object_err()
995 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,  in freelist_corrupted()
1009 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,  in slab_err()
1027 static void init_object(struct kmem_cache *s, void *object, u8 val)  in init_object()
1054 static void restore_bytes(struct kmem_cache *s, char *message, u8 data,  in restore_bytes()
1061 static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,  in check_bytes_and_report()
1133 static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)  in check_pad_bytes()
1155 static void slab_pad_check(struct kmem_cache *s, struct slab *slab)  in slab_pad_check()
1190 static int check_object(struct kmem_cache *s, struct slab *slab,  in check_object()
1258 static int check_slab(struct kmem_cache *s, struct slab *slab)  in check_slab()
1292 static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)  in on_freelist()
1341 static void trace(struct kmem_cache *s, struct slab *slab, void *object,  in trace()
1362 static void add_full(struct kmem_cache *s,  in add_full()
1372 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)  in remove_full()
1386 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)  in inc_slabs_node()
1401 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)  in dec_slabs_node()
1410 static void setup_object_debug(struct kmem_cache *s, void *object)  in setup_object_debug()
1420 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)  in setup_slab_debug()
1430 static inline int alloc_consistency_checks(struct kmem_cache *s,  in alloc_consistency_checks()
1447 static noinline bool alloc_debug_processing(struct kmem_cache *s,  in alloc_debug_processing()
1476 static inline int free_consistency_checks(struct kmem_cache *s,  in free_consistency_checks()
1722 static inline void setup_object_debug(struct kmem_cache *s, void *object) {}  in setup_object_debug()
1724 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}  in setup_slab_debug()
1726 static inline bool alloc_debug_processing(struct kmem_cache *s,  in alloc_debug_processing()
1729 static inline bool free_debug_processing(struct kmem_cache *s,  in free_debug_processing()
1733 static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}  in slab_pad_check()
1734 static inline int check_object(struct kmem_cache *s, struct slab *slab,  in check_object()
1737 static inline void set_track(struct kmem_cache *s, void *object,  in set_track()
1739 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,  in add_full()
1741 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,  in remove_full()
1754 static inline void inc_slabs_node(struct kmem_cache *s, int node,  in inc_slabs_node()
1756 static inline void dec_slabs_node(struct kmem_cache *s, int node,  in dec_slabs_node()
1760 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,  in freelist_corrupted()
1772 static __always_inline bool slab_free_hook(struct kmem_cache *s,  in slab_free_hook()
1809 static inline bool slab_free_freelist_hook(struct kmem_cache *s,  in slab_free_freelist_hook()
1853 static void *setup_object(struct kmem_cache *s, void *object)  in setup_object()
1895 static int init_cache_random_seq(struct kmem_cache *s)  in init_cache_random_seq()
1924 	struct kmem_cache *s;  in init_freelist_randomization()  local
1935 static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab,  in next_freelist_entry()
1957 static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)  in shuffle_freelist()
1991 static inline int init_cache_random_seq(struct kmem_cache *s)  in init_cache_random_seq()
1996 static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)  in shuffle_freelist()
2002 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)  in allocate_slab()
2069 static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)  in new_slab()
2080 static void __free_slab(struct kmem_cache *s, struct slab *slab)  in __free_slab()
2103 static void free_slab(struct kmem_cache *s, struct slab *slab)  in free_slab()
2119 static void discard_slab(struct kmem_cache *s, struct slab *slab)  in discard_slab()
2159 static void *alloc_single_from_partial(struct kmem_cache *s,  in alloc_single_from_partial()
2189 static void *alloc_single_from_new_slab(struct kmem_cache *s,  in alloc_single_from_new_slab()
2229 static inline void *acquire_slab(struct kmem_cache *s,  in acquire_slab()
2271 static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,  in put_cpu_partial()
2279 static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,  in get_partial_node()
2340 static void *get_any_partial(struct kmem_cache *s, struct partial_context *pc)  in get_any_partial()
2403 static void *get_partial(struct kmem_cache *s, int node, struct partial_context *pc)  in get_partial()
2458 		const struct kmem_cache *s, unsigned long tid)  in note_cmpxchg_failure()
2481 static void init_kmem_cache_cpus(struct kmem_cache *s)  in init_kmem_cache_cpus()
2499 static void deactivate_slab(struct kmem_cache *s, struct slab *slab,  in deactivate_slab()
2607 static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab)  in __unfreeze_partials()
2670 static void unfreeze_partials(struct kmem_cache *s)  in unfreeze_partials()
2684 static void unfreeze_partials_cpu(struct kmem_cache *s,  in unfreeze_partials_cpu()
2703 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)  in put_cpu_partial()
2745 static inline void unfreeze_partials(struct kmem_cache *s) { }  in unfreeze_partials()
2746 static inline void unfreeze_partials_cpu(struct kmem_cache *s,  in unfreeze_partials_cpu()
2751 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)  in flush_slab()
2774 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)  in __flush_cpu_slab()
2794 	struct kmem_cache *s;  member
2805 	struct kmem_cache *s;  in flush_cpu_slab()  local
2820 static bool has_cpu_slab(int cpu, struct kmem_cache *s)  in has_cpu_slab()
2830 static void flush_all_cpus_locked(struct kmem_cache *s)  in flush_all_cpus_locked()
2860 static void flush_all(struct kmem_cache *s)  in flush_all()
2873 	struct kmem_cache *s;  in slub_cpu_dead()  local
2883 static inline void flush_all_cpus_locked(struct kmem_cache *s) { }  in flush_all_cpus_locked()
2884 static inline void flush_all(struct kmem_cache *s) { }  in flush_all()
2885 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { }  in __flush_cpu_slab()
2914 static inline bool free_debug_processing(struct kmem_cache *s,  in free_debug_processing()
2990 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)  in slab_out_of_memory()
3025 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { }  in slab_out_of_memory()
3038 __update_cpu_freelist_fast(struct kmem_cache *s,  in __update_cpu_freelist_fast()
3057 static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)  in get_freelist()
3102 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,  in ___slab_alloc()
3315 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,  in __slab_alloc()
3336 static __always_inline void *__slab_alloc_node(struct kmem_cache *s,  in __slab_alloc_node()
3411 static void *__slab_alloc_node(struct kmem_cache *s,  in __slab_alloc_node()
3442 static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,  in maybe_wipe_obj_freeptr()
3460 static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,  in slab_alloc_node()
3490 static __fastpath_inline void *slab_alloc(struct kmem_cache *s, struct list_lru *lru,  in slab_alloc()
3497 void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,  in __kmem_cache_alloc_lru()
3507 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)  in kmem_cache_alloc()
3513 void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,  in kmem_cache_alloc_lru()
3520 void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,  in __kmem_cache_alloc_node()
3528 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)  in kmem_cache_alloc_node()
3539 	struct kmem_cache *s, struct slab *slab,  in free_to_partial_list()
3607 static void __slab_free(struct kmem_cache *s, struct slab *slab,  in __slab_free()
3741 static __always_inline void do_slab_free(struct kmem_cache *s,  in do_slab_free()
3797 static void do_slab_free(struct kmem_cache *s,  in do_slab_free()
3807 static __fastpath_inline void slab_free(struct kmem_cache *s, struct slab *slab,  in slab_free()
3827 void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller)  in __kmem_cache_free()
3832 void kmem_cache_free(struct kmem_cache *s, void *x)  in kmem_cache_free()
3847 	struct kmem_cache *s;  member
3863 int build_detached_freelist(struct kmem_cache *s, size_t size,  in build_detached_freelist()
3922 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)  in kmem_cache_free_bulk()
3941 static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,  in __kmem_cache_alloc_bulk()
4011 static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,  in __kmem_cache_alloc_bulk()
4042 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,  in kmem_cache_alloc_bulk()
4223 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)  in alloc_kmem_cache_cpus()
4244 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)  in alloc_kmem_cache_cpus()
4297 static void free_kmem_cache_nodes(struct kmem_cache *s)  in free_kmem_cache_nodes()
4308 void __kmem_cache_release(struct kmem_cache *s)  in __kmem_cache_release()
4317 static int init_kmem_cache_nodes(struct kmem_cache *s)  in init_kmem_cache_nodes()
4342 static void set_cpu_partial(struct kmem_cache *s)  in set_cpu_partial()
4379 static int calculate_sizes(struct kmem_cache *s)  in calculate_sizes()
4516 static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)  in kmem_cache_open()
4575 static void list_slab_objects(struct kmem_cache *s, struct slab *slab,  in list_slab_objects()
4603 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)  in free_partial()
4625 bool __kmem_cache_empty(struct kmem_cache *s)  in __kmem_cache_empty()
4639 int __kmem_cache_shutdown(struct kmem_cache *s)  in __kmem_cache_shutdown()
4662 	struct kmem_cache *s = slab->slab_cache;  in __kmem_obj_info()  local
4757 	struct kmem_cache *s;  in __check_heap_object()  local
4806 static int __kmem_cache_do_shrink(struct kmem_cache *s)  in __kmem_cache_do_shrink()
4868 int __kmem_cache_shrink(struct kmem_cache *s)  in __kmem_cache_shrink()
4876 	struct kmem_cache *s;  in slab_mem_going_offline_callback()  local
4915 	struct kmem_cache *s;  in slab_mem_going_online_callback()  local
5003 	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);  in bootstrap()  local
5096 	struct kmem_cache *s;  in __kmem_cache_alias()  local
5116 int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)  in __kmem_cache_create()
5153 static void validate_slab(struct kmem_cache *s, struct slab *slab,  in validate_slab()
5173 static int validate_slab_node(struct kmem_cache *s,  in validate_slab_node()
5210 long validate_slab_cache(struct kmem_cache *s)  in validate_slab_cache()
5287 static int add_location(struct loc_track *t, struct kmem_cache *s,  in add_location()
5379 static void process_slab(struct loc_track *t, struct kmem_cache *s,  in process_slab()
5413 static ssize_t show_slab_objects(struct kmem_cache *s,  in show_slab_objects()
5540 static ssize_t slab_size_show(struct kmem_cache *s, char *buf)  in slab_size_show()
5546 static ssize_t align_show(struct kmem_cache *s, char *buf)  in align_show()
5552 static ssize_t object_size_show(struct kmem_cache *s, char *buf)  in object_size_show()
5558 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)  in objs_per_slab_show()
5564 static ssize_t order_show(struct kmem_cache *s, char *buf)  in order_show()
5570 static ssize_t min_partial_show(struct kmem_cache *s, char *buf)  in min_partial_show()
5575 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,  in min_partial_store()
5590 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)  in cpu_partial_show()
5600 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,  in cpu_partial_store()
5618 static ssize_t ctor_show(struct kmem_cache *s, char *buf)  in ctor_show()
5626 static ssize_t aliases_show(struct kmem_cache *s, char *buf)  in aliases_show()
5632 static ssize_t partial_show(struct kmem_cache *s, char *buf)  in partial_show()
5638 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)  in cpu_slabs_show()
5644 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)  in objects_partial_show()
5650 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)  in slabs_cpu_partial_show()
5691 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)  in reclaim_account_show()
5697 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)  in hwcache_align_show()
5704 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)  in cache_dma_show()
5712 static ssize_t usersize_show(struct kmem_cache *s, char *buf)  in usersize_show()
5719 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)  in destroy_by_rcu_show()
5726 static ssize_t slabs_show(struct kmem_cache *s, char *buf)  in slabs_show()
5732 static ssize_t total_objects_show(struct kmem_cache *s, char *buf)  in total_objects_show()
5738 static ssize_t objects_show(struct kmem_cache *s, char *buf)  in objects_show()
5744 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)  in sanity_checks_show()
5750 static ssize_t trace_show(struct kmem_cache *s, char *buf)  in trace_show()
5756 static ssize_t red_zone_show(struct kmem_cache *s, char *buf)  in red_zone_show()
5763 static ssize_t poison_show(struct kmem_cache *s, char *buf)  in poison_show()
5770 static ssize_t store_user_show(struct kmem_cache *s, char *buf)  in store_user_show()
5777 static ssize_t validate_show(struct kmem_cache *s, char *buf)  in validate_show()
5782 static ssize_t validate_store(struct kmem_cache *s,  in validate_store()
5799 static ssize_t failslab_show(struct kmem_cache *s, char *buf)  in failslab_show()
5804 static ssize_t failslab_store(struct kmem_cache *s, const char *buf,  in failslab_store()
5820 static ssize_t shrink_show(struct kmem_cache *s, char *buf)  in shrink_show()
5825 static ssize_t shrink_store(struct kmem_cache *s,  in shrink_store()
5837 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)  in remote_node_defrag_ratio_show()
5842 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,  in remote_node_defrag_ratio_store()
5862 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)  in show_stat()
5894 static void clear_stat(struct kmem_cache *s, enum stat_item si)  in clear_stat()
5946 static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf)  in skip_kfence_show()
5951 static ssize_t skip_kfence_store(struct kmem_cache *s,  in skip_kfence_store()
6053 	struct kmem_cache *s;  in slab_attr_show()  local
6069 	struct kmem_cache *s;  in slab_attr_store()  local
6097 static inline struct kset *cache_kset(struct kmem_cache *s)  in cache_kset()
6108 static char *create_unique_id(struct kmem_cache *s)  in create_unique_id()
6146 static int sysfs_slab_add(struct kmem_cache *s)  in sysfs_slab_add()
6197 void sysfs_slab_unlink(struct kmem_cache *s)  in sysfs_slab_unlink()
6203 void sysfs_slab_release(struct kmem_cache *s)  in sysfs_slab_release()
6214 	struct kmem_cache *s;  member
6221 static int sysfs_slab_alias(struct kmem_cache *s, const char *name)  in sysfs_slab_alias()
6247 	struct kmem_cache *s;  in slab_sysfs_init()  local
6401 	struct kmem_cache *s = file_inode(filep)->i_private;  in slab_debug_trace_open()  local
6463 static void debugfs_slab_add(struct kmem_cache *s)  in debugfs_slab_add()
6479 void debugfs_slab_release(struct kmem_cache *s)  in debugfs_slab_release()
6486 	struct kmem_cache *s;  in slab_debugfs_init()  local
6503 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)  in get_slabinfo()
6525 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)  in slabinfo_show_stats()