1 // SPDX-License-Identifier: GPL-2.0 2 #include <stdlib.h> 3 #include <string.h> 4 #include <malloc.h> 5 #include <pthread.h> 6 #include <unistd.h> 7 #include <assert.h> 8 9 #include <linux/gfp.h> 10 #include <linux/poison.h> 11 #include <linux/slab.h> 12 #include <linux/radix-tree.h> 13 #include <urcu/uatomic.h> 14 15 int nr_allocated; 16 int preempt_count; 17 int test_verbose; 18 19 struct kmem_cache { 20 pthread_mutex_t lock; 21 unsigned int size; 22 unsigned int align; 23 int nr_objs; 24 void *objs; 25 void (*ctor)(void *); 26 unsigned int non_kernel; 27 unsigned long nr_allocated; 28 unsigned long nr_tallocated; 29 }; 30 31 void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) 32 { 33 cachep->non_kernel = val; 34 } 35 36 unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) 37 { 38 return cachep->size * cachep->nr_allocated; 39 } 40 41 unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) 42 { 43 return cachep->nr_allocated; 44 } 45 46 unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) 47 { 48 return cachep->nr_tallocated; 49 } 50 51 void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) 52 { 53 cachep->nr_tallocated = 0; 54 } 55 56 void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, 57 int gfp) 58 { 59 void *p; 60 61 if (!(gfp & __GFP_DIRECT_RECLAIM)) { 62 if (!cachep->non_kernel) 63 return NULL; 64 65 cachep->non_kernel--; 66 } 67 68 pthread_mutex_lock(&cachep->lock); 69 if (cachep->nr_objs) { 70 struct radix_tree_node *node = cachep->objs; 71 cachep->nr_objs--; 72 cachep->objs = node->parent; 73 pthread_mutex_unlock(&cachep->lock); 74 node->parent = NULL; 75 p = node; 76 } else { 77 pthread_mutex_unlock(&cachep->lock); 78 if (cachep->align) 79 posix_memalign(&p, cachep->align, cachep->size); 80 else 81 p = malloc(cachep->size); 82 if (cachep->ctor) 83 cachep->ctor(p); 84 else if (gfp & __GFP_ZERO) 85 memset(p, 0, cachep->size); 86 } 87 88 uatomic_inc(&cachep->nr_allocated); 89 uatomic_inc(&nr_allocated); 90 uatomic_inc(&cachep->nr_tallocated); 91 if (kmalloc_verbose) 92 printf("Allocating %p from slab\n", p); 93 return p; 94 } 95 96 void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp) 97 { 98 assert(objp); 99 uatomic_dec(&nr_allocated); 100 uatomic_dec(&cachep->nr_allocated); 101 if (kmalloc_verbose) 102 printf("Freeing %p to slab\n", objp); 103 if (cachep->nr_objs > 10 || cachep->align) { 104 memset(objp, POISON_FREE, cachep->size); 105 free(objp); 106 } else { 107 struct radix_tree_node *node = objp; 108 cachep->nr_objs++; 109 node->parent = cachep->objs; 110 cachep->objs = node; 111 } 112 } 113 114 void kmem_cache_free(struct kmem_cache *cachep, void *objp) 115 { 116 pthread_mutex_lock(&cachep->lock); 117 kmem_cache_free_locked(cachep, objp); 118 pthread_mutex_unlock(&cachep->lock); 119 } 120 121 void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) 122 { 123 if (kmalloc_verbose) 124 pr_debug("Bulk free %p[0-%lu]\n", list, size - 1); 125 126 pthread_mutex_lock(&cachep->lock); 127 for (int i = 0; i < size; i++) 128 kmem_cache_free_locked(cachep, list[i]); 129 pthread_mutex_unlock(&cachep->lock); 130 } 131 132 int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, 133 void **p) 134 { 135 size_t i; 136 137 if (kmalloc_verbose) 138 pr_debug("Bulk alloc %lu\n", size); 139 140 if (!(gfp & __GFP_DIRECT_RECLAIM)) { 141 if (cachep->non_kernel < size) 142 return 0; 143 144 cachep->non_kernel -= size; 145 } 146 147 pthread_mutex_lock(&cachep->lock); 148 if (cachep->nr_objs >= size) { 149 struct radix_tree_node *node; 150 151 for (i = 0; i < size; i++) { 152 node = cachep->objs; 153 cachep->nr_objs--; 154 cachep->objs = node->parent; 155 p[i] = node; 156 node->parent = NULL; 157 } 158 pthread_mutex_unlock(&cachep->lock); 159 } else { 160 pthread_mutex_unlock(&cachep->lock); 161 for (i = 0; i < size; i++) { 162 if (cachep->align) { 163 posix_memalign(&p[i], cachep->align, 164 cachep->size * size); 165 } else { 166 p[i] = malloc(cachep->size * size); 167 } 168 if (cachep->ctor) 169 cachep->ctor(p[i]); 170 else if (gfp & __GFP_ZERO) 171 memset(p[i], 0, cachep->size); 172 } 173 } 174 175 for (i = 0; i < size; i++) { 176 uatomic_inc(&nr_allocated); 177 uatomic_inc(&cachep->nr_allocated); 178 uatomic_inc(&cachep->nr_tallocated); 179 if (kmalloc_verbose) 180 printf("Allocating %p from slab\n", p[i]); 181 } 182 183 return size; 184 } 185 186 struct kmem_cache * 187 kmem_cache_create(const char *name, unsigned int size, unsigned int align, 188 unsigned int flags, void (*ctor)(void *)) 189 { 190 struct kmem_cache *ret = malloc(sizeof(*ret)); 191 192 pthread_mutex_init(&ret->lock, NULL); 193 ret->size = size; 194 ret->align = align; 195 ret->nr_objs = 0; 196 ret->nr_allocated = 0; 197 ret->nr_tallocated = 0; 198 ret->objs = NULL; 199 ret->ctor = ctor; 200 ret->non_kernel = 0; 201 return ret; 202 } 203 204 /* 205 * Test the test infrastructure for kem_cache_alloc/free and bulk counterparts. 206 */ 207 void test_kmem_cache_bulk(void) 208 { 209 int i; 210 void *list[12]; 211 static struct kmem_cache *test_cache, *test_cache2; 212 213 /* 214 * Testing the bulk allocators without aligned kmem_cache to force the 215 * bulk alloc/free to reuse 216 */ 217 test_cache = kmem_cache_create("test_cache", 256, 0, SLAB_PANIC, NULL); 218 219 for (i = 0; i < 5; i++) 220 list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); 221 222 for (i = 0; i < 5; i++) 223 kmem_cache_free(test_cache, list[i]); 224 assert(test_cache->nr_objs == 5); 225 226 kmem_cache_alloc_bulk(test_cache, __GFP_DIRECT_RECLAIM, 5, list); 227 kmem_cache_free_bulk(test_cache, 5, list); 228 229 for (i = 0; i < 12 ; i++) 230 list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); 231 232 for (i = 0; i < 12; i++) 233 kmem_cache_free(test_cache, list[i]); 234 235 /* The last free will not be kept around */ 236 assert(test_cache->nr_objs == 11); 237 238 /* Aligned caches will immediately free */ 239 test_cache2 = kmem_cache_create("test_cache2", 128, 128, SLAB_PANIC, NULL); 240 241 kmem_cache_alloc_bulk(test_cache2, __GFP_DIRECT_RECLAIM, 10, list); 242 kmem_cache_free_bulk(test_cache2, 10, list); 243 assert(!test_cache2->nr_objs); 244 245 246 } 247