1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Test cases for SL[AOU]B/page initialization at alloc/free time. 4 */ 5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 6 7 #include <linux/init.h> 8 #include <linux/kernel.h> 9 #include <linux/mm.h> 10 #include <linux/module.h> 11 #include <linux/slab.h> 12 #include <linux/string.h> 13 #include <linux/vmalloc.h> 14 15 #define GARBAGE_INT (0x09A7BA9E) 16 #define GARBAGE_BYTE (0x9E) 17 18 #define REPORT_FAILURES_IN_FN() \ 19 do { \ 20 if (failures) \ 21 pr_info("%s failed %d out of %d times\n", \ 22 __func__, failures, num_tests); \ 23 else \ 24 pr_info("all %d tests in %s passed\n", \ 25 num_tests, __func__); \ 26 } while (0) 27 28 /* Calculate the number of uninitialized bytes in the buffer. */ 29 static int __init count_nonzero_bytes(void *ptr, size_t size) 30 { 31 int i, ret = 0; 32 unsigned char *p = (unsigned char *)ptr; 33 34 for (i = 0; i < size; i++) 35 if (p[i]) 36 ret++; 37 return ret; 38 } 39 40 /* Fill a buffer with garbage, skipping |skip| first bytes. */ 41 static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) 42 { 43 unsigned int *p = (unsigned int *)((char *)ptr + skip); 44 int i = 0; 45 46 WARN_ON(skip > size); 47 size -= skip; 48 49 while (size >= sizeof(*p)) { 50 p[i] = GARBAGE_INT; 51 i++; 52 size -= sizeof(*p); 53 } 54 if (size) 55 memset(&p[i], GARBAGE_BYTE, size); 56 } 57 58 static void __init fill_with_garbage(void *ptr, size_t size) 59 { 60 fill_with_garbage_skip(ptr, size, 0); 61 } 62 63 static int __init do_alloc_pages_order(int order, int *total_failures) 64 { 65 struct page *page; 66 void *buf; 67 size_t size = PAGE_SIZE << order; 68 69 page = alloc_pages(GFP_KERNEL, order); 70 buf = page_address(page); 71 fill_with_garbage(buf, size); 72 __free_pages(page, order); 73 74 page = alloc_pages(GFP_KERNEL, order); 75 buf = page_address(page); 76 if (count_nonzero_bytes(buf, size)) 77 (*total_failures)++; 78 fill_with_garbage(buf, size); 79 __free_pages(page, order); 80 return 1; 81 } 82 83 /* Test the page allocator by calling alloc_pages with different orders. */ 84 static int __init test_pages(int *total_failures) 85 { 86 int failures = 0, num_tests = 0; 87 int i; 88 89 for (i = 0; i < 10; i++) 90 num_tests += do_alloc_pages_order(i, &failures); 91 92 REPORT_FAILURES_IN_FN(); 93 *total_failures += failures; 94 return num_tests; 95 } 96 97 /* Test kmalloc() with given parameters. */ 98 static int __init do_kmalloc_size(size_t size, int *total_failures) 99 { 100 void *buf; 101 102 buf = kmalloc(size, GFP_KERNEL); 103 fill_with_garbage(buf, size); 104 kfree(buf); 105 106 buf = kmalloc(size, GFP_KERNEL); 107 if (count_nonzero_bytes(buf, size)) 108 (*total_failures)++; 109 fill_with_garbage(buf, size); 110 kfree(buf); 111 return 1; 112 } 113 114 /* Test vmalloc() with given parameters. */ 115 static int __init do_vmalloc_size(size_t size, int *total_failures) 116 { 117 void *buf; 118 119 buf = vmalloc(size); 120 fill_with_garbage(buf, size); 121 vfree(buf); 122 123 buf = vmalloc(size); 124 if (count_nonzero_bytes(buf, size)) 125 (*total_failures)++; 126 fill_with_garbage(buf, size); 127 vfree(buf); 128 return 1; 129 } 130 131 /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ 132 static int __init test_kvmalloc(int *total_failures) 133 { 134 int failures = 0, num_tests = 0; 135 int i, size; 136 137 for (i = 0; i < 20; i++) { 138 size = 1 << i; 139 num_tests += do_kmalloc_size(size, &failures); 140 num_tests += do_vmalloc_size(size, &failures); 141 } 142 143 REPORT_FAILURES_IN_FN(); 144 *total_failures += failures; 145 return num_tests; 146 } 147 148 #define CTOR_BYTES (sizeof(unsigned int)) 149 #define CTOR_PATTERN (0x41414141) 150 /* Initialize the first 4 bytes of the object. */ 151 static void test_ctor(void *obj) 152 { 153 *(unsigned int *)obj = CTOR_PATTERN; 154 } 155 156 /* 157 * Check the invariants for the buffer allocated from a slab cache. 158 * If the cache has a test constructor, the first 4 bytes of the object must 159 * always remain equal to CTOR_PATTERN. 160 * If the cache isn't an RCU-typesafe one, or if the allocation is done with 161 * __GFP_ZERO, then the object contents must be zeroed after allocation. 162 * If the cache is an RCU-typesafe one, the object contents must never be 163 * zeroed after the first use. This is checked by memcmp() in 164 * do_kmem_cache_size(). 165 */ 166 static bool __init check_buf(void *buf, int size, bool want_ctor, 167 bool want_rcu, bool want_zero) 168 { 169 int bytes; 170 bool fail = false; 171 172 bytes = count_nonzero_bytes(buf, size); 173 WARN_ON(want_ctor && want_zero); 174 if (want_zero) 175 return bytes; 176 if (want_ctor) { 177 if (*(unsigned int *)buf != CTOR_PATTERN) 178 fail = 1; 179 } else { 180 if (bytes) 181 fail = !want_rcu; 182 } 183 return fail; 184 } 185 186 #define BULK_SIZE 100 187 static void *bulk_array[BULK_SIZE]; 188 189 /* 190 * Test kmem_cache with given parameters: 191 * want_ctor - use a constructor; 192 * want_rcu - use SLAB_TYPESAFE_BY_RCU; 193 * want_zero - use __GFP_ZERO. 194 */ 195 static int __init do_kmem_cache_size(size_t size, bool want_ctor, 196 bool want_rcu, bool want_zero, 197 int *total_failures) 198 { 199 struct kmem_cache *c; 200 int iter; 201 bool fail = false; 202 gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); 203 void *buf, *buf_copy; 204 205 c = kmem_cache_create("test_cache", size, 1, 206 want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, 207 want_ctor ? test_ctor : NULL); 208 for (iter = 0; iter < 10; iter++) { 209 /* Do a test of bulk allocations */ 210 if (!want_rcu && !want_ctor) { 211 int ret; 212 213 ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array); 214 if (!ret) { 215 fail = true; 216 } else { 217 int i; 218 for (i = 0; i < ret; i++) 219 fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero); 220 kmem_cache_free_bulk(c, ret, bulk_array); 221 } 222 } 223 224 buf = kmem_cache_alloc(c, alloc_mask); 225 /* Check that buf is zeroed, if it must be. */ 226 fail |= check_buf(buf, size, want_ctor, want_rcu, want_zero); 227 fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); 228 229 if (!want_rcu) { 230 kmem_cache_free(c, buf); 231 continue; 232 } 233 234 /* 235 * If this is an RCU cache, use a critical section to ensure we 236 * can touch objects after they're freed. 237 */ 238 rcu_read_lock(); 239 /* 240 * Copy the buffer to check that it's not wiped on 241 * free(). 242 */ 243 buf_copy = kmalloc(size, GFP_ATOMIC); 244 if (buf_copy) 245 memcpy(buf_copy, buf, size); 246 247 kmem_cache_free(c, buf); 248 /* 249 * Check that |buf| is intact after kmem_cache_free(). 250 * |want_zero| is false, because we wrote garbage to 251 * the buffer already. 252 */ 253 fail |= check_buf(buf, size, want_ctor, want_rcu, 254 false); 255 if (buf_copy) { 256 fail |= (bool)memcmp(buf, buf_copy, size); 257 kfree(buf_copy); 258 } 259 rcu_read_unlock(); 260 } 261 kmem_cache_destroy(c); 262 263 *total_failures += fail; 264 return 1; 265 } 266 267 /* 268 * Check that the data written to an RCU-allocated object survives 269 * reallocation. 270 */ 271 static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) 272 { 273 struct kmem_cache *c; 274 void *buf, *buf_contents, *saved_ptr; 275 void **used_objects; 276 int i, iter, maxiter = 1024; 277 bool fail = false; 278 279 c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, 280 NULL); 281 buf = kmem_cache_alloc(c, GFP_KERNEL); 282 if (!buf) 283 goto out; 284 saved_ptr = buf; 285 fill_with_garbage(buf, size); 286 buf_contents = kmalloc(size, GFP_KERNEL); 287 if (!buf_contents) { 288 kmem_cache_free(c, buf); 289 goto out; 290 } 291 used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); 292 if (!used_objects) { 293 kmem_cache_free(c, buf); 294 kfree(buf_contents); 295 goto out; 296 } 297 memcpy(buf_contents, buf, size); 298 kmem_cache_free(c, buf); 299 /* 300 * Run for a fixed number of iterations. If we never hit saved_ptr, 301 * assume the test passes. 302 */ 303 for (iter = 0; iter < maxiter; iter++) { 304 buf = kmem_cache_alloc(c, GFP_KERNEL); 305 used_objects[iter] = buf; 306 if (buf == saved_ptr) { 307 fail = memcmp(buf_contents, buf, size); 308 for (i = 0; i <= iter; i++) 309 kmem_cache_free(c, used_objects[i]); 310 goto free_out; 311 } 312 } 313 314 for (iter = 0; iter < maxiter; iter++) 315 kmem_cache_free(c, used_objects[iter]); 316 317 free_out: 318 kfree(buf_contents); 319 kfree(used_objects); 320 out: 321 kmem_cache_destroy(c); 322 *total_failures += fail; 323 return 1; 324 } 325 326 static int __init do_kmem_cache_size_bulk(int size, int *total_failures) 327 { 328 struct kmem_cache *c; 329 int i, iter, maxiter = 1024; 330 int num, bytes; 331 bool fail = false; 332 void *objects[10]; 333 334 c = kmem_cache_create("test_cache", size, size, 0, NULL); 335 for (iter = 0; (iter < maxiter) && !fail; iter++) { 336 num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), 337 objects); 338 for (i = 0; i < num; i++) { 339 bytes = count_nonzero_bytes(objects[i], size); 340 if (bytes) 341 fail = true; 342 fill_with_garbage(objects[i], size); 343 } 344 345 if (num) 346 kmem_cache_free_bulk(c, num, objects); 347 } 348 kmem_cache_destroy(c); 349 *total_failures += fail; 350 return 1; 351 } 352 353 /* 354 * Test kmem_cache allocation by creating caches of different sizes, with and 355 * without constructors, with and without SLAB_TYPESAFE_BY_RCU. 356 */ 357 static int __init test_kmemcache(int *total_failures) 358 { 359 int failures = 0, num_tests = 0; 360 int i, flags, size; 361 bool ctor, rcu, zero; 362 363 for (i = 0; i < 10; i++) { 364 size = 8 << i; 365 for (flags = 0; flags < 8; flags++) { 366 ctor = flags & 1; 367 rcu = flags & 2; 368 zero = flags & 4; 369 if (ctor & zero) 370 continue; 371 num_tests += do_kmem_cache_size(size, ctor, rcu, zero, 372 &failures); 373 } 374 num_tests += do_kmem_cache_size_bulk(size, &failures); 375 } 376 REPORT_FAILURES_IN_FN(); 377 *total_failures += failures; 378 return num_tests; 379 } 380 381 /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ 382 static int __init test_rcu_persistent(int *total_failures) 383 { 384 int failures = 0, num_tests = 0; 385 int i, size; 386 387 for (i = 0; i < 10; i++) { 388 size = 8 << i; 389 num_tests += do_kmem_cache_rcu_persistent(size, &failures); 390 } 391 REPORT_FAILURES_IN_FN(); 392 *total_failures += failures; 393 return num_tests; 394 } 395 396 /* 397 * Run the tests. Each test function returns the number of executed tests and 398 * updates |failures| with the number of failed tests. 399 */ 400 static int __init test_meminit_init(void) 401 { 402 int failures = 0, num_tests = 0; 403 404 num_tests += test_pages(&failures); 405 num_tests += test_kvmalloc(&failures); 406 num_tests += test_kmemcache(&failures); 407 num_tests += test_rcu_persistent(&failures); 408 409 if (failures == 0) 410 pr_info("all %d tests passed!\n", num_tests); 411 else 412 pr_info("failures: %d out of %d\n", failures, num_tests); 413 414 return failures ? -EINVAL : 0; 415 } 416 module_init(test_meminit_init); 417 418 MODULE_LICENSE("GPL"); 419