15015a300SAlexander Potapenko // SPDX-License-Identifier: GPL-2.0 25015a300SAlexander Potapenko /* 35015a300SAlexander Potapenko * Test cases for SL[AOU]B/page initialization at alloc/free time. 45015a300SAlexander Potapenko */ 55015a300SAlexander Potapenko #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 65015a300SAlexander Potapenko 75015a300SAlexander Potapenko #include <linux/init.h> 85015a300SAlexander Potapenko #include <linux/kernel.h> 95015a300SAlexander Potapenko #include <linux/mm.h> 105015a300SAlexander Potapenko #include <linux/module.h> 115015a300SAlexander Potapenko #include <linux/slab.h> 125015a300SAlexander Potapenko #include <linux/string.h> 135015a300SAlexander Potapenko #include <linux/vmalloc.h> 145015a300SAlexander Potapenko 155015a300SAlexander Potapenko #define GARBAGE_INT (0x09A7BA9E) 165015a300SAlexander Potapenko #define GARBAGE_BYTE (0x9E) 175015a300SAlexander Potapenko 185015a300SAlexander Potapenko #define REPORT_FAILURES_IN_FN() \ 195015a300SAlexander Potapenko do { \ 205015a300SAlexander Potapenko if (failures) \ 215015a300SAlexander Potapenko pr_info("%s failed %d out of %d times\n", \ 225015a300SAlexander Potapenko __func__, failures, num_tests); \ 235015a300SAlexander Potapenko else \ 245015a300SAlexander Potapenko pr_info("all %d tests in %s passed\n", \ 255015a300SAlexander Potapenko num_tests, __func__); \ 265015a300SAlexander Potapenko } while (0) 275015a300SAlexander Potapenko 285015a300SAlexander Potapenko /* Calculate the number of uninitialized bytes in the buffer. */ 295015a300SAlexander Potapenko static int __init count_nonzero_bytes(void *ptr, size_t size) 305015a300SAlexander Potapenko { 315015a300SAlexander Potapenko int i, ret = 0; 325015a300SAlexander Potapenko unsigned char *p = (unsigned char *)ptr; 335015a300SAlexander Potapenko 345015a300SAlexander Potapenko for (i = 0; i < size; i++) 355015a300SAlexander Potapenko if (p[i]) 365015a300SAlexander Potapenko ret++; 375015a300SAlexander Potapenko return ret; 385015a300SAlexander Potapenko } 395015a300SAlexander Potapenko 405015a300SAlexander Potapenko /* Fill a buffer with garbage, skipping |skip| first bytes. */ 415015a300SAlexander Potapenko static void __init fill_with_garbage_skip(void *ptr, size_t size, size_t skip) 425015a300SAlexander Potapenko { 435015a300SAlexander Potapenko unsigned int *p = (unsigned int *)ptr; 445015a300SAlexander Potapenko int i = 0; 455015a300SAlexander Potapenko 465015a300SAlexander Potapenko if (skip) { 475015a300SAlexander Potapenko WARN_ON(skip > size); 485015a300SAlexander Potapenko p += skip; 495015a300SAlexander Potapenko } 505015a300SAlexander Potapenko while (size >= sizeof(*p)) { 515015a300SAlexander Potapenko p[i] = GARBAGE_INT; 525015a300SAlexander Potapenko i++; 535015a300SAlexander Potapenko size -= sizeof(*p); 545015a300SAlexander Potapenko } 555015a300SAlexander Potapenko if (size) 565015a300SAlexander Potapenko memset(&p[i], GARBAGE_BYTE, size); 575015a300SAlexander Potapenko } 585015a300SAlexander Potapenko 595015a300SAlexander Potapenko static void __init fill_with_garbage(void *ptr, size_t size) 605015a300SAlexander Potapenko { 615015a300SAlexander Potapenko fill_with_garbage_skip(ptr, size, 0); 625015a300SAlexander Potapenko } 635015a300SAlexander Potapenko 645015a300SAlexander Potapenko static int __init do_alloc_pages_order(int order, int *total_failures) 655015a300SAlexander Potapenko { 665015a300SAlexander Potapenko struct page *page; 675015a300SAlexander Potapenko void *buf; 685015a300SAlexander Potapenko size_t size = PAGE_SIZE << order; 695015a300SAlexander Potapenko 705015a300SAlexander Potapenko page = alloc_pages(GFP_KERNEL, order); 715015a300SAlexander Potapenko buf = page_address(page); 725015a300SAlexander Potapenko fill_with_garbage(buf, size); 735015a300SAlexander Potapenko __free_pages(page, order); 745015a300SAlexander Potapenko 755015a300SAlexander Potapenko page = alloc_pages(GFP_KERNEL, order); 765015a300SAlexander Potapenko buf = page_address(page); 775015a300SAlexander Potapenko if (count_nonzero_bytes(buf, size)) 785015a300SAlexander Potapenko (*total_failures)++; 795015a300SAlexander Potapenko fill_with_garbage(buf, size); 805015a300SAlexander Potapenko __free_pages(page, order); 815015a300SAlexander Potapenko return 1; 825015a300SAlexander Potapenko } 835015a300SAlexander Potapenko 845015a300SAlexander Potapenko /* Test the page allocator by calling alloc_pages with different orders. */ 855015a300SAlexander Potapenko static int __init test_pages(int *total_failures) 865015a300SAlexander Potapenko { 875015a300SAlexander Potapenko int failures = 0, num_tests = 0; 885015a300SAlexander Potapenko int i; 895015a300SAlexander Potapenko 905015a300SAlexander Potapenko for (i = 0; i < 10; i++) 915015a300SAlexander Potapenko num_tests += do_alloc_pages_order(i, &failures); 925015a300SAlexander Potapenko 935015a300SAlexander Potapenko REPORT_FAILURES_IN_FN(); 945015a300SAlexander Potapenko *total_failures += failures; 955015a300SAlexander Potapenko return num_tests; 965015a300SAlexander Potapenko } 975015a300SAlexander Potapenko 985015a300SAlexander Potapenko /* Test kmalloc() with given parameters. */ 995015a300SAlexander Potapenko static int __init do_kmalloc_size(size_t size, int *total_failures) 1005015a300SAlexander Potapenko { 1015015a300SAlexander Potapenko void *buf; 1025015a300SAlexander Potapenko 1035015a300SAlexander Potapenko buf = kmalloc(size, GFP_KERNEL); 1045015a300SAlexander Potapenko fill_with_garbage(buf, size); 1055015a300SAlexander Potapenko kfree(buf); 1065015a300SAlexander Potapenko 1075015a300SAlexander Potapenko buf = kmalloc(size, GFP_KERNEL); 1085015a300SAlexander Potapenko if (count_nonzero_bytes(buf, size)) 1095015a300SAlexander Potapenko (*total_failures)++; 1105015a300SAlexander Potapenko fill_with_garbage(buf, size); 1115015a300SAlexander Potapenko kfree(buf); 1125015a300SAlexander Potapenko return 1; 1135015a300SAlexander Potapenko } 1145015a300SAlexander Potapenko 1155015a300SAlexander Potapenko /* Test vmalloc() with given parameters. */ 1165015a300SAlexander Potapenko static int __init do_vmalloc_size(size_t size, int *total_failures) 1175015a300SAlexander Potapenko { 1185015a300SAlexander Potapenko void *buf; 1195015a300SAlexander Potapenko 1205015a300SAlexander Potapenko buf = vmalloc(size); 1215015a300SAlexander Potapenko fill_with_garbage(buf, size); 1225015a300SAlexander Potapenko vfree(buf); 1235015a300SAlexander Potapenko 1245015a300SAlexander Potapenko buf = vmalloc(size); 1255015a300SAlexander Potapenko if (count_nonzero_bytes(buf, size)) 1265015a300SAlexander Potapenko (*total_failures)++; 1275015a300SAlexander Potapenko fill_with_garbage(buf, size); 1285015a300SAlexander Potapenko vfree(buf); 1295015a300SAlexander Potapenko return 1; 1305015a300SAlexander Potapenko } 1315015a300SAlexander Potapenko 1325015a300SAlexander Potapenko /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ 1335015a300SAlexander Potapenko static int __init test_kvmalloc(int *total_failures) 1345015a300SAlexander Potapenko { 1355015a300SAlexander Potapenko int failures = 0, num_tests = 0; 1365015a300SAlexander Potapenko int i, size; 1375015a300SAlexander Potapenko 1385015a300SAlexander Potapenko for (i = 0; i < 20; i++) { 1395015a300SAlexander Potapenko size = 1 << i; 1405015a300SAlexander Potapenko num_tests += do_kmalloc_size(size, &failures); 1415015a300SAlexander Potapenko num_tests += do_vmalloc_size(size, &failures); 1425015a300SAlexander Potapenko } 1435015a300SAlexander Potapenko 1445015a300SAlexander Potapenko REPORT_FAILURES_IN_FN(); 1455015a300SAlexander Potapenko *total_failures += failures; 1465015a300SAlexander Potapenko return num_tests; 1475015a300SAlexander Potapenko } 1485015a300SAlexander Potapenko 1495015a300SAlexander Potapenko #define CTOR_BYTES (sizeof(unsigned int)) 1505015a300SAlexander Potapenko #define CTOR_PATTERN (0x41414141) 1515015a300SAlexander Potapenko /* Initialize the first 4 bytes of the object. */ 1525015a300SAlexander Potapenko static void test_ctor(void *obj) 1535015a300SAlexander Potapenko { 1545015a300SAlexander Potapenko *(unsigned int *)obj = CTOR_PATTERN; 1555015a300SAlexander Potapenko } 1565015a300SAlexander Potapenko 1575015a300SAlexander Potapenko /* 1585015a300SAlexander Potapenko * Check the invariants for the buffer allocated from a slab cache. 1595015a300SAlexander Potapenko * If the cache has a test constructor, the first 4 bytes of the object must 1605015a300SAlexander Potapenko * always remain equal to CTOR_PATTERN. 1615015a300SAlexander Potapenko * If the cache isn't an RCU-typesafe one, or if the allocation is done with 1625015a300SAlexander Potapenko * __GFP_ZERO, then the object contents must be zeroed after allocation. 1635015a300SAlexander Potapenko * If the cache is an RCU-typesafe one, the object contents must never be 1645015a300SAlexander Potapenko * zeroed after the first use. This is checked by memcmp() in 1655015a300SAlexander Potapenko * do_kmem_cache_size(). 1665015a300SAlexander Potapenko */ 1675015a300SAlexander Potapenko static bool __init check_buf(void *buf, int size, bool want_ctor, 1685015a300SAlexander Potapenko bool want_rcu, bool want_zero) 1695015a300SAlexander Potapenko { 1705015a300SAlexander Potapenko int bytes; 1715015a300SAlexander Potapenko bool fail = false; 1725015a300SAlexander Potapenko 1735015a300SAlexander Potapenko bytes = count_nonzero_bytes(buf, size); 1745015a300SAlexander Potapenko WARN_ON(want_ctor && want_zero); 1755015a300SAlexander Potapenko if (want_zero) 1765015a300SAlexander Potapenko return bytes; 1775015a300SAlexander Potapenko if (want_ctor) { 1785015a300SAlexander Potapenko if (*(unsigned int *)buf != CTOR_PATTERN) 1795015a300SAlexander Potapenko fail = 1; 1805015a300SAlexander Potapenko } else { 1815015a300SAlexander Potapenko if (bytes) 1825015a300SAlexander Potapenko fail = !want_rcu; 1835015a300SAlexander Potapenko } 1845015a300SAlexander Potapenko return fail; 1855015a300SAlexander Potapenko } 1865015a300SAlexander Potapenko 1875015a300SAlexander Potapenko /* 1885015a300SAlexander Potapenko * Test kmem_cache with given parameters: 1895015a300SAlexander Potapenko * want_ctor - use a constructor; 1905015a300SAlexander Potapenko * want_rcu - use SLAB_TYPESAFE_BY_RCU; 1915015a300SAlexander Potapenko * want_zero - use __GFP_ZERO. 1925015a300SAlexander Potapenko */ 1935015a300SAlexander Potapenko static int __init do_kmem_cache_size(size_t size, bool want_ctor, 1945015a300SAlexander Potapenko bool want_rcu, bool want_zero, 1955015a300SAlexander Potapenko int *total_failures) 1965015a300SAlexander Potapenko { 1975015a300SAlexander Potapenko struct kmem_cache *c; 1985015a300SAlexander Potapenko int iter; 1995015a300SAlexander Potapenko bool fail = false; 2005015a300SAlexander Potapenko gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); 2015015a300SAlexander Potapenko void *buf, *buf_copy; 2025015a300SAlexander Potapenko 2035015a300SAlexander Potapenko c = kmem_cache_create("test_cache", size, 1, 2045015a300SAlexander Potapenko want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, 2055015a300SAlexander Potapenko want_ctor ? test_ctor : NULL); 2065015a300SAlexander Potapenko for (iter = 0; iter < 10; iter++) { 2075015a300SAlexander Potapenko buf = kmem_cache_alloc(c, alloc_mask); 2085015a300SAlexander Potapenko /* Check that buf is zeroed, if it must be. */ 2095015a300SAlexander Potapenko fail = check_buf(buf, size, want_ctor, want_rcu, want_zero); 2105015a300SAlexander Potapenko fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); 2115015a300SAlexander Potapenko /* 2125015a300SAlexander Potapenko * If this is an RCU cache, use a critical section to ensure we 2135015a300SAlexander Potapenko * can touch objects after they're freed. 2145015a300SAlexander Potapenko */ 2155015a300SAlexander Potapenko if (want_rcu) { 2165015a300SAlexander Potapenko rcu_read_lock(); 2175015a300SAlexander Potapenko /* 2185015a300SAlexander Potapenko * Copy the buffer to check that it's not wiped on 2195015a300SAlexander Potapenko * free(). 2205015a300SAlexander Potapenko */ 2215015a300SAlexander Potapenko buf_copy = kmalloc(size, GFP_KERNEL); 2225015a300SAlexander Potapenko if (buf_copy) 2235015a300SAlexander Potapenko memcpy(buf_copy, buf, size); 2245015a300SAlexander Potapenko } 2255015a300SAlexander Potapenko kmem_cache_free(c, buf); 2265015a300SAlexander Potapenko if (want_rcu) { 2275015a300SAlexander Potapenko /* 2285015a300SAlexander Potapenko * Check that |buf| is intact after kmem_cache_free(). 2295015a300SAlexander Potapenko * |want_zero| is false, because we wrote garbage to 2305015a300SAlexander Potapenko * the buffer already. 2315015a300SAlexander Potapenko */ 2325015a300SAlexander Potapenko fail |= check_buf(buf, size, want_ctor, want_rcu, 2335015a300SAlexander Potapenko false); 2345015a300SAlexander Potapenko if (buf_copy) { 2355015a300SAlexander Potapenko fail |= (bool)memcmp(buf, buf_copy, size); 2365015a300SAlexander Potapenko kfree(buf_copy); 2375015a300SAlexander Potapenko } 2385015a300SAlexander Potapenko rcu_read_unlock(); 2395015a300SAlexander Potapenko } 2405015a300SAlexander Potapenko } 2415015a300SAlexander Potapenko kmem_cache_destroy(c); 2425015a300SAlexander Potapenko 2435015a300SAlexander Potapenko *total_failures += fail; 2445015a300SAlexander Potapenko return 1; 2455015a300SAlexander Potapenko } 2465015a300SAlexander Potapenko 2475015a300SAlexander Potapenko /* 2485015a300SAlexander Potapenko * Check that the data written to an RCU-allocated object survives 2495015a300SAlexander Potapenko * reallocation. 2505015a300SAlexander Potapenko */ 2515015a300SAlexander Potapenko static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) 2525015a300SAlexander Potapenko { 2535015a300SAlexander Potapenko struct kmem_cache *c; 2545015a300SAlexander Potapenko void *buf, *buf_contents, *saved_ptr; 2555015a300SAlexander Potapenko void **used_objects; 2565015a300SAlexander Potapenko int i, iter, maxiter = 1024; 2575015a300SAlexander Potapenko bool fail = false; 2585015a300SAlexander Potapenko 2595015a300SAlexander Potapenko c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, 2605015a300SAlexander Potapenko NULL); 2615015a300SAlexander Potapenko buf = kmem_cache_alloc(c, GFP_KERNEL); 2625015a300SAlexander Potapenko saved_ptr = buf; 2635015a300SAlexander Potapenko fill_with_garbage(buf, size); 2645015a300SAlexander Potapenko buf_contents = kmalloc(size, GFP_KERNEL); 2655015a300SAlexander Potapenko if (!buf_contents) 2665015a300SAlexander Potapenko goto out; 2675015a300SAlexander Potapenko used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); 2685015a300SAlexander Potapenko if (!used_objects) { 2695015a300SAlexander Potapenko kfree(buf_contents); 2705015a300SAlexander Potapenko goto out; 2715015a300SAlexander Potapenko } 2725015a300SAlexander Potapenko memcpy(buf_contents, buf, size); 2735015a300SAlexander Potapenko kmem_cache_free(c, buf); 2745015a300SAlexander Potapenko /* 2755015a300SAlexander Potapenko * Run for a fixed number of iterations. If we never hit saved_ptr, 2765015a300SAlexander Potapenko * assume the test passes. 2775015a300SAlexander Potapenko */ 2785015a300SAlexander Potapenko for (iter = 0; iter < maxiter; iter++) { 2795015a300SAlexander Potapenko buf = kmem_cache_alloc(c, GFP_KERNEL); 2805015a300SAlexander Potapenko used_objects[iter] = buf; 2815015a300SAlexander Potapenko if (buf == saved_ptr) { 2825015a300SAlexander Potapenko fail = memcmp(buf_contents, buf, size); 2835015a300SAlexander Potapenko for (i = 0; i <= iter; i++) 2845015a300SAlexander Potapenko kmem_cache_free(c, used_objects[i]); 2855015a300SAlexander Potapenko goto free_out; 2865015a300SAlexander Potapenko } 2875015a300SAlexander Potapenko } 2885015a300SAlexander Potapenko 2895015a300SAlexander Potapenko free_out: 2905015a300SAlexander Potapenko kmem_cache_destroy(c); 2915015a300SAlexander Potapenko kfree(buf_contents); 2925015a300SAlexander Potapenko kfree(used_objects); 2935015a300SAlexander Potapenko out: 2945015a300SAlexander Potapenko *total_failures += fail; 2955015a300SAlexander Potapenko return 1; 2965015a300SAlexander Potapenko } 2975015a300SAlexander Potapenko 2985015a300SAlexander Potapenko /* 2995015a300SAlexander Potapenko * Test kmem_cache allocation by creating caches of different sizes, with and 3005015a300SAlexander Potapenko * without constructors, with and without SLAB_TYPESAFE_BY_RCU. 3015015a300SAlexander Potapenko */ 3025015a300SAlexander Potapenko static int __init test_kmemcache(int *total_failures) 3035015a300SAlexander Potapenko { 3045015a300SAlexander Potapenko int failures = 0, num_tests = 0; 3055015a300SAlexander Potapenko int i, flags, size; 3065015a300SAlexander Potapenko bool ctor, rcu, zero; 3075015a300SAlexander Potapenko 3085015a300SAlexander Potapenko for (i = 0; i < 10; i++) { 3095015a300SAlexander Potapenko size = 8 << i; 3105015a300SAlexander Potapenko for (flags = 0; flags < 8; flags++) { 3115015a300SAlexander Potapenko ctor = flags & 1; 3125015a300SAlexander Potapenko rcu = flags & 2; 3135015a300SAlexander Potapenko zero = flags & 4; 3145015a300SAlexander Potapenko if (ctor & zero) 3155015a300SAlexander Potapenko continue; 3165015a300SAlexander Potapenko num_tests += do_kmem_cache_size(size, ctor, rcu, zero, 3175015a300SAlexander Potapenko &failures); 3185015a300SAlexander Potapenko } 3195015a300SAlexander Potapenko } 3205015a300SAlexander Potapenko REPORT_FAILURES_IN_FN(); 3215015a300SAlexander Potapenko *total_failures += failures; 3225015a300SAlexander Potapenko return num_tests; 3235015a300SAlexander Potapenko } 3245015a300SAlexander Potapenko 3255015a300SAlexander Potapenko /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ 3265015a300SAlexander Potapenko static int __init test_rcu_persistent(int *total_failures) 3275015a300SAlexander Potapenko { 3285015a300SAlexander Potapenko int failures = 0, num_tests = 0; 3295015a300SAlexander Potapenko int i, size; 3305015a300SAlexander Potapenko 3315015a300SAlexander Potapenko for (i = 0; i < 10; i++) { 3325015a300SAlexander Potapenko size = 8 << i; 3335015a300SAlexander Potapenko num_tests += do_kmem_cache_rcu_persistent(size, &failures); 3345015a300SAlexander Potapenko } 3355015a300SAlexander Potapenko REPORT_FAILURES_IN_FN(); 3365015a300SAlexander Potapenko *total_failures += failures; 3375015a300SAlexander Potapenko return num_tests; 3385015a300SAlexander Potapenko } 3395015a300SAlexander Potapenko 3405015a300SAlexander Potapenko /* 3415015a300SAlexander Potapenko * Run the tests. Each test function returns the number of executed tests and 3425015a300SAlexander Potapenko * updates |failures| with the number of failed tests. 3435015a300SAlexander Potapenko */ 3445015a300SAlexander Potapenko static int __init test_meminit_init(void) 3455015a300SAlexander Potapenko { 3465015a300SAlexander Potapenko int failures = 0, num_tests = 0; 3475015a300SAlexander Potapenko 3485015a300SAlexander Potapenko num_tests += test_pages(&failures); 3495015a300SAlexander Potapenko num_tests += test_kvmalloc(&failures); 3505015a300SAlexander Potapenko num_tests += test_kmemcache(&failures); 3515015a300SAlexander Potapenko num_tests += test_rcu_persistent(&failures); 3525015a300SAlexander Potapenko 3535015a300SAlexander Potapenko if (failures == 0) 3545015a300SAlexander Potapenko pr_info("all %d tests passed!\n", num_tests); 3555015a300SAlexander Potapenko else 3565015a300SAlexander Potapenko pr_info("failures: %d out of %d\n", failures, num_tests); 3575015a300SAlexander Potapenko 3585015a300SAlexander Potapenko return failures ? -EINVAL : 0; 3595015a300SAlexander Potapenko } 3605015a300SAlexander Potapenko module_init(test_meminit_init); 3615015a300SAlexander Potapenko 3625015a300SAlexander Potapenko MODULE_LICENSE("GPL"); 363