1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Test cases for SL[AOU]B/page initialization at alloc/free time. 4 */ 5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 6 7 #include <linux/init.h> 8 #include <linux/kernel.h> 9 #include <linux/mm.h> 10 #include <linux/module.h> 11 #include <linux/slab.h> 12 #include <linux/string.h> 13 #include <linux/vmalloc.h> 14 15 #define GARBAGE_INT (0x09A7BA9E) 16 #define GARBAGE_BYTE (0x9E) 17 18 #define REPORT_FAILURES_IN_FN() \ 19 do { \ 20 if (failures) \ 21 pr_info("%s failed %d out of %d times\n", \ 22 __func__, failures, num_tests); \ 23 else \ 24 pr_info("all %d tests in %s passed\n", \ 25 num_tests, __func__); \ 26 } while (0) 27 28 /* Calculate the number of uninitialized bytes in the buffer. */ 29 static int __init count_nonzero_bytes(void *ptr, size_t size) 30 { 31 int i, ret = 0; 32 unsigned char *p = (unsigned char *)ptr; 33 34 for (i = 0; i < size; i++) 35 if (p[i]) 36 ret++; 37 return ret; 38 } 39 40 /* Fill a buffer with garbage, skipping |skip| first bytes. */ 41 static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) 42 { 43 unsigned int *p = (unsigned int *)((char *)ptr + skip); 44 int i = 0; 45 46 WARN_ON(skip > size); 47 size -= skip; 48 49 while (size >= sizeof(*p)) { 50 p[i] = GARBAGE_INT; 51 i++; 52 size -= sizeof(*p); 53 } 54 if (size) 55 memset(&p[i], GARBAGE_BYTE, size); 56 } 57 58 static void __init fill_with_garbage(void *ptr, size_t size) 59 { 60 fill_with_garbage_skip(ptr, size, 0); 61 } 62 63 static int __init do_alloc_pages_order(int order, int *total_failures) 64 { 65 struct page *page; 66 void *buf; 67 size_t size = PAGE_SIZE << order; 68 69 page = alloc_pages(GFP_KERNEL, order); 70 buf = page_address(page); 71 fill_with_garbage(buf, size); 72 __free_pages(page, order); 73 74 page = alloc_pages(GFP_KERNEL, order); 75 buf = page_address(page); 76 if (count_nonzero_bytes(buf, size)) 77 (*total_failures)++; 78 fill_with_garbage(buf, size); 79 __free_pages(page, order); 80 return 1; 81 } 82 83 /* Test the page allocator by calling alloc_pages with different orders. */ 84 static int __init test_pages(int *total_failures) 85 { 86 int failures = 0, num_tests = 0; 87 int i; 88 89 for (i = 0; i < 10; i++) 90 num_tests += do_alloc_pages_order(i, &failures); 91 92 REPORT_FAILURES_IN_FN(); 93 *total_failures += failures; 94 return num_tests; 95 } 96 97 /* Test kmalloc() with given parameters. */ 98 static int __init do_kmalloc_size(size_t size, int *total_failures) 99 { 100 void *buf; 101 102 buf = kmalloc(size, GFP_KERNEL); 103 fill_with_garbage(buf, size); 104 kfree(buf); 105 106 buf = kmalloc(size, GFP_KERNEL); 107 if (count_nonzero_bytes(buf, size)) 108 (*total_failures)++; 109 fill_with_garbage(buf, size); 110 kfree(buf); 111 return 1; 112 } 113 114 /* Test vmalloc() with given parameters. */ 115 static int __init do_vmalloc_size(size_t size, int *total_failures) 116 { 117 void *buf; 118 119 buf = vmalloc(size); 120 fill_with_garbage(buf, size); 121 vfree(buf); 122 123 buf = vmalloc(size); 124 if (count_nonzero_bytes(buf, size)) 125 (*total_failures)++; 126 fill_with_garbage(buf, size); 127 vfree(buf); 128 return 1; 129 } 130 131 /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ 132 static int __init test_kvmalloc(int *total_failures) 133 { 134 int failures = 0, num_tests = 0; 135 int i, size; 136 137 for (i = 0; i < 20; i++) { 138 size = 1 << i; 139 num_tests += do_kmalloc_size(size, &failures); 140 num_tests += do_vmalloc_size(size, &failures); 141 } 142 143 REPORT_FAILURES_IN_FN(); 144 *total_failures += failures; 145 return num_tests; 146 } 147 148 #define CTOR_BYTES (sizeof(unsigned int)) 149 #define CTOR_PATTERN (0x41414141) 150 /* Initialize the first 4 bytes of the object. */ 151 static void test_ctor(void *obj) 152 { 153 *(unsigned int *)obj = CTOR_PATTERN; 154 } 155 156 /* 157 * Check the invariants for the buffer allocated from a slab cache. 158 * If the cache has a test constructor, the first 4 bytes of the object must 159 * always remain equal to CTOR_PATTERN. 160 * If the cache isn't an RCU-typesafe one, or if the allocation is done with 161 * __GFP_ZERO, then the object contents must be zeroed after allocation. 162 * If the cache is an RCU-typesafe one, the object contents must never be 163 * zeroed after the first use. This is checked by memcmp() in 164 * do_kmem_cache_size(). 165 */ 166 static bool __init check_buf(void *buf, int size, bool want_ctor, 167 bool want_rcu, bool want_zero) 168 { 169 int bytes; 170 bool fail = false; 171 172 bytes = count_nonzero_bytes(buf, size); 173 WARN_ON(want_ctor && want_zero); 174 if (want_zero) 175 return bytes; 176 if (want_ctor) { 177 if (*(unsigned int *)buf != CTOR_PATTERN) 178 fail = 1; 179 } else { 180 if (bytes) 181 fail = !want_rcu; 182 } 183 return fail; 184 } 185 186 /* 187 * Test kmem_cache with given parameters: 188 * want_ctor - use a constructor; 189 * want_rcu - use SLAB_TYPESAFE_BY_RCU; 190 * want_zero - use __GFP_ZERO. 191 */ 192 static int __init do_kmem_cache_size(size_t size, bool want_ctor, 193 bool want_rcu, bool want_zero, 194 int *total_failures) 195 { 196 struct kmem_cache *c; 197 int iter; 198 bool fail = false; 199 gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); 200 void *buf, *buf_copy; 201 202 c = kmem_cache_create("test_cache", size, 1, 203 want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, 204 want_ctor ? test_ctor : NULL); 205 for (iter = 0; iter < 10; iter++) { 206 buf = kmem_cache_alloc(c, alloc_mask); 207 /* Check that buf is zeroed, if it must be. */ 208 fail = check_buf(buf, size, want_ctor, want_rcu, want_zero); 209 fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); 210 211 if (!want_rcu) { 212 kmem_cache_free(c, buf); 213 continue; 214 } 215 216 /* 217 * If this is an RCU cache, use a critical section to ensure we 218 * can touch objects after they're freed. 219 */ 220 rcu_read_lock(); 221 /* 222 * Copy the buffer to check that it's not wiped on 223 * free(). 224 */ 225 buf_copy = kmalloc(size, GFP_ATOMIC); 226 if (buf_copy) 227 memcpy(buf_copy, buf, size); 228 229 kmem_cache_free(c, buf); 230 /* 231 * Check that |buf| is intact after kmem_cache_free(). 232 * |want_zero| is false, because we wrote garbage to 233 * the buffer already. 234 */ 235 fail |= check_buf(buf, size, want_ctor, want_rcu, 236 false); 237 if (buf_copy) { 238 fail |= (bool)memcmp(buf, buf_copy, size); 239 kfree(buf_copy); 240 } 241 rcu_read_unlock(); 242 } 243 kmem_cache_destroy(c); 244 245 *total_failures += fail; 246 return 1; 247 } 248 249 /* 250 * Check that the data written to an RCU-allocated object survives 251 * reallocation. 252 */ 253 static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) 254 { 255 struct kmem_cache *c; 256 void *buf, *buf_contents, *saved_ptr; 257 void **used_objects; 258 int i, iter, maxiter = 1024; 259 bool fail = false; 260 261 c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, 262 NULL); 263 buf = kmem_cache_alloc(c, GFP_KERNEL); 264 saved_ptr = buf; 265 fill_with_garbage(buf, size); 266 buf_contents = kmalloc(size, GFP_KERNEL); 267 if (!buf_contents) 268 goto out; 269 used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); 270 if (!used_objects) { 271 kfree(buf_contents); 272 goto out; 273 } 274 memcpy(buf_contents, buf, size); 275 kmem_cache_free(c, buf); 276 /* 277 * Run for a fixed number of iterations. If we never hit saved_ptr, 278 * assume the test passes. 279 */ 280 for (iter = 0; iter < maxiter; iter++) { 281 buf = kmem_cache_alloc(c, GFP_KERNEL); 282 used_objects[iter] = buf; 283 if (buf == saved_ptr) { 284 fail = memcmp(buf_contents, buf, size); 285 for (i = 0; i <= iter; i++) 286 kmem_cache_free(c, used_objects[i]); 287 goto free_out; 288 } 289 } 290 291 free_out: 292 kmem_cache_destroy(c); 293 kfree(buf_contents); 294 kfree(used_objects); 295 out: 296 *total_failures += fail; 297 return 1; 298 } 299 300 /* 301 * Test kmem_cache allocation by creating caches of different sizes, with and 302 * without constructors, with and without SLAB_TYPESAFE_BY_RCU. 303 */ 304 static int __init test_kmemcache(int *total_failures) 305 { 306 int failures = 0, num_tests = 0; 307 int i, flags, size; 308 bool ctor, rcu, zero; 309 310 for (i = 0; i < 10; i++) { 311 size = 8 << i; 312 for (flags = 0; flags < 8; flags++) { 313 ctor = flags & 1; 314 rcu = flags & 2; 315 zero = flags & 4; 316 if (ctor & zero) 317 continue; 318 num_tests += do_kmem_cache_size(size, ctor, rcu, zero, 319 &failures); 320 } 321 } 322 REPORT_FAILURES_IN_FN(); 323 *total_failures += failures; 324 return num_tests; 325 } 326 327 /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ 328 static int __init test_rcu_persistent(int *total_failures) 329 { 330 int failures = 0, num_tests = 0; 331 int i, size; 332 333 for (i = 0; i < 10; i++) { 334 size = 8 << i; 335 num_tests += do_kmem_cache_rcu_persistent(size, &failures); 336 } 337 REPORT_FAILURES_IN_FN(); 338 *total_failures += failures; 339 return num_tests; 340 } 341 342 /* 343 * Run the tests. Each test function returns the number of executed tests and 344 * updates |failures| with the number of failed tests. 345 */ 346 static int __init test_meminit_init(void) 347 { 348 int failures = 0, num_tests = 0; 349 350 num_tests += test_pages(&failures); 351 num_tests += test_kvmalloc(&failures); 352 num_tests += test_kmemcache(&failures); 353 num_tests += test_rcu_persistent(&failures); 354 355 if (failures == 0) 356 pr_info("all %d tests passed!\n", num_tests); 357 else 358 pr_info("failures: %d out of %d\n", failures, num_tests); 359 360 return failures ? -EINVAL : 0; 361 } 362 module_init(test_meminit_init); 363 364 MODULE_LICENSE("GPL"); 365