1bc8fbc5fSMarco Elver // SPDX-License-Identifier: GPL-2.0 2bc8fbc5fSMarco Elver /* 3bc8fbc5fSMarco Elver * Test cases for KFENCE memory safety error detector. Since the interface with 4bc8fbc5fSMarco Elver * which KFENCE's reports are obtained is via the console, this is the output we 5bc8fbc5fSMarco Elver * should verify. For each test case checks the presence (or absence) of 6bc8fbc5fSMarco Elver * generated reports. Relies on 'console' tracepoint to capture reports as they 7bc8fbc5fSMarco Elver * appear in the kernel log. 8bc8fbc5fSMarco Elver * 9bc8fbc5fSMarco Elver * Copyright (C) 2020, Google LLC. 10bc8fbc5fSMarco Elver * Author: Alexander Potapenko <glider@google.com> 11bc8fbc5fSMarco Elver * Marco Elver <elver@google.com> 12bc8fbc5fSMarco Elver */ 13bc8fbc5fSMarco Elver 14bc8fbc5fSMarco Elver #include <kunit/test.h> 15bc8fbc5fSMarco Elver #include <linux/jiffies.h> 16bc8fbc5fSMarco Elver #include <linux/kernel.h> 17bc8fbc5fSMarco Elver #include <linux/kfence.h> 18bc8fbc5fSMarco Elver #include <linux/mm.h> 19bc8fbc5fSMarco Elver #include <linux/random.h> 20bc8fbc5fSMarco Elver #include <linux/slab.h> 21bc8fbc5fSMarco Elver #include <linux/spinlock.h> 22bc8fbc5fSMarco Elver #include <linux/string.h> 23bc8fbc5fSMarco Elver #include <linux/tracepoint.h> 24bc8fbc5fSMarco Elver #include <trace/events/printk.h> 25bc8fbc5fSMarco Elver 26f99e12b2SSven Schnelle #include <asm/kfence.h> 27f99e12b2SSven Schnelle 28bc8fbc5fSMarco Elver #include "kfence.h" 29bc8fbc5fSMarco Elver 30f99e12b2SSven Schnelle /* May be overridden by <asm/kfence.h>. */ 31f99e12b2SSven Schnelle #ifndef arch_kfence_test_address 32f99e12b2SSven Schnelle #define arch_kfence_test_address(addr) (addr) 33f99e12b2SSven Schnelle #endif 34f99e12b2SSven Schnelle 35f51733e2SMarco Elver #define KFENCE_TEST_REQUIRES(test, cond) do { \ 36f51733e2SMarco Elver if (!(cond)) \ 37f51733e2SMarco Elver kunit_skip((test), "Test requires: " #cond); \ 38f51733e2SMarco Elver } while (0) 39f51733e2SMarco Elver 40bc8fbc5fSMarco Elver /* Report as observed from console. */ 41bc8fbc5fSMarco Elver static struct { 42bc8fbc5fSMarco Elver spinlock_t lock; 43bc8fbc5fSMarco Elver int nlines; 44bc8fbc5fSMarco Elver char lines[2][256]; 45bc8fbc5fSMarco Elver } observed = { 46bc8fbc5fSMarco Elver .lock = __SPIN_LOCK_UNLOCKED(observed.lock), 47bc8fbc5fSMarco Elver }; 48bc8fbc5fSMarco Elver 49bc8fbc5fSMarco Elver /* Probe for console output: obtains observed lines of interest. */ 50bc8fbc5fSMarco Elver static void probe_console(void *ignore, const char *buf, size_t len) 51bc8fbc5fSMarco Elver { 52bc8fbc5fSMarco Elver unsigned long flags; 53bc8fbc5fSMarco Elver int nlines; 54bc8fbc5fSMarco Elver 55bc8fbc5fSMarco Elver spin_lock_irqsave(&observed.lock, flags); 56bc8fbc5fSMarco Elver nlines = observed.nlines; 57bc8fbc5fSMarco Elver 58bc8fbc5fSMarco Elver if (strnstr(buf, "BUG: KFENCE: ", len) && strnstr(buf, "test_", len)) { 59bc8fbc5fSMarco Elver /* 60bc8fbc5fSMarco Elver * KFENCE report and related to the test. 61bc8fbc5fSMarco Elver * 62bc8fbc5fSMarco Elver * The provided @buf is not NUL-terminated; copy no more than 63bc8fbc5fSMarco Elver * @len bytes and let strscpy() add the missing NUL-terminator. 64bc8fbc5fSMarco Elver */ 65bc8fbc5fSMarco Elver strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0]))); 66bc8fbc5fSMarco Elver nlines = 1; 67bc8fbc5fSMarco Elver } else if (nlines == 1 && (strnstr(buf, "at 0x", len) || strnstr(buf, "of 0x", len))) { 68bc8fbc5fSMarco Elver strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0]))); 69bc8fbc5fSMarco Elver } 70bc8fbc5fSMarco Elver 71bc8fbc5fSMarco Elver WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */ 72bc8fbc5fSMarco Elver spin_unlock_irqrestore(&observed.lock, flags); 73bc8fbc5fSMarco Elver } 74bc8fbc5fSMarco Elver 75bc8fbc5fSMarco Elver /* Check if a report related to the test exists. */ 76bc8fbc5fSMarco Elver static bool report_available(void) 77bc8fbc5fSMarco Elver { 78bc8fbc5fSMarco Elver return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines); 79bc8fbc5fSMarco Elver } 80bc8fbc5fSMarco Elver 81bc8fbc5fSMarco Elver /* Information we expect in a report. */ 82bc8fbc5fSMarco Elver struct expect_report { 83bc8fbc5fSMarco Elver enum kfence_error_type type; /* The type or error. */ 84bc8fbc5fSMarco Elver void *fn; /* Function pointer to expected function where access occurred. */ 85bc8fbc5fSMarco Elver char *addr; /* Address at which the bad access occurred. */ 86bc8fbc5fSMarco Elver bool is_write; /* Is access a write. */ 87bc8fbc5fSMarco Elver }; 88bc8fbc5fSMarco Elver 89bc8fbc5fSMarco Elver static const char *get_access_type(const struct expect_report *r) 90bc8fbc5fSMarco Elver { 91bc8fbc5fSMarco Elver return r->is_write ? "write" : "read"; 92bc8fbc5fSMarco Elver } 93bc8fbc5fSMarco Elver 94bc8fbc5fSMarco Elver /* Check observed report matches information in @r. */ 95bc8fbc5fSMarco Elver static bool report_matches(const struct expect_report *r) 96bc8fbc5fSMarco Elver { 97f99e12b2SSven Schnelle unsigned long addr = (unsigned long)r->addr; 98bc8fbc5fSMarco Elver bool ret = false; 99bc8fbc5fSMarco Elver unsigned long flags; 100bc8fbc5fSMarco Elver typeof(observed.lines) expect; 101bc8fbc5fSMarco Elver const char *end; 102bc8fbc5fSMarco Elver char *cur; 103bc8fbc5fSMarco Elver 104bc8fbc5fSMarco Elver /* Doubled-checked locking. */ 105bc8fbc5fSMarco Elver if (!report_available()) 106bc8fbc5fSMarco Elver return false; 107bc8fbc5fSMarco Elver 108bc8fbc5fSMarco Elver /* Generate expected report contents. */ 109bc8fbc5fSMarco Elver 110bc8fbc5fSMarco Elver /* Title */ 111bc8fbc5fSMarco Elver cur = expect[0]; 112bc8fbc5fSMarco Elver end = &expect[0][sizeof(expect[0]) - 1]; 113bc8fbc5fSMarco Elver switch (r->type) { 114bc8fbc5fSMarco Elver case KFENCE_ERROR_OOB: 115bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "BUG: KFENCE: out-of-bounds %s", 116bc8fbc5fSMarco Elver get_access_type(r)); 117bc8fbc5fSMarco Elver break; 118bc8fbc5fSMarco Elver case KFENCE_ERROR_UAF: 119bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "BUG: KFENCE: use-after-free %s", 120bc8fbc5fSMarco Elver get_access_type(r)); 121bc8fbc5fSMarco Elver break; 122bc8fbc5fSMarco Elver case KFENCE_ERROR_CORRUPTION: 123bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "BUG: KFENCE: memory corruption"); 124bc8fbc5fSMarco Elver break; 125bc8fbc5fSMarco Elver case KFENCE_ERROR_INVALID: 126bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid %s", 127bc8fbc5fSMarco Elver get_access_type(r)); 128bc8fbc5fSMarco Elver break; 129bc8fbc5fSMarco Elver case KFENCE_ERROR_INVALID_FREE: 130bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid free"); 131bc8fbc5fSMarco Elver break; 132bc8fbc5fSMarco Elver } 133bc8fbc5fSMarco Elver 134bc8fbc5fSMarco Elver scnprintf(cur, end - cur, " in %pS", r->fn); 135bc8fbc5fSMarco Elver /* The exact offset won't match, remove it; also strip module name. */ 136bc8fbc5fSMarco Elver cur = strchr(expect[0], '+'); 137bc8fbc5fSMarco Elver if (cur) 138bc8fbc5fSMarco Elver *cur = '\0'; 139bc8fbc5fSMarco Elver 140bc8fbc5fSMarco Elver /* Access information */ 141bc8fbc5fSMarco Elver cur = expect[1]; 142bc8fbc5fSMarco Elver end = &expect[1][sizeof(expect[1]) - 1]; 143bc8fbc5fSMarco Elver 144bc8fbc5fSMarco Elver switch (r->type) { 145bc8fbc5fSMarco Elver case KFENCE_ERROR_OOB: 146bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "Out-of-bounds %s at", get_access_type(r)); 147f99e12b2SSven Schnelle addr = arch_kfence_test_address(addr); 148bc8fbc5fSMarco Elver break; 149bc8fbc5fSMarco Elver case KFENCE_ERROR_UAF: 150bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "Use-after-free %s at", get_access_type(r)); 151f99e12b2SSven Schnelle addr = arch_kfence_test_address(addr); 152bc8fbc5fSMarco Elver break; 153bc8fbc5fSMarco Elver case KFENCE_ERROR_CORRUPTION: 154bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "Corrupted memory at"); 155bc8fbc5fSMarco Elver break; 156bc8fbc5fSMarco Elver case KFENCE_ERROR_INVALID: 157bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "Invalid %s at", get_access_type(r)); 158f99e12b2SSven Schnelle addr = arch_kfence_test_address(addr); 159bc8fbc5fSMarco Elver break; 160bc8fbc5fSMarco Elver case KFENCE_ERROR_INVALID_FREE: 161bc8fbc5fSMarco Elver cur += scnprintf(cur, end - cur, "Invalid free of"); 162bc8fbc5fSMarco Elver break; 163bc8fbc5fSMarco Elver } 164bc8fbc5fSMarco Elver 165f99e12b2SSven Schnelle cur += scnprintf(cur, end - cur, " 0x%p", (void *)addr); 166bc8fbc5fSMarco Elver 167bc8fbc5fSMarco Elver spin_lock_irqsave(&observed.lock, flags); 168bc8fbc5fSMarco Elver if (!report_available()) 169bc8fbc5fSMarco Elver goto out; /* A new report is being captured. */ 170bc8fbc5fSMarco Elver 171bc8fbc5fSMarco Elver /* Finally match expected output to what we actually observed. */ 172bc8fbc5fSMarco Elver ret = strstr(observed.lines[0], expect[0]) && strstr(observed.lines[1], expect[1]); 173bc8fbc5fSMarco Elver out: 174bc8fbc5fSMarco Elver spin_unlock_irqrestore(&observed.lock, flags); 175bc8fbc5fSMarco Elver return ret; 176bc8fbc5fSMarco Elver } 177bc8fbc5fSMarco Elver 178bc8fbc5fSMarco Elver /* ===== Test cases ===== */ 179bc8fbc5fSMarco Elver 180bc8fbc5fSMarco Elver #define TEST_PRIV_WANT_MEMCACHE ((void *)1) 181bc8fbc5fSMarco Elver 182bc8fbc5fSMarco Elver /* Cache used by tests; if NULL, allocate from kmalloc instead. */ 183bc8fbc5fSMarco Elver static struct kmem_cache *test_cache; 184bc8fbc5fSMarco Elver 185bc8fbc5fSMarco Elver static size_t setup_test_cache(struct kunit *test, size_t size, slab_flags_t flags, 186bc8fbc5fSMarco Elver void (*ctor)(void *)) 187bc8fbc5fSMarco Elver { 188bc8fbc5fSMarco Elver if (test->priv != TEST_PRIV_WANT_MEMCACHE) 189bc8fbc5fSMarco Elver return size; 190bc8fbc5fSMarco Elver 191bc8fbc5fSMarco Elver kunit_info(test, "%s: size=%zu, ctor=%ps\n", __func__, size, ctor); 192bc8fbc5fSMarco Elver 193bc8fbc5fSMarco Elver /* 194bc8fbc5fSMarco Elver * Use SLAB_NOLEAKTRACE to prevent merging with existing caches. Any 195bc8fbc5fSMarco Elver * other flag in SLAB_NEVER_MERGE also works. Use SLAB_ACCOUNT to 196bc8fbc5fSMarco Elver * allocate via memcg, if enabled. 197bc8fbc5fSMarco Elver */ 198bc8fbc5fSMarco Elver flags |= SLAB_NOLEAKTRACE | SLAB_ACCOUNT; 199bc8fbc5fSMarco Elver test_cache = kmem_cache_create("test", size, 1, flags, ctor); 200bc8fbc5fSMarco Elver KUNIT_ASSERT_TRUE_MSG(test, test_cache, "could not create cache"); 201bc8fbc5fSMarco Elver 202bc8fbc5fSMarco Elver return size; 203bc8fbc5fSMarco Elver } 204bc8fbc5fSMarco Elver 205bc8fbc5fSMarco Elver static void test_cache_destroy(void) 206bc8fbc5fSMarco Elver { 207bc8fbc5fSMarco Elver if (!test_cache) 208bc8fbc5fSMarco Elver return; 209bc8fbc5fSMarco Elver 210bc8fbc5fSMarco Elver kmem_cache_destroy(test_cache); 211bc8fbc5fSMarco Elver test_cache = NULL; 212bc8fbc5fSMarco Elver } 213bc8fbc5fSMarco Elver 214bc8fbc5fSMarco Elver static inline size_t kmalloc_cache_alignment(size_t size) 215bc8fbc5fSMarco Elver { 216588c7fa0SHyeonggon Yoo return kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)]->align; 217bc8fbc5fSMarco Elver } 218bc8fbc5fSMarco Elver 219bc8fbc5fSMarco Elver /* Must always inline to match stack trace against caller. */ 220bc8fbc5fSMarco Elver static __always_inline void test_free(void *ptr) 221bc8fbc5fSMarco Elver { 222bc8fbc5fSMarco Elver if (test_cache) 223bc8fbc5fSMarco Elver kmem_cache_free(test_cache, ptr); 224bc8fbc5fSMarco Elver else 225bc8fbc5fSMarco Elver kfree(ptr); 226bc8fbc5fSMarco Elver } 227bc8fbc5fSMarco Elver 228bc8fbc5fSMarco Elver /* 229bc8fbc5fSMarco Elver * If this should be a KFENCE allocation, and on which side the allocation and 230bc8fbc5fSMarco Elver * the closest guard page should be. 231bc8fbc5fSMarco Elver */ 232bc8fbc5fSMarco Elver enum allocation_policy { 233bc8fbc5fSMarco Elver ALLOCATE_ANY, /* KFENCE, any side. */ 234bc8fbc5fSMarco Elver ALLOCATE_LEFT, /* KFENCE, left side of page. */ 235bc8fbc5fSMarco Elver ALLOCATE_RIGHT, /* KFENCE, right side of page. */ 236bc8fbc5fSMarco Elver ALLOCATE_NONE, /* No KFENCE allocation. */ 237bc8fbc5fSMarco Elver }; 238bc8fbc5fSMarco Elver 239bc8fbc5fSMarco Elver /* 240bc8fbc5fSMarco Elver * Try to get a guarded allocation from KFENCE. Uses either kmalloc() or the 241bc8fbc5fSMarco Elver * current test_cache if set up. 242bc8fbc5fSMarco Elver */ 243bc8fbc5fSMarco Elver static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocation_policy policy) 244bc8fbc5fSMarco Elver { 245bc8fbc5fSMarco Elver void *alloc; 246bc8fbc5fSMarco Elver unsigned long timeout, resched_after; 247bc8fbc5fSMarco Elver const char *policy_name; 248bc8fbc5fSMarco Elver 249bc8fbc5fSMarco Elver switch (policy) { 250bc8fbc5fSMarco Elver case ALLOCATE_ANY: 251bc8fbc5fSMarco Elver policy_name = "any"; 252bc8fbc5fSMarco Elver break; 253bc8fbc5fSMarco Elver case ALLOCATE_LEFT: 254bc8fbc5fSMarco Elver policy_name = "left"; 255bc8fbc5fSMarco Elver break; 256bc8fbc5fSMarco Elver case ALLOCATE_RIGHT: 257bc8fbc5fSMarco Elver policy_name = "right"; 258bc8fbc5fSMarco Elver break; 259bc8fbc5fSMarco Elver case ALLOCATE_NONE: 260bc8fbc5fSMarco Elver policy_name = "none"; 261bc8fbc5fSMarco Elver break; 262bc8fbc5fSMarco Elver } 263bc8fbc5fSMarco Elver 264bc8fbc5fSMarco Elver kunit_info(test, "%s: size=%zu, gfp=%x, policy=%s, cache=%i\n", __func__, size, gfp, 265bc8fbc5fSMarco Elver policy_name, !!test_cache); 266bc8fbc5fSMarco Elver 267bc8fbc5fSMarco Elver /* 268bc8fbc5fSMarco Elver * 100x the sample interval should be more than enough to ensure we get 269bc8fbc5fSMarco Elver * a KFENCE allocation eventually. 270bc8fbc5fSMarco Elver */ 2718913c610SPeng Liu timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval); 272bc8fbc5fSMarco Elver /* 273bc8fbc5fSMarco Elver * Especially for non-preemption kernels, ensure the allocation-gate 274bc8fbc5fSMarco Elver * timer can catch up: after @resched_after, every failed allocation 275bc8fbc5fSMarco Elver * attempt yields, to ensure the allocation-gate timer is scheduled. 276bc8fbc5fSMarco Elver */ 2778913c610SPeng Liu resched_after = jiffies + msecs_to_jiffies(kfence_sample_interval); 278bc8fbc5fSMarco Elver do { 279bc8fbc5fSMarco Elver if (test_cache) 280bc8fbc5fSMarco Elver alloc = kmem_cache_alloc(test_cache, gfp); 281bc8fbc5fSMarco Elver else 282bc8fbc5fSMarco Elver alloc = kmalloc(size, gfp); 283bc8fbc5fSMarco Elver 284bc8fbc5fSMarco Elver if (is_kfence_address(alloc)) { 2858dae0cfeSVlastimil Babka struct slab *slab = virt_to_slab(alloc); 286588c7fa0SHyeonggon Yoo struct kmem_cache *s = test_cache ?: 287588c7fa0SHyeonggon Yoo kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)]; 288bc8fbc5fSMarco Elver 289bc8fbc5fSMarco Elver /* 290bc8fbc5fSMarco Elver * Verify that various helpers return the right values 291bc8fbc5fSMarco Elver * even for KFENCE objects; these are required so that 292bc8fbc5fSMarco Elver * memcg accounting works correctly. 293bc8fbc5fSMarco Elver */ 2948dae0cfeSVlastimil Babka KUNIT_EXPECT_EQ(test, obj_to_index(s, slab, alloc), 0U); 2958dae0cfeSVlastimil Babka KUNIT_EXPECT_EQ(test, objs_per_slab(s, slab), 1); 296bc8fbc5fSMarco Elver 297bc8fbc5fSMarco Elver if (policy == ALLOCATE_ANY) 298bc8fbc5fSMarco Elver return alloc; 299f403f22fSKefeng Wang if (policy == ALLOCATE_LEFT && PAGE_ALIGNED(alloc)) 300bc8fbc5fSMarco Elver return alloc; 301f403f22fSKefeng Wang if (policy == ALLOCATE_RIGHT && !PAGE_ALIGNED(alloc)) 302bc8fbc5fSMarco Elver return alloc; 303bc8fbc5fSMarco Elver } else if (policy == ALLOCATE_NONE) 304bc8fbc5fSMarco Elver return alloc; 305bc8fbc5fSMarco Elver 306bc8fbc5fSMarco Elver test_free(alloc); 307bc8fbc5fSMarco Elver 308bc8fbc5fSMarco Elver if (time_after(jiffies, resched_after)) 309bc8fbc5fSMarco Elver cond_resched(); 310bc8fbc5fSMarco Elver } while (time_before(jiffies, timeout)); 311bc8fbc5fSMarco Elver 312bc8fbc5fSMarco Elver KUNIT_ASSERT_TRUE_MSG(test, false, "failed to allocate from KFENCE"); 313bc8fbc5fSMarco Elver return NULL; /* Unreachable. */ 314bc8fbc5fSMarco Elver } 315bc8fbc5fSMarco Elver 316bc8fbc5fSMarco Elver static void test_out_of_bounds_read(struct kunit *test) 317bc8fbc5fSMarco Elver { 318bc8fbc5fSMarco Elver size_t size = 32; 319bc8fbc5fSMarco Elver struct expect_report expect = { 320bc8fbc5fSMarco Elver .type = KFENCE_ERROR_OOB, 321bc8fbc5fSMarco Elver .fn = test_out_of_bounds_read, 322bc8fbc5fSMarco Elver .is_write = false, 323bc8fbc5fSMarco Elver }; 324bc8fbc5fSMarco Elver char *buf; 325bc8fbc5fSMarco Elver 326bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 327bc8fbc5fSMarco Elver 328bc8fbc5fSMarco Elver /* 329bc8fbc5fSMarco Elver * If we don't have our own cache, adjust based on alignment, so that we 330bc8fbc5fSMarco Elver * actually access guard pages on either side. 331bc8fbc5fSMarco Elver */ 332bc8fbc5fSMarco Elver if (!test_cache) 333bc8fbc5fSMarco Elver size = kmalloc_cache_alignment(size); 334bc8fbc5fSMarco Elver 335bc8fbc5fSMarco Elver /* Test both sides. */ 336bc8fbc5fSMarco Elver 337bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT); 338bc8fbc5fSMarco Elver expect.addr = buf - 1; 339bc8fbc5fSMarco Elver READ_ONCE(*expect.addr); 340bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 341bc8fbc5fSMarco Elver test_free(buf); 342bc8fbc5fSMarco Elver 343bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT); 344bc8fbc5fSMarco Elver expect.addr = buf + size; 345bc8fbc5fSMarco Elver READ_ONCE(*expect.addr); 346bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 347bc8fbc5fSMarco Elver test_free(buf); 348bc8fbc5fSMarco Elver } 349bc8fbc5fSMarco Elver 350bc8fbc5fSMarco Elver static void test_out_of_bounds_write(struct kunit *test) 351bc8fbc5fSMarco Elver { 352bc8fbc5fSMarco Elver size_t size = 32; 353bc8fbc5fSMarco Elver struct expect_report expect = { 354bc8fbc5fSMarco Elver .type = KFENCE_ERROR_OOB, 355bc8fbc5fSMarco Elver .fn = test_out_of_bounds_write, 356bc8fbc5fSMarco Elver .is_write = true, 357bc8fbc5fSMarco Elver }; 358bc8fbc5fSMarco Elver char *buf; 359bc8fbc5fSMarco Elver 360bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 361bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT); 362bc8fbc5fSMarco Elver expect.addr = buf - 1; 363bc8fbc5fSMarco Elver WRITE_ONCE(*expect.addr, 42); 364bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 365bc8fbc5fSMarco Elver test_free(buf); 366bc8fbc5fSMarco Elver } 367bc8fbc5fSMarco Elver 368bc8fbc5fSMarco Elver static void test_use_after_free_read(struct kunit *test) 369bc8fbc5fSMarco Elver { 370bc8fbc5fSMarco Elver const size_t size = 32; 371bc8fbc5fSMarco Elver struct expect_report expect = { 372bc8fbc5fSMarco Elver .type = KFENCE_ERROR_UAF, 373bc8fbc5fSMarco Elver .fn = test_use_after_free_read, 374bc8fbc5fSMarco Elver .is_write = false, 375bc8fbc5fSMarco Elver }; 376bc8fbc5fSMarco Elver 377bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 378bc8fbc5fSMarco Elver expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 379bc8fbc5fSMarco Elver test_free(expect.addr); 380bc8fbc5fSMarco Elver READ_ONCE(*expect.addr); 381bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 382bc8fbc5fSMarco Elver } 383bc8fbc5fSMarco Elver 384bc8fbc5fSMarco Elver static void test_double_free(struct kunit *test) 385bc8fbc5fSMarco Elver { 386bc8fbc5fSMarco Elver const size_t size = 32; 387bc8fbc5fSMarco Elver struct expect_report expect = { 388bc8fbc5fSMarco Elver .type = KFENCE_ERROR_INVALID_FREE, 389bc8fbc5fSMarco Elver .fn = test_double_free, 390bc8fbc5fSMarco Elver }; 391bc8fbc5fSMarco Elver 392bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 393bc8fbc5fSMarco Elver expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 394bc8fbc5fSMarco Elver test_free(expect.addr); 395bc8fbc5fSMarco Elver test_free(expect.addr); /* Double-free. */ 396bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 397bc8fbc5fSMarco Elver } 398bc8fbc5fSMarco Elver 399bc8fbc5fSMarco Elver static void test_invalid_addr_free(struct kunit *test) 400bc8fbc5fSMarco Elver { 401bc8fbc5fSMarco Elver const size_t size = 32; 402bc8fbc5fSMarco Elver struct expect_report expect = { 403bc8fbc5fSMarco Elver .type = KFENCE_ERROR_INVALID_FREE, 404bc8fbc5fSMarco Elver .fn = test_invalid_addr_free, 405bc8fbc5fSMarco Elver }; 406bc8fbc5fSMarco Elver char *buf; 407bc8fbc5fSMarco Elver 408bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 409bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 410bc8fbc5fSMarco Elver expect.addr = buf + 1; /* Free on invalid address. */ 411bc8fbc5fSMarco Elver test_free(expect.addr); /* Invalid address free. */ 412bc8fbc5fSMarco Elver test_free(buf); /* No error. */ 413bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 414bc8fbc5fSMarco Elver } 415bc8fbc5fSMarco Elver 416bc8fbc5fSMarco Elver static void test_corruption(struct kunit *test) 417bc8fbc5fSMarco Elver { 418bc8fbc5fSMarco Elver size_t size = 32; 419bc8fbc5fSMarco Elver struct expect_report expect = { 420bc8fbc5fSMarco Elver .type = KFENCE_ERROR_CORRUPTION, 421bc8fbc5fSMarco Elver .fn = test_corruption, 422bc8fbc5fSMarco Elver }; 423bc8fbc5fSMarco Elver char *buf; 424bc8fbc5fSMarco Elver 425bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 426bc8fbc5fSMarco Elver 427bc8fbc5fSMarco Elver /* Test both sides. */ 428bc8fbc5fSMarco Elver 429bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT); 430bc8fbc5fSMarco Elver expect.addr = buf + size; 431bc8fbc5fSMarco Elver WRITE_ONCE(*expect.addr, 42); 432bc8fbc5fSMarco Elver test_free(buf); 433bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 434bc8fbc5fSMarco Elver 435bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT); 436bc8fbc5fSMarco Elver expect.addr = buf - 1; 437bc8fbc5fSMarco Elver WRITE_ONCE(*expect.addr, 42); 438bc8fbc5fSMarco Elver test_free(buf); 439bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 440bc8fbc5fSMarco Elver } 441bc8fbc5fSMarco Elver 442bc8fbc5fSMarco Elver /* 443bc8fbc5fSMarco Elver * KFENCE is unable to detect an OOB if the allocation's alignment requirements 444bc8fbc5fSMarco Elver * leave a gap between the object and the guard page. Specifically, an 445bc8fbc5fSMarco Elver * allocation of e.g. 73 bytes is aligned on 8 and 128 bytes for SLUB or SLAB 446bc8fbc5fSMarco Elver * respectively. Therefore it is impossible for the allocated object to 447bc8fbc5fSMarco Elver * contiguously line up with the right guard page. 448bc8fbc5fSMarco Elver * 449bc8fbc5fSMarco Elver * However, we test that an access to memory beyond the gap results in KFENCE 450bc8fbc5fSMarco Elver * detecting an OOB access. 451bc8fbc5fSMarco Elver */ 452bc8fbc5fSMarco Elver static void test_kmalloc_aligned_oob_read(struct kunit *test) 453bc8fbc5fSMarco Elver { 454bc8fbc5fSMarco Elver const size_t size = 73; 455bc8fbc5fSMarco Elver const size_t align = kmalloc_cache_alignment(size); 456bc8fbc5fSMarco Elver struct expect_report expect = { 457bc8fbc5fSMarco Elver .type = KFENCE_ERROR_OOB, 458bc8fbc5fSMarco Elver .fn = test_kmalloc_aligned_oob_read, 459bc8fbc5fSMarco Elver .is_write = false, 460bc8fbc5fSMarco Elver }; 461bc8fbc5fSMarco Elver char *buf; 462bc8fbc5fSMarco Elver 463bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT); 464bc8fbc5fSMarco Elver 465bc8fbc5fSMarco Elver /* 466bc8fbc5fSMarco Elver * The object is offset to the right, so there won't be an OOB to the 467bc8fbc5fSMarco Elver * left of it. 468bc8fbc5fSMarco Elver */ 469bc8fbc5fSMarco Elver READ_ONCE(*(buf - 1)); 470bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 471bc8fbc5fSMarco Elver 472bc8fbc5fSMarco Elver /* 473bc8fbc5fSMarco Elver * @buf must be aligned on @align, therefore buf + size belongs to the 474bc8fbc5fSMarco Elver * same page -> no OOB. 475bc8fbc5fSMarco Elver */ 476bc8fbc5fSMarco Elver READ_ONCE(*(buf + size)); 477bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 478bc8fbc5fSMarco Elver 479bc8fbc5fSMarco Elver /* Overflowing by @align bytes will result in an OOB. */ 480bc8fbc5fSMarco Elver expect.addr = buf + size + align; 481bc8fbc5fSMarco Elver READ_ONCE(*expect.addr); 482bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 483bc8fbc5fSMarco Elver 484bc8fbc5fSMarco Elver test_free(buf); 485bc8fbc5fSMarco Elver } 486bc8fbc5fSMarco Elver 487bc8fbc5fSMarco Elver static void test_kmalloc_aligned_oob_write(struct kunit *test) 488bc8fbc5fSMarco Elver { 489bc8fbc5fSMarco Elver const size_t size = 73; 490bc8fbc5fSMarco Elver struct expect_report expect = { 491bc8fbc5fSMarco Elver .type = KFENCE_ERROR_CORRUPTION, 492bc8fbc5fSMarco Elver .fn = test_kmalloc_aligned_oob_write, 493bc8fbc5fSMarco Elver }; 494bc8fbc5fSMarco Elver char *buf; 495bc8fbc5fSMarco Elver 496bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT); 497bc8fbc5fSMarco Elver /* 498bc8fbc5fSMarco Elver * The object is offset to the right, so we won't get a page 499bc8fbc5fSMarco Elver * fault immediately after it. 500bc8fbc5fSMarco Elver */ 501bc8fbc5fSMarco Elver expect.addr = buf + size; 502bc8fbc5fSMarco Elver WRITE_ONCE(*expect.addr, READ_ONCE(*expect.addr) + 1); 503bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 504bc8fbc5fSMarco Elver test_free(buf); 505bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 506bc8fbc5fSMarco Elver } 507bc8fbc5fSMarco Elver 508bc8fbc5fSMarco Elver /* Test cache shrinking and destroying with KFENCE. */ 509bc8fbc5fSMarco Elver static void test_shrink_memcache(struct kunit *test) 510bc8fbc5fSMarco Elver { 511bc8fbc5fSMarco Elver const size_t size = 32; 512bc8fbc5fSMarco Elver void *buf; 513bc8fbc5fSMarco Elver 514bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 515bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, test_cache); 516bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 517bc8fbc5fSMarco Elver kmem_cache_shrink(test_cache); 518bc8fbc5fSMarco Elver test_free(buf); 519bc8fbc5fSMarco Elver 520bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 521bc8fbc5fSMarco Elver } 522bc8fbc5fSMarco Elver 523bc8fbc5fSMarco Elver static void ctor_set_x(void *obj) 524bc8fbc5fSMarco Elver { 525bc8fbc5fSMarco Elver /* Every object has at least 8 bytes. */ 526bc8fbc5fSMarco Elver memset(obj, 'x', 8); 527bc8fbc5fSMarco Elver } 528bc8fbc5fSMarco Elver 529bc8fbc5fSMarco Elver /* Ensure that SL*B does not modify KFENCE objects on bulk free. */ 530bc8fbc5fSMarco Elver static void test_free_bulk(struct kunit *test) 531bc8fbc5fSMarco Elver { 532bc8fbc5fSMarco Elver int iter; 533bc8fbc5fSMarco Elver 534bc8fbc5fSMarco Elver for (iter = 0; iter < 5; iter++) { 535*e8a533cbSJason A. Donenfeld const size_t size = setup_test_cache(test, get_random_u32_inclusive(8, 307), 5368032bf12SJason A. Donenfeld 0, (iter & 1) ? ctor_set_x : NULL); 537bc8fbc5fSMarco Elver void *objects[] = { 538bc8fbc5fSMarco Elver test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT), 539bc8fbc5fSMarco Elver test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE), 540bc8fbc5fSMarco Elver test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT), 541bc8fbc5fSMarco Elver test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE), 542bc8fbc5fSMarco Elver test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE), 543bc8fbc5fSMarco Elver }; 544bc8fbc5fSMarco Elver 545bc8fbc5fSMarco Elver kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects); 546bc8fbc5fSMarco Elver KUNIT_ASSERT_FALSE(test, report_available()); 547bc8fbc5fSMarco Elver test_cache_destroy(); 548bc8fbc5fSMarco Elver } 549bc8fbc5fSMarco Elver } 550bc8fbc5fSMarco Elver 551bc8fbc5fSMarco Elver /* Test init-on-free works. */ 552bc8fbc5fSMarco Elver static void test_init_on_free(struct kunit *test) 553bc8fbc5fSMarco Elver { 554bc8fbc5fSMarco Elver const size_t size = 32; 555bc8fbc5fSMarco Elver struct expect_report expect = { 556bc8fbc5fSMarco Elver .type = KFENCE_ERROR_UAF, 557bc8fbc5fSMarco Elver .fn = test_init_on_free, 558bc8fbc5fSMarco Elver .is_write = false, 559bc8fbc5fSMarco Elver }; 560bc8fbc5fSMarco Elver int i; 561bc8fbc5fSMarco Elver 562f51733e2SMarco Elver KFENCE_TEST_REQUIRES(test, IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON)); 563bc8fbc5fSMarco Elver /* Assume it hasn't been disabled on command line. */ 564bc8fbc5fSMarco Elver 565bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 566bc8fbc5fSMarco Elver expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 567bc8fbc5fSMarco Elver for (i = 0; i < size; i++) 568bc8fbc5fSMarco Elver expect.addr[i] = i + 1; 569bc8fbc5fSMarco Elver test_free(expect.addr); 570bc8fbc5fSMarco Elver 571bc8fbc5fSMarco Elver for (i = 0; i < size; i++) { 572bc8fbc5fSMarco Elver /* 573bc8fbc5fSMarco Elver * This may fail if the page was recycled by KFENCE and then 574bc8fbc5fSMarco Elver * written to again -- this however, is near impossible with a 575bc8fbc5fSMarco Elver * default config. 576bc8fbc5fSMarco Elver */ 577bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, expect.addr[i], (char)0); 578bc8fbc5fSMarco Elver 579bc8fbc5fSMarco Elver if (!i) /* Only check first access to not fail test if page is ever re-protected. */ 580bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 581bc8fbc5fSMarco Elver } 582bc8fbc5fSMarco Elver } 583bc8fbc5fSMarco Elver 584bc8fbc5fSMarco Elver /* Ensure that constructors work properly. */ 585bc8fbc5fSMarco Elver static void test_memcache_ctor(struct kunit *test) 586bc8fbc5fSMarco Elver { 587bc8fbc5fSMarco Elver const size_t size = 32; 588bc8fbc5fSMarco Elver char *buf; 589bc8fbc5fSMarco Elver int i; 590bc8fbc5fSMarco Elver 591bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, ctor_set_x); 592bc8fbc5fSMarco Elver buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 593bc8fbc5fSMarco Elver 594bc8fbc5fSMarco Elver for (i = 0; i < 8; i++) 595bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, buf[i], (char)'x'); 596bc8fbc5fSMarco Elver 597bc8fbc5fSMarco Elver test_free(buf); 598bc8fbc5fSMarco Elver 599bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 600bc8fbc5fSMarco Elver } 601bc8fbc5fSMarco Elver 602bc8fbc5fSMarco Elver /* Test that memory is zeroed if requested. */ 603bc8fbc5fSMarco Elver static void test_gfpzero(struct kunit *test) 604bc8fbc5fSMarco Elver { 605bc8fbc5fSMarco Elver const size_t size = PAGE_SIZE; /* PAGE_SIZE so we can use ALLOCATE_ANY. */ 606bc8fbc5fSMarco Elver char *buf1, *buf2; 607bc8fbc5fSMarco Elver int i; 608bc8fbc5fSMarco Elver 609f51733e2SMarco Elver /* Skip if we think it'd take too long. */ 6108913c610SPeng Liu KFENCE_TEST_REQUIRES(test, kfence_sample_interval <= 100); 611bc8fbc5fSMarco Elver 612bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 613bc8fbc5fSMarco Elver buf1 = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 614bc8fbc5fSMarco Elver for (i = 0; i < size; i++) 615bc8fbc5fSMarco Elver buf1[i] = i + 1; 616bc8fbc5fSMarco Elver test_free(buf1); 617bc8fbc5fSMarco Elver 618bc8fbc5fSMarco Elver /* Try to get same address again -- this can take a while. */ 619bc8fbc5fSMarco Elver for (i = 0;; i++) { 620bc8fbc5fSMarco Elver buf2 = test_alloc(test, size, GFP_KERNEL | __GFP_ZERO, ALLOCATE_ANY); 621bc8fbc5fSMarco Elver if (buf1 == buf2) 622bc8fbc5fSMarco Elver break; 623bc8fbc5fSMarco Elver test_free(buf2); 624bc8fbc5fSMarco Elver 625adf50545SPeng Liu if (kthread_should_stop() || (i == CONFIG_KFENCE_NUM_OBJECTS)) { 626bc8fbc5fSMarco Elver kunit_warn(test, "giving up ... cannot get same object back\n"); 627bc8fbc5fSMarco Elver return; 628bc8fbc5fSMarco Elver } 6293cb1c962SPeng Liu cond_resched(); 630bc8fbc5fSMarco Elver } 631bc8fbc5fSMarco Elver 632bc8fbc5fSMarco Elver for (i = 0; i < size; i++) 633bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, buf2[i], (char)0); 634bc8fbc5fSMarco Elver 635bc8fbc5fSMarco Elver test_free(buf2); 636bc8fbc5fSMarco Elver 637bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 638bc8fbc5fSMarco Elver } 639bc8fbc5fSMarco Elver 640bc8fbc5fSMarco Elver static void test_invalid_access(struct kunit *test) 641bc8fbc5fSMarco Elver { 642bc8fbc5fSMarco Elver const struct expect_report expect = { 643bc8fbc5fSMarco Elver .type = KFENCE_ERROR_INVALID, 644bc8fbc5fSMarco Elver .fn = test_invalid_access, 645bc8fbc5fSMarco Elver .addr = &__kfence_pool[10], 646bc8fbc5fSMarco Elver .is_write = false, 647bc8fbc5fSMarco Elver }; 648bc8fbc5fSMarco Elver 649bc8fbc5fSMarco Elver READ_ONCE(__kfence_pool[10]); 650bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 651bc8fbc5fSMarco Elver } 652bc8fbc5fSMarco Elver 653bc8fbc5fSMarco Elver /* Test SLAB_TYPESAFE_BY_RCU works. */ 654bc8fbc5fSMarco Elver static void test_memcache_typesafe_by_rcu(struct kunit *test) 655bc8fbc5fSMarco Elver { 656bc8fbc5fSMarco Elver const size_t size = 32; 657bc8fbc5fSMarco Elver struct expect_report expect = { 658bc8fbc5fSMarco Elver .type = KFENCE_ERROR_UAF, 659bc8fbc5fSMarco Elver .fn = test_memcache_typesafe_by_rcu, 660bc8fbc5fSMarco Elver .is_write = false, 661bc8fbc5fSMarco Elver }; 662bc8fbc5fSMarco Elver 663bc8fbc5fSMarco Elver setup_test_cache(test, size, SLAB_TYPESAFE_BY_RCU, NULL); 664bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */ 665bc8fbc5fSMarco Elver 666bc8fbc5fSMarco Elver expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY); 667bc8fbc5fSMarco Elver *expect.addr = 42; 668bc8fbc5fSMarco Elver 669bc8fbc5fSMarco Elver rcu_read_lock(); 670bc8fbc5fSMarco Elver test_free(expect.addr); 671bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, *expect.addr, (char)42); 672bc8fbc5fSMarco Elver /* 673bc8fbc5fSMarco Elver * Up to this point, memory should not have been freed yet, and 674bc8fbc5fSMarco Elver * therefore there should be no KFENCE report from the above access. 675bc8fbc5fSMarco Elver */ 676bc8fbc5fSMarco Elver rcu_read_unlock(); 677bc8fbc5fSMarco Elver 678bc8fbc5fSMarco Elver /* Above access to @expect.addr should not have generated a report! */ 679bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 680bc8fbc5fSMarco Elver 681bc8fbc5fSMarco Elver /* Only after rcu_barrier() is the memory guaranteed to be freed. */ 682bc8fbc5fSMarco Elver rcu_barrier(); 683bc8fbc5fSMarco Elver 684bc8fbc5fSMarco Elver /* Expect use-after-free. */ 685bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, *expect.addr, (char)42); 686bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, report_matches(&expect)); 687bc8fbc5fSMarco Elver } 688bc8fbc5fSMarco Elver 689bc8fbc5fSMarco Elver /* Test krealloc(). */ 690bc8fbc5fSMarco Elver static void test_krealloc(struct kunit *test) 691bc8fbc5fSMarco Elver { 692bc8fbc5fSMarco Elver const size_t size = 32; 693bc8fbc5fSMarco Elver const struct expect_report expect = { 694bc8fbc5fSMarco Elver .type = KFENCE_ERROR_UAF, 695bc8fbc5fSMarco Elver .fn = test_krealloc, 696bc8fbc5fSMarco Elver .addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY), 697bc8fbc5fSMarco Elver .is_write = false, 698bc8fbc5fSMarco Elver }; 699bc8fbc5fSMarco Elver char *buf = expect.addr; 700bc8fbc5fSMarco Elver int i; 701bc8fbc5fSMarco Elver 702bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, test_cache); 703bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, ksize(buf), size); /* Precise size match after KFENCE alloc. */ 704bc8fbc5fSMarco Elver for (i = 0; i < size; i++) 705bc8fbc5fSMarco Elver buf[i] = i + 1; 706bc8fbc5fSMarco Elver 707bc8fbc5fSMarco Elver /* Check that we successfully change the size. */ 708bc8fbc5fSMarco Elver buf = krealloc(buf, size * 3, GFP_KERNEL); /* Grow. */ 709bc8fbc5fSMarco Elver /* Note: Might no longer be a KFENCE alloc. */ 710bc8fbc5fSMarco Elver KUNIT_EXPECT_GE(test, ksize(buf), size * 3); 711bc8fbc5fSMarco Elver for (i = 0; i < size; i++) 712bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1)); 713bc8fbc5fSMarco Elver for (; i < size * 3; i++) /* Fill to extra bytes. */ 714bc8fbc5fSMarco Elver buf[i] = i + 1; 715bc8fbc5fSMarco Elver 716bc8fbc5fSMarco Elver buf = krealloc(buf, size * 2, GFP_KERNEL); /* Shrink. */ 717bc8fbc5fSMarco Elver KUNIT_EXPECT_GE(test, ksize(buf), size * 2); 718bc8fbc5fSMarco Elver for (i = 0; i < size * 2; i++) 719bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1)); 720bc8fbc5fSMarco Elver 721bc8fbc5fSMarco Elver buf = krealloc(buf, 0, GFP_KERNEL); /* Free. */ 722bc8fbc5fSMarco Elver KUNIT_EXPECT_EQ(test, (unsigned long)buf, (unsigned long)ZERO_SIZE_PTR); 723bc8fbc5fSMarco Elver KUNIT_ASSERT_FALSE(test, report_available()); /* No reports yet! */ 724bc8fbc5fSMarco Elver 725bc8fbc5fSMarco Elver READ_ONCE(*expect.addr); /* Ensure krealloc() actually freed earlier KFENCE object. */ 726bc8fbc5fSMarco Elver KUNIT_ASSERT_TRUE(test, report_matches(&expect)); 727bc8fbc5fSMarco Elver } 728bc8fbc5fSMarco Elver 729bc8fbc5fSMarco Elver /* Test that some objects from a bulk allocation belong to KFENCE pool. */ 730bc8fbc5fSMarco Elver static void test_memcache_alloc_bulk(struct kunit *test) 731bc8fbc5fSMarco Elver { 732bc8fbc5fSMarco Elver const size_t size = 32; 733bc8fbc5fSMarco Elver bool pass = false; 734bc8fbc5fSMarco Elver unsigned long timeout; 735bc8fbc5fSMarco Elver 736bc8fbc5fSMarco Elver setup_test_cache(test, size, 0, NULL); 737bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */ 738bc8fbc5fSMarco Elver /* 739bc8fbc5fSMarco Elver * 100x the sample interval should be more than enough to ensure we get 740bc8fbc5fSMarco Elver * a KFENCE allocation eventually. 741bc8fbc5fSMarco Elver */ 7428913c610SPeng Liu timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval); 743bc8fbc5fSMarco Elver do { 744bc8fbc5fSMarco Elver void *objects[100]; 745bc8fbc5fSMarco Elver int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects), 746bc8fbc5fSMarco Elver objects); 747bc8fbc5fSMarco Elver if (!num) 748bc8fbc5fSMarco Elver continue; 749bc8fbc5fSMarco Elver for (i = 0; i < ARRAY_SIZE(objects); i++) { 750bc8fbc5fSMarco Elver if (is_kfence_address(objects[i])) { 751bc8fbc5fSMarco Elver pass = true; 752bc8fbc5fSMarco Elver break; 753bc8fbc5fSMarco Elver } 754bc8fbc5fSMarco Elver } 755bc8fbc5fSMarco Elver kmem_cache_free_bulk(test_cache, num, objects); 756bc8fbc5fSMarco Elver /* 757bc8fbc5fSMarco Elver * kmem_cache_alloc_bulk() disables interrupts, and calling it 758bc8fbc5fSMarco Elver * in a tight loop may not give KFENCE a chance to switch the 759bc8fbc5fSMarco Elver * static branch. Call cond_resched() to let KFENCE chime in. 760bc8fbc5fSMarco Elver */ 761bc8fbc5fSMarco Elver cond_resched(); 762bc8fbc5fSMarco Elver } while (!pass && time_before(jiffies, timeout)); 763bc8fbc5fSMarco Elver 764bc8fbc5fSMarco Elver KUNIT_EXPECT_TRUE(test, pass); 765bc8fbc5fSMarco Elver KUNIT_EXPECT_FALSE(test, report_available()); 766bc8fbc5fSMarco Elver } 767bc8fbc5fSMarco Elver 768bc8fbc5fSMarco Elver /* 769bc8fbc5fSMarco Elver * KUnit does not provide a way to provide arguments to tests, and we encode 770bc8fbc5fSMarco Elver * additional info in the name. Set up 2 tests per test case, one using the 771bc8fbc5fSMarco Elver * default allocator, and another using a custom memcache (suffix '-memcache'). 772bc8fbc5fSMarco Elver */ 773bc8fbc5fSMarco Elver #define KFENCE_KUNIT_CASE(test_name) \ 774bc8fbc5fSMarco Elver { .run_case = test_name, .name = #test_name }, \ 775bc8fbc5fSMarco Elver { .run_case = test_name, .name = #test_name "-memcache" } 776bc8fbc5fSMarco Elver 777bc8fbc5fSMarco Elver static struct kunit_case kfence_test_cases[] = { 778bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_out_of_bounds_read), 779bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_out_of_bounds_write), 780bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_use_after_free_read), 781bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_double_free), 782bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_invalid_addr_free), 783bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_corruption), 784bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_free_bulk), 785bc8fbc5fSMarco Elver KFENCE_KUNIT_CASE(test_init_on_free), 786bc8fbc5fSMarco Elver KUNIT_CASE(test_kmalloc_aligned_oob_read), 787bc8fbc5fSMarco Elver KUNIT_CASE(test_kmalloc_aligned_oob_write), 788bc8fbc5fSMarco Elver KUNIT_CASE(test_shrink_memcache), 789bc8fbc5fSMarco Elver KUNIT_CASE(test_memcache_ctor), 790bc8fbc5fSMarco Elver KUNIT_CASE(test_invalid_access), 791bc8fbc5fSMarco Elver KUNIT_CASE(test_gfpzero), 792bc8fbc5fSMarco Elver KUNIT_CASE(test_memcache_typesafe_by_rcu), 793bc8fbc5fSMarco Elver KUNIT_CASE(test_krealloc), 794bc8fbc5fSMarco Elver KUNIT_CASE(test_memcache_alloc_bulk), 795bc8fbc5fSMarco Elver {}, 796bc8fbc5fSMarco Elver }; 797bc8fbc5fSMarco Elver 798bc8fbc5fSMarco Elver /* ===== End test cases ===== */ 799bc8fbc5fSMarco Elver 800bc8fbc5fSMarco Elver static int test_init(struct kunit *test) 801bc8fbc5fSMarco Elver { 802bc8fbc5fSMarco Elver unsigned long flags; 803bc8fbc5fSMarco Elver int i; 804bc8fbc5fSMarco Elver 805c40c6e59SMarco Elver if (!__kfence_pool) 806c40c6e59SMarco Elver return -EINVAL; 807c40c6e59SMarco Elver 808bc8fbc5fSMarco Elver spin_lock_irqsave(&observed.lock, flags); 809bc8fbc5fSMarco Elver for (i = 0; i < ARRAY_SIZE(observed.lines); i++) 810bc8fbc5fSMarco Elver observed.lines[i][0] = '\0'; 811bc8fbc5fSMarco Elver observed.nlines = 0; 812bc8fbc5fSMarco Elver spin_unlock_irqrestore(&observed.lock, flags); 813bc8fbc5fSMarco Elver 814bc8fbc5fSMarco Elver /* Any test with 'memcache' in its name will want a memcache. */ 815bc8fbc5fSMarco Elver if (strstr(test->name, "memcache")) 816bc8fbc5fSMarco Elver test->priv = TEST_PRIV_WANT_MEMCACHE; 817bc8fbc5fSMarco Elver else 818bc8fbc5fSMarco Elver test->priv = NULL; 819bc8fbc5fSMarco Elver 820bc8fbc5fSMarco Elver return 0; 821bc8fbc5fSMarco Elver } 822bc8fbc5fSMarco Elver 823bc8fbc5fSMarco Elver static void test_exit(struct kunit *test) 824bc8fbc5fSMarco Elver { 825bc8fbc5fSMarco Elver test_cache_destroy(); 826bc8fbc5fSMarco Elver } 827bc8fbc5fSMarco Elver 828bc8fbc5fSMarco Elver static void register_tracepoints(struct tracepoint *tp, void *ignore) 829bc8fbc5fSMarco Elver { 830bc8fbc5fSMarco Elver check_trace_callback_type_console(probe_console); 831bc8fbc5fSMarco Elver if (!strcmp(tp->name, "console")) 832bc8fbc5fSMarco Elver WARN_ON(tracepoint_probe_register(tp, probe_console, NULL)); 833bc8fbc5fSMarco Elver } 834bc8fbc5fSMarco Elver 835bc8fbc5fSMarco Elver static void unregister_tracepoints(struct tracepoint *tp, void *ignore) 836bc8fbc5fSMarco Elver { 837bc8fbc5fSMarco Elver if (!strcmp(tp->name, "console")) 838bc8fbc5fSMarco Elver tracepoint_probe_unregister(tp, probe_console, NULL); 839bc8fbc5fSMarco Elver } 840bc8fbc5fSMarco Elver 8413b91f826SDaniel Latypov static int kfence_suite_init(struct kunit_suite *suite) 842bc8fbc5fSMarco Elver { 843bc8fbc5fSMarco Elver /* 844bc8fbc5fSMarco Elver * Because we want to be able to build the test as a module, we need to 845bc8fbc5fSMarco Elver * iterate through all known tracepoints, since the static registration 846bc8fbc5fSMarco Elver * won't work here. 847bc8fbc5fSMarco Elver */ 848bc8fbc5fSMarco Elver for_each_kernel_tracepoint(register_tracepoints, NULL); 8493b91f826SDaniel Latypov return 0; 850bc8fbc5fSMarco Elver } 851bc8fbc5fSMarco Elver 8523b91f826SDaniel Latypov static void kfence_suite_exit(struct kunit_suite *suite) 853bc8fbc5fSMarco Elver { 854bc8fbc5fSMarco Elver for_each_kernel_tracepoint(unregister_tracepoints, NULL); 855bc8fbc5fSMarco Elver tracepoint_synchronize_unregister(); 856bc8fbc5fSMarco Elver } 857bc8fbc5fSMarco Elver 8583b91f826SDaniel Latypov static struct kunit_suite kfence_test_suite = { 8593b91f826SDaniel Latypov .name = "kfence", 8603b91f826SDaniel Latypov .test_cases = kfence_test_cases, 8613b91f826SDaniel Latypov .init = test_init, 8623b91f826SDaniel Latypov .exit = test_exit, 8633b91f826SDaniel Latypov .suite_init = kfence_suite_init, 8643b91f826SDaniel Latypov .suite_exit = kfence_suite_exit, 8653b91f826SDaniel Latypov }; 8663b91f826SDaniel Latypov 8673b91f826SDaniel Latypov kunit_test_suites(&kfence_test_suite); 868bc8fbc5fSMarco Elver 869bc8fbc5fSMarco Elver MODULE_LICENSE("GPL v2"); 870bc8fbc5fSMarco Elver MODULE_AUTHOR("Alexander Potapenko <glider@google.com>, Marco Elver <elver@google.com>"); 871