cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

kfence_test.c (24416B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * Test cases for KFENCE memory safety error detector. Since the interface with
      4 * which KFENCE's reports are obtained is via the console, this is the output we
      5 * should verify. For each test case checks the presence (or absence) of
      6 * generated reports. Relies on 'console' tracepoint to capture reports as they
      7 * appear in the kernel log.
      8 *
      9 * Copyright (C) 2020, Google LLC.
     10 * Author: Alexander Potapenko <glider@google.com>
     11 *         Marco Elver <elver@google.com>
     12 */
     13
     14#include <kunit/test.h>
     15#include <linux/jiffies.h>
     16#include <linux/kernel.h>
     17#include <linux/kfence.h>
     18#include <linux/mm.h>
     19#include <linux/random.h>
     20#include <linux/slab.h>
     21#include <linux/spinlock.h>
     22#include <linux/string.h>
     23#include <linux/tracepoint.h>
     24#include <trace/events/printk.h>
     25
     26#include <asm/kfence.h>
     27
     28#include "kfence.h"
     29
     30/* May be overridden by <asm/kfence.h>. */
     31#ifndef arch_kfence_test_address
     32#define arch_kfence_test_address(addr) (addr)
     33#endif
     34
     35#define KFENCE_TEST_REQUIRES(test, cond) do {			\
     36	if (!(cond))						\
     37		kunit_skip((test), "Test requires: " #cond);	\
     38} while (0)
     39
     40/* Report as observed from console. */
     41static struct {
     42	spinlock_t lock;
     43	int nlines;
     44	char lines[2][256];
     45} observed = {
     46	.lock = __SPIN_LOCK_UNLOCKED(observed.lock),
     47};
     48
     49/* Probe for console output: obtains observed lines of interest. */
     50static void probe_console(void *ignore, const char *buf, size_t len)
     51{
     52	unsigned long flags;
     53	int nlines;
     54
     55	spin_lock_irqsave(&observed.lock, flags);
     56	nlines = observed.nlines;
     57
     58	if (strnstr(buf, "BUG: KFENCE: ", len) && strnstr(buf, "test_", len)) {
     59		/*
     60		 * KFENCE report and related to the test.
     61		 *
     62		 * The provided @buf is not NUL-terminated; copy no more than
     63		 * @len bytes and let strscpy() add the missing NUL-terminator.
     64		 */
     65		strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
     66		nlines = 1;
     67	} else if (nlines == 1 && (strnstr(buf, "at 0x", len) || strnstr(buf, "of 0x", len))) {
     68		strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
     69	}
     70
     71	WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
     72	spin_unlock_irqrestore(&observed.lock, flags);
     73}
     74
     75/* Check if a report related to the test exists. */
     76static bool report_available(void)
     77{
     78	return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
     79}
     80
     81/* Information we expect in a report. */
     82struct expect_report {
     83	enum kfence_error_type type; /* The type or error. */
     84	void *fn; /* Function pointer to expected function where access occurred. */
     85	char *addr; /* Address at which the bad access occurred. */
     86	bool is_write; /* Is access a write. */
     87};
     88
     89static const char *get_access_type(const struct expect_report *r)
     90{
     91	return r->is_write ? "write" : "read";
     92}
     93
     94/* Check observed report matches information in @r. */
     95static bool report_matches(const struct expect_report *r)
     96{
     97	unsigned long addr = (unsigned long)r->addr;
     98	bool ret = false;
     99	unsigned long flags;
    100	typeof(observed.lines) expect;
    101	const char *end;
    102	char *cur;
    103
    104	/* Doubled-checked locking. */
    105	if (!report_available())
    106		return false;
    107
    108	/* Generate expected report contents. */
    109
    110	/* Title */
    111	cur = expect[0];
    112	end = &expect[0][sizeof(expect[0]) - 1];
    113	switch (r->type) {
    114	case KFENCE_ERROR_OOB:
    115		cur += scnprintf(cur, end - cur, "BUG: KFENCE: out-of-bounds %s",
    116				 get_access_type(r));
    117		break;
    118	case KFENCE_ERROR_UAF:
    119		cur += scnprintf(cur, end - cur, "BUG: KFENCE: use-after-free %s",
    120				 get_access_type(r));
    121		break;
    122	case KFENCE_ERROR_CORRUPTION:
    123		cur += scnprintf(cur, end - cur, "BUG: KFENCE: memory corruption");
    124		break;
    125	case KFENCE_ERROR_INVALID:
    126		cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid %s",
    127				 get_access_type(r));
    128		break;
    129	case KFENCE_ERROR_INVALID_FREE:
    130		cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid free");
    131		break;
    132	}
    133
    134	scnprintf(cur, end - cur, " in %pS", r->fn);
    135	/* The exact offset won't match, remove it; also strip module name. */
    136	cur = strchr(expect[0], '+');
    137	if (cur)
    138		*cur = '\0';
    139
    140	/* Access information */
    141	cur = expect[1];
    142	end = &expect[1][sizeof(expect[1]) - 1];
    143
    144	switch (r->type) {
    145	case KFENCE_ERROR_OOB:
    146		cur += scnprintf(cur, end - cur, "Out-of-bounds %s at", get_access_type(r));
    147		addr = arch_kfence_test_address(addr);
    148		break;
    149	case KFENCE_ERROR_UAF:
    150		cur += scnprintf(cur, end - cur, "Use-after-free %s at", get_access_type(r));
    151		addr = arch_kfence_test_address(addr);
    152		break;
    153	case KFENCE_ERROR_CORRUPTION:
    154		cur += scnprintf(cur, end - cur, "Corrupted memory at");
    155		break;
    156	case KFENCE_ERROR_INVALID:
    157		cur += scnprintf(cur, end - cur, "Invalid %s at", get_access_type(r));
    158		addr = arch_kfence_test_address(addr);
    159		break;
    160	case KFENCE_ERROR_INVALID_FREE:
    161		cur += scnprintf(cur, end - cur, "Invalid free of");
    162		break;
    163	}
    164
    165	cur += scnprintf(cur, end - cur, " 0x%p", (void *)addr);
    166
    167	spin_lock_irqsave(&observed.lock, flags);
    168	if (!report_available())
    169		goto out; /* A new report is being captured. */
    170
    171	/* Finally match expected output to what we actually observed. */
    172	ret = strstr(observed.lines[0], expect[0]) && strstr(observed.lines[1], expect[1]);
    173out:
    174	spin_unlock_irqrestore(&observed.lock, flags);
    175	return ret;
    176}
    177
    178/* ===== Test cases ===== */
    179
    180#define TEST_PRIV_WANT_MEMCACHE ((void *)1)
    181
    182/* Cache used by tests; if NULL, allocate from kmalloc instead. */
    183static struct kmem_cache *test_cache;
    184
    185static size_t setup_test_cache(struct kunit *test, size_t size, slab_flags_t flags,
    186			       void (*ctor)(void *))
    187{
    188	if (test->priv != TEST_PRIV_WANT_MEMCACHE)
    189		return size;
    190
    191	kunit_info(test, "%s: size=%zu, ctor=%ps\n", __func__, size, ctor);
    192
    193	/*
    194	 * Use SLAB_NOLEAKTRACE to prevent merging with existing caches. Any
    195	 * other flag in SLAB_NEVER_MERGE also works. Use SLAB_ACCOUNT to
    196	 * allocate via memcg, if enabled.
    197	 */
    198	flags |= SLAB_NOLEAKTRACE | SLAB_ACCOUNT;
    199	test_cache = kmem_cache_create("test", size, 1, flags, ctor);
    200	KUNIT_ASSERT_TRUE_MSG(test, test_cache, "could not create cache");
    201
    202	return size;
    203}
    204
    205static void test_cache_destroy(void)
    206{
    207	if (!test_cache)
    208		return;
    209
    210	kmem_cache_destroy(test_cache);
    211	test_cache = NULL;
    212}
    213
    214static inline size_t kmalloc_cache_alignment(size_t size)
    215{
    216	return kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)]->align;
    217}
    218
    219/* Must always inline to match stack trace against caller. */
    220static __always_inline void test_free(void *ptr)
    221{
    222	if (test_cache)
    223		kmem_cache_free(test_cache, ptr);
    224	else
    225		kfree(ptr);
    226}
    227
    228/*
    229 * If this should be a KFENCE allocation, and on which side the allocation and
    230 * the closest guard page should be.
    231 */
    232enum allocation_policy {
    233	ALLOCATE_ANY, /* KFENCE, any side. */
    234	ALLOCATE_LEFT, /* KFENCE, left side of page. */
    235	ALLOCATE_RIGHT, /* KFENCE, right side of page. */
    236	ALLOCATE_NONE, /* No KFENCE allocation. */
    237};
    238
    239/*
    240 * Try to get a guarded allocation from KFENCE. Uses either kmalloc() or the
    241 * current test_cache if set up.
    242 */
    243static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocation_policy policy)
    244{
    245	void *alloc;
    246	unsigned long timeout, resched_after;
    247	const char *policy_name;
    248
    249	switch (policy) {
    250	case ALLOCATE_ANY:
    251		policy_name = "any";
    252		break;
    253	case ALLOCATE_LEFT:
    254		policy_name = "left";
    255		break;
    256	case ALLOCATE_RIGHT:
    257		policy_name = "right";
    258		break;
    259	case ALLOCATE_NONE:
    260		policy_name = "none";
    261		break;
    262	}
    263
    264	kunit_info(test, "%s: size=%zu, gfp=%x, policy=%s, cache=%i\n", __func__, size, gfp,
    265		   policy_name, !!test_cache);
    266
    267	/*
    268	 * 100x the sample interval should be more than enough to ensure we get
    269	 * a KFENCE allocation eventually.
    270	 */
    271	timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval);
    272	/*
    273	 * Especially for non-preemption kernels, ensure the allocation-gate
    274	 * timer can catch up: after @resched_after, every failed allocation
    275	 * attempt yields, to ensure the allocation-gate timer is scheduled.
    276	 */
    277	resched_after = jiffies + msecs_to_jiffies(kfence_sample_interval);
    278	do {
    279		if (test_cache)
    280			alloc = kmem_cache_alloc(test_cache, gfp);
    281		else
    282			alloc = kmalloc(size, gfp);
    283
    284		if (is_kfence_address(alloc)) {
    285			struct slab *slab = virt_to_slab(alloc);
    286			struct kmem_cache *s = test_cache ?:
    287					kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)];
    288
    289			/*
    290			 * Verify that various helpers return the right values
    291			 * even for KFENCE objects; these are required so that
    292			 * memcg accounting works correctly.
    293			 */
    294			KUNIT_EXPECT_EQ(test, obj_to_index(s, slab, alloc), 0U);
    295			KUNIT_EXPECT_EQ(test, objs_per_slab(s, slab), 1);
    296
    297			if (policy == ALLOCATE_ANY)
    298				return alloc;
    299			if (policy == ALLOCATE_LEFT && PAGE_ALIGNED(alloc))
    300				return alloc;
    301			if (policy == ALLOCATE_RIGHT && !PAGE_ALIGNED(alloc))
    302				return alloc;
    303		} else if (policy == ALLOCATE_NONE)
    304			return alloc;
    305
    306		test_free(alloc);
    307
    308		if (time_after(jiffies, resched_after))
    309			cond_resched();
    310	} while (time_before(jiffies, timeout));
    311
    312	KUNIT_ASSERT_TRUE_MSG(test, false, "failed to allocate from KFENCE");
    313	return NULL; /* Unreachable. */
    314}
    315
    316static void test_out_of_bounds_read(struct kunit *test)
    317{
    318	size_t size = 32;
    319	struct expect_report expect = {
    320		.type = KFENCE_ERROR_OOB,
    321		.fn = test_out_of_bounds_read,
    322		.is_write = false,
    323	};
    324	char *buf;
    325
    326	setup_test_cache(test, size, 0, NULL);
    327
    328	/*
    329	 * If we don't have our own cache, adjust based on alignment, so that we
    330	 * actually access guard pages on either side.
    331	 */
    332	if (!test_cache)
    333		size = kmalloc_cache_alignment(size);
    334
    335	/* Test both sides. */
    336
    337	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
    338	expect.addr = buf - 1;
    339	READ_ONCE(*expect.addr);
    340	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    341	test_free(buf);
    342
    343	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
    344	expect.addr = buf + size;
    345	READ_ONCE(*expect.addr);
    346	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    347	test_free(buf);
    348}
    349
    350static void test_out_of_bounds_write(struct kunit *test)
    351{
    352	size_t size = 32;
    353	struct expect_report expect = {
    354		.type = KFENCE_ERROR_OOB,
    355		.fn = test_out_of_bounds_write,
    356		.is_write = true,
    357	};
    358	char *buf;
    359
    360	setup_test_cache(test, size, 0, NULL);
    361	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
    362	expect.addr = buf - 1;
    363	WRITE_ONCE(*expect.addr, 42);
    364	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    365	test_free(buf);
    366}
    367
    368static void test_use_after_free_read(struct kunit *test)
    369{
    370	const size_t size = 32;
    371	struct expect_report expect = {
    372		.type = KFENCE_ERROR_UAF,
    373		.fn = test_use_after_free_read,
    374		.is_write = false,
    375	};
    376
    377	setup_test_cache(test, size, 0, NULL);
    378	expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    379	test_free(expect.addr);
    380	READ_ONCE(*expect.addr);
    381	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    382}
    383
    384static void test_double_free(struct kunit *test)
    385{
    386	const size_t size = 32;
    387	struct expect_report expect = {
    388		.type = KFENCE_ERROR_INVALID_FREE,
    389		.fn = test_double_free,
    390	};
    391
    392	setup_test_cache(test, size, 0, NULL);
    393	expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    394	test_free(expect.addr);
    395	test_free(expect.addr); /* Double-free. */
    396	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    397}
    398
    399static void test_invalid_addr_free(struct kunit *test)
    400{
    401	const size_t size = 32;
    402	struct expect_report expect = {
    403		.type = KFENCE_ERROR_INVALID_FREE,
    404		.fn = test_invalid_addr_free,
    405	};
    406	char *buf;
    407
    408	setup_test_cache(test, size, 0, NULL);
    409	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    410	expect.addr = buf + 1; /* Free on invalid address. */
    411	test_free(expect.addr); /* Invalid address free. */
    412	test_free(buf); /* No error. */
    413	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    414}
    415
    416static void test_corruption(struct kunit *test)
    417{
    418	size_t size = 32;
    419	struct expect_report expect = {
    420		.type = KFENCE_ERROR_CORRUPTION,
    421		.fn = test_corruption,
    422	};
    423	char *buf;
    424
    425	setup_test_cache(test, size, 0, NULL);
    426
    427	/* Test both sides. */
    428
    429	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
    430	expect.addr = buf + size;
    431	WRITE_ONCE(*expect.addr, 42);
    432	test_free(buf);
    433	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    434
    435	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
    436	expect.addr = buf - 1;
    437	WRITE_ONCE(*expect.addr, 42);
    438	test_free(buf);
    439	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    440}
    441
    442/*
    443 * KFENCE is unable to detect an OOB if the allocation's alignment requirements
    444 * leave a gap between the object and the guard page. Specifically, an
    445 * allocation of e.g. 73 bytes is aligned on 8 and 128 bytes for SLUB or SLAB
    446 * respectively. Therefore it is impossible for the allocated object to
    447 * contiguously line up with the right guard page.
    448 *
    449 * However, we test that an access to memory beyond the gap results in KFENCE
    450 * detecting an OOB access.
    451 */
    452static void test_kmalloc_aligned_oob_read(struct kunit *test)
    453{
    454	const size_t size = 73;
    455	const size_t align = kmalloc_cache_alignment(size);
    456	struct expect_report expect = {
    457		.type = KFENCE_ERROR_OOB,
    458		.fn = test_kmalloc_aligned_oob_read,
    459		.is_write = false,
    460	};
    461	char *buf;
    462
    463	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
    464
    465	/*
    466	 * The object is offset to the right, so there won't be an OOB to the
    467	 * left of it.
    468	 */
    469	READ_ONCE(*(buf - 1));
    470	KUNIT_EXPECT_FALSE(test, report_available());
    471
    472	/*
    473	 * @buf must be aligned on @align, therefore buf + size belongs to the
    474	 * same page -> no OOB.
    475	 */
    476	READ_ONCE(*(buf + size));
    477	KUNIT_EXPECT_FALSE(test, report_available());
    478
    479	/* Overflowing by @align bytes will result in an OOB. */
    480	expect.addr = buf + size + align;
    481	READ_ONCE(*expect.addr);
    482	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    483
    484	test_free(buf);
    485}
    486
    487static void test_kmalloc_aligned_oob_write(struct kunit *test)
    488{
    489	const size_t size = 73;
    490	struct expect_report expect = {
    491		.type = KFENCE_ERROR_CORRUPTION,
    492		.fn = test_kmalloc_aligned_oob_write,
    493	};
    494	char *buf;
    495
    496	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
    497	/*
    498	 * The object is offset to the right, so we won't get a page
    499	 * fault immediately after it.
    500	 */
    501	expect.addr = buf + size;
    502	WRITE_ONCE(*expect.addr, READ_ONCE(*expect.addr) + 1);
    503	KUNIT_EXPECT_FALSE(test, report_available());
    504	test_free(buf);
    505	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    506}
    507
    508/* Test cache shrinking and destroying with KFENCE. */
    509static void test_shrink_memcache(struct kunit *test)
    510{
    511	const size_t size = 32;
    512	void *buf;
    513
    514	setup_test_cache(test, size, 0, NULL);
    515	KUNIT_EXPECT_TRUE(test, test_cache);
    516	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    517	kmem_cache_shrink(test_cache);
    518	test_free(buf);
    519
    520	KUNIT_EXPECT_FALSE(test, report_available());
    521}
    522
    523static void ctor_set_x(void *obj)
    524{
    525	/* Every object has at least 8 bytes. */
    526	memset(obj, 'x', 8);
    527}
    528
    529/* Ensure that SL*B does not modify KFENCE objects on bulk free. */
    530static void test_free_bulk(struct kunit *test)
    531{
    532	int iter;
    533
    534	for (iter = 0; iter < 5; iter++) {
    535		const size_t size = setup_test_cache(test, 8 + prandom_u32_max(300), 0,
    536						     (iter & 1) ? ctor_set_x : NULL);
    537		void *objects[] = {
    538			test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT),
    539			test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
    540			test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT),
    541			test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
    542			test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
    543		};
    544
    545		kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects);
    546		KUNIT_ASSERT_FALSE(test, report_available());
    547		test_cache_destroy();
    548	}
    549}
    550
    551/* Test init-on-free works. */
    552static void test_init_on_free(struct kunit *test)
    553{
    554	const size_t size = 32;
    555	struct expect_report expect = {
    556		.type = KFENCE_ERROR_UAF,
    557		.fn = test_init_on_free,
    558		.is_write = false,
    559	};
    560	int i;
    561
    562	KFENCE_TEST_REQUIRES(test, IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON));
    563	/* Assume it hasn't been disabled on command line. */
    564
    565	setup_test_cache(test, size, 0, NULL);
    566	expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    567	for (i = 0; i < size; i++)
    568		expect.addr[i] = i + 1;
    569	test_free(expect.addr);
    570
    571	for (i = 0; i < size; i++) {
    572		/*
    573		 * This may fail if the page was recycled by KFENCE and then
    574		 * written to again -- this however, is near impossible with a
    575		 * default config.
    576		 */
    577		KUNIT_EXPECT_EQ(test, expect.addr[i], (char)0);
    578
    579		if (!i) /* Only check first access to not fail test if page is ever re-protected. */
    580			KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    581	}
    582}
    583
    584/* Ensure that constructors work properly. */
    585static void test_memcache_ctor(struct kunit *test)
    586{
    587	const size_t size = 32;
    588	char *buf;
    589	int i;
    590
    591	setup_test_cache(test, size, 0, ctor_set_x);
    592	buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    593
    594	for (i = 0; i < 8; i++)
    595		KUNIT_EXPECT_EQ(test, buf[i], (char)'x');
    596
    597	test_free(buf);
    598
    599	KUNIT_EXPECT_FALSE(test, report_available());
    600}
    601
    602/* Test that memory is zeroed if requested. */
    603static void test_gfpzero(struct kunit *test)
    604{
    605	const size_t size = PAGE_SIZE; /* PAGE_SIZE so we can use ALLOCATE_ANY. */
    606	char *buf1, *buf2;
    607	int i;
    608
    609	/* Skip if we think it'd take too long. */
    610	KFENCE_TEST_REQUIRES(test, kfence_sample_interval <= 100);
    611
    612	setup_test_cache(test, size, 0, NULL);
    613	buf1 = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    614	for (i = 0; i < size; i++)
    615		buf1[i] = i + 1;
    616	test_free(buf1);
    617
    618	/* Try to get same address again -- this can take a while. */
    619	for (i = 0;; i++) {
    620		buf2 = test_alloc(test, size, GFP_KERNEL | __GFP_ZERO, ALLOCATE_ANY);
    621		if (buf1 == buf2)
    622			break;
    623		test_free(buf2);
    624
    625		if (kthread_should_stop() || (i == CONFIG_KFENCE_NUM_OBJECTS)) {
    626			kunit_warn(test, "giving up ... cannot get same object back\n");
    627			return;
    628		}
    629		cond_resched();
    630	}
    631
    632	for (i = 0; i < size; i++)
    633		KUNIT_EXPECT_EQ(test, buf2[i], (char)0);
    634
    635	test_free(buf2);
    636
    637	KUNIT_EXPECT_FALSE(test, report_available());
    638}
    639
    640static void test_invalid_access(struct kunit *test)
    641{
    642	const struct expect_report expect = {
    643		.type = KFENCE_ERROR_INVALID,
    644		.fn = test_invalid_access,
    645		.addr = &__kfence_pool[10],
    646		.is_write = false,
    647	};
    648
    649	READ_ONCE(__kfence_pool[10]);
    650	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    651}
    652
    653/* Test SLAB_TYPESAFE_BY_RCU works. */
    654static void test_memcache_typesafe_by_rcu(struct kunit *test)
    655{
    656	const size_t size = 32;
    657	struct expect_report expect = {
    658		.type = KFENCE_ERROR_UAF,
    659		.fn = test_memcache_typesafe_by_rcu,
    660		.is_write = false,
    661	};
    662
    663	setup_test_cache(test, size, SLAB_TYPESAFE_BY_RCU, NULL);
    664	KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */
    665
    666	expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
    667	*expect.addr = 42;
    668
    669	rcu_read_lock();
    670	test_free(expect.addr);
    671	KUNIT_EXPECT_EQ(test, *expect.addr, (char)42);
    672	/*
    673	 * Up to this point, memory should not have been freed yet, and
    674	 * therefore there should be no KFENCE report from the above access.
    675	 */
    676	rcu_read_unlock();
    677
    678	/* Above access to @expect.addr should not have generated a report! */
    679	KUNIT_EXPECT_FALSE(test, report_available());
    680
    681	/* Only after rcu_barrier() is the memory guaranteed to be freed. */
    682	rcu_barrier();
    683
    684	/* Expect use-after-free. */
    685	KUNIT_EXPECT_EQ(test, *expect.addr, (char)42);
    686	KUNIT_EXPECT_TRUE(test, report_matches(&expect));
    687}
    688
    689/* Test krealloc(). */
    690static void test_krealloc(struct kunit *test)
    691{
    692	const size_t size = 32;
    693	const struct expect_report expect = {
    694		.type = KFENCE_ERROR_UAF,
    695		.fn = test_krealloc,
    696		.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY),
    697		.is_write = false,
    698	};
    699	char *buf = expect.addr;
    700	int i;
    701
    702	KUNIT_EXPECT_FALSE(test, test_cache);
    703	KUNIT_EXPECT_EQ(test, ksize(buf), size); /* Precise size match after KFENCE alloc. */
    704	for (i = 0; i < size; i++)
    705		buf[i] = i + 1;
    706
    707	/* Check that we successfully change the size. */
    708	buf = krealloc(buf, size * 3, GFP_KERNEL); /* Grow. */
    709	/* Note: Might no longer be a KFENCE alloc. */
    710	KUNIT_EXPECT_GE(test, ksize(buf), size * 3);
    711	for (i = 0; i < size; i++)
    712		KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1));
    713	for (; i < size * 3; i++) /* Fill to extra bytes. */
    714		buf[i] = i + 1;
    715
    716	buf = krealloc(buf, size * 2, GFP_KERNEL); /* Shrink. */
    717	KUNIT_EXPECT_GE(test, ksize(buf), size * 2);
    718	for (i = 0; i < size * 2; i++)
    719		KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1));
    720
    721	buf = krealloc(buf, 0, GFP_KERNEL); /* Free. */
    722	KUNIT_EXPECT_EQ(test, (unsigned long)buf, (unsigned long)ZERO_SIZE_PTR);
    723	KUNIT_ASSERT_FALSE(test, report_available()); /* No reports yet! */
    724
    725	READ_ONCE(*expect.addr); /* Ensure krealloc() actually freed earlier KFENCE object. */
    726	KUNIT_ASSERT_TRUE(test, report_matches(&expect));
    727}
    728
    729/* Test that some objects from a bulk allocation belong to KFENCE pool. */
    730static void test_memcache_alloc_bulk(struct kunit *test)
    731{
    732	const size_t size = 32;
    733	bool pass = false;
    734	unsigned long timeout;
    735
    736	setup_test_cache(test, size, 0, NULL);
    737	KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */
    738	/*
    739	 * 100x the sample interval should be more than enough to ensure we get
    740	 * a KFENCE allocation eventually.
    741	 */
    742	timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval);
    743	do {
    744		void *objects[100];
    745		int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects),
    746						   objects);
    747		if (!num)
    748			continue;
    749		for (i = 0; i < ARRAY_SIZE(objects); i++) {
    750			if (is_kfence_address(objects[i])) {
    751				pass = true;
    752				break;
    753			}
    754		}
    755		kmem_cache_free_bulk(test_cache, num, objects);
    756		/*
    757		 * kmem_cache_alloc_bulk() disables interrupts, and calling it
    758		 * in a tight loop may not give KFENCE a chance to switch the
    759		 * static branch. Call cond_resched() to let KFENCE chime in.
    760		 */
    761		cond_resched();
    762	} while (!pass && time_before(jiffies, timeout));
    763
    764	KUNIT_EXPECT_TRUE(test, pass);
    765	KUNIT_EXPECT_FALSE(test, report_available());
    766}
    767
    768/*
    769 * KUnit does not provide a way to provide arguments to tests, and we encode
    770 * additional info in the name. Set up 2 tests per test case, one using the
    771 * default allocator, and another using a custom memcache (suffix '-memcache').
    772 */
    773#define KFENCE_KUNIT_CASE(test_name)						\
    774	{ .run_case = test_name, .name = #test_name },				\
    775	{ .run_case = test_name, .name = #test_name "-memcache" }
    776
    777static struct kunit_case kfence_test_cases[] = {
    778	KFENCE_KUNIT_CASE(test_out_of_bounds_read),
    779	KFENCE_KUNIT_CASE(test_out_of_bounds_write),
    780	KFENCE_KUNIT_CASE(test_use_after_free_read),
    781	KFENCE_KUNIT_CASE(test_double_free),
    782	KFENCE_KUNIT_CASE(test_invalid_addr_free),
    783	KFENCE_KUNIT_CASE(test_corruption),
    784	KFENCE_KUNIT_CASE(test_free_bulk),
    785	KFENCE_KUNIT_CASE(test_init_on_free),
    786	KUNIT_CASE(test_kmalloc_aligned_oob_read),
    787	KUNIT_CASE(test_kmalloc_aligned_oob_write),
    788	KUNIT_CASE(test_shrink_memcache),
    789	KUNIT_CASE(test_memcache_ctor),
    790	KUNIT_CASE(test_invalid_access),
    791	KUNIT_CASE(test_gfpzero),
    792	KUNIT_CASE(test_memcache_typesafe_by_rcu),
    793	KUNIT_CASE(test_krealloc),
    794	KUNIT_CASE(test_memcache_alloc_bulk),
    795	{},
    796};
    797
    798/* ===== End test cases ===== */
    799
    800static int test_init(struct kunit *test)
    801{
    802	unsigned long flags;
    803	int i;
    804
    805	if (!__kfence_pool)
    806		return -EINVAL;
    807
    808	spin_lock_irqsave(&observed.lock, flags);
    809	for (i = 0; i < ARRAY_SIZE(observed.lines); i++)
    810		observed.lines[i][0] = '\0';
    811	observed.nlines = 0;
    812	spin_unlock_irqrestore(&observed.lock, flags);
    813
    814	/* Any test with 'memcache' in its name will want a memcache. */
    815	if (strstr(test->name, "memcache"))
    816		test->priv = TEST_PRIV_WANT_MEMCACHE;
    817	else
    818		test->priv = NULL;
    819
    820	return 0;
    821}
    822
    823static void test_exit(struct kunit *test)
    824{
    825	test_cache_destroy();
    826}
    827
    828static void register_tracepoints(struct tracepoint *tp, void *ignore)
    829{
    830	check_trace_callback_type_console(probe_console);
    831	if (!strcmp(tp->name, "console"))
    832		WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
    833}
    834
    835static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
    836{
    837	if (!strcmp(tp->name, "console"))
    838		tracepoint_probe_unregister(tp, probe_console, NULL);
    839}
    840
    841static int kfence_suite_init(struct kunit_suite *suite)
    842{
    843	/*
    844	 * Because we want to be able to build the test as a module, we need to
    845	 * iterate through all known tracepoints, since the static registration
    846	 * won't work here.
    847	 */
    848	for_each_kernel_tracepoint(register_tracepoints, NULL);
    849	return 0;
    850}
    851
    852static void kfence_suite_exit(struct kunit_suite *suite)
    853{
    854	for_each_kernel_tracepoint(unregister_tracepoints, NULL);
    855	tracepoint_synchronize_unregister();
    856}
    857
    858static struct kunit_suite kfence_test_suite = {
    859	.name = "kfence",
    860	.test_cases = kfence_test_cases,
    861	.init = test_init,
    862	.exit = test_exit,
    863	.suite_init = kfence_suite_init,
    864	.suite_exit = kfence_suite_exit,
    865};
    866
    867kunit_test_suites(&kfence_test_suite);
    868
    869MODULE_LICENSE("GPL v2");
    870MODULE_AUTHOR("Alexander Potapenko <glider@google.com>, Marco Elver <elver@google.com>");