Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Test cases for KFENCE memory safety error detector. Since the interface with
4 * which KFENCE's reports are obtained is via the console, this is the output we
5 * should verify. For each test case checks the presence (or absence) of
6 * generated reports. Relies on 'console' tracepoint to capture reports as they
7 * appear in the kernel log.
8 *
9 * Copyright (C) 2020, Google LLC.
10 * Author: Alexander Potapenko <glider@google.com>
11 * Marco Elver <elver@google.com>
12 */
13
14#include <kunit/test.h>
15#include <linux/jiffies.h>
16#include <linux/kernel.h>
17#include <linux/kfence.h>
18#include <linux/mm.h>
19#include <linux/random.h>
20#include <linux/slab.h>
21#include <linux/spinlock.h>
22#include <linux/string.h>
23#include <linux/tracepoint.h>
24#include <trace/events/printk.h>
25
26#include "kfence.h"
27
28/* Report as observed from console. */
29static struct {
30 spinlock_t lock;
31 int nlines;
32 char lines[2][256];
33} observed = {
34 .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
35};
36
37/* Probe for console output: obtains observed lines of interest. */
38static void probe_console(void *ignore, const char *buf, size_t len)
39{
40 unsigned long flags;
41 int nlines;
42
43 spin_lock_irqsave(&observed.lock, flags);
44 nlines = observed.nlines;
45
46 if (strnstr(buf, "BUG: KFENCE: ", len) && strnstr(buf, "test_", len)) {
47 /*
48 * KFENCE report and related to the test.
49 *
50 * The provided @buf is not NUL-terminated; copy no more than
51 * @len bytes and let strscpy() add the missing NUL-terminator.
52 */
53 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
54 nlines = 1;
55 } else if (nlines == 1 && (strnstr(buf, "at 0x", len) || strnstr(buf, "of 0x", len))) {
56 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
57 }
58
59 WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
60 spin_unlock_irqrestore(&observed.lock, flags);
61}
62
63/* Check if a report related to the test exists. */
64static bool report_available(void)
65{
66 return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
67}
68
69/* Information we expect in a report. */
70struct expect_report {
71 enum kfence_error_type type; /* The type or error. */
72 void *fn; /* Function pointer to expected function where access occurred. */
73 char *addr; /* Address at which the bad access occurred. */
74 bool is_write; /* Is access a write. */
75};
76
77static const char *get_access_type(const struct expect_report *r)
78{
79 return r->is_write ? "write" : "read";
80}
81
82/* Check observed report matches information in @r. */
83static bool report_matches(const struct expect_report *r)
84{
85 bool ret = false;
86 unsigned long flags;
87 typeof(observed.lines) expect;
88 const char *end;
89 char *cur;
90
91 /* Doubled-checked locking. */
92 if (!report_available())
93 return false;
94
95 /* Generate expected report contents. */
96
97 /* Title */
98 cur = expect[0];
99 end = &expect[0][sizeof(expect[0]) - 1];
100 switch (r->type) {
101 case KFENCE_ERROR_OOB:
102 cur += scnprintf(cur, end - cur, "BUG: KFENCE: out-of-bounds %s",
103 get_access_type(r));
104 break;
105 case KFENCE_ERROR_UAF:
106 cur += scnprintf(cur, end - cur, "BUG: KFENCE: use-after-free %s",
107 get_access_type(r));
108 break;
109 case KFENCE_ERROR_CORRUPTION:
110 cur += scnprintf(cur, end - cur, "BUG: KFENCE: memory corruption");
111 break;
112 case KFENCE_ERROR_INVALID:
113 cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid %s",
114 get_access_type(r));
115 break;
116 case KFENCE_ERROR_INVALID_FREE:
117 cur += scnprintf(cur, end - cur, "BUG: KFENCE: invalid free");
118 break;
119 }
120
121 scnprintf(cur, end - cur, " in %pS", r->fn);
122 /* The exact offset won't match, remove it; also strip module name. */
123 cur = strchr(expect[0], '+');
124 if (cur)
125 *cur = '\0';
126
127 /* Access information */
128 cur = expect[1];
129 end = &expect[1][sizeof(expect[1]) - 1];
130
131 switch (r->type) {
132 case KFENCE_ERROR_OOB:
133 cur += scnprintf(cur, end - cur, "Out-of-bounds %s at", get_access_type(r));
134 break;
135 case KFENCE_ERROR_UAF:
136 cur += scnprintf(cur, end - cur, "Use-after-free %s at", get_access_type(r));
137 break;
138 case KFENCE_ERROR_CORRUPTION:
139 cur += scnprintf(cur, end - cur, "Corrupted memory at");
140 break;
141 case KFENCE_ERROR_INVALID:
142 cur += scnprintf(cur, end - cur, "Invalid %s at", get_access_type(r));
143 break;
144 case KFENCE_ERROR_INVALID_FREE:
145 cur += scnprintf(cur, end - cur, "Invalid free of");
146 break;
147 }
148
149 cur += scnprintf(cur, end - cur, " 0x%p", (void *)r->addr);
150
151 spin_lock_irqsave(&observed.lock, flags);
152 if (!report_available())
153 goto out; /* A new report is being captured. */
154
155 /* Finally match expected output to what we actually observed. */
156 ret = strstr(observed.lines[0], expect[0]) && strstr(observed.lines[1], expect[1]);
157out:
158 spin_unlock_irqrestore(&observed.lock, flags);
159 return ret;
160}
161
162/* ===== Test cases ===== */
163
164#define TEST_PRIV_WANT_MEMCACHE ((void *)1)
165
166/* Cache used by tests; if NULL, allocate from kmalloc instead. */
167static struct kmem_cache *test_cache;
168
169static size_t setup_test_cache(struct kunit *test, size_t size, slab_flags_t flags,
170 void (*ctor)(void *))
171{
172 if (test->priv != TEST_PRIV_WANT_MEMCACHE)
173 return size;
174
175 kunit_info(test, "%s: size=%zu, ctor=%ps\n", __func__, size, ctor);
176
177 /*
178 * Use SLAB_NOLEAKTRACE to prevent merging with existing caches. Any
179 * other flag in SLAB_NEVER_MERGE also works. Use SLAB_ACCOUNT to
180 * allocate via memcg, if enabled.
181 */
182 flags |= SLAB_NOLEAKTRACE | SLAB_ACCOUNT;
183 test_cache = kmem_cache_create("test", size, 1, flags, ctor);
184 KUNIT_ASSERT_TRUE_MSG(test, test_cache, "could not create cache");
185
186 return size;
187}
188
189static void test_cache_destroy(void)
190{
191 if (!test_cache)
192 return;
193
194 kmem_cache_destroy(test_cache);
195 test_cache = NULL;
196}
197
198static inline size_t kmalloc_cache_alignment(size_t size)
199{
200 return kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)]->align;
201}
202
203/* Must always inline to match stack trace against caller. */
204static __always_inline void test_free(void *ptr)
205{
206 if (test_cache)
207 kmem_cache_free(test_cache, ptr);
208 else
209 kfree(ptr);
210}
211
212/*
213 * If this should be a KFENCE allocation, and on which side the allocation and
214 * the closest guard page should be.
215 */
216enum allocation_policy {
217 ALLOCATE_ANY, /* KFENCE, any side. */
218 ALLOCATE_LEFT, /* KFENCE, left side of page. */
219 ALLOCATE_RIGHT, /* KFENCE, right side of page. */
220 ALLOCATE_NONE, /* No KFENCE allocation. */
221};
222
223/*
224 * Try to get a guarded allocation from KFENCE. Uses either kmalloc() or the
225 * current test_cache if set up.
226 */
227static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocation_policy policy)
228{
229 void *alloc;
230 unsigned long timeout, resched_after;
231 const char *policy_name;
232
233 switch (policy) {
234 case ALLOCATE_ANY:
235 policy_name = "any";
236 break;
237 case ALLOCATE_LEFT:
238 policy_name = "left";
239 break;
240 case ALLOCATE_RIGHT:
241 policy_name = "right";
242 break;
243 case ALLOCATE_NONE:
244 policy_name = "none";
245 break;
246 }
247
248 kunit_info(test, "%s: size=%zu, gfp=%x, policy=%s, cache=%i\n", __func__, size, gfp,
249 policy_name, !!test_cache);
250
251 /*
252 * 100x the sample interval should be more than enough to ensure we get
253 * a KFENCE allocation eventually.
254 */
255 timeout = jiffies + msecs_to_jiffies(100 * CONFIG_KFENCE_SAMPLE_INTERVAL);
256 /*
257 * Especially for non-preemption kernels, ensure the allocation-gate
258 * timer can catch up: after @resched_after, every failed allocation
259 * attempt yields, to ensure the allocation-gate timer is scheduled.
260 */
261 resched_after = jiffies + msecs_to_jiffies(CONFIG_KFENCE_SAMPLE_INTERVAL);
262 do {
263 if (test_cache)
264 alloc = kmem_cache_alloc(test_cache, gfp);
265 else
266 alloc = kmalloc(size, gfp);
267
268 if (is_kfence_address(alloc)) {
269 struct page *page = virt_to_head_page(alloc);
270 struct kmem_cache *s = test_cache ?:
271 kmalloc_caches[kmalloc_type(GFP_KERNEL)][__kmalloc_index(size, false)];
272
273 /*
274 * Verify that various helpers return the right values
275 * even for KFENCE objects; these are required so that
276 * memcg accounting works correctly.
277 */
278 KUNIT_EXPECT_EQ(test, obj_to_index(s, page, alloc), 0U);
279 KUNIT_EXPECT_EQ(test, objs_per_slab_page(s, page), 1);
280
281 if (policy == ALLOCATE_ANY)
282 return alloc;
283 if (policy == ALLOCATE_LEFT && IS_ALIGNED((unsigned long)alloc, PAGE_SIZE))
284 return alloc;
285 if (policy == ALLOCATE_RIGHT &&
286 !IS_ALIGNED((unsigned long)alloc, PAGE_SIZE))
287 return alloc;
288 } else if (policy == ALLOCATE_NONE)
289 return alloc;
290
291 test_free(alloc);
292
293 if (time_after(jiffies, resched_after))
294 cond_resched();
295 } while (time_before(jiffies, timeout));
296
297 KUNIT_ASSERT_TRUE_MSG(test, false, "failed to allocate from KFENCE");
298 return NULL; /* Unreachable. */
299}
300
301static void test_out_of_bounds_read(struct kunit *test)
302{
303 size_t size = 32;
304 struct expect_report expect = {
305 .type = KFENCE_ERROR_OOB,
306 .fn = test_out_of_bounds_read,
307 .is_write = false,
308 };
309 char *buf;
310
311 setup_test_cache(test, size, 0, NULL);
312
313 /*
314 * If we don't have our own cache, adjust based on alignment, so that we
315 * actually access guard pages on either side.
316 */
317 if (!test_cache)
318 size = kmalloc_cache_alignment(size);
319
320 /* Test both sides. */
321
322 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
323 expect.addr = buf - 1;
324 READ_ONCE(*expect.addr);
325 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
326 test_free(buf);
327
328 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
329 expect.addr = buf + size;
330 READ_ONCE(*expect.addr);
331 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
332 test_free(buf);
333}
334
335static void test_out_of_bounds_write(struct kunit *test)
336{
337 size_t size = 32;
338 struct expect_report expect = {
339 .type = KFENCE_ERROR_OOB,
340 .fn = test_out_of_bounds_write,
341 .is_write = true,
342 };
343 char *buf;
344
345 setup_test_cache(test, size, 0, NULL);
346 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
347 expect.addr = buf - 1;
348 WRITE_ONCE(*expect.addr, 42);
349 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
350 test_free(buf);
351}
352
353static void test_use_after_free_read(struct kunit *test)
354{
355 const size_t size = 32;
356 struct expect_report expect = {
357 .type = KFENCE_ERROR_UAF,
358 .fn = test_use_after_free_read,
359 .is_write = false,
360 };
361
362 setup_test_cache(test, size, 0, NULL);
363 expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
364 test_free(expect.addr);
365 READ_ONCE(*expect.addr);
366 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
367}
368
369static void test_double_free(struct kunit *test)
370{
371 const size_t size = 32;
372 struct expect_report expect = {
373 .type = KFENCE_ERROR_INVALID_FREE,
374 .fn = test_double_free,
375 };
376
377 setup_test_cache(test, size, 0, NULL);
378 expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
379 test_free(expect.addr);
380 test_free(expect.addr); /* Double-free. */
381 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
382}
383
384static void test_invalid_addr_free(struct kunit *test)
385{
386 const size_t size = 32;
387 struct expect_report expect = {
388 .type = KFENCE_ERROR_INVALID_FREE,
389 .fn = test_invalid_addr_free,
390 };
391 char *buf;
392
393 setup_test_cache(test, size, 0, NULL);
394 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
395 expect.addr = buf + 1; /* Free on invalid address. */
396 test_free(expect.addr); /* Invalid address free. */
397 test_free(buf); /* No error. */
398 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
399}
400
401static void test_corruption(struct kunit *test)
402{
403 size_t size = 32;
404 struct expect_report expect = {
405 .type = KFENCE_ERROR_CORRUPTION,
406 .fn = test_corruption,
407 };
408 char *buf;
409
410 setup_test_cache(test, size, 0, NULL);
411
412 /* Test both sides. */
413
414 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT);
415 expect.addr = buf + size;
416 WRITE_ONCE(*expect.addr, 42);
417 test_free(buf);
418 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
419
420 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
421 expect.addr = buf - 1;
422 WRITE_ONCE(*expect.addr, 42);
423 test_free(buf);
424 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
425}
426
427/*
428 * KFENCE is unable to detect an OOB if the allocation's alignment requirements
429 * leave a gap between the object and the guard page. Specifically, an
430 * allocation of e.g. 73 bytes is aligned on 8 and 128 bytes for SLUB or SLAB
431 * respectively. Therefore it is impossible for the allocated object to
432 * contiguously line up with the right guard page.
433 *
434 * However, we test that an access to memory beyond the gap results in KFENCE
435 * detecting an OOB access.
436 */
437static void test_kmalloc_aligned_oob_read(struct kunit *test)
438{
439 const size_t size = 73;
440 const size_t align = kmalloc_cache_alignment(size);
441 struct expect_report expect = {
442 .type = KFENCE_ERROR_OOB,
443 .fn = test_kmalloc_aligned_oob_read,
444 .is_write = false,
445 };
446 char *buf;
447
448 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
449
450 /*
451 * The object is offset to the right, so there won't be an OOB to the
452 * left of it.
453 */
454 READ_ONCE(*(buf - 1));
455 KUNIT_EXPECT_FALSE(test, report_available());
456
457 /*
458 * @buf must be aligned on @align, therefore buf + size belongs to the
459 * same page -> no OOB.
460 */
461 READ_ONCE(*(buf + size));
462 KUNIT_EXPECT_FALSE(test, report_available());
463
464 /* Overflowing by @align bytes will result in an OOB. */
465 expect.addr = buf + size + align;
466 READ_ONCE(*expect.addr);
467 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
468
469 test_free(buf);
470}
471
472static void test_kmalloc_aligned_oob_write(struct kunit *test)
473{
474 const size_t size = 73;
475 struct expect_report expect = {
476 .type = KFENCE_ERROR_CORRUPTION,
477 .fn = test_kmalloc_aligned_oob_write,
478 };
479 char *buf;
480
481 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT);
482 /*
483 * The object is offset to the right, so we won't get a page
484 * fault immediately after it.
485 */
486 expect.addr = buf + size;
487 WRITE_ONCE(*expect.addr, READ_ONCE(*expect.addr) + 1);
488 KUNIT_EXPECT_FALSE(test, report_available());
489 test_free(buf);
490 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
491}
492
493/* Test cache shrinking and destroying with KFENCE. */
494static void test_shrink_memcache(struct kunit *test)
495{
496 const size_t size = 32;
497 void *buf;
498
499 setup_test_cache(test, size, 0, NULL);
500 KUNIT_EXPECT_TRUE(test, test_cache);
501 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
502 kmem_cache_shrink(test_cache);
503 test_free(buf);
504
505 KUNIT_EXPECT_FALSE(test, report_available());
506}
507
508static void ctor_set_x(void *obj)
509{
510 /* Every object has at least 8 bytes. */
511 memset(obj, 'x', 8);
512}
513
514/* Ensure that SL*B does not modify KFENCE objects on bulk free. */
515static void test_free_bulk(struct kunit *test)
516{
517 int iter;
518
519 for (iter = 0; iter < 5; iter++) {
520 const size_t size = setup_test_cache(test, 8 + prandom_u32_max(300), 0,
521 (iter & 1) ? ctor_set_x : NULL);
522 void *objects[] = {
523 test_alloc(test, size, GFP_KERNEL, ALLOCATE_RIGHT),
524 test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
525 test_alloc(test, size, GFP_KERNEL, ALLOCATE_LEFT),
526 test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
527 test_alloc(test, size, GFP_KERNEL, ALLOCATE_NONE),
528 };
529
530 kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects);
531 KUNIT_ASSERT_FALSE(test, report_available());
532 test_cache_destroy();
533 }
534}
535
536/* Test init-on-free works. */
537static void test_init_on_free(struct kunit *test)
538{
539 const size_t size = 32;
540 struct expect_report expect = {
541 .type = KFENCE_ERROR_UAF,
542 .fn = test_init_on_free,
543 .is_write = false,
544 };
545 int i;
546
547 if (!IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON))
548 return;
549 /* Assume it hasn't been disabled on command line. */
550
551 setup_test_cache(test, size, 0, NULL);
552 expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
553 for (i = 0; i < size; i++)
554 expect.addr[i] = i + 1;
555 test_free(expect.addr);
556
557 for (i = 0; i < size; i++) {
558 /*
559 * This may fail if the page was recycled by KFENCE and then
560 * written to again -- this however, is near impossible with a
561 * default config.
562 */
563 KUNIT_EXPECT_EQ(test, expect.addr[i], (char)0);
564
565 if (!i) /* Only check first access to not fail test if page is ever re-protected. */
566 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
567 }
568}
569
570/* Ensure that constructors work properly. */
571static void test_memcache_ctor(struct kunit *test)
572{
573 const size_t size = 32;
574 char *buf;
575 int i;
576
577 setup_test_cache(test, size, 0, ctor_set_x);
578 buf = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
579
580 for (i = 0; i < 8; i++)
581 KUNIT_EXPECT_EQ(test, buf[i], (char)'x');
582
583 test_free(buf);
584
585 KUNIT_EXPECT_FALSE(test, report_available());
586}
587
588/* Test that memory is zeroed if requested. */
589static void test_gfpzero(struct kunit *test)
590{
591 const size_t size = PAGE_SIZE; /* PAGE_SIZE so we can use ALLOCATE_ANY. */
592 char *buf1, *buf2;
593 int i;
594
595 if (CONFIG_KFENCE_SAMPLE_INTERVAL > 100) {
596 kunit_warn(test, "skipping ... would take too long\n");
597 return;
598 }
599
600 setup_test_cache(test, size, 0, NULL);
601 buf1 = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
602 for (i = 0; i < size; i++)
603 buf1[i] = i + 1;
604 test_free(buf1);
605
606 /* Try to get same address again -- this can take a while. */
607 for (i = 0;; i++) {
608 buf2 = test_alloc(test, size, GFP_KERNEL | __GFP_ZERO, ALLOCATE_ANY);
609 if (buf1 == buf2)
610 break;
611 test_free(buf2);
612
613 if (i == CONFIG_KFENCE_NUM_OBJECTS) {
614 kunit_warn(test, "giving up ... cannot get same object back\n");
615 return;
616 }
617 }
618
619 for (i = 0; i < size; i++)
620 KUNIT_EXPECT_EQ(test, buf2[i], (char)0);
621
622 test_free(buf2);
623
624 KUNIT_EXPECT_FALSE(test, report_available());
625}
626
627static void test_invalid_access(struct kunit *test)
628{
629 const struct expect_report expect = {
630 .type = KFENCE_ERROR_INVALID,
631 .fn = test_invalid_access,
632 .addr = &__kfence_pool[10],
633 .is_write = false,
634 };
635
636 READ_ONCE(__kfence_pool[10]);
637 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
638}
639
640/* Test SLAB_TYPESAFE_BY_RCU works. */
641static void test_memcache_typesafe_by_rcu(struct kunit *test)
642{
643 const size_t size = 32;
644 struct expect_report expect = {
645 .type = KFENCE_ERROR_UAF,
646 .fn = test_memcache_typesafe_by_rcu,
647 .is_write = false,
648 };
649
650 setup_test_cache(test, size, SLAB_TYPESAFE_BY_RCU, NULL);
651 KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */
652
653 expect.addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY);
654 *expect.addr = 42;
655
656 rcu_read_lock();
657 test_free(expect.addr);
658 KUNIT_EXPECT_EQ(test, *expect.addr, (char)42);
659 /*
660 * Up to this point, memory should not have been freed yet, and
661 * therefore there should be no KFENCE report from the above access.
662 */
663 rcu_read_unlock();
664
665 /* Above access to @expect.addr should not have generated a report! */
666 KUNIT_EXPECT_FALSE(test, report_available());
667
668 /* Only after rcu_barrier() is the memory guaranteed to be freed. */
669 rcu_barrier();
670
671 /* Expect use-after-free. */
672 KUNIT_EXPECT_EQ(test, *expect.addr, (char)42);
673 KUNIT_EXPECT_TRUE(test, report_matches(&expect));
674}
675
676/* Test krealloc(). */
677static void test_krealloc(struct kunit *test)
678{
679 const size_t size = 32;
680 const struct expect_report expect = {
681 .type = KFENCE_ERROR_UAF,
682 .fn = test_krealloc,
683 .addr = test_alloc(test, size, GFP_KERNEL, ALLOCATE_ANY),
684 .is_write = false,
685 };
686 char *buf = expect.addr;
687 int i;
688
689 KUNIT_EXPECT_FALSE(test, test_cache);
690 KUNIT_EXPECT_EQ(test, ksize(buf), size); /* Precise size match after KFENCE alloc. */
691 for (i = 0; i < size; i++)
692 buf[i] = i + 1;
693
694 /* Check that we successfully change the size. */
695 buf = krealloc(buf, size * 3, GFP_KERNEL); /* Grow. */
696 /* Note: Might no longer be a KFENCE alloc. */
697 KUNIT_EXPECT_GE(test, ksize(buf), size * 3);
698 for (i = 0; i < size; i++)
699 KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1));
700 for (; i < size * 3; i++) /* Fill to extra bytes. */
701 buf[i] = i + 1;
702
703 buf = krealloc(buf, size * 2, GFP_KERNEL); /* Shrink. */
704 KUNIT_EXPECT_GE(test, ksize(buf), size * 2);
705 for (i = 0; i < size * 2; i++)
706 KUNIT_EXPECT_EQ(test, buf[i], (char)(i + 1));
707
708 buf = krealloc(buf, 0, GFP_KERNEL); /* Free. */
709 KUNIT_EXPECT_EQ(test, (unsigned long)buf, (unsigned long)ZERO_SIZE_PTR);
710 KUNIT_ASSERT_FALSE(test, report_available()); /* No reports yet! */
711
712 READ_ONCE(*expect.addr); /* Ensure krealloc() actually freed earlier KFENCE object. */
713 KUNIT_ASSERT_TRUE(test, report_matches(&expect));
714}
715
716/* Test that some objects from a bulk allocation belong to KFENCE pool. */
717static void test_memcache_alloc_bulk(struct kunit *test)
718{
719 const size_t size = 32;
720 bool pass = false;
721 unsigned long timeout;
722
723 setup_test_cache(test, size, 0, NULL);
724 KUNIT_EXPECT_TRUE(test, test_cache); /* Want memcache. */
725 /*
726 * 100x the sample interval should be more than enough to ensure we get
727 * a KFENCE allocation eventually.
728 */
729 timeout = jiffies + msecs_to_jiffies(100 * CONFIG_KFENCE_SAMPLE_INTERVAL);
730 do {
731 void *objects[100];
732 int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects),
733 objects);
734 if (!num)
735 continue;
736 for (i = 0; i < ARRAY_SIZE(objects); i++) {
737 if (is_kfence_address(objects[i])) {
738 pass = true;
739 break;
740 }
741 }
742 kmem_cache_free_bulk(test_cache, num, objects);
743 /*
744 * kmem_cache_alloc_bulk() disables interrupts, and calling it
745 * in a tight loop may not give KFENCE a chance to switch the
746 * static branch. Call cond_resched() to let KFENCE chime in.
747 */
748 cond_resched();
749 } while (!pass && time_before(jiffies, timeout));
750
751 KUNIT_EXPECT_TRUE(test, pass);
752 KUNIT_EXPECT_FALSE(test, report_available());
753}
754
755/*
756 * KUnit does not provide a way to provide arguments to tests, and we encode
757 * additional info in the name. Set up 2 tests per test case, one using the
758 * default allocator, and another using a custom memcache (suffix '-memcache').
759 */
760#define KFENCE_KUNIT_CASE(test_name) \
761 { .run_case = test_name, .name = #test_name }, \
762 { .run_case = test_name, .name = #test_name "-memcache" }
763
764static struct kunit_case kfence_test_cases[] = {
765 KFENCE_KUNIT_CASE(test_out_of_bounds_read),
766 KFENCE_KUNIT_CASE(test_out_of_bounds_write),
767 KFENCE_KUNIT_CASE(test_use_after_free_read),
768 KFENCE_KUNIT_CASE(test_double_free),
769 KFENCE_KUNIT_CASE(test_invalid_addr_free),
770 KFENCE_KUNIT_CASE(test_corruption),
771 KFENCE_KUNIT_CASE(test_free_bulk),
772 KFENCE_KUNIT_CASE(test_init_on_free),
773 KUNIT_CASE(test_kmalloc_aligned_oob_read),
774 KUNIT_CASE(test_kmalloc_aligned_oob_write),
775 KUNIT_CASE(test_shrink_memcache),
776 KUNIT_CASE(test_memcache_ctor),
777 KUNIT_CASE(test_invalid_access),
778 KUNIT_CASE(test_gfpzero),
779 KUNIT_CASE(test_memcache_typesafe_by_rcu),
780 KUNIT_CASE(test_krealloc),
781 KUNIT_CASE(test_memcache_alloc_bulk),
782 {},
783};
784
785/* ===== End test cases ===== */
786
787static int test_init(struct kunit *test)
788{
789 unsigned long flags;
790 int i;
791
792 spin_lock_irqsave(&observed.lock, flags);
793 for (i = 0; i < ARRAY_SIZE(observed.lines); i++)
794 observed.lines[i][0] = '\0';
795 observed.nlines = 0;
796 spin_unlock_irqrestore(&observed.lock, flags);
797
798 /* Any test with 'memcache' in its name will want a memcache. */
799 if (strstr(test->name, "memcache"))
800 test->priv = TEST_PRIV_WANT_MEMCACHE;
801 else
802 test->priv = NULL;
803
804 return 0;
805}
806
807static void test_exit(struct kunit *test)
808{
809 test_cache_destroy();
810}
811
812static struct kunit_suite kfence_test_suite = {
813 .name = "kfence",
814 .test_cases = kfence_test_cases,
815 .init = test_init,
816 .exit = test_exit,
817};
818static struct kunit_suite *kfence_test_suites[] = { &kfence_test_suite, NULL };
819
820static void register_tracepoints(struct tracepoint *tp, void *ignore)
821{
822 check_trace_callback_type_console(probe_console);
823 if (!strcmp(tp->name, "console"))
824 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
825}
826
827static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
828{
829 if (!strcmp(tp->name, "console"))
830 tracepoint_probe_unregister(tp, probe_console, NULL);
831}
832
833/*
834 * We only want to do tracepoints setup and teardown once, therefore we have to
835 * customize the init and exit functions and cannot rely on kunit_test_suite().
836 */
837static int __init kfence_test_init(void)
838{
839 /*
840 * Because we want to be able to build the test as a module, we need to
841 * iterate through all known tracepoints, since the static registration
842 * won't work here.
843 */
844 for_each_kernel_tracepoint(register_tracepoints, NULL);
845 return __kunit_test_suites_init(kfence_test_suites);
846}
847
848static void kfence_test_exit(void)
849{
850 __kunit_test_suites_exit(kfence_test_suites);
851 for_each_kernel_tracepoint(unregister_tracepoints, NULL);
852 tracepoint_synchronize_unregister();
853}
854
855late_initcall_sync(kfence_test_init);
856module_exit(kfence_test_exit);
857
858MODULE_LICENSE("GPL v2");
859MODULE_AUTHOR("Alexander Potapenko <glider@google.com>, Marco Elver <elver@google.com>");