1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Test cases for SL[AOU]B/page initialization at alloc/free time.
  4 */
  5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  6
  7#include <linux/init.h>
  8#include <linux/kernel.h>
  9#include <linux/mm.h>
 10#include <linux/module.h>
 11#include <linux/slab.h>
 12#include <linux/string.h>
 13#include <linux/vmalloc.h>
 14
 15#define GARBAGE_INT (0x09A7BA9E)
 16#define GARBAGE_BYTE (0x9E)
 17
 18#define REPORT_FAILURES_IN_FN() \
 19	do {	\
 20		if (failures)	\
 21			pr_info("%s failed %d out of %d times\n",	\
 22				__func__, failures, num_tests);		\
 23		else		\
 24			pr_info("all %d tests in %s passed\n",		\
 25				num_tests, __func__);			\
 26	} while (0)
 27
 28/* Calculate the number of uninitialized bytes in the buffer. */
 29static int __init count_nonzero_bytes(void *ptr, size_t size)
 30{
 31	int i, ret = 0;
 32	unsigned char *p = (unsigned char *)ptr;
 33
 34	for (i = 0; i < size; i++)
 35		if (p[i])
 36			ret++;
 37	return ret;
 38}
 39
 40/* Fill a buffer with garbage, skipping |skip| first bytes. */
 41static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip)
 42{
 43	unsigned int *p = (unsigned int *)((char *)ptr + skip);
 44	int i = 0;
 45
 46	WARN_ON(skip > size);
 47	size -= skip;
 48
 49	while (size >= sizeof(*p)) {
 50		p[i] = GARBAGE_INT;
 51		i++;
 52		size -= sizeof(*p);
 53	}
 54	if (size)
 55		memset(&p[i], GARBAGE_BYTE, size);
 56}
 57
 58static void __init fill_with_garbage(void *ptr, size_t size)
 59{
 60	fill_with_garbage_skip(ptr, size, 0);
 61}
 62
 63static int __init do_alloc_pages_order(int order, int *total_failures)
 64{
 65	struct page *page;
 66	void *buf;
 67	size_t size = PAGE_SIZE << order;
 68
 69	page = alloc_pages(GFP_KERNEL, order);
 70	buf = page_address(page);
 71	fill_with_garbage(buf, size);
 72	__free_pages(page, order);
 73
 74	page = alloc_pages(GFP_KERNEL, order);
 75	buf = page_address(page);
 76	if (count_nonzero_bytes(buf, size))
 77		(*total_failures)++;
 78	fill_with_garbage(buf, size);
 79	__free_pages(page, order);
 80	return 1;
 81}
 82
 83/* Test the page allocator by calling alloc_pages with different orders. */
 84static int __init test_pages(int *total_failures)
 85{
 86	int failures = 0, num_tests = 0;
 87	int i;
 88
 89	for (i = 0; i < 10; i++)
 90		num_tests += do_alloc_pages_order(i, &failures);
 91
 92	REPORT_FAILURES_IN_FN();
 93	*total_failures += failures;
 94	return num_tests;
 95}
 96
 97/* Test kmalloc() with given parameters. */
 98static int __init do_kmalloc_size(size_t size, int *total_failures)
 99{
100	void *buf;
101
102	buf = kmalloc(size, GFP_KERNEL);
103	fill_with_garbage(buf, size);
104	kfree(buf);
105
106	buf = kmalloc(size, GFP_KERNEL);
107	if (count_nonzero_bytes(buf, size))
108		(*total_failures)++;
109	fill_with_garbage(buf, size);
110	kfree(buf);
111	return 1;
112}
113
114/* Test vmalloc() with given parameters. */
115static int __init do_vmalloc_size(size_t size, int *total_failures)
116{
117	void *buf;
118
119	buf = vmalloc(size);
120	fill_with_garbage(buf, size);
121	vfree(buf);
122
123	buf = vmalloc(size);
124	if (count_nonzero_bytes(buf, size))
125		(*total_failures)++;
126	fill_with_garbage(buf, size);
127	vfree(buf);
128	return 1;
129}
130
131/* Test kmalloc()/vmalloc() by allocating objects of different sizes. */
132static int __init test_kvmalloc(int *total_failures)
133{
134	int failures = 0, num_tests = 0;
135	int i, size;
136
137	for (i = 0; i < 20; i++) {
138		size = 1 << i;
139		num_tests += do_kmalloc_size(size, &failures);
140		num_tests += do_vmalloc_size(size, &failures);
141	}
142
143	REPORT_FAILURES_IN_FN();
144	*total_failures += failures;
145	return num_tests;
146}
147
148#define CTOR_BYTES (sizeof(unsigned int))
149#define CTOR_PATTERN (0x41414141)
150/* Initialize the first 4 bytes of the object. */
151static void test_ctor(void *obj)
152{
153	*(unsigned int *)obj = CTOR_PATTERN;
154}
155
156/*
157 * Check the invariants for the buffer allocated from a slab cache.
158 * If the cache has a test constructor, the first 4 bytes of the object must
159 * always remain equal to CTOR_PATTERN.
160 * If the cache isn't an RCU-typesafe one, or if the allocation is done with
161 * __GFP_ZERO, then the object contents must be zeroed after allocation.
162 * If the cache is an RCU-typesafe one, the object contents must never be
163 * zeroed after the first use. This is checked by memcmp() in
164 * do_kmem_cache_size().
165 */
166static bool __init check_buf(void *buf, int size, bool want_ctor,
167			     bool want_rcu, bool want_zero)
168{
169	int bytes;
170	bool fail = false;
171
172	bytes = count_nonzero_bytes(buf, size);
173	WARN_ON(want_ctor && want_zero);
174	if (want_zero)
175		return bytes;
176	if (want_ctor) {
177		if (*(unsigned int *)buf != CTOR_PATTERN)
178			fail = 1;
179	} else {
180		if (bytes)
181			fail = !want_rcu;
182	}
183	return fail;
184}
185
186/*
187 * Test kmem_cache with given parameters:
188 *  want_ctor - use a constructor;
189 *  want_rcu - use SLAB_TYPESAFE_BY_RCU;
190 *  want_zero - use __GFP_ZERO.
191 */
192static int __init do_kmem_cache_size(size_t size, bool want_ctor,
193				     bool want_rcu, bool want_zero,
194				     int *total_failures)
195{
196	struct kmem_cache *c;
197	int iter;
198	bool fail = false;
199	gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0);
200	void *buf, *buf_copy;
201
202	c = kmem_cache_create("test_cache", size, 1,
203			      want_rcu ? SLAB_TYPESAFE_BY_RCU : 0,
204			      want_ctor ? test_ctor : NULL);
205	for (iter = 0; iter < 10; iter++) {
206		buf = kmem_cache_alloc(c, alloc_mask);
207		/* Check that buf is zeroed, if it must be. */
208		fail = check_buf(buf, size, want_ctor, want_rcu, want_zero);
209		fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0);
210
211		if (!want_rcu) {
212			kmem_cache_free(c, buf);
213			continue;
214		}
215
216		/*
217		 * If this is an RCU cache, use a critical section to ensure we
218		 * can touch objects after they're freed.
219		 */
220		rcu_read_lock();
221		/*
222		 * Copy the buffer to check that it's not wiped on
223		 * free().
224		 */
225		buf_copy = kmalloc(size, GFP_ATOMIC);
226		if (buf_copy)
227			memcpy(buf_copy, buf, size);
228
229		kmem_cache_free(c, buf);
230		/*
231		 * Check that |buf| is intact after kmem_cache_free().
232		 * |want_zero| is false, because we wrote garbage to
233		 * the buffer already.
234		 */
235		fail |= check_buf(buf, size, want_ctor, want_rcu,
236				  false);
237		if (buf_copy) {
238			fail |= (bool)memcmp(buf, buf_copy, size);
239			kfree(buf_copy);
240		}
241		rcu_read_unlock();
242	}
243	kmem_cache_destroy(c);
244
245	*total_failures += fail;
246	return 1;
247}
248
249/*
250 * Check that the data written to an RCU-allocated object survives
251 * reallocation.
252 */
253static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures)
254{
255	struct kmem_cache *c;
256	void *buf, *buf_contents, *saved_ptr;
257	void **used_objects;
258	int i, iter, maxiter = 1024;
259	bool fail = false;
260
261	c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU,
262			      NULL);
263	buf = kmem_cache_alloc(c, GFP_KERNEL);
264	saved_ptr = buf;
265	fill_with_garbage(buf, size);
266	buf_contents = kmalloc(size, GFP_KERNEL);
267	if (!buf_contents)
268		goto out;
269	used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL);
270	if (!used_objects) {
271		kfree(buf_contents);
272		goto out;
273	}
274	memcpy(buf_contents, buf, size);
275	kmem_cache_free(c, buf);
276	/*
277	 * Run for a fixed number of iterations. If we never hit saved_ptr,
278	 * assume the test passes.
279	 */
280	for (iter = 0; iter < maxiter; iter++) {
281		buf = kmem_cache_alloc(c, GFP_KERNEL);
282		used_objects[iter] = buf;
283		if (buf == saved_ptr) {
284			fail = memcmp(buf_contents, buf, size);
285			for (i = 0; i <= iter; i++)
286				kmem_cache_free(c, used_objects[i]);
287			goto free_out;
288		}
289	}
290
291free_out:
292	kmem_cache_destroy(c);
293	kfree(buf_contents);
294	kfree(used_objects);
295out:
296	*total_failures += fail;
297	return 1;
298}
299
300static int __init do_kmem_cache_size_bulk(int size, int *total_failures)
301{
302	struct kmem_cache *c;
303	int i, iter, maxiter = 1024;
304	int num, bytes;
305	bool fail = false;
306	void *objects[10];
307
308	c = kmem_cache_create("test_cache", size, size, 0, NULL);
309	for (iter = 0; (iter < maxiter) && !fail; iter++) {
310		num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
311					    objects);
312		for (i = 0; i < num; i++) {
313			bytes = count_nonzero_bytes(objects[i], size);
314			if (bytes)
315				fail = true;
316			fill_with_garbage(objects[i], size);
317		}
318
319		if (num)
320			kmem_cache_free_bulk(c, num, objects);
321	}
322	*total_failures += fail;
323	return 1;
324}
325
326/*
327 * Test kmem_cache allocation by creating caches of different sizes, with and
328 * without constructors, with and without SLAB_TYPESAFE_BY_RCU.
329 */
330static int __init test_kmemcache(int *total_failures)
331{
332	int failures = 0, num_tests = 0;
333	int i, flags, size;
334	bool ctor, rcu, zero;
335
336	for (i = 0; i < 10; i++) {
337		size = 8 << i;
338		for (flags = 0; flags < 8; flags++) {
339			ctor = flags & 1;
340			rcu = flags & 2;
341			zero = flags & 4;
342			if (ctor & zero)
343				continue;
344			num_tests += do_kmem_cache_size(size, ctor, rcu, zero,
345							&failures);
346		}
347		num_tests += do_kmem_cache_size_bulk(size, &failures);
348	}
349	REPORT_FAILURES_IN_FN();
350	*total_failures += failures;
351	return num_tests;
352}
353
354/* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */
355static int __init test_rcu_persistent(int *total_failures)
356{
357	int failures = 0, num_tests = 0;
358	int i, size;
359
360	for (i = 0; i < 10; i++) {
361		size = 8 << i;
362		num_tests += do_kmem_cache_rcu_persistent(size, &failures);
363	}
364	REPORT_FAILURES_IN_FN();
365	*total_failures += failures;
366	return num_tests;
367}
368
369/*
370 * Run the tests. Each test function returns the number of executed tests and
371 * updates |failures| with the number of failed tests.
372 */
373static int __init test_meminit_init(void)
374{
375	int failures = 0, num_tests = 0;
376
377	num_tests += test_pages(&failures);
378	num_tests += test_kvmalloc(&failures);
379	num_tests += test_kmemcache(&failures);
380	num_tests += test_rcu_persistent(&failures);
381
382	if (failures == 0)
383		pr_info("all %d tests passed!\n", num_tests);
384	else
385		pr_info("failures: %d out of %d\n", failures, num_tests);
386
387	return failures ? -EINVAL : 0;
388}
389module_init(test_meminit_init);
390
391MODULE_LICENSE("GPL");