1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <linux/atomic.h>
  4#include <linux/bsearch.h>
  5#include <linux/bug.h>
  6#include <linux/debugfs.h>
  7#include <linux/init.h>
  8#include <linux/kallsyms.h>
  9#include <linux/sched.h>
 10#include <linux/seq_file.h>
 11#include <linux/slab.h>
 12#include <linux/sort.h>
 13#include <linux/string.h>
 14#include <linux/uaccess.h>
 15
 16#include "kcsan.h"
 17
 18/*
 19 * Statistics counters.
 20 */
 21static atomic_long_t counters[KCSAN_COUNTER_COUNT];
 22
 23/*
 24 * Addresses for filtering functions from reporting. This list can be used as a
 25 * whitelist or blacklist.
 26 */
 27static struct {
 28	unsigned long	*addrs;		/* array of addresses */
 29	size_t		size;		/* current size */
 30	int		used;		/* number of elements used */
 31	bool		sorted;		/* if elements are sorted */
 32	bool		whitelist;	/* if list is a blacklist or whitelist */
 33} report_filterlist = {
 34	.addrs		= NULL,
 35	.size		= 8,		/* small initial size */
 36	.used		= 0,
 37	.sorted		= false,
 38	.whitelist	= false,	/* default is blacklist */
 39};
 40static DEFINE_SPINLOCK(report_filterlist_lock);
 41
 42static const char *counter_to_name(enum kcsan_counter_id id)
 43{
 44	switch (id) {
 45	case KCSAN_COUNTER_USED_WATCHPOINTS:		return "used_watchpoints";
 46	case KCSAN_COUNTER_SETUP_WATCHPOINTS:		return "setup_watchpoints";
 47	case KCSAN_COUNTER_DATA_RACES:			return "data_races";
 48	case KCSAN_COUNTER_ASSERT_FAILURES:		return "assert_failures";
 49	case KCSAN_COUNTER_NO_CAPACITY:			return "no_capacity";
 50	case KCSAN_COUNTER_REPORT_RACES:		return "report_races";
 51	case KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN:	return "races_unknown_origin";
 52	case KCSAN_COUNTER_UNENCODABLE_ACCESSES:	return "unencodable_accesses";
 53	case KCSAN_COUNTER_ENCODING_FALSE_POSITIVES:	return "encoding_false_positives";
 54	case KCSAN_COUNTER_COUNT:
 55		BUG();
 56	}
 57	return NULL;
 58}
 59
 60void kcsan_counter_inc(enum kcsan_counter_id id)
 61{
 62	atomic_long_inc(&counters[id]);
 63}
 64
 65void kcsan_counter_dec(enum kcsan_counter_id id)
 66{
 67	atomic_long_dec(&counters[id]);
 68}
 69
 70/*
 71 * The microbenchmark allows benchmarking KCSAN core runtime only. To run
 72 * multiple threads, pipe 'microbench=<iters>' from multiple tasks into the
 73 * debugfs file. This will not generate any conflicts, and tests fast-path only.
 74 */
 75static noinline void microbenchmark(unsigned long iters)
 76{
 77	const struct kcsan_ctx ctx_save = current->kcsan_ctx;
 78	const bool was_enabled = READ_ONCE(kcsan_enabled);
 79	cycles_t cycles;
 80
 81	/* We may have been called from an atomic region; reset context. */
 82	memset(&current->kcsan_ctx, 0, sizeof(current->kcsan_ctx));
 83	/*
 84	 * Disable to benchmark fast-path for all accesses, and (expected
 85	 * negligible) call into slow-path, but never set up watchpoints.
 86	 */
 87	WRITE_ONCE(kcsan_enabled, false);
 88
 89	pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);
 90
 91	cycles = get_cycles();
 92	while (iters--) {
 93		unsigned long addr = iters & ((PAGE_SIZE << 8) - 1);
 94		int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC :
 95				(!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0);
 96		__kcsan_check_access((void *)addr, sizeof(long), type);
 97	}
 98	cycles = get_cycles() - cycles;
 99
100	pr_info("KCSAN: %s end   | cycles: %llu\n", __func__, cycles);
101
102	WRITE_ONCE(kcsan_enabled, was_enabled);
103	/* restore context */
104	current->kcsan_ctx = ctx_save;
105}
106
107/*
108 * Simple test to create conflicting accesses. Write 'test=<iters>' to KCSAN's
109 * debugfs file from multiple tasks to generate real conflicts and show reports.
110 */
111static long test_dummy;
112static long test_flags;
113static long test_scoped;
114static noinline void test_thread(unsigned long iters)
115{
116	const long CHANGE_BITS = 0xff00ff00ff00ff00L;
117	const struct kcsan_ctx ctx_save = current->kcsan_ctx;
118	cycles_t cycles;
119
120	/* We may have been called from an atomic region; reset context. */
121	memset(&current->kcsan_ctx, 0, sizeof(current->kcsan_ctx));
122
123	pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);
124	pr_info("test_dummy@%px, test_flags@%px, test_scoped@%px,\n",
125		&test_dummy, &test_flags, &test_scoped);
126
127	cycles = get_cycles();
128	while (iters--) {
129		/* These all should generate reports. */
130		__kcsan_check_read(&test_dummy, sizeof(test_dummy));
131		ASSERT_EXCLUSIVE_WRITER(test_dummy);
132		ASSERT_EXCLUSIVE_ACCESS(test_dummy);
133
134		ASSERT_EXCLUSIVE_BITS(test_flags, ~CHANGE_BITS); /* no report */
135		__kcsan_check_read(&test_flags, sizeof(test_flags)); /* no report */
136
137		ASSERT_EXCLUSIVE_BITS(test_flags, CHANGE_BITS); /* report */
138		__kcsan_check_read(&test_flags, sizeof(test_flags)); /* no report */
139
140		/* not actually instrumented */
141		WRITE_ONCE(test_dummy, iters);  /* to observe value-change */
142		__kcsan_check_write(&test_dummy, sizeof(test_dummy));
143
144		test_flags ^= CHANGE_BITS; /* generate value-change */
145		__kcsan_check_write(&test_flags, sizeof(test_flags));
146
147		BUG_ON(current->kcsan_ctx.scoped_accesses.prev);
148		{
149			/* Should generate reports anywhere in this block. */
150			ASSERT_EXCLUSIVE_WRITER_SCOPED(test_scoped);
151			ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_scoped);
152			BUG_ON(!current->kcsan_ctx.scoped_accesses.prev);
153			/* Unrelated accesses. */
154			__kcsan_check_access(&cycles, sizeof(cycles), 0);
155			__kcsan_check_access(&cycles, sizeof(cycles), KCSAN_ACCESS_ATOMIC);
156		}
157		BUG_ON(current->kcsan_ctx.scoped_accesses.prev);
158	}
159	cycles = get_cycles() - cycles;
160
161	pr_info("KCSAN: %s end   | cycles: %llu\n", __func__, cycles);
162
163	/* restore context */
164	current->kcsan_ctx = ctx_save;
165}
166
167static int cmp_filterlist_addrs(const void *rhs, const void *lhs)
168{
169	const unsigned long a = *(const unsigned long *)rhs;
170	const unsigned long b = *(const unsigned long *)lhs;
171
172	return a < b ? -1 : a == b ? 0 : 1;
173}
174
175bool kcsan_skip_report_debugfs(unsigned long func_addr)
176{
177	unsigned long symbolsize, offset;
178	unsigned long flags;
179	bool ret = false;
180
181	if (!kallsyms_lookup_size_offset(func_addr, &symbolsize, &offset))
182		return false;
183	func_addr -= offset; /* Get function start */
184
185	spin_lock_irqsave(&report_filterlist_lock, flags);
186	if (report_filterlist.used == 0)
187		goto out;
188
189	/* Sort array if it is unsorted, and then do a binary search. */
190	if (!report_filterlist.sorted) {
191		sort(report_filterlist.addrs, report_filterlist.used,
192		     sizeof(unsigned long), cmp_filterlist_addrs, NULL);
193		report_filterlist.sorted = true;
194	}
195	ret = !!bsearch(&func_addr, report_filterlist.addrs,
196			report_filterlist.used, sizeof(unsigned long),
197			cmp_filterlist_addrs);
198	if (report_filterlist.whitelist)
199		ret = !ret;
200
201out:
202	spin_unlock_irqrestore(&report_filterlist_lock, flags);
203	return ret;
204}
205
206static void set_report_filterlist_whitelist(bool whitelist)
207{
208	unsigned long flags;
209
210	spin_lock_irqsave(&report_filterlist_lock, flags);
211	report_filterlist.whitelist = whitelist;
212	spin_unlock_irqrestore(&report_filterlist_lock, flags);
213}
214
215/* Returns 0 on success, error-code otherwise. */
216static ssize_t insert_report_filterlist(const char *func)
217{
218	unsigned long flags;
219	unsigned long addr = kallsyms_lookup_name(func);
220	ssize_t ret = 0;
221
222	if (!addr) {
223		pr_err("KCSAN: could not find function: '%s'\n", func);
224		return -ENOENT;
225	}
226
227	spin_lock_irqsave(&report_filterlist_lock, flags);
228
229	if (report_filterlist.addrs == NULL) {
230		/* initial allocation */
231		report_filterlist.addrs =
232			kmalloc_array(report_filterlist.size,
233				      sizeof(unsigned long), GFP_ATOMIC);
234		if (report_filterlist.addrs == NULL) {
235			ret = -ENOMEM;
236			goto out;
237		}
238	} else if (report_filterlist.used == report_filterlist.size) {
239		/* resize filterlist */
240		size_t new_size = report_filterlist.size * 2;
241		unsigned long *new_addrs =
242			krealloc(report_filterlist.addrs,
243				 new_size * sizeof(unsigned long), GFP_ATOMIC);
244
245		if (new_addrs == NULL) {
246			/* leave filterlist itself untouched */
247			ret = -ENOMEM;
248			goto out;
249		}
250
251		report_filterlist.size = new_size;
252		report_filterlist.addrs = new_addrs;
253	}
254
255	/* Note: deduplicating should be done in userspace. */
256	report_filterlist.addrs[report_filterlist.used++] =
257		kallsyms_lookup_name(func);
258	report_filterlist.sorted = false;
259
260out:
261	spin_unlock_irqrestore(&report_filterlist_lock, flags);
262
263	return ret;
264}
265
266static int show_info(struct seq_file *file, void *v)
267{
268	int i;
269	unsigned long flags;
270
271	/* show stats */
272	seq_printf(file, "enabled: %i\n", READ_ONCE(kcsan_enabled));
273	for (i = 0; i < KCSAN_COUNTER_COUNT; ++i)
274		seq_printf(file, "%s: %ld\n", counter_to_name(i),
275			   atomic_long_read(&counters[i]));
276
277	/* show filter functions, and filter type */
278	spin_lock_irqsave(&report_filterlist_lock, flags);
279	seq_printf(file, "\n%s functions: %s\n",
280		   report_filterlist.whitelist ? "whitelisted" : "blacklisted",
281		   report_filterlist.used == 0 ? "none" : "");
282	for (i = 0; i < report_filterlist.used; ++i)
283		seq_printf(file, " %ps\n", (void *)report_filterlist.addrs[i]);
284	spin_unlock_irqrestore(&report_filterlist_lock, flags);
285
286	return 0;
287}
288
289static int debugfs_open(struct inode *inode, struct file *file)
290{
291	return single_open(file, show_info, NULL);
292}
293
294static ssize_t
295debugfs_write(struct file *file, const char __user *buf, size_t count, loff_t *off)
296{
297	char kbuf[KSYM_NAME_LEN];
298	char *arg;
299	int read_len = count < (sizeof(kbuf) - 1) ? count : (sizeof(kbuf) - 1);
300
301	if (copy_from_user(kbuf, buf, read_len))
302		return -EFAULT;
303	kbuf[read_len] = '\0';
304	arg = strstrip(kbuf);
305
306	if (!strcmp(arg, "on")) {
307		WRITE_ONCE(kcsan_enabled, true);
308	} else if (!strcmp(arg, "off")) {
309		WRITE_ONCE(kcsan_enabled, false);
310	} else if (!strncmp(arg, "microbench=", sizeof("microbench=") - 1)) {
311		unsigned long iters;
312
313		if (kstrtoul(&arg[sizeof("microbench=") - 1], 0, &iters))
314			return -EINVAL;
315		microbenchmark(iters);
316	} else if (!strncmp(arg, "test=", sizeof("test=") - 1)) {
317		unsigned long iters;
318
319		if (kstrtoul(&arg[sizeof("test=") - 1], 0, &iters))
320			return -EINVAL;
321		test_thread(iters);
322	} else if (!strcmp(arg, "whitelist")) {
323		set_report_filterlist_whitelist(true);
324	} else if (!strcmp(arg, "blacklist")) {
325		set_report_filterlist_whitelist(false);
326	} else if (arg[0] == '!') {
327		ssize_t ret = insert_report_filterlist(&arg[1]);
328
329		if (ret < 0)
330			return ret;
331	} else {
332		return -EINVAL;
333	}
334
335	return count;
336}
337
338static const struct file_operations debugfs_ops =
339{
340	.read	 = seq_read,
341	.open	 = debugfs_open,
342	.write	 = debugfs_write,
343	.release = single_release
344};
345
346void __init kcsan_debugfs_init(void)
347{
348	debugfs_create_file("kcsan", 0644, NULL, NULL, &debugfs_ops);
349}