Loading...
1// SPDX-License-Identifier: GPL-2.0
2#define pr_fmt(fmt) "kcov: " fmt
3
4#define DISABLE_BRANCH_PROFILING
5#include <linux/atomic.h>
6#include <linux/compiler.h>
7#include <linux/errno.h>
8#include <linux/export.h>
9#include <linux/types.h>
10#include <linux/file.h>
11#include <linux/fs.h>
12#include <linux/init.h>
13#include <linux/mm.h>
14#include <linux/preempt.h>
15#include <linux/printk.h>
16#include <linux/sched.h>
17#include <linux/slab.h>
18#include <linux/spinlock.h>
19#include <linux/vmalloc.h>
20#include <linux/debugfs.h>
21#include <linux/uaccess.h>
22#include <linux/kcov.h>
23#include <linux/refcount.h>
24#include <asm/setup.h>
25
26/* Number of 64-bit words written per one comparison: */
27#define KCOV_WORDS_PER_CMP 4
28
29/*
30 * kcov descriptor (one per opened debugfs file).
31 * State transitions of the descriptor:
32 * - initial state after open()
33 * - then there must be a single ioctl(KCOV_INIT_TRACE) call
34 * - then, mmap() call (several calls are allowed but not useful)
35 * - then, ioctl(KCOV_ENABLE, arg), where arg is
36 * KCOV_TRACE_PC - to trace only the PCs
37 * or
38 * KCOV_TRACE_CMP - to trace only the comparison operands
39 * - then, ioctl(KCOV_DISABLE) to disable the task.
40 * Enabling/disabling ioctls can be repeated (only one task a time allowed).
41 */
42struct kcov {
43 /*
44 * Reference counter. We keep one for:
45 * - opened file descriptor
46 * - task with enabled coverage (we can't unwire it from another task)
47 */
48 refcount_t refcount;
49 /* The lock protects mode, size, area and t. */
50 spinlock_t lock;
51 enum kcov_mode mode;
52 /* Size of arena (in long's for KCOV_MODE_TRACE). */
53 unsigned size;
54 /* Coverage buffer shared with user space. */
55 void *area;
56 /* Task for which we collect coverage, or NULL. */
57 struct task_struct *t;
58};
59
60static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
61{
62 unsigned int mode;
63
64 /*
65 * We are interested in code coverage as a function of a syscall inputs,
66 * so we ignore code executed in interrupts.
67 */
68 if (!in_task())
69 return false;
70 mode = READ_ONCE(t->kcov_mode);
71 /*
72 * There is some code that runs in interrupts but for which
73 * in_interrupt() returns false (e.g. preempt_schedule_irq()).
74 * READ_ONCE()/barrier() effectively provides load-acquire wrt
75 * interrupts, there are paired barrier()/WRITE_ONCE() in
76 * kcov_ioctl_locked().
77 */
78 barrier();
79 return mode == needed_mode;
80}
81
82static notrace unsigned long canonicalize_ip(unsigned long ip)
83{
84#ifdef CONFIG_RANDOMIZE_BASE
85 ip -= kaslr_offset();
86#endif
87 return ip;
88}
89
90/*
91 * Entry point from instrumented code.
92 * This is called once per basic-block/edge.
93 */
94void notrace __sanitizer_cov_trace_pc(void)
95{
96 struct task_struct *t;
97 unsigned long *area;
98 unsigned long ip = canonicalize_ip(_RET_IP_);
99 unsigned long pos;
100
101 t = current;
102 if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t))
103 return;
104
105 area = t->kcov_area;
106 /* The first 64-bit word is the number of subsequent PCs. */
107 pos = READ_ONCE(area[0]) + 1;
108 if (likely(pos < t->kcov_size)) {
109 area[pos] = ip;
110 WRITE_ONCE(area[0], pos);
111 }
112}
113EXPORT_SYMBOL(__sanitizer_cov_trace_pc);
114
115#ifdef CONFIG_KCOV_ENABLE_COMPARISONS
116static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
117{
118 struct task_struct *t;
119 u64 *area;
120 u64 count, start_index, end_pos, max_pos;
121
122 t = current;
123 if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t))
124 return;
125
126 ip = canonicalize_ip(ip);
127
128 /*
129 * We write all comparison arguments and types as u64.
130 * The buffer was allocated for t->kcov_size unsigned longs.
131 */
132 area = (u64 *)t->kcov_area;
133 max_pos = t->kcov_size * sizeof(unsigned long);
134
135 count = READ_ONCE(area[0]);
136
137 /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */
138 start_index = 1 + count * KCOV_WORDS_PER_CMP;
139 end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
140 if (likely(end_pos <= max_pos)) {
141 area[start_index] = type;
142 area[start_index + 1] = arg1;
143 area[start_index + 2] = arg2;
144 area[start_index + 3] = ip;
145 WRITE_ONCE(area[0], count + 1);
146 }
147}
148
149void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2)
150{
151 write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_);
152}
153EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1);
154
155void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2)
156{
157 write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_);
158}
159EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2);
160
161void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2)
162{
163 write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_);
164}
165EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4);
166
167void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2)
168{
169 write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_);
170}
171EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8);
172
173void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2)
174{
175 write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
176 _RET_IP_);
177}
178EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1);
179
180void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2)
181{
182 write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
183 _RET_IP_);
184}
185EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2);
186
187void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2)
188{
189 write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
190 _RET_IP_);
191}
192EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4);
193
194void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2)
195{
196 write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
197 _RET_IP_);
198}
199EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8);
200
201void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
202{
203 u64 i;
204 u64 count = cases[0];
205 u64 size = cases[1];
206 u64 type = KCOV_CMP_CONST;
207
208 switch (size) {
209 case 8:
210 type |= KCOV_CMP_SIZE(0);
211 break;
212 case 16:
213 type |= KCOV_CMP_SIZE(1);
214 break;
215 case 32:
216 type |= KCOV_CMP_SIZE(2);
217 break;
218 case 64:
219 type |= KCOV_CMP_SIZE(3);
220 break;
221 default:
222 return;
223 }
224 for (i = 0; i < count; i++)
225 write_comp_data(type, cases[i + 2], val, _RET_IP_);
226}
227EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
228#endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */
229
230static void kcov_get(struct kcov *kcov)
231{
232 refcount_inc(&kcov->refcount);
233}
234
235static void kcov_put(struct kcov *kcov)
236{
237 if (refcount_dec_and_test(&kcov->refcount)) {
238 vfree(kcov->area);
239 kfree(kcov);
240 }
241}
242
243void kcov_task_init(struct task_struct *t)
244{
245 WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
246 barrier();
247 t->kcov_size = 0;
248 t->kcov_area = NULL;
249 t->kcov = NULL;
250}
251
252void kcov_task_exit(struct task_struct *t)
253{
254 struct kcov *kcov;
255
256 kcov = t->kcov;
257 if (kcov == NULL)
258 return;
259 spin_lock(&kcov->lock);
260 if (WARN_ON(kcov->t != t)) {
261 spin_unlock(&kcov->lock);
262 return;
263 }
264 /* Just to not leave dangling references behind. */
265 kcov_task_init(t);
266 kcov->t = NULL;
267 kcov->mode = KCOV_MODE_INIT;
268 spin_unlock(&kcov->lock);
269 kcov_put(kcov);
270}
271
272static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
273{
274 int res = 0;
275 void *area;
276 struct kcov *kcov = vma->vm_file->private_data;
277 unsigned long size, off;
278 struct page *page;
279
280 area = vmalloc_user(vma->vm_end - vma->vm_start);
281 if (!area)
282 return -ENOMEM;
283
284 spin_lock(&kcov->lock);
285 size = kcov->size * sizeof(unsigned long);
286 if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 ||
287 vma->vm_end - vma->vm_start != size) {
288 res = -EINVAL;
289 goto exit;
290 }
291 if (!kcov->area) {
292 kcov->area = area;
293 vma->vm_flags |= VM_DONTEXPAND;
294 spin_unlock(&kcov->lock);
295 for (off = 0; off < size; off += PAGE_SIZE) {
296 page = vmalloc_to_page(kcov->area + off);
297 if (vm_insert_page(vma, vma->vm_start + off, page))
298 WARN_ONCE(1, "vm_insert_page() failed");
299 }
300 return 0;
301 }
302exit:
303 spin_unlock(&kcov->lock);
304 vfree(area);
305 return res;
306}
307
308static int kcov_open(struct inode *inode, struct file *filep)
309{
310 struct kcov *kcov;
311
312 kcov = kzalloc(sizeof(*kcov), GFP_KERNEL);
313 if (!kcov)
314 return -ENOMEM;
315 kcov->mode = KCOV_MODE_DISABLED;
316 refcount_set(&kcov->refcount, 1);
317 spin_lock_init(&kcov->lock);
318 filep->private_data = kcov;
319 return nonseekable_open(inode, filep);
320}
321
322static int kcov_close(struct inode *inode, struct file *filep)
323{
324 kcov_put(filep->private_data);
325 return 0;
326}
327
328/*
329 * Fault in a lazily-faulted vmalloc area before it can be used by
330 * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the
331 * vmalloc fault handling path is instrumented.
332 */
333static void kcov_fault_in_area(struct kcov *kcov)
334{
335 unsigned long stride = PAGE_SIZE / sizeof(unsigned long);
336 unsigned long *area = kcov->area;
337 unsigned long offset;
338
339 for (offset = 0; offset < kcov->size; offset += stride)
340 READ_ONCE(area[offset]);
341}
342
343static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
344 unsigned long arg)
345{
346 struct task_struct *t;
347 unsigned long size, unused;
348
349 switch (cmd) {
350 case KCOV_INIT_TRACE:
351 /*
352 * Enable kcov in trace mode and setup buffer size.
353 * Must happen before anything else.
354 */
355 if (kcov->mode != KCOV_MODE_DISABLED)
356 return -EBUSY;
357 /*
358 * Size must be at least 2 to hold current position and one PC.
359 * Later we allocate size * sizeof(unsigned long) memory,
360 * that must not overflow.
361 */
362 size = arg;
363 if (size < 2 || size > INT_MAX / sizeof(unsigned long))
364 return -EINVAL;
365 kcov->size = size;
366 kcov->mode = KCOV_MODE_INIT;
367 return 0;
368 case KCOV_ENABLE:
369 /*
370 * Enable coverage for the current task.
371 * At this point user must have been enabled trace mode,
372 * and mmapped the file. Coverage collection is disabled only
373 * at task exit or voluntary by KCOV_DISABLE. After that it can
374 * be enabled for another task.
375 */
376 if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
377 return -EINVAL;
378 t = current;
379 if (kcov->t != NULL || t->kcov != NULL)
380 return -EBUSY;
381 if (arg == KCOV_TRACE_PC)
382 kcov->mode = KCOV_MODE_TRACE_PC;
383 else if (arg == KCOV_TRACE_CMP)
384#ifdef CONFIG_KCOV_ENABLE_COMPARISONS
385 kcov->mode = KCOV_MODE_TRACE_CMP;
386#else
387 return -ENOTSUPP;
388#endif
389 else
390 return -EINVAL;
391 kcov_fault_in_area(kcov);
392 /* Cache in task struct for performance. */
393 t->kcov_size = kcov->size;
394 t->kcov_area = kcov->area;
395 /* See comment in check_kcov_mode(). */
396 barrier();
397 WRITE_ONCE(t->kcov_mode, kcov->mode);
398 t->kcov = kcov;
399 kcov->t = t;
400 /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */
401 kcov_get(kcov);
402 return 0;
403 case KCOV_DISABLE:
404 /* Disable coverage for the current task. */
405 unused = arg;
406 if (unused != 0 || current->kcov != kcov)
407 return -EINVAL;
408 t = current;
409 if (WARN_ON(kcov->t != t))
410 return -EINVAL;
411 kcov_task_init(t);
412 kcov->t = NULL;
413 kcov->mode = KCOV_MODE_INIT;
414 kcov_put(kcov);
415 return 0;
416 default:
417 return -ENOTTY;
418 }
419}
420
421static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
422{
423 struct kcov *kcov;
424 int res;
425
426 kcov = filep->private_data;
427 spin_lock(&kcov->lock);
428 res = kcov_ioctl_locked(kcov, cmd, arg);
429 spin_unlock(&kcov->lock);
430 return res;
431}
432
433static const struct file_operations kcov_fops = {
434 .open = kcov_open,
435 .unlocked_ioctl = kcov_ioctl,
436 .compat_ioctl = kcov_ioctl,
437 .mmap = kcov_mmap,
438 .release = kcov_close,
439};
440
441static int __init kcov_init(void)
442{
443 /*
444 * The kcov debugfs file won't ever get removed and thus,
445 * there is no need to protect it against removal races. The
446 * use of debugfs_create_file_unsafe() is actually safe here.
447 */
448 debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops);
449
450 return 0;
451}
452
453device_initcall(kcov_init);
1// SPDX-License-Identifier: GPL-2.0
2#define pr_fmt(fmt) "kcov: " fmt
3
4#define DISABLE_BRANCH_PROFILING
5#include <linux/atomic.h>
6#include <linux/compiler.h>
7#include <linux/errno.h>
8#include <linux/export.h>
9#include <linux/types.h>
10#include <linux/file.h>
11#include <linux/fs.h>
12#include <linux/init.h>
13#include <linux/mm.h>
14#include <linux/preempt.h>
15#include <linux/printk.h>
16#include <linux/sched.h>
17#include <linux/slab.h>
18#include <linux/spinlock.h>
19#include <linux/vmalloc.h>
20#include <linux/debugfs.h>
21#include <linux/uaccess.h>
22#include <linux/kcov.h>
23#include <asm/setup.h>
24
25/* Number of 64-bit words written per one comparison: */
26#define KCOV_WORDS_PER_CMP 4
27
28/*
29 * kcov descriptor (one per opened debugfs file).
30 * State transitions of the descriptor:
31 * - initial state after open()
32 * - then there must be a single ioctl(KCOV_INIT_TRACE) call
33 * - then, mmap() call (several calls are allowed but not useful)
34 * - then, ioctl(KCOV_ENABLE, arg), where arg is
35 * KCOV_TRACE_PC - to trace only the PCs
36 * or
37 * KCOV_TRACE_CMP - to trace only the comparison operands
38 * - then, ioctl(KCOV_DISABLE) to disable the task.
39 * Enabling/disabling ioctls can be repeated (only one task a time allowed).
40 */
41struct kcov {
42 /*
43 * Reference counter. We keep one for:
44 * - opened file descriptor
45 * - task with enabled coverage (we can't unwire it from another task)
46 */
47 atomic_t refcount;
48 /* The lock protects mode, size, area and t. */
49 spinlock_t lock;
50 enum kcov_mode mode;
51 /* Size of arena (in long's for KCOV_MODE_TRACE). */
52 unsigned size;
53 /* Coverage buffer shared with user space. */
54 void *area;
55 /* Task for which we collect coverage, or NULL. */
56 struct task_struct *t;
57};
58
59static bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
60{
61 enum kcov_mode mode;
62
63 /*
64 * We are interested in code coverage as a function of a syscall inputs,
65 * so we ignore code executed in interrupts.
66 */
67 if (!in_task())
68 return false;
69 mode = READ_ONCE(t->kcov_mode);
70 /*
71 * There is some code that runs in interrupts but for which
72 * in_interrupt() returns false (e.g. preempt_schedule_irq()).
73 * READ_ONCE()/barrier() effectively provides load-acquire wrt
74 * interrupts, there are paired barrier()/WRITE_ONCE() in
75 * kcov_ioctl_locked().
76 */
77 barrier();
78 return mode == needed_mode;
79}
80
81static unsigned long canonicalize_ip(unsigned long ip)
82{
83#ifdef CONFIG_RANDOMIZE_BASE
84 ip -= kaslr_offset();
85#endif
86 return ip;
87}
88
89/*
90 * Entry point from instrumented code.
91 * This is called once per basic-block/edge.
92 */
93void notrace __sanitizer_cov_trace_pc(void)
94{
95 struct task_struct *t;
96 unsigned long *area;
97 unsigned long ip = canonicalize_ip(_RET_IP_);
98 unsigned long pos;
99
100 t = current;
101 if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t))
102 return;
103
104 area = t->kcov_area;
105 /* The first 64-bit word is the number of subsequent PCs. */
106 pos = READ_ONCE(area[0]) + 1;
107 if (likely(pos < t->kcov_size)) {
108 area[pos] = ip;
109 WRITE_ONCE(area[0], pos);
110 }
111}
112EXPORT_SYMBOL(__sanitizer_cov_trace_pc);
113
114#ifdef CONFIG_KCOV_ENABLE_COMPARISONS
115static void write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
116{
117 struct task_struct *t;
118 u64 *area;
119 u64 count, start_index, end_pos, max_pos;
120
121 t = current;
122 if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t))
123 return;
124
125 ip = canonicalize_ip(ip);
126
127 /*
128 * We write all comparison arguments and types as u64.
129 * The buffer was allocated for t->kcov_size unsigned longs.
130 */
131 area = (u64 *)t->kcov_area;
132 max_pos = t->kcov_size * sizeof(unsigned long);
133
134 count = READ_ONCE(area[0]);
135
136 /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */
137 start_index = 1 + count * KCOV_WORDS_PER_CMP;
138 end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
139 if (likely(end_pos <= max_pos)) {
140 area[start_index] = type;
141 area[start_index + 1] = arg1;
142 area[start_index + 2] = arg2;
143 area[start_index + 3] = ip;
144 WRITE_ONCE(area[0], count + 1);
145 }
146}
147
148void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2)
149{
150 write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_);
151}
152EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1);
153
154void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2)
155{
156 write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_);
157}
158EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2);
159
160void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2)
161{
162 write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_);
163}
164EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4);
165
166void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2)
167{
168 write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_);
169}
170EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8);
171
172void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2)
173{
174 write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
175 _RET_IP_);
176}
177EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1);
178
179void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2)
180{
181 write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
182 _RET_IP_);
183}
184EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2);
185
186void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2)
187{
188 write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
189 _RET_IP_);
190}
191EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4);
192
193void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2)
194{
195 write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
196 _RET_IP_);
197}
198EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8);
199
200void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
201{
202 u64 i;
203 u64 count = cases[0];
204 u64 size = cases[1];
205 u64 type = KCOV_CMP_CONST;
206
207 switch (size) {
208 case 8:
209 type |= KCOV_CMP_SIZE(0);
210 break;
211 case 16:
212 type |= KCOV_CMP_SIZE(1);
213 break;
214 case 32:
215 type |= KCOV_CMP_SIZE(2);
216 break;
217 case 64:
218 type |= KCOV_CMP_SIZE(3);
219 break;
220 default:
221 return;
222 }
223 for (i = 0; i < count; i++)
224 write_comp_data(type, cases[i + 2], val, _RET_IP_);
225}
226EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
227#endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */
228
229static void kcov_get(struct kcov *kcov)
230{
231 atomic_inc(&kcov->refcount);
232}
233
234static void kcov_put(struct kcov *kcov)
235{
236 if (atomic_dec_and_test(&kcov->refcount)) {
237 vfree(kcov->area);
238 kfree(kcov);
239 }
240}
241
242void kcov_task_init(struct task_struct *t)
243{
244 t->kcov_mode = KCOV_MODE_DISABLED;
245 t->kcov_size = 0;
246 t->kcov_area = NULL;
247 t->kcov = NULL;
248}
249
250void kcov_task_exit(struct task_struct *t)
251{
252 struct kcov *kcov;
253
254 kcov = t->kcov;
255 if (kcov == NULL)
256 return;
257 spin_lock(&kcov->lock);
258 if (WARN_ON(kcov->t != t)) {
259 spin_unlock(&kcov->lock);
260 return;
261 }
262 /* Just to not leave dangling references behind. */
263 kcov_task_init(t);
264 kcov->t = NULL;
265 kcov->mode = KCOV_MODE_INIT;
266 spin_unlock(&kcov->lock);
267 kcov_put(kcov);
268}
269
270static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
271{
272 int res = 0;
273 void *area;
274 struct kcov *kcov = vma->vm_file->private_data;
275 unsigned long size, off;
276 struct page *page;
277
278 area = vmalloc_user(vma->vm_end - vma->vm_start);
279 if (!area)
280 return -ENOMEM;
281
282 spin_lock(&kcov->lock);
283 size = kcov->size * sizeof(unsigned long);
284 if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 ||
285 vma->vm_end - vma->vm_start != size) {
286 res = -EINVAL;
287 goto exit;
288 }
289 if (!kcov->area) {
290 kcov->area = area;
291 vma->vm_flags |= VM_DONTEXPAND;
292 spin_unlock(&kcov->lock);
293 for (off = 0; off < size; off += PAGE_SIZE) {
294 page = vmalloc_to_page(kcov->area + off);
295 if (vm_insert_page(vma, vma->vm_start + off, page))
296 WARN_ONCE(1, "vm_insert_page() failed");
297 }
298 return 0;
299 }
300exit:
301 spin_unlock(&kcov->lock);
302 vfree(area);
303 return res;
304}
305
306static int kcov_open(struct inode *inode, struct file *filep)
307{
308 struct kcov *kcov;
309
310 kcov = kzalloc(sizeof(*kcov), GFP_KERNEL);
311 if (!kcov)
312 return -ENOMEM;
313 kcov->mode = KCOV_MODE_DISABLED;
314 atomic_set(&kcov->refcount, 1);
315 spin_lock_init(&kcov->lock);
316 filep->private_data = kcov;
317 return nonseekable_open(inode, filep);
318}
319
320static int kcov_close(struct inode *inode, struct file *filep)
321{
322 kcov_put(filep->private_data);
323 return 0;
324}
325
326static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
327 unsigned long arg)
328{
329 struct task_struct *t;
330 unsigned long size, unused;
331
332 switch (cmd) {
333 case KCOV_INIT_TRACE:
334 /*
335 * Enable kcov in trace mode and setup buffer size.
336 * Must happen before anything else.
337 */
338 if (kcov->mode != KCOV_MODE_DISABLED)
339 return -EBUSY;
340 /*
341 * Size must be at least 2 to hold current position and one PC.
342 * Later we allocate size * sizeof(unsigned long) memory,
343 * that must not overflow.
344 */
345 size = arg;
346 if (size < 2 || size > INT_MAX / sizeof(unsigned long))
347 return -EINVAL;
348 kcov->size = size;
349 kcov->mode = KCOV_MODE_INIT;
350 return 0;
351 case KCOV_ENABLE:
352 /*
353 * Enable coverage for the current task.
354 * At this point user must have been enabled trace mode,
355 * and mmapped the file. Coverage collection is disabled only
356 * at task exit or voluntary by KCOV_DISABLE. After that it can
357 * be enabled for another task.
358 */
359 if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
360 return -EINVAL;
361 t = current;
362 if (kcov->t != NULL || t->kcov != NULL)
363 return -EBUSY;
364 if (arg == KCOV_TRACE_PC)
365 kcov->mode = KCOV_MODE_TRACE_PC;
366 else if (arg == KCOV_TRACE_CMP)
367#ifdef CONFIG_KCOV_ENABLE_COMPARISONS
368 kcov->mode = KCOV_MODE_TRACE_CMP;
369#else
370 return -ENOTSUPP;
371#endif
372 else
373 return -EINVAL;
374 /* Cache in task struct for performance. */
375 t->kcov_size = kcov->size;
376 t->kcov_area = kcov->area;
377 /* See comment in check_kcov_mode(). */
378 barrier();
379 WRITE_ONCE(t->kcov_mode, kcov->mode);
380 t->kcov = kcov;
381 kcov->t = t;
382 /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */
383 kcov_get(kcov);
384 return 0;
385 case KCOV_DISABLE:
386 /* Disable coverage for the current task. */
387 unused = arg;
388 if (unused != 0 || current->kcov != kcov)
389 return -EINVAL;
390 t = current;
391 if (WARN_ON(kcov->t != t))
392 return -EINVAL;
393 kcov_task_init(t);
394 kcov->t = NULL;
395 kcov->mode = KCOV_MODE_INIT;
396 kcov_put(kcov);
397 return 0;
398 default:
399 return -ENOTTY;
400 }
401}
402
403static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
404{
405 struct kcov *kcov;
406 int res;
407
408 kcov = filep->private_data;
409 spin_lock(&kcov->lock);
410 res = kcov_ioctl_locked(kcov, cmd, arg);
411 spin_unlock(&kcov->lock);
412 return res;
413}
414
415static const struct file_operations kcov_fops = {
416 .open = kcov_open,
417 .unlocked_ioctl = kcov_ioctl,
418 .compat_ioctl = kcov_ioctl,
419 .mmap = kcov_mmap,
420 .release = kcov_close,
421};
422
423static int __init kcov_init(void)
424{
425 /*
426 * The kcov debugfs file won't ever get removed and thus,
427 * there is no need to protect it against removal races. The
428 * use of debugfs_create_file_unsafe() is actually safe here.
429 */
430 if (!debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops)) {
431 pr_err("failed to create kcov in debugfs\n");
432 return -ENOMEM;
433 }
434 return 0;
435}
436
437device_initcall(kcov_init);