Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Fprobe-based tracing events
4 * Copyright (C) 2022 Google LLC.
5 */
6#define pr_fmt(fmt) "trace_fprobe: " fmt
7#include <asm/ptrace.h>
8
9#include <linux/fprobe.h>
10#include <linux/module.h>
11#include <linux/rculist.h>
12#include <linux/security.h>
13#include <linux/tracepoint.h>
14#include <linux/uaccess.h>
15
16#include "trace_dynevent.h"
17#include "trace_probe.h"
18#include "trace_probe_kernel.h"
19#include "trace_probe_tmpl.h"
20
21#define FPROBE_EVENT_SYSTEM "fprobes"
22#define TRACEPOINT_EVENT_SYSTEM "tracepoints"
23#define RETHOOK_MAXACTIVE_MAX 4096
24
25static int trace_fprobe_create(const char *raw_command);
26static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);
27static int trace_fprobe_release(struct dyn_event *ev);
28static bool trace_fprobe_is_busy(struct dyn_event *ev);
29static bool trace_fprobe_match(const char *system, const char *event,
30 int argc, const char **argv, struct dyn_event *ev);
31
32static struct dyn_event_operations trace_fprobe_ops = {
33 .create = trace_fprobe_create,
34 .show = trace_fprobe_show,
35 .is_busy = trace_fprobe_is_busy,
36 .free = trace_fprobe_release,
37 .match = trace_fprobe_match,
38};
39
40/*
41 * Fprobe event core functions
42 */
43struct trace_fprobe {
44 struct dyn_event devent;
45 struct fprobe fp;
46 const char *symbol;
47 struct tracepoint *tpoint;
48 struct module *mod;
49 struct trace_probe tp;
50};
51
52static bool is_trace_fprobe(struct dyn_event *ev)
53{
54 return ev->ops == &trace_fprobe_ops;
55}
56
57static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev)
58{
59 return container_of(ev, struct trace_fprobe, devent);
60}
61
62/**
63 * for_each_trace_fprobe - iterate over the trace_fprobe list
64 * @pos: the struct trace_fprobe * for each entry
65 * @dpos: the struct dyn_event * to use as a loop cursor
66 */
67#define for_each_trace_fprobe(pos, dpos) \
68 for_each_dyn_event(dpos) \
69 if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos)))
70
71static bool trace_fprobe_is_return(struct trace_fprobe *tf)
72{
73 return tf->fp.exit_handler != NULL;
74}
75
76static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)
77{
78 return tf->tpoint != NULL;
79}
80
81static const char *trace_fprobe_symbol(struct trace_fprobe *tf)
82{
83 return tf->symbol ? tf->symbol : "unknown";
84}
85
86static bool trace_fprobe_is_busy(struct dyn_event *ev)
87{
88 struct trace_fprobe *tf = to_trace_fprobe(ev);
89
90 return trace_probe_is_enabled(&tf->tp);
91}
92
93static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,
94 int argc, const char **argv)
95{
96 char buf[MAX_ARGSTR_LEN + 1];
97
98 if (!argc)
99 return true;
100
101 snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));
102 if (strcmp(buf, argv[0]))
103 return false;
104 argc--; argv++;
105
106 return trace_probe_match_command_args(&tf->tp, argc, argv);
107}
108
109static bool trace_fprobe_match(const char *system, const char *event,
110 int argc, const char **argv, struct dyn_event *ev)
111{
112 struct trace_fprobe *tf = to_trace_fprobe(ev);
113
114 if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))
115 return false;
116
117 if (system && strcmp(trace_probe_group_name(&tf->tp), system))
118 return false;
119
120 return trace_fprobe_match_command_head(tf, argc, argv);
121}
122
123static bool trace_fprobe_is_registered(struct trace_fprobe *tf)
124{
125 return fprobe_is_registered(&tf->fp);
126}
127
128/*
129 * Note that we don't verify the fetch_insn code, since it does not come
130 * from user space.
131 */
132static int
133process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
134 void *dest, void *base)
135{
136 struct pt_regs *regs = rec;
137 unsigned long val;
138 int ret;
139
140retry:
141 /* 1st stage: get value from context */
142 switch (code->op) {
143 case FETCH_OP_STACK:
144 val = regs_get_kernel_stack_nth(regs, code->param);
145 break;
146 case FETCH_OP_STACKP:
147 val = kernel_stack_pointer(regs);
148 break;
149 case FETCH_OP_RETVAL:
150 val = regs_return_value(regs);
151 break;
152#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
153 case FETCH_OP_ARG:
154 val = regs_get_kernel_argument(regs, code->param);
155 break;
156 case FETCH_OP_EDATA:
157 val = *(unsigned long *)((unsigned long)edata + code->offset);
158 break;
159#endif
160 case FETCH_NOP_SYMBOL: /* Ignore a place holder */
161 code++;
162 goto retry;
163 default:
164 ret = process_common_fetch_insn(code, &val);
165 if (ret < 0)
166 return ret;
167 }
168 code++;
169
170 return process_fetch_insn_bottom(code, val, dest, base);
171}
172NOKPROBE_SYMBOL(process_fetch_insn)
173
174/* function entry handler */
175static nokprobe_inline void
176__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
177 struct pt_regs *regs,
178 struct trace_event_file *trace_file)
179{
180 struct fentry_trace_entry_head *entry;
181 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
182 struct trace_event_buffer fbuffer;
183 int dsize;
184
185 if (WARN_ON_ONCE(call != trace_file->event_call))
186 return;
187
188 if (trace_trigger_soft_disabled(trace_file))
189 return;
190
191 dsize = __get_data_size(&tf->tp, regs, NULL);
192
193 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
194 sizeof(*entry) + tf->tp.size + dsize);
195 if (!entry)
196 return;
197
198 fbuffer.regs = regs;
199 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
200 entry->ip = entry_ip;
201 store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize);
202
203 trace_event_buffer_commit(&fbuffer);
204}
205
206static void
207fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
208 struct pt_regs *regs)
209{
210 struct event_file_link *link;
211
212 trace_probe_for_each_link_rcu(link, &tf->tp)
213 __fentry_trace_func(tf, entry_ip, regs, link->file);
214}
215NOKPROBE_SYMBOL(fentry_trace_func);
216
217/* function exit handler */
218static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip,
219 unsigned long ret_ip, struct pt_regs *regs,
220 void *entry_data)
221{
222 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
223
224 if (tf->tp.entry_arg)
225 store_trace_entry_data(entry_data, &tf->tp, regs);
226
227 return 0;
228}
229NOKPROBE_SYMBOL(trace_fprobe_entry_handler)
230
231static nokprobe_inline void
232__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
233 unsigned long ret_ip, struct pt_regs *regs,
234 void *entry_data, struct trace_event_file *trace_file)
235{
236 struct fexit_trace_entry_head *entry;
237 struct trace_event_buffer fbuffer;
238 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
239 int dsize;
240
241 if (WARN_ON_ONCE(call != trace_file->event_call))
242 return;
243
244 if (trace_trigger_soft_disabled(trace_file))
245 return;
246
247 dsize = __get_data_size(&tf->tp, regs, entry_data);
248
249 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
250 sizeof(*entry) + tf->tp.size + dsize);
251 if (!entry)
252 return;
253
254 fbuffer.regs = regs;
255 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
256 entry->func = entry_ip;
257 entry->ret_ip = ret_ip;
258 store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize);
259
260 trace_event_buffer_commit(&fbuffer);
261}
262
263static void
264fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
265 unsigned long ret_ip, struct pt_regs *regs, void *entry_data)
266{
267 struct event_file_link *link;
268
269 trace_probe_for_each_link_rcu(link, &tf->tp)
270 __fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data, link->file);
271}
272NOKPROBE_SYMBOL(fexit_trace_func);
273
274#ifdef CONFIG_PERF_EVENTS
275
276static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
277 struct pt_regs *regs)
278{
279 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
280 struct fentry_trace_entry_head *entry;
281 struct hlist_head *head;
282 int size, __size, dsize;
283 int rctx;
284
285 head = this_cpu_ptr(call->perf_events);
286 if (hlist_empty(head))
287 return 0;
288
289 dsize = __get_data_size(&tf->tp, regs, NULL);
290 __size = sizeof(*entry) + tf->tp.size + dsize;
291 size = ALIGN(__size + sizeof(u32), sizeof(u64));
292 size -= sizeof(u32);
293
294 entry = perf_trace_buf_alloc(size, NULL, &rctx);
295 if (!entry)
296 return 0;
297
298 entry->ip = entry_ip;
299 memset(&entry[1], 0, dsize);
300 store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize);
301 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
302 head, NULL);
303 return 0;
304}
305NOKPROBE_SYMBOL(fentry_perf_func);
306
307static void
308fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
309 unsigned long ret_ip, struct pt_regs *regs,
310 void *entry_data)
311{
312 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
313 struct fexit_trace_entry_head *entry;
314 struct hlist_head *head;
315 int size, __size, dsize;
316 int rctx;
317
318 head = this_cpu_ptr(call->perf_events);
319 if (hlist_empty(head))
320 return;
321
322 dsize = __get_data_size(&tf->tp, regs, entry_data);
323 __size = sizeof(*entry) + tf->tp.size + dsize;
324 size = ALIGN(__size + sizeof(u32), sizeof(u64));
325 size -= sizeof(u32);
326
327 entry = perf_trace_buf_alloc(size, NULL, &rctx);
328 if (!entry)
329 return;
330
331 entry->func = entry_ip;
332 entry->ret_ip = ret_ip;
333 store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize);
334 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
335 head, NULL);
336}
337NOKPROBE_SYMBOL(fexit_perf_func);
338#endif /* CONFIG_PERF_EVENTS */
339
340static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip,
341 unsigned long ret_ip, struct pt_regs *regs,
342 void *entry_data)
343{
344 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
345 int ret = 0;
346
347 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))
348 fentry_trace_func(tf, entry_ip, regs);
349#ifdef CONFIG_PERF_EVENTS
350 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))
351 ret = fentry_perf_func(tf, entry_ip, regs);
352#endif
353 return ret;
354}
355NOKPROBE_SYMBOL(fentry_dispatcher);
356
357static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip,
358 unsigned long ret_ip, struct pt_regs *regs,
359 void *entry_data)
360{
361 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
362
363 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))
364 fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data);
365#ifdef CONFIG_PERF_EVENTS
366 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))
367 fexit_perf_func(tf, entry_ip, ret_ip, regs, entry_data);
368#endif
369}
370NOKPROBE_SYMBOL(fexit_dispatcher);
371
372static void free_trace_fprobe(struct trace_fprobe *tf)
373{
374 if (tf) {
375 trace_probe_cleanup(&tf->tp);
376 kfree(tf->symbol);
377 kfree(tf);
378 }
379}
380
381/*
382 * Allocate new trace_probe and initialize it (including fprobe).
383 */
384static struct trace_fprobe *alloc_trace_fprobe(const char *group,
385 const char *event,
386 const char *symbol,
387 struct tracepoint *tpoint,
388 int maxactive,
389 int nargs, bool is_return)
390{
391 struct trace_fprobe *tf;
392 int ret = -ENOMEM;
393
394 tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);
395 if (!tf)
396 return ERR_PTR(ret);
397
398 tf->symbol = kstrdup(symbol, GFP_KERNEL);
399 if (!tf->symbol)
400 goto error;
401
402 if (is_return)
403 tf->fp.exit_handler = fexit_dispatcher;
404 else
405 tf->fp.entry_handler = fentry_dispatcher;
406
407 tf->tpoint = tpoint;
408 tf->fp.nr_maxactive = maxactive;
409
410 ret = trace_probe_init(&tf->tp, event, group, false, nargs);
411 if (ret < 0)
412 goto error;
413
414 dyn_event_init(&tf->devent, &trace_fprobe_ops);
415 return tf;
416error:
417 free_trace_fprobe(tf);
418 return ERR_PTR(ret);
419}
420
421static struct trace_fprobe *find_trace_fprobe(const char *event,
422 const char *group)
423{
424 struct dyn_event *pos;
425 struct trace_fprobe *tf;
426
427 for_each_trace_fprobe(tf, pos)
428 if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&
429 strcmp(trace_probe_group_name(&tf->tp), group) == 0)
430 return tf;
431 return NULL;
432}
433
434static inline int __enable_trace_fprobe(struct trace_fprobe *tf)
435{
436 if (trace_fprobe_is_registered(tf))
437 enable_fprobe(&tf->fp);
438
439 return 0;
440}
441
442static void __disable_trace_fprobe(struct trace_probe *tp)
443{
444 struct trace_fprobe *tf;
445
446 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
447 if (!trace_fprobe_is_registered(tf))
448 continue;
449 disable_fprobe(&tf->fp);
450 }
451}
452
453/*
454 * Enable trace_probe
455 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
456 */
457static int enable_trace_fprobe(struct trace_event_call *call,
458 struct trace_event_file *file)
459{
460 struct trace_probe *tp;
461 struct trace_fprobe *tf;
462 bool enabled;
463 int ret = 0;
464
465 tp = trace_probe_primary_from_call(call);
466 if (WARN_ON_ONCE(!tp))
467 return -ENODEV;
468 enabled = trace_probe_is_enabled(tp);
469
470 /* This also changes "enabled" state */
471 if (file) {
472 ret = trace_probe_add_file(tp, file);
473 if (ret)
474 return ret;
475 } else
476 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
477
478 if (!enabled) {
479 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
480 /* TODO: check the fprobe is gone */
481 __enable_trace_fprobe(tf);
482 }
483 }
484
485 return 0;
486}
487
488/*
489 * Disable trace_probe
490 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
491 */
492static int disable_trace_fprobe(struct trace_event_call *call,
493 struct trace_event_file *file)
494{
495 struct trace_probe *tp;
496
497 tp = trace_probe_primary_from_call(call);
498 if (WARN_ON_ONCE(!tp))
499 return -ENODEV;
500
501 if (file) {
502 if (!trace_probe_get_file_link(tp, file))
503 return -ENOENT;
504 if (!trace_probe_has_single_file(tp))
505 goto out;
506 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
507 } else
508 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
509
510 if (!trace_probe_is_enabled(tp))
511 __disable_trace_fprobe(tp);
512
513 out:
514 if (file)
515 /*
516 * Synchronization is done in below function. For perf event,
517 * file == NULL and perf_trace_event_unreg() calls
518 * tracepoint_synchronize_unregister() to ensure synchronize
519 * event. We don't need to care about it.
520 */
521 trace_probe_remove_file(tp, file);
522
523 return 0;
524}
525
526/* Event entry printers */
527static enum print_line_t
528print_fentry_event(struct trace_iterator *iter, int flags,
529 struct trace_event *event)
530{
531 struct fentry_trace_entry_head *field;
532 struct trace_seq *s = &iter->seq;
533 struct trace_probe *tp;
534
535 field = (struct fentry_trace_entry_head *)iter->ent;
536 tp = trace_probe_primary_from_call(
537 container_of(event, struct trace_event_call, event));
538 if (WARN_ON_ONCE(!tp))
539 goto out;
540
541 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
542
543 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
544 goto out;
545
546 trace_seq_putc(s, ')');
547
548 if (trace_probe_print_args(s, tp->args, tp->nr_args,
549 (u8 *)&field[1], field) < 0)
550 goto out;
551
552 trace_seq_putc(s, '\n');
553 out:
554 return trace_handle_return(s);
555}
556
557static enum print_line_t
558print_fexit_event(struct trace_iterator *iter, int flags,
559 struct trace_event *event)
560{
561 struct fexit_trace_entry_head *field;
562 struct trace_seq *s = &iter->seq;
563 struct trace_probe *tp;
564
565 field = (struct fexit_trace_entry_head *)iter->ent;
566 tp = trace_probe_primary_from_call(
567 container_of(event, struct trace_event_call, event));
568 if (WARN_ON_ONCE(!tp))
569 goto out;
570
571 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
572
573 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
574 goto out;
575
576 trace_seq_puts(s, " <- ");
577
578 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
579 goto out;
580
581 trace_seq_putc(s, ')');
582
583 if (trace_probe_print_args(s, tp->args, tp->nr_args,
584 (u8 *)&field[1], field) < 0)
585 goto out;
586
587 trace_seq_putc(s, '\n');
588
589 out:
590 return trace_handle_return(s);
591}
592
593static int fentry_event_define_fields(struct trace_event_call *event_call)
594{
595 int ret;
596 struct fentry_trace_entry_head field;
597 struct trace_probe *tp;
598
599 tp = trace_probe_primary_from_call(event_call);
600 if (WARN_ON_ONCE(!tp))
601 return -ENOENT;
602
603 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
604
605 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
606}
607
608static int fexit_event_define_fields(struct trace_event_call *event_call)
609{
610 int ret;
611 struct fexit_trace_entry_head field;
612 struct trace_probe *tp;
613
614 tp = trace_probe_primary_from_call(event_call);
615 if (WARN_ON_ONCE(!tp))
616 return -ENOENT;
617
618 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
619 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
620
621 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
622}
623
624static struct trace_event_functions fentry_funcs = {
625 .trace = print_fentry_event
626};
627
628static struct trace_event_functions fexit_funcs = {
629 .trace = print_fexit_event
630};
631
632static struct trace_event_fields fentry_fields_array[] = {
633 { .type = TRACE_FUNCTION_TYPE,
634 .define_fields = fentry_event_define_fields },
635 {}
636};
637
638static struct trace_event_fields fexit_fields_array[] = {
639 { .type = TRACE_FUNCTION_TYPE,
640 .define_fields = fexit_event_define_fields },
641 {}
642};
643
644static int fprobe_register(struct trace_event_call *event,
645 enum trace_reg type, void *data);
646
647static inline void init_trace_event_call(struct trace_fprobe *tf)
648{
649 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
650
651 if (trace_fprobe_is_return(tf)) {
652 call->event.funcs = &fexit_funcs;
653 call->class->fields_array = fexit_fields_array;
654 } else {
655 call->event.funcs = &fentry_funcs;
656 call->class->fields_array = fentry_fields_array;
657 }
658
659 call->flags = TRACE_EVENT_FL_FPROBE;
660 call->class->reg = fprobe_register;
661}
662
663static int register_fprobe_event(struct trace_fprobe *tf)
664{
665 init_trace_event_call(tf);
666
667 return trace_probe_register_event_call(&tf->tp);
668}
669
670static int unregister_fprobe_event(struct trace_fprobe *tf)
671{
672 return trace_probe_unregister_event_call(&tf->tp);
673}
674
675/* Internal register function - just handle fprobe and flags */
676static int __register_trace_fprobe(struct trace_fprobe *tf)
677{
678 int i, ret;
679
680 /* Should we need new LOCKDOWN flag for fprobe? */
681 ret = security_locked_down(LOCKDOWN_KPROBES);
682 if (ret)
683 return ret;
684
685 if (trace_fprobe_is_registered(tf))
686 return -EINVAL;
687
688 for (i = 0; i < tf->tp.nr_args; i++) {
689 ret = traceprobe_update_arg(&tf->tp.args[i]);
690 if (ret)
691 return ret;
692 }
693
694 /* Set/clear disabled flag according to tp->flag */
695 if (trace_probe_is_enabled(&tf->tp))
696 tf->fp.flags &= ~FPROBE_FL_DISABLED;
697 else
698 tf->fp.flags |= FPROBE_FL_DISABLED;
699
700 if (trace_fprobe_is_tracepoint(tf)) {
701 struct tracepoint *tpoint = tf->tpoint;
702 unsigned long ip = (unsigned long)tpoint->probestub;
703 /*
704 * Here, we do 2 steps to enable fprobe on a tracepoint.
705 * At first, put __probestub_##TP function on the tracepoint
706 * and put a fprobe on the stub function.
707 */
708 ret = tracepoint_probe_register_prio_may_exist(tpoint,
709 tpoint->probestub, NULL, 0);
710 if (ret < 0)
711 return ret;
712 return register_fprobe_ips(&tf->fp, &ip, 1);
713 }
714
715 /* TODO: handle filter, nofilter or symbol list */
716 return register_fprobe(&tf->fp, tf->symbol, NULL);
717}
718
719/* Internal unregister function - just handle fprobe and flags */
720static void __unregister_trace_fprobe(struct trace_fprobe *tf)
721{
722 if (trace_fprobe_is_registered(tf)) {
723 unregister_fprobe(&tf->fp);
724 memset(&tf->fp, 0, sizeof(tf->fp));
725 if (trace_fprobe_is_tracepoint(tf)) {
726 tracepoint_probe_unregister(tf->tpoint,
727 tf->tpoint->probestub, NULL);
728 tf->tpoint = NULL;
729 tf->mod = NULL;
730 }
731 }
732}
733
734/* TODO: make this trace_*probe common function */
735/* Unregister a trace_probe and probe_event */
736static int unregister_trace_fprobe(struct trace_fprobe *tf)
737{
738 /* If other probes are on the event, just unregister fprobe */
739 if (trace_probe_has_sibling(&tf->tp))
740 goto unreg;
741
742 /* Enabled event can not be unregistered */
743 if (trace_probe_is_enabled(&tf->tp))
744 return -EBUSY;
745
746 /* If there's a reference to the dynamic event */
747 if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))
748 return -EBUSY;
749
750 /* Will fail if probe is being used by ftrace or perf */
751 if (unregister_fprobe_event(tf))
752 return -EBUSY;
753
754unreg:
755 __unregister_trace_fprobe(tf);
756 dyn_event_remove(&tf->devent);
757 trace_probe_unlink(&tf->tp);
758
759 return 0;
760}
761
762static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig,
763 struct trace_fprobe *comp)
764{
765 struct trace_probe_event *tpe = orig->tp.event;
766 int i;
767
768 list_for_each_entry(orig, &tpe->probes, tp.list) {
769 if (strcmp(trace_fprobe_symbol(orig),
770 trace_fprobe_symbol(comp)))
771 continue;
772
773 /*
774 * trace_probe_compare_arg_type() ensured that nr_args and
775 * each argument name and type are same. Let's compare comm.
776 */
777 for (i = 0; i < orig->tp.nr_args; i++) {
778 if (strcmp(orig->tp.args[i].comm,
779 comp->tp.args[i].comm))
780 break;
781 }
782
783 if (i == orig->tp.nr_args)
784 return true;
785 }
786
787 return false;
788}
789
790static int append_trace_fprobe(struct trace_fprobe *tf, struct trace_fprobe *to)
791{
792 int ret;
793
794 if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) ||
795 trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) {
796 trace_probe_log_set_index(0);
797 trace_probe_log_err(0, DIFF_PROBE_TYPE);
798 return -EEXIST;
799 }
800 ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);
801 if (ret) {
802 /* Note that argument starts index = 2 */
803 trace_probe_log_set_index(ret + 1);
804 trace_probe_log_err(0, DIFF_ARG_TYPE);
805 return -EEXIST;
806 }
807 if (trace_fprobe_has_same_fprobe(to, tf)) {
808 trace_probe_log_set_index(0);
809 trace_probe_log_err(0, SAME_PROBE);
810 return -EEXIST;
811 }
812
813 /* Append to existing event */
814 ret = trace_probe_append(&tf->tp, &to->tp);
815 if (ret)
816 return ret;
817
818 ret = __register_trace_fprobe(tf);
819 if (ret)
820 trace_probe_unlink(&tf->tp);
821 else
822 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
823
824 return ret;
825}
826
827/* Register a trace_probe and probe_event */
828static int register_trace_fprobe(struct trace_fprobe *tf)
829{
830 struct trace_fprobe *old_tf;
831 int ret;
832
833 mutex_lock(&event_mutex);
834
835 old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),
836 trace_probe_group_name(&tf->tp));
837 if (old_tf) {
838 ret = append_trace_fprobe(tf, old_tf);
839 goto end;
840 }
841
842 /* Register new event */
843 ret = register_fprobe_event(tf);
844 if (ret) {
845 if (ret == -EEXIST) {
846 trace_probe_log_set_index(0);
847 trace_probe_log_err(0, EVENT_EXIST);
848 } else
849 pr_warn("Failed to register probe event(%d)\n", ret);
850 goto end;
851 }
852
853 /* Register fprobe */
854 ret = __register_trace_fprobe(tf);
855 if (ret < 0)
856 unregister_fprobe_event(tf);
857 else
858 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
859
860end:
861 mutex_unlock(&event_mutex);
862 return ret;
863}
864
865#ifdef CONFIG_MODULES
866static int __tracepoint_probe_module_cb(struct notifier_block *self,
867 unsigned long val, void *data)
868{
869 struct tp_module *tp_mod = data;
870 struct trace_fprobe *tf;
871 struct dyn_event *pos;
872
873 if (val != MODULE_STATE_GOING)
874 return NOTIFY_DONE;
875
876 mutex_lock(&event_mutex);
877 for_each_trace_fprobe(tf, pos) {
878 if (tp_mod->mod == tf->mod) {
879 tracepoint_probe_unregister(tf->tpoint,
880 tf->tpoint->probestub, NULL);
881 tf->tpoint = NULL;
882 tf->mod = NULL;
883 }
884 }
885 mutex_unlock(&event_mutex);
886
887 return NOTIFY_DONE;
888}
889
890static struct notifier_block tracepoint_module_nb = {
891 .notifier_call = __tracepoint_probe_module_cb,
892};
893#endif /* CONFIG_MODULES */
894
895struct __find_tracepoint_cb_data {
896 const char *tp_name;
897 struct tracepoint *tpoint;
898};
899
900static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)
901{
902 struct __find_tracepoint_cb_data *data = priv;
903
904 if (!data->tpoint && !strcmp(data->tp_name, tp->name))
905 data->tpoint = tp;
906}
907
908static struct tracepoint *find_tracepoint(const char *tp_name)
909{
910 struct __find_tracepoint_cb_data data = {
911 .tp_name = tp_name,
912 };
913
914 for_each_kernel_tracepoint(__find_tracepoint_cb, &data);
915
916 return data.tpoint;
917}
918
919static int parse_symbol_and_return(int argc, const char *argv[],
920 char **symbol, bool *is_return,
921 bool is_tracepoint)
922{
923 char *tmp = strchr(argv[1], '%');
924 int i;
925
926 if (tmp) {
927 int len = tmp - argv[1];
928
929 if (!is_tracepoint && !strcmp(tmp, "%return")) {
930 *is_return = true;
931 } else {
932 trace_probe_log_err(len, BAD_ADDR_SUFFIX);
933 return -EINVAL;
934 }
935 *symbol = kmemdup_nul(argv[1], len, GFP_KERNEL);
936 } else
937 *symbol = kstrdup(argv[1], GFP_KERNEL);
938 if (!*symbol)
939 return -ENOMEM;
940
941 if (*is_return)
942 return 0;
943
944 /* If there is $retval, this should be a return fprobe. */
945 for (i = 2; i < argc; i++) {
946 tmp = strstr(argv[i], "$retval");
947 if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') {
948 if (is_tracepoint) {
949 trace_probe_log_set_index(i);
950 trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE);
951 return -EINVAL;
952 }
953 *is_return = true;
954 break;
955 }
956 }
957 return 0;
958}
959
960static int __trace_fprobe_create(int argc, const char *argv[])
961{
962 /*
963 * Argument syntax:
964 * - Add fentry probe:
965 * f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS]
966 * - Add fexit probe:
967 * f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS]
968 * - Add tracepoint probe:
969 * t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS]
970 *
971 * Fetch args:
972 * $retval : fetch return value
973 * $stack : fetch stack address
974 * $stackN : fetch Nth entry of stack (N:0-)
975 * $argN : fetch Nth argument (N:1-)
976 * $comm : fetch current task comm
977 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
978 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
979 * Dereferencing memory fetch:
980 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
981 * Alias name of args:
982 * NAME=FETCHARG : set NAME as alias of FETCHARG.
983 * Type of args:
984 * FETCHARG:TYPE : use TYPE instead of unsigned long.
985 */
986 struct trace_fprobe *tf = NULL;
987 int i, len, new_argc = 0, ret = 0;
988 bool is_return = false;
989 char *symbol = NULL;
990 const char *event = NULL, *group = FPROBE_EVENT_SYSTEM;
991 const char **new_argv = NULL;
992 int maxactive = 0;
993 char buf[MAX_EVENT_NAME_LEN];
994 char gbuf[MAX_EVENT_NAME_LEN];
995 char sbuf[KSYM_NAME_LEN];
996 char abuf[MAX_BTF_ARGS_LEN];
997 bool is_tracepoint = false;
998 struct tracepoint *tpoint = NULL;
999 struct traceprobe_parse_context ctx = {
1000 .flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE,
1001 };
1002
1003 if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2)
1004 return -ECANCELED;
1005
1006 if (argv[0][0] == 't') {
1007 is_tracepoint = true;
1008 group = TRACEPOINT_EVENT_SYSTEM;
1009 }
1010
1011 trace_probe_log_init("trace_fprobe", argc, argv);
1012
1013 event = strchr(&argv[0][1], ':');
1014 if (event)
1015 event++;
1016
1017 if (isdigit(argv[0][1])) {
1018 if (event)
1019 len = event - &argv[0][1] - 1;
1020 else
1021 len = strlen(&argv[0][1]);
1022 if (len > MAX_EVENT_NAME_LEN - 1) {
1023 trace_probe_log_err(1, BAD_MAXACT);
1024 goto parse_error;
1025 }
1026 memcpy(buf, &argv[0][1], len);
1027 buf[len] = '\0';
1028 ret = kstrtouint(buf, 0, &maxactive);
1029 if (ret || !maxactive) {
1030 trace_probe_log_err(1, BAD_MAXACT);
1031 goto parse_error;
1032 }
1033 /* fprobe rethook instances are iterated over via a list. The
1034 * maximum should stay reasonable.
1035 */
1036 if (maxactive > RETHOOK_MAXACTIVE_MAX) {
1037 trace_probe_log_err(1, MAXACT_TOO_BIG);
1038 goto parse_error;
1039 }
1040 }
1041
1042 trace_probe_log_set_index(1);
1043
1044 /* a symbol(or tracepoint) must be specified */
1045 ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint);
1046 if (ret < 0)
1047 goto parse_error;
1048
1049 if (!is_return && maxactive) {
1050 trace_probe_log_set_index(0);
1051 trace_probe_log_err(1, BAD_MAXACT_TYPE);
1052 goto parse_error;
1053 }
1054
1055 trace_probe_log_set_index(0);
1056 if (event) {
1057 ret = traceprobe_parse_event_name(&event, &group, gbuf,
1058 event - argv[0]);
1059 if (ret)
1060 goto parse_error;
1061 }
1062
1063 if (!event) {
1064 /* Make a new event name */
1065 if (is_tracepoint)
1066 snprintf(buf, MAX_EVENT_NAME_LEN, "%s%s",
1067 isdigit(*symbol) ? "_" : "", symbol);
1068 else
1069 snprintf(buf, MAX_EVENT_NAME_LEN, "%s__%s", symbol,
1070 is_return ? "exit" : "entry");
1071 sanitize_event_name(buf);
1072 event = buf;
1073 }
1074
1075 if (is_return)
1076 ctx.flags |= TPARG_FL_RETURN;
1077 else
1078 ctx.flags |= TPARG_FL_FENTRY;
1079
1080 if (is_tracepoint) {
1081 ctx.flags |= TPARG_FL_TPOINT;
1082 tpoint = find_tracepoint(symbol);
1083 if (!tpoint) {
1084 trace_probe_log_set_index(1);
1085 trace_probe_log_err(0, NO_TRACEPOINT);
1086 goto parse_error;
1087 }
1088 ctx.funcname = kallsyms_lookup(
1089 (unsigned long)tpoint->probestub,
1090 NULL, NULL, NULL, sbuf);
1091 } else
1092 ctx.funcname = symbol;
1093
1094 argc -= 2; argv += 2;
1095 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1096 abuf, MAX_BTF_ARGS_LEN, &ctx);
1097 if (IS_ERR(new_argv)) {
1098 ret = PTR_ERR(new_argv);
1099 new_argv = NULL;
1100 goto out;
1101 }
1102 if (new_argv) {
1103 argc = new_argc;
1104 argv = new_argv;
1105 }
1106
1107 /* setup a probe */
1108 tf = alloc_trace_fprobe(group, event, symbol, tpoint, maxactive,
1109 argc, is_return);
1110 if (IS_ERR(tf)) {
1111 ret = PTR_ERR(tf);
1112 /* This must return -ENOMEM, else there is a bug */
1113 WARN_ON_ONCE(ret != -ENOMEM);
1114 goto out; /* We know tf is not allocated */
1115 }
1116
1117 if (is_tracepoint)
1118 tf->mod = __module_text_address(
1119 (unsigned long)tf->tpoint->probestub);
1120
1121 /* parse arguments */
1122 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
1123 trace_probe_log_set_index(i + 2);
1124 ctx.offset = 0;
1125 ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], &ctx);
1126 if (ret)
1127 goto error; /* This can be -ENOMEM */
1128 }
1129
1130 if (is_return && tf->tp.entry_arg) {
1131 tf->fp.entry_handler = trace_fprobe_entry_handler;
1132 tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp);
1133 }
1134
1135 ret = traceprobe_set_print_fmt(&tf->tp,
1136 is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL);
1137 if (ret < 0)
1138 goto error;
1139
1140 ret = register_trace_fprobe(tf);
1141 if (ret) {
1142 trace_probe_log_set_index(1);
1143 if (ret == -EILSEQ)
1144 trace_probe_log_err(0, BAD_INSN_BNDRY);
1145 else if (ret == -ENOENT)
1146 trace_probe_log_err(0, BAD_PROBE_ADDR);
1147 else if (ret != -ENOMEM && ret != -EEXIST)
1148 trace_probe_log_err(0, FAIL_REG_PROBE);
1149 goto error;
1150 }
1151
1152out:
1153 traceprobe_finish_parse(&ctx);
1154 trace_probe_log_clear();
1155 kfree(new_argv);
1156 kfree(symbol);
1157 return ret;
1158
1159parse_error:
1160 ret = -EINVAL;
1161error:
1162 free_trace_fprobe(tf);
1163 goto out;
1164}
1165
1166static int trace_fprobe_create(const char *raw_command)
1167{
1168 return trace_probe_create(raw_command, __trace_fprobe_create);
1169}
1170
1171static int trace_fprobe_release(struct dyn_event *ev)
1172{
1173 struct trace_fprobe *tf = to_trace_fprobe(ev);
1174 int ret = unregister_trace_fprobe(tf);
1175
1176 if (!ret)
1177 free_trace_fprobe(tf);
1178 return ret;
1179}
1180
1181static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
1182{
1183 struct trace_fprobe *tf = to_trace_fprobe(ev);
1184 int i;
1185
1186 if (trace_fprobe_is_tracepoint(tf))
1187 seq_putc(m, 't');
1188 else
1189 seq_putc(m, 'f');
1190 if (trace_fprobe_is_return(tf) && tf->fp.nr_maxactive)
1191 seq_printf(m, "%d", tf->fp.nr_maxactive);
1192 seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),
1193 trace_probe_name(&tf->tp));
1194
1195 seq_printf(m, " %s%s", trace_fprobe_symbol(tf),
1196 trace_fprobe_is_return(tf) ? "%return" : "");
1197
1198 for (i = 0; i < tf->tp.nr_args; i++)
1199 seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
1200 seq_putc(m, '\n');
1201
1202 return 0;
1203}
1204
1205/*
1206 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1207 */
1208static int fprobe_register(struct trace_event_call *event,
1209 enum trace_reg type, void *data)
1210{
1211 struct trace_event_file *file = data;
1212
1213 switch (type) {
1214 case TRACE_REG_REGISTER:
1215 return enable_trace_fprobe(event, file);
1216 case TRACE_REG_UNREGISTER:
1217 return disable_trace_fprobe(event, file);
1218
1219#ifdef CONFIG_PERF_EVENTS
1220 case TRACE_REG_PERF_REGISTER:
1221 return enable_trace_fprobe(event, NULL);
1222 case TRACE_REG_PERF_UNREGISTER:
1223 return disable_trace_fprobe(event, NULL);
1224 case TRACE_REG_PERF_OPEN:
1225 case TRACE_REG_PERF_CLOSE:
1226 case TRACE_REG_PERF_ADD:
1227 case TRACE_REG_PERF_DEL:
1228 return 0;
1229#endif
1230 }
1231 return 0;
1232}
1233
1234/*
1235 * Register dynevent at core_initcall. This allows kernel to setup fprobe
1236 * events in postcore_initcall without tracefs.
1237 */
1238static __init int init_fprobe_trace_early(void)
1239{
1240 int ret;
1241
1242 ret = dyn_event_register(&trace_fprobe_ops);
1243 if (ret)
1244 return ret;
1245
1246#ifdef CONFIG_MODULES
1247 ret = register_tracepoint_module_notifier(&tracepoint_module_nb);
1248 if (ret)
1249 return ret;
1250#endif
1251
1252 return 0;
1253}
1254core_initcall(init_fprobe_trace_early);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Fprobe-based tracing events
4 * Copyright (C) 2022 Google LLC.
5 */
6#define pr_fmt(fmt) "trace_fprobe: " fmt
7#include <asm/ptrace.h>
8
9#include <linux/fprobe.h>
10#include <linux/module.h>
11#include <linux/rculist.h>
12#include <linux/security.h>
13#include <linux/tracepoint.h>
14#include <linux/uaccess.h>
15
16#include "trace_dynevent.h"
17#include "trace_probe.h"
18#include "trace_probe_kernel.h"
19#include "trace_probe_tmpl.h"
20
21#define FPROBE_EVENT_SYSTEM "fprobes"
22#define TRACEPOINT_EVENT_SYSTEM "tracepoints"
23#define RETHOOK_MAXACTIVE_MAX 4096
24#define TRACEPOINT_STUB ERR_PTR(-ENOENT)
25
26static int trace_fprobe_create(const char *raw_command);
27static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);
28static int trace_fprobe_release(struct dyn_event *ev);
29static bool trace_fprobe_is_busy(struct dyn_event *ev);
30static bool trace_fprobe_match(const char *system, const char *event,
31 int argc, const char **argv, struct dyn_event *ev);
32
33static struct dyn_event_operations trace_fprobe_ops = {
34 .create = trace_fprobe_create,
35 .show = trace_fprobe_show,
36 .is_busy = trace_fprobe_is_busy,
37 .free = trace_fprobe_release,
38 .match = trace_fprobe_match,
39};
40
41/*
42 * Fprobe event core functions
43 */
44struct trace_fprobe {
45 struct dyn_event devent;
46 struct fprobe fp;
47 const char *symbol;
48 struct tracepoint *tpoint;
49 struct module *mod;
50 struct trace_probe tp;
51};
52
53static bool is_trace_fprobe(struct dyn_event *ev)
54{
55 return ev->ops == &trace_fprobe_ops;
56}
57
58static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev)
59{
60 return container_of(ev, struct trace_fprobe, devent);
61}
62
63/**
64 * for_each_trace_fprobe - iterate over the trace_fprobe list
65 * @pos: the struct trace_fprobe * for each entry
66 * @dpos: the struct dyn_event * to use as a loop cursor
67 */
68#define for_each_trace_fprobe(pos, dpos) \
69 for_each_dyn_event(dpos) \
70 if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos)))
71
72static bool trace_fprobe_is_return(struct trace_fprobe *tf)
73{
74 return tf->fp.exit_handler != NULL;
75}
76
77static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)
78{
79 return tf->tpoint != NULL;
80}
81
82static const char *trace_fprobe_symbol(struct trace_fprobe *tf)
83{
84 return tf->symbol ? tf->symbol : "unknown";
85}
86
87static bool trace_fprobe_is_busy(struct dyn_event *ev)
88{
89 struct trace_fprobe *tf = to_trace_fprobe(ev);
90
91 return trace_probe_is_enabled(&tf->tp);
92}
93
94static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,
95 int argc, const char **argv)
96{
97 char buf[MAX_ARGSTR_LEN + 1];
98
99 if (!argc)
100 return true;
101
102 snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));
103 if (strcmp(buf, argv[0]))
104 return false;
105 argc--; argv++;
106
107 return trace_probe_match_command_args(&tf->tp, argc, argv);
108}
109
110static bool trace_fprobe_match(const char *system, const char *event,
111 int argc, const char **argv, struct dyn_event *ev)
112{
113 struct trace_fprobe *tf = to_trace_fprobe(ev);
114
115 if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))
116 return false;
117
118 if (system && strcmp(trace_probe_group_name(&tf->tp), system))
119 return false;
120
121 return trace_fprobe_match_command_head(tf, argc, argv);
122}
123
124static bool trace_fprobe_is_registered(struct trace_fprobe *tf)
125{
126 return fprobe_is_registered(&tf->fp);
127}
128
129/*
130 * Note that we don't verify the fetch_insn code, since it does not come
131 * from user space.
132 */
133static int
134process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
135 void *dest, void *base)
136{
137 struct pt_regs *regs = rec;
138 unsigned long val;
139 int ret;
140
141retry:
142 /* 1st stage: get value from context */
143 switch (code->op) {
144 case FETCH_OP_STACK:
145 val = regs_get_kernel_stack_nth(regs, code->param);
146 break;
147 case FETCH_OP_STACKP:
148 val = kernel_stack_pointer(regs);
149 break;
150 case FETCH_OP_RETVAL:
151 val = regs_return_value(regs);
152 break;
153#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
154 case FETCH_OP_ARG:
155 val = regs_get_kernel_argument(regs, code->param);
156 break;
157 case FETCH_OP_EDATA:
158 val = *(unsigned long *)((unsigned long)edata + code->offset);
159 break;
160#endif
161 case FETCH_NOP_SYMBOL: /* Ignore a place holder */
162 code++;
163 goto retry;
164 default:
165 ret = process_common_fetch_insn(code, &val);
166 if (ret < 0)
167 return ret;
168 }
169 code++;
170
171 return process_fetch_insn_bottom(code, val, dest, base);
172}
173NOKPROBE_SYMBOL(process_fetch_insn)
174
175/* function entry handler */
176static nokprobe_inline void
177__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
178 struct pt_regs *regs,
179 struct trace_event_file *trace_file)
180{
181 struct fentry_trace_entry_head *entry;
182 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
183 struct trace_event_buffer fbuffer;
184 int dsize;
185
186 if (WARN_ON_ONCE(call != trace_file->event_call))
187 return;
188
189 if (trace_trigger_soft_disabled(trace_file))
190 return;
191
192 dsize = __get_data_size(&tf->tp, regs, NULL);
193
194 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
195 sizeof(*entry) + tf->tp.size + dsize);
196 if (!entry)
197 return;
198
199 fbuffer.regs = regs;
200 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
201 entry->ip = entry_ip;
202 store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize);
203
204 trace_event_buffer_commit(&fbuffer);
205}
206
207static void
208fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
209 struct pt_regs *regs)
210{
211 struct event_file_link *link;
212
213 trace_probe_for_each_link_rcu(link, &tf->tp)
214 __fentry_trace_func(tf, entry_ip, regs, link->file);
215}
216NOKPROBE_SYMBOL(fentry_trace_func);
217
218/* function exit handler */
219static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip,
220 unsigned long ret_ip, struct pt_regs *regs,
221 void *entry_data)
222{
223 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
224
225 if (tf->tp.entry_arg)
226 store_trace_entry_data(entry_data, &tf->tp, regs);
227
228 return 0;
229}
230NOKPROBE_SYMBOL(trace_fprobe_entry_handler)
231
232static nokprobe_inline void
233__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
234 unsigned long ret_ip, struct pt_regs *regs,
235 void *entry_data, struct trace_event_file *trace_file)
236{
237 struct fexit_trace_entry_head *entry;
238 struct trace_event_buffer fbuffer;
239 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
240 int dsize;
241
242 if (WARN_ON_ONCE(call != trace_file->event_call))
243 return;
244
245 if (trace_trigger_soft_disabled(trace_file))
246 return;
247
248 dsize = __get_data_size(&tf->tp, regs, entry_data);
249
250 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
251 sizeof(*entry) + tf->tp.size + dsize);
252 if (!entry)
253 return;
254
255 fbuffer.regs = regs;
256 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
257 entry->func = entry_ip;
258 entry->ret_ip = ret_ip;
259 store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize);
260
261 trace_event_buffer_commit(&fbuffer);
262}
263
264static void
265fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
266 unsigned long ret_ip, struct pt_regs *regs, void *entry_data)
267{
268 struct event_file_link *link;
269
270 trace_probe_for_each_link_rcu(link, &tf->tp)
271 __fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data, link->file);
272}
273NOKPROBE_SYMBOL(fexit_trace_func);
274
275#ifdef CONFIG_PERF_EVENTS
276
277static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
278 struct pt_regs *regs)
279{
280 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
281 struct fentry_trace_entry_head *entry;
282 struct hlist_head *head;
283 int size, __size, dsize;
284 int rctx;
285
286 head = this_cpu_ptr(call->perf_events);
287 if (hlist_empty(head))
288 return 0;
289
290 dsize = __get_data_size(&tf->tp, regs, NULL);
291 __size = sizeof(*entry) + tf->tp.size + dsize;
292 size = ALIGN(__size + sizeof(u32), sizeof(u64));
293 size -= sizeof(u32);
294
295 entry = perf_trace_buf_alloc(size, NULL, &rctx);
296 if (!entry)
297 return 0;
298
299 entry->ip = entry_ip;
300 memset(&entry[1], 0, dsize);
301 store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize);
302 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
303 head, NULL);
304 return 0;
305}
306NOKPROBE_SYMBOL(fentry_perf_func);
307
308static void
309fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
310 unsigned long ret_ip, struct pt_regs *regs,
311 void *entry_data)
312{
313 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
314 struct fexit_trace_entry_head *entry;
315 struct hlist_head *head;
316 int size, __size, dsize;
317 int rctx;
318
319 head = this_cpu_ptr(call->perf_events);
320 if (hlist_empty(head))
321 return;
322
323 dsize = __get_data_size(&tf->tp, regs, entry_data);
324 __size = sizeof(*entry) + tf->tp.size + dsize;
325 size = ALIGN(__size + sizeof(u32), sizeof(u64));
326 size -= sizeof(u32);
327
328 entry = perf_trace_buf_alloc(size, NULL, &rctx);
329 if (!entry)
330 return;
331
332 entry->func = entry_ip;
333 entry->ret_ip = ret_ip;
334 store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize);
335 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
336 head, NULL);
337}
338NOKPROBE_SYMBOL(fexit_perf_func);
339#endif /* CONFIG_PERF_EVENTS */
340
341static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip,
342 unsigned long ret_ip, struct pt_regs *regs,
343 void *entry_data)
344{
345 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
346 int ret = 0;
347
348 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))
349 fentry_trace_func(tf, entry_ip, regs);
350#ifdef CONFIG_PERF_EVENTS
351 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))
352 ret = fentry_perf_func(tf, entry_ip, regs);
353#endif
354 return ret;
355}
356NOKPROBE_SYMBOL(fentry_dispatcher);
357
358static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip,
359 unsigned long ret_ip, struct pt_regs *regs,
360 void *entry_data)
361{
362 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
363
364 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))
365 fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data);
366#ifdef CONFIG_PERF_EVENTS
367 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))
368 fexit_perf_func(tf, entry_ip, ret_ip, regs, entry_data);
369#endif
370}
371NOKPROBE_SYMBOL(fexit_dispatcher);
372
373static void free_trace_fprobe(struct trace_fprobe *tf)
374{
375 if (tf) {
376 trace_probe_cleanup(&tf->tp);
377 kfree(tf->symbol);
378 kfree(tf);
379 }
380}
381
382/*
383 * Allocate new trace_probe and initialize it (including fprobe).
384 */
385static struct trace_fprobe *alloc_trace_fprobe(const char *group,
386 const char *event,
387 const char *symbol,
388 struct tracepoint *tpoint,
389 struct module *mod,
390 int maxactive,
391 int nargs, bool is_return)
392{
393 struct trace_fprobe *tf;
394 int ret = -ENOMEM;
395
396 tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);
397 if (!tf)
398 return ERR_PTR(ret);
399
400 tf->symbol = kstrdup(symbol, GFP_KERNEL);
401 if (!tf->symbol)
402 goto error;
403
404 if (is_return)
405 tf->fp.exit_handler = fexit_dispatcher;
406 else
407 tf->fp.entry_handler = fentry_dispatcher;
408
409 tf->tpoint = tpoint;
410 tf->mod = mod;
411 tf->fp.nr_maxactive = maxactive;
412
413 ret = trace_probe_init(&tf->tp, event, group, false, nargs);
414 if (ret < 0)
415 goto error;
416
417 dyn_event_init(&tf->devent, &trace_fprobe_ops);
418 return tf;
419error:
420 free_trace_fprobe(tf);
421 return ERR_PTR(ret);
422}
423
424static struct trace_fprobe *find_trace_fprobe(const char *event,
425 const char *group)
426{
427 struct dyn_event *pos;
428 struct trace_fprobe *tf;
429
430 for_each_trace_fprobe(tf, pos)
431 if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&
432 strcmp(trace_probe_group_name(&tf->tp), group) == 0)
433 return tf;
434 return NULL;
435}
436
437static inline int __enable_trace_fprobe(struct trace_fprobe *tf)
438{
439 if (trace_fprobe_is_registered(tf))
440 enable_fprobe(&tf->fp);
441
442 return 0;
443}
444
445static void __disable_trace_fprobe(struct trace_probe *tp)
446{
447 struct trace_fprobe *tf;
448
449 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
450 if (!trace_fprobe_is_registered(tf))
451 continue;
452 disable_fprobe(&tf->fp);
453 }
454}
455
456/*
457 * Enable trace_probe
458 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
459 */
460static int enable_trace_fprobe(struct trace_event_call *call,
461 struct trace_event_file *file)
462{
463 struct trace_probe *tp;
464 struct trace_fprobe *tf;
465 bool enabled;
466 int ret = 0;
467
468 tp = trace_probe_primary_from_call(call);
469 if (WARN_ON_ONCE(!tp))
470 return -ENODEV;
471 enabled = trace_probe_is_enabled(tp);
472
473 /* This also changes "enabled" state */
474 if (file) {
475 ret = trace_probe_add_file(tp, file);
476 if (ret)
477 return ret;
478 } else
479 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
480
481 if (!enabled) {
482 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
483 /* TODO: check the fprobe is gone */
484 __enable_trace_fprobe(tf);
485 }
486 }
487
488 return 0;
489}
490
491/*
492 * Disable trace_probe
493 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
494 */
495static int disable_trace_fprobe(struct trace_event_call *call,
496 struct trace_event_file *file)
497{
498 struct trace_probe *tp;
499
500 tp = trace_probe_primary_from_call(call);
501 if (WARN_ON_ONCE(!tp))
502 return -ENODEV;
503
504 if (file) {
505 if (!trace_probe_get_file_link(tp, file))
506 return -ENOENT;
507 if (!trace_probe_has_single_file(tp))
508 goto out;
509 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
510 } else
511 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
512
513 if (!trace_probe_is_enabled(tp))
514 __disable_trace_fprobe(tp);
515
516 out:
517 if (file)
518 /*
519 * Synchronization is done in below function. For perf event,
520 * file == NULL and perf_trace_event_unreg() calls
521 * tracepoint_synchronize_unregister() to ensure synchronize
522 * event. We don't need to care about it.
523 */
524 trace_probe_remove_file(tp, file);
525
526 return 0;
527}
528
529/* Event entry printers */
530static enum print_line_t
531print_fentry_event(struct trace_iterator *iter, int flags,
532 struct trace_event *event)
533{
534 struct fentry_trace_entry_head *field;
535 struct trace_seq *s = &iter->seq;
536 struct trace_probe *tp;
537
538 field = (struct fentry_trace_entry_head *)iter->ent;
539 tp = trace_probe_primary_from_call(
540 container_of(event, struct trace_event_call, event));
541 if (WARN_ON_ONCE(!tp))
542 goto out;
543
544 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
545
546 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
547 goto out;
548
549 trace_seq_putc(s, ')');
550
551 if (trace_probe_print_args(s, tp->args, tp->nr_args,
552 (u8 *)&field[1], field) < 0)
553 goto out;
554
555 trace_seq_putc(s, '\n');
556 out:
557 return trace_handle_return(s);
558}
559
560static enum print_line_t
561print_fexit_event(struct trace_iterator *iter, int flags,
562 struct trace_event *event)
563{
564 struct fexit_trace_entry_head *field;
565 struct trace_seq *s = &iter->seq;
566 struct trace_probe *tp;
567
568 field = (struct fexit_trace_entry_head *)iter->ent;
569 tp = trace_probe_primary_from_call(
570 container_of(event, struct trace_event_call, event));
571 if (WARN_ON_ONCE(!tp))
572 goto out;
573
574 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
575
576 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
577 goto out;
578
579 trace_seq_puts(s, " <- ");
580
581 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
582 goto out;
583
584 trace_seq_putc(s, ')');
585
586 if (trace_probe_print_args(s, tp->args, tp->nr_args,
587 (u8 *)&field[1], field) < 0)
588 goto out;
589
590 trace_seq_putc(s, '\n');
591
592 out:
593 return trace_handle_return(s);
594}
595
596static int fentry_event_define_fields(struct trace_event_call *event_call)
597{
598 int ret;
599 struct fentry_trace_entry_head field;
600 struct trace_probe *tp;
601
602 tp = trace_probe_primary_from_call(event_call);
603 if (WARN_ON_ONCE(!tp))
604 return -ENOENT;
605
606 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
607
608 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
609}
610
611static int fexit_event_define_fields(struct trace_event_call *event_call)
612{
613 int ret;
614 struct fexit_trace_entry_head field;
615 struct trace_probe *tp;
616
617 tp = trace_probe_primary_from_call(event_call);
618 if (WARN_ON_ONCE(!tp))
619 return -ENOENT;
620
621 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
622 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
623
624 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
625}
626
627static struct trace_event_functions fentry_funcs = {
628 .trace = print_fentry_event
629};
630
631static struct trace_event_functions fexit_funcs = {
632 .trace = print_fexit_event
633};
634
635static struct trace_event_fields fentry_fields_array[] = {
636 { .type = TRACE_FUNCTION_TYPE,
637 .define_fields = fentry_event_define_fields },
638 {}
639};
640
641static struct trace_event_fields fexit_fields_array[] = {
642 { .type = TRACE_FUNCTION_TYPE,
643 .define_fields = fexit_event_define_fields },
644 {}
645};
646
647static int fprobe_register(struct trace_event_call *event,
648 enum trace_reg type, void *data);
649
650static inline void init_trace_event_call(struct trace_fprobe *tf)
651{
652 struct trace_event_call *call = trace_probe_event_call(&tf->tp);
653
654 if (trace_fprobe_is_return(tf)) {
655 call->event.funcs = &fexit_funcs;
656 call->class->fields_array = fexit_fields_array;
657 } else {
658 call->event.funcs = &fentry_funcs;
659 call->class->fields_array = fentry_fields_array;
660 }
661
662 call->flags = TRACE_EVENT_FL_FPROBE;
663 call->class->reg = fprobe_register;
664}
665
666static int register_fprobe_event(struct trace_fprobe *tf)
667{
668 init_trace_event_call(tf);
669
670 return trace_probe_register_event_call(&tf->tp);
671}
672
673static int unregister_fprobe_event(struct trace_fprobe *tf)
674{
675 return trace_probe_unregister_event_call(&tf->tp);
676}
677
678static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf)
679{
680 struct tracepoint *tpoint = tf->tpoint;
681 unsigned long ip = (unsigned long)tpoint->probestub;
682 int ret;
683
684 /*
685 * Here, we do 2 steps to enable fprobe on a tracepoint.
686 * At first, put __probestub_##TP function on the tracepoint
687 * and put a fprobe on the stub function.
688 */
689 ret = tracepoint_probe_register_prio_may_exist(tpoint,
690 tpoint->probestub, NULL, 0);
691 if (ret < 0)
692 return ret;
693 return register_fprobe_ips(&tf->fp, &ip, 1);
694}
695
696/* Internal register function - just handle fprobe and flags */
697static int __register_trace_fprobe(struct trace_fprobe *tf)
698{
699 int i, ret;
700
701 /* Should we need new LOCKDOWN flag for fprobe? */
702 ret = security_locked_down(LOCKDOWN_KPROBES);
703 if (ret)
704 return ret;
705
706 if (trace_fprobe_is_registered(tf))
707 return -EINVAL;
708
709 for (i = 0; i < tf->tp.nr_args; i++) {
710 ret = traceprobe_update_arg(&tf->tp.args[i]);
711 if (ret)
712 return ret;
713 }
714
715 /* Set/clear disabled flag according to tp->flag */
716 if (trace_probe_is_enabled(&tf->tp))
717 tf->fp.flags &= ~FPROBE_FL_DISABLED;
718 else
719 tf->fp.flags |= FPROBE_FL_DISABLED;
720
721 if (trace_fprobe_is_tracepoint(tf)) {
722
723 /* This tracepoint is not loaded yet */
724 if (tf->tpoint == TRACEPOINT_STUB)
725 return 0;
726
727 return __regsiter_tracepoint_fprobe(tf);
728 }
729
730 /* TODO: handle filter, nofilter or symbol list */
731 return register_fprobe(&tf->fp, tf->symbol, NULL);
732}
733
734/* Internal unregister function - just handle fprobe and flags */
735static void __unregister_trace_fprobe(struct trace_fprobe *tf)
736{
737 if (trace_fprobe_is_registered(tf)) {
738 unregister_fprobe(&tf->fp);
739 memset(&tf->fp, 0, sizeof(tf->fp));
740 if (trace_fprobe_is_tracepoint(tf)) {
741 tracepoint_probe_unregister(tf->tpoint,
742 tf->tpoint->probestub, NULL);
743 tf->tpoint = NULL;
744 tf->mod = NULL;
745 }
746 }
747}
748
749/* TODO: make this trace_*probe common function */
750/* Unregister a trace_probe and probe_event */
751static int unregister_trace_fprobe(struct trace_fprobe *tf)
752{
753 /* If other probes are on the event, just unregister fprobe */
754 if (trace_probe_has_sibling(&tf->tp))
755 goto unreg;
756
757 /* Enabled event can not be unregistered */
758 if (trace_probe_is_enabled(&tf->tp))
759 return -EBUSY;
760
761 /* If there's a reference to the dynamic event */
762 if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))
763 return -EBUSY;
764
765 /* Will fail if probe is being used by ftrace or perf */
766 if (unregister_fprobe_event(tf))
767 return -EBUSY;
768
769unreg:
770 __unregister_trace_fprobe(tf);
771 dyn_event_remove(&tf->devent);
772 trace_probe_unlink(&tf->tp);
773
774 return 0;
775}
776
777static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig,
778 struct trace_fprobe *comp)
779{
780 struct trace_probe_event *tpe = orig->tp.event;
781 int i;
782
783 list_for_each_entry(orig, &tpe->probes, tp.list) {
784 if (strcmp(trace_fprobe_symbol(orig),
785 trace_fprobe_symbol(comp)))
786 continue;
787
788 /*
789 * trace_probe_compare_arg_type() ensured that nr_args and
790 * each argument name and type are same. Let's compare comm.
791 */
792 for (i = 0; i < orig->tp.nr_args; i++) {
793 if (strcmp(orig->tp.args[i].comm,
794 comp->tp.args[i].comm))
795 break;
796 }
797
798 if (i == orig->tp.nr_args)
799 return true;
800 }
801
802 return false;
803}
804
805static int append_trace_fprobe(struct trace_fprobe *tf, struct trace_fprobe *to)
806{
807 int ret;
808
809 if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) ||
810 trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) {
811 trace_probe_log_set_index(0);
812 trace_probe_log_err(0, DIFF_PROBE_TYPE);
813 return -EEXIST;
814 }
815 ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);
816 if (ret) {
817 /* Note that argument starts index = 2 */
818 trace_probe_log_set_index(ret + 1);
819 trace_probe_log_err(0, DIFF_ARG_TYPE);
820 return -EEXIST;
821 }
822 if (trace_fprobe_has_same_fprobe(to, tf)) {
823 trace_probe_log_set_index(0);
824 trace_probe_log_err(0, SAME_PROBE);
825 return -EEXIST;
826 }
827
828 /* Append to existing event */
829 ret = trace_probe_append(&tf->tp, &to->tp);
830 if (ret)
831 return ret;
832
833 ret = __register_trace_fprobe(tf);
834 if (ret)
835 trace_probe_unlink(&tf->tp);
836 else
837 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
838
839 return ret;
840}
841
842/* Register a trace_probe and probe_event */
843static int register_trace_fprobe(struct trace_fprobe *tf)
844{
845 struct trace_fprobe *old_tf;
846 int ret;
847
848 mutex_lock(&event_mutex);
849
850 old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),
851 trace_probe_group_name(&tf->tp));
852 if (old_tf) {
853 ret = append_trace_fprobe(tf, old_tf);
854 goto end;
855 }
856
857 /* Register new event */
858 ret = register_fprobe_event(tf);
859 if (ret) {
860 if (ret == -EEXIST) {
861 trace_probe_log_set_index(0);
862 trace_probe_log_err(0, EVENT_EXIST);
863 } else
864 pr_warn("Failed to register probe event(%d)\n", ret);
865 goto end;
866 }
867
868 /* Register fprobe */
869 ret = __register_trace_fprobe(tf);
870 if (ret < 0)
871 unregister_fprobe_event(tf);
872 else
873 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
874
875end:
876 mutex_unlock(&event_mutex);
877 return ret;
878}
879
880struct __find_tracepoint_cb_data {
881 const char *tp_name;
882 struct tracepoint *tpoint;
883 struct module *mod;
884};
885
886static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv)
887{
888 struct __find_tracepoint_cb_data *data = priv;
889
890 if (!data->tpoint && !strcmp(data->tp_name, tp->name)) {
891 data->tpoint = tp;
892 if (!data->mod) {
893 data->mod = mod;
894 if (!try_module_get(data->mod)) {
895 data->tpoint = NULL;
896 data->mod = NULL;
897 }
898 }
899 }
900}
901
902static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)
903{
904 struct __find_tracepoint_cb_data *data = priv;
905
906 if (!data->tpoint && !strcmp(data->tp_name, tp->name))
907 data->tpoint = tp;
908}
909
910/*
911 * Find a tracepoint from kernel and module. If the tracepoint is in a module,
912 * this increments the module refcount to prevent unloading until the
913 * trace_fprobe is registered to the list. After registering the trace_fprobe
914 * on the trace_fprobe list, the module refcount is decremented because
915 * tracepoint_probe_module_cb will handle it.
916 */
917static struct tracepoint *find_tracepoint(const char *tp_name,
918 struct module **tp_mod)
919{
920 struct __find_tracepoint_cb_data data = {
921 .tp_name = tp_name,
922 .mod = NULL,
923 };
924
925 for_each_kernel_tracepoint(__find_tracepoint_cb, &data);
926
927 if (!data.tpoint && IS_ENABLED(CONFIG_MODULES)) {
928 for_each_module_tracepoint(__find_tracepoint_module_cb, &data);
929 *tp_mod = data.mod;
930 }
931
932 return data.tpoint;
933}
934
935#ifdef CONFIG_MODULES
936static void reenable_trace_fprobe(struct trace_fprobe *tf)
937{
938 struct trace_probe *tp = &tf->tp;
939
940 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
941 __enable_trace_fprobe(tf);
942 }
943}
944
945static struct tracepoint *find_tracepoint_in_module(struct module *mod,
946 const char *tp_name)
947{
948 struct __find_tracepoint_cb_data data = {
949 .tp_name = tp_name,
950 .mod = mod,
951 };
952
953 for_each_tracepoint_in_module(mod, __find_tracepoint_module_cb, &data);
954 return data.tpoint;
955}
956
957static int __tracepoint_probe_module_cb(struct notifier_block *self,
958 unsigned long val, void *data)
959{
960 struct tp_module *tp_mod = data;
961 struct tracepoint *tpoint;
962 struct trace_fprobe *tf;
963 struct dyn_event *pos;
964
965 if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)
966 return NOTIFY_DONE;
967
968 mutex_lock(&event_mutex);
969 for_each_trace_fprobe(tf, pos) {
970 if (val == MODULE_STATE_COMING && tf->tpoint == TRACEPOINT_STUB) {
971 tpoint = find_tracepoint_in_module(tp_mod->mod, tf->symbol);
972 if (tpoint) {
973 tf->tpoint = tpoint;
974 tf->mod = tp_mod->mod;
975 if (!WARN_ON_ONCE(__regsiter_tracepoint_fprobe(tf)) &&
976 trace_probe_is_enabled(&tf->tp))
977 reenable_trace_fprobe(tf);
978 }
979 } else if (val == MODULE_STATE_GOING && tp_mod->mod == tf->mod) {
980 tracepoint_probe_unregister(tf->tpoint,
981 tf->tpoint->probestub, NULL);
982 tf->tpoint = NULL;
983 tf->mod = NULL;
984 }
985 }
986 mutex_unlock(&event_mutex);
987
988 return NOTIFY_DONE;
989}
990
991static struct notifier_block tracepoint_module_nb = {
992 .notifier_call = __tracepoint_probe_module_cb,
993};
994#endif /* CONFIG_MODULES */
995
996static int parse_symbol_and_return(int argc, const char *argv[],
997 char **symbol, bool *is_return,
998 bool is_tracepoint)
999{
1000 char *tmp = strchr(argv[1], '%');
1001 int i;
1002
1003 if (tmp) {
1004 int len = tmp - argv[1];
1005
1006 if (!is_tracepoint && !strcmp(tmp, "%return")) {
1007 *is_return = true;
1008 } else {
1009 trace_probe_log_err(len, BAD_ADDR_SUFFIX);
1010 return -EINVAL;
1011 }
1012 *symbol = kmemdup_nul(argv[1], len, GFP_KERNEL);
1013 } else
1014 *symbol = kstrdup(argv[1], GFP_KERNEL);
1015 if (!*symbol)
1016 return -ENOMEM;
1017
1018 if (*is_return)
1019 return 0;
1020
1021 if (is_tracepoint) {
1022 tmp = *symbol;
1023 while (*tmp && (isalnum(*tmp) || *tmp == '_'))
1024 tmp++;
1025 if (*tmp) {
1026 /* find a wrong character. */
1027 trace_probe_log_err(tmp - *symbol, BAD_TP_NAME);
1028 kfree(*symbol);
1029 *symbol = NULL;
1030 return -EINVAL;
1031 }
1032 }
1033
1034 /* If there is $retval, this should be a return fprobe. */
1035 for (i = 2; i < argc; i++) {
1036 tmp = strstr(argv[i], "$retval");
1037 if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') {
1038 if (is_tracepoint) {
1039 trace_probe_log_set_index(i);
1040 trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE);
1041 kfree(*symbol);
1042 *symbol = NULL;
1043 return -EINVAL;
1044 }
1045 *is_return = true;
1046 break;
1047 }
1048 }
1049 return 0;
1050}
1051
1052static int __trace_fprobe_create(int argc, const char *argv[])
1053{
1054 /*
1055 * Argument syntax:
1056 * - Add fentry probe:
1057 * f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS]
1058 * - Add fexit probe:
1059 * f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS]
1060 * - Add tracepoint probe:
1061 * t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS]
1062 *
1063 * Fetch args:
1064 * $retval : fetch return value
1065 * $stack : fetch stack address
1066 * $stackN : fetch Nth entry of stack (N:0-)
1067 * $argN : fetch Nth argument (N:1-)
1068 * $comm : fetch current task comm
1069 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
1070 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
1071 * Dereferencing memory fetch:
1072 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
1073 * Alias name of args:
1074 * NAME=FETCHARG : set NAME as alias of FETCHARG.
1075 * Type of args:
1076 * FETCHARG:TYPE : use TYPE instead of unsigned long.
1077 */
1078 struct trace_fprobe *tf = NULL;
1079 int i, len, new_argc = 0, ret = 0;
1080 bool is_return = false;
1081 char *symbol = NULL;
1082 const char *event = NULL, *group = FPROBE_EVENT_SYSTEM;
1083 const char **new_argv = NULL;
1084 int maxactive = 0;
1085 char buf[MAX_EVENT_NAME_LEN];
1086 char gbuf[MAX_EVENT_NAME_LEN];
1087 char sbuf[KSYM_NAME_LEN];
1088 char abuf[MAX_BTF_ARGS_LEN];
1089 char *dbuf = NULL;
1090 bool is_tracepoint = false;
1091 struct module *tp_mod = NULL;
1092 struct tracepoint *tpoint = NULL;
1093 struct traceprobe_parse_context ctx = {
1094 .flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE,
1095 };
1096
1097 if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2)
1098 return -ECANCELED;
1099
1100 if (argv[0][0] == 't') {
1101 is_tracepoint = true;
1102 group = TRACEPOINT_EVENT_SYSTEM;
1103 }
1104
1105 trace_probe_log_init("trace_fprobe", argc, argv);
1106
1107 event = strchr(&argv[0][1], ':');
1108 if (event)
1109 event++;
1110
1111 if (isdigit(argv[0][1])) {
1112 if (event)
1113 len = event - &argv[0][1] - 1;
1114 else
1115 len = strlen(&argv[0][1]);
1116 if (len > MAX_EVENT_NAME_LEN - 1) {
1117 trace_probe_log_err(1, BAD_MAXACT);
1118 goto parse_error;
1119 }
1120 memcpy(buf, &argv[0][1], len);
1121 buf[len] = '\0';
1122 ret = kstrtouint(buf, 0, &maxactive);
1123 if (ret || !maxactive) {
1124 trace_probe_log_err(1, BAD_MAXACT);
1125 goto parse_error;
1126 }
1127 /* fprobe rethook instances are iterated over via a list. The
1128 * maximum should stay reasonable.
1129 */
1130 if (maxactive > RETHOOK_MAXACTIVE_MAX) {
1131 trace_probe_log_err(1, MAXACT_TOO_BIG);
1132 goto parse_error;
1133 }
1134 }
1135
1136 trace_probe_log_set_index(1);
1137
1138 /* a symbol(or tracepoint) must be specified */
1139 ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint);
1140 if (ret < 0)
1141 goto parse_error;
1142
1143 if (!is_return && maxactive) {
1144 trace_probe_log_set_index(0);
1145 trace_probe_log_err(1, BAD_MAXACT_TYPE);
1146 goto parse_error;
1147 }
1148
1149 trace_probe_log_set_index(0);
1150 if (event) {
1151 ret = traceprobe_parse_event_name(&event, &group, gbuf,
1152 event - argv[0]);
1153 if (ret)
1154 goto parse_error;
1155 }
1156
1157 if (!event) {
1158 /* Make a new event name */
1159 if (is_tracepoint)
1160 snprintf(buf, MAX_EVENT_NAME_LEN, "%s%s",
1161 isdigit(*symbol) ? "_" : "", symbol);
1162 else
1163 snprintf(buf, MAX_EVENT_NAME_LEN, "%s__%s", symbol,
1164 is_return ? "exit" : "entry");
1165 sanitize_event_name(buf);
1166 event = buf;
1167 }
1168
1169 if (is_return)
1170 ctx.flags |= TPARG_FL_RETURN;
1171 else
1172 ctx.flags |= TPARG_FL_FENTRY;
1173
1174 if (is_tracepoint) {
1175 ctx.flags |= TPARG_FL_TPOINT;
1176 tpoint = find_tracepoint(symbol, &tp_mod);
1177 if (tpoint) {
1178 ctx.funcname = kallsyms_lookup(
1179 (unsigned long)tpoint->probestub,
1180 NULL, NULL, NULL, sbuf);
1181 } else if (IS_ENABLED(CONFIG_MODULES)) {
1182 /* This *may* be loaded afterwards */
1183 tpoint = TRACEPOINT_STUB;
1184 ctx.funcname = symbol;
1185 } else {
1186 trace_probe_log_set_index(1);
1187 trace_probe_log_err(0, NO_TRACEPOINT);
1188 goto parse_error;
1189 }
1190 } else
1191 ctx.funcname = symbol;
1192
1193 argc -= 2; argv += 2;
1194 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1195 abuf, MAX_BTF_ARGS_LEN, &ctx);
1196 if (IS_ERR(new_argv)) {
1197 ret = PTR_ERR(new_argv);
1198 new_argv = NULL;
1199 goto out;
1200 }
1201 if (new_argv) {
1202 argc = new_argc;
1203 argv = new_argv;
1204 }
1205 if (argc > MAX_TRACE_ARGS) {
1206 ret = -E2BIG;
1207 goto out;
1208 }
1209
1210 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf);
1211 if (ret)
1212 goto out;
1213
1214 /* setup a probe */
1215 tf = alloc_trace_fprobe(group, event, symbol, tpoint, tp_mod,
1216 maxactive, argc, is_return);
1217 if (IS_ERR(tf)) {
1218 ret = PTR_ERR(tf);
1219 /* This must return -ENOMEM, else there is a bug */
1220 WARN_ON_ONCE(ret != -ENOMEM);
1221 goto out; /* We know tf is not allocated */
1222 }
1223
1224 /* parse arguments */
1225 for (i = 0; i < argc; i++) {
1226 trace_probe_log_set_index(i + 2);
1227 ctx.offset = 0;
1228 ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], &ctx);
1229 if (ret)
1230 goto error; /* This can be -ENOMEM */
1231 }
1232
1233 if (is_return && tf->tp.entry_arg) {
1234 tf->fp.entry_handler = trace_fprobe_entry_handler;
1235 tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp);
1236 }
1237
1238 ret = traceprobe_set_print_fmt(&tf->tp,
1239 is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL);
1240 if (ret < 0)
1241 goto error;
1242
1243 ret = register_trace_fprobe(tf);
1244 if (ret) {
1245 trace_probe_log_set_index(1);
1246 if (ret == -EILSEQ)
1247 trace_probe_log_err(0, BAD_INSN_BNDRY);
1248 else if (ret == -ENOENT)
1249 trace_probe_log_err(0, BAD_PROBE_ADDR);
1250 else if (ret != -ENOMEM && ret != -EEXIST)
1251 trace_probe_log_err(0, FAIL_REG_PROBE);
1252 goto error;
1253 }
1254
1255out:
1256 if (tp_mod)
1257 module_put(tp_mod);
1258 traceprobe_finish_parse(&ctx);
1259 trace_probe_log_clear();
1260 kfree(new_argv);
1261 kfree(symbol);
1262 kfree(dbuf);
1263 return ret;
1264
1265parse_error:
1266 ret = -EINVAL;
1267error:
1268 free_trace_fprobe(tf);
1269 goto out;
1270}
1271
1272static int trace_fprobe_create(const char *raw_command)
1273{
1274 return trace_probe_create(raw_command, __trace_fprobe_create);
1275}
1276
1277static int trace_fprobe_release(struct dyn_event *ev)
1278{
1279 struct trace_fprobe *tf = to_trace_fprobe(ev);
1280 int ret = unregister_trace_fprobe(tf);
1281
1282 if (!ret)
1283 free_trace_fprobe(tf);
1284 return ret;
1285}
1286
1287static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
1288{
1289 struct trace_fprobe *tf = to_trace_fprobe(ev);
1290 int i;
1291
1292 if (trace_fprobe_is_tracepoint(tf))
1293 seq_putc(m, 't');
1294 else
1295 seq_putc(m, 'f');
1296 if (trace_fprobe_is_return(tf) && tf->fp.nr_maxactive)
1297 seq_printf(m, "%d", tf->fp.nr_maxactive);
1298 seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),
1299 trace_probe_name(&tf->tp));
1300
1301 seq_printf(m, " %s%s", trace_fprobe_symbol(tf),
1302 trace_fprobe_is_return(tf) ? "%return" : "");
1303
1304 for (i = 0; i < tf->tp.nr_args; i++)
1305 seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
1306 seq_putc(m, '\n');
1307
1308 return 0;
1309}
1310
1311/*
1312 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1313 */
1314static int fprobe_register(struct trace_event_call *event,
1315 enum trace_reg type, void *data)
1316{
1317 struct trace_event_file *file = data;
1318
1319 switch (type) {
1320 case TRACE_REG_REGISTER:
1321 return enable_trace_fprobe(event, file);
1322 case TRACE_REG_UNREGISTER:
1323 return disable_trace_fprobe(event, file);
1324
1325#ifdef CONFIG_PERF_EVENTS
1326 case TRACE_REG_PERF_REGISTER:
1327 return enable_trace_fprobe(event, NULL);
1328 case TRACE_REG_PERF_UNREGISTER:
1329 return disable_trace_fprobe(event, NULL);
1330 case TRACE_REG_PERF_OPEN:
1331 case TRACE_REG_PERF_CLOSE:
1332 case TRACE_REG_PERF_ADD:
1333 case TRACE_REG_PERF_DEL:
1334 return 0;
1335#endif
1336 }
1337 return 0;
1338}
1339
1340/*
1341 * Register dynevent at core_initcall. This allows kernel to setup fprobe
1342 * events in postcore_initcall without tracefs.
1343 */
1344static __init int init_fprobe_trace_early(void)
1345{
1346 int ret;
1347
1348 ret = dyn_event_register(&trace_fprobe_ops);
1349 if (ret)
1350 return ret;
1351
1352#ifdef CONFIG_MODULES
1353 ret = register_tracepoint_module_notifier(&tracepoint_module_nb);
1354 if (ret)
1355 return ret;
1356#endif
1357
1358 return 0;
1359}
1360core_initcall(init_fprobe_trace_early);