Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fprobe - Simple ftrace probe wrapper for function entry.
4 */
5#define pr_fmt(fmt) "fprobe: " fmt
6
7#include <linux/err.h>
8#include <linux/fprobe.h>
9#include <linux/kallsyms.h>
10#include <linux/kprobes.h>
11#include <linux/rethook.h>
12#include <linux/slab.h>
13#include <linux/sort.h>
14
15#include "trace.h"
16
17struct fprobe_rethook_node {
18 struct rethook_node node;
19 unsigned long entry_ip;
20};
21
22static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
23 struct ftrace_ops *ops, struct ftrace_regs *fregs)
24{
25 struct fprobe_rethook_node *fpr;
26 struct rethook_node *rh;
27 struct fprobe *fp;
28 int bit;
29
30 fp = container_of(ops, struct fprobe, ops);
31 if (fprobe_disabled(fp))
32 return;
33
34 bit = ftrace_test_recursion_trylock(ip, parent_ip);
35 if (bit < 0) {
36 fp->nmissed++;
37 return;
38 }
39
40 if (fp->entry_handler)
41 fp->entry_handler(fp, ip, ftrace_get_regs(fregs));
42
43 if (fp->exit_handler) {
44 rh = rethook_try_get(fp->rethook);
45 if (!rh) {
46 fp->nmissed++;
47 goto out;
48 }
49 fpr = container_of(rh, struct fprobe_rethook_node, node);
50 fpr->entry_ip = ip;
51 rethook_hook(rh, ftrace_get_regs(fregs), true);
52 }
53
54out:
55 ftrace_test_recursion_unlock(bit);
56}
57NOKPROBE_SYMBOL(fprobe_handler);
58
59static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
60 struct ftrace_ops *ops, struct ftrace_regs *fregs)
61{
62 struct fprobe *fp = container_of(ops, struct fprobe, ops);
63
64 if (unlikely(kprobe_running())) {
65 fp->nmissed++;
66 return;
67 }
68 kprobe_busy_begin();
69 fprobe_handler(ip, parent_ip, ops, fregs);
70 kprobe_busy_end();
71}
72
73static void fprobe_exit_handler(struct rethook_node *rh, void *data,
74 struct pt_regs *regs)
75{
76 struct fprobe *fp = (struct fprobe *)data;
77 struct fprobe_rethook_node *fpr;
78
79 if (!fp || fprobe_disabled(fp))
80 return;
81
82 fpr = container_of(rh, struct fprobe_rethook_node, node);
83
84 fp->exit_handler(fp, fpr->entry_ip, regs);
85}
86NOKPROBE_SYMBOL(fprobe_exit_handler);
87
88static int symbols_cmp(const void *a, const void *b)
89{
90 const char **str_a = (const char **) a;
91 const char **str_b = (const char **) b;
92
93 return strcmp(*str_a, *str_b);
94}
95
96/* Convert ftrace location address from symbols */
97static unsigned long *get_ftrace_locations(const char **syms, int num)
98{
99 unsigned long *addrs;
100
101 /* Convert symbols to symbol address */
102 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
103 if (!addrs)
104 return ERR_PTR(-ENOMEM);
105
106 /* ftrace_lookup_symbols expects sorted symbols */
107 sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
108
109 if (!ftrace_lookup_symbols(syms, num, addrs))
110 return addrs;
111
112 kfree(addrs);
113 return ERR_PTR(-ENOENT);
114}
115
116static void fprobe_init(struct fprobe *fp)
117{
118 fp->nmissed = 0;
119 if (fprobe_shared_with_kprobes(fp))
120 fp->ops.func = fprobe_kprobe_handler;
121 else
122 fp->ops.func = fprobe_handler;
123 fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS;
124}
125
126static int fprobe_init_rethook(struct fprobe *fp, int num)
127{
128 int i, size;
129
130 if (num < 0)
131 return -EINVAL;
132
133 if (!fp->exit_handler) {
134 fp->rethook = NULL;
135 return 0;
136 }
137
138 /* Initialize rethook if needed */
139 size = num * num_possible_cpus() * 2;
140 if (size < 0)
141 return -E2BIG;
142
143 fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler);
144 if (!fp->rethook)
145 return -ENOMEM;
146 for (i = 0; i < size; i++) {
147 struct fprobe_rethook_node *node;
148
149 node = kzalloc(sizeof(*node), GFP_KERNEL);
150 if (!node) {
151 rethook_free(fp->rethook);
152 fp->rethook = NULL;
153 return -ENOMEM;
154 }
155 rethook_add_node(fp->rethook, &node->node);
156 }
157 return 0;
158}
159
160static void fprobe_fail_cleanup(struct fprobe *fp)
161{
162 if (fp->rethook) {
163 /* Don't need to cleanup rethook->handler because this is not used. */
164 rethook_free(fp->rethook);
165 fp->rethook = NULL;
166 }
167 ftrace_free_filter(&fp->ops);
168}
169
170/**
171 * register_fprobe() - Register fprobe to ftrace by pattern.
172 * @fp: A fprobe data structure to be registered.
173 * @filter: A wildcard pattern of probed symbols.
174 * @notfilter: A wildcard pattern of NOT probed symbols.
175 *
176 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
177 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
178 *
179 * Return 0 if @fp is registered successfully, -errno if not.
180 */
181int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
182{
183 struct ftrace_hash *hash;
184 unsigned char *str;
185 int ret, len;
186
187 if (!fp || !filter)
188 return -EINVAL;
189
190 fprobe_init(fp);
191
192 len = strlen(filter);
193 str = kstrdup(filter, GFP_KERNEL);
194 ret = ftrace_set_filter(&fp->ops, str, len, 0);
195 kfree(str);
196 if (ret)
197 return ret;
198
199 if (notfilter) {
200 len = strlen(notfilter);
201 str = kstrdup(notfilter, GFP_KERNEL);
202 ret = ftrace_set_notrace(&fp->ops, str, len, 0);
203 kfree(str);
204 if (ret)
205 goto out;
206 }
207
208 /* TODO:
209 * correctly calculate the total number of filtered symbols
210 * from both filter and notfilter.
211 */
212 hash = rcu_access_pointer(fp->ops.local_hash.filter_hash);
213 if (WARN_ON_ONCE(!hash))
214 goto out;
215
216 ret = fprobe_init_rethook(fp, (int)hash->count);
217 if (!ret)
218 ret = register_ftrace_function(&fp->ops);
219
220out:
221 if (ret)
222 fprobe_fail_cleanup(fp);
223 return ret;
224}
225EXPORT_SYMBOL_GPL(register_fprobe);
226
227/**
228 * register_fprobe_ips() - Register fprobe to ftrace by address.
229 * @fp: A fprobe data structure to be registered.
230 * @addrs: An array of target ftrace location addresses.
231 * @num: The number of entries of @addrs.
232 *
233 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
234 * The @addrs must be the addresses of ftrace location address, which may be
235 * the symbol address + arch-dependent offset.
236 * If you unsure what this mean, please use other registration functions.
237 *
238 * Return 0 if @fp is registered successfully, -errno if not.
239 */
240int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
241{
242 int ret;
243
244 if (!fp || !addrs || num <= 0)
245 return -EINVAL;
246
247 fprobe_init(fp);
248
249 ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0);
250 if (ret)
251 return ret;
252
253 ret = fprobe_init_rethook(fp, num);
254 if (!ret)
255 ret = register_ftrace_function(&fp->ops);
256
257 if (ret)
258 fprobe_fail_cleanup(fp);
259 return ret;
260}
261EXPORT_SYMBOL_GPL(register_fprobe_ips);
262
263/**
264 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
265 * @fp: A fprobe data structure to be registered.
266 * @syms: An array of target symbols.
267 * @num: The number of entries of @syms.
268 *
269 * Register @fp to the symbols given by @syms array. This will be useful if
270 * you are sure the symbols exist in the kernel.
271 *
272 * Return 0 if @fp is registered successfully, -errno if not.
273 */
274int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
275{
276 unsigned long *addrs;
277 int ret;
278
279 if (!fp || !syms || num <= 0)
280 return -EINVAL;
281
282 addrs = get_ftrace_locations(syms, num);
283 if (IS_ERR(addrs))
284 return PTR_ERR(addrs);
285
286 ret = register_fprobe_ips(fp, addrs, num);
287
288 kfree(addrs);
289
290 return ret;
291}
292EXPORT_SYMBOL_GPL(register_fprobe_syms);
293
294/**
295 * unregister_fprobe() - Unregister fprobe from ftrace
296 * @fp: A fprobe data structure to be unregistered.
297 *
298 * Unregister fprobe (and remove ftrace hooks from the function entries).
299 *
300 * Return 0 if @fp is unregistered successfully, -errno if not.
301 */
302int unregister_fprobe(struct fprobe *fp)
303{
304 int ret;
305
306 if (!fp || (fp->ops.saved_func != fprobe_handler &&
307 fp->ops.saved_func != fprobe_kprobe_handler))
308 return -EINVAL;
309
310 /*
311 * rethook_free() starts disabling the rethook, but the rethook handlers
312 * may be running on other processors at this point. To make sure that all
313 * current running handlers are finished, call unregister_ftrace_function()
314 * after this.
315 */
316 if (fp->rethook)
317 rethook_free(fp->rethook);
318
319 ret = unregister_ftrace_function(&fp->ops);
320 if (ret < 0)
321 return ret;
322
323 ftrace_free_filter(&fp->ops);
324
325 return ret;
326}
327EXPORT_SYMBOL_GPL(unregister_fprobe);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fprobe - Simple ftrace probe wrapper for function entry.
4 */
5#define pr_fmt(fmt) "fprobe: " fmt
6
7#include <linux/err.h>
8#include <linux/fprobe.h>
9#include <linux/kallsyms.h>
10#include <linux/kprobes.h>
11#include <linux/rethook.h>
12#include <linux/slab.h>
13#include <linux/sort.h>
14
15#include "trace.h"
16
17struct fprobe_rethook_node {
18 struct rethook_node node;
19 unsigned long entry_ip;
20 unsigned long entry_parent_ip;
21 char data[];
22};
23
24static inline void __fprobe_handler(unsigned long ip, unsigned long parent_ip,
25 struct ftrace_ops *ops, struct ftrace_regs *fregs)
26{
27 struct fprobe_rethook_node *fpr;
28 struct rethook_node *rh = NULL;
29 struct fprobe *fp;
30 void *entry_data = NULL;
31 int ret = 0;
32
33 fp = container_of(ops, struct fprobe, ops);
34
35 if (fp->exit_handler) {
36 rh = rethook_try_get(fp->rethook);
37 if (!rh) {
38 fp->nmissed++;
39 return;
40 }
41 fpr = container_of(rh, struct fprobe_rethook_node, node);
42 fpr->entry_ip = ip;
43 fpr->entry_parent_ip = parent_ip;
44 if (fp->entry_data_size)
45 entry_data = fpr->data;
46 }
47
48 if (fp->entry_handler)
49 ret = fp->entry_handler(fp, ip, parent_ip, ftrace_get_regs(fregs), entry_data);
50
51 /* If entry_handler returns !0, nmissed is not counted. */
52 if (rh) {
53 if (ret)
54 rethook_recycle(rh);
55 else
56 rethook_hook(rh, ftrace_get_regs(fregs), true);
57 }
58}
59
60static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
61 struct ftrace_ops *ops, struct ftrace_regs *fregs)
62{
63 struct fprobe *fp;
64 int bit;
65
66 fp = container_of(ops, struct fprobe, ops);
67 if (fprobe_disabled(fp))
68 return;
69
70 /* recursion detection has to go before any traceable function and
71 * all functions before this point should be marked as notrace
72 */
73 bit = ftrace_test_recursion_trylock(ip, parent_ip);
74 if (bit < 0) {
75 fp->nmissed++;
76 return;
77 }
78 __fprobe_handler(ip, parent_ip, ops, fregs);
79 ftrace_test_recursion_unlock(bit);
80
81}
82NOKPROBE_SYMBOL(fprobe_handler);
83
84static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
85 struct ftrace_ops *ops, struct ftrace_regs *fregs)
86{
87 struct fprobe *fp;
88 int bit;
89
90 fp = container_of(ops, struct fprobe, ops);
91 if (fprobe_disabled(fp))
92 return;
93
94 /* recursion detection has to go before any traceable function and
95 * all functions called before this point should be marked as notrace
96 */
97 bit = ftrace_test_recursion_trylock(ip, parent_ip);
98 if (bit < 0) {
99 fp->nmissed++;
100 return;
101 }
102
103 /*
104 * This user handler is shared with other kprobes and is not expected to be
105 * called recursively. So if any other kprobe handler is running, this will
106 * exit as kprobe does. See the section 'Share the callbacks with kprobes'
107 * in Documentation/trace/fprobe.rst for more information.
108 */
109 if (unlikely(kprobe_running())) {
110 fp->nmissed++;
111 goto recursion_unlock;
112 }
113
114 kprobe_busy_begin();
115 __fprobe_handler(ip, parent_ip, ops, fregs);
116 kprobe_busy_end();
117
118recursion_unlock:
119 ftrace_test_recursion_unlock(bit);
120}
121
122static void fprobe_exit_handler(struct rethook_node *rh, void *data,
123 unsigned long ret_ip, struct pt_regs *regs)
124{
125 struct fprobe *fp = (struct fprobe *)data;
126 struct fprobe_rethook_node *fpr;
127 int bit;
128
129 if (!fp || fprobe_disabled(fp))
130 return;
131
132 fpr = container_of(rh, struct fprobe_rethook_node, node);
133
134 /*
135 * we need to assure no calls to traceable functions in-between the
136 * end of fprobe_handler and the beginning of fprobe_exit_handler.
137 */
138 bit = ftrace_test_recursion_trylock(fpr->entry_ip, fpr->entry_parent_ip);
139 if (bit < 0) {
140 fp->nmissed++;
141 return;
142 }
143
144 fp->exit_handler(fp, fpr->entry_ip, ret_ip, regs,
145 fp->entry_data_size ? (void *)fpr->data : NULL);
146 ftrace_test_recursion_unlock(bit);
147}
148NOKPROBE_SYMBOL(fprobe_exit_handler);
149
150static int symbols_cmp(const void *a, const void *b)
151{
152 const char **str_a = (const char **) a;
153 const char **str_b = (const char **) b;
154
155 return strcmp(*str_a, *str_b);
156}
157
158/* Convert ftrace location address from symbols */
159static unsigned long *get_ftrace_locations(const char **syms, int num)
160{
161 unsigned long *addrs;
162
163 /* Convert symbols to symbol address */
164 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
165 if (!addrs)
166 return ERR_PTR(-ENOMEM);
167
168 /* ftrace_lookup_symbols expects sorted symbols */
169 sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
170
171 if (!ftrace_lookup_symbols(syms, num, addrs))
172 return addrs;
173
174 kfree(addrs);
175 return ERR_PTR(-ENOENT);
176}
177
178static void fprobe_init(struct fprobe *fp)
179{
180 fp->nmissed = 0;
181 if (fprobe_shared_with_kprobes(fp))
182 fp->ops.func = fprobe_kprobe_handler;
183 else
184 fp->ops.func = fprobe_handler;
185 fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS;
186}
187
188static int fprobe_init_rethook(struct fprobe *fp, int num)
189{
190 int size;
191
192 if (!fp->exit_handler) {
193 fp->rethook = NULL;
194 return 0;
195 }
196
197 /* Initialize rethook if needed */
198 if (fp->nr_maxactive)
199 num = fp->nr_maxactive;
200 else
201 num *= num_possible_cpus() * 2;
202 if (num <= 0)
203 return -EINVAL;
204
205 size = sizeof(struct fprobe_rethook_node) + fp->entry_data_size;
206
207 /* Initialize rethook */
208 fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler, size, num);
209 if (IS_ERR(fp->rethook))
210 return PTR_ERR(fp->rethook);
211
212 return 0;
213}
214
215static void fprobe_fail_cleanup(struct fprobe *fp)
216{
217 if (!IS_ERR_OR_NULL(fp->rethook)) {
218 /* Don't need to cleanup rethook->handler because this is not used. */
219 rethook_free(fp->rethook);
220 fp->rethook = NULL;
221 }
222 ftrace_free_filter(&fp->ops);
223}
224
225/**
226 * register_fprobe() - Register fprobe to ftrace by pattern.
227 * @fp: A fprobe data structure to be registered.
228 * @filter: A wildcard pattern of probed symbols.
229 * @notfilter: A wildcard pattern of NOT probed symbols.
230 *
231 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
232 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
233 *
234 * Return 0 if @fp is registered successfully, -errno if not.
235 */
236int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
237{
238 struct ftrace_hash *hash;
239 unsigned char *str;
240 int ret, len;
241
242 if (!fp || !filter)
243 return -EINVAL;
244
245 fprobe_init(fp);
246
247 len = strlen(filter);
248 str = kstrdup(filter, GFP_KERNEL);
249 ret = ftrace_set_filter(&fp->ops, str, len, 0);
250 kfree(str);
251 if (ret)
252 return ret;
253
254 if (notfilter) {
255 len = strlen(notfilter);
256 str = kstrdup(notfilter, GFP_KERNEL);
257 ret = ftrace_set_notrace(&fp->ops, str, len, 0);
258 kfree(str);
259 if (ret)
260 goto out;
261 }
262
263 /* TODO:
264 * correctly calculate the total number of filtered symbols
265 * from both filter and notfilter.
266 */
267 hash = rcu_access_pointer(fp->ops.local_hash.filter_hash);
268 if (WARN_ON_ONCE(!hash))
269 goto out;
270
271 ret = fprobe_init_rethook(fp, (int)hash->count);
272 if (!ret)
273 ret = register_ftrace_function(&fp->ops);
274
275out:
276 if (ret)
277 fprobe_fail_cleanup(fp);
278 return ret;
279}
280EXPORT_SYMBOL_GPL(register_fprobe);
281
282/**
283 * register_fprobe_ips() - Register fprobe to ftrace by address.
284 * @fp: A fprobe data structure to be registered.
285 * @addrs: An array of target ftrace location addresses.
286 * @num: The number of entries of @addrs.
287 *
288 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
289 * The @addrs must be the addresses of ftrace location address, which may be
290 * the symbol address + arch-dependent offset.
291 * If you unsure what this mean, please use other registration functions.
292 *
293 * Return 0 if @fp is registered successfully, -errno if not.
294 */
295int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
296{
297 int ret;
298
299 if (!fp || !addrs || num <= 0)
300 return -EINVAL;
301
302 fprobe_init(fp);
303
304 ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0);
305 if (ret)
306 return ret;
307
308 ret = fprobe_init_rethook(fp, num);
309 if (!ret)
310 ret = register_ftrace_function(&fp->ops);
311
312 if (ret)
313 fprobe_fail_cleanup(fp);
314 return ret;
315}
316EXPORT_SYMBOL_GPL(register_fprobe_ips);
317
318/**
319 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
320 * @fp: A fprobe data structure to be registered.
321 * @syms: An array of target symbols.
322 * @num: The number of entries of @syms.
323 *
324 * Register @fp to the symbols given by @syms array. This will be useful if
325 * you are sure the symbols exist in the kernel.
326 *
327 * Return 0 if @fp is registered successfully, -errno if not.
328 */
329int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
330{
331 unsigned long *addrs;
332 int ret;
333
334 if (!fp || !syms || num <= 0)
335 return -EINVAL;
336
337 addrs = get_ftrace_locations(syms, num);
338 if (IS_ERR(addrs))
339 return PTR_ERR(addrs);
340
341 ret = register_fprobe_ips(fp, addrs, num);
342
343 kfree(addrs);
344
345 return ret;
346}
347EXPORT_SYMBOL_GPL(register_fprobe_syms);
348
349bool fprobe_is_registered(struct fprobe *fp)
350{
351 if (!fp || (fp->ops.saved_func != fprobe_handler &&
352 fp->ops.saved_func != fprobe_kprobe_handler))
353 return false;
354 return true;
355}
356
357/**
358 * unregister_fprobe() - Unregister fprobe from ftrace
359 * @fp: A fprobe data structure to be unregistered.
360 *
361 * Unregister fprobe (and remove ftrace hooks from the function entries).
362 *
363 * Return 0 if @fp is unregistered successfully, -errno if not.
364 */
365int unregister_fprobe(struct fprobe *fp)
366{
367 int ret;
368
369 if (!fprobe_is_registered(fp))
370 return -EINVAL;
371
372 if (!IS_ERR_OR_NULL(fp->rethook))
373 rethook_stop(fp->rethook);
374
375 ret = unregister_ftrace_function(&fp->ops);
376 if (ret < 0)
377 return ret;
378
379 if (!IS_ERR_OR_NULL(fp->rethook))
380 rethook_free(fp->rethook);
381
382 ftrace_free_filter(&fp->ops);
383
384 return ret;
385}
386EXPORT_SYMBOL_GPL(unregister_fprobe);