1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * fprobe - Simple ftrace probe wrapper for function entry.
  4 */
  5#define pr_fmt(fmt) "fprobe: " fmt
  6
  7#include <linux/err.h>
  8#include <linux/fprobe.h>
  9#include <linux/kallsyms.h>
 10#include <linux/kprobes.h>
 11#include <linux/rethook.h>
 12#include <linux/slab.h>
 13#include <linux/sort.h>
 14
 15#include "trace.h"
 16
 17struct fprobe_rethook_node {
 18	struct rethook_node node;
 19	unsigned long entry_ip;
 20};
 21
 22static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
 23			   struct ftrace_ops *ops, struct ftrace_regs *fregs)
 24{
 25	struct fprobe_rethook_node *fpr;
 26	struct rethook_node *rh;
 27	struct fprobe *fp;
 28	int bit;
 29
 30	fp = container_of(ops, struct fprobe, ops);
 31	if (fprobe_disabled(fp))
 32		return;
 33
 34	bit = ftrace_test_recursion_trylock(ip, parent_ip);
 35	if (bit < 0) {
 36		fp->nmissed++;
 37		return;
 38	}
 39
 40	if (fp->entry_handler)
 41		fp->entry_handler(fp, ip, ftrace_get_regs(fregs));
 42
 43	if (fp->exit_handler) {
 44		rh = rethook_try_get(fp->rethook);
 45		if (!rh) {
 46			fp->nmissed++;
 47			goto out;
 48		}
 49		fpr = container_of(rh, struct fprobe_rethook_node, node);
 50		fpr->entry_ip = ip;
 51		rethook_hook(rh, ftrace_get_regs(fregs), true);
 52	}
 53
 54out:
 55	ftrace_test_recursion_unlock(bit);
 56}
 57NOKPROBE_SYMBOL(fprobe_handler);
 58
 59static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
 60				  struct ftrace_ops *ops, struct ftrace_regs *fregs)
 61{
 62	struct fprobe *fp = container_of(ops, struct fprobe, ops);
 63
 64	if (unlikely(kprobe_running())) {
 65		fp->nmissed++;
 66		return;
 67	}
 68	kprobe_busy_begin();
 69	fprobe_handler(ip, parent_ip, ops, fregs);
 70	kprobe_busy_end();
 71}
 72
 73static void fprobe_exit_handler(struct rethook_node *rh, void *data,
 74				struct pt_regs *regs)
 75{
 76	struct fprobe *fp = (struct fprobe *)data;
 77	struct fprobe_rethook_node *fpr;
 78
 79	if (!fp || fprobe_disabled(fp))
 80		return;
 81
 82	fpr = container_of(rh, struct fprobe_rethook_node, node);
 83
 84	fp->exit_handler(fp, fpr->entry_ip, regs);
 85}
 86NOKPROBE_SYMBOL(fprobe_exit_handler);
 87
 88static int symbols_cmp(const void *a, const void *b)
 89{
 90	const char **str_a = (const char **) a;
 91	const char **str_b = (const char **) b;
 92
 93	return strcmp(*str_a, *str_b);
 94}
 95
 96/* Convert ftrace location address from symbols */
 97static unsigned long *get_ftrace_locations(const char **syms, int num)
 98{
 99	unsigned long *addrs;
100
101	/* Convert symbols to symbol address */
102	addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
103	if (!addrs)
104		return ERR_PTR(-ENOMEM);
105
106	/* ftrace_lookup_symbols expects sorted symbols */
107	sort(syms, num, sizeof(*syms), symbols_cmp, NULL);
108
109	if (!ftrace_lookup_symbols(syms, num, addrs))
110		return addrs;
111
112	kfree(addrs);
113	return ERR_PTR(-ENOENT);
114}
115
116static void fprobe_init(struct fprobe *fp)
117{
118	fp->nmissed = 0;
119	if (fprobe_shared_with_kprobes(fp))
120		fp->ops.func = fprobe_kprobe_handler;
121	else
122		fp->ops.func = fprobe_handler;
123	fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS;
124}
125
126static int fprobe_init_rethook(struct fprobe *fp, int num)
127{
128	int i, size;
129
130	if (num < 0)
131		return -EINVAL;
132
133	if (!fp->exit_handler) {
134		fp->rethook = NULL;
135		return 0;
136	}
137
138	/* Initialize rethook if needed */
139	size = num * num_possible_cpus() * 2;
140	if (size < 0)
141		return -E2BIG;
142
143	fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler);
144	if (!fp->rethook)
145		return -ENOMEM;
146	for (i = 0; i < size; i++) {
147		struct fprobe_rethook_node *node;
148
149		node = kzalloc(sizeof(*node), GFP_KERNEL);
150		if (!node) {
151			rethook_free(fp->rethook);
152			fp->rethook = NULL;
153			return -ENOMEM;
154		}
155		rethook_add_node(fp->rethook, &node->node);
156	}
157	return 0;
158}
159
160static void fprobe_fail_cleanup(struct fprobe *fp)
161{
162	if (fp->rethook) {
163		/* Don't need to cleanup rethook->handler because this is not used. */
164		rethook_free(fp->rethook);
165		fp->rethook = NULL;
166	}
167	ftrace_free_filter(&fp->ops);
168}
169
170/**
171 * register_fprobe() - Register fprobe to ftrace by pattern.
172 * @fp: A fprobe data structure to be registered.
173 * @filter: A wildcard pattern of probed symbols.
174 * @notfilter: A wildcard pattern of NOT probed symbols.
175 *
176 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
177 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
178 *
179 * Return 0 if @fp is registered successfully, -errno if not.
180 */
181int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
182{
183	struct ftrace_hash *hash;
184	unsigned char *str;
185	int ret, len;
186
187	if (!fp || !filter)
188		return -EINVAL;
189
190	fprobe_init(fp);
191
192	len = strlen(filter);
193	str = kstrdup(filter, GFP_KERNEL);
194	ret = ftrace_set_filter(&fp->ops, str, len, 0);
195	kfree(str);
196	if (ret)
197		return ret;
198
199	if (notfilter) {
200		len = strlen(notfilter);
201		str = kstrdup(notfilter, GFP_KERNEL);
202		ret = ftrace_set_notrace(&fp->ops, str, len, 0);
203		kfree(str);
204		if (ret)
205			goto out;
206	}
207
208	/* TODO:
209	 * correctly calculate the total number of filtered symbols
210	 * from both filter and notfilter.
211	 */
212	hash = rcu_access_pointer(fp->ops.local_hash.filter_hash);
213	if (WARN_ON_ONCE(!hash))
214		goto out;
215
216	ret = fprobe_init_rethook(fp, (int)hash->count);
217	if (!ret)
218		ret = register_ftrace_function(&fp->ops);
219
220out:
221	if (ret)
222		fprobe_fail_cleanup(fp);
223	return ret;
224}
225EXPORT_SYMBOL_GPL(register_fprobe);
226
227/**
228 * register_fprobe_ips() - Register fprobe to ftrace by address.
229 * @fp: A fprobe data structure to be registered.
230 * @addrs: An array of target ftrace location addresses.
231 * @num: The number of entries of @addrs.
232 *
233 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
234 * The @addrs must be the addresses of ftrace location address, which may be
235 * the symbol address + arch-dependent offset.
236 * If you unsure what this mean, please use other registration functions.
237 *
238 * Return 0 if @fp is registered successfully, -errno if not.
239 */
240int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
241{
242	int ret;
243
244	if (!fp || !addrs || num <= 0)
245		return -EINVAL;
246
247	fprobe_init(fp);
248
249	ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0);
250	if (ret)
251		return ret;
252
253	ret = fprobe_init_rethook(fp, num);
254	if (!ret)
255		ret = register_ftrace_function(&fp->ops);
256
257	if (ret)
258		fprobe_fail_cleanup(fp);
259	return ret;
260}
261EXPORT_SYMBOL_GPL(register_fprobe_ips);
262
263/**
264 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
265 * @fp: A fprobe data structure to be registered.
266 * @syms: An array of target symbols.
267 * @num: The number of entries of @syms.
268 *
269 * Register @fp to the symbols given by @syms array. This will be useful if
270 * you are sure the symbols exist in the kernel.
271 *
272 * Return 0 if @fp is registered successfully, -errno if not.
273 */
274int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
275{
276	unsigned long *addrs;
277	int ret;
278
279	if (!fp || !syms || num <= 0)
280		return -EINVAL;
281
282	addrs = get_ftrace_locations(syms, num);
283	if (IS_ERR(addrs))
284		return PTR_ERR(addrs);
285
286	ret = register_fprobe_ips(fp, addrs, num);
287
288	kfree(addrs);
289
290	return ret;
291}
292EXPORT_SYMBOL_GPL(register_fprobe_syms);
293
294/**
295 * unregister_fprobe() - Unregister fprobe from ftrace
296 * @fp: A fprobe data structure to be unregistered.
297 *
298 * Unregister fprobe (and remove ftrace hooks from the function entries).
299 *
300 * Return 0 if @fp is unregistered successfully, -errno if not.
301 */
302int unregister_fprobe(struct fprobe *fp)
303{
304	int ret;
305
306	if (!fp || (fp->ops.saved_func != fprobe_handler &&
307		    fp->ops.saved_func != fprobe_kprobe_handler))
308		return -EINVAL;
309
310	/*
311	 * rethook_free() starts disabling the rethook, but the rethook handlers
312	 * may be running on other processors at this point. To make sure that all
313	 * current running handlers are finished, call unregister_ftrace_function()
314	 * after this.
315	 */
316	if (fp->rethook)
317		rethook_free(fp->rethook);
318
319	ret = unregister_ftrace_function(&fp->ops);
320	if (ret < 0)
321		return ret;
322
323	ftrace_free_filter(&fp->ops);
324
325	return ret;
326}
327EXPORT_SYMBOL_GPL(unregister_fprobe);