1// SPDX-License-Identifier: GPL-2.0
  2#include <test_progs.h>
  3#include "kprobe_multi.skel.h"
  4#include "trace_helpers.h"
  5#include "kprobe_multi_empty.skel.h"
  6#include "bpf/libbpf_internal.h"
  7#include "bpf/hashmap.h"
  8
  9static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
 10{
 11	LIBBPF_OPTS(bpf_test_run_opts, topts);
 12	int err, prog_fd;
 13
 14	prog_fd = bpf_program__fd(skel->progs.trigger);
 15	err = bpf_prog_test_run_opts(prog_fd, &topts);
 16	ASSERT_OK(err, "test_run");
 17	ASSERT_EQ(topts.retval, 0, "test_run");
 18
 19	ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
 20	ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
 21	ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
 22	ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
 23	ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
 24	ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
 25	ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
 26	ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
 27
 28	if (test_return) {
 29		ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
 30		ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
 31		ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
 32		ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
 33		ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
 34		ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
 35		ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
 36		ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
 37	}
 38}
 39
 40static void test_skel_api(void)
 41{
 42	struct kprobe_multi *skel = NULL;
 43	int err;
 44
 45	skel = kprobe_multi__open_and_load();
 46	if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
 47		goto cleanup;
 48
 49	skel->bss->pid = getpid();
 50	err = kprobe_multi__attach(skel);
 51	if (!ASSERT_OK(err, "kprobe_multi__attach"))
 52		goto cleanup;
 53
 54	kprobe_multi_test_run(skel, true);
 55
 56cleanup:
 57	kprobe_multi__destroy(skel);
 58}
 59
 60static void test_link_api(struct bpf_link_create_opts *opts)
 61{
 62	int prog_fd, link1_fd = -1, link2_fd = -1;
 63	struct kprobe_multi *skel = NULL;
 64
 65	skel = kprobe_multi__open_and_load();
 66	if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
 67		goto cleanup;
 68
 69	skel->bss->pid = getpid();
 70	prog_fd = bpf_program__fd(skel->progs.test_kprobe);
 71	link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
 72	if (!ASSERT_GE(link1_fd, 0, "link_fd"))
 73		goto cleanup;
 74
 75	opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
 76	prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
 77	link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
 78	if (!ASSERT_GE(link2_fd, 0, "link_fd"))
 79		goto cleanup;
 80
 81	kprobe_multi_test_run(skel, true);
 82
 83cleanup:
 84	if (link1_fd != -1)
 85		close(link1_fd);
 86	if (link2_fd != -1)
 87		close(link2_fd);
 88	kprobe_multi__destroy(skel);
 89}
 90
 91#define GET_ADDR(__sym, __addr) ({					\
 92	__addr = ksym_get_addr(__sym);					\
 93	if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym))	\
 94		return;							\
 95})
 96
 97static void test_link_api_addrs(void)
 98{
 99	LIBBPF_OPTS(bpf_link_create_opts, opts);
100	unsigned long long addrs[8];
101
102	GET_ADDR("bpf_fentry_test1", addrs[0]);
103	GET_ADDR("bpf_fentry_test2", addrs[1]);
104	GET_ADDR("bpf_fentry_test3", addrs[2]);
105	GET_ADDR("bpf_fentry_test4", addrs[3]);
106	GET_ADDR("bpf_fentry_test5", addrs[4]);
107	GET_ADDR("bpf_fentry_test6", addrs[5]);
108	GET_ADDR("bpf_fentry_test7", addrs[6]);
109	GET_ADDR("bpf_fentry_test8", addrs[7]);
110
111	opts.kprobe_multi.addrs = (const unsigned long*) addrs;
112	opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
113	test_link_api(&opts);
114}
115
116static void test_link_api_syms(void)
117{
118	LIBBPF_OPTS(bpf_link_create_opts, opts);
119	const char *syms[8] = {
120		"bpf_fentry_test1",
121		"bpf_fentry_test2",
122		"bpf_fentry_test3",
123		"bpf_fentry_test4",
124		"bpf_fentry_test5",
125		"bpf_fentry_test6",
126		"bpf_fentry_test7",
127		"bpf_fentry_test8",
128	};
129
130	opts.kprobe_multi.syms = syms;
131	opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
132	test_link_api(&opts);
133}
134
135static void
136test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
137{
138	struct bpf_link *link1 = NULL, *link2 = NULL;
139	struct kprobe_multi *skel = NULL;
140
141	skel = kprobe_multi__open_and_load();
142	if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
143		goto cleanup;
144
145	skel->bss->pid = getpid();
146	link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
147						      pattern, opts);
148	if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
149		goto cleanup;
150
151	if (opts) {
152		opts->retprobe = true;
153		link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
154							      pattern, opts);
155		if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
156			goto cleanup;
157	}
158
159	kprobe_multi_test_run(skel, !!opts);
160
161cleanup:
162	bpf_link__destroy(link2);
163	bpf_link__destroy(link1);
164	kprobe_multi__destroy(skel);
165}
166
167static void test_attach_api_pattern(void)
168{
169	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
170
171	test_attach_api("bpf_fentry_test*", &opts);
172	test_attach_api("bpf_fentry_test?", NULL);
173}
174
175static void test_attach_api_addrs(void)
176{
177	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
178	unsigned long long addrs[8];
179
180	GET_ADDR("bpf_fentry_test1", addrs[0]);
181	GET_ADDR("bpf_fentry_test2", addrs[1]);
182	GET_ADDR("bpf_fentry_test3", addrs[2]);
183	GET_ADDR("bpf_fentry_test4", addrs[3]);
184	GET_ADDR("bpf_fentry_test5", addrs[4]);
185	GET_ADDR("bpf_fentry_test6", addrs[5]);
186	GET_ADDR("bpf_fentry_test7", addrs[6]);
187	GET_ADDR("bpf_fentry_test8", addrs[7]);
188
189	opts.addrs = (const unsigned long *) addrs;
190	opts.cnt = ARRAY_SIZE(addrs);
191	test_attach_api(NULL, &opts);
192}
193
194static void test_attach_api_syms(void)
195{
196	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
197	const char *syms[8] = {
198		"bpf_fentry_test1",
199		"bpf_fentry_test2",
200		"bpf_fentry_test3",
201		"bpf_fentry_test4",
202		"bpf_fentry_test5",
203		"bpf_fentry_test6",
204		"bpf_fentry_test7",
205		"bpf_fentry_test8",
206	};
207
208	opts.syms = syms;
209	opts.cnt = ARRAY_SIZE(syms);
210	test_attach_api(NULL, &opts);
211}
212
213static void test_attach_api_fails(void)
214{
215	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
216	struct kprobe_multi *skel = NULL;
217	struct bpf_link *link = NULL;
218	unsigned long long addrs[2];
219	const char *syms[2] = {
220		"bpf_fentry_test1",
221		"bpf_fentry_test2",
222	};
223	__u64 cookies[2];
224
225	addrs[0] = ksym_get_addr("bpf_fentry_test1");
226	addrs[1] = ksym_get_addr("bpf_fentry_test2");
227
228	if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
229		goto cleanup;
230
231	skel = kprobe_multi__open_and_load();
232	if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
233		goto cleanup;
234
235	skel->bss->pid = getpid();
236
237	/* fail_1 - pattern and opts NULL */
238	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
239						     NULL, NULL);
240	if (!ASSERT_ERR_PTR(link, "fail_1"))
241		goto cleanup;
242
243	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
244		goto cleanup;
245
246	/* fail_2 - both addrs and syms set */
247	opts.addrs = (const unsigned long *) addrs;
248	opts.syms = syms;
249	opts.cnt = ARRAY_SIZE(syms);
250	opts.cookies = NULL;
251
252	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
253						     NULL, &opts);
254	if (!ASSERT_ERR_PTR(link, "fail_2"))
255		goto cleanup;
256
257	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
258		goto cleanup;
259
260	/* fail_3 - pattern and addrs set */
261	opts.addrs = (const unsigned long *) addrs;
262	opts.syms = NULL;
263	opts.cnt = ARRAY_SIZE(syms);
264	opts.cookies = NULL;
265
266	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
267						     "ksys_*", &opts);
268	if (!ASSERT_ERR_PTR(link, "fail_3"))
269		goto cleanup;
270
271	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
272		goto cleanup;
273
274	/* fail_4 - pattern and cnt set */
275	opts.addrs = NULL;
276	opts.syms = NULL;
277	opts.cnt = ARRAY_SIZE(syms);
278	opts.cookies = NULL;
279
280	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
281						     "ksys_*", &opts);
282	if (!ASSERT_ERR_PTR(link, "fail_4"))
283		goto cleanup;
284
285	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
286		goto cleanup;
287
288	/* fail_5 - pattern and cookies */
289	opts.addrs = NULL;
290	opts.syms = NULL;
291	opts.cnt = 0;
292	opts.cookies = cookies;
293
294	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
295						     "ksys_*", &opts);
296	if (!ASSERT_ERR_PTR(link, "fail_5"))
297		goto cleanup;
298
299	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
300		goto cleanup;
301
302cleanup:
303	bpf_link__destroy(link);
304	kprobe_multi__destroy(skel);
305}
306
307static inline __u64 get_time_ns(void)
308{
309	struct timespec t;
310
311	clock_gettime(CLOCK_MONOTONIC, &t);
312	return (__u64) t.tv_sec * 1000000000 + t.tv_nsec;
313}
314
315static size_t symbol_hash(long key, void *ctx __maybe_unused)
316{
317	return str_hash((const char *) key);
318}
319
320static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
321{
322	return strcmp((const char *) key1, (const char *) key2) == 0;
323}
324
325static int get_syms(char ***symsp, size_t *cntp)
326{
327	size_t cap = 0, cnt = 0, i;
328	char *name = NULL, **syms = NULL;
329	struct hashmap *map;
330	char buf[256];
331	FILE *f;
332	int err = 0;
333
334	/*
335	 * The available_filter_functions contains many duplicates,
336	 * but other than that all symbols are usable in kprobe multi
337	 * interface.
338	 * Filtering out duplicates by using hashmap__add, which won't
339	 * add existing entry.
340	 */
341	f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
342	if (!f)
343		return -EINVAL;
344
345	map = hashmap__new(symbol_hash, symbol_equal, NULL);
346	if (IS_ERR(map)) {
347		err = libbpf_get_error(map);
348		goto error;
349	}
350
351	while (fgets(buf, sizeof(buf), f)) {
352		/* skip modules */
353		if (strchr(buf, '['))
354			continue;
355
356		free(name);
357		if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
358			continue;
359		/*
360		 * We attach to almost all kernel functions and some of them
361		 * will cause 'suspicious RCU usage' when fprobe is attached
362		 * to them. Filter out the current culprits - arch_cpu_idle
363		 * default_idle and rcu_* functions.
364		 */
365		if (!strcmp(name, "arch_cpu_idle"))
366			continue;
367		if (!strcmp(name, "default_idle"))
368			continue;
369		if (!strncmp(name, "rcu_", 4))
370			continue;
371		if (!strcmp(name, "bpf_dispatcher_xdp_func"))
372			continue;
373		if (!strncmp(name, "__ftrace_invalid_address__",
374			     sizeof("__ftrace_invalid_address__") - 1))
375			continue;
376
377		err = hashmap__add(map, name, 0);
378		if (err == -EEXIST)
379			continue;
380		if (err)
381			goto error;
382
383		err = libbpf_ensure_mem((void **) &syms, &cap,
384					sizeof(*syms), cnt + 1);
385		if (err)
386			goto error;
387
388		syms[cnt++] = name;
389		name = NULL;
390	}
391
392	*symsp = syms;
393	*cntp = cnt;
394
395error:
396	free(name);
397	fclose(f);
398	hashmap__free(map);
399	if (err) {
400		for (i = 0; i < cnt; i++)
401			free(syms[i]);
402		free(syms);
403	}
404	return err;
405}
406
407void serial_test_kprobe_multi_bench_attach(void)
408{
409	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
410	struct kprobe_multi_empty *skel = NULL;
411	long attach_start_ns, attach_end_ns;
412	long detach_start_ns, detach_end_ns;
413	double attach_delta, detach_delta;
414	struct bpf_link *link = NULL;
415	char **syms = NULL;
416	size_t cnt = 0, i;
417
418	if (!ASSERT_OK(get_syms(&syms, &cnt), "get_syms"))
419		return;
420
421	skel = kprobe_multi_empty__open_and_load();
422	if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
423		goto cleanup;
424
425	opts.syms = (const char **) syms;
426	opts.cnt = cnt;
427
428	attach_start_ns = get_time_ns();
429	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
430						     NULL, &opts);
431	attach_end_ns = get_time_ns();
432
433	if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
434		goto cleanup;
435
436	detach_start_ns = get_time_ns();
437	bpf_link__destroy(link);
438	detach_end_ns = get_time_ns();
439
440	attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
441	detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
442
443	printf("%s: found %lu functions\n", __func__, cnt);
444	printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
445	printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
446
447cleanup:
448	kprobe_multi_empty__destroy(skel);
449	if (syms) {
450		for (i = 0; i < cnt; i++)
451			free(syms[i]);
452		free(syms);
453	}
454}
455
456void test_kprobe_multi_test(void)
457{
458	if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
459		return;
460
461	if (test__start_subtest("skel_api"))
462		test_skel_api();
463	if (test__start_subtest("link_api_addrs"))
464		test_link_api_syms();
465	if (test__start_subtest("link_api_syms"))
466		test_link_api_addrs();
467	if (test__start_subtest("attach_api_pattern"))
468		test_attach_api_pattern();
469	if (test__start_subtest("attach_api_addrs"))
470		test_attach_api_addrs();
471	if (test__start_subtest("attach_api_syms"))
472		test_attach_api_syms();
473	if (test__start_subtest("attach_api_fails"))
474		test_attach_api_fails();
475}