Loading...
1// SPDX-License-Identifier: GPL-2.0
2#include <test_progs.h>
3#include "kprobe_multi.skel.h"
4#include "trace_helpers.h"
5#include "kprobe_multi_empty.skel.h"
6#include "bpf/libbpf_internal.h"
7#include "bpf/hashmap.h"
8
9static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
10{
11 LIBBPF_OPTS(bpf_test_run_opts, topts);
12 int err, prog_fd;
13
14 prog_fd = bpf_program__fd(skel->progs.trigger);
15 err = bpf_prog_test_run_opts(prog_fd, &topts);
16 ASSERT_OK(err, "test_run");
17 ASSERT_EQ(topts.retval, 0, "test_run");
18
19 ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
20 ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
21 ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
22 ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
23 ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
24 ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
25 ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
26 ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
27
28 if (test_return) {
29 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
30 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
31 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
32 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
33 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
34 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
35 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
36 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
37 }
38}
39
40static void test_skel_api(void)
41{
42 struct kprobe_multi *skel = NULL;
43 int err;
44
45 skel = kprobe_multi__open_and_load();
46 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
47 goto cleanup;
48
49 skel->bss->pid = getpid();
50 err = kprobe_multi__attach(skel);
51 if (!ASSERT_OK(err, "kprobe_multi__attach"))
52 goto cleanup;
53
54 kprobe_multi_test_run(skel, true);
55
56cleanup:
57 kprobe_multi__destroy(skel);
58}
59
60static void test_link_api(struct bpf_link_create_opts *opts)
61{
62 int prog_fd, link1_fd = -1, link2_fd = -1;
63 struct kprobe_multi *skel = NULL;
64
65 skel = kprobe_multi__open_and_load();
66 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
67 goto cleanup;
68
69 skel->bss->pid = getpid();
70 prog_fd = bpf_program__fd(skel->progs.test_kprobe);
71 link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
72 if (!ASSERT_GE(link1_fd, 0, "link_fd"))
73 goto cleanup;
74
75 opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
76 prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
77 link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
78 if (!ASSERT_GE(link2_fd, 0, "link_fd"))
79 goto cleanup;
80
81 kprobe_multi_test_run(skel, true);
82
83cleanup:
84 if (link1_fd != -1)
85 close(link1_fd);
86 if (link2_fd != -1)
87 close(link2_fd);
88 kprobe_multi__destroy(skel);
89}
90
91#define GET_ADDR(__sym, __addr) ({ \
92 __addr = ksym_get_addr(__sym); \
93 if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
94 return; \
95})
96
97static void test_link_api_addrs(void)
98{
99 LIBBPF_OPTS(bpf_link_create_opts, opts);
100 unsigned long long addrs[8];
101
102 GET_ADDR("bpf_fentry_test1", addrs[0]);
103 GET_ADDR("bpf_fentry_test2", addrs[1]);
104 GET_ADDR("bpf_fentry_test3", addrs[2]);
105 GET_ADDR("bpf_fentry_test4", addrs[3]);
106 GET_ADDR("bpf_fentry_test5", addrs[4]);
107 GET_ADDR("bpf_fentry_test6", addrs[5]);
108 GET_ADDR("bpf_fentry_test7", addrs[6]);
109 GET_ADDR("bpf_fentry_test8", addrs[7]);
110
111 opts.kprobe_multi.addrs = (const unsigned long*) addrs;
112 opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
113 test_link_api(&opts);
114}
115
116static void test_link_api_syms(void)
117{
118 LIBBPF_OPTS(bpf_link_create_opts, opts);
119 const char *syms[8] = {
120 "bpf_fentry_test1",
121 "bpf_fentry_test2",
122 "bpf_fentry_test3",
123 "bpf_fentry_test4",
124 "bpf_fentry_test5",
125 "bpf_fentry_test6",
126 "bpf_fentry_test7",
127 "bpf_fentry_test8",
128 };
129
130 opts.kprobe_multi.syms = syms;
131 opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
132 test_link_api(&opts);
133}
134
135static void
136test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
137{
138 struct bpf_link *link1 = NULL, *link2 = NULL;
139 struct kprobe_multi *skel = NULL;
140
141 skel = kprobe_multi__open_and_load();
142 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
143 goto cleanup;
144
145 skel->bss->pid = getpid();
146 link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
147 pattern, opts);
148 if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
149 goto cleanup;
150
151 if (opts) {
152 opts->retprobe = true;
153 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
154 pattern, opts);
155 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
156 goto cleanup;
157 }
158
159 kprobe_multi_test_run(skel, !!opts);
160
161cleanup:
162 bpf_link__destroy(link2);
163 bpf_link__destroy(link1);
164 kprobe_multi__destroy(skel);
165}
166
167static void test_attach_api_pattern(void)
168{
169 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
170
171 test_attach_api("bpf_fentry_test*", &opts);
172 test_attach_api("bpf_fentry_test?", NULL);
173}
174
175static void test_attach_api_addrs(void)
176{
177 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
178 unsigned long long addrs[8];
179
180 GET_ADDR("bpf_fentry_test1", addrs[0]);
181 GET_ADDR("bpf_fentry_test2", addrs[1]);
182 GET_ADDR("bpf_fentry_test3", addrs[2]);
183 GET_ADDR("bpf_fentry_test4", addrs[3]);
184 GET_ADDR("bpf_fentry_test5", addrs[4]);
185 GET_ADDR("bpf_fentry_test6", addrs[5]);
186 GET_ADDR("bpf_fentry_test7", addrs[6]);
187 GET_ADDR("bpf_fentry_test8", addrs[7]);
188
189 opts.addrs = (const unsigned long *) addrs;
190 opts.cnt = ARRAY_SIZE(addrs);
191 test_attach_api(NULL, &opts);
192}
193
194static void test_attach_api_syms(void)
195{
196 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
197 const char *syms[8] = {
198 "bpf_fentry_test1",
199 "bpf_fentry_test2",
200 "bpf_fentry_test3",
201 "bpf_fentry_test4",
202 "bpf_fentry_test5",
203 "bpf_fentry_test6",
204 "bpf_fentry_test7",
205 "bpf_fentry_test8",
206 };
207
208 opts.syms = syms;
209 opts.cnt = ARRAY_SIZE(syms);
210 test_attach_api(NULL, &opts);
211}
212
213static void test_attach_api_fails(void)
214{
215 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
216 struct kprobe_multi *skel = NULL;
217 struct bpf_link *link = NULL;
218 unsigned long long addrs[2];
219 const char *syms[2] = {
220 "bpf_fentry_test1",
221 "bpf_fentry_test2",
222 };
223 __u64 cookies[2];
224
225 addrs[0] = ksym_get_addr("bpf_fentry_test1");
226 addrs[1] = ksym_get_addr("bpf_fentry_test2");
227
228 if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
229 goto cleanup;
230
231 skel = kprobe_multi__open_and_load();
232 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
233 goto cleanup;
234
235 skel->bss->pid = getpid();
236
237 /* fail_1 - pattern and opts NULL */
238 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
239 NULL, NULL);
240 if (!ASSERT_ERR_PTR(link, "fail_1"))
241 goto cleanup;
242
243 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
244 goto cleanup;
245
246 /* fail_2 - both addrs and syms set */
247 opts.addrs = (const unsigned long *) addrs;
248 opts.syms = syms;
249 opts.cnt = ARRAY_SIZE(syms);
250 opts.cookies = NULL;
251
252 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
253 NULL, &opts);
254 if (!ASSERT_ERR_PTR(link, "fail_2"))
255 goto cleanup;
256
257 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
258 goto cleanup;
259
260 /* fail_3 - pattern and addrs set */
261 opts.addrs = (const unsigned long *) addrs;
262 opts.syms = NULL;
263 opts.cnt = ARRAY_SIZE(syms);
264 opts.cookies = NULL;
265
266 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
267 "ksys_*", &opts);
268 if (!ASSERT_ERR_PTR(link, "fail_3"))
269 goto cleanup;
270
271 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
272 goto cleanup;
273
274 /* fail_4 - pattern and cnt set */
275 opts.addrs = NULL;
276 opts.syms = NULL;
277 opts.cnt = ARRAY_SIZE(syms);
278 opts.cookies = NULL;
279
280 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
281 "ksys_*", &opts);
282 if (!ASSERT_ERR_PTR(link, "fail_4"))
283 goto cleanup;
284
285 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
286 goto cleanup;
287
288 /* fail_5 - pattern and cookies */
289 opts.addrs = NULL;
290 opts.syms = NULL;
291 opts.cnt = 0;
292 opts.cookies = cookies;
293
294 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
295 "ksys_*", &opts);
296 if (!ASSERT_ERR_PTR(link, "fail_5"))
297 goto cleanup;
298
299 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
300 goto cleanup;
301
302cleanup:
303 bpf_link__destroy(link);
304 kprobe_multi__destroy(skel);
305}
306
307static inline __u64 get_time_ns(void)
308{
309 struct timespec t;
310
311 clock_gettime(CLOCK_MONOTONIC, &t);
312 return (__u64) t.tv_sec * 1000000000 + t.tv_nsec;
313}
314
315static size_t symbol_hash(long key, void *ctx __maybe_unused)
316{
317 return str_hash((const char *) key);
318}
319
320static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
321{
322 return strcmp((const char *) key1, (const char *) key2) == 0;
323}
324
325static int get_syms(char ***symsp, size_t *cntp)
326{
327 size_t cap = 0, cnt = 0, i;
328 char *name = NULL, **syms = NULL;
329 struct hashmap *map;
330 char buf[256];
331 FILE *f;
332 int err = 0;
333
334 /*
335 * The available_filter_functions contains many duplicates,
336 * but other than that all symbols are usable in kprobe multi
337 * interface.
338 * Filtering out duplicates by using hashmap__add, which won't
339 * add existing entry.
340 */
341 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
342 if (!f)
343 return -EINVAL;
344
345 map = hashmap__new(symbol_hash, symbol_equal, NULL);
346 if (IS_ERR(map)) {
347 err = libbpf_get_error(map);
348 goto error;
349 }
350
351 while (fgets(buf, sizeof(buf), f)) {
352 /* skip modules */
353 if (strchr(buf, '['))
354 continue;
355
356 free(name);
357 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
358 continue;
359 /*
360 * We attach to almost all kernel functions and some of them
361 * will cause 'suspicious RCU usage' when fprobe is attached
362 * to them. Filter out the current culprits - arch_cpu_idle
363 * default_idle and rcu_* functions.
364 */
365 if (!strcmp(name, "arch_cpu_idle"))
366 continue;
367 if (!strcmp(name, "default_idle"))
368 continue;
369 if (!strncmp(name, "rcu_", 4))
370 continue;
371 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
372 continue;
373 if (!strncmp(name, "__ftrace_invalid_address__",
374 sizeof("__ftrace_invalid_address__") - 1))
375 continue;
376
377 err = hashmap__add(map, name, 0);
378 if (err == -EEXIST)
379 continue;
380 if (err)
381 goto error;
382
383 err = libbpf_ensure_mem((void **) &syms, &cap,
384 sizeof(*syms), cnt + 1);
385 if (err)
386 goto error;
387
388 syms[cnt++] = name;
389 name = NULL;
390 }
391
392 *symsp = syms;
393 *cntp = cnt;
394
395error:
396 free(name);
397 fclose(f);
398 hashmap__free(map);
399 if (err) {
400 for (i = 0; i < cnt; i++)
401 free(syms[i]);
402 free(syms);
403 }
404 return err;
405}
406
407void serial_test_kprobe_multi_bench_attach(void)
408{
409 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
410 struct kprobe_multi_empty *skel = NULL;
411 long attach_start_ns, attach_end_ns;
412 long detach_start_ns, detach_end_ns;
413 double attach_delta, detach_delta;
414 struct bpf_link *link = NULL;
415 char **syms = NULL;
416 size_t cnt = 0, i;
417
418 if (!ASSERT_OK(get_syms(&syms, &cnt), "get_syms"))
419 return;
420
421 skel = kprobe_multi_empty__open_and_load();
422 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
423 goto cleanup;
424
425 opts.syms = (const char **) syms;
426 opts.cnt = cnt;
427
428 attach_start_ns = get_time_ns();
429 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
430 NULL, &opts);
431 attach_end_ns = get_time_ns();
432
433 if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
434 goto cleanup;
435
436 detach_start_ns = get_time_ns();
437 bpf_link__destroy(link);
438 detach_end_ns = get_time_ns();
439
440 attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
441 detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
442
443 printf("%s: found %lu functions\n", __func__, cnt);
444 printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
445 printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
446
447cleanup:
448 kprobe_multi_empty__destroy(skel);
449 if (syms) {
450 for (i = 0; i < cnt; i++)
451 free(syms[i]);
452 free(syms);
453 }
454}
455
456void test_kprobe_multi_test(void)
457{
458 if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
459 return;
460
461 if (test__start_subtest("skel_api"))
462 test_skel_api();
463 if (test__start_subtest("link_api_addrs"))
464 test_link_api_syms();
465 if (test__start_subtest("link_api_syms"))
466 test_link_api_addrs();
467 if (test__start_subtest("attach_api_pattern"))
468 test_attach_api_pattern();
469 if (test__start_subtest("attach_api_addrs"))
470 test_attach_api_addrs();
471 if (test__start_subtest("attach_api_syms"))
472 test_attach_api_syms();
473 if (test__start_subtest("attach_api_fails"))
474 test_attach_api_fails();
475}
1// SPDX-License-Identifier: GPL-2.0
2#include <test_progs.h>
3#include "kprobe_multi.skel.h"
4#include "trace_helpers.h"
5#include "kprobe_multi_empty.skel.h"
6#include "kprobe_multi_override.skel.h"
7#include "bpf/libbpf_internal.h"
8#include "bpf/hashmap.h"
9
10static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
11{
12 LIBBPF_OPTS(bpf_test_run_opts, topts);
13 int err, prog_fd;
14
15 prog_fd = bpf_program__fd(skel->progs.trigger);
16 err = bpf_prog_test_run_opts(prog_fd, &topts);
17 ASSERT_OK(err, "test_run");
18 ASSERT_EQ(topts.retval, 0, "test_run");
19
20 ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
21 ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
22 ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
23 ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
24 ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
25 ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
26 ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
27 ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
28
29 if (test_return) {
30 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
31 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
32 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
33 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
34 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
35 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
36 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
37 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
38 }
39}
40
41static void test_skel_api(void)
42{
43 struct kprobe_multi *skel = NULL;
44 int err;
45
46 skel = kprobe_multi__open_and_load();
47 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
48 goto cleanup;
49
50 skel->bss->pid = getpid();
51 err = kprobe_multi__attach(skel);
52 if (!ASSERT_OK(err, "kprobe_multi__attach"))
53 goto cleanup;
54
55 kprobe_multi_test_run(skel, true);
56
57cleanup:
58 kprobe_multi__destroy(skel);
59}
60
61static void test_link_api(struct bpf_link_create_opts *opts)
62{
63 int prog_fd, link1_fd = -1, link2_fd = -1;
64 struct kprobe_multi *skel = NULL;
65
66 skel = kprobe_multi__open_and_load();
67 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
68 goto cleanup;
69
70 skel->bss->pid = getpid();
71 prog_fd = bpf_program__fd(skel->progs.test_kprobe);
72 link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
73 if (!ASSERT_GE(link1_fd, 0, "link_fd"))
74 goto cleanup;
75
76 opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
77 prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
78 link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
79 if (!ASSERT_GE(link2_fd, 0, "link_fd"))
80 goto cleanup;
81
82 kprobe_multi_test_run(skel, true);
83
84cleanup:
85 if (link1_fd != -1)
86 close(link1_fd);
87 if (link2_fd != -1)
88 close(link2_fd);
89 kprobe_multi__destroy(skel);
90}
91
92#define GET_ADDR(__sym, __addr) ({ \
93 __addr = ksym_get_addr(__sym); \
94 if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
95 return; \
96})
97
98static void test_link_api_addrs(void)
99{
100 LIBBPF_OPTS(bpf_link_create_opts, opts);
101 unsigned long long addrs[8];
102
103 GET_ADDR("bpf_fentry_test1", addrs[0]);
104 GET_ADDR("bpf_fentry_test2", addrs[1]);
105 GET_ADDR("bpf_fentry_test3", addrs[2]);
106 GET_ADDR("bpf_fentry_test4", addrs[3]);
107 GET_ADDR("bpf_fentry_test5", addrs[4]);
108 GET_ADDR("bpf_fentry_test6", addrs[5]);
109 GET_ADDR("bpf_fentry_test7", addrs[6]);
110 GET_ADDR("bpf_fentry_test8", addrs[7]);
111
112 opts.kprobe_multi.addrs = (const unsigned long*) addrs;
113 opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
114 test_link_api(&opts);
115}
116
117static void test_link_api_syms(void)
118{
119 LIBBPF_OPTS(bpf_link_create_opts, opts);
120 const char *syms[8] = {
121 "bpf_fentry_test1",
122 "bpf_fentry_test2",
123 "bpf_fentry_test3",
124 "bpf_fentry_test4",
125 "bpf_fentry_test5",
126 "bpf_fentry_test6",
127 "bpf_fentry_test7",
128 "bpf_fentry_test8",
129 };
130
131 opts.kprobe_multi.syms = syms;
132 opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
133 test_link_api(&opts);
134}
135
136static void
137test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
138{
139 struct bpf_link *link1 = NULL, *link2 = NULL;
140 struct kprobe_multi *skel = NULL;
141
142 skel = kprobe_multi__open_and_load();
143 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
144 goto cleanup;
145
146 skel->bss->pid = getpid();
147 link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
148 pattern, opts);
149 if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
150 goto cleanup;
151
152 if (opts) {
153 opts->retprobe = true;
154 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
155 pattern, opts);
156 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
157 goto cleanup;
158 }
159
160 kprobe_multi_test_run(skel, !!opts);
161
162cleanup:
163 bpf_link__destroy(link2);
164 bpf_link__destroy(link1);
165 kprobe_multi__destroy(skel);
166}
167
168static void test_attach_api_pattern(void)
169{
170 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
171
172 test_attach_api("bpf_fentry_test*", &opts);
173 test_attach_api("bpf_fentry_test?", NULL);
174}
175
176static void test_attach_api_addrs(void)
177{
178 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
179 unsigned long long addrs[8];
180
181 GET_ADDR("bpf_fentry_test1", addrs[0]);
182 GET_ADDR("bpf_fentry_test2", addrs[1]);
183 GET_ADDR("bpf_fentry_test3", addrs[2]);
184 GET_ADDR("bpf_fentry_test4", addrs[3]);
185 GET_ADDR("bpf_fentry_test5", addrs[4]);
186 GET_ADDR("bpf_fentry_test6", addrs[5]);
187 GET_ADDR("bpf_fentry_test7", addrs[6]);
188 GET_ADDR("bpf_fentry_test8", addrs[7]);
189
190 opts.addrs = (const unsigned long *) addrs;
191 opts.cnt = ARRAY_SIZE(addrs);
192 test_attach_api(NULL, &opts);
193}
194
195static void test_attach_api_syms(void)
196{
197 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
198 const char *syms[8] = {
199 "bpf_fentry_test1",
200 "bpf_fentry_test2",
201 "bpf_fentry_test3",
202 "bpf_fentry_test4",
203 "bpf_fentry_test5",
204 "bpf_fentry_test6",
205 "bpf_fentry_test7",
206 "bpf_fentry_test8",
207 };
208
209 opts.syms = syms;
210 opts.cnt = ARRAY_SIZE(syms);
211 test_attach_api(NULL, &opts);
212}
213
214static void test_attach_api_fails(void)
215{
216 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
217 struct kprobe_multi *skel = NULL;
218 struct bpf_link *link = NULL;
219 unsigned long long addrs[2];
220 const char *syms[2] = {
221 "bpf_fentry_test1",
222 "bpf_fentry_test2",
223 };
224 __u64 cookies[2];
225 int saved_error;
226
227 addrs[0] = ksym_get_addr("bpf_fentry_test1");
228 addrs[1] = ksym_get_addr("bpf_fentry_test2");
229
230 if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
231 goto cleanup;
232
233 skel = kprobe_multi__open_and_load();
234 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
235 goto cleanup;
236
237 skel->bss->pid = getpid();
238
239 /* fail_1 - pattern and opts NULL */
240 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
241 NULL, NULL);
242 saved_error = -errno;
243 if (!ASSERT_ERR_PTR(link, "fail_1"))
244 goto cleanup;
245
246 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_1_error"))
247 goto cleanup;
248
249 /* fail_2 - both addrs and syms set */
250 opts.addrs = (const unsigned long *) addrs;
251 opts.syms = syms;
252 opts.cnt = ARRAY_SIZE(syms);
253 opts.cookies = NULL;
254
255 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
256 NULL, &opts);
257 saved_error = -errno;
258 if (!ASSERT_ERR_PTR(link, "fail_2"))
259 goto cleanup;
260
261 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_2_error"))
262 goto cleanup;
263
264 /* fail_3 - pattern and addrs set */
265 opts.addrs = (const unsigned long *) addrs;
266 opts.syms = NULL;
267 opts.cnt = ARRAY_SIZE(syms);
268 opts.cookies = NULL;
269
270 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
271 "ksys_*", &opts);
272 saved_error = -errno;
273 if (!ASSERT_ERR_PTR(link, "fail_3"))
274 goto cleanup;
275
276 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_3_error"))
277 goto cleanup;
278
279 /* fail_4 - pattern and cnt set */
280 opts.addrs = NULL;
281 opts.syms = NULL;
282 opts.cnt = ARRAY_SIZE(syms);
283 opts.cookies = NULL;
284
285 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
286 "ksys_*", &opts);
287 saved_error = -errno;
288 if (!ASSERT_ERR_PTR(link, "fail_4"))
289 goto cleanup;
290
291 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_4_error"))
292 goto cleanup;
293
294 /* fail_5 - pattern and cookies */
295 opts.addrs = NULL;
296 opts.syms = NULL;
297 opts.cnt = 0;
298 opts.cookies = cookies;
299
300 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
301 "ksys_*", &opts);
302 saved_error = -errno;
303 if (!ASSERT_ERR_PTR(link, "fail_5"))
304 goto cleanup;
305
306 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_5_error"))
307 goto cleanup;
308
309 /* fail_6 - abnormal cnt */
310 opts.addrs = (const unsigned long *) addrs;
311 opts.syms = NULL;
312 opts.cnt = INT_MAX;
313 opts.cookies = NULL;
314
315 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
316 NULL, &opts);
317 saved_error = -errno;
318 if (!ASSERT_ERR_PTR(link, "fail_6"))
319 goto cleanup;
320
321 if (!ASSERT_EQ(saved_error, -E2BIG, "fail_6_error"))
322 goto cleanup;
323
324cleanup:
325 bpf_link__destroy(link);
326 kprobe_multi__destroy(skel);
327}
328
329static size_t symbol_hash(long key, void *ctx __maybe_unused)
330{
331 return str_hash((const char *) key);
332}
333
334static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
335{
336 return strcmp((const char *) key1, (const char *) key2) == 0;
337}
338
339static int get_syms(char ***symsp, size_t *cntp, bool kernel)
340{
341 size_t cap = 0, cnt = 0, i;
342 char *name = NULL, **syms = NULL;
343 struct hashmap *map;
344 char buf[256];
345 FILE *f;
346 int err = 0;
347
348 /*
349 * The available_filter_functions contains many duplicates,
350 * but other than that all symbols are usable in kprobe multi
351 * interface.
352 * Filtering out duplicates by using hashmap__add, which won't
353 * add existing entry.
354 */
355
356 if (access("/sys/kernel/tracing/trace", F_OK) == 0)
357 f = fopen("/sys/kernel/tracing/available_filter_functions", "r");
358 else
359 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
360
361 if (!f)
362 return -EINVAL;
363
364 map = hashmap__new(symbol_hash, symbol_equal, NULL);
365 if (IS_ERR(map)) {
366 err = libbpf_get_error(map);
367 goto error;
368 }
369
370 while (fgets(buf, sizeof(buf), f)) {
371 if (kernel && strchr(buf, '['))
372 continue;
373 if (!kernel && !strchr(buf, '['))
374 continue;
375
376 free(name);
377 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
378 continue;
379 /*
380 * We attach to almost all kernel functions and some of them
381 * will cause 'suspicious RCU usage' when fprobe is attached
382 * to them. Filter out the current culprits - arch_cpu_idle
383 * default_idle and rcu_* functions.
384 */
385 if (!strcmp(name, "arch_cpu_idle"))
386 continue;
387 if (!strcmp(name, "default_idle"))
388 continue;
389 if (!strncmp(name, "rcu_", 4))
390 continue;
391 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
392 continue;
393 if (!strncmp(name, "__ftrace_invalid_address__",
394 sizeof("__ftrace_invalid_address__") - 1))
395 continue;
396
397 err = hashmap__add(map, name, 0);
398 if (err == -EEXIST) {
399 err = 0;
400 continue;
401 }
402 if (err)
403 goto error;
404
405 err = libbpf_ensure_mem((void **) &syms, &cap,
406 sizeof(*syms), cnt + 1);
407 if (err)
408 goto error;
409
410 syms[cnt++] = name;
411 name = NULL;
412 }
413
414 *symsp = syms;
415 *cntp = cnt;
416
417error:
418 free(name);
419 fclose(f);
420 hashmap__free(map);
421 if (err) {
422 for (i = 0; i < cnt; i++)
423 free(syms[i]);
424 free(syms);
425 }
426 return err;
427}
428
429static void test_kprobe_multi_bench_attach(bool kernel)
430{
431 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
432 struct kprobe_multi_empty *skel = NULL;
433 long attach_start_ns, attach_end_ns;
434 long detach_start_ns, detach_end_ns;
435 double attach_delta, detach_delta;
436 struct bpf_link *link = NULL;
437 char **syms = NULL;
438 size_t cnt = 0, i;
439
440 if (!ASSERT_OK(get_syms(&syms, &cnt, kernel), "get_syms"))
441 return;
442
443 skel = kprobe_multi_empty__open_and_load();
444 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
445 goto cleanup;
446
447 opts.syms = (const char **) syms;
448 opts.cnt = cnt;
449
450 attach_start_ns = get_time_ns();
451 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
452 NULL, &opts);
453 attach_end_ns = get_time_ns();
454
455 if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
456 goto cleanup;
457
458 detach_start_ns = get_time_ns();
459 bpf_link__destroy(link);
460 detach_end_ns = get_time_ns();
461
462 attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
463 detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
464
465 printf("%s: found %lu functions\n", __func__, cnt);
466 printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
467 printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
468
469cleanup:
470 kprobe_multi_empty__destroy(skel);
471 if (syms) {
472 for (i = 0; i < cnt; i++)
473 free(syms[i]);
474 free(syms);
475 }
476}
477
478static void test_attach_override(void)
479{
480 struct kprobe_multi_override *skel = NULL;
481 struct bpf_link *link = NULL;
482
483 skel = kprobe_multi_override__open_and_load();
484 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
485 goto cleanup;
486
487 /* The test_override calls bpf_override_return so it should fail
488 * to attach to bpf_fentry_test1 function, which is not on error
489 * injection list.
490 */
491 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
492 "bpf_fentry_test1", NULL);
493 if (!ASSERT_ERR_PTR(link, "override_attached_bpf_fentry_test1")) {
494 bpf_link__destroy(link);
495 goto cleanup;
496 }
497
498 /* The should_fail_bio function is on error injection list,
499 * attach should succeed.
500 */
501 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
502 "should_fail_bio", NULL);
503 if (!ASSERT_OK_PTR(link, "override_attached_should_fail_bio"))
504 goto cleanup;
505
506 bpf_link__destroy(link);
507
508cleanup:
509 kprobe_multi_override__destroy(skel);
510}
511
512void serial_test_kprobe_multi_bench_attach(void)
513{
514 if (test__start_subtest("kernel"))
515 test_kprobe_multi_bench_attach(true);
516 if (test__start_subtest("modules"))
517 test_kprobe_multi_bench_attach(false);
518}
519
520void test_kprobe_multi_test(void)
521{
522 if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
523 return;
524
525 if (test__start_subtest("skel_api"))
526 test_skel_api();
527 if (test__start_subtest("link_api_addrs"))
528 test_link_api_syms();
529 if (test__start_subtest("link_api_syms"))
530 test_link_api_addrs();
531 if (test__start_subtest("attach_api_pattern"))
532 test_attach_api_pattern();
533 if (test__start_subtest("attach_api_addrs"))
534 test_attach_api_addrs();
535 if (test__start_subtest("attach_api_syms"))
536 test_attach_api_syms();
537 if (test__start_subtest("attach_api_fails"))
538 test_attach_api_fails();
539 if (test__start_subtest("attach_override"))
540 test_attach_override();
541}