Loading...
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */
3#include <test_progs.h>
4#include <unistd.h>
5#include <sys/syscall.h>
6#include <task_local_storage_helpers.h>
7#include "bpf_iter_ipv6_route.skel.h"
8#include "bpf_iter_netlink.skel.h"
9#include "bpf_iter_bpf_map.skel.h"
10#include "bpf_iter_tasks.skel.h"
11#include "bpf_iter_task_stack.skel.h"
12#include "bpf_iter_task_file.skel.h"
13#include "bpf_iter_task_vmas.skel.h"
14#include "bpf_iter_task_btf.skel.h"
15#include "bpf_iter_tcp4.skel.h"
16#include "bpf_iter_tcp6.skel.h"
17#include "bpf_iter_udp4.skel.h"
18#include "bpf_iter_udp6.skel.h"
19#include "bpf_iter_unix.skel.h"
20#include "bpf_iter_vma_offset.skel.h"
21#include "bpf_iter_test_kern1.skel.h"
22#include "bpf_iter_test_kern2.skel.h"
23#include "bpf_iter_test_kern3.skel.h"
24#include "bpf_iter_test_kern4.skel.h"
25#include "bpf_iter_bpf_hash_map.skel.h"
26#include "bpf_iter_bpf_percpu_hash_map.skel.h"
27#include "bpf_iter_bpf_array_map.skel.h"
28#include "bpf_iter_bpf_percpu_array_map.skel.h"
29#include "bpf_iter_bpf_sk_storage_helpers.skel.h"
30#include "bpf_iter_bpf_sk_storage_map.skel.h"
31#include "bpf_iter_test_kern5.skel.h"
32#include "bpf_iter_test_kern6.skel.h"
33#include "bpf_iter_bpf_link.skel.h"
34#include "bpf_iter_ksym.skel.h"
35#include "bpf_iter_sockmap.skel.h"
36
37static void test_btf_id_or_null(void)
38{
39 struct bpf_iter_test_kern3 *skel;
40
41 skel = bpf_iter_test_kern3__open_and_load();
42 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
43 bpf_iter_test_kern3__destroy(skel);
44 return;
45 }
46}
47
48static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
49{
50 struct bpf_link *link;
51 char buf[16] = {};
52 int iter_fd, len;
53
54 link = bpf_program__attach_iter(prog, opts);
55 if (!ASSERT_OK_PTR(link, "attach_iter"))
56 return;
57
58 iter_fd = bpf_iter_create(bpf_link__fd(link));
59 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
60 goto free_link;
61
62 /* not check contents, but ensure read() ends without error */
63 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
64 ;
65 ASSERT_GE(len, 0, "read");
66
67 close(iter_fd);
68
69free_link:
70 bpf_link__destroy(link);
71}
72
73static void do_dummy_read(struct bpf_program *prog)
74{
75 do_dummy_read_opts(prog, NULL);
76}
77
78static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
79 struct bpf_map *map)
80{
81 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
82 union bpf_iter_link_info linfo;
83 struct bpf_link *link;
84 char buf[16] = {};
85 int iter_fd, len;
86
87 memset(&linfo, 0, sizeof(linfo));
88 linfo.map.map_fd = bpf_map__fd(map);
89 opts.link_info = &linfo;
90 opts.link_info_len = sizeof(linfo);
91 link = bpf_program__attach_iter(prog, &opts);
92 if (!ASSERT_OK_PTR(link, "attach_map_iter"))
93 return;
94
95 iter_fd = bpf_iter_create(bpf_link__fd(link));
96 if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
97 bpf_link__destroy(link);
98 return;
99 }
100
101 /* Close link and map fd prematurely */
102 bpf_link__destroy(link);
103 bpf_object__destroy_skeleton(*skel);
104 *skel = NULL;
105
106 /* Try to let map free work to run first if map is freed */
107 usleep(100);
108 /* Memory used by both sock map and sock local storage map are
109 * freed after two synchronize_rcu() calls, so wait for it
110 */
111 kern_sync_rcu();
112 kern_sync_rcu();
113
114 /* Read after both map fd and link fd are closed */
115 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
116 ;
117 ASSERT_GE(len, 0, "read_iterator");
118
119 close(iter_fd);
120}
121
122static int read_fd_into_buffer(int fd, char *buf, int size)
123{
124 int bufleft = size;
125 int len;
126
127 do {
128 len = read(fd, buf, bufleft);
129 if (len > 0) {
130 buf += len;
131 bufleft -= len;
132 }
133 } while (len > 0);
134
135 return len < 0 ? len : size - bufleft;
136}
137
138static void test_ipv6_route(void)
139{
140 struct bpf_iter_ipv6_route *skel;
141
142 skel = bpf_iter_ipv6_route__open_and_load();
143 if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
144 return;
145
146 do_dummy_read(skel->progs.dump_ipv6_route);
147
148 bpf_iter_ipv6_route__destroy(skel);
149}
150
151static void test_netlink(void)
152{
153 struct bpf_iter_netlink *skel;
154
155 skel = bpf_iter_netlink__open_and_load();
156 if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
157 return;
158
159 do_dummy_read(skel->progs.dump_netlink);
160
161 bpf_iter_netlink__destroy(skel);
162}
163
164static void test_bpf_map(void)
165{
166 struct bpf_iter_bpf_map *skel;
167
168 skel = bpf_iter_bpf_map__open_and_load();
169 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
170 return;
171
172 do_dummy_read(skel->progs.dump_bpf_map);
173
174 bpf_iter_bpf_map__destroy(skel);
175}
176
177static void check_bpf_link_info(const struct bpf_program *prog)
178{
179 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
180 union bpf_iter_link_info linfo;
181 struct bpf_link_info info = {};
182 struct bpf_link *link;
183 __u32 info_len;
184 int err;
185
186 memset(&linfo, 0, sizeof(linfo));
187 linfo.task.tid = getpid();
188 opts.link_info = &linfo;
189 opts.link_info_len = sizeof(linfo);
190
191 link = bpf_program__attach_iter(prog, &opts);
192 if (!ASSERT_OK_PTR(link, "attach_iter"))
193 return;
194
195 info_len = sizeof(info);
196 err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
197 ASSERT_OK(err, "bpf_link_get_info_by_fd");
198 ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
199
200 bpf_link__destroy(link);
201}
202
203static pthread_mutex_t do_nothing_mutex;
204
205static void *do_nothing_wait(void *arg)
206{
207 pthread_mutex_lock(&do_nothing_mutex);
208 pthread_mutex_unlock(&do_nothing_mutex);
209
210 pthread_exit(arg);
211}
212
213static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
214 int *num_unknown, int *num_known)
215{
216 struct bpf_iter_tasks *skel;
217 pthread_t thread_id;
218 void *ret;
219
220 skel = bpf_iter_tasks__open_and_load();
221 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
222 return;
223
224 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
225
226 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
227 "pthread_create");
228
229 skel->bss->tid = getpid();
230
231 do_dummy_read_opts(skel->progs.dump_task, opts);
232
233 *num_unknown = skel->bss->num_unknown_tid;
234 *num_known = skel->bss->num_known_tid;
235
236 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
237 ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
238 "pthread_join");
239
240 bpf_iter_tasks__destroy(skel);
241}
242
243static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
244{
245 int num_unknown_tid, num_known_tid;
246
247 test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
248 ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
249 ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
250}
251
252static void test_task_tid(void)
253{
254 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
255 union bpf_iter_link_info linfo;
256 int num_unknown_tid, num_known_tid;
257
258 memset(&linfo, 0, sizeof(linfo));
259 linfo.task.tid = getpid();
260 opts.link_info = &linfo;
261 opts.link_info_len = sizeof(linfo);
262 test_task_common(&opts, 0, 1);
263
264 linfo.task.tid = 0;
265 linfo.task.pid = getpid();
266 test_task_common(&opts, 1, 1);
267
268 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
269 ASSERT_GT(num_unknown_tid, 1, "check_num_unknown_tid");
270 ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
271}
272
273static void test_task_pid(void)
274{
275 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
276 union bpf_iter_link_info linfo;
277
278 memset(&linfo, 0, sizeof(linfo));
279 linfo.task.pid = getpid();
280 opts.link_info = &linfo;
281 opts.link_info_len = sizeof(linfo);
282
283 test_task_common(&opts, 1, 1);
284}
285
286static void test_task_pidfd(void)
287{
288 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
289 union bpf_iter_link_info linfo;
290 int pidfd;
291
292 pidfd = sys_pidfd_open(getpid(), 0);
293 if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
294 return;
295
296 memset(&linfo, 0, sizeof(linfo));
297 linfo.task.pid_fd = pidfd;
298 opts.link_info = &linfo;
299 opts.link_info_len = sizeof(linfo);
300
301 test_task_common(&opts, 1, 1);
302
303 close(pidfd);
304}
305
306static void test_task_sleepable(void)
307{
308 struct bpf_iter_tasks *skel;
309
310 skel = bpf_iter_tasks__open_and_load();
311 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
312 return;
313
314 do_dummy_read(skel->progs.dump_task_sleepable);
315
316 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
317 "num_expected_failure_copy_from_user_task");
318 ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
319 "num_success_copy_from_user_task");
320
321 bpf_iter_tasks__destroy(skel);
322}
323
324static void test_task_stack(void)
325{
326 struct bpf_iter_task_stack *skel;
327
328 skel = bpf_iter_task_stack__open_and_load();
329 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
330 return;
331
332 do_dummy_read(skel->progs.dump_task_stack);
333 do_dummy_read(skel->progs.get_task_user_stacks);
334
335 ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");
336
337 bpf_iter_task_stack__destroy(skel);
338}
339
340static void test_task_file(void)
341{
342 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
343 struct bpf_iter_task_file *skel;
344 union bpf_iter_link_info linfo;
345 pthread_t thread_id;
346 void *ret;
347
348 skel = bpf_iter_task_file__open_and_load();
349 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
350 return;
351
352 skel->bss->tgid = getpid();
353
354 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
355
356 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
357 "pthread_create");
358
359 memset(&linfo, 0, sizeof(linfo));
360 linfo.task.tid = getpid();
361 opts.link_info = &linfo;
362 opts.link_info_len = sizeof(linfo);
363
364 do_dummy_read_opts(skel->progs.dump_task_file, &opts);
365
366 ASSERT_EQ(skel->bss->count, 0, "check_count");
367 ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
368
369 skel->bss->last_tgid = 0;
370 skel->bss->count = 0;
371 skel->bss->unique_tgid_count = 0;
372
373 do_dummy_read(skel->progs.dump_task_file);
374
375 ASSERT_EQ(skel->bss->count, 0, "check_count");
376 ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
377
378 check_bpf_link_info(skel->progs.dump_task_file);
379
380 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
381 ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
382 ASSERT_NULL(ret, "pthread_join");
383
384 bpf_iter_task_file__destroy(skel);
385}
386
387#define TASKBUFSZ 32768
388
389static char taskbuf[TASKBUFSZ];
390
391static int do_btf_read(struct bpf_iter_task_btf *skel)
392{
393 struct bpf_program *prog = skel->progs.dump_task_struct;
394 struct bpf_iter_task_btf__bss *bss = skel->bss;
395 int iter_fd = -1, err;
396 struct bpf_link *link;
397 char *buf = taskbuf;
398 int ret = 0;
399
400 link = bpf_program__attach_iter(prog, NULL);
401 if (!ASSERT_OK_PTR(link, "attach_iter"))
402 return ret;
403
404 iter_fd = bpf_iter_create(bpf_link__fd(link));
405 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
406 goto free_link;
407
408 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
409 if (bss->skip) {
410 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
411 ret = 1;
412 test__skip();
413 goto free_link;
414 }
415
416 if (!ASSERT_GE(err, 0, "read"))
417 goto free_link;
418
419 ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
420 "check for btf representation of task_struct in iter data");
421free_link:
422 if (iter_fd > 0)
423 close(iter_fd);
424 bpf_link__destroy(link);
425 return ret;
426}
427
428static void test_task_btf(void)
429{
430 struct bpf_iter_task_btf__bss *bss;
431 struct bpf_iter_task_btf *skel;
432 int ret;
433
434 skel = bpf_iter_task_btf__open_and_load();
435 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
436 return;
437
438 bss = skel->bss;
439
440 ret = do_btf_read(skel);
441 if (ret)
442 goto cleanup;
443
444 if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
445 goto cleanup;
446
447 ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");
448
449cleanup:
450 bpf_iter_task_btf__destroy(skel);
451}
452
453static void test_tcp4(void)
454{
455 struct bpf_iter_tcp4 *skel;
456
457 skel = bpf_iter_tcp4__open_and_load();
458 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
459 return;
460
461 do_dummy_read(skel->progs.dump_tcp4);
462
463 bpf_iter_tcp4__destroy(skel);
464}
465
466static void test_tcp6(void)
467{
468 struct bpf_iter_tcp6 *skel;
469
470 skel = bpf_iter_tcp6__open_and_load();
471 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
472 return;
473
474 do_dummy_read(skel->progs.dump_tcp6);
475
476 bpf_iter_tcp6__destroy(skel);
477}
478
479static void test_udp4(void)
480{
481 struct bpf_iter_udp4 *skel;
482
483 skel = bpf_iter_udp4__open_and_load();
484 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
485 return;
486
487 do_dummy_read(skel->progs.dump_udp4);
488
489 bpf_iter_udp4__destroy(skel);
490}
491
492static void test_udp6(void)
493{
494 struct bpf_iter_udp6 *skel;
495
496 skel = bpf_iter_udp6__open_and_load();
497 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
498 return;
499
500 do_dummy_read(skel->progs.dump_udp6);
501
502 bpf_iter_udp6__destroy(skel);
503}
504
505static void test_unix(void)
506{
507 struct bpf_iter_unix *skel;
508
509 skel = bpf_iter_unix__open_and_load();
510 if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
511 return;
512
513 do_dummy_read(skel->progs.dump_unix);
514
515 bpf_iter_unix__destroy(skel);
516}
517
518/* The expected string is less than 16 bytes */
519static int do_read_with_fd(int iter_fd, const char *expected,
520 bool read_one_char)
521{
522 int len, read_buf_len, start;
523 char buf[16] = {};
524
525 read_buf_len = read_one_char ? 1 : 16;
526 start = 0;
527 while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
528 start += len;
529 if (!ASSERT_LT(start, 16, "read"))
530 return -1;
531 read_buf_len = read_one_char ? 1 : 16 - start;
532 }
533 if (!ASSERT_GE(len, 0, "read"))
534 return -1;
535
536 if (!ASSERT_STREQ(buf, expected, "read"))
537 return -1;
538
539 return 0;
540}
541
542static void test_anon_iter(bool read_one_char)
543{
544 struct bpf_iter_test_kern1 *skel;
545 struct bpf_link *link;
546 int iter_fd, err;
547
548 skel = bpf_iter_test_kern1__open_and_load();
549 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
550 return;
551
552 err = bpf_iter_test_kern1__attach(skel);
553 if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
554 goto out;
555 }
556
557 link = skel->links.dump_task;
558 iter_fd = bpf_iter_create(bpf_link__fd(link));
559 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
560 goto out;
561
562 do_read_with_fd(iter_fd, "abcd", read_one_char);
563 close(iter_fd);
564
565out:
566 bpf_iter_test_kern1__destroy(skel);
567}
568
569static int do_read(const char *path, const char *expected)
570{
571 int err, iter_fd;
572
573 iter_fd = open(path, O_RDONLY);
574 if (!ASSERT_GE(iter_fd, 0, "open"))
575 return -1;
576
577 err = do_read_with_fd(iter_fd, expected, false);
578 close(iter_fd);
579 return err;
580}
581
582static void test_file_iter(void)
583{
584 const char *path = "/sys/fs/bpf/bpf_iter_test1";
585 struct bpf_iter_test_kern1 *skel1;
586 struct bpf_iter_test_kern2 *skel2;
587 struct bpf_link *link;
588 int err;
589
590 skel1 = bpf_iter_test_kern1__open_and_load();
591 if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
592 return;
593
594 link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
595 if (!ASSERT_OK_PTR(link, "attach_iter"))
596 goto out;
597
598 /* unlink this path if it exists. */
599 unlink(path);
600
601 err = bpf_link__pin(link, path);
602 if (!ASSERT_OK(err, "pin_iter"))
603 goto free_link;
604
605 err = do_read(path, "abcd");
606 if (err)
607 goto unlink_path;
608
609 /* file based iterator seems working fine. Let us a link update
610 * of the underlying link and `cat` the iterator again, its content
611 * should change.
612 */
613 skel2 = bpf_iter_test_kern2__open_and_load();
614 if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
615 goto unlink_path;
616
617 err = bpf_link__update_program(link, skel2->progs.dump_task);
618 if (!ASSERT_OK(err, "update_prog"))
619 goto destroy_skel2;
620
621 do_read(path, "ABCD");
622
623destroy_skel2:
624 bpf_iter_test_kern2__destroy(skel2);
625unlink_path:
626 unlink(path);
627free_link:
628 bpf_link__destroy(link);
629out:
630 bpf_iter_test_kern1__destroy(skel1);
631}
632
633static void test_overflow(bool test_e2big_overflow, bool ret1)
634{
635 __u32 map_info_len, total_read_len, expected_read_len;
636 int err, iter_fd, map1_fd, map2_fd, len;
637 struct bpf_map_info map_info = {};
638 struct bpf_iter_test_kern4 *skel;
639 struct bpf_link *link;
640 __u32 iter_size;
641 char *buf;
642
643 skel = bpf_iter_test_kern4__open();
644 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
645 return;
646
647 /* create two maps: bpf program will only do bpf_seq_write
648 * for these two maps. The goal is one map output almost
649 * fills seq_file buffer and then the other will trigger
650 * overflow and needs restart.
651 */
652 map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
653 if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
654 goto out;
655 map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
656 if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
657 goto free_map1;
658
659 /* bpf_seq_printf kernel buffer is 8 pages, so one map
660 * bpf_seq_write will mostly fill it, and the other map
661 * will partially fill and then trigger overflow and need
662 * bpf_seq_read restart.
663 */
664 iter_size = sysconf(_SC_PAGE_SIZE) << 3;
665
666 if (test_e2big_overflow) {
667 skel->rodata->print_len = (iter_size + 8) / 8;
668 expected_read_len = 2 * (iter_size + 8);
669 } else if (!ret1) {
670 skel->rodata->print_len = (iter_size - 8) / 8;
671 expected_read_len = 2 * (iter_size - 8);
672 } else {
673 skel->rodata->print_len = 1;
674 expected_read_len = 2 * 8;
675 }
676 skel->rodata->ret1 = ret1;
677
678 if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
679 "bpf_iter_test_kern4__load"))
680 goto free_map2;
681
682 /* setup filtering map_id in bpf program */
683 map_info_len = sizeof(map_info);
684 err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
685 if (!ASSERT_OK(err, "get_map_info"))
686 goto free_map2;
687 skel->bss->map1_id = map_info.id;
688
689 err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
690 if (!ASSERT_OK(err, "get_map_info"))
691 goto free_map2;
692 skel->bss->map2_id = map_info.id;
693
694 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
695 if (!ASSERT_OK_PTR(link, "attach_iter"))
696 goto free_map2;
697
698 iter_fd = bpf_iter_create(bpf_link__fd(link));
699 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
700 goto free_link;
701
702 buf = malloc(expected_read_len);
703 if (!ASSERT_OK_PTR(buf, "malloc"))
704 goto close_iter;
705
706 /* do read */
707 total_read_len = 0;
708 if (test_e2big_overflow) {
709 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
710 total_read_len += len;
711
712 ASSERT_EQ(len, -1, "read");
713 ASSERT_EQ(errno, E2BIG, "read");
714 goto free_buf;
715 } else if (!ret1) {
716 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
717 total_read_len += len;
718
719 if (!ASSERT_GE(len, 0, "read"))
720 goto free_buf;
721 } else {
722 do {
723 len = read(iter_fd, buf, expected_read_len);
724 if (len > 0)
725 total_read_len += len;
726 } while (len > 0 || len == -EAGAIN);
727
728 if (!ASSERT_GE(len, 0, "read"))
729 goto free_buf;
730 }
731
732 if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
733 goto free_buf;
734
735 if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
736 goto free_buf;
737
738 if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
739 goto free_buf;
740
741 ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");
742
743free_buf:
744 free(buf);
745close_iter:
746 close(iter_fd);
747free_link:
748 bpf_link__destroy(link);
749free_map2:
750 close(map2_fd);
751free_map1:
752 close(map1_fd);
753out:
754 bpf_iter_test_kern4__destroy(skel);
755}
756
757static void test_bpf_hash_map(void)
758{
759 __u32 expected_key_a = 0, expected_key_b = 0;
760 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
761 struct bpf_iter_bpf_hash_map *skel;
762 int err, i, len, map_fd, iter_fd;
763 union bpf_iter_link_info linfo;
764 __u64 val, expected_val = 0;
765 struct bpf_link *link;
766 struct key_t {
767 int a;
768 int b;
769 int c;
770 } key;
771 char buf[64];
772
773 skel = bpf_iter_bpf_hash_map__open();
774 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
775 return;
776
777 skel->bss->in_test_mode = true;
778
779 err = bpf_iter_bpf_hash_map__load(skel);
780 if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
781 goto out;
782
783 /* iterator with hashmap2 and hashmap3 should fail */
784 memset(&linfo, 0, sizeof(linfo));
785 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
786 opts.link_info = &linfo;
787 opts.link_info_len = sizeof(linfo);
788 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
789 if (!ASSERT_ERR_PTR(link, "attach_iter"))
790 goto out;
791
792 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
793 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
794 if (!ASSERT_ERR_PTR(link, "attach_iter"))
795 goto out;
796
797 /* hashmap1 should be good, update map values here */
798 map_fd = bpf_map__fd(skel->maps.hashmap1);
799 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
800 key.a = i + 1;
801 key.b = i + 2;
802 key.c = i + 3;
803 val = i + 4;
804 expected_key_a += key.a;
805 expected_key_b += key.b;
806 expected_val += val;
807
808 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
809 if (!ASSERT_OK(err, "map_update"))
810 goto out;
811 }
812
813 /* Sleepable program is prohibited for hash map iterator */
814 linfo.map.map_fd = map_fd;
815 link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
816 if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
817 goto out;
818
819 linfo.map.map_fd = map_fd;
820 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
821 if (!ASSERT_OK_PTR(link, "attach_iter"))
822 goto out;
823
824 iter_fd = bpf_iter_create(bpf_link__fd(link));
825 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
826 goto free_link;
827
828 /* do some tests */
829 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
830 ;
831 if (!ASSERT_GE(len, 0, "read"))
832 goto close_iter;
833
834 /* test results */
835 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
836 goto close_iter;
837 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
838 goto close_iter;
839 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
840 goto close_iter;
841
842close_iter:
843 close(iter_fd);
844free_link:
845 bpf_link__destroy(link);
846out:
847 bpf_iter_bpf_hash_map__destroy(skel);
848}
849
850static void test_bpf_percpu_hash_map(void)
851{
852 __u32 expected_key_a = 0, expected_key_b = 0;
853 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
854 struct bpf_iter_bpf_percpu_hash_map *skel;
855 int err, i, j, len, map_fd, iter_fd;
856 union bpf_iter_link_info linfo;
857 __u32 expected_val = 0;
858 struct bpf_link *link;
859 struct key_t {
860 int a;
861 int b;
862 int c;
863 } key;
864 char buf[64];
865 void *val;
866
867 skel = bpf_iter_bpf_percpu_hash_map__open();
868 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
869 return;
870
871 skel->rodata->num_cpus = bpf_num_possible_cpus();
872 val = malloc(8 * bpf_num_possible_cpus());
873 if (!ASSERT_OK_PTR(val, "malloc"))
874 goto out;
875
876 err = bpf_iter_bpf_percpu_hash_map__load(skel);
877 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
878 goto out;
879
880 /* update map values here */
881 map_fd = bpf_map__fd(skel->maps.hashmap1);
882 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
883 key.a = i + 1;
884 key.b = i + 2;
885 key.c = i + 3;
886 expected_key_a += key.a;
887 expected_key_b += key.b;
888
889 for (j = 0; j < bpf_num_possible_cpus(); j++) {
890 *(__u32 *)(val + j * 8) = i + j;
891 expected_val += i + j;
892 }
893
894 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
895 if (!ASSERT_OK(err, "map_update"))
896 goto out;
897 }
898
899 memset(&linfo, 0, sizeof(linfo));
900 linfo.map.map_fd = map_fd;
901 opts.link_info = &linfo;
902 opts.link_info_len = sizeof(linfo);
903 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
904 if (!ASSERT_OK_PTR(link, "attach_iter"))
905 goto out;
906
907 iter_fd = bpf_iter_create(bpf_link__fd(link));
908 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
909 goto free_link;
910
911 /* do some tests */
912 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
913 ;
914 if (!ASSERT_GE(len, 0, "read"))
915 goto close_iter;
916
917 /* test results */
918 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
919 goto close_iter;
920 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
921 goto close_iter;
922 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
923 goto close_iter;
924
925close_iter:
926 close(iter_fd);
927free_link:
928 bpf_link__destroy(link);
929out:
930 bpf_iter_bpf_percpu_hash_map__destroy(skel);
931 free(val);
932}
933
934static void test_bpf_array_map(void)
935{
936 __u64 val, expected_val = 0, res_first_val, first_val = 0;
937 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
938 __u32 key, expected_key = 0, res_first_key;
939 int err, i, map_fd, hash_fd, iter_fd;
940 struct bpf_iter_bpf_array_map *skel;
941 union bpf_iter_link_info linfo;
942 struct bpf_link *link;
943 char buf[64] = {};
944 int len, start;
945
946 skel = bpf_iter_bpf_array_map__open_and_load();
947 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
948 return;
949
950 map_fd = bpf_map__fd(skel->maps.arraymap1);
951 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
952 val = i + 4;
953 expected_key += i;
954 expected_val += val;
955
956 if (i == 0)
957 first_val = val;
958
959 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
960 if (!ASSERT_OK(err, "map_update"))
961 goto out;
962 }
963
964 memset(&linfo, 0, sizeof(linfo));
965 linfo.map.map_fd = map_fd;
966 opts.link_info = &linfo;
967 opts.link_info_len = sizeof(linfo);
968 link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
969 if (!ASSERT_OK_PTR(link, "attach_iter"))
970 goto out;
971
972 iter_fd = bpf_iter_create(bpf_link__fd(link));
973 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
974 goto free_link;
975
976 /* do some tests */
977 start = 0;
978 while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
979 start += len;
980 if (!ASSERT_GE(len, 0, "read"))
981 goto close_iter;
982
983 /* test results */
984 res_first_key = *(__u32 *)buf;
985 res_first_val = *(__u64 *)(buf + sizeof(__u32));
986 if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
987 !ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
988 goto close_iter;
989
990 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
991 goto close_iter;
992 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
993 goto close_iter;
994
995 hash_fd = bpf_map__fd(skel->maps.hashmap1);
996 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
997 err = bpf_map_lookup_elem(map_fd, &i, &val);
998 if (!ASSERT_OK(err, "map_lookup arraymap1"))
999 goto close_iter;
1000 if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
1001 goto close_iter;
1002
1003 val = i + 4;
1004 err = bpf_map_lookup_elem(hash_fd, &val, &key);
1005 if (!ASSERT_OK(err, "map_lookup hashmap1"))
1006 goto close_iter;
1007 if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
1008 goto close_iter;
1009 }
1010
1011close_iter:
1012 close(iter_fd);
1013free_link:
1014 bpf_link__destroy(link);
1015out:
1016 bpf_iter_bpf_array_map__destroy(skel);
1017}
1018
1019static void test_bpf_array_map_iter_fd(void)
1020{
1021 struct bpf_iter_bpf_array_map *skel;
1022
1023 skel = bpf_iter_bpf_array_map__open_and_load();
1024 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1025 return;
1026
1027 do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,
1028 skel->maps.arraymap1);
1029
1030 bpf_iter_bpf_array_map__destroy(skel);
1031}
1032
1033static void test_bpf_percpu_array_map(void)
1034{
1035 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1036 struct bpf_iter_bpf_percpu_array_map *skel;
1037 __u32 expected_key = 0, expected_val = 0;
1038 union bpf_iter_link_info linfo;
1039 int err, i, j, map_fd, iter_fd;
1040 struct bpf_link *link;
1041 char buf[64];
1042 void *val;
1043 int len;
1044
1045 skel = bpf_iter_bpf_percpu_array_map__open();
1046 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
1047 return;
1048
1049 skel->rodata->num_cpus = bpf_num_possible_cpus();
1050 val = malloc(8 * bpf_num_possible_cpus());
1051 if (!ASSERT_OK_PTR(val, "malloc"))
1052 goto out;
1053
1054 err = bpf_iter_bpf_percpu_array_map__load(skel);
1055 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
1056 goto out;
1057
1058 /* update map values here */
1059 map_fd = bpf_map__fd(skel->maps.arraymap1);
1060 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1061 expected_key += i;
1062
1063 for (j = 0; j < bpf_num_possible_cpus(); j++) {
1064 *(__u32 *)(val + j * 8) = i + j;
1065 expected_val += i + j;
1066 }
1067
1068 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
1069 if (!ASSERT_OK(err, "map_update"))
1070 goto out;
1071 }
1072
1073 memset(&linfo, 0, sizeof(linfo));
1074 linfo.map.map_fd = map_fd;
1075 opts.link_info = &linfo;
1076 opts.link_info_len = sizeof(linfo);
1077 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
1078 if (!ASSERT_OK_PTR(link, "attach_iter"))
1079 goto out;
1080
1081 iter_fd = bpf_iter_create(bpf_link__fd(link));
1082 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1083 goto free_link;
1084
1085 /* do some tests */
1086 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1087 ;
1088 if (!ASSERT_GE(len, 0, "read"))
1089 goto close_iter;
1090
1091 /* test results */
1092 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1093 goto close_iter;
1094 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1095 goto close_iter;
1096
1097close_iter:
1098 close(iter_fd);
1099free_link:
1100 bpf_link__destroy(link);
1101out:
1102 bpf_iter_bpf_percpu_array_map__destroy(skel);
1103 free(val);
1104}
1105
1106/* An iterator program deletes all local storage in a map. */
1107static void test_bpf_sk_storage_delete(void)
1108{
1109 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1110 struct bpf_iter_bpf_sk_storage_helpers *skel;
1111 union bpf_iter_link_info linfo;
1112 int err, len, map_fd, iter_fd;
1113 struct bpf_link *link;
1114 int sock_fd = -1;
1115 __u32 val = 42;
1116 char buf[64];
1117
1118 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1119 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1120 return;
1121
1122 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1123
1124 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1125 if (!ASSERT_GE(sock_fd, 0, "socket"))
1126 goto out;
1127
1128 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1129 if (!ASSERT_OK(err, "map_update"))
1130 goto out;
1131
1132 memset(&linfo, 0, sizeof(linfo));
1133 linfo.map.map_fd = map_fd;
1134 opts.link_info = &linfo;
1135 opts.link_info_len = sizeof(linfo);
1136 link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
1137 &opts);
1138 if (!ASSERT_OK_PTR(link, "attach_iter"))
1139 goto out;
1140
1141 iter_fd = bpf_iter_create(bpf_link__fd(link));
1142 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1143 goto free_link;
1144
1145 /* do some tests */
1146 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1147 ;
1148 if (!ASSERT_GE(len, 0, "read"))
1149 goto close_iter;
1150
1151 /* test results */
1152 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1153
1154 /* Note: The following assertions serve to ensure
1155 * the value was deleted. It does so by asserting
1156 * that bpf_map_lookup_elem has failed. This might
1157 * seem counterintuitive at first.
1158 */
1159 ASSERT_ERR(err, "bpf_map_lookup_elem");
1160 ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");
1161
1162close_iter:
1163 close(iter_fd);
1164free_link:
1165 bpf_link__destroy(link);
1166out:
1167 if (sock_fd >= 0)
1168 close(sock_fd);
1169 bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1170}
1171
1172/* This creates a socket and its local storage. It then runs a task_iter BPF
1173 * program that replaces the existing socket local storage with the tgid of the
1174 * only task owning a file descriptor to this socket, this process, prog_tests.
1175 * It then runs a tcp socket iterator that negates the value in the existing
1176 * socket local storage, the test verifies that the resulting value is -pid.
1177 */
1178static void test_bpf_sk_storage_get(void)
1179{
1180 struct bpf_iter_bpf_sk_storage_helpers *skel;
1181 int err, map_fd, val = -1;
1182 int sock_fd = -1;
1183
1184 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1185 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1186 return;
1187
1188 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1189 if (!ASSERT_GE(sock_fd, 0, "socket"))
1190 goto out;
1191
1192 err = listen(sock_fd, 1);
1193 if (!ASSERT_OK(err, "listen"))
1194 goto close_socket;
1195
1196 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1197
1198 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1199 if (!ASSERT_OK(err, "bpf_map_update_elem"))
1200 goto close_socket;
1201
1202 do_dummy_read(skel->progs.fill_socket_owner);
1203
1204 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1205 if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
1206 !ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
1207 goto close_socket;
1208
1209 do_dummy_read(skel->progs.negate_socket_local_storage);
1210
1211 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1212 ASSERT_OK(err, "bpf_map_lookup_elem");
1213 ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");
1214
1215close_socket:
1216 close(sock_fd);
1217out:
1218 bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1219}
1220
1221static void test_bpf_sk_stoarge_map_iter_fd(void)
1222{
1223 struct bpf_iter_bpf_sk_storage_map *skel;
1224
1225 skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1226 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1227 return;
1228
1229 do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,
1230 skel->maps.sk_stg_map);
1231
1232 bpf_iter_bpf_sk_storage_map__destroy(skel);
1233}
1234
1235static void test_bpf_sk_storage_map(void)
1236{
1237 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1238 int err, i, len, map_fd, iter_fd, num_sockets;
1239 struct bpf_iter_bpf_sk_storage_map *skel;
1240 union bpf_iter_link_info linfo;
1241 int sock_fd[3] = {-1, -1, -1};
1242 __u32 val, expected_val = 0;
1243 struct bpf_link *link;
1244 char buf[64];
1245
1246 skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1247 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1248 return;
1249
1250 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1251 num_sockets = ARRAY_SIZE(sock_fd);
1252 for (i = 0; i < num_sockets; i++) {
1253 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
1254 if (!ASSERT_GE(sock_fd[i], 0, "socket"))
1255 goto out;
1256
1257 val = i + 1;
1258 expected_val += val;
1259
1260 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
1261 BPF_NOEXIST);
1262 if (!ASSERT_OK(err, "map_update"))
1263 goto out;
1264 }
1265
1266 memset(&linfo, 0, sizeof(linfo));
1267 linfo.map.map_fd = map_fd;
1268 opts.link_info = &linfo;
1269 opts.link_info_len = sizeof(linfo);
1270 link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
1271 err = libbpf_get_error(link);
1272 if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
1273 if (!err)
1274 bpf_link__destroy(link);
1275 goto out;
1276 }
1277
1278 link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
1279 if (!ASSERT_OK_PTR(link, "attach_iter"))
1280 goto out;
1281
1282 iter_fd = bpf_iter_create(bpf_link__fd(link));
1283 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1284 goto free_link;
1285
1286 skel->bss->to_add_val = time(NULL);
1287 /* do some tests */
1288 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1289 ;
1290 if (!ASSERT_GE(len, 0, "read"))
1291 goto close_iter;
1292
1293 /* test results */
1294 if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
1295 goto close_iter;
1296
1297 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1298 goto close_iter;
1299
1300 for (i = 0; i < num_sockets; i++) {
1301 err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
1302 if (!ASSERT_OK(err, "map_lookup") ||
1303 !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
1304 break;
1305 }
1306
1307close_iter:
1308 close(iter_fd);
1309free_link:
1310 bpf_link__destroy(link);
1311out:
1312 for (i = 0; i < num_sockets; i++) {
1313 if (sock_fd[i] >= 0)
1314 close(sock_fd[i]);
1315 }
1316 bpf_iter_bpf_sk_storage_map__destroy(skel);
1317}
1318
1319static void test_rdonly_buf_out_of_bound(void)
1320{
1321 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1322 struct bpf_iter_test_kern5 *skel;
1323 union bpf_iter_link_info linfo;
1324 struct bpf_link *link;
1325
1326 skel = bpf_iter_test_kern5__open_and_load();
1327 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
1328 return;
1329
1330 memset(&linfo, 0, sizeof(linfo));
1331 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
1332 opts.link_info = &linfo;
1333 opts.link_info_len = sizeof(linfo);
1334 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
1335 if (!ASSERT_ERR_PTR(link, "attach_iter"))
1336 bpf_link__destroy(link);
1337
1338 bpf_iter_test_kern5__destroy(skel);
1339}
1340
1341static void test_buf_neg_offset(void)
1342{
1343 struct bpf_iter_test_kern6 *skel;
1344
1345 skel = bpf_iter_test_kern6__open_and_load();
1346 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
1347 bpf_iter_test_kern6__destroy(skel);
1348}
1349
1350static void test_link_iter(void)
1351{
1352 struct bpf_iter_bpf_link *skel;
1353
1354 skel = bpf_iter_bpf_link__open_and_load();
1355 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
1356 return;
1357
1358 do_dummy_read(skel->progs.dump_bpf_link);
1359
1360 bpf_iter_bpf_link__destroy(skel);
1361}
1362
1363static void test_ksym_iter(void)
1364{
1365 struct bpf_iter_ksym *skel;
1366
1367 skel = bpf_iter_ksym__open_and_load();
1368 if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
1369 return;
1370
1371 do_dummy_read(skel->progs.dump_ksym);
1372
1373 bpf_iter_ksym__destroy(skel);
1374}
1375
1376#define CMP_BUFFER_SIZE 1024
1377static char task_vma_output[CMP_BUFFER_SIZE];
1378static char proc_maps_output[CMP_BUFFER_SIZE];
1379
1380/* remove \0 and \t from str, and only keep the first line */
1381static void str_strip_first_line(char *str)
1382{
1383 char *dst = str, *src = str;
1384
1385 do {
1386 if (*src == ' ' || *src == '\t')
1387 src++;
1388 else
1389 *(dst++) = *(src++);
1390
1391 } while (*src != '\0' && *src != '\n');
1392
1393 *dst = '\0';
1394}
1395
1396static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
1397{
1398 int err, iter_fd = -1, proc_maps_fd = -1;
1399 struct bpf_iter_task_vmas *skel;
1400 int len, read_size = 4;
1401 char maps_path[64];
1402
1403 skel = bpf_iter_task_vmas__open();
1404 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1405 return;
1406
1407 skel->bss->pid = getpid();
1408 skel->bss->one_task = opts ? 1 : 0;
1409
1410 err = bpf_iter_task_vmas__load(skel);
1411 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1412 goto out;
1413
1414 skel->links.proc_maps = bpf_program__attach_iter(
1415 skel->progs.proc_maps, opts);
1416
1417 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1418 skel->links.proc_maps = NULL;
1419 goto out;
1420 }
1421
1422 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1423 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1424 goto out;
1425
1426 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1427 * to trigger seq_file corner cases.
1428 */
1429 len = 0;
1430 while (len < CMP_BUFFER_SIZE) {
1431 err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1432 MIN(read_size, CMP_BUFFER_SIZE - len));
1433 if (!err)
1434 break;
1435 if (!ASSERT_GE(err, 0, "read_iter_fd"))
1436 goto out;
1437 len += err;
1438 }
1439 if (opts)
1440 ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
1441
1442 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1443 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1444 proc_maps_fd = open(maps_path, O_RDONLY);
1445 if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
1446 goto out;
1447 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1448 if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
1449 goto out;
1450
1451 /* strip and compare the first line of the two files */
1452 str_strip_first_line(task_vma_output);
1453 str_strip_first_line(proc_maps_output);
1454
1455 ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
1456
1457 check_bpf_link_info(skel->progs.proc_maps);
1458
1459out:
1460 close(proc_maps_fd);
1461 close(iter_fd);
1462 bpf_iter_task_vmas__destroy(skel);
1463}
1464
1465static void test_task_vma_dead_task(void)
1466{
1467 struct bpf_iter_task_vmas *skel;
1468 int wstatus, child_pid = -1;
1469 time_t start_tm, cur_tm;
1470 int err, iter_fd = -1;
1471 int wait_sec = 3;
1472
1473 skel = bpf_iter_task_vmas__open();
1474 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1475 return;
1476
1477 skel->bss->pid = getpid();
1478
1479 err = bpf_iter_task_vmas__load(skel);
1480 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1481 goto out;
1482
1483 skel->links.proc_maps = bpf_program__attach_iter(
1484 skel->progs.proc_maps, NULL);
1485
1486 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1487 skel->links.proc_maps = NULL;
1488 goto out;
1489 }
1490
1491 start_tm = time(NULL);
1492 cur_tm = start_tm;
1493
1494 child_pid = fork();
1495 if (child_pid == 0) {
1496 /* Fork short-lived processes in the background. */
1497 while (cur_tm < start_tm + wait_sec) {
1498 system("echo > /dev/null");
1499 cur_tm = time(NULL);
1500 }
1501 exit(0);
1502 }
1503
1504 if (!ASSERT_GE(child_pid, 0, "fork_child"))
1505 goto out;
1506
1507 while (cur_tm < start_tm + wait_sec) {
1508 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1509 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1510 goto out;
1511
1512 /* Drain all data from iter_fd. */
1513 while (cur_tm < start_tm + wait_sec) {
1514 err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);
1515 if (!ASSERT_GE(err, 0, "read_iter_fd"))
1516 goto out;
1517
1518 cur_tm = time(NULL);
1519
1520 if (err == 0)
1521 break;
1522 }
1523
1524 close(iter_fd);
1525 iter_fd = -1;
1526 }
1527
1528 check_bpf_link_info(skel->progs.proc_maps);
1529
1530out:
1531 waitpid(child_pid, &wstatus, 0);
1532 close(iter_fd);
1533 bpf_iter_task_vmas__destroy(skel);
1534}
1535
1536void test_bpf_sockmap_map_iter_fd(void)
1537{
1538 struct bpf_iter_sockmap *skel;
1539
1540 skel = bpf_iter_sockmap__open_and_load();
1541 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
1542 return;
1543
1544 do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);
1545
1546 bpf_iter_sockmap__destroy(skel);
1547}
1548
1549static void test_task_vma(void)
1550{
1551 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1552 union bpf_iter_link_info linfo;
1553
1554 memset(&linfo, 0, sizeof(linfo));
1555 linfo.task.tid = getpid();
1556 opts.link_info = &linfo;
1557 opts.link_info_len = sizeof(linfo);
1558
1559 test_task_vma_common(&opts);
1560 test_task_vma_common(NULL);
1561}
1562
1563/* uprobe attach point */
1564static noinline int trigger_func(int arg)
1565{
1566 asm volatile ("");
1567 return arg + 1;
1568}
1569
1570static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
1571{
1572 struct bpf_iter_vma_offset *skel;
1573 char buf[16] = {};
1574 int iter_fd, len;
1575 int pgsz, shift;
1576
1577 skel = bpf_iter_vma_offset__open_and_load();
1578 if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
1579 return;
1580
1581 skel->bss->pid = getpid();
1582 skel->bss->address = (uintptr_t)trigger_func;
1583 for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
1584 ;
1585 skel->bss->page_shift = shift;
1586
1587 skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
1588 if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
1589 goto exit;
1590
1591 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
1592 if (!ASSERT_GT(iter_fd, 0, "create_iter"))
1593 goto exit;
1594
1595 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1596 ;
1597 buf[15] = 0;
1598 ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
1599
1600 ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
1601 if (one_proc)
1602 ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1603 else
1604 ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1605
1606 close(iter_fd);
1607
1608exit:
1609 bpf_iter_vma_offset__destroy(skel);
1610}
1611
1612static void test_task_vma_offset(void)
1613{
1614 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1615 union bpf_iter_link_info linfo;
1616
1617 memset(&linfo, 0, sizeof(linfo));
1618 linfo.task.pid = getpid();
1619 opts.link_info = &linfo;
1620 opts.link_info_len = sizeof(linfo);
1621
1622 test_task_vma_offset_common(&opts, true);
1623
1624 linfo.task.pid = 0;
1625 linfo.task.tid = getpid();
1626 test_task_vma_offset_common(&opts, true);
1627
1628 test_task_vma_offset_common(NULL, false);
1629}
1630
1631void test_bpf_iter(void)
1632{
1633 ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
1634
1635 if (test__start_subtest("btf_id_or_null"))
1636 test_btf_id_or_null();
1637 if (test__start_subtest("ipv6_route"))
1638 test_ipv6_route();
1639 if (test__start_subtest("netlink"))
1640 test_netlink();
1641 if (test__start_subtest("bpf_map"))
1642 test_bpf_map();
1643 if (test__start_subtest("task_tid"))
1644 test_task_tid();
1645 if (test__start_subtest("task_pid"))
1646 test_task_pid();
1647 if (test__start_subtest("task_pidfd"))
1648 test_task_pidfd();
1649 if (test__start_subtest("task_sleepable"))
1650 test_task_sleepable();
1651 if (test__start_subtest("task_stack"))
1652 test_task_stack();
1653 if (test__start_subtest("task_file"))
1654 test_task_file();
1655 if (test__start_subtest("task_vma"))
1656 test_task_vma();
1657 if (test__start_subtest("task_vma_dead_task"))
1658 test_task_vma_dead_task();
1659 if (test__start_subtest("task_btf"))
1660 test_task_btf();
1661 if (test__start_subtest("tcp4"))
1662 test_tcp4();
1663 if (test__start_subtest("tcp6"))
1664 test_tcp6();
1665 if (test__start_subtest("udp4"))
1666 test_udp4();
1667 if (test__start_subtest("udp6"))
1668 test_udp6();
1669 if (test__start_subtest("unix"))
1670 test_unix();
1671 if (test__start_subtest("anon"))
1672 test_anon_iter(false);
1673 if (test__start_subtest("anon-read-one-char"))
1674 test_anon_iter(true);
1675 if (test__start_subtest("file"))
1676 test_file_iter();
1677 if (test__start_subtest("overflow"))
1678 test_overflow(false, false);
1679 if (test__start_subtest("overflow-e2big"))
1680 test_overflow(true, false);
1681 if (test__start_subtest("prog-ret-1"))
1682 test_overflow(false, true);
1683 if (test__start_subtest("bpf_hash_map"))
1684 test_bpf_hash_map();
1685 if (test__start_subtest("bpf_percpu_hash_map"))
1686 test_bpf_percpu_hash_map();
1687 if (test__start_subtest("bpf_array_map"))
1688 test_bpf_array_map();
1689 if (test__start_subtest("bpf_array_map_iter_fd"))
1690 test_bpf_array_map_iter_fd();
1691 if (test__start_subtest("bpf_percpu_array_map"))
1692 test_bpf_percpu_array_map();
1693 if (test__start_subtest("bpf_sk_storage_map"))
1694 test_bpf_sk_storage_map();
1695 if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
1696 test_bpf_sk_stoarge_map_iter_fd();
1697 if (test__start_subtest("bpf_sk_storage_delete"))
1698 test_bpf_sk_storage_delete();
1699 if (test__start_subtest("bpf_sk_storage_get"))
1700 test_bpf_sk_storage_get();
1701 if (test__start_subtest("rdonly-buf-out-of-bound"))
1702 test_rdonly_buf_out_of_bound();
1703 if (test__start_subtest("buf-neg-offset"))
1704 test_buf_neg_offset();
1705 if (test__start_subtest("link-iter"))
1706 test_link_iter();
1707 if (test__start_subtest("ksym"))
1708 test_ksym_iter();
1709 if (test__start_subtest("bpf_sockmap_map_iter_fd"))
1710 test_bpf_sockmap_map_iter_fd();
1711 if (test__start_subtest("vma_offset"))
1712 test_task_vma_offset();
1713}
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */
3#include <test_progs.h>
4#include <unistd.h>
5#include <sys/syscall.h>
6#include <task_local_storage_helpers.h>
7#include "bpf_iter_ipv6_route.skel.h"
8#include "bpf_iter_netlink.skel.h"
9#include "bpf_iter_bpf_map.skel.h"
10#include "bpf_iter_task.skel.h"
11#include "bpf_iter_task_stack.skel.h"
12#include "bpf_iter_task_file.skel.h"
13#include "bpf_iter_task_vma.skel.h"
14#include "bpf_iter_task_btf.skel.h"
15#include "bpf_iter_tcp4.skel.h"
16#include "bpf_iter_tcp6.skel.h"
17#include "bpf_iter_udp4.skel.h"
18#include "bpf_iter_udp6.skel.h"
19#include "bpf_iter_unix.skel.h"
20#include "bpf_iter_vma_offset.skel.h"
21#include "bpf_iter_test_kern1.skel.h"
22#include "bpf_iter_test_kern2.skel.h"
23#include "bpf_iter_test_kern3.skel.h"
24#include "bpf_iter_test_kern4.skel.h"
25#include "bpf_iter_bpf_hash_map.skel.h"
26#include "bpf_iter_bpf_percpu_hash_map.skel.h"
27#include "bpf_iter_bpf_array_map.skel.h"
28#include "bpf_iter_bpf_percpu_array_map.skel.h"
29#include "bpf_iter_bpf_sk_storage_helpers.skel.h"
30#include "bpf_iter_bpf_sk_storage_map.skel.h"
31#include "bpf_iter_test_kern5.skel.h"
32#include "bpf_iter_test_kern6.skel.h"
33#include "bpf_iter_bpf_link.skel.h"
34#include "bpf_iter_ksym.skel.h"
35#include "bpf_iter_sockmap.skel.h"
36
37static int duration;
38
39static void test_btf_id_or_null(void)
40{
41 struct bpf_iter_test_kern3 *skel;
42
43 skel = bpf_iter_test_kern3__open_and_load();
44 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
45 bpf_iter_test_kern3__destroy(skel);
46 return;
47 }
48}
49
50static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
51{
52 struct bpf_link *link;
53 char buf[16] = {};
54 int iter_fd, len;
55
56 link = bpf_program__attach_iter(prog, opts);
57 if (!ASSERT_OK_PTR(link, "attach_iter"))
58 return;
59
60 iter_fd = bpf_iter_create(bpf_link__fd(link));
61 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
62 goto free_link;
63
64 /* not check contents, but ensure read() ends without error */
65 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
66 ;
67 CHECK(len < 0, "read", "read failed: %s\n", strerror(errno));
68
69 close(iter_fd);
70
71free_link:
72 bpf_link__destroy(link);
73}
74
75static void do_dummy_read(struct bpf_program *prog)
76{
77 do_dummy_read_opts(prog, NULL);
78}
79
80static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
81 struct bpf_map *map)
82{
83 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
84 union bpf_iter_link_info linfo;
85 struct bpf_link *link;
86 char buf[16] = {};
87 int iter_fd, len;
88
89 memset(&linfo, 0, sizeof(linfo));
90 linfo.map.map_fd = bpf_map__fd(map);
91 opts.link_info = &linfo;
92 opts.link_info_len = sizeof(linfo);
93 link = bpf_program__attach_iter(prog, &opts);
94 if (!ASSERT_OK_PTR(link, "attach_map_iter"))
95 return;
96
97 iter_fd = bpf_iter_create(bpf_link__fd(link));
98 if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
99 bpf_link__destroy(link);
100 return;
101 }
102
103 /* Close link and map fd prematurely */
104 bpf_link__destroy(link);
105 bpf_object__destroy_skeleton(*skel);
106 *skel = NULL;
107
108 /* Try to let map free work to run first if map is freed */
109 usleep(100);
110 /* Memory used by both sock map and sock local storage map are
111 * freed after two synchronize_rcu() calls, so wait for it
112 */
113 kern_sync_rcu();
114 kern_sync_rcu();
115
116 /* Read after both map fd and link fd are closed */
117 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
118 ;
119 ASSERT_GE(len, 0, "read_iterator");
120
121 close(iter_fd);
122}
123
124static int read_fd_into_buffer(int fd, char *buf, int size)
125{
126 int bufleft = size;
127 int len;
128
129 do {
130 len = read(fd, buf, bufleft);
131 if (len > 0) {
132 buf += len;
133 bufleft -= len;
134 }
135 } while (len > 0);
136
137 return len < 0 ? len : size - bufleft;
138}
139
140static void test_ipv6_route(void)
141{
142 struct bpf_iter_ipv6_route *skel;
143
144 skel = bpf_iter_ipv6_route__open_and_load();
145 if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
146 return;
147
148 do_dummy_read(skel->progs.dump_ipv6_route);
149
150 bpf_iter_ipv6_route__destroy(skel);
151}
152
153static void test_netlink(void)
154{
155 struct bpf_iter_netlink *skel;
156
157 skel = bpf_iter_netlink__open_and_load();
158 if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
159 return;
160
161 do_dummy_read(skel->progs.dump_netlink);
162
163 bpf_iter_netlink__destroy(skel);
164}
165
166static void test_bpf_map(void)
167{
168 struct bpf_iter_bpf_map *skel;
169
170 skel = bpf_iter_bpf_map__open_and_load();
171 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
172 return;
173
174 do_dummy_read(skel->progs.dump_bpf_map);
175
176 bpf_iter_bpf_map__destroy(skel);
177}
178
179static void check_bpf_link_info(const struct bpf_program *prog)
180{
181 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
182 union bpf_iter_link_info linfo;
183 struct bpf_link_info info = {};
184 struct bpf_link *link;
185 __u32 info_len;
186 int err;
187
188 memset(&linfo, 0, sizeof(linfo));
189 linfo.task.tid = getpid();
190 opts.link_info = &linfo;
191 opts.link_info_len = sizeof(linfo);
192
193 link = bpf_program__attach_iter(prog, &opts);
194 if (!ASSERT_OK_PTR(link, "attach_iter"))
195 return;
196
197 info_len = sizeof(info);
198 err = bpf_obj_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
199 ASSERT_OK(err, "bpf_obj_get_info_by_fd");
200 ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
201
202 bpf_link__destroy(link);
203}
204
205static pthread_mutex_t do_nothing_mutex;
206
207static void *do_nothing_wait(void *arg)
208{
209 pthread_mutex_lock(&do_nothing_mutex);
210 pthread_mutex_unlock(&do_nothing_mutex);
211
212 pthread_exit(arg);
213}
214
215static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
216 int *num_unknown, int *num_known)
217{
218 struct bpf_iter_task *skel;
219 pthread_t thread_id;
220 void *ret;
221
222 skel = bpf_iter_task__open_and_load();
223 if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load"))
224 return;
225
226 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
227
228 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
229 "pthread_create");
230
231 skel->bss->tid = getpid();
232
233 do_dummy_read_opts(skel->progs.dump_task, opts);
234
235 *num_unknown = skel->bss->num_unknown_tid;
236 *num_known = skel->bss->num_known_tid;
237
238 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
239 ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
240 "pthread_join");
241
242 bpf_iter_task__destroy(skel);
243}
244
245static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
246{
247 int num_unknown_tid, num_known_tid;
248
249 test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
250 ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
251 ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
252}
253
254static void test_task_tid(void)
255{
256 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
257 union bpf_iter_link_info linfo;
258 int num_unknown_tid, num_known_tid;
259
260 memset(&linfo, 0, sizeof(linfo));
261 linfo.task.tid = getpid();
262 opts.link_info = &linfo;
263 opts.link_info_len = sizeof(linfo);
264 test_task_common(&opts, 0, 1);
265
266 linfo.task.tid = 0;
267 linfo.task.pid = getpid();
268 test_task_common(&opts, 1, 1);
269
270 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
271 ASSERT_GT(num_unknown_tid, 1, "check_num_unknown_tid");
272 ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
273}
274
275static void test_task_pid(void)
276{
277 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
278 union bpf_iter_link_info linfo;
279
280 memset(&linfo, 0, sizeof(linfo));
281 linfo.task.pid = getpid();
282 opts.link_info = &linfo;
283 opts.link_info_len = sizeof(linfo);
284
285 test_task_common(&opts, 1, 1);
286}
287
288static void test_task_pidfd(void)
289{
290 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
291 union bpf_iter_link_info linfo;
292 int pidfd;
293
294 pidfd = sys_pidfd_open(getpid(), 0);
295 if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
296 return;
297
298 memset(&linfo, 0, sizeof(linfo));
299 linfo.task.pid_fd = pidfd;
300 opts.link_info = &linfo;
301 opts.link_info_len = sizeof(linfo);
302
303 test_task_common(&opts, 1, 1);
304
305 close(pidfd);
306}
307
308static void test_task_sleepable(void)
309{
310 struct bpf_iter_task *skel;
311
312 skel = bpf_iter_task__open_and_load();
313 if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load"))
314 return;
315
316 do_dummy_read(skel->progs.dump_task_sleepable);
317
318 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
319 "num_expected_failure_copy_from_user_task");
320 ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
321 "num_success_copy_from_user_task");
322
323 bpf_iter_task__destroy(skel);
324}
325
326static void test_task_stack(void)
327{
328 struct bpf_iter_task_stack *skel;
329
330 skel = bpf_iter_task_stack__open_and_load();
331 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
332 return;
333
334 do_dummy_read(skel->progs.dump_task_stack);
335 do_dummy_read(skel->progs.get_task_user_stacks);
336
337 bpf_iter_task_stack__destroy(skel);
338}
339
340static void test_task_file(void)
341{
342 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
343 struct bpf_iter_task_file *skel;
344 union bpf_iter_link_info linfo;
345 pthread_t thread_id;
346 void *ret;
347
348 skel = bpf_iter_task_file__open_and_load();
349 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
350 return;
351
352 skel->bss->tgid = getpid();
353
354 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
355
356 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
357 "pthread_create");
358
359 memset(&linfo, 0, sizeof(linfo));
360 linfo.task.tid = getpid();
361 opts.link_info = &linfo;
362 opts.link_info_len = sizeof(linfo);
363
364 do_dummy_read_opts(skel->progs.dump_task_file, &opts);
365
366 ASSERT_EQ(skel->bss->count, 0, "check_count");
367 ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
368
369 skel->bss->last_tgid = 0;
370 skel->bss->count = 0;
371 skel->bss->unique_tgid_count = 0;
372
373 do_dummy_read(skel->progs.dump_task_file);
374
375 ASSERT_EQ(skel->bss->count, 0, "check_count");
376 ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
377
378 check_bpf_link_info(skel->progs.dump_task_file);
379
380 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
381 ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
382 ASSERT_NULL(ret, "pthread_join");
383
384 bpf_iter_task_file__destroy(skel);
385}
386
387#define TASKBUFSZ 32768
388
389static char taskbuf[TASKBUFSZ];
390
391static int do_btf_read(struct bpf_iter_task_btf *skel)
392{
393 struct bpf_program *prog = skel->progs.dump_task_struct;
394 struct bpf_iter_task_btf__bss *bss = skel->bss;
395 int iter_fd = -1, err;
396 struct bpf_link *link;
397 char *buf = taskbuf;
398 int ret = 0;
399
400 link = bpf_program__attach_iter(prog, NULL);
401 if (!ASSERT_OK_PTR(link, "attach_iter"))
402 return ret;
403
404 iter_fd = bpf_iter_create(bpf_link__fd(link));
405 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
406 goto free_link;
407
408 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
409 if (bss->skip) {
410 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
411 ret = 1;
412 test__skip();
413 goto free_link;
414 }
415
416 if (CHECK(err < 0, "read", "read failed: %s\n", strerror(errno)))
417 goto free_link;
418
419 ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
420 "check for btf representation of task_struct in iter data");
421free_link:
422 if (iter_fd > 0)
423 close(iter_fd);
424 bpf_link__destroy(link);
425 return ret;
426}
427
428static void test_task_btf(void)
429{
430 struct bpf_iter_task_btf__bss *bss;
431 struct bpf_iter_task_btf *skel;
432 int ret;
433
434 skel = bpf_iter_task_btf__open_and_load();
435 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
436 return;
437
438 bss = skel->bss;
439
440 ret = do_btf_read(skel);
441 if (ret)
442 goto cleanup;
443
444 if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
445 goto cleanup;
446
447 ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");
448
449cleanup:
450 bpf_iter_task_btf__destroy(skel);
451}
452
453static void test_tcp4(void)
454{
455 struct bpf_iter_tcp4 *skel;
456
457 skel = bpf_iter_tcp4__open_and_load();
458 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
459 return;
460
461 do_dummy_read(skel->progs.dump_tcp4);
462
463 bpf_iter_tcp4__destroy(skel);
464}
465
466static void test_tcp6(void)
467{
468 struct bpf_iter_tcp6 *skel;
469
470 skel = bpf_iter_tcp6__open_and_load();
471 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
472 return;
473
474 do_dummy_read(skel->progs.dump_tcp6);
475
476 bpf_iter_tcp6__destroy(skel);
477}
478
479static void test_udp4(void)
480{
481 struct bpf_iter_udp4 *skel;
482
483 skel = bpf_iter_udp4__open_and_load();
484 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
485 return;
486
487 do_dummy_read(skel->progs.dump_udp4);
488
489 bpf_iter_udp4__destroy(skel);
490}
491
492static void test_udp6(void)
493{
494 struct bpf_iter_udp6 *skel;
495
496 skel = bpf_iter_udp6__open_and_load();
497 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
498 return;
499
500 do_dummy_read(skel->progs.dump_udp6);
501
502 bpf_iter_udp6__destroy(skel);
503}
504
505static void test_unix(void)
506{
507 struct bpf_iter_unix *skel;
508
509 skel = bpf_iter_unix__open_and_load();
510 if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
511 return;
512
513 do_dummy_read(skel->progs.dump_unix);
514
515 bpf_iter_unix__destroy(skel);
516}
517
518/* The expected string is less than 16 bytes */
519static int do_read_with_fd(int iter_fd, const char *expected,
520 bool read_one_char)
521{
522 int len, read_buf_len, start;
523 char buf[16] = {};
524
525 read_buf_len = read_one_char ? 1 : 16;
526 start = 0;
527 while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
528 start += len;
529 if (CHECK(start >= 16, "read", "read len %d\n", len))
530 return -1;
531 read_buf_len = read_one_char ? 1 : 16 - start;
532 }
533 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
534 return -1;
535
536 if (!ASSERT_STREQ(buf, expected, "read"))
537 return -1;
538
539 return 0;
540}
541
542static void test_anon_iter(bool read_one_char)
543{
544 struct bpf_iter_test_kern1 *skel;
545 struct bpf_link *link;
546 int iter_fd, err;
547
548 skel = bpf_iter_test_kern1__open_and_load();
549 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
550 return;
551
552 err = bpf_iter_test_kern1__attach(skel);
553 if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
554 goto out;
555 }
556
557 link = skel->links.dump_task;
558 iter_fd = bpf_iter_create(bpf_link__fd(link));
559 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
560 goto out;
561
562 do_read_with_fd(iter_fd, "abcd", read_one_char);
563 close(iter_fd);
564
565out:
566 bpf_iter_test_kern1__destroy(skel);
567}
568
569static int do_read(const char *path, const char *expected)
570{
571 int err, iter_fd;
572
573 iter_fd = open(path, O_RDONLY);
574 if (CHECK(iter_fd < 0, "open", "open %s failed: %s\n",
575 path, strerror(errno)))
576 return -1;
577
578 err = do_read_with_fd(iter_fd, expected, false);
579 close(iter_fd);
580 return err;
581}
582
583static void test_file_iter(void)
584{
585 const char *path = "/sys/fs/bpf/bpf_iter_test1";
586 struct bpf_iter_test_kern1 *skel1;
587 struct bpf_iter_test_kern2 *skel2;
588 struct bpf_link *link;
589 int err;
590
591 skel1 = bpf_iter_test_kern1__open_and_load();
592 if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
593 return;
594
595 link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
596 if (!ASSERT_OK_PTR(link, "attach_iter"))
597 goto out;
598
599 /* unlink this path if it exists. */
600 unlink(path);
601
602 err = bpf_link__pin(link, path);
603 if (CHECK(err, "pin_iter", "pin_iter to %s failed: %d\n", path, err))
604 goto free_link;
605
606 err = do_read(path, "abcd");
607 if (err)
608 goto unlink_path;
609
610 /* file based iterator seems working fine. Let us a link update
611 * of the underlying link and `cat` the iterator again, its content
612 * should change.
613 */
614 skel2 = bpf_iter_test_kern2__open_and_load();
615 if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
616 goto unlink_path;
617
618 err = bpf_link__update_program(link, skel2->progs.dump_task);
619 if (!ASSERT_OK(err, "update_prog"))
620 goto destroy_skel2;
621
622 do_read(path, "ABCD");
623
624destroy_skel2:
625 bpf_iter_test_kern2__destroy(skel2);
626unlink_path:
627 unlink(path);
628free_link:
629 bpf_link__destroy(link);
630out:
631 bpf_iter_test_kern1__destroy(skel1);
632}
633
634static void test_overflow(bool test_e2big_overflow, bool ret1)
635{
636 __u32 map_info_len, total_read_len, expected_read_len;
637 int err, iter_fd, map1_fd, map2_fd, len;
638 struct bpf_map_info map_info = {};
639 struct bpf_iter_test_kern4 *skel;
640 struct bpf_link *link;
641 __u32 iter_size;
642 char *buf;
643
644 skel = bpf_iter_test_kern4__open();
645 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
646 return;
647
648 /* create two maps: bpf program will only do bpf_seq_write
649 * for these two maps. The goal is one map output almost
650 * fills seq_file buffer and then the other will trigger
651 * overflow and needs restart.
652 */
653 map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
654 if (CHECK(map1_fd < 0, "bpf_map_create",
655 "map_creation failed: %s\n", strerror(errno)))
656 goto out;
657 map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
658 if (CHECK(map2_fd < 0, "bpf_map_create",
659 "map_creation failed: %s\n", strerror(errno)))
660 goto free_map1;
661
662 /* bpf_seq_printf kernel buffer is 8 pages, so one map
663 * bpf_seq_write will mostly fill it, and the other map
664 * will partially fill and then trigger overflow and need
665 * bpf_seq_read restart.
666 */
667 iter_size = sysconf(_SC_PAGE_SIZE) << 3;
668
669 if (test_e2big_overflow) {
670 skel->rodata->print_len = (iter_size + 8) / 8;
671 expected_read_len = 2 * (iter_size + 8);
672 } else if (!ret1) {
673 skel->rodata->print_len = (iter_size - 8) / 8;
674 expected_read_len = 2 * (iter_size - 8);
675 } else {
676 skel->rodata->print_len = 1;
677 expected_read_len = 2 * 8;
678 }
679 skel->rodata->ret1 = ret1;
680
681 if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
682 "bpf_iter_test_kern4__load"))
683 goto free_map2;
684
685 /* setup filtering map_id in bpf program */
686 map_info_len = sizeof(map_info);
687 err = bpf_obj_get_info_by_fd(map1_fd, &map_info, &map_info_len);
688 if (CHECK(err, "get_map_info", "get map info failed: %s\n",
689 strerror(errno)))
690 goto free_map2;
691 skel->bss->map1_id = map_info.id;
692
693 err = bpf_obj_get_info_by_fd(map2_fd, &map_info, &map_info_len);
694 if (CHECK(err, "get_map_info", "get map info failed: %s\n",
695 strerror(errno)))
696 goto free_map2;
697 skel->bss->map2_id = map_info.id;
698
699 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
700 if (!ASSERT_OK_PTR(link, "attach_iter"))
701 goto free_map2;
702
703 iter_fd = bpf_iter_create(bpf_link__fd(link));
704 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
705 goto free_link;
706
707 buf = malloc(expected_read_len);
708 if (!buf)
709 goto close_iter;
710
711 /* do read */
712 total_read_len = 0;
713 if (test_e2big_overflow) {
714 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
715 total_read_len += len;
716
717 CHECK(len != -1 || errno != E2BIG, "read",
718 "expected ret -1, errno E2BIG, but get ret %d, error %s\n",
719 len, strerror(errno));
720 goto free_buf;
721 } else if (!ret1) {
722 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
723 total_read_len += len;
724
725 if (CHECK(len < 0, "read", "read failed: %s\n",
726 strerror(errno)))
727 goto free_buf;
728 } else {
729 do {
730 len = read(iter_fd, buf, expected_read_len);
731 if (len > 0)
732 total_read_len += len;
733 } while (len > 0 || len == -EAGAIN);
734
735 if (CHECK(len < 0, "read", "read failed: %s\n",
736 strerror(errno)))
737 goto free_buf;
738 }
739
740 if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
741 goto free_buf;
742
743 if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
744 goto free_buf;
745
746 if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
747 goto free_buf;
748
749 ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");
750
751free_buf:
752 free(buf);
753close_iter:
754 close(iter_fd);
755free_link:
756 bpf_link__destroy(link);
757free_map2:
758 close(map2_fd);
759free_map1:
760 close(map1_fd);
761out:
762 bpf_iter_test_kern4__destroy(skel);
763}
764
765static void test_bpf_hash_map(void)
766{
767 __u32 expected_key_a = 0, expected_key_b = 0;
768 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
769 struct bpf_iter_bpf_hash_map *skel;
770 int err, i, len, map_fd, iter_fd;
771 union bpf_iter_link_info linfo;
772 __u64 val, expected_val = 0;
773 struct bpf_link *link;
774 struct key_t {
775 int a;
776 int b;
777 int c;
778 } key;
779 char buf[64];
780
781 skel = bpf_iter_bpf_hash_map__open();
782 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
783 return;
784
785 skel->bss->in_test_mode = true;
786
787 err = bpf_iter_bpf_hash_map__load(skel);
788 if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
789 goto out;
790
791 /* iterator with hashmap2 and hashmap3 should fail */
792 memset(&linfo, 0, sizeof(linfo));
793 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
794 opts.link_info = &linfo;
795 opts.link_info_len = sizeof(linfo);
796 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
797 if (!ASSERT_ERR_PTR(link, "attach_iter"))
798 goto out;
799
800 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
801 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
802 if (!ASSERT_ERR_PTR(link, "attach_iter"))
803 goto out;
804
805 /* hashmap1 should be good, update map values here */
806 map_fd = bpf_map__fd(skel->maps.hashmap1);
807 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
808 key.a = i + 1;
809 key.b = i + 2;
810 key.c = i + 3;
811 val = i + 4;
812 expected_key_a += key.a;
813 expected_key_b += key.b;
814 expected_val += val;
815
816 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
817 if (!ASSERT_OK(err, "map_update"))
818 goto out;
819 }
820
821 /* Sleepable program is prohibited for hash map iterator */
822 linfo.map.map_fd = map_fd;
823 link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
824 if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
825 goto out;
826
827 linfo.map.map_fd = map_fd;
828 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
829 if (!ASSERT_OK_PTR(link, "attach_iter"))
830 goto out;
831
832 iter_fd = bpf_iter_create(bpf_link__fd(link));
833 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
834 goto free_link;
835
836 /* do some tests */
837 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
838 ;
839 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
840 goto close_iter;
841
842 /* test results */
843 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
844 goto close_iter;
845 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
846 goto close_iter;
847 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
848 goto close_iter;
849
850close_iter:
851 close(iter_fd);
852free_link:
853 bpf_link__destroy(link);
854out:
855 bpf_iter_bpf_hash_map__destroy(skel);
856}
857
858static void test_bpf_percpu_hash_map(void)
859{
860 __u32 expected_key_a = 0, expected_key_b = 0;
861 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
862 struct bpf_iter_bpf_percpu_hash_map *skel;
863 int err, i, j, len, map_fd, iter_fd;
864 union bpf_iter_link_info linfo;
865 __u32 expected_val = 0;
866 struct bpf_link *link;
867 struct key_t {
868 int a;
869 int b;
870 int c;
871 } key;
872 char buf[64];
873 void *val;
874
875 skel = bpf_iter_bpf_percpu_hash_map__open();
876 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
877 return;
878
879 skel->rodata->num_cpus = bpf_num_possible_cpus();
880 val = malloc(8 * bpf_num_possible_cpus());
881
882 err = bpf_iter_bpf_percpu_hash_map__load(skel);
883 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
884 goto out;
885
886 /* update map values here */
887 map_fd = bpf_map__fd(skel->maps.hashmap1);
888 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
889 key.a = i + 1;
890 key.b = i + 2;
891 key.c = i + 3;
892 expected_key_a += key.a;
893 expected_key_b += key.b;
894
895 for (j = 0; j < bpf_num_possible_cpus(); j++) {
896 *(__u32 *)(val + j * 8) = i + j;
897 expected_val += i + j;
898 }
899
900 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
901 if (!ASSERT_OK(err, "map_update"))
902 goto out;
903 }
904
905 memset(&linfo, 0, sizeof(linfo));
906 linfo.map.map_fd = map_fd;
907 opts.link_info = &linfo;
908 opts.link_info_len = sizeof(linfo);
909 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
910 if (!ASSERT_OK_PTR(link, "attach_iter"))
911 goto out;
912
913 iter_fd = bpf_iter_create(bpf_link__fd(link));
914 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
915 goto free_link;
916
917 /* do some tests */
918 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
919 ;
920 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
921 goto close_iter;
922
923 /* test results */
924 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
925 goto close_iter;
926 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
927 goto close_iter;
928 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
929 goto close_iter;
930
931close_iter:
932 close(iter_fd);
933free_link:
934 bpf_link__destroy(link);
935out:
936 bpf_iter_bpf_percpu_hash_map__destroy(skel);
937 free(val);
938}
939
940static void test_bpf_array_map(void)
941{
942 __u64 val, expected_val = 0, res_first_val, first_val = 0;
943 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
944 __u32 key, expected_key = 0, res_first_key;
945 int err, i, map_fd, hash_fd, iter_fd;
946 struct bpf_iter_bpf_array_map *skel;
947 union bpf_iter_link_info linfo;
948 struct bpf_link *link;
949 char buf[64] = {};
950 int len, start;
951
952 skel = bpf_iter_bpf_array_map__open_and_load();
953 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
954 return;
955
956 map_fd = bpf_map__fd(skel->maps.arraymap1);
957 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
958 val = i + 4;
959 expected_key += i;
960 expected_val += val;
961
962 if (i == 0)
963 first_val = val;
964
965 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
966 if (!ASSERT_OK(err, "map_update"))
967 goto out;
968 }
969
970 memset(&linfo, 0, sizeof(linfo));
971 linfo.map.map_fd = map_fd;
972 opts.link_info = &linfo;
973 opts.link_info_len = sizeof(linfo);
974 link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
975 if (!ASSERT_OK_PTR(link, "attach_iter"))
976 goto out;
977
978 iter_fd = bpf_iter_create(bpf_link__fd(link));
979 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
980 goto free_link;
981
982 /* do some tests */
983 start = 0;
984 while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
985 start += len;
986 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
987 goto close_iter;
988
989 /* test results */
990 res_first_key = *(__u32 *)buf;
991 res_first_val = *(__u64 *)(buf + sizeof(__u32));
992 if (CHECK(res_first_key != 0 || res_first_val != first_val,
993 "bpf_seq_write",
994 "seq_write failure: first key %u vs expected 0, "
995 " first value %llu vs expected %llu\n",
996 res_first_key, res_first_val, first_val))
997 goto close_iter;
998
999 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1000 goto close_iter;
1001 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1002 goto close_iter;
1003
1004 hash_fd = bpf_map__fd(skel->maps.hashmap1);
1005 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1006 err = bpf_map_lookup_elem(map_fd, &i, &val);
1007 if (!ASSERT_OK(err, "map_lookup arraymap1"))
1008 goto close_iter;
1009 if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
1010 goto close_iter;
1011
1012 val = i + 4;
1013 err = bpf_map_lookup_elem(hash_fd, &val, &key);
1014 if (!ASSERT_OK(err, "map_lookup hashmap1"))
1015 goto close_iter;
1016 if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
1017 goto close_iter;
1018 }
1019
1020close_iter:
1021 close(iter_fd);
1022free_link:
1023 bpf_link__destroy(link);
1024out:
1025 bpf_iter_bpf_array_map__destroy(skel);
1026}
1027
1028static void test_bpf_array_map_iter_fd(void)
1029{
1030 struct bpf_iter_bpf_array_map *skel;
1031
1032 skel = bpf_iter_bpf_array_map__open_and_load();
1033 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1034 return;
1035
1036 do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,
1037 skel->maps.arraymap1);
1038
1039 bpf_iter_bpf_array_map__destroy(skel);
1040}
1041
1042static void test_bpf_percpu_array_map(void)
1043{
1044 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1045 struct bpf_iter_bpf_percpu_array_map *skel;
1046 __u32 expected_key = 0, expected_val = 0;
1047 union bpf_iter_link_info linfo;
1048 int err, i, j, map_fd, iter_fd;
1049 struct bpf_link *link;
1050 char buf[64];
1051 void *val;
1052 int len;
1053
1054 skel = bpf_iter_bpf_percpu_array_map__open();
1055 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
1056 return;
1057
1058 skel->rodata->num_cpus = bpf_num_possible_cpus();
1059 val = malloc(8 * bpf_num_possible_cpus());
1060
1061 err = bpf_iter_bpf_percpu_array_map__load(skel);
1062 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
1063 goto out;
1064
1065 /* update map values here */
1066 map_fd = bpf_map__fd(skel->maps.arraymap1);
1067 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1068 expected_key += i;
1069
1070 for (j = 0; j < bpf_num_possible_cpus(); j++) {
1071 *(__u32 *)(val + j * 8) = i + j;
1072 expected_val += i + j;
1073 }
1074
1075 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
1076 if (!ASSERT_OK(err, "map_update"))
1077 goto out;
1078 }
1079
1080 memset(&linfo, 0, sizeof(linfo));
1081 linfo.map.map_fd = map_fd;
1082 opts.link_info = &linfo;
1083 opts.link_info_len = sizeof(linfo);
1084 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
1085 if (!ASSERT_OK_PTR(link, "attach_iter"))
1086 goto out;
1087
1088 iter_fd = bpf_iter_create(bpf_link__fd(link));
1089 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1090 goto free_link;
1091
1092 /* do some tests */
1093 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1094 ;
1095 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
1096 goto close_iter;
1097
1098 /* test results */
1099 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1100 goto close_iter;
1101 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1102 goto close_iter;
1103
1104close_iter:
1105 close(iter_fd);
1106free_link:
1107 bpf_link__destroy(link);
1108out:
1109 bpf_iter_bpf_percpu_array_map__destroy(skel);
1110 free(val);
1111}
1112
1113/* An iterator program deletes all local storage in a map. */
1114static void test_bpf_sk_storage_delete(void)
1115{
1116 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1117 struct bpf_iter_bpf_sk_storage_helpers *skel;
1118 union bpf_iter_link_info linfo;
1119 int err, len, map_fd, iter_fd;
1120 struct bpf_link *link;
1121 int sock_fd = -1;
1122 __u32 val = 42;
1123 char buf[64];
1124
1125 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1126 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1127 return;
1128
1129 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1130
1131 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1132 if (!ASSERT_GE(sock_fd, 0, "socket"))
1133 goto out;
1134 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1135 if (!ASSERT_OK(err, "map_update"))
1136 goto out;
1137
1138 memset(&linfo, 0, sizeof(linfo));
1139 linfo.map.map_fd = map_fd;
1140 opts.link_info = &linfo;
1141 opts.link_info_len = sizeof(linfo);
1142 link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
1143 &opts);
1144 if (!ASSERT_OK_PTR(link, "attach_iter"))
1145 goto out;
1146
1147 iter_fd = bpf_iter_create(bpf_link__fd(link));
1148 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1149 goto free_link;
1150
1151 /* do some tests */
1152 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1153 ;
1154 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
1155 goto close_iter;
1156
1157 /* test results */
1158 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1159 if (CHECK(!err || errno != ENOENT, "bpf_map_lookup_elem",
1160 "map value wasn't deleted (err=%d, errno=%d)\n", err, errno))
1161 goto close_iter;
1162
1163close_iter:
1164 close(iter_fd);
1165free_link:
1166 bpf_link__destroy(link);
1167out:
1168 if (sock_fd >= 0)
1169 close(sock_fd);
1170 bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1171}
1172
1173/* This creates a socket and its local storage. It then runs a task_iter BPF
1174 * program that replaces the existing socket local storage with the tgid of the
1175 * only task owning a file descriptor to this socket, this process, prog_tests.
1176 * It then runs a tcp socket iterator that negates the value in the existing
1177 * socket local storage, the test verifies that the resulting value is -pid.
1178 */
1179static void test_bpf_sk_storage_get(void)
1180{
1181 struct bpf_iter_bpf_sk_storage_helpers *skel;
1182 int err, map_fd, val = -1;
1183 int sock_fd = -1;
1184
1185 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1186 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1187 return;
1188
1189 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1190 if (!ASSERT_GE(sock_fd, 0, "socket"))
1191 goto out;
1192
1193 err = listen(sock_fd, 1);
1194 if (!ASSERT_OK(err, "listen"))
1195 goto close_socket;
1196
1197 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1198
1199 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1200 if (!ASSERT_OK(err, "bpf_map_update_elem"))
1201 goto close_socket;
1202
1203 do_dummy_read(skel->progs.fill_socket_owner);
1204
1205 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1206 if (CHECK(err || val != getpid(), "bpf_map_lookup_elem",
1207 "map value wasn't set correctly (expected %d, got %d, err=%d)\n",
1208 getpid(), val, err))
1209 goto close_socket;
1210
1211 do_dummy_read(skel->progs.negate_socket_local_storage);
1212
1213 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1214 CHECK(err || val != -getpid(), "bpf_map_lookup_elem",
1215 "map value wasn't set correctly (expected %d, got %d, err=%d)\n",
1216 -getpid(), val, err);
1217
1218close_socket:
1219 close(sock_fd);
1220out:
1221 bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1222}
1223
1224static void test_bpf_sk_stoarge_map_iter_fd(void)
1225{
1226 struct bpf_iter_bpf_sk_storage_map *skel;
1227
1228 skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1229 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1230 return;
1231
1232 do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,
1233 skel->maps.sk_stg_map);
1234
1235 bpf_iter_bpf_sk_storage_map__destroy(skel);
1236}
1237
1238static void test_bpf_sk_storage_map(void)
1239{
1240 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1241 int err, i, len, map_fd, iter_fd, num_sockets;
1242 struct bpf_iter_bpf_sk_storage_map *skel;
1243 union bpf_iter_link_info linfo;
1244 int sock_fd[3] = {-1, -1, -1};
1245 __u32 val, expected_val = 0;
1246 struct bpf_link *link;
1247 char buf[64];
1248
1249 skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1250 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1251 return;
1252
1253 map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1254 num_sockets = ARRAY_SIZE(sock_fd);
1255 for (i = 0; i < num_sockets; i++) {
1256 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
1257 if (!ASSERT_GE(sock_fd[i], 0, "socket"))
1258 goto out;
1259
1260 val = i + 1;
1261 expected_val += val;
1262
1263 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
1264 BPF_NOEXIST);
1265 if (!ASSERT_OK(err, "map_update"))
1266 goto out;
1267 }
1268
1269 memset(&linfo, 0, sizeof(linfo));
1270 linfo.map.map_fd = map_fd;
1271 opts.link_info = &linfo;
1272 opts.link_info_len = sizeof(linfo);
1273 link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
1274 err = libbpf_get_error(link);
1275 if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
1276 if (!err)
1277 bpf_link__destroy(link);
1278 goto out;
1279 }
1280
1281 link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
1282 if (!ASSERT_OK_PTR(link, "attach_iter"))
1283 goto out;
1284
1285 iter_fd = bpf_iter_create(bpf_link__fd(link));
1286 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1287 goto free_link;
1288
1289 skel->bss->to_add_val = time(NULL);
1290 /* do some tests */
1291 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1292 ;
1293 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
1294 goto close_iter;
1295
1296 /* test results */
1297 if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
1298 goto close_iter;
1299
1300 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1301 goto close_iter;
1302
1303 for (i = 0; i < num_sockets; i++) {
1304 err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
1305 if (!ASSERT_OK(err, "map_lookup") ||
1306 !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
1307 break;
1308 }
1309
1310close_iter:
1311 close(iter_fd);
1312free_link:
1313 bpf_link__destroy(link);
1314out:
1315 for (i = 0; i < num_sockets; i++) {
1316 if (sock_fd[i] >= 0)
1317 close(sock_fd[i]);
1318 }
1319 bpf_iter_bpf_sk_storage_map__destroy(skel);
1320}
1321
1322static void test_rdonly_buf_out_of_bound(void)
1323{
1324 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1325 struct bpf_iter_test_kern5 *skel;
1326 union bpf_iter_link_info linfo;
1327 struct bpf_link *link;
1328
1329 skel = bpf_iter_test_kern5__open_and_load();
1330 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
1331 return;
1332
1333 memset(&linfo, 0, sizeof(linfo));
1334 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
1335 opts.link_info = &linfo;
1336 opts.link_info_len = sizeof(linfo);
1337 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
1338 if (!ASSERT_ERR_PTR(link, "attach_iter"))
1339 bpf_link__destroy(link);
1340
1341 bpf_iter_test_kern5__destroy(skel);
1342}
1343
1344static void test_buf_neg_offset(void)
1345{
1346 struct bpf_iter_test_kern6 *skel;
1347
1348 skel = bpf_iter_test_kern6__open_and_load();
1349 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
1350 bpf_iter_test_kern6__destroy(skel);
1351}
1352
1353static void test_link_iter(void)
1354{
1355 struct bpf_iter_bpf_link *skel;
1356
1357 skel = bpf_iter_bpf_link__open_and_load();
1358 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
1359 return;
1360
1361 do_dummy_read(skel->progs.dump_bpf_link);
1362
1363 bpf_iter_bpf_link__destroy(skel);
1364}
1365
1366static void test_ksym_iter(void)
1367{
1368 struct bpf_iter_ksym *skel;
1369
1370 skel = bpf_iter_ksym__open_and_load();
1371 if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
1372 return;
1373
1374 do_dummy_read(skel->progs.dump_ksym);
1375
1376 bpf_iter_ksym__destroy(skel);
1377}
1378
1379#define CMP_BUFFER_SIZE 1024
1380static char task_vma_output[CMP_BUFFER_SIZE];
1381static char proc_maps_output[CMP_BUFFER_SIZE];
1382
1383/* remove \0 and \t from str, and only keep the first line */
1384static void str_strip_first_line(char *str)
1385{
1386 char *dst = str, *src = str;
1387
1388 do {
1389 if (*src == ' ' || *src == '\t')
1390 src++;
1391 else
1392 *(dst++) = *(src++);
1393
1394 } while (*src != '\0' && *src != '\n');
1395
1396 *dst = '\0';
1397}
1398
1399static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
1400{
1401 int err, iter_fd = -1, proc_maps_fd = -1;
1402 struct bpf_iter_task_vma *skel;
1403 int len, read_size = 4;
1404 char maps_path[64];
1405
1406 skel = bpf_iter_task_vma__open();
1407 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vma__open"))
1408 return;
1409
1410 skel->bss->pid = getpid();
1411 skel->bss->one_task = opts ? 1 : 0;
1412
1413 err = bpf_iter_task_vma__load(skel);
1414 if (!ASSERT_OK(err, "bpf_iter_task_vma__load"))
1415 goto out;
1416
1417 skel->links.proc_maps = bpf_program__attach_iter(
1418 skel->progs.proc_maps, opts);
1419
1420 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1421 skel->links.proc_maps = NULL;
1422 goto out;
1423 }
1424
1425 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1426 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1427 goto out;
1428
1429 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1430 * to trigger seq_file corner cases.
1431 */
1432 len = 0;
1433 while (len < CMP_BUFFER_SIZE) {
1434 err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1435 MIN(read_size, CMP_BUFFER_SIZE - len));
1436 if (!err)
1437 break;
1438 if (!ASSERT_GE(err, 0, "read_iter_fd"))
1439 goto out;
1440 len += err;
1441 }
1442 if (opts)
1443 ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
1444
1445 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1446 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1447 proc_maps_fd = open(maps_path, O_RDONLY);
1448 if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
1449 goto out;
1450 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1451 if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
1452 goto out;
1453
1454 /* strip and compare the first line of the two files */
1455 str_strip_first_line(task_vma_output);
1456 str_strip_first_line(proc_maps_output);
1457
1458 ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
1459
1460 check_bpf_link_info(skel->progs.proc_maps);
1461
1462out:
1463 close(proc_maps_fd);
1464 close(iter_fd);
1465 bpf_iter_task_vma__destroy(skel);
1466}
1467
1468static void test_task_vma_dead_task(void)
1469{
1470 struct bpf_iter_task_vma *skel;
1471 int wstatus, child_pid = -1;
1472 time_t start_tm, cur_tm;
1473 int err, iter_fd = -1;
1474 int wait_sec = 3;
1475
1476 skel = bpf_iter_task_vma__open();
1477 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vma__open"))
1478 return;
1479
1480 skel->bss->pid = getpid();
1481
1482 err = bpf_iter_task_vma__load(skel);
1483 if (!ASSERT_OK(err, "bpf_iter_task_vma__load"))
1484 goto out;
1485
1486 skel->links.proc_maps = bpf_program__attach_iter(
1487 skel->progs.proc_maps, NULL);
1488
1489 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1490 skel->links.proc_maps = NULL;
1491 goto out;
1492 }
1493
1494 start_tm = time(NULL);
1495 cur_tm = start_tm;
1496
1497 child_pid = fork();
1498 if (child_pid == 0) {
1499 /* Fork short-lived processes in the background. */
1500 while (cur_tm < start_tm + wait_sec) {
1501 system("echo > /dev/null");
1502 cur_tm = time(NULL);
1503 }
1504 exit(0);
1505 }
1506
1507 if (!ASSERT_GE(child_pid, 0, "fork_child"))
1508 goto out;
1509
1510 while (cur_tm < start_tm + wait_sec) {
1511 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1512 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1513 goto out;
1514
1515 /* Drain all data from iter_fd. */
1516 while (cur_tm < start_tm + wait_sec) {
1517 err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);
1518 if (!ASSERT_GE(err, 0, "read_iter_fd"))
1519 goto out;
1520
1521 cur_tm = time(NULL);
1522
1523 if (err == 0)
1524 break;
1525 }
1526
1527 close(iter_fd);
1528 iter_fd = -1;
1529 }
1530
1531 check_bpf_link_info(skel->progs.proc_maps);
1532
1533out:
1534 waitpid(child_pid, &wstatus, 0);
1535 close(iter_fd);
1536 bpf_iter_task_vma__destroy(skel);
1537}
1538
1539void test_bpf_sockmap_map_iter_fd(void)
1540{
1541 struct bpf_iter_sockmap *skel;
1542
1543 skel = bpf_iter_sockmap__open_and_load();
1544 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
1545 return;
1546
1547 do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);
1548
1549 bpf_iter_sockmap__destroy(skel);
1550}
1551
1552static void test_task_vma(void)
1553{
1554 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1555 union bpf_iter_link_info linfo;
1556
1557 memset(&linfo, 0, sizeof(linfo));
1558 linfo.task.tid = getpid();
1559 opts.link_info = &linfo;
1560 opts.link_info_len = sizeof(linfo);
1561
1562 test_task_vma_common(&opts);
1563 test_task_vma_common(NULL);
1564}
1565
1566/* uprobe attach point */
1567static noinline int trigger_func(int arg)
1568{
1569 asm volatile ("");
1570 return arg + 1;
1571}
1572
1573static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
1574{
1575 struct bpf_iter_vma_offset *skel;
1576 char buf[16] = {};
1577 int iter_fd, len;
1578 int pgsz, shift;
1579
1580 skel = bpf_iter_vma_offset__open_and_load();
1581 if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
1582 return;
1583
1584 skel->bss->pid = getpid();
1585 skel->bss->address = (uintptr_t)trigger_func;
1586 for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
1587 ;
1588 skel->bss->page_shift = shift;
1589
1590 skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
1591 if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
1592 goto exit;
1593
1594 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
1595 if (!ASSERT_GT(iter_fd, 0, "create_iter"))
1596 goto exit;
1597
1598 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1599 ;
1600 buf[15] = 0;
1601 ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
1602
1603 ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
1604 if (one_proc)
1605 ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1606 else
1607 ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1608
1609 close(iter_fd);
1610
1611exit:
1612 bpf_iter_vma_offset__destroy(skel);
1613}
1614
1615static void test_task_vma_offset(void)
1616{
1617 LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1618 union bpf_iter_link_info linfo;
1619
1620 memset(&linfo, 0, sizeof(linfo));
1621 linfo.task.pid = getpid();
1622 opts.link_info = &linfo;
1623 opts.link_info_len = sizeof(linfo);
1624
1625 test_task_vma_offset_common(&opts, true);
1626
1627 linfo.task.pid = 0;
1628 linfo.task.tid = getpid();
1629 test_task_vma_offset_common(&opts, true);
1630
1631 test_task_vma_offset_common(NULL, false);
1632}
1633
1634void test_bpf_iter(void)
1635{
1636 ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
1637
1638 if (test__start_subtest("btf_id_or_null"))
1639 test_btf_id_or_null();
1640 if (test__start_subtest("ipv6_route"))
1641 test_ipv6_route();
1642 if (test__start_subtest("netlink"))
1643 test_netlink();
1644 if (test__start_subtest("bpf_map"))
1645 test_bpf_map();
1646 if (test__start_subtest("task_tid"))
1647 test_task_tid();
1648 if (test__start_subtest("task_pid"))
1649 test_task_pid();
1650 if (test__start_subtest("task_pidfd"))
1651 test_task_pidfd();
1652 if (test__start_subtest("task_sleepable"))
1653 test_task_sleepable();
1654 if (test__start_subtest("task_stack"))
1655 test_task_stack();
1656 if (test__start_subtest("task_file"))
1657 test_task_file();
1658 if (test__start_subtest("task_vma"))
1659 test_task_vma();
1660 if (test__start_subtest("task_vma_dead_task"))
1661 test_task_vma_dead_task();
1662 if (test__start_subtest("task_btf"))
1663 test_task_btf();
1664 if (test__start_subtest("tcp4"))
1665 test_tcp4();
1666 if (test__start_subtest("tcp6"))
1667 test_tcp6();
1668 if (test__start_subtest("udp4"))
1669 test_udp4();
1670 if (test__start_subtest("udp6"))
1671 test_udp6();
1672 if (test__start_subtest("unix"))
1673 test_unix();
1674 if (test__start_subtest("anon"))
1675 test_anon_iter(false);
1676 if (test__start_subtest("anon-read-one-char"))
1677 test_anon_iter(true);
1678 if (test__start_subtest("file"))
1679 test_file_iter();
1680 if (test__start_subtest("overflow"))
1681 test_overflow(false, false);
1682 if (test__start_subtest("overflow-e2big"))
1683 test_overflow(true, false);
1684 if (test__start_subtest("prog-ret-1"))
1685 test_overflow(false, true);
1686 if (test__start_subtest("bpf_hash_map"))
1687 test_bpf_hash_map();
1688 if (test__start_subtest("bpf_percpu_hash_map"))
1689 test_bpf_percpu_hash_map();
1690 if (test__start_subtest("bpf_array_map"))
1691 test_bpf_array_map();
1692 if (test__start_subtest("bpf_array_map_iter_fd"))
1693 test_bpf_array_map_iter_fd();
1694 if (test__start_subtest("bpf_percpu_array_map"))
1695 test_bpf_percpu_array_map();
1696 if (test__start_subtest("bpf_sk_storage_map"))
1697 test_bpf_sk_storage_map();
1698 if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
1699 test_bpf_sk_stoarge_map_iter_fd();
1700 if (test__start_subtest("bpf_sk_storage_delete"))
1701 test_bpf_sk_storage_delete();
1702 if (test__start_subtest("bpf_sk_storage_get"))
1703 test_bpf_sk_storage_get();
1704 if (test__start_subtest("rdonly-buf-out-of-bound"))
1705 test_rdonly_buf_out_of_bound();
1706 if (test__start_subtest("buf-neg-offset"))
1707 test_buf_neg_offset();
1708 if (test__start_subtest("link-iter"))
1709 test_link_iter();
1710 if (test__start_subtest("ksym"))
1711 test_ksym_iter();
1712 if (test__start_subtest("bpf_sockmap_map_iter_fd"))
1713 test_bpf_sockmap_map_iter_fd();
1714 if (test__start_subtest("vma_offset"))
1715 test_task_vma_offset();
1716}