1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2020 Facebook */
   3#include <test_progs.h>
   4#include <unistd.h>
   5#include <sys/syscall.h>
   6#include <task_local_storage_helpers.h>
   7#include "bpf_iter_ipv6_route.skel.h"
   8#include "bpf_iter_netlink.skel.h"
   9#include "bpf_iter_bpf_map.skel.h"
  10#include "bpf_iter_tasks.skel.h"
  11#include "bpf_iter_task_stack.skel.h"
  12#include "bpf_iter_task_file.skel.h"
  13#include "bpf_iter_task_vmas.skel.h"
  14#include "bpf_iter_task_btf.skel.h"
  15#include "bpf_iter_tcp4.skel.h"
  16#include "bpf_iter_tcp6.skel.h"
  17#include "bpf_iter_udp4.skel.h"
  18#include "bpf_iter_udp6.skel.h"
  19#include "bpf_iter_unix.skel.h"
  20#include "bpf_iter_vma_offset.skel.h"
  21#include "bpf_iter_test_kern1.skel.h"
  22#include "bpf_iter_test_kern2.skel.h"
  23#include "bpf_iter_test_kern3.skel.h"
  24#include "bpf_iter_test_kern4.skel.h"
  25#include "bpf_iter_bpf_hash_map.skel.h"
  26#include "bpf_iter_bpf_percpu_hash_map.skel.h"
  27#include "bpf_iter_bpf_array_map.skel.h"
  28#include "bpf_iter_bpf_percpu_array_map.skel.h"
  29#include "bpf_iter_bpf_sk_storage_helpers.skel.h"
  30#include "bpf_iter_bpf_sk_storage_map.skel.h"
  31#include "bpf_iter_test_kern5.skel.h"
  32#include "bpf_iter_test_kern6.skel.h"
  33#include "bpf_iter_bpf_link.skel.h"
  34#include "bpf_iter_ksym.skel.h"
  35#include "bpf_iter_sockmap.skel.h"
  36
  37static void test_btf_id_or_null(void)
  38{
  39	struct bpf_iter_test_kern3 *skel;
  40
  41	skel = bpf_iter_test_kern3__open_and_load();
  42	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
  43		bpf_iter_test_kern3__destroy(skel);
  44		return;
  45	}
  46}
  47
  48static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
  49{
  50	struct bpf_link *link;
  51	char buf[16] = {};
  52	int iter_fd, len;
  53
  54	link = bpf_program__attach_iter(prog, opts);
  55	if (!ASSERT_OK_PTR(link, "attach_iter"))
  56		return;
  57
  58	iter_fd = bpf_iter_create(bpf_link__fd(link));
  59	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
  60		goto free_link;
  61
  62	/* not check contents, but ensure read() ends without error */
  63	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
  64		;
  65	ASSERT_GE(len, 0, "read");
  66
  67	close(iter_fd);
  68
  69free_link:
  70	bpf_link__destroy(link);
  71}
  72
  73static void do_dummy_read(struct bpf_program *prog)
  74{
  75	do_dummy_read_opts(prog, NULL);
  76}
  77
  78static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
  79				struct bpf_map *map)
  80{
  81	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
  82	union bpf_iter_link_info linfo;
  83	struct bpf_link *link;
  84	char buf[16] = {};
  85	int iter_fd, len;
  86
  87	memset(&linfo, 0, sizeof(linfo));
  88	linfo.map.map_fd = bpf_map__fd(map);
  89	opts.link_info = &linfo;
  90	opts.link_info_len = sizeof(linfo);
  91	link = bpf_program__attach_iter(prog, &opts);
  92	if (!ASSERT_OK_PTR(link, "attach_map_iter"))
  93		return;
  94
  95	iter_fd = bpf_iter_create(bpf_link__fd(link));
  96	if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
  97		bpf_link__destroy(link);
  98		return;
  99	}
 100
 101	/* Close link and map fd prematurely */
 102	bpf_link__destroy(link);
 103	bpf_object__destroy_skeleton(*skel);
 104	*skel = NULL;
 105
 106	/* Try to let map free work to run first if map is freed */
 107	usleep(100);
 108	/* Memory used by both sock map and sock local storage map are
 109	 * freed after two synchronize_rcu() calls, so wait for it
 110	 */
 111	kern_sync_rcu();
 112	kern_sync_rcu();
 113
 114	/* Read after both map fd and link fd are closed */
 115	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 116		;
 117	ASSERT_GE(len, 0, "read_iterator");
 118
 119	close(iter_fd);
 120}
 121
 122static int read_fd_into_buffer(int fd, char *buf, int size)
 123{
 124	int bufleft = size;
 125	int len;
 126
 127	do {
 128		len = read(fd, buf, bufleft);
 129		if (len > 0) {
 130			buf += len;
 131			bufleft -= len;
 132		}
 133	} while (len > 0);
 134
 135	return len < 0 ? len : size - bufleft;
 136}
 137
 138static void test_ipv6_route(void)
 139{
 140	struct bpf_iter_ipv6_route *skel;
 141
 142	skel = bpf_iter_ipv6_route__open_and_load();
 143	if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
 144		return;
 145
 146	do_dummy_read(skel->progs.dump_ipv6_route);
 147
 148	bpf_iter_ipv6_route__destroy(skel);
 149}
 150
 151static void test_netlink(void)
 152{
 153	struct bpf_iter_netlink *skel;
 154
 155	skel = bpf_iter_netlink__open_and_load();
 156	if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
 157		return;
 158
 159	do_dummy_read(skel->progs.dump_netlink);
 160
 161	bpf_iter_netlink__destroy(skel);
 162}
 163
 164static void test_bpf_map(void)
 165{
 166	struct bpf_iter_bpf_map *skel;
 167
 168	skel = bpf_iter_bpf_map__open_and_load();
 169	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
 170		return;
 171
 172	do_dummy_read(skel->progs.dump_bpf_map);
 173
 174	bpf_iter_bpf_map__destroy(skel);
 175}
 176
 177static void check_bpf_link_info(const struct bpf_program *prog)
 178{
 179	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 180	union bpf_iter_link_info linfo;
 181	struct bpf_link_info info = {};
 182	struct bpf_link *link;
 183	__u32 info_len;
 184	int err;
 185
 186	memset(&linfo, 0, sizeof(linfo));
 187	linfo.task.tid = getpid();
 188	opts.link_info = &linfo;
 189	opts.link_info_len = sizeof(linfo);
 190
 191	link = bpf_program__attach_iter(prog, &opts);
 192	if (!ASSERT_OK_PTR(link, "attach_iter"))
 193		return;
 194
 195	info_len = sizeof(info);
 196	err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
 197	ASSERT_OK(err, "bpf_link_get_info_by_fd");
 198	ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
 199
 200	bpf_link__destroy(link);
 201}
 202
 203static pthread_mutex_t do_nothing_mutex;
 204
 205static void *do_nothing_wait(void *arg)
 206{
 207	pthread_mutex_lock(&do_nothing_mutex);
 208	pthread_mutex_unlock(&do_nothing_mutex);
 209
 210	pthread_exit(arg);
 211}
 212
 213static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
 214				     int *num_unknown, int *num_known)
 215{
 216	struct bpf_iter_tasks *skel;
 217	pthread_t thread_id;
 218	void *ret;
 219
 220	skel = bpf_iter_tasks__open_and_load();
 221	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
 222		return;
 223
 224	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
 225
 226	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
 227		  "pthread_create");
 228
 229	skel->bss->tid = sys_gettid();
 230
 231	do_dummy_read_opts(skel->progs.dump_task, opts);
 232
 233	*num_unknown = skel->bss->num_unknown_tid;
 234	*num_known = skel->bss->num_known_tid;
 235
 236	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
 237	ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
 238		     "pthread_join");
 239
 240	bpf_iter_tasks__destroy(skel);
 241}
 242
 243static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
 244{
 245	int num_unknown_tid, num_known_tid;
 246
 247	test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
 248	ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
 249	ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
 250}
 251
 252static void *run_test_task_tid(void *arg)
 253{
 254	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 255	union bpf_iter_link_info linfo;
 256	int num_unknown_tid, num_known_tid;
 257
 258	ASSERT_NEQ(getpid(), sys_gettid(), "check_new_thread_id");
 259
 260	memset(&linfo, 0, sizeof(linfo));
 261	linfo.task.tid = sys_gettid();
 262	opts.link_info = &linfo;
 263	opts.link_info_len = sizeof(linfo);
 264	test_task_common(&opts, 0, 1);
 265
 266	linfo.task.tid = 0;
 267	linfo.task.pid = getpid();
 268	/* This includes the parent thread, this thread, watchdog timer thread
 269	 * and the do_nothing_wait thread
 270	 */
 271	test_task_common(&opts, 3, 1);
 272
 273	test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
 274	ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");
 275	ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
 276
 277	return NULL;
 278}
 279
 280static void test_task_tid(void)
 281{
 282	pthread_t thread_id;
 283
 284	/* Create a new thread so pid and tid aren't the same */
 285	ASSERT_OK(pthread_create(&thread_id, NULL, &run_test_task_tid, NULL),
 286		  "pthread_create");
 287	ASSERT_FALSE(pthread_join(thread_id, NULL), "pthread_join");
 288}
 289
 290static void test_task_pid(void)
 291{
 292	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 293	union bpf_iter_link_info linfo;
 294
 295	memset(&linfo, 0, sizeof(linfo));
 296	linfo.task.pid = getpid();
 297	opts.link_info = &linfo;
 298	opts.link_info_len = sizeof(linfo);
 299
 300	test_task_common(&opts, 2, 1);
 301}
 302
 303static void test_task_pidfd(void)
 304{
 305	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 306	union bpf_iter_link_info linfo;
 307	int pidfd;
 308
 309	pidfd = sys_pidfd_open(getpid(), 0);
 310	if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
 311		return;
 312
 313	memset(&linfo, 0, sizeof(linfo));
 314	linfo.task.pid_fd = pidfd;
 315	opts.link_info = &linfo;
 316	opts.link_info_len = sizeof(linfo);
 317
 318	test_task_common(&opts, 2, 1);
 319
 320	close(pidfd);
 321}
 322
 323static void test_task_sleepable(void)
 324{
 325	struct bpf_iter_tasks *skel;
 326
 327	skel = bpf_iter_tasks__open_and_load();
 328	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
 329		return;
 330
 331	do_dummy_read(skel->progs.dump_task_sleepable);
 332
 333	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
 334		  "num_expected_failure_copy_from_user_task");
 335	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
 336		  "num_success_copy_from_user_task");
 337
 338	bpf_iter_tasks__destroy(skel);
 339}
 340
 341static void test_task_stack(void)
 342{
 343	struct bpf_iter_task_stack *skel;
 344
 345	skel = bpf_iter_task_stack__open_and_load();
 346	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
 347		return;
 348
 349	do_dummy_read(skel->progs.dump_task_stack);
 350	do_dummy_read(skel->progs.get_task_user_stacks);
 351
 352	ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");
 353
 354	bpf_iter_task_stack__destroy(skel);
 355}
 356
 357static void test_task_file(void)
 358{
 359	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 360	struct bpf_iter_task_file *skel;
 361	union bpf_iter_link_info linfo;
 362	pthread_t thread_id;
 363	void *ret;
 364
 365	skel = bpf_iter_task_file__open_and_load();
 366	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
 367		return;
 368
 369	skel->bss->tgid = getpid();
 370
 371	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
 372
 373	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
 374		  "pthread_create");
 375
 376	memset(&linfo, 0, sizeof(linfo));
 377	linfo.task.tid = getpid();
 378	opts.link_info = &linfo;
 379	opts.link_info_len = sizeof(linfo);
 380
 381	do_dummy_read_opts(skel->progs.dump_task_file, &opts);
 382
 383	ASSERT_EQ(skel->bss->count, 0, "check_count");
 384	ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
 385
 386	skel->bss->last_tgid = 0;
 387	skel->bss->count = 0;
 388	skel->bss->unique_tgid_count = 0;
 389
 390	do_dummy_read(skel->progs.dump_task_file);
 391
 392	ASSERT_EQ(skel->bss->count, 0, "check_count");
 393	ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
 394
 395	check_bpf_link_info(skel->progs.dump_task_file);
 396
 397	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
 398	ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
 399	ASSERT_NULL(ret, "pthread_join");
 400
 401	bpf_iter_task_file__destroy(skel);
 402}
 403
 404#define TASKBUFSZ		32768
 405
 406static char taskbuf[TASKBUFSZ];
 407
 408static int do_btf_read(struct bpf_iter_task_btf *skel)
 409{
 410	struct bpf_program *prog = skel->progs.dump_task_struct;
 411	struct bpf_iter_task_btf__bss *bss = skel->bss;
 412	int iter_fd = -1, err;
 413	struct bpf_link *link;
 414	char *buf = taskbuf;
 415	int ret = 0;
 416
 417	link = bpf_program__attach_iter(prog, NULL);
 418	if (!ASSERT_OK_PTR(link, "attach_iter"))
 419		return ret;
 420
 421	iter_fd = bpf_iter_create(bpf_link__fd(link));
 422	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 423		goto free_link;
 424
 425	err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
 426	if (bss->skip) {
 427		printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
 428		ret = 1;
 429		test__skip();
 430		goto free_link;
 431	}
 432
 433	if (!ASSERT_GE(err, 0, "read"))
 434		goto free_link;
 435
 436	ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
 437	      "check for btf representation of task_struct in iter data");
 438free_link:
 439	if (iter_fd > 0)
 440		close(iter_fd);
 441	bpf_link__destroy(link);
 442	return ret;
 443}
 444
 445static void test_task_btf(void)
 446{
 447	struct bpf_iter_task_btf__bss *bss;
 448	struct bpf_iter_task_btf *skel;
 449	int ret;
 450
 451	skel = bpf_iter_task_btf__open_and_load();
 452	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
 453		return;
 454
 455	bss = skel->bss;
 456
 457	ret = do_btf_read(skel);
 458	if (ret)
 459		goto cleanup;
 460
 461	if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
 462		goto cleanup;
 463
 464	ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");
 465
 466cleanup:
 467	bpf_iter_task_btf__destroy(skel);
 468}
 469
 470static void test_tcp4(void)
 471{
 472	struct bpf_iter_tcp4 *skel;
 473
 474	skel = bpf_iter_tcp4__open_and_load();
 475	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
 476		return;
 477
 478	do_dummy_read(skel->progs.dump_tcp4);
 479
 480	bpf_iter_tcp4__destroy(skel);
 481}
 482
 483static void test_tcp6(void)
 484{
 485	struct bpf_iter_tcp6 *skel;
 486
 487	skel = bpf_iter_tcp6__open_and_load();
 488	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
 489		return;
 490
 491	do_dummy_read(skel->progs.dump_tcp6);
 492
 493	bpf_iter_tcp6__destroy(skel);
 494}
 495
 496static void test_udp4(void)
 497{
 498	struct bpf_iter_udp4 *skel;
 499
 500	skel = bpf_iter_udp4__open_and_load();
 501	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
 502		return;
 503
 504	do_dummy_read(skel->progs.dump_udp4);
 505
 506	bpf_iter_udp4__destroy(skel);
 507}
 508
 509static void test_udp6(void)
 510{
 511	struct bpf_iter_udp6 *skel;
 512
 513	skel = bpf_iter_udp6__open_and_load();
 514	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
 515		return;
 516
 517	do_dummy_read(skel->progs.dump_udp6);
 518
 519	bpf_iter_udp6__destroy(skel);
 520}
 521
 522static void test_unix(void)
 523{
 524	struct bpf_iter_unix *skel;
 525
 526	skel = bpf_iter_unix__open_and_load();
 527	if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
 528		return;
 529
 530	do_dummy_read(skel->progs.dump_unix);
 531
 532	bpf_iter_unix__destroy(skel);
 533}
 534
 535/* The expected string is less than 16 bytes */
 536static int do_read_with_fd(int iter_fd, const char *expected,
 537			   bool read_one_char)
 538{
 539	int len, read_buf_len, start;
 540	char buf[16] = {};
 541
 542	read_buf_len = read_one_char ? 1 : 16;
 543	start = 0;
 544	while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
 545		start += len;
 546		if (!ASSERT_LT(start, 16, "read"))
 547			return -1;
 548		read_buf_len = read_one_char ? 1 : 16 - start;
 549	}
 550	if (!ASSERT_GE(len, 0, "read"))
 551		return -1;
 552
 553	if (!ASSERT_STREQ(buf, expected, "read"))
 554		return -1;
 555
 556	return 0;
 557}
 558
 559static void test_anon_iter(bool read_one_char)
 560{
 561	struct bpf_iter_test_kern1 *skel;
 562	struct bpf_link *link;
 563	int iter_fd, err;
 564
 565	skel = bpf_iter_test_kern1__open_and_load();
 566	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
 567		return;
 568
 569	err = bpf_iter_test_kern1__attach(skel);
 570	if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
 571		goto out;
 572	}
 573
 574	link = skel->links.dump_task;
 575	iter_fd = bpf_iter_create(bpf_link__fd(link));
 576	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 577		goto out;
 578
 579	do_read_with_fd(iter_fd, "abcd", read_one_char);
 580	close(iter_fd);
 581
 582out:
 583	bpf_iter_test_kern1__destroy(skel);
 584}
 585
 586static int do_read(const char *path, const char *expected)
 587{
 588	int err, iter_fd;
 589
 590	iter_fd = open(path, O_RDONLY);
 591	if (!ASSERT_GE(iter_fd, 0, "open"))
 592		return -1;
 593
 594	err = do_read_with_fd(iter_fd, expected, false);
 595	close(iter_fd);
 596	return err;
 597}
 598
 599static void test_file_iter(void)
 600{
 601	const char *path = "/sys/fs/bpf/bpf_iter_test1";
 602	struct bpf_iter_test_kern1 *skel1;
 603	struct bpf_iter_test_kern2 *skel2;
 604	struct bpf_link *link;
 605	int err;
 606
 607	skel1 = bpf_iter_test_kern1__open_and_load();
 608	if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
 609		return;
 610
 611	link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
 612	if (!ASSERT_OK_PTR(link, "attach_iter"))
 613		goto out;
 614
 615	/* unlink this path if it exists. */
 616	unlink(path);
 617
 618	err = bpf_link__pin(link, path);
 619	if (!ASSERT_OK(err, "pin_iter"))
 620		goto free_link;
 621
 622	err = do_read(path, "abcd");
 623	if (err)
 624		goto unlink_path;
 625
 626	/* file based iterator seems working fine. Let us a link update
 627	 * of the underlying link and `cat` the iterator again, its content
 628	 * should change.
 629	 */
 630	skel2 = bpf_iter_test_kern2__open_and_load();
 631	if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
 632		goto unlink_path;
 633
 634	err = bpf_link__update_program(link, skel2->progs.dump_task);
 635	if (!ASSERT_OK(err, "update_prog"))
 636		goto destroy_skel2;
 637
 638	do_read(path, "ABCD");
 639
 640destroy_skel2:
 641	bpf_iter_test_kern2__destroy(skel2);
 642unlink_path:
 643	unlink(path);
 644free_link:
 645	bpf_link__destroy(link);
 646out:
 647	bpf_iter_test_kern1__destroy(skel1);
 648}
 649
 650static void test_overflow(bool test_e2big_overflow, bool ret1)
 651{
 652	__u32 map_info_len, total_read_len, expected_read_len;
 653	int err, iter_fd, map1_fd, map2_fd, len;
 654	struct bpf_map_info map_info = {};
 655	struct bpf_iter_test_kern4 *skel;
 656	struct bpf_link *link;
 657	__u32 iter_size;
 658	char *buf;
 659
 660	skel = bpf_iter_test_kern4__open();
 661	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
 662		return;
 663
 664	/* create two maps: bpf program will only do bpf_seq_write
 665	 * for these two maps. The goal is one map output almost
 666	 * fills seq_file buffer and then the other will trigger
 667	 * overflow and needs restart.
 668	 */
 669	map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
 670	if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
 671		goto out;
 672	map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
 673	if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
 674		goto free_map1;
 675
 676	/* bpf_seq_printf kernel buffer is 8 pages, so one map
 677	 * bpf_seq_write will mostly fill it, and the other map
 678	 * will partially fill and then trigger overflow and need
 679	 * bpf_seq_read restart.
 680	 */
 681	iter_size = sysconf(_SC_PAGE_SIZE) << 3;
 682
 683	if (test_e2big_overflow) {
 684		skel->rodata->print_len = (iter_size + 8) / 8;
 685		expected_read_len = 2 * (iter_size + 8);
 686	} else if (!ret1) {
 687		skel->rodata->print_len = (iter_size - 8) / 8;
 688		expected_read_len = 2 * (iter_size - 8);
 689	} else {
 690		skel->rodata->print_len = 1;
 691		expected_read_len = 2 * 8;
 692	}
 693	skel->rodata->ret1 = ret1;
 694
 695	if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
 696		  "bpf_iter_test_kern4__load"))
 697		goto free_map2;
 698
 699	/* setup filtering map_id in bpf program */
 700	map_info_len = sizeof(map_info);
 701	err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
 702	if (!ASSERT_OK(err, "get_map_info"))
 703		goto free_map2;
 704	skel->bss->map1_id = map_info.id;
 705
 706	err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
 707	if (!ASSERT_OK(err, "get_map_info"))
 708		goto free_map2;
 709	skel->bss->map2_id = map_info.id;
 710
 711	link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
 712	if (!ASSERT_OK_PTR(link, "attach_iter"))
 713		goto free_map2;
 714
 715	iter_fd = bpf_iter_create(bpf_link__fd(link));
 716	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 717		goto free_link;
 718
 719	buf = malloc(expected_read_len);
 720	if (!ASSERT_OK_PTR(buf, "malloc"))
 721		goto close_iter;
 722
 723	/* do read */
 724	total_read_len = 0;
 725	if (test_e2big_overflow) {
 726		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 727			total_read_len += len;
 728
 729		ASSERT_EQ(len, -1, "read");
 730		ASSERT_EQ(errno, E2BIG, "read");
 731		goto free_buf;
 732	} else if (!ret1) {
 733		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 734			total_read_len += len;
 735
 736		if (!ASSERT_GE(len, 0, "read"))
 737			goto free_buf;
 738	} else {
 739		do {
 740			len = read(iter_fd, buf, expected_read_len);
 741			if (len > 0)
 742				total_read_len += len;
 743		} while (len > 0 || len == -EAGAIN);
 744
 745		if (!ASSERT_GE(len, 0, "read"))
 746			goto free_buf;
 747	}
 748
 749	if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
 750		goto free_buf;
 751
 752	if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
 753		goto free_buf;
 754
 755	if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
 756		goto free_buf;
 757
 758	ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");
 759
 760free_buf:
 761	free(buf);
 762close_iter:
 763	close(iter_fd);
 764free_link:
 765	bpf_link__destroy(link);
 766free_map2:
 767	close(map2_fd);
 768free_map1:
 769	close(map1_fd);
 770out:
 771	bpf_iter_test_kern4__destroy(skel);
 772}
 773
 774static void test_bpf_hash_map(void)
 775{
 776	__u32 expected_key_a = 0, expected_key_b = 0;
 777	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 778	struct bpf_iter_bpf_hash_map *skel;
 779	int err, i, len, map_fd, iter_fd;
 780	union bpf_iter_link_info linfo;
 781	__u64 val, expected_val = 0;
 782	struct bpf_link *link;
 783	struct key_t {
 784		int a;
 785		int b;
 786		int c;
 787	} key;
 788	char buf[64];
 789
 790	skel = bpf_iter_bpf_hash_map__open();
 791	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
 792		return;
 793
 794	skel->bss->in_test_mode = true;
 795
 796	err = bpf_iter_bpf_hash_map__load(skel);
 797	if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
 798		goto out;
 799
 800	/* iterator with hashmap2 and hashmap3 should fail */
 801	memset(&linfo, 0, sizeof(linfo));
 802	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
 803	opts.link_info = &linfo;
 804	opts.link_info_len = sizeof(linfo);
 805	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 806	if (!ASSERT_ERR_PTR(link, "attach_iter"))
 807		goto out;
 808
 809	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
 810	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 811	if (!ASSERT_ERR_PTR(link, "attach_iter"))
 812		goto out;
 813
 814	/* hashmap1 should be good, update map values here */
 815	map_fd = bpf_map__fd(skel->maps.hashmap1);
 816	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
 817		key.a = i + 1;
 818		key.b = i + 2;
 819		key.c = i + 3;
 820		val = i + 4;
 821		expected_key_a += key.a;
 822		expected_key_b += key.b;
 823		expected_val += val;
 824
 825		err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
 826		if (!ASSERT_OK(err, "map_update"))
 827			goto out;
 828	}
 829
 830	/* Sleepable program is prohibited for hash map iterator */
 831	linfo.map.map_fd = map_fd;
 832	link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
 833	if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
 834		goto out;
 835
 836	linfo.map.map_fd = map_fd;
 837	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 838	if (!ASSERT_OK_PTR(link, "attach_iter"))
 839		goto out;
 840
 841	iter_fd = bpf_iter_create(bpf_link__fd(link));
 842	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 843		goto free_link;
 844
 845	/* do some tests */
 846	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 847		;
 848	if (!ASSERT_GE(len, 0, "read"))
 849		goto close_iter;
 850
 851	/* test results */
 852	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
 853		goto close_iter;
 854	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
 855		goto close_iter;
 856	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
 857		goto close_iter;
 858
 859close_iter:
 860	close(iter_fd);
 861free_link:
 862	bpf_link__destroy(link);
 863out:
 864	bpf_iter_bpf_hash_map__destroy(skel);
 865}
 866
 867static void test_bpf_percpu_hash_map(void)
 868{
 869	__u32 expected_key_a = 0, expected_key_b = 0;
 870	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 871	struct bpf_iter_bpf_percpu_hash_map *skel;
 872	int err, i, j, len, map_fd, iter_fd;
 873	union bpf_iter_link_info linfo;
 874	__u32 expected_val = 0;
 875	struct bpf_link *link;
 876	struct key_t {
 877		int a;
 878		int b;
 879		int c;
 880	} key;
 881	char buf[64];
 882	void *val;
 883
 884	skel = bpf_iter_bpf_percpu_hash_map__open();
 885	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
 886		return;
 887
 888	skel->rodata->num_cpus = bpf_num_possible_cpus();
 889	val = malloc(8 * bpf_num_possible_cpus());
 890	if (!ASSERT_OK_PTR(val, "malloc"))
 891		goto out;
 892
 893	err = bpf_iter_bpf_percpu_hash_map__load(skel);
 894	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
 895		goto out;
 896
 897	/* update map values here */
 898	map_fd = bpf_map__fd(skel->maps.hashmap1);
 899	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
 900		key.a = i + 1;
 901		key.b = i + 2;
 902		key.c = i + 3;
 903		expected_key_a += key.a;
 904		expected_key_b += key.b;
 905
 906		for (j = 0; j < bpf_num_possible_cpus(); j++) {
 907			*(__u32 *)(val + j * 8) = i + j;
 908			expected_val += i + j;
 909		}
 910
 911		err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
 912		if (!ASSERT_OK(err, "map_update"))
 913			goto out;
 914	}
 915
 916	memset(&linfo, 0, sizeof(linfo));
 917	linfo.map.map_fd = map_fd;
 918	opts.link_info = &linfo;
 919	opts.link_info_len = sizeof(linfo);
 920	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
 921	if (!ASSERT_OK_PTR(link, "attach_iter"))
 922		goto out;
 923
 924	iter_fd = bpf_iter_create(bpf_link__fd(link));
 925	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 926		goto free_link;
 927
 928	/* do some tests */
 929	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 930		;
 931	if (!ASSERT_GE(len, 0, "read"))
 932		goto close_iter;
 933
 934	/* test results */
 935	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
 936		goto close_iter;
 937	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
 938		goto close_iter;
 939	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
 940		goto close_iter;
 941
 942close_iter:
 943	close(iter_fd);
 944free_link:
 945	bpf_link__destroy(link);
 946out:
 947	bpf_iter_bpf_percpu_hash_map__destroy(skel);
 948	free(val);
 949}
 950
 951static void test_bpf_array_map(void)
 952{
 953	__u64 val, expected_val = 0, res_first_val, first_val = 0;
 954	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 955	__u32 key, expected_key = 0, res_first_key;
 956	int err, i, map_fd, hash_fd, iter_fd;
 957	struct bpf_iter_bpf_array_map *skel;
 958	union bpf_iter_link_info linfo;
 959	struct bpf_link *link;
 960	char buf[64] = {};
 961	int len, start;
 962
 963	skel = bpf_iter_bpf_array_map__open_and_load();
 964	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
 965		return;
 966
 967	map_fd = bpf_map__fd(skel->maps.arraymap1);
 968	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
 969		val = i + 4;
 970		expected_key += i;
 971		expected_val += val;
 972
 973		if (i == 0)
 974			first_val = val;
 975
 976		err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
 977		if (!ASSERT_OK(err, "map_update"))
 978			goto out;
 979	}
 980
 981	memset(&linfo, 0, sizeof(linfo));
 982	linfo.map.map_fd = map_fd;
 983	opts.link_info = &linfo;
 984	opts.link_info_len = sizeof(linfo);
 985	link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
 986	if (!ASSERT_OK_PTR(link, "attach_iter"))
 987		goto out;
 988
 989	iter_fd = bpf_iter_create(bpf_link__fd(link));
 990	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 991		goto free_link;
 992
 993	/* do some tests */
 994	start = 0;
 995	while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
 996		start += len;
 997	if (!ASSERT_GE(len, 0, "read"))
 998		goto close_iter;
 999
1000	/* test results */
1001	res_first_key = *(__u32 *)buf;
1002	res_first_val = *(__u64 *)(buf + sizeof(__u32));
1003	if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
1004			!ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
1005		goto close_iter;
1006
1007	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1008		goto close_iter;
1009	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1010		goto close_iter;
1011
1012	hash_fd = bpf_map__fd(skel->maps.hashmap1);
1013	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1014		err = bpf_map_lookup_elem(map_fd, &i, &val);
1015		if (!ASSERT_OK(err, "map_lookup arraymap1"))
1016			goto close_iter;
1017		if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
1018			goto close_iter;
1019
1020		val = i + 4;
1021		err = bpf_map_lookup_elem(hash_fd, &val, &key);
1022		if (!ASSERT_OK(err, "map_lookup hashmap1"))
1023			goto close_iter;
1024		if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
1025			goto close_iter;
1026	}
1027
1028close_iter:
1029	close(iter_fd);
1030free_link:
1031	bpf_link__destroy(link);
1032out:
1033	bpf_iter_bpf_array_map__destroy(skel);
1034}
1035
1036static void test_bpf_array_map_iter_fd(void)
1037{
1038	struct bpf_iter_bpf_array_map *skel;
1039
1040	skel = bpf_iter_bpf_array_map__open_and_load();
1041	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1042		return;
1043
1044	do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,
1045			    skel->maps.arraymap1);
1046
1047	bpf_iter_bpf_array_map__destroy(skel);
1048}
1049
1050static void test_bpf_percpu_array_map(void)
1051{
1052	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1053	struct bpf_iter_bpf_percpu_array_map *skel;
1054	__u32 expected_key = 0, expected_val = 0;
1055	union bpf_iter_link_info linfo;
1056	int err, i, j, map_fd, iter_fd;
1057	struct bpf_link *link;
1058	char buf[64];
1059	void *val;
1060	int len;
1061
1062	skel = bpf_iter_bpf_percpu_array_map__open();
1063	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
1064		return;
1065
1066	skel->rodata->num_cpus = bpf_num_possible_cpus();
1067	val = malloc(8 * bpf_num_possible_cpus());
1068	if (!ASSERT_OK_PTR(val, "malloc"))
1069		goto out;
1070
1071	err = bpf_iter_bpf_percpu_array_map__load(skel);
1072	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
1073		goto out;
1074
1075	/* update map values here */
1076	map_fd = bpf_map__fd(skel->maps.arraymap1);
1077	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1078		expected_key += i;
1079
1080		for (j = 0; j < bpf_num_possible_cpus(); j++) {
1081			*(__u32 *)(val + j * 8) = i + j;
1082			expected_val += i + j;
1083		}
1084
1085		err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
1086		if (!ASSERT_OK(err, "map_update"))
1087			goto out;
1088	}
1089
1090	memset(&linfo, 0, sizeof(linfo));
1091	linfo.map.map_fd = map_fd;
1092	opts.link_info = &linfo;
1093	opts.link_info_len = sizeof(linfo);
1094	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
1095	if (!ASSERT_OK_PTR(link, "attach_iter"))
1096		goto out;
1097
1098	iter_fd = bpf_iter_create(bpf_link__fd(link));
1099	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1100		goto free_link;
1101
1102	/* do some tests */
1103	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1104		;
1105	if (!ASSERT_GE(len, 0, "read"))
1106		goto close_iter;
1107
1108	/* test results */
1109	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1110		goto close_iter;
1111	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1112		goto close_iter;
1113
1114close_iter:
1115	close(iter_fd);
1116free_link:
1117	bpf_link__destroy(link);
1118out:
1119	bpf_iter_bpf_percpu_array_map__destroy(skel);
1120	free(val);
1121}
1122
1123/* An iterator program deletes all local storage in a map. */
1124static void test_bpf_sk_storage_delete(void)
1125{
1126	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1127	struct bpf_iter_bpf_sk_storage_helpers *skel;
1128	union bpf_iter_link_info linfo;
1129	int err, len, map_fd, iter_fd;
1130	struct bpf_link *link;
1131	int sock_fd = -1;
1132	__u32 val = 42;
1133	char buf[64];
1134
1135	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1136	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1137		return;
1138
1139	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1140
1141	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1142	if (!ASSERT_GE(sock_fd, 0, "socket"))
1143		goto out;
1144
1145	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1146	if (!ASSERT_OK(err, "map_update"))
1147		goto out;
1148
1149	memset(&linfo, 0, sizeof(linfo));
1150	linfo.map.map_fd = map_fd;
1151	opts.link_info = &linfo;
1152	opts.link_info_len = sizeof(linfo);
1153	link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
1154					&opts);
1155	if (!ASSERT_OK_PTR(link, "attach_iter"))
1156		goto out;
1157
1158	iter_fd = bpf_iter_create(bpf_link__fd(link));
1159	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1160		goto free_link;
1161
1162	/* do some tests */
1163	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1164		;
1165	if (!ASSERT_GE(len, 0, "read"))
1166		goto close_iter;
1167
1168	/* test results */
1169	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1170
1171	 /* Note: The following assertions serve to ensure
1172	  * the value was deleted. It does so by asserting
1173	  * that bpf_map_lookup_elem has failed. This might
1174	  * seem counterintuitive at first.
1175	  */
1176	ASSERT_ERR(err, "bpf_map_lookup_elem");
1177	ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");
1178
1179close_iter:
1180	close(iter_fd);
1181free_link:
1182	bpf_link__destroy(link);
1183out:
1184	if (sock_fd >= 0)
1185		close(sock_fd);
1186	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1187}
1188
1189/* This creates a socket and its local storage. It then runs a task_iter BPF
1190 * program that replaces the existing socket local storage with the tgid of the
1191 * only task owning a file descriptor to this socket, this process, prog_tests.
1192 * It then runs a tcp socket iterator that negates the value in the existing
1193 * socket local storage, the test verifies that the resulting value is -pid.
1194 */
1195static void test_bpf_sk_storage_get(void)
1196{
1197	struct bpf_iter_bpf_sk_storage_helpers *skel;
1198	int err, map_fd, val = -1;
1199	int sock_fd = -1;
1200
1201	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1202	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1203		return;
1204
1205	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1206	if (!ASSERT_GE(sock_fd, 0, "socket"))
1207		goto out;
1208
1209	err = listen(sock_fd, 1);
1210	if (!ASSERT_OK(err, "listen"))
1211		goto close_socket;
1212
1213	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1214
1215	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1216	if (!ASSERT_OK(err, "bpf_map_update_elem"))
1217		goto close_socket;
1218
1219	do_dummy_read(skel->progs.fill_socket_owner);
1220
1221	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1222	if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
1223			!ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
1224		goto close_socket;
1225
1226	do_dummy_read(skel->progs.negate_socket_local_storage);
1227
1228	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1229	ASSERT_OK(err, "bpf_map_lookup_elem");
1230	ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");
1231
1232close_socket:
1233	close(sock_fd);
1234out:
1235	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1236}
1237
1238static void test_bpf_sk_storage_map_iter_fd(void)
1239{
1240	struct bpf_iter_bpf_sk_storage_map *skel;
1241
1242	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1243	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1244		return;
1245
1246	do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,
1247			    skel->maps.sk_stg_map);
1248
1249	bpf_iter_bpf_sk_storage_map__destroy(skel);
1250}
1251
1252static void test_bpf_sk_storage_map(void)
1253{
1254	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1255	int err, i, len, map_fd, iter_fd, num_sockets;
1256	struct bpf_iter_bpf_sk_storage_map *skel;
1257	union bpf_iter_link_info linfo;
1258	int sock_fd[3] = {-1, -1, -1};
1259	__u32 val, expected_val = 0;
1260	struct bpf_link *link;
1261	char buf[64];
1262
1263	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1264	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1265		return;
1266
1267	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1268	num_sockets = ARRAY_SIZE(sock_fd);
1269	for (i = 0; i < num_sockets; i++) {
1270		sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
1271		if (!ASSERT_GE(sock_fd[i], 0, "socket"))
1272			goto out;
1273
1274		val = i + 1;
1275		expected_val += val;
1276
1277		err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
1278					  BPF_NOEXIST);
1279		if (!ASSERT_OK(err, "map_update"))
1280			goto out;
1281	}
1282
1283	memset(&linfo, 0, sizeof(linfo));
1284	linfo.map.map_fd = map_fd;
1285	opts.link_info = &linfo;
1286	opts.link_info_len = sizeof(linfo);
1287	link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
1288	err = libbpf_get_error(link);
1289	if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
1290		if (!err)
1291			bpf_link__destroy(link);
1292		goto out;
1293	}
1294
1295	link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
1296	if (!ASSERT_OK_PTR(link, "attach_iter"))
1297		goto out;
1298
1299	iter_fd = bpf_iter_create(bpf_link__fd(link));
1300	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1301		goto free_link;
1302
1303	skel->bss->to_add_val = time(NULL);
1304	/* do some tests */
1305	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1306		;
1307	if (!ASSERT_GE(len, 0, "read"))
1308		goto close_iter;
1309
1310	/* test results */
1311	if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
1312		goto close_iter;
1313
1314	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1315		goto close_iter;
1316
1317	for (i = 0; i < num_sockets; i++) {
1318		err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
1319		if (!ASSERT_OK(err, "map_lookup") ||
1320		    !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
1321			break;
1322	}
1323
1324close_iter:
1325	close(iter_fd);
1326free_link:
1327	bpf_link__destroy(link);
1328out:
1329	for (i = 0; i < num_sockets; i++) {
1330		if (sock_fd[i] >= 0)
1331			close(sock_fd[i]);
1332	}
1333	bpf_iter_bpf_sk_storage_map__destroy(skel);
1334}
1335
1336static void test_rdonly_buf_out_of_bound(void)
1337{
1338	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1339	struct bpf_iter_test_kern5 *skel;
1340	union bpf_iter_link_info linfo;
1341	struct bpf_link *link;
1342
1343	skel = bpf_iter_test_kern5__open_and_load();
1344	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
1345		return;
1346
1347	memset(&linfo, 0, sizeof(linfo));
1348	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
1349	opts.link_info = &linfo;
1350	opts.link_info_len = sizeof(linfo);
1351	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
1352	if (!ASSERT_ERR_PTR(link, "attach_iter"))
1353		bpf_link__destroy(link);
1354
1355	bpf_iter_test_kern5__destroy(skel);
1356}
1357
1358static void test_buf_neg_offset(void)
1359{
1360	struct bpf_iter_test_kern6 *skel;
1361
1362	skel = bpf_iter_test_kern6__open_and_load();
1363	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
1364		bpf_iter_test_kern6__destroy(skel);
1365}
1366
1367static void test_link_iter(void)
1368{
1369	struct bpf_iter_bpf_link *skel;
1370
1371	skel = bpf_iter_bpf_link__open_and_load();
1372	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
1373		return;
1374
1375	do_dummy_read(skel->progs.dump_bpf_link);
1376
1377	bpf_iter_bpf_link__destroy(skel);
1378}
1379
1380static void test_ksym_iter(void)
1381{
1382	struct bpf_iter_ksym *skel;
1383
1384	skel = bpf_iter_ksym__open_and_load();
1385	if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
1386		return;
1387
1388	do_dummy_read(skel->progs.dump_ksym);
1389
1390	bpf_iter_ksym__destroy(skel);
1391}
1392
1393#define CMP_BUFFER_SIZE 1024
1394static char task_vma_output[CMP_BUFFER_SIZE];
1395static char proc_maps_output[CMP_BUFFER_SIZE];
1396
1397/* remove \0 and \t from str, and only keep the first line */
1398static void str_strip_first_line(char *str)
1399{
1400	char *dst = str, *src = str;
1401
1402	do {
1403		if (*src == ' ' || *src == '\t')
1404			src++;
1405		else
1406			*(dst++) = *(src++);
1407
1408	} while (*src != '\0' && *src != '\n');
1409
1410	*dst = '\0';
1411}
1412
1413static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
1414{
1415	int err, iter_fd = -1, proc_maps_fd = -1;
1416	struct bpf_iter_task_vmas *skel;
1417	int len, read_size = 4;
1418	char maps_path[64];
1419
1420	skel = bpf_iter_task_vmas__open();
1421	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1422		return;
1423
1424	skel->bss->pid = getpid();
1425	skel->bss->one_task = opts ? 1 : 0;
1426
1427	err = bpf_iter_task_vmas__load(skel);
1428	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1429		goto out;
1430
1431	skel->links.proc_maps = bpf_program__attach_iter(
1432		skel->progs.proc_maps, opts);
1433
1434	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1435		skel->links.proc_maps = NULL;
1436		goto out;
1437	}
1438
1439	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1440	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1441		goto out;
1442
1443	/* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1444	 * to trigger seq_file corner cases.
1445	 */
1446	len = 0;
1447	while (len < CMP_BUFFER_SIZE) {
1448		err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1449					  MIN(read_size, CMP_BUFFER_SIZE - len));
1450		if (!err)
1451			break;
1452		if (!ASSERT_GE(err, 0, "read_iter_fd"))
1453			goto out;
1454		len += err;
1455	}
1456	if (opts)
1457		ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
1458
1459	/* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1460	snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1461	proc_maps_fd = open(maps_path, O_RDONLY);
1462	if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
1463		goto out;
1464	err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1465	if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
1466		goto out;
1467
1468	/* strip and compare the first line of the two files */
1469	str_strip_first_line(task_vma_output);
1470	str_strip_first_line(proc_maps_output);
1471
1472	ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
1473
1474	check_bpf_link_info(skel->progs.proc_maps);
1475
1476out:
1477	close(proc_maps_fd);
1478	close(iter_fd);
1479	bpf_iter_task_vmas__destroy(skel);
1480}
1481
1482static void test_task_vma_dead_task(void)
1483{
1484	struct bpf_iter_task_vmas *skel;
1485	int wstatus, child_pid = -1;
1486	time_t start_tm, cur_tm;
1487	int err, iter_fd = -1;
1488	int wait_sec = 3;
1489
1490	skel = bpf_iter_task_vmas__open();
1491	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1492		return;
1493
1494	skel->bss->pid = getpid();
1495
1496	err = bpf_iter_task_vmas__load(skel);
1497	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1498		goto out;
1499
1500	skel->links.proc_maps = bpf_program__attach_iter(
1501		skel->progs.proc_maps, NULL);
1502
1503	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1504		skel->links.proc_maps = NULL;
1505		goto out;
1506	}
1507
1508	start_tm = time(NULL);
1509	cur_tm = start_tm;
1510
1511	child_pid = fork();
1512	if (child_pid == 0) {
1513		/* Fork short-lived processes in the background. */
1514		while (cur_tm < start_tm + wait_sec) {
1515			system("echo > /dev/null");
1516			cur_tm = time(NULL);
1517		}
1518		exit(0);
1519	}
1520
1521	if (!ASSERT_GE(child_pid, 0, "fork_child"))
1522		goto out;
1523
1524	while (cur_tm < start_tm + wait_sec) {
1525		iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1526		if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1527			goto out;
1528
1529		/* Drain all data from iter_fd. */
1530		while (cur_tm < start_tm + wait_sec) {
1531			err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);
1532			if (!ASSERT_GE(err, 0, "read_iter_fd"))
1533				goto out;
1534
1535			cur_tm = time(NULL);
1536
1537			if (err == 0)
1538				break;
1539		}
1540
1541		close(iter_fd);
1542		iter_fd = -1;
1543	}
1544
1545	check_bpf_link_info(skel->progs.proc_maps);
1546
1547out:
1548	waitpid(child_pid, &wstatus, 0);
1549	close(iter_fd);
1550	bpf_iter_task_vmas__destroy(skel);
1551}
1552
1553void test_bpf_sockmap_map_iter_fd(void)
1554{
1555	struct bpf_iter_sockmap *skel;
1556
1557	skel = bpf_iter_sockmap__open_and_load();
1558	if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
1559		return;
1560
1561	do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);
1562
1563	bpf_iter_sockmap__destroy(skel);
1564}
1565
1566static void test_task_vma(void)
1567{
1568	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1569	union bpf_iter_link_info linfo;
1570
1571	memset(&linfo, 0, sizeof(linfo));
1572	linfo.task.tid = getpid();
1573	opts.link_info = &linfo;
1574	opts.link_info_len = sizeof(linfo);
1575
1576	test_task_vma_common(&opts);
1577	test_task_vma_common(NULL);
1578}
1579
1580/* uprobe attach point */
1581static noinline int trigger_func(int arg)
1582{
1583	asm volatile ("");
1584	return arg + 1;
1585}
1586
1587static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
1588{
1589	struct bpf_iter_vma_offset *skel;
1590	char buf[16] = {};
1591	int iter_fd, len;
1592	int pgsz, shift;
1593
1594	skel = bpf_iter_vma_offset__open_and_load();
1595	if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
1596		return;
1597
1598	skel->bss->pid = getpid();
1599	skel->bss->address = (uintptr_t)trigger_func;
1600	for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
1601		;
1602	skel->bss->page_shift = shift;
1603
1604	skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
1605	if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
1606		goto exit;
1607
1608	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
1609	if (!ASSERT_GT(iter_fd, 0, "create_iter"))
1610		goto exit;
1611
1612	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1613		;
1614	buf[15] = 0;
1615	ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
1616
1617	ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
1618	if (one_proc)
1619		ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1620	else
1621		ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1622
1623	close(iter_fd);
1624
1625exit:
1626	bpf_iter_vma_offset__destroy(skel);
1627}
1628
1629static void test_task_vma_offset(void)
1630{
1631	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1632	union bpf_iter_link_info linfo;
1633
1634	memset(&linfo, 0, sizeof(linfo));
1635	linfo.task.pid = getpid();
1636	opts.link_info = &linfo;
1637	opts.link_info_len = sizeof(linfo);
1638
1639	test_task_vma_offset_common(&opts, true);
1640
1641	linfo.task.pid = 0;
1642	linfo.task.tid = getpid();
1643	test_task_vma_offset_common(&opts, true);
1644
1645	test_task_vma_offset_common(NULL, false);
1646}
1647
1648void test_bpf_iter(void)
1649{
1650	ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
1651
1652	if (test__start_subtest("btf_id_or_null"))
1653		test_btf_id_or_null();
1654	if (test__start_subtest("ipv6_route"))
1655		test_ipv6_route();
1656	if (test__start_subtest("netlink"))
1657		test_netlink();
1658	if (test__start_subtest("bpf_map"))
1659		test_bpf_map();
1660	if (test__start_subtest("task_tid"))
1661		test_task_tid();
1662	if (test__start_subtest("task_pid"))
1663		test_task_pid();
1664	if (test__start_subtest("task_pidfd"))
1665		test_task_pidfd();
1666	if (test__start_subtest("task_sleepable"))
1667		test_task_sleepable();
1668	if (test__start_subtest("task_stack"))
1669		test_task_stack();
1670	if (test__start_subtest("task_file"))
1671		test_task_file();
1672	if (test__start_subtest("task_vma"))
1673		test_task_vma();
1674	if (test__start_subtest("task_vma_dead_task"))
1675		test_task_vma_dead_task();
1676	if (test__start_subtest("task_btf"))
1677		test_task_btf();
1678	if (test__start_subtest("tcp4"))
1679		test_tcp4();
1680	if (test__start_subtest("tcp6"))
1681		test_tcp6();
1682	if (test__start_subtest("udp4"))
1683		test_udp4();
1684	if (test__start_subtest("udp6"))
1685		test_udp6();
1686	if (test__start_subtest("unix"))
1687		test_unix();
1688	if (test__start_subtest("anon"))
1689		test_anon_iter(false);
1690	if (test__start_subtest("anon-read-one-char"))
1691		test_anon_iter(true);
1692	if (test__start_subtest("file"))
1693		test_file_iter();
1694	if (test__start_subtest("overflow"))
1695		test_overflow(false, false);
1696	if (test__start_subtest("overflow-e2big"))
1697		test_overflow(true, false);
1698	if (test__start_subtest("prog-ret-1"))
1699		test_overflow(false, true);
1700	if (test__start_subtest("bpf_hash_map"))
1701		test_bpf_hash_map();
1702	if (test__start_subtest("bpf_percpu_hash_map"))
1703		test_bpf_percpu_hash_map();
1704	if (test__start_subtest("bpf_array_map"))
1705		test_bpf_array_map();
1706	if (test__start_subtest("bpf_array_map_iter_fd"))
1707		test_bpf_array_map_iter_fd();
1708	if (test__start_subtest("bpf_percpu_array_map"))
1709		test_bpf_percpu_array_map();
1710	if (test__start_subtest("bpf_sk_storage_map"))
1711		test_bpf_sk_storage_map();
1712	if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
1713		test_bpf_sk_storage_map_iter_fd();
1714	if (test__start_subtest("bpf_sk_storage_delete"))
1715		test_bpf_sk_storage_delete();
1716	if (test__start_subtest("bpf_sk_storage_get"))
1717		test_bpf_sk_storage_get();
1718	if (test__start_subtest("rdonly-buf-out-of-bound"))
1719		test_rdonly_buf_out_of_bound();
1720	if (test__start_subtest("buf-neg-offset"))
1721		test_buf_neg_offset();
1722	if (test__start_subtest("link-iter"))
1723		test_link_iter();
1724	if (test__start_subtest("ksym"))
1725		test_ksym_iter();
1726	if (test__start_subtest("bpf_sockmap_map_iter_fd"))
1727		test_bpf_sockmap_map_iter_fd();
1728	if (test__start_subtest("vma_offset"))
1729		test_task_vma_offset();
1730}