1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
   3#include <stdio.h>
   4#include <stdlib.h>
   5#include <sys/socket.h>
   6#include <sys/ioctl.h>
   7#include <sys/select.h>
   8#include <netinet/in.h>
   9#include <arpa/inet.h>
  10#include <unistd.h>
  11#include <string.h>
  12#include <errno.h>
  13#include <stdbool.h>
  14#include <signal.h>
  15#include <fcntl.h>
  16#include <sys/wait.h>
  17#include <time.h>
  18#include <sched.h>
  19
  20#include <sys/time.h>
  21#include <sys/types.h>
  22#include <sys/sendfile.h>
  23
  24#include <linux/netlink.h>
  25#include <linux/socket.h>
  26#include <linux/sock_diag.h>
  27#include <linux/bpf.h>
  28#include <linux/if_link.h>
  29#include <linux/tls.h>
  30#include <assert.h>
  31#include <libgen.h>
  32
  33#include <getopt.h>
  34
  35#include <bpf/bpf.h>
  36#include <bpf/libbpf.h>
  37
  38#include "bpf_util.h"
  39#include "cgroup_helpers.h"
  40
  41int running;
  42static void running_handler(int a);
  43
  44#ifndef TCP_ULP
  45# define TCP_ULP 31
  46#endif
  47#ifndef SOL_TLS
  48# define SOL_TLS 282
  49#endif
  50
  51/* randomly selected ports for testing on lo */
  52#define S1_PORT 10000
  53#define S2_PORT 10001
  54
  55#define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.bpf.o"
  56#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
  57#define CG_PATH "/sockmap"
  58
  59#define EDATAINTEGRITY 2001
  60
  61/* global sockets */
  62int s1, s2, c1, c2, p1, p2;
  63int test_cnt;
  64int passed;
  65int failed;
  66int map_fd[9];
  67struct bpf_map *maps[9];
  68struct bpf_program *progs[9];
  69struct bpf_link *links[9];
  70
  71int txmsg_pass;
  72int txmsg_redir;
  73int txmsg_drop;
  74int txmsg_apply;
  75int txmsg_cork;
  76int txmsg_start;
  77int txmsg_end;
  78int txmsg_start_push;
  79int txmsg_end_push;
  80int txmsg_start_pop;
  81int txmsg_pop;
  82int txmsg_ingress;
  83int txmsg_redir_skb;
  84int txmsg_ktls_skb;
  85int txmsg_ktls_skb_drop;
  86int txmsg_ktls_skb_redir;
  87int ktls;
  88int peek_flag;
  89int skb_use_parser;
  90int txmsg_omit_skb_parser;
  91int verify_push_start;
  92int verify_push_len;
  93int verify_pop_start;
  94int verify_pop_len;
  95
  96static const struct option long_options[] = {
  97	{"help",	no_argument,		NULL, 'h' },
  98	{"cgroup",	required_argument,	NULL, 'c' },
  99	{"rate",	required_argument,	NULL, 'r' },
 100	{"verbose",	optional_argument,	NULL, 'v' },
 101	{"iov_count",	required_argument,	NULL, 'i' },
 102	{"length",	required_argument,	NULL, 'l' },
 103	{"test",	required_argument,	NULL, 't' },
 104	{"data_test",   no_argument,		NULL, 'd' },
 105	{"txmsg",		no_argument,	&txmsg_pass,  1  },
 106	{"txmsg_redir",		no_argument,	&txmsg_redir, 1  },
 107	{"txmsg_drop",		no_argument,	&txmsg_drop, 1 },
 108	{"txmsg_apply",	required_argument,	NULL, 'a'},
 109	{"txmsg_cork",	required_argument,	NULL, 'k'},
 110	{"txmsg_start", required_argument,	NULL, 's'},
 111	{"txmsg_end",	required_argument,	NULL, 'e'},
 112	{"txmsg_start_push", required_argument,	NULL, 'p'},
 113	{"txmsg_end_push",   required_argument,	NULL, 'q'},
 114	{"txmsg_start_pop",  required_argument,	NULL, 'w'},
 115	{"txmsg_pop",	     required_argument,	NULL, 'x'},
 116	{"txmsg_ingress", no_argument,		&txmsg_ingress, 1 },
 117	{"txmsg_redir_skb", no_argument,	&txmsg_redir_skb, 1 },
 118	{"ktls", no_argument,			&ktls, 1 },
 119	{"peek", no_argument,			&peek_flag, 1 },
 120	{"txmsg_omit_skb_parser", no_argument,      &txmsg_omit_skb_parser, 1},
 121	{"whitelist", required_argument,	NULL, 'n' },
 122	{"blacklist", required_argument,	NULL, 'b' },
 123	{0, 0, NULL, 0 }
 124};
 125
 126struct test_env {
 127	const char *type;
 128	const char *subtest;
 129	const char *prepend;
 130
 131	int test_num;
 132	int subtest_num;
 133
 134	int succ_cnt;
 135	int fail_cnt;
 136	int fail_last;
 137};
 138
 139struct test_env env;
 140
 141struct sockmap_options {
 142	int verbose;
 143	bool base;
 144	bool sendpage;
 145	bool data_test;
 146	bool drop_expected;
 147	bool check_recved_len;
 148	bool tx_wait_mem;
 149	int iov_count;
 150	int iov_length;
 151	int rate;
 152	char *map;
 153	char *whitelist;
 154	char *blacklist;
 155	char *prepend;
 156};
 157
 158struct _test {
 159	char *title;
 160	void (*tester)(int cg_fd, struct sockmap_options *opt);
 161};
 162
 163static void test_start(void)
 164{
 165	env.subtest_num++;
 166}
 167
 168static void test_fail(void)
 169{
 170	env.fail_cnt++;
 171}
 172
 173static void test_pass(void)
 174{
 175	env.succ_cnt++;
 176}
 177
 178static void test_reset(void)
 179{
 180	txmsg_start = txmsg_end = 0;
 181	txmsg_start_pop = txmsg_pop = 0;
 182	txmsg_start_push = txmsg_end_push = 0;
 183	txmsg_pass = txmsg_drop = txmsg_redir = 0;
 184	txmsg_apply = txmsg_cork = 0;
 185	txmsg_ingress = txmsg_redir_skb = 0;
 186	txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
 187	txmsg_omit_skb_parser = 0;
 188	skb_use_parser = 0;
 189}
 190
 191static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
 192{
 193	env.type = o->map;
 194	env.subtest = t->title;
 195	env.prepend = o->prepend;
 196	env.test_num++;
 197	env.subtest_num = 0;
 198	env.fail_last = env.fail_cnt;
 199	test_reset();
 200	return 0;
 201}
 202
 203static void test_end_subtest(void)
 204{
 205	int error = env.fail_cnt - env.fail_last;
 206	int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
 207
 208	if (!error)
 209		test_pass();
 210
 211	fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
 212		env.test_num, env.subtest_num,
 213		!type ? "sockmap" : "sockhash",
 214		env.prepend ? : "",
 215		env.subtest, error ? "FAIL" : "OK");
 216}
 217
 218static void test_print_results(void)
 219{
 220	fprintf(stdout, "Pass: %d Fail: %d\n",
 221		env.succ_cnt, env.fail_cnt);
 222}
 223
 224static void usage(char *argv[])
 225{
 226	int i;
 227
 228	printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
 229	printf(" options:\n");
 230	for (i = 0; long_options[i].name != 0; i++) {
 231		printf(" --%-12s", long_options[i].name);
 232		if (long_options[i].flag != NULL)
 233			printf(" flag (internal value:%d)\n",
 234				*long_options[i].flag);
 235		else
 236			printf(" -%c\n", long_options[i].val);
 237	}
 238	printf("\n");
 239}
 240
 241char *sock_to_string(int s)
 242{
 243	if (s == c1)
 244		return "client1";
 245	else if (s == c2)
 246		return "client2";
 247	else if (s == s1)
 248		return "server1";
 249	else if (s == s2)
 250		return "server2";
 251	else if (s == p1)
 252		return "peer1";
 253	else if (s == p2)
 254		return "peer2";
 255	else
 256		return "unknown";
 257}
 258
 259static int sockmap_init_ktls(int verbose, int s)
 260{
 261	struct tls12_crypto_info_aes_gcm_128 tls_tx = {
 262		.info = {
 263			.version     = TLS_1_2_VERSION,
 264			.cipher_type = TLS_CIPHER_AES_GCM_128,
 265		},
 266	};
 267	struct tls12_crypto_info_aes_gcm_128 tls_rx = {
 268		.info = {
 269			.version     = TLS_1_2_VERSION,
 270			.cipher_type = TLS_CIPHER_AES_GCM_128,
 271		},
 272	};
 273	int so_buf = 6553500;
 274	int err;
 275
 276	err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
 277	if (err) {
 278		fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
 279		return -EINVAL;
 280	}
 281	err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
 282	if (err) {
 283		fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
 284		return -EINVAL;
 285	}
 286	err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
 287	if (err) {
 288		fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
 289		return -EINVAL;
 290	}
 291	err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
 292	if (err) {
 293		fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
 294		return -EINVAL;
 295	}
 296	err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
 297	if (err) {
 298		fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
 299		return -EINVAL;
 300	}
 301
 302	if (verbose)
 303		fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
 304	return 0;
 305}
 306static int sockmap_init_sockets(int verbose)
 307{
 308	int i, err, one = 1;
 309	struct sockaddr_in addr;
 310	int *fds[4] = {&s1, &s2, &c1, &c2};
 311
 312	s1 = s2 = p1 = p2 = c1 = c2 = 0;
 313
 314	/* Init sockets */
 315	for (i = 0; i < 4; i++) {
 316		*fds[i] = socket(AF_INET, SOCK_STREAM, 0);
 317		if (*fds[i] < 0) {
 318			perror("socket s1 failed()");
 319			return errno;
 320		}
 321	}
 322
 323	/* Allow reuse */
 324	for (i = 0; i < 2; i++) {
 325		err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
 326				 (char *)&one, sizeof(one));
 327		if (err) {
 328			perror("setsockopt failed()");
 329			return errno;
 330		}
 331	}
 332
 333	/* Non-blocking sockets */
 334	for (i = 0; i < 2; i++) {
 335		err = ioctl(*fds[i], FIONBIO, (char *)&one);
 336		if (err < 0) {
 337			perror("ioctl s1 failed()");
 338			return errno;
 339		}
 340	}
 341
 342	/* Bind server sockets */
 343	memset(&addr, 0, sizeof(struct sockaddr_in));
 344	addr.sin_family = AF_INET;
 345	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
 346
 347	addr.sin_port = htons(S1_PORT);
 348	err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
 349	if (err < 0) {
 350		perror("bind s1 failed()");
 351		return errno;
 352	}
 353
 354	addr.sin_port = htons(S2_PORT);
 355	err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
 356	if (err < 0) {
 357		perror("bind s2 failed()");
 358		return errno;
 359	}
 360
 361	/* Listen server sockets */
 362	addr.sin_port = htons(S1_PORT);
 363	err = listen(s1, 32);
 364	if (err < 0) {
 365		perror("listen s1 failed()");
 366		return errno;
 367	}
 368
 369	addr.sin_port = htons(S2_PORT);
 370	err = listen(s2, 32);
 371	if (err < 0) {
 372		perror("listen s1 failed()");
 373		return errno;
 374	}
 375
 376	/* Initiate Connect */
 377	addr.sin_port = htons(S1_PORT);
 378	err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
 379	if (err < 0 && errno != EINPROGRESS) {
 380		perror("connect c1 failed()");
 381		return errno;
 382	}
 383
 384	addr.sin_port = htons(S2_PORT);
 385	err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
 386	if (err < 0 && errno != EINPROGRESS) {
 387		perror("connect c2 failed()");
 388		return errno;
 389	} else if (err < 0) {
 390		err = 0;
 391	}
 392
 393	/* Accept Connecrtions */
 394	p1 = accept(s1, NULL, NULL);
 395	if (p1 < 0) {
 396		perror("accept s1 failed()");
 397		return errno;
 398	}
 399
 400	p2 = accept(s2, NULL, NULL);
 401	if (p2 < 0) {
 402		perror("accept s1 failed()");
 403		return errno;
 404	}
 405
 406	if (verbose > 1) {
 407		printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
 408		printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
 409			c1, s1, c2, s2);
 410	}
 411	return 0;
 412}
 413
 414struct msg_stats {
 415	size_t bytes_sent;
 416	size_t bytes_recvd;
 417	struct timespec start;
 418	struct timespec end;
 419};
 420
 421static int msg_loop_sendpage(int fd, int iov_length, int cnt,
 422			     struct msg_stats *s,
 423			     struct sockmap_options *opt)
 424{
 425	bool drop = opt->drop_expected;
 426	unsigned char k = 0;
 427	int i, j, fp;
 428	FILE *file;
 
 429
 430	file = tmpfile();
 431	if (!file) {
 432		perror("create file for sendpage");
 433		return 1;
 434	}
 435	for (i = 0; i < cnt; i++, k = 0) {
 436		for (j = 0; j < iov_length; j++, k++)
 437			fwrite(&k, sizeof(char), 1, file);
 438	}
 439	fflush(file);
 440	fseek(file, 0, SEEK_SET);
 441
 442	fp = fileno(file);
 443
 444	clock_gettime(CLOCK_MONOTONIC, &s->start);
 445	for (i = 0; i < cnt; i++) {
 446		int sent;
 447
 448		errno = 0;
 449		sent = sendfile(fd, fp, NULL, iov_length);
 450
 451		if (!drop && sent < 0) {
 452			perror("sendpage loop error");
 453			fclose(file);
 454			return sent;
 455		} else if (drop && sent >= 0) {
 456			printf("sendpage loop error expected: %i errno %i\n",
 457			       sent, errno);
 458			fclose(file);
 459			return -EIO;
 460		}
 461
 462		if (sent > 0)
 463			s->bytes_sent += sent;
 464	}
 465	clock_gettime(CLOCK_MONOTONIC, &s->end);
 466	fclose(file);
 467	return 0;
 468}
 469
 470static void msg_free_iov(struct msghdr *msg)
 471{
 472	int i;
 473
 474	for (i = 0; i < msg->msg_iovlen; i++)
 475		free(msg->msg_iov[i].iov_base);
 476	free(msg->msg_iov);
 477	msg->msg_iov = NULL;
 478	msg->msg_iovlen = 0;
 479}
 480
 481static int msg_alloc_iov(struct msghdr *msg,
 482			 int iov_count, int iov_length,
 483			 bool data, bool xmit)
 484{
 485	unsigned char k = 0;
 486	struct iovec *iov;
 487	int i;
 488
 489	iov = calloc(iov_count, sizeof(struct iovec));
 490	if (!iov)
 491		return errno;
 492
 493	for (i = 0; i < iov_count; i++) {
 494		unsigned char *d = calloc(iov_length, sizeof(char));
 495
 496		if (!d) {
 497			fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
 498			goto unwind_iov;
 499		}
 500		iov[i].iov_base = d;
 501		iov[i].iov_len = iov_length;
 502
 503		if (data && xmit) {
 504			int j;
 505
 506			for (j = 0; j < iov_length; j++)
 507				d[j] = k++;
 508		}
 509	}
 510
 511	msg->msg_iov = iov;
 512	msg->msg_iovlen = iov_count;
 513
 514	return 0;
 515unwind_iov:
 516	for (i--; i >= 0 ; i--)
 517		free(msg->msg_iov[i].iov_base);
 518	return -ENOMEM;
 519}
 520
 521/* In push or pop test, we need to do some calculations for msg_verify_data */
 522static void msg_verify_date_prep(void)
 523{
 524	int push_range_end = txmsg_start_push + txmsg_end_push - 1;
 525	int pop_range_end = txmsg_start_pop + txmsg_pop - 1;
 526
 527	if (txmsg_end_push && txmsg_pop &&
 528	    txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) {
 529		/* The push range and the pop range overlap */
 530		int overlap_len;
 531
 532		verify_push_start = txmsg_start_push;
 533		verify_pop_start = txmsg_start_pop;
 534		if (txmsg_start_push < txmsg_start_pop)
 535			overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop);
 536		else
 537			overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push);
 538		verify_push_len = max(txmsg_end_push - overlap_len, 0);
 539		verify_pop_len = max(txmsg_pop - overlap_len, 0);
 540	} else {
 541		/* Otherwise */
 542		verify_push_start = txmsg_start_push;
 543		verify_pop_start = txmsg_start_pop;
 544		verify_push_len = txmsg_end_push;
 545		verify_pop_len = txmsg_pop;
 546	}
 547}
 548
 549static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz,
 550			   unsigned char *k_p, int *bytes_cnt_p,
 551			   int *check_cnt_p, int *push_p)
 552{
 553	int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p;
 554	unsigned char k = *k_p;
 555	int i, j;
 556
 557	for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) {
 558		unsigned char *d = msg->msg_iov[i].iov_base;
 559
 560		/* Special case test for skb ingress + ktls */
 561		if (i == 0 && txmsg_ktls_skb) {
 562			if (msg->msg_iov[i].iov_len < 4)
 563				return -EDATAINTEGRITY;
 564			if (memcmp(d, "PASS", 4) != 0) {
 565				fprintf(stderr,
 566					"detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
 567					i, 0, d[0], d[1], d[2], d[3]);
 568				return -EDATAINTEGRITY;
 569			}
 570			j = 4; /* advance index past PASS header */
 571		}
 572
 573		for (; j < msg->msg_iov[i].iov_len && size; j++) {
 574			if (push > 0 &&
 575			    check_cnt == verify_push_start + verify_push_len - push) {
 576				int skipped;
 577revisit_push:
 578				skipped = push;
 579				if (j + push >= msg->msg_iov[i].iov_len)
 580					skipped = msg->msg_iov[i].iov_len - j;
 581				push -= skipped;
 582				size -= skipped;
 583				j += skipped - 1;
 584				check_cnt += skipped;
 585				continue;
 586			}
 587
 588			if (verify_pop_len > 0 && check_cnt == verify_pop_start) {
 589				bytes_cnt += verify_pop_len;
 590				check_cnt += verify_pop_len;
 591				k += verify_pop_len;
 592
 593				if (bytes_cnt == chunk_sz) {
 594					k = 0;
 595					bytes_cnt = 0;
 596					check_cnt = 0;
 597					push = verify_push_len;
 598				}
 599
 600				if (push > 0 &&
 601				    check_cnt == verify_push_start + verify_push_len - push)
 602					goto revisit_push;
 603			}
 604
 605			if (d[j] != k++) {
 606				fprintf(stderr,
 607					"detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
 608					i, j, d[j], k - 1, d[j+1], k);
 609				return -EDATAINTEGRITY;
 610			}
 611			bytes_cnt++;
 612			check_cnt++;
 613			if (bytes_cnt == chunk_sz) {
 614				k = 0;
 615				bytes_cnt = 0;
 616				check_cnt = 0;
 617				push = verify_push_len;
 618			}
 619			size--;
 620		}
 621	}
 622	*k_p = k;
 623	*bytes_cnt_p = bytes_cnt;
 624	*check_cnt_p = check_cnt;
 625	*push_p = push;
 626	return 0;
 627}
 628
 629static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
 630		    struct msg_stats *s, bool tx,
 631		    struct sockmap_options *opt)
 632{
 633	struct msghdr msg = {0}, msg_peek = {0};
 634	int err, i, flags = MSG_NOSIGNAL;
 635	bool drop = opt->drop_expected;
 636	bool data = opt->data_test;
 637	int iov_alloc_length = iov_length;
 638
 639	if (!tx && opt->check_recved_len)
 640		iov_alloc_length *= 2;
 641
 642	err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
 643	if (err)
 644		goto out_errno;
 645	if (peek_flag) {
 646		err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
 647		if (err)
 648			goto out_errno;
 649	}
 650
 651	if (tx) {
 652		clock_gettime(CLOCK_MONOTONIC, &s->start);
 653		for (i = 0; i < cnt; i++) {
 654			int sent;
 655
 656			errno = 0;
 657			sent = sendmsg(fd, &msg, flags);
 658
 659			if (!drop && sent < 0) {
 660				if (opt->tx_wait_mem && errno == EACCES) {
 661					errno = 0;
 662					goto out_errno;
 663				}
 664				perror("sendmsg loop error");
 665				goto out_errno;
 666			} else if (drop && sent >= 0) {
 667				fprintf(stderr,
 668					"sendmsg loop error expected: %i errno %i\n",
 669					sent, errno);
 670				errno = -EIO;
 671				goto out_errno;
 672			}
 673			if (sent > 0)
 674				s->bytes_sent += sent;
 675		}
 676		clock_gettime(CLOCK_MONOTONIC, &s->end);
 677	} else {
 678		float total_bytes, txmsg_pop_total, txmsg_push_total;
 679		int slct, recvp = 0, recv, max_fd = fd;
 
 680		int fd_flags = O_NONBLOCK;
 681		struct timeval timeout;
 682		unsigned char k = 0;
 683		int bytes_cnt = 0;
 684		int check_cnt = 0;
 685		int push = 0;
 686		fd_set w;
 687
 688		fcntl(fd, fd_flags);
 689		/* Account for pop bytes noting each iteration of apply will
 690		 * call msg_pop_data helper so we need to account for this
 691		 * by calculating the number of apply iterations. Note user
 692		 * of the tool can create cases where no data is sent by
 693		 * manipulating pop/push/pull/etc. For example txmsg_apply 1
 694		 * with txmsg_pop 1 will try to apply 1B at a time but each
 695		 * iteration will then pop 1B so no data will ever be sent.
 696		 * This is really only useful for testing edge cases in code
 697		 * paths.
 698		 */
 699		total_bytes = (float)iov_length * (float)cnt;
 700		if (!opt->sendpage)
 701			total_bytes *= (float)iov_count;
 702		if (txmsg_apply) {
 703			txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply);
 704			txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
 705		} else {
 706			txmsg_push_total = txmsg_end_push * cnt;
 707			txmsg_pop_total = txmsg_pop * cnt;
 708		}
 709		total_bytes += txmsg_push_total;
 710		total_bytes -= txmsg_pop_total;
 711		if (data) {
 712			msg_verify_date_prep();
 713			push = verify_push_len;
 714		}
 715		err = clock_gettime(CLOCK_MONOTONIC, &s->start);
 716		if (err < 0)
 717			perror("recv start time");
 718		while (s->bytes_recvd < total_bytes) {
 719			if (txmsg_cork) {
 720				timeout.tv_sec = 0;
 721				timeout.tv_usec = 300000;
 722			} else {
 723				timeout.tv_sec = 3;
 724				timeout.tv_usec = 0;
 725			}
 726
 727			/* FD sets */
 728			FD_ZERO(&w);
 729			FD_SET(fd, &w);
 730
 731			slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
 732			if (slct == -1) {
 733				perror("select()");
 734				clock_gettime(CLOCK_MONOTONIC, &s->end);
 735				goto out_errno;
 736			} else if (!slct) {
 737				if (opt->verbose)
 738					fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
 739				errno = -EIO;
 740				clock_gettime(CLOCK_MONOTONIC, &s->end);
 741				goto out_errno;
 742			}
 743
 744			if (opt->tx_wait_mem) {
 745				FD_ZERO(&w);
 746				FD_SET(fd, &w);
 747				slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
 748				errno = 0;
 749				close(fd);
 750				goto out_errno;
 751			}
 752
 753			errno = 0;
 754			if (peek_flag) {
 755				flags |= MSG_PEEK;
 756				recvp = recvmsg(fd, &msg_peek, flags);
 757				if (recvp < 0) {
 758					if (errno != EWOULDBLOCK) {
 759						clock_gettime(CLOCK_MONOTONIC, &s->end);
 760						goto out_errno;
 761					}
 762				}
 763				flags = 0;
 764			}
 765
 766			recv = recvmsg(fd, &msg, flags);
 767			if (recv < 0) {
 768				if (errno != EWOULDBLOCK) {
 769					clock_gettime(CLOCK_MONOTONIC, &s->end);
 770					perror("recv failed()");
 771					goto out_errno;
 772				}
 773			}
 774
 775			if (recv > 0)
 776				s->bytes_recvd += recv;
 777
 778			if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
 779				errno = EMSGSIZE;
 780				fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
 781						s->bytes_recvd, total_bytes);
 782				goto out_errno;
 783			}
 784
 785			if (data) {
 786				int chunk_sz = opt->sendpage ?
 787						iov_length :
 788						iov_length * iov_count;
 789
 790				errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt,
 791							&check_cnt, &push);
 792				if (errno) {
 793					perror("data verify msg failed");
 794					goto out_errno;
 795				}
 796				if (recvp) {
 797					errno = msg_verify_data(&msg_peek,
 798								recvp,
 799								chunk_sz,
 800								&k,
 801								&bytes_cnt,
 802								&check_cnt,
 803								&push);
 804					if (errno) {
 805						perror("data verify msg_peek failed");
 806						goto out_errno;
 807					}
 808				}
 809			}
 810		}
 811		clock_gettime(CLOCK_MONOTONIC, &s->end);
 812	}
 813
 814	msg_free_iov(&msg);
 815	msg_free_iov(&msg_peek);
 816	return err;
 817out_errno:
 818	msg_free_iov(&msg);
 819	msg_free_iov(&msg_peek);
 820	return errno;
 821}
 822
 823static float giga = 1000000000;
 824
 825static inline float sentBps(struct msg_stats s)
 826{
 827	return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
 828}
 829
 830static inline float recvdBps(struct msg_stats s)
 831{
 832	return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
 833}
 834
 835static int sendmsg_test(struct sockmap_options *opt)
 836{
 837	float sent_Bps = 0, recvd_Bps = 0;
 838	int rx_fd, txpid, rxpid, err = 0;
 839	struct msg_stats s = {0};
 840	int iov_count = opt->iov_count;
 841	int iov_buf = opt->iov_length;
 842	int rx_status, tx_status;
 843	int cnt = opt->rate;
 844
 845	errno = 0;
 846
 847	if (opt->base)
 848		rx_fd = p1;
 849	else
 850		rx_fd = p2;
 851
 852	if (ktls) {
 853		/* Redirecting into non-TLS socket which sends into a TLS
 854		 * socket is not a valid test. So in this case lets not
 855		 * enable kTLS but still run the test.
 856		 */
 857		if (!txmsg_redir || txmsg_ingress) {
 858			err = sockmap_init_ktls(opt->verbose, rx_fd);
 859			if (err)
 860				return err;
 861		}
 862		err = sockmap_init_ktls(opt->verbose, c1);
 863		if (err)
 864			return err;
 865	}
 866
 867	if (opt->tx_wait_mem) {
 868		struct timeval timeout;
 869		int rxtx_buf_len = 1024;
 870
 871		timeout.tv_sec = 3;
 872		timeout.tv_usec = 0;
 873
 874		err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
 875		err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
 876		err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
 877		if (err) {
 878			perror("setsockopt failed()");
 879			return errno;
 880		}
 881	}
 882
 883	rxpid = fork();
 884	if (rxpid == 0) {
 
 
 885		if (opt->drop_expected || txmsg_ktls_skb_drop)
 886			_exit(0);
 887
 888		if (!iov_buf) /* zero bytes sent case */
 889			_exit(0);
 890
 891		if (opt->sendpage)
 892			iov_count = 1;
 893		err = msg_loop(rx_fd, iov_count, iov_buf,
 894			       cnt, &s, false, opt);
 895		if (opt->verbose > 1)
 896			fprintf(stderr,
 897				"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
 898				iov_count, iov_buf, cnt, err);
 899		if (s.end.tv_sec - s.start.tv_sec) {
 900			sent_Bps = sentBps(s);
 901			recvd_Bps = recvdBps(s);
 902		}
 903		if (opt->verbose > 1)
 904			fprintf(stdout,
 905				"rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
 906				s.bytes_sent, sent_Bps, sent_Bps/giga,
 907				s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
 908				peek_flag ? "(peek_msg)" : "");
 909		if (err && err != -EDATAINTEGRITY && txmsg_cork)
 910			err = 0;
 911		exit(err ? 1 : 0);
 912	} else if (rxpid == -1) {
 913		perror("msg_loop_rx");
 914		return errno;
 915	}
 916
 917	if (opt->tx_wait_mem)
 918		close(c2);
 919
 920	txpid = fork();
 921	if (txpid == 0) {
 922		if (opt->sendpage)
 923			err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
 924		else
 925			err = msg_loop(c1, iov_count, iov_buf,
 926				       cnt, &s, true, opt);
 927
 928		if (err)
 929			fprintf(stderr,
 930				"msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
 931				iov_count, iov_buf, cnt, err);
 932		if (s.end.tv_sec - s.start.tv_sec) {
 933			sent_Bps = sentBps(s);
 934			recvd_Bps = recvdBps(s);
 935		}
 936		if (opt->verbose > 1)
 937			fprintf(stdout,
 938				"tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
 939				s.bytes_sent, sent_Bps, sent_Bps/giga,
 940				s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
 941		exit(err ? 1 : 0);
 942	} else if (txpid == -1) {
 943		perror("msg_loop_tx");
 944		return errno;
 945	}
 946
 947	assert(waitpid(rxpid, &rx_status, 0) == rxpid);
 948	assert(waitpid(txpid, &tx_status, 0) == txpid);
 949	if (WIFEXITED(rx_status)) {
 950		err = WEXITSTATUS(rx_status);
 951		if (err) {
 952			fprintf(stderr, "rx thread exited with err %d.\n", err);
 953			goto out;
 954		}
 955	}
 956	if (WIFEXITED(tx_status)) {
 957		err = WEXITSTATUS(tx_status);
 958		if (err)
 959			fprintf(stderr, "tx thread exited with err %d.\n", err);
 960	}
 961out:
 962	return err;
 963}
 964
 965static int forever_ping_pong(int rate, struct sockmap_options *opt)
 966{
 967	struct timeval timeout;
 968	char buf[1024] = {0};
 969	int sc;
 970
 971	timeout.tv_sec = 10;
 972	timeout.tv_usec = 0;
 973
 974	/* Ping/Pong data from client to server */
 975	sc = send(c1, buf, sizeof(buf), 0);
 976	if (sc < 0) {
 977		perror("send failed()");
 978		return sc;
 979	}
 980
 981	do {
 982		int s, rc, i, max_fd = p2;
 983		fd_set w;
 984
 985		/* FD sets */
 986		FD_ZERO(&w);
 987		FD_SET(c1, &w);
 988		FD_SET(c2, &w);
 989		FD_SET(p1, &w);
 990		FD_SET(p2, &w);
 991
 992		s = select(max_fd + 1, &w, NULL, NULL, &timeout);
 993		if (s == -1) {
 994			perror("select()");
 995			break;
 996		} else if (!s) {
 997			fprintf(stderr, "unexpected timeout\n");
 998			break;
 999		}
1000
1001		for (i = 0; i <= max_fd && s > 0; ++i) {
1002			if (!FD_ISSET(i, &w))
1003				continue;
1004
1005			s--;
1006
1007			rc = recv(i, buf, sizeof(buf), 0);
1008			if (rc < 0) {
1009				if (errno != EWOULDBLOCK) {
1010					perror("recv failed()");
1011					return rc;
1012				}
1013			}
1014
1015			if (rc == 0) {
1016				close(i);
1017				break;
1018			}
1019
1020			sc = send(i, buf, rc, 0);
1021			if (sc < 0) {
1022				perror("send failed()");
1023				return sc;
1024			}
1025		}
1026
1027		if (rate)
1028			sleep(rate);
1029
1030		if (opt->verbose) {
1031			printf(".");
1032			fflush(stdout);
1033
1034		}
1035	} while (running);
1036
1037	return 0;
1038}
1039
1040enum {
1041	SELFTESTS,
1042	PING_PONG,
1043	SENDMSG,
1044	BASE,
1045	BASE_SENDPAGE,
1046	SENDPAGE,
1047};
1048
1049static int run_options(struct sockmap_options *options, int cg_fd,  int test)
1050{
1051	int i, key, next_key, err, zero = 0;
1052	struct bpf_program *tx_prog;
1053
1054	/* If base test skip BPF setup */
1055	if (test == BASE || test == BASE_SENDPAGE)
1056		goto run;
1057
1058	/* Attach programs to sockmap */
1059	if (!txmsg_omit_skb_parser) {
1060		links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]);
1061		if (!links[0]) {
 
1062			fprintf(stderr,
1063				"ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n",
1064				bpf_program__fd(progs[0]), map_fd[0], strerror(errno));
1065			return -1;
1066		}
1067	}
1068
1069	links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]);
1070	if (!links[1]) {
1071		fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n",
1072			strerror(errno));
1073		return -1;
 
1074	}
1075
1076	/* Attach programs to TLS sockmap */
1077	if (txmsg_ktls_skb) {
1078		if (!txmsg_omit_skb_parser) {
1079			links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]);
1080			if (!links[2]) {
 
1081				fprintf(stderr,
1082					"ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n",
1083					bpf_program__fd(progs[0]), map_fd[8], strerror(errno));
1084				return -1;
1085			}
1086		}
1087
1088		links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]);
1089		if (!links[3]) {
1090			fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n",
1091				strerror(errno));
1092			return -1;
 
1093		}
1094	}
1095
1096	/* Attach to cgroups */
1097	err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1098	if (err) {
1099		fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1100			err, strerror(errno));
1101		return err;
1102	}
1103
1104run:
1105	err = sockmap_init_sockets(options->verbose);
1106	if (err) {
1107		fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1108		goto out;
1109	}
1110
1111	/* Attach txmsg program to sockmap */
1112	if (txmsg_pass)
1113		tx_prog = progs[4];
1114	else if (txmsg_redir)
1115		tx_prog = progs[5];
1116	else if (txmsg_apply)
1117		tx_prog = progs[6];
1118	else if (txmsg_cork)
1119		tx_prog = progs[7];
1120	else if (txmsg_drop)
1121		tx_prog = progs[8];
1122	else
1123		tx_prog = NULL;
1124
1125	if (tx_prog) {
1126		int redir_fd;
1127
1128		links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]);
1129		if (!links[4]) {
 
1130			fprintf(stderr,
1131				"ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n",
1132				strerror(errno));
1133			err = -1;
1134			goto out;
1135		}
1136
1137		i = 0;
1138		err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1139		if (err) {
1140			fprintf(stderr,
1141				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1142				err, strerror(errno));
1143			goto out;
1144		}
1145
1146		if (txmsg_redir)
1147			redir_fd = c2;
1148		else
1149			redir_fd = c1;
1150
1151		err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1152		if (err) {
1153			fprintf(stderr,
1154				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1155				err, strerror(errno));
1156			goto out;
1157		}
1158
1159		if (txmsg_apply) {
1160			err = bpf_map_update_elem(map_fd[3],
1161						  &i, &txmsg_apply, BPF_ANY);
1162			if (err) {
1163				fprintf(stderr,
1164					"ERROR: bpf_map_update_elem (apply_bytes):  %d (%s\n",
1165					err, strerror(errno));
1166				goto out;
1167			}
1168		}
1169
1170		if (txmsg_cork) {
1171			err = bpf_map_update_elem(map_fd[4],
1172						  &i, &txmsg_cork, BPF_ANY);
1173			if (err) {
1174				fprintf(stderr,
1175					"ERROR: bpf_map_update_elem (cork_bytes):  %d (%s\n",
1176					err, strerror(errno));
1177				goto out;
1178			}
1179		}
1180
1181		if (txmsg_start) {
1182			err = bpf_map_update_elem(map_fd[5],
1183						  &i, &txmsg_start, BPF_ANY);
1184			if (err) {
1185				fprintf(stderr,
1186					"ERROR: bpf_map_update_elem (txmsg_start):  %d (%s)\n",
1187					err, strerror(errno));
1188				goto out;
1189			}
1190		}
1191
1192		if (txmsg_end) {
1193			i = 1;
1194			err = bpf_map_update_elem(map_fd[5],
1195						  &i, &txmsg_end, BPF_ANY);
1196			if (err) {
1197				fprintf(stderr,
1198					"ERROR: bpf_map_update_elem (txmsg_end):  %d (%s)\n",
1199					err, strerror(errno));
1200				goto out;
1201			}
1202		}
1203
1204		if (txmsg_start_push) {
1205			i = 2;
1206			err = bpf_map_update_elem(map_fd[5],
1207						  &i, &txmsg_start_push, BPF_ANY);
1208			if (err) {
1209				fprintf(stderr,
1210					"ERROR: bpf_map_update_elem (txmsg_start_push):  %d (%s)\n",
1211					err, strerror(errno));
1212				goto out;
1213			}
1214		}
1215
1216		if (txmsg_end_push) {
1217			i = 3;
1218			err = bpf_map_update_elem(map_fd[5],
1219						  &i, &txmsg_end_push, BPF_ANY);
1220			if (err) {
1221				fprintf(stderr,
1222					"ERROR: bpf_map_update_elem %i@%i (txmsg_end_push):  %d (%s)\n",
1223					txmsg_end_push, i, err, strerror(errno));
1224				goto out;
1225			}
1226		}
1227
1228		if (txmsg_start_pop) {
1229			i = 4;
1230			err = bpf_map_update_elem(map_fd[5],
1231						  &i, &txmsg_start_pop, BPF_ANY);
1232			if (err) {
1233				fprintf(stderr,
1234					"ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop):  %d (%s)\n",
1235					txmsg_start_pop, i, err, strerror(errno));
1236				goto out;
1237			}
1238		} else {
1239			i = 4;
1240			bpf_map_update_elem(map_fd[5],
1241						  &i, &txmsg_start_pop, BPF_ANY);
1242		}
1243
1244		if (txmsg_pop) {
1245			i = 5;
1246			err = bpf_map_update_elem(map_fd[5],
1247						  &i, &txmsg_pop, BPF_ANY);
1248			if (err) {
1249				fprintf(stderr,
1250					"ERROR: bpf_map_update_elem %i@%i (txmsg_pop):  %d (%s)\n",
1251					txmsg_pop, i, err, strerror(errno));
1252				goto out;
1253			}
1254		} else {
1255			i = 5;
1256			bpf_map_update_elem(map_fd[5],
1257					    &i, &txmsg_pop, BPF_ANY);
1258
1259		}
1260
1261		if (txmsg_ingress) {
1262			int in = BPF_F_INGRESS;
1263
1264			i = 0;
1265			err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1266			if (err) {
1267				fprintf(stderr,
1268					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1269					err, strerror(errno));
1270			}
1271			i = 1;
1272			err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1273			if (err) {
1274				fprintf(stderr,
1275					"ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1276					err, strerror(errno));
1277			}
1278			err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1279			if (err) {
1280				fprintf(stderr,
1281					"ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1282					err, strerror(errno));
1283			}
1284
1285			i = 2;
1286			err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1287			if (err) {
1288				fprintf(stderr,
1289					"ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1290					err, strerror(errno));
1291			}
1292		}
1293
1294		if (txmsg_ktls_skb) {
1295			int ingress = BPF_F_INGRESS;
1296
1297			i = 0;
1298			err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1299			if (err) {
1300				fprintf(stderr,
1301					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1302					err, strerror(errno));
1303			}
1304
1305			if (txmsg_ktls_skb_redir) {
1306				i = 1;
1307				err = bpf_map_update_elem(map_fd[7],
1308							  &i, &ingress, BPF_ANY);
1309				if (err) {
1310					fprintf(stderr,
1311						"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1312						err, strerror(errno));
1313				}
1314			}
1315
1316			if (txmsg_ktls_skb_drop) {
1317				i = 1;
1318				err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1319			}
1320		}
1321
1322		if (txmsg_redir_skb) {
1323			int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1324					p2 : p1;
1325			int ingress = BPF_F_INGRESS;
1326
1327			i = 0;
1328			err = bpf_map_update_elem(map_fd[7],
1329						  &i, &ingress, BPF_ANY);
1330			if (err) {
1331				fprintf(stderr,
1332					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1333					err, strerror(errno));
1334			}
1335
1336			i = 3;
1337			err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1338			if (err) {
1339				fprintf(stderr,
1340					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1341					err, strerror(errno));
1342			}
1343		}
1344	}
1345
1346	if (skb_use_parser) {
1347		i = 2;
1348		err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1349	}
1350
1351	if (txmsg_drop)
1352		options->drop_expected = true;
1353
1354	if (test == PING_PONG)
1355		err = forever_ping_pong(options->rate, options);
1356	else if (test == SENDMSG) {
1357		options->base = false;
1358		options->sendpage = false;
1359		err = sendmsg_test(options);
1360	} else if (test == SENDPAGE) {
1361		options->base = false;
1362		options->sendpage = true;
1363		err = sendmsg_test(options);
1364	} else if (test == BASE) {
1365		options->base = true;
1366		options->sendpage = false;
1367		err = sendmsg_test(options);
1368	} else if (test == BASE_SENDPAGE) {
1369		options->base = true;
1370		options->sendpage = true;
1371		err = sendmsg_test(options);
1372	} else
1373		fprintf(stderr, "unknown test\n");
1374out:
1375	/* Detatch and zero all the maps */
1376	bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS);
 
 
 
 
1377
1378	for (i = 0; i < ARRAY_SIZE(links); i++) {
1379		if (links[i])
1380			bpf_link__detach(links[i]);
1381	}
1382
1383	for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1384		key = next_key = 0;
1385		bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1386		while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1387			bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1388			key = next_key;
1389		}
1390	}
1391
1392	close(s1);
1393	close(s2);
1394	close(p1);
1395	close(p2);
1396	close(c1);
1397	close(c2);
1398	return err;
1399}
1400
1401static char *test_to_str(int test)
1402{
1403	switch (test) {
1404	case SENDMSG:
1405		return "sendmsg";
1406	case SENDPAGE:
1407		return "sendpage";
1408	}
1409	return "unknown";
1410}
1411
1412static void append_str(char *dst, const char *src, size_t dst_cap)
1413{
1414	size_t avail = dst_cap - strlen(dst);
1415
1416	if (avail <= 1) /* just zero byte could be written */
1417		return;
1418
1419	strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1420}
1421
1422#define OPTSTRING 60
1423static void test_options(char *options)
1424{
1425	char tstr[OPTSTRING];
1426
1427	memset(options, 0, OPTSTRING);
1428
1429	if (txmsg_pass)
1430		append_str(options, "pass,", OPTSTRING);
1431	if (txmsg_redir)
1432		append_str(options, "redir,", OPTSTRING);
1433	if (txmsg_drop)
1434		append_str(options, "drop,", OPTSTRING);
1435	if (txmsg_apply) {
1436		snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1437		append_str(options, tstr, OPTSTRING);
1438	}
1439	if (txmsg_cork) {
1440		snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1441		append_str(options, tstr, OPTSTRING);
1442	}
1443	if (txmsg_start) {
1444		snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1445		append_str(options, tstr, OPTSTRING);
1446	}
1447	if (txmsg_end) {
1448		snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1449		append_str(options, tstr, OPTSTRING);
1450	}
1451	if (txmsg_start_pop) {
1452		snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1453			 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1454		append_str(options, tstr, OPTSTRING);
1455	}
1456	if (txmsg_ingress)
1457		append_str(options, "ingress,", OPTSTRING);
1458	if (txmsg_redir_skb)
1459		append_str(options, "redir_skb,", OPTSTRING);
1460	if (txmsg_ktls_skb)
1461		append_str(options, "ktls_skb,", OPTSTRING);
1462	if (ktls)
1463		append_str(options, "ktls,", OPTSTRING);
1464	if (peek_flag)
1465		append_str(options, "peek,", OPTSTRING);
1466}
1467
1468static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1469{
1470	char *options = calloc(OPTSTRING, sizeof(char));
1471	int err;
1472
1473	if (test == SENDPAGE)
1474		opt->sendpage = true;
1475	else
1476		opt->sendpage = false;
1477
1478	if (txmsg_drop)
1479		opt->drop_expected = true;
1480	else
1481		opt->drop_expected = false;
1482
1483	test_options(options);
1484
1485	if (opt->verbose) {
1486		fprintf(stdout,
1487			" [TEST %i]: (%i, %i, %i, %s, %s): ",
1488			test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1489			test_to_str(test), options);
1490		fflush(stdout);
1491	}
1492	err = run_options(opt, cgrp, test);
1493	if (opt->verbose)
1494		fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1495	test_cnt++;
1496	!err ? passed++ : failed++;
1497	free(options);
1498	return err;
1499}
1500
1501static void test_exec(int cgrp, struct sockmap_options *opt)
1502{
1503	int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1504	int err;
1505
1506	if (type == 0) {
1507		test_start();
1508		err = __test_exec(cgrp, SENDMSG, opt);
1509		if (err)
1510			test_fail();
1511	} else {
1512		test_start();
1513		err = __test_exec(cgrp, SENDPAGE, opt);
1514		if (err)
1515			test_fail();
1516	}
1517}
1518
1519static void test_send_one(struct sockmap_options *opt, int cgrp)
1520{
1521	opt->iov_length = 1;
1522	opt->iov_count = 1;
1523	opt->rate = 1;
1524	test_exec(cgrp, opt);
1525
1526	opt->iov_length = 1;
1527	opt->iov_count = 1024;
1528	opt->rate = 1;
1529	test_exec(cgrp, opt);
1530
1531	opt->iov_length = 1024;
1532	opt->iov_count = 1;
1533	opt->rate = 1;
1534	test_exec(cgrp, opt);
1535
1536}
1537
1538static void test_send_many(struct sockmap_options *opt, int cgrp)
1539{
1540	opt->iov_length = 3;
1541	opt->iov_count = 1;
1542	opt->rate = 512;
1543	test_exec(cgrp, opt);
1544
1545	opt->rate = 100;
1546	opt->iov_count = 1;
1547	opt->iov_length = 5;
1548	test_exec(cgrp, opt);
1549}
1550
1551static void test_send_large(struct sockmap_options *opt, int cgrp)
1552{
1553	opt->iov_length = 8192;
1554	opt->iov_count = 32;
1555	opt->rate = 2;
1556	test_exec(cgrp, opt);
1557}
1558
1559static void test_send(struct sockmap_options *opt, int cgrp)
1560{
1561	test_send_one(opt, cgrp);
1562	test_send_many(opt, cgrp);
1563	test_send_large(opt, cgrp);
1564	sched_yield();
1565}
1566
1567static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1568{
1569	/* Test small and large iov_count values with pass/redir/apply/cork */
1570	txmsg_pass = 1;
1571	test_send(opt, cgrp);
1572}
1573
1574static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1575{
1576	txmsg_redir = 1;
1577	test_send(opt, cgrp);
1578}
1579
1580static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1581{
1582	opt->tx_wait_mem = true;
1583	txmsg_redir = 1;
1584	test_send_large(opt, cgrp);
1585
1586	txmsg_redir = 1;
1587	txmsg_apply = 4097;
1588	test_send_large(opt, cgrp);
1589	opt->tx_wait_mem = false;
1590}
1591
1592static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1593{
1594	txmsg_drop = 1;
1595	test_send(opt, cgrp);
1596}
1597
1598static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1599{
1600	txmsg_pass = txmsg_drop = 0;
1601	txmsg_ingress = txmsg_redir = 1;
1602	test_send(opt, cgrp);
1603}
1604
1605static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1606{
1607	bool data = opt->data_test;
1608	int k = ktls;
1609
1610	opt->data_test = true;
1611	ktls = 1;
1612
1613	txmsg_pass = txmsg_drop = 0;
1614	txmsg_ingress = txmsg_redir = 0;
1615	txmsg_ktls_skb = 1;
1616	txmsg_pass = 1;
1617
1618	/* Using data verification so ensure iov layout is
1619	 * expected from test receiver side. e.g. has enough
1620	 * bytes to write test code.
1621	 */
1622	opt->iov_length = 100;
1623	opt->iov_count = 1;
1624	opt->rate = 1;
1625	test_exec(cgrp, opt);
1626
1627	txmsg_ktls_skb_drop = 1;
1628	test_exec(cgrp, opt);
1629
1630	txmsg_ktls_skb_drop = 0;
1631	txmsg_ktls_skb_redir = 1;
1632	test_exec(cgrp, opt);
1633	txmsg_ktls_skb_redir = 0;
1634
1635	/* Tests that omit skb_parser */
1636	txmsg_omit_skb_parser = 1;
1637	ktls = 0;
1638	txmsg_ktls_skb = 0;
1639	test_exec(cgrp, opt);
1640
1641	txmsg_ktls_skb_drop = 1;
1642	test_exec(cgrp, opt);
1643	txmsg_ktls_skb_drop = 0;
1644
1645	txmsg_ktls_skb_redir = 1;
1646	test_exec(cgrp, opt);
1647
1648	ktls = 1;
1649	test_exec(cgrp, opt);
1650	txmsg_omit_skb_parser = 0;
1651
1652	opt->data_test = data;
1653	ktls = k;
1654}
1655
1656/* Test cork with hung data. This tests poor usage patterns where
1657 * cork can leave data on the ring if user program is buggy and
1658 * doesn't flush them somehow. They do take some time however
1659 * because they wait for a timeout. Test pass, redir and cork with
1660 * apply logic. Use cork size of 4097 with send_large to avoid
1661 * aligning cork size with send size.
1662 */
1663static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1664{
1665	txmsg_pass = 1;
1666	txmsg_redir = 0;
1667	txmsg_cork = 4097;
1668	txmsg_apply = 4097;
1669	test_send_large(opt, cgrp);
1670
1671	txmsg_pass = 0;
1672	txmsg_redir = 1;
1673	txmsg_apply = 0;
1674	txmsg_cork = 4097;
1675	test_send_large(opt, cgrp);
1676
1677	txmsg_pass = 0;
1678	txmsg_redir = 1;
1679	txmsg_apply = 4097;
1680	txmsg_cork = 4097;
1681	test_send_large(opt, cgrp);
1682}
1683
1684static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1685{
1686	/* Test basic start/end */
1687	txmsg_pass = 1;
1688	txmsg_start = 1;
1689	txmsg_end = 2;
1690	test_send(opt, cgrp);
1691
1692	/* Test >4k pull */
1693	txmsg_pass = 1;
1694	txmsg_start = 4096;
1695	txmsg_end = 9182;
1696	test_send_large(opt, cgrp);
1697
1698	/* Test pull + redirect */
1699	txmsg_redir = 1;
1700	txmsg_start = 1;
1701	txmsg_end = 2;
1702	test_send(opt, cgrp);
1703
1704	/* Test pull + cork */
1705	txmsg_redir = 0;
1706	txmsg_cork = 512;
1707	txmsg_start = 1;
1708	txmsg_end = 2;
1709	test_send_many(opt, cgrp);
1710
1711	/* Test pull + cork + redirect */
1712	txmsg_redir = 1;
1713	txmsg_cork = 512;
1714	txmsg_start = 1;
1715	txmsg_end = 2;
1716	test_send_many(opt, cgrp);
1717}
1718
1719static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1720{
1721	bool data = opt->data_test;
1722
1723	/* Test basic pop */
1724	txmsg_pass = 1;
1725	txmsg_start_pop = 1;
1726	txmsg_pop = 2;
1727	test_send_many(opt, cgrp);
1728
1729	/* Test pop with >4k */
1730	txmsg_pass = 1;
1731	txmsg_start_pop = 4096;
1732	txmsg_pop = 4096;
1733	test_send_large(opt, cgrp);
1734
1735	/* Test pop + redirect */
1736	txmsg_redir = 1;
1737	txmsg_start_pop = 1;
1738	txmsg_pop = 2;
1739	test_send_many(opt, cgrp);
1740
1741	/* TODO: Test for pop + cork should be different,
1742	 * - It makes the layout of the received data difficult
1743	 * - It makes it hard to calculate the total_bytes in the recvmsg
1744	 * Temporarily skip the data integrity test for this case now.
1745	 */
1746	opt->data_test = false;
1747	/* Test pop + cork */
1748	txmsg_redir = 0;
1749	txmsg_cork = 512;
1750	txmsg_start_pop = 1;
1751	txmsg_pop = 2;
1752	test_send_many(opt, cgrp);
1753
1754	/* Test pop + redirect + cork */
1755	txmsg_redir = 1;
1756	txmsg_cork = 4;
1757	txmsg_start_pop = 1;
1758	txmsg_pop = 2;
1759	test_send_many(opt, cgrp);
1760	opt->data_test = data;
1761}
1762
1763static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1764{
1765	bool data = opt->data_test;
1766
1767	/* Test basic push */
1768	txmsg_pass = 1;
1769	txmsg_start_push = 1;
1770	txmsg_end_push = 1;
1771	test_send(opt, cgrp);
1772
1773	/* Test push 4kB >4k */
1774	txmsg_pass = 1;
1775	txmsg_start_push = 4096;
1776	txmsg_end_push = 4096;
1777	test_send_large(opt, cgrp);
1778
1779	/* Test push + redirect */
1780	txmsg_redir = 1;
1781	txmsg_start_push = 1;
1782	txmsg_end_push = 2;
1783	test_send_many(opt, cgrp);
1784
1785	/* TODO: Test for push + cork should be different,
1786	 * - It makes the layout of the received data difficult
1787	 * - It makes it hard to calculate the total_bytes in the recvmsg
1788	 * Temporarily skip the data integrity test for this case now.
1789	 */
1790	opt->data_test = false;
1791	/* Test push + cork */
1792	txmsg_redir = 0;
1793	txmsg_cork = 512;
1794	txmsg_start_push = 1;
1795	txmsg_end_push = 2;
1796	test_send_many(opt, cgrp);
1797	opt->data_test = data;
1798}
1799
1800static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1801{
1802	/* Test push/pop range overlapping */
1803	txmsg_pass = 1;
1804	txmsg_start_push = 1;
1805	txmsg_end_push = 10;
1806	txmsg_start_pop = 5;
1807	txmsg_pop = 4;
1808	test_send_large(opt, cgrp);
1809
1810	txmsg_pass = 1;
1811	txmsg_start_push = 1;
1812	txmsg_end_push = 10;
1813	txmsg_start_pop = 5;
1814	txmsg_pop = 16;
1815	test_send_large(opt, cgrp);
1816
1817	txmsg_pass = 1;
1818	txmsg_start_push = 5;
1819	txmsg_end_push = 4;
1820	txmsg_start_pop = 1;
1821	txmsg_pop = 10;
1822	test_send_large(opt, cgrp);
1823
1824	txmsg_pass = 1;
1825	txmsg_start_push = 5;
1826	txmsg_end_push = 16;
1827	txmsg_start_pop = 1;
1828	txmsg_pop = 10;
1829	test_send_large(opt, cgrp);
1830
1831	/* Test push/pop range non-overlapping */
1832	txmsg_pass = 1;
1833	txmsg_start_push = 1;
1834	txmsg_end_push = 10;
1835	txmsg_start_pop = 16;
1836	txmsg_pop = 4;
1837	test_send_large(opt, cgrp);
1838
1839	txmsg_pass = 1;
1840	txmsg_start_push = 16;
1841	txmsg_end_push = 10;
1842	txmsg_start_pop = 5;
1843	txmsg_pop = 4;
1844	test_send_large(opt, cgrp);
1845}
1846
1847static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1848{
1849	txmsg_pass = 1;
1850	txmsg_redir = 0;
1851	txmsg_ingress = 0;
1852	txmsg_apply = 1;
1853	txmsg_cork = 0;
1854	test_send_one(opt, cgrp);
1855
1856	txmsg_pass = 0;
1857	txmsg_redir = 1;
1858	txmsg_ingress = 0;
1859	txmsg_apply = 1;
1860	txmsg_cork = 0;
1861	test_send_one(opt, cgrp);
1862
1863	txmsg_pass = 0;
1864	txmsg_redir = 1;
1865	txmsg_ingress = 1;
1866	txmsg_apply = 1;
1867	txmsg_cork = 0;
1868	test_send_one(opt, cgrp);
1869
1870	txmsg_pass = 1;
1871	txmsg_redir = 0;
1872	txmsg_ingress = 0;
1873	txmsg_apply = 1024;
1874	txmsg_cork = 0;
1875	test_send_large(opt, cgrp);
1876
1877	txmsg_pass = 0;
1878	txmsg_redir = 1;
1879	txmsg_ingress = 0;
1880	txmsg_apply = 1024;
1881	txmsg_cork = 0;
1882	test_send_large(opt, cgrp);
1883
1884	txmsg_pass = 0;
1885	txmsg_redir = 1;
1886	txmsg_ingress = 1;
1887	txmsg_apply = 1024;
1888	txmsg_cork = 0;
1889	test_send_large(opt, cgrp);
1890}
1891
1892static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1893{
1894	txmsg_pass = 1;
1895	txmsg_redir = 0;
1896	txmsg_apply = 0;
1897	txmsg_cork = 1;
1898	test_send(opt, cgrp);
1899
1900	txmsg_pass = 1;
1901	txmsg_redir = 0;
1902	txmsg_apply = 1;
1903	txmsg_cork = 1;
1904	test_send(opt, cgrp);
1905}
1906
1907static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1908{
1909	txmsg_pass = 1;
1910	skb_use_parser = 512;
1911	if (ktls == 1)
1912		skb_use_parser = 570;
1913	opt->iov_length = 256;
1914	opt->iov_count = 1;
1915	opt->rate = 2;
1916	test_exec(cgrp, opt);
1917}
1918
1919static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1920{
1921	if (ktls == 1)
1922		return;
1923	skb_use_parser = 10;
1924	opt->iov_length = 20;
1925	opt->iov_count = 1;
1926	opt->rate = 1;
1927	opt->check_recved_len = true;
1928	test_exec(cgrp, opt);
1929	opt->check_recved_len = false;
1930}
1931
1932char *map_names[] = {
1933	"sock_map",
1934	"sock_map_txmsg",
1935	"sock_map_redir",
1936	"sock_apply_bytes",
1937	"sock_cork_bytes",
1938	"sock_bytes",
1939	"sock_redir_flags",
1940	"sock_skb_opts",
1941	"tls_sock_map",
1942};
1943
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1944static int populate_progs(char *bpf_file)
1945{
1946	struct bpf_program *prog;
1947	struct bpf_object *obj;
1948	int i = 0;
1949	long err;
1950
1951	obj = bpf_object__open(bpf_file);
1952	err = libbpf_get_error(obj);
1953	if (err) {
1954		char err_buf[256];
1955
1956		libbpf_strerror(err, err_buf, sizeof(err_buf));
1957		printf("Unable to load eBPF objects in file '%s' : %s\n",
1958		       bpf_file, err_buf);
1959		return -1;
1960	}
1961
 
 
 
 
 
 
 
1962	i = bpf_object__load(obj);
1963	i = 0;
1964	bpf_object__for_each_program(prog, obj) {
1965		progs[i] = prog;
1966		i++;
1967	}
1968
1969	for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1970		maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1971		map_fd[i] = bpf_map__fd(maps[i]);
1972		if (map_fd[i] < 0) {
1973			fprintf(stderr, "load_bpf_file: (%i) %s\n",
1974				map_fd[i], strerror(errno));
1975			return -1;
1976		}
1977	}
1978
1979	for (i = 0; i < ARRAY_SIZE(links); i++)
1980		links[i] = NULL;
1981
1982	return 0;
1983}
1984
1985struct _test test[] = {
1986	{"txmsg test passthrough", test_txmsg_pass},
1987	{"txmsg test redirect", test_txmsg_redir},
1988	{"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1989	{"txmsg test drop", test_txmsg_drop},
1990	{"txmsg test ingress redirect", test_txmsg_ingress_redir},
1991	{"txmsg test skb", test_txmsg_skb},
1992	{"txmsg test apply", test_txmsg_apply},
1993	{"txmsg test cork", test_txmsg_cork},
1994	{"txmsg test hanging corks", test_txmsg_cork_hangs},
1995	{"txmsg test push_data", test_txmsg_push},
1996	{"txmsg test pull-data", test_txmsg_pull},
1997	{"txmsg test pop-data", test_txmsg_pop},
1998	{"txmsg test push/pop data", test_txmsg_push_pop},
1999	{"txmsg test ingress parser", test_txmsg_ingress_parser},
2000	{"txmsg test ingress parser2", test_txmsg_ingress_parser2},
2001};
2002
2003static int check_whitelist(struct _test *t, struct sockmap_options *opt)
2004{
2005	char *entry, *ptr;
2006
2007	if (!opt->whitelist)
2008		return 0;
2009	ptr = strdup(opt->whitelist);
2010	if (!ptr)
2011		return -ENOMEM;
2012	entry = strtok(ptr, ",");
2013	while (entry) {
2014		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
2015		    strstr(opt->map, entry) != 0 ||
2016		    strstr(t->title, entry) != 0) {
2017			free(ptr);
2018			return 0;
2019		}
2020		entry = strtok(NULL, ",");
2021	}
2022	free(ptr);
2023	return -EINVAL;
2024}
2025
2026static int check_blacklist(struct _test *t, struct sockmap_options *opt)
2027{
2028	char *entry, *ptr;
2029
2030	if (!opt->blacklist)
2031		return -EINVAL;
2032	ptr = strdup(opt->blacklist);
2033	if (!ptr)
2034		return -ENOMEM;
2035	entry = strtok(ptr, ",");
2036	while (entry) {
2037		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
2038		    strstr(opt->map, entry) != 0 ||
2039		    strstr(t->title, entry) != 0) {
2040			free(ptr);
2041			return 0;
2042		}
2043		entry = strtok(NULL, ",");
2044	}
2045	free(ptr);
2046	return -EINVAL;
2047}
2048
2049static int __test_selftests(int cg_fd, struct sockmap_options *opt)
2050{
2051	int i, err;
2052
2053	err = populate_progs(opt->map);
2054	if (err < 0) {
2055		fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
2056		return err;
2057	}
2058
2059	/* Tests basic commands and APIs */
2060	for (i = 0; i < ARRAY_SIZE(test); i++) {
2061		struct _test t = test[i];
2062
2063		if (check_whitelist(&t, opt) != 0)
2064			continue;
2065		if (check_blacklist(&t, opt) == 0)
2066			continue;
2067
2068		test_start_subtest(&t, opt);
2069		t.tester(cg_fd, opt);
2070		test_end_subtest();
2071	}
2072
2073	return err;
2074}
2075
2076static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
2077{
2078	opt->map = BPF_SOCKMAP_FILENAME;
2079	__test_selftests(cg_fd, opt);
2080}
2081
2082static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
2083{
2084	opt->map = BPF_SOCKHASH_FILENAME;
2085	__test_selftests(cg_fd, opt);
2086}
2087
2088static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
2089{
2090	opt->map = BPF_SOCKHASH_FILENAME;
2091	opt->prepend = "ktls";
2092	ktls = 1;
2093	__test_selftests(cg_fd, opt);
2094	ktls = 0;
2095}
2096
2097static int test_selftest(int cg_fd, struct sockmap_options *opt)
2098{
 
2099	test_selftests_sockmap(cg_fd, opt);
2100	test_selftests_sockhash(cg_fd, opt);
2101	test_selftests_ktls(cg_fd, opt);
2102	test_print_results();
2103	return 0;
2104}
2105
2106int main(int argc, char **argv)
2107{
2108	int iov_count = 1, length = 1024, rate = 1;
2109	struct sockmap_options options = {0};
2110	int opt, longindex, err, cg_fd = 0;
2111	char *bpf_file = BPF_SOCKMAP_FILENAME;
2112	int test = SELFTESTS;
2113	bool cg_created = 0;
2114
2115	while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
2116				  long_options, &longindex)) != -1) {
2117		switch (opt) {
2118		case 's':
2119			txmsg_start = atoi(optarg);
2120			break;
2121		case 'e':
2122			txmsg_end = atoi(optarg);
2123			break;
2124		case 'p':
2125			txmsg_start_push = atoi(optarg);
2126			break;
2127		case 'q':
2128			txmsg_end_push = atoi(optarg);
2129			break;
2130		case 'w':
2131			txmsg_start_pop = atoi(optarg);
2132			break;
2133		case 'x':
2134			txmsg_pop = atoi(optarg);
2135			break;
2136		case 'a':
2137			txmsg_apply = atoi(optarg);
2138			break;
2139		case 'k':
2140			txmsg_cork = atoi(optarg);
2141			break;
2142		case 'c':
2143			cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
2144			if (cg_fd < 0) {
2145				fprintf(stderr,
2146					"ERROR: (%i) open cg path failed: %s\n",
2147					cg_fd, optarg);
2148				return cg_fd;
2149			}
2150			break;
2151		case 'r':
2152			rate = atoi(optarg);
2153			break;
2154		case 'v':
2155			options.verbose = 1;
2156			if (optarg)
2157				options.verbose = atoi(optarg);
2158			break;
2159		case 'i':
2160			iov_count = atoi(optarg);
2161			break;
2162		case 'l':
2163			length = atoi(optarg);
2164			break;
2165		case 'd':
2166			options.data_test = true;
2167			break;
2168		case 't':
2169			if (strcmp(optarg, "ping") == 0) {
2170				test = PING_PONG;
2171			} else if (strcmp(optarg, "sendmsg") == 0) {
2172				test = SENDMSG;
2173			} else if (strcmp(optarg, "base") == 0) {
2174				test = BASE;
2175			} else if (strcmp(optarg, "base_sendpage") == 0) {
2176				test = BASE_SENDPAGE;
2177			} else if (strcmp(optarg, "sendpage") == 0) {
2178				test = SENDPAGE;
2179			} else {
2180				usage(argv);
2181				return -1;
2182			}
2183			break;
2184		case 'n':
2185			options.whitelist = strdup(optarg);
2186			if (!options.whitelist)
2187				return -ENOMEM;
2188			break;
2189		case 'b':
2190			options.blacklist = strdup(optarg);
2191			if (!options.blacklist)
2192				return -ENOMEM;
2193		case 0:
2194			break;
2195		case 'h':
2196		default:
2197			usage(argv);
2198			return -1;
2199		}
2200	}
2201
2202	if (!cg_fd) {
2203		cg_fd = cgroup_setup_and_join(CG_PATH);
2204		if (cg_fd < 0)
2205			return cg_fd;
2206		cg_created = 1;
2207	}
2208
2209	/* Use libbpf 1.0 API mode */
2210	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2211
2212	if (test == SELFTESTS) {
2213		err = test_selftest(cg_fd, &options);
2214		goto out;
2215	}
2216
2217	err = populate_progs(bpf_file);
2218	if (err) {
2219		fprintf(stderr, "populate program: (%s) %s\n",
2220			bpf_file, strerror(errno));
2221		return 1;
2222	}
2223	running = 1;
2224
2225	/* catch SIGINT */
2226	signal(SIGINT, running_handler);
2227
2228	options.iov_count = iov_count;
2229	options.iov_length = length;
2230	options.rate = rate;
2231
2232	err = run_options(&options, cg_fd, test);
2233out:
2234	if (options.whitelist)
2235		free(options.whitelist);
2236	if (options.blacklist)
2237		free(options.blacklist);
2238	close(cg_fd);
2239	if (cg_created)
2240		cleanup_cgroup_environment();
 
2241	return err;
2242}
2243
2244void running_handler(int a)
2245{
2246	running = 0;
2247}