1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
   3#include <stdio.h>
   4#include <stdlib.h>
   5#include <sys/socket.h>
   6#include <sys/ioctl.h>
   7#include <sys/select.h>
   8#include <netinet/in.h>
   9#include <arpa/inet.h>
  10#include <unistd.h>
  11#include <string.h>
  12#include <errno.h>
  13#include <stdbool.h>
  14#include <signal.h>
  15#include <fcntl.h>
  16#include <sys/wait.h>
  17#include <time.h>
  18#include <sched.h>
  19
  20#include <sys/time.h>
 
  21#include <sys/types.h>
  22#include <sys/sendfile.h>
  23
  24#include <linux/netlink.h>
  25#include <linux/socket.h>
  26#include <linux/sock_diag.h>
  27#include <linux/bpf.h>
  28#include <linux/if_link.h>
  29#include <linux/tls.h>
  30#include <assert.h>
  31#include <libgen.h>
  32
  33#include <getopt.h>
  34
  35#include <bpf/bpf.h>
  36#include <bpf/libbpf.h>
  37
  38#include "bpf_util.h"
 
  39#include "cgroup_helpers.h"
  40
  41int running;
  42static void running_handler(int a);
  43
  44#ifndef TCP_ULP
  45# define TCP_ULP 31
  46#endif
  47#ifndef SOL_TLS
  48# define SOL_TLS 282
  49#endif
  50
  51/* randomly selected ports for testing on lo */
  52#define S1_PORT 10000
  53#define S2_PORT 10001
  54
  55#define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.bpf.o"
  56#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
  57#define CG_PATH "/sockmap"
  58
  59/* global sockets */
  60int s1, s2, c1, c2, p1, p2;
  61int test_cnt;
  62int passed;
  63int failed;
  64int map_fd[9];
  65struct bpf_map *maps[9];
  66int prog_fd[11];
  67
  68int txmsg_pass;
 
  69int txmsg_redir;
 
  70int txmsg_drop;
  71int txmsg_apply;
  72int txmsg_cork;
  73int txmsg_start;
  74int txmsg_end;
  75int txmsg_start_push;
  76int txmsg_end_push;
  77int txmsg_start_pop;
  78int txmsg_pop;
  79int txmsg_ingress;
  80int txmsg_redir_skb;
  81int txmsg_ktls_skb;
  82int txmsg_ktls_skb_drop;
  83int txmsg_ktls_skb_redir;
  84int ktls;
  85int peek_flag;
  86int skb_use_parser;
  87int txmsg_omit_skb_parser;
  88
  89static const struct option long_options[] = {
  90	{"help",	no_argument,		NULL, 'h' },
  91	{"cgroup",	required_argument,	NULL, 'c' },
  92	{"rate",	required_argument,	NULL, 'r' },
  93	{"verbose",	optional_argument,	NULL, 'v' },
  94	{"iov_count",	required_argument,	NULL, 'i' },
  95	{"length",	required_argument,	NULL, 'l' },
  96	{"test",	required_argument,	NULL, 't' },
  97	{"data_test",   no_argument,		NULL, 'd' },
  98	{"txmsg",		no_argument,	&txmsg_pass,  1  },
 
  99	{"txmsg_redir",		no_argument,	&txmsg_redir, 1  },
 
 100	{"txmsg_drop",		no_argument,	&txmsg_drop, 1 },
 101	{"txmsg_apply",	required_argument,	NULL, 'a'},
 102	{"txmsg_cork",	required_argument,	NULL, 'k'},
 103	{"txmsg_start", required_argument,	NULL, 's'},
 104	{"txmsg_end",	required_argument,	NULL, 'e'},
 105	{"txmsg_start_push", required_argument,	NULL, 'p'},
 106	{"txmsg_end_push",   required_argument,	NULL, 'q'},
 107	{"txmsg_start_pop",  required_argument,	NULL, 'w'},
 108	{"txmsg_pop",	     required_argument,	NULL, 'x'},
 109	{"txmsg_ingress", no_argument,		&txmsg_ingress, 1 },
 110	{"txmsg_redir_skb", no_argument,	&txmsg_redir_skb, 1 },
 111	{"ktls", no_argument,			&ktls, 1 },
 112	{"peek", no_argument,			&peek_flag, 1 },
 113	{"txmsg_omit_skb_parser", no_argument,      &txmsg_omit_skb_parser, 1},
 114	{"whitelist", required_argument,	NULL, 'n' },
 115	{"blacklist", required_argument,	NULL, 'b' },
 116	{0, 0, NULL, 0 }
 117};
 118
 119struct test_env {
 120	const char *type;
 121	const char *subtest;
 122	const char *prepend;
 123
 124	int test_num;
 125	int subtest_num;
 126
 127	int succ_cnt;
 128	int fail_cnt;
 129	int fail_last;
 130};
 131
 132struct test_env env;
 133
 134struct sockmap_options {
 135	int verbose;
 136	bool base;
 137	bool sendpage;
 138	bool data_test;
 139	bool drop_expected;
 140	bool check_recved_len;
 141	bool tx_wait_mem;
 142	int iov_count;
 143	int iov_length;
 144	int rate;
 145	char *map;
 146	char *whitelist;
 147	char *blacklist;
 148	char *prepend;
 149};
 150
 151struct _test {
 152	char *title;
 153	void (*tester)(int cg_fd, struct sockmap_options *opt);
 154};
 155
 156static void test_start(void)
 157{
 158	env.subtest_num++;
 159}
 160
 161static void test_fail(void)
 162{
 163	env.fail_cnt++;
 164}
 165
 166static void test_pass(void)
 167{
 168	env.succ_cnt++;
 169}
 170
 171static void test_reset(void)
 172{
 173	txmsg_start = txmsg_end = 0;
 174	txmsg_start_pop = txmsg_pop = 0;
 175	txmsg_start_push = txmsg_end_push = 0;
 176	txmsg_pass = txmsg_drop = txmsg_redir = 0;
 177	txmsg_apply = txmsg_cork = 0;
 178	txmsg_ingress = txmsg_redir_skb = 0;
 179	txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
 180	txmsg_omit_skb_parser = 0;
 181	skb_use_parser = 0;
 182}
 183
 184static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
 185{
 186	env.type = o->map;
 187	env.subtest = t->title;
 188	env.prepend = o->prepend;
 189	env.test_num++;
 190	env.subtest_num = 0;
 191	env.fail_last = env.fail_cnt;
 192	test_reset();
 193	return 0;
 194}
 195
 196static void test_end_subtest(void)
 197{
 198	int error = env.fail_cnt - env.fail_last;
 199	int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
 200
 201	if (!error)
 202		test_pass();
 203
 204	fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
 205		env.test_num, env.subtest_num,
 206		!type ? "sockmap" : "sockhash",
 207		env.prepend ? : "",
 208		env.subtest, error ? "FAIL" : "OK");
 209}
 210
 211static void test_print_results(void)
 212{
 213	fprintf(stdout, "Pass: %d Fail: %d\n",
 214		env.succ_cnt, env.fail_cnt);
 215}
 216
 217static void usage(char *argv[])
 218{
 219	int i;
 220
 221	printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
 222	printf(" options:\n");
 223	for (i = 0; long_options[i].name != 0; i++) {
 224		printf(" --%-12s", long_options[i].name);
 225		if (long_options[i].flag != NULL)
 226			printf(" flag (internal value:%d)\n",
 227				*long_options[i].flag);
 228		else
 229			printf(" -%c\n", long_options[i].val);
 230	}
 231	printf("\n");
 232}
 233
 234char *sock_to_string(int s)
 235{
 236	if (s == c1)
 237		return "client1";
 238	else if (s == c2)
 239		return "client2";
 240	else if (s == s1)
 241		return "server1";
 242	else if (s == s2)
 243		return "server2";
 244	else if (s == p1)
 245		return "peer1";
 246	else if (s == p2)
 247		return "peer2";
 248	else
 249		return "unknown";
 250}
 251
 252static int sockmap_init_ktls(int verbose, int s)
 253{
 254	struct tls12_crypto_info_aes_gcm_128 tls_tx = {
 255		.info = {
 256			.version     = TLS_1_2_VERSION,
 257			.cipher_type = TLS_CIPHER_AES_GCM_128,
 258		},
 259	};
 260	struct tls12_crypto_info_aes_gcm_128 tls_rx = {
 261		.info = {
 262			.version     = TLS_1_2_VERSION,
 263			.cipher_type = TLS_CIPHER_AES_GCM_128,
 264		},
 265	};
 266	int so_buf = 6553500;
 267	int err;
 268
 269	err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
 270	if (err) {
 271		fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
 272		return -EINVAL;
 273	}
 274	err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
 275	if (err) {
 276		fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
 277		return -EINVAL;
 278	}
 279	err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
 280	if (err) {
 281		fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
 282		return -EINVAL;
 283	}
 284	err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
 285	if (err) {
 286		fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
 287		return -EINVAL;
 288	}
 289	err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
 290	if (err) {
 291		fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
 292		return -EINVAL;
 293	}
 294
 295	if (verbose)
 296		fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
 297	return 0;
 298}
 299static int sockmap_init_sockets(int verbose)
 300{
 301	int i, err, one = 1;
 302	struct sockaddr_in addr;
 303	int *fds[4] = {&s1, &s2, &c1, &c2};
 304
 305	s1 = s2 = p1 = p2 = c1 = c2 = 0;
 306
 307	/* Init sockets */
 308	for (i = 0; i < 4; i++) {
 309		*fds[i] = socket(AF_INET, SOCK_STREAM, 0);
 310		if (*fds[i] < 0) {
 311			perror("socket s1 failed()");
 312			return errno;
 313		}
 314	}
 315
 316	/* Allow reuse */
 317	for (i = 0; i < 2; i++) {
 318		err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
 319				 (char *)&one, sizeof(one));
 320		if (err) {
 321			perror("setsockopt failed()");
 322			return errno;
 323		}
 324	}
 325
 326	/* Non-blocking sockets */
 327	for (i = 0; i < 2; i++) {
 328		err = ioctl(*fds[i], FIONBIO, (char *)&one);
 329		if (err < 0) {
 330			perror("ioctl s1 failed()");
 331			return errno;
 332		}
 333	}
 334
 335	/* Bind server sockets */
 336	memset(&addr, 0, sizeof(struct sockaddr_in));
 337	addr.sin_family = AF_INET;
 338	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
 339
 340	addr.sin_port = htons(S1_PORT);
 341	err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
 342	if (err < 0) {
 343		perror("bind s1 failed()");
 344		return errno;
 345	}
 346
 347	addr.sin_port = htons(S2_PORT);
 348	err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
 349	if (err < 0) {
 350		perror("bind s2 failed()");
 351		return errno;
 352	}
 353
 354	/* Listen server sockets */
 355	addr.sin_port = htons(S1_PORT);
 356	err = listen(s1, 32);
 357	if (err < 0) {
 358		perror("listen s1 failed()");
 359		return errno;
 360	}
 361
 362	addr.sin_port = htons(S2_PORT);
 363	err = listen(s2, 32);
 364	if (err < 0) {
 365		perror("listen s1 failed()");
 366		return errno;
 367	}
 368
 369	/* Initiate Connect */
 370	addr.sin_port = htons(S1_PORT);
 371	err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
 372	if (err < 0 && errno != EINPROGRESS) {
 373		perror("connect c1 failed()");
 374		return errno;
 375	}
 376
 377	addr.sin_port = htons(S2_PORT);
 378	err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
 379	if (err < 0 && errno != EINPROGRESS) {
 380		perror("connect c2 failed()");
 381		return errno;
 382	} else if (err < 0) {
 383		err = 0;
 384	}
 385
 386	/* Accept Connecrtions */
 387	p1 = accept(s1, NULL, NULL);
 388	if (p1 < 0) {
 389		perror("accept s1 failed()");
 390		return errno;
 391	}
 392
 393	p2 = accept(s2, NULL, NULL);
 394	if (p2 < 0) {
 395		perror("accept s1 failed()");
 396		return errno;
 397	}
 398
 399	if (verbose > 1) {
 400		printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
 401		printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
 402			c1, s1, c2, s2);
 403	}
 404	return 0;
 405}
 406
 407struct msg_stats {
 408	size_t bytes_sent;
 409	size_t bytes_recvd;
 410	struct timespec start;
 411	struct timespec end;
 412};
 413
 
 
 
 
 
 
 
 
 
 
 
 414static int msg_loop_sendpage(int fd, int iov_length, int cnt,
 415			     struct msg_stats *s,
 416			     struct sockmap_options *opt)
 417{
 418	bool drop = opt->drop_expected;
 419	unsigned char k = 0;
 420	FILE *file;
 421	int i, fp;
 422
 423	file = tmpfile();
 424	if (!file) {
 425		perror("create file for sendpage");
 426		return 1;
 427	}
 428	for (i = 0; i < iov_length * cnt; i++, k++)
 429		fwrite(&k, sizeof(char), 1, file);
 430	fflush(file);
 431	fseek(file, 0, SEEK_SET);
 
 432
 433	fp = fileno(file);
 434
 435	clock_gettime(CLOCK_MONOTONIC, &s->start);
 436	for (i = 0; i < cnt; i++) {
 437		int sent;
 438
 439		errno = 0;
 440		sent = sendfile(fd, fp, NULL, iov_length);
 441
 442		if (!drop && sent < 0) {
 443			perror("sendpage loop error");
 444			fclose(file);
 445			return sent;
 446		} else if (drop && sent >= 0) {
 447			printf("sendpage loop error expected: %i errno %i\n",
 448			       sent, errno);
 449			fclose(file);
 450			return -EIO;
 451		}
 452
 453		if (sent > 0)
 454			s->bytes_sent += sent;
 455	}
 456	clock_gettime(CLOCK_MONOTONIC, &s->end);
 457	fclose(file);
 458	return 0;
 459}
 460
 461static void msg_free_iov(struct msghdr *msg)
 462{
 463	int i;
 464
 465	for (i = 0; i < msg->msg_iovlen; i++)
 466		free(msg->msg_iov[i].iov_base);
 467	free(msg->msg_iov);
 468	msg->msg_iov = NULL;
 469	msg->msg_iovlen = 0;
 470}
 471
 472static int msg_alloc_iov(struct msghdr *msg,
 473			 int iov_count, int iov_length,
 474			 bool data, bool xmit)
 475{
 476	unsigned char k = 0;
 477	struct iovec *iov;
 478	int i;
 479
 480	iov = calloc(iov_count, sizeof(struct iovec));
 481	if (!iov)
 482		return errno;
 483
 484	for (i = 0; i < iov_count; i++) {
 485		unsigned char *d = calloc(iov_length, sizeof(char));
 486
 487		if (!d) {
 488			fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
 489			goto unwind_iov;
 490		}
 491		iov[i].iov_base = d;
 492		iov[i].iov_len = iov_length;
 493
 494		if (data && xmit) {
 495			int j;
 496
 497			for (j = 0; j < iov_length; j++)
 498				d[j] = k++;
 499		}
 500	}
 501
 502	msg->msg_iov = iov;
 503	msg->msg_iovlen = iov_count;
 504
 505	return 0;
 506unwind_iov:
 507	for (i--; i >= 0 ; i--)
 508		free(msg->msg_iov[i].iov_base);
 509	return -ENOMEM;
 510}
 511
 512static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
 513{
 514	int i, j = 0, bytes_cnt = 0;
 515	unsigned char k = 0;
 516
 517	for (i = 0; i < msg->msg_iovlen; i++) {
 518		unsigned char *d = msg->msg_iov[i].iov_base;
 519
 520		/* Special case test for skb ingress + ktls */
 521		if (i == 0 && txmsg_ktls_skb) {
 522			if (msg->msg_iov[i].iov_len < 4)
 523				return -EIO;
 524			if (memcmp(d, "PASS", 4) != 0) {
 525				fprintf(stderr,
 526					"detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
 527					i, 0, d[0], d[1], d[2], d[3]);
 528				return -EIO;
 529			}
 530			j = 4; /* advance index past PASS header */
 531		}
 532
 533		for (; j < msg->msg_iov[i].iov_len && size; j++) {
 534			if (d[j] != k++) {
 535				fprintf(stderr,
 536					"detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
 537					i, j, d[j], k - 1, d[j+1], k);
 538				return -EIO;
 539			}
 540			bytes_cnt++;
 541			if (bytes_cnt == chunk_sz) {
 542				k = 0;
 543				bytes_cnt = 0;
 544			}
 545			size--;
 546		}
 547	}
 548	return 0;
 549}
 550
 551static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
 552		    struct msg_stats *s, bool tx,
 553		    struct sockmap_options *opt)
 554{
 555	struct msghdr msg = {0}, msg_peek = {0};
 556	int err, i, flags = MSG_NOSIGNAL;
 557	bool drop = opt->drop_expected;
 558	bool data = opt->data_test;
 559	int iov_alloc_length = iov_length;
 560
 561	if (!tx && opt->check_recved_len)
 562		iov_alloc_length *= 2;
 563
 564	err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
 565	if (err)
 566		goto out_errno;
 567	if (peek_flag) {
 568		err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
 569		if (err)
 570			goto out_errno;
 571	}
 572
 573	if (tx) {
 574		clock_gettime(CLOCK_MONOTONIC, &s->start);
 575		for (i = 0; i < cnt; i++) {
 576			int sent;
 577
 578			errno = 0;
 579			sent = sendmsg(fd, &msg, flags);
 580
 581			if (!drop && sent < 0) {
 582				if (opt->tx_wait_mem && errno == EACCES) {
 583					errno = 0;
 584					goto out_errno;
 585				}
 586				perror("sendmsg loop error");
 587				goto out_errno;
 588			} else if (drop && sent >= 0) {
 589				fprintf(stderr,
 590					"sendmsg loop error expected: %i errno %i\n",
 591					sent, errno);
 592				errno = -EIO;
 593				goto out_errno;
 594			}
 595			if (sent > 0)
 596				s->bytes_sent += sent;
 597		}
 598		clock_gettime(CLOCK_MONOTONIC, &s->end);
 599	} else {
 600		int slct, recvp = 0, recv, max_fd = fd;
 601		float total_bytes, txmsg_pop_total;
 602		int fd_flags = O_NONBLOCK;
 603		struct timeval timeout;
 604		fd_set w;
 605
 606		fcntl(fd, fd_flags);
 607		/* Account for pop bytes noting each iteration of apply will
 608		 * call msg_pop_data helper so we need to account for this
 609		 * by calculating the number of apply iterations. Note user
 610		 * of the tool can create cases where no data is sent by
 611		 * manipulating pop/push/pull/etc. For example txmsg_apply 1
 612		 * with txmsg_pop 1 will try to apply 1B at a time but each
 613		 * iteration will then pop 1B so no data will ever be sent.
 614		 * This is really only useful for testing edge cases in code
 615		 * paths.
 616		 */
 617		total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
 
 618		if (txmsg_apply)
 619			txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
 620		else
 621			txmsg_pop_total = txmsg_pop * cnt;
 622		total_bytes -= txmsg_pop_total;
 623		err = clock_gettime(CLOCK_MONOTONIC, &s->start);
 624		if (err < 0)
 625			perror("recv start time");
 626		while (s->bytes_recvd < total_bytes) {
 627			if (txmsg_cork) {
 628				timeout.tv_sec = 0;
 629				timeout.tv_usec = 300000;
 630			} else {
 631				timeout.tv_sec = 3;
 632				timeout.tv_usec = 0;
 633			}
 634
 635			/* FD sets */
 636			FD_ZERO(&w);
 637			FD_SET(fd, &w);
 638
 639			slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
 640			if (slct == -1) {
 641				perror("select()");
 642				clock_gettime(CLOCK_MONOTONIC, &s->end);
 643				goto out_errno;
 644			} else if (!slct) {
 645				if (opt->verbose)
 646					fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
 647				errno = -EIO;
 648				clock_gettime(CLOCK_MONOTONIC, &s->end);
 649				goto out_errno;
 650			}
 651
 652			if (opt->tx_wait_mem) {
 653				FD_ZERO(&w);
 654				FD_SET(fd, &w);
 655				slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
 656				errno = 0;
 657				close(fd);
 658				goto out_errno;
 659			}
 660
 661			errno = 0;
 662			if (peek_flag) {
 663				flags |= MSG_PEEK;
 664				recvp = recvmsg(fd, &msg_peek, flags);
 665				if (recvp < 0) {
 666					if (errno != EWOULDBLOCK) {
 667						clock_gettime(CLOCK_MONOTONIC, &s->end);
 668						goto out_errno;
 669					}
 670				}
 671				flags = 0;
 672			}
 673
 674			recv = recvmsg(fd, &msg, flags);
 675			if (recv < 0) {
 676				if (errno != EWOULDBLOCK) {
 677					clock_gettime(CLOCK_MONOTONIC, &s->end);
 678					perror("recv failed()");
 679					goto out_errno;
 680				}
 681			}
 682
 683			s->bytes_recvd += recv;
 684
 685			if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
 686				errno = EMSGSIZE;
 687				fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
 688						s->bytes_recvd, total_bytes);
 689				goto out_errno;
 690			}
 691
 692			if (data) {
 693				int chunk_sz = opt->sendpage ?
 694						iov_length * cnt :
 695						iov_length * iov_count;
 696
 697				errno = msg_verify_data(&msg, recv, chunk_sz);
 698				if (errno) {
 699					perror("data verify msg failed");
 700					goto out_errno;
 701				}
 702				if (recvp) {
 703					errno = msg_verify_data(&msg_peek,
 704								recvp,
 705								chunk_sz);
 706					if (errno) {
 707						perror("data verify msg_peek failed");
 708						goto out_errno;
 709					}
 710				}
 711			}
 712		}
 713		clock_gettime(CLOCK_MONOTONIC, &s->end);
 714	}
 715
 716	msg_free_iov(&msg);
 717	msg_free_iov(&msg_peek);
 718	return err;
 719out_errno:
 720	msg_free_iov(&msg);
 721	msg_free_iov(&msg_peek);
 722	return errno;
 723}
 724
 725static float giga = 1000000000;
 726
 727static inline float sentBps(struct msg_stats s)
 728{
 729	return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
 730}
 731
 732static inline float recvdBps(struct msg_stats s)
 733{
 734	return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
 735}
 736
 737static int sendmsg_test(struct sockmap_options *opt)
 738{
 739	float sent_Bps = 0, recvd_Bps = 0;
 740	int rx_fd, txpid, rxpid, err = 0;
 741	struct msg_stats s = {0};
 742	int iov_count = opt->iov_count;
 743	int iov_buf = opt->iov_length;
 744	int rx_status, tx_status;
 745	int cnt = opt->rate;
 746
 747	errno = 0;
 748
 749	if (opt->base)
 750		rx_fd = p1;
 751	else
 752		rx_fd = p2;
 753
 754	if (ktls) {
 755		/* Redirecting into non-TLS socket which sends into a TLS
 756		 * socket is not a valid test. So in this case lets not
 757		 * enable kTLS but still run the test.
 758		 */
 759		if (!txmsg_redir || txmsg_ingress) {
 760			err = sockmap_init_ktls(opt->verbose, rx_fd);
 761			if (err)
 762				return err;
 763		}
 764		err = sockmap_init_ktls(opt->verbose, c1);
 765		if (err)
 766			return err;
 767	}
 768
 769	if (opt->tx_wait_mem) {
 770		struct timeval timeout;
 771		int rxtx_buf_len = 1024;
 772
 773		timeout.tv_sec = 3;
 774		timeout.tv_usec = 0;
 775
 776		err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
 777		err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
 778		err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
 779		if (err) {
 780			perror("setsockopt failed()");
 781			return errno;
 782		}
 783	}
 784
 785	rxpid = fork();
 786	if (rxpid == 0) {
 787		if (txmsg_pop || txmsg_start_pop)
 788			iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
 789		if (opt->drop_expected || txmsg_ktls_skb_drop)
 790			_exit(0);
 791
 792		if (!iov_buf) /* zero bytes sent case */
 793			_exit(0);
 794
 795		if (opt->sendpage)
 796			iov_count = 1;
 797		err = msg_loop(rx_fd, iov_count, iov_buf,
 798			       cnt, &s, false, opt);
 799		if (opt->verbose > 1)
 800			fprintf(stderr,
 801				"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
 802				iov_count, iov_buf, cnt, err);
 803		if (s.end.tv_sec - s.start.tv_sec) {
 804			sent_Bps = sentBps(s);
 805			recvd_Bps = recvdBps(s);
 806		}
 807		if (opt->verbose > 1)
 808			fprintf(stdout,
 809				"rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
 810				s.bytes_sent, sent_Bps, sent_Bps/giga,
 811				s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
 812				peek_flag ? "(peek_msg)" : "");
 813		if (err && txmsg_cork)
 814			err = 0;
 815		exit(err ? 1 : 0);
 816	} else if (rxpid == -1) {
 817		perror("msg_loop_rx");
 818		return errno;
 819	}
 820
 821	if (opt->tx_wait_mem)
 822		close(c2);
 823
 824	txpid = fork();
 825	if (txpid == 0) {
 826		if (opt->sendpage)
 827			err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
 828		else
 829			err = msg_loop(c1, iov_count, iov_buf,
 830				       cnt, &s, true, opt);
 831
 832		if (err)
 833			fprintf(stderr,
 834				"msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
 835				iov_count, iov_buf, cnt, err);
 836		if (s.end.tv_sec - s.start.tv_sec) {
 837			sent_Bps = sentBps(s);
 838			recvd_Bps = recvdBps(s);
 839		}
 840		if (opt->verbose > 1)
 841			fprintf(stdout,
 842				"tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
 843				s.bytes_sent, sent_Bps, sent_Bps/giga,
 844				s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
 845		exit(err ? 1 : 0);
 846	} else if (txpid == -1) {
 847		perror("msg_loop_tx");
 848		return errno;
 849	}
 850
 851	assert(waitpid(rxpid, &rx_status, 0) == rxpid);
 852	assert(waitpid(txpid, &tx_status, 0) == txpid);
 853	if (WIFEXITED(rx_status)) {
 854		err = WEXITSTATUS(rx_status);
 855		if (err) {
 856			fprintf(stderr, "rx thread exited with err %d.\n", err);
 857			goto out;
 858		}
 859	}
 860	if (WIFEXITED(tx_status)) {
 861		err = WEXITSTATUS(tx_status);
 862		if (err)
 863			fprintf(stderr, "tx thread exited with err %d.\n", err);
 864	}
 865out:
 866	return err;
 867}
 868
 869static int forever_ping_pong(int rate, struct sockmap_options *opt)
 870{
 871	struct timeval timeout;
 872	char buf[1024] = {0};
 873	int sc;
 874
 875	timeout.tv_sec = 10;
 876	timeout.tv_usec = 0;
 877
 878	/* Ping/Pong data from client to server */
 879	sc = send(c1, buf, sizeof(buf), 0);
 880	if (sc < 0) {
 881		perror("send failed()");
 882		return sc;
 883	}
 884
 885	do {
 886		int s, rc, i, max_fd = p2;
 887		fd_set w;
 888
 889		/* FD sets */
 890		FD_ZERO(&w);
 891		FD_SET(c1, &w);
 892		FD_SET(c2, &w);
 893		FD_SET(p1, &w);
 894		FD_SET(p2, &w);
 895
 896		s = select(max_fd + 1, &w, NULL, NULL, &timeout);
 897		if (s == -1) {
 898			perror("select()");
 899			break;
 900		} else if (!s) {
 901			fprintf(stderr, "unexpected timeout\n");
 902			break;
 903		}
 904
 905		for (i = 0; i <= max_fd && s > 0; ++i) {
 906			if (!FD_ISSET(i, &w))
 907				continue;
 908
 909			s--;
 910
 911			rc = recv(i, buf, sizeof(buf), 0);
 912			if (rc < 0) {
 913				if (errno != EWOULDBLOCK) {
 914					perror("recv failed()");
 915					return rc;
 916				}
 917			}
 918
 919			if (rc == 0) {
 920				close(i);
 921				break;
 922			}
 923
 924			sc = send(i, buf, rc, 0);
 925			if (sc < 0) {
 926				perror("send failed()");
 927				return sc;
 928			}
 929		}
 930
 931		if (rate)
 932			sleep(rate);
 933
 934		if (opt->verbose) {
 935			printf(".");
 936			fflush(stdout);
 937
 938		}
 939	} while (running);
 940
 941	return 0;
 942}
 943
 944enum {
 945	SELFTESTS,
 946	PING_PONG,
 947	SENDMSG,
 948	BASE,
 949	BASE_SENDPAGE,
 950	SENDPAGE,
 951};
 952
 953static int run_options(struct sockmap_options *options, int cg_fd,  int test)
 954{
 955	int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
 956
 957	/* If base test skip BPF setup */
 958	if (test == BASE || test == BASE_SENDPAGE)
 959		goto run;
 960
 961	/* Attach programs to sockmap */
 962	if (!txmsg_omit_skb_parser) {
 963		err = bpf_prog_attach(prog_fd[0], map_fd[0],
 964				      BPF_SK_SKB_STREAM_PARSER, 0);
 965		if (err) {
 966			fprintf(stderr,
 967				"ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
 968				prog_fd[0], map_fd[0], err, strerror(errno));
 969			return err;
 970		}
 971	}
 972
 973	err = bpf_prog_attach(prog_fd[1], map_fd[0],
 974				BPF_SK_SKB_STREAM_VERDICT, 0);
 975	if (err) {
 976		fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
 977			err, strerror(errno));
 978		return err;
 979	}
 980
 981	/* Attach programs to TLS sockmap */
 982	if (txmsg_ktls_skb) {
 983		if (!txmsg_omit_skb_parser) {
 984			err = bpf_prog_attach(prog_fd[0], map_fd[8],
 985					      BPF_SK_SKB_STREAM_PARSER, 0);
 986			if (err) {
 987				fprintf(stderr,
 988					"ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
 989					prog_fd[0], map_fd[8], err, strerror(errno));
 990				return err;
 991			}
 992		}
 993
 994		err = bpf_prog_attach(prog_fd[2], map_fd[8],
 995				      BPF_SK_SKB_STREAM_VERDICT, 0);
 996		if (err) {
 997			fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
 998				err, strerror(errno));
 999			return err;
1000		}
1001	}
1002
1003	/* Attach to cgroups */
1004	err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1005	if (err) {
1006		fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1007			err, strerror(errno));
1008		return err;
1009	}
1010
1011run:
1012	err = sockmap_init_sockets(options->verbose);
1013	if (err) {
1014		fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1015		goto out;
1016	}
1017
1018	/* Attach txmsg program to sockmap */
1019	if (txmsg_pass)
 
 
1020		tx_prog_fd = prog_fd[4];
1021	else if (txmsg_redir)
1022		tx_prog_fd = prog_fd[5];
1023	else if (txmsg_apply)
1024		tx_prog_fd = prog_fd[6];
1025	else if (txmsg_cork)
1026		tx_prog_fd = prog_fd[7];
1027	else if (txmsg_drop)
 
 
 
 
 
1028		tx_prog_fd = prog_fd[8];
1029	else
1030		tx_prog_fd = 0;
1031
1032	if (tx_prog_fd) {
1033		int redir_fd, i = 0;
1034
1035		err = bpf_prog_attach(tx_prog_fd,
1036				      map_fd[1], BPF_SK_MSG_VERDICT, 0);
1037		if (err) {
1038			fprintf(stderr,
1039				"ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1040				err, strerror(errno));
1041			goto out;
1042		}
1043
1044		err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1045		if (err) {
1046			fprintf(stderr,
1047				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1048				err, strerror(errno));
1049			goto out;
1050		}
1051
1052		if (txmsg_redir)
1053			redir_fd = c2;
1054		else
1055			redir_fd = c1;
1056
1057		err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1058		if (err) {
1059			fprintf(stderr,
1060				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1061				err, strerror(errno));
1062			goto out;
1063		}
1064
1065		if (txmsg_apply) {
1066			err = bpf_map_update_elem(map_fd[3],
1067						  &i, &txmsg_apply, BPF_ANY);
1068			if (err) {
1069				fprintf(stderr,
1070					"ERROR: bpf_map_update_elem (apply_bytes):  %d (%s\n",
1071					err, strerror(errno));
1072				goto out;
1073			}
1074		}
1075
1076		if (txmsg_cork) {
1077			err = bpf_map_update_elem(map_fd[4],
1078						  &i, &txmsg_cork, BPF_ANY);
1079			if (err) {
1080				fprintf(stderr,
1081					"ERROR: bpf_map_update_elem (cork_bytes):  %d (%s\n",
1082					err, strerror(errno));
1083				goto out;
1084			}
1085		}
1086
1087		if (txmsg_start) {
1088			err = bpf_map_update_elem(map_fd[5],
1089						  &i, &txmsg_start, BPF_ANY);
1090			if (err) {
1091				fprintf(stderr,
1092					"ERROR: bpf_map_update_elem (txmsg_start):  %d (%s)\n",
1093					err, strerror(errno));
1094				goto out;
1095			}
1096		}
1097
1098		if (txmsg_end) {
1099			i = 1;
1100			err = bpf_map_update_elem(map_fd[5],
1101						  &i, &txmsg_end, BPF_ANY);
1102			if (err) {
1103				fprintf(stderr,
1104					"ERROR: bpf_map_update_elem (txmsg_end):  %d (%s)\n",
1105					err, strerror(errno));
1106				goto out;
1107			}
1108		}
1109
1110		if (txmsg_start_push) {
1111			i = 2;
1112			err = bpf_map_update_elem(map_fd[5],
1113						  &i, &txmsg_start_push, BPF_ANY);
1114			if (err) {
1115				fprintf(stderr,
1116					"ERROR: bpf_map_update_elem (txmsg_start_push):  %d (%s)\n",
1117					err, strerror(errno));
1118				goto out;
1119			}
1120		}
1121
1122		if (txmsg_end_push) {
1123			i = 3;
1124			err = bpf_map_update_elem(map_fd[5],
1125						  &i, &txmsg_end_push, BPF_ANY);
1126			if (err) {
1127				fprintf(stderr,
1128					"ERROR: bpf_map_update_elem %i@%i (txmsg_end_push):  %d (%s)\n",
1129					txmsg_end_push, i, err, strerror(errno));
1130				goto out;
1131			}
1132		}
1133
1134		if (txmsg_start_pop) {
1135			i = 4;
1136			err = bpf_map_update_elem(map_fd[5],
1137						  &i, &txmsg_start_pop, BPF_ANY);
1138			if (err) {
1139				fprintf(stderr,
1140					"ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop):  %d (%s)\n",
1141					txmsg_start_pop, i, err, strerror(errno));
1142				goto out;
1143			}
1144		} else {
1145			i = 4;
1146			bpf_map_update_elem(map_fd[5],
1147						  &i, &txmsg_start_pop, BPF_ANY);
1148		}
1149
1150		if (txmsg_pop) {
1151			i = 5;
1152			err = bpf_map_update_elem(map_fd[5],
1153						  &i, &txmsg_pop, BPF_ANY);
1154			if (err) {
1155				fprintf(stderr,
1156					"ERROR: bpf_map_update_elem %i@%i (txmsg_pop):  %d (%s)\n",
1157					txmsg_pop, i, err, strerror(errno));
1158				goto out;
1159			}
1160		} else {
1161			i = 5;
1162			bpf_map_update_elem(map_fd[5],
1163					    &i, &txmsg_pop, BPF_ANY);
1164
1165		}
1166
1167		if (txmsg_ingress) {
1168			int in = BPF_F_INGRESS;
1169
1170			i = 0;
1171			err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1172			if (err) {
1173				fprintf(stderr,
1174					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1175					err, strerror(errno));
1176			}
1177			i = 1;
1178			err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1179			if (err) {
1180				fprintf(stderr,
1181					"ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1182					err, strerror(errno));
1183			}
1184			err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1185			if (err) {
1186				fprintf(stderr,
1187					"ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1188					err, strerror(errno));
1189			}
1190
1191			i = 2;
1192			err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1193			if (err) {
1194				fprintf(stderr,
1195					"ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1196					err, strerror(errno));
1197			}
1198		}
1199
1200		if (txmsg_ktls_skb) {
1201			int ingress = BPF_F_INGRESS;
1202
1203			i = 0;
1204			err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1205			if (err) {
1206				fprintf(stderr,
1207					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1208					err, strerror(errno));
1209			}
1210
1211			if (txmsg_ktls_skb_redir) {
1212				i = 1;
1213				err = bpf_map_update_elem(map_fd[7],
1214							  &i, &ingress, BPF_ANY);
1215				if (err) {
1216					fprintf(stderr,
1217						"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1218						err, strerror(errno));
1219				}
1220			}
1221
1222			if (txmsg_ktls_skb_drop) {
1223				i = 1;
1224				err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1225			}
1226		}
1227
1228		if (txmsg_redir_skb) {
1229			int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1230					p2 : p1;
1231			int ingress = BPF_F_INGRESS;
1232
1233			i = 0;
1234			err = bpf_map_update_elem(map_fd[7],
1235						  &i, &ingress, BPF_ANY);
1236			if (err) {
1237				fprintf(stderr,
1238					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1239					err, strerror(errno));
1240			}
1241
1242			i = 3;
1243			err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
 
1244			if (err) {
1245				fprintf(stderr,
1246					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1247					err, strerror(errno));
1248			}
1249		}
1250	}
1251
1252	if (skb_use_parser) {
1253		i = 2;
1254		err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1255	}
1256
1257	if (txmsg_drop)
1258		options->drop_expected = true;
1259
1260	if (test == PING_PONG)
1261		err = forever_ping_pong(options->rate, options);
1262	else if (test == SENDMSG) {
1263		options->base = false;
1264		options->sendpage = false;
1265		err = sendmsg_test(options);
1266	} else if (test == SENDPAGE) {
1267		options->base = false;
1268		options->sendpage = true;
1269		err = sendmsg_test(options);
1270	} else if (test == BASE) {
1271		options->base = true;
1272		options->sendpage = false;
1273		err = sendmsg_test(options);
1274	} else if (test == BASE_SENDPAGE) {
1275		options->base = true;
1276		options->sendpage = true;
1277		err = sendmsg_test(options);
1278	} else
1279		fprintf(stderr, "unknown test\n");
1280out:
1281	/* Detatch and zero all the maps */
1282	bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1283	bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1284	bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1285	bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1286	bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1287
1288	if (tx_prog_fd >= 0)
1289		bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1290
1291	for (i = 0; i < 8; i++) {
1292		key = next_key = 0;
1293		bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1294		while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1295			bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1296			key = next_key;
1297		}
1298	}
1299
1300	close(s1);
1301	close(s2);
1302	close(p1);
1303	close(p2);
1304	close(c1);
1305	close(c2);
1306	return err;
1307}
1308
1309static char *test_to_str(int test)
1310{
1311	switch (test) {
1312	case SENDMSG:
1313		return "sendmsg";
1314	case SENDPAGE:
1315		return "sendpage";
1316	}
1317	return "unknown";
1318}
1319
1320static void append_str(char *dst, const char *src, size_t dst_cap)
1321{
1322	size_t avail = dst_cap - strlen(dst);
1323
1324	if (avail <= 1) /* just zero byte could be written */
1325		return;
1326
1327	strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1328}
1329
1330#define OPTSTRING 60
1331static void test_options(char *options)
1332{
1333	char tstr[OPTSTRING];
1334
1335	memset(options, 0, OPTSTRING);
1336
1337	if (txmsg_pass)
1338		append_str(options, "pass,", OPTSTRING);
 
 
1339	if (txmsg_redir)
1340		append_str(options, "redir,", OPTSTRING);
 
 
1341	if (txmsg_drop)
1342		append_str(options, "drop,", OPTSTRING);
1343	if (txmsg_apply) {
1344		snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1345		append_str(options, tstr, OPTSTRING);
1346	}
1347	if (txmsg_cork) {
1348		snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1349		append_str(options, tstr, OPTSTRING);
1350	}
1351	if (txmsg_start) {
1352		snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1353		append_str(options, tstr, OPTSTRING);
1354	}
1355	if (txmsg_end) {
1356		snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1357		append_str(options, tstr, OPTSTRING);
1358	}
1359	if (txmsg_start_pop) {
1360		snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1361			 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1362		append_str(options, tstr, OPTSTRING);
1363	}
1364	if (txmsg_ingress)
1365		append_str(options, "ingress,", OPTSTRING);
1366	if (txmsg_redir_skb)
1367		append_str(options, "redir_skb,", OPTSTRING);
1368	if (txmsg_ktls_skb)
1369		append_str(options, "ktls_skb,", OPTSTRING);
1370	if (ktls)
1371		append_str(options, "ktls,", OPTSTRING);
1372	if (peek_flag)
1373		append_str(options, "peek,", OPTSTRING);
1374}
1375
1376static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1377{
1378	char *options = calloc(OPTSTRING, sizeof(char));
1379	int err;
1380
1381	if (test == SENDPAGE)
1382		opt->sendpage = true;
1383	else
1384		opt->sendpage = false;
1385
1386	if (txmsg_drop)
1387		opt->drop_expected = true;
1388	else
1389		opt->drop_expected = false;
1390
1391	test_options(options);
1392
1393	if (opt->verbose) {
1394		fprintf(stdout,
1395			" [TEST %i]: (%i, %i, %i, %s, %s): ",
1396			test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1397			test_to_str(test), options);
1398		fflush(stdout);
1399	}
1400	err = run_options(opt, cgrp, test);
1401	if (opt->verbose)
1402		fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1403	test_cnt++;
1404	!err ? passed++ : failed++;
1405	free(options);
1406	return err;
1407}
1408
1409static void test_exec(int cgrp, struct sockmap_options *opt)
1410{
1411	int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1412	int err;
1413
1414	if (type == 0) {
1415		test_start();
1416		err = __test_exec(cgrp, SENDMSG, opt);
1417		if (err)
1418			test_fail();
1419	} else {
1420		test_start();
1421		err = __test_exec(cgrp, SENDPAGE, opt);
1422		if (err)
1423			test_fail();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1424	}
 
 
 
1425}
1426
1427static void test_send_one(struct sockmap_options *opt, int cgrp)
1428{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1429	opt->iov_length = 1;
1430	opt->iov_count = 1;
1431	opt->rate = 1;
1432	test_exec(cgrp, opt);
 
 
1433
1434	opt->iov_length = 1;
1435	opt->iov_count = 1024;
1436	opt->rate = 1;
1437	test_exec(cgrp, opt);
 
 
1438
1439	opt->iov_length = 1024;
1440	opt->iov_count = 1;
1441	opt->rate = 1;
1442	test_exec(cgrp, opt);
1443
1444}
1445
1446static void test_send_many(struct sockmap_options *opt, int cgrp)
1447{
1448	opt->iov_length = 3;
1449	opt->iov_count = 1;
1450	opt->rate = 512;
1451	test_exec(cgrp, opt);
1452
1453	opt->rate = 100;
1454	opt->iov_count = 1;
1455	opt->iov_length = 5;
1456	test_exec(cgrp, opt);
1457}
1458
1459static void test_send_large(struct sockmap_options *opt, int cgrp)
1460{
1461	opt->iov_length = 256;
1462	opt->iov_count = 1024;
1463	opt->rate = 2;
1464	test_exec(cgrp, opt);
1465}
 
1466
1467static void test_send(struct sockmap_options *opt, int cgrp)
1468{
1469	test_send_one(opt, cgrp);
1470	test_send_many(opt, cgrp);
1471	test_send_large(opt, cgrp);
 
 
1472	sched_yield();
 
1473}
1474
1475static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1476{
1477	/* Test small and large iov_count values with pass/redir/apply/cork */
1478	txmsg_pass = 1;
1479	test_send(opt, cgrp);
1480}
1481
1482static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1483{
1484	txmsg_redir = 1;
1485	test_send(opt, cgrp);
1486}
1487
1488static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1489{
1490	txmsg_redir = 1;
1491	opt->tx_wait_mem = true;
1492	test_send_large(opt, cgrp);
1493	opt->tx_wait_mem = false;
1494}
1495
1496static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1497{
1498	txmsg_drop = 1;
1499	test_send(opt, cgrp);
1500}
1501
1502static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1503{
1504	txmsg_pass = txmsg_drop = 0;
1505	txmsg_ingress = txmsg_redir = 1;
1506	test_send(opt, cgrp);
1507}
 
 
1508
1509static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1510{
1511	bool data = opt->data_test;
1512	int k = ktls;
 
 
 
1513
1514	opt->data_test = true;
1515	ktls = 1;
 
 
 
 
 
1516
1517	txmsg_pass = txmsg_drop = 0;
1518	txmsg_ingress = txmsg_redir = 0;
1519	txmsg_ktls_skb = 1;
1520	txmsg_pass = 1;
 
 
 
 
 
 
1521
1522	/* Using data verification so ensure iov layout is
1523	 * expected from test receiver side. e.g. has enough
1524	 * bytes to write test code.
1525	 */
1526	opt->iov_length = 100;
1527	opt->iov_count = 1;
1528	opt->rate = 1;
1529	test_exec(cgrp, opt);
1530
1531	txmsg_ktls_skb_drop = 1;
1532	test_exec(cgrp, opt);
 
 
 
 
 
1533
1534	txmsg_ktls_skb_drop = 0;
1535	txmsg_ktls_skb_redir = 1;
1536	test_exec(cgrp, opt);
1537	txmsg_ktls_skb_redir = 0;
1538
1539	/* Tests that omit skb_parser */
1540	txmsg_omit_skb_parser = 1;
1541	ktls = 0;
1542	txmsg_ktls_skb = 0;
1543	test_exec(cgrp, opt);
1544
1545	txmsg_ktls_skb_drop = 1;
1546	test_exec(cgrp, opt);
1547	txmsg_ktls_skb_drop = 0;
1548
1549	txmsg_ktls_skb_redir = 1;
1550	test_exec(cgrp, opt);
1551
1552	ktls = 1;
1553	test_exec(cgrp, opt);
1554	txmsg_omit_skb_parser = 0;
1555
1556	opt->data_test = data;
1557	ktls = k;
1558}
1559
1560/* Test cork with hung data. This tests poor usage patterns where
1561 * cork can leave data on the ring if user program is buggy and
1562 * doesn't flush them somehow. They do take some time however
1563 * because they wait for a timeout. Test pass, redir and cork with
1564 * apply logic. Use cork size of 4097 with send_large to avoid
1565 * aligning cork size with send size.
1566 */
1567static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1568{
1569	txmsg_pass = 1;
1570	txmsg_redir = 0;
1571	txmsg_cork = 4097;
1572	txmsg_apply = 4097;
1573	test_send_large(opt, cgrp);
 
 
1574
1575	txmsg_pass = 0;
1576	txmsg_redir = 1;
1577	txmsg_apply = 0;
1578	txmsg_cork = 4097;
1579	test_send_large(opt, cgrp);
 
 
1580
1581	txmsg_pass = 0;
1582	txmsg_redir = 1;
1583	txmsg_apply = 4097;
1584	txmsg_cork = 4097;
1585	test_send_large(opt, cgrp);
1586}
1587
1588static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1589{
1590	/* Test basic start/end */
1591	txmsg_start = 1;
1592	txmsg_end = 2;
1593	test_send(opt, cgrp);
1594
1595	/* Test >4k pull */
1596	txmsg_start = 4096;
1597	txmsg_end = 9182;
1598	test_send_large(opt, cgrp);
1599
1600	/* Test pull + redirect */
1601	txmsg_redir = 0;
1602	txmsg_start = 1;
1603	txmsg_end = 2;
1604	test_send(opt, cgrp);
1605
1606	/* Test pull + cork */
1607	txmsg_redir = 0;
1608	txmsg_cork = 512;
1609	txmsg_start = 1;
1610	txmsg_end = 2;
1611	test_send_many(opt, cgrp);
1612
1613	/* Test pull + cork + redirect */
1614	txmsg_redir = 1;
1615	txmsg_cork = 512;
1616	txmsg_start = 1;
1617	txmsg_end = 2;
1618	test_send_many(opt, cgrp);
1619}
1620
1621static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1622{
1623	/* Test basic pop */
1624	txmsg_start_pop = 1;
1625	txmsg_pop = 2;
1626	test_send_many(opt, cgrp);
1627
1628	/* Test pop with >4k */
1629	txmsg_start_pop = 4096;
1630	txmsg_pop = 4096;
1631	test_send_large(opt, cgrp);
1632
1633	/* Test pop + redirect */
1634	txmsg_redir = 1;
1635	txmsg_start_pop = 1;
1636	txmsg_pop = 2;
1637	test_send_many(opt, cgrp);
 
 
1638
1639	/* Test pop + cork */
1640	txmsg_redir = 0;
1641	txmsg_cork = 512;
1642	txmsg_start_pop = 1;
1643	txmsg_pop = 2;
1644	test_send_many(opt, cgrp);
 
1645
1646	/* Test pop + redirect + cork */
1647	txmsg_redir = 1;
1648	txmsg_cork = 4;
1649	txmsg_start_pop = 1;
1650	txmsg_pop = 2;
1651	test_send_many(opt, cgrp);
 
 
 
1652}
1653
1654static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1655{
1656	/* Test basic push */
1657	txmsg_start_push = 1;
1658	txmsg_end_push = 1;
1659	test_send(opt, cgrp);
1660
1661	/* Test push 4kB >4k */
1662	txmsg_start_push = 4096;
1663	txmsg_end_push = 4096;
1664	test_send_large(opt, cgrp);
1665
1666	/* Test push + redirect */
1667	txmsg_redir = 1;
1668	txmsg_start_push = 1;
1669	txmsg_end_push = 2;
1670	test_send_many(opt, cgrp);
 
 
 
 
1671
1672	/* Test push + cork */
1673	txmsg_redir = 0;
1674	txmsg_cork = 512;
1675	txmsg_start_push = 1;
1676	txmsg_end_push = 2;
1677	test_send_many(opt, cgrp);
1678}
 
 
 
1679
1680static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1681{
1682	txmsg_start_push = 1;
1683	txmsg_end_push = 10;
1684	txmsg_start_pop = 5;
1685	txmsg_pop = 4;
1686	test_send_large(opt, cgrp);
1687}
 
 
 
 
 
 
 
 
 
 
1688
1689static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1690{
1691	txmsg_pass = 1;
1692	txmsg_redir = 0;
1693	txmsg_ingress = 0;
1694	txmsg_apply = 1;
1695	txmsg_cork = 0;
1696	test_send_one(opt, cgrp);
 
 
1697
1698	txmsg_pass = 0;
1699	txmsg_redir = 1;
1700	txmsg_ingress = 0;
1701	txmsg_apply = 1;
1702	txmsg_cork = 0;
1703	test_send_one(opt, cgrp);
 
 
 
 
1704
1705	txmsg_pass = 0;
1706	txmsg_redir = 1;
1707	txmsg_ingress = 1;
1708	txmsg_apply = 1;
1709	txmsg_cork = 0;
1710	test_send_one(opt, cgrp);
 
 
1711
1712	txmsg_pass = 1;
1713	txmsg_redir = 0;
1714	txmsg_ingress = 0;
1715	txmsg_apply = 1024;
1716	txmsg_cork = 0;
1717	test_send_large(opt, cgrp);
 
 
1718
1719	txmsg_pass = 0;
1720	txmsg_redir = 1;
1721	txmsg_ingress = 0;
1722	txmsg_apply = 1024;
1723	txmsg_cork = 0;
1724	test_send_large(opt, cgrp);
 
 
1725
1726	txmsg_pass = 0;
1727	txmsg_redir = 1;
1728	txmsg_ingress = 1;
1729	txmsg_apply = 1024;
1730	txmsg_cork = 0;
1731	test_send_large(opt, cgrp);
1732}
1733
1734static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1735{
1736	txmsg_pass = 1;
1737	txmsg_redir = 0;
1738	txmsg_apply = 0;
1739	txmsg_cork = 1;
1740	test_send(opt, cgrp);
 
1741
1742	txmsg_pass = 1;
1743	txmsg_redir = 0;
1744	txmsg_apply = 1;
1745	txmsg_cork = 1;
1746	test_send(opt, cgrp);
1747}
 
 
1748
1749static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1750{
1751	txmsg_pass = 1;
1752	skb_use_parser = 512;
1753	if (ktls == 1)
1754		skb_use_parser = 570;
1755	opt->iov_length = 256;
1756	opt->iov_count = 1;
1757	opt->rate = 2;
1758	test_exec(cgrp, opt);
1759}
1760
1761static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1762{
1763	if (ktls == 1)
1764		return;
1765	skb_use_parser = 10;
1766	opt->iov_length = 20;
1767	opt->iov_count = 1;
1768	opt->rate = 1;
1769	opt->check_recved_len = true;
1770	test_exec(cgrp, opt);
1771	opt->check_recved_len = false;
 
 
 
 
 
1772}
1773
1774char *map_names[] = {
1775	"sock_map",
1776	"sock_map_txmsg",
1777	"sock_map_redir",
1778	"sock_apply_bytes",
1779	"sock_cork_bytes",
1780	"sock_bytes",
1781	"sock_redir_flags",
1782	"sock_skb_opts",
1783	"tls_sock_map",
1784};
1785
1786int prog_attach_type[] = {
1787	BPF_SK_SKB_STREAM_PARSER,
1788	BPF_SK_SKB_STREAM_VERDICT,
1789	BPF_SK_SKB_STREAM_VERDICT,
1790	BPF_CGROUP_SOCK_OPS,
1791	BPF_SK_MSG_VERDICT,
1792	BPF_SK_MSG_VERDICT,
1793	BPF_SK_MSG_VERDICT,
1794	BPF_SK_MSG_VERDICT,
1795	BPF_SK_MSG_VERDICT,
1796	BPF_SK_MSG_VERDICT,
1797	BPF_SK_MSG_VERDICT,
1798};
1799
1800int prog_type[] = {
1801	BPF_PROG_TYPE_SK_SKB,
1802	BPF_PROG_TYPE_SK_SKB,
1803	BPF_PROG_TYPE_SK_SKB,
1804	BPF_PROG_TYPE_SOCK_OPS,
1805	BPF_PROG_TYPE_SK_MSG,
1806	BPF_PROG_TYPE_SK_MSG,
1807	BPF_PROG_TYPE_SK_MSG,
1808	BPF_PROG_TYPE_SK_MSG,
1809	BPF_PROG_TYPE_SK_MSG,
1810	BPF_PROG_TYPE_SK_MSG,
1811	BPF_PROG_TYPE_SK_MSG,
1812};
1813
1814static int populate_progs(char *bpf_file)
1815{
1816	struct bpf_program *prog;
1817	struct bpf_object *obj;
1818	int i = 0;
1819	long err;
1820
1821	obj = bpf_object__open(bpf_file);
1822	err = libbpf_get_error(obj);
1823	if (err) {
1824		char err_buf[256];
1825
1826		libbpf_strerror(err, err_buf, sizeof(err_buf));
1827		printf("Unable to load eBPF objects in file '%s' : %s\n",
1828		       bpf_file, err_buf);
1829		return -1;
1830	}
1831
1832	bpf_object__for_each_program(prog, obj) {
1833		bpf_program__set_type(prog, prog_type[i]);
1834		bpf_program__set_expected_attach_type(prog,
1835						      prog_attach_type[i]);
1836		i++;
1837	}
1838
1839	i = bpf_object__load(obj);
1840	i = 0;
1841	bpf_object__for_each_program(prog, obj) {
1842		prog_fd[i] = bpf_program__fd(prog);
1843		i++;
1844	}
1845
1846	for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1847		maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1848		map_fd[i] = bpf_map__fd(maps[i]);
1849		if (map_fd[i] < 0) {
1850			fprintf(stderr, "load_bpf_file: (%i) %s\n",
1851				map_fd[i], strerror(errno));
1852			return -1;
1853		}
1854	}
1855
1856	return 0;
1857}
1858
1859struct _test test[] = {
1860	{"txmsg test passthrough", test_txmsg_pass},
1861	{"txmsg test redirect", test_txmsg_redir},
1862	{"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1863	{"txmsg test drop", test_txmsg_drop},
1864	{"txmsg test ingress redirect", test_txmsg_ingress_redir},
1865	{"txmsg test skb", test_txmsg_skb},
1866	{"txmsg test apply", test_txmsg_apply},
1867	{"txmsg test cork", test_txmsg_cork},
1868	{"txmsg test hanging corks", test_txmsg_cork_hangs},
1869	{"txmsg test push_data", test_txmsg_push},
1870	{"txmsg test pull-data", test_txmsg_pull},
1871	{"txmsg test pop-data", test_txmsg_pop},
1872	{"txmsg test push/pop data", test_txmsg_push_pop},
1873	{"txmsg test ingress parser", test_txmsg_ingress_parser},
1874	{"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1875};
1876
1877static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1878{
1879	char *entry, *ptr;
1880
1881	if (!opt->whitelist)
1882		return 0;
1883	ptr = strdup(opt->whitelist);
1884	if (!ptr)
1885		return -ENOMEM;
1886	entry = strtok(ptr, ",");
1887	while (entry) {
1888		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1889		    strstr(opt->map, entry) != 0 ||
1890		    strstr(t->title, entry) != 0)
1891			return 0;
1892		entry = strtok(NULL, ",");
1893	}
1894	return -EINVAL;
1895}
1896
1897static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1898{
1899	char *entry, *ptr;
1900
1901	if (!opt->blacklist)
1902		return -EINVAL;
1903	ptr = strdup(opt->blacklist);
1904	if (!ptr)
1905		return -ENOMEM;
1906	entry = strtok(ptr, ",");
1907	while (entry) {
1908		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1909		    strstr(opt->map, entry) != 0 ||
1910		    strstr(t->title, entry) != 0)
1911			return 0;
1912		entry = strtok(NULL, ",");
1913	}
1914	return -EINVAL;
1915}
1916
1917static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1918{
1919	int i, err;
1920
1921	err = populate_progs(opt->map);
1922	if (err < 0) {
1923		fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1924		return err;
1925	}
1926
1927	/* Tests basic commands and APIs */
1928	for (i = 0; i < ARRAY_SIZE(test); i++) {
1929		struct _test t = test[i];
1930
1931		if (check_whitelist(&t, opt) != 0)
1932			continue;
1933		if (check_blacklist(&t, opt) == 0)
1934			continue;
1935
1936		test_start_subtest(&t, opt);
1937		t.tester(cg_fd, opt);
1938		test_end_subtest();
 
 
 
 
 
 
1939	}
1940
1941	return err;
1942}
 
 
 
1943
1944static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1945{
1946	opt->map = BPF_SOCKMAP_FILENAME;
1947	__test_selftests(cg_fd, opt);
1948}
1949
1950static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1951{
1952	opt->map = BPF_SOCKHASH_FILENAME;
1953	__test_selftests(cg_fd, opt);
1954}
1955
1956static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1957{
1958	opt->map = BPF_SOCKHASH_FILENAME;
1959	opt->prepend = "ktls";
1960	ktls = 1;
1961	__test_selftests(cg_fd, opt);
1962	ktls = 0;
1963}
1964
1965static int test_selftest(int cg_fd, struct sockmap_options *opt)
1966{
 
1967
1968	test_selftests_sockmap(cg_fd, opt);
1969	test_selftests_sockhash(cg_fd, opt);
1970	test_selftests_ktls(cg_fd, opt);
1971	test_print_results();
1972	return 0;
 
 
 
1973}
1974
1975int main(int argc, char **argv)
1976{
1977	int iov_count = 1, length = 1024, rate = 1;
1978	struct sockmap_options options = {0};
1979	int opt, longindex, err, cg_fd = 0;
1980	char *bpf_file = BPF_SOCKMAP_FILENAME;
1981	int test = SELFTESTS;
1982	bool cg_created = 0;
1983
1984	while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
 
 
 
1985				  long_options, &longindex)) != -1) {
1986		switch (opt) {
1987		case 's':
1988			txmsg_start = atoi(optarg);
1989			break;
1990		case 'e':
1991			txmsg_end = atoi(optarg);
1992			break;
1993		case 'p':
1994			txmsg_start_push = atoi(optarg);
1995			break;
1996		case 'q':
1997			txmsg_end_push = atoi(optarg);
1998			break;
1999		case 'w':
2000			txmsg_start_pop = atoi(optarg);
2001			break;
2002		case 'x':
2003			txmsg_pop = atoi(optarg);
2004			break;
2005		case 'a':
2006			txmsg_apply = atoi(optarg);
2007			break;
2008		case 'k':
2009			txmsg_cork = atoi(optarg);
2010			break;
2011		case 'c':
2012			cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
2013			if (cg_fd < 0) {
2014				fprintf(stderr,
2015					"ERROR: (%i) open cg path failed: %s\n",
2016					cg_fd, optarg);
2017				return cg_fd;
2018			}
2019			break;
2020		case 'r':
2021			rate = atoi(optarg);
2022			break;
2023		case 'v':
2024			options.verbose = 1;
2025			if (optarg)
2026				options.verbose = atoi(optarg);
2027			break;
2028		case 'i':
2029			iov_count = atoi(optarg);
2030			break;
2031		case 'l':
2032			length = atoi(optarg);
2033			break;
2034		case 'd':
2035			options.data_test = true;
2036			break;
2037		case 't':
2038			if (strcmp(optarg, "ping") == 0) {
2039				test = PING_PONG;
2040			} else if (strcmp(optarg, "sendmsg") == 0) {
2041				test = SENDMSG;
2042			} else if (strcmp(optarg, "base") == 0) {
2043				test = BASE;
2044			} else if (strcmp(optarg, "base_sendpage") == 0) {
2045				test = BASE_SENDPAGE;
2046			} else if (strcmp(optarg, "sendpage") == 0) {
2047				test = SENDPAGE;
2048			} else {
2049				usage(argv);
2050				return -1;
2051			}
2052			break;
2053		case 'n':
2054			options.whitelist = strdup(optarg);
2055			if (!options.whitelist)
2056				return -ENOMEM;
2057			break;
2058		case 'b':
2059			options.blacklist = strdup(optarg);
2060			if (!options.blacklist)
2061				return -ENOMEM;
2062		case 0:
2063			break;
2064		case 'h':
2065		default:
2066			usage(argv);
2067			return -1;
2068		}
2069	}
2070
2071	if (!cg_fd) {
2072		cg_fd = cgroup_setup_and_join(CG_PATH);
2073		if (cg_fd < 0)
2074			return cg_fd;
2075		cg_created = 1;
2076	}
2077
2078	/* Use libbpf 1.0 API mode */
2079	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2080
2081	if (test == SELFTESTS) {
2082		err = test_selftest(cg_fd, &options);
2083		goto out;
 
2084	}
2085
2086	err = populate_progs(bpf_file);
2087	if (err) {
2088		fprintf(stderr, "populate program: (%s) %s\n",
2089			bpf_file, strerror(errno));
2090		return 1;
2091	}
2092	running = 1;
2093
2094	/* catch SIGINT */
2095	signal(SIGINT, running_handler);
2096
2097	options.iov_count = iov_count;
2098	options.iov_length = length;
2099	options.rate = rate;
2100
2101	err = run_options(&options, cg_fd, test);
2102out:
2103	if (options.whitelist)
2104		free(options.whitelist);
2105	if (options.blacklist)
2106		free(options.blacklist);
2107	if (cg_created)
2108		cleanup_cgroup_environment();
2109	close(cg_fd);
2110	return err;
2111}
2112
2113void running_handler(int a)
2114{
2115	running = 0;
2116}