Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3#include <stdio.h>
4#include <stdlib.h>
5#include <sys/socket.h>
6#include <sys/ioctl.h>
7#include <sys/select.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10#include <unistd.h>
11#include <string.h>
12#include <errno.h>
13#include <stdbool.h>
14#include <signal.h>
15#include <fcntl.h>
16#include <sys/wait.h>
17#include <time.h>
18#include <sched.h>
19
20#include <sys/time.h>
21#include <sys/types.h>
22#include <sys/sendfile.h>
23
24#include <linux/netlink.h>
25#include <linux/socket.h>
26#include <linux/sock_diag.h>
27#include <linux/bpf.h>
28#include <linux/if_link.h>
29#include <linux/tls.h>
30#include <assert.h>
31#include <libgen.h>
32
33#include <getopt.h>
34
35#include <bpf/bpf.h>
36#include <bpf/libbpf.h>
37
38#include "bpf_util.h"
39#include "cgroup_helpers.h"
40
41int running;
42static void running_handler(int a);
43
44#ifndef TCP_ULP
45# define TCP_ULP 31
46#endif
47#ifndef SOL_TLS
48# define SOL_TLS 282
49#endif
50
51/* randomly selected ports for testing on lo */
52#define S1_PORT 10000
53#define S2_PORT 10001
54
55#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
56#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57#define CG_PATH "/sockmap"
58
59#define EDATAINTEGRITY 2001
60
61/* global sockets */
62int s1, s2, c1, c2, p1, p2;
63int test_cnt;
64int passed;
65int failed;
66int map_fd[9];
67struct bpf_map *maps[9];
68struct bpf_program *progs[9];
69struct bpf_link *links[9];
70
71int txmsg_pass;
72int txmsg_redir;
73int txmsg_drop;
74int txmsg_apply;
75int txmsg_cork;
76int txmsg_start;
77int txmsg_end;
78int txmsg_start_push;
79int txmsg_end_push;
80int txmsg_start_pop;
81int txmsg_pop;
82int txmsg_ingress;
83int txmsg_redir_skb;
84int txmsg_ktls_skb;
85int txmsg_ktls_skb_drop;
86int txmsg_ktls_skb_redir;
87int ktls;
88int peek_flag;
89int skb_use_parser;
90int txmsg_omit_skb_parser;
91int verify_push_start;
92int verify_push_len;
93int verify_pop_start;
94int verify_pop_len;
95
96static const struct option long_options[] = {
97 {"help", no_argument, NULL, 'h' },
98 {"cgroup", required_argument, NULL, 'c' },
99 {"rate", required_argument, NULL, 'r' },
100 {"verbose", optional_argument, NULL, 'v' },
101 {"iov_count", required_argument, NULL, 'i' },
102 {"length", required_argument, NULL, 'l' },
103 {"test", required_argument, NULL, 't' },
104 {"data_test", no_argument, NULL, 'd' },
105 {"txmsg", no_argument, &txmsg_pass, 1 },
106 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
107 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
108 {"txmsg_apply", required_argument, NULL, 'a'},
109 {"txmsg_cork", required_argument, NULL, 'k'},
110 {"txmsg_start", required_argument, NULL, 's'},
111 {"txmsg_end", required_argument, NULL, 'e'},
112 {"txmsg_start_push", required_argument, NULL, 'p'},
113 {"txmsg_end_push", required_argument, NULL, 'q'},
114 {"txmsg_start_pop", required_argument, NULL, 'w'},
115 {"txmsg_pop", required_argument, NULL, 'x'},
116 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
117 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
118 {"ktls", no_argument, &ktls, 1 },
119 {"peek", no_argument, &peek_flag, 1 },
120 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
121 {"whitelist", required_argument, NULL, 'n' },
122 {"blacklist", required_argument, NULL, 'b' },
123 {0, 0, NULL, 0 }
124};
125
126struct test_env {
127 const char *type;
128 const char *subtest;
129 const char *prepend;
130
131 int test_num;
132 int subtest_num;
133
134 int succ_cnt;
135 int fail_cnt;
136 int fail_last;
137};
138
139struct test_env env;
140
141struct sockmap_options {
142 int verbose;
143 bool base;
144 bool sendpage;
145 bool data_test;
146 bool drop_expected;
147 bool check_recved_len;
148 bool tx_wait_mem;
149 int iov_count;
150 int iov_length;
151 int rate;
152 char *map;
153 char *whitelist;
154 char *blacklist;
155 char *prepend;
156};
157
158struct _test {
159 char *title;
160 void (*tester)(int cg_fd, struct sockmap_options *opt);
161};
162
163static void test_start(void)
164{
165 env.subtest_num++;
166}
167
168static void test_fail(void)
169{
170 env.fail_cnt++;
171}
172
173static void test_pass(void)
174{
175 env.succ_cnt++;
176}
177
178static void test_reset(void)
179{
180 txmsg_start = txmsg_end = 0;
181 txmsg_start_pop = txmsg_pop = 0;
182 txmsg_start_push = txmsg_end_push = 0;
183 txmsg_pass = txmsg_drop = txmsg_redir = 0;
184 txmsg_apply = txmsg_cork = 0;
185 txmsg_ingress = txmsg_redir_skb = 0;
186 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
187 txmsg_omit_skb_parser = 0;
188 skb_use_parser = 0;
189}
190
191static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
192{
193 env.type = o->map;
194 env.subtest = t->title;
195 env.prepend = o->prepend;
196 env.test_num++;
197 env.subtest_num = 0;
198 env.fail_last = env.fail_cnt;
199 test_reset();
200 return 0;
201}
202
203static void test_end_subtest(void)
204{
205 int error = env.fail_cnt - env.fail_last;
206 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
207
208 if (!error)
209 test_pass();
210
211 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
212 env.test_num, env.subtest_num,
213 !type ? "sockmap" : "sockhash",
214 env.prepend ? : "",
215 env.subtest, error ? "FAIL" : "OK");
216}
217
218static void test_print_results(void)
219{
220 fprintf(stdout, "Pass: %d Fail: %d\n",
221 env.succ_cnt, env.fail_cnt);
222}
223
224static void usage(char *argv[])
225{
226 int i;
227
228 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
229 printf(" options:\n");
230 for (i = 0; long_options[i].name != 0; i++) {
231 printf(" --%-12s", long_options[i].name);
232 if (long_options[i].flag != NULL)
233 printf(" flag (internal value:%d)\n",
234 *long_options[i].flag);
235 else
236 printf(" -%c\n", long_options[i].val);
237 }
238 printf("\n");
239}
240
241char *sock_to_string(int s)
242{
243 if (s == c1)
244 return "client1";
245 else if (s == c2)
246 return "client2";
247 else if (s == s1)
248 return "server1";
249 else if (s == s2)
250 return "server2";
251 else if (s == p1)
252 return "peer1";
253 else if (s == p2)
254 return "peer2";
255 else
256 return "unknown";
257}
258
259static int sockmap_init_ktls(int verbose, int s)
260{
261 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
262 .info = {
263 .version = TLS_1_2_VERSION,
264 .cipher_type = TLS_CIPHER_AES_GCM_128,
265 },
266 };
267 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
268 .info = {
269 .version = TLS_1_2_VERSION,
270 .cipher_type = TLS_CIPHER_AES_GCM_128,
271 },
272 };
273 int so_buf = 6553500;
274 int err;
275
276 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
277 if (err) {
278 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
279 return -EINVAL;
280 }
281 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
282 if (err) {
283 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
284 return -EINVAL;
285 }
286 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
287 if (err) {
288 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
289 return -EINVAL;
290 }
291 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
292 if (err) {
293 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
294 return -EINVAL;
295 }
296 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
297 if (err) {
298 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
299 return -EINVAL;
300 }
301
302 if (verbose)
303 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
304 return 0;
305}
306static int sockmap_init_sockets(int verbose)
307{
308 int i, err, one = 1;
309 struct sockaddr_in addr;
310 int *fds[4] = {&s1, &s2, &c1, &c2};
311
312 s1 = s2 = p1 = p2 = c1 = c2 = 0;
313
314 /* Init sockets */
315 for (i = 0; i < 4; i++) {
316 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
317 if (*fds[i] < 0) {
318 perror("socket s1 failed()");
319 return errno;
320 }
321 }
322
323 /* Allow reuse */
324 for (i = 0; i < 2; i++) {
325 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
326 (char *)&one, sizeof(one));
327 if (err) {
328 perror("setsockopt failed()");
329 return errno;
330 }
331 }
332
333 /* Non-blocking sockets */
334 for (i = 0; i < 2; i++) {
335 err = ioctl(*fds[i], FIONBIO, (char *)&one);
336 if (err < 0) {
337 perror("ioctl s1 failed()");
338 return errno;
339 }
340 }
341
342 /* Bind server sockets */
343 memset(&addr, 0, sizeof(struct sockaddr_in));
344 addr.sin_family = AF_INET;
345 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
346
347 addr.sin_port = htons(S1_PORT);
348 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
349 if (err < 0) {
350 perror("bind s1 failed()");
351 return errno;
352 }
353
354 addr.sin_port = htons(S2_PORT);
355 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
356 if (err < 0) {
357 perror("bind s2 failed()");
358 return errno;
359 }
360
361 /* Listen server sockets */
362 addr.sin_port = htons(S1_PORT);
363 err = listen(s1, 32);
364 if (err < 0) {
365 perror("listen s1 failed()");
366 return errno;
367 }
368
369 addr.sin_port = htons(S2_PORT);
370 err = listen(s2, 32);
371 if (err < 0) {
372 perror("listen s1 failed()");
373 return errno;
374 }
375
376 /* Initiate Connect */
377 addr.sin_port = htons(S1_PORT);
378 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
379 if (err < 0 && errno != EINPROGRESS) {
380 perror("connect c1 failed()");
381 return errno;
382 }
383
384 addr.sin_port = htons(S2_PORT);
385 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
386 if (err < 0 && errno != EINPROGRESS) {
387 perror("connect c2 failed()");
388 return errno;
389 } else if (err < 0) {
390 err = 0;
391 }
392
393 /* Accept Connecrtions */
394 p1 = accept(s1, NULL, NULL);
395 if (p1 < 0) {
396 perror("accept s1 failed()");
397 return errno;
398 }
399
400 p2 = accept(s2, NULL, NULL);
401 if (p2 < 0) {
402 perror("accept s1 failed()");
403 return errno;
404 }
405
406 if (verbose > 1) {
407 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
408 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
409 c1, s1, c2, s2);
410 }
411 return 0;
412}
413
414struct msg_stats {
415 size_t bytes_sent;
416 size_t bytes_recvd;
417 struct timespec start;
418 struct timespec end;
419};
420
421static int msg_loop_sendpage(int fd, int iov_length, int cnt,
422 struct msg_stats *s,
423 struct sockmap_options *opt)
424{
425 bool drop = opt->drop_expected;
426 unsigned char k = 0;
427 int i, j, fp;
428 FILE *file;
429
430 file = tmpfile();
431 if (!file) {
432 perror("create file for sendpage");
433 return 1;
434 }
435 for (i = 0; i < cnt; i++, k = 0) {
436 for (j = 0; j < iov_length; j++, k++)
437 fwrite(&k, sizeof(char), 1, file);
438 }
439 fflush(file);
440 fseek(file, 0, SEEK_SET);
441
442 fp = fileno(file);
443
444 clock_gettime(CLOCK_MONOTONIC, &s->start);
445 for (i = 0; i < cnt; i++) {
446 int sent;
447
448 errno = 0;
449 sent = sendfile(fd, fp, NULL, iov_length);
450
451 if (!drop && sent < 0) {
452 perror("sendpage loop error");
453 fclose(file);
454 return sent;
455 } else if (drop && sent >= 0) {
456 printf("sendpage loop error expected: %i errno %i\n",
457 sent, errno);
458 fclose(file);
459 return -EIO;
460 }
461
462 if (sent > 0)
463 s->bytes_sent += sent;
464 }
465 clock_gettime(CLOCK_MONOTONIC, &s->end);
466 fclose(file);
467 return 0;
468}
469
470static void msg_free_iov(struct msghdr *msg)
471{
472 int i;
473
474 for (i = 0; i < msg->msg_iovlen; i++)
475 free(msg->msg_iov[i].iov_base);
476 free(msg->msg_iov);
477 msg->msg_iov = NULL;
478 msg->msg_iovlen = 0;
479}
480
481static int msg_alloc_iov(struct msghdr *msg,
482 int iov_count, int iov_length,
483 bool data, bool xmit)
484{
485 unsigned char k = 0;
486 struct iovec *iov;
487 int i;
488
489 iov = calloc(iov_count, sizeof(struct iovec));
490 if (!iov)
491 return errno;
492
493 for (i = 0; i < iov_count; i++) {
494 unsigned char *d = calloc(iov_length, sizeof(char));
495
496 if (!d) {
497 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
498 goto unwind_iov;
499 }
500 iov[i].iov_base = d;
501 iov[i].iov_len = iov_length;
502
503 if (data && xmit) {
504 int j;
505
506 for (j = 0; j < iov_length; j++)
507 d[j] = k++;
508 }
509 }
510
511 msg->msg_iov = iov;
512 msg->msg_iovlen = iov_count;
513
514 return 0;
515unwind_iov:
516 for (i--; i >= 0 ; i--)
517 free(msg->msg_iov[i].iov_base);
518 return -ENOMEM;
519}
520
521/* In push or pop test, we need to do some calculations for msg_verify_data */
522static void msg_verify_date_prep(void)
523{
524 int push_range_end = txmsg_start_push + txmsg_end_push - 1;
525 int pop_range_end = txmsg_start_pop + txmsg_pop - 1;
526
527 if (txmsg_end_push && txmsg_pop &&
528 txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) {
529 /* The push range and the pop range overlap */
530 int overlap_len;
531
532 verify_push_start = txmsg_start_push;
533 verify_pop_start = txmsg_start_pop;
534 if (txmsg_start_push < txmsg_start_pop)
535 overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop);
536 else
537 overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push);
538 verify_push_len = max(txmsg_end_push - overlap_len, 0);
539 verify_pop_len = max(txmsg_pop - overlap_len, 0);
540 } else {
541 /* Otherwise */
542 verify_push_start = txmsg_start_push;
543 verify_pop_start = txmsg_start_pop;
544 verify_push_len = txmsg_end_push;
545 verify_pop_len = txmsg_pop;
546 }
547}
548
549static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz,
550 unsigned char *k_p, int *bytes_cnt_p,
551 int *check_cnt_p, int *push_p)
552{
553 int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p;
554 unsigned char k = *k_p;
555 int i, j;
556
557 for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) {
558 unsigned char *d = msg->msg_iov[i].iov_base;
559
560 /* Special case test for skb ingress + ktls */
561 if (i == 0 && txmsg_ktls_skb) {
562 if (msg->msg_iov[i].iov_len < 4)
563 return -EDATAINTEGRITY;
564 if (memcmp(d, "PASS", 4) != 0) {
565 fprintf(stderr,
566 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
567 i, 0, d[0], d[1], d[2], d[3]);
568 return -EDATAINTEGRITY;
569 }
570 j = 4; /* advance index past PASS header */
571 }
572
573 for (; j < msg->msg_iov[i].iov_len && size; j++) {
574 if (push > 0 &&
575 check_cnt == verify_push_start + verify_push_len - push) {
576 int skipped;
577revisit_push:
578 skipped = push;
579 if (j + push >= msg->msg_iov[i].iov_len)
580 skipped = msg->msg_iov[i].iov_len - j;
581 push -= skipped;
582 size -= skipped;
583 j += skipped - 1;
584 check_cnt += skipped;
585 continue;
586 }
587
588 if (verify_pop_len > 0 && check_cnt == verify_pop_start) {
589 bytes_cnt += verify_pop_len;
590 check_cnt += verify_pop_len;
591 k += verify_pop_len;
592
593 if (bytes_cnt == chunk_sz) {
594 k = 0;
595 bytes_cnt = 0;
596 check_cnt = 0;
597 push = verify_push_len;
598 }
599
600 if (push > 0 &&
601 check_cnt == verify_push_start + verify_push_len - push)
602 goto revisit_push;
603 }
604
605 if (d[j] != k++) {
606 fprintf(stderr,
607 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
608 i, j, d[j], k - 1, d[j+1], k);
609 return -EDATAINTEGRITY;
610 }
611 bytes_cnt++;
612 check_cnt++;
613 if (bytes_cnt == chunk_sz) {
614 k = 0;
615 bytes_cnt = 0;
616 check_cnt = 0;
617 push = verify_push_len;
618 }
619 size--;
620 }
621 }
622 *k_p = k;
623 *bytes_cnt_p = bytes_cnt;
624 *check_cnt_p = check_cnt;
625 *push_p = push;
626 return 0;
627}
628
629static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
630 struct msg_stats *s, bool tx,
631 struct sockmap_options *opt)
632{
633 struct msghdr msg = {0}, msg_peek = {0};
634 int err, i, flags = MSG_NOSIGNAL;
635 bool drop = opt->drop_expected;
636 bool data = opt->data_test;
637 int iov_alloc_length = iov_length;
638
639 if (!tx && opt->check_recved_len)
640 iov_alloc_length *= 2;
641
642 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
643 if (err)
644 goto out_errno;
645 if (peek_flag) {
646 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
647 if (err)
648 goto out_errno;
649 }
650
651 if (tx) {
652 clock_gettime(CLOCK_MONOTONIC, &s->start);
653 for (i = 0; i < cnt; i++) {
654 int sent;
655
656 errno = 0;
657 sent = sendmsg(fd, &msg, flags);
658
659 if (!drop && sent < 0) {
660 if (opt->tx_wait_mem && errno == EACCES) {
661 errno = 0;
662 goto out_errno;
663 }
664 perror("sendmsg loop error");
665 goto out_errno;
666 } else if (drop && sent >= 0) {
667 fprintf(stderr,
668 "sendmsg loop error expected: %i errno %i\n",
669 sent, errno);
670 errno = -EIO;
671 goto out_errno;
672 }
673 if (sent > 0)
674 s->bytes_sent += sent;
675 }
676 clock_gettime(CLOCK_MONOTONIC, &s->end);
677 } else {
678 float total_bytes, txmsg_pop_total, txmsg_push_total;
679 int slct, recvp = 0, recv, max_fd = fd;
680 int fd_flags = O_NONBLOCK;
681 struct timeval timeout;
682 unsigned char k = 0;
683 int bytes_cnt = 0;
684 int check_cnt = 0;
685 int push = 0;
686 fd_set w;
687
688 fcntl(fd, fd_flags);
689 /* Account for pop bytes noting each iteration of apply will
690 * call msg_pop_data helper so we need to account for this
691 * by calculating the number of apply iterations. Note user
692 * of the tool can create cases where no data is sent by
693 * manipulating pop/push/pull/etc. For example txmsg_apply 1
694 * with txmsg_pop 1 will try to apply 1B at a time but each
695 * iteration will then pop 1B so no data will ever be sent.
696 * This is really only useful for testing edge cases in code
697 * paths.
698 */
699 total_bytes = (float)iov_length * (float)cnt;
700 if (!opt->sendpage)
701 total_bytes *= (float)iov_count;
702 if (txmsg_apply) {
703 txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply);
704 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
705 } else {
706 txmsg_push_total = txmsg_end_push * cnt;
707 txmsg_pop_total = txmsg_pop * cnt;
708 }
709 total_bytes += txmsg_push_total;
710 total_bytes -= txmsg_pop_total;
711 if (data) {
712 msg_verify_date_prep();
713 push = verify_push_len;
714 }
715 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
716 if (err < 0)
717 perror("recv start time");
718 while (s->bytes_recvd < total_bytes) {
719 if (txmsg_cork) {
720 timeout.tv_sec = 0;
721 timeout.tv_usec = 300000;
722 } else {
723 timeout.tv_sec = 3;
724 timeout.tv_usec = 0;
725 }
726
727 /* FD sets */
728 FD_ZERO(&w);
729 FD_SET(fd, &w);
730
731 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
732 if (slct == -1) {
733 perror("select()");
734 clock_gettime(CLOCK_MONOTONIC, &s->end);
735 goto out_errno;
736 } else if (!slct) {
737 if (opt->verbose)
738 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
739 errno = -EIO;
740 clock_gettime(CLOCK_MONOTONIC, &s->end);
741 goto out_errno;
742 }
743
744 if (opt->tx_wait_mem) {
745 FD_ZERO(&w);
746 FD_SET(fd, &w);
747 slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
748 errno = 0;
749 close(fd);
750 goto out_errno;
751 }
752
753 errno = 0;
754 if (peek_flag) {
755 flags |= MSG_PEEK;
756 recvp = recvmsg(fd, &msg_peek, flags);
757 if (recvp < 0) {
758 if (errno != EWOULDBLOCK) {
759 clock_gettime(CLOCK_MONOTONIC, &s->end);
760 goto out_errno;
761 }
762 }
763 flags = 0;
764 }
765
766 recv = recvmsg(fd, &msg, flags);
767 if (recv < 0) {
768 if (errno != EWOULDBLOCK) {
769 clock_gettime(CLOCK_MONOTONIC, &s->end);
770 perror("recv failed()");
771 goto out_errno;
772 }
773 }
774
775 if (recv > 0)
776 s->bytes_recvd += recv;
777
778 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
779 errno = EMSGSIZE;
780 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
781 s->bytes_recvd, total_bytes);
782 goto out_errno;
783 }
784
785 if (data) {
786 int chunk_sz = opt->sendpage ?
787 iov_length :
788 iov_length * iov_count;
789
790 errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt,
791 &check_cnt, &push);
792 if (errno) {
793 perror("data verify msg failed");
794 goto out_errno;
795 }
796 if (recvp) {
797 errno = msg_verify_data(&msg_peek,
798 recvp,
799 chunk_sz,
800 &k,
801 &bytes_cnt,
802 &check_cnt,
803 &push);
804 if (errno) {
805 perror("data verify msg_peek failed");
806 goto out_errno;
807 }
808 }
809 }
810 }
811 clock_gettime(CLOCK_MONOTONIC, &s->end);
812 }
813
814 msg_free_iov(&msg);
815 msg_free_iov(&msg_peek);
816 return err;
817out_errno:
818 msg_free_iov(&msg);
819 msg_free_iov(&msg_peek);
820 return errno;
821}
822
823static float giga = 1000000000;
824
825static inline float sentBps(struct msg_stats s)
826{
827 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
828}
829
830static inline float recvdBps(struct msg_stats s)
831{
832 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
833}
834
835static int sendmsg_test(struct sockmap_options *opt)
836{
837 float sent_Bps = 0, recvd_Bps = 0;
838 int rx_fd, txpid, rxpid, err = 0;
839 struct msg_stats s = {0};
840 int iov_count = opt->iov_count;
841 int iov_buf = opt->iov_length;
842 int rx_status, tx_status;
843 int cnt = opt->rate;
844
845 errno = 0;
846
847 if (opt->base)
848 rx_fd = p1;
849 else
850 rx_fd = p2;
851
852 if (ktls) {
853 /* Redirecting into non-TLS socket which sends into a TLS
854 * socket is not a valid test. So in this case lets not
855 * enable kTLS but still run the test.
856 */
857 if (!txmsg_redir || txmsg_ingress) {
858 err = sockmap_init_ktls(opt->verbose, rx_fd);
859 if (err)
860 return err;
861 }
862 err = sockmap_init_ktls(opt->verbose, c1);
863 if (err)
864 return err;
865 }
866
867 if (opt->tx_wait_mem) {
868 struct timeval timeout;
869 int rxtx_buf_len = 1024;
870
871 timeout.tv_sec = 3;
872 timeout.tv_usec = 0;
873
874 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
875 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
876 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
877 if (err) {
878 perror("setsockopt failed()");
879 return errno;
880 }
881 }
882
883 rxpid = fork();
884 if (rxpid == 0) {
885 if (opt->drop_expected || txmsg_ktls_skb_drop)
886 _exit(0);
887
888 if (!iov_buf) /* zero bytes sent case */
889 _exit(0);
890
891 if (opt->sendpage)
892 iov_count = 1;
893 err = msg_loop(rx_fd, iov_count, iov_buf,
894 cnt, &s, false, opt);
895 if (opt->verbose > 1)
896 fprintf(stderr,
897 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
898 iov_count, iov_buf, cnt, err);
899 if (s.end.tv_sec - s.start.tv_sec) {
900 sent_Bps = sentBps(s);
901 recvd_Bps = recvdBps(s);
902 }
903 if (opt->verbose > 1)
904 fprintf(stdout,
905 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
906 s.bytes_sent, sent_Bps, sent_Bps/giga,
907 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
908 peek_flag ? "(peek_msg)" : "");
909 if (err && err != -EDATAINTEGRITY && txmsg_cork)
910 err = 0;
911 exit(err ? 1 : 0);
912 } else if (rxpid == -1) {
913 perror("msg_loop_rx");
914 return errno;
915 }
916
917 if (opt->tx_wait_mem)
918 close(c2);
919
920 txpid = fork();
921 if (txpid == 0) {
922 if (opt->sendpage)
923 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
924 else
925 err = msg_loop(c1, iov_count, iov_buf,
926 cnt, &s, true, opt);
927
928 if (err)
929 fprintf(stderr,
930 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
931 iov_count, iov_buf, cnt, err);
932 if (s.end.tv_sec - s.start.tv_sec) {
933 sent_Bps = sentBps(s);
934 recvd_Bps = recvdBps(s);
935 }
936 if (opt->verbose > 1)
937 fprintf(stdout,
938 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
939 s.bytes_sent, sent_Bps, sent_Bps/giga,
940 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
941 exit(err ? 1 : 0);
942 } else if (txpid == -1) {
943 perror("msg_loop_tx");
944 return errno;
945 }
946
947 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
948 assert(waitpid(txpid, &tx_status, 0) == txpid);
949 if (WIFEXITED(rx_status)) {
950 err = WEXITSTATUS(rx_status);
951 if (err) {
952 fprintf(stderr, "rx thread exited with err %d.\n", err);
953 goto out;
954 }
955 }
956 if (WIFEXITED(tx_status)) {
957 err = WEXITSTATUS(tx_status);
958 if (err)
959 fprintf(stderr, "tx thread exited with err %d.\n", err);
960 }
961out:
962 return err;
963}
964
965static int forever_ping_pong(int rate, struct sockmap_options *opt)
966{
967 struct timeval timeout;
968 char buf[1024] = {0};
969 int sc;
970
971 timeout.tv_sec = 10;
972 timeout.tv_usec = 0;
973
974 /* Ping/Pong data from client to server */
975 sc = send(c1, buf, sizeof(buf), 0);
976 if (sc < 0) {
977 perror("send failed()");
978 return sc;
979 }
980
981 do {
982 int s, rc, i, max_fd = p2;
983 fd_set w;
984
985 /* FD sets */
986 FD_ZERO(&w);
987 FD_SET(c1, &w);
988 FD_SET(c2, &w);
989 FD_SET(p1, &w);
990 FD_SET(p2, &w);
991
992 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
993 if (s == -1) {
994 perror("select()");
995 break;
996 } else if (!s) {
997 fprintf(stderr, "unexpected timeout\n");
998 break;
999 }
1000
1001 for (i = 0; i <= max_fd && s > 0; ++i) {
1002 if (!FD_ISSET(i, &w))
1003 continue;
1004
1005 s--;
1006
1007 rc = recv(i, buf, sizeof(buf), 0);
1008 if (rc < 0) {
1009 if (errno != EWOULDBLOCK) {
1010 perror("recv failed()");
1011 return rc;
1012 }
1013 }
1014
1015 if (rc == 0) {
1016 close(i);
1017 break;
1018 }
1019
1020 sc = send(i, buf, rc, 0);
1021 if (sc < 0) {
1022 perror("send failed()");
1023 return sc;
1024 }
1025 }
1026
1027 if (rate)
1028 sleep(rate);
1029
1030 if (opt->verbose) {
1031 printf(".");
1032 fflush(stdout);
1033
1034 }
1035 } while (running);
1036
1037 return 0;
1038}
1039
1040enum {
1041 SELFTESTS,
1042 PING_PONG,
1043 SENDMSG,
1044 BASE,
1045 BASE_SENDPAGE,
1046 SENDPAGE,
1047};
1048
1049static int run_options(struct sockmap_options *options, int cg_fd, int test)
1050{
1051 int i, key, next_key, err, zero = 0;
1052 struct bpf_program *tx_prog;
1053
1054 /* If base test skip BPF setup */
1055 if (test == BASE || test == BASE_SENDPAGE)
1056 goto run;
1057
1058 /* Attach programs to sockmap */
1059 if (!txmsg_omit_skb_parser) {
1060 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]);
1061 if (!links[0]) {
1062 fprintf(stderr,
1063 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n",
1064 bpf_program__fd(progs[0]), map_fd[0], strerror(errno));
1065 return -1;
1066 }
1067 }
1068
1069 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]);
1070 if (!links[1]) {
1071 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n",
1072 strerror(errno));
1073 return -1;
1074 }
1075
1076 /* Attach programs to TLS sockmap */
1077 if (txmsg_ktls_skb) {
1078 if (!txmsg_omit_skb_parser) {
1079 links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]);
1080 if (!links[2]) {
1081 fprintf(stderr,
1082 "ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n",
1083 bpf_program__fd(progs[0]), map_fd[8], strerror(errno));
1084 return -1;
1085 }
1086 }
1087
1088 links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]);
1089 if (!links[3]) {
1090 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n",
1091 strerror(errno));
1092 return -1;
1093 }
1094 }
1095
1096 /* Attach to cgroups */
1097 err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1098 if (err) {
1099 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1100 err, strerror(errno));
1101 return err;
1102 }
1103
1104run:
1105 err = sockmap_init_sockets(options->verbose);
1106 if (err) {
1107 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1108 goto out;
1109 }
1110
1111 /* Attach txmsg program to sockmap */
1112 if (txmsg_pass)
1113 tx_prog = progs[4];
1114 else if (txmsg_redir)
1115 tx_prog = progs[5];
1116 else if (txmsg_apply)
1117 tx_prog = progs[6];
1118 else if (txmsg_cork)
1119 tx_prog = progs[7];
1120 else if (txmsg_drop)
1121 tx_prog = progs[8];
1122 else
1123 tx_prog = NULL;
1124
1125 if (tx_prog) {
1126 int redir_fd;
1127
1128 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]);
1129 if (!links[4]) {
1130 fprintf(stderr,
1131 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n",
1132 strerror(errno));
1133 err = -1;
1134 goto out;
1135 }
1136
1137 i = 0;
1138 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1139 if (err) {
1140 fprintf(stderr,
1141 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1142 err, strerror(errno));
1143 goto out;
1144 }
1145
1146 if (txmsg_redir)
1147 redir_fd = c2;
1148 else
1149 redir_fd = c1;
1150
1151 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1152 if (err) {
1153 fprintf(stderr,
1154 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1155 err, strerror(errno));
1156 goto out;
1157 }
1158
1159 if (txmsg_apply) {
1160 err = bpf_map_update_elem(map_fd[3],
1161 &i, &txmsg_apply, BPF_ANY);
1162 if (err) {
1163 fprintf(stderr,
1164 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1165 err, strerror(errno));
1166 goto out;
1167 }
1168 }
1169
1170 if (txmsg_cork) {
1171 err = bpf_map_update_elem(map_fd[4],
1172 &i, &txmsg_cork, BPF_ANY);
1173 if (err) {
1174 fprintf(stderr,
1175 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1176 err, strerror(errno));
1177 goto out;
1178 }
1179 }
1180
1181 if (txmsg_start) {
1182 err = bpf_map_update_elem(map_fd[5],
1183 &i, &txmsg_start, BPF_ANY);
1184 if (err) {
1185 fprintf(stderr,
1186 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1187 err, strerror(errno));
1188 goto out;
1189 }
1190 }
1191
1192 if (txmsg_end) {
1193 i = 1;
1194 err = bpf_map_update_elem(map_fd[5],
1195 &i, &txmsg_end, BPF_ANY);
1196 if (err) {
1197 fprintf(stderr,
1198 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1199 err, strerror(errno));
1200 goto out;
1201 }
1202 }
1203
1204 if (txmsg_start_push) {
1205 i = 2;
1206 err = bpf_map_update_elem(map_fd[5],
1207 &i, &txmsg_start_push, BPF_ANY);
1208 if (err) {
1209 fprintf(stderr,
1210 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1211 err, strerror(errno));
1212 goto out;
1213 }
1214 }
1215
1216 if (txmsg_end_push) {
1217 i = 3;
1218 err = bpf_map_update_elem(map_fd[5],
1219 &i, &txmsg_end_push, BPF_ANY);
1220 if (err) {
1221 fprintf(stderr,
1222 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1223 txmsg_end_push, i, err, strerror(errno));
1224 goto out;
1225 }
1226 }
1227
1228 if (txmsg_start_pop) {
1229 i = 4;
1230 err = bpf_map_update_elem(map_fd[5],
1231 &i, &txmsg_start_pop, BPF_ANY);
1232 if (err) {
1233 fprintf(stderr,
1234 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1235 txmsg_start_pop, i, err, strerror(errno));
1236 goto out;
1237 }
1238 } else {
1239 i = 4;
1240 bpf_map_update_elem(map_fd[5],
1241 &i, &txmsg_start_pop, BPF_ANY);
1242 }
1243
1244 if (txmsg_pop) {
1245 i = 5;
1246 err = bpf_map_update_elem(map_fd[5],
1247 &i, &txmsg_pop, BPF_ANY);
1248 if (err) {
1249 fprintf(stderr,
1250 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1251 txmsg_pop, i, err, strerror(errno));
1252 goto out;
1253 }
1254 } else {
1255 i = 5;
1256 bpf_map_update_elem(map_fd[5],
1257 &i, &txmsg_pop, BPF_ANY);
1258
1259 }
1260
1261 if (txmsg_ingress) {
1262 int in = BPF_F_INGRESS;
1263
1264 i = 0;
1265 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1266 if (err) {
1267 fprintf(stderr,
1268 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1269 err, strerror(errno));
1270 }
1271 i = 1;
1272 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1273 if (err) {
1274 fprintf(stderr,
1275 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1276 err, strerror(errno));
1277 }
1278 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1279 if (err) {
1280 fprintf(stderr,
1281 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1282 err, strerror(errno));
1283 }
1284
1285 i = 2;
1286 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1287 if (err) {
1288 fprintf(stderr,
1289 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1290 err, strerror(errno));
1291 }
1292 }
1293
1294 if (txmsg_ktls_skb) {
1295 int ingress = BPF_F_INGRESS;
1296
1297 i = 0;
1298 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1299 if (err) {
1300 fprintf(stderr,
1301 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1302 err, strerror(errno));
1303 }
1304
1305 if (txmsg_ktls_skb_redir) {
1306 i = 1;
1307 err = bpf_map_update_elem(map_fd[7],
1308 &i, &ingress, BPF_ANY);
1309 if (err) {
1310 fprintf(stderr,
1311 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1312 err, strerror(errno));
1313 }
1314 }
1315
1316 if (txmsg_ktls_skb_drop) {
1317 i = 1;
1318 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1319 }
1320 }
1321
1322 if (txmsg_redir_skb) {
1323 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1324 p2 : p1;
1325 int ingress = BPF_F_INGRESS;
1326
1327 i = 0;
1328 err = bpf_map_update_elem(map_fd[7],
1329 &i, &ingress, BPF_ANY);
1330 if (err) {
1331 fprintf(stderr,
1332 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1333 err, strerror(errno));
1334 }
1335
1336 i = 3;
1337 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1338 if (err) {
1339 fprintf(stderr,
1340 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1341 err, strerror(errno));
1342 }
1343 }
1344 }
1345
1346 if (skb_use_parser) {
1347 i = 2;
1348 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1349 }
1350
1351 if (txmsg_drop)
1352 options->drop_expected = true;
1353
1354 if (test == PING_PONG)
1355 err = forever_ping_pong(options->rate, options);
1356 else if (test == SENDMSG) {
1357 options->base = false;
1358 options->sendpage = false;
1359 err = sendmsg_test(options);
1360 } else if (test == SENDPAGE) {
1361 options->base = false;
1362 options->sendpage = true;
1363 err = sendmsg_test(options);
1364 } else if (test == BASE) {
1365 options->base = true;
1366 options->sendpage = false;
1367 err = sendmsg_test(options);
1368 } else if (test == BASE_SENDPAGE) {
1369 options->base = true;
1370 options->sendpage = true;
1371 err = sendmsg_test(options);
1372 } else
1373 fprintf(stderr, "unknown test\n");
1374out:
1375 /* Detatch and zero all the maps */
1376 bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS);
1377
1378 for (i = 0; i < ARRAY_SIZE(links); i++) {
1379 if (links[i])
1380 bpf_link__detach(links[i]);
1381 }
1382
1383 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1384 key = next_key = 0;
1385 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1386 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1387 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1388 key = next_key;
1389 }
1390 }
1391
1392 close(s1);
1393 close(s2);
1394 close(p1);
1395 close(p2);
1396 close(c1);
1397 close(c2);
1398 return err;
1399}
1400
1401static char *test_to_str(int test)
1402{
1403 switch (test) {
1404 case SENDMSG:
1405 return "sendmsg";
1406 case SENDPAGE:
1407 return "sendpage";
1408 }
1409 return "unknown";
1410}
1411
1412static void append_str(char *dst, const char *src, size_t dst_cap)
1413{
1414 size_t avail = dst_cap - strlen(dst);
1415
1416 if (avail <= 1) /* just zero byte could be written */
1417 return;
1418
1419 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1420}
1421
1422#define OPTSTRING 60
1423static void test_options(char *options)
1424{
1425 char tstr[OPTSTRING];
1426
1427 memset(options, 0, OPTSTRING);
1428
1429 if (txmsg_pass)
1430 append_str(options, "pass,", OPTSTRING);
1431 if (txmsg_redir)
1432 append_str(options, "redir,", OPTSTRING);
1433 if (txmsg_drop)
1434 append_str(options, "drop,", OPTSTRING);
1435 if (txmsg_apply) {
1436 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1437 append_str(options, tstr, OPTSTRING);
1438 }
1439 if (txmsg_cork) {
1440 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1441 append_str(options, tstr, OPTSTRING);
1442 }
1443 if (txmsg_start) {
1444 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1445 append_str(options, tstr, OPTSTRING);
1446 }
1447 if (txmsg_end) {
1448 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1449 append_str(options, tstr, OPTSTRING);
1450 }
1451 if (txmsg_start_pop) {
1452 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1453 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1454 append_str(options, tstr, OPTSTRING);
1455 }
1456 if (txmsg_ingress)
1457 append_str(options, "ingress,", OPTSTRING);
1458 if (txmsg_redir_skb)
1459 append_str(options, "redir_skb,", OPTSTRING);
1460 if (txmsg_ktls_skb)
1461 append_str(options, "ktls_skb,", OPTSTRING);
1462 if (ktls)
1463 append_str(options, "ktls,", OPTSTRING);
1464 if (peek_flag)
1465 append_str(options, "peek,", OPTSTRING);
1466}
1467
1468static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1469{
1470 char *options = calloc(OPTSTRING, sizeof(char));
1471 int err;
1472
1473 if (test == SENDPAGE)
1474 opt->sendpage = true;
1475 else
1476 opt->sendpage = false;
1477
1478 if (txmsg_drop)
1479 opt->drop_expected = true;
1480 else
1481 opt->drop_expected = false;
1482
1483 test_options(options);
1484
1485 if (opt->verbose) {
1486 fprintf(stdout,
1487 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1488 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1489 test_to_str(test), options);
1490 fflush(stdout);
1491 }
1492 err = run_options(opt, cgrp, test);
1493 if (opt->verbose)
1494 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1495 test_cnt++;
1496 !err ? passed++ : failed++;
1497 free(options);
1498 return err;
1499}
1500
1501static void test_exec(int cgrp, struct sockmap_options *opt)
1502{
1503 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1504 int err;
1505
1506 if (type == 0) {
1507 test_start();
1508 err = __test_exec(cgrp, SENDMSG, opt);
1509 if (err)
1510 test_fail();
1511 } else {
1512 test_start();
1513 err = __test_exec(cgrp, SENDPAGE, opt);
1514 if (err)
1515 test_fail();
1516 }
1517}
1518
1519static void test_send_one(struct sockmap_options *opt, int cgrp)
1520{
1521 opt->iov_length = 1;
1522 opt->iov_count = 1;
1523 opt->rate = 1;
1524 test_exec(cgrp, opt);
1525
1526 opt->iov_length = 1;
1527 opt->iov_count = 1024;
1528 opt->rate = 1;
1529 test_exec(cgrp, opt);
1530
1531 opt->iov_length = 1024;
1532 opt->iov_count = 1;
1533 opt->rate = 1;
1534 test_exec(cgrp, opt);
1535
1536}
1537
1538static void test_send_many(struct sockmap_options *opt, int cgrp)
1539{
1540 opt->iov_length = 3;
1541 opt->iov_count = 1;
1542 opt->rate = 512;
1543 test_exec(cgrp, opt);
1544
1545 opt->rate = 100;
1546 opt->iov_count = 1;
1547 opt->iov_length = 5;
1548 test_exec(cgrp, opt);
1549}
1550
1551static void test_send_large(struct sockmap_options *opt, int cgrp)
1552{
1553 opt->iov_length = 8192;
1554 opt->iov_count = 32;
1555 opt->rate = 2;
1556 test_exec(cgrp, opt);
1557}
1558
1559static void test_send(struct sockmap_options *opt, int cgrp)
1560{
1561 test_send_one(opt, cgrp);
1562 test_send_many(opt, cgrp);
1563 test_send_large(opt, cgrp);
1564 sched_yield();
1565}
1566
1567static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1568{
1569 /* Test small and large iov_count values with pass/redir/apply/cork */
1570 txmsg_pass = 1;
1571 test_send(opt, cgrp);
1572}
1573
1574static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1575{
1576 txmsg_redir = 1;
1577 test_send(opt, cgrp);
1578}
1579
1580static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1581{
1582 opt->tx_wait_mem = true;
1583 txmsg_redir = 1;
1584 test_send_large(opt, cgrp);
1585
1586 txmsg_redir = 1;
1587 txmsg_apply = 4097;
1588 test_send_large(opt, cgrp);
1589 opt->tx_wait_mem = false;
1590}
1591
1592static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1593{
1594 txmsg_drop = 1;
1595 test_send(opt, cgrp);
1596}
1597
1598static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1599{
1600 txmsg_pass = txmsg_drop = 0;
1601 txmsg_ingress = txmsg_redir = 1;
1602 test_send(opt, cgrp);
1603}
1604
1605static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1606{
1607 bool data = opt->data_test;
1608 int k = ktls;
1609
1610 opt->data_test = true;
1611 ktls = 1;
1612
1613 txmsg_pass = txmsg_drop = 0;
1614 txmsg_ingress = txmsg_redir = 0;
1615 txmsg_ktls_skb = 1;
1616 txmsg_pass = 1;
1617
1618 /* Using data verification so ensure iov layout is
1619 * expected from test receiver side. e.g. has enough
1620 * bytes to write test code.
1621 */
1622 opt->iov_length = 100;
1623 opt->iov_count = 1;
1624 opt->rate = 1;
1625 test_exec(cgrp, opt);
1626
1627 txmsg_ktls_skb_drop = 1;
1628 test_exec(cgrp, opt);
1629
1630 txmsg_ktls_skb_drop = 0;
1631 txmsg_ktls_skb_redir = 1;
1632 test_exec(cgrp, opt);
1633 txmsg_ktls_skb_redir = 0;
1634
1635 /* Tests that omit skb_parser */
1636 txmsg_omit_skb_parser = 1;
1637 ktls = 0;
1638 txmsg_ktls_skb = 0;
1639 test_exec(cgrp, opt);
1640
1641 txmsg_ktls_skb_drop = 1;
1642 test_exec(cgrp, opt);
1643 txmsg_ktls_skb_drop = 0;
1644
1645 txmsg_ktls_skb_redir = 1;
1646 test_exec(cgrp, opt);
1647
1648 ktls = 1;
1649 test_exec(cgrp, opt);
1650 txmsg_omit_skb_parser = 0;
1651
1652 opt->data_test = data;
1653 ktls = k;
1654}
1655
1656/* Test cork with hung data. This tests poor usage patterns where
1657 * cork can leave data on the ring if user program is buggy and
1658 * doesn't flush them somehow. They do take some time however
1659 * because they wait for a timeout. Test pass, redir and cork with
1660 * apply logic. Use cork size of 4097 with send_large to avoid
1661 * aligning cork size with send size.
1662 */
1663static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1664{
1665 txmsg_pass = 1;
1666 txmsg_redir = 0;
1667 txmsg_cork = 4097;
1668 txmsg_apply = 4097;
1669 test_send_large(opt, cgrp);
1670
1671 txmsg_pass = 0;
1672 txmsg_redir = 1;
1673 txmsg_apply = 0;
1674 txmsg_cork = 4097;
1675 test_send_large(opt, cgrp);
1676
1677 txmsg_pass = 0;
1678 txmsg_redir = 1;
1679 txmsg_apply = 4097;
1680 txmsg_cork = 4097;
1681 test_send_large(opt, cgrp);
1682}
1683
1684static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1685{
1686 /* Test basic start/end */
1687 txmsg_pass = 1;
1688 txmsg_start = 1;
1689 txmsg_end = 2;
1690 test_send(opt, cgrp);
1691
1692 /* Test >4k pull */
1693 txmsg_pass = 1;
1694 txmsg_start = 4096;
1695 txmsg_end = 9182;
1696 test_send_large(opt, cgrp);
1697
1698 /* Test pull + redirect */
1699 txmsg_redir = 1;
1700 txmsg_start = 1;
1701 txmsg_end = 2;
1702 test_send(opt, cgrp);
1703
1704 /* Test pull + cork */
1705 txmsg_redir = 0;
1706 txmsg_cork = 512;
1707 txmsg_start = 1;
1708 txmsg_end = 2;
1709 test_send_many(opt, cgrp);
1710
1711 /* Test pull + cork + redirect */
1712 txmsg_redir = 1;
1713 txmsg_cork = 512;
1714 txmsg_start = 1;
1715 txmsg_end = 2;
1716 test_send_many(opt, cgrp);
1717}
1718
1719static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1720{
1721 bool data = opt->data_test;
1722
1723 /* Test basic pop */
1724 txmsg_pass = 1;
1725 txmsg_start_pop = 1;
1726 txmsg_pop = 2;
1727 test_send_many(opt, cgrp);
1728
1729 /* Test pop with >4k */
1730 txmsg_pass = 1;
1731 txmsg_start_pop = 4096;
1732 txmsg_pop = 4096;
1733 test_send_large(opt, cgrp);
1734
1735 /* Test pop + redirect */
1736 txmsg_redir = 1;
1737 txmsg_start_pop = 1;
1738 txmsg_pop = 2;
1739 test_send_many(opt, cgrp);
1740
1741 /* TODO: Test for pop + cork should be different,
1742 * - It makes the layout of the received data difficult
1743 * - It makes it hard to calculate the total_bytes in the recvmsg
1744 * Temporarily skip the data integrity test for this case now.
1745 */
1746 opt->data_test = false;
1747 /* Test pop + cork */
1748 txmsg_redir = 0;
1749 txmsg_cork = 512;
1750 txmsg_start_pop = 1;
1751 txmsg_pop = 2;
1752 test_send_many(opt, cgrp);
1753
1754 /* Test pop + redirect + cork */
1755 txmsg_redir = 1;
1756 txmsg_cork = 4;
1757 txmsg_start_pop = 1;
1758 txmsg_pop = 2;
1759 test_send_many(opt, cgrp);
1760 opt->data_test = data;
1761}
1762
1763static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1764{
1765 bool data = opt->data_test;
1766
1767 /* Test basic push */
1768 txmsg_pass = 1;
1769 txmsg_start_push = 1;
1770 txmsg_end_push = 1;
1771 test_send(opt, cgrp);
1772
1773 /* Test push 4kB >4k */
1774 txmsg_pass = 1;
1775 txmsg_start_push = 4096;
1776 txmsg_end_push = 4096;
1777 test_send_large(opt, cgrp);
1778
1779 /* Test push + redirect */
1780 txmsg_redir = 1;
1781 txmsg_start_push = 1;
1782 txmsg_end_push = 2;
1783 test_send_many(opt, cgrp);
1784
1785 /* TODO: Test for push + cork should be different,
1786 * - It makes the layout of the received data difficult
1787 * - It makes it hard to calculate the total_bytes in the recvmsg
1788 * Temporarily skip the data integrity test for this case now.
1789 */
1790 opt->data_test = false;
1791 /* Test push + cork */
1792 txmsg_redir = 0;
1793 txmsg_cork = 512;
1794 txmsg_start_push = 1;
1795 txmsg_end_push = 2;
1796 test_send_many(opt, cgrp);
1797 opt->data_test = data;
1798}
1799
1800static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1801{
1802 /* Test push/pop range overlapping */
1803 txmsg_pass = 1;
1804 txmsg_start_push = 1;
1805 txmsg_end_push = 10;
1806 txmsg_start_pop = 5;
1807 txmsg_pop = 4;
1808 test_send_large(opt, cgrp);
1809
1810 txmsg_pass = 1;
1811 txmsg_start_push = 1;
1812 txmsg_end_push = 10;
1813 txmsg_start_pop = 5;
1814 txmsg_pop = 16;
1815 test_send_large(opt, cgrp);
1816
1817 txmsg_pass = 1;
1818 txmsg_start_push = 5;
1819 txmsg_end_push = 4;
1820 txmsg_start_pop = 1;
1821 txmsg_pop = 10;
1822 test_send_large(opt, cgrp);
1823
1824 txmsg_pass = 1;
1825 txmsg_start_push = 5;
1826 txmsg_end_push = 16;
1827 txmsg_start_pop = 1;
1828 txmsg_pop = 10;
1829 test_send_large(opt, cgrp);
1830
1831 /* Test push/pop range non-overlapping */
1832 txmsg_pass = 1;
1833 txmsg_start_push = 1;
1834 txmsg_end_push = 10;
1835 txmsg_start_pop = 16;
1836 txmsg_pop = 4;
1837 test_send_large(opt, cgrp);
1838
1839 txmsg_pass = 1;
1840 txmsg_start_push = 16;
1841 txmsg_end_push = 10;
1842 txmsg_start_pop = 5;
1843 txmsg_pop = 4;
1844 test_send_large(opt, cgrp);
1845}
1846
1847static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1848{
1849 txmsg_pass = 1;
1850 txmsg_redir = 0;
1851 txmsg_ingress = 0;
1852 txmsg_apply = 1;
1853 txmsg_cork = 0;
1854 test_send_one(opt, cgrp);
1855
1856 txmsg_pass = 0;
1857 txmsg_redir = 1;
1858 txmsg_ingress = 0;
1859 txmsg_apply = 1;
1860 txmsg_cork = 0;
1861 test_send_one(opt, cgrp);
1862
1863 txmsg_pass = 0;
1864 txmsg_redir = 1;
1865 txmsg_ingress = 1;
1866 txmsg_apply = 1;
1867 txmsg_cork = 0;
1868 test_send_one(opt, cgrp);
1869
1870 txmsg_pass = 1;
1871 txmsg_redir = 0;
1872 txmsg_ingress = 0;
1873 txmsg_apply = 1024;
1874 txmsg_cork = 0;
1875 test_send_large(opt, cgrp);
1876
1877 txmsg_pass = 0;
1878 txmsg_redir = 1;
1879 txmsg_ingress = 0;
1880 txmsg_apply = 1024;
1881 txmsg_cork = 0;
1882 test_send_large(opt, cgrp);
1883
1884 txmsg_pass = 0;
1885 txmsg_redir = 1;
1886 txmsg_ingress = 1;
1887 txmsg_apply = 1024;
1888 txmsg_cork = 0;
1889 test_send_large(opt, cgrp);
1890}
1891
1892static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1893{
1894 txmsg_pass = 1;
1895 txmsg_redir = 0;
1896 txmsg_apply = 0;
1897 txmsg_cork = 1;
1898 test_send(opt, cgrp);
1899
1900 txmsg_pass = 1;
1901 txmsg_redir = 0;
1902 txmsg_apply = 1;
1903 txmsg_cork = 1;
1904 test_send(opt, cgrp);
1905}
1906
1907static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1908{
1909 txmsg_pass = 1;
1910 skb_use_parser = 512;
1911 if (ktls == 1)
1912 skb_use_parser = 570;
1913 opt->iov_length = 256;
1914 opt->iov_count = 1;
1915 opt->rate = 2;
1916 test_exec(cgrp, opt);
1917}
1918
1919static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1920{
1921 if (ktls == 1)
1922 return;
1923 skb_use_parser = 10;
1924 opt->iov_length = 20;
1925 opt->iov_count = 1;
1926 opt->rate = 1;
1927 opt->check_recved_len = true;
1928 test_exec(cgrp, opt);
1929 opt->check_recved_len = false;
1930}
1931
1932char *map_names[] = {
1933 "sock_map",
1934 "sock_map_txmsg",
1935 "sock_map_redir",
1936 "sock_apply_bytes",
1937 "sock_cork_bytes",
1938 "sock_bytes",
1939 "sock_redir_flags",
1940 "sock_skb_opts",
1941 "tls_sock_map",
1942};
1943
1944static int populate_progs(char *bpf_file)
1945{
1946 struct bpf_program *prog;
1947 struct bpf_object *obj;
1948 int i = 0;
1949 long err;
1950
1951 obj = bpf_object__open(bpf_file);
1952 err = libbpf_get_error(obj);
1953 if (err) {
1954 char err_buf[256];
1955
1956 libbpf_strerror(err, err_buf, sizeof(err_buf));
1957 printf("Unable to load eBPF objects in file '%s' : %s\n",
1958 bpf_file, err_buf);
1959 return -1;
1960 }
1961
1962 i = bpf_object__load(obj);
1963 i = 0;
1964 bpf_object__for_each_program(prog, obj) {
1965 progs[i] = prog;
1966 i++;
1967 }
1968
1969 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1970 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1971 map_fd[i] = bpf_map__fd(maps[i]);
1972 if (map_fd[i] < 0) {
1973 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1974 map_fd[i], strerror(errno));
1975 return -1;
1976 }
1977 }
1978
1979 for (i = 0; i < ARRAY_SIZE(links); i++)
1980 links[i] = NULL;
1981
1982 return 0;
1983}
1984
1985struct _test test[] = {
1986 {"txmsg test passthrough", test_txmsg_pass},
1987 {"txmsg test redirect", test_txmsg_redir},
1988 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1989 {"txmsg test drop", test_txmsg_drop},
1990 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1991 {"txmsg test skb", test_txmsg_skb},
1992 {"txmsg test apply", test_txmsg_apply},
1993 {"txmsg test cork", test_txmsg_cork},
1994 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1995 {"txmsg test push_data", test_txmsg_push},
1996 {"txmsg test pull-data", test_txmsg_pull},
1997 {"txmsg test pop-data", test_txmsg_pop},
1998 {"txmsg test push/pop data", test_txmsg_push_pop},
1999 {"txmsg test ingress parser", test_txmsg_ingress_parser},
2000 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
2001};
2002
2003static int check_whitelist(struct _test *t, struct sockmap_options *opt)
2004{
2005 char *entry, *ptr;
2006
2007 if (!opt->whitelist)
2008 return 0;
2009 ptr = strdup(opt->whitelist);
2010 if (!ptr)
2011 return -ENOMEM;
2012 entry = strtok(ptr, ",");
2013 while (entry) {
2014 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
2015 strstr(opt->map, entry) != 0 ||
2016 strstr(t->title, entry) != 0) {
2017 free(ptr);
2018 return 0;
2019 }
2020 entry = strtok(NULL, ",");
2021 }
2022 free(ptr);
2023 return -EINVAL;
2024}
2025
2026static int check_blacklist(struct _test *t, struct sockmap_options *opt)
2027{
2028 char *entry, *ptr;
2029
2030 if (!opt->blacklist)
2031 return -EINVAL;
2032 ptr = strdup(opt->blacklist);
2033 if (!ptr)
2034 return -ENOMEM;
2035 entry = strtok(ptr, ",");
2036 while (entry) {
2037 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
2038 strstr(opt->map, entry) != 0 ||
2039 strstr(t->title, entry) != 0) {
2040 free(ptr);
2041 return 0;
2042 }
2043 entry = strtok(NULL, ",");
2044 }
2045 free(ptr);
2046 return -EINVAL;
2047}
2048
2049static int __test_selftests(int cg_fd, struct sockmap_options *opt)
2050{
2051 int i, err;
2052
2053 err = populate_progs(opt->map);
2054 if (err < 0) {
2055 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
2056 return err;
2057 }
2058
2059 /* Tests basic commands and APIs */
2060 for (i = 0; i < ARRAY_SIZE(test); i++) {
2061 struct _test t = test[i];
2062
2063 if (check_whitelist(&t, opt) != 0)
2064 continue;
2065 if (check_blacklist(&t, opt) == 0)
2066 continue;
2067
2068 test_start_subtest(&t, opt);
2069 t.tester(cg_fd, opt);
2070 test_end_subtest();
2071 }
2072
2073 return err;
2074}
2075
2076static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
2077{
2078 opt->map = BPF_SOCKMAP_FILENAME;
2079 __test_selftests(cg_fd, opt);
2080}
2081
2082static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
2083{
2084 opt->map = BPF_SOCKHASH_FILENAME;
2085 __test_selftests(cg_fd, opt);
2086}
2087
2088static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
2089{
2090 opt->map = BPF_SOCKHASH_FILENAME;
2091 opt->prepend = "ktls";
2092 ktls = 1;
2093 __test_selftests(cg_fd, opt);
2094 ktls = 0;
2095}
2096
2097static int test_selftest(int cg_fd, struct sockmap_options *opt)
2098{
2099 test_selftests_sockmap(cg_fd, opt);
2100 test_selftests_sockhash(cg_fd, opt);
2101 test_selftests_ktls(cg_fd, opt);
2102 test_print_results();
2103 return 0;
2104}
2105
2106int main(int argc, char **argv)
2107{
2108 int iov_count = 1, length = 1024, rate = 1;
2109 struct sockmap_options options = {0};
2110 int opt, longindex, err, cg_fd = 0;
2111 char *bpf_file = BPF_SOCKMAP_FILENAME;
2112 int test = SELFTESTS;
2113 bool cg_created = 0;
2114
2115 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
2116 long_options, &longindex)) != -1) {
2117 switch (opt) {
2118 case 's':
2119 txmsg_start = atoi(optarg);
2120 break;
2121 case 'e':
2122 txmsg_end = atoi(optarg);
2123 break;
2124 case 'p':
2125 txmsg_start_push = atoi(optarg);
2126 break;
2127 case 'q':
2128 txmsg_end_push = atoi(optarg);
2129 break;
2130 case 'w':
2131 txmsg_start_pop = atoi(optarg);
2132 break;
2133 case 'x':
2134 txmsg_pop = atoi(optarg);
2135 break;
2136 case 'a':
2137 txmsg_apply = atoi(optarg);
2138 break;
2139 case 'k':
2140 txmsg_cork = atoi(optarg);
2141 break;
2142 case 'c':
2143 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
2144 if (cg_fd < 0) {
2145 fprintf(stderr,
2146 "ERROR: (%i) open cg path failed: %s\n",
2147 cg_fd, optarg);
2148 return cg_fd;
2149 }
2150 break;
2151 case 'r':
2152 rate = atoi(optarg);
2153 break;
2154 case 'v':
2155 options.verbose = 1;
2156 if (optarg)
2157 options.verbose = atoi(optarg);
2158 break;
2159 case 'i':
2160 iov_count = atoi(optarg);
2161 break;
2162 case 'l':
2163 length = atoi(optarg);
2164 break;
2165 case 'd':
2166 options.data_test = true;
2167 break;
2168 case 't':
2169 if (strcmp(optarg, "ping") == 0) {
2170 test = PING_PONG;
2171 } else if (strcmp(optarg, "sendmsg") == 0) {
2172 test = SENDMSG;
2173 } else if (strcmp(optarg, "base") == 0) {
2174 test = BASE;
2175 } else if (strcmp(optarg, "base_sendpage") == 0) {
2176 test = BASE_SENDPAGE;
2177 } else if (strcmp(optarg, "sendpage") == 0) {
2178 test = SENDPAGE;
2179 } else {
2180 usage(argv);
2181 return -1;
2182 }
2183 break;
2184 case 'n':
2185 options.whitelist = strdup(optarg);
2186 if (!options.whitelist)
2187 return -ENOMEM;
2188 break;
2189 case 'b':
2190 options.blacklist = strdup(optarg);
2191 if (!options.blacklist)
2192 return -ENOMEM;
2193 case 0:
2194 break;
2195 case 'h':
2196 default:
2197 usage(argv);
2198 return -1;
2199 }
2200 }
2201
2202 if (!cg_fd) {
2203 cg_fd = cgroup_setup_and_join(CG_PATH);
2204 if (cg_fd < 0)
2205 return cg_fd;
2206 cg_created = 1;
2207 }
2208
2209 /* Use libbpf 1.0 API mode */
2210 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2211
2212 if (test == SELFTESTS) {
2213 err = test_selftest(cg_fd, &options);
2214 goto out;
2215 }
2216
2217 err = populate_progs(bpf_file);
2218 if (err) {
2219 fprintf(stderr, "populate program: (%s) %s\n",
2220 bpf_file, strerror(errno));
2221 return 1;
2222 }
2223 running = 1;
2224
2225 /* catch SIGINT */
2226 signal(SIGINT, running_handler);
2227
2228 options.iov_count = iov_count;
2229 options.iov_length = length;
2230 options.rate = rate;
2231
2232 err = run_options(&options, cg_fd, test);
2233out:
2234 if (options.whitelist)
2235 free(options.whitelist);
2236 if (options.blacklist)
2237 free(options.blacklist);
2238 close(cg_fd);
2239 if (cg_created)
2240 cleanup_cgroup_environment();
2241 return err;
2242}
2243
2244void running_handler(int a)
2245{
2246 running = 0;
2247}