Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3#include <stdio.h>
4#include <stdlib.h>
5#include <sys/socket.h>
6#include <sys/ioctl.h>
7#include <sys/select.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10#include <unistd.h>
11#include <string.h>
12#include <errno.h>
13#include <stdbool.h>
14#include <signal.h>
15#include <fcntl.h>
16#include <sys/wait.h>
17#include <time.h>
18#include <sched.h>
19
20#include <sys/time.h>
21#include <sys/types.h>
22#include <sys/sendfile.h>
23
24#include <linux/netlink.h>
25#include <linux/socket.h>
26#include <linux/sock_diag.h>
27#include <linux/bpf.h>
28#include <linux/if_link.h>
29#include <linux/tls.h>
30#include <assert.h>
31#include <libgen.h>
32
33#include <getopt.h>
34
35#include <bpf/bpf.h>
36#include <bpf/libbpf.h>
37
38#include "bpf_util.h"
39#include "cgroup_helpers.h"
40
41int running;
42static void running_handler(int a);
43
44#ifndef TCP_ULP
45# define TCP_ULP 31
46#endif
47#ifndef SOL_TLS
48# define SOL_TLS 282
49#endif
50
51/* randomly selected ports for testing on lo */
52#define S1_PORT 10000
53#define S2_PORT 10001
54
55#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
56#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57#define CG_PATH "/sockmap"
58
59/* global sockets */
60int s1, s2, c1, c2, p1, p2;
61int test_cnt;
62int passed;
63int failed;
64int map_fd[9];
65struct bpf_map *maps[9];
66int prog_fd[11];
67
68int txmsg_pass;
69int txmsg_redir;
70int txmsg_drop;
71int txmsg_apply;
72int txmsg_cork;
73int txmsg_start;
74int txmsg_end;
75int txmsg_start_push;
76int txmsg_end_push;
77int txmsg_start_pop;
78int txmsg_pop;
79int txmsg_ingress;
80int txmsg_redir_skb;
81int txmsg_ktls_skb;
82int txmsg_ktls_skb_drop;
83int txmsg_ktls_skb_redir;
84int ktls;
85int peek_flag;
86int skb_use_parser;
87int txmsg_omit_skb_parser;
88
89static const struct option long_options[] = {
90 {"help", no_argument, NULL, 'h' },
91 {"cgroup", required_argument, NULL, 'c' },
92 {"rate", required_argument, NULL, 'r' },
93 {"verbose", optional_argument, NULL, 'v' },
94 {"iov_count", required_argument, NULL, 'i' },
95 {"length", required_argument, NULL, 'l' },
96 {"test", required_argument, NULL, 't' },
97 {"data_test", no_argument, NULL, 'd' },
98 {"txmsg", no_argument, &txmsg_pass, 1 },
99 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
100 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
101 {"txmsg_apply", required_argument, NULL, 'a'},
102 {"txmsg_cork", required_argument, NULL, 'k'},
103 {"txmsg_start", required_argument, NULL, 's'},
104 {"txmsg_end", required_argument, NULL, 'e'},
105 {"txmsg_start_push", required_argument, NULL, 'p'},
106 {"txmsg_end_push", required_argument, NULL, 'q'},
107 {"txmsg_start_pop", required_argument, NULL, 'w'},
108 {"txmsg_pop", required_argument, NULL, 'x'},
109 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
110 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
111 {"ktls", no_argument, &ktls, 1 },
112 {"peek", no_argument, &peek_flag, 1 },
113 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
114 {"whitelist", required_argument, NULL, 'n' },
115 {"blacklist", required_argument, NULL, 'b' },
116 {0, 0, NULL, 0 }
117};
118
119struct test_env {
120 const char *type;
121 const char *subtest;
122 const char *prepend;
123
124 int test_num;
125 int subtest_num;
126
127 int succ_cnt;
128 int fail_cnt;
129 int fail_last;
130};
131
132struct test_env env;
133
134struct sockmap_options {
135 int verbose;
136 bool base;
137 bool sendpage;
138 bool data_test;
139 bool drop_expected;
140 bool check_recved_len;
141 bool tx_wait_mem;
142 int iov_count;
143 int iov_length;
144 int rate;
145 char *map;
146 char *whitelist;
147 char *blacklist;
148 char *prepend;
149};
150
151struct _test {
152 char *title;
153 void (*tester)(int cg_fd, struct sockmap_options *opt);
154};
155
156static void test_start(void)
157{
158 env.subtest_num++;
159}
160
161static void test_fail(void)
162{
163 env.fail_cnt++;
164}
165
166static void test_pass(void)
167{
168 env.succ_cnt++;
169}
170
171static void test_reset(void)
172{
173 txmsg_start = txmsg_end = 0;
174 txmsg_start_pop = txmsg_pop = 0;
175 txmsg_start_push = txmsg_end_push = 0;
176 txmsg_pass = txmsg_drop = txmsg_redir = 0;
177 txmsg_apply = txmsg_cork = 0;
178 txmsg_ingress = txmsg_redir_skb = 0;
179 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
180 txmsg_omit_skb_parser = 0;
181 skb_use_parser = 0;
182}
183
184static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
185{
186 env.type = o->map;
187 env.subtest = t->title;
188 env.prepend = o->prepend;
189 env.test_num++;
190 env.subtest_num = 0;
191 env.fail_last = env.fail_cnt;
192 test_reset();
193 return 0;
194}
195
196static void test_end_subtest(void)
197{
198 int error = env.fail_cnt - env.fail_last;
199 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
200
201 if (!error)
202 test_pass();
203
204 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
205 env.test_num, env.subtest_num,
206 !type ? "sockmap" : "sockhash",
207 env.prepend ? : "",
208 env.subtest, error ? "FAIL" : "OK");
209}
210
211static void test_print_results(void)
212{
213 fprintf(stdout, "Pass: %d Fail: %d\n",
214 env.succ_cnt, env.fail_cnt);
215}
216
217static void usage(char *argv[])
218{
219 int i;
220
221 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
222 printf(" options:\n");
223 for (i = 0; long_options[i].name != 0; i++) {
224 printf(" --%-12s", long_options[i].name);
225 if (long_options[i].flag != NULL)
226 printf(" flag (internal value:%d)\n",
227 *long_options[i].flag);
228 else
229 printf(" -%c\n", long_options[i].val);
230 }
231 printf("\n");
232}
233
234char *sock_to_string(int s)
235{
236 if (s == c1)
237 return "client1";
238 else if (s == c2)
239 return "client2";
240 else if (s == s1)
241 return "server1";
242 else if (s == s2)
243 return "server2";
244 else if (s == p1)
245 return "peer1";
246 else if (s == p2)
247 return "peer2";
248 else
249 return "unknown";
250}
251
252static int sockmap_init_ktls(int verbose, int s)
253{
254 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
255 .info = {
256 .version = TLS_1_2_VERSION,
257 .cipher_type = TLS_CIPHER_AES_GCM_128,
258 },
259 };
260 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
261 .info = {
262 .version = TLS_1_2_VERSION,
263 .cipher_type = TLS_CIPHER_AES_GCM_128,
264 },
265 };
266 int so_buf = 6553500;
267 int err;
268
269 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
270 if (err) {
271 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
272 return -EINVAL;
273 }
274 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
275 if (err) {
276 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
277 return -EINVAL;
278 }
279 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
280 if (err) {
281 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
282 return -EINVAL;
283 }
284 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
285 if (err) {
286 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
287 return -EINVAL;
288 }
289 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
290 if (err) {
291 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
292 return -EINVAL;
293 }
294
295 if (verbose)
296 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
297 return 0;
298}
299static int sockmap_init_sockets(int verbose)
300{
301 int i, err, one = 1;
302 struct sockaddr_in addr;
303 int *fds[4] = {&s1, &s2, &c1, &c2};
304
305 s1 = s2 = p1 = p2 = c1 = c2 = 0;
306
307 /* Init sockets */
308 for (i = 0; i < 4; i++) {
309 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
310 if (*fds[i] < 0) {
311 perror("socket s1 failed()");
312 return errno;
313 }
314 }
315
316 /* Allow reuse */
317 for (i = 0; i < 2; i++) {
318 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
319 (char *)&one, sizeof(one));
320 if (err) {
321 perror("setsockopt failed()");
322 return errno;
323 }
324 }
325
326 /* Non-blocking sockets */
327 for (i = 0; i < 2; i++) {
328 err = ioctl(*fds[i], FIONBIO, (char *)&one);
329 if (err < 0) {
330 perror("ioctl s1 failed()");
331 return errno;
332 }
333 }
334
335 /* Bind server sockets */
336 memset(&addr, 0, sizeof(struct sockaddr_in));
337 addr.sin_family = AF_INET;
338 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
339
340 addr.sin_port = htons(S1_PORT);
341 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
342 if (err < 0) {
343 perror("bind s1 failed()");
344 return errno;
345 }
346
347 addr.sin_port = htons(S2_PORT);
348 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
349 if (err < 0) {
350 perror("bind s2 failed()");
351 return errno;
352 }
353
354 /* Listen server sockets */
355 addr.sin_port = htons(S1_PORT);
356 err = listen(s1, 32);
357 if (err < 0) {
358 perror("listen s1 failed()");
359 return errno;
360 }
361
362 addr.sin_port = htons(S2_PORT);
363 err = listen(s2, 32);
364 if (err < 0) {
365 perror("listen s1 failed()");
366 return errno;
367 }
368
369 /* Initiate Connect */
370 addr.sin_port = htons(S1_PORT);
371 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
372 if (err < 0 && errno != EINPROGRESS) {
373 perror("connect c1 failed()");
374 return errno;
375 }
376
377 addr.sin_port = htons(S2_PORT);
378 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
379 if (err < 0 && errno != EINPROGRESS) {
380 perror("connect c2 failed()");
381 return errno;
382 } else if (err < 0) {
383 err = 0;
384 }
385
386 /* Accept Connecrtions */
387 p1 = accept(s1, NULL, NULL);
388 if (p1 < 0) {
389 perror("accept s1 failed()");
390 return errno;
391 }
392
393 p2 = accept(s2, NULL, NULL);
394 if (p2 < 0) {
395 perror("accept s1 failed()");
396 return errno;
397 }
398
399 if (verbose > 1) {
400 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
401 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
402 c1, s1, c2, s2);
403 }
404 return 0;
405}
406
407struct msg_stats {
408 size_t bytes_sent;
409 size_t bytes_recvd;
410 struct timespec start;
411 struct timespec end;
412};
413
414static int msg_loop_sendpage(int fd, int iov_length, int cnt,
415 struct msg_stats *s,
416 struct sockmap_options *opt)
417{
418 bool drop = opt->drop_expected;
419 unsigned char k = 0;
420 FILE *file;
421 int i, fp;
422
423 file = tmpfile();
424 if (!file) {
425 perror("create file for sendpage");
426 return 1;
427 }
428 for (i = 0; i < iov_length * cnt; i++, k++)
429 fwrite(&k, sizeof(char), 1, file);
430 fflush(file);
431 fseek(file, 0, SEEK_SET);
432
433 fp = fileno(file);
434
435 clock_gettime(CLOCK_MONOTONIC, &s->start);
436 for (i = 0; i < cnt; i++) {
437 int sent;
438
439 errno = 0;
440 sent = sendfile(fd, fp, NULL, iov_length);
441
442 if (!drop && sent < 0) {
443 perror("sendpage loop error");
444 fclose(file);
445 return sent;
446 } else if (drop && sent >= 0) {
447 printf("sendpage loop error expected: %i errno %i\n",
448 sent, errno);
449 fclose(file);
450 return -EIO;
451 }
452
453 if (sent > 0)
454 s->bytes_sent += sent;
455 }
456 clock_gettime(CLOCK_MONOTONIC, &s->end);
457 fclose(file);
458 return 0;
459}
460
461static void msg_free_iov(struct msghdr *msg)
462{
463 int i;
464
465 for (i = 0; i < msg->msg_iovlen; i++)
466 free(msg->msg_iov[i].iov_base);
467 free(msg->msg_iov);
468 msg->msg_iov = NULL;
469 msg->msg_iovlen = 0;
470}
471
472static int msg_alloc_iov(struct msghdr *msg,
473 int iov_count, int iov_length,
474 bool data, bool xmit)
475{
476 unsigned char k = 0;
477 struct iovec *iov;
478 int i;
479
480 iov = calloc(iov_count, sizeof(struct iovec));
481 if (!iov)
482 return errno;
483
484 for (i = 0; i < iov_count; i++) {
485 unsigned char *d = calloc(iov_length, sizeof(char));
486
487 if (!d) {
488 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
489 goto unwind_iov;
490 }
491 iov[i].iov_base = d;
492 iov[i].iov_len = iov_length;
493
494 if (data && xmit) {
495 int j;
496
497 for (j = 0; j < iov_length; j++)
498 d[j] = k++;
499 }
500 }
501
502 msg->msg_iov = iov;
503 msg->msg_iovlen = iov_count;
504
505 return 0;
506unwind_iov:
507 for (i--; i >= 0 ; i--)
508 free(msg->msg_iov[i].iov_base);
509 return -ENOMEM;
510}
511
512static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
513{
514 int i, j = 0, bytes_cnt = 0;
515 unsigned char k = 0;
516
517 for (i = 0; i < msg->msg_iovlen; i++) {
518 unsigned char *d = msg->msg_iov[i].iov_base;
519
520 /* Special case test for skb ingress + ktls */
521 if (i == 0 && txmsg_ktls_skb) {
522 if (msg->msg_iov[i].iov_len < 4)
523 return -EIO;
524 if (memcmp(d, "PASS", 4) != 0) {
525 fprintf(stderr,
526 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
527 i, 0, d[0], d[1], d[2], d[3]);
528 return -EIO;
529 }
530 j = 4; /* advance index past PASS header */
531 }
532
533 for (; j < msg->msg_iov[i].iov_len && size; j++) {
534 if (d[j] != k++) {
535 fprintf(stderr,
536 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537 i, j, d[j], k - 1, d[j+1], k);
538 return -EIO;
539 }
540 bytes_cnt++;
541 if (bytes_cnt == chunk_sz) {
542 k = 0;
543 bytes_cnt = 0;
544 }
545 size--;
546 }
547 }
548 return 0;
549}
550
551static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
552 struct msg_stats *s, bool tx,
553 struct sockmap_options *opt)
554{
555 struct msghdr msg = {0}, msg_peek = {0};
556 int err, i, flags = MSG_NOSIGNAL;
557 bool drop = opt->drop_expected;
558 bool data = opt->data_test;
559 int iov_alloc_length = iov_length;
560
561 if (!tx && opt->check_recved_len)
562 iov_alloc_length *= 2;
563
564 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
565 if (err)
566 goto out_errno;
567 if (peek_flag) {
568 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
569 if (err)
570 goto out_errno;
571 }
572
573 if (tx) {
574 clock_gettime(CLOCK_MONOTONIC, &s->start);
575 for (i = 0; i < cnt; i++) {
576 int sent;
577
578 errno = 0;
579 sent = sendmsg(fd, &msg, flags);
580
581 if (!drop && sent < 0) {
582 if (opt->tx_wait_mem && errno == EACCES) {
583 errno = 0;
584 goto out_errno;
585 }
586 perror("sendmsg loop error");
587 goto out_errno;
588 } else if (drop && sent >= 0) {
589 fprintf(stderr,
590 "sendmsg loop error expected: %i errno %i\n",
591 sent, errno);
592 errno = -EIO;
593 goto out_errno;
594 }
595 if (sent > 0)
596 s->bytes_sent += sent;
597 }
598 clock_gettime(CLOCK_MONOTONIC, &s->end);
599 } else {
600 int slct, recvp = 0, recv, max_fd = fd;
601 float total_bytes, txmsg_pop_total;
602 int fd_flags = O_NONBLOCK;
603 struct timeval timeout;
604 fd_set w;
605
606 fcntl(fd, fd_flags);
607 /* Account for pop bytes noting each iteration of apply will
608 * call msg_pop_data helper so we need to account for this
609 * by calculating the number of apply iterations. Note user
610 * of the tool can create cases where no data is sent by
611 * manipulating pop/push/pull/etc. For example txmsg_apply 1
612 * with txmsg_pop 1 will try to apply 1B at a time but each
613 * iteration will then pop 1B so no data will ever be sent.
614 * This is really only useful for testing edge cases in code
615 * paths.
616 */
617 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
618 if (txmsg_apply)
619 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
620 else
621 txmsg_pop_total = txmsg_pop * cnt;
622 total_bytes -= txmsg_pop_total;
623 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
624 if (err < 0)
625 perror("recv start time");
626 while (s->bytes_recvd < total_bytes) {
627 if (txmsg_cork) {
628 timeout.tv_sec = 0;
629 timeout.tv_usec = 300000;
630 } else {
631 timeout.tv_sec = 3;
632 timeout.tv_usec = 0;
633 }
634
635 /* FD sets */
636 FD_ZERO(&w);
637 FD_SET(fd, &w);
638
639 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
640 if (slct == -1) {
641 perror("select()");
642 clock_gettime(CLOCK_MONOTONIC, &s->end);
643 goto out_errno;
644 } else if (!slct) {
645 if (opt->verbose)
646 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
647 errno = -EIO;
648 clock_gettime(CLOCK_MONOTONIC, &s->end);
649 goto out_errno;
650 }
651
652 if (opt->tx_wait_mem) {
653 FD_ZERO(&w);
654 FD_SET(fd, &w);
655 slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
656 errno = 0;
657 close(fd);
658 goto out_errno;
659 }
660
661 errno = 0;
662 if (peek_flag) {
663 flags |= MSG_PEEK;
664 recvp = recvmsg(fd, &msg_peek, flags);
665 if (recvp < 0) {
666 if (errno != EWOULDBLOCK) {
667 clock_gettime(CLOCK_MONOTONIC, &s->end);
668 goto out_errno;
669 }
670 }
671 flags = 0;
672 }
673
674 recv = recvmsg(fd, &msg, flags);
675 if (recv < 0) {
676 if (errno != EWOULDBLOCK) {
677 clock_gettime(CLOCK_MONOTONIC, &s->end);
678 perror("recv failed()");
679 goto out_errno;
680 }
681 }
682
683 s->bytes_recvd += recv;
684
685 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
686 errno = EMSGSIZE;
687 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
688 s->bytes_recvd, total_bytes);
689 goto out_errno;
690 }
691
692 if (data) {
693 int chunk_sz = opt->sendpage ?
694 iov_length * cnt :
695 iov_length * iov_count;
696
697 errno = msg_verify_data(&msg, recv, chunk_sz);
698 if (errno) {
699 perror("data verify msg failed");
700 goto out_errno;
701 }
702 if (recvp) {
703 errno = msg_verify_data(&msg_peek,
704 recvp,
705 chunk_sz);
706 if (errno) {
707 perror("data verify msg_peek failed");
708 goto out_errno;
709 }
710 }
711 }
712 }
713 clock_gettime(CLOCK_MONOTONIC, &s->end);
714 }
715
716 msg_free_iov(&msg);
717 msg_free_iov(&msg_peek);
718 return err;
719out_errno:
720 msg_free_iov(&msg);
721 msg_free_iov(&msg_peek);
722 return errno;
723}
724
725static float giga = 1000000000;
726
727static inline float sentBps(struct msg_stats s)
728{
729 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
730}
731
732static inline float recvdBps(struct msg_stats s)
733{
734 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
735}
736
737static int sendmsg_test(struct sockmap_options *opt)
738{
739 float sent_Bps = 0, recvd_Bps = 0;
740 int rx_fd, txpid, rxpid, err = 0;
741 struct msg_stats s = {0};
742 int iov_count = opt->iov_count;
743 int iov_buf = opt->iov_length;
744 int rx_status, tx_status;
745 int cnt = opt->rate;
746
747 errno = 0;
748
749 if (opt->base)
750 rx_fd = p1;
751 else
752 rx_fd = p2;
753
754 if (ktls) {
755 /* Redirecting into non-TLS socket which sends into a TLS
756 * socket is not a valid test. So in this case lets not
757 * enable kTLS but still run the test.
758 */
759 if (!txmsg_redir || txmsg_ingress) {
760 err = sockmap_init_ktls(opt->verbose, rx_fd);
761 if (err)
762 return err;
763 }
764 err = sockmap_init_ktls(opt->verbose, c1);
765 if (err)
766 return err;
767 }
768
769 if (opt->tx_wait_mem) {
770 struct timeval timeout;
771 int rxtx_buf_len = 1024;
772
773 timeout.tv_sec = 3;
774 timeout.tv_usec = 0;
775
776 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
777 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
778 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
779 if (err) {
780 perror("setsockopt failed()");
781 return errno;
782 }
783 }
784
785 rxpid = fork();
786 if (rxpid == 0) {
787 if (txmsg_pop || txmsg_start_pop)
788 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
789 if (opt->drop_expected || txmsg_ktls_skb_drop)
790 _exit(0);
791
792 if (!iov_buf) /* zero bytes sent case */
793 _exit(0);
794
795 if (opt->sendpage)
796 iov_count = 1;
797 err = msg_loop(rx_fd, iov_count, iov_buf,
798 cnt, &s, false, opt);
799 if (opt->verbose > 1)
800 fprintf(stderr,
801 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
802 iov_count, iov_buf, cnt, err);
803 if (s.end.tv_sec - s.start.tv_sec) {
804 sent_Bps = sentBps(s);
805 recvd_Bps = recvdBps(s);
806 }
807 if (opt->verbose > 1)
808 fprintf(stdout,
809 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
810 s.bytes_sent, sent_Bps, sent_Bps/giga,
811 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
812 peek_flag ? "(peek_msg)" : "");
813 if (err && txmsg_cork)
814 err = 0;
815 exit(err ? 1 : 0);
816 } else if (rxpid == -1) {
817 perror("msg_loop_rx");
818 return errno;
819 }
820
821 if (opt->tx_wait_mem)
822 close(c2);
823
824 txpid = fork();
825 if (txpid == 0) {
826 if (opt->sendpage)
827 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
828 else
829 err = msg_loop(c1, iov_count, iov_buf,
830 cnt, &s, true, opt);
831
832 if (err)
833 fprintf(stderr,
834 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
835 iov_count, iov_buf, cnt, err);
836 if (s.end.tv_sec - s.start.tv_sec) {
837 sent_Bps = sentBps(s);
838 recvd_Bps = recvdBps(s);
839 }
840 if (opt->verbose > 1)
841 fprintf(stdout,
842 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
843 s.bytes_sent, sent_Bps, sent_Bps/giga,
844 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
845 exit(err ? 1 : 0);
846 } else if (txpid == -1) {
847 perror("msg_loop_tx");
848 return errno;
849 }
850
851 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
852 assert(waitpid(txpid, &tx_status, 0) == txpid);
853 if (WIFEXITED(rx_status)) {
854 err = WEXITSTATUS(rx_status);
855 if (err) {
856 fprintf(stderr, "rx thread exited with err %d.\n", err);
857 goto out;
858 }
859 }
860 if (WIFEXITED(tx_status)) {
861 err = WEXITSTATUS(tx_status);
862 if (err)
863 fprintf(stderr, "tx thread exited with err %d.\n", err);
864 }
865out:
866 return err;
867}
868
869static int forever_ping_pong(int rate, struct sockmap_options *opt)
870{
871 struct timeval timeout;
872 char buf[1024] = {0};
873 int sc;
874
875 timeout.tv_sec = 10;
876 timeout.tv_usec = 0;
877
878 /* Ping/Pong data from client to server */
879 sc = send(c1, buf, sizeof(buf), 0);
880 if (sc < 0) {
881 perror("send failed()");
882 return sc;
883 }
884
885 do {
886 int s, rc, i, max_fd = p2;
887 fd_set w;
888
889 /* FD sets */
890 FD_ZERO(&w);
891 FD_SET(c1, &w);
892 FD_SET(c2, &w);
893 FD_SET(p1, &w);
894 FD_SET(p2, &w);
895
896 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
897 if (s == -1) {
898 perror("select()");
899 break;
900 } else if (!s) {
901 fprintf(stderr, "unexpected timeout\n");
902 break;
903 }
904
905 for (i = 0; i <= max_fd && s > 0; ++i) {
906 if (!FD_ISSET(i, &w))
907 continue;
908
909 s--;
910
911 rc = recv(i, buf, sizeof(buf), 0);
912 if (rc < 0) {
913 if (errno != EWOULDBLOCK) {
914 perror("recv failed()");
915 return rc;
916 }
917 }
918
919 if (rc == 0) {
920 close(i);
921 break;
922 }
923
924 sc = send(i, buf, rc, 0);
925 if (sc < 0) {
926 perror("send failed()");
927 return sc;
928 }
929 }
930
931 if (rate)
932 sleep(rate);
933
934 if (opt->verbose) {
935 printf(".");
936 fflush(stdout);
937
938 }
939 } while (running);
940
941 return 0;
942}
943
944enum {
945 SELFTESTS,
946 PING_PONG,
947 SENDMSG,
948 BASE,
949 BASE_SENDPAGE,
950 SENDPAGE,
951};
952
953static int run_options(struct sockmap_options *options, int cg_fd, int test)
954{
955 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
956
957 /* If base test skip BPF setup */
958 if (test == BASE || test == BASE_SENDPAGE)
959 goto run;
960
961 /* Attach programs to sockmap */
962 if (!txmsg_omit_skb_parser) {
963 err = bpf_prog_attach(prog_fd[0], map_fd[0],
964 BPF_SK_SKB_STREAM_PARSER, 0);
965 if (err) {
966 fprintf(stderr,
967 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
968 prog_fd[0], map_fd[0], err, strerror(errno));
969 return err;
970 }
971 }
972
973 err = bpf_prog_attach(prog_fd[1], map_fd[0],
974 BPF_SK_SKB_STREAM_VERDICT, 0);
975 if (err) {
976 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
977 err, strerror(errno));
978 return err;
979 }
980
981 /* Attach programs to TLS sockmap */
982 if (txmsg_ktls_skb) {
983 if (!txmsg_omit_skb_parser) {
984 err = bpf_prog_attach(prog_fd[0], map_fd[8],
985 BPF_SK_SKB_STREAM_PARSER, 0);
986 if (err) {
987 fprintf(stderr,
988 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
989 prog_fd[0], map_fd[8], err, strerror(errno));
990 return err;
991 }
992 }
993
994 err = bpf_prog_attach(prog_fd[2], map_fd[8],
995 BPF_SK_SKB_STREAM_VERDICT, 0);
996 if (err) {
997 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
998 err, strerror(errno));
999 return err;
1000 }
1001 }
1002
1003 /* Attach to cgroups */
1004 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1005 if (err) {
1006 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1007 err, strerror(errno));
1008 return err;
1009 }
1010
1011run:
1012 err = sockmap_init_sockets(options->verbose);
1013 if (err) {
1014 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1015 goto out;
1016 }
1017
1018 /* Attach txmsg program to sockmap */
1019 if (txmsg_pass)
1020 tx_prog_fd = prog_fd[4];
1021 else if (txmsg_redir)
1022 tx_prog_fd = prog_fd[5];
1023 else if (txmsg_apply)
1024 tx_prog_fd = prog_fd[6];
1025 else if (txmsg_cork)
1026 tx_prog_fd = prog_fd[7];
1027 else if (txmsg_drop)
1028 tx_prog_fd = prog_fd[8];
1029 else
1030 tx_prog_fd = 0;
1031
1032 if (tx_prog_fd) {
1033 int redir_fd, i = 0;
1034
1035 err = bpf_prog_attach(tx_prog_fd,
1036 map_fd[1], BPF_SK_MSG_VERDICT, 0);
1037 if (err) {
1038 fprintf(stderr,
1039 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1040 err, strerror(errno));
1041 goto out;
1042 }
1043
1044 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1045 if (err) {
1046 fprintf(stderr,
1047 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1048 err, strerror(errno));
1049 goto out;
1050 }
1051
1052 if (txmsg_redir)
1053 redir_fd = c2;
1054 else
1055 redir_fd = c1;
1056
1057 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1058 if (err) {
1059 fprintf(stderr,
1060 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1061 err, strerror(errno));
1062 goto out;
1063 }
1064
1065 if (txmsg_apply) {
1066 err = bpf_map_update_elem(map_fd[3],
1067 &i, &txmsg_apply, BPF_ANY);
1068 if (err) {
1069 fprintf(stderr,
1070 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1071 err, strerror(errno));
1072 goto out;
1073 }
1074 }
1075
1076 if (txmsg_cork) {
1077 err = bpf_map_update_elem(map_fd[4],
1078 &i, &txmsg_cork, BPF_ANY);
1079 if (err) {
1080 fprintf(stderr,
1081 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1082 err, strerror(errno));
1083 goto out;
1084 }
1085 }
1086
1087 if (txmsg_start) {
1088 err = bpf_map_update_elem(map_fd[5],
1089 &i, &txmsg_start, BPF_ANY);
1090 if (err) {
1091 fprintf(stderr,
1092 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1093 err, strerror(errno));
1094 goto out;
1095 }
1096 }
1097
1098 if (txmsg_end) {
1099 i = 1;
1100 err = bpf_map_update_elem(map_fd[5],
1101 &i, &txmsg_end, BPF_ANY);
1102 if (err) {
1103 fprintf(stderr,
1104 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1105 err, strerror(errno));
1106 goto out;
1107 }
1108 }
1109
1110 if (txmsg_start_push) {
1111 i = 2;
1112 err = bpf_map_update_elem(map_fd[5],
1113 &i, &txmsg_start_push, BPF_ANY);
1114 if (err) {
1115 fprintf(stderr,
1116 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1117 err, strerror(errno));
1118 goto out;
1119 }
1120 }
1121
1122 if (txmsg_end_push) {
1123 i = 3;
1124 err = bpf_map_update_elem(map_fd[5],
1125 &i, &txmsg_end_push, BPF_ANY);
1126 if (err) {
1127 fprintf(stderr,
1128 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1129 txmsg_end_push, i, err, strerror(errno));
1130 goto out;
1131 }
1132 }
1133
1134 if (txmsg_start_pop) {
1135 i = 4;
1136 err = bpf_map_update_elem(map_fd[5],
1137 &i, &txmsg_start_pop, BPF_ANY);
1138 if (err) {
1139 fprintf(stderr,
1140 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1141 txmsg_start_pop, i, err, strerror(errno));
1142 goto out;
1143 }
1144 } else {
1145 i = 4;
1146 bpf_map_update_elem(map_fd[5],
1147 &i, &txmsg_start_pop, BPF_ANY);
1148 }
1149
1150 if (txmsg_pop) {
1151 i = 5;
1152 err = bpf_map_update_elem(map_fd[5],
1153 &i, &txmsg_pop, BPF_ANY);
1154 if (err) {
1155 fprintf(stderr,
1156 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1157 txmsg_pop, i, err, strerror(errno));
1158 goto out;
1159 }
1160 } else {
1161 i = 5;
1162 bpf_map_update_elem(map_fd[5],
1163 &i, &txmsg_pop, BPF_ANY);
1164
1165 }
1166
1167 if (txmsg_ingress) {
1168 int in = BPF_F_INGRESS;
1169
1170 i = 0;
1171 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1172 if (err) {
1173 fprintf(stderr,
1174 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1175 err, strerror(errno));
1176 }
1177 i = 1;
1178 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1179 if (err) {
1180 fprintf(stderr,
1181 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1182 err, strerror(errno));
1183 }
1184 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1185 if (err) {
1186 fprintf(stderr,
1187 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1188 err, strerror(errno));
1189 }
1190
1191 i = 2;
1192 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1193 if (err) {
1194 fprintf(stderr,
1195 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1196 err, strerror(errno));
1197 }
1198 }
1199
1200 if (txmsg_ktls_skb) {
1201 int ingress = BPF_F_INGRESS;
1202
1203 i = 0;
1204 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1205 if (err) {
1206 fprintf(stderr,
1207 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1208 err, strerror(errno));
1209 }
1210
1211 if (txmsg_ktls_skb_redir) {
1212 i = 1;
1213 err = bpf_map_update_elem(map_fd[7],
1214 &i, &ingress, BPF_ANY);
1215 if (err) {
1216 fprintf(stderr,
1217 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1218 err, strerror(errno));
1219 }
1220 }
1221
1222 if (txmsg_ktls_skb_drop) {
1223 i = 1;
1224 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1225 }
1226 }
1227
1228 if (txmsg_redir_skb) {
1229 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1230 p2 : p1;
1231 int ingress = BPF_F_INGRESS;
1232
1233 i = 0;
1234 err = bpf_map_update_elem(map_fd[7],
1235 &i, &ingress, BPF_ANY);
1236 if (err) {
1237 fprintf(stderr,
1238 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1239 err, strerror(errno));
1240 }
1241
1242 i = 3;
1243 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1244 if (err) {
1245 fprintf(stderr,
1246 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1247 err, strerror(errno));
1248 }
1249 }
1250 }
1251
1252 if (skb_use_parser) {
1253 i = 2;
1254 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1255 }
1256
1257 if (txmsg_drop)
1258 options->drop_expected = true;
1259
1260 if (test == PING_PONG)
1261 err = forever_ping_pong(options->rate, options);
1262 else if (test == SENDMSG) {
1263 options->base = false;
1264 options->sendpage = false;
1265 err = sendmsg_test(options);
1266 } else if (test == SENDPAGE) {
1267 options->base = false;
1268 options->sendpage = true;
1269 err = sendmsg_test(options);
1270 } else if (test == BASE) {
1271 options->base = true;
1272 options->sendpage = false;
1273 err = sendmsg_test(options);
1274 } else if (test == BASE_SENDPAGE) {
1275 options->base = true;
1276 options->sendpage = true;
1277 err = sendmsg_test(options);
1278 } else
1279 fprintf(stderr, "unknown test\n");
1280out:
1281 /* Detatch and zero all the maps */
1282 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1283 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1284 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1285 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1286 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1287
1288 if (tx_prog_fd >= 0)
1289 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1290
1291 for (i = 0; i < 8; i++) {
1292 key = next_key = 0;
1293 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1294 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1295 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1296 key = next_key;
1297 }
1298 }
1299
1300 close(s1);
1301 close(s2);
1302 close(p1);
1303 close(p2);
1304 close(c1);
1305 close(c2);
1306 return err;
1307}
1308
1309static char *test_to_str(int test)
1310{
1311 switch (test) {
1312 case SENDMSG:
1313 return "sendmsg";
1314 case SENDPAGE:
1315 return "sendpage";
1316 }
1317 return "unknown";
1318}
1319
1320static void append_str(char *dst, const char *src, size_t dst_cap)
1321{
1322 size_t avail = dst_cap - strlen(dst);
1323
1324 if (avail <= 1) /* just zero byte could be written */
1325 return;
1326
1327 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1328}
1329
1330#define OPTSTRING 60
1331static void test_options(char *options)
1332{
1333 char tstr[OPTSTRING];
1334
1335 memset(options, 0, OPTSTRING);
1336
1337 if (txmsg_pass)
1338 append_str(options, "pass,", OPTSTRING);
1339 if (txmsg_redir)
1340 append_str(options, "redir,", OPTSTRING);
1341 if (txmsg_drop)
1342 append_str(options, "drop,", OPTSTRING);
1343 if (txmsg_apply) {
1344 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1345 append_str(options, tstr, OPTSTRING);
1346 }
1347 if (txmsg_cork) {
1348 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1349 append_str(options, tstr, OPTSTRING);
1350 }
1351 if (txmsg_start) {
1352 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1353 append_str(options, tstr, OPTSTRING);
1354 }
1355 if (txmsg_end) {
1356 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1357 append_str(options, tstr, OPTSTRING);
1358 }
1359 if (txmsg_start_pop) {
1360 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1361 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1362 append_str(options, tstr, OPTSTRING);
1363 }
1364 if (txmsg_ingress)
1365 append_str(options, "ingress,", OPTSTRING);
1366 if (txmsg_redir_skb)
1367 append_str(options, "redir_skb,", OPTSTRING);
1368 if (txmsg_ktls_skb)
1369 append_str(options, "ktls_skb,", OPTSTRING);
1370 if (ktls)
1371 append_str(options, "ktls,", OPTSTRING);
1372 if (peek_flag)
1373 append_str(options, "peek,", OPTSTRING);
1374}
1375
1376static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1377{
1378 char *options = calloc(OPTSTRING, sizeof(char));
1379 int err;
1380
1381 if (test == SENDPAGE)
1382 opt->sendpage = true;
1383 else
1384 opt->sendpage = false;
1385
1386 if (txmsg_drop)
1387 opt->drop_expected = true;
1388 else
1389 opt->drop_expected = false;
1390
1391 test_options(options);
1392
1393 if (opt->verbose) {
1394 fprintf(stdout,
1395 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1396 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1397 test_to_str(test), options);
1398 fflush(stdout);
1399 }
1400 err = run_options(opt, cgrp, test);
1401 if (opt->verbose)
1402 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1403 test_cnt++;
1404 !err ? passed++ : failed++;
1405 free(options);
1406 return err;
1407}
1408
1409static void test_exec(int cgrp, struct sockmap_options *opt)
1410{
1411 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1412 int err;
1413
1414 if (type == 0) {
1415 test_start();
1416 err = __test_exec(cgrp, SENDMSG, opt);
1417 if (err)
1418 test_fail();
1419 } else {
1420 test_start();
1421 err = __test_exec(cgrp, SENDPAGE, opt);
1422 if (err)
1423 test_fail();
1424 }
1425}
1426
1427static void test_send_one(struct sockmap_options *opt, int cgrp)
1428{
1429 opt->iov_length = 1;
1430 opt->iov_count = 1;
1431 opt->rate = 1;
1432 test_exec(cgrp, opt);
1433
1434 opt->iov_length = 1;
1435 opt->iov_count = 1024;
1436 opt->rate = 1;
1437 test_exec(cgrp, opt);
1438
1439 opt->iov_length = 1024;
1440 opt->iov_count = 1;
1441 opt->rate = 1;
1442 test_exec(cgrp, opt);
1443
1444}
1445
1446static void test_send_many(struct sockmap_options *opt, int cgrp)
1447{
1448 opt->iov_length = 3;
1449 opt->iov_count = 1;
1450 opt->rate = 512;
1451 test_exec(cgrp, opt);
1452
1453 opt->rate = 100;
1454 opt->iov_count = 1;
1455 opt->iov_length = 5;
1456 test_exec(cgrp, opt);
1457}
1458
1459static void test_send_large(struct sockmap_options *opt, int cgrp)
1460{
1461 opt->iov_length = 256;
1462 opt->iov_count = 1024;
1463 opt->rate = 2;
1464 test_exec(cgrp, opt);
1465}
1466
1467static void test_send(struct sockmap_options *opt, int cgrp)
1468{
1469 test_send_one(opt, cgrp);
1470 test_send_many(opt, cgrp);
1471 test_send_large(opt, cgrp);
1472 sched_yield();
1473}
1474
1475static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1476{
1477 /* Test small and large iov_count values with pass/redir/apply/cork */
1478 txmsg_pass = 1;
1479 test_send(opt, cgrp);
1480}
1481
1482static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1483{
1484 txmsg_redir = 1;
1485 test_send(opt, cgrp);
1486}
1487
1488static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1489{
1490 txmsg_redir = 1;
1491 opt->tx_wait_mem = true;
1492 test_send_large(opt, cgrp);
1493 opt->tx_wait_mem = false;
1494}
1495
1496static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1497{
1498 txmsg_drop = 1;
1499 test_send(opt, cgrp);
1500}
1501
1502static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1503{
1504 txmsg_pass = txmsg_drop = 0;
1505 txmsg_ingress = txmsg_redir = 1;
1506 test_send(opt, cgrp);
1507}
1508
1509static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1510{
1511 bool data = opt->data_test;
1512 int k = ktls;
1513
1514 opt->data_test = true;
1515 ktls = 1;
1516
1517 txmsg_pass = txmsg_drop = 0;
1518 txmsg_ingress = txmsg_redir = 0;
1519 txmsg_ktls_skb = 1;
1520 txmsg_pass = 1;
1521
1522 /* Using data verification so ensure iov layout is
1523 * expected from test receiver side. e.g. has enough
1524 * bytes to write test code.
1525 */
1526 opt->iov_length = 100;
1527 opt->iov_count = 1;
1528 opt->rate = 1;
1529 test_exec(cgrp, opt);
1530
1531 txmsg_ktls_skb_drop = 1;
1532 test_exec(cgrp, opt);
1533
1534 txmsg_ktls_skb_drop = 0;
1535 txmsg_ktls_skb_redir = 1;
1536 test_exec(cgrp, opt);
1537 txmsg_ktls_skb_redir = 0;
1538
1539 /* Tests that omit skb_parser */
1540 txmsg_omit_skb_parser = 1;
1541 ktls = 0;
1542 txmsg_ktls_skb = 0;
1543 test_exec(cgrp, opt);
1544
1545 txmsg_ktls_skb_drop = 1;
1546 test_exec(cgrp, opt);
1547 txmsg_ktls_skb_drop = 0;
1548
1549 txmsg_ktls_skb_redir = 1;
1550 test_exec(cgrp, opt);
1551
1552 ktls = 1;
1553 test_exec(cgrp, opt);
1554 txmsg_omit_skb_parser = 0;
1555
1556 opt->data_test = data;
1557 ktls = k;
1558}
1559
1560/* Test cork with hung data. This tests poor usage patterns where
1561 * cork can leave data on the ring if user program is buggy and
1562 * doesn't flush them somehow. They do take some time however
1563 * because they wait for a timeout. Test pass, redir and cork with
1564 * apply logic. Use cork size of 4097 with send_large to avoid
1565 * aligning cork size with send size.
1566 */
1567static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1568{
1569 txmsg_pass = 1;
1570 txmsg_redir = 0;
1571 txmsg_cork = 4097;
1572 txmsg_apply = 4097;
1573 test_send_large(opt, cgrp);
1574
1575 txmsg_pass = 0;
1576 txmsg_redir = 1;
1577 txmsg_apply = 0;
1578 txmsg_cork = 4097;
1579 test_send_large(opt, cgrp);
1580
1581 txmsg_pass = 0;
1582 txmsg_redir = 1;
1583 txmsg_apply = 4097;
1584 txmsg_cork = 4097;
1585 test_send_large(opt, cgrp);
1586}
1587
1588static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1589{
1590 /* Test basic start/end */
1591 txmsg_start = 1;
1592 txmsg_end = 2;
1593 test_send(opt, cgrp);
1594
1595 /* Test >4k pull */
1596 txmsg_start = 4096;
1597 txmsg_end = 9182;
1598 test_send_large(opt, cgrp);
1599
1600 /* Test pull + redirect */
1601 txmsg_redir = 0;
1602 txmsg_start = 1;
1603 txmsg_end = 2;
1604 test_send(opt, cgrp);
1605
1606 /* Test pull + cork */
1607 txmsg_redir = 0;
1608 txmsg_cork = 512;
1609 txmsg_start = 1;
1610 txmsg_end = 2;
1611 test_send_many(opt, cgrp);
1612
1613 /* Test pull + cork + redirect */
1614 txmsg_redir = 1;
1615 txmsg_cork = 512;
1616 txmsg_start = 1;
1617 txmsg_end = 2;
1618 test_send_many(opt, cgrp);
1619}
1620
1621static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1622{
1623 /* Test basic pop */
1624 txmsg_start_pop = 1;
1625 txmsg_pop = 2;
1626 test_send_many(opt, cgrp);
1627
1628 /* Test pop with >4k */
1629 txmsg_start_pop = 4096;
1630 txmsg_pop = 4096;
1631 test_send_large(opt, cgrp);
1632
1633 /* Test pop + redirect */
1634 txmsg_redir = 1;
1635 txmsg_start_pop = 1;
1636 txmsg_pop = 2;
1637 test_send_many(opt, cgrp);
1638
1639 /* Test pop + cork */
1640 txmsg_redir = 0;
1641 txmsg_cork = 512;
1642 txmsg_start_pop = 1;
1643 txmsg_pop = 2;
1644 test_send_many(opt, cgrp);
1645
1646 /* Test pop + redirect + cork */
1647 txmsg_redir = 1;
1648 txmsg_cork = 4;
1649 txmsg_start_pop = 1;
1650 txmsg_pop = 2;
1651 test_send_many(opt, cgrp);
1652}
1653
1654static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1655{
1656 /* Test basic push */
1657 txmsg_start_push = 1;
1658 txmsg_end_push = 1;
1659 test_send(opt, cgrp);
1660
1661 /* Test push 4kB >4k */
1662 txmsg_start_push = 4096;
1663 txmsg_end_push = 4096;
1664 test_send_large(opt, cgrp);
1665
1666 /* Test push + redirect */
1667 txmsg_redir = 1;
1668 txmsg_start_push = 1;
1669 txmsg_end_push = 2;
1670 test_send_many(opt, cgrp);
1671
1672 /* Test push + cork */
1673 txmsg_redir = 0;
1674 txmsg_cork = 512;
1675 txmsg_start_push = 1;
1676 txmsg_end_push = 2;
1677 test_send_many(opt, cgrp);
1678}
1679
1680static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1681{
1682 txmsg_start_push = 1;
1683 txmsg_end_push = 10;
1684 txmsg_start_pop = 5;
1685 txmsg_pop = 4;
1686 test_send_large(opt, cgrp);
1687}
1688
1689static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1690{
1691 txmsg_pass = 1;
1692 txmsg_redir = 0;
1693 txmsg_ingress = 0;
1694 txmsg_apply = 1;
1695 txmsg_cork = 0;
1696 test_send_one(opt, cgrp);
1697
1698 txmsg_pass = 0;
1699 txmsg_redir = 1;
1700 txmsg_ingress = 0;
1701 txmsg_apply = 1;
1702 txmsg_cork = 0;
1703 test_send_one(opt, cgrp);
1704
1705 txmsg_pass = 0;
1706 txmsg_redir = 1;
1707 txmsg_ingress = 1;
1708 txmsg_apply = 1;
1709 txmsg_cork = 0;
1710 test_send_one(opt, cgrp);
1711
1712 txmsg_pass = 1;
1713 txmsg_redir = 0;
1714 txmsg_ingress = 0;
1715 txmsg_apply = 1024;
1716 txmsg_cork = 0;
1717 test_send_large(opt, cgrp);
1718
1719 txmsg_pass = 0;
1720 txmsg_redir = 1;
1721 txmsg_ingress = 0;
1722 txmsg_apply = 1024;
1723 txmsg_cork = 0;
1724 test_send_large(opt, cgrp);
1725
1726 txmsg_pass = 0;
1727 txmsg_redir = 1;
1728 txmsg_ingress = 1;
1729 txmsg_apply = 1024;
1730 txmsg_cork = 0;
1731 test_send_large(opt, cgrp);
1732}
1733
1734static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1735{
1736 txmsg_pass = 1;
1737 txmsg_redir = 0;
1738 txmsg_apply = 0;
1739 txmsg_cork = 1;
1740 test_send(opt, cgrp);
1741
1742 txmsg_pass = 1;
1743 txmsg_redir = 0;
1744 txmsg_apply = 1;
1745 txmsg_cork = 1;
1746 test_send(opt, cgrp);
1747}
1748
1749static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1750{
1751 txmsg_pass = 1;
1752 skb_use_parser = 512;
1753 if (ktls == 1)
1754 skb_use_parser = 570;
1755 opt->iov_length = 256;
1756 opt->iov_count = 1;
1757 opt->rate = 2;
1758 test_exec(cgrp, opt);
1759}
1760
1761static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1762{
1763 if (ktls == 1)
1764 return;
1765 skb_use_parser = 10;
1766 opt->iov_length = 20;
1767 opt->iov_count = 1;
1768 opt->rate = 1;
1769 opt->check_recved_len = true;
1770 test_exec(cgrp, opt);
1771 opt->check_recved_len = false;
1772}
1773
1774char *map_names[] = {
1775 "sock_map",
1776 "sock_map_txmsg",
1777 "sock_map_redir",
1778 "sock_apply_bytes",
1779 "sock_cork_bytes",
1780 "sock_bytes",
1781 "sock_redir_flags",
1782 "sock_skb_opts",
1783 "tls_sock_map",
1784};
1785
1786int prog_attach_type[] = {
1787 BPF_SK_SKB_STREAM_PARSER,
1788 BPF_SK_SKB_STREAM_VERDICT,
1789 BPF_SK_SKB_STREAM_VERDICT,
1790 BPF_CGROUP_SOCK_OPS,
1791 BPF_SK_MSG_VERDICT,
1792 BPF_SK_MSG_VERDICT,
1793 BPF_SK_MSG_VERDICT,
1794 BPF_SK_MSG_VERDICT,
1795 BPF_SK_MSG_VERDICT,
1796 BPF_SK_MSG_VERDICT,
1797 BPF_SK_MSG_VERDICT,
1798};
1799
1800int prog_type[] = {
1801 BPF_PROG_TYPE_SK_SKB,
1802 BPF_PROG_TYPE_SK_SKB,
1803 BPF_PROG_TYPE_SK_SKB,
1804 BPF_PROG_TYPE_SOCK_OPS,
1805 BPF_PROG_TYPE_SK_MSG,
1806 BPF_PROG_TYPE_SK_MSG,
1807 BPF_PROG_TYPE_SK_MSG,
1808 BPF_PROG_TYPE_SK_MSG,
1809 BPF_PROG_TYPE_SK_MSG,
1810 BPF_PROG_TYPE_SK_MSG,
1811 BPF_PROG_TYPE_SK_MSG,
1812};
1813
1814static int populate_progs(char *bpf_file)
1815{
1816 struct bpf_program *prog;
1817 struct bpf_object *obj;
1818 int i = 0;
1819 long err;
1820
1821 obj = bpf_object__open(bpf_file);
1822 err = libbpf_get_error(obj);
1823 if (err) {
1824 char err_buf[256];
1825
1826 libbpf_strerror(err, err_buf, sizeof(err_buf));
1827 printf("Unable to load eBPF objects in file '%s' : %s\n",
1828 bpf_file, err_buf);
1829 return -1;
1830 }
1831
1832 bpf_object__for_each_program(prog, obj) {
1833 bpf_program__set_type(prog, prog_type[i]);
1834 bpf_program__set_expected_attach_type(prog,
1835 prog_attach_type[i]);
1836 i++;
1837 }
1838
1839 i = bpf_object__load(obj);
1840 i = 0;
1841 bpf_object__for_each_program(prog, obj) {
1842 prog_fd[i] = bpf_program__fd(prog);
1843 i++;
1844 }
1845
1846 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1847 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1848 map_fd[i] = bpf_map__fd(maps[i]);
1849 if (map_fd[i] < 0) {
1850 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1851 map_fd[i], strerror(errno));
1852 return -1;
1853 }
1854 }
1855
1856 return 0;
1857}
1858
1859struct _test test[] = {
1860 {"txmsg test passthrough", test_txmsg_pass},
1861 {"txmsg test redirect", test_txmsg_redir},
1862 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1863 {"txmsg test drop", test_txmsg_drop},
1864 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1865 {"txmsg test skb", test_txmsg_skb},
1866 {"txmsg test apply", test_txmsg_apply},
1867 {"txmsg test cork", test_txmsg_cork},
1868 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1869 {"txmsg test push_data", test_txmsg_push},
1870 {"txmsg test pull-data", test_txmsg_pull},
1871 {"txmsg test pop-data", test_txmsg_pop},
1872 {"txmsg test push/pop data", test_txmsg_push_pop},
1873 {"txmsg test ingress parser", test_txmsg_ingress_parser},
1874 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1875};
1876
1877static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1878{
1879 char *entry, *ptr;
1880
1881 if (!opt->whitelist)
1882 return 0;
1883 ptr = strdup(opt->whitelist);
1884 if (!ptr)
1885 return -ENOMEM;
1886 entry = strtok(ptr, ",");
1887 while (entry) {
1888 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1889 strstr(opt->map, entry) != 0 ||
1890 strstr(t->title, entry) != 0)
1891 return 0;
1892 entry = strtok(NULL, ",");
1893 }
1894 return -EINVAL;
1895}
1896
1897static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1898{
1899 char *entry, *ptr;
1900
1901 if (!opt->blacklist)
1902 return -EINVAL;
1903 ptr = strdup(opt->blacklist);
1904 if (!ptr)
1905 return -ENOMEM;
1906 entry = strtok(ptr, ",");
1907 while (entry) {
1908 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1909 strstr(opt->map, entry) != 0 ||
1910 strstr(t->title, entry) != 0)
1911 return 0;
1912 entry = strtok(NULL, ",");
1913 }
1914 return -EINVAL;
1915}
1916
1917static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1918{
1919 int i, err;
1920
1921 err = populate_progs(opt->map);
1922 if (err < 0) {
1923 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1924 return err;
1925 }
1926
1927 /* Tests basic commands and APIs */
1928 for (i = 0; i < ARRAY_SIZE(test); i++) {
1929 struct _test t = test[i];
1930
1931 if (check_whitelist(&t, opt) != 0)
1932 continue;
1933 if (check_blacklist(&t, opt) == 0)
1934 continue;
1935
1936 test_start_subtest(&t, opt);
1937 t.tester(cg_fd, opt);
1938 test_end_subtest();
1939 }
1940
1941 return err;
1942}
1943
1944static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1945{
1946 opt->map = BPF_SOCKMAP_FILENAME;
1947 __test_selftests(cg_fd, opt);
1948}
1949
1950static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1951{
1952 opt->map = BPF_SOCKHASH_FILENAME;
1953 __test_selftests(cg_fd, opt);
1954}
1955
1956static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1957{
1958 opt->map = BPF_SOCKHASH_FILENAME;
1959 opt->prepend = "ktls";
1960 ktls = 1;
1961 __test_selftests(cg_fd, opt);
1962 ktls = 0;
1963}
1964
1965static int test_selftest(int cg_fd, struct sockmap_options *opt)
1966{
1967
1968 test_selftests_sockmap(cg_fd, opt);
1969 test_selftests_sockhash(cg_fd, opt);
1970 test_selftests_ktls(cg_fd, opt);
1971 test_print_results();
1972 return 0;
1973}
1974
1975int main(int argc, char **argv)
1976{
1977 int iov_count = 1, length = 1024, rate = 1;
1978 struct sockmap_options options = {0};
1979 int opt, longindex, err, cg_fd = 0;
1980 char *bpf_file = BPF_SOCKMAP_FILENAME;
1981 int test = SELFTESTS;
1982 bool cg_created = 0;
1983
1984 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1985 long_options, &longindex)) != -1) {
1986 switch (opt) {
1987 case 's':
1988 txmsg_start = atoi(optarg);
1989 break;
1990 case 'e':
1991 txmsg_end = atoi(optarg);
1992 break;
1993 case 'p':
1994 txmsg_start_push = atoi(optarg);
1995 break;
1996 case 'q':
1997 txmsg_end_push = atoi(optarg);
1998 break;
1999 case 'w':
2000 txmsg_start_pop = atoi(optarg);
2001 break;
2002 case 'x':
2003 txmsg_pop = atoi(optarg);
2004 break;
2005 case 'a':
2006 txmsg_apply = atoi(optarg);
2007 break;
2008 case 'k':
2009 txmsg_cork = atoi(optarg);
2010 break;
2011 case 'c':
2012 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
2013 if (cg_fd < 0) {
2014 fprintf(stderr,
2015 "ERROR: (%i) open cg path failed: %s\n",
2016 cg_fd, optarg);
2017 return cg_fd;
2018 }
2019 break;
2020 case 'r':
2021 rate = atoi(optarg);
2022 break;
2023 case 'v':
2024 options.verbose = 1;
2025 if (optarg)
2026 options.verbose = atoi(optarg);
2027 break;
2028 case 'i':
2029 iov_count = atoi(optarg);
2030 break;
2031 case 'l':
2032 length = atoi(optarg);
2033 break;
2034 case 'd':
2035 options.data_test = true;
2036 break;
2037 case 't':
2038 if (strcmp(optarg, "ping") == 0) {
2039 test = PING_PONG;
2040 } else if (strcmp(optarg, "sendmsg") == 0) {
2041 test = SENDMSG;
2042 } else if (strcmp(optarg, "base") == 0) {
2043 test = BASE;
2044 } else if (strcmp(optarg, "base_sendpage") == 0) {
2045 test = BASE_SENDPAGE;
2046 } else if (strcmp(optarg, "sendpage") == 0) {
2047 test = SENDPAGE;
2048 } else {
2049 usage(argv);
2050 return -1;
2051 }
2052 break;
2053 case 'n':
2054 options.whitelist = strdup(optarg);
2055 if (!options.whitelist)
2056 return -ENOMEM;
2057 break;
2058 case 'b':
2059 options.blacklist = strdup(optarg);
2060 if (!options.blacklist)
2061 return -ENOMEM;
2062 case 0:
2063 break;
2064 case 'h':
2065 default:
2066 usage(argv);
2067 return -1;
2068 }
2069 }
2070
2071 if (!cg_fd) {
2072 cg_fd = cgroup_setup_and_join(CG_PATH);
2073 if (cg_fd < 0)
2074 return cg_fd;
2075 cg_created = 1;
2076 }
2077
2078 /* Use libbpf 1.0 API mode */
2079 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2080
2081 if (test == SELFTESTS) {
2082 err = test_selftest(cg_fd, &options);
2083 goto out;
2084 }
2085
2086 err = populate_progs(bpf_file);
2087 if (err) {
2088 fprintf(stderr, "populate program: (%s) %s\n",
2089 bpf_file, strerror(errno));
2090 return 1;
2091 }
2092 running = 1;
2093
2094 /* catch SIGINT */
2095 signal(SIGINT, running_handler);
2096
2097 options.iov_count = iov_count;
2098 options.iov_length = length;
2099 options.rate = rate;
2100
2101 err = run_options(&options, cg_fd, test);
2102out:
2103 if (options.whitelist)
2104 free(options.whitelist);
2105 if (options.blacklist)
2106 free(options.blacklist);
2107 if (cg_created)
2108 cleanup_cgroup_environment();
2109 close(cg_fd);
2110 return err;
2111}
2112
2113void running_handler(int a)
2114{
2115 running = 0;
2116}