Loading...
Note: File does not exist in v4.10.11.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2020 Intel Corporation. */
3
4/*
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
7 * for use.
8 *
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
11 *
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
15 *
16 * Tests Information:
17 * ------------------
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
20 *
21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall
24 * c. Socket Teardown
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
30 * mode is used
31 * e. Statistics
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
34 * i. rx dropped
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
37 * ii. tx invalid
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
39 * size + 1).
40 * iii. rx ring full
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
48 * g. unaligned mode
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames
53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
54 * using unaligned mode
55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to
56 * check if they pass the validation or not
57 *
58 * Flow:
59 * -----
60 * - Single process spawns two threads: Tx and Rx
61 * - Each of these two threads attach to a veth interface
62 * - Each thread creates one AF_XDP socket connected to a unique umem for each
63 * veth interface
64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
65 * - Rx thread verifies if all packets were received and delivered in-order,
66 * and have the right content
67 *
68 * Enable/disable packet dump mode:
69 * --------------------------
70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
72 */
73
74#define _GNU_SOURCE
75#include <assert.h>
76#include <fcntl.h>
77#include <errno.h>
78#include <getopt.h>
79#include <linux/if_link.h>
80#include <linux/if_ether.h>
81#include <linux/mman.h>
82#include <linux/netdev.h>
83#include <linux/bitmap.h>
84#include <arpa/inet.h>
85#include <net/if.h>
86#include <locale.h>
87#include <poll.h>
88#include <pthread.h>
89#include <signal.h>
90#include <stdio.h>
91#include <stdlib.h>
92#include <string.h>
93#include <stddef.h>
94#include <sys/mman.h>
95#include <sys/socket.h>
96#include <sys/time.h>
97#include <sys/types.h>
98#include <unistd.h>
99
100#include "xsk_xdp_progs.skel.h"
101#include "xsk.h"
102#include "xskxceiver.h"
103#include <bpf/bpf.h>
104#include <linux/filter.h>
105#include "../kselftest.h"
106#include "xsk_xdp_common.h"
107
108static bool opt_verbose;
109static bool opt_print_tests;
110static enum test_mode opt_mode = TEST_MODE_ALL;
111static u32 opt_run_test = RUN_ALL_TESTS;
112
113static void __exit_with_error(int error, const char *file, const char *func, int line)
114{
115 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
116 strerror(error));
117 ksft_exit_xfail();
118}
119
120#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
121#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
122static char *mode_string(struct test_spec *test)
123{
124 switch (test->mode) {
125 case TEST_MODE_SKB:
126 return "SKB";
127 case TEST_MODE_DRV:
128 return "DRV";
129 case TEST_MODE_ZC:
130 return "ZC";
131 default:
132 return "BOGUS";
133 }
134}
135
136static void report_failure(struct test_spec *test)
137{
138 if (test->fail)
139 return;
140
141 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
142 test->name);
143 test->fail = true;
144}
145
146/* The payload is a word consisting of a packet sequence number in the upper
147 * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's
148 * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0.
149 */
150static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size)
151{
152 u32 *ptr = (u32 *)dest, i;
153
154 start /= sizeof(*ptr);
155 size /= sizeof(*ptr);
156 for (i = 0; i < size; i++)
157 ptr[i] = htonl(pkt_nb << 16 | (i + start));
158}
159
160static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr)
161{
162 memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN);
163 memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN);
164 eth_hdr->h_proto = htons(ETH_P_LOOPBACK);
165}
166
167static bool is_umem_valid(struct ifobject *ifobj)
168{
169 return !!ifobj->umem->umem;
170}
171
172static u32 mode_to_xdp_flags(enum test_mode mode)
173{
174 return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
175}
176
177static u64 umem_size(struct xsk_umem_info *umem)
178{
179 return umem->num_frames * umem->frame_size;
180}
181
182static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,
183 u64 size)
184{
185 struct xsk_umem_config cfg = {
186 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
187 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
188 .frame_size = umem->frame_size,
189 .frame_headroom = umem->frame_headroom,
190 .flags = XSK_UMEM__DEFAULT_FLAGS
191 };
192 int ret;
193
194 if (umem->unaligned_mode)
195 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
196
197 ret = xsk_umem__create(&umem->umem, buffer, size,
198 &umem->fq, &umem->cq, &cfg);
199 if (ret)
200 return ret;
201
202 umem->buffer = buffer;
203 if (ifobj->shared_umem && ifobj->rx_on) {
204 umem->base_addr = umem_size(umem);
205 umem->next_buffer = umem_size(umem);
206 }
207
208 return 0;
209}
210
211static u64 umem_alloc_buffer(struct xsk_umem_info *umem)
212{
213 u64 addr;
214
215 addr = umem->next_buffer;
216 umem->next_buffer += umem->frame_size;
217 if (umem->next_buffer >= umem->base_addr + umem_size(umem))
218 umem->next_buffer = umem->base_addr;
219
220 return addr;
221}
222
223static void umem_reset_alloc(struct xsk_umem_info *umem)
224{
225 umem->next_buffer = 0;
226}
227
228static void enable_busy_poll(struct xsk_socket_info *xsk)
229{
230 int sock_opt;
231
232 sock_opt = 1;
233 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
234 (void *)&sock_opt, sizeof(sock_opt)) < 0)
235 exit_with_error(errno);
236
237 sock_opt = 20;
238 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
239 (void *)&sock_opt, sizeof(sock_opt)) < 0)
240 exit_with_error(errno);
241
242 sock_opt = BATCH_SIZE;
243 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
244 (void *)&sock_opt, sizeof(sock_opt)) < 0)
245 exit_with_error(errno);
246}
247
248static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
249 struct ifobject *ifobject, bool shared)
250{
251 struct xsk_socket_config cfg = {};
252 struct xsk_ring_cons *rxr;
253 struct xsk_ring_prod *txr;
254
255 xsk->umem = umem;
256 cfg.rx_size = xsk->rxqsize;
257 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
258 cfg.bind_flags = ifobject->bind_flags;
259 if (shared)
260 cfg.bind_flags |= XDP_SHARED_UMEM;
261 if (ifobject->mtu > MAX_ETH_PKT_SIZE)
262 cfg.bind_flags |= XDP_USE_SG;
263
264 txr = ifobject->tx_on ? &xsk->tx : NULL;
265 rxr = ifobject->rx_on ? &xsk->rx : NULL;
266 return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);
267}
268
269static bool ifobj_zc_avail(struct ifobject *ifobject)
270{
271 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
272 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
273 struct xsk_socket_info *xsk;
274 struct xsk_umem_info *umem;
275 bool zc_avail = false;
276 void *bufs;
277 int ret;
278
279 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
280 if (bufs == MAP_FAILED)
281 exit_with_error(errno);
282
283 umem = calloc(1, sizeof(struct xsk_umem_info));
284 if (!umem) {
285 munmap(bufs, umem_sz);
286 exit_with_error(ENOMEM);
287 }
288 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
289 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
290 if (ret)
291 exit_with_error(-ret);
292
293 xsk = calloc(1, sizeof(struct xsk_socket_info));
294 if (!xsk)
295 goto out;
296 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
297 ifobject->rx_on = true;
298 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
299 ret = __xsk_configure_socket(xsk, umem, ifobject, false);
300 if (!ret)
301 zc_avail = true;
302
303 xsk_socket__delete(xsk->xsk);
304 free(xsk);
305out:
306 munmap(umem->buffer, umem_sz);
307 xsk_umem__delete(umem->umem);
308 free(umem);
309 return zc_avail;
310}
311
312static struct option long_options[] = {
313 {"interface", required_argument, 0, 'i'},
314 {"busy-poll", no_argument, 0, 'b'},
315 {"verbose", no_argument, 0, 'v'},
316 {"mode", required_argument, 0, 'm'},
317 {"list", no_argument, 0, 'l'},
318 {"test", required_argument, 0, 't'},
319 {"help", no_argument, 0, 'h'},
320 {0, 0, 0, 0}
321};
322
323static void print_usage(char **argv)
324{
325 const char *str =
326 " Usage: xskxceiver [OPTIONS]\n"
327 " Options:\n"
328 " -i, --interface Use interface\n"
329 " -v, --verbose Verbose output\n"
330 " -b, --busy-poll Enable busy poll\n"
331 " -m, --mode Run only mode skb, drv, or zc\n"
332 " -l, --list List all available tests\n"
333 " -t, --test Run a specific test. Enter number from -l option.\n"
334 " -h, --help Display this help and exit\n";
335
336 ksft_print_msg(str, basename(argv[0]));
337 ksft_exit_xfail();
338}
339
340static bool validate_interface(struct ifobject *ifobj)
341{
342 if (!strcmp(ifobj->ifname, ""))
343 return false;
344 return true;
345}
346
347static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
348 char **argv)
349{
350 struct ifobject *ifobj;
351 u32 interface_nb = 0;
352 int option_index, c;
353
354 opterr = 0;
355
356 for (;;) {
357 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);
358 if (c == -1)
359 break;
360
361 switch (c) {
362 case 'i':
363 if (interface_nb == 0)
364 ifobj = ifobj_tx;
365 else if (interface_nb == 1)
366 ifobj = ifobj_rx;
367 else
368 break;
369
370 memcpy(ifobj->ifname, optarg,
371 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
372
373 ifobj->ifindex = if_nametoindex(ifobj->ifname);
374 if (!ifobj->ifindex)
375 exit_with_error(errno);
376
377 interface_nb++;
378 break;
379 case 'v':
380 opt_verbose = true;
381 break;
382 case 'b':
383 ifobj_tx->busy_poll = true;
384 ifobj_rx->busy_poll = true;
385 break;
386 case 'm':
387 if (!strncmp("skb", optarg, strlen(optarg)))
388 opt_mode = TEST_MODE_SKB;
389 else if (!strncmp("drv", optarg, strlen(optarg)))
390 opt_mode = TEST_MODE_DRV;
391 else if (!strncmp("zc", optarg, strlen(optarg)))
392 opt_mode = TEST_MODE_ZC;
393 else
394 print_usage(argv);
395 break;
396 case 'l':
397 opt_print_tests = true;
398 break;
399 case 't':
400 errno = 0;
401 opt_run_test = strtol(optarg, NULL, 0);
402 if (errno)
403 print_usage(argv);
404 break;
405 case 'h':
406 default:
407 print_usage(argv);
408 }
409 }
410}
411
412static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
413 struct ifobject *ifobj_rx)
414{
415 u32 i, j;
416
417 for (i = 0; i < MAX_INTERFACES; i++) {
418 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
419
420 ifobj->xsk = &ifobj->xsk_arr[0];
421 ifobj->use_poll = false;
422 ifobj->use_fill_ring = true;
423 ifobj->release_rx = true;
424 ifobj->validation_func = NULL;
425 ifobj->use_metadata = false;
426
427 if (i == 0) {
428 ifobj->rx_on = false;
429 ifobj->tx_on = true;
430 } else {
431 ifobj->rx_on = true;
432 ifobj->tx_on = false;
433 }
434
435 memset(ifobj->umem, 0, sizeof(*ifobj->umem));
436 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
437 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
438
439 for (j = 0; j < MAX_SOCKETS; j++) {
440 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
441 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
442 if (i == 0)
443 ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default;
444 else
445 ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default;
446
447 memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN);
448 memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN);
449 ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0);
450 ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1);
451 }
452 }
453
454 test->ifobj_tx = ifobj_tx;
455 test->ifobj_rx = ifobj_rx;
456 test->current_step = 0;
457 test->total_steps = 1;
458 test->nb_sockets = 1;
459 test->fail = false;
460 test->mtu = MAX_ETH_PKT_SIZE;
461 test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog;
462 test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk;
463 test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;
464 test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;
465}
466
467static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
468 struct ifobject *ifobj_rx, enum test_mode mode,
469 const struct test_spec *test_to_run)
470{
471 struct pkt_stream *tx_pkt_stream;
472 struct pkt_stream *rx_pkt_stream;
473 u32 i;
474
475 tx_pkt_stream = test->tx_pkt_stream_default;
476 rx_pkt_stream = test->rx_pkt_stream_default;
477 memset(test, 0, sizeof(*test));
478 test->tx_pkt_stream_default = tx_pkt_stream;
479 test->rx_pkt_stream_default = rx_pkt_stream;
480
481 for (i = 0; i < MAX_INTERFACES; i++) {
482 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
483
484 ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
485 if (mode == TEST_MODE_ZC)
486 ifobj->bind_flags |= XDP_ZEROCOPY;
487 else
488 ifobj->bind_flags |= XDP_COPY;
489 }
490
491 strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE);
492 test->test_func = test_to_run->test_func;
493 test->mode = mode;
494 __test_spec_init(test, ifobj_tx, ifobj_rx);
495}
496
497static void test_spec_reset(struct test_spec *test)
498{
499 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
500}
501
502static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx,
503 struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx,
504 struct bpf_map *xskmap_tx)
505{
506 test->xdp_prog_rx = xdp_prog_rx;
507 test->xdp_prog_tx = xdp_prog_tx;
508 test->xskmap_rx = xskmap_rx;
509 test->xskmap_tx = xskmap_tx;
510}
511
512static int test_spec_set_mtu(struct test_spec *test, int mtu)
513{
514 int err;
515
516 if (test->ifobj_rx->mtu != mtu) {
517 err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu);
518 if (err)
519 return err;
520 test->ifobj_rx->mtu = mtu;
521 }
522 if (test->ifobj_tx->mtu != mtu) {
523 err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu);
524 if (err)
525 return err;
526 test->ifobj_tx->mtu = mtu;
527 }
528
529 return 0;
530}
531
532static void pkt_stream_reset(struct pkt_stream *pkt_stream)
533{
534 if (pkt_stream) {
535 pkt_stream->current_pkt_nb = 0;
536 pkt_stream->nb_rx_pkts = 0;
537 }
538}
539
540static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream)
541{
542 if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts)
543 return NULL;
544
545 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
546}
547
548static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
549{
550 while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {
551 (*pkts_sent)++;
552 if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid)
553 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
554 pkt_stream->current_pkt_nb++;
555 }
556 return NULL;
557}
558
559static void pkt_stream_delete(struct pkt_stream *pkt_stream)
560{
561 free(pkt_stream->pkts);
562 free(pkt_stream);
563}
564
565static void pkt_stream_restore_default(struct test_spec *test)
566{
567 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream;
568 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream;
569
570 if (tx_pkt_stream != test->tx_pkt_stream_default) {
571 pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream);
572 test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default;
573 }
574
575 if (rx_pkt_stream != test->rx_pkt_stream_default) {
576 pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream);
577 test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default;
578 }
579}
580
581static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
582{
583 struct pkt_stream *pkt_stream;
584
585 pkt_stream = calloc(1, sizeof(*pkt_stream));
586 if (!pkt_stream)
587 return NULL;
588
589 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
590 if (!pkt_stream->pkts) {
591 free(pkt_stream);
592 return NULL;
593 }
594
595 pkt_stream->nb_pkts = nb_pkts;
596 return pkt_stream;
597}
598
599static bool pkt_continues(u32 options)
600{
601 return options & XDP_PKT_CONTD;
602}
603
604static u32 ceil_u32(u32 a, u32 b)
605{
606 return (a + b - 1) / b;
607}
608
609static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt)
610{
611 u32 nb_frags = 1, next_frag;
612
613 if (!pkt)
614 return 1;
615
616 if (!pkt_stream->verbatim) {
617 if (!pkt->valid || !pkt->len)
618 return 1;
619 return ceil_u32(pkt->len, frame_size);
620 }
621
622 /* Search for the end of the packet in verbatim mode */
623 if (!pkt_continues(pkt->options))
624 return nb_frags;
625
626 next_frag = pkt_stream->current_pkt_nb;
627 pkt++;
628 while (next_frag++ < pkt_stream->nb_pkts) {
629 nb_frags++;
630 if (!pkt_continues(pkt->options) || !pkt->valid)
631 break;
632 pkt++;
633 }
634 return nb_frags;
635}
636
637static bool set_pkt_valid(int offset, u32 len)
638{
639 return len <= MAX_ETH_JUMBO_SIZE;
640}
641
642static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
643{
644 pkt->offset = offset;
645 pkt->len = len;
646 pkt->valid = set_pkt_valid(offset, len);
647}
648
649static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
650{
651 bool prev_pkt_valid = pkt->valid;
652
653 pkt_set(pkt_stream, pkt, offset, len);
654 pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid;
655}
656
657static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)
658{
659 return ceil_u32(len, umem->frame_size) * umem->frame_size;
660}
661
662static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off)
663{
664 struct pkt_stream *pkt_stream;
665 u32 i;
666
667 pkt_stream = __pkt_stream_alloc(nb_pkts);
668 if (!pkt_stream)
669 exit_with_error(ENOMEM);
670
671 pkt_stream->nb_pkts = nb_pkts;
672 pkt_stream->max_pkt_len = pkt_len;
673 for (i = 0; i < nb_pkts; i++) {
674 struct pkt *pkt = &pkt_stream->pkts[i];
675
676 pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len);
677 pkt->pkt_nb = nb_start + i * nb_off;
678 }
679
680 return pkt_stream;
681}
682
683static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len)
684{
685 return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1);
686}
687
688static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream)
689{
690 return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
691}
692
693static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
694{
695 struct pkt_stream *pkt_stream;
696
697 pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
698 test->ifobj_tx->xsk->pkt_stream = pkt_stream;
699 pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
700 test->ifobj_rx->xsk->pkt_stream = pkt_stream;
701}
702
703static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
704 int offset)
705{
706 struct pkt_stream *pkt_stream;
707 u32 i;
708
709 pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream);
710 for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2)
711 pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len);
712
713 ifobj->xsk->pkt_stream = pkt_stream;
714}
715
716static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
717{
718 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
719 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
720}
721
722static void pkt_stream_receive_half(struct test_spec *test)
723{
724 struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream;
725 u32 i;
726
727 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts,
728 pkt_stream->pkts[0].len);
729 pkt_stream = test->ifobj_rx->xsk->pkt_stream;
730 for (i = 1; i < pkt_stream->nb_pkts; i += 2)
731 pkt_stream->pkts[i].valid = false;
732
733 pkt_stream->nb_valid_entries /= 2;
734}
735
736static void pkt_stream_even_odd_sequence(struct test_spec *test)
737{
738 struct pkt_stream *pkt_stream;
739 u32 i;
740
741 for (i = 0; i < test->nb_sockets; i++) {
742 pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream;
743 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
744 pkt_stream->pkts[0].len, i, 2);
745 test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream;
746
747 pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream;
748 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
749 pkt_stream->pkts[0].len, i, 2);
750 test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream;
751 }
752}
753
754static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem)
755{
756 if (!pkt->valid)
757 return pkt->offset;
758 return pkt->offset + umem_alloc_buffer(umem);
759}
760
761static void pkt_stream_cancel(struct pkt_stream *pkt_stream)
762{
763 pkt_stream->current_pkt_nb--;
764}
765
766static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len,
767 u32 pkt_nb, u32 bytes_written)
768{
769 void *data = xsk_umem__get_data(umem->buffer, addr);
770
771 if (len < MIN_PKT_SIZE)
772 return;
773
774 if (!bytes_written) {
775 gen_eth_hdr(xsk, data);
776
777 len -= PKT_HDR_SIZE;
778 data += PKT_HDR_SIZE;
779 } else {
780 bytes_written -= PKT_HDR_SIZE;
781 }
782
783 write_payload(data, pkt_nb, bytes_written, len);
784}
785
786static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames,
787 u32 nb_frames, bool verbatim)
788{
789 u32 i, len = 0, pkt_nb = 0, payload = 0;
790 struct pkt_stream *pkt_stream;
791
792 pkt_stream = __pkt_stream_alloc(nb_frames);
793 if (!pkt_stream)
794 exit_with_error(ENOMEM);
795
796 for (i = 0; i < nb_frames; i++) {
797 struct pkt *pkt = &pkt_stream->pkts[pkt_nb];
798 struct pkt *frame = &frames[i];
799
800 pkt->offset = frame->offset;
801 if (verbatim) {
802 *pkt = *frame;
803 pkt->pkt_nb = payload;
804 if (!frame->valid || !pkt_continues(frame->options))
805 payload++;
806 } else {
807 if (frame->valid)
808 len += frame->len;
809 if (frame->valid && pkt_continues(frame->options))
810 continue;
811
812 pkt->pkt_nb = pkt_nb;
813 pkt->len = len;
814 pkt->valid = frame->valid;
815 pkt->options = 0;
816
817 len = 0;
818 }
819
820 print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n",
821 pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb);
822
823 if (pkt->valid && pkt->len > pkt_stream->max_pkt_len)
824 pkt_stream->max_pkt_len = pkt->len;
825
826 if (pkt->valid)
827 pkt_stream->nb_valid_entries++;
828
829 pkt_nb++;
830 }
831
832 pkt_stream->nb_pkts = pkt_nb;
833 pkt_stream->verbatim = verbatim;
834 return pkt_stream;
835}
836
837static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
838{
839 struct pkt_stream *pkt_stream;
840
841 pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);
842 test->ifobj_tx->xsk->pkt_stream = pkt_stream;
843
844 pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false);
845 test->ifobj_rx->xsk->pkt_stream = pkt_stream;
846}
847
848static void pkt_print_data(u32 *data, u32 cnt)
849{
850 u32 i;
851
852 for (i = 0; i < cnt; i++) {
853 u32 seqnum, pkt_nb;
854
855 seqnum = ntohl(*data) & 0xffff;
856 pkt_nb = ntohl(*data) >> 16;
857 ksft_print_msg("%u:%u ", pkt_nb, seqnum);
858 data++;
859 }
860}
861
862static void pkt_dump(void *pkt, u32 len, bool eth_header)
863{
864 struct ethhdr *ethhdr = pkt;
865 u32 i, *data;
866
867 if (eth_header) {
868 /*extract L2 frame */
869 ksft_print_msg("DEBUG>> L2: dst mac: ");
870 for (i = 0; i < ETH_ALEN; i++)
871 ksft_print_msg("%02X", ethhdr->h_dest[i]);
872
873 ksft_print_msg("\nDEBUG>> L2: src mac: ");
874 for (i = 0; i < ETH_ALEN; i++)
875 ksft_print_msg("%02X", ethhdr->h_source[i]);
876
877 data = pkt + PKT_HDR_SIZE;
878 } else {
879 data = pkt;
880 }
881
882 /*extract L5 frame */
883 ksft_print_msg("\nDEBUG>> L5: seqnum: ");
884 pkt_print_data(data, PKT_DUMP_NB_TO_PRINT);
885 ksft_print_msg("....");
886 if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) {
887 ksft_print_msg("\n.... ");
888 pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT,
889 PKT_DUMP_NB_TO_PRINT);
890 }
891 ksft_print_msg("\n---------------------------------------\n");
892}
893
894static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr)
895{
896 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
897 u32 offset = addr % umem->frame_size, expected_offset;
898 int pkt_offset = pkt->valid ? pkt->offset : 0;
899
900 if (!umem->unaligned_mode)
901 pkt_offset = 0;
902
903 expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
904
905 if (offset == expected_offset)
906 return true;
907
908 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
909 return false;
910}
911
912static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr)
913{
914 void *data = xsk_umem__get_data(buffer, addr);
915 struct xdp_info *meta = data - sizeof(struct xdp_info);
916
917 if (meta->count != pkt->pkt_nb) {
918 ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n",
919 __func__, pkt->pkt_nb,
920 (unsigned long long)meta->count);
921 return false;
922 }
923
924 return true;
925}
926
927static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb,
928 u32 bytes_processed)
929{
930 u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum;
931 void *data = xsk_umem__get_data(umem->buffer, addr);
932
933 addr -= umem->base_addr;
934
935 if (addr >= umem->num_frames * umem->frame_size ||
936 addr + len > umem->num_frames * umem->frame_size) {
937 ksft_print_msg("Frag invalid addr: %llx len: %u\n",
938 (unsigned long long)addr, len);
939 return false;
940 }
941 if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) {
942 ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n",
943 (unsigned long long)addr, len);
944 return false;
945 }
946
947 pkt_data = data;
948 if (!bytes_processed) {
949 pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data);
950 len -= PKT_HDR_SIZE;
951 } else {
952 bytes_processed -= PKT_HDR_SIZE;
953 }
954
955 expected_seqnum = bytes_processed / sizeof(*pkt_data);
956 seqnum = ntohl(*pkt_data) & 0xffff;
957 pkt_nb = ntohl(*pkt_data) >> 16;
958
959 if (expected_pkt_nb != pkt_nb) {
960 ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n",
961 __func__, expected_pkt_nb, pkt_nb);
962 goto error;
963 }
964 if (expected_seqnum != seqnum) {
965 ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n",
966 __func__, expected_seqnum, seqnum);
967 goto error;
968 }
969
970 words_to_end = len / sizeof(*pkt_data) - 1;
971 pkt_data += words_to_end;
972 seqnum = ntohl(*pkt_data) & 0xffff;
973 expected_seqnum += words_to_end;
974 if (expected_seqnum != seqnum) {
975 ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n",
976 __func__, expected_seqnum, seqnum);
977 goto error;
978 }
979
980 return true;
981
982error:
983 pkt_dump(data, len, !bytes_processed);
984 return false;
985}
986
987static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
988{
989 if (pkt->len != len) {
990 ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n",
991 __func__, pkt->len, len);
992 pkt_dump(xsk_umem__get_data(buffer, addr), len, true);
993 return false;
994 }
995
996 return true;
997}
998
999static int kick_tx(struct xsk_socket_info *xsk)
1000{
1001 int ret;
1002
1003 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
1004 if (ret >= 0)
1005 return TEST_PASS;
1006 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
1007 usleep(100);
1008 return TEST_PASS;
1009 }
1010 return TEST_FAILURE;
1011}
1012
1013static int kick_rx(struct xsk_socket_info *xsk)
1014{
1015 int ret;
1016
1017 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
1018 if (ret < 0)
1019 return TEST_FAILURE;
1020
1021 return TEST_PASS;
1022}
1023
1024static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
1025{
1026 unsigned int rcvd;
1027 u32 idx;
1028 int ret;
1029
1030 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
1031 ret = kick_tx(xsk);
1032 if (ret)
1033 return TEST_FAILURE;
1034 }
1035
1036 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
1037 if (rcvd) {
1038 if (rcvd > xsk->outstanding_tx) {
1039 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
1040
1041 ksft_print_msg("[%s] Too many packets completed\n", __func__);
1042 ksft_print_msg("Last completion address: %llx\n",
1043 (unsigned long long)addr);
1044 return TEST_FAILURE;
1045 }
1046
1047 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
1048 xsk->outstanding_tx -= rcvd;
1049 }
1050
1051 return TEST_PASS;
1052}
1053
1054static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk)
1055{
1056 u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;
1057 u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0;
1058 struct pkt_stream *pkt_stream = xsk->pkt_stream;
1059 struct ifobject *ifobj = test->ifobj_rx;
1060 struct xsk_umem_info *umem = xsk->umem;
1061 struct pollfd fds = { };
1062 struct pkt *pkt;
1063 u64 first_addr = 0;
1064 int ret;
1065
1066 fds.fd = xsk_socket__fd(xsk->xsk);
1067 fds.events = POLLIN;
1068
1069 ret = kick_rx(xsk);
1070 if (ret)
1071 return TEST_FAILURE;
1072
1073 if (ifobj->use_poll) {
1074 ret = poll(&fds, 1, POLL_TMOUT);
1075 if (ret < 0)
1076 return TEST_FAILURE;
1077
1078 if (!ret) {
1079 if (!is_umem_valid(test->ifobj_tx))
1080 return TEST_PASS;
1081
1082 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
1083 return TEST_CONTINUE;
1084 }
1085
1086 if (!(fds.revents & POLLIN))
1087 return TEST_CONTINUE;
1088 }
1089
1090 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
1091 if (!rcvd)
1092 return TEST_CONTINUE;
1093
1094 if (ifobj->use_fill_ring) {
1095 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1096 while (ret != rcvd) {
1097 if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
1098 ret = poll(&fds, 1, POLL_TMOUT);
1099 if (ret < 0)
1100 return TEST_FAILURE;
1101 }
1102 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1103 }
1104 }
1105
1106 while (frags_processed < rcvd) {
1107 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
1108 u64 addr = desc->addr, orig;
1109
1110 orig = xsk_umem__extract_addr(addr);
1111 addr = xsk_umem__add_offset_to_addr(addr);
1112
1113 if (!nb_frags) {
1114 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
1115 if (!pkt) {
1116 ksft_print_msg("[%s] received too many packets addr: %lx len %u\n",
1117 __func__, addr, desc->len);
1118 return TEST_FAILURE;
1119 }
1120 }
1121
1122 print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n",
1123 addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid);
1124
1125 if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||
1126 !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata &&
1127 !is_metadata_correct(pkt, umem->buffer, addr)))
1128 return TEST_FAILURE;
1129
1130 if (!nb_frags++)
1131 first_addr = addr;
1132 frags_processed++;
1133 pkt_len += desc->len;
1134 if (ifobj->use_fill_ring)
1135 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
1136
1137 if (pkt_continues(desc->options))
1138 continue;
1139
1140 /* The complete packet has been received */
1141 if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||
1142 !is_offset_correct(umem, pkt, addr))
1143 return TEST_FAILURE;
1144
1145 pkt_stream->nb_rx_pkts++;
1146 nb_frags = 0;
1147 pkt_len = 0;
1148 }
1149
1150 if (nb_frags) {
1151 /* In the middle of a packet. Start over from beginning of packet. */
1152 idx_rx -= nb_frags;
1153 xsk_ring_cons__cancel(&xsk->rx, nb_frags);
1154 if (ifobj->use_fill_ring) {
1155 idx_fq -= nb_frags;
1156 xsk_ring_prod__cancel(&umem->fq, nb_frags);
1157 }
1158 frags_processed -= nb_frags;
1159 }
1160
1161 if (ifobj->use_fill_ring)
1162 xsk_ring_prod__submit(&umem->fq, frags_processed);
1163 if (ifobj->release_rx)
1164 xsk_ring_cons__release(&xsk->rx, frags_processed);
1165
1166 pthread_mutex_lock(&pacing_mutex);
1167 pkts_in_flight -= pkts_sent;
1168 pthread_mutex_unlock(&pacing_mutex);
1169 pkts_sent = 0;
1170
1171return TEST_CONTINUE;
1172}
1173
1174bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num,
1175 unsigned long *bitmap)
1176{
1177 struct pkt_stream *pkt_stream = xsk->pkt_stream;
1178
1179 if (!pkt_stream) {
1180 __set_bit(sock_num, bitmap);
1181 return false;
1182 }
1183
1184 if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) {
1185 __set_bit(sock_num, bitmap);
1186 if (bitmap_full(bitmap, test->nb_sockets))
1187 return true;
1188 }
1189
1190 return false;
1191}
1192
1193static int receive_pkts(struct test_spec *test)
1194{
1195 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1196 DECLARE_BITMAP(bitmap, test->nb_sockets);
1197 struct xsk_socket_info *xsk;
1198 u32 sock_num = 0;
1199 int res, ret;
1200
1201 ret = gettimeofday(&tv_now, NULL);
1202 if (ret)
1203 exit_with_error(errno);
1204
1205 timeradd(&tv_now, &tv_timeout, &tv_end);
1206
1207 while (1) {
1208 xsk = &test->ifobj_rx->xsk_arr[sock_num];
1209
1210 if ((all_packets_received(test, xsk, sock_num, bitmap)))
1211 break;
1212
1213 res = __receive_pkts(test, xsk);
1214 if (!(res == TEST_PASS || res == TEST_CONTINUE))
1215 return res;
1216
1217 ret = gettimeofday(&tv_now, NULL);
1218 if (ret)
1219 exit_with_error(errno);
1220
1221 if (timercmp(&tv_now, &tv_end, >)) {
1222 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
1223 return TEST_FAILURE;
1224 }
1225 sock_num = (sock_num + 1) % test->nb_sockets;
1226 }
1227
1228 return TEST_PASS;
1229}
1230
1231static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout)
1232{
1233 u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;
1234 struct pkt_stream *pkt_stream = xsk->pkt_stream;
1235 struct xsk_umem_info *umem = ifobject->umem;
1236 bool use_poll = ifobject->use_poll;
1237 struct pollfd fds = { };
1238 int ret;
1239
1240 buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len);
1241 /* pkts_in_flight might be negative if many invalid packets are sent */
1242 if (pkts_in_flight >= (int)((umem_size(umem) - BATCH_SIZE * buffer_len) / buffer_len)) {
1243 ret = kick_tx(xsk);
1244 if (ret)
1245 return TEST_FAILURE;
1246 return TEST_CONTINUE;
1247 }
1248
1249 fds.fd = xsk_socket__fd(xsk->xsk);
1250 fds.events = POLLOUT;
1251
1252 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
1253 if (use_poll) {
1254 ret = poll(&fds, 1, POLL_TMOUT);
1255 if (timeout) {
1256 if (ret < 0) {
1257 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1258 __func__, errno);
1259 return TEST_FAILURE;
1260 }
1261 if (ret == 0)
1262 return TEST_PASS;
1263 break;
1264 }
1265 if (ret <= 0) {
1266 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1267 __func__, errno);
1268 return TEST_FAILURE;
1269 }
1270 }
1271
1272 complete_pkts(xsk, BATCH_SIZE);
1273 }
1274
1275 for (i = 0; i < BATCH_SIZE; i++) {
1276 struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
1277 u32 nb_frags_left, nb_frags, bytes_written = 0;
1278
1279 if (!pkt)
1280 break;
1281
1282 nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt);
1283 if (nb_frags > BATCH_SIZE - i) {
1284 pkt_stream_cancel(pkt_stream);
1285 xsk_ring_prod__cancel(&xsk->tx, BATCH_SIZE - i);
1286 break;
1287 }
1288 nb_frags_left = nb_frags;
1289
1290 while (nb_frags_left--) {
1291 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
1292
1293 tx_desc->addr = pkt_get_addr(pkt, ifobject->umem);
1294 if (pkt_stream->verbatim) {
1295 tx_desc->len = pkt->len;
1296 tx_desc->options = pkt->options;
1297 } else if (nb_frags_left) {
1298 tx_desc->len = umem->frame_size;
1299 tx_desc->options = XDP_PKT_CONTD;
1300 } else {
1301 tx_desc->len = pkt->len - bytes_written;
1302 tx_desc->options = 0;
1303 }
1304 if (pkt->valid)
1305 pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb,
1306 bytes_written);
1307 bytes_written += tx_desc->len;
1308
1309 print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n",
1310 tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb);
1311
1312 if (nb_frags_left) {
1313 i++;
1314 if (pkt_stream->verbatim)
1315 pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
1316 }
1317 }
1318
1319 if (pkt && pkt->valid) {
1320 valid_pkts++;
1321 valid_frags += nb_frags;
1322 }
1323 }
1324
1325 pthread_mutex_lock(&pacing_mutex);
1326 pkts_in_flight += valid_pkts;
1327 pthread_mutex_unlock(&pacing_mutex);
1328
1329 xsk_ring_prod__submit(&xsk->tx, i);
1330 xsk->outstanding_tx += valid_frags;
1331
1332 if (use_poll) {
1333 ret = poll(&fds, 1, POLL_TMOUT);
1334 if (ret <= 0) {
1335 if (ret == 0 && timeout)
1336 return TEST_PASS;
1337
1338 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1339 return TEST_FAILURE;
1340 }
1341 }
1342
1343 if (!timeout) {
1344 if (complete_pkts(xsk, i))
1345 return TEST_FAILURE;
1346
1347 usleep(10);
1348 return TEST_PASS;
1349 }
1350
1351 return TEST_CONTINUE;
1352}
1353
1354static int wait_for_tx_completion(struct xsk_socket_info *xsk)
1355{
1356 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1357 int ret;
1358
1359 ret = gettimeofday(&tv_now, NULL);
1360 if (ret)
1361 exit_with_error(errno);
1362 timeradd(&tv_now, &tv_timeout, &tv_end);
1363
1364 while (xsk->outstanding_tx) {
1365 ret = gettimeofday(&tv_now, NULL);
1366 if (ret)
1367 exit_with_error(errno);
1368 if (timercmp(&tv_now, &tv_end, >)) {
1369 ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__);
1370 return TEST_FAILURE;
1371 }
1372
1373 complete_pkts(xsk, BATCH_SIZE);
1374 }
1375
1376 return TEST_PASS;
1377}
1378
1379bool all_packets_sent(struct test_spec *test, unsigned long *bitmap)
1380{
1381 return bitmap_full(bitmap, test->nb_sockets);
1382}
1383
1384static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1385{
1386 bool timeout = !is_umem_valid(test->ifobj_rx);
1387 DECLARE_BITMAP(bitmap, test->nb_sockets);
1388 u32 i, ret;
1389
1390 while (!(all_packets_sent(test, bitmap))) {
1391 for (i = 0; i < test->nb_sockets; i++) {
1392 struct pkt_stream *pkt_stream;
1393
1394 pkt_stream = ifobject->xsk_arr[i].pkt_stream;
1395 if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) {
1396 __set_bit(i, bitmap);
1397 continue;
1398 }
1399 ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout);
1400 if (ret == TEST_CONTINUE && !test->fail)
1401 continue;
1402
1403 if ((ret || test->fail) && !timeout)
1404 return TEST_FAILURE;
1405
1406 if (ret == TEST_PASS && timeout)
1407 return ret;
1408
1409 ret = wait_for_tx_completion(&ifobject->xsk_arr[i]);
1410 if (ret)
1411 return TEST_FAILURE;
1412 }
1413 }
1414
1415 return TEST_PASS;
1416}
1417
1418static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1419{
1420 int fd = xsk_socket__fd(xsk), err;
1421 socklen_t optlen, expected_len;
1422
1423 optlen = sizeof(*stats);
1424 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1425 if (err) {
1426 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1427 __func__, -err, strerror(-err));
1428 return TEST_FAILURE;
1429 }
1430
1431 expected_len = sizeof(struct xdp_statistics);
1432 if (optlen != expected_len) {
1433 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1434 __func__, expected_len, optlen);
1435 return TEST_FAILURE;
1436 }
1437
1438 return TEST_PASS;
1439}
1440
1441static int validate_rx_dropped(struct ifobject *ifobject)
1442{
1443 struct xsk_socket *xsk = ifobject->xsk->xsk;
1444 struct xdp_statistics stats;
1445 int err;
1446
1447 err = kick_rx(ifobject->xsk);
1448 if (err)
1449 return TEST_FAILURE;
1450
1451 err = get_xsk_stats(xsk, &stats);
1452 if (err)
1453 return TEST_FAILURE;
1454
1455 /* The receiver calls getsockopt after receiving the last (valid)
1456 * packet which is not the final packet sent in this test (valid and
1457 * invalid packets are sent in alternating fashion with the final
1458 * packet being invalid). Since the last packet may or may not have
1459 * been dropped already, both outcomes must be allowed.
1460 */
1461 if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 ||
1462 stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1)
1463 return TEST_PASS;
1464
1465 return TEST_FAILURE;
1466}
1467
1468static int validate_rx_full(struct ifobject *ifobject)
1469{
1470 struct xsk_socket *xsk = ifobject->xsk->xsk;
1471 struct xdp_statistics stats;
1472 int err;
1473
1474 usleep(1000);
1475 err = kick_rx(ifobject->xsk);
1476 if (err)
1477 return TEST_FAILURE;
1478
1479 err = get_xsk_stats(xsk, &stats);
1480 if (err)
1481 return TEST_FAILURE;
1482
1483 if (stats.rx_ring_full)
1484 return TEST_PASS;
1485
1486 return TEST_FAILURE;
1487}
1488
1489static int validate_fill_empty(struct ifobject *ifobject)
1490{
1491 struct xsk_socket *xsk = ifobject->xsk->xsk;
1492 struct xdp_statistics stats;
1493 int err;
1494
1495 usleep(1000);
1496 err = kick_rx(ifobject->xsk);
1497 if (err)
1498 return TEST_FAILURE;
1499
1500 err = get_xsk_stats(xsk, &stats);
1501 if (err)
1502 return TEST_FAILURE;
1503
1504 if (stats.rx_fill_ring_empty_descs)
1505 return TEST_PASS;
1506
1507 return TEST_FAILURE;
1508}
1509
1510static int validate_tx_invalid_descs(struct ifobject *ifobject)
1511{
1512 struct xsk_socket *xsk = ifobject->xsk->xsk;
1513 int fd = xsk_socket__fd(xsk);
1514 struct xdp_statistics stats;
1515 socklen_t optlen;
1516 int err;
1517
1518 optlen = sizeof(stats);
1519 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1520 if (err) {
1521 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1522 __func__, -err, strerror(-err));
1523 return TEST_FAILURE;
1524 }
1525
1526 if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) {
1527 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n",
1528 __func__,
1529 (unsigned long long)stats.tx_invalid_descs,
1530 ifobject->xsk->pkt_stream->nb_pkts);
1531 return TEST_FAILURE;
1532 }
1533
1534 return TEST_PASS;
1535}
1536
1537static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1538 struct xsk_umem_info *umem, bool tx)
1539{
1540 int i, ret;
1541
1542 for (i = 0; i < test->nb_sockets; i++) {
1543 bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1544 u32 ctr = 0;
1545
1546 while (ctr++ < SOCK_RECONF_CTR) {
1547 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1548 ifobject, shared);
1549 if (!ret)
1550 break;
1551
1552 /* Retry if it fails as xsk_socket__create() is asynchronous */
1553 if (ctr >= SOCK_RECONF_CTR)
1554 exit_with_error(-ret);
1555 usleep(USLEEP_MAX);
1556 }
1557 if (ifobject->busy_poll)
1558 enable_busy_poll(&ifobject->xsk_arr[i]);
1559 }
1560}
1561
1562static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1563{
1564 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1565 ifobject->xsk = &ifobject->xsk_arr[0];
1566 ifobject->xskmap = test->ifobj_rx->xskmap;
1567 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1568 ifobject->umem->base_addr = 0;
1569}
1570
1571static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream,
1572 bool fill_up)
1573{
1574 u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM;
1575 u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts;
1576 int ret;
1577
1578 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1579 buffers_to_fill = umem->num_frames;
1580 else
1581 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1582
1583 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1584 if (ret != buffers_to_fill)
1585 exit_with_error(ENOSPC);
1586
1587 while (filled < buffers_to_fill) {
1588 struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts);
1589 u64 addr;
1590 u32 i;
1591
1592 for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) {
1593 if (!pkt) {
1594 if (!fill_up)
1595 break;
1596 addr = filled * umem->frame_size + umem->base_addr;
1597 } else if (pkt->offset >= 0) {
1598 addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem);
1599 } else {
1600 addr = pkt->offset + umem_alloc_buffer(umem);
1601 }
1602
1603 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1604 if (++filled >= buffers_to_fill)
1605 break;
1606 }
1607 }
1608 xsk_ring_prod__submit(&umem->fq, filled);
1609 xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled);
1610
1611 pkt_stream_reset(pkt_stream);
1612 umem_reset_alloc(umem);
1613}
1614
1615static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1616{
1617 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1618 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1619 LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1620 void *bufs;
1621 int ret;
1622 u32 i;
1623
1624 if (ifobject->umem->unaligned_mode)
1625 mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB;
1626
1627 if (ifobject->shared_umem)
1628 umem_sz *= 2;
1629
1630 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1631 if (bufs == MAP_FAILED)
1632 exit_with_error(errno);
1633
1634 ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz);
1635 if (ret)
1636 exit_with_error(-ret);
1637
1638 xsk_configure_socket(test, ifobject, ifobject->umem, false);
1639
1640 ifobject->xsk = &ifobject->xsk_arr[0];
1641
1642 if (!ifobject->rx_on)
1643 return;
1644
1645 xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring);
1646
1647 for (i = 0; i < test->nb_sockets; i++) {
1648 ifobject->xsk = &ifobject->xsk_arr[i];
1649 ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i);
1650 if (ret)
1651 exit_with_error(errno);
1652 }
1653}
1654
1655static void *worker_testapp_validate_tx(void *arg)
1656{
1657 struct test_spec *test = (struct test_spec *)arg;
1658 struct ifobject *ifobject = test->ifobj_tx;
1659 int err;
1660
1661 if (test->current_step == 1) {
1662 if (!ifobject->shared_umem)
1663 thread_common_ops(test, ifobject);
1664 else
1665 thread_common_ops_tx(test, ifobject);
1666 }
1667
1668 err = send_pkts(test, ifobject);
1669
1670 if (!err && ifobject->validation_func)
1671 err = ifobject->validation_func(ifobject);
1672 if (err)
1673 report_failure(test);
1674
1675 pthread_exit(NULL);
1676}
1677
1678static void *worker_testapp_validate_rx(void *arg)
1679{
1680 struct test_spec *test = (struct test_spec *)arg;
1681 struct ifobject *ifobject = test->ifobj_rx;
1682 int err;
1683
1684 if (test->current_step == 1) {
1685 thread_common_ops(test, ifobject);
1686 } else {
1687 xsk_clear_xskmap(ifobject->xskmap);
1688 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0);
1689 if (err) {
1690 ksft_print_msg("Error: Failed to update xskmap, error %s\n",
1691 strerror(-err));
1692 exit_with_error(-err);
1693 }
1694 }
1695
1696 pthread_barrier_wait(&barr);
1697
1698 err = receive_pkts(test);
1699
1700 if (!err && ifobject->validation_func)
1701 err = ifobject->validation_func(ifobject);
1702 if (err)
1703 report_failure(test);
1704
1705 pthread_exit(NULL);
1706}
1707
1708static u64 ceil_u64(u64 a, u64 b)
1709{
1710 return (a + b - 1) / b;
1711}
1712
1713static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1714{
1715 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1716
1717 if (ifobj->shared_umem)
1718 umem_sz *= 2;
1719
1720 umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
1721 xsk_umem__delete(ifobj->umem->umem);
1722 munmap(ifobj->umem->buffer, umem_sz);
1723}
1724
1725static void handler(int signum)
1726{
1727 pthread_exit(NULL);
1728}
1729
1730static bool xdp_prog_changed_rx(struct test_spec *test)
1731{
1732 struct ifobject *ifobj = test->ifobj_rx;
1733
1734 return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode;
1735}
1736
1737static bool xdp_prog_changed_tx(struct test_spec *test)
1738{
1739 struct ifobject *ifobj = test->ifobj_tx;
1740
1741 return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode;
1742}
1743
1744static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,
1745 struct bpf_map *xskmap, enum test_mode mode)
1746{
1747 int err;
1748
1749 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode));
1750 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode));
1751 if (err) {
1752 ksft_print_msg("Error attaching XDP program\n");
1753 exit_with_error(-err);
1754 }
1755
1756 if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC))
1757 if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) {
1758 ksft_print_msg("ERROR: XDP prog not in DRV mode\n");
1759 exit_with_error(EINVAL);
1760 }
1761
1762 ifobj->xdp_prog = xdp_prog;
1763 ifobj->xskmap = xskmap;
1764 ifobj->mode = mode;
1765}
1766
1767static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,
1768 struct ifobject *ifobj_tx)
1769{
1770 if (xdp_prog_changed_rx(test))
1771 xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode);
1772
1773 if (!ifobj_tx || ifobj_tx->shared_umem)
1774 return;
1775
1776 if (xdp_prog_changed_tx(test))
1777 xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);
1778}
1779
1780static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,
1781 struct ifobject *ifobj2)
1782{
1783 pthread_t t0, t1;
1784 int err;
1785
1786 if (test->mtu > MAX_ETH_PKT_SIZE) {
1787 if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp ||
1788 (ifobj2 && !ifobj2->multi_buff_zc_supp))) {
1789 ksft_test_result_skip("Multi buffer for zero-copy not supported.\n");
1790 return TEST_SKIP;
1791 }
1792 if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp ||
1793 (ifobj2 && !ifobj2->multi_buff_supp))) {
1794 ksft_test_result_skip("Multi buffer not supported.\n");
1795 return TEST_SKIP;
1796 }
1797 }
1798 err = test_spec_set_mtu(test, test->mtu);
1799 if (err) {
1800 ksft_print_msg("Error, could not set mtu.\n");
1801 exit_with_error(err);
1802 }
1803
1804 if (ifobj2) {
1805 if (pthread_barrier_init(&barr, NULL, 2))
1806 exit_with_error(errno);
1807 pkt_stream_reset(ifobj2->xsk->pkt_stream);
1808 }
1809
1810 test->current_step++;
1811 pkt_stream_reset(ifobj1->xsk->pkt_stream);
1812 pkts_in_flight = 0;
1813
1814 signal(SIGUSR1, handler);
1815 /*Spawn RX thread */
1816 pthread_create(&t0, NULL, ifobj1->func_ptr, test);
1817
1818 if (ifobj2) {
1819 pthread_barrier_wait(&barr);
1820 if (pthread_barrier_destroy(&barr))
1821 exit_with_error(errno);
1822
1823 /*Spawn TX thread */
1824 pthread_create(&t1, NULL, ifobj2->func_ptr, test);
1825
1826 pthread_join(t1, NULL);
1827 }
1828
1829 if (!ifobj2)
1830 pthread_kill(t0, SIGUSR1);
1831 else
1832 pthread_join(t0, NULL);
1833
1834 if (test->total_steps == test->current_step || test->fail) {
1835 u32 i;
1836
1837 if (ifobj2)
1838 for (i = 0; i < test->nb_sockets; i++)
1839 xsk_socket__delete(ifobj2->xsk_arr[i].xsk);
1840
1841 for (i = 0; i < test->nb_sockets; i++)
1842 xsk_socket__delete(ifobj1->xsk_arr[i].xsk);
1843
1844 testapp_clean_xsk_umem(ifobj1);
1845 if (ifobj2 && !ifobj2->shared_umem)
1846 testapp_clean_xsk_umem(ifobj2);
1847 }
1848
1849 return !!test->fail;
1850}
1851
1852static int testapp_validate_traffic(struct test_spec *test)
1853{
1854 struct ifobject *ifobj_rx = test->ifobj_rx;
1855 struct ifobject *ifobj_tx = test->ifobj_tx;
1856
1857 if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) ||
1858 (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) {
1859 ksft_test_result_skip("No huge pages present.\n");
1860 return TEST_SKIP;
1861 }
1862
1863 xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx);
1864 return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);
1865}
1866
1867static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)
1868{
1869 return __testapp_validate_traffic(test, ifobj, NULL);
1870}
1871
1872static int testapp_teardown(struct test_spec *test)
1873{
1874 int i;
1875
1876 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1877 if (testapp_validate_traffic(test))
1878 return TEST_FAILURE;
1879 test_spec_reset(test);
1880 }
1881
1882 return TEST_PASS;
1883}
1884
1885static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1886{
1887 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1888 struct ifobject *tmp_ifobj = (*ifobj1);
1889
1890 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1891 (*ifobj2)->func_ptr = tmp_func_ptr;
1892
1893 *ifobj1 = *ifobj2;
1894 *ifobj2 = tmp_ifobj;
1895}
1896
1897static int testapp_bidirectional(struct test_spec *test)
1898{
1899 int res;
1900
1901 test->ifobj_tx->rx_on = true;
1902 test->ifobj_rx->tx_on = true;
1903 test->total_steps = 2;
1904 if (testapp_validate_traffic(test))
1905 return TEST_FAILURE;
1906
1907 print_verbose("Switching Tx/Rx direction\n");
1908 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1909 res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);
1910
1911 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1912 return res;
1913}
1914
1915static int swap_xsk_resources(struct test_spec *test)
1916{
1917 int ret;
1918
1919 test->ifobj_tx->xsk_arr[0].pkt_stream = NULL;
1920 test->ifobj_rx->xsk_arr[0].pkt_stream = NULL;
1921 test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default;
1922 test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default;
1923 test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1];
1924 test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1];
1925
1926 ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0);
1927 if (ret)
1928 return TEST_FAILURE;
1929
1930 return TEST_PASS;
1931}
1932
1933static int testapp_xdp_prog_cleanup(struct test_spec *test)
1934{
1935 test->total_steps = 2;
1936 test->nb_sockets = 2;
1937 if (testapp_validate_traffic(test))
1938 return TEST_FAILURE;
1939
1940 if (swap_xsk_resources(test))
1941 return TEST_FAILURE;
1942 return testapp_validate_traffic(test);
1943}
1944
1945static int testapp_headroom(struct test_spec *test)
1946{
1947 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1948 return testapp_validate_traffic(test);
1949}
1950
1951static int testapp_stats_rx_dropped(struct test_spec *test)
1952{
1953 if (test->mode == TEST_MODE_ZC) {
1954 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
1955 return TEST_SKIP;
1956 }
1957
1958 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1959 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1960 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1961 pkt_stream_receive_half(test);
1962 test->ifobj_rx->validation_func = validate_rx_dropped;
1963 return testapp_validate_traffic(test);
1964}
1965
1966static int testapp_stats_tx_invalid_descs(struct test_spec *test)
1967{
1968 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
1969 test->ifobj_tx->validation_func = validate_tx_invalid_descs;
1970 return testapp_validate_traffic(test);
1971}
1972
1973static int testapp_stats_rx_full(struct test_spec *test)
1974{
1975 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
1976 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
1977
1978 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
1979 test->ifobj_rx->release_rx = false;
1980 test->ifobj_rx->validation_func = validate_rx_full;
1981 return testapp_validate_traffic(test);
1982}
1983
1984static int testapp_stats_fill_empty(struct test_spec *test)
1985{
1986 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
1987 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
1988
1989 test->ifobj_rx->use_fill_ring = false;
1990 test->ifobj_rx->validation_func = validate_fill_empty;
1991 return testapp_validate_traffic(test);
1992}
1993
1994static int testapp_send_receive_unaligned(struct test_spec *test)
1995{
1996 test->ifobj_tx->umem->unaligned_mode = true;
1997 test->ifobj_rx->umem->unaligned_mode = true;
1998 /* Let half of the packets straddle a 4K buffer boundary */
1999 pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2);
2000
2001 return testapp_validate_traffic(test);
2002}
2003
2004static int testapp_send_receive_unaligned_mb(struct test_spec *test)
2005{
2006 test->mtu = MAX_ETH_JUMBO_SIZE;
2007 test->ifobj_tx->umem->unaligned_mode = true;
2008 test->ifobj_rx->umem->unaligned_mode = true;
2009 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
2010 return testapp_validate_traffic(test);
2011}
2012
2013static int testapp_single_pkt(struct test_spec *test)
2014{
2015 struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}};
2016
2017 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
2018 return testapp_validate_traffic(test);
2019}
2020
2021static int testapp_send_receive_mb(struct test_spec *test)
2022{
2023 test->mtu = MAX_ETH_JUMBO_SIZE;
2024 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
2025
2026 return testapp_validate_traffic(test);
2027}
2028
2029static int testapp_invalid_desc_mb(struct test_spec *test)
2030{
2031 struct xsk_umem_info *umem = test->ifobj_tx->umem;
2032 u64 umem_size = umem->num_frames * umem->frame_size;
2033 struct pkt pkts[] = {
2034 /* Valid packet for synch to start with */
2035 {0, MIN_PKT_SIZE, 0, true, 0},
2036 /* Zero frame len is not legal */
2037 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2038 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2039 {0, 0, 0, false, 0},
2040 /* Invalid address in the second frame */
2041 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2042 {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2043 /* Invalid len in the middle */
2044 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2045 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2046 /* Invalid options in the middle */
2047 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2048 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION},
2049 /* Transmit 2 frags, receive 3 */
2050 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD},
2051 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0},
2052 /* Middle frame crosses chunk boundary with small length */
2053 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2054 {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0},
2055 /* Valid packet for synch so that something is received */
2056 {0, MIN_PKT_SIZE, 0, true, 0}};
2057
2058 if (umem->unaligned_mode) {
2059 /* Crossing a chunk boundary allowed */
2060 pkts[12].valid = true;
2061 pkts[13].valid = true;
2062 }
2063
2064 test->mtu = MAX_ETH_JUMBO_SIZE;
2065 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
2066 return testapp_validate_traffic(test);
2067}
2068
2069static int testapp_invalid_desc(struct test_spec *test)
2070{
2071 struct xsk_umem_info *umem = test->ifobj_tx->umem;
2072 u64 umem_size = umem->num_frames * umem->frame_size;
2073 struct pkt pkts[] = {
2074 /* Zero packet address allowed */
2075 {0, MIN_PKT_SIZE, 0, true},
2076 /* Allowed packet */
2077 {0, MIN_PKT_SIZE, 0, true},
2078 /* Straddling the start of umem */
2079 {-2, MIN_PKT_SIZE, 0, false},
2080 /* Packet too large */
2081 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
2082 /* Up to end of umem allowed */
2083 {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true},
2084 /* After umem ends */
2085 {umem_size, MIN_PKT_SIZE, 0, false},
2086 /* Straddle the end of umem */
2087 {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
2088 /* Straddle a 4K boundary */
2089 {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
2090 /* Straddle a 2K boundary */
2091 {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true},
2092 /* Valid packet for synch so that something is received */
2093 {0, MIN_PKT_SIZE, 0, true}};
2094
2095 if (umem->unaligned_mode) {
2096 /* Crossing a page boundary allowed */
2097 pkts[7].valid = true;
2098 }
2099 if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
2100 /* Crossing a 2K frame size boundary not allowed */
2101 pkts[8].valid = false;
2102 }
2103
2104 if (test->ifobj_tx->shared_umem) {
2105 pkts[4].offset += umem_size;
2106 pkts[5].offset += umem_size;
2107 pkts[6].offset += umem_size;
2108 }
2109
2110 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
2111 return testapp_validate_traffic(test);
2112}
2113
2114static int testapp_xdp_drop(struct test_spec *test)
2115{
2116 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2117 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2118
2119 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop,
2120 skel_rx->maps.xsk, skel_tx->maps.xsk);
2121
2122 pkt_stream_receive_half(test);
2123 return testapp_validate_traffic(test);
2124}
2125
2126static int testapp_xdp_metadata_copy(struct test_spec *test)
2127{
2128 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2129 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2130 struct bpf_map *data_map;
2131 int count = 0;
2132 int key = 0;
2133
2134 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata,
2135 skel_tx->progs.xsk_xdp_populate_metadata,
2136 skel_rx->maps.xsk, skel_tx->maps.xsk);
2137 test->ifobj_rx->use_metadata = true;
2138
2139 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");
2140 if (!data_map || !bpf_map__is_internal(data_map)) {
2141 ksft_print_msg("Error: could not find bss section of XDP program\n");
2142 return TEST_FAILURE;
2143 }
2144
2145 if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) {
2146 ksft_print_msg("Error: could not update count element\n");
2147 return TEST_FAILURE;
2148 }
2149
2150 return testapp_validate_traffic(test);
2151}
2152
2153static int testapp_xdp_shared_umem(struct test_spec *test)
2154{
2155 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2156 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2157
2158 test->total_steps = 1;
2159 test->nb_sockets = 2;
2160
2161 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem,
2162 skel_tx->progs.xsk_xdp_shared_umem,
2163 skel_rx->maps.xsk, skel_tx->maps.xsk);
2164
2165 pkt_stream_even_odd_sequence(test);
2166
2167 return testapp_validate_traffic(test);
2168}
2169
2170static int testapp_poll_txq_tmout(struct test_spec *test)
2171{
2172 test->ifobj_tx->use_poll = true;
2173 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */
2174 test->ifobj_tx->umem->frame_size = 2048;
2175 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
2176 return testapp_validate_traffic_single_thread(test, test->ifobj_tx);
2177}
2178
2179static int testapp_poll_rxq_tmout(struct test_spec *test)
2180{
2181 test->ifobj_rx->use_poll = true;
2182 return testapp_validate_traffic_single_thread(test, test->ifobj_rx);
2183}
2184
2185static int testapp_too_many_frags(struct test_spec *test)
2186{
2187 struct pkt pkts[2 * XSK_DESC__MAX_SKB_FRAGS + 2] = {};
2188 u32 max_frags, i;
2189
2190 if (test->mode == TEST_MODE_ZC)
2191 max_frags = test->ifobj_tx->xdp_zc_max_segs;
2192 else
2193 max_frags = XSK_DESC__MAX_SKB_FRAGS;
2194
2195 test->mtu = MAX_ETH_JUMBO_SIZE;
2196
2197 /* Valid packet for synch */
2198 pkts[0].len = MIN_PKT_SIZE;
2199 pkts[0].valid = true;
2200
2201 /* One valid packet with the max amount of frags */
2202 for (i = 1; i < max_frags + 1; i++) {
2203 pkts[i].len = MIN_PKT_SIZE;
2204 pkts[i].options = XDP_PKT_CONTD;
2205 pkts[i].valid = true;
2206 }
2207 pkts[max_frags].options = 0;
2208
2209 /* An invalid packet with the max amount of frags but signals packet
2210 * continues on the last frag
2211 */
2212 for (i = max_frags + 1; i < 2 * max_frags + 1; i++) {
2213 pkts[i].len = MIN_PKT_SIZE;
2214 pkts[i].options = XDP_PKT_CONTD;
2215 pkts[i].valid = false;
2216 }
2217
2218 /* Valid packet for synch */
2219 pkts[2 * max_frags + 1].len = MIN_PKT_SIZE;
2220 pkts[2 * max_frags + 1].valid = true;
2221
2222 pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2);
2223 return testapp_validate_traffic(test);
2224}
2225
2226static int xsk_load_xdp_programs(struct ifobject *ifobj)
2227{
2228 ifobj->xdp_progs = xsk_xdp_progs__open_and_load();
2229 if (libbpf_get_error(ifobj->xdp_progs))
2230 return libbpf_get_error(ifobj->xdp_progs);
2231
2232 return 0;
2233}
2234
2235static void xsk_unload_xdp_programs(struct ifobject *ifobj)
2236{
2237 xsk_xdp_progs__destroy(ifobj->xdp_progs);
2238}
2239
2240/* Simple test */
2241static bool hugepages_present(void)
2242{
2243 size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
2244 void *bufs;
2245
2246 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
2247 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB);
2248 if (bufs == MAP_FAILED)
2249 return false;
2250
2251 mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
2252 munmap(bufs, mmap_sz);
2253 return true;
2254}
2255
2256static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr)
2257{
2258 LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);
2259 int err;
2260
2261 ifobj->func_ptr = func_ptr;
2262
2263 err = xsk_load_xdp_programs(ifobj);
2264 if (err) {
2265 ksft_print_msg("Error loading XDP program\n");
2266 exit_with_error(err);
2267 }
2268
2269 if (hugepages_present())
2270 ifobj->unaligned_supp = true;
2271
2272 err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts);
2273 if (err) {
2274 ksft_print_msg("Error querying XDP capabilities\n");
2275 exit_with_error(-err);
2276 }
2277 if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG)
2278 ifobj->multi_buff_supp = true;
2279 if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
2280 if (query_opts.xdp_zc_max_segs > 1) {
2281 ifobj->multi_buff_zc_supp = true;
2282 ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs;
2283 } else {
2284 ifobj->xdp_zc_max_segs = 0;
2285 }
2286 }
2287}
2288
2289static int testapp_send_receive(struct test_spec *test)
2290{
2291 return testapp_validate_traffic(test);
2292}
2293
2294static int testapp_send_receive_2k_frame(struct test_spec *test)
2295{
2296 test->ifobj_tx->umem->frame_size = 2048;
2297 test->ifobj_rx->umem->frame_size = 2048;
2298 pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2299 return testapp_validate_traffic(test);
2300}
2301
2302static int testapp_poll_rx(struct test_spec *test)
2303{
2304 test->ifobj_rx->use_poll = true;
2305 return testapp_validate_traffic(test);
2306}
2307
2308static int testapp_poll_tx(struct test_spec *test)
2309{
2310 test->ifobj_tx->use_poll = true;
2311 return testapp_validate_traffic(test);
2312}
2313
2314static int testapp_aligned_inv_desc(struct test_spec *test)
2315{
2316 return testapp_invalid_desc(test);
2317}
2318
2319static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test)
2320{
2321 test->ifobj_tx->umem->frame_size = 2048;
2322 test->ifobj_rx->umem->frame_size = 2048;
2323 return testapp_invalid_desc(test);
2324}
2325
2326static int testapp_unaligned_inv_desc(struct test_spec *test)
2327{
2328 test->ifobj_tx->umem->unaligned_mode = true;
2329 test->ifobj_rx->umem->unaligned_mode = true;
2330 return testapp_invalid_desc(test);
2331}
2332
2333static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test)
2334{
2335 u64 page_size, umem_size;
2336
2337 /* Odd frame size so the UMEM doesn't end near a page boundary. */
2338 test->ifobj_tx->umem->frame_size = 4001;
2339 test->ifobj_rx->umem->frame_size = 4001;
2340 test->ifobj_tx->umem->unaligned_mode = true;
2341 test->ifobj_rx->umem->unaligned_mode = true;
2342 /* This test exists to test descriptors that staddle the end of
2343 * the UMEM but not a page.
2344 */
2345 page_size = sysconf(_SC_PAGESIZE);
2346 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;
2347 assert(umem_size % page_size > MIN_PKT_SIZE);
2348 assert(umem_size % page_size < page_size - MIN_PKT_SIZE);
2349
2350 return testapp_invalid_desc(test);
2351}
2352
2353static int testapp_aligned_inv_desc_mb(struct test_spec *test)
2354{
2355 return testapp_invalid_desc_mb(test);
2356}
2357
2358static int testapp_unaligned_inv_desc_mb(struct test_spec *test)
2359{
2360 test->ifobj_tx->umem->unaligned_mode = true;
2361 test->ifobj_rx->umem->unaligned_mode = true;
2362 return testapp_invalid_desc_mb(test);
2363}
2364
2365static int testapp_xdp_metadata(struct test_spec *test)
2366{
2367 return testapp_xdp_metadata_copy(test);
2368}
2369
2370static int testapp_xdp_metadata_mb(struct test_spec *test)
2371{
2372 test->mtu = MAX_ETH_JUMBO_SIZE;
2373 return testapp_xdp_metadata_copy(test);
2374}
2375
2376static void run_pkt_test(struct test_spec *test)
2377{
2378 int ret;
2379
2380 ret = test->test_func(test);
2381
2382 if (ret == TEST_PASS)
2383 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
2384 test->name);
2385 pkt_stream_restore_default(test);
2386}
2387
2388static struct ifobject *ifobject_create(void)
2389{
2390 struct ifobject *ifobj;
2391
2392 ifobj = calloc(1, sizeof(struct ifobject));
2393 if (!ifobj)
2394 return NULL;
2395
2396 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
2397 if (!ifobj->xsk_arr)
2398 goto out_xsk_arr;
2399
2400 ifobj->umem = calloc(1, sizeof(*ifobj->umem));
2401 if (!ifobj->umem)
2402 goto out_umem;
2403
2404 return ifobj;
2405
2406out_umem:
2407 free(ifobj->xsk_arr);
2408out_xsk_arr:
2409 free(ifobj);
2410 return NULL;
2411}
2412
2413static void ifobject_delete(struct ifobject *ifobj)
2414{
2415 free(ifobj->umem);
2416 free(ifobj->xsk_arr);
2417 free(ifobj);
2418}
2419
2420static bool is_xdp_supported(int ifindex)
2421{
2422 int flags = XDP_FLAGS_DRV_MODE;
2423
2424 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
2425 struct bpf_insn insns[2] = {
2426 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
2427 BPF_EXIT_INSN()
2428 };
2429 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
2430 int err;
2431
2432 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
2433 if (prog_fd < 0)
2434 return false;
2435
2436 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
2437 if (err) {
2438 close(prog_fd);
2439 return false;
2440 }
2441
2442 bpf_xdp_detach(ifindex, flags, NULL);
2443 close(prog_fd);
2444
2445 return true;
2446}
2447
2448static const struct test_spec tests[] = {
2449 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive},
2450 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame},
2451 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt},
2452 {.name = "POLL_RX", .test_func = testapp_poll_rx},
2453 {.name = "POLL_TX", .test_func = testapp_poll_tx},
2454 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout},
2455 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout},
2456 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned},
2457 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc},
2458 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame},
2459 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc},
2460 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE",
2461 .test_func = testapp_unaligned_inv_desc_4001_frame},
2462 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom},
2463 {.name = "TEARDOWN", .test_func = testapp_teardown},
2464 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional},
2465 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped},
2466 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs},
2467 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full},
2468 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty},
2469 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup},
2470 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop},
2471 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem},
2472 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata},
2473 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb},
2474 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb},
2475 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS",
2476 .test_func = testapp_send_receive_unaligned_mb},
2477 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb},
2478 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb},
2479 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags},
2480};
2481
2482static void print_tests(void)
2483{
2484 u32 i;
2485
2486 printf("Tests:\n");
2487 for (i = 0; i < ARRAY_SIZE(tests); i++)
2488 printf("%u: %s\n", i, tests[i].name);
2489}
2490
2491int main(int argc, char **argv)
2492{
2493 struct pkt_stream *rx_pkt_stream_default;
2494 struct pkt_stream *tx_pkt_stream_default;
2495 struct ifobject *ifobj_tx, *ifobj_rx;
2496 u32 i, j, failed_tests = 0, nb_tests;
2497 int modes = TEST_MODE_SKB + 1;
2498 struct test_spec test;
2499 bool shared_netdev;
2500
2501 /* Use libbpf 1.0 API mode */
2502 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2503
2504 ifobj_tx = ifobject_create();
2505 if (!ifobj_tx)
2506 exit_with_error(ENOMEM);
2507 ifobj_rx = ifobject_create();
2508 if (!ifobj_rx)
2509 exit_with_error(ENOMEM);
2510
2511 setlocale(LC_ALL, "");
2512
2513 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
2514
2515 if (opt_print_tests) {
2516 print_tests();
2517 ksft_exit_xpass();
2518 }
2519 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) {
2520 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);
2521 ksft_exit_xfail();
2522 }
2523
2524 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
2525 ifobj_tx->shared_umem = shared_netdev;
2526 ifobj_rx->shared_umem = shared_netdev;
2527
2528 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))
2529 print_usage(argv);
2530
2531 if (is_xdp_supported(ifobj_tx->ifindex)) {
2532 modes++;
2533 if (ifobj_zc_avail(ifobj_tx))
2534 modes++;
2535 }
2536
2537 init_iface(ifobj_rx, worker_testapp_validate_rx);
2538 init_iface(ifobj_tx, worker_testapp_validate_tx);
2539
2540 test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
2541 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2542 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2543 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
2544 exit_with_error(ENOMEM);
2545 test.tx_pkt_stream_default = tx_pkt_stream_default;
2546 test.rx_pkt_stream_default = rx_pkt_stream_default;
2547
2548 if (opt_run_test == RUN_ALL_TESTS)
2549 nb_tests = ARRAY_SIZE(tests);
2550 else
2551 nb_tests = 1;
2552 if (opt_mode == TEST_MODE_ALL) {
2553 ksft_set_plan(modes * nb_tests);
2554 } else {
2555 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {
2556 ksft_print_msg("Error: XDP_DRV mode not supported.\n");
2557 ksft_exit_xfail();
2558 }
2559 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {
2560 ksft_print_msg("Error: zero-copy mode not supported.\n");
2561 ksft_exit_xfail();
2562 }
2563
2564 ksft_set_plan(nb_tests);
2565 }
2566
2567 for (i = 0; i < modes; i++) {
2568 if (opt_mode != TEST_MODE_ALL && i != opt_mode)
2569 continue;
2570
2571 for (j = 0; j < ARRAY_SIZE(tests); j++) {
2572 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)
2573 continue;
2574
2575 test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);
2576 run_pkt_test(&test);
2577 usleep(USLEEP_MAX);
2578
2579 if (test.fail)
2580 failed_tests++;
2581 }
2582 }
2583
2584 pkt_stream_delete(tx_pkt_stream_default);
2585 pkt_stream_delete(rx_pkt_stream_default);
2586 xsk_unload_xdp_programs(ifobj_tx);
2587 xsk_unload_xdp_programs(ifobj_rx);
2588 ifobject_delete(ifobj_tx);
2589 ifobject_delete(ifobj_rx);
2590
2591 if (failed_tests)
2592 ksft_exit_fail();
2593 else
2594 ksft_exit_pass();
2595}