Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2020 Intel Corporation. */
3
4/*
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
7 * for use.
8 *
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
11 *
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
15 *
16 * Tests Information:
17 * ------------------
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
20 *
21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall
24 * c. Socket Teardown
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
30 * mode is used
31 * e. Statistics
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
34 * i. rx dropped
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
37 * ii. tx invalid
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
39 * size + 1).
40 * iii. rx ring full
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
48 * g. unaligned mode
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
52 *
53 * Total tests: 12
54 *
55 * Flow:
56 * -----
57 * - Single process spawns two threads: Tx and Rx
58 * - Each of these two threads attach to a veth interface within their assigned
59 * namespaces
60 * - Each thread Creates one AF_XDP socket connected to a unique umem for each
61 * veth interface
62 * - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy>
63 * - Rx thread verifies if all 10k packets were received and delivered in-order,
64 * and have the right content
65 *
66 * Enable/disable packet dump mode:
67 * --------------------------
68 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
69 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
70 */
71
72#define _GNU_SOURCE
73#include <fcntl.h>
74#include <errno.h>
75#include <getopt.h>
76#include <asm/barrier.h>
77#include <linux/if_link.h>
78#include <linux/if_ether.h>
79#include <linux/ip.h>
80#include <linux/udp.h>
81#include <arpa/inet.h>
82#include <net/if.h>
83#include <locale.h>
84#include <poll.h>
85#include <pthread.h>
86#include <signal.h>
87#include <stdbool.h>
88#include <stdio.h>
89#include <stdlib.h>
90#include <string.h>
91#include <stddef.h>
92#include <sys/mman.h>
93#include <sys/socket.h>
94#include <sys/time.h>
95#include <sys/types.h>
96#include <sys/queue.h>
97#include <time.h>
98#include <unistd.h>
99#include <stdatomic.h>
100#include "xsk.h"
101#include "xskxceiver.h"
102#include <bpf/bpf.h>
103#include <linux/filter.h>
104#include "../kselftest.h"
105
106/* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf.
107 * Until xskxceiver is either moved or re-writed into libxdp, suppress
108 * deprecation warnings in this file
109 */
110#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
111
112static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
113static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
114static const char *IP1 = "192.168.100.162";
115static const char *IP2 = "192.168.100.161";
116static const u16 UDP_PORT1 = 2020;
117static const u16 UDP_PORT2 = 2121;
118
119static void __exit_with_error(int error, const char *file, const char *func, int line)
120{
121 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
122 strerror(error));
123 ksft_exit_xfail();
124}
125
126#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
127#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
128static char *mode_string(struct test_spec *test)
129{
130 switch (test->mode) {
131 case TEST_MODE_SKB:
132 return "SKB";
133 case TEST_MODE_DRV:
134 return "DRV";
135 case TEST_MODE_ZC:
136 return "ZC";
137 default:
138 return "BOGUS";
139 }
140}
141
142static void report_failure(struct test_spec *test)
143{
144 if (test->fail)
145 return;
146
147 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
148 test->name);
149 test->fail = true;
150}
151
152static void memset32_htonl(void *dest, u32 val, u32 size)
153{
154 u32 *ptr = (u32 *)dest;
155 int i;
156
157 val = htonl(val);
158
159 for (i = 0; i < (size & (~0x3)); i += 4)
160 ptr[i >> 2] = val;
161}
162
163/*
164 * Fold a partial checksum
165 * This function code has been taken from
166 * Linux kernel include/asm-generic/checksum.h
167 */
168static __u16 csum_fold(__u32 csum)
169{
170 u32 sum = (__force u32)csum;
171
172 sum = (sum & 0xffff) + (sum >> 16);
173 sum = (sum & 0xffff) + (sum >> 16);
174 return (__force __u16)~sum;
175}
176
177/*
178 * This function code has been taken from
179 * Linux kernel lib/checksum.c
180 */
181static u32 from64to32(u64 x)
182{
183 /* add up 32-bit and 32-bit for 32+c bit */
184 x = (x & 0xffffffff) + (x >> 32);
185 /* add up carry.. */
186 x = (x & 0xffffffff) + (x >> 32);
187 return (u32)x;
188}
189
190/*
191 * This function code has been taken from
192 * Linux kernel lib/checksum.c
193 */
194static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
195{
196 unsigned long long s = (__force u32)sum;
197
198 s += (__force u32)saddr;
199 s += (__force u32)daddr;
200#ifdef __BIG_ENDIAN__
201 s += proto + len;
202#else
203 s += (proto + len) << 8;
204#endif
205 return (__force __u32)from64to32(s);
206}
207
208/*
209 * This function has been taken from
210 * Linux kernel include/asm-generic/checksum.h
211 */
212static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
213{
214 return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
215}
216
217static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
218{
219 u32 csum = 0;
220 u32 cnt = 0;
221
222 /* udp hdr and data */
223 for (; cnt < len; cnt += 2)
224 csum += udp_pkt[cnt >> 1];
225
226 return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
227}
228
229static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
230{
231 memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
232 memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
233 eth_hdr->h_proto = htons(ETH_P_IP);
234}
235
236static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)
237{
238 ip_hdr->version = IP_PKT_VER;
239 ip_hdr->ihl = 0x5;
240 ip_hdr->tos = IP_PKT_TOS;
241 ip_hdr->tot_len = htons(IP_PKT_SIZE);
242 ip_hdr->id = 0;
243 ip_hdr->frag_off = 0;
244 ip_hdr->ttl = IPDEFTTL;
245 ip_hdr->protocol = IPPROTO_UDP;
246 ip_hdr->saddr = ifobject->src_ip;
247 ip_hdr->daddr = ifobject->dst_ip;
248 ip_hdr->check = 0;
249}
250
251static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject,
252 struct udphdr *udp_hdr)
253{
254 udp_hdr->source = htons(ifobject->src_port);
255 udp_hdr->dest = htons(ifobject->dst_port);
256 udp_hdr->len = htons(UDP_PKT_SIZE);
257 memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE);
258}
259
260static bool is_umem_valid(struct ifobject *ifobj)
261{
262 return !!ifobj->umem->umem;
263}
264
265static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
266{
267 udp_hdr->check = 0;
268 udp_hdr->check =
269 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);
270}
271
272static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size)
273{
274 struct xsk_umem_config cfg = {
275 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
276 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
277 .frame_size = umem->frame_size,
278 .frame_headroom = umem->frame_headroom,
279 .flags = XSK_UMEM__DEFAULT_FLAGS
280 };
281 int ret;
282
283 if (umem->unaligned_mode)
284 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
285
286 ret = xsk_umem__create(&umem->umem, buffer, size,
287 &umem->fq, &umem->cq, &cfg);
288 if (ret)
289 return ret;
290
291 umem->buffer = buffer;
292 return 0;
293}
294
295static void enable_busy_poll(struct xsk_socket_info *xsk)
296{
297 int sock_opt;
298
299 sock_opt = 1;
300 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
301 (void *)&sock_opt, sizeof(sock_opt)) < 0)
302 exit_with_error(errno);
303
304 sock_opt = 20;
305 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
306 (void *)&sock_opt, sizeof(sock_opt)) < 0)
307 exit_with_error(errno);
308
309 sock_opt = BATCH_SIZE;
310 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
311 (void *)&sock_opt, sizeof(sock_opt)) < 0)
312 exit_with_error(errno);
313}
314
315static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
316 struct ifobject *ifobject, bool shared)
317{
318 struct xsk_socket_config cfg = {};
319 struct xsk_ring_cons *rxr;
320 struct xsk_ring_prod *txr;
321
322 xsk->umem = umem;
323 cfg.rx_size = xsk->rxqsize;
324 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
325 cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
326 cfg.xdp_flags = ifobject->xdp_flags;
327 cfg.bind_flags = ifobject->bind_flags;
328 if (shared)
329 cfg.bind_flags |= XDP_SHARED_UMEM;
330
331 txr = ifobject->tx_on ? &xsk->tx : NULL;
332 rxr = ifobject->rx_on ? &xsk->rx : NULL;
333 return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);
334}
335
336static bool ifobj_zc_avail(struct ifobject *ifobject)
337{
338 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
339 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
340 struct xsk_socket_info *xsk;
341 struct xsk_umem_info *umem;
342 bool zc_avail = false;
343 void *bufs;
344 int ret;
345
346 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
347 if (bufs == MAP_FAILED)
348 exit_with_error(errno);
349
350 umem = calloc(1, sizeof(struct xsk_umem_info));
351 if (!umem) {
352 munmap(bufs, umem_sz);
353 exit_with_error(-ENOMEM);
354 }
355 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
356 ret = xsk_configure_umem(umem, bufs, umem_sz);
357 if (ret)
358 exit_with_error(-ret);
359
360 xsk = calloc(1, sizeof(struct xsk_socket_info));
361 if (!xsk)
362 goto out;
363 ifobject->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
364 ifobject->xdp_flags |= XDP_FLAGS_DRV_MODE;
365 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
366 ifobject->rx_on = true;
367 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
368 ret = __xsk_configure_socket(xsk, umem, ifobject, false);
369 if (!ret)
370 zc_avail = true;
371
372 xsk_socket__delete(xsk->xsk);
373 free(xsk);
374out:
375 munmap(umem->buffer, umem_sz);
376 xsk_umem__delete(umem->umem);
377 free(umem);
378 return zc_avail;
379}
380
381static struct option long_options[] = {
382 {"interface", required_argument, 0, 'i'},
383 {"busy-poll", no_argument, 0, 'b'},
384 {"dump-pkts", no_argument, 0, 'D'},
385 {"verbose", no_argument, 0, 'v'},
386 {0, 0, 0, 0}
387};
388
389static void usage(const char *prog)
390{
391 const char *str =
392 " Usage: %s [OPTIONS]\n"
393 " Options:\n"
394 " -i, --interface Use interface\n"
395 " -D, --dump-pkts Dump packets L2 - L5\n"
396 " -v, --verbose Verbose output\n"
397 " -b, --busy-poll Enable busy poll\n";
398
399 ksft_print_msg(str, prog);
400}
401
402static int switch_namespace(const char *nsname)
403{
404 char fqns[26] = "/var/run/netns/";
405 int nsfd;
406
407 if (!nsname || strlen(nsname) == 0)
408 return -1;
409
410 strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1);
411 nsfd = open(fqns, O_RDONLY);
412
413 if (nsfd == -1)
414 exit_with_error(errno);
415
416 if (setns(nsfd, 0) == -1)
417 exit_with_error(errno);
418
419 print_verbose("NS switched: %s\n", nsname);
420
421 return nsfd;
422}
423
424static bool validate_interface(struct ifobject *ifobj)
425{
426 if (!strcmp(ifobj->ifname, ""))
427 return false;
428 return true;
429}
430
431static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
432 char **argv)
433{
434 struct ifobject *ifobj;
435 u32 interface_nb = 0;
436 int option_index, c;
437
438 opterr = 0;
439
440 for (;;) {
441 char *sptr, *token;
442
443 c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index);
444 if (c == -1)
445 break;
446
447 switch (c) {
448 case 'i':
449 if (interface_nb == 0)
450 ifobj = ifobj_tx;
451 else if (interface_nb == 1)
452 ifobj = ifobj_rx;
453 else
454 break;
455
456 sptr = strndupa(optarg, strlen(optarg));
457 memcpy(ifobj->ifname, strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS);
458 token = strsep(&sptr, ",");
459 if (token)
460 memcpy(ifobj->nsname, token, MAX_INTERFACES_NAMESPACE_CHARS);
461 interface_nb++;
462 break;
463 case 'D':
464 opt_pkt_dump = true;
465 break;
466 case 'v':
467 opt_verbose = true;
468 break;
469 case 'b':
470 ifobj_tx->busy_poll = true;
471 ifobj_rx->busy_poll = true;
472 break;
473 default:
474 usage(basename(argv[0]));
475 ksft_exit_xfail();
476 }
477 }
478}
479
480static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
481 struct ifobject *ifobj_rx)
482{
483 u32 i, j;
484
485 for (i = 0; i < MAX_INTERFACES; i++) {
486 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
487
488 ifobj->xsk = &ifobj->xsk_arr[0];
489 ifobj->use_poll = false;
490 ifobj->use_fill_ring = true;
491 ifobj->release_rx = true;
492 ifobj->validation_func = NULL;
493
494 if (i == 0) {
495 ifobj->rx_on = false;
496 ifobj->tx_on = true;
497 ifobj->pkt_stream = test->tx_pkt_stream_default;
498 } else {
499 ifobj->rx_on = true;
500 ifobj->tx_on = false;
501 ifobj->pkt_stream = test->rx_pkt_stream_default;
502 }
503
504 memset(ifobj->umem, 0, sizeof(*ifobj->umem));
505 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
506 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
507 if (ifobj->shared_umem && ifobj->rx_on)
508 ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS *
509 XSK_UMEM__DEFAULT_FRAME_SIZE;
510
511 for (j = 0; j < MAX_SOCKETS; j++) {
512 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
513 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
514 }
515 }
516
517 test->ifobj_tx = ifobj_tx;
518 test->ifobj_rx = ifobj_rx;
519 test->current_step = 0;
520 test->total_steps = 1;
521 test->nb_sockets = 1;
522 test->fail = false;
523}
524
525static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
526 struct ifobject *ifobj_rx, enum test_mode mode)
527{
528 struct pkt_stream *tx_pkt_stream;
529 struct pkt_stream *rx_pkt_stream;
530 u32 i;
531
532 tx_pkt_stream = test->tx_pkt_stream_default;
533 rx_pkt_stream = test->rx_pkt_stream_default;
534 memset(test, 0, sizeof(*test));
535 test->tx_pkt_stream_default = tx_pkt_stream;
536 test->rx_pkt_stream_default = rx_pkt_stream;
537
538 for (i = 0; i < MAX_INTERFACES; i++) {
539 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
540
541 ifobj->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
542 if (mode == TEST_MODE_SKB)
543 ifobj->xdp_flags |= XDP_FLAGS_SKB_MODE;
544 else
545 ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE;
546
547 ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
548 if (mode == TEST_MODE_ZC)
549 ifobj->bind_flags |= XDP_ZEROCOPY;
550 else
551 ifobj->bind_flags |= XDP_COPY;
552 }
553
554 test->mode = mode;
555 __test_spec_init(test, ifobj_tx, ifobj_rx);
556}
557
558static void test_spec_reset(struct test_spec *test)
559{
560 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
561}
562
563static void test_spec_set_name(struct test_spec *test, const char *name)
564{
565 strncpy(test->name, name, MAX_TEST_NAME_SIZE);
566}
567
568static void pkt_stream_reset(struct pkt_stream *pkt_stream)
569{
570 if (pkt_stream)
571 pkt_stream->rx_pkt_nb = 0;
572}
573
574static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb)
575{
576 if (pkt_nb >= pkt_stream->nb_pkts)
577 return NULL;
578
579 return &pkt_stream->pkts[pkt_nb];
580}
581
582static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
583{
584 while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) {
585 (*pkts_sent)++;
586 if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid)
587 return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++];
588 pkt_stream->rx_pkt_nb++;
589 }
590 return NULL;
591}
592
593static void pkt_stream_delete(struct pkt_stream *pkt_stream)
594{
595 free(pkt_stream->pkts);
596 free(pkt_stream);
597}
598
599static void pkt_stream_restore_default(struct test_spec *test)
600{
601 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
602 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
603
604 if (tx_pkt_stream != test->tx_pkt_stream_default) {
605 pkt_stream_delete(test->ifobj_tx->pkt_stream);
606 test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
607 }
608
609 if (rx_pkt_stream != test->rx_pkt_stream_default) {
610 pkt_stream_delete(test->ifobj_rx->pkt_stream);
611 test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
612 }
613}
614
615static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
616{
617 struct pkt_stream *pkt_stream;
618
619 pkt_stream = calloc(1, sizeof(*pkt_stream));
620 if (!pkt_stream)
621 return NULL;
622
623 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
624 if (!pkt_stream->pkts) {
625 free(pkt_stream);
626 return NULL;
627 }
628
629 pkt_stream->nb_pkts = nb_pkts;
630 return pkt_stream;
631}
632
633static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len)
634{
635 pkt->addr = addr + umem->base_addr;
636 pkt->len = len;
637 if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)
638 pkt->valid = false;
639 else
640 pkt->valid = true;
641}
642
643static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
644{
645 struct pkt_stream *pkt_stream;
646 u32 i;
647
648 pkt_stream = __pkt_stream_alloc(nb_pkts);
649 if (!pkt_stream)
650 exit_with_error(ENOMEM);
651
652 pkt_stream->nb_pkts = nb_pkts;
653 for (i = 0; i < nb_pkts; i++) {
654 pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size,
655 pkt_len);
656 pkt_stream->pkts[i].payload = i;
657 }
658
659 return pkt_stream;
660}
661
662static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
663 struct pkt_stream *pkt_stream)
664{
665 return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
666}
667
668static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
669{
670 struct pkt_stream *pkt_stream;
671
672 pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
673 test->ifobj_tx->pkt_stream = pkt_stream;
674 pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
675 test->ifobj_rx->pkt_stream = pkt_stream;
676}
677
678static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
679 int offset)
680{
681 struct xsk_umem_info *umem = ifobj->umem;
682 struct pkt_stream *pkt_stream;
683 u32 i;
684
685 pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
686 for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
687 pkt_set(umem, &pkt_stream->pkts[i],
688 (i % umem->num_frames) * umem->frame_size + offset, pkt_len);
689
690 ifobj->pkt_stream = pkt_stream;
691}
692
693static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
694{
695 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
696 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
697}
698
699static void pkt_stream_receive_half(struct test_spec *test)
700{
701 struct xsk_umem_info *umem = test->ifobj_rx->umem;
702 struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream;
703 u32 i;
704
705 test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts,
706 pkt_stream->pkts[0].len);
707 pkt_stream = test->ifobj_rx->pkt_stream;
708 for (i = 1; i < pkt_stream->nb_pkts; i += 2)
709 pkt_stream->pkts[i].valid = false;
710}
711
712static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)
713{
714 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);
715 struct udphdr *udp_hdr;
716 struct ethhdr *eth_hdr;
717 struct iphdr *ip_hdr;
718 void *data;
719
720 if (!pkt)
721 return NULL;
722 if (!pkt->valid || pkt->len < MIN_PKT_SIZE)
723 return pkt;
724
725 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);
726 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr));
727 ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr));
728 eth_hdr = (struct ethhdr *)data;
729
730 gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr);
731 gen_ip_hdr(ifobject, ip_hdr);
732 gen_udp_csum(udp_hdr, ip_hdr);
733 gen_eth_hdr(ifobject, eth_hdr);
734
735 return pkt;
736}
737
738static void __pkt_stream_generate_custom(struct ifobject *ifobj,
739 struct pkt *pkts, u32 nb_pkts)
740{
741 struct pkt_stream *pkt_stream;
742 u32 i;
743
744 pkt_stream = __pkt_stream_alloc(nb_pkts);
745 if (!pkt_stream)
746 exit_with_error(ENOMEM);
747
748 for (i = 0; i < nb_pkts; i++) {
749 pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr;
750 pkt_stream->pkts[i].len = pkts[i].len;
751 pkt_stream->pkts[i].payload = i;
752 pkt_stream->pkts[i].valid = pkts[i].valid;
753 }
754
755 ifobj->pkt_stream = pkt_stream;
756}
757
758static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
759{
760 __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts);
761 __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts);
762}
763
764static void pkt_dump(void *pkt, u32 len)
765{
766 char s[INET_ADDRSTRLEN];
767 struct ethhdr *ethhdr;
768 struct udphdr *udphdr;
769 struct iphdr *iphdr;
770 int payload, i;
771
772 ethhdr = pkt;
773 iphdr = pkt + sizeof(*ethhdr);
774 udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr);
775
776 /*extract L2 frame */
777 fprintf(stdout, "DEBUG>> L2: dst mac: ");
778 for (i = 0; i < ETH_ALEN; i++)
779 fprintf(stdout, "%02X", ethhdr->h_dest[i]);
780
781 fprintf(stdout, "\nDEBUG>> L2: src mac: ");
782 for (i = 0; i < ETH_ALEN; i++)
783 fprintf(stdout, "%02X", ethhdr->h_source[i]);
784
785 /*extract L3 frame */
786 fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);
787 fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",
788 inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));
789 fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",
790 inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));
791 /*extract L4 frame */
792 fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));
793 fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));
794 /*extract L5 frame */
795 payload = *((uint32_t *)(pkt + PKT_HDR_SIZE));
796
797 fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload);
798 fprintf(stdout, "---------------------------------------\n");
799}
800
801static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr,
802 u64 pkt_stream_addr)
803{
804 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
805 u32 offset = addr % umem->frame_size, expected_offset = 0;
806
807 if (!pkt_stream->use_addr_for_fill)
808 pkt_stream_addr = 0;
809
810 expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
811
812 if (offset == expected_offset)
813 return true;
814
815 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
816 return false;
817}
818
819static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
820{
821 void *data = xsk_umem__get_data(buffer, addr);
822 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr));
823
824 if (!pkt) {
825 ksft_print_msg("[%s] too many packets received\n", __func__);
826 return false;
827 }
828
829 if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) {
830 /* Do not try to verify packets that are smaller than minimum size. */
831 return true;
832 }
833
834 if (pkt->len != len) {
835 ksft_print_msg("[%s] expected length [%d], got length [%d]\n",
836 __func__, pkt->len, len);
837 return false;
838 }
839
840 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
841 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE)));
842
843 if (opt_pkt_dump)
844 pkt_dump(data, PKT_SIZE);
845
846 if (pkt->payload != seqnum) {
847 ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n",
848 __func__, pkt->payload, seqnum);
849 return false;
850 }
851 } else {
852 ksft_print_msg("Invalid frame received: ");
853 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,
854 iphdr->tos);
855 return false;
856 }
857
858 return true;
859}
860
861static void kick_tx(struct xsk_socket_info *xsk)
862{
863 int ret;
864
865 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
866 if (ret >= 0)
867 return;
868 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
869 usleep(100);
870 return;
871 }
872 exit_with_error(errno);
873}
874
875static void kick_rx(struct xsk_socket_info *xsk)
876{
877 int ret;
878
879 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
880 if (ret < 0)
881 exit_with_error(errno);
882}
883
884static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
885{
886 unsigned int rcvd;
887 u32 idx;
888
889 if (xsk_ring_prod__needs_wakeup(&xsk->tx))
890 kick_tx(xsk);
891
892 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
893 if (rcvd) {
894 if (rcvd > xsk->outstanding_tx) {
895 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
896
897 ksft_print_msg("[%s] Too many packets completed\n", __func__);
898 ksft_print_msg("Last completion address: %llx\n", addr);
899 return TEST_FAILURE;
900 }
901
902 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
903 xsk->outstanding_tx -= rcvd;
904 }
905
906 return TEST_PASS;
907}
908
909static int receive_pkts(struct test_spec *test, struct pollfd *fds)
910{
911 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
912 struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream;
913 u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0;
914 struct xsk_socket_info *xsk = test->ifobj_rx->xsk;
915 struct ifobject *ifobj = test->ifobj_rx;
916 struct xsk_umem_info *umem = xsk->umem;
917 struct pkt *pkt;
918 int ret;
919
920 ret = gettimeofday(&tv_now, NULL);
921 if (ret)
922 exit_with_error(errno);
923 timeradd(&tv_now, &tv_timeout, &tv_end);
924
925 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
926 while (pkt) {
927 ret = gettimeofday(&tv_now, NULL);
928 if (ret)
929 exit_with_error(errno);
930 if (timercmp(&tv_now, &tv_end, >)) {
931 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
932 return TEST_FAILURE;
933 }
934
935 kick_rx(xsk);
936 if (ifobj->use_poll) {
937 ret = poll(fds, 1, POLL_TMOUT);
938 if (ret < 0)
939 exit_with_error(-ret);
940
941 if (!ret) {
942 if (!is_umem_valid(test->ifobj_tx))
943 return TEST_PASS;
944
945 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
946 return TEST_FAILURE;
947
948 }
949
950 if (!(fds->revents & POLLIN))
951 continue;
952 }
953
954 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
955 if (!rcvd)
956 continue;
957
958 if (ifobj->use_fill_ring) {
959 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
960 while (ret != rcvd) {
961 if (ret < 0)
962 exit_with_error(-ret);
963 if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
964 ret = poll(fds, 1, POLL_TMOUT);
965 if (ret < 0)
966 exit_with_error(-ret);
967 }
968 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
969 }
970 }
971
972 for (i = 0; i < rcvd; i++) {
973 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
974 u64 addr = desc->addr, orig;
975
976 orig = xsk_umem__extract_addr(addr);
977 addr = xsk_umem__add_offset_to_addr(addr);
978
979 if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) ||
980 !is_offset_correct(umem, pkt_stream, addr, pkt->addr))
981 return TEST_FAILURE;
982
983 if (ifobj->use_fill_ring)
984 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
985 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
986 }
987
988 if (ifobj->use_fill_ring)
989 xsk_ring_prod__submit(&umem->fq, rcvd);
990 if (ifobj->release_rx)
991 xsk_ring_cons__release(&xsk->rx, rcvd);
992
993 pthread_mutex_lock(&pacing_mutex);
994 pkts_in_flight -= pkts_sent;
995 if (pkts_in_flight < umem->num_frames)
996 pthread_cond_signal(&pacing_cond);
997 pthread_mutex_unlock(&pacing_mutex);
998 pkts_sent = 0;
999 }
1000
1001 return TEST_PASS;
1002}
1003
1004static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds,
1005 bool timeout)
1006{
1007 struct xsk_socket_info *xsk = ifobject->xsk;
1008 bool use_poll = ifobject->use_poll;
1009 u32 i, idx = 0, valid_pkts = 0;
1010 int ret;
1011
1012 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
1013 if (use_poll) {
1014 ret = poll(fds, 1, POLL_TMOUT);
1015 if (timeout) {
1016 if (ret < 0) {
1017 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1018 __func__, ret);
1019 return TEST_FAILURE;
1020 }
1021 if (ret == 0)
1022 return TEST_PASS;
1023 break;
1024 }
1025 if (ret <= 0) {
1026 ksft_print_msg("ERROR: [%s] Poll error %d\n",
1027 __func__, ret);
1028 return TEST_FAILURE;
1029 }
1030 }
1031
1032 complete_pkts(xsk, BATCH_SIZE);
1033 }
1034
1035 for (i = 0; i < BATCH_SIZE; i++) {
1036 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
1037 struct pkt *pkt = pkt_generate(ifobject, *pkt_nb);
1038
1039 if (!pkt)
1040 break;
1041
1042 tx_desc->addr = pkt->addr;
1043 tx_desc->len = pkt->len;
1044 (*pkt_nb)++;
1045 if (pkt->valid)
1046 valid_pkts++;
1047 }
1048
1049 pthread_mutex_lock(&pacing_mutex);
1050 pkts_in_flight += valid_pkts;
1051 /* pkts_in_flight might be negative if many invalid packets are sent */
1052 if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) {
1053 kick_tx(xsk);
1054 pthread_cond_wait(&pacing_cond, &pacing_mutex);
1055 }
1056 pthread_mutex_unlock(&pacing_mutex);
1057
1058 xsk_ring_prod__submit(&xsk->tx, i);
1059 xsk->outstanding_tx += valid_pkts;
1060
1061 if (use_poll) {
1062 ret = poll(fds, 1, POLL_TMOUT);
1063 if (ret <= 0) {
1064 if (ret == 0 && timeout)
1065 return TEST_PASS;
1066
1067 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1068 return TEST_FAILURE;
1069 }
1070 }
1071
1072 if (!timeout) {
1073 if (complete_pkts(xsk, i))
1074 return TEST_FAILURE;
1075
1076 usleep(10);
1077 return TEST_PASS;
1078 }
1079
1080 return TEST_CONTINUE;
1081}
1082
1083static void wait_for_tx_completion(struct xsk_socket_info *xsk)
1084{
1085 while (xsk->outstanding_tx)
1086 complete_pkts(xsk, BATCH_SIZE);
1087}
1088
1089static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1090{
1091 bool timeout = !is_umem_valid(test->ifobj_rx);
1092 struct pollfd fds = { };
1093 u32 pkt_cnt = 0, ret;
1094
1095 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1096 fds.events = POLLOUT;
1097
1098 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {
1099 ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout);
1100 if ((ret || test->fail) && !timeout)
1101 return TEST_FAILURE;
1102 else if (ret == TEST_PASS && timeout)
1103 return ret;
1104 }
1105
1106 wait_for_tx_completion(ifobject->xsk);
1107 return TEST_PASS;
1108}
1109
1110static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1111{
1112 int fd = xsk_socket__fd(xsk), err;
1113 socklen_t optlen, expected_len;
1114
1115 optlen = sizeof(*stats);
1116 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1117 if (err) {
1118 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1119 __func__, -err, strerror(-err));
1120 return TEST_FAILURE;
1121 }
1122
1123 expected_len = sizeof(struct xdp_statistics);
1124 if (optlen != expected_len) {
1125 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1126 __func__, expected_len, optlen);
1127 return TEST_FAILURE;
1128 }
1129
1130 return TEST_PASS;
1131}
1132
1133static int validate_rx_dropped(struct ifobject *ifobject)
1134{
1135 struct xsk_socket *xsk = ifobject->xsk->xsk;
1136 struct xdp_statistics stats;
1137 int err;
1138
1139 kick_rx(ifobject->xsk);
1140
1141 err = get_xsk_stats(xsk, &stats);
1142 if (err)
1143 return TEST_FAILURE;
1144
1145 if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2)
1146 return TEST_PASS;
1147
1148 return TEST_FAILURE;
1149}
1150
1151static int validate_rx_full(struct ifobject *ifobject)
1152{
1153 struct xsk_socket *xsk = ifobject->xsk->xsk;
1154 struct xdp_statistics stats;
1155 int err;
1156
1157 usleep(1000);
1158 kick_rx(ifobject->xsk);
1159
1160 err = get_xsk_stats(xsk, &stats);
1161 if (err)
1162 return TEST_FAILURE;
1163
1164 if (stats.rx_ring_full)
1165 return TEST_PASS;
1166
1167 return TEST_FAILURE;
1168}
1169
1170static int validate_fill_empty(struct ifobject *ifobject)
1171{
1172 struct xsk_socket *xsk = ifobject->xsk->xsk;
1173 struct xdp_statistics stats;
1174 int err;
1175
1176 usleep(1000);
1177 kick_rx(ifobject->xsk);
1178
1179 err = get_xsk_stats(xsk, &stats);
1180 if (err)
1181 return TEST_FAILURE;
1182
1183 if (stats.rx_fill_ring_empty_descs)
1184 return TEST_PASS;
1185
1186 return TEST_FAILURE;
1187}
1188
1189static int validate_tx_invalid_descs(struct ifobject *ifobject)
1190{
1191 struct xsk_socket *xsk = ifobject->xsk->xsk;
1192 int fd = xsk_socket__fd(xsk);
1193 struct xdp_statistics stats;
1194 socklen_t optlen;
1195 int err;
1196
1197 optlen = sizeof(stats);
1198 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1199 if (err) {
1200 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1201 __func__, -err, strerror(-err));
1202 return TEST_FAILURE;
1203 }
1204
1205 if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {
1206 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n",
1207 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
1208 return TEST_FAILURE;
1209 }
1210
1211 return TEST_PASS;
1212}
1213
1214static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1215 struct xsk_umem_info *umem, bool tx)
1216{
1217 int i, ret;
1218
1219 for (i = 0; i < test->nb_sockets; i++) {
1220 bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1221 u32 ctr = 0;
1222
1223 while (ctr++ < SOCK_RECONF_CTR) {
1224 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1225 ifobject, shared);
1226 if (!ret)
1227 break;
1228
1229 /* Retry if it fails as xsk_socket__create() is asynchronous */
1230 if (ctr >= SOCK_RECONF_CTR)
1231 exit_with_error(-ret);
1232 usleep(USLEEP_MAX);
1233 }
1234 if (ifobject->busy_poll)
1235 enable_busy_poll(&ifobject->xsk_arr[i]);
1236 }
1237}
1238
1239static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1240{
1241 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1242 ifobject->xsk = &ifobject->xsk_arr[0];
1243 ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd;
1244 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1245}
1246
1247static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
1248{
1249 u32 idx = 0, i, buffers_to_fill;
1250 int ret;
1251
1252 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1253 buffers_to_fill = umem->num_frames;
1254 else
1255 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1256
1257 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1258 if (ret != buffers_to_fill)
1259 exit_with_error(ENOSPC);
1260 for (i = 0; i < buffers_to_fill; i++) {
1261 u64 addr;
1262
1263 if (pkt_stream->use_addr_for_fill) {
1264 struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
1265
1266 if (!pkt)
1267 break;
1268 addr = pkt->addr;
1269 } else {
1270 addr = i * umem->frame_size;
1271 }
1272
1273 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1274 }
1275 xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
1276}
1277
1278static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1279{
1280 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1281 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1282 LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1283 int ret, ifindex;
1284 void *bufs;
1285
1286 ifobject->ns_fd = switch_namespace(ifobject->nsname);
1287
1288 if (ifobject->umem->unaligned_mode)
1289 mmap_flags |= MAP_HUGETLB;
1290
1291 if (ifobject->shared_umem)
1292 umem_sz *= 2;
1293
1294 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1295 if (bufs == MAP_FAILED)
1296 exit_with_error(errno);
1297
1298 ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz);
1299 if (ret)
1300 exit_with_error(-ret);
1301
1302 xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
1303
1304 xsk_configure_socket(test, ifobject, ifobject->umem, false);
1305
1306 ifobject->xsk = &ifobject->xsk_arr[0];
1307
1308 if (!ifobject->rx_on)
1309 return;
1310
1311 ifindex = if_nametoindex(ifobject->ifname);
1312 if (!ifindex)
1313 exit_with_error(errno);
1314
1315 ret = xsk_setup_xdp_prog_xsk(ifobject->xsk->xsk, &ifobject->xsk_map_fd);
1316 if (ret)
1317 exit_with_error(-ret);
1318
1319 ret = bpf_xdp_query(ifindex, ifobject->xdp_flags, &opts);
1320 if (ret)
1321 exit_with_error(-ret);
1322
1323 if (ifobject->xdp_flags & XDP_FLAGS_SKB_MODE) {
1324 if (opts.attach_mode != XDP_ATTACHED_SKB) {
1325 ksft_print_msg("ERROR: [%s] XDP prog not in SKB mode\n");
1326 exit_with_error(-EINVAL);
1327 }
1328 } else if (ifobject->xdp_flags & XDP_FLAGS_DRV_MODE) {
1329 if (opts.attach_mode != XDP_ATTACHED_DRV) {
1330 ksft_print_msg("ERROR: [%s] XDP prog not in DRV mode\n");
1331 exit_with_error(-EINVAL);
1332 }
1333 }
1334
1335 ret = xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
1336 if (ret)
1337 exit_with_error(-ret);
1338}
1339
1340static void *worker_testapp_validate_tx(void *arg)
1341{
1342 struct test_spec *test = (struct test_spec *)arg;
1343 struct ifobject *ifobject = test->ifobj_tx;
1344 int err;
1345
1346 if (test->current_step == 1) {
1347 if (!ifobject->shared_umem)
1348 thread_common_ops(test, ifobject);
1349 else
1350 thread_common_ops_tx(test, ifobject);
1351 }
1352
1353 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
1354 ifobject->ifname);
1355 err = send_pkts(test, ifobject);
1356
1357 if (!err && ifobject->validation_func)
1358 err = ifobject->validation_func(ifobject);
1359 if (err)
1360 report_failure(test);
1361
1362 pthread_exit(NULL);
1363}
1364
1365static void *worker_testapp_validate_rx(void *arg)
1366{
1367 struct test_spec *test = (struct test_spec *)arg;
1368 struct ifobject *ifobject = test->ifobj_rx;
1369 struct pollfd fds = { };
1370 int id = 0;
1371 int err;
1372
1373 if (test->current_step == 1) {
1374 thread_common_ops(test, ifobject);
1375 } else {
1376 bpf_map_delete_elem(ifobject->xsk_map_fd, &id);
1377 xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
1378 }
1379
1380 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1381 fds.events = POLLIN;
1382
1383 pthread_barrier_wait(&barr);
1384
1385 err = receive_pkts(test, &fds);
1386
1387 if (!err && ifobject->validation_func)
1388 err = ifobject->validation_func(ifobject);
1389 if (err) {
1390 report_failure(test);
1391 pthread_mutex_lock(&pacing_mutex);
1392 pthread_cond_signal(&pacing_cond);
1393 pthread_mutex_unlock(&pacing_mutex);
1394 }
1395
1396 pthread_exit(NULL);
1397}
1398
1399static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1400{
1401 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1402
1403 if (ifobj->shared_umem)
1404 umem_sz *= 2;
1405
1406 xsk_umem__delete(ifobj->umem->umem);
1407 munmap(ifobj->umem->buffer, umem_sz);
1408}
1409
1410static void handler(int signum)
1411{
1412 pthread_exit(NULL);
1413}
1414
1415static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj,
1416 enum test_type type)
1417{
1418 bool old_shared_umem = ifobj->shared_umem;
1419 pthread_t t0;
1420
1421 if (pthread_barrier_init(&barr, NULL, 2))
1422 exit_with_error(errno);
1423
1424 test->current_step++;
1425 if (type == TEST_TYPE_POLL_RXQ_TMOUT)
1426 pkt_stream_reset(ifobj->pkt_stream);
1427 pkts_in_flight = 0;
1428
1429 test->ifobj_rx->shared_umem = false;
1430 test->ifobj_tx->shared_umem = false;
1431
1432 signal(SIGUSR1, handler);
1433 /* Spawn thread */
1434 pthread_create(&t0, NULL, ifobj->func_ptr, test);
1435
1436 if (type != TEST_TYPE_POLL_TXQ_TMOUT)
1437 pthread_barrier_wait(&barr);
1438
1439 if (pthread_barrier_destroy(&barr))
1440 exit_with_error(errno);
1441
1442 pthread_kill(t0, SIGUSR1);
1443 pthread_join(t0, NULL);
1444
1445 if (test->total_steps == test->current_step || test->fail) {
1446 xsk_socket__delete(ifobj->xsk->xsk);
1447 testapp_clean_xsk_umem(ifobj);
1448 }
1449
1450 test->ifobj_rx->shared_umem = old_shared_umem;
1451 test->ifobj_tx->shared_umem = old_shared_umem;
1452
1453 return !!test->fail;
1454}
1455
1456static int testapp_validate_traffic(struct test_spec *test)
1457{
1458 struct ifobject *ifobj_tx = test->ifobj_tx;
1459 struct ifobject *ifobj_rx = test->ifobj_rx;
1460 pthread_t t0, t1;
1461
1462 if (pthread_barrier_init(&barr, NULL, 2))
1463 exit_with_error(errno);
1464
1465 test->current_step++;
1466 pkt_stream_reset(ifobj_rx->pkt_stream);
1467 pkts_in_flight = 0;
1468
1469 /*Spawn RX thread */
1470 pthread_create(&t0, NULL, ifobj_rx->func_ptr, test);
1471
1472 pthread_barrier_wait(&barr);
1473 if (pthread_barrier_destroy(&barr))
1474 exit_with_error(errno);
1475
1476 /*Spawn TX thread */
1477 pthread_create(&t1, NULL, ifobj_tx->func_ptr, test);
1478
1479 pthread_join(t1, NULL);
1480 pthread_join(t0, NULL);
1481
1482 if (test->total_steps == test->current_step || test->fail) {
1483 xsk_socket__delete(ifobj_tx->xsk->xsk);
1484 xsk_socket__delete(ifobj_rx->xsk->xsk);
1485 testapp_clean_xsk_umem(ifobj_rx);
1486 if (!ifobj_tx->shared_umem)
1487 testapp_clean_xsk_umem(ifobj_tx);
1488 }
1489
1490 return !!test->fail;
1491}
1492
1493static void testapp_teardown(struct test_spec *test)
1494{
1495 int i;
1496
1497 test_spec_set_name(test, "TEARDOWN");
1498 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1499 if (testapp_validate_traffic(test))
1500 return;
1501 test_spec_reset(test);
1502 }
1503}
1504
1505static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1506{
1507 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1508 struct ifobject *tmp_ifobj = (*ifobj1);
1509
1510 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1511 (*ifobj2)->func_ptr = tmp_func_ptr;
1512
1513 *ifobj1 = *ifobj2;
1514 *ifobj2 = tmp_ifobj;
1515}
1516
1517static void testapp_bidi(struct test_spec *test)
1518{
1519 test_spec_set_name(test, "BIDIRECTIONAL");
1520 test->ifobj_tx->rx_on = true;
1521 test->ifobj_rx->tx_on = true;
1522 test->total_steps = 2;
1523 if (testapp_validate_traffic(test))
1524 return;
1525
1526 print_verbose("Switching Tx/Rx vectors\n");
1527 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1528 testapp_validate_traffic(test);
1529
1530 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1531}
1532
1533static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
1534{
1535 int ret;
1536
1537 xsk_socket__delete(ifobj_tx->xsk->xsk);
1538 xsk_socket__delete(ifobj_rx->xsk->xsk);
1539 ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
1540 ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
1541
1542 ret = xsk_socket__update_xskmap(ifobj_rx->xsk->xsk, ifobj_rx->xsk_map_fd);
1543 if (ret)
1544 exit_with_error(-ret);
1545}
1546
1547static void testapp_bpf_res(struct test_spec *test)
1548{
1549 test_spec_set_name(test, "BPF_RES");
1550 test->total_steps = 2;
1551 test->nb_sockets = 2;
1552 if (testapp_validate_traffic(test))
1553 return;
1554
1555 swap_xsk_resources(test->ifobj_tx, test->ifobj_rx);
1556 testapp_validate_traffic(test);
1557}
1558
1559static void testapp_headroom(struct test_spec *test)
1560{
1561 test_spec_set_name(test, "UMEM_HEADROOM");
1562 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1563 testapp_validate_traffic(test);
1564}
1565
1566static void testapp_stats_rx_dropped(struct test_spec *test)
1567{
1568 test_spec_set_name(test, "STAT_RX_DROPPED");
1569 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1570 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1571 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1572 pkt_stream_receive_half(test);
1573 test->ifobj_rx->validation_func = validate_rx_dropped;
1574 testapp_validate_traffic(test);
1575}
1576
1577static void testapp_stats_tx_invalid_descs(struct test_spec *test)
1578{
1579 test_spec_set_name(test, "STAT_TX_INVALID");
1580 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
1581 test->ifobj_tx->validation_func = validate_tx_invalid_descs;
1582 testapp_validate_traffic(test);
1583
1584 pkt_stream_restore_default(test);
1585}
1586
1587static void testapp_stats_rx_full(struct test_spec *test)
1588{
1589 test_spec_set_name(test, "STAT_RX_FULL");
1590 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1591 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1592 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1593 if (!test->ifobj_rx->pkt_stream)
1594 exit_with_error(ENOMEM);
1595
1596 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
1597 test->ifobj_rx->release_rx = false;
1598 test->ifobj_rx->validation_func = validate_rx_full;
1599 testapp_validate_traffic(test);
1600
1601 pkt_stream_restore_default(test);
1602}
1603
1604static void testapp_stats_fill_empty(struct test_spec *test)
1605{
1606 test_spec_set_name(test, "STAT_RX_FILL_EMPTY");
1607 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1608 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1609 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1610 if (!test->ifobj_rx->pkt_stream)
1611 exit_with_error(ENOMEM);
1612
1613 test->ifobj_rx->use_fill_ring = false;
1614 test->ifobj_rx->validation_func = validate_fill_empty;
1615 testapp_validate_traffic(test);
1616
1617 pkt_stream_restore_default(test);
1618}
1619
1620/* Simple test */
1621static bool hugepages_present(struct ifobject *ifobject)
1622{
1623 const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size;
1624 void *bufs;
1625
1626 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1627 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
1628 if (bufs == MAP_FAILED)
1629 return false;
1630
1631 munmap(bufs, mmap_sz);
1632 return true;
1633}
1634
1635static bool testapp_unaligned(struct test_spec *test)
1636{
1637 if (!hugepages_present(test->ifobj_tx)) {
1638 ksft_test_result_skip("No 2M huge pages present.\n");
1639 return false;
1640 }
1641
1642 test_spec_set_name(test, "UNALIGNED_MODE");
1643 test->ifobj_tx->umem->unaligned_mode = true;
1644 test->ifobj_rx->umem->unaligned_mode = true;
1645 /* Let half of the packets straddle a buffer boundrary */
1646 pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2);
1647 test->ifobj_rx->pkt_stream->use_addr_for_fill = true;
1648 testapp_validate_traffic(test);
1649
1650 pkt_stream_restore_default(test);
1651 return true;
1652}
1653
1654static void testapp_single_pkt(struct test_spec *test)
1655{
1656 struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}};
1657
1658 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1659 testapp_validate_traffic(test);
1660 pkt_stream_restore_default(test);
1661}
1662
1663static void testapp_invalid_desc(struct test_spec *test)
1664{
1665 struct pkt pkts[] = {
1666 /* Zero packet address allowed */
1667 {0, PKT_SIZE, 0, true},
1668 /* Allowed packet */
1669 {0x1000, PKT_SIZE, 0, true},
1670 /* Straddling the start of umem */
1671 {-2, PKT_SIZE, 0, false},
1672 /* Packet too large */
1673 {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
1674 /* After umem ends */
1675 {UMEM_SIZE, PKT_SIZE, 0, false},
1676 /* Straddle the end of umem */
1677 {UMEM_SIZE - PKT_SIZE / 2, PKT_SIZE, 0, false},
1678 /* Straddle a page boundrary */
1679 {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false},
1680 /* Straddle a 2K boundrary */
1681 {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true},
1682 /* Valid packet for synch so that something is received */
1683 {0x4000, PKT_SIZE, 0, true}};
1684
1685 if (test->ifobj_tx->umem->unaligned_mode) {
1686 /* Crossing a page boundrary allowed */
1687 pkts[6].valid = true;
1688 }
1689 if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
1690 /* Crossing a 2K frame size boundrary not allowed */
1691 pkts[7].valid = false;
1692 }
1693
1694 if (test->ifobj_tx->shared_umem) {
1695 pkts[4].addr += UMEM_SIZE;
1696 pkts[5].addr += UMEM_SIZE;
1697 }
1698
1699 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1700 testapp_validate_traffic(test);
1701 pkt_stream_restore_default(test);
1702}
1703
1704static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
1705 const char *dst_ip, const char *src_ip, const u16 dst_port,
1706 const u16 src_port, thread_func_t func_ptr)
1707{
1708 struct in_addr ip;
1709
1710 memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
1711 memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
1712
1713 inet_aton(dst_ip, &ip);
1714 ifobj->dst_ip = ip.s_addr;
1715
1716 inet_aton(src_ip, &ip);
1717 ifobj->src_ip = ip.s_addr;
1718
1719 ifobj->dst_port = dst_port;
1720 ifobj->src_port = src_port;
1721
1722 ifobj->func_ptr = func_ptr;
1723}
1724
1725static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type)
1726{
1727 switch (type) {
1728 case TEST_TYPE_STATS_RX_DROPPED:
1729 if (mode == TEST_MODE_ZC) {
1730 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
1731 return;
1732 }
1733 testapp_stats_rx_dropped(test);
1734 break;
1735 case TEST_TYPE_STATS_TX_INVALID_DESCS:
1736 testapp_stats_tx_invalid_descs(test);
1737 break;
1738 case TEST_TYPE_STATS_RX_FULL:
1739 testapp_stats_rx_full(test);
1740 break;
1741 case TEST_TYPE_STATS_FILL_EMPTY:
1742 testapp_stats_fill_empty(test);
1743 break;
1744 case TEST_TYPE_TEARDOWN:
1745 testapp_teardown(test);
1746 break;
1747 case TEST_TYPE_BIDI:
1748 testapp_bidi(test);
1749 break;
1750 case TEST_TYPE_BPF_RES:
1751 testapp_bpf_res(test);
1752 break;
1753 case TEST_TYPE_RUN_TO_COMPLETION:
1754 test_spec_set_name(test, "RUN_TO_COMPLETION");
1755 testapp_validate_traffic(test);
1756 break;
1757 case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT:
1758 test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT");
1759 testapp_single_pkt(test);
1760 break;
1761 case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME:
1762 test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE");
1763 test->ifobj_tx->umem->frame_size = 2048;
1764 test->ifobj_rx->umem->frame_size = 2048;
1765 pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE);
1766 testapp_validate_traffic(test);
1767
1768 pkt_stream_restore_default(test);
1769 break;
1770 case TEST_TYPE_RX_POLL:
1771 test->ifobj_rx->use_poll = true;
1772 test_spec_set_name(test, "POLL_RX");
1773 testapp_validate_traffic(test);
1774 break;
1775 case TEST_TYPE_TX_POLL:
1776 test->ifobj_tx->use_poll = true;
1777 test_spec_set_name(test, "POLL_TX");
1778 testapp_validate_traffic(test);
1779 break;
1780 case TEST_TYPE_POLL_TXQ_TMOUT:
1781 test_spec_set_name(test, "POLL_TXQ_FULL");
1782 test->ifobj_tx->use_poll = true;
1783 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */
1784 test->ifobj_tx->umem->frame_size = 2048;
1785 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
1786 testapp_validate_traffic_single_thread(test, test->ifobj_tx, type);
1787 pkt_stream_restore_default(test);
1788 break;
1789 case TEST_TYPE_POLL_RXQ_TMOUT:
1790 test_spec_set_name(test, "POLL_RXQ_EMPTY");
1791 test->ifobj_rx->use_poll = true;
1792 testapp_validate_traffic_single_thread(test, test->ifobj_rx, type);
1793 break;
1794 case TEST_TYPE_ALIGNED_INV_DESC:
1795 test_spec_set_name(test, "ALIGNED_INV_DESC");
1796 testapp_invalid_desc(test);
1797 break;
1798 case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME:
1799 test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE");
1800 test->ifobj_tx->umem->frame_size = 2048;
1801 test->ifobj_rx->umem->frame_size = 2048;
1802 testapp_invalid_desc(test);
1803 break;
1804 case TEST_TYPE_UNALIGNED_INV_DESC:
1805 if (!hugepages_present(test->ifobj_tx)) {
1806 ksft_test_result_skip("No 2M huge pages present.\n");
1807 return;
1808 }
1809 test_spec_set_name(test, "UNALIGNED_INV_DESC");
1810 test->ifobj_tx->umem->unaligned_mode = true;
1811 test->ifobj_rx->umem->unaligned_mode = true;
1812 testapp_invalid_desc(test);
1813 break;
1814 case TEST_TYPE_UNALIGNED:
1815 if (!testapp_unaligned(test))
1816 return;
1817 break;
1818 case TEST_TYPE_HEADROOM:
1819 testapp_headroom(test);
1820 break;
1821 default:
1822 break;
1823 }
1824
1825 if (!test->fail)
1826 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
1827 test->name);
1828}
1829
1830static struct ifobject *ifobject_create(void)
1831{
1832 struct ifobject *ifobj;
1833
1834 ifobj = calloc(1, sizeof(struct ifobject));
1835 if (!ifobj)
1836 return NULL;
1837
1838 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
1839 if (!ifobj->xsk_arr)
1840 goto out_xsk_arr;
1841
1842 ifobj->umem = calloc(1, sizeof(*ifobj->umem));
1843 if (!ifobj->umem)
1844 goto out_umem;
1845
1846 ifobj->ns_fd = -1;
1847
1848 return ifobj;
1849
1850out_umem:
1851 free(ifobj->xsk_arr);
1852out_xsk_arr:
1853 free(ifobj);
1854 return NULL;
1855}
1856
1857static void ifobject_delete(struct ifobject *ifobj)
1858{
1859 if (ifobj->ns_fd != -1)
1860 close(ifobj->ns_fd);
1861 free(ifobj->umem);
1862 free(ifobj->xsk_arr);
1863 free(ifobj);
1864}
1865
1866static bool is_xdp_supported(struct ifobject *ifobject)
1867{
1868 int flags = XDP_FLAGS_DRV_MODE;
1869
1870 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
1871 struct bpf_insn insns[2] = {
1872 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
1873 BPF_EXIT_INSN()
1874 };
1875 int ifindex = if_nametoindex(ifobject->ifname);
1876 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
1877 int err;
1878
1879 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
1880 if (prog_fd < 0)
1881 return false;
1882
1883 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
1884 if (err) {
1885 close(prog_fd);
1886 return false;
1887 }
1888
1889 bpf_xdp_detach(ifindex, flags, NULL);
1890 close(prog_fd);
1891
1892 return true;
1893}
1894
1895int main(int argc, char **argv)
1896{
1897 struct pkt_stream *rx_pkt_stream_default;
1898 struct pkt_stream *tx_pkt_stream_default;
1899 struct ifobject *ifobj_tx, *ifobj_rx;
1900 int modes = TEST_MODE_SKB + 1;
1901 u32 i, j, failed_tests = 0;
1902 struct test_spec test;
1903 bool shared_umem;
1904
1905 /* Use libbpf 1.0 API mode */
1906 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1907
1908 ifobj_tx = ifobject_create();
1909 if (!ifobj_tx)
1910 exit_with_error(ENOMEM);
1911 ifobj_rx = ifobject_create();
1912 if (!ifobj_rx)
1913 exit_with_error(ENOMEM);
1914
1915 setlocale(LC_ALL, "");
1916
1917 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
1918 shared_umem = !strcmp(ifobj_tx->ifname, ifobj_rx->ifname);
1919
1920 ifobj_tx->shared_umem = shared_umem;
1921 ifobj_rx->shared_umem = shared_umem;
1922
1923 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
1924 usage(basename(argv[0]));
1925 ksft_exit_xfail();
1926 }
1927
1928 init_iface(ifobj_tx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2,
1929 worker_testapp_validate_tx);
1930 init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
1931 worker_testapp_validate_rx);
1932
1933 if (is_xdp_supported(ifobj_tx)) {
1934 modes++;
1935 if (ifobj_zc_avail(ifobj_tx))
1936 modes++;
1937 }
1938
1939 test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
1940 tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1941 rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1942 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
1943 exit_with_error(ENOMEM);
1944 test.tx_pkt_stream_default = tx_pkt_stream_default;
1945 test.rx_pkt_stream_default = rx_pkt_stream_default;
1946
1947 ksft_set_plan(modes * TEST_TYPE_MAX);
1948
1949 for (i = 0; i < modes; i++)
1950 for (j = 0; j < TEST_TYPE_MAX; j++) {
1951 test_spec_init(&test, ifobj_tx, ifobj_rx, i);
1952 run_pkt_test(&test, i, j);
1953 usleep(USLEEP_MAX);
1954
1955 if (test.fail)
1956 failed_tests++;
1957 }
1958
1959 pkt_stream_delete(tx_pkt_stream_default);
1960 pkt_stream_delete(rx_pkt_stream_default);
1961 ifobject_delete(ifobj_tx);
1962 ifobject_delete(ifobj_rx);
1963
1964 if (failed_tests)
1965 ksft_exit_fail();
1966 else
1967 ksft_exit_pass();
1968}