Loading...
1/*
2 * Test functionality of BPF filters for SO_REUSEPORT. The tests below will use
3 * a BPF program (both classic and extended) to read the first word from an
4 * incoming packet (expected to be in network byte-order), calculate a modulus
5 * of that number, and then dispatch the packet to the Nth socket using the
6 * result. These tests are run for each supported address family and protocol.
7 * Additionally, a few edge cases in the implementation are tested.
8 */
9
10#include <errno.h>
11#include <error.h>
12#include <fcntl.h>
13#include <linux/bpf.h>
14#include <linux/filter.h>
15#include <linux/unistd.h>
16#include <netinet/in.h>
17#include <netinet/tcp.h>
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <sys/epoll.h>
22#include <sys/types.h>
23#include <sys/socket.h>
24#include <unistd.h>
25
26#ifndef ARRAY_SIZE
27#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
28#endif
29
30struct test_params {
31 int recv_family;
32 int send_family;
33 int protocol;
34 size_t recv_socks;
35 uint16_t recv_port;
36 uint16_t send_port_min;
37};
38
39static size_t sockaddr_size(void)
40{
41 return sizeof(struct sockaddr_storage);
42}
43
44static struct sockaddr *new_any_sockaddr(int family, uint16_t port)
45{
46 struct sockaddr_storage *addr;
47 struct sockaddr_in *addr4;
48 struct sockaddr_in6 *addr6;
49
50 addr = malloc(sizeof(struct sockaddr_storage));
51 memset(addr, 0, sizeof(struct sockaddr_storage));
52
53 switch (family) {
54 case AF_INET:
55 addr4 = (struct sockaddr_in *)addr;
56 addr4->sin_family = AF_INET;
57 addr4->sin_addr.s_addr = htonl(INADDR_ANY);
58 addr4->sin_port = htons(port);
59 break;
60 case AF_INET6:
61 addr6 = (struct sockaddr_in6 *)addr;
62 addr6->sin6_family = AF_INET6;
63 addr6->sin6_addr = in6addr_any;
64 addr6->sin6_port = htons(port);
65 break;
66 default:
67 error(1, 0, "Unsupported family %d", family);
68 }
69 return (struct sockaddr *)addr;
70}
71
72static struct sockaddr *new_loopback_sockaddr(int family, uint16_t port)
73{
74 struct sockaddr *addr = new_any_sockaddr(family, port);
75 struct sockaddr_in *addr4;
76 struct sockaddr_in6 *addr6;
77
78 switch (family) {
79 case AF_INET:
80 addr4 = (struct sockaddr_in *)addr;
81 addr4->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
82 break;
83 case AF_INET6:
84 addr6 = (struct sockaddr_in6 *)addr;
85 addr6->sin6_addr = in6addr_loopback;
86 break;
87 default:
88 error(1, 0, "Unsupported family %d", family);
89 }
90 return addr;
91}
92
93static void attach_ebpf(int fd, uint16_t mod)
94{
95 static char bpf_log_buf[65536];
96 static const char bpf_license[] = "GPL";
97
98 int bpf_fd;
99 const struct bpf_insn prog[] = {
100 /* BPF_MOV64_REG(BPF_REG_6, BPF_REG_1) */
101 { BPF_ALU64 | BPF_MOV | BPF_X, BPF_REG_6, BPF_REG_1, 0, 0 },
102 /* BPF_LD_ABS(BPF_W, 0) R0 = (uint32_t)skb[0] */
103 { BPF_LD | BPF_ABS | BPF_W, 0, 0, 0, 0 },
104 /* BPF_ALU64_IMM(BPF_MOD, BPF_REG_0, mod) */
105 { BPF_ALU64 | BPF_MOD | BPF_K, BPF_REG_0, 0, 0, mod },
106 /* BPF_EXIT_INSN() */
107 { BPF_JMP | BPF_EXIT, 0, 0, 0, 0 }
108 };
109 union bpf_attr attr;
110
111 memset(&attr, 0, sizeof(attr));
112 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
113 attr.insn_cnt = ARRAY_SIZE(prog);
114 attr.insns = (unsigned long) &prog;
115 attr.license = (unsigned long) &bpf_license;
116 attr.log_buf = (unsigned long) &bpf_log_buf;
117 attr.log_size = sizeof(bpf_log_buf);
118 attr.log_level = 1;
119 attr.kern_version = 0;
120
121 bpf_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
122 if (bpf_fd < 0)
123 error(1, errno, "ebpf error. log:\n%s\n", bpf_log_buf);
124
125 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &bpf_fd,
126 sizeof(bpf_fd)))
127 error(1, errno, "failed to set SO_ATTACH_REUSEPORT_EBPF");
128
129 close(bpf_fd);
130}
131
132static void attach_cbpf(int fd, uint16_t mod)
133{
134 struct sock_filter code[] = {
135 /* A = (uint32_t)skb[0] */
136 { BPF_LD | BPF_W | BPF_ABS, 0, 0, 0 },
137 /* A = A % mod */
138 { BPF_ALU | BPF_MOD, 0, 0, mod },
139 /* return A */
140 { BPF_RET | BPF_A, 0, 0, 0 },
141 };
142 struct sock_fprog p = {
143 .len = ARRAY_SIZE(code),
144 .filter = code,
145 };
146
147 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, &p, sizeof(p)))
148 error(1, errno, "failed to set SO_ATTACH_REUSEPORT_CBPF");
149}
150
151static void build_recv_group(const struct test_params p, int fd[], uint16_t mod,
152 void (*attach_bpf)(int, uint16_t))
153{
154 struct sockaddr * const addr =
155 new_any_sockaddr(p.recv_family, p.recv_port);
156 int i, opt;
157
158 for (i = 0; i < p.recv_socks; ++i) {
159 fd[i] = socket(p.recv_family, p.protocol, 0);
160 if (fd[i] < 0)
161 error(1, errno, "failed to create recv %d", i);
162
163 opt = 1;
164 if (setsockopt(fd[i], SOL_SOCKET, SO_REUSEPORT, &opt,
165 sizeof(opt)))
166 error(1, errno, "failed to set SO_REUSEPORT on %d", i);
167
168 if (i == 0)
169 attach_bpf(fd[i], mod);
170
171 if (bind(fd[i], addr, sockaddr_size()))
172 error(1, errno, "failed to bind recv socket %d", i);
173
174 if (p.protocol == SOCK_STREAM) {
175 opt = 4;
176 if (setsockopt(fd[i], SOL_TCP, TCP_FASTOPEN, &opt,
177 sizeof(opt)))
178 error(1, errno,
179 "failed to set TCP_FASTOPEN on %d", i);
180 if (listen(fd[i], p.recv_socks * 10))
181 error(1, errno, "failed to listen on socket");
182 }
183 }
184 free(addr);
185}
186
187static void send_from(struct test_params p, uint16_t sport, char *buf,
188 size_t len)
189{
190 struct sockaddr * const saddr = new_any_sockaddr(p.send_family, sport);
191 struct sockaddr * const daddr =
192 new_loopback_sockaddr(p.send_family, p.recv_port);
193 const int fd = socket(p.send_family, p.protocol, 0);
194
195 if (fd < 0)
196 error(1, errno, "failed to create send socket");
197
198 if (bind(fd, saddr, sockaddr_size()))
199 error(1, errno, "failed to bind send socket");
200
201 if (sendto(fd, buf, len, MSG_FASTOPEN, daddr, sockaddr_size()) < 0)
202 error(1, errno, "failed to send message");
203
204 close(fd);
205 free(saddr);
206 free(daddr);
207}
208
209static void test_recv_order(const struct test_params p, int fd[], int mod)
210{
211 char recv_buf[8], send_buf[8];
212 struct msghdr msg;
213 struct iovec recv_io = { recv_buf, 8 };
214 struct epoll_event ev;
215 int epfd, conn, i, sport, expected;
216 uint32_t data, ndata;
217
218 epfd = epoll_create(1);
219 if (epfd < 0)
220 error(1, errno, "failed to create epoll");
221 for (i = 0; i < p.recv_socks; ++i) {
222 ev.events = EPOLLIN;
223 ev.data.fd = fd[i];
224 if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd[i], &ev))
225 error(1, errno, "failed to register sock %d epoll", i);
226 }
227
228 memset(&msg, 0, sizeof(msg));
229 msg.msg_iov = &recv_io;
230 msg.msg_iovlen = 1;
231
232 for (data = 0; data < p.recv_socks * 2; ++data) {
233 sport = p.send_port_min + data;
234 ndata = htonl(data);
235 memcpy(send_buf, &ndata, sizeof(ndata));
236 send_from(p, sport, send_buf, sizeof(ndata));
237
238 i = epoll_wait(epfd, &ev, 1, -1);
239 if (i < 0)
240 error(1, errno, "epoll wait failed");
241
242 if (p.protocol == SOCK_STREAM) {
243 conn = accept(ev.data.fd, NULL, NULL);
244 if (conn < 0)
245 error(1, errno, "error accepting");
246 i = recvmsg(conn, &msg, 0);
247 close(conn);
248 } else {
249 i = recvmsg(ev.data.fd, &msg, 0);
250 }
251 if (i < 0)
252 error(1, errno, "recvmsg error");
253 if (i != sizeof(ndata))
254 error(1, 0, "expected size %zd got %d",
255 sizeof(ndata), i);
256
257 for (i = 0; i < p.recv_socks; ++i)
258 if (ev.data.fd == fd[i])
259 break;
260 memcpy(&ndata, recv_buf, sizeof(ndata));
261 fprintf(stderr, "Socket %d: %d\n", i, ntohl(ndata));
262
263 expected = (sport % mod);
264 if (i != expected)
265 error(1, 0, "expected socket %d", expected);
266 }
267}
268
269static void test_reuseport_ebpf(struct test_params p)
270{
271 int i, fd[p.recv_socks];
272
273 fprintf(stderr, "Testing EBPF mod %zd...\n", p.recv_socks);
274 build_recv_group(p, fd, p.recv_socks, attach_ebpf);
275 test_recv_order(p, fd, p.recv_socks);
276
277 p.send_port_min += p.recv_socks * 2;
278 fprintf(stderr, "Reprograming, testing mod %zd...\n", p.recv_socks / 2);
279 attach_ebpf(fd[0], p.recv_socks / 2);
280 test_recv_order(p, fd, p.recv_socks / 2);
281
282 for (i = 0; i < p.recv_socks; ++i)
283 close(fd[i]);
284}
285
286static void test_reuseport_cbpf(struct test_params p)
287{
288 int i, fd[p.recv_socks];
289
290 fprintf(stderr, "Testing CBPF mod %zd...\n", p.recv_socks);
291 build_recv_group(p, fd, p.recv_socks, attach_cbpf);
292 test_recv_order(p, fd, p.recv_socks);
293
294 p.send_port_min += p.recv_socks * 2;
295 fprintf(stderr, "Reprograming, testing mod %zd...\n", p.recv_socks / 2);
296 attach_cbpf(fd[0], p.recv_socks / 2);
297 test_recv_order(p, fd, p.recv_socks / 2);
298
299 for (i = 0; i < p.recv_socks; ++i)
300 close(fd[i]);
301}
302
303static void test_extra_filter(const struct test_params p)
304{
305 struct sockaddr * const addr =
306 new_any_sockaddr(p.recv_family, p.recv_port);
307 int fd1, fd2, opt;
308
309 fprintf(stderr, "Testing too many filters...\n");
310 fd1 = socket(p.recv_family, p.protocol, 0);
311 if (fd1 < 0)
312 error(1, errno, "failed to create socket 1");
313 fd2 = socket(p.recv_family, p.protocol, 0);
314 if (fd2 < 0)
315 error(1, errno, "failed to create socket 2");
316
317 opt = 1;
318 if (setsockopt(fd1, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
319 error(1, errno, "failed to set SO_REUSEPORT on socket 1");
320 if (setsockopt(fd2, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
321 error(1, errno, "failed to set SO_REUSEPORT on socket 2");
322
323 attach_ebpf(fd1, 10);
324 attach_ebpf(fd2, 10);
325
326 if (bind(fd1, addr, sockaddr_size()))
327 error(1, errno, "failed to bind recv socket 1");
328
329 if (!bind(fd2, addr, sockaddr_size()) && errno != EADDRINUSE)
330 error(1, errno, "bind socket 2 should fail with EADDRINUSE");
331
332 free(addr);
333}
334
335static void test_filter_no_reuseport(const struct test_params p)
336{
337 struct sockaddr * const addr =
338 new_any_sockaddr(p.recv_family, p.recv_port);
339 const char bpf_license[] = "GPL";
340 struct bpf_insn ecode[] = {
341 { BPF_ALU64 | BPF_MOV | BPF_K, BPF_REG_0, 0, 0, 10 },
342 { BPF_JMP | BPF_EXIT, 0, 0, 0, 0 }
343 };
344 struct sock_filter ccode[] = {{ BPF_RET | BPF_A, 0, 0, 0 }};
345 union bpf_attr eprog;
346 struct sock_fprog cprog;
347 int fd, bpf_fd;
348
349 fprintf(stderr, "Testing filters on non-SO_REUSEPORT socket...\n");
350
351 memset(&eprog, 0, sizeof(eprog));
352 eprog.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
353 eprog.insn_cnt = ARRAY_SIZE(ecode);
354 eprog.insns = (unsigned long) &ecode;
355 eprog.license = (unsigned long) &bpf_license;
356 eprog.kern_version = 0;
357
358 memset(&cprog, 0, sizeof(cprog));
359 cprog.len = ARRAY_SIZE(ccode);
360 cprog.filter = ccode;
361
362
363 bpf_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &eprog, sizeof(eprog));
364 if (bpf_fd < 0)
365 error(1, errno, "ebpf error");
366 fd = socket(p.recv_family, p.protocol, 0);
367 if (fd < 0)
368 error(1, errno, "failed to create socket 1");
369
370 if (bind(fd, addr, sockaddr_size()))
371 error(1, errno, "failed to bind recv socket 1");
372
373 errno = 0;
374 if (!setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &bpf_fd,
375 sizeof(bpf_fd)) || errno != EINVAL)
376 error(1, errno, "setsockopt should have returned EINVAL");
377
378 errno = 0;
379 if (!setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, &cprog,
380 sizeof(cprog)) || errno != EINVAL)
381 error(1, errno, "setsockopt should have returned EINVAL");
382
383 free(addr);
384}
385
386static void test_filter_without_bind(void)
387{
388 int fd1, fd2, opt = 1;
389
390 fprintf(stderr, "Testing filter add without bind...\n");
391 fd1 = socket(AF_INET, SOCK_DGRAM, 0);
392 if (fd1 < 0)
393 error(1, errno, "failed to create socket 1");
394 fd2 = socket(AF_INET, SOCK_DGRAM, 0);
395 if (fd2 < 0)
396 error(1, errno, "failed to create socket 2");
397 if (setsockopt(fd1, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
398 error(1, errno, "failed to set SO_REUSEPORT on socket 1");
399 if (setsockopt(fd2, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
400 error(1, errno, "failed to set SO_REUSEPORT on socket 2");
401
402 attach_ebpf(fd1, 10);
403 attach_cbpf(fd2, 10);
404
405 close(fd1);
406 close(fd2);
407}
408
409void enable_fastopen(void)
410{
411 int fd = open("/proc/sys/net/ipv4/tcp_fastopen", 0);
412 int rw_mask = 3; /* bit 1: client side; bit-2 server side */
413 int val, size;
414 char buf[16];
415
416 if (fd < 0)
417 error(1, errno, "Unable to open tcp_fastopen sysctl");
418 if (read(fd, buf, sizeof(buf)) <= 0)
419 error(1, errno, "Unable to read tcp_fastopen sysctl");
420 val = atoi(buf);
421 close(fd);
422
423 if ((val & rw_mask) != rw_mask) {
424 fd = open("/proc/sys/net/ipv4/tcp_fastopen", O_RDWR);
425 if (fd < 0)
426 error(1, errno,
427 "Unable to open tcp_fastopen sysctl for writing");
428 val |= rw_mask;
429 size = snprintf(buf, 16, "%d", val);
430 if (write(fd, buf, size) <= 0)
431 error(1, errno, "Unable to write tcp_fastopen sysctl");
432 close(fd);
433 }
434}
435
436int main(void)
437{
438 fprintf(stderr, "---- IPv4 UDP ----\n");
439 /* NOTE: UDP socket lookups traverse a different code path when there
440 * are > 10 sockets in a group. Run the bpf test through both paths.
441 */
442 test_reuseport_ebpf((struct test_params) {
443 .recv_family = AF_INET,
444 .send_family = AF_INET,
445 .protocol = SOCK_DGRAM,
446 .recv_socks = 10,
447 .recv_port = 8000,
448 .send_port_min = 9000});
449 test_reuseport_ebpf((struct test_params) {
450 .recv_family = AF_INET,
451 .send_family = AF_INET,
452 .protocol = SOCK_DGRAM,
453 .recv_socks = 20,
454 .recv_port = 8000,
455 .send_port_min = 9000});
456 test_reuseport_cbpf((struct test_params) {
457 .recv_family = AF_INET,
458 .send_family = AF_INET,
459 .protocol = SOCK_DGRAM,
460 .recv_socks = 10,
461 .recv_port = 8001,
462 .send_port_min = 9020});
463 test_reuseport_cbpf((struct test_params) {
464 .recv_family = AF_INET,
465 .send_family = AF_INET,
466 .protocol = SOCK_DGRAM,
467 .recv_socks = 20,
468 .recv_port = 8001,
469 .send_port_min = 9020});
470 test_extra_filter((struct test_params) {
471 .recv_family = AF_INET,
472 .protocol = SOCK_DGRAM,
473 .recv_port = 8002});
474 test_filter_no_reuseport((struct test_params) {
475 .recv_family = AF_INET,
476 .protocol = SOCK_DGRAM,
477 .recv_port = 8008});
478
479 fprintf(stderr, "---- IPv6 UDP ----\n");
480 test_reuseport_ebpf((struct test_params) {
481 .recv_family = AF_INET6,
482 .send_family = AF_INET6,
483 .protocol = SOCK_DGRAM,
484 .recv_socks = 10,
485 .recv_port = 8003,
486 .send_port_min = 9040});
487 test_reuseport_ebpf((struct test_params) {
488 .recv_family = AF_INET6,
489 .send_family = AF_INET6,
490 .protocol = SOCK_DGRAM,
491 .recv_socks = 20,
492 .recv_port = 8003,
493 .send_port_min = 9040});
494 test_reuseport_cbpf((struct test_params) {
495 .recv_family = AF_INET6,
496 .send_family = AF_INET6,
497 .protocol = SOCK_DGRAM,
498 .recv_socks = 10,
499 .recv_port = 8004,
500 .send_port_min = 9060});
501 test_reuseport_cbpf((struct test_params) {
502 .recv_family = AF_INET6,
503 .send_family = AF_INET6,
504 .protocol = SOCK_DGRAM,
505 .recv_socks = 20,
506 .recv_port = 8004,
507 .send_port_min = 9060});
508 test_extra_filter((struct test_params) {
509 .recv_family = AF_INET6,
510 .protocol = SOCK_DGRAM,
511 .recv_port = 8005});
512 test_filter_no_reuseport((struct test_params) {
513 .recv_family = AF_INET6,
514 .protocol = SOCK_DGRAM,
515 .recv_port = 8009});
516
517 fprintf(stderr, "---- IPv6 UDP w/ mapped IPv4 ----\n");
518 test_reuseport_ebpf((struct test_params) {
519 .recv_family = AF_INET6,
520 .send_family = AF_INET,
521 .protocol = SOCK_DGRAM,
522 .recv_socks = 20,
523 .recv_port = 8006,
524 .send_port_min = 9080});
525 test_reuseport_ebpf((struct test_params) {
526 .recv_family = AF_INET6,
527 .send_family = AF_INET,
528 .protocol = SOCK_DGRAM,
529 .recv_socks = 10,
530 .recv_port = 8006,
531 .send_port_min = 9080});
532 test_reuseport_cbpf((struct test_params) {
533 .recv_family = AF_INET6,
534 .send_family = AF_INET,
535 .protocol = SOCK_DGRAM,
536 .recv_socks = 10,
537 .recv_port = 8007,
538 .send_port_min = 9100});
539 test_reuseport_cbpf((struct test_params) {
540 .recv_family = AF_INET6,
541 .send_family = AF_INET,
542 .protocol = SOCK_DGRAM,
543 .recv_socks = 20,
544 .recv_port = 8007,
545 .send_port_min = 9100});
546
547 /* TCP fastopen is required for the TCP tests */
548 enable_fastopen();
549 fprintf(stderr, "---- IPv4 TCP ----\n");
550 test_reuseport_ebpf((struct test_params) {
551 .recv_family = AF_INET,
552 .send_family = AF_INET,
553 .protocol = SOCK_STREAM,
554 .recv_socks = 10,
555 .recv_port = 8008,
556 .send_port_min = 9120});
557 test_reuseport_cbpf((struct test_params) {
558 .recv_family = AF_INET,
559 .send_family = AF_INET,
560 .protocol = SOCK_STREAM,
561 .recv_socks = 10,
562 .recv_port = 8009,
563 .send_port_min = 9160});
564 test_extra_filter((struct test_params) {
565 .recv_family = AF_INET,
566 .protocol = SOCK_STREAM,
567 .recv_port = 8010});
568 test_filter_no_reuseport((struct test_params) {
569 .recv_family = AF_INET,
570 .protocol = SOCK_STREAM,
571 .recv_port = 8011});
572
573 fprintf(stderr, "---- IPv6 TCP ----\n");
574 test_reuseport_ebpf((struct test_params) {
575 .recv_family = AF_INET6,
576 .send_family = AF_INET6,
577 .protocol = SOCK_STREAM,
578 .recv_socks = 10,
579 .recv_port = 8012,
580 .send_port_min = 9200});
581 test_reuseport_cbpf((struct test_params) {
582 .recv_family = AF_INET6,
583 .send_family = AF_INET6,
584 .protocol = SOCK_STREAM,
585 .recv_socks = 10,
586 .recv_port = 8013,
587 .send_port_min = 9240});
588 test_extra_filter((struct test_params) {
589 .recv_family = AF_INET6,
590 .protocol = SOCK_STREAM,
591 .recv_port = 8014});
592 test_filter_no_reuseport((struct test_params) {
593 .recv_family = AF_INET6,
594 .protocol = SOCK_STREAM,
595 .recv_port = 8015});
596
597 fprintf(stderr, "---- IPv6 TCP w/ mapped IPv4 ----\n");
598 test_reuseport_ebpf((struct test_params) {
599 .recv_family = AF_INET6,
600 .send_family = AF_INET,
601 .protocol = SOCK_STREAM,
602 .recv_socks = 10,
603 .recv_port = 8016,
604 .send_port_min = 9320});
605 test_reuseport_cbpf((struct test_params) {
606 .recv_family = AF_INET6,
607 .send_family = AF_INET,
608 .protocol = SOCK_STREAM,
609 .recv_socks = 10,
610 .recv_port = 8017,
611 .send_port_min = 9360});
612
613 test_filter_without_bind();
614
615 fprintf(stderr, "SUCCESS\n");
616 return 0;
617}
1/*
2 * Test functionality of BPF filters for SO_REUSEPORT. The tests below will use
3 * a BPF program (both classic and extended) to read the first word from an
4 * incoming packet (expected to be in network byte-order), calculate a modulus
5 * of that number, and then dispatch the packet to the Nth socket using the
6 * result. These tests are run for each supported address family and protocol.
7 * Additionally, a few edge cases in the implementation are tested.
8 */
9
10#include <errno.h>
11#include <error.h>
12#include <fcntl.h>
13#include <linux/bpf.h>
14#include <linux/filter.h>
15#include <linux/unistd.h>
16#include <netinet/in.h>
17#include <netinet/tcp.h>
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <sys/epoll.h>
22#include <sys/types.h>
23#include <sys/socket.h>
24#include <sys/resource.h>
25#include <unistd.h>
26
27#ifndef ARRAY_SIZE
28#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
29#endif
30
31struct test_params {
32 int recv_family;
33 int send_family;
34 int protocol;
35 size_t recv_socks;
36 uint16_t recv_port;
37 uint16_t send_port_min;
38};
39
40static size_t sockaddr_size(void)
41{
42 return sizeof(struct sockaddr_storage);
43}
44
45static struct sockaddr *new_any_sockaddr(int family, uint16_t port)
46{
47 struct sockaddr_storage *addr;
48 struct sockaddr_in *addr4;
49 struct sockaddr_in6 *addr6;
50
51 addr = malloc(sizeof(struct sockaddr_storage));
52 memset(addr, 0, sizeof(struct sockaddr_storage));
53
54 switch (family) {
55 case AF_INET:
56 addr4 = (struct sockaddr_in *)addr;
57 addr4->sin_family = AF_INET;
58 addr4->sin_addr.s_addr = htonl(INADDR_ANY);
59 addr4->sin_port = htons(port);
60 break;
61 case AF_INET6:
62 addr6 = (struct sockaddr_in6 *)addr;
63 addr6->sin6_family = AF_INET6;
64 addr6->sin6_addr = in6addr_any;
65 addr6->sin6_port = htons(port);
66 break;
67 default:
68 error(1, 0, "Unsupported family %d", family);
69 }
70 return (struct sockaddr *)addr;
71}
72
73static struct sockaddr *new_loopback_sockaddr(int family, uint16_t port)
74{
75 struct sockaddr *addr = new_any_sockaddr(family, port);
76 struct sockaddr_in *addr4;
77 struct sockaddr_in6 *addr6;
78
79 switch (family) {
80 case AF_INET:
81 addr4 = (struct sockaddr_in *)addr;
82 addr4->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
83 break;
84 case AF_INET6:
85 addr6 = (struct sockaddr_in6 *)addr;
86 addr6->sin6_addr = in6addr_loopback;
87 break;
88 default:
89 error(1, 0, "Unsupported family %d", family);
90 }
91 return addr;
92}
93
94static void attach_ebpf(int fd, uint16_t mod)
95{
96 static char bpf_log_buf[65536];
97 static const char bpf_license[] = "GPL";
98
99 int bpf_fd;
100 const struct bpf_insn prog[] = {
101 /* BPF_MOV64_REG(BPF_REG_6, BPF_REG_1) */
102 { BPF_ALU64 | BPF_MOV | BPF_X, BPF_REG_6, BPF_REG_1, 0, 0 },
103 /* BPF_LD_ABS(BPF_W, 0) R0 = (uint32_t)skb[0] */
104 { BPF_LD | BPF_ABS | BPF_W, 0, 0, 0, 0 },
105 /* BPF_ALU64_IMM(BPF_MOD, BPF_REG_0, mod) */
106 { BPF_ALU64 | BPF_MOD | BPF_K, BPF_REG_0, 0, 0, mod },
107 /* BPF_EXIT_INSN() */
108 { BPF_JMP | BPF_EXIT, 0, 0, 0, 0 }
109 };
110 union bpf_attr attr;
111
112 memset(&attr, 0, sizeof(attr));
113 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
114 attr.insn_cnt = ARRAY_SIZE(prog);
115 attr.insns = (unsigned long) &prog;
116 attr.license = (unsigned long) &bpf_license;
117 attr.log_buf = (unsigned long) &bpf_log_buf;
118 attr.log_size = sizeof(bpf_log_buf);
119 attr.log_level = 1;
120 attr.kern_version = 0;
121
122 bpf_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
123 if (bpf_fd < 0)
124 error(1, errno, "ebpf error. log:\n%s\n", bpf_log_buf);
125
126 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &bpf_fd,
127 sizeof(bpf_fd)))
128 error(1, errno, "failed to set SO_ATTACH_REUSEPORT_EBPF");
129
130 close(bpf_fd);
131}
132
133static void attach_cbpf(int fd, uint16_t mod)
134{
135 struct sock_filter code[] = {
136 /* A = (uint32_t)skb[0] */
137 { BPF_LD | BPF_W | BPF_ABS, 0, 0, 0 },
138 /* A = A % mod */
139 { BPF_ALU | BPF_MOD, 0, 0, mod },
140 /* return A */
141 { BPF_RET | BPF_A, 0, 0, 0 },
142 };
143 struct sock_fprog p = {
144 .len = ARRAY_SIZE(code),
145 .filter = code,
146 };
147
148 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, &p, sizeof(p)))
149 error(1, errno, "failed to set SO_ATTACH_REUSEPORT_CBPF");
150}
151
152static void build_recv_group(const struct test_params p, int fd[], uint16_t mod,
153 void (*attach_bpf)(int, uint16_t))
154{
155 struct sockaddr * const addr =
156 new_any_sockaddr(p.recv_family, p.recv_port);
157 int i, opt;
158
159 for (i = 0; i < p.recv_socks; ++i) {
160 fd[i] = socket(p.recv_family, p.protocol, 0);
161 if (fd[i] < 0)
162 error(1, errno, "failed to create recv %d", i);
163
164 opt = 1;
165 if (setsockopt(fd[i], SOL_SOCKET, SO_REUSEPORT, &opt,
166 sizeof(opt)))
167 error(1, errno, "failed to set SO_REUSEPORT on %d", i);
168
169 if (i == 0)
170 attach_bpf(fd[i], mod);
171
172 if (bind(fd[i], addr, sockaddr_size()))
173 error(1, errno, "failed to bind recv socket %d", i);
174
175 if (p.protocol == SOCK_STREAM) {
176 opt = 4;
177 if (setsockopt(fd[i], SOL_TCP, TCP_FASTOPEN, &opt,
178 sizeof(opt)))
179 error(1, errno,
180 "failed to set TCP_FASTOPEN on %d", i);
181 if (listen(fd[i], p.recv_socks * 10))
182 error(1, errno, "failed to listen on socket");
183 }
184 }
185 free(addr);
186}
187
188static void send_from(struct test_params p, uint16_t sport, char *buf,
189 size_t len)
190{
191 struct sockaddr * const saddr = new_any_sockaddr(p.send_family, sport);
192 struct sockaddr * const daddr =
193 new_loopback_sockaddr(p.send_family, p.recv_port);
194 const int fd = socket(p.send_family, p.protocol, 0), one = 1;
195
196 if (fd < 0)
197 error(1, errno, "failed to create send socket");
198
199 if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)))
200 error(1, errno, "failed to set reuseaddr");
201
202 if (bind(fd, saddr, sockaddr_size()))
203 error(1, errno, "failed to bind send socket");
204
205 if (sendto(fd, buf, len, MSG_FASTOPEN, daddr, sockaddr_size()) < 0)
206 error(1, errno, "failed to send message");
207
208 close(fd);
209 free(saddr);
210 free(daddr);
211}
212
213static void test_recv_order(const struct test_params p, int fd[], int mod)
214{
215 char recv_buf[8], send_buf[8];
216 struct msghdr msg;
217 struct iovec recv_io = { recv_buf, 8 };
218 struct epoll_event ev;
219 int epfd, conn, i, sport, expected;
220 uint32_t data, ndata;
221
222 epfd = epoll_create(1);
223 if (epfd < 0)
224 error(1, errno, "failed to create epoll");
225 for (i = 0; i < p.recv_socks; ++i) {
226 ev.events = EPOLLIN;
227 ev.data.fd = fd[i];
228 if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd[i], &ev))
229 error(1, errno, "failed to register sock %d epoll", i);
230 }
231
232 memset(&msg, 0, sizeof(msg));
233 msg.msg_iov = &recv_io;
234 msg.msg_iovlen = 1;
235
236 for (data = 0; data < p.recv_socks * 2; ++data) {
237 sport = p.send_port_min + data;
238 ndata = htonl(data);
239 memcpy(send_buf, &ndata, sizeof(ndata));
240 send_from(p, sport, send_buf, sizeof(ndata));
241
242 i = epoll_wait(epfd, &ev, 1, -1);
243 if (i < 0)
244 error(1, errno, "epoll wait failed");
245
246 if (p.protocol == SOCK_STREAM) {
247 conn = accept(ev.data.fd, NULL, NULL);
248 if (conn < 0)
249 error(1, errno, "error accepting");
250 i = recvmsg(conn, &msg, 0);
251 close(conn);
252 } else {
253 i = recvmsg(ev.data.fd, &msg, 0);
254 }
255 if (i < 0)
256 error(1, errno, "recvmsg error");
257 if (i != sizeof(ndata))
258 error(1, 0, "expected size %zd got %d",
259 sizeof(ndata), i);
260
261 for (i = 0; i < p.recv_socks; ++i)
262 if (ev.data.fd == fd[i])
263 break;
264 memcpy(&ndata, recv_buf, sizeof(ndata));
265 fprintf(stderr, "Socket %d: %d\n", i, ntohl(ndata));
266
267 expected = (sport % mod);
268 if (i != expected)
269 error(1, 0, "expected socket %d", expected);
270 }
271}
272
273static void test_reuseport_ebpf(struct test_params p)
274{
275 int i, fd[p.recv_socks];
276
277 fprintf(stderr, "Testing EBPF mod %zd...\n", p.recv_socks);
278 build_recv_group(p, fd, p.recv_socks, attach_ebpf);
279 test_recv_order(p, fd, p.recv_socks);
280
281 p.send_port_min += p.recv_socks * 2;
282 fprintf(stderr, "Reprograming, testing mod %zd...\n", p.recv_socks / 2);
283 attach_ebpf(fd[0], p.recv_socks / 2);
284 test_recv_order(p, fd, p.recv_socks / 2);
285
286 for (i = 0; i < p.recv_socks; ++i)
287 close(fd[i]);
288}
289
290static void test_reuseport_cbpf(struct test_params p)
291{
292 int i, fd[p.recv_socks];
293
294 fprintf(stderr, "Testing CBPF mod %zd...\n", p.recv_socks);
295 build_recv_group(p, fd, p.recv_socks, attach_cbpf);
296 test_recv_order(p, fd, p.recv_socks);
297
298 p.send_port_min += p.recv_socks * 2;
299 fprintf(stderr, "Reprograming, testing mod %zd...\n", p.recv_socks / 2);
300 attach_cbpf(fd[0], p.recv_socks / 2);
301 test_recv_order(p, fd, p.recv_socks / 2);
302
303 for (i = 0; i < p.recv_socks; ++i)
304 close(fd[i]);
305}
306
307static void test_extra_filter(const struct test_params p)
308{
309 struct sockaddr * const addr =
310 new_any_sockaddr(p.recv_family, p.recv_port);
311 int fd1, fd2, opt;
312
313 fprintf(stderr, "Testing too many filters...\n");
314 fd1 = socket(p.recv_family, p.protocol, 0);
315 if (fd1 < 0)
316 error(1, errno, "failed to create socket 1");
317 fd2 = socket(p.recv_family, p.protocol, 0);
318 if (fd2 < 0)
319 error(1, errno, "failed to create socket 2");
320
321 opt = 1;
322 if (setsockopt(fd1, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
323 error(1, errno, "failed to set SO_REUSEPORT on socket 1");
324 if (setsockopt(fd2, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
325 error(1, errno, "failed to set SO_REUSEPORT on socket 2");
326
327 attach_ebpf(fd1, 10);
328 attach_ebpf(fd2, 10);
329
330 if (bind(fd1, addr, sockaddr_size()))
331 error(1, errno, "failed to bind recv socket 1");
332
333 if (!bind(fd2, addr, sockaddr_size()) && errno != EADDRINUSE)
334 error(1, errno, "bind socket 2 should fail with EADDRINUSE");
335
336 free(addr);
337}
338
339static void test_filter_no_reuseport(const struct test_params p)
340{
341 struct sockaddr * const addr =
342 new_any_sockaddr(p.recv_family, p.recv_port);
343 const char bpf_license[] = "GPL";
344 struct bpf_insn ecode[] = {
345 { BPF_ALU64 | BPF_MOV | BPF_K, BPF_REG_0, 0, 0, 10 },
346 { BPF_JMP | BPF_EXIT, 0, 0, 0, 0 }
347 };
348 struct sock_filter ccode[] = {{ BPF_RET | BPF_A, 0, 0, 0 }};
349 union bpf_attr eprog;
350 struct sock_fprog cprog;
351 int fd, bpf_fd;
352
353 fprintf(stderr, "Testing filters on non-SO_REUSEPORT socket...\n");
354
355 memset(&eprog, 0, sizeof(eprog));
356 eprog.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
357 eprog.insn_cnt = ARRAY_SIZE(ecode);
358 eprog.insns = (unsigned long) &ecode;
359 eprog.license = (unsigned long) &bpf_license;
360 eprog.kern_version = 0;
361
362 memset(&cprog, 0, sizeof(cprog));
363 cprog.len = ARRAY_SIZE(ccode);
364 cprog.filter = ccode;
365
366
367 bpf_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &eprog, sizeof(eprog));
368 if (bpf_fd < 0)
369 error(1, errno, "ebpf error");
370 fd = socket(p.recv_family, p.protocol, 0);
371 if (fd < 0)
372 error(1, errno, "failed to create socket 1");
373
374 if (bind(fd, addr, sockaddr_size()))
375 error(1, errno, "failed to bind recv socket 1");
376
377 errno = 0;
378 if (!setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &bpf_fd,
379 sizeof(bpf_fd)) || errno != EINVAL)
380 error(1, errno, "setsockopt should have returned EINVAL");
381
382 errno = 0;
383 if (!setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, &cprog,
384 sizeof(cprog)) || errno != EINVAL)
385 error(1, errno, "setsockopt should have returned EINVAL");
386
387 free(addr);
388}
389
390static void test_filter_without_bind(void)
391{
392 int fd1, fd2, opt = 1;
393
394 fprintf(stderr, "Testing filter add without bind...\n");
395 fd1 = socket(AF_INET, SOCK_DGRAM, 0);
396 if (fd1 < 0)
397 error(1, errno, "failed to create socket 1");
398 fd2 = socket(AF_INET, SOCK_DGRAM, 0);
399 if (fd2 < 0)
400 error(1, errno, "failed to create socket 2");
401 if (setsockopt(fd1, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
402 error(1, errno, "failed to set SO_REUSEPORT on socket 1");
403 if (setsockopt(fd2, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)))
404 error(1, errno, "failed to set SO_REUSEPORT on socket 2");
405
406 attach_ebpf(fd1, 10);
407 attach_cbpf(fd2, 10);
408
409 close(fd1);
410 close(fd2);
411}
412
413void enable_fastopen(void)
414{
415 int fd = open("/proc/sys/net/ipv4/tcp_fastopen", 0);
416 int rw_mask = 3; /* bit 1: client side; bit-2 server side */
417 int val, size;
418 char buf[16];
419
420 if (fd < 0)
421 error(1, errno, "Unable to open tcp_fastopen sysctl");
422 if (read(fd, buf, sizeof(buf)) <= 0)
423 error(1, errno, "Unable to read tcp_fastopen sysctl");
424 val = atoi(buf);
425 close(fd);
426
427 if ((val & rw_mask) != rw_mask) {
428 fd = open("/proc/sys/net/ipv4/tcp_fastopen", O_RDWR);
429 if (fd < 0)
430 error(1, errno,
431 "Unable to open tcp_fastopen sysctl for writing");
432 val |= rw_mask;
433 size = snprintf(buf, 16, "%d", val);
434 if (write(fd, buf, size) <= 0)
435 error(1, errno, "Unable to write tcp_fastopen sysctl");
436 close(fd);
437 }
438}
439
440static struct rlimit rlim_old;
441
442static __attribute__((constructor)) void main_ctor(void)
443{
444 getrlimit(RLIMIT_MEMLOCK, &rlim_old);
445
446 if (rlim_old.rlim_cur != RLIM_INFINITY) {
447 struct rlimit rlim_new;
448
449 rlim_new.rlim_cur = rlim_old.rlim_cur + (1UL << 20);
450 rlim_new.rlim_max = rlim_old.rlim_max + (1UL << 20);
451 setrlimit(RLIMIT_MEMLOCK, &rlim_new);
452 }
453}
454
455static __attribute__((destructor)) void main_dtor(void)
456{
457 setrlimit(RLIMIT_MEMLOCK, &rlim_old);
458}
459
460int main(void)
461{
462 fprintf(stderr, "---- IPv4 UDP ----\n");
463 /* NOTE: UDP socket lookups traverse a different code path when there
464 * are > 10 sockets in a group. Run the bpf test through both paths.
465 */
466 test_reuseport_ebpf((struct test_params) {
467 .recv_family = AF_INET,
468 .send_family = AF_INET,
469 .protocol = SOCK_DGRAM,
470 .recv_socks = 10,
471 .recv_port = 8000,
472 .send_port_min = 9000});
473 test_reuseport_ebpf((struct test_params) {
474 .recv_family = AF_INET,
475 .send_family = AF_INET,
476 .protocol = SOCK_DGRAM,
477 .recv_socks = 20,
478 .recv_port = 8000,
479 .send_port_min = 9000});
480 test_reuseport_cbpf((struct test_params) {
481 .recv_family = AF_INET,
482 .send_family = AF_INET,
483 .protocol = SOCK_DGRAM,
484 .recv_socks = 10,
485 .recv_port = 8001,
486 .send_port_min = 9020});
487 test_reuseport_cbpf((struct test_params) {
488 .recv_family = AF_INET,
489 .send_family = AF_INET,
490 .protocol = SOCK_DGRAM,
491 .recv_socks = 20,
492 .recv_port = 8001,
493 .send_port_min = 9020});
494 test_extra_filter((struct test_params) {
495 .recv_family = AF_INET,
496 .protocol = SOCK_DGRAM,
497 .recv_port = 8002});
498 test_filter_no_reuseport((struct test_params) {
499 .recv_family = AF_INET,
500 .protocol = SOCK_DGRAM,
501 .recv_port = 8008});
502
503 fprintf(stderr, "---- IPv6 UDP ----\n");
504 test_reuseport_ebpf((struct test_params) {
505 .recv_family = AF_INET6,
506 .send_family = AF_INET6,
507 .protocol = SOCK_DGRAM,
508 .recv_socks = 10,
509 .recv_port = 8003,
510 .send_port_min = 9040});
511 test_reuseport_ebpf((struct test_params) {
512 .recv_family = AF_INET6,
513 .send_family = AF_INET6,
514 .protocol = SOCK_DGRAM,
515 .recv_socks = 20,
516 .recv_port = 8003,
517 .send_port_min = 9040});
518 test_reuseport_cbpf((struct test_params) {
519 .recv_family = AF_INET6,
520 .send_family = AF_INET6,
521 .protocol = SOCK_DGRAM,
522 .recv_socks = 10,
523 .recv_port = 8004,
524 .send_port_min = 9060});
525 test_reuseport_cbpf((struct test_params) {
526 .recv_family = AF_INET6,
527 .send_family = AF_INET6,
528 .protocol = SOCK_DGRAM,
529 .recv_socks = 20,
530 .recv_port = 8004,
531 .send_port_min = 9060});
532 test_extra_filter((struct test_params) {
533 .recv_family = AF_INET6,
534 .protocol = SOCK_DGRAM,
535 .recv_port = 8005});
536 test_filter_no_reuseport((struct test_params) {
537 .recv_family = AF_INET6,
538 .protocol = SOCK_DGRAM,
539 .recv_port = 8009});
540
541 fprintf(stderr, "---- IPv6 UDP w/ mapped IPv4 ----\n");
542 test_reuseport_ebpf((struct test_params) {
543 .recv_family = AF_INET6,
544 .send_family = AF_INET,
545 .protocol = SOCK_DGRAM,
546 .recv_socks = 20,
547 .recv_port = 8006,
548 .send_port_min = 9080});
549 test_reuseport_ebpf((struct test_params) {
550 .recv_family = AF_INET6,
551 .send_family = AF_INET,
552 .protocol = SOCK_DGRAM,
553 .recv_socks = 10,
554 .recv_port = 8006,
555 .send_port_min = 9080});
556 test_reuseport_cbpf((struct test_params) {
557 .recv_family = AF_INET6,
558 .send_family = AF_INET,
559 .protocol = SOCK_DGRAM,
560 .recv_socks = 10,
561 .recv_port = 8007,
562 .send_port_min = 9100});
563 test_reuseport_cbpf((struct test_params) {
564 .recv_family = AF_INET6,
565 .send_family = AF_INET,
566 .protocol = SOCK_DGRAM,
567 .recv_socks = 20,
568 .recv_port = 8007,
569 .send_port_min = 9100});
570
571 /* TCP fastopen is required for the TCP tests */
572 enable_fastopen();
573 fprintf(stderr, "---- IPv4 TCP ----\n");
574 test_reuseport_ebpf((struct test_params) {
575 .recv_family = AF_INET,
576 .send_family = AF_INET,
577 .protocol = SOCK_STREAM,
578 .recv_socks = 10,
579 .recv_port = 8008,
580 .send_port_min = 9120});
581 test_reuseport_cbpf((struct test_params) {
582 .recv_family = AF_INET,
583 .send_family = AF_INET,
584 .protocol = SOCK_STREAM,
585 .recv_socks = 10,
586 .recv_port = 8009,
587 .send_port_min = 9160});
588 test_extra_filter((struct test_params) {
589 .recv_family = AF_INET,
590 .protocol = SOCK_STREAM,
591 .recv_port = 8010});
592 test_filter_no_reuseport((struct test_params) {
593 .recv_family = AF_INET,
594 .protocol = SOCK_STREAM,
595 .recv_port = 8011});
596
597 fprintf(stderr, "---- IPv6 TCP ----\n");
598 test_reuseport_ebpf((struct test_params) {
599 .recv_family = AF_INET6,
600 .send_family = AF_INET6,
601 .protocol = SOCK_STREAM,
602 .recv_socks = 10,
603 .recv_port = 8012,
604 .send_port_min = 9200});
605 test_reuseport_cbpf((struct test_params) {
606 .recv_family = AF_INET6,
607 .send_family = AF_INET6,
608 .protocol = SOCK_STREAM,
609 .recv_socks = 10,
610 .recv_port = 8013,
611 .send_port_min = 9240});
612 test_extra_filter((struct test_params) {
613 .recv_family = AF_INET6,
614 .protocol = SOCK_STREAM,
615 .recv_port = 8014});
616 test_filter_no_reuseport((struct test_params) {
617 .recv_family = AF_INET6,
618 .protocol = SOCK_STREAM,
619 .recv_port = 8015});
620
621 fprintf(stderr, "---- IPv6 TCP w/ mapped IPv4 ----\n");
622 test_reuseport_ebpf((struct test_params) {
623 .recv_family = AF_INET6,
624 .send_family = AF_INET,
625 .protocol = SOCK_STREAM,
626 .recv_socks = 10,
627 .recv_port = 8016,
628 .send_port_min = 9320});
629 test_reuseport_cbpf((struct test_params) {
630 .recv_family = AF_INET6,
631 .send_family = AF_INET,
632 .protocol = SOCK_STREAM,
633 .recv_socks = 10,
634 .recv_port = 8017,
635 .send_port_min = 9360});
636
637 test_filter_without_bind();
638
639 fprintf(stderr, "SUCCESS\n");
640 return 0;
641}