Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright 2018 Google Inc.
4 * Author: Eric Dumazet (edumazet@google.com)
5 *
6 * Reference program demonstrating tcp mmap() usage,
7 * and SO_RCVLOWAT hints for receiver.
8 *
9 * Note : NIC with header split is needed to use mmap() on TCP :
10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
11 *
12 * How to use on loopback interface :
13 *
14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
15 * tcp_mmap -s -z &
16 * tcp_mmap -H ::1 -z
17 *
18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
19 * (4096 : page size on x86, 12: TCP TS option length)
20 * tcp_mmap -s -z -M $((4096+12)) &
21 * tcp_mmap -H ::1 -z -M $((4096+12))
22 *
23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
25 *
26 * $ ./tcp_mmap -s & # Without mmap()
27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
36 * $ kill %1 # kill tcp_mmap server
37 *
38 * $ ./tcp_mmap -s -z & # With mmap()
39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
48 */
49#define _GNU_SOURCE
50#include <pthread.h>
51#include <sys/types.h>
52#include <fcntl.h>
53#include <error.h>
54#include <sys/socket.h>
55#include <sys/mman.h>
56#include <sys/resource.h>
57#include <unistd.h>
58#include <string.h>
59#include <stdlib.h>
60#include <stdio.h>
61#include <errno.h>
62#include <time.h>
63#include <sys/time.h>
64#include <netinet/in.h>
65#include <arpa/inet.h>
66#include <poll.h>
67#include <linux/tcp.h>
68#include <assert.h>
69#include <openssl/pem.h>
70
71#ifndef MSG_ZEROCOPY
72#define MSG_ZEROCOPY 0x4000000
73#endif
74
75#ifndef min
76#define min(a, b) ((a) < (b) ? (a) : (b))
77#endif
78
79#define FILE_SZ (1ULL << 35)
80static int cfg_family = AF_INET6;
81static socklen_t cfg_alen = sizeof(struct sockaddr_in6);
82static int cfg_port = 8787;
83
84static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */
85static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */
86static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
87static int xflg; /* hash received data (simple xor) (-h option) */
88static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
89static int integrity; /* -i option: sender and receiver compute sha256 over the data.*/
90
91static size_t chunk_size = 512*1024;
92
93static size_t map_align;
94
95unsigned long htotal;
96unsigned int digest_len;
97
98static inline void prefetch(const void *x)
99{
100#if defined(__x86_64__)
101 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));
102#endif
103}
104
105void hash_zone(void *zone, unsigned int length)
106{
107 unsigned long temp = htotal;
108
109 while (length >= 8*sizeof(long)) {
110 prefetch(zone + 384);
111 temp ^= *(unsigned long *)zone;
112 temp ^= *(unsigned long *)(zone + sizeof(long));
113 temp ^= *(unsigned long *)(zone + 2*sizeof(long));
114 temp ^= *(unsigned long *)(zone + 3*sizeof(long));
115 temp ^= *(unsigned long *)(zone + 4*sizeof(long));
116 temp ^= *(unsigned long *)(zone + 5*sizeof(long));
117 temp ^= *(unsigned long *)(zone + 6*sizeof(long));
118 temp ^= *(unsigned long *)(zone + 7*sizeof(long));
119 zone += 8*sizeof(long);
120 length -= 8*sizeof(long);
121 }
122 while (length >= 1) {
123 temp ^= *(unsigned char *)zone;
124 zone += 1;
125 length--;
126 }
127 htotal = temp;
128}
129
130#define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))
131#define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
132
133
134static void *mmap_large_buffer(size_t need, size_t *allocated)
135{
136 void *buffer;
137 size_t sz;
138
139 /* Attempt to use huge pages if possible. */
140 sz = ALIGN_UP(need, map_align);
141 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
142 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
143
144 if (buffer == (void *)-1) {
145 sz = need;
146 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
147 MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE,
148 -1, 0);
149 if (buffer != (void *)-1)
150 fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n");
151 }
152 *allocated = sz;
153 return buffer;
154}
155
156static uint32_t tcp_info_get_rcv_mss(int fd)
157{
158 socklen_t sz = sizeof(struct tcp_info);
159 struct tcp_info info;
160
161 if (getsockopt(fd, IPPROTO_TCP, TCP_INFO, &info, &sz)) {
162 fprintf(stderr, "Error fetching TCP_INFO\n");
163 return 0;
164 }
165
166 return info.tcpi_rcv_mss;
167}
168
169void *child_thread(void *arg)
170{
171 unsigned char digest[SHA256_DIGEST_LENGTH];
172 unsigned long total_mmap = 0, total = 0;
173 struct tcp_zerocopy_receive zc;
174 unsigned char *buffer = NULL;
175 unsigned long delta_usec;
176 EVP_MD_CTX *ctx = NULL;
177 int flags = MAP_SHARED;
178 struct timeval t0, t1;
179 void *raddr = NULL;
180 void *addr = NULL;
181 double throughput;
182 struct rusage ru;
183 size_t buffer_sz;
184 int lu, fd;
185
186 fd = (int)(unsigned long)arg;
187
188 gettimeofday(&t0, NULL);
189
190 fcntl(fd, F_SETFL, O_NDELAY);
191 buffer = mmap_large_buffer(chunk_size, &buffer_sz);
192 if (buffer == (void *)-1) {
193 perror("mmap");
194 goto error;
195 }
196 if (zflg) {
197 raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0);
198 if (raddr == (void *)-1) {
199 perror("mmap");
200 zflg = 0;
201 } else {
202 addr = ALIGN_PTR_UP(raddr, map_align);
203 }
204 }
205 if (integrity) {
206 ctx = EVP_MD_CTX_new();
207 if (!ctx) {
208 perror("cannot enable SHA computing");
209 goto error;
210 }
211 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);
212 }
213 while (1) {
214 struct pollfd pfd = { .fd = fd, .events = POLLIN, };
215 int sub;
216
217 poll(&pfd, 1, 10000);
218 if (zflg) {
219 socklen_t zc_len = sizeof(zc);
220 int res;
221
222 memset(&zc, 0, sizeof(zc));
223 zc.address = (__u64)((unsigned long)addr);
224 zc.length = min(chunk_size, FILE_SZ - total);
225
226 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,
227 &zc, &zc_len);
228 if (res == -1)
229 break;
230
231 if (zc.length) {
232 assert(zc.length <= chunk_size);
233 if (integrity)
234 EVP_DigestUpdate(ctx, addr, zc.length);
235 total_mmap += zc.length;
236 if (xflg)
237 hash_zone(addr, zc.length);
238 /* It is more efficient to unmap the pages right now,
239 * instead of doing this in next TCP_ZEROCOPY_RECEIVE.
240 */
241 madvise(addr, zc.length, MADV_DONTNEED);
242 total += zc.length;
243 }
244 if (zc.recv_skip_hint) {
245 assert(zc.recv_skip_hint <= chunk_size);
246 lu = read(fd, buffer, min(zc.recv_skip_hint,
247 FILE_SZ - total));
248 if (lu > 0) {
249 if (integrity)
250 EVP_DigestUpdate(ctx, buffer, lu);
251 if (xflg)
252 hash_zone(buffer, lu);
253 total += lu;
254 }
255 if (lu == 0)
256 goto end;
257 }
258 continue;
259 }
260 sub = 0;
261 while (sub < chunk_size) {
262 lu = read(fd, buffer + sub, min(chunk_size - sub,
263 FILE_SZ - total));
264 if (lu == 0)
265 goto end;
266 if (lu < 0)
267 break;
268 if (integrity)
269 EVP_DigestUpdate(ctx, buffer + sub, lu);
270 if (xflg)
271 hash_zone(buffer + sub, lu);
272 total += lu;
273 sub += lu;
274 }
275 }
276end:
277 gettimeofday(&t1, NULL);
278 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;
279
280 if (integrity) {
281 fcntl(fd, F_SETFL, 0);
282 EVP_DigestFinal_ex(ctx, digest, &digest_len);
283 lu = read(fd, buffer, SHA256_DIGEST_LENGTH);
284 if (lu != SHA256_DIGEST_LENGTH)
285 perror("Error: Cannot read SHA256\n");
286
287 if (memcmp(digest, buffer,
288 SHA256_DIGEST_LENGTH))
289 fprintf(stderr, "Error: SHA256 of the data is not right\n");
290 else
291 printf("\nSHA256 is correct\n");
292 }
293
294 throughput = 0;
295 if (delta_usec)
296 throughput = total * 8.0 / (double)delta_usec / 1000.0;
297 getrusage(RUSAGE_THREAD, &ru);
298 if (total > 1024*1024) {
299 unsigned long total_usec;
300 unsigned long mb = total >> 20;
301 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +
302 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;
303 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
304 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches, rcv_mss %u\n",
305 total / (1024.0 * 1024.0),
306 100.0*total_mmap/total,
307 (double)delta_usec / 1000000.0,
308 throughput,
309 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,
310 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,
311 (double)total_usec/mb,
312 ru.ru_nvcsw,
313 tcp_info_get_rcv_mss(fd));
314 }
315error:
316 munmap(buffer, buffer_sz);
317 close(fd);
318 if (zflg)
319 munmap(raddr, chunk_size + map_align);
320 pthread_exit(0);
321}
322
323static void apply_rcvsnd_buf(int fd)
324{
325 if (rcvbuf && setsockopt(fd, SOL_SOCKET,
326 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {
327 perror("setsockopt SO_RCVBUF");
328 }
329
330 if (sndbuf && setsockopt(fd, SOL_SOCKET,
331 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {
332 perror("setsockopt SO_SNDBUF");
333 }
334}
335
336
337static void setup_sockaddr(int domain, const char *str_addr,
338 struct sockaddr_storage *sockaddr)
339{
340 struct sockaddr_in6 *addr6 = (void *) sockaddr;
341 struct sockaddr_in *addr4 = (void *) sockaddr;
342
343 switch (domain) {
344 case PF_INET:
345 memset(addr4, 0, sizeof(*addr4));
346 addr4->sin_family = AF_INET;
347 addr4->sin_port = htons(cfg_port);
348 if (str_addr &&
349 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
350 error(1, 0, "ipv4 parse error: %s", str_addr);
351 break;
352 case PF_INET6:
353 memset(addr6, 0, sizeof(*addr6));
354 addr6->sin6_family = AF_INET6;
355 addr6->sin6_port = htons(cfg_port);
356 if (str_addr &&
357 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
358 error(1, 0, "ipv6 parse error: %s", str_addr);
359 break;
360 default:
361 error(1, 0, "illegal domain");
362 }
363}
364
365static void do_accept(int fdlisten)
366{
367 pthread_attr_t attr;
368 int rcvlowat;
369
370 pthread_attr_init(&attr);
371 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
372
373 rcvlowat = chunk_size;
374 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,
375 &rcvlowat, sizeof(rcvlowat)) == -1) {
376 perror("setsockopt SO_RCVLOWAT");
377 }
378
379 apply_rcvsnd_buf(fdlisten);
380
381 while (1) {
382 struct sockaddr_in addr;
383 socklen_t addrlen = sizeof(addr);
384 pthread_t th;
385 int fd, res;
386
387 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);
388 if (fd == -1) {
389 perror("accept");
390 continue;
391 }
392 res = pthread_create(&th, &attr, child_thread,
393 (void *)(unsigned long)fd);
394 if (res) {
395 errno = res;
396 perror("pthread_create");
397 close(fd);
398 }
399 }
400}
401
402/* Each thread should reserve a big enough vma to avoid
403 * spinlock collisions in ptl locks.
404 * This size is 2MB on x86_64, and is exported in /proc/meminfo.
405 */
406static unsigned long default_huge_page_size(void)
407{
408 FILE *f = fopen("/proc/meminfo", "r");
409 unsigned long hps = 0;
410 size_t linelen = 0;
411 char *line = NULL;
412
413 if (!f)
414 return 0;
415 while (getline(&line, &linelen, f) > 0) {
416 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
417 hps <<= 10;
418 break;
419 }
420 }
421 free(line);
422 fclose(f);
423 return hps;
424}
425
426static void randomize(void *target, size_t count)
427{
428 static int urandom = -1;
429 ssize_t got;
430
431 urandom = open("/dev/urandom", O_RDONLY);
432 if (urandom < 0) {
433 perror("open /dev/urandom");
434 exit(1);
435 }
436 got = read(urandom, target, count);
437 if (got != count) {
438 perror("read /dev/urandom");
439 exit(1);
440 }
441}
442
443int main(int argc, char *argv[])
444{
445 unsigned char digest[SHA256_DIGEST_LENGTH];
446 struct sockaddr_storage listenaddr, addr;
447 unsigned int max_pacing_rate = 0;
448 EVP_MD_CTX *ctx = NULL;
449 unsigned char *buffer;
450 uint64_t total = 0;
451 char *host = NULL;
452 int fd, c, on = 1;
453 size_t buffer_sz;
454 int sflg = 0;
455 int mss = 0;
456
457 while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:C:a:i")) != -1) {
458 switch (c) {
459 case '4':
460 cfg_family = PF_INET;
461 cfg_alen = sizeof(struct sockaddr_in);
462 break;
463 case '6':
464 cfg_family = PF_INET6;
465 cfg_alen = sizeof(struct sockaddr_in6);
466 break;
467 case 'p':
468 cfg_port = atoi(optarg);
469 break;
470 case 'H':
471 host = optarg;
472 break;
473 case 's': /* server : listen for incoming connections */
474 sflg++;
475 break;
476 case 'r':
477 rcvbuf = atoi(optarg);
478 break;
479 case 'w':
480 sndbuf = atoi(optarg);
481 break;
482 case 'z':
483 zflg = 1;
484 break;
485 case 'M':
486 mss = atoi(optarg);
487 break;
488 case 'x':
489 xflg = 1;
490 break;
491 case 'k':
492 keepflag = 1;
493 break;
494 case 'P':
495 max_pacing_rate = atoi(optarg) ;
496 break;
497 case 'C':
498 chunk_size = atol(optarg);
499 break;
500 case 'a':
501 map_align = atol(optarg);
502 break;
503 case 'i':
504 integrity = 1;
505 break;
506 default:
507 exit(1);
508 }
509 }
510 if (!map_align) {
511 map_align = default_huge_page_size();
512 /* if really /proc/meminfo is not helping,
513 * we use the default x86_64 hugepagesize.
514 */
515 if (!map_align)
516 map_align = 2*1024*1024;
517 }
518 if (sflg) {
519 int fdlisten = socket(cfg_family, SOCK_STREAM, 0);
520
521 if (fdlisten == -1) {
522 perror("socket");
523 exit(1);
524 }
525 apply_rcvsnd_buf(fdlisten);
526 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
527
528 setup_sockaddr(cfg_family, host, &listenaddr);
529
530 if (mss &&
531 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,
532 &mss, sizeof(mss)) == -1) {
533 perror("setsockopt TCP_MAXSEG");
534 exit(1);
535 }
536 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {
537 perror("bind");
538 exit(1);
539 }
540 if (listen(fdlisten, 128) == -1) {
541 perror("listen");
542 exit(1);
543 }
544 do_accept(fdlisten);
545 }
546
547 buffer = mmap_large_buffer(chunk_size, &buffer_sz);
548 if (buffer == (unsigned char *)-1) {
549 perror("mmap");
550 exit(1);
551 }
552
553 fd = socket(cfg_family, SOCK_STREAM, 0);
554 if (fd == -1) {
555 perror("socket");
556 exit(1);
557 }
558 apply_rcvsnd_buf(fd);
559
560 setup_sockaddr(cfg_family, host, &addr);
561
562 if (mss &&
563 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {
564 perror("setsockopt TCP_MAXSEG");
565 exit(1);
566 }
567 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {
568 perror("connect");
569 exit(1);
570 }
571 if (max_pacing_rate &&
572 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,
573 &max_pacing_rate, sizeof(max_pacing_rate)) == -1)
574 perror("setsockopt SO_MAX_PACING_RATE");
575
576 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
577 &on, sizeof(on)) == -1) {
578 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
579 zflg = 0;
580 }
581 if (integrity) {
582 randomize(buffer, buffer_sz);
583 ctx = EVP_MD_CTX_new();
584 if (!ctx) {
585 perror("cannot enable SHA computing");
586 exit(1);
587 }
588 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);
589 }
590 while (total < FILE_SZ) {
591 size_t offset = total % chunk_size;
592 int64_t wr = FILE_SZ - total;
593
594 if (wr > chunk_size - offset)
595 wr = chunk_size - offset;
596 /* Note : we just want to fill the pipe with random bytes */
597 wr = send(fd, buffer + offset,
598 (size_t)wr, zflg ? MSG_ZEROCOPY : 0);
599 if (wr <= 0)
600 break;
601 if (integrity)
602 EVP_DigestUpdate(ctx, buffer + offset, wr);
603 total += wr;
604 }
605 if (integrity && total == FILE_SZ) {
606 EVP_DigestFinal_ex(ctx, digest, &digest_len);
607 send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0);
608 }
609 close(fd);
610 munmap(buffer, buffer_sz);
611 return 0;
612}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright 2018 Google Inc.
4 * Author: Eric Dumazet (edumazet@google.com)
5 *
6 * Reference program demonstrating tcp mmap() usage,
7 * and SO_RCVLOWAT hints for receiver.
8 *
9 * Note : NIC with header split is needed to use mmap() on TCP :
10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
11 *
12 * How to use on loopback interface :
13 *
14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
15 * tcp_mmap -s -z &
16 * tcp_mmap -H ::1 -z
17 *
18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
19 * (4096 : page size on x86, 12: TCP TS option length)
20 * tcp_mmap -s -z -M $((4096+12)) &
21 * tcp_mmap -H ::1 -z -M $((4096+12))
22 *
23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
25 *
26 * $ ./tcp_mmap -s & # Without mmap()
27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
36 * $ kill %1 # kill tcp_mmap server
37 *
38 * $ ./tcp_mmap -s -z & # With mmap()
39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
48 */
49#define _GNU_SOURCE
50#include <pthread.h>
51#include <sys/types.h>
52#include <fcntl.h>
53#include <error.h>
54#include <sys/socket.h>
55#include <sys/mman.h>
56#include <sys/resource.h>
57#include <unistd.h>
58#include <string.h>
59#include <stdlib.h>
60#include <stdio.h>
61#include <errno.h>
62#include <time.h>
63#include <sys/time.h>
64#include <netinet/in.h>
65#include <arpa/inet.h>
66#include <poll.h>
67#include <linux/tcp.h>
68#include <assert.h>
69
70#ifndef MSG_ZEROCOPY
71#define MSG_ZEROCOPY 0x4000000
72#endif
73
74#define FILE_SZ (1UL << 35)
75static int cfg_family = AF_INET6;
76static socklen_t cfg_alen = sizeof(struct sockaddr_in6);
77static int cfg_port = 8787;
78
79static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */
80static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */
81static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
82static int xflg; /* hash received data (simple xor) (-h option) */
83static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
84
85static int chunk_size = 512*1024;
86
87unsigned long htotal;
88
89static inline void prefetch(const void *x)
90{
91#if defined(__x86_64__)
92 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));
93#endif
94}
95
96void hash_zone(void *zone, unsigned int length)
97{
98 unsigned long temp = htotal;
99
100 while (length >= 8*sizeof(long)) {
101 prefetch(zone + 384);
102 temp ^= *(unsigned long *)zone;
103 temp ^= *(unsigned long *)(zone + sizeof(long));
104 temp ^= *(unsigned long *)(zone + 2*sizeof(long));
105 temp ^= *(unsigned long *)(zone + 3*sizeof(long));
106 temp ^= *(unsigned long *)(zone + 4*sizeof(long));
107 temp ^= *(unsigned long *)(zone + 5*sizeof(long));
108 temp ^= *(unsigned long *)(zone + 6*sizeof(long));
109 temp ^= *(unsigned long *)(zone + 7*sizeof(long));
110 zone += 8*sizeof(long);
111 length -= 8*sizeof(long);
112 }
113 while (length >= 1) {
114 temp ^= *(unsigned char *)zone;
115 zone += 1;
116 length--;
117 }
118 htotal = temp;
119}
120
121void *child_thread(void *arg)
122{
123 unsigned long total_mmap = 0, total = 0;
124 struct tcp_zerocopy_receive zc;
125 unsigned long delta_usec;
126 int flags = MAP_SHARED;
127 struct timeval t0, t1;
128 char *buffer = NULL;
129 void *addr = NULL;
130 double throughput;
131 struct rusage ru;
132 int lu, fd;
133
134 fd = (int)(unsigned long)arg;
135
136 gettimeofday(&t0, NULL);
137
138 fcntl(fd, F_SETFL, O_NDELAY);
139 buffer = malloc(chunk_size);
140 if (!buffer) {
141 perror("malloc");
142 goto error;
143 }
144 if (zflg) {
145 addr = mmap(NULL, chunk_size, PROT_READ, flags, fd, 0);
146 if (addr == (void *)-1)
147 zflg = 0;
148 }
149 while (1) {
150 struct pollfd pfd = { .fd = fd, .events = POLLIN, };
151 int sub;
152
153 poll(&pfd, 1, 10000);
154 if (zflg) {
155 socklen_t zc_len = sizeof(zc);
156 int res;
157
158 zc.address = (__u64)addr;
159 zc.length = chunk_size;
160 zc.recv_skip_hint = 0;
161 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,
162 &zc, &zc_len);
163 if (res == -1)
164 break;
165
166 if (zc.length) {
167 assert(zc.length <= chunk_size);
168 total_mmap += zc.length;
169 if (xflg)
170 hash_zone(addr, zc.length);
171 total += zc.length;
172 }
173 if (zc.recv_skip_hint) {
174 assert(zc.recv_skip_hint <= chunk_size);
175 lu = read(fd, buffer, zc.recv_skip_hint);
176 if (lu > 0) {
177 if (xflg)
178 hash_zone(buffer, lu);
179 total += lu;
180 }
181 }
182 continue;
183 }
184 sub = 0;
185 while (sub < chunk_size) {
186 lu = read(fd, buffer + sub, chunk_size - sub);
187 if (lu == 0)
188 goto end;
189 if (lu < 0)
190 break;
191 if (xflg)
192 hash_zone(buffer + sub, lu);
193 total += lu;
194 sub += lu;
195 }
196 }
197end:
198 gettimeofday(&t1, NULL);
199 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;
200
201 throughput = 0;
202 if (delta_usec)
203 throughput = total * 8.0 / (double)delta_usec / 1000.0;
204 getrusage(RUSAGE_THREAD, &ru);
205 if (total > 1024*1024) {
206 unsigned long total_usec;
207 unsigned long mb = total >> 20;
208 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +
209 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;
210 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
211 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches\n",
212 total / (1024.0 * 1024.0),
213 100.0*total_mmap/total,
214 (double)delta_usec / 1000000.0,
215 throughput,
216 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,
217 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,
218 (double)total_usec/mb,
219 ru.ru_nvcsw);
220 }
221error:
222 free(buffer);
223 close(fd);
224 if (zflg)
225 munmap(addr, chunk_size);
226 pthread_exit(0);
227}
228
229static void apply_rcvsnd_buf(int fd)
230{
231 if (rcvbuf && setsockopt(fd, SOL_SOCKET,
232 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {
233 perror("setsockopt SO_RCVBUF");
234 }
235
236 if (sndbuf && setsockopt(fd, SOL_SOCKET,
237 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {
238 perror("setsockopt SO_SNDBUF");
239 }
240}
241
242
243static void setup_sockaddr(int domain, const char *str_addr,
244 struct sockaddr_storage *sockaddr)
245{
246 struct sockaddr_in6 *addr6 = (void *) sockaddr;
247 struct sockaddr_in *addr4 = (void *) sockaddr;
248
249 switch (domain) {
250 case PF_INET:
251 memset(addr4, 0, sizeof(*addr4));
252 addr4->sin_family = AF_INET;
253 addr4->sin_port = htons(cfg_port);
254 if (str_addr &&
255 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
256 error(1, 0, "ipv4 parse error: %s", str_addr);
257 break;
258 case PF_INET6:
259 memset(addr6, 0, sizeof(*addr6));
260 addr6->sin6_family = AF_INET6;
261 addr6->sin6_port = htons(cfg_port);
262 if (str_addr &&
263 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
264 error(1, 0, "ipv6 parse error: %s", str_addr);
265 break;
266 default:
267 error(1, 0, "illegal domain");
268 }
269}
270
271static void do_accept(int fdlisten)
272{
273 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,
274 &chunk_size, sizeof(chunk_size)) == -1) {
275 perror("setsockopt SO_RCVLOWAT");
276 }
277
278 apply_rcvsnd_buf(fdlisten);
279
280 while (1) {
281 struct sockaddr_in addr;
282 socklen_t addrlen = sizeof(addr);
283 pthread_t th;
284 int fd, res;
285
286 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);
287 if (fd == -1) {
288 perror("accept");
289 continue;
290 }
291 res = pthread_create(&th, NULL, child_thread,
292 (void *)(unsigned long)fd);
293 if (res) {
294 errno = res;
295 perror("pthread_create");
296 close(fd);
297 }
298 }
299}
300
301int main(int argc, char *argv[])
302{
303 struct sockaddr_storage listenaddr, addr;
304 unsigned int max_pacing_rate = 0;
305 unsigned long total = 0;
306 char *host = NULL;
307 int fd, c, on = 1;
308 char *buffer;
309 int sflg = 0;
310 int mss = 0;
311
312 while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:")) != -1) {
313 switch (c) {
314 case '4':
315 cfg_family = PF_INET;
316 cfg_alen = sizeof(struct sockaddr_in);
317 break;
318 case '6':
319 cfg_family = PF_INET6;
320 cfg_alen = sizeof(struct sockaddr_in6);
321 break;
322 case 'p':
323 cfg_port = atoi(optarg);
324 break;
325 case 'H':
326 host = optarg;
327 break;
328 case 's': /* server : listen for incoming connections */
329 sflg++;
330 break;
331 case 'r':
332 rcvbuf = atoi(optarg);
333 break;
334 case 'w':
335 sndbuf = atoi(optarg);
336 break;
337 case 'z':
338 zflg = 1;
339 break;
340 case 'M':
341 mss = atoi(optarg);
342 break;
343 case 'x':
344 xflg = 1;
345 break;
346 case 'k':
347 keepflag = 1;
348 break;
349 case 'P':
350 max_pacing_rate = atoi(optarg) ;
351 break;
352 default:
353 exit(1);
354 }
355 }
356 if (sflg) {
357 int fdlisten = socket(cfg_family, SOCK_STREAM, 0);
358
359 if (fdlisten == -1) {
360 perror("socket");
361 exit(1);
362 }
363 apply_rcvsnd_buf(fdlisten);
364 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
365
366 setup_sockaddr(cfg_family, host, &listenaddr);
367
368 if (mss &&
369 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,
370 &mss, sizeof(mss)) == -1) {
371 perror("setsockopt TCP_MAXSEG");
372 exit(1);
373 }
374 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {
375 perror("bind");
376 exit(1);
377 }
378 if (listen(fdlisten, 128) == -1) {
379 perror("listen");
380 exit(1);
381 }
382 do_accept(fdlisten);
383 }
384 buffer = mmap(NULL, chunk_size, PROT_READ | PROT_WRITE,
385 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
386 if (buffer == (char *)-1) {
387 perror("mmap");
388 exit(1);
389 }
390
391 fd = socket(cfg_family, SOCK_STREAM, 0);
392 if (fd == -1) {
393 perror("socket");
394 exit(1);
395 }
396 apply_rcvsnd_buf(fd);
397
398 setup_sockaddr(cfg_family, host, &addr);
399
400 if (mss &&
401 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {
402 perror("setsockopt TCP_MAXSEG");
403 exit(1);
404 }
405 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {
406 perror("connect");
407 exit(1);
408 }
409 if (max_pacing_rate &&
410 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,
411 &max_pacing_rate, sizeof(max_pacing_rate)) == -1)
412 perror("setsockopt SO_MAX_PACING_RATE");
413
414 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
415 &on, sizeof(on)) == -1) {
416 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
417 zflg = 0;
418 }
419 while (total < FILE_SZ) {
420 long wr = FILE_SZ - total;
421
422 if (wr > chunk_size)
423 wr = chunk_size;
424 /* Note : we just want to fill the pipe with 0 bytes */
425 wr = send(fd, buffer, wr, zflg ? MSG_ZEROCOPY : 0);
426 if (wr <= 0)
427 break;
428 total += wr;
429 }
430 close(fd);
431 munmap(buffer, chunk_size);
432 return 0;
433}