1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2018 Google Inc.
4  * Author: Eric Dumazet (edumazet@google.com)
5  *
6  * Reference program demonstrating tcp mmap() usage,
7  * and SO_RCVLOWAT hints for receiver.
8  *
9  * Note : NIC with header split is needed to use mmap() on TCP :
10  * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
11  *
12  * How to use on loopback interface :
13  *
14  *  ifconfig lo mtu 61512  # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
15  *  tcp_mmap -s -z &
16  *  tcp_mmap -H ::1 -z
17  *
18  *  Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
19  *      (4096 : page size on x86, 12: TCP TS option length)
20  *  tcp_mmap -s -z -M $((4096+12)) &
21  *  tcp_mmap -H ::1 -z -M $((4096+12))
22  *
23  * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
24  *       We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
25  *
26  * $ ./tcp_mmap -s &                                 # Without mmap()
27  * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
28  * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
29  *   cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
30  * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
31  *  cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
32  * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
33  *   cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
34  * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
35  *   cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
36  * $ kill %1   # kill tcp_mmap server
37  *
38  * $ ./tcp_mmap -s -z &                              # With mmap()
39  * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
40  * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
41  *   cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
42  * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
43  *   cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
44  * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
45  *   cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
46  * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
47  *   cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
48  */
49 #define _GNU_SOURCE
50 #include <pthread.h>
51 #include <sys/types.h>
52 #include <fcntl.h>
53 #include <error.h>
54 #include <sys/socket.h>
55 #include <sys/mman.h>
56 #include <sys/resource.h>
57 #include <unistd.h>
58 #include <string.h>
59 #include <stdlib.h>
60 #include <stdio.h>
61 #include <errno.h>
62 #include <time.h>
63 #include <sys/time.h>
64 #include <netinet/in.h>
65 #include <arpa/inet.h>
66 #include <poll.h>
67 #include <linux/tcp.h>
68 #include <assert.h>
69 #include <openssl/pem.h>
70 
71 #ifndef MSG_ZEROCOPY
72 #define MSG_ZEROCOPY    0x4000000
73 #endif
74 
75 #ifndef min
76 #define min(a, b)  ((a) < (b) ? (a) : (b))
77 #endif
78 
79 #define FILE_SZ (1ULL << 35)
80 static int cfg_family = AF_INET6;
81 static socklen_t cfg_alen = sizeof(struct sockaddr_in6);
82 static int cfg_port = 8787;
83 
84 static int rcvbuf; /* Default: autotuning.  Can be set with -r <integer> option */
85 static int sndbuf; /* Default: autotuning.  Can be set with -w <integer> option */
86 static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
87 static int xflg; /* hash received data (simple xor) (-h option) */
88 static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
89 static int integrity; /* -i option: sender and receiver compute sha256 over the data.*/
90 
91 static size_t chunk_size  = 512*1024;
92 
93 static size_t map_align;
94 
95 unsigned long htotal;
96 unsigned int digest_len;
97 
98 static inline void prefetch(const void *x)
99 {
100 #if defined(__x86_64__)
101 	asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));
102 #endif
103 }
104 
105 void hash_zone(void *zone, unsigned int length)
106 {
107 	unsigned long temp = htotal;
108 
109 	while (length >= 8*sizeof(long)) {
110 		prefetch(zone + 384);
111 		temp ^= *(unsigned long *)zone;
112 		temp ^= *(unsigned long *)(zone + sizeof(long));
113 		temp ^= *(unsigned long *)(zone + 2*sizeof(long));
114 		temp ^= *(unsigned long *)(zone + 3*sizeof(long));
115 		temp ^= *(unsigned long *)(zone + 4*sizeof(long));
116 		temp ^= *(unsigned long *)(zone + 5*sizeof(long));
117 		temp ^= *(unsigned long *)(zone + 6*sizeof(long));
118 		temp ^= *(unsigned long *)(zone + 7*sizeof(long));
119 		zone += 8*sizeof(long);
120 		length -= 8*sizeof(long);
121 	}
122 	while (length >= 1) {
123 		temp ^= *(unsigned char *)zone;
124 		zone += 1;
125 		length--;
126 	}
127 	htotal = temp;
128 }
129 
130 #define ALIGN_UP(x, align_to)	(((x) + ((align_to)-1)) & ~((align_to)-1))
131 #define ALIGN_PTR_UP(p, ptr_align_to)	((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
132 
133 
134 static void *mmap_large_buffer(size_t need, size_t *allocated)
135 {
136 	void *buffer;
137 	size_t sz;
138 
139 	/* Attempt to use huge pages if possible. */
140 	sz = ALIGN_UP(need, map_align);
141 	buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
142 		      MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
143 
144 	if (buffer == (void *)-1) {
145 		sz = need;
146 		buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
147 			      MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE,
148 			      -1, 0);
149 		if (buffer != (void *)-1)
150 			fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n");
151 	}
152 	*allocated = sz;
153 	return buffer;
154 }
155 
156 void *child_thread(void *arg)
157 {
158 	unsigned char digest[SHA256_DIGEST_LENGTH];
159 	unsigned long total_mmap = 0, total = 0;
160 	struct tcp_zerocopy_receive zc;
161 	unsigned char *buffer = NULL;
162 	unsigned long delta_usec;
163 	EVP_MD_CTX *ctx = NULL;
164 	int flags = MAP_SHARED;
165 	struct timeval t0, t1;
166 	void *raddr = NULL;
167 	void *addr = NULL;
168 	double throughput;
169 	struct rusage ru;
170 	size_t buffer_sz;
171 	int lu, fd;
172 
173 	fd = (int)(unsigned long)arg;
174 
175 	gettimeofday(&t0, NULL);
176 
177 	fcntl(fd, F_SETFL, O_NDELAY);
178 	buffer = mmap_large_buffer(chunk_size, &buffer_sz);
179 	if (buffer == (void *)-1) {
180 		perror("mmap");
181 		goto error;
182 	}
183 	if (zflg) {
184 		raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0);
185 		if (raddr == (void *)-1) {
186 			perror("mmap");
187 			zflg = 0;
188 		} else {
189 			addr = ALIGN_PTR_UP(raddr, map_align);
190 		}
191 	}
192 	if (integrity) {
193 		ctx = EVP_MD_CTX_new();
194 		if (!ctx) {
195 			perror("cannot enable SHA computing");
196 			goto error;
197 		}
198 		EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);
199 	}
200 	while (1) {
201 		struct pollfd pfd = { .fd = fd, .events = POLLIN, };
202 		int sub;
203 
204 		poll(&pfd, 1, 10000);
205 		if (zflg) {
206 			socklen_t zc_len = sizeof(zc);
207 			int res;
208 
209 			memset(&zc, 0, sizeof(zc));
210 			zc.address = (__u64)((unsigned long)addr);
211 			zc.length = min(chunk_size, FILE_SZ - total);
212 
213 			res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,
214 					 &zc, &zc_len);
215 			if (res == -1)
216 				break;
217 
218 			if (zc.length) {
219 				assert(zc.length <= chunk_size);
220 				if (integrity)
221 					EVP_DigestUpdate(ctx, addr, zc.length);
222 				total_mmap += zc.length;
223 				if (xflg)
224 					hash_zone(addr, zc.length);
225 				/* It is more efficient to unmap the pages right now,
226 				 * instead of doing this in next TCP_ZEROCOPY_RECEIVE.
227 				 */
228 				madvise(addr, zc.length, MADV_DONTNEED);
229 				total += zc.length;
230 			}
231 			if (zc.recv_skip_hint) {
232 				assert(zc.recv_skip_hint <= chunk_size);
233 				lu = read(fd, buffer, min(zc.recv_skip_hint,
234 							  FILE_SZ - total));
235 				if (lu > 0) {
236 					if (integrity)
237 						EVP_DigestUpdate(ctx, buffer, lu);
238 					if (xflg)
239 						hash_zone(buffer, lu);
240 					total += lu;
241 				}
242 				if (lu == 0)
243 					goto end;
244 			}
245 			continue;
246 		}
247 		sub = 0;
248 		while (sub < chunk_size) {
249 			lu = read(fd, buffer + sub, min(chunk_size - sub,
250 							FILE_SZ - total));
251 			if (lu == 0)
252 				goto end;
253 			if (lu < 0)
254 				break;
255 			if (integrity)
256 				EVP_DigestUpdate(ctx, buffer + sub, lu);
257 			if (xflg)
258 				hash_zone(buffer + sub, lu);
259 			total += lu;
260 			sub += lu;
261 		}
262 	}
263 end:
264 	gettimeofday(&t1, NULL);
265 	delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;
266 
267 	if (integrity) {
268 		fcntl(fd, F_SETFL, 0);
269 		EVP_DigestFinal_ex(ctx, digest, &digest_len);
270 		lu = read(fd, buffer, SHA256_DIGEST_LENGTH);
271 		if (lu != SHA256_DIGEST_LENGTH)
272 			perror("Error: Cannot read SHA256\n");
273 
274 		if (memcmp(digest, buffer,
275 			   SHA256_DIGEST_LENGTH))
276 			fprintf(stderr, "Error: SHA256 of the data is not right\n");
277 		else
278 			printf("\nSHA256 is correct\n");
279 	}
280 
281 	throughput = 0;
282 	if (delta_usec)
283 		throughput = total * 8.0 / (double)delta_usec / 1000.0;
284 	getrusage(RUSAGE_THREAD, &ru);
285 	if (total > 1024*1024) {
286 		unsigned long total_usec;
287 		unsigned long mb = total >> 20;
288 		total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +
289 			     1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;
290 		printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
291 		       "  cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches\n",
292 				total / (1024.0 * 1024.0),
293 				100.0*total_mmap/total,
294 				(double)delta_usec / 1000000.0,
295 				throughput,
296 				(double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,
297 				(double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,
298 				(double)total_usec/mb,
299 				ru.ru_nvcsw);
300 	}
301 error:
302 	munmap(buffer, buffer_sz);
303 	close(fd);
304 	if (zflg)
305 		munmap(raddr, chunk_size + map_align);
306 	pthread_exit(0);
307 }
308 
309 static void apply_rcvsnd_buf(int fd)
310 {
311 	if (rcvbuf && setsockopt(fd, SOL_SOCKET,
312 				 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {
313 		perror("setsockopt SO_RCVBUF");
314 	}
315 
316 	if (sndbuf && setsockopt(fd, SOL_SOCKET,
317 				 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {
318 		perror("setsockopt SO_SNDBUF");
319 	}
320 }
321 
322 
323 static void setup_sockaddr(int domain, const char *str_addr,
324 			   struct sockaddr_storage *sockaddr)
325 {
326 	struct sockaddr_in6 *addr6 = (void *) sockaddr;
327 	struct sockaddr_in *addr4 = (void *) sockaddr;
328 
329 	switch (domain) {
330 	case PF_INET:
331 		memset(addr4, 0, sizeof(*addr4));
332 		addr4->sin_family = AF_INET;
333 		addr4->sin_port = htons(cfg_port);
334 		if (str_addr &&
335 		    inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
336 			error(1, 0, "ipv4 parse error: %s", str_addr);
337 		break;
338 	case PF_INET6:
339 		memset(addr6, 0, sizeof(*addr6));
340 		addr6->sin6_family = AF_INET6;
341 		addr6->sin6_port = htons(cfg_port);
342 		if (str_addr &&
343 		    inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
344 			error(1, 0, "ipv6 parse error: %s", str_addr);
345 		break;
346 	default:
347 		error(1, 0, "illegal domain");
348 	}
349 }
350 
351 static void do_accept(int fdlisten)
352 {
353 	pthread_attr_t attr;
354 	int rcvlowat;
355 
356 	pthread_attr_init(&attr);
357 	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
358 
359 	rcvlowat = chunk_size;
360 	if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,
361 		       &rcvlowat, sizeof(rcvlowat)) == -1) {
362 		perror("setsockopt SO_RCVLOWAT");
363 	}
364 
365 	apply_rcvsnd_buf(fdlisten);
366 
367 	while (1) {
368 		struct sockaddr_in addr;
369 		socklen_t addrlen = sizeof(addr);
370 		pthread_t th;
371 		int fd, res;
372 
373 		fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);
374 		if (fd == -1) {
375 			perror("accept");
376 			continue;
377 		}
378 		res = pthread_create(&th, &attr, child_thread,
379 				     (void *)(unsigned long)fd);
380 		if (res) {
381 			errno = res;
382 			perror("pthread_create");
383 			close(fd);
384 		}
385 	}
386 }
387 
388 /* Each thread should reserve a big enough vma to avoid
389  * spinlock collisions in ptl locks.
390  * This size is 2MB on x86_64, and is exported in /proc/meminfo.
391  */
392 static unsigned long default_huge_page_size(void)
393 {
394 	FILE *f = fopen("/proc/meminfo", "r");
395 	unsigned long hps = 0;
396 	size_t linelen = 0;
397 	char *line = NULL;
398 
399 	if (!f)
400 		return 0;
401 	while (getline(&line, &linelen, f) > 0) {
402 		if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
403 			hps <<= 10;
404 			break;
405 		}
406 	}
407 	free(line);
408 	fclose(f);
409 	return hps;
410 }
411 
412 static void randomize(void *target, size_t count)
413 {
414 	static int urandom = -1;
415 	ssize_t got;
416 
417 	urandom = open("/dev/urandom", O_RDONLY);
418 	if (urandom < 0) {
419 		perror("open /dev/urandom");
420 		exit(1);
421 	}
422 	got = read(urandom, target, count);
423 	if (got != count) {
424 		perror("read /dev/urandom");
425 		exit(1);
426 	}
427 }
428 
429 int main(int argc, char *argv[])
430 {
431 	unsigned char digest[SHA256_DIGEST_LENGTH];
432 	struct sockaddr_storage listenaddr, addr;
433 	unsigned int max_pacing_rate = 0;
434 	EVP_MD_CTX *ctx = NULL;
435 	unsigned char *buffer;
436 	uint64_t total = 0;
437 	char *host = NULL;
438 	int fd, c, on = 1;
439 	size_t buffer_sz;
440 	int sflg = 0;
441 	int mss = 0;
442 
443 	while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:C:a:i")) != -1) {
444 		switch (c) {
445 		case '4':
446 			cfg_family = PF_INET;
447 			cfg_alen = sizeof(struct sockaddr_in);
448 			break;
449 		case '6':
450 			cfg_family = PF_INET6;
451 			cfg_alen = sizeof(struct sockaddr_in6);
452 			break;
453 		case 'p':
454 			cfg_port = atoi(optarg);
455 			break;
456 		case 'H':
457 			host = optarg;
458 			break;
459 		case 's': /* server : listen for incoming connections */
460 			sflg++;
461 			break;
462 		case 'r':
463 			rcvbuf = atoi(optarg);
464 			break;
465 		case 'w':
466 			sndbuf = atoi(optarg);
467 			break;
468 		case 'z':
469 			zflg = 1;
470 			break;
471 		case 'M':
472 			mss = atoi(optarg);
473 			break;
474 		case 'x':
475 			xflg = 1;
476 			break;
477 		case 'k':
478 			keepflag = 1;
479 			break;
480 		case 'P':
481 			max_pacing_rate = atoi(optarg) ;
482 			break;
483 		case 'C':
484 			chunk_size = atol(optarg);
485 			break;
486 		case 'a':
487 			map_align = atol(optarg);
488 			break;
489 		case 'i':
490 			integrity = 1;
491 			break;
492 		default:
493 			exit(1);
494 		}
495 	}
496 	if (!map_align) {
497 		map_align = default_huge_page_size();
498 		/* if really /proc/meminfo is not helping,
499 		 * we use the default x86_64 hugepagesize.
500 		 */
501 		if (!map_align)
502 			map_align = 2*1024*1024;
503 	}
504 	if (sflg) {
505 		int fdlisten = socket(cfg_family, SOCK_STREAM, 0);
506 
507 		if (fdlisten == -1) {
508 			perror("socket");
509 			exit(1);
510 		}
511 		apply_rcvsnd_buf(fdlisten);
512 		setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
513 
514 		setup_sockaddr(cfg_family, host, &listenaddr);
515 
516 		if (mss &&
517 		    setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,
518 			       &mss, sizeof(mss)) == -1) {
519 			perror("setsockopt TCP_MAXSEG");
520 			exit(1);
521 		}
522 		if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {
523 			perror("bind");
524 			exit(1);
525 		}
526 		if (listen(fdlisten, 128) == -1) {
527 			perror("listen");
528 			exit(1);
529 		}
530 		do_accept(fdlisten);
531 	}
532 
533 	buffer = mmap_large_buffer(chunk_size, &buffer_sz);
534 	if (buffer == (unsigned char *)-1) {
535 		perror("mmap");
536 		exit(1);
537 	}
538 
539 	fd = socket(cfg_family, SOCK_STREAM, 0);
540 	if (fd == -1) {
541 		perror("socket");
542 		exit(1);
543 	}
544 	apply_rcvsnd_buf(fd);
545 
546 	setup_sockaddr(cfg_family, host, &addr);
547 
548 	if (mss &&
549 	    setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {
550 		perror("setsockopt TCP_MAXSEG");
551 		exit(1);
552 	}
553 	if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {
554 		perror("connect");
555 		exit(1);
556 	}
557 	if (max_pacing_rate &&
558 	    setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,
559 		       &max_pacing_rate, sizeof(max_pacing_rate)) == -1)
560 		perror("setsockopt SO_MAX_PACING_RATE");
561 
562 	if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
563 			       &on, sizeof(on)) == -1) {
564 		perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
565 		zflg = 0;
566 	}
567 	if (integrity) {
568 		randomize(buffer, buffer_sz);
569 		ctx = EVP_MD_CTX_new();
570 		if (!ctx) {
571 			perror("cannot enable SHA computing");
572 			exit(1);
573 		}
574 		EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);
575 	}
576 	while (total < FILE_SZ) {
577 		size_t offset = total % chunk_size;
578 		int64_t wr = FILE_SZ - total;
579 
580 		if (wr > chunk_size - offset)
581 			wr = chunk_size - offset;
582 		/* Note : we just want to fill the pipe with random bytes */
583 		wr = send(fd, buffer + offset,
584 			  (size_t)wr, zflg ? MSG_ZEROCOPY : 0);
585 		if (wr <= 0)
586 			break;
587 		if (integrity)
588 			EVP_DigestUpdate(ctx, buffer + offset, wr);
589 		total += wr;
590 	}
591 	if (integrity && total == FILE_SZ) {
592 		EVP_DigestFinal_ex(ctx, digest, &digest_len);
593 		send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0);
594 	}
595 	close(fd);
596 	munmap(buffer, buffer_sz);
597 	return 0;
598 }
599