1 // SPDX-License-Identifier: GPL-2.0 2 3 #define _GNU_SOURCE 4 5 #include <arpa/inet.h> 6 #include <errno.h> 7 #include <error.h> 8 #include <netinet/if_ether.h> 9 #include <netinet/in.h> 10 #include <netinet/ip.h> 11 #include <netinet/ip6.h> 12 #include <netinet/udp.h> 13 #include <poll.h> 14 #include <sched.h> 15 #include <signal.h> 16 #include <stdbool.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <string.h> 20 #include <sys/socket.h> 21 #include <sys/time.h> 22 #include <sys/types.h> 23 #include <unistd.h> 24 25 #ifndef ETH_MAX_MTU 26 #define ETH_MAX_MTU 0xFFFFU 27 #endif 28 29 #ifndef UDP_SEGMENT 30 #define UDP_SEGMENT 103 31 #endif 32 33 #ifndef SO_ZEROCOPY 34 #define SO_ZEROCOPY 60 35 #endif 36 37 #ifndef MSG_ZEROCOPY 38 #define MSG_ZEROCOPY 0x4000000 39 #endif 40 41 #define NUM_PKT 100 42 43 static bool cfg_cache_trash; 44 static int cfg_cpu = -1; 45 static int cfg_connected = true; 46 static int cfg_family = PF_UNSPEC; 47 static uint16_t cfg_mss; 48 static int cfg_payload_len = (1472 * 42); 49 static int cfg_port = 8000; 50 static int cfg_runtime_ms = -1; 51 static bool cfg_segment; 52 static bool cfg_sendmmsg; 53 static bool cfg_tcp; 54 static bool cfg_zerocopy; 55 static int cfg_msg_nr; 56 static uint16_t cfg_gso_size; 57 58 static socklen_t cfg_alen; 59 static struct sockaddr_storage cfg_dst_addr; 60 61 static bool interrupted; 62 static char buf[NUM_PKT][ETH_MAX_MTU]; 63 64 static void sigint_handler(int signum) 65 { 66 if (signum == SIGINT) 67 interrupted = true; 68 } 69 70 static unsigned long gettimeofday_ms(void) 71 { 72 struct timeval tv; 73 74 gettimeofday(&tv, NULL); 75 return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); 76 } 77 78 static int set_cpu(int cpu) 79 { 80 cpu_set_t mask; 81 82 CPU_ZERO(&mask); 83 CPU_SET(cpu, &mask); 84 if (sched_setaffinity(0, sizeof(mask), &mask)) 85 error(1, 0, "setaffinity %d", cpu); 86 87 return 0; 88 } 89 90 static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr) 91 { 92 struct sockaddr_in6 *addr6 = (void *) sockaddr; 93 struct sockaddr_in *addr4 = (void *) sockaddr; 94 95 switch (domain) { 96 case PF_INET: 97 addr4->sin_family = AF_INET; 98 addr4->sin_port = htons(cfg_port); 99 if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) 100 error(1, 0, "ipv4 parse error: %s", str_addr); 101 break; 102 case PF_INET6: 103 addr6->sin6_family = AF_INET6; 104 addr6->sin6_port = htons(cfg_port); 105 if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) 106 error(1, 0, "ipv6 parse error: %s", str_addr); 107 break; 108 default: 109 error(1, 0, "illegal domain"); 110 } 111 } 112 113 static void flush_zerocopy(int fd) 114 { 115 struct msghdr msg = {0}; /* flush */ 116 int ret; 117 118 while (1) { 119 ret = recvmsg(fd, &msg, MSG_ERRQUEUE); 120 if (ret == -1 && errno == EAGAIN) 121 break; 122 if (ret == -1) 123 error(1, errno, "errqueue"); 124 if (msg.msg_flags != (MSG_ERRQUEUE | MSG_CTRUNC)) 125 error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags); 126 msg.msg_flags = 0; 127 } 128 } 129 130 static int send_tcp(int fd, char *data) 131 { 132 int ret, done = 0, count = 0; 133 134 while (done < cfg_payload_len) { 135 ret = send(fd, data + done, cfg_payload_len - done, 136 cfg_zerocopy ? MSG_ZEROCOPY : 0); 137 if (ret == -1) 138 error(1, errno, "write"); 139 140 done += ret; 141 count++; 142 } 143 144 return count; 145 } 146 147 static int send_udp(int fd, char *data) 148 { 149 int ret, total_len, len, count = 0; 150 151 total_len = cfg_payload_len; 152 153 while (total_len) { 154 len = total_len < cfg_mss ? total_len : cfg_mss; 155 156 ret = sendto(fd, data, len, cfg_zerocopy ? MSG_ZEROCOPY : 0, 157 cfg_connected ? NULL : (void *)&cfg_dst_addr, 158 cfg_connected ? 0 : cfg_alen); 159 if (ret == -1) 160 error(1, errno, "write"); 161 if (ret != len) 162 error(1, errno, "write: %uB != %uB\n", ret, len); 163 164 total_len -= len; 165 count++; 166 } 167 168 return count; 169 } 170 171 static int send_udp_sendmmsg(int fd, char *data) 172 { 173 const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN; 174 struct mmsghdr mmsgs[max_nr_msg]; 175 struct iovec iov[max_nr_msg]; 176 unsigned int off = 0, left; 177 int i = 0, ret; 178 179 memset(mmsgs, 0, sizeof(mmsgs)); 180 181 left = cfg_payload_len; 182 while (left) { 183 if (i == max_nr_msg) 184 error(1, 0, "sendmmsg: exceeds max_nr_msg"); 185 186 iov[i].iov_base = data + off; 187 iov[i].iov_len = cfg_mss < left ? cfg_mss : left; 188 189 mmsgs[i].msg_hdr.msg_iov = iov + i; 190 mmsgs[i].msg_hdr.msg_iovlen = 1; 191 192 off += iov[i].iov_len; 193 left -= iov[i].iov_len; 194 i++; 195 } 196 197 ret = sendmmsg(fd, mmsgs, i, cfg_zerocopy ? MSG_ZEROCOPY : 0); 198 if (ret == -1) 199 error(1, errno, "sendmmsg"); 200 201 return ret; 202 } 203 204 static void send_udp_segment_cmsg(struct cmsghdr *cm) 205 { 206 uint16_t *valp; 207 208 cm->cmsg_level = SOL_UDP; 209 cm->cmsg_type = UDP_SEGMENT; 210 cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size)); 211 valp = (void *)CMSG_DATA(cm); 212 *valp = cfg_gso_size; 213 } 214 215 static int send_udp_segment(int fd, char *data) 216 { 217 char control[CMSG_SPACE(sizeof(cfg_gso_size))] = {0}; 218 struct msghdr msg = {0}; 219 struct iovec iov = {0}; 220 int ret; 221 222 iov.iov_base = data; 223 iov.iov_len = cfg_payload_len; 224 225 msg.msg_iov = &iov; 226 msg.msg_iovlen = 1; 227 228 msg.msg_control = control; 229 msg.msg_controllen = sizeof(control); 230 send_udp_segment_cmsg(CMSG_FIRSTHDR(&msg)); 231 232 msg.msg_name = (void *)&cfg_dst_addr; 233 msg.msg_namelen = cfg_alen; 234 235 ret = sendmsg(fd, &msg, cfg_zerocopy ? MSG_ZEROCOPY : 0); 236 if (ret == -1) 237 error(1, errno, "sendmsg"); 238 if (ret != iov.iov_len) 239 error(1, 0, "sendmsg: %u != %lu\n", ret, iov.iov_len); 240 241 return 1; 242 } 243 244 static void usage(const char *filepath) 245 { 246 error(1, 0, "Usage: %s [-46cmtuz] [-C cpu] [-D dst ip] [-l secs] [-m messagenr] [-p port] [-s sendsize] [-S gsosize]", 247 filepath); 248 } 249 250 static void parse_opts(int argc, char **argv) 251 { 252 int max_len, hdrlen; 253 int c; 254 255 while ((c = getopt(argc, argv, "46cC:D:l:mM:p:s:S:tuz")) != -1) { 256 switch (c) { 257 case '4': 258 if (cfg_family != PF_UNSPEC) 259 error(1, 0, "Pass one of -4 or -6"); 260 cfg_family = PF_INET; 261 cfg_alen = sizeof(struct sockaddr_in); 262 break; 263 case '6': 264 if (cfg_family != PF_UNSPEC) 265 error(1, 0, "Pass one of -4 or -6"); 266 cfg_family = PF_INET6; 267 cfg_alen = sizeof(struct sockaddr_in6); 268 break; 269 case 'c': 270 cfg_cache_trash = true; 271 break; 272 case 'C': 273 cfg_cpu = strtol(optarg, NULL, 0); 274 break; 275 case 'D': 276 setup_sockaddr(cfg_family, optarg, &cfg_dst_addr); 277 break; 278 case 'l': 279 cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000; 280 break; 281 case 'm': 282 cfg_sendmmsg = true; 283 break; 284 case 'M': 285 cfg_msg_nr = strtoul(optarg, NULL, 10); 286 break; 287 case 'p': 288 cfg_port = strtoul(optarg, NULL, 0); 289 break; 290 case 's': 291 cfg_payload_len = strtoul(optarg, NULL, 0); 292 break; 293 case 'S': 294 cfg_gso_size = strtoul(optarg, NULL, 0); 295 cfg_segment = true; 296 break; 297 case 't': 298 cfg_tcp = true; 299 break; 300 case 'u': 301 cfg_connected = false; 302 break; 303 case 'z': 304 cfg_zerocopy = true; 305 break; 306 } 307 } 308 309 if (optind != argc) 310 usage(argv[0]); 311 312 if (cfg_family == PF_UNSPEC) 313 error(1, 0, "must pass one of -4 or -6"); 314 if (cfg_tcp && !cfg_connected) 315 error(1, 0, "connectionless tcp makes no sense"); 316 if (cfg_segment && cfg_sendmmsg) 317 error(1, 0, "cannot combine segment offload and sendmmsg"); 318 319 if (cfg_family == PF_INET) 320 hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr); 321 else 322 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct udphdr); 323 324 cfg_mss = ETH_DATA_LEN - hdrlen; 325 max_len = ETH_MAX_MTU - hdrlen; 326 if (!cfg_gso_size) 327 cfg_gso_size = cfg_mss; 328 329 if (cfg_payload_len > max_len) 330 error(1, 0, "payload length %u exceeds max %u", 331 cfg_payload_len, max_len); 332 } 333 334 static void set_pmtu_discover(int fd, bool is_ipv4) 335 { 336 int level, name, val; 337 338 if (is_ipv4) { 339 level = SOL_IP; 340 name = IP_MTU_DISCOVER; 341 val = IP_PMTUDISC_DO; 342 } else { 343 level = SOL_IPV6; 344 name = IPV6_MTU_DISCOVER; 345 val = IPV6_PMTUDISC_DO; 346 } 347 348 if (setsockopt(fd, level, name, &val, sizeof(val))) 349 error(1, errno, "setsockopt path mtu"); 350 } 351 352 int main(int argc, char **argv) 353 { 354 unsigned long num_msgs, num_sends; 355 unsigned long tnow, treport, tstop; 356 int fd, i, val; 357 358 parse_opts(argc, argv); 359 360 if (cfg_cpu > 0) 361 set_cpu(cfg_cpu); 362 363 for (i = 0; i < sizeof(buf[0]); i++) 364 buf[0][i] = 'a' + (i % 26); 365 for (i = 1; i < NUM_PKT; i++) 366 memcpy(buf[i], buf[0], sizeof(buf[0])); 367 368 signal(SIGINT, sigint_handler); 369 370 fd = socket(cfg_family, cfg_tcp ? SOCK_STREAM : SOCK_DGRAM, 0); 371 if (fd == -1) 372 error(1, errno, "socket"); 373 374 if (cfg_zerocopy) { 375 val = 1; 376 if (setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) 377 error(1, errno, "setsockopt zerocopy"); 378 } 379 380 if (cfg_connected && 381 connect(fd, (void *)&cfg_dst_addr, cfg_alen)) 382 error(1, errno, "connect"); 383 384 if (cfg_segment) 385 set_pmtu_discover(fd, cfg_family == PF_INET); 386 387 num_msgs = num_sends = 0; 388 tnow = gettimeofday_ms(); 389 tstop = tnow + cfg_runtime_ms; 390 treport = tnow + 1000; 391 392 i = 0; 393 do { 394 if (cfg_tcp) 395 num_sends += send_tcp(fd, buf[i]); 396 else if (cfg_segment) 397 num_sends += send_udp_segment(fd, buf[i]); 398 else if (cfg_sendmmsg) 399 num_sends += send_udp_sendmmsg(fd, buf[i]); 400 else 401 num_sends += send_udp(fd, buf[i]); 402 num_msgs++; 403 if (cfg_zerocopy && ((num_msgs & 0xF) == 0)) 404 flush_zerocopy(fd); 405 406 if (cfg_msg_nr && num_msgs >= cfg_msg_nr) 407 break; 408 409 tnow = gettimeofday_ms(); 410 if (tnow > treport) { 411 fprintf(stderr, 412 "%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n", 413 cfg_tcp ? "tcp" : "udp", 414 (num_msgs * cfg_payload_len) >> 20, 415 num_sends, num_msgs); 416 num_msgs = num_sends = 0; 417 treport = tnow + 1000; 418 } 419 420 /* cold cache when writing buffer */ 421 if (cfg_cache_trash) 422 i = ++i < NUM_PKT ? i : 0; 423 424 } while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop)); 425 426 if (close(fd)) 427 error(1, errno, "close"); 428 429 return 0; 430 } 431