1 // SPDX-License-Identifier: GPL-2.0 2 /* Toeplitz test 3 * 4 * 1. Read packets and their rx_hash using PF_PACKET/TPACKET_V3 5 * 2. Compute the rx_hash in software based on the packet contents 6 * 3. Compare the two 7 * 8 * Optionally, either '-C $rx_irq_cpu_list' or '-r $rps_bitmap' may be given. 9 * 10 * If '-C $rx_irq_cpu_list' is given, also 11 * 12 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU 13 * 5. Compute the rxqueue that RSS would select based on this rx_hash 14 * 6. Using the $rx_irq_cpu_list map, identify the arriving cpu based on rxq irq 15 * 7. Compare the cpus from 4 and 6 16 * 17 * Else if '-r $rps_bitmap' is given, also 18 * 19 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU 20 * 5. Compute the cpu that RPS should select based on rx_hash and $rps_bitmap 21 * 6. Compare the cpus from 4 and 5 22 */ 23 24 #define _GNU_SOURCE 25 26 #include <arpa/inet.h> 27 #include <errno.h> 28 #include <error.h> 29 #include <fcntl.h> 30 #include <getopt.h> 31 #include <linux/filter.h> 32 #include <linux/if_ether.h> 33 #include <linux/if_packet.h> 34 #include <net/if.h> 35 #include <netdb.h> 36 #include <netinet/ip.h> 37 #include <netinet/ip6.h> 38 #include <netinet/tcp.h> 39 #include <netinet/udp.h> 40 #include <poll.h> 41 #include <stdbool.h> 42 #include <stddef.h> 43 #include <stdint.h> 44 #include <stdio.h> 45 #include <stdlib.h> 46 #include <string.h> 47 #include <sys/mman.h> 48 #include <sys/socket.h> 49 #include <sys/stat.h> 50 #include <sys/sysinfo.h> 51 #include <sys/time.h> 52 #include <sys/types.h> 53 #include <unistd.h> 54 55 #include "../kselftest.h" 56 57 #define TOEPLITZ_KEY_MIN_LEN 40 58 #define TOEPLITZ_KEY_MAX_LEN 60 59 60 #define TOEPLITZ_STR_LEN(K) (((K) * 3) - 1) /* hex encoded: AA:BB:CC:...:ZZ */ 61 #define TOEPLITZ_STR_MIN_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MIN_LEN) 62 #define TOEPLITZ_STR_MAX_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MAX_LEN) 63 64 #define FOUR_TUPLE_MAX_LEN ((sizeof(struct in6_addr) * 2) + (sizeof(uint16_t) * 2)) 65 66 #define RSS_MAX_CPUS (1 << 16) /* real constraint is PACKET_FANOUT_MAX */ 67 68 #define RPS_MAX_CPUS 16UL /* must be a power of 2 */ 69 70 /* configuration options (cmdline arguments) */ 71 static uint16_t cfg_dport = 8000; 72 static int cfg_family = AF_INET6; 73 static char *cfg_ifname = "eth0"; 74 static int cfg_num_queues; 75 static int cfg_num_rps_cpus; 76 static bool cfg_sink; 77 static int cfg_type = SOCK_STREAM; 78 static int cfg_timeout_msec = 1000; 79 static bool cfg_verbose; 80 81 /* global vars */ 82 static int num_cpus; 83 static int ring_block_nr; 84 static int ring_block_sz; 85 86 /* stats */ 87 static int frames_received; 88 static int frames_nohash; 89 static int frames_error; 90 91 #define log_verbose(args...) do { if (cfg_verbose) fprintf(stderr, args); } while (0) 92 93 /* tpacket ring */ 94 struct ring_state { 95 int fd; 96 char *mmap; 97 int idx; 98 int cpu; 99 }; 100 101 static unsigned int rx_irq_cpus[RSS_MAX_CPUS]; /* map from rxq to cpu */ 102 static int rps_silo_to_cpu[RPS_MAX_CPUS]; 103 static unsigned char toeplitz_key[TOEPLITZ_KEY_MAX_LEN]; 104 static struct ring_state rings[RSS_MAX_CPUS]; 105 toeplitz(const unsigned char * four_tuple,const unsigned char * key)106 static inline uint32_t toeplitz(const unsigned char *four_tuple, 107 const unsigned char *key) 108 { 109 int i, bit, ret = 0; 110 uint32_t key32; 111 112 key32 = ntohl(*((uint32_t *)key)); 113 key += 4; 114 115 for (i = 0; i < FOUR_TUPLE_MAX_LEN; i++) { 116 for (bit = 7; bit >= 0; bit--) { 117 if (four_tuple[i] & (1 << bit)) 118 ret ^= key32; 119 120 key32 <<= 1; 121 key32 |= !!(key[0] & (1 << bit)); 122 } 123 key++; 124 } 125 126 return ret; 127 } 128 129 /* Compare computed cpu with arrival cpu from packet_fanout_cpu */ verify_rss(uint32_t rx_hash,int cpu)130 static void verify_rss(uint32_t rx_hash, int cpu) 131 { 132 int queue = rx_hash % cfg_num_queues; 133 134 log_verbose(" rxq %d (cpu %d)", queue, rx_irq_cpus[queue]); 135 if (rx_irq_cpus[queue] != cpu) { 136 log_verbose(". error: rss cpu mismatch (%d)", cpu); 137 frames_error++; 138 } 139 } 140 verify_rps(uint64_t rx_hash,int cpu)141 static void verify_rps(uint64_t rx_hash, int cpu) 142 { 143 int silo = (rx_hash * cfg_num_rps_cpus) >> 32; 144 145 log_verbose(" silo %d (cpu %d)", silo, rps_silo_to_cpu[silo]); 146 if (rps_silo_to_cpu[silo] != cpu) { 147 log_verbose(". error: rps cpu mismatch (%d)", cpu); 148 frames_error++; 149 } 150 } 151 log_rxhash(int cpu,uint32_t rx_hash,const char * addrs,int addr_len)152 static void log_rxhash(int cpu, uint32_t rx_hash, 153 const char *addrs, int addr_len) 154 { 155 char saddr[INET6_ADDRSTRLEN], daddr[INET6_ADDRSTRLEN]; 156 uint16_t *ports; 157 158 if (!inet_ntop(cfg_family, addrs, saddr, sizeof(saddr)) || 159 !inet_ntop(cfg_family, addrs + addr_len, daddr, sizeof(daddr))) 160 error(1, 0, "address parse error"); 161 162 ports = (void *)addrs + (addr_len * 2); 163 log_verbose("cpu %d: rx_hash 0x%08x [saddr %s daddr %s sport %02hu dport %02hu]", 164 cpu, rx_hash, saddr, daddr, 165 ntohs(ports[0]), ntohs(ports[1])); 166 } 167 168 /* Compare computed rxhash with rxhash received from tpacket_v3 */ verify_rxhash(const char * pkt,uint32_t rx_hash,int cpu)169 static void verify_rxhash(const char *pkt, uint32_t rx_hash, int cpu) 170 { 171 unsigned char four_tuple[FOUR_TUPLE_MAX_LEN] = {0}; 172 uint32_t rx_hash_sw; 173 const char *addrs; 174 int addr_len; 175 176 if (cfg_family == AF_INET) { 177 addr_len = sizeof(struct in_addr); 178 addrs = pkt + offsetof(struct iphdr, saddr); 179 } else { 180 addr_len = sizeof(struct in6_addr); 181 addrs = pkt + offsetof(struct ip6_hdr, ip6_src); 182 } 183 184 memcpy(four_tuple, addrs, (addr_len * 2) + (sizeof(uint16_t) * 2)); 185 rx_hash_sw = toeplitz(four_tuple, toeplitz_key); 186 187 if (cfg_verbose) 188 log_rxhash(cpu, rx_hash, addrs, addr_len); 189 190 if (rx_hash != rx_hash_sw) { 191 log_verbose(" != expected 0x%x\n", rx_hash_sw); 192 frames_error++; 193 return; 194 } 195 196 log_verbose(" OK"); 197 if (cfg_num_queues) 198 verify_rss(rx_hash, cpu); 199 else if (cfg_num_rps_cpus) 200 verify_rps(rx_hash, cpu); 201 log_verbose("\n"); 202 } 203 recv_frame(const struct ring_state * ring,char * frame)204 static char *recv_frame(const struct ring_state *ring, char *frame) 205 { 206 struct tpacket3_hdr *hdr = (void *)frame; 207 208 if (hdr->hv1.tp_rxhash) 209 verify_rxhash(frame + hdr->tp_net, hdr->hv1.tp_rxhash, 210 ring->cpu); 211 else 212 frames_nohash++; 213 214 return frame + hdr->tp_next_offset; 215 } 216 217 /* A single TPACKET_V3 block can hold multiple frames */ recv_block(struct ring_state * ring)218 static bool recv_block(struct ring_state *ring) 219 { 220 struct tpacket_block_desc *block; 221 char *frame; 222 int i; 223 224 block = (void *)(ring->mmap + ring->idx * ring_block_sz); 225 if (!(block->hdr.bh1.block_status & TP_STATUS_USER)) 226 return false; 227 228 frame = (char *)block; 229 frame += block->hdr.bh1.offset_to_first_pkt; 230 231 for (i = 0; i < block->hdr.bh1.num_pkts; i++) { 232 frame = recv_frame(ring, frame); 233 frames_received++; 234 } 235 236 block->hdr.bh1.block_status = TP_STATUS_KERNEL; 237 ring->idx = (ring->idx + 1) % ring_block_nr; 238 239 return true; 240 } 241 242 /* simple test: sleep once unconditionally and then process all rings */ process_rings(void)243 static void process_rings(void) 244 { 245 int i; 246 247 usleep(1000 * cfg_timeout_msec); 248 249 for (i = 0; i < num_cpus; i++) 250 do {} while (recv_block(&rings[i])); 251 252 fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n", 253 frames_received - frames_nohash - frames_error, 254 frames_nohash, frames_error); 255 } 256 setup_ring(int fd)257 static char *setup_ring(int fd) 258 { 259 struct tpacket_req3 req3 = {0}; 260 void *ring; 261 262 req3.tp_retire_blk_tov = cfg_timeout_msec / 8; 263 req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH; 264 265 req3.tp_frame_size = 2048; 266 req3.tp_frame_nr = 1 << 10; 267 req3.tp_block_nr = 16; 268 269 req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr; 270 req3.tp_block_size /= req3.tp_block_nr; 271 272 if (setsockopt(fd, SOL_PACKET, PACKET_RX_RING, &req3, sizeof(req3))) 273 error(1, errno, "setsockopt PACKET_RX_RING"); 274 275 ring_block_sz = req3.tp_block_size; 276 ring_block_nr = req3.tp_block_nr; 277 278 ring = mmap(0, req3.tp_block_size * req3.tp_block_nr, 279 PROT_READ | PROT_WRITE, 280 MAP_SHARED | MAP_LOCKED | MAP_POPULATE, fd, 0); 281 if (ring == MAP_FAILED) 282 error(1, 0, "mmap failed"); 283 284 return ring; 285 } 286 __set_filter(int fd,int off_proto,uint8_t proto,int off_dport)287 static void __set_filter(int fd, int off_proto, uint8_t proto, int off_dport) 288 { 289 struct sock_filter filter[] = { 290 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, SKF_AD_OFF + SKF_AD_PKTTYPE), 291 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, PACKET_HOST, 0, 4), 292 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, off_proto), 293 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, proto, 0, 2), 294 BPF_STMT(BPF_LD + BPF_H + BPF_ABS, off_dport), 295 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, cfg_dport, 1, 0), 296 BPF_STMT(BPF_RET + BPF_K, 0), 297 BPF_STMT(BPF_RET + BPF_K, 0xFFFF), 298 }; 299 struct sock_fprog prog = {}; 300 301 prog.filter = filter; 302 prog.len = ARRAY_SIZE(filter); 303 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog))) 304 error(1, errno, "setsockopt filter"); 305 } 306 307 /* filter on transport protocol and destination port */ set_filter(int fd)308 static void set_filter(int fd) 309 { 310 const int off_dport = offsetof(struct tcphdr, dest); /* same for udp */ 311 uint8_t proto; 312 313 proto = cfg_type == SOCK_STREAM ? IPPROTO_TCP : IPPROTO_UDP; 314 if (cfg_family == AF_INET) 315 __set_filter(fd, offsetof(struct iphdr, protocol), proto, 316 sizeof(struct iphdr) + off_dport); 317 else 318 __set_filter(fd, offsetof(struct ip6_hdr, ip6_nxt), proto, 319 sizeof(struct ip6_hdr) + off_dport); 320 } 321 322 /* drop everything: used temporarily during setup */ set_filter_null(int fd)323 static void set_filter_null(int fd) 324 { 325 struct sock_filter filter[] = { 326 BPF_STMT(BPF_RET + BPF_K, 0), 327 }; 328 struct sock_fprog prog = {}; 329 330 prog.filter = filter; 331 prog.len = ARRAY_SIZE(filter); 332 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog))) 333 error(1, errno, "setsockopt filter"); 334 } 335 create_ring(char ** ring)336 static int create_ring(char **ring) 337 { 338 struct fanout_args args = { 339 .id = 1, 340 .type_flags = PACKET_FANOUT_CPU, 341 .max_num_members = RSS_MAX_CPUS 342 }; 343 struct sockaddr_ll ll = { 0 }; 344 int fd, val; 345 346 fd = socket(PF_PACKET, SOCK_DGRAM, 0); 347 if (fd == -1) 348 error(1, errno, "socket creation failed"); 349 350 val = TPACKET_V3; 351 if (setsockopt(fd, SOL_PACKET, PACKET_VERSION, &val, sizeof(val))) 352 error(1, errno, "setsockopt PACKET_VERSION"); 353 *ring = setup_ring(fd); 354 355 /* block packets until all rings are added to the fanout group: 356 * else packets can arrive during setup and get misclassified 357 */ 358 set_filter_null(fd); 359 360 ll.sll_family = AF_PACKET; 361 ll.sll_ifindex = if_nametoindex(cfg_ifname); 362 ll.sll_protocol = cfg_family == AF_INET ? htons(ETH_P_IP) : 363 htons(ETH_P_IPV6); 364 if (bind(fd, (void *)&ll, sizeof(ll))) 365 error(1, errno, "bind"); 366 367 /* must come after bind: verifies all programs in group match */ 368 if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &args, sizeof(args))) { 369 /* on failure, retry using old API if that is sufficient: 370 * it has a hard limit of 256 sockets, so only try if 371 * (a) only testing rxhash, not RSS or (b) <= 256 cpus. 372 * in this API, the third argument is left implicit. 373 */ 374 if (cfg_num_queues || num_cpus > 256 || 375 setsockopt(fd, SOL_PACKET, PACKET_FANOUT, 376 &args, sizeof(uint32_t))) 377 error(1, errno, "setsockopt PACKET_FANOUT cpu"); 378 } 379 380 return fd; 381 } 382 383 /* setup inet(6) socket to blackhole the test traffic, if arg '-s' */ setup_sink(void)384 static int setup_sink(void) 385 { 386 int fd, val; 387 388 fd = socket(cfg_family, cfg_type, 0); 389 if (fd == -1) 390 error(1, errno, "socket %d.%d", cfg_family, cfg_type); 391 392 val = 1 << 20; 393 if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &val, sizeof(val))) 394 error(1, errno, "setsockopt rcvbuf"); 395 396 return fd; 397 } 398 setup_rings(void)399 static void setup_rings(void) 400 { 401 int i; 402 403 for (i = 0; i < num_cpus; i++) { 404 rings[i].cpu = i; 405 rings[i].fd = create_ring(&rings[i].mmap); 406 } 407 408 /* accept packets once all rings in the fanout group are up */ 409 for (i = 0; i < num_cpus; i++) 410 set_filter(rings[i].fd); 411 } 412 cleanup_rings(void)413 static void cleanup_rings(void) 414 { 415 int i; 416 417 for (i = 0; i < num_cpus; i++) { 418 if (munmap(rings[i].mmap, ring_block_nr * ring_block_sz)) 419 error(1, errno, "munmap"); 420 if (close(rings[i].fd)) 421 error(1, errno, "close"); 422 } 423 } 424 parse_cpulist(const char * arg)425 static void parse_cpulist(const char *arg) 426 { 427 do { 428 rx_irq_cpus[cfg_num_queues++] = strtol(arg, NULL, 10); 429 430 arg = strchr(arg, ','); 431 if (!arg) 432 break; 433 arg++; // skip ',' 434 } while (1); 435 } 436 show_cpulist(void)437 static void show_cpulist(void) 438 { 439 int i; 440 441 for (i = 0; i < cfg_num_queues; i++) 442 fprintf(stderr, "rxq %d: cpu %d\n", i, rx_irq_cpus[i]); 443 } 444 show_silos(void)445 static void show_silos(void) 446 { 447 int i; 448 449 for (i = 0; i < cfg_num_rps_cpus; i++) 450 fprintf(stderr, "silo %d: cpu %d\n", i, rps_silo_to_cpu[i]); 451 } 452 parse_toeplitz_key(const char * str,int slen,unsigned char * key)453 static void parse_toeplitz_key(const char *str, int slen, unsigned char *key) 454 { 455 int i, ret, off; 456 457 if (slen < TOEPLITZ_STR_MIN_LEN || 458 slen > TOEPLITZ_STR_MAX_LEN + 1) 459 error(1, 0, "invalid toeplitz key"); 460 461 for (i = 0, off = 0; off < slen; i++, off += 3) { 462 ret = sscanf(str + off, "%hhx", &key[i]); 463 if (ret != 1) 464 error(1, 0, "key parse error at %d off %d len %d", 465 i, off, slen); 466 } 467 } 468 parse_rps_bitmap(const char * arg)469 static void parse_rps_bitmap(const char *arg) 470 { 471 unsigned long bitmap; 472 int i; 473 474 bitmap = strtoul(arg, NULL, 0); 475 476 if (bitmap & ~(RPS_MAX_CPUS - 1)) 477 error(1, 0, "rps bitmap 0x%lx out of bounds 0..%lu", 478 bitmap, RPS_MAX_CPUS - 1); 479 480 for (i = 0; i < RPS_MAX_CPUS; i++) 481 if (bitmap & 1UL << i) 482 rps_silo_to_cpu[cfg_num_rps_cpus++] = i; 483 } 484 parse_opts(int argc,char ** argv)485 static void parse_opts(int argc, char **argv) 486 { 487 static struct option long_options[] = { 488 {"dport", required_argument, 0, 'd'}, 489 {"cpus", required_argument, 0, 'C'}, 490 {"key", required_argument, 0, 'k'}, 491 {"iface", required_argument, 0, 'i'}, 492 {"ipv4", no_argument, 0, '4'}, 493 {"ipv6", no_argument, 0, '6'}, 494 {"sink", no_argument, 0, 's'}, 495 {"tcp", no_argument, 0, 't'}, 496 {"timeout", required_argument, 0, 'T'}, 497 {"udp", no_argument, 0, 'u'}, 498 {"verbose", no_argument, 0, 'v'}, 499 {"rps", required_argument, 0, 'r'}, 500 {0, 0, 0, 0} 501 }; 502 bool have_toeplitz = false; 503 int index, c; 504 505 while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) { 506 switch (c) { 507 case '4': 508 cfg_family = AF_INET; 509 break; 510 case '6': 511 cfg_family = AF_INET6; 512 break; 513 case 'C': 514 parse_cpulist(optarg); 515 break; 516 case 'd': 517 cfg_dport = strtol(optarg, NULL, 0); 518 break; 519 case 'i': 520 cfg_ifname = optarg; 521 break; 522 case 'k': 523 parse_toeplitz_key(optarg, strlen(optarg), 524 toeplitz_key); 525 have_toeplitz = true; 526 break; 527 case 'r': 528 parse_rps_bitmap(optarg); 529 break; 530 case 's': 531 cfg_sink = true; 532 break; 533 case 't': 534 cfg_type = SOCK_STREAM; 535 break; 536 case 'T': 537 cfg_timeout_msec = strtol(optarg, NULL, 0); 538 break; 539 case 'u': 540 cfg_type = SOCK_DGRAM; 541 break; 542 case 'v': 543 cfg_verbose = true; 544 break; 545 546 default: 547 error(1, 0, "unknown option %c", optopt); 548 break; 549 } 550 } 551 552 if (!have_toeplitz) 553 error(1, 0, "Must supply rss key ('-k')"); 554 555 num_cpus = get_nprocs(); 556 if (num_cpus > RSS_MAX_CPUS) 557 error(1, 0, "increase RSS_MAX_CPUS"); 558 559 if (cfg_num_queues && cfg_num_rps_cpus) 560 error(1, 0, 561 "Can't supply both RSS cpus ('-C') and RPS map ('-r')"); 562 if (cfg_verbose) { 563 show_cpulist(); 564 show_silos(); 565 } 566 } 567 main(int argc,char ** argv)568 int main(int argc, char **argv) 569 { 570 const int min_tests = 10; 571 int fd_sink = -1; 572 573 parse_opts(argc, argv); 574 575 if (cfg_sink) 576 fd_sink = setup_sink(); 577 578 setup_rings(); 579 process_rings(); 580 cleanup_rings(); 581 582 if (cfg_sink && close(fd_sink)) 583 error(1, errno, "close sink"); 584 585 if (frames_received - frames_nohash < min_tests) 586 error(1, 0, "too few frames for verification"); 587 588 return frames_error; 589 } 590