1 // SPDX-License-Identifier: GPL-2.0-only 2 #define _GNU_SOURCE 3 4 #include <arpa/inet.h> 5 #include <bpf/bpf.h> 6 #include <bpf/libbpf.h> 7 #include <errno.h> 8 #include <fcntl.h> 9 #include <getopt.h> 10 #include <linux/ethtool.h> 11 #include <linux/hashtable.h> 12 #include <linux/if_link.h> 13 #include <linux/jhash.h> 14 #include <linux/limits.h> 15 #include <linux/list.h> 16 #include <linux/sockios.h> 17 #include <locale.h> 18 #include <math.h> 19 #include <net/if.h> 20 #include <poll.h> 21 #include <signal.h> 22 #include <stdbool.h> 23 #include <stdio.h> 24 #include <stdlib.h> 25 #include <string.h> 26 #include <sys/ioctl.h> 27 #include <sys/mman.h> 28 #include <sys/signalfd.h> 29 #include <sys/sysinfo.h> 30 #include <sys/timerfd.h> 31 #include <sys/utsname.h> 32 #include <time.h> 33 #include <unistd.h> 34 35 #include "bpf_util.h" 36 #include "xdp_sample_user.h" 37 38 #define __sample_print(fmt, cond, ...) \ 39 ({ \ 40 if (cond) \ 41 printf(fmt, ##__VA_ARGS__); \ 42 }) 43 44 #define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__) 45 #define print_default(fmt, ...) \ 46 __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__) 47 #define __print_err(err, fmt, ...) \ 48 ({ \ 49 __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \ 50 ##__VA_ARGS__); \ 51 sample_err_exp = sample_err_exp ? true : err > 0; \ 52 }) 53 #define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__) 54 55 #define __COLUMN(x) "%'10" x " %-13s" 56 #define FMT_COLUMNf __COLUMN(".0f") 57 #define FMT_COLUMNd __COLUMN("d") 58 #define FMT_COLUMNl __COLUMN("llu") 59 #define RX(rx) rx, "rx/s" 60 #define PPS(pps) pps, "pkt/s" 61 #define DROP(drop) drop, "drop/s" 62 #define ERR(err) err, "error/s" 63 #define HITS(hits) hits, "hit/s" 64 #define XMIT(xmit) xmit, "xmit/s" 65 #define PASS(pass) pass, "pass/s" 66 #define REDIR(redir) redir, "redir/s" 67 #define NANOSEC_PER_SEC 1000000000 /* 10^9 */ 68 69 #define XDP_UNKNOWN (XDP_REDIRECT + 1) 70 #define XDP_ACTION_MAX (XDP_UNKNOWN + 1) 71 #define XDP_REDIRECT_ERR_MAX 7 72 73 enum map_type { 74 MAP_RX, 75 MAP_REDIRECT_ERR, 76 MAP_CPUMAP_ENQUEUE, 77 MAP_CPUMAP_KTHREAD, 78 MAP_EXCEPTION, 79 MAP_DEVMAP_XMIT, 80 MAP_DEVMAP_XMIT_MULTI, 81 NUM_MAP, 82 }; 83 84 enum log_level { 85 LL_DEFAULT = 1U << 0, 86 LL_SIMPLE = 1U << 1, 87 LL_DEBUG = 1U << 2, 88 }; 89 90 struct record { 91 __u64 timestamp; 92 struct datarec total; 93 struct datarec *cpu; 94 }; 95 96 struct map_entry { 97 struct hlist_node node; 98 __u64 pair; 99 struct record val; 100 }; 101 102 struct stats_record { 103 struct record rx_cnt; 104 struct record redir_err[XDP_REDIRECT_ERR_MAX]; 105 struct record kthread; 106 struct record exception[XDP_ACTION_MAX]; 107 struct record devmap_xmit; 108 DECLARE_HASHTABLE(xmit_map, 5); 109 struct record enq[]; 110 }; 111 112 struct sample_output { 113 struct { 114 __u64 rx; 115 __u64 redir; 116 __u64 drop; 117 __u64 drop_xmit; 118 __u64 err; 119 __u64 xmit; 120 } totals; 121 struct { 122 union { 123 __u64 pps; 124 __u64 num; 125 }; 126 __u64 drop; 127 __u64 err; 128 } rx_cnt; 129 struct { 130 __u64 suc; 131 __u64 err; 132 } redir_cnt; 133 struct { 134 __u64 hits; 135 } except_cnt; 136 struct { 137 __u64 pps; 138 __u64 drop; 139 __u64 err; 140 double bavg; 141 } xmit_cnt; 142 }; 143 144 struct xdp_desc { 145 int ifindex; 146 __u32 prog_id; 147 int flags; 148 } sample_xdp_progs[32]; 149 150 struct datarec *sample_mmap[NUM_MAP]; 151 struct bpf_map *sample_map[NUM_MAP]; 152 size_t sample_map_count[NUM_MAP]; 153 enum log_level sample_log_level; 154 struct sample_output sample_out; 155 unsigned long sample_interval; 156 bool sample_err_exp; 157 int sample_xdp_cnt; 158 int sample_n_cpus; 159 int sample_sig_fd; 160 int sample_mask; 161 162 static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = { 163 /* Key=1 keeps unknown errors */ 164 "Success", 165 "Unknown", 166 "EINVAL", 167 "ENETDOWN", 168 "EMSGSIZE", 169 "EOPNOTSUPP", 170 "ENOSPC", 171 }; 172 173 /* Keyed from Unknown */ 174 static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = { 175 "Unknown error", 176 "Invalid redirection", 177 "Device being redirected to is down", 178 "Packet length too large for device", 179 "Operation not supported", 180 "No space in ptr_ring of cpumap kthread", 181 }; 182 183 static const char *xdp_action_names[XDP_ACTION_MAX] = { 184 [XDP_ABORTED] = "XDP_ABORTED", 185 [XDP_DROP] = "XDP_DROP", 186 [XDP_PASS] = "XDP_PASS", 187 [XDP_TX] = "XDP_TX", 188 [XDP_REDIRECT] = "XDP_REDIRECT", 189 [XDP_UNKNOWN] = "XDP_UNKNOWN", 190 }; 191 192 static __u64 gettime(void) 193 { 194 struct timespec t; 195 int res; 196 197 res = clock_gettime(CLOCK_MONOTONIC, &t); 198 if (res < 0) { 199 fprintf(stderr, "Error with gettimeofday! (%i)\n", res); 200 return UINT64_MAX; 201 } 202 return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec; 203 } 204 205 static const char *action2str(int action) 206 { 207 if (action < XDP_ACTION_MAX) 208 return xdp_action_names[action]; 209 return NULL; 210 } 211 212 static void sample_print_help(int mask) 213 { 214 printf("Output format description\n\n" 215 "By default, redirect success statistics are disabled, use -s to enable.\n" 216 "The terse output mode is default, verbose mode can be activated using -v\n" 217 "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n" 218 "Terse mode displays at most the following fields:\n" 219 " rx/s Number of packets received per second\n" 220 " redir/s Number of packets successfully redirected per second\n" 221 " err,drop/s Aggregated count of errors per second (including dropped packets)\n" 222 " xmit/s Number of packets transmitted on the output device per second\n\n" 223 "Output description for verbose mode:\n" 224 " FIELD DESCRIPTION\n"); 225 226 if (mask & SAMPLE_RX_CNT) { 227 printf(" receive\t\tDisplays the number of packets received & errors encountered\n" 228 " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n" 229 " \t\t\tand drop statistics will be expanded inline in terse mode.\n" 230 " \t\t\t\tpkt/s - Packets received per second\n" 231 " \t\t\t\tdrop/s - Packets dropped per second\n" 232 " \t\t\t\terror/s - Errors encountered per second\n\n"); 233 } 234 if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { 235 printf(" redirect\t\tDisplays the number of packets successfully redirected\n" 236 " \t\t\tErrors encountered are expanded under redirect_err field\n" 237 " \t\t\tNote that passing -s to enable it has a per packet overhead\n" 238 " \t\t\t\tredir/s - Packets redirected successfully per second\n\n" 239 " redirect_err\t\tDisplays the number of packets that failed redirection\n" 240 " \t\t\tThe errno is expanded under this field with per CPU count\n" 241 " \t\t\tThe recognized errors are:\n"); 242 243 for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++) 244 printf("\t\t\t %s: %s\n", xdp_redirect_err_names[i], 245 xdp_redirect_err_help[i - 1]); 246 247 printf(" \n\t\t\t\terror/s - Packets that failed redirection per second\n\n"); 248 } 249 250 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) { 251 printf(" enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n" 252 " \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n" 253 " \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n" 254 " \t\t\tpackets to.\n" 255 " \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n" 256 " \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n" 257 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); 258 } 259 260 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) { 261 printf(" kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n" 262 " \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n" 263 " \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n" 264 " \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n" 265 " \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n" 266 " \t\t\t\tdrop/s - Packets dropped per second in kthread\n" 267 " \t\t\t\tsched - Number of times kthread called schedule()\n\n" 268 " \t\t\txdp_stats (also expands to per-CPU counts)\n" 269 " \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n" 270 " \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n" 271 " \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n"); 272 } 273 274 if (mask & SAMPLE_EXCEPTION_CNT) { 275 printf(" xdp_exception\t\tDisplays xdp_exception tracepoint events\n" 276 " \t\t\tThis can occur due to internal driver errors, unrecognized\n" 277 " \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n" 278 " \t\t\tEach action is expanded below this field with its count\n" 279 " \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n"); 280 } 281 282 if (mask & SAMPLE_DEVMAP_XMIT_CNT) { 283 printf(" devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n" 284 " \t\t\tThis tracepoint is invoked for successful transmissions on output\n" 285 " \t\t\tdevice but these statistics are not available for generic XDP mode,\n" 286 " \t\t\thence they will be omitted from the output when using SKB mode\n" 287 " \t\t\t\txmit/s - Number of packets that were transmitted per second\n" 288 " \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n" 289 " \t\t\t\tdrv_err/s - Number of internal driver errors per second\n" 290 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); 291 } 292 } 293 294 void sample_usage(char *argv[], const struct option *long_options, 295 const char *doc, int mask, bool error) 296 { 297 int i; 298 299 if (!error) 300 sample_print_help(mask); 301 302 printf("\n%s\nOption for %s:\n", doc, argv[0]); 303 for (i = 0; long_options[i].name != 0; i++) { 304 printf(" --%-15s", long_options[i].name); 305 if (long_options[i].flag != NULL) 306 printf(" flag (internal value: %d)", 307 *long_options[i].flag); 308 else 309 printf("\t short-option: -%c", long_options[i].val); 310 printf("\n"); 311 } 312 printf("\n"); 313 } 314 315 static struct datarec *alloc_record_per_cpu(void) 316 { 317 unsigned int nr_cpus = libbpf_num_possible_cpus(); 318 struct datarec *array; 319 320 array = calloc(nr_cpus, sizeof(*array)); 321 if (!array) { 322 fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n", 323 nr_cpus); 324 return NULL; 325 } 326 return array; 327 } 328 329 static int map_entry_init(struct map_entry *e, __u64 pair) 330 { 331 e->pair = pair; 332 INIT_HLIST_NODE(&e->node); 333 e->val.timestamp = gettime(); 334 e->val.cpu = alloc_record_per_cpu(); 335 if (!e->val.cpu) 336 return -ENOMEM; 337 return 0; 338 } 339 340 static void map_collect_percpu(struct datarec *values, struct record *rec) 341 { 342 /* For percpu maps, userspace gets a value per possible CPU */ 343 unsigned int nr_cpus = libbpf_num_possible_cpus(); 344 __u64 sum_xdp_redirect = 0; 345 __u64 sum_processed = 0; 346 __u64 sum_xdp_pass = 0; 347 __u64 sum_xdp_drop = 0; 348 __u64 sum_dropped = 0; 349 __u64 sum_issue = 0; 350 int i; 351 352 /* Get time as close as possible to reading map contents */ 353 rec->timestamp = gettime(); 354 355 /* Record and sum values from each CPU */ 356 for (i = 0; i < nr_cpus; i++) { 357 rec->cpu[i].processed = READ_ONCE(values[i].processed); 358 rec->cpu[i].dropped = READ_ONCE(values[i].dropped); 359 rec->cpu[i].issue = READ_ONCE(values[i].issue); 360 rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass); 361 rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop); 362 rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect); 363 364 sum_processed += rec->cpu[i].processed; 365 sum_dropped += rec->cpu[i].dropped; 366 sum_issue += rec->cpu[i].issue; 367 sum_xdp_pass += rec->cpu[i].xdp_pass; 368 sum_xdp_drop += rec->cpu[i].xdp_drop; 369 sum_xdp_redirect += rec->cpu[i].xdp_redirect; 370 } 371 372 rec->total.processed = sum_processed; 373 rec->total.dropped = sum_dropped; 374 rec->total.issue = sum_issue; 375 rec->total.xdp_pass = sum_xdp_pass; 376 rec->total.xdp_drop = sum_xdp_drop; 377 rec->total.xdp_redirect = sum_xdp_redirect; 378 } 379 380 static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec) 381 { 382 unsigned int nr_cpus = bpf_num_possible_cpus(); 383 __u32 batch, count = 32; 384 struct datarec *values; 385 bool init = false; 386 __u64 *keys; 387 int i, ret; 388 389 keys = calloc(count, sizeof(__u64)); 390 if (!keys) 391 return -ENOMEM; 392 values = calloc(count * nr_cpus, sizeof(struct datarec)); 393 if (!values) { 394 free(keys); 395 return -ENOMEM; 396 } 397 398 for (;;) { 399 bool exit = false; 400 401 ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch, 402 keys, values, &count, NULL); 403 if (ret < 0 && errno != ENOENT) 404 break; 405 if (errno == ENOENT) 406 exit = true; 407 408 init = true; 409 for (i = 0; i < count; i++) { 410 struct map_entry *e, *x = NULL; 411 __u64 pair = keys[i]; 412 struct datarec *arr; 413 414 arr = &values[i * nr_cpus]; 415 hash_for_each_possible(rec->xmit_map, e, node, pair) { 416 if (e->pair == pair) { 417 x = e; 418 break; 419 } 420 } 421 if (!x) { 422 x = calloc(1, sizeof(*x)); 423 if (!x) 424 goto cleanup; 425 if (map_entry_init(x, pair) < 0) { 426 free(x); 427 goto cleanup; 428 } 429 hash_add(rec->xmit_map, &x->node, pair); 430 } 431 map_collect_percpu(arr, &x->val); 432 } 433 434 if (exit) 435 break; 436 count = 32; 437 } 438 439 free(values); 440 free(keys); 441 return 0; 442 cleanup: 443 free(values); 444 free(keys); 445 return -ENOMEM; 446 } 447 448 static struct stats_record *alloc_stats_record(void) 449 { 450 struct stats_record *rec; 451 int i; 452 453 rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record)); 454 if (!rec) { 455 fprintf(stderr, "Failed to allocate memory\n"); 456 return NULL; 457 } 458 459 if (sample_mask & SAMPLE_RX_CNT) { 460 rec->rx_cnt.cpu = alloc_record_per_cpu(); 461 if (!rec->rx_cnt.cpu) { 462 fprintf(stderr, 463 "Failed to allocate rx_cnt per-CPU array\n"); 464 goto end_rec; 465 } 466 } 467 if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { 468 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) { 469 rec->redir_err[i].cpu = alloc_record_per_cpu(); 470 if (!rec->redir_err[i].cpu) { 471 fprintf(stderr, 472 "Failed to allocate redir_err per-CPU array for " 473 "\"%s\" case\n", 474 xdp_redirect_err_names[i]); 475 while (i--) 476 free(rec->redir_err[i].cpu); 477 goto end_rx_cnt; 478 } 479 } 480 } 481 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) { 482 rec->kthread.cpu = alloc_record_per_cpu(); 483 if (!rec->kthread.cpu) { 484 fprintf(stderr, 485 "Failed to allocate kthread per-CPU array\n"); 486 goto end_redir; 487 } 488 } 489 if (sample_mask & SAMPLE_EXCEPTION_CNT) { 490 for (i = 0; i < XDP_ACTION_MAX; i++) { 491 rec->exception[i].cpu = alloc_record_per_cpu(); 492 if (!rec->exception[i].cpu) { 493 fprintf(stderr, 494 "Failed to allocate exception per-CPU array for " 495 "\"%s\" case\n", 496 action2str(i)); 497 while (i--) 498 free(rec->exception[i].cpu); 499 goto end_kthread; 500 } 501 } 502 } 503 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) { 504 rec->devmap_xmit.cpu = alloc_record_per_cpu(); 505 if (!rec->devmap_xmit.cpu) { 506 fprintf(stderr, 507 "Failed to allocate devmap_xmit per-CPU array\n"); 508 goto end_exception; 509 } 510 } 511 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 512 hash_init(rec->xmit_map); 513 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) { 514 for (i = 0; i < sample_n_cpus; i++) { 515 rec->enq[i].cpu = alloc_record_per_cpu(); 516 if (!rec->enq[i].cpu) { 517 fprintf(stderr, 518 "Failed to allocate enqueue per-CPU array for " 519 "CPU %d\n", 520 i); 521 while (i--) 522 free(rec->enq[i].cpu); 523 goto end_devmap_xmit; 524 } 525 } 526 } 527 528 return rec; 529 530 end_devmap_xmit: 531 free(rec->devmap_xmit.cpu); 532 end_exception: 533 for (i = 0; i < XDP_ACTION_MAX; i++) 534 free(rec->exception[i].cpu); 535 end_kthread: 536 free(rec->kthread.cpu); 537 end_redir: 538 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) 539 free(rec->redir_err[i].cpu); 540 end_rx_cnt: 541 free(rec->rx_cnt.cpu); 542 end_rec: 543 free(rec); 544 return NULL; 545 } 546 547 static void free_stats_record(struct stats_record *r) 548 { 549 struct hlist_node *tmp; 550 struct map_entry *e; 551 int i; 552 553 for (i = 0; i < sample_n_cpus; i++) 554 free(r->enq[i].cpu); 555 hash_for_each_safe(r->xmit_map, i, tmp, e, node) { 556 hash_del(&e->node); 557 free(e->val.cpu); 558 free(e); 559 } 560 free(r->devmap_xmit.cpu); 561 for (i = 0; i < XDP_ACTION_MAX; i++) 562 free(r->exception[i].cpu); 563 free(r->kthread.cpu); 564 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) 565 free(r->redir_err[i].cpu); 566 free(r->rx_cnt.cpu); 567 free(r); 568 } 569 570 static double calc_period(struct record *r, struct record *p) 571 { 572 double period_ = 0; 573 __u64 period = 0; 574 575 period = r->timestamp - p->timestamp; 576 if (period > 0) 577 period_ = ((double)period / NANOSEC_PER_SEC); 578 579 return period_; 580 } 581 582 static double sample_round(double val) 583 { 584 if (val - floor(val) < 0.5) 585 return floor(val); 586 return ceil(val); 587 } 588 589 static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_) 590 { 591 __u64 packets = 0; 592 __u64 pps = 0; 593 594 if (period_ > 0) { 595 packets = r->processed - p->processed; 596 pps = sample_round(packets / period_); 597 } 598 return pps; 599 } 600 601 static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_) 602 { 603 __u64 packets = 0; 604 __u64 pps = 0; 605 606 if (period_ > 0) { 607 packets = r->dropped - p->dropped; 608 pps = sample_round(packets / period_); 609 } 610 return pps; 611 } 612 613 static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_) 614 { 615 __u64 packets = 0; 616 __u64 pps = 0; 617 618 if (period_ > 0) { 619 packets = r->issue - p->issue; 620 pps = sample_round(packets / period_); 621 } 622 return pps; 623 } 624 625 static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_) 626 { 627 __u64 packets = 0; 628 __u64 pps = 0; 629 630 if (period_ > 0) { 631 packets = r->info - p->info; 632 pps = sample_round(packets / period_); 633 } 634 return pps; 635 } 636 637 static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass, 638 double *xdp_drop, double *xdp_redirect, double period_) 639 { 640 *xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0; 641 if (period_ > 0) { 642 *xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_; 643 *xdp_pass = (r->xdp_pass - p->xdp_pass) / period_; 644 *xdp_drop = (r->xdp_drop - p->xdp_drop) / period_; 645 } 646 } 647 648 static void stats_get_rx_cnt(struct stats_record *stats_rec, 649 struct stats_record *stats_prev, 650 unsigned int nr_cpus, struct sample_output *out) 651 { 652 struct record *rec, *prev; 653 double t, pps, drop, err; 654 int i; 655 656 rec = &stats_rec->rx_cnt; 657 prev = &stats_prev->rx_cnt; 658 t = calc_period(rec, prev); 659 660 for (i = 0; i < nr_cpus; i++) { 661 struct datarec *r = &rec->cpu[i]; 662 struct datarec *p = &prev->cpu[i]; 663 char str[64]; 664 665 pps = calc_pps(r, p, t); 666 drop = calc_drop_pps(r, p, t); 667 err = calc_errs_pps(r, p, t); 668 if (!pps && !drop && !err) 669 continue; 670 671 snprintf(str, sizeof(str), "cpu:%d", i); 672 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 673 "\n", 674 str, PPS(pps), DROP(drop), ERR(err)); 675 } 676 677 if (out) { 678 pps = calc_pps(&rec->total, &prev->total, t); 679 drop = calc_drop_pps(&rec->total, &prev->total, t); 680 err = calc_errs_pps(&rec->total, &prev->total, t); 681 682 out->rx_cnt.pps = pps; 683 out->rx_cnt.drop = drop; 684 out->rx_cnt.err = err; 685 out->totals.rx += pps; 686 out->totals.drop += drop; 687 out->totals.err += err; 688 } 689 } 690 691 static void stats_get_cpumap_enqueue(struct stats_record *stats_rec, 692 struct stats_record *stats_prev, 693 unsigned int nr_cpus) 694 { 695 struct record *rec, *prev; 696 double t, pps, drop, err; 697 int i, to_cpu; 698 699 /* cpumap enqueue stats */ 700 for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) { 701 rec = &stats_rec->enq[to_cpu]; 702 prev = &stats_prev->enq[to_cpu]; 703 t = calc_period(rec, prev); 704 705 pps = calc_pps(&rec->total, &prev->total, t); 706 drop = calc_drop_pps(&rec->total, &prev->total, t); 707 err = calc_errs_pps(&rec->total, &prev->total, t); 708 709 if (pps > 0 || drop > 0) { 710 char str[64]; 711 712 snprintf(str, sizeof(str), "enqueue to cpu %d", to_cpu); 713 714 if (err > 0) 715 err = pps / err; /* calc average bulk size */ 716 717 print_err(drop, 718 " %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN( 719 ".2f") "\n", 720 str, PPS(pps), DROP(drop), err, "bulk-avg"); 721 } 722 723 for (i = 0; i < nr_cpus; i++) { 724 struct datarec *r = &rec->cpu[i]; 725 struct datarec *p = &prev->cpu[i]; 726 char str[64]; 727 728 pps = calc_pps(r, p, t); 729 drop = calc_drop_pps(r, p, t); 730 err = calc_errs_pps(r, p, t); 731 if (!pps && !drop && !err) 732 continue; 733 734 snprintf(str, sizeof(str), "cpu:%d->%d", i, to_cpu); 735 if (err > 0) 736 err = pps / err; /* calc average bulk size */ 737 print_default( 738 " %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN( 739 ".2f") "\n", 740 str, PPS(pps), DROP(drop), err, "bulk-avg"); 741 } 742 } 743 } 744 745 static void stats_get_cpumap_remote(struct stats_record *stats_rec, 746 struct stats_record *stats_prev, 747 unsigned int nr_cpus) 748 { 749 double xdp_pass, xdp_drop, xdp_redirect; 750 struct record *rec, *prev; 751 double t; 752 int i; 753 754 rec = &stats_rec->kthread; 755 prev = &stats_prev->kthread; 756 t = calc_period(rec, prev); 757 758 calc_xdp_pps(&rec->total, &prev->total, &xdp_pass, &xdp_drop, 759 &xdp_redirect, t); 760 if (xdp_pass || xdp_drop || xdp_redirect) { 761 print_err(xdp_drop, 762 " %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n", 763 "xdp_stats", PASS(xdp_pass), DROP(xdp_drop), 764 REDIR(xdp_redirect)); 765 } 766 767 for (i = 0; i < nr_cpus; i++) { 768 struct datarec *r = &rec->cpu[i]; 769 struct datarec *p = &prev->cpu[i]; 770 char str[64]; 771 772 calc_xdp_pps(r, p, &xdp_pass, &xdp_drop, &xdp_redirect, t); 773 if (!xdp_pass && !xdp_drop && !xdp_redirect) 774 continue; 775 776 snprintf(str, sizeof(str), "cpu:%d", i); 777 print_default(" %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 778 "\n", 779 str, PASS(xdp_pass), DROP(xdp_drop), 780 REDIR(xdp_redirect)); 781 } 782 } 783 784 static void stats_get_cpumap_kthread(struct stats_record *stats_rec, 785 struct stats_record *stats_prev, 786 unsigned int nr_cpus) 787 { 788 struct record *rec, *prev; 789 double t, pps, drop, err; 790 int i; 791 792 rec = &stats_rec->kthread; 793 prev = &stats_prev->kthread; 794 t = calc_period(rec, prev); 795 796 pps = calc_pps(&rec->total, &prev->total, t); 797 drop = calc_drop_pps(&rec->total, &prev->total, t); 798 err = calc_errs_pps(&rec->total, &prev->total, t); 799 800 print_err(drop, " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n", 801 pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err, 802 "sched"); 803 804 for (i = 0; i < nr_cpus; i++) { 805 struct datarec *r = &rec->cpu[i]; 806 struct datarec *p = &prev->cpu[i]; 807 char str[64]; 808 809 pps = calc_pps(r, p, t); 810 drop = calc_drop_pps(r, p, t); 811 err = calc_errs_pps(r, p, t); 812 if (!pps && !drop && !err) 813 continue; 814 815 snprintf(str, sizeof(str), "cpu:%d", i); 816 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 817 "\n", 818 str, PPS(pps), DROP(drop), err, "sched"); 819 } 820 } 821 822 static void stats_get_redirect_cnt(struct stats_record *stats_rec, 823 struct stats_record *stats_prev, 824 unsigned int nr_cpus, 825 struct sample_output *out) 826 { 827 struct record *rec, *prev; 828 double t, pps; 829 int i; 830 831 rec = &stats_rec->redir_err[0]; 832 prev = &stats_prev->redir_err[0]; 833 t = calc_period(rec, prev); 834 for (i = 0; i < nr_cpus; i++) { 835 struct datarec *r = &rec->cpu[i]; 836 struct datarec *p = &prev->cpu[i]; 837 char str[64]; 838 839 pps = calc_pps(r, p, t); 840 if (!pps) 841 continue; 842 843 snprintf(str, sizeof(str), "cpu:%d", i); 844 print_default(" %-18s " FMT_COLUMNf "\n", str, REDIR(pps)); 845 } 846 847 if (out) { 848 pps = calc_pps(&rec->total, &prev->total, t); 849 out->redir_cnt.suc = pps; 850 out->totals.redir += pps; 851 } 852 } 853 854 static void stats_get_redirect_err_cnt(struct stats_record *stats_rec, 855 struct stats_record *stats_prev, 856 unsigned int nr_cpus, 857 struct sample_output *out) 858 { 859 struct record *rec, *prev; 860 double t, drop, sum = 0; 861 int rec_i, i; 862 863 for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) { 864 char str[64]; 865 866 rec = &stats_rec->redir_err[rec_i]; 867 prev = &stats_prev->redir_err[rec_i]; 868 t = calc_period(rec, prev); 869 870 drop = calc_drop_pps(&rec->total, &prev->total, t); 871 if (drop > 0 && !out) { 872 snprintf(str, sizeof(str), 873 sample_log_level & LL_DEFAULT ? "%s total" : 874 "%s", 875 xdp_redirect_err_names[rec_i]); 876 print_err(drop, " %-18s " FMT_COLUMNf "\n", str, 877 ERR(drop)); 878 } 879 880 for (i = 0; i < nr_cpus; i++) { 881 struct datarec *r = &rec->cpu[i]; 882 struct datarec *p = &prev->cpu[i]; 883 double drop; 884 885 drop = calc_drop_pps(r, p, t); 886 if (!drop) 887 continue; 888 889 snprintf(str, sizeof(str), "cpu:%d", i); 890 print_default(" %-16s" FMT_COLUMNf "\n", str, 891 ERR(drop)); 892 } 893 894 sum += drop; 895 } 896 897 if (out) { 898 out->redir_cnt.err = sum; 899 out->totals.err += sum; 900 } 901 } 902 903 static void stats_get_exception_cnt(struct stats_record *stats_rec, 904 struct stats_record *stats_prev, 905 unsigned int nr_cpus, 906 struct sample_output *out) 907 { 908 double t, drop, sum = 0; 909 struct record *rec, *prev; 910 int rec_i, i; 911 912 for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) { 913 rec = &stats_rec->exception[rec_i]; 914 prev = &stats_prev->exception[rec_i]; 915 t = calc_period(rec, prev); 916 917 drop = calc_drop_pps(&rec->total, &prev->total, t); 918 /* Fold out errors after heading */ 919 sum += drop; 920 921 if (drop > 0 && !out) { 922 print_always(" %-18s " FMT_COLUMNf "\n", 923 action2str(rec_i), ERR(drop)); 924 925 for (i = 0; i < nr_cpus; i++) { 926 struct datarec *r = &rec->cpu[i]; 927 struct datarec *p = &prev->cpu[i]; 928 char str[64]; 929 double drop; 930 931 drop = calc_drop_pps(r, p, t); 932 if (!drop) 933 continue; 934 935 snprintf(str, sizeof(str), "cpu:%d", i); 936 print_default(" %-16s" FMT_COLUMNf "\n", 937 str, ERR(drop)); 938 } 939 } 940 } 941 942 if (out) { 943 out->except_cnt.hits = sum; 944 out->totals.err += sum; 945 } 946 } 947 948 static void stats_get_devmap_xmit(struct stats_record *stats_rec, 949 struct stats_record *stats_prev, 950 unsigned int nr_cpus, 951 struct sample_output *out) 952 { 953 double pps, drop, info, err; 954 struct record *rec, *prev; 955 double t; 956 int i; 957 958 rec = &stats_rec->devmap_xmit; 959 prev = &stats_prev->devmap_xmit; 960 t = calc_period(rec, prev); 961 for (i = 0; i < nr_cpus; i++) { 962 struct datarec *r = &rec->cpu[i]; 963 struct datarec *p = &prev->cpu[i]; 964 char str[64]; 965 966 pps = calc_pps(r, p, t); 967 drop = calc_drop_pps(r, p, t); 968 err = calc_errs_pps(r, p, t); 969 970 if (!pps && !drop && !err) 971 continue; 972 973 snprintf(str, sizeof(str), "cpu:%d", i); 974 info = calc_info_pps(r, p, t); 975 if (info > 0) 976 info = (pps + drop) / info; /* calc avg bulk */ 977 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 978 __COLUMN(".2f") "\n", 979 str, XMIT(pps), DROP(drop), err, "drv_err/s", 980 info, "bulk-avg"); 981 } 982 if (out) { 983 pps = calc_pps(&rec->total, &prev->total, t); 984 drop = calc_drop_pps(&rec->total, &prev->total, t); 985 info = calc_info_pps(&rec->total, &prev->total, t); 986 if (info > 0) 987 info = (pps + drop) / info; /* calc avg bulk */ 988 err = calc_errs_pps(&rec->total, &prev->total, t); 989 990 out->xmit_cnt.pps = pps; 991 out->xmit_cnt.drop = drop; 992 out->xmit_cnt.bavg = info; 993 out->xmit_cnt.err = err; 994 out->totals.xmit += pps; 995 out->totals.drop_xmit += drop; 996 out->totals.err += err; 997 } 998 } 999 1000 static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec, 1001 struct stats_record *stats_prev, 1002 unsigned int nr_cpus, 1003 struct sample_output *out, 1004 bool xmit_total) 1005 { 1006 double pps, drop, info, err; 1007 struct map_entry *entry; 1008 struct record *r, *p; 1009 double t; 1010 int bkt; 1011 1012 hash_for_each(stats_rec->xmit_map, bkt, entry, node) { 1013 struct map_entry *e, *x = NULL; 1014 char ifname_from[IFNAMSIZ]; 1015 char ifname_to[IFNAMSIZ]; 1016 const char *fstr, *tstr; 1017 unsigned long prev_time; 1018 struct record beg = {}; 1019 __u32 from_idx, to_idx; 1020 char str[128]; 1021 __u64 pair; 1022 int i; 1023 1024 prev_time = sample_interval * NANOSEC_PER_SEC; 1025 1026 pair = entry->pair; 1027 from_idx = pair >> 32; 1028 to_idx = pair & 0xFFFFFFFF; 1029 1030 r = &entry->val; 1031 beg.timestamp = r->timestamp - prev_time; 1032 1033 /* Find matching entry from stats_prev map */ 1034 hash_for_each_possible(stats_prev->xmit_map, e, node, pair) { 1035 if (e->pair == pair) { 1036 x = e; 1037 break; 1038 } 1039 } 1040 if (x) 1041 p = &x->val; 1042 else 1043 p = &beg; 1044 t = calc_period(r, p); 1045 pps = calc_pps(&r->total, &p->total, t); 1046 drop = calc_drop_pps(&r->total, &p->total, t); 1047 info = calc_info_pps(&r->total, &p->total, t); 1048 if (info > 0) 1049 info = (pps + drop) / info; /* calc avg bulk */ 1050 err = calc_errs_pps(&r->total, &p->total, t); 1051 1052 if (out) { 1053 /* We are responsible for filling out totals */ 1054 out->totals.xmit += pps; 1055 out->totals.drop_xmit += drop; 1056 out->totals.err += err; 1057 continue; 1058 } 1059 1060 fstr = tstr = NULL; 1061 if (if_indextoname(from_idx, ifname_from)) 1062 fstr = ifname_from; 1063 if (if_indextoname(to_idx, ifname_to)) 1064 tstr = ifname_to; 1065 1066 snprintf(str, sizeof(str), "xmit %s->%s", fstr ?: "?", 1067 tstr ?: "?"); 1068 /* Skip idle streams of redirection */ 1069 if (pps || drop || err) { 1070 print_err(drop, 1071 " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 1072 __COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop), 1073 err, "drv_err/s", info, "bulk-avg"); 1074 } 1075 1076 for (i = 0; i < nr_cpus; i++) { 1077 struct datarec *rc = &r->cpu[i]; 1078 struct datarec *pc, p_beg = {}; 1079 char str[64]; 1080 1081 pc = p == &beg ? &p_beg : &p->cpu[i]; 1082 1083 pps = calc_pps(rc, pc, t); 1084 drop = calc_drop_pps(rc, pc, t); 1085 err = calc_errs_pps(rc, pc, t); 1086 1087 if (!pps && !drop && !err) 1088 continue; 1089 1090 snprintf(str, sizeof(str), "cpu:%d", i); 1091 info = calc_info_pps(rc, pc, t); 1092 if (info > 0) 1093 info = (pps + drop) / info; /* calc avg bulk */ 1094 1095 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf 1096 __COLUMN(".2f") "\n", str, XMIT(pps), 1097 DROP(drop), err, "drv_err/s", info, "bulk-avg"); 1098 } 1099 } 1100 } 1101 1102 static void stats_print(const char *prefix, int mask, struct stats_record *r, 1103 struct stats_record *p, struct sample_output *out) 1104 { 1105 int nr_cpus = libbpf_num_possible_cpus(); 1106 const char *str; 1107 1108 print_always("%-23s", prefix ?: "Summary"); 1109 if (mask & SAMPLE_RX_CNT) 1110 print_always(FMT_COLUMNl, RX(out->totals.rx)); 1111 if (mask & SAMPLE_REDIRECT_CNT) 1112 print_always(FMT_COLUMNl, REDIR(out->totals.redir)); 1113 printf(FMT_COLUMNl, 1114 out->totals.err + out->totals.drop + out->totals.drop_xmit, 1115 "err,drop/s"); 1116 if (mask & SAMPLE_DEVMAP_XMIT_CNT || 1117 mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 1118 printf(FMT_COLUMNl, XMIT(out->totals.xmit)); 1119 printf("\n"); 1120 1121 if (mask & SAMPLE_RX_CNT) { 1122 str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ? 1123 "receive total" : 1124 "receive"; 1125 print_err((out->rx_cnt.err || out->rx_cnt.drop), 1126 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n", 1127 str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop), 1128 ERR(out->rx_cnt.err)); 1129 1130 stats_get_rx_cnt(r, p, nr_cpus, NULL); 1131 } 1132 1133 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) 1134 stats_get_cpumap_enqueue(r, p, nr_cpus); 1135 1136 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) { 1137 stats_get_cpumap_kthread(r, p, nr_cpus); 1138 stats_get_cpumap_remote(r, p, nr_cpus); 1139 } 1140 1141 if (mask & SAMPLE_REDIRECT_CNT) { 1142 str = out->redir_cnt.suc ? "redirect total" : "redirect"; 1143 print_default(" %-20s " FMT_COLUMNl "\n", str, 1144 REDIR(out->redir_cnt.suc)); 1145 1146 stats_get_redirect_cnt(r, p, nr_cpus, NULL); 1147 } 1148 1149 if (mask & SAMPLE_REDIRECT_ERR_CNT) { 1150 str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ? 1151 "redirect_err total" : 1152 "redirect_err"; 1153 print_err(out->redir_cnt.err, " %-20s " FMT_COLUMNl "\n", str, 1154 ERR(out->redir_cnt.err)); 1155 1156 stats_get_redirect_err_cnt(r, p, nr_cpus, NULL); 1157 } 1158 1159 if (mask & SAMPLE_EXCEPTION_CNT) { 1160 str = out->except_cnt.hits ? "xdp_exception total" : 1161 "xdp_exception"; 1162 1163 print_err(out->except_cnt.hits, " %-20s " FMT_COLUMNl "\n", str, 1164 HITS(out->except_cnt.hits)); 1165 1166 stats_get_exception_cnt(r, p, nr_cpus, NULL); 1167 } 1168 1169 if (mask & SAMPLE_DEVMAP_XMIT_CNT) { 1170 str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ? 1171 "devmap_xmit total" : 1172 "devmap_xmit"; 1173 1174 print_err(out->xmit_cnt.err || out->xmit_cnt.drop, 1175 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl 1176 __COLUMN(".2f") "\n", 1177 str, XMIT(out->xmit_cnt.pps), 1178 DROP(out->xmit_cnt.drop), out->xmit_cnt.err, 1179 "drv_err/s", out->xmit_cnt.bavg, "bulk-avg"); 1180 1181 stats_get_devmap_xmit(r, p, nr_cpus, NULL); 1182 } 1183 1184 if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 1185 stats_get_devmap_xmit_multi(r, p, nr_cpus, NULL, 1186 mask & SAMPLE_DEVMAP_XMIT_CNT); 1187 1188 if (sample_log_level & LL_DEFAULT || 1189 ((sample_log_level & LL_SIMPLE) && sample_err_exp)) { 1190 sample_err_exp = false; 1191 printf("\n"); 1192 } 1193 } 1194 1195 int sample_setup_maps(struct bpf_map **maps) 1196 { 1197 sample_n_cpus = libbpf_num_possible_cpus(); 1198 1199 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) { 1200 sample_map[i] = maps[i]; 1201 1202 switch (i) { 1203 case MAP_RX: 1204 case MAP_CPUMAP_KTHREAD: 1205 case MAP_DEVMAP_XMIT: 1206 sample_map_count[i] = sample_n_cpus; 1207 break; 1208 case MAP_REDIRECT_ERR: 1209 sample_map_count[i] = 1210 XDP_REDIRECT_ERR_MAX * sample_n_cpus; 1211 break; 1212 case MAP_EXCEPTION: 1213 sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus; 1214 case MAP_CPUMAP_ENQUEUE: 1215 sample_map_count[i] = sample_n_cpus * sample_n_cpus; 1216 break; 1217 default: 1218 return -EINVAL; 1219 } 1220 if (bpf_map__set_max_entries(sample_map[i], sample_map_count[i]) < 0) 1221 return -errno; 1222 } 1223 sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI]; 1224 return 0; 1225 } 1226 1227 static int sample_setup_maps_mappings(void) 1228 { 1229 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) { 1230 size_t size = sample_map_count[i] * sizeof(struct datarec); 1231 1232 sample_mmap[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, 1233 MAP_SHARED, bpf_map__fd(sample_map[i]), 0); 1234 if (sample_mmap[i] == MAP_FAILED) 1235 return -errno; 1236 } 1237 return 0; 1238 } 1239 1240 int __sample_init(int mask) 1241 { 1242 sigset_t st; 1243 1244 sigemptyset(&st); 1245 sigaddset(&st, SIGQUIT); 1246 sigaddset(&st, SIGINT); 1247 sigaddset(&st, SIGTERM); 1248 1249 if (sigprocmask(SIG_BLOCK, &st, NULL) < 0) 1250 return -errno; 1251 1252 sample_sig_fd = signalfd(-1, &st, SFD_CLOEXEC | SFD_NONBLOCK); 1253 if (sample_sig_fd < 0) 1254 return -errno; 1255 1256 sample_mask = mask; 1257 1258 return sample_setup_maps_mappings(); 1259 } 1260 1261 static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags) 1262 { 1263 __u32 cur_prog_id = 0; 1264 int ret; 1265 1266 if (prog_id) { 1267 ret = bpf_xdp_query_id(ifindex, xdp_flags, &cur_prog_id); 1268 if (ret < 0) 1269 return -errno; 1270 1271 if (prog_id != cur_prog_id) { 1272 print_always( 1273 "Program on ifindex %d does not match installed " 1274 "program, skipping unload\n", 1275 ifindex); 1276 return -ENOENT; 1277 } 1278 } 1279 1280 return bpf_xdp_detach(ifindex, xdp_flags, NULL); 1281 } 1282 1283 int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic, 1284 bool force) 1285 { 1286 int ret, xdp_flags = 0; 1287 __u32 prog_id = 0; 1288 1289 if (sample_xdp_cnt == 32) { 1290 fprintf(stderr, 1291 "Total limit for installed XDP programs in a sample reached\n"); 1292 return -ENOTSUP; 1293 } 1294 1295 xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0; 1296 xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; 1297 ret = bpf_xdp_attach(ifindex, bpf_program__fd(xdp_prog), xdp_flags, NULL); 1298 if (ret < 0) { 1299 ret = -errno; 1300 fprintf(stderr, 1301 "Failed to install program \"%s\" on ifindex %d, mode = %s, " 1302 "force = %s: %s\n", 1303 bpf_program__name(xdp_prog), ifindex, 1304 generic ? "skb" : "native", force ? "true" : "false", 1305 strerror(-ret)); 1306 return ret; 1307 } 1308 1309 ret = bpf_xdp_query_id(ifindex, xdp_flags, &prog_id); 1310 if (ret < 0) { 1311 ret = -errno; 1312 fprintf(stderr, 1313 "Failed to get XDP program id for ifindex %d, removing program: %s\n", 1314 ifindex, strerror(errno)); 1315 __sample_remove_xdp(ifindex, 0, xdp_flags); 1316 return ret; 1317 } 1318 sample_xdp_progs[sample_xdp_cnt++] = 1319 (struct xdp_desc){ ifindex, prog_id, xdp_flags }; 1320 1321 return 0; 1322 } 1323 1324 static void sample_summary_print(void) 1325 { 1326 double num = sample_out.rx_cnt.num; 1327 1328 if (sample_out.totals.rx) { 1329 double pkts = sample_out.totals.rx; 1330 1331 print_always(" Packets received : %'-10llu\n", 1332 sample_out.totals.rx); 1333 print_always(" Average packets/s : %'-10.0f\n", 1334 sample_round(pkts / num)); 1335 } 1336 if (sample_out.totals.redir) { 1337 double pkts = sample_out.totals.redir; 1338 1339 print_always(" Packets redirected : %'-10llu\n", 1340 sample_out.totals.redir); 1341 print_always(" Average redir/s : %'-10.0f\n", 1342 sample_round(pkts / num)); 1343 } 1344 if (sample_out.totals.drop) 1345 print_always(" Rx dropped : %'-10llu\n", 1346 sample_out.totals.drop); 1347 if (sample_out.totals.drop_xmit) 1348 print_always(" Tx dropped : %'-10llu\n", 1349 sample_out.totals.drop_xmit); 1350 if (sample_out.totals.err) 1351 print_always(" Errors recorded : %'-10llu\n", 1352 sample_out.totals.err); 1353 if (sample_out.totals.xmit) { 1354 double pkts = sample_out.totals.xmit; 1355 1356 print_always(" Packets transmitted : %'-10llu\n", 1357 sample_out.totals.xmit); 1358 print_always(" Average transmit/s : %'-10.0f\n", 1359 sample_round(pkts / num)); 1360 } 1361 } 1362 1363 void sample_exit(int status) 1364 { 1365 size_t size; 1366 1367 for (int i = 0; i < NUM_MAP; i++) { 1368 size = sample_map_count[i] * sizeof(**sample_mmap); 1369 munmap(sample_mmap[i], size); 1370 } 1371 while (sample_xdp_cnt--) { 1372 int i = sample_xdp_cnt, ifindex, xdp_flags; 1373 __u32 prog_id; 1374 1375 prog_id = sample_xdp_progs[i].prog_id; 1376 ifindex = sample_xdp_progs[i].ifindex; 1377 xdp_flags = sample_xdp_progs[i].flags; 1378 1379 __sample_remove_xdp(ifindex, prog_id, xdp_flags); 1380 } 1381 sample_summary_print(); 1382 close(sample_sig_fd); 1383 exit(status); 1384 } 1385 1386 static int sample_stats_collect(struct stats_record *rec) 1387 { 1388 int i; 1389 1390 if (sample_mask & SAMPLE_RX_CNT) 1391 map_collect_percpu(sample_mmap[MAP_RX], &rec->rx_cnt); 1392 1393 if (sample_mask & SAMPLE_REDIRECT_CNT) 1394 map_collect_percpu(sample_mmap[MAP_REDIRECT_ERR], &rec->redir_err[0]); 1395 1396 if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) { 1397 for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++) 1398 map_collect_percpu(&sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus], 1399 &rec->redir_err[i]); 1400 } 1401 1402 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) 1403 for (i = 0; i < sample_n_cpus; i++) 1404 map_collect_percpu(&sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus], 1405 &rec->enq[i]); 1406 1407 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) 1408 map_collect_percpu(sample_mmap[MAP_CPUMAP_KTHREAD], 1409 &rec->kthread); 1410 1411 if (sample_mask & SAMPLE_EXCEPTION_CNT) 1412 for (i = 0; i < XDP_ACTION_MAX; i++) 1413 map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus], 1414 &rec->exception[i]); 1415 1416 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) 1417 map_collect_percpu(sample_mmap[MAP_DEVMAP_XMIT], &rec->devmap_xmit); 1418 1419 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) { 1420 if (map_collect_percpu_devmap(bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0) 1421 return -EINVAL; 1422 } 1423 return 0; 1424 } 1425 1426 static void sample_summary_update(struct sample_output *out) 1427 { 1428 sample_out.totals.rx += out->totals.rx; 1429 sample_out.totals.redir += out->totals.redir; 1430 sample_out.totals.drop += out->totals.drop; 1431 sample_out.totals.drop_xmit += out->totals.drop_xmit; 1432 sample_out.totals.err += out->totals.err; 1433 sample_out.totals.xmit += out->totals.xmit; 1434 sample_out.rx_cnt.num++; 1435 } 1436 1437 static void sample_stats_print(int mask, struct stats_record *cur, 1438 struct stats_record *prev, char *prog_name) 1439 { 1440 struct sample_output out = {}; 1441 1442 if (mask & SAMPLE_RX_CNT) 1443 stats_get_rx_cnt(cur, prev, 0, &out); 1444 if (mask & SAMPLE_REDIRECT_CNT) 1445 stats_get_redirect_cnt(cur, prev, 0, &out); 1446 if (mask & SAMPLE_REDIRECT_ERR_CNT) 1447 stats_get_redirect_err_cnt(cur, prev, 0, &out); 1448 if (mask & SAMPLE_EXCEPTION_CNT) 1449 stats_get_exception_cnt(cur, prev, 0, &out); 1450 if (mask & SAMPLE_DEVMAP_XMIT_CNT) 1451 stats_get_devmap_xmit(cur, prev, 0, &out); 1452 else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) 1453 stats_get_devmap_xmit_multi(cur, prev, 0, &out, 1454 mask & SAMPLE_DEVMAP_XMIT_CNT); 1455 sample_summary_update(&out); 1456 1457 stats_print(prog_name, mask, cur, prev, &out); 1458 } 1459 1460 void sample_switch_mode(void) 1461 { 1462 sample_log_level ^= LL_DEBUG - 1; 1463 } 1464 1465 static int sample_signal_cb(void) 1466 { 1467 struct signalfd_siginfo si; 1468 int r; 1469 1470 r = read(sample_sig_fd, &si, sizeof(si)); 1471 if (r < 0) 1472 return -errno; 1473 1474 switch (si.ssi_signo) { 1475 case SIGQUIT: 1476 sample_switch_mode(); 1477 printf("\n"); 1478 break; 1479 default: 1480 printf("\n"); 1481 return 1; 1482 } 1483 1484 return 0; 1485 } 1486 1487 /* Pointer swap trick */ 1488 static void swap(struct stats_record **a, struct stats_record **b) 1489 { 1490 struct stats_record *tmp; 1491 1492 tmp = *a; 1493 *a = *b; 1494 *b = tmp; 1495 } 1496 1497 static int sample_timer_cb(int timerfd, struct stats_record **rec, 1498 struct stats_record **prev) 1499 { 1500 char line[64] = "Summary"; 1501 int ret; 1502 __u64 t; 1503 1504 ret = read(timerfd, &t, sizeof(t)); 1505 if (ret < 0) 1506 return -errno; 1507 1508 swap(prev, rec); 1509 ret = sample_stats_collect(*rec); 1510 if (ret < 0) 1511 return ret; 1512 1513 if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) { 1514 char fi[IFNAMSIZ]; 1515 char to[IFNAMSIZ]; 1516 const char *f, *t; 1517 1518 f = t = NULL; 1519 if (if_indextoname(sample_xdp_progs[0].ifindex, fi)) 1520 f = fi; 1521 if (if_indextoname(sample_xdp_progs[1].ifindex, to)) 1522 t = to; 1523 1524 snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?"); 1525 } 1526 1527 sample_stats_print(sample_mask, *rec, *prev, line); 1528 return 0; 1529 } 1530 1531 int sample_run(int interval, void (*post_cb)(void *), void *ctx) 1532 { 1533 struct timespec ts = { interval, 0 }; 1534 struct itimerspec its = { ts, ts }; 1535 struct stats_record *rec, *prev; 1536 struct pollfd pfd[2] = {}; 1537 int timerfd, ret; 1538 1539 if (!interval) { 1540 fprintf(stderr, "Incorrect interval 0\n"); 1541 return -EINVAL; 1542 } 1543 sample_interval = interval; 1544 /* Pretty print numbers */ 1545 setlocale(LC_NUMERIC, "en_US.UTF-8"); 1546 1547 timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK); 1548 if (timerfd < 0) 1549 return -errno; 1550 timerfd_settime(timerfd, 0, &its, NULL); 1551 1552 pfd[0].fd = sample_sig_fd; 1553 pfd[0].events = POLLIN; 1554 1555 pfd[1].fd = timerfd; 1556 pfd[1].events = POLLIN; 1557 1558 ret = -ENOMEM; 1559 rec = alloc_stats_record(); 1560 if (!rec) 1561 goto end; 1562 prev = alloc_stats_record(); 1563 if (!prev) 1564 goto end_rec; 1565 1566 ret = sample_stats_collect(rec); 1567 if (ret < 0) 1568 goto end_rec_prev; 1569 1570 for (;;) { 1571 ret = poll(pfd, 2, -1); 1572 if (ret < 0) { 1573 if (errno == EINTR) 1574 continue; 1575 else 1576 break; 1577 } 1578 1579 if (pfd[0].revents & POLLIN) 1580 ret = sample_signal_cb(); 1581 else if (pfd[1].revents & POLLIN) 1582 ret = sample_timer_cb(timerfd, &rec, &prev); 1583 1584 if (ret) 1585 break; 1586 1587 if (post_cb) 1588 post_cb(ctx); 1589 } 1590 1591 end_rec_prev: 1592 free_stats_record(prev); 1593 end_rec: 1594 free_stats_record(rec); 1595 end: 1596 close(timerfd); 1597 1598 return ret; 1599 } 1600 1601 const char *get_driver_name(int ifindex) 1602 { 1603 struct ethtool_drvinfo drv = {}; 1604 char ifname[IF_NAMESIZE]; 1605 static char drvname[32]; 1606 struct ifreq ifr = {}; 1607 int fd, r = 0; 1608 1609 fd = socket(AF_INET, SOCK_DGRAM, 0); 1610 if (fd < 0) 1611 return "[error]"; 1612 1613 if (!if_indextoname(ifindex, ifname)) 1614 goto end; 1615 1616 drv.cmd = ETHTOOL_GDRVINFO; 1617 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name)); 1618 ifr.ifr_data = (void *)&drv; 1619 1620 r = ioctl(fd, SIOCETHTOOL, &ifr); 1621 if (r) 1622 goto end; 1623 1624 safe_strncpy(drvname, drv.driver, sizeof(drvname)); 1625 1626 close(fd); 1627 return drvname; 1628 1629 end: 1630 r = errno; 1631 close(fd); 1632 return r == EOPNOTSUPP ? "loopback" : "[error]"; 1633 } 1634 1635 int get_mac_addr(int ifindex, void *mac_addr) 1636 { 1637 char ifname[IF_NAMESIZE]; 1638 struct ifreq ifr = {}; 1639 int fd, r; 1640 1641 fd = socket(AF_INET, SOCK_DGRAM, 0); 1642 if (fd < 0) 1643 return -errno; 1644 1645 if (!if_indextoname(ifindex, ifname)) { 1646 r = -errno; 1647 goto end; 1648 } 1649 1650 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name)); 1651 1652 r = ioctl(fd, SIOCGIFHWADDR, &ifr); 1653 if (r) { 1654 r = -errno; 1655 goto end; 1656 } 1657 1658 memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char)); 1659 1660 end: 1661 close(fd); 1662 return r; 1663 } 1664 1665 __attribute__((constructor)) static void sample_ctor(void) 1666 { 1667 if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) { 1668 fprintf(stderr, "Failed to set libbpf strict mode: %s\n", 1669 strerror(errno)); 1670 /* Just exit, nothing to cleanup right now */ 1671 exit(EXIT_FAIL_BPF); 1672 } 1673 } 1674