1 // SPDX-License-Identifier: GPL-2.0 2 /* GPLv2, Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc. */ 3 #include "xdp_sample.bpf.h" 4 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_core_read.h> 7 #include <bpf/bpf_helpers.h> 8 9 array_map rx_cnt SEC(".maps"); 10 array_map redir_err_cnt SEC(".maps"); 11 array_map cpumap_enqueue_cnt SEC(".maps"); 12 array_map cpumap_kthread_cnt SEC(".maps"); 13 array_map exception_cnt SEC(".maps"); 14 array_map devmap_xmit_cnt SEC(".maps"); 15 16 struct { 17 __uint(type, BPF_MAP_TYPE_PERCPU_HASH); 18 __uint(max_entries, 32 * 32); 19 __type(key, u64); 20 __type(value, struct datarec); 21 } devmap_xmit_cnt_multi SEC(".maps"); 22 23 const volatile int nr_cpus = 0; 24 25 /* These can be set before loading so that redundant comparisons can be DCE'd by 26 * the verifier, and only actual matches are tried after loading tp_btf program. 27 * This allows sample to filter tracepoint stats based on net_device. 28 */ 29 const volatile int from_match[32] = {}; 30 const volatile int to_match[32] = {}; 31 32 int cpumap_map_id = 0; 33 34 /* Find if b is part of set a, but if a is empty set then evaluate to true */ 35 #define IN_SET(a, b) \ 36 ({ \ 37 bool __res = !(a)[0]; \ 38 for (int i = 0; i < ARRAY_SIZE(a) && (a)[i]; i++) { \ 39 __res = (a)[i] == (b); \ 40 if (__res) \ 41 break; \ 42 } \ 43 __res; \ 44 }) 45 46 static __always_inline __u32 xdp_get_err_key(int err) 47 { 48 switch (err) { 49 case 0: 50 return 0; 51 case -EINVAL: 52 return 2; 53 case -ENETDOWN: 54 return 3; 55 case -EMSGSIZE: 56 return 4; 57 case -EOPNOTSUPP: 58 return 5; 59 case -ENOSPC: 60 return 6; 61 default: 62 return 1; 63 } 64 } 65 66 static __always_inline int xdp_redirect_collect_stat(int from, int err) 67 { 68 u32 cpu = bpf_get_smp_processor_id(); 69 u32 key = XDP_REDIRECT_ERROR; 70 struct datarec *rec; 71 u32 idx; 72 73 if (!IN_SET(from_match, from)) 74 return 0; 75 76 key = xdp_get_err_key(err); 77 78 idx = key * nr_cpus + cpu; 79 rec = bpf_map_lookup_elem(&redir_err_cnt, &idx); 80 if (!rec) 81 return 0; 82 if (key) 83 NO_TEAR_INC(rec->dropped); 84 else 85 NO_TEAR_INC(rec->processed); 86 return 0; /* Indicate event was filtered (no further processing)*/ 87 /* 88 * Returning 1 here would allow e.g. a perf-record tracepoint 89 * to see and record these events, but it doesn't work well 90 * in-practice as stopping perf-record also unload this 91 * bpf_prog. Plus, there is additional overhead of doing so. 92 */ 93 } 94 95 SEC("tp_btf/xdp_redirect_err") 96 int BPF_PROG(tp_xdp_redirect_err, const struct net_device *dev, 97 const struct bpf_prog *xdp, const void *tgt, int err, 98 const struct bpf_map *map, u32 index) 99 { 100 return xdp_redirect_collect_stat(dev->ifindex, err); 101 } 102 103 SEC("tp_btf/xdp_redirect_map_err") 104 int BPF_PROG(tp_xdp_redirect_map_err, const struct net_device *dev, 105 const struct bpf_prog *xdp, const void *tgt, int err, 106 const struct bpf_map *map, u32 index) 107 { 108 return xdp_redirect_collect_stat(dev->ifindex, err); 109 } 110 111 SEC("tp_btf/xdp_redirect") 112 int BPF_PROG(tp_xdp_redirect, const struct net_device *dev, 113 const struct bpf_prog *xdp, const void *tgt, int err, 114 const struct bpf_map *map, u32 index) 115 { 116 return xdp_redirect_collect_stat(dev->ifindex, err); 117 } 118 119 SEC("tp_btf/xdp_redirect_map") 120 int BPF_PROG(tp_xdp_redirect_map, const struct net_device *dev, 121 const struct bpf_prog *xdp, const void *tgt, int err, 122 const struct bpf_map *map, u32 index) 123 { 124 return xdp_redirect_collect_stat(dev->ifindex, err); 125 } 126 127 SEC("tp_btf/xdp_cpumap_enqueue") 128 int BPF_PROG(tp_xdp_cpumap_enqueue, int map_id, unsigned int processed, 129 unsigned int drops, int to_cpu) 130 { 131 u32 cpu = bpf_get_smp_processor_id(); 132 struct datarec *rec; 133 u32 idx; 134 135 if (cpumap_map_id && cpumap_map_id != map_id) 136 return 0; 137 138 idx = to_cpu * nr_cpus + cpu; 139 rec = bpf_map_lookup_elem(&cpumap_enqueue_cnt, &idx); 140 if (!rec) 141 return 0; 142 NO_TEAR_ADD(rec->processed, processed); 143 NO_TEAR_ADD(rec->dropped, drops); 144 /* Record bulk events, then userspace can calc average bulk size */ 145 if (processed > 0) 146 NO_TEAR_INC(rec->issue); 147 /* Inception: It's possible to detect overload situations, via 148 * this tracepoint. This can be used for creating a feedback 149 * loop to XDP, which can take appropriate actions to mitigate 150 * this overload situation. 151 */ 152 return 0; 153 } 154 155 SEC("tp_btf/xdp_cpumap_kthread") 156 int BPF_PROG(tp_xdp_cpumap_kthread, int map_id, unsigned int processed, 157 unsigned int drops, int sched, struct xdp_cpumap_stats *xdp_stats) 158 { 159 struct datarec *rec; 160 u32 cpu; 161 162 if (cpumap_map_id && cpumap_map_id != map_id) 163 return 0; 164 165 cpu = bpf_get_smp_processor_id(); 166 rec = bpf_map_lookup_elem(&cpumap_kthread_cnt, &cpu); 167 if (!rec) 168 return 0; 169 NO_TEAR_ADD(rec->processed, processed); 170 NO_TEAR_ADD(rec->dropped, drops); 171 NO_TEAR_ADD(rec->xdp_pass, xdp_stats->pass); 172 NO_TEAR_ADD(rec->xdp_drop, xdp_stats->drop); 173 NO_TEAR_ADD(rec->xdp_redirect, xdp_stats->redirect); 174 /* Count times kthread yielded CPU via schedule call */ 175 if (sched) 176 NO_TEAR_INC(rec->issue); 177 return 0; 178 } 179 180 SEC("tp_btf/xdp_exception") 181 int BPF_PROG(tp_xdp_exception, const struct net_device *dev, 182 const struct bpf_prog *xdp, u32 act) 183 { 184 u32 cpu = bpf_get_smp_processor_id(); 185 struct datarec *rec; 186 u32 key = act, idx; 187 188 if (!IN_SET(from_match, dev->ifindex)) 189 return 0; 190 if (!IN_SET(to_match, dev->ifindex)) 191 return 0; 192 193 if (key > XDP_REDIRECT) 194 key = XDP_REDIRECT + 1; 195 196 idx = key * nr_cpus + cpu; 197 rec = bpf_map_lookup_elem(&exception_cnt, &idx); 198 if (!rec) 199 return 0; 200 NO_TEAR_INC(rec->dropped); 201 202 return 0; 203 } 204 205 SEC("tp_btf/xdp_devmap_xmit") 206 int BPF_PROG(tp_xdp_devmap_xmit, const struct net_device *from_dev, 207 const struct net_device *to_dev, int sent, int drops, int err) 208 { 209 struct datarec *rec; 210 int idx_in, idx_out; 211 u32 cpu; 212 213 idx_in = from_dev->ifindex; 214 idx_out = to_dev->ifindex; 215 216 if (!IN_SET(from_match, idx_in)) 217 return 0; 218 if (!IN_SET(to_match, idx_out)) 219 return 0; 220 221 cpu = bpf_get_smp_processor_id(); 222 rec = bpf_map_lookup_elem(&devmap_xmit_cnt, &cpu); 223 if (!rec) 224 return 0; 225 NO_TEAR_ADD(rec->processed, sent); 226 NO_TEAR_ADD(rec->dropped, drops); 227 /* Record bulk events, then userspace can calc average bulk size */ 228 NO_TEAR_INC(rec->info); 229 /* Record error cases, where no frame were sent */ 230 /* Catch API error of drv ndo_xdp_xmit sent more than count */ 231 if (err || drops < 0) 232 NO_TEAR_INC(rec->issue); 233 return 0; 234 } 235 236 SEC("tp_btf/xdp_devmap_xmit") 237 int BPF_PROG(tp_xdp_devmap_xmit_multi, const struct net_device *from_dev, 238 const struct net_device *to_dev, int sent, int drops, int err) 239 { 240 struct datarec empty = {}; 241 struct datarec *rec; 242 int idx_in, idx_out; 243 u64 idx; 244 245 idx_in = from_dev->ifindex; 246 idx_out = to_dev->ifindex; 247 idx = idx_in; 248 idx = idx << 32 | idx_out; 249 250 if (!IN_SET(from_match, idx_in)) 251 return 0; 252 if (!IN_SET(to_match, idx_out)) 253 return 0; 254 255 bpf_map_update_elem(&devmap_xmit_cnt_multi, &idx, &empty, BPF_NOEXIST); 256 rec = bpf_map_lookup_elem(&devmap_xmit_cnt_multi, &idx); 257 if (!rec) 258 return 0; 259 260 NO_TEAR_ADD(rec->processed, sent); 261 NO_TEAR_ADD(rec->dropped, drops); 262 NO_TEAR_INC(rec->info); 263 if (err || drops < 0) 264 NO_TEAR_INC(rec->issue); 265 return 0; 266 } 267