1 /* 2 * eBPF RSS loader 3 * 4 * Developed by Daynix Computing LTD (http://www.daynix.com) 5 * 6 * Authors: 7 * Andrew Melnychenko <andrew@daynix.com> 8 * Yuri Benditovich <yuri.benditovich@daynix.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2. See 11 * the COPYING file in the top-level directory. 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/error-report.h" 16 #include "qapi/qapi-types-misc.h" 17 #include "qapi/qapi-commands-ebpf.h" 18 19 #include <bpf/libbpf.h> 20 #include <bpf/bpf.h> 21 22 #include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */ 23 24 #include "ebpf/ebpf_rss.h" 25 #include "ebpf/rss.bpf.skeleton.h" 26 #include "ebpf/ebpf.h" 27 28 #include "trace.h" 29 30 void ebpf_rss_init(struct EBPFRSSContext *ctx) 31 { 32 if (ctx != NULL) { 33 ctx->obj = NULL; 34 ctx->program_fd = -1; 35 ctx->map_configuration = -1; 36 ctx->map_toeplitz_key = -1; 37 ctx->map_indirections_table = -1; 38 39 ctx->mmap_configuration = NULL; 40 ctx->mmap_toeplitz_key = NULL; 41 ctx->mmap_indirections_table = NULL; 42 } 43 } 44 45 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx) 46 { 47 return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1); 48 } 49 50 static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx, Error **errp) 51 { 52 ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(), 53 PROT_READ | PROT_WRITE, MAP_SHARED, 54 ctx->map_configuration, 0); 55 if (ctx->mmap_configuration == MAP_FAILED) { 56 trace_ebpf_rss_mmap_error(ctx, "configuration"); 57 error_setg(errp, "Unable to map eBPF configuration array"); 58 return false; 59 } 60 ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(), 61 PROT_READ | PROT_WRITE, MAP_SHARED, 62 ctx->map_toeplitz_key, 0); 63 if (ctx->mmap_toeplitz_key == MAP_FAILED) { 64 trace_ebpf_rss_mmap_error(ctx, "toeplitz key"); 65 error_setg(errp, "Unable to map eBPF toeplitz array"); 66 goto toeplitz_fail; 67 } 68 ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(), 69 PROT_READ | PROT_WRITE, MAP_SHARED, 70 ctx->map_indirections_table, 0); 71 if (ctx->mmap_indirections_table == MAP_FAILED) { 72 trace_ebpf_rss_mmap_error(ctx, "indirections table"); 73 error_setg(errp, "Unable to map eBPF indirection array"); 74 goto indirection_fail; 75 } 76 77 trace_ebpf_rss_mmap(ctx, 78 ctx->mmap_configuration, 79 ctx->mmap_toeplitz_key, 80 ctx->mmap_indirections_table); 81 return true; 82 83 indirection_fail: 84 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size()); 85 ctx->mmap_toeplitz_key = NULL; 86 toeplitz_fail: 87 munmap(ctx->mmap_configuration, qemu_real_host_page_size()); 88 ctx->mmap_configuration = NULL; 89 90 ctx->mmap_indirections_table = NULL; 91 return false; 92 } 93 94 static void ebpf_rss_munmap(struct EBPFRSSContext *ctx) 95 { 96 munmap(ctx->mmap_indirections_table, qemu_real_host_page_size()); 97 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size()); 98 munmap(ctx->mmap_configuration, qemu_real_host_page_size()); 99 100 ctx->mmap_configuration = NULL; 101 ctx->mmap_toeplitz_key = NULL; 102 ctx->mmap_indirections_table = NULL; 103 } 104 105 bool ebpf_rss_load(struct EBPFRSSContext *ctx, Error **errp) 106 { 107 struct rss_bpf *rss_bpf_ctx; 108 109 if (ebpf_rss_is_loaded(ctx)) { 110 return false; 111 } 112 113 rss_bpf_ctx = rss_bpf__open(); 114 if (rss_bpf_ctx == NULL) { 115 trace_ebpf_rss_open_error(ctx); 116 error_setg(errp, "Unable to open eBPF RSS object"); 117 goto error; 118 } 119 120 bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER); 121 122 if (rss_bpf__load(rss_bpf_ctx)) { 123 trace_ebpf_rss_load_error(ctx); 124 error_setg(errp, "Unable to load eBPF program"); 125 goto error; 126 } 127 128 ctx->obj = rss_bpf_ctx; 129 ctx->program_fd = bpf_program__fd( 130 rss_bpf_ctx->progs.tun_rss_steering_prog); 131 ctx->map_configuration = bpf_map__fd( 132 rss_bpf_ctx->maps.tap_rss_map_configurations); 133 ctx->map_indirections_table = bpf_map__fd( 134 rss_bpf_ctx->maps.tap_rss_map_indirection_table); 135 ctx->map_toeplitz_key = bpf_map__fd( 136 rss_bpf_ctx->maps.tap_rss_map_toeplitz_key); 137 138 trace_ebpf_rss_load(ctx, 139 ctx->program_fd, 140 ctx->map_configuration, 141 ctx->map_indirections_table, 142 ctx->map_toeplitz_key); 143 if (!ebpf_rss_mmap(ctx, errp)) { 144 goto error; 145 } 146 147 return true; 148 error: 149 rss_bpf__destroy(rss_bpf_ctx); 150 ctx->obj = NULL; 151 ctx->program_fd = -1; 152 ctx->map_configuration = -1; 153 ctx->map_toeplitz_key = -1; 154 ctx->map_indirections_table = -1; 155 156 return false; 157 } 158 159 bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd, 160 int config_fd, int toeplitz_fd, int table_fd, 161 Error **errp) 162 { 163 if (ebpf_rss_is_loaded(ctx)) { 164 error_setg(errp, "eBPF program is already loaded"); 165 return false; 166 } 167 168 if (program_fd < 0) { 169 error_setg(errp, "eBPF program FD is not open"); 170 return false; 171 } 172 if (config_fd < 0) { 173 error_setg(errp, "eBPF config FD is not open"); 174 return false; 175 } 176 if (toeplitz_fd < 0) { 177 error_setg(errp, "eBPF toeplitz FD is not open"); 178 return false; 179 } 180 if (table_fd < 0) { 181 error_setg(errp, "eBPF indirection FD is not open"); 182 return false; 183 } 184 185 ctx->program_fd = program_fd; 186 ctx->map_configuration = config_fd; 187 ctx->map_toeplitz_key = toeplitz_fd; 188 ctx->map_indirections_table = table_fd; 189 190 trace_ebpf_rss_load(ctx, 191 ctx->program_fd, 192 ctx->map_configuration, 193 ctx->map_indirections_table, 194 ctx->map_toeplitz_key); 195 196 if (!ebpf_rss_mmap(ctx, errp)) { 197 ctx->program_fd = -1; 198 ctx->map_configuration = -1; 199 ctx->map_toeplitz_key = -1; 200 ctx->map_indirections_table = -1; 201 return false; 202 } 203 204 return true; 205 } 206 207 static void ebpf_rss_set_config(struct EBPFRSSContext *ctx, 208 struct EBPFRSSConfig *config) 209 { 210 memcpy(ctx->mmap_configuration, config, sizeof(*config)); 211 } 212 213 static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx, 214 uint16_t *indirections_table, 215 size_t len, 216 Error **errp) 217 { 218 char *cursor = ctx->mmap_indirections_table; 219 220 if (len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { 221 error_setg(errp, "Indirections table length %zu exceeds limit %d", 222 len, VIRTIO_NET_RSS_MAX_TABLE_LEN); 223 return false; 224 } 225 226 for (size_t i = 0; i < len; i++) { 227 *(uint16_t *)cursor = indirections_table[i]; 228 cursor += 8; 229 } 230 231 return true; 232 } 233 234 static void ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx, 235 uint8_t *toeplitz_key) 236 { 237 /* prepare toeplitz key */ 238 uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {}; 239 240 memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE); 241 *(uint32_t *)toe = ntohl(*(uint32_t *)toe); 242 243 memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE); 244 } 245 246 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, 247 uint16_t *indirections_table, uint8_t *toeplitz_key, 248 Error **errp) 249 { 250 if (!ebpf_rss_is_loaded(ctx)) { 251 error_setg(errp, "eBPF program is not loaded"); 252 return false; 253 } 254 if (config == NULL) { 255 error_setg(errp, "eBPF config table is NULL"); 256 return false; 257 } 258 if (indirections_table == NULL) { 259 error_setg(errp, "eBPF indirections table is NULL"); 260 return false; 261 } 262 if (toeplitz_key == NULL) { 263 error_setg(errp, "eBPF toeplitz key is NULL"); 264 return false; 265 } 266 267 ebpf_rss_set_config(ctx, config); 268 269 if (!ebpf_rss_set_indirections_table(ctx, indirections_table, 270 config->indirections_len, 271 errp)) { 272 return false; 273 } 274 275 ebpf_rss_set_toepliz_key(ctx, toeplitz_key); 276 277 trace_ebpf_rss_set_data(ctx, config, indirections_table, toeplitz_key); 278 279 return true; 280 } 281 282 void ebpf_rss_unload(struct EBPFRSSContext *ctx) 283 { 284 if (!ebpf_rss_is_loaded(ctx)) { 285 return; 286 } 287 288 trace_ebpf_rss_unload(ctx); 289 290 ebpf_rss_munmap(ctx); 291 292 if (ctx->obj) { 293 rss_bpf__destroy(ctx->obj); 294 } else { 295 close(ctx->program_fd); 296 close(ctx->map_configuration); 297 close(ctx->map_toeplitz_key); 298 close(ctx->map_indirections_table); 299 } 300 301 ctx->obj = NULL; 302 ctx->program_fd = -1; 303 ctx->map_configuration = -1; 304 ctx->map_toeplitz_key = -1; 305 ctx->map_indirections_table = -1; 306 } 307 308 ebpf_binary_init(EBPF_PROGRAM_ID_RSS, rss_bpf__elf_bytes) 309