1 /* 2 * eBPF RSS loader 3 * 4 * Developed by Daynix Computing LTD (http://www.daynix.com) 5 * 6 * Authors: 7 * Andrew Melnychenko <andrew@daynix.com> 8 * Yuri Benditovich <yuri.benditovich@daynix.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2. See 11 * the COPYING file in the top-level directory. 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/error-report.h" 16 #include "qapi/qapi-types-misc.h" 17 #include "qapi/qapi-commands-ebpf.h" 18 19 #include <bpf/libbpf.h> 20 #include <bpf/bpf.h> 21 22 #include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */ 23 24 #include "ebpf/ebpf_rss.h" 25 #include "ebpf/rss.bpf.skeleton.h" 26 #include "ebpf/ebpf.h" 27 28 #include "trace.h" 29 30 void ebpf_rss_init(struct EBPFRSSContext *ctx) 31 { 32 if (ctx != NULL) { 33 ctx->obj = NULL; 34 ctx->program_fd = -1; 35 ctx->map_configuration = -1; 36 ctx->map_toeplitz_key = -1; 37 ctx->map_indirections_table = -1; 38 39 ctx->mmap_configuration = NULL; 40 ctx->mmap_toeplitz_key = NULL; 41 ctx->mmap_indirections_table = NULL; 42 } 43 } 44 45 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx) 46 { 47 return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1); 48 } 49 50 static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx, Error **errp) 51 { 52 ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(), 53 PROT_READ | PROT_WRITE, MAP_SHARED, 54 ctx->map_configuration, 0); 55 if (ctx->mmap_configuration == MAP_FAILED) { 56 trace_ebpf_rss_mmap_error(ctx, "configuration"); 57 error_setg(errp, "Unable to map eBPF configuration array"); 58 return false; 59 } 60 ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(), 61 PROT_READ | PROT_WRITE, MAP_SHARED, 62 ctx->map_toeplitz_key, 0); 63 if (ctx->mmap_toeplitz_key == MAP_FAILED) { 64 trace_ebpf_rss_mmap_error(ctx, "toeplitz key"); 65 error_setg(errp, "Unable to map eBPF toeplitz array"); 66 goto toeplitz_fail; 67 } 68 ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(), 69 PROT_READ | PROT_WRITE, MAP_SHARED, 70 ctx->map_indirections_table, 0); 71 if (ctx->mmap_indirections_table == MAP_FAILED) { 72 trace_ebpf_rss_mmap_error(ctx, "indirections table"); 73 error_setg(errp, "Unable to map eBPF indirection array"); 74 goto indirection_fail; 75 } 76 77 return true; 78 79 indirection_fail: 80 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size()); 81 ctx->mmap_toeplitz_key = NULL; 82 toeplitz_fail: 83 munmap(ctx->mmap_configuration, qemu_real_host_page_size()); 84 ctx->mmap_configuration = NULL; 85 86 ctx->mmap_indirections_table = NULL; 87 return false; 88 } 89 90 static void ebpf_rss_munmap(struct EBPFRSSContext *ctx) 91 { 92 munmap(ctx->mmap_indirections_table, qemu_real_host_page_size()); 93 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size()); 94 munmap(ctx->mmap_configuration, qemu_real_host_page_size()); 95 96 ctx->mmap_configuration = NULL; 97 ctx->mmap_toeplitz_key = NULL; 98 ctx->mmap_indirections_table = NULL; 99 } 100 101 bool ebpf_rss_load(struct EBPFRSSContext *ctx, Error **errp) 102 { 103 struct rss_bpf *rss_bpf_ctx; 104 105 if (ebpf_rss_is_loaded(ctx)) { 106 return false; 107 } 108 109 rss_bpf_ctx = rss_bpf__open(); 110 if (rss_bpf_ctx == NULL) { 111 trace_ebpf_rss_open_error(ctx); 112 error_setg(errp, "Unable to open eBPF RSS object"); 113 goto error; 114 } 115 116 bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER); 117 118 if (rss_bpf__load(rss_bpf_ctx)) { 119 trace_ebpf_rss_load_error(ctx); 120 error_setg(errp, "Unable to load eBPF program"); 121 goto error; 122 } 123 124 ctx->obj = rss_bpf_ctx; 125 ctx->program_fd = bpf_program__fd( 126 rss_bpf_ctx->progs.tun_rss_steering_prog); 127 ctx->map_configuration = bpf_map__fd( 128 rss_bpf_ctx->maps.tap_rss_map_configurations); 129 ctx->map_indirections_table = bpf_map__fd( 130 rss_bpf_ctx->maps.tap_rss_map_indirection_table); 131 ctx->map_toeplitz_key = bpf_map__fd( 132 rss_bpf_ctx->maps.tap_rss_map_toeplitz_key); 133 134 if (!ebpf_rss_mmap(ctx, errp)) { 135 goto error; 136 } 137 138 return true; 139 error: 140 rss_bpf__destroy(rss_bpf_ctx); 141 ctx->obj = NULL; 142 ctx->program_fd = -1; 143 ctx->map_configuration = -1; 144 ctx->map_toeplitz_key = -1; 145 ctx->map_indirections_table = -1; 146 147 return false; 148 } 149 150 bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd, 151 int config_fd, int toeplitz_fd, int table_fd, 152 Error **errp) 153 { 154 if (ebpf_rss_is_loaded(ctx)) { 155 error_setg(errp, "eBPF program is already loaded"); 156 return false; 157 } 158 159 if (program_fd < 0) { 160 error_setg(errp, "eBPF program FD is not open"); 161 return false; 162 } 163 if (config_fd < 0) { 164 error_setg(errp, "eBPF config FD is not open"); 165 return false; 166 } 167 if (toeplitz_fd < 0) { 168 error_setg(errp, "eBPF toeplitz FD is not open"); 169 return false; 170 } 171 if (table_fd < 0) { 172 error_setg(errp, "eBPF indirection FD is not open"); 173 return false; 174 } 175 176 ctx->program_fd = program_fd; 177 ctx->map_configuration = config_fd; 178 ctx->map_toeplitz_key = toeplitz_fd; 179 ctx->map_indirections_table = table_fd; 180 181 if (!ebpf_rss_mmap(ctx, errp)) { 182 ctx->program_fd = -1; 183 ctx->map_configuration = -1; 184 ctx->map_toeplitz_key = -1; 185 ctx->map_indirections_table = -1; 186 return false; 187 } 188 189 return true; 190 } 191 192 static void ebpf_rss_set_config(struct EBPFRSSContext *ctx, 193 struct EBPFRSSConfig *config) 194 { 195 memcpy(ctx->mmap_configuration, config, sizeof(*config)); 196 } 197 198 static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx, 199 uint16_t *indirections_table, 200 size_t len, 201 Error **errp) 202 { 203 char *cursor = ctx->mmap_indirections_table; 204 205 if (len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { 206 error_setg(errp, "Indirections table length %zu exceeds limit %d", 207 len, VIRTIO_NET_RSS_MAX_TABLE_LEN); 208 return false; 209 } 210 211 for (size_t i = 0; i < len; i++) { 212 *(uint16_t *)cursor = indirections_table[i]; 213 cursor += 8; 214 } 215 216 return true; 217 } 218 219 static void ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx, 220 uint8_t *toeplitz_key) 221 { 222 /* prepare toeplitz key */ 223 uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {}; 224 225 memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE); 226 *(uint32_t *)toe = ntohl(*(uint32_t *)toe); 227 228 memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE); 229 } 230 231 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, 232 uint16_t *indirections_table, uint8_t *toeplitz_key, 233 Error **errp) 234 { 235 if (!ebpf_rss_is_loaded(ctx)) { 236 error_setg(errp, "eBPF program is not loaded"); 237 return false; 238 } 239 if (config == NULL) { 240 error_setg(errp, "eBPF config table is NULL"); 241 return false; 242 } 243 if (indirections_table == NULL) { 244 error_setg(errp, "eBPF indirections table is NULL"); 245 return false; 246 } 247 if (toeplitz_key == NULL) { 248 error_setg(errp, "eBPF toeplitz key is NULL"); 249 return false; 250 } 251 252 ebpf_rss_set_config(ctx, config); 253 254 if (!ebpf_rss_set_indirections_table(ctx, indirections_table, 255 config->indirections_len, 256 errp)) { 257 return false; 258 } 259 260 ebpf_rss_set_toepliz_key(ctx, toeplitz_key); 261 262 return true; 263 } 264 265 void ebpf_rss_unload(struct EBPFRSSContext *ctx) 266 { 267 if (!ebpf_rss_is_loaded(ctx)) { 268 return; 269 } 270 271 ebpf_rss_munmap(ctx); 272 273 if (ctx->obj) { 274 rss_bpf__destroy(ctx->obj); 275 } else { 276 close(ctx->program_fd); 277 close(ctx->map_configuration); 278 close(ctx->map_toeplitz_key); 279 close(ctx->map_indirections_table); 280 } 281 282 ctx->obj = NULL; 283 ctx->program_fd = -1; 284 ctx->map_configuration = -1; 285 ctx->map_toeplitz_key = -1; 286 ctx->map_indirections_table = -1; 287 } 288 289 ebpf_binary_init(EBPF_PROGRAM_ID_RSS, rss_bpf__elf_bytes) 290