1 /* 2 * eBPF RSS loader 3 * 4 * Developed by Daynix Computing LTD (http://www.daynix.com) 5 * 6 * Authors: 7 * Andrew Melnychenko <andrew@daynix.com> 8 * Yuri Benditovich <yuri.benditovich@daynix.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2. See 11 * the COPYING file in the top-level directory. 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/error-report.h" 16 #include "qapi/qapi-types-misc.h" 17 #include "qapi/qapi-commands-ebpf.h" 18 19 #include <bpf/libbpf.h> 20 #include <bpf/bpf.h> 21 22 #include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */ 23 24 #include "ebpf/ebpf_rss.h" 25 #include "ebpf/rss.bpf.skeleton.h" 26 #include "ebpf/ebpf.h" 27 28 #include "trace.h" 29 30 void ebpf_rss_init(struct EBPFRSSContext *ctx) 31 { 32 if (ctx != NULL) { 33 ctx->obj = NULL; 34 ctx->program_fd = -1; 35 ctx->map_configuration = -1; 36 ctx->map_toeplitz_key = -1; 37 ctx->map_indirections_table = -1; 38 39 ctx->mmap_configuration = NULL; 40 ctx->mmap_toeplitz_key = NULL; 41 ctx->mmap_indirections_table = NULL; 42 } 43 } 44 45 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx) 46 { 47 return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1); 48 } 49 50 static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx) 51 { 52 if (!ebpf_rss_is_loaded(ctx)) { 53 return false; 54 } 55 56 ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(), 57 PROT_READ | PROT_WRITE, MAP_SHARED, 58 ctx->map_configuration, 0); 59 if (ctx->mmap_configuration == MAP_FAILED) { 60 trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array"); 61 return false; 62 } 63 ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(), 64 PROT_READ | PROT_WRITE, MAP_SHARED, 65 ctx->map_toeplitz_key, 0); 66 if (ctx->mmap_toeplitz_key == MAP_FAILED) { 67 trace_ebpf_error("eBPF RSS", "can not mmap eBPF toeplitz key"); 68 goto toeplitz_fail; 69 } 70 ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(), 71 PROT_READ | PROT_WRITE, MAP_SHARED, 72 ctx->map_indirections_table, 0); 73 if (ctx->mmap_indirections_table == MAP_FAILED) { 74 trace_ebpf_error("eBPF RSS", "can not mmap eBPF indirection table"); 75 goto indirection_fail; 76 } 77 78 return true; 79 80 indirection_fail: 81 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size()); 82 ctx->mmap_toeplitz_key = NULL; 83 toeplitz_fail: 84 munmap(ctx->mmap_configuration, qemu_real_host_page_size()); 85 ctx->mmap_configuration = NULL; 86 87 ctx->mmap_indirections_table = NULL; 88 return false; 89 } 90 91 static void ebpf_rss_munmap(struct EBPFRSSContext *ctx) 92 { 93 if (!ebpf_rss_is_loaded(ctx)) { 94 return; 95 } 96 97 munmap(ctx->mmap_indirections_table, qemu_real_host_page_size()); 98 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size()); 99 munmap(ctx->mmap_configuration, qemu_real_host_page_size()); 100 101 ctx->mmap_configuration = NULL; 102 ctx->mmap_toeplitz_key = NULL; 103 ctx->mmap_indirections_table = NULL; 104 } 105 106 bool ebpf_rss_load(struct EBPFRSSContext *ctx) 107 { 108 struct rss_bpf *rss_bpf_ctx; 109 110 if (ebpf_rss_is_loaded(ctx)) { 111 return false; 112 } 113 114 rss_bpf_ctx = rss_bpf__open(); 115 if (rss_bpf_ctx == NULL) { 116 trace_ebpf_error("eBPF RSS", "can not open eBPF RSS object"); 117 goto error; 118 } 119 120 bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER); 121 122 if (rss_bpf__load(rss_bpf_ctx)) { 123 trace_ebpf_error("eBPF RSS", "can not load RSS program"); 124 goto error; 125 } 126 127 ctx->obj = rss_bpf_ctx; 128 ctx->program_fd = bpf_program__fd( 129 rss_bpf_ctx->progs.tun_rss_steering_prog); 130 ctx->map_configuration = bpf_map__fd( 131 rss_bpf_ctx->maps.tap_rss_map_configurations); 132 ctx->map_indirections_table = bpf_map__fd( 133 rss_bpf_ctx->maps.tap_rss_map_indirection_table); 134 ctx->map_toeplitz_key = bpf_map__fd( 135 rss_bpf_ctx->maps.tap_rss_map_toeplitz_key); 136 137 if (!ebpf_rss_mmap(ctx)) { 138 goto error; 139 } 140 141 return true; 142 error: 143 rss_bpf__destroy(rss_bpf_ctx); 144 ctx->obj = NULL; 145 ctx->program_fd = -1; 146 ctx->map_configuration = -1; 147 ctx->map_toeplitz_key = -1; 148 ctx->map_indirections_table = -1; 149 150 return false; 151 } 152 153 bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd, 154 int config_fd, int toeplitz_fd, int table_fd) 155 { 156 if (ebpf_rss_is_loaded(ctx)) { 157 return false; 158 } 159 160 if (program_fd < 0 || config_fd < 0 || toeplitz_fd < 0 || table_fd < 0) { 161 return false; 162 } 163 164 ctx->program_fd = program_fd; 165 ctx->map_configuration = config_fd; 166 ctx->map_toeplitz_key = toeplitz_fd; 167 ctx->map_indirections_table = table_fd; 168 169 if (!ebpf_rss_mmap(ctx)) { 170 ctx->program_fd = -1; 171 ctx->map_configuration = -1; 172 ctx->map_toeplitz_key = -1; 173 ctx->map_indirections_table = -1; 174 return false; 175 } 176 177 return true; 178 } 179 180 static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx, 181 struct EBPFRSSConfig *config) 182 { 183 if (!ebpf_rss_is_loaded(ctx)) { 184 return false; 185 } 186 187 memcpy(ctx->mmap_configuration, config, sizeof(*config)); 188 return true; 189 } 190 191 static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx, 192 uint16_t *indirections_table, 193 size_t len) 194 { 195 char *cursor = ctx->mmap_indirections_table; 196 197 if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL || 198 len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { 199 return false; 200 } 201 202 for (size_t i = 0; i < len; i++) { 203 *(uint16_t *)cursor = indirections_table[i]; 204 cursor += 8; 205 } 206 207 return true; 208 } 209 210 static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx, 211 uint8_t *toeplitz_key) 212 { 213 /* prepare toeplitz key */ 214 uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {}; 215 216 if (!ebpf_rss_is_loaded(ctx) || toeplitz_key == NULL) { 217 return false; 218 } 219 memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE); 220 *(uint32_t *)toe = ntohl(*(uint32_t *)toe); 221 222 memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE); 223 return true; 224 } 225 226 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config, 227 uint16_t *indirections_table, uint8_t *toeplitz_key) 228 { 229 if (!ebpf_rss_is_loaded(ctx) || config == NULL || 230 indirections_table == NULL || toeplitz_key == NULL) { 231 return false; 232 } 233 234 if (!ebpf_rss_set_config(ctx, config)) { 235 return false; 236 } 237 238 if (!ebpf_rss_set_indirections_table(ctx, indirections_table, 239 config->indirections_len)) { 240 return false; 241 } 242 243 if (!ebpf_rss_set_toepliz_key(ctx, toeplitz_key)) { 244 return false; 245 } 246 247 return true; 248 } 249 250 void ebpf_rss_unload(struct EBPFRSSContext *ctx) 251 { 252 if (!ebpf_rss_is_loaded(ctx)) { 253 return; 254 } 255 256 ebpf_rss_munmap(ctx); 257 258 if (ctx->obj) { 259 rss_bpf__destroy(ctx->obj); 260 } else { 261 close(ctx->program_fd); 262 close(ctx->map_configuration); 263 close(ctx->map_toeplitz_key); 264 close(ctx->map_indirections_table); 265 } 266 267 ctx->obj = NULL; 268 ctx->program_fd = -1; 269 ctx->map_configuration = -1; 270 ctx->map_toeplitz_key = -1; 271 ctx->map_indirections_table = -1; 272 } 273 274 ebpf_binary_init(EBPF_PROGRAM_ID_RSS, rss_bpf__elf_bytes) 275