1 /*
2 * eBPF RSS loader
3 *
4 * Developed by Daynix Computing LTD (http://www.daynix.com)
5 *
6 * Authors:
7 * Andrew Melnychenko <andrew@daynix.com>
8 * Yuri Benditovich <yuri.benditovich@daynix.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
12 */
13
14 #include "qemu/osdep.h"
15 #include "qemu/error-report.h"
16 #include "qapi/qapi-types-misc.h"
17 #include "qapi/qapi-commands-ebpf.h"
18
19 #include <bpf/libbpf.h>
20 #include <bpf/bpf.h>
21
22 #include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */
23
24 #include "ebpf/ebpf_rss.h"
25 #include "ebpf/rss.bpf.skeleton.h"
26 #include "ebpf/ebpf.h"
27
28 #include "trace.h"
29
ebpf_rss_init(struct EBPFRSSContext * ctx)30 void ebpf_rss_init(struct EBPFRSSContext *ctx)
31 {
32 if (ctx != NULL) {
33 ctx->obj = NULL;
34 ctx->program_fd = -1;
35 ctx->map_configuration = -1;
36 ctx->map_toeplitz_key = -1;
37 ctx->map_indirections_table = -1;
38
39 ctx->mmap_configuration = NULL;
40 ctx->mmap_toeplitz_key = NULL;
41 ctx->mmap_indirections_table = NULL;
42 }
43 }
44
ebpf_rss_is_loaded(struct EBPFRSSContext * ctx)45 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
46 {
47 return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
48 }
49
ebpf_rss_mmap(struct EBPFRSSContext * ctx,Error ** errp)50 static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx, Error **errp)
51 {
52 ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(),
53 PROT_READ | PROT_WRITE, MAP_SHARED,
54 ctx->map_configuration, 0);
55 if (ctx->mmap_configuration == MAP_FAILED) {
56 trace_ebpf_rss_mmap_error(ctx, "configuration");
57 error_setg(errp, "Unable to map eBPF configuration array");
58 return false;
59 }
60 ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(),
61 PROT_READ | PROT_WRITE, MAP_SHARED,
62 ctx->map_toeplitz_key, 0);
63 if (ctx->mmap_toeplitz_key == MAP_FAILED) {
64 trace_ebpf_rss_mmap_error(ctx, "toeplitz key");
65 error_setg(errp, "Unable to map eBPF toeplitz array");
66 goto toeplitz_fail;
67 }
68 ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(),
69 PROT_READ | PROT_WRITE, MAP_SHARED,
70 ctx->map_indirections_table, 0);
71 if (ctx->mmap_indirections_table == MAP_FAILED) {
72 trace_ebpf_rss_mmap_error(ctx, "indirections table");
73 error_setg(errp, "Unable to map eBPF indirection array");
74 goto indirection_fail;
75 }
76
77 trace_ebpf_rss_mmap(ctx,
78 ctx->mmap_configuration,
79 ctx->mmap_toeplitz_key,
80 ctx->mmap_indirections_table);
81 return true;
82
83 indirection_fail:
84 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
85 ctx->mmap_toeplitz_key = NULL;
86 toeplitz_fail:
87 munmap(ctx->mmap_configuration, qemu_real_host_page_size());
88 ctx->mmap_configuration = NULL;
89
90 ctx->mmap_indirections_table = NULL;
91 return false;
92 }
93
ebpf_rss_munmap(struct EBPFRSSContext * ctx)94 static void ebpf_rss_munmap(struct EBPFRSSContext *ctx)
95 {
96 munmap(ctx->mmap_indirections_table, qemu_real_host_page_size());
97 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
98 munmap(ctx->mmap_configuration, qemu_real_host_page_size());
99
100 ctx->mmap_configuration = NULL;
101 ctx->mmap_toeplitz_key = NULL;
102 ctx->mmap_indirections_table = NULL;
103 }
104
ebpf_rss_load(struct EBPFRSSContext * ctx,Error ** errp)105 bool ebpf_rss_load(struct EBPFRSSContext *ctx, Error **errp)
106 {
107 struct rss_bpf *rss_bpf_ctx;
108
109 if (ebpf_rss_is_loaded(ctx)) {
110 return false;
111 }
112
113 rss_bpf_ctx = rss_bpf__open();
114 if (rss_bpf_ctx == NULL) {
115 trace_ebpf_rss_open_error(ctx);
116 error_setg(errp, "Unable to open eBPF RSS object");
117 goto error;
118 }
119
120 bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER);
121
122 if (rss_bpf__load(rss_bpf_ctx)) {
123 trace_ebpf_rss_load_error(ctx);
124 error_setg(errp, "Unable to load eBPF program");
125 goto error;
126 }
127
128 ctx->obj = rss_bpf_ctx;
129 ctx->program_fd = bpf_program__fd(
130 rss_bpf_ctx->progs.tun_rss_steering_prog);
131 ctx->map_configuration = bpf_map__fd(
132 rss_bpf_ctx->maps.tap_rss_map_configurations);
133 ctx->map_indirections_table = bpf_map__fd(
134 rss_bpf_ctx->maps.tap_rss_map_indirection_table);
135 ctx->map_toeplitz_key = bpf_map__fd(
136 rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
137
138 trace_ebpf_rss_load(ctx,
139 ctx->program_fd,
140 ctx->map_configuration,
141 ctx->map_indirections_table,
142 ctx->map_toeplitz_key);
143 if (!ebpf_rss_mmap(ctx, errp)) {
144 goto error;
145 }
146
147 return true;
148 error:
149 rss_bpf__destroy(rss_bpf_ctx);
150 ctx->obj = NULL;
151 ctx->program_fd = -1;
152 ctx->map_configuration = -1;
153 ctx->map_toeplitz_key = -1;
154 ctx->map_indirections_table = -1;
155
156 return false;
157 }
158
ebpf_rss_load_fds(struct EBPFRSSContext * ctx,int program_fd,int config_fd,int toeplitz_fd,int table_fd,Error ** errp)159 bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
160 int config_fd, int toeplitz_fd, int table_fd,
161 Error **errp)
162 {
163 if (ebpf_rss_is_loaded(ctx)) {
164 error_setg(errp, "eBPF program is already loaded");
165 return false;
166 }
167
168 if (program_fd < 0) {
169 error_setg(errp, "eBPF program FD is not open");
170 return false;
171 }
172 if (config_fd < 0) {
173 error_setg(errp, "eBPF config FD is not open");
174 return false;
175 }
176 if (toeplitz_fd < 0) {
177 error_setg(errp, "eBPF toeplitz FD is not open");
178 return false;
179 }
180 if (table_fd < 0) {
181 error_setg(errp, "eBPF indirection FD is not open");
182 return false;
183 }
184
185 ctx->program_fd = program_fd;
186 ctx->map_configuration = config_fd;
187 ctx->map_toeplitz_key = toeplitz_fd;
188 ctx->map_indirections_table = table_fd;
189
190 trace_ebpf_rss_load(ctx,
191 ctx->program_fd,
192 ctx->map_configuration,
193 ctx->map_indirections_table,
194 ctx->map_toeplitz_key);
195
196 if (!ebpf_rss_mmap(ctx, errp)) {
197 ctx->program_fd = -1;
198 ctx->map_configuration = -1;
199 ctx->map_toeplitz_key = -1;
200 ctx->map_indirections_table = -1;
201 return false;
202 }
203
204 return true;
205 }
206
ebpf_rss_set_config(struct EBPFRSSContext * ctx,struct EBPFRSSConfig * config)207 static void ebpf_rss_set_config(struct EBPFRSSContext *ctx,
208 struct EBPFRSSConfig *config)
209 {
210 memcpy(ctx->mmap_configuration, config, sizeof(*config));
211 }
212
ebpf_rss_set_indirections_table(struct EBPFRSSContext * ctx,uint16_t * indirections_table,size_t len,Error ** errp)213 static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
214 uint16_t *indirections_table,
215 size_t len,
216 Error **errp)
217 {
218 char *cursor = ctx->mmap_indirections_table;
219
220 if (len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
221 error_setg(errp, "Indirections table length %zu exceeds limit %d",
222 len, VIRTIO_NET_RSS_MAX_TABLE_LEN);
223 return false;
224 }
225
226 for (size_t i = 0; i < len; i++) {
227 *(uint16_t *)cursor = indirections_table[i];
228 cursor += 8;
229 }
230
231 return true;
232 }
233
ebpf_rss_set_toepliz_key(struct EBPFRSSContext * ctx,uint8_t * toeplitz_key)234 static void ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
235 uint8_t *toeplitz_key)
236 {
237 /* prepare toeplitz key */
238 uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
239
240 memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
241 *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
242
243 memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE);
244 }
245
ebpf_rss_set_all(struct EBPFRSSContext * ctx,struct EBPFRSSConfig * config,uint16_t * indirections_table,uint8_t * toeplitz_key,Error ** errp)246 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
247 uint16_t *indirections_table, uint8_t *toeplitz_key,
248 Error **errp)
249 {
250 if (!ebpf_rss_is_loaded(ctx)) {
251 error_setg(errp, "eBPF program is not loaded");
252 return false;
253 }
254 if (config == NULL) {
255 error_setg(errp, "eBPF config table is NULL");
256 return false;
257 }
258 if (indirections_table == NULL) {
259 error_setg(errp, "eBPF indirections table is NULL");
260 return false;
261 }
262 if (toeplitz_key == NULL) {
263 error_setg(errp, "eBPF toeplitz key is NULL");
264 return false;
265 }
266
267 ebpf_rss_set_config(ctx, config);
268
269 if (!ebpf_rss_set_indirections_table(ctx, indirections_table,
270 config->indirections_len,
271 errp)) {
272 return false;
273 }
274
275 ebpf_rss_set_toepliz_key(ctx, toeplitz_key);
276
277 trace_ebpf_rss_set_data(ctx, config, indirections_table, toeplitz_key);
278
279 return true;
280 }
281
ebpf_rss_unload(struct EBPFRSSContext * ctx)282 void ebpf_rss_unload(struct EBPFRSSContext *ctx)
283 {
284 if (!ebpf_rss_is_loaded(ctx)) {
285 return;
286 }
287
288 trace_ebpf_rss_unload(ctx);
289
290 ebpf_rss_munmap(ctx);
291
292 if (ctx->obj) {
293 rss_bpf__destroy(ctx->obj);
294 } else {
295 close(ctx->program_fd);
296 close(ctx->map_configuration);
297 close(ctx->map_toeplitz_key);
298 close(ctx->map_indirections_table);
299 }
300
301 ctx->obj = NULL;
302 ctx->program_fd = -1;
303 ctx->map_configuration = -1;
304 ctx->map_toeplitz_key = -1;
305 ctx->map_indirections_table = -1;
306 }
307
308 ebpf_binary_init(EBPF_PROGRAM_ID_RSS, rss_bpf__elf_bytes)
309