1 /*
2 * eBPF RSS loader
3 *
4 * Developed by Daynix Computing LTD (http://www.daynix.com)
5 *
6 * Authors:
7 * Andrew Melnychenko <andrew@daynix.com>
8 * Yuri Benditovich <yuri.benditovich@daynix.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
12 */
13
14 #include "qemu/osdep.h"
15 #include "qemu/error-report.h"
16 #include "qapi/qapi-types-misc.h"
17 #include "qapi/qapi-commands-ebpf.h"
18
19 #include <bpf/libbpf.h>
20 #include <bpf/bpf.h>
21
22 #include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */
23
24 #include "ebpf/ebpf_rss.h"
25 #include "ebpf/rss.bpf.skeleton.h"
26 #include "ebpf/ebpf.h"
27
28 #include "trace.h"
29
ebpf_rss_init(struct EBPFRSSContext * ctx)30 void ebpf_rss_init(struct EBPFRSSContext *ctx)
31 {
32 if (ctx != NULL) {
33 ctx->obj = NULL;
34 ctx->program_fd = -1;
35 ctx->map_configuration = -1;
36 ctx->map_toeplitz_key = -1;
37 ctx->map_indirections_table = -1;
38
39 ctx->mmap_configuration = NULL;
40 ctx->mmap_toeplitz_key = NULL;
41 ctx->mmap_indirections_table = NULL;
42 }
43 }
44
ebpf_rss_is_loaded(struct EBPFRSSContext * ctx)45 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
46 {
47 return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
48 }
49
ebpf_rss_mmap(struct EBPFRSSContext * ctx)50 static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx)
51 {
52 if (!ebpf_rss_is_loaded(ctx)) {
53 return false;
54 }
55
56 ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(),
57 PROT_READ | PROT_WRITE, MAP_SHARED,
58 ctx->map_configuration, 0);
59 if (ctx->mmap_configuration == MAP_FAILED) {
60 trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array");
61 return false;
62 }
63 ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(),
64 PROT_READ | PROT_WRITE, MAP_SHARED,
65 ctx->map_toeplitz_key, 0);
66 if (ctx->mmap_toeplitz_key == MAP_FAILED) {
67 trace_ebpf_error("eBPF RSS", "can not mmap eBPF toeplitz key");
68 goto toeplitz_fail;
69 }
70 ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(),
71 PROT_READ | PROT_WRITE, MAP_SHARED,
72 ctx->map_indirections_table, 0);
73 if (ctx->mmap_indirections_table == MAP_FAILED) {
74 trace_ebpf_error("eBPF RSS", "can not mmap eBPF indirection table");
75 goto indirection_fail;
76 }
77
78 return true;
79
80 indirection_fail:
81 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
82 ctx->mmap_toeplitz_key = NULL;
83 toeplitz_fail:
84 munmap(ctx->mmap_configuration, qemu_real_host_page_size());
85 ctx->mmap_configuration = NULL;
86
87 ctx->mmap_indirections_table = NULL;
88 return false;
89 }
90
ebpf_rss_munmap(struct EBPFRSSContext * ctx)91 static void ebpf_rss_munmap(struct EBPFRSSContext *ctx)
92 {
93 if (!ebpf_rss_is_loaded(ctx)) {
94 return;
95 }
96
97 munmap(ctx->mmap_indirections_table, qemu_real_host_page_size());
98 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
99 munmap(ctx->mmap_configuration, qemu_real_host_page_size());
100
101 ctx->mmap_configuration = NULL;
102 ctx->mmap_toeplitz_key = NULL;
103 ctx->mmap_indirections_table = NULL;
104 }
105
ebpf_rss_load(struct EBPFRSSContext * ctx)106 bool ebpf_rss_load(struct EBPFRSSContext *ctx)
107 {
108 struct rss_bpf *rss_bpf_ctx;
109
110 if (ebpf_rss_is_loaded(ctx)) {
111 return false;
112 }
113
114 rss_bpf_ctx = rss_bpf__open();
115 if (rss_bpf_ctx == NULL) {
116 trace_ebpf_error("eBPF RSS", "can not open eBPF RSS object");
117 goto error;
118 }
119
120 bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER);
121
122 if (rss_bpf__load(rss_bpf_ctx)) {
123 trace_ebpf_error("eBPF RSS", "can not load RSS program");
124 goto error;
125 }
126
127 ctx->obj = rss_bpf_ctx;
128 ctx->program_fd = bpf_program__fd(
129 rss_bpf_ctx->progs.tun_rss_steering_prog);
130 ctx->map_configuration = bpf_map__fd(
131 rss_bpf_ctx->maps.tap_rss_map_configurations);
132 ctx->map_indirections_table = bpf_map__fd(
133 rss_bpf_ctx->maps.tap_rss_map_indirection_table);
134 ctx->map_toeplitz_key = bpf_map__fd(
135 rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
136
137 if (!ebpf_rss_mmap(ctx)) {
138 goto error;
139 }
140
141 return true;
142 error:
143 rss_bpf__destroy(rss_bpf_ctx);
144 ctx->obj = NULL;
145 ctx->program_fd = -1;
146 ctx->map_configuration = -1;
147 ctx->map_toeplitz_key = -1;
148 ctx->map_indirections_table = -1;
149
150 return false;
151 }
152
ebpf_rss_load_fds(struct EBPFRSSContext * ctx,int program_fd,int config_fd,int toeplitz_fd,int table_fd)153 bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
154 int config_fd, int toeplitz_fd, int table_fd)
155 {
156 if (ebpf_rss_is_loaded(ctx)) {
157 return false;
158 }
159
160 if (program_fd < 0 || config_fd < 0 || toeplitz_fd < 0 || table_fd < 0) {
161 return false;
162 }
163
164 ctx->program_fd = program_fd;
165 ctx->map_configuration = config_fd;
166 ctx->map_toeplitz_key = toeplitz_fd;
167 ctx->map_indirections_table = table_fd;
168
169 if (!ebpf_rss_mmap(ctx)) {
170 ctx->program_fd = -1;
171 ctx->map_configuration = -1;
172 ctx->map_toeplitz_key = -1;
173 ctx->map_indirections_table = -1;
174 return false;
175 }
176
177 return true;
178 }
179
ebpf_rss_set_config(struct EBPFRSSContext * ctx,struct EBPFRSSConfig * config)180 static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
181 struct EBPFRSSConfig *config)
182 {
183 if (!ebpf_rss_is_loaded(ctx)) {
184 return false;
185 }
186
187 memcpy(ctx->mmap_configuration, config, sizeof(*config));
188 return true;
189 }
190
ebpf_rss_set_indirections_table(struct EBPFRSSContext * ctx,uint16_t * indirections_table,size_t len)191 static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
192 uint16_t *indirections_table,
193 size_t len)
194 {
195 char *cursor = ctx->mmap_indirections_table;
196
197 if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
198 len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
199 return false;
200 }
201
202 for (size_t i = 0; i < len; i++) {
203 *(uint16_t *)cursor = indirections_table[i];
204 cursor += 8;
205 }
206
207 return true;
208 }
209
ebpf_rss_set_toepliz_key(struct EBPFRSSContext * ctx,uint8_t * toeplitz_key)210 static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
211 uint8_t *toeplitz_key)
212 {
213 /* prepare toeplitz key */
214 uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
215
216 if (!ebpf_rss_is_loaded(ctx) || toeplitz_key == NULL) {
217 return false;
218 }
219 memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
220 *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
221
222 memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE);
223 return true;
224 }
225
ebpf_rss_set_all(struct EBPFRSSContext * ctx,struct EBPFRSSConfig * config,uint16_t * indirections_table,uint8_t * toeplitz_key)226 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
227 uint16_t *indirections_table, uint8_t *toeplitz_key)
228 {
229 if (!ebpf_rss_is_loaded(ctx) || config == NULL ||
230 indirections_table == NULL || toeplitz_key == NULL) {
231 return false;
232 }
233
234 if (!ebpf_rss_set_config(ctx, config)) {
235 return false;
236 }
237
238 if (!ebpf_rss_set_indirections_table(ctx, indirections_table,
239 config->indirections_len)) {
240 return false;
241 }
242
243 if (!ebpf_rss_set_toepliz_key(ctx, toeplitz_key)) {
244 return false;
245 }
246
247 return true;
248 }
249
ebpf_rss_unload(struct EBPFRSSContext * ctx)250 void ebpf_rss_unload(struct EBPFRSSContext *ctx)
251 {
252 if (!ebpf_rss_is_loaded(ctx)) {
253 return;
254 }
255
256 ebpf_rss_munmap(ctx);
257
258 if (ctx->obj) {
259 rss_bpf__destroy(ctx->obj);
260 } else {
261 close(ctx->program_fd);
262 close(ctx->map_configuration);
263 close(ctx->map_toeplitz_key);
264 close(ctx->map_indirections_table);
265 }
266
267 ctx->obj = NULL;
268 ctx->program_fd = -1;
269 ctx->map_configuration = -1;
270 ctx->map_toeplitz_key = -1;
271 ctx->map_indirections_table = -1;
272 }
273
274 ebpf_binary_init(EBPF_PROGRAMID_RSS, rss_bpf__elf_bytes)
275