1 /*
2 * Multifd UADK compression accelerator implementation
3 *
4 * Copyright (c) 2024 Huawei Technologies R & D (UK) Ltd
5 *
6 * Authors:
7 * Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13 #include "qemu/osdep.h"
14 #include "qemu/module.h"
15 #include "qapi/error.h"
16 #include "exec/ramblock.h"
17 #include "migration.h"
18 #include "multifd.h"
19 #include "options.h"
20 #include "qemu/error-report.h"
21 #include "uadk/wd_comp.h"
22 #include "uadk/wd_sched.h"
23
24 struct wd_data {
25 handle_t handle;
26 uint8_t *buf;
27 uint32_t *buf_hdr;
28 };
29
uadk_hw_init(void)30 static bool uadk_hw_init(void)
31 {
32 char alg[] = "zlib";
33 int ret;
34
35 ret = wd_comp_init2(alg, SCHED_POLICY_RR, TASK_HW);
36 if (ret && ret != -WD_EEXIST) {
37 return false;
38 } else {
39 return true;
40 }
41 }
42
multifd_uadk_init_sess(uint32_t count,uint32_t page_size,bool compress,Error ** errp)43 static struct wd_data *multifd_uadk_init_sess(uint32_t count,
44 uint32_t page_size,
45 bool compress, Error **errp)
46 {
47 struct wd_comp_sess_setup ss = {0};
48 struct sched_params param = {0};
49 uint32_t size = count * page_size;
50 struct wd_data *wd;
51
52 wd = g_new0(struct wd_data, 1);
53
54 if (uadk_hw_init()) {
55 ss.alg_type = WD_ZLIB;
56 if (compress) {
57 ss.op_type = WD_DIR_COMPRESS;
58 /* Add an additional page for handling output > input */
59 size += page_size;
60 } else {
61 ss.op_type = WD_DIR_DECOMPRESS;
62 }
63 /* We use default level 1 compression and 4K window size */
64 param.type = ss.op_type;
65 ss.sched_param = ¶m;
66
67 wd->handle = wd_comp_alloc_sess(&ss);
68 if (!wd->handle) {
69 error_setg(errp, "multifd: failed wd_comp_alloc_sess");
70 goto out;
71 }
72 } else {
73 /* For CI test use */
74 warn_report_once("UADK hardware not available. Switch to no compression mode");
75 }
76
77 wd->buf = g_try_malloc(size);
78 if (!wd->buf) {
79 error_setg(errp, "multifd: out of mem for uadk buf");
80 goto out_free_sess;
81 }
82 wd->buf_hdr = g_new0(uint32_t, count);
83 return wd;
84
85 out_free_sess:
86 if (wd->handle) {
87 wd_comp_free_sess(wd->handle);
88 }
89 out:
90 wd_comp_uninit2();
91 g_free(wd);
92 return NULL;
93 }
94
multifd_uadk_uninit_sess(struct wd_data * wd)95 static void multifd_uadk_uninit_sess(struct wd_data *wd)
96 {
97 if (wd->handle) {
98 wd_comp_free_sess(wd->handle);
99 }
100 wd_comp_uninit2();
101 g_free(wd->buf);
102 g_free(wd->buf_hdr);
103 g_free(wd);
104 }
105
multifd_uadk_send_setup(MultiFDSendParams * p,Error ** errp)106 static int multifd_uadk_send_setup(MultiFDSendParams *p, Error **errp)
107 {
108 struct wd_data *wd;
109 uint32_t page_size = multifd_ram_page_size();
110 uint32_t page_count = multifd_ram_page_count();
111
112 wd = multifd_uadk_init_sess(page_count, page_size, true, errp);
113 if (!wd) {
114 return -1;
115 }
116
117 p->compress_data = wd;
118 assert(p->iov == NULL);
119 /*
120 * Each page will be compressed independently and sent using an IOV. The
121 * additional two IOVs are used to store packet header and compressed data
122 * length
123 */
124
125 p->iov = g_new0(struct iovec, page_count + 2);
126 return 0;
127 }
128
multifd_uadk_send_cleanup(MultiFDSendParams * p,Error ** errp)129 static void multifd_uadk_send_cleanup(MultiFDSendParams *p, Error **errp)
130 {
131 struct wd_data *wd = p->compress_data;
132
133 multifd_uadk_uninit_sess(wd);
134 p->compress_data = NULL;
135 g_free(p->iov);
136 p->iov = NULL;
137 }
138
prepare_next_iov(MultiFDSendParams * p,void * base,uint32_t len)139 static inline void prepare_next_iov(MultiFDSendParams *p, void *base,
140 uint32_t len)
141 {
142 p->iov[p->iovs_num].iov_base = (uint8_t *)base;
143 p->iov[p->iovs_num].iov_len = len;
144 p->next_packet_size += len;
145 p->iovs_num++;
146 }
147
multifd_uadk_send_prepare(MultiFDSendParams * p,Error ** errp)148 static int multifd_uadk_send_prepare(MultiFDSendParams *p, Error **errp)
149 {
150 struct wd_data *uadk_data = p->compress_data;
151 uint32_t hdr_size;
152 uint32_t page_size = multifd_ram_page_size();
153 uint8_t *buf = uadk_data->buf;
154 int ret = 0;
155 MultiFDPages_t *pages = &p->data->u.ram;
156
157 if (!multifd_send_prepare_common(p)) {
158 goto out;
159 }
160
161 hdr_size = pages->normal_num * sizeof(uint32_t);
162 /* prepare the header that stores the lengths of all compressed data */
163 prepare_next_iov(p, uadk_data->buf_hdr, hdr_size);
164
165 for (int i = 0; i < pages->normal_num; i++) {
166 struct wd_comp_req creq = {
167 .op_type = WD_DIR_COMPRESS,
168 .src = pages->block->host + pages->offset[i],
169 .src_len = page_size,
170 .dst = buf,
171 /* Set dst_len to double the src in case compressed out >= page_size */
172 .dst_len = p->page_size * 2,
173 };
174
175 if (uadk_data->handle) {
176 ret = wd_do_comp_sync(uadk_data->handle, &creq);
177 if (ret || creq.status) {
178 error_setg(errp, "multifd %u: failed compression, ret %d status %d",
179 p->id, ret, creq.status);
180 return -1;
181 }
182 if (creq.dst_len < page_size) {
183 uadk_data->buf_hdr[i] = cpu_to_be32(creq.dst_len);
184 prepare_next_iov(p, buf, creq.dst_len);
185 buf += creq.dst_len;
186 }
187 }
188 /*
189 * Send raw data if no UADK hardware or if compressed out >= page_size.
190 * We might be better off sending raw data if output is slightly less
191 * than page_size as well because at the receive end we can skip the
192 * decompression. But it is tricky to find the right number here.
193 */
194 if (!uadk_data->handle || creq.dst_len >= page_size) {
195 uadk_data->buf_hdr[i] = cpu_to_be32(page_size);
196 prepare_next_iov(p, pages->block->host + pages->offset[i],
197 page_size);
198 buf += page_size;
199 }
200 }
201 out:
202 p->flags |= MULTIFD_FLAG_UADK;
203 multifd_send_fill_packet(p);
204 return 0;
205 }
206
multifd_uadk_recv_setup(MultiFDRecvParams * p,Error ** errp)207 static int multifd_uadk_recv_setup(MultiFDRecvParams *p, Error **errp)
208 {
209 struct wd_data *wd;
210 uint32_t page_size = multifd_ram_page_size();
211 uint32_t page_count = multifd_ram_page_count();
212
213 wd = multifd_uadk_init_sess(page_count, page_size, false, errp);
214 if (!wd) {
215 return -1;
216 }
217 p->compress_data = wd;
218 return 0;
219 }
220
multifd_uadk_recv_cleanup(MultiFDRecvParams * p)221 static void multifd_uadk_recv_cleanup(MultiFDRecvParams *p)
222 {
223 struct wd_data *wd = p->compress_data;
224
225 multifd_uadk_uninit_sess(wd);
226 p->compress_data = NULL;
227 }
228
multifd_uadk_recv(MultiFDRecvParams * p,Error ** errp)229 static int multifd_uadk_recv(MultiFDRecvParams *p, Error **errp)
230 {
231 struct wd_data *uadk_data = p->compress_data;
232 uint32_t in_size = p->next_packet_size;
233 uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK;
234 uint32_t hdr_len = p->normal_num * sizeof(uint32_t);
235 uint32_t data_len = 0;
236 uint32_t page_size = multifd_ram_page_size();
237 uint8_t *buf = uadk_data->buf;
238 int ret = 0;
239
240 if (flags != MULTIFD_FLAG_UADK) {
241 error_setg(errp, "multifd %u: flags received %x flags expected %x",
242 p->id, flags, MULTIFD_FLAG_ZLIB);
243 return -1;
244 }
245
246 multifd_recv_zero_page_process(p);
247 if (!p->normal_num) {
248 assert(in_size == 0);
249 return 0;
250 }
251
252 /* read compressed data lengths */
253 assert(hdr_len < in_size);
254 ret = qio_channel_read_all(p->c, (void *) uadk_data->buf_hdr,
255 hdr_len, errp);
256 if (ret != 0) {
257 return ret;
258 }
259
260 for (int i = 0; i < p->normal_num; i++) {
261 uadk_data->buf_hdr[i] = be32_to_cpu(uadk_data->buf_hdr[i]);
262 data_len += uadk_data->buf_hdr[i];
263 assert(uadk_data->buf_hdr[i] <= page_size);
264 }
265
266 /* read compressed data */
267 assert(in_size == hdr_len + data_len);
268 ret = qio_channel_read_all(p->c, (void *)buf, data_len, errp);
269 if (ret != 0) {
270 return ret;
271 }
272
273 for (int i = 0; i < p->normal_num; i++) {
274 struct wd_comp_req creq = {
275 .op_type = WD_DIR_DECOMPRESS,
276 .src = buf,
277 .src_len = uadk_data->buf_hdr[i],
278 .dst = p->host + p->normal[i],
279 .dst_len = page_size,
280 };
281
282 if (uadk_data->buf_hdr[i] == page_size) {
283 memcpy(p->host + p->normal[i], buf, page_size);
284 buf += page_size;
285 continue;
286 }
287
288 if (unlikely(!uadk_data->handle)) {
289 error_setg(errp, "multifd %u: UADK HW not available for decompression",
290 p->id);
291 return -1;
292 }
293
294 ret = wd_do_comp_sync(uadk_data->handle, &creq);
295 if (ret || creq.status) {
296 error_setg(errp, "multifd %u: failed decompression, ret %d status %d",
297 p->id, ret, creq.status);
298 return -1;
299 }
300 if (creq.dst_len != page_size) {
301 error_setg(errp, "multifd %u: decompressed length error", p->id);
302 return -1;
303 }
304 buf += uadk_data->buf_hdr[i];
305 }
306
307 return 0;
308 }
309
310 static const MultiFDMethods multifd_uadk_ops = {
311 .send_setup = multifd_uadk_send_setup,
312 .send_cleanup = multifd_uadk_send_cleanup,
313 .send_prepare = multifd_uadk_send_prepare,
314 .recv_setup = multifd_uadk_recv_setup,
315 .recv_cleanup = multifd_uadk_recv_cleanup,
316 .recv = multifd_uadk_recv,
317 };
318
multifd_uadk_register(void)319 static void multifd_uadk_register(void)
320 {
321 multifd_register_ops(MULTIFD_COMPRESSION_UADK, &multifd_uadk_ops);
322 }
323 migration_init(multifd_uadk_register);
324