xref: /openbmc/qemu/migration/multifd-uadk.c (revision 154fd4d1) 
1 /*
2  * Multifd UADK compression accelerator implementation
3  *
4  * Copyright (c) 2024 Huawei Technologies R & D (UK) Ltd
5  *
6  * Authors:
7  *  Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2 or later.
10  * See the COPYING file in the top-level directory.
11  */
12 
13 #include "qemu/osdep.h"
14 #include "qemu/module.h"
15 #include "qapi/error.h"
16 #include "exec/ramblock.h"
17 #include "migration.h"
18 #include "multifd.h"
19 #include "options.h"
20 #include "qemu/error-report.h"
21 #include "uadk/wd_comp.h"
22 #include "uadk/wd_sched.h"
23 
24 struct wd_data {
25     handle_t handle;
26     uint8_t *buf;
27     uint32_t *buf_hdr;
28 };
29 
30 static bool uadk_hw_init(void)
31 {
32     char alg[] = "zlib";
33     int ret;
34 
35     ret = wd_comp_init2(alg, SCHED_POLICY_RR, TASK_HW);
36     if (ret && ret != -WD_EEXIST) {
37         return false;
38     } else {
39         return true;
40     }
41 }
42 
43 static struct wd_data *multifd_uadk_init_sess(uint32_t count,
44                                               uint32_t page_size,
45                                               bool compress, Error **errp)
46 {
47     struct wd_comp_sess_setup ss = {0};
48     struct sched_params param = {0};
49     uint32_t size = count * page_size;
50     struct wd_data *wd;
51 
52     wd = g_new0(struct wd_data, 1);
53 
54     if (uadk_hw_init()) {
55         ss.alg_type = WD_ZLIB;
56         if (compress) {
57             ss.op_type = WD_DIR_COMPRESS;
58             /* Add an additional page for handling output > input */
59             size += page_size;
60         } else {
61             ss.op_type = WD_DIR_DECOMPRESS;
62         }
63         /* We use default level 1 compression and 4K window size */
64         param.type = ss.op_type;
65         ss.sched_param = &param;
66 
67         wd->handle = wd_comp_alloc_sess(&ss);
68         if (!wd->handle) {
69             error_setg(errp, "multifd: failed wd_comp_alloc_sess");
70             goto out;
71         }
72     } else {
73         /* For CI test use */
74         warn_report_once("UADK hardware not available. Switch to no compression mode");
75     }
76 
77     wd->buf = g_try_malloc(size);
78     if (!wd->buf) {
79         error_setg(errp, "multifd: out of mem for uadk buf");
80         goto out_free_sess;
81     }
82     wd->buf_hdr = g_new0(uint32_t, count);
83     return wd;
84 
85 out_free_sess:
86     if (wd->handle) {
87         wd_comp_free_sess(wd->handle);
88     }
89 out:
90     wd_comp_uninit2();
91     g_free(wd);
92     return NULL;
93 }
94 
95 static void multifd_uadk_uninit_sess(struct wd_data *wd)
96 {
97     if (wd->handle) {
98         wd_comp_free_sess(wd->handle);
99     }
100     wd_comp_uninit2();
101     g_free(wd->buf);
102     g_free(wd->buf_hdr);
103     g_free(wd);
104 }
105 
106 static int multifd_uadk_send_setup(MultiFDSendParams *p, Error **errp)
107 {
108     struct wd_data *wd;
109     uint32_t page_size = multifd_ram_page_size();
110     uint32_t page_count = multifd_ram_page_count();
111 
112     wd = multifd_uadk_init_sess(page_count, page_size, true, errp);
113     if (!wd) {
114         return -1;
115     }
116 
117     p->compress_data = wd;
118     assert(p->iov == NULL);
119     /*
120      * Each page will be compressed independently and sent using an IOV. The
121      * additional two IOVs are used to store packet header and compressed data
122      * length
123      */
124 
125     p->iov = g_new0(struct iovec, page_count + 2);
126     return 0;
127 }
128 
129 static void multifd_uadk_send_cleanup(MultiFDSendParams *p, Error **errp)
130 {
131     struct wd_data *wd = p->compress_data;
132 
133     multifd_uadk_uninit_sess(wd);
134     p->compress_data = NULL;
135     g_free(p->iov);
136     p->iov = NULL;
137 }
138 
139 static inline void prepare_next_iov(MultiFDSendParams *p, void *base,
140                                     uint32_t len)
141 {
142     p->iov[p->iovs_num].iov_base = (uint8_t *)base;
143     p->iov[p->iovs_num].iov_len = len;
144     p->next_packet_size += len;
145     p->iovs_num++;
146 }
147 
148 static int multifd_uadk_send_prepare(MultiFDSendParams *p, Error **errp)
149 {
150     struct wd_data *uadk_data = p->compress_data;
151     uint32_t hdr_size;
152     uint32_t page_size = multifd_ram_page_size();
153     uint8_t *buf = uadk_data->buf;
154     int ret = 0;
155     MultiFDPages_t *pages = &p->data->u.ram;
156 
157     if (!multifd_send_prepare_common(p)) {
158         goto out;
159     }
160 
161     hdr_size = pages->normal_num * sizeof(uint32_t);
162     /* prepare the header that stores the lengths of all compressed data */
163     prepare_next_iov(p, uadk_data->buf_hdr, hdr_size);
164 
165     for (int i = 0; i < pages->normal_num; i++) {
166         struct wd_comp_req creq = {
167             .op_type = WD_DIR_COMPRESS,
168             .src     = pages->block->host + pages->offset[i],
169             .src_len = page_size,
170             .dst     = buf,
171             /* Set dst_len to double the src in case compressed out >= page_size */
172             .dst_len = p->page_size * 2,
173         };
174 
175         if (uadk_data->handle) {
176             ret = wd_do_comp_sync(uadk_data->handle, &creq);
177             if (ret || creq.status) {
178                 error_setg(errp, "multifd %u: failed compression, ret %d status %d",
179                            p->id, ret, creq.status);
180                 return -1;
181             }
182             if (creq.dst_len < page_size) {
183                 uadk_data->buf_hdr[i] = cpu_to_be32(creq.dst_len);
184                 prepare_next_iov(p, buf, creq.dst_len);
185                 buf += creq.dst_len;
186             }
187         }
188         /*
189          * Send raw data if no UADK hardware or if compressed out >= page_size.
190          * We might be better off sending raw data if output is slightly less
191          * than page_size as well because at the receive end we can skip the
192          * decompression. But it is tricky to find the right number here.
193          */
194         if (!uadk_data->handle || creq.dst_len >= page_size) {
195             uadk_data->buf_hdr[i] = cpu_to_be32(page_size);
196             prepare_next_iov(p, pages->block->host + pages->offset[i],
197                              page_size);
198             buf += page_size;
199         }
200     }
201 out:
202     p->flags |= MULTIFD_FLAG_UADK;
203     multifd_send_fill_packet(p);
204     return 0;
205 }
206 
207 static int multifd_uadk_recv_setup(MultiFDRecvParams *p, Error **errp)
208 {
209     struct wd_data *wd;
210     uint32_t page_size = multifd_ram_page_size();
211     uint32_t page_count = multifd_ram_page_count();
212 
213     wd = multifd_uadk_init_sess(page_count, page_size, false, errp);
214     if (!wd) {
215         return -1;
216     }
217     p->compress_data = wd;
218     return 0;
219 }
220 
221 static void multifd_uadk_recv_cleanup(MultiFDRecvParams *p)
222 {
223     struct wd_data *wd = p->compress_data;
224 
225     multifd_uadk_uninit_sess(wd);
226     p->compress_data = NULL;
227 }
228 
229 static int multifd_uadk_recv(MultiFDRecvParams *p, Error **errp)
230 {
231     struct wd_data *uadk_data = p->compress_data;
232     uint32_t in_size = p->next_packet_size;
233     uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK;
234     uint32_t hdr_len = p->normal_num * sizeof(uint32_t);
235     uint32_t data_len = 0;
236     uint32_t page_size = multifd_ram_page_size();
237     uint8_t *buf = uadk_data->buf;
238     int ret = 0;
239 
240     if (flags != MULTIFD_FLAG_UADK) {
241         error_setg(errp, "multifd %u: flags received %x flags expected %x",
242                    p->id, flags, MULTIFD_FLAG_ZLIB);
243         return -1;
244     }
245 
246     multifd_recv_zero_page_process(p);
247     if (!p->normal_num) {
248         assert(in_size == 0);
249         return 0;
250     }
251 
252     /* read compressed data lengths */
253     assert(hdr_len < in_size);
254     ret = qio_channel_read_all(p->c, (void *) uadk_data->buf_hdr,
255                                hdr_len, errp);
256     if (ret != 0) {
257         return ret;
258     }
259 
260     for (int i = 0; i < p->normal_num; i++) {
261         uadk_data->buf_hdr[i] = be32_to_cpu(uadk_data->buf_hdr[i]);
262         data_len += uadk_data->buf_hdr[i];
263         assert(uadk_data->buf_hdr[i] <= page_size);
264     }
265 
266     /* read compressed data */
267     assert(in_size == hdr_len + data_len);
268     ret = qio_channel_read_all(p->c, (void *)buf, data_len, errp);
269     if (ret != 0) {
270         return ret;
271     }
272 
273     for (int i = 0; i < p->normal_num; i++) {
274         struct wd_comp_req creq = {
275             .op_type = WD_DIR_DECOMPRESS,
276             .src     = buf,
277             .src_len = uadk_data->buf_hdr[i],
278             .dst     = p->host + p->normal[i],
279             .dst_len = page_size,
280         };
281 
282         if (uadk_data->buf_hdr[i] == page_size) {
283             memcpy(p->host + p->normal[i], buf, page_size);
284             buf += page_size;
285             continue;
286         }
287 
288         if (unlikely(!uadk_data->handle)) {
289             error_setg(errp, "multifd %u: UADK HW not available for decompression",
290                        p->id);
291             return -1;
292         }
293 
294         ret = wd_do_comp_sync(uadk_data->handle, &creq);
295         if (ret || creq.status) {
296             error_setg(errp, "multifd %u: failed decompression, ret %d status %d",
297                        p->id, ret, creq.status);
298             return -1;
299         }
300         if (creq.dst_len != page_size) {
301             error_setg(errp, "multifd %u: decompressed length error", p->id);
302             return -1;
303         }
304         buf += uadk_data->buf_hdr[i];
305      }
306 
307     return 0;
308 }
309 
310 static const MultiFDMethods multifd_uadk_ops = {
311     .send_setup = multifd_uadk_send_setup,
312     .send_cleanup = multifd_uadk_send_cleanup,
313     .send_prepare = multifd_uadk_send_prepare,
314     .recv_setup = multifd_uadk_recv_setup,
315     .recv_cleanup = multifd_uadk_recv_cleanup,
316     .recv = multifd_uadk_recv,
317 };
318 
319 static void multifd_uadk_register(void)
320 {
321     multifd_register_ops(MULTIFD_COMPRESSION_UADK, &multifd_uadk_ops);
322 }
323 migration_init(multifd_uadk_register);
324