1 /* 2 * Multifd UADK compression accelerator implementation 3 * 4 * Copyright (c) 2024 Huawei Technologies R & D (UK) Ltd 5 * 6 * Authors: 7 * Shameer Kolothum <shameerali.kolothum.thodi@huawei.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qemu/module.h" 15 #include "qapi/error.h" 16 #include "exec/ramblock.h" 17 #include "migration.h" 18 #include "multifd.h" 19 #include "options.h" 20 #include "qemu/error-report.h" 21 #include "uadk/wd_comp.h" 22 #include "uadk/wd_sched.h" 23 24 struct wd_data { 25 handle_t handle; 26 uint8_t *buf; 27 uint32_t *buf_hdr; 28 }; 29 30 static bool uadk_hw_init(void) 31 { 32 char alg[] = "zlib"; 33 int ret; 34 35 ret = wd_comp_init2(alg, SCHED_POLICY_RR, TASK_HW); 36 if (ret && ret != -WD_EEXIST) { 37 return false; 38 } else { 39 return true; 40 } 41 } 42 43 static struct wd_data *multifd_uadk_init_sess(uint32_t count, 44 uint32_t page_size, 45 bool compress, Error **errp) 46 { 47 struct wd_comp_sess_setup ss = {0}; 48 struct sched_params param = {0}; 49 uint32_t size = count * page_size; 50 struct wd_data *wd; 51 52 wd = g_new0(struct wd_data, 1); 53 54 if (uadk_hw_init()) { 55 ss.alg_type = WD_ZLIB; 56 if (compress) { 57 ss.op_type = WD_DIR_COMPRESS; 58 /* Add an additional page for handling output > input */ 59 size += page_size; 60 } else { 61 ss.op_type = WD_DIR_DECOMPRESS; 62 } 63 /* We use default level 1 compression and 4K window size */ 64 param.type = ss.op_type; 65 ss.sched_param = ¶m; 66 67 wd->handle = wd_comp_alloc_sess(&ss); 68 if (!wd->handle) { 69 error_setg(errp, "multifd: failed wd_comp_alloc_sess"); 70 goto out; 71 } 72 } else { 73 /* For CI test use */ 74 warn_report_once("UADK hardware not available. Switch to no compression mode"); 75 } 76 77 wd->buf = g_try_malloc(size); 78 if (!wd->buf) { 79 error_setg(errp, "multifd: out of mem for uadk buf"); 80 goto out_free_sess; 81 } 82 wd->buf_hdr = g_new0(uint32_t, count); 83 return wd; 84 85 out_free_sess: 86 if (wd->handle) { 87 wd_comp_free_sess(wd->handle); 88 } 89 out: 90 wd_comp_uninit2(); 91 g_free(wd); 92 return NULL; 93 } 94 95 static void multifd_uadk_uninit_sess(struct wd_data *wd) 96 { 97 if (wd->handle) { 98 wd_comp_free_sess(wd->handle); 99 } 100 wd_comp_uninit2(); 101 g_free(wd->buf); 102 g_free(wd->buf_hdr); 103 g_free(wd); 104 } 105 106 static int multifd_uadk_send_setup(MultiFDSendParams *p, Error **errp) 107 { 108 struct wd_data *wd; 109 uint32_t page_size = multifd_ram_page_size(); 110 uint32_t page_count = multifd_ram_page_count(); 111 112 wd = multifd_uadk_init_sess(page_count, page_size, true, errp); 113 if (!wd) { 114 return -1; 115 } 116 117 p->compress_data = wd; 118 assert(p->iov == NULL); 119 /* 120 * Each page will be compressed independently and sent using an IOV. The 121 * additional two IOVs are used to store packet header and compressed data 122 * length 123 */ 124 125 p->iov = g_new0(struct iovec, page_count + 2); 126 return 0; 127 } 128 129 static void multifd_uadk_send_cleanup(MultiFDSendParams *p, Error **errp) 130 { 131 struct wd_data *wd = p->compress_data; 132 133 multifd_uadk_uninit_sess(wd); 134 p->compress_data = NULL; 135 g_free(p->iov); 136 p->iov = NULL; 137 } 138 139 static inline void prepare_next_iov(MultiFDSendParams *p, void *base, 140 uint32_t len) 141 { 142 p->iov[p->iovs_num].iov_base = (uint8_t *)base; 143 p->iov[p->iovs_num].iov_len = len; 144 p->next_packet_size += len; 145 p->iovs_num++; 146 } 147 148 static int multifd_uadk_send_prepare(MultiFDSendParams *p, Error **errp) 149 { 150 struct wd_data *uadk_data = p->compress_data; 151 uint32_t hdr_size; 152 uint32_t page_size = multifd_ram_page_size(); 153 uint8_t *buf = uadk_data->buf; 154 int ret = 0; 155 MultiFDPages_t *pages = &p->data->u.ram; 156 157 if (!multifd_send_prepare_common(p)) { 158 goto out; 159 } 160 161 hdr_size = pages->normal_num * sizeof(uint32_t); 162 /* prepare the header that stores the lengths of all compressed data */ 163 prepare_next_iov(p, uadk_data->buf_hdr, hdr_size); 164 165 for (int i = 0; i < pages->normal_num; i++) { 166 struct wd_comp_req creq = { 167 .op_type = WD_DIR_COMPRESS, 168 .src = pages->block->host + pages->offset[i], 169 .src_len = page_size, 170 .dst = buf, 171 /* Set dst_len to double the src in case compressed out >= page_size */ 172 .dst_len = p->page_size * 2, 173 }; 174 175 if (uadk_data->handle) { 176 ret = wd_do_comp_sync(uadk_data->handle, &creq); 177 if (ret || creq.status) { 178 error_setg(errp, "multifd %u: failed compression, ret %d status %d", 179 p->id, ret, creq.status); 180 return -1; 181 } 182 if (creq.dst_len < page_size) { 183 uadk_data->buf_hdr[i] = cpu_to_be32(creq.dst_len); 184 prepare_next_iov(p, buf, creq.dst_len); 185 buf += creq.dst_len; 186 } 187 } 188 /* 189 * Send raw data if no UADK hardware or if compressed out >= page_size. 190 * We might be better off sending raw data if output is slightly less 191 * than page_size as well because at the receive end we can skip the 192 * decompression. But it is tricky to find the right number here. 193 */ 194 if (!uadk_data->handle || creq.dst_len >= page_size) { 195 uadk_data->buf_hdr[i] = cpu_to_be32(page_size); 196 prepare_next_iov(p, pages->block->host + pages->offset[i], 197 page_size); 198 buf += page_size; 199 } 200 } 201 out: 202 p->flags |= MULTIFD_FLAG_UADK; 203 multifd_send_fill_packet(p); 204 return 0; 205 } 206 207 static int multifd_uadk_recv_setup(MultiFDRecvParams *p, Error **errp) 208 { 209 struct wd_data *wd; 210 uint32_t page_size = multifd_ram_page_size(); 211 uint32_t page_count = multifd_ram_page_count(); 212 213 wd = multifd_uadk_init_sess(page_count, page_size, false, errp); 214 if (!wd) { 215 return -1; 216 } 217 p->compress_data = wd; 218 return 0; 219 } 220 221 static void multifd_uadk_recv_cleanup(MultiFDRecvParams *p) 222 { 223 struct wd_data *wd = p->compress_data; 224 225 multifd_uadk_uninit_sess(wd); 226 p->compress_data = NULL; 227 } 228 229 static int multifd_uadk_recv(MultiFDRecvParams *p, Error **errp) 230 { 231 struct wd_data *uadk_data = p->compress_data; 232 uint32_t in_size = p->next_packet_size; 233 uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; 234 uint32_t hdr_len = p->normal_num * sizeof(uint32_t); 235 uint32_t data_len = 0; 236 uint32_t page_size = multifd_ram_page_size(); 237 uint8_t *buf = uadk_data->buf; 238 int ret = 0; 239 240 if (flags != MULTIFD_FLAG_UADK) { 241 error_setg(errp, "multifd %u: flags received %x flags expected %x", 242 p->id, flags, MULTIFD_FLAG_ZLIB); 243 return -1; 244 } 245 246 multifd_recv_zero_page_process(p); 247 if (!p->normal_num) { 248 assert(in_size == 0); 249 return 0; 250 } 251 252 /* read compressed data lengths */ 253 assert(hdr_len < in_size); 254 ret = qio_channel_read_all(p->c, (void *) uadk_data->buf_hdr, 255 hdr_len, errp); 256 if (ret != 0) { 257 return ret; 258 } 259 260 for (int i = 0; i < p->normal_num; i++) { 261 uadk_data->buf_hdr[i] = be32_to_cpu(uadk_data->buf_hdr[i]); 262 data_len += uadk_data->buf_hdr[i]; 263 assert(uadk_data->buf_hdr[i] <= page_size); 264 } 265 266 /* read compressed data */ 267 assert(in_size == hdr_len + data_len); 268 ret = qio_channel_read_all(p->c, (void *)buf, data_len, errp); 269 if (ret != 0) { 270 return ret; 271 } 272 273 for (int i = 0; i < p->normal_num; i++) { 274 struct wd_comp_req creq = { 275 .op_type = WD_DIR_DECOMPRESS, 276 .src = buf, 277 .src_len = uadk_data->buf_hdr[i], 278 .dst = p->host + p->normal[i], 279 .dst_len = page_size, 280 }; 281 282 if (uadk_data->buf_hdr[i] == page_size) { 283 memcpy(p->host + p->normal[i], buf, page_size); 284 buf += page_size; 285 continue; 286 } 287 288 if (unlikely(!uadk_data->handle)) { 289 error_setg(errp, "multifd %u: UADK HW not available for decompression", 290 p->id); 291 return -1; 292 } 293 294 ret = wd_do_comp_sync(uadk_data->handle, &creq); 295 if (ret || creq.status) { 296 error_setg(errp, "multifd %u: failed decompression, ret %d status %d", 297 p->id, ret, creq.status); 298 return -1; 299 } 300 if (creq.dst_len != page_size) { 301 error_setg(errp, "multifd %u: decompressed length error", p->id); 302 return -1; 303 } 304 buf += uadk_data->buf_hdr[i]; 305 } 306 307 return 0; 308 } 309 310 static const MultiFDMethods multifd_uadk_ops = { 311 .send_setup = multifd_uadk_send_setup, 312 .send_cleanup = multifd_uadk_send_cleanup, 313 .send_prepare = multifd_uadk_send_prepare, 314 .recv_setup = multifd_uadk_recv_setup, 315 .recv_cleanup = multifd_uadk_recv_cleanup, 316 .recv = multifd_uadk_recv, 317 }; 318 319 static void multifd_uadk_register(void) 320 { 321 multifd_register_ops(MULTIFD_COMPRESSION_UADK, &multifd_uadk_ops); 322 } 323 migration_init(multifd_uadk_register); 324