1 /* 2 * Multifd RAM migration without compression 3 * 4 * Copyright (c) 2019-2020 Red Hat Inc 5 * 6 * Authors: 7 * Juan Quintela <quintela@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "exec/ramblock.h" 15 #include "exec/target_page.h" 16 #include "file.h" 17 #include "multifd.h" 18 #include "options.h" 19 #include "qapi/error.h" 20 #include "qemu/error-report.h" 21 #include "trace.h" 22 23 static MultiFDSendData *multifd_ram_send; 24 25 size_t multifd_ram_payload_size(void) 26 { 27 uint32_t n = multifd_ram_page_count(); 28 29 /* 30 * We keep an array of page offsets at the end of MultiFDPages_t, 31 * add space for it in the allocation. 32 */ 33 return sizeof(MultiFDPages_t) + n * sizeof(ram_addr_t); 34 } 35 36 void multifd_ram_save_setup(void) 37 { 38 multifd_ram_send = multifd_send_data_alloc(); 39 } 40 41 void multifd_ram_save_cleanup(void) 42 { 43 g_free(multifd_ram_send); 44 multifd_ram_send = NULL; 45 } 46 47 static void multifd_set_file_bitmap(MultiFDSendParams *p) 48 { 49 MultiFDPages_t *pages = &p->data->u.ram; 50 51 assert(pages->block); 52 53 for (int i = 0; i < pages->normal_num; i++) { 54 ramblock_set_file_bmap_atomic(pages->block, pages->offset[i], true); 55 } 56 57 for (int i = pages->normal_num; i < pages->num; i++) { 58 ramblock_set_file_bmap_atomic(pages->block, pages->offset[i], false); 59 } 60 } 61 62 static int multifd_nocomp_send_setup(MultiFDSendParams *p, Error **errp) 63 { 64 uint32_t page_count = multifd_ram_page_count(); 65 66 if (migrate_zero_copy_send()) { 67 p->write_flags |= QIO_CHANNEL_WRITE_FLAG_ZERO_COPY; 68 } 69 70 if (!migrate_mapped_ram()) { 71 /* We need one extra place for the packet header */ 72 p->iov = g_new0(struct iovec, page_count + 1); 73 } else { 74 p->iov = g_new0(struct iovec, page_count); 75 } 76 77 return 0; 78 } 79 80 static void multifd_nocomp_send_cleanup(MultiFDSendParams *p, Error **errp) 81 { 82 g_free(p->iov); 83 p->iov = NULL; 84 return; 85 } 86 87 static void multifd_send_prepare_iovs(MultiFDSendParams *p) 88 { 89 MultiFDPages_t *pages = &p->data->u.ram; 90 uint32_t page_size = multifd_ram_page_size(); 91 92 for (int i = 0; i < pages->normal_num; i++) { 93 p->iov[p->iovs_num].iov_base = pages->block->host + pages->offset[i]; 94 p->iov[p->iovs_num].iov_len = page_size; 95 p->iovs_num++; 96 } 97 98 p->next_packet_size = pages->normal_num * page_size; 99 } 100 101 static int multifd_nocomp_send_prepare(MultiFDSendParams *p, Error **errp) 102 { 103 bool use_zero_copy_send = migrate_zero_copy_send(); 104 int ret; 105 106 multifd_send_zero_page_detect(p); 107 108 if (migrate_mapped_ram()) { 109 multifd_send_prepare_iovs(p); 110 multifd_set_file_bitmap(p); 111 112 return 0; 113 } 114 115 if (!use_zero_copy_send) { 116 /* 117 * Only !zerocopy needs the header in IOV; zerocopy will 118 * send it separately. 119 */ 120 multifd_send_prepare_header(p); 121 } 122 123 multifd_send_prepare_iovs(p); 124 p->flags |= MULTIFD_FLAG_NOCOMP; 125 126 multifd_send_fill_packet(p); 127 128 if (use_zero_copy_send) { 129 /* Send header first, without zerocopy */ 130 ret = qio_channel_write_all(p->c, (void *)p->packet, 131 p->packet_len, errp); 132 if (ret != 0) { 133 return -1; 134 } 135 } 136 137 return 0; 138 } 139 140 static int multifd_nocomp_recv_setup(MultiFDRecvParams *p, Error **errp) 141 { 142 p->iov = g_new0(struct iovec, multifd_ram_page_count()); 143 return 0; 144 } 145 146 static void multifd_nocomp_recv_cleanup(MultiFDRecvParams *p) 147 { 148 g_free(p->iov); 149 p->iov = NULL; 150 } 151 152 static int multifd_nocomp_recv(MultiFDRecvParams *p, Error **errp) 153 { 154 uint32_t flags; 155 156 if (migrate_mapped_ram()) { 157 return multifd_file_recv_data(p, errp); 158 } 159 160 flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; 161 162 if (flags != MULTIFD_FLAG_NOCOMP) { 163 error_setg(errp, "multifd %u: flags received %x flags expected %x", 164 p->id, flags, MULTIFD_FLAG_NOCOMP); 165 return -1; 166 } 167 168 multifd_recv_zero_page_process(p); 169 170 if (!p->normal_num) { 171 return 0; 172 } 173 174 for (int i = 0; i < p->normal_num; i++) { 175 p->iov[i].iov_base = p->host + p->normal[i]; 176 p->iov[i].iov_len = multifd_ram_page_size(); 177 ramblock_recv_bitmap_set_offset(p->block, p->normal[i]); 178 } 179 return qio_channel_readv_all(p->c, p->iov, p->normal_num, errp); 180 } 181 182 static void multifd_pages_reset(MultiFDPages_t *pages) 183 { 184 /* 185 * We don't need to touch offset[] array, because it will be 186 * overwritten later when reused. 187 */ 188 pages->num = 0; 189 pages->normal_num = 0; 190 pages->block = NULL; 191 } 192 193 void multifd_ram_fill_packet(MultiFDSendParams *p) 194 { 195 MultiFDPacket_t *packet = p->packet; 196 MultiFDPages_t *pages = &p->data->u.ram; 197 uint32_t zero_num = pages->num - pages->normal_num; 198 199 packet->pages_alloc = cpu_to_be32(multifd_ram_page_count()); 200 packet->normal_pages = cpu_to_be32(pages->normal_num); 201 packet->zero_pages = cpu_to_be32(zero_num); 202 203 if (pages->block) { 204 strncpy(packet->ramblock, pages->block->idstr, 256); 205 } 206 207 for (int i = 0; i < pages->num; i++) { 208 /* there are architectures where ram_addr_t is 32 bit */ 209 uint64_t temp = pages->offset[i]; 210 211 packet->offset[i] = cpu_to_be64(temp); 212 } 213 214 trace_multifd_send_ram_fill(p->id, pages->normal_num, 215 zero_num); 216 } 217 218 int multifd_ram_unfill_packet(MultiFDRecvParams *p, Error **errp) 219 { 220 MultiFDPacket_t *packet = p->packet; 221 uint32_t page_count = multifd_ram_page_count(); 222 uint32_t page_size = multifd_ram_page_size(); 223 uint32_t pages_per_packet = be32_to_cpu(packet->pages_alloc); 224 int i; 225 226 if (pages_per_packet > page_count) { 227 error_setg(errp, "multifd: received packet with %u pages, expected %u", 228 pages_per_packet, page_count); 229 return -1; 230 } 231 232 p->normal_num = be32_to_cpu(packet->normal_pages); 233 if (p->normal_num > pages_per_packet) { 234 error_setg(errp, "multifd: received packet with %u non-zero pages, " 235 "which exceeds maximum expected pages %u", 236 p->normal_num, pages_per_packet); 237 return -1; 238 } 239 240 p->zero_num = be32_to_cpu(packet->zero_pages); 241 if (p->zero_num > pages_per_packet - p->normal_num) { 242 error_setg(errp, 243 "multifd: received packet with %u zero pages, expected maximum %u", 244 p->zero_num, pages_per_packet - p->normal_num); 245 return -1; 246 } 247 248 if (p->normal_num == 0 && p->zero_num == 0) { 249 return 0; 250 } 251 252 /* make sure that ramblock is 0 terminated */ 253 packet->ramblock[255] = 0; 254 p->block = qemu_ram_block_by_name(packet->ramblock); 255 if (!p->block) { 256 error_setg(errp, "multifd: unknown ram block %s", 257 packet->ramblock); 258 return -1; 259 } 260 261 p->host = p->block->host; 262 for (i = 0; i < p->normal_num; i++) { 263 uint64_t offset = be64_to_cpu(packet->offset[i]); 264 265 if (offset > (p->block->used_length - page_size)) { 266 error_setg(errp, "multifd: offset too long %" PRIu64 267 " (max " RAM_ADDR_FMT ")", 268 offset, p->block->used_length); 269 return -1; 270 } 271 p->normal[i] = offset; 272 } 273 274 for (i = 0; i < p->zero_num; i++) { 275 uint64_t offset = be64_to_cpu(packet->offset[p->normal_num + i]); 276 277 if (offset > (p->block->used_length - page_size)) { 278 error_setg(errp, "multifd: offset too long %" PRIu64 279 " (max " RAM_ADDR_FMT ")", 280 offset, p->block->used_length); 281 return -1; 282 } 283 p->zero[i] = offset; 284 } 285 286 return 0; 287 } 288 289 static inline bool multifd_queue_empty(MultiFDPages_t *pages) 290 { 291 return pages->num == 0; 292 } 293 294 static inline bool multifd_queue_full(MultiFDPages_t *pages) 295 { 296 return pages->num == multifd_ram_page_count(); 297 } 298 299 static inline void multifd_enqueue(MultiFDPages_t *pages, ram_addr_t offset) 300 { 301 pages->offset[pages->num++] = offset; 302 } 303 304 /* Returns true if enqueue successful, false otherwise */ 305 bool multifd_queue_page(RAMBlock *block, ram_addr_t offset) 306 { 307 MultiFDPages_t *pages; 308 309 retry: 310 pages = &multifd_ram_send->u.ram; 311 312 if (multifd_payload_empty(multifd_ram_send)) { 313 multifd_pages_reset(pages); 314 multifd_set_payload_type(multifd_ram_send, MULTIFD_PAYLOAD_RAM); 315 } 316 317 /* If the queue is empty, we can already enqueue now */ 318 if (multifd_queue_empty(pages)) { 319 pages->block = block; 320 multifd_enqueue(pages, offset); 321 return true; 322 } 323 324 /* 325 * Not empty, meanwhile we need a flush. It can because of either: 326 * 327 * (1) The page is not on the same ramblock of previous ones, or, 328 * (2) The queue is full. 329 * 330 * After flush, always retry. 331 */ 332 if (pages->block != block || multifd_queue_full(pages)) { 333 if (!multifd_send(&multifd_ram_send)) { 334 return false; 335 } 336 goto retry; 337 } 338 339 /* Not empty, and we still have space, do it! */ 340 multifd_enqueue(pages, offset); 341 return true; 342 } 343 344 int multifd_ram_flush_and_sync(void) 345 { 346 if (!migrate_multifd()) { 347 return 0; 348 } 349 350 if (!multifd_payload_empty(multifd_ram_send)) { 351 if (!multifd_send(&multifd_ram_send)) { 352 error_report("%s: multifd_send fail", __func__); 353 return -1; 354 } 355 } 356 357 return multifd_send_sync_main(); 358 } 359 360 bool multifd_send_prepare_common(MultiFDSendParams *p) 361 { 362 MultiFDPages_t *pages = &p->data->u.ram; 363 multifd_send_zero_page_detect(p); 364 365 if (!pages->normal_num) { 366 p->next_packet_size = 0; 367 return false; 368 } 369 370 multifd_send_prepare_header(p); 371 372 return true; 373 } 374 375 static const MultiFDMethods multifd_nocomp_ops = { 376 .send_setup = multifd_nocomp_send_setup, 377 .send_cleanup = multifd_nocomp_send_cleanup, 378 .send_prepare = multifd_nocomp_send_prepare, 379 .recv_setup = multifd_nocomp_recv_setup, 380 .recv_cleanup = multifd_nocomp_recv_cleanup, 381 .recv = multifd_nocomp_recv 382 }; 383 384 static void multifd_nocomp_register(void) 385 { 386 multifd_register_ops(MULTIFD_COMPRESSION_NONE, &multifd_nocomp_ops); 387 } 388 389 migration_init(multifd_nocomp_register); 390