Lines Matching refs:p
111 static void multifd_set_file_bitmap(MultiFDSendParams *p) in multifd_set_file_bitmap() argument
113 MultiFDPages_t *pages = p->pages; in multifd_set_file_bitmap()
117 for (int i = 0; i < p->pages->normal_num; i++) { in multifd_set_file_bitmap()
121 for (int i = p->pages->normal_num; i < p->pages->num; i++) { in multifd_set_file_bitmap()
134 static int nocomp_send_setup(MultiFDSendParams *p, Error **errp) in nocomp_send_setup() argument
137 p->write_flags |= QIO_CHANNEL_WRITE_FLAG_ZERO_COPY; in nocomp_send_setup()
142 p->iov = g_new0(struct iovec, p->page_count + 1); in nocomp_send_setup()
144 p->iov = g_new0(struct iovec, p->page_count); in nocomp_send_setup()
158 static void nocomp_send_cleanup(MultiFDSendParams *p, Error **errp) in nocomp_send_cleanup() argument
160 g_free(p->iov); in nocomp_send_cleanup()
161 p->iov = NULL; in nocomp_send_cleanup()
165 static void multifd_send_prepare_iovs(MultiFDSendParams *p) in multifd_send_prepare_iovs() argument
167 MultiFDPages_t *pages = p->pages; in multifd_send_prepare_iovs()
170 p->iov[p->iovs_num].iov_base = pages->block->host + pages->offset[i]; in multifd_send_prepare_iovs()
171 p->iov[p->iovs_num].iov_len = p->page_size; in multifd_send_prepare_iovs()
172 p->iovs_num++; in multifd_send_prepare_iovs()
175 p->next_packet_size = pages->normal_num * p->page_size; in multifd_send_prepare_iovs()
189 static int nocomp_send_prepare(MultiFDSendParams *p, Error **errp) in nocomp_send_prepare() argument
194 multifd_send_zero_page_detect(p); in nocomp_send_prepare()
197 multifd_send_prepare_iovs(p); in nocomp_send_prepare()
198 multifd_set_file_bitmap(p); in nocomp_send_prepare()
208 multifd_send_prepare_header(p); in nocomp_send_prepare()
211 multifd_send_prepare_iovs(p); in nocomp_send_prepare()
212 p->flags |= MULTIFD_FLAG_NOCOMP; in nocomp_send_prepare()
214 multifd_send_fill_packet(p); in nocomp_send_prepare()
218 ret = qio_channel_write_all(p->c, (void *)p->packet, in nocomp_send_prepare()
219 p->packet_len, errp); in nocomp_send_prepare()
238 static int nocomp_recv_setup(MultiFDRecvParams *p, Error **errp) in nocomp_recv_setup() argument
240 p->iov = g_new0(struct iovec, p->page_count); in nocomp_recv_setup()
251 static void nocomp_recv_cleanup(MultiFDRecvParams *p) in nocomp_recv_cleanup() argument
253 g_free(p->iov); in nocomp_recv_cleanup()
254 p->iov = NULL; in nocomp_recv_cleanup()
267 static int nocomp_recv(MultiFDRecvParams *p, Error **errp) in nocomp_recv() argument
272 return multifd_file_recv_data(p, errp); in nocomp_recv()
275 flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; in nocomp_recv()
279 p->id, flags, MULTIFD_FLAG_NOCOMP); in nocomp_recv()
283 multifd_recv_zero_page_process(p); in nocomp_recv()
285 if (!p->normal_num) { in nocomp_recv()
289 for (int i = 0; i < p->normal_num; i++) { in nocomp_recv()
290 p->iov[i].iov_base = p->host + p->normal[i]; in nocomp_recv()
291 p->iov[i].iov_len = p->page_size; in nocomp_recv()
292 ramblock_recv_bitmap_set_offset(p->block, p->normal[i]); in nocomp_recv()
294 return qio_channel_readv_all(p->c, p->iov, p->normal_num, errp); in nocomp_recv()
328 static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp) in multifd_send_initial_packet() argument
336 msg.id = p->id; in multifd_send_initial_packet()
339 ret = qio_channel_write_all(p->c, (char *)&msg, size, errp); in multifd_send_initial_packet()
411 void multifd_send_fill_packet(MultiFDSendParams *p) in multifd_send_fill_packet() argument
413 MultiFDPacket_t *packet = p->packet; in multifd_send_fill_packet()
414 MultiFDPages_t *pages = p->pages; in multifd_send_fill_packet()
419 packet->flags = cpu_to_be32(p->flags); in multifd_send_fill_packet()
420 packet->pages_alloc = cpu_to_be32(p->pages->allocated); in multifd_send_fill_packet()
423 packet->next_packet_size = cpu_to_be32(p->next_packet_size); in multifd_send_fill_packet()
439 p->packets_sent++; in multifd_send_fill_packet()
440 p->total_normal_pages += pages->normal_num; in multifd_send_fill_packet()
441 p->total_zero_pages += zero_num; in multifd_send_fill_packet()
443 trace_multifd_send(p->id, packet_num, pages->normal_num, zero_num, in multifd_send_fill_packet()
444 p->flags, p->next_packet_size); in multifd_send_fill_packet()
447 static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) in multifd_recv_unfill_packet() argument
449 MultiFDPacket_t *packet = p->packet; in multifd_recv_unfill_packet()
468 p->flags = be32_to_cpu(packet->flags); in multifd_recv_unfill_packet()
475 if (packet->pages_alloc > p->page_count) { in multifd_recv_unfill_packet()
478 packet->pages_alloc, p->page_count) ; in multifd_recv_unfill_packet()
482 p->normal_num = be32_to_cpu(packet->normal_pages); in multifd_recv_unfill_packet()
483 if (p->normal_num > packet->pages_alloc) { in multifd_recv_unfill_packet()
486 p->normal_num, packet->pages_alloc) ; in multifd_recv_unfill_packet()
490 p->zero_num = be32_to_cpu(packet->zero_pages); in multifd_recv_unfill_packet()
491 if (p->zero_num > packet->pages_alloc - p->normal_num) { in multifd_recv_unfill_packet()
494 p->zero_num, packet->pages_alloc - p->normal_num) ; in multifd_recv_unfill_packet()
498 p->next_packet_size = be32_to_cpu(packet->next_packet_size); in multifd_recv_unfill_packet()
499 p->packet_num = be64_to_cpu(packet->packet_num); in multifd_recv_unfill_packet()
500 p->packets_recved++; in multifd_recv_unfill_packet()
501 p->total_normal_pages += p->normal_num; in multifd_recv_unfill_packet()
502 p->total_zero_pages += p->zero_num; in multifd_recv_unfill_packet()
504 trace_multifd_recv(p->id, p->packet_num, p->normal_num, p->zero_num, in multifd_recv_unfill_packet()
505 p->flags, p->next_packet_size); in multifd_recv_unfill_packet()
507 if (p->normal_num == 0 && p->zero_num == 0) { in multifd_recv_unfill_packet()
513 p->block = qemu_ram_block_by_name(packet->ramblock); in multifd_recv_unfill_packet()
514 if (!p->block) { in multifd_recv_unfill_packet()
520 p->host = p->block->host; in multifd_recv_unfill_packet()
521 for (i = 0; i < p->normal_num; i++) { in multifd_recv_unfill_packet()
524 if (offset > (p->block->used_length - p->page_size)) { in multifd_recv_unfill_packet()
527 offset, p->block->used_length); in multifd_recv_unfill_packet()
530 p->normal[i] = offset; in multifd_recv_unfill_packet()
533 for (i = 0; i < p->zero_num; i++) { in multifd_recv_unfill_packet()
534 uint64_t offset = be64_to_cpu(packet->offset[p->normal_num + i]); in multifd_recv_unfill_packet()
536 if (offset > (p->block->used_length - p->page_size)) { in multifd_recv_unfill_packet()
539 offset, p->block->used_length); in multifd_recv_unfill_packet()
542 p->zero[i] = offset; in multifd_recv_unfill_packet()
564 static void multifd_send_kick_main(MultiFDSendParams *p) in multifd_send_kick_main() argument
566 qemu_sem_post(&p->sem_sync); in multifd_send_kick_main()
593 MultiFDSendParams *p = NULL; /* make happy gcc */ in multifd_send_pages() local
613 p = &multifd_send_state->params[i]; in multifd_send_pages()
618 if (qatomic_read(&p->pending_job) == false) { in multifd_send_pages()
629 assert(!p->pages->num); in multifd_send_pages()
630 multifd_send_state->pages = p->pages; in multifd_send_pages()
631 p->pages = pages; in multifd_send_pages()
636 qatomic_store_release(&p->pending_job, true); in multifd_send_pages()
637 qemu_sem_post(&p->sem); in multifd_send_pages()
735 MultiFDSendParams *p = &multifd_send_state->params[i]; in multifd_send_terminate_threads() local
737 qemu_sem_post(&p->sem); in multifd_send_terminate_threads()
738 if (p->c) { in multifd_send_terminate_threads()
739 qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); in multifd_send_terminate_threads()
747 MultiFDSendParams *p = &multifd_send_state->params[i]; in multifd_send_terminate_threads() local
749 if (p->tls_thread_created) { in multifd_send_terminate_threads()
750 qemu_thread_join(&p->tls_thread); in multifd_send_terminate_threads()
753 if (p->thread_created) { in multifd_send_terminate_threads()
754 qemu_thread_join(&p->thread); in multifd_send_terminate_threads()
759 static bool multifd_send_cleanup_channel(MultiFDSendParams *p, Error **errp) in multifd_send_cleanup_channel() argument
761 if (p->c) { in multifd_send_cleanup_channel()
762 migration_ioc_unregister_yank(p->c); in multifd_send_cleanup_channel()
785 qio_channel_close(p->c, &error_abort); in multifd_send_cleanup_channel()
786 object_unref(OBJECT(p->c)); in multifd_send_cleanup_channel()
787 p->c = NULL; in multifd_send_cleanup_channel()
789 qemu_sem_destroy(&p->sem); in multifd_send_cleanup_channel()
790 qemu_sem_destroy(&p->sem_sync); in multifd_send_cleanup_channel()
791 g_free(p->name); in multifd_send_cleanup_channel()
792 p->name = NULL; in multifd_send_cleanup_channel()
793 multifd_pages_clear(p->pages); in multifd_send_cleanup_channel()
794 p->pages = NULL; in multifd_send_cleanup_channel()
795 p->packet_len = 0; in multifd_send_cleanup_channel()
796 g_free(p->packet); in multifd_send_cleanup_channel()
797 p->packet = NULL; in multifd_send_cleanup_channel()
798 multifd_send_state->ops->send_cleanup(p, errp); in multifd_send_cleanup_channel()
828 MultiFDSendParams *p = &multifd_send_state->params[i]; in multifd_send_shutdown() local
831 if (!multifd_send_cleanup_channel(p, &local_err)) { in multifd_send_shutdown()
875 MultiFDSendParams *p = &multifd_send_state->params[i]; in multifd_send_sync_main() local
881 trace_multifd_send_sync_main_signal(p->id); in multifd_send_sync_main()
887 assert(qatomic_read(&p->pending_sync) == false); in multifd_send_sync_main()
888 qatomic_set(&p->pending_sync, true); in multifd_send_sync_main()
889 qemu_sem_post(&p->sem); in multifd_send_sync_main()
892 MultiFDSendParams *p = &multifd_send_state->params[i]; in multifd_send_sync_main() local
899 trace_multifd_send_sync_main_wait(p->id); in multifd_send_sync_main()
900 qemu_sem_wait(&p->sem_sync); in multifd_send_sync_main()
902 if (flush_zero_copy && p->c && (multifd_zero_copy_flush(p->c) < 0)) { in multifd_send_sync_main()
913 MultiFDSendParams *p = opaque; in multifd_send_thread() local
919 thread = migration_threads_add(p->name, qemu_get_thread_id()); in multifd_send_thread()
921 trace_multifd_send_thread_start(p->id); in multifd_send_thread()
925 if (multifd_send_initial_packet(p, &local_err) < 0) { in multifd_send_thread()
933 qemu_sem_wait(&p->sem); in multifd_send_thread()
943 if (qatomic_load_acquire(&p->pending_job)) { in multifd_send_thread()
944 MultiFDPages_t *pages = p->pages; in multifd_send_thread()
946 p->iovs_num = 0; in multifd_send_thread()
949 ret = multifd_send_state->ops->send_prepare(p, &local_err); in multifd_send_thread()
955 ret = file_write_ramblock_iov(p->c, p->iov, p->iovs_num, in multifd_send_thread()
956 p->pages->block, &local_err); in multifd_send_thread()
958 ret = qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, in multifd_send_thread()
959 NULL, 0, p->write_flags, in multifd_send_thread()
968 p->next_packet_size + p->packet_len); in multifd_send_thread()
972 multifd_pages_reset(p->pages); in multifd_send_thread()
973 p->next_packet_size = 0; in multifd_send_thread()
980 qatomic_store_release(&p->pending_job, false); in multifd_send_thread()
987 assert(qatomic_read(&p->pending_sync)); in multifd_send_thread()
990 p->flags = MULTIFD_FLAG_SYNC; in multifd_send_thread()
991 multifd_send_fill_packet(p); in multifd_send_thread()
992 ret = qio_channel_write_all(p->c, (void *)p->packet, in multifd_send_thread()
993 p->packet_len, &local_err); in multifd_send_thread()
998 stat64_add(&mig_stats.multifd_bytes, p->packet_len); in multifd_send_thread()
999 p->flags = 0; in multifd_send_thread()
1002 qatomic_set(&p->pending_sync, false); in multifd_send_thread()
1003 qemu_sem_post(&p->sem_sync); in multifd_send_thread()
1010 trace_multifd_send_error(p->id); in multifd_send_thread()
1012 multifd_send_kick_main(p); in multifd_send_thread()
1018 trace_multifd_send_thread_end(p->id, p->packets_sent, p->total_normal_pages, in multifd_send_thread()
1019 p->total_zero_pages); in multifd_send_thread()
1027 MultiFDSendParams *p; member
1037 args->p, in multifd_tls_handshake_thread()
1045 static bool multifd_tls_channel_connect(MultiFDSendParams *p, in multifd_tls_channel_connect() argument
1069 args->p = p; in multifd_tls_channel_connect()
1071 p->tls_thread_created = true; in multifd_tls_channel_connect()
1072 qemu_thread_create(&p->tls_thread, "mig/src/tls", in multifd_tls_channel_connect()
1078 void multifd_channel_connect(MultiFDSendParams *p, QIOChannel *ioc) in multifd_channel_connect() argument
1084 p->c = ioc; in multifd_channel_connect()
1086 p->thread_created = true; in multifd_channel_connect()
1087 qemu_thread_create(&p->thread, p->name, multifd_send_thread, p, in multifd_channel_connect()
1099 MultiFDSendParams *p = opaque; in multifd_new_send_channel_async() local
1104 trace_multifd_new_send_channel_async(p->id); in multifd_new_send_channel_async()
1115 ret = multifd_tls_channel_connect(p, ioc, &local_err); in multifd_new_send_channel_async()
1120 multifd_channel_connect(p, ioc); in multifd_new_send_channel_async()
1135 trace_multifd_new_send_channel_async_error(p->id, local_err); in multifd_new_send_channel_async()
1178 MultiFDSendParams *p = &multifd_send_state->params[i]; in multifd_send_setup() local
1181 qemu_sem_init(&p->sem, 0); in multifd_send_setup()
1182 qemu_sem_init(&p->sem_sync, 0); in multifd_send_setup()
1183 p->id = i; in multifd_send_setup()
1184 p->pages = multifd_pages_init(page_count); in multifd_send_setup()
1187 p->packet_len = sizeof(MultiFDPacket_t) in multifd_send_setup()
1189 p->packet = g_malloc0(p->packet_len); in multifd_send_setup()
1190 p->packet->magic = cpu_to_be32(MULTIFD_MAGIC); in multifd_send_setup()
1191 p->packet->version = cpu_to_be32(MULTIFD_VERSION); in multifd_send_setup()
1193 p->name = g_strdup_printf("mig/src/send_%d", i); in multifd_send_setup()
1194 p->page_size = qemu_target_page_size(); in multifd_send_setup()
1195 p->page_count = page_count; in multifd_send_setup()
1196 p->write_flags = 0; in multifd_send_setup()
1198 if (!multifd_new_send_channel_create(p, &local_err)) { in multifd_send_setup()
1218 MultiFDSendParams *p = &multifd_send_state->params[i]; in multifd_send_setup() local
1221 ret = multifd_send_state->ops->send_setup(p, &local_err); in multifd_send_setup()
1240 MultiFDRecvParams *p = NULL; in multifd_recv() local
1254 p = &multifd_recv_state->params[i]; in multifd_recv()
1256 if (qatomic_read(&p->pending_job) == false) { in multifd_recv()
1268 assert(!p->data->size); in multifd_recv()
1269 multifd_recv_state->data = p->data; in multifd_recv()
1270 p->data = data; in multifd_recv()
1276 qatomic_store_release(&p->pending_job, true); in multifd_recv()
1277 qemu_sem_post(&p->sem); in multifd_recv()
1308 MultiFDRecvParams *p = &multifd_recv_state->params[i]; in multifd_recv_terminate_threads() local
1321 qemu_sem_post(&p->sem_sync); in multifd_recv_terminate_threads()
1330 qemu_sem_post(&p->sem); in multifd_recv_terminate_threads()
1339 if (p->c) { in multifd_recv_terminate_threads()
1340 qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); in multifd_recv_terminate_threads()
1352 static void multifd_recv_cleanup_channel(MultiFDRecvParams *p) in multifd_recv_cleanup_channel() argument
1354 migration_ioc_unregister_yank(p->c); in multifd_recv_cleanup_channel()
1355 object_unref(OBJECT(p->c)); in multifd_recv_cleanup_channel()
1356 p->c = NULL; in multifd_recv_cleanup_channel()
1357 qemu_mutex_destroy(&p->mutex); in multifd_recv_cleanup_channel()
1358 qemu_sem_destroy(&p->sem_sync); in multifd_recv_cleanup_channel()
1359 qemu_sem_destroy(&p->sem); in multifd_recv_cleanup_channel()
1360 g_free(p->data); in multifd_recv_cleanup_channel()
1361 p->data = NULL; in multifd_recv_cleanup_channel()
1362 g_free(p->name); in multifd_recv_cleanup_channel()
1363 p->name = NULL; in multifd_recv_cleanup_channel()
1364 p->packet_len = 0; in multifd_recv_cleanup_channel()
1365 g_free(p->packet); in multifd_recv_cleanup_channel()
1366 p->packet = NULL; in multifd_recv_cleanup_channel()
1367 g_free(p->normal); in multifd_recv_cleanup_channel()
1368 p->normal = NULL; in multifd_recv_cleanup_channel()
1369 g_free(p->zero); in multifd_recv_cleanup_channel()
1370 p->zero = NULL; in multifd_recv_cleanup_channel()
1371 multifd_recv_state->ops->recv_cleanup(p); in multifd_recv_cleanup_channel()
1394 MultiFDRecvParams *p = &multifd_recv_state->params[i]; in multifd_recv_cleanup() local
1396 if (p->thread_created) { in multifd_recv_cleanup()
1397 qemu_thread_join(&p->thread); in multifd_recv_cleanup()
1422 MultiFDRecvParams *p = &multifd_recv_state->params[i]; in multifd_recv_sync_main() local
1424 trace_multifd_recv_sync_main_signal(p->id); in multifd_recv_sync_main()
1425 qemu_sem_post(&p->sem); in multifd_recv_sync_main()
1455 MultiFDRecvParams *p = &multifd_recv_state->params[i]; in multifd_recv_sync_main() local
1457 WITH_QEMU_LOCK_GUARD(&p->mutex) { in multifd_recv_sync_main()
1458 if (multifd_recv_state->packet_num < p->packet_num) { in multifd_recv_sync_main()
1459 multifd_recv_state->packet_num = p->packet_num; in multifd_recv_sync_main()
1462 trace_multifd_recv_sync_main_signal(p->id); in multifd_recv_sync_main()
1463 qemu_sem_post(&p->sem_sync); in multifd_recv_sync_main()
1470 MultiFDRecvParams *p = opaque; in multifd_recv_thread() local
1475 trace_multifd_recv_thread_start(p->id); in multifd_recv_thread()
1481 p->normal_num = 0; in multifd_recv_thread()
1488 ret = qio_channel_read_all_eof(p->c, (void *)p->packet, in multifd_recv_thread()
1489 p->packet_len, &local_err); in multifd_recv_thread()
1494 qemu_mutex_lock(&p->mutex); in multifd_recv_thread()
1495 ret = multifd_recv_unfill_packet(p, &local_err); in multifd_recv_thread()
1497 qemu_mutex_unlock(&p->mutex); in multifd_recv_thread()
1501 flags = p->flags; in multifd_recv_thread()
1503 p->flags &= ~MULTIFD_FLAG_SYNC; in multifd_recv_thread()
1504 has_data = p->normal_num || p->zero_num; in multifd_recv_thread()
1505 qemu_mutex_unlock(&p->mutex); in multifd_recv_thread()
1511 qemu_sem_wait(&p->sem); in multifd_recv_thread()
1518 if (!qatomic_load_acquire(&p->pending_job)) { in multifd_recv_thread()
1529 has_data = !!p->data->size; in multifd_recv_thread()
1533 ret = multifd_recv_state->ops->recv(p, &local_err); in multifd_recv_thread()
1542 qemu_sem_wait(&p->sem_sync); in multifd_recv_thread()
1545 p->total_normal_pages += p->data->size / qemu_target_page_size(); in multifd_recv_thread()
1546 p->data->size = 0; in multifd_recv_thread()
1552 qatomic_store_release(&p->pending_job, false); in multifd_recv_thread()
1562 trace_multifd_recv_thread_end(p->id, p->packets_recved, in multifd_recv_thread()
1563 p->total_normal_pages, in multifd_recv_thread()
1564 p->total_zero_pages); in multifd_recv_thread()
1597 MultiFDRecvParams *p = &multifd_recv_state->params[i]; in multifd_recv_setup() local
1599 qemu_mutex_init(&p->mutex); in multifd_recv_setup()
1600 qemu_sem_init(&p->sem_sync, 0); in multifd_recv_setup()
1601 qemu_sem_init(&p->sem, 0); in multifd_recv_setup()
1602 p->pending_job = false; in multifd_recv_setup()
1603 p->id = i; in multifd_recv_setup()
1605 p->data = g_new0(MultiFDRecvData, 1); in multifd_recv_setup()
1606 p->data->size = 0; in multifd_recv_setup()
1609 p->packet_len = sizeof(MultiFDPacket_t) in multifd_recv_setup()
1611 p->packet = g_malloc0(p->packet_len); in multifd_recv_setup()
1613 p->name = g_strdup_printf("mig/dst/recv_%d", i); in multifd_recv_setup()
1614 p->normal = g_new0(ram_addr_t, page_count); in multifd_recv_setup()
1615 p->zero = g_new0(ram_addr_t, page_count); in multifd_recv_setup()
1616 p->page_count = page_count; in multifd_recv_setup()
1617 p->page_size = qemu_target_page_size(); in multifd_recv_setup()
1621 MultiFDRecvParams *p = &multifd_recv_state->params[i]; in multifd_recv_setup() local
1624 ret = multifd_recv_state->ops->recv_setup(p, errp); in multifd_recv_setup()
1654 MultiFDRecvParams *p; in multifd_recv_new_channel() local
1674 p = &multifd_recv_state->params[id]; in multifd_recv_new_channel()
1675 if (p->c != NULL) { in multifd_recv_new_channel()
1682 p->c = ioc; in multifd_recv_new_channel()
1685 p->thread_created = true; in multifd_recv_new_channel()
1686 qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p, in multifd_recv_new_channel()
1691 bool multifd_send_prepare_common(MultiFDSendParams *p) in multifd_send_prepare_common() argument
1693 multifd_send_zero_page_detect(p); in multifd_send_prepare_common()
1695 if (!p->pages->normal_num) { in multifd_send_prepare_common()
1696 p->next_packet_size = 0; in multifd_send_prepare_common()
1700 multifd_send_prepare_header(p); in multifd_send_prepare_common()