Lines Matching full:rdma
2 * RDMA protocol and interfaces
21 #include "rdma.h"
37 #include <rdma/rdma_cma.h>
85 * The last two bit ranges are only used for RDMA writes,
102 * RDMA migration protocol:
103 * 1. RDMA Writes (data messages, i.e. RAM)
114 * Work request IDs for IB SEND messages only (not RDMA writes).
148 * This is *not* used for RDMA writes, only IB Send/Recv.
158 * Negotiate RDMA capabilities during connection-setup time.
178 * Representation of a RAMBlock from an RDMA perspective.
207 * the information needed to perform the actual RDMA.
288 * Main data structure for RDMA state.
291 * having more than one RDMA connection open at the same time.
380 #define TYPE_QIO_CHANNEL_RDMA "qio-channel-rdma"
422 * the actual RDMA operation.
434 static bool rdma_errored(RDMAContext *rdma) in rdma_errored() argument
436 if (rdma->errored && !rdma->error_reported) { in rdma_errored()
437 error_report("RDMA is in an error state waiting migration" in rdma_errored()
439 rdma->error_reported = true; in rdma_errored()
441 return rdma->errored; in rdma_errored()
444 static void register_to_network(RDMAContext *rdma, RDMARegister *reg) in register_to_network() argument
447 local_block = &rdma->local_ram_blocks.block[reg->current_index]; in register_to_network()
455 reg->key.current_addr += rdma->dest_blocks[reg->current_index].offset; in register_to_network()
476 static void compress_to_network(RDMAContext *rdma, RDMACompress *comp) in compress_to_network() argument
483 comp->offset -= rdma->local_ram_blocks.block[comp->block_idx].offset; in compress_to_network()
484 comp->offset += rdma->dest_blocks[comp->block_idx].offset; in compress_to_network()
501 * the RDMA operation.
521 static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
524 int (*callback)(RDMAContext *rdma,
554 static void rdma_add_block(RDMAContext *rdma, const char *block_name, in rdma_add_block() argument
558 RDMALocalBlocks *local = &rdma->local_ram_blocks; in rdma_add_block()
565 if (rdma->blockmap) { in rdma_add_block()
567 g_hash_table_remove(rdma->blockmap, in rdma_add_block()
569 g_hash_table_insert(rdma->blockmap, in rdma_add_block()
595 if (rdma->blockmap) { in rdma_add_block()
596 g_hash_table_insert(rdma->blockmap, (void *)(uintptr_t)block_offset, block); in rdma_add_block()
630 static void qemu_rdma_init_ram_blocks(RDMAContext *rdma) in qemu_rdma_init_ram_blocks() argument
632 RDMALocalBlocks *local = &rdma->local_ram_blocks; in qemu_rdma_init_ram_blocks()
635 assert(rdma->blockmap == NULL); in qemu_rdma_init_ram_blocks()
637 ret = foreach_not_ignored_block(qemu_rdma_init_one_block, rdma); in qemu_rdma_init_ram_blocks()
640 rdma->dest_blocks = g_new0(RDMADestBlock, in qemu_rdma_init_ram_blocks()
641 rdma->local_ram_blocks.nb_blocks); in qemu_rdma_init_ram_blocks()
649 static void rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block) in rdma_delete_block() argument
651 RDMALocalBlocks *local = &rdma->local_ram_blocks; in rdma_delete_block()
654 if (rdma->blockmap) { in rdma_delete_block()
655 g_hash_table_remove(rdma->blockmap, (void *)(uintptr_t)block->offset); in rdma_delete_block()
663 rdma->total_registrations--; in rdma_delete_block()
671 rdma->total_registrations--; in rdma_delete_block()
687 if (rdma->blockmap) { in rdma_delete_block()
689 g_hash_table_remove(rdma->blockmap, in rdma_delete_block()
725 if (local->nb_blocks && rdma->blockmap) { in rdma_delete_block()
727 g_hash_table_insert(rdma->blockmap, in rdma_delete_block()
735 * Trace RDMA device open, with device details.
758 * Trace RDMA gid addressing information.
759 * Useful for understanding the RDMA device hierarchy in the kernel.
811 * Patches are being reviewed on linux-rdma.
847 "could not open RDMA device context"); in qemu_rdma_broken_ipv6_kernel()
855 "RDMA ERROR: Could not query initial IB port"); in qemu_rdma_broken_ipv6_kernel()
878 error_setg(errp, "RDMA ERROR: " in qemu_rdma_broken_ipv6_kernel()
897 error_setg(errp, "RDMA ERROR: Could not query initial IB port"); in qemu_rdma_broken_ipv6_kernel()
902 error_setg(errp, "RDMA ERROR: " in qemu_rdma_broken_ipv6_kernel()
904 "(but patches on linux-rdma in progress)"); in qemu_rdma_broken_ipv6_kernel()
914 * Figure out which RDMA device corresponds to the requested IP hostname
918 static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp) in qemu_rdma_resolve_host() argument
927 if (rdma->host == NULL || !strcmp(rdma->host, "")) { in qemu_rdma_resolve_host()
928 error_setg(errp, "RDMA ERROR: RDMA hostname has not been set"); in qemu_rdma_resolve_host()
933 rdma->channel = rdma_create_event_channel(); in qemu_rdma_resolve_host()
934 if (!rdma->channel) { in qemu_rdma_resolve_host()
935 error_setg(errp, "RDMA ERROR: could not create CM channel"); in qemu_rdma_resolve_host()
940 ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP); in qemu_rdma_resolve_host()
942 error_setg(errp, "RDMA ERROR: could not create channel id"); in qemu_rdma_resolve_host()
946 snprintf(port_str, 16, "%d", rdma->port); in qemu_rdma_resolve_host()
949 ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res); in qemu_rdma_resolve_host()
951 error_setg(errp, "RDMA ERROR: could not rdma_getaddrinfo address %s", in qemu_rdma_resolve_host()
952 rdma->host); in qemu_rdma_resolve_host()
962 trace_qemu_rdma_resolve_host_trying(rdma->host, ip); in qemu_rdma_resolve_host()
964 ret = rdma_resolve_addr(rdma->cm_id, NULL, e->ai_dst_addr, in qemu_rdma_resolve_host()
968 ret = qemu_rdma_broken_ipv6_kernel(rdma->cm_id->verbs, in qemu_rdma_resolve_host()
983 error_setg(errp, "RDMA ERROR: could not resolve address %s", in qemu_rdma_resolve_host()
984 rdma->host); in qemu_rdma_resolve_host()
990 qemu_rdma_dump_gid("source_resolve_addr", rdma->cm_id); in qemu_rdma_resolve_host()
992 ret = rdma_get_cm_event(rdma->channel, &cm_event); in qemu_rdma_resolve_host()
994 error_setg(errp, "RDMA ERROR: could not perform event_addr_resolved"); in qemu_rdma_resolve_host()
1000 "RDMA ERROR: result not equal to event_addr_resolved %s", in qemu_rdma_resolve_host()
1008 ret = rdma_resolve_route(rdma->cm_id, RDMA_RESOLVE_TIMEOUT_MS); in qemu_rdma_resolve_host()
1010 error_setg(errp, "RDMA ERROR: could not resolve rdma route"); in qemu_rdma_resolve_host()
1014 ret = rdma_get_cm_event(rdma->channel, &cm_event); in qemu_rdma_resolve_host()
1016 error_setg(errp, "RDMA ERROR: could not perform event_route_resolved"); in qemu_rdma_resolve_host()
1020 error_setg(errp, "RDMA ERROR: " in qemu_rdma_resolve_host()
1027 rdma->verbs = rdma->cm_id->verbs; in qemu_rdma_resolve_host()
1028 qemu_rdma_dump_id("source_resolve_host", rdma->cm_id->verbs); in qemu_rdma_resolve_host()
1029 qemu_rdma_dump_gid("source_resolve_host", rdma->cm_id); in qemu_rdma_resolve_host()
1033 rdma_destroy_id(rdma->cm_id); in qemu_rdma_resolve_host()
1034 rdma->cm_id = NULL; in qemu_rdma_resolve_host()
1036 rdma_destroy_event_channel(rdma->channel); in qemu_rdma_resolve_host()
1037 rdma->channel = NULL; in qemu_rdma_resolve_host()
1044 static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma, Error **errp) in qemu_rdma_alloc_pd_cq() argument
1047 rdma->pd = ibv_alloc_pd(rdma->verbs); in qemu_rdma_alloc_pd_cq()
1048 if (!rdma->pd) { in qemu_rdma_alloc_pd_cq()
1054 rdma->recv_comp_channel = ibv_create_comp_channel(rdma->verbs); in qemu_rdma_alloc_pd_cq()
1055 if (!rdma->recv_comp_channel) { in qemu_rdma_alloc_pd_cq()
1063 rdma->recv_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3), in qemu_rdma_alloc_pd_cq()
1064 NULL, rdma->recv_comp_channel, 0); in qemu_rdma_alloc_pd_cq()
1065 if (!rdma->recv_cq) { in qemu_rdma_alloc_pd_cq()
1071 rdma->send_comp_channel = ibv_create_comp_channel(rdma->verbs); in qemu_rdma_alloc_pd_cq()
1072 if (!rdma->send_comp_channel) { in qemu_rdma_alloc_pd_cq()
1077 rdma->send_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3), in qemu_rdma_alloc_pd_cq()
1078 NULL, rdma->send_comp_channel, 0); in qemu_rdma_alloc_pd_cq()
1079 if (!rdma->send_cq) { in qemu_rdma_alloc_pd_cq()
1087 if (rdma->pd) { in qemu_rdma_alloc_pd_cq()
1088 ibv_dealloc_pd(rdma->pd); in qemu_rdma_alloc_pd_cq()
1090 if (rdma->recv_comp_channel) { in qemu_rdma_alloc_pd_cq()
1091 ibv_destroy_comp_channel(rdma->recv_comp_channel); in qemu_rdma_alloc_pd_cq()
1093 if (rdma->send_comp_channel) { in qemu_rdma_alloc_pd_cq()
1094 ibv_destroy_comp_channel(rdma->send_comp_channel); in qemu_rdma_alloc_pd_cq()
1096 if (rdma->recv_cq) { in qemu_rdma_alloc_pd_cq()
1097 ibv_destroy_cq(rdma->recv_cq); in qemu_rdma_alloc_pd_cq()
1098 rdma->recv_cq = NULL; in qemu_rdma_alloc_pd_cq()
1100 rdma->pd = NULL; in qemu_rdma_alloc_pd_cq()
1101 rdma->recv_comp_channel = NULL; in qemu_rdma_alloc_pd_cq()
1102 rdma->send_comp_channel = NULL; in qemu_rdma_alloc_pd_cq()
1110 static int qemu_rdma_alloc_qp(RDMAContext *rdma) in qemu_rdma_alloc_qp() argument
1118 attr.send_cq = rdma->send_cq; in qemu_rdma_alloc_qp()
1119 attr.recv_cq = rdma->recv_cq; in qemu_rdma_alloc_qp()
1122 if (rdma_create_qp(rdma->cm_id, rdma->pd, &attr) < 0) { in qemu_rdma_alloc_qp()
1126 rdma->qp = rdma->cm_id->qp; in qemu_rdma_alloc_qp()
1168 static int qemu_rdma_reg_whole_ram_blocks(RDMAContext *rdma, Error **errp) in qemu_rdma_reg_whole_ram_blocks() argument
1171 RDMALocalBlocks *local = &rdma->local_ram_blocks; in qemu_rdma_reg_whole_ram_blocks()
1177 ibv_reg_mr(rdma->pd, in qemu_rdma_reg_whole_ram_blocks()
1188 errno == ENOTSUP && rdma_support_odp(rdma->verbs)) { in qemu_rdma_reg_whole_ram_blocks()
1192 ibv_reg_mr(rdma->pd, in qemu_rdma_reg_whole_ram_blocks()
1198 qemu_rdma_advise_prefetch_mr(rdma->pd, in qemu_rdma_reg_whole_ram_blocks()
1212 rdma->total_registrations++; in qemu_rdma_reg_whole_ram_blocks()
1221 rdma->total_registrations--; in qemu_rdma_reg_whole_ram_blocks()
1235 static void qemu_rdma_search_ram_block(RDMAContext *rdma, in qemu_rdma_search_ram_block() argument
1243 RDMALocalBlock *block = g_hash_table_lookup(rdma->blockmap, in qemu_rdma_search_ram_block()
1259 * to perform the actual RDMA operation.
1261 static int qemu_rdma_register_and_get_keys(RDMAContext *rdma, in qemu_rdma_register_and_get_keys() argument
1293 block->pmr[chunk] = ibv_reg_mr(rdma->pd, chunk_start, len, access); in qemu_rdma_register_and_get_keys()
1301 errno == ENOTSUP && rdma_support_odp(rdma->verbs)) { in qemu_rdma_register_and_get_keys()
1304 block->pmr[chunk] = ibv_reg_mr(rdma->pd, chunk_start, len, access); in qemu_rdma_register_and_get_keys()
1308 qemu_rdma_advise_prefetch_mr(rdma->pd, (uintptr_t)chunk_start, in qemu_rdma_register_and_get_keys()
1318 rdma->total_registrations++; in qemu_rdma_register_and_get_keys()
1333 static int qemu_rdma_reg_control(RDMAContext *rdma, int idx) in qemu_rdma_reg_control() argument
1335 rdma->wr_data[idx].control_mr = ibv_reg_mr(rdma->pd, in qemu_rdma_reg_control()
1336 rdma->wr_data[idx].control, RDMA_CONTROL_MAX_BUFFER, in qemu_rdma_reg_control()
1338 if (rdma->wr_data[idx].control_mr) { in qemu_rdma_reg_control()
1339 rdma->total_registrations++; in qemu_rdma_reg_control()
1356 static int qemu_rdma_unregister_waiting(RDMAContext *rdma) in qemu_rdma_unregister_waiting() argument
1360 while (rdma->unregistrations[rdma->unregister_current]) { in qemu_rdma_unregister_waiting()
1362 uint64_t wr_id = rdma->unregistrations[rdma->unregister_current]; in qemu_rdma_unregister_waiting()
1368 &(rdma->local_ram_blocks.block[index]); in qemu_rdma_unregister_waiting()
1378 rdma->unregister_current); in qemu_rdma_unregister_waiting()
1380 rdma->unregistrations[rdma->unregister_current] = 0; in qemu_rdma_unregister_waiting()
1381 rdma->unregister_current++; in qemu_rdma_unregister_waiting()
1383 if (rdma->unregister_current == RDMA_SIGNALED_SEND_MAX) { in qemu_rdma_unregister_waiting()
1384 rdma->unregister_current = 0; in qemu_rdma_unregister_waiting()
1413 rdma->total_registrations--; in qemu_rdma_unregister_waiting()
1416 register_to_network(rdma, ®); in qemu_rdma_unregister_waiting()
1417 ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) ®, in qemu_rdma_unregister_waiting()
1446 static int qemu_rdma_poll(RDMAContext *rdma, struct ibv_cq *cq, in qemu_rdma_poll() argument
1470 if (rdma->control_ready_expected && in qemu_rdma_poll()
1473 rdma->nb_sent); in qemu_rdma_poll()
1474 rdma->control_ready_expected = 0; in qemu_rdma_poll()
1482 RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]); in qemu_rdma_poll()
1484 trace_qemu_rdma_poll_write(wr_id, rdma->nb_sent, in qemu_rdma_poll()
1490 if (rdma->nb_sent > 0) { in qemu_rdma_poll()
1491 rdma->nb_sent--; in qemu_rdma_poll()
1494 trace_qemu_rdma_poll_other(wr_id, rdma->nb_sent); in qemu_rdma_poll()
1508 static int qemu_rdma_wait_comp_channel(RDMAContext *rdma, in qemu_rdma_wait_comp_channel() argument
1517 if (rdma->migration_started_on_destination && in qemu_rdma_wait_comp_channel()
1528 while (!rdma->errored && !rdma->received_error) { in qemu_rdma_wait_comp_channel()
1534 pfds[1].fd = rdma->channel->fd; in qemu_rdma_wait_comp_channel()
1547 if (rdma_get_cm_event(rdma->channel, &cm_event) < 0) { in qemu_rdma_wait_comp_channel()
1576 if (rdma->received_error) { in qemu_rdma_wait_comp_channel()
1579 return -rdma->errored; in qemu_rdma_wait_comp_channel()
1582 static struct ibv_comp_channel *to_channel(RDMAContext *rdma, uint64_t wrid) in to_channel() argument
1584 return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_comp_channel : in to_channel()
1585 rdma->recv_comp_channel; in to_channel()
1588 static struct ibv_cq *to_cq(RDMAContext *rdma, uint64_t wrid) in to_cq() argument
1590 return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_cq : rdma->recv_cq; in to_cq()
1602 * The only exception is actual RDMA Write completions. These
1606 static int qemu_rdma_block_for_wrid(RDMAContext *rdma, in qemu_rdma_block_for_wrid() argument
1614 struct ibv_comp_channel *ch = to_channel(rdma, wrid_requested); in qemu_rdma_block_for_wrid()
1615 struct ibv_cq *poll_cq = to_cq(rdma, wrid_requested); in qemu_rdma_block_for_wrid()
1622 ret = qemu_rdma_poll(rdma, poll_cq, &wr_id_in, byte_len); in qemu_rdma_block_for_wrid()
1642 ret = qemu_rdma_wait_comp_channel(rdma, ch); in qemu_rdma_block_for_wrid()
1659 ret = qemu_rdma_poll(rdma, poll_cq, &wr_id_in, byte_len); in qemu_rdma_block_for_wrid()
1690 rdma->errored = true; in qemu_rdma_block_for_wrid()
1698 static int qemu_rdma_post_send_control(RDMAContext *rdma, uint8_t *buf, in qemu_rdma_post_send_control() argument
1703 RDMAWorkRequestData *wr = &rdma->wr_data[RDMA_WRID_CONTROL]; in qemu_rdma_post_send_control()
1737 ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr); in qemu_rdma_post_send_control()
1744 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL); in qemu_rdma_post_send_control()
1746 error_setg(errp, "rdma migration: send polling control error"); in qemu_rdma_post_send_control()
1757 static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx, in qemu_rdma_post_recv_control() argument
1762 .addr = (uintptr_t)(rdma->wr_data[idx].control), in qemu_rdma_post_recv_control()
1764 .lkey = rdma->wr_data[idx].control_mr->lkey, in qemu_rdma_post_recv_control()
1774 if (ibv_post_recv(rdma->qp, &recv_wr, &bad_wr)) { in qemu_rdma_post_recv_control()
1785 static int qemu_rdma_exchange_get_response(RDMAContext *rdma, in qemu_rdma_exchange_get_response() argument
1790 int ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RECV_CONTROL + idx, in qemu_rdma_exchange_get_response()
1794 error_setg(errp, "rdma migration: recv polling control error!"); in qemu_rdma_exchange_get_response()
1798 network_to_control((void *) rdma->wr_data[idx].control); in qemu_rdma_exchange_get_response()
1799 memcpy(head, rdma->wr_data[idx].control, sizeof(RDMAControlHeader)); in qemu_rdma_exchange_get_response()
1812 rdma->received_error = true; in qemu_rdma_exchange_get_response()
1837 static void qemu_rdma_move_header(RDMAContext *rdma, int idx, in qemu_rdma_move_header() argument
1840 rdma->wr_data[idx].control_len = head->len; in qemu_rdma_move_header()
1841 rdma->wr_data[idx].control_curr = in qemu_rdma_move_header()
1842 rdma->wr_data[idx].control + sizeof(RDMAControlHeader); in qemu_rdma_move_header()
1858 static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head, in qemu_rdma_exchange_send() argument
1861 int (*callback)(RDMAContext *rdma, in qemu_rdma_exchange_send() argument
1871 if (rdma->control_ready_expected) { in qemu_rdma_exchange_send()
1874 ret = qemu_rdma_exchange_get_response(rdma, &resp_ignored, in qemu_rdma_exchange_send()
1886 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA, errp); in qemu_rdma_exchange_send()
1895 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp); in qemu_rdma_exchange_send()
1903 ret = qemu_rdma_post_send_control(rdma, data, head, errp); in qemu_rdma_exchange_send()
1915 ret = callback(rdma, errp); in qemu_rdma_exchange_send()
1922 ret = qemu_rdma_exchange_get_response(rdma, resp, in qemu_rdma_exchange_send()
1930 qemu_rdma_move_header(rdma, RDMA_WRID_DATA, resp); in qemu_rdma_exchange_send()
1937 rdma->control_ready_expected = 1; in qemu_rdma_exchange_send()
1946 static int qemu_rdma_exchange_recv(RDMAContext *rdma, RDMAControlHeader *head, in qemu_rdma_exchange_recv() argument
1959 ret = qemu_rdma_post_send_control(rdma, NULL, &ready, errp); in qemu_rdma_exchange_recv()
1968 ret = qemu_rdma_exchange_get_response(rdma, head, in qemu_rdma_exchange_recv()
1975 qemu_rdma_move_header(rdma, RDMA_WRID_READY, head); in qemu_rdma_exchange_recv()
1980 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp); in qemu_rdma_exchange_recv()
1989 * Write an actual chunk of memory using RDMA.
1994 static int qemu_rdma_write_one(RDMAContext *rdma, in qemu_rdma_write_one() argument
2004 RDMALocalBlock *block = &(rdma->local_ram_blocks.block[current_index]); in qemu_rdma_write_one()
2046 sge.addr, length, rdma->nb_sent, block->nb_chunks); in qemu_rdma_write_one()
2048 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL); in qemu_rdma_write_one()
2054 current_index, chunk, sge.addr, length, rdma->nb_sent); in qemu_rdma_write_one()
2059 if (!rdma->pin_all || !block->is_ram_block) { in qemu_rdma_write_one()
2064 * memset() + madvise() the entire chunk without RDMA. in qemu_rdma_write_one()
2081 compress_to_network(rdma, &comp); in qemu_rdma_write_one()
2082 ret = qemu_rdma_exchange_send(rdma, &head, in qemu_rdma_write_one()
2118 register_to_network(rdma, ®); in qemu_rdma_write_one()
2119 ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) ®, in qemu_rdma_write_one()
2126 if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr, in qemu_rdma_write_one()
2134 rdma->wr_data[reg_result_idx].control_curr; in qemu_rdma_write_one()
2145 if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr, in qemu_rdma_write_one()
2153 send_wr.wr.rdma.rkey = block->remote_keys[chunk]; in qemu_rdma_write_one()
2155 send_wr.wr.rdma.rkey = block->remote_rkey; in qemu_rdma_write_one()
2157 if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr, in qemu_rdma_write_one()
2178 send_wr.wr.rdma.remote_addr = block->remote_host_addr + in qemu_rdma_write_one()
2181 trace_qemu_rdma_write_one_post(chunk, sge.addr, send_wr.wr.rdma.remote_addr, in qemu_rdma_write_one()
2188 ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr); in qemu_rdma_write_one()
2192 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL); in qemu_rdma_write_one()
2194 error_setg(errp, "rdma migration: failed to make " in qemu_rdma_write_one()
2203 "rdma migration: post rdma write failed"); in qemu_rdma_write_one()
2211 * overhead at all. I will assume that RDMA is magicaly and don't in qemu_rdma_write_one()
2216 * but this being RDMA, who knows. in qemu_rdma_write_one()
2220 rdma->total_writes++; in qemu_rdma_write_one()
2226 * Push out any unwritten RDMA operations.
2231 static int qemu_rdma_write_flush(RDMAContext *rdma, Error **errp) in qemu_rdma_write_flush() argument
2235 if (!rdma->current_length) { in qemu_rdma_write_flush()
2239 ret = qemu_rdma_write_one(rdma, rdma->current_index, rdma->current_addr, in qemu_rdma_write_flush()
2240 rdma->current_length, errp); in qemu_rdma_write_flush()
2247 rdma->nb_sent++; in qemu_rdma_write_flush()
2248 trace_qemu_rdma_write_flush(rdma->nb_sent); in qemu_rdma_write_flush()
2251 rdma->current_length = 0; in qemu_rdma_write_flush()
2252 rdma->current_addr = 0; in qemu_rdma_write_flush()
2257 static inline bool qemu_rdma_buffer_mergeable(RDMAContext *rdma, in qemu_rdma_buffer_mergeable() argument
2264 if (rdma->current_index < 0) { in qemu_rdma_buffer_mergeable()
2268 if (rdma->current_chunk < 0) { in qemu_rdma_buffer_mergeable()
2272 block = &(rdma->local_ram_blocks.block[rdma->current_index]); in qemu_rdma_buffer_mergeable()
2274 chunk_end = ram_chunk_end(block, rdma->current_chunk); in qemu_rdma_buffer_mergeable()
2276 if (rdma->current_length == 0) { in qemu_rdma_buffer_mergeable()
2283 if (offset != (rdma->current_addr + rdma->current_length)) { in qemu_rdma_buffer_mergeable()
2312 static int qemu_rdma_write(RDMAContext *rdma, in qemu_rdma_write() argument
2317 uint64_t index = rdma->current_index; in qemu_rdma_write()
2318 uint64_t chunk = rdma->current_chunk; in qemu_rdma_write()
2321 if (!qemu_rdma_buffer_mergeable(rdma, current_addr, len)) { in qemu_rdma_write()
2322 if (qemu_rdma_write_flush(rdma, errp) < 0) { in qemu_rdma_write()
2325 rdma->current_length = 0; in qemu_rdma_write()
2326 rdma->current_addr = current_addr; in qemu_rdma_write()
2328 qemu_rdma_search_ram_block(rdma, block_offset, in qemu_rdma_write()
2330 rdma->current_index = index; in qemu_rdma_write()
2331 rdma->current_chunk = chunk; in qemu_rdma_write()
2335 rdma->current_length += len; in qemu_rdma_write()
2338 if (rdma->current_length >= RDMA_MERGE_MAX) { in qemu_rdma_write()
2339 return qemu_rdma_write_flush(rdma, errp); in qemu_rdma_write()
2345 static void qemu_rdma_cleanup(RDMAContext *rdma) in qemu_rdma_cleanup() argument
2349 if (rdma->cm_id && rdma->connected) { in qemu_rdma_cleanup()
2350 if ((rdma->errored || in qemu_rdma_cleanup()
2352 !rdma->received_error) { in qemu_rdma_cleanup()
2358 if (qemu_rdma_post_send_control(rdma, NULL, &head, &err) < 0) { in qemu_rdma_cleanup()
2363 rdma_disconnect(rdma->cm_id); in qemu_rdma_cleanup()
2365 rdma->connected = false; in qemu_rdma_cleanup()
2368 if (rdma->channel) { in qemu_rdma_cleanup()
2369 qemu_set_fd_handler(rdma->channel->fd, NULL, NULL, NULL); in qemu_rdma_cleanup()
2371 g_free(rdma->dest_blocks); in qemu_rdma_cleanup()
2372 rdma->dest_blocks = NULL; in qemu_rdma_cleanup()
2375 if (rdma->wr_data[i].control_mr) { in qemu_rdma_cleanup()
2376 rdma->total_registrations--; in qemu_rdma_cleanup()
2377 ibv_dereg_mr(rdma->wr_data[i].control_mr); in qemu_rdma_cleanup()
2379 rdma->wr_data[i].control_mr = NULL; in qemu_rdma_cleanup()
2382 if (rdma->local_ram_blocks.block) { in qemu_rdma_cleanup()
2383 while (rdma->local_ram_blocks.nb_blocks) { in qemu_rdma_cleanup()
2384 rdma_delete_block(rdma, &rdma->local_ram_blocks.block[0]); in qemu_rdma_cleanup()
2388 if (rdma->qp) { in qemu_rdma_cleanup()
2389 rdma_destroy_qp(rdma->cm_id); in qemu_rdma_cleanup()
2390 rdma->qp = NULL; in qemu_rdma_cleanup()
2392 if (rdma->recv_cq) { in qemu_rdma_cleanup()
2393 ibv_destroy_cq(rdma->recv_cq); in qemu_rdma_cleanup()
2394 rdma->recv_cq = NULL; in qemu_rdma_cleanup()
2396 if (rdma->send_cq) { in qemu_rdma_cleanup()
2397 ibv_destroy_cq(rdma->send_cq); in qemu_rdma_cleanup()
2398 rdma->send_cq = NULL; in qemu_rdma_cleanup()
2400 if (rdma->recv_comp_channel) { in qemu_rdma_cleanup()
2401 ibv_destroy_comp_channel(rdma->recv_comp_channel); in qemu_rdma_cleanup()
2402 rdma->recv_comp_channel = NULL; in qemu_rdma_cleanup()
2404 if (rdma->send_comp_channel) { in qemu_rdma_cleanup()
2405 ibv_destroy_comp_channel(rdma->send_comp_channel); in qemu_rdma_cleanup()
2406 rdma->send_comp_channel = NULL; in qemu_rdma_cleanup()
2408 if (rdma->pd) { in qemu_rdma_cleanup()
2409 ibv_dealloc_pd(rdma->pd); in qemu_rdma_cleanup()
2410 rdma->pd = NULL; in qemu_rdma_cleanup()
2412 if (rdma->cm_id) { in qemu_rdma_cleanup()
2413 rdma_destroy_id(rdma->cm_id); in qemu_rdma_cleanup()
2414 rdma->cm_id = NULL; in qemu_rdma_cleanup()
2418 if (rdma->listen_id) { in qemu_rdma_cleanup()
2419 if (!rdma->is_return_path) { in qemu_rdma_cleanup()
2420 rdma_destroy_id(rdma->listen_id); in qemu_rdma_cleanup()
2422 rdma->listen_id = NULL; in qemu_rdma_cleanup()
2424 if (rdma->channel) { in qemu_rdma_cleanup()
2425 if (!rdma->is_return_path) { in qemu_rdma_cleanup()
2426 rdma_destroy_event_channel(rdma->channel); in qemu_rdma_cleanup()
2428 rdma->channel = NULL; in qemu_rdma_cleanup()
2432 if (rdma->channel) { in qemu_rdma_cleanup()
2433 rdma_destroy_event_channel(rdma->channel); in qemu_rdma_cleanup()
2434 rdma->channel = NULL; in qemu_rdma_cleanup()
2436 g_free(rdma->host); in qemu_rdma_cleanup()
2437 rdma->host = NULL; in qemu_rdma_cleanup()
2441 static int qemu_rdma_source_init(RDMAContext *rdma, bool pin_all, Error **errp) in qemu_rdma_source_init() argument
2449 rdma->pin_all = pin_all; in qemu_rdma_source_init()
2451 ret = qemu_rdma_resolve_host(rdma, errp); in qemu_rdma_source_init()
2456 ret = qemu_rdma_alloc_pd_cq(rdma, errp); in qemu_rdma_source_init()
2461 ret = qemu_rdma_alloc_qp(rdma); in qemu_rdma_source_init()
2463 error_setg(errp, "RDMA ERROR: rdma migration: error allocating qp!"); in qemu_rdma_source_init()
2467 qemu_rdma_init_ram_blocks(rdma); in qemu_rdma_source_init()
2470 rdma->blockmap = g_hash_table_new(g_direct_hash, g_direct_equal); in qemu_rdma_source_init()
2471 for (int i = 0; i < rdma->local_ram_blocks.nb_blocks; i++) { in qemu_rdma_source_init()
2472 g_hash_table_insert(rdma->blockmap, in qemu_rdma_source_init()
2473 (void *)(uintptr_t)rdma->local_ram_blocks.block[i].offset, in qemu_rdma_source_init()
2474 &rdma->local_ram_blocks.block[i]); in qemu_rdma_source_init()
2478 ret = qemu_rdma_reg_control(rdma, i); in qemu_rdma_source_init()
2480 error_setg(errp, "RDMA ERROR: rdma migration: error " in qemu_rdma_source_init()
2489 qemu_rdma_cleanup(rdma); in qemu_rdma_source_init()
2493 static int qemu_get_cm_event_timeout(RDMAContext *rdma, in qemu_get_cm_event_timeout() argument
2499 .fd = rdma->channel->fd, in qemu_get_cm_event_timeout()
2509 error_setg(errp, "RDMA ERROR: poll cm event timeout"); in qemu_get_cm_event_timeout()
2512 error_setg(errp, "RDMA ERROR: failed to poll cm event, errno=%i", in qemu_get_cm_event_timeout()
2516 if (rdma_get_cm_event(rdma->channel, cm_event) < 0) { in qemu_get_cm_event_timeout()
2517 error_setg(errp, "RDMA ERROR: failed to get cm event"); in qemu_get_cm_event_timeout()
2522 error_setg(errp, "RDMA ERROR: no POLLIN event, revent=%x", in qemu_get_cm_event_timeout()
2528 static int qemu_rdma_connect(RDMAContext *rdma, bool return_path, in qemu_rdma_connect() argument
2547 if (rdma->pin_all) { in qemu_rdma_connect()
2554 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp); in qemu_rdma_connect()
2559 ret = rdma_connect(rdma->cm_id, &conn_param); in qemu_rdma_connect()
2562 "RDMA ERROR: connecting to destination!"); in qemu_rdma_connect()
2567 ret = qemu_get_cm_event_timeout(rdma, &cm_event, 5000, errp); in qemu_rdma_connect()
2569 ret = rdma_get_cm_event(rdma->channel, &cm_event); in qemu_rdma_connect()
2572 "RDMA ERROR: failed to get cm event"); in qemu_rdma_connect()
2580 error_setg(errp, "RDMA ERROR: connecting to destination!"); in qemu_rdma_connect()
2584 rdma->connected = true; in qemu_rdma_connect()
2593 if (rdma->pin_all && !(cap.flags & RDMA_CAPABILITY_PIN_ALL)) { in qemu_rdma_connect()
2594 warn_report("RDMA: Server cannot support pinning all memory. " in qemu_rdma_connect()
2596 rdma->pin_all = false; in qemu_rdma_connect()
2599 trace_qemu_rdma_connect_pin_all_outcome(rdma->pin_all); in qemu_rdma_connect()
2603 rdma->control_ready_expected = 1; in qemu_rdma_connect()
2604 rdma->nb_sent = 0; in qemu_rdma_connect()
2608 qemu_rdma_cleanup(rdma); in qemu_rdma_connect()
2612 static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp) in qemu_rdma_dest_init() argument
2623 rdma->wr_data[i].control_len = 0; in qemu_rdma_dest_init()
2624 rdma->wr_data[i].control_curr = NULL; in qemu_rdma_dest_init()
2627 if (!rdma->host || !rdma->host[0]) { in qemu_rdma_dest_init()
2628 error_setg(errp, "RDMA ERROR: RDMA host is not set!"); in qemu_rdma_dest_init()
2629 rdma->errored = true; in qemu_rdma_dest_init()
2633 rdma->channel = rdma_create_event_channel(); in qemu_rdma_dest_init()
2634 if (!rdma->channel) { in qemu_rdma_dest_init()
2635 error_setg(errp, "RDMA ERROR: could not create rdma event channel"); in qemu_rdma_dest_init()
2636 rdma->errored = true; in qemu_rdma_dest_init()
2641 ret = rdma_create_id(rdma->channel, &listen_id, NULL, RDMA_PS_TCP); in qemu_rdma_dest_init()
2643 error_setg(errp, "RDMA ERROR: could not create cm_id!"); in qemu_rdma_dest_init()
2647 snprintf(port_str, 16, "%d", rdma->port); in qemu_rdma_dest_init()
2650 ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res); in qemu_rdma_dest_init()
2652 error_setg(errp, "RDMA ERROR: could not rdma_getaddrinfo address %s", in qemu_rdma_dest_init()
2653 rdma->host); in qemu_rdma_dest_init()
2660 error_setg(errp, "RDMA ERROR: Error: could not set REUSEADDR option"); in qemu_rdma_dest_init()
2670 trace_qemu_rdma_dest_init_trying(rdma->host, ip); in qemu_rdma_dest_init()
2691 error_setg(errp, "RDMA ERROR: Error: could not rdma_bind_addr!"); in qemu_rdma_dest_init()
2696 rdma->listen_id = listen_id; in qemu_rdma_dest_init()
2703 rdma_destroy_event_channel(rdma->channel); in qemu_rdma_dest_init()
2704 rdma->channel = NULL; in qemu_rdma_dest_init()
2705 rdma->errored = true; in qemu_rdma_dest_init()
2711 RDMAContext *rdma) in qemu_rdma_return_path_dest_init()
2719 rdma_return_path->channel = rdma->channel; in qemu_rdma_return_path_dest_init()
2720 rdma_return_path->listen_id = rdma->listen_id; in qemu_rdma_return_path_dest_init()
2722 rdma->return_path = rdma_return_path; in qemu_rdma_return_path_dest_init()
2723 rdma_return_path->return_path = rdma; in qemu_rdma_return_path_dest_init()
2729 RDMAContext *rdma = NULL; in qemu_rdma_data_init() local
2731 rdma = g_new0(RDMAContext, 1); in qemu_rdma_data_init()
2732 rdma->current_index = -1; in qemu_rdma_data_init()
2733 rdma->current_chunk = -1; in qemu_rdma_data_init()
2735 rdma->host = g_strdup(saddr->host); in qemu_rdma_data_init()
2736 rdma->port = atoi(saddr->port); in qemu_rdma_data_init()
2737 return rdma; in qemu_rdma_data_init()
2743 * VM's ram is handled with regular RDMA messages.
2754 RDMAContext *rdma; in qio_channel_rdma_writev() local
2760 rdma = qatomic_rcu_read(&rioc->rdmaout); in qio_channel_rdma_writev()
2762 if (!rdma) { in qio_channel_rdma_writev()
2763 error_setg(errp, "RDMA control channel output is not set"); in qio_channel_rdma_writev()
2767 if (rdma->errored) { in qio_channel_rdma_writev()
2769 "RDMA is in an error state waiting migration to abort!"); in qio_channel_rdma_writev()
2777 ret = qemu_rdma_write_flush(rdma, errp); in qio_channel_rdma_writev()
2779 rdma->errored = true; in qio_channel_rdma_writev()
2795 ret = qemu_rdma_exchange_send(rdma, &head, in qio_channel_rdma_writev()
2799 rdma->errored = true; in qio_channel_rdma_writev()
2811 static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf, in qemu_rdma_fill() argument
2816 if (rdma->wr_data[idx].control_len) { in qemu_rdma_fill()
2817 trace_qemu_rdma_fill(rdma->wr_data[idx].control_len, size); in qemu_rdma_fill()
2819 len = MIN(size, rdma->wr_data[idx].control_len); in qemu_rdma_fill()
2820 memcpy(buf, rdma->wr_data[idx].control_curr, len); in qemu_rdma_fill()
2821 rdma->wr_data[idx].control_curr += len; in qemu_rdma_fill()
2822 rdma->wr_data[idx].control_len -= len; in qemu_rdma_fill()
2830 * RDMA links don't use bytestreams, so we have to
2842 RDMAContext *rdma; in qio_channel_rdma_readv() local
2849 rdma = qatomic_rcu_read(&rioc->rdmain); in qio_channel_rdma_readv()
2851 if (!rdma) { in qio_channel_rdma_readv()
2852 error_setg(errp, "RDMA control channel input is not set"); in qio_channel_rdma_readv()
2856 if (rdma->errored) { in qio_channel_rdma_readv()
2858 "RDMA is in an error state waiting migration to abort!"); in qio_channel_rdma_readv()
2871 len = qemu_rdma_fill(rdma, data, want, 0); in qio_channel_rdma_readv()
2889 ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_QEMU_FILE, in qio_channel_rdma_readv()
2893 rdma->errored = true; in qio_channel_rdma_readv()
2900 len = qemu_rdma_fill(rdma, data, want, 0); in qio_channel_rdma_readv()
2919 static int qemu_rdma_drain_cq(RDMAContext *rdma) in qemu_rdma_drain_cq() argument
2923 if (qemu_rdma_write_flush(rdma, &err) < 0) { in qemu_rdma_drain_cq()
2928 while (rdma->nb_sent) { in qemu_rdma_drain_cq()
2929 if (qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL) < 0) { in qemu_rdma_drain_cq()
2930 error_report("rdma migration: complete polling error!"); in qemu_rdma_drain_cq()
2935 qemu_rdma_unregister_waiting(rdma); in qemu_rdma_drain_cq()
2964 RDMAContext *rdma; in qio_channel_rdma_source_prepare() local
2970 rdma = qatomic_rcu_read(&rsource->rioc->rdmain); in qio_channel_rdma_source_prepare()
2972 rdma = qatomic_rcu_read(&rsource->rioc->rdmaout); in qio_channel_rdma_source_prepare()
2975 if (!rdma) { in qio_channel_rdma_source_prepare()
2980 if (rdma->wr_data[0].control_len) { in qio_channel_rdma_source_prepare()
2992 RDMAContext *rdma; in qio_channel_rdma_source_check() local
2997 rdma = qatomic_rcu_read(&rsource->rioc->rdmain); in qio_channel_rdma_source_check()
2999 rdma = qatomic_rcu_read(&rsource->rioc->rdmaout); in qio_channel_rdma_source_check()
3002 if (!rdma) { in qio_channel_rdma_source_check()
3007 if (rdma->wr_data[0].control_len) { in qio_channel_rdma_source_check()
3022 RDMAContext *rdma; in qio_channel_rdma_source_dispatch() local
3027 rdma = qatomic_rcu_read(&rsource->rioc->rdmain); in qio_channel_rdma_source_dispatch()
3029 rdma = qatomic_rcu_read(&rsource->rioc->rdmaout); in qio_channel_rdma_source_dispatch()
3032 if (!rdma) { in qio_channel_rdma_source_dispatch()
3037 if (rdma->wr_data[0].control_len) { in qio_channel_rdma_source_dispatch()
3211 RDMAContext *rdma; in qemu_rdma_save_page() local
3215 rdma = qatomic_rcu_read(&rioc->rdmaout); in qemu_rdma_save_page()
3217 if (!rdma) { in qemu_rdma_save_page()
3221 if (rdma_errored(rdma)) { in qemu_rdma_save_page()
3230 * an actual RDMA write will occur and a new chunk will be formed. in qemu_rdma_save_page()
3232 ret = qemu_rdma_write(rdma, block_offset, offset, size, &err); in qemu_rdma_save_page()
3247 ret = qemu_rdma_poll(rdma, rdma->recv_cq, &wr_id_in, NULL); in qemu_rdma_save_page()
3250 error_report("rdma migration: polling error"); in qemu_rdma_save_page()
3263 ret = qemu_rdma_poll(rdma, rdma->send_cq, &wr_id_in, NULL); in qemu_rdma_save_page()
3266 error_report("rdma migration: polling error"); in qemu_rdma_save_page()
3280 rdma->errored = true; in qemu_rdma_save_page()
3306 RDMAContext *rdma = opaque; in rdma_cm_poll_handler() local
3310 if (rdma_get_cm_event(rdma->channel, &cm_event) < 0) { in rdma_cm_poll_handler()
3317 if (!rdma->errored && in rdma_cm_poll_handler()
3321 rdma->errored = true; in rdma_cm_poll_handler()
3322 if (rdma->return_path) { in rdma_cm_poll_handler()
3323 rdma->return_path->errored = true; in rdma_cm_poll_handler()
3335 static int qemu_rdma_accept(RDMAContext *rdma) in qemu_rdma_accept() argument
3350 ret = rdma_get_cm_event(rdma->channel, &cm_event); in qemu_rdma_accept()
3360 isock->host = g_strdup(rdma->host); in qemu_rdma_accept()
3361 isock->port = g_strdup_printf("%d", rdma->port); in qemu_rdma_accept()
3368 && !rdma->is_return_path) { in qemu_rdma_accept()
3375 qemu_rdma_return_path_dest_init(rdma_return_path, rdma); in qemu_rdma_accept()
3383 error_report("Unknown source RDMA version: %d, bailing...", in qemu_rdma_accept()
3399 rdma->pin_all = true; in qemu_rdma_accept()
3402 rdma->cm_id = cm_event->id; in qemu_rdma_accept()
3407 trace_qemu_rdma_accept_pin_state(rdma->pin_all); in qemu_rdma_accept()
3413 if (!rdma->verbs) { in qemu_rdma_accept()
3414 rdma->verbs = verbs; in qemu_rdma_accept()
3415 } else if (rdma->verbs != verbs) { in qemu_rdma_accept()
3416 error_report("ibv context not matching %p, %p!", rdma->verbs, in qemu_rdma_accept()
3423 ret = qemu_rdma_alloc_pd_cq(rdma, &err); in qemu_rdma_accept()
3429 ret = qemu_rdma_alloc_qp(rdma); in qemu_rdma_accept()
3431 error_report("rdma migration: error allocating qp!"); in qemu_rdma_accept()
3435 qemu_rdma_init_ram_blocks(rdma); in qemu_rdma_accept()
3438 ret = qemu_rdma_reg_control(rdma, i); in qemu_rdma_accept()
3440 error_report("rdma: error registering %d control", i); in qemu_rdma_accept()
3447 && !rdma->is_return_path) { in qemu_rdma_accept()
3448 qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration, in qemu_rdma_accept()
3450 (void *)(intptr_t)rdma->return_path); in qemu_rdma_accept()
3452 qemu_set_fd_handler(rdma->channel->fd, rdma_cm_poll_handler, in qemu_rdma_accept()
3453 NULL, rdma); in qemu_rdma_accept()
3456 ret = rdma_accept(rdma->cm_id, &conn_param); in qemu_rdma_accept()
3462 ret = rdma_get_cm_event(rdma->channel, &cm_event); in qemu_rdma_accept()
3475 rdma->connected = true; in qemu_rdma_accept()
3477 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, &err); in qemu_rdma_accept()
3483 qemu_rdma_dump_gid("dest_connect", rdma->cm_id); in qemu_rdma_accept()
3488 rdma->errored = true; in qemu_rdma_accept()
3489 qemu_rdma_cleanup(rdma); in qemu_rdma_accept()
3505 * can perform RDMA operations.
3525 RDMAContext *rdma; in rdma_registration_handle() local
3543 rdma = qatomic_rcu_read(&rioc->rdmain); in rdma_registration_handle()
3545 if (!rdma) { in rdma_registration_handle()
3549 if (rdma_errored(rdma)) { in rdma_registration_handle()
3553 local = &rdma->local_ram_blocks; in rdma_registration_handle()
3557 ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_NONE, &err); in rdma_registration_handle()
3565 error_report("rdma: Too many requests in this message (%d)." in rdma_registration_handle()
3572 comp = (RDMACompress *) rdma->wr_data[idx].control_curr; in rdma_registration_handle()
3578 if (comp->block_idx >= rdma->local_ram_blocks.nb_blocks) { in rdma_registration_handle()
3579 error_report("rdma: 'compress' bad block index %u (vs %d)", in rdma_registration_handle()
3581 rdma->local_ram_blocks.nb_blocks); in rdma_registration_handle()
3584 block = &(rdma->local_ram_blocks.block[comp->block_idx]); in rdma_registration_handle()
3589 error_report("rdma: Zero page with non-zero (%d) value", in rdma_registration_handle()
3607 qsort(rdma->local_ram_blocks.block, in rdma_registration_handle()
3608 rdma->local_ram_blocks.nb_blocks, in rdma_registration_handle()
3614 if (rdma->pin_all) { in rdma_registration_handle()
3615 ret = qemu_rdma_reg_whole_ram_blocks(rdma, &err); in rdma_registration_handle()
3629 rdma->dest_blocks[i].remote_host_addr = in rdma_registration_handle()
3632 if (rdma->pin_all) { in rdma_registration_handle()
3633 rdma->dest_blocks[i].remote_rkey = local->block[i].mr->rkey; in rdma_registration_handle()
3636 rdma->dest_blocks[i].offset = local->block[i].offset; in rdma_registration_handle()
3637 rdma->dest_blocks[i].length = local->block[i].length; in rdma_registration_handle()
3639 dest_block_to_network(&rdma->dest_blocks[i]); in rdma_registration_handle()
3648 blocks.len = rdma->local_ram_blocks.nb_blocks in rdma_registration_handle()
3652 ret = qemu_rdma_post_send_control(rdma, in rdma_registration_handle()
3653 (uint8_t *) rdma->dest_blocks, &blocks, in rdma_registration_handle()
3666 registers = (RDMARegister *) rdma->wr_data[idx].control_curr; in rdma_registration_handle()
3680 if (reg->current_index >= rdma->local_ram_blocks.nb_blocks) { in rdma_registration_handle()
3681 error_report("rdma: 'register' bad block index %u (vs %d)", in rdma_registration_handle()
3683 rdma->local_ram_blocks.nb_blocks); in rdma_registration_handle()
3686 block = &(rdma->local_ram_blocks.block[reg->current_index]); in rdma_registration_handle()
3689 error_report("rdma: bad register address for block %s" in rdma_registration_handle()
3705 error_report("rdma: bad chunk for block %s" in rdma_registration_handle()
3715 if (qemu_rdma_register_and_get_keys(rdma, block, in rdma_registration_handle()
3730 ret = qemu_rdma_post_send_control(rdma, in rdma_registration_handle()
3741 registers = (RDMARegister *) rdma->wr_data[idx].control_curr; in rdma_registration_handle()
3750 block = &(rdma->local_ram_blocks.block[reg->current_index]); in rdma_registration_handle()
3756 error_report("rdma unregistration chunk failed: %s", in rdma_registration_handle()
3761 rdma->total_registrations--; in rdma_registration_handle()
3766 ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp, &err); in rdma_registration_handle()
3783 rdma->errored = true; in rdma_registration_handle()
3804 RDMAContext *rdma = qatomic_rcu_read(&rioc->rdmain); in rdma_block_notification_handle() local
3806 if (!rdma) { in rdma_block_notification_handle()
3811 for (curr = 0; curr < rdma->local_ram_blocks.nb_blocks; curr++) { in rdma_block_notification_handle()
3812 if (!strcmp(rdma->local_ram_blocks.block[curr].block_name, name)) { in rdma_block_notification_handle()
3823 rdma->local_ram_blocks.block[curr].src_index = rdma->next_src_index; in rdma_block_notification_handle()
3824 trace_rdma_block_notification_handle(name, rdma->next_src_index); in rdma_block_notification_handle()
3825 rdma->next_src_index++; in rdma_block_notification_handle()
3838 RDMAContext *rdma = qatomic_rcu_read(&rioc->rdmaout); in rdma_registration_start() local
3839 if (!rdma) { in rdma_registration_start()
3843 if (rdma_errored(rdma)) { in rdma_registration_start()
3860 RDMAContext *rdma; in rdma_registration_stop() local
3870 rdma = qatomic_rcu_read(&rioc->rdmaout); in rdma_registration_stop()
3871 if (!rdma) { in rdma_registration_stop()
3875 if (rdma_errored(rdma)) { in rdma_registration_stop()
3880 ret = qemu_rdma_drain_cq(rdma); in rdma_registration_stop()
3888 RDMALocalBlocks *local = &rdma->local_ram_blocks; in rdma_registration_stop()
3902 ret = qemu_rdma_exchange_send(rdma, &head, NULL, &resp, in rdma_registration_stop()
3903 ®_result_idx, rdma->pin_all ? in rdma_registration_stop()
3930 rdma->errored = true; in rdma_registration_stop()
3934 qemu_rdma_move_header(rdma, reg_result_idx, &resp); in rdma_registration_stop()
3935 memcpy(rdma->dest_blocks, in rdma_registration_stop()
3936 rdma->wr_data[reg_result_idx].control_curr, resp.len); in rdma_registration_stop()
3938 network_to_dest_block(&rdma->dest_blocks[i]); in rdma_registration_stop()
3941 if (rdma->dest_blocks[i].length != local->block[i].length) { in rdma_registration_stop()
3946 rdma->dest_blocks[i].length); in rdma_registration_stop()
3947 rdma->errored = true; in rdma_registration_stop()
3951 rdma->dest_blocks[i].remote_host_addr; in rdma_registration_stop()
3952 local->block[i].remote_rkey = rdma->dest_blocks[i].remote_rkey; in rdma_registration_stop()
3959 ret = qemu_rdma_exchange_send(rdma, &head, NULL, NULL, NULL, NULL, &err); in rdma_registration_stop()
3968 rdma->errored = true; in rdma_registration_stop()
4016 static QEMUFile *rdma_new_input(RDMAContext *rdma) in rdma_new_input() argument
4021 rioc->rdmain = rdma; in rdma_new_input()
4022 rioc->rdmaout = rdma->return_path; in rdma_new_input()
4027 static QEMUFile *rdma_new_output(RDMAContext *rdma) in rdma_new_output() argument
4032 rioc->rdmaout = rdma; in rdma_new_output()
4033 rioc->rdmain = rdma->return_path; in rdma_new_output()
4040 RDMAContext *rdma = opaque; in rdma_accept_incoming_migration() local
4044 if (qemu_rdma_accept(rdma) < 0) { in rdma_accept_incoming_migration()
4045 error_report("RDMA ERROR: Migration initialization failed"); in rdma_accept_incoming_migration()
4051 if (rdma->is_return_path) { in rdma_accept_incoming_migration()
4055 f = rdma_new_input(rdma); in rdma_accept_incoming_migration()
4057 error_report("RDMA ERROR: could not open RDMA for input"); in rdma_accept_incoming_migration()
4058 qemu_rdma_cleanup(rdma); in rdma_accept_incoming_migration()
4062 rdma->migration_started_on_destination = 1; in rdma_accept_incoming_migration()
4071 RDMAContext *rdma; in rdma_start_incoming_migration() local
4077 error_setg(errp, "RDMA: cannot disable RAM discard"); in rdma_start_incoming_migration()
4081 rdma = qemu_rdma_data_init(host_port, errp); in rdma_start_incoming_migration()
4082 if (rdma == NULL) { in rdma_start_incoming_migration()
4086 ret = qemu_rdma_dest_init(rdma, errp); in rdma_start_incoming_migration()
4093 ret = rdma_listen(rdma->listen_id, 5); in rdma_start_incoming_migration()
4096 error_setg(errp, "RDMA ERROR: listening on socket!"); in rdma_start_incoming_migration()
4102 qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration, in rdma_start_incoming_migration()
4103 NULL, (void *)(intptr_t)rdma); in rdma_start_incoming_migration()
4107 qemu_rdma_cleanup(rdma); in rdma_start_incoming_migration()
4109 if (rdma) { in rdma_start_incoming_migration()
4110 g_free(rdma->host); in rdma_start_incoming_migration()
4112 g_free(rdma); in rdma_start_incoming_migration()
4120 RDMAContext *rdma; in rdma_start_outgoing_migration() local
4125 error_setg(errp, "RDMA: cannot disable RAM discard"); in rdma_start_outgoing_migration()
4129 rdma = qemu_rdma_data_init(host_port, errp); in rdma_start_outgoing_migration()
4130 if (rdma == NULL) { in rdma_start_outgoing_migration()
4134 ret = qemu_rdma_source_init(rdma, migrate_rdma_pin_all(), errp); in rdma_start_outgoing_migration()
4141 ret = qemu_rdma_connect(rdma, false, errp); in rdma_start_outgoing_migration()
4147 /* RDMA postcopy need a separate queue pair for return path */ in rdma_start_outgoing_migration()
4168 rdma->return_path = rdma_return_path; in rdma_start_outgoing_migration()
4169 rdma_return_path->return_path = rdma; in rdma_start_outgoing_migration()
4175 s->to_dst_file = rdma_new_output(rdma); in rdma_start_outgoing_migration()
4180 qemu_rdma_cleanup(rdma); in rdma_start_outgoing_migration()
4182 g_free(rdma); in rdma_start_outgoing_migration()