Lines Matching refs:backend_dev
76 static void clean_recv_mads(RdmaBackendDev *backend_dev) in clean_recv_mads() argument
81 cqe_ctx_id = rdma_protected_gqueue_pop_int64(&backend_dev-> in clean_recv_mads()
84 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe); in clean_recv_mads()
86 backend_dev->rdma_dev_res); in clean_recv_mads()
140 RdmaBackendDev *backend_dev = (RdmaBackendDev *)arg; in comp_handler_thread() local
148 flags = fcntl(backend_dev->channel->fd, F_GETFL); in comp_handler_thread()
149 rc = fcntl(backend_dev->channel->fd, F_SETFL, flags | O_NONBLOCK); in comp_handler_thread()
155 pfds[0].fd = backend_dev->channel->fd; in comp_handler_thread()
158 backend_dev->comp_thread.is_running = true; in comp_handler_thread()
160 while (backend_dev->comp_thread.run) { in comp_handler_thread()
164 backend_dev->rdma_dev_res->stats.poll_cq_ppoll_to++; in comp_handler_thread()
166 } while (!rc && backend_dev->comp_thread.run); in comp_handler_thread()
168 if (backend_dev->comp_thread.run) { in comp_handler_thread()
169 rc = ibv_get_cq_event(backend_dev->channel, &ev_cq, &ev_ctx); in comp_handler_thread()
182 backend_dev->rdma_dev_res->stats.poll_cq_from_bk++; in comp_handler_thread()
183 rdma_poll_cq(backend_dev->rdma_dev_res, ev_cq); in comp_handler_thread()
189 backend_dev->comp_thread.is_running = false; in comp_handler_thread()
196 static inline void disable_rdmacm_mux_async(RdmaBackendDev *backend_dev) in disable_rdmacm_mux_async() argument
198 qatomic_set(&backend_dev->rdmacm_mux.can_receive, 0); in disable_rdmacm_mux_async()
201 static inline void enable_rdmacm_mux_async(RdmaBackendDev *backend_dev) in enable_rdmacm_mux_async() argument
203 qatomic_set(&backend_dev->rdmacm_mux.can_receive, sizeof(RdmaCmMuxMsg)); in enable_rdmacm_mux_async()
206 static inline int rdmacm_mux_can_process_async(RdmaBackendDev *backend_dev) in rdmacm_mux_can_process_async() argument
208 return qatomic_read(&backend_dev->rdmacm_mux.can_receive); in rdmacm_mux_can_process_async()
240 static int rdmacm_mux_send(RdmaBackendDev *backend_dev, RdmaCmMuxMsg *msg) in rdmacm_mux_send() argument
246 disable_rdmacm_mux_async(backend_dev); in rdmacm_mux_send()
247 rc = qemu_chr_fe_write(backend_dev->rdmacm_mux.chr_be, in rdmacm_mux_send()
250 enable_rdmacm_mux_async(backend_dev); in rdmacm_mux_send()
255 rc = rdmacm_mux_check_op_status(backend_dev->rdmacm_mux.chr_be); in rdmacm_mux_send()
261 enable_rdmacm_mux_async(backend_dev); in rdmacm_mux_send()
274 static void start_comp_thread(RdmaBackendDev *backend_dev) in start_comp_thread() argument
278 stop_backend_thread(&backend_dev->comp_thread); in start_comp_thread()
281 ibv_get_device_name(backend_dev->ib_dev)); in start_comp_thread()
282 backend_dev->comp_thread.run = true; in start_comp_thread()
283 qemu_thread_create(&backend_dev->comp_thread.thread, thread_name, in start_comp_thread()
284 comp_handler_thread, backend_dev, QEMU_THREAD_DETACHED); in start_comp_thread()
298 int rdma_backend_query_port(RdmaBackendDev *backend_dev, in rdma_backend_query_port() argument
303 rc = ibv_query_port(backend_dev->context, backend_dev->port_num, port_attr); in rdma_backend_query_port()
325 static struct ibv_ah *create_ah(RdmaBackendDev *backend_dev, struct ibv_pd *pd, in create_ah() argument
338 .port_num = backend_dev->port_num, in create_ah()
433 static int mad_send(RdmaBackendDev *backend_dev, uint8_t sgid_idx, in mad_send() argument
459 hdr = rdma_pci_dma_map(backend_dev->dev, sge[0].addr, sge[0].length); in mad_send()
463 data = rdma_pci_dma_map(backend_dev->dev, sge[1].addr, sge[1].length); in mad_send()
465 rdma_pci_dma_unmap(backend_dev->dev, hdr, sge[0].length); in mad_send()
472 rdma_pci_dma_unmap(backend_dev->dev, data, sge[1].length); in mad_send()
473 rdma_pci_dma_unmap(backend_dev->dev, hdr, sge[0].length); in mad_send()
477 ret = rdmacm_mux_send(backend_dev, &msg); in mad_send()
486 void rdma_backend_post_send(RdmaBackendDev *backend_dev, in rdma_backend_post_send() argument
503 rc = mad_send(backend_dev, sgid_idx, sgid, sge, num_sge); in rdma_backend_post_send()
506 backend_dev->rdma_dev_res->stats.mad_tx_err++; in rdma_backend_post_send()
509 backend_dev->rdma_dev_res->stats.mad_tx++; in rdma_backend_post_send()
519 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx); in rdma_backend_post_send()
527 rc = build_host_sge_array(backend_dev->rdma_dev_res, sge, num_sge, in rdma_backend_post_send()
528 &backend_dev->rdma_dev_res->stats.tx_len); in rdma_backend_post_send()
535 wr.wr.ud.ah = create_ah(backend_dev, qp->ibpd, sgid_idx, dgid); in rdma_backend_post_send()
558 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe); in rdma_backend_post_send()
559 backend_dev->rdma_dev_res->stats.tx++; in rdma_backend_post_send()
564 backend_dev->rdma_dev_res->stats.tx_err++; in rdma_backend_post_send()
565 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id); in rdma_backend_post_send()
571 static unsigned int save_mad_recv_buffer(RdmaBackendDev *backend_dev, in save_mad_recv_buffer() argument
591 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx); in save_mad_recv_buffer()
600 rdma_protected_gqueue_append_int64(&backend_dev->recv_mads_list, bctx_id); in save_mad_recv_buffer()
605 void rdma_backend_post_recv(RdmaBackendDev *backend_dev, in rdma_backend_post_recv() argument
620 rc = save_mad_recv_buffer(backend_dev, sge, num_sge, ctx); in rdma_backend_post_recv()
623 backend_dev->rdma_dev_res->stats.mad_rx_bufs_err++; in rdma_backend_post_recv()
625 backend_dev->rdma_dev_res->stats.mad_rx_bufs++; in rdma_backend_post_recv()
635 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx); in rdma_backend_post_recv()
643 rc = build_host_sge_array(backend_dev->rdma_dev_res, sge, num_sge, in rdma_backend_post_recv()
644 &backend_dev->rdma_dev_res->stats.rx_bufs_len); in rdma_backend_post_recv()
661 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe); in rdma_backend_post_recv()
662 backend_dev->rdma_dev_res->stats.rx_bufs++; in rdma_backend_post_recv()
667 backend_dev->rdma_dev_res->stats.rx_bufs_err++; in rdma_backend_post_recv()
668 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id); in rdma_backend_post_recv()
674 void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev, in rdma_backend_post_srq_recv() argument
687 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx); in rdma_backend_post_srq_recv()
695 rc = build_host_sge_array(backend_dev->rdma_dev_res, sge, num_sge, in rdma_backend_post_srq_recv()
696 &backend_dev->rdma_dev_res->stats.rx_bufs_len); in rdma_backend_post_srq_recv()
713 qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe); in rdma_backend_post_srq_recv()
714 backend_dev->rdma_dev_res->stats.rx_bufs++; in rdma_backend_post_srq_recv()
715 backend_dev->rdma_dev_res->stats.rx_srq++; in rdma_backend_post_srq_recv()
720 backend_dev->rdma_dev_res->stats.rx_bufs_err++; in rdma_backend_post_srq_recv()
721 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id); in rdma_backend_post_srq_recv()
727 int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd) in rdma_backend_create_pd() argument
729 pd->ibpd = ibv_alloc_pd(backend_dev->context); in rdma_backend_create_pd()
771 int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq, in rdma_backend_create_cq() argument
776 cq->ibcq = ibv_create_cq(backend_dev->context, cqe + 1, NULL, in rdma_backend_create_cq()
777 backend_dev->channel, 0); in rdma_backend_create_cq()
788 cq->backend_dev = backend_dev; in rdma_backend_create_cq()
851 int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp, in rdma_backend_qp_state_init() argument
860 attr.port_num = backend_dev->port_num; in rdma_backend_qp_state_init()
888 int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp, in rdma_backend_qp_state_rtr() argument
911 attr.ah_attr.port_num = backend_dev->port_num; in rdma_backend_qp_state_rtr()
1068 static int init_device_caps(RdmaBackendDev *backend_dev, in init_device_caps() argument
1074 rc = ibv_query_device(backend_dev->context, &bk_dev_attr); in init_device_caps()
1105 static void process_incoming_mad_req(RdmaBackendDev *backend_dev, in process_incoming_mad_req() argument
1114 cqe_ctx_id = rdma_protected_gqueue_pop_int64(&backend_dev->recv_mads_list); in process_incoming_mad_req()
1121 bctx = rdma_rm_get_cqe_ctx(backend_dev->rdma_dev_res, cqe_ctx_id); in process_incoming_mad_req()
1124 backend_dev->rdma_dev_res->stats.mad_rx_err++; in process_incoming_mad_req()
1128 mad = rdma_pci_dma_map(backend_dev->dev, bctx->sge.addr, in process_incoming_mad_req()
1131 backend_dev->rdma_dev_res->stats.mad_rx_err++; in process_incoming_mad_req()
1141 rdma_pci_dma_unmap(backend_dev->dev, mad, bctx->sge.length); in process_incoming_mad_req()
1146 backend_dev->rdma_dev_res->stats.mad_rx++; in process_incoming_mad_req()
1151 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, cqe_ctx_id); in process_incoming_mad_req()
1156 RdmaBackendDev *backend_dev = (RdmaBackendDev *)opaque; in rdmacm_mux_can_receive() local
1158 return rdmacm_mux_can_process_async(backend_dev); in rdmacm_mux_can_receive()
1163 RdmaBackendDev *backend_dev = (RdmaBackendDev *)opaque; in rdmacm_mux_read() local
1173 process_incoming_mad_req(backend_dev, msg); in rdmacm_mux_read()
1176 static int mad_init(RdmaBackendDev *backend_dev, CharBackend *mad_chr_be) in mad_init() argument
1180 backend_dev->rdmacm_mux.chr_be = mad_chr_be; in mad_init()
1182 ret = qemu_chr_fe_backend_connected(backend_dev->rdmacm_mux.chr_be); in mad_init()
1188 rdma_protected_gqueue_init(&backend_dev->recv_mads_list); in mad_init()
1190 enable_rdmacm_mux_async(backend_dev); in mad_init()
1192 qemu_chr_fe_set_handlers(backend_dev->rdmacm_mux.chr_be, in mad_init()
1194 NULL, backend_dev, NULL, true); in mad_init()
1199 static void mad_stop(RdmaBackendDev *backend_dev) in mad_stop() argument
1201 clean_recv_mads(backend_dev); in mad_stop()
1204 static void mad_fini(RdmaBackendDev *backend_dev) in mad_fini() argument
1206 disable_rdmacm_mux_async(backend_dev); in mad_fini()
1207 qemu_chr_fe_disconnect(backend_dev->rdmacm_mux.chr_be); in mad_fini()
1208 rdma_protected_gqueue_destroy(&backend_dev->recv_mads_list); in mad_fini()
1211 int rdma_backend_get_gid_index(RdmaBackendDev *backend_dev, in rdma_backend_get_gid_index() argument
1219 ret = ibv_query_gid(backend_dev->context, backend_dev->port_num, i, in rdma_backend_get_gid_index()
1231 int rdma_backend_add_gid(RdmaBackendDev *backend_dev, const char *ifname, in rdma_backend_add_gid() argument
1243 ret = rdmacm_mux_send(backend_dev, &msg); in rdma_backend_add_gid()
1256 int rdma_backend_del_gid(RdmaBackendDev *backend_dev, const char *ifname, in rdma_backend_del_gid() argument
1268 ret = rdmacm_mux_send(backend_dev, &msg); in rdma_backend_del_gid()
1282 int rdma_backend_init(RdmaBackendDev *backend_dev, PCIDevice *pdev, in rdma_backend_init() argument
1292 memset(backend_dev, 0, sizeof(*backend_dev)); in rdma_backend_init()
1294 backend_dev->dev = pdev; in rdma_backend_init()
1295 backend_dev->port_num = port_num; in rdma_backend_init()
1296 backend_dev->rdma_dev_res = rdma_dev_res; in rdma_backend_init()
1320 backend_dev->ib_dev = dev_list[i]; in rdma_backend_init()
1321 if (!backend_dev->ib_dev) { in rdma_backend_init()
1328 backend_dev->ib_dev = *dev_list; in rdma_backend_init()
1331 rdma_info_report("uverb device %s", backend_dev->ib_dev->dev_name); in rdma_backend_init()
1333 backend_dev->context = ibv_open_device(backend_dev->ib_dev); in rdma_backend_init()
1334 if (!backend_dev->context) { in rdma_backend_init()
1336 ibv_get_device_name(backend_dev->ib_dev)); in rdma_backend_init()
1341 backend_dev->channel = ibv_create_comp_channel(backend_dev->context); in rdma_backend_init()
1342 if (!backend_dev->channel) { in rdma_backend_init()
1348 ret = init_device_caps(backend_dev, dev_attr); in rdma_backend_init()
1356 ret = mad_init(backend_dev, mad_chr_be); in rdma_backend_init()
1363 backend_dev->comp_thread.run = false; in rdma_backend_init()
1364 backend_dev->comp_thread.is_running = false; in rdma_backend_init()
1371 ibv_destroy_comp_channel(backend_dev->channel); in rdma_backend_init()
1374 ibv_close_device(backend_dev->context); in rdma_backend_init()
1384 void rdma_backend_start(RdmaBackendDev *backend_dev) in rdma_backend_start() argument
1386 start_comp_thread(backend_dev); in rdma_backend_start()
1389 void rdma_backend_stop(RdmaBackendDev *backend_dev) in rdma_backend_stop() argument
1391 mad_stop(backend_dev); in rdma_backend_stop()
1392 stop_backend_thread(&backend_dev->comp_thread); in rdma_backend_stop()
1395 void rdma_backend_fini(RdmaBackendDev *backend_dev) in rdma_backend_fini() argument
1397 mad_fini(backend_dev); in rdma_backend_fini()
1399 ibv_destroy_comp_channel(backend_dev->channel); in rdma_backend_fini()
1400 ibv_close_device(backend_dev->context); in rdma_backend_fini()