73 	struct nvme_rdma_queue  *queue;  member
159 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)  in nvme_rdma_queue_idx()  argument
161 	return queue - queue->ctrl->queues;  in nvme_rdma_queue_idx()
164 static bool nvme_rdma_poll_queue(struct nvme_rdma_queue *queue)  in nvme_rdma_poll_queue()  argument
166 	return nvme_rdma_queue_idx(queue) >  in nvme_rdma_poll_queue()
167 		queue->ctrl->io_queues[HCTX_TYPE_DEFAULT] +  in nvme_rdma_poll_queue()
168 		queue->ctrl->io_queues[HCTX_TYPE_READ];  in nvme_rdma_poll_queue()
171 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)  in nvme_rdma_inline_data_size()  argument
173 	return queue->cmnd_capsule_len - sizeof(struct nvme_command);  in nvme_rdma_inline_data_size()
246 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)  in nvme_rdma_wait_for_cm()  argument
250 	ret = wait_for_completion_interruptible(&queue->cm_done);  in nvme_rdma_wait_for_cm()
253 	WARN_ON_ONCE(queue->cm_error > 0);  in nvme_rdma_wait_for_cm()
254 	return queue->cm_error;  in nvme_rdma_wait_for_cm()
257 static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)  in nvme_rdma_create_qp()  argument
259 	struct nvme_rdma_device *dev = queue->device;  in nvme_rdma_create_qp()
266 	init_attr.cap.max_send_wr = factor * queue->queue_size + 1;  in nvme_rdma_create_qp()
268 	init_attr.cap.max_recv_wr = queue->queue_size + 1;  in nvme_rdma_create_qp()
273 	init_attr.send_cq = queue->ib_cq;  in nvme_rdma_create_qp()
274 	init_attr.recv_cq = queue->ib_cq;  in nvme_rdma_create_qp()
275 	if (queue->pi_support)  in nvme_rdma_create_qp()
277 	init_attr.qp_context = queue;  in nvme_rdma_create_qp()
279 	ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);  in nvme_rdma_create_qp()
281 	queue->qp = queue->cm_id->qp;  in nvme_rdma_create_qp()
300 	struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];  in nvme_rdma_init_request()  local
308 	if (queue->pi_support)  in nvme_rdma_init_request()
313 	req->queue = queue;  in nvme_rdma_init_request()
323 	struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];  in nvme_rdma_init_hctx()  local
327 	hctx->driver_data = queue;  in nvme_rdma_init_hctx()
335 	struct nvme_rdma_queue *queue = &ctrl->queues[0];  in nvme_rdma_init_admin_hctx()  local
339 	hctx->driver_data = queue;  in nvme_rdma_init_admin_hctx()
413 static void nvme_rdma_free_cq(struct nvme_rdma_queue *queue)  in nvme_rdma_free_cq()  argument
415 	if (nvme_rdma_poll_queue(queue))  in nvme_rdma_free_cq()
416 		ib_free_cq(queue->ib_cq);  in nvme_rdma_free_cq()
418 		ib_cq_pool_put(queue->ib_cq, queue->cq_size);  in nvme_rdma_free_cq()
421 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)  in nvme_rdma_destroy_queue_ib()  argument
426 	if (!test_and_clear_bit(NVME_RDMA_Q_TR_READY, &queue->flags))  in nvme_rdma_destroy_queue_ib()
429 	dev = queue->device;  in nvme_rdma_destroy_queue_ib()
432 	if (queue->pi_support)  in nvme_rdma_destroy_queue_ib()
433 		ib_mr_pool_destroy(queue->qp, &queue->qp->sig_mrs);  in nvme_rdma_destroy_queue_ib()
434 	ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs);  in nvme_rdma_destroy_queue_ib()
441 	ib_destroy_qp(queue->qp);  in nvme_rdma_destroy_queue_ib()
442 	nvme_rdma_free_cq(queue);  in nvme_rdma_destroy_queue_ib()
444 	nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,  in nvme_rdma_destroy_queue_ib()
463 		struct nvme_rdma_queue *queue)  in nvme_rdma_create_cq()  argument
465 	int ret, comp_vector, idx = nvme_rdma_queue_idx(queue);  in nvme_rdma_create_cq()
474 	if (nvme_rdma_poll_queue(queue))  in nvme_rdma_create_cq()
475 		queue->ib_cq = ib_alloc_cq(ibdev, queue, queue->cq_size,  in nvme_rdma_create_cq()
478 		queue->ib_cq = ib_cq_pool_get(ibdev, queue->cq_size,  in nvme_rdma_create_cq()
481 	if (IS_ERR(queue->ib_cq)) {  in nvme_rdma_create_cq()
482 		ret = PTR_ERR(queue->ib_cq);  in nvme_rdma_create_cq()
489 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)  in nvme_rdma_create_queue_ib()  argument
496 	queue->device = nvme_rdma_find_get_device(queue->cm_id);  in nvme_rdma_create_queue_ib()
497 	if (!queue->device) {  in nvme_rdma_create_queue_ib()
498 		dev_err(queue->cm_id->device->dev.parent,  in nvme_rdma_create_queue_ib()
502 	ibdev = queue->device->dev;  in nvme_rdma_create_queue_ib()
505 	queue->cq_size = cq_factor * queue->queue_size + 1;  in nvme_rdma_create_queue_ib()
507 	ret = nvme_rdma_create_cq(ibdev, queue);  in nvme_rdma_create_queue_ib()
511 	ret = nvme_rdma_create_qp(queue, send_wr_factor);  in nvme_rdma_create_queue_ib()
515 	queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,  in nvme_rdma_create_queue_ib()
517 	if (!queue->rsp_ring) {  in nvme_rdma_create_queue_ib()
527 	pages_per_mr = nvme_rdma_get_max_fr_pages(ibdev, queue->pi_support) + 1;  in nvme_rdma_create_queue_ib()
528 	ret = ib_mr_pool_init(queue->qp, &queue->qp->rdma_mrs,  in nvme_rdma_create_queue_ib()
529 			      queue->queue_size,  in nvme_rdma_create_queue_ib()
533 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_create_queue_ib()
535 			queue->queue_size, nvme_rdma_queue_idx(queue));  in nvme_rdma_create_queue_ib()
539 	if (queue->pi_support) {  in nvme_rdma_create_queue_ib()
540 		ret = ib_mr_pool_init(queue->qp, &queue->qp->sig_mrs,  in nvme_rdma_create_queue_ib()
541 				      queue->queue_size, IB_MR_TYPE_INTEGRITY,  in nvme_rdma_create_queue_ib()
544 			dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_create_queue_ib()
546 				queue->queue_size, nvme_rdma_queue_idx(queue));  in nvme_rdma_create_queue_ib()
551 	set_bit(NVME_RDMA_Q_TR_READY, &queue->flags);  in nvme_rdma_create_queue_ib()
556 	ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs);  in nvme_rdma_create_queue_ib()
558 	nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,  in nvme_rdma_create_queue_ib()
561 	rdma_destroy_qp(queue->cm_id);  in nvme_rdma_create_queue_ib()
563 	nvme_rdma_free_cq(queue);  in nvme_rdma_create_queue_ib()
565 	nvme_rdma_dev_put(queue->device);  in nvme_rdma_create_queue_ib()
572 	struct nvme_rdma_queue *queue;  in nvme_rdma_alloc_queue()  local
576 	queue = &ctrl->queues[idx];  in nvme_rdma_alloc_queue()
577 	mutex_init(&queue->queue_lock);  in nvme_rdma_alloc_queue()
578 	queue->ctrl = ctrl;  in nvme_rdma_alloc_queue()
580 		queue->pi_support = true;  in nvme_rdma_alloc_queue()
582 		queue->pi_support = false;  in nvme_rdma_alloc_queue()
583 	init_completion(&queue->cm_done);  in nvme_rdma_alloc_queue()
586 		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;  in nvme_rdma_alloc_queue()
588 		queue->cmnd_capsule_len = sizeof(struct nvme_command);  in nvme_rdma_alloc_queue()
590 	queue->queue_size = queue_size;  in nvme_rdma_alloc_queue()
592 	queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,  in nvme_rdma_alloc_queue()
594 	if (IS_ERR(queue->cm_id)) {  in nvme_rdma_alloc_queue()
596 			"failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));  in nvme_rdma_alloc_queue()
597 		ret = PTR_ERR(queue->cm_id);  in nvme_rdma_alloc_queue()
604 	queue->cm_error = -ETIMEDOUT;  in nvme_rdma_alloc_queue()
605 	ret = rdma_resolve_addr(queue->cm_id, src_addr,  in nvme_rdma_alloc_queue()
614 	ret = nvme_rdma_wait_for_cm(queue);  in nvme_rdma_alloc_queue()
621 	set_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags);  in nvme_rdma_alloc_queue()
626 	rdma_destroy_id(queue->cm_id);  in nvme_rdma_alloc_queue()
627 	nvme_rdma_destroy_queue_ib(queue);  in nvme_rdma_alloc_queue()
629 	mutex_destroy(&queue->queue_lock);  in nvme_rdma_alloc_queue()
633 static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)  in __nvme_rdma_stop_queue()  argument
635 	rdma_disconnect(queue->cm_id);  in __nvme_rdma_stop_queue()
636 	ib_drain_qp(queue->qp);  in __nvme_rdma_stop_queue()
639 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)  in nvme_rdma_stop_queue()  argument
641 	if (!test_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))  in nvme_rdma_stop_queue()
644 	mutex_lock(&queue->queue_lock);  in nvme_rdma_stop_queue()
645 	if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))  in nvme_rdma_stop_queue()
646 		__nvme_rdma_stop_queue(queue);  in nvme_rdma_stop_queue()
647 	mutex_unlock(&queue->queue_lock);  in nvme_rdma_stop_queue()
650 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)  in nvme_rdma_free_queue()  argument
652 	if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))  in nvme_rdma_free_queue()
655 	rdma_destroy_id(queue->cm_id);  in nvme_rdma_free_queue()
656 	nvme_rdma_destroy_queue_ib(queue);  in nvme_rdma_free_queue()
657 	mutex_destroy(&queue->queue_lock);  in nvme_rdma_free_queue()
678 	struct nvme_rdma_queue *queue = &ctrl->queues[idx];  in nvme_rdma_start_queue()  local
687 		set_bit(NVME_RDMA_Q_LIVE, &queue->flags);  in nvme_rdma_start_queue()
689 		if (test_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))  in nvme_rdma_start_queue()
690 			__nvme_rdma_stop_queue(queue);  in nvme_rdma_start_queue()
1166 	struct nvme_rdma_queue *queue = wc->qp->qp_context;  in nvme_rdma_wr_error()  local
1167 	struct nvme_rdma_ctrl *ctrl = queue->ctrl;  in nvme_rdma_wr_error()
1194 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,  in nvme_rdma_inv_rkey()  argument
1208 	return ib_post_send(queue->qp, &wr, NULL);  in nvme_rdma_inv_rkey()
1227 static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,  in nvme_rdma_unmap_data()  argument
1231 	struct nvme_rdma_device *dev = queue->device;  in nvme_rdma_unmap_data()
1233 	struct list_head *pool = &queue->qp->rdma_mrs;  in nvme_rdma_unmap_data()
1239 		pool = &queue->qp->sig_mrs;  in nvme_rdma_unmap_data()
1242 		ib_mr_pool_put(queue->qp, pool, req->mr);  in nvme_rdma_unmap_data()
1260 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,  in nvme_rdma_map_sg_inline()  argument
1273 		sge->lkey = queue->device->pd->local_dma_lkey;  in nvme_rdma_map_sg_inline()
1278 	sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);  in nvme_rdma_map_sg_inline()
1286 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,  in nvme_rdma_map_sg_single()  argument
1293 	put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);  in nvme_rdma_map_sg_single()
1298 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,  in nvme_rdma_map_sg_fr()  argument
1305 	req->mr = ib_mr_pool_get(queue->qp, &queue->qp->rdma_mrs);  in nvme_rdma_map_sg_fr()
1316 		ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr);  in nvme_rdma_map_sg_fr()
1402 static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue,  in nvme_rdma_map_sg_pi()  argument
1414 	req->mr = ib_mr_pool_get(queue->qp, &queue->qp->sig_mrs);  in nvme_rdma_map_sg_pi()
1450 	ib_mr_pool_put(queue->qp, &queue->qp->sig_mrs, req->mr);  in nvme_rdma_map_sg_pi()
1517 static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,  in nvme_rdma_map_data()  argument
1521 	struct nvme_rdma_device *dev = queue->device;  in nvme_rdma_map_data()
1539 		ret = nvme_rdma_map_sg_pi(queue, req, c, count, pi_count);  in nvme_rdma_map_data()
1544 		if (rq_data_dir(rq) == WRITE && nvme_rdma_queue_idx(queue) &&  in nvme_rdma_map_data()
1545 		    queue->ctrl->use_inline_data &&  in nvme_rdma_map_data()
1547 				nvme_rdma_inline_data_size(queue)) {  in nvme_rdma_map_data()
1548 			ret = nvme_rdma_map_sg_inline(queue, req, c, count);  in nvme_rdma_map_data()
1553 			ret = nvme_rdma_map_sg_single(queue, req, c);  in nvme_rdma_map_data()
1558 	ret = nvme_rdma_map_sg_fr(queue, req, c, count);  in nvme_rdma_map_data()
1583 static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,  in nvme_rdma_post_send()  argument
1592 	sge->lkey   = queue->device->pd->local_dma_lkey;  in nvme_rdma_post_send()
1606 	ret = ib_post_send(queue->qp, first, NULL);  in nvme_rdma_post_send()
1608 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_post_send()
1614 static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,  in nvme_rdma_post_recv()  argument
1623 	list.lkey   = queue->device->pd->local_dma_lkey;  in nvme_rdma_post_recv()
1632 	ret = ib_post_recv(queue->qp, &wr, NULL);  in nvme_rdma_post_recv()
1634 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_post_recv()
1640 static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)  in nvme_rdma_tagset()  argument
1642 	u32 queue_idx = nvme_rdma_queue_idx(queue);  in nvme_rdma_tagset()
1645 		return queue->ctrl->admin_tag_set.tags[queue_idx];  in nvme_rdma_tagset()
1646 	return queue->ctrl->tag_set.tags[queue_idx - 1];  in nvme_rdma_tagset()
1658 	struct nvme_rdma_queue *queue = &ctrl->queues[0];  in nvme_rdma_submit_async_event()  local
1659 	struct ib_device *dev = queue->device->dev;  in nvme_rdma_submit_async_event()
1678 	ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL);  in nvme_rdma_submit_async_event()
1682 static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,  in nvme_rdma_process_nvme_rsp()  argument
1688 	rq = nvme_find_rq(nvme_rdma_tagset(queue), cqe->command_id);  in nvme_rdma_process_nvme_rsp()
1690 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_process_nvme_rsp()
1692 			cqe->command_id, queue->qp->qp_num);  in nvme_rdma_process_nvme_rsp()
1693 		nvme_rdma_error_recovery(queue->ctrl);  in nvme_rdma_process_nvme_rsp()
1704 			dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_process_nvme_rsp()
1707 			nvme_rdma_error_recovery(queue->ctrl);  in nvme_rdma_process_nvme_rsp()
1712 		ret = nvme_rdma_inv_rkey(queue, req);  in nvme_rdma_process_nvme_rsp()
1714 			dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_process_nvme_rsp()
1717 			nvme_rdma_error_recovery(queue->ctrl);  in nvme_rdma_process_nvme_rsp()
1730 	struct nvme_rdma_queue *queue = wc->qp->qp_context;  in nvme_rdma_recv_done()  local
1731 	struct ib_device *ibdev = queue->device->dev;  in nvme_rdma_recv_done()
1742 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_recv_done()
1744 		nvme_rdma_error_recovery(queue->ctrl);  in nvme_rdma_recv_done()
1755 	if (unlikely(nvme_is_aen_req(nvme_rdma_queue_idx(queue),  in nvme_rdma_recv_done()
1757 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,  in nvme_rdma_recv_done()
1760 		nvme_rdma_process_nvme_rsp(queue, cqe, wc);  in nvme_rdma_recv_done()
1763 	nvme_rdma_post_recv(queue, qe);  in nvme_rdma_recv_done()
1766 static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)  in nvme_rdma_conn_established()  argument
1770 	for (i = 0; i < queue->queue_size; i++) {  in nvme_rdma_conn_established()
1771 		ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);  in nvme_rdma_conn_established()
1779 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,  in nvme_rdma_conn_rejected()  argument
1782 	struct rdma_cm_id *cm_id = queue->cm_id;  in nvme_rdma_conn_rejected()
1794 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_conn_rejected()
1798 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_conn_rejected()
1805 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)  in nvme_rdma_addr_resolved()  argument
1807 	struct nvme_ctrl *ctrl = &queue->ctrl->ctrl;  in nvme_rdma_addr_resolved()
1810 	ret = nvme_rdma_create_queue_ib(queue);  in nvme_rdma_addr_resolved()
1815 		rdma_set_service_type(queue->cm_id, ctrl->opts->tos);  in nvme_rdma_addr_resolved()
1816 	ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CM_TIMEOUT_MS);  in nvme_rdma_addr_resolved()
1819 			queue->cm_error);  in nvme_rdma_addr_resolved()
1826 	nvme_rdma_destroy_queue_ib(queue);  in nvme_rdma_addr_resolved()
1830 static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)  in nvme_rdma_route_resolved()  argument
1832 	struct nvme_rdma_ctrl *ctrl = queue->ctrl;  in nvme_rdma_route_resolved()
1837 	param.qp_num = queue->qp->qp_num;  in nvme_rdma_route_resolved()
1840 	param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;  in nvme_rdma_route_resolved()
1848 	priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));  in nvme_rdma_route_resolved()
1862 		priv.hrqsize = cpu_to_le16(queue->queue_size);  in nvme_rdma_route_resolved()
1863 		priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);  in nvme_rdma_route_resolved()
1866 	ret = rdma_connect_locked(queue->cm_id, &param);  in nvme_rdma_route_resolved()
1879 	struct nvme_rdma_queue *queue = cm_id->context;  in nvme_rdma_cm_handler()  local
1882 	dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",  in nvme_rdma_cm_handler()
1888 		cm_error = nvme_rdma_addr_resolved(queue);  in nvme_rdma_cm_handler()
1891 		cm_error = nvme_rdma_route_resolved(queue);  in nvme_rdma_cm_handler()
1894 		queue->cm_error = nvme_rdma_conn_established(queue);  in nvme_rdma_cm_handler()
1896 		complete(&queue->cm_done);  in nvme_rdma_cm_handler()
1899 		cm_error = nvme_rdma_conn_rejected(queue, ev);  in nvme_rdma_cm_handler()
1905 		dev_dbg(queue->ctrl->ctrl.device,  in nvme_rdma_cm_handler()
1912 		dev_dbg(queue->ctrl->ctrl.device,  in nvme_rdma_cm_handler()
1914 		nvme_rdma_error_recovery(queue->ctrl);  in nvme_rdma_cm_handler()
1920 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_cm_handler()
1922 		nvme_rdma_error_recovery(queue->ctrl);  in nvme_rdma_cm_handler()
1927 		queue->cm_error = cm_error;  in nvme_rdma_cm_handler()
1928 		complete(&queue->cm_done);  in nvme_rdma_cm_handler()
1937 	struct nvme_rdma_queue *queue = req->queue;  in nvme_rdma_complete_timed_out()  local
1939 	nvme_rdma_stop_queue(queue);  in nvme_rdma_complete_timed_out()
1946 	struct nvme_rdma_queue *queue = req->queue;  in nvme_rdma_timeout()  local
1947 	struct nvme_rdma_ctrl *ctrl = queue->ctrl;  in nvme_rdma_timeout()
1950 		 rq->tag, nvme_rdma_queue_idx(queue));  in nvme_rdma_timeout()
1981 	struct nvme_ns *ns = hctx->queue->queuedata;  in nvme_rdma_queue_rq()
1982 	struct nvme_rdma_queue *queue = hctx->driver_data;  in nvme_rdma_queue_rq()  local
1988 	bool queue_ready = test_bit(NVME_RDMA_Q_LIVE, &queue->flags);  in nvme_rdma_queue_rq()
1994 	if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))  in nvme_rdma_queue_rq()
1995 		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);  in nvme_rdma_queue_rq()
1997 	dev = queue->device->dev;  in nvme_rdma_queue_rq()
2016 	    queue->pi_support &&  in nvme_rdma_queue_rq()
2024 	err = nvme_rdma_map_data(queue, rq, c);  in nvme_rdma_queue_rq()
2026 		dev_err(queue->ctrl->ctrl.device,  in nvme_rdma_queue_rq()
2036 	err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,  in nvme_rdma_queue_rq()
2044 	nvme_rdma_unmap_data(queue, rq);  in nvme_rdma_queue_rq()
2061 	struct nvme_rdma_queue *queue = hctx->driver_data;  in nvme_rdma_poll()  local
2063 	return ib_process_cq_direct(queue->ib_cq, -1);  in nvme_rdma_poll()
2100 	struct nvme_rdma_queue *queue = req->queue;  in nvme_rdma_complete_rq()  local
2101 	struct ib_device *ibdev = queue->device->dev;  in nvme_rdma_complete_rq()
2106 	nvme_rdma_unmap_data(queue, rq);  in nvme_rdma_complete_rq()