Lines Matching refs:ic
59 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_set_flow_control() local
63 ic->i_flowctl = 1; in rds_ib_set_flow_control()
66 ic->i_flowctl = 0; in rds_ib_set_flow_control()
76 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_connect_complete() local
124 ic->i_active_side ? "Active" : "Passive", in rds_ib_cm_connect_complete()
128 ic->i_flowctl ? ", flow control" : ""); in rds_ib_cm_connect_complete()
131 ic->i_sl = ic->i_cm_id->route.path_rec->sl; in rds_ib_cm_connect_complete()
133 atomic_set(&ic->i_cq_quiesce, 0); in rds_ib_cm_connect_complete()
139 rds_ib_send_init_ring(ic); in rds_ib_cm_connect_complete()
140 rds_ib_recv_init_ring(ic); in rds_ib_cm_connect_complete()
146 err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr); in rds_ib_cm_connect_complete()
171 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_fill_conn_param() local
172 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_cm_fill_conn_param()
195 cpu_to_be64(rds_ib_piggyb_ack(ic)); in rds_ib_cm_fill_conn_param()
210 cpu_to_be64(rds_ib_piggyb_ack(ic)); in rds_ib_cm_fill_conn_param()
218 if (ic->i_flowctl) { in rds_ib_cm_fill_conn_param()
222 (atomic_read(&ic->i_credits)); in rds_ib_cm_fill_conn_param()
228 &ic->i_credits); in rds_ib_cm_fill_conn_param()
250 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_recv() local
256 tasklet_schedule(&ic->i_recv_tasklet); in rds_ib_cq_comp_handler_recv()
259 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq, in poll_scq() argument
272 if (wc->wr_id <= ic->i_send_ring.w_nr || in poll_scq()
274 rds_ib_send_cqe_handler(ic, wc); in poll_scq()
276 rds_ib_mr_cqe_handler(ic, wc); in poll_scq()
284 struct rds_ib_connection *ic = (struct rds_ib_connection *)data; in rds_ib_tasklet_fn_send() local
285 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_send()
290 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_send()
293 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
294 ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_tasklet_fn_send()
295 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
300 rds_send_xmit(&ic->conn->c_path[0]); in rds_ib_tasklet_fn_send()
303 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq, in poll_rcq() argument
317 rds_ib_recv_cqe_handler(ic, wc, ack_state); in poll_rcq()
324 struct rds_ib_connection *ic = (struct rds_ib_connection *)data; in rds_ib_tasklet_fn_recv() local
325 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_recv()
326 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_tasklet_fn_recv()
335 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_recv()
339 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
340 ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_tasklet_fn_recv()
341 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
344 rds_ib_set_ack(ic, state.ack_next, state.ack_required); in rds_ib_tasklet_fn_recv()
345 if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) { in rds_ib_tasklet_fn_recv()
347 ic->i_ack_recv = state.ack_recv; in rds_ib_tasklet_fn_recv()
351 rds_ib_attempt_ack(ic); in rds_ib_tasklet_fn_recv()
357 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_qp_event_handler() local
359 rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event, in rds_ib_qp_event_handler()
364 rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST); in rds_ib_qp_event_handler()
378 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_send() local
384 tasklet_schedule(&ic->i_send_tasklet); in rds_ib_cq_comp_handler_send()
504 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_setup_qp() local
505 struct ib_device *dev = ic->i_cm_id->device; in rds_ib_setup_qp()
531 if (ic->i_send_ring.w_nr != max_wrs) in rds_ib_setup_qp()
532 rds_ib_ring_resize(&ic->i_send_ring, max_wrs); in rds_ib_setup_qp()
536 if (ic->i_recv_ring.w_nr != max_wrs) in rds_ib_setup_qp()
537 rds_ib_ring_resize(&ic->i_recv_ring, max_wrs); in rds_ib_setup_qp()
540 ic->i_pd = rds_ibdev->pd; in rds_ib_setup_qp()
542 ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
543 cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
544 cq_attr.comp_vector = ic->i_scq_vector; in rds_ib_setup_qp()
545 ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send, in rds_ib_setup_qp()
548 if (IS_ERR(ic->i_send_cq)) { in rds_ib_setup_qp()
549 ret = PTR_ERR(ic->i_send_cq); in rds_ib_setup_qp()
550 ic->i_send_cq = NULL; in rds_ib_setup_qp()
551 ibdev_put_vector(rds_ibdev, ic->i_scq_vector); in rds_ib_setup_qp()
556 ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
557 cq_attr.cqe = ic->i_recv_ring.w_nr; in rds_ib_setup_qp()
558 cq_attr.comp_vector = ic->i_rcq_vector; in rds_ib_setup_qp()
559 ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv, in rds_ib_setup_qp()
562 if (IS_ERR(ic->i_recv_cq)) { in rds_ib_setup_qp()
563 ret = PTR_ERR(ic->i_recv_cq); in rds_ib_setup_qp()
564 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
565 ibdev_put_vector(rds_ibdev, ic->i_rcq_vector); in rds_ib_setup_qp()
570 ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_setup_qp()
576 ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_setup_qp()
587 attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
588 attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1; in rds_ib_setup_qp()
593 attr.send_cq = ic->i_send_cq; in rds_ib_setup_qp()
594 attr.recv_cq = ic->i_recv_cq; in rds_ib_setup_qp()
600 ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr); in rds_ib_setup_qp()
606 ic->i_send_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_send_hdrs_dma, in rds_ib_setup_qp()
607 ic->i_send_ring.w_nr, in rds_ib_setup_qp()
609 if (!ic->i_send_hdrs) { in rds_ib_setup_qp()
615 ic->i_recv_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
616 ic->i_recv_ring.w_nr, in rds_ib_setup_qp()
618 if (!ic->i_recv_hdrs) { in rds_ib_setup_qp()
624 ic->i_ack = rds_dma_hdr_alloc(rds_ibdev->dev, &ic->i_ack_dma, in rds_ib_setup_qp()
626 if (!ic->i_ack) { in rds_ib_setup_qp()
632 ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work), in rds_ib_setup_qp()
633 ic->i_send_ring.w_nr), in rds_ib_setup_qp()
635 if (!ic->i_sends) { in rds_ib_setup_qp()
641 ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work), in rds_ib_setup_qp()
642 ic->i_recv_ring.w_nr), in rds_ib_setup_qp()
644 if (!ic->i_recvs) { in rds_ib_setup_qp()
650 rds_ib_recv_init_ack(ic); in rds_ib_setup_qp()
652 rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd, in rds_ib_setup_qp()
653 ic->i_send_cq, ic->i_recv_cq); in rds_ib_setup_qp()
658 vfree(ic->i_sends); in rds_ib_setup_qp()
661 rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma, in rds_ib_setup_qp()
663 ic->i_ack = NULL; in rds_ib_setup_qp()
666 rds_dma_hdrs_free(rds_ibdev, ic->i_recv_hdrs, ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
667 ic->i_recv_ring.w_nr, DMA_FROM_DEVICE); in rds_ib_setup_qp()
668 ic->i_recv_hdrs = NULL; in rds_ib_setup_qp()
669 ic->i_recv_hdrs_dma = NULL; in rds_ib_setup_qp()
672 rds_dma_hdrs_free(rds_ibdev, ic->i_send_hdrs, ic->i_send_hdrs_dma, in rds_ib_setup_qp()
673 ic->i_send_ring.w_nr, DMA_TO_DEVICE); in rds_ib_setup_qp()
674 ic->i_send_hdrs = NULL; in rds_ib_setup_qp()
675 ic->i_send_hdrs_dma = NULL; in rds_ib_setup_qp()
678 rdma_destroy_qp(ic->i_cm_id); in rds_ib_setup_qp()
680 ib_destroy_cq(ic->i_recv_cq); in rds_ib_setup_qp()
681 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
683 ib_destroy_cq(ic->i_send_cq); in rds_ib_setup_qp()
684 ic->i_send_cq = NULL; in rds_ib_setup_qp()
789 struct rds_ib_connection *ic = NULL; in rds_ib_cm_handle_connect() local
885 ic = conn->c_transport_data; in rds_ib_cm_handle_connect()
897 BUG_ON(ic->i_cm_id); in rds_ib_cm_handle_connect()
899 ic->i_cm_id = cm_id; in rds_ib_cm_handle_connect()
934 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_initiate_connect() local
942 ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */ in rds_ib_cm_initiate_connect()
963 if (ic->i_cm_id == cm_id) in rds_ib_cm_initiate_connect()
966 ic->i_active_side = true; in rds_ib_cm_initiate_connect()
975 struct rds_ib_connection *ic; in rds_ib_conn_path_connect() local
978 ic = conn->c_transport_data; in rds_ib_conn_path_connect()
988 ic->i_cm_id = rdma_create_id(&init_net, handler, conn, in rds_ib_conn_path_connect()
990 if (IS_ERR(ic->i_cm_id)) { in rds_ib_conn_path_connect()
991 ret = PTR_ERR(ic->i_cm_id); in rds_ib_conn_path_connect()
992 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
997 rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn); in rds_ib_conn_path_connect()
1027 ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src, in rds_ib_conn_path_connect()
1031 rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id, in rds_ib_conn_path_connect()
1033 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_connect()
1034 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
1049 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_conn_path_shutdown() local
1052 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id, in rds_ib_conn_path_shutdown()
1053 ic->i_pd, ic->i_send_cq, ic->i_recv_cq, in rds_ib_conn_path_shutdown()
1054 ic->i_cm_id ? ic->i_cm_id->qp : NULL); in rds_ib_conn_path_shutdown()
1056 if (ic->i_cm_id) { in rds_ib_conn_path_shutdown()
1057 rdsdebug("disconnecting cm %p\n", ic->i_cm_id); in rds_ib_conn_path_shutdown()
1058 err = rdma_disconnect(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1064 ic->i_cm_id, err); in rds_ib_conn_path_shutdown()
1082 rds_ib_ring_empty(&ic->i_recv_ring) && in rds_ib_conn_path_shutdown()
1083 (atomic_read(&ic->i_signaled_sends) == 0) && in rds_ib_conn_path_shutdown()
1084 (atomic_read(&ic->i_fastreg_inuse_count) == 0) && in rds_ib_conn_path_shutdown()
1085 (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR)); in rds_ib_conn_path_shutdown()
1086 tasklet_kill(&ic->i_send_tasklet); in rds_ib_conn_path_shutdown()
1087 tasklet_kill(&ic->i_recv_tasklet); in rds_ib_conn_path_shutdown()
1089 atomic_set(&ic->i_cq_quiesce, 1); in rds_ib_conn_path_shutdown()
1092 if (ic->i_cm_id->qp) in rds_ib_conn_path_shutdown()
1093 rdma_destroy_qp(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1094 if (ic->i_send_cq) { in rds_ib_conn_path_shutdown()
1095 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1096 ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector); in rds_ib_conn_path_shutdown()
1097 ib_destroy_cq(ic->i_send_cq); in rds_ib_conn_path_shutdown()
1100 if (ic->i_recv_cq) { in rds_ib_conn_path_shutdown()
1101 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1102 ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector); in rds_ib_conn_path_shutdown()
1103 ib_destroy_cq(ic->i_recv_cq); in rds_ib_conn_path_shutdown()
1106 if (ic->rds_ibdev) { in rds_ib_conn_path_shutdown()
1108 if (ic->i_send_hdrs) { in rds_ib_conn_path_shutdown()
1109 rds_dma_hdrs_free(ic->rds_ibdev, in rds_ib_conn_path_shutdown()
1110 ic->i_send_hdrs, in rds_ib_conn_path_shutdown()
1111 ic->i_send_hdrs_dma, in rds_ib_conn_path_shutdown()
1112 ic->i_send_ring.w_nr, in rds_ib_conn_path_shutdown()
1114 ic->i_send_hdrs = NULL; in rds_ib_conn_path_shutdown()
1115 ic->i_send_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1118 if (ic->i_recv_hdrs) { in rds_ib_conn_path_shutdown()
1119 rds_dma_hdrs_free(ic->rds_ibdev, in rds_ib_conn_path_shutdown()
1120 ic->i_recv_hdrs, in rds_ib_conn_path_shutdown()
1121 ic->i_recv_hdrs_dma, in rds_ib_conn_path_shutdown()
1122 ic->i_recv_ring.w_nr, in rds_ib_conn_path_shutdown()
1124 ic->i_recv_hdrs = NULL; in rds_ib_conn_path_shutdown()
1125 ic->i_recv_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1128 if (ic->i_ack) { in rds_ib_conn_path_shutdown()
1129 rds_dma_hdr_free(ic->rds_ibdev->dev, ic->i_ack, in rds_ib_conn_path_shutdown()
1130 ic->i_ack_dma, DMA_TO_DEVICE); in rds_ib_conn_path_shutdown()
1131 ic->i_ack = NULL; in rds_ib_conn_path_shutdown()
1134 WARN_ON(ic->i_send_hdrs); in rds_ib_conn_path_shutdown()
1135 WARN_ON(ic->i_send_hdrs_dma); in rds_ib_conn_path_shutdown()
1136 WARN_ON(ic->i_recv_hdrs); in rds_ib_conn_path_shutdown()
1137 WARN_ON(ic->i_recv_hdrs_dma); in rds_ib_conn_path_shutdown()
1138 WARN_ON(ic->i_ack); in rds_ib_conn_path_shutdown()
1141 if (ic->i_sends) in rds_ib_conn_path_shutdown()
1142 rds_ib_send_clear_ring(ic); in rds_ib_conn_path_shutdown()
1143 if (ic->i_recvs) in rds_ib_conn_path_shutdown()
1144 rds_ib_recv_clear_ring(ic); in rds_ib_conn_path_shutdown()
1146 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1151 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1152 rds_ib_remove_conn(ic->rds_ibdev, conn); in rds_ib_conn_path_shutdown()
1154 ic->i_cm_id = NULL; in rds_ib_conn_path_shutdown()
1155 ic->i_pd = NULL; in rds_ib_conn_path_shutdown()
1156 ic->i_send_cq = NULL; in rds_ib_conn_path_shutdown()
1157 ic->i_recv_cq = NULL; in rds_ib_conn_path_shutdown()
1159 BUG_ON(ic->rds_ibdev); in rds_ib_conn_path_shutdown()
1162 if (ic->i_data_op) { in rds_ib_conn_path_shutdown()
1165 rm = container_of(ic->i_data_op, struct rds_message, data); in rds_ib_conn_path_shutdown()
1167 ic->i_data_op = NULL; in rds_ib_conn_path_shutdown()
1171 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_conn_path_shutdown()
1173 atomic64_set(&ic->i_ack_next, 0); in rds_ib_conn_path_shutdown()
1175 ic->i_ack_next = 0; in rds_ib_conn_path_shutdown()
1177 ic->i_ack_recv = 0; in rds_ib_conn_path_shutdown()
1180 ic->i_flowctl = 0; in rds_ib_conn_path_shutdown()
1181 atomic_set(&ic->i_credits, 0); in rds_ib_conn_path_shutdown()
1184 rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr); in rds_ib_conn_path_shutdown()
1185 rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr); in rds_ib_conn_path_shutdown()
1187 if (ic->i_ibinc) { in rds_ib_conn_path_shutdown()
1188 rds_inc_put(&ic->i_ibinc->ii_inc); in rds_ib_conn_path_shutdown()
1189 ic->i_ibinc = NULL; in rds_ib_conn_path_shutdown()
1192 vfree(ic->i_sends); in rds_ib_conn_path_shutdown()
1193 ic->i_sends = NULL; in rds_ib_conn_path_shutdown()
1194 vfree(ic->i_recvs); in rds_ib_conn_path_shutdown()
1195 ic->i_recvs = NULL; in rds_ib_conn_path_shutdown()
1196 ic->i_active_side = false; in rds_ib_conn_path_shutdown()
1201 struct rds_ib_connection *ic; in rds_ib_conn_alloc() local
1206 ic = kzalloc(sizeof(struct rds_ib_connection), gfp); in rds_ib_conn_alloc()
1207 if (!ic) in rds_ib_conn_alloc()
1210 ret = rds_ib_recv_alloc_caches(ic, gfp); in rds_ib_conn_alloc()
1212 kfree(ic); in rds_ib_conn_alloc()
1216 INIT_LIST_HEAD(&ic->ib_node); in rds_ib_conn_alloc()
1217 tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send, in rds_ib_conn_alloc()
1218 (unsigned long)ic); in rds_ib_conn_alloc()
1219 tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv, in rds_ib_conn_alloc()
1220 (unsigned long)ic); in rds_ib_conn_alloc()
1221 mutex_init(&ic->i_recv_mutex); in rds_ib_conn_alloc()
1223 spin_lock_init(&ic->i_ack_lock); in rds_ib_conn_alloc()
1225 atomic_set(&ic->i_signaled_sends, 0); in rds_ib_conn_alloc()
1226 atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR); in rds_ib_conn_alloc()
1232 rds_ib_ring_init(&ic->i_send_ring, 0); in rds_ib_conn_alloc()
1233 rds_ib_ring_init(&ic->i_recv_ring, 0); in rds_ib_conn_alloc()
1235 ic->conn = conn; in rds_ib_conn_alloc()
1236 conn->c_transport_data = ic; in rds_ib_conn_alloc()
1239 list_add_tail(&ic->ib_node, &ib_nodev_conns); in rds_ib_conn_alloc()
1252 struct rds_ib_connection *ic = arg; in rds_ib_conn_free() local
1255 rdsdebug("ic %p\n", ic); in rds_ib_conn_free()
1262 lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock; in rds_ib_conn_free()
1265 list_del(&ic->ib_node); in rds_ib_conn_free()
1268 rds_ib_recv_free_caches(ic); in rds_ib_conn_free()
1270 kfree(ic); in rds_ib_conn_free()