Lines Matching refs:qp
58 static int irdma_nop_1(struct irdma_qp_uk *qp) in irdma_nop_1() argument
65 if (!qp->sq_ring.head) in irdma_nop_1()
68 wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); in irdma_nop_1()
69 wqe = qp->sq_base[wqe_idx].elem; in irdma_nop_1()
71 qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA; in irdma_nop_1()
79 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_nop_1()
94 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx) in irdma_clr_wqes() argument
100 wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size; in irdma_clr_wqes()
101 sq = qp->sq_base + wqe_idx; in irdma_clr_wqes()
103 memset(sq, qp->swqe_polarity ? 0 : 0xFF, in irdma_clr_wqes()
106 memset(sq, qp->swqe_polarity ? 0xFF : 0, in irdma_clr_wqes()
115 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp) in irdma_uk_qp_post_wr() argument
125 get_64bit_val(qp->shadow_area, 0, &temp); in irdma_uk_qp_post_wr()
128 sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); in irdma_uk_qp_post_wr()
129 if (sw_sq_head != qp->initial_ring.head) { in irdma_uk_qp_post_wr()
131 if (sw_sq_head > qp->initial_ring.head) { in irdma_uk_qp_post_wr()
132 if (hw_sq_tail >= qp->initial_ring.head && in irdma_uk_qp_post_wr()
134 writel(qp->qp_id, qp->wqe_alloc_db); in irdma_uk_qp_post_wr()
136 if (hw_sq_tail >= qp->initial_ring.head || in irdma_uk_qp_post_wr()
138 writel(qp->qp_id, qp->wqe_alloc_db); in irdma_uk_qp_post_wr()
143 qp->initial_ring.head = qp->sq_ring.head; in irdma_uk_qp_post_wr()
154 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, in irdma_qp_get_next_send_wqe() argument
163 avail_quanta = qp->uk_attrs->max_hw_sq_chunk - in irdma_qp_get_next_send_wqe()
164 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) % in irdma_qp_get_next_send_wqe()
165 qp->uk_attrs->max_hw_sq_chunk); in irdma_qp_get_next_send_wqe()
168 if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) in irdma_qp_get_next_send_wqe()
173 IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) in irdma_qp_get_next_send_wqe()
177 irdma_nop_1(qp); in irdma_qp_get_next_send_wqe()
178 IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring); in irdma_qp_get_next_send_wqe()
182 *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); in irdma_qp_get_next_send_wqe()
184 qp->swqe_polarity = !qp->swqe_polarity; in irdma_qp_get_next_send_wqe()
186 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); in irdma_qp_get_next_send_wqe()
188 wqe = qp->sq_base[*wqe_idx].elem; in irdma_qp_get_next_send_wqe()
189 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 && in irdma_qp_get_next_send_wqe()
190 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { in irdma_qp_get_next_send_wqe()
191 wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; in irdma_qp_get_next_send_wqe()
192 wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity ? 0 : 1)); in irdma_qp_get_next_send_wqe()
194 qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; in irdma_qp_get_next_send_wqe()
195 qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; in irdma_qp_get_next_send_wqe()
196 qp->sq_wrtrk_array[*wqe_idx].quanta = quanta; in irdma_qp_get_next_send_wqe()
206 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx) in irdma_qp_get_next_recv_wqe() argument
211 if (IRDMA_RING_FULL_ERR(qp->rq_ring)) in irdma_qp_get_next_recv_wqe()
214 IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); in irdma_qp_get_next_recv_wqe()
219 qp->rwqe_polarity = !qp->rwqe_polarity; in irdma_qp_get_next_recv_wqe()
221 wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem; in irdma_qp_get_next_recv_wqe()
232 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, in irdma_uk_rdma_write() argument
246 if (op_info->num_lo_sges > qp->max_sq_frag_cnt) in irdma_uk_rdma_write()
263 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, in irdma_uk_rdma_write()
268 irdma_clr_wqes(qp, wqe_idx); in irdma_uk_rdma_write()
278 qp->wqe_ops.iw_set_fragment(wqe, 0, in irdma_uk_rdma_write()
280 qp->swqe_polarity); in irdma_uk_rdma_write()
285 qp->wqe_ops.iw_set_fragment(wqe, byte_off, in irdma_uk_rdma_write()
287 qp->swqe_polarity); in irdma_uk_rdma_write()
292 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && in irdma_uk_rdma_write()
294 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_rdma_write()
295 qp->swqe_polarity); in irdma_uk_rdma_write()
296 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_rdma_write()
308 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_rdma_write()
315 irdma_uk_qp_post_wr(qp); in irdma_uk_rdma_write()
327 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, in irdma_uk_rdma_read() argument
341 if (qp->max_sq_frag_cnt < op_info->num_lo_sges) in irdma_uk_rdma_read()
351 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, in irdma_uk_rdma_read()
356 irdma_clr_wqes(qp, wqe_idx); in irdma_uk_rdma_read()
362 qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->lo_sg_list, in irdma_uk_rdma_read()
363 qp->swqe_polarity); in irdma_uk_rdma_read()
365 qp->wqe_ops.iw_set_fragment(wqe, byte_off, in irdma_uk_rdma_read()
367 qp->swqe_polarity); in irdma_uk_rdma_read()
372 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && in irdma_uk_rdma_read()
374 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_rdma_read()
375 qp->swqe_polarity); in irdma_uk_rdma_read()
376 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_rdma_read()
389 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_rdma_read()
396 irdma_uk_qp_post_wr(qp); in irdma_uk_rdma_read()
407 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, in irdma_uk_send() argument
420 if (qp->max_sq_frag_cnt < op_info->num_sges) in irdma_uk_send()
434 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, in irdma_uk_send()
439 irdma_clr_wqes(qp, wqe_idx); in irdma_uk_send()
448 qp->wqe_ops.iw_set_fragment(wqe, 0, in irdma_uk_send()
450 qp->swqe_polarity); in irdma_uk_send()
455 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i], in irdma_uk_send()
456 qp->swqe_polarity); in irdma_uk_send()
461 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && in irdma_uk_send()
463 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_send()
464 qp->swqe_polarity); in irdma_uk_send()
465 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_send()
484 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_send()
491 irdma_uk_qp_post_wr(qp); in irdma_uk_send()
654 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, in irdma_uk_inline_rdma_write() argument
667 if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges)) in irdma_uk_inline_rdma_write()
673 if (unlikely(total_size > qp->max_inline_data)) in irdma_uk_inline_rdma_write()
676 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); in irdma_uk_inline_rdma_write()
677 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, in irdma_uk_inline_rdma_write()
682 irdma_clr_wqes(qp, wqe_idx); in irdma_uk_inline_rdma_write()
697 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_inline_rdma_write()
703 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list, in irdma_uk_inline_rdma_write()
705 qp->swqe_polarity); in irdma_uk_inline_rdma_write()
711 irdma_uk_qp_post_wr(qp); in irdma_uk_inline_rdma_write()
722 int irdma_uk_inline_send(struct irdma_qp_uk *qp, in irdma_uk_inline_send() argument
735 if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges)) in irdma_uk_inline_send()
741 if (unlikely(total_size > qp->max_inline_data)) in irdma_uk_inline_send()
744 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); in irdma_uk_inline_send()
745 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, in irdma_uk_inline_send()
750 irdma_clr_wqes(qp, wqe_idx); in irdma_uk_inline_send()
770 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_inline_send()
775 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list, in irdma_uk_inline_send()
776 op_info->num_sges, qp->swqe_polarity); in irdma_uk_inline_send()
783 irdma_uk_qp_post_wr(qp); in irdma_uk_inline_send()
794 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, in irdma_uk_stag_local_invalidate() argument
808 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, in irdma_uk_stag_local_invalidate()
813 irdma_clr_wqes(qp, wqe_idx); in irdma_uk_stag_local_invalidate()
816 qp->wqe_ops.iw_set_fragment(wqe, 0, &sge, 0); in irdma_uk_stag_local_invalidate()
824 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_stag_local_invalidate()
831 irdma_uk_qp_post_wr(qp); in irdma_uk_stag_local_invalidate()
841 int irdma_uk_post_receive(struct irdma_qp_uk *qp, in irdma_uk_post_receive() argument
849 if (qp->max_rq_frag_cnt < info->num_sges) in irdma_uk_post_receive()
852 wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx); in irdma_uk_post_receive()
856 qp->rq_wrid_array[wqe_idx] = info->wr_id; in irdma_uk_post_receive()
858 qp->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list, in irdma_uk_post_receive()
859 qp->rwqe_polarity); in irdma_uk_post_receive()
862 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i], in irdma_uk_post_receive()
863 qp->rwqe_polarity); in irdma_uk_post_receive()
868 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) && in irdma_uk_post_receive()
870 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_post_receive()
871 qp->rwqe_polarity); in irdma_uk_post_receive()
872 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_post_receive()
878 FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity); in irdma_uk_post_receive()
975 struct irdma_qp_uk *qp; in irdma_uk_cq_poll_cmpl() local
1079 qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx; in irdma_uk_cq_poll_cmpl()
1080 if (!qp || qp->destroy_pending) { in irdma_uk_cq_poll_cmpl()
1085 info->qp_handle = (irdma_qp_handle)(unsigned long)qp; in irdma_uk_cq_poll_cmpl()
1091 array_idx = wqe_idx / qp->rq_wqe_size_multiplier; in irdma_uk_cq_poll_cmpl()
1095 if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) { in irdma_uk_cq_poll_cmpl()
1100 info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail]; in irdma_uk_cq_poll_cmpl()
1101 array_idx = qp->rq_ring.tail; in irdma_uk_cq_poll_cmpl()
1103 info->wr_id = qp->rq_wrid_array[array_idx]; in irdma_uk_cq_poll_cmpl()
1114 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1); in irdma_uk_cq_poll_cmpl()
1116 qp->rq_flush_seen = true; in irdma_uk_cq_poll_cmpl()
1117 if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) in irdma_uk_cq_poll_cmpl()
1118 qp->rq_flush_complete = true; in irdma_uk_cq_poll_cmpl()
1122 pring = &qp->rq_ring; in irdma_uk_cq_poll_cmpl()
1124 if (qp->first_sq_wq) { in irdma_uk_cq_poll_cmpl()
1125 if (wqe_idx + 1 >= qp->conn_wqes) in irdma_uk_cq_poll_cmpl()
1126 qp->first_sq_wq = false; in irdma_uk_cq_poll_cmpl()
1128 if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) { in irdma_uk_cq_poll_cmpl()
1139 info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; in irdma_uk_cq_poll_cmpl()
1141 info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; in irdma_uk_cq_poll_cmpl()
1143 IRDMA_RING_SET_TAIL(qp->sq_ring, in irdma_uk_cq_poll_cmpl()
1144 wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); in irdma_uk_cq_poll_cmpl()
1146 if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) { in irdma_uk_cq_poll_cmpl()
1156 tail = qp->sq_ring.tail; in irdma_uk_cq_poll_cmpl()
1157 sw_wqe = qp->sq_base[tail].elem; in irdma_uk_cq_poll_cmpl()
1162 IRDMA_RING_SET_TAIL(qp->sq_ring, in irdma_uk_cq_poll_cmpl()
1163 tail + qp->sq_wrtrk_array[tail].quanta); in irdma_uk_cq_poll_cmpl()
1165 info->wr_id = qp->sq_wrtrk_array[tail].wrid; in irdma_uk_cq_poll_cmpl()
1166 info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; in irdma_uk_cq_poll_cmpl()
1173 qp->sq_flush_seen = true; in irdma_uk_cq_poll_cmpl()
1174 if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) in irdma_uk_cq_poll_cmpl()
1175 qp->sq_flush_complete = true; in irdma_uk_cq_poll_cmpl()
1177 pring = &qp->sq_ring; in irdma_uk_cq_poll_cmpl()
1323 static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp, in irdma_setup_connection_wqes() argument
1329 (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)) in irdma_setup_connection_wqes()
1332 qp->conn_wqes = move_cnt; in irdma_setup_connection_wqes()
1333 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt); in irdma_setup_connection_wqes()
1334 IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt); in irdma_setup_connection_wqes()
1335 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt); in irdma_setup_connection_wqes()
1420 int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) in irdma_uk_qp_init() argument
1425 qp->uk_attrs = info->uk_attrs; in irdma_uk_qp_init()
1426 if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags || in irdma_uk_qp_init()
1427 info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags) in irdma_uk_qp_init()
1430 qp->qp_caps = info->qp_caps; in irdma_uk_qp_init()
1431 qp->sq_base = info->sq; in irdma_uk_qp_init()
1432 qp->rq_base = info->rq; in irdma_uk_qp_init()
1433 qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP; in irdma_uk_qp_init()
1434 qp->shadow_area = info->shadow_area; in irdma_uk_qp_init()
1435 qp->sq_wrtrk_array = info->sq_wrtrk_array; in irdma_uk_qp_init()
1437 qp->rq_wrid_array = info->rq_wrid_array; in irdma_uk_qp_init()
1438 qp->wqe_alloc_db = info->wqe_alloc_db; in irdma_uk_qp_init()
1439 qp->qp_id = info->qp_id; in irdma_uk_qp_init()
1440 qp->sq_size = info->sq_size; in irdma_uk_qp_init()
1441 qp->max_sq_frag_cnt = info->max_sq_frag_cnt; in irdma_uk_qp_init()
1442 sq_ring_size = qp->sq_size << info->sq_shift; in irdma_uk_qp_init()
1443 IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); in irdma_uk_qp_init()
1444 IRDMA_RING_INIT(qp->initial_ring, sq_ring_size); in irdma_uk_qp_init()
1446 irdma_setup_connection_wqes(qp, info); in irdma_uk_qp_init()
1447 qp->swqe_polarity = 1; in irdma_uk_qp_init()
1448 qp->first_sq_wq = true; in irdma_uk_qp_init()
1450 qp->swqe_polarity = 0; in irdma_uk_qp_init()
1452 qp->swqe_polarity_deferred = 1; in irdma_uk_qp_init()
1453 qp->rwqe_polarity = 0; in irdma_uk_qp_init()
1454 qp->rq_size = info->rq_size; in irdma_uk_qp_init()
1455 qp->max_rq_frag_cnt = info->max_rq_frag_cnt; in irdma_uk_qp_init()
1456 qp->max_inline_data = info->max_inline_data; in irdma_uk_qp_init()
1457 qp->rq_wqe_size = info->rq_shift; in irdma_uk_qp_init()
1458 IRDMA_RING_INIT(qp->rq_ring, qp->rq_size); in irdma_uk_qp_init()
1459 qp->rq_wqe_size_multiplier = 1 << info->rq_shift; in irdma_uk_qp_init()
1460 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) in irdma_uk_qp_init()
1461 qp->wqe_ops = iw_wqe_uk_ops_gen_1; in irdma_uk_qp_init()
1463 qp->wqe_ops = iw_wqe_uk_ops; in irdma_uk_qp_init()
1531 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq) in irdma_nop() argument
1539 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, in irdma_nop()
1544 irdma_clr_wqes(qp, wqe_idx); in irdma_nop()
1552 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_nop()
1558 irdma_uk_qp_post_wr(qp); in irdma_nop()