Lines Matching +full:link +full:-
1 // SPDX-License-Identifier: GPL-2.0
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
17 * SMC uses this to exchange LLC (link layer control)
44 struct smc_link *link; member
52 /*------------------------------- completion --------------------------------*/
54 /* returns true if at least one tx work request is pending on the given link */
55 static inline bool smc_wr_is_tx_pend(struct smc_link *link) in smc_wr_is_tx_pend() argument
57 return !bitmap_empty(link->wr_tx_mask, link->wr_tx_cnt); in smc_wr_is_tx_pend()
60 /* wait till all pending tx work requests on the given link are completed */
61 void smc_wr_tx_wait_no_pending_sends(struct smc_link *link) in smc_wr_tx_wait_no_pending_sends() argument
63 wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link)); in smc_wr_tx_wait_no_pending_sends()
66 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) in smc_wr_tx_find_pending_index() argument
70 for (i = 0; i < link->wr_tx_cnt; i++) { in smc_wr_tx_find_pending_index()
71 if (link->wr_tx_pends[i].wr_id == wr_id) in smc_wr_tx_find_pending_index()
74 return link->wr_tx_cnt; in smc_wr_tx_find_pending_index()
80 struct smc_link *link; in smc_wr_tx_process_cqe() local
83 link = wc->qp->qp_context; in smc_wr_tx_process_cqe()
85 if (wc->opcode == IB_WC_REG_MR) { in smc_wr_tx_process_cqe()
86 if (wc->status) in smc_wr_tx_process_cqe()
87 link->wr_reg_state = FAILED; in smc_wr_tx_process_cqe()
89 link->wr_reg_state = CONFIRMED; in smc_wr_tx_process_cqe()
90 smc_wr_wakeup_reg_wait(link); in smc_wr_tx_process_cqe()
94 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); in smc_wr_tx_process_cqe()
95 if (pnd_snd_idx == link->wr_tx_cnt) { in smc_wr_tx_process_cqe()
96 if (link->lgr->smc_version != SMC_V2 || in smc_wr_tx_process_cqe()
97 link->wr_tx_v2_pend->wr_id != wc->wr_id) in smc_wr_tx_process_cqe()
99 link->wr_tx_v2_pend->wc_status = wc->status; in smc_wr_tx_process_cqe()
100 memcpy(&pnd_snd, link->wr_tx_v2_pend, sizeof(pnd_snd)); in smc_wr_tx_process_cqe()
102 memset(link->wr_tx_v2_pend, 0, in smc_wr_tx_process_cqe()
103 sizeof(*link->wr_tx_v2_pend)); in smc_wr_tx_process_cqe()
104 memset(link->lgr->wr_tx_buf_v2, 0, in smc_wr_tx_process_cqe()
105 sizeof(*link->lgr->wr_tx_buf_v2)); in smc_wr_tx_process_cqe()
107 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; in smc_wr_tx_process_cqe()
108 if (link->wr_tx_pends[pnd_snd_idx].compl_requested) in smc_wr_tx_process_cqe()
109 complete(&link->wr_tx_compl[pnd_snd_idx]); in smc_wr_tx_process_cqe()
110 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], in smc_wr_tx_process_cqe()
113 memset(&link->wr_tx_pends[pnd_snd_idx], 0, in smc_wr_tx_process_cqe()
114 sizeof(link->wr_tx_pends[pnd_snd_idx])); in smc_wr_tx_process_cqe()
115 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, in smc_wr_tx_process_cqe()
116 sizeof(link->wr_tx_bufs[pnd_snd_idx])); in smc_wr_tx_process_cqe()
117 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) in smc_wr_tx_process_cqe()
121 if (wc->status) { in smc_wr_tx_process_cqe()
122 if (link->lgr->smc_version == SMC_V2) { in smc_wr_tx_process_cqe()
123 memset(link->wr_tx_v2_pend, 0, in smc_wr_tx_process_cqe()
124 sizeof(*link->wr_tx_v2_pend)); in smc_wr_tx_process_cqe()
125 memset(link->lgr->wr_tx_buf_v2, 0, in smc_wr_tx_process_cqe()
126 sizeof(*link->lgr->wr_tx_buf_v2)); in smc_wr_tx_process_cqe()
128 /* terminate link */ in smc_wr_tx_process_cqe()
129 smcr_link_down_cond_sched(link); in smc_wr_tx_process_cqe()
132 pnd_snd.handler(&pnd_snd.priv, link, wc->status); in smc_wr_tx_process_cqe()
133 wake_up(&link->wr_tx_wait); in smc_wr_tx_process_cqe()
147 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); in smc_wr_tx_tasklet_fn()
149 ib_req_notify_cq(dev->roce_cq_send, in smc_wr_tx_tasklet_fn()
166 tasklet_schedule(&dev->send_tasklet); in smc_wr_tx_cq_handler()
169 /*---------------------------- request submission ---------------------------*/
171 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) in smc_wr_tx_get_free_slot_index() argument
173 *idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot_index()
174 if (!smc_link_sendable(link)) in smc_wr_tx_get_free_slot_index()
175 return -ENOLINK; in smc_wr_tx_get_free_slot_index()
176 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { in smc_wr_tx_get_free_slot_index()
177 if (!test_and_set_bit(*idx, link->wr_tx_mask)) in smc_wr_tx_get_free_slot_index()
180 *idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot_index()
181 return -EBUSY; in smc_wr_tx_get_free_slot_index()
185 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
187 * @link: Pointer to smc_link used to later send the message.
193 * Return: 0 on success, or -errno on error.
195 int smc_wr_tx_get_free_slot(struct smc_link *link, in smc_wr_tx_get_free_slot() argument
201 struct smc_link_group *lgr = smc_get_lgr(link); in smc_wr_tx_get_free_slot()
203 u32 idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot()
210 if (in_softirq() || lgr->terminating) { in smc_wr_tx_get_free_slot()
211 rc = smc_wr_tx_get_free_slot_index(link, &idx); in smc_wr_tx_get_free_slot()
216 link->wr_tx_wait, in smc_wr_tx_get_free_slot()
217 !smc_link_sendable(link) || in smc_wr_tx_get_free_slot()
218 lgr->terminating || in smc_wr_tx_get_free_slot()
219 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), in smc_wr_tx_get_free_slot()
222 /* timeout - terminate link */ in smc_wr_tx_get_free_slot()
223 smcr_link_down_cond_sched(link); in smc_wr_tx_get_free_slot()
224 return -EPIPE; in smc_wr_tx_get_free_slot()
226 if (idx == link->wr_tx_cnt) in smc_wr_tx_get_free_slot()
227 return -EPIPE; in smc_wr_tx_get_free_slot()
229 wr_id = smc_wr_tx_get_next_wr_id(link); in smc_wr_tx_get_free_slot()
230 wr_pend = &link->wr_tx_pends[idx]; in smc_wr_tx_get_free_slot()
231 wr_pend->wr_id = wr_id; in smc_wr_tx_get_free_slot()
232 wr_pend->handler = handler; in smc_wr_tx_get_free_slot()
233 wr_pend->link = link; in smc_wr_tx_get_free_slot()
234 wr_pend->idx = idx; in smc_wr_tx_get_free_slot()
235 wr_ib = &link->wr_tx_ibs[idx]; in smc_wr_tx_get_free_slot()
236 wr_ib->wr_id = wr_id; in smc_wr_tx_get_free_slot()
237 *wr_buf = &link->wr_tx_bufs[idx]; in smc_wr_tx_get_free_slot()
239 *wr_rdma_buf = &link->wr_tx_rdmas[idx]; in smc_wr_tx_get_free_slot()
240 *wr_pend_priv = &wr_pend->priv; in smc_wr_tx_get_free_slot()
244 int smc_wr_tx_get_v2_slot(struct smc_link *link, in smc_wr_tx_get_v2_slot() argument
253 if (link->wr_tx_v2_pend->idx == link->wr_tx_cnt) in smc_wr_tx_get_v2_slot()
254 return -EBUSY; in smc_wr_tx_get_v2_slot()
258 wr_id = smc_wr_tx_get_next_wr_id(link); in smc_wr_tx_get_v2_slot()
259 wr_pend = link->wr_tx_v2_pend; in smc_wr_tx_get_v2_slot()
260 wr_pend->wr_id = wr_id; in smc_wr_tx_get_v2_slot()
261 wr_pend->handler = handler; in smc_wr_tx_get_v2_slot()
262 wr_pend->link = link; in smc_wr_tx_get_v2_slot()
263 wr_pend->idx = link->wr_tx_cnt; in smc_wr_tx_get_v2_slot()
264 wr_ib = link->wr_tx_v2_ib; in smc_wr_tx_get_v2_slot()
265 wr_ib->wr_id = wr_id; in smc_wr_tx_get_v2_slot()
266 *wr_buf = link->lgr->wr_tx_buf_v2; in smc_wr_tx_get_v2_slot()
267 *wr_pend_priv = &wr_pend->priv; in smc_wr_tx_get_v2_slot()
271 int smc_wr_tx_put_slot(struct smc_link *link, in smc_wr_tx_put_slot() argument
277 if (pend->idx < link->wr_tx_cnt) { in smc_wr_tx_put_slot()
278 u32 idx = pend->idx; in smc_wr_tx_put_slot()
281 memset(&link->wr_tx_pends[idx], 0, in smc_wr_tx_put_slot()
282 sizeof(link->wr_tx_pends[idx])); in smc_wr_tx_put_slot()
283 memset(&link->wr_tx_bufs[idx], 0, in smc_wr_tx_put_slot()
284 sizeof(link->wr_tx_bufs[idx])); in smc_wr_tx_put_slot()
285 test_and_clear_bit(idx, link->wr_tx_mask); in smc_wr_tx_put_slot()
286 wake_up(&link->wr_tx_wait); in smc_wr_tx_put_slot()
288 } else if (link->lgr->smc_version == SMC_V2 && in smc_wr_tx_put_slot()
289 pend->idx == link->wr_tx_cnt) { in smc_wr_tx_put_slot()
291 memset(&link->wr_tx_v2_pend, 0, in smc_wr_tx_put_slot()
292 sizeof(link->wr_tx_v2_pend)); in smc_wr_tx_put_slot()
293 memset(&link->lgr->wr_tx_buf_v2, 0, in smc_wr_tx_put_slot()
294 sizeof(link->lgr->wr_tx_buf_v2)); in smc_wr_tx_put_slot()
304 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) in smc_wr_tx_send() argument
309 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_tx_send()
312 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL); in smc_wr_tx_send()
314 smc_wr_tx_put_slot(link, priv); in smc_wr_tx_send()
315 smcr_link_down_cond_sched(link); in smc_wr_tx_send()
320 int smc_wr_tx_v2_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, in smc_wr_tx_v2_send() argument
325 link->wr_tx_v2_ib->sg_list[0].length = len; in smc_wr_tx_v2_send()
326 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_tx_v2_send()
328 rc = ib_post_send(link->roce_qp, link->wr_tx_v2_ib, NULL); in smc_wr_tx_v2_send()
330 smc_wr_tx_put_slot(link, priv); in smc_wr_tx_v2_send()
331 smcr_link_down_cond_sched(link); in smc_wr_tx_v2_send()
340 int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, in smc_wr_tx_send_wait() argument
348 pend->compl_requested = 1; in smc_wr_tx_send_wait()
349 pnd_idx = pend->idx; in smc_wr_tx_send_wait()
350 init_completion(&link->wr_tx_compl[pnd_idx]); in smc_wr_tx_send_wait()
352 rc = smc_wr_tx_send(link, priv); in smc_wr_tx_send_wait()
357 &link->wr_tx_compl[pnd_idx], timeout); in smc_wr_tx_send_wait()
359 rc = -ENODATA; in smc_wr_tx_send_wait()
366 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) in smc_wr_reg_send() argument
370 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_reg_send()
372 link->wr_reg_state = POSTED; in smc_wr_reg_send()
373 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; in smc_wr_reg_send()
374 link->wr_reg.mr = mr; in smc_wr_reg_send()
375 link->wr_reg.key = mr->rkey; in smc_wr_reg_send()
376 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL); in smc_wr_reg_send()
380 percpu_ref_get(&link->wr_reg_refs); in smc_wr_reg_send()
381 rc = wait_event_interruptible_timeout(link->wr_reg_wait, in smc_wr_reg_send()
382 (link->wr_reg_state != POSTED), in smc_wr_reg_send()
384 percpu_ref_put(&link->wr_reg_refs); in smc_wr_reg_send()
386 /* timeout - terminate link */ in smc_wr_reg_send()
387 smcr_link_down_cond_sched(link); in smc_wr_reg_send()
388 return -EPIPE; in smc_wr_reg_send()
390 if (rc == -ERESTARTSYS) in smc_wr_reg_send()
391 return -EINTR; in smc_wr_reg_send()
392 switch (link->wr_reg_state) { in smc_wr_reg_send()
397 rc = -EIO; in smc_wr_reg_send()
400 rc = -EPIPE; in smc_wr_reg_send()
414 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { in smc_wr_rx_register_handler()
415 if (h_iter->type == handler->type) { in smc_wr_rx_register_handler()
416 rc = -EEXIST; in smc_wr_rx_register_handler()
420 hash_add(smc_wr_rx_hash, &handler->list, handler->type); in smc_wr_rx_register_handler()
432 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; in smc_wr_rx_demultiplex() local
438 if (wc->byte_len < sizeof(*wr_rx)) in smc_wr_rx_demultiplex()
440 temp_wr_id = wc->wr_id; in smc_wr_rx_demultiplex()
441 index = do_div(temp_wr_id, link->wr_rx_cnt); in smc_wr_rx_demultiplex()
442 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; in smc_wr_rx_demultiplex()
443 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { in smc_wr_rx_demultiplex()
444 if (handler->type == wr_rx->type) in smc_wr_rx_demultiplex()
445 handler->handler(wc, wr_rx); in smc_wr_rx_demultiplex()
451 struct smc_link *link; in smc_wr_rx_process_cqes() local
455 link = wc[i].qp->qp_context; in smc_wr_rx_process_cqes()
456 link->wr_rx_id_compl = wc[i].wr_id; in smc_wr_rx_process_cqes()
458 link->wr_rx_tstamp = jiffies; in smc_wr_rx_process_cqes()
460 smc_wr_rx_post(link); /* refill WR RX */ in smc_wr_rx_process_cqes()
467 smcr_link_down_cond_sched(link); in smc_wr_rx_process_cqes()
468 if (link->wr_rx_id_compl == link->wr_rx_id) in smc_wr_rx_process_cqes()
469 wake_up(&link->wr_rx_empty_wait); in smc_wr_rx_process_cqes()
472 smc_wr_rx_post(link); /* refill WR RX */ in smc_wr_rx_process_cqes()
490 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); in smc_wr_rx_tasklet_fn()
492 ib_req_notify_cq(dev->roce_cq_recv, in smc_wr_rx_tasklet_fn()
508 tasklet_schedule(&dev->recv_tasklet); in smc_wr_rx_cq_handler()
511 int smc_wr_rx_post_init(struct smc_link *link) in smc_wr_rx_post_init() argument
516 for (i = 0; i < link->wr_rx_cnt; i++) in smc_wr_rx_post_init()
517 rc = smc_wr_rx_post(link); in smc_wr_rx_post_init()
525 struct ib_qp_attr *attr = &lnk->qp_attr; in smc_wr_remember_qp_attr()
530 ib_query_qp(lnk->roce_qp, attr, in smc_wr_remember_qp_attr()
550 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, in smc_wr_remember_qp_attr()
551 lnk->qp_attr.cap.max_send_wr); in smc_wr_remember_qp_attr()
552 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, in smc_wr_remember_qp_attr()
553 lnk->qp_attr.cap.max_recv_wr); in smc_wr_remember_qp_attr()
558 int sges_per_buf = (lnk->lgr->smc_version == SMC_V2) ? 2 : 1; in smc_wr_init_sge()
559 bool send_inline = (lnk->qp_attr.cap.max_inline_data > SMC_WR_TX_SIZE); in smc_wr_init_sge()
562 for (i = 0; i < lnk->wr_tx_cnt; i++) { in smc_wr_init_sge()
563 lnk->wr_tx_sges[i].addr = send_inline ? (uintptr_t)(&lnk->wr_tx_bufs[i]) : in smc_wr_init_sge()
564 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
565 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
566 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
567 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
568 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
569 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
570 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
571 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
572 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
573 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
574 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
575 lnk->wr_tx_ibs[i].next = NULL; in smc_wr_init_sge()
576 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; in smc_wr_init_sge()
577 lnk->wr_tx_ibs[i].num_sge = 1; in smc_wr_init_sge()
578 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; in smc_wr_init_sge()
579 lnk->wr_tx_ibs[i].send_flags = in smc_wr_init_sge()
582 lnk->wr_tx_ibs[i].send_flags |= IB_SEND_INLINE; in smc_wr_init_sge()
583 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
584 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
585 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list = in smc_wr_init_sge()
586 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge; in smc_wr_init_sge()
587 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list = in smc_wr_init_sge()
588 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge; in smc_wr_init_sge()
591 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
592 lnk->wr_tx_v2_sge->addr = lnk->wr_tx_v2_dma_addr; in smc_wr_init_sge()
593 lnk->wr_tx_v2_sge->length = SMC_WR_BUF_V2_SIZE; in smc_wr_init_sge()
594 lnk->wr_tx_v2_sge->lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
596 lnk->wr_tx_v2_ib->next = NULL; in smc_wr_init_sge()
597 lnk->wr_tx_v2_ib->sg_list = lnk->wr_tx_v2_sge; in smc_wr_init_sge()
598 lnk->wr_tx_v2_ib->num_sge = 1; in smc_wr_init_sge()
599 lnk->wr_tx_v2_ib->opcode = IB_WR_SEND; in smc_wr_init_sge()
600 lnk->wr_tx_v2_ib->send_flags = in smc_wr_init_sge()
604 /* With SMC-Rv2 there can be messages larger than SMC_WR_TX_SIZE. in smc_wr_init_sge()
610 for (i = 0; i < lnk->wr_rx_cnt; i++) { in smc_wr_init_sge()
613 lnk->wr_rx_sges[x].addr = in smc_wr_init_sge()
614 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
615 lnk->wr_rx_sges[x].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
616 lnk->wr_rx_sges[x].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
617 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
618 lnk->wr_rx_sges[x + 1].addr = in smc_wr_init_sge()
619 lnk->wr_rx_v2_dma_addr + SMC_WR_TX_SIZE; in smc_wr_init_sge()
620 lnk->wr_rx_sges[x + 1].length = in smc_wr_init_sge()
621 SMC_WR_BUF_V2_SIZE - SMC_WR_TX_SIZE; in smc_wr_init_sge()
622 lnk->wr_rx_sges[x + 1].lkey = in smc_wr_init_sge()
623 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
625 lnk->wr_rx_ibs[i].next = NULL; in smc_wr_init_sge()
626 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[x]; in smc_wr_init_sge()
627 lnk->wr_rx_ibs[i].num_sge = sges_per_buf; in smc_wr_init_sge()
629 lnk->wr_reg.wr.next = NULL; in smc_wr_init_sge()
630 lnk->wr_reg.wr.num_sge = 0; in smc_wr_init_sge()
631 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; in smc_wr_init_sge()
632 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; in smc_wr_init_sge()
633 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; in smc_wr_init_sge()
640 if (!lnk->smcibdev) in smc_wr_free_link()
642 ibdev = lnk->smcibdev->ibdev; in smc_wr_free_link()
649 percpu_ref_kill(&lnk->wr_reg_refs); in smc_wr_free_link()
650 wait_for_completion(&lnk->reg_ref_comp); in smc_wr_free_link()
651 percpu_ref_exit(&lnk->wr_reg_refs); in smc_wr_free_link()
652 percpu_ref_kill(&lnk->wr_tx_refs); in smc_wr_free_link()
653 wait_for_completion(&lnk->tx_ref_comp); in smc_wr_free_link()
654 percpu_ref_exit(&lnk->wr_tx_refs); in smc_wr_free_link()
656 if (lnk->wr_rx_dma_addr) { in smc_wr_free_link()
657 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_free_link()
658 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_free_link()
660 lnk->wr_rx_dma_addr = 0; in smc_wr_free_link()
662 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_free_link()
663 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_free_link()
666 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_free_link()
668 if (lnk->wr_tx_dma_addr) { in smc_wr_free_link()
669 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, in smc_wr_free_link()
670 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_free_link()
672 lnk->wr_tx_dma_addr = 0; in smc_wr_free_link()
674 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_free_link()
675 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_free_link()
678 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_free_link()
684 if (lgr->smc_version < SMC_V2) in smc_wr_free_lgr_mem()
687 kfree(lgr->wr_rx_buf_v2); in smc_wr_free_lgr_mem()
688 lgr->wr_rx_buf_v2 = NULL; in smc_wr_free_lgr_mem()
689 kfree(lgr->wr_tx_buf_v2); in smc_wr_free_lgr_mem()
690 lgr->wr_tx_buf_v2 = NULL; in smc_wr_free_lgr_mem()
695 kfree(lnk->wr_tx_v2_ib); in smc_wr_free_link_mem()
696 lnk->wr_tx_v2_ib = NULL; in smc_wr_free_link_mem()
697 kfree(lnk->wr_tx_v2_sge); in smc_wr_free_link_mem()
698 lnk->wr_tx_v2_sge = NULL; in smc_wr_free_link_mem()
699 kfree(lnk->wr_tx_v2_pend); in smc_wr_free_link_mem()
700 lnk->wr_tx_v2_pend = NULL; in smc_wr_free_link_mem()
701 kfree(lnk->wr_tx_compl); in smc_wr_free_link_mem()
702 lnk->wr_tx_compl = NULL; in smc_wr_free_link_mem()
703 kfree(lnk->wr_tx_pends); in smc_wr_free_link_mem()
704 lnk->wr_tx_pends = NULL; in smc_wr_free_link_mem()
705 bitmap_free(lnk->wr_tx_mask); in smc_wr_free_link_mem()
706 lnk->wr_tx_mask = NULL; in smc_wr_free_link_mem()
707 kfree(lnk->wr_tx_sges); in smc_wr_free_link_mem()
708 lnk->wr_tx_sges = NULL; in smc_wr_free_link_mem()
709 kfree(lnk->wr_tx_rdma_sges); in smc_wr_free_link_mem()
710 lnk->wr_tx_rdma_sges = NULL; in smc_wr_free_link_mem()
711 kfree(lnk->wr_rx_sges); in smc_wr_free_link_mem()
712 lnk->wr_rx_sges = NULL; in smc_wr_free_link_mem()
713 kfree(lnk->wr_tx_rdmas); in smc_wr_free_link_mem()
714 lnk->wr_tx_rdmas = NULL; in smc_wr_free_link_mem()
715 kfree(lnk->wr_rx_ibs); in smc_wr_free_link_mem()
716 lnk->wr_rx_ibs = NULL; in smc_wr_free_link_mem()
717 kfree(lnk->wr_tx_ibs); in smc_wr_free_link_mem()
718 lnk->wr_tx_ibs = NULL; in smc_wr_free_link_mem()
719 kfree(lnk->wr_tx_bufs); in smc_wr_free_link_mem()
720 lnk->wr_tx_bufs = NULL; in smc_wr_free_link_mem()
721 kfree(lnk->wr_rx_bufs); in smc_wr_free_link_mem()
722 lnk->wr_rx_bufs = NULL; in smc_wr_free_link_mem()
727 if (lgr->smc_version < SMC_V2) in smc_wr_alloc_lgr_mem()
730 lgr->wr_rx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL); in smc_wr_alloc_lgr_mem()
731 if (!lgr->wr_rx_buf_v2) in smc_wr_alloc_lgr_mem()
732 return -ENOMEM; in smc_wr_alloc_lgr_mem()
733 lgr->wr_tx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL); in smc_wr_alloc_lgr_mem()
734 if (!lgr->wr_tx_buf_v2) { in smc_wr_alloc_lgr_mem()
735 kfree(lgr->wr_rx_buf_v2); in smc_wr_alloc_lgr_mem()
736 return -ENOMEM; in smc_wr_alloc_lgr_mem()
741 int smc_wr_alloc_link_mem(struct smc_link *link) in smc_wr_alloc_link_mem() argument
743 int sges_per_buf = link->lgr->smc_version == SMC_V2 ? 2 : 1; in smc_wr_alloc_link_mem()
745 /* allocate link related memory */ in smc_wr_alloc_link_mem()
746 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); in smc_wr_alloc_link_mem()
747 if (!link->wr_tx_bufs) in smc_wr_alloc_link_mem()
749 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, in smc_wr_alloc_link_mem()
751 if (!link->wr_rx_bufs) in smc_wr_alloc_link_mem()
753 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), in smc_wr_alloc_link_mem()
755 if (!link->wr_tx_ibs) in smc_wr_alloc_link_mem()
757 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, in smc_wr_alloc_link_mem()
758 sizeof(link->wr_rx_ibs[0]), in smc_wr_alloc_link_mem()
760 if (!link->wr_rx_ibs) in smc_wr_alloc_link_mem()
762 link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
763 sizeof(link->wr_tx_rdmas[0]), in smc_wr_alloc_link_mem()
765 if (!link->wr_tx_rdmas) in smc_wr_alloc_link_mem()
767 link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
768 sizeof(link->wr_tx_rdma_sges[0]), in smc_wr_alloc_link_mem()
770 if (!link->wr_tx_rdma_sges) in smc_wr_alloc_link_mem()
772 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), in smc_wr_alloc_link_mem()
774 if (!link->wr_tx_sges) in smc_wr_alloc_link_mem()
776 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, in smc_wr_alloc_link_mem()
777 sizeof(link->wr_rx_sges[0]) * sges_per_buf, in smc_wr_alloc_link_mem()
779 if (!link->wr_rx_sges) in smc_wr_alloc_link_mem()
781 link->wr_tx_mask = bitmap_zalloc(SMC_WR_BUF_CNT, GFP_KERNEL); in smc_wr_alloc_link_mem()
782 if (!link->wr_tx_mask) in smc_wr_alloc_link_mem()
784 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
785 sizeof(link->wr_tx_pends[0]), in smc_wr_alloc_link_mem()
787 if (!link->wr_tx_pends) in smc_wr_alloc_link_mem()
789 link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
790 sizeof(link->wr_tx_compl[0]), in smc_wr_alloc_link_mem()
792 if (!link->wr_tx_compl) in smc_wr_alloc_link_mem()
795 if (link->lgr->smc_version == SMC_V2) { in smc_wr_alloc_link_mem()
796 link->wr_tx_v2_ib = kzalloc(sizeof(*link->wr_tx_v2_ib), in smc_wr_alloc_link_mem()
798 if (!link->wr_tx_v2_ib) in smc_wr_alloc_link_mem()
800 link->wr_tx_v2_sge = kzalloc(sizeof(*link->wr_tx_v2_sge), in smc_wr_alloc_link_mem()
802 if (!link->wr_tx_v2_sge) in smc_wr_alloc_link_mem()
804 link->wr_tx_v2_pend = kzalloc(sizeof(*link->wr_tx_v2_pend), in smc_wr_alloc_link_mem()
806 if (!link->wr_tx_v2_pend) in smc_wr_alloc_link_mem()
812 kfree(link->wr_tx_v2_sge); in smc_wr_alloc_link_mem()
814 kfree(link->wr_tx_v2_ib); in smc_wr_alloc_link_mem()
816 kfree(link->wr_tx_compl); in smc_wr_alloc_link_mem()
818 kfree(link->wr_tx_pends); in smc_wr_alloc_link_mem()
820 kfree(link->wr_tx_mask); in smc_wr_alloc_link_mem()
822 kfree(link->wr_rx_sges); in smc_wr_alloc_link_mem()
824 kfree(link->wr_tx_sges); in smc_wr_alloc_link_mem()
826 kfree(link->wr_tx_rdma_sges); in smc_wr_alloc_link_mem()
828 kfree(link->wr_tx_rdmas); in smc_wr_alloc_link_mem()
830 kfree(link->wr_rx_ibs); in smc_wr_alloc_link_mem()
832 kfree(link->wr_tx_ibs); in smc_wr_alloc_link_mem()
834 kfree(link->wr_rx_bufs); in smc_wr_alloc_link_mem()
836 kfree(link->wr_tx_bufs); in smc_wr_alloc_link_mem()
838 return -ENOMEM; in smc_wr_alloc_link_mem()
843 tasklet_kill(&smcibdev->recv_tasklet); in smc_wr_remove_dev()
844 tasklet_kill(&smcibdev->send_tasklet); in smc_wr_remove_dev()
849 tasklet_setup(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn); in smc_wr_add_dev()
850 tasklet_setup(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn); in smc_wr_add_dev()
857 complete(&lnk->tx_ref_comp); in smcr_wr_tx_refs_free()
864 complete(&lnk->reg_ref_comp); in smcr_wr_reg_refs_free()
869 struct ib_device *ibdev = lnk->smcibdev->ibdev; in smc_wr_create_link()
872 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); in smc_wr_create_link()
873 lnk->wr_rx_id = 0; in smc_wr_create_link()
874 lnk->wr_rx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
875 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
877 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { in smc_wr_create_link()
878 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()
879 rc = -EIO; in smc_wr_create_link()
882 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_create_link()
883 lnk->wr_rx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
884 lnk->lgr->wr_rx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
886 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_v2_dma_addr)) { in smc_wr_create_link()
887 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
888 rc = -EIO; in smc_wr_create_link()
891 lnk->wr_tx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
892 lnk->lgr->wr_tx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
894 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_v2_dma_addr)) { in smc_wr_create_link()
895 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
896 rc = -EIO; in smc_wr_create_link()
900 lnk->wr_tx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
901 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_create_link()
903 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { in smc_wr_create_link()
904 rc = -EIO; in smc_wr_create_link()
908 bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT); in smc_wr_create_link()
909 init_waitqueue_head(&lnk->wr_tx_wait); in smc_wr_create_link()
910 rc = percpu_ref_init(&lnk->wr_tx_refs, smcr_wr_tx_refs_free, 0, GFP_KERNEL); in smc_wr_create_link()
913 init_completion(&lnk->tx_ref_comp); in smc_wr_create_link()
914 init_waitqueue_head(&lnk->wr_reg_wait); in smc_wr_create_link()
915 rc = percpu_ref_init(&lnk->wr_reg_refs, smcr_wr_reg_refs_free, 0, GFP_KERNEL); in smc_wr_create_link()
918 init_completion(&lnk->reg_ref_comp); in smc_wr_create_link()
919 init_waitqueue_head(&lnk->wr_rx_empty_wait); in smc_wr_create_link()
923 percpu_ref_exit(&lnk->wr_tx_refs); in smc_wr_create_link()
925 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_create_link()
926 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_create_link()
929 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
931 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_create_link()
932 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_create_link()
935 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
937 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_create_link()
938 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
940 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()