Lines Matching full:lnk
523 void smc_wr_remember_qp_attr(struct smc_link *lnk) in smc_wr_remember_qp_attr() argument
525 struct ib_qp_attr *attr = &lnk->qp_attr; in smc_wr_remember_qp_attr()
530 ib_query_qp(lnk->roce_qp, attr, in smc_wr_remember_qp_attr()
550 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, in smc_wr_remember_qp_attr()
551 lnk->qp_attr.cap.max_send_wr); in smc_wr_remember_qp_attr()
552 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, in smc_wr_remember_qp_attr()
553 lnk->qp_attr.cap.max_recv_wr); in smc_wr_remember_qp_attr()
556 static void smc_wr_init_sge(struct smc_link *lnk) in smc_wr_init_sge() argument
558 int sges_per_buf = (lnk->lgr->smc_version == SMC_V2) ? 2 : 1; in smc_wr_init_sge()
559 bool send_inline = (lnk->qp_attr.cap.max_inline_data > SMC_WR_TX_SIZE); in smc_wr_init_sge()
562 for (i = 0; i < lnk->wr_tx_cnt; i++) { in smc_wr_init_sge()
563 lnk->wr_tx_sges[i].addr = send_inline ? (uintptr_t)(&lnk->wr_tx_bufs[i]) : in smc_wr_init_sge()
564 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
565 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
566 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
567 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
568 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
569 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
570 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
571 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
572 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
573 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
574 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
575 lnk->wr_tx_ibs[i].next = NULL; in smc_wr_init_sge()
576 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; in smc_wr_init_sge()
577 lnk->wr_tx_ibs[i].num_sge = 1; in smc_wr_init_sge()
578 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; in smc_wr_init_sge()
579 lnk->wr_tx_ibs[i].send_flags = in smc_wr_init_sge()
582 lnk->wr_tx_ibs[i].send_flags |= IB_SEND_INLINE; in smc_wr_init_sge()
583 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
584 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
585 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list = in smc_wr_init_sge()
586 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge; in smc_wr_init_sge()
587 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list = in smc_wr_init_sge()
588 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge; in smc_wr_init_sge()
591 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
592 lnk->wr_tx_v2_sge->addr = lnk->wr_tx_v2_dma_addr; in smc_wr_init_sge()
593 lnk->wr_tx_v2_sge->length = SMC_WR_BUF_V2_SIZE; in smc_wr_init_sge()
594 lnk->wr_tx_v2_sge->lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
596 lnk->wr_tx_v2_ib->next = NULL; in smc_wr_init_sge()
597 lnk->wr_tx_v2_ib->sg_list = lnk->wr_tx_v2_sge; in smc_wr_init_sge()
598 lnk->wr_tx_v2_ib->num_sge = 1; in smc_wr_init_sge()
599 lnk->wr_tx_v2_ib->opcode = IB_WR_SEND; in smc_wr_init_sge()
600 lnk->wr_tx_v2_ib->send_flags = in smc_wr_init_sge()
610 for (i = 0; i < lnk->wr_rx_cnt; i++) { in smc_wr_init_sge()
613 lnk->wr_rx_sges[x].addr = in smc_wr_init_sge()
614 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
615 lnk->wr_rx_sges[x].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
616 lnk->wr_rx_sges[x].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
617 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
618 lnk->wr_rx_sges[x + 1].addr = in smc_wr_init_sge()
619 lnk->wr_rx_v2_dma_addr + SMC_WR_TX_SIZE; in smc_wr_init_sge()
620 lnk->wr_rx_sges[x + 1].length = in smc_wr_init_sge()
622 lnk->wr_rx_sges[x + 1].lkey = in smc_wr_init_sge()
623 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
625 lnk->wr_rx_ibs[i].next = NULL; in smc_wr_init_sge()
626 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[x]; in smc_wr_init_sge()
627 lnk->wr_rx_ibs[i].num_sge = sges_per_buf; in smc_wr_init_sge()
629 lnk->wr_reg.wr.next = NULL; in smc_wr_init_sge()
630 lnk->wr_reg.wr.num_sge = 0; in smc_wr_init_sge()
631 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; in smc_wr_init_sge()
632 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; in smc_wr_init_sge()
633 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; in smc_wr_init_sge()
636 void smc_wr_free_link(struct smc_link *lnk) in smc_wr_free_link() argument
640 if (!lnk->smcibdev) in smc_wr_free_link()
642 ibdev = lnk->smcibdev->ibdev; in smc_wr_free_link()
644 smc_wr_drain_cq(lnk); in smc_wr_free_link()
645 smc_wr_wakeup_reg_wait(lnk); in smc_wr_free_link()
646 smc_wr_wakeup_tx_wait(lnk); in smc_wr_free_link()
648 smc_wr_tx_wait_no_pending_sends(lnk); in smc_wr_free_link()
649 percpu_ref_kill(&lnk->wr_reg_refs); in smc_wr_free_link()
650 wait_for_completion(&lnk->reg_ref_comp); in smc_wr_free_link()
651 percpu_ref_exit(&lnk->wr_reg_refs); in smc_wr_free_link()
652 percpu_ref_kill(&lnk->wr_tx_refs); in smc_wr_free_link()
653 wait_for_completion(&lnk->tx_ref_comp); in smc_wr_free_link()
654 percpu_ref_exit(&lnk->wr_tx_refs); in smc_wr_free_link()
656 if (lnk->wr_rx_dma_addr) { in smc_wr_free_link()
657 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_free_link()
658 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_free_link()
660 lnk->wr_rx_dma_addr = 0; in smc_wr_free_link()
662 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_free_link()
663 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_free_link()
666 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_free_link()
668 if (lnk->wr_tx_dma_addr) { in smc_wr_free_link()
669 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, in smc_wr_free_link()
670 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_free_link()
672 lnk->wr_tx_dma_addr = 0; in smc_wr_free_link()
674 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_free_link()
675 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_free_link()
678 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_free_link()
693 void smc_wr_free_link_mem(struct smc_link *lnk) in smc_wr_free_link_mem() argument
695 kfree(lnk->wr_tx_v2_ib); in smc_wr_free_link_mem()
696 lnk->wr_tx_v2_ib = NULL; in smc_wr_free_link_mem()
697 kfree(lnk->wr_tx_v2_sge); in smc_wr_free_link_mem()
698 lnk->wr_tx_v2_sge = NULL; in smc_wr_free_link_mem()
699 kfree(lnk->wr_tx_v2_pend); in smc_wr_free_link_mem()
700 lnk->wr_tx_v2_pend = NULL; in smc_wr_free_link_mem()
701 kfree(lnk->wr_tx_compl); in smc_wr_free_link_mem()
702 lnk->wr_tx_compl = NULL; in smc_wr_free_link_mem()
703 kfree(lnk->wr_tx_pends); in smc_wr_free_link_mem()
704 lnk->wr_tx_pends = NULL; in smc_wr_free_link_mem()
705 bitmap_free(lnk->wr_tx_mask); in smc_wr_free_link_mem()
706 lnk->wr_tx_mask = NULL; in smc_wr_free_link_mem()
707 kfree(lnk->wr_tx_sges); in smc_wr_free_link_mem()
708 lnk->wr_tx_sges = NULL; in smc_wr_free_link_mem()
709 kfree(lnk->wr_tx_rdma_sges); in smc_wr_free_link_mem()
710 lnk->wr_tx_rdma_sges = NULL; in smc_wr_free_link_mem()
711 kfree(lnk->wr_rx_sges); in smc_wr_free_link_mem()
712 lnk->wr_rx_sges = NULL; in smc_wr_free_link_mem()
713 kfree(lnk->wr_tx_rdmas); in smc_wr_free_link_mem()
714 lnk->wr_tx_rdmas = NULL; in smc_wr_free_link_mem()
715 kfree(lnk->wr_rx_ibs); in smc_wr_free_link_mem()
716 lnk->wr_rx_ibs = NULL; in smc_wr_free_link_mem()
717 kfree(lnk->wr_tx_ibs); in smc_wr_free_link_mem()
718 lnk->wr_tx_ibs = NULL; in smc_wr_free_link_mem()
719 kfree(lnk->wr_tx_bufs); in smc_wr_free_link_mem()
720 lnk->wr_tx_bufs = NULL; in smc_wr_free_link_mem()
721 kfree(lnk->wr_rx_bufs); in smc_wr_free_link_mem()
722 lnk->wr_rx_bufs = NULL; in smc_wr_free_link_mem()
855 struct smc_link *lnk = container_of(ref, struct smc_link, wr_tx_refs); in smcr_wr_tx_refs_free() local
857 complete(&lnk->tx_ref_comp); in smcr_wr_tx_refs_free()
862 struct smc_link *lnk = container_of(ref, struct smc_link, wr_reg_refs); in smcr_wr_reg_refs_free() local
864 complete(&lnk->reg_ref_comp); in smcr_wr_reg_refs_free()
867 int smc_wr_create_link(struct smc_link *lnk) in smc_wr_create_link() argument
869 struct ib_device *ibdev = lnk->smcibdev->ibdev; in smc_wr_create_link()
872 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); in smc_wr_create_link()
873 lnk->wr_rx_id = 0; in smc_wr_create_link()
874 lnk->wr_rx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
875 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
877 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { in smc_wr_create_link()
878 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()
882 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_create_link()
883 lnk->wr_rx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
884 lnk->lgr->wr_rx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
886 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_v2_dma_addr)) { in smc_wr_create_link()
887 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
891 lnk->wr_tx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
892 lnk->lgr->wr_tx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
894 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_v2_dma_addr)) { in smc_wr_create_link()
895 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
900 lnk->wr_tx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
901 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_create_link()
903 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { in smc_wr_create_link()
907 smc_wr_init_sge(lnk); in smc_wr_create_link()
908 bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT); in smc_wr_create_link()
909 init_waitqueue_head(&lnk->wr_tx_wait); in smc_wr_create_link()
910 rc = percpu_ref_init(&lnk->wr_tx_refs, smcr_wr_tx_refs_free, 0, GFP_KERNEL); in smc_wr_create_link()
913 init_completion(&lnk->tx_ref_comp); in smc_wr_create_link()
914 init_waitqueue_head(&lnk->wr_reg_wait); in smc_wr_create_link()
915 rc = percpu_ref_init(&lnk->wr_reg_refs, smcr_wr_reg_refs_free, 0, GFP_KERNEL); in smc_wr_create_link()
918 init_completion(&lnk->reg_ref_comp); in smc_wr_create_link()
919 init_waitqueue_head(&lnk->wr_rx_empty_wait); in smc_wr_create_link()
923 percpu_ref_exit(&lnk->wr_tx_refs); in smc_wr_create_link()
925 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_create_link()
926 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_create_link()
929 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
931 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_create_link()
932 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_create_link()
935 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
937 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_create_link()
938 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
940 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()