71 static void smc_ibdev_cnt_inc(struct smc_link *lnk)  in smc_ibdev_cnt_inc()  argument
73 	atomic_inc(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);  in smc_ibdev_cnt_inc()
76 static void smc_ibdev_cnt_dec(struct smc_link *lnk)  in smc_ibdev_cnt_dec()  argument
78 	atomic_dec(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);  in smc_ibdev_cnt_dec()
130 	conn->lnk = NULL;	/* reset conn->lnk first */  in smcr_lgr_conn_assign_link()
132 		struct smc_link *lnk = &conn->lgr->lnk[i];  in smcr_lgr_conn_assign_link()  local
134 		if (lnk->state != expected || lnk->link_is_asym)  in smcr_lgr_conn_assign_link()
137 			conn->lnk = lnk; /* temporary, SMC server assigns link*/  in smcr_lgr_conn_assign_link()
144 				lnk2 = &conn->lgr->lnk[j];  in smcr_lgr_conn_assign_link()
147 					conn->lnk = lnk2;  in smcr_lgr_conn_assign_link()
152 		if (!conn->lnk)  in smcr_lgr_conn_assign_link()
153 			conn->lnk = lnk;  in smcr_lgr_conn_assign_link()
156 	if (!conn->lnk)  in smcr_lgr_conn_assign_link()
158 	atomic_inc(&conn->lnk->conn_cnt);  in smcr_lgr_conn_assign_link()
202 	if (conn->lnk)  in __smc_lgr_unregister_conn()
203 		atomic_dec(&conn->lnk->conn_cnt);  in __smc_lgr_unregister_conn()
465 		if (!smc_link_usable(&lgr->lnk[i]))  in smc_nl_handle_lgr()
467 		if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb))  in smc_nl_handle_lgr()
660 		struct smc_link *lnk = &lgr->lnk[i];  in smcr_lgr_link_deactivate_all()  local
662 		if (smc_link_sendable(lnk))  in smcr_lgr_link_deactivate_all()
663 			lnk->state = SMC_LNK_INACTIVE;  in smcr_lgr_link_deactivate_all()
727 			if (smc_link_usable(&lgr->lnk[i]) &&  in smcr_next_link_id()
728 			    lgr->lnk[i].link_id == link_id)  in smcr_next_link_id()
745 int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk,  in smcr_link_init()  argument
753 		lnk->smcibdev = ini->smcrv2.ib_dev_v2;  in smcr_link_init()
754 		lnk->ibport = ini->smcrv2.ib_port_v2;  in smcr_link_init()
756 		lnk->smcibdev = ini->ib_dev;  in smcr_link_init()
757 		lnk->ibport = ini->ib_port;  in smcr_link_init()
759 	get_device(&lnk->smcibdev->ibdev->dev);  in smcr_link_init()
760 	atomic_inc(&lnk->smcibdev->lnk_cnt);  in smcr_link_init()
761 	refcount_set(&lnk->refcnt, 1); /* link refcnt is set to 1 */  in smcr_link_init()
762 	lnk->clearing = 0;  in smcr_link_init()
763 	lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu;  in smcr_link_init()
764 	lnk->link_id = smcr_next_link_id(lgr);  in smcr_link_init()
765 	lnk->lgr = lgr;  in smcr_link_init()
767 	lnk->link_idx = link_idx;  in smcr_link_init()
768 	lnk->wr_rx_id_compl = 0;  in smcr_link_init()
769 	smc_ibdev_cnt_inc(lnk);  in smcr_link_init()
770 	smcr_copy_dev_info_to_link(lnk);  in smcr_link_init()
771 	atomic_set(&lnk->conn_cnt, 0);  in smcr_link_init()
772 	smc_llc_link_set_uid(lnk);  in smcr_link_init()
773 	INIT_WORK(&lnk->link_down_wrk, smc_link_down_work);  in smcr_link_init()
774 	if (!lnk->smcibdev->initialized) {  in smcr_link_init()
775 		rc = (int)smc_ib_setup_per_ibdev(lnk->smcibdev);  in smcr_link_init()
780 	lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +  in smcr_link_init()
782 	rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,  in smcr_link_init()
783 				  ini->vlan_id, lnk->gid, &lnk->sgid_index,  in smcr_link_init()
788 	rc = smc_llc_link_init(lnk);  in smcr_link_init()
791 	rc = smc_wr_alloc_link_mem(lnk);  in smcr_link_init()
794 	rc = smc_ib_create_protection_domain(lnk);  in smcr_link_init()
797 	rc = smc_ib_create_queue_pair(lnk);  in smcr_link_init()
800 	rc = smc_wr_create_link(lnk);  in smcr_link_init()
803 	lnk->state = SMC_LNK_ACTIVATING;  in smcr_link_init()
807 	smc_ib_destroy_queue_pair(lnk);  in smcr_link_init()
809 	smc_ib_dealloc_protection_domain(lnk);  in smcr_link_init()
811 	smc_wr_free_link_mem(lnk);  in smcr_link_init()
813 	smc_llc_link_clear(lnk, false);  in smcr_link_init()
815 	smc_ibdev_cnt_dec(lnk);  in smcr_link_init()
816 	put_device(&lnk->smcibdev->ibdev->dev);  in smcr_link_init()
817 	smcibdev = lnk->smcibdev;  in smcr_link_init()
818 	memset(lnk, 0, sizeof(struct smc_link));  in smcr_link_init()
819 	lnk->state = SMC_LNK_UNUSED;  in smcr_link_init()
832 	struct smc_link *lnk;  in smc_lgr_create()  local
922 		lnk = &lgr->lnk[link_idx];  in smc_lgr_create()
923 		rc = smcr_link_init(lgr, lnk, link_idx, ini);  in smc_lgr_create()
928 		lgr->net = smc_ib_net(lnk->smcibdev);  in smc_lgr_create()
1015 		smc_wr_tx_put_slot(conn->lnk,  in smc_switch_cursor()
1024 	atomic_dec(&conn->lnk->conn_cnt);  in smc_switch_link_and_count()
1026 	smcr_link_put(conn->lnk);  in smc_switch_link_and_count()
1027 	conn->lnk = to_lnk;  in smc_switch_link_and_count()
1028 	atomic_inc(&conn->lnk->conn_cnt);  in smc_switch_link_and_count()
1030 	smcr_link_hold(conn->lnk);  in smc_switch_link_and_count()
1048 		if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx)  in smc_switch_conns()
1050 		if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev &&  in smc_switch_conns()
1051 		    from_lnk->ibport == lgr->lnk[i].ibport) {  in smc_switch_conns()
1054 		to_lnk = &lgr->lnk[i];  in smc_switch_conns()
1065 		if (conn->lnk != from_lnk)  in smc_switch_conns()
1068 		/* conn->lnk not yet set in SMC_INIT state */  in smc_switch_conns()
1220 		smcr_link_put(conn->lnk); /* link_hold in smc_conn_create() */  in smc_conn_free()
1226 				struct smc_link *lnk)  in smcr_buf_unmap_link()  argument
1229 		buf_desc->is_reg_mr[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1230 	if (!buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_unmap_link()
1234 	    buf_desc->mr[lnk->link_idx]) {  in smcr_buf_unmap_link()
1235 		smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]);  in smcr_buf_unmap_link()
1236 		buf_desc->mr[lnk->link_idx] = NULL;  in smcr_buf_unmap_link()
1239 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE);  in smcr_buf_unmap_link()
1241 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE);  in smcr_buf_unmap_link()
1243 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_unmap_link()
1244 	buf_desc->is_map_ib[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1248 static void smcr_buf_unmap_lgr(struct smc_link *lnk)  in smcr_buf_unmap_lgr()  argument
1250 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_unmap_lgr()
1257 			smcr_buf_unmap_link(buf_desc, true, lnk);  in smcr_buf_unmap_lgr()
1263 			smcr_buf_unmap_link(buf_desc, false, lnk);  in smcr_buf_unmap_lgr()
1268 static void smcr_rtoken_clear_link(struct smc_link *lnk)  in smcr_rtoken_clear_link()  argument
1270 	struct smc_link_group *lgr = lnk->lgr;  in smcr_rtoken_clear_link()
1274 		lgr->rtokens[i][lnk->link_idx].rkey = 0;  in smcr_rtoken_clear_link()
1275 		lgr->rtokens[i][lnk->link_idx].dma_addr = 0;  in smcr_rtoken_clear_link()
1279 static void __smcr_link_clear(struct smc_link *lnk)  in __smcr_link_clear()  argument
1281 	struct smc_link_group *lgr = lnk->lgr;  in __smcr_link_clear()
1284 	smc_wr_free_link_mem(lnk);  in __smcr_link_clear()
1285 	smc_ibdev_cnt_dec(lnk);  in __smcr_link_clear()
1286 	put_device(&lnk->smcibdev->ibdev->dev);  in __smcr_link_clear()
1287 	smcibdev = lnk->smcibdev;  in __smcr_link_clear()
1288 	memset(lnk, 0, sizeof(struct smc_link));  in __smcr_link_clear()
1289 	lnk->state = SMC_LNK_UNUSED;  in __smcr_link_clear()
1296 void smcr_link_clear(struct smc_link *lnk, bool log)  in smcr_link_clear()  argument
1298 	if (!lnk->lgr || lnk->clearing ||  in smcr_link_clear()
1299 	    lnk->state == SMC_LNK_UNUSED)  in smcr_link_clear()
1301 	lnk->clearing = 1;  in smcr_link_clear()
1302 	lnk->peer_qpn = 0;  in smcr_link_clear()
1303 	smc_llc_link_clear(lnk, log);  in smcr_link_clear()
1304 	smcr_buf_unmap_lgr(lnk);  in smcr_link_clear()
1305 	smcr_rtoken_clear_link(lnk);  in smcr_link_clear()
1306 	smc_ib_modify_qp_error(lnk);  in smcr_link_clear()
1307 	smc_wr_free_link(lnk);  in smcr_link_clear()
1308 	smc_ib_destroy_queue_pair(lnk);  in smcr_link_clear()
1309 	smc_ib_dealloc_protection_domain(lnk);  in smcr_link_clear()
1310 	smcr_link_put(lnk); /* theoretically last link_put */  in smcr_link_clear()
1313 void smcr_link_hold(struct smc_link *lnk)  in smcr_link_hold()  argument
1315 	refcount_inc(&lnk->refcnt);  in smcr_link_hold()
1318 void smcr_link_put(struct smc_link *lnk)  in smcr_link_put()  argument
1320 	if (refcount_dec_and_test(&lnk->refcnt))  in smcr_link_put()
1321 		__smcr_link_clear(lnk);  in smcr_link_put()
1330 		smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]);  in smcr_buf_free()
1411 			if (lgr->lnk[i].state != SMC_LNK_UNUSED)  in smc_lgr_free()
1412 				smcr_link_clear(&lgr->lnk[i], false);  in smc_lgr_free()
1607 				if (lgr->lnk[i].smcibdev == smcibdev)  in smc_smcr_terminate_all()
1608 					smcr_link_down_cond_sched(&lgr->lnk[i]);  in smc_smcr_terminate_all()
1637 		if (smc_link_usable(&lgr->lnk[i]))  in smcr_lgr_set_type()
1638 			lgr->lnk[i].link_is_asym = false;  in smcr_lgr_set_type()
1670 	lgr->lnk[asym_lnk_idx].link_is_asym = true;  in smcr_lgr_set_type_asym()
1716 static void smcr_link_down(struct smc_link *lnk)  in smcr_link_down()  argument
1718 	struct smc_link_group *lgr = lnk->lgr;  in smcr_link_down()
1722 	if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list))  in smcr_link_down()
1725 	to_lnk = smc_switch_conns(lgr, lnk, true);  in smcr_link_down()
1727 		smcr_link_clear(lnk, true);  in smcr_link_down()
1731 	del_link_id = lnk->link_id;  in smcr_link_down()
1750 			smcr_link_clear(lnk, true);  in smcr_link_down()
1757 void smcr_link_down_cond(struct smc_link *lnk)  in smcr_link_down_cond()  argument
1759 	if (smc_link_downing(&lnk->state)) {  in smcr_link_down_cond()
1760 		trace_smcr_link_down(lnk, __builtin_return_address(0));  in smcr_link_down_cond()
1761 		smcr_link_down(lnk);  in smcr_link_down_cond()
1766 void smcr_link_down_cond_sched(struct smc_link *lnk)  in smcr_link_down_cond_sched()  argument
1768 	if (smc_link_downing(&lnk->state)) {  in smcr_link_down_cond_sched()
1769 		trace_smcr_link_down(lnk, __builtin_return_address(0));  in smcr_link_down_cond_sched()
1770 		smcr_link_hold(lnk); /* smcr_link_put in link_down_wrk */  in smcr_link_down_cond_sched()
1771 		if (!schedule_work(&lnk->link_down_wrk))  in smcr_link_down_cond_sched()
1772 			smcr_link_put(lnk);  in smcr_link_down_cond_sched()
1788 			struct smc_link *lnk = &lgr->lnk[i];  in smcr_port_err()  local
1790 			if (smc_link_usable(lnk) &&  in smcr_port_err()
1791 			    lnk->smcibdev == smcibdev && lnk->ibport == ibport)  in smcr_port_err()
1792 				smcr_link_down_cond_sched(lnk);  in smcr_port_err()
1869 	struct smc_link *lnk;  in smcr_lgr_match()  local
1877 		lnk = &lgr->lnk[i];  in smcr_lgr_match()
1879 		if (!smc_link_active(lnk))  in smcr_lgr_match()
1882 		if (!rdma_dev_access_netns(lnk->smcibdev->ibdev, net))  in smcr_lgr_match()
1884 		if ((lgr->role == SMC_SERV || lnk->peer_qpn == clcqpn) &&  in smcr_lgr_match()
1885 		    !memcmp(lnk->peer_gid, peer_gid, SMC_GID_SIZE) &&  in smcr_lgr_match()
1887 		     !memcmp(lnk->peer_mac, peer_mac_v1, ETH_ALEN)))  in smcr_lgr_match()
1986 		smcr_link_hold(conn->lnk); /* link_put in smc_conn_free() */  in smc_conn_create()
2075 			     struct smc_link *lnk)  in smcr_buf_map_link()  argument
2081 	if (buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_map_link()
2093 	rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL);  in smcr_buf_map_link()
2099 		for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) {  in smcr_buf_map_link()
2108 		sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl,  in smcr_buf_map_link()
2113 	rc = smc_ib_buf_map_sg(lnk, buf_desc,  in smcr_buf_map_link()
2122 		smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx;  in smcr_buf_map_link()
2130 		rc = smc_ib_get_memory_region(lnk->roce_pd, access_flags,  in smcr_buf_map_link()
2131 					      buf_desc, lnk->link_idx);  in smcr_buf_map_link()
2134 		smc_ib_sync_sg_for_device(lnk, buf_desc,  in smcr_buf_map_link()
2137 	buf_desc->is_map_ib[lnk->link_idx] = true;  in smcr_buf_map_link()
2141 	smc_ib_buf_unmap_sg(lnk, buf_desc,  in smcr_buf_map_link()
2144 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_map_link()
2169 static int _smcr_buf_map_lgr(struct smc_link *lnk, struct rw_semaphore *lock,  in _smcr_buf_map_lgr()  argument
2179 		rc = smcr_buf_map_link(buf_desc, is_rmb, lnk);  in _smcr_buf_map_lgr()
2189 int smcr_buf_map_lgr(struct smc_link *lnk)  in smcr_buf_map_lgr()  argument
2191 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_map_lgr()
2195 		rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock,  in smcr_buf_map_lgr()
2199 		rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock,  in smcr_buf_map_lgr()
2210 int smcr_buf_reg_lgr(struct smc_link *lnk)  in smcr_buf_reg_lgr()  argument
2212 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_reg_lgr()
2222 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2240 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2307 		struct smc_link *lnk = &lgr->lnk[i];  in smcr_buf_map_usable_links()  local
2309 		if (!smc_link_usable(lnk))  in smcr_buf_map_usable_links()
2311 		if (smcr_buf_map_link(buf_desc, is_rmb, lnk)) {  in smcr_buf_map_usable_links()
2453 	    !smc_link_active(conn->lnk))  in smc_sndbuf_sync_sg_for_device()
2455 	smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE);  in smc_sndbuf_sync_sg_for_device()
2467 		if (!smc_link_active(&conn->lgr->lnk[i]))  in smc_rmb_sync_sg_for_cpu()
2469 		smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc,  in smc_rmb_sync_sg_for_cpu()
2588 		if (lgr->lnk[link_idx].link_id == link_id) {  in smc_rtoken_set2()
2600 int smc_rtoken_add(struct smc_link *lnk, __be64 nw_vaddr, __be32 nw_rkey)  in smc_rtoken_add()  argument
2602 	struct smc_link_group *lgr = smc_get_lgr(lnk);  in smc_rtoken_add()
2608 		if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&  in smc_rtoken_add()
2609 		    lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr &&  in smc_rtoken_add()
2618 	lgr->rtokens[i][lnk->link_idx].rkey = rkey;  in smc_rtoken_add()
2619 	lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr;  in smc_rtoken_add()
2624 int smc_rtoken_delete(struct smc_link *lnk, __be32 nw_rkey)  in smc_rtoken_delete()  argument
2626 	struct smc_link_group *lgr = smc_get_lgr(lnk);  in smc_rtoken_delete()
2631 		if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&  in smc_rtoken_delete()
2646 			    struct smc_link *lnk,  in smc_rmb_rtoken_handling()  argument
2649 	conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr,  in smc_rmb_rtoken_handling()