1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2a4cf0443SUrsula Braun /* 3a4cf0443SUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 4a4cf0443SUrsula Braun * 5a4cf0443SUrsula Braun * IB infrastructure: 6a4cf0443SUrsula Braun * Establish SMC-R as an Infiniband Client to be notified about added and 7a4cf0443SUrsula Braun * removed IB devices of type RDMA. 8a4cf0443SUrsula Braun * Determine device and port characteristics for these IB devices. 9a4cf0443SUrsula Braun * 10a4cf0443SUrsula Braun * Copyright IBM Corp. 2016 11a4cf0443SUrsula Braun * 12a4cf0443SUrsula Braun * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 13a4cf0443SUrsula Braun */ 14a4cf0443SUrsula Braun 15a4cf0443SUrsula Braun #include <linux/random.h> 16bd4ad577SUrsula Braun #include <linux/workqueue.h> 1710428dd8SUrsula Braun #include <linux/scatterlist.h> 186dabd405SUrsula Braun #include <linux/wait.h> 1992f3cb0eSUrsula Braun #include <linux/mutex.h> 20*24fb6811SKarsten Graul #include <linux/inetdevice.h> 21a4cf0443SUrsula Braun #include <rdma/ib_verbs.h> 22ddb457c6SParav Pandit #include <rdma/ib_cache.h> 23a4cf0443SUrsula Braun 246812baabSThomas Richter #include "smc_pnet.h" 25a4cf0443SUrsula Braun #include "smc_ib.h" 26cd6851f3SUrsula Braun #include "smc_core.h" 27f38ba179SUrsula Braun #include "smc_wr.h" 28a4cf0443SUrsula Braun #include "smc.h" 29a3db10efSGuvenc Gulce #include "smc_netlink.h" 30a4cf0443SUrsula Braun 31c9f4c6cfSUrsula Braun #define SMC_MAX_CQE 32766 /* max. # of completion queue elements */ 32c9f4c6cfSUrsula Braun 33bd4ad577SUrsula Braun #define SMC_QP_MIN_RNR_TIMER 5 34bd4ad577SUrsula Braun #define SMC_QP_TIMEOUT 15 /* 4096 * 2 ** timeout usec */ 35bd4ad577SUrsula Braun #define SMC_QP_RETRY_CNT 7 /* 7: infinite */ 36bd4ad577SUrsula Braun #define SMC_QP_RNR_RETRY 7 /* 7: infinite */ 37bd4ad577SUrsula Braun 38a4cf0443SUrsula Braun struct smc_ib_devices smc_ib_devices = { /* smc-registered ib devices */ 3992f3cb0eSUrsula Braun .mutex = __MUTEX_INITIALIZER(smc_ib_devices.mutex), 40a4cf0443SUrsula Braun .list = LIST_HEAD_INIT(smc_ib_devices.list), 41a4cf0443SUrsula Braun }; 42a4cf0443SUrsula Braun 43366bb249SHans Wippel u8 local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */ 44a4cf0443SUrsula Braun 45bd4ad577SUrsula Braun static int smc_ib_modify_qp_init(struct smc_link *lnk) 46bd4ad577SUrsula Braun { 47bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 48bd4ad577SUrsula Braun 49bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 50bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_INIT; 51bd4ad577SUrsula Braun qp_attr.pkey_index = 0; 52bd4ad577SUrsula Braun qp_attr.port_num = lnk->ibport; 53bd4ad577SUrsula Braun qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE 54bd4ad577SUrsula Braun | IB_ACCESS_REMOTE_WRITE; 55bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, 56bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_PKEY_INDEX | 57bd4ad577SUrsula Braun IB_QP_ACCESS_FLAGS | IB_QP_PORT); 58bd4ad577SUrsula Braun } 59bd4ad577SUrsula Braun 60bd4ad577SUrsula Braun static int smc_ib_modify_qp_rtr(struct smc_link *lnk) 61bd4ad577SUrsula Braun { 62bd4ad577SUrsula Braun enum ib_qp_attr_mask qp_attr_mask = 63bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | 64bd4ad577SUrsula Braun IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; 65bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 66*24fb6811SKarsten Graul u8 hop_lim = 1; 67bd4ad577SUrsula Braun 68bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 69bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RTR; 70bd4ad577SUrsula Braun qp_attr.path_mtu = min(lnk->path_mtu, lnk->peer_mtu); 7144c58487SDasaratharaman Chandramouli qp_attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 72d8966fcdSDasaratharaman Chandramouli rdma_ah_set_port_num(&qp_attr.ah_attr, lnk->ibport); 73*24fb6811SKarsten Graul if (lnk->lgr->smc_version == SMC_V2 && lnk->lgr->uses_gateway) 74*24fb6811SKarsten Graul hop_lim = IPV6_DEFAULT_HOPLIMIT; 75*24fb6811SKarsten Graul rdma_ah_set_grh(&qp_attr.ah_attr, NULL, 0, lnk->sgid_index, hop_lim, 0); 76d8966fcdSDasaratharaman Chandramouli rdma_ah_set_dgid_raw(&qp_attr.ah_attr, lnk->peer_gid); 77*24fb6811SKarsten Graul if (lnk->lgr->smc_version == SMC_V2 && lnk->lgr->uses_gateway) 78*24fb6811SKarsten Graul memcpy(&qp_attr.ah_attr.roce.dmac, lnk->lgr->nexthop_mac, 79*24fb6811SKarsten Graul sizeof(lnk->lgr->nexthop_mac)); 80*24fb6811SKarsten Graul else 8144c58487SDasaratharaman Chandramouli memcpy(&qp_attr.ah_attr.roce.dmac, lnk->peer_mac, 82bd4ad577SUrsula Braun sizeof(lnk->peer_mac)); 83bd4ad577SUrsula Braun qp_attr.dest_qp_num = lnk->peer_qpn; 84bd4ad577SUrsula Braun qp_attr.rq_psn = lnk->peer_psn; /* starting receive packet seq # */ 85bd4ad577SUrsula Braun qp_attr.max_dest_rd_atomic = 1; /* max # of resources for incoming 86bd4ad577SUrsula Braun * requests 87bd4ad577SUrsula Braun */ 88bd4ad577SUrsula Braun qp_attr.min_rnr_timer = SMC_QP_MIN_RNR_TIMER; 89bd4ad577SUrsula Braun 90bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, qp_attr_mask); 91bd4ad577SUrsula Braun } 92bd4ad577SUrsula Braun 93bd4ad577SUrsula Braun int smc_ib_modify_qp_rts(struct smc_link *lnk) 94bd4ad577SUrsula Braun { 95bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 96bd4ad577SUrsula Braun 97bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 98bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RTS; 99bd4ad577SUrsula Braun qp_attr.timeout = SMC_QP_TIMEOUT; /* local ack timeout */ 100bd4ad577SUrsula Braun qp_attr.retry_cnt = SMC_QP_RETRY_CNT; /* retry count */ 101bd4ad577SUrsula Braun qp_attr.rnr_retry = SMC_QP_RNR_RETRY; /* RNR retries, 7=infinite */ 102bd4ad577SUrsula Braun qp_attr.sq_psn = lnk->psn_initial; /* starting send packet seq # */ 103bd4ad577SUrsula Braun qp_attr.max_rd_atomic = 1; /* # of outstanding RDMA reads and 104bd4ad577SUrsula Braun * atomic ops allowed 105bd4ad577SUrsula Braun */ 106bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, 107bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | 108bd4ad577SUrsula Braun IB_QP_SQ_PSN | IB_QP_RNR_RETRY | 109bd4ad577SUrsula Braun IB_QP_MAX_QP_RD_ATOMIC); 110bd4ad577SUrsula Braun } 111bd4ad577SUrsula Braun 112bd4ad577SUrsula Braun int smc_ib_modify_qp_reset(struct smc_link *lnk) 113bd4ad577SUrsula Braun { 114bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 115bd4ad577SUrsula Braun 116bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 117bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RESET; 118bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, IB_QP_STATE); 119bd4ad577SUrsula Braun } 120bd4ad577SUrsula Braun 121bd4ad577SUrsula Braun int smc_ib_ready_link(struct smc_link *lnk) 122bd4ad577SUrsula Braun { 12300e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(lnk); 124bd4ad577SUrsula Braun int rc = 0; 125bd4ad577SUrsula Braun 126bd4ad577SUrsula Braun rc = smc_ib_modify_qp_init(lnk); 127bd4ad577SUrsula Braun if (rc) 128bd4ad577SUrsula Braun goto out; 129bd4ad577SUrsula Braun 130bd4ad577SUrsula Braun rc = smc_ib_modify_qp_rtr(lnk); 131bd4ad577SUrsula Braun if (rc) 132bd4ad577SUrsula Braun goto out; 133bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 134bd4ad577SUrsula Braun rc = ib_req_notify_cq(lnk->smcibdev->roce_cq_recv, 135bd4ad577SUrsula Braun IB_CQ_SOLICITED_MASK); 136bd4ad577SUrsula Braun if (rc) 137bd4ad577SUrsula Braun goto out; 138bd4ad577SUrsula Braun rc = smc_wr_rx_post_init(lnk); 139bd4ad577SUrsula Braun if (rc) 140bd4ad577SUrsula Braun goto out; 141bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 142bd4ad577SUrsula Braun 143bd4ad577SUrsula Braun if (lgr->role == SMC_SERV) { 144bd4ad577SUrsula Braun rc = smc_ib_modify_qp_rts(lnk); 145bd4ad577SUrsula Braun if (rc) 146bd4ad577SUrsula Braun goto out; 147bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 148bd4ad577SUrsula Braun } 149bd4ad577SUrsula Braun out: 150bd4ad577SUrsula Braun return rc; 151bd4ad577SUrsula Braun } 152bd4ad577SUrsula Braun 1537005ada6SUrsula Braun static int smc_ib_fill_mac(struct smc_ib_device *smcibdev, u8 ibport) 154be6a3f38SUrsula Braun { 155b4c296f9SJason Gunthorpe const struct ib_gid_attr *attr; 1565102eca9SParav Pandit int rc; 157be6a3f38SUrsula Braun 158b4c296f9SJason Gunthorpe attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, 0); 159b4c296f9SJason Gunthorpe if (IS_ERR(attr)) 160be6a3f38SUrsula Braun return -ENODEV; 161be6a3f38SUrsula Braun 1625102eca9SParav Pandit rc = rdma_read_gid_l2_fields(attr, NULL, smcibdev->mac[ibport - 1]); 163b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 164b4c296f9SJason Gunthorpe return rc; 165be6a3f38SUrsula Braun } 166be6a3f38SUrsula Braun 167be6a3f38SUrsula Braun /* Create an identifier unique for this instance of SMC-R. 168be6a3f38SUrsula Braun * The MAC-address of the first active registered IB device 169be6a3f38SUrsula Braun * plus a random 2-byte number is used to create this identifier. 170be6a3f38SUrsula Braun * This name is delivered to the peer during connection initialization. 171be6a3f38SUrsula Braun */ 172be6a3f38SUrsula Braun static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev, 173be6a3f38SUrsula Braun u8 ibport) 174be6a3f38SUrsula Braun { 175be6a3f38SUrsula Braun memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1], 176be6a3f38SUrsula Braun sizeof(smcibdev->mac[ibport - 1])); 177366bb249SHans Wippel } 178366bb249SHans Wippel 179a082ec89SHans Wippel bool smc_ib_is_valid_local_systemid(void) 180366bb249SHans Wippel { 181366bb249SHans Wippel return !is_zero_ether_addr(&local_systemid[2]); 182366bb249SHans Wippel } 183366bb249SHans Wippel 184366bb249SHans Wippel static void smc_ib_init_local_systemid(void) 185366bb249SHans Wippel { 186be6a3f38SUrsula Braun get_random_bytes(&local_systemid[0], 2); 187be6a3f38SUrsula Braun } 188be6a3f38SUrsula Braun 189be6a3f38SUrsula Braun bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport) 190be6a3f38SUrsula Braun { 191be6a3f38SUrsula Braun return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE; 192be6a3f38SUrsula Braun } 193be6a3f38SUrsula Braun 194e5c4744cSKarsten Graul int smc_ib_find_route(__be32 saddr, __be32 daddr, 195e5c4744cSKarsten Graul u8 nexthop_mac[], u8 *uses_gateway) 196e5c4744cSKarsten Graul { 197e5c4744cSKarsten Graul struct neighbour *neigh = NULL; 198e5c4744cSKarsten Graul struct rtable *rt = NULL; 199e5c4744cSKarsten Graul struct flowi4 fl4 = { 200e5c4744cSKarsten Graul .saddr = saddr, 201e5c4744cSKarsten Graul .daddr = daddr 202e5c4744cSKarsten Graul }; 203e5c4744cSKarsten Graul 204e5c4744cSKarsten Graul if (daddr == cpu_to_be32(INADDR_NONE)) 205e5c4744cSKarsten Graul goto out; 206e5c4744cSKarsten Graul rt = ip_route_output_flow(&init_net, &fl4, NULL); 207e5c4744cSKarsten Graul if (IS_ERR(rt)) 208e5c4744cSKarsten Graul goto out; 209e5c4744cSKarsten Graul if (rt->rt_uses_gateway && rt->rt_gw_family != AF_INET) 210e5c4744cSKarsten Graul goto out; 211e5c4744cSKarsten Graul neigh = rt->dst.ops->neigh_lookup(&rt->dst, NULL, &fl4.daddr); 212e5c4744cSKarsten Graul if (neigh) { 213e5c4744cSKarsten Graul memcpy(nexthop_mac, neigh->ha, ETH_ALEN); 214e5c4744cSKarsten Graul *uses_gateway = rt->rt_uses_gateway; 215e5c4744cSKarsten Graul return 0; 216e5c4744cSKarsten Graul } 217e5c4744cSKarsten Graul out: 218e5c4744cSKarsten Graul return -ENOENT; 219e5c4744cSKarsten Graul } 220e5c4744cSKarsten Graul 221*24fb6811SKarsten Graul static int smc_ib_determine_gid_rcu(const struct net_device *ndev, 222*24fb6811SKarsten Graul const struct ib_gid_attr *attr, 223*24fb6811SKarsten Graul u8 gid[], u8 *sgid_index, 224*24fb6811SKarsten Graul struct smc_init_info_smcrv2 *smcrv2) 225*24fb6811SKarsten Graul { 226*24fb6811SKarsten Graul if (!smcrv2 && attr->gid_type == IB_GID_TYPE_ROCE) { 227*24fb6811SKarsten Graul if (gid) 228*24fb6811SKarsten Graul memcpy(gid, &attr->gid, SMC_GID_SIZE); 229*24fb6811SKarsten Graul if (sgid_index) 230*24fb6811SKarsten Graul *sgid_index = attr->index; 231*24fb6811SKarsten Graul return 0; 232*24fb6811SKarsten Graul } 233*24fb6811SKarsten Graul if (smcrv2 && attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP && 234*24fb6811SKarsten Graul smc_ib_gid_to_ipv4((u8 *)&attr->gid) != cpu_to_be32(INADDR_NONE)) { 235*24fb6811SKarsten Graul struct in_device *in_dev = __in_dev_get_rcu(ndev); 236*24fb6811SKarsten Graul const struct in_ifaddr *ifa; 237*24fb6811SKarsten Graul bool subnet_match = false; 238*24fb6811SKarsten Graul 239*24fb6811SKarsten Graul if (!in_dev) 240*24fb6811SKarsten Graul goto out; 241*24fb6811SKarsten Graul in_dev_for_each_ifa_rcu(ifa, in_dev) { 242*24fb6811SKarsten Graul if (!inet_ifa_match(smcrv2->saddr, ifa)) 243*24fb6811SKarsten Graul continue; 244*24fb6811SKarsten Graul subnet_match = true; 245*24fb6811SKarsten Graul break; 246*24fb6811SKarsten Graul } 247*24fb6811SKarsten Graul if (!subnet_match) 248*24fb6811SKarsten Graul goto out; 249*24fb6811SKarsten Graul if (smcrv2->daddr && smc_ib_find_route(smcrv2->saddr, 250*24fb6811SKarsten Graul smcrv2->daddr, 251*24fb6811SKarsten Graul smcrv2->nexthop_mac, 252*24fb6811SKarsten Graul &smcrv2->uses_gateway)) 253*24fb6811SKarsten Graul goto out; 254*24fb6811SKarsten Graul 255*24fb6811SKarsten Graul if (gid) 256*24fb6811SKarsten Graul memcpy(gid, &attr->gid, SMC_GID_SIZE); 257*24fb6811SKarsten Graul if (sgid_index) 258*24fb6811SKarsten Graul *sgid_index = attr->index; 259*24fb6811SKarsten Graul return 0; 260*24fb6811SKarsten Graul } 261*24fb6811SKarsten Graul out: 262*24fb6811SKarsten Graul return -ENODEV; 263*24fb6811SKarsten Graul } 264*24fb6811SKarsten Graul 2657005ada6SUrsula Braun /* determine the gid for an ib-device port and vlan id */ 2667005ada6SUrsula Braun int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport, 267*24fb6811SKarsten Graul unsigned short vlan_id, u8 gid[], u8 *sgid_index, 268*24fb6811SKarsten Graul struct smc_init_info_smcrv2 *smcrv2) 2697005ada6SUrsula Braun { 270b4c296f9SJason Gunthorpe const struct ib_gid_attr *attr; 2715102eca9SParav Pandit const struct net_device *ndev; 2727005ada6SUrsula Braun int i; 2737005ada6SUrsula Braun 2747005ada6SUrsula Braun for (i = 0; i < smcibdev->pattr[ibport - 1].gid_tbl_len; i++) { 275b4c296f9SJason Gunthorpe attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, i); 276b4c296f9SJason Gunthorpe if (IS_ERR(attr)) 2777005ada6SUrsula Braun continue; 278b4c296f9SJason Gunthorpe 2795102eca9SParav Pandit rcu_read_lock(); 2805102eca9SParav Pandit ndev = rdma_read_gid_attr_ndev_rcu(attr); 2815102eca9SParav Pandit if (!IS_ERR(ndev) && 28241a0be3fSKarsten Graul ((!vlan_id && !is_vlan_dev(ndev)) || 28341a0be3fSKarsten Graul (vlan_id && is_vlan_dev(ndev) && 284*24fb6811SKarsten Graul vlan_dev_vlan_id(ndev) == vlan_id))) { 285*24fb6811SKarsten Graul if (!smc_ib_determine_gid_rcu(ndev, attr, gid, 286*24fb6811SKarsten Graul sgid_index, smcrv2)) { 2875102eca9SParav Pandit rcu_read_unlock(); 288b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 2897005ada6SUrsula Braun return 0; 2907005ada6SUrsula Braun } 291*24fb6811SKarsten Graul } 2925102eca9SParav Pandit rcu_read_unlock(); 293b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 2947005ada6SUrsula Braun } 2957005ada6SUrsula Braun return -ENODEV; 2967005ada6SUrsula Braun } 2977005ada6SUrsula Braun 298be6a3f38SUrsula Braun static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport) 299be6a3f38SUrsula Braun { 300be6a3f38SUrsula Braun int rc; 301be6a3f38SUrsula Braun 302be6a3f38SUrsula Braun memset(&smcibdev->pattr[ibport - 1], 0, 303be6a3f38SUrsula Braun sizeof(smcibdev->pattr[ibport - 1])); 304be6a3f38SUrsula Braun rc = ib_query_port(smcibdev->ibdev, ibport, 305be6a3f38SUrsula Braun &smcibdev->pattr[ibport - 1]); 306be6a3f38SUrsula Braun if (rc) 307be6a3f38SUrsula Braun goto out; 308be6a3f38SUrsula Braun /* the SMC protocol requires specification of the RoCE MAC address */ 3097005ada6SUrsula Braun rc = smc_ib_fill_mac(smcibdev, ibport); 310be6a3f38SUrsula Braun if (rc) 311be6a3f38SUrsula Braun goto out; 312366bb249SHans Wippel if (!smc_ib_is_valid_local_systemid() && 313be6a3f38SUrsula Braun smc_ib_port_active(smcibdev, ibport)) 314be6a3f38SUrsula Braun /* create unique system identifier */ 315be6a3f38SUrsula Braun smc_ib_define_local_systemid(smcibdev, ibport); 316be6a3f38SUrsula Braun out: 317be6a3f38SUrsula Braun return rc; 318be6a3f38SUrsula Braun } 319be6a3f38SUrsula Braun 320bd4ad577SUrsula Braun /* process context wrapper for might_sleep smc_ib_remember_port_attr */ 321bd4ad577SUrsula Braun static void smc_ib_port_event_work(struct work_struct *work) 322bd4ad577SUrsula Braun { 323bd4ad577SUrsula Braun struct smc_ib_device *smcibdev = container_of( 324bd4ad577SUrsula Braun work, struct smc_ib_device, port_event_work); 325bd4ad577SUrsula Braun u8 port_idx; 326bd4ad577SUrsula Braun 327bd4ad577SUrsula Braun for_each_set_bit(port_idx, &smcibdev->port_event_mask, SMC_MAX_PORTS) { 328bd4ad577SUrsula Braun smc_ib_remember_port_attr(smcibdev, port_idx + 1); 329bd4ad577SUrsula Braun clear_bit(port_idx, &smcibdev->port_event_mask); 330c3d9494eSUrsula Braun if (!smc_ib_port_active(smcibdev, port_idx + 1)) { 331c3d9494eSUrsula Braun set_bit(port_idx, smcibdev->ports_going_away); 332541afa10SKarsten Graul smcr_port_err(smcibdev, port_idx + 1); 333c3d9494eSUrsula Braun } else { 334c3d9494eSUrsula Braun clear_bit(port_idx, smcibdev->ports_going_away); 3351f90a05dSKarsten Graul smcr_port_add(smcibdev, port_idx + 1); 336c3d9494eSUrsula Braun } 337bd4ad577SUrsula Braun } 338bd4ad577SUrsula Braun } 339bd4ad577SUrsula Braun 340bd4ad577SUrsula Braun /* can be called in IRQ context */ 341bd4ad577SUrsula Braun static void smc_ib_global_event_handler(struct ib_event_handler *handler, 342bd4ad577SUrsula Braun struct ib_event *ibevent) 343bd4ad577SUrsula Braun { 344bd4ad577SUrsula Braun struct smc_ib_device *smcibdev; 3455613f20cSUrsula Braun bool schedule = false; 346bd4ad577SUrsula Braun u8 port_idx; 347bd4ad577SUrsula Braun 348bd4ad577SUrsula Braun smcibdev = container_of(handler, struct smc_ib_device, event_handler); 349bd4ad577SUrsula Braun 350bd4ad577SUrsula Braun switch (ibevent->event) { 351bd4ad577SUrsula Braun case IB_EVENT_DEVICE_FATAL: 35281cf6430SKarsten Graul /* terminate all ports on device */ 353c3d9494eSUrsula Braun for (port_idx = 0; port_idx < SMC_MAX_PORTS; port_idx++) { 354bd4ad577SUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 3555613f20cSUrsula Braun if (!test_and_set_bit(port_idx, 3565613f20cSUrsula Braun smcibdev->ports_going_away)) 3575613f20cSUrsula Braun schedule = true; 358c3d9494eSUrsula Braun } 3595613f20cSUrsula Braun if (schedule) 3605613f20cSUrsula Braun schedule_work(&smcibdev->port_event_work); 3615613f20cSUrsula Braun break; 3625613f20cSUrsula Braun case IB_EVENT_PORT_ACTIVE: 3635613f20cSUrsula Braun port_idx = ibevent->element.port_num - 1; 3645613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3655613f20cSUrsula Braun break; 3665613f20cSUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 3675613f20cSUrsula Braun if (test_and_clear_bit(port_idx, smcibdev->ports_going_away)) 368bd4ad577SUrsula Braun schedule_work(&smcibdev->port_event_work); 369bd4ad577SUrsula Braun break; 37081cf6430SKarsten Graul case IB_EVENT_PORT_ERR: 3715613f20cSUrsula Braun port_idx = ibevent->element.port_num - 1; 3725613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3735613f20cSUrsula Braun break; 3745613f20cSUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 3755613f20cSUrsula Braun if (!test_and_set_bit(port_idx, smcibdev->ports_going_away)) 3765613f20cSUrsula Braun schedule_work(&smcibdev->port_event_work); 3775613f20cSUrsula Braun break; 37881cf6430SKarsten Graul case IB_EVENT_GID_CHANGE: 37981cf6430SKarsten Graul port_idx = ibevent->element.port_num - 1; 3805613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3815613f20cSUrsula Braun break; 38281cf6430SKarsten Graul set_bit(port_idx, &smcibdev->port_event_mask); 38381cf6430SKarsten Graul schedule_work(&smcibdev->port_event_work); 38481cf6430SKarsten Graul break; 385bd4ad577SUrsula Braun default: 386bd4ad577SUrsula Braun break; 387bd4ad577SUrsula Braun } 388bd4ad577SUrsula Braun } 389bd4ad577SUrsula Braun 390f38ba179SUrsula Braun void smc_ib_dealloc_protection_domain(struct smc_link *lnk) 391f38ba179SUrsula Braun { 392da05bf29SUrsula Braun if (lnk->roce_pd) 393f38ba179SUrsula Braun ib_dealloc_pd(lnk->roce_pd); 394f38ba179SUrsula Braun lnk->roce_pd = NULL; 395f38ba179SUrsula Braun } 396f38ba179SUrsula Braun 397f38ba179SUrsula Braun int smc_ib_create_protection_domain(struct smc_link *lnk) 398f38ba179SUrsula Braun { 399f38ba179SUrsula Braun int rc; 400f38ba179SUrsula Braun 401897e1c24SUrsula Braun lnk->roce_pd = ib_alloc_pd(lnk->smcibdev->ibdev, 0); 402f38ba179SUrsula Braun rc = PTR_ERR_OR_ZERO(lnk->roce_pd); 403f38ba179SUrsula Braun if (IS_ERR(lnk->roce_pd)) 404f38ba179SUrsula Braun lnk->roce_pd = NULL; 405f38ba179SUrsula Braun return rc; 406f38ba179SUrsula Braun } 407f38ba179SUrsula Braun 408a3db10efSGuvenc Gulce static bool smcr_diag_is_dev_critical(struct smc_lgr_list *smc_lgr, 409a3db10efSGuvenc Gulce struct smc_ib_device *smcibdev) 410a3db10efSGuvenc Gulce { 411a3db10efSGuvenc Gulce struct smc_link_group *lgr; 412a3db10efSGuvenc Gulce bool rc = false; 413a3db10efSGuvenc Gulce int i; 414a3db10efSGuvenc Gulce 415a3db10efSGuvenc Gulce spin_lock_bh(&smc_lgr->lock); 416a3db10efSGuvenc Gulce list_for_each_entry(lgr, &smc_lgr->list, list) { 417a3db10efSGuvenc Gulce if (lgr->is_smcd) 418a3db10efSGuvenc Gulce continue; 419a3db10efSGuvenc Gulce for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 420a3db10efSGuvenc Gulce if (lgr->lnk[i].state == SMC_LNK_UNUSED || 421a3db10efSGuvenc Gulce lgr->lnk[i].smcibdev != smcibdev) 422a3db10efSGuvenc Gulce continue; 423a3db10efSGuvenc Gulce if (lgr->type == SMC_LGR_SINGLE || 424a3db10efSGuvenc Gulce lgr->type == SMC_LGR_ASYMMETRIC_LOCAL) { 425a3db10efSGuvenc Gulce rc = true; 426a3db10efSGuvenc Gulce goto out; 427a3db10efSGuvenc Gulce } 428a3db10efSGuvenc Gulce } 429a3db10efSGuvenc Gulce } 430a3db10efSGuvenc Gulce out: 431a3db10efSGuvenc Gulce spin_unlock_bh(&smc_lgr->lock); 432a3db10efSGuvenc Gulce return rc; 433a3db10efSGuvenc Gulce } 434a3db10efSGuvenc Gulce 435a3db10efSGuvenc Gulce static int smc_nl_handle_dev_port(struct sk_buff *skb, 436a3db10efSGuvenc Gulce struct ib_device *ibdev, 437a3db10efSGuvenc Gulce struct smc_ib_device *smcibdev, 438a3db10efSGuvenc Gulce int port) 439a3db10efSGuvenc Gulce { 440a3db10efSGuvenc Gulce char smc_pnet[SMC_MAX_PNETID_LEN + 1]; 441a3db10efSGuvenc Gulce struct nlattr *port_attrs; 442a3db10efSGuvenc Gulce unsigned char port_state; 443a3db10efSGuvenc Gulce int lnk_count = 0; 444a3db10efSGuvenc Gulce 445a3db10efSGuvenc Gulce port_attrs = nla_nest_start(skb, SMC_NLA_DEV_PORT + port); 446a3db10efSGuvenc Gulce if (!port_attrs) 447a3db10efSGuvenc Gulce goto errout; 448a3db10efSGuvenc Gulce 449a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_PORT_PNET_USR, 450a3db10efSGuvenc Gulce smcibdev->pnetid_by_user[port])) 451a3db10efSGuvenc Gulce goto errattr; 4528a446536SGuvenc Gulce memcpy(smc_pnet, &smcibdev->pnetid[port], SMC_MAX_PNETID_LEN); 4538a446536SGuvenc Gulce smc_pnet[SMC_MAX_PNETID_LEN] = 0; 454a3db10efSGuvenc Gulce if (nla_put_string(skb, SMC_NLA_DEV_PORT_PNETID, smc_pnet)) 455a3db10efSGuvenc Gulce goto errattr; 456a3db10efSGuvenc Gulce if (nla_put_u32(skb, SMC_NLA_DEV_PORT_NETDEV, 457a3db10efSGuvenc Gulce smcibdev->ndev_ifidx[port])) 458a3db10efSGuvenc Gulce goto errattr; 459a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_PORT_VALID, 1)) 460a3db10efSGuvenc Gulce goto errattr; 461a3db10efSGuvenc Gulce port_state = smc_ib_port_active(smcibdev, port + 1); 462a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_PORT_STATE, port_state)) 463a3db10efSGuvenc Gulce goto errattr; 464a3db10efSGuvenc Gulce lnk_count = atomic_read(&smcibdev->lnk_cnt_by_port[port]); 465a3db10efSGuvenc Gulce if (nla_put_u32(skb, SMC_NLA_DEV_PORT_LNK_CNT, lnk_count)) 466a3db10efSGuvenc Gulce goto errattr; 467a3db10efSGuvenc Gulce nla_nest_end(skb, port_attrs); 468a3db10efSGuvenc Gulce return 0; 469a3db10efSGuvenc Gulce errattr: 470a3db10efSGuvenc Gulce nla_nest_cancel(skb, port_attrs); 471a3db10efSGuvenc Gulce errout: 472a3db10efSGuvenc Gulce return -EMSGSIZE; 473a3db10efSGuvenc Gulce } 474a3db10efSGuvenc Gulce 475995433b7SKarsten Graul static bool smc_nl_handle_pci_values(const struct smc_pci_dev *smc_pci_dev, 476995433b7SKarsten Graul struct sk_buff *skb) 477995433b7SKarsten Graul { 478995433b7SKarsten Graul if (nla_put_u32(skb, SMC_NLA_DEV_PCI_FID, smc_pci_dev->pci_fid)) 479995433b7SKarsten Graul return false; 480995433b7SKarsten Graul if (nla_put_u16(skb, SMC_NLA_DEV_PCI_CHID, smc_pci_dev->pci_pchid)) 481995433b7SKarsten Graul return false; 482995433b7SKarsten Graul if (nla_put_u16(skb, SMC_NLA_DEV_PCI_VENDOR, smc_pci_dev->pci_vendor)) 483995433b7SKarsten Graul return false; 484995433b7SKarsten Graul if (nla_put_u16(skb, SMC_NLA_DEV_PCI_DEVICE, smc_pci_dev->pci_device)) 485995433b7SKarsten Graul return false; 486995433b7SKarsten Graul if (nla_put_string(skb, SMC_NLA_DEV_PCI_ID, smc_pci_dev->pci_id)) 487995433b7SKarsten Graul return false; 488995433b7SKarsten Graul return true; 489995433b7SKarsten Graul } 490995433b7SKarsten Graul 491a3db10efSGuvenc Gulce static int smc_nl_handle_smcr_dev(struct smc_ib_device *smcibdev, 492a3db10efSGuvenc Gulce struct sk_buff *skb, 493a3db10efSGuvenc Gulce struct netlink_callback *cb) 494a3db10efSGuvenc Gulce { 4958a446536SGuvenc Gulce char smc_ibname[IB_DEVICE_NAME_MAX]; 496a3db10efSGuvenc Gulce struct smc_pci_dev smc_pci_dev; 497a3db10efSGuvenc Gulce struct pci_dev *pci_dev; 498a3db10efSGuvenc Gulce unsigned char is_crit; 499a3db10efSGuvenc Gulce struct nlattr *attrs; 500a3db10efSGuvenc Gulce void *nlh; 501a3db10efSGuvenc Gulce int i; 502a3db10efSGuvenc Gulce 503a3db10efSGuvenc Gulce nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 504a3db10efSGuvenc Gulce &smc_gen_nl_family, NLM_F_MULTI, 505a3db10efSGuvenc Gulce SMC_NETLINK_GET_DEV_SMCR); 506a3db10efSGuvenc Gulce if (!nlh) 507a3db10efSGuvenc Gulce goto errmsg; 508a3db10efSGuvenc Gulce attrs = nla_nest_start(skb, SMC_GEN_DEV_SMCR); 509a3db10efSGuvenc Gulce if (!attrs) 510a3db10efSGuvenc Gulce goto errout; 511a3db10efSGuvenc Gulce is_crit = smcr_diag_is_dev_critical(&smc_lgr_list, smcibdev); 512a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_IS_CRIT, is_crit)) 513a3db10efSGuvenc Gulce goto errattr; 514995433b7SKarsten Graul if (smcibdev->ibdev->dev.parent) { 515a3db10efSGuvenc Gulce memset(&smc_pci_dev, 0, sizeof(smc_pci_dev)); 516a3db10efSGuvenc Gulce pci_dev = to_pci_dev(smcibdev->ibdev->dev.parent); 517a3db10efSGuvenc Gulce smc_set_pci_values(pci_dev, &smc_pci_dev); 518995433b7SKarsten Graul if (!smc_nl_handle_pci_values(&smc_pci_dev, skb)) 519a3db10efSGuvenc Gulce goto errattr; 520995433b7SKarsten Graul } 521a3db10efSGuvenc Gulce snprintf(smc_ibname, sizeof(smc_ibname), "%s", smcibdev->ibdev->name); 522a3db10efSGuvenc Gulce if (nla_put_string(skb, SMC_NLA_DEV_IB_NAME, smc_ibname)) 523a3db10efSGuvenc Gulce goto errattr; 524a3db10efSGuvenc Gulce for (i = 1; i <= SMC_MAX_PORTS; i++) { 525a3db10efSGuvenc Gulce if (!rdma_is_port_valid(smcibdev->ibdev, i)) 526a3db10efSGuvenc Gulce continue; 527a3db10efSGuvenc Gulce if (smc_nl_handle_dev_port(skb, smcibdev->ibdev, 528a3db10efSGuvenc Gulce smcibdev, i - 1)) 529a3db10efSGuvenc Gulce goto errattr; 530a3db10efSGuvenc Gulce } 531a3db10efSGuvenc Gulce 532a3db10efSGuvenc Gulce nla_nest_end(skb, attrs); 533a3db10efSGuvenc Gulce genlmsg_end(skb, nlh); 534a3db10efSGuvenc Gulce return 0; 535a3db10efSGuvenc Gulce 536a3db10efSGuvenc Gulce errattr: 537a3db10efSGuvenc Gulce nla_nest_cancel(skb, attrs); 538a3db10efSGuvenc Gulce errout: 539a3db10efSGuvenc Gulce genlmsg_cancel(skb, nlh); 540a3db10efSGuvenc Gulce errmsg: 541a3db10efSGuvenc Gulce return -EMSGSIZE; 542a3db10efSGuvenc Gulce } 543a3db10efSGuvenc Gulce 544a3db10efSGuvenc Gulce static void smc_nl_prep_smcr_dev(struct smc_ib_devices *dev_list, 545a3db10efSGuvenc Gulce struct sk_buff *skb, 546a3db10efSGuvenc Gulce struct netlink_callback *cb) 547a3db10efSGuvenc Gulce { 548a3db10efSGuvenc Gulce struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); 549a3db10efSGuvenc Gulce struct smc_ib_device *smcibdev; 550a3db10efSGuvenc Gulce int snum = cb_ctx->pos[0]; 551a3db10efSGuvenc Gulce int num = 0; 552a3db10efSGuvenc Gulce 553a3db10efSGuvenc Gulce mutex_lock(&dev_list->mutex); 554a3db10efSGuvenc Gulce list_for_each_entry(smcibdev, &dev_list->list, list) { 555a3db10efSGuvenc Gulce if (num < snum) 556a3db10efSGuvenc Gulce goto next; 557a3db10efSGuvenc Gulce if (smc_nl_handle_smcr_dev(smcibdev, skb, cb)) 558a3db10efSGuvenc Gulce goto errout; 559a3db10efSGuvenc Gulce next: 560a3db10efSGuvenc Gulce num++; 561a3db10efSGuvenc Gulce } 562a3db10efSGuvenc Gulce errout: 563a3db10efSGuvenc Gulce mutex_unlock(&dev_list->mutex); 564a3db10efSGuvenc Gulce cb_ctx->pos[0] = num; 565a3db10efSGuvenc Gulce } 566a3db10efSGuvenc Gulce 567a3db10efSGuvenc Gulce int smcr_nl_get_device(struct sk_buff *skb, struct netlink_callback *cb) 568a3db10efSGuvenc Gulce { 569a3db10efSGuvenc Gulce smc_nl_prep_smcr_dev(&smc_ib_devices, skb, cb); 570a3db10efSGuvenc Gulce return skb->len; 571a3db10efSGuvenc Gulce } 572a3db10efSGuvenc Gulce 573f38ba179SUrsula Braun static void smc_ib_qp_event_handler(struct ib_event *ibevent, void *priv) 574f38ba179SUrsula Braun { 575e5f3aa04SKarsten Graul struct smc_link *lnk = (struct smc_link *)priv; 576e5f3aa04SKarsten Graul struct smc_ib_device *smcibdev = lnk->smcibdev; 577da05bf29SUrsula Braun u8 port_idx; 578da05bf29SUrsula Braun 579f38ba179SUrsula Braun switch (ibevent->event) { 58081cf6430SKarsten Graul case IB_EVENT_QP_FATAL: 581f38ba179SUrsula Braun case IB_EVENT_QP_ACCESS_ERR: 582e5f3aa04SKarsten Graul port_idx = ibevent->element.qp->port - 1; 5835613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 5845613f20cSUrsula Braun break; 585da05bf29SUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 5865613f20cSUrsula Braun if (!test_and_set_bit(port_idx, smcibdev->ports_going_away)) 587da05bf29SUrsula Braun schedule_work(&smcibdev->port_event_work); 588f38ba179SUrsula Braun break; 589f38ba179SUrsula Braun default: 590f38ba179SUrsula Braun break; 591f38ba179SUrsula Braun } 592f38ba179SUrsula Braun } 593f38ba179SUrsula Braun 594f38ba179SUrsula Braun void smc_ib_destroy_queue_pair(struct smc_link *lnk) 595f38ba179SUrsula Braun { 596da05bf29SUrsula Braun if (lnk->roce_qp) 597f38ba179SUrsula Braun ib_destroy_qp(lnk->roce_qp); 598f38ba179SUrsula Braun lnk->roce_qp = NULL; 599f38ba179SUrsula Braun } 600f38ba179SUrsula Braun 601f38ba179SUrsula Braun /* create a queue pair within the protection domain for a link */ 602f38ba179SUrsula Braun int smc_ib_create_queue_pair(struct smc_link *lnk) 603f38ba179SUrsula Braun { 604f38ba179SUrsula Braun struct ib_qp_init_attr qp_attr = { 605f38ba179SUrsula Braun .event_handler = smc_ib_qp_event_handler, 606f38ba179SUrsula Braun .qp_context = lnk, 607f38ba179SUrsula Braun .send_cq = lnk->smcibdev->roce_cq_send, 608f38ba179SUrsula Braun .recv_cq = lnk->smcibdev->roce_cq_recv, 609f38ba179SUrsula Braun .srq = NULL, 610f38ba179SUrsula Braun .cap = { 611f38ba179SUrsula Braun /* include unsolicited rdma_writes as well, 612f38ba179SUrsula Braun * there are max. 2 RDMA_WRITE per 1 WR_SEND 613f38ba179SUrsula Braun */ 614652a1e41SUrsula Braun .max_send_wr = SMC_WR_BUF_CNT * 3, 615f38ba179SUrsula Braun .max_recv_wr = SMC_WR_BUF_CNT * 3, 616f38ba179SUrsula Braun .max_send_sge = SMC_IB_MAX_SEND_SGE, 617f38ba179SUrsula Braun .max_recv_sge = 1, 618f38ba179SUrsula Braun }, 619f38ba179SUrsula Braun .sq_sig_type = IB_SIGNAL_REQ_WR, 620f38ba179SUrsula Braun .qp_type = IB_QPT_RC, 621f38ba179SUrsula Braun }; 622f38ba179SUrsula Braun int rc; 623f38ba179SUrsula Braun 624f38ba179SUrsula Braun lnk->roce_qp = ib_create_qp(lnk->roce_pd, &qp_attr); 625f38ba179SUrsula Braun rc = PTR_ERR_OR_ZERO(lnk->roce_qp); 626f38ba179SUrsula Braun if (IS_ERR(lnk->roce_qp)) 627f38ba179SUrsula Braun lnk->roce_qp = NULL; 628f38ba179SUrsula Braun else 629f38ba179SUrsula Braun smc_wr_remember_qp_attr(lnk); 630f38ba179SUrsula Braun return rc; 631f38ba179SUrsula Braun } 632f38ba179SUrsula Braun 633897e1c24SUrsula Braun void smc_ib_put_memory_region(struct ib_mr *mr) 634897e1c24SUrsula Braun { 635897e1c24SUrsula Braun ib_dereg_mr(mr); 636897e1c24SUrsula Braun } 637897e1c24SUrsula Braun 638387707fdSKarsten Graul static int smc_ib_map_mr_sg(struct smc_buf_desc *buf_slot, u8 link_idx) 639897e1c24SUrsula Braun { 640897e1c24SUrsula Braun unsigned int offset = 0; 641897e1c24SUrsula Braun int sg_num; 642897e1c24SUrsula Braun 643897e1c24SUrsula Braun /* map the largest prefix of a dma mapped SG list */ 644387707fdSKarsten Graul sg_num = ib_map_mr_sg(buf_slot->mr_rx[link_idx], 645387707fdSKarsten Graul buf_slot->sgt[link_idx].sgl, 646387707fdSKarsten Graul buf_slot->sgt[link_idx].orig_nents, 647897e1c24SUrsula Braun &offset, PAGE_SIZE); 648897e1c24SUrsula Braun 649897e1c24SUrsula Braun return sg_num; 650897e1c24SUrsula Braun } 651897e1c24SUrsula Braun 652897e1c24SUrsula Braun /* Allocate a memory region and map the dma mapped SG list of buf_slot */ 653897e1c24SUrsula Braun int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags, 654387707fdSKarsten Graul struct smc_buf_desc *buf_slot, u8 link_idx) 655897e1c24SUrsula Braun { 656387707fdSKarsten Graul if (buf_slot->mr_rx[link_idx]) 657897e1c24SUrsula Braun return 0; /* already done */ 658897e1c24SUrsula Braun 659387707fdSKarsten Graul buf_slot->mr_rx[link_idx] = 660897e1c24SUrsula Braun ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 1 << buf_slot->order); 661387707fdSKarsten Graul if (IS_ERR(buf_slot->mr_rx[link_idx])) { 662897e1c24SUrsula Braun int rc; 663897e1c24SUrsula Braun 664387707fdSKarsten Graul rc = PTR_ERR(buf_slot->mr_rx[link_idx]); 665387707fdSKarsten Graul buf_slot->mr_rx[link_idx] = NULL; 666897e1c24SUrsula Braun return rc; 667897e1c24SUrsula Braun } 668897e1c24SUrsula Braun 669387707fdSKarsten Graul if (smc_ib_map_mr_sg(buf_slot, link_idx) != 1) 670897e1c24SUrsula Braun return -EINVAL; 671897e1c24SUrsula Braun 672897e1c24SUrsula Braun return 0; 673897e1c24SUrsula Braun } 674897e1c24SUrsula Braun 67510428dd8SUrsula Braun /* synchronize buffer usage for cpu access */ 676387707fdSKarsten Graul void smc_ib_sync_sg_for_cpu(struct smc_link *lnk, 67710428dd8SUrsula Braun struct smc_buf_desc *buf_slot, 67810428dd8SUrsula Braun enum dma_data_direction data_direction) 67910428dd8SUrsula Braun { 68010428dd8SUrsula Braun struct scatterlist *sg; 68110428dd8SUrsula Braun unsigned int i; 68210428dd8SUrsula Braun 68310428dd8SUrsula Braun /* for now there is just one DMA address */ 684387707fdSKarsten Graul for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, 685387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].nents, i) { 68610428dd8SUrsula Braun if (!sg_dma_len(sg)) 68710428dd8SUrsula Braun break; 688387707fdSKarsten Graul ib_dma_sync_single_for_cpu(lnk->smcibdev->ibdev, 68910428dd8SUrsula Braun sg_dma_address(sg), 69010428dd8SUrsula Braun sg_dma_len(sg), 69110428dd8SUrsula Braun data_direction); 69210428dd8SUrsula Braun } 69310428dd8SUrsula Braun } 69410428dd8SUrsula Braun 69510428dd8SUrsula Braun /* synchronize buffer usage for device access */ 696387707fdSKarsten Graul void smc_ib_sync_sg_for_device(struct smc_link *lnk, 69710428dd8SUrsula Braun struct smc_buf_desc *buf_slot, 69810428dd8SUrsula Braun enum dma_data_direction data_direction) 69910428dd8SUrsula Braun { 70010428dd8SUrsula Braun struct scatterlist *sg; 70110428dd8SUrsula Braun unsigned int i; 70210428dd8SUrsula Braun 70310428dd8SUrsula Braun /* for now there is just one DMA address */ 704387707fdSKarsten Graul for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, 705387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].nents, i) { 70610428dd8SUrsula Braun if (!sg_dma_len(sg)) 70710428dd8SUrsula Braun break; 708387707fdSKarsten Graul ib_dma_sync_single_for_device(lnk->smcibdev->ibdev, 70910428dd8SUrsula Braun sg_dma_address(sg), 71010428dd8SUrsula Braun sg_dma_len(sg), 71110428dd8SUrsula Braun data_direction); 71210428dd8SUrsula Braun } 71310428dd8SUrsula Braun } 71410428dd8SUrsula Braun 715a3fe3d01SUrsula Braun /* Map a new TX or RX buffer SG-table to DMA */ 716387707fdSKarsten Graul int smc_ib_buf_map_sg(struct smc_link *lnk, 717a3fe3d01SUrsula Braun struct smc_buf_desc *buf_slot, 718a3fe3d01SUrsula Braun enum dma_data_direction data_direction) 719a3fe3d01SUrsula Braun { 720a3fe3d01SUrsula Braun int mapped_nents; 721a3fe3d01SUrsula Braun 722387707fdSKarsten Graul mapped_nents = ib_dma_map_sg(lnk->smcibdev->ibdev, 723387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl, 724387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].orig_nents, 725a3fe3d01SUrsula Braun data_direction); 726a3fe3d01SUrsula Braun if (!mapped_nents) 727a3fe3d01SUrsula Braun return -ENOMEM; 728a3fe3d01SUrsula Braun 729a3fe3d01SUrsula Braun return mapped_nents; 730a3fe3d01SUrsula Braun } 731a3fe3d01SUrsula Braun 732387707fdSKarsten Graul void smc_ib_buf_unmap_sg(struct smc_link *lnk, 733a3fe3d01SUrsula Braun struct smc_buf_desc *buf_slot, 734a3fe3d01SUrsula Braun enum dma_data_direction data_direction) 735a3fe3d01SUrsula Braun { 736387707fdSKarsten Graul if (!buf_slot->sgt[lnk->link_idx].sgl->dma_address) 737a3fe3d01SUrsula Braun return; /* already unmapped */ 738a3fe3d01SUrsula Braun 739387707fdSKarsten Graul ib_dma_unmap_sg(lnk->smcibdev->ibdev, 740387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl, 741387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].orig_nents, 742a3fe3d01SUrsula Braun data_direction); 743387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl->dma_address = 0; 744a3fe3d01SUrsula Braun } 745a3fe3d01SUrsula Braun 746bd4ad577SUrsula Braun long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev) 747bd4ad577SUrsula Braun { 748bd4ad577SUrsula Braun struct ib_cq_init_attr cqattr = { 749c9f4c6cfSUrsula Braun .cqe = SMC_MAX_CQE, .comp_vector = 0 }; 750c9f4c6cfSUrsula Braun int cqe_size_order, smc_order; 751bd4ad577SUrsula Braun long rc; 752bd4ad577SUrsula Braun 75363673597SKarsten Graul mutex_lock(&smcibdev->mutex); 75463673597SKarsten Graul rc = 0; 75563673597SKarsten Graul if (smcibdev->initialized) 75663673597SKarsten Graul goto out; 757c9f4c6cfSUrsula Braun /* the calculated number of cq entries fits to mlx5 cq allocation */ 758c9f4c6cfSUrsula Braun cqe_size_order = cache_line_size() == 128 ? 7 : 6; 759c9f4c6cfSUrsula Braun smc_order = MAX_ORDER - cqe_size_order - 1; 760c9f4c6cfSUrsula Braun if (SMC_MAX_CQE + 2 > (0x00000001 << smc_order) * PAGE_SIZE) 761c9f4c6cfSUrsula Braun cqattr.cqe = (0x00000001 << smc_order) * PAGE_SIZE - 2; 762bd4ad577SUrsula Braun smcibdev->roce_cq_send = ib_create_cq(smcibdev->ibdev, 763bd4ad577SUrsula Braun smc_wr_tx_cq_handler, NULL, 764bd4ad577SUrsula Braun smcibdev, &cqattr); 765bd4ad577SUrsula Braun rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_send); 766bd4ad577SUrsula Braun if (IS_ERR(smcibdev->roce_cq_send)) { 767bd4ad577SUrsula Braun smcibdev->roce_cq_send = NULL; 76863673597SKarsten Graul goto out; 769bd4ad577SUrsula Braun } 770bd4ad577SUrsula Braun smcibdev->roce_cq_recv = ib_create_cq(smcibdev->ibdev, 771bd4ad577SUrsula Braun smc_wr_rx_cq_handler, NULL, 772bd4ad577SUrsula Braun smcibdev, &cqattr); 773bd4ad577SUrsula Braun rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_recv); 774bd4ad577SUrsula Braun if (IS_ERR(smcibdev->roce_cq_recv)) { 775bd4ad577SUrsula Braun smcibdev->roce_cq_recv = NULL; 776bd4ad577SUrsula Braun goto err; 777bd4ad577SUrsula Braun } 778bd4ad577SUrsula Braun smc_wr_add_dev(smcibdev); 779bd4ad577SUrsula Braun smcibdev->initialized = 1; 78063673597SKarsten Graul goto out; 781bd4ad577SUrsula Braun 782bd4ad577SUrsula Braun err: 783bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_send); 78463673597SKarsten Graul out: 78563673597SKarsten Graul mutex_unlock(&smcibdev->mutex); 786bd4ad577SUrsula Braun return rc; 787bd4ad577SUrsula Braun } 788bd4ad577SUrsula Braun 789bd4ad577SUrsula Braun static void smc_ib_cleanup_per_ibdev(struct smc_ib_device *smcibdev) 790bd4ad577SUrsula Braun { 79163673597SKarsten Graul mutex_lock(&smcibdev->mutex); 792bd4ad577SUrsula Braun if (!smcibdev->initialized) 79363673597SKarsten Graul goto out; 794da05bf29SUrsula Braun smcibdev->initialized = 0; 795bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_recv); 796bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_send); 7976a37ad3dSUrsula Braun smc_wr_remove_dev(smcibdev); 79863673597SKarsten Graul out: 79963673597SKarsten Graul mutex_unlock(&smcibdev->mutex); 800bd4ad577SUrsula Braun } 801bd4ad577SUrsula Braun 802a4cf0443SUrsula Braun static struct ib_client smc_ib_client; 803a4cf0443SUrsula Braun 8043d453f53SGuvenc Gulce static void smc_copy_netdev_ifindex(struct smc_ib_device *smcibdev, int port) 8053d453f53SGuvenc Gulce { 8063d453f53SGuvenc Gulce struct ib_device *ibdev = smcibdev->ibdev; 8073d453f53SGuvenc Gulce struct net_device *ndev; 8083d453f53SGuvenc Gulce 8093d453f53SGuvenc Gulce if (!ibdev->ops.get_netdev) 8103d453f53SGuvenc Gulce return; 8113d453f53SGuvenc Gulce ndev = ibdev->ops.get_netdev(ibdev, port + 1); 8123d453f53SGuvenc Gulce if (ndev) { 8133d453f53SGuvenc Gulce smcibdev->ndev_ifidx[port] = ndev->ifindex; 8143d453f53SGuvenc Gulce dev_put(ndev); 8153d453f53SGuvenc Gulce } 8163d453f53SGuvenc Gulce } 8173d453f53SGuvenc Gulce 8183d453f53SGuvenc Gulce void smc_ib_ndev_change(struct net_device *ndev, unsigned long event) 8193d453f53SGuvenc Gulce { 8203d453f53SGuvenc Gulce struct smc_ib_device *smcibdev; 8213d453f53SGuvenc Gulce struct ib_device *libdev; 8223d453f53SGuvenc Gulce struct net_device *lndev; 8233d453f53SGuvenc Gulce u8 port_cnt; 8243d453f53SGuvenc Gulce int i; 8253d453f53SGuvenc Gulce 8263d453f53SGuvenc Gulce mutex_lock(&smc_ib_devices.mutex); 8273d453f53SGuvenc Gulce list_for_each_entry(smcibdev, &smc_ib_devices.list, list) { 8283d453f53SGuvenc Gulce port_cnt = smcibdev->ibdev->phys_port_cnt; 8293d453f53SGuvenc Gulce for (i = 0; i < min_t(size_t, port_cnt, SMC_MAX_PORTS); i++) { 8303d453f53SGuvenc Gulce libdev = smcibdev->ibdev; 8313d453f53SGuvenc Gulce if (!libdev->ops.get_netdev) 8323d453f53SGuvenc Gulce continue; 8333d453f53SGuvenc Gulce lndev = libdev->ops.get_netdev(libdev, i + 1); 8343d453f53SGuvenc Gulce dev_put(lndev); 8353d453f53SGuvenc Gulce if (lndev != ndev) 8363d453f53SGuvenc Gulce continue; 8373d453f53SGuvenc Gulce if (event == NETDEV_REGISTER) 8383d453f53SGuvenc Gulce smcibdev->ndev_ifidx[i] = ndev->ifindex; 8393d453f53SGuvenc Gulce if (event == NETDEV_UNREGISTER) 8403d453f53SGuvenc Gulce smcibdev->ndev_ifidx[i] = 0; 8413d453f53SGuvenc Gulce } 8423d453f53SGuvenc Gulce } 8433d453f53SGuvenc Gulce mutex_unlock(&smc_ib_devices.mutex); 8443d453f53SGuvenc Gulce } 8453d453f53SGuvenc Gulce 846a4cf0443SUrsula Braun /* callback function for ib_register_client() */ 84711a0ae4cSJason Gunthorpe static int smc_ib_add_dev(struct ib_device *ibdev) 848a4cf0443SUrsula Braun { 849a4cf0443SUrsula Braun struct smc_ib_device *smcibdev; 850be6a3f38SUrsula Braun u8 port_cnt; 851be6a3f38SUrsula Braun int i; 852a4cf0443SUrsula Braun 853a4cf0443SUrsula Braun if (ibdev->node_type != RDMA_NODE_IB_CA) 85411a0ae4cSJason Gunthorpe return -EOPNOTSUPP; 855a4cf0443SUrsula Braun 856a4cf0443SUrsula Braun smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL); 857a4cf0443SUrsula Braun if (!smcibdev) 85811a0ae4cSJason Gunthorpe return -ENOMEM; 859a4cf0443SUrsula Braun 860a4cf0443SUrsula Braun smcibdev->ibdev = ibdev; 861bd4ad577SUrsula Braun INIT_WORK(&smcibdev->port_event_work, smc_ib_port_event_work); 8626dabd405SUrsula Braun atomic_set(&smcibdev->lnk_cnt, 0); 8636dabd405SUrsula Braun init_waitqueue_head(&smcibdev->lnks_deleted); 86463673597SKarsten Graul mutex_init(&smcibdev->mutex); 86592f3cb0eSUrsula Braun mutex_lock(&smc_ib_devices.mutex); 866a4cf0443SUrsula Braun list_add_tail(&smcibdev->list, &smc_ib_devices.list); 86792f3cb0eSUrsula Braun mutex_unlock(&smc_ib_devices.mutex); 868a4cf0443SUrsula Braun ib_set_client_data(ibdev, &smc_ib_client, smcibdev); 869be6a3f38SUrsula Braun INIT_IB_EVENT_HANDLER(&smcibdev->event_handler, smcibdev->ibdev, 870be6a3f38SUrsula Braun smc_ib_global_event_handler); 871be6a3f38SUrsula Braun ib_register_event_handler(&smcibdev->event_handler); 872be6a3f38SUrsula Braun 873be6a3f38SUrsula Braun /* trigger reading of the port attributes */ 874be6a3f38SUrsula Braun port_cnt = smcibdev->ibdev->phys_port_cnt; 8750a99be43SKarsten Graul pr_warn_ratelimited("smc: adding ib device %s with port count %d\n", 8760a99be43SKarsten Graul smcibdev->ibdev->name, port_cnt); 877be6a3f38SUrsula Braun for (i = 0; 878be6a3f38SUrsula Braun i < min_t(size_t, port_cnt, SMC_MAX_PORTS); 8790afff91cSUrsula Braun i++) { 880be6a3f38SUrsula Braun set_bit(i, &smcibdev->port_event_mask); 8810afff91cSUrsula Braun /* determine pnetids of the port */ 882fdff704dSKarsten Graul if (smc_pnetid_by_dev_port(ibdev->dev.parent, i, 883fdff704dSKarsten Graul smcibdev->pnetid[i])) 884fdff704dSKarsten Graul smc_pnetid_by_table_ib(smcibdev, i + 1); 8853d453f53SGuvenc Gulce smc_copy_netdev_ifindex(smcibdev, i); 8860a99be43SKarsten Graul pr_warn_ratelimited("smc: ib device %s port %d has pnetid " 8870a99be43SKarsten Graul "%.16s%s\n", 8880a99be43SKarsten Graul smcibdev->ibdev->name, i + 1, 8890a99be43SKarsten Graul smcibdev->pnetid[i], 8900a99be43SKarsten Graul smcibdev->pnetid_by_user[i] ? 8910a99be43SKarsten Graul " (user defined)" : 8920a99be43SKarsten Graul ""); 8930afff91cSUrsula Braun } 894be6a3f38SUrsula Braun schedule_work(&smcibdev->port_event_work); 89511a0ae4cSJason Gunthorpe return 0; 896a4cf0443SUrsula Braun } 897a4cf0443SUrsula Braun 8980b29ec64SUrsula Braun /* callback function for ib_unregister_client() */ 899a4cf0443SUrsula Braun static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data) 900a4cf0443SUrsula Braun { 9011587982eSJason Gunthorpe struct smc_ib_device *smcibdev = client_data; 902a4cf0443SUrsula Braun 90392f3cb0eSUrsula Braun mutex_lock(&smc_ib_devices.mutex); 904a4cf0443SUrsula Braun list_del_init(&smcibdev->list); /* remove from smc_ib_devices */ 90592f3cb0eSUrsula Braun mutex_unlock(&smc_ib_devices.mutex); 9060a99be43SKarsten Graul pr_warn_ratelimited("smc: removing ib device %s\n", 9070a99be43SKarsten Graul smcibdev->ibdev->name); 9080b29ec64SUrsula Braun smc_smcr_terminate_all(smcibdev); 909bd4ad577SUrsula Braun smc_ib_cleanup_per_ibdev(smcibdev); 910be6a3f38SUrsula Braun ib_unregister_event_handler(&smcibdev->event_handler); 911ece0d7bdSKarsten Graul cancel_work_sync(&smcibdev->port_event_work); 912a4cf0443SUrsula Braun kfree(smcibdev); 913a4cf0443SUrsula Braun } 914a4cf0443SUrsula Braun 915a4cf0443SUrsula Braun static struct ib_client smc_ib_client = { 916a4cf0443SUrsula Braun .name = "smc_ib", 917a4cf0443SUrsula Braun .add = smc_ib_add_dev, 918a4cf0443SUrsula Braun .remove = smc_ib_remove_dev, 919a4cf0443SUrsula Braun }; 920a4cf0443SUrsula Braun 921a4cf0443SUrsula Braun int __init smc_ib_register_client(void) 922a4cf0443SUrsula Braun { 923366bb249SHans Wippel smc_ib_init_local_systemid(); 924a4cf0443SUrsula Braun return ib_register_client(&smc_ib_client); 925a4cf0443SUrsula Braun } 926a4cf0443SUrsula Braun 927a4cf0443SUrsula Braun void smc_ib_unregister_client(void) 928a4cf0443SUrsula Braun { 929a4cf0443SUrsula Braun ib_unregister_client(&smc_ib_client); 930a4cf0443SUrsula Braun } 931