1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2a4cf0443SUrsula Braun /* 3a4cf0443SUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 4a4cf0443SUrsula Braun * 5a4cf0443SUrsula Braun * IB infrastructure: 6a4cf0443SUrsula Braun * Establish SMC-R as an Infiniband Client to be notified about added and 7a4cf0443SUrsula Braun * removed IB devices of type RDMA. 8a4cf0443SUrsula Braun * Determine device and port characteristics for these IB devices. 9a4cf0443SUrsula Braun * 10a4cf0443SUrsula Braun * Copyright IBM Corp. 2016 11a4cf0443SUrsula Braun * 12a4cf0443SUrsula Braun * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 13a4cf0443SUrsula Braun */ 14a4cf0443SUrsula Braun 15a4cf0443SUrsula Braun #include <linux/random.h> 16bd4ad577SUrsula Braun #include <linux/workqueue.h> 1710428dd8SUrsula Braun #include <linux/scatterlist.h> 186dabd405SUrsula Braun #include <linux/wait.h> 1992f3cb0eSUrsula Braun #include <linux/mutex.h> 20a4cf0443SUrsula Braun #include <rdma/ib_verbs.h> 21ddb457c6SParav Pandit #include <rdma/ib_cache.h> 22a4cf0443SUrsula Braun 236812baabSThomas Richter #include "smc_pnet.h" 24a4cf0443SUrsula Braun #include "smc_ib.h" 25cd6851f3SUrsula Braun #include "smc_core.h" 26f38ba179SUrsula Braun #include "smc_wr.h" 27a4cf0443SUrsula Braun #include "smc.h" 28a3db10efSGuvenc Gulce #include "smc_netlink.h" 29a4cf0443SUrsula Braun 30c9f4c6cfSUrsula Braun #define SMC_MAX_CQE 32766 /* max. # of completion queue elements */ 31c9f4c6cfSUrsula Braun 32bd4ad577SUrsula Braun #define SMC_QP_MIN_RNR_TIMER 5 33bd4ad577SUrsula Braun #define SMC_QP_TIMEOUT 15 /* 4096 * 2 ** timeout usec */ 34bd4ad577SUrsula Braun #define SMC_QP_RETRY_CNT 7 /* 7: infinite */ 35bd4ad577SUrsula Braun #define SMC_QP_RNR_RETRY 7 /* 7: infinite */ 36bd4ad577SUrsula Braun 37a4cf0443SUrsula Braun struct smc_ib_devices smc_ib_devices = { /* smc-registered ib devices */ 3892f3cb0eSUrsula Braun .mutex = __MUTEX_INITIALIZER(smc_ib_devices.mutex), 39a4cf0443SUrsula Braun .list = LIST_HEAD_INIT(smc_ib_devices.list), 40a4cf0443SUrsula Braun }; 41a4cf0443SUrsula Braun 42366bb249SHans Wippel u8 local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */ 43a4cf0443SUrsula Braun 44bd4ad577SUrsula Braun static int smc_ib_modify_qp_init(struct smc_link *lnk) 45bd4ad577SUrsula Braun { 46bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 47bd4ad577SUrsula Braun 48bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 49bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_INIT; 50bd4ad577SUrsula Braun qp_attr.pkey_index = 0; 51bd4ad577SUrsula Braun qp_attr.port_num = lnk->ibport; 52bd4ad577SUrsula Braun qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE 53bd4ad577SUrsula Braun | IB_ACCESS_REMOTE_WRITE; 54bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, 55bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_PKEY_INDEX | 56bd4ad577SUrsula Braun IB_QP_ACCESS_FLAGS | IB_QP_PORT); 57bd4ad577SUrsula Braun } 58bd4ad577SUrsula Braun 59bd4ad577SUrsula Braun static int smc_ib_modify_qp_rtr(struct smc_link *lnk) 60bd4ad577SUrsula Braun { 61bd4ad577SUrsula Braun enum ib_qp_attr_mask qp_attr_mask = 62bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | 63bd4ad577SUrsula Braun IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; 64bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 65bd4ad577SUrsula Braun 66bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 67bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RTR; 68bd4ad577SUrsula Braun qp_attr.path_mtu = min(lnk->path_mtu, lnk->peer_mtu); 6944c58487SDasaratharaman Chandramouli qp_attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 70d8966fcdSDasaratharaman Chandramouli rdma_ah_set_port_num(&qp_attr.ah_attr, lnk->ibport); 717005ada6SUrsula Braun rdma_ah_set_grh(&qp_attr.ah_attr, NULL, 0, lnk->sgid_index, 1, 0); 72d8966fcdSDasaratharaman Chandramouli rdma_ah_set_dgid_raw(&qp_attr.ah_attr, lnk->peer_gid); 7344c58487SDasaratharaman Chandramouli memcpy(&qp_attr.ah_attr.roce.dmac, lnk->peer_mac, 74bd4ad577SUrsula Braun sizeof(lnk->peer_mac)); 75bd4ad577SUrsula Braun qp_attr.dest_qp_num = lnk->peer_qpn; 76bd4ad577SUrsula Braun qp_attr.rq_psn = lnk->peer_psn; /* starting receive packet seq # */ 77bd4ad577SUrsula Braun qp_attr.max_dest_rd_atomic = 1; /* max # of resources for incoming 78bd4ad577SUrsula Braun * requests 79bd4ad577SUrsula Braun */ 80bd4ad577SUrsula Braun qp_attr.min_rnr_timer = SMC_QP_MIN_RNR_TIMER; 81bd4ad577SUrsula Braun 82bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, qp_attr_mask); 83bd4ad577SUrsula Braun } 84bd4ad577SUrsula Braun 85bd4ad577SUrsula Braun int smc_ib_modify_qp_rts(struct smc_link *lnk) 86bd4ad577SUrsula Braun { 87bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 88bd4ad577SUrsula Braun 89bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 90bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RTS; 91bd4ad577SUrsula Braun qp_attr.timeout = SMC_QP_TIMEOUT; /* local ack timeout */ 92bd4ad577SUrsula Braun qp_attr.retry_cnt = SMC_QP_RETRY_CNT; /* retry count */ 93bd4ad577SUrsula Braun qp_attr.rnr_retry = SMC_QP_RNR_RETRY; /* RNR retries, 7=infinite */ 94bd4ad577SUrsula Braun qp_attr.sq_psn = lnk->psn_initial; /* starting send packet seq # */ 95bd4ad577SUrsula Braun qp_attr.max_rd_atomic = 1; /* # of outstanding RDMA reads and 96bd4ad577SUrsula Braun * atomic ops allowed 97bd4ad577SUrsula Braun */ 98bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, 99bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | 100bd4ad577SUrsula Braun IB_QP_SQ_PSN | IB_QP_RNR_RETRY | 101bd4ad577SUrsula Braun IB_QP_MAX_QP_RD_ATOMIC); 102bd4ad577SUrsula Braun } 103bd4ad577SUrsula Braun 104bd4ad577SUrsula Braun int smc_ib_modify_qp_reset(struct smc_link *lnk) 105bd4ad577SUrsula Braun { 106bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 107bd4ad577SUrsula Braun 108bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 109bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RESET; 110bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, IB_QP_STATE); 111bd4ad577SUrsula Braun } 112bd4ad577SUrsula Braun 113bd4ad577SUrsula Braun int smc_ib_ready_link(struct smc_link *lnk) 114bd4ad577SUrsula Braun { 11500e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(lnk); 116bd4ad577SUrsula Braun int rc = 0; 117bd4ad577SUrsula Braun 118bd4ad577SUrsula Braun rc = smc_ib_modify_qp_init(lnk); 119bd4ad577SUrsula Braun if (rc) 120bd4ad577SUrsula Braun goto out; 121bd4ad577SUrsula Braun 122bd4ad577SUrsula Braun rc = smc_ib_modify_qp_rtr(lnk); 123bd4ad577SUrsula Braun if (rc) 124bd4ad577SUrsula Braun goto out; 125bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 126bd4ad577SUrsula Braun rc = ib_req_notify_cq(lnk->smcibdev->roce_cq_recv, 127bd4ad577SUrsula Braun IB_CQ_SOLICITED_MASK); 128bd4ad577SUrsula Braun if (rc) 129bd4ad577SUrsula Braun goto out; 130bd4ad577SUrsula Braun rc = smc_wr_rx_post_init(lnk); 131bd4ad577SUrsula Braun if (rc) 132bd4ad577SUrsula Braun goto out; 133bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 134bd4ad577SUrsula Braun 135bd4ad577SUrsula Braun if (lgr->role == SMC_SERV) { 136bd4ad577SUrsula Braun rc = smc_ib_modify_qp_rts(lnk); 137bd4ad577SUrsula Braun if (rc) 138bd4ad577SUrsula Braun goto out; 139bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 140bd4ad577SUrsula Braun } 141bd4ad577SUrsula Braun out: 142bd4ad577SUrsula Braun return rc; 143bd4ad577SUrsula Braun } 144bd4ad577SUrsula Braun 1457005ada6SUrsula Braun static int smc_ib_fill_mac(struct smc_ib_device *smcibdev, u8 ibport) 146be6a3f38SUrsula Braun { 147b4c296f9SJason Gunthorpe const struct ib_gid_attr *attr; 1485102eca9SParav Pandit int rc; 149be6a3f38SUrsula Braun 150b4c296f9SJason Gunthorpe attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, 0); 151b4c296f9SJason Gunthorpe if (IS_ERR(attr)) 152be6a3f38SUrsula Braun return -ENODEV; 153be6a3f38SUrsula Braun 1545102eca9SParav Pandit rc = rdma_read_gid_l2_fields(attr, NULL, smcibdev->mac[ibport - 1]); 155b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 156b4c296f9SJason Gunthorpe return rc; 157be6a3f38SUrsula Braun } 158be6a3f38SUrsula Braun 159be6a3f38SUrsula Braun /* Create an identifier unique for this instance of SMC-R. 160be6a3f38SUrsula Braun * The MAC-address of the first active registered IB device 161be6a3f38SUrsula Braun * plus a random 2-byte number is used to create this identifier. 162be6a3f38SUrsula Braun * This name is delivered to the peer during connection initialization. 163be6a3f38SUrsula Braun */ 164be6a3f38SUrsula Braun static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev, 165be6a3f38SUrsula Braun u8 ibport) 166be6a3f38SUrsula Braun { 167be6a3f38SUrsula Braun memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1], 168be6a3f38SUrsula Braun sizeof(smcibdev->mac[ibport - 1])); 169366bb249SHans Wippel } 170366bb249SHans Wippel 171a082ec89SHans Wippel bool smc_ib_is_valid_local_systemid(void) 172366bb249SHans Wippel { 173366bb249SHans Wippel return !is_zero_ether_addr(&local_systemid[2]); 174366bb249SHans Wippel } 175366bb249SHans Wippel 176366bb249SHans Wippel static void smc_ib_init_local_systemid(void) 177366bb249SHans Wippel { 178be6a3f38SUrsula Braun get_random_bytes(&local_systemid[0], 2); 179be6a3f38SUrsula Braun } 180be6a3f38SUrsula Braun 181be6a3f38SUrsula Braun bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport) 182be6a3f38SUrsula Braun { 183be6a3f38SUrsula Braun return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE; 184be6a3f38SUrsula Braun } 185be6a3f38SUrsula Braun 186*e5c4744cSKarsten Graul int smc_ib_find_route(__be32 saddr, __be32 daddr, 187*e5c4744cSKarsten Graul u8 nexthop_mac[], u8 *uses_gateway) 188*e5c4744cSKarsten Graul { 189*e5c4744cSKarsten Graul struct neighbour *neigh = NULL; 190*e5c4744cSKarsten Graul struct rtable *rt = NULL; 191*e5c4744cSKarsten Graul struct flowi4 fl4 = { 192*e5c4744cSKarsten Graul .saddr = saddr, 193*e5c4744cSKarsten Graul .daddr = daddr 194*e5c4744cSKarsten Graul }; 195*e5c4744cSKarsten Graul 196*e5c4744cSKarsten Graul if (daddr == cpu_to_be32(INADDR_NONE)) 197*e5c4744cSKarsten Graul goto out; 198*e5c4744cSKarsten Graul rt = ip_route_output_flow(&init_net, &fl4, NULL); 199*e5c4744cSKarsten Graul if (IS_ERR(rt)) 200*e5c4744cSKarsten Graul goto out; 201*e5c4744cSKarsten Graul if (rt->rt_uses_gateway && rt->rt_gw_family != AF_INET) 202*e5c4744cSKarsten Graul goto out; 203*e5c4744cSKarsten Graul neigh = rt->dst.ops->neigh_lookup(&rt->dst, NULL, &fl4.daddr); 204*e5c4744cSKarsten Graul if (neigh) { 205*e5c4744cSKarsten Graul memcpy(nexthop_mac, neigh->ha, ETH_ALEN); 206*e5c4744cSKarsten Graul *uses_gateway = rt->rt_uses_gateway; 207*e5c4744cSKarsten Graul return 0; 208*e5c4744cSKarsten Graul } 209*e5c4744cSKarsten Graul out: 210*e5c4744cSKarsten Graul return -ENOENT; 211*e5c4744cSKarsten Graul } 212*e5c4744cSKarsten Graul 2137005ada6SUrsula Braun /* determine the gid for an ib-device port and vlan id */ 2147005ada6SUrsula Braun int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport, 2157005ada6SUrsula Braun unsigned short vlan_id, u8 gid[], u8 *sgid_index) 2167005ada6SUrsula Braun { 217b4c296f9SJason Gunthorpe const struct ib_gid_attr *attr; 2185102eca9SParav Pandit const struct net_device *ndev; 2197005ada6SUrsula Braun int i; 2207005ada6SUrsula Braun 2217005ada6SUrsula Braun for (i = 0; i < smcibdev->pattr[ibport - 1].gid_tbl_len; i++) { 222b4c296f9SJason Gunthorpe attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, i); 223b4c296f9SJason Gunthorpe if (IS_ERR(attr)) 2247005ada6SUrsula Braun continue; 225b4c296f9SJason Gunthorpe 2265102eca9SParav Pandit rcu_read_lock(); 2275102eca9SParav Pandit ndev = rdma_read_gid_attr_ndev_rcu(attr); 2285102eca9SParav Pandit if (!IS_ERR(ndev) && 22941a0be3fSKarsten Graul ((!vlan_id && !is_vlan_dev(ndev)) || 23041a0be3fSKarsten Graul (vlan_id && is_vlan_dev(ndev) && 23141a0be3fSKarsten Graul vlan_dev_vlan_id(ndev) == vlan_id)) && 232b4c296f9SJason Gunthorpe attr->gid_type == IB_GID_TYPE_ROCE) { 2335102eca9SParav Pandit rcu_read_unlock(); 2347005ada6SUrsula Braun if (gid) 235b4c296f9SJason Gunthorpe memcpy(gid, &attr->gid, SMC_GID_SIZE); 2367005ada6SUrsula Braun if (sgid_index) 237b4c296f9SJason Gunthorpe *sgid_index = attr->index; 238b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 2397005ada6SUrsula Braun return 0; 2407005ada6SUrsula Braun } 2415102eca9SParav Pandit rcu_read_unlock(); 242b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 2437005ada6SUrsula Braun } 2447005ada6SUrsula Braun return -ENODEV; 2457005ada6SUrsula Braun } 2467005ada6SUrsula Braun 247be6a3f38SUrsula Braun static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport) 248be6a3f38SUrsula Braun { 249be6a3f38SUrsula Braun int rc; 250be6a3f38SUrsula Braun 251be6a3f38SUrsula Braun memset(&smcibdev->pattr[ibport - 1], 0, 252be6a3f38SUrsula Braun sizeof(smcibdev->pattr[ibport - 1])); 253be6a3f38SUrsula Braun rc = ib_query_port(smcibdev->ibdev, ibport, 254be6a3f38SUrsula Braun &smcibdev->pattr[ibport - 1]); 255be6a3f38SUrsula Braun if (rc) 256be6a3f38SUrsula Braun goto out; 257be6a3f38SUrsula Braun /* the SMC protocol requires specification of the RoCE MAC address */ 2587005ada6SUrsula Braun rc = smc_ib_fill_mac(smcibdev, ibport); 259be6a3f38SUrsula Braun if (rc) 260be6a3f38SUrsula Braun goto out; 261366bb249SHans Wippel if (!smc_ib_is_valid_local_systemid() && 262be6a3f38SUrsula Braun smc_ib_port_active(smcibdev, ibport)) 263be6a3f38SUrsula Braun /* create unique system identifier */ 264be6a3f38SUrsula Braun smc_ib_define_local_systemid(smcibdev, ibport); 265be6a3f38SUrsula Braun out: 266be6a3f38SUrsula Braun return rc; 267be6a3f38SUrsula Braun } 268be6a3f38SUrsula Braun 269bd4ad577SUrsula Braun /* process context wrapper for might_sleep smc_ib_remember_port_attr */ 270bd4ad577SUrsula Braun static void smc_ib_port_event_work(struct work_struct *work) 271bd4ad577SUrsula Braun { 272bd4ad577SUrsula Braun struct smc_ib_device *smcibdev = container_of( 273bd4ad577SUrsula Braun work, struct smc_ib_device, port_event_work); 274bd4ad577SUrsula Braun u8 port_idx; 275bd4ad577SUrsula Braun 276bd4ad577SUrsula Braun for_each_set_bit(port_idx, &smcibdev->port_event_mask, SMC_MAX_PORTS) { 277bd4ad577SUrsula Braun smc_ib_remember_port_attr(smcibdev, port_idx + 1); 278bd4ad577SUrsula Braun clear_bit(port_idx, &smcibdev->port_event_mask); 279c3d9494eSUrsula Braun if (!smc_ib_port_active(smcibdev, port_idx + 1)) { 280c3d9494eSUrsula Braun set_bit(port_idx, smcibdev->ports_going_away); 281541afa10SKarsten Graul smcr_port_err(smcibdev, port_idx + 1); 282c3d9494eSUrsula Braun } else { 283c3d9494eSUrsula Braun clear_bit(port_idx, smcibdev->ports_going_away); 2841f90a05dSKarsten Graul smcr_port_add(smcibdev, port_idx + 1); 285c3d9494eSUrsula Braun } 286bd4ad577SUrsula Braun } 287bd4ad577SUrsula Braun } 288bd4ad577SUrsula Braun 289bd4ad577SUrsula Braun /* can be called in IRQ context */ 290bd4ad577SUrsula Braun static void smc_ib_global_event_handler(struct ib_event_handler *handler, 291bd4ad577SUrsula Braun struct ib_event *ibevent) 292bd4ad577SUrsula Braun { 293bd4ad577SUrsula Braun struct smc_ib_device *smcibdev; 2945613f20cSUrsula Braun bool schedule = false; 295bd4ad577SUrsula Braun u8 port_idx; 296bd4ad577SUrsula Braun 297bd4ad577SUrsula Braun smcibdev = container_of(handler, struct smc_ib_device, event_handler); 298bd4ad577SUrsula Braun 299bd4ad577SUrsula Braun switch (ibevent->event) { 300bd4ad577SUrsula Braun case IB_EVENT_DEVICE_FATAL: 30181cf6430SKarsten Graul /* terminate all ports on device */ 302c3d9494eSUrsula Braun for (port_idx = 0; port_idx < SMC_MAX_PORTS; port_idx++) { 303bd4ad577SUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 3045613f20cSUrsula Braun if (!test_and_set_bit(port_idx, 3055613f20cSUrsula Braun smcibdev->ports_going_away)) 3065613f20cSUrsula Braun schedule = true; 307c3d9494eSUrsula Braun } 3085613f20cSUrsula Braun if (schedule) 3095613f20cSUrsula Braun schedule_work(&smcibdev->port_event_work); 3105613f20cSUrsula Braun break; 3115613f20cSUrsula Braun case IB_EVENT_PORT_ACTIVE: 3125613f20cSUrsula Braun port_idx = ibevent->element.port_num - 1; 3135613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3145613f20cSUrsula Braun break; 3155613f20cSUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 3165613f20cSUrsula Braun if (test_and_clear_bit(port_idx, smcibdev->ports_going_away)) 317bd4ad577SUrsula Braun schedule_work(&smcibdev->port_event_work); 318bd4ad577SUrsula Braun break; 31981cf6430SKarsten Graul case IB_EVENT_PORT_ERR: 3205613f20cSUrsula Braun port_idx = ibevent->element.port_num - 1; 3215613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3225613f20cSUrsula Braun break; 3235613f20cSUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 3245613f20cSUrsula Braun if (!test_and_set_bit(port_idx, smcibdev->ports_going_away)) 3255613f20cSUrsula Braun schedule_work(&smcibdev->port_event_work); 3265613f20cSUrsula Braun break; 32781cf6430SKarsten Graul case IB_EVENT_GID_CHANGE: 32881cf6430SKarsten Graul port_idx = ibevent->element.port_num - 1; 3295613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3305613f20cSUrsula Braun break; 33181cf6430SKarsten Graul set_bit(port_idx, &smcibdev->port_event_mask); 33281cf6430SKarsten Graul schedule_work(&smcibdev->port_event_work); 33381cf6430SKarsten Graul break; 334bd4ad577SUrsula Braun default: 335bd4ad577SUrsula Braun break; 336bd4ad577SUrsula Braun } 337bd4ad577SUrsula Braun } 338bd4ad577SUrsula Braun 339f38ba179SUrsula Braun void smc_ib_dealloc_protection_domain(struct smc_link *lnk) 340f38ba179SUrsula Braun { 341da05bf29SUrsula Braun if (lnk->roce_pd) 342f38ba179SUrsula Braun ib_dealloc_pd(lnk->roce_pd); 343f38ba179SUrsula Braun lnk->roce_pd = NULL; 344f38ba179SUrsula Braun } 345f38ba179SUrsula Braun 346f38ba179SUrsula Braun int smc_ib_create_protection_domain(struct smc_link *lnk) 347f38ba179SUrsula Braun { 348f38ba179SUrsula Braun int rc; 349f38ba179SUrsula Braun 350897e1c24SUrsula Braun lnk->roce_pd = ib_alloc_pd(lnk->smcibdev->ibdev, 0); 351f38ba179SUrsula Braun rc = PTR_ERR_OR_ZERO(lnk->roce_pd); 352f38ba179SUrsula Braun if (IS_ERR(lnk->roce_pd)) 353f38ba179SUrsula Braun lnk->roce_pd = NULL; 354f38ba179SUrsula Braun return rc; 355f38ba179SUrsula Braun } 356f38ba179SUrsula Braun 357a3db10efSGuvenc Gulce static bool smcr_diag_is_dev_critical(struct smc_lgr_list *smc_lgr, 358a3db10efSGuvenc Gulce struct smc_ib_device *smcibdev) 359a3db10efSGuvenc Gulce { 360a3db10efSGuvenc Gulce struct smc_link_group *lgr; 361a3db10efSGuvenc Gulce bool rc = false; 362a3db10efSGuvenc Gulce int i; 363a3db10efSGuvenc Gulce 364a3db10efSGuvenc Gulce spin_lock_bh(&smc_lgr->lock); 365a3db10efSGuvenc Gulce list_for_each_entry(lgr, &smc_lgr->list, list) { 366a3db10efSGuvenc Gulce if (lgr->is_smcd) 367a3db10efSGuvenc Gulce continue; 368a3db10efSGuvenc Gulce for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 369a3db10efSGuvenc Gulce if (lgr->lnk[i].state == SMC_LNK_UNUSED || 370a3db10efSGuvenc Gulce lgr->lnk[i].smcibdev != smcibdev) 371a3db10efSGuvenc Gulce continue; 372a3db10efSGuvenc Gulce if (lgr->type == SMC_LGR_SINGLE || 373a3db10efSGuvenc Gulce lgr->type == SMC_LGR_ASYMMETRIC_LOCAL) { 374a3db10efSGuvenc Gulce rc = true; 375a3db10efSGuvenc Gulce goto out; 376a3db10efSGuvenc Gulce } 377a3db10efSGuvenc Gulce } 378a3db10efSGuvenc Gulce } 379a3db10efSGuvenc Gulce out: 380a3db10efSGuvenc Gulce spin_unlock_bh(&smc_lgr->lock); 381a3db10efSGuvenc Gulce return rc; 382a3db10efSGuvenc Gulce } 383a3db10efSGuvenc Gulce 384a3db10efSGuvenc Gulce static int smc_nl_handle_dev_port(struct sk_buff *skb, 385a3db10efSGuvenc Gulce struct ib_device *ibdev, 386a3db10efSGuvenc Gulce struct smc_ib_device *smcibdev, 387a3db10efSGuvenc Gulce int port) 388a3db10efSGuvenc Gulce { 389a3db10efSGuvenc Gulce char smc_pnet[SMC_MAX_PNETID_LEN + 1]; 390a3db10efSGuvenc Gulce struct nlattr *port_attrs; 391a3db10efSGuvenc Gulce unsigned char port_state; 392a3db10efSGuvenc Gulce int lnk_count = 0; 393a3db10efSGuvenc Gulce 394a3db10efSGuvenc Gulce port_attrs = nla_nest_start(skb, SMC_NLA_DEV_PORT + port); 395a3db10efSGuvenc Gulce if (!port_attrs) 396a3db10efSGuvenc Gulce goto errout; 397a3db10efSGuvenc Gulce 398a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_PORT_PNET_USR, 399a3db10efSGuvenc Gulce smcibdev->pnetid_by_user[port])) 400a3db10efSGuvenc Gulce goto errattr; 4018a446536SGuvenc Gulce memcpy(smc_pnet, &smcibdev->pnetid[port], SMC_MAX_PNETID_LEN); 4028a446536SGuvenc Gulce smc_pnet[SMC_MAX_PNETID_LEN] = 0; 403a3db10efSGuvenc Gulce if (nla_put_string(skb, SMC_NLA_DEV_PORT_PNETID, smc_pnet)) 404a3db10efSGuvenc Gulce goto errattr; 405a3db10efSGuvenc Gulce if (nla_put_u32(skb, SMC_NLA_DEV_PORT_NETDEV, 406a3db10efSGuvenc Gulce smcibdev->ndev_ifidx[port])) 407a3db10efSGuvenc Gulce goto errattr; 408a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_PORT_VALID, 1)) 409a3db10efSGuvenc Gulce goto errattr; 410a3db10efSGuvenc Gulce port_state = smc_ib_port_active(smcibdev, port + 1); 411a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_PORT_STATE, port_state)) 412a3db10efSGuvenc Gulce goto errattr; 413a3db10efSGuvenc Gulce lnk_count = atomic_read(&smcibdev->lnk_cnt_by_port[port]); 414a3db10efSGuvenc Gulce if (nla_put_u32(skb, SMC_NLA_DEV_PORT_LNK_CNT, lnk_count)) 415a3db10efSGuvenc Gulce goto errattr; 416a3db10efSGuvenc Gulce nla_nest_end(skb, port_attrs); 417a3db10efSGuvenc Gulce return 0; 418a3db10efSGuvenc Gulce errattr: 419a3db10efSGuvenc Gulce nla_nest_cancel(skb, port_attrs); 420a3db10efSGuvenc Gulce errout: 421a3db10efSGuvenc Gulce return -EMSGSIZE; 422a3db10efSGuvenc Gulce } 423a3db10efSGuvenc Gulce 424995433b7SKarsten Graul static bool smc_nl_handle_pci_values(const struct smc_pci_dev *smc_pci_dev, 425995433b7SKarsten Graul struct sk_buff *skb) 426995433b7SKarsten Graul { 427995433b7SKarsten Graul if (nla_put_u32(skb, SMC_NLA_DEV_PCI_FID, smc_pci_dev->pci_fid)) 428995433b7SKarsten Graul return false; 429995433b7SKarsten Graul if (nla_put_u16(skb, SMC_NLA_DEV_PCI_CHID, smc_pci_dev->pci_pchid)) 430995433b7SKarsten Graul return false; 431995433b7SKarsten Graul if (nla_put_u16(skb, SMC_NLA_DEV_PCI_VENDOR, smc_pci_dev->pci_vendor)) 432995433b7SKarsten Graul return false; 433995433b7SKarsten Graul if (nla_put_u16(skb, SMC_NLA_DEV_PCI_DEVICE, smc_pci_dev->pci_device)) 434995433b7SKarsten Graul return false; 435995433b7SKarsten Graul if (nla_put_string(skb, SMC_NLA_DEV_PCI_ID, smc_pci_dev->pci_id)) 436995433b7SKarsten Graul return false; 437995433b7SKarsten Graul return true; 438995433b7SKarsten Graul } 439995433b7SKarsten Graul 440a3db10efSGuvenc Gulce static int smc_nl_handle_smcr_dev(struct smc_ib_device *smcibdev, 441a3db10efSGuvenc Gulce struct sk_buff *skb, 442a3db10efSGuvenc Gulce struct netlink_callback *cb) 443a3db10efSGuvenc Gulce { 4448a446536SGuvenc Gulce char smc_ibname[IB_DEVICE_NAME_MAX]; 445a3db10efSGuvenc Gulce struct smc_pci_dev smc_pci_dev; 446a3db10efSGuvenc Gulce struct pci_dev *pci_dev; 447a3db10efSGuvenc Gulce unsigned char is_crit; 448a3db10efSGuvenc Gulce struct nlattr *attrs; 449a3db10efSGuvenc Gulce void *nlh; 450a3db10efSGuvenc Gulce int i; 451a3db10efSGuvenc Gulce 452a3db10efSGuvenc Gulce nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 453a3db10efSGuvenc Gulce &smc_gen_nl_family, NLM_F_MULTI, 454a3db10efSGuvenc Gulce SMC_NETLINK_GET_DEV_SMCR); 455a3db10efSGuvenc Gulce if (!nlh) 456a3db10efSGuvenc Gulce goto errmsg; 457a3db10efSGuvenc Gulce attrs = nla_nest_start(skb, SMC_GEN_DEV_SMCR); 458a3db10efSGuvenc Gulce if (!attrs) 459a3db10efSGuvenc Gulce goto errout; 460a3db10efSGuvenc Gulce is_crit = smcr_diag_is_dev_critical(&smc_lgr_list, smcibdev); 461a3db10efSGuvenc Gulce if (nla_put_u8(skb, SMC_NLA_DEV_IS_CRIT, is_crit)) 462a3db10efSGuvenc Gulce goto errattr; 463995433b7SKarsten Graul if (smcibdev->ibdev->dev.parent) { 464a3db10efSGuvenc Gulce memset(&smc_pci_dev, 0, sizeof(smc_pci_dev)); 465a3db10efSGuvenc Gulce pci_dev = to_pci_dev(smcibdev->ibdev->dev.parent); 466a3db10efSGuvenc Gulce smc_set_pci_values(pci_dev, &smc_pci_dev); 467995433b7SKarsten Graul if (!smc_nl_handle_pci_values(&smc_pci_dev, skb)) 468a3db10efSGuvenc Gulce goto errattr; 469995433b7SKarsten Graul } 470a3db10efSGuvenc Gulce snprintf(smc_ibname, sizeof(smc_ibname), "%s", smcibdev->ibdev->name); 471a3db10efSGuvenc Gulce if (nla_put_string(skb, SMC_NLA_DEV_IB_NAME, smc_ibname)) 472a3db10efSGuvenc Gulce goto errattr; 473a3db10efSGuvenc Gulce for (i = 1; i <= SMC_MAX_PORTS; i++) { 474a3db10efSGuvenc Gulce if (!rdma_is_port_valid(smcibdev->ibdev, i)) 475a3db10efSGuvenc Gulce continue; 476a3db10efSGuvenc Gulce if (smc_nl_handle_dev_port(skb, smcibdev->ibdev, 477a3db10efSGuvenc Gulce smcibdev, i - 1)) 478a3db10efSGuvenc Gulce goto errattr; 479a3db10efSGuvenc Gulce } 480a3db10efSGuvenc Gulce 481a3db10efSGuvenc Gulce nla_nest_end(skb, attrs); 482a3db10efSGuvenc Gulce genlmsg_end(skb, nlh); 483a3db10efSGuvenc Gulce return 0; 484a3db10efSGuvenc Gulce 485a3db10efSGuvenc Gulce errattr: 486a3db10efSGuvenc Gulce nla_nest_cancel(skb, attrs); 487a3db10efSGuvenc Gulce errout: 488a3db10efSGuvenc Gulce genlmsg_cancel(skb, nlh); 489a3db10efSGuvenc Gulce errmsg: 490a3db10efSGuvenc Gulce return -EMSGSIZE; 491a3db10efSGuvenc Gulce } 492a3db10efSGuvenc Gulce 493a3db10efSGuvenc Gulce static void smc_nl_prep_smcr_dev(struct smc_ib_devices *dev_list, 494a3db10efSGuvenc Gulce struct sk_buff *skb, 495a3db10efSGuvenc Gulce struct netlink_callback *cb) 496a3db10efSGuvenc Gulce { 497a3db10efSGuvenc Gulce struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); 498a3db10efSGuvenc Gulce struct smc_ib_device *smcibdev; 499a3db10efSGuvenc Gulce int snum = cb_ctx->pos[0]; 500a3db10efSGuvenc Gulce int num = 0; 501a3db10efSGuvenc Gulce 502a3db10efSGuvenc Gulce mutex_lock(&dev_list->mutex); 503a3db10efSGuvenc Gulce list_for_each_entry(smcibdev, &dev_list->list, list) { 504a3db10efSGuvenc Gulce if (num < snum) 505a3db10efSGuvenc Gulce goto next; 506a3db10efSGuvenc Gulce if (smc_nl_handle_smcr_dev(smcibdev, skb, cb)) 507a3db10efSGuvenc Gulce goto errout; 508a3db10efSGuvenc Gulce next: 509a3db10efSGuvenc Gulce num++; 510a3db10efSGuvenc Gulce } 511a3db10efSGuvenc Gulce errout: 512a3db10efSGuvenc Gulce mutex_unlock(&dev_list->mutex); 513a3db10efSGuvenc Gulce cb_ctx->pos[0] = num; 514a3db10efSGuvenc Gulce } 515a3db10efSGuvenc Gulce 516a3db10efSGuvenc Gulce int smcr_nl_get_device(struct sk_buff *skb, struct netlink_callback *cb) 517a3db10efSGuvenc Gulce { 518a3db10efSGuvenc Gulce smc_nl_prep_smcr_dev(&smc_ib_devices, skb, cb); 519a3db10efSGuvenc Gulce return skb->len; 520a3db10efSGuvenc Gulce } 521a3db10efSGuvenc Gulce 522f38ba179SUrsula Braun static void smc_ib_qp_event_handler(struct ib_event *ibevent, void *priv) 523f38ba179SUrsula Braun { 524e5f3aa04SKarsten Graul struct smc_link *lnk = (struct smc_link *)priv; 525e5f3aa04SKarsten Graul struct smc_ib_device *smcibdev = lnk->smcibdev; 526da05bf29SUrsula Braun u8 port_idx; 527da05bf29SUrsula Braun 528f38ba179SUrsula Braun switch (ibevent->event) { 52981cf6430SKarsten Graul case IB_EVENT_QP_FATAL: 530f38ba179SUrsula Braun case IB_EVENT_QP_ACCESS_ERR: 531e5f3aa04SKarsten Graul port_idx = ibevent->element.qp->port - 1; 5325613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 5335613f20cSUrsula Braun break; 534da05bf29SUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 5355613f20cSUrsula Braun if (!test_and_set_bit(port_idx, smcibdev->ports_going_away)) 536da05bf29SUrsula Braun schedule_work(&smcibdev->port_event_work); 537f38ba179SUrsula Braun break; 538f38ba179SUrsula Braun default: 539f38ba179SUrsula Braun break; 540f38ba179SUrsula Braun } 541f38ba179SUrsula Braun } 542f38ba179SUrsula Braun 543f38ba179SUrsula Braun void smc_ib_destroy_queue_pair(struct smc_link *lnk) 544f38ba179SUrsula Braun { 545da05bf29SUrsula Braun if (lnk->roce_qp) 546f38ba179SUrsula Braun ib_destroy_qp(lnk->roce_qp); 547f38ba179SUrsula Braun lnk->roce_qp = NULL; 548f38ba179SUrsula Braun } 549f38ba179SUrsula Braun 550f38ba179SUrsula Braun /* create a queue pair within the protection domain for a link */ 551f38ba179SUrsula Braun int smc_ib_create_queue_pair(struct smc_link *lnk) 552f38ba179SUrsula Braun { 553f38ba179SUrsula Braun struct ib_qp_init_attr qp_attr = { 554f38ba179SUrsula Braun .event_handler = smc_ib_qp_event_handler, 555f38ba179SUrsula Braun .qp_context = lnk, 556f38ba179SUrsula Braun .send_cq = lnk->smcibdev->roce_cq_send, 557f38ba179SUrsula Braun .recv_cq = lnk->smcibdev->roce_cq_recv, 558f38ba179SUrsula Braun .srq = NULL, 559f38ba179SUrsula Braun .cap = { 560f38ba179SUrsula Braun /* include unsolicited rdma_writes as well, 561f38ba179SUrsula Braun * there are max. 2 RDMA_WRITE per 1 WR_SEND 562f38ba179SUrsula Braun */ 563652a1e41SUrsula Braun .max_send_wr = SMC_WR_BUF_CNT * 3, 564f38ba179SUrsula Braun .max_recv_wr = SMC_WR_BUF_CNT * 3, 565f38ba179SUrsula Braun .max_send_sge = SMC_IB_MAX_SEND_SGE, 566f38ba179SUrsula Braun .max_recv_sge = 1, 567f38ba179SUrsula Braun }, 568f38ba179SUrsula Braun .sq_sig_type = IB_SIGNAL_REQ_WR, 569f38ba179SUrsula Braun .qp_type = IB_QPT_RC, 570f38ba179SUrsula Braun }; 571f38ba179SUrsula Braun int rc; 572f38ba179SUrsula Braun 573f38ba179SUrsula Braun lnk->roce_qp = ib_create_qp(lnk->roce_pd, &qp_attr); 574f38ba179SUrsula Braun rc = PTR_ERR_OR_ZERO(lnk->roce_qp); 575f38ba179SUrsula Braun if (IS_ERR(lnk->roce_qp)) 576f38ba179SUrsula Braun lnk->roce_qp = NULL; 577f38ba179SUrsula Braun else 578f38ba179SUrsula Braun smc_wr_remember_qp_attr(lnk); 579f38ba179SUrsula Braun return rc; 580f38ba179SUrsula Braun } 581f38ba179SUrsula Braun 582897e1c24SUrsula Braun void smc_ib_put_memory_region(struct ib_mr *mr) 583897e1c24SUrsula Braun { 584897e1c24SUrsula Braun ib_dereg_mr(mr); 585897e1c24SUrsula Braun } 586897e1c24SUrsula Braun 587387707fdSKarsten Graul static int smc_ib_map_mr_sg(struct smc_buf_desc *buf_slot, u8 link_idx) 588897e1c24SUrsula Braun { 589897e1c24SUrsula Braun unsigned int offset = 0; 590897e1c24SUrsula Braun int sg_num; 591897e1c24SUrsula Braun 592897e1c24SUrsula Braun /* map the largest prefix of a dma mapped SG list */ 593387707fdSKarsten Graul sg_num = ib_map_mr_sg(buf_slot->mr_rx[link_idx], 594387707fdSKarsten Graul buf_slot->sgt[link_idx].sgl, 595387707fdSKarsten Graul buf_slot->sgt[link_idx].orig_nents, 596897e1c24SUrsula Braun &offset, PAGE_SIZE); 597897e1c24SUrsula Braun 598897e1c24SUrsula Braun return sg_num; 599897e1c24SUrsula Braun } 600897e1c24SUrsula Braun 601897e1c24SUrsula Braun /* Allocate a memory region and map the dma mapped SG list of buf_slot */ 602897e1c24SUrsula Braun int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags, 603387707fdSKarsten Graul struct smc_buf_desc *buf_slot, u8 link_idx) 604897e1c24SUrsula Braun { 605387707fdSKarsten Graul if (buf_slot->mr_rx[link_idx]) 606897e1c24SUrsula Braun return 0; /* already done */ 607897e1c24SUrsula Braun 608387707fdSKarsten Graul buf_slot->mr_rx[link_idx] = 609897e1c24SUrsula Braun ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 1 << buf_slot->order); 610387707fdSKarsten Graul if (IS_ERR(buf_slot->mr_rx[link_idx])) { 611897e1c24SUrsula Braun int rc; 612897e1c24SUrsula Braun 613387707fdSKarsten Graul rc = PTR_ERR(buf_slot->mr_rx[link_idx]); 614387707fdSKarsten Graul buf_slot->mr_rx[link_idx] = NULL; 615897e1c24SUrsula Braun return rc; 616897e1c24SUrsula Braun } 617897e1c24SUrsula Braun 618387707fdSKarsten Graul if (smc_ib_map_mr_sg(buf_slot, link_idx) != 1) 619897e1c24SUrsula Braun return -EINVAL; 620897e1c24SUrsula Braun 621897e1c24SUrsula Braun return 0; 622897e1c24SUrsula Braun } 623897e1c24SUrsula Braun 62410428dd8SUrsula Braun /* synchronize buffer usage for cpu access */ 625387707fdSKarsten Graul void smc_ib_sync_sg_for_cpu(struct smc_link *lnk, 62610428dd8SUrsula Braun struct smc_buf_desc *buf_slot, 62710428dd8SUrsula Braun enum dma_data_direction data_direction) 62810428dd8SUrsula Braun { 62910428dd8SUrsula Braun struct scatterlist *sg; 63010428dd8SUrsula Braun unsigned int i; 63110428dd8SUrsula Braun 63210428dd8SUrsula Braun /* for now there is just one DMA address */ 633387707fdSKarsten Graul for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, 634387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].nents, i) { 63510428dd8SUrsula Braun if (!sg_dma_len(sg)) 63610428dd8SUrsula Braun break; 637387707fdSKarsten Graul ib_dma_sync_single_for_cpu(lnk->smcibdev->ibdev, 63810428dd8SUrsula Braun sg_dma_address(sg), 63910428dd8SUrsula Braun sg_dma_len(sg), 64010428dd8SUrsula Braun data_direction); 64110428dd8SUrsula Braun } 64210428dd8SUrsula Braun } 64310428dd8SUrsula Braun 64410428dd8SUrsula Braun /* synchronize buffer usage for device access */ 645387707fdSKarsten Graul void smc_ib_sync_sg_for_device(struct smc_link *lnk, 64610428dd8SUrsula Braun struct smc_buf_desc *buf_slot, 64710428dd8SUrsula Braun enum dma_data_direction data_direction) 64810428dd8SUrsula Braun { 64910428dd8SUrsula Braun struct scatterlist *sg; 65010428dd8SUrsula Braun unsigned int i; 65110428dd8SUrsula Braun 65210428dd8SUrsula Braun /* for now there is just one DMA address */ 653387707fdSKarsten Graul for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, 654387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].nents, i) { 65510428dd8SUrsula Braun if (!sg_dma_len(sg)) 65610428dd8SUrsula Braun break; 657387707fdSKarsten Graul ib_dma_sync_single_for_device(lnk->smcibdev->ibdev, 65810428dd8SUrsula Braun sg_dma_address(sg), 65910428dd8SUrsula Braun sg_dma_len(sg), 66010428dd8SUrsula Braun data_direction); 66110428dd8SUrsula Braun } 66210428dd8SUrsula Braun } 66310428dd8SUrsula Braun 664a3fe3d01SUrsula Braun /* Map a new TX or RX buffer SG-table to DMA */ 665387707fdSKarsten Graul int smc_ib_buf_map_sg(struct smc_link *lnk, 666a3fe3d01SUrsula Braun struct smc_buf_desc *buf_slot, 667a3fe3d01SUrsula Braun enum dma_data_direction data_direction) 668a3fe3d01SUrsula Braun { 669a3fe3d01SUrsula Braun int mapped_nents; 670a3fe3d01SUrsula Braun 671387707fdSKarsten Graul mapped_nents = ib_dma_map_sg(lnk->smcibdev->ibdev, 672387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl, 673387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].orig_nents, 674a3fe3d01SUrsula Braun data_direction); 675a3fe3d01SUrsula Braun if (!mapped_nents) 676a3fe3d01SUrsula Braun return -ENOMEM; 677a3fe3d01SUrsula Braun 678a3fe3d01SUrsula Braun return mapped_nents; 679a3fe3d01SUrsula Braun } 680a3fe3d01SUrsula Braun 681387707fdSKarsten Graul void smc_ib_buf_unmap_sg(struct smc_link *lnk, 682a3fe3d01SUrsula Braun struct smc_buf_desc *buf_slot, 683a3fe3d01SUrsula Braun enum dma_data_direction data_direction) 684a3fe3d01SUrsula Braun { 685387707fdSKarsten Graul if (!buf_slot->sgt[lnk->link_idx].sgl->dma_address) 686a3fe3d01SUrsula Braun return; /* already unmapped */ 687a3fe3d01SUrsula Braun 688387707fdSKarsten Graul ib_dma_unmap_sg(lnk->smcibdev->ibdev, 689387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl, 690387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].orig_nents, 691a3fe3d01SUrsula Braun data_direction); 692387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl->dma_address = 0; 693a3fe3d01SUrsula Braun } 694a3fe3d01SUrsula Braun 695bd4ad577SUrsula Braun long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev) 696bd4ad577SUrsula Braun { 697bd4ad577SUrsula Braun struct ib_cq_init_attr cqattr = { 698c9f4c6cfSUrsula Braun .cqe = SMC_MAX_CQE, .comp_vector = 0 }; 699c9f4c6cfSUrsula Braun int cqe_size_order, smc_order; 700bd4ad577SUrsula Braun long rc; 701bd4ad577SUrsula Braun 70263673597SKarsten Graul mutex_lock(&smcibdev->mutex); 70363673597SKarsten Graul rc = 0; 70463673597SKarsten Graul if (smcibdev->initialized) 70563673597SKarsten Graul goto out; 706c9f4c6cfSUrsula Braun /* the calculated number of cq entries fits to mlx5 cq allocation */ 707c9f4c6cfSUrsula Braun cqe_size_order = cache_line_size() == 128 ? 7 : 6; 708c9f4c6cfSUrsula Braun smc_order = MAX_ORDER - cqe_size_order - 1; 709c9f4c6cfSUrsula Braun if (SMC_MAX_CQE + 2 > (0x00000001 << smc_order) * PAGE_SIZE) 710c9f4c6cfSUrsula Braun cqattr.cqe = (0x00000001 << smc_order) * PAGE_SIZE - 2; 711bd4ad577SUrsula Braun smcibdev->roce_cq_send = ib_create_cq(smcibdev->ibdev, 712bd4ad577SUrsula Braun smc_wr_tx_cq_handler, NULL, 713bd4ad577SUrsula Braun smcibdev, &cqattr); 714bd4ad577SUrsula Braun rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_send); 715bd4ad577SUrsula Braun if (IS_ERR(smcibdev->roce_cq_send)) { 716bd4ad577SUrsula Braun smcibdev->roce_cq_send = NULL; 71763673597SKarsten Graul goto out; 718bd4ad577SUrsula Braun } 719bd4ad577SUrsula Braun smcibdev->roce_cq_recv = ib_create_cq(smcibdev->ibdev, 720bd4ad577SUrsula Braun smc_wr_rx_cq_handler, NULL, 721bd4ad577SUrsula Braun smcibdev, &cqattr); 722bd4ad577SUrsula Braun rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_recv); 723bd4ad577SUrsula Braun if (IS_ERR(smcibdev->roce_cq_recv)) { 724bd4ad577SUrsula Braun smcibdev->roce_cq_recv = NULL; 725bd4ad577SUrsula Braun goto err; 726bd4ad577SUrsula Braun } 727bd4ad577SUrsula Braun smc_wr_add_dev(smcibdev); 728bd4ad577SUrsula Braun smcibdev->initialized = 1; 72963673597SKarsten Graul goto out; 730bd4ad577SUrsula Braun 731bd4ad577SUrsula Braun err: 732bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_send); 73363673597SKarsten Graul out: 73463673597SKarsten Graul mutex_unlock(&smcibdev->mutex); 735bd4ad577SUrsula Braun return rc; 736bd4ad577SUrsula Braun } 737bd4ad577SUrsula Braun 738bd4ad577SUrsula Braun static void smc_ib_cleanup_per_ibdev(struct smc_ib_device *smcibdev) 739bd4ad577SUrsula Braun { 74063673597SKarsten Graul mutex_lock(&smcibdev->mutex); 741bd4ad577SUrsula Braun if (!smcibdev->initialized) 74263673597SKarsten Graul goto out; 743da05bf29SUrsula Braun smcibdev->initialized = 0; 744bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_recv); 745bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_send); 7466a37ad3dSUrsula Braun smc_wr_remove_dev(smcibdev); 74763673597SKarsten Graul out: 74863673597SKarsten Graul mutex_unlock(&smcibdev->mutex); 749bd4ad577SUrsula Braun } 750bd4ad577SUrsula Braun 751a4cf0443SUrsula Braun static struct ib_client smc_ib_client; 752a4cf0443SUrsula Braun 7533d453f53SGuvenc Gulce static void smc_copy_netdev_ifindex(struct smc_ib_device *smcibdev, int port) 7543d453f53SGuvenc Gulce { 7553d453f53SGuvenc Gulce struct ib_device *ibdev = smcibdev->ibdev; 7563d453f53SGuvenc Gulce struct net_device *ndev; 7573d453f53SGuvenc Gulce 7583d453f53SGuvenc Gulce if (!ibdev->ops.get_netdev) 7593d453f53SGuvenc Gulce return; 7603d453f53SGuvenc Gulce ndev = ibdev->ops.get_netdev(ibdev, port + 1); 7613d453f53SGuvenc Gulce if (ndev) { 7623d453f53SGuvenc Gulce smcibdev->ndev_ifidx[port] = ndev->ifindex; 7633d453f53SGuvenc Gulce dev_put(ndev); 7643d453f53SGuvenc Gulce } 7653d453f53SGuvenc Gulce } 7663d453f53SGuvenc Gulce 7673d453f53SGuvenc Gulce void smc_ib_ndev_change(struct net_device *ndev, unsigned long event) 7683d453f53SGuvenc Gulce { 7693d453f53SGuvenc Gulce struct smc_ib_device *smcibdev; 7703d453f53SGuvenc Gulce struct ib_device *libdev; 7713d453f53SGuvenc Gulce struct net_device *lndev; 7723d453f53SGuvenc Gulce u8 port_cnt; 7733d453f53SGuvenc Gulce int i; 7743d453f53SGuvenc Gulce 7753d453f53SGuvenc Gulce mutex_lock(&smc_ib_devices.mutex); 7763d453f53SGuvenc Gulce list_for_each_entry(smcibdev, &smc_ib_devices.list, list) { 7773d453f53SGuvenc Gulce port_cnt = smcibdev->ibdev->phys_port_cnt; 7783d453f53SGuvenc Gulce for (i = 0; i < min_t(size_t, port_cnt, SMC_MAX_PORTS); i++) { 7793d453f53SGuvenc Gulce libdev = smcibdev->ibdev; 7803d453f53SGuvenc Gulce if (!libdev->ops.get_netdev) 7813d453f53SGuvenc Gulce continue; 7823d453f53SGuvenc Gulce lndev = libdev->ops.get_netdev(libdev, i + 1); 7833d453f53SGuvenc Gulce dev_put(lndev); 7843d453f53SGuvenc Gulce if (lndev != ndev) 7853d453f53SGuvenc Gulce continue; 7863d453f53SGuvenc Gulce if (event == NETDEV_REGISTER) 7873d453f53SGuvenc Gulce smcibdev->ndev_ifidx[i] = ndev->ifindex; 7883d453f53SGuvenc Gulce if (event == NETDEV_UNREGISTER) 7893d453f53SGuvenc Gulce smcibdev->ndev_ifidx[i] = 0; 7903d453f53SGuvenc Gulce } 7913d453f53SGuvenc Gulce } 7923d453f53SGuvenc Gulce mutex_unlock(&smc_ib_devices.mutex); 7933d453f53SGuvenc Gulce } 7943d453f53SGuvenc Gulce 795a4cf0443SUrsula Braun /* callback function for ib_register_client() */ 79611a0ae4cSJason Gunthorpe static int smc_ib_add_dev(struct ib_device *ibdev) 797a4cf0443SUrsula Braun { 798a4cf0443SUrsula Braun struct smc_ib_device *smcibdev; 799be6a3f38SUrsula Braun u8 port_cnt; 800be6a3f38SUrsula Braun int i; 801a4cf0443SUrsula Braun 802a4cf0443SUrsula Braun if (ibdev->node_type != RDMA_NODE_IB_CA) 80311a0ae4cSJason Gunthorpe return -EOPNOTSUPP; 804a4cf0443SUrsula Braun 805a4cf0443SUrsula Braun smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL); 806a4cf0443SUrsula Braun if (!smcibdev) 80711a0ae4cSJason Gunthorpe return -ENOMEM; 808a4cf0443SUrsula Braun 809a4cf0443SUrsula Braun smcibdev->ibdev = ibdev; 810bd4ad577SUrsula Braun INIT_WORK(&smcibdev->port_event_work, smc_ib_port_event_work); 8116dabd405SUrsula Braun atomic_set(&smcibdev->lnk_cnt, 0); 8126dabd405SUrsula Braun init_waitqueue_head(&smcibdev->lnks_deleted); 81363673597SKarsten Graul mutex_init(&smcibdev->mutex); 81492f3cb0eSUrsula Braun mutex_lock(&smc_ib_devices.mutex); 815a4cf0443SUrsula Braun list_add_tail(&smcibdev->list, &smc_ib_devices.list); 81692f3cb0eSUrsula Braun mutex_unlock(&smc_ib_devices.mutex); 817a4cf0443SUrsula Braun ib_set_client_data(ibdev, &smc_ib_client, smcibdev); 818be6a3f38SUrsula Braun INIT_IB_EVENT_HANDLER(&smcibdev->event_handler, smcibdev->ibdev, 819be6a3f38SUrsula Braun smc_ib_global_event_handler); 820be6a3f38SUrsula Braun ib_register_event_handler(&smcibdev->event_handler); 821be6a3f38SUrsula Braun 822be6a3f38SUrsula Braun /* trigger reading of the port attributes */ 823be6a3f38SUrsula Braun port_cnt = smcibdev->ibdev->phys_port_cnt; 8240a99be43SKarsten Graul pr_warn_ratelimited("smc: adding ib device %s with port count %d\n", 8250a99be43SKarsten Graul smcibdev->ibdev->name, port_cnt); 826be6a3f38SUrsula Braun for (i = 0; 827be6a3f38SUrsula Braun i < min_t(size_t, port_cnt, SMC_MAX_PORTS); 8280afff91cSUrsula Braun i++) { 829be6a3f38SUrsula Braun set_bit(i, &smcibdev->port_event_mask); 8300afff91cSUrsula Braun /* determine pnetids of the port */ 831fdff704dSKarsten Graul if (smc_pnetid_by_dev_port(ibdev->dev.parent, i, 832fdff704dSKarsten Graul smcibdev->pnetid[i])) 833fdff704dSKarsten Graul smc_pnetid_by_table_ib(smcibdev, i + 1); 8343d453f53SGuvenc Gulce smc_copy_netdev_ifindex(smcibdev, i); 8350a99be43SKarsten Graul pr_warn_ratelimited("smc: ib device %s port %d has pnetid " 8360a99be43SKarsten Graul "%.16s%s\n", 8370a99be43SKarsten Graul smcibdev->ibdev->name, i + 1, 8380a99be43SKarsten Graul smcibdev->pnetid[i], 8390a99be43SKarsten Graul smcibdev->pnetid_by_user[i] ? 8400a99be43SKarsten Graul " (user defined)" : 8410a99be43SKarsten Graul ""); 8420afff91cSUrsula Braun } 843be6a3f38SUrsula Braun schedule_work(&smcibdev->port_event_work); 84411a0ae4cSJason Gunthorpe return 0; 845a4cf0443SUrsula Braun } 846a4cf0443SUrsula Braun 8470b29ec64SUrsula Braun /* callback function for ib_unregister_client() */ 848a4cf0443SUrsula Braun static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data) 849a4cf0443SUrsula Braun { 8501587982eSJason Gunthorpe struct smc_ib_device *smcibdev = client_data; 851a4cf0443SUrsula Braun 85292f3cb0eSUrsula Braun mutex_lock(&smc_ib_devices.mutex); 853a4cf0443SUrsula Braun list_del_init(&smcibdev->list); /* remove from smc_ib_devices */ 85492f3cb0eSUrsula Braun mutex_unlock(&smc_ib_devices.mutex); 8550a99be43SKarsten Graul pr_warn_ratelimited("smc: removing ib device %s\n", 8560a99be43SKarsten Graul smcibdev->ibdev->name); 8570b29ec64SUrsula Braun smc_smcr_terminate_all(smcibdev); 858bd4ad577SUrsula Braun smc_ib_cleanup_per_ibdev(smcibdev); 859be6a3f38SUrsula Braun ib_unregister_event_handler(&smcibdev->event_handler); 860ece0d7bdSKarsten Graul cancel_work_sync(&smcibdev->port_event_work); 861a4cf0443SUrsula Braun kfree(smcibdev); 862a4cf0443SUrsula Braun } 863a4cf0443SUrsula Braun 864a4cf0443SUrsula Braun static struct ib_client smc_ib_client = { 865a4cf0443SUrsula Braun .name = "smc_ib", 866a4cf0443SUrsula Braun .add = smc_ib_add_dev, 867a4cf0443SUrsula Braun .remove = smc_ib_remove_dev, 868a4cf0443SUrsula Braun }; 869a4cf0443SUrsula Braun 870a4cf0443SUrsula Braun int __init smc_ib_register_client(void) 871a4cf0443SUrsula Braun { 872366bb249SHans Wippel smc_ib_init_local_systemid(); 873a4cf0443SUrsula Braun return ib_register_client(&smc_ib_client); 874a4cf0443SUrsula Braun } 875a4cf0443SUrsula Braun 876a4cf0443SUrsula Braun void smc_ib_unregister_client(void) 877a4cf0443SUrsula Braun { 878a4cf0443SUrsula Braun ib_unregister_client(&smc_ib_client); 879a4cf0443SUrsula Braun } 880