1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2a4cf0443SUrsula Braun /* 3a4cf0443SUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 4a4cf0443SUrsula Braun * 5a4cf0443SUrsula Braun * IB infrastructure: 6a4cf0443SUrsula Braun * Establish SMC-R as an Infiniband Client to be notified about added and 7a4cf0443SUrsula Braun * removed IB devices of type RDMA. 8a4cf0443SUrsula Braun * Determine device and port characteristics for these IB devices. 9a4cf0443SUrsula Braun * 10a4cf0443SUrsula Braun * Copyright IBM Corp. 2016 11a4cf0443SUrsula Braun * 12a4cf0443SUrsula Braun * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 13a4cf0443SUrsula Braun */ 14a4cf0443SUrsula Braun 15a4cf0443SUrsula Braun #include <linux/random.h> 16bd4ad577SUrsula Braun #include <linux/workqueue.h> 1710428dd8SUrsula Braun #include <linux/scatterlist.h> 186dabd405SUrsula Braun #include <linux/wait.h> 1992f3cb0eSUrsula Braun #include <linux/mutex.h> 20a4cf0443SUrsula Braun #include <rdma/ib_verbs.h> 21ddb457c6SParav Pandit #include <rdma/ib_cache.h> 22a4cf0443SUrsula Braun 236812baabSThomas Richter #include "smc_pnet.h" 24a4cf0443SUrsula Braun #include "smc_ib.h" 25cd6851f3SUrsula Braun #include "smc_core.h" 26f38ba179SUrsula Braun #include "smc_wr.h" 27a4cf0443SUrsula Braun #include "smc.h" 28a4cf0443SUrsula Braun 29c9f4c6cfSUrsula Braun #define SMC_MAX_CQE 32766 /* max. # of completion queue elements */ 30c9f4c6cfSUrsula Braun 31bd4ad577SUrsula Braun #define SMC_QP_MIN_RNR_TIMER 5 32bd4ad577SUrsula Braun #define SMC_QP_TIMEOUT 15 /* 4096 * 2 ** timeout usec */ 33bd4ad577SUrsula Braun #define SMC_QP_RETRY_CNT 7 /* 7: infinite */ 34bd4ad577SUrsula Braun #define SMC_QP_RNR_RETRY 7 /* 7: infinite */ 35bd4ad577SUrsula Braun 36a4cf0443SUrsula Braun struct smc_ib_devices smc_ib_devices = { /* smc-registered ib devices */ 3792f3cb0eSUrsula Braun .mutex = __MUTEX_INITIALIZER(smc_ib_devices.mutex), 38a4cf0443SUrsula Braun .list = LIST_HEAD_INIT(smc_ib_devices.list), 39a4cf0443SUrsula Braun }; 40a4cf0443SUrsula Braun 41366bb249SHans Wippel u8 local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */ 42a4cf0443SUrsula Braun 43bd4ad577SUrsula Braun static int smc_ib_modify_qp_init(struct smc_link *lnk) 44bd4ad577SUrsula Braun { 45bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 46bd4ad577SUrsula Braun 47bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 48bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_INIT; 49bd4ad577SUrsula Braun qp_attr.pkey_index = 0; 50bd4ad577SUrsula Braun qp_attr.port_num = lnk->ibport; 51bd4ad577SUrsula Braun qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE 52bd4ad577SUrsula Braun | IB_ACCESS_REMOTE_WRITE; 53bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, 54bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_PKEY_INDEX | 55bd4ad577SUrsula Braun IB_QP_ACCESS_FLAGS | IB_QP_PORT); 56bd4ad577SUrsula Braun } 57bd4ad577SUrsula Braun 58bd4ad577SUrsula Braun static int smc_ib_modify_qp_rtr(struct smc_link *lnk) 59bd4ad577SUrsula Braun { 60bd4ad577SUrsula Braun enum ib_qp_attr_mask qp_attr_mask = 61bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | 62bd4ad577SUrsula Braun IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; 63bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 64bd4ad577SUrsula Braun 65bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 66bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RTR; 67bd4ad577SUrsula Braun qp_attr.path_mtu = min(lnk->path_mtu, lnk->peer_mtu); 6844c58487SDasaratharaman Chandramouli qp_attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 69d8966fcdSDasaratharaman Chandramouli rdma_ah_set_port_num(&qp_attr.ah_attr, lnk->ibport); 707005ada6SUrsula Braun rdma_ah_set_grh(&qp_attr.ah_attr, NULL, 0, lnk->sgid_index, 1, 0); 71d8966fcdSDasaratharaman Chandramouli rdma_ah_set_dgid_raw(&qp_attr.ah_attr, lnk->peer_gid); 7244c58487SDasaratharaman Chandramouli memcpy(&qp_attr.ah_attr.roce.dmac, lnk->peer_mac, 73bd4ad577SUrsula Braun sizeof(lnk->peer_mac)); 74bd4ad577SUrsula Braun qp_attr.dest_qp_num = lnk->peer_qpn; 75bd4ad577SUrsula Braun qp_attr.rq_psn = lnk->peer_psn; /* starting receive packet seq # */ 76bd4ad577SUrsula Braun qp_attr.max_dest_rd_atomic = 1; /* max # of resources for incoming 77bd4ad577SUrsula Braun * requests 78bd4ad577SUrsula Braun */ 79bd4ad577SUrsula Braun qp_attr.min_rnr_timer = SMC_QP_MIN_RNR_TIMER; 80bd4ad577SUrsula Braun 81bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, qp_attr_mask); 82bd4ad577SUrsula Braun } 83bd4ad577SUrsula Braun 84bd4ad577SUrsula Braun int smc_ib_modify_qp_rts(struct smc_link *lnk) 85bd4ad577SUrsula Braun { 86bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 87bd4ad577SUrsula Braun 88bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 89bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RTS; 90bd4ad577SUrsula Braun qp_attr.timeout = SMC_QP_TIMEOUT; /* local ack timeout */ 91bd4ad577SUrsula Braun qp_attr.retry_cnt = SMC_QP_RETRY_CNT; /* retry count */ 92bd4ad577SUrsula Braun qp_attr.rnr_retry = SMC_QP_RNR_RETRY; /* RNR retries, 7=infinite */ 93bd4ad577SUrsula Braun qp_attr.sq_psn = lnk->psn_initial; /* starting send packet seq # */ 94bd4ad577SUrsula Braun qp_attr.max_rd_atomic = 1; /* # of outstanding RDMA reads and 95bd4ad577SUrsula Braun * atomic ops allowed 96bd4ad577SUrsula Braun */ 97bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, 98bd4ad577SUrsula Braun IB_QP_STATE | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | 99bd4ad577SUrsula Braun IB_QP_SQ_PSN | IB_QP_RNR_RETRY | 100bd4ad577SUrsula Braun IB_QP_MAX_QP_RD_ATOMIC); 101bd4ad577SUrsula Braun } 102bd4ad577SUrsula Braun 103bd4ad577SUrsula Braun int smc_ib_modify_qp_reset(struct smc_link *lnk) 104bd4ad577SUrsula Braun { 105bd4ad577SUrsula Braun struct ib_qp_attr qp_attr; 106bd4ad577SUrsula Braun 107bd4ad577SUrsula Braun memset(&qp_attr, 0, sizeof(qp_attr)); 108bd4ad577SUrsula Braun qp_attr.qp_state = IB_QPS_RESET; 109bd4ad577SUrsula Braun return ib_modify_qp(lnk->roce_qp, &qp_attr, IB_QP_STATE); 110bd4ad577SUrsula Braun } 111bd4ad577SUrsula Braun 112bd4ad577SUrsula Braun int smc_ib_ready_link(struct smc_link *lnk) 113bd4ad577SUrsula Braun { 11400e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(lnk); 115bd4ad577SUrsula Braun int rc = 0; 116bd4ad577SUrsula Braun 117bd4ad577SUrsula Braun rc = smc_ib_modify_qp_init(lnk); 118bd4ad577SUrsula Braun if (rc) 119bd4ad577SUrsula Braun goto out; 120bd4ad577SUrsula Braun 121bd4ad577SUrsula Braun rc = smc_ib_modify_qp_rtr(lnk); 122bd4ad577SUrsula Braun if (rc) 123bd4ad577SUrsula Braun goto out; 124bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 125bd4ad577SUrsula Braun rc = ib_req_notify_cq(lnk->smcibdev->roce_cq_recv, 126bd4ad577SUrsula Braun IB_CQ_SOLICITED_MASK); 127bd4ad577SUrsula Braun if (rc) 128bd4ad577SUrsula Braun goto out; 129bd4ad577SUrsula Braun rc = smc_wr_rx_post_init(lnk); 130bd4ad577SUrsula Braun if (rc) 131bd4ad577SUrsula Braun goto out; 132bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 133bd4ad577SUrsula Braun 134bd4ad577SUrsula Braun if (lgr->role == SMC_SERV) { 135bd4ad577SUrsula Braun rc = smc_ib_modify_qp_rts(lnk); 136bd4ad577SUrsula Braun if (rc) 137bd4ad577SUrsula Braun goto out; 138bd4ad577SUrsula Braun smc_wr_remember_qp_attr(lnk); 139bd4ad577SUrsula Braun } 140bd4ad577SUrsula Braun out: 141bd4ad577SUrsula Braun return rc; 142bd4ad577SUrsula Braun } 143bd4ad577SUrsula Braun 1447005ada6SUrsula Braun static int smc_ib_fill_mac(struct smc_ib_device *smcibdev, u8 ibport) 145be6a3f38SUrsula Braun { 146b4c296f9SJason Gunthorpe const struct ib_gid_attr *attr; 1475102eca9SParav Pandit int rc; 148be6a3f38SUrsula Braun 149b4c296f9SJason Gunthorpe attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, 0); 150b4c296f9SJason Gunthorpe if (IS_ERR(attr)) 151be6a3f38SUrsula Braun return -ENODEV; 152be6a3f38SUrsula Braun 1535102eca9SParav Pandit rc = rdma_read_gid_l2_fields(attr, NULL, smcibdev->mac[ibport - 1]); 154b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 155b4c296f9SJason Gunthorpe return rc; 156be6a3f38SUrsula Braun } 157be6a3f38SUrsula Braun 158be6a3f38SUrsula Braun /* Create an identifier unique for this instance of SMC-R. 159be6a3f38SUrsula Braun * The MAC-address of the first active registered IB device 160be6a3f38SUrsula Braun * plus a random 2-byte number is used to create this identifier. 161be6a3f38SUrsula Braun * This name is delivered to the peer during connection initialization. 162be6a3f38SUrsula Braun */ 163be6a3f38SUrsula Braun static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev, 164be6a3f38SUrsula Braun u8 ibport) 165be6a3f38SUrsula Braun { 166be6a3f38SUrsula Braun memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1], 167be6a3f38SUrsula Braun sizeof(smcibdev->mac[ibport - 1])); 168366bb249SHans Wippel } 169366bb249SHans Wippel 170a082ec89SHans Wippel bool smc_ib_is_valid_local_systemid(void) 171366bb249SHans Wippel { 172366bb249SHans Wippel return !is_zero_ether_addr(&local_systemid[2]); 173366bb249SHans Wippel } 174366bb249SHans Wippel 175366bb249SHans Wippel static void smc_ib_init_local_systemid(void) 176366bb249SHans Wippel { 177be6a3f38SUrsula Braun get_random_bytes(&local_systemid[0], 2); 178be6a3f38SUrsula Braun } 179be6a3f38SUrsula Braun 180be6a3f38SUrsula Braun bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport) 181be6a3f38SUrsula Braun { 182be6a3f38SUrsula Braun return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE; 183be6a3f38SUrsula Braun } 184be6a3f38SUrsula Braun 1857005ada6SUrsula Braun /* determine the gid for an ib-device port and vlan id */ 1867005ada6SUrsula Braun int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport, 1877005ada6SUrsula Braun unsigned short vlan_id, u8 gid[], u8 *sgid_index) 1887005ada6SUrsula Braun { 189b4c296f9SJason Gunthorpe const struct ib_gid_attr *attr; 1905102eca9SParav Pandit const struct net_device *ndev; 1917005ada6SUrsula Braun int i; 1927005ada6SUrsula Braun 1937005ada6SUrsula Braun for (i = 0; i < smcibdev->pattr[ibport - 1].gid_tbl_len; i++) { 194b4c296f9SJason Gunthorpe attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, i); 195b4c296f9SJason Gunthorpe if (IS_ERR(attr)) 1967005ada6SUrsula Braun continue; 197b4c296f9SJason Gunthorpe 1985102eca9SParav Pandit rcu_read_lock(); 1995102eca9SParav Pandit ndev = rdma_read_gid_attr_ndev_rcu(attr); 2005102eca9SParav Pandit if (!IS_ERR(ndev) && 20141a0be3fSKarsten Graul ((!vlan_id && !is_vlan_dev(ndev)) || 20241a0be3fSKarsten Graul (vlan_id && is_vlan_dev(ndev) && 20341a0be3fSKarsten Graul vlan_dev_vlan_id(ndev) == vlan_id)) && 204b4c296f9SJason Gunthorpe attr->gid_type == IB_GID_TYPE_ROCE) { 2055102eca9SParav Pandit rcu_read_unlock(); 2067005ada6SUrsula Braun if (gid) 207b4c296f9SJason Gunthorpe memcpy(gid, &attr->gid, SMC_GID_SIZE); 2087005ada6SUrsula Braun if (sgid_index) 209b4c296f9SJason Gunthorpe *sgid_index = attr->index; 210b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 2117005ada6SUrsula Braun return 0; 2127005ada6SUrsula Braun } 2135102eca9SParav Pandit rcu_read_unlock(); 214b4c296f9SJason Gunthorpe rdma_put_gid_attr(attr); 2157005ada6SUrsula Braun } 2167005ada6SUrsula Braun return -ENODEV; 2177005ada6SUrsula Braun } 2187005ada6SUrsula Braun 219be6a3f38SUrsula Braun static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport) 220be6a3f38SUrsula Braun { 221be6a3f38SUrsula Braun int rc; 222be6a3f38SUrsula Braun 223be6a3f38SUrsula Braun memset(&smcibdev->pattr[ibport - 1], 0, 224be6a3f38SUrsula Braun sizeof(smcibdev->pattr[ibport - 1])); 225be6a3f38SUrsula Braun rc = ib_query_port(smcibdev->ibdev, ibport, 226be6a3f38SUrsula Braun &smcibdev->pattr[ibport - 1]); 227be6a3f38SUrsula Braun if (rc) 228be6a3f38SUrsula Braun goto out; 229be6a3f38SUrsula Braun /* the SMC protocol requires specification of the RoCE MAC address */ 2307005ada6SUrsula Braun rc = smc_ib_fill_mac(smcibdev, ibport); 231be6a3f38SUrsula Braun if (rc) 232be6a3f38SUrsula Braun goto out; 233366bb249SHans Wippel if (!smc_ib_is_valid_local_systemid() && 234be6a3f38SUrsula Braun smc_ib_port_active(smcibdev, ibport)) 235be6a3f38SUrsula Braun /* create unique system identifier */ 236be6a3f38SUrsula Braun smc_ib_define_local_systemid(smcibdev, ibport); 237be6a3f38SUrsula Braun out: 238be6a3f38SUrsula Braun return rc; 239be6a3f38SUrsula Braun } 240be6a3f38SUrsula Braun 241bd4ad577SUrsula Braun /* process context wrapper for might_sleep smc_ib_remember_port_attr */ 242bd4ad577SUrsula Braun static void smc_ib_port_event_work(struct work_struct *work) 243bd4ad577SUrsula Braun { 244bd4ad577SUrsula Braun struct smc_ib_device *smcibdev = container_of( 245bd4ad577SUrsula Braun work, struct smc_ib_device, port_event_work); 246bd4ad577SUrsula Braun u8 port_idx; 247bd4ad577SUrsula Braun 248bd4ad577SUrsula Braun for_each_set_bit(port_idx, &smcibdev->port_event_mask, SMC_MAX_PORTS) { 249bd4ad577SUrsula Braun smc_ib_remember_port_attr(smcibdev, port_idx + 1); 250bd4ad577SUrsula Braun clear_bit(port_idx, &smcibdev->port_event_mask); 251c3d9494eSUrsula Braun if (!smc_ib_port_active(smcibdev, port_idx + 1)) { 252c3d9494eSUrsula Braun set_bit(port_idx, smcibdev->ports_going_away); 253541afa10SKarsten Graul smcr_port_err(smcibdev, port_idx + 1); 254c3d9494eSUrsula Braun } else { 255c3d9494eSUrsula Braun clear_bit(port_idx, smcibdev->ports_going_away); 2561f90a05dSKarsten Graul smcr_port_add(smcibdev, port_idx + 1); 257c3d9494eSUrsula Braun } 258bd4ad577SUrsula Braun } 259bd4ad577SUrsula Braun } 260bd4ad577SUrsula Braun 261bd4ad577SUrsula Braun /* can be called in IRQ context */ 262bd4ad577SUrsula Braun static void smc_ib_global_event_handler(struct ib_event_handler *handler, 263bd4ad577SUrsula Braun struct ib_event *ibevent) 264bd4ad577SUrsula Braun { 265bd4ad577SUrsula Braun struct smc_ib_device *smcibdev; 2665613f20cSUrsula Braun bool schedule = false; 267bd4ad577SUrsula Braun u8 port_idx; 268bd4ad577SUrsula Braun 269bd4ad577SUrsula Braun smcibdev = container_of(handler, struct smc_ib_device, event_handler); 270bd4ad577SUrsula Braun 271bd4ad577SUrsula Braun switch (ibevent->event) { 272bd4ad577SUrsula Braun case IB_EVENT_DEVICE_FATAL: 27381cf6430SKarsten Graul /* terminate all ports on device */ 274c3d9494eSUrsula Braun for (port_idx = 0; port_idx < SMC_MAX_PORTS; port_idx++) { 275bd4ad577SUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 2765613f20cSUrsula Braun if (!test_and_set_bit(port_idx, 2775613f20cSUrsula Braun smcibdev->ports_going_away)) 2785613f20cSUrsula Braun schedule = true; 279c3d9494eSUrsula Braun } 2805613f20cSUrsula Braun if (schedule) 2815613f20cSUrsula Braun schedule_work(&smcibdev->port_event_work); 2825613f20cSUrsula Braun break; 2835613f20cSUrsula Braun case IB_EVENT_PORT_ACTIVE: 2845613f20cSUrsula Braun port_idx = ibevent->element.port_num - 1; 2855613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 2865613f20cSUrsula Braun break; 2875613f20cSUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 2885613f20cSUrsula Braun if (test_and_clear_bit(port_idx, smcibdev->ports_going_away)) 289bd4ad577SUrsula Braun schedule_work(&smcibdev->port_event_work); 290bd4ad577SUrsula Braun break; 29181cf6430SKarsten Graul case IB_EVENT_PORT_ERR: 2925613f20cSUrsula Braun port_idx = ibevent->element.port_num - 1; 2935613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 2945613f20cSUrsula Braun break; 2955613f20cSUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 2965613f20cSUrsula Braun if (!test_and_set_bit(port_idx, smcibdev->ports_going_away)) 2975613f20cSUrsula Braun schedule_work(&smcibdev->port_event_work); 2985613f20cSUrsula Braun break; 29981cf6430SKarsten Graul case IB_EVENT_GID_CHANGE: 30081cf6430SKarsten Graul port_idx = ibevent->element.port_num - 1; 3015613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3025613f20cSUrsula Braun break; 30381cf6430SKarsten Graul set_bit(port_idx, &smcibdev->port_event_mask); 30481cf6430SKarsten Graul schedule_work(&smcibdev->port_event_work); 30581cf6430SKarsten Graul break; 306bd4ad577SUrsula Braun default: 307bd4ad577SUrsula Braun break; 308bd4ad577SUrsula Braun } 309bd4ad577SUrsula Braun } 310bd4ad577SUrsula Braun 311f38ba179SUrsula Braun void smc_ib_dealloc_protection_domain(struct smc_link *lnk) 312f38ba179SUrsula Braun { 313da05bf29SUrsula Braun if (lnk->roce_pd) 314f38ba179SUrsula Braun ib_dealloc_pd(lnk->roce_pd); 315f38ba179SUrsula Braun lnk->roce_pd = NULL; 316f38ba179SUrsula Braun } 317f38ba179SUrsula Braun 318f38ba179SUrsula Braun int smc_ib_create_protection_domain(struct smc_link *lnk) 319f38ba179SUrsula Braun { 320f38ba179SUrsula Braun int rc; 321f38ba179SUrsula Braun 322897e1c24SUrsula Braun lnk->roce_pd = ib_alloc_pd(lnk->smcibdev->ibdev, 0); 323f38ba179SUrsula Braun rc = PTR_ERR_OR_ZERO(lnk->roce_pd); 324f38ba179SUrsula Braun if (IS_ERR(lnk->roce_pd)) 325f38ba179SUrsula Braun lnk->roce_pd = NULL; 326f38ba179SUrsula Braun return rc; 327f38ba179SUrsula Braun } 328f38ba179SUrsula Braun 329f38ba179SUrsula Braun static void smc_ib_qp_event_handler(struct ib_event *ibevent, void *priv) 330f38ba179SUrsula Braun { 331e5f3aa04SKarsten Graul struct smc_link *lnk = (struct smc_link *)priv; 332e5f3aa04SKarsten Graul struct smc_ib_device *smcibdev = lnk->smcibdev; 333da05bf29SUrsula Braun u8 port_idx; 334da05bf29SUrsula Braun 335f38ba179SUrsula Braun switch (ibevent->event) { 33681cf6430SKarsten Graul case IB_EVENT_QP_FATAL: 337f38ba179SUrsula Braun case IB_EVENT_QP_ACCESS_ERR: 338e5f3aa04SKarsten Graul port_idx = ibevent->element.qp->port - 1; 3395613f20cSUrsula Braun if (port_idx >= SMC_MAX_PORTS) 3405613f20cSUrsula Braun break; 341da05bf29SUrsula Braun set_bit(port_idx, &smcibdev->port_event_mask); 3425613f20cSUrsula Braun if (!test_and_set_bit(port_idx, smcibdev->ports_going_away)) 343da05bf29SUrsula Braun schedule_work(&smcibdev->port_event_work); 344f38ba179SUrsula Braun break; 345f38ba179SUrsula Braun default: 346f38ba179SUrsula Braun break; 347f38ba179SUrsula Braun } 348f38ba179SUrsula Braun } 349f38ba179SUrsula Braun 350f38ba179SUrsula Braun void smc_ib_destroy_queue_pair(struct smc_link *lnk) 351f38ba179SUrsula Braun { 352da05bf29SUrsula Braun if (lnk->roce_qp) 353f38ba179SUrsula Braun ib_destroy_qp(lnk->roce_qp); 354f38ba179SUrsula Braun lnk->roce_qp = NULL; 355f38ba179SUrsula Braun } 356f38ba179SUrsula Braun 357f38ba179SUrsula Braun /* create a queue pair within the protection domain for a link */ 358f38ba179SUrsula Braun int smc_ib_create_queue_pair(struct smc_link *lnk) 359f38ba179SUrsula Braun { 360f38ba179SUrsula Braun struct ib_qp_init_attr qp_attr = { 361f38ba179SUrsula Braun .event_handler = smc_ib_qp_event_handler, 362f38ba179SUrsula Braun .qp_context = lnk, 363f38ba179SUrsula Braun .send_cq = lnk->smcibdev->roce_cq_send, 364f38ba179SUrsula Braun .recv_cq = lnk->smcibdev->roce_cq_recv, 365f38ba179SUrsula Braun .srq = NULL, 366f38ba179SUrsula Braun .cap = { 367f38ba179SUrsula Braun /* include unsolicited rdma_writes as well, 368f38ba179SUrsula Braun * there are max. 2 RDMA_WRITE per 1 WR_SEND 369f38ba179SUrsula Braun */ 370652a1e41SUrsula Braun .max_send_wr = SMC_WR_BUF_CNT * 3, 371f38ba179SUrsula Braun .max_recv_wr = SMC_WR_BUF_CNT * 3, 372f38ba179SUrsula Braun .max_send_sge = SMC_IB_MAX_SEND_SGE, 373f38ba179SUrsula Braun .max_recv_sge = 1, 374f38ba179SUrsula Braun }, 375f38ba179SUrsula Braun .sq_sig_type = IB_SIGNAL_REQ_WR, 376f38ba179SUrsula Braun .qp_type = IB_QPT_RC, 377f38ba179SUrsula Braun }; 378f38ba179SUrsula Braun int rc; 379f38ba179SUrsula Braun 380f38ba179SUrsula Braun lnk->roce_qp = ib_create_qp(lnk->roce_pd, &qp_attr); 381f38ba179SUrsula Braun rc = PTR_ERR_OR_ZERO(lnk->roce_qp); 382f38ba179SUrsula Braun if (IS_ERR(lnk->roce_qp)) 383f38ba179SUrsula Braun lnk->roce_qp = NULL; 384f38ba179SUrsula Braun else 385f38ba179SUrsula Braun smc_wr_remember_qp_attr(lnk); 386f38ba179SUrsula Braun return rc; 387f38ba179SUrsula Braun } 388f38ba179SUrsula Braun 389897e1c24SUrsula Braun void smc_ib_put_memory_region(struct ib_mr *mr) 390897e1c24SUrsula Braun { 391897e1c24SUrsula Braun ib_dereg_mr(mr); 392897e1c24SUrsula Braun } 393897e1c24SUrsula Braun 394387707fdSKarsten Graul static int smc_ib_map_mr_sg(struct smc_buf_desc *buf_slot, u8 link_idx) 395897e1c24SUrsula Braun { 396897e1c24SUrsula Braun unsigned int offset = 0; 397897e1c24SUrsula Braun int sg_num; 398897e1c24SUrsula Braun 399897e1c24SUrsula Braun /* map the largest prefix of a dma mapped SG list */ 400387707fdSKarsten Graul sg_num = ib_map_mr_sg(buf_slot->mr_rx[link_idx], 401387707fdSKarsten Graul buf_slot->sgt[link_idx].sgl, 402387707fdSKarsten Graul buf_slot->sgt[link_idx].orig_nents, 403897e1c24SUrsula Braun &offset, PAGE_SIZE); 404897e1c24SUrsula Braun 405897e1c24SUrsula Braun return sg_num; 406897e1c24SUrsula Braun } 407897e1c24SUrsula Braun 408897e1c24SUrsula Braun /* Allocate a memory region and map the dma mapped SG list of buf_slot */ 409897e1c24SUrsula Braun int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags, 410387707fdSKarsten Graul struct smc_buf_desc *buf_slot, u8 link_idx) 411897e1c24SUrsula Braun { 412387707fdSKarsten Graul if (buf_slot->mr_rx[link_idx]) 413897e1c24SUrsula Braun return 0; /* already done */ 414897e1c24SUrsula Braun 415387707fdSKarsten Graul buf_slot->mr_rx[link_idx] = 416897e1c24SUrsula Braun ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 1 << buf_slot->order); 417387707fdSKarsten Graul if (IS_ERR(buf_slot->mr_rx[link_idx])) { 418897e1c24SUrsula Braun int rc; 419897e1c24SUrsula Braun 420387707fdSKarsten Graul rc = PTR_ERR(buf_slot->mr_rx[link_idx]); 421387707fdSKarsten Graul buf_slot->mr_rx[link_idx] = NULL; 422897e1c24SUrsula Braun return rc; 423897e1c24SUrsula Braun } 424897e1c24SUrsula Braun 425387707fdSKarsten Graul if (smc_ib_map_mr_sg(buf_slot, link_idx) != 1) 426897e1c24SUrsula Braun return -EINVAL; 427897e1c24SUrsula Braun 428897e1c24SUrsula Braun return 0; 429897e1c24SUrsula Braun } 430897e1c24SUrsula Braun 43110428dd8SUrsula Braun /* synchronize buffer usage for cpu access */ 432387707fdSKarsten Graul void smc_ib_sync_sg_for_cpu(struct smc_link *lnk, 43310428dd8SUrsula Braun struct smc_buf_desc *buf_slot, 43410428dd8SUrsula Braun enum dma_data_direction data_direction) 43510428dd8SUrsula Braun { 43610428dd8SUrsula Braun struct scatterlist *sg; 43710428dd8SUrsula Braun unsigned int i; 43810428dd8SUrsula Braun 43910428dd8SUrsula Braun /* for now there is just one DMA address */ 440387707fdSKarsten Graul for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, 441387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].nents, i) { 44210428dd8SUrsula Braun if (!sg_dma_len(sg)) 44310428dd8SUrsula Braun break; 444387707fdSKarsten Graul ib_dma_sync_single_for_cpu(lnk->smcibdev->ibdev, 44510428dd8SUrsula Braun sg_dma_address(sg), 44610428dd8SUrsula Braun sg_dma_len(sg), 44710428dd8SUrsula Braun data_direction); 44810428dd8SUrsula Braun } 44910428dd8SUrsula Braun } 45010428dd8SUrsula Braun 45110428dd8SUrsula Braun /* synchronize buffer usage for device access */ 452387707fdSKarsten Graul void smc_ib_sync_sg_for_device(struct smc_link *lnk, 45310428dd8SUrsula Braun struct smc_buf_desc *buf_slot, 45410428dd8SUrsula Braun enum dma_data_direction data_direction) 45510428dd8SUrsula Braun { 45610428dd8SUrsula Braun struct scatterlist *sg; 45710428dd8SUrsula Braun unsigned int i; 45810428dd8SUrsula Braun 45910428dd8SUrsula Braun /* for now there is just one DMA address */ 460387707fdSKarsten Graul for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, 461387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].nents, i) { 46210428dd8SUrsula Braun if (!sg_dma_len(sg)) 46310428dd8SUrsula Braun break; 464387707fdSKarsten Graul ib_dma_sync_single_for_device(lnk->smcibdev->ibdev, 46510428dd8SUrsula Braun sg_dma_address(sg), 46610428dd8SUrsula Braun sg_dma_len(sg), 46710428dd8SUrsula Braun data_direction); 46810428dd8SUrsula Braun } 46910428dd8SUrsula Braun } 47010428dd8SUrsula Braun 471a3fe3d01SUrsula Braun /* Map a new TX or RX buffer SG-table to DMA */ 472387707fdSKarsten Graul int smc_ib_buf_map_sg(struct smc_link *lnk, 473a3fe3d01SUrsula Braun struct smc_buf_desc *buf_slot, 474a3fe3d01SUrsula Braun enum dma_data_direction data_direction) 475a3fe3d01SUrsula Braun { 476a3fe3d01SUrsula Braun int mapped_nents; 477a3fe3d01SUrsula Braun 478387707fdSKarsten Graul mapped_nents = ib_dma_map_sg(lnk->smcibdev->ibdev, 479387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl, 480387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].orig_nents, 481a3fe3d01SUrsula Braun data_direction); 482a3fe3d01SUrsula Braun if (!mapped_nents) 483a3fe3d01SUrsula Braun return -ENOMEM; 484a3fe3d01SUrsula Braun 485a3fe3d01SUrsula Braun return mapped_nents; 486a3fe3d01SUrsula Braun } 487a3fe3d01SUrsula Braun 488387707fdSKarsten Graul void smc_ib_buf_unmap_sg(struct smc_link *lnk, 489a3fe3d01SUrsula Braun struct smc_buf_desc *buf_slot, 490a3fe3d01SUrsula Braun enum dma_data_direction data_direction) 491a3fe3d01SUrsula Braun { 492387707fdSKarsten Graul if (!buf_slot->sgt[lnk->link_idx].sgl->dma_address) 493a3fe3d01SUrsula Braun return; /* already unmapped */ 494a3fe3d01SUrsula Braun 495387707fdSKarsten Graul ib_dma_unmap_sg(lnk->smcibdev->ibdev, 496387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl, 497387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].orig_nents, 498a3fe3d01SUrsula Braun data_direction); 499387707fdSKarsten Graul buf_slot->sgt[lnk->link_idx].sgl->dma_address = 0; 500a3fe3d01SUrsula Braun } 501a3fe3d01SUrsula Braun 502bd4ad577SUrsula Braun long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev) 503bd4ad577SUrsula Braun { 504bd4ad577SUrsula Braun struct ib_cq_init_attr cqattr = { 505c9f4c6cfSUrsula Braun .cqe = SMC_MAX_CQE, .comp_vector = 0 }; 506c9f4c6cfSUrsula Braun int cqe_size_order, smc_order; 507bd4ad577SUrsula Braun long rc; 508bd4ad577SUrsula Braun 50963673597SKarsten Graul mutex_lock(&smcibdev->mutex); 51063673597SKarsten Graul rc = 0; 51163673597SKarsten Graul if (smcibdev->initialized) 51263673597SKarsten Graul goto out; 513c9f4c6cfSUrsula Braun /* the calculated number of cq entries fits to mlx5 cq allocation */ 514c9f4c6cfSUrsula Braun cqe_size_order = cache_line_size() == 128 ? 7 : 6; 515c9f4c6cfSUrsula Braun smc_order = MAX_ORDER - cqe_size_order - 1; 516c9f4c6cfSUrsula Braun if (SMC_MAX_CQE + 2 > (0x00000001 << smc_order) * PAGE_SIZE) 517c9f4c6cfSUrsula Braun cqattr.cqe = (0x00000001 << smc_order) * PAGE_SIZE - 2; 518bd4ad577SUrsula Braun smcibdev->roce_cq_send = ib_create_cq(smcibdev->ibdev, 519bd4ad577SUrsula Braun smc_wr_tx_cq_handler, NULL, 520bd4ad577SUrsula Braun smcibdev, &cqattr); 521bd4ad577SUrsula Braun rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_send); 522bd4ad577SUrsula Braun if (IS_ERR(smcibdev->roce_cq_send)) { 523bd4ad577SUrsula Braun smcibdev->roce_cq_send = NULL; 52463673597SKarsten Graul goto out; 525bd4ad577SUrsula Braun } 526bd4ad577SUrsula Braun smcibdev->roce_cq_recv = ib_create_cq(smcibdev->ibdev, 527bd4ad577SUrsula Braun smc_wr_rx_cq_handler, NULL, 528bd4ad577SUrsula Braun smcibdev, &cqattr); 529bd4ad577SUrsula Braun rc = PTR_ERR_OR_ZERO(smcibdev->roce_cq_recv); 530bd4ad577SUrsula Braun if (IS_ERR(smcibdev->roce_cq_recv)) { 531bd4ad577SUrsula Braun smcibdev->roce_cq_recv = NULL; 532bd4ad577SUrsula Braun goto err; 533bd4ad577SUrsula Braun } 534bd4ad577SUrsula Braun smc_wr_add_dev(smcibdev); 535bd4ad577SUrsula Braun smcibdev->initialized = 1; 53663673597SKarsten Graul goto out; 537bd4ad577SUrsula Braun 538bd4ad577SUrsula Braun err: 539bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_send); 54063673597SKarsten Graul out: 54163673597SKarsten Graul mutex_unlock(&smcibdev->mutex); 542bd4ad577SUrsula Braun return rc; 543bd4ad577SUrsula Braun } 544bd4ad577SUrsula Braun 545bd4ad577SUrsula Braun static void smc_ib_cleanup_per_ibdev(struct smc_ib_device *smcibdev) 546bd4ad577SUrsula Braun { 54763673597SKarsten Graul mutex_lock(&smcibdev->mutex); 548bd4ad577SUrsula Braun if (!smcibdev->initialized) 54963673597SKarsten Graul goto out; 550da05bf29SUrsula Braun smcibdev->initialized = 0; 551bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_recv); 552bd4ad577SUrsula Braun ib_destroy_cq(smcibdev->roce_cq_send); 5536a37ad3dSUrsula Braun smc_wr_remove_dev(smcibdev); 55463673597SKarsten Graul out: 55563673597SKarsten Graul mutex_unlock(&smcibdev->mutex); 556bd4ad577SUrsula Braun } 557bd4ad577SUrsula Braun 558a4cf0443SUrsula Braun static struct ib_client smc_ib_client; 559a4cf0443SUrsula Braun 560a4cf0443SUrsula Braun /* callback function for ib_register_client() */ 56111a0ae4cSJason Gunthorpe static int smc_ib_add_dev(struct ib_device *ibdev) 562a4cf0443SUrsula Braun { 563a4cf0443SUrsula Braun struct smc_ib_device *smcibdev; 564be6a3f38SUrsula Braun u8 port_cnt; 565be6a3f38SUrsula Braun int i; 566a4cf0443SUrsula Braun 567a4cf0443SUrsula Braun if (ibdev->node_type != RDMA_NODE_IB_CA) 56811a0ae4cSJason Gunthorpe return -EOPNOTSUPP; 569a4cf0443SUrsula Braun 570a4cf0443SUrsula Braun smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL); 571a4cf0443SUrsula Braun if (!smcibdev) 57211a0ae4cSJason Gunthorpe return -ENOMEM; 573a4cf0443SUrsula Braun 574a4cf0443SUrsula Braun smcibdev->ibdev = ibdev; 575bd4ad577SUrsula Braun INIT_WORK(&smcibdev->port_event_work, smc_ib_port_event_work); 5766dabd405SUrsula Braun atomic_set(&smcibdev->lnk_cnt, 0); 5776dabd405SUrsula Braun init_waitqueue_head(&smcibdev->lnks_deleted); 57863673597SKarsten Graul mutex_init(&smcibdev->mutex); 57992f3cb0eSUrsula Braun mutex_lock(&smc_ib_devices.mutex); 580a4cf0443SUrsula Braun list_add_tail(&smcibdev->list, &smc_ib_devices.list); 58192f3cb0eSUrsula Braun mutex_unlock(&smc_ib_devices.mutex); 582a4cf0443SUrsula Braun ib_set_client_data(ibdev, &smc_ib_client, smcibdev); 583be6a3f38SUrsula Braun INIT_IB_EVENT_HANDLER(&smcibdev->event_handler, smcibdev->ibdev, 584be6a3f38SUrsula Braun smc_ib_global_event_handler); 585be6a3f38SUrsula Braun ib_register_event_handler(&smcibdev->event_handler); 586be6a3f38SUrsula Braun 587be6a3f38SUrsula Braun /* trigger reading of the port attributes */ 588be6a3f38SUrsula Braun port_cnt = smcibdev->ibdev->phys_port_cnt; 5890a99be43SKarsten Graul pr_warn_ratelimited("smc: adding ib device %s with port count %d\n", 5900a99be43SKarsten Graul smcibdev->ibdev->name, port_cnt); 591be6a3f38SUrsula Braun for (i = 0; 592be6a3f38SUrsula Braun i < min_t(size_t, port_cnt, SMC_MAX_PORTS); 5930afff91cSUrsula Braun i++) { 594be6a3f38SUrsula Braun set_bit(i, &smcibdev->port_event_mask); 5950afff91cSUrsula Braun /* determine pnetids of the port */ 596fdff704dSKarsten Graul if (smc_pnetid_by_dev_port(ibdev->dev.parent, i, 597fdff704dSKarsten Graul smcibdev->pnetid[i])) 598fdff704dSKarsten Graul smc_pnetid_by_table_ib(smcibdev, i + 1); 5990a99be43SKarsten Graul pr_warn_ratelimited("smc: ib device %s port %d has pnetid " 6000a99be43SKarsten Graul "%.16s%s\n", 6010a99be43SKarsten Graul smcibdev->ibdev->name, i + 1, 6020a99be43SKarsten Graul smcibdev->pnetid[i], 6030a99be43SKarsten Graul smcibdev->pnetid_by_user[i] ? 6040a99be43SKarsten Graul " (user defined)" : 6050a99be43SKarsten Graul ""); 6060afff91cSUrsula Braun } 607be6a3f38SUrsula Braun schedule_work(&smcibdev->port_event_work); 60811a0ae4cSJason Gunthorpe return 0; 609a4cf0443SUrsula Braun } 610a4cf0443SUrsula Braun 6110b29ec64SUrsula Braun /* callback function for ib_unregister_client() */ 612a4cf0443SUrsula Braun static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data) 613a4cf0443SUrsula Braun { 6141587982eSJason Gunthorpe struct smc_ib_device *smcibdev = client_data; 615a4cf0443SUrsula Braun 61692f3cb0eSUrsula Braun mutex_lock(&smc_ib_devices.mutex); 617a4cf0443SUrsula Braun list_del_init(&smcibdev->list); /* remove from smc_ib_devices */ 61892f3cb0eSUrsula Braun mutex_unlock(&smc_ib_devices.mutex); 6190a99be43SKarsten Graul pr_warn_ratelimited("smc: removing ib device %s\n", 6200a99be43SKarsten Graul smcibdev->ibdev->name); 6210b29ec64SUrsula Braun smc_smcr_terminate_all(smcibdev); 622bd4ad577SUrsula Braun smc_ib_cleanup_per_ibdev(smcibdev); 623be6a3f38SUrsula Braun ib_unregister_event_handler(&smcibdev->event_handler); 624ece0d7bdSKarsten Graul cancel_work_sync(&smcibdev->port_event_work); 625a4cf0443SUrsula Braun kfree(smcibdev); 626a4cf0443SUrsula Braun } 627a4cf0443SUrsula Braun 628a4cf0443SUrsula Braun static struct ib_client smc_ib_client = { 629a4cf0443SUrsula Braun .name = "smc_ib", 630a4cf0443SUrsula Braun .add = smc_ib_add_dev, 631a4cf0443SUrsula Braun .remove = smc_ib_remove_dev, 632a4cf0443SUrsula Braun }; 633a4cf0443SUrsula Braun 634a4cf0443SUrsula Braun int __init smc_ib_register_client(void) 635a4cf0443SUrsula Braun { 636366bb249SHans Wippel smc_ib_init_local_systemid(); 637a4cf0443SUrsula Braun return ib_register_client(&smc_ib_client); 638a4cf0443SUrsula Braun } 639a4cf0443SUrsula Braun 640a4cf0443SUrsula Braun void smc_ib_unregister_client(void) 641a4cf0443SUrsula Braun { 642a4cf0443SUrsula Braun ib_unregister_client(&smc_ib_client); 643a4cf0443SUrsula Braun } 644