1*e126ba97SEli Cohen /* 2*e126ba97SEli Cohen * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved. 3*e126ba97SEli Cohen * 4*e126ba97SEli Cohen * This software is available to you under a choice of one of two 5*e126ba97SEli Cohen * licenses. You may choose to be licensed under the terms of the GNU 6*e126ba97SEli Cohen * General Public License (GPL) Version 2, available from the file 7*e126ba97SEli Cohen * COPYING in the main directory of this source tree, or the 8*e126ba97SEli Cohen * OpenIB.org BSD license below: 9*e126ba97SEli Cohen * 10*e126ba97SEli Cohen * Redistribution and use in source and binary forms, with or 11*e126ba97SEli Cohen * without modification, are permitted provided that the following 12*e126ba97SEli Cohen * conditions are met: 13*e126ba97SEli Cohen * 14*e126ba97SEli Cohen * - Redistributions of source code must retain the above 15*e126ba97SEli Cohen * copyright notice, this list of conditions and the following 16*e126ba97SEli Cohen * disclaimer. 17*e126ba97SEli Cohen * 18*e126ba97SEli Cohen * - Redistributions in binary form must reproduce the above 19*e126ba97SEli Cohen * copyright notice, this list of conditions and the following 20*e126ba97SEli Cohen * disclaimer in the documentation and/or other materials 21*e126ba97SEli Cohen * provided with the distribution. 22*e126ba97SEli Cohen * 23*e126ba97SEli Cohen * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24*e126ba97SEli Cohen * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25*e126ba97SEli Cohen * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26*e126ba97SEli Cohen * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27*e126ba97SEli Cohen * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28*e126ba97SEli Cohen * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29*e126ba97SEli Cohen * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30*e126ba97SEli Cohen * SOFTWARE. 31*e126ba97SEli Cohen */ 32*e126ba97SEli Cohen 33*e126ba97SEli Cohen #include <asm-generic/kmap_types.h> 34*e126ba97SEli Cohen #include <linux/module.h> 35*e126ba97SEli Cohen #include <linux/init.h> 36*e126ba97SEli Cohen #include <linux/errno.h> 37*e126ba97SEli Cohen #include <linux/pci.h> 38*e126ba97SEli Cohen #include <linux/dma-mapping.h> 39*e126ba97SEli Cohen #include <linux/slab.h> 40*e126ba97SEli Cohen #include <linux/io-mapping.h> 41*e126ba97SEli Cohen #include <linux/sched.h> 42*e126ba97SEli Cohen #include <rdma/ib_user_verbs.h> 43*e126ba97SEli Cohen #include <rdma/ib_smi.h> 44*e126ba97SEli Cohen #include <rdma/ib_umem.h> 45*e126ba97SEli Cohen #include "user.h" 46*e126ba97SEli Cohen #include "mlx5_ib.h" 47*e126ba97SEli Cohen 48*e126ba97SEli Cohen #define DRIVER_NAME "mlx5_ib" 49*e126ba97SEli Cohen #define DRIVER_VERSION "1.0" 50*e126ba97SEli Cohen #define DRIVER_RELDATE "June 2013" 51*e126ba97SEli Cohen 52*e126ba97SEli Cohen MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 53*e126ba97SEli Cohen MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver"); 54*e126ba97SEli Cohen MODULE_LICENSE("Dual BSD/GPL"); 55*e126ba97SEli Cohen MODULE_VERSION(DRIVER_VERSION); 56*e126ba97SEli Cohen 57*e126ba97SEli Cohen static int prof_sel = 2; 58*e126ba97SEli Cohen module_param_named(prof_sel, prof_sel, int, 0444); 59*e126ba97SEli Cohen MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 60*e126ba97SEli Cohen 61*e126ba97SEli Cohen static char mlx5_version[] = 62*e126ba97SEli Cohen DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v" 63*e126ba97SEli Cohen DRIVER_VERSION " (" DRIVER_RELDATE ")\n"; 64*e126ba97SEli Cohen 65*e126ba97SEli Cohen struct mlx5_profile profile[] = { 66*e126ba97SEli Cohen [0] = { 67*e126ba97SEli Cohen .mask = 0, 68*e126ba97SEli Cohen }, 69*e126ba97SEli Cohen [1] = { 70*e126ba97SEli Cohen .mask = MLX5_PROF_MASK_QP_SIZE, 71*e126ba97SEli Cohen .log_max_qp = 12, 72*e126ba97SEli Cohen }, 73*e126ba97SEli Cohen [2] = { 74*e126ba97SEli Cohen .mask = MLX5_PROF_MASK_QP_SIZE | 75*e126ba97SEli Cohen MLX5_PROF_MASK_MR_CACHE, 76*e126ba97SEli Cohen .log_max_qp = 17, 77*e126ba97SEli Cohen .mr_cache[0] = { 78*e126ba97SEli Cohen .size = 500, 79*e126ba97SEli Cohen .limit = 250 80*e126ba97SEli Cohen }, 81*e126ba97SEli Cohen .mr_cache[1] = { 82*e126ba97SEli Cohen .size = 500, 83*e126ba97SEli Cohen .limit = 250 84*e126ba97SEli Cohen }, 85*e126ba97SEli Cohen .mr_cache[2] = { 86*e126ba97SEli Cohen .size = 500, 87*e126ba97SEli Cohen .limit = 250 88*e126ba97SEli Cohen }, 89*e126ba97SEli Cohen .mr_cache[3] = { 90*e126ba97SEli Cohen .size = 500, 91*e126ba97SEli Cohen .limit = 250 92*e126ba97SEli Cohen }, 93*e126ba97SEli Cohen .mr_cache[4] = { 94*e126ba97SEli Cohen .size = 500, 95*e126ba97SEli Cohen .limit = 250 96*e126ba97SEli Cohen }, 97*e126ba97SEli Cohen .mr_cache[5] = { 98*e126ba97SEli Cohen .size = 500, 99*e126ba97SEli Cohen .limit = 250 100*e126ba97SEli Cohen }, 101*e126ba97SEli Cohen .mr_cache[6] = { 102*e126ba97SEli Cohen .size = 500, 103*e126ba97SEli Cohen .limit = 250 104*e126ba97SEli Cohen }, 105*e126ba97SEli Cohen .mr_cache[7] = { 106*e126ba97SEli Cohen .size = 500, 107*e126ba97SEli Cohen .limit = 250 108*e126ba97SEli Cohen }, 109*e126ba97SEli Cohen .mr_cache[8] = { 110*e126ba97SEli Cohen .size = 500, 111*e126ba97SEli Cohen .limit = 250 112*e126ba97SEli Cohen }, 113*e126ba97SEli Cohen .mr_cache[9] = { 114*e126ba97SEli Cohen .size = 500, 115*e126ba97SEli Cohen .limit = 250 116*e126ba97SEli Cohen }, 117*e126ba97SEli Cohen .mr_cache[10] = { 118*e126ba97SEli Cohen .size = 500, 119*e126ba97SEli Cohen .limit = 250 120*e126ba97SEli Cohen }, 121*e126ba97SEli Cohen .mr_cache[11] = { 122*e126ba97SEli Cohen .size = 500, 123*e126ba97SEli Cohen .limit = 250 124*e126ba97SEli Cohen }, 125*e126ba97SEli Cohen .mr_cache[12] = { 126*e126ba97SEli Cohen .size = 64, 127*e126ba97SEli Cohen .limit = 32 128*e126ba97SEli Cohen }, 129*e126ba97SEli Cohen .mr_cache[13] = { 130*e126ba97SEli Cohen .size = 32, 131*e126ba97SEli Cohen .limit = 16 132*e126ba97SEli Cohen }, 133*e126ba97SEli Cohen .mr_cache[14] = { 134*e126ba97SEli Cohen .size = 16, 135*e126ba97SEli Cohen .limit = 8 136*e126ba97SEli Cohen }, 137*e126ba97SEli Cohen .mr_cache[15] = { 138*e126ba97SEli Cohen .size = 8, 139*e126ba97SEli Cohen .limit = 4 140*e126ba97SEli Cohen }, 141*e126ba97SEli Cohen }, 142*e126ba97SEli Cohen }; 143*e126ba97SEli Cohen 144*e126ba97SEli Cohen int mlx5_vector2eqn(struct mlx5_ib_dev *dev, int vector, int *eqn, int *irqn) 145*e126ba97SEli Cohen { 146*e126ba97SEli Cohen struct mlx5_eq_table *table = &dev->mdev.priv.eq_table; 147*e126ba97SEli Cohen struct mlx5_eq *eq, *n; 148*e126ba97SEli Cohen int err = -ENOENT; 149*e126ba97SEli Cohen 150*e126ba97SEli Cohen spin_lock(&table->lock); 151*e126ba97SEli Cohen list_for_each_entry_safe(eq, n, &dev->eqs_list, list) { 152*e126ba97SEli Cohen if (eq->index == vector) { 153*e126ba97SEli Cohen *eqn = eq->eqn; 154*e126ba97SEli Cohen *irqn = eq->irqn; 155*e126ba97SEli Cohen err = 0; 156*e126ba97SEli Cohen break; 157*e126ba97SEli Cohen } 158*e126ba97SEli Cohen } 159*e126ba97SEli Cohen spin_unlock(&table->lock); 160*e126ba97SEli Cohen 161*e126ba97SEli Cohen return err; 162*e126ba97SEli Cohen } 163*e126ba97SEli Cohen 164*e126ba97SEli Cohen static int alloc_comp_eqs(struct mlx5_ib_dev *dev) 165*e126ba97SEli Cohen { 166*e126ba97SEli Cohen struct mlx5_eq_table *table = &dev->mdev.priv.eq_table; 167*e126ba97SEli Cohen struct mlx5_eq *eq, *n; 168*e126ba97SEli Cohen int ncomp_vec; 169*e126ba97SEli Cohen int nent; 170*e126ba97SEli Cohen int err; 171*e126ba97SEli Cohen int i; 172*e126ba97SEli Cohen 173*e126ba97SEli Cohen INIT_LIST_HEAD(&dev->eqs_list); 174*e126ba97SEli Cohen ncomp_vec = table->num_comp_vectors; 175*e126ba97SEli Cohen nent = MLX5_COMP_EQ_SIZE; 176*e126ba97SEli Cohen for (i = 0; i < ncomp_vec; i++) { 177*e126ba97SEli Cohen eq = kzalloc(sizeof(*eq), GFP_KERNEL); 178*e126ba97SEli Cohen if (!eq) { 179*e126ba97SEli Cohen err = -ENOMEM; 180*e126ba97SEli Cohen goto clean; 181*e126ba97SEli Cohen } 182*e126ba97SEli Cohen 183*e126ba97SEli Cohen snprintf(eq->name, MLX5_MAX_EQ_NAME, "mlx5_comp%d", i); 184*e126ba97SEli Cohen err = mlx5_create_map_eq(&dev->mdev, eq, 185*e126ba97SEli Cohen i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 186*e126ba97SEli Cohen eq->name, 187*e126ba97SEli Cohen &dev->mdev.priv.uuari.uars[0]); 188*e126ba97SEli Cohen if (err) { 189*e126ba97SEli Cohen kfree(eq); 190*e126ba97SEli Cohen goto clean; 191*e126ba97SEli Cohen } 192*e126ba97SEli Cohen mlx5_ib_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 193*e126ba97SEli Cohen eq->index = i; 194*e126ba97SEli Cohen spin_lock(&table->lock); 195*e126ba97SEli Cohen list_add_tail(&eq->list, &dev->eqs_list); 196*e126ba97SEli Cohen spin_unlock(&table->lock); 197*e126ba97SEli Cohen } 198*e126ba97SEli Cohen 199*e126ba97SEli Cohen dev->num_comp_vectors = ncomp_vec; 200*e126ba97SEli Cohen return 0; 201*e126ba97SEli Cohen 202*e126ba97SEli Cohen clean: 203*e126ba97SEli Cohen spin_lock(&table->lock); 204*e126ba97SEli Cohen list_for_each_entry_safe(eq, n, &dev->eqs_list, list) { 205*e126ba97SEli Cohen list_del(&eq->list); 206*e126ba97SEli Cohen spin_unlock(&table->lock); 207*e126ba97SEli Cohen if (mlx5_destroy_unmap_eq(&dev->mdev, eq)) 208*e126ba97SEli Cohen mlx5_ib_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn); 209*e126ba97SEli Cohen kfree(eq); 210*e126ba97SEli Cohen spin_lock(&table->lock); 211*e126ba97SEli Cohen } 212*e126ba97SEli Cohen spin_unlock(&table->lock); 213*e126ba97SEli Cohen return err; 214*e126ba97SEli Cohen } 215*e126ba97SEli Cohen 216*e126ba97SEli Cohen static void free_comp_eqs(struct mlx5_ib_dev *dev) 217*e126ba97SEli Cohen { 218*e126ba97SEli Cohen struct mlx5_eq_table *table = &dev->mdev.priv.eq_table; 219*e126ba97SEli Cohen struct mlx5_eq *eq, *n; 220*e126ba97SEli Cohen 221*e126ba97SEli Cohen spin_lock(&table->lock); 222*e126ba97SEli Cohen list_for_each_entry_safe(eq, n, &dev->eqs_list, list) { 223*e126ba97SEli Cohen list_del(&eq->list); 224*e126ba97SEli Cohen spin_unlock(&table->lock); 225*e126ba97SEli Cohen if (mlx5_destroy_unmap_eq(&dev->mdev, eq)) 226*e126ba97SEli Cohen mlx5_ib_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn); 227*e126ba97SEli Cohen kfree(eq); 228*e126ba97SEli Cohen spin_lock(&table->lock); 229*e126ba97SEli Cohen } 230*e126ba97SEli Cohen spin_unlock(&table->lock); 231*e126ba97SEli Cohen } 232*e126ba97SEli Cohen 233*e126ba97SEli Cohen static int mlx5_ib_query_device(struct ib_device *ibdev, 234*e126ba97SEli Cohen struct ib_device_attr *props) 235*e126ba97SEli Cohen { 236*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibdev); 237*e126ba97SEli Cohen struct ib_smp *in_mad = NULL; 238*e126ba97SEli Cohen struct ib_smp *out_mad = NULL; 239*e126ba97SEli Cohen int err = -ENOMEM; 240*e126ba97SEli Cohen int max_rq_sg; 241*e126ba97SEli Cohen int max_sq_sg; 242*e126ba97SEli Cohen u64 flags; 243*e126ba97SEli Cohen 244*e126ba97SEli Cohen in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL); 245*e126ba97SEli Cohen out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL); 246*e126ba97SEli Cohen if (!in_mad || !out_mad) 247*e126ba97SEli Cohen goto out; 248*e126ba97SEli Cohen 249*e126ba97SEli Cohen init_query_mad(in_mad); 250*e126ba97SEli Cohen in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; 251*e126ba97SEli Cohen 252*e126ba97SEli Cohen err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad, out_mad); 253*e126ba97SEli Cohen if (err) 254*e126ba97SEli Cohen goto out; 255*e126ba97SEli Cohen 256*e126ba97SEli Cohen memset(props, 0, sizeof(*props)); 257*e126ba97SEli Cohen 258*e126ba97SEli Cohen props->fw_ver = ((u64)fw_rev_maj(&dev->mdev) << 32) | 259*e126ba97SEli Cohen (fw_rev_min(&dev->mdev) << 16) | 260*e126ba97SEli Cohen fw_rev_sub(&dev->mdev); 261*e126ba97SEli Cohen props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | 262*e126ba97SEli Cohen IB_DEVICE_PORT_ACTIVE_EVENT | 263*e126ba97SEli Cohen IB_DEVICE_SYS_IMAGE_GUID | 264*e126ba97SEli Cohen IB_DEVICE_RC_RNR_NAK_GEN | 265*e126ba97SEli Cohen IB_DEVICE_BLOCK_MULTICAST_LOOPBACK; 266*e126ba97SEli Cohen flags = dev->mdev.caps.flags; 267*e126ba97SEli Cohen if (flags & MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR) 268*e126ba97SEli Cohen props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; 269*e126ba97SEli Cohen if (flags & MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR) 270*e126ba97SEli Cohen props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; 271*e126ba97SEli Cohen if (flags & MLX5_DEV_CAP_FLAG_APM) 272*e126ba97SEli Cohen props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; 273*e126ba97SEli Cohen props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY; 274*e126ba97SEli Cohen if (flags & MLX5_DEV_CAP_FLAG_XRC) 275*e126ba97SEli Cohen props->device_cap_flags |= IB_DEVICE_XRC; 276*e126ba97SEli Cohen props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; 277*e126ba97SEli Cohen 278*e126ba97SEli Cohen props->vendor_id = be32_to_cpup((__be32 *)(out_mad->data + 36)) & 279*e126ba97SEli Cohen 0xffffff; 280*e126ba97SEli Cohen props->vendor_part_id = be16_to_cpup((__be16 *)(out_mad->data + 30)); 281*e126ba97SEli Cohen props->hw_ver = be32_to_cpup((__be32 *)(out_mad->data + 32)); 282*e126ba97SEli Cohen memcpy(&props->sys_image_guid, out_mad->data + 4, 8); 283*e126ba97SEli Cohen 284*e126ba97SEli Cohen props->max_mr_size = ~0ull; 285*e126ba97SEli Cohen props->page_size_cap = dev->mdev.caps.min_page_sz; 286*e126ba97SEli Cohen props->max_qp = 1 << dev->mdev.caps.log_max_qp; 287*e126ba97SEli Cohen props->max_qp_wr = dev->mdev.caps.max_wqes; 288*e126ba97SEli Cohen max_rq_sg = dev->mdev.caps.max_rq_desc_sz / sizeof(struct mlx5_wqe_data_seg); 289*e126ba97SEli Cohen max_sq_sg = (dev->mdev.caps.max_sq_desc_sz - sizeof(struct mlx5_wqe_ctrl_seg)) / 290*e126ba97SEli Cohen sizeof(struct mlx5_wqe_data_seg); 291*e126ba97SEli Cohen props->max_sge = min(max_rq_sg, max_sq_sg); 292*e126ba97SEli Cohen props->max_cq = 1 << dev->mdev.caps.log_max_cq; 293*e126ba97SEli Cohen props->max_cqe = dev->mdev.caps.max_cqes - 1; 294*e126ba97SEli Cohen props->max_mr = 1 << dev->mdev.caps.log_max_mkey; 295*e126ba97SEli Cohen props->max_pd = 1 << dev->mdev.caps.log_max_pd; 296*e126ba97SEli Cohen props->max_qp_rd_atom = dev->mdev.caps.max_ra_req_qp; 297*e126ba97SEli Cohen props->max_qp_init_rd_atom = dev->mdev.caps.max_ra_res_qp; 298*e126ba97SEli Cohen props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; 299*e126ba97SEli Cohen props->max_srq = 1 << dev->mdev.caps.log_max_srq; 300*e126ba97SEli Cohen props->max_srq_wr = dev->mdev.caps.max_srq_wqes - 1; 301*e126ba97SEli Cohen props->max_srq_sge = max_rq_sg - 1; 302*e126ba97SEli Cohen props->max_fast_reg_page_list_len = (unsigned int)-1; 303*e126ba97SEli Cohen props->local_ca_ack_delay = dev->mdev.caps.local_ca_ack_delay; 304*e126ba97SEli Cohen props->atomic_cap = dev->mdev.caps.flags & MLX5_DEV_CAP_FLAG_ATOMIC ? 305*e126ba97SEli Cohen IB_ATOMIC_HCA : IB_ATOMIC_NONE; 306*e126ba97SEli Cohen props->masked_atomic_cap = IB_ATOMIC_HCA; 307*e126ba97SEli Cohen props->max_pkeys = be16_to_cpup((__be16 *)(out_mad->data + 28)); 308*e126ba97SEli Cohen props->max_mcast_grp = 1 << dev->mdev.caps.log_max_mcg; 309*e126ba97SEli Cohen props->max_mcast_qp_attach = dev->mdev.caps.max_qp_mcg; 310*e126ba97SEli Cohen props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 311*e126ba97SEli Cohen props->max_mcast_grp; 312*e126ba97SEli Cohen props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */ 313*e126ba97SEli Cohen 314*e126ba97SEli Cohen out: 315*e126ba97SEli Cohen kfree(in_mad); 316*e126ba97SEli Cohen kfree(out_mad); 317*e126ba97SEli Cohen 318*e126ba97SEli Cohen return err; 319*e126ba97SEli Cohen } 320*e126ba97SEli Cohen 321*e126ba97SEli Cohen int mlx5_ib_query_port(struct ib_device *ibdev, u8 port, 322*e126ba97SEli Cohen struct ib_port_attr *props) 323*e126ba97SEli Cohen { 324*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibdev); 325*e126ba97SEli Cohen struct ib_smp *in_mad = NULL; 326*e126ba97SEli Cohen struct ib_smp *out_mad = NULL; 327*e126ba97SEli Cohen int ext_active_speed; 328*e126ba97SEli Cohen int err = -ENOMEM; 329*e126ba97SEli Cohen 330*e126ba97SEli Cohen if (port < 1 || port > dev->mdev.caps.num_ports) { 331*e126ba97SEli Cohen mlx5_ib_warn(dev, "invalid port number %d\n", port); 332*e126ba97SEli Cohen return -EINVAL; 333*e126ba97SEli Cohen } 334*e126ba97SEli Cohen 335*e126ba97SEli Cohen in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL); 336*e126ba97SEli Cohen out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL); 337*e126ba97SEli Cohen if (!in_mad || !out_mad) 338*e126ba97SEli Cohen goto out; 339*e126ba97SEli Cohen 340*e126ba97SEli Cohen memset(props, 0, sizeof(*props)); 341*e126ba97SEli Cohen 342*e126ba97SEli Cohen init_query_mad(in_mad); 343*e126ba97SEli Cohen in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; 344*e126ba97SEli Cohen in_mad->attr_mod = cpu_to_be32(port); 345*e126ba97SEli Cohen 346*e126ba97SEli Cohen err = mlx5_MAD_IFC(dev, 1, 1, port, NULL, NULL, in_mad, out_mad); 347*e126ba97SEli Cohen if (err) { 348*e126ba97SEli Cohen mlx5_ib_warn(dev, "err %d\n", err); 349*e126ba97SEli Cohen goto out; 350*e126ba97SEli Cohen } 351*e126ba97SEli Cohen 352*e126ba97SEli Cohen 353*e126ba97SEli Cohen props->lid = be16_to_cpup((__be16 *)(out_mad->data + 16)); 354*e126ba97SEli Cohen props->lmc = out_mad->data[34] & 0x7; 355*e126ba97SEli Cohen props->sm_lid = be16_to_cpup((__be16 *)(out_mad->data + 18)); 356*e126ba97SEli Cohen props->sm_sl = out_mad->data[36] & 0xf; 357*e126ba97SEli Cohen props->state = out_mad->data[32] & 0xf; 358*e126ba97SEli Cohen props->phys_state = out_mad->data[33] >> 4; 359*e126ba97SEli Cohen props->port_cap_flags = be32_to_cpup((__be32 *)(out_mad->data + 20)); 360*e126ba97SEli Cohen props->gid_tbl_len = out_mad->data[50]; 361*e126ba97SEli Cohen props->max_msg_sz = 1 << to_mdev(ibdev)->mdev.caps.log_max_msg; 362*e126ba97SEli Cohen props->pkey_tbl_len = to_mdev(ibdev)->mdev.caps.port[port - 1].pkey_table_len; 363*e126ba97SEli Cohen props->bad_pkey_cntr = be16_to_cpup((__be16 *)(out_mad->data + 46)); 364*e126ba97SEli Cohen props->qkey_viol_cntr = be16_to_cpup((__be16 *)(out_mad->data + 48)); 365*e126ba97SEli Cohen props->active_width = out_mad->data[31] & 0xf; 366*e126ba97SEli Cohen props->active_speed = out_mad->data[35] >> 4; 367*e126ba97SEli Cohen props->max_mtu = out_mad->data[41] & 0xf; 368*e126ba97SEli Cohen props->active_mtu = out_mad->data[36] >> 4; 369*e126ba97SEli Cohen props->subnet_timeout = out_mad->data[51] & 0x1f; 370*e126ba97SEli Cohen props->max_vl_num = out_mad->data[37] >> 4; 371*e126ba97SEli Cohen props->init_type_reply = out_mad->data[41] >> 4; 372*e126ba97SEli Cohen 373*e126ba97SEli Cohen /* Check if extended speeds (EDR/FDR/...) are supported */ 374*e126ba97SEli Cohen if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) { 375*e126ba97SEli Cohen ext_active_speed = out_mad->data[62] >> 4; 376*e126ba97SEli Cohen 377*e126ba97SEli Cohen switch (ext_active_speed) { 378*e126ba97SEli Cohen case 1: 379*e126ba97SEli Cohen props->active_speed = 16; /* FDR */ 380*e126ba97SEli Cohen break; 381*e126ba97SEli Cohen case 2: 382*e126ba97SEli Cohen props->active_speed = 32; /* EDR */ 383*e126ba97SEli Cohen break; 384*e126ba97SEli Cohen } 385*e126ba97SEli Cohen } 386*e126ba97SEli Cohen 387*e126ba97SEli Cohen /* If reported active speed is QDR, check if is FDR-10 */ 388*e126ba97SEli Cohen if (props->active_speed == 4) { 389*e126ba97SEli Cohen if (dev->mdev.caps.ext_port_cap[port - 1] & 390*e126ba97SEli Cohen MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) { 391*e126ba97SEli Cohen init_query_mad(in_mad); 392*e126ba97SEli Cohen in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO; 393*e126ba97SEli Cohen in_mad->attr_mod = cpu_to_be32(port); 394*e126ba97SEli Cohen 395*e126ba97SEli Cohen err = mlx5_MAD_IFC(dev, 1, 1, port, 396*e126ba97SEli Cohen NULL, NULL, in_mad, out_mad); 397*e126ba97SEli Cohen if (err) 398*e126ba97SEli Cohen goto out; 399*e126ba97SEli Cohen 400*e126ba97SEli Cohen /* Checking LinkSpeedActive for FDR-10 */ 401*e126ba97SEli Cohen if (out_mad->data[15] & 0x1) 402*e126ba97SEli Cohen props->active_speed = 8; 403*e126ba97SEli Cohen } 404*e126ba97SEli Cohen } 405*e126ba97SEli Cohen 406*e126ba97SEli Cohen out: 407*e126ba97SEli Cohen kfree(in_mad); 408*e126ba97SEli Cohen kfree(out_mad); 409*e126ba97SEli Cohen 410*e126ba97SEli Cohen return err; 411*e126ba97SEli Cohen } 412*e126ba97SEli Cohen 413*e126ba97SEli Cohen static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index, 414*e126ba97SEli Cohen union ib_gid *gid) 415*e126ba97SEli Cohen { 416*e126ba97SEli Cohen struct ib_smp *in_mad = NULL; 417*e126ba97SEli Cohen struct ib_smp *out_mad = NULL; 418*e126ba97SEli Cohen int err = -ENOMEM; 419*e126ba97SEli Cohen 420*e126ba97SEli Cohen in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL); 421*e126ba97SEli Cohen out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL); 422*e126ba97SEli Cohen if (!in_mad || !out_mad) 423*e126ba97SEli Cohen goto out; 424*e126ba97SEli Cohen 425*e126ba97SEli Cohen init_query_mad(in_mad); 426*e126ba97SEli Cohen in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; 427*e126ba97SEli Cohen in_mad->attr_mod = cpu_to_be32(port); 428*e126ba97SEli Cohen 429*e126ba97SEli Cohen err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); 430*e126ba97SEli Cohen if (err) 431*e126ba97SEli Cohen goto out; 432*e126ba97SEli Cohen 433*e126ba97SEli Cohen memcpy(gid->raw, out_mad->data + 8, 8); 434*e126ba97SEli Cohen 435*e126ba97SEli Cohen init_query_mad(in_mad); 436*e126ba97SEli Cohen in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; 437*e126ba97SEli Cohen in_mad->attr_mod = cpu_to_be32(index / 8); 438*e126ba97SEli Cohen 439*e126ba97SEli Cohen err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); 440*e126ba97SEli Cohen if (err) 441*e126ba97SEli Cohen goto out; 442*e126ba97SEli Cohen 443*e126ba97SEli Cohen memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); 444*e126ba97SEli Cohen 445*e126ba97SEli Cohen out: 446*e126ba97SEli Cohen kfree(in_mad); 447*e126ba97SEli Cohen kfree(out_mad); 448*e126ba97SEli Cohen return err; 449*e126ba97SEli Cohen } 450*e126ba97SEli Cohen 451*e126ba97SEli Cohen static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, 452*e126ba97SEli Cohen u16 *pkey) 453*e126ba97SEli Cohen { 454*e126ba97SEli Cohen struct ib_smp *in_mad = NULL; 455*e126ba97SEli Cohen struct ib_smp *out_mad = NULL; 456*e126ba97SEli Cohen int err = -ENOMEM; 457*e126ba97SEli Cohen 458*e126ba97SEli Cohen in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL); 459*e126ba97SEli Cohen out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL); 460*e126ba97SEli Cohen if (!in_mad || !out_mad) 461*e126ba97SEli Cohen goto out; 462*e126ba97SEli Cohen 463*e126ba97SEli Cohen init_query_mad(in_mad); 464*e126ba97SEli Cohen in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; 465*e126ba97SEli Cohen in_mad->attr_mod = cpu_to_be32(index / 32); 466*e126ba97SEli Cohen 467*e126ba97SEli Cohen err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); 468*e126ba97SEli Cohen if (err) 469*e126ba97SEli Cohen goto out; 470*e126ba97SEli Cohen 471*e126ba97SEli Cohen *pkey = be16_to_cpu(((__be16 *)out_mad->data)[index % 32]); 472*e126ba97SEli Cohen 473*e126ba97SEli Cohen out: 474*e126ba97SEli Cohen kfree(in_mad); 475*e126ba97SEli Cohen kfree(out_mad); 476*e126ba97SEli Cohen return err; 477*e126ba97SEli Cohen } 478*e126ba97SEli Cohen 479*e126ba97SEli Cohen struct mlx5_reg_node_desc { 480*e126ba97SEli Cohen u8 desc[64]; 481*e126ba97SEli Cohen }; 482*e126ba97SEli Cohen 483*e126ba97SEli Cohen static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask, 484*e126ba97SEli Cohen struct ib_device_modify *props) 485*e126ba97SEli Cohen { 486*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibdev); 487*e126ba97SEli Cohen struct mlx5_reg_node_desc in; 488*e126ba97SEli Cohen struct mlx5_reg_node_desc out; 489*e126ba97SEli Cohen int err; 490*e126ba97SEli Cohen 491*e126ba97SEli Cohen if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) 492*e126ba97SEli Cohen return -EOPNOTSUPP; 493*e126ba97SEli Cohen 494*e126ba97SEli Cohen if (!(mask & IB_DEVICE_MODIFY_NODE_DESC)) 495*e126ba97SEli Cohen return 0; 496*e126ba97SEli Cohen 497*e126ba97SEli Cohen /* 498*e126ba97SEli Cohen * If possible, pass node desc to FW, so it can generate 499*e126ba97SEli Cohen * a 144 trap. If cmd fails, just ignore. 500*e126ba97SEli Cohen */ 501*e126ba97SEli Cohen memcpy(&in, props->node_desc, 64); 502*e126ba97SEli Cohen err = mlx5_core_access_reg(&dev->mdev, &in, sizeof(in), &out, 503*e126ba97SEli Cohen sizeof(out), MLX5_REG_NODE_DESC, 0, 1); 504*e126ba97SEli Cohen if (err) 505*e126ba97SEli Cohen return err; 506*e126ba97SEli Cohen 507*e126ba97SEli Cohen memcpy(ibdev->node_desc, props->node_desc, 64); 508*e126ba97SEli Cohen 509*e126ba97SEli Cohen return err; 510*e126ba97SEli Cohen } 511*e126ba97SEli Cohen 512*e126ba97SEli Cohen static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask, 513*e126ba97SEli Cohen struct ib_port_modify *props) 514*e126ba97SEli Cohen { 515*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibdev); 516*e126ba97SEli Cohen struct ib_port_attr attr; 517*e126ba97SEli Cohen u32 tmp; 518*e126ba97SEli Cohen int err; 519*e126ba97SEli Cohen 520*e126ba97SEli Cohen mutex_lock(&dev->cap_mask_mutex); 521*e126ba97SEli Cohen 522*e126ba97SEli Cohen err = mlx5_ib_query_port(ibdev, port, &attr); 523*e126ba97SEli Cohen if (err) 524*e126ba97SEli Cohen goto out; 525*e126ba97SEli Cohen 526*e126ba97SEli Cohen tmp = (attr.port_cap_flags | props->set_port_cap_mask) & 527*e126ba97SEli Cohen ~props->clr_port_cap_mask; 528*e126ba97SEli Cohen 529*e126ba97SEli Cohen err = mlx5_set_port_caps(&dev->mdev, port, tmp); 530*e126ba97SEli Cohen 531*e126ba97SEli Cohen out: 532*e126ba97SEli Cohen mutex_unlock(&dev->cap_mask_mutex); 533*e126ba97SEli Cohen return err; 534*e126ba97SEli Cohen } 535*e126ba97SEli Cohen 536*e126ba97SEli Cohen static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev, 537*e126ba97SEli Cohen struct ib_udata *udata) 538*e126ba97SEli Cohen { 539*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibdev); 540*e126ba97SEli Cohen struct mlx5_ib_alloc_ucontext_req req; 541*e126ba97SEli Cohen struct mlx5_ib_alloc_ucontext_resp resp; 542*e126ba97SEli Cohen struct mlx5_ib_ucontext *context; 543*e126ba97SEli Cohen struct mlx5_uuar_info *uuari; 544*e126ba97SEli Cohen struct mlx5_uar *uars; 545*e126ba97SEli Cohen int num_uars; 546*e126ba97SEli Cohen int uuarn; 547*e126ba97SEli Cohen int err; 548*e126ba97SEli Cohen int i; 549*e126ba97SEli Cohen 550*e126ba97SEli Cohen if (!dev->ib_active) 551*e126ba97SEli Cohen return ERR_PTR(-EAGAIN); 552*e126ba97SEli Cohen 553*e126ba97SEli Cohen err = ib_copy_from_udata(&req, udata, sizeof(req)); 554*e126ba97SEli Cohen if (err) 555*e126ba97SEli Cohen return ERR_PTR(err); 556*e126ba97SEli Cohen 557*e126ba97SEli Cohen if (req.total_num_uuars > MLX5_MAX_UUARS) 558*e126ba97SEli Cohen return ERR_PTR(-ENOMEM); 559*e126ba97SEli Cohen 560*e126ba97SEli Cohen if (req.total_num_uuars == 0) 561*e126ba97SEli Cohen return ERR_PTR(-EINVAL); 562*e126ba97SEli Cohen 563*e126ba97SEli Cohen req.total_num_uuars = ALIGN(req.total_num_uuars, MLX5_BF_REGS_PER_PAGE); 564*e126ba97SEli Cohen if (req.num_low_latency_uuars > req.total_num_uuars - 1) 565*e126ba97SEli Cohen return ERR_PTR(-EINVAL); 566*e126ba97SEli Cohen 567*e126ba97SEli Cohen num_uars = req.total_num_uuars / MLX5_BF_REGS_PER_PAGE; 568*e126ba97SEli Cohen resp.qp_tab_size = 1 << dev->mdev.caps.log_max_qp; 569*e126ba97SEli Cohen resp.bf_reg_size = dev->mdev.caps.bf_reg_size; 570*e126ba97SEli Cohen resp.cache_line_size = L1_CACHE_BYTES; 571*e126ba97SEli Cohen resp.max_sq_desc_sz = dev->mdev.caps.max_sq_desc_sz; 572*e126ba97SEli Cohen resp.max_rq_desc_sz = dev->mdev.caps.max_rq_desc_sz; 573*e126ba97SEli Cohen resp.max_send_wqebb = dev->mdev.caps.max_wqes; 574*e126ba97SEli Cohen resp.max_recv_wr = dev->mdev.caps.max_wqes; 575*e126ba97SEli Cohen resp.max_srq_recv_wr = dev->mdev.caps.max_srq_wqes; 576*e126ba97SEli Cohen 577*e126ba97SEli Cohen context = kzalloc(sizeof(*context), GFP_KERNEL); 578*e126ba97SEli Cohen if (!context) 579*e126ba97SEli Cohen return ERR_PTR(-ENOMEM); 580*e126ba97SEli Cohen 581*e126ba97SEli Cohen uuari = &context->uuari; 582*e126ba97SEli Cohen mutex_init(&uuari->lock); 583*e126ba97SEli Cohen uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL); 584*e126ba97SEli Cohen if (!uars) { 585*e126ba97SEli Cohen err = -ENOMEM; 586*e126ba97SEli Cohen goto out_ctx; 587*e126ba97SEli Cohen } 588*e126ba97SEli Cohen 589*e126ba97SEli Cohen uuari->bitmap = kcalloc(BITS_TO_LONGS(req.total_num_uuars), 590*e126ba97SEli Cohen sizeof(*uuari->bitmap), 591*e126ba97SEli Cohen GFP_KERNEL); 592*e126ba97SEli Cohen if (!uuari->bitmap) { 593*e126ba97SEli Cohen err = -ENOMEM; 594*e126ba97SEli Cohen goto out_uar_ctx; 595*e126ba97SEli Cohen } 596*e126ba97SEli Cohen /* 597*e126ba97SEli Cohen * clear all fast path uuars 598*e126ba97SEli Cohen */ 599*e126ba97SEli Cohen for (i = 0; i < req.total_num_uuars; i++) { 600*e126ba97SEli Cohen uuarn = i & 3; 601*e126ba97SEli Cohen if (uuarn == 2 || uuarn == 3) 602*e126ba97SEli Cohen set_bit(i, uuari->bitmap); 603*e126ba97SEli Cohen } 604*e126ba97SEli Cohen 605*e126ba97SEli Cohen uuari->count = kcalloc(req.total_num_uuars, sizeof(*uuari->count), GFP_KERNEL); 606*e126ba97SEli Cohen if (!uuari->count) { 607*e126ba97SEli Cohen err = -ENOMEM; 608*e126ba97SEli Cohen goto out_bitmap; 609*e126ba97SEli Cohen } 610*e126ba97SEli Cohen 611*e126ba97SEli Cohen for (i = 0; i < num_uars; i++) { 612*e126ba97SEli Cohen err = mlx5_cmd_alloc_uar(&dev->mdev, &uars[i].index); 613*e126ba97SEli Cohen if (err) 614*e126ba97SEli Cohen goto out_count; 615*e126ba97SEli Cohen } 616*e126ba97SEli Cohen 617*e126ba97SEli Cohen INIT_LIST_HEAD(&context->db_page_list); 618*e126ba97SEli Cohen mutex_init(&context->db_page_mutex); 619*e126ba97SEli Cohen 620*e126ba97SEli Cohen resp.tot_uuars = req.total_num_uuars; 621*e126ba97SEli Cohen resp.num_ports = dev->mdev.caps.num_ports; 622*e126ba97SEli Cohen err = ib_copy_to_udata(udata, &resp, sizeof(resp)); 623*e126ba97SEli Cohen if (err) 624*e126ba97SEli Cohen goto out_uars; 625*e126ba97SEli Cohen 626*e126ba97SEli Cohen uuari->num_low_latency_uuars = req.num_low_latency_uuars; 627*e126ba97SEli Cohen uuari->uars = uars; 628*e126ba97SEli Cohen uuari->num_uars = num_uars; 629*e126ba97SEli Cohen return &context->ibucontext; 630*e126ba97SEli Cohen 631*e126ba97SEli Cohen out_uars: 632*e126ba97SEli Cohen for (i--; i >= 0; i--) 633*e126ba97SEli Cohen mlx5_cmd_free_uar(&dev->mdev, uars[i].index); 634*e126ba97SEli Cohen out_count: 635*e126ba97SEli Cohen kfree(uuari->count); 636*e126ba97SEli Cohen 637*e126ba97SEli Cohen out_bitmap: 638*e126ba97SEli Cohen kfree(uuari->bitmap); 639*e126ba97SEli Cohen 640*e126ba97SEli Cohen out_uar_ctx: 641*e126ba97SEli Cohen kfree(uars); 642*e126ba97SEli Cohen 643*e126ba97SEli Cohen out_ctx: 644*e126ba97SEli Cohen kfree(context); 645*e126ba97SEli Cohen return ERR_PTR(err); 646*e126ba97SEli Cohen } 647*e126ba97SEli Cohen 648*e126ba97SEli Cohen static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) 649*e126ba97SEli Cohen { 650*e126ba97SEli Cohen struct mlx5_ib_ucontext *context = to_mucontext(ibcontext); 651*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibcontext->device); 652*e126ba97SEli Cohen struct mlx5_uuar_info *uuari = &context->uuari; 653*e126ba97SEli Cohen int i; 654*e126ba97SEli Cohen 655*e126ba97SEli Cohen for (i = 0; i < uuari->num_uars; i++) { 656*e126ba97SEli Cohen if (mlx5_cmd_free_uar(&dev->mdev, uuari->uars[i].index)) 657*e126ba97SEli Cohen mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index); 658*e126ba97SEli Cohen } 659*e126ba97SEli Cohen 660*e126ba97SEli Cohen kfree(uuari->count); 661*e126ba97SEli Cohen kfree(uuari->bitmap); 662*e126ba97SEli Cohen kfree(uuari->uars); 663*e126ba97SEli Cohen kfree(context); 664*e126ba97SEli Cohen 665*e126ba97SEli Cohen return 0; 666*e126ba97SEli Cohen } 667*e126ba97SEli Cohen 668*e126ba97SEli Cohen static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index) 669*e126ba97SEli Cohen { 670*e126ba97SEli Cohen return (pci_resource_start(dev->mdev.pdev, 0) >> PAGE_SHIFT) + index; 671*e126ba97SEli Cohen } 672*e126ba97SEli Cohen 673*e126ba97SEli Cohen static int get_command(unsigned long offset) 674*e126ba97SEli Cohen { 675*e126ba97SEli Cohen return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK; 676*e126ba97SEli Cohen } 677*e126ba97SEli Cohen 678*e126ba97SEli Cohen static int get_arg(unsigned long offset) 679*e126ba97SEli Cohen { 680*e126ba97SEli Cohen return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1); 681*e126ba97SEli Cohen } 682*e126ba97SEli Cohen 683*e126ba97SEli Cohen static int get_index(unsigned long offset) 684*e126ba97SEli Cohen { 685*e126ba97SEli Cohen return get_arg(offset); 686*e126ba97SEli Cohen } 687*e126ba97SEli Cohen 688*e126ba97SEli Cohen static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) 689*e126ba97SEli Cohen { 690*e126ba97SEli Cohen struct mlx5_ib_ucontext *context = to_mucontext(ibcontext); 691*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibcontext->device); 692*e126ba97SEli Cohen struct mlx5_uuar_info *uuari = &context->uuari; 693*e126ba97SEli Cohen unsigned long command; 694*e126ba97SEli Cohen unsigned long idx; 695*e126ba97SEli Cohen phys_addr_t pfn; 696*e126ba97SEli Cohen 697*e126ba97SEli Cohen command = get_command(vma->vm_pgoff); 698*e126ba97SEli Cohen switch (command) { 699*e126ba97SEli Cohen case MLX5_IB_MMAP_REGULAR_PAGE: 700*e126ba97SEli Cohen if (vma->vm_end - vma->vm_start != PAGE_SIZE) 701*e126ba97SEli Cohen return -EINVAL; 702*e126ba97SEli Cohen 703*e126ba97SEli Cohen idx = get_index(vma->vm_pgoff); 704*e126ba97SEli Cohen pfn = uar_index2pfn(dev, uuari->uars[idx].index); 705*e126ba97SEli Cohen mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx, 706*e126ba97SEli Cohen (unsigned long long)pfn); 707*e126ba97SEli Cohen 708*e126ba97SEli Cohen if (idx >= uuari->num_uars) 709*e126ba97SEli Cohen return -EINVAL; 710*e126ba97SEli Cohen 711*e126ba97SEli Cohen vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 712*e126ba97SEli Cohen if (io_remap_pfn_range(vma, vma->vm_start, pfn, 713*e126ba97SEli Cohen PAGE_SIZE, vma->vm_page_prot)) 714*e126ba97SEli Cohen return -EAGAIN; 715*e126ba97SEli Cohen 716*e126ba97SEli Cohen mlx5_ib_dbg(dev, "mapped WC at 0x%lx, PA 0x%llx\n", 717*e126ba97SEli Cohen vma->vm_start, 718*e126ba97SEli Cohen (unsigned long long)pfn << PAGE_SHIFT); 719*e126ba97SEli Cohen break; 720*e126ba97SEli Cohen 721*e126ba97SEli Cohen case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES: 722*e126ba97SEli Cohen return -ENOSYS; 723*e126ba97SEli Cohen 724*e126ba97SEli Cohen default: 725*e126ba97SEli Cohen return -EINVAL; 726*e126ba97SEli Cohen } 727*e126ba97SEli Cohen 728*e126ba97SEli Cohen return 0; 729*e126ba97SEli Cohen } 730*e126ba97SEli Cohen 731*e126ba97SEli Cohen static int alloc_pa_mkey(struct mlx5_ib_dev *dev, u32 *key, u32 pdn) 732*e126ba97SEli Cohen { 733*e126ba97SEli Cohen struct mlx5_create_mkey_mbox_in *in; 734*e126ba97SEli Cohen struct mlx5_mkey_seg *seg; 735*e126ba97SEli Cohen struct mlx5_core_mr mr; 736*e126ba97SEli Cohen int err; 737*e126ba97SEli Cohen 738*e126ba97SEli Cohen in = kzalloc(sizeof(*in), GFP_KERNEL); 739*e126ba97SEli Cohen if (!in) 740*e126ba97SEli Cohen return -ENOMEM; 741*e126ba97SEli Cohen 742*e126ba97SEli Cohen seg = &in->seg; 743*e126ba97SEli Cohen seg->flags = MLX5_PERM_LOCAL_READ | MLX5_ACCESS_MODE_PA; 744*e126ba97SEli Cohen seg->flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64); 745*e126ba97SEli Cohen seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 746*e126ba97SEli Cohen seg->start_addr = 0; 747*e126ba97SEli Cohen 748*e126ba97SEli Cohen err = mlx5_core_create_mkey(&dev->mdev, &mr, in, sizeof(*in)); 749*e126ba97SEli Cohen if (err) { 750*e126ba97SEli Cohen mlx5_ib_warn(dev, "failed to create mkey, %d\n", err); 751*e126ba97SEli Cohen goto err_in; 752*e126ba97SEli Cohen } 753*e126ba97SEli Cohen 754*e126ba97SEli Cohen kfree(in); 755*e126ba97SEli Cohen *key = mr.key; 756*e126ba97SEli Cohen 757*e126ba97SEli Cohen return 0; 758*e126ba97SEli Cohen 759*e126ba97SEli Cohen err_in: 760*e126ba97SEli Cohen kfree(in); 761*e126ba97SEli Cohen 762*e126ba97SEli Cohen return err; 763*e126ba97SEli Cohen } 764*e126ba97SEli Cohen 765*e126ba97SEli Cohen static void free_pa_mkey(struct mlx5_ib_dev *dev, u32 key) 766*e126ba97SEli Cohen { 767*e126ba97SEli Cohen struct mlx5_core_mr mr; 768*e126ba97SEli Cohen int err; 769*e126ba97SEli Cohen 770*e126ba97SEli Cohen memset(&mr, 0, sizeof(mr)); 771*e126ba97SEli Cohen mr.key = key; 772*e126ba97SEli Cohen err = mlx5_core_destroy_mkey(&dev->mdev, &mr); 773*e126ba97SEli Cohen if (err) 774*e126ba97SEli Cohen mlx5_ib_warn(dev, "failed to destroy mkey 0x%x\n", key); 775*e126ba97SEli Cohen } 776*e126ba97SEli Cohen 777*e126ba97SEli Cohen static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev, 778*e126ba97SEli Cohen struct ib_ucontext *context, 779*e126ba97SEli Cohen struct ib_udata *udata) 780*e126ba97SEli Cohen { 781*e126ba97SEli Cohen struct mlx5_ib_alloc_pd_resp resp; 782*e126ba97SEli Cohen struct mlx5_ib_pd *pd; 783*e126ba97SEli Cohen int err; 784*e126ba97SEli Cohen 785*e126ba97SEli Cohen pd = kmalloc(sizeof(*pd), GFP_KERNEL); 786*e126ba97SEli Cohen if (!pd) 787*e126ba97SEli Cohen return ERR_PTR(-ENOMEM); 788*e126ba97SEli Cohen 789*e126ba97SEli Cohen err = mlx5_core_alloc_pd(&to_mdev(ibdev)->mdev, &pd->pdn); 790*e126ba97SEli Cohen if (err) { 791*e126ba97SEli Cohen kfree(pd); 792*e126ba97SEli Cohen return ERR_PTR(err); 793*e126ba97SEli Cohen } 794*e126ba97SEli Cohen 795*e126ba97SEli Cohen if (context) { 796*e126ba97SEli Cohen resp.pdn = pd->pdn; 797*e126ba97SEli Cohen if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { 798*e126ba97SEli Cohen mlx5_core_dealloc_pd(&to_mdev(ibdev)->mdev, pd->pdn); 799*e126ba97SEli Cohen kfree(pd); 800*e126ba97SEli Cohen return ERR_PTR(-EFAULT); 801*e126ba97SEli Cohen } 802*e126ba97SEli Cohen } else { 803*e126ba97SEli Cohen err = alloc_pa_mkey(to_mdev(ibdev), &pd->pa_lkey, pd->pdn); 804*e126ba97SEli Cohen if (err) { 805*e126ba97SEli Cohen mlx5_core_dealloc_pd(&to_mdev(ibdev)->mdev, pd->pdn); 806*e126ba97SEli Cohen kfree(pd); 807*e126ba97SEli Cohen return ERR_PTR(err); 808*e126ba97SEli Cohen } 809*e126ba97SEli Cohen } 810*e126ba97SEli Cohen 811*e126ba97SEli Cohen return &pd->ibpd; 812*e126ba97SEli Cohen } 813*e126ba97SEli Cohen 814*e126ba97SEli Cohen static int mlx5_ib_dealloc_pd(struct ib_pd *pd) 815*e126ba97SEli Cohen { 816*e126ba97SEli Cohen struct mlx5_ib_dev *mdev = to_mdev(pd->device); 817*e126ba97SEli Cohen struct mlx5_ib_pd *mpd = to_mpd(pd); 818*e126ba97SEli Cohen 819*e126ba97SEli Cohen if (!pd->uobject) 820*e126ba97SEli Cohen free_pa_mkey(mdev, mpd->pa_lkey); 821*e126ba97SEli Cohen 822*e126ba97SEli Cohen mlx5_core_dealloc_pd(&mdev->mdev, mpd->pdn); 823*e126ba97SEli Cohen kfree(mpd); 824*e126ba97SEli Cohen 825*e126ba97SEli Cohen return 0; 826*e126ba97SEli Cohen } 827*e126ba97SEli Cohen 828*e126ba97SEli Cohen static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 829*e126ba97SEli Cohen { 830*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 831*e126ba97SEli Cohen int err; 832*e126ba97SEli Cohen 833*e126ba97SEli Cohen err = mlx5_core_attach_mcg(&dev->mdev, gid, ibqp->qp_num); 834*e126ba97SEli Cohen if (err) 835*e126ba97SEli Cohen mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n", 836*e126ba97SEli Cohen ibqp->qp_num, gid->raw); 837*e126ba97SEli Cohen 838*e126ba97SEli Cohen return err; 839*e126ba97SEli Cohen } 840*e126ba97SEli Cohen 841*e126ba97SEli Cohen static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 842*e126ba97SEli Cohen { 843*e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 844*e126ba97SEli Cohen int err; 845*e126ba97SEli Cohen 846*e126ba97SEli Cohen err = mlx5_core_detach_mcg(&dev->mdev, gid, ibqp->qp_num); 847*e126ba97SEli Cohen if (err) 848*e126ba97SEli Cohen mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n", 849*e126ba97SEli Cohen ibqp->qp_num, gid->raw); 850*e126ba97SEli Cohen 851*e126ba97SEli Cohen return err; 852*e126ba97SEli Cohen } 853*e126ba97SEli Cohen 854*e126ba97SEli Cohen static int init_node_data(struct mlx5_ib_dev *dev) 855*e126ba97SEli Cohen { 856*e126ba97SEli Cohen struct ib_smp *in_mad = NULL; 857*e126ba97SEli Cohen struct ib_smp *out_mad = NULL; 858*e126ba97SEli Cohen int err = -ENOMEM; 859*e126ba97SEli Cohen 860*e126ba97SEli Cohen in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL); 861*e126ba97SEli Cohen out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL); 862*e126ba97SEli Cohen if (!in_mad || !out_mad) 863*e126ba97SEli Cohen goto out; 864*e126ba97SEli Cohen 865*e126ba97SEli Cohen init_query_mad(in_mad); 866*e126ba97SEli Cohen in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; 867*e126ba97SEli Cohen 868*e126ba97SEli Cohen err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad); 869*e126ba97SEli Cohen if (err) 870*e126ba97SEli Cohen goto out; 871*e126ba97SEli Cohen 872*e126ba97SEli Cohen memcpy(dev->ib_dev.node_desc, out_mad->data, 64); 873*e126ba97SEli Cohen 874*e126ba97SEli Cohen in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; 875*e126ba97SEli Cohen 876*e126ba97SEli Cohen err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad); 877*e126ba97SEli Cohen if (err) 878*e126ba97SEli Cohen goto out; 879*e126ba97SEli Cohen 880*e126ba97SEli Cohen dev->mdev.rev_id = be32_to_cpup((__be32 *)(out_mad->data + 32)); 881*e126ba97SEli Cohen memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); 882*e126ba97SEli Cohen 883*e126ba97SEli Cohen out: 884*e126ba97SEli Cohen kfree(in_mad); 885*e126ba97SEli Cohen kfree(out_mad); 886*e126ba97SEli Cohen return err; 887*e126ba97SEli Cohen } 888*e126ba97SEli Cohen 889*e126ba97SEli Cohen static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr, 890*e126ba97SEli Cohen char *buf) 891*e126ba97SEli Cohen { 892*e126ba97SEli Cohen struct mlx5_ib_dev *dev = 893*e126ba97SEli Cohen container_of(device, struct mlx5_ib_dev, ib_dev.dev); 894*e126ba97SEli Cohen 895*e126ba97SEli Cohen return sprintf(buf, "%d\n", dev->mdev.priv.fw_pages); 896*e126ba97SEli Cohen } 897*e126ba97SEli Cohen 898*e126ba97SEli Cohen static ssize_t show_reg_pages(struct device *device, 899*e126ba97SEli Cohen struct device_attribute *attr, char *buf) 900*e126ba97SEli Cohen { 901*e126ba97SEli Cohen struct mlx5_ib_dev *dev = 902*e126ba97SEli Cohen container_of(device, struct mlx5_ib_dev, ib_dev.dev); 903*e126ba97SEli Cohen 904*e126ba97SEli Cohen return sprintf(buf, "%d\n", dev->mdev.priv.reg_pages); 905*e126ba97SEli Cohen } 906*e126ba97SEli Cohen 907*e126ba97SEli Cohen static ssize_t show_hca(struct device *device, struct device_attribute *attr, 908*e126ba97SEli Cohen char *buf) 909*e126ba97SEli Cohen { 910*e126ba97SEli Cohen struct mlx5_ib_dev *dev = 911*e126ba97SEli Cohen container_of(device, struct mlx5_ib_dev, ib_dev.dev); 912*e126ba97SEli Cohen return sprintf(buf, "MT%d\n", dev->mdev.pdev->device); 913*e126ba97SEli Cohen } 914*e126ba97SEli Cohen 915*e126ba97SEli Cohen static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, 916*e126ba97SEli Cohen char *buf) 917*e126ba97SEli Cohen { 918*e126ba97SEli Cohen struct mlx5_ib_dev *dev = 919*e126ba97SEli Cohen container_of(device, struct mlx5_ib_dev, ib_dev.dev); 920*e126ba97SEli Cohen return sprintf(buf, "%d.%d.%d\n", fw_rev_maj(&dev->mdev), 921*e126ba97SEli Cohen fw_rev_min(&dev->mdev), fw_rev_sub(&dev->mdev)); 922*e126ba97SEli Cohen } 923*e126ba97SEli Cohen 924*e126ba97SEli Cohen static ssize_t show_rev(struct device *device, struct device_attribute *attr, 925*e126ba97SEli Cohen char *buf) 926*e126ba97SEli Cohen { 927*e126ba97SEli Cohen struct mlx5_ib_dev *dev = 928*e126ba97SEli Cohen container_of(device, struct mlx5_ib_dev, ib_dev.dev); 929*e126ba97SEli Cohen return sprintf(buf, "%x\n", dev->mdev.rev_id); 930*e126ba97SEli Cohen } 931*e126ba97SEli Cohen 932*e126ba97SEli Cohen static ssize_t show_board(struct device *device, struct device_attribute *attr, 933*e126ba97SEli Cohen char *buf) 934*e126ba97SEli Cohen { 935*e126ba97SEli Cohen struct mlx5_ib_dev *dev = 936*e126ba97SEli Cohen container_of(device, struct mlx5_ib_dev, ib_dev.dev); 937*e126ba97SEli Cohen return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN, 938*e126ba97SEli Cohen dev->mdev.board_id); 939*e126ba97SEli Cohen } 940*e126ba97SEli Cohen 941*e126ba97SEli Cohen static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); 942*e126ba97SEli Cohen static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); 943*e126ba97SEli Cohen static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); 944*e126ba97SEli Cohen static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); 945*e126ba97SEli Cohen static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL); 946*e126ba97SEli Cohen static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL); 947*e126ba97SEli Cohen 948*e126ba97SEli Cohen static struct device_attribute *mlx5_class_attributes[] = { 949*e126ba97SEli Cohen &dev_attr_hw_rev, 950*e126ba97SEli Cohen &dev_attr_fw_ver, 951*e126ba97SEli Cohen &dev_attr_hca_type, 952*e126ba97SEli Cohen &dev_attr_board_id, 953*e126ba97SEli Cohen &dev_attr_fw_pages, 954*e126ba97SEli Cohen &dev_attr_reg_pages, 955*e126ba97SEli Cohen }; 956*e126ba97SEli Cohen 957*e126ba97SEli Cohen static void mlx5_ib_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 958*e126ba97SEli Cohen void *data) 959*e126ba97SEli Cohen { 960*e126ba97SEli Cohen struct mlx5_ib_dev *ibdev = container_of(dev, struct mlx5_ib_dev, mdev); 961*e126ba97SEli Cohen struct ib_event ibev; 962*e126ba97SEli Cohen u8 port = 0; 963*e126ba97SEli Cohen 964*e126ba97SEli Cohen switch (event) { 965*e126ba97SEli Cohen case MLX5_DEV_EVENT_SYS_ERROR: 966*e126ba97SEli Cohen ibdev->ib_active = false; 967*e126ba97SEli Cohen ibev.event = IB_EVENT_DEVICE_FATAL; 968*e126ba97SEli Cohen break; 969*e126ba97SEli Cohen 970*e126ba97SEli Cohen case MLX5_DEV_EVENT_PORT_UP: 971*e126ba97SEli Cohen ibev.event = IB_EVENT_PORT_ACTIVE; 972*e126ba97SEli Cohen port = *(u8 *)data; 973*e126ba97SEli Cohen break; 974*e126ba97SEli Cohen 975*e126ba97SEli Cohen case MLX5_DEV_EVENT_PORT_DOWN: 976*e126ba97SEli Cohen ibev.event = IB_EVENT_PORT_ERR; 977*e126ba97SEli Cohen port = *(u8 *)data; 978*e126ba97SEli Cohen break; 979*e126ba97SEli Cohen 980*e126ba97SEli Cohen case MLX5_DEV_EVENT_PORT_INITIALIZED: 981*e126ba97SEli Cohen /* not used by ULPs */ 982*e126ba97SEli Cohen return; 983*e126ba97SEli Cohen 984*e126ba97SEli Cohen case MLX5_DEV_EVENT_LID_CHANGE: 985*e126ba97SEli Cohen ibev.event = IB_EVENT_LID_CHANGE; 986*e126ba97SEli Cohen port = *(u8 *)data; 987*e126ba97SEli Cohen break; 988*e126ba97SEli Cohen 989*e126ba97SEli Cohen case MLX5_DEV_EVENT_PKEY_CHANGE: 990*e126ba97SEli Cohen ibev.event = IB_EVENT_PKEY_CHANGE; 991*e126ba97SEli Cohen port = *(u8 *)data; 992*e126ba97SEli Cohen break; 993*e126ba97SEli Cohen 994*e126ba97SEli Cohen case MLX5_DEV_EVENT_GUID_CHANGE: 995*e126ba97SEli Cohen ibev.event = IB_EVENT_GID_CHANGE; 996*e126ba97SEli Cohen port = *(u8 *)data; 997*e126ba97SEli Cohen break; 998*e126ba97SEli Cohen 999*e126ba97SEli Cohen case MLX5_DEV_EVENT_CLIENT_REREG: 1000*e126ba97SEli Cohen ibev.event = IB_EVENT_CLIENT_REREGISTER; 1001*e126ba97SEli Cohen port = *(u8 *)data; 1002*e126ba97SEli Cohen break; 1003*e126ba97SEli Cohen } 1004*e126ba97SEli Cohen 1005*e126ba97SEli Cohen ibev.device = &ibdev->ib_dev; 1006*e126ba97SEli Cohen ibev.element.port_num = port; 1007*e126ba97SEli Cohen 1008*e126ba97SEli Cohen if (ibdev->ib_active) 1009*e126ba97SEli Cohen ib_dispatch_event(&ibev); 1010*e126ba97SEli Cohen } 1011*e126ba97SEli Cohen 1012*e126ba97SEli Cohen static void get_ext_port_caps(struct mlx5_ib_dev *dev) 1013*e126ba97SEli Cohen { 1014*e126ba97SEli Cohen int port; 1015*e126ba97SEli Cohen 1016*e126ba97SEli Cohen for (port = 1; port <= dev->mdev.caps.num_ports; port++) 1017*e126ba97SEli Cohen mlx5_query_ext_port_caps(dev, port); 1018*e126ba97SEli Cohen } 1019*e126ba97SEli Cohen 1020*e126ba97SEli Cohen static int get_port_caps(struct mlx5_ib_dev *dev) 1021*e126ba97SEli Cohen { 1022*e126ba97SEli Cohen struct ib_device_attr *dprops = NULL; 1023*e126ba97SEli Cohen struct ib_port_attr *pprops = NULL; 1024*e126ba97SEli Cohen int err = 0; 1025*e126ba97SEli Cohen int port; 1026*e126ba97SEli Cohen 1027*e126ba97SEli Cohen pprops = kmalloc(sizeof(*pprops), GFP_KERNEL); 1028*e126ba97SEli Cohen if (!pprops) 1029*e126ba97SEli Cohen goto out; 1030*e126ba97SEli Cohen 1031*e126ba97SEli Cohen dprops = kmalloc(sizeof(*dprops), GFP_KERNEL); 1032*e126ba97SEli Cohen if (!dprops) 1033*e126ba97SEli Cohen goto out; 1034*e126ba97SEli Cohen 1035*e126ba97SEli Cohen err = mlx5_ib_query_device(&dev->ib_dev, dprops); 1036*e126ba97SEli Cohen if (err) { 1037*e126ba97SEli Cohen mlx5_ib_warn(dev, "query_device failed %d\n", err); 1038*e126ba97SEli Cohen goto out; 1039*e126ba97SEli Cohen } 1040*e126ba97SEli Cohen 1041*e126ba97SEli Cohen for (port = 1; port <= dev->mdev.caps.num_ports; port++) { 1042*e126ba97SEli Cohen err = mlx5_ib_query_port(&dev->ib_dev, port, pprops); 1043*e126ba97SEli Cohen if (err) { 1044*e126ba97SEli Cohen mlx5_ib_warn(dev, "query_port %d failed %d\n", port, err); 1045*e126ba97SEli Cohen break; 1046*e126ba97SEli Cohen } 1047*e126ba97SEli Cohen dev->mdev.caps.port[port - 1].pkey_table_len = dprops->max_pkeys; 1048*e126ba97SEli Cohen dev->mdev.caps.port[port - 1].gid_table_len = pprops->gid_tbl_len; 1049*e126ba97SEli Cohen mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n", 1050*e126ba97SEli Cohen dprops->max_pkeys, pprops->gid_tbl_len); 1051*e126ba97SEli Cohen } 1052*e126ba97SEli Cohen 1053*e126ba97SEli Cohen out: 1054*e126ba97SEli Cohen kfree(pprops); 1055*e126ba97SEli Cohen kfree(dprops); 1056*e126ba97SEli Cohen 1057*e126ba97SEli Cohen return err; 1058*e126ba97SEli Cohen } 1059*e126ba97SEli Cohen 1060*e126ba97SEli Cohen static void destroy_umrc_res(struct mlx5_ib_dev *dev) 1061*e126ba97SEli Cohen { 1062*e126ba97SEli Cohen int err; 1063*e126ba97SEli Cohen 1064*e126ba97SEli Cohen err = mlx5_mr_cache_cleanup(dev); 1065*e126ba97SEli Cohen if (err) 1066*e126ba97SEli Cohen mlx5_ib_warn(dev, "mr cache cleanup failed\n"); 1067*e126ba97SEli Cohen 1068*e126ba97SEli Cohen mlx5_ib_destroy_qp(dev->umrc.qp); 1069*e126ba97SEli Cohen ib_destroy_cq(dev->umrc.cq); 1070*e126ba97SEli Cohen ib_dereg_mr(dev->umrc.mr); 1071*e126ba97SEli Cohen ib_dealloc_pd(dev->umrc.pd); 1072*e126ba97SEli Cohen } 1073*e126ba97SEli Cohen 1074*e126ba97SEli Cohen enum { 1075*e126ba97SEli Cohen MAX_UMR_WR = 128, 1076*e126ba97SEli Cohen }; 1077*e126ba97SEli Cohen 1078*e126ba97SEli Cohen static int create_umr_res(struct mlx5_ib_dev *dev) 1079*e126ba97SEli Cohen { 1080*e126ba97SEli Cohen struct ib_qp_init_attr *init_attr = NULL; 1081*e126ba97SEli Cohen struct ib_qp_attr *attr = NULL; 1082*e126ba97SEli Cohen struct ib_pd *pd; 1083*e126ba97SEli Cohen struct ib_cq *cq; 1084*e126ba97SEli Cohen struct ib_qp *qp; 1085*e126ba97SEli Cohen struct ib_mr *mr; 1086*e126ba97SEli Cohen int ret; 1087*e126ba97SEli Cohen 1088*e126ba97SEli Cohen attr = kzalloc(sizeof(*attr), GFP_KERNEL); 1089*e126ba97SEli Cohen init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL); 1090*e126ba97SEli Cohen if (!attr || !init_attr) { 1091*e126ba97SEli Cohen ret = -ENOMEM; 1092*e126ba97SEli Cohen goto error_0; 1093*e126ba97SEli Cohen } 1094*e126ba97SEli Cohen 1095*e126ba97SEli Cohen pd = ib_alloc_pd(&dev->ib_dev); 1096*e126ba97SEli Cohen if (IS_ERR(pd)) { 1097*e126ba97SEli Cohen mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n"); 1098*e126ba97SEli Cohen ret = PTR_ERR(pd); 1099*e126ba97SEli Cohen goto error_0; 1100*e126ba97SEli Cohen } 1101*e126ba97SEli Cohen 1102*e126ba97SEli Cohen mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE); 1103*e126ba97SEli Cohen if (IS_ERR(mr)) { 1104*e126ba97SEli Cohen mlx5_ib_dbg(dev, "Couldn't create DMA MR for sync UMR QP\n"); 1105*e126ba97SEli Cohen ret = PTR_ERR(mr); 1106*e126ba97SEli Cohen goto error_1; 1107*e126ba97SEli Cohen } 1108*e126ba97SEli Cohen 1109*e126ba97SEli Cohen cq = ib_create_cq(&dev->ib_dev, mlx5_umr_cq_handler, NULL, NULL, 128, 1110*e126ba97SEli Cohen 0); 1111*e126ba97SEli Cohen if (IS_ERR(cq)) { 1112*e126ba97SEli Cohen mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n"); 1113*e126ba97SEli Cohen ret = PTR_ERR(cq); 1114*e126ba97SEli Cohen goto error_2; 1115*e126ba97SEli Cohen } 1116*e126ba97SEli Cohen ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 1117*e126ba97SEli Cohen 1118*e126ba97SEli Cohen init_attr->send_cq = cq; 1119*e126ba97SEli Cohen init_attr->recv_cq = cq; 1120*e126ba97SEli Cohen init_attr->sq_sig_type = IB_SIGNAL_ALL_WR; 1121*e126ba97SEli Cohen init_attr->cap.max_send_wr = MAX_UMR_WR; 1122*e126ba97SEli Cohen init_attr->cap.max_send_sge = 1; 1123*e126ba97SEli Cohen init_attr->qp_type = MLX5_IB_QPT_REG_UMR; 1124*e126ba97SEli Cohen init_attr->port_num = 1; 1125*e126ba97SEli Cohen qp = mlx5_ib_create_qp(pd, init_attr, NULL); 1126*e126ba97SEli Cohen if (IS_ERR(qp)) { 1127*e126ba97SEli Cohen mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n"); 1128*e126ba97SEli Cohen ret = PTR_ERR(qp); 1129*e126ba97SEli Cohen goto error_3; 1130*e126ba97SEli Cohen } 1131*e126ba97SEli Cohen qp->device = &dev->ib_dev; 1132*e126ba97SEli Cohen qp->real_qp = qp; 1133*e126ba97SEli Cohen qp->uobject = NULL; 1134*e126ba97SEli Cohen qp->qp_type = MLX5_IB_QPT_REG_UMR; 1135*e126ba97SEli Cohen 1136*e126ba97SEli Cohen attr->qp_state = IB_QPS_INIT; 1137*e126ba97SEli Cohen attr->port_num = 1; 1138*e126ba97SEli Cohen ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX | 1139*e126ba97SEli Cohen IB_QP_PORT, NULL); 1140*e126ba97SEli Cohen if (ret) { 1141*e126ba97SEli Cohen mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n"); 1142*e126ba97SEli Cohen goto error_4; 1143*e126ba97SEli Cohen } 1144*e126ba97SEli Cohen 1145*e126ba97SEli Cohen memset(attr, 0, sizeof(*attr)); 1146*e126ba97SEli Cohen attr->qp_state = IB_QPS_RTR; 1147*e126ba97SEli Cohen attr->path_mtu = IB_MTU_256; 1148*e126ba97SEli Cohen 1149*e126ba97SEli Cohen ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); 1150*e126ba97SEli Cohen if (ret) { 1151*e126ba97SEli Cohen mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n"); 1152*e126ba97SEli Cohen goto error_4; 1153*e126ba97SEli Cohen } 1154*e126ba97SEli Cohen 1155*e126ba97SEli Cohen memset(attr, 0, sizeof(*attr)); 1156*e126ba97SEli Cohen attr->qp_state = IB_QPS_RTS; 1157*e126ba97SEli Cohen ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL); 1158*e126ba97SEli Cohen if (ret) { 1159*e126ba97SEli Cohen mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n"); 1160*e126ba97SEli Cohen goto error_4; 1161*e126ba97SEli Cohen } 1162*e126ba97SEli Cohen 1163*e126ba97SEli Cohen dev->umrc.qp = qp; 1164*e126ba97SEli Cohen dev->umrc.cq = cq; 1165*e126ba97SEli Cohen dev->umrc.mr = mr; 1166*e126ba97SEli Cohen dev->umrc.pd = pd; 1167*e126ba97SEli Cohen 1168*e126ba97SEli Cohen sema_init(&dev->umrc.sem, MAX_UMR_WR); 1169*e126ba97SEli Cohen ret = mlx5_mr_cache_init(dev); 1170*e126ba97SEli Cohen if (ret) { 1171*e126ba97SEli Cohen mlx5_ib_warn(dev, "mr cache init failed %d\n", ret); 1172*e126ba97SEli Cohen goto error_4; 1173*e126ba97SEli Cohen } 1174*e126ba97SEli Cohen 1175*e126ba97SEli Cohen kfree(attr); 1176*e126ba97SEli Cohen kfree(init_attr); 1177*e126ba97SEli Cohen 1178*e126ba97SEli Cohen return 0; 1179*e126ba97SEli Cohen 1180*e126ba97SEli Cohen error_4: 1181*e126ba97SEli Cohen mlx5_ib_destroy_qp(qp); 1182*e126ba97SEli Cohen 1183*e126ba97SEli Cohen error_3: 1184*e126ba97SEli Cohen ib_destroy_cq(cq); 1185*e126ba97SEli Cohen 1186*e126ba97SEli Cohen error_2: 1187*e126ba97SEli Cohen ib_dereg_mr(mr); 1188*e126ba97SEli Cohen 1189*e126ba97SEli Cohen error_1: 1190*e126ba97SEli Cohen ib_dealloc_pd(pd); 1191*e126ba97SEli Cohen 1192*e126ba97SEli Cohen error_0: 1193*e126ba97SEli Cohen kfree(attr); 1194*e126ba97SEli Cohen kfree(init_attr); 1195*e126ba97SEli Cohen return ret; 1196*e126ba97SEli Cohen } 1197*e126ba97SEli Cohen 1198*e126ba97SEli Cohen static int create_dev_resources(struct mlx5_ib_resources *devr) 1199*e126ba97SEli Cohen { 1200*e126ba97SEli Cohen struct ib_srq_init_attr attr; 1201*e126ba97SEli Cohen struct mlx5_ib_dev *dev; 1202*e126ba97SEli Cohen int ret = 0; 1203*e126ba97SEli Cohen 1204*e126ba97SEli Cohen dev = container_of(devr, struct mlx5_ib_dev, devr); 1205*e126ba97SEli Cohen 1206*e126ba97SEli Cohen devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL); 1207*e126ba97SEli Cohen if (IS_ERR(devr->p0)) { 1208*e126ba97SEli Cohen ret = PTR_ERR(devr->p0); 1209*e126ba97SEli Cohen goto error0; 1210*e126ba97SEli Cohen } 1211*e126ba97SEli Cohen devr->p0->device = &dev->ib_dev; 1212*e126ba97SEli Cohen devr->p0->uobject = NULL; 1213*e126ba97SEli Cohen atomic_set(&devr->p0->usecnt, 0); 1214*e126ba97SEli Cohen 1215*e126ba97SEli Cohen devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, 1, 0, NULL, NULL); 1216*e126ba97SEli Cohen if (IS_ERR(devr->c0)) { 1217*e126ba97SEli Cohen ret = PTR_ERR(devr->c0); 1218*e126ba97SEli Cohen goto error1; 1219*e126ba97SEli Cohen } 1220*e126ba97SEli Cohen devr->c0->device = &dev->ib_dev; 1221*e126ba97SEli Cohen devr->c0->uobject = NULL; 1222*e126ba97SEli Cohen devr->c0->comp_handler = NULL; 1223*e126ba97SEli Cohen devr->c0->event_handler = NULL; 1224*e126ba97SEli Cohen devr->c0->cq_context = NULL; 1225*e126ba97SEli Cohen atomic_set(&devr->c0->usecnt, 0); 1226*e126ba97SEli Cohen 1227*e126ba97SEli Cohen devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL); 1228*e126ba97SEli Cohen if (IS_ERR(devr->x0)) { 1229*e126ba97SEli Cohen ret = PTR_ERR(devr->x0); 1230*e126ba97SEli Cohen goto error2; 1231*e126ba97SEli Cohen } 1232*e126ba97SEli Cohen devr->x0->device = &dev->ib_dev; 1233*e126ba97SEli Cohen devr->x0->inode = NULL; 1234*e126ba97SEli Cohen atomic_set(&devr->x0->usecnt, 0); 1235*e126ba97SEli Cohen mutex_init(&devr->x0->tgt_qp_mutex); 1236*e126ba97SEli Cohen INIT_LIST_HEAD(&devr->x0->tgt_qp_list); 1237*e126ba97SEli Cohen 1238*e126ba97SEli Cohen devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL); 1239*e126ba97SEli Cohen if (IS_ERR(devr->x1)) { 1240*e126ba97SEli Cohen ret = PTR_ERR(devr->x1); 1241*e126ba97SEli Cohen goto error3; 1242*e126ba97SEli Cohen } 1243*e126ba97SEli Cohen devr->x1->device = &dev->ib_dev; 1244*e126ba97SEli Cohen devr->x1->inode = NULL; 1245*e126ba97SEli Cohen atomic_set(&devr->x1->usecnt, 0); 1246*e126ba97SEli Cohen mutex_init(&devr->x1->tgt_qp_mutex); 1247*e126ba97SEli Cohen INIT_LIST_HEAD(&devr->x1->tgt_qp_list); 1248*e126ba97SEli Cohen 1249*e126ba97SEli Cohen memset(&attr, 0, sizeof(attr)); 1250*e126ba97SEli Cohen attr.attr.max_sge = 1; 1251*e126ba97SEli Cohen attr.attr.max_wr = 1; 1252*e126ba97SEli Cohen attr.srq_type = IB_SRQT_XRC; 1253*e126ba97SEli Cohen attr.ext.xrc.cq = devr->c0; 1254*e126ba97SEli Cohen attr.ext.xrc.xrcd = devr->x0; 1255*e126ba97SEli Cohen 1256*e126ba97SEli Cohen devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL); 1257*e126ba97SEli Cohen if (IS_ERR(devr->s0)) { 1258*e126ba97SEli Cohen ret = PTR_ERR(devr->s0); 1259*e126ba97SEli Cohen goto error4; 1260*e126ba97SEli Cohen } 1261*e126ba97SEli Cohen devr->s0->device = &dev->ib_dev; 1262*e126ba97SEli Cohen devr->s0->pd = devr->p0; 1263*e126ba97SEli Cohen devr->s0->uobject = NULL; 1264*e126ba97SEli Cohen devr->s0->event_handler = NULL; 1265*e126ba97SEli Cohen devr->s0->srq_context = NULL; 1266*e126ba97SEli Cohen devr->s0->srq_type = IB_SRQT_XRC; 1267*e126ba97SEli Cohen devr->s0->ext.xrc.xrcd = devr->x0; 1268*e126ba97SEli Cohen devr->s0->ext.xrc.cq = devr->c0; 1269*e126ba97SEli Cohen atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt); 1270*e126ba97SEli Cohen atomic_inc(&devr->s0->ext.xrc.cq->usecnt); 1271*e126ba97SEli Cohen atomic_inc(&devr->p0->usecnt); 1272*e126ba97SEli Cohen atomic_set(&devr->s0->usecnt, 0); 1273*e126ba97SEli Cohen 1274*e126ba97SEli Cohen return 0; 1275*e126ba97SEli Cohen 1276*e126ba97SEli Cohen error4: 1277*e126ba97SEli Cohen mlx5_ib_dealloc_xrcd(devr->x1); 1278*e126ba97SEli Cohen error3: 1279*e126ba97SEli Cohen mlx5_ib_dealloc_xrcd(devr->x0); 1280*e126ba97SEli Cohen error2: 1281*e126ba97SEli Cohen mlx5_ib_destroy_cq(devr->c0); 1282*e126ba97SEli Cohen error1: 1283*e126ba97SEli Cohen mlx5_ib_dealloc_pd(devr->p0); 1284*e126ba97SEli Cohen error0: 1285*e126ba97SEli Cohen return ret; 1286*e126ba97SEli Cohen } 1287*e126ba97SEli Cohen 1288*e126ba97SEli Cohen static void destroy_dev_resources(struct mlx5_ib_resources *devr) 1289*e126ba97SEli Cohen { 1290*e126ba97SEli Cohen mlx5_ib_destroy_srq(devr->s0); 1291*e126ba97SEli Cohen mlx5_ib_dealloc_xrcd(devr->x0); 1292*e126ba97SEli Cohen mlx5_ib_dealloc_xrcd(devr->x1); 1293*e126ba97SEli Cohen mlx5_ib_destroy_cq(devr->c0); 1294*e126ba97SEli Cohen mlx5_ib_dealloc_pd(devr->p0); 1295*e126ba97SEli Cohen } 1296*e126ba97SEli Cohen 1297*e126ba97SEli Cohen static int init_one(struct pci_dev *pdev, 1298*e126ba97SEli Cohen const struct pci_device_id *id) 1299*e126ba97SEli Cohen { 1300*e126ba97SEli Cohen struct mlx5_core_dev *mdev; 1301*e126ba97SEli Cohen struct mlx5_ib_dev *dev; 1302*e126ba97SEli Cohen int err; 1303*e126ba97SEli Cohen int i; 1304*e126ba97SEli Cohen 1305*e126ba97SEli Cohen printk_once(KERN_INFO "%s", mlx5_version); 1306*e126ba97SEli Cohen 1307*e126ba97SEli Cohen dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev)); 1308*e126ba97SEli Cohen if (!dev) 1309*e126ba97SEli Cohen return -ENOMEM; 1310*e126ba97SEli Cohen 1311*e126ba97SEli Cohen mdev = &dev->mdev; 1312*e126ba97SEli Cohen mdev->event = mlx5_ib_event; 1313*e126ba97SEli Cohen if (prof_sel >= ARRAY_SIZE(profile)) { 1314*e126ba97SEli Cohen pr_warn("selected pofile out of range, selceting default\n"); 1315*e126ba97SEli Cohen prof_sel = 0; 1316*e126ba97SEli Cohen } 1317*e126ba97SEli Cohen mdev->profile = &profile[prof_sel]; 1318*e126ba97SEli Cohen err = mlx5_dev_init(mdev, pdev); 1319*e126ba97SEli Cohen if (err) 1320*e126ba97SEli Cohen goto err_free; 1321*e126ba97SEli Cohen 1322*e126ba97SEli Cohen err = get_port_caps(dev); 1323*e126ba97SEli Cohen if (err) 1324*e126ba97SEli Cohen goto err_cleanup; 1325*e126ba97SEli Cohen 1326*e126ba97SEli Cohen get_ext_port_caps(dev); 1327*e126ba97SEli Cohen 1328*e126ba97SEli Cohen err = alloc_comp_eqs(dev); 1329*e126ba97SEli Cohen if (err) 1330*e126ba97SEli Cohen goto err_cleanup; 1331*e126ba97SEli Cohen 1332*e126ba97SEli Cohen MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock); 1333*e126ba97SEli Cohen 1334*e126ba97SEli Cohen strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX); 1335*e126ba97SEli Cohen dev->ib_dev.owner = THIS_MODULE; 1336*e126ba97SEli Cohen dev->ib_dev.node_type = RDMA_NODE_IB_CA; 1337*e126ba97SEli Cohen dev->ib_dev.local_dma_lkey = mdev->caps.reserved_lkey; 1338*e126ba97SEli Cohen dev->num_ports = mdev->caps.num_ports; 1339*e126ba97SEli Cohen dev->ib_dev.phys_port_cnt = dev->num_ports; 1340*e126ba97SEli Cohen dev->ib_dev.num_comp_vectors = dev->num_comp_vectors; 1341*e126ba97SEli Cohen dev->ib_dev.dma_device = &mdev->pdev->dev; 1342*e126ba97SEli Cohen 1343*e126ba97SEli Cohen dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION; 1344*e126ba97SEli Cohen dev->ib_dev.uverbs_cmd_mask = 1345*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 1346*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 1347*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 1348*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 1349*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 1350*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_REG_MR) | 1351*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 1352*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 1353*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 1354*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | 1355*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 1356*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 1357*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 1358*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_QUERY_QP) | 1359*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 1360*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | 1361*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | 1362*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | 1363*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | 1364*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | 1365*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | 1366*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) | 1367*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_OPEN_QP); 1368*e126ba97SEli Cohen 1369*e126ba97SEli Cohen dev->ib_dev.query_device = mlx5_ib_query_device; 1370*e126ba97SEli Cohen dev->ib_dev.query_port = mlx5_ib_query_port; 1371*e126ba97SEli Cohen dev->ib_dev.query_gid = mlx5_ib_query_gid; 1372*e126ba97SEli Cohen dev->ib_dev.query_pkey = mlx5_ib_query_pkey; 1373*e126ba97SEli Cohen dev->ib_dev.modify_device = mlx5_ib_modify_device; 1374*e126ba97SEli Cohen dev->ib_dev.modify_port = mlx5_ib_modify_port; 1375*e126ba97SEli Cohen dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext; 1376*e126ba97SEli Cohen dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext; 1377*e126ba97SEli Cohen dev->ib_dev.mmap = mlx5_ib_mmap; 1378*e126ba97SEli Cohen dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd; 1379*e126ba97SEli Cohen dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd; 1380*e126ba97SEli Cohen dev->ib_dev.create_ah = mlx5_ib_create_ah; 1381*e126ba97SEli Cohen dev->ib_dev.query_ah = mlx5_ib_query_ah; 1382*e126ba97SEli Cohen dev->ib_dev.destroy_ah = mlx5_ib_destroy_ah; 1383*e126ba97SEli Cohen dev->ib_dev.create_srq = mlx5_ib_create_srq; 1384*e126ba97SEli Cohen dev->ib_dev.modify_srq = mlx5_ib_modify_srq; 1385*e126ba97SEli Cohen dev->ib_dev.query_srq = mlx5_ib_query_srq; 1386*e126ba97SEli Cohen dev->ib_dev.destroy_srq = mlx5_ib_destroy_srq; 1387*e126ba97SEli Cohen dev->ib_dev.post_srq_recv = mlx5_ib_post_srq_recv; 1388*e126ba97SEli Cohen dev->ib_dev.create_qp = mlx5_ib_create_qp; 1389*e126ba97SEli Cohen dev->ib_dev.modify_qp = mlx5_ib_modify_qp; 1390*e126ba97SEli Cohen dev->ib_dev.query_qp = mlx5_ib_query_qp; 1391*e126ba97SEli Cohen dev->ib_dev.destroy_qp = mlx5_ib_destroy_qp; 1392*e126ba97SEli Cohen dev->ib_dev.post_send = mlx5_ib_post_send; 1393*e126ba97SEli Cohen dev->ib_dev.post_recv = mlx5_ib_post_recv; 1394*e126ba97SEli Cohen dev->ib_dev.create_cq = mlx5_ib_create_cq; 1395*e126ba97SEli Cohen dev->ib_dev.modify_cq = mlx5_ib_modify_cq; 1396*e126ba97SEli Cohen dev->ib_dev.resize_cq = mlx5_ib_resize_cq; 1397*e126ba97SEli Cohen dev->ib_dev.destroy_cq = mlx5_ib_destroy_cq; 1398*e126ba97SEli Cohen dev->ib_dev.poll_cq = mlx5_ib_poll_cq; 1399*e126ba97SEli Cohen dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq; 1400*e126ba97SEli Cohen dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr; 1401*e126ba97SEli Cohen dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr; 1402*e126ba97SEli Cohen dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr; 1403*e126ba97SEli Cohen dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach; 1404*e126ba97SEli Cohen dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach; 1405*e126ba97SEli Cohen dev->ib_dev.process_mad = mlx5_ib_process_mad; 1406*e126ba97SEli Cohen dev->ib_dev.alloc_fast_reg_mr = mlx5_ib_alloc_fast_reg_mr; 1407*e126ba97SEli Cohen dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list; 1408*e126ba97SEli Cohen dev->ib_dev.free_fast_reg_page_list = mlx5_ib_free_fast_reg_page_list; 1409*e126ba97SEli Cohen 1410*e126ba97SEli Cohen if (mdev->caps.flags & MLX5_DEV_CAP_FLAG_XRC) { 1411*e126ba97SEli Cohen dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd; 1412*e126ba97SEli Cohen dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd; 1413*e126ba97SEli Cohen dev->ib_dev.uverbs_cmd_mask |= 1414*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) | 1415*e126ba97SEli Cohen (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD); 1416*e126ba97SEli Cohen } 1417*e126ba97SEli Cohen 1418*e126ba97SEli Cohen err = init_node_data(dev); 1419*e126ba97SEli Cohen if (err) 1420*e126ba97SEli Cohen goto err_eqs; 1421*e126ba97SEli Cohen 1422*e126ba97SEli Cohen mutex_init(&dev->cap_mask_mutex); 1423*e126ba97SEli Cohen spin_lock_init(&dev->mr_lock); 1424*e126ba97SEli Cohen 1425*e126ba97SEli Cohen err = create_dev_resources(&dev->devr); 1426*e126ba97SEli Cohen if (err) 1427*e126ba97SEli Cohen goto err_eqs; 1428*e126ba97SEli Cohen 1429*e126ba97SEli Cohen if (ib_register_device(&dev->ib_dev, NULL)) 1430*e126ba97SEli Cohen goto err_rsrc; 1431*e126ba97SEli Cohen 1432*e126ba97SEli Cohen err = create_umr_res(dev); 1433*e126ba97SEli Cohen if (err) 1434*e126ba97SEli Cohen goto err_dev; 1435*e126ba97SEli Cohen 1436*e126ba97SEli Cohen for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) { 1437*e126ba97SEli Cohen if (device_create_file(&dev->ib_dev.dev, 1438*e126ba97SEli Cohen mlx5_class_attributes[i])) 1439*e126ba97SEli Cohen goto err_umrc; 1440*e126ba97SEli Cohen } 1441*e126ba97SEli Cohen 1442*e126ba97SEli Cohen dev->ib_active = true; 1443*e126ba97SEli Cohen 1444*e126ba97SEli Cohen return 0; 1445*e126ba97SEli Cohen 1446*e126ba97SEli Cohen err_umrc: 1447*e126ba97SEli Cohen destroy_umrc_res(dev); 1448*e126ba97SEli Cohen 1449*e126ba97SEli Cohen err_dev: 1450*e126ba97SEli Cohen ib_unregister_device(&dev->ib_dev); 1451*e126ba97SEli Cohen 1452*e126ba97SEli Cohen err_rsrc: 1453*e126ba97SEli Cohen destroy_dev_resources(&dev->devr); 1454*e126ba97SEli Cohen 1455*e126ba97SEli Cohen err_eqs: 1456*e126ba97SEli Cohen free_comp_eqs(dev); 1457*e126ba97SEli Cohen 1458*e126ba97SEli Cohen err_cleanup: 1459*e126ba97SEli Cohen mlx5_dev_cleanup(mdev); 1460*e126ba97SEli Cohen 1461*e126ba97SEli Cohen err_free: 1462*e126ba97SEli Cohen ib_dealloc_device((struct ib_device *)dev); 1463*e126ba97SEli Cohen 1464*e126ba97SEli Cohen return err; 1465*e126ba97SEli Cohen } 1466*e126ba97SEli Cohen 1467*e126ba97SEli Cohen static void remove_one(struct pci_dev *pdev) 1468*e126ba97SEli Cohen { 1469*e126ba97SEli Cohen struct mlx5_ib_dev *dev = mlx5_pci2ibdev(pdev); 1470*e126ba97SEli Cohen 1471*e126ba97SEli Cohen destroy_umrc_res(dev); 1472*e126ba97SEli Cohen ib_unregister_device(&dev->ib_dev); 1473*e126ba97SEli Cohen destroy_dev_resources(&dev->devr); 1474*e126ba97SEli Cohen free_comp_eqs(dev); 1475*e126ba97SEli Cohen mlx5_dev_cleanup(&dev->mdev); 1476*e126ba97SEli Cohen ib_dealloc_device(&dev->ib_dev); 1477*e126ba97SEli Cohen } 1478*e126ba97SEli Cohen 1479*e126ba97SEli Cohen static DEFINE_PCI_DEVICE_TABLE(mlx5_ib_pci_table) = { 1480*e126ba97SEli Cohen { PCI_VDEVICE(MELLANOX, 4113) }, /* MT4113 Connect-IB */ 1481*e126ba97SEli Cohen { 0, } 1482*e126ba97SEli Cohen }; 1483*e126ba97SEli Cohen 1484*e126ba97SEli Cohen MODULE_DEVICE_TABLE(pci, mlx5_ib_pci_table); 1485*e126ba97SEli Cohen 1486*e126ba97SEli Cohen static struct pci_driver mlx5_ib_driver = { 1487*e126ba97SEli Cohen .name = DRIVER_NAME, 1488*e126ba97SEli Cohen .id_table = mlx5_ib_pci_table, 1489*e126ba97SEli Cohen .probe = init_one, 1490*e126ba97SEli Cohen .remove = remove_one 1491*e126ba97SEli Cohen }; 1492*e126ba97SEli Cohen 1493*e126ba97SEli Cohen static int __init mlx5_ib_init(void) 1494*e126ba97SEli Cohen { 1495*e126ba97SEli Cohen return pci_register_driver(&mlx5_ib_driver); 1496*e126ba97SEli Cohen } 1497*e126ba97SEli Cohen 1498*e126ba97SEli Cohen static void __exit mlx5_ib_cleanup(void) 1499*e126ba97SEli Cohen { 1500*e126ba97SEli Cohen pci_unregister_driver(&mlx5_ib_driver); 1501*e126ba97SEli Cohen } 1502*e126ba97SEli Cohen 1503*e126ba97SEli Cohen module_init(mlx5_ib_init); 1504*e126ba97SEli Cohen module_exit(mlx5_ib_cleanup); 1505