1e126ba97SEli Cohen /* 2e126ba97SEli Cohen * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved. 3e126ba97SEli Cohen * 4e126ba97SEli Cohen * This software is available to you under a choice of one of two 5e126ba97SEli Cohen * licenses. You may choose to be licensed under the terms of the GNU 6e126ba97SEli Cohen * General Public License (GPL) Version 2, available from the file 7e126ba97SEli Cohen * COPYING in the main directory of this source tree, or the 8e126ba97SEli Cohen * OpenIB.org BSD license below: 9e126ba97SEli Cohen * 10e126ba97SEli Cohen * Redistribution and use in source and binary forms, with or 11e126ba97SEli Cohen * without modification, are permitted provided that the following 12e126ba97SEli Cohen * conditions are met: 13e126ba97SEli Cohen * 14e126ba97SEli Cohen * - Redistributions of source code must retain the above 15e126ba97SEli Cohen * copyright notice, this list of conditions and the following 16e126ba97SEli Cohen * disclaimer. 17e126ba97SEli Cohen * 18e126ba97SEli Cohen * - Redistributions in binary form must reproduce the above 19e126ba97SEli Cohen * copyright notice, this list of conditions and the following 20e126ba97SEli Cohen * disclaimer in the documentation and/or other materials 21e126ba97SEli Cohen * provided with the distribution. 22e126ba97SEli Cohen * 23e126ba97SEli Cohen * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e126ba97SEli Cohen * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e126ba97SEli Cohen * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e126ba97SEli Cohen * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e126ba97SEli Cohen * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e126ba97SEli Cohen * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e126ba97SEli Cohen * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e126ba97SEli Cohen * SOFTWARE. 31e126ba97SEli Cohen */ 32e126ba97SEli Cohen 33e126ba97SEli Cohen #include <asm-generic/kmap_types.h> 34e126ba97SEli Cohen #include <linux/module.h> 35e126ba97SEli Cohen #include <linux/init.h> 36e126ba97SEli Cohen #include <linux/errno.h> 37e126ba97SEli Cohen #include <linux/pci.h> 38e126ba97SEli Cohen #include <linux/dma-mapping.h> 39e126ba97SEli Cohen #include <linux/slab.h> 40e126ba97SEli Cohen #include <linux/io-mapping.h> 41e126ba97SEli Cohen #include <linux/mlx5/driver.h> 42e126ba97SEli Cohen #include <linux/mlx5/cq.h> 43e126ba97SEli Cohen #include <linux/mlx5/qp.h> 44e126ba97SEli Cohen #include <linux/mlx5/srq.h> 45e126ba97SEli Cohen #include <linux/debugfs.h> 46f66f049fSEli Cohen #include <linux/kmod.h> 47b775516bSEli Cohen #include <linux/mlx5/mlx5_ifc.h> 48e126ba97SEli Cohen #include "mlx5_core.h" 49e126ba97SEli Cohen 50e126ba97SEli Cohen #define DRIVER_NAME "mlx5_core" 51169a1d85SAmir Vadai #define DRIVER_VERSION "2.2-1" 52169a1d85SAmir Vadai #define DRIVER_RELDATE "Feb 2014" 53e126ba97SEli Cohen 54e126ba97SEli Cohen MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 55e126ba97SEli Cohen MODULE_DESCRIPTION("Mellanox ConnectX-IB HCA core library"); 56e126ba97SEli Cohen MODULE_LICENSE("Dual BSD/GPL"); 57e126ba97SEli Cohen MODULE_VERSION(DRIVER_VERSION); 58e126ba97SEli Cohen 59e126ba97SEli Cohen int mlx5_core_debug_mask; 60e126ba97SEli Cohen module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 61e126ba97SEli Cohen MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 62e126ba97SEli Cohen 639603b61dSJack Morgenstein #define MLX5_DEFAULT_PROF 2 649603b61dSJack Morgenstein static int prof_sel = MLX5_DEFAULT_PROF; 659603b61dSJack Morgenstein module_param_named(prof_sel, prof_sel, int, 0444); 669603b61dSJack Morgenstein MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 679603b61dSJack Morgenstein 68e126ba97SEli Cohen struct workqueue_struct *mlx5_core_wq; 699603b61dSJack Morgenstein static LIST_HEAD(intf_list); 709603b61dSJack Morgenstein static LIST_HEAD(dev_list); 719603b61dSJack Morgenstein static DEFINE_MUTEX(intf_mutex); 729603b61dSJack Morgenstein 739603b61dSJack Morgenstein struct mlx5_device_context { 749603b61dSJack Morgenstein struct list_head list; 759603b61dSJack Morgenstein struct mlx5_interface *intf; 769603b61dSJack Morgenstein void *context; 779603b61dSJack Morgenstein }; 789603b61dSJack Morgenstein 799603b61dSJack Morgenstein static struct mlx5_profile profile[] = { 809603b61dSJack Morgenstein [0] = { 819603b61dSJack Morgenstein .mask = 0, 829603b61dSJack Morgenstein }, 839603b61dSJack Morgenstein [1] = { 849603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE, 859603b61dSJack Morgenstein .log_max_qp = 12, 869603b61dSJack Morgenstein }, 879603b61dSJack Morgenstein [2] = { 889603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE | 899603b61dSJack Morgenstein MLX5_PROF_MASK_MR_CACHE, 909603b61dSJack Morgenstein .log_max_qp = 17, 919603b61dSJack Morgenstein .mr_cache[0] = { 929603b61dSJack Morgenstein .size = 500, 939603b61dSJack Morgenstein .limit = 250 949603b61dSJack Morgenstein }, 959603b61dSJack Morgenstein .mr_cache[1] = { 969603b61dSJack Morgenstein .size = 500, 979603b61dSJack Morgenstein .limit = 250 989603b61dSJack Morgenstein }, 999603b61dSJack Morgenstein .mr_cache[2] = { 1009603b61dSJack Morgenstein .size = 500, 1019603b61dSJack Morgenstein .limit = 250 1029603b61dSJack Morgenstein }, 1039603b61dSJack Morgenstein .mr_cache[3] = { 1049603b61dSJack Morgenstein .size = 500, 1059603b61dSJack Morgenstein .limit = 250 1069603b61dSJack Morgenstein }, 1079603b61dSJack Morgenstein .mr_cache[4] = { 1089603b61dSJack Morgenstein .size = 500, 1099603b61dSJack Morgenstein .limit = 250 1109603b61dSJack Morgenstein }, 1119603b61dSJack Morgenstein .mr_cache[5] = { 1129603b61dSJack Morgenstein .size = 500, 1139603b61dSJack Morgenstein .limit = 250 1149603b61dSJack Morgenstein }, 1159603b61dSJack Morgenstein .mr_cache[6] = { 1169603b61dSJack Morgenstein .size = 500, 1179603b61dSJack Morgenstein .limit = 250 1189603b61dSJack Morgenstein }, 1199603b61dSJack Morgenstein .mr_cache[7] = { 1209603b61dSJack Morgenstein .size = 500, 1219603b61dSJack Morgenstein .limit = 250 1229603b61dSJack Morgenstein }, 1239603b61dSJack Morgenstein .mr_cache[8] = { 1249603b61dSJack Morgenstein .size = 500, 1259603b61dSJack Morgenstein .limit = 250 1269603b61dSJack Morgenstein }, 1279603b61dSJack Morgenstein .mr_cache[9] = { 1289603b61dSJack Morgenstein .size = 500, 1299603b61dSJack Morgenstein .limit = 250 1309603b61dSJack Morgenstein }, 1319603b61dSJack Morgenstein .mr_cache[10] = { 1329603b61dSJack Morgenstein .size = 500, 1339603b61dSJack Morgenstein .limit = 250 1349603b61dSJack Morgenstein }, 1359603b61dSJack Morgenstein .mr_cache[11] = { 1369603b61dSJack Morgenstein .size = 500, 1379603b61dSJack Morgenstein .limit = 250 1389603b61dSJack Morgenstein }, 1399603b61dSJack Morgenstein .mr_cache[12] = { 1409603b61dSJack Morgenstein .size = 64, 1419603b61dSJack Morgenstein .limit = 32 1429603b61dSJack Morgenstein }, 1439603b61dSJack Morgenstein .mr_cache[13] = { 1449603b61dSJack Morgenstein .size = 32, 1459603b61dSJack Morgenstein .limit = 16 1469603b61dSJack Morgenstein }, 1479603b61dSJack Morgenstein .mr_cache[14] = { 1489603b61dSJack Morgenstein .size = 16, 1499603b61dSJack Morgenstein .limit = 8 1509603b61dSJack Morgenstein }, 1519603b61dSJack Morgenstein .mr_cache[15] = { 1529603b61dSJack Morgenstein .size = 8, 1539603b61dSJack Morgenstein .limit = 4 1549603b61dSJack Morgenstein }, 1559603b61dSJack Morgenstein }, 1569603b61dSJack Morgenstein }; 157e126ba97SEli Cohen 158e126ba97SEli Cohen static int set_dma_caps(struct pci_dev *pdev) 159e126ba97SEli Cohen { 160e126ba97SEli Cohen int err; 161e126ba97SEli Cohen 162e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 163e126ba97SEli Cohen if (err) { 1641a91de28SJoe Perches dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 165e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 166e126ba97SEli Cohen if (err) { 1671a91de28SJoe Perches dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 168e126ba97SEli Cohen return err; 169e126ba97SEli Cohen } 170e126ba97SEli Cohen } 171e126ba97SEli Cohen 172e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 173e126ba97SEli Cohen if (err) { 174e126ba97SEli Cohen dev_warn(&pdev->dev, 1751a91de28SJoe Perches "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 176e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 177e126ba97SEli Cohen if (err) { 178e126ba97SEli Cohen dev_err(&pdev->dev, 1791a91de28SJoe Perches "Can't set consistent PCI DMA mask, aborting\n"); 180e126ba97SEli Cohen return err; 181e126ba97SEli Cohen } 182e126ba97SEli Cohen } 183e126ba97SEli Cohen 184e126ba97SEli Cohen dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 185e126ba97SEli Cohen return err; 186e126ba97SEli Cohen } 187e126ba97SEli Cohen 188e126ba97SEli Cohen static int request_bar(struct pci_dev *pdev) 189e126ba97SEli Cohen { 190e126ba97SEli Cohen int err = 0; 191e126ba97SEli Cohen 192e126ba97SEli Cohen if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 1931a91de28SJoe Perches dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 194e126ba97SEli Cohen return -ENODEV; 195e126ba97SEli Cohen } 196e126ba97SEli Cohen 197e126ba97SEli Cohen err = pci_request_regions(pdev, DRIVER_NAME); 198e126ba97SEli Cohen if (err) 199e126ba97SEli Cohen dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 200e126ba97SEli Cohen 201e126ba97SEli Cohen return err; 202e126ba97SEli Cohen } 203e126ba97SEli Cohen 204e126ba97SEli Cohen static void release_bar(struct pci_dev *pdev) 205e126ba97SEli Cohen { 206e126ba97SEli Cohen pci_release_regions(pdev); 207e126ba97SEli Cohen } 208e126ba97SEli Cohen 209e126ba97SEli Cohen static int mlx5_enable_msix(struct mlx5_core_dev *dev) 210e126ba97SEli Cohen { 211e126ba97SEli Cohen struct mlx5_eq_table *table = &dev->priv.eq_table; 212c7a08ac7SEli Cohen int num_eqs = 1 << dev->caps.gen.log_max_eq; 213e126ba97SEli Cohen int nvec; 214e126ba97SEli Cohen int i; 215e126ba97SEli Cohen 216c7a08ac7SEli Cohen nvec = dev->caps.gen.num_ports * num_online_cpus() + MLX5_EQ_VEC_COMP_BASE; 217e126ba97SEli Cohen nvec = min_t(int, nvec, num_eqs); 218e126ba97SEli Cohen if (nvec <= MLX5_EQ_VEC_COMP_BASE) 219e126ba97SEli Cohen return -ENOMEM; 220e126ba97SEli Cohen 221e126ba97SEli Cohen table->msix_arr = kzalloc(nvec * sizeof(*table->msix_arr), GFP_KERNEL); 222e126ba97SEli Cohen if (!table->msix_arr) 223e126ba97SEli Cohen return -ENOMEM; 224e126ba97SEli Cohen 225e126ba97SEli Cohen for (i = 0; i < nvec; i++) 226e126ba97SEli Cohen table->msix_arr[i].entry = i; 227e126ba97SEli Cohen 228f3c9407bSAlexander Gordeev nvec = pci_enable_msix_range(dev->pdev, table->msix_arr, 2293a9e161aSEli Cohen MLX5_EQ_VEC_COMP_BASE + 1, nvec); 230f3c9407bSAlexander Gordeev if (nvec < 0) 231f3c9407bSAlexander Gordeev return nvec; 232e126ba97SEli Cohen 233f3c9407bSAlexander Gordeev table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 234e126ba97SEli Cohen 235e126ba97SEli Cohen return 0; 236e126ba97SEli Cohen } 237e126ba97SEli Cohen 238e126ba97SEli Cohen static void mlx5_disable_msix(struct mlx5_core_dev *dev) 239e126ba97SEli Cohen { 240e126ba97SEli Cohen struct mlx5_eq_table *table = &dev->priv.eq_table; 241e126ba97SEli Cohen 242e126ba97SEli Cohen pci_disable_msix(dev->pdev); 243e126ba97SEli Cohen kfree(table->msix_arr); 244e126ba97SEli Cohen } 245e126ba97SEli Cohen 246e126ba97SEli Cohen struct mlx5_reg_host_endianess { 247e126ba97SEli Cohen u8 he; 248e126ba97SEli Cohen u8 rsvd[15]; 249e126ba97SEli Cohen }; 250e126ba97SEli Cohen 25187b8de49SEli Cohen 25287b8de49SEli Cohen #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 25387b8de49SEli Cohen 25487b8de49SEli Cohen enum { 25587b8de49SEli Cohen MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 256c7a08ac7SEli Cohen MLX5_DEV_CAP_FLAG_DCT, 25787b8de49SEli Cohen }; 25887b8de49SEli Cohen 259c7a08ac7SEli Cohen static u16 to_fw_pkey_sz(u32 size) 260c7a08ac7SEli Cohen { 261c7a08ac7SEli Cohen switch (size) { 262c7a08ac7SEli Cohen case 128: 263c7a08ac7SEli Cohen return 0; 264c7a08ac7SEli Cohen case 256: 265c7a08ac7SEli Cohen return 1; 266c7a08ac7SEli Cohen case 512: 267c7a08ac7SEli Cohen return 2; 268c7a08ac7SEli Cohen case 1024: 269c7a08ac7SEli Cohen return 3; 270c7a08ac7SEli Cohen case 2048: 271c7a08ac7SEli Cohen return 4; 272c7a08ac7SEli Cohen case 4096: 273c7a08ac7SEli Cohen return 5; 274c7a08ac7SEli Cohen default: 275c7a08ac7SEli Cohen pr_warn("invalid pkey table size %d\n", size); 276c7a08ac7SEli Cohen return 0; 277c7a08ac7SEli Cohen } 278c7a08ac7SEli Cohen } 279c7a08ac7SEli Cohen 28087b8de49SEli Cohen /* selectively copy writable fields clearing any reserved area 28187b8de49SEli Cohen */ 282b775516bSEli Cohen static void copy_rw_fields(void *to, struct mlx5_caps *from) 28387b8de49SEli Cohen { 284b775516bSEli Cohen __be64 *flags_off = (__be64 *)MLX5_ADDR_OF(cmd_hca_cap, to, reserved_22); 28587b8de49SEli Cohen u64 v64; 28687b8de49SEli Cohen 287b775516bSEli Cohen MLX5_SET(cmd_hca_cap, to, log_max_qp, from->gen.log_max_qp); 288b775516bSEli Cohen MLX5_SET(cmd_hca_cap, to, log_max_ra_req_qp, from->gen.log_max_ra_req_qp); 289b775516bSEli Cohen MLX5_SET(cmd_hca_cap, to, log_max_ra_res_qp, from->gen.log_max_ra_res_qp); 290b775516bSEli Cohen MLX5_SET(cmd_hca_cap, to, pkey_table_size, from->gen.pkey_table_size); 291b775516bSEli Cohen MLX5_SET(cmd_hca_cap, to, log_max_ra_req_dc, from->gen.log_max_ra_req_dc); 292b775516bSEli Cohen MLX5_SET(cmd_hca_cap, to, log_max_ra_res_dc, from->gen.log_max_ra_res_dc); 293b775516bSEli Cohen MLX5_SET(cmd_hca_cap, to, pkey_table_size, to_fw_pkey_sz(from->gen.pkey_table_size)); 294b775516bSEli Cohen v64 = from->gen.flags & MLX5_CAP_BITS_RW_MASK; 295b775516bSEli Cohen *flags_off = cpu_to_be64(v64); 29687b8de49SEli Cohen } 29787b8de49SEli Cohen 298c7a08ac7SEli Cohen static u16 get_pkey_table_size(int pkey) 299c7a08ac7SEli Cohen { 300c7a08ac7SEli Cohen if (pkey > MLX5_MAX_LOG_PKEY_TABLE) 301c7a08ac7SEli Cohen return 0; 302c7a08ac7SEli Cohen 303c7a08ac7SEli Cohen return MLX5_MIN_PKEY_TABLE_SIZE << pkey; 304c7a08ac7SEli Cohen } 305c7a08ac7SEli Cohen 306b775516bSEli Cohen static void fw2drv_caps(struct mlx5_caps *caps, void *out) 307c7a08ac7SEli Cohen { 308c7a08ac7SEli Cohen struct mlx5_general_caps *gen = &caps->gen; 309c7a08ac7SEli Cohen 310b775516bSEli Cohen gen->max_srq_wqes = 1 << MLX5_GET_PR(cmd_hca_cap, out, log_max_srq_sz); 311b775516bSEli Cohen gen->max_wqes = 1 << MLX5_GET_PR(cmd_hca_cap, out, log_max_qp_sz); 312b775516bSEli Cohen gen->log_max_qp = MLX5_GET_PR(cmd_hca_cap, out, log_max_qp); 313b775516bSEli Cohen gen->log_max_strq = MLX5_GET_PR(cmd_hca_cap, out, log_max_strq_sz); 314b775516bSEli Cohen gen->log_max_srq = MLX5_GET_PR(cmd_hca_cap, out, log_max_srqs); 315b775516bSEli Cohen gen->max_cqes = 1 << MLX5_GET_PR(cmd_hca_cap, out, log_max_cq_sz); 316b775516bSEli Cohen gen->log_max_cq = MLX5_GET_PR(cmd_hca_cap, out, log_max_cq); 317b775516bSEli Cohen gen->max_eqes = 1 << MLX5_GET_PR(cmd_hca_cap, out, log_max_eq_sz); 318b775516bSEli Cohen gen->log_max_mkey = MLX5_GET_PR(cmd_hca_cap, out, log_max_mkey); 319b775516bSEli Cohen gen->log_max_eq = MLX5_GET_PR(cmd_hca_cap, out, log_max_eq); 320b775516bSEli Cohen gen->max_indirection = MLX5_GET_PR(cmd_hca_cap, out, max_indirection); 321b775516bSEli Cohen gen->log_max_mrw_sz = MLX5_GET_PR(cmd_hca_cap, out, log_max_mrw_sz); 322b775516bSEli Cohen gen->log_max_bsf_list_size = MLX5_GET_PR(cmd_hca_cap, out, log_max_bsf_list_size); 323b775516bSEli Cohen gen->log_max_klm_list_size = MLX5_GET_PR(cmd_hca_cap, out, log_max_klm_list_size); 324b775516bSEli Cohen gen->log_max_ra_req_dc = MLX5_GET_PR(cmd_hca_cap, out, log_max_ra_req_dc); 325b775516bSEli Cohen gen->log_max_ra_res_dc = MLX5_GET_PR(cmd_hca_cap, out, log_max_ra_res_dc); 326b775516bSEli Cohen gen->log_max_ra_req_qp = MLX5_GET_PR(cmd_hca_cap, out, log_max_ra_req_qp); 327b775516bSEli Cohen gen->log_max_ra_res_qp = MLX5_GET_PR(cmd_hca_cap, out, log_max_ra_res_qp); 328b775516bSEli Cohen gen->max_qp_counters = MLX5_GET_PR(cmd_hca_cap, out, max_qp_cnt); 329b775516bSEli Cohen gen->pkey_table_size = get_pkey_table_size(MLX5_GET_PR(cmd_hca_cap, out, pkey_table_size)); 330b775516bSEli Cohen gen->local_ca_ack_delay = MLX5_GET_PR(cmd_hca_cap, out, local_ca_ack_delay); 331b775516bSEli Cohen gen->num_ports = MLX5_GET_PR(cmd_hca_cap, out, num_ports); 332b775516bSEli Cohen gen->log_max_msg = MLX5_GET_PR(cmd_hca_cap, out, log_max_msg); 333b775516bSEli Cohen gen->stat_rate_support = MLX5_GET_PR(cmd_hca_cap, out, stat_rate_support); 334b775516bSEli Cohen gen->flags = be64_to_cpu(*(__be64 *)MLX5_ADDR_OF(cmd_hca_cap, out, reserved_22)); 335c7a08ac7SEli Cohen pr_debug("flags = 0x%llx\n", gen->flags); 336b775516bSEli Cohen gen->uar_sz = MLX5_GET_PR(cmd_hca_cap, out, uar_sz); 337b775516bSEli Cohen gen->min_log_pg_sz = MLX5_GET_PR(cmd_hca_cap, out, log_pg_sz); 338b775516bSEli Cohen gen->bf_reg_size = MLX5_GET_PR(cmd_hca_cap, out, bf); 339b775516bSEli Cohen gen->bf_reg_size = 1 << MLX5_GET_PR(cmd_hca_cap, out, log_bf_reg_size); 340b775516bSEli Cohen gen->max_sq_desc_sz = MLX5_GET_PR(cmd_hca_cap, out, max_wqe_sz_sq); 341b775516bSEli Cohen gen->max_rq_desc_sz = MLX5_GET_PR(cmd_hca_cap, out, max_wqe_sz_rq); 342b775516bSEli Cohen gen->max_dc_sq_desc_sz = MLX5_GET_PR(cmd_hca_cap, out, max_wqe_sz_sq_dc); 343b775516bSEli Cohen gen->max_qp_mcg = MLX5_GET_PR(cmd_hca_cap, out, max_qp_mcg); 344b775516bSEli Cohen gen->log_max_pd = MLX5_GET_PR(cmd_hca_cap, out, log_max_pd); 345b775516bSEli Cohen gen->log_max_xrcd = MLX5_GET_PR(cmd_hca_cap, out, log_max_xrcd); 346b775516bSEli Cohen gen->log_uar_page_sz = MLX5_GET_PR(cmd_hca_cap, out, log_uar_page_sz); 347c7a08ac7SEli Cohen } 348c7a08ac7SEli Cohen 349c7a08ac7SEli Cohen static const char *caps_opmod_str(u16 opmod) 350c7a08ac7SEli Cohen { 351c7a08ac7SEli Cohen switch (opmod) { 352c7a08ac7SEli Cohen case HCA_CAP_OPMOD_GET_MAX: 353c7a08ac7SEli Cohen return "GET_MAX"; 354c7a08ac7SEli Cohen case HCA_CAP_OPMOD_GET_CUR: 355c7a08ac7SEli Cohen return "GET_CUR"; 356c7a08ac7SEli Cohen default: 357c7a08ac7SEli Cohen return "Invalid"; 358c7a08ac7SEli Cohen } 359c7a08ac7SEli Cohen } 360c7a08ac7SEli Cohen 361c7a08ac7SEli Cohen int mlx5_core_get_caps(struct mlx5_core_dev *dev, struct mlx5_caps *caps, 362c7a08ac7SEli Cohen u16 opmod) 363c7a08ac7SEli Cohen { 364b775516bSEli Cohen u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 365b775516bSEli Cohen int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 366b775516bSEli Cohen void *out; 367c7a08ac7SEli Cohen int err; 368c7a08ac7SEli Cohen 369b775516bSEli Cohen memset(in, 0, sizeof(in)); 370b775516bSEli Cohen out = kzalloc(out_sz, GFP_KERNEL); 371c7a08ac7SEli Cohen if (!out) 372c7a08ac7SEli Cohen return -ENOMEM; 373b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 374b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 375b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 376b775516bSEli Cohen if (err) 377b775516bSEli Cohen goto query_ex; 378c7a08ac7SEli Cohen 379b775516bSEli Cohen err = mlx5_cmd_status_to_err_v2(out); 380c7a08ac7SEli Cohen if (err) { 381c7a08ac7SEli Cohen mlx5_core_warn(dev, "query max hca cap failed, %d\n", err); 382c7a08ac7SEli Cohen goto query_ex; 383c7a08ac7SEli Cohen } 384c7a08ac7SEli Cohen mlx5_core_dbg(dev, "%s\n", caps_opmod_str(opmod)); 385b775516bSEli Cohen fw2drv_caps(caps, MLX5_ADDR_OF(query_hca_cap_out, out, capability_struct)); 386c7a08ac7SEli Cohen 387c7a08ac7SEli Cohen query_ex: 388c7a08ac7SEli Cohen kfree(out); 389c7a08ac7SEli Cohen return err; 390c7a08ac7SEli Cohen } 391c7a08ac7SEli Cohen 392b775516bSEli Cohen static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz) 393c7a08ac7SEli Cohen { 394b775516bSEli Cohen u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)]; 395c7a08ac7SEli Cohen int err; 396c7a08ac7SEli Cohen 397b775516bSEli Cohen memset(out, 0, sizeof(out)); 398c7a08ac7SEli Cohen 399b775516bSEli Cohen MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 400b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 401c7a08ac7SEli Cohen if (err) 402c7a08ac7SEli Cohen return err; 403c7a08ac7SEli Cohen 404b775516bSEli Cohen err = mlx5_cmd_status_to_err_v2(out); 405c7a08ac7SEli Cohen 406c7a08ac7SEli Cohen return err; 407c7a08ac7SEli Cohen } 40887b8de49SEli Cohen 409e126ba97SEli Cohen static int handle_hca_cap(struct mlx5_core_dev *dev) 410e126ba97SEli Cohen { 411b775516bSEli Cohen void *set_ctx = NULL; 412c7a08ac7SEli Cohen struct mlx5_profile *prof = dev->profile; 413c7a08ac7SEli Cohen struct mlx5_caps *cur_caps = NULL; 414c7a08ac7SEli Cohen struct mlx5_caps *max_caps = NULL; 415c7a08ac7SEli Cohen int err = -ENOMEM; 416b775516bSEli Cohen int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 417e126ba97SEli Cohen 418b775516bSEli Cohen set_ctx = kzalloc(set_sz, GFP_KERNEL); 419c7a08ac7SEli Cohen if (!set_ctx) 420e126ba97SEli Cohen goto query_ex; 421e126ba97SEli Cohen 422c7a08ac7SEli Cohen max_caps = kzalloc(sizeof(*max_caps), GFP_KERNEL); 423c7a08ac7SEli Cohen if (!max_caps) 424c7a08ac7SEli Cohen goto query_ex; 425c7a08ac7SEli Cohen 426c7a08ac7SEli Cohen cur_caps = kzalloc(sizeof(*cur_caps), GFP_KERNEL); 427c7a08ac7SEli Cohen if (!cur_caps) 428c7a08ac7SEli Cohen goto query_ex; 429c7a08ac7SEli Cohen 430c7a08ac7SEli Cohen err = mlx5_core_get_caps(dev, max_caps, HCA_CAP_OPMOD_GET_MAX); 431e126ba97SEli Cohen if (err) 432e126ba97SEli Cohen goto query_ex; 433e126ba97SEli Cohen 434c7a08ac7SEli Cohen err = mlx5_core_get_caps(dev, cur_caps, HCA_CAP_OPMOD_GET_CUR); 435c7a08ac7SEli Cohen if (err) 436e126ba97SEli Cohen goto query_ex; 437e126ba97SEli Cohen 438c7a08ac7SEli Cohen /* we limit the size of the pkey table to 128 entries for now */ 439c7a08ac7SEli Cohen cur_caps->gen.pkey_table_size = 128; 440e126ba97SEli Cohen 441c7a08ac7SEli Cohen if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 442c7a08ac7SEli Cohen cur_caps->gen.log_max_qp = prof->log_max_qp; 443e126ba97SEli Cohen 444c1868b82SEli Cohen /* disable checksum */ 445c7a08ac7SEli Cohen cur_caps->gen.flags &= ~MLX5_DEV_CAP_FLAG_CMDIF_CSUM; 446c1868b82SEli Cohen 447b775516bSEli Cohen copy_rw_fields(MLX5_ADDR_OF(set_hca_cap_in, set_ctx, hca_capability_struct), 448b775516bSEli Cohen cur_caps); 449b775516bSEli Cohen err = set_caps(dev, set_ctx, set_sz); 450e126ba97SEli Cohen 451e126ba97SEli Cohen query_ex: 452c7a08ac7SEli Cohen kfree(cur_caps); 453c7a08ac7SEli Cohen kfree(max_caps); 454e126ba97SEli Cohen kfree(set_ctx); 455e126ba97SEli Cohen 456e126ba97SEli Cohen return err; 457e126ba97SEli Cohen } 458e126ba97SEli Cohen 459e126ba97SEli Cohen static int set_hca_ctrl(struct mlx5_core_dev *dev) 460e126ba97SEli Cohen { 461e126ba97SEli Cohen struct mlx5_reg_host_endianess he_in; 462e126ba97SEli Cohen struct mlx5_reg_host_endianess he_out; 463e126ba97SEli Cohen int err; 464e126ba97SEli Cohen 465e126ba97SEli Cohen memset(&he_in, 0, sizeof(he_in)); 466e126ba97SEli Cohen he_in.he = MLX5_SET_HOST_ENDIANNESS; 467e126ba97SEli Cohen err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 468e126ba97SEli Cohen &he_out, sizeof(he_out), 469e126ba97SEli Cohen MLX5_REG_HOST_ENDIANNESS, 0, 1); 470e126ba97SEli Cohen return err; 471e126ba97SEli Cohen } 472e126ba97SEli Cohen 473cd23b14bSEli Cohen static int mlx5_core_enable_hca(struct mlx5_core_dev *dev) 474cd23b14bSEli Cohen { 475cd23b14bSEli Cohen int err; 476cd23b14bSEli Cohen struct mlx5_enable_hca_mbox_in in; 477cd23b14bSEli Cohen struct mlx5_enable_hca_mbox_out out; 478cd23b14bSEli Cohen 479cd23b14bSEli Cohen memset(&in, 0, sizeof(in)); 480cd23b14bSEli Cohen memset(&out, 0, sizeof(out)); 481cd23b14bSEli Cohen in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ENABLE_HCA); 482cd23b14bSEli Cohen err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 483cd23b14bSEli Cohen if (err) 484cd23b14bSEli Cohen return err; 485cd23b14bSEli Cohen 486cd23b14bSEli Cohen if (out.hdr.status) 487cd23b14bSEli Cohen return mlx5_cmd_status_to_err(&out.hdr); 488cd23b14bSEli Cohen 489cd23b14bSEli Cohen return 0; 490cd23b14bSEli Cohen } 491cd23b14bSEli Cohen 492cd23b14bSEli Cohen static int mlx5_core_disable_hca(struct mlx5_core_dev *dev) 493cd23b14bSEli Cohen { 494cd23b14bSEli Cohen int err; 495cd23b14bSEli Cohen struct mlx5_disable_hca_mbox_in in; 496cd23b14bSEli Cohen struct mlx5_disable_hca_mbox_out out; 497cd23b14bSEli Cohen 498cd23b14bSEli Cohen memset(&in, 0, sizeof(in)); 499cd23b14bSEli Cohen memset(&out, 0, sizeof(out)); 500cd23b14bSEli Cohen in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DISABLE_HCA); 501cd23b14bSEli Cohen err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 502cd23b14bSEli Cohen if (err) 503cd23b14bSEli Cohen return err; 504cd23b14bSEli Cohen 505cd23b14bSEli Cohen if (out.hdr.status) 506cd23b14bSEli Cohen return mlx5_cmd_status_to_err(&out.hdr); 507cd23b14bSEli Cohen 508cd23b14bSEli Cohen return 0; 509cd23b14bSEli Cohen } 510cd23b14bSEli Cohen 5119603b61dSJack Morgenstein static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev) 512e126ba97SEli Cohen { 513e126ba97SEli Cohen struct mlx5_priv *priv = &dev->priv; 514e126ba97SEli Cohen int err; 515e126ba97SEli Cohen 516e126ba97SEli Cohen dev->pdev = pdev; 517e126ba97SEli Cohen pci_set_drvdata(dev->pdev, dev); 518e126ba97SEli Cohen strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 519e126ba97SEli Cohen priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 520e126ba97SEli Cohen 521e126ba97SEli Cohen mutex_init(&priv->pgdir_mutex); 522e126ba97SEli Cohen INIT_LIST_HEAD(&priv->pgdir_list); 523e126ba97SEli Cohen spin_lock_init(&priv->mkey_lock); 524e126ba97SEli Cohen 525e126ba97SEli Cohen priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root); 526e126ba97SEli Cohen if (!priv->dbg_root) 527e126ba97SEli Cohen return -ENOMEM; 528e126ba97SEli Cohen 529e126ba97SEli Cohen err = pci_enable_device(pdev); 530e126ba97SEli Cohen if (err) { 5311a91de28SJoe Perches dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 532e126ba97SEli Cohen goto err_dbg; 533e126ba97SEli Cohen } 534e126ba97SEli Cohen 535e126ba97SEli Cohen err = request_bar(pdev); 536e126ba97SEli Cohen if (err) { 5371a91de28SJoe Perches dev_err(&pdev->dev, "error requesting BARs, aborting\n"); 538e126ba97SEli Cohen goto err_disable; 539e126ba97SEli Cohen } 540e126ba97SEli Cohen 541e126ba97SEli Cohen pci_set_master(pdev); 542e126ba97SEli Cohen 543e126ba97SEli Cohen err = set_dma_caps(pdev); 544e126ba97SEli Cohen if (err) { 545e126ba97SEli Cohen dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); 546e126ba97SEli Cohen goto err_clr_master; 547e126ba97SEli Cohen } 548e126ba97SEli Cohen 549e126ba97SEli Cohen dev->iseg_base = pci_resource_start(dev->pdev, 0); 550e126ba97SEli Cohen dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 551e126ba97SEli Cohen if (!dev->iseg) { 552e126ba97SEli Cohen err = -ENOMEM; 553e126ba97SEli Cohen dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); 554e126ba97SEli Cohen goto err_clr_master; 555e126ba97SEli Cohen } 556e126ba97SEli Cohen dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 557e126ba97SEli Cohen fw_rev_min(dev), fw_rev_sub(dev)); 558e126ba97SEli Cohen 559e126ba97SEli Cohen err = mlx5_cmd_init(dev); 560e126ba97SEli Cohen if (err) { 561e126ba97SEli Cohen dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); 562e126ba97SEli Cohen goto err_unmap; 563e126ba97SEli Cohen } 564e126ba97SEli Cohen 565e126ba97SEli Cohen mlx5_pagealloc_init(dev); 566cd23b14bSEli Cohen 567cd23b14bSEli Cohen err = mlx5_core_enable_hca(dev); 568cd23b14bSEli Cohen if (err) { 569cd23b14bSEli Cohen dev_err(&pdev->dev, "enable hca failed\n"); 570cd23b14bSEli Cohen goto err_pagealloc_cleanup; 571cd23b14bSEli Cohen } 572cd23b14bSEli Cohen 573cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 1); 574cd23b14bSEli Cohen if (err) { 575cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate boot pages\n"); 576cd23b14bSEli Cohen goto err_disable_hca; 577cd23b14bSEli Cohen } 578cd23b14bSEli Cohen 579e126ba97SEli Cohen err = set_hca_ctrl(dev); 580e126ba97SEli Cohen if (err) { 581e126ba97SEli Cohen dev_err(&pdev->dev, "set_hca_ctrl failed\n"); 582cd23b14bSEli Cohen goto reclaim_boot_pages; 583e126ba97SEli Cohen } 584e126ba97SEli Cohen 585e126ba97SEli Cohen err = handle_hca_cap(dev); 586e126ba97SEli Cohen if (err) { 587e126ba97SEli Cohen dev_err(&pdev->dev, "handle_hca_cap failed\n"); 588cd23b14bSEli Cohen goto reclaim_boot_pages; 589e126ba97SEli Cohen } 590e126ba97SEli Cohen 591cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 0); 592e126ba97SEli Cohen if (err) { 593cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate init pages\n"); 594cd23b14bSEli Cohen goto reclaim_boot_pages; 595e126ba97SEli Cohen } 596e126ba97SEli Cohen 597e126ba97SEli Cohen err = mlx5_pagealloc_start(dev); 598e126ba97SEli Cohen if (err) { 599e126ba97SEli Cohen dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n"); 600cd23b14bSEli Cohen goto reclaim_boot_pages; 601e126ba97SEli Cohen } 602e126ba97SEli Cohen 603e126ba97SEli Cohen err = mlx5_cmd_init_hca(dev); 604e126ba97SEli Cohen if (err) { 605e126ba97SEli Cohen dev_err(&pdev->dev, "init hca failed\n"); 606e126ba97SEli Cohen goto err_pagealloc_stop; 607e126ba97SEli Cohen } 608e126ba97SEli Cohen 609e126ba97SEli Cohen mlx5_start_health_poll(dev); 610e126ba97SEli Cohen 611e126ba97SEli Cohen err = mlx5_cmd_query_hca_cap(dev, &dev->caps); 612e126ba97SEli Cohen if (err) { 613e126ba97SEli Cohen dev_err(&pdev->dev, "query hca failed\n"); 614e126ba97SEli Cohen goto err_stop_poll; 615e126ba97SEli Cohen } 616e126ba97SEli Cohen 617e126ba97SEli Cohen err = mlx5_cmd_query_adapter(dev); 618e126ba97SEli Cohen if (err) { 619e126ba97SEli Cohen dev_err(&pdev->dev, "query adapter failed\n"); 620e126ba97SEli Cohen goto err_stop_poll; 621e126ba97SEli Cohen } 622e126ba97SEli Cohen 623e126ba97SEli Cohen err = mlx5_enable_msix(dev); 624e126ba97SEli Cohen if (err) { 625e126ba97SEli Cohen dev_err(&pdev->dev, "enable msix failed\n"); 626e126ba97SEli Cohen goto err_stop_poll; 627e126ba97SEli Cohen } 628e126ba97SEli Cohen 629e126ba97SEli Cohen err = mlx5_eq_init(dev); 630e126ba97SEli Cohen if (err) { 631e126ba97SEli Cohen dev_err(&pdev->dev, "failed to initialize eq\n"); 632e126ba97SEli Cohen goto disable_msix; 633e126ba97SEli Cohen } 634e126ba97SEli Cohen 635e126ba97SEli Cohen err = mlx5_alloc_uuars(dev, &priv->uuari); 636e126ba97SEli Cohen if (err) { 637e126ba97SEli Cohen dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 638e126ba97SEli Cohen goto err_eq_cleanup; 639e126ba97SEli Cohen } 640e126ba97SEli Cohen 641e126ba97SEli Cohen err = mlx5_start_eqs(dev); 642e126ba97SEli Cohen if (err) { 643e126ba97SEli Cohen dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 644e126ba97SEli Cohen goto err_free_uar; 645e126ba97SEli Cohen } 646e126ba97SEli Cohen 647e126ba97SEli Cohen MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 648e126ba97SEli Cohen 649e126ba97SEli Cohen mlx5_init_cq_table(dev); 650e126ba97SEli Cohen mlx5_init_qp_table(dev); 651e126ba97SEli Cohen mlx5_init_srq_table(dev); 6523bcdb17aSSagi Grimberg mlx5_init_mr_table(dev); 653e126ba97SEli Cohen 654e126ba97SEli Cohen return 0; 655e126ba97SEli Cohen 656e126ba97SEli Cohen err_free_uar: 657e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 658e126ba97SEli Cohen 659e126ba97SEli Cohen err_eq_cleanup: 660e126ba97SEli Cohen mlx5_eq_cleanup(dev); 661e126ba97SEli Cohen 662e126ba97SEli Cohen disable_msix: 663e126ba97SEli Cohen mlx5_disable_msix(dev); 664e126ba97SEli Cohen 665e126ba97SEli Cohen err_stop_poll: 666e126ba97SEli Cohen mlx5_stop_health_poll(dev); 6671bde6e30SEli Cohen if (mlx5_cmd_teardown_hca(dev)) { 6681bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 6691bde6e30SEli Cohen return err; 6701bde6e30SEli Cohen } 671e126ba97SEli Cohen 672e126ba97SEli Cohen err_pagealloc_stop: 673e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 674e126ba97SEli Cohen 675cd23b14bSEli Cohen reclaim_boot_pages: 676e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 677e126ba97SEli Cohen 678cd23b14bSEli Cohen err_disable_hca: 679cd23b14bSEli Cohen mlx5_core_disable_hca(dev); 680cd23b14bSEli Cohen 681e126ba97SEli Cohen err_pagealloc_cleanup: 682e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 683e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 684e126ba97SEli Cohen 685e126ba97SEli Cohen err_unmap: 686e126ba97SEli Cohen iounmap(dev->iseg); 687e126ba97SEli Cohen 688e126ba97SEli Cohen err_clr_master: 689e126ba97SEli Cohen pci_clear_master(dev->pdev); 690e126ba97SEli Cohen release_bar(dev->pdev); 691e126ba97SEli Cohen 692e126ba97SEli Cohen err_disable: 693e126ba97SEli Cohen pci_disable_device(dev->pdev); 694e126ba97SEli Cohen 695e126ba97SEli Cohen err_dbg: 696e126ba97SEli Cohen debugfs_remove(priv->dbg_root); 697e126ba97SEli Cohen return err; 698e126ba97SEli Cohen } 699e126ba97SEli Cohen EXPORT_SYMBOL(mlx5_dev_init); 700e126ba97SEli Cohen 7019603b61dSJack Morgenstein static void mlx5_dev_cleanup(struct mlx5_core_dev *dev) 702e126ba97SEli Cohen { 703e126ba97SEli Cohen struct mlx5_priv *priv = &dev->priv; 704e126ba97SEli Cohen 705e126ba97SEli Cohen mlx5_cleanup_srq_table(dev); 706e126ba97SEli Cohen mlx5_cleanup_qp_table(dev); 707e126ba97SEli Cohen mlx5_cleanup_cq_table(dev); 708e126ba97SEli Cohen mlx5_stop_eqs(dev); 709e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 710e126ba97SEli Cohen mlx5_eq_cleanup(dev); 711e126ba97SEli Cohen mlx5_disable_msix(dev); 712e126ba97SEli Cohen mlx5_stop_health_poll(dev); 7131bde6e30SEli Cohen if (mlx5_cmd_teardown_hca(dev)) { 7141bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 7151bde6e30SEli Cohen return; 7161bde6e30SEli Cohen } 717e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 718e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 719cd23b14bSEli Cohen mlx5_core_disable_hca(dev); 720e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 721e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 722e126ba97SEli Cohen iounmap(dev->iseg); 723e126ba97SEli Cohen pci_clear_master(dev->pdev); 724e126ba97SEli Cohen release_bar(dev->pdev); 725e126ba97SEli Cohen pci_disable_device(dev->pdev); 726e126ba97SEli Cohen debugfs_remove(priv->dbg_root); 727e126ba97SEli Cohen } 7289603b61dSJack Morgenstein 7299603b61dSJack Morgenstein static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 7309603b61dSJack Morgenstein { 7319603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 7329603b61dSJack Morgenstein struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 7339603b61dSJack Morgenstein 7349603b61dSJack Morgenstein dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL); 7359603b61dSJack Morgenstein if (!dev_ctx) { 7369603b61dSJack Morgenstein pr_warn("mlx5_add_device: alloc context failed\n"); 7379603b61dSJack Morgenstein return; 7389603b61dSJack Morgenstein } 7399603b61dSJack Morgenstein 7409603b61dSJack Morgenstein dev_ctx->intf = intf; 7419603b61dSJack Morgenstein dev_ctx->context = intf->add(dev); 7429603b61dSJack Morgenstein 7439603b61dSJack Morgenstein if (dev_ctx->context) { 7449603b61dSJack Morgenstein spin_lock_irq(&priv->ctx_lock); 7459603b61dSJack Morgenstein list_add_tail(&dev_ctx->list, &priv->ctx_list); 7469603b61dSJack Morgenstein spin_unlock_irq(&priv->ctx_lock); 7479603b61dSJack Morgenstein } else { 7489603b61dSJack Morgenstein kfree(dev_ctx); 7499603b61dSJack Morgenstein } 7509603b61dSJack Morgenstein } 7519603b61dSJack Morgenstein 7529603b61dSJack Morgenstein static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 7539603b61dSJack Morgenstein { 7549603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 7559603b61dSJack Morgenstein struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 7569603b61dSJack Morgenstein 7579603b61dSJack Morgenstein list_for_each_entry(dev_ctx, &priv->ctx_list, list) 7589603b61dSJack Morgenstein if (dev_ctx->intf == intf) { 7599603b61dSJack Morgenstein spin_lock_irq(&priv->ctx_lock); 7609603b61dSJack Morgenstein list_del(&dev_ctx->list); 7619603b61dSJack Morgenstein spin_unlock_irq(&priv->ctx_lock); 7629603b61dSJack Morgenstein 7639603b61dSJack Morgenstein intf->remove(dev, dev_ctx->context); 7649603b61dSJack Morgenstein kfree(dev_ctx); 7659603b61dSJack Morgenstein return; 7669603b61dSJack Morgenstein } 7679603b61dSJack Morgenstein } 7689603b61dSJack Morgenstein static int mlx5_register_device(struct mlx5_core_dev *dev) 7699603b61dSJack Morgenstein { 7709603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 7719603b61dSJack Morgenstein struct mlx5_interface *intf; 7729603b61dSJack Morgenstein 7739603b61dSJack Morgenstein mutex_lock(&intf_mutex); 7749603b61dSJack Morgenstein list_add_tail(&priv->dev_list, &dev_list); 7759603b61dSJack Morgenstein list_for_each_entry(intf, &intf_list, list) 7769603b61dSJack Morgenstein mlx5_add_device(intf, priv); 7779603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 7789603b61dSJack Morgenstein 7799603b61dSJack Morgenstein return 0; 7809603b61dSJack Morgenstein } 7819603b61dSJack Morgenstein static void mlx5_unregister_device(struct mlx5_core_dev *dev) 7829603b61dSJack Morgenstein { 7839603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 7849603b61dSJack Morgenstein struct mlx5_interface *intf; 7859603b61dSJack Morgenstein 7869603b61dSJack Morgenstein mutex_lock(&intf_mutex); 7879603b61dSJack Morgenstein list_for_each_entry(intf, &intf_list, list) 7889603b61dSJack Morgenstein mlx5_remove_device(intf, priv); 7899603b61dSJack Morgenstein list_del(&priv->dev_list); 7909603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 7919603b61dSJack Morgenstein } 7929603b61dSJack Morgenstein 7939603b61dSJack Morgenstein int mlx5_register_interface(struct mlx5_interface *intf) 7949603b61dSJack Morgenstein { 7959603b61dSJack Morgenstein struct mlx5_priv *priv; 7969603b61dSJack Morgenstein 7979603b61dSJack Morgenstein if (!intf->add || !intf->remove) 7989603b61dSJack Morgenstein return -EINVAL; 7999603b61dSJack Morgenstein 8009603b61dSJack Morgenstein mutex_lock(&intf_mutex); 8019603b61dSJack Morgenstein list_add_tail(&intf->list, &intf_list); 8029603b61dSJack Morgenstein list_for_each_entry(priv, &dev_list, dev_list) 8039603b61dSJack Morgenstein mlx5_add_device(intf, priv); 8049603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 8059603b61dSJack Morgenstein 8069603b61dSJack Morgenstein return 0; 8079603b61dSJack Morgenstein } 8089603b61dSJack Morgenstein EXPORT_SYMBOL(mlx5_register_interface); 8099603b61dSJack Morgenstein 8109603b61dSJack Morgenstein void mlx5_unregister_interface(struct mlx5_interface *intf) 8119603b61dSJack Morgenstein { 8129603b61dSJack Morgenstein struct mlx5_priv *priv; 8139603b61dSJack Morgenstein 8149603b61dSJack Morgenstein mutex_lock(&intf_mutex); 8159603b61dSJack Morgenstein list_for_each_entry(priv, &dev_list, dev_list) 8169603b61dSJack Morgenstein mlx5_remove_device(intf, priv); 8179603b61dSJack Morgenstein list_del(&intf->list); 8189603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 8199603b61dSJack Morgenstein } 8209603b61dSJack Morgenstein EXPORT_SYMBOL(mlx5_unregister_interface); 8219603b61dSJack Morgenstein 8229603b61dSJack Morgenstein static void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 8234d2f9bbbSJack Morgenstein unsigned long param) 8249603b61dSJack Morgenstein { 8259603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 8269603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 8279603b61dSJack Morgenstein unsigned long flags; 8289603b61dSJack Morgenstein 8299603b61dSJack Morgenstein spin_lock_irqsave(&priv->ctx_lock, flags); 8309603b61dSJack Morgenstein 8319603b61dSJack Morgenstein list_for_each_entry(dev_ctx, &priv->ctx_list, list) 8329603b61dSJack Morgenstein if (dev_ctx->intf->event) 8334d2f9bbbSJack Morgenstein dev_ctx->intf->event(dev, dev_ctx->context, event, param); 8349603b61dSJack Morgenstein 8359603b61dSJack Morgenstein spin_unlock_irqrestore(&priv->ctx_lock, flags); 8369603b61dSJack Morgenstein } 8379603b61dSJack Morgenstein 8389603b61dSJack Morgenstein struct mlx5_core_event_handler { 8399603b61dSJack Morgenstein void (*event)(struct mlx5_core_dev *dev, 8409603b61dSJack Morgenstein enum mlx5_dev_event event, 8419603b61dSJack Morgenstein void *data); 8429603b61dSJack Morgenstein }; 8439603b61dSJack Morgenstein 844f66f049fSEli Cohen #define MLX5_IB_MOD "mlx5_ib" 845f66f049fSEli Cohen 8469603b61dSJack Morgenstein static int init_one(struct pci_dev *pdev, 8479603b61dSJack Morgenstein const struct pci_device_id *id) 8489603b61dSJack Morgenstein { 8499603b61dSJack Morgenstein struct mlx5_core_dev *dev; 8509603b61dSJack Morgenstein struct mlx5_priv *priv; 8519603b61dSJack Morgenstein int err; 8529603b61dSJack Morgenstein 8539603b61dSJack Morgenstein dev = kzalloc(sizeof(*dev), GFP_KERNEL); 8549603b61dSJack Morgenstein if (!dev) { 8559603b61dSJack Morgenstein dev_err(&pdev->dev, "kzalloc failed\n"); 8569603b61dSJack Morgenstein return -ENOMEM; 8579603b61dSJack Morgenstein } 8589603b61dSJack Morgenstein priv = &dev->priv; 8599603b61dSJack Morgenstein 8609603b61dSJack Morgenstein pci_set_drvdata(pdev, dev); 8619603b61dSJack Morgenstein 8629603b61dSJack Morgenstein if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profile)) { 8639603b61dSJack Morgenstein pr_warn("selected profile out of range, selecting default (%d)\n", 8649603b61dSJack Morgenstein MLX5_DEFAULT_PROF); 8659603b61dSJack Morgenstein prof_sel = MLX5_DEFAULT_PROF; 8669603b61dSJack Morgenstein } 8679603b61dSJack Morgenstein dev->profile = &profile[prof_sel]; 8689603b61dSJack Morgenstein dev->event = mlx5_core_event; 8699603b61dSJack Morgenstein 870364d1798SEli Cohen INIT_LIST_HEAD(&priv->ctx_list); 871364d1798SEli Cohen spin_lock_init(&priv->ctx_lock); 8729603b61dSJack Morgenstein err = mlx5_dev_init(dev, pdev); 8739603b61dSJack Morgenstein if (err) { 8749603b61dSJack Morgenstein dev_err(&pdev->dev, "mlx5_dev_init failed %d\n", err); 8759603b61dSJack Morgenstein goto out; 8769603b61dSJack Morgenstein } 8779603b61dSJack Morgenstein 8789603b61dSJack Morgenstein err = mlx5_register_device(dev); 8799603b61dSJack Morgenstein if (err) { 8809603b61dSJack Morgenstein dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 8819603b61dSJack Morgenstein goto out_init; 8829603b61dSJack Morgenstein } 8839603b61dSJack Morgenstein 884f66f049fSEli Cohen err = request_module_nowait(MLX5_IB_MOD); 885f66f049fSEli Cohen if (err) 886f66f049fSEli Cohen pr_info("failed request module on %s\n", MLX5_IB_MOD); 887f66f049fSEli Cohen 8889603b61dSJack Morgenstein return 0; 8899603b61dSJack Morgenstein 8909603b61dSJack Morgenstein out_init: 8919603b61dSJack Morgenstein mlx5_dev_cleanup(dev); 8929603b61dSJack Morgenstein out: 8939603b61dSJack Morgenstein kfree(dev); 8949603b61dSJack Morgenstein return err; 8959603b61dSJack Morgenstein } 8969603b61dSJack Morgenstein static void remove_one(struct pci_dev *pdev) 8979603b61dSJack Morgenstein { 8989603b61dSJack Morgenstein struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 8999603b61dSJack Morgenstein 9009603b61dSJack Morgenstein mlx5_unregister_device(dev); 9019603b61dSJack Morgenstein mlx5_dev_cleanup(dev); 9029603b61dSJack Morgenstein kfree(dev); 9039603b61dSJack Morgenstein } 9049603b61dSJack Morgenstein 9059603b61dSJack Morgenstein static const struct pci_device_id mlx5_core_pci_table[] = { 9061c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */ 9071c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1012) }, /* Connect-IB VF */ 9081c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */ 9091c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1014) }, /* ConnectX-4 VF */ 9101c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */ 9111c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1016) }, /* ConnectX-4LX VF */ 9129603b61dSJack Morgenstein { 0, } 9139603b61dSJack Morgenstein }; 9149603b61dSJack Morgenstein 9159603b61dSJack Morgenstein MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 9169603b61dSJack Morgenstein 9179603b61dSJack Morgenstein static struct pci_driver mlx5_core_driver = { 9189603b61dSJack Morgenstein .name = DRIVER_NAME, 9199603b61dSJack Morgenstein .id_table = mlx5_core_pci_table, 9209603b61dSJack Morgenstein .probe = init_one, 9219603b61dSJack Morgenstein .remove = remove_one 9229603b61dSJack Morgenstein }; 923e126ba97SEli Cohen 924e126ba97SEli Cohen static int __init init(void) 925e126ba97SEli Cohen { 926e126ba97SEli Cohen int err; 927e126ba97SEli Cohen 928e126ba97SEli Cohen mlx5_register_debugfs(); 929e126ba97SEli Cohen mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq"); 930e126ba97SEli Cohen if (!mlx5_core_wq) { 931e126ba97SEli Cohen err = -ENOMEM; 932e126ba97SEli Cohen goto err_debug; 933e126ba97SEli Cohen } 934e126ba97SEli Cohen mlx5_health_init(); 935e126ba97SEli Cohen 9369603b61dSJack Morgenstein err = pci_register_driver(&mlx5_core_driver); 9379603b61dSJack Morgenstein if (err) 9389603b61dSJack Morgenstein goto err_health; 9399603b61dSJack Morgenstein 940e126ba97SEli Cohen return 0; 941e126ba97SEli Cohen 9429603b61dSJack Morgenstein err_health: 9439603b61dSJack Morgenstein mlx5_health_cleanup(); 9449603b61dSJack Morgenstein destroy_workqueue(mlx5_core_wq); 945e126ba97SEli Cohen err_debug: 946e126ba97SEli Cohen mlx5_unregister_debugfs(); 947e126ba97SEli Cohen return err; 948e126ba97SEli Cohen } 949e126ba97SEli Cohen 950e126ba97SEli Cohen static void __exit cleanup(void) 951e126ba97SEli Cohen { 9529603b61dSJack Morgenstein pci_unregister_driver(&mlx5_core_driver); 953e126ba97SEli Cohen mlx5_health_cleanup(); 954e126ba97SEli Cohen destroy_workqueue(mlx5_core_wq); 955e126ba97SEli Cohen mlx5_unregister_debugfs(); 956e126ba97SEli Cohen } 957e126ba97SEli Cohen 958e126ba97SEli Cohen module_init(init); 959e126ba97SEli Cohen module_exit(cleanup); 960