1e126ba97SEli Cohen /* 2302bdf68SSaeed Mahameed * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3e126ba97SEli Cohen * 4e126ba97SEli Cohen * This software is available to you under a choice of one of two 5e126ba97SEli Cohen * licenses. You may choose to be licensed under the terms of the GNU 6e126ba97SEli Cohen * General Public License (GPL) Version 2, available from the file 7e126ba97SEli Cohen * COPYING in the main directory of this source tree, or the 8e126ba97SEli Cohen * OpenIB.org BSD license below: 9e126ba97SEli Cohen * 10e126ba97SEli Cohen * Redistribution and use in source and binary forms, with or 11e126ba97SEli Cohen * without modification, are permitted provided that the following 12e126ba97SEli Cohen * conditions are met: 13e126ba97SEli Cohen * 14e126ba97SEli Cohen * - Redistributions of source code must retain the above 15e126ba97SEli Cohen * copyright notice, this list of conditions and the following 16e126ba97SEli Cohen * disclaimer. 17e126ba97SEli Cohen * 18e126ba97SEli Cohen * - Redistributions in binary form must reproduce the above 19e126ba97SEli Cohen * copyright notice, this list of conditions and the following 20e126ba97SEli Cohen * disclaimer in the documentation and/or other materials 21e126ba97SEli Cohen * provided with the distribution. 22e126ba97SEli Cohen * 23e126ba97SEli Cohen * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e126ba97SEli Cohen * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e126ba97SEli Cohen * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e126ba97SEli Cohen * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e126ba97SEli Cohen * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e126ba97SEli Cohen * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e126ba97SEli Cohen * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e126ba97SEli Cohen * SOFTWARE. 31e126ba97SEli Cohen */ 32e126ba97SEli Cohen 33e126ba97SEli Cohen #include <asm-generic/kmap_types.h> 34e126ba97SEli Cohen #include <linux/module.h> 35e126ba97SEli Cohen #include <linux/init.h> 36e126ba97SEli Cohen #include <linux/errno.h> 37e126ba97SEli Cohen #include <linux/pci.h> 38e126ba97SEli Cohen #include <linux/dma-mapping.h> 39e126ba97SEli Cohen #include <linux/slab.h> 40e126ba97SEli Cohen #include <linux/io-mapping.h> 41db058a18SSaeed Mahameed #include <linux/interrupt.h> 42e126ba97SEli Cohen #include <linux/mlx5/driver.h> 43e126ba97SEli Cohen #include <linux/mlx5/cq.h> 44e126ba97SEli Cohen #include <linux/mlx5/qp.h> 45e126ba97SEli Cohen #include <linux/mlx5/srq.h> 46e126ba97SEli Cohen #include <linux/debugfs.h> 47f66f049fSEli Cohen #include <linux/kmod.h> 48b775516bSEli Cohen #include <linux/mlx5/mlx5_ifc.h> 49e126ba97SEli Cohen #include "mlx5_core.h" 50e126ba97SEli Cohen 51e126ba97SEli Cohen MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 524ae6c18cSAchiad Shochat MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 53e126ba97SEli Cohen MODULE_LICENSE("Dual BSD/GPL"); 54e126ba97SEli Cohen MODULE_VERSION(DRIVER_VERSION); 55e126ba97SEli Cohen 56e126ba97SEli Cohen int mlx5_core_debug_mask; 57e126ba97SEli Cohen module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 58e126ba97SEli Cohen MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 59e126ba97SEli Cohen 609603b61dSJack Morgenstein #define MLX5_DEFAULT_PROF 2 619603b61dSJack Morgenstein static int prof_sel = MLX5_DEFAULT_PROF; 629603b61dSJack Morgenstein module_param_named(prof_sel, prof_sel, int, 0444); 639603b61dSJack Morgenstein MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 649603b61dSJack Morgenstein 65e126ba97SEli Cohen struct workqueue_struct *mlx5_core_wq; 669603b61dSJack Morgenstein static LIST_HEAD(intf_list); 679603b61dSJack Morgenstein static LIST_HEAD(dev_list); 689603b61dSJack Morgenstein static DEFINE_MUTEX(intf_mutex); 699603b61dSJack Morgenstein 709603b61dSJack Morgenstein struct mlx5_device_context { 719603b61dSJack Morgenstein struct list_head list; 729603b61dSJack Morgenstein struct mlx5_interface *intf; 739603b61dSJack Morgenstein void *context; 749603b61dSJack Morgenstein }; 759603b61dSJack Morgenstein 769603b61dSJack Morgenstein static struct mlx5_profile profile[] = { 779603b61dSJack Morgenstein [0] = { 789603b61dSJack Morgenstein .mask = 0, 799603b61dSJack Morgenstein }, 809603b61dSJack Morgenstein [1] = { 819603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE, 829603b61dSJack Morgenstein .log_max_qp = 12, 839603b61dSJack Morgenstein }, 849603b61dSJack Morgenstein [2] = { 859603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE | 869603b61dSJack Morgenstein MLX5_PROF_MASK_MR_CACHE, 879603b61dSJack Morgenstein .log_max_qp = 17, 889603b61dSJack Morgenstein .mr_cache[0] = { 899603b61dSJack Morgenstein .size = 500, 909603b61dSJack Morgenstein .limit = 250 919603b61dSJack Morgenstein }, 929603b61dSJack Morgenstein .mr_cache[1] = { 939603b61dSJack Morgenstein .size = 500, 949603b61dSJack Morgenstein .limit = 250 959603b61dSJack Morgenstein }, 969603b61dSJack Morgenstein .mr_cache[2] = { 979603b61dSJack Morgenstein .size = 500, 989603b61dSJack Morgenstein .limit = 250 999603b61dSJack Morgenstein }, 1009603b61dSJack Morgenstein .mr_cache[3] = { 1019603b61dSJack Morgenstein .size = 500, 1029603b61dSJack Morgenstein .limit = 250 1039603b61dSJack Morgenstein }, 1049603b61dSJack Morgenstein .mr_cache[4] = { 1059603b61dSJack Morgenstein .size = 500, 1069603b61dSJack Morgenstein .limit = 250 1079603b61dSJack Morgenstein }, 1089603b61dSJack Morgenstein .mr_cache[5] = { 1099603b61dSJack Morgenstein .size = 500, 1109603b61dSJack Morgenstein .limit = 250 1119603b61dSJack Morgenstein }, 1129603b61dSJack Morgenstein .mr_cache[6] = { 1139603b61dSJack Morgenstein .size = 500, 1149603b61dSJack Morgenstein .limit = 250 1159603b61dSJack Morgenstein }, 1169603b61dSJack Morgenstein .mr_cache[7] = { 1179603b61dSJack Morgenstein .size = 500, 1189603b61dSJack Morgenstein .limit = 250 1199603b61dSJack Morgenstein }, 1209603b61dSJack Morgenstein .mr_cache[8] = { 1219603b61dSJack Morgenstein .size = 500, 1229603b61dSJack Morgenstein .limit = 250 1239603b61dSJack Morgenstein }, 1249603b61dSJack Morgenstein .mr_cache[9] = { 1259603b61dSJack Morgenstein .size = 500, 1269603b61dSJack Morgenstein .limit = 250 1279603b61dSJack Morgenstein }, 1289603b61dSJack Morgenstein .mr_cache[10] = { 1299603b61dSJack Morgenstein .size = 500, 1309603b61dSJack Morgenstein .limit = 250 1319603b61dSJack Morgenstein }, 1329603b61dSJack Morgenstein .mr_cache[11] = { 1339603b61dSJack Morgenstein .size = 500, 1349603b61dSJack Morgenstein .limit = 250 1359603b61dSJack Morgenstein }, 1369603b61dSJack Morgenstein .mr_cache[12] = { 1379603b61dSJack Morgenstein .size = 64, 1389603b61dSJack Morgenstein .limit = 32 1399603b61dSJack Morgenstein }, 1409603b61dSJack Morgenstein .mr_cache[13] = { 1419603b61dSJack Morgenstein .size = 32, 1429603b61dSJack Morgenstein .limit = 16 1439603b61dSJack Morgenstein }, 1449603b61dSJack Morgenstein .mr_cache[14] = { 1459603b61dSJack Morgenstein .size = 16, 1469603b61dSJack Morgenstein .limit = 8 1479603b61dSJack Morgenstein }, 1489603b61dSJack Morgenstein .mr_cache[15] = { 1499603b61dSJack Morgenstein .size = 8, 1509603b61dSJack Morgenstein .limit = 4 1519603b61dSJack Morgenstein }, 1529603b61dSJack Morgenstein }, 1539603b61dSJack Morgenstein }; 154e126ba97SEli Cohen 155e126ba97SEli Cohen static int set_dma_caps(struct pci_dev *pdev) 156e126ba97SEli Cohen { 157e126ba97SEli Cohen int err; 158e126ba97SEli Cohen 159e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 160e126ba97SEli Cohen if (err) { 1611a91de28SJoe Perches dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 162e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 163e126ba97SEli Cohen if (err) { 1641a91de28SJoe Perches dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 165e126ba97SEli Cohen return err; 166e126ba97SEli Cohen } 167e126ba97SEli Cohen } 168e126ba97SEli Cohen 169e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 170e126ba97SEli Cohen if (err) { 171e126ba97SEli Cohen dev_warn(&pdev->dev, 1721a91de28SJoe Perches "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 173e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 174e126ba97SEli Cohen if (err) { 175e126ba97SEli Cohen dev_err(&pdev->dev, 1761a91de28SJoe Perches "Can't set consistent PCI DMA mask, aborting\n"); 177e126ba97SEli Cohen return err; 178e126ba97SEli Cohen } 179e126ba97SEli Cohen } 180e126ba97SEli Cohen 181e126ba97SEli Cohen dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 182e126ba97SEli Cohen return err; 183e126ba97SEli Cohen } 184e126ba97SEli Cohen 185e126ba97SEli Cohen static int request_bar(struct pci_dev *pdev) 186e126ba97SEli Cohen { 187e126ba97SEli Cohen int err = 0; 188e126ba97SEli Cohen 189e126ba97SEli Cohen if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 1901a91de28SJoe Perches dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 191e126ba97SEli Cohen return -ENODEV; 192e126ba97SEli Cohen } 193e126ba97SEli Cohen 194e126ba97SEli Cohen err = pci_request_regions(pdev, DRIVER_NAME); 195e126ba97SEli Cohen if (err) 196e126ba97SEli Cohen dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 197e126ba97SEli Cohen 198e126ba97SEli Cohen return err; 199e126ba97SEli Cohen } 200e126ba97SEli Cohen 201e126ba97SEli Cohen static void release_bar(struct pci_dev *pdev) 202e126ba97SEli Cohen { 203e126ba97SEli Cohen pci_release_regions(pdev); 204e126ba97SEli Cohen } 205e126ba97SEli Cohen 206e126ba97SEli Cohen static int mlx5_enable_msix(struct mlx5_core_dev *dev) 207e126ba97SEli Cohen { 208db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 209db058a18SSaeed Mahameed struct mlx5_eq_table *table = &priv->eq_table; 210938fe83cSSaeed Mahameed int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 211e126ba97SEli Cohen int nvec; 212e126ba97SEli Cohen int i; 213e126ba97SEli Cohen 214938fe83cSSaeed Mahameed nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 215938fe83cSSaeed Mahameed MLX5_EQ_VEC_COMP_BASE; 216e126ba97SEli Cohen nvec = min_t(int, nvec, num_eqs); 217e126ba97SEli Cohen if (nvec <= MLX5_EQ_VEC_COMP_BASE) 218e126ba97SEli Cohen return -ENOMEM; 219e126ba97SEli Cohen 220db058a18SSaeed Mahameed priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL); 221db058a18SSaeed Mahameed 222db058a18SSaeed Mahameed priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL); 223db058a18SSaeed Mahameed if (!priv->msix_arr || !priv->irq_info) 224db058a18SSaeed Mahameed goto err_free_msix; 225e126ba97SEli Cohen 226e126ba97SEli Cohen for (i = 0; i < nvec; i++) 227db058a18SSaeed Mahameed priv->msix_arr[i].entry = i; 228e126ba97SEli Cohen 229db058a18SSaeed Mahameed nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 2303a9e161aSEli Cohen MLX5_EQ_VEC_COMP_BASE + 1, nvec); 231f3c9407bSAlexander Gordeev if (nvec < 0) 232f3c9407bSAlexander Gordeev return nvec; 233e126ba97SEli Cohen 234f3c9407bSAlexander Gordeev table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 235e126ba97SEli Cohen 236e126ba97SEli Cohen return 0; 237db058a18SSaeed Mahameed 238db058a18SSaeed Mahameed err_free_msix: 239db058a18SSaeed Mahameed kfree(priv->irq_info); 240db058a18SSaeed Mahameed kfree(priv->msix_arr); 241db058a18SSaeed Mahameed return -ENOMEM; 242e126ba97SEli Cohen } 243e126ba97SEli Cohen 244e126ba97SEli Cohen static void mlx5_disable_msix(struct mlx5_core_dev *dev) 245e126ba97SEli Cohen { 246db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 247e126ba97SEli Cohen 248e126ba97SEli Cohen pci_disable_msix(dev->pdev); 249db058a18SSaeed Mahameed kfree(priv->irq_info); 250db058a18SSaeed Mahameed kfree(priv->msix_arr); 251e126ba97SEli Cohen } 252e126ba97SEli Cohen 253e126ba97SEli Cohen struct mlx5_reg_host_endianess { 254e126ba97SEli Cohen u8 he; 255e126ba97SEli Cohen u8 rsvd[15]; 256e126ba97SEli Cohen }; 257e126ba97SEli Cohen 25887b8de49SEli Cohen 25987b8de49SEli Cohen #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 26087b8de49SEli Cohen 26187b8de49SEli Cohen enum { 26287b8de49SEli Cohen MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 263c7a08ac7SEli Cohen MLX5_DEV_CAP_FLAG_DCT, 26487b8de49SEli Cohen }; 26587b8de49SEli Cohen 266c7a08ac7SEli Cohen static u16 to_fw_pkey_sz(u32 size) 267c7a08ac7SEli Cohen { 268c7a08ac7SEli Cohen switch (size) { 269c7a08ac7SEli Cohen case 128: 270c7a08ac7SEli Cohen return 0; 271c7a08ac7SEli Cohen case 256: 272c7a08ac7SEli Cohen return 1; 273c7a08ac7SEli Cohen case 512: 274c7a08ac7SEli Cohen return 2; 275c7a08ac7SEli Cohen case 1024: 276c7a08ac7SEli Cohen return 3; 277c7a08ac7SEli Cohen case 2048: 278c7a08ac7SEli Cohen return 4; 279c7a08ac7SEli Cohen case 4096: 280c7a08ac7SEli Cohen return 5; 281c7a08ac7SEli Cohen default: 282c7a08ac7SEli Cohen pr_warn("invalid pkey table size %d\n", size); 283c7a08ac7SEli Cohen return 0; 284c7a08ac7SEli Cohen } 285c7a08ac7SEli Cohen } 286c7a08ac7SEli Cohen 287938fe83cSSaeed Mahameed static u16 to_sw_pkey_sz(int pkey_sz) 28887b8de49SEli Cohen { 289938fe83cSSaeed Mahameed if (pkey_sz > MLX5_MAX_LOG_PKEY_TABLE) 290c7a08ac7SEli Cohen return 0; 291c7a08ac7SEli Cohen 292938fe83cSSaeed Mahameed return MLX5_MIN_PKEY_TABLE_SIZE << pkey_sz; 293c7a08ac7SEli Cohen } 294c7a08ac7SEli Cohen 295938fe83cSSaeed Mahameed int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, 296938fe83cSSaeed Mahameed enum mlx5_cap_mode cap_mode) 297c7a08ac7SEli Cohen { 298b775516bSEli Cohen u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 299b775516bSEli Cohen int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 300938fe83cSSaeed Mahameed void *out, *hca_caps; 301938fe83cSSaeed Mahameed u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 302c7a08ac7SEli Cohen int err; 303c7a08ac7SEli Cohen 304b775516bSEli Cohen memset(in, 0, sizeof(in)); 305b775516bSEli Cohen out = kzalloc(out_sz, GFP_KERNEL); 306c7a08ac7SEli Cohen if (!out) 307c7a08ac7SEli Cohen return -ENOMEM; 308938fe83cSSaeed Mahameed 309b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 310b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 311b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 312b775516bSEli Cohen if (err) 313b775516bSEli Cohen goto query_ex; 314c7a08ac7SEli Cohen 315b775516bSEli Cohen err = mlx5_cmd_status_to_err_v2(out); 316c7a08ac7SEli Cohen if (err) { 317938fe83cSSaeed Mahameed mlx5_core_warn(dev, 318938fe83cSSaeed Mahameed "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 319938fe83cSSaeed Mahameed cap_type, cap_mode, err); 320c7a08ac7SEli Cohen goto query_ex; 321c7a08ac7SEli Cohen } 322c7a08ac7SEli Cohen 323938fe83cSSaeed Mahameed hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 324938fe83cSSaeed Mahameed 325938fe83cSSaeed Mahameed switch (cap_mode) { 326938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_MAX: 327938fe83cSSaeed Mahameed memcpy(dev->hca_caps_max[cap_type], hca_caps, 328938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 329938fe83cSSaeed Mahameed break; 330938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_CUR: 331938fe83cSSaeed Mahameed memcpy(dev->hca_caps_cur[cap_type], hca_caps, 332938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 333938fe83cSSaeed Mahameed break; 334938fe83cSSaeed Mahameed default: 335938fe83cSSaeed Mahameed mlx5_core_warn(dev, 336938fe83cSSaeed Mahameed "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 337938fe83cSSaeed Mahameed cap_type, cap_mode); 338938fe83cSSaeed Mahameed err = -EINVAL; 339938fe83cSSaeed Mahameed break; 340938fe83cSSaeed Mahameed } 341c7a08ac7SEli Cohen query_ex: 342c7a08ac7SEli Cohen kfree(out); 343c7a08ac7SEli Cohen return err; 344c7a08ac7SEli Cohen } 345c7a08ac7SEli Cohen 346b775516bSEli Cohen static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz) 347c7a08ac7SEli Cohen { 348b775516bSEli Cohen u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)]; 349c7a08ac7SEli Cohen int err; 350c7a08ac7SEli Cohen 351b775516bSEli Cohen memset(out, 0, sizeof(out)); 352c7a08ac7SEli Cohen 353b775516bSEli Cohen MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 354b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 355c7a08ac7SEli Cohen if (err) 356c7a08ac7SEli Cohen return err; 357c7a08ac7SEli Cohen 358b775516bSEli Cohen err = mlx5_cmd_status_to_err_v2(out); 359c7a08ac7SEli Cohen 360c7a08ac7SEli Cohen return err; 361c7a08ac7SEli Cohen } 36287b8de49SEli Cohen 363e126ba97SEli Cohen static int handle_hca_cap(struct mlx5_core_dev *dev) 364e126ba97SEli Cohen { 365b775516bSEli Cohen void *set_ctx = NULL; 366c7a08ac7SEli Cohen struct mlx5_profile *prof = dev->profile; 367c7a08ac7SEli Cohen int err = -ENOMEM; 368b775516bSEli Cohen int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 369938fe83cSSaeed Mahameed void *set_hca_cap; 370e126ba97SEli Cohen 371b775516bSEli Cohen set_ctx = kzalloc(set_sz, GFP_KERNEL); 372c7a08ac7SEli Cohen if (!set_ctx) 373e126ba97SEli Cohen goto query_ex; 374e126ba97SEli Cohen 375938fe83cSSaeed Mahameed err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX); 376e126ba97SEli Cohen if (err) 377e126ba97SEli Cohen goto query_ex; 378e126ba97SEli Cohen 379938fe83cSSaeed Mahameed err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR); 380c7a08ac7SEli Cohen if (err) 381e126ba97SEli Cohen goto query_ex; 382e126ba97SEli Cohen 383938fe83cSSaeed Mahameed set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 384938fe83cSSaeed Mahameed capability); 385938fe83cSSaeed Mahameed memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], 386938fe83cSSaeed Mahameed MLX5_ST_SZ_BYTES(cmd_hca_cap)); 387938fe83cSSaeed Mahameed 388938fe83cSSaeed Mahameed mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 389938fe83cSSaeed Mahameed to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 390938fe83cSSaeed Mahameed 128); 391c7a08ac7SEli Cohen /* we limit the size of the pkey table to 128 entries for now */ 392938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 393938fe83cSSaeed Mahameed to_fw_pkey_sz(128)); 394e126ba97SEli Cohen 395c7a08ac7SEli Cohen if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 396938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 397938fe83cSSaeed Mahameed prof->log_max_qp); 398e126ba97SEli Cohen 399938fe83cSSaeed Mahameed /* disable cmdif checksum */ 400938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 401c1868b82SEli Cohen 402b775516bSEli Cohen err = set_caps(dev, set_ctx, set_sz); 403e126ba97SEli Cohen 404e126ba97SEli Cohen query_ex: 405e126ba97SEli Cohen kfree(set_ctx); 406e126ba97SEli Cohen return err; 407e126ba97SEli Cohen } 408e126ba97SEli Cohen 409e126ba97SEli Cohen static int set_hca_ctrl(struct mlx5_core_dev *dev) 410e126ba97SEli Cohen { 411e126ba97SEli Cohen struct mlx5_reg_host_endianess he_in; 412e126ba97SEli Cohen struct mlx5_reg_host_endianess he_out; 413e126ba97SEli Cohen int err; 414e126ba97SEli Cohen 415e126ba97SEli Cohen memset(&he_in, 0, sizeof(he_in)); 416e126ba97SEli Cohen he_in.he = MLX5_SET_HOST_ENDIANNESS; 417e126ba97SEli Cohen err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 418e126ba97SEli Cohen &he_out, sizeof(he_out), 419e126ba97SEli Cohen MLX5_REG_HOST_ENDIANNESS, 0, 1); 420e126ba97SEli Cohen return err; 421e126ba97SEli Cohen } 422e126ba97SEli Cohen 423cd23b14bSEli Cohen static int mlx5_core_enable_hca(struct mlx5_core_dev *dev) 424cd23b14bSEli Cohen { 425cd23b14bSEli Cohen int err; 426cd23b14bSEli Cohen struct mlx5_enable_hca_mbox_in in; 427cd23b14bSEli Cohen struct mlx5_enable_hca_mbox_out out; 428cd23b14bSEli Cohen 429cd23b14bSEli Cohen memset(&in, 0, sizeof(in)); 430cd23b14bSEli Cohen memset(&out, 0, sizeof(out)); 431cd23b14bSEli Cohen in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ENABLE_HCA); 432cd23b14bSEli Cohen err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 433cd23b14bSEli Cohen if (err) 434cd23b14bSEli Cohen return err; 435cd23b14bSEli Cohen 436cd23b14bSEli Cohen if (out.hdr.status) 437cd23b14bSEli Cohen return mlx5_cmd_status_to_err(&out.hdr); 438cd23b14bSEli Cohen 439cd23b14bSEli Cohen return 0; 440cd23b14bSEli Cohen } 441cd23b14bSEli Cohen 442cd23b14bSEli Cohen static int mlx5_core_disable_hca(struct mlx5_core_dev *dev) 443cd23b14bSEli Cohen { 444cd23b14bSEli Cohen int err; 445cd23b14bSEli Cohen struct mlx5_disable_hca_mbox_in in; 446cd23b14bSEli Cohen struct mlx5_disable_hca_mbox_out out; 447cd23b14bSEli Cohen 448cd23b14bSEli Cohen memset(&in, 0, sizeof(in)); 449cd23b14bSEli Cohen memset(&out, 0, sizeof(out)); 450cd23b14bSEli Cohen in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DISABLE_HCA); 451cd23b14bSEli Cohen err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 452cd23b14bSEli Cohen if (err) 453cd23b14bSEli Cohen return err; 454cd23b14bSEli Cohen 455cd23b14bSEli Cohen if (out.hdr.status) 456cd23b14bSEli Cohen return mlx5_cmd_status_to_err(&out.hdr); 457cd23b14bSEli Cohen 458cd23b14bSEli Cohen return 0; 459cd23b14bSEli Cohen } 460cd23b14bSEli Cohen 461db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) 462db058a18SSaeed Mahameed { 463db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 464db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 465db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 466db058a18SSaeed Mahameed int numa_node = dev_to_node(&mdev->pdev->dev); 467db058a18SSaeed Mahameed int err; 468db058a18SSaeed Mahameed 469db058a18SSaeed Mahameed if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) { 470db058a18SSaeed Mahameed mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); 471db058a18SSaeed Mahameed return -ENOMEM; 472db058a18SSaeed Mahameed } 473db058a18SSaeed Mahameed 474dda922c8SDavid S. Miller cpumask_set_cpu(cpumask_local_spread(i, numa_node), 475dda922c8SDavid S. Miller priv->irq_info[i].mask); 476db058a18SSaeed Mahameed 477db058a18SSaeed Mahameed err = irq_set_affinity_hint(irq, priv->irq_info[i].mask); 478db058a18SSaeed Mahameed if (err) { 479db058a18SSaeed Mahameed mlx5_core_warn(mdev, "irq_set_affinity_hint failed,irq 0x%.4x", 480db058a18SSaeed Mahameed irq); 481db058a18SSaeed Mahameed goto err_clear_mask; 482db058a18SSaeed Mahameed } 483db058a18SSaeed Mahameed 484db058a18SSaeed Mahameed return 0; 485db058a18SSaeed Mahameed 486db058a18SSaeed Mahameed err_clear_mask: 487db058a18SSaeed Mahameed free_cpumask_var(priv->irq_info[i].mask); 488db058a18SSaeed Mahameed return err; 489db058a18SSaeed Mahameed } 490db058a18SSaeed Mahameed 491db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i) 492db058a18SSaeed Mahameed { 493db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 494db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 495db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 496db058a18SSaeed Mahameed 497db058a18SSaeed Mahameed irq_set_affinity_hint(irq, NULL); 498db058a18SSaeed Mahameed free_cpumask_var(priv->irq_info[i].mask); 499db058a18SSaeed Mahameed } 500db058a18SSaeed Mahameed 501db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev) 502db058a18SSaeed Mahameed { 503db058a18SSaeed Mahameed int err; 504db058a18SSaeed Mahameed int i; 505db058a18SSaeed Mahameed 506db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) { 507db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hint(mdev, i); 508db058a18SSaeed Mahameed if (err) 509db058a18SSaeed Mahameed goto err_out; 510db058a18SSaeed Mahameed } 511db058a18SSaeed Mahameed 512db058a18SSaeed Mahameed return 0; 513db058a18SSaeed Mahameed 514db058a18SSaeed Mahameed err_out: 515db058a18SSaeed Mahameed for (i--; i >= 0; i--) 516db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 517db058a18SSaeed Mahameed 518db058a18SSaeed Mahameed return err; 519db058a18SSaeed Mahameed } 520db058a18SSaeed Mahameed 521db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev) 522db058a18SSaeed Mahameed { 523db058a18SSaeed Mahameed int i; 524db058a18SSaeed Mahameed 525db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) 526db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 527db058a18SSaeed Mahameed } 528db058a18SSaeed Mahameed 529233d05d2SSaeed Mahameed int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn) 530233d05d2SSaeed Mahameed { 531233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 532233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 533233d05d2SSaeed Mahameed int err = -ENOENT; 534233d05d2SSaeed Mahameed 535233d05d2SSaeed Mahameed spin_lock(&table->lock); 536233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 537233d05d2SSaeed Mahameed if (eq->index == vector) { 538233d05d2SSaeed Mahameed *eqn = eq->eqn; 539233d05d2SSaeed Mahameed *irqn = eq->irqn; 540233d05d2SSaeed Mahameed err = 0; 541233d05d2SSaeed Mahameed break; 542233d05d2SSaeed Mahameed } 543233d05d2SSaeed Mahameed } 544233d05d2SSaeed Mahameed spin_unlock(&table->lock); 545233d05d2SSaeed Mahameed 546233d05d2SSaeed Mahameed return err; 547233d05d2SSaeed Mahameed } 548233d05d2SSaeed Mahameed EXPORT_SYMBOL(mlx5_vector2eqn); 549233d05d2SSaeed Mahameed 550233d05d2SSaeed Mahameed static void free_comp_eqs(struct mlx5_core_dev *dev) 551233d05d2SSaeed Mahameed { 552233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 553233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 554233d05d2SSaeed Mahameed 555233d05d2SSaeed Mahameed spin_lock(&table->lock); 556233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 557233d05d2SSaeed Mahameed list_del(&eq->list); 558233d05d2SSaeed Mahameed spin_unlock(&table->lock); 559233d05d2SSaeed Mahameed if (mlx5_destroy_unmap_eq(dev, eq)) 560233d05d2SSaeed Mahameed mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 561233d05d2SSaeed Mahameed eq->eqn); 562233d05d2SSaeed Mahameed kfree(eq); 563233d05d2SSaeed Mahameed spin_lock(&table->lock); 564233d05d2SSaeed Mahameed } 565233d05d2SSaeed Mahameed spin_unlock(&table->lock); 566233d05d2SSaeed Mahameed } 567233d05d2SSaeed Mahameed 568233d05d2SSaeed Mahameed static int alloc_comp_eqs(struct mlx5_core_dev *dev) 569233d05d2SSaeed Mahameed { 570233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 571db058a18SSaeed Mahameed char name[MLX5_MAX_IRQ_NAME]; 572233d05d2SSaeed Mahameed struct mlx5_eq *eq; 573233d05d2SSaeed Mahameed int ncomp_vec; 574233d05d2SSaeed Mahameed int nent; 575233d05d2SSaeed Mahameed int err; 576233d05d2SSaeed Mahameed int i; 577233d05d2SSaeed Mahameed 578233d05d2SSaeed Mahameed INIT_LIST_HEAD(&table->comp_eqs_list); 579233d05d2SSaeed Mahameed ncomp_vec = table->num_comp_vectors; 580233d05d2SSaeed Mahameed nent = MLX5_COMP_EQ_SIZE; 581233d05d2SSaeed Mahameed for (i = 0; i < ncomp_vec; i++) { 582233d05d2SSaeed Mahameed eq = kzalloc(sizeof(*eq), GFP_KERNEL); 583233d05d2SSaeed Mahameed if (!eq) { 584233d05d2SSaeed Mahameed err = -ENOMEM; 585233d05d2SSaeed Mahameed goto clean; 586233d05d2SSaeed Mahameed } 587233d05d2SSaeed Mahameed 588db058a18SSaeed Mahameed snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 589233d05d2SSaeed Mahameed err = mlx5_create_map_eq(dev, eq, 590233d05d2SSaeed Mahameed i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 591233d05d2SSaeed Mahameed name, &dev->priv.uuari.uars[0]); 592233d05d2SSaeed Mahameed if (err) { 593233d05d2SSaeed Mahameed kfree(eq); 594233d05d2SSaeed Mahameed goto clean; 595233d05d2SSaeed Mahameed } 596233d05d2SSaeed Mahameed mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 597233d05d2SSaeed Mahameed eq->index = i; 598233d05d2SSaeed Mahameed spin_lock(&table->lock); 599233d05d2SSaeed Mahameed list_add_tail(&eq->list, &table->comp_eqs_list); 600233d05d2SSaeed Mahameed spin_unlock(&table->lock); 601233d05d2SSaeed Mahameed } 602233d05d2SSaeed Mahameed 603233d05d2SSaeed Mahameed return 0; 604233d05d2SSaeed Mahameed 605233d05d2SSaeed Mahameed clean: 606233d05d2SSaeed Mahameed free_comp_eqs(dev); 607233d05d2SSaeed Mahameed return err; 608233d05d2SSaeed Mahameed } 609233d05d2SSaeed Mahameed 610f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 611f62b8bb8SAmir Vadai static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 612f62b8bb8SAmir Vadai { 613f62b8bb8SAmir Vadai u32 query_in[MLX5_ST_SZ_DW(query_issi_in)]; 614f62b8bb8SAmir Vadai u32 query_out[MLX5_ST_SZ_DW(query_issi_out)]; 615f62b8bb8SAmir Vadai u32 set_in[MLX5_ST_SZ_DW(set_issi_in)]; 616f62b8bb8SAmir Vadai u32 set_out[MLX5_ST_SZ_DW(set_issi_out)]; 617f62b8bb8SAmir Vadai int err; 618f62b8bb8SAmir Vadai u32 sup_issi; 619f62b8bb8SAmir Vadai 620f62b8bb8SAmir Vadai memset(query_in, 0, sizeof(query_in)); 621f62b8bb8SAmir Vadai memset(query_out, 0, sizeof(query_out)); 622f62b8bb8SAmir Vadai 623f62b8bb8SAmir Vadai MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 624f62b8bb8SAmir Vadai 625f62b8bb8SAmir Vadai err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in), 626f62b8bb8SAmir Vadai query_out, sizeof(query_out)); 627f62b8bb8SAmir Vadai if (err) { 628f62b8bb8SAmir Vadai if (((struct mlx5_outbox_hdr *)query_out)->status == 629f62b8bb8SAmir Vadai MLX5_CMD_STAT_BAD_OP_ERR) { 630f62b8bb8SAmir Vadai pr_debug("Only ISSI 0 is supported\n"); 631f62b8bb8SAmir Vadai return 0; 632f62b8bb8SAmir Vadai } 633f62b8bb8SAmir Vadai 634f62b8bb8SAmir Vadai pr_err("failed to query ISSI\n"); 635f62b8bb8SAmir Vadai return err; 636f62b8bb8SAmir Vadai } 637f62b8bb8SAmir Vadai 638f62b8bb8SAmir Vadai sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 639f62b8bb8SAmir Vadai 640f62b8bb8SAmir Vadai if (sup_issi & (1 << 1)) { 641f62b8bb8SAmir Vadai memset(set_in, 0, sizeof(set_in)); 642f62b8bb8SAmir Vadai memset(set_out, 0, sizeof(set_out)); 643f62b8bb8SAmir Vadai 644f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 645f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, current_issi, 1); 646f62b8bb8SAmir Vadai 647f62b8bb8SAmir Vadai err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in), 648f62b8bb8SAmir Vadai set_out, sizeof(set_out)); 649f62b8bb8SAmir Vadai if (err) { 650f62b8bb8SAmir Vadai pr_err("failed to set ISSI=1\n"); 651f62b8bb8SAmir Vadai return err; 652f62b8bb8SAmir Vadai } 653f62b8bb8SAmir Vadai 654f62b8bb8SAmir Vadai dev->issi = 1; 655f62b8bb8SAmir Vadai 656f62b8bb8SAmir Vadai return 0; 657f62b8bb8SAmir Vadai } else if (sup_issi & (1 << 0)) { 658f62b8bb8SAmir Vadai return 0; 659f62b8bb8SAmir Vadai } 660f62b8bb8SAmir Vadai 661f62b8bb8SAmir Vadai return -ENOTSUPP; 662f62b8bb8SAmir Vadai } 663f62b8bb8SAmir Vadai #endif 664f62b8bb8SAmir Vadai 6659603b61dSJack Morgenstein static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev) 666e126ba97SEli Cohen { 667e126ba97SEli Cohen struct mlx5_priv *priv = &dev->priv; 668e126ba97SEli Cohen int err; 669e126ba97SEli Cohen 670e126ba97SEli Cohen dev->pdev = pdev; 671e126ba97SEli Cohen pci_set_drvdata(dev->pdev, dev); 672e126ba97SEli Cohen strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 673e126ba97SEli Cohen priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 674e126ba97SEli Cohen 675e126ba97SEli Cohen mutex_init(&priv->pgdir_mutex); 676e126ba97SEli Cohen INIT_LIST_HEAD(&priv->pgdir_list); 677e126ba97SEli Cohen spin_lock_init(&priv->mkey_lock); 678e126ba97SEli Cohen 679e126ba97SEli Cohen priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root); 680e126ba97SEli Cohen if (!priv->dbg_root) 681e126ba97SEli Cohen return -ENOMEM; 682e126ba97SEli Cohen 683e126ba97SEli Cohen err = pci_enable_device(pdev); 684e126ba97SEli Cohen if (err) { 6851a91de28SJoe Perches dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 686e126ba97SEli Cohen goto err_dbg; 687e126ba97SEli Cohen } 688e126ba97SEli Cohen 689e126ba97SEli Cohen err = request_bar(pdev); 690e126ba97SEli Cohen if (err) { 6911a91de28SJoe Perches dev_err(&pdev->dev, "error requesting BARs, aborting\n"); 692e126ba97SEli Cohen goto err_disable; 693e126ba97SEli Cohen } 694e126ba97SEli Cohen 695e126ba97SEli Cohen pci_set_master(pdev); 696e126ba97SEli Cohen 697e126ba97SEli Cohen err = set_dma_caps(pdev); 698e126ba97SEli Cohen if (err) { 699e126ba97SEli Cohen dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); 700e126ba97SEli Cohen goto err_clr_master; 701e126ba97SEli Cohen } 702e126ba97SEli Cohen 703e126ba97SEli Cohen dev->iseg_base = pci_resource_start(dev->pdev, 0); 704e126ba97SEli Cohen dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 705e126ba97SEli Cohen if (!dev->iseg) { 706e126ba97SEli Cohen err = -ENOMEM; 707e126ba97SEli Cohen dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); 708e126ba97SEli Cohen goto err_clr_master; 709e126ba97SEli Cohen } 710e126ba97SEli Cohen dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 711e126ba97SEli Cohen fw_rev_min(dev), fw_rev_sub(dev)); 712e126ba97SEli Cohen 713e126ba97SEli Cohen err = mlx5_cmd_init(dev); 714e126ba97SEli Cohen if (err) { 715e126ba97SEli Cohen dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); 716e126ba97SEli Cohen goto err_unmap; 717e126ba97SEli Cohen } 718e126ba97SEli Cohen 719e126ba97SEli Cohen mlx5_pagealloc_init(dev); 720cd23b14bSEli Cohen 721cd23b14bSEli Cohen err = mlx5_core_enable_hca(dev); 722cd23b14bSEli Cohen if (err) { 723cd23b14bSEli Cohen dev_err(&pdev->dev, "enable hca failed\n"); 724cd23b14bSEli Cohen goto err_pagealloc_cleanup; 725cd23b14bSEli Cohen } 726cd23b14bSEli Cohen 727f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 728f62b8bb8SAmir Vadai err = mlx5_core_set_issi(dev); 729f62b8bb8SAmir Vadai if (err) { 730f62b8bb8SAmir Vadai dev_err(&pdev->dev, "failed to set issi\n"); 731f62b8bb8SAmir Vadai goto err_disable_hca; 732f62b8bb8SAmir Vadai } 733f62b8bb8SAmir Vadai #endif 734f62b8bb8SAmir Vadai 735cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 1); 736cd23b14bSEli Cohen if (err) { 737cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate boot pages\n"); 738cd23b14bSEli Cohen goto err_disable_hca; 739cd23b14bSEli Cohen } 740cd23b14bSEli Cohen 741e126ba97SEli Cohen err = set_hca_ctrl(dev); 742e126ba97SEli Cohen if (err) { 743e126ba97SEli Cohen dev_err(&pdev->dev, "set_hca_ctrl failed\n"); 744cd23b14bSEli Cohen goto reclaim_boot_pages; 745e126ba97SEli Cohen } 746e126ba97SEli Cohen 747e126ba97SEli Cohen err = handle_hca_cap(dev); 748e126ba97SEli Cohen if (err) { 749e126ba97SEli Cohen dev_err(&pdev->dev, "handle_hca_cap failed\n"); 750cd23b14bSEli Cohen goto reclaim_boot_pages; 751e126ba97SEli Cohen } 752e126ba97SEli Cohen 753cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 0); 754e126ba97SEli Cohen if (err) { 755cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate init pages\n"); 756cd23b14bSEli Cohen goto reclaim_boot_pages; 757e126ba97SEli Cohen } 758e126ba97SEli Cohen 759e126ba97SEli Cohen err = mlx5_pagealloc_start(dev); 760e126ba97SEli Cohen if (err) { 761e126ba97SEli Cohen dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n"); 762cd23b14bSEli Cohen goto reclaim_boot_pages; 763e126ba97SEli Cohen } 764e126ba97SEli Cohen 765e126ba97SEli Cohen err = mlx5_cmd_init_hca(dev); 766e126ba97SEli Cohen if (err) { 767e126ba97SEli Cohen dev_err(&pdev->dev, "init hca failed\n"); 768e126ba97SEli Cohen goto err_pagealloc_stop; 769e126ba97SEli Cohen } 770e126ba97SEli Cohen 771e126ba97SEli Cohen mlx5_start_health_poll(dev); 772e126ba97SEli Cohen 773938fe83cSSaeed Mahameed err = mlx5_query_hca_caps(dev); 774e126ba97SEli Cohen if (err) { 775e126ba97SEli Cohen dev_err(&pdev->dev, "query hca failed\n"); 776e126ba97SEli Cohen goto err_stop_poll; 777e126ba97SEli Cohen } 778e126ba97SEli Cohen 779e126ba97SEli Cohen err = mlx5_cmd_query_adapter(dev); 780e126ba97SEli Cohen if (err) { 781e126ba97SEli Cohen dev_err(&pdev->dev, "query adapter failed\n"); 782e126ba97SEli Cohen goto err_stop_poll; 783e126ba97SEli Cohen } 784e126ba97SEli Cohen 785e126ba97SEli Cohen err = mlx5_enable_msix(dev); 786e126ba97SEli Cohen if (err) { 787e126ba97SEli Cohen dev_err(&pdev->dev, "enable msix failed\n"); 788e126ba97SEli Cohen goto err_stop_poll; 789e126ba97SEli Cohen } 790e126ba97SEli Cohen 791e126ba97SEli Cohen err = mlx5_eq_init(dev); 792e126ba97SEli Cohen if (err) { 793e126ba97SEli Cohen dev_err(&pdev->dev, "failed to initialize eq\n"); 794e126ba97SEli Cohen goto disable_msix; 795e126ba97SEli Cohen } 796e126ba97SEli Cohen 797e126ba97SEli Cohen err = mlx5_alloc_uuars(dev, &priv->uuari); 798e126ba97SEli Cohen if (err) { 799e126ba97SEli Cohen dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 800e126ba97SEli Cohen goto err_eq_cleanup; 801e126ba97SEli Cohen } 802e126ba97SEli Cohen 803e126ba97SEli Cohen err = mlx5_start_eqs(dev); 804e126ba97SEli Cohen if (err) { 805e126ba97SEli Cohen dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 806e126ba97SEli Cohen goto err_free_uar; 807e126ba97SEli Cohen } 808e126ba97SEli Cohen 809233d05d2SSaeed Mahameed err = alloc_comp_eqs(dev); 810233d05d2SSaeed Mahameed if (err) { 811233d05d2SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc completion EQs\n"); 812233d05d2SSaeed Mahameed goto err_stop_eqs; 813233d05d2SSaeed Mahameed } 814233d05d2SSaeed Mahameed 815db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hints(dev); 816db058a18SSaeed Mahameed if (err) { 817db058a18SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n"); 818db058a18SSaeed Mahameed goto err_free_comp_eqs; 819db058a18SSaeed Mahameed } 820db058a18SSaeed Mahameed 821e126ba97SEli Cohen MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 822e126ba97SEli Cohen 823e126ba97SEli Cohen mlx5_init_cq_table(dev); 824e126ba97SEli Cohen mlx5_init_qp_table(dev); 825e126ba97SEli Cohen mlx5_init_srq_table(dev); 8263bcdb17aSSagi Grimberg mlx5_init_mr_table(dev); 827e126ba97SEli Cohen 828e126ba97SEli Cohen return 0; 829e126ba97SEli Cohen 830db058a18SSaeed Mahameed err_free_comp_eqs: 831db058a18SSaeed Mahameed free_comp_eqs(dev); 832db058a18SSaeed Mahameed 833233d05d2SSaeed Mahameed err_stop_eqs: 834233d05d2SSaeed Mahameed mlx5_stop_eqs(dev); 835233d05d2SSaeed Mahameed 836e126ba97SEli Cohen err_free_uar: 837e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 838e126ba97SEli Cohen 839e126ba97SEli Cohen err_eq_cleanup: 840e126ba97SEli Cohen mlx5_eq_cleanup(dev); 841e126ba97SEli Cohen 842e126ba97SEli Cohen disable_msix: 843e126ba97SEli Cohen mlx5_disable_msix(dev); 844e126ba97SEli Cohen 845e126ba97SEli Cohen err_stop_poll: 846e126ba97SEli Cohen mlx5_stop_health_poll(dev); 8471bde6e30SEli Cohen if (mlx5_cmd_teardown_hca(dev)) { 8481bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 8491bde6e30SEli Cohen return err; 8501bde6e30SEli Cohen } 851e126ba97SEli Cohen 852e126ba97SEli Cohen err_pagealloc_stop: 853e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 854e126ba97SEli Cohen 855cd23b14bSEli Cohen reclaim_boot_pages: 856e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 857e126ba97SEli Cohen 858cd23b14bSEli Cohen err_disable_hca: 859cd23b14bSEli Cohen mlx5_core_disable_hca(dev); 860cd23b14bSEli Cohen 861e126ba97SEli Cohen err_pagealloc_cleanup: 862e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 863e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 864e126ba97SEli Cohen 865e126ba97SEli Cohen err_unmap: 866e126ba97SEli Cohen iounmap(dev->iseg); 867e126ba97SEli Cohen 868e126ba97SEli Cohen err_clr_master: 869e126ba97SEli Cohen pci_clear_master(dev->pdev); 870e126ba97SEli Cohen release_bar(dev->pdev); 871e126ba97SEli Cohen 872e126ba97SEli Cohen err_disable: 873e126ba97SEli Cohen pci_disable_device(dev->pdev); 874e126ba97SEli Cohen 875e126ba97SEli Cohen err_dbg: 876e126ba97SEli Cohen debugfs_remove(priv->dbg_root); 877e126ba97SEli Cohen return err; 878e126ba97SEli Cohen } 879e126ba97SEli Cohen 8809603b61dSJack Morgenstein static void mlx5_dev_cleanup(struct mlx5_core_dev *dev) 881e126ba97SEli Cohen { 882e126ba97SEli Cohen struct mlx5_priv *priv = &dev->priv; 883e126ba97SEli Cohen 884e126ba97SEli Cohen mlx5_cleanup_srq_table(dev); 885e126ba97SEli Cohen mlx5_cleanup_qp_table(dev); 886e126ba97SEli Cohen mlx5_cleanup_cq_table(dev); 887db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hints(dev); 888233d05d2SSaeed Mahameed free_comp_eqs(dev); 889e126ba97SEli Cohen mlx5_stop_eqs(dev); 890e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 891e126ba97SEli Cohen mlx5_eq_cleanup(dev); 892e126ba97SEli Cohen mlx5_disable_msix(dev); 893e126ba97SEli Cohen mlx5_stop_health_poll(dev); 8941bde6e30SEli Cohen if (mlx5_cmd_teardown_hca(dev)) { 8951bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 8961bde6e30SEli Cohen return; 8971bde6e30SEli Cohen } 898e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 899e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 900cd23b14bSEli Cohen mlx5_core_disable_hca(dev); 901e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 902e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 903e126ba97SEli Cohen iounmap(dev->iseg); 904e126ba97SEli Cohen pci_clear_master(dev->pdev); 905e126ba97SEli Cohen release_bar(dev->pdev); 906e126ba97SEli Cohen pci_disable_device(dev->pdev); 907e126ba97SEli Cohen debugfs_remove(priv->dbg_root); 908e126ba97SEli Cohen } 9099603b61dSJack Morgenstein 9109603b61dSJack Morgenstein static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 9119603b61dSJack Morgenstein { 9129603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 9139603b61dSJack Morgenstein struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 9149603b61dSJack Morgenstein 9159603b61dSJack Morgenstein dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL); 9169603b61dSJack Morgenstein if (!dev_ctx) { 9179603b61dSJack Morgenstein pr_warn("mlx5_add_device: alloc context failed\n"); 9189603b61dSJack Morgenstein return; 9199603b61dSJack Morgenstein } 9209603b61dSJack Morgenstein 9219603b61dSJack Morgenstein dev_ctx->intf = intf; 9229603b61dSJack Morgenstein dev_ctx->context = intf->add(dev); 9239603b61dSJack Morgenstein 9249603b61dSJack Morgenstein if (dev_ctx->context) { 9259603b61dSJack Morgenstein spin_lock_irq(&priv->ctx_lock); 9269603b61dSJack Morgenstein list_add_tail(&dev_ctx->list, &priv->ctx_list); 9279603b61dSJack Morgenstein spin_unlock_irq(&priv->ctx_lock); 9289603b61dSJack Morgenstein } else { 9299603b61dSJack Morgenstein kfree(dev_ctx); 9309603b61dSJack Morgenstein } 9319603b61dSJack Morgenstein } 9329603b61dSJack Morgenstein 9339603b61dSJack Morgenstein static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 9349603b61dSJack Morgenstein { 9359603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 9369603b61dSJack Morgenstein struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 9379603b61dSJack Morgenstein 9389603b61dSJack Morgenstein list_for_each_entry(dev_ctx, &priv->ctx_list, list) 9399603b61dSJack Morgenstein if (dev_ctx->intf == intf) { 9409603b61dSJack Morgenstein spin_lock_irq(&priv->ctx_lock); 9419603b61dSJack Morgenstein list_del(&dev_ctx->list); 9429603b61dSJack Morgenstein spin_unlock_irq(&priv->ctx_lock); 9439603b61dSJack Morgenstein 9449603b61dSJack Morgenstein intf->remove(dev, dev_ctx->context); 9459603b61dSJack Morgenstein kfree(dev_ctx); 9469603b61dSJack Morgenstein return; 9479603b61dSJack Morgenstein } 9489603b61dSJack Morgenstein } 9499603b61dSJack Morgenstein static int mlx5_register_device(struct mlx5_core_dev *dev) 9509603b61dSJack Morgenstein { 9519603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 9529603b61dSJack Morgenstein struct mlx5_interface *intf; 9539603b61dSJack Morgenstein 9549603b61dSJack Morgenstein mutex_lock(&intf_mutex); 9559603b61dSJack Morgenstein list_add_tail(&priv->dev_list, &dev_list); 9569603b61dSJack Morgenstein list_for_each_entry(intf, &intf_list, list) 9579603b61dSJack Morgenstein mlx5_add_device(intf, priv); 9589603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 9599603b61dSJack Morgenstein 9609603b61dSJack Morgenstein return 0; 9619603b61dSJack Morgenstein } 9629603b61dSJack Morgenstein static void mlx5_unregister_device(struct mlx5_core_dev *dev) 9639603b61dSJack Morgenstein { 9649603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 9659603b61dSJack Morgenstein struct mlx5_interface *intf; 9669603b61dSJack Morgenstein 9679603b61dSJack Morgenstein mutex_lock(&intf_mutex); 9689603b61dSJack Morgenstein list_for_each_entry(intf, &intf_list, list) 9699603b61dSJack Morgenstein mlx5_remove_device(intf, priv); 9709603b61dSJack Morgenstein list_del(&priv->dev_list); 9719603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 9729603b61dSJack Morgenstein } 9739603b61dSJack Morgenstein 9749603b61dSJack Morgenstein int mlx5_register_interface(struct mlx5_interface *intf) 9759603b61dSJack Morgenstein { 9769603b61dSJack Morgenstein struct mlx5_priv *priv; 9779603b61dSJack Morgenstein 9789603b61dSJack Morgenstein if (!intf->add || !intf->remove) 9799603b61dSJack Morgenstein return -EINVAL; 9809603b61dSJack Morgenstein 9819603b61dSJack Morgenstein mutex_lock(&intf_mutex); 9829603b61dSJack Morgenstein list_add_tail(&intf->list, &intf_list); 9839603b61dSJack Morgenstein list_for_each_entry(priv, &dev_list, dev_list) 9849603b61dSJack Morgenstein mlx5_add_device(intf, priv); 9859603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 9869603b61dSJack Morgenstein 9879603b61dSJack Morgenstein return 0; 9889603b61dSJack Morgenstein } 9899603b61dSJack Morgenstein EXPORT_SYMBOL(mlx5_register_interface); 9909603b61dSJack Morgenstein 9919603b61dSJack Morgenstein void mlx5_unregister_interface(struct mlx5_interface *intf) 9929603b61dSJack Morgenstein { 9939603b61dSJack Morgenstein struct mlx5_priv *priv; 9949603b61dSJack Morgenstein 9959603b61dSJack Morgenstein mutex_lock(&intf_mutex); 9969603b61dSJack Morgenstein list_for_each_entry(priv, &dev_list, dev_list) 9979603b61dSJack Morgenstein mlx5_remove_device(intf, priv); 9989603b61dSJack Morgenstein list_del(&intf->list); 9999603b61dSJack Morgenstein mutex_unlock(&intf_mutex); 10009603b61dSJack Morgenstein } 10019603b61dSJack Morgenstein EXPORT_SYMBOL(mlx5_unregister_interface); 10029603b61dSJack Morgenstein 100364613d94SSaeed Mahameed void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) 100464613d94SSaeed Mahameed { 100564613d94SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 100664613d94SSaeed Mahameed struct mlx5_device_context *dev_ctx; 100764613d94SSaeed Mahameed unsigned long flags; 100864613d94SSaeed Mahameed void *result = NULL; 100964613d94SSaeed Mahameed 101064613d94SSaeed Mahameed spin_lock_irqsave(&priv->ctx_lock, flags); 101164613d94SSaeed Mahameed 101264613d94SSaeed Mahameed list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) 101364613d94SSaeed Mahameed if ((dev_ctx->intf->protocol == protocol) && 101464613d94SSaeed Mahameed dev_ctx->intf->get_dev) { 101564613d94SSaeed Mahameed result = dev_ctx->intf->get_dev(dev_ctx->context); 101664613d94SSaeed Mahameed break; 101764613d94SSaeed Mahameed } 101864613d94SSaeed Mahameed 101964613d94SSaeed Mahameed spin_unlock_irqrestore(&priv->ctx_lock, flags); 102064613d94SSaeed Mahameed 102164613d94SSaeed Mahameed return result; 102264613d94SSaeed Mahameed } 102364613d94SSaeed Mahameed EXPORT_SYMBOL(mlx5_get_protocol_dev); 102464613d94SSaeed Mahameed 10259603b61dSJack Morgenstein static void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 10264d2f9bbbSJack Morgenstein unsigned long param) 10279603b61dSJack Morgenstein { 10289603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 10299603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 10309603b61dSJack Morgenstein unsigned long flags; 10319603b61dSJack Morgenstein 10329603b61dSJack Morgenstein spin_lock_irqsave(&priv->ctx_lock, flags); 10339603b61dSJack Morgenstein 10349603b61dSJack Morgenstein list_for_each_entry(dev_ctx, &priv->ctx_list, list) 10359603b61dSJack Morgenstein if (dev_ctx->intf->event) 10364d2f9bbbSJack Morgenstein dev_ctx->intf->event(dev, dev_ctx->context, event, param); 10379603b61dSJack Morgenstein 10389603b61dSJack Morgenstein spin_unlock_irqrestore(&priv->ctx_lock, flags); 10399603b61dSJack Morgenstein } 10409603b61dSJack Morgenstein 10419603b61dSJack Morgenstein struct mlx5_core_event_handler { 10429603b61dSJack Morgenstein void (*event)(struct mlx5_core_dev *dev, 10439603b61dSJack Morgenstein enum mlx5_dev_event event, 10449603b61dSJack Morgenstein void *data); 10459603b61dSJack Morgenstein }; 10469603b61dSJack Morgenstein 1047f66f049fSEli Cohen #define MLX5_IB_MOD "mlx5_ib" 1048f66f049fSEli Cohen 10499603b61dSJack Morgenstein static int init_one(struct pci_dev *pdev, 10509603b61dSJack Morgenstein const struct pci_device_id *id) 10519603b61dSJack Morgenstein { 10529603b61dSJack Morgenstein struct mlx5_core_dev *dev; 10539603b61dSJack Morgenstein struct mlx5_priv *priv; 10549603b61dSJack Morgenstein int err; 10559603b61dSJack Morgenstein 10569603b61dSJack Morgenstein dev = kzalloc(sizeof(*dev), GFP_KERNEL); 10579603b61dSJack Morgenstein if (!dev) { 10589603b61dSJack Morgenstein dev_err(&pdev->dev, "kzalloc failed\n"); 10599603b61dSJack Morgenstein return -ENOMEM; 10609603b61dSJack Morgenstein } 10619603b61dSJack Morgenstein priv = &dev->priv; 10629603b61dSJack Morgenstein 10639603b61dSJack Morgenstein pci_set_drvdata(pdev, dev); 10649603b61dSJack Morgenstein 10659603b61dSJack Morgenstein if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profile)) { 10669603b61dSJack Morgenstein pr_warn("selected profile out of range, selecting default (%d)\n", 10679603b61dSJack Morgenstein MLX5_DEFAULT_PROF); 10689603b61dSJack Morgenstein prof_sel = MLX5_DEFAULT_PROF; 10699603b61dSJack Morgenstein } 10709603b61dSJack Morgenstein dev->profile = &profile[prof_sel]; 10719603b61dSJack Morgenstein dev->event = mlx5_core_event; 10729603b61dSJack Morgenstein 1073364d1798SEli Cohen INIT_LIST_HEAD(&priv->ctx_list); 1074364d1798SEli Cohen spin_lock_init(&priv->ctx_lock); 10759603b61dSJack Morgenstein err = mlx5_dev_init(dev, pdev); 10769603b61dSJack Morgenstein if (err) { 10779603b61dSJack Morgenstein dev_err(&pdev->dev, "mlx5_dev_init failed %d\n", err); 10789603b61dSJack Morgenstein goto out; 10799603b61dSJack Morgenstein } 10809603b61dSJack Morgenstein 10819603b61dSJack Morgenstein err = mlx5_register_device(dev); 10829603b61dSJack Morgenstein if (err) { 10839603b61dSJack Morgenstein dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 10849603b61dSJack Morgenstein goto out_init; 10859603b61dSJack Morgenstein } 10869603b61dSJack Morgenstein 1087f66f049fSEli Cohen err = request_module_nowait(MLX5_IB_MOD); 1088f66f049fSEli Cohen if (err) 1089f66f049fSEli Cohen pr_info("failed request module on %s\n", MLX5_IB_MOD); 1090f66f049fSEli Cohen 10919603b61dSJack Morgenstein return 0; 10929603b61dSJack Morgenstein 10939603b61dSJack Morgenstein out_init: 10949603b61dSJack Morgenstein mlx5_dev_cleanup(dev); 10959603b61dSJack Morgenstein out: 10969603b61dSJack Morgenstein kfree(dev); 10979603b61dSJack Morgenstein return err; 10989603b61dSJack Morgenstein } 10999603b61dSJack Morgenstein static void remove_one(struct pci_dev *pdev) 11009603b61dSJack Morgenstein { 11019603b61dSJack Morgenstein struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 11029603b61dSJack Morgenstein 11039603b61dSJack Morgenstein mlx5_unregister_device(dev); 11049603b61dSJack Morgenstein mlx5_dev_cleanup(dev); 11059603b61dSJack Morgenstein kfree(dev); 11069603b61dSJack Morgenstein } 11079603b61dSJack Morgenstein 11089603b61dSJack Morgenstein static const struct pci_device_id mlx5_core_pci_table[] = { 11091c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */ 11101c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1012) }, /* Connect-IB VF */ 11111c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */ 11121c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1014) }, /* ConnectX-4 VF */ 11131c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */ 11141c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1016) }, /* ConnectX-4LX VF */ 11159603b61dSJack Morgenstein { 0, } 11169603b61dSJack Morgenstein }; 11179603b61dSJack Morgenstein 11189603b61dSJack Morgenstein MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 11199603b61dSJack Morgenstein 11209603b61dSJack Morgenstein static struct pci_driver mlx5_core_driver = { 11219603b61dSJack Morgenstein .name = DRIVER_NAME, 11229603b61dSJack Morgenstein .id_table = mlx5_core_pci_table, 11239603b61dSJack Morgenstein .probe = init_one, 11249603b61dSJack Morgenstein .remove = remove_one 11259603b61dSJack Morgenstein }; 1126e126ba97SEli Cohen 1127e126ba97SEli Cohen static int __init init(void) 1128e126ba97SEli Cohen { 1129e126ba97SEli Cohen int err; 1130e126ba97SEli Cohen 1131e126ba97SEli Cohen mlx5_register_debugfs(); 1132e126ba97SEli Cohen mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq"); 1133e126ba97SEli Cohen if (!mlx5_core_wq) { 1134e126ba97SEli Cohen err = -ENOMEM; 1135e126ba97SEli Cohen goto err_debug; 1136e126ba97SEli Cohen } 1137e126ba97SEli Cohen mlx5_health_init(); 1138e126ba97SEli Cohen 11399603b61dSJack Morgenstein err = pci_register_driver(&mlx5_core_driver); 11409603b61dSJack Morgenstein if (err) 11419603b61dSJack Morgenstein goto err_health; 11429603b61dSJack Morgenstein 1143f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1144f62b8bb8SAmir Vadai mlx5e_init(); 1145f62b8bb8SAmir Vadai #endif 1146f62b8bb8SAmir Vadai 1147e126ba97SEli Cohen return 0; 1148e126ba97SEli Cohen 11499603b61dSJack Morgenstein err_health: 11509603b61dSJack Morgenstein mlx5_health_cleanup(); 11519603b61dSJack Morgenstein destroy_workqueue(mlx5_core_wq); 1152e126ba97SEli Cohen err_debug: 1153e126ba97SEli Cohen mlx5_unregister_debugfs(); 1154e126ba97SEli Cohen return err; 1155e126ba97SEli Cohen } 1156e126ba97SEli Cohen 1157e126ba97SEli Cohen static void __exit cleanup(void) 1158e126ba97SEli Cohen { 1159f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1160f62b8bb8SAmir Vadai mlx5e_cleanup(); 1161f62b8bb8SAmir Vadai #endif 11629603b61dSJack Morgenstein pci_unregister_driver(&mlx5_core_driver); 1163e126ba97SEli Cohen mlx5_health_cleanup(); 1164e126ba97SEli Cohen destroy_workqueue(mlx5_core_wq); 1165e126ba97SEli Cohen mlx5_unregister_debugfs(); 1166e126ba97SEli Cohen } 1167e126ba97SEli Cohen 1168e126ba97SEli Cohen module_init(init); 1169e126ba97SEli Cohen module_exit(cleanup); 1170