1e126ba97SEli Cohen /* 2302bdf68SSaeed Mahameed * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3e126ba97SEli Cohen * 4e126ba97SEli Cohen * This software is available to you under a choice of one of two 5e126ba97SEli Cohen * licenses. You may choose to be licensed under the terms of the GNU 6e126ba97SEli Cohen * General Public License (GPL) Version 2, available from the file 7e126ba97SEli Cohen * COPYING in the main directory of this source tree, or the 8e126ba97SEli Cohen * OpenIB.org BSD license below: 9e126ba97SEli Cohen * 10e126ba97SEli Cohen * Redistribution and use in source and binary forms, with or 11e126ba97SEli Cohen * without modification, are permitted provided that the following 12e126ba97SEli Cohen * conditions are met: 13e126ba97SEli Cohen * 14e126ba97SEli Cohen * - Redistributions of source code must retain the above 15e126ba97SEli Cohen * copyright notice, this list of conditions and the following 16e126ba97SEli Cohen * disclaimer. 17e126ba97SEli Cohen * 18e126ba97SEli Cohen * - Redistributions in binary form must reproduce the above 19e126ba97SEli Cohen * copyright notice, this list of conditions and the following 20e126ba97SEli Cohen * disclaimer in the documentation and/or other materials 21e126ba97SEli Cohen * provided with the distribution. 22e126ba97SEli Cohen * 23e126ba97SEli Cohen * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e126ba97SEli Cohen * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e126ba97SEli Cohen * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e126ba97SEli Cohen * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e126ba97SEli Cohen * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e126ba97SEli Cohen * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e126ba97SEli Cohen * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e126ba97SEli Cohen * SOFTWARE. 31e126ba97SEli Cohen */ 32e126ba97SEli Cohen 33adec640eSChristoph Hellwig #include <linux/highmem.h> 34e126ba97SEli Cohen #include <linux/module.h> 35e126ba97SEli Cohen #include <linux/init.h> 36e126ba97SEli Cohen #include <linux/errno.h> 37e126ba97SEli Cohen #include <linux/pci.h> 38e126ba97SEli Cohen #include <linux/dma-mapping.h> 39e126ba97SEli Cohen #include <linux/slab.h> 40e126ba97SEli Cohen #include <linux/io-mapping.h> 41db058a18SSaeed Mahameed #include <linux/interrupt.h> 42e3297246SEli Cohen #include <linux/delay.h> 43e126ba97SEli Cohen #include <linux/mlx5/driver.h> 44e126ba97SEli Cohen #include <linux/mlx5/cq.h> 45e126ba97SEli Cohen #include <linux/mlx5/qp.h> 46e126ba97SEli Cohen #include <linux/mlx5/srq.h> 47e126ba97SEli Cohen #include <linux/debugfs.h> 48f66f049fSEli Cohen #include <linux/kmod.h> 4989d44f0aSMajd Dibbiny #include <linux/delay.h> 50b775516bSEli Cohen #include <linux/mlx5/mlx5_ifc.h> 51e126ba97SEli Cohen #include "mlx5_core.h" 5286d722adSMaor Gottlieb #include "fs_core.h" 53073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 54073bb189SSaeed Mahameed #include "eswitch.h" 55073bb189SSaeed Mahameed #endif 56e126ba97SEli Cohen 57e126ba97SEli Cohen MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 584ae6c18cSAchiad Shochat MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 59e126ba97SEli Cohen MODULE_LICENSE("Dual BSD/GPL"); 60e126ba97SEli Cohen MODULE_VERSION(DRIVER_VERSION); 61e126ba97SEli Cohen 62e126ba97SEli Cohen int mlx5_core_debug_mask; 63e126ba97SEli Cohen module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 64e126ba97SEli Cohen MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 65e126ba97SEli Cohen 669603b61dSJack Morgenstein #define MLX5_DEFAULT_PROF 2 679603b61dSJack Morgenstein static int prof_sel = MLX5_DEFAULT_PROF; 689603b61dSJack Morgenstein module_param_named(prof_sel, prof_sel, int, 0444); 699603b61dSJack Morgenstein MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 709603b61dSJack Morgenstein 719603b61dSJack Morgenstein static LIST_HEAD(intf_list); 729603b61dSJack Morgenstein static LIST_HEAD(dev_list); 739603b61dSJack Morgenstein static DEFINE_MUTEX(intf_mutex); 749603b61dSJack Morgenstein 759603b61dSJack Morgenstein struct mlx5_device_context { 769603b61dSJack Morgenstein struct list_head list; 779603b61dSJack Morgenstein struct mlx5_interface *intf; 789603b61dSJack Morgenstein void *context; 799603b61dSJack Morgenstein }; 809603b61dSJack Morgenstein 81f91e6d89SEran Ben Elisha enum { 82f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_BE = 0x0, 83f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, 84f91e6d89SEran Ben Elisha }; 85f91e6d89SEran Ben Elisha 869603b61dSJack Morgenstein static struct mlx5_profile profile[] = { 879603b61dSJack Morgenstein [0] = { 889603b61dSJack Morgenstein .mask = 0, 899603b61dSJack Morgenstein }, 909603b61dSJack Morgenstein [1] = { 919603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE, 929603b61dSJack Morgenstein .log_max_qp = 12, 939603b61dSJack Morgenstein }, 949603b61dSJack Morgenstein [2] = { 959603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE | 969603b61dSJack Morgenstein MLX5_PROF_MASK_MR_CACHE, 979603b61dSJack Morgenstein .log_max_qp = 17, 989603b61dSJack Morgenstein .mr_cache[0] = { 999603b61dSJack Morgenstein .size = 500, 1009603b61dSJack Morgenstein .limit = 250 1019603b61dSJack Morgenstein }, 1029603b61dSJack Morgenstein .mr_cache[1] = { 1039603b61dSJack Morgenstein .size = 500, 1049603b61dSJack Morgenstein .limit = 250 1059603b61dSJack Morgenstein }, 1069603b61dSJack Morgenstein .mr_cache[2] = { 1079603b61dSJack Morgenstein .size = 500, 1089603b61dSJack Morgenstein .limit = 250 1099603b61dSJack Morgenstein }, 1109603b61dSJack Morgenstein .mr_cache[3] = { 1119603b61dSJack Morgenstein .size = 500, 1129603b61dSJack Morgenstein .limit = 250 1139603b61dSJack Morgenstein }, 1149603b61dSJack Morgenstein .mr_cache[4] = { 1159603b61dSJack Morgenstein .size = 500, 1169603b61dSJack Morgenstein .limit = 250 1179603b61dSJack Morgenstein }, 1189603b61dSJack Morgenstein .mr_cache[5] = { 1199603b61dSJack Morgenstein .size = 500, 1209603b61dSJack Morgenstein .limit = 250 1219603b61dSJack Morgenstein }, 1229603b61dSJack Morgenstein .mr_cache[6] = { 1239603b61dSJack Morgenstein .size = 500, 1249603b61dSJack Morgenstein .limit = 250 1259603b61dSJack Morgenstein }, 1269603b61dSJack Morgenstein .mr_cache[7] = { 1279603b61dSJack Morgenstein .size = 500, 1289603b61dSJack Morgenstein .limit = 250 1299603b61dSJack Morgenstein }, 1309603b61dSJack Morgenstein .mr_cache[8] = { 1319603b61dSJack Morgenstein .size = 500, 1329603b61dSJack Morgenstein .limit = 250 1339603b61dSJack Morgenstein }, 1349603b61dSJack Morgenstein .mr_cache[9] = { 1359603b61dSJack Morgenstein .size = 500, 1369603b61dSJack Morgenstein .limit = 250 1379603b61dSJack Morgenstein }, 1389603b61dSJack Morgenstein .mr_cache[10] = { 1399603b61dSJack Morgenstein .size = 500, 1409603b61dSJack Morgenstein .limit = 250 1419603b61dSJack Morgenstein }, 1429603b61dSJack Morgenstein .mr_cache[11] = { 1439603b61dSJack Morgenstein .size = 500, 1449603b61dSJack Morgenstein .limit = 250 1459603b61dSJack Morgenstein }, 1469603b61dSJack Morgenstein .mr_cache[12] = { 1479603b61dSJack Morgenstein .size = 64, 1489603b61dSJack Morgenstein .limit = 32 1499603b61dSJack Morgenstein }, 1509603b61dSJack Morgenstein .mr_cache[13] = { 1519603b61dSJack Morgenstein .size = 32, 1529603b61dSJack Morgenstein .limit = 16 1539603b61dSJack Morgenstein }, 1549603b61dSJack Morgenstein .mr_cache[14] = { 1559603b61dSJack Morgenstein .size = 16, 1569603b61dSJack Morgenstein .limit = 8 1579603b61dSJack Morgenstein }, 1589603b61dSJack Morgenstein .mr_cache[15] = { 1599603b61dSJack Morgenstein .size = 8, 1609603b61dSJack Morgenstein .limit = 4 1619603b61dSJack Morgenstein }, 1629603b61dSJack Morgenstein }, 1639603b61dSJack Morgenstein }; 164e126ba97SEli Cohen 165e3297246SEli Cohen #define FW_INIT_TIMEOUT_MILI 2000 166e3297246SEli Cohen #define FW_INIT_WAIT_MS 2 167e3297246SEli Cohen 168e3297246SEli Cohen static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 169e3297246SEli Cohen { 170e3297246SEli Cohen unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); 171e3297246SEli Cohen int err = 0; 172e3297246SEli Cohen 173e3297246SEli Cohen while (fw_initializing(dev)) { 174e3297246SEli Cohen if (time_after(jiffies, end)) { 175e3297246SEli Cohen err = -EBUSY; 176e3297246SEli Cohen break; 177e3297246SEli Cohen } 178e3297246SEli Cohen msleep(FW_INIT_WAIT_MS); 179e3297246SEli Cohen } 180e3297246SEli Cohen 181e3297246SEli Cohen return err; 182e3297246SEli Cohen } 183e3297246SEli Cohen 184e126ba97SEli Cohen static int set_dma_caps(struct pci_dev *pdev) 185e126ba97SEli Cohen { 186e126ba97SEli Cohen int err; 187e126ba97SEli Cohen 188e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 189e126ba97SEli Cohen if (err) { 1901a91de28SJoe Perches dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 191e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 192e126ba97SEli Cohen if (err) { 1931a91de28SJoe Perches dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 194e126ba97SEli Cohen return err; 195e126ba97SEli Cohen } 196e126ba97SEli Cohen } 197e126ba97SEli Cohen 198e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 199e126ba97SEli Cohen if (err) { 200e126ba97SEli Cohen dev_warn(&pdev->dev, 2011a91de28SJoe Perches "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 202e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 203e126ba97SEli Cohen if (err) { 204e126ba97SEli Cohen dev_err(&pdev->dev, 2051a91de28SJoe Perches "Can't set consistent PCI DMA mask, aborting\n"); 206e126ba97SEli Cohen return err; 207e126ba97SEli Cohen } 208e126ba97SEli Cohen } 209e126ba97SEli Cohen 210e126ba97SEli Cohen dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 211e126ba97SEli Cohen return err; 212e126ba97SEli Cohen } 213e126ba97SEli Cohen 21489d44f0aSMajd Dibbiny static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) 21589d44f0aSMajd Dibbiny { 21689d44f0aSMajd Dibbiny struct pci_dev *pdev = dev->pdev; 21789d44f0aSMajd Dibbiny int err = 0; 21889d44f0aSMajd Dibbiny 21989d44f0aSMajd Dibbiny mutex_lock(&dev->pci_status_mutex); 22089d44f0aSMajd Dibbiny if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { 22189d44f0aSMajd Dibbiny err = pci_enable_device(pdev); 22289d44f0aSMajd Dibbiny if (!err) 22389d44f0aSMajd Dibbiny dev->pci_status = MLX5_PCI_STATUS_ENABLED; 22489d44f0aSMajd Dibbiny } 22589d44f0aSMajd Dibbiny mutex_unlock(&dev->pci_status_mutex); 22689d44f0aSMajd Dibbiny 22789d44f0aSMajd Dibbiny return err; 22889d44f0aSMajd Dibbiny } 22989d44f0aSMajd Dibbiny 23089d44f0aSMajd Dibbiny static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) 23189d44f0aSMajd Dibbiny { 23289d44f0aSMajd Dibbiny struct pci_dev *pdev = dev->pdev; 23389d44f0aSMajd Dibbiny 23489d44f0aSMajd Dibbiny mutex_lock(&dev->pci_status_mutex); 23589d44f0aSMajd Dibbiny if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { 23689d44f0aSMajd Dibbiny pci_disable_device(pdev); 23789d44f0aSMajd Dibbiny dev->pci_status = MLX5_PCI_STATUS_DISABLED; 23889d44f0aSMajd Dibbiny } 23989d44f0aSMajd Dibbiny mutex_unlock(&dev->pci_status_mutex); 24089d44f0aSMajd Dibbiny } 24189d44f0aSMajd Dibbiny 242e126ba97SEli Cohen static int request_bar(struct pci_dev *pdev) 243e126ba97SEli Cohen { 244e126ba97SEli Cohen int err = 0; 245e126ba97SEli Cohen 246e126ba97SEli Cohen if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 2471a91de28SJoe Perches dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 248e126ba97SEli Cohen return -ENODEV; 249e126ba97SEli Cohen } 250e126ba97SEli Cohen 251e126ba97SEli Cohen err = pci_request_regions(pdev, DRIVER_NAME); 252e126ba97SEli Cohen if (err) 253e126ba97SEli Cohen dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 254e126ba97SEli Cohen 255e126ba97SEli Cohen return err; 256e126ba97SEli Cohen } 257e126ba97SEli Cohen 258e126ba97SEli Cohen static void release_bar(struct pci_dev *pdev) 259e126ba97SEli Cohen { 260e126ba97SEli Cohen pci_release_regions(pdev); 261e126ba97SEli Cohen } 262e126ba97SEli Cohen 263e126ba97SEli Cohen static int mlx5_enable_msix(struct mlx5_core_dev *dev) 264e126ba97SEli Cohen { 265db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 266db058a18SSaeed Mahameed struct mlx5_eq_table *table = &priv->eq_table; 267938fe83cSSaeed Mahameed int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 268e126ba97SEli Cohen int nvec; 269e126ba97SEli Cohen int i; 270e126ba97SEli Cohen 271938fe83cSSaeed Mahameed nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 272938fe83cSSaeed Mahameed MLX5_EQ_VEC_COMP_BASE; 273e126ba97SEli Cohen nvec = min_t(int, nvec, num_eqs); 274e126ba97SEli Cohen if (nvec <= MLX5_EQ_VEC_COMP_BASE) 275e126ba97SEli Cohen return -ENOMEM; 276e126ba97SEli Cohen 277db058a18SSaeed Mahameed priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL); 278db058a18SSaeed Mahameed 279db058a18SSaeed Mahameed priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL); 280db058a18SSaeed Mahameed if (!priv->msix_arr || !priv->irq_info) 281db058a18SSaeed Mahameed goto err_free_msix; 282e126ba97SEli Cohen 283e126ba97SEli Cohen for (i = 0; i < nvec; i++) 284db058a18SSaeed Mahameed priv->msix_arr[i].entry = i; 285e126ba97SEli Cohen 286db058a18SSaeed Mahameed nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 2873a9e161aSEli Cohen MLX5_EQ_VEC_COMP_BASE + 1, nvec); 288f3c9407bSAlexander Gordeev if (nvec < 0) 289f3c9407bSAlexander Gordeev return nvec; 290e126ba97SEli Cohen 291f3c9407bSAlexander Gordeev table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 292e126ba97SEli Cohen 293e126ba97SEli Cohen return 0; 294db058a18SSaeed Mahameed 295db058a18SSaeed Mahameed err_free_msix: 296db058a18SSaeed Mahameed kfree(priv->irq_info); 297db058a18SSaeed Mahameed kfree(priv->msix_arr); 298db058a18SSaeed Mahameed return -ENOMEM; 299e126ba97SEli Cohen } 300e126ba97SEli Cohen 301e126ba97SEli Cohen static void mlx5_disable_msix(struct mlx5_core_dev *dev) 302e126ba97SEli Cohen { 303db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 304e126ba97SEli Cohen 305e126ba97SEli Cohen pci_disable_msix(dev->pdev); 306db058a18SSaeed Mahameed kfree(priv->irq_info); 307db058a18SSaeed Mahameed kfree(priv->msix_arr); 308e126ba97SEli Cohen } 309e126ba97SEli Cohen 310e126ba97SEli Cohen struct mlx5_reg_host_endianess { 311e126ba97SEli Cohen u8 he; 312e126ba97SEli Cohen u8 rsvd[15]; 313e126ba97SEli Cohen }; 314e126ba97SEli Cohen 31587b8de49SEli Cohen 31687b8de49SEli Cohen #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 31787b8de49SEli Cohen 31887b8de49SEli Cohen enum { 31987b8de49SEli Cohen MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 320c7a08ac7SEli Cohen MLX5_DEV_CAP_FLAG_DCT, 32187b8de49SEli Cohen }; 32287b8de49SEli Cohen 323c7a08ac7SEli Cohen static u16 to_fw_pkey_sz(u32 size) 324c7a08ac7SEli Cohen { 325c7a08ac7SEli Cohen switch (size) { 326c7a08ac7SEli Cohen case 128: 327c7a08ac7SEli Cohen return 0; 328c7a08ac7SEli Cohen case 256: 329c7a08ac7SEli Cohen return 1; 330c7a08ac7SEli Cohen case 512: 331c7a08ac7SEli Cohen return 2; 332c7a08ac7SEli Cohen case 1024: 333c7a08ac7SEli Cohen return 3; 334c7a08ac7SEli Cohen case 2048: 335c7a08ac7SEli Cohen return 4; 336c7a08ac7SEli Cohen case 4096: 337c7a08ac7SEli Cohen return 5; 338c7a08ac7SEli Cohen default: 339c7a08ac7SEli Cohen pr_warn("invalid pkey table size %d\n", size); 340c7a08ac7SEli Cohen return 0; 341c7a08ac7SEli Cohen } 342c7a08ac7SEli Cohen } 343c7a08ac7SEli Cohen 344938fe83cSSaeed Mahameed int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, 345938fe83cSSaeed Mahameed enum mlx5_cap_mode cap_mode) 346c7a08ac7SEli Cohen { 347b775516bSEli Cohen u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 348b775516bSEli Cohen int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 349938fe83cSSaeed Mahameed void *out, *hca_caps; 350938fe83cSSaeed Mahameed u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 351c7a08ac7SEli Cohen int err; 352c7a08ac7SEli Cohen 353b775516bSEli Cohen memset(in, 0, sizeof(in)); 354b775516bSEli Cohen out = kzalloc(out_sz, GFP_KERNEL); 355c7a08ac7SEli Cohen if (!out) 356c7a08ac7SEli Cohen return -ENOMEM; 357938fe83cSSaeed Mahameed 358b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 359b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 360b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 361b775516bSEli Cohen if (err) 362b775516bSEli Cohen goto query_ex; 363c7a08ac7SEli Cohen 364b775516bSEli Cohen err = mlx5_cmd_status_to_err_v2(out); 365c7a08ac7SEli Cohen if (err) { 366938fe83cSSaeed Mahameed mlx5_core_warn(dev, 367938fe83cSSaeed Mahameed "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 368938fe83cSSaeed Mahameed cap_type, cap_mode, err); 369c7a08ac7SEli Cohen goto query_ex; 370c7a08ac7SEli Cohen } 371c7a08ac7SEli Cohen 372938fe83cSSaeed Mahameed hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 373938fe83cSSaeed Mahameed 374938fe83cSSaeed Mahameed switch (cap_mode) { 375938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_MAX: 376938fe83cSSaeed Mahameed memcpy(dev->hca_caps_max[cap_type], hca_caps, 377938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 378938fe83cSSaeed Mahameed break; 379938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_CUR: 380938fe83cSSaeed Mahameed memcpy(dev->hca_caps_cur[cap_type], hca_caps, 381938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 382938fe83cSSaeed Mahameed break; 383938fe83cSSaeed Mahameed default: 384938fe83cSSaeed Mahameed mlx5_core_warn(dev, 385938fe83cSSaeed Mahameed "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 386938fe83cSSaeed Mahameed cap_type, cap_mode); 387938fe83cSSaeed Mahameed err = -EINVAL; 388938fe83cSSaeed Mahameed break; 389938fe83cSSaeed Mahameed } 390c7a08ac7SEli Cohen query_ex: 391c7a08ac7SEli Cohen kfree(out); 392c7a08ac7SEli Cohen return err; 393c7a08ac7SEli Cohen } 394c7a08ac7SEli Cohen 395f91e6d89SEran Ben Elisha static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz, int opmod) 396c7a08ac7SEli Cohen { 397b775516bSEli Cohen u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)]; 398c7a08ac7SEli Cohen int err; 399c7a08ac7SEli Cohen 400b775516bSEli Cohen memset(out, 0, sizeof(out)); 401c7a08ac7SEli Cohen 402b775516bSEli Cohen MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 403f91e6d89SEran Ben Elisha MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1); 404b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 405c7a08ac7SEli Cohen if (err) 406c7a08ac7SEli Cohen return err; 407c7a08ac7SEli Cohen 408b775516bSEli Cohen err = mlx5_cmd_status_to_err_v2(out); 409c7a08ac7SEli Cohen 410c7a08ac7SEli Cohen return err; 411c7a08ac7SEli Cohen } 41287b8de49SEli Cohen 413f91e6d89SEran Ben Elisha static int handle_hca_cap_atomic(struct mlx5_core_dev *dev) 414f91e6d89SEran Ben Elisha { 415f91e6d89SEran Ben Elisha void *set_ctx; 416f91e6d89SEran Ben Elisha void *set_hca_cap; 417f91e6d89SEran Ben Elisha int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 418f91e6d89SEran Ben Elisha int req_endianness; 419f91e6d89SEran Ben Elisha int err; 420f91e6d89SEran Ben Elisha 421f91e6d89SEran Ben Elisha if (MLX5_CAP_GEN(dev, atomic)) { 422f91e6d89SEran Ben Elisha err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC, 423f91e6d89SEran Ben Elisha HCA_CAP_OPMOD_GET_CUR); 424f91e6d89SEran Ben Elisha if (err) 425f91e6d89SEran Ben Elisha return err; 426f91e6d89SEran Ben Elisha } else { 427f91e6d89SEran Ben Elisha return 0; 428f91e6d89SEran Ben Elisha } 429f91e6d89SEran Ben Elisha 430f91e6d89SEran Ben Elisha req_endianness = 431f91e6d89SEran Ben Elisha MLX5_CAP_ATOMIC(dev, 432f91e6d89SEran Ben Elisha supported_atomic_req_8B_endianess_mode_1); 433f91e6d89SEran Ben Elisha 434f91e6d89SEran Ben Elisha if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) 435f91e6d89SEran Ben Elisha return 0; 436f91e6d89SEran Ben Elisha 437f91e6d89SEran Ben Elisha set_ctx = kzalloc(set_sz, GFP_KERNEL); 438f91e6d89SEran Ben Elisha if (!set_ctx) 439f91e6d89SEran Ben Elisha return -ENOMEM; 440f91e6d89SEran Ben Elisha 441f91e6d89SEran Ben Elisha set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 442f91e6d89SEran Ben Elisha 443f91e6d89SEran Ben Elisha /* Set requestor to host endianness */ 444f91e6d89SEran Ben Elisha MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode, 445f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); 446f91e6d89SEran Ben Elisha 447f91e6d89SEran Ben Elisha err = set_caps(dev, set_ctx, set_sz, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC); 448f91e6d89SEran Ben Elisha 449f91e6d89SEran Ben Elisha kfree(set_ctx); 450f91e6d89SEran Ben Elisha return err; 451f91e6d89SEran Ben Elisha } 452f91e6d89SEran Ben Elisha 453e126ba97SEli Cohen static int handle_hca_cap(struct mlx5_core_dev *dev) 454e126ba97SEli Cohen { 455b775516bSEli Cohen void *set_ctx = NULL; 456c7a08ac7SEli Cohen struct mlx5_profile *prof = dev->profile; 457c7a08ac7SEli Cohen int err = -ENOMEM; 458b775516bSEli Cohen int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 459938fe83cSSaeed Mahameed void *set_hca_cap; 460e126ba97SEli Cohen 461b775516bSEli Cohen set_ctx = kzalloc(set_sz, GFP_KERNEL); 462c7a08ac7SEli Cohen if (!set_ctx) 463e126ba97SEli Cohen goto query_ex; 464e126ba97SEli Cohen 465938fe83cSSaeed Mahameed err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX); 466e126ba97SEli Cohen if (err) 467e126ba97SEli Cohen goto query_ex; 468e126ba97SEli Cohen 469938fe83cSSaeed Mahameed err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR); 470c7a08ac7SEli Cohen if (err) 471e126ba97SEli Cohen goto query_ex; 472e126ba97SEli Cohen 473938fe83cSSaeed Mahameed set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 474938fe83cSSaeed Mahameed capability); 475938fe83cSSaeed Mahameed memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], 476938fe83cSSaeed Mahameed MLX5_ST_SZ_BYTES(cmd_hca_cap)); 477938fe83cSSaeed Mahameed 478938fe83cSSaeed Mahameed mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 479707c4602SMajd Dibbiny mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 480938fe83cSSaeed Mahameed 128); 481c7a08ac7SEli Cohen /* we limit the size of the pkey table to 128 entries for now */ 482938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 483938fe83cSSaeed Mahameed to_fw_pkey_sz(128)); 484e126ba97SEli Cohen 485c7a08ac7SEli Cohen if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 486938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 487938fe83cSSaeed Mahameed prof->log_max_qp); 488e126ba97SEli Cohen 489938fe83cSSaeed Mahameed /* disable cmdif checksum */ 490938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 491c1868b82SEli Cohen 492fe1e1876SCarol L Soto MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 493fe1e1876SCarol L Soto 494f91e6d89SEran Ben Elisha err = set_caps(dev, set_ctx, set_sz, 495f91e6d89SEran Ben Elisha MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 496e126ba97SEli Cohen 497e126ba97SEli Cohen query_ex: 498e126ba97SEli Cohen kfree(set_ctx); 499e126ba97SEli Cohen return err; 500e126ba97SEli Cohen } 501e126ba97SEli Cohen 502e126ba97SEli Cohen static int set_hca_ctrl(struct mlx5_core_dev *dev) 503e126ba97SEli Cohen { 504e126ba97SEli Cohen struct mlx5_reg_host_endianess he_in; 505e126ba97SEli Cohen struct mlx5_reg_host_endianess he_out; 506e126ba97SEli Cohen int err; 507e126ba97SEli Cohen 508fc50db98SEli Cohen if (!mlx5_core_is_pf(dev)) 509fc50db98SEli Cohen return 0; 510fc50db98SEli Cohen 511e126ba97SEli Cohen memset(&he_in, 0, sizeof(he_in)); 512e126ba97SEli Cohen he_in.he = MLX5_SET_HOST_ENDIANNESS; 513e126ba97SEli Cohen err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 514e126ba97SEli Cohen &he_out, sizeof(he_out), 515e126ba97SEli Cohen MLX5_REG_HOST_ENDIANNESS, 0, 1); 516e126ba97SEli Cohen return err; 517e126ba97SEli Cohen } 518e126ba97SEli Cohen 5190b107106SEli Cohen int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id) 520cd23b14bSEli Cohen { 5210b107106SEli Cohen u32 out[MLX5_ST_SZ_DW(enable_hca_out)]; 5220b107106SEli Cohen u32 in[MLX5_ST_SZ_DW(enable_hca_in)]; 523cd23b14bSEli Cohen int err; 524cd23b14bSEli Cohen 5250b107106SEli Cohen memset(in, 0, sizeof(in)); 5260b107106SEli Cohen MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 5270b107106SEli Cohen MLX5_SET(enable_hca_in, in, function_id, func_id); 5280b107106SEli Cohen memset(out, 0, sizeof(out)); 5290b107106SEli Cohen 530cd23b14bSEli Cohen err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 531cd23b14bSEli Cohen if (err) 532cd23b14bSEli Cohen return err; 533cd23b14bSEli Cohen 5340b107106SEli Cohen return mlx5_cmd_status_to_err_v2(out); 535cd23b14bSEli Cohen } 536cd23b14bSEli Cohen 5370b107106SEli Cohen int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) 538cd23b14bSEli Cohen { 5390b107106SEli Cohen u32 out[MLX5_ST_SZ_DW(disable_hca_out)]; 5400b107106SEli Cohen u32 in[MLX5_ST_SZ_DW(disable_hca_in)]; 541cd23b14bSEli Cohen int err; 542cd23b14bSEli Cohen 5430b107106SEli Cohen memset(in, 0, sizeof(in)); 5440b107106SEli Cohen MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 5450b107106SEli Cohen MLX5_SET(disable_hca_in, in, function_id, func_id); 5460b107106SEli Cohen memset(out, 0, sizeof(out)); 5470b107106SEli Cohen err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 548cd23b14bSEli Cohen if (err) 549cd23b14bSEli Cohen return err; 550cd23b14bSEli Cohen 5510b107106SEli Cohen return mlx5_cmd_status_to_err_v2(out); 552cd23b14bSEli Cohen } 553cd23b14bSEli Cohen 554b0844444SEran Ben Elisha cycle_t mlx5_read_internal_timer(struct mlx5_core_dev *dev) 555b0844444SEran Ben Elisha { 556b0844444SEran Ben Elisha u32 timer_h, timer_h1, timer_l; 557b0844444SEran Ben Elisha 558b0844444SEran Ben Elisha timer_h = ioread32be(&dev->iseg->internal_timer_h); 559b0844444SEran Ben Elisha timer_l = ioread32be(&dev->iseg->internal_timer_l); 560b0844444SEran Ben Elisha timer_h1 = ioread32be(&dev->iseg->internal_timer_h); 561b0844444SEran Ben Elisha if (timer_h != timer_h1) /* wrap around */ 562b0844444SEran Ben Elisha timer_l = ioread32be(&dev->iseg->internal_timer_l); 563b0844444SEran Ben Elisha 564b0844444SEran Ben Elisha return (cycle_t)timer_l | (cycle_t)timer_h1 << 32; 565b0844444SEran Ben Elisha } 566b0844444SEran Ben Elisha 567db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) 568db058a18SSaeed Mahameed { 569db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 570db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 571db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 572311c7c71SSaeed Mahameed int numa_node = priv->numa_node; 573db058a18SSaeed Mahameed int err; 574db058a18SSaeed Mahameed 575db058a18SSaeed Mahameed if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) { 576db058a18SSaeed Mahameed mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); 577db058a18SSaeed Mahameed return -ENOMEM; 578db058a18SSaeed Mahameed } 579db058a18SSaeed Mahameed 580dda922c8SDavid S. Miller cpumask_set_cpu(cpumask_local_spread(i, numa_node), 581dda922c8SDavid S. Miller priv->irq_info[i].mask); 582db058a18SSaeed Mahameed 583db058a18SSaeed Mahameed err = irq_set_affinity_hint(irq, priv->irq_info[i].mask); 584db058a18SSaeed Mahameed if (err) { 585db058a18SSaeed Mahameed mlx5_core_warn(mdev, "irq_set_affinity_hint failed,irq 0x%.4x", 586db058a18SSaeed Mahameed irq); 587db058a18SSaeed Mahameed goto err_clear_mask; 588db058a18SSaeed Mahameed } 589db058a18SSaeed Mahameed 590db058a18SSaeed Mahameed return 0; 591db058a18SSaeed Mahameed 592db058a18SSaeed Mahameed err_clear_mask: 593db058a18SSaeed Mahameed free_cpumask_var(priv->irq_info[i].mask); 594db058a18SSaeed Mahameed return err; 595db058a18SSaeed Mahameed } 596db058a18SSaeed Mahameed 597db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i) 598db058a18SSaeed Mahameed { 599db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 600db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 601db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 602db058a18SSaeed Mahameed 603db058a18SSaeed Mahameed irq_set_affinity_hint(irq, NULL); 604db058a18SSaeed Mahameed free_cpumask_var(priv->irq_info[i].mask); 605db058a18SSaeed Mahameed } 606db058a18SSaeed Mahameed 607db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev) 608db058a18SSaeed Mahameed { 609db058a18SSaeed Mahameed int err; 610db058a18SSaeed Mahameed int i; 611db058a18SSaeed Mahameed 612db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) { 613db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hint(mdev, i); 614db058a18SSaeed Mahameed if (err) 615db058a18SSaeed Mahameed goto err_out; 616db058a18SSaeed Mahameed } 617db058a18SSaeed Mahameed 618db058a18SSaeed Mahameed return 0; 619db058a18SSaeed Mahameed 620db058a18SSaeed Mahameed err_out: 621db058a18SSaeed Mahameed for (i--; i >= 0; i--) 622db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 623db058a18SSaeed Mahameed 624db058a18SSaeed Mahameed return err; 625db058a18SSaeed Mahameed } 626db058a18SSaeed Mahameed 627db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev) 628db058a18SSaeed Mahameed { 629db058a18SSaeed Mahameed int i; 630db058a18SSaeed Mahameed 631db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) 632db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 633db058a18SSaeed Mahameed } 634db058a18SSaeed Mahameed 6350b6e26ceSDoron Tsur int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, 6360b6e26ceSDoron Tsur unsigned int *irqn) 637233d05d2SSaeed Mahameed { 638233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 639233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 640233d05d2SSaeed Mahameed int err = -ENOENT; 641233d05d2SSaeed Mahameed 642233d05d2SSaeed Mahameed spin_lock(&table->lock); 643233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 644233d05d2SSaeed Mahameed if (eq->index == vector) { 645233d05d2SSaeed Mahameed *eqn = eq->eqn; 646233d05d2SSaeed Mahameed *irqn = eq->irqn; 647233d05d2SSaeed Mahameed err = 0; 648233d05d2SSaeed Mahameed break; 649233d05d2SSaeed Mahameed } 650233d05d2SSaeed Mahameed } 651233d05d2SSaeed Mahameed spin_unlock(&table->lock); 652233d05d2SSaeed Mahameed 653233d05d2SSaeed Mahameed return err; 654233d05d2SSaeed Mahameed } 655233d05d2SSaeed Mahameed EXPORT_SYMBOL(mlx5_vector2eqn); 656233d05d2SSaeed Mahameed 657233d05d2SSaeed Mahameed static void free_comp_eqs(struct mlx5_core_dev *dev) 658233d05d2SSaeed Mahameed { 659233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 660233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 661233d05d2SSaeed Mahameed 662233d05d2SSaeed Mahameed spin_lock(&table->lock); 663233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 664233d05d2SSaeed Mahameed list_del(&eq->list); 665233d05d2SSaeed Mahameed spin_unlock(&table->lock); 666233d05d2SSaeed Mahameed if (mlx5_destroy_unmap_eq(dev, eq)) 667233d05d2SSaeed Mahameed mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 668233d05d2SSaeed Mahameed eq->eqn); 669233d05d2SSaeed Mahameed kfree(eq); 670233d05d2SSaeed Mahameed spin_lock(&table->lock); 671233d05d2SSaeed Mahameed } 672233d05d2SSaeed Mahameed spin_unlock(&table->lock); 673233d05d2SSaeed Mahameed } 674233d05d2SSaeed Mahameed 675233d05d2SSaeed Mahameed static int alloc_comp_eqs(struct mlx5_core_dev *dev) 676233d05d2SSaeed Mahameed { 677233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 678db058a18SSaeed Mahameed char name[MLX5_MAX_IRQ_NAME]; 679233d05d2SSaeed Mahameed struct mlx5_eq *eq; 680233d05d2SSaeed Mahameed int ncomp_vec; 681233d05d2SSaeed Mahameed int nent; 682233d05d2SSaeed Mahameed int err; 683233d05d2SSaeed Mahameed int i; 684233d05d2SSaeed Mahameed 685233d05d2SSaeed Mahameed INIT_LIST_HEAD(&table->comp_eqs_list); 686233d05d2SSaeed Mahameed ncomp_vec = table->num_comp_vectors; 687233d05d2SSaeed Mahameed nent = MLX5_COMP_EQ_SIZE; 688233d05d2SSaeed Mahameed for (i = 0; i < ncomp_vec; i++) { 689233d05d2SSaeed Mahameed eq = kzalloc(sizeof(*eq), GFP_KERNEL); 690233d05d2SSaeed Mahameed if (!eq) { 691233d05d2SSaeed Mahameed err = -ENOMEM; 692233d05d2SSaeed Mahameed goto clean; 693233d05d2SSaeed Mahameed } 694233d05d2SSaeed Mahameed 695db058a18SSaeed Mahameed snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 696233d05d2SSaeed Mahameed err = mlx5_create_map_eq(dev, eq, 697233d05d2SSaeed Mahameed i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 698233d05d2SSaeed Mahameed name, &dev->priv.uuari.uars[0]); 699233d05d2SSaeed Mahameed if (err) { 700233d05d2SSaeed Mahameed kfree(eq); 701233d05d2SSaeed Mahameed goto clean; 702233d05d2SSaeed Mahameed } 703233d05d2SSaeed Mahameed mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 704233d05d2SSaeed Mahameed eq->index = i; 705233d05d2SSaeed Mahameed spin_lock(&table->lock); 706233d05d2SSaeed Mahameed list_add_tail(&eq->list, &table->comp_eqs_list); 707233d05d2SSaeed Mahameed spin_unlock(&table->lock); 708233d05d2SSaeed Mahameed } 709233d05d2SSaeed Mahameed 710233d05d2SSaeed Mahameed return 0; 711233d05d2SSaeed Mahameed 712233d05d2SSaeed Mahameed clean: 713233d05d2SSaeed Mahameed free_comp_eqs(dev); 714233d05d2SSaeed Mahameed return err; 715233d05d2SSaeed Mahameed } 716233d05d2SSaeed Mahameed 717f62b8bb8SAmir Vadai static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 718f62b8bb8SAmir Vadai { 719f62b8bb8SAmir Vadai u32 query_in[MLX5_ST_SZ_DW(query_issi_in)]; 720f62b8bb8SAmir Vadai u32 query_out[MLX5_ST_SZ_DW(query_issi_out)]; 721f62b8bb8SAmir Vadai u32 set_in[MLX5_ST_SZ_DW(set_issi_in)]; 722f62b8bb8SAmir Vadai u32 set_out[MLX5_ST_SZ_DW(set_issi_out)]; 723f62b8bb8SAmir Vadai int err; 724f62b8bb8SAmir Vadai u32 sup_issi; 725f62b8bb8SAmir Vadai 726f62b8bb8SAmir Vadai memset(query_in, 0, sizeof(query_in)); 727f62b8bb8SAmir Vadai memset(query_out, 0, sizeof(query_out)); 728f62b8bb8SAmir Vadai 729f62b8bb8SAmir Vadai MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 730f62b8bb8SAmir Vadai 731f62b8bb8SAmir Vadai err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in), 732f62b8bb8SAmir Vadai query_out, sizeof(query_out)); 733f62b8bb8SAmir Vadai if (err) { 734f62b8bb8SAmir Vadai if (((struct mlx5_outbox_hdr *)query_out)->status == 735f62b8bb8SAmir Vadai MLX5_CMD_STAT_BAD_OP_ERR) { 736f62b8bb8SAmir Vadai pr_debug("Only ISSI 0 is supported\n"); 737f62b8bb8SAmir Vadai return 0; 738f62b8bb8SAmir Vadai } 739f62b8bb8SAmir Vadai 740f62b8bb8SAmir Vadai pr_err("failed to query ISSI\n"); 741f62b8bb8SAmir Vadai return err; 742f62b8bb8SAmir Vadai } 743f62b8bb8SAmir Vadai 744f62b8bb8SAmir Vadai sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 745f62b8bb8SAmir Vadai 746f62b8bb8SAmir Vadai if (sup_issi & (1 << 1)) { 747f62b8bb8SAmir Vadai memset(set_in, 0, sizeof(set_in)); 748f62b8bb8SAmir Vadai memset(set_out, 0, sizeof(set_out)); 749f62b8bb8SAmir Vadai 750f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 751f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, current_issi, 1); 752f62b8bb8SAmir Vadai 753f62b8bb8SAmir Vadai err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in), 754f62b8bb8SAmir Vadai set_out, sizeof(set_out)); 755f62b8bb8SAmir Vadai if (err) { 756f62b8bb8SAmir Vadai pr_err("failed to set ISSI=1\n"); 757f62b8bb8SAmir Vadai return err; 758f62b8bb8SAmir Vadai } 759f62b8bb8SAmir Vadai 760f62b8bb8SAmir Vadai dev->issi = 1; 761f62b8bb8SAmir Vadai 762f62b8bb8SAmir Vadai return 0; 763e74a1db0SHaggai Abramonvsky } else if (sup_issi & (1 << 0) || !sup_issi) { 764f62b8bb8SAmir Vadai return 0; 765f62b8bb8SAmir Vadai } 766f62b8bb8SAmir Vadai 767f62b8bb8SAmir Vadai return -ENOTSUPP; 768f62b8bb8SAmir Vadai } 769f62b8bb8SAmir Vadai 77088a85f99SAchiad Shochat static int map_bf_area(struct mlx5_core_dev *dev) 77188a85f99SAchiad Shochat { 77288a85f99SAchiad Shochat resource_size_t bf_start = pci_resource_start(dev->pdev, 0); 77388a85f99SAchiad Shochat resource_size_t bf_len = pci_resource_len(dev->pdev, 0); 77488a85f99SAchiad Shochat 77588a85f99SAchiad Shochat dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len); 77688a85f99SAchiad Shochat 77788a85f99SAchiad Shochat return dev->priv.bf_mapping ? 0 : -ENOMEM; 77888a85f99SAchiad Shochat } 77988a85f99SAchiad Shochat 78088a85f99SAchiad Shochat static void unmap_bf_area(struct mlx5_core_dev *dev) 78188a85f99SAchiad Shochat { 78288a85f99SAchiad Shochat if (dev->priv.bf_mapping) 78388a85f99SAchiad Shochat io_mapping_free(dev->priv.bf_mapping); 78488a85f99SAchiad Shochat } 78588a85f99SAchiad Shochat 786a31208b1SMajd Dibbiny static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 787a31208b1SMajd Dibbiny { 788a31208b1SMajd Dibbiny struct mlx5_device_context *dev_ctx; 789a31208b1SMajd Dibbiny struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 790a31208b1SMajd Dibbiny 791a31208b1SMajd Dibbiny dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL); 792a31208b1SMajd Dibbiny if (!dev_ctx) 793a31208b1SMajd Dibbiny return; 794a31208b1SMajd Dibbiny 795a31208b1SMajd Dibbiny dev_ctx->intf = intf; 796a31208b1SMajd Dibbiny dev_ctx->context = intf->add(dev); 797a31208b1SMajd Dibbiny 798a31208b1SMajd Dibbiny if (dev_ctx->context) { 799a31208b1SMajd Dibbiny spin_lock_irq(&priv->ctx_lock); 800a31208b1SMajd Dibbiny list_add_tail(&dev_ctx->list, &priv->ctx_list); 801a31208b1SMajd Dibbiny spin_unlock_irq(&priv->ctx_lock); 802a31208b1SMajd Dibbiny } else { 803a31208b1SMajd Dibbiny kfree(dev_ctx); 804a31208b1SMajd Dibbiny } 805a31208b1SMajd Dibbiny } 806a31208b1SMajd Dibbiny 807a31208b1SMajd Dibbiny static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 808a31208b1SMajd Dibbiny { 809a31208b1SMajd Dibbiny struct mlx5_device_context *dev_ctx; 810a31208b1SMajd Dibbiny struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 811a31208b1SMajd Dibbiny 812a31208b1SMajd Dibbiny list_for_each_entry(dev_ctx, &priv->ctx_list, list) 813a31208b1SMajd Dibbiny if (dev_ctx->intf == intf) { 814a31208b1SMajd Dibbiny spin_lock_irq(&priv->ctx_lock); 815a31208b1SMajd Dibbiny list_del(&dev_ctx->list); 816a31208b1SMajd Dibbiny spin_unlock_irq(&priv->ctx_lock); 817a31208b1SMajd Dibbiny 818a31208b1SMajd Dibbiny intf->remove(dev, dev_ctx->context); 819a31208b1SMajd Dibbiny kfree(dev_ctx); 820a31208b1SMajd Dibbiny return; 821a31208b1SMajd Dibbiny } 822a31208b1SMajd Dibbiny } 823a31208b1SMajd Dibbiny 824a31208b1SMajd Dibbiny static int mlx5_register_device(struct mlx5_core_dev *dev) 825e126ba97SEli Cohen { 826e126ba97SEli Cohen struct mlx5_priv *priv = &dev->priv; 827a31208b1SMajd Dibbiny struct mlx5_interface *intf; 828e126ba97SEli Cohen 829a31208b1SMajd Dibbiny mutex_lock(&intf_mutex); 830a31208b1SMajd Dibbiny list_add_tail(&priv->dev_list, &dev_list); 831a31208b1SMajd Dibbiny list_for_each_entry(intf, &intf_list, list) 832a31208b1SMajd Dibbiny mlx5_add_device(intf, priv); 833a31208b1SMajd Dibbiny mutex_unlock(&intf_mutex); 834a31208b1SMajd Dibbiny 835a31208b1SMajd Dibbiny return 0; 836a31208b1SMajd Dibbiny } 837a31208b1SMajd Dibbiny 838a31208b1SMajd Dibbiny static void mlx5_unregister_device(struct mlx5_core_dev *dev) 839a31208b1SMajd Dibbiny { 840a31208b1SMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 841a31208b1SMajd Dibbiny struct mlx5_interface *intf; 842a31208b1SMajd Dibbiny 843a31208b1SMajd Dibbiny mutex_lock(&intf_mutex); 844a31208b1SMajd Dibbiny list_for_each_entry(intf, &intf_list, list) 845a31208b1SMajd Dibbiny mlx5_remove_device(intf, priv); 846a31208b1SMajd Dibbiny list_del(&priv->dev_list); 847a31208b1SMajd Dibbiny mutex_unlock(&intf_mutex); 848a31208b1SMajd Dibbiny } 849a31208b1SMajd Dibbiny 850a31208b1SMajd Dibbiny int mlx5_register_interface(struct mlx5_interface *intf) 851a31208b1SMajd Dibbiny { 852a31208b1SMajd Dibbiny struct mlx5_priv *priv; 853a31208b1SMajd Dibbiny 854a31208b1SMajd Dibbiny if (!intf->add || !intf->remove) 855a31208b1SMajd Dibbiny return -EINVAL; 856a31208b1SMajd Dibbiny 857a31208b1SMajd Dibbiny mutex_lock(&intf_mutex); 858a31208b1SMajd Dibbiny list_add_tail(&intf->list, &intf_list); 859a31208b1SMajd Dibbiny list_for_each_entry(priv, &dev_list, dev_list) 860a31208b1SMajd Dibbiny mlx5_add_device(intf, priv); 861a31208b1SMajd Dibbiny mutex_unlock(&intf_mutex); 862a31208b1SMajd Dibbiny 863a31208b1SMajd Dibbiny return 0; 864a31208b1SMajd Dibbiny } 865a31208b1SMajd Dibbiny EXPORT_SYMBOL(mlx5_register_interface); 866a31208b1SMajd Dibbiny 867a31208b1SMajd Dibbiny void mlx5_unregister_interface(struct mlx5_interface *intf) 868a31208b1SMajd Dibbiny { 869a31208b1SMajd Dibbiny struct mlx5_priv *priv; 870a31208b1SMajd Dibbiny 871a31208b1SMajd Dibbiny mutex_lock(&intf_mutex); 872a31208b1SMajd Dibbiny list_for_each_entry(priv, &dev_list, dev_list) 873a31208b1SMajd Dibbiny mlx5_remove_device(intf, priv); 874a31208b1SMajd Dibbiny list_del(&intf->list); 875a31208b1SMajd Dibbiny mutex_unlock(&intf_mutex); 876a31208b1SMajd Dibbiny } 877a31208b1SMajd Dibbiny EXPORT_SYMBOL(mlx5_unregister_interface); 878a31208b1SMajd Dibbiny 879a31208b1SMajd Dibbiny void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) 880a31208b1SMajd Dibbiny { 881a31208b1SMajd Dibbiny struct mlx5_priv *priv = &mdev->priv; 882a31208b1SMajd Dibbiny struct mlx5_device_context *dev_ctx; 883a31208b1SMajd Dibbiny unsigned long flags; 884a31208b1SMajd Dibbiny void *result = NULL; 885a31208b1SMajd Dibbiny 886a31208b1SMajd Dibbiny spin_lock_irqsave(&priv->ctx_lock, flags); 887a31208b1SMajd Dibbiny 888a31208b1SMajd Dibbiny list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) 889a31208b1SMajd Dibbiny if ((dev_ctx->intf->protocol == protocol) && 890a31208b1SMajd Dibbiny dev_ctx->intf->get_dev) { 891a31208b1SMajd Dibbiny result = dev_ctx->intf->get_dev(dev_ctx->context); 892a31208b1SMajd Dibbiny break; 893a31208b1SMajd Dibbiny } 894a31208b1SMajd Dibbiny 895a31208b1SMajd Dibbiny spin_unlock_irqrestore(&priv->ctx_lock, flags); 896a31208b1SMajd Dibbiny 897a31208b1SMajd Dibbiny return result; 898a31208b1SMajd Dibbiny } 899a31208b1SMajd Dibbiny EXPORT_SYMBOL(mlx5_get_protocol_dev); 900a31208b1SMajd Dibbiny 901a31208b1SMajd Dibbiny static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 902a31208b1SMajd Dibbiny { 903a31208b1SMajd Dibbiny struct pci_dev *pdev = dev->pdev; 904a31208b1SMajd Dibbiny int err = 0; 905a31208b1SMajd Dibbiny 906e126ba97SEli Cohen pci_set_drvdata(dev->pdev, dev); 907e126ba97SEli Cohen strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 908e126ba97SEli Cohen priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 909e126ba97SEli Cohen 910e126ba97SEli Cohen mutex_init(&priv->pgdir_mutex); 911e126ba97SEli Cohen INIT_LIST_HEAD(&priv->pgdir_list); 912e126ba97SEli Cohen spin_lock_init(&priv->mkey_lock); 913e126ba97SEli Cohen 914311c7c71SSaeed Mahameed mutex_init(&priv->alloc_mutex); 915311c7c71SSaeed Mahameed 916311c7c71SSaeed Mahameed priv->numa_node = dev_to_node(&dev->pdev->dev); 917311c7c71SSaeed Mahameed 918e126ba97SEli Cohen priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root); 919e126ba97SEli Cohen if (!priv->dbg_root) 920e126ba97SEli Cohen return -ENOMEM; 921e126ba97SEli Cohen 92289d44f0aSMajd Dibbiny err = mlx5_pci_enable_device(dev); 923e126ba97SEli Cohen if (err) { 9241a91de28SJoe Perches dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 925e126ba97SEli Cohen goto err_dbg; 926e126ba97SEli Cohen } 927e126ba97SEli Cohen 928e126ba97SEli Cohen err = request_bar(pdev); 929e126ba97SEli Cohen if (err) { 9301a91de28SJoe Perches dev_err(&pdev->dev, "error requesting BARs, aborting\n"); 931e126ba97SEli Cohen goto err_disable; 932e126ba97SEli Cohen } 933e126ba97SEli Cohen 934e126ba97SEli Cohen pci_set_master(pdev); 935e126ba97SEli Cohen 936e126ba97SEli Cohen err = set_dma_caps(pdev); 937e126ba97SEli Cohen if (err) { 938e126ba97SEli Cohen dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); 939e126ba97SEli Cohen goto err_clr_master; 940e126ba97SEli Cohen } 941e126ba97SEli Cohen 942e126ba97SEli Cohen dev->iseg_base = pci_resource_start(dev->pdev, 0); 943e126ba97SEli Cohen dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 944e126ba97SEli Cohen if (!dev->iseg) { 945e126ba97SEli Cohen err = -ENOMEM; 946e126ba97SEli Cohen dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); 947e126ba97SEli Cohen goto err_clr_master; 948e126ba97SEli Cohen } 949a31208b1SMajd Dibbiny 950a31208b1SMajd Dibbiny return 0; 951a31208b1SMajd Dibbiny 952a31208b1SMajd Dibbiny err_clr_master: 953a31208b1SMajd Dibbiny pci_clear_master(dev->pdev); 954a31208b1SMajd Dibbiny release_bar(dev->pdev); 955a31208b1SMajd Dibbiny err_disable: 95689d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 957a31208b1SMajd Dibbiny 958a31208b1SMajd Dibbiny err_dbg: 959a31208b1SMajd Dibbiny debugfs_remove(priv->dbg_root); 960a31208b1SMajd Dibbiny return err; 961a31208b1SMajd Dibbiny } 962a31208b1SMajd Dibbiny 963a31208b1SMajd Dibbiny static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 964a31208b1SMajd Dibbiny { 965a31208b1SMajd Dibbiny iounmap(dev->iseg); 966a31208b1SMajd Dibbiny pci_clear_master(dev->pdev); 967a31208b1SMajd Dibbiny release_bar(dev->pdev); 96889d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 969a31208b1SMajd Dibbiny debugfs_remove(priv->dbg_root); 970a31208b1SMajd Dibbiny } 971a31208b1SMajd Dibbiny 972a31208b1SMajd Dibbiny #define MLX5_IB_MOD "mlx5_ib" 973a31208b1SMajd Dibbiny static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 974a31208b1SMajd Dibbiny { 975a31208b1SMajd Dibbiny struct pci_dev *pdev = dev->pdev; 976a31208b1SMajd Dibbiny int err; 977a31208b1SMajd Dibbiny 97889d44f0aSMajd Dibbiny mutex_lock(&dev->intf_state_mutex); 97989d44f0aSMajd Dibbiny if (dev->interface_state == MLX5_INTERFACE_STATE_UP) { 98089d44f0aSMajd Dibbiny dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", 98189d44f0aSMajd Dibbiny __func__); 98289d44f0aSMajd Dibbiny goto out; 98389d44f0aSMajd Dibbiny } 98489d44f0aSMajd Dibbiny 985e126ba97SEli Cohen dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 986e126ba97SEli Cohen fw_rev_min(dev), fw_rev_sub(dev)); 987e126ba97SEli Cohen 98889d44f0aSMajd Dibbiny /* on load removing any previous indication of internal error, device is 98989d44f0aSMajd Dibbiny * up 99089d44f0aSMajd Dibbiny */ 99189d44f0aSMajd Dibbiny dev->state = MLX5_DEVICE_STATE_UP; 99289d44f0aSMajd Dibbiny 993e126ba97SEli Cohen err = mlx5_cmd_init(dev); 994e126ba97SEli Cohen if (err) { 995e126ba97SEli Cohen dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); 99689d44f0aSMajd Dibbiny goto out_err; 997e126ba97SEli Cohen } 998e126ba97SEli Cohen 999e3297246SEli Cohen err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 1000e3297246SEli Cohen if (err) { 1001e3297246SEli Cohen dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n", 1002e3297246SEli Cohen FW_INIT_TIMEOUT_MILI); 1003e3297246SEli Cohen goto out_err; 1004e3297246SEli Cohen } 1005e3297246SEli Cohen 1006e126ba97SEli Cohen mlx5_pagealloc_init(dev); 1007cd23b14bSEli Cohen 10080b107106SEli Cohen err = mlx5_core_enable_hca(dev, 0); 1009cd23b14bSEli Cohen if (err) { 1010cd23b14bSEli Cohen dev_err(&pdev->dev, "enable hca failed\n"); 1011cd23b14bSEli Cohen goto err_pagealloc_cleanup; 1012cd23b14bSEli Cohen } 1013cd23b14bSEli Cohen 1014f62b8bb8SAmir Vadai err = mlx5_core_set_issi(dev); 1015f62b8bb8SAmir Vadai if (err) { 1016f62b8bb8SAmir Vadai dev_err(&pdev->dev, "failed to set issi\n"); 1017f62b8bb8SAmir Vadai goto err_disable_hca; 1018f62b8bb8SAmir Vadai } 1019f62b8bb8SAmir Vadai 1020cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 1); 1021cd23b14bSEli Cohen if (err) { 1022cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate boot pages\n"); 1023cd23b14bSEli Cohen goto err_disable_hca; 1024cd23b14bSEli Cohen } 1025cd23b14bSEli Cohen 1026e126ba97SEli Cohen err = set_hca_ctrl(dev); 1027e126ba97SEli Cohen if (err) { 1028e126ba97SEli Cohen dev_err(&pdev->dev, "set_hca_ctrl failed\n"); 1029cd23b14bSEli Cohen goto reclaim_boot_pages; 1030e126ba97SEli Cohen } 1031e126ba97SEli Cohen 1032e126ba97SEli Cohen err = handle_hca_cap(dev); 1033e126ba97SEli Cohen if (err) { 1034e126ba97SEli Cohen dev_err(&pdev->dev, "handle_hca_cap failed\n"); 1035cd23b14bSEli Cohen goto reclaim_boot_pages; 1036e126ba97SEli Cohen } 1037e126ba97SEli Cohen 1038f91e6d89SEran Ben Elisha err = handle_hca_cap_atomic(dev); 1039f91e6d89SEran Ben Elisha if (err) { 1040f91e6d89SEran Ben Elisha dev_err(&pdev->dev, "handle_hca_cap_atomic failed\n"); 1041f91e6d89SEran Ben Elisha goto reclaim_boot_pages; 1042f91e6d89SEran Ben Elisha } 1043f91e6d89SEran Ben Elisha 1044cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 0); 1045e126ba97SEli Cohen if (err) { 1046cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate init pages\n"); 1047cd23b14bSEli Cohen goto reclaim_boot_pages; 1048e126ba97SEli Cohen } 1049e126ba97SEli Cohen 1050e126ba97SEli Cohen err = mlx5_pagealloc_start(dev); 1051e126ba97SEli Cohen if (err) { 1052e126ba97SEli Cohen dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n"); 1053cd23b14bSEli Cohen goto reclaim_boot_pages; 1054e126ba97SEli Cohen } 1055e126ba97SEli Cohen 1056e126ba97SEli Cohen err = mlx5_cmd_init_hca(dev); 1057e126ba97SEli Cohen if (err) { 1058e126ba97SEli Cohen dev_err(&pdev->dev, "init hca failed\n"); 1059e126ba97SEli Cohen goto err_pagealloc_stop; 1060e126ba97SEli Cohen } 1061e126ba97SEli Cohen 1062e126ba97SEli Cohen mlx5_start_health_poll(dev); 1063e126ba97SEli Cohen 1064938fe83cSSaeed Mahameed err = mlx5_query_hca_caps(dev); 1065e126ba97SEli Cohen if (err) { 1066e126ba97SEli Cohen dev_err(&pdev->dev, "query hca failed\n"); 1067e126ba97SEli Cohen goto err_stop_poll; 1068e126ba97SEli Cohen } 1069e126ba97SEli Cohen 1070211e6c80SMajd Dibbiny err = mlx5_query_board_id(dev); 1071e126ba97SEli Cohen if (err) { 1072211e6c80SMajd Dibbiny dev_err(&pdev->dev, "query board id failed\n"); 1073e126ba97SEli Cohen goto err_stop_poll; 1074e126ba97SEli Cohen } 1075e126ba97SEli Cohen 1076e126ba97SEli Cohen err = mlx5_enable_msix(dev); 1077e126ba97SEli Cohen if (err) { 1078e126ba97SEli Cohen dev_err(&pdev->dev, "enable msix failed\n"); 1079e126ba97SEli Cohen goto err_stop_poll; 1080e126ba97SEli Cohen } 1081e126ba97SEli Cohen 1082e126ba97SEli Cohen err = mlx5_eq_init(dev); 1083e126ba97SEli Cohen if (err) { 1084e126ba97SEli Cohen dev_err(&pdev->dev, "failed to initialize eq\n"); 1085e126ba97SEli Cohen goto disable_msix; 1086e126ba97SEli Cohen } 1087e126ba97SEli Cohen 1088e126ba97SEli Cohen err = mlx5_alloc_uuars(dev, &priv->uuari); 1089e126ba97SEli Cohen if (err) { 1090e126ba97SEli Cohen dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 1091e126ba97SEli Cohen goto err_eq_cleanup; 1092e126ba97SEli Cohen } 1093e126ba97SEli Cohen 1094e126ba97SEli Cohen err = mlx5_start_eqs(dev); 1095e126ba97SEli Cohen if (err) { 1096e126ba97SEli Cohen dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 1097e126ba97SEli Cohen goto err_free_uar; 1098e126ba97SEli Cohen } 1099e126ba97SEli Cohen 1100233d05d2SSaeed Mahameed err = alloc_comp_eqs(dev); 1101233d05d2SSaeed Mahameed if (err) { 1102233d05d2SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc completion EQs\n"); 1103233d05d2SSaeed Mahameed goto err_stop_eqs; 1104233d05d2SSaeed Mahameed } 1105233d05d2SSaeed Mahameed 110688a85f99SAchiad Shochat if (map_bf_area(dev)) 110788a85f99SAchiad Shochat dev_err(&pdev->dev, "Failed to map blue flame area\n"); 110888a85f99SAchiad Shochat 1109db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hints(dev); 1110db058a18SSaeed Mahameed if (err) { 1111db058a18SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n"); 111288a85f99SAchiad Shochat goto err_unmap_bf_area; 1113db058a18SSaeed Mahameed } 1114db058a18SSaeed Mahameed 1115e126ba97SEli Cohen MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 1116e126ba97SEli Cohen 1117e126ba97SEli Cohen mlx5_init_cq_table(dev); 1118e126ba97SEli Cohen mlx5_init_qp_table(dev); 1119e126ba97SEli Cohen mlx5_init_srq_table(dev); 1120a606b0f6SMatan Barak mlx5_init_mkey_table(dev); 1121e126ba97SEli Cohen 112286d722adSMaor Gottlieb err = mlx5_init_fs(dev); 112386d722adSMaor Gottlieb if (err) { 112486d722adSMaor Gottlieb dev_err(&pdev->dev, "Failed to init flow steering\n"); 112586d722adSMaor Gottlieb goto err_fs; 112686d722adSMaor Gottlieb } 1127073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 1128073bb189SSaeed Mahameed err = mlx5_eswitch_init(dev); 1129073bb189SSaeed Mahameed if (err) { 1130073bb189SSaeed Mahameed dev_err(&pdev->dev, "eswitch init failed %d\n", err); 1131073bb189SSaeed Mahameed goto err_reg_dev; 1132073bb189SSaeed Mahameed } 1133073bb189SSaeed Mahameed #endif 1134073bb189SSaeed Mahameed 1135fc50db98SEli Cohen err = mlx5_sriov_init(dev); 1136fc50db98SEli Cohen if (err) { 1137fc50db98SEli Cohen dev_err(&pdev->dev, "sriov init failed %d\n", err); 1138fc50db98SEli Cohen goto err_sriov; 1139fc50db98SEli Cohen } 1140fc50db98SEli Cohen 1141a31208b1SMajd Dibbiny err = mlx5_register_device(dev); 1142a31208b1SMajd Dibbiny if (err) { 1143a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 1144a31208b1SMajd Dibbiny goto err_reg_dev; 1145a31208b1SMajd Dibbiny } 1146a31208b1SMajd Dibbiny 1147a31208b1SMajd Dibbiny err = request_module_nowait(MLX5_IB_MOD); 1148a31208b1SMajd Dibbiny if (err) 1149a31208b1SMajd Dibbiny pr_info("failed request module on %s\n", MLX5_IB_MOD); 1150a31208b1SMajd Dibbiny 115189d44f0aSMajd Dibbiny dev->interface_state = MLX5_INTERFACE_STATE_UP; 115289d44f0aSMajd Dibbiny out: 115389d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 115489d44f0aSMajd Dibbiny 1155e126ba97SEli Cohen return 0; 1156e126ba97SEli Cohen 1157fc50db98SEli Cohen err_sriov: 1158fc50db98SEli Cohen if (mlx5_sriov_cleanup(dev)) 1159fc50db98SEli Cohen dev_err(&dev->pdev->dev, "sriov cleanup failed\n"); 1160fc50db98SEli Cohen 1161073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 1162073bb189SSaeed Mahameed mlx5_eswitch_cleanup(dev->priv.eswitch); 1163073bb189SSaeed Mahameed #endif 1164a31208b1SMajd Dibbiny err_reg_dev: 116586d722adSMaor Gottlieb mlx5_cleanup_fs(dev); 116686d722adSMaor Gottlieb err_fs: 1167a606b0f6SMatan Barak mlx5_cleanup_mkey_table(dev); 1168a31208b1SMajd Dibbiny mlx5_cleanup_srq_table(dev); 1169a31208b1SMajd Dibbiny mlx5_cleanup_qp_table(dev); 1170a31208b1SMajd Dibbiny mlx5_cleanup_cq_table(dev); 1171a31208b1SMajd Dibbiny mlx5_irq_clear_affinity_hints(dev); 1172a31208b1SMajd Dibbiny 117388a85f99SAchiad Shochat err_unmap_bf_area: 117488a85f99SAchiad Shochat unmap_bf_area(dev); 117588a85f99SAchiad Shochat 1176db058a18SSaeed Mahameed free_comp_eqs(dev); 1177db058a18SSaeed Mahameed 1178233d05d2SSaeed Mahameed err_stop_eqs: 1179233d05d2SSaeed Mahameed mlx5_stop_eqs(dev); 1180233d05d2SSaeed Mahameed 1181e126ba97SEli Cohen err_free_uar: 1182e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 1183e126ba97SEli Cohen 1184e126ba97SEli Cohen err_eq_cleanup: 1185e126ba97SEli Cohen mlx5_eq_cleanup(dev); 1186e126ba97SEli Cohen 1187e126ba97SEli Cohen disable_msix: 1188e126ba97SEli Cohen mlx5_disable_msix(dev); 1189e126ba97SEli Cohen 1190e126ba97SEli Cohen err_stop_poll: 1191e126ba97SEli Cohen mlx5_stop_health_poll(dev); 11921bde6e30SEli Cohen if (mlx5_cmd_teardown_hca(dev)) { 11931bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 119489d44f0aSMajd Dibbiny goto out_err; 11951bde6e30SEli Cohen } 1196e126ba97SEli Cohen 1197e126ba97SEli Cohen err_pagealloc_stop: 1198e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 1199e126ba97SEli Cohen 1200cd23b14bSEli Cohen reclaim_boot_pages: 1201e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 1202e126ba97SEli Cohen 1203cd23b14bSEli Cohen err_disable_hca: 12040b107106SEli Cohen mlx5_core_disable_hca(dev, 0); 1205cd23b14bSEli Cohen 1206e126ba97SEli Cohen err_pagealloc_cleanup: 1207e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 1208e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 1209e126ba97SEli Cohen 121089d44f0aSMajd Dibbiny out_err: 121189d44f0aSMajd Dibbiny dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 121289d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 121389d44f0aSMajd Dibbiny 1214e126ba97SEli Cohen return err; 1215e126ba97SEli Cohen } 1216e126ba97SEli Cohen 1217a31208b1SMajd Dibbiny static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 1218e126ba97SEli Cohen { 121989d44f0aSMajd Dibbiny int err = 0; 1220e126ba97SEli Cohen 1221fc50db98SEli Cohen err = mlx5_sriov_cleanup(dev); 1222fc50db98SEli Cohen if (err) { 1223fc50db98SEli Cohen dev_warn(&dev->pdev->dev, "%s: sriov cleanup failed - abort\n", 1224fc50db98SEli Cohen __func__); 1225fc50db98SEli Cohen return err; 1226fc50db98SEli Cohen } 1227fc50db98SEli Cohen 122889d44f0aSMajd Dibbiny mutex_lock(&dev->intf_state_mutex); 122989d44f0aSMajd Dibbiny if (dev->interface_state == MLX5_INTERFACE_STATE_DOWN) { 123089d44f0aSMajd Dibbiny dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", 123189d44f0aSMajd Dibbiny __func__); 123289d44f0aSMajd Dibbiny goto out; 123389d44f0aSMajd Dibbiny } 1234a31208b1SMajd Dibbiny mlx5_unregister_device(dev); 1235073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 1236073bb189SSaeed Mahameed mlx5_eswitch_cleanup(dev->priv.eswitch); 1237073bb189SSaeed Mahameed #endif 1238073bb189SSaeed Mahameed 123986d722adSMaor Gottlieb mlx5_cleanup_fs(dev); 1240a606b0f6SMatan Barak mlx5_cleanup_mkey_table(dev); 1241e126ba97SEli Cohen mlx5_cleanup_srq_table(dev); 1242e126ba97SEli Cohen mlx5_cleanup_qp_table(dev); 1243e126ba97SEli Cohen mlx5_cleanup_cq_table(dev); 1244db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hints(dev); 124588a85f99SAchiad Shochat unmap_bf_area(dev); 1246233d05d2SSaeed Mahameed free_comp_eqs(dev); 1247e126ba97SEli Cohen mlx5_stop_eqs(dev); 1248e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 1249e126ba97SEli Cohen mlx5_eq_cleanup(dev); 1250e126ba97SEli Cohen mlx5_disable_msix(dev); 1251e126ba97SEli Cohen mlx5_stop_health_poll(dev); 1252ac6ea6e8SEli Cohen err = mlx5_cmd_teardown_hca(dev); 1253ac6ea6e8SEli Cohen if (err) { 12541bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 1255ac6ea6e8SEli Cohen goto out; 12561bde6e30SEli Cohen } 1257e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 1258e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 12590b107106SEli Cohen mlx5_core_disable_hca(dev, 0); 1260e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 1261e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 12629603b61dSJack Morgenstein 1263ac6ea6e8SEli Cohen out: 126489d44f0aSMajd Dibbiny dev->interface_state = MLX5_INTERFACE_STATE_DOWN; 126589d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 1266ac6ea6e8SEli Cohen return err; 12679603b61dSJack Morgenstein } 126864613d94SSaeed Mahameed 126989d44f0aSMajd Dibbiny void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 12704d2f9bbbSJack Morgenstein unsigned long param) 12719603b61dSJack Morgenstein { 12729603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 12739603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 12749603b61dSJack Morgenstein unsigned long flags; 12759603b61dSJack Morgenstein 12769603b61dSJack Morgenstein spin_lock_irqsave(&priv->ctx_lock, flags); 12779603b61dSJack Morgenstein 12789603b61dSJack Morgenstein list_for_each_entry(dev_ctx, &priv->ctx_list, list) 12799603b61dSJack Morgenstein if (dev_ctx->intf->event) 12804d2f9bbbSJack Morgenstein dev_ctx->intf->event(dev, dev_ctx->context, event, param); 12819603b61dSJack Morgenstein 12829603b61dSJack Morgenstein spin_unlock_irqrestore(&priv->ctx_lock, flags); 12839603b61dSJack Morgenstein } 12849603b61dSJack Morgenstein 12859603b61dSJack Morgenstein struct mlx5_core_event_handler { 12869603b61dSJack Morgenstein void (*event)(struct mlx5_core_dev *dev, 12879603b61dSJack Morgenstein enum mlx5_dev_event event, 12889603b61dSJack Morgenstein void *data); 12899603b61dSJack Morgenstein }; 12909603b61dSJack Morgenstein 1291f66f049fSEli Cohen 12929603b61dSJack Morgenstein static int init_one(struct pci_dev *pdev, 12939603b61dSJack Morgenstein const struct pci_device_id *id) 12949603b61dSJack Morgenstein { 12959603b61dSJack Morgenstein struct mlx5_core_dev *dev; 12969603b61dSJack Morgenstein struct mlx5_priv *priv; 12979603b61dSJack Morgenstein int err; 12989603b61dSJack Morgenstein 12999603b61dSJack Morgenstein dev = kzalloc(sizeof(*dev), GFP_KERNEL); 13009603b61dSJack Morgenstein if (!dev) { 13019603b61dSJack Morgenstein dev_err(&pdev->dev, "kzalloc failed\n"); 13029603b61dSJack Morgenstein return -ENOMEM; 13039603b61dSJack Morgenstein } 13049603b61dSJack Morgenstein priv = &dev->priv; 1305fc50db98SEli Cohen priv->pci_dev_data = id->driver_data; 13069603b61dSJack Morgenstein 13079603b61dSJack Morgenstein pci_set_drvdata(pdev, dev); 13089603b61dSJack Morgenstein 13099603b61dSJack Morgenstein if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profile)) { 13109603b61dSJack Morgenstein pr_warn("selected profile out of range, selecting default (%d)\n", 13119603b61dSJack Morgenstein MLX5_DEFAULT_PROF); 13129603b61dSJack Morgenstein prof_sel = MLX5_DEFAULT_PROF; 13139603b61dSJack Morgenstein } 13149603b61dSJack Morgenstein dev->profile = &profile[prof_sel]; 1315a31208b1SMajd Dibbiny dev->pdev = pdev; 13169603b61dSJack Morgenstein dev->event = mlx5_core_event; 13179603b61dSJack Morgenstein 1318364d1798SEli Cohen INIT_LIST_HEAD(&priv->ctx_list); 1319364d1798SEli Cohen spin_lock_init(&priv->ctx_lock); 132089d44f0aSMajd Dibbiny mutex_init(&dev->pci_status_mutex); 132189d44f0aSMajd Dibbiny mutex_init(&dev->intf_state_mutex); 1322a31208b1SMajd Dibbiny err = mlx5_pci_init(dev, priv); 13239603b61dSJack Morgenstein if (err) { 1324a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err); 1325a31208b1SMajd Dibbiny goto clean_dev; 13269603b61dSJack Morgenstein } 13279603b61dSJack Morgenstein 1328ac6ea6e8SEli Cohen err = mlx5_health_init(dev); 1329ac6ea6e8SEli Cohen if (err) { 1330ac6ea6e8SEli Cohen dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err); 1331ac6ea6e8SEli Cohen goto close_pci; 1332ac6ea6e8SEli Cohen } 1333ac6ea6e8SEli Cohen 1334a31208b1SMajd Dibbiny err = mlx5_load_one(dev, priv); 13359603b61dSJack Morgenstein if (err) { 1336a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err); 1337ac6ea6e8SEli Cohen goto clean_health; 13389603b61dSJack Morgenstein } 13399603b61dSJack Morgenstein 13409603b61dSJack Morgenstein return 0; 13419603b61dSJack Morgenstein 1342ac6ea6e8SEli Cohen clean_health: 1343ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1344a31208b1SMajd Dibbiny close_pci: 1345a31208b1SMajd Dibbiny mlx5_pci_close(dev, priv); 1346a31208b1SMajd Dibbiny clean_dev: 1347a31208b1SMajd Dibbiny pci_set_drvdata(pdev, NULL); 13489603b61dSJack Morgenstein kfree(dev); 1349a31208b1SMajd Dibbiny 13509603b61dSJack Morgenstein return err; 13519603b61dSJack Morgenstein } 1352a31208b1SMajd Dibbiny 13539603b61dSJack Morgenstein static void remove_one(struct pci_dev *pdev) 13549603b61dSJack Morgenstein { 13559603b61dSJack Morgenstein struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1356a31208b1SMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 13579603b61dSJack Morgenstein 1358a31208b1SMajd Dibbiny if (mlx5_unload_one(dev, priv)) { 1359a31208b1SMajd Dibbiny dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); 1360ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1361a31208b1SMajd Dibbiny return; 1362a31208b1SMajd Dibbiny } 1363ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1364a31208b1SMajd Dibbiny mlx5_pci_close(dev, priv); 1365a31208b1SMajd Dibbiny pci_set_drvdata(pdev, NULL); 13669603b61dSJack Morgenstein kfree(dev); 13679603b61dSJack Morgenstein } 13689603b61dSJack Morgenstein 136989d44f0aSMajd Dibbiny static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, 137089d44f0aSMajd Dibbiny pci_channel_state_t state) 137189d44f0aSMajd Dibbiny { 137289d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 137389d44f0aSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 137489d44f0aSMajd Dibbiny 137589d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 137689d44f0aSMajd Dibbiny mlx5_enter_error_state(dev); 137789d44f0aSMajd Dibbiny mlx5_unload_one(dev, priv); 137889d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 137989d44f0aSMajd Dibbiny return state == pci_channel_io_perm_failure ? 138089d44f0aSMajd Dibbiny PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 138189d44f0aSMajd Dibbiny } 138289d44f0aSMajd Dibbiny 138389d44f0aSMajd Dibbiny static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) 138489d44f0aSMajd Dibbiny { 138589d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 138689d44f0aSMajd Dibbiny int err = 0; 138789d44f0aSMajd Dibbiny 138889d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 138989d44f0aSMajd Dibbiny 139089d44f0aSMajd Dibbiny err = mlx5_pci_enable_device(dev); 139189d44f0aSMajd Dibbiny if (err) { 139289d44f0aSMajd Dibbiny dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" 139389d44f0aSMajd Dibbiny , __func__, err); 139489d44f0aSMajd Dibbiny return PCI_ERS_RESULT_DISCONNECT; 139589d44f0aSMajd Dibbiny } 139689d44f0aSMajd Dibbiny pci_set_master(pdev); 139789d44f0aSMajd Dibbiny pci_set_power_state(pdev, PCI_D0); 139889d44f0aSMajd Dibbiny pci_restore_state(pdev); 139989d44f0aSMajd Dibbiny 140089d44f0aSMajd Dibbiny return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 140189d44f0aSMajd Dibbiny } 140289d44f0aSMajd Dibbiny 140389d44f0aSMajd Dibbiny void mlx5_disable_device(struct mlx5_core_dev *dev) 140489d44f0aSMajd Dibbiny { 140589d44f0aSMajd Dibbiny mlx5_pci_err_detected(dev->pdev, 0); 140689d44f0aSMajd Dibbiny } 140789d44f0aSMajd Dibbiny 140889d44f0aSMajd Dibbiny /* wait for the device to show vital signs. For now we check 140989d44f0aSMajd Dibbiny * that we can read the device ID and that the health buffer 141089d44f0aSMajd Dibbiny * shows a non zero value which is different than 0xffffffff 141189d44f0aSMajd Dibbiny */ 141289d44f0aSMajd Dibbiny static void wait_vital(struct pci_dev *pdev) 141389d44f0aSMajd Dibbiny { 141489d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 141589d44f0aSMajd Dibbiny struct mlx5_core_health *health = &dev->priv.health; 141689d44f0aSMajd Dibbiny const int niter = 100; 141789d44f0aSMajd Dibbiny u32 count; 141889d44f0aSMajd Dibbiny u16 did; 141989d44f0aSMajd Dibbiny int i; 142089d44f0aSMajd Dibbiny 142189d44f0aSMajd Dibbiny /* Wait for firmware to be ready after reset */ 142289d44f0aSMajd Dibbiny msleep(1000); 142389d44f0aSMajd Dibbiny for (i = 0; i < niter; i++) { 142489d44f0aSMajd Dibbiny if (pci_read_config_word(pdev, 2, &did)) { 142589d44f0aSMajd Dibbiny dev_warn(&pdev->dev, "failed reading config word\n"); 142689d44f0aSMajd Dibbiny break; 142789d44f0aSMajd Dibbiny } 142889d44f0aSMajd Dibbiny if (did == pdev->device) { 142989d44f0aSMajd Dibbiny dev_info(&pdev->dev, "device ID correctly read after %d iterations\n", i); 143089d44f0aSMajd Dibbiny break; 143189d44f0aSMajd Dibbiny } 143289d44f0aSMajd Dibbiny msleep(50); 143389d44f0aSMajd Dibbiny } 143489d44f0aSMajd Dibbiny if (i == niter) 143589d44f0aSMajd Dibbiny dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); 143689d44f0aSMajd Dibbiny 143789d44f0aSMajd Dibbiny for (i = 0; i < niter; i++) { 143889d44f0aSMajd Dibbiny count = ioread32be(health->health_counter); 143989d44f0aSMajd Dibbiny if (count && count != 0xffffffff) { 144089d44f0aSMajd Dibbiny dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); 144189d44f0aSMajd Dibbiny break; 144289d44f0aSMajd Dibbiny } 144389d44f0aSMajd Dibbiny msleep(50); 144489d44f0aSMajd Dibbiny } 144589d44f0aSMajd Dibbiny 144689d44f0aSMajd Dibbiny if (i == niter) 144789d44f0aSMajd Dibbiny dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); 144889d44f0aSMajd Dibbiny } 144989d44f0aSMajd Dibbiny 145089d44f0aSMajd Dibbiny static void mlx5_pci_resume(struct pci_dev *pdev) 145189d44f0aSMajd Dibbiny { 145289d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 145389d44f0aSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 145489d44f0aSMajd Dibbiny int err; 145589d44f0aSMajd Dibbiny 145689d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 145789d44f0aSMajd Dibbiny 145889d44f0aSMajd Dibbiny pci_save_state(pdev); 145989d44f0aSMajd Dibbiny wait_vital(pdev); 146089d44f0aSMajd Dibbiny 146189d44f0aSMajd Dibbiny err = mlx5_load_one(dev, priv); 146289d44f0aSMajd Dibbiny if (err) 146389d44f0aSMajd Dibbiny dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" 146489d44f0aSMajd Dibbiny , __func__, err); 146589d44f0aSMajd Dibbiny else 146689d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s: device recovered\n", __func__); 146789d44f0aSMajd Dibbiny } 146889d44f0aSMajd Dibbiny 146989d44f0aSMajd Dibbiny static const struct pci_error_handlers mlx5_err_handler = { 147089d44f0aSMajd Dibbiny .error_detected = mlx5_pci_err_detected, 147189d44f0aSMajd Dibbiny .slot_reset = mlx5_pci_slot_reset, 147289d44f0aSMajd Dibbiny .resume = mlx5_pci_resume 147389d44f0aSMajd Dibbiny }; 147489d44f0aSMajd Dibbiny 14759603b61dSJack Morgenstein static const struct pci_device_id mlx5_core_pci_table[] = { 14761c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */ 1477fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */ 14781c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */ 1479fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */ 14801c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */ 1481fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */ 14829603b61dSJack Morgenstein { 0, } 14839603b61dSJack Morgenstein }; 14849603b61dSJack Morgenstein 14859603b61dSJack Morgenstein MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 14869603b61dSJack Morgenstein 14879603b61dSJack Morgenstein static struct pci_driver mlx5_core_driver = { 14889603b61dSJack Morgenstein .name = DRIVER_NAME, 14899603b61dSJack Morgenstein .id_table = mlx5_core_pci_table, 14909603b61dSJack Morgenstein .probe = init_one, 149189d44f0aSMajd Dibbiny .remove = remove_one, 1492fc50db98SEli Cohen .err_handler = &mlx5_err_handler, 1493fc50db98SEli Cohen .sriov_configure = mlx5_core_sriov_configure, 14949603b61dSJack Morgenstein }; 1495e126ba97SEli Cohen 1496e126ba97SEli Cohen static int __init init(void) 1497e126ba97SEli Cohen { 1498e126ba97SEli Cohen int err; 1499e126ba97SEli Cohen 1500e126ba97SEli Cohen mlx5_register_debugfs(); 1501e126ba97SEli Cohen 15029603b61dSJack Morgenstein err = pci_register_driver(&mlx5_core_driver); 15039603b61dSJack Morgenstein if (err) 1504ac6ea6e8SEli Cohen goto err_debug; 15059603b61dSJack Morgenstein 1506f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1507f62b8bb8SAmir Vadai mlx5e_init(); 1508f62b8bb8SAmir Vadai #endif 1509f62b8bb8SAmir Vadai 1510e126ba97SEli Cohen return 0; 1511e126ba97SEli Cohen 1512e126ba97SEli Cohen err_debug: 1513e126ba97SEli Cohen mlx5_unregister_debugfs(); 1514e126ba97SEli Cohen return err; 1515e126ba97SEli Cohen } 1516e126ba97SEli Cohen 1517e126ba97SEli Cohen static void __exit cleanup(void) 1518e126ba97SEli Cohen { 1519f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1520f62b8bb8SAmir Vadai mlx5e_cleanup(); 1521f62b8bb8SAmir Vadai #endif 15229603b61dSJack Morgenstein pci_unregister_driver(&mlx5_core_driver); 1523e126ba97SEli Cohen mlx5_unregister_debugfs(); 1524e126ba97SEli Cohen } 1525e126ba97SEli Cohen 1526e126ba97SEli Cohen module_init(init); 1527e126ba97SEli Cohen module_exit(cleanup); 1528