1e126ba97SEli Cohen /* 2302bdf68SSaeed Mahameed * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3e126ba97SEli Cohen * 4e126ba97SEli Cohen * This software is available to you under a choice of one of two 5e126ba97SEli Cohen * licenses. You may choose to be licensed under the terms of the GNU 6e126ba97SEli Cohen * General Public License (GPL) Version 2, available from the file 7e126ba97SEli Cohen * COPYING in the main directory of this source tree, or the 8e126ba97SEli Cohen * OpenIB.org BSD license below: 9e126ba97SEli Cohen * 10e126ba97SEli Cohen * Redistribution and use in source and binary forms, with or 11e126ba97SEli Cohen * without modification, are permitted provided that the following 12e126ba97SEli Cohen * conditions are met: 13e126ba97SEli Cohen * 14e126ba97SEli Cohen * - Redistributions of source code must retain the above 15e126ba97SEli Cohen * copyright notice, this list of conditions and the following 16e126ba97SEli Cohen * disclaimer. 17e126ba97SEli Cohen * 18e126ba97SEli Cohen * - Redistributions in binary form must reproduce the above 19e126ba97SEli Cohen * copyright notice, this list of conditions and the following 20e126ba97SEli Cohen * disclaimer in the documentation and/or other materials 21e126ba97SEli Cohen * provided with the distribution. 22e126ba97SEli Cohen * 23e126ba97SEli Cohen * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e126ba97SEli Cohen * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e126ba97SEli Cohen * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e126ba97SEli Cohen * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e126ba97SEli Cohen * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e126ba97SEli Cohen * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e126ba97SEli Cohen * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e126ba97SEli Cohen * SOFTWARE. 31e126ba97SEli Cohen */ 32e126ba97SEli Cohen 33adec640eSChristoph Hellwig #include <linux/highmem.h> 34e126ba97SEli Cohen #include <linux/module.h> 35e126ba97SEli Cohen #include <linux/init.h> 36e126ba97SEli Cohen #include <linux/errno.h> 37e126ba97SEli Cohen #include <linux/pci.h> 38e126ba97SEli Cohen #include <linux/dma-mapping.h> 39e126ba97SEli Cohen #include <linux/slab.h> 40e126ba97SEli Cohen #include <linux/io-mapping.h> 41db058a18SSaeed Mahameed #include <linux/interrupt.h> 42e3297246SEli Cohen #include <linux/delay.h> 43e126ba97SEli Cohen #include <linux/mlx5/driver.h> 44e126ba97SEli Cohen #include <linux/mlx5/cq.h> 45e126ba97SEli Cohen #include <linux/mlx5/qp.h> 46e126ba97SEli Cohen #include <linux/mlx5/srq.h> 47e126ba97SEli Cohen #include <linux/debugfs.h> 48f66f049fSEli Cohen #include <linux/kmod.h> 4989d44f0aSMajd Dibbiny #include <linux/delay.h> 50b775516bSEli Cohen #include <linux/mlx5/mlx5_ifc.h> 515a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 525a7b27ebSMaor Gottlieb #include <linux/cpu_rmap.h> 535a7b27ebSMaor Gottlieb #endif 54feae9087SOr Gerlitz #include <net/devlink.h> 55e126ba97SEli Cohen #include "mlx5_core.h" 5686d722adSMaor Gottlieb #include "fs_core.h" 57073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 58073bb189SSaeed Mahameed #include "eswitch.h" 59073bb189SSaeed Mahameed #endif 60e126ba97SEli Cohen 61e126ba97SEli Cohen MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 624ae6c18cSAchiad Shochat MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 63e126ba97SEli Cohen MODULE_LICENSE("Dual BSD/GPL"); 64e126ba97SEli Cohen MODULE_VERSION(DRIVER_VERSION); 65e126ba97SEli Cohen 66e126ba97SEli Cohen int mlx5_core_debug_mask; 67e126ba97SEli Cohen module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 68e126ba97SEli Cohen MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 69e126ba97SEli Cohen 709603b61dSJack Morgenstein #define MLX5_DEFAULT_PROF 2 719603b61dSJack Morgenstein static int prof_sel = MLX5_DEFAULT_PROF; 729603b61dSJack Morgenstein module_param_named(prof_sel, prof_sel, int, 0444); 739603b61dSJack Morgenstein MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 749603b61dSJack Morgenstein 759603b61dSJack Morgenstein static LIST_HEAD(intf_list); 767907f23aSAviv Heller 777907f23aSAviv Heller LIST_HEAD(mlx5_dev_list); 787907f23aSAviv Heller DEFINE_MUTEX(mlx5_intf_mutex); 799603b61dSJack Morgenstein 809603b61dSJack Morgenstein struct mlx5_device_context { 819603b61dSJack Morgenstein struct list_head list; 829603b61dSJack Morgenstein struct mlx5_interface *intf; 839603b61dSJack Morgenstein void *context; 849603b61dSJack Morgenstein }; 859603b61dSJack Morgenstein 86f91e6d89SEran Ben Elisha enum { 87f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_BE = 0x0, 88f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, 89f91e6d89SEran Ben Elisha }; 90f91e6d89SEran Ben Elisha 919603b61dSJack Morgenstein static struct mlx5_profile profile[] = { 929603b61dSJack Morgenstein [0] = { 939603b61dSJack Morgenstein .mask = 0, 949603b61dSJack Morgenstein }, 959603b61dSJack Morgenstein [1] = { 969603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE, 979603b61dSJack Morgenstein .log_max_qp = 12, 989603b61dSJack Morgenstein }, 999603b61dSJack Morgenstein [2] = { 1009603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE | 1019603b61dSJack Morgenstein MLX5_PROF_MASK_MR_CACHE, 1029603b61dSJack Morgenstein .log_max_qp = 17, 1039603b61dSJack Morgenstein .mr_cache[0] = { 1049603b61dSJack Morgenstein .size = 500, 1059603b61dSJack Morgenstein .limit = 250 1069603b61dSJack Morgenstein }, 1079603b61dSJack Morgenstein .mr_cache[1] = { 1089603b61dSJack Morgenstein .size = 500, 1099603b61dSJack Morgenstein .limit = 250 1109603b61dSJack Morgenstein }, 1119603b61dSJack Morgenstein .mr_cache[2] = { 1129603b61dSJack Morgenstein .size = 500, 1139603b61dSJack Morgenstein .limit = 250 1149603b61dSJack Morgenstein }, 1159603b61dSJack Morgenstein .mr_cache[3] = { 1169603b61dSJack Morgenstein .size = 500, 1179603b61dSJack Morgenstein .limit = 250 1189603b61dSJack Morgenstein }, 1199603b61dSJack Morgenstein .mr_cache[4] = { 1209603b61dSJack Morgenstein .size = 500, 1219603b61dSJack Morgenstein .limit = 250 1229603b61dSJack Morgenstein }, 1239603b61dSJack Morgenstein .mr_cache[5] = { 1249603b61dSJack Morgenstein .size = 500, 1259603b61dSJack Morgenstein .limit = 250 1269603b61dSJack Morgenstein }, 1279603b61dSJack Morgenstein .mr_cache[6] = { 1289603b61dSJack Morgenstein .size = 500, 1299603b61dSJack Morgenstein .limit = 250 1309603b61dSJack Morgenstein }, 1319603b61dSJack Morgenstein .mr_cache[7] = { 1329603b61dSJack Morgenstein .size = 500, 1339603b61dSJack Morgenstein .limit = 250 1349603b61dSJack Morgenstein }, 1359603b61dSJack Morgenstein .mr_cache[8] = { 1369603b61dSJack Morgenstein .size = 500, 1379603b61dSJack Morgenstein .limit = 250 1389603b61dSJack Morgenstein }, 1399603b61dSJack Morgenstein .mr_cache[9] = { 1409603b61dSJack Morgenstein .size = 500, 1419603b61dSJack Morgenstein .limit = 250 1429603b61dSJack Morgenstein }, 1439603b61dSJack Morgenstein .mr_cache[10] = { 1449603b61dSJack Morgenstein .size = 500, 1459603b61dSJack Morgenstein .limit = 250 1469603b61dSJack Morgenstein }, 1479603b61dSJack Morgenstein .mr_cache[11] = { 1489603b61dSJack Morgenstein .size = 500, 1499603b61dSJack Morgenstein .limit = 250 1509603b61dSJack Morgenstein }, 1519603b61dSJack Morgenstein .mr_cache[12] = { 1529603b61dSJack Morgenstein .size = 64, 1539603b61dSJack Morgenstein .limit = 32 1549603b61dSJack Morgenstein }, 1559603b61dSJack Morgenstein .mr_cache[13] = { 1569603b61dSJack Morgenstein .size = 32, 1579603b61dSJack Morgenstein .limit = 16 1589603b61dSJack Morgenstein }, 1599603b61dSJack Morgenstein .mr_cache[14] = { 1609603b61dSJack Morgenstein .size = 16, 1619603b61dSJack Morgenstein .limit = 8 1629603b61dSJack Morgenstein }, 1639603b61dSJack Morgenstein .mr_cache[15] = { 1649603b61dSJack Morgenstein .size = 8, 1659603b61dSJack Morgenstein .limit = 4 1669603b61dSJack Morgenstein }, 1679603b61dSJack Morgenstein }, 1689603b61dSJack Morgenstein }; 169e126ba97SEli Cohen 170e3297246SEli Cohen #define FW_INIT_TIMEOUT_MILI 2000 171e3297246SEli Cohen #define FW_INIT_WAIT_MS 2 172e3297246SEli Cohen 173e3297246SEli Cohen static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 174e3297246SEli Cohen { 175e3297246SEli Cohen unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); 176e3297246SEli Cohen int err = 0; 177e3297246SEli Cohen 178e3297246SEli Cohen while (fw_initializing(dev)) { 179e3297246SEli Cohen if (time_after(jiffies, end)) { 180e3297246SEli Cohen err = -EBUSY; 181e3297246SEli Cohen break; 182e3297246SEli Cohen } 183e3297246SEli Cohen msleep(FW_INIT_WAIT_MS); 184e3297246SEli Cohen } 185e3297246SEli Cohen 186e3297246SEli Cohen return err; 187e3297246SEli Cohen } 188e3297246SEli Cohen 189e126ba97SEli Cohen static int set_dma_caps(struct pci_dev *pdev) 190e126ba97SEli Cohen { 191e126ba97SEli Cohen int err; 192e126ba97SEli Cohen 193e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 194e126ba97SEli Cohen if (err) { 1951a91de28SJoe Perches dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 196e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 197e126ba97SEli Cohen if (err) { 1981a91de28SJoe Perches dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 199e126ba97SEli Cohen return err; 200e126ba97SEli Cohen } 201e126ba97SEli Cohen } 202e126ba97SEli Cohen 203e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 204e126ba97SEli Cohen if (err) { 205e126ba97SEli Cohen dev_warn(&pdev->dev, 2061a91de28SJoe Perches "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 207e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 208e126ba97SEli Cohen if (err) { 209e126ba97SEli Cohen dev_err(&pdev->dev, 2101a91de28SJoe Perches "Can't set consistent PCI DMA mask, aborting\n"); 211e126ba97SEli Cohen return err; 212e126ba97SEli Cohen } 213e126ba97SEli Cohen } 214e126ba97SEli Cohen 215e126ba97SEli Cohen dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 216e126ba97SEli Cohen return err; 217e126ba97SEli Cohen } 218e126ba97SEli Cohen 21989d44f0aSMajd Dibbiny static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) 22089d44f0aSMajd Dibbiny { 22189d44f0aSMajd Dibbiny struct pci_dev *pdev = dev->pdev; 22289d44f0aSMajd Dibbiny int err = 0; 22389d44f0aSMajd Dibbiny 22489d44f0aSMajd Dibbiny mutex_lock(&dev->pci_status_mutex); 22589d44f0aSMajd Dibbiny if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { 22689d44f0aSMajd Dibbiny err = pci_enable_device(pdev); 22789d44f0aSMajd Dibbiny if (!err) 22889d44f0aSMajd Dibbiny dev->pci_status = MLX5_PCI_STATUS_ENABLED; 22989d44f0aSMajd Dibbiny } 23089d44f0aSMajd Dibbiny mutex_unlock(&dev->pci_status_mutex); 23189d44f0aSMajd Dibbiny 23289d44f0aSMajd Dibbiny return err; 23389d44f0aSMajd Dibbiny } 23489d44f0aSMajd Dibbiny 23589d44f0aSMajd Dibbiny static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) 23689d44f0aSMajd Dibbiny { 23789d44f0aSMajd Dibbiny struct pci_dev *pdev = dev->pdev; 23889d44f0aSMajd Dibbiny 23989d44f0aSMajd Dibbiny mutex_lock(&dev->pci_status_mutex); 24089d44f0aSMajd Dibbiny if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { 24189d44f0aSMajd Dibbiny pci_disable_device(pdev); 24289d44f0aSMajd Dibbiny dev->pci_status = MLX5_PCI_STATUS_DISABLED; 24389d44f0aSMajd Dibbiny } 24489d44f0aSMajd Dibbiny mutex_unlock(&dev->pci_status_mutex); 24589d44f0aSMajd Dibbiny } 24689d44f0aSMajd Dibbiny 247e126ba97SEli Cohen static int request_bar(struct pci_dev *pdev) 248e126ba97SEli Cohen { 249e126ba97SEli Cohen int err = 0; 250e126ba97SEli Cohen 251e126ba97SEli Cohen if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 2521a91de28SJoe Perches dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 253e126ba97SEli Cohen return -ENODEV; 254e126ba97SEli Cohen } 255e126ba97SEli Cohen 256e126ba97SEli Cohen err = pci_request_regions(pdev, DRIVER_NAME); 257e126ba97SEli Cohen if (err) 258e126ba97SEli Cohen dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 259e126ba97SEli Cohen 260e126ba97SEli Cohen return err; 261e126ba97SEli Cohen } 262e126ba97SEli Cohen 263e126ba97SEli Cohen static void release_bar(struct pci_dev *pdev) 264e126ba97SEli Cohen { 265e126ba97SEli Cohen pci_release_regions(pdev); 266e126ba97SEli Cohen } 267e126ba97SEli Cohen 268e126ba97SEli Cohen static int mlx5_enable_msix(struct mlx5_core_dev *dev) 269e126ba97SEli Cohen { 270db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 271db058a18SSaeed Mahameed struct mlx5_eq_table *table = &priv->eq_table; 272938fe83cSSaeed Mahameed int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 273e126ba97SEli Cohen int nvec; 274e126ba97SEli Cohen int i; 275e126ba97SEli Cohen 276938fe83cSSaeed Mahameed nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 277938fe83cSSaeed Mahameed MLX5_EQ_VEC_COMP_BASE; 278e126ba97SEli Cohen nvec = min_t(int, nvec, num_eqs); 279e126ba97SEli Cohen if (nvec <= MLX5_EQ_VEC_COMP_BASE) 280e126ba97SEli Cohen return -ENOMEM; 281e126ba97SEli Cohen 282db058a18SSaeed Mahameed priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL); 283db058a18SSaeed Mahameed 284db058a18SSaeed Mahameed priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL); 285db058a18SSaeed Mahameed if (!priv->msix_arr || !priv->irq_info) 286db058a18SSaeed Mahameed goto err_free_msix; 287e126ba97SEli Cohen 288e126ba97SEli Cohen for (i = 0; i < nvec; i++) 289db058a18SSaeed Mahameed priv->msix_arr[i].entry = i; 290e126ba97SEli Cohen 291db058a18SSaeed Mahameed nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 2923a9e161aSEli Cohen MLX5_EQ_VEC_COMP_BASE + 1, nvec); 293f3c9407bSAlexander Gordeev if (nvec < 0) 294f3c9407bSAlexander Gordeev return nvec; 295e126ba97SEli Cohen 296f3c9407bSAlexander Gordeev table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 297e126ba97SEli Cohen 298e126ba97SEli Cohen return 0; 299db058a18SSaeed Mahameed 300db058a18SSaeed Mahameed err_free_msix: 301db058a18SSaeed Mahameed kfree(priv->irq_info); 302db058a18SSaeed Mahameed kfree(priv->msix_arr); 303db058a18SSaeed Mahameed return -ENOMEM; 304e126ba97SEli Cohen } 305e126ba97SEli Cohen 306e126ba97SEli Cohen static void mlx5_disable_msix(struct mlx5_core_dev *dev) 307e126ba97SEli Cohen { 308db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 309e126ba97SEli Cohen 310e126ba97SEli Cohen pci_disable_msix(dev->pdev); 311db058a18SSaeed Mahameed kfree(priv->irq_info); 312db058a18SSaeed Mahameed kfree(priv->msix_arr); 313e126ba97SEli Cohen } 314e126ba97SEli Cohen 315e126ba97SEli Cohen struct mlx5_reg_host_endianess { 316e126ba97SEli Cohen u8 he; 317e126ba97SEli Cohen u8 rsvd[15]; 318e126ba97SEli Cohen }; 319e126ba97SEli Cohen 32087b8de49SEli Cohen 32187b8de49SEli Cohen #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 32287b8de49SEli Cohen 32387b8de49SEli Cohen enum { 32487b8de49SEli Cohen MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 325c7a08ac7SEli Cohen MLX5_DEV_CAP_FLAG_DCT, 32687b8de49SEli Cohen }; 32787b8de49SEli Cohen 3282974ab6eSSaeed Mahameed static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) 329c7a08ac7SEli Cohen { 330c7a08ac7SEli Cohen switch (size) { 331c7a08ac7SEli Cohen case 128: 332c7a08ac7SEli Cohen return 0; 333c7a08ac7SEli Cohen case 256: 334c7a08ac7SEli Cohen return 1; 335c7a08ac7SEli Cohen case 512: 336c7a08ac7SEli Cohen return 2; 337c7a08ac7SEli Cohen case 1024: 338c7a08ac7SEli Cohen return 3; 339c7a08ac7SEli Cohen case 2048: 340c7a08ac7SEli Cohen return 4; 341c7a08ac7SEli Cohen case 4096: 342c7a08ac7SEli Cohen return 5; 343c7a08ac7SEli Cohen default: 3442974ab6eSSaeed Mahameed mlx5_core_warn(dev, "invalid pkey table size %d\n", size); 345c7a08ac7SEli Cohen return 0; 346c7a08ac7SEli Cohen } 347c7a08ac7SEli Cohen } 348c7a08ac7SEli Cohen 349b06e7de8SLeon Romanovsky static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, 350b06e7de8SLeon Romanovsky enum mlx5_cap_type cap_type, 351938fe83cSSaeed Mahameed enum mlx5_cap_mode cap_mode) 352c7a08ac7SEli Cohen { 353b775516bSEli Cohen u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 354b775516bSEli Cohen int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 355938fe83cSSaeed Mahameed void *out, *hca_caps; 356938fe83cSSaeed Mahameed u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 357c7a08ac7SEli Cohen int err; 358c7a08ac7SEli Cohen 359b775516bSEli Cohen memset(in, 0, sizeof(in)); 360b775516bSEli Cohen out = kzalloc(out_sz, GFP_KERNEL); 361c7a08ac7SEli Cohen if (!out) 362c7a08ac7SEli Cohen return -ENOMEM; 363938fe83cSSaeed Mahameed 364b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 365b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 366b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 367c7a08ac7SEli Cohen if (err) { 368938fe83cSSaeed Mahameed mlx5_core_warn(dev, 369938fe83cSSaeed Mahameed "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 370938fe83cSSaeed Mahameed cap_type, cap_mode, err); 371c7a08ac7SEli Cohen goto query_ex; 372c7a08ac7SEli Cohen } 373c7a08ac7SEli Cohen 374938fe83cSSaeed Mahameed hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 375938fe83cSSaeed Mahameed 376938fe83cSSaeed Mahameed switch (cap_mode) { 377938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_MAX: 378938fe83cSSaeed Mahameed memcpy(dev->hca_caps_max[cap_type], hca_caps, 379938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 380938fe83cSSaeed Mahameed break; 381938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_CUR: 382938fe83cSSaeed Mahameed memcpy(dev->hca_caps_cur[cap_type], hca_caps, 383938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 384938fe83cSSaeed Mahameed break; 385938fe83cSSaeed Mahameed default: 386938fe83cSSaeed Mahameed mlx5_core_warn(dev, 387938fe83cSSaeed Mahameed "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 388938fe83cSSaeed Mahameed cap_type, cap_mode); 389938fe83cSSaeed Mahameed err = -EINVAL; 390938fe83cSSaeed Mahameed break; 391938fe83cSSaeed Mahameed } 392c7a08ac7SEli Cohen query_ex: 393c7a08ac7SEli Cohen kfree(out); 394c7a08ac7SEli Cohen return err; 395c7a08ac7SEli Cohen } 396c7a08ac7SEli Cohen 397b06e7de8SLeon Romanovsky int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) 398b06e7de8SLeon Romanovsky { 399b06e7de8SLeon Romanovsky int ret; 400b06e7de8SLeon Romanovsky 401b06e7de8SLeon Romanovsky ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR); 402b06e7de8SLeon Romanovsky if (ret) 403b06e7de8SLeon Romanovsky return ret; 404b06e7de8SLeon Romanovsky return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX); 405b06e7de8SLeon Romanovsky } 406b06e7de8SLeon Romanovsky 407f91e6d89SEran Ben Elisha static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz, int opmod) 408c7a08ac7SEli Cohen { 409c4f287c4SSaeed Mahameed u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)] = {0}; 410c7a08ac7SEli Cohen 411b775516bSEli Cohen MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 412f91e6d89SEran Ben Elisha MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1); 413c4f287c4SSaeed Mahameed return mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 414c7a08ac7SEli Cohen } 41587b8de49SEli Cohen 416f91e6d89SEran Ben Elisha static int handle_hca_cap_atomic(struct mlx5_core_dev *dev) 417f91e6d89SEran Ben Elisha { 418f91e6d89SEran Ben Elisha void *set_ctx; 419f91e6d89SEran Ben Elisha void *set_hca_cap; 420f91e6d89SEran Ben Elisha int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 421f91e6d89SEran Ben Elisha int req_endianness; 422f91e6d89SEran Ben Elisha int err; 423f91e6d89SEran Ben Elisha 424f91e6d89SEran Ben Elisha if (MLX5_CAP_GEN(dev, atomic)) { 425b06e7de8SLeon Romanovsky err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); 426f91e6d89SEran Ben Elisha if (err) 427f91e6d89SEran Ben Elisha return err; 428f91e6d89SEran Ben Elisha } else { 429f91e6d89SEran Ben Elisha return 0; 430f91e6d89SEran Ben Elisha } 431f91e6d89SEran Ben Elisha 432f91e6d89SEran Ben Elisha req_endianness = 433f91e6d89SEran Ben Elisha MLX5_CAP_ATOMIC(dev, 434f91e6d89SEran Ben Elisha supported_atomic_req_8B_endianess_mode_1); 435f91e6d89SEran Ben Elisha 436f91e6d89SEran Ben Elisha if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) 437f91e6d89SEran Ben Elisha return 0; 438f91e6d89SEran Ben Elisha 439f91e6d89SEran Ben Elisha set_ctx = kzalloc(set_sz, GFP_KERNEL); 440f91e6d89SEran Ben Elisha if (!set_ctx) 441f91e6d89SEran Ben Elisha return -ENOMEM; 442f91e6d89SEran Ben Elisha 443f91e6d89SEran Ben Elisha set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 444f91e6d89SEran Ben Elisha 445f91e6d89SEran Ben Elisha /* Set requestor to host endianness */ 446f91e6d89SEran Ben Elisha MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode, 447f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); 448f91e6d89SEran Ben Elisha 449f91e6d89SEran Ben Elisha err = set_caps(dev, set_ctx, set_sz, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC); 450f91e6d89SEran Ben Elisha 451f91e6d89SEran Ben Elisha kfree(set_ctx); 452f91e6d89SEran Ben Elisha return err; 453f91e6d89SEran Ben Elisha } 454f91e6d89SEran Ben Elisha 455e126ba97SEli Cohen static int handle_hca_cap(struct mlx5_core_dev *dev) 456e126ba97SEli Cohen { 457b775516bSEli Cohen void *set_ctx = NULL; 458c7a08ac7SEli Cohen struct mlx5_profile *prof = dev->profile; 459c7a08ac7SEli Cohen int err = -ENOMEM; 460b775516bSEli Cohen int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 461938fe83cSSaeed Mahameed void *set_hca_cap; 462e126ba97SEli Cohen 463b775516bSEli Cohen set_ctx = kzalloc(set_sz, GFP_KERNEL); 464c7a08ac7SEli Cohen if (!set_ctx) 465e126ba97SEli Cohen goto query_ex; 466e126ba97SEli Cohen 467b06e7de8SLeon Romanovsky err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); 468c7a08ac7SEli Cohen if (err) 469e126ba97SEli Cohen goto query_ex; 470e126ba97SEli Cohen 471938fe83cSSaeed Mahameed set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 472938fe83cSSaeed Mahameed capability); 473938fe83cSSaeed Mahameed memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], 474938fe83cSSaeed Mahameed MLX5_ST_SZ_BYTES(cmd_hca_cap)); 475938fe83cSSaeed Mahameed 476938fe83cSSaeed Mahameed mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 477707c4602SMajd Dibbiny mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 478938fe83cSSaeed Mahameed 128); 479c7a08ac7SEli Cohen /* we limit the size of the pkey table to 128 entries for now */ 480938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 4812974ab6eSSaeed Mahameed to_fw_pkey_sz(dev, 128)); 482e126ba97SEli Cohen 483c7a08ac7SEli Cohen if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 484938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 485938fe83cSSaeed Mahameed prof->log_max_qp); 486e126ba97SEli Cohen 487938fe83cSSaeed Mahameed /* disable cmdif checksum */ 488938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 489c1868b82SEli Cohen 490fe1e1876SCarol L Soto MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 491fe1e1876SCarol L Soto 492f91e6d89SEran Ben Elisha err = set_caps(dev, set_ctx, set_sz, 493f91e6d89SEran Ben Elisha MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 494e126ba97SEli Cohen 495e126ba97SEli Cohen query_ex: 496e126ba97SEli Cohen kfree(set_ctx); 497e126ba97SEli Cohen return err; 498e126ba97SEli Cohen } 499e126ba97SEli Cohen 500e126ba97SEli Cohen static int set_hca_ctrl(struct mlx5_core_dev *dev) 501e126ba97SEli Cohen { 502e126ba97SEli Cohen struct mlx5_reg_host_endianess he_in; 503e126ba97SEli Cohen struct mlx5_reg_host_endianess he_out; 504e126ba97SEli Cohen int err; 505e126ba97SEli Cohen 506fc50db98SEli Cohen if (!mlx5_core_is_pf(dev)) 507fc50db98SEli Cohen return 0; 508fc50db98SEli Cohen 509e126ba97SEli Cohen memset(&he_in, 0, sizeof(he_in)); 510e126ba97SEli Cohen he_in.he = MLX5_SET_HOST_ENDIANNESS; 511e126ba97SEli Cohen err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 512e126ba97SEli Cohen &he_out, sizeof(he_out), 513e126ba97SEli Cohen MLX5_REG_HOST_ENDIANNESS, 0, 1); 514e126ba97SEli Cohen return err; 515e126ba97SEli Cohen } 516e126ba97SEli Cohen 5170b107106SEli Cohen int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id) 518cd23b14bSEli Cohen { 519c4f287c4SSaeed Mahameed u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0}; 520c4f287c4SSaeed Mahameed u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {0}; 521cd23b14bSEli Cohen 5220b107106SEli Cohen MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 5230b107106SEli Cohen MLX5_SET(enable_hca_in, in, function_id, func_id); 524c4f287c4SSaeed Mahameed return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 525cd23b14bSEli Cohen } 526cd23b14bSEli Cohen 5270b107106SEli Cohen int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) 528cd23b14bSEli Cohen { 529c4f287c4SSaeed Mahameed u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {0}; 530c4f287c4SSaeed Mahameed u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {0}; 531cd23b14bSEli Cohen 5320b107106SEli Cohen MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 5330b107106SEli Cohen MLX5_SET(disable_hca_in, in, function_id, func_id); 534c4f287c4SSaeed Mahameed return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 535cd23b14bSEli Cohen } 536cd23b14bSEli Cohen 537b0844444SEran Ben Elisha cycle_t mlx5_read_internal_timer(struct mlx5_core_dev *dev) 538b0844444SEran Ben Elisha { 539b0844444SEran Ben Elisha u32 timer_h, timer_h1, timer_l; 540b0844444SEran Ben Elisha 541b0844444SEran Ben Elisha timer_h = ioread32be(&dev->iseg->internal_timer_h); 542b0844444SEran Ben Elisha timer_l = ioread32be(&dev->iseg->internal_timer_l); 543b0844444SEran Ben Elisha timer_h1 = ioread32be(&dev->iseg->internal_timer_h); 544b0844444SEran Ben Elisha if (timer_h != timer_h1) /* wrap around */ 545b0844444SEran Ben Elisha timer_l = ioread32be(&dev->iseg->internal_timer_l); 546b0844444SEran Ben Elisha 547b0844444SEran Ben Elisha return (cycle_t)timer_l | (cycle_t)timer_h1 << 32; 548b0844444SEran Ben Elisha } 549b0844444SEran Ben Elisha 550db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) 551db058a18SSaeed Mahameed { 552db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 553db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 554db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 555311c7c71SSaeed Mahameed int numa_node = priv->numa_node; 556db058a18SSaeed Mahameed int err; 557db058a18SSaeed Mahameed 558db058a18SSaeed Mahameed if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) { 559db058a18SSaeed Mahameed mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); 560db058a18SSaeed Mahameed return -ENOMEM; 561db058a18SSaeed Mahameed } 562db058a18SSaeed Mahameed 563dda922c8SDavid S. Miller cpumask_set_cpu(cpumask_local_spread(i, numa_node), 564dda922c8SDavid S. Miller priv->irq_info[i].mask); 565db058a18SSaeed Mahameed 566db058a18SSaeed Mahameed err = irq_set_affinity_hint(irq, priv->irq_info[i].mask); 567db058a18SSaeed Mahameed if (err) { 568db058a18SSaeed Mahameed mlx5_core_warn(mdev, "irq_set_affinity_hint failed,irq 0x%.4x", 569db058a18SSaeed Mahameed irq); 570db058a18SSaeed Mahameed goto err_clear_mask; 571db058a18SSaeed Mahameed } 572db058a18SSaeed Mahameed 573db058a18SSaeed Mahameed return 0; 574db058a18SSaeed Mahameed 575db058a18SSaeed Mahameed err_clear_mask: 576db058a18SSaeed Mahameed free_cpumask_var(priv->irq_info[i].mask); 577db058a18SSaeed Mahameed return err; 578db058a18SSaeed Mahameed } 579db058a18SSaeed Mahameed 580db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i) 581db058a18SSaeed Mahameed { 582db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 583db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 584db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 585db058a18SSaeed Mahameed 586db058a18SSaeed Mahameed irq_set_affinity_hint(irq, NULL); 587db058a18SSaeed Mahameed free_cpumask_var(priv->irq_info[i].mask); 588db058a18SSaeed Mahameed } 589db058a18SSaeed Mahameed 590db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev) 591db058a18SSaeed Mahameed { 592db058a18SSaeed Mahameed int err; 593db058a18SSaeed Mahameed int i; 594db058a18SSaeed Mahameed 595db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) { 596db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hint(mdev, i); 597db058a18SSaeed Mahameed if (err) 598db058a18SSaeed Mahameed goto err_out; 599db058a18SSaeed Mahameed } 600db058a18SSaeed Mahameed 601db058a18SSaeed Mahameed return 0; 602db058a18SSaeed Mahameed 603db058a18SSaeed Mahameed err_out: 604db058a18SSaeed Mahameed for (i--; i >= 0; i--) 605db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 606db058a18SSaeed Mahameed 607db058a18SSaeed Mahameed return err; 608db058a18SSaeed Mahameed } 609db058a18SSaeed Mahameed 610db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev) 611db058a18SSaeed Mahameed { 612db058a18SSaeed Mahameed int i; 613db058a18SSaeed Mahameed 614db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) 615db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 616db058a18SSaeed Mahameed } 617db058a18SSaeed Mahameed 6180b6e26ceSDoron Tsur int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, 6190b6e26ceSDoron Tsur unsigned int *irqn) 620233d05d2SSaeed Mahameed { 621233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 622233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 623233d05d2SSaeed Mahameed int err = -ENOENT; 624233d05d2SSaeed Mahameed 625233d05d2SSaeed Mahameed spin_lock(&table->lock); 626233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 627233d05d2SSaeed Mahameed if (eq->index == vector) { 628233d05d2SSaeed Mahameed *eqn = eq->eqn; 629233d05d2SSaeed Mahameed *irqn = eq->irqn; 630233d05d2SSaeed Mahameed err = 0; 631233d05d2SSaeed Mahameed break; 632233d05d2SSaeed Mahameed } 633233d05d2SSaeed Mahameed } 634233d05d2SSaeed Mahameed spin_unlock(&table->lock); 635233d05d2SSaeed Mahameed 636233d05d2SSaeed Mahameed return err; 637233d05d2SSaeed Mahameed } 638233d05d2SSaeed Mahameed EXPORT_SYMBOL(mlx5_vector2eqn); 639233d05d2SSaeed Mahameed 64094c6825eSMatan Barak struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn) 64194c6825eSMatan Barak { 64294c6825eSMatan Barak struct mlx5_eq_table *table = &dev->priv.eq_table; 64394c6825eSMatan Barak struct mlx5_eq *eq; 64494c6825eSMatan Barak 64594c6825eSMatan Barak spin_lock(&table->lock); 64694c6825eSMatan Barak list_for_each_entry(eq, &table->comp_eqs_list, list) 64794c6825eSMatan Barak if (eq->eqn == eqn) { 64894c6825eSMatan Barak spin_unlock(&table->lock); 64994c6825eSMatan Barak return eq; 65094c6825eSMatan Barak } 65194c6825eSMatan Barak 65294c6825eSMatan Barak spin_unlock(&table->lock); 65394c6825eSMatan Barak 65494c6825eSMatan Barak return ERR_PTR(-ENOENT); 65594c6825eSMatan Barak } 65694c6825eSMatan Barak 657233d05d2SSaeed Mahameed static void free_comp_eqs(struct mlx5_core_dev *dev) 658233d05d2SSaeed Mahameed { 659233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 660233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 661233d05d2SSaeed Mahameed 6625a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 6635a7b27ebSMaor Gottlieb if (dev->rmap) { 6645a7b27ebSMaor Gottlieb free_irq_cpu_rmap(dev->rmap); 6655a7b27ebSMaor Gottlieb dev->rmap = NULL; 6665a7b27ebSMaor Gottlieb } 6675a7b27ebSMaor Gottlieb #endif 668233d05d2SSaeed Mahameed spin_lock(&table->lock); 669233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 670233d05d2SSaeed Mahameed list_del(&eq->list); 671233d05d2SSaeed Mahameed spin_unlock(&table->lock); 672233d05d2SSaeed Mahameed if (mlx5_destroy_unmap_eq(dev, eq)) 673233d05d2SSaeed Mahameed mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 674233d05d2SSaeed Mahameed eq->eqn); 675233d05d2SSaeed Mahameed kfree(eq); 676233d05d2SSaeed Mahameed spin_lock(&table->lock); 677233d05d2SSaeed Mahameed } 678233d05d2SSaeed Mahameed spin_unlock(&table->lock); 679233d05d2SSaeed Mahameed } 680233d05d2SSaeed Mahameed 681233d05d2SSaeed Mahameed static int alloc_comp_eqs(struct mlx5_core_dev *dev) 682233d05d2SSaeed Mahameed { 683233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 684db058a18SSaeed Mahameed char name[MLX5_MAX_IRQ_NAME]; 685233d05d2SSaeed Mahameed struct mlx5_eq *eq; 686233d05d2SSaeed Mahameed int ncomp_vec; 687233d05d2SSaeed Mahameed int nent; 688233d05d2SSaeed Mahameed int err; 689233d05d2SSaeed Mahameed int i; 690233d05d2SSaeed Mahameed 691233d05d2SSaeed Mahameed INIT_LIST_HEAD(&table->comp_eqs_list); 692233d05d2SSaeed Mahameed ncomp_vec = table->num_comp_vectors; 693233d05d2SSaeed Mahameed nent = MLX5_COMP_EQ_SIZE; 6945a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 6955a7b27ebSMaor Gottlieb dev->rmap = alloc_irq_cpu_rmap(ncomp_vec); 6965a7b27ebSMaor Gottlieb if (!dev->rmap) 6975a7b27ebSMaor Gottlieb return -ENOMEM; 6985a7b27ebSMaor Gottlieb #endif 699233d05d2SSaeed Mahameed for (i = 0; i < ncomp_vec; i++) { 700233d05d2SSaeed Mahameed eq = kzalloc(sizeof(*eq), GFP_KERNEL); 701233d05d2SSaeed Mahameed if (!eq) { 702233d05d2SSaeed Mahameed err = -ENOMEM; 703233d05d2SSaeed Mahameed goto clean; 704233d05d2SSaeed Mahameed } 705233d05d2SSaeed Mahameed 7065a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 7075a7b27ebSMaor Gottlieb irq_cpu_rmap_add(dev->rmap, 7085a7b27ebSMaor Gottlieb dev->priv.msix_arr[i + MLX5_EQ_VEC_COMP_BASE].vector); 7095a7b27ebSMaor Gottlieb #endif 710db058a18SSaeed Mahameed snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 711233d05d2SSaeed Mahameed err = mlx5_create_map_eq(dev, eq, 712233d05d2SSaeed Mahameed i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 713233d05d2SSaeed Mahameed name, &dev->priv.uuari.uars[0]); 714233d05d2SSaeed Mahameed if (err) { 715233d05d2SSaeed Mahameed kfree(eq); 716233d05d2SSaeed Mahameed goto clean; 717233d05d2SSaeed Mahameed } 718233d05d2SSaeed Mahameed mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 719233d05d2SSaeed Mahameed eq->index = i; 720233d05d2SSaeed Mahameed spin_lock(&table->lock); 721233d05d2SSaeed Mahameed list_add_tail(&eq->list, &table->comp_eqs_list); 722233d05d2SSaeed Mahameed spin_unlock(&table->lock); 723233d05d2SSaeed Mahameed } 724233d05d2SSaeed Mahameed 725233d05d2SSaeed Mahameed return 0; 726233d05d2SSaeed Mahameed 727233d05d2SSaeed Mahameed clean: 728233d05d2SSaeed Mahameed free_comp_eqs(dev); 729233d05d2SSaeed Mahameed return err; 730233d05d2SSaeed Mahameed } 731233d05d2SSaeed Mahameed 732f62b8bb8SAmir Vadai static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 733f62b8bb8SAmir Vadai { 734c4f287c4SSaeed Mahameed u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {0}; 735c4f287c4SSaeed Mahameed u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {0}; 736f62b8bb8SAmir Vadai u32 sup_issi; 737c4f287c4SSaeed Mahameed int err; 738f62b8bb8SAmir Vadai 739f62b8bb8SAmir Vadai MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 740c4f287c4SSaeed Mahameed err = mlx5_cmd_exec(dev, query_in, sizeof(query_in), 741f62b8bb8SAmir Vadai query_out, sizeof(query_out)); 742f62b8bb8SAmir Vadai if (err) { 743c4f287c4SSaeed Mahameed u32 syndrome; 744c4f287c4SSaeed Mahameed u8 status; 745c4f287c4SSaeed Mahameed 746c4f287c4SSaeed Mahameed mlx5_cmd_mbox_status(query_out, &status, &syndrome); 747c4f287c4SSaeed Mahameed if (status == MLX5_CMD_STAT_BAD_OP_ERR) { 748f62b8bb8SAmir Vadai pr_debug("Only ISSI 0 is supported\n"); 749f62b8bb8SAmir Vadai return 0; 750f62b8bb8SAmir Vadai } 751f62b8bb8SAmir Vadai 752c4f287c4SSaeed Mahameed pr_err("failed to query ISSI err(%d)\n", err); 753f62b8bb8SAmir Vadai return err; 754f62b8bb8SAmir Vadai } 755f62b8bb8SAmir Vadai 756f62b8bb8SAmir Vadai sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 757f62b8bb8SAmir Vadai 758f62b8bb8SAmir Vadai if (sup_issi & (1 << 1)) { 759c4f287c4SSaeed Mahameed u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {0}; 760c4f287c4SSaeed Mahameed u32 set_out[MLX5_ST_SZ_DW(set_issi_out)] = {0}; 761f62b8bb8SAmir Vadai 762f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 763f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, current_issi, 1); 764c4f287c4SSaeed Mahameed err = mlx5_cmd_exec(dev, set_in, sizeof(set_in), 765f62b8bb8SAmir Vadai set_out, sizeof(set_out)); 766f62b8bb8SAmir Vadai if (err) { 767c4f287c4SSaeed Mahameed pr_err("failed to set ISSI=1 err(%d)\n", err); 768f62b8bb8SAmir Vadai return err; 769f62b8bb8SAmir Vadai } 770f62b8bb8SAmir Vadai 771f62b8bb8SAmir Vadai dev->issi = 1; 772f62b8bb8SAmir Vadai 773f62b8bb8SAmir Vadai return 0; 774e74a1db0SHaggai Abramonvsky } else if (sup_issi & (1 << 0) || !sup_issi) { 775f62b8bb8SAmir Vadai return 0; 776f62b8bb8SAmir Vadai } 777f62b8bb8SAmir Vadai 778f62b8bb8SAmir Vadai return -ENOTSUPP; 779f62b8bb8SAmir Vadai } 780f62b8bb8SAmir Vadai 781a31208b1SMajd Dibbiny static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 782a31208b1SMajd Dibbiny { 783a31208b1SMajd Dibbiny struct mlx5_device_context *dev_ctx; 784a31208b1SMajd Dibbiny struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 785a31208b1SMajd Dibbiny 786a31208b1SMajd Dibbiny dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL); 787a31208b1SMajd Dibbiny if (!dev_ctx) 788a31208b1SMajd Dibbiny return; 789a31208b1SMajd Dibbiny 790a31208b1SMajd Dibbiny dev_ctx->intf = intf; 791a31208b1SMajd Dibbiny dev_ctx->context = intf->add(dev); 792a31208b1SMajd Dibbiny 793a31208b1SMajd Dibbiny if (dev_ctx->context) { 794a31208b1SMajd Dibbiny spin_lock_irq(&priv->ctx_lock); 795a31208b1SMajd Dibbiny list_add_tail(&dev_ctx->list, &priv->ctx_list); 796a31208b1SMajd Dibbiny spin_unlock_irq(&priv->ctx_lock); 797a31208b1SMajd Dibbiny } else { 798a31208b1SMajd Dibbiny kfree(dev_ctx); 799a31208b1SMajd Dibbiny } 800a31208b1SMajd Dibbiny } 801a31208b1SMajd Dibbiny 802a31208b1SMajd Dibbiny static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 803a31208b1SMajd Dibbiny { 804a31208b1SMajd Dibbiny struct mlx5_device_context *dev_ctx; 805a31208b1SMajd Dibbiny struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 806a31208b1SMajd Dibbiny 807a31208b1SMajd Dibbiny list_for_each_entry(dev_ctx, &priv->ctx_list, list) 808a31208b1SMajd Dibbiny if (dev_ctx->intf == intf) { 809a31208b1SMajd Dibbiny spin_lock_irq(&priv->ctx_lock); 810a31208b1SMajd Dibbiny list_del(&dev_ctx->list); 811a31208b1SMajd Dibbiny spin_unlock_irq(&priv->ctx_lock); 812a31208b1SMajd Dibbiny 813a31208b1SMajd Dibbiny intf->remove(dev, dev_ctx->context); 814a31208b1SMajd Dibbiny kfree(dev_ctx); 815a31208b1SMajd Dibbiny return; 816a31208b1SMajd Dibbiny } 817a31208b1SMajd Dibbiny } 818a31208b1SMajd Dibbiny 819a31208b1SMajd Dibbiny static int mlx5_register_device(struct mlx5_core_dev *dev) 820e126ba97SEli Cohen { 821e126ba97SEli Cohen struct mlx5_priv *priv = &dev->priv; 822a31208b1SMajd Dibbiny struct mlx5_interface *intf; 823e126ba97SEli Cohen 8247907f23aSAviv Heller mutex_lock(&mlx5_intf_mutex); 8257907f23aSAviv Heller list_add_tail(&priv->dev_list, &mlx5_dev_list); 826a31208b1SMajd Dibbiny list_for_each_entry(intf, &intf_list, list) 827a31208b1SMajd Dibbiny mlx5_add_device(intf, priv); 8287907f23aSAviv Heller mutex_unlock(&mlx5_intf_mutex); 829a31208b1SMajd Dibbiny 830a31208b1SMajd Dibbiny return 0; 831a31208b1SMajd Dibbiny } 832a31208b1SMajd Dibbiny 833a31208b1SMajd Dibbiny static void mlx5_unregister_device(struct mlx5_core_dev *dev) 834a31208b1SMajd Dibbiny { 835a31208b1SMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 836a31208b1SMajd Dibbiny struct mlx5_interface *intf; 837a31208b1SMajd Dibbiny 8387907f23aSAviv Heller mutex_lock(&mlx5_intf_mutex); 839a31208b1SMajd Dibbiny list_for_each_entry(intf, &intf_list, list) 840a31208b1SMajd Dibbiny mlx5_remove_device(intf, priv); 841a31208b1SMajd Dibbiny list_del(&priv->dev_list); 8427907f23aSAviv Heller mutex_unlock(&mlx5_intf_mutex); 843a31208b1SMajd Dibbiny } 844a31208b1SMajd Dibbiny 845a31208b1SMajd Dibbiny int mlx5_register_interface(struct mlx5_interface *intf) 846a31208b1SMajd Dibbiny { 847a31208b1SMajd Dibbiny struct mlx5_priv *priv; 848a31208b1SMajd Dibbiny 849a31208b1SMajd Dibbiny if (!intf->add || !intf->remove) 850a31208b1SMajd Dibbiny return -EINVAL; 851a31208b1SMajd Dibbiny 8527907f23aSAviv Heller mutex_lock(&mlx5_intf_mutex); 853a31208b1SMajd Dibbiny list_add_tail(&intf->list, &intf_list); 8547907f23aSAviv Heller list_for_each_entry(priv, &mlx5_dev_list, dev_list) 855a31208b1SMajd Dibbiny mlx5_add_device(intf, priv); 8567907f23aSAviv Heller mutex_unlock(&mlx5_intf_mutex); 857a31208b1SMajd Dibbiny 858a31208b1SMajd Dibbiny return 0; 859a31208b1SMajd Dibbiny } 860a31208b1SMajd Dibbiny EXPORT_SYMBOL(mlx5_register_interface); 861a31208b1SMajd Dibbiny 862a31208b1SMajd Dibbiny void mlx5_unregister_interface(struct mlx5_interface *intf) 863a31208b1SMajd Dibbiny { 864a31208b1SMajd Dibbiny struct mlx5_priv *priv; 865a31208b1SMajd Dibbiny 8667907f23aSAviv Heller mutex_lock(&mlx5_intf_mutex); 8677907f23aSAviv Heller list_for_each_entry(priv, &mlx5_dev_list, dev_list) 868a31208b1SMajd Dibbiny mlx5_remove_device(intf, priv); 869a31208b1SMajd Dibbiny list_del(&intf->list); 8707907f23aSAviv Heller mutex_unlock(&mlx5_intf_mutex); 871a31208b1SMajd Dibbiny } 872a31208b1SMajd Dibbiny EXPORT_SYMBOL(mlx5_unregister_interface); 873a31208b1SMajd Dibbiny 874a31208b1SMajd Dibbiny void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) 875a31208b1SMajd Dibbiny { 876a31208b1SMajd Dibbiny struct mlx5_priv *priv = &mdev->priv; 877a31208b1SMajd Dibbiny struct mlx5_device_context *dev_ctx; 878a31208b1SMajd Dibbiny unsigned long flags; 879a31208b1SMajd Dibbiny void *result = NULL; 880a31208b1SMajd Dibbiny 881a31208b1SMajd Dibbiny spin_lock_irqsave(&priv->ctx_lock, flags); 882a31208b1SMajd Dibbiny 883a31208b1SMajd Dibbiny list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) 884a31208b1SMajd Dibbiny if ((dev_ctx->intf->protocol == protocol) && 885a31208b1SMajd Dibbiny dev_ctx->intf->get_dev) { 886a31208b1SMajd Dibbiny result = dev_ctx->intf->get_dev(dev_ctx->context); 887a31208b1SMajd Dibbiny break; 888a31208b1SMajd Dibbiny } 889a31208b1SMajd Dibbiny 890a31208b1SMajd Dibbiny spin_unlock_irqrestore(&priv->ctx_lock, flags); 891a31208b1SMajd Dibbiny 892a31208b1SMajd Dibbiny return result; 893a31208b1SMajd Dibbiny } 894a31208b1SMajd Dibbiny EXPORT_SYMBOL(mlx5_get_protocol_dev); 895a31208b1SMajd Dibbiny 8967907f23aSAviv Heller /* Must be called with intf_mutex held */ 8977907f23aSAviv Heller void mlx5_add_dev_by_protocol(struct mlx5_core_dev *dev, int protocol) 8987907f23aSAviv Heller { 8997907f23aSAviv Heller struct mlx5_interface *intf; 9007907f23aSAviv Heller 9017907f23aSAviv Heller list_for_each_entry(intf, &intf_list, list) 9027907f23aSAviv Heller if (intf->protocol == protocol) { 9037907f23aSAviv Heller mlx5_add_device(intf, &dev->priv); 9047907f23aSAviv Heller break; 9057907f23aSAviv Heller } 9067907f23aSAviv Heller } 9077907f23aSAviv Heller 9087907f23aSAviv Heller /* Must be called with intf_mutex held */ 9097907f23aSAviv Heller void mlx5_remove_dev_by_protocol(struct mlx5_core_dev *dev, int protocol) 9107907f23aSAviv Heller { 9117907f23aSAviv Heller struct mlx5_interface *intf; 9127907f23aSAviv Heller 9137907f23aSAviv Heller list_for_each_entry(intf, &intf_list, list) 9147907f23aSAviv Heller if (intf->protocol == protocol) { 9157907f23aSAviv Heller mlx5_remove_device(intf, &dev->priv); 9167907f23aSAviv Heller break; 9177907f23aSAviv Heller } 9187907f23aSAviv Heller } 9197907f23aSAviv Heller 920a31208b1SMajd Dibbiny static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 921a31208b1SMajd Dibbiny { 922a31208b1SMajd Dibbiny struct pci_dev *pdev = dev->pdev; 923a31208b1SMajd Dibbiny int err = 0; 924a31208b1SMajd Dibbiny 925e126ba97SEli Cohen pci_set_drvdata(dev->pdev, dev); 926e126ba97SEli Cohen strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 927e126ba97SEli Cohen priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 928e126ba97SEli Cohen 929e126ba97SEli Cohen mutex_init(&priv->pgdir_mutex); 930e126ba97SEli Cohen INIT_LIST_HEAD(&priv->pgdir_list); 931e126ba97SEli Cohen spin_lock_init(&priv->mkey_lock); 932e126ba97SEli Cohen 933311c7c71SSaeed Mahameed mutex_init(&priv->alloc_mutex); 934311c7c71SSaeed Mahameed 935311c7c71SSaeed Mahameed priv->numa_node = dev_to_node(&dev->pdev->dev); 936311c7c71SSaeed Mahameed 937e126ba97SEli Cohen priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root); 938e126ba97SEli Cohen if (!priv->dbg_root) 939e126ba97SEli Cohen return -ENOMEM; 940e126ba97SEli Cohen 94189d44f0aSMajd Dibbiny err = mlx5_pci_enable_device(dev); 942e126ba97SEli Cohen if (err) { 9431a91de28SJoe Perches dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 944e126ba97SEli Cohen goto err_dbg; 945e126ba97SEli Cohen } 946e126ba97SEli Cohen 947e126ba97SEli Cohen err = request_bar(pdev); 948e126ba97SEli Cohen if (err) { 9491a91de28SJoe Perches dev_err(&pdev->dev, "error requesting BARs, aborting\n"); 950e126ba97SEli Cohen goto err_disable; 951e126ba97SEli Cohen } 952e126ba97SEli Cohen 953e126ba97SEli Cohen pci_set_master(pdev); 954e126ba97SEli Cohen 955e126ba97SEli Cohen err = set_dma_caps(pdev); 956e126ba97SEli Cohen if (err) { 957e126ba97SEli Cohen dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); 958e126ba97SEli Cohen goto err_clr_master; 959e126ba97SEli Cohen } 960e126ba97SEli Cohen 961e126ba97SEli Cohen dev->iseg_base = pci_resource_start(dev->pdev, 0); 962e126ba97SEli Cohen dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 963e126ba97SEli Cohen if (!dev->iseg) { 964e126ba97SEli Cohen err = -ENOMEM; 965e126ba97SEli Cohen dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); 966e126ba97SEli Cohen goto err_clr_master; 967e126ba97SEli Cohen } 968a31208b1SMajd Dibbiny 969a31208b1SMajd Dibbiny return 0; 970a31208b1SMajd Dibbiny 971a31208b1SMajd Dibbiny err_clr_master: 972a31208b1SMajd Dibbiny pci_clear_master(dev->pdev); 973a31208b1SMajd Dibbiny release_bar(dev->pdev); 974a31208b1SMajd Dibbiny err_disable: 97589d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 976a31208b1SMajd Dibbiny 977a31208b1SMajd Dibbiny err_dbg: 978a31208b1SMajd Dibbiny debugfs_remove(priv->dbg_root); 979a31208b1SMajd Dibbiny return err; 980a31208b1SMajd Dibbiny } 981a31208b1SMajd Dibbiny 982a31208b1SMajd Dibbiny static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 983a31208b1SMajd Dibbiny { 984a31208b1SMajd Dibbiny iounmap(dev->iseg); 985a31208b1SMajd Dibbiny pci_clear_master(dev->pdev); 986a31208b1SMajd Dibbiny release_bar(dev->pdev); 98789d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 988a31208b1SMajd Dibbiny debugfs_remove(priv->dbg_root); 989a31208b1SMajd Dibbiny } 990a31208b1SMajd Dibbiny 991a31208b1SMajd Dibbiny #define MLX5_IB_MOD "mlx5_ib" 992a31208b1SMajd Dibbiny static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 993a31208b1SMajd Dibbiny { 994a31208b1SMajd Dibbiny struct pci_dev *pdev = dev->pdev; 995a31208b1SMajd Dibbiny int err; 996a31208b1SMajd Dibbiny 99789d44f0aSMajd Dibbiny mutex_lock(&dev->intf_state_mutex); 9985fc7197dSMajd Dibbiny if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 99989d44f0aSMajd Dibbiny dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", 100089d44f0aSMajd Dibbiny __func__); 100189d44f0aSMajd Dibbiny goto out; 100289d44f0aSMajd Dibbiny } 100389d44f0aSMajd Dibbiny 1004e126ba97SEli Cohen dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 1005e126ba97SEli Cohen fw_rev_min(dev), fw_rev_sub(dev)); 1006e126ba97SEli Cohen 100789d44f0aSMajd Dibbiny /* on load removing any previous indication of internal error, device is 100889d44f0aSMajd Dibbiny * up 100989d44f0aSMajd Dibbiny */ 101089d44f0aSMajd Dibbiny dev->state = MLX5_DEVICE_STATE_UP; 101189d44f0aSMajd Dibbiny 1012e126ba97SEli Cohen err = mlx5_cmd_init(dev); 1013e126ba97SEli Cohen if (err) { 1014e126ba97SEli Cohen dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); 101589d44f0aSMajd Dibbiny goto out_err; 1016e126ba97SEli Cohen } 1017e126ba97SEli Cohen 1018e3297246SEli Cohen err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 1019e3297246SEli Cohen if (err) { 1020e3297246SEli Cohen dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n", 1021e3297246SEli Cohen FW_INIT_TIMEOUT_MILI); 1022e3297246SEli Cohen goto out_err; 1023e3297246SEli Cohen } 1024e3297246SEli Cohen 1025e126ba97SEli Cohen mlx5_pagealloc_init(dev); 1026cd23b14bSEli Cohen 10270b107106SEli Cohen err = mlx5_core_enable_hca(dev, 0); 1028cd23b14bSEli Cohen if (err) { 1029cd23b14bSEli Cohen dev_err(&pdev->dev, "enable hca failed\n"); 1030cd23b14bSEli Cohen goto err_pagealloc_cleanup; 1031cd23b14bSEli Cohen } 1032cd23b14bSEli Cohen 1033f62b8bb8SAmir Vadai err = mlx5_core_set_issi(dev); 1034f62b8bb8SAmir Vadai if (err) { 1035f62b8bb8SAmir Vadai dev_err(&pdev->dev, "failed to set issi\n"); 1036f62b8bb8SAmir Vadai goto err_disable_hca; 1037f62b8bb8SAmir Vadai } 1038f62b8bb8SAmir Vadai 1039cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 1); 1040cd23b14bSEli Cohen if (err) { 1041cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate boot pages\n"); 1042cd23b14bSEli Cohen goto err_disable_hca; 1043cd23b14bSEli Cohen } 1044cd23b14bSEli Cohen 1045e126ba97SEli Cohen err = set_hca_ctrl(dev); 1046e126ba97SEli Cohen if (err) { 1047e126ba97SEli Cohen dev_err(&pdev->dev, "set_hca_ctrl failed\n"); 1048cd23b14bSEli Cohen goto reclaim_boot_pages; 1049e126ba97SEli Cohen } 1050e126ba97SEli Cohen 1051e126ba97SEli Cohen err = handle_hca_cap(dev); 1052e126ba97SEli Cohen if (err) { 1053e126ba97SEli Cohen dev_err(&pdev->dev, "handle_hca_cap failed\n"); 1054cd23b14bSEli Cohen goto reclaim_boot_pages; 1055e126ba97SEli Cohen } 1056e126ba97SEli Cohen 1057f91e6d89SEran Ben Elisha err = handle_hca_cap_atomic(dev); 1058f91e6d89SEran Ben Elisha if (err) { 1059f91e6d89SEran Ben Elisha dev_err(&pdev->dev, "handle_hca_cap_atomic failed\n"); 1060f91e6d89SEran Ben Elisha goto reclaim_boot_pages; 1061f91e6d89SEran Ben Elisha } 1062f91e6d89SEran Ben Elisha 1063cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 0); 1064e126ba97SEli Cohen if (err) { 1065cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate init pages\n"); 1066cd23b14bSEli Cohen goto reclaim_boot_pages; 1067e126ba97SEli Cohen } 1068e126ba97SEli Cohen 1069e126ba97SEli Cohen err = mlx5_pagealloc_start(dev); 1070e126ba97SEli Cohen if (err) { 1071e126ba97SEli Cohen dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n"); 1072cd23b14bSEli Cohen goto reclaim_boot_pages; 1073e126ba97SEli Cohen } 1074e126ba97SEli Cohen 1075e126ba97SEli Cohen err = mlx5_cmd_init_hca(dev); 1076e126ba97SEli Cohen if (err) { 1077e126ba97SEli Cohen dev_err(&pdev->dev, "init hca failed\n"); 1078e126ba97SEli Cohen goto err_pagealloc_stop; 1079e126ba97SEli Cohen } 1080e126ba97SEli Cohen 1081e126ba97SEli Cohen mlx5_start_health_poll(dev); 1082e126ba97SEli Cohen 1083938fe83cSSaeed Mahameed err = mlx5_query_hca_caps(dev); 1084e126ba97SEli Cohen if (err) { 1085e126ba97SEli Cohen dev_err(&pdev->dev, "query hca failed\n"); 1086e126ba97SEli Cohen goto err_stop_poll; 1087e126ba97SEli Cohen } 1088e126ba97SEli Cohen 1089211e6c80SMajd Dibbiny err = mlx5_query_board_id(dev); 1090e126ba97SEli Cohen if (err) { 1091211e6c80SMajd Dibbiny dev_err(&pdev->dev, "query board id failed\n"); 1092e126ba97SEli Cohen goto err_stop_poll; 1093e126ba97SEli Cohen } 1094e126ba97SEli Cohen 1095e126ba97SEli Cohen err = mlx5_enable_msix(dev); 1096e126ba97SEli Cohen if (err) { 1097e126ba97SEli Cohen dev_err(&pdev->dev, "enable msix failed\n"); 1098e126ba97SEli Cohen goto err_stop_poll; 1099e126ba97SEli Cohen } 1100e126ba97SEli Cohen 1101e126ba97SEli Cohen err = mlx5_eq_init(dev); 1102e126ba97SEli Cohen if (err) { 1103e126ba97SEli Cohen dev_err(&pdev->dev, "failed to initialize eq\n"); 1104e126ba97SEli Cohen goto disable_msix; 1105e126ba97SEli Cohen } 1106e126ba97SEli Cohen 1107e126ba97SEli Cohen err = mlx5_alloc_uuars(dev, &priv->uuari); 1108e126ba97SEli Cohen if (err) { 1109e126ba97SEli Cohen dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 1110e126ba97SEli Cohen goto err_eq_cleanup; 1111e126ba97SEli Cohen } 1112e126ba97SEli Cohen 1113e126ba97SEli Cohen err = mlx5_start_eqs(dev); 1114e126ba97SEli Cohen if (err) { 1115e126ba97SEli Cohen dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 1116e126ba97SEli Cohen goto err_free_uar; 1117e126ba97SEli Cohen } 1118e126ba97SEli Cohen 1119233d05d2SSaeed Mahameed err = alloc_comp_eqs(dev); 1120233d05d2SSaeed Mahameed if (err) { 1121233d05d2SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc completion EQs\n"); 1122233d05d2SSaeed Mahameed goto err_stop_eqs; 1123233d05d2SSaeed Mahameed } 1124233d05d2SSaeed Mahameed 1125db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hints(dev); 11260ba42241SMoshe Lazer if (err) 1127db058a18SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n"); 1128db058a18SSaeed Mahameed 1129e126ba97SEli Cohen MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 1130e126ba97SEli Cohen 1131e126ba97SEli Cohen mlx5_init_cq_table(dev); 1132e126ba97SEli Cohen mlx5_init_qp_table(dev); 1133e126ba97SEli Cohen mlx5_init_srq_table(dev); 1134a606b0f6SMatan Barak mlx5_init_mkey_table(dev); 1135e126ba97SEli Cohen 113686d722adSMaor Gottlieb err = mlx5_init_fs(dev); 113786d722adSMaor Gottlieb if (err) { 113886d722adSMaor Gottlieb dev_err(&pdev->dev, "Failed to init flow steering\n"); 113986d722adSMaor Gottlieb goto err_fs; 114086d722adSMaor Gottlieb } 11411466cc5bSYevgeny Petrilin 11421466cc5bSYevgeny Petrilin err = mlx5_init_rl_table(dev); 11431466cc5bSYevgeny Petrilin if (err) { 11441466cc5bSYevgeny Petrilin dev_err(&pdev->dev, "Failed to init rate limiting\n"); 11451466cc5bSYevgeny Petrilin goto err_rl; 11461466cc5bSYevgeny Petrilin } 11471466cc5bSYevgeny Petrilin 1148073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 1149073bb189SSaeed Mahameed err = mlx5_eswitch_init(dev); 1150073bb189SSaeed Mahameed if (err) { 1151073bb189SSaeed Mahameed dev_err(&pdev->dev, "eswitch init failed %d\n", err); 1152073bb189SSaeed Mahameed goto err_reg_dev; 1153073bb189SSaeed Mahameed } 1154073bb189SSaeed Mahameed #endif 1155073bb189SSaeed Mahameed 1156fc50db98SEli Cohen err = mlx5_sriov_init(dev); 1157fc50db98SEli Cohen if (err) { 1158fc50db98SEli Cohen dev_err(&pdev->dev, "sriov init failed %d\n", err); 1159fc50db98SEli Cohen goto err_sriov; 1160fc50db98SEli Cohen } 1161fc50db98SEli Cohen 1162a31208b1SMajd Dibbiny err = mlx5_register_device(dev); 1163a31208b1SMajd Dibbiny if (err) { 1164a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 1165a31208b1SMajd Dibbiny goto err_reg_dev; 1166a31208b1SMajd Dibbiny } 1167a31208b1SMajd Dibbiny 1168a31208b1SMajd Dibbiny err = request_module_nowait(MLX5_IB_MOD); 1169a31208b1SMajd Dibbiny if (err) 1170a31208b1SMajd Dibbiny pr_info("failed request module on %s\n", MLX5_IB_MOD); 1171a31208b1SMajd Dibbiny 11725fc7197dSMajd Dibbiny clear_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state); 11735fc7197dSMajd Dibbiny set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 117489d44f0aSMajd Dibbiny out: 117589d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 117689d44f0aSMajd Dibbiny 1177e126ba97SEli Cohen return 0; 1178e126ba97SEli Cohen 1179fc50db98SEli Cohen err_sriov: 1180fc50db98SEli Cohen if (mlx5_sriov_cleanup(dev)) 1181fc50db98SEli Cohen dev_err(&dev->pdev->dev, "sriov cleanup failed\n"); 1182fc50db98SEli Cohen 1183073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 1184073bb189SSaeed Mahameed mlx5_eswitch_cleanup(dev->priv.eswitch); 1185073bb189SSaeed Mahameed #endif 1186a31208b1SMajd Dibbiny err_reg_dev: 11871466cc5bSYevgeny Petrilin mlx5_cleanup_rl_table(dev); 11881466cc5bSYevgeny Petrilin err_rl: 118986d722adSMaor Gottlieb mlx5_cleanup_fs(dev); 119086d722adSMaor Gottlieb err_fs: 1191a606b0f6SMatan Barak mlx5_cleanup_mkey_table(dev); 1192a31208b1SMajd Dibbiny mlx5_cleanup_srq_table(dev); 1193a31208b1SMajd Dibbiny mlx5_cleanup_qp_table(dev); 1194a31208b1SMajd Dibbiny mlx5_cleanup_cq_table(dev); 1195a31208b1SMajd Dibbiny mlx5_irq_clear_affinity_hints(dev); 1196db058a18SSaeed Mahameed free_comp_eqs(dev); 1197db058a18SSaeed Mahameed 1198233d05d2SSaeed Mahameed err_stop_eqs: 1199233d05d2SSaeed Mahameed mlx5_stop_eqs(dev); 1200233d05d2SSaeed Mahameed 1201e126ba97SEli Cohen err_free_uar: 1202e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 1203e126ba97SEli Cohen 1204e126ba97SEli Cohen err_eq_cleanup: 1205e126ba97SEli Cohen mlx5_eq_cleanup(dev); 1206e126ba97SEli Cohen 1207e126ba97SEli Cohen disable_msix: 1208e126ba97SEli Cohen mlx5_disable_msix(dev); 1209e126ba97SEli Cohen 1210e126ba97SEli Cohen err_stop_poll: 1211e126ba97SEli Cohen mlx5_stop_health_poll(dev); 12121bde6e30SEli Cohen if (mlx5_cmd_teardown_hca(dev)) { 12131bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 121489d44f0aSMajd Dibbiny goto out_err; 12151bde6e30SEli Cohen } 1216e126ba97SEli Cohen 1217e126ba97SEli Cohen err_pagealloc_stop: 1218e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 1219e126ba97SEli Cohen 1220cd23b14bSEli Cohen reclaim_boot_pages: 1221e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 1222e126ba97SEli Cohen 1223cd23b14bSEli Cohen err_disable_hca: 12240b107106SEli Cohen mlx5_core_disable_hca(dev, 0); 1225cd23b14bSEli Cohen 1226e126ba97SEli Cohen err_pagealloc_cleanup: 1227e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 1228e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 1229e126ba97SEli Cohen 123089d44f0aSMajd Dibbiny out_err: 123189d44f0aSMajd Dibbiny dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 123289d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 123389d44f0aSMajd Dibbiny 1234e126ba97SEli Cohen return err; 1235e126ba97SEli Cohen } 1236e126ba97SEli Cohen 1237a31208b1SMajd Dibbiny static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 1238e126ba97SEli Cohen { 123989d44f0aSMajd Dibbiny int err = 0; 1240e126ba97SEli Cohen 1241fc50db98SEli Cohen err = mlx5_sriov_cleanup(dev); 1242fc50db98SEli Cohen if (err) { 1243fc50db98SEli Cohen dev_warn(&dev->pdev->dev, "%s: sriov cleanup failed - abort\n", 1244fc50db98SEli Cohen __func__); 1245fc50db98SEli Cohen return err; 1246fc50db98SEli Cohen } 1247fc50db98SEli Cohen 124889d44f0aSMajd Dibbiny mutex_lock(&dev->intf_state_mutex); 12495fc7197dSMajd Dibbiny if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) { 125089d44f0aSMajd Dibbiny dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", 125189d44f0aSMajd Dibbiny __func__); 125289d44f0aSMajd Dibbiny goto out; 125389d44f0aSMajd Dibbiny } 1254a31208b1SMajd Dibbiny mlx5_unregister_device(dev); 1255073bb189SSaeed Mahameed #ifdef CONFIG_MLX5_CORE_EN 1256073bb189SSaeed Mahameed mlx5_eswitch_cleanup(dev->priv.eswitch); 1257073bb189SSaeed Mahameed #endif 1258073bb189SSaeed Mahameed 12591466cc5bSYevgeny Petrilin mlx5_cleanup_rl_table(dev); 126086d722adSMaor Gottlieb mlx5_cleanup_fs(dev); 1261a606b0f6SMatan Barak mlx5_cleanup_mkey_table(dev); 1262e126ba97SEli Cohen mlx5_cleanup_srq_table(dev); 1263e126ba97SEli Cohen mlx5_cleanup_qp_table(dev); 1264e126ba97SEli Cohen mlx5_cleanup_cq_table(dev); 1265db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hints(dev); 1266233d05d2SSaeed Mahameed free_comp_eqs(dev); 1267e126ba97SEli Cohen mlx5_stop_eqs(dev); 1268e126ba97SEli Cohen mlx5_free_uuars(dev, &priv->uuari); 1269e126ba97SEli Cohen mlx5_eq_cleanup(dev); 1270e126ba97SEli Cohen mlx5_disable_msix(dev); 1271e126ba97SEli Cohen mlx5_stop_health_poll(dev); 1272ac6ea6e8SEli Cohen err = mlx5_cmd_teardown_hca(dev); 1273ac6ea6e8SEli Cohen if (err) { 12741bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 1275ac6ea6e8SEli Cohen goto out; 12761bde6e30SEli Cohen } 1277e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 1278e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 12790b107106SEli Cohen mlx5_core_disable_hca(dev, 0); 1280e126ba97SEli Cohen mlx5_pagealloc_cleanup(dev); 1281e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 12829603b61dSJack Morgenstein 1283ac6ea6e8SEli Cohen out: 12845fc7197dSMajd Dibbiny clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 12855fc7197dSMajd Dibbiny set_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state); 128689d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 1287ac6ea6e8SEli Cohen return err; 12889603b61dSJack Morgenstein } 128964613d94SSaeed Mahameed 129089d44f0aSMajd Dibbiny void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 12914d2f9bbbSJack Morgenstein unsigned long param) 12929603b61dSJack Morgenstein { 12939603b61dSJack Morgenstein struct mlx5_priv *priv = &dev->priv; 12949603b61dSJack Morgenstein struct mlx5_device_context *dev_ctx; 12959603b61dSJack Morgenstein unsigned long flags; 12969603b61dSJack Morgenstein 12979603b61dSJack Morgenstein spin_lock_irqsave(&priv->ctx_lock, flags); 12989603b61dSJack Morgenstein 12999603b61dSJack Morgenstein list_for_each_entry(dev_ctx, &priv->ctx_list, list) 13009603b61dSJack Morgenstein if (dev_ctx->intf->event) 13014d2f9bbbSJack Morgenstein dev_ctx->intf->event(dev, dev_ctx->context, event, param); 13029603b61dSJack Morgenstein 13039603b61dSJack Morgenstein spin_unlock_irqrestore(&priv->ctx_lock, flags); 13049603b61dSJack Morgenstein } 13059603b61dSJack Morgenstein 13069603b61dSJack Morgenstein struct mlx5_core_event_handler { 13079603b61dSJack Morgenstein void (*event)(struct mlx5_core_dev *dev, 13089603b61dSJack Morgenstein enum mlx5_dev_event event, 13099603b61dSJack Morgenstein void *data); 13109603b61dSJack Morgenstein }; 13119603b61dSJack Morgenstein 1312feae9087SOr Gerlitz static const struct devlink_ops mlx5_devlink_ops = { 1313feae9087SOr Gerlitz #ifdef CONFIG_MLX5_CORE_EN 1314feae9087SOr Gerlitz .eswitch_mode_set = mlx5_devlink_eswitch_mode_set, 1315feae9087SOr Gerlitz .eswitch_mode_get = mlx5_devlink_eswitch_mode_get, 1316feae9087SOr Gerlitz #endif 1317feae9087SOr Gerlitz }; 1318f66f049fSEli Cohen 13199603b61dSJack Morgenstein static int init_one(struct pci_dev *pdev, 13209603b61dSJack Morgenstein const struct pci_device_id *id) 13219603b61dSJack Morgenstein { 13229603b61dSJack Morgenstein struct mlx5_core_dev *dev; 1323feae9087SOr Gerlitz struct devlink *devlink; 13249603b61dSJack Morgenstein struct mlx5_priv *priv; 13259603b61dSJack Morgenstein int err; 13269603b61dSJack Morgenstein 1327feae9087SOr Gerlitz devlink = devlink_alloc(&mlx5_devlink_ops, sizeof(*dev)); 1328feae9087SOr Gerlitz if (!devlink) { 13299603b61dSJack Morgenstein dev_err(&pdev->dev, "kzalloc failed\n"); 13309603b61dSJack Morgenstein return -ENOMEM; 13319603b61dSJack Morgenstein } 1332feae9087SOr Gerlitz 1333feae9087SOr Gerlitz dev = devlink_priv(devlink); 13349603b61dSJack Morgenstein priv = &dev->priv; 1335fc50db98SEli Cohen priv->pci_dev_data = id->driver_data; 13369603b61dSJack Morgenstein 13379603b61dSJack Morgenstein pci_set_drvdata(pdev, dev); 13389603b61dSJack Morgenstein 13399603b61dSJack Morgenstein if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profile)) { 13402974ab6eSSaeed Mahameed mlx5_core_warn(dev, 13412974ab6eSSaeed Mahameed "selected profile out of range, selecting default (%d)\n", 13429603b61dSJack Morgenstein MLX5_DEFAULT_PROF); 13439603b61dSJack Morgenstein prof_sel = MLX5_DEFAULT_PROF; 13449603b61dSJack Morgenstein } 13459603b61dSJack Morgenstein dev->profile = &profile[prof_sel]; 1346a31208b1SMajd Dibbiny dev->pdev = pdev; 13479603b61dSJack Morgenstein dev->event = mlx5_core_event; 13489603b61dSJack Morgenstein 1349364d1798SEli Cohen INIT_LIST_HEAD(&priv->ctx_list); 1350364d1798SEli Cohen spin_lock_init(&priv->ctx_lock); 135189d44f0aSMajd Dibbiny mutex_init(&dev->pci_status_mutex); 135289d44f0aSMajd Dibbiny mutex_init(&dev->intf_state_mutex); 1353a31208b1SMajd Dibbiny err = mlx5_pci_init(dev, priv); 13549603b61dSJack Morgenstein if (err) { 1355a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err); 1356a31208b1SMajd Dibbiny goto clean_dev; 13579603b61dSJack Morgenstein } 13589603b61dSJack Morgenstein 1359ac6ea6e8SEli Cohen err = mlx5_health_init(dev); 1360ac6ea6e8SEli Cohen if (err) { 1361ac6ea6e8SEli Cohen dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err); 1362ac6ea6e8SEli Cohen goto close_pci; 1363ac6ea6e8SEli Cohen } 1364ac6ea6e8SEli Cohen 1365a31208b1SMajd Dibbiny err = mlx5_load_one(dev, priv); 13669603b61dSJack Morgenstein if (err) { 1367a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err); 1368ac6ea6e8SEli Cohen goto clean_health; 13699603b61dSJack Morgenstein } 13709603b61dSJack Morgenstein 1371feae9087SOr Gerlitz err = devlink_register(devlink, &pdev->dev); 1372feae9087SOr Gerlitz if (err) 1373feae9087SOr Gerlitz goto clean_load; 1374feae9087SOr Gerlitz 13759603b61dSJack Morgenstein return 0; 13769603b61dSJack Morgenstein 1377feae9087SOr Gerlitz clean_load: 1378feae9087SOr Gerlitz mlx5_unload_one(dev, priv); 1379ac6ea6e8SEli Cohen clean_health: 1380ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1381a31208b1SMajd Dibbiny close_pci: 1382a31208b1SMajd Dibbiny mlx5_pci_close(dev, priv); 1383a31208b1SMajd Dibbiny clean_dev: 1384a31208b1SMajd Dibbiny pci_set_drvdata(pdev, NULL); 1385feae9087SOr Gerlitz devlink_free(devlink); 1386a31208b1SMajd Dibbiny 13879603b61dSJack Morgenstein return err; 13889603b61dSJack Morgenstein } 1389a31208b1SMajd Dibbiny 13909603b61dSJack Morgenstein static void remove_one(struct pci_dev *pdev) 13919603b61dSJack Morgenstein { 13929603b61dSJack Morgenstein struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1393feae9087SOr Gerlitz struct devlink *devlink = priv_to_devlink(dev); 1394a31208b1SMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 13959603b61dSJack Morgenstein 1396feae9087SOr Gerlitz devlink_unregister(devlink); 1397a31208b1SMajd Dibbiny if (mlx5_unload_one(dev, priv)) { 1398a31208b1SMajd Dibbiny dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); 1399ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1400a31208b1SMajd Dibbiny return; 1401a31208b1SMajd Dibbiny } 1402ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1403a31208b1SMajd Dibbiny mlx5_pci_close(dev, priv); 1404a31208b1SMajd Dibbiny pci_set_drvdata(pdev, NULL); 1405feae9087SOr Gerlitz devlink_free(devlink); 14069603b61dSJack Morgenstein } 14079603b61dSJack Morgenstein 140889d44f0aSMajd Dibbiny static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, 140989d44f0aSMajd Dibbiny pci_channel_state_t state) 141089d44f0aSMajd Dibbiny { 141189d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 141289d44f0aSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 141389d44f0aSMajd Dibbiny 141489d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 141589d44f0aSMajd Dibbiny mlx5_enter_error_state(dev); 141689d44f0aSMajd Dibbiny mlx5_unload_one(dev, priv); 141789d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 141889d44f0aSMajd Dibbiny return state == pci_channel_io_perm_failure ? 141989d44f0aSMajd Dibbiny PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 142089d44f0aSMajd Dibbiny } 142189d44f0aSMajd Dibbiny 142289d44f0aSMajd Dibbiny static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) 142389d44f0aSMajd Dibbiny { 142489d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 142589d44f0aSMajd Dibbiny int err = 0; 142689d44f0aSMajd Dibbiny 142789d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 142889d44f0aSMajd Dibbiny 142989d44f0aSMajd Dibbiny err = mlx5_pci_enable_device(dev); 143089d44f0aSMajd Dibbiny if (err) { 143189d44f0aSMajd Dibbiny dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" 143289d44f0aSMajd Dibbiny , __func__, err); 143389d44f0aSMajd Dibbiny return PCI_ERS_RESULT_DISCONNECT; 143489d44f0aSMajd Dibbiny } 143589d44f0aSMajd Dibbiny pci_set_master(pdev); 143689d44f0aSMajd Dibbiny pci_set_power_state(pdev, PCI_D0); 143789d44f0aSMajd Dibbiny pci_restore_state(pdev); 143889d44f0aSMajd Dibbiny 143989d44f0aSMajd Dibbiny return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 144089d44f0aSMajd Dibbiny } 144189d44f0aSMajd Dibbiny 144289d44f0aSMajd Dibbiny void mlx5_disable_device(struct mlx5_core_dev *dev) 144389d44f0aSMajd Dibbiny { 144489d44f0aSMajd Dibbiny mlx5_pci_err_detected(dev->pdev, 0); 144589d44f0aSMajd Dibbiny } 144689d44f0aSMajd Dibbiny 1447d57847dcSDaniel Jurgens /* wait for the device to show vital signs by waiting 1448d57847dcSDaniel Jurgens * for the health counter to start counting. 144989d44f0aSMajd Dibbiny */ 1450d57847dcSDaniel Jurgens static int wait_vital(struct pci_dev *pdev) 145189d44f0aSMajd Dibbiny { 145289d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 145389d44f0aSMajd Dibbiny struct mlx5_core_health *health = &dev->priv.health; 145489d44f0aSMajd Dibbiny const int niter = 100; 1455d57847dcSDaniel Jurgens u32 last_count = 0; 145689d44f0aSMajd Dibbiny u32 count; 145789d44f0aSMajd Dibbiny int i; 145889d44f0aSMajd Dibbiny 145989d44f0aSMajd Dibbiny for (i = 0; i < niter; i++) { 146089d44f0aSMajd Dibbiny count = ioread32be(health->health_counter); 146189d44f0aSMajd Dibbiny if (count && count != 0xffffffff) { 1462d57847dcSDaniel Jurgens if (last_count && last_count != count) { 146389d44f0aSMajd Dibbiny dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); 1464d57847dcSDaniel Jurgens return 0; 1465d57847dcSDaniel Jurgens } 1466d57847dcSDaniel Jurgens last_count = count; 146789d44f0aSMajd Dibbiny } 146889d44f0aSMajd Dibbiny msleep(50); 146989d44f0aSMajd Dibbiny } 147089d44f0aSMajd Dibbiny 1471d57847dcSDaniel Jurgens return -ETIMEDOUT; 147289d44f0aSMajd Dibbiny } 147389d44f0aSMajd Dibbiny 147489d44f0aSMajd Dibbiny static void mlx5_pci_resume(struct pci_dev *pdev) 147589d44f0aSMajd Dibbiny { 147689d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 147789d44f0aSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 147889d44f0aSMajd Dibbiny int err; 147989d44f0aSMajd Dibbiny 148089d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 148189d44f0aSMajd Dibbiny 148289d44f0aSMajd Dibbiny pci_save_state(pdev); 1483d57847dcSDaniel Jurgens err = wait_vital(pdev); 1484d57847dcSDaniel Jurgens if (err) { 1485d57847dcSDaniel Jurgens dev_err(&pdev->dev, "%s: wait_vital timed out\n", __func__); 1486d57847dcSDaniel Jurgens return; 1487d57847dcSDaniel Jurgens } 148889d44f0aSMajd Dibbiny 148989d44f0aSMajd Dibbiny err = mlx5_load_one(dev, priv); 149089d44f0aSMajd Dibbiny if (err) 149189d44f0aSMajd Dibbiny dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" 149289d44f0aSMajd Dibbiny , __func__, err); 149389d44f0aSMajd Dibbiny else 149489d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s: device recovered\n", __func__); 149589d44f0aSMajd Dibbiny } 149689d44f0aSMajd Dibbiny 149789d44f0aSMajd Dibbiny static const struct pci_error_handlers mlx5_err_handler = { 149889d44f0aSMajd Dibbiny .error_detected = mlx5_pci_err_detected, 149989d44f0aSMajd Dibbiny .slot_reset = mlx5_pci_slot_reset, 150089d44f0aSMajd Dibbiny .resume = mlx5_pci_resume 150189d44f0aSMajd Dibbiny }; 150289d44f0aSMajd Dibbiny 15035fc7197dSMajd Dibbiny static void shutdown(struct pci_dev *pdev) 15045fc7197dSMajd Dibbiny { 15055fc7197dSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 15065fc7197dSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 15075fc7197dSMajd Dibbiny 15085fc7197dSMajd Dibbiny dev_info(&pdev->dev, "Shutdown was called\n"); 15095fc7197dSMajd Dibbiny /* Notify mlx5 clients that the kernel is being shut down */ 15105fc7197dSMajd Dibbiny set_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &dev->intf_state); 15115fc7197dSMajd Dibbiny mlx5_unload_one(dev, priv); 15125fc7197dSMajd Dibbiny mlx5_pci_disable_device(dev); 15135fc7197dSMajd Dibbiny } 15145fc7197dSMajd Dibbiny 15159603b61dSJack Morgenstein static const struct pci_device_id mlx5_core_pci_table[] = { 15161c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */ 1517fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */ 15181c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */ 1519fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */ 15201c755cc5SOr Gerlitz { PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */ 1521fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */ 15227092fe86SMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */ 152364dbbdfeSMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */ 15247092fe86SMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5, PCIe 4.0 */ 15259603b61dSJack Morgenstein { 0, } 15269603b61dSJack Morgenstein }; 15279603b61dSJack Morgenstein 15289603b61dSJack Morgenstein MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 15299603b61dSJack Morgenstein 15309603b61dSJack Morgenstein static struct pci_driver mlx5_core_driver = { 15319603b61dSJack Morgenstein .name = DRIVER_NAME, 15329603b61dSJack Morgenstein .id_table = mlx5_core_pci_table, 15339603b61dSJack Morgenstein .probe = init_one, 153489d44f0aSMajd Dibbiny .remove = remove_one, 15355fc7197dSMajd Dibbiny .shutdown = shutdown, 1536fc50db98SEli Cohen .err_handler = &mlx5_err_handler, 1537fc50db98SEli Cohen .sriov_configure = mlx5_core_sriov_configure, 15389603b61dSJack Morgenstein }; 1539e126ba97SEli Cohen 1540e126ba97SEli Cohen static int __init init(void) 1541e126ba97SEli Cohen { 1542e126ba97SEli Cohen int err; 1543e126ba97SEli Cohen 1544e126ba97SEli Cohen mlx5_register_debugfs(); 1545e126ba97SEli Cohen 15469603b61dSJack Morgenstein err = pci_register_driver(&mlx5_core_driver); 15479603b61dSJack Morgenstein if (err) 1548ac6ea6e8SEli Cohen goto err_debug; 15499603b61dSJack Morgenstein 1550f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1551f62b8bb8SAmir Vadai mlx5e_init(); 1552f62b8bb8SAmir Vadai #endif 1553f62b8bb8SAmir Vadai 1554e126ba97SEli Cohen return 0; 1555e126ba97SEli Cohen 1556e126ba97SEli Cohen err_debug: 1557e126ba97SEli Cohen mlx5_unregister_debugfs(); 1558e126ba97SEli Cohen return err; 1559e126ba97SEli Cohen } 1560e126ba97SEli Cohen 1561e126ba97SEli Cohen static void __exit cleanup(void) 1562e126ba97SEli Cohen { 1563f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1564f62b8bb8SAmir Vadai mlx5e_cleanup(); 1565f62b8bb8SAmir Vadai #endif 15669603b61dSJack Morgenstein pci_unregister_driver(&mlx5_core_driver); 1567e126ba97SEli Cohen mlx5_unregister_debugfs(); 1568e126ba97SEli Cohen } 1569e126ba97SEli Cohen 1570e126ba97SEli Cohen module_init(init); 1571e126ba97SEli Cohen module_exit(cleanup); 1572