1e126ba97SEli Cohen /* 2302bdf68SSaeed Mahameed * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3e126ba97SEli Cohen * 4e126ba97SEli Cohen * This software is available to you under a choice of one of two 5e126ba97SEli Cohen * licenses. You may choose to be licensed under the terms of the GNU 6e126ba97SEli Cohen * General Public License (GPL) Version 2, available from the file 7e126ba97SEli Cohen * COPYING in the main directory of this source tree, or the 8e126ba97SEli Cohen * OpenIB.org BSD license below: 9e126ba97SEli Cohen * 10e126ba97SEli Cohen * Redistribution and use in source and binary forms, with or 11e126ba97SEli Cohen * without modification, are permitted provided that the following 12e126ba97SEli Cohen * conditions are met: 13e126ba97SEli Cohen * 14e126ba97SEli Cohen * - Redistributions of source code must retain the above 15e126ba97SEli Cohen * copyright notice, this list of conditions and the following 16e126ba97SEli Cohen * disclaimer. 17e126ba97SEli Cohen * 18e126ba97SEli Cohen * - Redistributions in binary form must reproduce the above 19e126ba97SEli Cohen * copyright notice, this list of conditions and the following 20e126ba97SEli Cohen * disclaimer in the documentation and/or other materials 21e126ba97SEli Cohen * provided with the distribution. 22e126ba97SEli Cohen * 23e126ba97SEli Cohen * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e126ba97SEli Cohen * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e126ba97SEli Cohen * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e126ba97SEli Cohen * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e126ba97SEli Cohen * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e126ba97SEli Cohen * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e126ba97SEli Cohen * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e126ba97SEli Cohen * SOFTWARE. 31e126ba97SEli Cohen */ 32e126ba97SEli Cohen 33adec640eSChristoph Hellwig #include <linux/highmem.h> 34e126ba97SEli Cohen #include <linux/module.h> 35e126ba97SEli Cohen #include <linux/init.h> 36e126ba97SEli Cohen #include <linux/errno.h> 37e126ba97SEli Cohen #include <linux/pci.h> 38e126ba97SEli Cohen #include <linux/dma-mapping.h> 39e126ba97SEli Cohen #include <linux/slab.h> 40e126ba97SEli Cohen #include <linux/io-mapping.h> 41db058a18SSaeed Mahameed #include <linux/interrupt.h> 42e3297246SEli Cohen #include <linux/delay.h> 43e126ba97SEli Cohen #include <linux/mlx5/driver.h> 44e126ba97SEli Cohen #include <linux/mlx5/cq.h> 45e126ba97SEli Cohen #include <linux/mlx5/qp.h> 46e126ba97SEli Cohen #include <linux/mlx5/srq.h> 47e126ba97SEli Cohen #include <linux/debugfs.h> 48f66f049fSEli Cohen #include <linux/kmod.h> 49b775516bSEli Cohen #include <linux/mlx5/mlx5_ifc.h> 505a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 515a7b27ebSMaor Gottlieb #include <linux/cpu_rmap.h> 525a7b27ebSMaor Gottlieb #endif 53feae9087SOr Gerlitz #include <net/devlink.h> 54e126ba97SEli Cohen #include "mlx5_core.h" 5586d722adSMaor Gottlieb #include "fs_core.h" 56eeb66cdbSSaeed Mahameed #include "lib/mpfs.h" 57073bb189SSaeed Mahameed #include "eswitch.h" 5852ec462eSIlan Tayari #include "lib/mlx5.h" 59e29341fbSIlan Tayari #include "fpga/core.h" 60bebb23e6SIlan Tayari #include "accel/ipsec.h" 61e126ba97SEli Cohen 62e126ba97SEli Cohen MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 634ae6c18cSAchiad Shochat MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 64e126ba97SEli Cohen MODULE_LICENSE("Dual BSD/GPL"); 65e126ba97SEli Cohen MODULE_VERSION(DRIVER_VERSION); 66e126ba97SEli Cohen 67f663ad98SKamal Heib unsigned int mlx5_core_debug_mask; 68f663ad98SKamal Heib module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644); 69e126ba97SEli Cohen MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 70e126ba97SEli Cohen 719603b61dSJack Morgenstein #define MLX5_DEFAULT_PROF 2 72f663ad98SKamal Heib static unsigned int prof_sel = MLX5_DEFAULT_PROF; 73f663ad98SKamal Heib module_param_named(prof_sel, prof_sel, uint, 0444); 749603b61dSJack Morgenstein MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 759603b61dSJack Morgenstein 76f91e6d89SEran Ben Elisha enum { 77f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_BE = 0x0, 78f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, 79f91e6d89SEran Ben Elisha }; 80f91e6d89SEran Ben Elisha 819603b61dSJack Morgenstein static struct mlx5_profile profile[] = { 829603b61dSJack Morgenstein [0] = { 839603b61dSJack Morgenstein .mask = 0, 849603b61dSJack Morgenstein }, 859603b61dSJack Morgenstein [1] = { 869603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE, 879603b61dSJack Morgenstein .log_max_qp = 12, 889603b61dSJack Morgenstein }, 899603b61dSJack Morgenstein [2] = { 909603b61dSJack Morgenstein .mask = MLX5_PROF_MASK_QP_SIZE | 919603b61dSJack Morgenstein MLX5_PROF_MASK_MR_CACHE, 925f40b4edSMaor Gottlieb .log_max_qp = 18, 939603b61dSJack Morgenstein .mr_cache[0] = { 949603b61dSJack Morgenstein .size = 500, 959603b61dSJack Morgenstein .limit = 250 969603b61dSJack Morgenstein }, 979603b61dSJack Morgenstein .mr_cache[1] = { 989603b61dSJack Morgenstein .size = 500, 999603b61dSJack Morgenstein .limit = 250 1009603b61dSJack Morgenstein }, 1019603b61dSJack Morgenstein .mr_cache[2] = { 1029603b61dSJack Morgenstein .size = 500, 1039603b61dSJack Morgenstein .limit = 250 1049603b61dSJack Morgenstein }, 1059603b61dSJack Morgenstein .mr_cache[3] = { 1069603b61dSJack Morgenstein .size = 500, 1079603b61dSJack Morgenstein .limit = 250 1089603b61dSJack Morgenstein }, 1099603b61dSJack Morgenstein .mr_cache[4] = { 1109603b61dSJack Morgenstein .size = 500, 1119603b61dSJack Morgenstein .limit = 250 1129603b61dSJack Morgenstein }, 1139603b61dSJack Morgenstein .mr_cache[5] = { 1149603b61dSJack Morgenstein .size = 500, 1159603b61dSJack Morgenstein .limit = 250 1169603b61dSJack Morgenstein }, 1179603b61dSJack Morgenstein .mr_cache[6] = { 1189603b61dSJack Morgenstein .size = 500, 1199603b61dSJack Morgenstein .limit = 250 1209603b61dSJack Morgenstein }, 1219603b61dSJack Morgenstein .mr_cache[7] = { 1229603b61dSJack Morgenstein .size = 500, 1239603b61dSJack Morgenstein .limit = 250 1249603b61dSJack Morgenstein }, 1259603b61dSJack Morgenstein .mr_cache[8] = { 1269603b61dSJack Morgenstein .size = 500, 1279603b61dSJack Morgenstein .limit = 250 1289603b61dSJack Morgenstein }, 1299603b61dSJack Morgenstein .mr_cache[9] = { 1309603b61dSJack Morgenstein .size = 500, 1319603b61dSJack Morgenstein .limit = 250 1329603b61dSJack Morgenstein }, 1339603b61dSJack Morgenstein .mr_cache[10] = { 1349603b61dSJack Morgenstein .size = 500, 1359603b61dSJack Morgenstein .limit = 250 1369603b61dSJack Morgenstein }, 1379603b61dSJack Morgenstein .mr_cache[11] = { 1389603b61dSJack Morgenstein .size = 500, 1399603b61dSJack Morgenstein .limit = 250 1409603b61dSJack Morgenstein }, 1419603b61dSJack Morgenstein .mr_cache[12] = { 1429603b61dSJack Morgenstein .size = 64, 1439603b61dSJack Morgenstein .limit = 32 1449603b61dSJack Morgenstein }, 1459603b61dSJack Morgenstein .mr_cache[13] = { 1469603b61dSJack Morgenstein .size = 32, 1479603b61dSJack Morgenstein .limit = 16 1489603b61dSJack Morgenstein }, 1499603b61dSJack Morgenstein .mr_cache[14] = { 1509603b61dSJack Morgenstein .size = 16, 1519603b61dSJack Morgenstein .limit = 8 1529603b61dSJack Morgenstein }, 1539603b61dSJack Morgenstein .mr_cache[15] = { 1549603b61dSJack Morgenstein .size = 8, 1559603b61dSJack Morgenstein .limit = 4 1569603b61dSJack Morgenstein }, 1577d0cc6edSArtemy Kovalyov .mr_cache[16] = { 1587d0cc6edSArtemy Kovalyov .size = 8, 1597d0cc6edSArtemy Kovalyov .limit = 4 1607d0cc6edSArtemy Kovalyov }, 1617d0cc6edSArtemy Kovalyov .mr_cache[17] = { 1627d0cc6edSArtemy Kovalyov .size = 8, 1637d0cc6edSArtemy Kovalyov .limit = 4 1647d0cc6edSArtemy Kovalyov }, 1657d0cc6edSArtemy Kovalyov .mr_cache[18] = { 1667d0cc6edSArtemy Kovalyov .size = 8, 1677d0cc6edSArtemy Kovalyov .limit = 4 1687d0cc6edSArtemy Kovalyov }, 1697d0cc6edSArtemy Kovalyov .mr_cache[19] = { 1707d0cc6edSArtemy Kovalyov .size = 4, 1717d0cc6edSArtemy Kovalyov .limit = 2 1727d0cc6edSArtemy Kovalyov }, 1737d0cc6edSArtemy Kovalyov .mr_cache[20] = { 1747d0cc6edSArtemy Kovalyov .size = 4, 1757d0cc6edSArtemy Kovalyov .limit = 2 1767d0cc6edSArtemy Kovalyov }, 1779603b61dSJack Morgenstein }, 1789603b61dSJack Morgenstein }; 179e126ba97SEli Cohen 180e3297246SEli Cohen #define FW_INIT_TIMEOUT_MILI 2000 181e3297246SEli Cohen #define FW_INIT_WAIT_MS 2 1826c780a02SEli Cohen #define FW_PRE_INIT_TIMEOUT_MILI 10000 183e3297246SEli Cohen 184e3297246SEli Cohen static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 185e3297246SEli Cohen { 186e3297246SEli Cohen unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); 187e3297246SEli Cohen int err = 0; 188e3297246SEli Cohen 189e3297246SEli Cohen while (fw_initializing(dev)) { 190e3297246SEli Cohen if (time_after(jiffies, end)) { 191e3297246SEli Cohen err = -EBUSY; 192e3297246SEli Cohen break; 193e3297246SEli Cohen } 194e3297246SEli Cohen msleep(FW_INIT_WAIT_MS); 195e3297246SEli Cohen } 196e3297246SEli Cohen 197e3297246SEli Cohen return err; 198e3297246SEli Cohen } 199e3297246SEli Cohen 200012e50e1SHuy Nguyen static void mlx5_set_driver_version(struct mlx5_core_dev *dev) 201012e50e1SHuy Nguyen { 202012e50e1SHuy Nguyen int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in, 203012e50e1SHuy Nguyen driver_version); 204012e50e1SHuy Nguyen u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {0}; 205012e50e1SHuy Nguyen u8 out[MLX5_ST_SZ_BYTES(set_driver_version_out)] = {0}; 206012e50e1SHuy Nguyen int remaining_size = driver_ver_sz; 207012e50e1SHuy Nguyen char *string; 208012e50e1SHuy Nguyen 209012e50e1SHuy Nguyen if (!MLX5_CAP_GEN(dev, driver_version)) 210012e50e1SHuy Nguyen return; 211012e50e1SHuy Nguyen 212012e50e1SHuy Nguyen string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version); 213012e50e1SHuy Nguyen 214012e50e1SHuy Nguyen strncpy(string, "Linux", remaining_size); 215012e50e1SHuy Nguyen 216012e50e1SHuy Nguyen remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 217012e50e1SHuy Nguyen strncat(string, ",", remaining_size); 218012e50e1SHuy Nguyen 219012e50e1SHuy Nguyen remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 220012e50e1SHuy Nguyen strncat(string, DRIVER_NAME, remaining_size); 221012e50e1SHuy Nguyen 222012e50e1SHuy Nguyen remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 223012e50e1SHuy Nguyen strncat(string, ",", remaining_size); 224012e50e1SHuy Nguyen 225012e50e1SHuy Nguyen remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 226012e50e1SHuy Nguyen strncat(string, DRIVER_VERSION, remaining_size); 227012e50e1SHuy Nguyen 228012e50e1SHuy Nguyen /*Send the command*/ 229012e50e1SHuy Nguyen MLX5_SET(set_driver_version_in, in, opcode, 230012e50e1SHuy Nguyen MLX5_CMD_OP_SET_DRIVER_VERSION); 231012e50e1SHuy Nguyen 232012e50e1SHuy Nguyen mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 233012e50e1SHuy Nguyen } 234012e50e1SHuy Nguyen 235e126ba97SEli Cohen static int set_dma_caps(struct pci_dev *pdev) 236e126ba97SEli Cohen { 237e126ba97SEli Cohen int err; 238e126ba97SEli Cohen 239e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 240e126ba97SEli Cohen if (err) { 2411a91de28SJoe Perches dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 242e126ba97SEli Cohen err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 243e126ba97SEli Cohen if (err) { 2441a91de28SJoe Perches dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 245e126ba97SEli Cohen return err; 246e126ba97SEli Cohen } 247e126ba97SEli Cohen } 248e126ba97SEli Cohen 249e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 250e126ba97SEli Cohen if (err) { 251e126ba97SEli Cohen dev_warn(&pdev->dev, 2521a91de28SJoe Perches "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 253e126ba97SEli Cohen err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 254e126ba97SEli Cohen if (err) { 255e126ba97SEli Cohen dev_err(&pdev->dev, 2561a91de28SJoe Perches "Can't set consistent PCI DMA mask, aborting\n"); 257e126ba97SEli Cohen return err; 258e126ba97SEli Cohen } 259e126ba97SEli Cohen } 260e126ba97SEli Cohen 261e126ba97SEli Cohen dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 262e126ba97SEli Cohen return err; 263e126ba97SEli Cohen } 264e126ba97SEli Cohen 26589d44f0aSMajd Dibbiny static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) 26689d44f0aSMajd Dibbiny { 26789d44f0aSMajd Dibbiny struct pci_dev *pdev = dev->pdev; 26889d44f0aSMajd Dibbiny int err = 0; 26989d44f0aSMajd Dibbiny 27089d44f0aSMajd Dibbiny mutex_lock(&dev->pci_status_mutex); 27189d44f0aSMajd Dibbiny if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { 27289d44f0aSMajd Dibbiny err = pci_enable_device(pdev); 27389d44f0aSMajd Dibbiny if (!err) 27489d44f0aSMajd Dibbiny dev->pci_status = MLX5_PCI_STATUS_ENABLED; 27589d44f0aSMajd Dibbiny } 27689d44f0aSMajd Dibbiny mutex_unlock(&dev->pci_status_mutex); 27789d44f0aSMajd Dibbiny 27889d44f0aSMajd Dibbiny return err; 27989d44f0aSMajd Dibbiny } 28089d44f0aSMajd Dibbiny 28189d44f0aSMajd Dibbiny static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) 28289d44f0aSMajd Dibbiny { 28389d44f0aSMajd Dibbiny struct pci_dev *pdev = dev->pdev; 28489d44f0aSMajd Dibbiny 28589d44f0aSMajd Dibbiny mutex_lock(&dev->pci_status_mutex); 28689d44f0aSMajd Dibbiny if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { 28789d44f0aSMajd Dibbiny pci_disable_device(pdev); 28889d44f0aSMajd Dibbiny dev->pci_status = MLX5_PCI_STATUS_DISABLED; 28989d44f0aSMajd Dibbiny } 29089d44f0aSMajd Dibbiny mutex_unlock(&dev->pci_status_mutex); 29189d44f0aSMajd Dibbiny } 29289d44f0aSMajd Dibbiny 293e126ba97SEli Cohen static int request_bar(struct pci_dev *pdev) 294e126ba97SEli Cohen { 295e126ba97SEli Cohen int err = 0; 296e126ba97SEli Cohen 297e126ba97SEli Cohen if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 2981a91de28SJoe Perches dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 299e126ba97SEli Cohen return -ENODEV; 300e126ba97SEli Cohen } 301e126ba97SEli Cohen 302e126ba97SEli Cohen err = pci_request_regions(pdev, DRIVER_NAME); 303e126ba97SEli Cohen if (err) 304e126ba97SEli Cohen dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 305e126ba97SEli Cohen 306e126ba97SEli Cohen return err; 307e126ba97SEli Cohen } 308e126ba97SEli Cohen 309e126ba97SEli Cohen static void release_bar(struct pci_dev *pdev) 310e126ba97SEli Cohen { 311e126ba97SEli Cohen pci_release_regions(pdev); 312e126ba97SEli Cohen } 313e126ba97SEli Cohen 314e126ba97SEli Cohen static int mlx5_enable_msix(struct mlx5_core_dev *dev) 315e126ba97SEli Cohen { 316db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 317db058a18SSaeed Mahameed struct mlx5_eq_table *table = &priv->eq_table; 318938fe83cSSaeed Mahameed int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 319e126ba97SEli Cohen int nvec; 320e126ba97SEli Cohen int i; 321e126ba97SEli Cohen 322938fe83cSSaeed Mahameed nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 323938fe83cSSaeed Mahameed MLX5_EQ_VEC_COMP_BASE; 324e126ba97SEli Cohen nvec = min_t(int, nvec, num_eqs); 325e126ba97SEli Cohen if (nvec <= MLX5_EQ_VEC_COMP_BASE) 326e126ba97SEli Cohen return -ENOMEM; 327e126ba97SEli Cohen 328db058a18SSaeed Mahameed priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL); 329db058a18SSaeed Mahameed 330db058a18SSaeed Mahameed priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL); 331db058a18SSaeed Mahameed if (!priv->msix_arr || !priv->irq_info) 332db058a18SSaeed Mahameed goto err_free_msix; 333e126ba97SEli Cohen 334e126ba97SEli Cohen for (i = 0; i < nvec; i++) 335db058a18SSaeed Mahameed priv->msix_arr[i].entry = i; 336e126ba97SEli Cohen 337db058a18SSaeed Mahameed nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 3383a9e161aSEli Cohen MLX5_EQ_VEC_COMP_BASE + 1, nvec); 339f3c9407bSAlexander Gordeev if (nvec < 0) 340f3c9407bSAlexander Gordeev return nvec; 341e126ba97SEli Cohen 342f3c9407bSAlexander Gordeev table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 343e126ba97SEli Cohen 344e126ba97SEli Cohen return 0; 345db058a18SSaeed Mahameed 346db058a18SSaeed Mahameed err_free_msix: 347db058a18SSaeed Mahameed kfree(priv->irq_info); 348db058a18SSaeed Mahameed kfree(priv->msix_arr); 349db058a18SSaeed Mahameed return -ENOMEM; 350e126ba97SEli Cohen } 351e126ba97SEli Cohen 352e126ba97SEli Cohen static void mlx5_disable_msix(struct mlx5_core_dev *dev) 353e126ba97SEli Cohen { 354db058a18SSaeed Mahameed struct mlx5_priv *priv = &dev->priv; 355e126ba97SEli Cohen 356e126ba97SEli Cohen pci_disable_msix(dev->pdev); 357db058a18SSaeed Mahameed kfree(priv->irq_info); 358db058a18SSaeed Mahameed kfree(priv->msix_arr); 359e126ba97SEli Cohen } 360e126ba97SEli Cohen 361bd10838aSOr Gerlitz struct mlx5_reg_host_endianness { 362e126ba97SEli Cohen u8 he; 363e126ba97SEli Cohen u8 rsvd[15]; 364e126ba97SEli Cohen }; 365e126ba97SEli Cohen 36687b8de49SEli Cohen #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 36787b8de49SEli Cohen 36887b8de49SEli Cohen enum { 36987b8de49SEli Cohen MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 370c7a08ac7SEli Cohen MLX5_DEV_CAP_FLAG_DCT, 37187b8de49SEli Cohen }; 37287b8de49SEli Cohen 3732974ab6eSSaeed Mahameed static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) 374c7a08ac7SEli Cohen { 375c7a08ac7SEli Cohen switch (size) { 376c7a08ac7SEli Cohen case 128: 377c7a08ac7SEli Cohen return 0; 378c7a08ac7SEli Cohen case 256: 379c7a08ac7SEli Cohen return 1; 380c7a08ac7SEli Cohen case 512: 381c7a08ac7SEli Cohen return 2; 382c7a08ac7SEli Cohen case 1024: 383c7a08ac7SEli Cohen return 3; 384c7a08ac7SEli Cohen case 2048: 385c7a08ac7SEli Cohen return 4; 386c7a08ac7SEli Cohen case 4096: 387c7a08ac7SEli Cohen return 5; 388c7a08ac7SEli Cohen default: 3892974ab6eSSaeed Mahameed mlx5_core_warn(dev, "invalid pkey table size %d\n", size); 390c7a08ac7SEli Cohen return 0; 391c7a08ac7SEli Cohen } 392c7a08ac7SEli Cohen } 393c7a08ac7SEli Cohen 394b06e7de8SLeon Romanovsky static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, 395b06e7de8SLeon Romanovsky enum mlx5_cap_type cap_type, 396938fe83cSSaeed Mahameed enum mlx5_cap_mode cap_mode) 397c7a08ac7SEli Cohen { 398b775516bSEli Cohen u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 399b775516bSEli Cohen int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 400938fe83cSSaeed Mahameed void *out, *hca_caps; 401938fe83cSSaeed Mahameed u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 402c7a08ac7SEli Cohen int err; 403c7a08ac7SEli Cohen 404b775516bSEli Cohen memset(in, 0, sizeof(in)); 405b775516bSEli Cohen out = kzalloc(out_sz, GFP_KERNEL); 406c7a08ac7SEli Cohen if (!out) 407c7a08ac7SEli Cohen return -ENOMEM; 408938fe83cSSaeed Mahameed 409b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 410b775516bSEli Cohen MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 411b775516bSEli Cohen err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 412c7a08ac7SEli Cohen if (err) { 413938fe83cSSaeed Mahameed mlx5_core_warn(dev, 414938fe83cSSaeed Mahameed "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 415938fe83cSSaeed Mahameed cap_type, cap_mode, err); 416c7a08ac7SEli Cohen goto query_ex; 417c7a08ac7SEli Cohen } 418c7a08ac7SEli Cohen 419938fe83cSSaeed Mahameed hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 420938fe83cSSaeed Mahameed 421938fe83cSSaeed Mahameed switch (cap_mode) { 422938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_MAX: 423701052c5SGal Pressman memcpy(dev->caps.hca_max[cap_type], hca_caps, 424938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 425938fe83cSSaeed Mahameed break; 426938fe83cSSaeed Mahameed case HCA_CAP_OPMOD_GET_CUR: 427701052c5SGal Pressman memcpy(dev->caps.hca_cur[cap_type], hca_caps, 428938fe83cSSaeed Mahameed MLX5_UN_SZ_BYTES(hca_cap_union)); 429938fe83cSSaeed Mahameed break; 430938fe83cSSaeed Mahameed default: 431938fe83cSSaeed Mahameed mlx5_core_warn(dev, 432938fe83cSSaeed Mahameed "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 433938fe83cSSaeed Mahameed cap_type, cap_mode); 434938fe83cSSaeed Mahameed err = -EINVAL; 435938fe83cSSaeed Mahameed break; 436938fe83cSSaeed Mahameed } 437c7a08ac7SEli Cohen query_ex: 438c7a08ac7SEli Cohen kfree(out); 439c7a08ac7SEli Cohen return err; 440c7a08ac7SEli Cohen } 441c7a08ac7SEli Cohen 442b06e7de8SLeon Romanovsky int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) 443b06e7de8SLeon Romanovsky { 444b06e7de8SLeon Romanovsky int ret; 445b06e7de8SLeon Romanovsky 446b06e7de8SLeon Romanovsky ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR); 447b06e7de8SLeon Romanovsky if (ret) 448b06e7de8SLeon Romanovsky return ret; 449b06e7de8SLeon Romanovsky return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX); 450b06e7de8SLeon Romanovsky } 451b06e7de8SLeon Romanovsky 452f91e6d89SEran Ben Elisha static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz, int opmod) 453c7a08ac7SEli Cohen { 454c4f287c4SSaeed Mahameed u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)] = {0}; 455c7a08ac7SEli Cohen 456b775516bSEli Cohen MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 457f91e6d89SEran Ben Elisha MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1); 458c4f287c4SSaeed Mahameed return mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 459c7a08ac7SEli Cohen } 46087b8de49SEli Cohen 461f91e6d89SEran Ben Elisha static int handle_hca_cap_atomic(struct mlx5_core_dev *dev) 462f91e6d89SEran Ben Elisha { 463f91e6d89SEran Ben Elisha void *set_ctx; 464f91e6d89SEran Ben Elisha void *set_hca_cap; 465f91e6d89SEran Ben Elisha int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 466f91e6d89SEran Ben Elisha int req_endianness; 467f91e6d89SEran Ben Elisha int err; 468f91e6d89SEran Ben Elisha 469f91e6d89SEran Ben Elisha if (MLX5_CAP_GEN(dev, atomic)) { 470b06e7de8SLeon Romanovsky err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); 471f91e6d89SEran Ben Elisha if (err) 472f91e6d89SEran Ben Elisha return err; 473f91e6d89SEran Ben Elisha } else { 474f91e6d89SEran Ben Elisha return 0; 475f91e6d89SEran Ben Elisha } 476f91e6d89SEran Ben Elisha 477f91e6d89SEran Ben Elisha req_endianness = 478f91e6d89SEran Ben Elisha MLX5_CAP_ATOMIC(dev, 479bd10838aSOr Gerlitz supported_atomic_req_8B_endianness_mode_1); 480f91e6d89SEran Ben Elisha 481f91e6d89SEran Ben Elisha if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) 482f91e6d89SEran Ben Elisha return 0; 483f91e6d89SEran Ben Elisha 484f91e6d89SEran Ben Elisha set_ctx = kzalloc(set_sz, GFP_KERNEL); 485f91e6d89SEran Ben Elisha if (!set_ctx) 486f91e6d89SEran Ben Elisha return -ENOMEM; 487f91e6d89SEran Ben Elisha 488f91e6d89SEran Ben Elisha set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 489f91e6d89SEran Ben Elisha 490f91e6d89SEran Ben Elisha /* Set requestor to host endianness */ 491bd10838aSOr Gerlitz MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianness_mode, 492f91e6d89SEran Ben Elisha MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); 493f91e6d89SEran Ben Elisha 494f91e6d89SEran Ben Elisha err = set_caps(dev, set_ctx, set_sz, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC); 495f91e6d89SEran Ben Elisha 496f91e6d89SEran Ben Elisha kfree(set_ctx); 497f91e6d89SEran Ben Elisha return err; 498f91e6d89SEran Ben Elisha } 499f91e6d89SEran Ben Elisha 500e126ba97SEli Cohen static int handle_hca_cap(struct mlx5_core_dev *dev) 501e126ba97SEli Cohen { 502b775516bSEli Cohen void *set_ctx = NULL; 503c7a08ac7SEli Cohen struct mlx5_profile *prof = dev->profile; 504c7a08ac7SEli Cohen int err = -ENOMEM; 505b775516bSEli Cohen int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 506938fe83cSSaeed Mahameed void *set_hca_cap; 507e126ba97SEli Cohen 508b775516bSEli Cohen set_ctx = kzalloc(set_sz, GFP_KERNEL); 509c7a08ac7SEli Cohen if (!set_ctx) 510e126ba97SEli Cohen goto query_ex; 511e126ba97SEli Cohen 512b06e7de8SLeon Romanovsky err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); 513c7a08ac7SEli Cohen if (err) 514e126ba97SEli Cohen goto query_ex; 515e126ba97SEli Cohen 516938fe83cSSaeed Mahameed set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 517938fe83cSSaeed Mahameed capability); 518701052c5SGal Pressman memcpy(set_hca_cap, dev->caps.hca_cur[MLX5_CAP_GENERAL], 519938fe83cSSaeed Mahameed MLX5_ST_SZ_BYTES(cmd_hca_cap)); 520938fe83cSSaeed Mahameed 521938fe83cSSaeed Mahameed mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 522707c4602SMajd Dibbiny mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 523938fe83cSSaeed Mahameed 128); 524c7a08ac7SEli Cohen /* we limit the size of the pkey table to 128 entries for now */ 525938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 5262974ab6eSSaeed Mahameed to_fw_pkey_sz(dev, 128)); 527e126ba97SEli Cohen 528883371c4SNoa Osherovich /* Check log_max_qp from HCA caps to set in current profile */ 529883371c4SNoa Osherovich if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < profile[prof_sel].log_max_qp) { 530883371c4SNoa Osherovich mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n", 531883371c4SNoa Osherovich profile[prof_sel].log_max_qp, 532883371c4SNoa Osherovich MLX5_CAP_GEN_MAX(dev, log_max_qp)); 533883371c4SNoa Osherovich profile[prof_sel].log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); 534883371c4SNoa Osherovich } 535c7a08ac7SEli Cohen if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 536938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 537938fe83cSSaeed Mahameed prof->log_max_qp); 538e126ba97SEli Cohen 539938fe83cSSaeed Mahameed /* disable cmdif checksum */ 540938fe83cSSaeed Mahameed MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 541c1868b82SEli Cohen 54291828bd8SMajd Dibbiny /* Enable 4K UAR only when HCA supports it and page size is bigger 54391828bd8SMajd Dibbiny * than 4K. 54491828bd8SMajd Dibbiny */ 54591828bd8SMajd Dibbiny if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096) 546f502d834SEli Cohen MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1); 547f502d834SEli Cohen 548fe1e1876SCarol L Soto MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 549fe1e1876SCarol L Soto 550f32f5bd2SDaniel Jurgens if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte)) 551f32f5bd2SDaniel Jurgens MLX5_SET(cmd_hca_cap, 552f32f5bd2SDaniel Jurgens set_hca_cap, 553f32f5bd2SDaniel Jurgens cache_line_128byte, 554f32f5bd2SDaniel Jurgens cache_line_size() == 128 ? 1 : 0); 555f32f5bd2SDaniel Jurgens 556f91e6d89SEran Ben Elisha err = set_caps(dev, set_ctx, set_sz, 557f91e6d89SEran Ben Elisha MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 558e126ba97SEli Cohen 559e126ba97SEli Cohen query_ex: 560e126ba97SEli Cohen kfree(set_ctx); 561e126ba97SEli Cohen return err; 562e126ba97SEli Cohen } 563e126ba97SEli Cohen 564e126ba97SEli Cohen static int set_hca_ctrl(struct mlx5_core_dev *dev) 565e126ba97SEli Cohen { 566bd10838aSOr Gerlitz struct mlx5_reg_host_endianness he_in; 567bd10838aSOr Gerlitz struct mlx5_reg_host_endianness he_out; 568e126ba97SEli Cohen int err; 569e126ba97SEli Cohen 570fc50db98SEli Cohen if (!mlx5_core_is_pf(dev)) 571fc50db98SEli Cohen return 0; 572fc50db98SEli Cohen 573e126ba97SEli Cohen memset(&he_in, 0, sizeof(he_in)); 574e126ba97SEli Cohen he_in.he = MLX5_SET_HOST_ENDIANNESS; 575e126ba97SEli Cohen err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 576e126ba97SEli Cohen &he_out, sizeof(he_out), 577e126ba97SEli Cohen MLX5_REG_HOST_ENDIANNESS, 0, 1); 578e126ba97SEli Cohen return err; 579e126ba97SEli Cohen } 580e126ba97SEli Cohen 5810b107106SEli Cohen int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id) 582cd23b14bSEli Cohen { 583c4f287c4SSaeed Mahameed u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0}; 584c4f287c4SSaeed Mahameed u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {0}; 585cd23b14bSEli Cohen 5860b107106SEli Cohen MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 5870b107106SEli Cohen MLX5_SET(enable_hca_in, in, function_id, func_id); 588c4f287c4SSaeed Mahameed return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 589cd23b14bSEli Cohen } 590cd23b14bSEli Cohen 5910b107106SEli Cohen int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) 592cd23b14bSEli Cohen { 593c4f287c4SSaeed Mahameed u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {0}; 594c4f287c4SSaeed Mahameed u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {0}; 595cd23b14bSEli Cohen 5960b107106SEli Cohen MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 5970b107106SEli Cohen MLX5_SET(disable_hca_in, in, function_id, func_id); 598c4f287c4SSaeed Mahameed return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 599cd23b14bSEli Cohen } 600cd23b14bSEli Cohen 601a5a1d1c2SThomas Gleixner u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev) 602b0844444SEran Ben Elisha { 603b0844444SEran Ben Elisha u32 timer_h, timer_h1, timer_l; 604b0844444SEran Ben Elisha 605b0844444SEran Ben Elisha timer_h = ioread32be(&dev->iseg->internal_timer_h); 606b0844444SEran Ben Elisha timer_l = ioread32be(&dev->iseg->internal_timer_l); 607b0844444SEran Ben Elisha timer_h1 = ioread32be(&dev->iseg->internal_timer_h); 608b0844444SEran Ben Elisha if (timer_h != timer_h1) /* wrap around */ 609b0844444SEran Ben Elisha timer_l = ioread32be(&dev->iseg->internal_timer_l); 610b0844444SEran Ben Elisha 611a5a1d1c2SThomas Gleixner return (u64)timer_l | (u64)timer_h1 << 32; 612b0844444SEran Ben Elisha } 613b0844444SEran Ben Elisha 614db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) 615db058a18SSaeed Mahameed { 616db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 617db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 618db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 619db058a18SSaeed Mahameed 620db058a18SSaeed Mahameed if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) { 621db058a18SSaeed Mahameed mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); 622db058a18SSaeed Mahameed return -ENOMEM; 623db058a18SSaeed Mahameed } 624db058a18SSaeed Mahameed 625d151d73dSEli Cohen cpumask_set_cpu(cpumask_local_spread(i, priv->numa_node), 626dda922c8SDavid S. Miller priv->irq_info[i].mask); 627db058a18SSaeed Mahameed 628f0d7ae95SArnd Bergmann if (IS_ENABLED(CONFIG_SMP) && 629f0d7ae95SArnd Bergmann irq_set_affinity_hint(irq, priv->irq_info[i].mask)) 630b665d98eSTariq Toukan mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x", irq); 631db058a18SSaeed Mahameed 632db058a18SSaeed Mahameed return 0; 633db058a18SSaeed Mahameed } 634db058a18SSaeed Mahameed 635db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i) 636db058a18SSaeed Mahameed { 637db058a18SSaeed Mahameed struct mlx5_priv *priv = &mdev->priv; 638db058a18SSaeed Mahameed struct msix_entry *msix = priv->msix_arr; 639db058a18SSaeed Mahameed int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 640db058a18SSaeed Mahameed 641db058a18SSaeed Mahameed irq_set_affinity_hint(irq, NULL); 642db058a18SSaeed Mahameed free_cpumask_var(priv->irq_info[i].mask); 643db058a18SSaeed Mahameed } 644db058a18SSaeed Mahameed 645db058a18SSaeed Mahameed static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev) 646db058a18SSaeed Mahameed { 647db058a18SSaeed Mahameed int err; 648db058a18SSaeed Mahameed int i; 649db058a18SSaeed Mahameed 650db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) { 651db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hint(mdev, i); 652db058a18SSaeed Mahameed if (err) 653db058a18SSaeed Mahameed goto err_out; 654db058a18SSaeed Mahameed } 655db058a18SSaeed Mahameed 656db058a18SSaeed Mahameed return 0; 657db058a18SSaeed Mahameed 658db058a18SSaeed Mahameed err_out: 659db058a18SSaeed Mahameed for (i--; i >= 0; i--) 660db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 661db058a18SSaeed Mahameed 662db058a18SSaeed Mahameed return err; 663db058a18SSaeed Mahameed } 664db058a18SSaeed Mahameed 665db058a18SSaeed Mahameed static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev) 666db058a18SSaeed Mahameed { 667db058a18SSaeed Mahameed int i; 668db058a18SSaeed Mahameed 669db058a18SSaeed Mahameed for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) 670db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hint(mdev, i); 671db058a18SSaeed Mahameed } 672db058a18SSaeed Mahameed 6730b6e26ceSDoron Tsur int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, 6740b6e26ceSDoron Tsur unsigned int *irqn) 675233d05d2SSaeed Mahameed { 676233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 677233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 678233d05d2SSaeed Mahameed int err = -ENOENT; 679233d05d2SSaeed Mahameed 680233d05d2SSaeed Mahameed spin_lock(&table->lock); 681233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 682233d05d2SSaeed Mahameed if (eq->index == vector) { 683233d05d2SSaeed Mahameed *eqn = eq->eqn; 684233d05d2SSaeed Mahameed *irqn = eq->irqn; 685233d05d2SSaeed Mahameed err = 0; 686233d05d2SSaeed Mahameed break; 687233d05d2SSaeed Mahameed } 688233d05d2SSaeed Mahameed } 689233d05d2SSaeed Mahameed spin_unlock(&table->lock); 690233d05d2SSaeed Mahameed 691233d05d2SSaeed Mahameed return err; 692233d05d2SSaeed Mahameed } 693233d05d2SSaeed Mahameed EXPORT_SYMBOL(mlx5_vector2eqn); 694233d05d2SSaeed Mahameed 69594c6825eSMatan Barak struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn) 69694c6825eSMatan Barak { 69794c6825eSMatan Barak struct mlx5_eq_table *table = &dev->priv.eq_table; 69894c6825eSMatan Barak struct mlx5_eq *eq; 69994c6825eSMatan Barak 70094c6825eSMatan Barak spin_lock(&table->lock); 70194c6825eSMatan Barak list_for_each_entry(eq, &table->comp_eqs_list, list) 70294c6825eSMatan Barak if (eq->eqn == eqn) { 70394c6825eSMatan Barak spin_unlock(&table->lock); 70494c6825eSMatan Barak return eq; 70594c6825eSMatan Barak } 70694c6825eSMatan Barak 70794c6825eSMatan Barak spin_unlock(&table->lock); 70894c6825eSMatan Barak 70994c6825eSMatan Barak return ERR_PTR(-ENOENT); 71094c6825eSMatan Barak } 71194c6825eSMatan Barak 712233d05d2SSaeed Mahameed static void free_comp_eqs(struct mlx5_core_dev *dev) 713233d05d2SSaeed Mahameed { 714233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 715233d05d2SSaeed Mahameed struct mlx5_eq *eq, *n; 716233d05d2SSaeed Mahameed 7175a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 7185a7b27ebSMaor Gottlieb if (dev->rmap) { 7195a7b27ebSMaor Gottlieb free_irq_cpu_rmap(dev->rmap); 7205a7b27ebSMaor Gottlieb dev->rmap = NULL; 7215a7b27ebSMaor Gottlieb } 7225a7b27ebSMaor Gottlieb #endif 723233d05d2SSaeed Mahameed spin_lock(&table->lock); 724233d05d2SSaeed Mahameed list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 725233d05d2SSaeed Mahameed list_del(&eq->list); 726233d05d2SSaeed Mahameed spin_unlock(&table->lock); 727233d05d2SSaeed Mahameed if (mlx5_destroy_unmap_eq(dev, eq)) 728233d05d2SSaeed Mahameed mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 729233d05d2SSaeed Mahameed eq->eqn); 730233d05d2SSaeed Mahameed kfree(eq); 731233d05d2SSaeed Mahameed spin_lock(&table->lock); 732233d05d2SSaeed Mahameed } 733233d05d2SSaeed Mahameed spin_unlock(&table->lock); 734233d05d2SSaeed Mahameed } 735233d05d2SSaeed Mahameed 736233d05d2SSaeed Mahameed static int alloc_comp_eqs(struct mlx5_core_dev *dev) 737233d05d2SSaeed Mahameed { 738233d05d2SSaeed Mahameed struct mlx5_eq_table *table = &dev->priv.eq_table; 739db058a18SSaeed Mahameed char name[MLX5_MAX_IRQ_NAME]; 740233d05d2SSaeed Mahameed struct mlx5_eq *eq; 741233d05d2SSaeed Mahameed int ncomp_vec; 742233d05d2SSaeed Mahameed int nent; 743233d05d2SSaeed Mahameed int err; 744233d05d2SSaeed Mahameed int i; 745233d05d2SSaeed Mahameed 746233d05d2SSaeed Mahameed INIT_LIST_HEAD(&table->comp_eqs_list); 747233d05d2SSaeed Mahameed ncomp_vec = table->num_comp_vectors; 748233d05d2SSaeed Mahameed nent = MLX5_COMP_EQ_SIZE; 7495a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 7505a7b27ebSMaor Gottlieb dev->rmap = alloc_irq_cpu_rmap(ncomp_vec); 7515a7b27ebSMaor Gottlieb if (!dev->rmap) 7525a7b27ebSMaor Gottlieb return -ENOMEM; 7535a7b27ebSMaor Gottlieb #endif 754233d05d2SSaeed Mahameed for (i = 0; i < ncomp_vec; i++) { 755233d05d2SSaeed Mahameed eq = kzalloc(sizeof(*eq), GFP_KERNEL); 756233d05d2SSaeed Mahameed if (!eq) { 757233d05d2SSaeed Mahameed err = -ENOMEM; 758233d05d2SSaeed Mahameed goto clean; 759233d05d2SSaeed Mahameed } 760233d05d2SSaeed Mahameed 7615a7b27ebSMaor Gottlieb #ifdef CONFIG_RFS_ACCEL 7625a7b27ebSMaor Gottlieb irq_cpu_rmap_add(dev->rmap, 7635a7b27ebSMaor Gottlieb dev->priv.msix_arr[i + MLX5_EQ_VEC_COMP_BASE].vector); 7645a7b27ebSMaor Gottlieb #endif 765db058a18SSaeed Mahameed snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 766233d05d2SSaeed Mahameed err = mlx5_create_map_eq(dev, eq, 767233d05d2SSaeed Mahameed i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 76801187175SEli Cohen name, MLX5_EQ_TYPE_COMP); 769233d05d2SSaeed Mahameed if (err) { 770233d05d2SSaeed Mahameed kfree(eq); 771233d05d2SSaeed Mahameed goto clean; 772233d05d2SSaeed Mahameed } 773233d05d2SSaeed Mahameed mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 774233d05d2SSaeed Mahameed eq->index = i; 775233d05d2SSaeed Mahameed spin_lock(&table->lock); 776233d05d2SSaeed Mahameed list_add_tail(&eq->list, &table->comp_eqs_list); 777233d05d2SSaeed Mahameed spin_unlock(&table->lock); 778233d05d2SSaeed Mahameed } 779233d05d2SSaeed Mahameed 780233d05d2SSaeed Mahameed return 0; 781233d05d2SSaeed Mahameed 782233d05d2SSaeed Mahameed clean: 783233d05d2SSaeed Mahameed free_comp_eqs(dev); 784233d05d2SSaeed Mahameed return err; 785233d05d2SSaeed Mahameed } 786233d05d2SSaeed Mahameed 787f62b8bb8SAmir Vadai static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 788f62b8bb8SAmir Vadai { 789c4f287c4SSaeed Mahameed u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {0}; 790c4f287c4SSaeed Mahameed u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {0}; 791f62b8bb8SAmir Vadai u32 sup_issi; 792c4f287c4SSaeed Mahameed int err; 793f62b8bb8SAmir Vadai 794f62b8bb8SAmir Vadai MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 795c4f287c4SSaeed Mahameed err = mlx5_cmd_exec(dev, query_in, sizeof(query_in), 796f62b8bb8SAmir Vadai query_out, sizeof(query_out)); 797f62b8bb8SAmir Vadai if (err) { 798c4f287c4SSaeed Mahameed u32 syndrome; 799c4f287c4SSaeed Mahameed u8 status; 800c4f287c4SSaeed Mahameed 801c4f287c4SSaeed Mahameed mlx5_cmd_mbox_status(query_out, &status, &syndrome); 802f9c14e46SKamal Heib if (!status || syndrome == MLX5_DRIVER_SYND) { 803f9c14e46SKamal Heib mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n", 804f9c14e46SKamal Heib err, status, syndrome); 805f9c14e46SKamal Heib return err; 806f62b8bb8SAmir Vadai } 807f62b8bb8SAmir Vadai 808f9c14e46SKamal Heib mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n"); 809f9c14e46SKamal Heib dev->issi = 0; 810f9c14e46SKamal Heib return 0; 811f62b8bb8SAmir Vadai } 812f62b8bb8SAmir Vadai 813f62b8bb8SAmir Vadai sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 814f62b8bb8SAmir Vadai 815f62b8bb8SAmir Vadai if (sup_issi & (1 << 1)) { 816c4f287c4SSaeed Mahameed u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {0}; 817c4f287c4SSaeed Mahameed u32 set_out[MLX5_ST_SZ_DW(set_issi_out)] = {0}; 818f62b8bb8SAmir Vadai 819f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 820f62b8bb8SAmir Vadai MLX5_SET(set_issi_in, set_in, current_issi, 1); 821c4f287c4SSaeed Mahameed err = mlx5_cmd_exec(dev, set_in, sizeof(set_in), 822f62b8bb8SAmir Vadai set_out, sizeof(set_out)); 823f62b8bb8SAmir Vadai if (err) { 824f9c14e46SKamal Heib mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n", 825f9c14e46SKamal Heib err); 826f62b8bb8SAmir Vadai return err; 827f62b8bb8SAmir Vadai } 828f62b8bb8SAmir Vadai 829f62b8bb8SAmir Vadai dev->issi = 1; 830f62b8bb8SAmir Vadai 831f62b8bb8SAmir Vadai return 0; 832e74a1db0SHaggai Abramonvsky } else if (sup_issi & (1 << 0) || !sup_issi) { 833f62b8bb8SAmir Vadai return 0; 834f62b8bb8SAmir Vadai } 835f62b8bb8SAmir Vadai 8369eb78923SOr Gerlitz return -EOPNOTSUPP; 837f62b8bb8SAmir Vadai } 838f62b8bb8SAmir Vadai 8397907f23aSAviv Heller 840a31208b1SMajd Dibbiny static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 841a31208b1SMajd Dibbiny { 842a31208b1SMajd Dibbiny struct pci_dev *pdev = dev->pdev; 843a31208b1SMajd Dibbiny int err = 0; 844a31208b1SMajd Dibbiny 845e126ba97SEli Cohen pci_set_drvdata(dev->pdev, dev); 846e126ba97SEli Cohen strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 847e126ba97SEli Cohen priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 848e126ba97SEli Cohen 849e126ba97SEli Cohen mutex_init(&priv->pgdir_mutex); 850e126ba97SEli Cohen INIT_LIST_HEAD(&priv->pgdir_list); 851e126ba97SEli Cohen spin_lock_init(&priv->mkey_lock); 852e126ba97SEli Cohen 853311c7c71SSaeed Mahameed mutex_init(&priv->alloc_mutex); 854311c7c71SSaeed Mahameed 855311c7c71SSaeed Mahameed priv->numa_node = dev_to_node(&dev->pdev->dev); 856311c7c71SSaeed Mahameed 857e126ba97SEli Cohen priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root); 858e126ba97SEli Cohen if (!priv->dbg_root) 859e126ba97SEli Cohen return -ENOMEM; 860e126ba97SEli Cohen 86189d44f0aSMajd Dibbiny err = mlx5_pci_enable_device(dev); 862e126ba97SEli Cohen if (err) { 8631a91de28SJoe Perches dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 864e126ba97SEli Cohen goto err_dbg; 865e126ba97SEli Cohen } 866e126ba97SEli Cohen 867e126ba97SEli Cohen err = request_bar(pdev); 868e126ba97SEli Cohen if (err) { 8691a91de28SJoe Perches dev_err(&pdev->dev, "error requesting BARs, aborting\n"); 870e126ba97SEli Cohen goto err_disable; 871e126ba97SEli Cohen } 872e126ba97SEli Cohen 873e126ba97SEli Cohen pci_set_master(pdev); 874e126ba97SEli Cohen 875e126ba97SEli Cohen err = set_dma_caps(pdev); 876e126ba97SEli Cohen if (err) { 877e126ba97SEli Cohen dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); 878e126ba97SEli Cohen goto err_clr_master; 879e126ba97SEli Cohen } 880e126ba97SEli Cohen 881e126ba97SEli Cohen dev->iseg_base = pci_resource_start(dev->pdev, 0); 882e126ba97SEli Cohen dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 883e126ba97SEli Cohen if (!dev->iseg) { 884e126ba97SEli Cohen err = -ENOMEM; 885e126ba97SEli Cohen dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); 886e126ba97SEli Cohen goto err_clr_master; 887e126ba97SEli Cohen } 888a31208b1SMajd Dibbiny 889a31208b1SMajd Dibbiny return 0; 890a31208b1SMajd Dibbiny 891a31208b1SMajd Dibbiny err_clr_master: 892a31208b1SMajd Dibbiny pci_clear_master(dev->pdev); 893a31208b1SMajd Dibbiny release_bar(dev->pdev); 894a31208b1SMajd Dibbiny err_disable: 89589d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 896a31208b1SMajd Dibbiny 897a31208b1SMajd Dibbiny err_dbg: 898a31208b1SMajd Dibbiny debugfs_remove(priv->dbg_root); 899a31208b1SMajd Dibbiny return err; 900a31208b1SMajd Dibbiny } 901a31208b1SMajd Dibbiny 902a31208b1SMajd Dibbiny static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 903a31208b1SMajd Dibbiny { 904a31208b1SMajd Dibbiny iounmap(dev->iseg); 905a31208b1SMajd Dibbiny pci_clear_master(dev->pdev); 906a31208b1SMajd Dibbiny release_bar(dev->pdev); 90789d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 908a31208b1SMajd Dibbiny debugfs_remove(priv->dbg_root); 909a31208b1SMajd Dibbiny } 910a31208b1SMajd Dibbiny 91159211bd3SMohamad Haj Yahia static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 91259211bd3SMohamad Haj Yahia { 91359211bd3SMohamad Haj Yahia struct pci_dev *pdev = dev->pdev; 91459211bd3SMohamad Haj Yahia int err; 91559211bd3SMohamad Haj Yahia 91659211bd3SMohamad Haj Yahia err = mlx5_query_board_id(dev); 91759211bd3SMohamad Haj Yahia if (err) { 91859211bd3SMohamad Haj Yahia dev_err(&pdev->dev, "query board id failed\n"); 91959211bd3SMohamad Haj Yahia goto out; 92059211bd3SMohamad Haj Yahia } 92159211bd3SMohamad Haj Yahia 92259211bd3SMohamad Haj Yahia err = mlx5_eq_init(dev); 92359211bd3SMohamad Haj Yahia if (err) { 92459211bd3SMohamad Haj Yahia dev_err(&pdev->dev, "failed to initialize eq\n"); 92559211bd3SMohamad Haj Yahia goto out; 92659211bd3SMohamad Haj Yahia } 92759211bd3SMohamad Haj Yahia 92859211bd3SMohamad Haj Yahia err = mlx5_init_cq_table(dev); 92959211bd3SMohamad Haj Yahia if (err) { 93059211bd3SMohamad Haj Yahia dev_err(&pdev->dev, "failed to initialize cq table\n"); 93159211bd3SMohamad Haj Yahia goto err_eq_cleanup; 93259211bd3SMohamad Haj Yahia } 93359211bd3SMohamad Haj Yahia 93459211bd3SMohamad Haj Yahia mlx5_init_qp_table(dev); 93559211bd3SMohamad Haj Yahia 93659211bd3SMohamad Haj Yahia mlx5_init_srq_table(dev); 93759211bd3SMohamad Haj Yahia 93859211bd3SMohamad Haj Yahia mlx5_init_mkey_table(dev); 93959211bd3SMohamad Haj Yahia 94052ec462eSIlan Tayari mlx5_init_reserved_gids(dev); 94152ec462eSIlan Tayari 94259211bd3SMohamad Haj Yahia err = mlx5_init_rl_table(dev); 94359211bd3SMohamad Haj Yahia if (err) { 94459211bd3SMohamad Haj Yahia dev_err(&pdev->dev, "Failed to init rate limiting\n"); 94559211bd3SMohamad Haj Yahia goto err_tables_cleanup; 94659211bd3SMohamad Haj Yahia } 94759211bd3SMohamad Haj Yahia 948eeb66cdbSSaeed Mahameed err = mlx5_mpfs_init(dev); 949eeb66cdbSSaeed Mahameed if (err) { 950eeb66cdbSSaeed Mahameed dev_err(&pdev->dev, "Failed to init l2 table %d\n", err); 951eeb66cdbSSaeed Mahameed goto err_rl_cleanup; 952eeb66cdbSSaeed Mahameed } 953eeb66cdbSSaeed Mahameed 954c2d6e31aSMohamad Haj Yahia err = mlx5_eswitch_init(dev); 955c2d6e31aSMohamad Haj Yahia if (err) { 956c2d6e31aSMohamad Haj Yahia dev_err(&pdev->dev, "Failed to init eswitch %d\n", err); 957eeb66cdbSSaeed Mahameed goto err_mpfs_cleanup; 958c2d6e31aSMohamad Haj Yahia } 959c2d6e31aSMohamad Haj Yahia 960c2d6e31aSMohamad Haj Yahia err = mlx5_sriov_init(dev); 961c2d6e31aSMohamad Haj Yahia if (err) { 962c2d6e31aSMohamad Haj Yahia dev_err(&pdev->dev, "Failed to init sriov %d\n", err); 963c2d6e31aSMohamad Haj Yahia goto err_eswitch_cleanup; 964c2d6e31aSMohamad Haj Yahia } 965c2d6e31aSMohamad Haj Yahia 9669410733cSIlan Tayari err = mlx5_fpga_init(dev); 9679410733cSIlan Tayari if (err) { 9689410733cSIlan Tayari dev_err(&pdev->dev, "Failed to init fpga device %d\n", err); 9699410733cSIlan Tayari goto err_sriov_cleanup; 9709410733cSIlan Tayari } 9719410733cSIlan Tayari 97259211bd3SMohamad Haj Yahia return 0; 97359211bd3SMohamad Haj Yahia 9749410733cSIlan Tayari err_sriov_cleanup: 9759410733cSIlan Tayari mlx5_sriov_cleanup(dev); 976c2d6e31aSMohamad Haj Yahia err_eswitch_cleanup: 977c2d6e31aSMohamad Haj Yahia mlx5_eswitch_cleanup(dev->priv.eswitch); 978eeb66cdbSSaeed Mahameed err_mpfs_cleanup: 979eeb66cdbSSaeed Mahameed mlx5_mpfs_cleanup(dev); 980eeb66cdbSSaeed Mahameed err_rl_cleanup: 981c2d6e31aSMohamad Haj Yahia mlx5_cleanup_rl_table(dev); 98259211bd3SMohamad Haj Yahia err_tables_cleanup: 98359211bd3SMohamad Haj Yahia mlx5_cleanup_mkey_table(dev); 98459211bd3SMohamad Haj Yahia mlx5_cleanup_srq_table(dev); 98559211bd3SMohamad Haj Yahia mlx5_cleanup_qp_table(dev); 98659211bd3SMohamad Haj Yahia mlx5_cleanup_cq_table(dev); 98759211bd3SMohamad Haj Yahia 98859211bd3SMohamad Haj Yahia err_eq_cleanup: 98959211bd3SMohamad Haj Yahia mlx5_eq_cleanup(dev); 99059211bd3SMohamad Haj Yahia 99159211bd3SMohamad Haj Yahia out: 99259211bd3SMohamad Haj Yahia return err; 99359211bd3SMohamad Haj Yahia } 99459211bd3SMohamad Haj Yahia 99559211bd3SMohamad Haj Yahia static void mlx5_cleanup_once(struct mlx5_core_dev *dev) 99659211bd3SMohamad Haj Yahia { 9979410733cSIlan Tayari mlx5_fpga_cleanup(dev); 998c2d6e31aSMohamad Haj Yahia mlx5_sriov_cleanup(dev); 999c2d6e31aSMohamad Haj Yahia mlx5_eswitch_cleanup(dev->priv.eswitch); 1000eeb66cdbSSaeed Mahameed mlx5_mpfs_cleanup(dev); 100159211bd3SMohamad Haj Yahia mlx5_cleanup_rl_table(dev); 100252ec462eSIlan Tayari mlx5_cleanup_reserved_gids(dev); 100359211bd3SMohamad Haj Yahia mlx5_cleanup_mkey_table(dev); 100459211bd3SMohamad Haj Yahia mlx5_cleanup_srq_table(dev); 100559211bd3SMohamad Haj Yahia mlx5_cleanup_qp_table(dev); 100659211bd3SMohamad Haj Yahia mlx5_cleanup_cq_table(dev); 100759211bd3SMohamad Haj Yahia mlx5_eq_cleanup(dev); 100859211bd3SMohamad Haj Yahia } 100959211bd3SMohamad Haj Yahia 101059211bd3SMohamad Haj Yahia static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, 101159211bd3SMohamad Haj Yahia bool boot) 1012a31208b1SMajd Dibbiny { 1013a31208b1SMajd Dibbiny struct pci_dev *pdev = dev->pdev; 1014a31208b1SMajd Dibbiny int err; 1015a31208b1SMajd Dibbiny 101689d44f0aSMajd Dibbiny mutex_lock(&dev->intf_state_mutex); 10175fc7197dSMajd Dibbiny if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 101889d44f0aSMajd Dibbiny dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", 101989d44f0aSMajd Dibbiny __func__); 102089d44f0aSMajd Dibbiny goto out; 102189d44f0aSMajd Dibbiny } 102289d44f0aSMajd Dibbiny 1023e126ba97SEli Cohen dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 1024e126ba97SEli Cohen fw_rev_min(dev), fw_rev_sub(dev)); 1025e126ba97SEli Cohen 102689d44f0aSMajd Dibbiny /* on load removing any previous indication of internal error, device is 102789d44f0aSMajd Dibbiny * up 102889d44f0aSMajd Dibbiny */ 102989d44f0aSMajd Dibbiny dev->state = MLX5_DEVICE_STATE_UP; 103089d44f0aSMajd Dibbiny 10316c780a02SEli Cohen /* wait for firmware to accept initialization segments configurations 10326c780a02SEli Cohen */ 10336c780a02SEli Cohen err = wait_fw_init(dev, FW_PRE_INIT_TIMEOUT_MILI); 10346c780a02SEli Cohen if (err) { 10356c780a02SEli Cohen dev_err(&dev->pdev->dev, "Firmware over %d MS in pre-initializing state, aborting\n", 10366c780a02SEli Cohen FW_PRE_INIT_TIMEOUT_MILI); 10378ce59b16SGal Pressman goto out_err; 10386c780a02SEli Cohen } 10396c780a02SEli Cohen 1040e126ba97SEli Cohen err = mlx5_cmd_init(dev); 1041e126ba97SEli Cohen if (err) { 1042e126ba97SEli Cohen dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); 104389d44f0aSMajd Dibbiny goto out_err; 1044e126ba97SEli Cohen } 1045e126ba97SEli Cohen 1046e3297246SEli Cohen err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 1047e3297246SEli Cohen if (err) { 1048e3297246SEli Cohen dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n", 1049e3297246SEli Cohen FW_INIT_TIMEOUT_MILI); 105055378a23SMohamad Haj Yahia goto err_cmd_cleanup; 1051e3297246SEli Cohen } 1052e3297246SEli Cohen 10530b107106SEli Cohen err = mlx5_core_enable_hca(dev, 0); 1054cd23b14bSEli Cohen if (err) { 1055cd23b14bSEli Cohen dev_err(&pdev->dev, "enable hca failed\n"); 105659211bd3SMohamad Haj Yahia goto err_cmd_cleanup; 1057cd23b14bSEli Cohen } 1058cd23b14bSEli Cohen 1059f62b8bb8SAmir Vadai err = mlx5_core_set_issi(dev); 1060f62b8bb8SAmir Vadai if (err) { 1061f62b8bb8SAmir Vadai dev_err(&pdev->dev, "failed to set issi\n"); 1062f62b8bb8SAmir Vadai goto err_disable_hca; 1063f62b8bb8SAmir Vadai } 1064f62b8bb8SAmir Vadai 1065cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 1); 1066cd23b14bSEli Cohen if (err) { 1067cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate boot pages\n"); 1068cd23b14bSEli Cohen goto err_disable_hca; 1069cd23b14bSEli Cohen } 1070cd23b14bSEli Cohen 1071e126ba97SEli Cohen err = set_hca_ctrl(dev); 1072e126ba97SEli Cohen if (err) { 1073e126ba97SEli Cohen dev_err(&pdev->dev, "set_hca_ctrl failed\n"); 1074cd23b14bSEli Cohen goto reclaim_boot_pages; 1075e126ba97SEli Cohen } 1076e126ba97SEli Cohen 1077e126ba97SEli Cohen err = handle_hca_cap(dev); 1078e126ba97SEli Cohen if (err) { 1079e126ba97SEli Cohen dev_err(&pdev->dev, "handle_hca_cap failed\n"); 1080cd23b14bSEli Cohen goto reclaim_boot_pages; 1081e126ba97SEli Cohen } 1082e126ba97SEli Cohen 1083f91e6d89SEran Ben Elisha err = handle_hca_cap_atomic(dev); 1084f91e6d89SEran Ben Elisha if (err) { 1085f91e6d89SEran Ben Elisha dev_err(&pdev->dev, "handle_hca_cap_atomic failed\n"); 1086f91e6d89SEran Ben Elisha goto reclaim_boot_pages; 1087f91e6d89SEran Ben Elisha } 1088f91e6d89SEran Ben Elisha 1089cd23b14bSEli Cohen err = mlx5_satisfy_startup_pages(dev, 0); 1090e126ba97SEli Cohen if (err) { 1091cd23b14bSEli Cohen dev_err(&pdev->dev, "failed to allocate init pages\n"); 1092cd23b14bSEli Cohen goto reclaim_boot_pages; 1093e126ba97SEli Cohen } 1094e126ba97SEli Cohen 1095e126ba97SEli Cohen err = mlx5_pagealloc_start(dev); 1096e126ba97SEli Cohen if (err) { 1097e126ba97SEli Cohen dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n"); 1098cd23b14bSEli Cohen goto reclaim_boot_pages; 1099e126ba97SEli Cohen } 1100e126ba97SEli Cohen 1101e126ba97SEli Cohen err = mlx5_cmd_init_hca(dev); 1102e126ba97SEli Cohen if (err) { 1103e126ba97SEli Cohen dev_err(&pdev->dev, "init hca failed\n"); 1104e126ba97SEli Cohen goto err_pagealloc_stop; 1105e126ba97SEli Cohen } 1106e126ba97SEli Cohen 1107012e50e1SHuy Nguyen mlx5_set_driver_version(dev); 1108012e50e1SHuy Nguyen 1109e126ba97SEli Cohen mlx5_start_health_poll(dev); 1110e126ba97SEli Cohen 1111bba1574cSDaniel Jurgens err = mlx5_query_hca_caps(dev); 1112bba1574cSDaniel Jurgens if (err) { 1113bba1574cSDaniel Jurgens dev_err(&pdev->dev, "query hca failed\n"); 1114bba1574cSDaniel Jurgens goto err_stop_poll; 1115bba1574cSDaniel Jurgens } 1116bba1574cSDaniel Jurgens 111759211bd3SMohamad Haj Yahia if (boot && mlx5_init_once(dev, priv)) { 111859211bd3SMohamad Haj Yahia dev_err(&pdev->dev, "sw objs init failed\n"); 1119e126ba97SEli Cohen goto err_stop_poll; 1120e126ba97SEli Cohen } 1121e126ba97SEli Cohen 1122e126ba97SEli Cohen err = mlx5_enable_msix(dev); 1123e126ba97SEli Cohen if (err) { 1124e126ba97SEli Cohen dev_err(&pdev->dev, "enable msix failed\n"); 112559211bd3SMohamad Haj Yahia goto err_cleanup_once; 1126e126ba97SEli Cohen } 1127e126ba97SEli Cohen 112801187175SEli Cohen dev->priv.uar = mlx5_get_uars_page(dev); 112901187175SEli Cohen if (!dev->priv.uar) { 1130e126ba97SEli Cohen dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 113159211bd3SMohamad Haj Yahia goto err_disable_msix; 1132e126ba97SEli Cohen } 1133e126ba97SEli Cohen 1134e126ba97SEli Cohen err = mlx5_start_eqs(dev); 1135e126ba97SEli Cohen if (err) { 1136e126ba97SEli Cohen dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 11379410733cSIlan Tayari goto err_put_uars; 1138e126ba97SEli Cohen } 1139e126ba97SEli Cohen 1140233d05d2SSaeed Mahameed err = alloc_comp_eqs(dev); 1141233d05d2SSaeed Mahameed if (err) { 1142233d05d2SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc completion EQs\n"); 1143233d05d2SSaeed Mahameed goto err_stop_eqs; 1144233d05d2SSaeed Mahameed } 1145233d05d2SSaeed Mahameed 1146db058a18SSaeed Mahameed err = mlx5_irq_set_affinity_hints(dev); 114759211bd3SMohamad Haj Yahia if (err) { 1148db058a18SSaeed Mahameed dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n"); 114959211bd3SMohamad Haj Yahia goto err_affinity_hints; 115059211bd3SMohamad Haj Yahia } 1151e126ba97SEli Cohen 115286d722adSMaor Gottlieb err = mlx5_init_fs(dev); 115386d722adSMaor Gottlieb if (err) { 115486d722adSMaor Gottlieb dev_err(&pdev->dev, "Failed to init flow steering\n"); 115586d722adSMaor Gottlieb goto err_fs; 115686d722adSMaor Gottlieb } 11571466cc5bSYevgeny Petrilin 1158c2d6e31aSMohamad Haj Yahia err = mlx5_sriov_attach(dev); 1159fc50db98SEli Cohen if (err) { 1160fc50db98SEli Cohen dev_err(&pdev->dev, "sriov init failed %d\n", err); 1161fc50db98SEli Cohen goto err_sriov; 1162fc50db98SEli Cohen } 1163fc50db98SEli Cohen 1164e29341fbSIlan Tayari err = mlx5_fpga_device_start(dev); 1165e29341fbSIlan Tayari if (err) { 1166e29341fbSIlan Tayari dev_err(&pdev->dev, "fpga device start failed %d\n", err); 116752ec462eSIlan Tayari goto err_fpga_start; 1168e29341fbSIlan Tayari } 1169bebb23e6SIlan Tayari err = mlx5_accel_ipsec_init(dev); 1170bebb23e6SIlan Tayari if (err) { 1171bebb23e6SIlan Tayari dev_err(&pdev->dev, "IPSec device start failed %d\n", err); 1172bebb23e6SIlan Tayari goto err_ipsec_start; 1173bebb23e6SIlan Tayari } 1174e29341fbSIlan Tayari 1175737a234bSMohamad Haj Yahia if (mlx5_device_registered(dev)) { 1176737a234bSMohamad Haj Yahia mlx5_attach_device(dev); 1177737a234bSMohamad Haj Yahia } else { 1178a31208b1SMajd Dibbiny err = mlx5_register_device(dev); 1179a31208b1SMajd Dibbiny if (err) { 1180a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 1181a31208b1SMajd Dibbiny goto err_reg_dev; 1182a31208b1SMajd Dibbiny } 1183737a234bSMohamad Haj Yahia } 1184a31208b1SMajd Dibbiny 11855fc7197dSMajd Dibbiny clear_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state); 11865fc7197dSMajd Dibbiny set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 118789d44f0aSMajd Dibbiny out: 118889d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 118989d44f0aSMajd Dibbiny 1190e126ba97SEli Cohen return 0; 1191e126ba97SEli Cohen 119259211bd3SMohamad Haj Yahia err_reg_dev: 1193bebb23e6SIlan Tayari mlx5_accel_ipsec_cleanup(dev); 1194bebb23e6SIlan Tayari err_ipsec_start: 119552ec462eSIlan Tayari mlx5_fpga_device_stop(dev); 119652ec462eSIlan Tayari 119752ec462eSIlan Tayari err_fpga_start: 1198c2d6e31aSMohamad Haj Yahia mlx5_sriov_detach(dev); 1199fc50db98SEli Cohen 120059211bd3SMohamad Haj Yahia err_sriov: 120186d722adSMaor Gottlieb mlx5_cleanup_fs(dev); 120259211bd3SMohamad Haj Yahia 120386d722adSMaor Gottlieb err_fs: 1204a31208b1SMajd Dibbiny mlx5_irq_clear_affinity_hints(dev); 120559211bd3SMohamad Haj Yahia 120659211bd3SMohamad Haj Yahia err_affinity_hints: 1207db058a18SSaeed Mahameed free_comp_eqs(dev); 1208db058a18SSaeed Mahameed 1209233d05d2SSaeed Mahameed err_stop_eqs: 1210233d05d2SSaeed Mahameed mlx5_stop_eqs(dev); 1211233d05d2SSaeed Mahameed 12125fe9dec0SEli Cohen err_put_uars: 121301187175SEli Cohen mlx5_put_uars_page(dev, priv->uar); 1214e126ba97SEli Cohen 121559211bd3SMohamad Haj Yahia err_disable_msix: 1216e126ba97SEli Cohen mlx5_disable_msix(dev); 1217e126ba97SEli Cohen 121859211bd3SMohamad Haj Yahia err_cleanup_once: 121959211bd3SMohamad Haj Yahia if (boot) 122059211bd3SMohamad Haj Yahia mlx5_cleanup_once(dev); 122159211bd3SMohamad Haj Yahia 1222e126ba97SEli Cohen err_stop_poll: 1223e126ba97SEli Cohen mlx5_stop_health_poll(dev); 12241bde6e30SEli Cohen if (mlx5_cmd_teardown_hca(dev)) { 12251bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 122689d44f0aSMajd Dibbiny goto out_err; 12271bde6e30SEli Cohen } 1228e126ba97SEli Cohen 1229e126ba97SEli Cohen err_pagealloc_stop: 1230e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 1231e126ba97SEli Cohen 1232cd23b14bSEli Cohen reclaim_boot_pages: 1233e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 1234e126ba97SEli Cohen 1235cd23b14bSEli Cohen err_disable_hca: 12360b107106SEli Cohen mlx5_core_disable_hca(dev, 0); 1237cd23b14bSEli Cohen 123859211bd3SMohamad Haj Yahia err_cmd_cleanup: 1239e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 1240e126ba97SEli Cohen 124189d44f0aSMajd Dibbiny out_err: 124289d44f0aSMajd Dibbiny dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 124389d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 124489d44f0aSMajd Dibbiny 1245e126ba97SEli Cohen return err; 1246e126ba97SEli Cohen } 1247e126ba97SEli Cohen 124859211bd3SMohamad Haj Yahia static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, 124959211bd3SMohamad Haj Yahia bool cleanup) 1250e126ba97SEli Cohen { 125189d44f0aSMajd Dibbiny int err = 0; 1252e126ba97SEli Cohen 12535e44fca5SDaniel Jurgens if (cleanup) 12542a0165a0SMohamad Haj Yahia mlx5_drain_health_recovery(dev); 1255689a248dSDaniel Jurgens 125689d44f0aSMajd Dibbiny mutex_lock(&dev->intf_state_mutex); 12575fc7197dSMajd Dibbiny if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) { 125889d44f0aSMajd Dibbiny dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", 125989d44f0aSMajd Dibbiny __func__); 126059211bd3SMohamad Haj Yahia if (cleanup) 126159211bd3SMohamad Haj Yahia mlx5_cleanup_once(dev); 126289d44f0aSMajd Dibbiny goto out; 126389d44f0aSMajd Dibbiny } 12646b6adee3SMohamad Haj Yahia 12659ade8c7cSIlan Tayari clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 12669ade8c7cSIlan Tayari set_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state); 12679ade8c7cSIlan Tayari 1268737a234bSMohamad Haj Yahia if (mlx5_device_registered(dev)) 1269737a234bSMohamad Haj Yahia mlx5_detach_device(dev); 1270737a234bSMohamad Haj Yahia 1271bebb23e6SIlan Tayari mlx5_accel_ipsec_cleanup(dev); 127252ec462eSIlan Tayari mlx5_fpga_device_stop(dev); 127352ec462eSIlan Tayari 1274c2d6e31aSMohamad Haj Yahia mlx5_sriov_detach(dev); 127586d722adSMaor Gottlieb mlx5_cleanup_fs(dev); 1276db058a18SSaeed Mahameed mlx5_irq_clear_affinity_hints(dev); 1277233d05d2SSaeed Mahameed free_comp_eqs(dev); 1278e126ba97SEli Cohen mlx5_stop_eqs(dev); 127901187175SEli Cohen mlx5_put_uars_page(dev, priv->uar); 1280e126ba97SEli Cohen mlx5_disable_msix(dev); 128159211bd3SMohamad Haj Yahia if (cleanup) 128259211bd3SMohamad Haj Yahia mlx5_cleanup_once(dev); 1283e126ba97SEli Cohen mlx5_stop_health_poll(dev); 1284ac6ea6e8SEli Cohen err = mlx5_cmd_teardown_hca(dev); 1285ac6ea6e8SEli Cohen if (err) { 12861bde6e30SEli Cohen dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 1287ac6ea6e8SEli Cohen goto out; 12881bde6e30SEli Cohen } 1289e126ba97SEli Cohen mlx5_pagealloc_stop(dev); 1290e126ba97SEli Cohen mlx5_reclaim_startup_pages(dev); 12910b107106SEli Cohen mlx5_core_disable_hca(dev, 0); 1292e126ba97SEli Cohen mlx5_cmd_cleanup(dev); 12939603b61dSJack Morgenstein 1294ac6ea6e8SEli Cohen out: 129589d44f0aSMajd Dibbiny mutex_unlock(&dev->intf_state_mutex); 1296ac6ea6e8SEli Cohen return err; 12979603b61dSJack Morgenstein } 129864613d94SSaeed Mahameed 12999603b61dSJack Morgenstein struct mlx5_core_event_handler { 13009603b61dSJack Morgenstein void (*event)(struct mlx5_core_dev *dev, 13019603b61dSJack Morgenstein enum mlx5_dev_event event, 13029603b61dSJack Morgenstein void *data); 13039603b61dSJack Morgenstein }; 13049603b61dSJack Morgenstein 1305feae9087SOr Gerlitz static const struct devlink_ops mlx5_devlink_ops = { 1306e80541ecSSaeed Mahameed #ifdef CONFIG_MLX5_ESWITCH 1307feae9087SOr Gerlitz .eswitch_mode_set = mlx5_devlink_eswitch_mode_set, 1308feae9087SOr Gerlitz .eswitch_mode_get = mlx5_devlink_eswitch_mode_get, 1309bffaa916SRoi Dayan .eswitch_inline_mode_set = mlx5_devlink_eswitch_inline_mode_set, 1310bffaa916SRoi Dayan .eswitch_inline_mode_get = mlx5_devlink_eswitch_inline_mode_get, 13117768d197SRoi Dayan .eswitch_encap_mode_set = mlx5_devlink_eswitch_encap_mode_set, 13127768d197SRoi Dayan .eswitch_encap_mode_get = mlx5_devlink_eswitch_encap_mode_get, 1313feae9087SOr Gerlitz #endif 1314feae9087SOr Gerlitz }; 1315f66f049fSEli Cohen 131659211bd3SMohamad Haj Yahia #define MLX5_IB_MOD "mlx5_ib" 13179603b61dSJack Morgenstein static int init_one(struct pci_dev *pdev, 13189603b61dSJack Morgenstein const struct pci_device_id *id) 13199603b61dSJack Morgenstein { 13209603b61dSJack Morgenstein struct mlx5_core_dev *dev; 1321feae9087SOr Gerlitz struct devlink *devlink; 13229603b61dSJack Morgenstein struct mlx5_priv *priv; 13239603b61dSJack Morgenstein int err; 13249603b61dSJack Morgenstein 1325feae9087SOr Gerlitz devlink = devlink_alloc(&mlx5_devlink_ops, sizeof(*dev)); 1326feae9087SOr Gerlitz if (!devlink) { 13279603b61dSJack Morgenstein dev_err(&pdev->dev, "kzalloc failed\n"); 13289603b61dSJack Morgenstein return -ENOMEM; 13299603b61dSJack Morgenstein } 1330feae9087SOr Gerlitz 1331feae9087SOr Gerlitz dev = devlink_priv(devlink); 13329603b61dSJack Morgenstein priv = &dev->priv; 1333fc50db98SEli Cohen priv->pci_dev_data = id->driver_data; 13349603b61dSJack Morgenstein 13359603b61dSJack Morgenstein pci_set_drvdata(pdev, dev); 13369603b61dSJack Morgenstein 13370e97a340SHuy Nguyen dev->pdev = pdev; 13380e97a340SHuy Nguyen dev->event = mlx5_core_event; 13399603b61dSJack Morgenstein dev->profile = &profile[prof_sel]; 13409603b61dSJack Morgenstein 1341364d1798SEli Cohen INIT_LIST_HEAD(&priv->ctx_list); 1342364d1798SEli Cohen spin_lock_init(&priv->ctx_lock); 134389d44f0aSMajd Dibbiny mutex_init(&dev->pci_status_mutex); 134489d44f0aSMajd Dibbiny mutex_init(&dev->intf_state_mutex); 1345d9aaed83SArtemy Kovalyov 1346d9aaed83SArtemy Kovalyov #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1347d9aaed83SArtemy Kovalyov err = init_srcu_struct(&priv->pfault_srcu); 1348d9aaed83SArtemy Kovalyov if (err) { 1349d9aaed83SArtemy Kovalyov dev_err(&pdev->dev, "init_srcu_struct failed with error code %d\n", 1350d9aaed83SArtemy Kovalyov err); 1351d9aaed83SArtemy Kovalyov goto clean_dev; 1352d9aaed83SArtemy Kovalyov } 1353d9aaed83SArtemy Kovalyov #endif 135401187175SEli Cohen mutex_init(&priv->bfregs.reg_head.lock); 135501187175SEli Cohen mutex_init(&priv->bfregs.wc_head.lock); 135601187175SEli Cohen INIT_LIST_HEAD(&priv->bfregs.reg_head.list); 135701187175SEli Cohen INIT_LIST_HEAD(&priv->bfregs.wc_head.list); 135801187175SEli Cohen 1359a31208b1SMajd Dibbiny err = mlx5_pci_init(dev, priv); 13609603b61dSJack Morgenstein if (err) { 1361a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err); 1362d9aaed83SArtemy Kovalyov goto clean_srcu; 13639603b61dSJack Morgenstein } 13649603b61dSJack Morgenstein 1365ac6ea6e8SEli Cohen err = mlx5_health_init(dev); 1366ac6ea6e8SEli Cohen if (err) { 1367ac6ea6e8SEli Cohen dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err); 1368ac6ea6e8SEli Cohen goto close_pci; 1369ac6ea6e8SEli Cohen } 1370ac6ea6e8SEli Cohen 137159211bd3SMohamad Haj Yahia mlx5_pagealloc_init(dev); 137259211bd3SMohamad Haj Yahia 137359211bd3SMohamad Haj Yahia err = mlx5_load_one(dev, priv, true); 13749603b61dSJack Morgenstein if (err) { 1375a31208b1SMajd Dibbiny dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err); 1376ac6ea6e8SEli Cohen goto clean_health; 13779603b61dSJack Morgenstein } 137859211bd3SMohamad Haj Yahia 1379f82eed45SLeon Romanovsky request_module_nowait(MLX5_IB_MOD); 13809603b61dSJack Morgenstein 1381feae9087SOr Gerlitz err = devlink_register(devlink, &pdev->dev); 1382feae9087SOr Gerlitz if (err) 1383feae9087SOr Gerlitz goto clean_load; 1384feae9087SOr Gerlitz 13855d47f6c8SDaniel Jurgens pci_save_state(pdev); 13869603b61dSJack Morgenstein return 0; 13879603b61dSJack Morgenstein 1388feae9087SOr Gerlitz clean_load: 138959211bd3SMohamad Haj Yahia mlx5_unload_one(dev, priv, true); 1390ac6ea6e8SEli Cohen clean_health: 139159211bd3SMohamad Haj Yahia mlx5_pagealloc_cleanup(dev); 1392ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1393a31208b1SMajd Dibbiny close_pci: 1394a31208b1SMajd Dibbiny mlx5_pci_close(dev, priv); 1395d9aaed83SArtemy Kovalyov clean_srcu: 1396d9aaed83SArtemy Kovalyov #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1397d9aaed83SArtemy Kovalyov cleanup_srcu_struct(&priv->pfault_srcu); 1398a31208b1SMajd Dibbiny clean_dev: 1399d9aaed83SArtemy Kovalyov #endif 1400a31208b1SMajd Dibbiny pci_set_drvdata(pdev, NULL); 1401feae9087SOr Gerlitz devlink_free(devlink); 1402a31208b1SMajd Dibbiny 14039603b61dSJack Morgenstein return err; 14049603b61dSJack Morgenstein } 1405a31208b1SMajd Dibbiny 14069603b61dSJack Morgenstein static void remove_one(struct pci_dev *pdev) 14079603b61dSJack Morgenstein { 14089603b61dSJack Morgenstein struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1409feae9087SOr Gerlitz struct devlink *devlink = priv_to_devlink(dev); 1410a31208b1SMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 14119603b61dSJack Morgenstein 1412feae9087SOr Gerlitz devlink_unregister(devlink); 1413737a234bSMohamad Haj Yahia mlx5_unregister_device(dev); 1414737a234bSMohamad Haj Yahia 141559211bd3SMohamad Haj Yahia if (mlx5_unload_one(dev, priv, true)) { 1416a31208b1SMajd Dibbiny dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); 1417ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1418a31208b1SMajd Dibbiny return; 1419a31208b1SMajd Dibbiny } 1420737a234bSMohamad Haj Yahia 142159211bd3SMohamad Haj Yahia mlx5_pagealloc_cleanup(dev); 1422ac6ea6e8SEli Cohen mlx5_health_cleanup(dev); 1423a31208b1SMajd Dibbiny mlx5_pci_close(dev, priv); 1424d9aaed83SArtemy Kovalyov #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1425d9aaed83SArtemy Kovalyov cleanup_srcu_struct(&priv->pfault_srcu); 1426d9aaed83SArtemy Kovalyov #endif 1427a31208b1SMajd Dibbiny pci_set_drvdata(pdev, NULL); 1428feae9087SOr Gerlitz devlink_free(devlink); 14299603b61dSJack Morgenstein } 14309603b61dSJack Morgenstein 143189d44f0aSMajd Dibbiny static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, 143289d44f0aSMajd Dibbiny pci_channel_state_t state) 143389d44f0aSMajd Dibbiny { 143489d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 143589d44f0aSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 143689d44f0aSMajd Dibbiny 143789d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 143804c0c1abSMohamad Haj Yahia 14398812c24dSMajd Dibbiny mlx5_enter_error_state(dev, false); 144059211bd3SMohamad Haj Yahia mlx5_unload_one(dev, priv, false); 14415d47f6c8SDaniel Jurgens /* In case of kernel call drain the health wq */ 144205ac2c0bSMohamad Haj Yahia if (state) { 14435e44fca5SDaniel Jurgens mlx5_drain_health_wq(dev); 144489d44f0aSMajd Dibbiny mlx5_pci_disable_device(dev); 144505ac2c0bSMohamad Haj Yahia } 144605ac2c0bSMohamad Haj Yahia 144789d44f0aSMajd Dibbiny return state == pci_channel_io_perm_failure ? 144889d44f0aSMajd Dibbiny PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 144989d44f0aSMajd Dibbiny } 145089d44f0aSMajd Dibbiny 1451d57847dcSDaniel Jurgens /* wait for the device to show vital signs by waiting 1452d57847dcSDaniel Jurgens * for the health counter to start counting. 145389d44f0aSMajd Dibbiny */ 1454d57847dcSDaniel Jurgens static int wait_vital(struct pci_dev *pdev) 145589d44f0aSMajd Dibbiny { 145689d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 145789d44f0aSMajd Dibbiny struct mlx5_core_health *health = &dev->priv.health; 145889d44f0aSMajd Dibbiny const int niter = 100; 1459d57847dcSDaniel Jurgens u32 last_count = 0; 146089d44f0aSMajd Dibbiny u32 count; 146189d44f0aSMajd Dibbiny int i; 146289d44f0aSMajd Dibbiny 146389d44f0aSMajd Dibbiny for (i = 0; i < niter; i++) { 146489d44f0aSMajd Dibbiny count = ioread32be(health->health_counter); 146589d44f0aSMajd Dibbiny if (count && count != 0xffffffff) { 1466d57847dcSDaniel Jurgens if (last_count && last_count != count) { 146789d44f0aSMajd Dibbiny dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); 1468d57847dcSDaniel Jurgens return 0; 1469d57847dcSDaniel Jurgens } 1470d57847dcSDaniel Jurgens last_count = count; 147189d44f0aSMajd Dibbiny } 147289d44f0aSMajd Dibbiny msleep(50); 147389d44f0aSMajd Dibbiny } 147489d44f0aSMajd Dibbiny 1475d57847dcSDaniel Jurgens return -ETIMEDOUT; 147689d44f0aSMajd Dibbiny } 147789d44f0aSMajd Dibbiny 14781061c90fSMohamad Haj Yahia static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) 14791061c90fSMohamad Haj Yahia { 14801061c90fSMohamad Haj Yahia struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 14811061c90fSMohamad Haj Yahia int err; 14821061c90fSMohamad Haj Yahia 14831061c90fSMohamad Haj Yahia dev_info(&pdev->dev, "%s was called\n", __func__); 14841061c90fSMohamad Haj Yahia 14851061c90fSMohamad Haj Yahia err = mlx5_pci_enable_device(dev); 14861061c90fSMohamad Haj Yahia if (err) { 14871061c90fSMohamad Haj Yahia dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" 14881061c90fSMohamad Haj Yahia , __func__, err); 14891061c90fSMohamad Haj Yahia return PCI_ERS_RESULT_DISCONNECT; 14901061c90fSMohamad Haj Yahia } 14911061c90fSMohamad Haj Yahia 14921061c90fSMohamad Haj Yahia pci_set_master(pdev); 14931061c90fSMohamad Haj Yahia pci_restore_state(pdev); 14945d47f6c8SDaniel Jurgens pci_save_state(pdev); 14951061c90fSMohamad Haj Yahia 14961061c90fSMohamad Haj Yahia if (wait_vital(pdev)) { 14971061c90fSMohamad Haj Yahia dev_err(&pdev->dev, "%s: wait_vital timed out\n", __func__); 14981061c90fSMohamad Haj Yahia return PCI_ERS_RESULT_DISCONNECT; 14991061c90fSMohamad Haj Yahia } 15001061c90fSMohamad Haj Yahia 15011061c90fSMohamad Haj Yahia return PCI_ERS_RESULT_RECOVERED; 15021061c90fSMohamad Haj Yahia } 15031061c90fSMohamad Haj Yahia 150489d44f0aSMajd Dibbiny static void mlx5_pci_resume(struct pci_dev *pdev) 150589d44f0aSMajd Dibbiny { 150689d44f0aSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 150789d44f0aSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 150889d44f0aSMajd Dibbiny int err; 150989d44f0aSMajd Dibbiny 151089d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s was called\n", __func__); 151189d44f0aSMajd Dibbiny 151259211bd3SMohamad Haj Yahia err = mlx5_load_one(dev, priv, false); 151389d44f0aSMajd Dibbiny if (err) 151489d44f0aSMajd Dibbiny dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" 151589d44f0aSMajd Dibbiny , __func__, err); 151689d44f0aSMajd Dibbiny else 151789d44f0aSMajd Dibbiny dev_info(&pdev->dev, "%s: device recovered\n", __func__); 151889d44f0aSMajd Dibbiny } 151989d44f0aSMajd Dibbiny 152089d44f0aSMajd Dibbiny static const struct pci_error_handlers mlx5_err_handler = { 152189d44f0aSMajd Dibbiny .error_detected = mlx5_pci_err_detected, 152289d44f0aSMajd Dibbiny .slot_reset = mlx5_pci_slot_reset, 152389d44f0aSMajd Dibbiny .resume = mlx5_pci_resume 152489d44f0aSMajd Dibbiny }; 152589d44f0aSMajd Dibbiny 15268812c24dSMajd Dibbiny static int mlx5_try_fast_unload(struct mlx5_core_dev *dev) 15278812c24dSMajd Dibbiny { 15288812c24dSMajd Dibbiny int ret; 15298812c24dSMajd Dibbiny 15308812c24dSMajd Dibbiny if (!MLX5_CAP_GEN(dev, force_teardown)) { 15318812c24dSMajd Dibbiny mlx5_core_dbg(dev, "force teardown is not supported in the firmware\n"); 15328812c24dSMajd Dibbiny return -EOPNOTSUPP; 15338812c24dSMajd Dibbiny } 15348812c24dSMajd Dibbiny 15358812c24dSMajd Dibbiny if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 15368812c24dSMajd Dibbiny mlx5_core_dbg(dev, "Device in internal error state, giving up\n"); 15378812c24dSMajd Dibbiny return -EAGAIN; 15388812c24dSMajd Dibbiny } 15398812c24dSMajd Dibbiny 15408812c24dSMajd Dibbiny ret = mlx5_cmd_force_teardown_hca(dev); 15418812c24dSMajd Dibbiny if (ret) { 15428812c24dSMajd Dibbiny mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", ret); 15438812c24dSMajd Dibbiny return ret; 15448812c24dSMajd Dibbiny } 15458812c24dSMajd Dibbiny 15468812c24dSMajd Dibbiny mlx5_enter_error_state(dev, true); 15478812c24dSMajd Dibbiny 15488812c24dSMajd Dibbiny return 0; 15498812c24dSMajd Dibbiny } 15508812c24dSMajd Dibbiny 15515fc7197dSMajd Dibbiny static void shutdown(struct pci_dev *pdev) 15525fc7197dSMajd Dibbiny { 15535fc7197dSMajd Dibbiny struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 15545fc7197dSMajd Dibbiny struct mlx5_priv *priv = &dev->priv; 15558812c24dSMajd Dibbiny int err; 15565fc7197dSMajd Dibbiny 15575fc7197dSMajd Dibbiny dev_info(&pdev->dev, "Shutdown was called\n"); 15585fc7197dSMajd Dibbiny /* Notify mlx5 clients that the kernel is being shut down */ 15595fc7197dSMajd Dibbiny set_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &dev->intf_state); 15608812c24dSMajd Dibbiny err = mlx5_try_fast_unload(dev); 15618812c24dSMajd Dibbiny if (err) 156259211bd3SMohamad Haj Yahia mlx5_unload_one(dev, priv, false); 15635fc7197dSMajd Dibbiny mlx5_pci_disable_device(dev); 15645fc7197dSMajd Dibbiny } 15655fc7197dSMajd Dibbiny 15669603b61dSJack Morgenstein static const struct pci_device_id mlx5_core_pci_table[] = { 1567bbad7c21SMyron Stowe { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) }, 1568fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */ 1569bbad7c21SMyron Stowe { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) }, 1570fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */ 1571bbad7c21SMyron Stowe { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) }, 1572fc50db98SEli Cohen { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */ 15737092fe86SMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */ 157464dbbdfeSMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */ 1575d0dd989fSMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5 Ex */ 1576d0dd989fSMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 Ex VF */ 1577d0dd989fSMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x101b) }, /* ConnectX-6 */ 1578d0dd989fSMajd Dibbiny { PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF}, /* ConnectX-6 VF */ 15792e9d3e83SNoa Osherovich { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */ 15802e9d3e83SNoa Osherovich { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */ 15819603b61dSJack Morgenstein { 0, } 15829603b61dSJack Morgenstein }; 15839603b61dSJack Morgenstein 15849603b61dSJack Morgenstein MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 15859603b61dSJack Morgenstein 158604c0c1abSMohamad Haj Yahia void mlx5_disable_device(struct mlx5_core_dev *dev) 158704c0c1abSMohamad Haj Yahia { 158804c0c1abSMohamad Haj Yahia mlx5_pci_err_detected(dev->pdev, 0); 158904c0c1abSMohamad Haj Yahia } 159004c0c1abSMohamad Haj Yahia 159104c0c1abSMohamad Haj Yahia void mlx5_recover_device(struct mlx5_core_dev *dev) 159204c0c1abSMohamad Haj Yahia { 159304c0c1abSMohamad Haj Yahia mlx5_pci_disable_device(dev); 159404c0c1abSMohamad Haj Yahia if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED) 159504c0c1abSMohamad Haj Yahia mlx5_pci_resume(dev->pdev); 159604c0c1abSMohamad Haj Yahia } 159704c0c1abSMohamad Haj Yahia 15989603b61dSJack Morgenstein static struct pci_driver mlx5_core_driver = { 15999603b61dSJack Morgenstein .name = DRIVER_NAME, 16009603b61dSJack Morgenstein .id_table = mlx5_core_pci_table, 16019603b61dSJack Morgenstein .probe = init_one, 160289d44f0aSMajd Dibbiny .remove = remove_one, 16035fc7197dSMajd Dibbiny .shutdown = shutdown, 1604fc50db98SEli Cohen .err_handler = &mlx5_err_handler, 1605fc50db98SEli Cohen .sriov_configure = mlx5_core_sriov_configure, 16069603b61dSJack Morgenstein }; 1607e126ba97SEli Cohen 1608f663ad98SKamal Heib static void mlx5_core_verify_params(void) 1609f663ad98SKamal Heib { 1610f663ad98SKamal Heib if (prof_sel >= ARRAY_SIZE(profile)) { 1611f663ad98SKamal Heib pr_warn("mlx5_core: WARNING: Invalid module parameter prof_sel %d, valid range 0-%zu, changing back to default(%d)\n", 1612f663ad98SKamal Heib prof_sel, 1613f663ad98SKamal Heib ARRAY_SIZE(profile) - 1, 1614f663ad98SKamal Heib MLX5_DEFAULT_PROF); 1615f663ad98SKamal Heib prof_sel = MLX5_DEFAULT_PROF; 1616f663ad98SKamal Heib } 1617f663ad98SKamal Heib } 1618f663ad98SKamal Heib 1619e126ba97SEli Cohen static int __init init(void) 1620e126ba97SEli Cohen { 1621e126ba97SEli Cohen int err; 1622e126ba97SEli Cohen 1623f663ad98SKamal Heib mlx5_core_verify_params(); 1624e126ba97SEli Cohen mlx5_register_debugfs(); 1625e126ba97SEli Cohen 16269603b61dSJack Morgenstein err = pci_register_driver(&mlx5_core_driver); 16279603b61dSJack Morgenstein if (err) 1628ac6ea6e8SEli Cohen goto err_debug; 16299603b61dSJack Morgenstein 1630f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1631f62b8bb8SAmir Vadai mlx5e_init(); 1632f62b8bb8SAmir Vadai #endif 1633f62b8bb8SAmir Vadai 1634e126ba97SEli Cohen return 0; 1635e126ba97SEli Cohen 1636e126ba97SEli Cohen err_debug: 1637e126ba97SEli Cohen mlx5_unregister_debugfs(); 1638e126ba97SEli Cohen return err; 1639e126ba97SEli Cohen } 1640e126ba97SEli Cohen 1641e126ba97SEli Cohen static void __exit cleanup(void) 1642e126ba97SEli Cohen { 1643f62b8bb8SAmir Vadai #ifdef CONFIG_MLX5_CORE_EN 1644f62b8bb8SAmir Vadai mlx5e_cleanup(); 1645f62b8bb8SAmir Vadai #endif 16469603b61dSJack Morgenstein pci_unregister_driver(&mlx5_core_driver); 1647e126ba97SEli Cohen mlx5_unregister_debugfs(); 1648e126ba97SEli Cohen } 1649e126ba97SEli Cohen 1650e126ba97SEli Cohen module_init(init); 1651e126ba97SEli Cohen module_exit(cleanup); 1652