1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/highmem.h> 34 #include <linux/module.h> 35 #include <linux/init.h> 36 #include <linux/errno.h> 37 #include <linux/pci.h> 38 #include <linux/dma-mapping.h> 39 #include <linux/slab.h> 40 #include <linux/io-mapping.h> 41 #include <linux/interrupt.h> 42 #include <linux/delay.h> 43 #include <linux/mlx5/driver.h> 44 #include <linux/mlx5/cq.h> 45 #include <linux/mlx5/qp.h> 46 #include <linux/mlx5/srq.h> 47 #include <linux/debugfs.h> 48 #include <linux/kmod.h> 49 #include <linux/mlx5/mlx5_ifc.h> 50 #ifdef CONFIG_RFS_ACCEL 51 #include <linux/cpu_rmap.h> 52 #endif 53 #include <net/devlink.h> 54 #include "mlx5_core.h" 55 #include "fs_core.h" 56 #ifdef CONFIG_MLX5_CORE_EN 57 #include "eswitch.h" 58 #endif 59 60 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 61 MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 62 MODULE_LICENSE("Dual BSD/GPL"); 63 MODULE_VERSION(DRIVER_VERSION); 64 65 unsigned int mlx5_core_debug_mask; 66 module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644); 67 MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 68 69 #define MLX5_DEFAULT_PROF 2 70 static unsigned int prof_sel = MLX5_DEFAULT_PROF; 71 module_param_named(prof_sel, prof_sel, uint, 0444); 72 MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 73 74 enum { 75 MLX5_ATOMIC_REQ_MODE_BE = 0x0, 76 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, 77 }; 78 79 static struct mlx5_profile profile[] = { 80 [0] = { 81 .mask = 0, 82 }, 83 [1] = { 84 .mask = MLX5_PROF_MASK_QP_SIZE, 85 .log_max_qp = 12, 86 }, 87 [2] = { 88 .mask = MLX5_PROF_MASK_QP_SIZE | 89 MLX5_PROF_MASK_MR_CACHE, 90 .log_max_qp = 18, 91 .mr_cache[0] = { 92 .size = 500, 93 .limit = 250 94 }, 95 .mr_cache[1] = { 96 .size = 500, 97 .limit = 250 98 }, 99 .mr_cache[2] = { 100 .size = 500, 101 .limit = 250 102 }, 103 .mr_cache[3] = { 104 .size = 500, 105 .limit = 250 106 }, 107 .mr_cache[4] = { 108 .size = 500, 109 .limit = 250 110 }, 111 .mr_cache[5] = { 112 .size = 500, 113 .limit = 250 114 }, 115 .mr_cache[6] = { 116 .size = 500, 117 .limit = 250 118 }, 119 .mr_cache[7] = { 120 .size = 500, 121 .limit = 250 122 }, 123 .mr_cache[8] = { 124 .size = 500, 125 .limit = 250 126 }, 127 .mr_cache[9] = { 128 .size = 500, 129 .limit = 250 130 }, 131 .mr_cache[10] = { 132 .size = 500, 133 .limit = 250 134 }, 135 .mr_cache[11] = { 136 .size = 500, 137 .limit = 250 138 }, 139 .mr_cache[12] = { 140 .size = 64, 141 .limit = 32 142 }, 143 .mr_cache[13] = { 144 .size = 32, 145 .limit = 16 146 }, 147 .mr_cache[14] = { 148 .size = 16, 149 .limit = 8 150 }, 151 .mr_cache[15] = { 152 .size = 8, 153 .limit = 4 154 }, 155 .mr_cache[16] = { 156 .size = 8, 157 .limit = 4 158 }, 159 .mr_cache[17] = { 160 .size = 8, 161 .limit = 4 162 }, 163 .mr_cache[18] = { 164 .size = 8, 165 .limit = 4 166 }, 167 .mr_cache[19] = { 168 .size = 4, 169 .limit = 2 170 }, 171 .mr_cache[20] = { 172 .size = 4, 173 .limit = 2 174 }, 175 }, 176 }; 177 178 #define FW_INIT_TIMEOUT_MILI 2000 179 #define FW_INIT_WAIT_MS 2 180 181 static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 182 { 183 unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); 184 int err = 0; 185 186 while (fw_initializing(dev)) { 187 if (time_after(jiffies, end)) { 188 err = -EBUSY; 189 break; 190 } 191 msleep(FW_INIT_WAIT_MS); 192 } 193 194 return err; 195 } 196 197 static void mlx5_set_driver_version(struct mlx5_core_dev *dev) 198 { 199 int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in, 200 driver_version); 201 u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {0}; 202 u8 out[MLX5_ST_SZ_BYTES(set_driver_version_out)] = {0}; 203 int remaining_size = driver_ver_sz; 204 char *string; 205 206 if (!MLX5_CAP_GEN(dev, driver_version)) 207 return; 208 209 string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version); 210 211 strncpy(string, "Linux", remaining_size); 212 213 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 214 strncat(string, ",", remaining_size); 215 216 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 217 strncat(string, DRIVER_NAME, remaining_size); 218 219 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 220 strncat(string, ",", remaining_size); 221 222 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string)); 223 strncat(string, DRIVER_VERSION, remaining_size); 224 225 /*Send the command*/ 226 MLX5_SET(set_driver_version_in, in, opcode, 227 MLX5_CMD_OP_SET_DRIVER_VERSION); 228 229 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 230 } 231 232 static int set_dma_caps(struct pci_dev *pdev) 233 { 234 int err; 235 236 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 237 if (err) { 238 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 239 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 240 if (err) { 241 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 242 return err; 243 } 244 } 245 246 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 247 if (err) { 248 dev_warn(&pdev->dev, 249 "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 250 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 251 if (err) { 252 dev_err(&pdev->dev, 253 "Can't set consistent PCI DMA mask, aborting\n"); 254 return err; 255 } 256 } 257 258 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 259 return err; 260 } 261 262 static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) 263 { 264 struct pci_dev *pdev = dev->pdev; 265 int err = 0; 266 267 mutex_lock(&dev->pci_status_mutex); 268 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { 269 err = pci_enable_device(pdev); 270 if (!err) 271 dev->pci_status = MLX5_PCI_STATUS_ENABLED; 272 } 273 mutex_unlock(&dev->pci_status_mutex); 274 275 return err; 276 } 277 278 static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) 279 { 280 struct pci_dev *pdev = dev->pdev; 281 282 mutex_lock(&dev->pci_status_mutex); 283 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { 284 pci_disable_device(pdev); 285 dev->pci_status = MLX5_PCI_STATUS_DISABLED; 286 } 287 mutex_unlock(&dev->pci_status_mutex); 288 } 289 290 static int request_bar(struct pci_dev *pdev) 291 { 292 int err = 0; 293 294 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 295 dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 296 return -ENODEV; 297 } 298 299 err = pci_request_regions(pdev, DRIVER_NAME); 300 if (err) 301 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 302 303 return err; 304 } 305 306 static void release_bar(struct pci_dev *pdev) 307 { 308 pci_release_regions(pdev); 309 } 310 311 static int mlx5_enable_msix(struct mlx5_core_dev *dev) 312 { 313 struct mlx5_priv *priv = &dev->priv; 314 struct mlx5_eq_table *table = &priv->eq_table; 315 int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 316 int nvec; 317 int i; 318 319 nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 320 MLX5_EQ_VEC_COMP_BASE; 321 nvec = min_t(int, nvec, num_eqs); 322 if (nvec <= MLX5_EQ_VEC_COMP_BASE) 323 return -ENOMEM; 324 325 priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL); 326 327 priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL); 328 if (!priv->msix_arr || !priv->irq_info) 329 goto err_free_msix; 330 331 for (i = 0; i < nvec; i++) 332 priv->msix_arr[i].entry = i; 333 334 nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 335 MLX5_EQ_VEC_COMP_BASE + 1, nvec); 336 if (nvec < 0) 337 return nvec; 338 339 table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 340 341 return 0; 342 343 err_free_msix: 344 kfree(priv->irq_info); 345 kfree(priv->msix_arr); 346 return -ENOMEM; 347 } 348 349 static void mlx5_disable_msix(struct mlx5_core_dev *dev) 350 { 351 struct mlx5_priv *priv = &dev->priv; 352 353 pci_disable_msix(dev->pdev); 354 kfree(priv->irq_info); 355 kfree(priv->msix_arr); 356 } 357 358 struct mlx5_reg_host_endianess { 359 u8 he; 360 u8 rsvd[15]; 361 }; 362 363 364 #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 365 366 enum { 367 MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 368 MLX5_DEV_CAP_FLAG_DCT, 369 }; 370 371 static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) 372 { 373 switch (size) { 374 case 128: 375 return 0; 376 case 256: 377 return 1; 378 case 512: 379 return 2; 380 case 1024: 381 return 3; 382 case 2048: 383 return 4; 384 case 4096: 385 return 5; 386 default: 387 mlx5_core_warn(dev, "invalid pkey table size %d\n", size); 388 return 0; 389 } 390 } 391 392 static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, 393 enum mlx5_cap_type cap_type, 394 enum mlx5_cap_mode cap_mode) 395 { 396 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 397 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 398 void *out, *hca_caps; 399 u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 400 int err; 401 402 memset(in, 0, sizeof(in)); 403 out = kzalloc(out_sz, GFP_KERNEL); 404 if (!out) 405 return -ENOMEM; 406 407 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 408 MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 409 err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 410 if (err) { 411 mlx5_core_warn(dev, 412 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 413 cap_type, cap_mode, err); 414 goto query_ex; 415 } 416 417 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 418 419 switch (cap_mode) { 420 case HCA_CAP_OPMOD_GET_MAX: 421 memcpy(dev->caps.hca_max[cap_type], hca_caps, 422 MLX5_UN_SZ_BYTES(hca_cap_union)); 423 break; 424 case HCA_CAP_OPMOD_GET_CUR: 425 memcpy(dev->caps.hca_cur[cap_type], hca_caps, 426 MLX5_UN_SZ_BYTES(hca_cap_union)); 427 break; 428 default: 429 mlx5_core_warn(dev, 430 "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 431 cap_type, cap_mode); 432 err = -EINVAL; 433 break; 434 } 435 query_ex: 436 kfree(out); 437 return err; 438 } 439 440 int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) 441 { 442 int ret; 443 444 ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR); 445 if (ret) 446 return ret; 447 return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX); 448 } 449 450 static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz, int opmod) 451 { 452 u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)] = {0}; 453 454 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 455 MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1); 456 return mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 457 } 458 459 static int handle_hca_cap_atomic(struct mlx5_core_dev *dev) 460 { 461 void *set_ctx; 462 void *set_hca_cap; 463 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 464 int req_endianness; 465 int err; 466 467 if (MLX5_CAP_GEN(dev, atomic)) { 468 err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC); 469 if (err) 470 return err; 471 } else { 472 return 0; 473 } 474 475 req_endianness = 476 MLX5_CAP_ATOMIC(dev, 477 supported_atomic_req_8B_endianess_mode_1); 478 479 if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) 480 return 0; 481 482 set_ctx = kzalloc(set_sz, GFP_KERNEL); 483 if (!set_ctx) 484 return -ENOMEM; 485 486 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); 487 488 /* Set requestor to host endianness */ 489 MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode, 490 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); 491 492 err = set_caps(dev, set_ctx, set_sz, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC); 493 494 kfree(set_ctx); 495 return err; 496 } 497 498 static int handle_hca_cap(struct mlx5_core_dev *dev) 499 { 500 void *set_ctx = NULL; 501 struct mlx5_profile *prof = dev->profile; 502 int err = -ENOMEM; 503 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 504 void *set_hca_cap; 505 506 set_ctx = kzalloc(set_sz, GFP_KERNEL); 507 if (!set_ctx) 508 goto query_ex; 509 510 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL); 511 if (err) 512 goto query_ex; 513 514 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 515 capability); 516 memcpy(set_hca_cap, dev->caps.hca_cur[MLX5_CAP_GENERAL], 517 MLX5_ST_SZ_BYTES(cmd_hca_cap)); 518 519 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 520 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 521 128); 522 /* we limit the size of the pkey table to 128 entries for now */ 523 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 524 to_fw_pkey_sz(dev, 128)); 525 526 /* Check log_max_qp from HCA caps to set in current profile */ 527 if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < profile[prof_sel].log_max_qp) { 528 mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n", 529 profile[prof_sel].log_max_qp, 530 MLX5_CAP_GEN_MAX(dev, log_max_qp)); 531 profile[prof_sel].log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); 532 } 533 if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 534 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 535 prof->log_max_qp); 536 537 /* disable cmdif checksum */ 538 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 539 540 /* If the HCA supports 4K UARs use it */ 541 if (MLX5_CAP_GEN_MAX(dev, uar_4k)) 542 MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1); 543 544 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 545 546 if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte)) 547 MLX5_SET(cmd_hca_cap, 548 set_hca_cap, 549 cache_line_128byte, 550 cache_line_size() == 128 ? 1 : 0); 551 552 err = set_caps(dev, set_ctx, set_sz, 553 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 554 555 query_ex: 556 kfree(set_ctx); 557 return err; 558 } 559 560 static int set_hca_ctrl(struct mlx5_core_dev *dev) 561 { 562 struct mlx5_reg_host_endianess he_in; 563 struct mlx5_reg_host_endianess he_out; 564 int err; 565 566 if (!mlx5_core_is_pf(dev)) 567 return 0; 568 569 memset(&he_in, 0, sizeof(he_in)); 570 he_in.he = MLX5_SET_HOST_ENDIANNESS; 571 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 572 &he_out, sizeof(he_out), 573 MLX5_REG_HOST_ENDIANNESS, 0, 1); 574 return err; 575 } 576 577 int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id) 578 { 579 u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0}; 580 u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {0}; 581 582 MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); 583 MLX5_SET(enable_hca_in, in, function_id, func_id); 584 return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 585 } 586 587 int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) 588 { 589 u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {0}; 590 u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {0}; 591 592 MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); 593 MLX5_SET(disable_hca_in, in, function_id, func_id); 594 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 595 } 596 597 u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev) 598 { 599 u32 timer_h, timer_h1, timer_l; 600 601 timer_h = ioread32be(&dev->iseg->internal_timer_h); 602 timer_l = ioread32be(&dev->iseg->internal_timer_l); 603 timer_h1 = ioread32be(&dev->iseg->internal_timer_h); 604 if (timer_h != timer_h1) /* wrap around */ 605 timer_l = ioread32be(&dev->iseg->internal_timer_l); 606 607 return (u64)timer_l | (u64)timer_h1 << 32; 608 } 609 610 static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) 611 { 612 struct mlx5_priv *priv = &mdev->priv; 613 struct msix_entry *msix = priv->msix_arr; 614 int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 615 616 if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) { 617 mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); 618 return -ENOMEM; 619 } 620 621 cpumask_set_cpu(cpumask_local_spread(i, priv->numa_node), 622 priv->irq_info[i].mask); 623 624 #ifdef CONFIG_SMP 625 if (irq_set_affinity_hint(irq, priv->irq_info[i].mask)) 626 mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x", irq); 627 #endif 628 629 return 0; 630 } 631 632 static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i) 633 { 634 struct mlx5_priv *priv = &mdev->priv; 635 struct msix_entry *msix = priv->msix_arr; 636 int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 637 638 irq_set_affinity_hint(irq, NULL); 639 free_cpumask_var(priv->irq_info[i].mask); 640 } 641 642 static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev) 643 { 644 int err; 645 int i; 646 647 for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) { 648 err = mlx5_irq_set_affinity_hint(mdev, i); 649 if (err) 650 goto err_out; 651 } 652 653 return 0; 654 655 err_out: 656 for (i--; i >= 0; i--) 657 mlx5_irq_clear_affinity_hint(mdev, i); 658 659 return err; 660 } 661 662 static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev) 663 { 664 int i; 665 666 for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) 667 mlx5_irq_clear_affinity_hint(mdev, i); 668 } 669 670 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, 671 unsigned int *irqn) 672 { 673 struct mlx5_eq_table *table = &dev->priv.eq_table; 674 struct mlx5_eq *eq, *n; 675 int err = -ENOENT; 676 677 spin_lock(&table->lock); 678 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 679 if (eq->index == vector) { 680 *eqn = eq->eqn; 681 *irqn = eq->irqn; 682 err = 0; 683 break; 684 } 685 } 686 spin_unlock(&table->lock); 687 688 return err; 689 } 690 EXPORT_SYMBOL(mlx5_vector2eqn); 691 692 struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn) 693 { 694 struct mlx5_eq_table *table = &dev->priv.eq_table; 695 struct mlx5_eq *eq; 696 697 spin_lock(&table->lock); 698 list_for_each_entry(eq, &table->comp_eqs_list, list) 699 if (eq->eqn == eqn) { 700 spin_unlock(&table->lock); 701 return eq; 702 } 703 704 spin_unlock(&table->lock); 705 706 return ERR_PTR(-ENOENT); 707 } 708 709 static void free_comp_eqs(struct mlx5_core_dev *dev) 710 { 711 struct mlx5_eq_table *table = &dev->priv.eq_table; 712 struct mlx5_eq *eq, *n; 713 714 #ifdef CONFIG_RFS_ACCEL 715 if (dev->rmap) { 716 free_irq_cpu_rmap(dev->rmap); 717 dev->rmap = NULL; 718 } 719 #endif 720 spin_lock(&table->lock); 721 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 722 list_del(&eq->list); 723 spin_unlock(&table->lock); 724 if (mlx5_destroy_unmap_eq(dev, eq)) 725 mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 726 eq->eqn); 727 kfree(eq); 728 spin_lock(&table->lock); 729 } 730 spin_unlock(&table->lock); 731 } 732 733 static int alloc_comp_eqs(struct mlx5_core_dev *dev) 734 { 735 struct mlx5_eq_table *table = &dev->priv.eq_table; 736 char name[MLX5_MAX_IRQ_NAME]; 737 struct mlx5_eq *eq; 738 int ncomp_vec; 739 int nent; 740 int err; 741 int i; 742 743 INIT_LIST_HEAD(&table->comp_eqs_list); 744 ncomp_vec = table->num_comp_vectors; 745 nent = MLX5_COMP_EQ_SIZE; 746 #ifdef CONFIG_RFS_ACCEL 747 dev->rmap = alloc_irq_cpu_rmap(ncomp_vec); 748 if (!dev->rmap) 749 return -ENOMEM; 750 #endif 751 for (i = 0; i < ncomp_vec; i++) { 752 eq = kzalloc(sizeof(*eq), GFP_KERNEL); 753 if (!eq) { 754 err = -ENOMEM; 755 goto clean; 756 } 757 758 #ifdef CONFIG_RFS_ACCEL 759 irq_cpu_rmap_add(dev->rmap, 760 dev->priv.msix_arr[i + MLX5_EQ_VEC_COMP_BASE].vector); 761 #endif 762 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 763 err = mlx5_create_map_eq(dev, eq, 764 i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 765 name, MLX5_EQ_TYPE_COMP); 766 if (err) { 767 kfree(eq); 768 goto clean; 769 } 770 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 771 eq->index = i; 772 spin_lock(&table->lock); 773 list_add_tail(&eq->list, &table->comp_eqs_list); 774 spin_unlock(&table->lock); 775 } 776 777 return 0; 778 779 clean: 780 free_comp_eqs(dev); 781 return err; 782 } 783 784 static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 785 { 786 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {0}; 787 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {0}; 788 u32 sup_issi; 789 int err; 790 791 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 792 err = mlx5_cmd_exec(dev, query_in, sizeof(query_in), 793 query_out, sizeof(query_out)); 794 if (err) { 795 u32 syndrome; 796 u8 status; 797 798 mlx5_cmd_mbox_status(query_out, &status, &syndrome); 799 if (!status || syndrome == MLX5_DRIVER_SYND) { 800 mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n", 801 err, status, syndrome); 802 return err; 803 } 804 805 mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n"); 806 dev->issi = 0; 807 return 0; 808 } 809 810 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 811 812 if (sup_issi & (1 << 1)) { 813 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {0}; 814 u32 set_out[MLX5_ST_SZ_DW(set_issi_out)] = {0}; 815 816 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 817 MLX5_SET(set_issi_in, set_in, current_issi, 1); 818 err = mlx5_cmd_exec(dev, set_in, sizeof(set_in), 819 set_out, sizeof(set_out)); 820 if (err) { 821 mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n", 822 err); 823 return err; 824 } 825 826 dev->issi = 1; 827 828 return 0; 829 } else if (sup_issi & (1 << 0) || !sup_issi) { 830 return 0; 831 } 832 833 return -EOPNOTSUPP; 834 } 835 836 837 static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 838 { 839 struct pci_dev *pdev = dev->pdev; 840 int err = 0; 841 842 pci_set_drvdata(dev->pdev, dev); 843 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 844 priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 845 846 mutex_init(&priv->pgdir_mutex); 847 INIT_LIST_HEAD(&priv->pgdir_list); 848 spin_lock_init(&priv->mkey_lock); 849 850 mutex_init(&priv->alloc_mutex); 851 852 priv->numa_node = dev_to_node(&dev->pdev->dev); 853 854 priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root); 855 if (!priv->dbg_root) 856 return -ENOMEM; 857 858 err = mlx5_pci_enable_device(dev); 859 if (err) { 860 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 861 goto err_dbg; 862 } 863 864 err = request_bar(pdev); 865 if (err) { 866 dev_err(&pdev->dev, "error requesting BARs, aborting\n"); 867 goto err_disable; 868 } 869 870 pci_set_master(pdev); 871 872 err = set_dma_caps(pdev); 873 if (err) { 874 dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); 875 goto err_clr_master; 876 } 877 878 dev->iseg_base = pci_resource_start(dev->pdev, 0); 879 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 880 if (!dev->iseg) { 881 err = -ENOMEM; 882 dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); 883 goto err_clr_master; 884 } 885 886 return 0; 887 888 err_clr_master: 889 pci_clear_master(dev->pdev); 890 release_bar(dev->pdev); 891 err_disable: 892 mlx5_pci_disable_device(dev); 893 894 err_dbg: 895 debugfs_remove(priv->dbg_root); 896 return err; 897 } 898 899 static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 900 { 901 iounmap(dev->iseg); 902 pci_clear_master(dev->pdev); 903 release_bar(dev->pdev); 904 mlx5_pci_disable_device(dev); 905 debugfs_remove(priv->dbg_root); 906 } 907 908 static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 909 { 910 struct pci_dev *pdev = dev->pdev; 911 int err; 912 913 err = mlx5_query_board_id(dev); 914 if (err) { 915 dev_err(&pdev->dev, "query board id failed\n"); 916 goto out; 917 } 918 919 err = mlx5_eq_init(dev); 920 if (err) { 921 dev_err(&pdev->dev, "failed to initialize eq\n"); 922 goto out; 923 } 924 925 err = mlx5_init_cq_table(dev); 926 if (err) { 927 dev_err(&pdev->dev, "failed to initialize cq table\n"); 928 goto err_eq_cleanup; 929 } 930 931 mlx5_init_qp_table(dev); 932 933 mlx5_init_srq_table(dev); 934 935 mlx5_init_mkey_table(dev); 936 937 err = mlx5_init_rl_table(dev); 938 if (err) { 939 dev_err(&pdev->dev, "Failed to init rate limiting\n"); 940 goto err_tables_cleanup; 941 } 942 943 #ifdef CONFIG_MLX5_CORE_EN 944 err = mlx5_eswitch_init(dev); 945 if (err) { 946 dev_err(&pdev->dev, "Failed to init eswitch %d\n", err); 947 goto err_rl_cleanup; 948 } 949 #endif 950 951 err = mlx5_sriov_init(dev); 952 if (err) { 953 dev_err(&pdev->dev, "Failed to init sriov %d\n", err); 954 goto err_eswitch_cleanup; 955 } 956 957 return 0; 958 959 err_eswitch_cleanup: 960 #ifdef CONFIG_MLX5_CORE_EN 961 mlx5_eswitch_cleanup(dev->priv.eswitch); 962 963 err_rl_cleanup: 964 #endif 965 mlx5_cleanup_rl_table(dev); 966 967 err_tables_cleanup: 968 mlx5_cleanup_mkey_table(dev); 969 mlx5_cleanup_srq_table(dev); 970 mlx5_cleanup_qp_table(dev); 971 mlx5_cleanup_cq_table(dev); 972 973 err_eq_cleanup: 974 mlx5_eq_cleanup(dev); 975 976 out: 977 return err; 978 } 979 980 static void mlx5_cleanup_once(struct mlx5_core_dev *dev) 981 { 982 mlx5_sriov_cleanup(dev); 983 #ifdef CONFIG_MLX5_CORE_EN 984 mlx5_eswitch_cleanup(dev->priv.eswitch); 985 #endif 986 mlx5_cleanup_rl_table(dev); 987 mlx5_cleanup_mkey_table(dev); 988 mlx5_cleanup_srq_table(dev); 989 mlx5_cleanup_qp_table(dev); 990 mlx5_cleanup_cq_table(dev); 991 mlx5_eq_cleanup(dev); 992 } 993 994 static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, 995 bool boot) 996 { 997 struct pci_dev *pdev = dev->pdev; 998 int err; 999 1000 mutex_lock(&dev->intf_state_mutex); 1001 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { 1002 dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", 1003 __func__); 1004 goto out; 1005 } 1006 1007 dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 1008 fw_rev_min(dev), fw_rev_sub(dev)); 1009 1010 /* on load removing any previous indication of internal error, device is 1011 * up 1012 */ 1013 dev->state = MLX5_DEVICE_STATE_UP; 1014 1015 err = mlx5_cmd_init(dev); 1016 if (err) { 1017 dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); 1018 goto out_err; 1019 } 1020 1021 err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 1022 if (err) { 1023 dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n", 1024 FW_INIT_TIMEOUT_MILI); 1025 goto err_cmd_cleanup; 1026 } 1027 1028 err = mlx5_core_enable_hca(dev, 0); 1029 if (err) { 1030 dev_err(&pdev->dev, "enable hca failed\n"); 1031 goto err_cmd_cleanup; 1032 } 1033 1034 err = mlx5_core_set_issi(dev); 1035 if (err) { 1036 dev_err(&pdev->dev, "failed to set issi\n"); 1037 goto err_disable_hca; 1038 } 1039 1040 err = mlx5_satisfy_startup_pages(dev, 1); 1041 if (err) { 1042 dev_err(&pdev->dev, "failed to allocate boot pages\n"); 1043 goto err_disable_hca; 1044 } 1045 1046 err = set_hca_ctrl(dev); 1047 if (err) { 1048 dev_err(&pdev->dev, "set_hca_ctrl failed\n"); 1049 goto reclaim_boot_pages; 1050 } 1051 1052 err = handle_hca_cap(dev); 1053 if (err) { 1054 dev_err(&pdev->dev, "handle_hca_cap failed\n"); 1055 goto reclaim_boot_pages; 1056 } 1057 1058 err = handle_hca_cap_atomic(dev); 1059 if (err) { 1060 dev_err(&pdev->dev, "handle_hca_cap_atomic failed\n"); 1061 goto reclaim_boot_pages; 1062 } 1063 1064 err = mlx5_satisfy_startup_pages(dev, 0); 1065 if (err) { 1066 dev_err(&pdev->dev, "failed to allocate init pages\n"); 1067 goto reclaim_boot_pages; 1068 } 1069 1070 err = mlx5_pagealloc_start(dev); 1071 if (err) { 1072 dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n"); 1073 goto reclaim_boot_pages; 1074 } 1075 1076 err = mlx5_cmd_init_hca(dev); 1077 if (err) { 1078 dev_err(&pdev->dev, "init hca failed\n"); 1079 goto err_pagealloc_stop; 1080 } 1081 1082 mlx5_set_driver_version(dev); 1083 1084 mlx5_start_health_poll(dev); 1085 1086 err = mlx5_query_hca_caps(dev); 1087 if (err) { 1088 dev_err(&pdev->dev, "query hca failed\n"); 1089 goto err_stop_poll; 1090 } 1091 1092 if (boot && mlx5_init_once(dev, priv)) { 1093 dev_err(&pdev->dev, "sw objs init failed\n"); 1094 goto err_stop_poll; 1095 } 1096 1097 err = mlx5_enable_msix(dev); 1098 if (err) { 1099 dev_err(&pdev->dev, "enable msix failed\n"); 1100 goto err_cleanup_once; 1101 } 1102 1103 dev->priv.uar = mlx5_get_uars_page(dev); 1104 if (!dev->priv.uar) { 1105 dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 1106 goto err_disable_msix; 1107 } 1108 1109 err = mlx5_start_eqs(dev); 1110 if (err) { 1111 dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 1112 goto err_put_uars; 1113 } 1114 1115 err = alloc_comp_eqs(dev); 1116 if (err) { 1117 dev_err(&pdev->dev, "Failed to alloc completion EQs\n"); 1118 goto err_stop_eqs; 1119 } 1120 1121 err = mlx5_irq_set_affinity_hints(dev); 1122 if (err) { 1123 dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n"); 1124 goto err_affinity_hints; 1125 } 1126 1127 err = mlx5_init_fs(dev); 1128 if (err) { 1129 dev_err(&pdev->dev, "Failed to init flow steering\n"); 1130 goto err_fs; 1131 } 1132 1133 #ifdef CONFIG_MLX5_CORE_EN 1134 mlx5_eswitch_attach(dev->priv.eswitch); 1135 #endif 1136 1137 err = mlx5_sriov_attach(dev); 1138 if (err) { 1139 dev_err(&pdev->dev, "sriov init failed %d\n", err); 1140 goto err_sriov; 1141 } 1142 1143 if (mlx5_device_registered(dev)) { 1144 mlx5_attach_device(dev); 1145 } else { 1146 err = mlx5_register_device(dev); 1147 if (err) { 1148 dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 1149 goto err_reg_dev; 1150 } 1151 } 1152 1153 clear_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state); 1154 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1155 out: 1156 mutex_unlock(&dev->intf_state_mutex); 1157 1158 return 0; 1159 1160 err_reg_dev: 1161 mlx5_sriov_detach(dev); 1162 1163 err_sriov: 1164 #ifdef CONFIG_MLX5_CORE_EN 1165 mlx5_eswitch_detach(dev->priv.eswitch); 1166 #endif 1167 mlx5_cleanup_fs(dev); 1168 1169 err_fs: 1170 mlx5_irq_clear_affinity_hints(dev); 1171 1172 err_affinity_hints: 1173 free_comp_eqs(dev); 1174 1175 err_stop_eqs: 1176 mlx5_stop_eqs(dev); 1177 1178 err_put_uars: 1179 mlx5_put_uars_page(dev, priv->uar); 1180 1181 err_disable_msix: 1182 mlx5_disable_msix(dev); 1183 1184 err_cleanup_once: 1185 if (boot) 1186 mlx5_cleanup_once(dev); 1187 1188 err_stop_poll: 1189 mlx5_stop_health_poll(dev); 1190 if (mlx5_cmd_teardown_hca(dev)) { 1191 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 1192 goto out_err; 1193 } 1194 1195 err_pagealloc_stop: 1196 mlx5_pagealloc_stop(dev); 1197 1198 reclaim_boot_pages: 1199 mlx5_reclaim_startup_pages(dev); 1200 1201 err_disable_hca: 1202 mlx5_core_disable_hca(dev, 0); 1203 1204 err_cmd_cleanup: 1205 mlx5_cmd_cleanup(dev); 1206 1207 out_err: 1208 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1209 mutex_unlock(&dev->intf_state_mutex); 1210 1211 return err; 1212 } 1213 1214 static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, 1215 bool cleanup) 1216 { 1217 int err = 0; 1218 1219 if (cleanup) 1220 mlx5_drain_health_wq(dev); 1221 1222 mutex_lock(&dev->intf_state_mutex); 1223 if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) { 1224 dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", 1225 __func__); 1226 if (cleanup) 1227 mlx5_cleanup_once(dev); 1228 goto out; 1229 } 1230 1231 if (mlx5_device_registered(dev)) 1232 mlx5_detach_device(dev); 1233 1234 mlx5_sriov_detach(dev); 1235 #ifdef CONFIG_MLX5_CORE_EN 1236 mlx5_eswitch_detach(dev->priv.eswitch); 1237 #endif 1238 mlx5_cleanup_fs(dev); 1239 mlx5_irq_clear_affinity_hints(dev); 1240 free_comp_eqs(dev); 1241 mlx5_stop_eqs(dev); 1242 mlx5_put_uars_page(dev, priv->uar); 1243 mlx5_disable_msix(dev); 1244 if (cleanup) 1245 mlx5_cleanup_once(dev); 1246 mlx5_stop_health_poll(dev); 1247 err = mlx5_cmd_teardown_hca(dev); 1248 if (err) { 1249 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 1250 goto out; 1251 } 1252 mlx5_pagealloc_stop(dev); 1253 mlx5_reclaim_startup_pages(dev); 1254 mlx5_core_disable_hca(dev, 0); 1255 mlx5_cmd_cleanup(dev); 1256 1257 out: 1258 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); 1259 set_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state); 1260 mutex_unlock(&dev->intf_state_mutex); 1261 return err; 1262 } 1263 1264 struct mlx5_core_event_handler { 1265 void (*event)(struct mlx5_core_dev *dev, 1266 enum mlx5_dev_event event, 1267 void *data); 1268 }; 1269 1270 static const struct devlink_ops mlx5_devlink_ops = { 1271 #ifdef CONFIG_MLX5_CORE_EN 1272 .eswitch_mode_set = mlx5_devlink_eswitch_mode_set, 1273 .eswitch_mode_get = mlx5_devlink_eswitch_mode_get, 1274 .eswitch_inline_mode_set = mlx5_devlink_eswitch_inline_mode_set, 1275 .eswitch_inline_mode_get = mlx5_devlink_eswitch_inline_mode_get, 1276 .eswitch_encap_mode_set = mlx5_devlink_eswitch_encap_mode_set, 1277 .eswitch_encap_mode_get = mlx5_devlink_eswitch_encap_mode_get, 1278 #endif 1279 }; 1280 1281 #define MLX5_IB_MOD "mlx5_ib" 1282 static int init_one(struct pci_dev *pdev, 1283 const struct pci_device_id *id) 1284 { 1285 struct mlx5_core_dev *dev; 1286 struct devlink *devlink; 1287 struct mlx5_priv *priv; 1288 int err; 1289 1290 devlink = devlink_alloc(&mlx5_devlink_ops, sizeof(*dev)); 1291 if (!devlink) { 1292 dev_err(&pdev->dev, "kzalloc failed\n"); 1293 return -ENOMEM; 1294 } 1295 1296 dev = devlink_priv(devlink); 1297 priv = &dev->priv; 1298 priv->pci_dev_data = id->driver_data; 1299 1300 pci_set_drvdata(pdev, dev); 1301 1302 dev->pdev = pdev; 1303 dev->event = mlx5_core_event; 1304 dev->profile = &profile[prof_sel]; 1305 1306 INIT_LIST_HEAD(&priv->ctx_list); 1307 spin_lock_init(&priv->ctx_lock); 1308 mutex_init(&dev->pci_status_mutex); 1309 mutex_init(&dev->intf_state_mutex); 1310 1311 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1312 err = init_srcu_struct(&priv->pfault_srcu); 1313 if (err) { 1314 dev_err(&pdev->dev, "init_srcu_struct failed with error code %d\n", 1315 err); 1316 goto clean_dev; 1317 } 1318 #endif 1319 mutex_init(&priv->bfregs.reg_head.lock); 1320 mutex_init(&priv->bfregs.wc_head.lock); 1321 INIT_LIST_HEAD(&priv->bfregs.reg_head.list); 1322 INIT_LIST_HEAD(&priv->bfregs.wc_head.list); 1323 1324 err = mlx5_pci_init(dev, priv); 1325 if (err) { 1326 dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err); 1327 goto clean_srcu; 1328 } 1329 1330 err = mlx5_health_init(dev); 1331 if (err) { 1332 dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err); 1333 goto close_pci; 1334 } 1335 1336 mlx5_pagealloc_init(dev); 1337 1338 err = mlx5_load_one(dev, priv, true); 1339 if (err) { 1340 dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err); 1341 goto clean_health; 1342 } 1343 1344 request_module_nowait(MLX5_IB_MOD); 1345 1346 err = devlink_register(devlink, &pdev->dev); 1347 if (err) 1348 goto clean_load; 1349 1350 pci_save_state(pdev); 1351 return 0; 1352 1353 clean_load: 1354 mlx5_unload_one(dev, priv, true); 1355 clean_health: 1356 mlx5_pagealloc_cleanup(dev); 1357 mlx5_health_cleanup(dev); 1358 close_pci: 1359 mlx5_pci_close(dev, priv); 1360 clean_srcu: 1361 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1362 cleanup_srcu_struct(&priv->pfault_srcu); 1363 clean_dev: 1364 #endif 1365 pci_set_drvdata(pdev, NULL); 1366 devlink_free(devlink); 1367 1368 return err; 1369 } 1370 1371 static void remove_one(struct pci_dev *pdev) 1372 { 1373 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1374 struct devlink *devlink = priv_to_devlink(dev); 1375 struct mlx5_priv *priv = &dev->priv; 1376 1377 devlink_unregister(devlink); 1378 mlx5_unregister_device(dev); 1379 1380 if (mlx5_unload_one(dev, priv, true)) { 1381 dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); 1382 mlx5_health_cleanup(dev); 1383 return; 1384 } 1385 1386 mlx5_pagealloc_cleanup(dev); 1387 mlx5_health_cleanup(dev); 1388 mlx5_pci_close(dev, priv); 1389 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1390 cleanup_srcu_struct(&priv->pfault_srcu); 1391 #endif 1392 pci_set_drvdata(pdev, NULL); 1393 devlink_free(devlink); 1394 } 1395 1396 static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, 1397 pci_channel_state_t state) 1398 { 1399 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1400 struct mlx5_priv *priv = &dev->priv; 1401 1402 dev_info(&pdev->dev, "%s was called\n", __func__); 1403 1404 mlx5_enter_error_state(dev); 1405 mlx5_unload_one(dev, priv, false); 1406 /* In case of kernel call drain the health wq */ 1407 if (state) { 1408 mlx5_drain_health_wq(dev); 1409 mlx5_pci_disable_device(dev); 1410 } 1411 1412 return state == pci_channel_io_perm_failure ? 1413 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 1414 } 1415 1416 /* wait for the device to show vital signs by waiting 1417 * for the health counter to start counting. 1418 */ 1419 static int wait_vital(struct pci_dev *pdev) 1420 { 1421 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1422 struct mlx5_core_health *health = &dev->priv.health; 1423 const int niter = 100; 1424 u32 last_count = 0; 1425 u32 count; 1426 int i; 1427 1428 for (i = 0; i < niter; i++) { 1429 count = ioread32be(health->health_counter); 1430 if (count && count != 0xffffffff) { 1431 if (last_count && last_count != count) { 1432 dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); 1433 return 0; 1434 } 1435 last_count = count; 1436 } 1437 msleep(50); 1438 } 1439 1440 return -ETIMEDOUT; 1441 } 1442 1443 static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) 1444 { 1445 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1446 int err; 1447 1448 dev_info(&pdev->dev, "%s was called\n", __func__); 1449 1450 err = mlx5_pci_enable_device(dev); 1451 if (err) { 1452 dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" 1453 , __func__, err); 1454 return PCI_ERS_RESULT_DISCONNECT; 1455 } 1456 1457 pci_set_master(pdev); 1458 pci_restore_state(pdev); 1459 pci_save_state(pdev); 1460 1461 if (wait_vital(pdev)) { 1462 dev_err(&pdev->dev, "%s: wait_vital timed out\n", __func__); 1463 return PCI_ERS_RESULT_DISCONNECT; 1464 } 1465 1466 return PCI_ERS_RESULT_RECOVERED; 1467 } 1468 1469 static void mlx5_pci_resume(struct pci_dev *pdev) 1470 { 1471 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1472 struct mlx5_priv *priv = &dev->priv; 1473 int err; 1474 1475 dev_info(&pdev->dev, "%s was called\n", __func__); 1476 1477 err = mlx5_load_one(dev, priv, false); 1478 if (err) 1479 dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" 1480 , __func__, err); 1481 else 1482 dev_info(&pdev->dev, "%s: device recovered\n", __func__); 1483 } 1484 1485 static const struct pci_error_handlers mlx5_err_handler = { 1486 .error_detected = mlx5_pci_err_detected, 1487 .slot_reset = mlx5_pci_slot_reset, 1488 .resume = mlx5_pci_resume 1489 }; 1490 1491 static void shutdown(struct pci_dev *pdev) 1492 { 1493 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1494 struct mlx5_priv *priv = &dev->priv; 1495 1496 dev_info(&pdev->dev, "Shutdown was called\n"); 1497 /* Notify mlx5 clients that the kernel is being shut down */ 1498 set_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &dev->intf_state); 1499 mlx5_unload_one(dev, priv, false); 1500 mlx5_pci_disable_device(dev); 1501 } 1502 1503 static const struct pci_device_id mlx5_core_pci_table[] = { 1504 { PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */ 1505 { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */ 1506 { PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */ 1507 { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */ 1508 { PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */ 1509 { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */ 1510 { PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */ 1511 { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */ 1512 { PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5 Ex */ 1513 { PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 Ex VF */ 1514 { PCI_VDEVICE(MELLANOX, 0x101b) }, /* ConnectX-6 */ 1515 { PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF}, /* ConnectX-6 VF */ 1516 { 0, } 1517 }; 1518 1519 MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 1520 1521 void mlx5_disable_device(struct mlx5_core_dev *dev) 1522 { 1523 mlx5_pci_err_detected(dev->pdev, 0); 1524 } 1525 1526 void mlx5_recover_device(struct mlx5_core_dev *dev) 1527 { 1528 mlx5_pci_disable_device(dev); 1529 if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED) 1530 mlx5_pci_resume(dev->pdev); 1531 } 1532 1533 static struct pci_driver mlx5_core_driver = { 1534 .name = DRIVER_NAME, 1535 .id_table = mlx5_core_pci_table, 1536 .probe = init_one, 1537 .remove = remove_one, 1538 .shutdown = shutdown, 1539 .err_handler = &mlx5_err_handler, 1540 .sriov_configure = mlx5_core_sriov_configure, 1541 }; 1542 1543 static void mlx5_core_verify_params(void) 1544 { 1545 if (prof_sel >= ARRAY_SIZE(profile)) { 1546 pr_warn("mlx5_core: WARNING: Invalid module parameter prof_sel %d, valid range 0-%zu, changing back to default(%d)\n", 1547 prof_sel, 1548 ARRAY_SIZE(profile) - 1, 1549 MLX5_DEFAULT_PROF); 1550 prof_sel = MLX5_DEFAULT_PROF; 1551 } 1552 } 1553 1554 static int __init init(void) 1555 { 1556 int err; 1557 1558 mlx5_core_verify_params(); 1559 mlx5_register_debugfs(); 1560 1561 err = pci_register_driver(&mlx5_core_driver); 1562 if (err) 1563 goto err_debug; 1564 1565 #ifdef CONFIG_MLX5_CORE_EN 1566 mlx5e_init(); 1567 #endif 1568 1569 return 0; 1570 1571 err_debug: 1572 mlx5_unregister_debugfs(); 1573 return err; 1574 } 1575 1576 static void __exit cleanup(void) 1577 { 1578 #ifdef CONFIG_MLX5_CORE_EN 1579 mlx5e_cleanup(); 1580 #endif 1581 pci_unregister_driver(&mlx5_core_driver); 1582 mlx5_unregister_debugfs(); 1583 } 1584 1585 module_init(init); 1586 module_exit(cleanup); 1587