1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/mlx5/driver.h> 34 #include <linux/mlx5/eswitch.h> 35 #include <linux/mlx5/mlx5_ifc_vdpa.h> 36 #include <linux/mlx5/vport.h> 37 #include "mlx5_core.h" 38 #include "devlink.h" 39 #include "lag/lag.h" 40 41 /* intf dev list mutex */ 42 static DEFINE_MUTEX(mlx5_intf_mutex); 43 static DEFINE_IDA(mlx5_adev_ida); 44 45 static bool is_eth_rep_supported(struct mlx5_core_dev *dev) 46 { 47 if (!IS_ENABLED(CONFIG_MLX5_ESWITCH)) 48 return false; 49 50 if (!MLX5_ESWITCH_MANAGER(dev)) 51 return false; 52 53 if (!is_mdev_switchdev_mode(dev)) 54 return false; 55 56 return true; 57 } 58 59 bool mlx5_eth_supported(struct mlx5_core_dev *dev) 60 { 61 if (!IS_ENABLED(CONFIG_MLX5_CORE_EN)) 62 return false; 63 64 if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH) 65 return false; 66 67 if (!MLX5_CAP_GEN(dev, eth_net_offloads)) { 68 mlx5_core_warn(dev, "Missing eth_net_offloads capability\n"); 69 return false; 70 } 71 72 if (!MLX5_CAP_GEN(dev, nic_flow_table)) { 73 mlx5_core_warn(dev, "Missing nic_flow_table capability\n"); 74 return false; 75 } 76 77 if (!MLX5_CAP_ETH(dev, csum_cap)) { 78 mlx5_core_warn(dev, "Missing csum_cap capability\n"); 79 return false; 80 } 81 82 if (!MLX5_CAP_ETH(dev, max_lso_cap)) { 83 mlx5_core_warn(dev, "Missing max_lso_cap capability\n"); 84 return false; 85 } 86 87 if (!MLX5_CAP_ETH(dev, vlan_cap)) { 88 mlx5_core_warn(dev, "Missing vlan_cap capability\n"); 89 return false; 90 } 91 92 if (!MLX5_CAP_ETH(dev, rss_ind_tbl_cap)) { 93 mlx5_core_warn(dev, "Missing rss_ind_tbl_cap capability\n"); 94 return false; 95 } 96 97 if (MLX5_CAP_FLOWTABLE(dev, 98 flow_table_properties_nic_receive.max_ft_level) < 3) { 99 mlx5_core_warn(dev, "max_ft_level < 3\n"); 100 return false; 101 } 102 103 if (!MLX5_CAP_ETH(dev, self_lb_en_modifiable)) 104 mlx5_core_warn(dev, "Self loop back prevention is not supported\n"); 105 if (!MLX5_CAP_GEN(dev, cq_moderation)) 106 mlx5_core_warn(dev, "CQ moderation is not supported\n"); 107 108 return true; 109 } 110 111 bool mlx5_vnet_supported(struct mlx5_core_dev *dev) 112 { 113 if (!IS_ENABLED(CONFIG_MLX5_VDPA_NET)) 114 return false; 115 116 if (mlx5_core_is_pf(dev)) 117 return false; 118 119 if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & 120 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q)) 121 return false; 122 123 if (!(MLX5_CAP_DEV_VDPA_EMULATION(dev, event_mode) & 124 MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE)) 125 return false; 126 127 if (!MLX5_CAP_DEV_VDPA_EMULATION(dev, eth_frame_offload_type)) 128 return false; 129 130 return true; 131 } 132 133 static bool is_vnet_enabled(struct mlx5_core_dev *dev) 134 { 135 union devlink_param_value val; 136 int err; 137 138 err = devl_param_driverinit_value_get(priv_to_devlink(dev), 139 DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET, 140 &val); 141 return err ? false : val.vbool; 142 } 143 144 static bool is_ib_rep_supported(struct mlx5_core_dev *dev) 145 { 146 if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND)) 147 return false; 148 149 if (dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV) 150 return false; 151 152 if (!is_eth_rep_supported(dev)) 153 return false; 154 155 if (mlx5_core_mp_enabled(dev)) 156 return false; 157 158 return true; 159 } 160 161 static bool is_mp_supported(struct mlx5_core_dev *dev) 162 { 163 if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND)) 164 return false; 165 166 if (dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV) 167 return false; 168 169 if (is_ib_rep_supported(dev)) 170 return false; 171 172 if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH) 173 return false; 174 175 if (!mlx5_core_is_mp_slave(dev)) 176 return false; 177 178 return true; 179 } 180 181 bool mlx5_rdma_supported(struct mlx5_core_dev *dev) 182 { 183 if (!IS_ENABLED(CONFIG_MLX5_INFINIBAND)) 184 return false; 185 186 if (dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV) 187 return false; 188 189 if (is_ib_rep_supported(dev)) 190 return false; 191 192 if (is_mp_supported(dev)) 193 return false; 194 195 return true; 196 } 197 198 static bool is_ib_enabled(struct mlx5_core_dev *dev) 199 { 200 union devlink_param_value val; 201 int err; 202 203 err = devl_param_driverinit_value_get(priv_to_devlink(dev), 204 DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA, 205 &val); 206 return err ? false : val.vbool; 207 } 208 209 enum { 210 MLX5_INTERFACE_PROTOCOL_ETH, 211 MLX5_INTERFACE_PROTOCOL_ETH_REP, 212 213 MLX5_INTERFACE_PROTOCOL_IB, 214 MLX5_INTERFACE_PROTOCOL_IB_REP, 215 MLX5_INTERFACE_PROTOCOL_MPIB, 216 217 MLX5_INTERFACE_PROTOCOL_VNET, 218 }; 219 220 static const struct mlx5_adev_device { 221 const char *suffix; 222 bool (*is_supported)(struct mlx5_core_dev *dev); 223 bool (*is_enabled)(struct mlx5_core_dev *dev); 224 } mlx5_adev_devices[] = { 225 [MLX5_INTERFACE_PROTOCOL_VNET] = { .suffix = "vnet", 226 .is_supported = &mlx5_vnet_supported, 227 .is_enabled = &is_vnet_enabled }, 228 [MLX5_INTERFACE_PROTOCOL_IB] = { .suffix = "rdma", 229 .is_supported = &mlx5_rdma_supported, 230 .is_enabled = &is_ib_enabled }, 231 [MLX5_INTERFACE_PROTOCOL_ETH] = { .suffix = "eth", 232 .is_supported = &mlx5_eth_supported, 233 .is_enabled = &mlx5_core_is_eth_enabled }, 234 [MLX5_INTERFACE_PROTOCOL_ETH_REP] = { .suffix = "eth-rep", 235 .is_supported = &is_eth_rep_supported }, 236 [MLX5_INTERFACE_PROTOCOL_IB_REP] = { .suffix = "rdma-rep", 237 .is_supported = &is_ib_rep_supported }, 238 [MLX5_INTERFACE_PROTOCOL_MPIB] = { .suffix = "multiport", 239 .is_supported = &is_mp_supported }, 240 }; 241 242 int mlx5_adev_idx_alloc(void) 243 { 244 return ida_alloc(&mlx5_adev_ida, GFP_KERNEL); 245 } 246 247 void mlx5_adev_idx_free(int idx) 248 { 249 ida_free(&mlx5_adev_ida, idx); 250 } 251 252 int mlx5_adev_init(struct mlx5_core_dev *dev) 253 { 254 struct mlx5_priv *priv = &dev->priv; 255 256 priv->adev = kcalloc(ARRAY_SIZE(mlx5_adev_devices), 257 sizeof(struct mlx5_adev *), GFP_KERNEL); 258 if (!priv->adev) 259 return -ENOMEM; 260 261 return 0; 262 } 263 264 void mlx5_adev_cleanup(struct mlx5_core_dev *dev) 265 { 266 struct mlx5_priv *priv = &dev->priv; 267 268 kfree(priv->adev); 269 } 270 271 static void adev_release(struct device *dev) 272 { 273 struct mlx5_adev *mlx5_adev = 274 container_of(dev, struct mlx5_adev, adev.dev); 275 struct mlx5_priv *priv = &mlx5_adev->mdev->priv; 276 int idx = mlx5_adev->idx; 277 278 kfree(mlx5_adev); 279 priv->adev[idx] = NULL; 280 } 281 282 static struct mlx5_adev *add_adev(struct mlx5_core_dev *dev, int idx) 283 { 284 const char *suffix = mlx5_adev_devices[idx].suffix; 285 struct auxiliary_device *adev; 286 struct mlx5_adev *madev; 287 int ret; 288 289 madev = kzalloc(sizeof(*madev), GFP_KERNEL); 290 if (!madev) 291 return ERR_PTR(-ENOMEM); 292 293 adev = &madev->adev; 294 adev->id = dev->priv.adev_idx; 295 adev->name = suffix; 296 adev->dev.parent = dev->device; 297 adev->dev.release = adev_release; 298 madev->mdev = dev; 299 madev->idx = idx; 300 301 ret = auxiliary_device_init(adev); 302 if (ret) { 303 kfree(madev); 304 return ERR_PTR(ret); 305 } 306 307 ret = auxiliary_device_add(adev); 308 if (ret) { 309 auxiliary_device_uninit(adev); 310 return ERR_PTR(ret); 311 } 312 return madev; 313 } 314 315 static void del_adev(struct auxiliary_device *adev) 316 { 317 auxiliary_device_delete(adev); 318 auxiliary_device_uninit(adev); 319 } 320 321 void mlx5_dev_set_lightweight(struct mlx5_core_dev *dev) 322 { 323 mutex_lock(&mlx5_intf_mutex); 324 dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; 325 mutex_unlock(&mlx5_intf_mutex); 326 } 327 328 bool mlx5_dev_is_lightweight(struct mlx5_core_dev *dev) 329 { 330 return dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; 331 } 332 333 int mlx5_attach_device(struct mlx5_core_dev *dev) 334 { 335 struct mlx5_priv *priv = &dev->priv; 336 struct auxiliary_device *adev; 337 struct auxiliary_driver *adrv; 338 int ret = 0, i; 339 340 devl_assert_locked(priv_to_devlink(dev)); 341 mutex_lock(&mlx5_intf_mutex); 342 priv->flags &= ~MLX5_PRIV_FLAGS_DETACH; 343 for (i = 0; i < ARRAY_SIZE(mlx5_adev_devices); i++) { 344 if (!priv->adev[i]) { 345 bool is_supported = false; 346 347 if (mlx5_adev_devices[i].is_enabled) { 348 bool enabled; 349 350 enabled = mlx5_adev_devices[i].is_enabled(dev); 351 if (!enabled) 352 continue; 353 } 354 355 if (mlx5_adev_devices[i].is_supported) 356 is_supported = mlx5_adev_devices[i].is_supported(dev); 357 358 if (!is_supported) 359 continue; 360 361 priv->adev[i] = add_adev(dev, i); 362 if (IS_ERR(priv->adev[i])) { 363 ret = PTR_ERR(priv->adev[i]); 364 priv->adev[i] = NULL; 365 } 366 } else { 367 adev = &priv->adev[i]->adev; 368 369 /* Pay attention that this is not PCI driver that 370 * mlx5_core_dev is connected, but auxiliary driver. 371 */ 372 if (!adev->dev.driver) 373 continue; 374 adrv = to_auxiliary_drv(adev->dev.driver); 375 376 if (adrv->resume) 377 ret = adrv->resume(adev); 378 } 379 if (ret) { 380 mlx5_core_warn(dev, "Device[%d] (%s) failed to load\n", 381 i, mlx5_adev_devices[i].suffix); 382 383 break; 384 } 385 } 386 mutex_unlock(&mlx5_intf_mutex); 387 return ret; 388 } 389 390 void mlx5_detach_device(struct mlx5_core_dev *dev, bool suspend) 391 { 392 struct mlx5_priv *priv = &dev->priv; 393 struct auxiliary_device *adev; 394 struct auxiliary_driver *adrv; 395 pm_message_t pm = {}; 396 int i; 397 398 devl_assert_locked(priv_to_devlink(dev)); 399 mutex_lock(&mlx5_intf_mutex); 400 for (i = ARRAY_SIZE(mlx5_adev_devices) - 1; i >= 0; i--) { 401 if (!priv->adev[i]) 402 continue; 403 404 if (mlx5_adev_devices[i].is_enabled) { 405 bool enabled; 406 407 enabled = mlx5_adev_devices[i].is_enabled(dev); 408 if (!enabled) 409 goto skip_suspend; 410 } 411 412 adev = &priv->adev[i]->adev; 413 /* Auxiliary driver was unbind manually through sysfs */ 414 if (!adev->dev.driver) 415 goto skip_suspend; 416 417 adrv = to_auxiliary_drv(adev->dev.driver); 418 419 if (adrv->suspend && suspend) { 420 adrv->suspend(adev, pm); 421 continue; 422 } 423 424 skip_suspend: 425 del_adev(&priv->adev[i]->adev); 426 priv->adev[i] = NULL; 427 } 428 priv->flags |= MLX5_PRIV_FLAGS_DETACH; 429 mutex_unlock(&mlx5_intf_mutex); 430 } 431 432 int mlx5_register_device(struct mlx5_core_dev *dev) 433 { 434 int ret; 435 436 devl_assert_locked(priv_to_devlink(dev)); 437 mutex_lock(&mlx5_intf_mutex); 438 dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; 439 ret = mlx5_rescan_drivers_locked(dev); 440 mutex_unlock(&mlx5_intf_mutex); 441 if (ret) 442 mlx5_unregister_device(dev); 443 444 return ret; 445 } 446 447 void mlx5_unregister_device(struct mlx5_core_dev *dev) 448 { 449 devl_assert_locked(priv_to_devlink(dev)); 450 mutex_lock(&mlx5_intf_mutex); 451 dev->priv.flags = MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; 452 mlx5_rescan_drivers_locked(dev); 453 mutex_unlock(&mlx5_intf_mutex); 454 } 455 456 static int add_drivers(struct mlx5_core_dev *dev) 457 { 458 struct mlx5_priv *priv = &dev->priv; 459 int i, ret = 0; 460 461 for (i = 0; i < ARRAY_SIZE(mlx5_adev_devices); i++) { 462 bool is_supported = false; 463 464 if (priv->adev[i]) 465 continue; 466 467 if (mlx5_adev_devices[i].is_enabled && 468 !(mlx5_adev_devices[i].is_enabled(dev))) 469 continue; 470 471 if (mlx5_adev_devices[i].is_supported) 472 is_supported = mlx5_adev_devices[i].is_supported(dev); 473 474 if (!is_supported) 475 continue; 476 477 priv->adev[i] = add_adev(dev, i); 478 if (IS_ERR(priv->adev[i])) { 479 mlx5_core_warn(dev, "Device[%d] (%s) failed to load\n", 480 i, mlx5_adev_devices[i].suffix); 481 /* We continue to rescan drivers and leave to the caller 482 * to make decision if to release everything or continue. 483 */ 484 ret = PTR_ERR(priv->adev[i]); 485 priv->adev[i] = NULL; 486 } 487 } 488 return ret; 489 } 490 491 static void delete_drivers(struct mlx5_core_dev *dev) 492 { 493 struct mlx5_priv *priv = &dev->priv; 494 bool delete_all; 495 int i; 496 497 delete_all = priv->flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; 498 499 for (i = ARRAY_SIZE(mlx5_adev_devices) - 1; i >= 0; i--) { 500 bool is_supported = false; 501 502 if (!priv->adev[i]) 503 continue; 504 505 if (mlx5_adev_devices[i].is_enabled) { 506 bool enabled; 507 508 enabled = mlx5_adev_devices[i].is_enabled(dev); 509 if (!enabled) 510 goto del_adev; 511 } 512 513 if (mlx5_adev_devices[i].is_supported && !delete_all) 514 is_supported = mlx5_adev_devices[i].is_supported(dev); 515 516 if (is_supported) 517 continue; 518 519 del_adev: 520 del_adev(&priv->adev[i]->adev); 521 priv->adev[i] = NULL; 522 } 523 } 524 525 /* This function is used after mlx5_core_dev is reconfigured. 526 */ 527 int mlx5_rescan_drivers_locked(struct mlx5_core_dev *dev) 528 { 529 struct mlx5_priv *priv = &dev->priv; 530 531 lockdep_assert_held(&mlx5_intf_mutex); 532 if (priv->flags & MLX5_PRIV_FLAGS_DETACH) 533 return 0; 534 535 delete_drivers(dev); 536 if (priv->flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) 537 return 0; 538 539 return add_drivers(dev); 540 } 541 542 bool mlx5_same_hw_devs(struct mlx5_core_dev *dev, struct mlx5_core_dev *peer_dev) 543 { 544 u64 fsystem_guid, psystem_guid; 545 546 fsystem_guid = mlx5_query_nic_system_image_guid(dev); 547 psystem_guid = mlx5_query_nic_system_image_guid(peer_dev); 548 549 return (fsystem_guid && psystem_guid && fsystem_guid == psystem_guid); 550 } 551 552 static u32 mlx5_gen_pci_id(const struct mlx5_core_dev *dev) 553 { 554 return (u32)((pci_domain_nr(dev->pdev->bus) << 16) | 555 (dev->pdev->bus->number << 8) | 556 PCI_SLOT(dev->pdev->devfn)); 557 } 558 559 static int _next_phys_dev(struct mlx5_core_dev *mdev, 560 const struct mlx5_core_dev *curr) 561 { 562 if (!mlx5_core_is_pf(mdev)) 563 return 0; 564 565 if (mdev == curr) 566 return 0; 567 568 if (!mlx5_same_hw_devs(mdev, (struct mlx5_core_dev *)curr) && 569 mlx5_gen_pci_id(mdev) != mlx5_gen_pci_id(curr)) 570 return 0; 571 572 return 1; 573 } 574 575 static void *pci_get_other_drvdata(struct device *this, struct device *other) 576 { 577 if (this->driver != other->driver) 578 return NULL; 579 580 return pci_get_drvdata(to_pci_dev(other)); 581 } 582 583 static int next_phys_dev_lag(struct device *dev, const void *data) 584 { 585 struct mlx5_core_dev *mdev, *this = (struct mlx5_core_dev *)data; 586 587 mdev = pci_get_other_drvdata(this->device, dev); 588 if (!mdev) 589 return 0; 590 591 if (!mlx5_lag_is_supported(mdev)) 592 return 0; 593 594 return _next_phys_dev(mdev, data); 595 } 596 597 static struct mlx5_core_dev *mlx5_get_next_dev(struct mlx5_core_dev *dev, 598 int (*match)(struct device *dev, const void *data)) 599 { 600 struct device *next; 601 602 if (!mlx5_core_is_pf(dev)) 603 return NULL; 604 605 next = bus_find_device(&pci_bus_type, NULL, dev, match); 606 if (!next) 607 return NULL; 608 609 put_device(next); 610 return pci_get_drvdata(to_pci_dev(next)); 611 } 612 613 /* Must be called with intf_mutex held */ 614 struct mlx5_core_dev *mlx5_get_next_phys_dev_lag(struct mlx5_core_dev *dev) 615 { 616 lockdep_assert_held(&mlx5_intf_mutex); 617 return mlx5_get_next_dev(dev, &next_phys_dev_lag); 618 } 619 620 void mlx5_dev_list_lock(void) 621 { 622 mutex_lock(&mlx5_intf_mutex); 623 } 624 void mlx5_dev_list_unlock(void) 625 { 626 mutex_unlock(&mlx5_intf_mutex); 627 } 628 629 int mlx5_dev_list_trylock(void) 630 { 631 return mutex_trylock(&mlx5_intf_mutex); 632 } 633