1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 #include <linux/string.h> 36 #include <linux/errno.h> 37 #include <linux/kernel.h> 38 #include <linux/slab.h> 39 #include <linux/init.h> 40 #include <linux/mutex.h> 41 #include <linux/netdevice.h> 42 #include <linux/security.h> 43 #include <linux/notifier.h> 44 #include <rdma/rdma_netlink.h> 45 #include <rdma/ib_addr.h> 46 #include <rdma/ib_cache.h> 47 48 #include "core_priv.h" 49 50 MODULE_AUTHOR("Roland Dreier"); 51 MODULE_DESCRIPTION("core kernel InfiniBand API"); 52 MODULE_LICENSE("Dual BSD/GPL"); 53 54 struct ib_client_data { 55 struct list_head list; 56 struct ib_client *client; 57 void * data; 58 /* The device or client is going down. Do not call client or device 59 * callbacks other than remove(). */ 60 bool going_down; 61 }; 62 63 struct workqueue_struct *ib_comp_wq; 64 struct workqueue_struct *ib_wq; 65 EXPORT_SYMBOL_GPL(ib_wq); 66 67 /* The device_list and client_list contain devices and clients after their 68 * registration has completed, and the devices and clients are removed 69 * during unregistration. */ 70 static LIST_HEAD(device_list); 71 static LIST_HEAD(client_list); 72 73 /* 74 * device_mutex and lists_rwsem protect access to both device_list and 75 * client_list. device_mutex protects writer access by device and client 76 * registration / de-registration. lists_rwsem protects reader access to 77 * these lists. Iterators of these lists must lock it for read, while updates 78 * to the lists must be done with a write lock. A special case is when the 79 * device_mutex is locked. In this case locking the lists for read access is 80 * not necessary as the device_mutex implies it. 81 * 82 * lists_rwsem also protects access to the client data list. 83 */ 84 static DEFINE_MUTEX(device_mutex); 85 static DECLARE_RWSEM(lists_rwsem); 86 87 static int ib_security_change(struct notifier_block *nb, unsigned long event, 88 void *lsm_data); 89 static void ib_policy_change_task(struct work_struct *work); 90 static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task); 91 92 static struct notifier_block ibdev_lsm_nb = { 93 .notifier_call = ib_security_change, 94 }; 95 96 static int ib_device_check_mandatory(struct ib_device *device) 97 { 98 #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device, x), #x } 99 static const struct { 100 size_t offset; 101 char *name; 102 } mandatory_table[] = { 103 IB_MANDATORY_FUNC(query_device), 104 IB_MANDATORY_FUNC(query_port), 105 IB_MANDATORY_FUNC(query_pkey), 106 IB_MANDATORY_FUNC(query_gid), 107 IB_MANDATORY_FUNC(alloc_pd), 108 IB_MANDATORY_FUNC(dealloc_pd), 109 IB_MANDATORY_FUNC(create_ah), 110 IB_MANDATORY_FUNC(destroy_ah), 111 IB_MANDATORY_FUNC(create_qp), 112 IB_MANDATORY_FUNC(modify_qp), 113 IB_MANDATORY_FUNC(destroy_qp), 114 IB_MANDATORY_FUNC(post_send), 115 IB_MANDATORY_FUNC(post_recv), 116 IB_MANDATORY_FUNC(create_cq), 117 IB_MANDATORY_FUNC(destroy_cq), 118 IB_MANDATORY_FUNC(poll_cq), 119 IB_MANDATORY_FUNC(req_notify_cq), 120 IB_MANDATORY_FUNC(get_dma_mr), 121 IB_MANDATORY_FUNC(dereg_mr), 122 IB_MANDATORY_FUNC(get_port_immutable) 123 }; 124 int i; 125 126 for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { 127 if (!*(void **) ((void *) device + mandatory_table[i].offset)) { 128 pr_warn("Device %s is missing mandatory function %s\n", 129 device->name, mandatory_table[i].name); 130 return -EINVAL; 131 } 132 } 133 134 return 0; 135 } 136 137 static struct ib_device *__ib_device_get_by_index(u32 index) 138 { 139 struct ib_device *device; 140 141 list_for_each_entry(device, &device_list, core_list) 142 if (device->index == index) 143 return device; 144 145 return NULL; 146 } 147 148 /* 149 * Caller is responsible to return refrerence count by calling put_device() 150 */ 151 struct ib_device *ib_device_get_by_index(u32 index) 152 { 153 struct ib_device *device; 154 155 down_read(&lists_rwsem); 156 device = __ib_device_get_by_index(index); 157 if (device) 158 get_device(&device->dev); 159 160 up_read(&lists_rwsem); 161 return device; 162 } 163 164 static struct ib_device *__ib_device_get_by_name(const char *name) 165 { 166 struct ib_device *device; 167 168 list_for_each_entry(device, &device_list, core_list) 169 if (!strncmp(name, device->name, IB_DEVICE_NAME_MAX)) 170 return device; 171 172 return NULL; 173 } 174 175 static int alloc_name(char *name) 176 { 177 unsigned long *inuse; 178 char buf[IB_DEVICE_NAME_MAX]; 179 struct ib_device *device; 180 int i; 181 182 inuse = (unsigned long *) get_zeroed_page(GFP_KERNEL); 183 if (!inuse) 184 return -ENOMEM; 185 186 list_for_each_entry(device, &device_list, core_list) { 187 if (!sscanf(device->name, name, &i)) 188 continue; 189 if (i < 0 || i >= PAGE_SIZE * 8) 190 continue; 191 snprintf(buf, sizeof buf, name, i); 192 if (!strncmp(buf, device->name, IB_DEVICE_NAME_MAX)) 193 set_bit(i, inuse); 194 } 195 196 i = find_first_zero_bit(inuse, PAGE_SIZE * 8); 197 free_page((unsigned long) inuse); 198 snprintf(buf, sizeof buf, name, i); 199 200 if (__ib_device_get_by_name(buf)) 201 return -ENFILE; 202 203 strlcpy(name, buf, IB_DEVICE_NAME_MAX); 204 return 0; 205 } 206 207 static void ib_device_release(struct device *device) 208 { 209 struct ib_device *dev = container_of(device, struct ib_device, dev); 210 211 WARN_ON(dev->reg_state == IB_DEV_REGISTERED); 212 if (dev->reg_state == IB_DEV_UNREGISTERED) { 213 /* 214 * In IB_DEV_UNINITIALIZED state, cache or port table 215 * is not even created. Free cache and port table only when 216 * device reaches UNREGISTERED state. 217 */ 218 ib_cache_release_one(dev); 219 kfree(dev->port_immutable); 220 } 221 kfree(dev); 222 } 223 224 static int ib_device_uevent(struct device *device, 225 struct kobj_uevent_env *env) 226 { 227 struct ib_device *dev = container_of(device, struct ib_device, dev); 228 229 if (add_uevent_var(env, "NAME=%s", dev->name)) 230 return -ENOMEM; 231 232 /* 233 * It would be nice to pass the node GUID with the event... 234 */ 235 236 return 0; 237 } 238 239 static struct class ib_class = { 240 .name = "infiniband", 241 .dev_release = ib_device_release, 242 .dev_uevent = ib_device_uevent, 243 }; 244 245 /** 246 * ib_alloc_device - allocate an IB device struct 247 * @size:size of structure to allocate 248 * 249 * Low-level drivers should use ib_alloc_device() to allocate &struct 250 * ib_device. @size is the size of the structure to be allocated, 251 * including any private data used by the low-level driver. 252 * ib_dealloc_device() must be used to free structures allocated with 253 * ib_alloc_device(). 254 */ 255 struct ib_device *ib_alloc_device(size_t size) 256 { 257 struct ib_device *device; 258 259 if (WARN_ON(size < sizeof(struct ib_device))) 260 return NULL; 261 262 device = kzalloc(size, GFP_KERNEL); 263 if (!device) 264 return NULL; 265 266 rdma_restrack_init(&device->res); 267 268 device->dev.class = &ib_class; 269 device_initialize(&device->dev); 270 271 dev_set_drvdata(&device->dev, device); 272 273 INIT_LIST_HEAD(&device->event_handler_list); 274 spin_lock_init(&device->event_handler_lock); 275 spin_lock_init(&device->client_data_lock); 276 INIT_LIST_HEAD(&device->client_data_list); 277 INIT_LIST_HEAD(&device->port_list); 278 279 return device; 280 } 281 EXPORT_SYMBOL(ib_alloc_device); 282 283 /** 284 * ib_dealloc_device - free an IB device struct 285 * @device:structure to free 286 * 287 * Free a structure allocated with ib_alloc_device(). 288 */ 289 void ib_dealloc_device(struct ib_device *device) 290 { 291 WARN_ON(device->reg_state != IB_DEV_UNREGISTERED && 292 device->reg_state != IB_DEV_UNINITIALIZED); 293 put_device(&device->dev); 294 } 295 EXPORT_SYMBOL(ib_dealloc_device); 296 297 static int add_client_context(struct ib_device *device, struct ib_client *client) 298 { 299 struct ib_client_data *context; 300 unsigned long flags; 301 302 context = kmalloc(sizeof *context, GFP_KERNEL); 303 if (!context) 304 return -ENOMEM; 305 306 context->client = client; 307 context->data = NULL; 308 context->going_down = false; 309 310 down_write(&lists_rwsem); 311 spin_lock_irqsave(&device->client_data_lock, flags); 312 list_add(&context->list, &device->client_data_list); 313 spin_unlock_irqrestore(&device->client_data_lock, flags); 314 up_write(&lists_rwsem); 315 316 return 0; 317 } 318 319 static int verify_immutable(const struct ib_device *dev, u8 port) 320 { 321 return WARN_ON(!rdma_cap_ib_mad(dev, port) && 322 rdma_max_mad_size(dev, port) != 0); 323 } 324 325 static int read_port_immutable(struct ib_device *device) 326 { 327 int ret; 328 u8 start_port = rdma_start_port(device); 329 u8 end_port = rdma_end_port(device); 330 u8 port; 331 332 /** 333 * device->port_immutable is indexed directly by the port number to make 334 * access to this data as efficient as possible. 335 * 336 * Therefore port_immutable is declared as a 1 based array with 337 * potential empty slots at the beginning. 338 */ 339 device->port_immutable = kzalloc(sizeof(*device->port_immutable) 340 * (end_port + 1), 341 GFP_KERNEL); 342 if (!device->port_immutable) 343 return -ENOMEM; 344 345 for (port = start_port; port <= end_port; ++port) { 346 ret = device->get_port_immutable(device, port, 347 &device->port_immutable[port]); 348 if (ret) 349 return ret; 350 351 if (verify_immutable(device, port)) 352 return -EINVAL; 353 } 354 return 0; 355 } 356 357 void ib_get_device_fw_str(struct ib_device *dev, char *str) 358 { 359 if (dev->get_dev_fw_str) 360 dev->get_dev_fw_str(dev, str); 361 else 362 str[0] = '\0'; 363 } 364 EXPORT_SYMBOL(ib_get_device_fw_str); 365 366 static int setup_port_pkey_list(struct ib_device *device) 367 { 368 int i; 369 370 /** 371 * device->port_pkey_list is indexed directly by the port number, 372 * Therefore it is declared as a 1 based array with potential empty 373 * slots at the beginning. 374 */ 375 device->port_pkey_list = kcalloc(rdma_end_port(device) + 1, 376 sizeof(*device->port_pkey_list), 377 GFP_KERNEL); 378 379 if (!device->port_pkey_list) 380 return -ENOMEM; 381 382 for (i = 0; i < (rdma_end_port(device) + 1); i++) { 383 spin_lock_init(&device->port_pkey_list[i].list_lock); 384 INIT_LIST_HEAD(&device->port_pkey_list[i].pkey_list); 385 } 386 387 return 0; 388 } 389 390 static void ib_policy_change_task(struct work_struct *work) 391 { 392 struct ib_device *dev; 393 394 down_read(&lists_rwsem); 395 list_for_each_entry(dev, &device_list, core_list) { 396 int i; 397 398 for (i = rdma_start_port(dev); i <= rdma_end_port(dev); i++) { 399 u64 sp; 400 int ret = ib_get_cached_subnet_prefix(dev, 401 i, 402 &sp); 403 404 WARN_ONCE(ret, 405 "ib_get_cached_subnet_prefix err: %d, this should never happen here\n", 406 ret); 407 if (!ret) 408 ib_security_cache_change(dev, i, sp); 409 } 410 } 411 up_read(&lists_rwsem); 412 } 413 414 static int ib_security_change(struct notifier_block *nb, unsigned long event, 415 void *lsm_data) 416 { 417 if (event != LSM_POLICY_CHANGE) 418 return NOTIFY_DONE; 419 420 schedule_work(&ib_policy_change_work); 421 422 return NOTIFY_OK; 423 } 424 425 /** 426 * __dev_new_index - allocate an device index 427 * 428 * Returns a suitable unique value for a new device interface 429 * number. It assumes that there are less than 2^32-1 ib devices 430 * will be present in the system. 431 */ 432 static u32 __dev_new_index(void) 433 { 434 /* 435 * The device index to allow stable naming. 436 * Similar to struct net -> ifindex. 437 */ 438 static u32 index; 439 440 for (;;) { 441 if (!(++index)) 442 index = 1; 443 444 if (!__ib_device_get_by_index(index)) 445 return index; 446 } 447 } 448 449 /** 450 * ib_register_device - Register an IB device with IB core 451 * @device:Device to register 452 * 453 * Low-level drivers use ib_register_device() to register their 454 * devices with the IB core. All registered clients will receive a 455 * callback for each device that is added. @device must be allocated 456 * with ib_alloc_device(). 457 */ 458 int ib_register_device(struct ib_device *device, 459 int (*port_callback)(struct ib_device *, 460 u8, struct kobject *)) 461 { 462 int ret; 463 struct ib_client *client; 464 struct ib_udata uhw = {.outlen = 0, .inlen = 0}; 465 struct device *parent = device->dev.parent; 466 467 WARN_ON_ONCE(device->dma_device); 468 if (device->dev.dma_ops) { 469 /* 470 * The caller provided custom DMA operations. Copy the 471 * DMA-related fields that are used by e.g. dma_alloc_coherent() 472 * into device->dev. 473 */ 474 device->dma_device = &device->dev; 475 if (!device->dev.dma_mask) { 476 if (parent) 477 device->dev.dma_mask = parent->dma_mask; 478 else 479 WARN_ON_ONCE(true); 480 } 481 if (!device->dev.coherent_dma_mask) { 482 if (parent) 483 device->dev.coherent_dma_mask = 484 parent->coherent_dma_mask; 485 else 486 WARN_ON_ONCE(true); 487 } 488 } else { 489 /* 490 * The caller did not provide custom DMA operations. Use the 491 * DMA mapping operations of the parent device. 492 */ 493 WARN_ON_ONCE(!parent); 494 device->dma_device = parent; 495 } 496 497 mutex_lock(&device_mutex); 498 499 if (strchr(device->name, '%')) { 500 ret = alloc_name(device->name); 501 if (ret) 502 goto out; 503 } 504 505 if (ib_device_check_mandatory(device)) { 506 ret = -EINVAL; 507 goto out; 508 } 509 510 ret = read_port_immutable(device); 511 if (ret) { 512 pr_warn("Couldn't create per port immutable data %s\n", 513 device->name); 514 goto out; 515 } 516 517 ret = setup_port_pkey_list(device); 518 if (ret) { 519 pr_warn("Couldn't create per port_pkey_list\n"); 520 goto out; 521 } 522 523 ret = ib_cache_setup_one(device); 524 if (ret) { 525 pr_warn("Couldn't set up InfiniBand P_Key/GID cache\n"); 526 goto port_cleanup; 527 } 528 529 ret = ib_device_register_rdmacg(device); 530 if (ret) { 531 pr_warn("Couldn't register device with rdma cgroup\n"); 532 goto cache_cleanup; 533 } 534 535 memset(&device->attrs, 0, sizeof(device->attrs)); 536 ret = device->query_device(device, &device->attrs, &uhw); 537 if (ret) { 538 pr_warn("Couldn't query the device attributes\n"); 539 goto cache_cleanup; 540 } 541 542 ret = ib_device_register_sysfs(device, port_callback); 543 if (ret) { 544 pr_warn("Couldn't register device %s with driver model\n", 545 device->name); 546 goto cache_cleanup; 547 } 548 549 device->reg_state = IB_DEV_REGISTERED; 550 551 list_for_each_entry(client, &client_list, list) 552 if (!add_client_context(device, client) && client->add) 553 client->add(device); 554 555 device->index = __dev_new_index(); 556 down_write(&lists_rwsem); 557 list_add_tail(&device->core_list, &device_list); 558 up_write(&lists_rwsem); 559 mutex_unlock(&device_mutex); 560 return 0; 561 562 cache_cleanup: 563 ib_cache_cleanup_one(device); 564 ib_cache_release_one(device); 565 port_cleanup: 566 kfree(device->port_immutable); 567 out: 568 mutex_unlock(&device_mutex); 569 return ret; 570 } 571 EXPORT_SYMBOL(ib_register_device); 572 573 /** 574 * ib_unregister_device - Unregister an IB device 575 * @device:Device to unregister 576 * 577 * Unregister an IB device. All clients will receive a remove callback. 578 */ 579 void ib_unregister_device(struct ib_device *device) 580 { 581 struct ib_client_data *context, *tmp; 582 unsigned long flags; 583 584 mutex_lock(&device_mutex); 585 586 down_write(&lists_rwsem); 587 list_del(&device->core_list); 588 spin_lock_irqsave(&device->client_data_lock, flags); 589 list_for_each_entry_safe(context, tmp, &device->client_data_list, list) 590 context->going_down = true; 591 spin_unlock_irqrestore(&device->client_data_lock, flags); 592 downgrade_write(&lists_rwsem); 593 594 list_for_each_entry_safe(context, tmp, &device->client_data_list, 595 list) { 596 if (context->client->remove) 597 context->client->remove(device, context->data); 598 } 599 up_read(&lists_rwsem); 600 601 rdma_restrack_clean(&device->res); 602 603 ib_device_unregister_rdmacg(device); 604 ib_device_unregister_sysfs(device); 605 606 mutex_unlock(&device_mutex); 607 608 ib_cache_cleanup_one(device); 609 610 ib_security_destroy_port_pkey_list(device); 611 kfree(device->port_pkey_list); 612 613 down_write(&lists_rwsem); 614 spin_lock_irqsave(&device->client_data_lock, flags); 615 list_for_each_entry_safe(context, tmp, &device->client_data_list, list) 616 kfree(context); 617 spin_unlock_irqrestore(&device->client_data_lock, flags); 618 up_write(&lists_rwsem); 619 620 device->reg_state = IB_DEV_UNREGISTERED; 621 } 622 EXPORT_SYMBOL(ib_unregister_device); 623 624 /** 625 * ib_register_client - Register an IB client 626 * @client:Client to register 627 * 628 * Upper level users of the IB drivers can use ib_register_client() to 629 * register callbacks for IB device addition and removal. When an IB 630 * device is added, each registered client's add method will be called 631 * (in the order the clients were registered), and when a device is 632 * removed, each client's remove method will be called (in the reverse 633 * order that clients were registered). In addition, when 634 * ib_register_client() is called, the client will receive an add 635 * callback for all devices already registered. 636 */ 637 int ib_register_client(struct ib_client *client) 638 { 639 struct ib_device *device; 640 641 mutex_lock(&device_mutex); 642 643 list_for_each_entry(device, &device_list, core_list) 644 if (!add_client_context(device, client) && client->add) 645 client->add(device); 646 647 down_write(&lists_rwsem); 648 list_add_tail(&client->list, &client_list); 649 up_write(&lists_rwsem); 650 651 mutex_unlock(&device_mutex); 652 653 return 0; 654 } 655 EXPORT_SYMBOL(ib_register_client); 656 657 /** 658 * ib_unregister_client - Unregister an IB client 659 * @client:Client to unregister 660 * 661 * Upper level users use ib_unregister_client() to remove their client 662 * registration. When ib_unregister_client() is called, the client 663 * will receive a remove callback for each IB device still registered. 664 */ 665 void ib_unregister_client(struct ib_client *client) 666 { 667 struct ib_client_data *context, *tmp; 668 struct ib_device *device; 669 unsigned long flags; 670 671 mutex_lock(&device_mutex); 672 673 down_write(&lists_rwsem); 674 list_del(&client->list); 675 up_write(&lists_rwsem); 676 677 list_for_each_entry(device, &device_list, core_list) { 678 struct ib_client_data *found_context = NULL; 679 680 down_write(&lists_rwsem); 681 spin_lock_irqsave(&device->client_data_lock, flags); 682 list_for_each_entry_safe(context, tmp, &device->client_data_list, list) 683 if (context->client == client) { 684 context->going_down = true; 685 found_context = context; 686 break; 687 } 688 spin_unlock_irqrestore(&device->client_data_lock, flags); 689 up_write(&lists_rwsem); 690 691 if (client->remove) 692 client->remove(device, found_context ? 693 found_context->data : NULL); 694 695 if (!found_context) { 696 pr_warn("No client context found for %s/%s\n", 697 device->name, client->name); 698 continue; 699 } 700 701 down_write(&lists_rwsem); 702 spin_lock_irqsave(&device->client_data_lock, flags); 703 list_del(&found_context->list); 704 kfree(found_context); 705 spin_unlock_irqrestore(&device->client_data_lock, flags); 706 up_write(&lists_rwsem); 707 } 708 709 mutex_unlock(&device_mutex); 710 } 711 EXPORT_SYMBOL(ib_unregister_client); 712 713 /** 714 * ib_get_client_data - Get IB client context 715 * @device:Device to get context for 716 * @client:Client to get context for 717 * 718 * ib_get_client_data() returns client context set with 719 * ib_set_client_data(). 720 */ 721 void *ib_get_client_data(struct ib_device *device, struct ib_client *client) 722 { 723 struct ib_client_data *context; 724 void *ret = NULL; 725 unsigned long flags; 726 727 spin_lock_irqsave(&device->client_data_lock, flags); 728 list_for_each_entry(context, &device->client_data_list, list) 729 if (context->client == client) { 730 ret = context->data; 731 break; 732 } 733 spin_unlock_irqrestore(&device->client_data_lock, flags); 734 735 return ret; 736 } 737 EXPORT_SYMBOL(ib_get_client_data); 738 739 /** 740 * ib_set_client_data - Set IB client context 741 * @device:Device to set context for 742 * @client:Client to set context for 743 * @data:Context to set 744 * 745 * ib_set_client_data() sets client context that can be retrieved with 746 * ib_get_client_data(). 747 */ 748 void ib_set_client_data(struct ib_device *device, struct ib_client *client, 749 void *data) 750 { 751 struct ib_client_data *context; 752 unsigned long flags; 753 754 spin_lock_irqsave(&device->client_data_lock, flags); 755 list_for_each_entry(context, &device->client_data_list, list) 756 if (context->client == client) { 757 context->data = data; 758 goto out; 759 } 760 761 pr_warn("No client context found for %s/%s\n", 762 device->name, client->name); 763 764 out: 765 spin_unlock_irqrestore(&device->client_data_lock, flags); 766 } 767 EXPORT_SYMBOL(ib_set_client_data); 768 769 /** 770 * ib_register_event_handler - Register an IB event handler 771 * @event_handler:Handler to register 772 * 773 * ib_register_event_handler() registers an event handler that will be 774 * called back when asynchronous IB events occur (as defined in 775 * chapter 11 of the InfiniBand Architecture Specification). This 776 * callback may occur in interrupt context. 777 */ 778 void ib_register_event_handler(struct ib_event_handler *event_handler) 779 { 780 unsigned long flags; 781 782 spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 783 list_add_tail(&event_handler->list, 784 &event_handler->device->event_handler_list); 785 spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 786 } 787 EXPORT_SYMBOL(ib_register_event_handler); 788 789 /** 790 * ib_unregister_event_handler - Unregister an event handler 791 * @event_handler:Handler to unregister 792 * 793 * Unregister an event handler registered with 794 * ib_register_event_handler(). 795 */ 796 void ib_unregister_event_handler(struct ib_event_handler *event_handler) 797 { 798 unsigned long flags; 799 800 spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 801 list_del(&event_handler->list); 802 spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 803 } 804 EXPORT_SYMBOL(ib_unregister_event_handler); 805 806 /** 807 * ib_dispatch_event - Dispatch an asynchronous event 808 * @event:Event to dispatch 809 * 810 * Low-level drivers must call ib_dispatch_event() to dispatch the 811 * event to all registered event handlers when an asynchronous event 812 * occurs. 813 */ 814 void ib_dispatch_event(struct ib_event *event) 815 { 816 unsigned long flags; 817 struct ib_event_handler *handler; 818 819 spin_lock_irqsave(&event->device->event_handler_lock, flags); 820 821 list_for_each_entry(handler, &event->device->event_handler_list, list) 822 handler->handler(handler, event); 823 824 spin_unlock_irqrestore(&event->device->event_handler_lock, flags); 825 } 826 EXPORT_SYMBOL(ib_dispatch_event); 827 828 /** 829 * ib_query_port - Query IB port attributes 830 * @device:Device to query 831 * @port_num:Port number to query 832 * @port_attr:Port attributes 833 * 834 * ib_query_port() returns the attributes of a port through the 835 * @port_attr pointer. 836 */ 837 int ib_query_port(struct ib_device *device, 838 u8 port_num, 839 struct ib_port_attr *port_attr) 840 { 841 union ib_gid gid; 842 int err; 843 844 if (!rdma_is_port_valid(device, port_num)) 845 return -EINVAL; 846 847 memset(port_attr, 0, sizeof(*port_attr)); 848 err = device->query_port(device, port_num, port_attr); 849 if (err || port_attr->subnet_prefix) 850 return err; 851 852 if (rdma_port_get_link_layer(device, port_num) != IB_LINK_LAYER_INFINIBAND) 853 return 0; 854 855 err = ib_query_gid(device, port_num, 0, &gid, NULL); 856 if (err) 857 return err; 858 859 port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix); 860 return 0; 861 } 862 EXPORT_SYMBOL(ib_query_port); 863 864 /** 865 * ib_query_gid - Get GID table entry 866 * @device:Device to query 867 * @port_num:Port number to query 868 * @index:GID table index to query 869 * @gid:Returned GID 870 * @attr: Returned GID attributes related to this GID index (only in RoCE). 871 * NULL means ignore. 872 * 873 * ib_query_gid() fetches the specified GID table entry. 874 */ 875 int ib_query_gid(struct ib_device *device, 876 u8 port_num, int index, union ib_gid *gid, 877 struct ib_gid_attr *attr) 878 { 879 if (rdma_cap_roce_gid_table(device, port_num)) 880 return ib_get_cached_gid(device, port_num, index, gid, attr); 881 882 if (attr) 883 return -EINVAL; 884 885 return device->query_gid(device, port_num, index, gid); 886 } 887 EXPORT_SYMBOL(ib_query_gid); 888 889 /** 890 * ib_enum_roce_netdev - enumerate all RoCE ports 891 * @ib_dev : IB device we want to query 892 * @filter: Should we call the callback? 893 * @filter_cookie: Cookie passed to filter 894 * @cb: Callback to call for each found RoCE ports 895 * @cookie: Cookie passed back to the callback 896 * 897 * Enumerates all of the physical RoCE ports of ib_dev 898 * which are related to netdevice and calls callback() on each 899 * device for which filter() function returns non zero. 900 */ 901 void ib_enum_roce_netdev(struct ib_device *ib_dev, 902 roce_netdev_filter filter, 903 void *filter_cookie, 904 roce_netdev_callback cb, 905 void *cookie) 906 { 907 u8 port; 908 909 for (port = rdma_start_port(ib_dev); port <= rdma_end_port(ib_dev); 910 port++) 911 if (rdma_protocol_roce(ib_dev, port)) { 912 struct net_device *idev = NULL; 913 914 if (ib_dev->get_netdev) 915 idev = ib_dev->get_netdev(ib_dev, port); 916 917 if (idev && 918 idev->reg_state >= NETREG_UNREGISTERED) { 919 dev_put(idev); 920 idev = NULL; 921 } 922 923 if (filter(ib_dev, port, idev, filter_cookie)) 924 cb(ib_dev, port, idev, cookie); 925 926 if (idev) 927 dev_put(idev); 928 } 929 } 930 931 /** 932 * ib_enum_all_roce_netdevs - enumerate all RoCE devices 933 * @filter: Should we call the callback? 934 * @filter_cookie: Cookie passed to filter 935 * @cb: Callback to call for each found RoCE ports 936 * @cookie: Cookie passed back to the callback 937 * 938 * Enumerates all RoCE devices' physical ports which are related 939 * to netdevices and calls callback() on each device for which 940 * filter() function returns non zero. 941 */ 942 void ib_enum_all_roce_netdevs(roce_netdev_filter filter, 943 void *filter_cookie, 944 roce_netdev_callback cb, 945 void *cookie) 946 { 947 struct ib_device *dev; 948 949 down_read(&lists_rwsem); 950 list_for_each_entry(dev, &device_list, core_list) 951 ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); 952 up_read(&lists_rwsem); 953 } 954 955 /** 956 * ib_enum_all_devs - enumerate all ib_devices 957 * @cb: Callback to call for each found ib_device 958 * 959 * Enumerates all ib_devices and calls callback() on each device. 960 */ 961 int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb, 962 struct netlink_callback *cb) 963 { 964 struct ib_device *dev; 965 unsigned int idx = 0; 966 int ret = 0; 967 968 down_read(&lists_rwsem); 969 list_for_each_entry(dev, &device_list, core_list) { 970 ret = nldev_cb(dev, skb, cb, idx); 971 if (ret) 972 break; 973 idx++; 974 } 975 976 up_read(&lists_rwsem); 977 return ret; 978 } 979 980 /** 981 * ib_query_pkey - Get P_Key table entry 982 * @device:Device to query 983 * @port_num:Port number to query 984 * @index:P_Key table index to query 985 * @pkey:Returned P_Key 986 * 987 * ib_query_pkey() fetches the specified P_Key table entry. 988 */ 989 int ib_query_pkey(struct ib_device *device, 990 u8 port_num, u16 index, u16 *pkey) 991 { 992 return device->query_pkey(device, port_num, index, pkey); 993 } 994 EXPORT_SYMBOL(ib_query_pkey); 995 996 /** 997 * ib_modify_device - Change IB device attributes 998 * @device:Device to modify 999 * @device_modify_mask:Mask of attributes to change 1000 * @device_modify:New attribute values 1001 * 1002 * ib_modify_device() changes a device's attributes as specified by 1003 * the @device_modify_mask and @device_modify structure. 1004 */ 1005 int ib_modify_device(struct ib_device *device, 1006 int device_modify_mask, 1007 struct ib_device_modify *device_modify) 1008 { 1009 if (!device->modify_device) 1010 return -ENOSYS; 1011 1012 return device->modify_device(device, device_modify_mask, 1013 device_modify); 1014 } 1015 EXPORT_SYMBOL(ib_modify_device); 1016 1017 /** 1018 * ib_modify_port - Modifies the attributes for the specified port. 1019 * @device: The device to modify. 1020 * @port_num: The number of the port to modify. 1021 * @port_modify_mask: Mask used to specify which attributes of the port 1022 * to change. 1023 * @port_modify: New attribute values for the port. 1024 * 1025 * ib_modify_port() changes a port's attributes as specified by the 1026 * @port_modify_mask and @port_modify structure. 1027 */ 1028 int ib_modify_port(struct ib_device *device, 1029 u8 port_num, int port_modify_mask, 1030 struct ib_port_modify *port_modify) 1031 { 1032 int rc; 1033 1034 if (!rdma_is_port_valid(device, port_num)) 1035 return -EINVAL; 1036 1037 if (device->modify_port) 1038 rc = device->modify_port(device, port_num, port_modify_mask, 1039 port_modify); 1040 else 1041 rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS; 1042 return rc; 1043 } 1044 EXPORT_SYMBOL(ib_modify_port); 1045 1046 /** 1047 * ib_find_gid - Returns the port number and GID table index where 1048 * a specified GID value occurs. Its searches only for IB link layer. 1049 * @device: The device to query. 1050 * @gid: The GID value to search for. 1051 * @ndev: The ndev related to the GID to search for. 1052 * @port_num: The port number of the device where the GID value was found. 1053 * @index: The index into the GID table where the GID was found. This 1054 * parameter may be NULL. 1055 */ 1056 int ib_find_gid(struct ib_device *device, union ib_gid *gid, 1057 struct net_device *ndev, u8 *port_num, u16 *index) 1058 { 1059 union ib_gid tmp_gid; 1060 int ret, port, i; 1061 1062 for (port = rdma_start_port(device); port <= rdma_end_port(device); ++port) { 1063 if (rdma_cap_roce_gid_table(device, port)) 1064 continue; 1065 1066 for (i = 0; i < device->port_immutable[port].gid_tbl_len; ++i) { 1067 ret = ib_query_gid(device, port, i, &tmp_gid, NULL); 1068 if (ret) 1069 return ret; 1070 if (!memcmp(&tmp_gid, gid, sizeof *gid)) { 1071 *port_num = port; 1072 if (index) 1073 *index = i; 1074 return 0; 1075 } 1076 } 1077 } 1078 1079 return -ENOENT; 1080 } 1081 EXPORT_SYMBOL(ib_find_gid); 1082 1083 /** 1084 * ib_find_pkey - Returns the PKey table index where a specified 1085 * PKey value occurs. 1086 * @device: The device to query. 1087 * @port_num: The port number of the device to search for the PKey. 1088 * @pkey: The PKey value to search for. 1089 * @index: The index into the PKey table where the PKey was found. 1090 */ 1091 int ib_find_pkey(struct ib_device *device, 1092 u8 port_num, u16 pkey, u16 *index) 1093 { 1094 int ret, i; 1095 u16 tmp_pkey; 1096 int partial_ix = -1; 1097 1098 for (i = 0; i < device->port_immutable[port_num].pkey_tbl_len; ++i) { 1099 ret = ib_query_pkey(device, port_num, i, &tmp_pkey); 1100 if (ret) 1101 return ret; 1102 if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) { 1103 /* if there is full-member pkey take it.*/ 1104 if (tmp_pkey & 0x8000) { 1105 *index = i; 1106 return 0; 1107 } 1108 if (partial_ix < 0) 1109 partial_ix = i; 1110 } 1111 } 1112 1113 /*no full-member, if exists take the limited*/ 1114 if (partial_ix >= 0) { 1115 *index = partial_ix; 1116 return 0; 1117 } 1118 return -ENOENT; 1119 } 1120 EXPORT_SYMBOL(ib_find_pkey); 1121 1122 /** 1123 * ib_get_net_dev_by_params() - Return the appropriate net_dev 1124 * for a received CM request 1125 * @dev: An RDMA device on which the request has been received. 1126 * @port: Port number on the RDMA device. 1127 * @pkey: The Pkey the request came on. 1128 * @gid: A GID that the net_dev uses to communicate. 1129 * @addr: Contains the IP address that the request specified as its 1130 * destination. 1131 */ 1132 struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, 1133 u8 port, 1134 u16 pkey, 1135 const union ib_gid *gid, 1136 const struct sockaddr *addr) 1137 { 1138 struct net_device *net_dev = NULL; 1139 struct ib_client_data *context; 1140 1141 if (!rdma_protocol_ib(dev, port)) 1142 return NULL; 1143 1144 down_read(&lists_rwsem); 1145 1146 list_for_each_entry(context, &dev->client_data_list, list) { 1147 struct ib_client *client = context->client; 1148 1149 if (context->going_down) 1150 continue; 1151 1152 if (client->get_net_dev_by_params) { 1153 net_dev = client->get_net_dev_by_params(dev, port, pkey, 1154 gid, addr, 1155 context->data); 1156 if (net_dev) 1157 break; 1158 } 1159 } 1160 1161 up_read(&lists_rwsem); 1162 1163 return net_dev; 1164 } 1165 EXPORT_SYMBOL(ib_get_net_dev_by_params); 1166 1167 static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = { 1168 [RDMA_NL_LS_OP_RESOLVE] = { 1169 .doit = ib_nl_handle_resolve_resp, 1170 .flags = RDMA_NL_ADMIN_PERM, 1171 }, 1172 [RDMA_NL_LS_OP_SET_TIMEOUT] = { 1173 .doit = ib_nl_handle_set_timeout, 1174 .flags = RDMA_NL_ADMIN_PERM, 1175 }, 1176 [RDMA_NL_LS_OP_IP_RESOLVE] = { 1177 .doit = ib_nl_handle_ip_res_resp, 1178 .flags = RDMA_NL_ADMIN_PERM, 1179 }, 1180 }; 1181 1182 static int __init ib_core_init(void) 1183 { 1184 int ret; 1185 1186 ib_wq = alloc_workqueue("infiniband", 0, 0); 1187 if (!ib_wq) 1188 return -ENOMEM; 1189 1190 ib_comp_wq = alloc_workqueue("ib-comp-wq", 1191 WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 1192 if (!ib_comp_wq) { 1193 ret = -ENOMEM; 1194 goto err; 1195 } 1196 1197 ret = class_register(&ib_class); 1198 if (ret) { 1199 pr_warn("Couldn't create InfiniBand device class\n"); 1200 goto err_comp; 1201 } 1202 1203 ret = rdma_nl_init(); 1204 if (ret) { 1205 pr_warn("Couldn't init IB netlink interface: err %d\n", ret); 1206 goto err_sysfs; 1207 } 1208 1209 ret = addr_init(); 1210 if (ret) { 1211 pr_warn("Could't init IB address resolution\n"); 1212 goto err_ibnl; 1213 } 1214 1215 ret = ib_mad_init(); 1216 if (ret) { 1217 pr_warn("Couldn't init IB MAD\n"); 1218 goto err_addr; 1219 } 1220 1221 ret = ib_sa_init(); 1222 if (ret) { 1223 pr_warn("Couldn't init SA\n"); 1224 goto err_mad; 1225 } 1226 1227 ret = register_lsm_notifier(&ibdev_lsm_nb); 1228 if (ret) { 1229 pr_warn("Couldn't register LSM notifier. ret %d\n", ret); 1230 goto err_sa; 1231 } 1232 1233 nldev_init(); 1234 rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table); 1235 ib_cache_setup(); 1236 1237 return 0; 1238 1239 err_sa: 1240 ib_sa_cleanup(); 1241 err_mad: 1242 ib_mad_cleanup(); 1243 err_addr: 1244 addr_cleanup(); 1245 err_ibnl: 1246 rdma_nl_exit(); 1247 err_sysfs: 1248 class_unregister(&ib_class); 1249 err_comp: 1250 destroy_workqueue(ib_comp_wq); 1251 err: 1252 destroy_workqueue(ib_wq); 1253 return ret; 1254 } 1255 1256 static void __exit ib_core_cleanup(void) 1257 { 1258 ib_cache_cleanup(); 1259 nldev_exit(); 1260 rdma_nl_unregister(RDMA_NL_LS); 1261 unregister_lsm_notifier(&ibdev_lsm_nb); 1262 ib_sa_cleanup(); 1263 ib_mad_cleanup(); 1264 addr_cleanup(); 1265 rdma_nl_exit(); 1266 class_unregister(&ib_class); 1267 destroy_workqueue(ib_comp_wq); 1268 /* Make sure that any pending umem accounting work is done. */ 1269 destroy_workqueue(ib_wq); 1270 } 1271 1272 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4); 1273 1274 subsys_initcall(ib_core_init); 1275 module_exit(ib_core_cleanup); 1276