11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Communications. All rights reserved. 32a1d9b7fSRoland Dreier * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * This software is available to you under a choice of one of two 61da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 71da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 81da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 91da177e4SLinus Torvalds * OpenIB.org BSD license below: 101da177e4SLinus Torvalds * 111da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 121da177e4SLinus Torvalds * without modification, are permitted provided that the following 131da177e4SLinus Torvalds * conditions are met: 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * - Redistributions of source code must retain the above 161da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 171da177e4SLinus Torvalds * disclaimer. 181da177e4SLinus Torvalds * 191da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 201da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 211da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 221da177e4SLinus Torvalds * provided with the distribution. 231da177e4SLinus Torvalds * 241da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 251da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 261da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 271da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 281da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 291da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 301da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 311da177e4SLinus Torvalds * SOFTWARE. 321da177e4SLinus Torvalds */ 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds #include <linux/module.h> 351da177e4SLinus Torvalds #include <linux/string.h> 361da177e4SLinus Torvalds #include <linux/errno.h> 379a6b090cSAhmed S. Darwish #include <linux/kernel.h> 381da177e4SLinus Torvalds #include <linux/slab.h> 391da177e4SLinus Torvalds #include <linux/init.h> 409268f72dSYotam Kenneth #include <linux/netdevice.h> 418f408ab6SDaniel Jurgens #include <linux/security.h> 428f408ab6SDaniel Jurgens #include <linux/notifier.h> 43324e227eSJason Gunthorpe #include <linux/hashtable.h> 44b2cbae2cSRoland Dreier #include <rdma/rdma_netlink.h> 4503db3a2dSMatan Barak #include <rdma/ib_addr.h> 4603db3a2dSMatan Barak #include <rdma/ib_cache.h> 471da177e4SLinus Torvalds 481da177e4SLinus Torvalds #include "core_priv.h" 4941eda65cSLeon Romanovsky #include "restrack.h" 501da177e4SLinus Torvalds 511da177e4SLinus Torvalds MODULE_AUTHOR("Roland Dreier"); 521da177e4SLinus Torvalds MODULE_DESCRIPTION("core kernel InfiniBand API"); 531da177e4SLinus Torvalds MODULE_LICENSE("Dual BSD/GPL"); 541da177e4SLinus Torvalds 5514d3a3b2SChristoph Hellwig struct workqueue_struct *ib_comp_wq; 56f794809aSJack Morgenstein struct workqueue_struct *ib_comp_unbound_wq; 57f0626710STejun Heo struct workqueue_struct *ib_wq; 58f0626710STejun Heo EXPORT_SYMBOL_GPL(ib_wq); 59f0626710STejun Heo 600df91bb6SJason Gunthorpe /* 61921eab11SJason Gunthorpe * Each of the three rwsem locks (devices, clients, client_data) protects the 62921eab11SJason Gunthorpe * xarray of the same name. Specifically it allows the caller to assert that 63921eab11SJason Gunthorpe * the MARK will/will not be changing under the lock, and for devices and 64921eab11SJason Gunthorpe * clients, that the value in the xarray is still a valid pointer. Change of 65921eab11SJason Gunthorpe * the MARK is linked to the object state, so holding the lock and testing the 66921eab11SJason Gunthorpe * MARK also asserts that the contained object is in a certain state. 67921eab11SJason Gunthorpe * 68921eab11SJason Gunthorpe * This is used to build a two stage register/unregister flow where objects 69921eab11SJason Gunthorpe * can continue to be in the xarray even though they are still in progress to 70921eab11SJason Gunthorpe * register/unregister. 71921eab11SJason Gunthorpe * 72921eab11SJason Gunthorpe * The xarray itself provides additional locking, and restartable iteration, 73921eab11SJason Gunthorpe * which is also relied on. 74921eab11SJason Gunthorpe * 75921eab11SJason Gunthorpe * Locks should not be nested, with the exception of client_data, which is 76921eab11SJason Gunthorpe * allowed to nest under the read side of the other two locks. 77921eab11SJason Gunthorpe * 78921eab11SJason Gunthorpe * The devices_rwsem also protects the device name list, any change or 79921eab11SJason Gunthorpe * assignment of device name must also hold the write side to guarantee unique 80921eab11SJason Gunthorpe * names. 81921eab11SJason Gunthorpe */ 82921eab11SJason Gunthorpe 83921eab11SJason Gunthorpe /* 840df91bb6SJason Gunthorpe * devices contains devices that have had their names assigned. The 850df91bb6SJason Gunthorpe * devices may not be registered. Users that care about the registration 860df91bb6SJason Gunthorpe * status need to call ib_device_try_get() on the device to ensure it is 870df91bb6SJason Gunthorpe * registered, and keep it registered, for the required duration. 880df91bb6SJason Gunthorpe * 890df91bb6SJason Gunthorpe */ 900df91bb6SJason Gunthorpe static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC); 91921eab11SJason Gunthorpe static DECLARE_RWSEM(devices_rwsem); 920df91bb6SJason Gunthorpe #define DEVICE_REGISTERED XA_MARK_1 930df91bb6SJason Gunthorpe 941da177e4SLinus Torvalds static LIST_HEAD(client_list); 95e59178d8SJason Gunthorpe #define CLIENT_REGISTERED XA_MARK_1 96e59178d8SJason Gunthorpe static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC); 97921eab11SJason Gunthorpe static DECLARE_RWSEM(clients_rwsem); 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds /* 1000df91bb6SJason Gunthorpe * If client_data is registered then the corresponding client must also still 1010df91bb6SJason Gunthorpe * be registered. 1020df91bb6SJason Gunthorpe */ 1030df91bb6SJason Gunthorpe #define CLIENT_DATA_REGISTERED XA_MARK_1 1040df91bb6SJason Gunthorpe /* 1050df91bb6SJason Gunthorpe * xarray has this behavior where it won't iterate over NULL values stored in 1060df91bb6SJason Gunthorpe * allocated arrays. So we need our own iterator to see all values stored in 1070df91bb6SJason Gunthorpe * the array. This does the same thing as xa_for_each except that it also 1080df91bb6SJason Gunthorpe * returns NULL valued entries if the array is allocating. Simplified to only 1090df91bb6SJason Gunthorpe * work on simple xarrays. 1100df91bb6SJason Gunthorpe */ 1110df91bb6SJason Gunthorpe static void *xan_find_marked(struct xarray *xa, unsigned long *indexp, 1120df91bb6SJason Gunthorpe xa_mark_t filter) 1130df91bb6SJason Gunthorpe { 1140df91bb6SJason Gunthorpe XA_STATE(xas, xa, *indexp); 1150df91bb6SJason Gunthorpe void *entry; 1160df91bb6SJason Gunthorpe 1170df91bb6SJason Gunthorpe rcu_read_lock(); 1180df91bb6SJason Gunthorpe do { 1190df91bb6SJason Gunthorpe entry = xas_find_marked(&xas, ULONG_MAX, filter); 1200df91bb6SJason Gunthorpe if (xa_is_zero(entry)) 1210df91bb6SJason Gunthorpe break; 1220df91bb6SJason Gunthorpe } while (xas_retry(&xas, entry)); 1230df91bb6SJason Gunthorpe rcu_read_unlock(); 1240df91bb6SJason Gunthorpe 1250df91bb6SJason Gunthorpe if (entry) { 1260df91bb6SJason Gunthorpe *indexp = xas.xa_index; 1270df91bb6SJason Gunthorpe if (xa_is_zero(entry)) 1280df91bb6SJason Gunthorpe return NULL; 1290df91bb6SJason Gunthorpe return entry; 1300df91bb6SJason Gunthorpe } 1310df91bb6SJason Gunthorpe return XA_ERROR(-ENOENT); 1320df91bb6SJason Gunthorpe } 1330df91bb6SJason Gunthorpe #define xan_for_each_marked(xa, index, entry, filter) \ 1340df91bb6SJason Gunthorpe for (index = 0, entry = xan_find_marked(xa, &(index), filter); \ 1350df91bb6SJason Gunthorpe !xa_is_err(entry); \ 1360df91bb6SJason Gunthorpe (index)++, entry = xan_find_marked(xa, &(index), filter)) 1370df91bb6SJason Gunthorpe 138324e227eSJason Gunthorpe /* RCU hash table mapping netdevice pointers to struct ib_port_data */ 139324e227eSJason Gunthorpe static DEFINE_SPINLOCK(ndev_hash_lock); 140324e227eSJason Gunthorpe static DECLARE_HASHTABLE(ndev_hash, 5); 141324e227eSJason Gunthorpe 142c2261dd7SJason Gunthorpe static void free_netdevs(struct ib_device *ib_dev); 143d0899892SJason Gunthorpe static void ib_unregister_work(struct work_struct *work); 144d0899892SJason Gunthorpe static void __ib_unregister_device(struct ib_device *device); 1458f408ab6SDaniel Jurgens static int ib_security_change(struct notifier_block *nb, unsigned long event, 1468f408ab6SDaniel Jurgens void *lsm_data); 1478f408ab6SDaniel Jurgens static void ib_policy_change_task(struct work_struct *work); 1488f408ab6SDaniel Jurgens static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task); 1498f408ab6SDaniel Jurgens 1508f408ab6SDaniel Jurgens static struct notifier_block ibdev_lsm_nb = { 1518f408ab6SDaniel Jurgens .notifier_call = ib_security_change, 1528f408ab6SDaniel Jurgens }; 1531da177e4SLinus Torvalds 154324e227eSJason Gunthorpe /* Pointer to the RCU head at the start of the ib_port_data array */ 155324e227eSJason Gunthorpe struct ib_port_data_rcu { 156324e227eSJason Gunthorpe struct rcu_head rcu_head; 157324e227eSJason Gunthorpe struct ib_port_data pdata[]; 158324e227eSJason Gunthorpe }; 159324e227eSJason Gunthorpe 1601da177e4SLinus Torvalds static int ib_device_check_mandatory(struct ib_device *device) 1611da177e4SLinus Torvalds { 1623023a1e9SKamal Heib #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x } 1631da177e4SLinus Torvalds static const struct { 1641da177e4SLinus Torvalds size_t offset; 1651da177e4SLinus Torvalds char *name; 1661da177e4SLinus Torvalds } mandatory_table[] = { 1671da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_device), 1681da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_port), 1691da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_pkey), 1701da177e4SLinus Torvalds IB_MANDATORY_FUNC(alloc_pd), 1711da177e4SLinus Torvalds IB_MANDATORY_FUNC(dealloc_pd), 1721da177e4SLinus Torvalds IB_MANDATORY_FUNC(create_qp), 1731da177e4SLinus Torvalds IB_MANDATORY_FUNC(modify_qp), 1741da177e4SLinus Torvalds IB_MANDATORY_FUNC(destroy_qp), 1751da177e4SLinus Torvalds IB_MANDATORY_FUNC(post_send), 1761da177e4SLinus Torvalds IB_MANDATORY_FUNC(post_recv), 1771da177e4SLinus Torvalds IB_MANDATORY_FUNC(create_cq), 1781da177e4SLinus Torvalds IB_MANDATORY_FUNC(destroy_cq), 1791da177e4SLinus Torvalds IB_MANDATORY_FUNC(poll_cq), 1801da177e4SLinus Torvalds IB_MANDATORY_FUNC(req_notify_cq), 1811da177e4SLinus Torvalds IB_MANDATORY_FUNC(get_dma_mr), 1827738613eSIra Weiny IB_MANDATORY_FUNC(dereg_mr), 1837738613eSIra Weiny IB_MANDATORY_FUNC(get_port_immutable) 1841da177e4SLinus Torvalds }; 1851da177e4SLinus Torvalds int i; 1861da177e4SLinus Torvalds 1876780c4faSGal Pressman device->kverbs_provider = true; 1889a6b090cSAhmed S. Darwish for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { 1893023a1e9SKamal Heib if (!*(void **) ((void *) &device->ops + 1903023a1e9SKamal Heib mandatory_table[i].offset)) { 1916780c4faSGal Pressman device->kverbs_provider = false; 1926780c4faSGal Pressman break; 1931da177e4SLinus Torvalds } 1941da177e4SLinus Torvalds } 1951da177e4SLinus Torvalds 1961da177e4SLinus Torvalds return 0; 1971da177e4SLinus Torvalds } 1981da177e4SLinus Torvalds 199f8978bd9SLeon Romanovsky /* 20001b67117SParav Pandit * Caller must perform ib_device_put() to return the device reference count 20101b67117SParav Pandit * when ib_device_get_by_index() returns valid device pointer. 202f8978bd9SLeon Romanovsky */ 203f8978bd9SLeon Romanovsky struct ib_device *ib_device_get_by_index(u32 index) 204f8978bd9SLeon Romanovsky { 205f8978bd9SLeon Romanovsky struct ib_device *device; 206f8978bd9SLeon Romanovsky 207921eab11SJason Gunthorpe down_read(&devices_rwsem); 2080df91bb6SJason Gunthorpe device = xa_load(&devices, index); 20901b67117SParav Pandit if (device) { 210d79af724SJason Gunthorpe if (!ib_device_try_get(device)) 21101b67117SParav Pandit device = NULL; 21201b67117SParav Pandit } 213921eab11SJason Gunthorpe up_read(&devices_rwsem); 214f8978bd9SLeon Romanovsky return device; 215f8978bd9SLeon Romanovsky } 216f8978bd9SLeon Romanovsky 217d79af724SJason Gunthorpe /** 218d79af724SJason Gunthorpe * ib_device_put - Release IB device reference 219d79af724SJason Gunthorpe * @device: device whose reference to be released 220d79af724SJason Gunthorpe * 221d79af724SJason Gunthorpe * ib_device_put() releases reference to the IB device to allow it to be 222d79af724SJason Gunthorpe * unregistered and eventually free. 223d79af724SJason Gunthorpe */ 22401b67117SParav Pandit void ib_device_put(struct ib_device *device) 22501b67117SParav Pandit { 22601b67117SParav Pandit if (refcount_dec_and_test(&device->refcount)) 22701b67117SParav Pandit complete(&device->unreg_completion); 22801b67117SParav Pandit } 229d79af724SJason Gunthorpe EXPORT_SYMBOL(ib_device_put); 23001b67117SParav Pandit 2311da177e4SLinus Torvalds static struct ib_device *__ib_device_get_by_name(const char *name) 2321da177e4SLinus Torvalds { 2331da177e4SLinus Torvalds struct ib_device *device; 2340df91bb6SJason Gunthorpe unsigned long index; 2351da177e4SLinus Torvalds 2360df91bb6SJason Gunthorpe xa_for_each (&devices, index, device) 237896de009SJason Gunthorpe if (!strcmp(name, dev_name(&device->dev))) 2381da177e4SLinus Torvalds return device; 2391da177e4SLinus Torvalds 2401da177e4SLinus Torvalds return NULL; 2411da177e4SLinus Torvalds } 2421da177e4SLinus Torvalds 2436cc2c8e5SJason Gunthorpe /** 2446cc2c8e5SJason Gunthorpe * ib_device_get_by_name - Find an IB device by name 2456cc2c8e5SJason Gunthorpe * @name: The name to look for 2466cc2c8e5SJason Gunthorpe * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 2476cc2c8e5SJason Gunthorpe * 2486cc2c8e5SJason Gunthorpe * Find and hold an ib_device by its name. The caller must call 2496cc2c8e5SJason Gunthorpe * ib_device_put() on the returned pointer. 2506cc2c8e5SJason Gunthorpe */ 2516cc2c8e5SJason Gunthorpe struct ib_device *ib_device_get_by_name(const char *name, 2526cc2c8e5SJason Gunthorpe enum rdma_driver_id driver_id) 2536cc2c8e5SJason Gunthorpe { 2546cc2c8e5SJason Gunthorpe struct ib_device *device; 2556cc2c8e5SJason Gunthorpe 2566cc2c8e5SJason Gunthorpe down_read(&devices_rwsem); 2576cc2c8e5SJason Gunthorpe device = __ib_device_get_by_name(name); 2586cc2c8e5SJason Gunthorpe if (device && driver_id != RDMA_DRIVER_UNKNOWN && 2596cc2c8e5SJason Gunthorpe device->driver_id != driver_id) 2606cc2c8e5SJason Gunthorpe device = NULL; 2616cc2c8e5SJason Gunthorpe 2626cc2c8e5SJason Gunthorpe if (device) { 2636cc2c8e5SJason Gunthorpe if (!ib_device_try_get(device)) 2646cc2c8e5SJason Gunthorpe device = NULL; 2656cc2c8e5SJason Gunthorpe } 2666cc2c8e5SJason Gunthorpe up_read(&devices_rwsem); 2676cc2c8e5SJason Gunthorpe return device; 2686cc2c8e5SJason Gunthorpe } 2696cc2c8e5SJason Gunthorpe EXPORT_SYMBOL(ib_device_get_by_name); 2706cc2c8e5SJason Gunthorpe 271d21943ddSLeon Romanovsky int ib_device_rename(struct ib_device *ibdev, const char *name) 272d21943ddSLeon Romanovsky { 273e3593b56SJason Gunthorpe int ret; 274d21943ddSLeon Romanovsky 275921eab11SJason Gunthorpe down_write(&devices_rwsem); 276e3593b56SJason Gunthorpe if (!strcmp(name, dev_name(&ibdev->dev))) { 277e3593b56SJason Gunthorpe ret = 0; 278e3593b56SJason Gunthorpe goto out; 279e3593b56SJason Gunthorpe } 280e3593b56SJason Gunthorpe 281344684e6SJason Gunthorpe if (__ib_device_get_by_name(name)) { 282d21943ddSLeon Romanovsky ret = -EEXIST; 283d21943ddSLeon Romanovsky goto out; 284d21943ddSLeon Romanovsky } 285d21943ddSLeon Romanovsky 286d21943ddSLeon Romanovsky ret = device_rename(&ibdev->dev, name); 287d21943ddSLeon Romanovsky if (ret) 288d21943ddSLeon Romanovsky goto out; 289d21943ddSLeon Romanovsky strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); 290d21943ddSLeon Romanovsky out: 291921eab11SJason Gunthorpe up_write(&devices_rwsem); 292d21943ddSLeon Romanovsky return ret; 293d21943ddSLeon Romanovsky } 294d21943ddSLeon Romanovsky 295e349f858SJason Gunthorpe static int alloc_name(struct ib_device *ibdev, const char *name) 2961da177e4SLinus Torvalds { 2971da177e4SLinus Torvalds struct ib_device *device; 2980df91bb6SJason Gunthorpe unsigned long index; 2993b88afd3SJason Gunthorpe struct ida inuse; 3003b88afd3SJason Gunthorpe int rc; 3011da177e4SLinus Torvalds int i; 3021da177e4SLinus Torvalds 303921eab11SJason Gunthorpe lockdep_assert_held_exclusive(&devices_rwsem); 3043b88afd3SJason Gunthorpe ida_init(&inuse); 3050df91bb6SJason Gunthorpe xa_for_each (&devices, index, device) { 306e349f858SJason Gunthorpe char buf[IB_DEVICE_NAME_MAX]; 307e349f858SJason Gunthorpe 308896de009SJason Gunthorpe if (sscanf(dev_name(&device->dev), name, &i) != 1) 3091da177e4SLinus Torvalds continue; 3103b88afd3SJason Gunthorpe if (i < 0 || i >= INT_MAX) 3111da177e4SLinus Torvalds continue; 3121da177e4SLinus Torvalds snprintf(buf, sizeof buf, name, i); 3133b88afd3SJason Gunthorpe if (strcmp(buf, dev_name(&device->dev)) != 0) 3143b88afd3SJason Gunthorpe continue; 3153b88afd3SJason Gunthorpe 3163b88afd3SJason Gunthorpe rc = ida_alloc_range(&inuse, i, i, GFP_KERNEL); 3173b88afd3SJason Gunthorpe if (rc < 0) 3183b88afd3SJason Gunthorpe goto out; 3191da177e4SLinus Torvalds } 3201da177e4SLinus Torvalds 3213b88afd3SJason Gunthorpe rc = ida_alloc(&inuse, GFP_KERNEL); 3223b88afd3SJason Gunthorpe if (rc < 0) 3233b88afd3SJason Gunthorpe goto out; 3241da177e4SLinus Torvalds 3253b88afd3SJason Gunthorpe rc = dev_set_name(&ibdev->dev, name, rc); 3263b88afd3SJason Gunthorpe out: 3273b88afd3SJason Gunthorpe ida_destroy(&inuse); 3283b88afd3SJason Gunthorpe return rc; 3291da177e4SLinus Torvalds } 3301da177e4SLinus Torvalds 33155aeed06SJason Gunthorpe static void ib_device_release(struct device *device) 33255aeed06SJason Gunthorpe { 33355aeed06SJason Gunthorpe struct ib_device *dev = container_of(device, struct ib_device, dev); 33455aeed06SJason Gunthorpe 335c2261dd7SJason Gunthorpe free_netdevs(dev); 336652432f3SJason Gunthorpe WARN_ON(refcount_read(&dev->refcount)); 33703db3a2dSMatan Barak ib_cache_release_one(dev); 338b34b269aSJason Gunthorpe ib_security_release_port_pkey_list(dev); 3390df91bb6SJason Gunthorpe xa_destroy(&dev->client_data); 340324e227eSJason Gunthorpe if (dev->port_data) 341324e227eSJason Gunthorpe kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu, 342324e227eSJason Gunthorpe pdata[0]), 343324e227eSJason Gunthorpe rcu_head); 344324e227eSJason Gunthorpe kfree_rcu(dev, rcu_head); 34555aeed06SJason Gunthorpe } 34655aeed06SJason Gunthorpe 34755aeed06SJason Gunthorpe static int ib_device_uevent(struct device *device, 34855aeed06SJason Gunthorpe struct kobj_uevent_env *env) 34955aeed06SJason Gunthorpe { 350896de009SJason Gunthorpe if (add_uevent_var(env, "NAME=%s", dev_name(device))) 35155aeed06SJason Gunthorpe return -ENOMEM; 35255aeed06SJason Gunthorpe 35355aeed06SJason Gunthorpe /* 35455aeed06SJason Gunthorpe * It would be nice to pass the node GUID with the event... 35555aeed06SJason Gunthorpe */ 35655aeed06SJason Gunthorpe 35755aeed06SJason Gunthorpe return 0; 35855aeed06SJason Gunthorpe } 35955aeed06SJason Gunthorpe 36055aeed06SJason Gunthorpe static struct class ib_class = { 36155aeed06SJason Gunthorpe .name = "infiniband", 36255aeed06SJason Gunthorpe .dev_release = ib_device_release, 36355aeed06SJason Gunthorpe .dev_uevent = ib_device_uevent, 36455aeed06SJason Gunthorpe }; 36555aeed06SJason Gunthorpe 3661da177e4SLinus Torvalds /** 367459cc69fSLeon Romanovsky * _ib_alloc_device - allocate an IB device struct 3681da177e4SLinus Torvalds * @size:size of structure to allocate 3691da177e4SLinus Torvalds * 3701da177e4SLinus Torvalds * Low-level drivers should use ib_alloc_device() to allocate &struct 3711da177e4SLinus Torvalds * ib_device. @size is the size of the structure to be allocated, 3721da177e4SLinus Torvalds * including any private data used by the low-level driver. 3731da177e4SLinus Torvalds * ib_dealloc_device() must be used to free structures allocated with 3741da177e4SLinus Torvalds * ib_alloc_device(). 3751da177e4SLinus Torvalds */ 376459cc69fSLeon Romanovsky struct ib_device *_ib_alloc_device(size_t size) 3771da177e4SLinus Torvalds { 37855aeed06SJason Gunthorpe struct ib_device *device; 3791da177e4SLinus Torvalds 38055aeed06SJason Gunthorpe if (WARN_ON(size < sizeof(struct ib_device))) 38155aeed06SJason Gunthorpe return NULL; 38255aeed06SJason Gunthorpe 38355aeed06SJason Gunthorpe device = kzalloc(size, GFP_KERNEL); 38455aeed06SJason Gunthorpe if (!device) 38555aeed06SJason Gunthorpe return NULL; 38655aeed06SJason Gunthorpe 38741eda65cSLeon Romanovsky if (rdma_restrack_init(device)) { 38841eda65cSLeon Romanovsky kfree(device); 38941eda65cSLeon Romanovsky return NULL; 39041eda65cSLeon Romanovsky } 39102d8883fSLeon Romanovsky 39255aeed06SJason Gunthorpe device->dev.class = &ib_class; 3935f8f5499SParav Pandit device->groups[0] = &ib_dev_attr_group; 3945f8f5499SParav Pandit device->dev.groups = device->groups; 39555aeed06SJason Gunthorpe device_initialize(&device->dev); 39655aeed06SJason Gunthorpe 39755aeed06SJason Gunthorpe INIT_LIST_HEAD(&device->event_handler_list); 39855aeed06SJason Gunthorpe spin_lock_init(&device->event_handler_lock); 399d0899892SJason Gunthorpe mutex_init(&device->unregistration_lock); 4000df91bb6SJason Gunthorpe /* 4010df91bb6SJason Gunthorpe * client_data needs to be alloc because we don't want our mark to be 4020df91bb6SJason Gunthorpe * destroyed if the user stores NULL in the client data. 4030df91bb6SJason Gunthorpe */ 4040df91bb6SJason Gunthorpe xa_init_flags(&device->client_data, XA_FLAGS_ALLOC); 405921eab11SJason Gunthorpe init_rwsem(&device->client_data_rwsem); 40655aeed06SJason Gunthorpe INIT_LIST_HEAD(&device->port_list); 40701b67117SParav Pandit init_completion(&device->unreg_completion); 408d0899892SJason Gunthorpe INIT_WORK(&device->unregistration_work, ib_unregister_work); 40955aeed06SJason Gunthorpe 41055aeed06SJason Gunthorpe return device; 4111da177e4SLinus Torvalds } 412459cc69fSLeon Romanovsky EXPORT_SYMBOL(_ib_alloc_device); 4131da177e4SLinus Torvalds 4141da177e4SLinus Torvalds /** 4151da177e4SLinus Torvalds * ib_dealloc_device - free an IB device struct 4161da177e4SLinus Torvalds * @device:structure to free 4171da177e4SLinus Torvalds * 4181da177e4SLinus Torvalds * Free a structure allocated with ib_alloc_device(). 4191da177e4SLinus Torvalds */ 4201da177e4SLinus Torvalds void ib_dealloc_device(struct ib_device *device) 4211da177e4SLinus Torvalds { 422d0899892SJason Gunthorpe if (device->ops.dealloc_driver) 423d0899892SJason Gunthorpe device->ops.dealloc_driver(device); 424d0899892SJason Gunthorpe 425d0899892SJason Gunthorpe /* 426d0899892SJason Gunthorpe * ib_unregister_driver() requires all devices to remain in the xarray 427d0899892SJason Gunthorpe * while their ops are callable. The last op we call is dealloc_driver 428d0899892SJason Gunthorpe * above. This is needed to create a fence on op callbacks prior to 429d0899892SJason Gunthorpe * allowing the driver module to unload. 430d0899892SJason Gunthorpe */ 431d0899892SJason Gunthorpe down_write(&devices_rwsem); 432d0899892SJason Gunthorpe if (xa_load(&devices, device->index) == device) 433d0899892SJason Gunthorpe xa_erase(&devices, device->index); 434d0899892SJason Gunthorpe up_write(&devices_rwsem); 435d0899892SJason Gunthorpe 436c2261dd7SJason Gunthorpe /* Expedite releasing netdev references */ 437c2261dd7SJason Gunthorpe free_netdevs(device); 438c2261dd7SJason Gunthorpe 4390df91bb6SJason Gunthorpe WARN_ON(!xa_empty(&device->client_data)); 440652432f3SJason Gunthorpe WARN_ON(refcount_read(&device->refcount)); 4410ad699c0SLeon Romanovsky rdma_restrack_clean(device); 442e155755eSParav Pandit /* Balances with device_initialize */ 443924b8900SLeon Romanovsky put_device(&device->dev); 4441da177e4SLinus Torvalds } 4451da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_device); 4461da177e4SLinus Torvalds 447921eab11SJason Gunthorpe /* 448921eab11SJason Gunthorpe * add_client_context() and remove_client_context() must be safe against 449921eab11SJason Gunthorpe * parallel calls on the same device - registration/unregistration of both the 450921eab11SJason Gunthorpe * device and client can be occurring in parallel. 451921eab11SJason Gunthorpe * 452921eab11SJason Gunthorpe * The routines need to be a fence, any caller must not return until the add 453921eab11SJason Gunthorpe * or remove is fully completed. 454921eab11SJason Gunthorpe */ 455921eab11SJason Gunthorpe static int add_client_context(struct ib_device *device, 456921eab11SJason Gunthorpe struct ib_client *client) 4571da177e4SLinus Torvalds { 458921eab11SJason Gunthorpe int ret = 0; 4591da177e4SLinus Torvalds 4606780c4faSGal Pressman if (!device->kverbs_provider && !client->no_kverbs_req) 461921eab11SJason Gunthorpe return 0; 4626780c4faSGal Pressman 463921eab11SJason Gunthorpe down_write(&device->client_data_rwsem); 464921eab11SJason Gunthorpe /* 465921eab11SJason Gunthorpe * Another caller to add_client_context got here first and has already 466921eab11SJason Gunthorpe * completely initialized context. 467921eab11SJason Gunthorpe */ 468921eab11SJason Gunthorpe if (xa_get_mark(&device->client_data, client->client_id, 469921eab11SJason Gunthorpe CLIENT_DATA_REGISTERED)) 470921eab11SJason Gunthorpe goto out; 471921eab11SJason Gunthorpe 472921eab11SJason Gunthorpe ret = xa_err(xa_store(&device->client_data, client->client_id, NULL, 473921eab11SJason Gunthorpe GFP_KERNEL)); 474921eab11SJason Gunthorpe if (ret) 475921eab11SJason Gunthorpe goto out; 476921eab11SJason Gunthorpe downgrade_write(&device->client_data_rwsem); 477921eab11SJason Gunthorpe if (client->add) 478921eab11SJason Gunthorpe client->add(device); 479921eab11SJason Gunthorpe 480921eab11SJason Gunthorpe /* Readers shall not see a client until add has been completed */ 4810df91bb6SJason Gunthorpe xa_set_mark(&device->client_data, client->client_id, 4820df91bb6SJason Gunthorpe CLIENT_DATA_REGISTERED); 483921eab11SJason Gunthorpe up_read(&device->client_data_rwsem); 484921eab11SJason Gunthorpe return 0; 4851da177e4SLinus Torvalds 486921eab11SJason Gunthorpe out: 487921eab11SJason Gunthorpe up_write(&device->client_data_rwsem); 488921eab11SJason Gunthorpe return ret; 489921eab11SJason Gunthorpe } 490921eab11SJason Gunthorpe 491921eab11SJason Gunthorpe static void remove_client_context(struct ib_device *device, 492921eab11SJason Gunthorpe unsigned int client_id) 493921eab11SJason Gunthorpe { 494921eab11SJason Gunthorpe struct ib_client *client; 495921eab11SJason Gunthorpe void *client_data; 496921eab11SJason Gunthorpe 497921eab11SJason Gunthorpe down_write(&device->client_data_rwsem); 498921eab11SJason Gunthorpe if (!xa_get_mark(&device->client_data, client_id, 499921eab11SJason Gunthorpe CLIENT_DATA_REGISTERED)) { 500921eab11SJason Gunthorpe up_write(&device->client_data_rwsem); 501921eab11SJason Gunthorpe return; 502921eab11SJason Gunthorpe } 503921eab11SJason Gunthorpe client_data = xa_load(&device->client_data, client_id); 504921eab11SJason Gunthorpe xa_clear_mark(&device->client_data, client_id, CLIENT_DATA_REGISTERED); 505921eab11SJason Gunthorpe client = xa_load(&clients, client_id); 506921eab11SJason Gunthorpe downgrade_write(&device->client_data_rwsem); 507921eab11SJason Gunthorpe 508921eab11SJason Gunthorpe /* 509921eab11SJason Gunthorpe * Notice we cannot be holding any exclusive locks when calling the 510921eab11SJason Gunthorpe * remove callback as the remove callback can recurse back into any 511921eab11SJason Gunthorpe * public functions in this module and thus try for any locks those 512921eab11SJason Gunthorpe * functions take. 513921eab11SJason Gunthorpe * 514921eab11SJason Gunthorpe * For this reason clients and drivers should not call the 515921eab11SJason Gunthorpe * unregistration functions will holdling any locks. 516921eab11SJason Gunthorpe * 517921eab11SJason Gunthorpe * It tempting to drop the client_data_rwsem too, but this is required 518921eab11SJason Gunthorpe * to ensure that unregister_client does not return until all clients 519921eab11SJason Gunthorpe * are completely unregistered, which is required to avoid module 520921eab11SJason Gunthorpe * unloading races. 521921eab11SJason Gunthorpe */ 522921eab11SJason Gunthorpe if (client->remove) 523921eab11SJason Gunthorpe client->remove(device, client_data); 524921eab11SJason Gunthorpe 525921eab11SJason Gunthorpe xa_erase(&device->client_data, client_id); 526921eab11SJason Gunthorpe up_read(&device->client_data_rwsem); 5271da177e4SLinus Torvalds } 5281da177e4SLinus Torvalds 529c2261dd7SJason Gunthorpe static int alloc_port_data(struct ib_device *device) 5305eb620c8SYosef Etigin { 531324e227eSJason Gunthorpe struct ib_port_data_rcu *pdata_rcu; 532ea1075edSJason Gunthorpe unsigned int port; 533c2261dd7SJason Gunthorpe 534c2261dd7SJason Gunthorpe if (device->port_data) 535c2261dd7SJason Gunthorpe return 0; 536c2261dd7SJason Gunthorpe 537c2261dd7SJason Gunthorpe /* This can only be called once the physical port range is defined */ 538c2261dd7SJason Gunthorpe if (WARN_ON(!device->phys_port_cnt)) 539c2261dd7SJason Gunthorpe return -EINVAL; 5405eb620c8SYosef Etigin 5418ceb1357SJason Gunthorpe /* 5428ceb1357SJason Gunthorpe * device->port_data is indexed directly by the port number to make 5437738613eSIra Weiny * access to this data as efficient as possible. 5447738613eSIra Weiny * 5458ceb1357SJason Gunthorpe * Therefore port_data is declared as a 1 based array with potential 5468ceb1357SJason Gunthorpe * empty slots at the beginning. 5477738613eSIra Weiny */ 548324e227eSJason Gunthorpe pdata_rcu = kzalloc(struct_size(pdata_rcu, pdata, 549324e227eSJason Gunthorpe rdma_end_port(device) + 1), 550324e227eSJason Gunthorpe GFP_KERNEL); 551324e227eSJason Gunthorpe if (!pdata_rcu) 55255aeed06SJason Gunthorpe return -ENOMEM; 553324e227eSJason Gunthorpe /* 554324e227eSJason Gunthorpe * The rcu_head is put in front of the port data array and the stored 555324e227eSJason Gunthorpe * pointer is adjusted since we never need to see that member until 556324e227eSJason Gunthorpe * kfree_rcu. 557324e227eSJason Gunthorpe */ 558324e227eSJason Gunthorpe device->port_data = pdata_rcu->pdata; 5595eb620c8SYosef Etigin 560ea1075edSJason Gunthorpe rdma_for_each_port (device, port) { 5618ceb1357SJason Gunthorpe struct ib_port_data *pdata = &device->port_data[port]; 5628ceb1357SJason Gunthorpe 563324e227eSJason Gunthorpe pdata->ib_dev = device; 5648ceb1357SJason Gunthorpe spin_lock_init(&pdata->pkey_list_lock); 5658ceb1357SJason Gunthorpe INIT_LIST_HEAD(&pdata->pkey_list); 566c2261dd7SJason Gunthorpe spin_lock_init(&pdata->netdev_lock); 567324e227eSJason Gunthorpe INIT_HLIST_NODE(&pdata->ndev_hash_link); 568c2261dd7SJason Gunthorpe } 569c2261dd7SJason Gunthorpe return 0; 570c2261dd7SJason Gunthorpe } 571c2261dd7SJason Gunthorpe 572c2261dd7SJason Gunthorpe static int verify_immutable(const struct ib_device *dev, u8 port) 573c2261dd7SJason Gunthorpe { 574c2261dd7SJason Gunthorpe return WARN_ON(!rdma_cap_ib_mad(dev, port) && 575c2261dd7SJason Gunthorpe rdma_max_mad_size(dev, port) != 0); 576c2261dd7SJason Gunthorpe } 577c2261dd7SJason Gunthorpe 578c2261dd7SJason Gunthorpe static int setup_port_data(struct ib_device *device) 579c2261dd7SJason Gunthorpe { 580c2261dd7SJason Gunthorpe unsigned int port; 581c2261dd7SJason Gunthorpe int ret; 582c2261dd7SJason Gunthorpe 583c2261dd7SJason Gunthorpe ret = alloc_port_data(device); 584c2261dd7SJason Gunthorpe if (ret) 585c2261dd7SJason Gunthorpe return ret; 586c2261dd7SJason Gunthorpe 587c2261dd7SJason Gunthorpe rdma_for_each_port (device, port) { 588c2261dd7SJason Gunthorpe struct ib_port_data *pdata = &device->port_data[port]; 5898ceb1357SJason Gunthorpe 5908ceb1357SJason Gunthorpe ret = device->ops.get_port_immutable(device, port, 5918ceb1357SJason Gunthorpe &pdata->immutable); 5925eb620c8SYosef Etigin if (ret) 5935eb620c8SYosef Etigin return ret; 59455aeed06SJason Gunthorpe 59555aeed06SJason Gunthorpe if (verify_immutable(device, port)) 59655aeed06SJason Gunthorpe return -EINVAL; 59755aeed06SJason Gunthorpe } 59855aeed06SJason Gunthorpe return 0; 5995eb620c8SYosef Etigin } 6005eb620c8SYosef Etigin 6019abb0d1bSLeon Romanovsky void ib_get_device_fw_str(struct ib_device *dev, char *str) 6025fa76c20SIra Weiny { 6033023a1e9SKamal Heib if (dev->ops.get_dev_fw_str) 6043023a1e9SKamal Heib dev->ops.get_dev_fw_str(dev, str); 6055fa76c20SIra Weiny else 6065fa76c20SIra Weiny str[0] = '\0'; 6075fa76c20SIra Weiny } 6085fa76c20SIra Weiny EXPORT_SYMBOL(ib_get_device_fw_str); 6095fa76c20SIra Weiny 6108f408ab6SDaniel Jurgens static void ib_policy_change_task(struct work_struct *work) 6118f408ab6SDaniel Jurgens { 6128f408ab6SDaniel Jurgens struct ib_device *dev; 6130df91bb6SJason Gunthorpe unsigned long index; 6148f408ab6SDaniel Jurgens 615921eab11SJason Gunthorpe down_read(&devices_rwsem); 6160df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 617ea1075edSJason Gunthorpe unsigned int i; 6188f408ab6SDaniel Jurgens 619ea1075edSJason Gunthorpe rdma_for_each_port (dev, i) { 6208f408ab6SDaniel Jurgens u64 sp; 6218f408ab6SDaniel Jurgens int ret = ib_get_cached_subnet_prefix(dev, 6228f408ab6SDaniel Jurgens i, 6238f408ab6SDaniel Jurgens &sp); 6248f408ab6SDaniel Jurgens 6258f408ab6SDaniel Jurgens WARN_ONCE(ret, 6268f408ab6SDaniel Jurgens "ib_get_cached_subnet_prefix err: %d, this should never happen here\n", 6278f408ab6SDaniel Jurgens ret); 628a750cfdeSDaniel Jurgens if (!ret) 6298f408ab6SDaniel Jurgens ib_security_cache_change(dev, i, sp); 6308f408ab6SDaniel Jurgens } 6318f408ab6SDaniel Jurgens } 632921eab11SJason Gunthorpe up_read(&devices_rwsem); 6338f408ab6SDaniel Jurgens } 6348f408ab6SDaniel Jurgens 6358f408ab6SDaniel Jurgens static int ib_security_change(struct notifier_block *nb, unsigned long event, 6368f408ab6SDaniel Jurgens void *lsm_data) 6378f408ab6SDaniel Jurgens { 6388f408ab6SDaniel Jurgens if (event != LSM_POLICY_CHANGE) 6398f408ab6SDaniel Jurgens return NOTIFY_DONE; 6408f408ab6SDaniel Jurgens 6418f408ab6SDaniel Jurgens schedule_work(&ib_policy_change_work); 642c66f6741SDaniel Jurgens ib_mad_agent_security_change(); 6438f408ab6SDaniel Jurgens 6448f408ab6SDaniel Jurgens return NOTIFY_OK; 6458f408ab6SDaniel Jurgens } 6468f408ab6SDaniel Jurgens 647ecc82c53SLeon Romanovsky /* 648d0899892SJason Gunthorpe * Assign the unique string device name and the unique device index. This is 649d0899892SJason Gunthorpe * undone by ib_dealloc_device. 650ecc82c53SLeon Romanovsky */ 6510df91bb6SJason Gunthorpe static int assign_name(struct ib_device *device, const char *name) 6520df91bb6SJason Gunthorpe { 6530df91bb6SJason Gunthorpe static u32 last_id; 6540df91bb6SJason Gunthorpe int ret; 655ecc82c53SLeon Romanovsky 656921eab11SJason Gunthorpe down_write(&devices_rwsem); 6570df91bb6SJason Gunthorpe /* Assign a unique name to the device */ 6580df91bb6SJason Gunthorpe if (strchr(name, '%')) 6590df91bb6SJason Gunthorpe ret = alloc_name(device, name); 6600df91bb6SJason Gunthorpe else 6610df91bb6SJason Gunthorpe ret = dev_set_name(&device->dev, name); 6620df91bb6SJason Gunthorpe if (ret) 6630df91bb6SJason Gunthorpe goto out; 664ecc82c53SLeon Romanovsky 6650df91bb6SJason Gunthorpe if (__ib_device_get_by_name(dev_name(&device->dev))) { 6660df91bb6SJason Gunthorpe ret = -ENFILE; 6670df91bb6SJason Gunthorpe goto out; 668ecc82c53SLeon Romanovsky } 6690df91bb6SJason Gunthorpe strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); 6700df91bb6SJason Gunthorpe 6710df91bb6SJason Gunthorpe /* Cyclically allocate a user visible ID for the device */ 6720df91bb6SJason Gunthorpe device->index = last_id; 6730df91bb6SJason Gunthorpe ret = xa_alloc(&devices, &device->index, INT_MAX, device, GFP_KERNEL); 6740df91bb6SJason Gunthorpe if (ret == -ENOSPC) { 6750df91bb6SJason Gunthorpe device->index = 0; 6760df91bb6SJason Gunthorpe ret = xa_alloc(&devices, &device->index, INT_MAX, device, 6770df91bb6SJason Gunthorpe GFP_KERNEL); 6780df91bb6SJason Gunthorpe } 6790df91bb6SJason Gunthorpe if (ret) 6800df91bb6SJason Gunthorpe goto out; 6810df91bb6SJason Gunthorpe last_id = device->index + 1; 6820df91bb6SJason Gunthorpe 6830df91bb6SJason Gunthorpe ret = 0; 684921eab11SJason Gunthorpe 6850df91bb6SJason Gunthorpe out: 686921eab11SJason Gunthorpe up_write(&devices_rwsem); 6870df91bb6SJason Gunthorpe return ret; 6880df91bb6SJason Gunthorpe } 6890df91bb6SJason Gunthorpe 690548cb4fbSParav Pandit static void setup_dma_device(struct ib_device *device) 6911da177e4SLinus Torvalds { 69299db9494SBart Van Assche struct device *parent = device->dev.parent; 6931da177e4SLinus Torvalds 6940957c29fSBart Van Assche WARN_ON_ONCE(device->dma_device); 6950957c29fSBart Van Assche if (device->dev.dma_ops) { 6960957c29fSBart Van Assche /* 6970957c29fSBart Van Assche * The caller provided custom DMA operations. Copy the 6980957c29fSBart Van Assche * DMA-related fields that are used by e.g. dma_alloc_coherent() 6990957c29fSBart Van Assche * into device->dev. 7000957c29fSBart Van Assche */ 7010957c29fSBart Van Assche device->dma_device = &device->dev; 70202ee9da3SBart Van Assche if (!device->dev.dma_mask) { 70302ee9da3SBart Van Assche if (parent) 70499db9494SBart Van Assche device->dev.dma_mask = parent->dma_mask; 70502ee9da3SBart Van Assche else 70602ee9da3SBart Van Assche WARN_ON_ONCE(true); 70702ee9da3SBart Van Assche } 70802ee9da3SBart Van Assche if (!device->dev.coherent_dma_mask) { 70902ee9da3SBart Van Assche if (parent) 7100957c29fSBart Van Assche device->dev.coherent_dma_mask = 7110957c29fSBart Van Assche parent->coherent_dma_mask; 71202ee9da3SBart Van Assche else 71302ee9da3SBart Van Assche WARN_ON_ONCE(true); 71402ee9da3SBart Van Assche } 7150957c29fSBart Van Assche } else { 7160957c29fSBart Van Assche /* 7170957c29fSBart Van Assche * The caller did not provide custom DMA operations. Use the 7180957c29fSBart Van Assche * DMA mapping operations of the parent device. 7190957c29fSBart Van Assche */ 72002ee9da3SBart Van Assche WARN_ON_ONCE(!parent); 7210957c29fSBart Van Assche device->dma_device = parent; 7220957c29fSBart Van Assche } 723548cb4fbSParav Pandit } 724548cb4fbSParav Pandit 725921eab11SJason Gunthorpe /* 726921eab11SJason Gunthorpe * setup_device() allocates memory and sets up data that requires calling the 727921eab11SJason Gunthorpe * device ops, this is the only reason these actions are not done during 728921eab11SJason Gunthorpe * ib_alloc_device. It is undone by ib_dealloc_device(). 729921eab11SJason Gunthorpe */ 730548cb4fbSParav Pandit static int setup_device(struct ib_device *device) 731548cb4fbSParav Pandit { 732548cb4fbSParav Pandit struct ib_udata uhw = {.outlen = 0, .inlen = 0}; 733548cb4fbSParav Pandit int ret; 734548cb4fbSParav Pandit 735921eab11SJason Gunthorpe setup_dma_device(device); 736921eab11SJason Gunthorpe 737548cb4fbSParav Pandit ret = ib_device_check_mandatory(device); 738548cb4fbSParav Pandit if (ret) 739548cb4fbSParav Pandit return ret; 740548cb4fbSParav Pandit 7418ceb1357SJason Gunthorpe ret = setup_port_data(device); 742548cb4fbSParav Pandit if (ret) { 7438ceb1357SJason Gunthorpe dev_warn(&device->dev, "Couldn't create per-port data\n"); 744548cb4fbSParav Pandit return ret; 745548cb4fbSParav Pandit } 746548cb4fbSParav Pandit 747548cb4fbSParav Pandit memset(&device->attrs, 0, sizeof(device->attrs)); 7483023a1e9SKamal Heib ret = device->ops.query_device(device, &device->attrs, &uhw); 749548cb4fbSParav Pandit if (ret) { 750548cb4fbSParav Pandit dev_warn(&device->dev, 751548cb4fbSParav Pandit "Couldn't query the device attributes\n"); 752d45f89d5SJason Gunthorpe return ret; 753548cb4fbSParav Pandit } 754548cb4fbSParav Pandit 755548cb4fbSParav Pandit return 0; 756548cb4fbSParav Pandit } 757548cb4fbSParav Pandit 758921eab11SJason Gunthorpe static void disable_device(struct ib_device *device) 759921eab11SJason Gunthorpe { 760921eab11SJason Gunthorpe struct ib_client *client; 761921eab11SJason Gunthorpe 762921eab11SJason Gunthorpe WARN_ON(!refcount_read(&device->refcount)); 763921eab11SJason Gunthorpe 764921eab11SJason Gunthorpe down_write(&devices_rwsem); 765921eab11SJason Gunthorpe xa_clear_mark(&devices, device->index, DEVICE_REGISTERED); 766921eab11SJason Gunthorpe up_write(&devices_rwsem); 767921eab11SJason Gunthorpe 768921eab11SJason Gunthorpe down_read(&clients_rwsem); 769921eab11SJason Gunthorpe list_for_each_entry_reverse(client, &client_list, list) 770921eab11SJason Gunthorpe remove_client_context(device, client->client_id); 771921eab11SJason Gunthorpe up_read(&clients_rwsem); 772921eab11SJason Gunthorpe 773921eab11SJason Gunthorpe /* Pairs with refcount_set in enable_device */ 774921eab11SJason Gunthorpe ib_device_put(device); 775921eab11SJason Gunthorpe wait_for_completion(&device->unreg_completion); 776c2261dd7SJason Gunthorpe 777c2261dd7SJason Gunthorpe /* Expedite removing unregistered pointers from the hash table */ 778c2261dd7SJason Gunthorpe free_netdevs(device); 779921eab11SJason Gunthorpe } 780921eab11SJason Gunthorpe 781921eab11SJason Gunthorpe /* 782921eab11SJason Gunthorpe * An enabled device is visible to all clients and to all the public facing 783d0899892SJason Gunthorpe * APIs that return a device pointer. This always returns with a new get, even 784d0899892SJason Gunthorpe * if it fails. 785921eab11SJason Gunthorpe */ 786d0899892SJason Gunthorpe static int enable_device_and_get(struct ib_device *device) 787921eab11SJason Gunthorpe { 788921eab11SJason Gunthorpe struct ib_client *client; 789921eab11SJason Gunthorpe unsigned long index; 790d0899892SJason Gunthorpe int ret = 0; 791921eab11SJason Gunthorpe 792d0899892SJason Gunthorpe /* 793d0899892SJason Gunthorpe * One ref belongs to the xa and the other belongs to this 794d0899892SJason Gunthorpe * thread. This is needed to guard against parallel unregistration. 795d0899892SJason Gunthorpe */ 796d0899892SJason Gunthorpe refcount_set(&device->refcount, 2); 797921eab11SJason Gunthorpe down_write(&devices_rwsem); 798921eab11SJason Gunthorpe xa_set_mark(&devices, device->index, DEVICE_REGISTERED); 799d0899892SJason Gunthorpe 800d0899892SJason Gunthorpe /* 801d0899892SJason Gunthorpe * By using downgrade_write() we ensure that no other thread can clear 802d0899892SJason Gunthorpe * DEVICE_REGISTERED while we are completing the client setup. 803d0899892SJason Gunthorpe */ 804d0899892SJason Gunthorpe downgrade_write(&devices_rwsem); 805921eab11SJason Gunthorpe 806921eab11SJason Gunthorpe down_read(&clients_rwsem); 807921eab11SJason Gunthorpe xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) { 808921eab11SJason Gunthorpe ret = add_client_context(device, client); 809d0899892SJason Gunthorpe if (ret) 810d0899892SJason Gunthorpe break; 811d0899892SJason Gunthorpe } 812921eab11SJason Gunthorpe up_read(&clients_rwsem); 813d0899892SJason Gunthorpe up_read(&devices_rwsem); 814921eab11SJason Gunthorpe return ret; 815921eab11SJason Gunthorpe } 816921eab11SJason Gunthorpe 817548cb4fbSParav Pandit /** 818548cb4fbSParav Pandit * ib_register_device - Register an IB device with IB core 819548cb4fbSParav Pandit * @device:Device to register 820548cb4fbSParav Pandit * 821548cb4fbSParav Pandit * Low-level drivers use ib_register_device() to register their 822548cb4fbSParav Pandit * devices with the IB core. All registered clients will receive a 823548cb4fbSParav Pandit * callback for each device that is added. @device must be allocated 824548cb4fbSParav Pandit * with ib_alloc_device(). 825d0899892SJason Gunthorpe * 826d0899892SJason Gunthorpe * If the driver uses ops.dealloc_driver and calls any ib_unregister_device() 827d0899892SJason Gunthorpe * asynchronously then the device pointer may become freed as soon as this 828d0899892SJason Gunthorpe * function returns. 829548cb4fbSParav Pandit */ 830ea4baf7fSParav Pandit int ib_register_device(struct ib_device *device, const char *name) 831548cb4fbSParav Pandit { 832548cb4fbSParav Pandit int ret; 8331da177e4SLinus Torvalds 8340df91bb6SJason Gunthorpe ret = assign_name(device, name); 835e349f858SJason Gunthorpe if (ret) 836921eab11SJason Gunthorpe return ret; 8371da177e4SLinus Torvalds 838548cb4fbSParav Pandit ret = setup_device(device); 839548cb4fbSParav Pandit if (ret) 840d0899892SJason Gunthorpe return ret; 84103db3a2dSMatan Barak 842d45f89d5SJason Gunthorpe ret = ib_cache_setup_one(device); 843d45f89d5SJason Gunthorpe if (ret) { 844d45f89d5SJason Gunthorpe dev_warn(&device->dev, 845d45f89d5SJason Gunthorpe "Couldn't set up InfiniBand P_Key/GID cache\n"); 846d0899892SJason Gunthorpe return ret; 847d45f89d5SJason Gunthorpe } 848d45f89d5SJason Gunthorpe 8497527a7b1SParav Pandit ib_device_register_rdmacg(device); 8503e153a93SIra Weiny 8515f8f5499SParav Pandit ret = device_add(&device->dev); 8525f8f5499SParav Pandit if (ret) 8535f8f5499SParav Pandit goto cg_cleanup; 8545f8f5499SParav Pandit 855ea4baf7fSParav Pandit ret = ib_device_register_sysfs(device); 8561da177e4SLinus Torvalds if (ret) { 85743c7c851SJason Gunthorpe dev_warn(&device->dev, 85843c7c851SJason Gunthorpe "Couldn't register device with driver model\n"); 8595f8f5499SParav Pandit goto dev_cleanup; 8601da177e4SLinus Torvalds } 8611da177e4SLinus Torvalds 862d0899892SJason Gunthorpe ret = enable_device_and_get(device); 863d0899892SJason Gunthorpe if (ret) { 864d0899892SJason Gunthorpe void (*dealloc_fn)(struct ib_device *); 865d0899892SJason Gunthorpe 866d0899892SJason Gunthorpe /* 867d0899892SJason Gunthorpe * If we hit this error flow then we don't want to 868d0899892SJason Gunthorpe * automatically dealloc the device since the caller is 869d0899892SJason Gunthorpe * expected to call ib_dealloc_device() after 870d0899892SJason Gunthorpe * ib_register_device() fails. This is tricky due to the 871d0899892SJason Gunthorpe * possibility for a parallel unregistration along with this 872d0899892SJason Gunthorpe * error flow. Since we have a refcount here we know any 873d0899892SJason Gunthorpe * parallel flow is stopped in disable_device and will see the 874d0899892SJason Gunthorpe * NULL pointers, causing the responsibility to 875d0899892SJason Gunthorpe * ib_dealloc_device() to revert back to this thread. 876d0899892SJason Gunthorpe */ 877d0899892SJason Gunthorpe dealloc_fn = device->ops.dealloc_driver; 878d0899892SJason Gunthorpe device->ops.dealloc_driver = NULL; 879d0899892SJason Gunthorpe ib_device_put(device); 880d0899892SJason Gunthorpe __ib_unregister_device(device); 881d0899892SJason Gunthorpe device->ops.dealloc_driver = dealloc_fn; 882d0899892SJason Gunthorpe return ret; 883d0899892SJason Gunthorpe } 884d0899892SJason Gunthorpe ib_device_put(device); 8851da177e4SLinus Torvalds 8864be3a4faSParav Pandit return 0; 8874be3a4faSParav Pandit 8885f8f5499SParav Pandit dev_cleanup: 8895f8f5499SParav Pandit device_del(&device->dev); 8902fb4f4eaSParav Pandit cg_cleanup: 8912fb4f4eaSParav Pandit ib_device_unregister_rdmacg(device); 892d45f89d5SJason Gunthorpe ib_cache_cleanup_one(device); 8931da177e4SLinus Torvalds return ret; 8941da177e4SLinus Torvalds } 8951da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_device); 8961da177e4SLinus Torvalds 897d0899892SJason Gunthorpe /* Callers must hold a get on the device. */ 898d0899892SJason Gunthorpe static void __ib_unregister_device(struct ib_device *ib_dev) 899d0899892SJason Gunthorpe { 900d0899892SJason Gunthorpe /* 901d0899892SJason Gunthorpe * We have a registration lock so that all the calls to unregister are 902d0899892SJason Gunthorpe * fully fenced, once any unregister returns the device is truely 903d0899892SJason Gunthorpe * unregistered even if multiple callers are unregistering it at the 904d0899892SJason Gunthorpe * same time. This also interacts with the registration flow and 905d0899892SJason Gunthorpe * provides sane semantics if register and unregister are racing. 906d0899892SJason Gunthorpe */ 907d0899892SJason Gunthorpe mutex_lock(&ib_dev->unregistration_lock); 908d0899892SJason Gunthorpe if (!refcount_read(&ib_dev->refcount)) 909d0899892SJason Gunthorpe goto out; 910d0899892SJason Gunthorpe 911d0899892SJason Gunthorpe disable_device(ib_dev); 912d0899892SJason Gunthorpe ib_device_unregister_sysfs(ib_dev); 913d0899892SJason Gunthorpe device_del(&ib_dev->dev); 914d0899892SJason Gunthorpe ib_device_unregister_rdmacg(ib_dev); 915d0899892SJason Gunthorpe ib_cache_cleanup_one(ib_dev); 916d0899892SJason Gunthorpe 917d0899892SJason Gunthorpe /* 918d0899892SJason Gunthorpe * Drivers using the new flow may not call ib_dealloc_device except 919d0899892SJason Gunthorpe * in error unwind prior to registration success. 920d0899892SJason Gunthorpe */ 921d0899892SJason Gunthorpe if (ib_dev->ops.dealloc_driver) { 922d0899892SJason Gunthorpe WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1); 923d0899892SJason Gunthorpe ib_dealloc_device(ib_dev); 924d0899892SJason Gunthorpe } 925d0899892SJason Gunthorpe out: 926d0899892SJason Gunthorpe mutex_unlock(&ib_dev->unregistration_lock); 927d0899892SJason Gunthorpe } 928d0899892SJason Gunthorpe 9291da177e4SLinus Torvalds /** 9301da177e4SLinus Torvalds * ib_unregister_device - Unregister an IB device 931d0899892SJason Gunthorpe * @device: The device to unregister 9321da177e4SLinus Torvalds * 9331da177e4SLinus Torvalds * Unregister an IB device. All clients will receive a remove callback. 934d0899892SJason Gunthorpe * 935d0899892SJason Gunthorpe * Callers should call this routine only once, and protect against races with 936d0899892SJason Gunthorpe * registration. Typically it should only be called as part of a remove 937d0899892SJason Gunthorpe * callback in an implementation of driver core's struct device_driver and 938d0899892SJason Gunthorpe * related. 939d0899892SJason Gunthorpe * 940d0899892SJason Gunthorpe * If ops.dealloc_driver is used then ib_dev will be freed upon return from 941d0899892SJason Gunthorpe * this function. 9421da177e4SLinus Torvalds */ 943d0899892SJason Gunthorpe void ib_unregister_device(struct ib_device *ib_dev) 9441da177e4SLinus Torvalds { 945d0899892SJason Gunthorpe get_device(&ib_dev->dev); 946d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 947d0899892SJason Gunthorpe put_device(&ib_dev->dev); 9481da177e4SLinus Torvalds } 9491da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_device); 9501da177e4SLinus Torvalds 951d0899892SJason Gunthorpe /** 952d0899892SJason Gunthorpe * ib_unregister_device_and_put - Unregister a device while holding a 'get' 953d0899892SJason Gunthorpe * device: The device to unregister 954d0899892SJason Gunthorpe * 955d0899892SJason Gunthorpe * This is the same as ib_unregister_device(), except it includes an internal 956d0899892SJason Gunthorpe * ib_device_put() that should match a 'get' obtained by the caller. 957d0899892SJason Gunthorpe * 958d0899892SJason Gunthorpe * It is safe to call this routine concurrently from multiple threads while 959d0899892SJason Gunthorpe * holding the 'get'. When the function returns the device is fully 960d0899892SJason Gunthorpe * unregistered. 961d0899892SJason Gunthorpe * 962d0899892SJason Gunthorpe * Drivers using this flow MUST use the driver_unregister callback to clean up 963d0899892SJason Gunthorpe * their resources associated with the device and dealloc it. 964d0899892SJason Gunthorpe */ 965d0899892SJason Gunthorpe void ib_unregister_device_and_put(struct ib_device *ib_dev) 966d0899892SJason Gunthorpe { 967d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 968d0899892SJason Gunthorpe get_device(&ib_dev->dev); 969d0899892SJason Gunthorpe ib_device_put(ib_dev); 970d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 971d0899892SJason Gunthorpe put_device(&ib_dev->dev); 972d0899892SJason Gunthorpe } 973d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_device_and_put); 974d0899892SJason Gunthorpe 975d0899892SJason Gunthorpe /** 976d0899892SJason Gunthorpe * ib_unregister_driver - Unregister all IB devices for a driver 977d0899892SJason Gunthorpe * @driver_id: The driver to unregister 978d0899892SJason Gunthorpe * 979d0899892SJason Gunthorpe * This implements a fence for device unregistration. It only returns once all 980d0899892SJason Gunthorpe * devices associated with the driver_id have fully completed their 981d0899892SJason Gunthorpe * unregistration and returned from ib_unregister_device*(). 982d0899892SJason Gunthorpe * 983d0899892SJason Gunthorpe * If device's are not yet unregistered it goes ahead and starts unregistering 984d0899892SJason Gunthorpe * them. 985d0899892SJason Gunthorpe * 986d0899892SJason Gunthorpe * This does not block creation of new devices with the given driver_id, that 987d0899892SJason Gunthorpe * is the responsibility of the caller. 988d0899892SJason Gunthorpe */ 989d0899892SJason Gunthorpe void ib_unregister_driver(enum rdma_driver_id driver_id) 990d0899892SJason Gunthorpe { 991d0899892SJason Gunthorpe struct ib_device *ib_dev; 992d0899892SJason Gunthorpe unsigned long index; 993d0899892SJason Gunthorpe 994d0899892SJason Gunthorpe down_read(&devices_rwsem); 995d0899892SJason Gunthorpe xa_for_each (&devices, index, ib_dev) { 996d0899892SJason Gunthorpe if (ib_dev->driver_id != driver_id) 997d0899892SJason Gunthorpe continue; 998d0899892SJason Gunthorpe 999d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1000d0899892SJason Gunthorpe up_read(&devices_rwsem); 1001d0899892SJason Gunthorpe 1002d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 1003d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1004d0899892SJason Gunthorpe 1005d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1006d0899892SJason Gunthorpe down_read(&devices_rwsem); 1007d0899892SJason Gunthorpe } 1008d0899892SJason Gunthorpe up_read(&devices_rwsem); 1009d0899892SJason Gunthorpe } 1010d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_driver); 1011d0899892SJason Gunthorpe 1012d0899892SJason Gunthorpe static void ib_unregister_work(struct work_struct *work) 1013d0899892SJason Gunthorpe { 1014d0899892SJason Gunthorpe struct ib_device *ib_dev = 1015d0899892SJason Gunthorpe container_of(work, struct ib_device, unregistration_work); 1016d0899892SJason Gunthorpe 1017d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1018d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1019d0899892SJason Gunthorpe } 1020d0899892SJason Gunthorpe 1021d0899892SJason Gunthorpe /** 1022d0899892SJason Gunthorpe * ib_unregister_device_queued - Unregister a device using a work queue 1023d0899892SJason Gunthorpe * device: The device to unregister 1024d0899892SJason Gunthorpe * 1025d0899892SJason Gunthorpe * This schedules an asynchronous unregistration using a WQ for the device. A 1026d0899892SJason Gunthorpe * driver should use this to avoid holding locks while doing unregistration, 1027d0899892SJason Gunthorpe * such as holding the RTNL lock. 1028d0899892SJason Gunthorpe * 1029d0899892SJason Gunthorpe * Drivers using this API must use ib_unregister_driver before module unload 1030d0899892SJason Gunthorpe * to ensure that all scheduled unregistrations have completed. 1031d0899892SJason Gunthorpe */ 1032d0899892SJason Gunthorpe void ib_unregister_device_queued(struct ib_device *ib_dev) 1033d0899892SJason Gunthorpe { 1034d0899892SJason Gunthorpe WARN_ON(!refcount_read(&ib_dev->refcount)); 1035d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 1036d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1037d0899892SJason Gunthorpe if (!queue_work(system_unbound_wq, &ib_dev->unregistration_work)) 1038d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1039d0899892SJason Gunthorpe } 1040d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_device_queued); 1041d0899892SJason Gunthorpe 1042e59178d8SJason Gunthorpe static int assign_client_id(struct ib_client *client) 1043e59178d8SJason Gunthorpe { 1044e59178d8SJason Gunthorpe int ret; 1045e59178d8SJason Gunthorpe 1046921eab11SJason Gunthorpe down_write(&clients_rwsem); 1047e59178d8SJason Gunthorpe /* 1048e59178d8SJason Gunthorpe * The add/remove callbacks must be called in FIFO/LIFO order. To 1049e59178d8SJason Gunthorpe * achieve this we assign client_ids so they are sorted in 1050e59178d8SJason Gunthorpe * registration order, and retain a linked list we can reverse iterate 1051e59178d8SJason Gunthorpe * to get the LIFO order. The extra linked list can go away if xarray 1052e59178d8SJason Gunthorpe * learns to reverse iterate. 1053e59178d8SJason Gunthorpe */ 1054e59178d8SJason Gunthorpe if (list_empty(&client_list)) 1055e59178d8SJason Gunthorpe client->client_id = 0; 1056e59178d8SJason Gunthorpe else 1057e59178d8SJason Gunthorpe client->client_id = 1058e59178d8SJason Gunthorpe list_last_entry(&client_list, struct ib_client, list) 1059e59178d8SJason Gunthorpe ->client_id; 1060e59178d8SJason Gunthorpe ret = xa_alloc(&clients, &client->client_id, INT_MAX, client, 1061e59178d8SJason Gunthorpe GFP_KERNEL); 1062e59178d8SJason Gunthorpe if (ret) 1063e59178d8SJason Gunthorpe goto out; 1064e59178d8SJason Gunthorpe 1065921eab11SJason Gunthorpe xa_set_mark(&clients, client->client_id, CLIENT_REGISTERED); 1066921eab11SJason Gunthorpe list_add_tail(&client->list, &client_list); 1067921eab11SJason Gunthorpe 1068e59178d8SJason Gunthorpe out: 1069921eab11SJason Gunthorpe up_write(&clients_rwsem); 1070e59178d8SJason Gunthorpe return ret; 1071e59178d8SJason Gunthorpe } 1072e59178d8SJason Gunthorpe 10731da177e4SLinus Torvalds /** 10741da177e4SLinus Torvalds * ib_register_client - Register an IB client 10751da177e4SLinus Torvalds * @client:Client to register 10761da177e4SLinus Torvalds * 10771da177e4SLinus Torvalds * Upper level users of the IB drivers can use ib_register_client() to 10781da177e4SLinus Torvalds * register callbacks for IB device addition and removal. When an IB 10791da177e4SLinus Torvalds * device is added, each registered client's add method will be called 10801da177e4SLinus Torvalds * (in the order the clients were registered), and when a device is 10811da177e4SLinus Torvalds * removed, each client's remove method will be called (in the reverse 10821da177e4SLinus Torvalds * order that clients were registered). In addition, when 10831da177e4SLinus Torvalds * ib_register_client() is called, the client will receive an add 10841da177e4SLinus Torvalds * callback for all devices already registered. 10851da177e4SLinus Torvalds */ 10861da177e4SLinus Torvalds int ib_register_client(struct ib_client *client) 10871da177e4SLinus Torvalds { 10881da177e4SLinus Torvalds struct ib_device *device; 10890df91bb6SJason Gunthorpe unsigned long index; 1090e59178d8SJason Gunthorpe int ret; 10911da177e4SLinus Torvalds 1092e59178d8SJason Gunthorpe ret = assign_client_id(client); 1093921eab11SJason Gunthorpe if (ret) 1094921eab11SJason Gunthorpe return ret; 1095921eab11SJason Gunthorpe 1096921eab11SJason Gunthorpe down_read(&devices_rwsem); 1097921eab11SJason Gunthorpe xa_for_each_marked (&devices, index, device, DEVICE_REGISTERED) { 1098921eab11SJason Gunthorpe ret = add_client_context(device, client); 1099e59178d8SJason Gunthorpe if (ret) { 1100921eab11SJason Gunthorpe up_read(&devices_rwsem); 1101921eab11SJason Gunthorpe ib_unregister_client(client); 1102e59178d8SJason Gunthorpe return ret; 1103e59178d8SJason Gunthorpe } 1104921eab11SJason Gunthorpe } 1105921eab11SJason Gunthorpe up_read(&devices_rwsem); 11061da177e4SLinus Torvalds return 0; 11071da177e4SLinus Torvalds } 11081da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_client); 11091da177e4SLinus Torvalds 11101da177e4SLinus Torvalds /** 11111da177e4SLinus Torvalds * ib_unregister_client - Unregister an IB client 11121da177e4SLinus Torvalds * @client:Client to unregister 11131da177e4SLinus Torvalds * 11141da177e4SLinus Torvalds * Upper level users use ib_unregister_client() to remove their client 11151da177e4SLinus Torvalds * registration. When ib_unregister_client() is called, the client 11161da177e4SLinus Torvalds * will receive a remove callback for each IB device still registered. 1117921eab11SJason Gunthorpe * 1118921eab11SJason Gunthorpe * This is a full fence, once it returns no client callbacks will be called, 1119921eab11SJason Gunthorpe * or are running in another thread. 11201da177e4SLinus Torvalds */ 11211da177e4SLinus Torvalds void ib_unregister_client(struct ib_client *client) 11221da177e4SLinus Torvalds { 11231da177e4SLinus Torvalds struct ib_device *device; 11240df91bb6SJason Gunthorpe unsigned long index; 11251da177e4SLinus Torvalds 1126921eab11SJason Gunthorpe down_write(&clients_rwsem); 1127e59178d8SJason Gunthorpe xa_clear_mark(&clients, client->client_id, CLIENT_REGISTERED); 1128921eab11SJason Gunthorpe up_write(&clients_rwsem); 1129921eab11SJason Gunthorpe /* 1130921eab11SJason Gunthorpe * Every device still known must be serialized to make sure we are 1131921eab11SJason Gunthorpe * done with the client callbacks before we return. 1132921eab11SJason Gunthorpe */ 1133921eab11SJason Gunthorpe down_read(&devices_rwsem); 1134921eab11SJason Gunthorpe xa_for_each (&devices, index, device) 1135921eab11SJason Gunthorpe remove_client_context(device, client->client_id); 1136921eab11SJason Gunthorpe up_read(&devices_rwsem); 11375aa44bb9SHaggai Eran 1138921eab11SJason Gunthorpe down_write(&clients_rwsem); 1139e59178d8SJason Gunthorpe list_del(&client->list); 1140e59178d8SJason Gunthorpe xa_erase(&clients, client->client_id); 1141921eab11SJason Gunthorpe up_write(&clients_rwsem); 11421da177e4SLinus Torvalds } 11431da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_client); 11441da177e4SLinus Torvalds 11451da177e4SLinus Torvalds /** 11469cd330d3SKrishna Kumar * ib_set_client_data - Set IB client context 11471da177e4SLinus Torvalds * @device:Device to set context for 11481da177e4SLinus Torvalds * @client:Client to set context for 11491da177e4SLinus Torvalds * @data:Context to set 11501da177e4SLinus Torvalds * 11510df91bb6SJason Gunthorpe * ib_set_client_data() sets client context data that can be retrieved with 11520df91bb6SJason Gunthorpe * ib_get_client_data(). This can only be called while the client is 11530df91bb6SJason Gunthorpe * registered to the device, once the ib_client remove() callback returns this 11540df91bb6SJason Gunthorpe * cannot be called. 11551da177e4SLinus Torvalds */ 11561da177e4SLinus Torvalds void ib_set_client_data(struct ib_device *device, struct ib_client *client, 11571da177e4SLinus Torvalds void *data) 11581da177e4SLinus Torvalds { 11590df91bb6SJason Gunthorpe void *rc; 11601da177e4SLinus Torvalds 11610df91bb6SJason Gunthorpe if (WARN_ON(IS_ERR(data))) 11620df91bb6SJason Gunthorpe data = NULL; 11631da177e4SLinus Torvalds 11640df91bb6SJason Gunthorpe rc = xa_store(&device->client_data, client->client_id, data, 11650df91bb6SJason Gunthorpe GFP_KERNEL); 11660df91bb6SJason Gunthorpe WARN_ON(xa_is_err(rc)); 11671da177e4SLinus Torvalds } 11681da177e4SLinus Torvalds EXPORT_SYMBOL(ib_set_client_data); 11691da177e4SLinus Torvalds 11701da177e4SLinus Torvalds /** 11711da177e4SLinus Torvalds * ib_register_event_handler - Register an IB event handler 11721da177e4SLinus Torvalds * @event_handler:Handler to register 11731da177e4SLinus Torvalds * 11741da177e4SLinus Torvalds * ib_register_event_handler() registers an event handler that will be 11751da177e4SLinus Torvalds * called back when asynchronous IB events occur (as defined in 11761da177e4SLinus Torvalds * chapter 11 of the InfiniBand Architecture Specification). This 11771da177e4SLinus Torvalds * callback may occur in interrupt context. 11781da177e4SLinus Torvalds */ 1179dcc9881eSLeon Romanovsky void ib_register_event_handler(struct ib_event_handler *event_handler) 11801da177e4SLinus Torvalds { 11811da177e4SLinus Torvalds unsigned long flags; 11821da177e4SLinus Torvalds 11831da177e4SLinus Torvalds spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 11841da177e4SLinus Torvalds list_add_tail(&event_handler->list, 11851da177e4SLinus Torvalds &event_handler->device->event_handler_list); 11861da177e4SLinus Torvalds spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 11871da177e4SLinus Torvalds } 11881da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_event_handler); 11891da177e4SLinus Torvalds 11901da177e4SLinus Torvalds /** 11911da177e4SLinus Torvalds * ib_unregister_event_handler - Unregister an event handler 11921da177e4SLinus Torvalds * @event_handler:Handler to unregister 11931da177e4SLinus Torvalds * 11941da177e4SLinus Torvalds * Unregister an event handler registered with 11951da177e4SLinus Torvalds * ib_register_event_handler(). 11961da177e4SLinus Torvalds */ 1197dcc9881eSLeon Romanovsky void ib_unregister_event_handler(struct ib_event_handler *event_handler) 11981da177e4SLinus Torvalds { 11991da177e4SLinus Torvalds unsigned long flags; 12001da177e4SLinus Torvalds 12011da177e4SLinus Torvalds spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 12021da177e4SLinus Torvalds list_del(&event_handler->list); 12031da177e4SLinus Torvalds spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 12041da177e4SLinus Torvalds } 12051da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_event_handler); 12061da177e4SLinus Torvalds 12071da177e4SLinus Torvalds /** 12081da177e4SLinus Torvalds * ib_dispatch_event - Dispatch an asynchronous event 12091da177e4SLinus Torvalds * @event:Event to dispatch 12101da177e4SLinus Torvalds * 12111da177e4SLinus Torvalds * Low-level drivers must call ib_dispatch_event() to dispatch the 12121da177e4SLinus Torvalds * event to all registered event handlers when an asynchronous event 12131da177e4SLinus Torvalds * occurs. 12141da177e4SLinus Torvalds */ 12151da177e4SLinus Torvalds void ib_dispatch_event(struct ib_event *event) 12161da177e4SLinus Torvalds { 12171da177e4SLinus Torvalds unsigned long flags; 12181da177e4SLinus Torvalds struct ib_event_handler *handler; 12191da177e4SLinus Torvalds 12201da177e4SLinus Torvalds spin_lock_irqsave(&event->device->event_handler_lock, flags); 12211da177e4SLinus Torvalds 12221da177e4SLinus Torvalds list_for_each_entry(handler, &event->device->event_handler_list, list) 12231da177e4SLinus Torvalds handler->handler(handler, event); 12241da177e4SLinus Torvalds 12251da177e4SLinus Torvalds spin_unlock_irqrestore(&event->device->event_handler_lock, flags); 12261da177e4SLinus Torvalds } 12271da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dispatch_event); 12281da177e4SLinus Torvalds 12291da177e4SLinus Torvalds /** 12301da177e4SLinus Torvalds * ib_query_port - Query IB port attributes 12311da177e4SLinus Torvalds * @device:Device to query 12321da177e4SLinus Torvalds * @port_num:Port number to query 12331da177e4SLinus Torvalds * @port_attr:Port attributes 12341da177e4SLinus Torvalds * 12351da177e4SLinus Torvalds * ib_query_port() returns the attributes of a port through the 12361da177e4SLinus Torvalds * @port_attr pointer. 12371da177e4SLinus Torvalds */ 12381da177e4SLinus Torvalds int ib_query_port(struct ib_device *device, 12391da177e4SLinus Torvalds u8 port_num, 12401da177e4SLinus Torvalds struct ib_port_attr *port_attr) 12411da177e4SLinus Torvalds { 1242fad61ad4SEli Cohen union ib_gid gid; 1243fad61ad4SEli Cohen int err; 1244fad61ad4SEli Cohen 124524dc831bSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 1246116c0074SRoland Dreier return -EINVAL; 1247116c0074SRoland Dreier 1248fad61ad4SEli Cohen memset(port_attr, 0, sizeof(*port_attr)); 12493023a1e9SKamal Heib err = device->ops.query_port(device, port_num, port_attr); 1250fad61ad4SEli Cohen if (err || port_attr->subnet_prefix) 1251fad61ad4SEli Cohen return err; 1252fad61ad4SEli Cohen 1253d7012467SEli Cohen if (rdma_port_get_link_layer(device, port_num) != IB_LINK_LAYER_INFINIBAND) 1254d7012467SEli Cohen return 0; 1255d7012467SEli Cohen 12563023a1e9SKamal Heib err = device->ops.query_gid(device, port_num, 0, &gid); 1257fad61ad4SEli Cohen if (err) 1258fad61ad4SEli Cohen return err; 1259fad61ad4SEli Cohen 1260fad61ad4SEli Cohen port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix); 1261fad61ad4SEli Cohen return 0; 12621da177e4SLinus Torvalds } 12631da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_port); 12641da177e4SLinus Torvalds 1265324e227eSJason Gunthorpe static void add_ndev_hash(struct ib_port_data *pdata) 1266324e227eSJason Gunthorpe { 1267324e227eSJason Gunthorpe unsigned long flags; 1268324e227eSJason Gunthorpe 1269324e227eSJason Gunthorpe might_sleep(); 1270324e227eSJason Gunthorpe 1271324e227eSJason Gunthorpe spin_lock_irqsave(&ndev_hash_lock, flags); 1272324e227eSJason Gunthorpe if (hash_hashed(&pdata->ndev_hash_link)) { 1273324e227eSJason Gunthorpe hash_del_rcu(&pdata->ndev_hash_link); 1274324e227eSJason Gunthorpe spin_unlock_irqrestore(&ndev_hash_lock, flags); 1275324e227eSJason Gunthorpe /* 1276324e227eSJason Gunthorpe * We cannot do hash_add_rcu after a hash_del_rcu until the 1277324e227eSJason Gunthorpe * grace period 1278324e227eSJason Gunthorpe */ 1279324e227eSJason Gunthorpe synchronize_rcu(); 1280324e227eSJason Gunthorpe spin_lock_irqsave(&ndev_hash_lock, flags); 1281324e227eSJason Gunthorpe } 1282324e227eSJason Gunthorpe if (pdata->netdev) 1283324e227eSJason Gunthorpe hash_add_rcu(ndev_hash, &pdata->ndev_hash_link, 1284324e227eSJason Gunthorpe (uintptr_t)pdata->netdev); 1285324e227eSJason Gunthorpe spin_unlock_irqrestore(&ndev_hash_lock, flags); 1286324e227eSJason Gunthorpe } 1287324e227eSJason Gunthorpe 12881da177e4SLinus Torvalds /** 1289c2261dd7SJason Gunthorpe * ib_device_set_netdev - Associate the ib_dev with an underlying net_device 1290c2261dd7SJason Gunthorpe * @ib_dev: Device to modify 1291c2261dd7SJason Gunthorpe * @ndev: net_device to affiliate, may be NULL 1292c2261dd7SJason Gunthorpe * @port: IB port the net_device is connected to 1293c2261dd7SJason Gunthorpe * 1294c2261dd7SJason Gunthorpe * Drivers should use this to link the ib_device to a netdev so the netdev 1295c2261dd7SJason Gunthorpe * shows up in interfaces like ib_enum_roce_netdev. Only one netdev may be 1296c2261dd7SJason Gunthorpe * affiliated with any port. 1297c2261dd7SJason Gunthorpe * 1298c2261dd7SJason Gunthorpe * The caller must ensure that the given ndev is not unregistered or 1299c2261dd7SJason Gunthorpe * unregistering, and that either the ib_device is unregistered or 1300c2261dd7SJason Gunthorpe * ib_device_set_netdev() is called with NULL when the ndev sends a 1301c2261dd7SJason Gunthorpe * NETDEV_UNREGISTER event. 1302c2261dd7SJason Gunthorpe */ 1303c2261dd7SJason Gunthorpe int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev, 1304c2261dd7SJason Gunthorpe unsigned int port) 1305c2261dd7SJason Gunthorpe { 1306c2261dd7SJason Gunthorpe struct net_device *old_ndev; 1307c2261dd7SJason Gunthorpe struct ib_port_data *pdata; 1308c2261dd7SJason Gunthorpe unsigned long flags; 1309c2261dd7SJason Gunthorpe int ret; 1310c2261dd7SJason Gunthorpe 1311c2261dd7SJason Gunthorpe /* 1312c2261dd7SJason Gunthorpe * Drivers wish to call this before ib_register_driver, so we have to 1313c2261dd7SJason Gunthorpe * setup the port data early. 1314c2261dd7SJason Gunthorpe */ 1315c2261dd7SJason Gunthorpe ret = alloc_port_data(ib_dev); 1316c2261dd7SJason Gunthorpe if (ret) 1317c2261dd7SJason Gunthorpe return ret; 1318c2261dd7SJason Gunthorpe 1319c2261dd7SJason Gunthorpe if (!rdma_is_port_valid(ib_dev, port)) 1320c2261dd7SJason Gunthorpe return -EINVAL; 1321c2261dd7SJason Gunthorpe 1322c2261dd7SJason Gunthorpe pdata = &ib_dev->port_data[port]; 1323c2261dd7SJason Gunthorpe spin_lock_irqsave(&pdata->netdev_lock, flags); 1324324e227eSJason Gunthorpe old_ndev = rcu_dereference_protected( 1325324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1326324e227eSJason Gunthorpe if (old_ndev == ndev) { 1327c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1328c2261dd7SJason Gunthorpe return 0; 1329c2261dd7SJason Gunthorpe } 1330c2261dd7SJason Gunthorpe 1331c2261dd7SJason Gunthorpe if (ndev) 1332c2261dd7SJason Gunthorpe dev_hold(ndev); 1333324e227eSJason Gunthorpe rcu_assign_pointer(pdata->netdev, ndev); 1334c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1335c2261dd7SJason Gunthorpe 1336324e227eSJason Gunthorpe add_ndev_hash(pdata); 1337c2261dd7SJason Gunthorpe if (old_ndev) 1338c2261dd7SJason Gunthorpe dev_put(old_ndev); 1339c2261dd7SJason Gunthorpe 1340c2261dd7SJason Gunthorpe return 0; 1341c2261dd7SJason Gunthorpe } 1342c2261dd7SJason Gunthorpe EXPORT_SYMBOL(ib_device_set_netdev); 1343c2261dd7SJason Gunthorpe 1344c2261dd7SJason Gunthorpe static void free_netdevs(struct ib_device *ib_dev) 1345c2261dd7SJason Gunthorpe { 1346c2261dd7SJason Gunthorpe unsigned long flags; 1347c2261dd7SJason Gunthorpe unsigned int port; 1348c2261dd7SJason Gunthorpe 1349c2261dd7SJason Gunthorpe rdma_for_each_port (ib_dev, port) { 1350c2261dd7SJason Gunthorpe struct ib_port_data *pdata = &ib_dev->port_data[port]; 1351324e227eSJason Gunthorpe struct net_device *ndev; 1352c2261dd7SJason Gunthorpe 1353c2261dd7SJason Gunthorpe spin_lock_irqsave(&pdata->netdev_lock, flags); 1354324e227eSJason Gunthorpe ndev = rcu_dereference_protected( 1355324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1356324e227eSJason Gunthorpe if (ndev) { 1357324e227eSJason Gunthorpe spin_lock(&ndev_hash_lock); 1358324e227eSJason Gunthorpe hash_del_rcu(&pdata->ndev_hash_link); 1359324e227eSJason Gunthorpe spin_unlock(&ndev_hash_lock); 1360324e227eSJason Gunthorpe 1361324e227eSJason Gunthorpe /* 1362324e227eSJason Gunthorpe * If this is the last dev_put there is still a 1363324e227eSJason Gunthorpe * synchronize_rcu before the netdev is kfreed, so we 1364324e227eSJason Gunthorpe * can continue to rely on unlocked pointer 1365324e227eSJason Gunthorpe * comparisons after the put 1366324e227eSJason Gunthorpe */ 1367324e227eSJason Gunthorpe rcu_assign_pointer(pdata->netdev, NULL); 1368324e227eSJason Gunthorpe dev_put(ndev); 1369c2261dd7SJason Gunthorpe } 1370c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1371c2261dd7SJason Gunthorpe } 1372c2261dd7SJason Gunthorpe } 1373c2261dd7SJason Gunthorpe 1374c2261dd7SJason Gunthorpe struct net_device *ib_device_get_netdev(struct ib_device *ib_dev, 1375c2261dd7SJason Gunthorpe unsigned int port) 1376c2261dd7SJason Gunthorpe { 1377c2261dd7SJason Gunthorpe struct ib_port_data *pdata; 1378c2261dd7SJason Gunthorpe struct net_device *res; 1379c2261dd7SJason Gunthorpe 1380c2261dd7SJason Gunthorpe if (!rdma_is_port_valid(ib_dev, port)) 1381c2261dd7SJason Gunthorpe return NULL; 1382c2261dd7SJason Gunthorpe 1383c2261dd7SJason Gunthorpe pdata = &ib_dev->port_data[port]; 1384c2261dd7SJason Gunthorpe 1385c2261dd7SJason Gunthorpe /* 1386c2261dd7SJason Gunthorpe * New drivers should use ib_device_set_netdev() not the legacy 1387c2261dd7SJason Gunthorpe * get_netdev(). 1388c2261dd7SJason Gunthorpe */ 1389c2261dd7SJason Gunthorpe if (ib_dev->ops.get_netdev) 1390c2261dd7SJason Gunthorpe res = ib_dev->ops.get_netdev(ib_dev, port); 1391c2261dd7SJason Gunthorpe else { 1392c2261dd7SJason Gunthorpe spin_lock(&pdata->netdev_lock); 1393324e227eSJason Gunthorpe res = rcu_dereference_protected( 1394324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1395c2261dd7SJason Gunthorpe if (res) 1396c2261dd7SJason Gunthorpe dev_hold(res); 1397c2261dd7SJason Gunthorpe spin_unlock(&pdata->netdev_lock); 1398c2261dd7SJason Gunthorpe } 1399c2261dd7SJason Gunthorpe 1400c2261dd7SJason Gunthorpe /* 1401c2261dd7SJason Gunthorpe * If we are starting to unregister expedite things by preventing 1402c2261dd7SJason Gunthorpe * propagation of an unregistering netdev. 1403c2261dd7SJason Gunthorpe */ 1404c2261dd7SJason Gunthorpe if (res && res->reg_state != NETREG_REGISTERED) { 1405c2261dd7SJason Gunthorpe dev_put(res); 1406c2261dd7SJason Gunthorpe return NULL; 1407c2261dd7SJason Gunthorpe } 1408c2261dd7SJason Gunthorpe 1409c2261dd7SJason Gunthorpe return res; 1410c2261dd7SJason Gunthorpe } 1411c2261dd7SJason Gunthorpe 1412c2261dd7SJason Gunthorpe /** 1413324e227eSJason Gunthorpe * ib_device_get_by_netdev - Find an IB device associated with a netdev 1414324e227eSJason Gunthorpe * @ndev: netdev to locate 1415324e227eSJason Gunthorpe * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 1416324e227eSJason Gunthorpe * 1417324e227eSJason Gunthorpe * Find and hold an ib_device that is associated with a netdev via 1418324e227eSJason Gunthorpe * ib_device_set_netdev(). The caller must call ib_device_put() on the 1419324e227eSJason Gunthorpe * returned pointer. 1420324e227eSJason Gunthorpe */ 1421324e227eSJason Gunthorpe struct ib_device *ib_device_get_by_netdev(struct net_device *ndev, 1422324e227eSJason Gunthorpe enum rdma_driver_id driver_id) 1423324e227eSJason Gunthorpe { 1424324e227eSJason Gunthorpe struct ib_device *res = NULL; 1425324e227eSJason Gunthorpe struct ib_port_data *cur; 1426324e227eSJason Gunthorpe 1427324e227eSJason Gunthorpe rcu_read_lock(); 1428324e227eSJason Gunthorpe hash_for_each_possible_rcu (ndev_hash, cur, ndev_hash_link, 1429324e227eSJason Gunthorpe (uintptr_t)ndev) { 1430324e227eSJason Gunthorpe if (rcu_access_pointer(cur->netdev) == ndev && 1431324e227eSJason Gunthorpe (driver_id == RDMA_DRIVER_UNKNOWN || 1432324e227eSJason Gunthorpe cur->ib_dev->driver_id == driver_id) && 1433324e227eSJason Gunthorpe ib_device_try_get(cur->ib_dev)) { 1434324e227eSJason Gunthorpe res = cur->ib_dev; 1435324e227eSJason Gunthorpe break; 1436324e227eSJason Gunthorpe } 1437324e227eSJason Gunthorpe } 1438324e227eSJason Gunthorpe rcu_read_unlock(); 1439324e227eSJason Gunthorpe 1440324e227eSJason Gunthorpe return res; 1441324e227eSJason Gunthorpe } 1442324e227eSJason Gunthorpe EXPORT_SYMBOL(ib_device_get_by_netdev); 1443324e227eSJason Gunthorpe 1444324e227eSJason Gunthorpe /** 144503db3a2dSMatan Barak * ib_enum_roce_netdev - enumerate all RoCE ports 144603db3a2dSMatan Barak * @ib_dev : IB device we want to query 144703db3a2dSMatan Barak * @filter: Should we call the callback? 144803db3a2dSMatan Barak * @filter_cookie: Cookie passed to filter 144903db3a2dSMatan Barak * @cb: Callback to call for each found RoCE ports 145003db3a2dSMatan Barak * @cookie: Cookie passed back to the callback 145103db3a2dSMatan Barak * 145203db3a2dSMatan Barak * Enumerates all of the physical RoCE ports of ib_dev 145303db3a2dSMatan Barak * which are related to netdevice and calls callback() on each 145403db3a2dSMatan Barak * device for which filter() function returns non zero. 145503db3a2dSMatan Barak */ 145603db3a2dSMatan Barak void ib_enum_roce_netdev(struct ib_device *ib_dev, 145703db3a2dSMatan Barak roce_netdev_filter filter, 145803db3a2dSMatan Barak void *filter_cookie, 145903db3a2dSMatan Barak roce_netdev_callback cb, 146003db3a2dSMatan Barak void *cookie) 146103db3a2dSMatan Barak { 1462ea1075edSJason Gunthorpe unsigned int port; 146303db3a2dSMatan Barak 1464ea1075edSJason Gunthorpe rdma_for_each_port (ib_dev, port) 146503db3a2dSMatan Barak if (rdma_protocol_roce(ib_dev, port)) { 1466c2261dd7SJason Gunthorpe struct net_device *idev = 1467c2261dd7SJason Gunthorpe ib_device_get_netdev(ib_dev, port); 146803db3a2dSMatan Barak 146903db3a2dSMatan Barak if (filter(ib_dev, port, idev, filter_cookie)) 147003db3a2dSMatan Barak cb(ib_dev, port, idev, cookie); 147103db3a2dSMatan Barak 147203db3a2dSMatan Barak if (idev) 147303db3a2dSMatan Barak dev_put(idev); 147403db3a2dSMatan Barak } 147503db3a2dSMatan Barak } 147603db3a2dSMatan Barak 147703db3a2dSMatan Barak /** 147803db3a2dSMatan Barak * ib_enum_all_roce_netdevs - enumerate all RoCE devices 147903db3a2dSMatan Barak * @filter: Should we call the callback? 148003db3a2dSMatan Barak * @filter_cookie: Cookie passed to filter 148103db3a2dSMatan Barak * @cb: Callback to call for each found RoCE ports 148203db3a2dSMatan Barak * @cookie: Cookie passed back to the callback 148303db3a2dSMatan Barak * 148403db3a2dSMatan Barak * Enumerates all RoCE devices' physical ports which are related 148503db3a2dSMatan Barak * to netdevices and calls callback() on each device for which 148603db3a2dSMatan Barak * filter() function returns non zero. 148703db3a2dSMatan Barak */ 148803db3a2dSMatan Barak void ib_enum_all_roce_netdevs(roce_netdev_filter filter, 148903db3a2dSMatan Barak void *filter_cookie, 149003db3a2dSMatan Barak roce_netdev_callback cb, 149103db3a2dSMatan Barak void *cookie) 149203db3a2dSMatan Barak { 149303db3a2dSMatan Barak struct ib_device *dev; 14940df91bb6SJason Gunthorpe unsigned long index; 149503db3a2dSMatan Barak 1496921eab11SJason Gunthorpe down_read(&devices_rwsem); 14970df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) 149803db3a2dSMatan Barak ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); 1499921eab11SJason Gunthorpe up_read(&devices_rwsem); 150003db3a2dSMatan Barak } 150103db3a2dSMatan Barak 150203db3a2dSMatan Barak /** 15038030c835SLeon Romanovsky * ib_enum_all_devs - enumerate all ib_devices 15048030c835SLeon Romanovsky * @cb: Callback to call for each found ib_device 15058030c835SLeon Romanovsky * 15068030c835SLeon Romanovsky * Enumerates all ib_devices and calls callback() on each device. 15078030c835SLeon Romanovsky */ 15088030c835SLeon Romanovsky int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb, 15098030c835SLeon Romanovsky struct netlink_callback *cb) 15108030c835SLeon Romanovsky { 15110df91bb6SJason Gunthorpe unsigned long index; 15128030c835SLeon Romanovsky struct ib_device *dev; 15138030c835SLeon Romanovsky unsigned int idx = 0; 15148030c835SLeon Romanovsky int ret = 0; 15158030c835SLeon Romanovsky 1516921eab11SJason Gunthorpe down_read(&devices_rwsem); 15170df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 15188030c835SLeon Romanovsky ret = nldev_cb(dev, skb, cb, idx); 15198030c835SLeon Romanovsky if (ret) 15208030c835SLeon Romanovsky break; 15218030c835SLeon Romanovsky idx++; 15228030c835SLeon Romanovsky } 1523921eab11SJason Gunthorpe up_read(&devices_rwsem); 15248030c835SLeon Romanovsky return ret; 15258030c835SLeon Romanovsky } 15268030c835SLeon Romanovsky 15278030c835SLeon Romanovsky /** 15281da177e4SLinus Torvalds * ib_query_pkey - Get P_Key table entry 15291da177e4SLinus Torvalds * @device:Device to query 15301da177e4SLinus Torvalds * @port_num:Port number to query 15311da177e4SLinus Torvalds * @index:P_Key table index to query 15321da177e4SLinus Torvalds * @pkey:Returned P_Key 15331da177e4SLinus Torvalds * 15341da177e4SLinus Torvalds * ib_query_pkey() fetches the specified P_Key table entry. 15351da177e4SLinus Torvalds */ 15361da177e4SLinus Torvalds int ib_query_pkey(struct ib_device *device, 15371da177e4SLinus Torvalds u8 port_num, u16 index, u16 *pkey) 15381da177e4SLinus Torvalds { 15399af3f5cfSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 15409af3f5cfSYuval Shaia return -EINVAL; 15419af3f5cfSYuval Shaia 15423023a1e9SKamal Heib return device->ops.query_pkey(device, port_num, index, pkey); 15431da177e4SLinus Torvalds } 15441da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_pkey); 15451da177e4SLinus Torvalds 15461da177e4SLinus Torvalds /** 15471da177e4SLinus Torvalds * ib_modify_device - Change IB device attributes 15481da177e4SLinus Torvalds * @device:Device to modify 15491da177e4SLinus Torvalds * @device_modify_mask:Mask of attributes to change 15501da177e4SLinus Torvalds * @device_modify:New attribute values 15511da177e4SLinus Torvalds * 15521da177e4SLinus Torvalds * ib_modify_device() changes a device's attributes as specified by 15531da177e4SLinus Torvalds * the @device_modify_mask and @device_modify structure. 15541da177e4SLinus Torvalds */ 15551da177e4SLinus Torvalds int ib_modify_device(struct ib_device *device, 15561da177e4SLinus Torvalds int device_modify_mask, 15571da177e4SLinus Torvalds struct ib_device_modify *device_modify) 15581da177e4SLinus Torvalds { 15593023a1e9SKamal Heib if (!device->ops.modify_device) 156010e1b54bSBart Van Assche return -ENOSYS; 156110e1b54bSBart Van Assche 15623023a1e9SKamal Heib return device->ops.modify_device(device, device_modify_mask, 15631da177e4SLinus Torvalds device_modify); 15641da177e4SLinus Torvalds } 15651da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_device); 15661da177e4SLinus Torvalds 15671da177e4SLinus Torvalds /** 15681da177e4SLinus Torvalds * ib_modify_port - Modifies the attributes for the specified port. 15691da177e4SLinus Torvalds * @device: The device to modify. 15701da177e4SLinus Torvalds * @port_num: The number of the port to modify. 15711da177e4SLinus Torvalds * @port_modify_mask: Mask used to specify which attributes of the port 15721da177e4SLinus Torvalds * to change. 15731da177e4SLinus Torvalds * @port_modify: New attribute values for the port. 15741da177e4SLinus Torvalds * 15751da177e4SLinus Torvalds * ib_modify_port() changes a port's attributes as specified by the 15761da177e4SLinus Torvalds * @port_modify_mask and @port_modify structure. 15771da177e4SLinus Torvalds */ 15781da177e4SLinus Torvalds int ib_modify_port(struct ib_device *device, 15791da177e4SLinus Torvalds u8 port_num, int port_modify_mask, 15801da177e4SLinus Torvalds struct ib_port_modify *port_modify) 15811da177e4SLinus Torvalds { 158261e0962dSSelvin Xavier int rc; 158310e1b54bSBart Van Assche 158424dc831bSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 1585116c0074SRoland Dreier return -EINVAL; 1586116c0074SRoland Dreier 15873023a1e9SKamal Heib if (device->ops.modify_port) 15883023a1e9SKamal Heib rc = device->ops.modify_port(device, port_num, 15893023a1e9SKamal Heib port_modify_mask, 15901da177e4SLinus Torvalds port_modify); 159161e0962dSSelvin Xavier else 159261e0962dSSelvin Xavier rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS; 159361e0962dSSelvin Xavier return rc; 15941da177e4SLinus Torvalds } 15951da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_port); 15961da177e4SLinus Torvalds 15975eb620c8SYosef Etigin /** 15985eb620c8SYosef Etigin * ib_find_gid - Returns the port number and GID table index where 1599dbb12562SParav Pandit * a specified GID value occurs. Its searches only for IB link layer. 16005eb620c8SYosef Etigin * @device: The device to query. 16015eb620c8SYosef Etigin * @gid: The GID value to search for. 16025eb620c8SYosef Etigin * @port_num: The port number of the device where the GID value was found. 16035eb620c8SYosef Etigin * @index: The index into the GID table where the GID was found. This 16045eb620c8SYosef Etigin * parameter may be NULL. 16055eb620c8SYosef Etigin */ 16065eb620c8SYosef Etigin int ib_find_gid(struct ib_device *device, union ib_gid *gid, 1607b26c4a11SParav Pandit u8 *port_num, u16 *index) 16085eb620c8SYosef Etigin { 16095eb620c8SYosef Etigin union ib_gid tmp_gid; 1610ea1075edSJason Gunthorpe unsigned int port; 1611ea1075edSJason Gunthorpe int ret, i; 16125eb620c8SYosef Etigin 1613ea1075edSJason Gunthorpe rdma_for_each_port (device, port) { 161422d24f75SParav Pandit if (!rdma_protocol_ib(device, port)) 1615b39ffa1dSMatan Barak continue; 1616b39ffa1dSMatan Barak 16178ceb1357SJason Gunthorpe for (i = 0; i < device->port_data[port].immutable.gid_tbl_len; 16188ceb1357SJason Gunthorpe ++i) { 16191dfce294SParav Pandit ret = rdma_query_gid(device, port, i, &tmp_gid); 16205eb620c8SYosef Etigin if (ret) 16215eb620c8SYosef Etigin return ret; 16225eb620c8SYosef Etigin if (!memcmp(&tmp_gid, gid, sizeof *gid)) { 16235eb620c8SYosef Etigin *port_num = port; 16245eb620c8SYosef Etigin if (index) 16255eb620c8SYosef Etigin *index = i; 16265eb620c8SYosef Etigin return 0; 16275eb620c8SYosef Etigin } 16285eb620c8SYosef Etigin } 16295eb620c8SYosef Etigin } 16305eb620c8SYosef Etigin 16315eb620c8SYosef Etigin return -ENOENT; 16325eb620c8SYosef Etigin } 16335eb620c8SYosef Etigin EXPORT_SYMBOL(ib_find_gid); 16345eb620c8SYosef Etigin 16355eb620c8SYosef Etigin /** 16365eb620c8SYosef Etigin * ib_find_pkey - Returns the PKey table index where a specified 16375eb620c8SYosef Etigin * PKey value occurs. 16385eb620c8SYosef Etigin * @device: The device to query. 16395eb620c8SYosef Etigin * @port_num: The port number of the device to search for the PKey. 16405eb620c8SYosef Etigin * @pkey: The PKey value to search for. 16415eb620c8SYosef Etigin * @index: The index into the PKey table where the PKey was found. 16425eb620c8SYosef Etigin */ 16435eb620c8SYosef Etigin int ib_find_pkey(struct ib_device *device, 16445eb620c8SYosef Etigin u8 port_num, u16 pkey, u16 *index) 16455eb620c8SYosef Etigin { 16465eb620c8SYosef Etigin int ret, i; 16475eb620c8SYosef Etigin u16 tmp_pkey; 1648ff7166c4SJack Morgenstein int partial_ix = -1; 16495eb620c8SYosef Etigin 16508ceb1357SJason Gunthorpe for (i = 0; i < device->port_data[port_num].immutable.pkey_tbl_len; 16518ceb1357SJason Gunthorpe ++i) { 16525eb620c8SYosef Etigin ret = ib_query_pkey(device, port_num, i, &tmp_pkey); 16535eb620c8SYosef Etigin if (ret) 16545eb620c8SYosef Etigin return ret; 165536026eccSMoni Shoua if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) { 1656ff7166c4SJack Morgenstein /* if there is full-member pkey take it.*/ 1657ff7166c4SJack Morgenstein if (tmp_pkey & 0x8000) { 16585eb620c8SYosef Etigin *index = i; 16595eb620c8SYosef Etigin return 0; 16605eb620c8SYosef Etigin } 1661ff7166c4SJack Morgenstein if (partial_ix < 0) 1662ff7166c4SJack Morgenstein partial_ix = i; 1663ff7166c4SJack Morgenstein } 16645eb620c8SYosef Etigin } 16655eb620c8SYosef Etigin 1666ff7166c4SJack Morgenstein /*no full-member, if exists take the limited*/ 1667ff7166c4SJack Morgenstein if (partial_ix >= 0) { 1668ff7166c4SJack Morgenstein *index = partial_ix; 1669ff7166c4SJack Morgenstein return 0; 1670ff7166c4SJack Morgenstein } 16715eb620c8SYosef Etigin return -ENOENT; 16725eb620c8SYosef Etigin } 16735eb620c8SYosef Etigin EXPORT_SYMBOL(ib_find_pkey); 16745eb620c8SYosef Etigin 16759268f72dSYotam Kenneth /** 16769268f72dSYotam Kenneth * ib_get_net_dev_by_params() - Return the appropriate net_dev 16779268f72dSYotam Kenneth * for a received CM request 16789268f72dSYotam Kenneth * @dev: An RDMA device on which the request has been received. 16799268f72dSYotam Kenneth * @port: Port number on the RDMA device. 16809268f72dSYotam Kenneth * @pkey: The Pkey the request came on. 16819268f72dSYotam Kenneth * @gid: A GID that the net_dev uses to communicate. 16829268f72dSYotam Kenneth * @addr: Contains the IP address that the request specified as its 16839268f72dSYotam Kenneth * destination. 1684921eab11SJason Gunthorpe * 16859268f72dSYotam Kenneth */ 16869268f72dSYotam Kenneth struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, 16879268f72dSYotam Kenneth u8 port, 16889268f72dSYotam Kenneth u16 pkey, 16899268f72dSYotam Kenneth const union ib_gid *gid, 16909268f72dSYotam Kenneth const struct sockaddr *addr) 16919268f72dSYotam Kenneth { 16929268f72dSYotam Kenneth struct net_device *net_dev = NULL; 16930df91bb6SJason Gunthorpe unsigned long index; 16940df91bb6SJason Gunthorpe void *client_data; 16959268f72dSYotam Kenneth 16969268f72dSYotam Kenneth if (!rdma_protocol_ib(dev, port)) 16979268f72dSYotam Kenneth return NULL; 16989268f72dSYotam Kenneth 1699921eab11SJason Gunthorpe /* 1700921eab11SJason Gunthorpe * Holding the read side guarantees that the client will not become 1701921eab11SJason Gunthorpe * unregistered while we are calling get_net_dev_by_params() 1702921eab11SJason Gunthorpe */ 1703921eab11SJason Gunthorpe down_read(&dev->client_data_rwsem); 17040df91bb6SJason Gunthorpe xan_for_each_marked (&dev->client_data, index, client_data, 17050df91bb6SJason Gunthorpe CLIENT_DATA_REGISTERED) { 17060df91bb6SJason Gunthorpe struct ib_client *client = xa_load(&clients, index); 17079268f72dSYotam Kenneth 17080df91bb6SJason Gunthorpe if (!client || !client->get_net_dev_by_params) 17099268f72dSYotam Kenneth continue; 17109268f72dSYotam Kenneth 17110df91bb6SJason Gunthorpe net_dev = client->get_net_dev_by_params(dev, port, pkey, gid, 17120df91bb6SJason Gunthorpe addr, client_data); 17139268f72dSYotam Kenneth if (net_dev) 17149268f72dSYotam Kenneth break; 17159268f72dSYotam Kenneth } 1716921eab11SJason Gunthorpe up_read(&dev->client_data_rwsem); 17179268f72dSYotam Kenneth 17189268f72dSYotam Kenneth return net_dev; 17199268f72dSYotam Kenneth } 17209268f72dSYotam Kenneth EXPORT_SYMBOL(ib_get_net_dev_by_params); 17219268f72dSYotam Kenneth 1722521ed0d9SKamal Heib void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) 1723521ed0d9SKamal Heib { 17243023a1e9SKamal Heib struct ib_device_ops *dev_ops = &dev->ops; 1725521ed0d9SKamal Heib #define SET_DEVICE_OP(ptr, name) \ 1726521ed0d9SKamal Heib do { \ 1727521ed0d9SKamal Heib if (ops->name) \ 1728521ed0d9SKamal Heib if (!((ptr)->name)) \ 1729521ed0d9SKamal Heib (ptr)->name = ops->name; \ 1730521ed0d9SKamal Heib } while (0) 1731521ed0d9SKamal Heib 173230471d4bSLeon Romanovsky #define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name) 173330471d4bSLeon Romanovsky 17343023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, add_gid); 17352f1927b0SMoni Shoua SET_DEVICE_OP(dev_ops, advise_mr); 17363023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_dm); 17373023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_fmr); 17383023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_hw_stats); 17393023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_mr); 17403023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_mw); 17413023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_pd); 17423023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_rdma_netdev); 17433023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_ucontext); 17443023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_xrcd); 17453023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, attach_mcast); 17463023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, check_mr_status); 17473023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_ah); 17483023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_counters); 17493023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_cq); 17503023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_flow); 17513023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_flow_action_esp); 17523023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_qp); 17533023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_rwq_ind_table); 17543023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_srq); 17553023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_wq); 17563023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_dm); 1757d0899892SJason Gunthorpe SET_DEVICE_OP(dev_ops, dealloc_driver); 17583023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_fmr); 17593023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_mw); 17603023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_pd); 17613023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_ucontext); 17623023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_xrcd); 17633023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, del_gid); 17643023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dereg_mr); 17653023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_ah); 17663023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_counters); 17673023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_cq); 17683023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_flow); 17693023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_flow_action); 17703023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_qp); 17713023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table); 17723023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_srq); 17733023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_wq); 17743023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, detach_mcast); 17753023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, disassociate_ucontext); 17763023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, drain_rq); 17773023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, drain_sq); 177802da3750SLeon Romanovsky SET_DEVICE_OP(dev_ops, fill_res_entry); 17793023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_dev_fw_str); 17803023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_dma_mr); 17813023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_hw_stats); 17823023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_link_layer); 17833023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_netdev); 17843023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_port_immutable); 17853023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vector_affinity); 17863023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vf_config); 17873023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vf_stats); 1788ea4baf7fSParav Pandit SET_DEVICE_OP(dev_ops, init_port); 17893023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, map_mr_sg); 17903023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, map_phys_fmr); 17913023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, mmap); 17923023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_ah); 17933023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_cq); 17943023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_device); 17953023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_flow_action_esp); 17963023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_port); 17973023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_qp); 17983023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_srq); 17993023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_wq); 18003023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, peek_cq); 18013023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, poll_cq); 18023023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_recv); 18033023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_send); 18043023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_srq_recv); 18053023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, process_mad); 18063023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_ah); 18073023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_device); 18083023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_gid); 18093023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_pkey); 18103023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_port); 18113023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_qp); 18123023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_srq); 18133023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); 18143023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, read_counters); 18153023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, reg_dm_mr); 18163023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, reg_user_mr); 18173023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, req_ncomp_notif); 18183023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, req_notify_cq); 18193023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, rereg_user_mr); 18203023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, resize_cq); 18213023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, set_vf_guid); 18223023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, set_vf_link_state); 18233023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, unmap_fmr); 182421a428a0SLeon Romanovsky 182521a428a0SLeon Romanovsky SET_OBJ_SIZE(dev_ops, ib_pd); 1826521ed0d9SKamal Heib } 1827521ed0d9SKamal Heib EXPORT_SYMBOL(ib_set_device_ops); 1828521ed0d9SKamal Heib 1829d0e312feSLeon Romanovsky static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = { 1830735c631aSMark Bloch [RDMA_NL_LS_OP_RESOLVE] = { 1831647c75acSLeon Romanovsky .doit = ib_nl_handle_resolve_resp, 1832e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 1833e3a2b93dSLeon Romanovsky }, 1834735c631aSMark Bloch [RDMA_NL_LS_OP_SET_TIMEOUT] = { 1835647c75acSLeon Romanovsky .doit = ib_nl_handle_set_timeout, 1836e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 1837e3a2b93dSLeon Romanovsky }, 1838ae43f828SMark Bloch [RDMA_NL_LS_OP_IP_RESOLVE] = { 1839647c75acSLeon Romanovsky .doit = ib_nl_handle_ip_res_resp, 1840e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 1841e3a2b93dSLeon Romanovsky }, 1842735c631aSMark Bloch }; 1843735c631aSMark Bloch 18441da177e4SLinus Torvalds static int __init ib_core_init(void) 18451da177e4SLinus Torvalds { 18461da177e4SLinus Torvalds int ret; 18471da177e4SLinus Torvalds 1848f0626710STejun Heo ib_wq = alloc_workqueue("infiniband", 0, 0); 1849f0626710STejun Heo if (!ib_wq) 1850f0626710STejun Heo return -ENOMEM; 1851f0626710STejun Heo 185214d3a3b2SChristoph Hellwig ib_comp_wq = alloc_workqueue("ib-comp-wq", 1853b7363e67SSagi Grimberg WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 185414d3a3b2SChristoph Hellwig if (!ib_comp_wq) { 185514d3a3b2SChristoph Hellwig ret = -ENOMEM; 185614d3a3b2SChristoph Hellwig goto err; 185714d3a3b2SChristoph Hellwig } 185814d3a3b2SChristoph Hellwig 1859f794809aSJack Morgenstein ib_comp_unbound_wq = 1860f794809aSJack Morgenstein alloc_workqueue("ib-comp-unb-wq", 1861f794809aSJack Morgenstein WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM | 1862f794809aSJack Morgenstein WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE); 1863f794809aSJack Morgenstein if (!ib_comp_unbound_wq) { 1864f794809aSJack Morgenstein ret = -ENOMEM; 1865f794809aSJack Morgenstein goto err_comp; 1866f794809aSJack Morgenstein } 1867f794809aSJack Morgenstein 186855aeed06SJason Gunthorpe ret = class_register(&ib_class); 1869fd75c789SNir Muchtar if (ret) { 1870aba25a3eSParav Pandit pr_warn("Couldn't create InfiniBand device class\n"); 1871f794809aSJack Morgenstein goto err_comp_unbound; 1872fd75c789SNir Muchtar } 18731da177e4SLinus Torvalds 1874c9901724SLeon Romanovsky ret = rdma_nl_init(); 18751da177e4SLinus Torvalds if (ret) { 1876c9901724SLeon Romanovsky pr_warn("Couldn't init IB netlink interface: err %d\n", ret); 1877fd75c789SNir Muchtar goto err_sysfs; 18781da177e4SLinus Torvalds } 18791da177e4SLinus Torvalds 1880e3f20f02SLeon Romanovsky ret = addr_init(); 1881e3f20f02SLeon Romanovsky if (ret) { 1882e3f20f02SLeon Romanovsky pr_warn("Could't init IB address resolution\n"); 1883e3f20f02SLeon Romanovsky goto err_ibnl; 1884e3f20f02SLeon Romanovsky } 1885e3f20f02SLeon Romanovsky 18864c2cb422SMark Bloch ret = ib_mad_init(); 18874c2cb422SMark Bloch if (ret) { 18884c2cb422SMark Bloch pr_warn("Couldn't init IB MAD\n"); 18894c2cb422SMark Bloch goto err_addr; 18904c2cb422SMark Bloch } 18914c2cb422SMark Bloch 1892c2e49c92SMark Bloch ret = ib_sa_init(); 1893c2e49c92SMark Bloch if (ret) { 1894c2e49c92SMark Bloch pr_warn("Couldn't init SA\n"); 1895c2e49c92SMark Bloch goto err_mad; 1896c2e49c92SMark Bloch } 1897c2e49c92SMark Bloch 18988f408ab6SDaniel Jurgens ret = register_lsm_notifier(&ibdev_lsm_nb); 18998f408ab6SDaniel Jurgens if (ret) { 19008f408ab6SDaniel Jurgens pr_warn("Couldn't register LSM notifier. ret %d\n", ret); 1901c9901724SLeon Romanovsky goto err_sa; 19028f408ab6SDaniel Jurgens } 19038f408ab6SDaniel Jurgens 19046c80b41aSLeon Romanovsky nldev_init(); 1905c9901724SLeon Romanovsky rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table); 19065ef8c0c1SJason Gunthorpe roce_gid_mgmt_init(); 1907b2cbae2cSRoland Dreier 1908fd75c789SNir Muchtar return 0; 1909fd75c789SNir Muchtar 1910735c631aSMark Bloch err_sa: 1911735c631aSMark Bloch ib_sa_cleanup(); 1912c2e49c92SMark Bloch err_mad: 1913c2e49c92SMark Bloch ib_mad_cleanup(); 19144c2cb422SMark Bloch err_addr: 19154c2cb422SMark Bloch addr_cleanup(); 1916e3f20f02SLeon Romanovsky err_ibnl: 1917c9901724SLeon Romanovsky rdma_nl_exit(); 1918fd75c789SNir Muchtar err_sysfs: 191955aeed06SJason Gunthorpe class_unregister(&ib_class); 1920f794809aSJack Morgenstein err_comp_unbound: 1921f794809aSJack Morgenstein destroy_workqueue(ib_comp_unbound_wq); 192214d3a3b2SChristoph Hellwig err_comp: 192314d3a3b2SChristoph Hellwig destroy_workqueue(ib_comp_wq); 1924fd75c789SNir Muchtar err: 1925fd75c789SNir Muchtar destroy_workqueue(ib_wq); 19261da177e4SLinus Torvalds return ret; 19271da177e4SLinus Torvalds } 19281da177e4SLinus Torvalds 19291da177e4SLinus Torvalds static void __exit ib_core_cleanup(void) 19301da177e4SLinus Torvalds { 19315ef8c0c1SJason Gunthorpe roce_gid_mgmt_cleanup(); 19326c80b41aSLeon Romanovsky nldev_exit(); 1933c9901724SLeon Romanovsky rdma_nl_unregister(RDMA_NL_LS); 1934c9901724SLeon Romanovsky unregister_lsm_notifier(&ibdev_lsm_nb); 1935c2e49c92SMark Bloch ib_sa_cleanup(); 19364c2cb422SMark Bloch ib_mad_cleanup(); 1937e3f20f02SLeon Romanovsky addr_cleanup(); 1938c9901724SLeon Romanovsky rdma_nl_exit(); 193955aeed06SJason Gunthorpe class_unregister(&ib_class); 1940f794809aSJack Morgenstein destroy_workqueue(ib_comp_unbound_wq); 194114d3a3b2SChristoph Hellwig destroy_workqueue(ib_comp_wq); 1942f7c6a7b5SRoland Dreier /* Make sure that any pending umem accounting work is done. */ 1943f0626710STejun Heo destroy_workqueue(ib_wq); 1944d0899892SJason Gunthorpe flush_workqueue(system_unbound_wq); 1945e59178d8SJason Gunthorpe WARN_ON(!xa_empty(&clients)); 19460df91bb6SJason Gunthorpe WARN_ON(!xa_empty(&devices)); 19471da177e4SLinus Torvalds } 19481da177e4SLinus Torvalds 1949e3bf14bdSJason Gunthorpe MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4); 1950e3bf14bdSJason Gunthorpe 1951a9cd1a67SDmitry Monakhov subsys_initcall(ib_core_init); 19521da177e4SLinus Torvalds module_exit(ib_core_cleanup); 1953