11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Communications. All rights reserved. 32a1d9b7fSRoland Dreier * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * This software is available to you under a choice of one of two 61da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 71da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 81da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 91da177e4SLinus Torvalds * OpenIB.org BSD license below: 101da177e4SLinus Torvalds * 111da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 121da177e4SLinus Torvalds * without modification, are permitted provided that the following 131da177e4SLinus Torvalds * conditions are met: 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * - Redistributions of source code must retain the above 161da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 171da177e4SLinus Torvalds * disclaimer. 181da177e4SLinus Torvalds * 191da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 201da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 211da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 221da177e4SLinus Torvalds * provided with the distribution. 231da177e4SLinus Torvalds * 241da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 251da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 261da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 271da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 281da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 291da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 301da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 311da177e4SLinus Torvalds * SOFTWARE. 321da177e4SLinus Torvalds */ 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds #include <linux/module.h> 351da177e4SLinus Torvalds #include <linux/string.h> 361da177e4SLinus Torvalds #include <linux/errno.h> 379a6b090cSAhmed S. Darwish #include <linux/kernel.h> 381da177e4SLinus Torvalds #include <linux/slab.h> 391da177e4SLinus Torvalds #include <linux/init.h> 409268f72dSYotam Kenneth #include <linux/netdevice.h> 418f408ab6SDaniel Jurgens #include <linux/security.h> 428f408ab6SDaniel Jurgens #include <linux/notifier.h> 43324e227eSJason Gunthorpe #include <linux/hashtable.h> 44b2cbae2cSRoland Dreier #include <rdma/rdma_netlink.h> 4503db3a2dSMatan Barak #include <rdma/ib_addr.h> 4603db3a2dSMatan Barak #include <rdma/ib_cache.h> 471da177e4SLinus Torvalds 481da177e4SLinus Torvalds #include "core_priv.h" 4941eda65cSLeon Romanovsky #include "restrack.h" 501da177e4SLinus Torvalds 511da177e4SLinus Torvalds MODULE_AUTHOR("Roland Dreier"); 521da177e4SLinus Torvalds MODULE_DESCRIPTION("core kernel InfiniBand API"); 531da177e4SLinus Torvalds MODULE_LICENSE("Dual BSD/GPL"); 541da177e4SLinus Torvalds 5514d3a3b2SChristoph Hellwig struct workqueue_struct *ib_comp_wq; 56f794809aSJack Morgenstein struct workqueue_struct *ib_comp_unbound_wq; 57f0626710STejun Heo struct workqueue_struct *ib_wq; 58f0626710STejun Heo EXPORT_SYMBOL_GPL(ib_wq); 59f0626710STejun Heo 600df91bb6SJason Gunthorpe /* 61921eab11SJason Gunthorpe * Each of the three rwsem locks (devices, clients, client_data) protects the 62921eab11SJason Gunthorpe * xarray of the same name. Specifically it allows the caller to assert that 63921eab11SJason Gunthorpe * the MARK will/will not be changing under the lock, and for devices and 64921eab11SJason Gunthorpe * clients, that the value in the xarray is still a valid pointer. Change of 65921eab11SJason Gunthorpe * the MARK is linked to the object state, so holding the lock and testing the 66921eab11SJason Gunthorpe * MARK also asserts that the contained object is in a certain state. 67921eab11SJason Gunthorpe * 68921eab11SJason Gunthorpe * This is used to build a two stage register/unregister flow where objects 69921eab11SJason Gunthorpe * can continue to be in the xarray even though they are still in progress to 70921eab11SJason Gunthorpe * register/unregister. 71921eab11SJason Gunthorpe * 72921eab11SJason Gunthorpe * The xarray itself provides additional locking, and restartable iteration, 73921eab11SJason Gunthorpe * which is also relied on. 74921eab11SJason Gunthorpe * 75921eab11SJason Gunthorpe * Locks should not be nested, with the exception of client_data, which is 76921eab11SJason Gunthorpe * allowed to nest under the read side of the other two locks. 77921eab11SJason Gunthorpe * 78921eab11SJason Gunthorpe * The devices_rwsem also protects the device name list, any change or 79921eab11SJason Gunthorpe * assignment of device name must also hold the write side to guarantee unique 80921eab11SJason Gunthorpe * names. 81921eab11SJason Gunthorpe */ 82921eab11SJason Gunthorpe 83921eab11SJason Gunthorpe /* 840df91bb6SJason Gunthorpe * devices contains devices that have had their names assigned. The 850df91bb6SJason Gunthorpe * devices may not be registered. Users that care about the registration 860df91bb6SJason Gunthorpe * status need to call ib_device_try_get() on the device to ensure it is 870df91bb6SJason Gunthorpe * registered, and keep it registered, for the required duration. 880df91bb6SJason Gunthorpe * 890df91bb6SJason Gunthorpe */ 900df91bb6SJason Gunthorpe static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC); 91921eab11SJason Gunthorpe static DECLARE_RWSEM(devices_rwsem); 920df91bb6SJason Gunthorpe #define DEVICE_REGISTERED XA_MARK_1 930df91bb6SJason Gunthorpe 941da177e4SLinus Torvalds static LIST_HEAD(client_list); 95e59178d8SJason Gunthorpe #define CLIENT_REGISTERED XA_MARK_1 96e59178d8SJason Gunthorpe static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC); 97921eab11SJason Gunthorpe static DECLARE_RWSEM(clients_rwsem); 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds /* 1000df91bb6SJason Gunthorpe * If client_data is registered then the corresponding client must also still 1010df91bb6SJason Gunthorpe * be registered. 1020df91bb6SJason Gunthorpe */ 1030df91bb6SJason Gunthorpe #define CLIENT_DATA_REGISTERED XA_MARK_1 1040df91bb6SJason Gunthorpe /* 1050df91bb6SJason Gunthorpe * xarray has this behavior where it won't iterate over NULL values stored in 1060df91bb6SJason Gunthorpe * allocated arrays. So we need our own iterator to see all values stored in 1070df91bb6SJason Gunthorpe * the array. This does the same thing as xa_for_each except that it also 1080df91bb6SJason Gunthorpe * returns NULL valued entries if the array is allocating. Simplified to only 1090df91bb6SJason Gunthorpe * work on simple xarrays. 1100df91bb6SJason Gunthorpe */ 1110df91bb6SJason Gunthorpe static void *xan_find_marked(struct xarray *xa, unsigned long *indexp, 1120df91bb6SJason Gunthorpe xa_mark_t filter) 1130df91bb6SJason Gunthorpe { 1140df91bb6SJason Gunthorpe XA_STATE(xas, xa, *indexp); 1150df91bb6SJason Gunthorpe void *entry; 1160df91bb6SJason Gunthorpe 1170df91bb6SJason Gunthorpe rcu_read_lock(); 1180df91bb6SJason Gunthorpe do { 1190df91bb6SJason Gunthorpe entry = xas_find_marked(&xas, ULONG_MAX, filter); 1200df91bb6SJason Gunthorpe if (xa_is_zero(entry)) 1210df91bb6SJason Gunthorpe break; 1220df91bb6SJason Gunthorpe } while (xas_retry(&xas, entry)); 1230df91bb6SJason Gunthorpe rcu_read_unlock(); 1240df91bb6SJason Gunthorpe 1250df91bb6SJason Gunthorpe if (entry) { 1260df91bb6SJason Gunthorpe *indexp = xas.xa_index; 1270df91bb6SJason Gunthorpe if (xa_is_zero(entry)) 1280df91bb6SJason Gunthorpe return NULL; 1290df91bb6SJason Gunthorpe return entry; 1300df91bb6SJason Gunthorpe } 1310df91bb6SJason Gunthorpe return XA_ERROR(-ENOENT); 1320df91bb6SJason Gunthorpe } 1330df91bb6SJason Gunthorpe #define xan_for_each_marked(xa, index, entry, filter) \ 1340df91bb6SJason Gunthorpe for (index = 0, entry = xan_find_marked(xa, &(index), filter); \ 1350df91bb6SJason Gunthorpe !xa_is_err(entry); \ 1360df91bb6SJason Gunthorpe (index)++, entry = xan_find_marked(xa, &(index), filter)) 1370df91bb6SJason Gunthorpe 138324e227eSJason Gunthorpe /* RCU hash table mapping netdevice pointers to struct ib_port_data */ 139324e227eSJason Gunthorpe static DEFINE_SPINLOCK(ndev_hash_lock); 140324e227eSJason Gunthorpe static DECLARE_HASHTABLE(ndev_hash, 5); 141324e227eSJason Gunthorpe 142c2261dd7SJason Gunthorpe static void free_netdevs(struct ib_device *ib_dev); 143d0899892SJason Gunthorpe static void ib_unregister_work(struct work_struct *work); 144d0899892SJason Gunthorpe static void __ib_unregister_device(struct ib_device *device); 1458f408ab6SDaniel Jurgens static int ib_security_change(struct notifier_block *nb, unsigned long event, 1468f408ab6SDaniel Jurgens void *lsm_data); 1478f408ab6SDaniel Jurgens static void ib_policy_change_task(struct work_struct *work); 1488f408ab6SDaniel Jurgens static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task); 1498f408ab6SDaniel Jurgens 1508f408ab6SDaniel Jurgens static struct notifier_block ibdev_lsm_nb = { 1518f408ab6SDaniel Jurgens .notifier_call = ib_security_change, 1528f408ab6SDaniel Jurgens }; 1531da177e4SLinus Torvalds 154324e227eSJason Gunthorpe /* Pointer to the RCU head at the start of the ib_port_data array */ 155324e227eSJason Gunthorpe struct ib_port_data_rcu { 156324e227eSJason Gunthorpe struct rcu_head rcu_head; 157324e227eSJason Gunthorpe struct ib_port_data pdata[]; 158324e227eSJason Gunthorpe }; 159324e227eSJason Gunthorpe 1601da177e4SLinus Torvalds static int ib_device_check_mandatory(struct ib_device *device) 1611da177e4SLinus Torvalds { 1623023a1e9SKamal Heib #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x } 1631da177e4SLinus Torvalds static const struct { 1641da177e4SLinus Torvalds size_t offset; 1651da177e4SLinus Torvalds char *name; 1661da177e4SLinus Torvalds } mandatory_table[] = { 1671da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_device), 1681da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_port), 1691da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_pkey), 1701da177e4SLinus Torvalds IB_MANDATORY_FUNC(alloc_pd), 1711da177e4SLinus Torvalds IB_MANDATORY_FUNC(dealloc_pd), 1721da177e4SLinus Torvalds IB_MANDATORY_FUNC(create_qp), 1731da177e4SLinus Torvalds IB_MANDATORY_FUNC(modify_qp), 1741da177e4SLinus Torvalds IB_MANDATORY_FUNC(destroy_qp), 1751da177e4SLinus Torvalds IB_MANDATORY_FUNC(post_send), 1761da177e4SLinus Torvalds IB_MANDATORY_FUNC(post_recv), 1771da177e4SLinus Torvalds IB_MANDATORY_FUNC(create_cq), 1781da177e4SLinus Torvalds IB_MANDATORY_FUNC(destroy_cq), 1791da177e4SLinus Torvalds IB_MANDATORY_FUNC(poll_cq), 1801da177e4SLinus Torvalds IB_MANDATORY_FUNC(req_notify_cq), 1811da177e4SLinus Torvalds IB_MANDATORY_FUNC(get_dma_mr), 1827738613eSIra Weiny IB_MANDATORY_FUNC(dereg_mr), 1837738613eSIra Weiny IB_MANDATORY_FUNC(get_port_immutable) 1841da177e4SLinus Torvalds }; 1851da177e4SLinus Torvalds int i; 1861da177e4SLinus Torvalds 1876780c4faSGal Pressman device->kverbs_provider = true; 1889a6b090cSAhmed S. Darwish for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { 1893023a1e9SKamal Heib if (!*(void **) ((void *) &device->ops + 1903023a1e9SKamal Heib mandatory_table[i].offset)) { 1916780c4faSGal Pressman device->kverbs_provider = false; 1926780c4faSGal Pressman break; 1931da177e4SLinus Torvalds } 1941da177e4SLinus Torvalds } 1951da177e4SLinus Torvalds 1961da177e4SLinus Torvalds return 0; 1971da177e4SLinus Torvalds } 1981da177e4SLinus Torvalds 199f8978bd9SLeon Romanovsky /* 20001b67117SParav Pandit * Caller must perform ib_device_put() to return the device reference count 20101b67117SParav Pandit * when ib_device_get_by_index() returns valid device pointer. 202f8978bd9SLeon Romanovsky */ 203f8978bd9SLeon Romanovsky struct ib_device *ib_device_get_by_index(u32 index) 204f8978bd9SLeon Romanovsky { 205f8978bd9SLeon Romanovsky struct ib_device *device; 206f8978bd9SLeon Romanovsky 207921eab11SJason Gunthorpe down_read(&devices_rwsem); 2080df91bb6SJason Gunthorpe device = xa_load(&devices, index); 20901b67117SParav Pandit if (device) { 210d79af724SJason Gunthorpe if (!ib_device_try_get(device)) 21101b67117SParav Pandit device = NULL; 21201b67117SParav Pandit } 213921eab11SJason Gunthorpe up_read(&devices_rwsem); 214f8978bd9SLeon Romanovsky return device; 215f8978bd9SLeon Romanovsky } 216f8978bd9SLeon Romanovsky 217d79af724SJason Gunthorpe /** 218d79af724SJason Gunthorpe * ib_device_put - Release IB device reference 219d79af724SJason Gunthorpe * @device: device whose reference to be released 220d79af724SJason Gunthorpe * 221d79af724SJason Gunthorpe * ib_device_put() releases reference to the IB device to allow it to be 222d79af724SJason Gunthorpe * unregistered and eventually free. 223d79af724SJason Gunthorpe */ 22401b67117SParav Pandit void ib_device_put(struct ib_device *device) 22501b67117SParav Pandit { 22601b67117SParav Pandit if (refcount_dec_and_test(&device->refcount)) 22701b67117SParav Pandit complete(&device->unreg_completion); 22801b67117SParav Pandit } 229d79af724SJason Gunthorpe EXPORT_SYMBOL(ib_device_put); 23001b67117SParav Pandit 2311da177e4SLinus Torvalds static struct ib_device *__ib_device_get_by_name(const char *name) 2321da177e4SLinus Torvalds { 2331da177e4SLinus Torvalds struct ib_device *device; 2340df91bb6SJason Gunthorpe unsigned long index; 2351da177e4SLinus Torvalds 2360df91bb6SJason Gunthorpe xa_for_each (&devices, index, device) 237896de009SJason Gunthorpe if (!strcmp(name, dev_name(&device->dev))) 2381da177e4SLinus Torvalds return device; 2391da177e4SLinus Torvalds 2401da177e4SLinus Torvalds return NULL; 2411da177e4SLinus Torvalds } 2421da177e4SLinus Torvalds 2436cc2c8e5SJason Gunthorpe /** 2446cc2c8e5SJason Gunthorpe * ib_device_get_by_name - Find an IB device by name 2456cc2c8e5SJason Gunthorpe * @name: The name to look for 2466cc2c8e5SJason Gunthorpe * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 2476cc2c8e5SJason Gunthorpe * 2486cc2c8e5SJason Gunthorpe * Find and hold an ib_device by its name. The caller must call 2496cc2c8e5SJason Gunthorpe * ib_device_put() on the returned pointer. 2506cc2c8e5SJason Gunthorpe */ 2516cc2c8e5SJason Gunthorpe struct ib_device *ib_device_get_by_name(const char *name, 2526cc2c8e5SJason Gunthorpe enum rdma_driver_id driver_id) 2536cc2c8e5SJason Gunthorpe { 2546cc2c8e5SJason Gunthorpe struct ib_device *device; 2556cc2c8e5SJason Gunthorpe 2566cc2c8e5SJason Gunthorpe down_read(&devices_rwsem); 2576cc2c8e5SJason Gunthorpe device = __ib_device_get_by_name(name); 2586cc2c8e5SJason Gunthorpe if (device && driver_id != RDMA_DRIVER_UNKNOWN && 2596cc2c8e5SJason Gunthorpe device->driver_id != driver_id) 2606cc2c8e5SJason Gunthorpe device = NULL; 2616cc2c8e5SJason Gunthorpe 2626cc2c8e5SJason Gunthorpe if (device) { 2636cc2c8e5SJason Gunthorpe if (!ib_device_try_get(device)) 2646cc2c8e5SJason Gunthorpe device = NULL; 2656cc2c8e5SJason Gunthorpe } 2666cc2c8e5SJason Gunthorpe up_read(&devices_rwsem); 2676cc2c8e5SJason Gunthorpe return device; 2686cc2c8e5SJason Gunthorpe } 2696cc2c8e5SJason Gunthorpe EXPORT_SYMBOL(ib_device_get_by_name); 2706cc2c8e5SJason Gunthorpe 271d21943ddSLeon Romanovsky int ib_device_rename(struct ib_device *ibdev, const char *name) 272d21943ddSLeon Romanovsky { 273e3593b56SJason Gunthorpe int ret; 274d21943ddSLeon Romanovsky 275921eab11SJason Gunthorpe down_write(&devices_rwsem); 276e3593b56SJason Gunthorpe if (!strcmp(name, dev_name(&ibdev->dev))) { 277e3593b56SJason Gunthorpe ret = 0; 278e3593b56SJason Gunthorpe goto out; 279e3593b56SJason Gunthorpe } 280e3593b56SJason Gunthorpe 281344684e6SJason Gunthorpe if (__ib_device_get_by_name(name)) { 282d21943ddSLeon Romanovsky ret = -EEXIST; 283d21943ddSLeon Romanovsky goto out; 284d21943ddSLeon Romanovsky } 285d21943ddSLeon Romanovsky 286d21943ddSLeon Romanovsky ret = device_rename(&ibdev->dev, name); 287d21943ddSLeon Romanovsky if (ret) 288d21943ddSLeon Romanovsky goto out; 289d21943ddSLeon Romanovsky strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); 290d21943ddSLeon Romanovsky out: 291921eab11SJason Gunthorpe up_write(&devices_rwsem); 292d21943ddSLeon Romanovsky return ret; 293d21943ddSLeon Romanovsky } 294d21943ddSLeon Romanovsky 295e349f858SJason Gunthorpe static int alloc_name(struct ib_device *ibdev, const char *name) 2961da177e4SLinus Torvalds { 2971da177e4SLinus Torvalds struct ib_device *device; 2980df91bb6SJason Gunthorpe unsigned long index; 2993b88afd3SJason Gunthorpe struct ida inuse; 3003b88afd3SJason Gunthorpe int rc; 3011da177e4SLinus Torvalds int i; 3021da177e4SLinus Torvalds 303921eab11SJason Gunthorpe lockdep_assert_held_exclusive(&devices_rwsem); 3043b88afd3SJason Gunthorpe ida_init(&inuse); 3050df91bb6SJason Gunthorpe xa_for_each (&devices, index, device) { 306e349f858SJason Gunthorpe char buf[IB_DEVICE_NAME_MAX]; 307e349f858SJason Gunthorpe 308896de009SJason Gunthorpe if (sscanf(dev_name(&device->dev), name, &i) != 1) 3091da177e4SLinus Torvalds continue; 3103b88afd3SJason Gunthorpe if (i < 0 || i >= INT_MAX) 3111da177e4SLinus Torvalds continue; 3121da177e4SLinus Torvalds snprintf(buf, sizeof buf, name, i); 3133b88afd3SJason Gunthorpe if (strcmp(buf, dev_name(&device->dev)) != 0) 3143b88afd3SJason Gunthorpe continue; 3153b88afd3SJason Gunthorpe 3163b88afd3SJason Gunthorpe rc = ida_alloc_range(&inuse, i, i, GFP_KERNEL); 3173b88afd3SJason Gunthorpe if (rc < 0) 3183b88afd3SJason Gunthorpe goto out; 3191da177e4SLinus Torvalds } 3201da177e4SLinus Torvalds 3213b88afd3SJason Gunthorpe rc = ida_alloc(&inuse, GFP_KERNEL); 3223b88afd3SJason Gunthorpe if (rc < 0) 3233b88afd3SJason Gunthorpe goto out; 3241da177e4SLinus Torvalds 3253b88afd3SJason Gunthorpe rc = dev_set_name(&ibdev->dev, name, rc); 3263b88afd3SJason Gunthorpe out: 3273b88afd3SJason Gunthorpe ida_destroy(&inuse); 3283b88afd3SJason Gunthorpe return rc; 3291da177e4SLinus Torvalds } 3301da177e4SLinus Torvalds 33155aeed06SJason Gunthorpe static void ib_device_release(struct device *device) 33255aeed06SJason Gunthorpe { 33355aeed06SJason Gunthorpe struct ib_device *dev = container_of(device, struct ib_device, dev); 33455aeed06SJason Gunthorpe 335c2261dd7SJason Gunthorpe free_netdevs(dev); 336652432f3SJason Gunthorpe WARN_ON(refcount_read(&dev->refcount)); 33703db3a2dSMatan Barak ib_cache_release_one(dev); 338b34b269aSJason Gunthorpe ib_security_release_port_pkey_list(dev); 3390df91bb6SJason Gunthorpe xa_destroy(&dev->client_data); 340324e227eSJason Gunthorpe if (dev->port_data) 341324e227eSJason Gunthorpe kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu, 342324e227eSJason Gunthorpe pdata[0]), 343324e227eSJason Gunthorpe rcu_head); 344324e227eSJason Gunthorpe kfree_rcu(dev, rcu_head); 34555aeed06SJason Gunthorpe } 34655aeed06SJason Gunthorpe 34755aeed06SJason Gunthorpe static int ib_device_uevent(struct device *device, 34855aeed06SJason Gunthorpe struct kobj_uevent_env *env) 34955aeed06SJason Gunthorpe { 350896de009SJason Gunthorpe if (add_uevent_var(env, "NAME=%s", dev_name(device))) 35155aeed06SJason Gunthorpe return -ENOMEM; 35255aeed06SJason Gunthorpe 35355aeed06SJason Gunthorpe /* 35455aeed06SJason Gunthorpe * It would be nice to pass the node GUID with the event... 35555aeed06SJason Gunthorpe */ 35655aeed06SJason Gunthorpe 35755aeed06SJason Gunthorpe return 0; 35855aeed06SJason Gunthorpe } 35955aeed06SJason Gunthorpe 36055aeed06SJason Gunthorpe static struct class ib_class = { 36155aeed06SJason Gunthorpe .name = "infiniband", 36255aeed06SJason Gunthorpe .dev_release = ib_device_release, 36355aeed06SJason Gunthorpe .dev_uevent = ib_device_uevent, 36455aeed06SJason Gunthorpe }; 36555aeed06SJason Gunthorpe 366*cebe556bSParav Pandit static void rdma_init_coredev(struct ib_core_device *coredev, 367*cebe556bSParav Pandit struct ib_device *dev) 368*cebe556bSParav Pandit { 369*cebe556bSParav Pandit /* This BUILD_BUG_ON is intended to catch layout change 370*cebe556bSParav Pandit * of union of ib_core_device and device. 371*cebe556bSParav Pandit * dev must be the first element as ib_core and providers 372*cebe556bSParav Pandit * driver uses it. Adding anything in ib_core_device before 373*cebe556bSParav Pandit * device will break this assumption. 374*cebe556bSParav Pandit */ 375*cebe556bSParav Pandit BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) != 376*cebe556bSParav Pandit offsetof(struct ib_device, dev)); 377*cebe556bSParav Pandit 378*cebe556bSParav Pandit coredev->dev.class = &ib_class; 379*cebe556bSParav Pandit coredev->dev.groups = dev->groups; 380*cebe556bSParav Pandit device_initialize(&coredev->dev); 381*cebe556bSParav Pandit coredev->owner = dev; 382*cebe556bSParav Pandit INIT_LIST_HEAD(&coredev->port_list); 383*cebe556bSParav Pandit } 384*cebe556bSParav Pandit 3851da177e4SLinus Torvalds /** 386459cc69fSLeon Romanovsky * _ib_alloc_device - allocate an IB device struct 3871da177e4SLinus Torvalds * @size:size of structure to allocate 3881da177e4SLinus Torvalds * 3891da177e4SLinus Torvalds * Low-level drivers should use ib_alloc_device() to allocate &struct 3901da177e4SLinus Torvalds * ib_device. @size is the size of the structure to be allocated, 3911da177e4SLinus Torvalds * including any private data used by the low-level driver. 3921da177e4SLinus Torvalds * ib_dealloc_device() must be used to free structures allocated with 3931da177e4SLinus Torvalds * ib_alloc_device(). 3941da177e4SLinus Torvalds */ 395459cc69fSLeon Romanovsky struct ib_device *_ib_alloc_device(size_t size) 3961da177e4SLinus Torvalds { 39755aeed06SJason Gunthorpe struct ib_device *device; 3981da177e4SLinus Torvalds 39955aeed06SJason Gunthorpe if (WARN_ON(size < sizeof(struct ib_device))) 40055aeed06SJason Gunthorpe return NULL; 40155aeed06SJason Gunthorpe 40255aeed06SJason Gunthorpe device = kzalloc(size, GFP_KERNEL); 40355aeed06SJason Gunthorpe if (!device) 40455aeed06SJason Gunthorpe return NULL; 40555aeed06SJason Gunthorpe 40641eda65cSLeon Romanovsky if (rdma_restrack_init(device)) { 40741eda65cSLeon Romanovsky kfree(device); 40841eda65cSLeon Romanovsky return NULL; 40941eda65cSLeon Romanovsky } 41002d8883fSLeon Romanovsky 4115f8f5499SParav Pandit device->groups[0] = &ib_dev_attr_group; 412*cebe556bSParav Pandit rdma_init_coredev(&device->coredev, device); 41355aeed06SJason Gunthorpe 41455aeed06SJason Gunthorpe INIT_LIST_HEAD(&device->event_handler_list); 41555aeed06SJason Gunthorpe spin_lock_init(&device->event_handler_lock); 416d0899892SJason Gunthorpe mutex_init(&device->unregistration_lock); 4170df91bb6SJason Gunthorpe /* 4180df91bb6SJason Gunthorpe * client_data needs to be alloc because we don't want our mark to be 4190df91bb6SJason Gunthorpe * destroyed if the user stores NULL in the client data. 4200df91bb6SJason Gunthorpe */ 4210df91bb6SJason Gunthorpe xa_init_flags(&device->client_data, XA_FLAGS_ALLOC); 422921eab11SJason Gunthorpe init_rwsem(&device->client_data_rwsem); 42301b67117SParav Pandit init_completion(&device->unreg_completion); 424d0899892SJason Gunthorpe INIT_WORK(&device->unregistration_work, ib_unregister_work); 42555aeed06SJason Gunthorpe 42655aeed06SJason Gunthorpe return device; 4271da177e4SLinus Torvalds } 428459cc69fSLeon Romanovsky EXPORT_SYMBOL(_ib_alloc_device); 4291da177e4SLinus Torvalds 4301da177e4SLinus Torvalds /** 4311da177e4SLinus Torvalds * ib_dealloc_device - free an IB device struct 4321da177e4SLinus Torvalds * @device:structure to free 4331da177e4SLinus Torvalds * 4341da177e4SLinus Torvalds * Free a structure allocated with ib_alloc_device(). 4351da177e4SLinus Torvalds */ 4361da177e4SLinus Torvalds void ib_dealloc_device(struct ib_device *device) 4371da177e4SLinus Torvalds { 438d0899892SJason Gunthorpe if (device->ops.dealloc_driver) 439d0899892SJason Gunthorpe device->ops.dealloc_driver(device); 440d0899892SJason Gunthorpe 441d0899892SJason Gunthorpe /* 442d0899892SJason Gunthorpe * ib_unregister_driver() requires all devices to remain in the xarray 443d0899892SJason Gunthorpe * while their ops are callable. The last op we call is dealloc_driver 444d0899892SJason Gunthorpe * above. This is needed to create a fence on op callbacks prior to 445d0899892SJason Gunthorpe * allowing the driver module to unload. 446d0899892SJason Gunthorpe */ 447d0899892SJason Gunthorpe down_write(&devices_rwsem); 448d0899892SJason Gunthorpe if (xa_load(&devices, device->index) == device) 449d0899892SJason Gunthorpe xa_erase(&devices, device->index); 450d0899892SJason Gunthorpe up_write(&devices_rwsem); 451d0899892SJason Gunthorpe 452c2261dd7SJason Gunthorpe /* Expedite releasing netdev references */ 453c2261dd7SJason Gunthorpe free_netdevs(device); 454c2261dd7SJason Gunthorpe 4550df91bb6SJason Gunthorpe WARN_ON(!xa_empty(&device->client_data)); 456652432f3SJason Gunthorpe WARN_ON(refcount_read(&device->refcount)); 4570ad699c0SLeon Romanovsky rdma_restrack_clean(device); 458e155755eSParav Pandit /* Balances with device_initialize */ 459924b8900SLeon Romanovsky put_device(&device->dev); 4601da177e4SLinus Torvalds } 4611da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_device); 4621da177e4SLinus Torvalds 463921eab11SJason Gunthorpe /* 464921eab11SJason Gunthorpe * add_client_context() and remove_client_context() must be safe against 465921eab11SJason Gunthorpe * parallel calls on the same device - registration/unregistration of both the 466921eab11SJason Gunthorpe * device and client can be occurring in parallel. 467921eab11SJason Gunthorpe * 468921eab11SJason Gunthorpe * The routines need to be a fence, any caller must not return until the add 469921eab11SJason Gunthorpe * or remove is fully completed. 470921eab11SJason Gunthorpe */ 471921eab11SJason Gunthorpe static int add_client_context(struct ib_device *device, 472921eab11SJason Gunthorpe struct ib_client *client) 4731da177e4SLinus Torvalds { 474921eab11SJason Gunthorpe int ret = 0; 4751da177e4SLinus Torvalds 4766780c4faSGal Pressman if (!device->kverbs_provider && !client->no_kverbs_req) 477921eab11SJason Gunthorpe return 0; 4786780c4faSGal Pressman 479921eab11SJason Gunthorpe down_write(&device->client_data_rwsem); 480921eab11SJason Gunthorpe /* 481921eab11SJason Gunthorpe * Another caller to add_client_context got here first and has already 482921eab11SJason Gunthorpe * completely initialized context. 483921eab11SJason Gunthorpe */ 484921eab11SJason Gunthorpe if (xa_get_mark(&device->client_data, client->client_id, 485921eab11SJason Gunthorpe CLIENT_DATA_REGISTERED)) 486921eab11SJason Gunthorpe goto out; 487921eab11SJason Gunthorpe 488921eab11SJason Gunthorpe ret = xa_err(xa_store(&device->client_data, client->client_id, NULL, 489921eab11SJason Gunthorpe GFP_KERNEL)); 490921eab11SJason Gunthorpe if (ret) 491921eab11SJason Gunthorpe goto out; 492921eab11SJason Gunthorpe downgrade_write(&device->client_data_rwsem); 493921eab11SJason Gunthorpe if (client->add) 494921eab11SJason Gunthorpe client->add(device); 495921eab11SJason Gunthorpe 496921eab11SJason Gunthorpe /* Readers shall not see a client until add has been completed */ 4970df91bb6SJason Gunthorpe xa_set_mark(&device->client_data, client->client_id, 4980df91bb6SJason Gunthorpe CLIENT_DATA_REGISTERED); 499921eab11SJason Gunthorpe up_read(&device->client_data_rwsem); 500921eab11SJason Gunthorpe return 0; 5011da177e4SLinus Torvalds 502921eab11SJason Gunthorpe out: 503921eab11SJason Gunthorpe up_write(&device->client_data_rwsem); 504921eab11SJason Gunthorpe return ret; 505921eab11SJason Gunthorpe } 506921eab11SJason Gunthorpe 507921eab11SJason Gunthorpe static void remove_client_context(struct ib_device *device, 508921eab11SJason Gunthorpe unsigned int client_id) 509921eab11SJason Gunthorpe { 510921eab11SJason Gunthorpe struct ib_client *client; 511921eab11SJason Gunthorpe void *client_data; 512921eab11SJason Gunthorpe 513921eab11SJason Gunthorpe down_write(&device->client_data_rwsem); 514921eab11SJason Gunthorpe if (!xa_get_mark(&device->client_data, client_id, 515921eab11SJason Gunthorpe CLIENT_DATA_REGISTERED)) { 516921eab11SJason Gunthorpe up_write(&device->client_data_rwsem); 517921eab11SJason Gunthorpe return; 518921eab11SJason Gunthorpe } 519921eab11SJason Gunthorpe client_data = xa_load(&device->client_data, client_id); 520921eab11SJason Gunthorpe xa_clear_mark(&device->client_data, client_id, CLIENT_DATA_REGISTERED); 521921eab11SJason Gunthorpe client = xa_load(&clients, client_id); 522921eab11SJason Gunthorpe downgrade_write(&device->client_data_rwsem); 523921eab11SJason Gunthorpe 524921eab11SJason Gunthorpe /* 525921eab11SJason Gunthorpe * Notice we cannot be holding any exclusive locks when calling the 526921eab11SJason Gunthorpe * remove callback as the remove callback can recurse back into any 527921eab11SJason Gunthorpe * public functions in this module and thus try for any locks those 528921eab11SJason Gunthorpe * functions take. 529921eab11SJason Gunthorpe * 530921eab11SJason Gunthorpe * For this reason clients and drivers should not call the 531921eab11SJason Gunthorpe * unregistration functions will holdling any locks. 532921eab11SJason Gunthorpe * 533921eab11SJason Gunthorpe * It tempting to drop the client_data_rwsem too, but this is required 534921eab11SJason Gunthorpe * to ensure that unregister_client does not return until all clients 535921eab11SJason Gunthorpe * are completely unregistered, which is required to avoid module 536921eab11SJason Gunthorpe * unloading races. 537921eab11SJason Gunthorpe */ 538921eab11SJason Gunthorpe if (client->remove) 539921eab11SJason Gunthorpe client->remove(device, client_data); 540921eab11SJason Gunthorpe 541921eab11SJason Gunthorpe xa_erase(&device->client_data, client_id); 542921eab11SJason Gunthorpe up_read(&device->client_data_rwsem); 5431da177e4SLinus Torvalds } 5441da177e4SLinus Torvalds 545c2261dd7SJason Gunthorpe static int alloc_port_data(struct ib_device *device) 5465eb620c8SYosef Etigin { 547324e227eSJason Gunthorpe struct ib_port_data_rcu *pdata_rcu; 548ea1075edSJason Gunthorpe unsigned int port; 549c2261dd7SJason Gunthorpe 550c2261dd7SJason Gunthorpe if (device->port_data) 551c2261dd7SJason Gunthorpe return 0; 552c2261dd7SJason Gunthorpe 553c2261dd7SJason Gunthorpe /* This can only be called once the physical port range is defined */ 554c2261dd7SJason Gunthorpe if (WARN_ON(!device->phys_port_cnt)) 555c2261dd7SJason Gunthorpe return -EINVAL; 5565eb620c8SYosef Etigin 5578ceb1357SJason Gunthorpe /* 5588ceb1357SJason Gunthorpe * device->port_data is indexed directly by the port number to make 5597738613eSIra Weiny * access to this data as efficient as possible. 5607738613eSIra Weiny * 5618ceb1357SJason Gunthorpe * Therefore port_data is declared as a 1 based array with potential 5628ceb1357SJason Gunthorpe * empty slots at the beginning. 5637738613eSIra Weiny */ 564324e227eSJason Gunthorpe pdata_rcu = kzalloc(struct_size(pdata_rcu, pdata, 565324e227eSJason Gunthorpe rdma_end_port(device) + 1), 566324e227eSJason Gunthorpe GFP_KERNEL); 567324e227eSJason Gunthorpe if (!pdata_rcu) 56855aeed06SJason Gunthorpe return -ENOMEM; 569324e227eSJason Gunthorpe /* 570324e227eSJason Gunthorpe * The rcu_head is put in front of the port data array and the stored 571324e227eSJason Gunthorpe * pointer is adjusted since we never need to see that member until 572324e227eSJason Gunthorpe * kfree_rcu. 573324e227eSJason Gunthorpe */ 574324e227eSJason Gunthorpe device->port_data = pdata_rcu->pdata; 5755eb620c8SYosef Etigin 576ea1075edSJason Gunthorpe rdma_for_each_port (device, port) { 5778ceb1357SJason Gunthorpe struct ib_port_data *pdata = &device->port_data[port]; 5788ceb1357SJason Gunthorpe 579324e227eSJason Gunthorpe pdata->ib_dev = device; 5808ceb1357SJason Gunthorpe spin_lock_init(&pdata->pkey_list_lock); 5818ceb1357SJason Gunthorpe INIT_LIST_HEAD(&pdata->pkey_list); 582c2261dd7SJason Gunthorpe spin_lock_init(&pdata->netdev_lock); 583324e227eSJason Gunthorpe INIT_HLIST_NODE(&pdata->ndev_hash_link); 584c2261dd7SJason Gunthorpe } 585c2261dd7SJason Gunthorpe return 0; 586c2261dd7SJason Gunthorpe } 587c2261dd7SJason Gunthorpe 588c2261dd7SJason Gunthorpe static int verify_immutable(const struct ib_device *dev, u8 port) 589c2261dd7SJason Gunthorpe { 590c2261dd7SJason Gunthorpe return WARN_ON(!rdma_cap_ib_mad(dev, port) && 591c2261dd7SJason Gunthorpe rdma_max_mad_size(dev, port) != 0); 592c2261dd7SJason Gunthorpe } 593c2261dd7SJason Gunthorpe 594c2261dd7SJason Gunthorpe static int setup_port_data(struct ib_device *device) 595c2261dd7SJason Gunthorpe { 596c2261dd7SJason Gunthorpe unsigned int port; 597c2261dd7SJason Gunthorpe int ret; 598c2261dd7SJason Gunthorpe 599c2261dd7SJason Gunthorpe ret = alloc_port_data(device); 600c2261dd7SJason Gunthorpe if (ret) 601c2261dd7SJason Gunthorpe return ret; 602c2261dd7SJason Gunthorpe 603c2261dd7SJason Gunthorpe rdma_for_each_port (device, port) { 604c2261dd7SJason Gunthorpe struct ib_port_data *pdata = &device->port_data[port]; 6058ceb1357SJason Gunthorpe 6068ceb1357SJason Gunthorpe ret = device->ops.get_port_immutable(device, port, 6078ceb1357SJason Gunthorpe &pdata->immutable); 6085eb620c8SYosef Etigin if (ret) 6095eb620c8SYosef Etigin return ret; 61055aeed06SJason Gunthorpe 61155aeed06SJason Gunthorpe if (verify_immutable(device, port)) 61255aeed06SJason Gunthorpe return -EINVAL; 61355aeed06SJason Gunthorpe } 61455aeed06SJason Gunthorpe return 0; 6155eb620c8SYosef Etigin } 6165eb620c8SYosef Etigin 6179abb0d1bSLeon Romanovsky void ib_get_device_fw_str(struct ib_device *dev, char *str) 6185fa76c20SIra Weiny { 6193023a1e9SKamal Heib if (dev->ops.get_dev_fw_str) 6203023a1e9SKamal Heib dev->ops.get_dev_fw_str(dev, str); 6215fa76c20SIra Weiny else 6225fa76c20SIra Weiny str[0] = '\0'; 6235fa76c20SIra Weiny } 6245fa76c20SIra Weiny EXPORT_SYMBOL(ib_get_device_fw_str); 6255fa76c20SIra Weiny 6268f408ab6SDaniel Jurgens static void ib_policy_change_task(struct work_struct *work) 6278f408ab6SDaniel Jurgens { 6288f408ab6SDaniel Jurgens struct ib_device *dev; 6290df91bb6SJason Gunthorpe unsigned long index; 6308f408ab6SDaniel Jurgens 631921eab11SJason Gunthorpe down_read(&devices_rwsem); 6320df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 633ea1075edSJason Gunthorpe unsigned int i; 6348f408ab6SDaniel Jurgens 635ea1075edSJason Gunthorpe rdma_for_each_port (dev, i) { 6368f408ab6SDaniel Jurgens u64 sp; 6378f408ab6SDaniel Jurgens int ret = ib_get_cached_subnet_prefix(dev, 6388f408ab6SDaniel Jurgens i, 6398f408ab6SDaniel Jurgens &sp); 6408f408ab6SDaniel Jurgens 6418f408ab6SDaniel Jurgens WARN_ONCE(ret, 6428f408ab6SDaniel Jurgens "ib_get_cached_subnet_prefix err: %d, this should never happen here\n", 6438f408ab6SDaniel Jurgens ret); 644a750cfdeSDaniel Jurgens if (!ret) 6458f408ab6SDaniel Jurgens ib_security_cache_change(dev, i, sp); 6468f408ab6SDaniel Jurgens } 6478f408ab6SDaniel Jurgens } 648921eab11SJason Gunthorpe up_read(&devices_rwsem); 6498f408ab6SDaniel Jurgens } 6508f408ab6SDaniel Jurgens 6518f408ab6SDaniel Jurgens static int ib_security_change(struct notifier_block *nb, unsigned long event, 6528f408ab6SDaniel Jurgens void *lsm_data) 6538f408ab6SDaniel Jurgens { 6548f408ab6SDaniel Jurgens if (event != LSM_POLICY_CHANGE) 6558f408ab6SDaniel Jurgens return NOTIFY_DONE; 6568f408ab6SDaniel Jurgens 6578f408ab6SDaniel Jurgens schedule_work(&ib_policy_change_work); 658c66f6741SDaniel Jurgens ib_mad_agent_security_change(); 6598f408ab6SDaniel Jurgens 6608f408ab6SDaniel Jurgens return NOTIFY_OK; 6618f408ab6SDaniel Jurgens } 6628f408ab6SDaniel Jurgens 663ecc82c53SLeon Romanovsky /* 664d0899892SJason Gunthorpe * Assign the unique string device name and the unique device index. This is 665d0899892SJason Gunthorpe * undone by ib_dealloc_device. 666ecc82c53SLeon Romanovsky */ 6670df91bb6SJason Gunthorpe static int assign_name(struct ib_device *device, const char *name) 6680df91bb6SJason Gunthorpe { 6690df91bb6SJason Gunthorpe static u32 last_id; 6700df91bb6SJason Gunthorpe int ret; 671ecc82c53SLeon Romanovsky 672921eab11SJason Gunthorpe down_write(&devices_rwsem); 6730df91bb6SJason Gunthorpe /* Assign a unique name to the device */ 6740df91bb6SJason Gunthorpe if (strchr(name, '%')) 6750df91bb6SJason Gunthorpe ret = alloc_name(device, name); 6760df91bb6SJason Gunthorpe else 6770df91bb6SJason Gunthorpe ret = dev_set_name(&device->dev, name); 6780df91bb6SJason Gunthorpe if (ret) 6790df91bb6SJason Gunthorpe goto out; 680ecc82c53SLeon Romanovsky 6810df91bb6SJason Gunthorpe if (__ib_device_get_by_name(dev_name(&device->dev))) { 6820df91bb6SJason Gunthorpe ret = -ENFILE; 6830df91bb6SJason Gunthorpe goto out; 684ecc82c53SLeon Romanovsky } 6850df91bb6SJason Gunthorpe strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); 6860df91bb6SJason Gunthorpe 687ea295481SLinus Torvalds ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b, 688ea295481SLinus Torvalds &last_id, GFP_KERNEL); 689ea295481SLinus Torvalds if (ret > 0) 6900df91bb6SJason Gunthorpe ret = 0; 691921eab11SJason Gunthorpe 6920df91bb6SJason Gunthorpe out: 693921eab11SJason Gunthorpe up_write(&devices_rwsem); 6940df91bb6SJason Gunthorpe return ret; 6950df91bb6SJason Gunthorpe } 6960df91bb6SJason Gunthorpe 697548cb4fbSParav Pandit static void setup_dma_device(struct ib_device *device) 6981da177e4SLinus Torvalds { 69999db9494SBart Van Assche struct device *parent = device->dev.parent; 7001da177e4SLinus Torvalds 7010957c29fSBart Van Assche WARN_ON_ONCE(device->dma_device); 7020957c29fSBart Van Assche if (device->dev.dma_ops) { 7030957c29fSBart Van Assche /* 7040957c29fSBart Van Assche * The caller provided custom DMA operations. Copy the 7050957c29fSBart Van Assche * DMA-related fields that are used by e.g. dma_alloc_coherent() 7060957c29fSBart Van Assche * into device->dev. 7070957c29fSBart Van Assche */ 7080957c29fSBart Van Assche device->dma_device = &device->dev; 70902ee9da3SBart Van Assche if (!device->dev.dma_mask) { 71002ee9da3SBart Van Assche if (parent) 71199db9494SBart Van Assche device->dev.dma_mask = parent->dma_mask; 71202ee9da3SBart Van Assche else 71302ee9da3SBart Van Assche WARN_ON_ONCE(true); 71402ee9da3SBart Van Assche } 71502ee9da3SBart Van Assche if (!device->dev.coherent_dma_mask) { 71602ee9da3SBart Van Assche if (parent) 7170957c29fSBart Van Assche device->dev.coherent_dma_mask = 7180957c29fSBart Van Assche parent->coherent_dma_mask; 71902ee9da3SBart Van Assche else 72002ee9da3SBart Van Assche WARN_ON_ONCE(true); 72102ee9da3SBart Van Assche } 7220957c29fSBart Van Assche } else { 7230957c29fSBart Van Assche /* 7240957c29fSBart Van Assche * The caller did not provide custom DMA operations. Use the 7250957c29fSBart Van Assche * DMA mapping operations of the parent device. 7260957c29fSBart Van Assche */ 72702ee9da3SBart Van Assche WARN_ON_ONCE(!parent); 7280957c29fSBart Van Assche device->dma_device = parent; 7290957c29fSBart Van Assche } 730548cb4fbSParav Pandit } 731548cb4fbSParav Pandit 732921eab11SJason Gunthorpe /* 733921eab11SJason Gunthorpe * setup_device() allocates memory and sets up data that requires calling the 734921eab11SJason Gunthorpe * device ops, this is the only reason these actions are not done during 735921eab11SJason Gunthorpe * ib_alloc_device. It is undone by ib_dealloc_device(). 736921eab11SJason Gunthorpe */ 737548cb4fbSParav Pandit static int setup_device(struct ib_device *device) 738548cb4fbSParav Pandit { 739548cb4fbSParav Pandit struct ib_udata uhw = {.outlen = 0, .inlen = 0}; 740548cb4fbSParav Pandit int ret; 741548cb4fbSParav Pandit 742921eab11SJason Gunthorpe setup_dma_device(device); 743921eab11SJason Gunthorpe 744548cb4fbSParav Pandit ret = ib_device_check_mandatory(device); 745548cb4fbSParav Pandit if (ret) 746548cb4fbSParav Pandit return ret; 747548cb4fbSParav Pandit 7488ceb1357SJason Gunthorpe ret = setup_port_data(device); 749548cb4fbSParav Pandit if (ret) { 7508ceb1357SJason Gunthorpe dev_warn(&device->dev, "Couldn't create per-port data\n"); 751548cb4fbSParav Pandit return ret; 752548cb4fbSParav Pandit } 753548cb4fbSParav Pandit 754548cb4fbSParav Pandit memset(&device->attrs, 0, sizeof(device->attrs)); 7553023a1e9SKamal Heib ret = device->ops.query_device(device, &device->attrs, &uhw); 756548cb4fbSParav Pandit if (ret) { 757548cb4fbSParav Pandit dev_warn(&device->dev, 758548cb4fbSParav Pandit "Couldn't query the device attributes\n"); 759d45f89d5SJason Gunthorpe return ret; 760548cb4fbSParav Pandit } 761548cb4fbSParav Pandit 762548cb4fbSParav Pandit return 0; 763548cb4fbSParav Pandit } 764548cb4fbSParav Pandit 765921eab11SJason Gunthorpe static void disable_device(struct ib_device *device) 766921eab11SJason Gunthorpe { 767921eab11SJason Gunthorpe struct ib_client *client; 768921eab11SJason Gunthorpe 769921eab11SJason Gunthorpe WARN_ON(!refcount_read(&device->refcount)); 770921eab11SJason Gunthorpe 771921eab11SJason Gunthorpe down_write(&devices_rwsem); 772921eab11SJason Gunthorpe xa_clear_mark(&devices, device->index, DEVICE_REGISTERED); 773921eab11SJason Gunthorpe up_write(&devices_rwsem); 774921eab11SJason Gunthorpe 775921eab11SJason Gunthorpe down_read(&clients_rwsem); 776921eab11SJason Gunthorpe list_for_each_entry_reverse(client, &client_list, list) 777921eab11SJason Gunthorpe remove_client_context(device, client->client_id); 778921eab11SJason Gunthorpe up_read(&clients_rwsem); 779921eab11SJason Gunthorpe 780921eab11SJason Gunthorpe /* Pairs with refcount_set in enable_device */ 781921eab11SJason Gunthorpe ib_device_put(device); 782921eab11SJason Gunthorpe wait_for_completion(&device->unreg_completion); 783c2261dd7SJason Gunthorpe 784c2261dd7SJason Gunthorpe /* Expedite removing unregistered pointers from the hash table */ 785c2261dd7SJason Gunthorpe free_netdevs(device); 786921eab11SJason Gunthorpe } 787921eab11SJason Gunthorpe 788921eab11SJason Gunthorpe /* 789921eab11SJason Gunthorpe * An enabled device is visible to all clients and to all the public facing 790d0899892SJason Gunthorpe * APIs that return a device pointer. This always returns with a new get, even 791d0899892SJason Gunthorpe * if it fails. 792921eab11SJason Gunthorpe */ 793d0899892SJason Gunthorpe static int enable_device_and_get(struct ib_device *device) 794921eab11SJason Gunthorpe { 795921eab11SJason Gunthorpe struct ib_client *client; 796921eab11SJason Gunthorpe unsigned long index; 797d0899892SJason Gunthorpe int ret = 0; 798921eab11SJason Gunthorpe 799d0899892SJason Gunthorpe /* 800d0899892SJason Gunthorpe * One ref belongs to the xa and the other belongs to this 801d0899892SJason Gunthorpe * thread. This is needed to guard against parallel unregistration. 802d0899892SJason Gunthorpe */ 803d0899892SJason Gunthorpe refcount_set(&device->refcount, 2); 804921eab11SJason Gunthorpe down_write(&devices_rwsem); 805921eab11SJason Gunthorpe xa_set_mark(&devices, device->index, DEVICE_REGISTERED); 806d0899892SJason Gunthorpe 807d0899892SJason Gunthorpe /* 808d0899892SJason Gunthorpe * By using downgrade_write() we ensure that no other thread can clear 809d0899892SJason Gunthorpe * DEVICE_REGISTERED while we are completing the client setup. 810d0899892SJason Gunthorpe */ 811d0899892SJason Gunthorpe downgrade_write(&devices_rwsem); 812921eab11SJason Gunthorpe 813ca22354bSJason Gunthorpe if (device->ops.enable_driver) { 814ca22354bSJason Gunthorpe ret = device->ops.enable_driver(device); 815ca22354bSJason Gunthorpe if (ret) 816ca22354bSJason Gunthorpe goto out; 817ca22354bSJason Gunthorpe } 818ca22354bSJason Gunthorpe 819921eab11SJason Gunthorpe down_read(&clients_rwsem); 820921eab11SJason Gunthorpe xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) { 821921eab11SJason Gunthorpe ret = add_client_context(device, client); 822d0899892SJason Gunthorpe if (ret) 823d0899892SJason Gunthorpe break; 824d0899892SJason Gunthorpe } 825921eab11SJason Gunthorpe up_read(&clients_rwsem); 826ca22354bSJason Gunthorpe 827ca22354bSJason Gunthorpe out: 828d0899892SJason Gunthorpe up_read(&devices_rwsem); 829921eab11SJason Gunthorpe return ret; 830921eab11SJason Gunthorpe } 831921eab11SJason Gunthorpe 832548cb4fbSParav Pandit /** 833548cb4fbSParav Pandit * ib_register_device - Register an IB device with IB core 834548cb4fbSParav Pandit * @device:Device to register 835548cb4fbSParav Pandit * 836548cb4fbSParav Pandit * Low-level drivers use ib_register_device() to register their 837548cb4fbSParav Pandit * devices with the IB core. All registered clients will receive a 838548cb4fbSParav Pandit * callback for each device that is added. @device must be allocated 839548cb4fbSParav Pandit * with ib_alloc_device(). 840d0899892SJason Gunthorpe * 841d0899892SJason Gunthorpe * If the driver uses ops.dealloc_driver and calls any ib_unregister_device() 842d0899892SJason Gunthorpe * asynchronously then the device pointer may become freed as soon as this 843d0899892SJason Gunthorpe * function returns. 844548cb4fbSParav Pandit */ 845ea4baf7fSParav Pandit int ib_register_device(struct ib_device *device, const char *name) 846548cb4fbSParav Pandit { 847548cb4fbSParav Pandit int ret; 8481da177e4SLinus Torvalds 8490df91bb6SJason Gunthorpe ret = assign_name(device, name); 850e349f858SJason Gunthorpe if (ret) 851921eab11SJason Gunthorpe return ret; 8521da177e4SLinus Torvalds 853548cb4fbSParav Pandit ret = setup_device(device); 854548cb4fbSParav Pandit if (ret) 855d0899892SJason Gunthorpe return ret; 85603db3a2dSMatan Barak 857d45f89d5SJason Gunthorpe ret = ib_cache_setup_one(device); 858d45f89d5SJason Gunthorpe if (ret) { 859d45f89d5SJason Gunthorpe dev_warn(&device->dev, 860d45f89d5SJason Gunthorpe "Couldn't set up InfiniBand P_Key/GID cache\n"); 861d0899892SJason Gunthorpe return ret; 862d45f89d5SJason Gunthorpe } 863d45f89d5SJason Gunthorpe 8647527a7b1SParav Pandit ib_device_register_rdmacg(device); 8653e153a93SIra Weiny 8665f8f5499SParav Pandit ret = device_add(&device->dev); 8675f8f5499SParav Pandit if (ret) 8685f8f5499SParav Pandit goto cg_cleanup; 8695f8f5499SParav Pandit 870ea4baf7fSParav Pandit ret = ib_device_register_sysfs(device); 8711da177e4SLinus Torvalds if (ret) { 87243c7c851SJason Gunthorpe dev_warn(&device->dev, 87343c7c851SJason Gunthorpe "Couldn't register device with driver model\n"); 8745f8f5499SParav Pandit goto dev_cleanup; 8751da177e4SLinus Torvalds } 8761da177e4SLinus Torvalds 877d0899892SJason Gunthorpe ret = enable_device_and_get(device); 878d0899892SJason Gunthorpe if (ret) { 879d0899892SJason Gunthorpe void (*dealloc_fn)(struct ib_device *); 880d0899892SJason Gunthorpe 881d0899892SJason Gunthorpe /* 882d0899892SJason Gunthorpe * If we hit this error flow then we don't want to 883d0899892SJason Gunthorpe * automatically dealloc the device since the caller is 884d0899892SJason Gunthorpe * expected to call ib_dealloc_device() after 885d0899892SJason Gunthorpe * ib_register_device() fails. This is tricky due to the 886d0899892SJason Gunthorpe * possibility for a parallel unregistration along with this 887d0899892SJason Gunthorpe * error flow. Since we have a refcount here we know any 888d0899892SJason Gunthorpe * parallel flow is stopped in disable_device and will see the 889d0899892SJason Gunthorpe * NULL pointers, causing the responsibility to 890d0899892SJason Gunthorpe * ib_dealloc_device() to revert back to this thread. 891d0899892SJason Gunthorpe */ 892d0899892SJason Gunthorpe dealloc_fn = device->ops.dealloc_driver; 893d0899892SJason Gunthorpe device->ops.dealloc_driver = NULL; 894d0899892SJason Gunthorpe ib_device_put(device); 895d0899892SJason Gunthorpe __ib_unregister_device(device); 896d0899892SJason Gunthorpe device->ops.dealloc_driver = dealloc_fn; 897d0899892SJason Gunthorpe return ret; 898d0899892SJason Gunthorpe } 899d0899892SJason Gunthorpe ib_device_put(device); 9001da177e4SLinus Torvalds 9014be3a4faSParav Pandit return 0; 9024be3a4faSParav Pandit 9035f8f5499SParav Pandit dev_cleanup: 9045f8f5499SParav Pandit device_del(&device->dev); 9052fb4f4eaSParav Pandit cg_cleanup: 9062fb4f4eaSParav Pandit ib_device_unregister_rdmacg(device); 907d45f89d5SJason Gunthorpe ib_cache_cleanup_one(device); 9081da177e4SLinus Torvalds return ret; 9091da177e4SLinus Torvalds } 9101da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_device); 9111da177e4SLinus Torvalds 912d0899892SJason Gunthorpe /* Callers must hold a get on the device. */ 913d0899892SJason Gunthorpe static void __ib_unregister_device(struct ib_device *ib_dev) 914d0899892SJason Gunthorpe { 915d0899892SJason Gunthorpe /* 916d0899892SJason Gunthorpe * We have a registration lock so that all the calls to unregister are 917d0899892SJason Gunthorpe * fully fenced, once any unregister returns the device is truely 918d0899892SJason Gunthorpe * unregistered even if multiple callers are unregistering it at the 919d0899892SJason Gunthorpe * same time. This also interacts with the registration flow and 920d0899892SJason Gunthorpe * provides sane semantics if register and unregister are racing. 921d0899892SJason Gunthorpe */ 922d0899892SJason Gunthorpe mutex_lock(&ib_dev->unregistration_lock); 923d0899892SJason Gunthorpe if (!refcount_read(&ib_dev->refcount)) 924d0899892SJason Gunthorpe goto out; 925d0899892SJason Gunthorpe 926d0899892SJason Gunthorpe disable_device(ib_dev); 927d0899892SJason Gunthorpe ib_device_unregister_sysfs(ib_dev); 928d0899892SJason Gunthorpe device_del(&ib_dev->dev); 929d0899892SJason Gunthorpe ib_device_unregister_rdmacg(ib_dev); 930d0899892SJason Gunthorpe ib_cache_cleanup_one(ib_dev); 931d0899892SJason Gunthorpe 932d0899892SJason Gunthorpe /* 933d0899892SJason Gunthorpe * Drivers using the new flow may not call ib_dealloc_device except 934d0899892SJason Gunthorpe * in error unwind prior to registration success. 935d0899892SJason Gunthorpe */ 936d0899892SJason Gunthorpe if (ib_dev->ops.dealloc_driver) { 937d0899892SJason Gunthorpe WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1); 938d0899892SJason Gunthorpe ib_dealloc_device(ib_dev); 939d0899892SJason Gunthorpe } 940d0899892SJason Gunthorpe out: 941d0899892SJason Gunthorpe mutex_unlock(&ib_dev->unregistration_lock); 942d0899892SJason Gunthorpe } 943d0899892SJason Gunthorpe 9441da177e4SLinus Torvalds /** 9451da177e4SLinus Torvalds * ib_unregister_device - Unregister an IB device 946d0899892SJason Gunthorpe * @device: The device to unregister 9471da177e4SLinus Torvalds * 9481da177e4SLinus Torvalds * Unregister an IB device. All clients will receive a remove callback. 949d0899892SJason Gunthorpe * 950d0899892SJason Gunthorpe * Callers should call this routine only once, and protect against races with 951d0899892SJason Gunthorpe * registration. Typically it should only be called as part of a remove 952d0899892SJason Gunthorpe * callback in an implementation of driver core's struct device_driver and 953d0899892SJason Gunthorpe * related. 954d0899892SJason Gunthorpe * 955d0899892SJason Gunthorpe * If ops.dealloc_driver is used then ib_dev will be freed upon return from 956d0899892SJason Gunthorpe * this function. 9571da177e4SLinus Torvalds */ 958d0899892SJason Gunthorpe void ib_unregister_device(struct ib_device *ib_dev) 9591da177e4SLinus Torvalds { 960d0899892SJason Gunthorpe get_device(&ib_dev->dev); 961d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 962d0899892SJason Gunthorpe put_device(&ib_dev->dev); 9631da177e4SLinus Torvalds } 9641da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_device); 9651da177e4SLinus Torvalds 966d0899892SJason Gunthorpe /** 967d0899892SJason Gunthorpe * ib_unregister_device_and_put - Unregister a device while holding a 'get' 968d0899892SJason Gunthorpe * device: The device to unregister 969d0899892SJason Gunthorpe * 970d0899892SJason Gunthorpe * This is the same as ib_unregister_device(), except it includes an internal 971d0899892SJason Gunthorpe * ib_device_put() that should match a 'get' obtained by the caller. 972d0899892SJason Gunthorpe * 973d0899892SJason Gunthorpe * It is safe to call this routine concurrently from multiple threads while 974d0899892SJason Gunthorpe * holding the 'get'. When the function returns the device is fully 975d0899892SJason Gunthorpe * unregistered. 976d0899892SJason Gunthorpe * 977d0899892SJason Gunthorpe * Drivers using this flow MUST use the driver_unregister callback to clean up 978d0899892SJason Gunthorpe * their resources associated with the device and dealloc it. 979d0899892SJason Gunthorpe */ 980d0899892SJason Gunthorpe void ib_unregister_device_and_put(struct ib_device *ib_dev) 981d0899892SJason Gunthorpe { 982d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 983d0899892SJason Gunthorpe get_device(&ib_dev->dev); 984d0899892SJason Gunthorpe ib_device_put(ib_dev); 985d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 986d0899892SJason Gunthorpe put_device(&ib_dev->dev); 987d0899892SJason Gunthorpe } 988d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_device_and_put); 989d0899892SJason Gunthorpe 990d0899892SJason Gunthorpe /** 991d0899892SJason Gunthorpe * ib_unregister_driver - Unregister all IB devices for a driver 992d0899892SJason Gunthorpe * @driver_id: The driver to unregister 993d0899892SJason Gunthorpe * 994d0899892SJason Gunthorpe * This implements a fence for device unregistration. It only returns once all 995d0899892SJason Gunthorpe * devices associated with the driver_id have fully completed their 996d0899892SJason Gunthorpe * unregistration and returned from ib_unregister_device*(). 997d0899892SJason Gunthorpe * 998d0899892SJason Gunthorpe * If device's are not yet unregistered it goes ahead and starts unregistering 999d0899892SJason Gunthorpe * them. 1000d0899892SJason Gunthorpe * 1001d0899892SJason Gunthorpe * This does not block creation of new devices with the given driver_id, that 1002d0899892SJason Gunthorpe * is the responsibility of the caller. 1003d0899892SJason Gunthorpe */ 1004d0899892SJason Gunthorpe void ib_unregister_driver(enum rdma_driver_id driver_id) 1005d0899892SJason Gunthorpe { 1006d0899892SJason Gunthorpe struct ib_device *ib_dev; 1007d0899892SJason Gunthorpe unsigned long index; 1008d0899892SJason Gunthorpe 1009d0899892SJason Gunthorpe down_read(&devices_rwsem); 1010d0899892SJason Gunthorpe xa_for_each (&devices, index, ib_dev) { 1011d0899892SJason Gunthorpe if (ib_dev->driver_id != driver_id) 1012d0899892SJason Gunthorpe continue; 1013d0899892SJason Gunthorpe 1014d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1015d0899892SJason Gunthorpe up_read(&devices_rwsem); 1016d0899892SJason Gunthorpe 1017d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 1018d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1019d0899892SJason Gunthorpe 1020d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1021d0899892SJason Gunthorpe down_read(&devices_rwsem); 1022d0899892SJason Gunthorpe } 1023d0899892SJason Gunthorpe up_read(&devices_rwsem); 1024d0899892SJason Gunthorpe } 1025d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_driver); 1026d0899892SJason Gunthorpe 1027d0899892SJason Gunthorpe static void ib_unregister_work(struct work_struct *work) 1028d0899892SJason Gunthorpe { 1029d0899892SJason Gunthorpe struct ib_device *ib_dev = 1030d0899892SJason Gunthorpe container_of(work, struct ib_device, unregistration_work); 1031d0899892SJason Gunthorpe 1032d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1033d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1034d0899892SJason Gunthorpe } 1035d0899892SJason Gunthorpe 1036d0899892SJason Gunthorpe /** 1037d0899892SJason Gunthorpe * ib_unregister_device_queued - Unregister a device using a work queue 1038d0899892SJason Gunthorpe * device: The device to unregister 1039d0899892SJason Gunthorpe * 1040d0899892SJason Gunthorpe * This schedules an asynchronous unregistration using a WQ for the device. A 1041d0899892SJason Gunthorpe * driver should use this to avoid holding locks while doing unregistration, 1042d0899892SJason Gunthorpe * such as holding the RTNL lock. 1043d0899892SJason Gunthorpe * 1044d0899892SJason Gunthorpe * Drivers using this API must use ib_unregister_driver before module unload 1045d0899892SJason Gunthorpe * to ensure that all scheduled unregistrations have completed. 1046d0899892SJason Gunthorpe */ 1047d0899892SJason Gunthorpe void ib_unregister_device_queued(struct ib_device *ib_dev) 1048d0899892SJason Gunthorpe { 1049d0899892SJason Gunthorpe WARN_ON(!refcount_read(&ib_dev->refcount)); 1050d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 1051d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1052d0899892SJason Gunthorpe if (!queue_work(system_unbound_wq, &ib_dev->unregistration_work)) 1053d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1054d0899892SJason Gunthorpe } 1055d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_device_queued); 1056d0899892SJason Gunthorpe 1057e59178d8SJason Gunthorpe static int assign_client_id(struct ib_client *client) 1058e59178d8SJason Gunthorpe { 1059e59178d8SJason Gunthorpe int ret; 1060e59178d8SJason Gunthorpe 1061921eab11SJason Gunthorpe down_write(&clients_rwsem); 1062e59178d8SJason Gunthorpe /* 1063e59178d8SJason Gunthorpe * The add/remove callbacks must be called in FIFO/LIFO order. To 1064e59178d8SJason Gunthorpe * achieve this we assign client_ids so they are sorted in 1065e59178d8SJason Gunthorpe * registration order, and retain a linked list we can reverse iterate 1066e59178d8SJason Gunthorpe * to get the LIFO order. The extra linked list can go away if xarray 1067e59178d8SJason Gunthorpe * learns to reverse iterate. 1068e59178d8SJason Gunthorpe */ 1069ea295481SLinus Torvalds if (list_empty(&client_list)) { 1070e59178d8SJason Gunthorpe client->client_id = 0; 1071ea295481SLinus Torvalds } else { 1072ea295481SLinus Torvalds struct ib_client *last; 1073ea295481SLinus Torvalds 1074ea295481SLinus Torvalds last = list_last_entry(&client_list, struct ib_client, list); 1075ea295481SLinus Torvalds client->client_id = last->client_id + 1; 1076ea295481SLinus Torvalds } 1077ea295481SLinus Torvalds ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL); 1078e59178d8SJason Gunthorpe if (ret) 1079e59178d8SJason Gunthorpe goto out; 1080e59178d8SJason Gunthorpe 1081921eab11SJason Gunthorpe xa_set_mark(&clients, client->client_id, CLIENT_REGISTERED); 1082921eab11SJason Gunthorpe list_add_tail(&client->list, &client_list); 1083921eab11SJason Gunthorpe 1084e59178d8SJason Gunthorpe out: 1085921eab11SJason Gunthorpe up_write(&clients_rwsem); 1086e59178d8SJason Gunthorpe return ret; 1087e59178d8SJason Gunthorpe } 1088e59178d8SJason Gunthorpe 10891da177e4SLinus Torvalds /** 10901da177e4SLinus Torvalds * ib_register_client - Register an IB client 10911da177e4SLinus Torvalds * @client:Client to register 10921da177e4SLinus Torvalds * 10931da177e4SLinus Torvalds * Upper level users of the IB drivers can use ib_register_client() to 10941da177e4SLinus Torvalds * register callbacks for IB device addition and removal. When an IB 10951da177e4SLinus Torvalds * device is added, each registered client's add method will be called 10961da177e4SLinus Torvalds * (in the order the clients were registered), and when a device is 10971da177e4SLinus Torvalds * removed, each client's remove method will be called (in the reverse 10981da177e4SLinus Torvalds * order that clients were registered). In addition, when 10991da177e4SLinus Torvalds * ib_register_client() is called, the client will receive an add 11001da177e4SLinus Torvalds * callback for all devices already registered. 11011da177e4SLinus Torvalds */ 11021da177e4SLinus Torvalds int ib_register_client(struct ib_client *client) 11031da177e4SLinus Torvalds { 11041da177e4SLinus Torvalds struct ib_device *device; 11050df91bb6SJason Gunthorpe unsigned long index; 1106e59178d8SJason Gunthorpe int ret; 11071da177e4SLinus Torvalds 1108e59178d8SJason Gunthorpe ret = assign_client_id(client); 1109921eab11SJason Gunthorpe if (ret) 1110921eab11SJason Gunthorpe return ret; 1111921eab11SJason Gunthorpe 1112921eab11SJason Gunthorpe down_read(&devices_rwsem); 1113921eab11SJason Gunthorpe xa_for_each_marked (&devices, index, device, DEVICE_REGISTERED) { 1114921eab11SJason Gunthorpe ret = add_client_context(device, client); 1115e59178d8SJason Gunthorpe if (ret) { 1116921eab11SJason Gunthorpe up_read(&devices_rwsem); 1117921eab11SJason Gunthorpe ib_unregister_client(client); 1118e59178d8SJason Gunthorpe return ret; 1119e59178d8SJason Gunthorpe } 1120921eab11SJason Gunthorpe } 1121921eab11SJason Gunthorpe up_read(&devices_rwsem); 11221da177e4SLinus Torvalds return 0; 11231da177e4SLinus Torvalds } 11241da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_client); 11251da177e4SLinus Torvalds 11261da177e4SLinus Torvalds /** 11271da177e4SLinus Torvalds * ib_unregister_client - Unregister an IB client 11281da177e4SLinus Torvalds * @client:Client to unregister 11291da177e4SLinus Torvalds * 11301da177e4SLinus Torvalds * Upper level users use ib_unregister_client() to remove their client 11311da177e4SLinus Torvalds * registration. When ib_unregister_client() is called, the client 11321da177e4SLinus Torvalds * will receive a remove callback for each IB device still registered. 1133921eab11SJason Gunthorpe * 1134921eab11SJason Gunthorpe * This is a full fence, once it returns no client callbacks will be called, 1135921eab11SJason Gunthorpe * or are running in another thread. 11361da177e4SLinus Torvalds */ 11371da177e4SLinus Torvalds void ib_unregister_client(struct ib_client *client) 11381da177e4SLinus Torvalds { 11391da177e4SLinus Torvalds struct ib_device *device; 11400df91bb6SJason Gunthorpe unsigned long index; 11411da177e4SLinus Torvalds 1142921eab11SJason Gunthorpe down_write(&clients_rwsem); 1143e59178d8SJason Gunthorpe xa_clear_mark(&clients, client->client_id, CLIENT_REGISTERED); 1144921eab11SJason Gunthorpe up_write(&clients_rwsem); 1145921eab11SJason Gunthorpe /* 1146921eab11SJason Gunthorpe * Every device still known must be serialized to make sure we are 1147921eab11SJason Gunthorpe * done with the client callbacks before we return. 1148921eab11SJason Gunthorpe */ 1149921eab11SJason Gunthorpe down_read(&devices_rwsem); 1150921eab11SJason Gunthorpe xa_for_each (&devices, index, device) 1151921eab11SJason Gunthorpe remove_client_context(device, client->client_id); 1152921eab11SJason Gunthorpe up_read(&devices_rwsem); 11535aa44bb9SHaggai Eran 1154921eab11SJason Gunthorpe down_write(&clients_rwsem); 1155e59178d8SJason Gunthorpe list_del(&client->list); 1156e59178d8SJason Gunthorpe xa_erase(&clients, client->client_id); 1157921eab11SJason Gunthorpe up_write(&clients_rwsem); 11581da177e4SLinus Torvalds } 11591da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_client); 11601da177e4SLinus Torvalds 11611da177e4SLinus Torvalds /** 11629cd330d3SKrishna Kumar * ib_set_client_data - Set IB client context 11631da177e4SLinus Torvalds * @device:Device to set context for 11641da177e4SLinus Torvalds * @client:Client to set context for 11651da177e4SLinus Torvalds * @data:Context to set 11661da177e4SLinus Torvalds * 11670df91bb6SJason Gunthorpe * ib_set_client_data() sets client context data that can be retrieved with 11680df91bb6SJason Gunthorpe * ib_get_client_data(). This can only be called while the client is 11690df91bb6SJason Gunthorpe * registered to the device, once the ib_client remove() callback returns this 11700df91bb6SJason Gunthorpe * cannot be called. 11711da177e4SLinus Torvalds */ 11721da177e4SLinus Torvalds void ib_set_client_data(struct ib_device *device, struct ib_client *client, 11731da177e4SLinus Torvalds void *data) 11741da177e4SLinus Torvalds { 11750df91bb6SJason Gunthorpe void *rc; 11761da177e4SLinus Torvalds 11770df91bb6SJason Gunthorpe if (WARN_ON(IS_ERR(data))) 11780df91bb6SJason Gunthorpe data = NULL; 11791da177e4SLinus Torvalds 11800df91bb6SJason Gunthorpe rc = xa_store(&device->client_data, client->client_id, data, 11810df91bb6SJason Gunthorpe GFP_KERNEL); 11820df91bb6SJason Gunthorpe WARN_ON(xa_is_err(rc)); 11831da177e4SLinus Torvalds } 11841da177e4SLinus Torvalds EXPORT_SYMBOL(ib_set_client_data); 11851da177e4SLinus Torvalds 11861da177e4SLinus Torvalds /** 11871da177e4SLinus Torvalds * ib_register_event_handler - Register an IB event handler 11881da177e4SLinus Torvalds * @event_handler:Handler to register 11891da177e4SLinus Torvalds * 11901da177e4SLinus Torvalds * ib_register_event_handler() registers an event handler that will be 11911da177e4SLinus Torvalds * called back when asynchronous IB events occur (as defined in 11921da177e4SLinus Torvalds * chapter 11 of the InfiniBand Architecture Specification). This 11931da177e4SLinus Torvalds * callback may occur in interrupt context. 11941da177e4SLinus Torvalds */ 1195dcc9881eSLeon Romanovsky void ib_register_event_handler(struct ib_event_handler *event_handler) 11961da177e4SLinus Torvalds { 11971da177e4SLinus Torvalds unsigned long flags; 11981da177e4SLinus Torvalds 11991da177e4SLinus Torvalds spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 12001da177e4SLinus Torvalds list_add_tail(&event_handler->list, 12011da177e4SLinus Torvalds &event_handler->device->event_handler_list); 12021da177e4SLinus Torvalds spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 12031da177e4SLinus Torvalds } 12041da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_event_handler); 12051da177e4SLinus Torvalds 12061da177e4SLinus Torvalds /** 12071da177e4SLinus Torvalds * ib_unregister_event_handler - Unregister an event handler 12081da177e4SLinus Torvalds * @event_handler:Handler to unregister 12091da177e4SLinus Torvalds * 12101da177e4SLinus Torvalds * Unregister an event handler registered with 12111da177e4SLinus Torvalds * ib_register_event_handler(). 12121da177e4SLinus Torvalds */ 1213dcc9881eSLeon Romanovsky void ib_unregister_event_handler(struct ib_event_handler *event_handler) 12141da177e4SLinus Torvalds { 12151da177e4SLinus Torvalds unsigned long flags; 12161da177e4SLinus Torvalds 12171da177e4SLinus Torvalds spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 12181da177e4SLinus Torvalds list_del(&event_handler->list); 12191da177e4SLinus Torvalds spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 12201da177e4SLinus Torvalds } 12211da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_event_handler); 12221da177e4SLinus Torvalds 12231da177e4SLinus Torvalds /** 12241da177e4SLinus Torvalds * ib_dispatch_event - Dispatch an asynchronous event 12251da177e4SLinus Torvalds * @event:Event to dispatch 12261da177e4SLinus Torvalds * 12271da177e4SLinus Torvalds * Low-level drivers must call ib_dispatch_event() to dispatch the 12281da177e4SLinus Torvalds * event to all registered event handlers when an asynchronous event 12291da177e4SLinus Torvalds * occurs. 12301da177e4SLinus Torvalds */ 12311da177e4SLinus Torvalds void ib_dispatch_event(struct ib_event *event) 12321da177e4SLinus Torvalds { 12331da177e4SLinus Torvalds unsigned long flags; 12341da177e4SLinus Torvalds struct ib_event_handler *handler; 12351da177e4SLinus Torvalds 12361da177e4SLinus Torvalds spin_lock_irqsave(&event->device->event_handler_lock, flags); 12371da177e4SLinus Torvalds 12381da177e4SLinus Torvalds list_for_each_entry(handler, &event->device->event_handler_list, list) 12391da177e4SLinus Torvalds handler->handler(handler, event); 12401da177e4SLinus Torvalds 12411da177e4SLinus Torvalds spin_unlock_irqrestore(&event->device->event_handler_lock, flags); 12421da177e4SLinus Torvalds } 12431da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dispatch_event); 12441da177e4SLinus Torvalds 12451da177e4SLinus Torvalds /** 12461da177e4SLinus Torvalds * ib_query_port - Query IB port attributes 12471da177e4SLinus Torvalds * @device:Device to query 12481da177e4SLinus Torvalds * @port_num:Port number to query 12491da177e4SLinus Torvalds * @port_attr:Port attributes 12501da177e4SLinus Torvalds * 12511da177e4SLinus Torvalds * ib_query_port() returns the attributes of a port through the 12521da177e4SLinus Torvalds * @port_attr pointer. 12531da177e4SLinus Torvalds */ 12541da177e4SLinus Torvalds int ib_query_port(struct ib_device *device, 12551da177e4SLinus Torvalds u8 port_num, 12561da177e4SLinus Torvalds struct ib_port_attr *port_attr) 12571da177e4SLinus Torvalds { 1258fad61ad4SEli Cohen union ib_gid gid; 1259fad61ad4SEli Cohen int err; 1260fad61ad4SEli Cohen 126124dc831bSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 1262116c0074SRoland Dreier return -EINVAL; 1263116c0074SRoland Dreier 1264fad61ad4SEli Cohen memset(port_attr, 0, sizeof(*port_attr)); 12653023a1e9SKamal Heib err = device->ops.query_port(device, port_num, port_attr); 1266fad61ad4SEli Cohen if (err || port_attr->subnet_prefix) 1267fad61ad4SEli Cohen return err; 1268fad61ad4SEli Cohen 1269d7012467SEli Cohen if (rdma_port_get_link_layer(device, port_num) != IB_LINK_LAYER_INFINIBAND) 1270d7012467SEli Cohen return 0; 1271d7012467SEli Cohen 12723023a1e9SKamal Heib err = device->ops.query_gid(device, port_num, 0, &gid); 1273fad61ad4SEli Cohen if (err) 1274fad61ad4SEli Cohen return err; 1275fad61ad4SEli Cohen 1276fad61ad4SEli Cohen port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix); 1277fad61ad4SEli Cohen return 0; 12781da177e4SLinus Torvalds } 12791da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_port); 12801da177e4SLinus Torvalds 1281324e227eSJason Gunthorpe static void add_ndev_hash(struct ib_port_data *pdata) 1282324e227eSJason Gunthorpe { 1283324e227eSJason Gunthorpe unsigned long flags; 1284324e227eSJason Gunthorpe 1285324e227eSJason Gunthorpe might_sleep(); 1286324e227eSJason Gunthorpe 1287324e227eSJason Gunthorpe spin_lock_irqsave(&ndev_hash_lock, flags); 1288324e227eSJason Gunthorpe if (hash_hashed(&pdata->ndev_hash_link)) { 1289324e227eSJason Gunthorpe hash_del_rcu(&pdata->ndev_hash_link); 1290324e227eSJason Gunthorpe spin_unlock_irqrestore(&ndev_hash_lock, flags); 1291324e227eSJason Gunthorpe /* 1292324e227eSJason Gunthorpe * We cannot do hash_add_rcu after a hash_del_rcu until the 1293324e227eSJason Gunthorpe * grace period 1294324e227eSJason Gunthorpe */ 1295324e227eSJason Gunthorpe synchronize_rcu(); 1296324e227eSJason Gunthorpe spin_lock_irqsave(&ndev_hash_lock, flags); 1297324e227eSJason Gunthorpe } 1298324e227eSJason Gunthorpe if (pdata->netdev) 1299324e227eSJason Gunthorpe hash_add_rcu(ndev_hash, &pdata->ndev_hash_link, 1300324e227eSJason Gunthorpe (uintptr_t)pdata->netdev); 1301324e227eSJason Gunthorpe spin_unlock_irqrestore(&ndev_hash_lock, flags); 1302324e227eSJason Gunthorpe } 1303324e227eSJason Gunthorpe 13041da177e4SLinus Torvalds /** 1305c2261dd7SJason Gunthorpe * ib_device_set_netdev - Associate the ib_dev with an underlying net_device 1306c2261dd7SJason Gunthorpe * @ib_dev: Device to modify 1307c2261dd7SJason Gunthorpe * @ndev: net_device to affiliate, may be NULL 1308c2261dd7SJason Gunthorpe * @port: IB port the net_device is connected to 1309c2261dd7SJason Gunthorpe * 1310c2261dd7SJason Gunthorpe * Drivers should use this to link the ib_device to a netdev so the netdev 1311c2261dd7SJason Gunthorpe * shows up in interfaces like ib_enum_roce_netdev. Only one netdev may be 1312c2261dd7SJason Gunthorpe * affiliated with any port. 1313c2261dd7SJason Gunthorpe * 1314c2261dd7SJason Gunthorpe * The caller must ensure that the given ndev is not unregistered or 1315c2261dd7SJason Gunthorpe * unregistering, and that either the ib_device is unregistered or 1316c2261dd7SJason Gunthorpe * ib_device_set_netdev() is called with NULL when the ndev sends a 1317c2261dd7SJason Gunthorpe * NETDEV_UNREGISTER event. 1318c2261dd7SJason Gunthorpe */ 1319c2261dd7SJason Gunthorpe int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev, 1320c2261dd7SJason Gunthorpe unsigned int port) 1321c2261dd7SJason Gunthorpe { 1322c2261dd7SJason Gunthorpe struct net_device *old_ndev; 1323c2261dd7SJason Gunthorpe struct ib_port_data *pdata; 1324c2261dd7SJason Gunthorpe unsigned long flags; 1325c2261dd7SJason Gunthorpe int ret; 1326c2261dd7SJason Gunthorpe 1327c2261dd7SJason Gunthorpe /* 1328c2261dd7SJason Gunthorpe * Drivers wish to call this before ib_register_driver, so we have to 1329c2261dd7SJason Gunthorpe * setup the port data early. 1330c2261dd7SJason Gunthorpe */ 1331c2261dd7SJason Gunthorpe ret = alloc_port_data(ib_dev); 1332c2261dd7SJason Gunthorpe if (ret) 1333c2261dd7SJason Gunthorpe return ret; 1334c2261dd7SJason Gunthorpe 1335c2261dd7SJason Gunthorpe if (!rdma_is_port_valid(ib_dev, port)) 1336c2261dd7SJason Gunthorpe return -EINVAL; 1337c2261dd7SJason Gunthorpe 1338c2261dd7SJason Gunthorpe pdata = &ib_dev->port_data[port]; 1339c2261dd7SJason Gunthorpe spin_lock_irqsave(&pdata->netdev_lock, flags); 1340324e227eSJason Gunthorpe old_ndev = rcu_dereference_protected( 1341324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1342324e227eSJason Gunthorpe if (old_ndev == ndev) { 1343c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1344c2261dd7SJason Gunthorpe return 0; 1345c2261dd7SJason Gunthorpe } 1346c2261dd7SJason Gunthorpe 1347c2261dd7SJason Gunthorpe if (ndev) 1348c2261dd7SJason Gunthorpe dev_hold(ndev); 1349324e227eSJason Gunthorpe rcu_assign_pointer(pdata->netdev, ndev); 1350c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1351c2261dd7SJason Gunthorpe 1352324e227eSJason Gunthorpe add_ndev_hash(pdata); 1353c2261dd7SJason Gunthorpe if (old_ndev) 1354c2261dd7SJason Gunthorpe dev_put(old_ndev); 1355c2261dd7SJason Gunthorpe 1356c2261dd7SJason Gunthorpe return 0; 1357c2261dd7SJason Gunthorpe } 1358c2261dd7SJason Gunthorpe EXPORT_SYMBOL(ib_device_set_netdev); 1359c2261dd7SJason Gunthorpe 1360c2261dd7SJason Gunthorpe static void free_netdevs(struct ib_device *ib_dev) 1361c2261dd7SJason Gunthorpe { 1362c2261dd7SJason Gunthorpe unsigned long flags; 1363c2261dd7SJason Gunthorpe unsigned int port; 1364c2261dd7SJason Gunthorpe 1365c2261dd7SJason Gunthorpe rdma_for_each_port (ib_dev, port) { 1366c2261dd7SJason Gunthorpe struct ib_port_data *pdata = &ib_dev->port_data[port]; 1367324e227eSJason Gunthorpe struct net_device *ndev; 1368c2261dd7SJason Gunthorpe 1369c2261dd7SJason Gunthorpe spin_lock_irqsave(&pdata->netdev_lock, flags); 1370324e227eSJason Gunthorpe ndev = rcu_dereference_protected( 1371324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1372324e227eSJason Gunthorpe if (ndev) { 1373324e227eSJason Gunthorpe spin_lock(&ndev_hash_lock); 1374324e227eSJason Gunthorpe hash_del_rcu(&pdata->ndev_hash_link); 1375324e227eSJason Gunthorpe spin_unlock(&ndev_hash_lock); 1376324e227eSJason Gunthorpe 1377324e227eSJason Gunthorpe /* 1378324e227eSJason Gunthorpe * If this is the last dev_put there is still a 1379324e227eSJason Gunthorpe * synchronize_rcu before the netdev is kfreed, so we 1380324e227eSJason Gunthorpe * can continue to rely on unlocked pointer 1381324e227eSJason Gunthorpe * comparisons after the put 1382324e227eSJason Gunthorpe */ 1383324e227eSJason Gunthorpe rcu_assign_pointer(pdata->netdev, NULL); 1384324e227eSJason Gunthorpe dev_put(ndev); 1385c2261dd7SJason Gunthorpe } 1386c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1387c2261dd7SJason Gunthorpe } 1388c2261dd7SJason Gunthorpe } 1389c2261dd7SJason Gunthorpe 1390c2261dd7SJason Gunthorpe struct net_device *ib_device_get_netdev(struct ib_device *ib_dev, 1391c2261dd7SJason Gunthorpe unsigned int port) 1392c2261dd7SJason Gunthorpe { 1393c2261dd7SJason Gunthorpe struct ib_port_data *pdata; 1394c2261dd7SJason Gunthorpe struct net_device *res; 1395c2261dd7SJason Gunthorpe 1396c2261dd7SJason Gunthorpe if (!rdma_is_port_valid(ib_dev, port)) 1397c2261dd7SJason Gunthorpe return NULL; 1398c2261dd7SJason Gunthorpe 1399c2261dd7SJason Gunthorpe pdata = &ib_dev->port_data[port]; 1400c2261dd7SJason Gunthorpe 1401c2261dd7SJason Gunthorpe /* 1402c2261dd7SJason Gunthorpe * New drivers should use ib_device_set_netdev() not the legacy 1403c2261dd7SJason Gunthorpe * get_netdev(). 1404c2261dd7SJason Gunthorpe */ 1405c2261dd7SJason Gunthorpe if (ib_dev->ops.get_netdev) 1406c2261dd7SJason Gunthorpe res = ib_dev->ops.get_netdev(ib_dev, port); 1407c2261dd7SJason Gunthorpe else { 1408c2261dd7SJason Gunthorpe spin_lock(&pdata->netdev_lock); 1409324e227eSJason Gunthorpe res = rcu_dereference_protected( 1410324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1411c2261dd7SJason Gunthorpe if (res) 1412c2261dd7SJason Gunthorpe dev_hold(res); 1413c2261dd7SJason Gunthorpe spin_unlock(&pdata->netdev_lock); 1414c2261dd7SJason Gunthorpe } 1415c2261dd7SJason Gunthorpe 1416c2261dd7SJason Gunthorpe /* 1417c2261dd7SJason Gunthorpe * If we are starting to unregister expedite things by preventing 1418c2261dd7SJason Gunthorpe * propagation of an unregistering netdev. 1419c2261dd7SJason Gunthorpe */ 1420c2261dd7SJason Gunthorpe if (res && res->reg_state != NETREG_REGISTERED) { 1421c2261dd7SJason Gunthorpe dev_put(res); 1422c2261dd7SJason Gunthorpe return NULL; 1423c2261dd7SJason Gunthorpe } 1424c2261dd7SJason Gunthorpe 1425c2261dd7SJason Gunthorpe return res; 1426c2261dd7SJason Gunthorpe } 1427c2261dd7SJason Gunthorpe 1428c2261dd7SJason Gunthorpe /** 1429324e227eSJason Gunthorpe * ib_device_get_by_netdev - Find an IB device associated with a netdev 1430324e227eSJason Gunthorpe * @ndev: netdev to locate 1431324e227eSJason Gunthorpe * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 1432324e227eSJason Gunthorpe * 1433324e227eSJason Gunthorpe * Find and hold an ib_device that is associated with a netdev via 1434324e227eSJason Gunthorpe * ib_device_set_netdev(). The caller must call ib_device_put() on the 1435324e227eSJason Gunthorpe * returned pointer. 1436324e227eSJason Gunthorpe */ 1437324e227eSJason Gunthorpe struct ib_device *ib_device_get_by_netdev(struct net_device *ndev, 1438324e227eSJason Gunthorpe enum rdma_driver_id driver_id) 1439324e227eSJason Gunthorpe { 1440324e227eSJason Gunthorpe struct ib_device *res = NULL; 1441324e227eSJason Gunthorpe struct ib_port_data *cur; 1442324e227eSJason Gunthorpe 1443324e227eSJason Gunthorpe rcu_read_lock(); 1444324e227eSJason Gunthorpe hash_for_each_possible_rcu (ndev_hash, cur, ndev_hash_link, 1445324e227eSJason Gunthorpe (uintptr_t)ndev) { 1446324e227eSJason Gunthorpe if (rcu_access_pointer(cur->netdev) == ndev && 1447324e227eSJason Gunthorpe (driver_id == RDMA_DRIVER_UNKNOWN || 1448324e227eSJason Gunthorpe cur->ib_dev->driver_id == driver_id) && 1449324e227eSJason Gunthorpe ib_device_try_get(cur->ib_dev)) { 1450324e227eSJason Gunthorpe res = cur->ib_dev; 1451324e227eSJason Gunthorpe break; 1452324e227eSJason Gunthorpe } 1453324e227eSJason Gunthorpe } 1454324e227eSJason Gunthorpe rcu_read_unlock(); 1455324e227eSJason Gunthorpe 1456324e227eSJason Gunthorpe return res; 1457324e227eSJason Gunthorpe } 1458324e227eSJason Gunthorpe EXPORT_SYMBOL(ib_device_get_by_netdev); 1459324e227eSJason Gunthorpe 1460324e227eSJason Gunthorpe /** 146103db3a2dSMatan Barak * ib_enum_roce_netdev - enumerate all RoCE ports 146203db3a2dSMatan Barak * @ib_dev : IB device we want to query 146303db3a2dSMatan Barak * @filter: Should we call the callback? 146403db3a2dSMatan Barak * @filter_cookie: Cookie passed to filter 146503db3a2dSMatan Barak * @cb: Callback to call for each found RoCE ports 146603db3a2dSMatan Barak * @cookie: Cookie passed back to the callback 146703db3a2dSMatan Barak * 146803db3a2dSMatan Barak * Enumerates all of the physical RoCE ports of ib_dev 146903db3a2dSMatan Barak * which are related to netdevice and calls callback() on each 147003db3a2dSMatan Barak * device for which filter() function returns non zero. 147103db3a2dSMatan Barak */ 147203db3a2dSMatan Barak void ib_enum_roce_netdev(struct ib_device *ib_dev, 147303db3a2dSMatan Barak roce_netdev_filter filter, 147403db3a2dSMatan Barak void *filter_cookie, 147503db3a2dSMatan Barak roce_netdev_callback cb, 147603db3a2dSMatan Barak void *cookie) 147703db3a2dSMatan Barak { 1478ea1075edSJason Gunthorpe unsigned int port; 147903db3a2dSMatan Barak 1480ea1075edSJason Gunthorpe rdma_for_each_port (ib_dev, port) 148103db3a2dSMatan Barak if (rdma_protocol_roce(ib_dev, port)) { 1482c2261dd7SJason Gunthorpe struct net_device *idev = 1483c2261dd7SJason Gunthorpe ib_device_get_netdev(ib_dev, port); 148403db3a2dSMatan Barak 148503db3a2dSMatan Barak if (filter(ib_dev, port, idev, filter_cookie)) 148603db3a2dSMatan Barak cb(ib_dev, port, idev, cookie); 148703db3a2dSMatan Barak 148803db3a2dSMatan Barak if (idev) 148903db3a2dSMatan Barak dev_put(idev); 149003db3a2dSMatan Barak } 149103db3a2dSMatan Barak } 149203db3a2dSMatan Barak 149303db3a2dSMatan Barak /** 149403db3a2dSMatan Barak * ib_enum_all_roce_netdevs - enumerate all RoCE devices 149503db3a2dSMatan Barak * @filter: Should we call the callback? 149603db3a2dSMatan Barak * @filter_cookie: Cookie passed to filter 149703db3a2dSMatan Barak * @cb: Callback to call for each found RoCE ports 149803db3a2dSMatan Barak * @cookie: Cookie passed back to the callback 149903db3a2dSMatan Barak * 150003db3a2dSMatan Barak * Enumerates all RoCE devices' physical ports which are related 150103db3a2dSMatan Barak * to netdevices and calls callback() on each device for which 150203db3a2dSMatan Barak * filter() function returns non zero. 150303db3a2dSMatan Barak */ 150403db3a2dSMatan Barak void ib_enum_all_roce_netdevs(roce_netdev_filter filter, 150503db3a2dSMatan Barak void *filter_cookie, 150603db3a2dSMatan Barak roce_netdev_callback cb, 150703db3a2dSMatan Barak void *cookie) 150803db3a2dSMatan Barak { 150903db3a2dSMatan Barak struct ib_device *dev; 15100df91bb6SJason Gunthorpe unsigned long index; 151103db3a2dSMatan Barak 1512921eab11SJason Gunthorpe down_read(&devices_rwsem); 15130df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) 151403db3a2dSMatan Barak ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); 1515921eab11SJason Gunthorpe up_read(&devices_rwsem); 151603db3a2dSMatan Barak } 151703db3a2dSMatan Barak 151803db3a2dSMatan Barak /** 15198030c835SLeon Romanovsky * ib_enum_all_devs - enumerate all ib_devices 15208030c835SLeon Romanovsky * @cb: Callback to call for each found ib_device 15218030c835SLeon Romanovsky * 15228030c835SLeon Romanovsky * Enumerates all ib_devices and calls callback() on each device. 15238030c835SLeon Romanovsky */ 15248030c835SLeon Romanovsky int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb, 15258030c835SLeon Romanovsky struct netlink_callback *cb) 15268030c835SLeon Romanovsky { 15270df91bb6SJason Gunthorpe unsigned long index; 15288030c835SLeon Romanovsky struct ib_device *dev; 15298030c835SLeon Romanovsky unsigned int idx = 0; 15308030c835SLeon Romanovsky int ret = 0; 15318030c835SLeon Romanovsky 1532921eab11SJason Gunthorpe down_read(&devices_rwsem); 15330df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 15348030c835SLeon Romanovsky ret = nldev_cb(dev, skb, cb, idx); 15358030c835SLeon Romanovsky if (ret) 15368030c835SLeon Romanovsky break; 15378030c835SLeon Romanovsky idx++; 15388030c835SLeon Romanovsky } 1539921eab11SJason Gunthorpe up_read(&devices_rwsem); 15408030c835SLeon Romanovsky return ret; 15418030c835SLeon Romanovsky } 15428030c835SLeon Romanovsky 15438030c835SLeon Romanovsky /** 15441da177e4SLinus Torvalds * ib_query_pkey - Get P_Key table entry 15451da177e4SLinus Torvalds * @device:Device to query 15461da177e4SLinus Torvalds * @port_num:Port number to query 15471da177e4SLinus Torvalds * @index:P_Key table index to query 15481da177e4SLinus Torvalds * @pkey:Returned P_Key 15491da177e4SLinus Torvalds * 15501da177e4SLinus Torvalds * ib_query_pkey() fetches the specified P_Key table entry. 15511da177e4SLinus Torvalds */ 15521da177e4SLinus Torvalds int ib_query_pkey(struct ib_device *device, 15531da177e4SLinus Torvalds u8 port_num, u16 index, u16 *pkey) 15541da177e4SLinus Torvalds { 15559af3f5cfSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 15569af3f5cfSYuval Shaia return -EINVAL; 15579af3f5cfSYuval Shaia 15583023a1e9SKamal Heib return device->ops.query_pkey(device, port_num, index, pkey); 15591da177e4SLinus Torvalds } 15601da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_pkey); 15611da177e4SLinus Torvalds 15621da177e4SLinus Torvalds /** 15631da177e4SLinus Torvalds * ib_modify_device - Change IB device attributes 15641da177e4SLinus Torvalds * @device:Device to modify 15651da177e4SLinus Torvalds * @device_modify_mask:Mask of attributes to change 15661da177e4SLinus Torvalds * @device_modify:New attribute values 15671da177e4SLinus Torvalds * 15681da177e4SLinus Torvalds * ib_modify_device() changes a device's attributes as specified by 15691da177e4SLinus Torvalds * the @device_modify_mask and @device_modify structure. 15701da177e4SLinus Torvalds */ 15711da177e4SLinus Torvalds int ib_modify_device(struct ib_device *device, 15721da177e4SLinus Torvalds int device_modify_mask, 15731da177e4SLinus Torvalds struct ib_device_modify *device_modify) 15741da177e4SLinus Torvalds { 15753023a1e9SKamal Heib if (!device->ops.modify_device) 157610e1b54bSBart Van Assche return -ENOSYS; 157710e1b54bSBart Van Assche 15783023a1e9SKamal Heib return device->ops.modify_device(device, device_modify_mask, 15791da177e4SLinus Torvalds device_modify); 15801da177e4SLinus Torvalds } 15811da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_device); 15821da177e4SLinus Torvalds 15831da177e4SLinus Torvalds /** 15841da177e4SLinus Torvalds * ib_modify_port - Modifies the attributes for the specified port. 15851da177e4SLinus Torvalds * @device: The device to modify. 15861da177e4SLinus Torvalds * @port_num: The number of the port to modify. 15871da177e4SLinus Torvalds * @port_modify_mask: Mask used to specify which attributes of the port 15881da177e4SLinus Torvalds * to change. 15891da177e4SLinus Torvalds * @port_modify: New attribute values for the port. 15901da177e4SLinus Torvalds * 15911da177e4SLinus Torvalds * ib_modify_port() changes a port's attributes as specified by the 15921da177e4SLinus Torvalds * @port_modify_mask and @port_modify structure. 15931da177e4SLinus Torvalds */ 15941da177e4SLinus Torvalds int ib_modify_port(struct ib_device *device, 15951da177e4SLinus Torvalds u8 port_num, int port_modify_mask, 15961da177e4SLinus Torvalds struct ib_port_modify *port_modify) 15971da177e4SLinus Torvalds { 159861e0962dSSelvin Xavier int rc; 159910e1b54bSBart Van Assche 160024dc831bSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 1601116c0074SRoland Dreier return -EINVAL; 1602116c0074SRoland Dreier 16033023a1e9SKamal Heib if (device->ops.modify_port) 16043023a1e9SKamal Heib rc = device->ops.modify_port(device, port_num, 16053023a1e9SKamal Heib port_modify_mask, 16061da177e4SLinus Torvalds port_modify); 160761e0962dSSelvin Xavier else 160861e0962dSSelvin Xavier rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS; 160961e0962dSSelvin Xavier return rc; 16101da177e4SLinus Torvalds } 16111da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_port); 16121da177e4SLinus Torvalds 16135eb620c8SYosef Etigin /** 16145eb620c8SYosef Etigin * ib_find_gid - Returns the port number and GID table index where 1615dbb12562SParav Pandit * a specified GID value occurs. Its searches only for IB link layer. 16165eb620c8SYosef Etigin * @device: The device to query. 16175eb620c8SYosef Etigin * @gid: The GID value to search for. 16185eb620c8SYosef Etigin * @port_num: The port number of the device where the GID value was found. 16195eb620c8SYosef Etigin * @index: The index into the GID table where the GID was found. This 16205eb620c8SYosef Etigin * parameter may be NULL. 16215eb620c8SYosef Etigin */ 16225eb620c8SYosef Etigin int ib_find_gid(struct ib_device *device, union ib_gid *gid, 1623b26c4a11SParav Pandit u8 *port_num, u16 *index) 16245eb620c8SYosef Etigin { 16255eb620c8SYosef Etigin union ib_gid tmp_gid; 1626ea1075edSJason Gunthorpe unsigned int port; 1627ea1075edSJason Gunthorpe int ret, i; 16285eb620c8SYosef Etigin 1629ea1075edSJason Gunthorpe rdma_for_each_port (device, port) { 163022d24f75SParav Pandit if (!rdma_protocol_ib(device, port)) 1631b39ffa1dSMatan Barak continue; 1632b39ffa1dSMatan Barak 16338ceb1357SJason Gunthorpe for (i = 0; i < device->port_data[port].immutable.gid_tbl_len; 16348ceb1357SJason Gunthorpe ++i) { 16351dfce294SParav Pandit ret = rdma_query_gid(device, port, i, &tmp_gid); 16365eb620c8SYosef Etigin if (ret) 16375eb620c8SYosef Etigin return ret; 16385eb620c8SYosef Etigin if (!memcmp(&tmp_gid, gid, sizeof *gid)) { 16395eb620c8SYosef Etigin *port_num = port; 16405eb620c8SYosef Etigin if (index) 16415eb620c8SYosef Etigin *index = i; 16425eb620c8SYosef Etigin return 0; 16435eb620c8SYosef Etigin } 16445eb620c8SYosef Etigin } 16455eb620c8SYosef Etigin } 16465eb620c8SYosef Etigin 16475eb620c8SYosef Etigin return -ENOENT; 16485eb620c8SYosef Etigin } 16495eb620c8SYosef Etigin EXPORT_SYMBOL(ib_find_gid); 16505eb620c8SYosef Etigin 16515eb620c8SYosef Etigin /** 16525eb620c8SYosef Etigin * ib_find_pkey - Returns the PKey table index where a specified 16535eb620c8SYosef Etigin * PKey value occurs. 16545eb620c8SYosef Etigin * @device: The device to query. 16555eb620c8SYosef Etigin * @port_num: The port number of the device to search for the PKey. 16565eb620c8SYosef Etigin * @pkey: The PKey value to search for. 16575eb620c8SYosef Etigin * @index: The index into the PKey table where the PKey was found. 16585eb620c8SYosef Etigin */ 16595eb620c8SYosef Etigin int ib_find_pkey(struct ib_device *device, 16605eb620c8SYosef Etigin u8 port_num, u16 pkey, u16 *index) 16615eb620c8SYosef Etigin { 16625eb620c8SYosef Etigin int ret, i; 16635eb620c8SYosef Etigin u16 tmp_pkey; 1664ff7166c4SJack Morgenstein int partial_ix = -1; 16655eb620c8SYosef Etigin 16668ceb1357SJason Gunthorpe for (i = 0; i < device->port_data[port_num].immutable.pkey_tbl_len; 16678ceb1357SJason Gunthorpe ++i) { 16685eb620c8SYosef Etigin ret = ib_query_pkey(device, port_num, i, &tmp_pkey); 16695eb620c8SYosef Etigin if (ret) 16705eb620c8SYosef Etigin return ret; 167136026eccSMoni Shoua if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) { 1672ff7166c4SJack Morgenstein /* if there is full-member pkey take it.*/ 1673ff7166c4SJack Morgenstein if (tmp_pkey & 0x8000) { 16745eb620c8SYosef Etigin *index = i; 16755eb620c8SYosef Etigin return 0; 16765eb620c8SYosef Etigin } 1677ff7166c4SJack Morgenstein if (partial_ix < 0) 1678ff7166c4SJack Morgenstein partial_ix = i; 1679ff7166c4SJack Morgenstein } 16805eb620c8SYosef Etigin } 16815eb620c8SYosef Etigin 1682ff7166c4SJack Morgenstein /*no full-member, if exists take the limited*/ 1683ff7166c4SJack Morgenstein if (partial_ix >= 0) { 1684ff7166c4SJack Morgenstein *index = partial_ix; 1685ff7166c4SJack Morgenstein return 0; 1686ff7166c4SJack Morgenstein } 16875eb620c8SYosef Etigin return -ENOENT; 16885eb620c8SYosef Etigin } 16895eb620c8SYosef Etigin EXPORT_SYMBOL(ib_find_pkey); 16905eb620c8SYosef Etigin 16919268f72dSYotam Kenneth /** 16929268f72dSYotam Kenneth * ib_get_net_dev_by_params() - Return the appropriate net_dev 16939268f72dSYotam Kenneth * for a received CM request 16949268f72dSYotam Kenneth * @dev: An RDMA device on which the request has been received. 16959268f72dSYotam Kenneth * @port: Port number on the RDMA device. 16969268f72dSYotam Kenneth * @pkey: The Pkey the request came on. 16979268f72dSYotam Kenneth * @gid: A GID that the net_dev uses to communicate. 16989268f72dSYotam Kenneth * @addr: Contains the IP address that the request specified as its 16999268f72dSYotam Kenneth * destination. 1700921eab11SJason Gunthorpe * 17019268f72dSYotam Kenneth */ 17029268f72dSYotam Kenneth struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, 17039268f72dSYotam Kenneth u8 port, 17049268f72dSYotam Kenneth u16 pkey, 17059268f72dSYotam Kenneth const union ib_gid *gid, 17069268f72dSYotam Kenneth const struct sockaddr *addr) 17079268f72dSYotam Kenneth { 17089268f72dSYotam Kenneth struct net_device *net_dev = NULL; 17090df91bb6SJason Gunthorpe unsigned long index; 17100df91bb6SJason Gunthorpe void *client_data; 17119268f72dSYotam Kenneth 17129268f72dSYotam Kenneth if (!rdma_protocol_ib(dev, port)) 17139268f72dSYotam Kenneth return NULL; 17149268f72dSYotam Kenneth 1715921eab11SJason Gunthorpe /* 1716921eab11SJason Gunthorpe * Holding the read side guarantees that the client will not become 1717921eab11SJason Gunthorpe * unregistered while we are calling get_net_dev_by_params() 1718921eab11SJason Gunthorpe */ 1719921eab11SJason Gunthorpe down_read(&dev->client_data_rwsem); 17200df91bb6SJason Gunthorpe xan_for_each_marked (&dev->client_data, index, client_data, 17210df91bb6SJason Gunthorpe CLIENT_DATA_REGISTERED) { 17220df91bb6SJason Gunthorpe struct ib_client *client = xa_load(&clients, index); 17239268f72dSYotam Kenneth 17240df91bb6SJason Gunthorpe if (!client || !client->get_net_dev_by_params) 17259268f72dSYotam Kenneth continue; 17269268f72dSYotam Kenneth 17270df91bb6SJason Gunthorpe net_dev = client->get_net_dev_by_params(dev, port, pkey, gid, 17280df91bb6SJason Gunthorpe addr, client_data); 17299268f72dSYotam Kenneth if (net_dev) 17309268f72dSYotam Kenneth break; 17319268f72dSYotam Kenneth } 1732921eab11SJason Gunthorpe up_read(&dev->client_data_rwsem); 17339268f72dSYotam Kenneth 17349268f72dSYotam Kenneth return net_dev; 17359268f72dSYotam Kenneth } 17369268f72dSYotam Kenneth EXPORT_SYMBOL(ib_get_net_dev_by_params); 17379268f72dSYotam Kenneth 1738521ed0d9SKamal Heib void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) 1739521ed0d9SKamal Heib { 17403023a1e9SKamal Heib struct ib_device_ops *dev_ops = &dev->ops; 1741521ed0d9SKamal Heib #define SET_DEVICE_OP(ptr, name) \ 1742521ed0d9SKamal Heib do { \ 1743521ed0d9SKamal Heib if (ops->name) \ 1744521ed0d9SKamal Heib if (!((ptr)->name)) \ 1745521ed0d9SKamal Heib (ptr)->name = ops->name; \ 1746521ed0d9SKamal Heib } while (0) 1747521ed0d9SKamal Heib 174830471d4bSLeon Romanovsky #define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name) 174930471d4bSLeon Romanovsky 17503023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, add_gid); 17512f1927b0SMoni Shoua SET_DEVICE_OP(dev_ops, advise_mr); 17523023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_dm); 17533023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_fmr); 17543023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_hw_stats); 17553023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_mr); 17563023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_mw); 17573023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_pd); 17583023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_rdma_netdev); 17593023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_ucontext); 17603023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_xrcd); 17613023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, attach_mcast); 17623023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, check_mr_status); 17633023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_ah); 17643023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_counters); 17653023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_cq); 17663023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_flow); 17673023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_flow_action_esp); 17683023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_qp); 17693023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_rwq_ind_table); 17703023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_srq); 17713023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_wq); 17723023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_dm); 1773d0899892SJason Gunthorpe SET_DEVICE_OP(dev_ops, dealloc_driver); 17743023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_fmr); 17753023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_mw); 17763023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_pd); 17773023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_ucontext); 17783023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_xrcd); 17793023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, del_gid); 17803023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dereg_mr); 17813023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_ah); 17823023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_counters); 17833023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_cq); 17843023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_flow); 17853023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_flow_action); 17863023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_qp); 17873023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table); 17883023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_srq); 17893023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_wq); 17903023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, detach_mcast); 17913023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, disassociate_ucontext); 17923023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, drain_rq); 17933023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, drain_sq); 1794ca22354bSJason Gunthorpe SET_DEVICE_OP(dev_ops, enable_driver); 179502da3750SLeon Romanovsky SET_DEVICE_OP(dev_ops, fill_res_entry); 17963023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_dev_fw_str); 17973023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_dma_mr); 17983023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_hw_stats); 17993023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_link_layer); 18003023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_netdev); 18013023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_port_immutable); 18023023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vector_affinity); 18033023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vf_config); 18043023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vf_stats); 1805ea4baf7fSParav Pandit SET_DEVICE_OP(dev_ops, init_port); 18063023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, map_mr_sg); 18073023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, map_phys_fmr); 18083023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, mmap); 18093023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_ah); 18103023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_cq); 18113023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_device); 18123023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_flow_action_esp); 18133023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_port); 18143023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_qp); 18153023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_srq); 18163023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_wq); 18173023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, peek_cq); 18183023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, poll_cq); 18193023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_recv); 18203023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_send); 18213023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_srq_recv); 18223023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, process_mad); 18233023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_ah); 18243023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_device); 18253023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_gid); 18263023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_pkey); 18273023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_port); 18283023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_qp); 18293023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_srq); 18303023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); 18313023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, read_counters); 18323023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, reg_dm_mr); 18333023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, reg_user_mr); 18343023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, req_ncomp_notif); 18353023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, req_notify_cq); 18363023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, rereg_user_mr); 18373023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, resize_cq); 18383023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, set_vf_guid); 18393023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, set_vf_link_state); 18403023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, unmap_fmr); 184121a428a0SLeon Romanovsky 184221a428a0SLeon Romanovsky SET_OBJ_SIZE(dev_ops, ib_pd); 1843a2a074efSLeon Romanovsky SET_OBJ_SIZE(dev_ops, ib_ucontext); 1844521ed0d9SKamal Heib } 1845521ed0d9SKamal Heib EXPORT_SYMBOL(ib_set_device_ops); 1846521ed0d9SKamal Heib 1847d0e312feSLeon Romanovsky static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = { 1848735c631aSMark Bloch [RDMA_NL_LS_OP_RESOLVE] = { 1849647c75acSLeon Romanovsky .doit = ib_nl_handle_resolve_resp, 1850e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 1851e3a2b93dSLeon Romanovsky }, 1852735c631aSMark Bloch [RDMA_NL_LS_OP_SET_TIMEOUT] = { 1853647c75acSLeon Romanovsky .doit = ib_nl_handle_set_timeout, 1854e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 1855e3a2b93dSLeon Romanovsky }, 1856ae43f828SMark Bloch [RDMA_NL_LS_OP_IP_RESOLVE] = { 1857647c75acSLeon Romanovsky .doit = ib_nl_handle_ip_res_resp, 1858e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 1859e3a2b93dSLeon Romanovsky }, 1860735c631aSMark Bloch }; 1861735c631aSMark Bloch 18621da177e4SLinus Torvalds static int __init ib_core_init(void) 18631da177e4SLinus Torvalds { 18641da177e4SLinus Torvalds int ret; 18651da177e4SLinus Torvalds 1866f0626710STejun Heo ib_wq = alloc_workqueue("infiniband", 0, 0); 1867f0626710STejun Heo if (!ib_wq) 1868f0626710STejun Heo return -ENOMEM; 1869f0626710STejun Heo 187014d3a3b2SChristoph Hellwig ib_comp_wq = alloc_workqueue("ib-comp-wq", 1871b7363e67SSagi Grimberg WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 187214d3a3b2SChristoph Hellwig if (!ib_comp_wq) { 187314d3a3b2SChristoph Hellwig ret = -ENOMEM; 187414d3a3b2SChristoph Hellwig goto err; 187514d3a3b2SChristoph Hellwig } 187614d3a3b2SChristoph Hellwig 1877f794809aSJack Morgenstein ib_comp_unbound_wq = 1878f794809aSJack Morgenstein alloc_workqueue("ib-comp-unb-wq", 1879f794809aSJack Morgenstein WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM | 1880f794809aSJack Morgenstein WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE); 1881f794809aSJack Morgenstein if (!ib_comp_unbound_wq) { 1882f794809aSJack Morgenstein ret = -ENOMEM; 1883f794809aSJack Morgenstein goto err_comp; 1884f794809aSJack Morgenstein } 1885f794809aSJack Morgenstein 188655aeed06SJason Gunthorpe ret = class_register(&ib_class); 1887fd75c789SNir Muchtar if (ret) { 1888aba25a3eSParav Pandit pr_warn("Couldn't create InfiniBand device class\n"); 1889f794809aSJack Morgenstein goto err_comp_unbound; 1890fd75c789SNir Muchtar } 18911da177e4SLinus Torvalds 1892c9901724SLeon Romanovsky ret = rdma_nl_init(); 18931da177e4SLinus Torvalds if (ret) { 1894c9901724SLeon Romanovsky pr_warn("Couldn't init IB netlink interface: err %d\n", ret); 1895fd75c789SNir Muchtar goto err_sysfs; 18961da177e4SLinus Torvalds } 18971da177e4SLinus Torvalds 1898e3f20f02SLeon Romanovsky ret = addr_init(); 1899e3f20f02SLeon Romanovsky if (ret) { 1900e3f20f02SLeon Romanovsky pr_warn("Could't init IB address resolution\n"); 1901e3f20f02SLeon Romanovsky goto err_ibnl; 1902e3f20f02SLeon Romanovsky } 1903e3f20f02SLeon Romanovsky 19044c2cb422SMark Bloch ret = ib_mad_init(); 19054c2cb422SMark Bloch if (ret) { 19064c2cb422SMark Bloch pr_warn("Couldn't init IB MAD\n"); 19074c2cb422SMark Bloch goto err_addr; 19084c2cb422SMark Bloch } 19094c2cb422SMark Bloch 1910c2e49c92SMark Bloch ret = ib_sa_init(); 1911c2e49c92SMark Bloch if (ret) { 1912c2e49c92SMark Bloch pr_warn("Couldn't init SA\n"); 1913c2e49c92SMark Bloch goto err_mad; 1914c2e49c92SMark Bloch } 1915c2e49c92SMark Bloch 19168f408ab6SDaniel Jurgens ret = register_lsm_notifier(&ibdev_lsm_nb); 19178f408ab6SDaniel Jurgens if (ret) { 19188f408ab6SDaniel Jurgens pr_warn("Couldn't register LSM notifier. ret %d\n", ret); 1919c9901724SLeon Romanovsky goto err_sa; 19208f408ab6SDaniel Jurgens } 19218f408ab6SDaniel Jurgens 19226c80b41aSLeon Romanovsky nldev_init(); 1923c9901724SLeon Romanovsky rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table); 19245ef8c0c1SJason Gunthorpe roce_gid_mgmt_init(); 1925b2cbae2cSRoland Dreier 1926fd75c789SNir Muchtar return 0; 1927fd75c789SNir Muchtar 1928735c631aSMark Bloch err_sa: 1929735c631aSMark Bloch ib_sa_cleanup(); 1930c2e49c92SMark Bloch err_mad: 1931c2e49c92SMark Bloch ib_mad_cleanup(); 19324c2cb422SMark Bloch err_addr: 19334c2cb422SMark Bloch addr_cleanup(); 1934e3f20f02SLeon Romanovsky err_ibnl: 1935c9901724SLeon Romanovsky rdma_nl_exit(); 1936fd75c789SNir Muchtar err_sysfs: 193755aeed06SJason Gunthorpe class_unregister(&ib_class); 1938f794809aSJack Morgenstein err_comp_unbound: 1939f794809aSJack Morgenstein destroy_workqueue(ib_comp_unbound_wq); 194014d3a3b2SChristoph Hellwig err_comp: 194114d3a3b2SChristoph Hellwig destroy_workqueue(ib_comp_wq); 1942fd75c789SNir Muchtar err: 1943fd75c789SNir Muchtar destroy_workqueue(ib_wq); 19441da177e4SLinus Torvalds return ret; 19451da177e4SLinus Torvalds } 19461da177e4SLinus Torvalds 19471da177e4SLinus Torvalds static void __exit ib_core_cleanup(void) 19481da177e4SLinus Torvalds { 19495ef8c0c1SJason Gunthorpe roce_gid_mgmt_cleanup(); 19506c80b41aSLeon Romanovsky nldev_exit(); 1951c9901724SLeon Romanovsky rdma_nl_unregister(RDMA_NL_LS); 1952c9901724SLeon Romanovsky unregister_lsm_notifier(&ibdev_lsm_nb); 1953c2e49c92SMark Bloch ib_sa_cleanup(); 19544c2cb422SMark Bloch ib_mad_cleanup(); 1955e3f20f02SLeon Romanovsky addr_cleanup(); 1956c9901724SLeon Romanovsky rdma_nl_exit(); 195755aeed06SJason Gunthorpe class_unregister(&ib_class); 1958f794809aSJack Morgenstein destroy_workqueue(ib_comp_unbound_wq); 195914d3a3b2SChristoph Hellwig destroy_workqueue(ib_comp_wq); 1960f7c6a7b5SRoland Dreier /* Make sure that any pending umem accounting work is done. */ 1961f0626710STejun Heo destroy_workqueue(ib_wq); 1962d0899892SJason Gunthorpe flush_workqueue(system_unbound_wq); 1963e59178d8SJason Gunthorpe WARN_ON(!xa_empty(&clients)); 19640df91bb6SJason Gunthorpe WARN_ON(!xa_empty(&devices)); 19651da177e4SLinus Torvalds } 19661da177e4SLinus Torvalds 1967e3bf14bdSJason Gunthorpe MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4); 1968e3bf14bdSJason Gunthorpe 1969a9cd1a67SDmitry Monakhov subsys_initcall(ib_core_init); 19701da177e4SLinus Torvalds module_exit(ib_core_cleanup); 1971