11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Communications. All rights reserved. 32a1d9b7fSRoland Dreier * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * This software is available to you under a choice of one of two 61da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 71da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 81da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 91da177e4SLinus Torvalds * OpenIB.org BSD license below: 101da177e4SLinus Torvalds * 111da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 121da177e4SLinus Torvalds * without modification, are permitted provided that the following 131da177e4SLinus Torvalds * conditions are met: 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * - Redistributions of source code must retain the above 161da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 171da177e4SLinus Torvalds * disclaimer. 181da177e4SLinus Torvalds * 191da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 201da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 211da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 221da177e4SLinus Torvalds * provided with the distribution. 231da177e4SLinus Torvalds * 241da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 251da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 261da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 271da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 281da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 291da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 301da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 311da177e4SLinus Torvalds * SOFTWARE. 321da177e4SLinus Torvalds */ 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds #include <linux/module.h> 351da177e4SLinus Torvalds #include <linux/string.h> 361da177e4SLinus Torvalds #include <linux/errno.h> 379a6b090cSAhmed S. Darwish #include <linux/kernel.h> 381da177e4SLinus Torvalds #include <linux/slab.h> 391da177e4SLinus Torvalds #include <linux/init.h> 409268f72dSYotam Kenneth #include <linux/netdevice.h> 414e0f7b90SParav Pandit #include <net/net_namespace.h> 424e0f7b90SParav Pandit #include <net/netns/generic.h> 438f408ab6SDaniel Jurgens #include <linux/security.h> 448f408ab6SDaniel Jurgens #include <linux/notifier.h> 45324e227eSJason Gunthorpe #include <linux/hashtable.h> 46b2cbae2cSRoland Dreier #include <rdma/rdma_netlink.h> 4703db3a2dSMatan Barak #include <rdma/ib_addr.h> 4803db3a2dSMatan Barak #include <rdma/ib_cache.h> 491da177e4SLinus Torvalds 501da177e4SLinus Torvalds #include "core_priv.h" 5141eda65cSLeon Romanovsky #include "restrack.h" 521da177e4SLinus Torvalds 531da177e4SLinus Torvalds MODULE_AUTHOR("Roland Dreier"); 541da177e4SLinus Torvalds MODULE_DESCRIPTION("core kernel InfiniBand API"); 551da177e4SLinus Torvalds MODULE_LICENSE("Dual BSD/GPL"); 561da177e4SLinus Torvalds 5714d3a3b2SChristoph Hellwig struct workqueue_struct *ib_comp_wq; 58f794809aSJack Morgenstein struct workqueue_struct *ib_comp_unbound_wq; 59f0626710STejun Heo struct workqueue_struct *ib_wq; 60f0626710STejun Heo EXPORT_SYMBOL_GPL(ib_wq); 61f0626710STejun Heo 620df91bb6SJason Gunthorpe /* 63921eab11SJason Gunthorpe * Each of the three rwsem locks (devices, clients, client_data) protects the 64921eab11SJason Gunthorpe * xarray of the same name. Specifically it allows the caller to assert that 65921eab11SJason Gunthorpe * the MARK will/will not be changing under the lock, and for devices and 66921eab11SJason Gunthorpe * clients, that the value in the xarray is still a valid pointer. Change of 67921eab11SJason Gunthorpe * the MARK is linked to the object state, so holding the lock and testing the 68921eab11SJason Gunthorpe * MARK also asserts that the contained object is in a certain state. 69921eab11SJason Gunthorpe * 70921eab11SJason Gunthorpe * This is used to build a two stage register/unregister flow where objects 71921eab11SJason Gunthorpe * can continue to be in the xarray even though they are still in progress to 72921eab11SJason Gunthorpe * register/unregister. 73921eab11SJason Gunthorpe * 74921eab11SJason Gunthorpe * The xarray itself provides additional locking, and restartable iteration, 75921eab11SJason Gunthorpe * which is also relied on. 76921eab11SJason Gunthorpe * 77921eab11SJason Gunthorpe * Locks should not be nested, with the exception of client_data, which is 78921eab11SJason Gunthorpe * allowed to nest under the read side of the other two locks. 79921eab11SJason Gunthorpe * 80921eab11SJason Gunthorpe * The devices_rwsem also protects the device name list, any change or 81921eab11SJason Gunthorpe * assignment of device name must also hold the write side to guarantee unique 82921eab11SJason Gunthorpe * names. 83921eab11SJason Gunthorpe */ 84921eab11SJason Gunthorpe 85921eab11SJason Gunthorpe /* 860df91bb6SJason Gunthorpe * devices contains devices that have had their names assigned. The 870df91bb6SJason Gunthorpe * devices may not be registered. Users that care about the registration 880df91bb6SJason Gunthorpe * status need to call ib_device_try_get() on the device to ensure it is 890df91bb6SJason Gunthorpe * registered, and keep it registered, for the required duration. 900df91bb6SJason Gunthorpe * 910df91bb6SJason Gunthorpe */ 920df91bb6SJason Gunthorpe static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC); 93921eab11SJason Gunthorpe static DECLARE_RWSEM(devices_rwsem); 940df91bb6SJason Gunthorpe #define DEVICE_REGISTERED XA_MARK_1 950df91bb6SJason Gunthorpe 961da177e4SLinus Torvalds static LIST_HEAD(client_list); 97e59178d8SJason Gunthorpe #define CLIENT_REGISTERED XA_MARK_1 98e59178d8SJason Gunthorpe static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC); 99921eab11SJason Gunthorpe static DECLARE_RWSEM(clients_rwsem); 1001da177e4SLinus Torvalds 1011da177e4SLinus Torvalds /* 1020df91bb6SJason Gunthorpe * If client_data is registered then the corresponding client must also still 1030df91bb6SJason Gunthorpe * be registered. 1040df91bb6SJason Gunthorpe */ 1050df91bb6SJason Gunthorpe #define CLIENT_DATA_REGISTERED XA_MARK_1 1064e0f7b90SParav Pandit 1074e0f7b90SParav Pandit /** 1084e0f7b90SParav Pandit * struct rdma_dev_net - rdma net namespace metadata for a net 1094e0f7b90SParav Pandit * @net: Pointer to owner net namespace 1104e0f7b90SParav Pandit * @id: xarray id to identify the net namespace. 1114e0f7b90SParav Pandit */ 1124e0f7b90SParav Pandit struct rdma_dev_net { 1134e0f7b90SParav Pandit possible_net_t net; 1144e0f7b90SParav Pandit u32 id; 1154e0f7b90SParav Pandit }; 1164e0f7b90SParav Pandit 1174e0f7b90SParav Pandit static unsigned int rdma_dev_net_id; 1184e0f7b90SParav Pandit 1194e0f7b90SParav Pandit /* 1204e0f7b90SParav Pandit * A list of net namespaces is maintained in an xarray. This is necessary 1214e0f7b90SParav Pandit * because we can't get the locking right using the existing net ns list. We 1224e0f7b90SParav Pandit * would require a init_net callback after the list is updated. 1234e0f7b90SParav Pandit */ 1244e0f7b90SParav Pandit static DEFINE_XARRAY_FLAGS(rdma_nets, XA_FLAGS_ALLOC); 1254e0f7b90SParav Pandit /* 1264e0f7b90SParav Pandit * rwsem to protect accessing the rdma_nets xarray entries. 1274e0f7b90SParav Pandit */ 1284e0f7b90SParav Pandit static DECLARE_RWSEM(rdma_nets_rwsem); 1294e0f7b90SParav Pandit 130cb7e0e13SParav Pandit bool ib_devices_shared_netns = true; 131a56bc45bSParav Pandit module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444); 132a56bc45bSParav Pandit MODULE_PARM_DESC(netns_mode, 133a56bc45bSParav Pandit "Share device among net namespaces; default=1 (shared)"); 13441c61401SParav Pandit /** 13541c61401SParav Pandit * rdma_dev_access_netns() - Return whether a rdma device can be accessed 13641c61401SParav Pandit * from a specified net namespace or not. 13741c61401SParav Pandit * @device: Pointer to rdma device which needs to be checked 13841c61401SParav Pandit * @net: Pointer to net namesapce for which access to be checked 13941c61401SParav Pandit * 14041c61401SParav Pandit * rdma_dev_access_netns() - Return whether a rdma device can be accessed 14141c61401SParav Pandit * from a specified net namespace or not. When 14241c61401SParav Pandit * rdma device is in shared mode, it ignores the 14341c61401SParav Pandit * net namespace. When rdma device is exclusive 14441c61401SParav Pandit * to a net namespace, rdma device net namespace is 14541c61401SParav Pandit * checked against the specified one. 14641c61401SParav Pandit */ 14741c61401SParav Pandit bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net) 14841c61401SParav Pandit { 14941c61401SParav Pandit return (ib_devices_shared_netns || 15041c61401SParav Pandit net_eq(read_pnet(&dev->coredev.rdma_net), net)); 15141c61401SParav Pandit } 15241c61401SParav Pandit EXPORT_SYMBOL(rdma_dev_access_netns); 15341c61401SParav Pandit 1540df91bb6SJason Gunthorpe /* 1550df91bb6SJason Gunthorpe * xarray has this behavior where it won't iterate over NULL values stored in 1560df91bb6SJason Gunthorpe * allocated arrays. So we need our own iterator to see all values stored in 1570df91bb6SJason Gunthorpe * the array. This does the same thing as xa_for_each except that it also 1580df91bb6SJason Gunthorpe * returns NULL valued entries if the array is allocating. Simplified to only 1590df91bb6SJason Gunthorpe * work on simple xarrays. 1600df91bb6SJason Gunthorpe */ 1610df91bb6SJason Gunthorpe static void *xan_find_marked(struct xarray *xa, unsigned long *indexp, 1620df91bb6SJason Gunthorpe xa_mark_t filter) 1630df91bb6SJason Gunthorpe { 1640df91bb6SJason Gunthorpe XA_STATE(xas, xa, *indexp); 1650df91bb6SJason Gunthorpe void *entry; 1660df91bb6SJason Gunthorpe 1670df91bb6SJason Gunthorpe rcu_read_lock(); 1680df91bb6SJason Gunthorpe do { 1690df91bb6SJason Gunthorpe entry = xas_find_marked(&xas, ULONG_MAX, filter); 1700df91bb6SJason Gunthorpe if (xa_is_zero(entry)) 1710df91bb6SJason Gunthorpe break; 1720df91bb6SJason Gunthorpe } while (xas_retry(&xas, entry)); 1730df91bb6SJason Gunthorpe rcu_read_unlock(); 1740df91bb6SJason Gunthorpe 1750df91bb6SJason Gunthorpe if (entry) { 1760df91bb6SJason Gunthorpe *indexp = xas.xa_index; 1770df91bb6SJason Gunthorpe if (xa_is_zero(entry)) 1780df91bb6SJason Gunthorpe return NULL; 1790df91bb6SJason Gunthorpe return entry; 1800df91bb6SJason Gunthorpe } 1810df91bb6SJason Gunthorpe return XA_ERROR(-ENOENT); 1820df91bb6SJason Gunthorpe } 1830df91bb6SJason Gunthorpe #define xan_for_each_marked(xa, index, entry, filter) \ 1840df91bb6SJason Gunthorpe for (index = 0, entry = xan_find_marked(xa, &(index), filter); \ 1850df91bb6SJason Gunthorpe !xa_is_err(entry); \ 1860df91bb6SJason Gunthorpe (index)++, entry = xan_find_marked(xa, &(index), filter)) 1870df91bb6SJason Gunthorpe 188324e227eSJason Gunthorpe /* RCU hash table mapping netdevice pointers to struct ib_port_data */ 189324e227eSJason Gunthorpe static DEFINE_SPINLOCK(ndev_hash_lock); 190324e227eSJason Gunthorpe static DECLARE_HASHTABLE(ndev_hash, 5); 191324e227eSJason Gunthorpe 192c2261dd7SJason Gunthorpe static void free_netdevs(struct ib_device *ib_dev); 193d0899892SJason Gunthorpe static void ib_unregister_work(struct work_struct *work); 194d0899892SJason Gunthorpe static void __ib_unregister_device(struct ib_device *device); 1958f408ab6SDaniel Jurgens static int ib_security_change(struct notifier_block *nb, unsigned long event, 1968f408ab6SDaniel Jurgens void *lsm_data); 1978f408ab6SDaniel Jurgens static void ib_policy_change_task(struct work_struct *work); 1988f408ab6SDaniel Jurgens static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task); 1998f408ab6SDaniel Jurgens 2008f408ab6SDaniel Jurgens static struct notifier_block ibdev_lsm_nb = { 2018f408ab6SDaniel Jurgens .notifier_call = ib_security_change, 2028f408ab6SDaniel Jurgens }; 2031da177e4SLinus Torvalds 204324e227eSJason Gunthorpe /* Pointer to the RCU head at the start of the ib_port_data array */ 205324e227eSJason Gunthorpe struct ib_port_data_rcu { 206324e227eSJason Gunthorpe struct rcu_head rcu_head; 207324e227eSJason Gunthorpe struct ib_port_data pdata[]; 208324e227eSJason Gunthorpe }; 209324e227eSJason Gunthorpe 2101da177e4SLinus Torvalds static int ib_device_check_mandatory(struct ib_device *device) 2111da177e4SLinus Torvalds { 2123023a1e9SKamal Heib #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x } 2131da177e4SLinus Torvalds static const struct { 2141da177e4SLinus Torvalds size_t offset; 2151da177e4SLinus Torvalds char *name; 2161da177e4SLinus Torvalds } mandatory_table[] = { 2171da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_device), 2181da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_port), 2191da177e4SLinus Torvalds IB_MANDATORY_FUNC(query_pkey), 2201da177e4SLinus Torvalds IB_MANDATORY_FUNC(alloc_pd), 2211da177e4SLinus Torvalds IB_MANDATORY_FUNC(dealloc_pd), 2221da177e4SLinus Torvalds IB_MANDATORY_FUNC(create_qp), 2231da177e4SLinus Torvalds IB_MANDATORY_FUNC(modify_qp), 2241da177e4SLinus Torvalds IB_MANDATORY_FUNC(destroy_qp), 2251da177e4SLinus Torvalds IB_MANDATORY_FUNC(post_send), 2261da177e4SLinus Torvalds IB_MANDATORY_FUNC(post_recv), 2271da177e4SLinus Torvalds IB_MANDATORY_FUNC(create_cq), 2281da177e4SLinus Torvalds IB_MANDATORY_FUNC(destroy_cq), 2291da177e4SLinus Torvalds IB_MANDATORY_FUNC(poll_cq), 2301da177e4SLinus Torvalds IB_MANDATORY_FUNC(req_notify_cq), 2311da177e4SLinus Torvalds IB_MANDATORY_FUNC(get_dma_mr), 2327738613eSIra Weiny IB_MANDATORY_FUNC(dereg_mr), 2337738613eSIra Weiny IB_MANDATORY_FUNC(get_port_immutable) 2341da177e4SLinus Torvalds }; 2351da177e4SLinus Torvalds int i; 2361da177e4SLinus Torvalds 2376780c4faSGal Pressman device->kverbs_provider = true; 2389a6b090cSAhmed S. Darwish for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) { 2393023a1e9SKamal Heib if (!*(void **) ((void *) &device->ops + 2403023a1e9SKamal Heib mandatory_table[i].offset)) { 2416780c4faSGal Pressman device->kverbs_provider = false; 2426780c4faSGal Pressman break; 2431da177e4SLinus Torvalds } 2441da177e4SLinus Torvalds } 2451da177e4SLinus Torvalds 2461da177e4SLinus Torvalds return 0; 2471da177e4SLinus Torvalds } 2481da177e4SLinus Torvalds 249f8978bd9SLeon Romanovsky /* 25001b67117SParav Pandit * Caller must perform ib_device_put() to return the device reference count 25101b67117SParav Pandit * when ib_device_get_by_index() returns valid device pointer. 252f8978bd9SLeon Romanovsky */ 25337eeab55SParav Pandit struct ib_device *ib_device_get_by_index(const struct net *net, u32 index) 254f8978bd9SLeon Romanovsky { 255f8978bd9SLeon Romanovsky struct ib_device *device; 256f8978bd9SLeon Romanovsky 257921eab11SJason Gunthorpe down_read(&devices_rwsem); 2580df91bb6SJason Gunthorpe device = xa_load(&devices, index); 25901b67117SParav Pandit if (device) { 26037eeab55SParav Pandit if (!rdma_dev_access_netns(device, net)) { 26137eeab55SParav Pandit device = NULL; 26237eeab55SParav Pandit goto out; 26337eeab55SParav Pandit } 26437eeab55SParav Pandit 265d79af724SJason Gunthorpe if (!ib_device_try_get(device)) 26601b67117SParav Pandit device = NULL; 26701b67117SParav Pandit } 26837eeab55SParav Pandit out: 269921eab11SJason Gunthorpe up_read(&devices_rwsem); 270f8978bd9SLeon Romanovsky return device; 271f8978bd9SLeon Romanovsky } 272f8978bd9SLeon Romanovsky 273d79af724SJason Gunthorpe /** 274d79af724SJason Gunthorpe * ib_device_put - Release IB device reference 275d79af724SJason Gunthorpe * @device: device whose reference to be released 276d79af724SJason Gunthorpe * 277d79af724SJason Gunthorpe * ib_device_put() releases reference to the IB device to allow it to be 278d79af724SJason Gunthorpe * unregistered and eventually free. 279d79af724SJason Gunthorpe */ 28001b67117SParav Pandit void ib_device_put(struct ib_device *device) 28101b67117SParav Pandit { 28201b67117SParav Pandit if (refcount_dec_and_test(&device->refcount)) 28301b67117SParav Pandit complete(&device->unreg_completion); 28401b67117SParav Pandit } 285d79af724SJason Gunthorpe EXPORT_SYMBOL(ib_device_put); 28601b67117SParav Pandit 2871da177e4SLinus Torvalds static struct ib_device *__ib_device_get_by_name(const char *name) 2881da177e4SLinus Torvalds { 2891da177e4SLinus Torvalds struct ib_device *device; 2900df91bb6SJason Gunthorpe unsigned long index; 2911da177e4SLinus Torvalds 2920df91bb6SJason Gunthorpe xa_for_each (&devices, index, device) 293896de009SJason Gunthorpe if (!strcmp(name, dev_name(&device->dev))) 2941da177e4SLinus Torvalds return device; 2951da177e4SLinus Torvalds 2961da177e4SLinus Torvalds return NULL; 2971da177e4SLinus Torvalds } 2981da177e4SLinus Torvalds 2996cc2c8e5SJason Gunthorpe /** 3006cc2c8e5SJason Gunthorpe * ib_device_get_by_name - Find an IB device by name 3016cc2c8e5SJason Gunthorpe * @name: The name to look for 3026cc2c8e5SJason Gunthorpe * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 3036cc2c8e5SJason Gunthorpe * 3046cc2c8e5SJason Gunthorpe * Find and hold an ib_device by its name. The caller must call 3056cc2c8e5SJason Gunthorpe * ib_device_put() on the returned pointer. 3066cc2c8e5SJason Gunthorpe */ 3076cc2c8e5SJason Gunthorpe struct ib_device *ib_device_get_by_name(const char *name, 3086cc2c8e5SJason Gunthorpe enum rdma_driver_id driver_id) 3096cc2c8e5SJason Gunthorpe { 3106cc2c8e5SJason Gunthorpe struct ib_device *device; 3116cc2c8e5SJason Gunthorpe 3126cc2c8e5SJason Gunthorpe down_read(&devices_rwsem); 3136cc2c8e5SJason Gunthorpe device = __ib_device_get_by_name(name); 3146cc2c8e5SJason Gunthorpe if (device && driver_id != RDMA_DRIVER_UNKNOWN && 3156cc2c8e5SJason Gunthorpe device->driver_id != driver_id) 3166cc2c8e5SJason Gunthorpe device = NULL; 3176cc2c8e5SJason Gunthorpe 3186cc2c8e5SJason Gunthorpe if (device) { 3196cc2c8e5SJason Gunthorpe if (!ib_device_try_get(device)) 3206cc2c8e5SJason Gunthorpe device = NULL; 3216cc2c8e5SJason Gunthorpe } 3226cc2c8e5SJason Gunthorpe up_read(&devices_rwsem); 3236cc2c8e5SJason Gunthorpe return device; 3246cc2c8e5SJason Gunthorpe } 3256cc2c8e5SJason Gunthorpe EXPORT_SYMBOL(ib_device_get_by_name); 3266cc2c8e5SJason Gunthorpe 3274e0f7b90SParav Pandit static int rename_compat_devs(struct ib_device *device) 3284e0f7b90SParav Pandit { 3294e0f7b90SParav Pandit struct ib_core_device *cdev; 3304e0f7b90SParav Pandit unsigned long index; 3314e0f7b90SParav Pandit int ret = 0; 3324e0f7b90SParav Pandit 3334e0f7b90SParav Pandit mutex_lock(&device->compat_devs_mutex); 3344e0f7b90SParav Pandit xa_for_each (&device->compat_devs, index, cdev) { 3354e0f7b90SParav Pandit ret = device_rename(&cdev->dev, dev_name(&device->dev)); 3364e0f7b90SParav Pandit if (ret) { 3374e0f7b90SParav Pandit dev_warn(&cdev->dev, 3384e0f7b90SParav Pandit "Fail to rename compatdev to new name %s\n", 3394e0f7b90SParav Pandit dev_name(&device->dev)); 3404e0f7b90SParav Pandit break; 3414e0f7b90SParav Pandit } 3424e0f7b90SParav Pandit } 3434e0f7b90SParav Pandit mutex_unlock(&device->compat_devs_mutex); 3444e0f7b90SParav Pandit return ret; 3454e0f7b90SParav Pandit } 3464e0f7b90SParav Pandit 347d21943ddSLeon Romanovsky int ib_device_rename(struct ib_device *ibdev, const char *name) 348d21943ddSLeon Romanovsky { 349e3593b56SJason Gunthorpe int ret; 350d21943ddSLeon Romanovsky 351921eab11SJason Gunthorpe down_write(&devices_rwsem); 352e3593b56SJason Gunthorpe if (!strcmp(name, dev_name(&ibdev->dev))) { 353e3593b56SJason Gunthorpe ret = 0; 354e3593b56SJason Gunthorpe goto out; 355e3593b56SJason Gunthorpe } 356e3593b56SJason Gunthorpe 357344684e6SJason Gunthorpe if (__ib_device_get_by_name(name)) { 358d21943ddSLeon Romanovsky ret = -EEXIST; 359d21943ddSLeon Romanovsky goto out; 360d21943ddSLeon Romanovsky } 361d21943ddSLeon Romanovsky 362d21943ddSLeon Romanovsky ret = device_rename(&ibdev->dev, name); 363d21943ddSLeon Romanovsky if (ret) 364d21943ddSLeon Romanovsky goto out; 365d21943ddSLeon Romanovsky strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); 3664e0f7b90SParav Pandit ret = rename_compat_devs(ibdev); 367d21943ddSLeon Romanovsky out: 368921eab11SJason Gunthorpe up_write(&devices_rwsem); 369d21943ddSLeon Romanovsky return ret; 370d21943ddSLeon Romanovsky } 371d21943ddSLeon Romanovsky 372e349f858SJason Gunthorpe static int alloc_name(struct ib_device *ibdev, const char *name) 3731da177e4SLinus Torvalds { 3741da177e4SLinus Torvalds struct ib_device *device; 3750df91bb6SJason Gunthorpe unsigned long index; 3763b88afd3SJason Gunthorpe struct ida inuse; 3773b88afd3SJason Gunthorpe int rc; 3781da177e4SLinus Torvalds int i; 3791da177e4SLinus Torvalds 380921eab11SJason Gunthorpe lockdep_assert_held_exclusive(&devices_rwsem); 3813b88afd3SJason Gunthorpe ida_init(&inuse); 3820df91bb6SJason Gunthorpe xa_for_each (&devices, index, device) { 383e349f858SJason Gunthorpe char buf[IB_DEVICE_NAME_MAX]; 384e349f858SJason Gunthorpe 385896de009SJason Gunthorpe if (sscanf(dev_name(&device->dev), name, &i) != 1) 3861da177e4SLinus Torvalds continue; 3873b88afd3SJason Gunthorpe if (i < 0 || i >= INT_MAX) 3881da177e4SLinus Torvalds continue; 3891da177e4SLinus Torvalds snprintf(buf, sizeof buf, name, i); 3903b88afd3SJason Gunthorpe if (strcmp(buf, dev_name(&device->dev)) != 0) 3913b88afd3SJason Gunthorpe continue; 3923b88afd3SJason Gunthorpe 3933b88afd3SJason Gunthorpe rc = ida_alloc_range(&inuse, i, i, GFP_KERNEL); 3943b88afd3SJason Gunthorpe if (rc < 0) 3953b88afd3SJason Gunthorpe goto out; 3961da177e4SLinus Torvalds } 3971da177e4SLinus Torvalds 3983b88afd3SJason Gunthorpe rc = ida_alloc(&inuse, GFP_KERNEL); 3993b88afd3SJason Gunthorpe if (rc < 0) 4003b88afd3SJason Gunthorpe goto out; 4011da177e4SLinus Torvalds 4023b88afd3SJason Gunthorpe rc = dev_set_name(&ibdev->dev, name, rc); 4033b88afd3SJason Gunthorpe out: 4043b88afd3SJason Gunthorpe ida_destroy(&inuse); 4053b88afd3SJason Gunthorpe return rc; 4061da177e4SLinus Torvalds } 4071da177e4SLinus Torvalds 40855aeed06SJason Gunthorpe static void ib_device_release(struct device *device) 40955aeed06SJason Gunthorpe { 41055aeed06SJason Gunthorpe struct ib_device *dev = container_of(device, struct ib_device, dev); 41155aeed06SJason Gunthorpe 412c2261dd7SJason Gunthorpe free_netdevs(dev); 413652432f3SJason Gunthorpe WARN_ON(refcount_read(&dev->refcount)); 41403db3a2dSMatan Barak ib_cache_release_one(dev); 415b34b269aSJason Gunthorpe ib_security_release_port_pkey_list(dev); 4164e0f7b90SParav Pandit xa_destroy(&dev->compat_devs); 4170df91bb6SJason Gunthorpe xa_destroy(&dev->client_data); 418324e227eSJason Gunthorpe if (dev->port_data) 419324e227eSJason Gunthorpe kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu, 420324e227eSJason Gunthorpe pdata[0]), 421324e227eSJason Gunthorpe rcu_head); 422324e227eSJason Gunthorpe kfree_rcu(dev, rcu_head); 42355aeed06SJason Gunthorpe } 42455aeed06SJason Gunthorpe 42555aeed06SJason Gunthorpe static int ib_device_uevent(struct device *device, 42655aeed06SJason Gunthorpe struct kobj_uevent_env *env) 42755aeed06SJason Gunthorpe { 428896de009SJason Gunthorpe if (add_uevent_var(env, "NAME=%s", dev_name(device))) 42955aeed06SJason Gunthorpe return -ENOMEM; 43055aeed06SJason Gunthorpe 43155aeed06SJason Gunthorpe /* 43255aeed06SJason Gunthorpe * It would be nice to pass the node GUID with the event... 43355aeed06SJason Gunthorpe */ 43455aeed06SJason Gunthorpe 43555aeed06SJason Gunthorpe return 0; 43655aeed06SJason Gunthorpe } 43755aeed06SJason Gunthorpe 43862dfa795SParav Pandit static const void *net_namespace(struct device *d) 43962dfa795SParav Pandit { 4404e0f7b90SParav Pandit struct ib_core_device *coredev = 4414e0f7b90SParav Pandit container_of(d, struct ib_core_device, dev); 4424e0f7b90SParav Pandit 4434e0f7b90SParav Pandit return read_pnet(&coredev->rdma_net); 44462dfa795SParav Pandit } 44562dfa795SParav Pandit 44655aeed06SJason Gunthorpe static struct class ib_class = { 44755aeed06SJason Gunthorpe .name = "infiniband", 44855aeed06SJason Gunthorpe .dev_release = ib_device_release, 44955aeed06SJason Gunthorpe .dev_uevent = ib_device_uevent, 45062dfa795SParav Pandit .ns_type = &net_ns_type_operations, 45162dfa795SParav Pandit .namespace = net_namespace, 45255aeed06SJason Gunthorpe }; 45355aeed06SJason Gunthorpe 454cebe556bSParav Pandit static void rdma_init_coredev(struct ib_core_device *coredev, 4554e0f7b90SParav Pandit struct ib_device *dev, struct net *net) 456cebe556bSParav Pandit { 457cebe556bSParav Pandit /* This BUILD_BUG_ON is intended to catch layout change 458cebe556bSParav Pandit * of union of ib_core_device and device. 459cebe556bSParav Pandit * dev must be the first element as ib_core and providers 460cebe556bSParav Pandit * driver uses it. Adding anything in ib_core_device before 461cebe556bSParav Pandit * device will break this assumption. 462cebe556bSParav Pandit */ 463cebe556bSParav Pandit BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) != 464cebe556bSParav Pandit offsetof(struct ib_device, dev)); 465cebe556bSParav Pandit 466cebe556bSParav Pandit coredev->dev.class = &ib_class; 467cebe556bSParav Pandit coredev->dev.groups = dev->groups; 468cebe556bSParav Pandit device_initialize(&coredev->dev); 469cebe556bSParav Pandit coredev->owner = dev; 470cebe556bSParav Pandit INIT_LIST_HEAD(&coredev->port_list); 4714e0f7b90SParav Pandit write_pnet(&coredev->rdma_net, net); 472cebe556bSParav Pandit } 473cebe556bSParav Pandit 4741da177e4SLinus Torvalds /** 475459cc69fSLeon Romanovsky * _ib_alloc_device - allocate an IB device struct 4761da177e4SLinus Torvalds * @size:size of structure to allocate 4771da177e4SLinus Torvalds * 4781da177e4SLinus Torvalds * Low-level drivers should use ib_alloc_device() to allocate &struct 4791da177e4SLinus Torvalds * ib_device. @size is the size of the structure to be allocated, 4801da177e4SLinus Torvalds * including any private data used by the low-level driver. 4811da177e4SLinus Torvalds * ib_dealloc_device() must be used to free structures allocated with 4821da177e4SLinus Torvalds * ib_alloc_device(). 4831da177e4SLinus Torvalds */ 484459cc69fSLeon Romanovsky struct ib_device *_ib_alloc_device(size_t size) 4851da177e4SLinus Torvalds { 48655aeed06SJason Gunthorpe struct ib_device *device; 4871da177e4SLinus Torvalds 48855aeed06SJason Gunthorpe if (WARN_ON(size < sizeof(struct ib_device))) 48955aeed06SJason Gunthorpe return NULL; 49055aeed06SJason Gunthorpe 49155aeed06SJason Gunthorpe device = kzalloc(size, GFP_KERNEL); 49255aeed06SJason Gunthorpe if (!device) 49355aeed06SJason Gunthorpe return NULL; 49455aeed06SJason Gunthorpe 49541eda65cSLeon Romanovsky if (rdma_restrack_init(device)) { 49641eda65cSLeon Romanovsky kfree(device); 49741eda65cSLeon Romanovsky return NULL; 49841eda65cSLeon Romanovsky } 49902d8883fSLeon Romanovsky 5005f8f5499SParav Pandit device->groups[0] = &ib_dev_attr_group; 5014e0f7b90SParav Pandit rdma_init_coredev(&device->coredev, device, &init_net); 50255aeed06SJason Gunthorpe 50355aeed06SJason Gunthorpe INIT_LIST_HEAD(&device->event_handler_list); 50455aeed06SJason Gunthorpe spin_lock_init(&device->event_handler_lock); 505d0899892SJason Gunthorpe mutex_init(&device->unregistration_lock); 5060df91bb6SJason Gunthorpe /* 5070df91bb6SJason Gunthorpe * client_data needs to be alloc because we don't want our mark to be 5080df91bb6SJason Gunthorpe * destroyed if the user stores NULL in the client data. 5090df91bb6SJason Gunthorpe */ 5100df91bb6SJason Gunthorpe xa_init_flags(&device->client_data, XA_FLAGS_ALLOC); 511921eab11SJason Gunthorpe init_rwsem(&device->client_data_rwsem); 5124e0f7b90SParav Pandit xa_init_flags(&device->compat_devs, XA_FLAGS_ALLOC); 5134e0f7b90SParav Pandit mutex_init(&device->compat_devs_mutex); 51401b67117SParav Pandit init_completion(&device->unreg_completion); 515d0899892SJason Gunthorpe INIT_WORK(&device->unregistration_work, ib_unregister_work); 51655aeed06SJason Gunthorpe 51755aeed06SJason Gunthorpe return device; 5181da177e4SLinus Torvalds } 519459cc69fSLeon Romanovsky EXPORT_SYMBOL(_ib_alloc_device); 5201da177e4SLinus Torvalds 5211da177e4SLinus Torvalds /** 5221da177e4SLinus Torvalds * ib_dealloc_device - free an IB device struct 5231da177e4SLinus Torvalds * @device:structure to free 5241da177e4SLinus Torvalds * 5251da177e4SLinus Torvalds * Free a structure allocated with ib_alloc_device(). 5261da177e4SLinus Torvalds */ 5271da177e4SLinus Torvalds void ib_dealloc_device(struct ib_device *device) 5281da177e4SLinus Torvalds { 529d0899892SJason Gunthorpe if (device->ops.dealloc_driver) 530d0899892SJason Gunthorpe device->ops.dealloc_driver(device); 531d0899892SJason Gunthorpe 532d0899892SJason Gunthorpe /* 533d0899892SJason Gunthorpe * ib_unregister_driver() requires all devices to remain in the xarray 534d0899892SJason Gunthorpe * while their ops are callable. The last op we call is dealloc_driver 535d0899892SJason Gunthorpe * above. This is needed to create a fence on op callbacks prior to 536d0899892SJason Gunthorpe * allowing the driver module to unload. 537d0899892SJason Gunthorpe */ 538d0899892SJason Gunthorpe down_write(&devices_rwsem); 539d0899892SJason Gunthorpe if (xa_load(&devices, device->index) == device) 540d0899892SJason Gunthorpe xa_erase(&devices, device->index); 541d0899892SJason Gunthorpe up_write(&devices_rwsem); 542d0899892SJason Gunthorpe 543c2261dd7SJason Gunthorpe /* Expedite releasing netdev references */ 544c2261dd7SJason Gunthorpe free_netdevs(device); 545c2261dd7SJason Gunthorpe 5464e0f7b90SParav Pandit WARN_ON(!xa_empty(&device->compat_devs)); 5470df91bb6SJason Gunthorpe WARN_ON(!xa_empty(&device->client_data)); 548652432f3SJason Gunthorpe WARN_ON(refcount_read(&device->refcount)); 5490ad699c0SLeon Romanovsky rdma_restrack_clean(device); 550e155755eSParav Pandit /* Balances with device_initialize */ 551924b8900SLeon Romanovsky put_device(&device->dev); 5521da177e4SLinus Torvalds } 5531da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_device); 5541da177e4SLinus Torvalds 555921eab11SJason Gunthorpe /* 556921eab11SJason Gunthorpe * add_client_context() and remove_client_context() must be safe against 557921eab11SJason Gunthorpe * parallel calls on the same device - registration/unregistration of both the 558921eab11SJason Gunthorpe * device and client can be occurring in parallel. 559921eab11SJason Gunthorpe * 560921eab11SJason Gunthorpe * The routines need to be a fence, any caller must not return until the add 561921eab11SJason Gunthorpe * or remove is fully completed. 562921eab11SJason Gunthorpe */ 563921eab11SJason Gunthorpe static int add_client_context(struct ib_device *device, 564921eab11SJason Gunthorpe struct ib_client *client) 5651da177e4SLinus Torvalds { 566921eab11SJason Gunthorpe int ret = 0; 5671da177e4SLinus Torvalds 5686780c4faSGal Pressman if (!device->kverbs_provider && !client->no_kverbs_req) 569921eab11SJason Gunthorpe return 0; 5706780c4faSGal Pressman 571921eab11SJason Gunthorpe down_write(&device->client_data_rwsem); 572921eab11SJason Gunthorpe /* 573921eab11SJason Gunthorpe * Another caller to add_client_context got here first and has already 574921eab11SJason Gunthorpe * completely initialized context. 575921eab11SJason Gunthorpe */ 576921eab11SJason Gunthorpe if (xa_get_mark(&device->client_data, client->client_id, 577921eab11SJason Gunthorpe CLIENT_DATA_REGISTERED)) 578921eab11SJason Gunthorpe goto out; 579921eab11SJason Gunthorpe 580921eab11SJason Gunthorpe ret = xa_err(xa_store(&device->client_data, client->client_id, NULL, 581921eab11SJason Gunthorpe GFP_KERNEL)); 582921eab11SJason Gunthorpe if (ret) 583921eab11SJason Gunthorpe goto out; 584921eab11SJason Gunthorpe downgrade_write(&device->client_data_rwsem); 585921eab11SJason Gunthorpe if (client->add) 586921eab11SJason Gunthorpe client->add(device); 587921eab11SJason Gunthorpe 588921eab11SJason Gunthorpe /* Readers shall not see a client until add has been completed */ 5890df91bb6SJason Gunthorpe xa_set_mark(&device->client_data, client->client_id, 5900df91bb6SJason Gunthorpe CLIENT_DATA_REGISTERED); 591921eab11SJason Gunthorpe up_read(&device->client_data_rwsem); 592921eab11SJason Gunthorpe return 0; 5931da177e4SLinus Torvalds 594921eab11SJason Gunthorpe out: 595921eab11SJason Gunthorpe up_write(&device->client_data_rwsem); 596921eab11SJason Gunthorpe return ret; 597921eab11SJason Gunthorpe } 598921eab11SJason Gunthorpe 599921eab11SJason Gunthorpe static void remove_client_context(struct ib_device *device, 600921eab11SJason Gunthorpe unsigned int client_id) 601921eab11SJason Gunthorpe { 602921eab11SJason Gunthorpe struct ib_client *client; 603921eab11SJason Gunthorpe void *client_data; 604921eab11SJason Gunthorpe 605921eab11SJason Gunthorpe down_write(&device->client_data_rwsem); 606921eab11SJason Gunthorpe if (!xa_get_mark(&device->client_data, client_id, 607921eab11SJason Gunthorpe CLIENT_DATA_REGISTERED)) { 608921eab11SJason Gunthorpe up_write(&device->client_data_rwsem); 609921eab11SJason Gunthorpe return; 610921eab11SJason Gunthorpe } 611921eab11SJason Gunthorpe client_data = xa_load(&device->client_data, client_id); 612921eab11SJason Gunthorpe xa_clear_mark(&device->client_data, client_id, CLIENT_DATA_REGISTERED); 613921eab11SJason Gunthorpe client = xa_load(&clients, client_id); 614921eab11SJason Gunthorpe downgrade_write(&device->client_data_rwsem); 615921eab11SJason Gunthorpe 616921eab11SJason Gunthorpe /* 617921eab11SJason Gunthorpe * Notice we cannot be holding any exclusive locks when calling the 618921eab11SJason Gunthorpe * remove callback as the remove callback can recurse back into any 619921eab11SJason Gunthorpe * public functions in this module and thus try for any locks those 620921eab11SJason Gunthorpe * functions take. 621921eab11SJason Gunthorpe * 622921eab11SJason Gunthorpe * For this reason clients and drivers should not call the 623921eab11SJason Gunthorpe * unregistration functions will holdling any locks. 624921eab11SJason Gunthorpe * 625921eab11SJason Gunthorpe * It tempting to drop the client_data_rwsem too, but this is required 626921eab11SJason Gunthorpe * to ensure that unregister_client does not return until all clients 627921eab11SJason Gunthorpe * are completely unregistered, which is required to avoid module 628921eab11SJason Gunthorpe * unloading races. 629921eab11SJason Gunthorpe */ 630921eab11SJason Gunthorpe if (client->remove) 631921eab11SJason Gunthorpe client->remove(device, client_data); 632921eab11SJason Gunthorpe 633921eab11SJason Gunthorpe xa_erase(&device->client_data, client_id); 634921eab11SJason Gunthorpe up_read(&device->client_data_rwsem); 6351da177e4SLinus Torvalds } 6361da177e4SLinus Torvalds 637c2261dd7SJason Gunthorpe static int alloc_port_data(struct ib_device *device) 6385eb620c8SYosef Etigin { 639324e227eSJason Gunthorpe struct ib_port_data_rcu *pdata_rcu; 640ea1075edSJason Gunthorpe unsigned int port; 641c2261dd7SJason Gunthorpe 642c2261dd7SJason Gunthorpe if (device->port_data) 643c2261dd7SJason Gunthorpe return 0; 644c2261dd7SJason Gunthorpe 645c2261dd7SJason Gunthorpe /* This can only be called once the physical port range is defined */ 646c2261dd7SJason Gunthorpe if (WARN_ON(!device->phys_port_cnt)) 647c2261dd7SJason Gunthorpe return -EINVAL; 6485eb620c8SYosef Etigin 6498ceb1357SJason Gunthorpe /* 6508ceb1357SJason Gunthorpe * device->port_data is indexed directly by the port number to make 6517738613eSIra Weiny * access to this data as efficient as possible. 6527738613eSIra Weiny * 6538ceb1357SJason Gunthorpe * Therefore port_data is declared as a 1 based array with potential 6548ceb1357SJason Gunthorpe * empty slots at the beginning. 6557738613eSIra Weiny */ 656324e227eSJason Gunthorpe pdata_rcu = kzalloc(struct_size(pdata_rcu, pdata, 657324e227eSJason Gunthorpe rdma_end_port(device) + 1), 658324e227eSJason Gunthorpe GFP_KERNEL); 659324e227eSJason Gunthorpe if (!pdata_rcu) 66055aeed06SJason Gunthorpe return -ENOMEM; 661324e227eSJason Gunthorpe /* 662324e227eSJason Gunthorpe * The rcu_head is put in front of the port data array and the stored 663324e227eSJason Gunthorpe * pointer is adjusted since we never need to see that member until 664324e227eSJason Gunthorpe * kfree_rcu. 665324e227eSJason Gunthorpe */ 666324e227eSJason Gunthorpe device->port_data = pdata_rcu->pdata; 6675eb620c8SYosef Etigin 668ea1075edSJason Gunthorpe rdma_for_each_port (device, port) { 6698ceb1357SJason Gunthorpe struct ib_port_data *pdata = &device->port_data[port]; 6708ceb1357SJason Gunthorpe 671324e227eSJason Gunthorpe pdata->ib_dev = device; 6728ceb1357SJason Gunthorpe spin_lock_init(&pdata->pkey_list_lock); 6738ceb1357SJason Gunthorpe INIT_LIST_HEAD(&pdata->pkey_list); 674c2261dd7SJason Gunthorpe spin_lock_init(&pdata->netdev_lock); 675324e227eSJason Gunthorpe INIT_HLIST_NODE(&pdata->ndev_hash_link); 676c2261dd7SJason Gunthorpe } 677c2261dd7SJason Gunthorpe return 0; 678c2261dd7SJason Gunthorpe } 679c2261dd7SJason Gunthorpe 680c2261dd7SJason Gunthorpe static int verify_immutable(const struct ib_device *dev, u8 port) 681c2261dd7SJason Gunthorpe { 682c2261dd7SJason Gunthorpe return WARN_ON(!rdma_cap_ib_mad(dev, port) && 683c2261dd7SJason Gunthorpe rdma_max_mad_size(dev, port) != 0); 684c2261dd7SJason Gunthorpe } 685c2261dd7SJason Gunthorpe 686c2261dd7SJason Gunthorpe static int setup_port_data(struct ib_device *device) 687c2261dd7SJason Gunthorpe { 688c2261dd7SJason Gunthorpe unsigned int port; 689c2261dd7SJason Gunthorpe int ret; 690c2261dd7SJason Gunthorpe 691c2261dd7SJason Gunthorpe ret = alloc_port_data(device); 692c2261dd7SJason Gunthorpe if (ret) 693c2261dd7SJason Gunthorpe return ret; 694c2261dd7SJason Gunthorpe 695c2261dd7SJason Gunthorpe rdma_for_each_port (device, port) { 696c2261dd7SJason Gunthorpe struct ib_port_data *pdata = &device->port_data[port]; 6978ceb1357SJason Gunthorpe 6988ceb1357SJason Gunthorpe ret = device->ops.get_port_immutable(device, port, 6998ceb1357SJason Gunthorpe &pdata->immutable); 7005eb620c8SYosef Etigin if (ret) 7015eb620c8SYosef Etigin return ret; 70255aeed06SJason Gunthorpe 70355aeed06SJason Gunthorpe if (verify_immutable(device, port)) 70455aeed06SJason Gunthorpe return -EINVAL; 70555aeed06SJason Gunthorpe } 70655aeed06SJason Gunthorpe return 0; 7075eb620c8SYosef Etigin } 7085eb620c8SYosef Etigin 7099abb0d1bSLeon Romanovsky void ib_get_device_fw_str(struct ib_device *dev, char *str) 7105fa76c20SIra Weiny { 7113023a1e9SKamal Heib if (dev->ops.get_dev_fw_str) 7123023a1e9SKamal Heib dev->ops.get_dev_fw_str(dev, str); 7135fa76c20SIra Weiny else 7145fa76c20SIra Weiny str[0] = '\0'; 7155fa76c20SIra Weiny } 7165fa76c20SIra Weiny EXPORT_SYMBOL(ib_get_device_fw_str); 7175fa76c20SIra Weiny 7188f408ab6SDaniel Jurgens static void ib_policy_change_task(struct work_struct *work) 7198f408ab6SDaniel Jurgens { 7208f408ab6SDaniel Jurgens struct ib_device *dev; 7210df91bb6SJason Gunthorpe unsigned long index; 7228f408ab6SDaniel Jurgens 723921eab11SJason Gunthorpe down_read(&devices_rwsem); 7240df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 725ea1075edSJason Gunthorpe unsigned int i; 7268f408ab6SDaniel Jurgens 727ea1075edSJason Gunthorpe rdma_for_each_port (dev, i) { 7288f408ab6SDaniel Jurgens u64 sp; 7298f408ab6SDaniel Jurgens int ret = ib_get_cached_subnet_prefix(dev, 7308f408ab6SDaniel Jurgens i, 7318f408ab6SDaniel Jurgens &sp); 7328f408ab6SDaniel Jurgens 7338f408ab6SDaniel Jurgens WARN_ONCE(ret, 7348f408ab6SDaniel Jurgens "ib_get_cached_subnet_prefix err: %d, this should never happen here\n", 7358f408ab6SDaniel Jurgens ret); 736a750cfdeSDaniel Jurgens if (!ret) 7378f408ab6SDaniel Jurgens ib_security_cache_change(dev, i, sp); 7388f408ab6SDaniel Jurgens } 7398f408ab6SDaniel Jurgens } 740921eab11SJason Gunthorpe up_read(&devices_rwsem); 7418f408ab6SDaniel Jurgens } 7428f408ab6SDaniel Jurgens 7438f408ab6SDaniel Jurgens static int ib_security_change(struct notifier_block *nb, unsigned long event, 7448f408ab6SDaniel Jurgens void *lsm_data) 7458f408ab6SDaniel Jurgens { 7468f408ab6SDaniel Jurgens if (event != LSM_POLICY_CHANGE) 7478f408ab6SDaniel Jurgens return NOTIFY_DONE; 7488f408ab6SDaniel Jurgens 7498f408ab6SDaniel Jurgens schedule_work(&ib_policy_change_work); 750c66f6741SDaniel Jurgens ib_mad_agent_security_change(); 7518f408ab6SDaniel Jurgens 7528f408ab6SDaniel Jurgens return NOTIFY_OK; 7538f408ab6SDaniel Jurgens } 7548f408ab6SDaniel Jurgens 7554e0f7b90SParav Pandit static void compatdev_release(struct device *dev) 7564e0f7b90SParav Pandit { 7574e0f7b90SParav Pandit struct ib_core_device *cdev = 7584e0f7b90SParav Pandit container_of(dev, struct ib_core_device, dev); 7594e0f7b90SParav Pandit 7604e0f7b90SParav Pandit kfree(cdev); 7614e0f7b90SParav Pandit } 7624e0f7b90SParav Pandit 7634e0f7b90SParav Pandit static int add_one_compat_dev(struct ib_device *device, 7644e0f7b90SParav Pandit struct rdma_dev_net *rnet) 7654e0f7b90SParav Pandit { 7664e0f7b90SParav Pandit struct ib_core_device *cdev; 7674e0f7b90SParav Pandit int ret; 7684e0f7b90SParav Pandit 7692b34c558SParav Pandit lockdep_assert_held(&rdma_nets_rwsem); 770a56bc45bSParav Pandit if (!ib_devices_shared_netns) 771a56bc45bSParav Pandit return 0; 772a56bc45bSParav Pandit 7734e0f7b90SParav Pandit /* 7744e0f7b90SParav Pandit * Create and add compat device in all namespaces other than where it 7754e0f7b90SParav Pandit * is currently bound to. 7764e0f7b90SParav Pandit */ 7774e0f7b90SParav Pandit if (net_eq(read_pnet(&rnet->net), 7784e0f7b90SParav Pandit read_pnet(&device->coredev.rdma_net))) 7794e0f7b90SParav Pandit return 0; 7804e0f7b90SParav Pandit 7814e0f7b90SParav Pandit /* 7824e0f7b90SParav Pandit * The first of init_net() or ib_register_device() to take the 7834e0f7b90SParav Pandit * compat_devs_mutex wins and gets to add the device. Others will wait 7844e0f7b90SParav Pandit * for completion here. 7854e0f7b90SParav Pandit */ 7864e0f7b90SParav Pandit mutex_lock(&device->compat_devs_mutex); 7874e0f7b90SParav Pandit cdev = xa_load(&device->compat_devs, rnet->id); 7884e0f7b90SParav Pandit if (cdev) { 7894e0f7b90SParav Pandit ret = 0; 7904e0f7b90SParav Pandit goto done; 7914e0f7b90SParav Pandit } 7924e0f7b90SParav Pandit ret = xa_reserve(&device->compat_devs, rnet->id, GFP_KERNEL); 7934e0f7b90SParav Pandit if (ret) 7944e0f7b90SParav Pandit goto done; 7954e0f7b90SParav Pandit 7964e0f7b90SParav Pandit cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); 7974e0f7b90SParav Pandit if (!cdev) { 7984e0f7b90SParav Pandit ret = -ENOMEM; 7994e0f7b90SParav Pandit goto cdev_err; 8004e0f7b90SParav Pandit } 8014e0f7b90SParav Pandit 8024e0f7b90SParav Pandit cdev->dev.parent = device->dev.parent; 8034e0f7b90SParav Pandit rdma_init_coredev(cdev, device, read_pnet(&rnet->net)); 8044e0f7b90SParav Pandit cdev->dev.release = compatdev_release; 8054e0f7b90SParav Pandit dev_set_name(&cdev->dev, "%s", dev_name(&device->dev)); 8064e0f7b90SParav Pandit 8074e0f7b90SParav Pandit ret = device_add(&cdev->dev); 8084e0f7b90SParav Pandit if (ret) 8094e0f7b90SParav Pandit goto add_err; 8105417783eSParav Pandit ret = ib_setup_port_attrs(cdev, false); 8115417783eSParav Pandit if (ret) 8125417783eSParav Pandit goto port_err; 8134e0f7b90SParav Pandit 8144e0f7b90SParav Pandit ret = xa_err(xa_store(&device->compat_devs, rnet->id, 8154e0f7b90SParav Pandit cdev, GFP_KERNEL)); 8164e0f7b90SParav Pandit if (ret) 8174e0f7b90SParav Pandit goto insert_err; 8184e0f7b90SParav Pandit 8194e0f7b90SParav Pandit mutex_unlock(&device->compat_devs_mutex); 8204e0f7b90SParav Pandit return 0; 8214e0f7b90SParav Pandit 8224e0f7b90SParav Pandit insert_err: 8235417783eSParav Pandit ib_free_port_attrs(cdev); 8245417783eSParav Pandit port_err: 8254e0f7b90SParav Pandit device_del(&cdev->dev); 8264e0f7b90SParav Pandit add_err: 8274e0f7b90SParav Pandit put_device(&cdev->dev); 8284e0f7b90SParav Pandit cdev_err: 8294e0f7b90SParav Pandit xa_release(&device->compat_devs, rnet->id); 8304e0f7b90SParav Pandit done: 8314e0f7b90SParav Pandit mutex_unlock(&device->compat_devs_mutex); 8324e0f7b90SParav Pandit return ret; 8334e0f7b90SParav Pandit } 8344e0f7b90SParav Pandit 8354e0f7b90SParav Pandit static void remove_one_compat_dev(struct ib_device *device, u32 id) 8364e0f7b90SParav Pandit { 8374e0f7b90SParav Pandit struct ib_core_device *cdev; 8384e0f7b90SParav Pandit 8394e0f7b90SParav Pandit mutex_lock(&device->compat_devs_mutex); 8404e0f7b90SParav Pandit cdev = xa_erase(&device->compat_devs, id); 8414e0f7b90SParav Pandit mutex_unlock(&device->compat_devs_mutex); 8424e0f7b90SParav Pandit if (cdev) { 8435417783eSParav Pandit ib_free_port_attrs(cdev); 8444e0f7b90SParav Pandit device_del(&cdev->dev); 8454e0f7b90SParav Pandit put_device(&cdev->dev); 8464e0f7b90SParav Pandit } 8474e0f7b90SParav Pandit } 8484e0f7b90SParav Pandit 8494e0f7b90SParav Pandit static void remove_compat_devs(struct ib_device *device) 8504e0f7b90SParav Pandit { 8514e0f7b90SParav Pandit struct ib_core_device *cdev; 8524e0f7b90SParav Pandit unsigned long index; 8534e0f7b90SParav Pandit 8544e0f7b90SParav Pandit xa_for_each (&device->compat_devs, index, cdev) 8554e0f7b90SParav Pandit remove_one_compat_dev(device, index); 8564e0f7b90SParav Pandit } 8574e0f7b90SParav Pandit 8584e0f7b90SParav Pandit static int add_compat_devs(struct ib_device *device) 8594e0f7b90SParav Pandit { 8604e0f7b90SParav Pandit struct rdma_dev_net *rnet; 8614e0f7b90SParav Pandit unsigned long index; 8624e0f7b90SParav Pandit int ret = 0; 8634e0f7b90SParav Pandit 8644e0f7b90SParav Pandit down_read(&rdma_nets_rwsem); 8654e0f7b90SParav Pandit xa_for_each (&rdma_nets, index, rnet) { 8664e0f7b90SParav Pandit ret = add_one_compat_dev(device, rnet); 8674e0f7b90SParav Pandit if (ret) 8684e0f7b90SParav Pandit break; 8694e0f7b90SParav Pandit } 8704e0f7b90SParav Pandit up_read(&rdma_nets_rwsem); 8714e0f7b90SParav Pandit return ret; 8724e0f7b90SParav Pandit } 8734e0f7b90SParav Pandit 8742b34c558SParav Pandit static void remove_all_compat_devs(void) 8752b34c558SParav Pandit { 8762b34c558SParav Pandit struct ib_compat_device *cdev; 8772b34c558SParav Pandit struct ib_device *dev; 8782b34c558SParav Pandit unsigned long index; 8792b34c558SParav Pandit 8802b34c558SParav Pandit down_read(&devices_rwsem); 8812b34c558SParav Pandit xa_for_each (&devices, index, dev) { 8822b34c558SParav Pandit unsigned long c_index = 0; 8832b34c558SParav Pandit 8842b34c558SParav Pandit /* Hold nets_rwsem so that any other thread modifying this 8852b34c558SParav Pandit * system param can sync with this thread. 8862b34c558SParav Pandit */ 8872b34c558SParav Pandit down_read(&rdma_nets_rwsem); 8882b34c558SParav Pandit xa_for_each (&dev->compat_devs, c_index, cdev) 8892b34c558SParav Pandit remove_one_compat_dev(dev, c_index); 8902b34c558SParav Pandit up_read(&rdma_nets_rwsem); 8912b34c558SParav Pandit } 8922b34c558SParav Pandit up_read(&devices_rwsem); 8932b34c558SParav Pandit } 8942b34c558SParav Pandit 8952b34c558SParav Pandit static int add_all_compat_devs(void) 8962b34c558SParav Pandit { 8972b34c558SParav Pandit struct rdma_dev_net *rnet; 8982b34c558SParav Pandit struct ib_device *dev; 8992b34c558SParav Pandit unsigned long index; 9002b34c558SParav Pandit int ret = 0; 9012b34c558SParav Pandit 9022b34c558SParav Pandit down_read(&devices_rwsem); 9032b34c558SParav Pandit xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 9042b34c558SParav Pandit unsigned long net_index = 0; 9052b34c558SParav Pandit 9062b34c558SParav Pandit /* Hold nets_rwsem so that any other thread modifying this 9072b34c558SParav Pandit * system param can sync with this thread. 9082b34c558SParav Pandit */ 9092b34c558SParav Pandit down_read(&rdma_nets_rwsem); 9102b34c558SParav Pandit xa_for_each (&rdma_nets, net_index, rnet) { 9112b34c558SParav Pandit ret = add_one_compat_dev(dev, rnet); 9122b34c558SParav Pandit if (ret) 9132b34c558SParav Pandit break; 9142b34c558SParav Pandit } 9152b34c558SParav Pandit up_read(&rdma_nets_rwsem); 9162b34c558SParav Pandit } 9172b34c558SParav Pandit up_read(&devices_rwsem); 9182b34c558SParav Pandit if (ret) 9192b34c558SParav Pandit remove_all_compat_devs(); 9202b34c558SParav Pandit return ret; 9212b34c558SParav Pandit } 9222b34c558SParav Pandit 9232b34c558SParav Pandit int rdma_compatdev_set(u8 enable) 9242b34c558SParav Pandit { 9252b34c558SParav Pandit struct rdma_dev_net *rnet; 9262b34c558SParav Pandit unsigned long index; 9272b34c558SParav Pandit int ret = 0; 9282b34c558SParav Pandit 9292b34c558SParav Pandit down_write(&rdma_nets_rwsem); 9302b34c558SParav Pandit if (ib_devices_shared_netns == enable) { 9312b34c558SParav Pandit up_write(&rdma_nets_rwsem); 9322b34c558SParav Pandit return 0; 9332b34c558SParav Pandit } 9342b34c558SParav Pandit 9352b34c558SParav Pandit /* enable/disable of compat devices is not supported 9362b34c558SParav Pandit * when more than default init_net exists. 9372b34c558SParav Pandit */ 9382b34c558SParav Pandit xa_for_each (&rdma_nets, index, rnet) { 9392b34c558SParav Pandit ret++; 9402b34c558SParav Pandit break; 9412b34c558SParav Pandit } 9422b34c558SParav Pandit if (!ret) 9432b34c558SParav Pandit ib_devices_shared_netns = enable; 9442b34c558SParav Pandit up_write(&rdma_nets_rwsem); 9452b34c558SParav Pandit if (ret) 9462b34c558SParav Pandit return -EBUSY; 9472b34c558SParav Pandit 9482b34c558SParav Pandit if (enable) 9492b34c558SParav Pandit ret = add_all_compat_devs(); 9502b34c558SParav Pandit else 9512b34c558SParav Pandit remove_all_compat_devs(); 9522b34c558SParav Pandit return ret; 9532b34c558SParav Pandit } 9542b34c558SParav Pandit 9554e0f7b90SParav Pandit static void rdma_dev_exit_net(struct net *net) 9564e0f7b90SParav Pandit { 9574e0f7b90SParav Pandit struct rdma_dev_net *rnet = net_generic(net, rdma_dev_net_id); 9584e0f7b90SParav Pandit struct ib_device *dev; 9594e0f7b90SParav Pandit unsigned long index; 9604e0f7b90SParav Pandit int ret; 9614e0f7b90SParav Pandit 9624e0f7b90SParav Pandit down_write(&rdma_nets_rwsem); 9634e0f7b90SParav Pandit /* 9644e0f7b90SParav Pandit * Prevent the ID from being re-used and hide the id from xa_for_each. 9654e0f7b90SParav Pandit */ 9664e0f7b90SParav Pandit ret = xa_err(xa_store(&rdma_nets, rnet->id, NULL, GFP_KERNEL)); 9674e0f7b90SParav Pandit WARN_ON(ret); 9684e0f7b90SParav Pandit up_write(&rdma_nets_rwsem); 9694e0f7b90SParav Pandit 9704e0f7b90SParav Pandit down_read(&devices_rwsem); 9714e0f7b90SParav Pandit xa_for_each (&devices, index, dev) { 9724e0f7b90SParav Pandit get_device(&dev->dev); 9734e0f7b90SParav Pandit /* 9744e0f7b90SParav Pandit * Release the devices_rwsem so that pontentially blocking 9754e0f7b90SParav Pandit * device_del, doesn't hold the devices_rwsem for too long. 9764e0f7b90SParav Pandit */ 9774e0f7b90SParav Pandit up_read(&devices_rwsem); 9784e0f7b90SParav Pandit 9794e0f7b90SParav Pandit remove_one_compat_dev(dev, rnet->id); 9804e0f7b90SParav Pandit 9814e0f7b90SParav Pandit put_device(&dev->dev); 9824e0f7b90SParav Pandit down_read(&devices_rwsem); 9834e0f7b90SParav Pandit } 9844e0f7b90SParav Pandit up_read(&devices_rwsem); 9854e0f7b90SParav Pandit 9864e0f7b90SParav Pandit xa_erase(&rdma_nets, rnet->id); 9874e0f7b90SParav Pandit } 9884e0f7b90SParav Pandit 9894e0f7b90SParav Pandit static __net_init int rdma_dev_init_net(struct net *net) 9904e0f7b90SParav Pandit { 9914e0f7b90SParav Pandit struct rdma_dev_net *rnet = net_generic(net, rdma_dev_net_id); 9924e0f7b90SParav Pandit unsigned long index; 9934e0f7b90SParav Pandit struct ib_device *dev; 9944e0f7b90SParav Pandit int ret; 9954e0f7b90SParav Pandit 9964e0f7b90SParav Pandit /* No need to create any compat devices in default init_net. */ 9974e0f7b90SParav Pandit if (net_eq(net, &init_net)) 9984e0f7b90SParav Pandit return 0; 9994e0f7b90SParav Pandit 10004e0f7b90SParav Pandit write_pnet(&rnet->net, net); 10014e0f7b90SParav Pandit 10024e0f7b90SParav Pandit ret = xa_alloc(&rdma_nets, &rnet->id, rnet, xa_limit_32b, GFP_KERNEL); 10034e0f7b90SParav Pandit if (ret) 10044e0f7b90SParav Pandit return ret; 10054e0f7b90SParav Pandit 10064e0f7b90SParav Pandit down_read(&devices_rwsem); 10074e0f7b90SParav Pandit xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 10082b34c558SParav Pandit /* Hold nets_rwsem so that netlink command cannot change 10092b34c558SParav Pandit * system configuration for device sharing mode. 10102b34c558SParav Pandit */ 10112b34c558SParav Pandit down_read(&rdma_nets_rwsem); 10124e0f7b90SParav Pandit ret = add_one_compat_dev(dev, rnet); 10132b34c558SParav Pandit up_read(&rdma_nets_rwsem); 10144e0f7b90SParav Pandit if (ret) 10154e0f7b90SParav Pandit break; 10164e0f7b90SParav Pandit } 10174e0f7b90SParav Pandit up_read(&devices_rwsem); 10184e0f7b90SParav Pandit 10194e0f7b90SParav Pandit if (ret) 10204e0f7b90SParav Pandit rdma_dev_exit_net(net); 10214e0f7b90SParav Pandit 10224e0f7b90SParav Pandit return ret; 10234e0f7b90SParav Pandit } 10244e0f7b90SParav Pandit 1025ecc82c53SLeon Romanovsky /* 1026d0899892SJason Gunthorpe * Assign the unique string device name and the unique device index. This is 1027d0899892SJason Gunthorpe * undone by ib_dealloc_device. 1028ecc82c53SLeon Romanovsky */ 10290df91bb6SJason Gunthorpe static int assign_name(struct ib_device *device, const char *name) 10300df91bb6SJason Gunthorpe { 10310df91bb6SJason Gunthorpe static u32 last_id; 10320df91bb6SJason Gunthorpe int ret; 1033ecc82c53SLeon Romanovsky 1034921eab11SJason Gunthorpe down_write(&devices_rwsem); 10350df91bb6SJason Gunthorpe /* Assign a unique name to the device */ 10360df91bb6SJason Gunthorpe if (strchr(name, '%')) 10370df91bb6SJason Gunthorpe ret = alloc_name(device, name); 10380df91bb6SJason Gunthorpe else 10390df91bb6SJason Gunthorpe ret = dev_set_name(&device->dev, name); 10400df91bb6SJason Gunthorpe if (ret) 10410df91bb6SJason Gunthorpe goto out; 1042ecc82c53SLeon Romanovsky 10430df91bb6SJason Gunthorpe if (__ib_device_get_by_name(dev_name(&device->dev))) { 10440df91bb6SJason Gunthorpe ret = -ENFILE; 10450df91bb6SJason Gunthorpe goto out; 1046ecc82c53SLeon Romanovsky } 10470df91bb6SJason Gunthorpe strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); 10480df91bb6SJason Gunthorpe 1049ea295481SLinus Torvalds ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b, 1050ea295481SLinus Torvalds &last_id, GFP_KERNEL); 1051ea295481SLinus Torvalds if (ret > 0) 10520df91bb6SJason Gunthorpe ret = 0; 1053921eab11SJason Gunthorpe 10540df91bb6SJason Gunthorpe out: 1055921eab11SJason Gunthorpe up_write(&devices_rwsem); 10560df91bb6SJason Gunthorpe return ret; 10570df91bb6SJason Gunthorpe } 10580df91bb6SJason Gunthorpe 1059548cb4fbSParav Pandit static void setup_dma_device(struct ib_device *device) 10601da177e4SLinus Torvalds { 106199db9494SBart Van Assche struct device *parent = device->dev.parent; 10621da177e4SLinus Torvalds 10630957c29fSBart Van Assche WARN_ON_ONCE(device->dma_device); 10640957c29fSBart Van Assche if (device->dev.dma_ops) { 10650957c29fSBart Van Assche /* 10660957c29fSBart Van Assche * The caller provided custom DMA operations. Copy the 10670957c29fSBart Van Assche * DMA-related fields that are used by e.g. dma_alloc_coherent() 10680957c29fSBart Van Assche * into device->dev. 10690957c29fSBart Van Assche */ 10700957c29fSBart Van Assche device->dma_device = &device->dev; 107102ee9da3SBart Van Assche if (!device->dev.dma_mask) { 107202ee9da3SBart Van Assche if (parent) 107399db9494SBart Van Assche device->dev.dma_mask = parent->dma_mask; 107402ee9da3SBart Van Assche else 107502ee9da3SBart Van Assche WARN_ON_ONCE(true); 107602ee9da3SBart Van Assche } 107702ee9da3SBart Van Assche if (!device->dev.coherent_dma_mask) { 107802ee9da3SBart Van Assche if (parent) 10790957c29fSBart Van Assche device->dev.coherent_dma_mask = 10800957c29fSBart Van Assche parent->coherent_dma_mask; 108102ee9da3SBart Van Assche else 108202ee9da3SBart Van Assche WARN_ON_ONCE(true); 108302ee9da3SBart Van Assche } 10840957c29fSBart Van Assche } else { 10850957c29fSBart Van Assche /* 10860957c29fSBart Van Assche * The caller did not provide custom DMA operations. Use the 10870957c29fSBart Van Assche * DMA mapping operations of the parent device. 10880957c29fSBart Van Assche */ 108902ee9da3SBart Van Assche WARN_ON_ONCE(!parent); 10900957c29fSBart Van Assche device->dma_device = parent; 10910957c29fSBart Van Assche } 1092548cb4fbSParav Pandit } 1093548cb4fbSParav Pandit 1094921eab11SJason Gunthorpe /* 1095921eab11SJason Gunthorpe * setup_device() allocates memory and sets up data that requires calling the 1096921eab11SJason Gunthorpe * device ops, this is the only reason these actions are not done during 1097921eab11SJason Gunthorpe * ib_alloc_device. It is undone by ib_dealloc_device(). 1098921eab11SJason Gunthorpe */ 1099548cb4fbSParav Pandit static int setup_device(struct ib_device *device) 1100548cb4fbSParav Pandit { 1101548cb4fbSParav Pandit struct ib_udata uhw = {.outlen = 0, .inlen = 0}; 1102548cb4fbSParav Pandit int ret; 1103548cb4fbSParav Pandit 1104921eab11SJason Gunthorpe setup_dma_device(device); 1105921eab11SJason Gunthorpe 1106548cb4fbSParav Pandit ret = ib_device_check_mandatory(device); 1107548cb4fbSParav Pandit if (ret) 1108548cb4fbSParav Pandit return ret; 1109548cb4fbSParav Pandit 11108ceb1357SJason Gunthorpe ret = setup_port_data(device); 1111548cb4fbSParav Pandit if (ret) { 11128ceb1357SJason Gunthorpe dev_warn(&device->dev, "Couldn't create per-port data\n"); 1113548cb4fbSParav Pandit return ret; 1114548cb4fbSParav Pandit } 1115548cb4fbSParav Pandit 1116548cb4fbSParav Pandit memset(&device->attrs, 0, sizeof(device->attrs)); 11173023a1e9SKamal Heib ret = device->ops.query_device(device, &device->attrs, &uhw); 1118548cb4fbSParav Pandit if (ret) { 1119548cb4fbSParav Pandit dev_warn(&device->dev, 1120548cb4fbSParav Pandit "Couldn't query the device attributes\n"); 1121d45f89d5SJason Gunthorpe return ret; 1122548cb4fbSParav Pandit } 1123548cb4fbSParav Pandit 1124548cb4fbSParav Pandit return 0; 1125548cb4fbSParav Pandit } 1126548cb4fbSParav Pandit 1127921eab11SJason Gunthorpe static void disable_device(struct ib_device *device) 1128921eab11SJason Gunthorpe { 1129921eab11SJason Gunthorpe struct ib_client *client; 1130921eab11SJason Gunthorpe 1131921eab11SJason Gunthorpe WARN_ON(!refcount_read(&device->refcount)); 1132921eab11SJason Gunthorpe 1133921eab11SJason Gunthorpe down_write(&devices_rwsem); 1134921eab11SJason Gunthorpe xa_clear_mark(&devices, device->index, DEVICE_REGISTERED); 1135921eab11SJason Gunthorpe up_write(&devices_rwsem); 1136921eab11SJason Gunthorpe 1137921eab11SJason Gunthorpe down_read(&clients_rwsem); 1138921eab11SJason Gunthorpe list_for_each_entry_reverse(client, &client_list, list) 1139921eab11SJason Gunthorpe remove_client_context(device, client->client_id); 1140921eab11SJason Gunthorpe up_read(&clients_rwsem); 1141921eab11SJason Gunthorpe 1142921eab11SJason Gunthorpe /* Pairs with refcount_set in enable_device */ 1143921eab11SJason Gunthorpe ib_device_put(device); 1144921eab11SJason Gunthorpe wait_for_completion(&device->unreg_completion); 1145c2261dd7SJason Gunthorpe 11464e0f7b90SParav Pandit /* 11474e0f7b90SParav Pandit * compat devices must be removed after device refcount drops to zero. 11484e0f7b90SParav Pandit * Otherwise init_net() may add more compatdevs after removing compat 11494e0f7b90SParav Pandit * devices and before device is disabled. 11504e0f7b90SParav Pandit */ 11514e0f7b90SParav Pandit remove_compat_devs(device); 11524e0f7b90SParav Pandit 1153c2261dd7SJason Gunthorpe /* Expedite removing unregistered pointers from the hash table */ 1154c2261dd7SJason Gunthorpe free_netdevs(device); 1155921eab11SJason Gunthorpe } 1156921eab11SJason Gunthorpe 1157921eab11SJason Gunthorpe /* 1158921eab11SJason Gunthorpe * An enabled device is visible to all clients and to all the public facing 1159d0899892SJason Gunthorpe * APIs that return a device pointer. This always returns with a new get, even 1160d0899892SJason Gunthorpe * if it fails. 1161921eab11SJason Gunthorpe */ 1162d0899892SJason Gunthorpe static int enable_device_and_get(struct ib_device *device) 1163921eab11SJason Gunthorpe { 1164921eab11SJason Gunthorpe struct ib_client *client; 1165921eab11SJason Gunthorpe unsigned long index; 1166d0899892SJason Gunthorpe int ret = 0; 1167921eab11SJason Gunthorpe 1168d0899892SJason Gunthorpe /* 1169d0899892SJason Gunthorpe * One ref belongs to the xa and the other belongs to this 1170d0899892SJason Gunthorpe * thread. This is needed to guard against parallel unregistration. 1171d0899892SJason Gunthorpe */ 1172d0899892SJason Gunthorpe refcount_set(&device->refcount, 2); 1173921eab11SJason Gunthorpe down_write(&devices_rwsem); 1174921eab11SJason Gunthorpe xa_set_mark(&devices, device->index, DEVICE_REGISTERED); 1175d0899892SJason Gunthorpe 1176d0899892SJason Gunthorpe /* 1177d0899892SJason Gunthorpe * By using downgrade_write() we ensure that no other thread can clear 1178d0899892SJason Gunthorpe * DEVICE_REGISTERED while we are completing the client setup. 1179d0899892SJason Gunthorpe */ 1180d0899892SJason Gunthorpe downgrade_write(&devices_rwsem); 1181921eab11SJason Gunthorpe 1182ca22354bSJason Gunthorpe if (device->ops.enable_driver) { 1183ca22354bSJason Gunthorpe ret = device->ops.enable_driver(device); 1184ca22354bSJason Gunthorpe if (ret) 1185ca22354bSJason Gunthorpe goto out; 1186ca22354bSJason Gunthorpe } 1187ca22354bSJason Gunthorpe 1188921eab11SJason Gunthorpe down_read(&clients_rwsem); 1189921eab11SJason Gunthorpe xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) { 1190921eab11SJason Gunthorpe ret = add_client_context(device, client); 1191d0899892SJason Gunthorpe if (ret) 1192d0899892SJason Gunthorpe break; 1193d0899892SJason Gunthorpe } 1194921eab11SJason Gunthorpe up_read(&clients_rwsem); 11954e0f7b90SParav Pandit if (!ret) 11964e0f7b90SParav Pandit ret = add_compat_devs(device); 1197ca22354bSJason Gunthorpe out: 1198d0899892SJason Gunthorpe up_read(&devices_rwsem); 1199921eab11SJason Gunthorpe return ret; 1200921eab11SJason Gunthorpe } 1201921eab11SJason Gunthorpe 1202548cb4fbSParav Pandit /** 1203548cb4fbSParav Pandit * ib_register_device - Register an IB device with IB core 1204548cb4fbSParav Pandit * @device:Device to register 1205548cb4fbSParav Pandit * 1206548cb4fbSParav Pandit * Low-level drivers use ib_register_device() to register their 1207548cb4fbSParav Pandit * devices with the IB core. All registered clients will receive a 1208548cb4fbSParav Pandit * callback for each device that is added. @device must be allocated 1209548cb4fbSParav Pandit * with ib_alloc_device(). 1210d0899892SJason Gunthorpe * 1211d0899892SJason Gunthorpe * If the driver uses ops.dealloc_driver and calls any ib_unregister_device() 1212d0899892SJason Gunthorpe * asynchronously then the device pointer may become freed as soon as this 1213d0899892SJason Gunthorpe * function returns. 1214548cb4fbSParav Pandit */ 1215ea4baf7fSParav Pandit int ib_register_device(struct ib_device *device, const char *name) 1216548cb4fbSParav Pandit { 1217548cb4fbSParav Pandit int ret; 12181da177e4SLinus Torvalds 12190df91bb6SJason Gunthorpe ret = assign_name(device, name); 1220e349f858SJason Gunthorpe if (ret) 1221921eab11SJason Gunthorpe return ret; 12221da177e4SLinus Torvalds 1223548cb4fbSParav Pandit ret = setup_device(device); 1224548cb4fbSParav Pandit if (ret) 1225d0899892SJason Gunthorpe return ret; 122603db3a2dSMatan Barak 1227d45f89d5SJason Gunthorpe ret = ib_cache_setup_one(device); 1228d45f89d5SJason Gunthorpe if (ret) { 1229d45f89d5SJason Gunthorpe dev_warn(&device->dev, 1230d45f89d5SJason Gunthorpe "Couldn't set up InfiniBand P_Key/GID cache\n"); 1231d0899892SJason Gunthorpe return ret; 1232d45f89d5SJason Gunthorpe } 1233d45f89d5SJason Gunthorpe 12347527a7b1SParav Pandit ib_device_register_rdmacg(device); 12353e153a93SIra Weiny 12365f8f5499SParav Pandit ret = device_add(&device->dev); 12375f8f5499SParav Pandit if (ret) 12385f8f5499SParav Pandit goto cg_cleanup; 12395f8f5499SParav Pandit 1240ea4baf7fSParav Pandit ret = ib_device_register_sysfs(device); 12411da177e4SLinus Torvalds if (ret) { 124243c7c851SJason Gunthorpe dev_warn(&device->dev, 124343c7c851SJason Gunthorpe "Couldn't register device with driver model\n"); 12445f8f5499SParav Pandit goto dev_cleanup; 12451da177e4SLinus Torvalds } 12461da177e4SLinus Torvalds 1247d0899892SJason Gunthorpe ret = enable_device_and_get(device); 1248d0899892SJason Gunthorpe if (ret) { 1249d0899892SJason Gunthorpe void (*dealloc_fn)(struct ib_device *); 1250d0899892SJason Gunthorpe 1251d0899892SJason Gunthorpe /* 1252d0899892SJason Gunthorpe * If we hit this error flow then we don't want to 1253d0899892SJason Gunthorpe * automatically dealloc the device since the caller is 1254d0899892SJason Gunthorpe * expected to call ib_dealloc_device() after 1255d0899892SJason Gunthorpe * ib_register_device() fails. This is tricky due to the 1256d0899892SJason Gunthorpe * possibility for a parallel unregistration along with this 1257d0899892SJason Gunthorpe * error flow. Since we have a refcount here we know any 1258d0899892SJason Gunthorpe * parallel flow is stopped in disable_device and will see the 1259d0899892SJason Gunthorpe * NULL pointers, causing the responsibility to 1260d0899892SJason Gunthorpe * ib_dealloc_device() to revert back to this thread. 1261d0899892SJason Gunthorpe */ 1262d0899892SJason Gunthorpe dealloc_fn = device->ops.dealloc_driver; 1263d0899892SJason Gunthorpe device->ops.dealloc_driver = NULL; 1264d0899892SJason Gunthorpe ib_device_put(device); 1265d0899892SJason Gunthorpe __ib_unregister_device(device); 1266d0899892SJason Gunthorpe device->ops.dealloc_driver = dealloc_fn; 1267d0899892SJason Gunthorpe return ret; 1268d0899892SJason Gunthorpe } 1269d0899892SJason Gunthorpe ib_device_put(device); 12701da177e4SLinus Torvalds 12714be3a4faSParav Pandit return 0; 12724be3a4faSParav Pandit 12735f8f5499SParav Pandit dev_cleanup: 12745f8f5499SParav Pandit device_del(&device->dev); 12752fb4f4eaSParav Pandit cg_cleanup: 12762fb4f4eaSParav Pandit ib_device_unregister_rdmacg(device); 1277d45f89d5SJason Gunthorpe ib_cache_cleanup_one(device); 12781da177e4SLinus Torvalds return ret; 12791da177e4SLinus Torvalds } 12801da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_device); 12811da177e4SLinus Torvalds 1282d0899892SJason Gunthorpe /* Callers must hold a get on the device. */ 1283d0899892SJason Gunthorpe static void __ib_unregister_device(struct ib_device *ib_dev) 1284d0899892SJason Gunthorpe { 1285d0899892SJason Gunthorpe /* 1286d0899892SJason Gunthorpe * We have a registration lock so that all the calls to unregister are 1287d0899892SJason Gunthorpe * fully fenced, once any unregister returns the device is truely 1288d0899892SJason Gunthorpe * unregistered even if multiple callers are unregistering it at the 1289d0899892SJason Gunthorpe * same time. This also interacts with the registration flow and 1290d0899892SJason Gunthorpe * provides sane semantics if register and unregister are racing. 1291d0899892SJason Gunthorpe */ 1292d0899892SJason Gunthorpe mutex_lock(&ib_dev->unregistration_lock); 1293d0899892SJason Gunthorpe if (!refcount_read(&ib_dev->refcount)) 1294d0899892SJason Gunthorpe goto out; 1295d0899892SJason Gunthorpe 1296d0899892SJason Gunthorpe disable_device(ib_dev); 1297d0899892SJason Gunthorpe ib_device_unregister_sysfs(ib_dev); 1298d0899892SJason Gunthorpe device_del(&ib_dev->dev); 1299d0899892SJason Gunthorpe ib_device_unregister_rdmacg(ib_dev); 1300d0899892SJason Gunthorpe ib_cache_cleanup_one(ib_dev); 1301d0899892SJason Gunthorpe 1302d0899892SJason Gunthorpe /* 1303d0899892SJason Gunthorpe * Drivers using the new flow may not call ib_dealloc_device except 1304d0899892SJason Gunthorpe * in error unwind prior to registration success. 1305d0899892SJason Gunthorpe */ 1306d0899892SJason Gunthorpe if (ib_dev->ops.dealloc_driver) { 1307d0899892SJason Gunthorpe WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1); 1308d0899892SJason Gunthorpe ib_dealloc_device(ib_dev); 1309d0899892SJason Gunthorpe } 1310d0899892SJason Gunthorpe out: 1311d0899892SJason Gunthorpe mutex_unlock(&ib_dev->unregistration_lock); 1312d0899892SJason Gunthorpe } 1313d0899892SJason Gunthorpe 13141da177e4SLinus Torvalds /** 13151da177e4SLinus Torvalds * ib_unregister_device - Unregister an IB device 1316d0899892SJason Gunthorpe * @device: The device to unregister 13171da177e4SLinus Torvalds * 13181da177e4SLinus Torvalds * Unregister an IB device. All clients will receive a remove callback. 1319d0899892SJason Gunthorpe * 1320d0899892SJason Gunthorpe * Callers should call this routine only once, and protect against races with 1321d0899892SJason Gunthorpe * registration. Typically it should only be called as part of a remove 1322d0899892SJason Gunthorpe * callback in an implementation of driver core's struct device_driver and 1323d0899892SJason Gunthorpe * related. 1324d0899892SJason Gunthorpe * 1325d0899892SJason Gunthorpe * If ops.dealloc_driver is used then ib_dev will be freed upon return from 1326d0899892SJason Gunthorpe * this function. 13271da177e4SLinus Torvalds */ 1328d0899892SJason Gunthorpe void ib_unregister_device(struct ib_device *ib_dev) 13291da177e4SLinus Torvalds { 1330d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1331d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1332d0899892SJason Gunthorpe put_device(&ib_dev->dev); 13331da177e4SLinus Torvalds } 13341da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_device); 13351da177e4SLinus Torvalds 1336d0899892SJason Gunthorpe /** 1337d0899892SJason Gunthorpe * ib_unregister_device_and_put - Unregister a device while holding a 'get' 1338d0899892SJason Gunthorpe * device: The device to unregister 1339d0899892SJason Gunthorpe * 1340d0899892SJason Gunthorpe * This is the same as ib_unregister_device(), except it includes an internal 1341d0899892SJason Gunthorpe * ib_device_put() that should match a 'get' obtained by the caller. 1342d0899892SJason Gunthorpe * 1343d0899892SJason Gunthorpe * It is safe to call this routine concurrently from multiple threads while 1344d0899892SJason Gunthorpe * holding the 'get'. When the function returns the device is fully 1345d0899892SJason Gunthorpe * unregistered. 1346d0899892SJason Gunthorpe * 1347d0899892SJason Gunthorpe * Drivers using this flow MUST use the driver_unregister callback to clean up 1348d0899892SJason Gunthorpe * their resources associated with the device and dealloc it. 1349d0899892SJason Gunthorpe */ 1350d0899892SJason Gunthorpe void ib_unregister_device_and_put(struct ib_device *ib_dev) 1351d0899892SJason Gunthorpe { 1352d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 1353d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1354d0899892SJason Gunthorpe ib_device_put(ib_dev); 1355d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1356d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1357d0899892SJason Gunthorpe } 1358d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_device_and_put); 1359d0899892SJason Gunthorpe 1360d0899892SJason Gunthorpe /** 1361d0899892SJason Gunthorpe * ib_unregister_driver - Unregister all IB devices for a driver 1362d0899892SJason Gunthorpe * @driver_id: The driver to unregister 1363d0899892SJason Gunthorpe * 1364d0899892SJason Gunthorpe * This implements a fence for device unregistration. It only returns once all 1365d0899892SJason Gunthorpe * devices associated with the driver_id have fully completed their 1366d0899892SJason Gunthorpe * unregistration and returned from ib_unregister_device*(). 1367d0899892SJason Gunthorpe * 1368d0899892SJason Gunthorpe * If device's are not yet unregistered it goes ahead and starts unregistering 1369d0899892SJason Gunthorpe * them. 1370d0899892SJason Gunthorpe * 1371d0899892SJason Gunthorpe * This does not block creation of new devices with the given driver_id, that 1372d0899892SJason Gunthorpe * is the responsibility of the caller. 1373d0899892SJason Gunthorpe */ 1374d0899892SJason Gunthorpe void ib_unregister_driver(enum rdma_driver_id driver_id) 1375d0899892SJason Gunthorpe { 1376d0899892SJason Gunthorpe struct ib_device *ib_dev; 1377d0899892SJason Gunthorpe unsigned long index; 1378d0899892SJason Gunthorpe 1379d0899892SJason Gunthorpe down_read(&devices_rwsem); 1380d0899892SJason Gunthorpe xa_for_each (&devices, index, ib_dev) { 1381d0899892SJason Gunthorpe if (ib_dev->driver_id != driver_id) 1382d0899892SJason Gunthorpe continue; 1383d0899892SJason Gunthorpe 1384d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1385d0899892SJason Gunthorpe up_read(&devices_rwsem); 1386d0899892SJason Gunthorpe 1387d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 1388d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1389d0899892SJason Gunthorpe 1390d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1391d0899892SJason Gunthorpe down_read(&devices_rwsem); 1392d0899892SJason Gunthorpe } 1393d0899892SJason Gunthorpe up_read(&devices_rwsem); 1394d0899892SJason Gunthorpe } 1395d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_driver); 1396d0899892SJason Gunthorpe 1397d0899892SJason Gunthorpe static void ib_unregister_work(struct work_struct *work) 1398d0899892SJason Gunthorpe { 1399d0899892SJason Gunthorpe struct ib_device *ib_dev = 1400d0899892SJason Gunthorpe container_of(work, struct ib_device, unregistration_work); 1401d0899892SJason Gunthorpe 1402d0899892SJason Gunthorpe __ib_unregister_device(ib_dev); 1403d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1404d0899892SJason Gunthorpe } 1405d0899892SJason Gunthorpe 1406d0899892SJason Gunthorpe /** 1407d0899892SJason Gunthorpe * ib_unregister_device_queued - Unregister a device using a work queue 1408d0899892SJason Gunthorpe * device: The device to unregister 1409d0899892SJason Gunthorpe * 1410d0899892SJason Gunthorpe * This schedules an asynchronous unregistration using a WQ for the device. A 1411d0899892SJason Gunthorpe * driver should use this to avoid holding locks while doing unregistration, 1412d0899892SJason Gunthorpe * such as holding the RTNL lock. 1413d0899892SJason Gunthorpe * 1414d0899892SJason Gunthorpe * Drivers using this API must use ib_unregister_driver before module unload 1415d0899892SJason Gunthorpe * to ensure that all scheduled unregistrations have completed. 1416d0899892SJason Gunthorpe */ 1417d0899892SJason Gunthorpe void ib_unregister_device_queued(struct ib_device *ib_dev) 1418d0899892SJason Gunthorpe { 1419d0899892SJason Gunthorpe WARN_ON(!refcount_read(&ib_dev->refcount)); 1420d0899892SJason Gunthorpe WARN_ON(!ib_dev->ops.dealloc_driver); 1421d0899892SJason Gunthorpe get_device(&ib_dev->dev); 1422d0899892SJason Gunthorpe if (!queue_work(system_unbound_wq, &ib_dev->unregistration_work)) 1423d0899892SJason Gunthorpe put_device(&ib_dev->dev); 1424d0899892SJason Gunthorpe } 1425d0899892SJason Gunthorpe EXPORT_SYMBOL(ib_unregister_device_queued); 1426d0899892SJason Gunthorpe 14274e0f7b90SParav Pandit static struct pernet_operations rdma_dev_net_ops = { 14284e0f7b90SParav Pandit .init = rdma_dev_init_net, 14294e0f7b90SParav Pandit .exit = rdma_dev_exit_net, 14304e0f7b90SParav Pandit .id = &rdma_dev_net_id, 14314e0f7b90SParav Pandit .size = sizeof(struct rdma_dev_net), 14324e0f7b90SParav Pandit }; 14334e0f7b90SParav Pandit 1434e59178d8SJason Gunthorpe static int assign_client_id(struct ib_client *client) 1435e59178d8SJason Gunthorpe { 1436e59178d8SJason Gunthorpe int ret; 1437e59178d8SJason Gunthorpe 1438921eab11SJason Gunthorpe down_write(&clients_rwsem); 1439e59178d8SJason Gunthorpe /* 1440e59178d8SJason Gunthorpe * The add/remove callbacks must be called in FIFO/LIFO order. To 1441e59178d8SJason Gunthorpe * achieve this we assign client_ids so they are sorted in 1442e59178d8SJason Gunthorpe * registration order, and retain a linked list we can reverse iterate 1443e59178d8SJason Gunthorpe * to get the LIFO order. The extra linked list can go away if xarray 1444e59178d8SJason Gunthorpe * learns to reverse iterate. 1445e59178d8SJason Gunthorpe */ 1446ea295481SLinus Torvalds if (list_empty(&client_list)) { 1447e59178d8SJason Gunthorpe client->client_id = 0; 1448ea295481SLinus Torvalds } else { 1449ea295481SLinus Torvalds struct ib_client *last; 1450ea295481SLinus Torvalds 1451ea295481SLinus Torvalds last = list_last_entry(&client_list, struct ib_client, list); 1452ea295481SLinus Torvalds client->client_id = last->client_id + 1; 1453ea295481SLinus Torvalds } 1454ea295481SLinus Torvalds ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL); 1455e59178d8SJason Gunthorpe if (ret) 1456e59178d8SJason Gunthorpe goto out; 1457e59178d8SJason Gunthorpe 1458921eab11SJason Gunthorpe xa_set_mark(&clients, client->client_id, CLIENT_REGISTERED); 1459921eab11SJason Gunthorpe list_add_tail(&client->list, &client_list); 1460921eab11SJason Gunthorpe 1461e59178d8SJason Gunthorpe out: 1462921eab11SJason Gunthorpe up_write(&clients_rwsem); 1463e59178d8SJason Gunthorpe return ret; 1464e59178d8SJason Gunthorpe } 1465e59178d8SJason Gunthorpe 14661da177e4SLinus Torvalds /** 14671da177e4SLinus Torvalds * ib_register_client - Register an IB client 14681da177e4SLinus Torvalds * @client:Client to register 14691da177e4SLinus Torvalds * 14701da177e4SLinus Torvalds * Upper level users of the IB drivers can use ib_register_client() to 14711da177e4SLinus Torvalds * register callbacks for IB device addition and removal. When an IB 14721da177e4SLinus Torvalds * device is added, each registered client's add method will be called 14731da177e4SLinus Torvalds * (in the order the clients were registered), and when a device is 14741da177e4SLinus Torvalds * removed, each client's remove method will be called (in the reverse 14751da177e4SLinus Torvalds * order that clients were registered). In addition, when 14761da177e4SLinus Torvalds * ib_register_client() is called, the client will receive an add 14771da177e4SLinus Torvalds * callback for all devices already registered. 14781da177e4SLinus Torvalds */ 14791da177e4SLinus Torvalds int ib_register_client(struct ib_client *client) 14801da177e4SLinus Torvalds { 14811da177e4SLinus Torvalds struct ib_device *device; 14820df91bb6SJason Gunthorpe unsigned long index; 1483e59178d8SJason Gunthorpe int ret; 14841da177e4SLinus Torvalds 1485e59178d8SJason Gunthorpe ret = assign_client_id(client); 1486921eab11SJason Gunthorpe if (ret) 1487921eab11SJason Gunthorpe return ret; 1488921eab11SJason Gunthorpe 1489921eab11SJason Gunthorpe down_read(&devices_rwsem); 1490921eab11SJason Gunthorpe xa_for_each_marked (&devices, index, device, DEVICE_REGISTERED) { 1491921eab11SJason Gunthorpe ret = add_client_context(device, client); 1492e59178d8SJason Gunthorpe if (ret) { 1493921eab11SJason Gunthorpe up_read(&devices_rwsem); 1494921eab11SJason Gunthorpe ib_unregister_client(client); 1495e59178d8SJason Gunthorpe return ret; 1496e59178d8SJason Gunthorpe } 1497921eab11SJason Gunthorpe } 1498921eab11SJason Gunthorpe up_read(&devices_rwsem); 14991da177e4SLinus Torvalds return 0; 15001da177e4SLinus Torvalds } 15011da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_client); 15021da177e4SLinus Torvalds 15031da177e4SLinus Torvalds /** 15041da177e4SLinus Torvalds * ib_unregister_client - Unregister an IB client 15051da177e4SLinus Torvalds * @client:Client to unregister 15061da177e4SLinus Torvalds * 15071da177e4SLinus Torvalds * Upper level users use ib_unregister_client() to remove their client 15081da177e4SLinus Torvalds * registration. When ib_unregister_client() is called, the client 15091da177e4SLinus Torvalds * will receive a remove callback for each IB device still registered. 1510921eab11SJason Gunthorpe * 1511921eab11SJason Gunthorpe * This is a full fence, once it returns no client callbacks will be called, 1512921eab11SJason Gunthorpe * or are running in another thread. 15131da177e4SLinus Torvalds */ 15141da177e4SLinus Torvalds void ib_unregister_client(struct ib_client *client) 15151da177e4SLinus Torvalds { 15161da177e4SLinus Torvalds struct ib_device *device; 15170df91bb6SJason Gunthorpe unsigned long index; 15181da177e4SLinus Torvalds 1519921eab11SJason Gunthorpe down_write(&clients_rwsem); 1520e59178d8SJason Gunthorpe xa_clear_mark(&clients, client->client_id, CLIENT_REGISTERED); 1521921eab11SJason Gunthorpe up_write(&clients_rwsem); 1522921eab11SJason Gunthorpe /* 1523921eab11SJason Gunthorpe * Every device still known must be serialized to make sure we are 1524921eab11SJason Gunthorpe * done with the client callbacks before we return. 1525921eab11SJason Gunthorpe */ 1526921eab11SJason Gunthorpe down_read(&devices_rwsem); 1527921eab11SJason Gunthorpe xa_for_each (&devices, index, device) 1528921eab11SJason Gunthorpe remove_client_context(device, client->client_id); 1529921eab11SJason Gunthorpe up_read(&devices_rwsem); 15305aa44bb9SHaggai Eran 1531921eab11SJason Gunthorpe down_write(&clients_rwsem); 1532e59178d8SJason Gunthorpe list_del(&client->list); 1533e59178d8SJason Gunthorpe xa_erase(&clients, client->client_id); 1534921eab11SJason Gunthorpe up_write(&clients_rwsem); 15351da177e4SLinus Torvalds } 15361da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_client); 15371da177e4SLinus Torvalds 15381da177e4SLinus Torvalds /** 15399cd330d3SKrishna Kumar * ib_set_client_data - Set IB client context 15401da177e4SLinus Torvalds * @device:Device to set context for 15411da177e4SLinus Torvalds * @client:Client to set context for 15421da177e4SLinus Torvalds * @data:Context to set 15431da177e4SLinus Torvalds * 15440df91bb6SJason Gunthorpe * ib_set_client_data() sets client context data that can be retrieved with 15450df91bb6SJason Gunthorpe * ib_get_client_data(). This can only be called while the client is 15460df91bb6SJason Gunthorpe * registered to the device, once the ib_client remove() callback returns this 15470df91bb6SJason Gunthorpe * cannot be called. 15481da177e4SLinus Torvalds */ 15491da177e4SLinus Torvalds void ib_set_client_data(struct ib_device *device, struct ib_client *client, 15501da177e4SLinus Torvalds void *data) 15511da177e4SLinus Torvalds { 15520df91bb6SJason Gunthorpe void *rc; 15531da177e4SLinus Torvalds 15540df91bb6SJason Gunthorpe if (WARN_ON(IS_ERR(data))) 15550df91bb6SJason Gunthorpe data = NULL; 15561da177e4SLinus Torvalds 15570df91bb6SJason Gunthorpe rc = xa_store(&device->client_data, client->client_id, data, 15580df91bb6SJason Gunthorpe GFP_KERNEL); 15590df91bb6SJason Gunthorpe WARN_ON(xa_is_err(rc)); 15601da177e4SLinus Torvalds } 15611da177e4SLinus Torvalds EXPORT_SYMBOL(ib_set_client_data); 15621da177e4SLinus Torvalds 15631da177e4SLinus Torvalds /** 15641da177e4SLinus Torvalds * ib_register_event_handler - Register an IB event handler 15651da177e4SLinus Torvalds * @event_handler:Handler to register 15661da177e4SLinus Torvalds * 15671da177e4SLinus Torvalds * ib_register_event_handler() registers an event handler that will be 15681da177e4SLinus Torvalds * called back when asynchronous IB events occur (as defined in 15691da177e4SLinus Torvalds * chapter 11 of the InfiniBand Architecture Specification). This 15701da177e4SLinus Torvalds * callback may occur in interrupt context. 15711da177e4SLinus Torvalds */ 1572dcc9881eSLeon Romanovsky void ib_register_event_handler(struct ib_event_handler *event_handler) 15731da177e4SLinus Torvalds { 15741da177e4SLinus Torvalds unsigned long flags; 15751da177e4SLinus Torvalds 15761da177e4SLinus Torvalds spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 15771da177e4SLinus Torvalds list_add_tail(&event_handler->list, 15781da177e4SLinus Torvalds &event_handler->device->event_handler_list); 15791da177e4SLinus Torvalds spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 15801da177e4SLinus Torvalds } 15811da177e4SLinus Torvalds EXPORT_SYMBOL(ib_register_event_handler); 15821da177e4SLinus Torvalds 15831da177e4SLinus Torvalds /** 15841da177e4SLinus Torvalds * ib_unregister_event_handler - Unregister an event handler 15851da177e4SLinus Torvalds * @event_handler:Handler to unregister 15861da177e4SLinus Torvalds * 15871da177e4SLinus Torvalds * Unregister an event handler registered with 15881da177e4SLinus Torvalds * ib_register_event_handler(). 15891da177e4SLinus Torvalds */ 1590dcc9881eSLeon Romanovsky void ib_unregister_event_handler(struct ib_event_handler *event_handler) 15911da177e4SLinus Torvalds { 15921da177e4SLinus Torvalds unsigned long flags; 15931da177e4SLinus Torvalds 15941da177e4SLinus Torvalds spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); 15951da177e4SLinus Torvalds list_del(&event_handler->list); 15961da177e4SLinus Torvalds spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); 15971da177e4SLinus Torvalds } 15981da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unregister_event_handler); 15991da177e4SLinus Torvalds 16001da177e4SLinus Torvalds /** 16011da177e4SLinus Torvalds * ib_dispatch_event - Dispatch an asynchronous event 16021da177e4SLinus Torvalds * @event:Event to dispatch 16031da177e4SLinus Torvalds * 16041da177e4SLinus Torvalds * Low-level drivers must call ib_dispatch_event() to dispatch the 16051da177e4SLinus Torvalds * event to all registered event handlers when an asynchronous event 16061da177e4SLinus Torvalds * occurs. 16071da177e4SLinus Torvalds */ 16081da177e4SLinus Torvalds void ib_dispatch_event(struct ib_event *event) 16091da177e4SLinus Torvalds { 16101da177e4SLinus Torvalds unsigned long flags; 16111da177e4SLinus Torvalds struct ib_event_handler *handler; 16121da177e4SLinus Torvalds 16131da177e4SLinus Torvalds spin_lock_irqsave(&event->device->event_handler_lock, flags); 16141da177e4SLinus Torvalds 16151da177e4SLinus Torvalds list_for_each_entry(handler, &event->device->event_handler_list, list) 16161da177e4SLinus Torvalds handler->handler(handler, event); 16171da177e4SLinus Torvalds 16181da177e4SLinus Torvalds spin_unlock_irqrestore(&event->device->event_handler_lock, flags); 16191da177e4SLinus Torvalds } 16201da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dispatch_event); 16211da177e4SLinus Torvalds 16221da177e4SLinus Torvalds /** 16231da177e4SLinus Torvalds * ib_query_port - Query IB port attributes 16241da177e4SLinus Torvalds * @device:Device to query 16251da177e4SLinus Torvalds * @port_num:Port number to query 16261da177e4SLinus Torvalds * @port_attr:Port attributes 16271da177e4SLinus Torvalds * 16281da177e4SLinus Torvalds * ib_query_port() returns the attributes of a port through the 16291da177e4SLinus Torvalds * @port_attr pointer. 16301da177e4SLinus Torvalds */ 16311da177e4SLinus Torvalds int ib_query_port(struct ib_device *device, 16321da177e4SLinus Torvalds u8 port_num, 16331da177e4SLinus Torvalds struct ib_port_attr *port_attr) 16341da177e4SLinus Torvalds { 1635fad61ad4SEli Cohen union ib_gid gid; 1636fad61ad4SEli Cohen int err; 1637fad61ad4SEli Cohen 163824dc831bSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 1639116c0074SRoland Dreier return -EINVAL; 1640116c0074SRoland Dreier 1641fad61ad4SEli Cohen memset(port_attr, 0, sizeof(*port_attr)); 16423023a1e9SKamal Heib err = device->ops.query_port(device, port_num, port_attr); 1643fad61ad4SEli Cohen if (err || port_attr->subnet_prefix) 1644fad61ad4SEli Cohen return err; 1645fad61ad4SEli Cohen 1646d7012467SEli Cohen if (rdma_port_get_link_layer(device, port_num) != IB_LINK_LAYER_INFINIBAND) 1647d7012467SEli Cohen return 0; 1648d7012467SEli Cohen 16493023a1e9SKamal Heib err = device->ops.query_gid(device, port_num, 0, &gid); 1650fad61ad4SEli Cohen if (err) 1651fad61ad4SEli Cohen return err; 1652fad61ad4SEli Cohen 1653fad61ad4SEli Cohen port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix); 1654fad61ad4SEli Cohen return 0; 16551da177e4SLinus Torvalds } 16561da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_port); 16571da177e4SLinus Torvalds 1658324e227eSJason Gunthorpe static void add_ndev_hash(struct ib_port_data *pdata) 1659324e227eSJason Gunthorpe { 1660324e227eSJason Gunthorpe unsigned long flags; 1661324e227eSJason Gunthorpe 1662324e227eSJason Gunthorpe might_sleep(); 1663324e227eSJason Gunthorpe 1664324e227eSJason Gunthorpe spin_lock_irqsave(&ndev_hash_lock, flags); 1665324e227eSJason Gunthorpe if (hash_hashed(&pdata->ndev_hash_link)) { 1666324e227eSJason Gunthorpe hash_del_rcu(&pdata->ndev_hash_link); 1667324e227eSJason Gunthorpe spin_unlock_irqrestore(&ndev_hash_lock, flags); 1668324e227eSJason Gunthorpe /* 1669324e227eSJason Gunthorpe * We cannot do hash_add_rcu after a hash_del_rcu until the 1670324e227eSJason Gunthorpe * grace period 1671324e227eSJason Gunthorpe */ 1672324e227eSJason Gunthorpe synchronize_rcu(); 1673324e227eSJason Gunthorpe spin_lock_irqsave(&ndev_hash_lock, flags); 1674324e227eSJason Gunthorpe } 1675324e227eSJason Gunthorpe if (pdata->netdev) 1676324e227eSJason Gunthorpe hash_add_rcu(ndev_hash, &pdata->ndev_hash_link, 1677324e227eSJason Gunthorpe (uintptr_t)pdata->netdev); 1678324e227eSJason Gunthorpe spin_unlock_irqrestore(&ndev_hash_lock, flags); 1679324e227eSJason Gunthorpe } 1680324e227eSJason Gunthorpe 16811da177e4SLinus Torvalds /** 1682c2261dd7SJason Gunthorpe * ib_device_set_netdev - Associate the ib_dev with an underlying net_device 1683c2261dd7SJason Gunthorpe * @ib_dev: Device to modify 1684c2261dd7SJason Gunthorpe * @ndev: net_device to affiliate, may be NULL 1685c2261dd7SJason Gunthorpe * @port: IB port the net_device is connected to 1686c2261dd7SJason Gunthorpe * 1687c2261dd7SJason Gunthorpe * Drivers should use this to link the ib_device to a netdev so the netdev 1688c2261dd7SJason Gunthorpe * shows up in interfaces like ib_enum_roce_netdev. Only one netdev may be 1689c2261dd7SJason Gunthorpe * affiliated with any port. 1690c2261dd7SJason Gunthorpe * 1691c2261dd7SJason Gunthorpe * The caller must ensure that the given ndev is not unregistered or 1692c2261dd7SJason Gunthorpe * unregistering, and that either the ib_device is unregistered or 1693c2261dd7SJason Gunthorpe * ib_device_set_netdev() is called with NULL when the ndev sends a 1694c2261dd7SJason Gunthorpe * NETDEV_UNREGISTER event. 1695c2261dd7SJason Gunthorpe */ 1696c2261dd7SJason Gunthorpe int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev, 1697c2261dd7SJason Gunthorpe unsigned int port) 1698c2261dd7SJason Gunthorpe { 1699c2261dd7SJason Gunthorpe struct net_device *old_ndev; 1700c2261dd7SJason Gunthorpe struct ib_port_data *pdata; 1701c2261dd7SJason Gunthorpe unsigned long flags; 1702c2261dd7SJason Gunthorpe int ret; 1703c2261dd7SJason Gunthorpe 1704c2261dd7SJason Gunthorpe /* 1705c2261dd7SJason Gunthorpe * Drivers wish to call this before ib_register_driver, so we have to 1706c2261dd7SJason Gunthorpe * setup the port data early. 1707c2261dd7SJason Gunthorpe */ 1708c2261dd7SJason Gunthorpe ret = alloc_port_data(ib_dev); 1709c2261dd7SJason Gunthorpe if (ret) 1710c2261dd7SJason Gunthorpe return ret; 1711c2261dd7SJason Gunthorpe 1712c2261dd7SJason Gunthorpe if (!rdma_is_port_valid(ib_dev, port)) 1713c2261dd7SJason Gunthorpe return -EINVAL; 1714c2261dd7SJason Gunthorpe 1715c2261dd7SJason Gunthorpe pdata = &ib_dev->port_data[port]; 1716c2261dd7SJason Gunthorpe spin_lock_irqsave(&pdata->netdev_lock, flags); 1717324e227eSJason Gunthorpe old_ndev = rcu_dereference_protected( 1718324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1719324e227eSJason Gunthorpe if (old_ndev == ndev) { 1720c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1721c2261dd7SJason Gunthorpe return 0; 1722c2261dd7SJason Gunthorpe } 1723c2261dd7SJason Gunthorpe 1724c2261dd7SJason Gunthorpe if (ndev) 1725c2261dd7SJason Gunthorpe dev_hold(ndev); 1726324e227eSJason Gunthorpe rcu_assign_pointer(pdata->netdev, ndev); 1727c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1728c2261dd7SJason Gunthorpe 1729324e227eSJason Gunthorpe add_ndev_hash(pdata); 1730c2261dd7SJason Gunthorpe if (old_ndev) 1731c2261dd7SJason Gunthorpe dev_put(old_ndev); 1732c2261dd7SJason Gunthorpe 1733c2261dd7SJason Gunthorpe return 0; 1734c2261dd7SJason Gunthorpe } 1735c2261dd7SJason Gunthorpe EXPORT_SYMBOL(ib_device_set_netdev); 1736c2261dd7SJason Gunthorpe 1737c2261dd7SJason Gunthorpe static void free_netdevs(struct ib_device *ib_dev) 1738c2261dd7SJason Gunthorpe { 1739c2261dd7SJason Gunthorpe unsigned long flags; 1740c2261dd7SJason Gunthorpe unsigned int port; 1741c2261dd7SJason Gunthorpe 1742c2261dd7SJason Gunthorpe rdma_for_each_port (ib_dev, port) { 1743c2261dd7SJason Gunthorpe struct ib_port_data *pdata = &ib_dev->port_data[port]; 1744324e227eSJason Gunthorpe struct net_device *ndev; 1745c2261dd7SJason Gunthorpe 1746c2261dd7SJason Gunthorpe spin_lock_irqsave(&pdata->netdev_lock, flags); 1747324e227eSJason Gunthorpe ndev = rcu_dereference_protected( 1748324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1749324e227eSJason Gunthorpe if (ndev) { 1750324e227eSJason Gunthorpe spin_lock(&ndev_hash_lock); 1751324e227eSJason Gunthorpe hash_del_rcu(&pdata->ndev_hash_link); 1752324e227eSJason Gunthorpe spin_unlock(&ndev_hash_lock); 1753324e227eSJason Gunthorpe 1754324e227eSJason Gunthorpe /* 1755324e227eSJason Gunthorpe * If this is the last dev_put there is still a 1756324e227eSJason Gunthorpe * synchronize_rcu before the netdev is kfreed, so we 1757324e227eSJason Gunthorpe * can continue to rely on unlocked pointer 1758324e227eSJason Gunthorpe * comparisons after the put 1759324e227eSJason Gunthorpe */ 1760324e227eSJason Gunthorpe rcu_assign_pointer(pdata->netdev, NULL); 1761324e227eSJason Gunthorpe dev_put(ndev); 1762c2261dd7SJason Gunthorpe } 1763c2261dd7SJason Gunthorpe spin_unlock_irqrestore(&pdata->netdev_lock, flags); 1764c2261dd7SJason Gunthorpe } 1765c2261dd7SJason Gunthorpe } 1766c2261dd7SJason Gunthorpe 1767c2261dd7SJason Gunthorpe struct net_device *ib_device_get_netdev(struct ib_device *ib_dev, 1768c2261dd7SJason Gunthorpe unsigned int port) 1769c2261dd7SJason Gunthorpe { 1770c2261dd7SJason Gunthorpe struct ib_port_data *pdata; 1771c2261dd7SJason Gunthorpe struct net_device *res; 1772c2261dd7SJason Gunthorpe 1773c2261dd7SJason Gunthorpe if (!rdma_is_port_valid(ib_dev, port)) 1774c2261dd7SJason Gunthorpe return NULL; 1775c2261dd7SJason Gunthorpe 1776c2261dd7SJason Gunthorpe pdata = &ib_dev->port_data[port]; 1777c2261dd7SJason Gunthorpe 1778c2261dd7SJason Gunthorpe /* 1779c2261dd7SJason Gunthorpe * New drivers should use ib_device_set_netdev() not the legacy 1780c2261dd7SJason Gunthorpe * get_netdev(). 1781c2261dd7SJason Gunthorpe */ 1782c2261dd7SJason Gunthorpe if (ib_dev->ops.get_netdev) 1783c2261dd7SJason Gunthorpe res = ib_dev->ops.get_netdev(ib_dev, port); 1784c2261dd7SJason Gunthorpe else { 1785c2261dd7SJason Gunthorpe spin_lock(&pdata->netdev_lock); 1786324e227eSJason Gunthorpe res = rcu_dereference_protected( 1787324e227eSJason Gunthorpe pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); 1788c2261dd7SJason Gunthorpe if (res) 1789c2261dd7SJason Gunthorpe dev_hold(res); 1790c2261dd7SJason Gunthorpe spin_unlock(&pdata->netdev_lock); 1791c2261dd7SJason Gunthorpe } 1792c2261dd7SJason Gunthorpe 1793c2261dd7SJason Gunthorpe /* 1794c2261dd7SJason Gunthorpe * If we are starting to unregister expedite things by preventing 1795c2261dd7SJason Gunthorpe * propagation of an unregistering netdev. 1796c2261dd7SJason Gunthorpe */ 1797c2261dd7SJason Gunthorpe if (res && res->reg_state != NETREG_REGISTERED) { 1798c2261dd7SJason Gunthorpe dev_put(res); 1799c2261dd7SJason Gunthorpe return NULL; 1800c2261dd7SJason Gunthorpe } 1801c2261dd7SJason Gunthorpe 1802c2261dd7SJason Gunthorpe return res; 1803c2261dd7SJason Gunthorpe } 1804c2261dd7SJason Gunthorpe 1805c2261dd7SJason Gunthorpe /** 1806324e227eSJason Gunthorpe * ib_device_get_by_netdev - Find an IB device associated with a netdev 1807324e227eSJason Gunthorpe * @ndev: netdev to locate 1808324e227eSJason Gunthorpe * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all) 1809324e227eSJason Gunthorpe * 1810324e227eSJason Gunthorpe * Find and hold an ib_device that is associated with a netdev via 1811324e227eSJason Gunthorpe * ib_device_set_netdev(). The caller must call ib_device_put() on the 1812324e227eSJason Gunthorpe * returned pointer. 1813324e227eSJason Gunthorpe */ 1814324e227eSJason Gunthorpe struct ib_device *ib_device_get_by_netdev(struct net_device *ndev, 1815324e227eSJason Gunthorpe enum rdma_driver_id driver_id) 1816324e227eSJason Gunthorpe { 1817324e227eSJason Gunthorpe struct ib_device *res = NULL; 1818324e227eSJason Gunthorpe struct ib_port_data *cur; 1819324e227eSJason Gunthorpe 1820324e227eSJason Gunthorpe rcu_read_lock(); 1821324e227eSJason Gunthorpe hash_for_each_possible_rcu (ndev_hash, cur, ndev_hash_link, 1822324e227eSJason Gunthorpe (uintptr_t)ndev) { 1823324e227eSJason Gunthorpe if (rcu_access_pointer(cur->netdev) == ndev && 1824324e227eSJason Gunthorpe (driver_id == RDMA_DRIVER_UNKNOWN || 1825324e227eSJason Gunthorpe cur->ib_dev->driver_id == driver_id) && 1826324e227eSJason Gunthorpe ib_device_try_get(cur->ib_dev)) { 1827324e227eSJason Gunthorpe res = cur->ib_dev; 1828324e227eSJason Gunthorpe break; 1829324e227eSJason Gunthorpe } 1830324e227eSJason Gunthorpe } 1831324e227eSJason Gunthorpe rcu_read_unlock(); 1832324e227eSJason Gunthorpe 1833324e227eSJason Gunthorpe return res; 1834324e227eSJason Gunthorpe } 1835324e227eSJason Gunthorpe EXPORT_SYMBOL(ib_device_get_by_netdev); 1836324e227eSJason Gunthorpe 1837324e227eSJason Gunthorpe /** 183803db3a2dSMatan Barak * ib_enum_roce_netdev - enumerate all RoCE ports 183903db3a2dSMatan Barak * @ib_dev : IB device we want to query 184003db3a2dSMatan Barak * @filter: Should we call the callback? 184103db3a2dSMatan Barak * @filter_cookie: Cookie passed to filter 184203db3a2dSMatan Barak * @cb: Callback to call for each found RoCE ports 184303db3a2dSMatan Barak * @cookie: Cookie passed back to the callback 184403db3a2dSMatan Barak * 184503db3a2dSMatan Barak * Enumerates all of the physical RoCE ports of ib_dev 184603db3a2dSMatan Barak * which are related to netdevice and calls callback() on each 184703db3a2dSMatan Barak * device for which filter() function returns non zero. 184803db3a2dSMatan Barak */ 184903db3a2dSMatan Barak void ib_enum_roce_netdev(struct ib_device *ib_dev, 185003db3a2dSMatan Barak roce_netdev_filter filter, 185103db3a2dSMatan Barak void *filter_cookie, 185203db3a2dSMatan Barak roce_netdev_callback cb, 185303db3a2dSMatan Barak void *cookie) 185403db3a2dSMatan Barak { 1855ea1075edSJason Gunthorpe unsigned int port; 185603db3a2dSMatan Barak 1857ea1075edSJason Gunthorpe rdma_for_each_port (ib_dev, port) 185803db3a2dSMatan Barak if (rdma_protocol_roce(ib_dev, port)) { 1859c2261dd7SJason Gunthorpe struct net_device *idev = 1860c2261dd7SJason Gunthorpe ib_device_get_netdev(ib_dev, port); 186103db3a2dSMatan Barak 186203db3a2dSMatan Barak if (filter(ib_dev, port, idev, filter_cookie)) 186303db3a2dSMatan Barak cb(ib_dev, port, idev, cookie); 186403db3a2dSMatan Barak 186503db3a2dSMatan Barak if (idev) 186603db3a2dSMatan Barak dev_put(idev); 186703db3a2dSMatan Barak } 186803db3a2dSMatan Barak } 186903db3a2dSMatan Barak 187003db3a2dSMatan Barak /** 187103db3a2dSMatan Barak * ib_enum_all_roce_netdevs - enumerate all RoCE devices 187203db3a2dSMatan Barak * @filter: Should we call the callback? 187303db3a2dSMatan Barak * @filter_cookie: Cookie passed to filter 187403db3a2dSMatan Barak * @cb: Callback to call for each found RoCE ports 187503db3a2dSMatan Barak * @cookie: Cookie passed back to the callback 187603db3a2dSMatan Barak * 187703db3a2dSMatan Barak * Enumerates all RoCE devices' physical ports which are related 187803db3a2dSMatan Barak * to netdevices and calls callback() on each device for which 187903db3a2dSMatan Barak * filter() function returns non zero. 188003db3a2dSMatan Barak */ 188103db3a2dSMatan Barak void ib_enum_all_roce_netdevs(roce_netdev_filter filter, 188203db3a2dSMatan Barak void *filter_cookie, 188303db3a2dSMatan Barak roce_netdev_callback cb, 188403db3a2dSMatan Barak void *cookie) 188503db3a2dSMatan Barak { 188603db3a2dSMatan Barak struct ib_device *dev; 18870df91bb6SJason Gunthorpe unsigned long index; 188803db3a2dSMatan Barak 1889921eab11SJason Gunthorpe down_read(&devices_rwsem); 18900df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) 189103db3a2dSMatan Barak ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); 1892921eab11SJason Gunthorpe up_read(&devices_rwsem); 189303db3a2dSMatan Barak } 189403db3a2dSMatan Barak 189503db3a2dSMatan Barak /** 18968030c835SLeon Romanovsky * ib_enum_all_devs - enumerate all ib_devices 18978030c835SLeon Romanovsky * @cb: Callback to call for each found ib_device 18988030c835SLeon Romanovsky * 18998030c835SLeon Romanovsky * Enumerates all ib_devices and calls callback() on each device. 19008030c835SLeon Romanovsky */ 19018030c835SLeon Romanovsky int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb, 19028030c835SLeon Romanovsky struct netlink_callback *cb) 19038030c835SLeon Romanovsky { 19040df91bb6SJason Gunthorpe unsigned long index; 19058030c835SLeon Romanovsky struct ib_device *dev; 19068030c835SLeon Romanovsky unsigned int idx = 0; 19078030c835SLeon Romanovsky int ret = 0; 19088030c835SLeon Romanovsky 1909921eab11SJason Gunthorpe down_read(&devices_rwsem); 19100df91bb6SJason Gunthorpe xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) { 191137eeab55SParav Pandit if (!rdma_dev_access_netns(dev, sock_net(skb->sk))) 191237eeab55SParav Pandit continue; 191337eeab55SParav Pandit 19148030c835SLeon Romanovsky ret = nldev_cb(dev, skb, cb, idx); 19158030c835SLeon Romanovsky if (ret) 19168030c835SLeon Romanovsky break; 19178030c835SLeon Romanovsky idx++; 19188030c835SLeon Romanovsky } 1919921eab11SJason Gunthorpe up_read(&devices_rwsem); 19208030c835SLeon Romanovsky return ret; 19218030c835SLeon Romanovsky } 19228030c835SLeon Romanovsky 19238030c835SLeon Romanovsky /** 19241da177e4SLinus Torvalds * ib_query_pkey - Get P_Key table entry 19251da177e4SLinus Torvalds * @device:Device to query 19261da177e4SLinus Torvalds * @port_num:Port number to query 19271da177e4SLinus Torvalds * @index:P_Key table index to query 19281da177e4SLinus Torvalds * @pkey:Returned P_Key 19291da177e4SLinus Torvalds * 19301da177e4SLinus Torvalds * ib_query_pkey() fetches the specified P_Key table entry. 19311da177e4SLinus Torvalds */ 19321da177e4SLinus Torvalds int ib_query_pkey(struct ib_device *device, 19331da177e4SLinus Torvalds u8 port_num, u16 index, u16 *pkey) 19341da177e4SLinus Torvalds { 19359af3f5cfSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 19369af3f5cfSYuval Shaia return -EINVAL; 19379af3f5cfSYuval Shaia 19383023a1e9SKamal Heib return device->ops.query_pkey(device, port_num, index, pkey); 19391da177e4SLinus Torvalds } 19401da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_pkey); 19411da177e4SLinus Torvalds 19421da177e4SLinus Torvalds /** 19431da177e4SLinus Torvalds * ib_modify_device - Change IB device attributes 19441da177e4SLinus Torvalds * @device:Device to modify 19451da177e4SLinus Torvalds * @device_modify_mask:Mask of attributes to change 19461da177e4SLinus Torvalds * @device_modify:New attribute values 19471da177e4SLinus Torvalds * 19481da177e4SLinus Torvalds * ib_modify_device() changes a device's attributes as specified by 19491da177e4SLinus Torvalds * the @device_modify_mask and @device_modify structure. 19501da177e4SLinus Torvalds */ 19511da177e4SLinus Torvalds int ib_modify_device(struct ib_device *device, 19521da177e4SLinus Torvalds int device_modify_mask, 19531da177e4SLinus Torvalds struct ib_device_modify *device_modify) 19541da177e4SLinus Torvalds { 19553023a1e9SKamal Heib if (!device->ops.modify_device) 195610e1b54bSBart Van Assche return -ENOSYS; 195710e1b54bSBart Van Assche 19583023a1e9SKamal Heib return device->ops.modify_device(device, device_modify_mask, 19591da177e4SLinus Torvalds device_modify); 19601da177e4SLinus Torvalds } 19611da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_device); 19621da177e4SLinus Torvalds 19631da177e4SLinus Torvalds /** 19641da177e4SLinus Torvalds * ib_modify_port - Modifies the attributes for the specified port. 19651da177e4SLinus Torvalds * @device: The device to modify. 19661da177e4SLinus Torvalds * @port_num: The number of the port to modify. 19671da177e4SLinus Torvalds * @port_modify_mask: Mask used to specify which attributes of the port 19681da177e4SLinus Torvalds * to change. 19691da177e4SLinus Torvalds * @port_modify: New attribute values for the port. 19701da177e4SLinus Torvalds * 19711da177e4SLinus Torvalds * ib_modify_port() changes a port's attributes as specified by the 19721da177e4SLinus Torvalds * @port_modify_mask and @port_modify structure. 19731da177e4SLinus Torvalds */ 19741da177e4SLinus Torvalds int ib_modify_port(struct ib_device *device, 19751da177e4SLinus Torvalds u8 port_num, int port_modify_mask, 19761da177e4SLinus Torvalds struct ib_port_modify *port_modify) 19771da177e4SLinus Torvalds { 197861e0962dSSelvin Xavier int rc; 197910e1b54bSBart Van Assche 198024dc831bSYuval Shaia if (!rdma_is_port_valid(device, port_num)) 1981116c0074SRoland Dreier return -EINVAL; 1982116c0074SRoland Dreier 19833023a1e9SKamal Heib if (device->ops.modify_port) 19843023a1e9SKamal Heib rc = device->ops.modify_port(device, port_num, 19853023a1e9SKamal Heib port_modify_mask, 19861da177e4SLinus Torvalds port_modify); 198761e0962dSSelvin Xavier else 198861e0962dSSelvin Xavier rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS; 198961e0962dSSelvin Xavier return rc; 19901da177e4SLinus Torvalds } 19911da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_port); 19921da177e4SLinus Torvalds 19935eb620c8SYosef Etigin /** 19945eb620c8SYosef Etigin * ib_find_gid - Returns the port number and GID table index where 1995dbb12562SParav Pandit * a specified GID value occurs. Its searches only for IB link layer. 19965eb620c8SYosef Etigin * @device: The device to query. 19975eb620c8SYosef Etigin * @gid: The GID value to search for. 19985eb620c8SYosef Etigin * @port_num: The port number of the device where the GID value was found. 19995eb620c8SYosef Etigin * @index: The index into the GID table where the GID was found. This 20005eb620c8SYosef Etigin * parameter may be NULL. 20015eb620c8SYosef Etigin */ 20025eb620c8SYosef Etigin int ib_find_gid(struct ib_device *device, union ib_gid *gid, 2003b26c4a11SParav Pandit u8 *port_num, u16 *index) 20045eb620c8SYosef Etigin { 20055eb620c8SYosef Etigin union ib_gid tmp_gid; 2006ea1075edSJason Gunthorpe unsigned int port; 2007ea1075edSJason Gunthorpe int ret, i; 20085eb620c8SYosef Etigin 2009ea1075edSJason Gunthorpe rdma_for_each_port (device, port) { 201022d24f75SParav Pandit if (!rdma_protocol_ib(device, port)) 2011b39ffa1dSMatan Barak continue; 2012b39ffa1dSMatan Barak 20138ceb1357SJason Gunthorpe for (i = 0; i < device->port_data[port].immutable.gid_tbl_len; 20148ceb1357SJason Gunthorpe ++i) { 20151dfce294SParav Pandit ret = rdma_query_gid(device, port, i, &tmp_gid); 20165eb620c8SYosef Etigin if (ret) 20175eb620c8SYosef Etigin return ret; 20185eb620c8SYosef Etigin if (!memcmp(&tmp_gid, gid, sizeof *gid)) { 20195eb620c8SYosef Etigin *port_num = port; 20205eb620c8SYosef Etigin if (index) 20215eb620c8SYosef Etigin *index = i; 20225eb620c8SYosef Etigin return 0; 20235eb620c8SYosef Etigin } 20245eb620c8SYosef Etigin } 20255eb620c8SYosef Etigin } 20265eb620c8SYosef Etigin 20275eb620c8SYosef Etigin return -ENOENT; 20285eb620c8SYosef Etigin } 20295eb620c8SYosef Etigin EXPORT_SYMBOL(ib_find_gid); 20305eb620c8SYosef Etigin 20315eb620c8SYosef Etigin /** 20325eb620c8SYosef Etigin * ib_find_pkey - Returns the PKey table index where a specified 20335eb620c8SYosef Etigin * PKey value occurs. 20345eb620c8SYosef Etigin * @device: The device to query. 20355eb620c8SYosef Etigin * @port_num: The port number of the device to search for the PKey. 20365eb620c8SYosef Etigin * @pkey: The PKey value to search for. 20375eb620c8SYosef Etigin * @index: The index into the PKey table where the PKey was found. 20385eb620c8SYosef Etigin */ 20395eb620c8SYosef Etigin int ib_find_pkey(struct ib_device *device, 20405eb620c8SYosef Etigin u8 port_num, u16 pkey, u16 *index) 20415eb620c8SYosef Etigin { 20425eb620c8SYosef Etigin int ret, i; 20435eb620c8SYosef Etigin u16 tmp_pkey; 2044ff7166c4SJack Morgenstein int partial_ix = -1; 20455eb620c8SYosef Etigin 20468ceb1357SJason Gunthorpe for (i = 0; i < device->port_data[port_num].immutable.pkey_tbl_len; 20478ceb1357SJason Gunthorpe ++i) { 20485eb620c8SYosef Etigin ret = ib_query_pkey(device, port_num, i, &tmp_pkey); 20495eb620c8SYosef Etigin if (ret) 20505eb620c8SYosef Etigin return ret; 205136026eccSMoni Shoua if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) { 2052ff7166c4SJack Morgenstein /* if there is full-member pkey take it.*/ 2053ff7166c4SJack Morgenstein if (tmp_pkey & 0x8000) { 20545eb620c8SYosef Etigin *index = i; 20555eb620c8SYosef Etigin return 0; 20565eb620c8SYosef Etigin } 2057ff7166c4SJack Morgenstein if (partial_ix < 0) 2058ff7166c4SJack Morgenstein partial_ix = i; 2059ff7166c4SJack Morgenstein } 20605eb620c8SYosef Etigin } 20615eb620c8SYosef Etigin 2062ff7166c4SJack Morgenstein /*no full-member, if exists take the limited*/ 2063ff7166c4SJack Morgenstein if (partial_ix >= 0) { 2064ff7166c4SJack Morgenstein *index = partial_ix; 2065ff7166c4SJack Morgenstein return 0; 2066ff7166c4SJack Morgenstein } 20675eb620c8SYosef Etigin return -ENOENT; 20685eb620c8SYosef Etigin } 20695eb620c8SYosef Etigin EXPORT_SYMBOL(ib_find_pkey); 20705eb620c8SYosef Etigin 20719268f72dSYotam Kenneth /** 20729268f72dSYotam Kenneth * ib_get_net_dev_by_params() - Return the appropriate net_dev 20739268f72dSYotam Kenneth * for a received CM request 20749268f72dSYotam Kenneth * @dev: An RDMA device on which the request has been received. 20759268f72dSYotam Kenneth * @port: Port number on the RDMA device. 20769268f72dSYotam Kenneth * @pkey: The Pkey the request came on. 20779268f72dSYotam Kenneth * @gid: A GID that the net_dev uses to communicate. 20789268f72dSYotam Kenneth * @addr: Contains the IP address that the request specified as its 20799268f72dSYotam Kenneth * destination. 2080921eab11SJason Gunthorpe * 20819268f72dSYotam Kenneth */ 20829268f72dSYotam Kenneth struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, 20839268f72dSYotam Kenneth u8 port, 20849268f72dSYotam Kenneth u16 pkey, 20859268f72dSYotam Kenneth const union ib_gid *gid, 20869268f72dSYotam Kenneth const struct sockaddr *addr) 20879268f72dSYotam Kenneth { 20889268f72dSYotam Kenneth struct net_device *net_dev = NULL; 20890df91bb6SJason Gunthorpe unsigned long index; 20900df91bb6SJason Gunthorpe void *client_data; 20919268f72dSYotam Kenneth 20929268f72dSYotam Kenneth if (!rdma_protocol_ib(dev, port)) 20939268f72dSYotam Kenneth return NULL; 20949268f72dSYotam Kenneth 2095921eab11SJason Gunthorpe /* 2096921eab11SJason Gunthorpe * Holding the read side guarantees that the client will not become 2097921eab11SJason Gunthorpe * unregistered while we are calling get_net_dev_by_params() 2098921eab11SJason Gunthorpe */ 2099921eab11SJason Gunthorpe down_read(&dev->client_data_rwsem); 21000df91bb6SJason Gunthorpe xan_for_each_marked (&dev->client_data, index, client_data, 21010df91bb6SJason Gunthorpe CLIENT_DATA_REGISTERED) { 21020df91bb6SJason Gunthorpe struct ib_client *client = xa_load(&clients, index); 21039268f72dSYotam Kenneth 21040df91bb6SJason Gunthorpe if (!client || !client->get_net_dev_by_params) 21059268f72dSYotam Kenneth continue; 21069268f72dSYotam Kenneth 21070df91bb6SJason Gunthorpe net_dev = client->get_net_dev_by_params(dev, port, pkey, gid, 21080df91bb6SJason Gunthorpe addr, client_data); 21099268f72dSYotam Kenneth if (net_dev) 21109268f72dSYotam Kenneth break; 21119268f72dSYotam Kenneth } 2112921eab11SJason Gunthorpe up_read(&dev->client_data_rwsem); 21139268f72dSYotam Kenneth 21149268f72dSYotam Kenneth return net_dev; 21159268f72dSYotam Kenneth } 21169268f72dSYotam Kenneth EXPORT_SYMBOL(ib_get_net_dev_by_params); 21179268f72dSYotam Kenneth 2118521ed0d9SKamal Heib void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) 2119521ed0d9SKamal Heib { 21203023a1e9SKamal Heib struct ib_device_ops *dev_ops = &dev->ops; 2121521ed0d9SKamal Heib #define SET_DEVICE_OP(ptr, name) \ 2122521ed0d9SKamal Heib do { \ 2123521ed0d9SKamal Heib if (ops->name) \ 2124521ed0d9SKamal Heib if (!((ptr)->name)) \ 2125521ed0d9SKamal Heib (ptr)->name = ops->name; \ 2126521ed0d9SKamal Heib } while (0) 2127521ed0d9SKamal Heib 212830471d4bSLeon Romanovsky #define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name) 212930471d4bSLeon Romanovsky 21303023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, add_gid); 21312f1927b0SMoni Shoua SET_DEVICE_OP(dev_ops, advise_mr); 21323023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_dm); 21333023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_fmr); 21343023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_hw_stats); 21353023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_mr); 21363023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_mw); 21373023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_pd); 21383023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_rdma_netdev); 21393023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_ucontext); 21403023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, alloc_xrcd); 21413023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, attach_mcast); 21423023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, check_mr_status); 21433023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_ah); 21443023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_counters); 21453023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_cq); 21463023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_flow); 21473023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_flow_action_esp); 21483023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_qp); 21493023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_rwq_ind_table); 21503023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_srq); 21513023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, create_wq); 21523023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_dm); 2153d0899892SJason Gunthorpe SET_DEVICE_OP(dev_ops, dealloc_driver); 21543023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_fmr); 21553023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_mw); 21563023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_pd); 21573023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_ucontext); 21583023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dealloc_xrcd); 21593023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, del_gid); 21603023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, dereg_mr); 21613023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_ah); 21623023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_counters); 21633023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_cq); 21643023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_flow); 21653023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_flow_action); 21663023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_qp); 21673023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table); 21683023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_srq); 21693023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, destroy_wq); 21703023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, detach_mcast); 21713023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, disassociate_ucontext); 21723023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, drain_rq); 21733023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, drain_sq); 2174ca22354bSJason Gunthorpe SET_DEVICE_OP(dev_ops, enable_driver); 217502da3750SLeon Romanovsky SET_DEVICE_OP(dev_ops, fill_res_entry); 21763023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_dev_fw_str); 21773023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_dma_mr); 21783023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_hw_stats); 21793023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_link_layer); 21803023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_netdev); 21813023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_port_immutable); 21823023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vector_affinity); 21833023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vf_config); 21843023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, get_vf_stats); 2185ea4baf7fSParav Pandit SET_DEVICE_OP(dev_ops, init_port); 21863023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, map_mr_sg); 21873023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, map_phys_fmr); 21883023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, mmap); 21893023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_ah); 21903023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_cq); 21913023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_device); 21923023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_flow_action_esp); 21933023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_port); 21943023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_qp); 21953023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_srq); 21963023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, modify_wq); 21973023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, peek_cq); 21983023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, poll_cq); 21993023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_recv); 22003023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_send); 22013023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, post_srq_recv); 22023023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, process_mad); 22033023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_ah); 22043023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_device); 22053023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_gid); 22063023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_pkey); 22073023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_port); 22083023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_qp); 22093023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, query_srq); 22103023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); 22113023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, read_counters); 22123023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, reg_dm_mr); 22133023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, reg_user_mr); 22143023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, req_ncomp_notif); 22153023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, req_notify_cq); 22163023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, rereg_user_mr); 22173023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, resize_cq); 22183023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, set_vf_guid); 22193023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, set_vf_link_state); 22203023a1e9SKamal Heib SET_DEVICE_OP(dev_ops, unmap_fmr); 222121a428a0SLeon Romanovsky 222221a428a0SLeon Romanovsky SET_OBJ_SIZE(dev_ops, ib_pd); 2223a2a074efSLeon Romanovsky SET_OBJ_SIZE(dev_ops, ib_ucontext); 2224521ed0d9SKamal Heib } 2225521ed0d9SKamal Heib EXPORT_SYMBOL(ib_set_device_ops); 2226521ed0d9SKamal Heib 2227d0e312feSLeon Romanovsky static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = { 2228735c631aSMark Bloch [RDMA_NL_LS_OP_RESOLVE] = { 2229647c75acSLeon Romanovsky .doit = ib_nl_handle_resolve_resp, 2230e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 2231e3a2b93dSLeon Romanovsky }, 2232735c631aSMark Bloch [RDMA_NL_LS_OP_SET_TIMEOUT] = { 2233647c75acSLeon Romanovsky .doit = ib_nl_handle_set_timeout, 2234e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 2235e3a2b93dSLeon Romanovsky }, 2236ae43f828SMark Bloch [RDMA_NL_LS_OP_IP_RESOLVE] = { 2237647c75acSLeon Romanovsky .doit = ib_nl_handle_ip_res_resp, 2238e3a2b93dSLeon Romanovsky .flags = RDMA_NL_ADMIN_PERM, 2239e3a2b93dSLeon Romanovsky }, 2240735c631aSMark Bloch }; 2241735c631aSMark Bloch 22421da177e4SLinus Torvalds static int __init ib_core_init(void) 22431da177e4SLinus Torvalds { 22441da177e4SLinus Torvalds int ret; 22451da177e4SLinus Torvalds 2246f0626710STejun Heo ib_wq = alloc_workqueue("infiniband", 0, 0); 2247f0626710STejun Heo if (!ib_wq) 2248f0626710STejun Heo return -ENOMEM; 2249f0626710STejun Heo 225014d3a3b2SChristoph Hellwig ib_comp_wq = alloc_workqueue("ib-comp-wq", 2251b7363e67SSagi Grimberg WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 225214d3a3b2SChristoph Hellwig if (!ib_comp_wq) { 225314d3a3b2SChristoph Hellwig ret = -ENOMEM; 225414d3a3b2SChristoph Hellwig goto err; 225514d3a3b2SChristoph Hellwig } 225614d3a3b2SChristoph Hellwig 2257f794809aSJack Morgenstein ib_comp_unbound_wq = 2258f794809aSJack Morgenstein alloc_workqueue("ib-comp-unb-wq", 2259f794809aSJack Morgenstein WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM | 2260f794809aSJack Morgenstein WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE); 2261f794809aSJack Morgenstein if (!ib_comp_unbound_wq) { 2262f794809aSJack Morgenstein ret = -ENOMEM; 2263f794809aSJack Morgenstein goto err_comp; 2264f794809aSJack Morgenstein } 2265f794809aSJack Morgenstein 226655aeed06SJason Gunthorpe ret = class_register(&ib_class); 2267fd75c789SNir Muchtar if (ret) { 2268aba25a3eSParav Pandit pr_warn("Couldn't create InfiniBand device class\n"); 2269f794809aSJack Morgenstein goto err_comp_unbound; 2270fd75c789SNir Muchtar } 22711da177e4SLinus Torvalds 2272c9901724SLeon Romanovsky ret = rdma_nl_init(); 22731da177e4SLinus Torvalds if (ret) { 2274c9901724SLeon Romanovsky pr_warn("Couldn't init IB netlink interface: err %d\n", ret); 2275fd75c789SNir Muchtar goto err_sysfs; 22761da177e4SLinus Torvalds } 22771da177e4SLinus Torvalds 2278e3f20f02SLeon Romanovsky ret = addr_init(); 2279e3f20f02SLeon Romanovsky if (ret) { 2280e3f20f02SLeon Romanovsky pr_warn("Could't init IB address resolution\n"); 2281e3f20f02SLeon Romanovsky goto err_ibnl; 2282e3f20f02SLeon Romanovsky } 2283e3f20f02SLeon Romanovsky 22844c2cb422SMark Bloch ret = ib_mad_init(); 22854c2cb422SMark Bloch if (ret) { 22864c2cb422SMark Bloch pr_warn("Couldn't init IB MAD\n"); 22874c2cb422SMark Bloch goto err_addr; 22884c2cb422SMark Bloch } 22894c2cb422SMark Bloch 2290c2e49c92SMark Bloch ret = ib_sa_init(); 2291c2e49c92SMark Bloch if (ret) { 2292c2e49c92SMark Bloch pr_warn("Couldn't init SA\n"); 2293c2e49c92SMark Bloch goto err_mad; 2294c2e49c92SMark Bloch } 2295c2e49c92SMark Bloch 22968f408ab6SDaniel Jurgens ret = register_lsm_notifier(&ibdev_lsm_nb); 22978f408ab6SDaniel Jurgens if (ret) { 22988f408ab6SDaniel Jurgens pr_warn("Couldn't register LSM notifier. ret %d\n", ret); 2299c9901724SLeon Romanovsky goto err_sa; 23008f408ab6SDaniel Jurgens } 23018f408ab6SDaniel Jurgens 23024e0f7b90SParav Pandit ret = register_pernet_device(&rdma_dev_net_ops); 23034e0f7b90SParav Pandit if (ret) { 23044e0f7b90SParav Pandit pr_warn("Couldn't init compat dev. ret %d\n", ret); 23054e0f7b90SParav Pandit goto err_compat; 23064e0f7b90SParav Pandit } 23074e0f7b90SParav Pandit 23086c80b41aSLeon Romanovsky nldev_init(); 2309c9901724SLeon Romanovsky rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table); 23105ef8c0c1SJason Gunthorpe roce_gid_mgmt_init(); 2311b2cbae2cSRoland Dreier 2312fd75c789SNir Muchtar return 0; 2313fd75c789SNir Muchtar 23144e0f7b90SParav Pandit err_compat: 23154e0f7b90SParav Pandit unregister_lsm_notifier(&ibdev_lsm_nb); 2316735c631aSMark Bloch err_sa: 2317735c631aSMark Bloch ib_sa_cleanup(); 2318c2e49c92SMark Bloch err_mad: 2319c2e49c92SMark Bloch ib_mad_cleanup(); 23204c2cb422SMark Bloch err_addr: 23214c2cb422SMark Bloch addr_cleanup(); 2322e3f20f02SLeon Romanovsky err_ibnl: 2323c9901724SLeon Romanovsky rdma_nl_exit(); 2324fd75c789SNir Muchtar err_sysfs: 232555aeed06SJason Gunthorpe class_unregister(&ib_class); 2326f794809aSJack Morgenstein err_comp_unbound: 2327f794809aSJack Morgenstein destroy_workqueue(ib_comp_unbound_wq); 232814d3a3b2SChristoph Hellwig err_comp: 232914d3a3b2SChristoph Hellwig destroy_workqueue(ib_comp_wq); 2330fd75c789SNir Muchtar err: 2331fd75c789SNir Muchtar destroy_workqueue(ib_wq); 23321da177e4SLinus Torvalds return ret; 23331da177e4SLinus Torvalds } 23341da177e4SLinus Torvalds 23351da177e4SLinus Torvalds static void __exit ib_core_cleanup(void) 23361da177e4SLinus Torvalds { 23375ef8c0c1SJason Gunthorpe roce_gid_mgmt_cleanup(); 23386c80b41aSLeon Romanovsky nldev_exit(); 2339c9901724SLeon Romanovsky rdma_nl_unregister(RDMA_NL_LS); 23404e0f7b90SParav Pandit unregister_pernet_device(&rdma_dev_net_ops); 2341c9901724SLeon Romanovsky unregister_lsm_notifier(&ibdev_lsm_nb); 2342c2e49c92SMark Bloch ib_sa_cleanup(); 23434c2cb422SMark Bloch ib_mad_cleanup(); 2344e3f20f02SLeon Romanovsky addr_cleanup(); 2345c9901724SLeon Romanovsky rdma_nl_exit(); 234655aeed06SJason Gunthorpe class_unregister(&ib_class); 2347f794809aSJack Morgenstein destroy_workqueue(ib_comp_unbound_wq); 234814d3a3b2SChristoph Hellwig destroy_workqueue(ib_comp_wq); 2349f7c6a7b5SRoland Dreier /* Make sure that any pending umem accounting work is done. */ 2350f0626710STejun Heo destroy_workqueue(ib_wq); 2351d0899892SJason Gunthorpe flush_workqueue(system_unbound_wq); 2352e59178d8SJason Gunthorpe WARN_ON(!xa_empty(&clients)); 23530df91bb6SJason Gunthorpe WARN_ON(!xa_empty(&devices)); 23541da177e4SLinus Torvalds } 23551da177e4SLinus Torvalds 2356e3bf14bdSJason Gunthorpe MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4); 2357e3bf14bdSJason Gunthorpe 235862dfa795SParav Pandit /* ib core relies on netdev stack to first register net_ns_type_operations 235962dfa795SParav Pandit * ns kobject type before ib_core initialization. 236062dfa795SParav Pandit */ 236162dfa795SParav Pandit fs_initcall(ib_core_init); 23621da177e4SLinus Torvalds module_exit(ib_core_cleanup); 2363