138321256SMatan Barak /*
238321256SMatan Barak * Copyright (c) 2016, Mellanox Technologies inc. All rights reserved.
338321256SMatan Barak *
438321256SMatan Barak * This software is available to you under a choice of one of two
538321256SMatan Barak * licenses. You may choose to be licensed under the terms of the GNU
638321256SMatan Barak * General Public License (GPL) Version 2, available from the file
738321256SMatan Barak * COPYING in the main directory of this source tree, or the
838321256SMatan Barak * OpenIB.org BSD license below:
938321256SMatan Barak *
1038321256SMatan Barak * Redistribution and use in source and binary forms, with or
1138321256SMatan Barak * without modification, are permitted provided that the following
1238321256SMatan Barak * conditions are met:
1338321256SMatan Barak *
1438321256SMatan Barak * - Redistributions of source code must retain the above
1538321256SMatan Barak * copyright notice, this list of conditions and the following
1638321256SMatan Barak * disclaimer.
1738321256SMatan Barak *
1838321256SMatan Barak * - Redistributions in binary form must reproduce the above
1938321256SMatan Barak * copyright notice, this list of conditions and the following
2038321256SMatan Barak * disclaimer in the documentation and/or other materials
2138321256SMatan Barak * provided with the distribution.
2238321256SMatan Barak *
2338321256SMatan Barak * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
2438321256SMatan Barak * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
2538321256SMatan Barak * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
2638321256SMatan Barak * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
2738321256SMatan Barak * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
2838321256SMatan Barak * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
2938321256SMatan Barak * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
3038321256SMatan Barak * SOFTWARE.
3138321256SMatan Barak */
3238321256SMatan Barak
3338321256SMatan Barak #include <linux/file.h>
3438321256SMatan Barak #include <linux/anon_inodes.h>
35e951747aSJason Gunthorpe #include <linux/sched/mm.h>
3638321256SMatan Barak #include <rdma/ib_verbs.h>
3738321256SMatan Barak #include <rdma/uverbs_types.h>
3838321256SMatan Barak #include <linux/rcupdate.h>
39a0aa309cSMatan Barak #include <rdma/uverbs_ioctl.h>
40fac9658cSMatan Barak #include <rdma/rdma_user_ioctl.h>
4138321256SMatan Barak #include "uverbs.h"
4238321256SMatan Barak #include "core_priv.h"
4338321256SMatan Barak #include "rdma_core.h"
4438321256SMatan Barak
uverbs_uobject_free(struct kref * ref)4530004b86SMatan Barak static void uverbs_uobject_free(struct kref *ref)
4638321256SMatan Barak {
478bdf9dd9SJason Gunthorpe kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu);
4838321256SMatan Barak }
4938321256SMatan Barak
506898d1c6SJason Gunthorpe /*
516898d1c6SJason Gunthorpe * In order to indicate we no longer needs this uobject, uverbs_uobject_put
526898d1c6SJason Gunthorpe * is called. When the reference count is decreased, the uobject is freed.
536898d1c6SJason Gunthorpe * For example, this is used when attaching a completion channel to a CQ.
546898d1c6SJason Gunthorpe */
uverbs_uobject_put(struct ib_uobject * uobject)5538321256SMatan Barak void uverbs_uobject_put(struct ib_uobject *uobject)
5638321256SMatan Barak {
5730004b86SMatan Barak kref_put(&uobject->ref, uverbs_uobject_free);
5838321256SMatan Barak }
596898d1c6SJason Gunthorpe EXPORT_SYMBOL(uverbs_uobject_put);
6038321256SMatan Barak
uverbs_try_lock_object(struct ib_uobject * uobj,enum rdma_lookup_mode mode)619867f5c6SJason Gunthorpe static int uverbs_try_lock_object(struct ib_uobject *uobj,
629867f5c6SJason Gunthorpe enum rdma_lookup_mode mode)
6338321256SMatan Barak {
6438321256SMatan Barak /*
6530004b86SMatan Barak * When a shared access is required, we use a positive counter. Each
6630004b86SMatan Barak * shared access request checks that the value != -1 and increment it.
6730004b86SMatan Barak * Exclusive access is required for operations like write or destroy.
6830004b86SMatan Barak * In exclusive access mode, we check that the counter is zero (nobody
6930004b86SMatan Barak * claimed this object) and we set it to -1. Releasing a shared access
7030004b86SMatan Barak * lock is done simply by decreasing the counter. As for exclusive
71*fd46ef3dSJiang Jian * access locks, since only a single one of them is allowed
7230004b86SMatan Barak * concurrently, setting the counter to zero is enough for releasing
7330004b86SMatan Barak * this lock.
7438321256SMatan Barak */
759867f5c6SJason Gunthorpe switch (mode) {
769867f5c6SJason Gunthorpe case UVERBS_LOOKUP_READ:
77bfc18e38SMark Rutland return atomic_fetch_add_unless(&uobj->usecnt, 1, -1) == -1 ?
7838321256SMatan Barak -EBUSY : 0;
799867f5c6SJason Gunthorpe case UVERBS_LOOKUP_WRITE:
807452a3c7SJason Gunthorpe /* lock is exclusive */
8138321256SMatan Barak return atomic_cmpxchg(&uobj->usecnt, 0, -1) == 0 ? 0 : -EBUSY;
827452a3c7SJason Gunthorpe case UVERBS_LOOKUP_DESTROY:
837452a3c7SJason Gunthorpe return 0;
8438321256SMatan Barak }
859867f5c6SJason Gunthorpe return 0;
869867f5c6SJason Gunthorpe }
8738321256SMatan Barak
assert_uverbs_usecnt(struct ib_uobject * uobj,enum rdma_lookup_mode mode)889867f5c6SJason Gunthorpe static void assert_uverbs_usecnt(struct ib_uobject *uobj,
899867f5c6SJason Gunthorpe enum rdma_lookup_mode mode)
9087ad80abSJason Gunthorpe {
9187ad80abSJason Gunthorpe #ifdef CONFIG_LOCKDEP
929867f5c6SJason Gunthorpe switch (mode) {
939867f5c6SJason Gunthorpe case UVERBS_LOOKUP_READ:
9487ad80abSJason Gunthorpe WARN_ON(atomic_read(&uobj->usecnt) <= 0);
959867f5c6SJason Gunthorpe break;
969867f5c6SJason Gunthorpe case UVERBS_LOOKUP_WRITE:
979867f5c6SJason Gunthorpe WARN_ON(atomic_read(&uobj->usecnt) != -1);
989867f5c6SJason Gunthorpe break;
997452a3c7SJason Gunthorpe case UVERBS_LOOKUP_DESTROY:
1007452a3c7SJason Gunthorpe break;
1019867f5c6SJason Gunthorpe }
10287ad80abSJason Gunthorpe #endif
10387ad80abSJason Gunthorpe }
10487ad80abSJason Gunthorpe
10587ad80abSJason Gunthorpe /*
1061e857e65SJason Gunthorpe * This must be called with the hw_destroy_rwsem locked for read or write,
1071e857e65SJason Gunthorpe * also the uobject itself must be locked for write.
10887ad80abSJason Gunthorpe *
10987ad80abSJason Gunthorpe * Upon return the HW object is guaranteed to be destroyed.
11087ad80abSJason Gunthorpe *
11187ad80abSJason Gunthorpe * For RDMA_REMOVE_ABORT, the hw_destroy_rwsem is not required to be held,
11287ad80abSJason Gunthorpe * however the type's allocat_commit function cannot have been called and the
11387ad80abSJason Gunthorpe * uobject cannot be on the uobjects_lists
11487ad80abSJason Gunthorpe *
115016b26afSYangyang Li * For RDMA_REMOVE_DESTROY the caller should be holding a kref (eg via
11687ad80abSJason Gunthorpe * rdma_lookup_get_uobject) and the object is left in a state where the caller
11787ad80abSJason Gunthorpe * needs to call rdma_lookup_put_uobject.
11887ad80abSJason Gunthorpe *
11987ad80abSJason Gunthorpe * For all other destroy modes this function internally unlocks the uobject
12087ad80abSJason Gunthorpe * and consumes the kref on the uobj.
12187ad80abSJason Gunthorpe */
uverbs_destroy_uobject(struct ib_uobject * uobj,enum rdma_remove_reason reason,struct uverbs_attr_bundle * attrs)12287ad80abSJason Gunthorpe static int uverbs_destroy_uobject(struct ib_uobject *uobj,
123a6a3797dSShamir Rabinovitch enum rdma_remove_reason reason,
124a6a3797dSShamir Rabinovitch struct uverbs_attr_bundle *attrs)
12587ad80abSJason Gunthorpe {
126feec576aSJason Gunthorpe struct ib_uverbs_file *ufile = attrs->ufile;
12787ad80abSJason Gunthorpe unsigned long flags;
12887ad80abSJason Gunthorpe int ret;
12987ad80abSJason Gunthorpe
1301e857e65SJason Gunthorpe lockdep_assert_held(&ufile->hw_destroy_rwsem);
1319867f5c6SJason Gunthorpe assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE);
13287ad80abSJason Gunthorpe
133849e1490SJason Gunthorpe if (reason == RDMA_REMOVE_ABORT) {
134849e1490SJason Gunthorpe WARN_ON(!list_empty(&uobj->list));
135849e1490SJason Gunthorpe WARN_ON(!uobj->context);
136849e1490SJason Gunthorpe uobj->uapi_object->type_class->alloc_abort(uobj);
137849e1490SJason Gunthorpe } else if (uobj->object) {
138a6a3797dSShamir Rabinovitch ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason,
139a6a3797dSShamir Rabinovitch attrs);
140efa968eeSLeon Romanovsky if (ret)
141efa968eeSLeon Romanovsky /* Nothing to be done, wait till ucontext will clean it */
14287ad80abSJason Gunthorpe return ret;
14387ad80abSJason Gunthorpe
14487ad80abSJason Gunthorpe uobj->object = NULL;
14587ad80abSJason Gunthorpe }
14687ad80abSJason Gunthorpe
14787ad80abSJason Gunthorpe uobj->context = NULL;
14887ad80abSJason Gunthorpe
14987ad80abSJason Gunthorpe /*
150c85f4abeSJason Gunthorpe * For DESTROY the usecnt is not changed, the caller is expected to
151c85f4abeSJason Gunthorpe * manage it via uobj_put_destroy(). Only DESTROY can remove the IDR
152c85f4abeSJason Gunthorpe * handle.
15387ad80abSJason Gunthorpe */
15487ad80abSJason Gunthorpe if (reason != RDMA_REMOVE_DESTROY)
15587ad80abSJason Gunthorpe atomic_set(&uobj->usecnt, 0);
1560f50d88aSJason Gunthorpe else
1576b0d08f4SJason Gunthorpe uobj->uapi_object->type_class->remove_handle(uobj);
15887ad80abSJason Gunthorpe
15987ad80abSJason Gunthorpe if (!list_empty(&uobj->list)) {
16087ad80abSJason Gunthorpe spin_lock_irqsave(&ufile->uobjects_lock, flags);
16187ad80abSJason Gunthorpe list_del_init(&uobj->list);
16287ad80abSJason Gunthorpe spin_unlock_irqrestore(&ufile->uobjects_lock, flags);
16387ad80abSJason Gunthorpe
16487ad80abSJason Gunthorpe /*
16587ad80abSJason Gunthorpe * Pairs with the get in rdma_alloc_commit_uobject(), could
16687ad80abSJason Gunthorpe * destroy uobj.
16787ad80abSJason Gunthorpe */
16887ad80abSJason Gunthorpe uverbs_uobject_put(uobj);
16987ad80abSJason Gunthorpe }
17087ad80abSJason Gunthorpe
17187ad80abSJason Gunthorpe /*
17287ad80abSJason Gunthorpe * When aborting the stack kref remains owned by the core code, and is
17387ad80abSJason Gunthorpe * not transferred into the type. Pairs with the get in alloc_uobj
17487ad80abSJason Gunthorpe */
17587ad80abSJason Gunthorpe if (reason == RDMA_REMOVE_ABORT)
17687ad80abSJason Gunthorpe uverbs_uobject_put(uobj);
17787ad80abSJason Gunthorpe
17887ad80abSJason Gunthorpe return 0;
17987ad80abSJason Gunthorpe }
18087ad80abSJason Gunthorpe
181c33e73afSJason Gunthorpe /*
1827452a3c7SJason Gunthorpe * This calls uverbs_destroy_uobject() using the RDMA_REMOVE_DESTROY
1837452a3c7SJason Gunthorpe * sequence. It should only be used from command callbacks. On success the
184c85f4abeSJason Gunthorpe * caller must pair this with uobj_put_destroy(). This
1857452a3c7SJason Gunthorpe * version requires the caller to have already obtained an
1867452a3c7SJason Gunthorpe * LOOKUP_DESTROY uobject kref.
1877452a3c7SJason Gunthorpe */
uobj_destroy(struct ib_uobject * uobj,struct uverbs_attr_bundle * attrs)188a6a3797dSShamir Rabinovitch int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs)
1897452a3c7SJason Gunthorpe {
190feec576aSJason Gunthorpe struct ib_uverbs_file *ufile = attrs->ufile;
1917452a3c7SJason Gunthorpe int ret;
1927452a3c7SJason Gunthorpe
1937452a3c7SJason Gunthorpe down_read(&ufile->hw_destroy_rwsem);
1947452a3c7SJason Gunthorpe
195c85f4abeSJason Gunthorpe /*
196c85f4abeSJason Gunthorpe * Once the uobject is destroyed by RDMA_REMOVE_DESTROY then it is left
197c85f4abeSJason Gunthorpe * write locked as the callers put it back with UVERBS_LOOKUP_DESTROY.
198c85f4abeSJason Gunthorpe * This is because any other concurrent thread can still see the object
199c85f4abeSJason Gunthorpe * in the xarray due to RCU. Leaving it locked ensures nothing else will
200c85f4abeSJason Gunthorpe * touch it.
201c85f4abeSJason Gunthorpe */
2027452a3c7SJason Gunthorpe ret = uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE);
2037452a3c7SJason Gunthorpe if (ret)
2047452a3c7SJason Gunthorpe goto out_unlock;
2057452a3c7SJason Gunthorpe
206a6a3797dSShamir Rabinovitch ret = uverbs_destroy_uobject(uobj, RDMA_REMOVE_DESTROY, attrs);
2077452a3c7SJason Gunthorpe if (ret) {
2087452a3c7SJason Gunthorpe atomic_set(&uobj->usecnt, 0);
2097452a3c7SJason Gunthorpe goto out_unlock;
2107452a3c7SJason Gunthorpe }
2117452a3c7SJason Gunthorpe
2127452a3c7SJason Gunthorpe out_unlock:
2137452a3c7SJason Gunthorpe up_read(&ufile->hw_destroy_rwsem);
2147452a3c7SJason Gunthorpe return ret;
2157452a3c7SJason Gunthorpe }
2167452a3c7SJason Gunthorpe
2177452a3c7SJason Gunthorpe /*
21832ed5c00SJason Gunthorpe * uobj_get_destroy destroys the HW object and returns a handle to the uobj
21932ed5c00SJason Gunthorpe * with a NULL object pointer. The caller must pair this with
220c85f4abeSJason Gunthorpe * uobj_put_destroy().
221c33e73afSJason Gunthorpe */
__uobj_get_destroy(const struct uverbs_api_object * obj,u32 id,struct uverbs_attr_bundle * attrs)2226b0d08f4SJason Gunthorpe struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
22370f06b26SShamir Rabinovitch u32 id, struct uverbs_attr_bundle *attrs)
224c33e73afSJason Gunthorpe {
225c33e73afSJason Gunthorpe struct ib_uobject *uobj;
226c33e73afSJason Gunthorpe int ret;
227c33e73afSJason Gunthorpe
2288313c10fSJason Gunthorpe uobj = rdma_lookup_get_uobject(obj, attrs->ufile, id,
22970f06b26SShamir Rabinovitch UVERBS_LOOKUP_DESTROY, attrs);
230c33e73afSJason Gunthorpe if (IS_ERR(uobj))
23132ed5c00SJason Gunthorpe return uobj;
23232ed5c00SJason Gunthorpe
233a6a3797dSShamir Rabinovitch ret = uobj_destroy(uobj, attrs);
23432ed5c00SJason Gunthorpe if (ret) {
2357452a3c7SJason Gunthorpe rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY);
23632ed5c00SJason Gunthorpe return ERR_PTR(ret);
23732ed5c00SJason Gunthorpe }
23832ed5c00SJason Gunthorpe
23932ed5c00SJason Gunthorpe return uobj;
24032ed5c00SJason Gunthorpe }
24132ed5c00SJason Gunthorpe
24232ed5c00SJason Gunthorpe /*
2437106a976SJason Gunthorpe * Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success
2447106a976SJason Gunthorpe * (negative errno on failure). For use by callers that do not need the uobj.
24532ed5c00SJason Gunthorpe */
__uobj_perform_destroy(const struct uverbs_api_object * obj,u32 id,struct uverbs_attr_bundle * attrs)2466b0d08f4SJason Gunthorpe int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,
24770f06b26SShamir Rabinovitch struct uverbs_attr_bundle *attrs)
24832ed5c00SJason Gunthorpe {
24932ed5c00SJason Gunthorpe struct ib_uobject *uobj;
25032ed5c00SJason Gunthorpe
2518313c10fSJason Gunthorpe uobj = __uobj_get_destroy(obj, id, attrs);
25232ed5c00SJason Gunthorpe if (IS_ERR(uobj))
253c33e73afSJason Gunthorpe return PTR_ERR(uobj);
254c85f4abeSJason Gunthorpe uobj_put_destroy(uobj);
2557106a976SJason Gunthorpe return 0;
256c33e73afSJason Gunthorpe }
257c33e73afSJason Gunthorpe
25887ad80abSJason Gunthorpe /* alloc_uobj must be undone by uverbs_destroy_uobject() */
alloc_uobj(struct uverbs_attr_bundle * attrs,const struct uverbs_api_object * obj)25939e83af8SJason Gunthorpe static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs,
2606b0d08f4SJason Gunthorpe const struct uverbs_api_object *obj)
26138321256SMatan Barak {
26239e83af8SJason Gunthorpe struct ib_uverbs_file *ufile = attrs->ufile;
26322fa27fbSJason Gunthorpe struct ib_uobject *uobj;
26438321256SMatan Barak
26539e83af8SJason Gunthorpe if (!attrs->context) {
26639e83af8SJason Gunthorpe struct ib_ucontext *ucontext =
26739e83af8SJason Gunthorpe ib_uverbs_get_ucontext_file(ufile);
26839e83af8SJason Gunthorpe
26922fa27fbSJason Gunthorpe if (IS_ERR(ucontext))
27022fa27fbSJason Gunthorpe return ERR_CAST(ucontext);
27139e83af8SJason Gunthorpe attrs->context = ucontext;
27239e83af8SJason Gunthorpe }
27322fa27fbSJason Gunthorpe
2746b0d08f4SJason Gunthorpe uobj = kzalloc(obj->type_attrs->obj_size, GFP_KERNEL);
27538321256SMatan Barak if (!uobj)
27638321256SMatan Barak return ERR_PTR(-ENOMEM);
27738321256SMatan Barak /*
27838321256SMatan Barak * user_handle should be filled by the handler,
27938321256SMatan Barak * The object is added to the list in the commit stage.
28038321256SMatan Barak */
2816ef1c828SJason Gunthorpe uobj->ufile = ufile;
28239e83af8SJason Gunthorpe uobj->context = attrs->context;
2835671f79bSJason Gunthorpe INIT_LIST_HEAD(&uobj->list);
2846b0d08f4SJason Gunthorpe uobj->uapi_object = obj;
285d9dc7a35SJason Gunthorpe /*
286d9dc7a35SJason Gunthorpe * Allocated objects start out as write locked to deny any other
287d9dc7a35SJason Gunthorpe * syscalls from accessing them until they are committed. See
288d9dc7a35SJason Gunthorpe * rdma_alloc_commit_uobject
289d9dc7a35SJason Gunthorpe */
290d9dc7a35SJason Gunthorpe atomic_set(&uobj->usecnt, -1);
29138321256SMatan Barak kref_init(&uobj->ref);
29238321256SMatan Barak
29338321256SMatan Barak return uobj;
29438321256SMatan Barak }
29538321256SMatan Barak
idr_add_uobj(struct ib_uobject * uobj)29638321256SMatan Barak static int idr_add_uobj(struct ib_uobject *uobj)
29738321256SMatan Barak {
29838321256SMatan Barak /*
29938321256SMatan Barak * We start with allocating an idr pointing to NULL. This represents an
30038321256SMatan Barak * object which isn't initialized yet. We'll replace it later on with
30138321256SMatan Barak * the real object once we commit.
30238321256SMatan Barak */
303b9b0f345SMatthew Wilcox return xa_alloc(&uobj->ufile->idr, &uobj->id, NULL, xa_limit_32b,
304b9b0f345SMatthew Wilcox GFP_KERNEL);
30538321256SMatan Barak }
30638321256SMatan Barak
30738321256SMatan Barak /* Returns the ib_uobject or an error. The caller should check for IS_ERR. */
3086ef1c828SJason Gunthorpe static struct ib_uobject *
lookup_get_idr_uobject(const struct uverbs_api_object * obj,struct ib_uverbs_file * ufile,s64 id,enum rdma_lookup_mode mode)3096b0d08f4SJason Gunthorpe lookup_get_idr_uobject(const struct uverbs_api_object *obj,
3109867f5c6SJason Gunthorpe struct ib_uverbs_file *ufile, s64 id,
3119867f5c6SJason Gunthorpe enum rdma_lookup_mode mode)
31238321256SMatan Barak {
31338321256SMatan Barak struct ib_uobject *uobj;
3141250c304SJason Gunthorpe
3151250c304SJason Gunthorpe if (id < 0 || id > ULONG_MAX)
3161250c304SJason Gunthorpe return ERR_PTR(-EINVAL);
31738321256SMatan Barak
31838321256SMatan Barak rcu_read_lock();
3196623e3e3SLeon Romanovsky /*
3206623e3e3SLeon Romanovsky * The idr_find is guaranteed to return a pointer to something that
3216623e3e3SLeon Romanovsky * isn't freed yet, or NULL, as the free after idr_remove goes through
3226623e3e3SLeon Romanovsky * kfree_rcu(). However the object may still have been released and
3236623e3e3SLeon Romanovsky * kfree() could be called at any time.
3246623e3e3SLeon Romanovsky */
325b9b0f345SMatthew Wilcox uobj = xa_load(&ufile->idr, id);
326b9b0f345SMatthew Wilcox if (!uobj || !kref_get_unless_zero(&uobj->ref))
3276623e3e3SLeon Romanovsky uobj = ERR_PTR(-ENOENT);
32838321256SMatan Barak rcu_read_unlock();
32938321256SMatan Barak return uobj;
33038321256SMatan Barak }
33138321256SMatan Barak
3329867f5c6SJason Gunthorpe static struct ib_uobject *
lookup_get_fd_uobject(const struct uverbs_api_object * obj,struct ib_uverbs_file * ufile,s64 id,enum rdma_lookup_mode mode)3336b0d08f4SJason Gunthorpe lookup_get_fd_uobject(const struct uverbs_api_object *obj,
3349867f5c6SJason Gunthorpe struct ib_uverbs_file *ufile, s64 id,
3359867f5c6SJason Gunthorpe enum rdma_lookup_mode mode)
336cf8966b3SMatan Barak {
3376b0d08f4SJason Gunthorpe const struct uverbs_obj_fd_type *fd_type;
338cf8966b3SMatan Barak struct file *f;
339cf8966b3SMatan Barak struct ib_uobject *uobject;
3401250c304SJason Gunthorpe int fdno = id;
341cf8966b3SMatan Barak
3421250c304SJason Gunthorpe if (fdno != id)
3431250c304SJason Gunthorpe return ERR_PTR(-EINVAL);
3441250c304SJason Gunthorpe
3459867f5c6SJason Gunthorpe if (mode != UVERBS_LOOKUP_READ)
346cf8966b3SMatan Barak return ERR_PTR(-EOPNOTSUPP);
347cf8966b3SMatan Barak
3486b0d08f4SJason Gunthorpe if (!obj->type_attrs)
3496b0d08f4SJason Gunthorpe return ERR_PTR(-EIO);
3506b0d08f4SJason Gunthorpe fd_type =
3516b0d08f4SJason Gunthorpe container_of(obj->type_attrs, struct uverbs_obj_fd_type, type);
3526b0d08f4SJason Gunthorpe
3531250c304SJason Gunthorpe f = fget(fdno);
354cf8966b3SMatan Barak if (!f)
355cf8966b3SMatan Barak return ERR_PTR(-EBADF);
356cf8966b3SMatan Barak
357cf8966b3SMatan Barak uobject = f->private_data;
358cf8966b3SMatan Barak /*
359f7c8416cSJason Gunthorpe * fget(id) ensures we are not currently running
360f7c8416cSJason Gunthorpe * uverbs_uobject_fd_release(), and the caller is expected to ensure
361f7c8416cSJason Gunthorpe * that release is never done while a call to lookup is possible.
362cf8966b3SMatan Barak */
3630fb00941SLeon Romanovsky if (f->f_op != fd_type->fops || uobject->ufile != ufile) {
364cf8966b3SMatan Barak fput(f);
365cf8966b3SMatan Barak return ERR_PTR(-EBADF);
366cf8966b3SMatan Barak }
367cf8966b3SMatan Barak
368cf8966b3SMatan Barak uverbs_uobject_get(uobject);
369cf8966b3SMatan Barak return uobject;
370cf8966b3SMatan Barak }
371cf8966b3SMatan Barak
rdma_lookup_get_uobject(const struct uverbs_api_object * obj,struct ib_uverbs_file * ufile,s64 id,enum rdma_lookup_mode mode,struct uverbs_attr_bundle * attrs)3726b0d08f4SJason Gunthorpe struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
3731250c304SJason Gunthorpe struct ib_uverbs_file *ufile, s64 id,
37470f06b26SShamir Rabinovitch enum rdma_lookup_mode mode,
37570f06b26SShamir Rabinovitch struct uverbs_attr_bundle *attrs)
37638321256SMatan Barak {
37738321256SMatan Barak struct ib_uobject *uobj;
37838321256SMatan Barak int ret;
37938321256SMatan Barak
380b9b0f345SMatthew Wilcox if (obj == ERR_PTR(-ENOMSG)) {
3814d7e8cc5SYishai Hadas /* must be UVERBS_IDR_ANY_OBJECT, see uapi_get_object() */
3824d7e8cc5SYishai Hadas uobj = lookup_get_idr_uobject(NULL, ufile, id, mode);
3834d7e8cc5SYishai Hadas if (IS_ERR(uobj))
3844d7e8cc5SYishai Hadas return uobj;
3854d7e8cc5SYishai Hadas } else {
3864d7e8cc5SYishai Hadas if (IS_ERR(obj))
3876b0d08f4SJason Gunthorpe return ERR_PTR(-EINVAL);
3886b0d08f4SJason Gunthorpe
3896b0d08f4SJason Gunthorpe uobj = obj->type_class->lookup_get(obj, ufile, id, mode);
39038321256SMatan Barak if (IS_ERR(uobj))
39138321256SMatan Barak return uobj;
39238321256SMatan Barak
3936b0d08f4SJason Gunthorpe if (uobj->uapi_object != obj) {
39438321256SMatan Barak ret = -EINVAL;
39538321256SMatan Barak goto free;
39638321256SMatan Barak }
3974d7e8cc5SYishai Hadas }
39838321256SMatan Barak
399cc2e14e6SJason Gunthorpe /*
400cc2e14e6SJason Gunthorpe * If we have been disassociated block every command except for
401cc2e14e6SJason Gunthorpe * DESTROY based commands.
402cc2e14e6SJason Gunthorpe */
403cc2e14e6SJason Gunthorpe if (mode != UVERBS_LOOKUP_DESTROY &&
404cc2e14e6SJason Gunthorpe !srcu_dereference(ufile->device->ib_dev,
405cc2e14e6SJason Gunthorpe &ufile->device->disassociate_srcu)) {
406cc2e14e6SJason Gunthorpe ret = -EIO;
407cc2e14e6SJason Gunthorpe goto free;
408cc2e14e6SJason Gunthorpe }
409cc2e14e6SJason Gunthorpe
4109867f5c6SJason Gunthorpe ret = uverbs_try_lock_object(uobj, mode);
411e951747aSJason Gunthorpe if (ret)
41238321256SMatan Barak goto free;
41370f06b26SShamir Rabinovitch if (attrs)
41470f06b26SShamir Rabinovitch attrs->context = uobj->context;
41538321256SMatan Barak
41638321256SMatan Barak return uobj;
41738321256SMatan Barak free:
4184d7e8cc5SYishai Hadas uobj->uapi_object->type_class->lookup_put(uobj, mode);
41938321256SMatan Barak uverbs_uobject_put(uobj);
42038321256SMatan Barak return ERR_PTR(ret);
42138321256SMatan Barak }
42238321256SMatan Barak
4236b0d08f4SJason Gunthorpe static struct ib_uobject *
alloc_begin_idr_uobject(const struct uverbs_api_object * obj,struct uverbs_attr_bundle * attrs)4246b0d08f4SJason Gunthorpe alloc_begin_idr_uobject(const struct uverbs_api_object *obj,
42539e83af8SJason Gunthorpe struct uverbs_attr_bundle *attrs)
42638321256SMatan Barak {
42738321256SMatan Barak int ret;
42838321256SMatan Barak struct ib_uobject *uobj;
42938321256SMatan Barak
43039e83af8SJason Gunthorpe uobj = alloc_uobj(attrs, obj);
43138321256SMatan Barak if (IS_ERR(uobj))
43238321256SMatan Barak return uobj;
43338321256SMatan Barak
43438321256SMatan Barak ret = idr_add_uobj(uobj);
43538321256SMatan Barak if (ret)
43638321256SMatan Barak goto uobj_put;
43738321256SMatan Barak
43822fa27fbSJason Gunthorpe ret = ib_rdmacg_try_charge(&uobj->cg_obj, uobj->context->device,
43938321256SMatan Barak RDMACG_RESOURCE_HCA_OBJECT);
44038321256SMatan Barak if (ret)
441b9b0f345SMatthew Wilcox goto remove;
44238321256SMatan Barak
44338321256SMatan Barak return uobj;
44438321256SMatan Barak
445b9b0f345SMatthew Wilcox remove:
44639e83af8SJason Gunthorpe xa_erase(&attrs->ufile->idr, uobj->id);
44738321256SMatan Barak uobj_put:
44838321256SMatan Barak uverbs_uobject_put(uobj);
44938321256SMatan Barak return ERR_PTR(ret);
45038321256SMatan Barak }
45138321256SMatan Barak
4526b0d08f4SJason Gunthorpe static struct ib_uobject *
alloc_begin_fd_uobject(const struct uverbs_api_object * obj,struct uverbs_attr_bundle * attrs)4536b0d08f4SJason Gunthorpe alloc_begin_fd_uobject(const struct uverbs_api_object *obj,
45439e83af8SJason Gunthorpe struct uverbs_attr_bundle *attrs)
455cf8966b3SMatan Barak {
4564121fb0dSLeon Romanovsky const struct uverbs_obj_fd_type *fd_type;
457cf8966b3SMatan Barak int new_fd;
4584121fb0dSLeon Romanovsky struct ib_uobject *uobj, *ret;
459849e1490SJason Gunthorpe struct file *filp;
460849e1490SJason Gunthorpe
46139e83af8SJason Gunthorpe uobj = alloc_uobj(attrs, obj);
462849e1490SJason Gunthorpe if (IS_ERR(uobj))
4634121fb0dSLeon Romanovsky return uobj;
4644121fb0dSLeon Romanovsky
4654121fb0dSLeon Romanovsky fd_type =
4664121fb0dSLeon Romanovsky container_of(obj->type_attrs, struct uverbs_obj_fd_type, type);
4674121fb0dSLeon Romanovsky if (WARN_ON(fd_type->fops->release != &uverbs_uobject_fd_release &&
4684121fb0dSLeon Romanovsky fd_type->fops->release != &uverbs_async_event_release)) {
4694121fb0dSLeon Romanovsky ret = ERR_PTR(-EINVAL);
470849e1490SJason Gunthorpe goto err_fd;
4714121fb0dSLeon Romanovsky }
4724121fb0dSLeon Romanovsky
4734121fb0dSLeon Romanovsky new_fd = get_unused_fd_flags(O_CLOEXEC);
4744121fb0dSLeon Romanovsky if (new_fd < 0) {
4754121fb0dSLeon Romanovsky ret = ERR_PTR(new_fd);
4764121fb0dSLeon Romanovsky goto err_fd;
4774121fb0dSLeon Romanovsky }
478849e1490SJason Gunthorpe
479849e1490SJason Gunthorpe /* Note that uverbs_uobject_fd_release() is called during abort */
480849e1490SJason Gunthorpe filp = anon_inode_getfile(fd_type->name, fd_type->fops, NULL,
481849e1490SJason Gunthorpe fd_type->flags);
482849e1490SJason Gunthorpe if (IS_ERR(filp)) {
4834121fb0dSLeon Romanovsky ret = ERR_CAST(filp);
4844121fb0dSLeon Romanovsky goto err_getfile;
485cf8966b3SMatan Barak }
486849e1490SJason Gunthorpe uobj->object = filp;
487cf8966b3SMatan Barak
488d0259e82SJason Gunthorpe uobj->id = new_fd;
489849e1490SJason Gunthorpe return uobj;
490cf8966b3SMatan Barak
4914121fb0dSLeon Romanovsky err_getfile:
492849e1490SJason Gunthorpe put_unused_fd(new_fd);
4934121fb0dSLeon Romanovsky err_fd:
4944121fb0dSLeon Romanovsky uverbs_uobject_put(uobj);
4954121fb0dSLeon Romanovsky return ret;
496cf8966b3SMatan Barak }
497cf8966b3SMatan Barak
rdma_alloc_begin_uobject(const struct uverbs_api_object * obj,struct uverbs_attr_bundle * attrs)4986b0d08f4SJason Gunthorpe struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
49970f06b26SShamir Rabinovitch struct uverbs_attr_bundle *attrs)
50038321256SMatan Barak {
50139e83af8SJason Gunthorpe struct ib_uverbs_file *ufile = attrs->ufile;
5021e857e65SJason Gunthorpe struct ib_uobject *ret;
5031e857e65SJason Gunthorpe
5044d7e8cc5SYishai Hadas if (IS_ERR(obj))
5056b0d08f4SJason Gunthorpe return ERR_PTR(-EINVAL);
5066b0d08f4SJason Gunthorpe
5071e857e65SJason Gunthorpe /*
5081e857e65SJason Gunthorpe * The hw_destroy_rwsem is held across the entire object creation and
5091e857e65SJason Gunthorpe * released during rdma_alloc_commit_uobject or
5101e857e65SJason Gunthorpe * rdma_alloc_abort_uobject
5111e857e65SJason Gunthorpe */
5121e857e65SJason Gunthorpe if (!down_read_trylock(&ufile->hw_destroy_rwsem))
5131e857e65SJason Gunthorpe return ERR_PTR(-EIO);
5141e857e65SJason Gunthorpe
51539e83af8SJason Gunthorpe ret = obj->type_class->alloc_begin(obj, attrs);
5161e857e65SJason Gunthorpe if (IS_ERR(ret)) {
5171e857e65SJason Gunthorpe up_read(&ufile->hw_destroy_rwsem);
5181e857e65SJason Gunthorpe return ret;
5191e857e65SJason Gunthorpe }
5201e857e65SJason Gunthorpe return ret;
52138321256SMatan Barak }
52238321256SMatan Barak
alloc_abort_idr_uobject(struct ib_uobject * uobj)52387ad80abSJason Gunthorpe static void alloc_abort_idr_uobject(struct ib_uobject *uobj)
52487ad80abSJason Gunthorpe {
52587ad80abSJason Gunthorpe ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device,
52687ad80abSJason Gunthorpe RDMACG_RESOURCE_HCA_OBJECT);
52787ad80abSJason Gunthorpe
528b9b0f345SMatthew Wilcox xa_erase(&uobj->ufile->idr, uobj->id);
52987ad80abSJason Gunthorpe }
53087ad80abSJason Gunthorpe
destroy_hw_idr_uobject(struct ib_uobject * uobj,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)5310f50d88aSJason Gunthorpe static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj,
532a6a3797dSShamir Rabinovitch enum rdma_remove_reason why,
533a6a3797dSShamir Rabinovitch struct uverbs_attr_bundle *attrs)
53438321256SMatan Barak {
53538321256SMatan Barak const struct uverbs_obj_idr_type *idr_type =
5366b0d08f4SJason Gunthorpe container_of(uobj->uapi_object->type_attrs,
5376b0d08f4SJason Gunthorpe struct uverbs_obj_idr_type, type);
538a6a3797dSShamir Rabinovitch int ret = idr_type->destroy_object(uobj, why, attrs);
53938321256SMatan Barak
540efa968eeSLeon Romanovsky if (ret)
54138321256SMatan Barak return ret;
54238321256SMatan Barak
54387ad80abSJason Gunthorpe if (why == RDMA_REMOVE_ABORT)
54487ad80abSJason Gunthorpe return 0;
5455671f79bSJason Gunthorpe
5460f50d88aSJason Gunthorpe ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device,
5470f50d88aSJason Gunthorpe RDMACG_RESOURCE_HCA_OBJECT);
54838321256SMatan Barak
54987ad80abSJason Gunthorpe return 0;
55038321256SMatan Barak }
55138321256SMatan Barak
remove_handle_idr_uobject(struct ib_uobject * uobj)5520f50d88aSJason Gunthorpe static void remove_handle_idr_uobject(struct ib_uobject *uobj)
5530f50d88aSJason Gunthorpe {
554b9b0f345SMatthew Wilcox xa_erase(&uobj->ufile->idr, uobj->id);
5550f50d88aSJason Gunthorpe /* Matches the kref in alloc_commit_idr_uobject */
5560f50d88aSJason Gunthorpe uverbs_uobject_put(uobj);
5570f50d88aSJason Gunthorpe }
5580f50d88aSJason Gunthorpe
alloc_abort_fd_uobject(struct ib_uobject * uobj)559cf8966b3SMatan Barak static void alloc_abort_fd_uobject(struct ib_uobject *uobj)
560cf8966b3SMatan Barak {
561849e1490SJason Gunthorpe struct file *filp = uobj->object;
562849e1490SJason Gunthorpe
563849e1490SJason Gunthorpe fput(filp);
564aba94548SJason Gunthorpe put_unused_fd(uobj->id);
565cf8966b3SMatan Barak }
566cf8966b3SMatan Barak
destroy_hw_fd_uobject(struct ib_uobject * uobj,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)5670f50d88aSJason Gunthorpe static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj,
568a6a3797dSShamir Rabinovitch enum rdma_remove_reason why,
569a6a3797dSShamir Rabinovitch struct uverbs_attr_bundle *attrs)
570cf8966b3SMatan Barak {
5716b0d08f4SJason Gunthorpe const struct uverbs_obj_fd_type *fd_type = container_of(
5726b0d08f4SJason Gunthorpe uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type);
573cf8966b3SMatan Barak
574c5633a72SLeon Romanovsky fd_type->destroy_object(uobj, why);
575c5633a72SLeon Romanovsky return 0;
57638321256SMatan Barak }
57738321256SMatan Barak
remove_handle_fd_uobject(struct ib_uobject * uobj)5780f50d88aSJason Gunthorpe static void remove_handle_fd_uobject(struct ib_uobject *uobj)
5790f50d88aSJason Gunthorpe {
5800f50d88aSJason Gunthorpe }
5810f50d88aSJason Gunthorpe
alloc_commit_idr_uobject(struct ib_uobject * uobj)582849e1490SJason Gunthorpe static void alloc_commit_idr_uobject(struct ib_uobject *uobj)
58338321256SMatan Barak {
584c561c288SJason Gunthorpe struct ib_uverbs_file *ufile = uobj->ufile;
585b9b0f345SMatthew Wilcox void *old;
586c561c288SJason Gunthorpe
58738321256SMatan Barak /*
58838321256SMatan Barak * We already allocated this IDR with a NULL object, so
58938321256SMatan Barak * this shouldn't fail.
590c561c288SJason Gunthorpe *
591b9b0f345SMatthew Wilcox * NOTE: Storing the uobj transfers our kref on uobj to the XArray.
592c561c288SJason Gunthorpe * It will be put by remove_commit_idr_uobject()
59338321256SMatan Barak */
594b9b0f345SMatthew Wilcox old = xa_store(&ufile->idr, uobj->id, uobj, GFP_KERNEL);
595b9b0f345SMatthew Wilcox WARN_ON(old != NULL);
59638321256SMatan Barak }
59738321256SMatan Barak
swap_idr_uobjects(struct ib_uobject * obj_old,struct ib_uobject * obj_new)5986e0954b1SJason Gunthorpe static void swap_idr_uobjects(struct ib_uobject *obj_old,
5996e0954b1SJason Gunthorpe struct ib_uobject *obj_new)
6006e0954b1SJason Gunthorpe {
6016e0954b1SJason Gunthorpe struct ib_uverbs_file *ufile = obj_old->ufile;
6026e0954b1SJason Gunthorpe void *old;
6036e0954b1SJason Gunthorpe
6046e0954b1SJason Gunthorpe /*
6056e0954b1SJason Gunthorpe * New must be an object that been allocated but not yet committed, this
6066e0954b1SJason Gunthorpe * moves the pre-committed state to obj_old, new still must be comitted.
6076e0954b1SJason Gunthorpe */
6086e0954b1SJason Gunthorpe old = xa_cmpxchg(&ufile->idr, obj_old->id, obj_old, XA_ZERO_ENTRY,
6096e0954b1SJason Gunthorpe GFP_KERNEL);
6106e0954b1SJason Gunthorpe if (WARN_ON(old != obj_old))
6116e0954b1SJason Gunthorpe return;
6126e0954b1SJason Gunthorpe
6136e0954b1SJason Gunthorpe swap(obj_old->id, obj_new->id);
6146e0954b1SJason Gunthorpe
6156e0954b1SJason Gunthorpe old = xa_cmpxchg(&ufile->idr, obj_old->id, NULL, obj_old, GFP_KERNEL);
6166e0954b1SJason Gunthorpe WARN_ON(old != NULL);
6176e0954b1SJason Gunthorpe }
6186e0954b1SJason Gunthorpe
alloc_commit_fd_uobject(struct ib_uobject * uobj)619849e1490SJason Gunthorpe static void alloc_commit_fd_uobject(struct ib_uobject *uobj)
620cf8966b3SMatan Barak {
621c561c288SJason Gunthorpe int fd = uobj->id;
622849e1490SJason Gunthorpe struct file *filp = uobj->object;
623aba94548SJason Gunthorpe
624f7c8416cSJason Gunthorpe /* Matching put will be done in uverbs_uobject_fd_release() */
625aba94548SJason Gunthorpe kref_get(&uobj->ufile->ref);
626c561c288SJason Gunthorpe
627cf8966b3SMatan Barak /* This shouldn't be used anymore. Use the file object instead */
628d0259e82SJason Gunthorpe uobj->id = 0;
629c561c288SJason Gunthorpe
630c561c288SJason Gunthorpe /*
631c561c288SJason Gunthorpe * NOTE: Once we install the file we loose ownership of our kref on
632f7c8416cSJason Gunthorpe * uobj. It will be put by uverbs_uobject_fd_release()
633c561c288SJason Gunthorpe */
634849e1490SJason Gunthorpe filp->private_data = uobj;
635aba94548SJason Gunthorpe fd_install(fd, filp);
636cf8966b3SMatan Barak }
637cf8966b3SMatan Barak
638c561c288SJason Gunthorpe /*
639c561c288SJason Gunthorpe * In all cases rdma_alloc_commit_uobject() consumes the kref to uobj and the
6402c96eb7dSJason Gunthorpe * caller can no longer assume uobj is valid. If this function fails it
6412c96eb7dSJason Gunthorpe * destroys the uboject, including the attached HW object.
642c561c288SJason Gunthorpe */
rdma_alloc_commit_uobject(struct ib_uobject * uobj,struct uverbs_attr_bundle * attrs)643849e1490SJason Gunthorpe void rdma_alloc_commit_uobject(struct ib_uobject *uobj,
644a6a3797dSShamir Rabinovitch struct uverbs_attr_bundle *attrs)
64538321256SMatan Barak {
646feec576aSJason Gunthorpe struct ib_uverbs_file *ufile = attrs->ufile;
6476a5e9c88SJason Gunthorpe
6485671f79bSJason Gunthorpe /* kref is held so long as the uobj is on the uobj list. */
6495671f79bSJason Gunthorpe uverbs_uobject_get(uobj);
65087064277SJason Gunthorpe spin_lock_irq(&ufile->uobjects_lock);
6516a5e9c88SJason Gunthorpe list_add(&uobj->list, &ufile->uobjects);
65287064277SJason Gunthorpe spin_unlock_irq(&ufile->uobjects_lock);
6538efe991eSJason Gunthorpe
654aba94548SJason Gunthorpe /* matches atomic_set(-1) in alloc_uobj */
655aba94548SJason Gunthorpe atomic_set(&uobj->usecnt, 0);
656aba94548SJason Gunthorpe
6570d1fd39bSLeon Romanovsky /* alloc_commit consumes the uobj kref */
6580d1fd39bSLeon Romanovsky uobj->uapi_object->type_class->alloc_commit(uobj);
6590d1fd39bSLeon Romanovsky
6601e857e65SJason Gunthorpe /* Matches the down_read in rdma_alloc_begin_uobject */
66187064277SJason Gunthorpe up_read(&ufile->hw_destroy_rwsem);
66238321256SMatan Barak }
66338321256SMatan Barak
6645671f79bSJason Gunthorpe /*
6656e0954b1SJason Gunthorpe * new_uobj will be assigned to the handle currently used by to_uobj, and
6666e0954b1SJason Gunthorpe * to_uobj will be destroyed.
6676e0954b1SJason Gunthorpe *
6686e0954b1SJason Gunthorpe * Upon return the caller must do:
6696e0954b1SJason Gunthorpe * rdma_alloc_commit_uobject(new_uobj)
6706e0954b1SJason Gunthorpe * uobj_put_destroy(to_uobj)
6716e0954b1SJason Gunthorpe *
6726e0954b1SJason Gunthorpe * to_uobj must have a write get but the put mode switches to destroy once
6736e0954b1SJason Gunthorpe * this is called.
6746e0954b1SJason Gunthorpe */
rdma_assign_uobject(struct ib_uobject * to_uobj,struct ib_uobject * new_uobj,struct uverbs_attr_bundle * attrs)6756e0954b1SJason Gunthorpe void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj,
6766e0954b1SJason Gunthorpe struct uverbs_attr_bundle *attrs)
6776e0954b1SJason Gunthorpe {
6786e0954b1SJason Gunthorpe assert_uverbs_usecnt(new_uobj, UVERBS_LOOKUP_WRITE);
6796e0954b1SJason Gunthorpe
6806e0954b1SJason Gunthorpe if (WARN_ON(to_uobj->uapi_object != new_uobj->uapi_object ||
6816e0954b1SJason Gunthorpe !to_uobj->uapi_object->type_class->swap_uobjects))
6826e0954b1SJason Gunthorpe return;
6836e0954b1SJason Gunthorpe
6846e0954b1SJason Gunthorpe to_uobj->uapi_object->type_class->swap_uobjects(to_uobj, new_uobj);
6856e0954b1SJason Gunthorpe
6866e0954b1SJason Gunthorpe /*
6876e0954b1SJason Gunthorpe * If this fails then the uobject is still completely valid (though with
6886e0954b1SJason Gunthorpe * a new ID) and we leak it until context close.
6896e0954b1SJason Gunthorpe */
6906e0954b1SJason Gunthorpe uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs);
6916e0954b1SJason Gunthorpe }
6926e0954b1SJason Gunthorpe
6936e0954b1SJason Gunthorpe /*
6945671f79bSJason Gunthorpe * This consumes the kref for uobj. It is up to the caller to unwind the HW
6955671f79bSJason Gunthorpe * object and anything else connected to uobj before calling this.
6965671f79bSJason Gunthorpe */
rdma_alloc_abort_uobject(struct ib_uobject * uobj,struct uverbs_attr_bundle * attrs,bool hw_obj_valid)697a6a3797dSShamir Rabinovitch void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
6980ac8903cSJason Gunthorpe struct uverbs_attr_bundle *attrs,
6990ac8903cSJason Gunthorpe bool hw_obj_valid)
70038321256SMatan Barak {
7011e857e65SJason Gunthorpe struct ib_uverbs_file *ufile = uobj->ufile;
702f553246fSJason Gunthorpe int ret;
7031e857e65SJason Gunthorpe
704f553246fSJason Gunthorpe if (hw_obj_valid) {
705f553246fSJason Gunthorpe ret = uobj->uapi_object->type_class->destroy_hw(
706f553246fSJason Gunthorpe uobj, RDMA_REMOVE_ABORT, attrs);
707f553246fSJason Gunthorpe /*
708f553246fSJason Gunthorpe * If the driver couldn't destroy the object then go ahead and
709f553246fSJason Gunthorpe * commit it. Leaking objects that can't be destroyed is only
710f553246fSJason Gunthorpe * done during FD close after the driver has a few more tries to
711f553246fSJason Gunthorpe * destroy it.
712f553246fSJason Gunthorpe */
713f553246fSJason Gunthorpe if (WARN_ON(ret))
714f553246fSJason Gunthorpe return rdma_alloc_commit_uobject(uobj, attrs);
715f553246fSJason Gunthorpe }
716f553246fSJason Gunthorpe
717f553246fSJason Gunthorpe uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);
7181e857e65SJason Gunthorpe
7191e857e65SJason Gunthorpe /* Matches the down_read in rdma_alloc_begin_uobject */
7201e857e65SJason Gunthorpe up_read(&ufile->hw_destroy_rwsem);
72138321256SMatan Barak }
72238321256SMatan Barak
lookup_put_idr_uobject(struct ib_uobject * uobj,enum rdma_lookup_mode mode)7239867f5c6SJason Gunthorpe static void lookup_put_idr_uobject(struct ib_uobject *uobj,
7249867f5c6SJason Gunthorpe enum rdma_lookup_mode mode)
72538321256SMatan Barak {
72638321256SMatan Barak }
72738321256SMatan Barak
lookup_put_fd_uobject(struct ib_uobject * uobj,enum rdma_lookup_mode mode)7289867f5c6SJason Gunthorpe static void lookup_put_fd_uobject(struct ib_uobject *uobj,
7299867f5c6SJason Gunthorpe enum rdma_lookup_mode mode)
730cf8966b3SMatan Barak {
731cf8966b3SMatan Barak struct file *filp = uobj->object;
732cf8966b3SMatan Barak
7339867f5c6SJason Gunthorpe WARN_ON(mode != UVERBS_LOOKUP_READ);
734f7c8416cSJason Gunthorpe /*
735f7c8416cSJason Gunthorpe * This indirectly calls uverbs_uobject_fd_release() and free the
736f7c8416cSJason Gunthorpe * object
737f7c8416cSJason Gunthorpe */
738cf8966b3SMatan Barak fput(filp);
739cf8966b3SMatan Barak }
740cf8966b3SMatan Barak
rdma_lookup_put_uobject(struct ib_uobject * uobj,enum rdma_lookup_mode mode)7419867f5c6SJason Gunthorpe void rdma_lookup_put_uobject(struct ib_uobject *uobj,
7429867f5c6SJason Gunthorpe enum rdma_lookup_mode mode)
74338321256SMatan Barak {
7449867f5c6SJason Gunthorpe assert_uverbs_usecnt(uobj, mode);
74538321256SMatan Barak /*
74638321256SMatan Barak * In order to unlock an object, either decrease its usecnt for
74730004b86SMatan Barak * read access or zero it in case of exclusive access. See
74838321256SMatan Barak * uverbs_try_lock_object for locking schema information.
74938321256SMatan Barak */
7509867f5c6SJason Gunthorpe switch (mode) {
7519867f5c6SJason Gunthorpe case UVERBS_LOOKUP_READ:
75238321256SMatan Barak atomic_dec(&uobj->usecnt);
7539867f5c6SJason Gunthorpe break;
7549867f5c6SJason Gunthorpe case UVERBS_LOOKUP_WRITE:
75538321256SMatan Barak atomic_set(&uobj->usecnt, 0);
7569867f5c6SJason Gunthorpe break;
7577452a3c7SJason Gunthorpe case UVERBS_LOOKUP_DESTROY:
7587452a3c7SJason Gunthorpe break;
7599867f5c6SJason Gunthorpe }
76038321256SMatan Barak
761f0abc761SLeon Romanovsky uobj->uapi_object->type_class->lookup_put(uobj, mode);
7625671f79bSJason Gunthorpe /* Pairs with the kref obtained by type->lookup_get */
76338321256SMatan Barak uverbs_uobject_put(uobj);
76438321256SMatan Barak }
76538321256SMatan Barak
setup_ufile_idr_uobject(struct ib_uverbs_file * ufile)7660f50d88aSJason Gunthorpe void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile)
7670f50d88aSJason Gunthorpe {
768b9b0f345SMatthew Wilcox xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC);
7690f50d88aSJason Gunthorpe }
7700f50d88aSJason Gunthorpe
release_ufile_idr_uobject(struct ib_uverbs_file * ufile)7710f50d88aSJason Gunthorpe void release_ufile_idr_uobject(struct ib_uverbs_file *ufile)
7720f50d88aSJason Gunthorpe {
7730f50d88aSJason Gunthorpe struct ib_uobject *entry;
774b9b0f345SMatthew Wilcox unsigned long id;
7750f50d88aSJason Gunthorpe
7760f50d88aSJason Gunthorpe /*
7770f50d88aSJason Gunthorpe * At this point uverbs_cleanup_ufile() is guaranteed to have run, and
778b9b0f345SMatthew Wilcox * there are no HW objects left, however the xarray is still populated
7790f50d88aSJason Gunthorpe * with anything that has not been cleaned up by userspace. Since the
7800f50d88aSJason Gunthorpe * kref on ufile is 0, nothing is allowed to call lookup_get.
7810f50d88aSJason Gunthorpe *
7820f50d88aSJason Gunthorpe * This is an optimized equivalent to remove_handle_idr_uobject
7830f50d88aSJason Gunthorpe */
784b9b0f345SMatthew Wilcox xa_for_each(&ufile->idr, id, entry) {
7850f50d88aSJason Gunthorpe WARN_ON(entry->object);
7860f50d88aSJason Gunthorpe uverbs_uobject_put(entry);
7870f50d88aSJason Gunthorpe }
7880f50d88aSJason Gunthorpe
789b9b0f345SMatthew Wilcox xa_destroy(&ufile->idr);
7900f50d88aSJason Gunthorpe }
7910f50d88aSJason Gunthorpe
79238321256SMatan Barak const struct uverbs_obj_type_class uverbs_idr_class = {
79338321256SMatan Barak .alloc_begin = alloc_begin_idr_uobject,
79438321256SMatan Barak .lookup_get = lookup_get_idr_uobject,
79538321256SMatan Barak .alloc_commit = alloc_commit_idr_uobject,
79638321256SMatan Barak .alloc_abort = alloc_abort_idr_uobject,
79738321256SMatan Barak .lookup_put = lookup_put_idr_uobject,
7980f50d88aSJason Gunthorpe .destroy_hw = destroy_hw_idr_uobject,
7990f50d88aSJason Gunthorpe .remove_handle = remove_handle_idr_uobject,
8006e0954b1SJason Gunthorpe .swap_uobjects = swap_idr_uobjects,
80138321256SMatan Barak };
8021114b0a8SMatan Barak EXPORT_SYMBOL(uverbs_idr_class);
80338321256SMatan Barak
804f7c8416cSJason Gunthorpe /*
805f7c8416cSJason Gunthorpe * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct
806f7c8416cSJason Gunthorpe * file_operations release method.
807f7c8416cSJason Gunthorpe */
uverbs_uobject_fd_release(struct inode * inode,struct file * filp)808f7c8416cSJason Gunthorpe int uverbs_uobject_fd_release(struct inode *inode, struct file *filp)
809cf8966b3SMatan Barak {
810849e1490SJason Gunthorpe struct ib_uverbs_file *ufile;
811849e1490SJason Gunthorpe struct ib_uobject *uobj;
812849e1490SJason Gunthorpe
813849e1490SJason Gunthorpe /*
814849e1490SJason Gunthorpe * This can only happen if the fput came from alloc_abort_fd_uobject()
815849e1490SJason Gunthorpe */
816849e1490SJason Gunthorpe if (!filp->private_data)
817849e1490SJason Gunthorpe return 0;
818849e1490SJason Gunthorpe uobj = filp->private_data;
819849e1490SJason Gunthorpe ufile = uobj->ufile;
820849e1490SJason Gunthorpe
821849e1490SJason Gunthorpe if (down_read_trylock(&ufile->hw_destroy_rwsem)) {
822a6a3797dSShamir Rabinovitch struct uverbs_attr_bundle attrs = {
823a6a3797dSShamir Rabinovitch .context = uobj->context,
824a6a3797dSShamir Rabinovitch .ufile = ufile,
825a6a3797dSShamir Rabinovitch };
826cf8966b3SMatan Barak
82787ad80abSJason Gunthorpe /*
82887ad80abSJason Gunthorpe * lookup_get_fd_uobject holds the kref on the struct file any
82987ad80abSJason Gunthorpe * time a FD uobj is locked, which prevents this release
83087ad80abSJason Gunthorpe * method from being invoked. Meaning we can always get the
83187ad80abSJason Gunthorpe * write lock here, or we have a kernel bug.
83287ad80abSJason Gunthorpe */
8339867f5c6SJason Gunthorpe WARN_ON(uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE));
834a6a3797dSShamir Rabinovitch uverbs_destroy_uobject(uobj, RDMA_REMOVE_CLOSE, &attrs);
83587064277SJason Gunthorpe up_read(&ufile->hw_destroy_rwsem);
836e6d5d5ddSJason Gunthorpe }
837e6d5d5ddSJason Gunthorpe
838849e1490SJason Gunthorpe /* Matches the get in alloc_commit_fd_uobject() */
839e6d5d5ddSJason Gunthorpe kref_put(&ufile->ref, ib_uverbs_release_file);
840e6d5d5ddSJason Gunthorpe
8415671f79bSJason Gunthorpe /* Pairs with filp->private_data in alloc_begin_fd_uobject */
842d0259e82SJason Gunthorpe uverbs_uobject_put(uobj);
843f7c8416cSJason Gunthorpe return 0;
844cf8966b3SMatan Barak }
845f7c8416cSJason Gunthorpe EXPORT_SYMBOL(uverbs_uobject_fd_release);
846cf8966b3SMatan Barak
847e951747aSJason Gunthorpe /*
848e951747aSJason Gunthorpe * Drop the ucontext off the ufile and completely disconnect it from the
849e951747aSJason Gunthorpe * ib_device
850e951747aSJason Gunthorpe */
ufile_destroy_ucontext(struct ib_uverbs_file * ufile,enum rdma_remove_reason reason)851e951747aSJason Gunthorpe static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile,
852e951747aSJason Gunthorpe enum rdma_remove_reason reason)
853e951747aSJason Gunthorpe {
854e951747aSJason Gunthorpe struct ib_ucontext *ucontext = ufile->ucontext;
855ce92db1cSJason Gunthorpe struct ib_device *ib_dev = ucontext->device;
856e951747aSJason Gunthorpe
857ce92db1cSJason Gunthorpe /*
858ce92db1cSJason Gunthorpe * If we are closing the FD then the user mmap VMAs must have
859ce92db1cSJason Gunthorpe * already been destroyed as they hold on to the filep, otherwise
860ce92db1cSJason Gunthorpe * they need to be zap'd.
861ce92db1cSJason Gunthorpe */
8625f9794dcSJason Gunthorpe if (reason == RDMA_REMOVE_DRIVER_REMOVE) {
8635f9794dcSJason Gunthorpe uverbs_user_mmap_disassociate(ufile);
8643023a1e9SKamal Heib if (ib_dev->ops.disassociate_ucontext)
8653023a1e9SKamal Heib ib_dev->ops.disassociate_ucontext(ucontext);
8665f9794dcSJason Gunthorpe }
867e951747aSJason Gunthorpe
868ce92db1cSJason Gunthorpe ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev,
869e951747aSJason Gunthorpe RDMACG_RESOURCE_HCA_HANDLE);
870e951747aSJason Gunthorpe
87112d23a91SLeon Romanovsky rdma_restrack_del(&ucontext->res);
87212d23a91SLeon Romanovsky
873a2a074efSLeon Romanovsky ib_dev->ops.dealloc_ucontext(ucontext);
8743411f9f0SMichal Kalderon WARN_ON(!xa_empty(&ucontext->mmap_xa));
875a2a074efSLeon Romanovsky kfree(ucontext);
876e951747aSJason Gunthorpe
877e951747aSJason Gunthorpe ufile->ucontext = NULL;
878e951747aSJason Gunthorpe }
879e951747aSJason Gunthorpe
__uverbs_cleanup_ufile(struct ib_uverbs_file * ufile,enum rdma_remove_reason reason)8806ef1c828SJason Gunthorpe static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
8811c77483eSYishai Hadas enum rdma_remove_reason reason)
88238321256SMatan Barak {
88338321256SMatan Barak struct ib_uobject *obj, *next_obj;
8841c77483eSYishai Hadas int ret = -EINVAL;
885a6a3797dSShamir Rabinovitch struct uverbs_attr_bundle attrs = { .ufile = ufile };
88638321256SMatan Barak
88738321256SMatan Barak /*
88838321256SMatan Barak * This shouldn't run while executing other commands on this
889cf8966b3SMatan Barak * context. Thus, the only thing we should take care of is
890cf8966b3SMatan Barak * releasing a FD while traversing this list. The FD could be
891cf8966b3SMatan Barak * closed and released from the _release fop of this FD.
892cf8966b3SMatan Barak * In order to mitigate this, we add a lock.
8931c77483eSYishai Hadas * We take and release the lock per traversal in order to let
8941c77483eSYishai Hadas * other threads (which might still use the FDs) chance to run.
89538321256SMatan Barak */
8966a5e9c88SJason Gunthorpe list_for_each_entry_safe(obj, next_obj, &ufile->uobjects, list) {
897a6a3797dSShamir Rabinovitch attrs.context = obj->context;
89838321256SMatan Barak /*
89938321256SMatan Barak * if we hit this WARN_ON, that means we are
90038321256SMatan Barak * racing with a lookup_get.
90138321256SMatan Barak */
9029867f5c6SJason Gunthorpe WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE));
903efa968eeSLeon Romanovsky if (reason == RDMA_REMOVE_DRIVER_FAILURE)
904efa968eeSLeon Romanovsky obj->object = NULL;
905a6a3797dSShamir Rabinovitch if (!uverbs_destroy_uobject(obj, reason, &attrs))
9061c77483eSYishai Hadas ret = 0;
907e4ff3d22SArtemy Kovalyov else
908e4ff3d22SArtemy Kovalyov atomic_set(&obj->usecnt, 0);
909f025c489SMatan Barak }
910efa968eeSLeon Romanovsky
911efa968eeSLeon Romanovsky if (reason == RDMA_REMOVE_DRIVER_FAILURE) {
912efa968eeSLeon Romanovsky WARN_ON(!list_empty(&ufile->uobjects));
913efa968eeSLeon Romanovsky return 0;
914efa968eeSLeon Romanovsky }
9151c77483eSYishai Hadas return ret;
91638321256SMatan Barak }
9171c77483eSYishai Hadas
918e951747aSJason Gunthorpe /*
919016b26afSYangyang Li * Destroy the ucontext and every uobject associated with it.
920e951747aSJason Gunthorpe *
921e951747aSJason Gunthorpe * This is internally locked and can be called in parallel from multiple
922e951747aSJason Gunthorpe * contexts.
923e951747aSJason Gunthorpe */
uverbs_destroy_ufile_hw(struct ib_uverbs_file * ufile,enum rdma_remove_reason reason)924e951747aSJason Gunthorpe void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
925e951747aSJason Gunthorpe enum rdma_remove_reason reason)
9261c77483eSYishai Hadas {
927e951747aSJason Gunthorpe down_write(&ufile->hw_destroy_rwsem);
9286a5e9c88SJason Gunthorpe
9291c77483eSYishai Hadas /*
930e951747aSJason Gunthorpe * If a ucontext was never created then we can't have any uobjects to
931e951747aSJason Gunthorpe * cleanup, nothing to do.
9321c77483eSYishai Hadas */
933e951747aSJason Gunthorpe if (!ufile->ucontext)
934e951747aSJason Gunthorpe goto done;
935e951747aSJason Gunthorpe
936efa968eeSLeon Romanovsky while (!list_empty(&ufile->uobjects) &&
937efa968eeSLeon Romanovsky !__uverbs_cleanup_ufile(ufile, reason)) {
9381c77483eSYishai Hadas }
9391c77483eSYishai Hadas
940efa968eeSLeon Romanovsky if (WARN_ON(!list_empty(&ufile->uobjects)))
941efa968eeSLeon Romanovsky __uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE);
942e951747aSJason Gunthorpe ufile_destroy_ucontext(ufile, reason);
943e951747aSJason Gunthorpe
944e951747aSJason Gunthorpe done:
94587064277SJason Gunthorpe up_write(&ufile->hw_destroy_rwsem);
94638321256SMatan Barak }
94738321256SMatan Barak
948cf8966b3SMatan Barak const struct uverbs_obj_type_class uverbs_fd_class = {
949cf8966b3SMatan Barak .alloc_begin = alloc_begin_fd_uobject,
950cf8966b3SMatan Barak .lookup_get = lookup_get_fd_uobject,
951cf8966b3SMatan Barak .alloc_commit = alloc_commit_fd_uobject,
952cf8966b3SMatan Barak .alloc_abort = alloc_abort_fd_uobject,
953cf8966b3SMatan Barak .lookup_put = lookup_put_fd_uobject,
9540f50d88aSJason Gunthorpe .destroy_hw = destroy_hw_fd_uobject,
9550f50d88aSJason Gunthorpe .remove_handle = remove_handle_fd_uobject,
956cf8966b3SMatan Barak };
9571114b0a8SMatan Barak EXPORT_SYMBOL(uverbs_fd_class);
958cf8966b3SMatan Barak
9596ef1c828SJason Gunthorpe struct ib_uobject *
uverbs_get_uobject_from_file(u16 object_id,enum uverbs_obj_access access,s64 id,struct uverbs_attr_bundle * attrs)96070f06b26SShamir Rabinovitch uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
96170f06b26SShamir Rabinovitch s64 id, struct uverbs_attr_bundle *attrs)
962a0aa309cSMatan Barak {
9636b0d08f4SJason Gunthorpe const struct uverbs_api_object *obj =
96470f06b26SShamir Rabinovitch uapi_get_object(attrs->ufile->device->uapi, object_id);
9656b0d08f4SJason Gunthorpe
966a0aa309cSMatan Barak switch (access) {
967a0aa309cSMatan Barak case UVERBS_ACCESS_READ:
96870f06b26SShamir Rabinovitch return rdma_lookup_get_uobject(obj, attrs->ufile, id,
96970f06b26SShamir Rabinovitch UVERBS_LOOKUP_READ, attrs);
970a0aa309cSMatan Barak case UVERBS_ACCESS_DESTROY:
9717452a3c7SJason Gunthorpe /* Actual destruction is done inside uverbs_handle_method */
97270f06b26SShamir Rabinovitch return rdma_lookup_get_uobject(obj, attrs->ufile, id,
97370f06b26SShamir Rabinovitch UVERBS_LOOKUP_DESTROY, attrs);
974a0aa309cSMatan Barak case UVERBS_ACCESS_WRITE:
97570f06b26SShamir Rabinovitch return rdma_lookup_get_uobject(obj, attrs->ufile, id,
97670f06b26SShamir Rabinovitch UVERBS_LOOKUP_WRITE, attrs);
977a0aa309cSMatan Barak case UVERBS_ACCESS_NEW:
97839e83af8SJason Gunthorpe return rdma_alloc_begin_uobject(obj, attrs);
979a0aa309cSMatan Barak default:
980a0aa309cSMatan Barak WARN_ON(true);
981a0aa309cSMatan Barak return ERR_PTR(-EOPNOTSUPP);
982a0aa309cSMatan Barak }
983a0aa309cSMatan Barak }
984a0aa309cSMatan Barak
uverbs_finalize_object(struct ib_uobject * uobj,enum uverbs_obj_access access,bool hw_obj_valid,bool commit,struct uverbs_attr_bundle * attrs)985849e1490SJason Gunthorpe void uverbs_finalize_object(struct ib_uobject *uobj,
9860ac8903cSJason Gunthorpe enum uverbs_obj_access access, bool hw_obj_valid,
9870ac8903cSJason Gunthorpe bool commit, struct uverbs_attr_bundle *attrs)
988a0aa309cSMatan Barak {
989a0aa309cSMatan Barak /*
990a0aa309cSMatan Barak * refcounts should be handled at the object level and not at the
991a0aa309cSMatan Barak * uobject level. Refcounts of the objects themselves are done in
992a0aa309cSMatan Barak * handlers.
993a0aa309cSMatan Barak */
994a0aa309cSMatan Barak
995a0aa309cSMatan Barak switch (access) {
996a0aa309cSMatan Barak case UVERBS_ACCESS_READ:
9979867f5c6SJason Gunthorpe rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_READ);
998a0aa309cSMatan Barak break;
999a0aa309cSMatan Barak case UVERBS_ACCESS_WRITE:
10009867f5c6SJason Gunthorpe rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);
1001a0aa309cSMatan Barak break;
1002a0aa309cSMatan Barak case UVERBS_ACCESS_DESTROY:
10037452a3c7SJason Gunthorpe if (uobj)
10047452a3c7SJason Gunthorpe rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY);
1005a0aa309cSMatan Barak break;
1006a0aa309cSMatan Barak case UVERBS_ACCESS_NEW:
1007a0aa309cSMatan Barak if (commit)
1008849e1490SJason Gunthorpe rdma_alloc_commit_uobject(uobj, attrs);
1009a0aa309cSMatan Barak else
10100ac8903cSJason Gunthorpe rdma_alloc_abort_uobject(uobj, attrs, hw_obj_valid);
1011a0aa309cSMatan Barak break;
1012a0aa309cSMatan Barak default:
1013a0aa309cSMatan Barak WARN_ON(true);
1014a0aa309cSMatan Barak }
1015a0aa309cSMatan Barak }
1016