1 /* 2 * Copyright (c) 2016, Mellanox Technologies inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/file.h> 34 #include <linux/anon_inodes.h> 35 #include <linux/sched/mm.h> 36 #include <rdma/ib_verbs.h> 37 #include <rdma/uverbs_types.h> 38 #include <linux/rcupdate.h> 39 #include <rdma/uverbs_ioctl.h> 40 #include <rdma/rdma_user_ioctl.h> 41 #include "uverbs.h" 42 #include "core_priv.h" 43 #include "rdma_core.h" 44 45 static void uverbs_uobject_free(struct kref *ref) 46 { 47 kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu); 48 } 49 50 /* 51 * In order to indicate we no longer needs this uobject, uverbs_uobject_put 52 * is called. When the reference count is decreased, the uobject is freed. 53 * For example, this is used when attaching a completion channel to a CQ. 54 */ 55 void uverbs_uobject_put(struct ib_uobject *uobject) 56 { 57 kref_put(&uobject->ref, uverbs_uobject_free); 58 } 59 EXPORT_SYMBOL(uverbs_uobject_put); 60 61 static int uverbs_try_lock_object(struct ib_uobject *uobj, 62 enum rdma_lookup_mode mode) 63 { 64 /* 65 * When a shared access is required, we use a positive counter. Each 66 * shared access request checks that the value != -1 and increment it. 67 * Exclusive access is required for operations like write or destroy. 68 * In exclusive access mode, we check that the counter is zero (nobody 69 * claimed this object) and we set it to -1. Releasing a shared access 70 * lock is done simply by decreasing the counter. As for exclusive 71 * access locks, since only a single one of them is is allowed 72 * concurrently, setting the counter to zero is enough for releasing 73 * this lock. 74 */ 75 switch (mode) { 76 case UVERBS_LOOKUP_READ: 77 return atomic_fetch_add_unless(&uobj->usecnt, 1, -1) == -1 ? 78 -EBUSY : 0; 79 case UVERBS_LOOKUP_WRITE: 80 /* lock is exclusive */ 81 return atomic_cmpxchg(&uobj->usecnt, 0, -1) == 0 ? 0 : -EBUSY; 82 case UVERBS_LOOKUP_DESTROY: 83 return 0; 84 } 85 return 0; 86 } 87 88 static void assert_uverbs_usecnt(struct ib_uobject *uobj, 89 enum rdma_lookup_mode mode) 90 { 91 #ifdef CONFIG_LOCKDEP 92 switch (mode) { 93 case UVERBS_LOOKUP_READ: 94 WARN_ON(atomic_read(&uobj->usecnt) <= 0); 95 break; 96 case UVERBS_LOOKUP_WRITE: 97 WARN_ON(atomic_read(&uobj->usecnt) != -1); 98 break; 99 case UVERBS_LOOKUP_DESTROY: 100 break; 101 } 102 #endif 103 } 104 105 /* 106 * This must be called with the hw_destroy_rwsem locked for read or write, 107 * also the uobject itself must be locked for write. 108 * 109 * Upon return the HW object is guaranteed to be destroyed. 110 * 111 * For RDMA_REMOVE_ABORT, the hw_destroy_rwsem is not required to be held, 112 * however the type's allocat_commit function cannot have been called and the 113 * uobject cannot be on the uobjects_lists 114 * 115 * For RDMA_REMOVE_DESTROY the caller should be holding a kref (eg via 116 * rdma_lookup_get_uobject) and the object is left in a state where the caller 117 * needs to call rdma_lookup_put_uobject. 118 * 119 * For all other destroy modes this function internally unlocks the uobject 120 * and consumes the kref on the uobj. 121 */ 122 static int uverbs_destroy_uobject(struct ib_uobject *uobj, 123 enum rdma_remove_reason reason, 124 struct uverbs_attr_bundle *attrs) 125 { 126 struct ib_uverbs_file *ufile = attrs->ufile; 127 unsigned long flags; 128 int ret; 129 130 lockdep_assert_held(&ufile->hw_destroy_rwsem); 131 assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE); 132 133 if (reason == RDMA_REMOVE_ABORT) { 134 WARN_ON(!list_empty(&uobj->list)); 135 WARN_ON(!uobj->context); 136 uobj->uapi_object->type_class->alloc_abort(uobj); 137 } else if (uobj->object) { 138 ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason, 139 attrs); 140 if (ret) 141 /* Nothing to be done, wait till ucontext will clean it */ 142 return ret; 143 144 uobj->object = NULL; 145 } 146 147 uobj->context = NULL; 148 149 /* 150 * For DESTROY the usecnt is not changed, the caller is expected to 151 * manage it via uobj_put_destroy(). Only DESTROY can remove the IDR 152 * handle. 153 */ 154 if (reason != RDMA_REMOVE_DESTROY) 155 atomic_set(&uobj->usecnt, 0); 156 else 157 uobj->uapi_object->type_class->remove_handle(uobj); 158 159 if (!list_empty(&uobj->list)) { 160 spin_lock_irqsave(&ufile->uobjects_lock, flags); 161 list_del_init(&uobj->list); 162 spin_unlock_irqrestore(&ufile->uobjects_lock, flags); 163 164 /* 165 * Pairs with the get in rdma_alloc_commit_uobject(), could 166 * destroy uobj. 167 */ 168 uverbs_uobject_put(uobj); 169 } 170 171 /* 172 * When aborting the stack kref remains owned by the core code, and is 173 * not transferred into the type. Pairs with the get in alloc_uobj 174 */ 175 if (reason == RDMA_REMOVE_ABORT) 176 uverbs_uobject_put(uobj); 177 178 return 0; 179 } 180 181 /* 182 * This calls uverbs_destroy_uobject() using the RDMA_REMOVE_DESTROY 183 * sequence. It should only be used from command callbacks. On success the 184 * caller must pair this with uobj_put_destroy(). This 185 * version requires the caller to have already obtained an 186 * LOOKUP_DESTROY uobject kref. 187 */ 188 int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs) 189 { 190 struct ib_uverbs_file *ufile = attrs->ufile; 191 int ret; 192 193 down_read(&ufile->hw_destroy_rwsem); 194 195 /* 196 * Once the uobject is destroyed by RDMA_REMOVE_DESTROY then it is left 197 * write locked as the callers put it back with UVERBS_LOOKUP_DESTROY. 198 * This is because any other concurrent thread can still see the object 199 * in the xarray due to RCU. Leaving it locked ensures nothing else will 200 * touch it. 201 */ 202 ret = uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE); 203 if (ret) 204 goto out_unlock; 205 206 ret = uverbs_destroy_uobject(uobj, RDMA_REMOVE_DESTROY, attrs); 207 if (ret) { 208 atomic_set(&uobj->usecnt, 0); 209 goto out_unlock; 210 } 211 212 out_unlock: 213 up_read(&ufile->hw_destroy_rwsem); 214 return ret; 215 } 216 217 /* 218 * uobj_get_destroy destroys the HW object and returns a handle to the uobj 219 * with a NULL object pointer. The caller must pair this with 220 * uobj_put_destroy(). 221 */ 222 struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj, 223 u32 id, struct uverbs_attr_bundle *attrs) 224 { 225 struct ib_uobject *uobj; 226 int ret; 227 228 uobj = rdma_lookup_get_uobject(obj, attrs->ufile, id, 229 UVERBS_LOOKUP_DESTROY, attrs); 230 if (IS_ERR(uobj)) 231 return uobj; 232 233 ret = uobj_destroy(uobj, attrs); 234 if (ret) { 235 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY); 236 return ERR_PTR(ret); 237 } 238 239 return uobj; 240 } 241 242 /* 243 * Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success 244 * (negative errno on failure). For use by callers that do not need the uobj. 245 */ 246 int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id, 247 struct uverbs_attr_bundle *attrs) 248 { 249 struct ib_uobject *uobj; 250 251 uobj = __uobj_get_destroy(obj, id, attrs); 252 if (IS_ERR(uobj)) 253 return PTR_ERR(uobj); 254 uobj_put_destroy(uobj); 255 return 0; 256 } 257 258 /* alloc_uobj must be undone by uverbs_destroy_uobject() */ 259 static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs, 260 const struct uverbs_api_object *obj) 261 { 262 struct ib_uverbs_file *ufile = attrs->ufile; 263 struct ib_uobject *uobj; 264 265 if (!attrs->context) { 266 struct ib_ucontext *ucontext = 267 ib_uverbs_get_ucontext_file(ufile); 268 269 if (IS_ERR(ucontext)) 270 return ERR_CAST(ucontext); 271 attrs->context = ucontext; 272 } 273 274 uobj = kzalloc(obj->type_attrs->obj_size, GFP_KERNEL); 275 if (!uobj) 276 return ERR_PTR(-ENOMEM); 277 /* 278 * user_handle should be filled by the handler, 279 * The object is added to the list in the commit stage. 280 */ 281 uobj->ufile = ufile; 282 uobj->context = attrs->context; 283 INIT_LIST_HEAD(&uobj->list); 284 uobj->uapi_object = obj; 285 /* 286 * Allocated objects start out as write locked to deny any other 287 * syscalls from accessing them until they are committed. See 288 * rdma_alloc_commit_uobject 289 */ 290 atomic_set(&uobj->usecnt, -1); 291 kref_init(&uobj->ref); 292 293 return uobj; 294 } 295 296 static int idr_add_uobj(struct ib_uobject *uobj) 297 { 298 /* 299 * We start with allocating an idr pointing to NULL. This represents an 300 * object which isn't initialized yet. We'll replace it later on with 301 * the real object once we commit. 302 */ 303 return xa_alloc(&uobj->ufile->idr, &uobj->id, NULL, xa_limit_32b, 304 GFP_KERNEL); 305 } 306 307 /* Returns the ib_uobject or an error. The caller should check for IS_ERR. */ 308 static struct ib_uobject * 309 lookup_get_idr_uobject(const struct uverbs_api_object *obj, 310 struct ib_uverbs_file *ufile, s64 id, 311 enum rdma_lookup_mode mode) 312 { 313 struct ib_uobject *uobj; 314 315 if (id < 0 || id > ULONG_MAX) 316 return ERR_PTR(-EINVAL); 317 318 rcu_read_lock(); 319 /* 320 * The idr_find is guaranteed to return a pointer to something that 321 * isn't freed yet, or NULL, as the free after idr_remove goes through 322 * kfree_rcu(). However the object may still have been released and 323 * kfree() could be called at any time. 324 */ 325 uobj = xa_load(&ufile->idr, id); 326 if (!uobj || !kref_get_unless_zero(&uobj->ref)) 327 uobj = ERR_PTR(-ENOENT); 328 rcu_read_unlock(); 329 return uobj; 330 } 331 332 static struct ib_uobject * 333 lookup_get_fd_uobject(const struct uverbs_api_object *obj, 334 struct ib_uverbs_file *ufile, s64 id, 335 enum rdma_lookup_mode mode) 336 { 337 const struct uverbs_obj_fd_type *fd_type; 338 struct file *f; 339 struct ib_uobject *uobject; 340 int fdno = id; 341 342 if (fdno != id) 343 return ERR_PTR(-EINVAL); 344 345 if (mode != UVERBS_LOOKUP_READ) 346 return ERR_PTR(-EOPNOTSUPP); 347 348 if (!obj->type_attrs) 349 return ERR_PTR(-EIO); 350 fd_type = 351 container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); 352 353 f = fget(fdno); 354 if (!f) 355 return ERR_PTR(-EBADF); 356 357 uobject = f->private_data; 358 /* 359 * fget(id) ensures we are not currently running 360 * uverbs_uobject_fd_release(), and the caller is expected to ensure 361 * that release is never done while a call to lookup is possible. 362 */ 363 if (f->f_op != fd_type->fops || uobject->ufile != ufile) { 364 fput(f); 365 return ERR_PTR(-EBADF); 366 } 367 368 uverbs_uobject_get(uobject); 369 return uobject; 370 } 371 372 struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj, 373 struct ib_uverbs_file *ufile, s64 id, 374 enum rdma_lookup_mode mode, 375 struct uverbs_attr_bundle *attrs) 376 { 377 struct ib_uobject *uobj; 378 int ret; 379 380 if (obj == ERR_PTR(-ENOMSG)) { 381 /* must be UVERBS_IDR_ANY_OBJECT, see uapi_get_object() */ 382 uobj = lookup_get_idr_uobject(NULL, ufile, id, mode); 383 if (IS_ERR(uobj)) 384 return uobj; 385 } else { 386 if (IS_ERR(obj)) 387 return ERR_PTR(-EINVAL); 388 389 uobj = obj->type_class->lookup_get(obj, ufile, id, mode); 390 if (IS_ERR(uobj)) 391 return uobj; 392 393 if (uobj->uapi_object != obj) { 394 ret = -EINVAL; 395 goto free; 396 } 397 } 398 399 /* 400 * If we have been disassociated block every command except for 401 * DESTROY based commands. 402 */ 403 if (mode != UVERBS_LOOKUP_DESTROY && 404 !srcu_dereference(ufile->device->ib_dev, 405 &ufile->device->disassociate_srcu)) { 406 ret = -EIO; 407 goto free; 408 } 409 410 ret = uverbs_try_lock_object(uobj, mode); 411 if (ret) 412 goto free; 413 if (attrs) 414 attrs->context = uobj->context; 415 416 return uobj; 417 free: 418 uobj->uapi_object->type_class->lookup_put(uobj, mode); 419 uverbs_uobject_put(uobj); 420 return ERR_PTR(ret); 421 } 422 423 static struct ib_uobject * 424 alloc_begin_idr_uobject(const struct uverbs_api_object *obj, 425 struct uverbs_attr_bundle *attrs) 426 { 427 int ret; 428 struct ib_uobject *uobj; 429 430 uobj = alloc_uobj(attrs, obj); 431 if (IS_ERR(uobj)) 432 return uobj; 433 434 ret = idr_add_uobj(uobj); 435 if (ret) 436 goto uobj_put; 437 438 ret = ib_rdmacg_try_charge(&uobj->cg_obj, uobj->context->device, 439 RDMACG_RESOURCE_HCA_OBJECT); 440 if (ret) 441 goto remove; 442 443 return uobj; 444 445 remove: 446 xa_erase(&attrs->ufile->idr, uobj->id); 447 uobj_put: 448 uverbs_uobject_put(uobj); 449 return ERR_PTR(ret); 450 } 451 452 static struct ib_uobject * 453 alloc_begin_fd_uobject(const struct uverbs_api_object *obj, 454 struct uverbs_attr_bundle *attrs) 455 { 456 const struct uverbs_obj_fd_type *fd_type; 457 int new_fd; 458 struct ib_uobject *uobj, *ret; 459 struct file *filp; 460 461 uobj = alloc_uobj(attrs, obj); 462 if (IS_ERR(uobj)) 463 return uobj; 464 465 fd_type = 466 container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); 467 if (WARN_ON(fd_type->fops->release != &uverbs_uobject_fd_release && 468 fd_type->fops->release != &uverbs_async_event_release)) { 469 ret = ERR_PTR(-EINVAL); 470 goto err_fd; 471 } 472 473 new_fd = get_unused_fd_flags(O_CLOEXEC); 474 if (new_fd < 0) { 475 ret = ERR_PTR(new_fd); 476 goto err_fd; 477 } 478 479 /* Note that uverbs_uobject_fd_release() is called during abort */ 480 filp = anon_inode_getfile(fd_type->name, fd_type->fops, NULL, 481 fd_type->flags); 482 if (IS_ERR(filp)) { 483 ret = ERR_CAST(filp); 484 goto err_getfile; 485 } 486 uobj->object = filp; 487 488 uobj->id = new_fd; 489 return uobj; 490 491 err_getfile: 492 put_unused_fd(new_fd); 493 err_fd: 494 uverbs_uobject_put(uobj); 495 return ret; 496 } 497 498 struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, 499 struct uverbs_attr_bundle *attrs) 500 { 501 struct ib_uverbs_file *ufile = attrs->ufile; 502 struct ib_uobject *ret; 503 504 if (IS_ERR(obj)) 505 return ERR_PTR(-EINVAL); 506 507 /* 508 * The hw_destroy_rwsem is held across the entire object creation and 509 * released during rdma_alloc_commit_uobject or 510 * rdma_alloc_abort_uobject 511 */ 512 if (!down_read_trylock(&ufile->hw_destroy_rwsem)) 513 return ERR_PTR(-EIO); 514 515 ret = obj->type_class->alloc_begin(obj, attrs); 516 if (IS_ERR(ret)) { 517 up_read(&ufile->hw_destroy_rwsem); 518 return ret; 519 } 520 return ret; 521 } 522 523 static void alloc_abort_idr_uobject(struct ib_uobject *uobj) 524 { 525 ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device, 526 RDMACG_RESOURCE_HCA_OBJECT); 527 528 xa_erase(&uobj->ufile->idr, uobj->id); 529 } 530 531 static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj, 532 enum rdma_remove_reason why, 533 struct uverbs_attr_bundle *attrs) 534 { 535 const struct uverbs_obj_idr_type *idr_type = 536 container_of(uobj->uapi_object->type_attrs, 537 struct uverbs_obj_idr_type, type); 538 int ret = idr_type->destroy_object(uobj, why, attrs); 539 540 if (ret) 541 return ret; 542 543 if (why == RDMA_REMOVE_ABORT) 544 return 0; 545 546 ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device, 547 RDMACG_RESOURCE_HCA_OBJECT); 548 549 return 0; 550 } 551 552 static void remove_handle_idr_uobject(struct ib_uobject *uobj) 553 { 554 xa_erase(&uobj->ufile->idr, uobj->id); 555 /* Matches the kref in alloc_commit_idr_uobject */ 556 uverbs_uobject_put(uobj); 557 } 558 559 static void alloc_abort_fd_uobject(struct ib_uobject *uobj) 560 { 561 struct file *filp = uobj->object; 562 563 fput(filp); 564 put_unused_fd(uobj->id); 565 } 566 567 static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj, 568 enum rdma_remove_reason why, 569 struct uverbs_attr_bundle *attrs) 570 { 571 const struct uverbs_obj_fd_type *fd_type = container_of( 572 uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type); 573 574 fd_type->destroy_object(uobj, why); 575 return 0; 576 } 577 578 static void remove_handle_fd_uobject(struct ib_uobject *uobj) 579 { 580 } 581 582 static void alloc_commit_idr_uobject(struct ib_uobject *uobj) 583 { 584 struct ib_uverbs_file *ufile = uobj->ufile; 585 void *old; 586 587 /* 588 * We already allocated this IDR with a NULL object, so 589 * this shouldn't fail. 590 * 591 * NOTE: Storing the uobj transfers our kref on uobj to the XArray. 592 * It will be put by remove_commit_idr_uobject() 593 */ 594 old = xa_store(&ufile->idr, uobj->id, uobj, GFP_KERNEL); 595 WARN_ON(old != NULL); 596 } 597 598 static void swap_idr_uobjects(struct ib_uobject *obj_old, 599 struct ib_uobject *obj_new) 600 { 601 struct ib_uverbs_file *ufile = obj_old->ufile; 602 void *old; 603 604 /* 605 * New must be an object that been allocated but not yet committed, this 606 * moves the pre-committed state to obj_old, new still must be comitted. 607 */ 608 old = xa_cmpxchg(&ufile->idr, obj_old->id, obj_old, XA_ZERO_ENTRY, 609 GFP_KERNEL); 610 if (WARN_ON(old != obj_old)) 611 return; 612 613 swap(obj_old->id, obj_new->id); 614 615 old = xa_cmpxchg(&ufile->idr, obj_old->id, NULL, obj_old, GFP_KERNEL); 616 WARN_ON(old != NULL); 617 } 618 619 static void alloc_commit_fd_uobject(struct ib_uobject *uobj) 620 { 621 int fd = uobj->id; 622 struct file *filp = uobj->object; 623 624 /* Matching put will be done in uverbs_uobject_fd_release() */ 625 kref_get(&uobj->ufile->ref); 626 627 /* This shouldn't be used anymore. Use the file object instead */ 628 uobj->id = 0; 629 630 /* 631 * NOTE: Once we install the file we loose ownership of our kref on 632 * uobj. It will be put by uverbs_uobject_fd_release() 633 */ 634 filp->private_data = uobj; 635 fd_install(fd, filp); 636 } 637 638 /* 639 * In all cases rdma_alloc_commit_uobject() consumes the kref to uobj and the 640 * caller can no longer assume uobj is valid. If this function fails it 641 * destroys the uboject, including the attached HW object. 642 */ 643 void rdma_alloc_commit_uobject(struct ib_uobject *uobj, 644 struct uverbs_attr_bundle *attrs) 645 { 646 struct ib_uverbs_file *ufile = attrs->ufile; 647 648 /* kref is held so long as the uobj is on the uobj list. */ 649 uverbs_uobject_get(uobj); 650 spin_lock_irq(&ufile->uobjects_lock); 651 list_add(&uobj->list, &ufile->uobjects); 652 spin_unlock_irq(&ufile->uobjects_lock); 653 654 /* matches atomic_set(-1) in alloc_uobj */ 655 atomic_set(&uobj->usecnt, 0); 656 657 /* alloc_commit consumes the uobj kref */ 658 uobj->uapi_object->type_class->alloc_commit(uobj); 659 660 /* Matches the down_read in rdma_alloc_begin_uobject */ 661 up_read(&ufile->hw_destroy_rwsem); 662 } 663 664 /* 665 * new_uobj will be assigned to the handle currently used by to_uobj, and 666 * to_uobj will be destroyed. 667 * 668 * Upon return the caller must do: 669 * rdma_alloc_commit_uobject(new_uobj) 670 * uobj_put_destroy(to_uobj) 671 * 672 * to_uobj must have a write get but the put mode switches to destroy once 673 * this is called. 674 */ 675 void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj, 676 struct uverbs_attr_bundle *attrs) 677 { 678 assert_uverbs_usecnt(new_uobj, UVERBS_LOOKUP_WRITE); 679 680 if (WARN_ON(to_uobj->uapi_object != new_uobj->uapi_object || 681 !to_uobj->uapi_object->type_class->swap_uobjects)) 682 return; 683 684 to_uobj->uapi_object->type_class->swap_uobjects(to_uobj, new_uobj); 685 686 /* 687 * If this fails then the uobject is still completely valid (though with 688 * a new ID) and we leak it until context close. 689 */ 690 uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs); 691 } 692 693 /* 694 * This consumes the kref for uobj. It is up to the caller to unwind the HW 695 * object and anything else connected to uobj before calling this. 696 */ 697 void rdma_alloc_abort_uobject(struct ib_uobject *uobj, 698 struct uverbs_attr_bundle *attrs, 699 bool hw_obj_valid) 700 { 701 struct ib_uverbs_file *ufile = uobj->ufile; 702 int ret; 703 704 if (hw_obj_valid) { 705 ret = uobj->uapi_object->type_class->destroy_hw( 706 uobj, RDMA_REMOVE_ABORT, attrs); 707 /* 708 * If the driver couldn't destroy the object then go ahead and 709 * commit it. Leaking objects that can't be destroyed is only 710 * done during FD close after the driver has a few more tries to 711 * destroy it. 712 */ 713 if (WARN_ON(ret)) 714 return rdma_alloc_commit_uobject(uobj, attrs); 715 } 716 717 uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs); 718 719 /* Matches the down_read in rdma_alloc_begin_uobject */ 720 up_read(&ufile->hw_destroy_rwsem); 721 } 722 723 static void lookup_put_idr_uobject(struct ib_uobject *uobj, 724 enum rdma_lookup_mode mode) 725 { 726 } 727 728 static void lookup_put_fd_uobject(struct ib_uobject *uobj, 729 enum rdma_lookup_mode mode) 730 { 731 struct file *filp = uobj->object; 732 733 WARN_ON(mode != UVERBS_LOOKUP_READ); 734 /* 735 * This indirectly calls uverbs_uobject_fd_release() and free the 736 * object 737 */ 738 fput(filp); 739 } 740 741 void rdma_lookup_put_uobject(struct ib_uobject *uobj, 742 enum rdma_lookup_mode mode) 743 { 744 assert_uverbs_usecnt(uobj, mode); 745 /* 746 * In order to unlock an object, either decrease its usecnt for 747 * read access or zero it in case of exclusive access. See 748 * uverbs_try_lock_object for locking schema information. 749 */ 750 switch (mode) { 751 case UVERBS_LOOKUP_READ: 752 atomic_dec(&uobj->usecnt); 753 break; 754 case UVERBS_LOOKUP_WRITE: 755 atomic_set(&uobj->usecnt, 0); 756 break; 757 case UVERBS_LOOKUP_DESTROY: 758 break; 759 } 760 761 uobj->uapi_object->type_class->lookup_put(uobj, mode); 762 /* Pairs with the kref obtained by type->lookup_get */ 763 uverbs_uobject_put(uobj); 764 } 765 766 void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile) 767 { 768 xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC); 769 } 770 771 void release_ufile_idr_uobject(struct ib_uverbs_file *ufile) 772 { 773 struct ib_uobject *entry; 774 unsigned long id; 775 776 /* 777 * At this point uverbs_cleanup_ufile() is guaranteed to have run, and 778 * there are no HW objects left, however the xarray is still populated 779 * with anything that has not been cleaned up by userspace. Since the 780 * kref on ufile is 0, nothing is allowed to call lookup_get. 781 * 782 * This is an optimized equivalent to remove_handle_idr_uobject 783 */ 784 xa_for_each(&ufile->idr, id, entry) { 785 WARN_ON(entry->object); 786 uverbs_uobject_put(entry); 787 } 788 789 xa_destroy(&ufile->idr); 790 } 791 792 const struct uverbs_obj_type_class uverbs_idr_class = { 793 .alloc_begin = alloc_begin_idr_uobject, 794 .lookup_get = lookup_get_idr_uobject, 795 .alloc_commit = alloc_commit_idr_uobject, 796 .alloc_abort = alloc_abort_idr_uobject, 797 .lookup_put = lookup_put_idr_uobject, 798 .destroy_hw = destroy_hw_idr_uobject, 799 .remove_handle = remove_handle_idr_uobject, 800 .swap_uobjects = swap_idr_uobjects, 801 }; 802 EXPORT_SYMBOL(uverbs_idr_class); 803 804 /* 805 * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct 806 * file_operations release method. 807 */ 808 int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) 809 { 810 struct ib_uverbs_file *ufile; 811 struct ib_uobject *uobj; 812 813 /* 814 * This can only happen if the fput came from alloc_abort_fd_uobject() 815 */ 816 if (!filp->private_data) 817 return 0; 818 uobj = filp->private_data; 819 ufile = uobj->ufile; 820 821 if (down_read_trylock(&ufile->hw_destroy_rwsem)) { 822 struct uverbs_attr_bundle attrs = { 823 .context = uobj->context, 824 .ufile = ufile, 825 }; 826 827 /* 828 * lookup_get_fd_uobject holds the kref on the struct file any 829 * time a FD uobj is locked, which prevents this release 830 * method from being invoked. Meaning we can always get the 831 * write lock here, or we have a kernel bug. 832 */ 833 WARN_ON(uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE)); 834 uverbs_destroy_uobject(uobj, RDMA_REMOVE_CLOSE, &attrs); 835 up_read(&ufile->hw_destroy_rwsem); 836 } 837 838 /* Matches the get in alloc_commit_fd_uobject() */ 839 kref_put(&ufile->ref, ib_uverbs_release_file); 840 841 /* Pairs with filp->private_data in alloc_begin_fd_uobject */ 842 uverbs_uobject_put(uobj); 843 return 0; 844 } 845 EXPORT_SYMBOL(uverbs_uobject_fd_release); 846 847 /* 848 * Drop the ucontext off the ufile and completely disconnect it from the 849 * ib_device 850 */ 851 static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile, 852 enum rdma_remove_reason reason) 853 { 854 struct ib_ucontext *ucontext = ufile->ucontext; 855 struct ib_device *ib_dev = ucontext->device; 856 857 /* 858 * If we are closing the FD then the user mmap VMAs must have 859 * already been destroyed as they hold on to the filep, otherwise 860 * they need to be zap'd. 861 */ 862 if (reason == RDMA_REMOVE_DRIVER_REMOVE) { 863 uverbs_user_mmap_disassociate(ufile); 864 if (ib_dev->ops.disassociate_ucontext) 865 ib_dev->ops.disassociate_ucontext(ucontext); 866 } 867 868 ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev, 869 RDMACG_RESOURCE_HCA_HANDLE); 870 871 rdma_restrack_del(&ucontext->res); 872 873 ib_dev->ops.dealloc_ucontext(ucontext); 874 WARN_ON(!xa_empty(&ucontext->mmap_xa)); 875 kfree(ucontext); 876 877 ufile->ucontext = NULL; 878 } 879 880 static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, 881 enum rdma_remove_reason reason) 882 { 883 struct ib_uobject *obj, *next_obj; 884 int ret = -EINVAL; 885 struct uverbs_attr_bundle attrs = { .ufile = ufile }; 886 887 /* 888 * This shouldn't run while executing other commands on this 889 * context. Thus, the only thing we should take care of is 890 * releasing a FD while traversing this list. The FD could be 891 * closed and released from the _release fop of this FD. 892 * In order to mitigate this, we add a lock. 893 * We take and release the lock per traversal in order to let 894 * other threads (which might still use the FDs) chance to run. 895 */ 896 list_for_each_entry_safe(obj, next_obj, &ufile->uobjects, list) { 897 attrs.context = obj->context; 898 /* 899 * if we hit this WARN_ON, that means we are 900 * racing with a lookup_get. 901 */ 902 WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE)); 903 if (reason == RDMA_REMOVE_DRIVER_FAILURE) 904 obj->object = NULL; 905 if (!uverbs_destroy_uobject(obj, reason, &attrs)) 906 ret = 0; 907 else 908 atomic_set(&obj->usecnt, 0); 909 } 910 911 if (reason == RDMA_REMOVE_DRIVER_FAILURE) { 912 WARN_ON(!list_empty(&ufile->uobjects)); 913 return 0; 914 } 915 return ret; 916 } 917 918 /* 919 * Destroy the ucontext and every uobject associated with it. 920 * 921 * This is internally locked and can be called in parallel from multiple 922 * contexts. 923 */ 924 void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, 925 enum rdma_remove_reason reason) 926 { 927 down_write(&ufile->hw_destroy_rwsem); 928 929 /* 930 * If a ucontext was never created then we can't have any uobjects to 931 * cleanup, nothing to do. 932 */ 933 if (!ufile->ucontext) 934 goto done; 935 936 while (!list_empty(&ufile->uobjects) && 937 !__uverbs_cleanup_ufile(ufile, reason)) { 938 } 939 940 if (WARN_ON(!list_empty(&ufile->uobjects))) 941 __uverbs_cleanup_ufile(ufile, RDMA_REMOVE_DRIVER_FAILURE); 942 ufile_destroy_ucontext(ufile, reason); 943 944 done: 945 up_write(&ufile->hw_destroy_rwsem); 946 } 947 948 const struct uverbs_obj_type_class uverbs_fd_class = { 949 .alloc_begin = alloc_begin_fd_uobject, 950 .lookup_get = lookup_get_fd_uobject, 951 .alloc_commit = alloc_commit_fd_uobject, 952 .alloc_abort = alloc_abort_fd_uobject, 953 .lookup_put = lookup_put_fd_uobject, 954 .destroy_hw = destroy_hw_fd_uobject, 955 .remove_handle = remove_handle_fd_uobject, 956 }; 957 EXPORT_SYMBOL(uverbs_fd_class); 958 959 struct ib_uobject * 960 uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, 961 s64 id, struct uverbs_attr_bundle *attrs) 962 { 963 const struct uverbs_api_object *obj = 964 uapi_get_object(attrs->ufile->device->uapi, object_id); 965 966 switch (access) { 967 case UVERBS_ACCESS_READ: 968 return rdma_lookup_get_uobject(obj, attrs->ufile, id, 969 UVERBS_LOOKUP_READ, attrs); 970 case UVERBS_ACCESS_DESTROY: 971 /* Actual destruction is done inside uverbs_handle_method */ 972 return rdma_lookup_get_uobject(obj, attrs->ufile, id, 973 UVERBS_LOOKUP_DESTROY, attrs); 974 case UVERBS_ACCESS_WRITE: 975 return rdma_lookup_get_uobject(obj, attrs->ufile, id, 976 UVERBS_LOOKUP_WRITE, attrs); 977 case UVERBS_ACCESS_NEW: 978 return rdma_alloc_begin_uobject(obj, attrs); 979 default: 980 WARN_ON(true); 981 return ERR_PTR(-EOPNOTSUPP); 982 } 983 } 984 985 void uverbs_finalize_object(struct ib_uobject *uobj, 986 enum uverbs_obj_access access, bool hw_obj_valid, 987 bool commit, struct uverbs_attr_bundle *attrs) 988 { 989 /* 990 * refcounts should be handled at the object level and not at the 991 * uobject level. Refcounts of the objects themselves are done in 992 * handlers. 993 */ 994 995 switch (access) { 996 case UVERBS_ACCESS_READ: 997 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_READ); 998 break; 999 case UVERBS_ACCESS_WRITE: 1000 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE); 1001 break; 1002 case UVERBS_ACCESS_DESTROY: 1003 if (uobj) 1004 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY); 1005 break; 1006 case UVERBS_ACCESS_NEW: 1007 if (commit) 1008 rdma_alloc_commit_uobject(uobj, attrs); 1009 else 1010 rdma_alloc_abort_uobject(uobj, attrs, hw_obj_valid); 1011 break; 1012 default: 1013 WARN_ON(true); 1014 } 1015 } 1016