1 /* 2 * Copyright (c) 2016, Mellanox Technologies inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/file.h> 34 #include <linux/anon_inodes.h> 35 #include <linux/sched/mm.h> 36 #include <rdma/ib_verbs.h> 37 #include <rdma/uverbs_types.h> 38 #include <linux/rcupdate.h> 39 #include <rdma/uverbs_ioctl.h> 40 #include <rdma/rdma_user_ioctl.h> 41 #include "uverbs.h" 42 #include "core_priv.h" 43 #include "rdma_core.h" 44 45 void uverbs_uobject_get(struct ib_uobject *uobject) 46 { 47 kref_get(&uobject->ref); 48 } 49 50 static void uverbs_uobject_free(struct kref *ref) 51 { 52 struct ib_uobject *uobj = 53 container_of(ref, struct ib_uobject, ref); 54 55 if (uobj->uapi_object->type_class->needs_kfree_rcu) 56 kfree_rcu(uobj, rcu); 57 else 58 kfree(uobj); 59 } 60 61 void uverbs_uobject_put(struct ib_uobject *uobject) 62 { 63 kref_put(&uobject->ref, uverbs_uobject_free); 64 } 65 66 static int uverbs_try_lock_object(struct ib_uobject *uobj, 67 enum rdma_lookup_mode mode) 68 { 69 /* 70 * When a shared access is required, we use a positive counter. Each 71 * shared access request checks that the value != -1 and increment it. 72 * Exclusive access is required for operations like write or destroy. 73 * In exclusive access mode, we check that the counter is zero (nobody 74 * claimed this object) and we set it to -1. Releasing a shared access 75 * lock is done simply by decreasing the counter. As for exclusive 76 * access locks, since only a single one of them is is allowed 77 * concurrently, setting the counter to zero is enough for releasing 78 * this lock. 79 */ 80 switch (mode) { 81 case UVERBS_LOOKUP_READ: 82 return atomic_fetch_add_unless(&uobj->usecnt, 1, -1) == -1 ? 83 -EBUSY : 0; 84 case UVERBS_LOOKUP_WRITE: 85 /* lock is exclusive */ 86 return atomic_cmpxchg(&uobj->usecnt, 0, -1) == 0 ? 0 : -EBUSY; 87 case UVERBS_LOOKUP_DESTROY: 88 return 0; 89 } 90 return 0; 91 } 92 93 static void assert_uverbs_usecnt(struct ib_uobject *uobj, 94 enum rdma_lookup_mode mode) 95 { 96 #ifdef CONFIG_LOCKDEP 97 switch (mode) { 98 case UVERBS_LOOKUP_READ: 99 WARN_ON(atomic_read(&uobj->usecnt) <= 0); 100 break; 101 case UVERBS_LOOKUP_WRITE: 102 WARN_ON(atomic_read(&uobj->usecnt) != -1); 103 break; 104 case UVERBS_LOOKUP_DESTROY: 105 break; 106 } 107 #endif 108 } 109 110 /* 111 * This must be called with the hw_destroy_rwsem locked for read or write, 112 * also the uobject itself must be locked for write. 113 * 114 * Upon return the HW object is guaranteed to be destroyed. 115 * 116 * For RDMA_REMOVE_ABORT, the hw_destroy_rwsem is not required to be held, 117 * however the type's allocat_commit function cannot have been called and the 118 * uobject cannot be on the uobjects_lists 119 * 120 * For RDMA_REMOVE_DESTROY the caller shold be holding a kref (eg via 121 * rdma_lookup_get_uobject) and the object is left in a state where the caller 122 * needs to call rdma_lookup_put_uobject. 123 * 124 * For all other destroy modes this function internally unlocks the uobject 125 * and consumes the kref on the uobj. 126 */ 127 static int uverbs_destroy_uobject(struct ib_uobject *uobj, 128 enum rdma_remove_reason reason) 129 { 130 struct ib_uverbs_file *ufile = uobj->ufile; 131 unsigned long flags; 132 int ret; 133 134 lockdep_assert_held(&ufile->hw_destroy_rwsem); 135 assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE); 136 137 if (uobj->object) { 138 ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason); 139 if (ret) { 140 if (ib_is_destroy_retryable(ret, reason, uobj)) 141 return ret; 142 143 /* Nothing to be done, dangle the memory and move on */ 144 WARN(true, 145 "ib_uverbs: failed to remove uobject id %d, driver err=%d", 146 uobj->id, ret); 147 } 148 149 uobj->object = NULL; 150 } 151 152 if (reason == RDMA_REMOVE_ABORT) { 153 WARN_ON(!list_empty(&uobj->list)); 154 WARN_ON(!uobj->context); 155 uobj->uapi_object->type_class->alloc_abort(uobj); 156 } 157 158 uobj->context = NULL; 159 160 /* 161 * For DESTROY the usecnt is held write locked, the caller is expected 162 * to put it unlock and put the object when done with it. Only DESTROY 163 * can remove the IDR handle. 164 */ 165 if (reason != RDMA_REMOVE_DESTROY) 166 atomic_set(&uobj->usecnt, 0); 167 else 168 uobj->uapi_object->type_class->remove_handle(uobj); 169 170 if (!list_empty(&uobj->list)) { 171 spin_lock_irqsave(&ufile->uobjects_lock, flags); 172 list_del_init(&uobj->list); 173 spin_unlock_irqrestore(&ufile->uobjects_lock, flags); 174 175 /* 176 * Pairs with the get in rdma_alloc_commit_uobject(), could 177 * destroy uobj. 178 */ 179 uverbs_uobject_put(uobj); 180 } 181 182 /* 183 * When aborting the stack kref remains owned by the core code, and is 184 * not transferred into the type. Pairs with the get in alloc_uobj 185 */ 186 if (reason == RDMA_REMOVE_ABORT) 187 uverbs_uobject_put(uobj); 188 189 return 0; 190 } 191 192 /* 193 * This calls uverbs_destroy_uobject() using the RDMA_REMOVE_DESTROY 194 * sequence. It should only be used from command callbacks. On success the 195 * caller must pair this with rdma_lookup_put_uobject(LOOKUP_WRITE). This 196 * version requires the caller to have already obtained an 197 * LOOKUP_DESTROY uobject kref. 198 */ 199 int uobj_destroy(struct ib_uobject *uobj) 200 { 201 struct ib_uverbs_file *ufile = uobj->ufile; 202 int ret; 203 204 down_read(&ufile->hw_destroy_rwsem); 205 206 ret = uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE); 207 if (ret) 208 goto out_unlock; 209 210 ret = uverbs_destroy_uobject(uobj, RDMA_REMOVE_DESTROY); 211 if (ret) { 212 atomic_set(&uobj->usecnt, 0); 213 goto out_unlock; 214 } 215 216 out_unlock: 217 up_read(&ufile->hw_destroy_rwsem); 218 return ret; 219 } 220 221 /* 222 * uobj_get_destroy destroys the HW object and returns a handle to the uobj 223 * with a NULL object pointer. The caller must pair this with 224 * uverbs_put_destroy. 225 */ 226 struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj, 227 u32 id, 228 const struct uverbs_attr_bundle *attrs) 229 { 230 struct ib_uobject *uobj; 231 int ret; 232 233 uobj = rdma_lookup_get_uobject(obj, attrs->ufile, id, 234 UVERBS_LOOKUP_DESTROY); 235 if (IS_ERR(uobj)) 236 return uobj; 237 238 ret = uobj_destroy(uobj); 239 if (ret) { 240 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY); 241 return ERR_PTR(ret); 242 } 243 244 return uobj; 245 } 246 247 /* 248 * Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success 249 * (negative errno on failure). For use by callers that do not need the uobj. 250 */ 251 int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id, 252 const struct uverbs_attr_bundle *attrs) 253 { 254 struct ib_uobject *uobj; 255 256 uobj = __uobj_get_destroy(obj, id, attrs); 257 if (IS_ERR(uobj)) 258 return PTR_ERR(uobj); 259 260 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE); 261 return 0; 262 } 263 264 /* alloc_uobj must be undone by uverbs_destroy_uobject() */ 265 static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile, 266 const struct uverbs_api_object *obj) 267 { 268 struct ib_uobject *uobj; 269 struct ib_ucontext *ucontext; 270 271 ucontext = ib_uverbs_get_ucontext_file(ufile); 272 if (IS_ERR(ucontext)) 273 return ERR_CAST(ucontext); 274 275 uobj = kzalloc(obj->type_attrs->obj_size, GFP_KERNEL); 276 if (!uobj) 277 return ERR_PTR(-ENOMEM); 278 /* 279 * user_handle should be filled by the handler, 280 * The object is added to the list in the commit stage. 281 */ 282 uobj->ufile = ufile; 283 uobj->context = ucontext; 284 INIT_LIST_HEAD(&uobj->list); 285 uobj->uapi_object = obj; 286 /* 287 * Allocated objects start out as write locked to deny any other 288 * syscalls from accessing them until they are committed. See 289 * rdma_alloc_commit_uobject 290 */ 291 atomic_set(&uobj->usecnt, -1); 292 kref_init(&uobj->ref); 293 294 return uobj; 295 } 296 297 static int idr_add_uobj(struct ib_uobject *uobj) 298 { 299 int ret; 300 301 idr_preload(GFP_KERNEL); 302 spin_lock(&uobj->ufile->idr_lock); 303 304 /* 305 * We start with allocating an idr pointing to NULL. This represents an 306 * object which isn't initialized yet. We'll replace it later on with 307 * the real object once we commit. 308 */ 309 ret = idr_alloc(&uobj->ufile->idr, NULL, 0, 310 min_t(unsigned long, U32_MAX - 1, INT_MAX), GFP_NOWAIT); 311 if (ret >= 0) 312 uobj->id = ret; 313 314 spin_unlock(&uobj->ufile->idr_lock); 315 idr_preload_end(); 316 317 return ret < 0 ? ret : 0; 318 } 319 320 /* Returns the ib_uobject or an error. The caller should check for IS_ERR. */ 321 static struct ib_uobject * 322 lookup_get_idr_uobject(const struct uverbs_api_object *obj, 323 struct ib_uverbs_file *ufile, s64 id, 324 enum rdma_lookup_mode mode) 325 { 326 struct ib_uobject *uobj; 327 unsigned long idrno = id; 328 329 if (id < 0 || id > ULONG_MAX) 330 return ERR_PTR(-EINVAL); 331 332 rcu_read_lock(); 333 /* object won't be released as we're protected in rcu */ 334 uobj = idr_find(&ufile->idr, idrno); 335 if (!uobj) { 336 uobj = ERR_PTR(-ENOENT); 337 goto free; 338 } 339 340 /* 341 * The idr_find is guaranteed to return a pointer to something that 342 * isn't freed yet, or NULL, as the free after idr_remove goes through 343 * kfree_rcu(). However the object may still have been released and 344 * kfree() could be called at any time. 345 */ 346 if (!kref_get_unless_zero(&uobj->ref)) 347 uobj = ERR_PTR(-ENOENT); 348 349 free: 350 rcu_read_unlock(); 351 return uobj; 352 } 353 354 static struct ib_uobject * 355 lookup_get_fd_uobject(const struct uverbs_api_object *obj, 356 struct ib_uverbs_file *ufile, s64 id, 357 enum rdma_lookup_mode mode) 358 { 359 const struct uverbs_obj_fd_type *fd_type; 360 struct file *f; 361 struct ib_uobject *uobject; 362 int fdno = id; 363 364 if (fdno != id) 365 return ERR_PTR(-EINVAL); 366 367 if (mode != UVERBS_LOOKUP_READ) 368 return ERR_PTR(-EOPNOTSUPP); 369 370 if (!obj->type_attrs) 371 return ERR_PTR(-EIO); 372 fd_type = 373 container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); 374 375 f = fget(fdno); 376 if (!f) 377 return ERR_PTR(-EBADF); 378 379 uobject = f->private_data; 380 /* 381 * fget(id) ensures we are not currently running uverbs_close_fd, 382 * and the caller is expected to ensure that uverbs_close_fd is never 383 * done while a call top lookup is possible. 384 */ 385 if (f->f_op != fd_type->fops) { 386 fput(f); 387 return ERR_PTR(-EBADF); 388 } 389 390 uverbs_uobject_get(uobject); 391 return uobject; 392 } 393 394 struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj, 395 struct ib_uverbs_file *ufile, s64 id, 396 enum rdma_lookup_mode mode) 397 { 398 struct ib_uobject *uobj; 399 int ret; 400 401 if (IS_ERR(obj) && PTR_ERR(obj) == -ENOMSG) { 402 /* must be UVERBS_IDR_ANY_OBJECT, see uapi_get_object() */ 403 uobj = lookup_get_idr_uobject(NULL, ufile, id, mode); 404 if (IS_ERR(uobj)) 405 return uobj; 406 } else { 407 if (IS_ERR(obj)) 408 return ERR_PTR(-EINVAL); 409 410 uobj = obj->type_class->lookup_get(obj, ufile, id, mode); 411 if (IS_ERR(uobj)) 412 return uobj; 413 414 if (uobj->uapi_object != obj) { 415 ret = -EINVAL; 416 goto free; 417 } 418 } 419 420 /* 421 * If we have been disassociated block every command except for 422 * DESTROY based commands. 423 */ 424 if (mode != UVERBS_LOOKUP_DESTROY && 425 !srcu_dereference(ufile->device->ib_dev, 426 &ufile->device->disassociate_srcu)) { 427 ret = -EIO; 428 goto free; 429 } 430 431 ret = uverbs_try_lock_object(uobj, mode); 432 if (ret) 433 goto free; 434 435 return uobj; 436 free: 437 uobj->uapi_object->type_class->lookup_put(uobj, mode); 438 uverbs_uobject_put(uobj); 439 return ERR_PTR(ret); 440 } 441 442 static struct ib_uobject * 443 alloc_begin_idr_uobject(const struct uverbs_api_object *obj, 444 struct ib_uverbs_file *ufile) 445 { 446 int ret; 447 struct ib_uobject *uobj; 448 449 uobj = alloc_uobj(ufile, obj); 450 if (IS_ERR(uobj)) 451 return uobj; 452 453 ret = idr_add_uobj(uobj); 454 if (ret) 455 goto uobj_put; 456 457 ret = ib_rdmacg_try_charge(&uobj->cg_obj, uobj->context->device, 458 RDMACG_RESOURCE_HCA_OBJECT); 459 if (ret) 460 goto idr_remove; 461 462 return uobj; 463 464 idr_remove: 465 spin_lock(&ufile->idr_lock); 466 idr_remove(&ufile->idr, uobj->id); 467 spin_unlock(&ufile->idr_lock); 468 uobj_put: 469 uverbs_uobject_put(uobj); 470 return ERR_PTR(ret); 471 } 472 473 static struct ib_uobject * 474 alloc_begin_fd_uobject(const struct uverbs_api_object *obj, 475 struct ib_uverbs_file *ufile) 476 { 477 int new_fd; 478 struct ib_uobject *uobj; 479 480 new_fd = get_unused_fd_flags(O_CLOEXEC); 481 if (new_fd < 0) 482 return ERR_PTR(new_fd); 483 484 uobj = alloc_uobj(ufile, obj); 485 if (IS_ERR(uobj)) { 486 put_unused_fd(new_fd); 487 return uobj; 488 } 489 490 uobj->id = new_fd; 491 uobj->ufile = ufile; 492 493 return uobj; 494 } 495 496 struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, 497 struct ib_uverbs_file *ufile) 498 { 499 struct ib_uobject *ret; 500 501 if (IS_ERR(obj)) 502 return ERR_PTR(-EINVAL); 503 504 /* 505 * The hw_destroy_rwsem is held across the entire object creation and 506 * released during rdma_alloc_commit_uobject or 507 * rdma_alloc_abort_uobject 508 */ 509 if (!down_read_trylock(&ufile->hw_destroy_rwsem)) 510 return ERR_PTR(-EIO); 511 512 ret = obj->type_class->alloc_begin(obj, ufile); 513 if (IS_ERR(ret)) { 514 up_read(&ufile->hw_destroy_rwsem); 515 return ret; 516 } 517 return ret; 518 } 519 520 static void alloc_abort_idr_uobject(struct ib_uobject *uobj) 521 { 522 ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device, 523 RDMACG_RESOURCE_HCA_OBJECT); 524 525 spin_lock(&uobj->ufile->idr_lock); 526 idr_remove(&uobj->ufile->idr, uobj->id); 527 spin_unlock(&uobj->ufile->idr_lock); 528 } 529 530 static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj, 531 enum rdma_remove_reason why) 532 { 533 const struct uverbs_obj_idr_type *idr_type = 534 container_of(uobj->uapi_object->type_attrs, 535 struct uverbs_obj_idr_type, type); 536 int ret = idr_type->destroy_object(uobj, why); 537 538 /* 539 * We can only fail gracefully if the user requested to destroy the 540 * object or when a retry may be called upon an error. 541 * In the rest of the cases, just remove whatever you can. 542 */ 543 if (ib_is_destroy_retryable(ret, why, uobj)) 544 return ret; 545 546 if (why == RDMA_REMOVE_ABORT) 547 return 0; 548 549 ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device, 550 RDMACG_RESOURCE_HCA_OBJECT); 551 552 return 0; 553 } 554 555 static void remove_handle_idr_uobject(struct ib_uobject *uobj) 556 { 557 spin_lock(&uobj->ufile->idr_lock); 558 idr_remove(&uobj->ufile->idr, uobj->id); 559 spin_unlock(&uobj->ufile->idr_lock); 560 /* Matches the kref in alloc_commit_idr_uobject */ 561 uverbs_uobject_put(uobj); 562 } 563 564 static void alloc_abort_fd_uobject(struct ib_uobject *uobj) 565 { 566 put_unused_fd(uobj->id); 567 } 568 569 static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj, 570 enum rdma_remove_reason why) 571 { 572 const struct uverbs_obj_fd_type *fd_type = container_of( 573 uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type); 574 int ret = fd_type->context_closed(uobj, why); 575 576 if (ib_is_destroy_retryable(ret, why, uobj)) 577 return ret; 578 579 return 0; 580 } 581 582 static void remove_handle_fd_uobject(struct ib_uobject *uobj) 583 { 584 } 585 586 static int alloc_commit_idr_uobject(struct ib_uobject *uobj) 587 { 588 struct ib_uverbs_file *ufile = uobj->ufile; 589 590 spin_lock(&ufile->idr_lock); 591 /* 592 * We already allocated this IDR with a NULL object, so 593 * this shouldn't fail. 594 * 595 * NOTE: Once we set the IDR we loose ownership of our kref on uobj. 596 * It will be put by remove_commit_idr_uobject() 597 */ 598 WARN_ON(idr_replace(&ufile->idr, uobj, uobj->id)); 599 spin_unlock(&ufile->idr_lock); 600 601 return 0; 602 } 603 604 static int alloc_commit_fd_uobject(struct ib_uobject *uobj) 605 { 606 const struct uverbs_obj_fd_type *fd_type = container_of( 607 uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type); 608 int fd = uobj->id; 609 struct file *filp; 610 611 /* 612 * The kref for uobj is moved into filp->private data and put in 613 * uverbs_close_fd(). Once alloc_commit() succeeds uverbs_close_fd() 614 * must be guaranteed to be called from the provided fops release 615 * callback. 616 */ 617 filp = anon_inode_getfile(fd_type->name, 618 fd_type->fops, 619 uobj, 620 fd_type->flags); 621 if (IS_ERR(filp)) 622 return PTR_ERR(filp); 623 624 uobj->object = filp; 625 626 /* Matching put will be done in uverbs_close_fd() */ 627 kref_get(&uobj->ufile->ref); 628 629 /* This shouldn't be used anymore. Use the file object instead */ 630 uobj->id = 0; 631 632 /* 633 * NOTE: Once we install the file we loose ownership of our kref on 634 * uobj. It will be put by uverbs_close_fd() 635 */ 636 fd_install(fd, filp); 637 638 return 0; 639 } 640 641 /* 642 * In all cases rdma_alloc_commit_uobject() consumes the kref to uobj and the 643 * caller can no longer assume uobj is valid. If this function fails it 644 * destroys the uboject, including the attached HW object. 645 */ 646 int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj) 647 { 648 struct ib_uverbs_file *ufile = uobj->ufile; 649 int ret; 650 651 /* alloc_commit consumes the uobj kref */ 652 ret = uobj->uapi_object->type_class->alloc_commit(uobj); 653 if (ret) { 654 uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT); 655 up_read(&ufile->hw_destroy_rwsem); 656 return ret; 657 } 658 659 /* kref is held so long as the uobj is on the uobj list. */ 660 uverbs_uobject_get(uobj); 661 spin_lock_irq(&ufile->uobjects_lock); 662 list_add(&uobj->list, &ufile->uobjects); 663 spin_unlock_irq(&ufile->uobjects_lock); 664 665 /* matches atomic_set(-1) in alloc_uobj */ 666 atomic_set(&uobj->usecnt, 0); 667 668 /* Matches the down_read in rdma_alloc_begin_uobject */ 669 up_read(&ufile->hw_destroy_rwsem); 670 671 return 0; 672 } 673 674 /* 675 * This consumes the kref for uobj. It is up to the caller to unwind the HW 676 * object and anything else connected to uobj before calling this. 677 */ 678 void rdma_alloc_abort_uobject(struct ib_uobject *uobj) 679 { 680 struct ib_uverbs_file *ufile = uobj->ufile; 681 682 uobj->object = NULL; 683 uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT); 684 685 /* Matches the down_read in rdma_alloc_begin_uobject */ 686 up_read(&ufile->hw_destroy_rwsem); 687 } 688 689 static void lookup_put_idr_uobject(struct ib_uobject *uobj, 690 enum rdma_lookup_mode mode) 691 { 692 } 693 694 static void lookup_put_fd_uobject(struct ib_uobject *uobj, 695 enum rdma_lookup_mode mode) 696 { 697 struct file *filp = uobj->object; 698 699 WARN_ON(mode != UVERBS_LOOKUP_READ); 700 /* This indirectly calls uverbs_close_fd and free the object */ 701 fput(filp); 702 } 703 704 void rdma_lookup_put_uobject(struct ib_uobject *uobj, 705 enum rdma_lookup_mode mode) 706 { 707 assert_uverbs_usecnt(uobj, mode); 708 uobj->uapi_object->type_class->lookup_put(uobj, mode); 709 /* 710 * In order to unlock an object, either decrease its usecnt for 711 * read access or zero it in case of exclusive access. See 712 * uverbs_try_lock_object for locking schema information. 713 */ 714 switch (mode) { 715 case UVERBS_LOOKUP_READ: 716 atomic_dec(&uobj->usecnt); 717 break; 718 case UVERBS_LOOKUP_WRITE: 719 atomic_set(&uobj->usecnt, 0); 720 break; 721 case UVERBS_LOOKUP_DESTROY: 722 break; 723 } 724 725 /* Pairs with the kref obtained by type->lookup_get */ 726 uverbs_uobject_put(uobj); 727 } 728 729 void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile) 730 { 731 spin_lock_init(&ufile->idr_lock); 732 idr_init(&ufile->idr); 733 } 734 735 void release_ufile_idr_uobject(struct ib_uverbs_file *ufile) 736 { 737 struct ib_uobject *entry; 738 int id; 739 740 /* 741 * At this point uverbs_cleanup_ufile() is guaranteed to have run, and 742 * there are no HW objects left, however the IDR is still populated 743 * with anything that has not been cleaned up by userspace. Since the 744 * kref on ufile is 0, nothing is allowed to call lookup_get. 745 * 746 * This is an optimized equivalent to remove_handle_idr_uobject 747 */ 748 idr_for_each_entry(&ufile->idr, entry, id) { 749 WARN_ON(entry->object); 750 uverbs_uobject_put(entry); 751 } 752 753 idr_destroy(&ufile->idr); 754 } 755 756 const struct uverbs_obj_type_class uverbs_idr_class = { 757 .alloc_begin = alloc_begin_idr_uobject, 758 .lookup_get = lookup_get_idr_uobject, 759 .alloc_commit = alloc_commit_idr_uobject, 760 .alloc_abort = alloc_abort_idr_uobject, 761 .lookup_put = lookup_put_idr_uobject, 762 .destroy_hw = destroy_hw_idr_uobject, 763 .remove_handle = remove_handle_idr_uobject, 764 /* 765 * When we destroy an object, we first just lock it for WRITE and 766 * actually DESTROY it in the finalize stage. So, the problematic 767 * scenario is when we just started the finalize stage of the 768 * destruction (nothing was executed yet). Now, the other thread 769 * fetched the object for READ access, but it didn't lock it yet. 770 * The DESTROY thread continues and starts destroying the object. 771 * When the other thread continue - without the RCU, it would 772 * access freed memory. However, the rcu_read_lock delays the free 773 * until the rcu_read_lock of the READ operation quits. Since the 774 * exclusive lock of the object is still taken by the DESTROY flow, the 775 * READ operation will get -EBUSY and it'll just bail out. 776 */ 777 .needs_kfree_rcu = true, 778 }; 779 EXPORT_SYMBOL(uverbs_idr_class); 780 781 void uverbs_close_fd(struct file *f) 782 { 783 struct ib_uobject *uobj = f->private_data; 784 struct ib_uverbs_file *ufile = uobj->ufile; 785 786 if (down_read_trylock(&ufile->hw_destroy_rwsem)) { 787 /* 788 * lookup_get_fd_uobject holds the kref on the struct file any 789 * time a FD uobj is locked, which prevents this release 790 * method from being invoked. Meaning we can always get the 791 * write lock here, or we have a kernel bug. 792 */ 793 WARN_ON(uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE)); 794 uverbs_destroy_uobject(uobj, RDMA_REMOVE_CLOSE); 795 up_read(&ufile->hw_destroy_rwsem); 796 } 797 798 /* Matches the get in alloc_begin_fd_uobject */ 799 kref_put(&ufile->ref, ib_uverbs_release_file); 800 801 /* Pairs with filp->private_data in alloc_begin_fd_uobject */ 802 uverbs_uobject_put(uobj); 803 } 804 805 /* 806 * Drop the ucontext off the ufile and completely disconnect it from the 807 * ib_device 808 */ 809 static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile, 810 enum rdma_remove_reason reason) 811 { 812 struct ib_ucontext *ucontext = ufile->ucontext; 813 struct ib_device *ib_dev = ucontext->device; 814 int ret; 815 816 /* 817 * If we are closing the FD then the user mmap VMAs must have 818 * already been destroyed as they hold on to the filep, otherwise 819 * they need to be zap'd. 820 */ 821 if (reason == RDMA_REMOVE_DRIVER_REMOVE) { 822 uverbs_user_mmap_disassociate(ufile); 823 if (ib_dev->ops.disassociate_ucontext) 824 ib_dev->ops.disassociate_ucontext(ucontext); 825 } 826 827 ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev, 828 RDMACG_RESOURCE_HCA_HANDLE); 829 830 rdma_restrack_del(&ucontext->res); 831 832 /* 833 * FIXME: Drivers are not permitted to fail dealloc_ucontext, remove 834 * the error return. 835 */ 836 ret = ib_dev->ops.dealloc_ucontext(ucontext); 837 WARN_ON(ret); 838 839 ufile->ucontext = NULL; 840 } 841 842 static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, 843 enum rdma_remove_reason reason) 844 { 845 struct ib_uobject *obj, *next_obj; 846 int ret = -EINVAL; 847 848 /* 849 * This shouldn't run while executing other commands on this 850 * context. Thus, the only thing we should take care of is 851 * releasing a FD while traversing this list. The FD could be 852 * closed and released from the _release fop of this FD. 853 * In order to mitigate this, we add a lock. 854 * We take and release the lock per traversal in order to let 855 * other threads (which might still use the FDs) chance to run. 856 */ 857 list_for_each_entry_safe(obj, next_obj, &ufile->uobjects, list) { 858 /* 859 * if we hit this WARN_ON, that means we are 860 * racing with a lookup_get. 861 */ 862 WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE)); 863 if (!uverbs_destroy_uobject(obj, reason)) 864 ret = 0; 865 else 866 atomic_set(&obj->usecnt, 0); 867 } 868 return ret; 869 } 870 871 /* 872 * Destroy the uncontext and every uobject associated with it. If called with 873 * reason != RDMA_REMOVE_CLOSE this will not return until the destruction has 874 * been completed and ufile->ucontext is NULL. 875 * 876 * This is internally locked and can be called in parallel from multiple 877 * contexts. 878 */ 879 void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, 880 enum rdma_remove_reason reason) 881 { 882 if (reason == RDMA_REMOVE_CLOSE) { 883 /* 884 * During destruction we might trigger something that 885 * synchronously calls release on any file descriptor. For 886 * this reason all paths that come from file_operations 887 * release must use try_lock. They can progress knowing that 888 * there is an ongoing uverbs_destroy_ufile_hw that will clean 889 * up the driver resources. 890 */ 891 if (!mutex_trylock(&ufile->ucontext_lock)) 892 return; 893 894 } else { 895 mutex_lock(&ufile->ucontext_lock); 896 } 897 898 down_write(&ufile->hw_destroy_rwsem); 899 900 /* 901 * If a ucontext was never created then we can't have any uobjects to 902 * cleanup, nothing to do. 903 */ 904 if (!ufile->ucontext) 905 goto done; 906 907 ufile->ucontext->closing = true; 908 ufile->ucontext->cleanup_retryable = true; 909 while (!list_empty(&ufile->uobjects)) 910 if (__uverbs_cleanup_ufile(ufile, reason)) { 911 /* 912 * No entry was cleaned-up successfully during this 913 * iteration 914 */ 915 break; 916 } 917 918 ufile->ucontext->cleanup_retryable = false; 919 if (!list_empty(&ufile->uobjects)) 920 __uverbs_cleanup_ufile(ufile, reason); 921 922 ufile_destroy_ucontext(ufile, reason); 923 924 done: 925 up_write(&ufile->hw_destroy_rwsem); 926 mutex_unlock(&ufile->ucontext_lock); 927 } 928 929 const struct uverbs_obj_type_class uverbs_fd_class = { 930 .alloc_begin = alloc_begin_fd_uobject, 931 .lookup_get = lookup_get_fd_uobject, 932 .alloc_commit = alloc_commit_fd_uobject, 933 .alloc_abort = alloc_abort_fd_uobject, 934 .lookup_put = lookup_put_fd_uobject, 935 .destroy_hw = destroy_hw_fd_uobject, 936 .remove_handle = remove_handle_fd_uobject, 937 .needs_kfree_rcu = false, 938 }; 939 EXPORT_SYMBOL(uverbs_fd_class); 940 941 struct ib_uobject * 942 uverbs_get_uobject_from_file(u16 object_id, 943 struct ib_uverbs_file *ufile, 944 enum uverbs_obj_access access, s64 id) 945 { 946 const struct uverbs_api_object *obj = 947 uapi_get_object(ufile->device->uapi, object_id); 948 949 switch (access) { 950 case UVERBS_ACCESS_READ: 951 return rdma_lookup_get_uobject(obj, ufile, id, 952 UVERBS_LOOKUP_READ); 953 case UVERBS_ACCESS_DESTROY: 954 /* Actual destruction is done inside uverbs_handle_method */ 955 return rdma_lookup_get_uobject(obj, ufile, id, 956 UVERBS_LOOKUP_DESTROY); 957 case UVERBS_ACCESS_WRITE: 958 return rdma_lookup_get_uobject(obj, ufile, id, 959 UVERBS_LOOKUP_WRITE); 960 case UVERBS_ACCESS_NEW: 961 return rdma_alloc_begin_uobject(obj, ufile); 962 default: 963 WARN_ON(true); 964 return ERR_PTR(-EOPNOTSUPP); 965 } 966 } 967 968 int uverbs_finalize_object(struct ib_uobject *uobj, 969 enum uverbs_obj_access access, 970 bool commit) 971 { 972 int ret = 0; 973 974 /* 975 * refcounts should be handled at the object level and not at the 976 * uobject level. Refcounts of the objects themselves are done in 977 * handlers. 978 */ 979 980 switch (access) { 981 case UVERBS_ACCESS_READ: 982 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_READ); 983 break; 984 case UVERBS_ACCESS_WRITE: 985 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE); 986 break; 987 case UVERBS_ACCESS_DESTROY: 988 if (uobj) 989 rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY); 990 break; 991 case UVERBS_ACCESS_NEW: 992 if (commit) 993 ret = rdma_alloc_commit_uobject(uobj); 994 else 995 rdma_alloc_abort_uobject(uobj); 996 break; 997 default: 998 WARN_ON(true); 999 ret = -EOPNOTSUPP; 1000 } 1001 1002 return ret; 1003 } 1004