1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 5 * Copyright (c) 2005 Voltaire, Inc. All rights reserved. 6 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 */ 36 37 #include <linux/module.h> 38 #include <linux/init.h> 39 #include <linux/device.h> 40 #include <linux/err.h> 41 #include <linux/fs.h> 42 #include <linux/poll.h> 43 #include <linux/sched.h> 44 #include <linux/file.h> 45 #include <linux/cdev.h> 46 #include <linux/anon_inodes.h> 47 #include <linux/slab.h> 48 #include <linux/sched/mm.h> 49 50 #include <linux/uaccess.h> 51 52 #include <rdma/ib.h> 53 #include <rdma/uverbs_std_types.h> 54 #include <rdma/rdma_netlink.h> 55 56 #include "uverbs.h" 57 #include "core_priv.h" 58 #include "rdma_core.h" 59 60 MODULE_AUTHOR("Roland Dreier"); 61 MODULE_DESCRIPTION("InfiniBand userspace verbs access"); 62 MODULE_LICENSE("Dual BSD/GPL"); 63 64 enum { 65 IB_UVERBS_MAJOR = 231, 66 IB_UVERBS_BASE_MINOR = 192, 67 IB_UVERBS_MAX_DEVICES = RDMA_MAX_PORTS, 68 IB_UVERBS_NUM_FIXED_MINOR = 32, 69 IB_UVERBS_NUM_DYNAMIC_MINOR = IB_UVERBS_MAX_DEVICES - IB_UVERBS_NUM_FIXED_MINOR, 70 }; 71 72 #define IB_UVERBS_BASE_DEV MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR) 73 74 static dev_t dynamic_uverbs_dev; 75 static struct class *uverbs_class; 76 77 static DEFINE_IDA(uverbs_ida); 78 static int ib_uverbs_add_one(struct ib_device *device); 79 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data); 80 81 /* 82 * Must be called with the ufile->device->disassociate_srcu held, and the lock 83 * must be held until use of the ucontext is finished. 84 */ 85 struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile) 86 { 87 /* 88 * We do not hold the hw_destroy_rwsem lock for this flow, instead 89 * srcu is used. It does not matter if someone races this with 90 * get_context, we get NULL or valid ucontext. 91 */ 92 struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext); 93 94 if (!srcu_dereference(ufile->device->ib_dev, 95 &ufile->device->disassociate_srcu)) 96 return ERR_PTR(-EIO); 97 98 if (!ucontext) 99 return ERR_PTR(-EINVAL); 100 101 return ucontext; 102 } 103 EXPORT_SYMBOL(ib_uverbs_get_ucontext_file); 104 105 int uverbs_dealloc_mw(struct ib_mw *mw) 106 { 107 struct ib_pd *pd = mw->pd; 108 int ret; 109 110 ret = mw->device->ops.dealloc_mw(mw); 111 if (ret) 112 return ret; 113 114 atomic_dec(&pd->usecnt); 115 kfree(mw); 116 return ret; 117 } 118 119 static void ib_uverbs_release_dev(struct device *device) 120 { 121 struct ib_uverbs_device *dev = 122 container_of(device, struct ib_uverbs_device, dev); 123 124 uverbs_destroy_api(dev->uapi); 125 cleanup_srcu_struct(&dev->disassociate_srcu); 126 mutex_destroy(&dev->lists_mutex); 127 mutex_destroy(&dev->xrcd_tree_mutex); 128 kfree(dev); 129 } 130 131 void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file, 132 struct ib_ucq_object *uobj) 133 { 134 struct ib_uverbs_event *evt, *tmp; 135 136 if (ev_file) { 137 spin_lock_irq(&ev_file->ev_queue.lock); 138 list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) { 139 list_del(&evt->list); 140 kfree(evt); 141 } 142 spin_unlock_irq(&ev_file->ev_queue.lock); 143 144 uverbs_uobject_put(&ev_file->uobj); 145 } 146 147 ib_uverbs_release_uevent(&uobj->uevent); 148 } 149 150 void ib_uverbs_release_uevent(struct ib_uevent_object *uobj) 151 { 152 struct ib_uverbs_async_event_file *async_file = uobj->event_file; 153 struct ib_uverbs_event *evt, *tmp; 154 155 if (!async_file) 156 return; 157 158 spin_lock_irq(&async_file->ev_queue.lock); 159 list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) { 160 list_del(&evt->list); 161 kfree(evt); 162 } 163 spin_unlock_irq(&async_file->ev_queue.lock); 164 uverbs_uobject_put(&async_file->uobj); 165 } 166 167 void ib_uverbs_detach_umcast(struct ib_qp *qp, 168 struct ib_uqp_object *uobj) 169 { 170 struct ib_uverbs_mcast_entry *mcast, *tmp; 171 172 list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) { 173 ib_detach_mcast(qp, &mcast->gid, mcast->lid); 174 list_del(&mcast->list); 175 kfree(mcast); 176 } 177 } 178 179 static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev) 180 { 181 complete(&dev->comp); 182 } 183 184 void ib_uverbs_release_file(struct kref *ref) 185 { 186 struct ib_uverbs_file *file = 187 container_of(ref, struct ib_uverbs_file, ref); 188 struct ib_device *ib_dev; 189 int srcu_key; 190 191 release_ufile_idr_uobject(file); 192 193 srcu_key = srcu_read_lock(&file->device->disassociate_srcu); 194 ib_dev = srcu_dereference(file->device->ib_dev, 195 &file->device->disassociate_srcu); 196 if (ib_dev && !ib_dev->ops.disassociate_ucontext) 197 module_put(ib_dev->ops.owner); 198 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); 199 200 if (refcount_dec_and_test(&file->device->refcount)) 201 ib_uverbs_comp_dev(file->device); 202 203 if (file->default_async_file) 204 uverbs_uobject_put(&file->default_async_file->uobj); 205 put_device(&file->device->dev); 206 207 if (file->disassociate_page) 208 __free_pages(file->disassociate_page, 0); 209 mutex_destroy(&file->umap_lock); 210 mutex_destroy(&file->ucontext_lock); 211 kfree(file); 212 } 213 214 static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, 215 struct file *filp, char __user *buf, 216 size_t count, loff_t *pos, 217 size_t eventsz) 218 { 219 struct ib_uverbs_event *event; 220 int ret = 0; 221 222 spin_lock_irq(&ev_queue->lock); 223 224 while (list_empty(&ev_queue->event_list)) { 225 if (ev_queue->is_closed) { 226 spin_unlock_irq(&ev_queue->lock); 227 return -EIO; 228 } 229 230 spin_unlock_irq(&ev_queue->lock); 231 if (filp->f_flags & O_NONBLOCK) 232 return -EAGAIN; 233 234 if (wait_event_interruptible(ev_queue->poll_wait, 235 (!list_empty(&ev_queue->event_list) || 236 ev_queue->is_closed))) 237 return -ERESTARTSYS; 238 239 spin_lock_irq(&ev_queue->lock); 240 } 241 242 event = list_entry(ev_queue->event_list.next, struct ib_uverbs_event, list); 243 244 if (eventsz > count) { 245 ret = -EINVAL; 246 event = NULL; 247 } else { 248 list_del(ev_queue->event_list.next); 249 if (event->counter) { 250 ++(*event->counter); 251 list_del(&event->obj_list); 252 } 253 } 254 255 spin_unlock_irq(&ev_queue->lock); 256 257 if (event) { 258 if (copy_to_user(buf, event, eventsz)) 259 ret = -EFAULT; 260 else 261 ret = eventsz; 262 } 263 264 kfree(event); 265 266 return ret; 267 } 268 269 static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf, 270 size_t count, loff_t *pos) 271 { 272 struct ib_uverbs_async_event_file *file = filp->private_data; 273 274 return ib_uverbs_event_read(&file->ev_queue, filp, buf, count, pos, 275 sizeof(struct ib_uverbs_async_event_desc)); 276 } 277 278 static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf, 279 size_t count, loff_t *pos) 280 { 281 struct ib_uverbs_completion_event_file *comp_ev_file = 282 filp->private_data; 283 284 return ib_uverbs_event_read(&comp_ev_file->ev_queue, filp, buf, count, 285 pos, 286 sizeof(struct ib_uverbs_comp_event_desc)); 287 } 288 289 static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue, 290 struct file *filp, 291 struct poll_table_struct *wait) 292 { 293 __poll_t pollflags = 0; 294 295 poll_wait(filp, &ev_queue->poll_wait, wait); 296 297 spin_lock_irq(&ev_queue->lock); 298 if (!list_empty(&ev_queue->event_list)) 299 pollflags = EPOLLIN | EPOLLRDNORM; 300 else if (ev_queue->is_closed) 301 pollflags = EPOLLERR; 302 spin_unlock_irq(&ev_queue->lock); 303 304 return pollflags; 305 } 306 307 static __poll_t ib_uverbs_async_event_poll(struct file *filp, 308 struct poll_table_struct *wait) 309 { 310 struct ib_uverbs_async_event_file *file = filp->private_data; 311 312 return ib_uverbs_event_poll(&file->ev_queue, filp, wait); 313 } 314 315 static __poll_t ib_uverbs_comp_event_poll(struct file *filp, 316 struct poll_table_struct *wait) 317 { 318 struct ib_uverbs_completion_event_file *comp_ev_file = 319 filp->private_data; 320 321 return ib_uverbs_event_poll(&comp_ev_file->ev_queue, filp, wait); 322 } 323 324 static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on) 325 { 326 struct ib_uverbs_async_event_file *file = filp->private_data; 327 328 return fasync_helper(fd, filp, on, &file->ev_queue.async_queue); 329 } 330 331 static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on) 332 { 333 struct ib_uverbs_completion_event_file *comp_ev_file = 334 filp->private_data; 335 336 return fasync_helper(fd, filp, on, &comp_ev_file->ev_queue.async_queue); 337 } 338 339 const struct file_operations uverbs_event_fops = { 340 .owner = THIS_MODULE, 341 .read = ib_uverbs_comp_event_read, 342 .poll = ib_uverbs_comp_event_poll, 343 .release = uverbs_uobject_fd_release, 344 .fasync = ib_uverbs_comp_event_fasync, 345 .llseek = no_llseek, 346 }; 347 348 const struct file_operations uverbs_async_event_fops = { 349 .owner = THIS_MODULE, 350 .read = ib_uverbs_async_event_read, 351 .poll = ib_uverbs_async_event_poll, 352 .release = uverbs_async_event_release, 353 .fasync = ib_uverbs_async_event_fasync, 354 .llseek = no_llseek, 355 }; 356 357 void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) 358 { 359 struct ib_uverbs_event_queue *ev_queue = cq_context; 360 struct ib_ucq_object *uobj; 361 struct ib_uverbs_event *entry; 362 unsigned long flags; 363 364 if (!ev_queue) 365 return; 366 367 spin_lock_irqsave(&ev_queue->lock, flags); 368 if (ev_queue->is_closed) { 369 spin_unlock_irqrestore(&ev_queue->lock, flags); 370 return; 371 } 372 373 entry = kmalloc(sizeof(*entry), GFP_ATOMIC); 374 if (!entry) { 375 spin_unlock_irqrestore(&ev_queue->lock, flags); 376 return; 377 } 378 379 uobj = cq->uobject; 380 381 entry->desc.comp.cq_handle = cq->uobject->uevent.uobject.user_handle; 382 entry->counter = &uobj->comp_events_reported; 383 384 list_add_tail(&entry->list, &ev_queue->event_list); 385 list_add_tail(&entry->obj_list, &uobj->comp_list); 386 spin_unlock_irqrestore(&ev_queue->lock, flags); 387 388 wake_up_interruptible(&ev_queue->poll_wait); 389 kill_fasync(&ev_queue->async_queue, SIGIO, POLL_IN); 390 } 391 392 void ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file, 393 __u64 element, __u64 event, 394 struct list_head *obj_list, u32 *counter) 395 { 396 struct ib_uverbs_event *entry; 397 unsigned long flags; 398 399 if (!async_file) 400 return; 401 402 spin_lock_irqsave(&async_file->ev_queue.lock, flags); 403 if (async_file->ev_queue.is_closed) { 404 spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); 405 return; 406 } 407 408 entry = kmalloc(sizeof(*entry), GFP_ATOMIC); 409 if (!entry) { 410 spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); 411 return; 412 } 413 414 entry->desc.async.element = element; 415 entry->desc.async.event_type = event; 416 entry->desc.async.reserved = 0; 417 entry->counter = counter; 418 419 list_add_tail(&entry->list, &async_file->ev_queue.event_list); 420 if (obj_list) 421 list_add_tail(&entry->obj_list, obj_list); 422 spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); 423 424 wake_up_interruptible(&async_file->ev_queue.poll_wait); 425 kill_fasync(&async_file->ev_queue.async_queue, SIGIO, POLL_IN); 426 } 427 428 static void uverbs_uobj_event(struct ib_uevent_object *eobj, 429 struct ib_event *event) 430 { 431 ib_uverbs_async_handler(eobj->event_file, 432 eobj->uobject.user_handle, event->event, 433 &eobj->event_list, &eobj->events_reported); 434 } 435 436 void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) 437 { 438 uverbs_uobj_event(&event->element.cq->uobject->uevent, event); 439 } 440 441 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) 442 { 443 /* for XRC target qp's, check that qp is live */ 444 if (!event->element.qp->uobject) 445 return; 446 447 uverbs_uobj_event(&event->element.qp->uobject->uevent, event); 448 } 449 450 void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr) 451 { 452 uverbs_uobj_event(&event->element.wq->uobject->uevent, event); 453 } 454 455 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr) 456 { 457 uverbs_uobj_event(&event->element.srq->uobject->uevent, event); 458 } 459 460 static void ib_uverbs_event_handler(struct ib_event_handler *handler, 461 struct ib_event *event) 462 { 463 ib_uverbs_async_handler( 464 container_of(handler, struct ib_uverbs_async_event_file, 465 event_handler), 466 event->element.port_num, event->event, NULL, NULL); 467 } 468 469 void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue) 470 { 471 spin_lock_init(&ev_queue->lock); 472 INIT_LIST_HEAD(&ev_queue->event_list); 473 init_waitqueue_head(&ev_queue->poll_wait); 474 ev_queue->is_closed = 0; 475 ev_queue->async_queue = NULL; 476 } 477 478 void ib_uverbs_init_async_event_file( 479 struct ib_uverbs_async_event_file *async_file) 480 { 481 struct ib_uverbs_file *uverbs_file = async_file->uobj.ufile; 482 struct ib_device *ib_dev = async_file->uobj.context->device; 483 484 ib_uverbs_init_event_queue(&async_file->ev_queue); 485 486 /* The first async_event_file becomes the default one for the file. */ 487 mutex_lock(&uverbs_file->ucontext_lock); 488 if (!uverbs_file->default_async_file) { 489 /* Pairs with the put in ib_uverbs_release_file */ 490 uverbs_uobject_get(&async_file->uobj); 491 smp_store_release(&uverbs_file->default_async_file, async_file); 492 } 493 mutex_unlock(&uverbs_file->ucontext_lock); 494 495 INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev, 496 ib_uverbs_event_handler); 497 ib_register_event_handler(&async_file->event_handler); 498 } 499 500 static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr, 501 struct ib_uverbs_ex_cmd_hdr *ex_hdr, size_t count, 502 const struct uverbs_api_write_method *method_elm) 503 { 504 if (method_elm->is_ex) { 505 count -= sizeof(*hdr) + sizeof(*ex_hdr); 506 507 if ((hdr->in_words + ex_hdr->provider_in_words) * 8 != count) 508 return -EINVAL; 509 510 if (hdr->in_words * 8 < method_elm->req_size) 511 return -ENOSPC; 512 513 if (ex_hdr->cmd_hdr_reserved) 514 return -EINVAL; 515 516 if (ex_hdr->response) { 517 if (!hdr->out_words && !ex_hdr->provider_out_words) 518 return -EINVAL; 519 520 if (hdr->out_words * 8 < method_elm->resp_size) 521 return -ENOSPC; 522 523 if (!access_ok(u64_to_user_ptr(ex_hdr->response), 524 (hdr->out_words + ex_hdr->provider_out_words) * 8)) 525 return -EFAULT; 526 } else { 527 if (hdr->out_words || ex_hdr->provider_out_words) 528 return -EINVAL; 529 } 530 531 return 0; 532 } 533 534 /* not extended command */ 535 if (hdr->in_words * 4 != count) 536 return -EINVAL; 537 538 if (count < method_elm->req_size + sizeof(hdr)) { 539 /* 540 * rdma-core v18 and v19 have a bug where they send DESTROY_CQ 541 * with a 16 byte write instead of 24. Old kernels didn't 542 * check the size so they allowed this. Now that the size is 543 * checked provide a compatibility work around to not break 544 * those userspaces. 545 */ 546 if (hdr->command == IB_USER_VERBS_CMD_DESTROY_CQ && 547 count == 16) { 548 hdr->in_words = 6; 549 return 0; 550 } 551 return -ENOSPC; 552 } 553 if (hdr->out_words * 4 < method_elm->resp_size) 554 return -ENOSPC; 555 556 return 0; 557 } 558 559 static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, 560 size_t count, loff_t *pos) 561 { 562 struct ib_uverbs_file *file = filp->private_data; 563 const struct uverbs_api_write_method *method_elm; 564 struct uverbs_api *uapi = file->device->uapi; 565 struct ib_uverbs_ex_cmd_hdr ex_hdr; 566 struct ib_uverbs_cmd_hdr hdr; 567 struct uverbs_attr_bundle bundle; 568 int srcu_key; 569 ssize_t ret; 570 571 if (!ib_safe_file_access(filp)) { 572 pr_err_once("uverbs_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n", 573 task_tgid_vnr(current), current->comm); 574 return -EACCES; 575 } 576 577 if (count < sizeof(hdr)) 578 return -EINVAL; 579 580 if (copy_from_user(&hdr, buf, sizeof(hdr))) 581 return -EFAULT; 582 583 method_elm = uapi_get_method(uapi, hdr.command); 584 if (IS_ERR(method_elm)) 585 return PTR_ERR(method_elm); 586 587 if (method_elm->is_ex) { 588 if (count < (sizeof(hdr) + sizeof(ex_hdr))) 589 return -EINVAL; 590 if (copy_from_user(&ex_hdr, buf + sizeof(hdr), sizeof(ex_hdr))) 591 return -EFAULT; 592 } 593 594 ret = verify_hdr(&hdr, &ex_hdr, count, method_elm); 595 if (ret) 596 return ret; 597 598 srcu_key = srcu_read_lock(&file->device->disassociate_srcu); 599 600 buf += sizeof(hdr); 601 602 memset(bundle.attr_present, 0, sizeof(bundle.attr_present)); 603 bundle.ufile = file; 604 bundle.context = NULL; /* only valid if bundle has uobject */ 605 bundle.uobject = NULL; 606 if (!method_elm->is_ex) { 607 size_t in_len = hdr.in_words * 4 - sizeof(hdr); 608 size_t out_len = hdr.out_words * 4; 609 u64 response = 0; 610 611 if (method_elm->has_udata) { 612 bundle.driver_udata.inlen = 613 in_len - method_elm->req_size; 614 in_len = method_elm->req_size; 615 if (bundle.driver_udata.inlen) 616 bundle.driver_udata.inbuf = buf + in_len; 617 else 618 bundle.driver_udata.inbuf = NULL; 619 } else { 620 memset(&bundle.driver_udata, 0, 621 sizeof(bundle.driver_udata)); 622 } 623 624 if (method_elm->has_resp) { 625 /* 626 * The macros check that if has_resp is set 627 * then the command request structure starts 628 * with a '__aligned u64 response' member. 629 */ 630 ret = get_user(response, (const u64 __user *)buf); 631 if (ret) 632 goto out_unlock; 633 634 if (method_elm->has_udata) { 635 bundle.driver_udata.outlen = 636 out_len - method_elm->resp_size; 637 out_len = method_elm->resp_size; 638 if (bundle.driver_udata.outlen) 639 bundle.driver_udata.outbuf = 640 u64_to_user_ptr(response + 641 out_len); 642 else 643 bundle.driver_udata.outbuf = NULL; 644 } 645 } else { 646 bundle.driver_udata.outlen = 0; 647 bundle.driver_udata.outbuf = NULL; 648 } 649 650 ib_uverbs_init_udata_buf_or_null( 651 &bundle.ucore, buf, u64_to_user_ptr(response), 652 in_len, out_len); 653 } else { 654 buf += sizeof(ex_hdr); 655 656 ib_uverbs_init_udata_buf_or_null(&bundle.ucore, buf, 657 u64_to_user_ptr(ex_hdr.response), 658 hdr.in_words * 8, hdr.out_words * 8); 659 660 ib_uverbs_init_udata_buf_or_null( 661 &bundle.driver_udata, buf + bundle.ucore.inlen, 662 u64_to_user_ptr(ex_hdr.response) + bundle.ucore.outlen, 663 ex_hdr.provider_in_words * 8, 664 ex_hdr.provider_out_words * 8); 665 666 } 667 668 ret = method_elm->handler(&bundle); 669 if (bundle.uobject) 670 uverbs_finalize_object(bundle.uobject, UVERBS_ACCESS_NEW, true, 671 !ret, &bundle); 672 out_unlock: 673 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); 674 return (ret) ? : count; 675 } 676 677 static const struct vm_operations_struct rdma_umap_ops; 678 679 static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma) 680 { 681 struct ib_uverbs_file *file = filp->private_data; 682 struct ib_ucontext *ucontext; 683 int ret = 0; 684 int srcu_key; 685 686 srcu_key = srcu_read_lock(&file->device->disassociate_srcu); 687 ucontext = ib_uverbs_get_ucontext_file(file); 688 if (IS_ERR(ucontext)) { 689 ret = PTR_ERR(ucontext); 690 goto out; 691 } 692 vma->vm_ops = &rdma_umap_ops; 693 ret = ucontext->device->ops.mmap(ucontext, vma); 694 out: 695 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); 696 return ret; 697 } 698 699 /* 700 * The VMA has been dup'd, initialize the vm_private_data with a new tracking 701 * struct 702 */ 703 static void rdma_umap_open(struct vm_area_struct *vma) 704 { 705 struct ib_uverbs_file *ufile = vma->vm_file->private_data; 706 struct rdma_umap_priv *opriv = vma->vm_private_data; 707 struct rdma_umap_priv *priv; 708 709 if (!opriv) 710 return; 711 712 /* We are racing with disassociation */ 713 if (!down_read_trylock(&ufile->hw_destroy_rwsem)) 714 goto out_zap; 715 /* 716 * Disassociation already completed, the VMA should already be zapped. 717 */ 718 if (!ufile->ucontext) 719 goto out_unlock; 720 721 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 722 if (!priv) 723 goto out_unlock; 724 rdma_umap_priv_init(priv, vma, opriv->entry); 725 726 up_read(&ufile->hw_destroy_rwsem); 727 return; 728 729 out_unlock: 730 up_read(&ufile->hw_destroy_rwsem); 731 out_zap: 732 /* 733 * We can't allow the VMA to be created with the actual IO pages, that 734 * would break our API contract, and it can't be stopped at this 735 * point, so zap it. 736 */ 737 vma->vm_private_data = NULL; 738 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 739 } 740 741 static void rdma_umap_close(struct vm_area_struct *vma) 742 { 743 struct ib_uverbs_file *ufile = vma->vm_file->private_data; 744 struct rdma_umap_priv *priv = vma->vm_private_data; 745 746 if (!priv) 747 return; 748 749 /* 750 * The vma holds a reference on the struct file that created it, which 751 * in turn means that the ib_uverbs_file is guaranteed to exist at 752 * this point. 753 */ 754 mutex_lock(&ufile->umap_lock); 755 if (priv->entry) 756 rdma_user_mmap_entry_put(priv->entry); 757 758 list_del(&priv->list); 759 mutex_unlock(&ufile->umap_lock); 760 kfree(priv); 761 } 762 763 /* 764 * Once the zap_vma_ptes has been called touches to the VMA will come here and 765 * we return a dummy writable zero page for all the pfns. 766 */ 767 static vm_fault_t rdma_umap_fault(struct vm_fault *vmf) 768 { 769 struct ib_uverbs_file *ufile = vmf->vma->vm_file->private_data; 770 struct rdma_umap_priv *priv = vmf->vma->vm_private_data; 771 vm_fault_t ret = 0; 772 773 if (!priv) 774 return VM_FAULT_SIGBUS; 775 776 /* Read only pages can just use the system zero page. */ 777 if (!(vmf->vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) { 778 vmf->page = ZERO_PAGE(vmf->address); 779 get_page(vmf->page); 780 return 0; 781 } 782 783 mutex_lock(&ufile->umap_lock); 784 if (!ufile->disassociate_page) 785 ufile->disassociate_page = 786 alloc_pages(vmf->gfp_mask | __GFP_ZERO, 0); 787 788 if (ufile->disassociate_page) { 789 /* 790 * This VMA is forced to always be shared so this doesn't have 791 * to worry about COW. 792 */ 793 vmf->page = ufile->disassociate_page; 794 get_page(vmf->page); 795 } else { 796 ret = VM_FAULT_SIGBUS; 797 } 798 mutex_unlock(&ufile->umap_lock); 799 800 return ret; 801 } 802 803 static const struct vm_operations_struct rdma_umap_ops = { 804 .open = rdma_umap_open, 805 .close = rdma_umap_close, 806 .fault = rdma_umap_fault, 807 }; 808 809 void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) 810 { 811 struct rdma_umap_priv *priv, *next_priv; 812 813 lockdep_assert_held(&ufile->hw_destroy_rwsem); 814 815 while (1) { 816 struct mm_struct *mm = NULL; 817 818 /* Get an arbitrary mm pointer that hasn't been cleaned yet */ 819 mutex_lock(&ufile->umap_lock); 820 while (!list_empty(&ufile->umaps)) { 821 int ret; 822 823 priv = list_first_entry(&ufile->umaps, 824 struct rdma_umap_priv, list); 825 mm = priv->vma->vm_mm; 826 ret = mmget_not_zero(mm); 827 if (!ret) { 828 list_del_init(&priv->list); 829 if (priv->entry) { 830 rdma_user_mmap_entry_put(priv->entry); 831 priv->entry = NULL; 832 } 833 mm = NULL; 834 continue; 835 } 836 break; 837 } 838 mutex_unlock(&ufile->umap_lock); 839 if (!mm) 840 return; 841 842 /* 843 * The umap_lock is nested under mmap_lock since it used within 844 * the vma_ops callbacks, so we have to clean the list one mm 845 * at a time to get the lock ordering right. Typically there 846 * will only be one mm, so no big deal. 847 */ 848 mmap_read_lock(mm); 849 mutex_lock(&ufile->umap_lock); 850 list_for_each_entry_safe (priv, next_priv, &ufile->umaps, 851 list) { 852 struct vm_area_struct *vma = priv->vma; 853 854 if (vma->vm_mm != mm) 855 continue; 856 list_del_init(&priv->list); 857 858 zap_vma_ptes(vma, vma->vm_start, 859 vma->vm_end - vma->vm_start); 860 861 if (priv->entry) { 862 rdma_user_mmap_entry_put(priv->entry); 863 priv->entry = NULL; 864 } 865 } 866 mutex_unlock(&ufile->umap_lock); 867 mmap_read_unlock(mm); 868 mmput(mm); 869 } 870 } 871 872 /* 873 * ib_uverbs_open() does not need the BKL: 874 * 875 * - the ib_uverbs_device structures are properly reference counted and 876 * everything else is purely local to the file being created, so 877 * races against other open calls are not a problem; 878 * - there is no ioctl method to race against; 879 * - the open method will either immediately run -ENXIO, or all 880 * required initialization will be done. 881 */ 882 static int ib_uverbs_open(struct inode *inode, struct file *filp) 883 { 884 struct ib_uverbs_device *dev; 885 struct ib_uverbs_file *file; 886 struct ib_device *ib_dev; 887 int ret; 888 int module_dependent; 889 int srcu_key; 890 891 dev = container_of(inode->i_cdev, struct ib_uverbs_device, cdev); 892 if (!refcount_inc_not_zero(&dev->refcount)) 893 return -ENXIO; 894 895 get_device(&dev->dev); 896 srcu_key = srcu_read_lock(&dev->disassociate_srcu); 897 mutex_lock(&dev->lists_mutex); 898 ib_dev = srcu_dereference(dev->ib_dev, 899 &dev->disassociate_srcu); 900 if (!ib_dev) { 901 ret = -EIO; 902 goto err; 903 } 904 905 if (!rdma_dev_access_netns(ib_dev, current->nsproxy->net_ns)) { 906 ret = -EPERM; 907 goto err; 908 } 909 910 /* In case IB device supports disassociate ucontext, there is no hard 911 * dependency between uverbs device and its low level device. 912 */ 913 module_dependent = !(ib_dev->ops.disassociate_ucontext); 914 915 if (module_dependent) { 916 if (!try_module_get(ib_dev->ops.owner)) { 917 ret = -ENODEV; 918 goto err; 919 } 920 } 921 922 file = kzalloc(sizeof(*file), GFP_KERNEL); 923 if (!file) { 924 ret = -ENOMEM; 925 if (module_dependent) 926 goto err_module; 927 928 goto err; 929 } 930 931 file->device = dev; 932 kref_init(&file->ref); 933 mutex_init(&file->ucontext_lock); 934 935 spin_lock_init(&file->uobjects_lock); 936 INIT_LIST_HEAD(&file->uobjects); 937 init_rwsem(&file->hw_destroy_rwsem); 938 mutex_init(&file->umap_lock); 939 INIT_LIST_HEAD(&file->umaps); 940 941 filp->private_data = file; 942 list_add_tail(&file->list, &dev->uverbs_file_list); 943 mutex_unlock(&dev->lists_mutex); 944 srcu_read_unlock(&dev->disassociate_srcu, srcu_key); 945 946 setup_ufile_idr_uobject(file); 947 948 return stream_open(inode, filp); 949 950 err_module: 951 module_put(ib_dev->ops.owner); 952 953 err: 954 mutex_unlock(&dev->lists_mutex); 955 srcu_read_unlock(&dev->disassociate_srcu, srcu_key); 956 if (refcount_dec_and_test(&dev->refcount)) 957 ib_uverbs_comp_dev(dev); 958 959 put_device(&dev->dev); 960 return ret; 961 } 962 963 static int ib_uverbs_close(struct inode *inode, struct file *filp) 964 { 965 struct ib_uverbs_file *file = filp->private_data; 966 967 uverbs_destroy_ufile_hw(file, RDMA_REMOVE_CLOSE); 968 969 mutex_lock(&file->device->lists_mutex); 970 list_del_init(&file->list); 971 mutex_unlock(&file->device->lists_mutex); 972 973 kref_put(&file->ref, ib_uverbs_release_file); 974 975 return 0; 976 } 977 978 static const struct file_operations uverbs_fops = { 979 .owner = THIS_MODULE, 980 .write = ib_uverbs_write, 981 .open = ib_uverbs_open, 982 .release = ib_uverbs_close, 983 .llseek = no_llseek, 984 .unlocked_ioctl = ib_uverbs_ioctl, 985 .compat_ioctl = compat_ptr_ioctl, 986 }; 987 988 static const struct file_operations uverbs_mmap_fops = { 989 .owner = THIS_MODULE, 990 .write = ib_uverbs_write, 991 .mmap = ib_uverbs_mmap, 992 .open = ib_uverbs_open, 993 .release = ib_uverbs_close, 994 .llseek = no_llseek, 995 .unlocked_ioctl = ib_uverbs_ioctl, 996 .compat_ioctl = compat_ptr_ioctl, 997 }; 998 999 static int ib_uverbs_get_nl_info(struct ib_device *ibdev, void *client_data, 1000 struct ib_client_nl_info *res) 1001 { 1002 struct ib_uverbs_device *uverbs_dev = client_data; 1003 int ret; 1004 1005 if (res->port != -1) 1006 return -EINVAL; 1007 1008 res->abi = ibdev->ops.uverbs_abi_ver; 1009 res->cdev = &uverbs_dev->dev; 1010 1011 /* 1012 * To support DRIVER_ID binding in userspace some of the driver need 1013 * upgrading to expose their PCI dependent revision information 1014 * through get_context instead of relying on modalias matching. When 1015 * the drivers are fixed they can drop this flag. 1016 */ 1017 if (!ibdev->ops.uverbs_no_driver_id_binding) { 1018 ret = nla_put_u32(res->nl_msg, RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID, 1019 ibdev->ops.driver_id); 1020 if (ret) 1021 return ret; 1022 } 1023 return 0; 1024 } 1025 1026 static struct ib_client uverbs_client = { 1027 .name = "uverbs", 1028 .no_kverbs_req = true, 1029 .add = ib_uverbs_add_one, 1030 .remove = ib_uverbs_remove_one, 1031 .get_nl_info = ib_uverbs_get_nl_info, 1032 }; 1033 MODULE_ALIAS_RDMA_CLIENT("uverbs"); 1034 1035 static ssize_t ibdev_show(struct device *device, struct device_attribute *attr, 1036 char *buf) 1037 { 1038 struct ib_uverbs_device *dev = 1039 container_of(device, struct ib_uverbs_device, dev); 1040 int ret = -ENODEV; 1041 int srcu_key; 1042 struct ib_device *ib_dev; 1043 1044 srcu_key = srcu_read_lock(&dev->disassociate_srcu); 1045 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu); 1046 if (ib_dev) 1047 ret = sysfs_emit(buf, "%s\n", dev_name(&ib_dev->dev)); 1048 srcu_read_unlock(&dev->disassociate_srcu, srcu_key); 1049 1050 return ret; 1051 } 1052 static DEVICE_ATTR_RO(ibdev); 1053 1054 static ssize_t abi_version_show(struct device *device, 1055 struct device_attribute *attr, char *buf) 1056 { 1057 struct ib_uverbs_device *dev = 1058 container_of(device, struct ib_uverbs_device, dev); 1059 int ret = -ENODEV; 1060 int srcu_key; 1061 struct ib_device *ib_dev; 1062 1063 srcu_key = srcu_read_lock(&dev->disassociate_srcu); 1064 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu); 1065 if (ib_dev) 1066 ret = sysfs_emit(buf, "%u\n", ib_dev->ops.uverbs_abi_ver); 1067 srcu_read_unlock(&dev->disassociate_srcu, srcu_key); 1068 1069 return ret; 1070 } 1071 static DEVICE_ATTR_RO(abi_version); 1072 1073 static struct attribute *ib_dev_attrs[] = { 1074 &dev_attr_abi_version.attr, 1075 &dev_attr_ibdev.attr, 1076 NULL, 1077 }; 1078 1079 static const struct attribute_group dev_attr_group = { 1080 .attrs = ib_dev_attrs, 1081 }; 1082 1083 static CLASS_ATTR_STRING(abi_version, S_IRUGO, 1084 __stringify(IB_USER_VERBS_ABI_VERSION)); 1085 1086 static int ib_uverbs_create_uapi(struct ib_device *device, 1087 struct ib_uverbs_device *uverbs_dev) 1088 { 1089 struct uverbs_api *uapi; 1090 1091 uapi = uverbs_alloc_api(device); 1092 if (IS_ERR(uapi)) 1093 return PTR_ERR(uapi); 1094 1095 uverbs_dev->uapi = uapi; 1096 return 0; 1097 } 1098 1099 static int ib_uverbs_add_one(struct ib_device *device) 1100 { 1101 int devnum; 1102 dev_t base; 1103 struct ib_uverbs_device *uverbs_dev; 1104 int ret; 1105 1106 if (!device->ops.alloc_ucontext) 1107 return -EOPNOTSUPP; 1108 1109 uverbs_dev = kzalloc(sizeof(*uverbs_dev), GFP_KERNEL); 1110 if (!uverbs_dev) 1111 return -ENOMEM; 1112 1113 ret = init_srcu_struct(&uverbs_dev->disassociate_srcu); 1114 if (ret) { 1115 kfree(uverbs_dev); 1116 return -ENOMEM; 1117 } 1118 1119 device_initialize(&uverbs_dev->dev); 1120 uverbs_dev->dev.class = uverbs_class; 1121 uverbs_dev->dev.parent = device->dev.parent; 1122 uverbs_dev->dev.release = ib_uverbs_release_dev; 1123 uverbs_dev->groups[0] = &dev_attr_group; 1124 uverbs_dev->dev.groups = uverbs_dev->groups; 1125 refcount_set(&uverbs_dev->refcount, 1); 1126 init_completion(&uverbs_dev->comp); 1127 uverbs_dev->xrcd_tree = RB_ROOT; 1128 mutex_init(&uverbs_dev->xrcd_tree_mutex); 1129 mutex_init(&uverbs_dev->lists_mutex); 1130 INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list); 1131 rcu_assign_pointer(uverbs_dev->ib_dev, device); 1132 uverbs_dev->num_comp_vectors = device->num_comp_vectors; 1133 1134 devnum = ida_alloc_max(&uverbs_ida, IB_UVERBS_MAX_DEVICES - 1, 1135 GFP_KERNEL); 1136 if (devnum < 0) { 1137 ret = -ENOMEM; 1138 goto err; 1139 } 1140 uverbs_dev->devnum = devnum; 1141 if (devnum >= IB_UVERBS_NUM_FIXED_MINOR) 1142 base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR; 1143 else 1144 base = IB_UVERBS_BASE_DEV + devnum; 1145 1146 ret = ib_uverbs_create_uapi(device, uverbs_dev); 1147 if (ret) 1148 goto err_uapi; 1149 1150 uverbs_dev->dev.devt = base; 1151 dev_set_name(&uverbs_dev->dev, "uverbs%d", uverbs_dev->devnum); 1152 1153 cdev_init(&uverbs_dev->cdev, 1154 device->ops.mmap ? &uverbs_mmap_fops : &uverbs_fops); 1155 uverbs_dev->cdev.owner = THIS_MODULE; 1156 1157 ret = cdev_device_add(&uverbs_dev->cdev, &uverbs_dev->dev); 1158 if (ret) 1159 goto err_uapi; 1160 1161 ib_set_client_data(device, &uverbs_client, uverbs_dev); 1162 return 0; 1163 1164 err_uapi: 1165 ida_free(&uverbs_ida, devnum); 1166 err: 1167 if (refcount_dec_and_test(&uverbs_dev->refcount)) 1168 ib_uverbs_comp_dev(uverbs_dev); 1169 wait_for_completion(&uverbs_dev->comp); 1170 put_device(&uverbs_dev->dev); 1171 return ret; 1172 } 1173 1174 static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, 1175 struct ib_device *ib_dev) 1176 { 1177 struct ib_uverbs_file *file; 1178 1179 /* Pending running commands to terminate */ 1180 uverbs_disassociate_api_pre(uverbs_dev); 1181 1182 mutex_lock(&uverbs_dev->lists_mutex); 1183 while (!list_empty(&uverbs_dev->uverbs_file_list)) { 1184 file = list_first_entry(&uverbs_dev->uverbs_file_list, 1185 struct ib_uverbs_file, list); 1186 list_del_init(&file->list); 1187 kref_get(&file->ref); 1188 1189 /* We must release the mutex before going ahead and calling 1190 * uverbs_cleanup_ufile, as it might end up indirectly calling 1191 * uverbs_close, for example due to freeing the resources (e.g 1192 * mmput). 1193 */ 1194 mutex_unlock(&uverbs_dev->lists_mutex); 1195 1196 uverbs_destroy_ufile_hw(file, RDMA_REMOVE_DRIVER_REMOVE); 1197 kref_put(&file->ref, ib_uverbs_release_file); 1198 1199 mutex_lock(&uverbs_dev->lists_mutex); 1200 } 1201 mutex_unlock(&uverbs_dev->lists_mutex); 1202 1203 uverbs_disassociate_api(uverbs_dev->uapi); 1204 } 1205 1206 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data) 1207 { 1208 struct ib_uverbs_device *uverbs_dev = client_data; 1209 int wait_clients = 1; 1210 1211 cdev_device_del(&uverbs_dev->cdev, &uverbs_dev->dev); 1212 ida_free(&uverbs_ida, uverbs_dev->devnum); 1213 1214 if (device->ops.disassociate_ucontext) { 1215 /* We disassociate HW resources and immediately return. 1216 * Userspace will see a EIO errno for all future access. 1217 * Upon returning, ib_device may be freed internally and is not 1218 * valid any more. 1219 * uverbs_device is still available until all clients close 1220 * their files, then the uverbs device ref count will be zero 1221 * and its resources will be freed. 1222 * Note: At this point no more files can be opened since the 1223 * cdev was deleted, however active clients can still issue 1224 * commands and close their open files. 1225 */ 1226 ib_uverbs_free_hw_resources(uverbs_dev, device); 1227 wait_clients = 0; 1228 } 1229 1230 if (refcount_dec_and_test(&uverbs_dev->refcount)) 1231 ib_uverbs_comp_dev(uverbs_dev); 1232 if (wait_clients) 1233 wait_for_completion(&uverbs_dev->comp); 1234 1235 put_device(&uverbs_dev->dev); 1236 } 1237 1238 static char *uverbs_devnode(const struct device *dev, umode_t *mode) 1239 { 1240 if (mode) 1241 *mode = 0666; 1242 return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev)); 1243 } 1244 1245 static int __init ib_uverbs_init(void) 1246 { 1247 int ret; 1248 1249 ret = register_chrdev_region(IB_UVERBS_BASE_DEV, 1250 IB_UVERBS_NUM_FIXED_MINOR, 1251 "infiniband_verbs"); 1252 if (ret) { 1253 pr_err("user_verbs: couldn't register device number\n"); 1254 goto out; 1255 } 1256 1257 ret = alloc_chrdev_region(&dynamic_uverbs_dev, 0, 1258 IB_UVERBS_NUM_DYNAMIC_MINOR, 1259 "infiniband_verbs"); 1260 if (ret) { 1261 pr_err("couldn't register dynamic device number\n"); 1262 goto out_alloc; 1263 } 1264 1265 uverbs_class = class_create("infiniband_verbs"); 1266 if (IS_ERR(uverbs_class)) { 1267 ret = PTR_ERR(uverbs_class); 1268 pr_err("user_verbs: couldn't create class infiniband_verbs\n"); 1269 goto out_chrdev; 1270 } 1271 1272 uverbs_class->devnode = uverbs_devnode; 1273 1274 ret = class_create_file(uverbs_class, &class_attr_abi_version.attr); 1275 if (ret) { 1276 pr_err("user_verbs: couldn't create abi_version attribute\n"); 1277 goto out_class; 1278 } 1279 1280 ret = ib_register_client(&uverbs_client); 1281 if (ret) { 1282 pr_err("user_verbs: couldn't register client\n"); 1283 goto out_class; 1284 } 1285 1286 return 0; 1287 1288 out_class: 1289 class_destroy(uverbs_class); 1290 1291 out_chrdev: 1292 unregister_chrdev_region(dynamic_uverbs_dev, 1293 IB_UVERBS_NUM_DYNAMIC_MINOR); 1294 1295 out_alloc: 1296 unregister_chrdev_region(IB_UVERBS_BASE_DEV, 1297 IB_UVERBS_NUM_FIXED_MINOR); 1298 1299 out: 1300 return ret; 1301 } 1302 1303 static void __exit ib_uverbs_cleanup(void) 1304 { 1305 ib_unregister_client(&uverbs_client); 1306 class_destroy(uverbs_class); 1307 unregister_chrdev_region(IB_UVERBS_BASE_DEV, 1308 IB_UVERBS_NUM_FIXED_MINOR); 1309 unregister_chrdev_region(dynamic_uverbs_dev, 1310 IB_UVERBS_NUM_DYNAMIC_MINOR); 1311 mmu_notifier_synchronize(); 1312 } 1313 1314 module_init(ib_uverbs_init); 1315 module_exit(ib_uverbs_cleanup); 1316