1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <linux/uaccess.h> 42 43 #include <rdma/uverbs_types.h> 44 #include <rdma/uverbs_std_types.h> 45 #include "rdma_core.h" 46 47 #include "uverbs.h" 48 #include "core_priv.h" 49 50 /* 51 * Copy a response to userspace. If the provided 'resp' is larger than the 52 * user buffer it is silently truncated. If the user provided a larger buffer 53 * then the trailing portion is zero filled. 54 * 55 * These semantics are intended to support future extension of the output 56 * structures. 57 */ 58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp, 59 size_t resp_len) 60 { 61 int ret; 62 63 if (copy_to_user(attrs->ucore.outbuf, resp, 64 min(attrs->ucore.outlen, resp_len))) 65 return -EFAULT; 66 67 if (resp_len < attrs->ucore.outlen) { 68 /* 69 * Zero fill any extra memory that user 70 * space might have provided. 71 */ 72 ret = clear_user(attrs->ucore.outbuf + resp_len, 73 attrs->ucore.outlen - resp_len); 74 if (ret) 75 return -EFAULT; 76 } 77 78 return 0; 79 } 80 81 /* 82 * Copy a request from userspace. If the provided 'req' is larger than the 83 * user buffer then the user buffer is zero extended into the 'req'. If 'req' 84 * is smaller than the user buffer then the uncopied bytes in the user buffer 85 * must be zero. 86 */ 87 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req, 88 size_t req_len) 89 { 90 if (copy_from_user(req, attrs->ucore.inbuf, 91 min(attrs->ucore.inlen, req_len))) 92 return -EFAULT; 93 94 if (attrs->ucore.inlen < req_len) { 95 memset(req + attrs->ucore.inlen, 0, 96 req_len - attrs->ucore.inlen); 97 } else if (attrs->ucore.inlen > req_len) { 98 if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len, 99 attrs->ucore.inlen - req_len)) 100 return -EOPNOTSUPP; 101 } 102 return 0; 103 } 104 105 /* 106 * Generate the value for the 'response_length' protocol used by write_ex. 107 * This is the number of bytes the kernel actually wrote. Userspace can use 108 * this to detect what structure members in the response the kernel 109 * understood. 110 */ 111 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs, 112 size_t resp_len) 113 { 114 return min_t(size_t, attrs->ucore.outlen, resp_len); 115 } 116 117 /* 118 * The iterator version of the request interface is for handlers that need to 119 * step over a flex array at the end of a command header. 120 */ 121 struct uverbs_req_iter { 122 const void __user *cur; 123 const void __user *end; 124 }; 125 126 static int uverbs_request_start(struct uverbs_attr_bundle *attrs, 127 struct uverbs_req_iter *iter, 128 void *req, 129 size_t req_len) 130 { 131 if (attrs->ucore.inlen < req_len) 132 return -ENOSPC; 133 134 if (copy_from_user(req, attrs->ucore.inbuf, req_len)) 135 return -EFAULT; 136 137 iter->cur = attrs->ucore.inbuf + req_len; 138 iter->end = attrs->ucore.inbuf + attrs->ucore.inlen; 139 return 0; 140 } 141 142 static int uverbs_request_next(struct uverbs_req_iter *iter, void *val, 143 size_t len) 144 { 145 if (iter->cur + len > iter->end) 146 return -ENOSPC; 147 148 if (copy_from_user(val, iter->cur, len)) 149 return -EFAULT; 150 151 iter->cur += len; 152 return 0; 153 } 154 155 static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter, 156 size_t len) 157 { 158 const void __user *res = iter->cur; 159 160 if (iter->cur + len > iter->end) 161 return ERR_PTR(-ENOSPC); 162 iter->cur += len; 163 return res; 164 } 165 166 static int uverbs_request_finish(struct uverbs_req_iter *iter) 167 { 168 if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur)) 169 return -EOPNOTSUPP; 170 return 0; 171 } 172 173 static struct ib_uverbs_completion_event_file * 174 _ib_uverbs_lookup_comp_file(s32 fd, const struct uverbs_attr_bundle *attrs) 175 { 176 struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL, 177 fd, attrs); 178 179 if (IS_ERR(uobj)) 180 return (void *)uobj; 181 182 uverbs_uobject_get(uobj); 183 uobj_put_read(uobj); 184 185 return container_of(uobj, struct ib_uverbs_completion_event_file, 186 uobj); 187 } 188 #define ib_uverbs_lookup_comp_file(_fd, _ufile) \ 189 _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile) 190 191 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) 192 { 193 struct ib_uverbs_file *file = attrs->ufile; 194 struct ib_uverbs_get_context cmd; 195 struct ib_uverbs_get_context_resp resp; 196 struct ib_ucontext *ucontext; 197 struct file *filp; 198 struct ib_rdmacg_object cg_obj; 199 struct ib_device *ib_dev; 200 int ret; 201 202 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 203 if (ret) 204 return ret; 205 206 mutex_lock(&file->ucontext_lock); 207 ib_dev = srcu_dereference(file->device->ib_dev, 208 &file->device->disassociate_srcu); 209 if (!ib_dev) { 210 ret = -EIO; 211 goto err; 212 } 213 214 if (file->ucontext) { 215 ret = -EINVAL; 216 goto err; 217 } 218 219 ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 220 if (ret) 221 goto err; 222 223 ucontext = ib_dev->ops.alloc_ucontext(ib_dev, &attrs->driver_udata); 224 if (IS_ERR(ucontext)) { 225 ret = PTR_ERR(ucontext); 226 goto err_alloc; 227 } 228 229 ucontext->device = ib_dev; 230 ucontext->cg_obj = cg_obj; 231 /* ufile is required when some objects are released */ 232 ucontext->ufile = file; 233 234 ucontext->closing = false; 235 ucontext->cleanup_retryable = false; 236 237 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 238 mutex_init(&ucontext->per_mm_list_lock); 239 INIT_LIST_HEAD(&ucontext->per_mm_list); 240 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) 241 ucontext->invalidate_range = NULL; 242 243 #endif 244 245 resp.num_comp_vectors = file->device->num_comp_vectors; 246 247 ret = get_unused_fd_flags(O_CLOEXEC); 248 if (ret < 0) 249 goto err_free; 250 resp.async_fd = ret; 251 252 filp = ib_uverbs_alloc_async_event_file(file, ib_dev); 253 if (IS_ERR(filp)) { 254 ret = PTR_ERR(filp); 255 goto err_fd; 256 } 257 258 ret = uverbs_response(attrs, &resp, sizeof(resp)); 259 if (ret) 260 goto err_file; 261 262 fd_install(resp.async_fd, filp); 263 264 ucontext->res.type = RDMA_RESTRACK_CTX; 265 rdma_restrack_uadd(&ucontext->res); 266 267 /* 268 * Make sure that ib_uverbs_get_ucontext() sees the pointer update 269 * only after all writes to setup the ucontext have completed 270 */ 271 smp_store_release(&file->ucontext, ucontext); 272 273 mutex_unlock(&file->ucontext_lock); 274 275 return 0; 276 277 err_file: 278 ib_uverbs_free_async_event_file(file); 279 fput(filp); 280 281 err_fd: 282 put_unused_fd(resp.async_fd); 283 284 err_free: 285 ib_dev->ops.dealloc_ucontext(ucontext); 286 287 err_alloc: 288 ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 289 290 err: 291 mutex_unlock(&file->ucontext_lock); 292 return ret; 293 } 294 295 static void copy_query_dev_fields(struct ib_ucontext *ucontext, 296 struct ib_uverbs_query_device_resp *resp, 297 struct ib_device_attr *attr) 298 { 299 struct ib_device *ib_dev = ucontext->device; 300 301 resp->fw_ver = attr->fw_ver; 302 resp->node_guid = ib_dev->node_guid; 303 resp->sys_image_guid = attr->sys_image_guid; 304 resp->max_mr_size = attr->max_mr_size; 305 resp->page_size_cap = attr->page_size_cap; 306 resp->vendor_id = attr->vendor_id; 307 resp->vendor_part_id = attr->vendor_part_id; 308 resp->hw_ver = attr->hw_ver; 309 resp->max_qp = attr->max_qp; 310 resp->max_qp_wr = attr->max_qp_wr; 311 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags); 312 resp->max_sge = min(attr->max_send_sge, attr->max_recv_sge); 313 resp->max_sge_rd = attr->max_sge_rd; 314 resp->max_cq = attr->max_cq; 315 resp->max_cqe = attr->max_cqe; 316 resp->max_mr = attr->max_mr; 317 resp->max_pd = attr->max_pd; 318 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 319 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 320 resp->max_res_rd_atom = attr->max_res_rd_atom; 321 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 322 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 323 resp->atomic_cap = attr->atomic_cap; 324 resp->max_ee = attr->max_ee; 325 resp->max_rdd = attr->max_rdd; 326 resp->max_mw = attr->max_mw; 327 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 328 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 329 resp->max_mcast_grp = attr->max_mcast_grp; 330 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 331 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 332 resp->max_ah = attr->max_ah; 333 resp->max_fmr = attr->max_fmr; 334 resp->max_map_per_fmr = attr->max_map_per_fmr; 335 resp->max_srq = attr->max_srq; 336 resp->max_srq_wr = attr->max_srq_wr; 337 resp->max_srq_sge = attr->max_srq_sge; 338 resp->max_pkeys = attr->max_pkeys; 339 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 340 resp->phys_port_cnt = ib_dev->phys_port_cnt; 341 } 342 343 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs) 344 { 345 struct ib_uverbs_query_device cmd; 346 struct ib_uverbs_query_device_resp resp; 347 struct ib_ucontext *ucontext; 348 int ret; 349 350 ucontext = ib_uverbs_get_ucontext(attrs); 351 if (IS_ERR(ucontext)) 352 return PTR_ERR(ucontext); 353 354 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 355 if (ret) 356 return ret; 357 358 memset(&resp, 0, sizeof resp); 359 copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs); 360 361 return uverbs_response(attrs, &resp, sizeof(resp)); 362 } 363 364 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs) 365 { 366 struct ib_uverbs_query_port cmd; 367 struct ib_uverbs_query_port_resp resp; 368 struct ib_port_attr attr; 369 int ret; 370 struct ib_ucontext *ucontext; 371 struct ib_device *ib_dev; 372 373 ucontext = ib_uverbs_get_ucontext(attrs); 374 if (IS_ERR(ucontext)) 375 return PTR_ERR(ucontext); 376 ib_dev = ucontext->device; 377 378 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 379 if (ret) 380 return ret; 381 382 ret = ib_query_port(ib_dev, cmd.port_num, &attr); 383 if (ret) 384 return ret; 385 386 memset(&resp, 0, sizeof resp); 387 copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num); 388 389 return uverbs_response(attrs, &resp, sizeof(resp)); 390 } 391 392 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) 393 { 394 struct ib_uverbs_alloc_pd cmd; 395 struct ib_uverbs_alloc_pd_resp resp; 396 struct ib_uobject *uobj; 397 struct ib_pd *pd; 398 int ret; 399 struct ib_device *ib_dev; 400 401 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 402 if (ret) 403 return ret; 404 405 uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev); 406 if (IS_ERR(uobj)) 407 return PTR_ERR(uobj); 408 409 pd = ib_dev->ops.alloc_pd(ib_dev, uobj->context, &attrs->driver_udata); 410 if (IS_ERR(pd)) { 411 ret = PTR_ERR(pd); 412 goto err; 413 } 414 415 pd->device = ib_dev; 416 pd->uobject = uobj; 417 pd->__internal_mr = NULL; 418 atomic_set(&pd->usecnt, 0); 419 420 uobj->object = pd; 421 memset(&resp, 0, sizeof resp); 422 resp.pd_handle = uobj->id; 423 pd->res.type = RDMA_RESTRACK_PD; 424 rdma_restrack_uadd(&pd->res); 425 426 ret = uverbs_response(attrs, &resp, sizeof(resp)); 427 if (ret) 428 goto err_copy; 429 430 return uobj_alloc_commit(uobj); 431 432 err_copy: 433 ib_dealloc_pd(pd); 434 435 err: 436 uobj_alloc_abort(uobj); 437 return ret; 438 } 439 440 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs) 441 { 442 struct ib_uverbs_dealloc_pd cmd; 443 int ret; 444 445 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 446 if (ret) 447 return ret; 448 449 return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 450 } 451 452 struct xrcd_table_entry { 453 struct rb_node node; 454 struct ib_xrcd *xrcd; 455 struct inode *inode; 456 }; 457 458 static int xrcd_table_insert(struct ib_uverbs_device *dev, 459 struct inode *inode, 460 struct ib_xrcd *xrcd) 461 { 462 struct xrcd_table_entry *entry, *scan; 463 struct rb_node **p = &dev->xrcd_tree.rb_node; 464 struct rb_node *parent = NULL; 465 466 entry = kmalloc(sizeof *entry, GFP_KERNEL); 467 if (!entry) 468 return -ENOMEM; 469 470 entry->xrcd = xrcd; 471 entry->inode = inode; 472 473 while (*p) { 474 parent = *p; 475 scan = rb_entry(parent, struct xrcd_table_entry, node); 476 477 if (inode < scan->inode) { 478 p = &(*p)->rb_left; 479 } else if (inode > scan->inode) { 480 p = &(*p)->rb_right; 481 } else { 482 kfree(entry); 483 return -EEXIST; 484 } 485 } 486 487 rb_link_node(&entry->node, parent, p); 488 rb_insert_color(&entry->node, &dev->xrcd_tree); 489 igrab(inode); 490 return 0; 491 } 492 493 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 494 struct inode *inode) 495 { 496 struct xrcd_table_entry *entry; 497 struct rb_node *p = dev->xrcd_tree.rb_node; 498 499 while (p) { 500 entry = rb_entry(p, struct xrcd_table_entry, node); 501 502 if (inode < entry->inode) 503 p = p->rb_left; 504 else if (inode > entry->inode) 505 p = p->rb_right; 506 else 507 return entry; 508 } 509 510 return NULL; 511 } 512 513 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 514 { 515 struct xrcd_table_entry *entry; 516 517 entry = xrcd_table_search(dev, inode); 518 if (!entry) 519 return NULL; 520 521 return entry->xrcd; 522 } 523 524 static void xrcd_table_delete(struct ib_uverbs_device *dev, 525 struct inode *inode) 526 { 527 struct xrcd_table_entry *entry; 528 529 entry = xrcd_table_search(dev, inode); 530 if (entry) { 531 iput(inode); 532 rb_erase(&entry->node, &dev->xrcd_tree); 533 kfree(entry); 534 } 535 } 536 537 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) 538 { 539 struct ib_uverbs_device *ibudev = attrs->ufile->device; 540 struct ib_uverbs_open_xrcd cmd; 541 struct ib_uverbs_open_xrcd_resp resp; 542 struct ib_uxrcd_object *obj; 543 struct ib_xrcd *xrcd = NULL; 544 struct fd f = {NULL, 0}; 545 struct inode *inode = NULL; 546 int ret = 0; 547 int new_xrcd = 0; 548 struct ib_device *ib_dev; 549 550 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 551 if (ret) 552 return ret; 553 554 mutex_lock(&ibudev->xrcd_tree_mutex); 555 556 if (cmd.fd != -1) { 557 /* search for file descriptor */ 558 f = fdget(cmd.fd); 559 if (!f.file) { 560 ret = -EBADF; 561 goto err_tree_mutex_unlock; 562 } 563 564 inode = file_inode(f.file); 565 xrcd = find_xrcd(ibudev, inode); 566 if (!xrcd && !(cmd.oflags & O_CREAT)) { 567 /* no file descriptor. Need CREATE flag */ 568 ret = -EAGAIN; 569 goto err_tree_mutex_unlock; 570 } 571 572 if (xrcd && cmd.oflags & O_EXCL) { 573 ret = -EINVAL; 574 goto err_tree_mutex_unlock; 575 } 576 } 577 578 obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs, 579 &ib_dev); 580 if (IS_ERR(obj)) { 581 ret = PTR_ERR(obj); 582 goto err_tree_mutex_unlock; 583 } 584 585 if (!xrcd) { 586 xrcd = ib_dev->ops.alloc_xrcd(ib_dev, obj->uobject.context, 587 &attrs->driver_udata); 588 if (IS_ERR(xrcd)) { 589 ret = PTR_ERR(xrcd); 590 goto err; 591 } 592 593 xrcd->inode = inode; 594 xrcd->device = ib_dev; 595 atomic_set(&xrcd->usecnt, 0); 596 mutex_init(&xrcd->tgt_qp_mutex); 597 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 598 new_xrcd = 1; 599 } 600 601 atomic_set(&obj->refcnt, 0); 602 obj->uobject.object = xrcd; 603 memset(&resp, 0, sizeof resp); 604 resp.xrcd_handle = obj->uobject.id; 605 606 if (inode) { 607 if (new_xrcd) { 608 /* create new inode/xrcd table entry */ 609 ret = xrcd_table_insert(ibudev, inode, xrcd); 610 if (ret) 611 goto err_dealloc_xrcd; 612 } 613 atomic_inc(&xrcd->usecnt); 614 } 615 616 ret = uverbs_response(attrs, &resp, sizeof(resp)); 617 if (ret) 618 goto err_copy; 619 620 if (f.file) 621 fdput(f); 622 623 mutex_unlock(&ibudev->xrcd_tree_mutex); 624 625 return uobj_alloc_commit(&obj->uobject); 626 627 err_copy: 628 if (inode) { 629 if (new_xrcd) 630 xrcd_table_delete(ibudev, inode); 631 atomic_dec(&xrcd->usecnt); 632 } 633 634 err_dealloc_xrcd: 635 ib_dealloc_xrcd(xrcd); 636 637 err: 638 uobj_alloc_abort(&obj->uobject); 639 640 err_tree_mutex_unlock: 641 if (f.file) 642 fdput(f); 643 644 mutex_unlock(&ibudev->xrcd_tree_mutex); 645 646 return ret; 647 } 648 649 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs) 650 { 651 struct ib_uverbs_close_xrcd cmd; 652 int ret; 653 654 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 655 if (ret) 656 return ret; 657 658 return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs); 659 } 660 661 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, 662 struct ib_xrcd *xrcd, 663 enum rdma_remove_reason why) 664 { 665 struct inode *inode; 666 int ret; 667 struct ib_uverbs_device *dev = uobject->context->ufile->device; 668 669 inode = xrcd->inode; 670 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 671 return 0; 672 673 ret = ib_dealloc_xrcd(xrcd); 674 675 if (ib_is_destroy_retryable(ret, why, uobject)) { 676 atomic_inc(&xrcd->usecnt); 677 return ret; 678 } 679 680 if (inode) 681 xrcd_table_delete(dev, inode); 682 683 return ret; 684 } 685 686 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) 687 { 688 struct ib_uverbs_reg_mr cmd; 689 struct ib_uverbs_reg_mr_resp resp; 690 struct ib_uobject *uobj; 691 struct ib_pd *pd; 692 struct ib_mr *mr; 693 int ret; 694 struct ib_device *ib_dev; 695 696 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 697 if (ret) 698 return ret; 699 700 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 701 return -EINVAL; 702 703 ret = ib_check_mr_access(cmd.access_flags); 704 if (ret) 705 return ret; 706 707 uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev); 708 if (IS_ERR(uobj)) 709 return PTR_ERR(uobj); 710 711 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 712 if (!pd) { 713 ret = -EINVAL; 714 goto err_free; 715 } 716 717 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 718 if (!(pd->device->attrs.device_cap_flags & 719 IB_DEVICE_ON_DEMAND_PAGING)) { 720 pr_debug("ODP support not available\n"); 721 ret = -EINVAL; 722 goto err_put; 723 } 724 } 725 726 mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 727 cmd.access_flags, 728 &attrs->driver_udata); 729 if (IS_ERR(mr)) { 730 ret = PTR_ERR(mr); 731 goto err_put; 732 } 733 734 mr->device = pd->device; 735 mr->pd = pd; 736 mr->dm = NULL; 737 mr->uobject = uobj; 738 atomic_inc(&pd->usecnt); 739 mr->res.type = RDMA_RESTRACK_MR; 740 rdma_restrack_uadd(&mr->res); 741 742 uobj->object = mr; 743 744 memset(&resp, 0, sizeof resp); 745 resp.lkey = mr->lkey; 746 resp.rkey = mr->rkey; 747 resp.mr_handle = uobj->id; 748 749 ret = uverbs_response(attrs, &resp, sizeof(resp)); 750 if (ret) 751 goto err_copy; 752 753 uobj_put_obj_read(pd); 754 755 return uobj_alloc_commit(uobj); 756 757 err_copy: 758 ib_dereg_mr(mr); 759 760 err_put: 761 uobj_put_obj_read(pd); 762 763 err_free: 764 uobj_alloc_abort(uobj); 765 return ret; 766 } 767 768 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) 769 { 770 struct ib_uverbs_rereg_mr cmd; 771 struct ib_uverbs_rereg_mr_resp resp; 772 struct ib_pd *pd = NULL; 773 struct ib_mr *mr; 774 struct ib_pd *old_pd; 775 int ret; 776 struct ib_uobject *uobj; 777 778 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 779 if (ret) 780 return ret; 781 782 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 783 return -EINVAL; 784 785 if ((cmd.flags & IB_MR_REREG_TRANS) && 786 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 787 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 788 return -EINVAL; 789 790 uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs); 791 if (IS_ERR(uobj)) 792 return PTR_ERR(uobj); 793 794 mr = uobj->object; 795 796 if (mr->dm) { 797 ret = -EINVAL; 798 goto put_uobjs; 799 } 800 801 if (cmd.flags & IB_MR_REREG_ACCESS) { 802 ret = ib_check_mr_access(cmd.access_flags); 803 if (ret) 804 goto put_uobjs; 805 } 806 807 if (cmd.flags & IB_MR_REREG_PD) { 808 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, 809 attrs); 810 if (!pd) { 811 ret = -EINVAL; 812 goto put_uobjs; 813 } 814 } 815 816 old_pd = mr->pd; 817 ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start, 818 cmd.length, cmd.hca_va, 819 cmd.access_flags, pd, 820 &attrs->driver_udata); 821 if (!ret) { 822 if (cmd.flags & IB_MR_REREG_PD) { 823 atomic_inc(&pd->usecnt); 824 mr->pd = pd; 825 atomic_dec(&old_pd->usecnt); 826 } 827 } else { 828 goto put_uobj_pd; 829 } 830 831 memset(&resp, 0, sizeof(resp)); 832 resp.lkey = mr->lkey; 833 resp.rkey = mr->rkey; 834 835 ret = uverbs_response(attrs, &resp, sizeof(resp)); 836 837 put_uobj_pd: 838 if (cmd.flags & IB_MR_REREG_PD) 839 uobj_put_obj_read(pd); 840 841 put_uobjs: 842 uobj_put_write(uobj); 843 844 return ret; 845 } 846 847 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs) 848 { 849 struct ib_uverbs_dereg_mr cmd; 850 int ret; 851 852 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 853 if (ret) 854 return ret; 855 856 return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs); 857 } 858 859 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) 860 { 861 struct ib_uverbs_alloc_mw cmd; 862 struct ib_uverbs_alloc_mw_resp resp; 863 struct ib_uobject *uobj; 864 struct ib_pd *pd; 865 struct ib_mw *mw; 866 int ret; 867 struct ib_device *ib_dev; 868 869 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 870 if (ret) 871 return ret; 872 873 uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev); 874 if (IS_ERR(uobj)) 875 return PTR_ERR(uobj); 876 877 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 878 if (!pd) { 879 ret = -EINVAL; 880 goto err_free; 881 } 882 883 mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata); 884 if (IS_ERR(mw)) { 885 ret = PTR_ERR(mw); 886 goto err_put; 887 } 888 889 mw->device = pd->device; 890 mw->pd = pd; 891 mw->uobject = uobj; 892 atomic_inc(&pd->usecnt); 893 894 uobj->object = mw; 895 896 memset(&resp, 0, sizeof(resp)); 897 resp.rkey = mw->rkey; 898 resp.mw_handle = uobj->id; 899 900 ret = uverbs_response(attrs, &resp, sizeof(resp)); 901 if (ret) 902 goto err_copy; 903 904 uobj_put_obj_read(pd); 905 return uobj_alloc_commit(uobj); 906 907 err_copy: 908 uverbs_dealloc_mw(mw); 909 err_put: 910 uobj_put_obj_read(pd); 911 err_free: 912 uobj_alloc_abort(uobj); 913 return ret; 914 } 915 916 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs) 917 { 918 struct ib_uverbs_dealloc_mw cmd; 919 int ret; 920 921 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 922 if (ret) 923 return ret; 924 925 return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs); 926 } 927 928 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs) 929 { 930 struct ib_uverbs_create_comp_channel cmd; 931 struct ib_uverbs_create_comp_channel_resp resp; 932 struct ib_uobject *uobj; 933 struct ib_uverbs_completion_event_file *ev_file; 934 struct ib_device *ib_dev; 935 int ret; 936 937 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 938 if (ret) 939 return ret; 940 941 uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev); 942 if (IS_ERR(uobj)) 943 return PTR_ERR(uobj); 944 945 resp.fd = uobj->id; 946 947 ev_file = container_of(uobj, struct ib_uverbs_completion_event_file, 948 uobj); 949 ib_uverbs_init_event_queue(&ev_file->ev_queue); 950 951 ret = uverbs_response(attrs, &resp, sizeof(resp)); 952 if (ret) { 953 uobj_alloc_abort(uobj); 954 return ret; 955 } 956 957 return uobj_alloc_commit(uobj); 958 } 959 960 static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, 961 struct ib_uverbs_ex_create_cq *cmd) 962 { 963 struct ib_ucq_object *obj; 964 struct ib_uverbs_completion_event_file *ev_file = NULL; 965 struct ib_cq *cq; 966 int ret; 967 struct ib_uverbs_ex_create_cq_resp resp; 968 struct ib_cq_init_attr attr = {}; 969 struct ib_device *ib_dev; 970 971 if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors) 972 return ERR_PTR(-EINVAL); 973 974 obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs, 975 &ib_dev); 976 if (IS_ERR(obj)) 977 return obj; 978 979 if (cmd->comp_channel >= 0) { 980 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs); 981 if (IS_ERR(ev_file)) { 982 ret = PTR_ERR(ev_file); 983 goto err; 984 } 985 } 986 987 obj->uobject.user_handle = cmd->user_handle; 988 obj->comp_events_reported = 0; 989 obj->async_events_reported = 0; 990 INIT_LIST_HEAD(&obj->comp_list); 991 INIT_LIST_HEAD(&obj->async_list); 992 993 attr.cqe = cmd->cqe; 994 attr.comp_vector = cmd->comp_vector; 995 attr.flags = cmd->flags; 996 997 cq = ib_dev->ops.create_cq(ib_dev, &attr, obj->uobject.context, 998 &attrs->driver_udata); 999 if (IS_ERR(cq)) { 1000 ret = PTR_ERR(cq); 1001 goto err_file; 1002 } 1003 1004 cq->device = ib_dev; 1005 cq->uobject = &obj->uobject; 1006 cq->comp_handler = ib_uverbs_comp_handler; 1007 cq->event_handler = ib_uverbs_cq_event_handler; 1008 cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; 1009 atomic_set(&cq->usecnt, 0); 1010 1011 obj->uobject.object = cq; 1012 memset(&resp, 0, sizeof resp); 1013 resp.base.cq_handle = obj->uobject.id; 1014 resp.base.cqe = cq->cqe; 1015 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 1016 1017 cq->res.type = RDMA_RESTRACK_CQ; 1018 rdma_restrack_uadd(&cq->res); 1019 1020 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1021 if (ret) 1022 goto err_cb; 1023 1024 ret = uobj_alloc_commit(&obj->uobject); 1025 if (ret) 1026 return ERR_PTR(ret); 1027 return obj; 1028 1029 err_cb: 1030 ib_destroy_cq(cq); 1031 1032 err_file: 1033 if (ev_file) 1034 ib_uverbs_release_ucq(attrs->ufile, ev_file, obj); 1035 1036 err: 1037 uobj_alloc_abort(&obj->uobject); 1038 1039 return ERR_PTR(ret); 1040 } 1041 1042 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs) 1043 { 1044 struct ib_uverbs_create_cq cmd; 1045 struct ib_uverbs_ex_create_cq cmd_ex; 1046 struct ib_ucq_object *obj; 1047 int ret; 1048 1049 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1050 if (ret) 1051 return ret; 1052 1053 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1054 cmd_ex.user_handle = cmd.user_handle; 1055 cmd_ex.cqe = cmd.cqe; 1056 cmd_ex.comp_vector = cmd.comp_vector; 1057 cmd_ex.comp_channel = cmd.comp_channel; 1058 1059 obj = create_cq(attrs, &cmd_ex); 1060 return PTR_ERR_OR_ZERO(obj); 1061 } 1062 1063 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs) 1064 { 1065 struct ib_uverbs_ex_create_cq cmd; 1066 struct ib_ucq_object *obj; 1067 int ret; 1068 1069 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1070 if (ret) 1071 return ret; 1072 1073 if (cmd.comp_mask) 1074 return -EINVAL; 1075 1076 if (cmd.reserved) 1077 return -EINVAL; 1078 1079 obj = create_cq(attrs, &cmd); 1080 return PTR_ERR_OR_ZERO(obj); 1081 } 1082 1083 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) 1084 { 1085 struct ib_uverbs_resize_cq cmd; 1086 struct ib_uverbs_resize_cq_resp resp = {}; 1087 struct ib_cq *cq; 1088 int ret = -EINVAL; 1089 1090 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1091 if (ret) 1092 return ret; 1093 1094 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1095 if (!cq) 1096 return -EINVAL; 1097 1098 ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata); 1099 if (ret) 1100 goto out; 1101 1102 resp.cqe = cq->cqe; 1103 1104 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1105 out: 1106 uobj_put_obj_read(cq); 1107 1108 return ret; 1109 } 1110 1111 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest, 1112 struct ib_wc *wc) 1113 { 1114 struct ib_uverbs_wc tmp; 1115 1116 tmp.wr_id = wc->wr_id; 1117 tmp.status = wc->status; 1118 tmp.opcode = wc->opcode; 1119 tmp.vendor_err = wc->vendor_err; 1120 tmp.byte_len = wc->byte_len; 1121 tmp.ex.imm_data = wc->ex.imm_data; 1122 tmp.qp_num = wc->qp->qp_num; 1123 tmp.src_qp = wc->src_qp; 1124 tmp.wc_flags = wc->wc_flags; 1125 tmp.pkey_index = wc->pkey_index; 1126 if (rdma_cap_opa_ah(ib_dev, wc->port_num)) 1127 tmp.slid = OPA_TO_IB_UCAST_LID(wc->slid); 1128 else 1129 tmp.slid = ib_lid_cpu16(wc->slid); 1130 tmp.sl = wc->sl; 1131 tmp.dlid_path_bits = wc->dlid_path_bits; 1132 tmp.port_num = wc->port_num; 1133 tmp.reserved = 0; 1134 1135 if (copy_to_user(dest, &tmp, sizeof tmp)) 1136 return -EFAULT; 1137 1138 return 0; 1139 } 1140 1141 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs) 1142 { 1143 struct ib_uverbs_poll_cq cmd; 1144 struct ib_uverbs_poll_cq_resp resp; 1145 u8 __user *header_ptr; 1146 u8 __user *data_ptr; 1147 struct ib_cq *cq; 1148 struct ib_wc wc; 1149 int ret; 1150 1151 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1152 if (ret) 1153 return ret; 1154 1155 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1156 if (!cq) 1157 return -EINVAL; 1158 1159 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1160 header_ptr = attrs->ucore.outbuf; 1161 data_ptr = header_ptr + sizeof resp; 1162 1163 memset(&resp, 0, sizeof resp); 1164 while (resp.count < cmd.ne) { 1165 ret = ib_poll_cq(cq, 1, &wc); 1166 if (ret < 0) 1167 goto out_put; 1168 if (!ret) 1169 break; 1170 1171 ret = copy_wc_to_user(cq->device, data_ptr, &wc); 1172 if (ret) 1173 goto out_put; 1174 1175 data_ptr += sizeof(struct ib_uverbs_wc); 1176 ++resp.count; 1177 } 1178 1179 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1180 ret = -EFAULT; 1181 goto out_put; 1182 } 1183 1184 ret = 0; 1185 1186 out_put: 1187 uobj_put_obj_read(cq); 1188 return ret; 1189 } 1190 1191 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs) 1192 { 1193 struct ib_uverbs_req_notify_cq cmd; 1194 struct ib_cq *cq; 1195 int ret; 1196 1197 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1198 if (ret) 1199 return ret; 1200 1201 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1202 if (!cq) 1203 return -EINVAL; 1204 1205 ib_req_notify_cq(cq, cmd.solicited_only ? 1206 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1207 1208 uobj_put_obj_read(cq); 1209 1210 return 0; 1211 } 1212 1213 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs) 1214 { 1215 struct ib_uverbs_destroy_cq cmd; 1216 struct ib_uverbs_destroy_cq_resp resp; 1217 struct ib_uobject *uobj; 1218 struct ib_ucq_object *obj; 1219 int ret; 1220 1221 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1222 if (ret) 1223 return ret; 1224 1225 uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1226 if (IS_ERR(uobj)) 1227 return PTR_ERR(uobj); 1228 1229 obj = container_of(uobj, struct ib_ucq_object, uobject); 1230 memset(&resp, 0, sizeof(resp)); 1231 resp.comp_events_reported = obj->comp_events_reported; 1232 resp.async_events_reported = obj->async_events_reported; 1233 1234 uobj_put_destroy(uobj); 1235 1236 return uverbs_response(attrs, &resp, sizeof(resp)); 1237 } 1238 1239 static int create_qp(struct uverbs_attr_bundle *attrs, 1240 struct ib_uverbs_ex_create_qp *cmd) 1241 { 1242 struct ib_uqp_object *obj; 1243 struct ib_device *device; 1244 struct ib_pd *pd = NULL; 1245 struct ib_xrcd *xrcd = NULL; 1246 struct ib_uobject *xrcd_uobj = ERR_PTR(-ENOENT); 1247 struct ib_cq *scq = NULL, *rcq = NULL; 1248 struct ib_srq *srq = NULL; 1249 struct ib_qp *qp; 1250 struct ib_qp_init_attr attr = {}; 1251 struct ib_uverbs_ex_create_qp_resp resp; 1252 int ret; 1253 struct ib_rwq_ind_table *ind_tbl = NULL; 1254 bool has_sq = true; 1255 struct ib_device *ib_dev; 1256 1257 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1258 return -EPERM; 1259 1260 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs, 1261 &ib_dev); 1262 if (IS_ERR(obj)) 1263 return PTR_ERR(obj); 1264 obj->uxrcd = NULL; 1265 obj->uevent.uobject.user_handle = cmd->user_handle; 1266 mutex_init(&obj->mcast_lock); 1267 1268 if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) { 1269 ind_tbl = uobj_get_obj_read(rwq_ind_table, 1270 UVERBS_OBJECT_RWQ_IND_TBL, 1271 cmd->rwq_ind_tbl_handle, attrs); 1272 if (!ind_tbl) { 1273 ret = -EINVAL; 1274 goto err_put; 1275 } 1276 1277 attr.rwq_ind_tbl = ind_tbl; 1278 } 1279 1280 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) { 1281 ret = -EINVAL; 1282 goto err_put; 1283 } 1284 1285 if (ind_tbl && !cmd->max_send_wr) 1286 has_sq = false; 1287 1288 if (cmd->qp_type == IB_QPT_XRC_TGT) { 1289 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle, 1290 attrs); 1291 1292 if (IS_ERR(xrcd_uobj)) { 1293 ret = -EINVAL; 1294 goto err_put; 1295 } 1296 1297 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1298 if (!xrcd) { 1299 ret = -EINVAL; 1300 goto err_put; 1301 } 1302 device = xrcd->device; 1303 } else { 1304 if (cmd->qp_type == IB_QPT_XRC_INI) { 1305 cmd->max_recv_wr = 0; 1306 cmd->max_recv_sge = 0; 1307 } else { 1308 if (cmd->is_srq) { 1309 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, 1310 cmd->srq_handle, attrs); 1311 if (!srq || srq->srq_type == IB_SRQT_XRC) { 1312 ret = -EINVAL; 1313 goto err_put; 1314 } 1315 } 1316 1317 if (!ind_tbl) { 1318 if (cmd->recv_cq_handle != cmd->send_cq_handle) { 1319 rcq = uobj_get_obj_read( 1320 cq, UVERBS_OBJECT_CQ, 1321 cmd->recv_cq_handle, attrs); 1322 if (!rcq) { 1323 ret = -EINVAL; 1324 goto err_put; 1325 } 1326 } 1327 } 1328 } 1329 1330 if (has_sq) 1331 scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 1332 cmd->send_cq_handle, attrs); 1333 if (!ind_tbl) 1334 rcq = rcq ?: scq; 1335 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, 1336 attrs); 1337 if (!pd || (!scq && has_sq)) { 1338 ret = -EINVAL; 1339 goto err_put; 1340 } 1341 1342 device = pd->device; 1343 } 1344 1345 attr.event_handler = ib_uverbs_qp_event_handler; 1346 attr.qp_context = attrs->ufile; 1347 attr.send_cq = scq; 1348 attr.recv_cq = rcq; 1349 attr.srq = srq; 1350 attr.xrcd = xrcd; 1351 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : 1352 IB_SIGNAL_REQ_WR; 1353 attr.qp_type = cmd->qp_type; 1354 attr.create_flags = 0; 1355 1356 attr.cap.max_send_wr = cmd->max_send_wr; 1357 attr.cap.max_recv_wr = cmd->max_recv_wr; 1358 attr.cap.max_send_sge = cmd->max_send_sge; 1359 attr.cap.max_recv_sge = cmd->max_recv_sge; 1360 attr.cap.max_inline_data = cmd->max_inline_data; 1361 1362 obj->uevent.events_reported = 0; 1363 INIT_LIST_HEAD(&obj->uevent.event_list); 1364 INIT_LIST_HEAD(&obj->mcast_list); 1365 1366 attr.create_flags = cmd->create_flags; 1367 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 1368 IB_QP_CREATE_CROSS_CHANNEL | 1369 IB_QP_CREATE_MANAGED_SEND | 1370 IB_QP_CREATE_MANAGED_RECV | 1371 IB_QP_CREATE_SCATTER_FCS | 1372 IB_QP_CREATE_CVLAN_STRIPPING | 1373 IB_QP_CREATE_SOURCE_QPN | 1374 IB_QP_CREATE_PCI_WRITE_END_PADDING)) { 1375 ret = -EINVAL; 1376 goto err_put; 1377 } 1378 1379 if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) { 1380 if (!capable(CAP_NET_RAW)) { 1381 ret = -EPERM; 1382 goto err_put; 1383 } 1384 1385 attr.source_qpn = cmd->source_qpn; 1386 } 1387 1388 if (cmd->qp_type == IB_QPT_XRC_TGT) 1389 qp = ib_create_qp(pd, &attr); 1390 else 1391 qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, 1392 &obj->uevent.uobject); 1393 1394 if (IS_ERR(qp)) { 1395 ret = PTR_ERR(qp); 1396 goto err_put; 1397 } 1398 1399 if (cmd->qp_type != IB_QPT_XRC_TGT) { 1400 ret = ib_create_qp_security(qp, device); 1401 if (ret) 1402 goto err_cb; 1403 1404 qp->real_qp = qp; 1405 qp->pd = pd; 1406 qp->send_cq = attr.send_cq; 1407 qp->recv_cq = attr.recv_cq; 1408 qp->srq = attr.srq; 1409 qp->rwq_ind_tbl = ind_tbl; 1410 qp->event_handler = attr.event_handler; 1411 qp->qp_context = attr.qp_context; 1412 qp->qp_type = attr.qp_type; 1413 atomic_set(&qp->usecnt, 0); 1414 atomic_inc(&pd->usecnt); 1415 qp->port = 0; 1416 if (attr.send_cq) 1417 atomic_inc(&attr.send_cq->usecnt); 1418 if (attr.recv_cq) 1419 atomic_inc(&attr.recv_cq->usecnt); 1420 if (attr.srq) 1421 atomic_inc(&attr.srq->usecnt); 1422 if (ind_tbl) 1423 atomic_inc(&ind_tbl->usecnt); 1424 } else { 1425 /* It is done in _ib_create_qp for other QP types */ 1426 qp->uobject = &obj->uevent.uobject; 1427 } 1428 1429 obj->uevent.uobject.object = qp; 1430 1431 memset(&resp, 0, sizeof resp); 1432 resp.base.qpn = qp->qp_num; 1433 resp.base.qp_handle = obj->uevent.uobject.id; 1434 resp.base.max_recv_sge = attr.cap.max_recv_sge; 1435 resp.base.max_send_sge = attr.cap.max_send_sge; 1436 resp.base.max_recv_wr = attr.cap.max_recv_wr; 1437 resp.base.max_send_wr = attr.cap.max_send_wr; 1438 resp.base.max_inline_data = attr.cap.max_inline_data; 1439 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 1440 1441 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1442 if (ret) 1443 goto err_cb; 1444 1445 if (xrcd) { 1446 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 1447 uobject); 1448 atomic_inc(&obj->uxrcd->refcnt); 1449 uobj_put_read(xrcd_uobj); 1450 } 1451 1452 if (pd) 1453 uobj_put_obj_read(pd); 1454 if (scq) 1455 uobj_put_obj_read(scq); 1456 if (rcq && rcq != scq) 1457 uobj_put_obj_read(rcq); 1458 if (srq) 1459 uobj_put_obj_read(srq); 1460 if (ind_tbl) 1461 uobj_put_obj_read(ind_tbl); 1462 1463 return uobj_alloc_commit(&obj->uevent.uobject); 1464 err_cb: 1465 ib_destroy_qp(qp); 1466 1467 err_put: 1468 if (!IS_ERR(xrcd_uobj)) 1469 uobj_put_read(xrcd_uobj); 1470 if (pd) 1471 uobj_put_obj_read(pd); 1472 if (scq) 1473 uobj_put_obj_read(scq); 1474 if (rcq && rcq != scq) 1475 uobj_put_obj_read(rcq); 1476 if (srq) 1477 uobj_put_obj_read(srq); 1478 if (ind_tbl) 1479 uobj_put_obj_read(ind_tbl); 1480 1481 uobj_alloc_abort(&obj->uevent.uobject); 1482 return ret; 1483 } 1484 1485 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs) 1486 { 1487 struct ib_uverbs_create_qp cmd; 1488 struct ib_uverbs_ex_create_qp cmd_ex; 1489 int ret; 1490 1491 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1492 if (ret) 1493 return ret; 1494 1495 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1496 cmd_ex.user_handle = cmd.user_handle; 1497 cmd_ex.pd_handle = cmd.pd_handle; 1498 cmd_ex.send_cq_handle = cmd.send_cq_handle; 1499 cmd_ex.recv_cq_handle = cmd.recv_cq_handle; 1500 cmd_ex.srq_handle = cmd.srq_handle; 1501 cmd_ex.max_send_wr = cmd.max_send_wr; 1502 cmd_ex.max_recv_wr = cmd.max_recv_wr; 1503 cmd_ex.max_send_sge = cmd.max_send_sge; 1504 cmd_ex.max_recv_sge = cmd.max_recv_sge; 1505 cmd_ex.max_inline_data = cmd.max_inline_data; 1506 cmd_ex.sq_sig_all = cmd.sq_sig_all; 1507 cmd_ex.qp_type = cmd.qp_type; 1508 cmd_ex.is_srq = cmd.is_srq; 1509 1510 return create_qp(attrs, &cmd_ex); 1511 } 1512 1513 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs) 1514 { 1515 struct ib_uverbs_ex_create_qp cmd; 1516 int ret; 1517 1518 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1519 if (ret) 1520 return ret; 1521 1522 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK) 1523 return -EINVAL; 1524 1525 if (cmd.reserved) 1526 return -EINVAL; 1527 1528 return create_qp(attrs, &cmd); 1529 } 1530 1531 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) 1532 { 1533 struct ib_uverbs_open_qp cmd; 1534 struct ib_uverbs_create_qp_resp resp; 1535 struct ib_uqp_object *obj; 1536 struct ib_xrcd *xrcd; 1537 struct ib_uobject *uninitialized_var(xrcd_uobj); 1538 struct ib_qp *qp; 1539 struct ib_qp_open_attr attr; 1540 int ret; 1541 struct ib_device *ib_dev; 1542 1543 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1544 if (ret) 1545 return ret; 1546 1547 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs, 1548 &ib_dev); 1549 if (IS_ERR(obj)) 1550 return PTR_ERR(obj); 1551 1552 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs); 1553 if (IS_ERR(xrcd_uobj)) { 1554 ret = -EINVAL; 1555 goto err_put; 1556 } 1557 1558 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1559 if (!xrcd) { 1560 ret = -EINVAL; 1561 goto err_xrcd; 1562 } 1563 1564 attr.event_handler = ib_uverbs_qp_event_handler; 1565 attr.qp_context = attrs->ufile; 1566 attr.qp_num = cmd.qpn; 1567 attr.qp_type = cmd.qp_type; 1568 1569 obj->uevent.events_reported = 0; 1570 INIT_LIST_HEAD(&obj->uevent.event_list); 1571 INIT_LIST_HEAD(&obj->mcast_list); 1572 1573 qp = ib_open_qp(xrcd, &attr); 1574 if (IS_ERR(qp)) { 1575 ret = PTR_ERR(qp); 1576 goto err_xrcd; 1577 } 1578 1579 obj->uevent.uobject.object = qp; 1580 obj->uevent.uobject.user_handle = cmd.user_handle; 1581 1582 memset(&resp, 0, sizeof resp); 1583 resp.qpn = qp->qp_num; 1584 resp.qp_handle = obj->uevent.uobject.id; 1585 1586 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1587 if (ret) 1588 goto err_destroy; 1589 1590 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 1591 atomic_inc(&obj->uxrcd->refcnt); 1592 qp->uobject = &obj->uevent.uobject; 1593 uobj_put_read(xrcd_uobj); 1594 1595 return uobj_alloc_commit(&obj->uevent.uobject); 1596 1597 err_destroy: 1598 ib_destroy_qp(qp); 1599 err_xrcd: 1600 uobj_put_read(xrcd_uobj); 1601 err_put: 1602 uobj_alloc_abort(&obj->uevent.uobject); 1603 return ret; 1604 } 1605 1606 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr, 1607 struct rdma_ah_attr *rdma_attr) 1608 { 1609 const struct ib_global_route *grh; 1610 1611 uverb_attr->dlid = rdma_ah_get_dlid(rdma_attr); 1612 uverb_attr->sl = rdma_ah_get_sl(rdma_attr); 1613 uverb_attr->src_path_bits = rdma_ah_get_path_bits(rdma_attr); 1614 uverb_attr->static_rate = rdma_ah_get_static_rate(rdma_attr); 1615 uverb_attr->is_global = !!(rdma_ah_get_ah_flags(rdma_attr) & 1616 IB_AH_GRH); 1617 if (uverb_attr->is_global) { 1618 grh = rdma_ah_read_grh(rdma_attr); 1619 memcpy(uverb_attr->dgid, grh->dgid.raw, 16); 1620 uverb_attr->flow_label = grh->flow_label; 1621 uverb_attr->sgid_index = grh->sgid_index; 1622 uverb_attr->hop_limit = grh->hop_limit; 1623 uverb_attr->traffic_class = grh->traffic_class; 1624 } 1625 uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr); 1626 } 1627 1628 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs) 1629 { 1630 struct ib_uverbs_query_qp cmd; 1631 struct ib_uverbs_query_qp_resp resp; 1632 struct ib_qp *qp; 1633 struct ib_qp_attr *attr; 1634 struct ib_qp_init_attr *init_attr; 1635 int ret; 1636 1637 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1638 if (ret) 1639 return ret; 1640 1641 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1642 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1643 if (!attr || !init_attr) { 1644 ret = -ENOMEM; 1645 goto out; 1646 } 1647 1648 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 1649 if (!qp) { 1650 ret = -EINVAL; 1651 goto out; 1652 } 1653 1654 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1655 1656 uobj_put_obj_read(qp); 1657 1658 if (ret) 1659 goto out; 1660 1661 memset(&resp, 0, sizeof resp); 1662 1663 resp.qp_state = attr->qp_state; 1664 resp.cur_qp_state = attr->cur_qp_state; 1665 resp.path_mtu = attr->path_mtu; 1666 resp.path_mig_state = attr->path_mig_state; 1667 resp.qkey = attr->qkey; 1668 resp.rq_psn = attr->rq_psn; 1669 resp.sq_psn = attr->sq_psn; 1670 resp.dest_qp_num = attr->dest_qp_num; 1671 resp.qp_access_flags = attr->qp_access_flags; 1672 resp.pkey_index = attr->pkey_index; 1673 resp.alt_pkey_index = attr->alt_pkey_index; 1674 resp.sq_draining = attr->sq_draining; 1675 resp.max_rd_atomic = attr->max_rd_atomic; 1676 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1677 resp.min_rnr_timer = attr->min_rnr_timer; 1678 resp.port_num = attr->port_num; 1679 resp.timeout = attr->timeout; 1680 resp.retry_cnt = attr->retry_cnt; 1681 resp.rnr_retry = attr->rnr_retry; 1682 resp.alt_port_num = attr->alt_port_num; 1683 resp.alt_timeout = attr->alt_timeout; 1684 1685 copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr); 1686 copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr); 1687 1688 resp.max_send_wr = init_attr->cap.max_send_wr; 1689 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1690 resp.max_send_sge = init_attr->cap.max_send_sge; 1691 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1692 resp.max_inline_data = init_attr->cap.max_inline_data; 1693 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1694 1695 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1696 1697 out: 1698 kfree(attr); 1699 kfree(init_attr); 1700 1701 return ret; 1702 } 1703 1704 /* Remove ignored fields set in the attribute mask */ 1705 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 1706 { 1707 switch (qp_type) { 1708 case IB_QPT_XRC_INI: 1709 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 1710 case IB_QPT_XRC_TGT: 1711 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 1712 IB_QP_RNR_RETRY); 1713 default: 1714 return mask; 1715 } 1716 } 1717 1718 static void copy_ah_attr_from_uverbs(struct ib_device *dev, 1719 struct rdma_ah_attr *rdma_attr, 1720 struct ib_uverbs_qp_dest *uverb_attr) 1721 { 1722 rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num); 1723 if (uverb_attr->is_global) { 1724 rdma_ah_set_grh(rdma_attr, NULL, 1725 uverb_attr->flow_label, 1726 uverb_attr->sgid_index, 1727 uverb_attr->hop_limit, 1728 uverb_attr->traffic_class); 1729 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid); 1730 } else { 1731 rdma_ah_set_ah_flags(rdma_attr, 0); 1732 } 1733 rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid); 1734 rdma_ah_set_sl(rdma_attr, uverb_attr->sl); 1735 rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits); 1736 rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate); 1737 rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num); 1738 rdma_ah_set_make_grd(rdma_attr, false); 1739 } 1740 1741 static int modify_qp(struct uverbs_attr_bundle *attrs, 1742 struct ib_uverbs_ex_modify_qp *cmd) 1743 { 1744 struct ib_qp_attr *attr; 1745 struct ib_qp *qp; 1746 int ret; 1747 1748 attr = kzalloc(sizeof(*attr), GFP_KERNEL); 1749 if (!attr) 1750 return -ENOMEM; 1751 1752 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, 1753 attrs); 1754 if (!qp) { 1755 ret = -EINVAL; 1756 goto out; 1757 } 1758 1759 if ((cmd->base.attr_mask & IB_QP_PORT) && 1760 !rdma_is_port_valid(qp->device, cmd->base.port_num)) { 1761 ret = -EINVAL; 1762 goto release_qp; 1763 } 1764 1765 if ((cmd->base.attr_mask & IB_QP_AV)) { 1766 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) { 1767 ret = -EINVAL; 1768 goto release_qp; 1769 } 1770 1771 if (cmd->base.attr_mask & IB_QP_STATE && 1772 cmd->base.qp_state == IB_QPS_RTR) { 1773 /* We are in INIT->RTR TRANSITION (if we are not, 1774 * this transition will be rejected in subsequent checks). 1775 * In the INIT->RTR transition, we cannot have IB_QP_PORT set, 1776 * but the IB_QP_STATE flag is required. 1777 * 1778 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver, 1779 * when IB_QP_AV is set, has required inclusion of a valid 1780 * port number in the primary AV. (AVs are created and handled 1781 * differently for infiniband and ethernet (RoCE) ports). 1782 * 1783 * Check the port number included in the primary AV against 1784 * the port number in the qp struct, which was set (and saved) 1785 * in the RST->INIT transition. 1786 */ 1787 if (cmd->base.dest.port_num != qp->real_qp->port) { 1788 ret = -EINVAL; 1789 goto release_qp; 1790 } 1791 } else { 1792 /* We are in SQD->SQD. (If we are not, this transition will 1793 * be rejected later in the verbs layer checks). 1794 * Check for both IB_QP_PORT and IB_QP_AV, these can be set 1795 * together in the SQD->SQD transition. 1796 * 1797 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the 1798 * verbs layer driver does not track primary port changes 1799 * resulting from path migration. Thus, in SQD, if the primary 1800 * AV is modified, the primary port should also be modified). 1801 * 1802 * Note that in this transition, the IB_QP_STATE flag 1803 * is not allowed. 1804 */ 1805 if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 1806 == (IB_QP_AV | IB_QP_PORT)) && 1807 cmd->base.port_num != cmd->base.dest.port_num) { 1808 ret = -EINVAL; 1809 goto release_qp; 1810 } 1811 if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 1812 == IB_QP_AV) { 1813 cmd->base.attr_mask |= IB_QP_PORT; 1814 cmd->base.port_num = cmd->base.dest.port_num; 1815 } 1816 } 1817 } 1818 1819 if ((cmd->base.attr_mask & IB_QP_ALT_PATH) && 1820 (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) || 1821 !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) || 1822 cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) { 1823 ret = -EINVAL; 1824 goto release_qp; 1825 } 1826 1827 if ((cmd->base.attr_mask & IB_QP_CUR_STATE && 1828 cmd->base.cur_qp_state > IB_QPS_ERR) || 1829 (cmd->base.attr_mask & IB_QP_STATE && 1830 cmd->base.qp_state > IB_QPS_ERR)) { 1831 ret = -EINVAL; 1832 goto release_qp; 1833 } 1834 1835 if (cmd->base.attr_mask & IB_QP_STATE) 1836 attr->qp_state = cmd->base.qp_state; 1837 if (cmd->base.attr_mask & IB_QP_CUR_STATE) 1838 attr->cur_qp_state = cmd->base.cur_qp_state; 1839 if (cmd->base.attr_mask & IB_QP_PATH_MTU) 1840 attr->path_mtu = cmd->base.path_mtu; 1841 if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE) 1842 attr->path_mig_state = cmd->base.path_mig_state; 1843 if (cmd->base.attr_mask & IB_QP_QKEY) 1844 attr->qkey = cmd->base.qkey; 1845 if (cmd->base.attr_mask & IB_QP_RQ_PSN) 1846 attr->rq_psn = cmd->base.rq_psn; 1847 if (cmd->base.attr_mask & IB_QP_SQ_PSN) 1848 attr->sq_psn = cmd->base.sq_psn; 1849 if (cmd->base.attr_mask & IB_QP_DEST_QPN) 1850 attr->dest_qp_num = cmd->base.dest_qp_num; 1851 if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS) 1852 attr->qp_access_flags = cmd->base.qp_access_flags; 1853 if (cmd->base.attr_mask & IB_QP_PKEY_INDEX) 1854 attr->pkey_index = cmd->base.pkey_index; 1855 if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 1856 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify; 1857 if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC) 1858 attr->max_rd_atomic = cmd->base.max_rd_atomic; 1859 if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 1860 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic; 1861 if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER) 1862 attr->min_rnr_timer = cmd->base.min_rnr_timer; 1863 if (cmd->base.attr_mask & IB_QP_PORT) 1864 attr->port_num = cmd->base.port_num; 1865 if (cmd->base.attr_mask & IB_QP_TIMEOUT) 1866 attr->timeout = cmd->base.timeout; 1867 if (cmd->base.attr_mask & IB_QP_RETRY_CNT) 1868 attr->retry_cnt = cmd->base.retry_cnt; 1869 if (cmd->base.attr_mask & IB_QP_RNR_RETRY) 1870 attr->rnr_retry = cmd->base.rnr_retry; 1871 if (cmd->base.attr_mask & IB_QP_ALT_PATH) { 1872 attr->alt_port_num = cmd->base.alt_port_num; 1873 attr->alt_timeout = cmd->base.alt_timeout; 1874 attr->alt_pkey_index = cmd->base.alt_pkey_index; 1875 } 1876 if (cmd->base.attr_mask & IB_QP_RATE_LIMIT) 1877 attr->rate_limit = cmd->rate_limit; 1878 1879 if (cmd->base.attr_mask & IB_QP_AV) 1880 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr, 1881 &cmd->base.dest); 1882 1883 if (cmd->base.attr_mask & IB_QP_ALT_PATH) 1884 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr, 1885 &cmd->base.alt_dest); 1886 1887 ret = ib_modify_qp_with_udata(qp, attr, 1888 modify_qp_mask(qp->qp_type, 1889 cmd->base.attr_mask), 1890 &attrs->driver_udata); 1891 1892 release_qp: 1893 uobj_put_obj_read(qp); 1894 out: 1895 kfree(attr); 1896 1897 return ret; 1898 } 1899 1900 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs) 1901 { 1902 struct ib_uverbs_ex_modify_qp cmd; 1903 int ret; 1904 1905 ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base)); 1906 if (ret) 1907 return ret; 1908 1909 if (cmd.base.attr_mask & 1910 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1)) 1911 return -EOPNOTSUPP; 1912 1913 return modify_qp(attrs, &cmd); 1914 } 1915 1916 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs) 1917 { 1918 struct ib_uverbs_ex_modify_qp cmd; 1919 struct ib_uverbs_ex_modify_qp_resp resp = { 1920 .response_length = uverbs_response_length(attrs, sizeof(resp)) 1921 }; 1922 int ret; 1923 1924 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1925 if (ret) 1926 return ret; 1927 1928 /* 1929 * Last bit is reserved for extending the attr_mask by 1930 * using another field. 1931 */ 1932 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); 1933 1934 if (cmd.base.attr_mask & 1935 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) 1936 return -EOPNOTSUPP; 1937 1938 ret = modify_qp(attrs, &cmd); 1939 if (ret) 1940 return ret; 1941 1942 return uverbs_response(attrs, &resp, sizeof(resp)); 1943 } 1944 1945 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs) 1946 { 1947 struct ib_uverbs_destroy_qp cmd; 1948 struct ib_uverbs_destroy_qp_resp resp; 1949 struct ib_uobject *uobj; 1950 struct ib_uqp_object *obj; 1951 int ret; 1952 1953 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1954 if (ret) 1955 return ret; 1956 1957 uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 1958 if (IS_ERR(uobj)) 1959 return PTR_ERR(uobj); 1960 1961 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 1962 memset(&resp, 0, sizeof(resp)); 1963 resp.events_reported = obj->uevent.events_reported; 1964 1965 uobj_put_destroy(uobj); 1966 1967 return uverbs_response(attrs, &resp, sizeof(resp)); 1968 } 1969 1970 static void *alloc_wr(size_t wr_size, __u32 num_sge) 1971 { 1972 if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) / 1973 sizeof (struct ib_sge)) 1974 return NULL; 1975 1976 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) + 1977 num_sge * sizeof (struct ib_sge), GFP_KERNEL); 1978 } 1979 1980 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) 1981 { 1982 struct ib_uverbs_post_send cmd; 1983 struct ib_uverbs_post_send_resp resp; 1984 struct ib_uverbs_send_wr *user_wr; 1985 struct ib_send_wr *wr = NULL, *last, *next; 1986 const struct ib_send_wr *bad_wr; 1987 struct ib_qp *qp; 1988 int i, sg_ind; 1989 int is_ud; 1990 int ret, ret2; 1991 size_t next_size; 1992 const struct ib_sge __user *sgls; 1993 const void __user *wqes; 1994 struct uverbs_req_iter iter; 1995 1996 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 1997 if (ret) 1998 return ret; 1999 wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count); 2000 if (IS_ERR(wqes)) 2001 return PTR_ERR(wqes); 2002 sgls = uverbs_request_next_ptr( 2003 &iter, cmd.sge_count * sizeof(struct ib_uverbs_sge)); 2004 if (IS_ERR(sgls)) 2005 return PTR_ERR(sgls); 2006 ret = uverbs_request_finish(&iter); 2007 if (ret) 2008 return ret; 2009 2010 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2011 if (!user_wr) 2012 return -ENOMEM; 2013 2014 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2015 if (!qp) 2016 goto out; 2017 2018 is_ud = qp->qp_type == IB_QPT_UD; 2019 sg_ind = 0; 2020 last = NULL; 2021 for (i = 0; i < cmd.wr_count; ++i) { 2022 if (copy_from_user(user_wr, wqes + i * cmd.wqe_size, 2023 cmd.wqe_size)) { 2024 ret = -EFAULT; 2025 goto out_put; 2026 } 2027 2028 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2029 ret = -EINVAL; 2030 goto out_put; 2031 } 2032 2033 if (is_ud) { 2034 struct ib_ud_wr *ud; 2035 2036 if (user_wr->opcode != IB_WR_SEND && 2037 user_wr->opcode != IB_WR_SEND_WITH_IMM) { 2038 ret = -EINVAL; 2039 goto out_put; 2040 } 2041 2042 next_size = sizeof(*ud); 2043 ud = alloc_wr(next_size, user_wr->num_sge); 2044 if (!ud) { 2045 ret = -ENOMEM; 2046 goto out_put; 2047 } 2048 2049 ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH, 2050 user_wr->wr.ud.ah, attrs); 2051 if (!ud->ah) { 2052 kfree(ud); 2053 ret = -EINVAL; 2054 goto out_put; 2055 } 2056 ud->remote_qpn = user_wr->wr.ud.remote_qpn; 2057 ud->remote_qkey = user_wr->wr.ud.remote_qkey; 2058 2059 next = &ud->wr; 2060 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2061 user_wr->opcode == IB_WR_RDMA_WRITE || 2062 user_wr->opcode == IB_WR_RDMA_READ) { 2063 struct ib_rdma_wr *rdma; 2064 2065 next_size = sizeof(*rdma); 2066 rdma = alloc_wr(next_size, user_wr->num_sge); 2067 if (!rdma) { 2068 ret = -ENOMEM; 2069 goto out_put; 2070 } 2071 2072 rdma->remote_addr = user_wr->wr.rdma.remote_addr; 2073 rdma->rkey = user_wr->wr.rdma.rkey; 2074 2075 next = &rdma->wr; 2076 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2077 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2078 struct ib_atomic_wr *atomic; 2079 2080 next_size = sizeof(*atomic); 2081 atomic = alloc_wr(next_size, user_wr->num_sge); 2082 if (!atomic) { 2083 ret = -ENOMEM; 2084 goto out_put; 2085 } 2086 2087 atomic->remote_addr = user_wr->wr.atomic.remote_addr; 2088 atomic->compare_add = user_wr->wr.atomic.compare_add; 2089 atomic->swap = user_wr->wr.atomic.swap; 2090 atomic->rkey = user_wr->wr.atomic.rkey; 2091 2092 next = &atomic->wr; 2093 } else if (user_wr->opcode == IB_WR_SEND || 2094 user_wr->opcode == IB_WR_SEND_WITH_IMM || 2095 user_wr->opcode == IB_WR_SEND_WITH_INV) { 2096 next_size = sizeof(*next); 2097 next = alloc_wr(next_size, user_wr->num_sge); 2098 if (!next) { 2099 ret = -ENOMEM; 2100 goto out_put; 2101 } 2102 } else { 2103 ret = -EINVAL; 2104 goto out_put; 2105 } 2106 2107 if (user_wr->opcode == IB_WR_SEND_WITH_IMM || 2108 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 2109 next->ex.imm_data = 2110 (__be32 __force) user_wr->ex.imm_data; 2111 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) { 2112 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; 2113 } 2114 2115 if (!last) 2116 wr = next; 2117 else 2118 last->next = next; 2119 last = next; 2120 2121 next->next = NULL; 2122 next->wr_id = user_wr->wr_id; 2123 next->num_sge = user_wr->num_sge; 2124 next->opcode = user_wr->opcode; 2125 next->send_flags = user_wr->send_flags; 2126 2127 if (next->num_sge) { 2128 next->sg_list = (void *) next + 2129 ALIGN(next_size, sizeof(struct ib_sge)); 2130 if (copy_from_user(next->sg_list, sgls + sg_ind, 2131 next->num_sge * 2132 sizeof(struct ib_sge))) { 2133 ret = -EFAULT; 2134 goto out_put; 2135 } 2136 sg_ind += next->num_sge; 2137 } else 2138 next->sg_list = NULL; 2139 } 2140 2141 resp.bad_wr = 0; 2142 ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr); 2143 if (ret) 2144 for (next = wr; next; next = next->next) { 2145 ++resp.bad_wr; 2146 if (next == bad_wr) 2147 break; 2148 } 2149 2150 ret2 = uverbs_response(attrs, &resp, sizeof(resp)); 2151 if (ret2) 2152 ret = ret2; 2153 2154 out_put: 2155 uobj_put_obj_read(qp); 2156 2157 while (wr) { 2158 if (is_ud && ud_wr(wr)->ah) 2159 uobj_put_obj_read(ud_wr(wr)->ah); 2160 next = wr->next; 2161 kfree(wr); 2162 wr = next; 2163 } 2164 2165 out: 2166 kfree(user_wr); 2167 2168 return ret; 2169 } 2170 2171 static struct ib_recv_wr * 2172 ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count, 2173 u32 wqe_size, u32 sge_count) 2174 { 2175 struct ib_uverbs_recv_wr *user_wr; 2176 struct ib_recv_wr *wr = NULL, *last, *next; 2177 int sg_ind; 2178 int i; 2179 int ret; 2180 const struct ib_sge __user *sgls; 2181 const void __user *wqes; 2182 2183 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2184 return ERR_PTR(-EINVAL); 2185 2186 wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count); 2187 if (IS_ERR(wqes)) 2188 return ERR_CAST(wqes); 2189 sgls = uverbs_request_next_ptr( 2190 iter, sge_count * sizeof(struct ib_uverbs_sge)); 2191 if (IS_ERR(sgls)) 2192 return ERR_CAST(sgls); 2193 ret = uverbs_request_finish(iter); 2194 if (ret) 2195 return ERR_PTR(ret); 2196 2197 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2198 if (!user_wr) 2199 return ERR_PTR(-ENOMEM); 2200 2201 sg_ind = 0; 2202 last = NULL; 2203 for (i = 0; i < wr_count; ++i) { 2204 if (copy_from_user(user_wr, wqes + i * wqe_size, 2205 wqe_size)) { 2206 ret = -EFAULT; 2207 goto err; 2208 } 2209 2210 if (user_wr->num_sge + sg_ind > sge_count) { 2211 ret = -EINVAL; 2212 goto err; 2213 } 2214 2215 if (user_wr->num_sge >= 2216 (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) / 2217 sizeof (struct ib_sge)) { 2218 ret = -EINVAL; 2219 goto err; 2220 } 2221 2222 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2223 user_wr->num_sge * sizeof (struct ib_sge), 2224 GFP_KERNEL); 2225 if (!next) { 2226 ret = -ENOMEM; 2227 goto err; 2228 } 2229 2230 if (!last) 2231 wr = next; 2232 else 2233 last->next = next; 2234 last = next; 2235 2236 next->next = NULL; 2237 next->wr_id = user_wr->wr_id; 2238 next->num_sge = user_wr->num_sge; 2239 2240 if (next->num_sge) { 2241 next->sg_list = (void *) next + 2242 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2243 if (copy_from_user(next->sg_list, sgls + sg_ind, 2244 next->num_sge * 2245 sizeof(struct ib_sge))) { 2246 ret = -EFAULT; 2247 goto err; 2248 } 2249 sg_ind += next->num_sge; 2250 } else 2251 next->sg_list = NULL; 2252 } 2253 2254 kfree(user_wr); 2255 return wr; 2256 2257 err: 2258 kfree(user_wr); 2259 2260 while (wr) { 2261 next = wr->next; 2262 kfree(wr); 2263 wr = next; 2264 } 2265 2266 return ERR_PTR(ret); 2267 } 2268 2269 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs) 2270 { 2271 struct ib_uverbs_post_recv cmd; 2272 struct ib_uverbs_post_recv_resp resp; 2273 struct ib_recv_wr *wr, *next; 2274 const struct ib_recv_wr *bad_wr; 2275 struct ib_qp *qp; 2276 int ret, ret2; 2277 struct uverbs_req_iter iter; 2278 2279 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 2280 if (ret) 2281 return ret; 2282 2283 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size, 2284 cmd.sge_count); 2285 if (IS_ERR(wr)) 2286 return PTR_ERR(wr); 2287 2288 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2289 if (!qp) { 2290 ret = -EINVAL; 2291 goto out; 2292 } 2293 2294 resp.bad_wr = 0; 2295 ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr); 2296 2297 uobj_put_obj_read(qp); 2298 if (ret) { 2299 for (next = wr; next; next = next->next) { 2300 ++resp.bad_wr; 2301 if (next == bad_wr) 2302 break; 2303 } 2304 } 2305 2306 ret2 = uverbs_response(attrs, &resp, sizeof(resp)); 2307 if (ret2) 2308 ret = ret2; 2309 out: 2310 while (wr) { 2311 next = wr->next; 2312 kfree(wr); 2313 wr = next; 2314 } 2315 2316 return ret; 2317 } 2318 2319 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs) 2320 { 2321 struct ib_uverbs_post_srq_recv cmd; 2322 struct ib_uverbs_post_srq_recv_resp resp; 2323 struct ib_recv_wr *wr, *next; 2324 const struct ib_recv_wr *bad_wr; 2325 struct ib_srq *srq; 2326 int ret, ret2; 2327 struct uverbs_req_iter iter; 2328 2329 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 2330 if (ret) 2331 return ret; 2332 2333 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size, 2334 cmd.sge_count); 2335 if (IS_ERR(wr)) 2336 return PTR_ERR(wr); 2337 2338 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 2339 if (!srq) { 2340 ret = -EINVAL; 2341 goto out; 2342 } 2343 2344 resp.bad_wr = 0; 2345 ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr); 2346 2347 uobj_put_obj_read(srq); 2348 2349 if (ret) 2350 for (next = wr; next; next = next->next) { 2351 ++resp.bad_wr; 2352 if (next == bad_wr) 2353 break; 2354 } 2355 2356 ret2 = uverbs_response(attrs, &resp, sizeof(resp)); 2357 if (ret2) 2358 ret = ret2; 2359 2360 out: 2361 while (wr) { 2362 next = wr->next; 2363 kfree(wr); 2364 wr = next; 2365 } 2366 2367 return ret; 2368 } 2369 2370 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs) 2371 { 2372 struct ib_uverbs_create_ah cmd; 2373 struct ib_uverbs_create_ah_resp resp; 2374 struct ib_uobject *uobj; 2375 struct ib_pd *pd; 2376 struct ib_ah *ah; 2377 struct rdma_ah_attr attr = {}; 2378 int ret; 2379 struct ib_device *ib_dev; 2380 2381 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2382 if (ret) 2383 return ret; 2384 2385 uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev); 2386 if (IS_ERR(uobj)) 2387 return PTR_ERR(uobj); 2388 2389 if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) { 2390 ret = -EINVAL; 2391 goto err; 2392 } 2393 2394 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 2395 if (!pd) { 2396 ret = -EINVAL; 2397 goto err; 2398 } 2399 2400 attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num); 2401 rdma_ah_set_make_grd(&attr, false); 2402 rdma_ah_set_dlid(&attr, cmd.attr.dlid); 2403 rdma_ah_set_sl(&attr, cmd.attr.sl); 2404 rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits); 2405 rdma_ah_set_static_rate(&attr, cmd.attr.static_rate); 2406 rdma_ah_set_port_num(&attr, cmd.attr.port_num); 2407 2408 if (cmd.attr.is_global) { 2409 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label, 2410 cmd.attr.grh.sgid_index, 2411 cmd.attr.grh.hop_limit, 2412 cmd.attr.grh.traffic_class); 2413 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid); 2414 } else { 2415 rdma_ah_set_ah_flags(&attr, 0); 2416 } 2417 2418 ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata); 2419 if (IS_ERR(ah)) { 2420 ret = PTR_ERR(ah); 2421 goto err_put; 2422 } 2423 2424 ah->uobject = uobj; 2425 uobj->user_handle = cmd.user_handle; 2426 uobj->object = ah; 2427 2428 resp.ah_handle = uobj->id; 2429 2430 ret = uverbs_response(attrs, &resp, sizeof(resp)); 2431 if (ret) 2432 goto err_copy; 2433 2434 uobj_put_obj_read(pd); 2435 return uobj_alloc_commit(uobj); 2436 2437 err_copy: 2438 rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE); 2439 2440 err_put: 2441 uobj_put_obj_read(pd); 2442 2443 err: 2444 uobj_alloc_abort(uobj); 2445 return ret; 2446 } 2447 2448 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs) 2449 { 2450 struct ib_uverbs_destroy_ah cmd; 2451 int ret; 2452 2453 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2454 if (ret) 2455 return ret; 2456 2457 return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs); 2458 } 2459 2460 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs) 2461 { 2462 struct ib_uverbs_attach_mcast cmd; 2463 struct ib_qp *qp; 2464 struct ib_uqp_object *obj; 2465 struct ib_uverbs_mcast_entry *mcast; 2466 int ret; 2467 2468 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2469 if (ret) 2470 return ret; 2471 2472 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2473 if (!qp) 2474 return -EINVAL; 2475 2476 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2477 2478 mutex_lock(&obj->mcast_lock); 2479 list_for_each_entry(mcast, &obj->mcast_list, list) 2480 if (cmd.mlid == mcast->lid && 2481 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2482 ret = 0; 2483 goto out_put; 2484 } 2485 2486 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 2487 if (!mcast) { 2488 ret = -ENOMEM; 2489 goto out_put; 2490 } 2491 2492 mcast->lid = cmd.mlid; 2493 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 2494 2495 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 2496 if (!ret) 2497 list_add_tail(&mcast->list, &obj->mcast_list); 2498 else 2499 kfree(mcast); 2500 2501 out_put: 2502 mutex_unlock(&obj->mcast_lock); 2503 uobj_put_obj_read(qp); 2504 2505 return ret; 2506 } 2507 2508 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) 2509 { 2510 struct ib_uverbs_detach_mcast cmd; 2511 struct ib_uqp_object *obj; 2512 struct ib_qp *qp; 2513 struct ib_uverbs_mcast_entry *mcast; 2514 int ret = -EINVAL; 2515 bool found = false; 2516 2517 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2518 if (ret) 2519 return ret; 2520 2521 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2522 if (!qp) 2523 return -EINVAL; 2524 2525 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2526 mutex_lock(&obj->mcast_lock); 2527 2528 list_for_each_entry(mcast, &obj->mcast_list, list) 2529 if (cmd.mlid == mcast->lid && 2530 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2531 list_del(&mcast->list); 2532 kfree(mcast); 2533 found = true; 2534 break; 2535 } 2536 2537 if (!found) { 2538 ret = -EINVAL; 2539 goto out_put; 2540 } 2541 2542 ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid); 2543 2544 out_put: 2545 mutex_unlock(&obj->mcast_lock); 2546 uobj_put_obj_read(qp); 2547 return ret; 2548 } 2549 2550 struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) 2551 { 2552 struct ib_uflow_resources *resources; 2553 2554 resources = kzalloc(sizeof(*resources), GFP_KERNEL); 2555 2556 if (!resources) 2557 return NULL; 2558 2559 if (!num_specs) 2560 goto out; 2561 2562 resources->counters = 2563 kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL); 2564 resources->collection = 2565 kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL); 2566 2567 if (!resources->counters || !resources->collection) 2568 goto err; 2569 2570 out: 2571 resources->max = num_specs; 2572 return resources; 2573 2574 err: 2575 kfree(resources->counters); 2576 kfree(resources); 2577 2578 return NULL; 2579 } 2580 EXPORT_SYMBOL(flow_resources_alloc); 2581 2582 void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) 2583 { 2584 unsigned int i; 2585 2586 if (!uflow_res) 2587 return; 2588 2589 for (i = 0; i < uflow_res->collection_num; i++) 2590 atomic_dec(&uflow_res->collection[i]->usecnt); 2591 2592 for (i = 0; i < uflow_res->counters_num; i++) 2593 atomic_dec(&uflow_res->counters[i]->usecnt); 2594 2595 kfree(uflow_res->collection); 2596 kfree(uflow_res->counters); 2597 kfree(uflow_res); 2598 } 2599 EXPORT_SYMBOL(ib_uverbs_flow_resources_free); 2600 2601 void flow_resources_add(struct ib_uflow_resources *uflow_res, 2602 enum ib_flow_spec_type type, 2603 void *ibobj) 2604 { 2605 WARN_ON(uflow_res->num >= uflow_res->max); 2606 2607 switch (type) { 2608 case IB_FLOW_SPEC_ACTION_HANDLE: 2609 atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); 2610 uflow_res->collection[uflow_res->collection_num++] = 2611 (struct ib_flow_action *)ibobj; 2612 break; 2613 case IB_FLOW_SPEC_ACTION_COUNT: 2614 atomic_inc(&((struct ib_counters *)ibobj)->usecnt); 2615 uflow_res->counters[uflow_res->counters_num++] = 2616 (struct ib_counters *)ibobj; 2617 break; 2618 default: 2619 WARN_ON(1); 2620 } 2621 2622 uflow_res->num++; 2623 } 2624 EXPORT_SYMBOL(flow_resources_add); 2625 2626 static int kern_spec_to_ib_spec_action(const struct uverbs_attr_bundle *attrs, 2627 struct ib_uverbs_flow_spec *kern_spec, 2628 union ib_flow_spec *ib_spec, 2629 struct ib_uflow_resources *uflow_res) 2630 { 2631 ib_spec->type = kern_spec->type; 2632 switch (ib_spec->type) { 2633 case IB_FLOW_SPEC_ACTION_TAG: 2634 if (kern_spec->flow_tag.size != 2635 sizeof(struct ib_uverbs_flow_spec_action_tag)) 2636 return -EINVAL; 2637 2638 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag); 2639 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id; 2640 break; 2641 case IB_FLOW_SPEC_ACTION_DROP: 2642 if (kern_spec->drop.size != 2643 sizeof(struct ib_uverbs_flow_spec_action_drop)) 2644 return -EINVAL; 2645 2646 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop); 2647 break; 2648 case IB_FLOW_SPEC_ACTION_HANDLE: 2649 if (kern_spec->action.size != 2650 sizeof(struct ib_uverbs_flow_spec_action_handle)) 2651 return -EOPNOTSUPP; 2652 ib_spec->action.act = uobj_get_obj_read(flow_action, 2653 UVERBS_OBJECT_FLOW_ACTION, 2654 kern_spec->action.handle, 2655 attrs); 2656 if (!ib_spec->action.act) 2657 return -EINVAL; 2658 ib_spec->action.size = 2659 sizeof(struct ib_flow_spec_action_handle); 2660 flow_resources_add(uflow_res, 2661 IB_FLOW_SPEC_ACTION_HANDLE, 2662 ib_spec->action.act); 2663 uobj_put_obj_read(ib_spec->action.act); 2664 break; 2665 case IB_FLOW_SPEC_ACTION_COUNT: 2666 if (kern_spec->flow_count.size != 2667 sizeof(struct ib_uverbs_flow_spec_action_count)) 2668 return -EINVAL; 2669 ib_spec->flow_count.counters = 2670 uobj_get_obj_read(counters, 2671 UVERBS_OBJECT_COUNTERS, 2672 kern_spec->flow_count.handle, 2673 attrs); 2674 if (!ib_spec->flow_count.counters) 2675 return -EINVAL; 2676 ib_spec->flow_count.size = 2677 sizeof(struct ib_flow_spec_action_count); 2678 flow_resources_add(uflow_res, 2679 IB_FLOW_SPEC_ACTION_COUNT, 2680 ib_spec->flow_count.counters); 2681 uobj_put_obj_read(ib_spec->flow_count.counters); 2682 break; 2683 default: 2684 return -EINVAL; 2685 } 2686 return 0; 2687 } 2688 2689 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec) 2690 { 2691 /* Returns user space filter size, includes padding */ 2692 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2; 2693 } 2694 2695 static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size, 2696 u16 ib_real_filter_sz) 2697 { 2698 /* 2699 * User space filter structures must be 64 bit aligned, otherwise this 2700 * may pass, but we won't handle additional new attributes. 2701 */ 2702 2703 if (kern_filter_size > ib_real_filter_sz) { 2704 if (memchr_inv(kern_spec_filter + 2705 ib_real_filter_sz, 0, 2706 kern_filter_size - ib_real_filter_sz)) 2707 return -EINVAL; 2708 return ib_real_filter_sz; 2709 } 2710 return kern_filter_size; 2711 } 2712 2713 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type, 2714 const void *kern_spec_mask, 2715 const void *kern_spec_val, 2716 size_t kern_filter_sz, 2717 union ib_flow_spec *ib_spec) 2718 { 2719 ssize_t actual_filter_sz; 2720 ssize_t ib_filter_sz; 2721 2722 /* User flow spec size must be aligned to 4 bytes */ 2723 if (kern_filter_sz != ALIGN(kern_filter_sz, 4)) 2724 return -EINVAL; 2725 2726 ib_spec->type = type; 2727 2728 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL)) 2729 return -EINVAL; 2730 2731 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 2732 case IB_FLOW_SPEC_ETH: 2733 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz); 2734 actual_filter_sz = spec_filter_size(kern_spec_mask, 2735 kern_filter_sz, 2736 ib_filter_sz); 2737 if (actual_filter_sz <= 0) 2738 return -EINVAL; 2739 ib_spec->size = sizeof(struct ib_flow_spec_eth); 2740 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz); 2741 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz); 2742 break; 2743 case IB_FLOW_SPEC_IPV4: 2744 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz); 2745 actual_filter_sz = spec_filter_size(kern_spec_mask, 2746 kern_filter_sz, 2747 ib_filter_sz); 2748 if (actual_filter_sz <= 0) 2749 return -EINVAL; 2750 ib_spec->size = sizeof(struct ib_flow_spec_ipv4); 2751 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz); 2752 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz); 2753 break; 2754 case IB_FLOW_SPEC_IPV6: 2755 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz); 2756 actual_filter_sz = spec_filter_size(kern_spec_mask, 2757 kern_filter_sz, 2758 ib_filter_sz); 2759 if (actual_filter_sz <= 0) 2760 return -EINVAL; 2761 ib_spec->size = sizeof(struct ib_flow_spec_ipv6); 2762 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz); 2763 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz); 2764 2765 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) || 2766 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20)) 2767 return -EINVAL; 2768 break; 2769 case IB_FLOW_SPEC_TCP: 2770 case IB_FLOW_SPEC_UDP: 2771 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz); 2772 actual_filter_sz = spec_filter_size(kern_spec_mask, 2773 kern_filter_sz, 2774 ib_filter_sz); 2775 if (actual_filter_sz <= 0) 2776 return -EINVAL; 2777 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp); 2778 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz); 2779 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz); 2780 break; 2781 case IB_FLOW_SPEC_VXLAN_TUNNEL: 2782 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz); 2783 actual_filter_sz = spec_filter_size(kern_spec_mask, 2784 kern_filter_sz, 2785 ib_filter_sz); 2786 if (actual_filter_sz <= 0) 2787 return -EINVAL; 2788 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel); 2789 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz); 2790 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz); 2791 2792 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) || 2793 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24)) 2794 return -EINVAL; 2795 break; 2796 case IB_FLOW_SPEC_ESP: 2797 ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz); 2798 actual_filter_sz = spec_filter_size(kern_spec_mask, 2799 kern_filter_sz, 2800 ib_filter_sz); 2801 if (actual_filter_sz <= 0) 2802 return -EINVAL; 2803 ib_spec->esp.size = sizeof(struct ib_flow_spec_esp); 2804 memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz); 2805 memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz); 2806 break; 2807 case IB_FLOW_SPEC_GRE: 2808 ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz); 2809 actual_filter_sz = spec_filter_size(kern_spec_mask, 2810 kern_filter_sz, 2811 ib_filter_sz); 2812 if (actual_filter_sz <= 0) 2813 return -EINVAL; 2814 ib_spec->gre.size = sizeof(struct ib_flow_spec_gre); 2815 memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz); 2816 memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz); 2817 break; 2818 case IB_FLOW_SPEC_MPLS: 2819 ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz); 2820 actual_filter_sz = spec_filter_size(kern_spec_mask, 2821 kern_filter_sz, 2822 ib_filter_sz); 2823 if (actual_filter_sz <= 0) 2824 return -EINVAL; 2825 ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls); 2826 memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz); 2827 memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz); 2828 break; 2829 default: 2830 return -EINVAL; 2831 } 2832 return 0; 2833 } 2834 2835 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec, 2836 union ib_flow_spec *ib_spec) 2837 { 2838 ssize_t kern_filter_sz; 2839 void *kern_spec_mask; 2840 void *kern_spec_val; 2841 2842 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr); 2843 2844 kern_spec_val = (void *)kern_spec + 2845 sizeof(struct ib_uverbs_flow_spec_hdr); 2846 kern_spec_mask = kern_spec_val + kern_filter_sz; 2847 2848 return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type, 2849 kern_spec_mask, 2850 kern_spec_val, 2851 kern_filter_sz, ib_spec); 2852 } 2853 2854 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs, 2855 struct ib_uverbs_flow_spec *kern_spec, 2856 union ib_flow_spec *ib_spec, 2857 struct ib_uflow_resources *uflow_res) 2858 { 2859 if (kern_spec->reserved) 2860 return -EINVAL; 2861 2862 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG) 2863 return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec, 2864 uflow_res); 2865 else 2866 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec); 2867 } 2868 2869 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) 2870 { 2871 struct ib_uverbs_ex_create_wq cmd; 2872 struct ib_uverbs_ex_create_wq_resp resp = {}; 2873 struct ib_uwq_object *obj; 2874 int err = 0; 2875 struct ib_cq *cq; 2876 struct ib_pd *pd; 2877 struct ib_wq *wq; 2878 struct ib_wq_init_attr wq_init_attr = {}; 2879 struct ib_device *ib_dev; 2880 2881 err = uverbs_request(attrs, &cmd, sizeof(cmd)); 2882 if (err) 2883 return err; 2884 2885 if (cmd.comp_mask) 2886 return -EOPNOTSUPP; 2887 2888 obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs, 2889 &ib_dev); 2890 if (IS_ERR(obj)) 2891 return PTR_ERR(obj); 2892 2893 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 2894 if (!pd) { 2895 err = -EINVAL; 2896 goto err_uobj; 2897 } 2898 2899 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 2900 if (!cq) { 2901 err = -EINVAL; 2902 goto err_put_pd; 2903 } 2904 2905 wq_init_attr.cq = cq; 2906 wq_init_attr.max_sge = cmd.max_sge; 2907 wq_init_attr.max_wr = cmd.max_wr; 2908 wq_init_attr.wq_context = attrs->ufile; 2909 wq_init_attr.wq_type = cmd.wq_type; 2910 wq_init_attr.event_handler = ib_uverbs_wq_event_handler; 2911 wq_init_attr.create_flags = cmd.create_flags; 2912 obj->uevent.events_reported = 0; 2913 INIT_LIST_HEAD(&obj->uevent.event_list); 2914 2915 wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata); 2916 if (IS_ERR(wq)) { 2917 err = PTR_ERR(wq); 2918 goto err_put_cq; 2919 } 2920 2921 wq->uobject = &obj->uevent.uobject; 2922 obj->uevent.uobject.object = wq; 2923 wq->wq_type = wq_init_attr.wq_type; 2924 wq->cq = cq; 2925 wq->pd = pd; 2926 wq->device = pd->device; 2927 wq->wq_context = wq_init_attr.wq_context; 2928 atomic_set(&wq->usecnt, 0); 2929 atomic_inc(&pd->usecnt); 2930 atomic_inc(&cq->usecnt); 2931 wq->uobject = &obj->uevent.uobject; 2932 obj->uevent.uobject.object = wq; 2933 2934 memset(&resp, 0, sizeof(resp)); 2935 resp.wq_handle = obj->uevent.uobject.id; 2936 resp.max_sge = wq_init_attr.max_sge; 2937 resp.max_wr = wq_init_attr.max_wr; 2938 resp.wqn = wq->wq_num; 2939 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 2940 err = uverbs_response(attrs, &resp, sizeof(resp)); 2941 if (err) 2942 goto err_copy; 2943 2944 uobj_put_obj_read(pd); 2945 uobj_put_obj_read(cq); 2946 return uobj_alloc_commit(&obj->uevent.uobject); 2947 2948 err_copy: 2949 ib_destroy_wq(wq); 2950 err_put_cq: 2951 uobj_put_obj_read(cq); 2952 err_put_pd: 2953 uobj_put_obj_read(pd); 2954 err_uobj: 2955 uobj_alloc_abort(&obj->uevent.uobject); 2956 2957 return err; 2958 } 2959 2960 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs) 2961 { 2962 struct ib_uverbs_ex_destroy_wq cmd; 2963 struct ib_uverbs_ex_destroy_wq_resp resp = {}; 2964 struct ib_uobject *uobj; 2965 struct ib_uwq_object *obj; 2966 int ret; 2967 2968 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2969 if (ret) 2970 return ret; 2971 2972 if (cmd.comp_mask) 2973 return -EOPNOTSUPP; 2974 2975 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 2976 uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs); 2977 if (IS_ERR(uobj)) 2978 return PTR_ERR(uobj); 2979 2980 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); 2981 resp.events_reported = obj->uevent.events_reported; 2982 2983 uobj_put_destroy(uobj); 2984 2985 return uverbs_response(attrs, &resp, sizeof(resp)); 2986 } 2987 2988 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs) 2989 { 2990 struct ib_uverbs_ex_modify_wq cmd; 2991 struct ib_wq *wq; 2992 struct ib_wq_attr wq_attr = {}; 2993 int ret; 2994 2995 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2996 if (ret) 2997 return ret; 2998 2999 if (!cmd.attr_mask) 3000 return -EINVAL; 3001 3002 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS)) 3003 return -EINVAL; 3004 3005 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs); 3006 if (!wq) 3007 return -EINVAL; 3008 3009 wq_attr.curr_wq_state = cmd.curr_wq_state; 3010 wq_attr.wq_state = cmd.wq_state; 3011 if (cmd.attr_mask & IB_WQ_FLAGS) { 3012 wq_attr.flags = cmd.flags; 3013 wq_attr.flags_mask = cmd.flags_mask; 3014 } 3015 ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask, 3016 &attrs->driver_udata); 3017 uobj_put_obj_read(wq); 3018 return ret; 3019 } 3020 3021 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) 3022 { 3023 struct ib_uverbs_ex_create_rwq_ind_table cmd; 3024 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; 3025 struct ib_uobject *uobj; 3026 int err; 3027 struct ib_rwq_ind_table_init_attr init_attr = {}; 3028 struct ib_rwq_ind_table *rwq_ind_tbl; 3029 struct ib_wq **wqs = NULL; 3030 u32 *wqs_handles = NULL; 3031 struct ib_wq *wq = NULL; 3032 int i, j, num_read_wqs; 3033 u32 num_wq_handles; 3034 struct uverbs_req_iter iter; 3035 struct ib_device *ib_dev; 3036 3037 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 3038 if (err) 3039 return err; 3040 3041 if (cmd.comp_mask) 3042 return -EOPNOTSUPP; 3043 3044 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE) 3045 return -EINVAL; 3046 3047 num_wq_handles = 1 << cmd.log_ind_tbl_size; 3048 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles), 3049 GFP_KERNEL); 3050 if (!wqs_handles) 3051 return -ENOMEM; 3052 3053 err = uverbs_request_next(&iter, wqs_handles, 3054 num_wq_handles * sizeof(__u32)); 3055 if (err) 3056 goto err_free; 3057 3058 err = uverbs_request_finish(&iter); 3059 if (err) 3060 goto err_free; 3061 3062 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL); 3063 if (!wqs) { 3064 err = -ENOMEM; 3065 goto err_free; 3066 } 3067 3068 for (num_read_wqs = 0; num_read_wqs < num_wq_handles; 3069 num_read_wqs++) { 3070 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, 3071 wqs_handles[num_read_wqs], attrs); 3072 if (!wq) { 3073 err = -EINVAL; 3074 goto put_wqs; 3075 } 3076 3077 wqs[num_read_wqs] = wq; 3078 } 3079 3080 uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev); 3081 if (IS_ERR(uobj)) { 3082 err = PTR_ERR(uobj); 3083 goto put_wqs; 3084 } 3085 3086 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; 3087 init_attr.ind_tbl = wqs; 3088 3089 rwq_ind_tbl = ib_dev->ops.create_rwq_ind_table(ib_dev, &init_attr, 3090 &attrs->driver_udata); 3091 3092 if (IS_ERR(rwq_ind_tbl)) { 3093 err = PTR_ERR(rwq_ind_tbl); 3094 goto err_uobj; 3095 } 3096 3097 rwq_ind_tbl->ind_tbl = wqs; 3098 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; 3099 rwq_ind_tbl->uobject = uobj; 3100 uobj->object = rwq_ind_tbl; 3101 rwq_ind_tbl->device = ib_dev; 3102 atomic_set(&rwq_ind_tbl->usecnt, 0); 3103 3104 for (i = 0; i < num_wq_handles; i++) 3105 atomic_inc(&wqs[i]->usecnt); 3106 3107 resp.ind_tbl_handle = uobj->id; 3108 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; 3109 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 3110 3111 err = uverbs_response(attrs, &resp, sizeof(resp)); 3112 if (err) 3113 goto err_copy; 3114 3115 kfree(wqs_handles); 3116 3117 for (j = 0; j < num_read_wqs; j++) 3118 uobj_put_obj_read(wqs[j]); 3119 3120 return uobj_alloc_commit(uobj); 3121 3122 err_copy: 3123 ib_destroy_rwq_ind_table(rwq_ind_tbl); 3124 err_uobj: 3125 uobj_alloc_abort(uobj); 3126 put_wqs: 3127 for (j = 0; j < num_read_wqs; j++) 3128 uobj_put_obj_read(wqs[j]); 3129 err_free: 3130 kfree(wqs_handles); 3131 kfree(wqs); 3132 return err; 3133 } 3134 3135 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs) 3136 { 3137 struct ib_uverbs_ex_destroy_rwq_ind_table cmd; 3138 int ret; 3139 3140 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3141 if (ret) 3142 return ret; 3143 3144 if (cmd.comp_mask) 3145 return -EOPNOTSUPP; 3146 3147 return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL, 3148 cmd.ind_tbl_handle, attrs); 3149 } 3150 3151 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) 3152 { 3153 struct ib_uverbs_create_flow cmd; 3154 struct ib_uverbs_create_flow_resp resp; 3155 struct ib_uobject *uobj; 3156 struct ib_flow *flow_id; 3157 struct ib_uverbs_flow_attr *kern_flow_attr; 3158 struct ib_flow_attr *flow_attr; 3159 struct ib_qp *qp; 3160 struct ib_uflow_resources *uflow_res; 3161 struct ib_uverbs_flow_spec_hdr *kern_spec; 3162 struct uverbs_req_iter iter; 3163 int err; 3164 void *ib_spec; 3165 int i; 3166 struct ib_device *ib_dev; 3167 3168 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 3169 if (err) 3170 return err; 3171 3172 if (cmd.comp_mask) 3173 return -EINVAL; 3174 3175 if (!capable(CAP_NET_RAW)) 3176 return -EPERM; 3177 3178 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED) 3179 return -EINVAL; 3180 3181 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && 3182 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) || 3183 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT))) 3184 return -EINVAL; 3185 3186 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 3187 return -EINVAL; 3188 3189 if (cmd.flow_attr.size > 3190 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 3191 return -EINVAL; 3192 3193 if (cmd.flow_attr.reserved[0] || 3194 cmd.flow_attr.reserved[1]) 3195 return -EINVAL; 3196 3197 if (cmd.flow_attr.num_of_specs) { 3198 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 3199 GFP_KERNEL); 3200 if (!kern_flow_attr) 3201 return -ENOMEM; 3202 3203 *kern_flow_attr = cmd.flow_attr; 3204 err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs, 3205 cmd.flow_attr.size); 3206 if (err) 3207 goto err_free_attr; 3208 } else { 3209 kern_flow_attr = &cmd.flow_attr; 3210 } 3211 3212 err = uverbs_request_finish(&iter); 3213 if (err) 3214 goto err_free_attr; 3215 3216 uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev); 3217 if (IS_ERR(uobj)) { 3218 err = PTR_ERR(uobj); 3219 goto err_free_attr; 3220 } 3221 3222 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 3223 if (!qp) { 3224 err = -EINVAL; 3225 goto err_uobj; 3226 } 3227 3228 if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) { 3229 err = -EINVAL; 3230 goto err_put; 3231 } 3232 3233 flow_attr = kzalloc(struct_size(flow_attr, flows, 3234 cmd.flow_attr.num_of_specs), GFP_KERNEL); 3235 if (!flow_attr) { 3236 err = -ENOMEM; 3237 goto err_put; 3238 } 3239 uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs); 3240 if (!uflow_res) { 3241 err = -ENOMEM; 3242 goto err_free_flow_attr; 3243 } 3244 3245 flow_attr->type = kern_flow_attr->type; 3246 flow_attr->priority = kern_flow_attr->priority; 3247 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 3248 flow_attr->port = kern_flow_attr->port; 3249 flow_attr->flags = kern_flow_attr->flags; 3250 flow_attr->size = sizeof(*flow_attr); 3251 3252 kern_spec = kern_flow_attr->flow_specs; 3253 ib_spec = flow_attr + 1; 3254 for (i = 0; i < flow_attr->num_of_specs && 3255 cmd.flow_attr.size >= sizeof(*kern_spec) && 3256 cmd.flow_attr.size >= kern_spec->size; 3257 i++) { 3258 err = kern_spec_to_ib_spec( 3259 attrs, (struct ib_uverbs_flow_spec *)kern_spec, 3260 ib_spec, uflow_res); 3261 if (err) 3262 goto err_free; 3263 3264 flow_attr->size += 3265 ((union ib_flow_spec *) ib_spec)->size; 3266 cmd.flow_attr.size -= kern_spec->size; 3267 kern_spec = ((void *)kern_spec) + kern_spec->size; 3268 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 3269 } 3270 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 3271 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 3272 i, cmd.flow_attr.size); 3273 err = -EINVAL; 3274 goto err_free; 3275 } 3276 3277 flow_id = qp->device->ops.create_flow( 3278 qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata); 3279 3280 if (IS_ERR(flow_id)) { 3281 err = PTR_ERR(flow_id); 3282 goto err_free; 3283 } 3284 3285 ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res); 3286 3287 memset(&resp, 0, sizeof(resp)); 3288 resp.flow_handle = uobj->id; 3289 3290 err = uverbs_response(attrs, &resp, sizeof(resp)); 3291 if (err) 3292 goto err_copy; 3293 3294 uobj_put_obj_read(qp); 3295 kfree(flow_attr); 3296 if (cmd.flow_attr.num_of_specs) 3297 kfree(kern_flow_attr); 3298 return uobj_alloc_commit(uobj); 3299 err_copy: 3300 if (!qp->device->ops.destroy_flow(flow_id)) 3301 atomic_dec(&qp->usecnt); 3302 err_free: 3303 ib_uverbs_flow_resources_free(uflow_res); 3304 err_free_flow_attr: 3305 kfree(flow_attr); 3306 err_put: 3307 uobj_put_obj_read(qp); 3308 err_uobj: 3309 uobj_alloc_abort(uobj); 3310 err_free_attr: 3311 if (cmd.flow_attr.num_of_specs) 3312 kfree(kern_flow_attr); 3313 return err; 3314 } 3315 3316 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs) 3317 { 3318 struct ib_uverbs_destroy_flow cmd; 3319 int ret; 3320 3321 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3322 if (ret) 3323 return ret; 3324 3325 if (cmd.comp_mask) 3326 return -EINVAL; 3327 3328 return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs); 3329 } 3330 3331 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, 3332 struct ib_uverbs_create_xsrq *cmd, 3333 struct ib_udata *udata) 3334 { 3335 struct ib_uverbs_create_srq_resp resp; 3336 struct ib_usrq_object *obj; 3337 struct ib_pd *pd; 3338 struct ib_srq *srq; 3339 struct ib_uobject *uninitialized_var(xrcd_uobj); 3340 struct ib_srq_init_attr attr; 3341 int ret; 3342 struct ib_device *ib_dev; 3343 3344 obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs, 3345 &ib_dev); 3346 if (IS_ERR(obj)) 3347 return PTR_ERR(obj); 3348 3349 if (cmd->srq_type == IB_SRQT_TM) 3350 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags; 3351 3352 if (cmd->srq_type == IB_SRQT_XRC) { 3353 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle, 3354 attrs); 3355 if (IS_ERR(xrcd_uobj)) { 3356 ret = -EINVAL; 3357 goto err; 3358 } 3359 3360 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object; 3361 if (!attr.ext.xrc.xrcd) { 3362 ret = -EINVAL; 3363 goto err_put_xrcd; 3364 } 3365 3366 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 3367 atomic_inc(&obj->uxrcd->refcnt); 3368 } 3369 3370 if (ib_srq_has_cq(cmd->srq_type)) { 3371 attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 3372 cmd->cq_handle, attrs); 3373 if (!attr.ext.cq) { 3374 ret = -EINVAL; 3375 goto err_put_xrcd; 3376 } 3377 } 3378 3379 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs); 3380 if (!pd) { 3381 ret = -EINVAL; 3382 goto err_put_cq; 3383 } 3384 3385 attr.event_handler = ib_uverbs_srq_event_handler; 3386 attr.srq_context = attrs->ufile; 3387 attr.srq_type = cmd->srq_type; 3388 attr.attr.max_wr = cmd->max_wr; 3389 attr.attr.max_sge = cmd->max_sge; 3390 attr.attr.srq_limit = cmd->srq_limit; 3391 3392 obj->uevent.events_reported = 0; 3393 INIT_LIST_HEAD(&obj->uevent.event_list); 3394 3395 srq = pd->device->ops.create_srq(pd, &attr, udata); 3396 if (IS_ERR(srq)) { 3397 ret = PTR_ERR(srq); 3398 goto err_put; 3399 } 3400 3401 srq->device = pd->device; 3402 srq->pd = pd; 3403 srq->srq_type = cmd->srq_type; 3404 srq->uobject = &obj->uevent.uobject; 3405 srq->event_handler = attr.event_handler; 3406 srq->srq_context = attr.srq_context; 3407 3408 if (ib_srq_has_cq(cmd->srq_type)) { 3409 srq->ext.cq = attr.ext.cq; 3410 atomic_inc(&attr.ext.cq->usecnt); 3411 } 3412 3413 if (cmd->srq_type == IB_SRQT_XRC) { 3414 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 3415 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 3416 } 3417 3418 atomic_inc(&pd->usecnt); 3419 atomic_set(&srq->usecnt, 0); 3420 3421 obj->uevent.uobject.object = srq; 3422 obj->uevent.uobject.user_handle = cmd->user_handle; 3423 3424 memset(&resp, 0, sizeof resp); 3425 resp.srq_handle = obj->uevent.uobject.id; 3426 resp.max_wr = attr.attr.max_wr; 3427 resp.max_sge = attr.attr.max_sge; 3428 if (cmd->srq_type == IB_SRQT_XRC) 3429 resp.srqn = srq->ext.xrc.srq_num; 3430 3431 ret = uverbs_response(attrs, &resp, sizeof(resp)); 3432 if (ret) 3433 goto err_copy; 3434 3435 if (cmd->srq_type == IB_SRQT_XRC) 3436 uobj_put_read(xrcd_uobj); 3437 3438 if (ib_srq_has_cq(cmd->srq_type)) 3439 uobj_put_obj_read(attr.ext.cq); 3440 3441 uobj_put_obj_read(pd); 3442 return uobj_alloc_commit(&obj->uevent.uobject); 3443 3444 err_copy: 3445 ib_destroy_srq(srq); 3446 3447 err_put: 3448 uobj_put_obj_read(pd); 3449 3450 err_put_cq: 3451 if (ib_srq_has_cq(cmd->srq_type)) 3452 uobj_put_obj_read(attr.ext.cq); 3453 3454 err_put_xrcd: 3455 if (cmd->srq_type == IB_SRQT_XRC) { 3456 atomic_dec(&obj->uxrcd->refcnt); 3457 uobj_put_read(xrcd_uobj); 3458 } 3459 3460 err: 3461 uobj_alloc_abort(&obj->uevent.uobject); 3462 return ret; 3463 } 3464 3465 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs) 3466 { 3467 struct ib_uverbs_create_srq cmd; 3468 struct ib_uverbs_create_xsrq xcmd; 3469 int ret; 3470 3471 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3472 if (ret) 3473 return ret; 3474 3475 memset(&xcmd, 0, sizeof(xcmd)); 3476 xcmd.response = cmd.response; 3477 xcmd.user_handle = cmd.user_handle; 3478 xcmd.srq_type = IB_SRQT_BASIC; 3479 xcmd.pd_handle = cmd.pd_handle; 3480 xcmd.max_wr = cmd.max_wr; 3481 xcmd.max_sge = cmd.max_sge; 3482 xcmd.srq_limit = cmd.srq_limit; 3483 3484 return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata); 3485 } 3486 3487 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs) 3488 { 3489 struct ib_uverbs_create_xsrq cmd; 3490 int ret; 3491 3492 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3493 if (ret) 3494 return ret; 3495 3496 return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata); 3497 } 3498 3499 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs) 3500 { 3501 struct ib_uverbs_modify_srq cmd; 3502 struct ib_srq *srq; 3503 struct ib_srq_attr attr; 3504 int ret; 3505 3506 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3507 if (ret) 3508 return ret; 3509 3510 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 3511 if (!srq) 3512 return -EINVAL; 3513 3514 attr.max_wr = cmd.max_wr; 3515 attr.srq_limit = cmd.srq_limit; 3516 3517 ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask, 3518 &attrs->driver_udata); 3519 3520 uobj_put_obj_read(srq); 3521 3522 return ret; 3523 } 3524 3525 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs) 3526 { 3527 struct ib_uverbs_query_srq cmd; 3528 struct ib_uverbs_query_srq_resp resp; 3529 struct ib_srq_attr attr; 3530 struct ib_srq *srq; 3531 int ret; 3532 3533 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3534 if (ret) 3535 return ret; 3536 3537 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 3538 if (!srq) 3539 return -EINVAL; 3540 3541 ret = ib_query_srq(srq, &attr); 3542 3543 uobj_put_obj_read(srq); 3544 3545 if (ret) 3546 return ret; 3547 3548 memset(&resp, 0, sizeof resp); 3549 3550 resp.max_wr = attr.max_wr; 3551 resp.max_sge = attr.max_sge; 3552 resp.srq_limit = attr.srq_limit; 3553 3554 return uverbs_response(attrs, &resp, sizeof(resp)); 3555 } 3556 3557 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs) 3558 { 3559 struct ib_uverbs_destroy_srq cmd; 3560 struct ib_uverbs_destroy_srq_resp resp; 3561 struct ib_uobject *uobj; 3562 struct ib_uevent_object *obj; 3563 int ret; 3564 3565 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3566 if (ret) 3567 return ret; 3568 3569 uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 3570 if (IS_ERR(uobj)) 3571 return PTR_ERR(uobj); 3572 3573 obj = container_of(uobj, struct ib_uevent_object, uobject); 3574 memset(&resp, 0, sizeof(resp)); 3575 resp.events_reported = obj->events_reported; 3576 3577 uobj_put_destroy(uobj); 3578 3579 return uverbs_response(attrs, &resp, sizeof(resp)); 3580 } 3581 3582 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs) 3583 { 3584 struct ib_uverbs_ex_query_device_resp resp = {}; 3585 struct ib_uverbs_ex_query_device cmd; 3586 struct ib_device_attr attr = {0}; 3587 struct ib_ucontext *ucontext; 3588 struct ib_device *ib_dev; 3589 int err; 3590 3591 ucontext = ib_uverbs_get_ucontext(attrs); 3592 if (IS_ERR(ucontext)) 3593 return PTR_ERR(ucontext); 3594 ib_dev = ucontext->device; 3595 3596 err = uverbs_request(attrs, &cmd, sizeof(cmd)); 3597 if (err) 3598 return err; 3599 3600 if (cmd.comp_mask) 3601 return -EINVAL; 3602 3603 if (cmd.reserved) 3604 return -EINVAL; 3605 3606 err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata); 3607 if (err) 3608 return err; 3609 3610 copy_query_dev_fields(ucontext, &resp.base, &attr); 3611 3612 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 3613 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 3614 resp.odp_caps.per_transport_caps.rc_odp_caps = 3615 attr.odp_caps.per_transport_caps.rc_odp_caps; 3616 resp.odp_caps.per_transport_caps.uc_odp_caps = 3617 attr.odp_caps.per_transport_caps.uc_odp_caps; 3618 resp.odp_caps.per_transport_caps.ud_odp_caps = 3619 attr.odp_caps.per_transport_caps.ud_odp_caps; 3620 #endif 3621 3622 resp.timestamp_mask = attr.timestamp_mask; 3623 resp.hca_core_clock = attr.hca_core_clock; 3624 resp.device_cap_flags_ex = attr.device_cap_flags; 3625 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; 3626 resp.rss_caps.max_rwq_indirection_tables = 3627 attr.rss_caps.max_rwq_indirection_tables; 3628 resp.rss_caps.max_rwq_indirection_table_size = 3629 attr.rss_caps.max_rwq_indirection_table_size; 3630 resp.max_wq_type_rq = attr.max_wq_type_rq; 3631 resp.raw_packet_caps = attr.raw_packet_caps; 3632 resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size; 3633 resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags; 3634 resp.tm_caps.max_ops = attr.tm_caps.max_ops; 3635 resp.tm_caps.max_sge = attr.tm_caps.max_sge; 3636 resp.tm_caps.flags = attr.tm_caps.flags; 3637 resp.cq_moderation_caps.max_cq_moderation_count = 3638 attr.cq_caps.max_cq_moderation_count; 3639 resp.cq_moderation_caps.max_cq_moderation_period = 3640 attr.cq_caps.max_cq_moderation_period; 3641 resp.max_dm_size = attr.max_dm_size; 3642 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 3643 3644 return uverbs_response(attrs, &resp, sizeof(resp)); 3645 } 3646 3647 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs) 3648 { 3649 struct ib_uverbs_ex_modify_cq cmd; 3650 struct ib_cq *cq; 3651 int ret; 3652 3653 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3654 if (ret) 3655 return ret; 3656 3657 if (!cmd.attr_mask || cmd.reserved) 3658 return -EINVAL; 3659 3660 if (cmd.attr_mask > IB_CQ_MODERATE) 3661 return -EOPNOTSUPP; 3662 3663 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 3664 if (!cq) 3665 return -EINVAL; 3666 3667 ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period); 3668 3669 uobj_put_obj_read(cq); 3670 3671 return ret; 3672 } 3673 3674 /* 3675 * Describe the input structs for write(). Some write methods have an input 3676 * only struct, most have an input and output. If the struct has an output then 3677 * the 'response' u64 must be the first field in the request structure. 3678 * 3679 * If udata is present then both the request and response structs have a 3680 * trailing driver_data flex array. In this case the size of the base struct 3681 * cannot be changed. 3682 */ 3683 #define offsetof_after(_struct, _member) \ 3684 (offsetof(_struct, _member) + sizeof(((_struct *)NULL)->_member)) 3685 3686 #define UAPI_DEF_WRITE_IO(req, resp) \ 3687 .write.has_resp = 1 + \ 3688 BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \ 3689 BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) != \ 3690 sizeof(u64)), \ 3691 .write.req_size = sizeof(req), .write.resp_size = sizeof(resp) 3692 3693 #define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req) 3694 3695 #define UAPI_DEF_WRITE_UDATA_IO(req, resp) \ 3696 UAPI_DEF_WRITE_IO(req, resp), \ 3697 .write.has_udata = \ 3698 1 + \ 3699 BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \ 3700 sizeof(req)) + \ 3701 BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \ 3702 sizeof(resp)) 3703 3704 #define UAPI_DEF_WRITE_UDATA_I(req) \ 3705 UAPI_DEF_WRITE_I(req), \ 3706 .write.has_udata = \ 3707 1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \ 3708 sizeof(req)) 3709 3710 /* 3711 * The _EX versions are for use with WRITE_EX and allow the last struct member 3712 * to be specified. Buffers that do not include that member will be rejected. 3713 */ 3714 #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \ 3715 .write.has_resp = 1, \ 3716 .write.req_size = offsetof_after(req, req_last_member), \ 3717 .write.resp_size = offsetof_after(resp, resp_last_member) 3718 3719 #define UAPI_DEF_WRITE_I_EX(req, req_last_member) \ 3720 .write.req_size = offsetof_after(req, req_last_member) 3721 3722 const struct uapi_definition uverbs_def_write_intf[] = { 3723 DECLARE_UVERBS_OBJECT( 3724 UVERBS_OBJECT_AH, 3725 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH, 3726 ib_uverbs_create_ah, 3727 UAPI_DEF_WRITE_UDATA_IO( 3728 struct ib_uverbs_create_ah, 3729 struct ib_uverbs_create_ah_resp), 3730 UAPI_DEF_METHOD_NEEDS_FN(create_ah)), 3731 DECLARE_UVERBS_WRITE( 3732 IB_USER_VERBS_CMD_DESTROY_AH, 3733 ib_uverbs_destroy_ah, 3734 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah), 3735 UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))), 3736 3737 DECLARE_UVERBS_OBJECT( 3738 UVERBS_OBJECT_COMP_CHANNEL, 3739 DECLARE_UVERBS_WRITE( 3740 IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL, 3741 ib_uverbs_create_comp_channel, 3742 UAPI_DEF_WRITE_IO( 3743 struct ib_uverbs_create_comp_channel, 3744 struct ib_uverbs_create_comp_channel_resp))), 3745 3746 DECLARE_UVERBS_OBJECT( 3747 UVERBS_OBJECT_CQ, 3748 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ, 3749 ib_uverbs_create_cq, 3750 UAPI_DEF_WRITE_UDATA_IO( 3751 struct ib_uverbs_create_cq, 3752 struct ib_uverbs_create_cq_resp), 3753 UAPI_DEF_METHOD_NEEDS_FN(create_cq)), 3754 DECLARE_UVERBS_WRITE( 3755 IB_USER_VERBS_CMD_DESTROY_CQ, 3756 ib_uverbs_destroy_cq, 3757 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq, 3758 struct ib_uverbs_destroy_cq_resp), 3759 UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)), 3760 DECLARE_UVERBS_WRITE( 3761 IB_USER_VERBS_CMD_POLL_CQ, 3762 ib_uverbs_poll_cq, 3763 UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq, 3764 struct ib_uverbs_poll_cq_resp), 3765 UAPI_DEF_METHOD_NEEDS_FN(poll_cq)), 3766 DECLARE_UVERBS_WRITE( 3767 IB_USER_VERBS_CMD_REQ_NOTIFY_CQ, 3768 ib_uverbs_req_notify_cq, 3769 UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq), 3770 UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)), 3771 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ, 3772 ib_uverbs_resize_cq, 3773 UAPI_DEF_WRITE_UDATA_IO( 3774 struct ib_uverbs_resize_cq, 3775 struct ib_uverbs_resize_cq_resp), 3776 UAPI_DEF_METHOD_NEEDS_FN(resize_cq)), 3777 DECLARE_UVERBS_WRITE_EX( 3778 IB_USER_VERBS_EX_CMD_CREATE_CQ, 3779 ib_uverbs_ex_create_cq, 3780 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq, 3781 reserved, 3782 struct ib_uverbs_ex_create_cq_resp, 3783 response_length), 3784 UAPI_DEF_METHOD_NEEDS_FN(create_cq)), 3785 DECLARE_UVERBS_WRITE_EX( 3786 IB_USER_VERBS_EX_CMD_MODIFY_CQ, 3787 ib_uverbs_ex_modify_cq, 3788 UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq), 3789 UAPI_DEF_METHOD_NEEDS_FN(create_cq))), 3790 3791 DECLARE_UVERBS_OBJECT( 3792 UVERBS_OBJECT_DEVICE, 3793 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT, 3794 ib_uverbs_get_context, 3795 UAPI_DEF_WRITE_UDATA_IO( 3796 struct ib_uverbs_get_context, 3797 struct ib_uverbs_get_context_resp)), 3798 DECLARE_UVERBS_WRITE( 3799 IB_USER_VERBS_CMD_QUERY_DEVICE, 3800 ib_uverbs_query_device, 3801 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device, 3802 struct ib_uverbs_query_device_resp)), 3803 DECLARE_UVERBS_WRITE( 3804 IB_USER_VERBS_CMD_QUERY_PORT, 3805 ib_uverbs_query_port, 3806 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port, 3807 struct ib_uverbs_query_port_resp), 3808 UAPI_DEF_METHOD_NEEDS_FN(query_port)), 3809 DECLARE_UVERBS_WRITE_EX( 3810 IB_USER_VERBS_EX_CMD_QUERY_DEVICE, 3811 ib_uverbs_ex_query_device, 3812 UAPI_DEF_WRITE_IO_EX( 3813 struct ib_uverbs_ex_query_device, 3814 reserved, 3815 struct ib_uverbs_ex_query_device_resp, 3816 response_length), 3817 UAPI_DEF_METHOD_NEEDS_FN(query_device)), 3818 UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext), 3819 UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)), 3820 3821 DECLARE_UVERBS_OBJECT( 3822 UVERBS_OBJECT_FLOW, 3823 DECLARE_UVERBS_WRITE_EX( 3824 IB_USER_VERBS_EX_CMD_CREATE_FLOW, 3825 ib_uverbs_ex_create_flow, 3826 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow, 3827 flow_attr, 3828 struct ib_uverbs_create_flow_resp, 3829 flow_handle), 3830 UAPI_DEF_METHOD_NEEDS_FN(create_flow)), 3831 DECLARE_UVERBS_WRITE_EX( 3832 IB_USER_VERBS_EX_CMD_DESTROY_FLOW, 3833 ib_uverbs_ex_destroy_flow, 3834 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow), 3835 UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))), 3836 3837 DECLARE_UVERBS_OBJECT( 3838 UVERBS_OBJECT_MR, 3839 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR, 3840 ib_uverbs_dereg_mr, 3841 UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr), 3842 UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)), 3843 DECLARE_UVERBS_WRITE( 3844 IB_USER_VERBS_CMD_REG_MR, 3845 ib_uverbs_reg_mr, 3846 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr, 3847 struct ib_uverbs_reg_mr_resp), 3848 UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)), 3849 DECLARE_UVERBS_WRITE( 3850 IB_USER_VERBS_CMD_REREG_MR, 3851 ib_uverbs_rereg_mr, 3852 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr, 3853 struct ib_uverbs_rereg_mr_resp), 3854 UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))), 3855 3856 DECLARE_UVERBS_OBJECT( 3857 UVERBS_OBJECT_MW, 3858 DECLARE_UVERBS_WRITE( 3859 IB_USER_VERBS_CMD_ALLOC_MW, 3860 ib_uverbs_alloc_mw, 3861 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw, 3862 struct ib_uverbs_alloc_mw_resp), 3863 UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)), 3864 DECLARE_UVERBS_WRITE( 3865 IB_USER_VERBS_CMD_DEALLOC_MW, 3866 ib_uverbs_dealloc_mw, 3867 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw), 3868 UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))), 3869 3870 DECLARE_UVERBS_OBJECT( 3871 UVERBS_OBJECT_PD, 3872 DECLARE_UVERBS_WRITE( 3873 IB_USER_VERBS_CMD_ALLOC_PD, 3874 ib_uverbs_alloc_pd, 3875 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd, 3876 struct ib_uverbs_alloc_pd_resp), 3877 UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)), 3878 DECLARE_UVERBS_WRITE( 3879 IB_USER_VERBS_CMD_DEALLOC_PD, 3880 ib_uverbs_dealloc_pd, 3881 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd), 3882 UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))), 3883 3884 DECLARE_UVERBS_OBJECT( 3885 UVERBS_OBJECT_QP, 3886 DECLARE_UVERBS_WRITE( 3887 IB_USER_VERBS_CMD_ATTACH_MCAST, 3888 ib_uverbs_attach_mcast, 3889 UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast), 3890 UAPI_DEF_METHOD_NEEDS_FN(attach_mcast), 3891 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)), 3892 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP, 3893 ib_uverbs_create_qp, 3894 UAPI_DEF_WRITE_UDATA_IO( 3895 struct ib_uverbs_create_qp, 3896 struct ib_uverbs_create_qp_resp), 3897 UAPI_DEF_METHOD_NEEDS_FN(create_qp)), 3898 DECLARE_UVERBS_WRITE( 3899 IB_USER_VERBS_CMD_DESTROY_QP, 3900 ib_uverbs_destroy_qp, 3901 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp, 3902 struct ib_uverbs_destroy_qp_resp), 3903 UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)), 3904 DECLARE_UVERBS_WRITE( 3905 IB_USER_VERBS_CMD_DETACH_MCAST, 3906 ib_uverbs_detach_mcast, 3907 UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast), 3908 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)), 3909 DECLARE_UVERBS_WRITE( 3910 IB_USER_VERBS_CMD_MODIFY_QP, 3911 ib_uverbs_modify_qp, 3912 UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp), 3913 UAPI_DEF_METHOD_NEEDS_FN(modify_qp)), 3914 DECLARE_UVERBS_WRITE( 3915 IB_USER_VERBS_CMD_POST_RECV, 3916 ib_uverbs_post_recv, 3917 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv, 3918 struct ib_uverbs_post_recv_resp), 3919 UAPI_DEF_METHOD_NEEDS_FN(post_recv)), 3920 DECLARE_UVERBS_WRITE( 3921 IB_USER_VERBS_CMD_POST_SEND, 3922 ib_uverbs_post_send, 3923 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send, 3924 struct ib_uverbs_post_send_resp), 3925 UAPI_DEF_METHOD_NEEDS_FN(post_send)), 3926 DECLARE_UVERBS_WRITE( 3927 IB_USER_VERBS_CMD_QUERY_QP, 3928 ib_uverbs_query_qp, 3929 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp, 3930 struct ib_uverbs_query_qp_resp), 3931 UAPI_DEF_METHOD_NEEDS_FN(query_qp)), 3932 DECLARE_UVERBS_WRITE_EX( 3933 IB_USER_VERBS_EX_CMD_CREATE_QP, 3934 ib_uverbs_ex_create_qp, 3935 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp, 3936 comp_mask, 3937 struct ib_uverbs_ex_create_qp_resp, 3938 response_length), 3939 UAPI_DEF_METHOD_NEEDS_FN(create_qp)), 3940 DECLARE_UVERBS_WRITE_EX( 3941 IB_USER_VERBS_EX_CMD_MODIFY_QP, 3942 ib_uverbs_ex_modify_qp, 3943 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp, 3944 base, 3945 struct ib_uverbs_ex_modify_qp_resp, 3946 response_length), 3947 UAPI_DEF_METHOD_NEEDS_FN(modify_qp))), 3948 3949 DECLARE_UVERBS_OBJECT( 3950 UVERBS_OBJECT_RWQ_IND_TBL, 3951 DECLARE_UVERBS_WRITE_EX( 3952 IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL, 3953 ib_uverbs_ex_create_rwq_ind_table, 3954 UAPI_DEF_WRITE_IO_EX( 3955 struct ib_uverbs_ex_create_rwq_ind_table, 3956 log_ind_tbl_size, 3957 struct ib_uverbs_ex_create_rwq_ind_table_resp, 3958 ind_tbl_num), 3959 UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)), 3960 DECLARE_UVERBS_WRITE_EX( 3961 IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL, 3962 ib_uverbs_ex_destroy_rwq_ind_table, 3963 UAPI_DEF_WRITE_I( 3964 struct ib_uverbs_ex_destroy_rwq_ind_table), 3965 UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))), 3966 3967 DECLARE_UVERBS_OBJECT( 3968 UVERBS_OBJECT_WQ, 3969 DECLARE_UVERBS_WRITE_EX( 3970 IB_USER_VERBS_EX_CMD_CREATE_WQ, 3971 ib_uverbs_ex_create_wq, 3972 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq, 3973 max_sge, 3974 struct ib_uverbs_ex_create_wq_resp, 3975 wqn), 3976 UAPI_DEF_METHOD_NEEDS_FN(create_wq)), 3977 DECLARE_UVERBS_WRITE_EX( 3978 IB_USER_VERBS_EX_CMD_DESTROY_WQ, 3979 ib_uverbs_ex_destroy_wq, 3980 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq, 3981 wq_handle, 3982 struct ib_uverbs_ex_destroy_wq_resp, 3983 reserved), 3984 UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)), 3985 DECLARE_UVERBS_WRITE_EX( 3986 IB_USER_VERBS_EX_CMD_MODIFY_WQ, 3987 ib_uverbs_ex_modify_wq, 3988 UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq, 3989 curr_wq_state), 3990 UAPI_DEF_METHOD_NEEDS_FN(modify_wq))), 3991 3992 DECLARE_UVERBS_OBJECT( 3993 UVERBS_OBJECT_SRQ, 3994 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ, 3995 ib_uverbs_create_srq, 3996 UAPI_DEF_WRITE_UDATA_IO( 3997 struct ib_uverbs_create_srq, 3998 struct ib_uverbs_create_srq_resp), 3999 UAPI_DEF_METHOD_NEEDS_FN(create_srq)), 4000 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ, 4001 ib_uverbs_create_xsrq, 4002 UAPI_DEF_WRITE_UDATA_IO( 4003 struct ib_uverbs_create_xsrq, 4004 struct ib_uverbs_create_srq_resp), 4005 UAPI_DEF_METHOD_NEEDS_FN(create_srq)), 4006 DECLARE_UVERBS_WRITE( 4007 IB_USER_VERBS_CMD_DESTROY_SRQ, 4008 ib_uverbs_destroy_srq, 4009 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq, 4010 struct ib_uverbs_destroy_srq_resp), 4011 UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)), 4012 DECLARE_UVERBS_WRITE( 4013 IB_USER_VERBS_CMD_MODIFY_SRQ, 4014 ib_uverbs_modify_srq, 4015 UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq), 4016 UAPI_DEF_METHOD_NEEDS_FN(modify_srq)), 4017 DECLARE_UVERBS_WRITE( 4018 IB_USER_VERBS_CMD_POST_SRQ_RECV, 4019 ib_uverbs_post_srq_recv, 4020 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv, 4021 struct ib_uverbs_post_srq_recv_resp), 4022 UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)), 4023 DECLARE_UVERBS_WRITE( 4024 IB_USER_VERBS_CMD_QUERY_SRQ, 4025 ib_uverbs_query_srq, 4026 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq, 4027 struct ib_uverbs_query_srq_resp), 4028 UAPI_DEF_METHOD_NEEDS_FN(query_srq))), 4029 4030 DECLARE_UVERBS_OBJECT( 4031 UVERBS_OBJECT_XRCD, 4032 DECLARE_UVERBS_WRITE( 4033 IB_USER_VERBS_CMD_CLOSE_XRCD, 4034 ib_uverbs_close_xrcd, 4035 UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd), 4036 UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)), 4037 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP, 4038 ib_uverbs_open_qp, 4039 UAPI_DEF_WRITE_UDATA_IO( 4040 struct ib_uverbs_open_qp, 4041 struct ib_uverbs_create_qp_resp)), 4042 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD, 4043 ib_uverbs_open_xrcd, 4044 UAPI_DEF_WRITE_UDATA_IO( 4045 struct ib_uverbs_open_xrcd, 4046 struct ib_uverbs_open_xrcd_resp), 4047 UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))), 4048 4049 {}, 4050 }; 4051