1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <linux/uaccess.h> 42 43 #include "uverbs.h" 44 #include "core_priv.h" 45 46 struct uverbs_lock_class { 47 struct lock_class_key key; 48 char name[16]; 49 }; 50 51 static struct uverbs_lock_class pd_lock_class = { .name = "PD-uobj" }; 52 static struct uverbs_lock_class mr_lock_class = { .name = "MR-uobj" }; 53 static struct uverbs_lock_class mw_lock_class = { .name = "MW-uobj" }; 54 static struct uverbs_lock_class cq_lock_class = { .name = "CQ-uobj" }; 55 static struct uverbs_lock_class qp_lock_class = { .name = "QP-uobj" }; 56 static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" }; 57 static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" }; 58 static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" }; 59 static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" }; 60 static struct uverbs_lock_class wq_lock_class = { .name = "WQ-uobj" }; 61 static struct uverbs_lock_class rwq_ind_table_lock_class = { .name = "IND_TBL-uobj" }; 62 63 /* 64 * The ib_uobject locking scheme is as follows: 65 * 66 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it 67 * needs to be held during all idr write operations. When an object is 68 * looked up, a reference must be taken on the object's kref before 69 * dropping this lock. For read operations, the rcu_read_lock() 70 * and rcu_write_lock() but similarly the kref reference is grabbed 71 * before the rcu_read_unlock(). 72 * 73 * - Each object also has an rwsem. This rwsem must be held for 74 * reading while an operation that uses the object is performed. 75 * For example, while registering an MR, the associated PD's 76 * uobject.mutex must be held for reading. The rwsem must be held 77 * for writing while initializing or destroying an object. 78 * 79 * - In addition, each object has a "live" flag. If this flag is not 80 * set, then lookups of the object will fail even if it is found in 81 * the idr. This handles a reader that blocks and does not acquire 82 * the rwsem until after the object is destroyed. The destroy 83 * operation will set the live flag to 0 and then drop the rwsem; 84 * this will allow the reader to acquire the rwsem, see that the 85 * live flag is 0, and then drop the rwsem and its reference to 86 * object. The underlying storage will not be freed until the last 87 * reference to the object is dropped. 88 */ 89 90 static void init_uobj(struct ib_uobject *uobj, u64 user_handle, 91 struct ib_ucontext *context, struct uverbs_lock_class *c) 92 { 93 uobj->user_handle = user_handle; 94 uobj->context = context; 95 kref_init(&uobj->ref); 96 init_rwsem(&uobj->mutex); 97 lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name); 98 uobj->live = 0; 99 } 100 101 static void release_uobj(struct kref *kref) 102 { 103 kfree_rcu(container_of(kref, struct ib_uobject, ref), rcu); 104 } 105 106 static void put_uobj(struct ib_uobject *uobj) 107 { 108 kref_put(&uobj->ref, release_uobj); 109 } 110 111 static void put_uobj_read(struct ib_uobject *uobj) 112 { 113 up_read(&uobj->mutex); 114 put_uobj(uobj); 115 } 116 117 static void put_uobj_write(struct ib_uobject *uobj) 118 { 119 up_write(&uobj->mutex); 120 put_uobj(uobj); 121 } 122 123 static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) 124 { 125 int ret; 126 127 idr_preload(GFP_KERNEL); 128 spin_lock(&ib_uverbs_idr_lock); 129 130 ret = idr_alloc(idr, uobj, 0, 0, GFP_NOWAIT); 131 if (ret >= 0) 132 uobj->id = ret; 133 134 spin_unlock(&ib_uverbs_idr_lock); 135 idr_preload_end(); 136 137 return ret < 0 ? ret : 0; 138 } 139 140 void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) 141 { 142 spin_lock(&ib_uverbs_idr_lock); 143 idr_remove(idr, uobj->id); 144 spin_unlock(&ib_uverbs_idr_lock); 145 } 146 147 static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, 148 struct ib_ucontext *context) 149 { 150 struct ib_uobject *uobj; 151 152 rcu_read_lock(); 153 uobj = idr_find(idr, id); 154 if (uobj) { 155 if (uobj->context == context) 156 kref_get(&uobj->ref); 157 else 158 uobj = NULL; 159 } 160 rcu_read_unlock(); 161 162 return uobj; 163 } 164 165 static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, 166 struct ib_ucontext *context, int nested) 167 { 168 struct ib_uobject *uobj; 169 170 uobj = __idr_get_uobj(idr, id, context); 171 if (!uobj) 172 return NULL; 173 174 if (nested) 175 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); 176 else 177 down_read(&uobj->mutex); 178 if (!uobj->live) { 179 put_uobj_read(uobj); 180 return NULL; 181 } 182 183 return uobj; 184 } 185 186 static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, 187 struct ib_ucontext *context) 188 { 189 struct ib_uobject *uobj; 190 191 uobj = __idr_get_uobj(idr, id, context); 192 if (!uobj) 193 return NULL; 194 195 down_write(&uobj->mutex); 196 if (!uobj->live) { 197 put_uobj_write(uobj); 198 return NULL; 199 } 200 201 return uobj; 202 } 203 204 static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, 205 int nested) 206 { 207 struct ib_uobject *uobj; 208 209 uobj = idr_read_uobj(idr, id, context, nested); 210 return uobj ? uobj->object : NULL; 211 } 212 213 static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) 214 { 215 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); 216 } 217 218 static void put_pd_read(struct ib_pd *pd) 219 { 220 put_uobj_read(pd->uobject); 221 } 222 223 static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) 224 { 225 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); 226 } 227 228 static void put_cq_read(struct ib_cq *cq) 229 { 230 put_uobj_read(cq->uobject); 231 } 232 233 static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) 234 { 235 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); 236 } 237 238 static void put_ah_read(struct ib_ah *ah) 239 { 240 put_uobj_read(ah->uobject); 241 } 242 243 static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) 244 { 245 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); 246 } 247 248 static struct ib_wq *idr_read_wq(int wq_handle, struct ib_ucontext *context) 249 { 250 return idr_read_obj(&ib_uverbs_wq_idr, wq_handle, context, 0); 251 } 252 253 static void put_wq_read(struct ib_wq *wq) 254 { 255 put_uobj_read(wq->uobject); 256 } 257 258 static struct ib_rwq_ind_table *idr_read_rwq_indirection_table(int ind_table_handle, 259 struct ib_ucontext *context) 260 { 261 return idr_read_obj(&ib_uverbs_rwq_ind_tbl_idr, ind_table_handle, context, 0); 262 } 263 264 static void put_rwq_indirection_table_read(struct ib_rwq_ind_table *ind_table) 265 { 266 put_uobj_read(ind_table->uobject); 267 } 268 269 static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context) 270 { 271 struct ib_uobject *uobj; 272 273 uobj = idr_write_uobj(&ib_uverbs_qp_idr, qp_handle, context); 274 return uobj ? uobj->object : NULL; 275 } 276 277 static void put_qp_read(struct ib_qp *qp) 278 { 279 put_uobj_read(qp->uobject); 280 } 281 282 static void put_qp_write(struct ib_qp *qp) 283 { 284 put_uobj_write(qp->uobject); 285 } 286 287 static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) 288 { 289 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); 290 } 291 292 static void put_srq_read(struct ib_srq *srq) 293 { 294 put_uobj_read(srq->uobject); 295 } 296 297 static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context, 298 struct ib_uobject **uobj) 299 { 300 *uobj = idr_read_uobj(&ib_uverbs_xrcd_idr, xrcd_handle, context, 0); 301 return *uobj ? (*uobj)->object : NULL; 302 } 303 304 static void put_xrcd_read(struct ib_uobject *uobj) 305 { 306 put_uobj_read(uobj); 307 } 308 309 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 310 struct ib_device *ib_dev, 311 const char __user *buf, 312 int in_len, int out_len) 313 { 314 struct ib_uverbs_get_context cmd; 315 struct ib_uverbs_get_context_resp resp; 316 struct ib_udata udata; 317 struct ib_ucontext *ucontext; 318 struct file *filp; 319 int ret; 320 321 if (out_len < sizeof resp) 322 return -ENOSPC; 323 324 if (copy_from_user(&cmd, buf, sizeof cmd)) 325 return -EFAULT; 326 327 mutex_lock(&file->mutex); 328 329 if (file->ucontext) { 330 ret = -EINVAL; 331 goto err; 332 } 333 334 INIT_UDATA(&udata, buf + sizeof cmd, 335 (unsigned long) cmd.response + sizeof resp, 336 in_len - sizeof cmd, out_len - sizeof resp); 337 338 ucontext = ib_dev->alloc_ucontext(ib_dev, &udata); 339 if (IS_ERR(ucontext)) { 340 ret = PTR_ERR(ucontext); 341 goto err; 342 } 343 344 ucontext->device = ib_dev; 345 INIT_LIST_HEAD(&ucontext->pd_list); 346 INIT_LIST_HEAD(&ucontext->mr_list); 347 INIT_LIST_HEAD(&ucontext->mw_list); 348 INIT_LIST_HEAD(&ucontext->cq_list); 349 INIT_LIST_HEAD(&ucontext->qp_list); 350 INIT_LIST_HEAD(&ucontext->srq_list); 351 INIT_LIST_HEAD(&ucontext->ah_list); 352 INIT_LIST_HEAD(&ucontext->wq_list); 353 INIT_LIST_HEAD(&ucontext->rwq_ind_tbl_list); 354 INIT_LIST_HEAD(&ucontext->xrcd_list); 355 INIT_LIST_HEAD(&ucontext->rule_list); 356 rcu_read_lock(); 357 ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); 358 rcu_read_unlock(); 359 ucontext->closing = 0; 360 361 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 362 ucontext->umem_tree = RB_ROOT; 363 init_rwsem(&ucontext->umem_rwsem); 364 ucontext->odp_mrs_count = 0; 365 INIT_LIST_HEAD(&ucontext->no_private_counters); 366 367 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) 368 ucontext->invalidate_range = NULL; 369 370 #endif 371 372 resp.num_comp_vectors = file->device->num_comp_vectors; 373 374 ret = get_unused_fd_flags(O_CLOEXEC); 375 if (ret < 0) 376 goto err_free; 377 resp.async_fd = ret; 378 379 filp = ib_uverbs_alloc_event_file(file, ib_dev, 1); 380 if (IS_ERR(filp)) { 381 ret = PTR_ERR(filp); 382 goto err_fd; 383 } 384 385 if (copy_to_user((void __user *) (unsigned long) cmd.response, 386 &resp, sizeof resp)) { 387 ret = -EFAULT; 388 goto err_file; 389 } 390 391 file->ucontext = ucontext; 392 393 fd_install(resp.async_fd, filp); 394 395 mutex_unlock(&file->mutex); 396 397 return in_len; 398 399 err_file: 400 ib_uverbs_free_async_event_file(file); 401 fput(filp); 402 403 err_fd: 404 put_unused_fd(resp.async_fd); 405 406 err_free: 407 put_pid(ucontext->tgid); 408 ib_dev->dealloc_ucontext(ucontext); 409 410 err: 411 mutex_unlock(&file->mutex); 412 return ret; 413 } 414 415 static void copy_query_dev_fields(struct ib_uverbs_file *file, 416 struct ib_device *ib_dev, 417 struct ib_uverbs_query_device_resp *resp, 418 struct ib_device_attr *attr) 419 { 420 resp->fw_ver = attr->fw_ver; 421 resp->node_guid = ib_dev->node_guid; 422 resp->sys_image_guid = attr->sys_image_guid; 423 resp->max_mr_size = attr->max_mr_size; 424 resp->page_size_cap = attr->page_size_cap; 425 resp->vendor_id = attr->vendor_id; 426 resp->vendor_part_id = attr->vendor_part_id; 427 resp->hw_ver = attr->hw_ver; 428 resp->max_qp = attr->max_qp; 429 resp->max_qp_wr = attr->max_qp_wr; 430 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags); 431 resp->max_sge = attr->max_sge; 432 resp->max_sge_rd = attr->max_sge_rd; 433 resp->max_cq = attr->max_cq; 434 resp->max_cqe = attr->max_cqe; 435 resp->max_mr = attr->max_mr; 436 resp->max_pd = attr->max_pd; 437 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 438 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 439 resp->max_res_rd_atom = attr->max_res_rd_atom; 440 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 441 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 442 resp->atomic_cap = attr->atomic_cap; 443 resp->max_ee = attr->max_ee; 444 resp->max_rdd = attr->max_rdd; 445 resp->max_mw = attr->max_mw; 446 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 447 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 448 resp->max_mcast_grp = attr->max_mcast_grp; 449 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 450 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 451 resp->max_ah = attr->max_ah; 452 resp->max_fmr = attr->max_fmr; 453 resp->max_map_per_fmr = attr->max_map_per_fmr; 454 resp->max_srq = attr->max_srq; 455 resp->max_srq_wr = attr->max_srq_wr; 456 resp->max_srq_sge = attr->max_srq_sge; 457 resp->max_pkeys = attr->max_pkeys; 458 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 459 resp->phys_port_cnt = ib_dev->phys_port_cnt; 460 } 461 462 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 463 struct ib_device *ib_dev, 464 const char __user *buf, 465 int in_len, int out_len) 466 { 467 struct ib_uverbs_query_device cmd; 468 struct ib_uverbs_query_device_resp resp; 469 470 if (out_len < sizeof resp) 471 return -ENOSPC; 472 473 if (copy_from_user(&cmd, buf, sizeof cmd)) 474 return -EFAULT; 475 476 memset(&resp, 0, sizeof resp); 477 copy_query_dev_fields(file, ib_dev, &resp, &ib_dev->attrs); 478 479 if (copy_to_user((void __user *) (unsigned long) cmd.response, 480 &resp, sizeof resp)) 481 return -EFAULT; 482 483 return in_len; 484 } 485 486 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 487 struct ib_device *ib_dev, 488 const char __user *buf, 489 int in_len, int out_len) 490 { 491 struct ib_uverbs_query_port cmd; 492 struct ib_uverbs_query_port_resp resp; 493 struct ib_port_attr attr; 494 int ret; 495 496 if (out_len < sizeof resp) 497 return -ENOSPC; 498 499 if (copy_from_user(&cmd, buf, sizeof cmd)) 500 return -EFAULT; 501 502 ret = ib_query_port(ib_dev, cmd.port_num, &attr); 503 if (ret) 504 return ret; 505 506 memset(&resp, 0, sizeof resp); 507 508 resp.state = attr.state; 509 resp.max_mtu = attr.max_mtu; 510 resp.active_mtu = attr.active_mtu; 511 resp.gid_tbl_len = attr.gid_tbl_len; 512 resp.port_cap_flags = attr.port_cap_flags; 513 resp.max_msg_sz = attr.max_msg_sz; 514 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 515 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 516 resp.pkey_tbl_len = attr.pkey_tbl_len; 517 resp.lid = attr.lid; 518 resp.sm_lid = attr.sm_lid; 519 resp.lmc = attr.lmc; 520 resp.max_vl_num = attr.max_vl_num; 521 resp.sm_sl = attr.sm_sl; 522 resp.subnet_timeout = attr.subnet_timeout; 523 resp.init_type_reply = attr.init_type_reply; 524 resp.active_width = attr.active_width; 525 resp.active_speed = attr.active_speed; 526 resp.phys_state = attr.phys_state; 527 resp.link_layer = rdma_port_get_link_layer(ib_dev, 528 cmd.port_num); 529 530 if (copy_to_user((void __user *) (unsigned long) cmd.response, 531 &resp, sizeof resp)) 532 return -EFAULT; 533 534 return in_len; 535 } 536 537 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 538 struct ib_device *ib_dev, 539 const char __user *buf, 540 int in_len, int out_len) 541 { 542 struct ib_uverbs_alloc_pd cmd; 543 struct ib_uverbs_alloc_pd_resp resp; 544 struct ib_udata udata; 545 struct ib_uobject *uobj; 546 struct ib_pd *pd; 547 int ret; 548 549 if (out_len < sizeof resp) 550 return -ENOSPC; 551 552 if (copy_from_user(&cmd, buf, sizeof cmd)) 553 return -EFAULT; 554 555 INIT_UDATA(&udata, buf + sizeof cmd, 556 (unsigned long) cmd.response + sizeof resp, 557 in_len - sizeof cmd, out_len - sizeof resp); 558 559 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 560 if (!uobj) 561 return -ENOMEM; 562 563 init_uobj(uobj, 0, file->ucontext, &pd_lock_class); 564 down_write(&uobj->mutex); 565 566 pd = ib_dev->alloc_pd(ib_dev, file->ucontext, &udata); 567 if (IS_ERR(pd)) { 568 ret = PTR_ERR(pd); 569 goto err; 570 } 571 572 pd->device = ib_dev; 573 pd->uobject = uobj; 574 pd->__internal_mr = NULL; 575 atomic_set(&pd->usecnt, 0); 576 577 uobj->object = pd; 578 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); 579 if (ret) 580 goto err_idr; 581 582 memset(&resp, 0, sizeof resp); 583 resp.pd_handle = uobj->id; 584 585 if (copy_to_user((void __user *) (unsigned long) cmd.response, 586 &resp, sizeof resp)) { 587 ret = -EFAULT; 588 goto err_copy; 589 } 590 591 mutex_lock(&file->mutex); 592 list_add_tail(&uobj->list, &file->ucontext->pd_list); 593 mutex_unlock(&file->mutex); 594 595 uobj->live = 1; 596 597 up_write(&uobj->mutex); 598 599 return in_len; 600 601 err_copy: 602 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 603 604 err_idr: 605 ib_dealloc_pd(pd); 606 607 err: 608 put_uobj_write(uobj); 609 return ret; 610 } 611 612 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 613 struct ib_device *ib_dev, 614 const char __user *buf, 615 int in_len, int out_len) 616 { 617 struct ib_uverbs_dealloc_pd cmd; 618 struct ib_uobject *uobj; 619 struct ib_pd *pd; 620 int ret; 621 622 if (copy_from_user(&cmd, buf, sizeof cmd)) 623 return -EFAULT; 624 625 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); 626 if (!uobj) 627 return -EINVAL; 628 pd = uobj->object; 629 630 if (atomic_read(&pd->usecnt)) { 631 ret = -EBUSY; 632 goto err_put; 633 } 634 635 ret = pd->device->dealloc_pd(uobj->object); 636 WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd"); 637 if (ret) 638 goto err_put; 639 640 uobj->live = 0; 641 put_uobj_write(uobj); 642 643 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 644 645 mutex_lock(&file->mutex); 646 list_del(&uobj->list); 647 mutex_unlock(&file->mutex); 648 649 put_uobj(uobj); 650 651 return in_len; 652 653 err_put: 654 put_uobj_write(uobj); 655 return ret; 656 } 657 658 struct xrcd_table_entry { 659 struct rb_node node; 660 struct ib_xrcd *xrcd; 661 struct inode *inode; 662 }; 663 664 static int xrcd_table_insert(struct ib_uverbs_device *dev, 665 struct inode *inode, 666 struct ib_xrcd *xrcd) 667 { 668 struct xrcd_table_entry *entry, *scan; 669 struct rb_node **p = &dev->xrcd_tree.rb_node; 670 struct rb_node *parent = NULL; 671 672 entry = kmalloc(sizeof *entry, GFP_KERNEL); 673 if (!entry) 674 return -ENOMEM; 675 676 entry->xrcd = xrcd; 677 entry->inode = inode; 678 679 while (*p) { 680 parent = *p; 681 scan = rb_entry(parent, struct xrcd_table_entry, node); 682 683 if (inode < scan->inode) { 684 p = &(*p)->rb_left; 685 } else if (inode > scan->inode) { 686 p = &(*p)->rb_right; 687 } else { 688 kfree(entry); 689 return -EEXIST; 690 } 691 } 692 693 rb_link_node(&entry->node, parent, p); 694 rb_insert_color(&entry->node, &dev->xrcd_tree); 695 igrab(inode); 696 return 0; 697 } 698 699 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 700 struct inode *inode) 701 { 702 struct xrcd_table_entry *entry; 703 struct rb_node *p = dev->xrcd_tree.rb_node; 704 705 while (p) { 706 entry = rb_entry(p, struct xrcd_table_entry, node); 707 708 if (inode < entry->inode) 709 p = p->rb_left; 710 else if (inode > entry->inode) 711 p = p->rb_right; 712 else 713 return entry; 714 } 715 716 return NULL; 717 } 718 719 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 720 { 721 struct xrcd_table_entry *entry; 722 723 entry = xrcd_table_search(dev, inode); 724 if (!entry) 725 return NULL; 726 727 return entry->xrcd; 728 } 729 730 static void xrcd_table_delete(struct ib_uverbs_device *dev, 731 struct inode *inode) 732 { 733 struct xrcd_table_entry *entry; 734 735 entry = xrcd_table_search(dev, inode); 736 if (entry) { 737 iput(inode); 738 rb_erase(&entry->node, &dev->xrcd_tree); 739 kfree(entry); 740 } 741 } 742 743 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, 744 struct ib_device *ib_dev, 745 const char __user *buf, int in_len, 746 int out_len) 747 { 748 struct ib_uverbs_open_xrcd cmd; 749 struct ib_uverbs_open_xrcd_resp resp; 750 struct ib_udata udata; 751 struct ib_uxrcd_object *obj; 752 struct ib_xrcd *xrcd = NULL; 753 struct fd f = {NULL, 0}; 754 struct inode *inode = NULL; 755 int ret = 0; 756 int new_xrcd = 0; 757 758 if (out_len < sizeof resp) 759 return -ENOSPC; 760 761 if (copy_from_user(&cmd, buf, sizeof cmd)) 762 return -EFAULT; 763 764 INIT_UDATA(&udata, buf + sizeof cmd, 765 (unsigned long) cmd.response + sizeof resp, 766 in_len - sizeof cmd, out_len - sizeof resp); 767 768 mutex_lock(&file->device->xrcd_tree_mutex); 769 770 if (cmd.fd != -1) { 771 /* search for file descriptor */ 772 f = fdget(cmd.fd); 773 if (!f.file) { 774 ret = -EBADF; 775 goto err_tree_mutex_unlock; 776 } 777 778 inode = file_inode(f.file); 779 xrcd = find_xrcd(file->device, inode); 780 if (!xrcd && !(cmd.oflags & O_CREAT)) { 781 /* no file descriptor. Need CREATE flag */ 782 ret = -EAGAIN; 783 goto err_tree_mutex_unlock; 784 } 785 786 if (xrcd && cmd.oflags & O_EXCL) { 787 ret = -EINVAL; 788 goto err_tree_mutex_unlock; 789 } 790 } 791 792 obj = kmalloc(sizeof *obj, GFP_KERNEL); 793 if (!obj) { 794 ret = -ENOMEM; 795 goto err_tree_mutex_unlock; 796 } 797 798 init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class); 799 800 down_write(&obj->uobject.mutex); 801 802 if (!xrcd) { 803 xrcd = ib_dev->alloc_xrcd(ib_dev, file->ucontext, &udata); 804 if (IS_ERR(xrcd)) { 805 ret = PTR_ERR(xrcd); 806 goto err; 807 } 808 809 xrcd->inode = inode; 810 xrcd->device = ib_dev; 811 atomic_set(&xrcd->usecnt, 0); 812 mutex_init(&xrcd->tgt_qp_mutex); 813 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 814 new_xrcd = 1; 815 } 816 817 atomic_set(&obj->refcnt, 0); 818 obj->uobject.object = xrcd; 819 ret = idr_add_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 820 if (ret) 821 goto err_idr; 822 823 memset(&resp, 0, sizeof resp); 824 resp.xrcd_handle = obj->uobject.id; 825 826 if (inode) { 827 if (new_xrcd) { 828 /* create new inode/xrcd table entry */ 829 ret = xrcd_table_insert(file->device, inode, xrcd); 830 if (ret) 831 goto err_insert_xrcd; 832 } 833 atomic_inc(&xrcd->usecnt); 834 } 835 836 if (copy_to_user((void __user *) (unsigned long) cmd.response, 837 &resp, sizeof resp)) { 838 ret = -EFAULT; 839 goto err_copy; 840 } 841 842 if (f.file) 843 fdput(f); 844 845 mutex_lock(&file->mutex); 846 list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list); 847 mutex_unlock(&file->mutex); 848 849 obj->uobject.live = 1; 850 up_write(&obj->uobject.mutex); 851 852 mutex_unlock(&file->device->xrcd_tree_mutex); 853 return in_len; 854 855 err_copy: 856 if (inode) { 857 if (new_xrcd) 858 xrcd_table_delete(file->device, inode); 859 atomic_dec(&xrcd->usecnt); 860 } 861 862 err_insert_xrcd: 863 idr_remove_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 864 865 err_idr: 866 ib_dealloc_xrcd(xrcd); 867 868 err: 869 put_uobj_write(&obj->uobject); 870 871 err_tree_mutex_unlock: 872 if (f.file) 873 fdput(f); 874 875 mutex_unlock(&file->device->xrcd_tree_mutex); 876 877 return ret; 878 } 879 880 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file, 881 struct ib_device *ib_dev, 882 const char __user *buf, int in_len, 883 int out_len) 884 { 885 struct ib_uverbs_close_xrcd cmd; 886 struct ib_uobject *uobj; 887 struct ib_xrcd *xrcd = NULL; 888 struct inode *inode = NULL; 889 struct ib_uxrcd_object *obj; 890 int live; 891 int ret = 0; 892 893 if (copy_from_user(&cmd, buf, sizeof cmd)) 894 return -EFAULT; 895 896 mutex_lock(&file->device->xrcd_tree_mutex); 897 uobj = idr_write_uobj(&ib_uverbs_xrcd_idr, cmd.xrcd_handle, file->ucontext); 898 if (!uobj) { 899 ret = -EINVAL; 900 goto out; 901 } 902 903 xrcd = uobj->object; 904 inode = xrcd->inode; 905 obj = container_of(uobj, struct ib_uxrcd_object, uobject); 906 if (atomic_read(&obj->refcnt)) { 907 put_uobj_write(uobj); 908 ret = -EBUSY; 909 goto out; 910 } 911 912 if (!inode || atomic_dec_and_test(&xrcd->usecnt)) { 913 ret = ib_dealloc_xrcd(uobj->object); 914 if (!ret) 915 uobj->live = 0; 916 } 917 918 live = uobj->live; 919 if (inode && ret) 920 atomic_inc(&xrcd->usecnt); 921 922 put_uobj_write(uobj); 923 924 if (ret) 925 goto out; 926 927 if (inode && !live) 928 xrcd_table_delete(file->device, inode); 929 930 idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj); 931 mutex_lock(&file->mutex); 932 list_del(&uobj->list); 933 mutex_unlock(&file->mutex); 934 935 put_uobj(uobj); 936 ret = in_len; 937 938 out: 939 mutex_unlock(&file->device->xrcd_tree_mutex); 940 return ret; 941 } 942 943 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, 944 struct ib_xrcd *xrcd) 945 { 946 struct inode *inode; 947 948 inode = xrcd->inode; 949 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 950 return; 951 952 ib_dealloc_xrcd(xrcd); 953 954 if (inode) 955 xrcd_table_delete(dev, inode); 956 } 957 958 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 959 struct ib_device *ib_dev, 960 const char __user *buf, int in_len, 961 int out_len) 962 { 963 struct ib_uverbs_reg_mr cmd; 964 struct ib_uverbs_reg_mr_resp resp; 965 struct ib_udata udata; 966 struct ib_uobject *uobj; 967 struct ib_pd *pd; 968 struct ib_mr *mr; 969 int ret; 970 971 if (out_len < sizeof resp) 972 return -ENOSPC; 973 974 if (copy_from_user(&cmd, buf, sizeof cmd)) 975 return -EFAULT; 976 977 INIT_UDATA(&udata, buf + sizeof cmd, 978 (unsigned long) cmd.response + sizeof resp, 979 in_len - sizeof cmd, out_len - sizeof resp); 980 981 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 982 return -EINVAL; 983 984 ret = ib_check_mr_access(cmd.access_flags); 985 if (ret) 986 return ret; 987 988 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 989 if (!uobj) 990 return -ENOMEM; 991 992 init_uobj(uobj, 0, file->ucontext, &mr_lock_class); 993 down_write(&uobj->mutex); 994 995 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 996 if (!pd) { 997 ret = -EINVAL; 998 goto err_free; 999 } 1000 1001 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 1002 if (!(pd->device->attrs.device_cap_flags & 1003 IB_DEVICE_ON_DEMAND_PAGING)) { 1004 pr_debug("ODP support not available\n"); 1005 ret = -EINVAL; 1006 goto err_put; 1007 } 1008 } 1009 1010 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 1011 cmd.access_flags, &udata); 1012 if (IS_ERR(mr)) { 1013 ret = PTR_ERR(mr); 1014 goto err_put; 1015 } 1016 1017 mr->device = pd->device; 1018 mr->pd = pd; 1019 mr->uobject = uobj; 1020 atomic_inc(&pd->usecnt); 1021 1022 uobj->object = mr; 1023 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); 1024 if (ret) 1025 goto err_unreg; 1026 1027 memset(&resp, 0, sizeof resp); 1028 resp.lkey = mr->lkey; 1029 resp.rkey = mr->rkey; 1030 resp.mr_handle = uobj->id; 1031 1032 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1033 &resp, sizeof resp)) { 1034 ret = -EFAULT; 1035 goto err_copy; 1036 } 1037 1038 put_pd_read(pd); 1039 1040 mutex_lock(&file->mutex); 1041 list_add_tail(&uobj->list, &file->ucontext->mr_list); 1042 mutex_unlock(&file->mutex); 1043 1044 uobj->live = 1; 1045 1046 up_write(&uobj->mutex); 1047 1048 return in_len; 1049 1050 err_copy: 1051 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1052 1053 err_unreg: 1054 ib_dereg_mr(mr); 1055 1056 err_put: 1057 put_pd_read(pd); 1058 1059 err_free: 1060 put_uobj_write(uobj); 1061 return ret; 1062 } 1063 1064 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, 1065 struct ib_device *ib_dev, 1066 const char __user *buf, int in_len, 1067 int out_len) 1068 { 1069 struct ib_uverbs_rereg_mr cmd; 1070 struct ib_uverbs_rereg_mr_resp resp; 1071 struct ib_udata udata; 1072 struct ib_pd *pd = NULL; 1073 struct ib_mr *mr; 1074 struct ib_pd *old_pd; 1075 int ret; 1076 struct ib_uobject *uobj; 1077 1078 if (out_len < sizeof(resp)) 1079 return -ENOSPC; 1080 1081 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1082 return -EFAULT; 1083 1084 INIT_UDATA(&udata, buf + sizeof(cmd), 1085 (unsigned long) cmd.response + sizeof(resp), 1086 in_len - sizeof(cmd), out_len - sizeof(resp)); 1087 1088 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 1089 return -EINVAL; 1090 1091 if ((cmd.flags & IB_MR_REREG_TRANS) && 1092 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 1093 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 1094 return -EINVAL; 1095 1096 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, 1097 file->ucontext); 1098 1099 if (!uobj) 1100 return -EINVAL; 1101 1102 mr = uobj->object; 1103 1104 if (cmd.flags & IB_MR_REREG_ACCESS) { 1105 ret = ib_check_mr_access(cmd.access_flags); 1106 if (ret) 1107 goto put_uobjs; 1108 } 1109 1110 if (cmd.flags & IB_MR_REREG_PD) { 1111 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1112 if (!pd) { 1113 ret = -EINVAL; 1114 goto put_uobjs; 1115 } 1116 } 1117 1118 old_pd = mr->pd; 1119 ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start, 1120 cmd.length, cmd.hca_va, 1121 cmd.access_flags, pd, &udata); 1122 if (!ret) { 1123 if (cmd.flags & IB_MR_REREG_PD) { 1124 atomic_inc(&pd->usecnt); 1125 mr->pd = pd; 1126 atomic_dec(&old_pd->usecnt); 1127 } 1128 } else { 1129 goto put_uobj_pd; 1130 } 1131 1132 memset(&resp, 0, sizeof(resp)); 1133 resp.lkey = mr->lkey; 1134 resp.rkey = mr->rkey; 1135 1136 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1137 &resp, sizeof(resp))) 1138 ret = -EFAULT; 1139 else 1140 ret = in_len; 1141 1142 put_uobj_pd: 1143 if (cmd.flags & IB_MR_REREG_PD) 1144 put_pd_read(pd); 1145 1146 put_uobjs: 1147 1148 put_uobj_write(mr->uobject); 1149 1150 return ret; 1151 } 1152 1153 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 1154 struct ib_device *ib_dev, 1155 const char __user *buf, int in_len, 1156 int out_len) 1157 { 1158 struct ib_uverbs_dereg_mr cmd; 1159 struct ib_mr *mr; 1160 struct ib_uobject *uobj; 1161 int ret = -EINVAL; 1162 1163 if (copy_from_user(&cmd, buf, sizeof cmd)) 1164 return -EFAULT; 1165 1166 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); 1167 if (!uobj) 1168 return -EINVAL; 1169 1170 mr = uobj->object; 1171 1172 ret = ib_dereg_mr(mr); 1173 if (!ret) 1174 uobj->live = 0; 1175 1176 put_uobj_write(uobj); 1177 1178 if (ret) 1179 return ret; 1180 1181 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1182 1183 mutex_lock(&file->mutex); 1184 list_del(&uobj->list); 1185 mutex_unlock(&file->mutex); 1186 1187 put_uobj(uobj); 1188 1189 return in_len; 1190 } 1191 1192 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, 1193 struct ib_device *ib_dev, 1194 const char __user *buf, int in_len, 1195 int out_len) 1196 { 1197 struct ib_uverbs_alloc_mw cmd; 1198 struct ib_uverbs_alloc_mw_resp resp; 1199 struct ib_uobject *uobj; 1200 struct ib_pd *pd; 1201 struct ib_mw *mw; 1202 struct ib_udata udata; 1203 int ret; 1204 1205 if (out_len < sizeof(resp)) 1206 return -ENOSPC; 1207 1208 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1209 return -EFAULT; 1210 1211 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 1212 if (!uobj) 1213 return -ENOMEM; 1214 1215 init_uobj(uobj, 0, file->ucontext, &mw_lock_class); 1216 down_write(&uobj->mutex); 1217 1218 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1219 if (!pd) { 1220 ret = -EINVAL; 1221 goto err_free; 1222 } 1223 1224 INIT_UDATA(&udata, buf + sizeof(cmd), 1225 (unsigned long)cmd.response + sizeof(resp), 1226 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1227 out_len - sizeof(resp)); 1228 1229 mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata); 1230 if (IS_ERR(mw)) { 1231 ret = PTR_ERR(mw); 1232 goto err_put; 1233 } 1234 1235 mw->device = pd->device; 1236 mw->pd = pd; 1237 mw->uobject = uobj; 1238 atomic_inc(&pd->usecnt); 1239 1240 uobj->object = mw; 1241 ret = idr_add_uobj(&ib_uverbs_mw_idr, uobj); 1242 if (ret) 1243 goto err_unalloc; 1244 1245 memset(&resp, 0, sizeof(resp)); 1246 resp.rkey = mw->rkey; 1247 resp.mw_handle = uobj->id; 1248 1249 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1250 &resp, sizeof(resp))) { 1251 ret = -EFAULT; 1252 goto err_copy; 1253 } 1254 1255 put_pd_read(pd); 1256 1257 mutex_lock(&file->mutex); 1258 list_add_tail(&uobj->list, &file->ucontext->mw_list); 1259 mutex_unlock(&file->mutex); 1260 1261 uobj->live = 1; 1262 1263 up_write(&uobj->mutex); 1264 1265 return in_len; 1266 1267 err_copy: 1268 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1269 1270 err_unalloc: 1271 uverbs_dealloc_mw(mw); 1272 1273 err_put: 1274 put_pd_read(pd); 1275 1276 err_free: 1277 put_uobj_write(uobj); 1278 return ret; 1279 } 1280 1281 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file, 1282 struct ib_device *ib_dev, 1283 const char __user *buf, int in_len, 1284 int out_len) 1285 { 1286 struct ib_uverbs_dealloc_mw cmd; 1287 struct ib_mw *mw; 1288 struct ib_uobject *uobj; 1289 int ret = -EINVAL; 1290 1291 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1292 return -EFAULT; 1293 1294 uobj = idr_write_uobj(&ib_uverbs_mw_idr, cmd.mw_handle, file->ucontext); 1295 if (!uobj) 1296 return -EINVAL; 1297 1298 mw = uobj->object; 1299 1300 ret = uverbs_dealloc_mw(mw); 1301 if (!ret) 1302 uobj->live = 0; 1303 1304 put_uobj_write(uobj); 1305 1306 if (ret) 1307 return ret; 1308 1309 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1310 1311 mutex_lock(&file->mutex); 1312 list_del(&uobj->list); 1313 mutex_unlock(&file->mutex); 1314 1315 put_uobj(uobj); 1316 1317 return in_len; 1318 } 1319 1320 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 1321 struct ib_device *ib_dev, 1322 const char __user *buf, int in_len, 1323 int out_len) 1324 { 1325 struct ib_uverbs_create_comp_channel cmd; 1326 struct ib_uverbs_create_comp_channel_resp resp; 1327 struct file *filp; 1328 int ret; 1329 1330 if (out_len < sizeof resp) 1331 return -ENOSPC; 1332 1333 if (copy_from_user(&cmd, buf, sizeof cmd)) 1334 return -EFAULT; 1335 1336 ret = get_unused_fd_flags(O_CLOEXEC); 1337 if (ret < 0) 1338 return ret; 1339 resp.fd = ret; 1340 1341 filp = ib_uverbs_alloc_event_file(file, ib_dev, 0); 1342 if (IS_ERR(filp)) { 1343 put_unused_fd(resp.fd); 1344 return PTR_ERR(filp); 1345 } 1346 1347 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1348 &resp, sizeof resp)) { 1349 put_unused_fd(resp.fd); 1350 fput(filp); 1351 return -EFAULT; 1352 } 1353 1354 fd_install(resp.fd, filp); 1355 return in_len; 1356 } 1357 1358 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, 1359 struct ib_device *ib_dev, 1360 struct ib_udata *ucore, 1361 struct ib_udata *uhw, 1362 struct ib_uverbs_ex_create_cq *cmd, 1363 size_t cmd_sz, 1364 int (*cb)(struct ib_uverbs_file *file, 1365 struct ib_ucq_object *obj, 1366 struct ib_uverbs_ex_create_cq_resp *resp, 1367 struct ib_udata *udata, 1368 void *context), 1369 void *context) 1370 { 1371 struct ib_ucq_object *obj; 1372 struct ib_uverbs_event_file *ev_file = NULL; 1373 struct ib_cq *cq; 1374 int ret; 1375 struct ib_uverbs_ex_create_cq_resp resp; 1376 struct ib_cq_init_attr attr = {}; 1377 1378 if (cmd->comp_vector >= file->device->num_comp_vectors) 1379 return ERR_PTR(-EINVAL); 1380 1381 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1382 if (!obj) 1383 return ERR_PTR(-ENOMEM); 1384 1385 init_uobj(&obj->uobject, cmd->user_handle, file->ucontext, &cq_lock_class); 1386 down_write(&obj->uobject.mutex); 1387 1388 if (cmd->comp_channel >= 0) { 1389 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel); 1390 if (!ev_file) { 1391 ret = -EINVAL; 1392 goto err; 1393 } 1394 } 1395 1396 obj->uverbs_file = file; 1397 obj->comp_events_reported = 0; 1398 obj->async_events_reported = 0; 1399 INIT_LIST_HEAD(&obj->comp_list); 1400 INIT_LIST_HEAD(&obj->async_list); 1401 1402 attr.cqe = cmd->cqe; 1403 attr.comp_vector = cmd->comp_vector; 1404 1405 if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags)) 1406 attr.flags = cmd->flags; 1407 1408 cq = ib_dev->create_cq(ib_dev, &attr, 1409 file->ucontext, uhw); 1410 if (IS_ERR(cq)) { 1411 ret = PTR_ERR(cq); 1412 goto err_file; 1413 } 1414 1415 cq->device = ib_dev; 1416 cq->uobject = &obj->uobject; 1417 cq->comp_handler = ib_uverbs_comp_handler; 1418 cq->event_handler = ib_uverbs_cq_event_handler; 1419 cq->cq_context = ev_file; 1420 atomic_set(&cq->usecnt, 0); 1421 1422 obj->uobject.object = cq; 1423 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1424 if (ret) 1425 goto err_free; 1426 1427 memset(&resp, 0, sizeof resp); 1428 resp.base.cq_handle = obj->uobject.id; 1429 resp.base.cqe = cq->cqe; 1430 1431 resp.response_length = offsetof(typeof(resp), response_length) + 1432 sizeof(resp.response_length); 1433 1434 ret = cb(file, obj, &resp, ucore, context); 1435 if (ret) 1436 goto err_cb; 1437 1438 mutex_lock(&file->mutex); 1439 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); 1440 mutex_unlock(&file->mutex); 1441 1442 obj->uobject.live = 1; 1443 1444 up_write(&obj->uobject.mutex); 1445 1446 return obj; 1447 1448 err_cb: 1449 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1450 1451 err_free: 1452 ib_destroy_cq(cq); 1453 1454 err_file: 1455 if (ev_file) 1456 ib_uverbs_release_ucq(file, ev_file, obj); 1457 1458 err: 1459 put_uobj_write(&obj->uobject); 1460 1461 return ERR_PTR(ret); 1462 } 1463 1464 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file, 1465 struct ib_ucq_object *obj, 1466 struct ib_uverbs_ex_create_cq_resp *resp, 1467 struct ib_udata *ucore, void *context) 1468 { 1469 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 1470 return -EFAULT; 1471 1472 return 0; 1473 } 1474 1475 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 1476 struct ib_device *ib_dev, 1477 const char __user *buf, int in_len, 1478 int out_len) 1479 { 1480 struct ib_uverbs_create_cq cmd; 1481 struct ib_uverbs_ex_create_cq cmd_ex; 1482 struct ib_uverbs_create_cq_resp resp; 1483 struct ib_udata ucore; 1484 struct ib_udata uhw; 1485 struct ib_ucq_object *obj; 1486 1487 if (out_len < sizeof(resp)) 1488 return -ENOSPC; 1489 1490 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1491 return -EFAULT; 1492 1493 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), sizeof(resp)); 1494 1495 INIT_UDATA(&uhw, buf + sizeof(cmd), 1496 (unsigned long)cmd.response + sizeof(resp), 1497 in_len - sizeof(cmd), out_len - sizeof(resp)); 1498 1499 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1500 cmd_ex.user_handle = cmd.user_handle; 1501 cmd_ex.cqe = cmd.cqe; 1502 cmd_ex.comp_vector = cmd.comp_vector; 1503 cmd_ex.comp_channel = cmd.comp_channel; 1504 1505 obj = create_cq(file, ib_dev, &ucore, &uhw, &cmd_ex, 1506 offsetof(typeof(cmd_ex), comp_channel) + 1507 sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb, 1508 NULL); 1509 1510 if (IS_ERR(obj)) 1511 return PTR_ERR(obj); 1512 1513 return in_len; 1514 } 1515 1516 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file, 1517 struct ib_ucq_object *obj, 1518 struct ib_uverbs_ex_create_cq_resp *resp, 1519 struct ib_udata *ucore, void *context) 1520 { 1521 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 1522 return -EFAULT; 1523 1524 return 0; 1525 } 1526 1527 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file, 1528 struct ib_device *ib_dev, 1529 struct ib_udata *ucore, 1530 struct ib_udata *uhw) 1531 { 1532 struct ib_uverbs_ex_create_cq_resp resp; 1533 struct ib_uverbs_ex_create_cq cmd; 1534 struct ib_ucq_object *obj; 1535 int err; 1536 1537 if (ucore->inlen < sizeof(cmd)) 1538 return -EINVAL; 1539 1540 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 1541 if (err) 1542 return err; 1543 1544 if (cmd.comp_mask) 1545 return -EINVAL; 1546 1547 if (cmd.reserved) 1548 return -EINVAL; 1549 1550 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 1551 sizeof(resp.response_length))) 1552 return -ENOSPC; 1553 1554 obj = create_cq(file, ib_dev, ucore, uhw, &cmd, 1555 min(ucore->inlen, sizeof(cmd)), 1556 ib_uverbs_ex_create_cq_cb, NULL); 1557 1558 if (IS_ERR(obj)) 1559 return PTR_ERR(obj); 1560 1561 return 0; 1562 } 1563 1564 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 1565 struct ib_device *ib_dev, 1566 const char __user *buf, int in_len, 1567 int out_len) 1568 { 1569 struct ib_uverbs_resize_cq cmd; 1570 struct ib_uverbs_resize_cq_resp resp; 1571 struct ib_udata udata; 1572 struct ib_cq *cq; 1573 int ret = -EINVAL; 1574 1575 if (copy_from_user(&cmd, buf, sizeof cmd)) 1576 return -EFAULT; 1577 1578 INIT_UDATA(&udata, buf + sizeof cmd, 1579 (unsigned long) cmd.response + sizeof resp, 1580 in_len - sizeof cmd, out_len - sizeof resp); 1581 1582 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1583 if (!cq) 1584 return -EINVAL; 1585 1586 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 1587 if (ret) 1588 goto out; 1589 1590 resp.cqe = cq->cqe; 1591 1592 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1593 &resp, sizeof resp.cqe)) 1594 ret = -EFAULT; 1595 1596 out: 1597 put_cq_read(cq); 1598 1599 return ret ? ret : in_len; 1600 } 1601 1602 static int copy_wc_to_user(void __user *dest, struct ib_wc *wc) 1603 { 1604 struct ib_uverbs_wc tmp; 1605 1606 tmp.wr_id = wc->wr_id; 1607 tmp.status = wc->status; 1608 tmp.opcode = wc->opcode; 1609 tmp.vendor_err = wc->vendor_err; 1610 tmp.byte_len = wc->byte_len; 1611 tmp.ex.imm_data = (__u32 __force) wc->ex.imm_data; 1612 tmp.qp_num = wc->qp->qp_num; 1613 tmp.src_qp = wc->src_qp; 1614 tmp.wc_flags = wc->wc_flags; 1615 tmp.pkey_index = wc->pkey_index; 1616 tmp.slid = wc->slid; 1617 tmp.sl = wc->sl; 1618 tmp.dlid_path_bits = wc->dlid_path_bits; 1619 tmp.port_num = wc->port_num; 1620 tmp.reserved = 0; 1621 1622 if (copy_to_user(dest, &tmp, sizeof tmp)) 1623 return -EFAULT; 1624 1625 return 0; 1626 } 1627 1628 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 1629 struct ib_device *ib_dev, 1630 const char __user *buf, int in_len, 1631 int out_len) 1632 { 1633 struct ib_uverbs_poll_cq cmd; 1634 struct ib_uverbs_poll_cq_resp resp; 1635 u8 __user *header_ptr; 1636 u8 __user *data_ptr; 1637 struct ib_cq *cq; 1638 struct ib_wc wc; 1639 int ret; 1640 1641 if (copy_from_user(&cmd, buf, sizeof cmd)) 1642 return -EFAULT; 1643 1644 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1645 if (!cq) 1646 return -EINVAL; 1647 1648 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1649 header_ptr = (void __user *)(unsigned long) cmd.response; 1650 data_ptr = header_ptr + sizeof resp; 1651 1652 memset(&resp, 0, sizeof resp); 1653 while (resp.count < cmd.ne) { 1654 ret = ib_poll_cq(cq, 1, &wc); 1655 if (ret < 0) 1656 goto out_put; 1657 if (!ret) 1658 break; 1659 1660 ret = copy_wc_to_user(data_ptr, &wc); 1661 if (ret) 1662 goto out_put; 1663 1664 data_ptr += sizeof(struct ib_uverbs_wc); 1665 ++resp.count; 1666 } 1667 1668 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1669 ret = -EFAULT; 1670 goto out_put; 1671 } 1672 1673 ret = in_len; 1674 1675 out_put: 1676 put_cq_read(cq); 1677 return ret; 1678 } 1679 1680 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 1681 struct ib_device *ib_dev, 1682 const char __user *buf, int in_len, 1683 int out_len) 1684 { 1685 struct ib_uverbs_req_notify_cq cmd; 1686 struct ib_cq *cq; 1687 1688 if (copy_from_user(&cmd, buf, sizeof cmd)) 1689 return -EFAULT; 1690 1691 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1692 if (!cq) 1693 return -EINVAL; 1694 1695 ib_req_notify_cq(cq, cmd.solicited_only ? 1696 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1697 1698 put_cq_read(cq); 1699 1700 return in_len; 1701 } 1702 1703 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 1704 struct ib_device *ib_dev, 1705 const char __user *buf, int in_len, 1706 int out_len) 1707 { 1708 struct ib_uverbs_destroy_cq cmd; 1709 struct ib_uverbs_destroy_cq_resp resp; 1710 struct ib_uobject *uobj; 1711 struct ib_cq *cq; 1712 struct ib_ucq_object *obj; 1713 struct ib_uverbs_event_file *ev_file; 1714 int ret = -EINVAL; 1715 1716 if (copy_from_user(&cmd, buf, sizeof cmd)) 1717 return -EFAULT; 1718 1719 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); 1720 if (!uobj) 1721 return -EINVAL; 1722 cq = uobj->object; 1723 ev_file = cq->cq_context; 1724 obj = container_of(cq->uobject, struct ib_ucq_object, uobject); 1725 1726 ret = ib_destroy_cq(cq); 1727 if (!ret) 1728 uobj->live = 0; 1729 1730 put_uobj_write(uobj); 1731 1732 if (ret) 1733 return ret; 1734 1735 idr_remove_uobj(&ib_uverbs_cq_idr, uobj); 1736 1737 mutex_lock(&file->mutex); 1738 list_del(&uobj->list); 1739 mutex_unlock(&file->mutex); 1740 1741 ib_uverbs_release_ucq(file, ev_file, obj); 1742 1743 memset(&resp, 0, sizeof resp); 1744 resp.comp_events_reported = obj->comp_events_reported; 1745 resp.async_events_reported = obj->async_events_reported; 1746 1747 put_uobj(uobj); 1748 1749 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1750 &resp, sizeof resp)) 1751 return -EFAULT; 1752 1753 return in_len; 1754 } 1755 1756 static int create_qp(struct ib_uverbs_file *file, 1757 struct ib_udata *ucore, 1758 struct ib_udata *uhw, 1759 struct ib_uverbs_ex_create_qp *cmd, 1760 size_t cmd_sz, 1761 int (*cb)(struct ib_uverbs_file *file, 1762 struct ib_uverbs_ex_create_qp_resp *resp, 1763 struct ib_udata *udata), 1764 void *context) 1765 { 1766 struct ib_uqp_object *obj; 1767 struct ib_device *device; 1768 struct ib_pd *pd = NULL; 1769 struct ib_xrcd *xrcd = NULL; 1770 struct ib_uobject *uninitialized_var(xrcd_uobj); 1771 struct ib_cq *scq = NULL, *rcq = NULL; 1772 struct ib_srq *srq = NULL; 1773 struct ib_qp *qp; 1774 char *buf; 1775 struct ib_qp_init_attr attr = {}; 1776 struct ib_uverbs_ex_create_qp_resp resp; 1777 int ret; 1778 struct ib_rwq_ind_table *ind_tbl = NULL; 1779 bool has_sq = true; 1780 1781 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1782 return -EPERM; 1783 1784 obj = kzalloc(sizeof *obj, GFP_KERNEL); 1785 if (!obj) 1786 return -ENOMEM; 1787 1788 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, 1789 &qp_lock_class); 1790 down_write(&obj->uevent.uobject.mutex); 1791 if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) + 1792 sizeof(cmd->rwq_ind_tbl_handle) && 1793 (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) { 1794 ind_tbl = idr_read_rwq_indirection_table(cmd->rwq_ind_tbl_handle, 1795 file->ucontext); 1796 if (!ind_tbl) { 1797 ret = -EINVAL; 1798 goto err_put; 1799 } 1800 1801 attr.rwq_ind_tbl = ind_tbl; 1802 } 1803 1804 if ((cmd_sz >= offsetof(typeof(*cmd), reserved1) + 1805 sizeof(cmd->reserved1)) && cmd->reserved1) { 1806 ret = -EOPNOTSUPP; 1807 goto err_put; 1808 } 1809 1810 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) { 1811 ret = -EINVAL; 1812 goto err_put; 1813 } 1814 1815 if (ind_tbl && !cmd->max_send_wr) 1816 has_sq = false; 1817 1818 if (cmd->qp_type == IB_QPT_XRC_TGT) { 1819 xrcd = idr_read_xrcd(cmd->pd_handle, file->ucontext, 1820 &xrcd_uobj); 1821 if (!xrcd) { 1822 ret = -EINVAL; 1823 goto err_put; 1824 } 1825 device = xrcd->device; 1826 } else { 1827 if (cmd->qp_type == IB_QPT_XRC_INI) { 1828 cmd->max_recv_wr = 0; 1829 cmd->max_recv_sge = 0; 1830 } else { 1831 if (cmd->is_srq) { 1832 srq = idr_read_srq(cmd->srq_handle, 1833 file->ucontext); 1834 if (!srq || srq->srq_type != IB_SRQT_BASIC) { 1835 ret = -EINVAL; 1836 goto err_put; 1837 } 1838 } 1839 1840 if (!ind_tbl) { 1841 if (cmd->recv_cq_handle != cmd->send_cq_handle) { 1842 rcq = idr_read_cq(cmd->recv_cq_handle, 1843 file->ucontext, 0); 1844 if (!rcq) { 1845 ret = -EINVAL; 1846 goto err_put; 1847 } 1848 } 1849 } 1850 } 1851 1852 if (has_sq) 1853 scq = idr_read_cq(cmd->send_cq_handle, file->ucontext, !!rcq); 1854 if (!ind_tbl) 1855 rcq = rcq ?: scq; 1856 pd = idr_read_pd(cmd->pd_handle, file->ucontext); 1857 if (!pd || (!scq && has_sq)) { 1858 ret = -EINVAL; 1859 goto err_put; 1860 } 1861 1862 device = pd->device; 1863 } 1864 1865 attr.event_handler = ib_uverbs_qp_event_handler; 1866 attr.qp_context = file; 1867 attr.send_cq = scq; 1868 attr.recv_cq = rcq; 1869 attr.srq = srq; 1870 attr.xrcd = xrcd; 1871 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : 1872 IB_SIGNAL_REQ_WR; 1873 attr.qp_type = cmd->qp_type; 1874 attr.create_flags = 0; 1875 1876 attr.cap.max_send_wr = cmd->max_send_wr; 1877 attr.cap.max_recv_wr = cmd->max_recv_wr; 1878 attr.cap.max_send_sge = cmd->max_send_sge; 1879 attr.cap.max_recv_sge = cmd->max_recv_sge; 1880 attr.cap.max_inline_data = cmd->max_inline_data; 1881 1882 obj->uevent.events_reported = 0; 1883 INIT_LIST_HEAD(&obj->uevent.event_list); 1884 INIT_LIST_HEAD(&obj->mcast_list); 1885 1886 if (cmd_sz >= offsetof(typeof(*cmd), create_flags) + 1887 sizeof(cmd->create_flags)) 1888 attr.create_flags = cmd->create_flags; 1889 1890 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 1891 IB_QP_CREATE_CROSS_CHANNEL | 1892 IB_QP_CREATE_MANAGED_SEND | 1893 IB_QP_CREATE_MANAGED_RECV | 1894 IB_QP_CREATE_SCATTER_FCS | 1895 IB_QP_CREATE_CVLAN_STRIPPING)) { 1896 ret = -EINVAL; 1897 goto err_put; 1898 } 1899 1900 buf = (void *)cmd + sizeof(*cmd); 1901 if (cmd_sz > sizeof(*cmd)) 1902 if (!(buf[0] == 0 && !memcmp(buf, buf + 1, 1903 cmd_sz - sizeof(*cmd) - 1))) { 1904 ret = -EINVAL; 1905 goto err_put; 1906 } 1907 1908 if (cmd->qp_type == IB_QPT_XRC_TGT) 1909 qp = ib_create_qp(pd, &attr); 1910 else 1911 qp = device->create_qp(pd, &attr, uhw); 1912 1913 if (IS_ERR(qp)) { 1914 ret = PTR_ERR(qp); 1915 goto err_put; 1916 } 1917 1918 if (cmd->qp_type != IB_QPT_XRC_TGT) { 1919 qp->real_qp = qp; 1920 qp->device = device; 1921 qp->pd = pd; 1922 qp->send_cq = attr.send_cq; 1923 qp->recv_cq = attr.recv_cq; 1924 qp->srq = attr.srq; 1925 qp->rwq_ind_tbl = ind_tbl; 1926 qp->event_handler = attr.event_handler; 1927 qp->qp_context = attr.qp_context; 1928 qp->qp_type = attr.qp_type; 1929 atomic_set(&qp->usecnt, 0); 1930 atomic_inc(&pd->usecnt); 1931 if (attr.send_cq) 1932 atomic_inc(&attr.send_cq->usecnt); 1933 if (attr.recv_cq) 1934 atomic_inc(&attr.recv_cq->usecnt); 1935 if (attr.srq) 1936 atomic_inc(&attr.srq->usecnt); 1937 if (ind_tbl) 1938 atomic_inc(&ind_tbl->usecnt); 1939 } 1940 qp->uobject = &obj->uevent.uobject; 1941 1942 obj->uevent.uobject.object = qp; 1943 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1944 if (ret) 1945 goto err_destroy; 1946 1947 memset(&resp, 0, sizeof resp); 1948 resp.base.qpn = qp->qp_num; 1949 resp.base.qp_handle = obj->uevent.uobject.id; 1950 resp.base.max_recv_sge = attr.cap.max_recv_sge; 1951 resp.base.max_send_sge = attr.cap.max_send_sge; 1952 resp.base.max_recv_wr = attr.cap.max_recv_wr; 1953 resp.base.max_send_wr = attr.cap.max_send_wr; 1954 resp.base.max_inline_data = attr.cap.max_inline_data; 1955 1956 resp.response_length = offsetof(typeof(resp), response_length) + 1957 sizeof(resp.response_length); 1958 1959 ret = cb(file, &resp, ucore); 1960 if (ret) 1961 goto err_cb; 1962 1963 if (xrcd) { 1964 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 1965 uobject); 1966 atomic_inc(&obj->uxrcd->refcnt); 1967 put_xrcd_read(xrcd_uobj); 1968 } 1969 1970 if (pd) 1971 put_pd_read(pd); 1972 if (scq) 1973 put_cq_read(scq); 1974 if (rcq && rcq != scq) 1975 put_cq_read(rcq); 1976 if (srq) 1977 put_srq_read(srq); 1978 if (ind_tbl) 1979 put_rwq_indirection_table_read(ind_tbl); 1980 1981 mutex_lock(&file->mutex); 1982 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 1983 mutex_unlock(&file->mutex); 1984 1985 obj->uevent.uobject.live = 1; 1986 1987 up_write(&obj->uevent.uobject.mutex); 1988 1989 return 0; 1990 err_cb: 1991 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1992 1993 err_destroy: 1994 ib_destroy_qp(qp); 1995 1996 err_put: 1997 if (xrcd) 1998 put_xrcd_read(xrcd_uobj); 1999 if (pd) 2000 put_pd_read(pd); 2001 if (scq) 2002 put_cq_read(scq); 2003 if (rcq && rcq != scq) 2004 put_cq_read(rcq); 2005 if (srq) 2006 put_srq_read(srq); 2007 if (ind_tbl) 2008 put_rwq_indirection_table_read(ind_tbl); 2009 2010 put_uobj_write(&obj->uevent.uobject); 2011 return ret; 2012 } 2013 2014 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file, 2015 struct ib_uverbs_ex_create_qp_resp *resp, 2016 struct ib_udata *ucore) 2017 { 2018 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 2019 return -EFAULT; 2020 2021 return 0; 2022 } 2023 2024 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 2025 struct ib_device *ib_dev, 2026 const char __user *buf, int in_len, 2027 int out_len) 2028 { 2029 struct ib_uverbs_create_qp cmd; 2030 struct ib_uverbs_ex_create_qp cmd_ex; 2031 struct ib_udata ucore; 2032 struct ib_udata uhw; 2033 ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp); 2034 int err; 2035 2036 if (out_len < resp_size) 2037 return -ENOSPC; 2038 2039 if (copy_from_user(&cmd, buf, sizeof(cmd))) 2040 return -EFAULT; 2041 2042 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), 2043 resp_size); 2044 INIT_UDATA(&uhw, buf + sizeof(cmd), 2045 (unsigned long)cmd.response + resp_size, 2046 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 2047 out_len - resp_size); 2048 2049 memset(&cmd_ex, 0, sizeof(cmd_ex)); 2050 cmd_ex.user_handle = cmd.user_handle; 2051 cmd_ex.pd_handle = cmd.pd_handle; 2052 cmd_ex.send_cq_handle = cmd.send_cq_handle; 2053 cmd_ex.recv_cq_handle = cmd.recv_cq_handle; 2054 cmd_ex.srq_handle = cmd.srq_handle; 2055 cmd_ex.max_send_wr = cmd.max_send_wr; 2056 cmd_ex.max_recv_wr = cmd.max_recv_wr; 2057 cmd_ex.max_send_sge = cmd.max_send_sge; 2058 cmd_ex.max_recv_sge = cmd.max_recv_sge; 2059 cmd_ex.max_inline_data = cmd.max_inline_data; 2060 cmd_ex.sq_sig_all = cmd.sq_sig_all; 2061 cmd_ex.qp_type = cmd.qp_type; 2062 cmd_ex.is_srq = cmd.is_srq; 2063 2064 err = create_qp(file, &ucore, &uhw, &cmd_ex, 2065 offsetof(typeof(cmd_ex), is_srq) + 2066 sizeof(cmd.is_srq), ib_uverbs_create_qp_cb, 2067 NULL); 2068 2069 if (err) 2070 return err; 2071 2072 return in_len; 2073 } 2074 2075 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file, 2076 struct ib_uverbs_ex_create_qp_resp *resp, 2077 struct ib_udata *ucore) 2078 { 2079 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 2080 return -EFAULT; 2081 2082 return 0; 2083 } 2084 2085 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file, 2086 struct ib_device *ib_dev, 2087 struct ib_udata *ucore, 2088 struct ib_udata *uhw) 2089 { 2090 struct ib_uverbs_ex_create_qp_resp resp; 2091 struct ib_uverbs_ex_create_qp cmd = {0}; 2092 int err; 2093 2094 if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) + 2095 sizeof(cmd.comp_mask))) 2096 return -EINVAL; 2097 2098 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2099 if (err) 2100 return err; 2101 2102 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK) 2103 return -EINVAL; 2104 2105 if (cmd.reserved) 2106 return -EINVAL; 2107 2108 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 2109 sizeof(resp.response_length))) 2110 return -ENOSPC; 2111 2112 err = create_qp(file, ucore, uhw, &cmd, 2113 min(ucore->inlen, sizeof(cmd)), 2114 ib_uverbs_ex_create_qp_cb, NULL); 2115 2116 if (err) 2117 return err; 2118 2119 return 0; 2120 } 2121 2122 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, 2123 struct ib_device *ib_dev, 2124 const char __user *buf, int in_len, int out_len) 2125 { 2126 struct ib_uverbs_open_qp cmd; 2127 struct ib_uverbs_create_qp_resp resp; 2128 struct ib_udata udata; 2129 struct ib_uqp_object *obj; 2130 struct ib_xrcd *xrcd; 2131 struct ib_uobject *uninitialized_var(xrcd_uobj); 2132 struct ib_qp *qp; 2133 struct ib_qp_open_attr attr; 2134 int ret; 2135 2136 if (out_len < sizeof resp) 2137 return -ENOSPC; 2138 2139 if (copy_from_user(&cmd, buf, sizeof cmd)) 2140 return -EFAULT; 2141 2142 INIT_UDATA(&udata, buf + sizeof cmd, 2143 (unsigned long) cmd.response + sizeof resp, 2144 in_len - sizeof cmd, out_len - sizeof resp); 2145 2146 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2147 if (!obj) 2148 return -ENOMEM; 2149 2150 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class); 2151 down_write(&obj->uevent.uobject.mutex); 2152 2153 xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj); 2154 if (!xrcd) { 2155 ret = -EINVAL; 2156 goto err_put; 2157 } 2158 2159 attr.event_handler = ib_uverbs_qp_event_handler; 2160 attr.qp_context = file; 2161 attr.qp_num = cmd.qpn; 2162 attr.qp_type = cmd.qp_type; 2163 2164 obj->uevent.events_reported = 0; 2165 INIT_LIST_HEAD(&obj->uevent.event_list); 2166 INIT_LIST_HEAD(&obj->mcast_list); 2167 2168 qp = ib_open_qp(xrcd, &attr); 2169 if (IS_ERR(qp)) { 2170 ret = PTR_ERR(qp); 2171 goto err_put; 2172 } 2173 2174 qp->uobject = &obj->uevent.uobject; 2175 2176 obj->uevent.uobject.object = qp; 2177 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 2178 if (ret) 2179 goto err_destroy; 2180 2181 memset(&resp, 0, sizeof resp); 2182 resp.qpn = qp->qp_num; 2183 resp.qp_handle = obj->uevent.uobject.id; 2184 2185 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2186 &resp, sizeof resp)) { 2187 ret = -EFAULT; 2188 goto err_remove; 2189 } 2190 2191 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 2192 atomic_inc(&obj->uxrcd->refcnt); 2193 put_xrcd_read(xrcd_uobj); 2194 2195 mutex_lock(&file->mutex); 2196 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 2197 mutex_unlock(&file->mutex); 2198 2199 obj->uevent.uobject.live = 1; 2200 2201 up_write(&obj->uevent.uobject.mutex); 2202 2203 return in_len; 2204 2205 err_remove: 2206 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 2207 2208 err_destroy: 2209 ib_destroy_qp(qp); 2210 2211 err_put: 2212 put_xrcd_read(xrcd_uobj); 2213 put_uobj_write(&obj->uevent.uobject); 2214 return ret; 2215 } 2216 2217 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 2218 struct ib_device *ib_dev, 2219 const char __user *buf, int in_len, 2220 int out_len) 2221 { 2222 struct ib_uverbs_query_qp cmd; 2223 struct ib_uverbs_query_qp_resp resp; 2224 struct ib_qp *qp; 2225 struct ib_qp_attr *attr; 2226 struct ib_qp_init_attr *init_attr; 2227 int ret; 2228 2229 if (copy_from_user(&cmd, buf, sizeof cmd)) 2230 return -EFAULT; 2231 2232 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2233 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 2234 if (!attr || !init_attr) { 2235 ret = -ENOMEM; 2236 goto out; 2237 } 2238 2239 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2240 if (!qp) { 2241 ret = -EINVAL; 2242 goto out; 2243 } 2244 2245 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 2246 2247 put_qp_read(qp); 2248 2249 if (ret) 2250 goto out; 2251 2252 memset(&resp, 0, sizeof resp); 2253 2254 resp.qp_state = attr->qp_state; 2255 resp.cur_qp_state = attr->cur_qp_state; 2256 resp.path_mtu = attr->path_mtu; 2257 resp.path_mig_state = attr->path_mig_state; 2258 resp.qkey = attr->qkey; 2259 resp.rq_psn = attr->rq_psn; 2260 resp.sq_psn = attr->sq_psn; 2261 resp.dest_qp_num = attr->dest_qp_num; 2262 resp.qp_access_flags = attr->qp_access_flags; 2263 resp.pkey_index = attr->pkey_index; 2264 resp.alt_pkey_index = attr->alt_pkey_index; 2265 resp.sq_draining = attr->sq_draining; 2266 resp.max_rd_atomic = attr->max_rd_atomic; 2267 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 2268 resp.min_rnr_timer = attr->min_rnr_timer; 2269 resp.port_num = attr->port_num; 2270 resp.timeout = attr->timeout; 2271 resp.retry_cnt = attr->retry_cnt; 2272 resp.rnr_retry = attr->rnr_retry; 2273 resp.alt_port_num = attr->alt_port_num; 2274 resp.alt_timeout = attr->alt_timeout; 2275 2276 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 2277 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 2278 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 2279 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 2280 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 2281 resp.dest.dlid = attr->ah_attr.dlid; 2282 resp.dest.sl = attr->ah_attr.sl; 2283 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 2284 resp.dest.static_rate = attr->ah_attr.static_rate; 2285 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 2286 resp.dest.port_num = attr->ah_attr.port_num; 2287 2288 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 2289 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 2290 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 2291 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 2292 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 2293 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 2294 resp.alt_dest.sl = attr->alt_ah_attr.sl; 2295 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 2296 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 2297 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 2298 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 2299 2300 resp.max_send_wr = init_attr->cap.max_send_wr; 2301 resp.max_recv_wr = init_attr->cap.max_recv_wr; 2302 resp.max_send_sge = init_attr->cap.max_send_sge; 2303 resp.max_recv_sge = init_attr->cap.max_recv_sge; 2304 resp.max_inline_data = init_attr->cap.max_inline_data; 2305 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2306 2307 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2308 &resp, sizeof resp)) 2309 ret = -EFAULT; 2310 2311 out: 2312 kfree(attr); 2313 kfree(init_attr); 2314 2315 return ret ? ret : in_len; 2316 } 2317 2318 /* Remove ignored fields set in the attribute mask */ 2319 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 2320 { 2321 switch (qp_type) { 2322 case IB_QPT_XRC_INI: 2323 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 2324 case IB_QPT_XRC_TGT: 2325 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 2326 IB_QP_RNR_RETRY); 2327 default: 2328 return mask; 2329 } 2330 } 2331 2332 static int modify_qp(struct ib_uverbs_file *file, 2333 struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata) 2334 { 2335 struct ib_qp_attr *attr; 2336 struct ib_qp *qp; 2337 int ret; 2338 2339 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2340 if (!attr) 2341 return -ENOMEM; 2342 2343 qp = idr_read_qp(cmd->base.qp_handle, file->ucontext); 2344 if (!qp) { 2345 ret = -EINVAL; 2346 goto out; 2347 } 2348 2349 attr->qp_state = cmd->base.qp_state; 2350 attr->cur_qp_state = cmd->base.cur_qp_state; 2351 attr->path_mtu = cmd->base.path_mtu; 2352 attr->path_mig_state = cmd->base.path_mig_state; 2353 attr->qkey = cmd->base.qkey; 2354 attr->rq_psn = cmd->base.rq_psn; 2355 attr->sq_psn = cmd->base.sq_psn; 2356 attr->dest_qp_num = cmd->base.dest_qp_num; 2357 attr->qp_access_flags = cmd->base.qp_access_flags; 2358 attr->pkey_index = cmd->base.pkey_index; 2359 attr->alt_pkey_index = cmd->base.alt_pkey_index; 2360 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify; 2361 attr->max_rd_atomic = cmd->base.max_rd_atomic; 2362 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic; 2363 attr->min_rnr_timer = cmd->base.min_rnr_timer; 2364 attr->port_num = cmd->base.port_num; 2365 attr->timeout = cmd->base.timeout; 2366 attr->retry_cnt = cmd->base.retry_cnt; 2367 attr->rnr_retry = cmd->base.rnr_retry; 2368 attr->alt_port_num = cmd->base.alt_port_num; 2369 attr->alt_timeout = cmd->base.alt_timeout; 2370 attr->rate_limit = cmd->rate_limit; 2371 2372 memcpy(attr->ah_attr.grh.dgid.raw, cmd->base.dest.dgid, 16); 2373 attr->ah_attr.grh.flow_label = cmd->base.dest.flow_label; 2374 attr->ah_attr.grh.sgid_index = cmd->base.dest.sgid_index; 2375 attr->ah_attr.grh.hop_limit = cmd->base.dest.hop_limit; 2376 attr->ah_attr.grh.traffic_class = cmd->base.dest.traffic_class; 2377 attr->ah_attr.dlid = cmd->base.dest.dlid; 2378 attr->ah_attr.sl = cmd->base.dest.sl; 2379 attr->ah_attr.src_path_bits = cmd->base.dest.src_path_bits; 2380 attr->ah_attr.static_rate = cmd->base.dest.static_rate; 2381 attr->ah_attr.ah_flags = cmd->base.dest.is_global ? 2382 IB_AH_GRH : 0; 2383 attr->ah_attr.port_num = cmd->base.dest.port_num; 2384 2385 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd->base.alt_dest.dgid, 16); 2386 attr->alt_ah_attr.grh.flow_label = cmd->base.alt_dest.flow_label; 2387 attr->alt_ah_attr.grh.sgid_index = cmd->base.alt_dest.sgid_index; 2388 attr->alt_ah_attr.grh.hop_limit = cmd->base.alt_dest.hop_limit; 2389 attr->alt_ah_attr.grh.traffic_class = cmd->base.alt_dest.traffic_class; 2390 attr->alt_ah_attr.dlid = cmd->base.alt_dest.dlid; 2391 attr->alt_ah_attr.sl = cmd->base.alt_dest.sl; 2392 attr->alt_ah_attr.src_path_bits = cmd->base.alt_dest.src_path_bits; 2393 attr->alt_ah_attr.static_rate = cmd->base.alt_dest.static_rate; 2394 attr->alt_ah_attr.ah_flags = cmd->base.alt_dest.is_global ? 2395 IB_AH_GRH : 0; 2396 attr->alt_ah_attr.port_num = cmd->base.alt_dest.port_num; 2397 2398 if (qp->real_qp == qp) { 2399 if (cmd->base.attr_mask & IB_QP_AV) { 2400 ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr); 2401 if (ret) 2402 goto release_qp; 2403 } 2404 ret = qp->device->modify_qp(qp, attr, 2405 modify_qp_mask(qp->qp_type, 2406 cmd->base.attr_mask), 2407 udata); 2408 } else { 2409 ret = ib_modify_qp(qp, attr, 2410 modify_qp_mask(qp->qp_type, 2411 cmd->base.attr_mask)); 2412 } 2413 2414 release_qp: 2415 put_qp_read(qp); 2416 2417 out: 2418 kfree(attr); 2419 2420 return ret; 2421 } 2422 2423 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 2424 struct ib_device *ib_dev, 2425 const char __user *buf, int in_len, 2426 int out_len) 2427 { 2428 struct ib_uverbs_ex_modify_qp cmd = {}; 2429 struct ib_udata udata; 2430 int ret; 2431 2432 if (copy_from_user(&cmd.base, buf, sizeof(cmd.base))) 2433 return -EFAULT; 2434 2435 if (cmd.base.attr_mask & 2436 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1)) 2437 return -EOPNOTSUPP; 2438 2439 INIT_UDATA(&udata, buf + sizeof(cmd.base), NULL, 2440 in_len - sizeof(cmd.base), out_len); 2441 2442 ret = modify_qp(file, &cmd, &udata); 2443 if (ret) 2444 return ret; 2445 2446 return in_len; 2447 } 2448 2449 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file, 2450 struct ib_device *ib_dev, 2451 struct ib_udata *ucore, 2452 struct ib_udata *uhw) 2453 { 2454 struct ib_uverbs_ex_modify_qp cmd = {}; 2455 int ret; 2456 2457 /* 2458 * Last bit is reserved for extending the attr_mask by 2459 * using another field. 2460 */ 2461 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); 2462 2463 if (ucore->inlen < sizeof(cmd.base)) 2464 return -EINVAL; 2465 2466 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2467 if (ret) 2468 return ret; 2469 2470 if (cmd.base.attr_mask & 2471 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) 2472 return -EOPNOTSUPP; 2473 2474 if (ucore->inlen > sizeof(cmd)) { 2475 if (ib_is_udata_cleared(ucore, sizeof(cmd), 2476 ucore->inlen - sizeof(cmd))) 2477 return -EOPNOTSUPP; 2478 } 2479 2480 ret = modify_qp(file, &cmd, uhw); 2481 2482 return ret; 2483 } 2484 2485 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 2486 struct ib_device *ib_dev, 2487 const char __user *buf, int in_len, 2488 int out_len) 2489 { 2490 struct ib_uverbs_destroy_qp cmd; 2491 struct ib_uverbs_destroy_qp_resp resp; 2492 struct ib_uobject *uobj; 2493 struct ib_qp *qp; 2494 struct ib_uqp_object *obj; 2495 int ret = -EINVAL; 2496 2497 if (copy_from_user(&cmd, buf, sizeof cmd)) 2498 return -EFAULT; 2499 2500 memset(&resp, 0, sizeof resp); 2501 2502 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); 2503 if (!uobj) 2504 return -EINVAL; 2505 qp = uobj->object; 2506 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 2507 2508 if (!list_empty(&obj->mcast_list)) { 2509 put_uobj_write(uobj); 2510 return -EBUSY; 2511 } 2512 2513 ret = ib_destroy_qp(qp); 2514 if (!ret) 2515 uobj->live = 0; 2516 2517 put_uobj_write(uobj); 2518 2519 if (ret) 2520 return ret; 2521 2522 if (obj->uxrcd) 2523 atomic_dec(&obj->uxrcd->refcnt); 2524 2525 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 2526 2527 mutex_lock(&file->mutex); 2528 list_del(&uobj->list); 2529 mutex_unlock(&file->mutex); 2530 2531 ib_uverbs_release_uevent(file, &obj->uevent); 2532 2533 resp.events_reported = obj->uevent.events_reported; 2534 2535 put_uobj(uobj); 2536 2537 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2538 &resp, sizeof resp)) 2539 return -EFAULT; 2540 2541 return in_len; 2542 } 2543 2544 static void *alloc_wr(size_t wr_size, __u32 num_sge) 2545 { 2546 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) + 2547 num_sge * sizeof (struct ib_sge), GFP_KERNEL); 2548 }; 2549 2550 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 2551 struct ib_device *ib_dev, 2552 const char __user *buf, int in_len, 2553 int out_len) 2554 { 2555 struct ib_uverbs_post_send cmd; 2556 struct ib_uverbs_post_send_resp resp; 2557 struct ib_uverbs_send_wr *user_wr; 2558 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; 2559 struct ib_qp *qp; 2560 int i, sg_ind; 2561 int is_ud; 2562 ssize_t ret = -EINVAL; 2563 size_t next_size; 2564 2565 if (copy_from_user(&cmd, buf, sizeof cmd)) 2566 return -EFAULT; 2567 2568 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 2569 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 2570 return -EINVAL; 2571 2572 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 2573 return -EINVAL; 2574 2575 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2576 if (!user_wr) 2577 return -ENOMEM; 2578 2579 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2580 if (!qp) 2581 goto out; 2582 2583 is_ud = qp->qp_type == IB_QPT_UD; 2584 sg_ind = 0; 2585 last = NULL; 2586 for (i = 0; i < cmd.wr_count; ++i) { 2587 if (copy_from_user(user_wr, 2588 buf + sizeof cmd + i * cmd.wqe_size, 2589 cmd.wqe_size)) { 2590 ret = -EFAULT; 2591 goto out_put; 2592 } 2593 2594 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2595 ret = -EINVAL; 2596 goto out_put; 2597 } 2598 2599 if (is_ud) { 2600 struct ib_ud_wr *ud; 2601 2602 if (user_wr->opcode != IB_WR_SEND && 2603 user_wr->opcode != IB_WR_SEND_WITH_IMM) { 2604 ret = -EINVAL; 2605 goto out_put; 2606 } 2607 2608 next_size = sizeof(*ud); 2609 ud = alloc_wr(next_size, user_wr->num_sge); 2610 if (!ud) { 2611 ret = -ENOMEM; 2612 goto out_put; 2613 } 2614 2615 ud->ah = idr_read_ah(user_wr->wr.ud.ah, file->ucontext); 2616 if (!ud->ah) { 2617 kfree(ud); 2618 ret = -EINVAL; 2619 goto out_put; 2620 } 2621 ud->remote_qpn = user_wr->wr.ud.remote_qpn; 2622 ud->remote_qkey = user_wr->wr.ud.remote_qkey; 2623 2624 next = &ud->wr; 2625 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2626 user_wr->opcode == IB_WR_RDMA_WRITE || 2627 user_wr->opcode == IB_WR_RDMA_READ) { 2628 struct ib_rdma_wr *rdma; 2629 2630 next_size = sizeof(*rdma); 2631 rdma = alloc_wr(next_size, user_wr->num_sge); 2632 if (!rdma) { 2633 ret = -ENOMEM; 2634 goto out_put; 2635 } 2636 2637 rdma->remote_addr = user_wr->wr.rdma.remote_addr; 2638 rdma->rkey = user_wr->wr.rdma.rkey; 2639 2640 next = &rdma->wr; 2641 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2642 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2643 struct ib_atomic_wr *atomic; 2644 2645 next_size = sizeof(*atomic); 2646 atomic = alloc_wr(next_size, user_wr->num_sge); 2647 if (!atomic) { 2648 ret = -ENOMEM; 2649 goto out_put; 2650 } 2651 2652 atomic->remote_addr = user_wr->wr.atomic.remote_addr; 2653 atomic->compare_add = user_wr->wr.atomic.compare_add; 2654 atomic->swap = user_wr->wr.atomic.swap; 2655 atomic->rkey = user_wr->wr.atomic.rkey; 2656 2657 next = &atomic->wr; 2658 } else if (user_wr->opcode == IB_WR_SEND || 2659 user_wr->opcode == IB_WR_SEND_WITH_IMM || 2660 user_wr->opcode == IB_WR_SEND_WITH_INV) { 2661 next_size = sizeof(*next); 2662 next = alloc_wr(next_size, user_wr->num_sge); 2663 if (!next) { 2664 ret = -ENOMEM; 2665 goto out_put; 2666 } 2667 } else { 2668 ret = -EINVAL; 2669 goto out_put; 2670 } 2671 2672 if (user_wr->opcode == IB_WR_SEND_WITH_IMM || 2673 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 2674 next->ex.imm_data = 2675 (__be32 __force) user_wr->ex.imm_data; 2676 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) { 2677 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; 2678 } 2679 2680 if (!last) 2681 wr = next; 2682 else 2683 last->next = next; 2684 last = next; 2685 2686 next->next = NULL; 2687 next->wr_id = user_wr->wr_id; 2688 next->num_sge = user_wr->num_sge; 2689 next->opcode = user_wr->opcode; 2690 next->send_flags = user_wr->send_flags; 2691 2692 if (next->num_sge) { 2693 next->sg_list = (void *) next + 2694 ALIGN(next_size, sizeof(struct ib_sge)); 2695 if (copy_from_user(next->sg_list, 2696 buf + sizeof cmd + 2697 cmd.wr_count * cmd.wqe_size + 2698 sg_ind * sizeof (struct ib_sge), 2699 next->num_sge * sizeof (struct ib_sge))) { 2700 ret = -EFAULT; 2701 goto out_put; 2702 } 2703 sg_ind += next->num_sge; 2704 } else 2705 next->sg_list = NULL; 2706 } 2707 2708 resp.bad_wr = 0; 2709 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr); 2710 if (ret) 2711 for (next = wr; next; next = next->next) { 2712 ++resp.bad_wr; 2713 if (next == bad_wr) 2714 break; 2715 } 2716 2717 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2718 &resp, sizeof resp)) 2719 ret = -EFAULT; 2720 2721 out_put: 2722 put_qp_read(qp); 2723 2724 while (wr) { 2725 if (is_ud && ud_wr(wr)->ah) 2726 put_ah_read(ud_wr(wr)->ah); 2727 next = wr->next; 2728 kfree(wr); 2729 wr = next; 2730 } 2731 2732 out: 2733 kfree(user_wr); 2734 2735 return ret ? ret : in_len; 2736 } 2737 2738 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 2739 int in_len, 2740 u32 wr_count, 2741 u32 sge_count, 2742 u32 wqe_size) 2743 { 2744 struct ib_uverbs_recv_wr *user_wr; 2745 struct ib_recv_wr *wr = NULL, *last, *next; 2746 int sg_ind; 2747 int i; 2748 int ret; 2749 2750 if (in_len < wqe_size * wr_count + 2751 sge_count * sizeof (struct ib_uverbs_sge)) 2752 return ERR_PTR(-EINVAL); 2753 2754 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2755 return ERR_PTR(-EINVAL); 2756 2757 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2758 if (!user_wr) 2759 return ERR_PTR(-ENOMEM); 2760 2761 sg_ind = 0; 2762 last = NULL; 2763 for (i = 0; i < wr_count; ++i) { 2764 if (copy_from_user(user_wr, buf + i * wqe_size, 2765 wqe_size)) { 2766 ret = -EFAULT; 2767 goto err; 2768 } 2769 2770 if (user_wr->num_sge + sg_ind > sge_count) { 2771 ret = -EINVAL; 2772 goto err; 2773 } 2774 2775 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2776 user_wr->num_sge * sizeof (struct ib_sge), 2777 GFP_KERNEL); 2778 if (!next) { 2779 ret = -ENOMEM; 2780 goto err; 2781 } 2782 2783 if (!last) 2784 wr = next; 2785 else 2786 last->next = next; 2787 last = next; 2788 2789 next->next = NULL; 2790 next->wr_id = user_wr->wr_id; 2791 next->num_sge = user_wr->num_sge; 2792 2793 if (next->num_sge) { 2794 next->sg_list = (void *) next + 2795 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2796 if (copy_from_user(next->sg_list, 2797 buf + wr_count * wqe_size + 2798 sg_ind * sizeof (struct ib_sge), 2799 next->num_sge * sizeof (struct ib_sge))) { 2800 ret = -EFAULT; 2801 goto err; 2802 } 2803 sg_ind += next->num_sge; 2804 } else 2805 next->sg_list = NULL; 2806 } 2807 2808 kfree(user_wr); 2809 return wr; 2810 2811 err: 2812 kfree(user_wr); 2813 2814 while (wr) { 2815 next = wr->next; 2816 kfree(wr); 2817 wr = next; 2818 } 2819 2820 return ERR_PTR(ret); 2821 } 2822 2823 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 2824 struct ib_device *ib_dev, 2825 const char __user *buf, int in_len, 2826 int out_len) 2827 { 2828 struct ib_uverbs_post_recv cmd; 2829 struct ib_uverbs_post_recv_resp resp; 2830 struct ib_recv_wr *wr, *next, *bad_wr; 2831 struct ib_qp *qp; 2832 ssize_t ret = -EINVAL; 2833 2834 if (copy_from_user(&cmd, buf, sizeof cmd)) 2835 return -EFAULT; 2836 2837 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2838 in_len - sizeof cmd, cmd.wr_count, 2839 cmd.sge_count, cmd.wqe_size); 2840 if (IS_ERR(wr)) 2841 return PTR_ERR(wr); 2842 2843 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2844 if (!qp) 2845 goto out; 2846 2847 resp.bad_wr = 0; 2848 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr); 2849 2850 put_qp_read(qp); 2851 2852 if (ret) 2853 for (next = wr; next; next = next->next) { 2854 ++resp.bad_wr; 2855 if (next == bad_wr) 2856 break; 2857 } 2858 2859 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2860 &resp, sizeof resp)) 2861 ret = -EFAULT; 2862 2863 out: 2864 while (wr) { 2865 next = wr->next; 2866 kfree(wr); 2867 wr = next; 2868 } 2869 2870 return ret ? ret : in_len; 2871 } 2872 2873 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 2874 struct ib_device *ib_dev, 2875 const char __user *buf, int in_len, 2876 int out_len) 2877 { 2878 struct ib_uverbs_post_srq_recv cmd; 2879 struct ib_uverbs_post_srq_recv_resp resp; 2880 struct ib_recv_wr *wr, *next, *bad_wr; 2881 struct ib_srq *srq; 2882 ssize_t ret = -EINVAL; 2883 2884 if (copy_from_user(&cmd, buf, sizeof cmd)) 2885 return -EFAULT; 2886 2887 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2888 in_len - sizeof cmd, cmd.wr_count, 2889 cmd.sge_count, cmd.wqe_size); 2890 if (IS_ERR(wr)) 2891 return PTR_ERR(wr); 2892 2893 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2894 if (!srq) 2895 goto out; 2896 2897 resp.bad_wr = 0; 2898 ret = srq->device->post_srq_recv(srq, wr, &bad_wr); 2899 2900 put_srq_read(srq); 2901 2902 if (ret) 2903 for (next = wr; next; next = next->next) { 2904 ++resp.bad_wr; 2905 if (next == bad_wr) 2906 break; 2907 } 2908 2909 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2910 &resp, sizeof resp)) 2911 ret = -EFAULT; 2912 2913 out: 2914 while (wr) { 2915 next = wr->next; 2916 kfree(wr); 2917 wr = next; 2918 } 2919 2920 return ret ? ret : in_len; 2921 } 2922 2923 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 2924 struct ib_device *ib_dev, 2925 const char __user *buf, int in_len, 2926 int out_len) 2927 { 2928 struct ib_uverbs_create_ah cmd; 2929 struct ib_uverbs_create_ah_resp resp; 2930 struct ib_uobject *uobj; 2931 struct ib_pd *pd; 2932 struct ib_ah *ah; 2933 struct ib_ah_attr attr; 2934 int ret; 2935 struct ib_udata udata; 2936 2937 if (out_len < sizeof resp) 2938 return -ENOSPC; 2939 2940 if (copy_from_user(&cmd, buf, sizeof cmd)) 2941 return -EFAULT; 2942 2943 INIT_UDATA(&udata, buf + sizeof(cmd), 2944 (unsigned long)cmd.response + sizeof(resp), 2945 in_len - sizeof(cmd), out_len - sizeof(resp)); 2946 2947 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 2948 if (!uobj) 2949 return -ENOMEM; 2950 2951 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class); 2952 down_write(&uobj->mutex); 2953 2954 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2955 if (!pd) { 2956 ret = -EINVAL; 2957 goto err; 2958 } 2959 2960 attr.dlid = cmd.attr.dlid; 2961 attr.sl = cmd.attr.sl; 2962 attr.src_path_bits = cmd.attr.src_path_bits; 2963 attr.static_rate = cmd.attr.static_rate; 2964 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; 2965 attr.port_num = cmd.attr.port_num; 2966 attr.grh.flow_label = cmd.attr.grh.flow_label; 2967 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 2968 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 2969 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 2970 memset(&attr.dmac, 0, sizeof(attr.dmac)); 2971 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 2972 2973 ah = pd->device->create_ah(pd, &attr, &udata); 2974 2975 if (IS_ERR(ah)) { 2976 ret = PTR_ERR(ah); 2977 goto err_put; 2978 } 2979 2980 ah->device = pd->device; 2981 ah->pd = pd; 2982 atomic_inc(&pd->usecnt); 2983 ah->uobject = uobj; 2984 uobj->object = ah; 2985 2986 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); 2987 if (ret) 2988 goto err_destroy; 2989 2990 resp.ah_handle = uobj->id; 2991 2992 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2993 &resp, sizeof resp)) { 2994 ret = -EFAULT; 2995 goto err_copy; 2996 } 2997 2998 put_pd_read(pd); 2999 3000 mutex_lock(&file->mutex); 3001 list_add_tail(&uobj->list, &file->ucontext->ah_list); 3002 mutex_unlock(&file->mutex); 3003 3004 uobj->live = 1; 3005 3006 up_write(&uobj->mutex); 3007 3008 return in_len; 3009 3010 err_copy: 3011 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 3012 3013 err_destroy: 3014 ib_destroy_ah(ah); 3015 3016 err_put: 3017 put_pd_read(pd); 3018 3019 err: 3020 put_uobj_write(uobj); 3021 return ret; 3022 } 3023 3024 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 3025 struct ib_device *ib_dev, 3026 const char __user *buf, int in_len, int out_len) 3027 { 3028 struct ib_uverbs_destroy_ah cmd; 3029 struct ib_ah *ah; 3030 struct ib_uobject *uobj; 3031 int ret; 3032 3033 if (copy_from_user(&cmd, buf, sizeof cmd)) 3034 return -EFAULT; 3035 3036 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); 3037 if (!uobj) 3038 return -EINVAL; 3039 ah = uobj->object; 3040 3041 ret = ib_destroy_ah(ah); 3042 if (!ret) 3043 uobj->live = 0; 3044 3045 put_uobj_write(uobj); 3046 3047 if (ret) 3048 return ret; 3049 3050 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 3051 3052 mutex_lock(&file->mutex); 3053 list_del(&uobj->list); 3054 mutex_unlock(&file->mutex); 3055 3056 put_uobj(uobj); 3057 3058 return in_len; 3059 } 3060 3061 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 3062 struct ib_device *ib_dev, 3063 const char __user *buf, int in_len, 3064 int out_len) 3065 { 3066 struct ib_uverbs_attach_mcast cmd; 3067 struct ib_qp *qp; 3068 struct ib_uqp_object *obj; 3069 struct ib_uverbs_mcast_entry *mcast; 3070 int ret; 3071 3072 if (copy_from_user(&cmd, buf, sizeof cmd)) 3073 return -EFAULT; 3074 3075 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 3076 if (!qp) 3077 return -EINVAL; 3078 3079 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 3080 3081 list_for_each_entry(mcast, &obj->mcast_list, list) 3082 if (cmd.mlid == mcast->lid && 3083 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 3084 ret = 0; 3085 goto out_put; 3086 } 3087 3088 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 3089 if (!mcast) { 3090 ret = -ENOMEM; 3091 goto out_put; 3092 } 3093 3094 mcast->lid = cmd.mlid; 3095 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 3096 3097 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 3098 if (!ret) 3099 list_add_tail(&mcast->list, &obj->mcast_list); 3100 else 3101 kfree(mcast); 3102 3103 out_put: 3104 put_qp_write(qp); 3105 3106 return ret ? ret : in_len; 3107 } 3108 3109 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 3110 struct ib_device *ib_dev, 3111 const char __user *buf, int in_len, 3112 int out_len) 3113 { 3114 struct ib_uverbs_detach_mcast cmd; 3115 struct ib_uqp_object *obj; 3116 struct ib_qp *qp; 3117 struct ib_uverbs_mcast_entry *mcast; 3118 int ret = -EINVAL; 3119 3120 if (copy_from_user(&cmd, buf, sizeof cmd)) 3121 return -EFAULT; 3122 3123 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 3124 if (!qp) 3125 return -EINVAL; 3126 3127 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); 3128 if (ret) 3129 goto out_put; 3130 3131 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 3132 3133 list_for_each_entry(mcast, &obj->mcast_list, list) 3134 if (cmd.mlid == mcast->lid && 3135 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 3136 list_del(&mcast->list); 3137 kfree(mcast); 3138 break; 3139 } 3140 3141 out_put: 3142 put_qp_write(qp); 3143 3144 return ret ? ret : in_len; 3145 } 3146 3147 static int kern_spec_to_ib_spec_action(struct ib_uverbs_flow_spec *kern_spec, 3148 union ib_flow_spec *ib_spec) 3149 { 3150 ib_spec->type = kern_spec->type; 3151 switch (ib_spec->type) { 3152 case IB_FLOW_SPEC_ACTION_TAG: 3153 if (kern_spec->flow_tag.size != 3154 sizeof(struct ib_uverbs_flow_spec_action_tag)) 3155 return -EINVAL; 3156 3157 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag); 3158 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id; 3159 break; 3160 default: 3161 return -EINVAL; 3162 } 3163 return 0; 3164 } 3165 3166 static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr *spec) 3167 { 3168 /* Returns user space filter size, includes padding */ 3169 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2; 3170 } 3171 3172 static ssize_t spec_filter_size(void *kern_spec_filter, u16 kern_filter_size, 3173 u16 ib_real_filter_sz) 3174 { 3175 /* 3176 * User space filter structures must be 64 bit aligned, otherwise this 3177 * may pass, but we won't handle additional new attributes. 3178 */ 3179 3180 if (kern_filter_size > ib_real_filter_sz) { 3181 if (memchr_inv(kern_spec_filter + 3182 ib_real_filter_sz, 0, 3183 kern_filter_size - ib_real_filter_sz)) 3184 return -EINVAL; 3185 return ib_real_filter_sz; 3186 } 3187 return kern_filter_size; 3188 } 3189 3190 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec, 3191 union ib_flow_spec *ib_spec) 3192 { 3193 ssize_t actual_filter_sz; 3194 ssize_t kern_filter_sz; 3195 ssize_t ib_filter_sz; 3196 void *kern_spec_mask; 3197 void *kern_spec_val; 3198 3199 if (kern_spec->reserved) 3200 return -EINVAL; 3201 3202 ib_spec->type = kern_spec->type; 3203 3204 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr); 3205 /* User flow spec size must be aligned to 4 bytes */ 3206 if (kern_filter_sz != ALIGN(kern_filter_sz, 4)) 3207 return -EINVAL; 3208 3209 kern_spec_val = (void *)kern_spec + 3210 sizeof(struct ib_uverbs_flow_spec_hdr); 3211 kern_spec_mask = kern_spec_val + kern_filter_sz; 3212 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL)) 3213 return -EINVAL; 3214 3215 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 3216 case IB_FLOW_SPEC_ETH: 3217 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz); 3218 actual_filter_sz = spec_filter_size(kern_spec_mask, 3219 kern_filter_sz, 3220 ib_filter_sz); 3221 if (actual_filter_sz <= 0) 3222 return -EINVAL; 3223 ib_spec->size = sizeof(struct ib_flow_spec_eth); 3224 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz); 3225 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz); 3226 break; 3227 case IB_FLOW_SPEC_IPV4: 3228 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz); 3229 actual_filter_sz = spec_filter_size(kern_spec_mask, 3230 kern_filter_sz, 3231 ib_filter_sz); 3232 if (actual_filter_sz <= 0) 3233 return -EINVAL; 3234 ib_spec->size = sizeof(struct ib_flow_spec_ipv4); 3235 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz); 3236 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz); 3237 break; 3238 case IB_FLOW_SPEC_IPV6: 3239 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz); 3240 actual_filter_sz = spec_filter_size(kern_spec_mask, 3241 kern_filter_sz, 3242 ib_filter_sz); 3243 if (actual_filter_sz <= 0) 3244 return -EINVAL; 3245 ib_spec->size = sizeof(struct ib_flow_spec_ipv6); 3246 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz); 3247 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz); 3248 3249 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) || 3250 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20)) 3251 return -EINVAL; 3252 break; 3253 case IB_FLOW_SPEC_TCP: 3254 case IB_FLOW_SPEC_UDP: 3255 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz); 3256 actual_filter_sz = spec_filter_size(kern_spec_mask, 3257 kern_filter_sz, 3258 ib_filter_sz); 3259 if (actual_filter_sz <= 0) 3260 return -EINVAL; 3261 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp); 3262 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz); 3263 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz); 3264 break; 3265 case IB_FLOW_SPEC_VXLAN_TUNNEL: 3266 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz); 3267 actual_filter_sz = spec_filter_size(kern_spec_mask, 3268 kern_filter_sz, 3269 ib_filter_sz); 3270 if (actual_filter_sz <= 0) 3271 return -EINVAL; 3272 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel); 3273 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz); 3274 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz); 3275 3276 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) || 3277 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24)) 3278 return -EINVAL; 3279 break; 3280 default: 3281 return -EINVAL; 3282 } 3283 return 0; 3284 } 3285 3286 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec, 3287 union ib_flow_spec *ib_spec) 3288 { 3289 if (kern_spec->reserved) 3290 return -EINVAL; 3291 3292 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG) 3293 return kern_spec_to_ib_spec_action(kern_spec, ib_spec); 3294 else 3295 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec); 3296 } 3297 3298 int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, 3299 struct ib_device *ib_dev, 3300 struct ib_udata *ucore, 3301 struct ib_udata *uhw) 3302 { 3303 struct ib_uverbs_ex_create_wq cmd = {}; 3304 struct ib_uverbs_ex_create_wq_resp resp = {}; 3305 struct ib_uwq_object *obj; 3306 int err = 0; 3307 struct ib_cq *cq; 3308 struct ib_pd *pd; 3309 struct ib_wq *wq; 3310 struct ib_wq_init_attr wq_init_attr = {}; 3311 size_t required_cmd_sz; 3312 size_t required_resp_len; 3313 3314 required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge); 3315 required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn); 3316 3317 if (ucore->inlen < required_cmd_sz) 3318 return -EINVAL; 3319 3320 if (ucore->outlen < required_resp_len) 3321 return -ENOSPC; 3322 3323 if (ucore->inlen > sizeof(cmd) && 3324 !ib_is_udata_cleared(ucore, sizeof(cmd), 3325 ucore->inlen - sizeof(cmd))) 3326 return -EOPNOTSUPP; 3327 3328 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3329 if (err) 3330 return err; 3331 3332 if (cmd.comp_mask) 3333 return -EOPNOTSUPP; 3334 3335 obj = kmalloc(sizeof(*obj), GFP_KERNEL); 3336 if (!obj) 3337 return -ENOMEM; 3338 3339 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, 3340 &wq_lock_class); 3341 down_write(&obj->uevent.uobject.mutex); 3342 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 3343 if (!pd) { 3344 err = -EINVAL; 3345 goto err_uobj; 3346 } 3347 3348 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 3349 if (!cq) { 3350 err = -EINVAL; 3351 goto err_put_pd; 3352 } 3353 3354 wq_init_attr.cq = cq; 3355 wq_init_attr.max_sge = cmd.max_sge; 3356 wq_init_attr.max_wr = cmd.max_wr; 3357 wq_init_attr.wq_context = file; 3358 wq_init_attr.wq_type = cmd.wq_type; 3359 wq_init_attr.event_handler = ib_uverbs_wq_event_handler; 3360 if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) + 3361 sizeof(cmd.create_flags))) 3362 wq_init_attr.create_flags = cmd.create_flags; 3363 obj->uevent.events_reported = 0; 3364 INIT_LIST_HEAD(&obj->uevent.event_list); 3365 wq = pd->device->create_wq(pd, &wq_init_attr, uhw); 3366 if (IS_ERR(wq)) { 3367 err = PTR_ERR(wq); 3368 goto err_put_cq; 3369 } 3370 3371 wq->uobject = &obj->uevent.uobject; 3372 obj->uevent.uobject.object = wq; 3373 wq->wq_type = wq_init_attr.wq_type; 3374 wq->cq = cq; 3375 wq->pd = pd; 3376 wq->device = pd->device; 3377 wq->wq_context = wq_init_attr.wq_context; 3378 atomic_set(&wq->usecnt, 0); 3379 atomic_inc(&pd->usecnt); 3380 atomic_inc(&cq->usecnt); 3381 wq->uobject = &obj->uevent.uobject; 3382 obj->uevent.uobject.object = wq; 3383 err = idr_add_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); 3384 if (err) 3385 goto destroy_wq; 3386 3387 memset(&resp, 0, sizeof(resp)); 3388 resp.wq_handle = obj->uevent.uobject.id; 3389 resp.max_sge = wq_init_attr.max_sge; 3390 resp.max_wr = wq_init_attr.max_wr; 3391 resp.wqn = wq->wq_num; 3392 resp.response_length = required_resp_len; 3393 err = ib_copy_to_udata(ucore, 3394 &resp, resp.response_length); 3395 if (err) 3396 goto err_copy; 3397 3398 put_pd_read(pd); 3399 put_cq_read(cq); 3400 3401 mutex_lock(&file->mutex); 3402 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->wq_list); 3403 mutex_unlock(&file->mutex); 3404 3405 obj->uevent.uobject.live = 1; 3406 up_write(&obj->uevent.uobject.mutex); 3407 return 0; 3408 3409 err_copy: 3410 idr_remove_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); 3411 destroy_wq: 3412 ib_destroy_wq(wq); 3413 err_put_cq: 3414 put_cq_read(cq); 3415 err_put_pd: 3416 put_pd_read(pd); 3417 err_uobj: 3418 put_uobj_write(&obj->uevent.uobject); 3419 3420 return err; 3421 } 3422 3423 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file, 3424 struct ib_device *ib_dev, 3425 struct ib_udata *ucore, 3426 struct ib_udata *uhw) 3427 { 3428 struct ib_uverbs_ex_destroy_wq cmd = {}; 3429 struct ib_uverbs_ex_destroy_wq_resp resp = {}; 3430 struct ib_wq *wq; 3431 struct ib_uobject *uobj; 3432 struct ib_uwq_object *obj; 3433 size_t required_cmd_sz; 3434 size_t required_resp_len; 3435 int ret; 3436 3437 required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle); 3438 required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); 3439 3440 if (ucore->inlen < required_cmd_sz) 3441 return -EINVAL; 3442 3443 if (ucore->outlen < required_resp_len) 3444 return -ENOSPC; 3445 3446 if (ucore->inlen > sizeof(cmd) && 3447 !ib_is_udata_cleared(ucore, sizeof(cmd), 3448 ucore->inlen - sizeof(cmd))) 3449 return -EOPNOTSUPP; 3450 3451 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3452 if (ret) 3453 return ret; 3454 3455 if (cmd.comp_mask) 3456 return -EOPNOTSUPP; 3457 3458 resp.response_length = required_resp_len; 3459 uobj = idr_write_uobj(&ib_uverbs_wq_idr, cmd.wq_handle, 3460 file->ucontext); 3461 if (!uobj) 3462 return -EINVAL; 3463 3464 wq = uobj->object; 3465 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); 3466 ret = ib_destroy_wq(wq); 3467 if (!ret) 3468 uobj->live = 0; 3469 3470 put_uobj_write(uobj); 3471 if (ret) 3472 return ret; 3473 3474 idr_remove_uobj(&ib_uverbs_wq_idr, uobj); 3475 3476 mutex_lock(&file->mutex); 3477 list_del(&uobj->list); 3478 mutex_unlock(&file->mutex); 3479 3480 ib_uverbs_release_uevent(file, &obj->uevent); 3481 resp.events_reported = obj->uevent.events_reported; 3482 put_uobj(uobj); 3483 3484 ret = ib_copy_to_udata(ucore, &resp, resp.response_length); 3485 if (ret) 3486 return ret; 3487 3488 return 0; 3489 } 3490 3491 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file, 3492 struct ib_device *ib_dev, 3493 struct ib_udata *ucore, 3494 struct ib_udata *uhw) 3495 { 3496 struct ib_uverbs_ex_modify_wq cmd = {}; 3497 struct ib_wq *wq; 3498 struct ib_wq_attr wq_attr = {}; 3499 size_t required_cmd_sz; 3500 int ret; 3501 3502 required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state); 3503 if (ucore->inlen < required_cmd_sz) 3504 return -EINVAL; 3505 3506 if (ucore->inlen > sizeof(cmd) && 3507 !ib_is_udata_cleared(ucore, sizeof(cmd), 3508 ucore->inlen - sizeof(cmd))) 3509 return -EOPNOTSUPP; 3510 3511 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3512 if (ret) 3513 return ret; 3514 3515 if (!cmd.attr_mask) 3516 return -EINVAL; 3517 3518 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS)) 3519 return -EINVAL; 3520 3521 wq = idr_read_wq(cmd.wq_handle, file->ucontext); 3522 if (!wq) 3523 return -EINVAL; 3524 3525 wq_attr.curr_wq_state = cmd.curr_wq_state; 3526 wq_attr.wq_state = cmd.wq_state; 3527 if (cmd.attr_mask & IB_WQ_FLAGS) { 3528 wq_attr.flags = cmd.flags; 3529 wq_attr.flags_mask = cmd.flags_mask; 3530 } 3531 ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw); 3532 put_wq_read(wq); 3533 return ret; 3534 } 3535 3536 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, 3537 struct ib_device *ib_dev, 3538 struct ib_udata *ucore, 3539 struct ib_udata *uhw) 3540 { 3541 struct ib_uverbs_ex_create_rwq_ind_table cmd = {}; 3542 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; 3543 struct ib_uobject *uobj; 3544 int err = 0; 3545 struct ib_rwq_ind_table_init_attr init_attr = {}; 3546 struct ib_rwq_ind_table *rwq_ind_tbl; 3547 struct ib_wq **wqs = NULL; 3548 u32 *wqs_handles = NULL; 3549 struct ib_wq *wq = NULL; 3550 int i, j, num_read_wqs; 3551 u32 num_wq_handles; 3552 u32 expected_in_size; 3553 size_t required_cmd_sz_header; 3554 size_t required_resp_len; 3555 3556 required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size); 3557 required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num); 3558 3559 if (ucore->inlen < required_cmd_sz_header) 3560 return -EINVAL; 3561 3562 if (ucore->outlen < required_resp_len) 3563 return -ENOSPC; 3564 3565 err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header); 3566 if (err) 3567 return err; 3568 3569 ucore->inbuf += required_cmd_sz_header; 3570 ucore->inlen -= required_cmd_sz_header; 3571 3572 if (cmd.comp_mask) 3573 return -EOPNOTSUPP; 3574 3575 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE) 3576 return -EINVAL; 3577 3578 num_wq_handles = 1 << cmd.log_ind_tbl_size; 3579 expected_in_size = num_wq_handles * sizeof(__u32); 3580 if (num_wq_handles == 1) 3581 /* input size for wq handles is u64 aligned */ 3582 expected_in_size += sizeof(__u32); 3583 3584 if (ucore->inlen < expected_in_size) 3585 return -EINVAL; 3586 3587 if (ucore->inlen > expected_in_size && 3588 !ib_is_udata_cleared(ucore, expected_in_size, 3589 ucore->inlen - expected_in_size)) 3590 return -EOPNOTSUPP; 3591 3592 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles), 3593 GFP_KERNEL); 3594 if (!wqs_handles) 3595 return -ENOMEM; 3596 3597 err = ib_copy_from_udata(wqs_handles, ucore, 3598 num_wq_handles * sizeof(__u32)); 3599 if (err) 3600 goto err_free; 3601 3602 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL); 3603 if (!wqs) { 3604 err = -ENOMEM; 3605 goto err_free; 3606 } 3607 3608 for (num_read_wqs = 0; num_read_wqs < num_wq_handles; 3609 num_read_wqs++) { 3610 wq = idr_read_wq(wqs_handles[num_read_wqs], file->ucontext); 3611 if (!wq) { 3612 err = -EINVAL; 3613 goto put_wqs; 3614 } 3615 3616 wqs[num_read_wqs] = wq; 3617 } 3618 3619 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 3620 if (!uobj) { 3621 err = -ENOMEM; 3622 goto put_wqs; 3623 } 3624 3625 init_uobj(uobj, 0, file->ucontext, &rwq_ind_table_lock_class); 3626 down_write(&uobj->mutex); 3627 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; 3628 init_attr.ind_tbl = wqs; 3629 rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw); 3630 3631 if (IS_ERR(rwq_ind_tbl)) { 3632 err = PTR_ERR(rwq_ind_tbl); 3633 goto err_uobj; 3634 } 3635 3636 rwq_ind_tbl->ind_tbl = wqs; 3637 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; 3638 rwq_ind_tbl->uobject = uobj; 3639 uobj->object = rwq_ind_tbl; 3640 rwq_ind_tbl->device = ib_dev; 3641 atomic_set(&rwq_ind_tbl->usecnt, 0); 3642 3643 for (i = 0; i < num_wq_handles; i++) 3644 atomic_inc(&wqs[i]->usecnt); 3645 3646 err = idr_add_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3647 if (err) 3648 goto destroy_ind_tbl; 3649 3650 resp.ind_tbl_handle = uobj->id; 3651 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; 3652 resp.response_length = required_resp_len; 3653 3654 err = ib_copy_to_udata(ucore, 3655 &resp, resp.response_length); 3656 if (err) 3657 goto err_copy; 3658 3659 kfree(wqs_handles); 3660 3661 for (j = 0; j < num_read_wqs; j++) 3662 put_wq_read(wqs[j]); 3663 3664 mutex_lock(&file->mutex); 3665 list_add_tail(&uobj->list, &file->ucontext->rwq_ind_tbl_list); 3666 mutex_unlock(&file->mutex); 3667 3668 uobj->live = 1; 3669 3670 up_write(&uobj->mutex); 3671 return 0; 3672 3673 err_copy: 3674 idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3675 destroy_ind_tbl: 3676 ib_destroy_rwq_ind_table(rwq_ind_tbl); 3677 err_uobj: 3678 put_uobj_write(uobj); 3679 put_wqs: 3680 for (j = 0; j < num_read_wqs; j++) 3681 put_wq_read(wqs[j]); 3682 err_free: 3683 kfree(wqs_handles); 3684 kfree(wqs); 3685 return err; 3686 } 3687 3688 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file, 3689 struct ib_device *ib_dev, 3690 struct ib_udata *ucore, 3691 struct ib_udata *uhw) 3692 { 3693 struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {}; 3694 struct ib_rwq_ind_table *rwq_ind_tbl; 3695 struct ib_uobject *uobj; 3696 int ret; 3697 struct ib_wq **ind_tbl; 3698 size_t required_cmd_sz; 3699 3700 required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle); 3701 3702 if (ucore->inlen < required_cmd_sz) 3703 return -EINVAL; 3704 3705 if (ucore->inlen > sizeof(cmd) && 3706 !ib_is_udata_cleared(ucore, sizeof(cmd), 3707 ucore->inlen - sizeof(cmd))) 3708 return -EOPNOTSUPP; 3709 3710 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3711 if (ret) 3712 return ret; 3713 3714 if (cmd.comp_mask) 3715 return -EOPNOTSUPP; 3716 3717 uobj = idr_write_uobj(&ib_uverbs_rwq_ind_tbl_idr, cmd.ind_tbl_handle, 3718 file->ucontext); 3719 if (!uobj) 3720 return -EINVAL; 3721 rwq_ind_tbl = uobj->object; 3722 ind_tbl = rwq_ind_tbl->ind_tbl; 3723 3724 ret = ib_destroy_rwq_ind_table(rwq_ind_tbl); 3725 if (!ret) 3726 uobj->live = 0; 3727 3728 put_uobj_write(uobj); 3729 3730 if (ret) 3731 return ret; 3732 3733 idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3734 3735 mutex_lock(&file->mutex); 3736 list_del(&uobj->list); 3737 mutex_unlock(&file->mutex); 3738 3739 put_uobj(uobj); 3740 kfree(ind_tbl); 3741 return ret; 3742 } 3743 3744 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, 3745 struct ib_device *ib_dev, 3746 struct ib_udata *ucore, 3747 struct ib_udata *uhw) 3748 { 3749 struct ib_uverbs_create_flow cmd; 3750 struct ib_uverbs_create_flow_resp resp; 3751 struct ib_uobject *uobj; 3752 struct ib_flow *flow_id; 3753 struct ib_uverbs_flow_attr *kern_flow_attr; 3754 struct ib_flow_attr *flow_attr; 3755 struct ib_qp *qp; 3756 int err = 0; 3757 void *kern_spec; 3758 void *ib_spec; 3759 int i; 3760 3761 if (ucore->inlen < sizeof(cmd)) 3762 return -EINVAL; 3763 3764 if (ucore->outlen < sizeof(resp)) 3765 return -ENOSPC; 3766 3767 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3768 if (err) 3769 return err; 3770 3771 ucore->inbuf += sizeof(cmd); 3772 ucore->inlen -= sizeof(cmd); 3773 3774 if (cmd.comp_mask) 3775 return -EINVAL; 3776 3777 if (!capable(CAP_NET_RAW)) 3778 return -EPERM; 3779 3780 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED) 3781 return -EINVAL; 3782 3783 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && 3784 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) || 3785 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT))) 3786 return -EINVAL; 3787 3788 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 3789 return -EINVAL; 3790 3791 if (cmd.flow_attr.size > ucore->inlen || 3792 cmd.flow_attr.size > 3793 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 3794 return -EINVAL; 3795 3796 if (cmd.flow_attr.reserved[0] || 3797 cmd.flow_attr.reserved[1]) 3798 return -EINVAL; 3799 3800 if (cmd.flow_attr.num_of_specs) { 3801 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 3802 GFP_KERNEL); 3803 if (!kern_flow_attr) 3804 return -ENOMEM; 3805 3806 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr)); 3807 err = ib_copy_from_udata(kern_flow_attr + 1, ucore, 3808 cmd.flow_attr.size); 3809 if (err) 3810 goto err_free_attr; 3811 } else { 3812 kern_flow_attr = &cmd.flow_attr; 3813 } 3814 3815 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 3816 if (!uobj) { 3817 err = -ENOMEM; 3818 goto err_free_attr; 3819 } 3820 init_uobj(uobj, 0, file->ucontext, &rule_lock_class); 3821 down_write(&uobj->mutex); 3822 3823 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 3824 if (!qp) { 3825 err = -EINVAL; 3826 goto err_uobj; 3827 } 3828 3829 flow_attr = kzalloc(sizeof(*flow_attr) + cmd.flow_attr.num_of_specs * 3830 sizeof(union ib_flow_spec), GFP_KERNEL); 3831 if (!flow_attr) { 3832 err = -ENOMEM; 3833 goto err_put; 3834 } 3835 3836 flow_attr->type = kern_flow_attr->type; 3837 flow_attr->priority = kern_flow_attr->priority; 3838 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 3839 flow_attr->port = kern_flow_attr->port; 3840 flow_attr->flags = kern_flow_attr->flags; 3841 flow_attr->size = sizeof(*flow_attr); 3842 3843 kern_spec = kern_flow_attr + 1; 3844 ib_spec = flow_attr + 1; 3845 for (i = 0; i < flow_attr->num_of_specs && 3846 cmd.flow_attr.size > offsetof(struct ib_uverbs_flow_spec, reserved) && 3847 cmd.flow_attr.size >= 3848 ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) { 3849 err = kern_spec_to_ib_spec(kern_spec, ib_spec); 3850 if (err) 3851 goto err_free; 3852 flow_attr->size += 3853 ((union ib_flow_spec *) ib_spec)->size; 3854 cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size; 3855 kern_spec += ((struct ib_uverbs_flow_spec *) kern_spec)->size; 3856 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 3857 } 3858 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 3859 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 3860 i, cmd.flow_attr.size); 3861 err = -EINVAL; 3862 goto err_free; 3863 } 3864 flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER); 3865 if (IS_ERR(flow_id)) { 3866 err = PTR_ERR(flow_id); 3867 goto err_free; 3868 } 3869 flow_id->uobject = uobj; 3870 uobj->object = flow_id; 3871 3872 err = idr_add_uobj(&ib_uverbs_rule_idr, uobj); 3873 if (err) 3874 goto destroy_flow; 3875 3876 memset(&resp, 0, sizeof(resp)); 3877 resp.flow_handle = uobj->id; 3878 3879 err = ib_copy_to_udata(ucore, 3880 &resp, sizeof(resp)); 3881 if (err) 3882 goto err_copy; 3883 3884 put_qp_read(qp); 3885 mutex_lock(&file->mutex); 3886 list_add_tail(&uobj->list, &file->ucontext->rule_list); 3887 mutex_unlock(&file->mutex); 3888 3889 uobj->live = 1; 3890 3891 up_write(&uobj->mutex); 3892 kfree(flow_attr); 3893 if (cmd.flow_attr.num_of_specs) 3894 kfree(kern_flow_attr); 3895 return 0; 3896 err_copy: 3897 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 3898 destroy_flow: 3899 ib_destroy_flow(flow_id); 3900 err_free: 3901 kfree(flow_attr); 3902 err_put: 3903 put_qp_read(qp); 3904 err_uobj: 3905 put_uobj_write(uobj); 3906 err_free_attr: 3907 if (cmd.flow_attr.num_of_specs) 3908 kfree(kern_flow_attr); 3909 return err; 3910 } 3911 3912 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file, 3913 struct ib_device *ib_dev, 3914 struct ib_udata *ucore, 3915 struct ib_udata *uhw) 3916 { 3917 struct ib_uverbs_destroy_flow cmd; 3918 struct ib_flow *flow_id; 3919 struct ib_uobject *uobj; 3920 int ret; 3921 3922 if (ucore->inlen < sizeof(cmd)) 3923 return -EINVAL; 3924 3925 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3926 if (ret) 3927 return ret; 3928 3929 if (cmd.comp_mask) 3930 return -EINVAL; 3931 3932 uobj = idr_write_uobj(&ib_uverbs_rule_idr, cmd.flow_handle, 3933 file->ucontext); 3934 if (!uobj) 3935 return -EINVAL; 3936 flow_id = uobj->object; 3937 3938 ret = ib_destroy_flow(flow_id); 3939 if (!ret) 3940 uobj->live = 0; 3941 3942 put_uobj_write(uobj); 3943 3944 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 3945 3946 mutex_lock(&file->mutex); 3947 list_del(&uobj->list); 3948 mutex_unlock(&file->mutex); 3949 3950 put_uobj(uobj); 3951 3952 return ret; 3953 } 3954 3955 static int __uverbs_create_xsrq(struct ib_uverbs_file *file, 3956 struct ib_device *ib_dev, 3957 struct ib_uverbs_create_xsrq *cmd, 3958 struct ib_udata *udata) 3959 { 3960 struct ib_uverbs_create_srq_resp resp; 3961 struct ib_usrq_object *obj; 3962 struct ib_pd *pd; 3963 struct ib_srq *srq; 3964 struct ib_uobject *uninitialized_var(xrcd_uobj); 3965 struct ib_srq_init_attr attr; 3966 int ret; 3967 3968 obj = kmalloc(sizeof *obj, GFP_KERNEL); 3969 if (!obj) 3970 return -ENOMEM; 3971 3972 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class); 3973 down_write(&obj->uevent.uobject.mutex); 3974 3975 if (cmd->srq_type == IB_SRQT_XRC) { 3976 attr.ext.xrc.xrcd = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj); 3977 if (!attr.ext.xrc.xrcd) { 3978 ret = -EINVAL; 3979 goto err; 3980 } 3981 3982 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 3983 atomic_inc(&obj->uxrcd->refcnt); 3984 3985 attr.ext.xrc.cq = idr_read_cq(cmd->cq_handle, file->ucontext, 0); 3986 if (!attr.ext.xrc.cq) { 3987 ret = -EINVAL; 3988 goto err_put_xrcd; 3989 } 3990 } 3991 3992 pd = idr_read_pd(cmd->pd_handle, file->ucontext); 3993 if (!pd) { 3994 ret = -EINVAL; 3995 goto err_put_cq; 3996 } 3997 3998 attr.event_handler = ib_uverbs_srq_event_handler; 3999 attr.srq_context = file; 4000 attr.srq_type = cmd->srq_type; 4001 attr.attr.max_wr = cmd->max_wr; 4002 attr.attr.max_sge = cmd->max_sge; 4003 attr.attr.srq_limit = cmd->srq_limit; 4004 4005 obj->uevent.events_reported = 0; 4006 INIT_LIST_HEAD(&obj->uevent.event_list); 4007 4008 srq = pd->device->create_srq(pd, &attr, udata); 4009 if (IS_ERR(srq)) { 4010 ret = PTR_ERR(srq); 4011 goto err_put; 4012 } 4013 4014 srq->device = pd->device; 4015 srq->pd = pd; 4016 srq->srq_type = cmd->srq_type; 4017 srq->uobject = &obj->uevent.uobject; 4018 srq->event_handler = attr.event_handler; 4019 srq->srq_context = attr.srq_context; 4020 4021 if (cmd->srq_type == IB_SRQT_XRC) { 4022 srq->ext.xrc.cq = attr.ext.xrc.cq; 4023 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 4024 atomic_inc(&attr.ext.xrc.cq->usecnt); 4025 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 4026 } 4027 4028 atomic_inc(&pd->usecnt); 4029 atomic_set(&srq->usecnt, 0); 4030 4031 obj->uevent.uobject.object = srq; 4032 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 4033 if (ret) 4034 goto err_destroy; 4035 4036 memset(&resp, 0, sizeof resp); 4037 resp.srq_handle = obj->uevent.uobject.id; 4038 resp.max_wr = attr.attr.max_wr; 4039 resp.max_sge = attr.attr.max_sge; 4040 if (cmd->srq_type == IB_SRQT_XRC) 4041 resp.srqn = srq->ext.xrc.srq_num; 4042 4043 if (copy_to_user((void __user *) (unsigned long) cmd->response, 4044 &resp, sizeof resp)) { 4045 ret = -EFAULT; 4046 goto err_copy; 4047 } 4048 4049 if (cmd->srq_type == IB_SRQT_XRC) { 4050 put_uobj_read(xrcd_uobj); 4051 put_cq_read(attr.ext.xrc.cq); 4052 } 4053 put_pd_read(pd); 4054 4055 mutex_lock(&file->mutex); 4056 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->srq_list); 4057 mutex_unlock(&file->mutex); 4058 4059 obj->uevent.uobject.live = 1; 4060 4061 up_write(&obj->uevent.uobject.mutex); 4062 4063 return 0; 4064 4065 err_copy: 4066 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 4067 4068 err_destroy: 4069 ib_destroy_srq(srq); 4070 4071 err_put: 4072 put_pd_read(pd); 4073 4074 err_put_cq: 4075 if (cmd->srq_type == IB_SRQT_XRC) 4076 put_cq_read(attr.ext.xrc.cq); 4077 4078 err_put_xrcd: 4079 if (cmd->srq_type == IB_SRQT_XRC) { 4080 atomic_dec(&obj->uxrcd->refcnt); 4081 put_uobj_read(xrcd_uobj); 4082 } 4083 4084 err: 4085 put_uobj_write(&obj->uevent.uobject); 4086 return ret; 4087 } 4088 4089 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 4090 struct ib_device *ib_dev, 4091 const char __user *buf, int in_len, 4092 int out_len) 4093 { 4094 struct ib_uverbs_create_srq cmd; 4095 struct ib_uverbs_create_xsrq xcmd; 4096 struct ib_uverbs_create_srq_resp resp; 4097 struct ib_udata udata; 4098 int ret; 4099 4100 if (out_len < sizeof resp) 4101 return -ENOSPC; 4102 4103 if (copy_from_user(&cmd, buf, sizeof cmd)) 4104 return -EFAULT; 4105 4106 xcmd.response = cmd.response; 4107 xcmd.user_handle = cmd.user_handle; 4108 xcmd.srq_type = IB_SRQT_BASIC; 4109 xcmd.pd_handle = cmd.pd_handle; 4110 xcmd.max_wr = cmd.max_wr; 4111 xcmd.max_sge = cmd.max_sge; 4112 xcmd.srq_limit = cmd.srq_limit; 4113 4114 INIT_UDATA(&udata, buf + sizeof cmd, 4115 (unsigned long) cmd.response + sizeof resp, 4116 in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr), 4117 out_len - sizeof resp); 4118 4119 ret = __uverbs_create_xsrq(file, ib_dev, &xcmd, &udata); 4120 if (ret) 4121 return ret; 4122 4123 return in_len; 4124 } 4125 4126 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file, 4127 struct ib_device *ib_dev, 4128 const char __user *buf, int in_len, int out_len) 4129 { 4130 struct ib_uverbs_create_xsrq cmd; 4131 struct ib_uverbs_create_srq_resp resp; 4132 struct ib_udata udata; 4133 int ret; 4134 4135 if (out_len < sizeof resp) 4136 return -ENOSPC; 4137 4138 if (copy_from_user(&cmd, buf, sizeof cmd)) 4139 return -EFAULT; 4140 4141 INIT_UDATA(&udata, buf + sizeof cmd, 4142 (unsigned long) cmd.response + sizeof resp, 4143 in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr), 4144 out_len - sizeof resp); 4145 4146 ret = __uverbs_create_xsrq(file, ib_dev, &cmd, &udata); 4147 if (ret) 4148 return ret; 4149 4150 return in_len; 4151 } 4152 4153 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 4154 struct ib_device *ib_dev, 4155 const char __user *buf, int in_len, 4156 int out_len) 4157 { 4158 struct ib_uverbs_modify_srq cmd; 4159 struct ib_udata udata; 4160 struct ib_srq *srq; 4161 struct ib_srq_attr attr; 4162 int ret; 4163 4164 if (copy_from_user(&cmd, buf, sizeof cmd)) 4165 return -EFAULT; 4166 4167 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 4168 out_len); 4169 4170 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 4171 if (!srq) 4172 return -EINVAL; 4173 4174 attr.max_wr = cmd.max_wr; 4175 attr.srq_limit = cmd.srq_limit; 4176 4177 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 4178 4179 put_srq_read(srq); 4180 4181 return ret ? ret : in_len; 4182 } 4183 4184 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 4185 struct ib_device *ib_dev, 4186 const char __user *buf, 4187 int in_len, int out_len) 4188 { 4189 struct ib_uverbs_query_srq cmd; 4190 struct ib_uverbs_query_srq_resp resp; 4191 struct ib_srq_attr attr; 4192 struct ib_srq *srq; 4193 int ret; 4194 4195 if (out_len < sizeof resp) 4196 return -ENOSPC; 4197 4198 if (copy_from_user(&cmd, buf, sizeof cmd)) 4199 return -EFAULT; 4200 4201 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 4202 if (!srq) 4203 return -EINVAL; 4204 4205 ret = ib_query_srq(srq, &attr); 4206 4207 put_srq_read(srq); 4208 4209 if (ret) 4210 return ret; 4211 4212 memset(&resp, 0, sizeof resp); 4213 4214 resp.max_wr = attr.max_wr; 4215 resp.max_sge = attr.max_sge; 4216 resp.srq_limit = attr.srq_limit; 4217 4218 if (copy_to_user((void __user *) (unsigned long) cmd.response, 4219 &resp, sizeof resp)) 4220 return -EFAULT; 4221 4222 return in_len; 4223 } 4224 4225 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 4226 struct ib_device *ib_dev, 4227 const char __user *buf, int in_len, 4228 int out_len) 4229 { 4230 struct ib_uverbs_destroy_srq cmd; 4231 struct ib_uverbs_destroy_srq_resp resp; 4232 struct ib_uobject *uobj; 4233 struct ib_srq *srq; 4234 struct ib_uevent_object *obj; 4235 int ret = -EINVAL; 4236 struct ib_usrq_object *us; 4237 enum ib_srq_type srq_type; 4238 4239 if (copy_from_user(&cmd, buf, sizeof cmd)) 4240 return -EFAULT; 4241 4242 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); 4243 if (!uobj) 4244 return -EINVAL; 4245 srq = uobj->object; 4246 obj = container_of(uobj, struct ib_uevent_object, uobject); 4247 srq_type = srq->srq_type; 4248 4249 ret = ib_destroy_srq(srq); 4250 if (!ret) 4251 uobj->live = 0; 4252 4253 put_uobj_write(uobj); 4254 4255 if (ret) 4256 return ret; 4257 4258 if (srq_type == IB_SRQT_XRC) { 4259 us = container_of(obj, struct ib_usrq_object, uevent); 4260 atomic_dec(&us->uxrcd->refcnt); 4261 } 4262 4263 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 4264 4265 mutex_lock(&file->mutex); 4266 list_del(&uobj->list); 4267 mutex_unlock(&file->mutex); 4268 4269 ib_uverbs_release_uevent(file, obj); 4270 4271 memset(&resp, 0, sizeof resp); 4272 resp.events_reported = obj->events_reported; 4273 4274 put_uobj(uobj); 4275 4276 if (copy_to_user((void __user *) (unsigned long) cmd.response, 4277 &resp, sizeof resp)) 4278 ret = -EFAULT; 4279 4280 return ret ? ret : in_len; 4281 } 4282 4283 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, 4284 struct ib_device *ib_dev, 4285 struct ib_udata *ucore, 4286 struct ib_udata *uhw) 4287 { 4288 struct ib_uverbs_ex_query_device_resp resp = { {0} }; 4289 struct ib_uverbs_ex_query_device cmd; 4290 struct ib_device_attr attr = {0}; 4291 int err; 4292 4293 if (ucore->inlen < sizeof(cmd)) 4294 return -EINVAL; 4295 4296 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 4297 if (err) 4298 return err; 4299 4300 if (cmd.comp_mask) 4301 return -EINVAL; 4302 4303 if (cmd.reserved) 4304 return -EINVAL; 4305 4306 resp.response_length = offsetof(typeof(resp), odp_caps); 4307 4308 if (ucore->outlen < resp.response_length) 4309 return -ENOSPC; 4310 4311 err = ib_dev->query_device(ib_dev, &attr, uhw); 4312 if (err) 4313 return err; 4314 4315 copy_query_dev_fields(file, ib_dev, &resp.base, &attr); 4316 4317 if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps)) 4318 goto end; 4319 4320 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 4321 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 4322 resp.odp_caps.per_transport_caps.rc_odp_caps = 4323 attr.odp_caps.per_transport_caps.rc_odp_caps; 4324 resp.odp_caps.per_transport_caps.uc_odp_caps = 4325 attr.odp_caps.per_transport_caps.uc_odp_caps; 4326 resp.odp_caps.per_transport_caps.ud_odp_caps = 4327 attr.odp_caps.per_transport_caps.ud_odp_caps; 4328 #endif 4329 resp.response_length += sizeof(resp.odp_caps); 4330 4331 if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask)) 4332 goto end; 4333 4334 resp.timestamp_mask = attr.timestamp_mask; 4335 resp.response_length += sizeof(resp.timestamp_mask); 4336 4337 if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock)) 4338 goto end; 4339 4340 resp.hca_core_clock = attr.hca_core_clock; 4341 resp.response_length += sizeof(resp.hca_core_clock); 4342 4343 if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex)) 4344 goto end; 4345 4346 resp.device_cap_flags_ex = attr.device_cap_flags; 4347 resp.response_length += sizeof(resp.device_cap_flags_ex); 4348 4349 if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps)) 4350 goto end; 4351 4352 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; 4353 resp.rss_caps.max_rwq_indirection_tables = 4354 attr.rss_caps.max_rwq_indirection_tables; 4355 resp.rss_caps.max_rwq_indirection_table_size = 4356 attr.rss_caps.max_rwq_indirection_table_size; 4357 4358 resp.response_length += sizeof(resp.rss_caps); 4359 4360 if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq)) 4361 goto end; 4362 4363 resp.max_wq_type_rq = attr.max_wq_type_rq; 4364 resp.response_length += sizeof(resp.max_wq_type_rq); 4365 4366 if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps)) 4367 goto end; 4368 4369 resp.raw_packet_caps = attr.raw_packet_caps; 4370 resp.response_length += sizeof(resp.raw_packet_caps); 4371 end: 4372 err = ib_copy_to_udata(ucore, &resp, resp.response_length); 4373 return err; 4374 } 4375