1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <linux/uaccess.h> 42 43 #include "uverbs.h" 44 #include "core_priv.h" 45 46 struct uverbs_lock_class { 47 struct lock_class_key key; 48 char name[16]; 49 }; 50 51 static struct uverbs_lock_class pd_lock_class = { .name = "PD-uobj" }; 52 static struct uverbs_lock_class mr_lock_class = { .name = "MR-uobj" }; 53 static struct uverbs_lock_class mw_lock_class = { .name = "MW-uobj" }; 54 static struct uverbs_lock_class cq_lock_class = { .name = "CQ-uobj" }; 55 static struct uverbs_lock_class qp_lock_class = { .name = "QP-uobj" }; 56 static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" }; 57 static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" }; 58 static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" }; 59 static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" }; 60 static struct uverbs_lock_class wq_lock_class = { .name = "WQ-uobj" }; 61 static struct uverbs_lock_class rwq_ind_table_lock_class = { .name = "IND_TBL-uobj" }; 62 63 /* 64 * The ib_uobject locking scheme is as follows: 65 * 66 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it 67 * needs to be held during all idr write operations. When an object is 68 * looked up, a reference must be taken on the object's kref before 69 * dropping this lock. For read operations, the rcu_read_lock() 70 * and rcu_write_lock() but similarly the kref reference is grabbed 71 * before the rcu_read_unlock(). 72 * 73 * - Each object also has an rwsem. This rwsem must be held for 74 * reading while an operation that uses the object is performed. 75 * For example, while registering an MR, the associated PD's 76 * uobject.mutex must be held for reading. The rwsem must be held 77 * for writing while initializing or destroying an object. 78 * 79 * - In addition, each object has a "live" flag. If this flag is not 80 * set, then lookups of the object will fail even if it is found in 81 * the idr. This handles a reader that blocks and does not acquire 82 * the rwsem until after the object is destroyed. The destroy 83 * operation will set the live flag to 0 and then drop the rwsem; 84 * this will allow the reader to acquire the rwsem, see that the 85 * live flag is 0, and then drop the rwsem and its reference to 86 * object. The underlying storage will not be freed until the last 87 * reference to the object is dropped. 88 */ 89 90 static void init_uobj(struct ib_uobject *uobj, u64 user_handle, 91 struct ib_ucontext *context, struct uverbs_lock_class *c) 92 { 93 uobj->user_handle = user_handle; 94 uobj->context = context; 95 kref_init(&uobj->ref); 96 init_rwsem(&uobj->mutex); 97 lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name); 98 uobj->live = 0; 99 } 100 101 static void release_uobj(struct kref *kref) 102 { 103 kfree_rcu(container_of(kref, struct ib_uobject, ref), rcu); 104 } 105 106 static void put_uobj(struct ib_uobject *uobj) 107 { 108 kref_put(&uobj->ref, release_uobj); 109 } 110 111 static void put_uobj_read(struct ib_uobject *uobj) 112 { 113 up_read(&uobj->mutex); 114 put_uobj(uobj); 115 } 116 117 static void put_uobj_write(struct ib_uobject *uobj) 118 { 119 up_write(&uobj->mutex); 120 put_uobj(uobj); 121 } 122 123 static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) 124 { 125 int ret; 126 127 idr_preload(GFP_KERNEL); 128 spin_lock(&ib_uverbs_idr_lock); 129 130 ret = idr_alloc(idr, uobj, 0, 0, GFP_NOWAIT); 131 if (ret >= 0) 132 uobj->id = ret; 133 134 spin_unlock(&ib_uverbs_idr_lock); 135 idr_preload_end(); 136 137 return ret < 0 ? ret : 0; 138 } 139 140 void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) 141 { 142 spin_lock(&ib_uverbs_idr_lock); 143 idr_remove(idr, uobj->id); 144 spin_unlock(&ib_uverbs_idr_lock); 145 } 146 147 static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, 148 struct ib_ucontext *context) 149 { 150 struct ib_uobject *uobj; 151 152 rcu_read_lock(); 153 uobj = idr_find(idr, id); 154 if (uobj) { 155 if (uobj->context == context) 156 kref_get(&uobj->ref); 157 else 158 uobj = NULL; 159 } 160 rcu_read_unlock(); 161 162 return uobj; 163 } 164 165 static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, 166 struct ib_ucontext *context, int nested) 167 { 168 struct ib_uobject *uobj; 169 170 uobj = __idr_get_uobj(idr, id, context); 171 if (!uobj) 172 return NULL; 173 174 if (nested) 175 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); 176 else 177 down_read(&uobj->mutex); 178 if (!uobj->live) { 179 put_uobj_read(uobj); 180 return NULL; 181 } 182 183 return uobj; 184 } 185 186 static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, 187 struct ib_ucontext *context) 188 { 189 struct ib_uobject *uobj; 190 191 uobj = __idr_get_uobj(idr, id, context); 192 if (!uobj) 193 return NULL; 194 195 down_write(&uobj->mutex); 196 if (!uobj->live) { 197 put_uobj_write(uobj); 198 return NULL; 199 } 200 201 return uobj; 202 } 203 204 static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, 205 int nested) 206 { 207 struct ib_uobject *uobj; 208 209 uobj = idr_read_uobj(idr, id, context, nested); 210 return uobj ? uobj->object : NULL; 211 } 212 213 static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) 214 { 215 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); 216 } 217 218 static void put_pd_read(struct ib_pd *pd) 219 { 220 put_uobj_read(pd->uobject); 221 } 222 223 static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) 224 { 225 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); 226 } 227 228 static void put_cq_read(struct ib_cq *cq) 229 { 230 put_uobj_read(cq->uobject); 231 } 232 233 static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) 234 { 235 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); 236 } 237 238 static void put_ah_read(struct ib_ah *ah) 239 { 240 put_uobj_read(ah->uobject); 241 } 242 243 static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) 244 { 245 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); 246 } 247 248 static struct ib_wq *idr_read_wq(int wq_handle, struct ib_ucontext *context) 249 { 250 return idr_read_obj(&ib_uverbs_wq_idr, wq_handle, context, 0); 251 } 252 253 static void put_wq_read(struct ib_wq *wq) 254 { 255 put_uobj_read(wq->uobject); 256 } 257 258 static struct ib_rwq_ind_table *idr_read_rwq_indirection_table(int ind_table_handle, 259 struct ib_ucontext *context) 260 { 261 return idr_read_obj(&ib_uverbs_rwq_ind_tbl_idr, ind_table_handle, context, 0); 262 } 263 264 static void put_rwq_indirection_table_read(struct ib_rwq_ind_table *ind_table) 265 { 266 put_uobj_read(ind_table->uobject); 267 } 268 269 static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context) 270 { 271 struct ib_uobject *uobj; 272 273 uobj = idr_write_uobj(&ib_uverbs_qp_idr, qp_handle, context); 274 return uobj ? uobj->object : NULL; 275 } 276 277 static void put_qp_read(struct ib_qp *qp) 278 { 279 put_uobj_read(qp->uobject); 280 } 281 282 static void put_qp_write(struct ib_qp *qp) 283 { 284 put_uobj_write(qp->uobject); 285 } 286 287 static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) 288 { 289 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); 290 } 291 292 static void put_srq_read(struct ib_srq *srq) 293 { 294 put_uobj_read(srq->uobject); 295 } 296 297 static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context, 298 struct ib_uobject **uobj) 299 { 300 *uobj = idr_read_uobj(&ib_uverbs_xrcd_idr, xrcd_handle, context, 0); 301 return *uobj ? (*uobj)->object : NULL; 302 } 303 304 static void put_xrcd_read(struct ib_uobject *uobj) 305 { 306 put_uobj_read(uobj); 307 } 308 309 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 310 struct ib_device *ib_dev, 311 const char __user *buf, 312 int in_len, int out_len) 313 { 314 struct ib_uverbs_get_context cmd; 315 struct ib_uverbs_get_context_resp resp; 316 struct ib_udata udata; 317 struct ib_ucontext *ucontext; 318 struct file *filp; 319 struct ib_rdmacg_object cg_obj; 320 int ret; 321 322 if (out_len < sizeof resp) 323 return -ENOSPC; 324 325 if (copy_from_user(&cmd, buf, sizeof cmd)) 326 return -EFAULT; 327 328 mutex_lock(&file->mutex); 329 330 if (file->ucontext) { 331 ret = -EINVAL; 332 goto err; 333 } 334 335 INIT_UDATA(&udata, buf + sizeof cmd, 336 (unsigned long) cmd.response + sizeof resp, 337 in_len - sizeof cmd, out_len - sizeof resp); 338 339 ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 340 if (ret) 341 goto err; 342 343 ucontext = ib_dev->alloc_ucontext(ib_dev, &udata); 344 if (IS_ERR(ucontext)) { 345 ret = PTR_ERR(ucontext); 346 goto err_alloc; 347 } 348 349 ucontext->device = ib_dev; 350 ucontext->cg_obj = cg_obj; 351 INIT_LIST_HEAD(&ucontext->pd_list); 352 INIT_LIST_HEAD(&ucontext->mr_list); 353 INIT_LIST_HEAD(&ucontext->mw_list); 354 INIT_LIST_HEAD(&ucontext->cq_list); 355 INIT_LIST_HEAD(&ucontext->qp_list); 356 INIT_LIST_HEAD(&ucontext->srq_list); 357 INIT_LIST_HEAD(&ucontext->ah_list); 358 INIT_LIST_HEAD(&ucontext->wq_list); 359 INIT_LIST_HEAD(&ucontext->rwq_ind_tbl_list); 360 INIT_LIST_HEAD(&ucontext->xrcd_list); 361 INIT_LIST_HEAD(&ucontext->rule_list); 362 rcu_read_lock(); 363 ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); 364 rcu_read_unlock(); 365 ucontext->closing = 0; 366 367 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 368 ucontext->umem_tree = RB_ROOT; 369 init_rwsem(&ucontext->umem_rwsem); 370 ucontext->odp_mrs_count = 0; 371 INIT_LIST_HEAD(&ucontext->no_private_counters); 372 373 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) 374 ucontext->invalidate_range = NULL; 375 376 #endif 377 378 resp.num_comp_vectors = file->device->num_comp_vectors; 379 380 ret = get_unused_fd_flags(O_CLOEXEC); 381 if (ret < 0) 382 goto err_free; 383 resp.async_fd = ret; 384 385 filp = ib_uverbs_alloc_event_file(file, ib_dev, 1); 386 if (IS_ERR(filp)) { 387 ret = PTR_ERR(filp); 388 goto err_fd; 389 } 390 391 if (copy_to_user((void __user *) (unsigned long) cmd.response, 392 &resp, sizeof resp)) { 393 ret = -EFAULT; 394 goto err_file; 395 } 396 397 file->ucontext = ucontext; 398 399 fd_install(resp.async_fd, filp); 400 401 mutex_unlock(&file->mutex); 402 403 return in_len; 404 405 err_file: 406 ib_uverbs_free_async_event_file(file); 407 fput(filp); 408 409 err_fd: 410 put_unused_fd(resp.async_fd); 411 412 err_free: 413 put_pid(ucontext->tgid); 414 ib_dev->dealloc_ucontext(ucontext); 415 416 err_alloc: 417 ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 418 419 err: 420 mutex_unlock(&file->mutex); 421 return ret; 422 } 423 424 static void copy_query_dev_fields(struct ib_uverbs_file *file, 425 struct ib_device *ib_dev, 426 struct ib_uverbs_query_device_resp *resp, 427 struct ib_device_attr *attr) 428 { 429 resp->fw_ver = attr->fw_ver; 430 resp->node_guid = ib_dev->node_guid; 431 resp->sys_image_guid = attr->sys_image_guid; 432 resp->max_mr_size = attr->max_mr_size; 433 resp->page_size_cap = attr->page_size_cap; 434 resp->vendor_id = attr->vendor_id; 435 resp->vendor_part_id = attr->vendor_part_id; 436 resp->hw_ver = attr->hw_ver; 437 resp->max_qp = attr->max_qp; 438 resp->max_qp_wr = attr->max_qp_wr; 439 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags); 440 resp->max_sge = attr->max_sge; 441 resp->max_sge_rd = attr->max_sge_rd; 442 resp->max_cq = attr->max_cq; 443 resp->max_cqe = attr->max_cqe; 444 resp->max_mr = attr->max_mr; 445 resp->max_pd = attr->max_pd; 446 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 447 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 448 resp->max_res_rd_atom = attr->max_res_rd_atom; 449 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 450 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 451 resp->atomic_cap = attr->atomic_cap; 452 resp->max_ee = attr->max_ee; 453 resp->max_rdd = attr->max_rdd; 454 resp->max_mw = attr->max_mw; 455 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 456 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 457 resp->max_mcast_grp = attr->max_mcast_grp; 458 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 459 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 460 resp->max_ah = attr->max_ah; 461 resp->max_fmr = attr->max_fmr; 462 resp->max_map_per_fmr = attr->max_map_per_fmr; 463 resp->max_srq = attr->max_srq; 464 resp->max_srq_wr = attr->max_srq_wr; 465 resp->max_srq_sge = attr->max_srq_sge; 466 resp->max_pkeys = attr->max_pkeys; 467 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 468 resp->phys_port_cnt = ib_dev->phys_port_cnt; 469 } 470 471 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 472 struct ib_device *ib_dev, 473 const char __user *buf, 474 int in_len, int out_len) 475 { 476 struct ib_uverbs_query_device cmd; 477 struct ib_uverbs_query_device_resp resp; 478 479 if (out_len < sizeof resp) 480 return -ENOSPC; 481 482 if (copy_from_user(&cmd, buf, sizeof cmd)) 483 return -EFAULT; 484 485 memset(&resp, 0, sizeof resp); 486 copy_query_dev_fields(file, ib_dev, &resp, &ib_dev->attrs); 487 488 if (copy_to_user((void __user *) (unsigned long) cmd.response, 489 &resp, sizeof resp)) 490 return -EFAULT; 491 492 return in_len; 493 } 494 495 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 496 struct ib_device *ib_dev, 497 const char __user *buf, 498 int in_len, int out_len) 499 { 500 struct ib_uverbs_query_port cmd; 501 struct ib_uverbs_query_port_resp resp; 502 struct ib_port_attr attr; 503 int ret; 504 505 if (out_len < sizeof resp) 506 return -ENOSPC; 507 508 if (copy_from_user(&cmd, buf, sizeof cmd)) 509 return -EFAULT; 510 511 ret = ib_query_port(ib_dev, cmd.port_num, &attr); 512 if (ret) 513 return ret; 514 515 memset(&resp, 0, sizeof resp); 516 517 resp.state = attr.state; 518 resp.max_mtu = attr.max_mtu; 519 resp.active_mtu = attr.active_mtu; 520 resp.gid_tbl_len = attr.gid_tbl_len; 521 resp.port_cap_flags = attr.port_cap_flags; 522 resp.max_msg_sz = attr.max_msg_sz; 523 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 524 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 525 resp.pkey_tbl_len = attr.pkey_tbl_len; 526 resp.lid = attr.lid; 527 resp.sm_lid = attr.sm_lid; 528 resp.lmc = attr.lmc; 529 resp.max_vl_num = attr.max_vl_num; 530 resp.sm_sl = attr.sm_sl; 531 resp.subnet_timeout = attr.subnet_timeout; 532 resp.init_type_reply = attr.init_type_reply; 533 resp.active_width = attr.active_width; 534 resp.active_speed = attr.active_speed; 535 resp.phys_state = attr.phys_state; 536 resp.link_layer = rdma_port_get_link_layer(ib_dev, 537 cmd.port_num); 538 539 if (copy_to_user((void __user *) (unsigned long) cmd.response, 540 &resp, sizeof resp)) 541 return -EFAULT; 542 543 return in_len; 544 } 545 546 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 547 struct ib_device *ib_dev, 548 const char __user *buf, 549 int in_len, int out_len) 550 { 551 struct ib_uverbs_alloc_pd cmd; 552 struct ib_uverbs_alloc_pd_resp resp; 553 struct ib_udata udata; 554 struct ib_uobject *uobj; 555 struct ib_pd *pd; 556 int ret; 557 558 if (out_len < sizeof resp) 559 return -ENOSPC; 560 561 if (copy_from_user(&cmd, buf, sizeof cmd)) 562 return -EFAULT; 563 564 INIT_UDATA(&udata, buf + sizeof cmd, 565 (unsigned long) cmd.response + sizeof resp, 566 in_len - sizeof cmd, out_len - sizeof resp); 567 568 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 569 if (!uobj) 570 return -ENOMEM; 571 572 init_uobj(uobj, 0, file->ucontext, &pd_lock_class); 573 ret = ib_rdmacg_try_charge(&uobj->cg_obj, ib_dev, 574 RDMACG_RESOURCE_HCA_OBJECT); 575 if (ret) { 576 kfree(uobj); 577 return ret; 578 } 579 580 down_write(&uobj->mutex); 581 582 pd = ib_dev->alloc_pd(ib_dev, file->ucontext, &udata); 583 if (IS_ERR(pd)) { 584 ret = PTR_ERR(pd); 585 goto err; 586 } 587 588 pd->device = ib_dev; 589 pd->uobject = uobj; 590 pd->__internal_mr = NULL; 591 atomic_set(&pd->usecnt, 0); 592 593 uobj->object = pd; 594 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); 595 if (ret) 596 goto err_idr; 597 598 memset(&resp, 0, sizeof resp); 599 resp.pd_handle = uobj->id; 600 601 if (copy_to_user((void __user *) (unsigned long) cmd.response, 602 &resp, sizeof resp)) { 603 ret = -EFAULT; 604 goto err_copy; 605 } 606 607 mutex_lock(&file->mutex); 608 list_add_tail(&uobj->list, &file->ucontext->pd_list); 609 mutex_unlock(&file->mutex); 610 611 uobj->live = 1; 612 613 up_write(&uobj->mutex); 614 615 return in_len; 616 617 err_copy: 618 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 619 620 err_idr: 621 ib_dealloc_pd(pd); 622 623 err: 624 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 625 put_uobj_write(uobj); 626 return ret; 627 } 628 629 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 630 struct ib_device *ib_dev, 631 const char __user *buf, 632 int in_len, int out_len) 633 { 634 struct ib_uverbs_dealloc_pd cmd; 635 struct ib_uobject *uobj; 636 struct ib_pd *pd; 637 int ret; 638 639 if (copy_from_user(&cmd, buf, sizeof cmd)) 640 return -EFAULT; 641 642 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); 643 if (!uobj) 644 return -EINVAL; 645 pd = uobj->object; 646 647 if (atomic_read(&pd->usecnt)) { 648 ret = -EBUSY; 649 goto err_put; 650 } 651 652 ret = pd->device->dealloc_pd(uobj->object); 653 WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd"); 654 if (ret) 655 goto err_put; 656 657 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 658 659 uobj->live = 0; 660 put_uobj_write(uobj); 661 662 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 663 664 mutex_lock(&file->mutex); 665 list_del(&uobj->list); 666 mutex_unlock(&file->mutex); 667 668 put_uobj(uobj); 669 670 return in_len; 671 672 err_put: 673 put_uobj_write(uobj); 674 return ret; 675 } 676 677 struct xrcd_table_entry { 678 struct rb_node node; 679 struct ib_xrcd *xrcd; 680 struct inode *inode; 681 }; 682 683 static int xrcd_table_insert(struct ib_uverbs_device *dev, 684 struct inode *inode, 685 struct ib_xrcd *xrcd) 686 { 687 struct xrcd_table_entry *entry, *scan; 688 struct rb_node **p = &dev->xrcd_tree.rb_node; 689 struct rb_node *parent = NULL; 690 691 entry = kmalloc(sizeof *entry, GFP_KERNEL); 692 if (!entry) 693 return -ENOMEM; 694 695 entry->xrcd = xrcd; 696 entry->inode = inode; 697 698 while (*p) { 699 parent = *p; 700 scan = rb_entry(parent, struct xrcd_table_entry, node); 701 702 if (inode < scan->inode) { 703 p = &(*p)->rb_left; 704 } else if (inode > scan->inode) { 705 p = &(*p)->rb_right; 706 } else { 707 kfree(entry); 708 return -EEXIST; 709 } 710 } 711 712 rb_link_node(&entry->node, parent, p); 713 rb_insert_color(&entry->node, &dev->xrcd_tree); 714 igrab(inode); 715 return 0; 716 } 717 718 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 719 struct inode *inode) 720 { 721 struct xrcd_table_entry *entry; 722 struct rb_node *p = dev->xrcd_tree.rb_node; 723 724 while (p) { 725 entry = rb_entry(p, struct xrcd_table_entry, node); 726 727 if (inode < entry->inode) 728 p = p->rb_left; 729 else if (inode > entry->inode) 730 p = p->rb_right; 731 else 732 return entry; 733 } 734 735 return NULL; 736 } 737 738 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 739 { 740 struct xrcd_table_entry *entry; 741 742 entry = xrcd_table_search(dev, inode); 743 if (!entry) 744 return NULL; 745 746 return entry->xrcd; 747 } 748 749 static void xrcd_table_delete(struct ib_uverbs_device *dev, 750 struct inode *inode) 751 { 752 struct xrcd_table_entry *entry; 753 754 entry = xrcd_table_search(dev, inode); 755 if (entry) { 756 iput(inode); 757 rb_erase(&entry->node, &dev->xrcd_tree); 758 kfree(entry); 759 } 760 } 761 762 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, 763 struct ib_device *ib_dev, 764 const char __user *buf, int in_len, 765 int out_len) 766 { 767 struct ib_uverbs_open_xrcd cmd; 768 struct ib_uverbs_open_xrcd_resp resp; 769 struct ib_udata udata; 770 struct ib_uxrcd_object *obj; 771 struct ib_xrcd *xrcd = NULL; 772 struct fd f = {NULL, 0}; 773 struct inode *inode = NULL; 774 int ret = 0; 775 int new_xrcd = 0; 776 777 if (out_len < sizeof resp) 778 return -ENOSPC; 779 780 if (copy_from_user(&cmd, buf, sizeof cmd)) 781 return -EFAULT; 782 783 INIT_UDATA(&udata, buf + sizeof cmd, 784 (unsigned long) cmd.response + sizeof resp, 785 in_len - sizeof cmd, out_len - sizeof resp); 786 787 mutex_lock(&file->device->xrcd_tree_mutex); 788 789 if (cmd.fd != -1) { 790 /* search for file descriptor */ 791 f = fdget(cmd.fd); 792 if (!f.file) { 793 ret = -EBADF; 794 goto err_tree_mutex_unlock; 795 } 796 797 inode = file_inode(f.file); 798 xrcd = find_xrcd(file->device, inode); 799 if (!xrcd && !(cmd.oflags & O_CREAT)) { 800 /* no file descriptor. Need CREATE flag */ 801 ret = -EAGAIN; 802 goto err_tree_mutex_unlock; 803 } 804 805 if (xrcd && cmd.oflags & O_EXCL) { 806 ret = -EINVAL; 807 goto err_tree_mutex_unlock; 808 } 809 } 810 811 obj = kmalloc(sizeof *obj, GFP_KERNEL); 812 if (!obj) { 813 ret = -ENOMEM; 814 goto err_tree_mutex_unlock; 815 } 816 817 init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class); 818 819 down_write(&obj->uobject.mutex); 820 821 if (!xrcd) { 822 xrcd = ib_dev->alloc_xrcd(ib_dev, file->ucontext, &udata); 823 if (IS_ERR(xrcd)) { 824 ret = PTR_ERR(xrcd); 825 goto err; 826 } 827 828 xrcd->inode = inode; 829 xrcd->device = ib_dev; 830 atomic_set(&xrcd->usecnt, 0); 831 mutex_init(&xrcd->tgt_qp_mutex); 832 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 833 new_xrcd = 1; 834 } 835 836 atomic_set(&obj->refcnt, 0); 837 obj->uobject.object = xrcd; 838 ret = idr_add_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 839 if (ret) 840 goto err_idr; 841 842 memset(&resp, 0, sizeof resp); 843 resp.xrcd_handle = obj->uobject.id; 844 845 if (inode) { 846 if (new_xrcd) { 847 /* create new inode/xrcd table entry */ 848 ret = xrcd_table_insert(file->device, inode, xrcd); 849 if (ret) 850 goto err_insert_xrcd; 851 } 852 atomic_inc(&xrcd->usecnt); 853 } 854 855 if (copy_to_user((void __user *) (unsigned long) cmd.response, 856 &resp, sizeof resp)) { 857 ret = -EFAULT; 858 goto err_copy; 859 } 860 861 if (f.file) 862 fdput(f); 863 864 mutex_lock(&file->mutex); 865 list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list); 866 mutex_unlock(&file->mutex); 867 868 obj->uobject.live = 1; 869 up_write(&obj->uobject.mutex); 870 871 mutex_unlock(&file->device->xrcd_tree_mutex); 872 return in_len; 873 874 err_copy: 875 if (inode) { 876 if (new_xrcd) 877 xrcd_table_delete(file->device, inode); 878 atomic_dec(&xrcd->usecnt); 879 } 880 881 err_insert_xrcd: 882 idr_remove_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 883 884 err_idr: 885 ib_dealloc_xrcd(xrcd); 886 887 err: 888 put_uobj_write(&obj->uobject); 889 890 err_tree_mutex_unlock: 891 if (f.file) 892 fdput(f); 893 894 mutex_unlock(&file->device->xrcd_tree_mutex); 895 896 return ret; 897 } 898 899 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file, 900 struct ib_device *ib_dev, 901 const char __user *buf, int in_len, 902 int out_len) 903 { 904 struct ib_uverbs_close_xrcd cmd; 905 struct ib_uobject *uobj; 906 struct ib_xrcd *xrcd = NULL; 907 struct inode *inode = NULL; 908 struct ib_uxrcd_object *obj; 909 int live; 910 int ret = 0; 911 912 if (copy_from_user(&cmd, buf, sizeof cmd)) 913 return -EFAULT; 914 915 mutex_lock(&file->device->xrcd_tree_mutex); 916 uobj = idr_write_uobj(&ib_uverbs_xrcd_idr, cmd.xrcd_handle, file->ucontext); 917 if (!uobj) { 918 ret = -EINVAL; 919 goto out; 920 } 921 922 xrcd = uobj->object; 923 inode = xrcd->inode; 924 obj = container_of(uobj, struct ib_uxrcd_object, uobject); 925 if (atomic_read(&obj->refcnt)) { 926 put_uobj_write(uobj); 927 ret = -EBUSY; 928 goto out; 929 } 930 931 if (!inode || atomic_dec_and_test(&xrcd->usecnt)) { 932 ret = ib_dealloc_xrcd(uobj->object); 933 if (!ret) 934 uobj->live = 0; 935 } 936 937 live = uobj->live; 938 if (inode && ret) 939 atomic_inc(&xrcd->usecnt); 940 941 put_uobj_write(uobj); 942 943 if (ret) 944 goto out; 945 946 if (inode && !live) 947 xrcd_table_delete(file->device, inode); 948 949 idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj); 950 mutex_lock(&file->mutex); 951 list_del(&uobj->list); 952 mutex_unlock(&file->mutex); 953 954 put_uobj(uobj); 955 ret = in_len; 956 957 out: 958 mutex_unlock(&file->device->xrcd_tree_mutex); 959 return ret; 960 } 961 962 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, 963 struct ib_xrcd *xrcd) 964 { 965 struct inode *inode; 966 967 inode = xrcd->inode; 968 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 969 return; 970 971 ib_dealloc_xrcd(xrcd); 972 973 if (inode) 974 xrcd_table_delete(dev, inode); 975 } 976 977 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 978 struct ib_device *ib_dev, 979 const char __user *buf, int in_len, 980 int out_len) 981 { 982 struct ib_uverbs_reg_mr cmd; 983 struct ib_uverbs_reg_mr_resp resp; 984 struct ib_udata udata; 985 struct ib_uobject *uobj; 986 struct ib_pd *pd; 987 struct ib_mr *mr; 988 int ret; 989 990 if (out_len < sizeof resp) 991 return -ENOSPC; 992 993 if (copy_from_user(&cmd, buf, sizeof cmd)) 994 return -EFAULT; 995 996 INIT_UDATA(&udata, buf + sizeof cmd, 997 (unsigned long) cmd.response + sizeof resp, 998 in_len - sizeof cmd, out_len - sizeof resp); 999 1000 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 1001 return -EINVAL; 1002 1003 ret = ib_check_mr_access(cmd.access_flags); 1004 if (ret) 1005 return ret; 1006 1007 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 1008 if (!uobj) 1009 return -ENOMEM; 1010 1011 init_uobj(uobj, 0, file->ucontext, &mr_lock_class); 1012 down_write(&uobj->mutex); 1013 1014 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1015 if (!pd) { 1016 ret = -EINVAL; 1017 goto err_free; 1018 } 1019 1020 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 1021 if (!(pd->device->attrs.device_cap_flags & 1022 IB_DEVICE_ON_DEMAND_PAGING)) { 1023 pr_debug("ODP support not available\n"); 1024 ret = -EINVAL; 1025 goto err_put; 1026 } 1027 } 1028 ret = ib_rdmacg_try_charge(&uobj->cg_obj, ib_dev, 1029 RDMACG_RESOURCE_HCA_OBJECT); 1030 if (ret) 1031 goto err_charge; 1032 1033 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 1034 cmd.access_flags, &udata); 1035 if (IS_ERR(mr)) { 1036 ret = PTR_ERR(mr); 1037 goto err_put; 1038 } 1039 1040 mr->device = pd->device; 1041 mr->pd = pd; 1042 mr->uobject = uobj; 1043 atomic_inc(&pd->usecnt); 1044 1045 uobj->object = mr; 1046 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); 1047 if (ret) 1048 goto err_unreg; 1049 1050 memset(&resp, 0, sizeof resp); 1051 resp.lkey = mr->lkey; 1052 resp.rkey = mr->rkey; 1053 resp.mr_handle = uobj->id; 1054 1055 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1056 &resp, sizeof resp)) { 1057 ret = -EFAULT; 1058 goto err_copy; 1059 } 1060 1061 put_pd_read(pd); 1062 1063 mutex_lock(&file->mutex); 1064 list_add_tail(&uobj->list, &file->ucontext->mr_list); 1065 mutex_unlock(&file->mutex); 1066 1067 uobj->live = 1; 1068 1069 up_write(&uobj->mutex); 1070 1071 return in_len; 1072 1073 err_copy: 1074 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1075 1076 err_unreg: 1077 ib_dereg_mr(mr); 1078 1079 err_put: 1080 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 1081 1082 err_charge: 1083 put_pd_read(pd); 1084 1085 err_free: 1086 put_uobj_write(uobj); 1087 return ret; 1088 } 1089 1090 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, 1091 struct ib_device *ib_dev, 1092 const char __user *buf, int in_len, 1093 int out_len) 1094 { 1095 struct ib_uverbs_rereg_mr cmd; 1096 struct ib_uverbs_rereg_mr_resp resp; 1097 struct ib_udata udata; 1098 struct ib_pd *pd = NULL; 1099 struct ib_mr *mr; 1100 struct ib_pd *old_pd; 1101 int ret; 1102 struct ib_uobject *uobj; 1103 1104 if (out_len < sizeof(resp)) 1105 return -ENOSPC; 1106 1107 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1108 return -EFAULT; 1109 1110 INIT_UDATA(&udata, buf + sizeof(cmd), 1111 (unsigned long) cmd.response + sizeof(resp), 1112 in_len - sizeof(cmd), out_len - sizeof(resp)); 1113 1114 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 1115 return -EINVAL; 1116 1117 if ((cmd.flags & IB_MR_REREG_TRANS) && 1118 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 1119 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 1120 return -EINVAL; 1121 1122 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, 1123 file->ucontext); 1124 1125 if (!uobj) 1126 return -EINVAL; 1127 1128 mr = uobj->object; 1129 1130 if (cmd.flags & IB_MR_REREG_ACCESS) { 1131 ret = ib_check_mr_access(cmd.access_flags); 1132 if (ret) 1133 goto put_uobjs; 1134 } 1135 1136 if (cmd.flags & IB_MR_REREG_PD) { 1137 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1138 if (!pd) { 1139 ret = -EINVAL; 1140 goto put_uobjs; 1141 } 1142 } 1143 1144 old_pd = mr->pd; 1145 ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start, 1146 cmd.length, cmd.hca_va, 1147 cmd.access_flags, pd, &udata); 1148 if (!ret) { 1149 if (cmd.flags & IB_MR_REREG_PD) { 1150 atomic_inc(&pd->usecnt); 1151 mr->pd = pd; 1152 atomic_dec(&old_pd->usecnt); 1153 } 1154 } else { 1155 goto put_uobj_pd; 1156 } 1157 1158 memset(&resp, 0, sizeof(resp)); 1159 resp.lkey = mr->lkey; 1160 resp.rkey = mr->rkey; 1161 1162 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1163 &resp, sizeof(resp))) 1164 ret = -EFAULT; 1165 else 1166 ret = in_len; 1167 1168 put_uobj_pd: 1169 if (cmd.flags & IB_MR_REREG_PD) 1170 put_pd_read(pd); 1171 1172 put_uobjs: 1173 1174 put_uobj_write(mr->uobject); 1175 1176 return ret; 1177 } 1178 1179 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 1180 struct ib_device *ib_dev, 1181 const char __user *buf, int in_len, 1182 int out_len) 1183 { 1184 struct ib_uverbs_dereg_mr cmd; 1185 struct ib_mr *mr; 1186 struct ib_uobject *uobj; 1187 int ret = -EINVAL; 1188 1189 if (copy_from_user(&cmd, buf, sizeof cmd)) 1190 return -EFAULT; 1191 1192 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); 1193 if (!uobj) 1194 return -EINVAL; 1195 1196 mr = uobj->object; 1197 1198 ret = ib_dereg_mr(mr); 1199 if (!ret) 1200 uobj->live = 0; 1201 1202 put_uobj_write(uobj); 1203 1204 if (ret) 1205 return ret; 1206 1207 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 1208 1209 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1210 1211 mutex_lock(&file->mutex); 1212 list_del(&uobj->list); 1213 mutex_unlock(&file->mutex); 1214 1215 put_uobj(uobj); 1216 1217 return in_len; 1218 } 1219 1220 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, 1221 struct ib_device *ib_dev, 1222 const char __user *buf, int in_len, 1223 int out_len) 1224 { 1225 struct ib_uverbs_alloc_mw cmd; 1226 struct ib_uverbs_alloc_mw_resp resp; 1227 struct ib_uobject *uobj; 1228 struct ib_pd *pd; 1229 struct ib_mw *mw; 1230 struct ib_udata udata; 1231 int ret; 1232 1233 if (out_len < sizeof(resp)) 1234 return -ENOSPC; 1235 1236 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1237 return -EFAULT; 1238 1239 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 1240 if (!uobj) 1241 return -ENOMEM; 1242 1243 init_uobj(uobj, 0, file->ucontext, &mw_lock_class); 1244 down_write(&uobj->mutex); 1245 1246 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1247 if (!pd) { 1248 ret = -EINVAL; 1249 goto err_free; 1250 } 1251 1252 INIT_UDATA(&udata, buf + sizeof(cmd), 1253 (unsigned long)cmd.response + sizeof(resp), 1254 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1255 out_len - sizeof(resp)); 1256 1257 ret = ib_rdmacg_try_charge(&uobj->cg_obj, ib_dev, 1258 RDMACG_RESOURCE_HCA_OBJECT); 1259 if (ret) 1260 goto err_charge; 1261 1262 mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata); 1263 if (IS_ERR(mw)) { 1264 ret = PTR_ERR(mw); 1265 goto err_put; 1266 } 1267 1268 mw->device = pd->device; 1269 mw->pd = pd; 1270 mw->uobject = uobj; 1271 atomic_inc(&pd->usecnt); 1272 1273 uobj->object = mw; 1274 ret = idr_add_uobj(&ib_uverbs_mw_idr, uobj); 1275 if (ret) 1276 goto err_unalloc; 1277 1278 memset(&resp, 0, sizeof(resp)); 1279 resp.rkey = mw->rkey; 1280 resp.mw_handle = uobj->id; 1281 1282 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1283 &resp, sizeof(resp))) { 1284 ret = -EFAULT; 1285 goto err_copy; 1286 } 1287 1288 put_pd_read(pd); 1289 1290 mutex_lock(&file->mutex); 1291 list_add_tail(&uobj->list, &file->ucontext->mw_list); 1292 mutex_unlock(&file->mutex); 1293 1294 uobj->live = 1; 1295 1296 up_write(&uobj->mutex); 1297 1298 return in_len; 1299 1300 err_copy: 1301 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1302 1303 err_unalloc: 1304 uverbs_dealloc_mw(mw); 1305 1306 err_put: 1307 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 1308 1309 err_charge: 1310 put_pd_read(pd); 1311 1312 err_free: 1313 put_uobj_write(uobj); 1314 return ret; 1315 } 1316 1317 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file, 1318 struct ib_device *ib_dev, 1319 const char __user *buf, int in_len, 1320 int out_len) 1321 { 1322 struct ib_uverbs_dealloc_mw cmd; 1323 struct ib_mw *mw; 1324 struct ib_uobject *uobj; 1325 int ret = -EINVAL; 1326 1327 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1328 return -EFAULT; 1329 1330 uobj = idr_write_uobj(&ib_uverbs_mw_idr, cmd.mw_handle, file->ucontext); 1331 if (!uobj) 1332 return -EINVAL; 1333 1334 mw = uobj->object; 1335 1336 ret = uverbs_dealloc_mw(mw); 1337 if (!ret) 1338 uobj->live = 0; 1339 1340 put_uobj_write(uobj); 1341 1342 if (ret) 1343 return ret; 1344 1345 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 1346 1347 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1348 1349 mutex_lock(&file->mutex); 1350 list_del(&uobj->list); 1351 mutex_unlock(&file->mutex); 1352 1353 put_uobj(uobj); 1354 1355 return in_len; 1356 } 1357 1358 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 1359 struct ib_device *ib_dev, 1360 const char __user *buf, int in_len, 1361 int out_len) 1362 { 1363 struct ib_uverbs_create_comp_channel cmd; 1364 struct ib_uverbs_create_comp_channel_resp resp; 1365 struct file *filp; 1366 int ret; 1367 1368 if (out_len < sizeof resp) 1369 return -ENOSPC; 1370 1371 if (copy_from_user(&cmd, buf, sizeof cmd)) 1372 return -EFAULT; 1373 1374 ret = get_unused_fd_flags(O_CLOEXEC); 1375 if (ret < 0) 1376 return ret; 1377 resp.fd = ret; 1378 1379 filp = ib_uverbs_alloc_event_file(file, ib_dev, 0); 1380 if (IS_ERR(filp)) { 1381 put_unused_fd(resp.fd); 1382 return PTR_ERR(filp); 1383 } 1384 1385 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1386 &resp, sizeof resp)) { 1387 put_unused_fd(resp.fd); 1388 fput(filp); 1389 return -EFAULT; 1390 } 1391 1392 fd_install(resp.fd, filp); 1393 return in_len; 1394 } 1395 1396 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, 1397 struct ib_device *ib_dev, 1398 struct ib_udata *ucore, 1399 struct ib_udata *uhw, 1400 struct ib_uverbs_ex_create_cq *cmd, 1401 size_t cmd_sz, 1402 int (*cb)(struct ib_uverbs_file *file, 1403 struct ib_ucq_object *obj, 1404 struct ib_uverbs_ex_create_cq_resp *resp, 1405 struct ib_udata *udata, 1406 void *context), 1407 void *context) 1408 { 1409 struct ib_ucq_object *obj; 1410 struct ib_uverbs_event_file *ev_file = NULL; 1411 struct ib_cq *cq; 1412 int ret; 1413 struct ib_uverbs_ex_create_cq_resp resp; 1414 struct ib_cq_init_attr attr = {}; 1415 1416 if (cmd->comp_vector >= file->device->num_comp_vectors) 1417 return ERR_PTR(-EINVAL); 1418 1419 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1420 if (!obj) 1421 return ERR_PTR(-ENOMEM); 1422 1423 init_uobj(&obj->uobject, cmd->user_handle, file->ucontext, &cq_lock_class); 1424 down_write(&obj->uobject.mutex); 1425 1426 if (cmd->comp_channel >= 0) { 1427 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel); 1428 if (!ev_file) { 1429 ret = -EINVAL; 1430 goto err; 1431 } 1432 } 1433 1434 obj->uverbs_file = file; 1435 obj->comp_events_reported = 0; 1436 obj->async_events_reported = 0; 1437 INIT_LIST_HEAD(&obj->comp_list); 1438 INIT_LIST_HEAD(&obj->async_list); 1439 1440 attr.cqe = cmd->cqe; 1441 attr.comp_vector = cmd->comp_vector; 1442 1443 if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags)) 1444 attr.flags = cmd->flags; 1445 1446 ret = ib_rdmacg_try_charge(&obj->uobject.cg_obj, ib_dev, 1447 RDMACG_RESOURCE_HCA_OBJECT); 1448 if (ret) 1449 goto err_charge; 1450 1451 cq = ib_dev->create_cq(ib_dev, &attr, 1452 file->ucontext, uhw); 1453 if (IS_ERR(cq)) { 1454 ret = PTR_ERR(cq); 1455 goto err_file; 1456 } 1457 1458 cq->device = ib_dev; 1459 cq->uobject = &obj->uobject; 1460 cq->comp_handler = ib_uverbs_comp_handler; 1461 cq->event_handler = ib_uverbs_cq_event_handler; 1462 cq->cq_context = ev_file; 1463 atomic_set(&cq->usecnt, 0); 1464 1465 obj->uobject.object = cq; 1466 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1467 if (ret) 1468 goto err_free; 1469 1470 memset(&resp, 0, sizeof resp); 1471 resp.base.cq_handle = obj->uobject.id; 1472 resp.base.cqe = cq->cqe; 1473 1474 resp.response_length = offsetof(typeof(resp), response_length) + 1475 sizeof(resp.response_length); 1476 1477 ret = cb(file, obj, &resp, ucore, context); 1478 if (ret) 1479 goto err_cb; 1480 1481 mutex_lock(&file->mutex); 1482 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); 1483 mutex_unlock(&file->mutex); 1484 1485 obj->uobject.live = 1; 1486 1487 up_write(&obj->uobject.mutex); 1488 1489 return obj; 1490 1491 err_cb: 1492 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1493 1494 err_free: 1495 ib_destroy_cq(cq); 1496 1497 err_file: 1498 ib_rdmacg_uncharge(&obj->uobject.cg_obj, ib_dev, 1499 RDMACG_RESOURCE_HCA_OBJECT); 1500 1501 err_charge: 1502 if (ev_file) 1503 ib_uverbs_release_ucq(file, ev_file, obj); 1504 1505 err: 1506 put_uobj_write(&obj->uobject); 1507 1508 return ERR_PTR(ret); 1509 } 1510 1511 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file, 1512 struct ib_ucq_object *obj, 1513 struct ib_uverbs_ex_create_cq_resp *resp, 1514 struct ib_udata *ucore, void *context) 1515 { 1516 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 1517 return -EFAULT; 1518 1519 return 0; 1520 } 1521 1522 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 1523 struct ib_device *ib_dev, 1524 const char __user *buf, int in_len, 1525 int out_len) 1526 { 1527 struct ib_uverbs_create_cq cmd; 1528 struct ib_uverbs_ex_create_cq cmd_ex; 1529 struct ib_uverbs_create_cq_resp resp; 1530 struct ib_udata ucore; 1531 struct ib_udata uhw; 1532 struct ib_ucq_object *obj; 1533 1534 if (out_len < sizeof(resp)) 1535 return -ENOSPC; 1536 1537 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1538 return -EFAULT; 1539 1540 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), sizeof(resp)); 1541 1542 INIT_UDATA(&uhw, buf + sizeof(cmd), 1543 (unsigned long)cmd.response + sizeof(resp), 1544 in_len - sizeof(cmd), out_len - sizeof(resp)); 1545 1546 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1547 cmd_ex.user_handle = cmd.user_handle; 1548 cmd_ex.cqe = cmd.cqe; 1549 cmd_ex.comp_vector = cmd.comp_vector; 1550 cmd_ex.comp_channel = cmd.comp_channel; 1551 1552 obj = create_cq(file, ib_dev, &ucore, &uhw, &cmd_ex, 1553 offsetof(typeof(cmd_ex), comp_channel) + 1554 sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb, 1555 NULL); 1556 1557 if (IS_ERR(obj)) 1558 return PTR_ERR(obj); 1559 1560 return in_len; 1561 } 1562 1563 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file, 1564 struct ib_ucq_object *obj, 1565 struct ib_uverbs_ex_create_cq_resp *resp, 1566 struct ib_udata *ucore, void *context) 1567 { 1568 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 1569 return -EFAULT; 1570 1571 return 0; 1572 } 1573 1574 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file, 1575 struct ib_device *ib_dev, 1576 struct ib_udata *ucore, 1577 struct ib_udata *uhw) 1578 { 1579 struct ib_uverbs_ex_create_cq_resp resp; 1580 struct ib_uverbs_ex_create_cq cmd; 1581 struct ib_ucq_object *obj; 1582 int err; 1583 1584 if (ucore->inlen < sizeof(cmd)) 1585 return -EINVAL; 1586 1587 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 1588 if (err) 1589 return err; 1590 1591 if (cmd.comp_mask) 1592 return -EINVAL; 1593 1594 if (cmd.reserved) 1595 return -EINVAL; 1596 1597 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 1598 sizeof(resp.response_length))) 1599 return -ENOSPC; 1600 1601 obj = create_cq(file, ib_dev, ucore, uhw, &cmd, 1602 min(ucore->inlen, sizeof(cmd)), 1603 ib_uverbs_ex_create_cq_cb, NULL); 1604 1605 if (IS_ERR(obj)) 1606 return PTR_ERR(obj); 1607 1608 return 0; 1609 } 1610 1611 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 1612 struct ib_device *ib_dev, 1613 const char __user *buf, int in_len, 1614 int out_len) 1615 { 1616 struct ib_uverbs_resize_cq cmd; 1617 struct ib_uverbs_resize_cq_resp resp; 1618 struct ib_udata udata; 1619 struct ib_cq *cq; 1620 int ret = -EINVAL; 1621 1622 if (copy_from_user(&cmd, buf, sizeof cmd)) 1623 return -EFAULT; 1624 1625 INIT_UDATA(&udata, buf + sizeof cmd, 1626 (unsigned long) cmd.response + sizeof resp, 1627 in_len - sizeof cmd, out_len - sizeof resp); 1628 1629 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1630 if (!cq) 1631 return -EINVAL; 1632 1633 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 1634 if (ret) 1635 goto out; 1636 1637 resp.cqe = cq->cqe; 1638 1639 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1640 &resp, sizeof resp.cqe)) 1641 ret = -EFAULT; 1642 1643 out: 1644 put_cq_read(cq); 1645 1646 return ret ? ret : in_len; 1647 } 1648 1649 static int copy_wc_to_user(void __user *dest, struct ib_wc *wc) 1650 { 1651 struct ib_uverbs_wc tmp; 1652 1653 tmp.wr_id = wc->wr_id; 1654 tmp.status = wc->status; 1655 tmp.opcode = wc->opcode; 1656 tmp.vendor_err = wc->vendor_err; 1657 tmp.byte_len = wc->byte_len; 1658 tmp.ex.imm_data = (__u32 __force) wc->ex.imm_data; 1659 tmp.qp_num = wc->qp->qp_num; 1660 tmp.src_qp = wc->src_qp; 1661 tmp.wc_flags = wc->wc_flags; 1662 tmp.pkey_index = wc->pkey_index; 1663 tmp.slid = wc->slid; 1664 tmp.sl = wc->sl; 1665 tmp.dlid_path_bits = wc->dlid_path_bits; 1666 tmp.port_num = wc->port_num; 1667 tmp.reserved = 0; 1668 1669 if (copy_to_user(dest, &tmp, sizeof tmp)) 1670 return -EFAULT; 1671 1672 return 0; 1673 } 1674 1675 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 1676 struct ib_device *ib_dev, 1677 const char __user *buf, int in_len, 1678 int out_len) 1679 { 1680 struct ib_uverbs_poll_cq cmd; 1681 struct ib_uverbs_poll_cq_resp resp; 1682 u8 __user *header_ptr; 1683 u8 __user *data_ptr; 1684 struct ib_cq *cq; 1685 struct ib_wc wc; 1686 int ret; 1687 1688 if (copy_from_user(&cmd, buf, sizeof cmd)) 1689 return -EFAULT; 1690 1691 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1692 if (!cq) 1693 return -EINVAL; 1694 1695 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1696 header_ptr = (void __user *)(unsigned long) cmd.response; 1697 data_ptr = header_ptr + sizeof resp; 1698 1699 memset(&resp, 0, sizeof resp); 1700 while (resp.count < cmd.ne) { 1701 ret = ib_poll_cq(cq, 1, &wc); 1702 if (ret < 0) 1703 goto out_put; 1704 if (!ret) 1705 break; 1706 1707 ret = copy_wc_to_user(data_ptr, &wc); 1708 if (ret) 1709 goto out_put; 1710 1711 data_ptr += sizeof(struct ib_uverbs_wc); 1712 ++resp.count; 1713 } 1714 1715 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1716 ret = -EFAULT; 1717 goto out_put; 1718 } 1719 1720 ret = in_len; 1721 1722 out_put: 1723 put_cq_read(cq); 1724 return ret; 1725 } 1726 1727 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 1728 struct ib_device *ib_dev, 1729 const char __user *buf, int in_len, 1730 int out_len) 1731 { 1732 struct ib_uverbs_req_notify_cq cmd; 1733 struct ib_cq *cq; 1734 1735 if (copy_from_user(&cmd, buf, sizeof cmd)) 1736 return -EFAULT; 1737 1738 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1739 if (!cq) 1740 return -EINVAL; 1741 1742 ib_req_notify_cq(cq, cmd.solicited_only ? 1743 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1744 1745 put_cq_read(cq); 1746 1747 return in_len; 1748 } 1749 1750 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 1751 struct ib_device *ib_dev, 1752 const char __user *buf, int in_len, 1753 int out_len) 1754 { 1755 struct ib_uverbs_destroy_cq cmd; 1756 struct ib_uverbs_destroy_cq_resp resp; 1757 struct ib_uobject *uobj; 1758 struct ib_cq *cq; 1759 struct ib_ucq_object *obj; 1760 struct ib_uverbs_event_file *ev_file; 1761 int ret = -EINVAL; 1762 1763 if (copy_from_user(&cmd, buf, sizeof cmd)) 1764 return -EFAULT; 1765 1766 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); 1767 if (!uobj) 1768 return -EINVAL; 1769 cq = uobj->object; 1770 ev_file = cq->cq_context; 1771 obj = container_of(cq->uobject, struct ib_ucq_object, uobject); 1772 1773 ret = ib_destroy_cq(cq); 1774 if (!ret) 1775 uobj->live = 0; 1776 1777 put_uobj_write(uobj); 1778 1779 if (ret) 1780 return ret; 1781 1782 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 1783 1784 idr_remove_uobj(&ib_uverbs_cq_idr, uobj); 1785 1786 mutex_lock(&file->mutex); 1787 list_del(&uobj->list); 1788 mutex_unlock(&file->mutex); 1789 1790 ib_uverbs_release_ucq(file, ev_file, obj); 1791 1792 memset(&resp, 0, sizeof resp); 1793 resp.comp_events_reported = obj->comp_events_reported; 1794 resp.async_events_reported = obj->async_events_reported; 1795 1796 put_uobj(uobj); 1797 1798 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1799 &resp, sizeof resp)) 1800 return -EFAULT; 1801 1802 return in_len; 1803 } 1804 1805 static int create_qp(struct ib_uverbs_file *file, 1806 struct ib_udata *ucore, 1807 struct ib_udata *uhw, 1808 struct ib_uverbs_ex_create_qp *cmd, 1809 size_t cmd_sz, 1810 int (*cb)(struct ib_uverbs_file *file, 1811 struct ib_uverbs_ex_create_qp_resp *resp, 1812 struct ib_udata *udata), 1813 void *context) 1814 { 1815 struct ib_uqp_object *obj; 1816 struct ib_device *device; 1817 struct ib_pd *pd = NULL; 1818 struct ib_xrcd *xrcd = NULL; 1819 struct ib_uobject *uninitialized_var(xrcd_uobj); 1820 struct ib_cq *scq = NULL, *rcq = NULL; 1821 struct ib_srq *srq = NULL; 1822 struct ib_qp *qp; 1823 char *buf; 1824 struct ib_qp_init_attr attr = {}; 1825 struct ib_uverbs_ex_create_qp_resp resp; 1826 int ret; 1827 struct ib_rwq_ind_table *ind_tbl = NULL; 1828 bool has_sq = true; 1829 1830 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1831 return -EPERM; 1832 1833 obj = kzalloc(sizeof *obj, GFP_KERNEL); 1834 if (!obj) 1835 return -ENOMEM; 1836 1837 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, 1838 &qp_lock_class); 1839 down_write(&obj->uevent.uobject.mutex); 1840 if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) + 1841 sizeof(cmd->rwq_ind_tbl_handle) && 1842 (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) { 1843 ind_tbl = idr_read_rwq_indirection_table(cmd->rwq_ind_tbl_handle, 1844 file->ucontext); 1845 if (!ind_tbl) { 1846 ret = -EINVAL; 1847 goto err_put; 1848 } 1849 1850 attr.rwq_ind_tbl = ind_tbl; 1851 } 1852 1853 if ((cmd_sz >= offsetof(typeof(*cmd), reserved1) + 1854 sizeof(cmd->reserved1)) && cmd->reserved1) { 1855 ret = -EOPNOTSUPP; 1856 goto err_put; 1857 } 1858 1859 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) { 1860 ret = -EINVAL; 1861 goto err_put; 1862 } 1863 1864 if (ind_tbl && !cmd->max_send_wr) 1865 has_sq = false; 1866 1867 if (cmd->qp_type == IB_QPT_XRC_TGT) { 1868 xrcd = idr_read_xrcd(cmd->pd_handle, file->ucontext, 1869 &xrcd_uobj); 1870 if (!xrcd) { 1871 ret = -EINVAL; 1872 goto err_put; 1873 } 1874 device = xrcd->device; 1875 } else { 1876 if (cmd->qp_type == IB_QPT_XRC_INI) { 1877 cmd->max_recv_wr = 0; 1878 cmd->max_recv_sge = 0; 1879 } else { 1880 if (cmd->is_srq) { 1881 srq = idr_read_srq(cmd->srq_handle, 1882 file->ucontext); 1883 if (!srq || srq->srq_type != IB_SRQT_BASIC) { 1884 ret = -EINVAL; 1885 goto err_put; 1886 } 1887 } 1888 1889 if (!ind_tbl) { 1890 if (cmd->recv_cq_handle != cmd->send_cq_handle) { 1891 rcq = idr_read_cq(cmd->recv_cq_handle, 1892 file->ucontext, 0); 1893 if (!rcq) { 1894 ret = -EINVAL; 1895 goto err_put; 1896 } 1897 } 1898 } 1899 } 1900 1901 if (has_sq) 1902 scq = idr_read_cq(cmd->send_cq_handle, file->ucontext, !!rcq); 1903 if (!ind_tbl) 1904 rcq = rcq ?: scq; 1905 pd = idr_read_pd(cmd->pd_handle, file->ucontext); 1906 if (!pd || (!scq && has_sq)) { 1907 ret = -EINVAL; 1908 goto err_put; 1909 } 1910 1911 device = pd->device; 1912 } 1913 1914 attr.event_handler = ib_uverbs_qp_event_handler; 1915 attr.qp_context = file; 1916 attr.send_cq = scq; 1917 attr.recv_cq = rcq; 1918 attr.srq = srq; 1919 attr.xrcd = xrcd; 1920 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : 1921 IB_SIGNAL_REQ_WR; 1922 attr.qp_type = cmd->qp_type; 1923 attr.create_flags = 0; 1924 1925 attr.cap.max_send_wr = cmd->max_send_wr; 1926 attr.cap.max_recv_wr = cmd->max_recv_wr; 1927 attr.cap.max_send_sge = cmd->max_send_sge; 1928 attr.cap.max_recv_sge = cmd->max_recv_sge; 1929 attr.cap.max_inline_data = cmd->max_inline_data; 1930 1931 obj->uevent.events_reported = 0; 1932 INIT_LIST_HEAD(&obj->uevent.event_list); 1933 INIT_LIST_HEAD(&obj->mcast_list); 1934 1935 if (cmd_sz >= offsetof(typeof(*cmd), create_flags) + 1936 sizeof(cmd->create_flags)) 1937 attr.create_flags = cmd->create_flags; 1938 1939 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 1940 IB_QP_CREATE_CROSS_CHANNEL | 1941 IB_QP_CREATE_MANAGED_SEND | 1942 IB_QP_CREATE_MANAGED_RECV | 1943 IB_QP_CREATE_SCATTER_FCS | 1944 IB_QP_CREATE_CVLAN_STRIPPING)) { 1945 ret = -EINVAL; 1946 goto err_put; 1947 } 1948 1949 buf = (void *)cmd + sizeof(*cmd); 1950 if (cmd_sz > sizeof(*cmd)) 1951 if (!(buf[0] == 0 && !memcmp(buf, buf + 1, 1952 cmd_sz - sizeof(*cmd) - 1))) { 1953 ret = -EINVAL; 1954 goto err_put; 1955 } 1956 1957 ret = ib_rdmacg_try_charge(&obj->uevent.uobject.cg_obj, device, 1958 RDMACG_RESOURCE_HCA_OBJECT); 1959 if (ret) 1960 goto err_put; 1961 1962 if (cmd->qp_type == IB_QPT_XRC_TGT) 1963 qp = ib_create_qp(pd, &attr); 1964 else 1965 qp = device->create_qp(pd, &attr, uhw); 1966 1967 if (IS_ERR(qp)) { 1968 ret = PTR_ERR(qp); 1969 goto err_create; 1970 } 1971 1972 if (cmd->qp_type != IB_QPT_XRC_TGT) { 1973 qp->real_qp = qp; 1974 qp->device = device; 1975 qp->pd = pd; 1976 qp->send_cq = attr.send_cq; 1977 qp->recv_cq = attr.recv_cq; 1978 qp->srq = attr.srq; 1979 qp->rwq_ind_tbl = ind_tbl; 1980 qp->event_handler = attr.event_handler; 1981 qp->qp_context = attr.qp_context; 1982 qp->qp_type = attr.qp_type; 1983 atomic_set(&qp->usecnt, 0); 1984 atomic_inc(&pd->usecnt); 1985 if (attr.send_cq) 1986 atomic_inc(&attr.send_cq->usecnt); 1987 if (attr.recv_cq) 1988 atomic_inc(&attr.recv_cq->usecnt); 1989 if (attr.srq) 1990 atomic_inc(&attr.srq->usecnt); 1991 if (ind_tbl) 1992 atomic_inc(&ind_tbl->usecnt); 1993 } 1994 qp->uobject = &obj->uevent.uobject; 1995 1996 obj->uevent.uobject.object = qp; 1997 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1998 if (ret) 1999 goto err_destroy; 2000 2001 memset(&resp, 0, sizeof resp); 2002 resp.base.qpn = qp->qp_num; 2003 resp.base.qp_handle = obj->uevent.uobject.id; 2004 resp.base.max_recv_sge = attr.cap.max_recv_sge; 2005 resp.base.max_send_sge = attr.cap.max_send_sge; 2006 resp.base.max_recv_wr = attr.cap.max_recv_wr; 2007 resp.base.max_send_wr = attr.cap.max_send_wr; 2008 resp.base.max_inline_data = attr.cap.max_inline_data; 2009 2010 resp.response_length = offsetof(typeof(resp), response_length) + 2011 sizeof(resp.response_length); 2012 2013 ret = cb(file, &resp, ucore); 2014 if (ret) 2015 goto err_cb; 2016 2017 if (xrcd) { 2018 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 2019 uobject); 2020 atomic_inc(&obj->uxrcd->refcnt); 2021 put_xrcd_read(xrcd_uobj); 2022 } 2023 2024 if (pd) 2025 put_pd_read(pd); 2026 if (scq) 2027 put_cq_read(scq); 2028 if (rcq && rcq != scq) 2029 put_cq_read(rcq); 2030 if (srq) 2031 put_srq_read(srq); 2032 if (ind_tbl) 2033 put_rwq_indirection_table_read(ind_tbl); 2034 2035 mutex_lock(&file->mutex); 2036 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 2037 mutex_unlock(&file->mutex); 2038 2039 obj->uevent.uobject.live = 1; 2040 2041 up_write(&obj->uevent.uobject.mutex); 2042 2043 return 0; 2044 err_cb: 2045 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 2046 2047 err_destroy: 2048 ib_destroy_qp(qp); 2049 2050 err_create: 2051 ib_rdmacg_uncharge(&obj->uevent.uobject.cg_obj, device, 2052 RDMACG_RESOURCE_HCA_OBJECT); 2053 2054 err_put: 2055 if (xrcd) 2056 put_xrcd_read(xrcd_uobj); 2057 if (pd) 2058 put_pd_read(pd); 2059 if (scq) 2060 put_cq_read(scq); 2061 if (rcq && rcq != scq) 2062 put_cq_read(rcq); 2063 if (srq) 2064 put_srq_read(srq); 2065 if (ind_tbl) 2066 put_rwq_indirection_table_read(ind_tbl); 2067 2068 put_uobj_write(&obj->uevent.uobject); 2069 return ret; 2070 } 2071 2072 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file, 2073 struct ib_uverbs_ex_create_qp_resp *resp, 2074 struct ib_udata *ucore) 2075 { 2076 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 2077 return -EFAULT; 2078 2079 return 0; 2080 } 2081 2082 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 2083 struct ib_device *ib_dev, 2084 const char __user *buf, int in_len, 2085 int out_len) 2086 { 2087 struct ib_uverbs_create_qp cmd; 2088 struct ib_uverbs_ex_create_qp cmd_ex; 2089 struct ib_udata ucore; 2090 struct ib_udata uhw; 2091 ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp); 2092 int err; 2093 2094 if (out_len < resp_size) 2095 return -ENOSPC; 2096 2097 if (copy_from_user(&cmd, buf, sizeof(cmd))) 2098 return -EFAULT; 2099 2100 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), 2101 resp_size); 2102 INIT_UDATA(&uhw, buf + sizeof(cmd), 2103 (unsigned long)cmd.response + resp_size, 2104 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 2105 out_len - resp_size); 2106 2107 memset(&cmd_ex, 0, sizeof(cmd_ex)); 2108 cmd_ex.user_handle = cmd.user_handle; 2109 cmd_ex.pd_handle = cmd.pd_handle; 2110 cmd_ex.send_cq_handle = cmd.send_cq_handle; 2111 cmd_ex.recv_cq_handle = cmd.recv_cq_handle; 2112 cmd_ex.srq_handle = cmd.srq_handle; 2113 cmd_ex.max_send_wr = cmd.max_send_wr; 2114 cmd_ex.max_recv_wr = cmd.max_recv_wr; 2115 cmd_ex.max_send_sge = cmd.max_send_sge; 2116 cmd_ex.max_recv_sge = cmd.max_recv_sge; 2117 cmd_ex.max_inline_data = cmd.max_inline_data; 2118 cmd_ex.sq_sig_all = cmd.sq_sig_all; 2119 cmd_ex.qp_type = cmd.qp_type; 2120 cmd_ex.is_srq = cmd.is_srq; 2121 2122 err = create_qp(file, &ucore, &uhw, &cmd_ex, 2123 offsetof(typeof(cmd_ex), is_srq) + 2124 sizeof(cmd.is_srq), ib_uverbs_create_qp_cb, 2125 NULL); 2126 2127 if (err) 2128 return err; 2129 2130 return in_len; 2131 } 2132 2133 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file, 2134 struct ib_uverbs_ex_create_qp_resp *resp, 2135 struct ib_udata *ucore) 2136 { 2137 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 2138 return -EFAULT; 2139 2140 return 0; 2141 } 2142 2143 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file, 2144 struct ib_device *ib_dev, 2145 struct ib_udata *ucore, 2146 struct ib_udata *uhw) 2147 { 2148 struct ib_uverbs_ex_create_qp_resp resp; 2149 struct ib_uverbs_ex_create_qp cmd = {0}; 2150 int err; 2151 2152 if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) + 2153 sizeof(cmd.comp_mask))) 2154 return -EINVAL; 2155 2156 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2157 if (err) 2158 return err; 2159 2160 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK) 2161 return -EINVAL; 2162 2163 if (cmd.reserved) 2164 return -EINVAL; 2165 2166 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 2167 sizeof(resp.response_length))) 2168 return -ENOSPC; 2169 2170 err = create_qp(file, ucore, uhw, &cmd, 2171 min(ucore->inlen, sizeof(cmd)), 2172 ib_uverbs_ex_create_qp_cb, NULL); 2173 2174 if (err) 2175 return err; 2176 2177 return 0; 2178 } 2179 2180 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, 2181 struct ib_device *ib_dev, 2182 const char __user *buf, int in_len, int out_len) 2183 { 2184 struct ib_uverbs_open_qp cmd; 2185 struct ib_uverbs_create_qp_resp resp; 2186 struct ib_udata udata; 2187 struct ib_uqp_object *obj; 2188 struct ib_xrcd *xrcd; 2189 struct ib_uobject *uninitialized_var(xrcd_uobj); 2190 struct ib_qp *qp; 2191 struct ib_qp_open_attr attr; 2192 int ret; 2193 2194 if (out_len < sizeof resp) 2195 return -ENOSPC; 2196 2197 if (copy_from_user(&cmd, buf, sizeof cmd)) 2198 return -EFAULT; 2199 2200 INIT_UDATA(&udata, buf + sizeof cmd, 2201 (unsigned long) cmd.response + sizeof resp, 2202 in_len - sizeof cmd, out_len - sizeof resp); 2203 2204 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2205 if (!obj) 2206 return -ENOMEM; 2207 2208 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class); 2209 down_write(&obj->uevent.uobject.mutex); 2210 2211 xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj); 2212 if (!xrcd) { 2213 ret = -EINVAL; 2214 goto err_put; 2215 } 2216 2217 attr.event_handler = ib_uverbs_qp_event_handler; 2218 attr.qp_context = file; 2219 attr.qp_num = cmd.qpn; 2220 attr.qp_type = cmd.qp_type; 2221 2222 obj->uevent.events_reported = 0; 2223 INIT_LIST_HEAD(&obj->uevent.event_list); 2224 INIT_LIST_HEAD(&obj->mcast_list); 2225 2226 qp = ib_open_qp(xrcd, &attr); 2227 if (IS_ERR(qp)) { 2228 ret = PTR_ERR(qp); 2229 goto err_put; 2230 } 2231 2232 qp->uobject = &obj->uevent.uobject; 2233 2234 obj->uevent.uobject.object = qp; 2235 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 2236 if (ret) 2237 goto err_destroy; 2238 2239 memset(&resp, 0, sizeof resp); 2240 resp.qpn = qp->qp_num; 2241 resp.qp_handle = obj->uevent.uobject.id; 2242 2243 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2244 &resp, sizeof resp)) { 2245 ret = -EFAULT; 2246 goto err_remove; 2247 } 2248 2249 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 2250 atomic_inc(&obj->uxrcd->refcnt); 2251 put_xrcd_read(xrcd_uobj); 2252 2253 mutex_lock(&file->mutex); 2254 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 2255 mutex_unlock(&file->mutex); 2256 2257 obj->uevent.uobject.live = 1; 2258 2259 up_write(&obj->uevent.uobject.mutex); 2260 2261 return in_len; 2262 2263 err_remove: 2264 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 2265 2266 err_destroy: 2267 ib_destroy_qp(qp); 2268 2269 err_put: 2270 put_xrcd_read(xrcd_uobj); 2271 put_uobj_write(&obj->uevent.uobject); 2272 return ret; 2273 } 2274 2275 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 2276 struct ib_device *ib_dev, 2277 const char __user *buf, int in_len, 2278 int out_len) 2279 { 2280 struct ib_uverbs_query_qp cmd; 2281 struct ib_uverbs_query_qp_resp resp; 2282 struct ib_qp *qp; 2283 struct ib_qp_attr *attr; 2284 struct ib_qp_init_attr *init_attr; 2285 int ret; 2286 2287 if (copy_from_user(&cmd, buf, sizeof cmd)) 2288 return -EFAULT; 2289 2290 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2291 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 2292 if (!attr || !init_attr) { 2293 ret = -ENOMEM; 2294 goto out; 2295 } 2296 2297 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2298 if (!qp) { 2299 ret = -EINVAL; 2300 goto out; 2301 } 2302 2303 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 2304 2305 put_qp_read(qp); 2306 2307 if (ret) 2308 goto out; 2309 2310 memset(&resp, 0, sizeof resp); 2311 2312 resp.qp_state = attr->qp_state; 2313 resp.cur_qp_state = attr->cur_qp_state; 2314 resp.path_mtu = attr->path_mtu; 2315 resp.path_mig_state = attr->path_mig_state; 2316 resp.qkey = attr->qkey; 2317 resp.rq_psn = attr->rq_psn; 2318 resp.sq_psn = attr->sq_psn; 2319 resp.dest_qp_num = attr->dest_qp_num; 2320 resp.qp_access_flags = attr->qp_access_flags; 2321 resp.pkey_index = attr->pkey_index; 2322 resp.alt_pkey_index = attr->alt_pkey_index; 2323 resp.sq_draining = attr->sq_draining; 2324 resp.max_rd_atomic = attr->max_rd_atomic; 2325 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 2326 resp.min_rnr_timer = attr->min_rnr_timer; 2327 resp.port_num = attr->port_num; 2328 resp.timeout = attr->timeout; 2329 resp.retry_cnt = attr->retry_cnt; 2330 resp.rnr_retry = attr->rnr_retry; 2331 resp.alt_port_num = attr->alt_port_num; 2332 resp.alt_timeout = attr->alt_timeout; 2333 2334 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 2335 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 2336 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 2337 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 2338 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 2339 resp.dest.dlid = attr->ah_attr.dlid; 2340 resp.dest.sl = attr->ah_attr.sl; 2341 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 2342 resp.dest.static_rate = attr->ah_attr.static_rate; 2343 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 2344 resp.dest.port_num = attr->ah_attr.port_num; 2345 2346 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 2347 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 2348 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 2349 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 2350 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 2351 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 2352 resp.alt_dest.sl = attr->alt_ah_attr.sl; 2353 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 2354 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 2355 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 2356 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 2357 2358 resp.max_send_wr = init_attr->cap.max_send_wr; 2359 resp.max_recv_wr = init_attr->cap.max_recv_wr; 2360 resp.max_send_sge = init_attr->cap.max_send_sge; 2361 resp.max_recv_sge = init_attr->cap.max_recv_sge; 2362 resp.max_inline_data = init_attr->cap.max_inline_data; 2363 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2364 2365 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2366 &resp, sizeof resp)) 2367 ret = -EFAULT; 2368 2369 out: 2370 kfree(attr); 2371 kfree(init_attr); 2372 2373 return ret ? ret : in_len; 2374 } 2375 2376 /* Remove ignored fields set in the attribute mask */ 2377 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 2378 { 2379 switch (qp_type) { 2380 case IB_QPT_XRC_INI: 2381 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 2382 case IB_QPT_XRC_TGT: 2383 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 2384 IB_QP_RNR_RETRY); 2385 default: 2386 return mask; 2387 } 2388 } 2389 2390 static int modify_qp(struct ib_uverbs_file *file, 2391 struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata) 2392 { 2393 struct ib_qp_attr *attr; 2394 struct ib_qp *qp; 2395 int ret; 2396 2397 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2398 if (!attr) 2399 return -ENOMEM; 2400 2401 qp = idr_read_qp(cmd->base.qp_handle, file->ucontext); 2402 if (!qp) { 2403 ret = -EINVAL; 2404 goto out; 2405 } 2406 2407 attr->qp_state = cmd->base.qp_state; 2408 attr->cur_qp_state = cmd->base.cur_qp_state; 2409 attr->path_mtu = cmd->base.path_mtu; 2410 attr->path_mig_state = cmd->base.path_mig_state; 2411 attr->qkey = cmd->base.qkey; 2412 attr->rq_psn = cmd->base.rq_psn; 2413 attr->sq_psn = cmd->base.sq_psn; 2414 attr->dest_qp_num = cmd->base.dest_qp_num; 2415 attr->qp_access_flags = cmd->base.qp_access_flags; 2416 attr->pkey_index = cmd->base.pkey_index; 2417 attr->alt_pkey_index = cmd->base.alt_pkey_index; 2418 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify; 2419 attr->max_rd_atomic = cmd->base.max_rd_atomic; 2420 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic; 2421 attr->min_rnr_timer = cmd->base.min_rnr_timer; 2422 attr->port_num = cmd->base.port_num; 2423 attr->timeout = cmd->base.timeout; 2424 attr->retry_cnt = cmd->base.retry_cnt; 2425 attr->rnr_retry = cmd->base.rnr_retry; 2426 attr->alt_port_num = cmd->base.alt_port_num; 2427 attr->alt_timeout = cmd->base.alt_timeout; 2428 attr->rate_limit = cmd->rate_limit; 2429 2430 memcpy(attr->ah_attr.grh.dgid.raw, cmd->base.dest.dgid, 16); 2431 attr->ah_attr.grh.flow_label = cmd->base.dest.flow_label; 2432 attr->ah_attr.grh.sgid_index = cmd->base.dest.sgid_index; 2433 attr->ah_attr.grh.hop_limit = cmd->base.dest.hop_limit; 2434 attr->ah_attr.grh.traffic_class = cmd->base.dest.traffic_class; 2435 attr->ah_attr.dlid = cmd->base.dest.dlid; 2436 attr->ah_attr.sl = cmd->base.dest.sl; 2437 attr->ah_attr.src_path_bits = cmd->base.dest.src_path_bits; 2438 attr->ah_attr.static_rate = cmd->base.dest.static_rate; 2439 attr->ah_attr.ah_flags = cmd->base.dest.is_global ? 2440 IB_AH_GRH : 0; 2441 attr->ah_attr.port_num = cmd->base.dest.port_num; 2442 2443 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd->base.alt_dest.dgid, 16); 2444 attr->alt_ah_attr.grh.flow_label = cmd->base.alt_dest.flow_label; 2445 attr->alt_ah_attr.grh.sgid_index = cmd->base.alt_dest.sgid_index; 2446 attr->alt_ah_attr.grh.hop_limit = cmd->base.alt_dest.hop_limit; 2447 attr->alt_ah_attr.grh.traffic_class = cmd->base.alt_dest.traffic_class; 2448 attr->alt_ah_attr.dlid = cmd->base.alt_dest.dlid; 2449 attr->alt_ah_attr.sl = cmd->base.alt_dest.sl; 2450 attr->alt_ah_attr.src_path_bits = cmd->base.alt_dest.src_path_bits; 2451 attr->alt_ah_attr.static_rate = cmd->base.alt_dest.static_rate; 2452 attr->alt_ah_attr.ah_flags = cmd->base.alt_dest.is_global ? 2453 IB_AH_GRH : 0; 2454 attr->alt_ah_attr.port_num = cmd->base.alt_dest.port_num; 2455 2456 if (qp->real_qp == qp) { 2457 if (cmd->base.attr_mask & IB_QP_AV) { 2458 ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr); 2459 if (ret) 2460 goto release_qp; 2461 } 2462 ret = qp->device->modify_qp(qp, attr, 2463 modify_qp_mask(qp->qp_type, 2464 cmd->base.attr_mask), 2465 udata); 2466 } else { 2467 ret = ib_modify_qp(qp, attr, 2468 modify_qp_mask(qp->qp_type, 2469 cmd->base.attr_mask)); 2470 } 2471 2472 release_qp: 2473 put_qp_read(qp); 2474 2475 out: 2476 kfree(attr); 2477 2478 return ret; 2479 } 2480 2481 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 2482 struct ib_device *ib_dev, 2483 const char __user *buf, int in_len, 2484 int out_len) 2485 { 2486 struct ib_uverbs_ex_modify_qp cmd = {}; 2487 struct ib_udata udata; 2488 int ret; 2489 2490 if (copy_from_user(&cmd.base, buf, sizeof(cmd.base))) 2491 return -EFAULT; 2492 2493 if (cmd.base.attr_mask & 2494 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1)) 2495 return -EOPNOTSUPP; 2496 2497 INIT_UDATA(&udata, buf + sizeof(cmd.base), NULL, 2498 in_len - sizeof(cmd.base), out_len); 2499 2500 ret = modify_qp(file, &cmd, &udata); 2501 if (ret) 2502 return ret; 2503 2504 return in_len; 2505 } 2506 2507 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file, 2508 struct ib_device *ib_dev, 2509 struct ib_udata *ucore, 2510 struct ib_udata *uhw) 2511 { 2512 struct ib_uverbs_ex_modify_qp cmd = {}; 2513 int ret; 2514 2515 /* 2516 * Last bit is reserved for extending the attr_mask by 2517 * using another field. 2518 */ 2519 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); 2520 2521 if (ucore->inlen < sizeof(cmd.base)) 2522 return -EINVAL; 2523 2524 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2525 if (ret) 2526 return ret; 2527 2528 if (cmd.base.attr_mask & 2529 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) 2530 return -EOPNOTSUPP; 2531 2532 if (ucore->inlen > sizeof(cmd)) { 2533 if (ib_is_udata_cleared(ucore, sizeof(cmd), 2534 ucore->inlen - sizeof(cmd))) 2535 return -EOPNOTSUPP; 2536 } 2537 2538 ret = modify_qp(file, &cmd, uhw); 2539 2540 return ret; 2541 } 2542 2543 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 2544 struct ib_device *ib_dev, 2545 const char __user *buf, int in_len, 2546 int out_len) 2547 { 2548 struct ib_uverbs_destroy_qp cmd; 2549 struct ib_uverbs_destroy_qp_resp resp; 2550 struct ib_uobject *uobj; 2551 struct ib_qp *qp; 2552 struct ib_uqp_object *obj; 2553 int ret = -EINVAL; 2554 2555 if (copy_from_user(&cmd, buf, sizeof cmd)) 2556 return -EFAULT; 2557 2558 memset(&resp, 0, sizeof resp); 2559 2560 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); 2561 if (!uobj) 2562 return -EINVAL; 2563 qp = uobj->object; 2564 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 2565 2566 if (!list_empty(&obj->mcast_list)) { 2567 put_uobj_write(uobj); 2568 return -EBUSY; 2569 } 2570 2571 ret = ib_destroy_qp(qp); 2572 if (!ret) 2573 uobj->live = 0; 2574 2575 put_uobj_write(uobj); 2576 2577 if (ret) 2578 return ret; 2579 2580 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 2581 2582 if (obj->uxrcd) 2583 atomic_dec(&obj->uxrcd->refcnt); 2584 2585 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 2586 2587 mutex_lock(&file->mutex); 2588 list_del(&uobj->list); 2589 mutex_unlock(&file->mutex); 2590 2591 ib_uverbs_release_uevent(file, &obj->uevent); 2592 2593 resp.events_reported = obj->uevent.events_reported; 2594 2595 put_uobj(uobj); 2596 2597 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2598 &resp, sizeof resp)) 2599 return -EFAULT; 2600 2601 return in_len; 2602 } 2603 2604 static void *alloc_wr(size_t wr_size, __u32 num_sge) 2605 { 2606 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) + 2607 num_sge * sizeof (struct ib_sge), GFP_KERNEL); 2608 }; 2609 2610 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 2611 struct ib_device *ib_dev, 2612 const char __user *buf, int in_len, 2613 int out_len) 2614 { 2615 struct ib_uverbs_post_send cmd; 2616 struct ib_uverbs_post_send_resp resp; 2617 struct ib_uverbs_send_wr *user_wr; 2618 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; 2619 struct ib_qp *qp; 2620 int i, sg_ind; 2621 int is_ud; 2622 ssize_t ret = -EINVAL; 2623 size_t next_size; 2624 2625 if (copy_from_user(&cmd, buf, sizeof cmd)) 2626 return -EFAULT; 2627 2628 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 2629 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 2630 return -EINVAL; 2631 2632 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 2633 return -EINVAL; 2634 2635 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2636 if (!user_wr) 2637 return -ENOMEM; 2638 2639 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2640 if (!qp) 2641 goto out; 2642 2643 is_ud = qp->qp_type == IB_QPT_UD; 2644 sg_ind = 0; 2645 last = NULL; 2646 for (i = 0; i < cmd.wr_count; ++i) { 2647 if (copy_from_user(user_wr, 2648 buf + sizeof cmd + i * cmd.wqe_size, 2649 cmd.wqe_size)) { 2650 ret = -EFAULT; 2651 goto out_put; 2652 } 2653 2654 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2655 ret = -EINVAL; 2656 goto out_put; 2657 } 2658 2659 if (is_ud) { 2660 struct ib_ud_wr *ud; 2661 2662 if (user_wr->opcode != IB_WR_SEND && 2663 user_wr->opcode != IB_WR_SEND_WITH_IMM) { 2664 ret = -EINVAL; 2665 goto out_put; 2666 } 2667 2668 next_size = sizeof(*ud); 2669 ud = alloc_wr(next_size, user_wr->num_sge); 2670 if (!ud) { 2671 ret = -ENOMEM; 2672 goto out_put; 2673 } 2674 2675 ud->ah = idr_read_ah(user_wr->wr.ud.ah, file->ucontext); 2676 if (!ud->ah) { 2677 kfree(ud); 2678 ret = -EINVAL; 2679 goto out_put; 2680 } 2681 ud->remote_qpn = user_wr->wr.ud.remote_qpn; 2682 ud->remote_qkey = user_wr->wr.ud.remote_qkey; 2683 2684 next = &ud->wr; 2685 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2686 user_wr->opcode == IB_WR_RDMA_WRITE || 2687 user_wr->opcode == IB_WR_RDMA_READ) { 2688 struct ib_rdma_wr *rdma; 2689 2690 next_size = sizeof(*rdma); 2691 rdma = alloc_wr(next_size, user_wr->num_sge); 2692 if (!rdma) { 2693 ret = -ENOMEM; 2694 goto out_put; 2695 } 2696 2697 rdma->remote_addr = user_wr->wr.rdma.remote_addr; 2698 rdma->rkey = user_wr->wr.rdma.rkey; 2699 2700 next = &rdma->wr; 2701 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2702 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2703 struct ib_atomic_wr *atomic; 2704 2705 next_size = sizeof(*atomic); 2706 atomic = alloc_wr(next_size, user_wr->num_sge); 2707 if (!atomic) { 2708 ret = -ENOMEM; 2709 goto out_put; 2710 } 2711 2712 atomic->remote_addr = user_wr->wr.atomic.remote_addr; 2713 atomic->compare_add = user_wr->wr.atomic.compare_add; 2714 atomic->swap = user_wr->wr.atomic.swap; 2715 atomic->rkey = user_wr->wr.atomic.rkey; 2716 2717 next = &atomic->wr; 2718 } else if (user_wr->opcode == IB_WR_SEND || 2719 user_wr->opcode == IB_WR_SEND_WITH_IMM || 2720 user_wr->opcode == IB_WR_SEND_WITH_INV) { 2721 next_size = sizeof(*next); 2722 next = alloc_wr(next_size, user_wr->num_sge); 2723 if (!next) { 2724 ret = -ENOMEM; 2725 goto out_put; 2726 } 2727 } else { 2728 ret = -EINVAL; 2729 goto out_put; 2730 } 2731 2732 if (user_wr->opcode == IB_WR_SEND_WITH_IMM || 2733 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 2734 next->ex.imm_data = 2735 (__be32 __force) user_wr->ex.imm_data; 2736 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) { 2737 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; 2738 } 2739 2740 if (!last) 2741 wr = next; 2742 else 2743 last->next = next; 2744 last = next; 2745 2746 next->next = NULL; 2747 next->wr_id = user_wr->wr_id; 2748 next->num_sge = user_wr->num_sge; 2749 next->opcode = user_wr->opcode; 2750 next->send_flags = user_wr->send_flags; 2751 2752 if (next->num_sge) { 2753 next->sg_list = (void *) next + 2754 ALIGN(next_size, sizeof(struct ib_sge)); 2755 if (copy_from_user(next->sg_list, 2756 buf + sizeof cmd + 2757 cmd.wr_count * cmd.wqe_size + 2758 sg_ind * sizeof (struct ib_sge), 2759 next->num_sge * sizeof (struct ib_sge))) { 2760 ret = -EFAULT; 2761 goto out_put; 2762 } 2763 sg_ind += next->num_sge; 2764 } else 2765 next->sg_list = NULL; 2766 } 2767 2768 resp.bad_wr = 0; 2769 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr); 2770 if (ret) 2771 for (next = wr; next; next = next->next) { 2772 ++resp.bad_wr; 2773 if (next == bad_wr) 2774 break; 2775 } 2776 2777 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2778 &resp, sizeof resp)) 2779 ret = -EFAULT; 2780 2781 out_put: 2782 put_qp_read(qp); 2783 2784 while (wr) { 2785 if (is_ud && ud_wr(wr)->ah) 2786 put_ah_read(ud_wr(wr)->ah); 2787 next = wr->next; 2788 kfree(wr); 2789 wr = next; 2790 } 2791 2792 out: 2793 kfree(user_wr); 2794 2795 return ret ? ret : in_len; 2796 } 2797 2798 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 2799 int in_len, 2800 u32 wr_count, 2801 u32 sge_count, 2802 u32 wqe_size) 2803 { 2804 struct ib_uverbs_recv_wr *user_wr; 2805 struct ib_recv_wr *wr = NULL, *last, *next; 2806 int sg_ind; 2807 int i; 2808 int ret; 2809 2810 if (in_len < wqe_size * wr_count + 2811 sge_count * sizeof (struct ib_uverbs_sge)) 2812 return ERR_PTR(-EINVAL); 2813 2814 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2815 return ERR_PTR(-EINVAL); 2816 2817 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2818 if (!user_wr) 2819 return ERR_PTR(-ENOMEM); 2820 2821 sg_ind = 0; 2822 last = NULL; 2823 for (i = 0; i < wr_count; ++i) { 2824 if (copy_from_user(user_wr, buf + i * wqe_size, 2825 wqe_size)) { 2826 ret = -EFAULT; 2827 goto err; 2828 } 2829 2830 if (user_wr->num_sge + sg_ind > sge_count) { 2831 ret = -EINVAL; 2832 goto err; 2833 } 2834 2835 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2836 user_wr->num_sge * sizeof (struct ib_sge), 2837 GFP_KERNEL); 2838 if (!next) { 2839 ret = -ENOMEM; 2840 goto err; 2841 } 2842 2843 if (!last) 2844 wr = next; 2845 else 2846 last->next = next; 2847 last = next; 2848 2849 next->next = NULL; 2850 next->wr_id = user_wr->wr_id; 2851 next->num_sge = user_wr->num_sge; 2852 2853 if (next->num_sge) { 2854 next->sg_list = (void *) next + 2855 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2856 if (copy_from_user(next->sg_list, 2857 buf + wr_count * wqe_size + 2858 sg_ind * sizeof (struct ib_sge), 2859 next->num_sge * sizeof (struct ib_sge))) { 2860 ret = -EFAULT; 2861 goto err; 2862 } 2863 sg_ind += next->num_sge; 2864 } else 2865 next->sg_list = NULL; 2866 } 2867 2868 kfree(user_wr); 2869 return wr; 2870 2871 err: 2872 kfree(user_wr); 2873 2874 while (wr) { 2875 next = wr->next; 2876 kfree(wr); 2877 wr = next; 2878 } 2879 2880 return ERR_PTR(ret); 2881 } 2882 2883 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 2884 struct ib_device *ib_dev, 2885 const char __user *buf, int in_len, 2886 int out_len) 2887 { 2888 struct ib_uverbs_post_recv cmd; 2889 struct ib_uverbs_post_recv_resp resp; 2890 struct ib_recv_wr *wr, *next, *bad_wr; 2891 struct ib_qp *qp; 2892 ssize_t ret = -EINVAL; 2893 2894 if (copy_from_user(&cmd, buf, sizeof cmd)) 2895 return -EFAULT; 2896 2897 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2898 in_len - sizeof cmd, cmd.wr_count, 2899 cmd.sge_count, cmd.wqe_size); 2900 if (IS_ERR(wr)) 2901 return PTR_ERR(wr); 2902 2903 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2904 if (!qp) 2905 goto out; 2906 2907 resp.bad_wr = 0; 2908 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr); 2909 2910 put_qp_read(qp); 2911 2912 if (ret) 2913 for (next = wr; next; next = next->next) { 2914 ++resp.bad_wr; 2915 if (next == bad_wr) 2916 break; 2917 } 2918 2919 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2920 &resp, sizeof resp)) 2921 ret = -EFAULT; 2922 2923 out: 2924 while (wr) { 2925 next = wr->next; 2926 kfree(wr); 2927 wr = next; 2928 } 2929 2930 return ret ? ret : in_len; 2931 } 2932 2933 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 2934 struct ib_device *ib_dev, 2935 const char __user *buf, int in_len, 2936 int out_len) 2937 { 2938 struct ib_uverbs_post_srq_recv cmd; 2939 struct ib_uverbs_post_srq_recv_resp resp; 2940 struct ib_recv_wr *wr, *next, *bad_wr; 2941 struct ib_srq *srq; 2942 ssize_t ret = -EINVAL; 2943 2944 if (copy_from_user(&cmd, buf, sizeof cmd)) 2945 return -EFAULT; 2946 2947 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2948 in_len - sizeof cmd, cmd.wr_count, 2949 cmd.sge_count, cmd.wqe_size); 2950 if (IS_ERR(wr)) 2951 return PTR_ERR(wr); 2952 2953 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2954 if (!srq) 2955 goto out; 2956 2957 resp.bad_wr = 0; 2958 ret = srq->device->post_srq_recv(srq, wr, &bad_wr); 2959 2960 put_srq_read(srq); 2961 2962 if (ret) 2963 for (next = wr; next; next = next->next) { 2964 ++resp.bad_wr; 2965 if (next == bad_wr) 2966 break; 2967 } 2968 2969 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2970 &resp, sizeof resp)) 2971 ret = -EFAULT; 2972 2973 out: 2974 while (wr) { 2975 next = wr->next; 2976 kfree(wr); 2977 wr = next; 2978 } 2979 2980 return ret ? ret : in_len; 2981 } 2982 2983 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 2984 struct ib_device *ib_dev, 2985 const char __user *buf, int in_len, 2986 int out_len) 2987 { 2988 struct ib_uverbs_create_ah cmd; 2989 struct ib_uverbs_create_ah_resp resp; 2990 struct ib_uobject *uobj; 2991 struct ib_pd *pd; 2992 struct ib_ah *ah; 2993 struct ib_ah_attr attr; 2994 int ret; 2995 struct ib_udata udata; 2996 2997 if (out_len < sizeof resp) 2998 return -ENOSPC; 2999 3000 if (copy_from_user(&cmd, buf, sizeof cmd)) 3001 return -EFAULT; 3002 3003 INIT_UDATA(&udata, buf + sizeof(cmd), 3004 (unsigned long)cmd.response + sizeof(resp), 3005 in_len - sizeof(cmd), out_len - sizeof(resp)); 3006 3007 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 3008 if (!uobj) 3009 return -ENOMEM; 3010 3011 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class); 3012 down_write(&uobj->mutex); 3013 3014 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 3015 if (!pd) { 3016 ret = -EINVAL; 3017 goto err; 3018 } 3019 3020 attr.dlid = cmd.attr.dlid; 3021 attr.sl = cmd.attr.sl; 3022 attr.src_path_bits = cmd.attr.src_path_bits; 3023 attr.static_rate = cmd.attr.static_rate; 3024 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; 3025 attr.port_num = cmd.attr.port_num; 3026 attr.grh.flow_label = cmd.attr.grh.flow_label; 3027 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 3028 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 3029 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 3030 memset(&attr.dmac, 0, sizeof(attr.dmac)); 3031 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 3032 3033 ret = ib_rdmacg_try_charge(&uobj->cg_obj, ib_dev, 3034 RDMACG_RESOURCE_HCA_OBJECT); 3035 if (ret) 3036 goto err_charge; 3037 3038 ah = pd->device->create_ah(pd, &attr, &udata); 3039 3040 if (IS_ERR(ah)) { 3041 ret = PTR_ERR(ah); 3042 goto err_create; 3043 } 3044 3045 ah->device = pd->device; 3046 ah->pd = pd; 3047 atomic_inc(&pd->usecnt); 3048 ah->uobject = uobj; 3049 uobj->object = ah; 3050 3051 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); 3052 if (ret) 3053 goto err_destroy; 3054 3055 resp.ah_handle = uobj->id; 3056 3057 if (copy_to_user((void __user *) (unsigned long) cmd.response, 3058 &resp, sizeof resp)) { 3059 ret = -EFAULT; 3060 goto err_copy; 3061 } 3062 3063 put_pd_read(pd); 3064 3065 mutex_lock(&file->mutex); 3066 list_add_tail(&uobj->list, &file->ucontext->ah_list); 3067 mutex_unlock(&file->mutex); 3068 3069 uobj->live = 1; 3070 3071 up_write(&uobj->mutex); 3072 3073 return in_len; 3074 3075 err_copy: 3076 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 3077 3078 err_destroy: 3079 ib_destroy_ah(ah); 3080 3081 err_create: 3082 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 3083 3084 err_charge: 3085 put_pd_read(pd); 3086 3087 err: 3088 put_uobj_write(uobj); 3089 return ret; 3090 } 3091 3092 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 3093 struct ib_device *ib_dev, 3094 const char __user *buf, int in_len, int out_len) 3095 { 3096 struct ib_uverbs_destroy_ah cmd; 3097 struct ib_ah *ah; 3098 struct ib_uobject *uobj; 3099 int ret; 3100 3101 if (copy_from_user(&cmd, buf, sizeof cmd)) 3102 return -EFAULT; 3103 3104 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); 3105 if (!uobj) 3106 return -EINVAL; 3107 ah = uobj->object; 3108 3109 ret = ib_destroy_ah(ah); 3110 if (!ret) 3111 uobj->live = 0; 3112 3113 put_uobj_write(uobj); 3114 3115 if (ret) 3116 return ret; 3117 3118 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 3119 3120 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 3121 3122 mutex_lock(&file->mutex); 3123 list_del(&uobj->list); 3124 mutex_unlock(&file->mutex); 3125 3126 put_uobj(uobj); 3127 3128 return in_len; 3129 } 3130 3131 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 3132 struct ib_device *ib_dev, 3133 const char __user *buf, int in_len, 3134 int out_len) 3135 { 3136 struct ib_uverbs_attach_mcast cmd; 3137 struct ib_qp *qp; 3138 struct ib_uqp_object *obj; 3139 struct ib_uverbs_mcast_entry *mcast; 3140 int ret; 3141 3142 if (copy_from_user(&cmd, buf, sizeof cmd)) 3143 return -EFAULT; 3144 3145 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 3146 if (!qp) 3147 return -EINVAL; 3148 3149 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 3150 3151 list_for_each_entry(mcast, &obj->mcast_list, list) 3152 if (cmd.mlid == mcast->lid && 3153 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 3154 ret = 0; 3155 goto out_put; 3156 } 3157 3158 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 3159 if (!mcast) { 3160 ret = -ENOMEM; 3161 goto out_put; 3162 } 3163 3164 mcast->lid = cmd.mlid; 3165 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 3166 3167 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 3168 if (!ret) 3169 list_add_tail(&mcast->list, &obj->mcast_list); 3170 else 3171 kfree(mcast); 3172 3173 out_put: 3174 put_qp_write(qp); 3175 3176 return ret ? ret : in_len; 3177 } 3178 3179 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 3180 struct ib_device *ib_dev, 3181 const char __user *buf, int in_len, 3182 int out_len) 3183 { 3184 struct ib_uverbs_detach_mcast cmd; 3185 struct ib_uqp_object *obj; 3186 struct ib_qp *qp; 3187 struct ib_uverbs_mcast_entry *mcast; 3188 int ret = -EINVAL; 3189 3190 if (copy_from_user(&cmd, buf, sizeof cmd)) 3191 return -EFAULT; 3192 3193 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 3194 if (!qp) 3195 return -EINVAL; 3196 3197 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); 3198 if (ret) 3199 goto out_put; 3200 3201 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 3202 3203 list_for_each_entry(mcast, &obj->mcast_list, list) 3204 if (cmd.mlid == mcast->lid && 3205 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 3206 list_del(&mcast->list); 3207 kfree(mcast); 3208 break; 3209 } 3210 3211 out_put: 3212 put_qp_write(qp); 3213 3214 return ret ? ret : in_len; 3215 } 3216 3217 static int kern_spec_to_ib_spec_action(struct ib_uverbs_flow_spec *kern_spec, 3218 union ib_flow_spec *ib_spec) 3219 { 3220 ib_spec->type = kern_spec->type; 3221 switch (ib_spec->type) { 3222 case IB_FLOW_SPEC_ACTION_TAG: 3223 if (kern_spec->flow_tag.size != 3224 sizeof(struct ib_uverbs_flow_spec_action_tag)) 3225 return -EINVAL; 3226 3227 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag); 3228 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id; 3229 break; 3230 default: 3231 return -EINVAL; 3232 } 3233 return 0; 3234 } 3235 3236 static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr *spec) 3237 { 3238 /* Returns user space filter size, includes padding */ 3239 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2; 3240 } 3241 3242 static ssize_t spec_filter_size(void *kern_spec_filter, u16 kern_filter_size, 3243 u16 ib_real_filter_sz) 3244 { 3245 /* 3246 * User space filter structures must be 64 bit aligned, otherwise this 3247 * may pass, but we won't handle additional new attributes. 3248 */ 3249 3250 if (kern_filter_size > ib_real_filter_sz) { 3251 if (memchr_inv(kern_spec_filter + 3252 ib_real_filter_sz, 0, 3253 kern_filter_size - ib_real_filter_sz)) 3254 return -EINVAL; 3255 return ib_real_filter_sz; 3256 } 3257 return kern_filter_size; 3258 } 3259 3260 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec, 3261 union ib_flow_spec *ib_spec) 3262 { 3263 ssize_t actual_filter_sz; 3264 ssize_t kern_filter_sz; 3265 ssize_t ib_filter_sz; 3266 void *kern_spec_mask; 3267 void *kern_spec_val; 3268 3269 if (kern_spec->reserved) 3270 return -EINVAL; 3271 3272 ib_spec->type = kern_spec->type; 3273 3274 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr); 3275 /* User flow spec size must be aligned to 4 bytes */ 3276 if (kern_filter_sz != ALIGN(kern_filter_sz, 4)) 3277 return -EINVAL; 3278 3279 kern_spec_val = (void *)kern_spec + 3280 sizeof(struct ib_uverbs_flow_spec_hdr); 3281 kern_spec_mask = kern_spec_val + kern_filter_sz; 3282 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL)) 3283 return -EINVAL; 3284 3285 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 3286 case IB_FLOW_SPEC_ETH: 3287 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz); 3288 actual_filter_sz = spec_filter_size(kern_spec_mask, 3289 kern_filter_sz, 3290 ib_filter_sz); 3291 if (actual_filter_sz <= 0) 3292 return -EINVAL; 3293 ib_spec->size = sizeof(struct ib_flow_spec_eth); 3294 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz); 3295 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz); 3296 break; 3297 case IB_FLOW_SPEC_IPV4: 3298 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz); 3299 actual_filter_sz = spec_filter_size(kern_spec_mask, 3300 kern_filter_sz, 3301 ib_filter_sz); 3302 if (actual_filter_sz <= 0) 3303 return -EINVAL; 3304 ib_spec->size = sizeof(struct ib_flow_spec_ipv4); 3305 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz); 3306 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz); 3307 break; 3308 case IB_FLOW_SPEC_IPV6: 3309 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz); 3310 actual_filter_sz = spec_filter_size(kern_spec_mask, 3311 kern_filter_sz, 3312 ib_filter_sz); 3313 if (actual_filter_sz <= 0) 3314 return -EINVAL; 3315 ib_spec->size = sizeof(struct ib_flow_spec_ipv6); 3316 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz); 3317 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz); 3318 3319 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) || 3320 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20)) 3321 return -EINVAL; 3322 break; 3323 case IB_FLOW_SPEC_TCP: 3324 case IB_FLOW_SPEC_UDP: 3325 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz); 3326 actual_filter_sz = spec_filter_size(kern_spec_mask, 3327 kern_filter_sz, 3328 ib_filter_sz); 3329 if (actual_filter_sz <= 0) 3330 return -EINVAL; 3331 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp); 3332 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz); 3333 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz); 3334 break; 3335 case IB_FLOW_SPEC_VXLAN_TUNNEL: 3336 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz); 3337 actual_filter_sz = spec_filter_size(kern_spec_mask, 3338 kern_filter_sz, 3339 ib_filter_sz); 3340 if (actual_filter_sz <= 0) 3341 return -EINVAL; 3342 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel); 3343 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz); 3344 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz); 3345 3346 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) || 3347 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24)) 3348 return -EINVAL; 3349 break; 3350 default: 3351 return -EINVAL; 3352 } 3353 return 0; 3354 } 3355 3356 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec, 3357 union ib_flow_spec *ib_spec) 3358 { 3359 if (kern_spec->reserved) 3360 return -EINVAL; 3361 3362 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG) 3363 return kern_spec_to_ib_spec_action(kern_spec, ib_spec); 3364 else 3365 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec); 3366 } 3367 3368 int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, 3369 struct ib_device *ib_dev, 3370 struct ib_udata *ucore, 3371 struct ib_udata *uhw) 3372 { 3373 struct ib_uverbs_ex_create_wq cmd = {}; 3374 struct ib_uverbs_ex_create_wq_resp resp = {}; 3375 struct ib_uwq_object *obj; 3376 int err = 0; 3377 struct ib_cq *cq; 3378 struct ib_pd *pd; 3379 struct ib_wq *wq; 3380 struct ib_wq_init_attr wq_init_attr = {}; 3381 size_t required_cmd_sz; 3382 size_t required_resp_len; 3383 3384 required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge); 3385 required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn); 3386 3387 if (ucore->inlen < required_cmd_sz) 3388 return -EINVAL; 3389 3390 if (ucore->outlen < required_resp_len) 3391 return -ENOSPC; 3392 3393 if (ucore->inlen > sizeof(cmd) && 3394 !ib_is_udata_cleared(ucore, sizeof(cmd), 3395 ucore->inlen - sizeof(cmd))) 3396 return -EOPNOTSUPP; 3397 3398 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3399 if (err) 3400 return err; 3401 3402 if (cmd.comp_mask) 3403 return -EOPNOTSUPP; 3404 3405 obj = kmalloc(sizeof(*obj), GFP_KERNEL); 3406 if (!obj) 3407 return -ENOMEM; 3408 3409 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, 3410 &wq_lock_class); 3411 down_write(&obj->uevent.uobject.mutex); 3412 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 3413 if (!pd) { 3414 err = -EINVAL; 3415 goto err_uobj; 3416 } 3417 3418 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 3419 if (!cq) { 3420 err = -EINVAL; 3421 goto err_put_pd; 3422 } 3423 3424 wq_init_attr.cq = cq; 3425 wq_init_attr.max_sge = cmd.max_sge; 3426 wq_init_attr.max_wr = cmd.max_wr; 3427 wq_init_attr.wq_context = file; 3428 wq_init_attr.wq_type = cmd.wq_type; 3429 wq_init_attr.event_handler = ib_uverbs_wq_event_handler; 3430 if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) + 3431 sizeof(cmd.create_flags))) 3432 wq_init_attr.create_flags = cmd.create_flags; 3433 obj->uevent.events_reported = 0; 3434 INIT_LIST_HEAD(&obj->uevent.event_list); 3435 wq = pd->device->create_wq(pd, &wq_init_attr, uhw); 3436 if (IS_ERR(wq)) { 3437 err = PTR_ERR(wq); 3438 goto err_put_cq; 3439 } 3440 3441 wq->uobject = &obj->uevent.uobject; 3442 obj->uevent.uobject.object = wq; 3443 wq->wq_type = wq_init_attr.wq_type; 3444 wq->cq = cq; 3445 wq->pd = pd; 3446 wq->device = pd->device; 3447 wq->wq_context = wq_init_attr.wq_context; 3448 atomic_set(&wq->usecnt, 0); 3449 atomic_inc(&pd->usecnt); 3450 atomic_inc(&cq->usecnt); 3451 wq->uobject = &obj->uevent.uobject; 3452 obj->uevent.uobject.object = wq; 3453 err = idr_add_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); 3454 if (err) 3455 goto destroy_wq; 3456 3457 memset(&resp, 0, sizeof(resp)); 3458 resp.wq_handle = obj->uevent.uobject.id; 3459 resp.max_sge = wq_init_attr.max_sge; 3460 resp.max_wr = wq_init_attr.max_wr; 3461 resp.wqn = wq->wq_num; 3462 resp.response_length = required_resp_len; 3463 err = ib_copy_to_udata(ucore, 3464 &resp, resp.response_length); 3465 if (err) 3466 goto err_copy; 3467 3468 put_pd_read(pd); 3469 put_cq_read(cq); 3470 3471 mutex_lock(&file->mutex); 3472 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->wq_list); 3473 mutex_unlock(&file->mutex); 3474 3475 obj->uevent.uobject.live = 1; 3476 up_write(&obj->uevent.uobject.mutex); 3477 return 0; 3478 3479 err_copy: 3480 idr_remove_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); 3481 destroy_wq: 3482 ib_destroy_wq(wq); 3483 err_put_cq: 3484 put_cq_read(cq); 3485 err_put_pd: 3486 put_pd_read(pd); 3487 err_uobj: 3488 put_uobj_write(&obj->uevent.uobject); 3489 3490 return err; 3491 } 3492 3493 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file, 3494 struct ib_device *ib_dev, 3495 struct ib_udata *ucore, 3496 struct ib_udata *uhw) 3497 { 3498 struct ib_uverbs_ex_destroy_wq cmd = {}; 3499 struct ib_uverbs_ex_destroy_wq_resp resp = {}; 3500 struct ib_wq *wq; 3501 struct ib_uobject *uobj; 3502 struct ib_uwq_object *obj; 3503 size_t required_cmd_sz; 3504 size_t required_resp_len; 3505 int ret; 3506 3507 required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle); 3508 required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); 3509 3510 if (ucore->inlen < required_cmd_sz) 3511 return -EINVAL; 3512 3513 if (ucore->outlen < required_resp_len) 3514 return -ENOSPC; 3515 3516 if (ucore->inlen > sizeof(cmd) && 3517 !ib_is_udata_cleared(ucore, sizeof(cmd), 3518 ucore->inlen - sizeof(cmd))) 3519 return -EOPNOTSUPP; 3520 3521 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3522 if (ret) 3523 return ret; 3524 3525 if (cmd.comp_mask) 3526 return -EOPNOTSUPP; 3527 3528 resp.response_length = required_resp_len; 3529 uobj = idr_write_uobj(&ib_uverbs_wq_idr, cmd.wq_handle, 3530 file->ucontext); 3531 if (!uobj) 3532 return -EINVAL; 3533 3534 wq = uobj->object; 3535 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); 3536 ret = ib_destroy_wq(wq); 3537 if (!ret) 3538 uobj->live = 0; 3539 3540 put_uobj_write(uobj); 3541 if (ret) 3542 return ret; 3543 3544 idr_remove_uobj(&ib_uverbs_wq_idr, uobj); 3545 3546 mutex_lock(&file->mutex); 3547 list_del(&uobj->list); 3548 mutex_unlock(&file->mutex); 3549 3550 ib_uverbs_release_uevent(file, &obj->uevent); 3551 resp.events_reported = obj->uevent.events_reported; 3552 put_uobj(uobj); 3553 3554 ret = ib_copy_to_udata(ucore, &resp, resp.response_length); 3555 if (ret) 3556 return ret; 3557 3558 return 0; 3559 } 3560 3561 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file, 3562 struct ib_device *ib_dev, 3563 struct ib_udata *ucore, 3564 struct ib_udata *uhw) 3565 { 3566 struct ib_uverbs_ex_modify_wq cmd = {}; 3567 struct ib_wq *wq; 3568 struct ib_wq_attr wq_attr = {}; 3569 size_t required_cmd_sz; 3570 int ret; 3571 3572 required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state); 3573 if (ucore->inlen < required_cmd_sz) 3574 return -EINVAL; 3575 3576 if (ucore->inlen > sizeof(cmd) && 3577 !ib_is_udata_cleared(ucore, sizeof(cmd), 3578 ucore->inlen - sizeof(cmd))) 3579 return -EOPNOTSUPP; 3580 3581 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3582 if (ret) 3583 return ret; 3584 3585 if (!cmd.attr_mask) 3586 return -EINVAL; 3587 3588 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS)) 3589 return -EINVAL; 3590 3591 wq = idr_read_wq(cmd.wq_handle, file->ucontext); 3592 if (!wq) 3593 return -EINVAL; 3594 3595 wq_attr.curr_wq_state = cmd.curr_wq_state; 3596 wq_attr.wq_state = cmd.wq_state; 3597 if (cmd.attr_mask & IB_WQ_FLAGS) { 3598 wq_attr.flags = cmd.flags; 3599 wq_attr.flags_mask = cmd.flags_mask; 3600 } 3601 ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw); 3602 put_wq_read(wq); 3603 return ret; 3604 } 3605 3606 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, 3607 struct ib_device *ib_dev, 3608 struct ib_udata *ucore, 3609 struct ib_udata *uhw) 3610 { 3611 struct ib_uverbs_ex_create_rwq_ind_table cmd = {}; 3612 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; 3613 struct ib_uobject *uobj; 3614 int err = 0; 3615 struct ib_rwq_ind_table_init_attr init_attr = {}; 3616 struct ib_rwq_ind_table *rwq_ind_tbl; 3617 struct ib_wq **wqs = NULL; 3618 u32 *wqs_handles = NULL; 3619 struct ib_wq *wq = NULL; 3620 int i, j, num_read_wqs; 3621 u32 num_wq_handles; 3622 u32 expected_in_size; 3623 size_t required_cmd_sz_header; 3624 size_t required_resp_len; 3625 3626 required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size); 3627 required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num); 3628 3629 if (ucore->inlen < required_cmd_sz_header) 3630 return -EINVAL; 3631 3632 if (ucore->outlen < required_resp_len) 3633 return -ENOSPC; 3634 3635 err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header); 3636 if (err) 3637 return err; 3638 3639 ucore->inbuf += required_cmd_sz_header; 3640 ucore->inlen -= required_cmd_sz_header; 3641 3642 if (cmd.comp_mask) 3643 return -EOPNOTSUPP; 3644 3645 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE) 3646 return -EINVAL; 3647 3648 num_wq_handles = 1 << cmd.log_ind_tbl_size; 3649 expected_in_size = num_wq_handles * sizeof(__u32); 3650 if (num_wq_handles == 1) 3651 /* input size for wq handles is u64 aligned */ 3652 expected_in_size += sizeof(__u32); 3653 3654 if (ucore->inlen < expected_in_size) 3655 return -EINVAL; 3656 3657 if (ucore->inlen > expected_in_size && 3658 !ib_is_udata_cleared(ucore, expected_in_size, 3659 ucore->inlen - expected_in_size)) 3660 return -EOPNOTSUPP; 3661 3662 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles), 3663 GFP_KERNEL); 3664 if (!wqs_handles) 3665 return -ENOMEM; 3666 3667 err = ib_copy_from_udata(wqs_handles, ucore, 3668 num_wq_handles * sizeof(__u32)); 3669 if (err) 3670 goto err_free; 3671 3672 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL); 3673 if (!wqs) { 3674 err = -ENOMEM; 3675 goto err_free; 3676 } 3677 3678 for (num_read_wqs = 0; num_read_wqs < num_wq_handles; 3679 num_read_wqs++) { 3680 wq = idr_read_wq(wqs_handles[num_read_wqs], file->ucontext); 3681 if (!wq) { 3682 err = -EINVAL; 3683 goto put_wqs; 3684 } 3685 3686 wqs[num_read_wqs] = wq; 3687 } 3688 3689 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 3690 if (!uobj) { 3691 err = -ENOMEM; 3692 goto put_wqs; 3693 } 3694 3695 init_uobj(uobj, 0, file->ucontext, &rwq_ind_table_lock_class); 3696 down_write(&uobj->mutex); 3697 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; 3698 init_attr.ind_tbl = wqs; 3699 rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw); 3700 3701 if (IS_ERR(rwq_ind_tbl)) { 3702 err = PTR_ERR(rwq_ind_tbl); 3703 goto err_uobj; 3704 } 3705 3706 rwq_ind_tbl->ind_tbl = wqs; 3707 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; 3708 rwq_ind_tbl->uobject = uobj; 3709 uobj->object = rwq_ind_tbl; 3710 rwq_ind_tbl->device = ib_dev; 3711 atomic_set(&rwq_ind_tbl->usecnt, 0); 3712 3713 for (i = 0; i < num_wq_handles; i++) 3714 atomic_inc(&wqs[i]->usecnt); 3715 3716 err = idr_add_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3717 if (err) 3718 goto destroy_ind_tbl; 3719 3720 resp.ind_tbl_handle = uobj->id; 3721 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; 3722 resp.response_length = required_resp_len; 3723 3724 err = ib_copy_to_udata(ucore, 3725 &resp, resp.response_length); 3726 if (err) 3727 goto err_copy; 3728 3729 kfree(wqs_handles); 3730 3731 for (j = 0; j < num_read_wqs; j++) 3732 put_wq_read(wqs[j]); 3733 3734 mutex_lock(&file->mutex); 3735 list_add_tail(&uobj->list, &file->ucontext->rwq_ind_tbl_list); 3736 mutex_unlock(&file->mutex); 3737 3738 uobj->live = 1; 3739 3740 up_write(&uobj->mutex); 3741 return 0; 3742 3743 err_copy: 3744 idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3745 destroy_ind_tbl: 3746 ib_destroy_rwq_ind_table(rwq_ind_tbl); 3747 err_uobj: 3748 put_uobj_write(uobj); 3749 put_wqs: 3750 for (j = 0; j < num_read_wqs; j++) 3751 put_wq_read(wqs[j]); 3752 err_free: 3753 kfree(wqs_handles); 3754 kfree(wqs); 3755 return err; 3756 } 3757 3758 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file, 3759 struct ib_device *ib_dev, 3760 struct ib_udata *ucore, 3761 struct ib_udata *uhw) 3762 { 3763 struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {}; 3764 struct ib_rwq_ind_table *rwq_ind_tbl; 3765 struct ib_uobject *uobj; 3766 int ret; 3767 struct ib_wq **ind_tbl; 3768 size_t required_cmd_sz; 3769 3770 required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle); 3771 3772 if (ucore->inlen < required_cmd_sz) 3773 return -EINVAL; 3774 3775 if (ucore->inlen > sizeof(cmd) && 3776 !ib_is_udata_cleared(ucore, sizeof(cmd), 3777 ucore->inlen - sizeof(cmd))) 3778 return -EOPNOTSUPP; 3779 3780 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3781 if (ret) 3782 return ret; 3783 3784 if (cmd.comp_mask) 3785 return -EOPNOTSUPP; 3786 3787 uobj = idr_write_uobj(&ib_uverbs_rwq_ind_tbl_idr, cmd.ind_tbl_handle, 3788 file->ucontext); 3789 if (!uobj) 3790 return -EINVAL; 3791 rwq_ind_tbl = uobj->object; 3792 ind_tbl = rwq_ind_tbl->ind_tbl; 3793 3794 ret = ib_destroy_rwq_ind_table(rwq_ind_tbl); 3795 if (!ret) 3796 uobj->live = 0; 3797 3798 put_uobj_write(uobj); 3799 3800 if (ret) 3801 return ret; 3802 3803 idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3804 3805 mutex_lock(&file->mutex); 3806 list_del(&uobj->list); 3807 mutex_unlock(&file->mutex); 3808 3809 put_uobj(uobj); 3810 kfree(ind_tbl); 3811 return ret; 3812 } 3813 3814 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, 3815 struct ib_device *ib_dev, 3816 struct ib_udata *ucore, 3817 struct ib_udata *uhw) 3818 { 3819 struct ib_uverbs_create_flow cmd; 3820 struct ib_uverbs_create_flow_resp resp; 3821 struct ib_uobject *uobj; 3822 struct ib_flow *flow_id; 3823 struct ib_uverbs_flow_attr *kern_flow_attr; 3824 struct ib_flow_attr *flow_attr; 3825 struct ib_qp *qp; 3826 int err = 0; 3827 void *kern_spec; 3828 void *ib_spec; 3829 int i; 3830 3831 if (ucore->inlen < sizeof(cmd)) 3832 return -EINVAL; 3833 3834 if (ucore->outlen < sizeof(resp)) 3835 return -ENOSPC; 3836 3837 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3838 if (err) 3839 return err; 3840 3841 ucore->inbuf += sizeof(cmd); 3842 ucore->inlen -= sizeof(cmd); 3843 3844 if (cmd.comp_mask) 3845 return -EINVAL; 3846 3847 if (!capable(CAP_NET_RAW)) 3848 return -EPERM; 3849 3850 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED) 3851 return -EINVAL; 3852 3853 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && 3854 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) || 3855 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT))) 3856 return -EINVAL; 3857 3858 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 3859 return -EINVAL; 3860 3861 if (cmd.flow_attr.size > ucore->inlen || 3862 cmd.flow_attr.size > 3863 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 3864 return -EINVAL; 3865 3866 if (cmd.flow_attr.reserved[0] || 3867 cmd.flow_attr.reserved[1]) 3868 return -EINVAL; 3869 3870 if (cmd.flow_attr.num_of_specs) { 3871 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 3872 GFP_KERNEL); 3873 if (!kern_flow_attr) 3874 return -ENOMEM; 3875 3876 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr)); 3877 err = ib_copy_from_udata(kern_flow_attr + 1, ucore, 3878 cmd.flow_attr.size); 3879 if (err) 3880 goto err_free_attr; 3881 } else { 3882 kern_flow_attr = &cmd.flow_attr; 3883 } 3884 3885 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 3886 if (!uobj) { 3887 err = -ENOMEM; 3888 goto err_free_attr; 3889 } 3890 init_uobj(uobj, 0, file->ucontext, &rule_lock_class); 3891 down_write(&uobj->mutex); 3892 3893 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 3894 if (!qp) { 3895 err = -EINVAL; 3896 goto err_uobj; 3897 } 3898 3899 flow_attr = kzalloc(sizeof(*flow_attr) + cmd.flow_attr.num_of_specs * 3900 sizeof(union ib_flow_spec), GFP_KERNEL); 3901 if (!flow_attr) { 3902 err = -ENOMEM; 3903 goto err_put; 3904 } 3905 3906 flow_attr->type = kern_flow_attr->type; 3907 flow_attr->priority = kern_flow_attr->priority; 3908 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 3909 flow_attr->port = kern_flow_attr->port; 3910 flow_attr->flags = kern_flow_attr->flags; 3911 flow_attr->size = sizeof(*flow_attr); 3912 3913 kern_spec = kern_flow_attr + 1; 3914 ib_spec = flow_attr + 1; 3915 for (i = 0; i < flow_attr->num_of_specs && 3916 cmd.flow_attr.size > offsetof(struct ib_uverbs_flow_spec, reserved) && 3917 cmd.flow_attr.size >= 3918 ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) { 3919 err = kern_spec_to_ib_spec(kern_spec, ib_spec); 3920 if (err) 3921 goto err_free; 3922 flow_attr->size += 3923 ((union ib_flow_spec *) ib_spec)->size; 3924 cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size; 3925 kern_spec += ((struct ib_uverbs_flow_spec *) kern_spec)->size; 3926 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 3927 } 3928 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 3929 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 3930 i, cmd.flow_attr.size); 3931 err = -EINVAL; 3932 goto err_free; 3933 } 3934 3935 err = ib_rdmacg_try_charge(&uobj->cg_obj, ib_dev, 3936 RDMACG_RESOURCE_HCA_OBJECT); 3937 if (err) 3938 goto err_free; 3939 3940 flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER); 3941 if (IS_ERR(flow_id)) { 3942 err = PTR_ERR(flow_id); 3943 goto err_create; 3944 } 3945 flow_id->uobject = uobj; 3946 uobj->object = flow_id; 3947 3948 err = idr_add_uobj(&ib_uverbs_rule_idr, uobj); 3949 if (err) 3950 goto destroy_flow; 3951 3952 memset(&resp, 0, sizeof(resp)); 3953 resp.flow_handle = uobj->id; 3954 3955 err = ib_copy_to_udata(ucore, 3956 &resp, sizeof(resp)); 3957 if (err) 3958 goto err_copy; 3959 3960 put_qp_read(qp); 3961 mutex_lock(&file->mutex); 3962 list_add_tail(&uobj->list, &file->ucontext->rule_list); 3963 mutex_unlock(&file->mutex); 3964 3965 uobj->live = 1; 3966 3967 up_write(&uobj->mutex); 3968 kfree(flow_attr); 3969 if (cmd.flow_attr.num_of_specs) 3970 kfree(kern_flow_attr); 3971 return 0; 3972 err_copy: 3973 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 3974 destroy_flow: 3975 ib_destroy_flow(flow_id); 3976 err_create: 3977 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 3978 err_free: 3979 kfree(flow_attr); 3980 err_put: 3981 put_qp_read(qp); 3982 err_uobj: 3983 put_uobj_write(uobj); 3984 err_free_attr: 3985 if (cmd.flow_attr.num_of_specs) 3986 kfree(kern_flow_attr); 3987 return err; 3988 } 3989 3990 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file, 3991 struct ib_device *ib_dev, 3992 struct ib_udata *ucore, 3993 struct ib_udata *uhw) 3994 { 3995 struct ib_uverbs_destroy_flow cmd; 3996 struct ib_flow *flow_id; 3997 struct ib_uobject *uobj; 3998 int ret; 3999 4000 if (ucore->inlen < sizeof(cmd)) 4001 return -EINVAL; 4002 4003 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 4004 if (ret) 4005 return ret; 4006 4007 if (cmd.comp_mask) 4008 return -EINVAL; 4009 4010 uobj = idr_write_uobj(&ib_uverbs_rule_idr, cmd.flow_handle, 4011 file->ucontext); 4012 if (!uobj) 4013 return -EINVAL; 4014 flow_id = uobj->object; 4015 4016 ret = ib_destroy_flow(flow_id); 4017 if (!ret) { 4018 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, 4019 RDMACG_RESOURCE_HCA_OBJECT); 4020 uobj->live = 0; 4021 } 4022 4023 put_uobj_write(uobj); 4024 4025 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 4026 4027 mutex_lock(&file->mutex); 4028 list_del(&uobj->list); 4029 mutex_unlock(&file->mutex); 4030 4031 put_uobj(uobj); 4032 4033 return ret; 4034 } 4035 4036 static int __uverbs_create_xsrq(struct ib_uverbs_file *file, 4037 struct ib_device *ib_dev, 4038 struct ib_uverbs_create_xsrq *cmd, 4039 struct ib_udata *udata) 4040 { 4041 struct ib_uverbs_create_srq_resp resp; 4042 struct ib_usrq_object *obj; 4043 struct ib_pd *pd; 4044 struct ib_srq *srq; 4045 struct ib_uobject *uninitialized_var(xrcd_uobj); 4046 struct ib_srq_init_attr attr; 4047 int ret; 4048 4049 obj = kmalloc(sizeof *obj, GFP_KERNEL); 4050 if (!obj) 4051 return -ENOMEM; 4052 4053 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class); 4054 down_write(&obj->uevent.uobject.mutex); 4055 4056 if (cmd->srq_type == IB_SRQT_XRC) { 4057 attr.ext.xrc.xrcd = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj); 4058 if (!attr.ext.xrc.xrcd) { 4059 ret = -EINVAL; 4060 goto err; 4061 } 4062 4063 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 4064 atomic_inc(&obj->uxrcd->refcnt); 4065 4066 attr.ext.xrc.cq = idr_read_cq(cmd->cq_handle, file->ucontext, 0); 4067 if (!attr.ext.xrc.cq) { 4068 ret = -EINVAL; 4069 goto err_put_xrcd; 4070 } 4071 } 4072 4073 pd = idr_read_pd(cmd->pd_handle, file->ucontext); 4074 if (!pd) { 4075 ret = -EINVAL; 4076 goto err_put_cq; 4077 } 4078 4079 attr.event_handler = ib_uverbs_srq_event_handler; 4080 attr.srq_context = file; 4081 attr.srq_type = cmd->srq_type; 4082 attr.attr.max_wr = cmd->max_wr; 4083 attr.attr.max_sge = cmd->max_sge; 4084 attr.attr.srq_limit = cmd->srq_limit; 4085 4086 obj->uevent.events_reported = 0; 4087 INIT_LIST_HEAD(&obj->uevent.event_list); 4088 4089 ret = ib_rdmacg_try_charge(&obj->uevent.uobject.cg_obj, ib_dev, 4090 RDMACG_RESOURCE_HCA_OBJECT); 4091 if (ret) 4092 goto err_put_cq; 4093 4094 srq = pd->device->create_srq(pd, &attr, udata); 4095 if (IS_ERR(srq)) { 4096 ret = PTR_ERR(srq); 4097 goto err_put; 4098 } 4099 4100 srq->device = pd->device; 4101 srq->pd = pd; 4102 srq->srq_type = cmd->srq_type; 4103 srq->uobject = &obj->uevent.uobject; 4104 srq->event_handler = attr.event_handler; 4105 srq->srq_context = attr.srq_context; 4106 4107 if (cmd->srq_type == IB_SRQT_XRC) { 4108 srq->ext.xrc.cq = attr.ext.xrc.cq; 4109 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 4110 atomic_inc(&attr.ext.xrc.cq->usecnt); 4111 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 4112 } 4113 4114 atomic_inc(&pd->usecnt); 4115 atomic_set(&srq->usecnt, 0); 4116 4117 obj->uevent.uobject.object = srq; 4118 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 4119 if (ret) 4120 goto err_destroy; 4121 4122 memset(&resp, 0, sizeof resp); 4123 resp.srq_handle = obj->uevent.uobject.id; 4124 resp.max_wr = attr.attr.max_wr; 4125 resp.max_sge = attr.attr.max_sge; 4126 if (cmd->srq_type == IB_SRQT_XRC) 4127 resp.srqn = srq->ext.xrc.srq_num; 4128 4129 if (copy_to_user((void __user *) (unsigned long) cmd->response, 4130 &resp, sizeof resp)) { 4131 ret = -EFAULT; 4132 goto err_copy; 4133 } 4134 4135 if (cmd->srq_type == IB_SRQT_XRC) { 4136 put_uobj_read(xrcd_uobj); 4137 put_cq_read(attr.ext.xrc.cq); 4138 } 4139 put_pd_read(pd); 4140 4141 mutex_lock(&file->mutex); 4142 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->srq_list); 4143 mutex_unlock(&file->mutex); 4144 4145 obj->uevent.uobject.live = 1; 4146 4147 up_write(&obj->uevent.uobject.mutex); 4148 4149 return 0; 4150 4151 err_copy: 4152 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 4153 4154 err_destroy: 4155 ib_destroy_srq(srq); 4156 4157 err_put: 4158 ib_rdmacg_uncharge(&obj->uevent.uobject.cg_obj, ib_dev, 4159 RDMACG_RESOURCE_HCA_OBJECT); 4160 put_pd_read(pd); 4161 4162 err_put_cq: 4163 if (cmd->srq_type == IB_SRQT_XRC) 4164 put_cq_read(attr.ext.xrc.cq); 4165 4166 err_put_xrcd: 4167 if (cmd->srq_type == IB_SRQT_XRC) { 4168 atomic_dec(&obj->uxrcd->refcnt); 4169 put_uobj_read(xrcd_uobj); 4170 } 4171 4172 err: 4173 put_uobj_write(&obj->uevent.uobject); 4174 return ret; 4175 } 4176 4177 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 4178 struct ib_device *ib_dev, 4179 const char __user *buf, int in_len, 4180 int out_len) 4181 { 4182 struct ib_uverbs_create_srq cmd; 4183 struct ib_uverbs_create_xsrq xcmd; 4184 struct ib_uverbs_create_srq_resp resp; 4185 struct ib_udata udata; 4186 int ret; 4187 4188 if (out_len < sizeof resp) 4189 return -ENOSPC; 4190 4191 if (copy_from_user(&cmd, buf, sizeof cmd)) 4192 return -EFAULT; 4193 4194 xcmd.response = cmd.response; 4195 xcmd.user_handle = cmd.user_handle; 4196 xcmd.srq_type = IB_SRQT_BASIC; 4197 xcmd.pd_handle = cmd.pd_handle; 4198 xcmd.max_wr = cmd.max_wr; 4199 xcmd.max_sge = cmd.max_sge; 4200 xcmd.srq_limit = cmd.srq_limit; 4201 4202 INIT_UDATA(&udata, buf + sizeof cmd, 4203 (unsigned long) cmd.response + sizeof resp, 4204 in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr), 4205 out_len - sizeof resp); 4206 4207 ret = __uverbs_create_xsrq(file, ib_dev, &xcmd, &udata); 4208 if (ret) 4209 return ret; 4210 4211 return in_len; 4212 } 4213 4214 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file, 4215 struct ib_device *ib_dev, 4216 const char __user *buf, int in_len, int out_len) 4217 { 4218 struct ib_uverbs_create_xsrq cmd; 4219 struct ib_uverbs_create_srq_resp resp; 4220 struct ib_udata udata; 4221 int ret; 4222 4223 if (out_len < sizeof resp) 4224 return -ENOSPC; 4225 4226 if (copy_from_user(&cmd, buf, sizeof cmd)) 4227 return -EFAULT; 4228 4229 INIT_UDATA(&udata, buf + sizeof cmd, 4230 (unsigned long) cmd.response + sizeof resp, 4231 in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr), 4232 out_len - sizeof resp); 4233 4234 ret = __uverbs_create_xsrq(file, ib_dev, &cmd, &udata); 4235 if (ret) 4236 return ret; 4237 4238 return in_len; 4239 } 4240 4241 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 4242 struct ib_device *ib_dev, 4243 const char __user *buf, int in_len, 4244 int out_len) 4245 { 4246 struct ib_uverbs_modify_srq cmd; 4247 struct ib_udata udata; 4248 struct ib_srq *srq; 4249 struct ib_srq_attr attr; 4250 int ret; 4251 4252 if (copy_from_user(&cmd, buf, sizeof cmd)) 4253 return -EFAULT; 4254 4255 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 4256 out_len); 4257 4258 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 4259 if (!srq) 4260 return -EINVAL; 4261 4262 attr.max_wr = cmd.max_wr; 4263 attr.srq_limit = cmd.srq_limit; 4264 4265 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 4266 4267 put_srq_read(srq); 4268 4269 return ret ? ret : in_len; 4270 } 4271 4272 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 4273 struct ib_device *ib_dev, 4274 const char __user *buf, 4275 int in_len, int out_len) 4276 { 4277 struct ib_uverbs_query_srq cmd; 4278 struct ib_uverbs_query_srq_resp resp; 4279 struct ib_srq_attr attr; 4280 struct ib_srq *srq; 4281 int ret; 4282 4283 if (out_len < sizeof resp) 4284 return -ENOSPC; 4285 4286 if (copy_from_user(&cmd, buf, sizeof cmd)) 4287 return -EFAULT; 4288 4289 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 4290 if (!srq) 4291 return -EINVAL; 4292 4293 ret = ib_query_srq(srq, &attr); 4294 4295 put_srq_read(srq); 4296 4297 if (ret) 4298 return ret; 4299 4300 memset(&resp, 0, sizeof resp); 4301 4302 resp.max_wr = attr.max_wr; 4303 resp.max_sge = attr.max_sge; 4304 resp.srq_limit = attr.srq_limit; 4305 4306 if (copy_to_user((void __user *) (unsigned long) cmd.response, 4307 &resp, sizeof resp)) 4308 return -EFAULT; 4309 4310 return in_len; 4311 } 4312 4313 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 4314 struct ib_device *ib_dev, 4315 const char __user *buf, int in_len, 4316 int out_len) 4317 { 4318 struct ib_uverbs_destroy_srq cmd; 4319 struct ib_uverbs_destroy_srq_resp resp; 4320 struct ib_uobject *uobj; 4321 struct ib_srq *srq; 4322 struct ib_uevent_object *obj; 4323 int ret = -EINVAL; 4324 struct ib_usrq_object *us; 4325 enum ib_srq_type srq_type; 4326 4327 if (copy_from_user(&cmd, buf, sizeof cmd)) 4328 return -EFAULT; 4329 4330 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); 4331 if (!uobj) 4332 return -EINVAL; 4333 srq = uobj->object; 4334 obj = container_of(uobj, struct ib_uevent_object, uobject); 4335 srq_type = srq->srq_type; 4336 4337 ret = ib_destroy_srq(srq); 4338 if (!ret) 4339 uobj->live = 0; 4340 4341 put_uobj_write(uobj); 4342 4343 if (ret) 4344 return ret; 4345 4346 ib_rdmacg_uncharge(&uobj->cg_obj, ib_dev, RDMACG_RESOURCE_HCA_OBJECT); 4347 4348 if (srq_type == IB_SRQT_XRC) { 4349 us = container_of(obj, struct ib_usrq_object, uevent); 4350 atomic_dec(&us->uxrcd->refcnt); 4351 } 4352 4353 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 4354 4355 mutex_lock(&file->mutex); 4356 list_del(&uobj->list); 4357 mutex_unlock(&file->mutex); 4358 4359 ib_uverbs_release_uevent(file, obj); 4360 4361 memset(&resp, 0, sizeof resp); 4362 resp.events_reported = obj->events_reported; 4363 4364 put_uobj(uobj); 4365 4366 if (copy_to_user((void __user *) (unsigned long) cmd.response, 4367 &resp, sizeof resp)) 4368 ret = -EFAULT; 4369 4370 return ret ? ret : in_len; 4371 } 4372 4373 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, 4374 struct ib_device *ib_dev, 4375 struct ib_udata *ucore, 4376 struct ib_udata *uhw) 4377 { 4378 struct ib_uverbs_ex_query_device_resp resp = { {0} }; 4379 struct ib_uverbs_ex_query_device cmd; 4380 struct ib_device_attr attr = {0}; 4381 int err; 4382 4383 if (ucore->inlen < sizeof(cmd)) 4384 return -EINVAL; 4385 4386 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 4387 if (err) 4388 return err; 4389 4390 if (cmd.comp_mask) 4391 return -EINVAL; 4392 4393 if (cmd.reserved) 4394 return -EINVAL; 4395 4396 resp.response_length = offsetof(typeof(resp), odp_caps); 4397 4398 if (ucore->outlen < resp.response_length) 4399 return -ENOSPC; 4400 4401 err = ib_dev->query_device(ib_dev, &attr, uhw); 4402 if (err) 4403 return err; 4404 4405 copy_query_dev_fields(file, ib_dev, &resp.base, &attr); 4406 4407 if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps)) 4408 goto end; 4409 4410 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 4411 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 4412 resp.odp_caps.per_transport_caps.rc_odp_caps = 4413 attr.odp_caps.per_transport_caps.rc_odp_caps; 4414 resp.odp_caps.per_transport_caps.uc_odp_caps = 4415 attr.odp_caps.per_transport_caps.uc_odp_caps; 4416 resp.odp_caps.per_transport_caps.ud_odp_caps = 4417 attr.odp_caps.per_transport_caps.ud_odp_caps; 4418 #endif 4419 resp.response_length += sizeof(resp.odp_caps); 4420 4421 if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask)) 4422 goto end; 4423 4424 resp.timestamp_mask = attr.timestamp_mask; 4425 resp.response_length += sizeof(resp.timestamp_mask); 4426 4427 if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock)) 4428 goto end; 4429 4430 resp.hca_core_clock = attr.hca_core_clock; 4431 resp.response_length += sizeof(resp.hca_core_clock); 4432 4433 if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex)) 4434 goto end; 4435 4436 resp.device_cap_flags_ex = attr.device_cap_flags; 4437 resp.response_length += sizeof(resp.device_cap_flags_ex); 4438 4439 if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps)) 4440 goto end; 4441 4442 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; 4443 resp.rss_caps.max_rwq_indirection_tables = 4444 attr.rss_caps.max_rwq_indirection_tables; 4445 resp.rss_caps.max_rwq_indirection_table_size = 4446 attr.rss_caps.max_rwq_indirection_table_size; 4447 4448 resp.response_length += sizeof(resp.rss_caps); 4449 4450 if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq)) 4451 goto end; 4452 4453 resp.max_wq_type_rq = attr.max_wq_type_rq; 4454 resp.response_length += sizeof(resp.max_wq_type_rq); 4455 4456 if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps)) 4457 goto end; 4458 4459 resp.raw_packet_caps = attr.raw_packet_caps; 4460 resp.response_length += sizeof(resp.raw_packet_caps); 4461 end: 4462 err = ib_copy_to_udata(ucore, &resp, resp.response_length); 4463 return err; 4464 } 4465