1 /* 2 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/rdma_user_ioctl.h> 34 #include <rdma/uverbs_ioctl.h> 35 #include "rdma_core.h" 36 #include "uverbs.h" 37 38 struct bundle_alloc_head { 39 struct bundle_alloc_head *next; 40 u8 data[]; 41 }; 42 43 struct bundle_priv { 44 /* Must be first */ 45 struct bundle_alloc_head alloc_head; 46 struct bundle_alloc_head *allocated_mem; 47 size_t internal_avail; 48 size_t internal_used; 49 50 struct radix_tree_root *radix; 51 const struct uverbs_api_ioctl_method *method_elm; 52 void __rcu **radix_slots; 53 unsigned long radix_slots_len; 54 u32 method_key; 55 56 struct ib_uverbs_attr __user *user_attrs; 57 struct ib_uverbs_attr *uattrs; 58 59 DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN); 60 DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN); 61 DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN); 62 63 /* 64 * Must be last. bundle ends in a flex array which overlaps 65 * internal_buffer. 66 */ 67 struct uverbs_attr_bundle bundle; 68 u64 internal_buffer[32]; 69 }; 70 71 /* 72 * Each method has an absolute minimum amount of memory it needs to allocate, 73 * precompute that amount and determine if the onstack memory can be used or 74 * if allocation is need. 75 */ 76 void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, 77 unsigned int num_attrs) 78 { 79 struct bundle_priv *pbundle; 80 size_t bundle_size = 81 offsetof(struct bundle_priv, internal_buffer) + 82 sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len + 83 sizeof(*pbundle->uattrs) * num_attrs; 84 85 method_elm->use_stack = bundle_size <= sizeof(*pbundle); 86 method_elm->bundle_size = 87 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer)); 88 89 /* Do not want order-2 allocations for this. */ 90 WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE); 91 } 92 93 /** 94 * _uverbs_alloc() - Quickly allocate memory for use with a bundle 95 * @bundle: The bundle 96 * @size: Number of bytes to allocate 97 * @flags: Allocator flags 98 * 99 * The bundle allocator is intended for allocations that are connected with 100 * processing the system call related to the bundle. The allocated memory is 101 * always freed once the system call completes, and cannot be freed any other 102 * way. 103 * 104 * This tries to use a small pool of pre-allocated memory for performance. 105 */ 106 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, 107 gfp_t flags) 108 { 109 struct bundle_priv *pbundle = 110 container_of(bundle, struct bundle_priv, bundle); 111 size_t new_used; 112 void *res; 113 114 if (check_add_overflow(size, pbundle->internal_used, &new_used)) 115 return ERR_PTR(-EOVERFLOW); 116 117 if (new_used > pbundle->internal_avail) { 118 struct bundle_alloc_head *buf; 119 120 buf = kvmalloc(struct_size(buf, data, size), flags); 121 if (!buf) 122 return ERR_PTR(-ENOMEM); 123 buf->next = pbundle->allocated_mem; 124 pbundle->allocated_mem = buf; 125 return buf->data; 126 } 127 128 res = (void *)pbundle->internal_buffer + pbundle->internal_used; 129 pbundle->internal_used = 130 ALIGN(new_used, sizeof(*pbundle->internal_buffer)); 131 if (want_init_on_alloc(flags)) 132 memset(res, 0, size); 133 return res; 134 } 135 EXPORT_SYMBOL(_uverbs_alloc); 136 137 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, 138 u16 len) 139 { 140 if (uattr->len > sizeof_field(struct ib_uverbs_attr, data)) 141 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len, 142 uattr->len - len); 143 144 return !memchr_inv((const void *)&uattr->data + len, 145 0, uattr->len - len); 146 } 147 148 static int uverbs_set_output(const struct uverbs_attr_bundle *bundle, 149 const struct uverbs_attr *attr) 150 { 151 struct bundle_priv *pbundle = 152 container_of(bundle, struct bundle_priv, bundle); 153 u16 flags; 154 155 flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | 156 UVERBS_ATTR_F_VALID_OUTPUT; 157 if (put_user(flags, 158 &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) 159 return -EFAULT; 160 return 0; 161 } 162 163 static int uverbs_process_idrs_array(struct bundle_priv *pbundle, 164 const struct uverbs_api_attr *attr_uapi, 165 struct uverbs_objs_arr_attr *attr, 166 struct ib_uverbs_attr *uattr, 167 u32 attr_bkey) 168 { 169 const struct uverbs_attr_spec *spec = &attr_uapi->spec; 170 size_t array_len; 171 u32 *idr_vals; 172 int ret = 0; 173 size_t i; 174 175 if (uattr->attr_data.reserved) 176 return -EINVAL; 177 178 if (uattr->len % sizeof(u32)) 179 return -EINVAL; 180 181 array_len = uattr->len / sizeof(u32); 182 if (array_len < spec->u2.objs_arr.min_len || 183 array_len > spec->u2.objs_arr.max_len) 184 return -EINVAL; 185 186 attr->uobjects = 187 uverbs_alloc(&pbundle->bundle, 188 array_size(array_len, sizeof(*attr->uobjects))); 189 if (IS_ERR(attr->uobjects)) 190 return PTR_ERR(attr->uobjects); 191 192 /* 193 * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects 194 * to store idrs array and avoid additional memory allocation. The 195 * idrs array is offset to the end of the uobjects array so we will be 196 * able to read idr and replace with a pointer. 197 */ 198 idr_vals = (u32 *)(attr->uobjects + array_len) - array_len; 199 200 if (uattr->len > sizeof(uattr->data)) { 201 ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data), 202 uattr->len); 203 if (ret) 204 return -EFAULT; 205 } else { 206 memcpy(idr_vals, &uattr->data, uattr->len); 207 } 208 209 for (i = 0; i != array_len; i++) { 210 attr->uobjects[i] = uverbs_get_uobject_from_file( 211 spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access, 212 idr_vals[i], &pbundle->bundle); 213 if (IS_ERR(attr->uobjects[i])) { 214 ret = PTR_ERR(attr->uobjects[i]); 215 break; 216 } 217 } 218 219 attr->len = i; 220 __set_bit(attr_bkey, pbundle->spec_finalize); 221 return ret; 222 } 223 224 static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi, 225 struct uverbs_objs_arr_attr *attr, 226 bool commit, 227 struct uverbs_attr_bundle *attrs) 228 { 229 const struct uverbs_attr_spec *spec = &attr_uapi->spec; 230 size_t i; 231 232 for (i = 0; i != attr->len; i++) 233 uverbs_finalize_object(attr->uobjects[i], 234 spec->u2.objs_arr.access, false, commit, 235 attrs); 236 } 237 238 static int uverbs_process_attr(struct bundle_priv *pbundle, 239 const struct uverbs_api_attr *attr_uapi, 240 struct ib_uverbs_attr *uattr, u32 attr_bkey) 241 { 242 const struct uverbs_attr_spec *spec = &attr_uapi->spec; 243 struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey]; 244 const struct uverbs_attr_spec *val_spec = spec; 245 struct uverbs_obj_attr *o_attr; 246 247 switch (spec->type) { 248 case UVERBS_ATTR_TYPE_ENUM_IN: 249 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems) 250 return -EOPNOTSUPP; 251 252 if (uattr->attr_data.enum_data.reserved) 253 return -EINVAL; 254 255 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id]; 256 257 /* Currently we only support PTR_IN based enums */ 258 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN) 259 return -EOPNOTSUPP; 260 261 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id; 262 fallthrough; 263 case UVERBS_ATTR_TYPE_PTR_IN: 264 /* Ensure that any data provided by userspace beyond the known 265 * struct is zero. Userspace that knows how to use some future 266 * longer struct will fail here if used with an old kernel and 267 * non-zero content, making ABI compat/discovery simpler. 268 */ 269 if (uattr->len > val_spec->u.ptr.len && 270 val_spec->zero_trailing && 271 !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len)) 272 return -EOPNOTSUPP; 273 274 fallthrough; 275 case UVERBS_ATTR_TYPE_PTR_OUT: 276 if (uattr->len < val_spec->u.ptr.min_len || 277 (!val_spec->zero_trailing && 278 uattr->len > val_spec->u.ptr.len)) 279 return -EINVAL; 280 281 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN && 282 uattr->attr_data.reserved) 283 return -EINVAL; 284 285 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs; 286 e->ptr_attr.len = uattr->len; 287 288 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) { 289 void *p; 290 291 p = uverbs_alloc(&pbundle->bundle, uattr->len); 292 if (IS_ERR(p)) 293 return PTR_ERR(p); 294 295 e->ptr_attr.ptr = p; 296 297 if (copy_from_user(p, u64_to_user_ptr(uattr->data), 298 uattr->len)) 299 return -EFAULT; 300 } else { 301 e->ptr_attr.data = uattr->data; 302 } 303 break; 304 305 case UVERBS_ATTR_TYPE_IDR: 306 case UVERBS_ATTR_TYPE_FD: 307 if (uattr->attr_data.reserved) 308 return -EINVAL; 309 310 if (uattr->len != 0) 311 return -EINVAL; 312 313 o_attr = &e->obj_attr; 314 o_attr->attr_elm = attr_uapi; 315 316 /* 317 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and 318 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64 319 * here without caring about truncation as we know that the 320 * IDR implementation today rejects negative IDs 321 */ 322 o_attr->uobject = uverbs_get_uobject_from_file( 323 spec->u.obj.obj_type, spec->u.obj.access, 324 uattr->data_s64, &pbundle->bundle); 325 if (IS_ERR(o_attr->uobject)) 326 return PTR_ERR(o_attr->uobject); 327 __set_bit(attr_bkey, pbundle->uobj_finalize); 328 329 if (spec->u.obj.access == UVERBS_ACCESS_NEW) { 330 unsigned int uattr_idx = uattr - pbundle->uattrs; 331 s64 id = o_attr->uobject->id; 332 333 /* Copy the allocated id to the user-space */ 334 if (put_user(id, &pbundle->user_attrs[uattr_idx].data)) 335 return -EFAULT; 336 } 337 338 break; 339 340 case UVERBS_ATTR_TYPE_RAW_FD: 341 if (uattr->attr_data.reserved || uattr->len != 0 || 342 uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX) 343 return -EINVAL; 344 /* _uverbs_get_const_signed() is the accessor */ 345 e->ptr_attr.data = uattr->data_s64; 346 break; 347 348 case UVERBS_ATTR_TYPE_IDRS_ARRAY: 349 return uverbs_process_idrs_array(pbundle, attr_uapi, 350 &e->objs_arr_attr, uattr, 351 attr_bkey); 352 default: 353 return -EOPNOTSUPP; 354 } 355 356 return 0; 357 } 358 359 /* 360 * We search the radix tree with the method prefix and now we want to fast 361 * search the suffix bits to get a particular attribute pointer. It is not 362 * totally clear to me if this breaks the radix tree encasulation or not, but 363 * it uses the iter data to determine if the method iter points at the same 364 * chunk that will store the attribute, if so it just derefs it directly. By 365 * construction in most kernel configs the method and attrs will all fit in a 366 * single radix chunk, so in most cases this will have no search. Other cases 367 * this falls back to a full search. 368 */ 369 static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle, 370 u32 attr_key) 371 { 372 void __rcu **slot; 373 374 if (likely(attr_key < pbundle->radix_slots_len)) { 375 void *entry; 376 377 slot = pbundle->radix_slots + attr_key; 378 entry = rcu_dereference_raw(*slot); 379 if (likely(!radix_tree_is_internal_node(entry) && entry)) 380 return slot; 381 } 382 383 return radix_tree_lookup_slot(pbundle->radix, 384 pbundle->method_key | attr_key); 385 } 386 387 static int uverbs_set_attr(struct bundle_priv *pbundle, 388 struct ib_uverbs_attr *uattr) 389 { 390 u32 attr_key = uapi_key_attr(uattr->attr_id); 391 u32 attr_bkey = uapi_bkey_attr(attr_key); 392 const struct uverbs_api_attr *attr; 393 void __rcu **slot; 394 int ret; 395 396 slot = uapi_get_attr_for_method(pbundle, attr_key); 397 if (!slot) { 398 /* 399 * Kernel does not support the attribute but user-space says it 400 * is mandatory 401 */ 402 if (uattr->flags & UVERBS_ATTR_F_MANDATORY) 403 return -EPROTONOSUPPORT; 404 return 0; 405 } 406 attr = rcu_dereference_protected(*slot, true); 407 408 /* Reject duplicate attributes from user-space */ 409 if (test_bit(attr_bkey, pbundle->bundle.attr_present)) 410 return -EINVAL; 411 412 ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey); 413 if (ret) 414 return ret; 415 416 __set_bit(attr_bkey, pbundle->bundle.attr_present); 417 418 return 0; 419 } 420 421 static int ib_uverbs_run_method(struct bundle_priv *pbundle, 422 unsigned int num_attrs) 423 { 424 int (*handler)(struct uverbs_attr_bundle *attrs); 425 size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs); 426 unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey; 427 unsigned int i; 428 int ret; 429 430 /* See uverbs_disassociate_api() */ 431 handler = srcu_dereference( 432 pbundle->method_elm->handler, 433 &pbundle->bundle.ufile->device->disassociate_srcu); 434 if (!handler) 435 return -EIO; 436 437 pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size); 438 if (IS_ERR(pbundle->uattrs)) 439 return PTR_ERR(pbundle->uattrs); 440 if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size)) 441 return -EFAULT; 442 443 for (i = 0; i != num_attrs; i++) { 444 ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]); 445 if (unlikely(ret)) 446 return ret; 447 } 448 449 /* User space did not provide all the mandatory attributes */ 450 if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory, 451 pbundle->bundle.attr_present, 452 pbundle->method_elm->key_bitmap_len))) 453 return -EINVAL; 454 455 if (pbundle->method_elm->has_udata) 456 uverbs_fill_udata(&pbundle->bundle, 457 &pbundle->bundle.driver_udata, 458 UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT); 459 else 460 pbundle->bundle.driver_udata = (struct ib_udata){}; 461 462 if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) { 463 struct uverbs_obj_attr *destroy_attr = 464 &pbundle->bundle.attrs[destroy_bkey].obj_attr; 465 466 ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle); 467 if (ret) 468 return ret; 469 __clear_bit(destroy_bkey, pbundle->uobj_finalize); 470 471 ret = handler(&pbundle->bundle); 472 uobj_put_destroy(destroy_attr->uobject); 473 } else { 474 ret = handler(&pbundle->bundle); 475 } 476 477 /* 478 * Until the drivers are revised to use the bundle directly we have to 479 * assume that the driver wrote to its UHW_OUT and flag userspace 480 * appropriately. 481 */ 482 if (!ret && pbundle->method_elm->has_udata) { 483 const struct uverbs_attr *attr = 484 uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT); 485 486 if (!IS_ERR(attr)) 487 ret = uverbs_set_output(&pbundle->bundle, attr); 488 } 489 490 /* 491 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can 492 * not invoke the method because the request is not supported. No 493 * other cases should return this code. 494 */ 495 if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT)) 496 return -EINVAL; 497 498 return ret; 499 } 500 501 static void bundle_destroy(struct bundle_priv *pbundle, bool commit) 502 { 503 unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len; 504 struct bundle_alloc_head *memblock; 505 unsigned int i; 506 507 /* fast path for simple uobjects */ 508 i = -1; 509 while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len, 510 i + 1)) < key_bitmap_len) { 511 struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; 512 513 uverbs_finalize_object( 514 attr->obj_attr.uobject, 515 attr->obj_attr.attr_elm->spec.u.obj.access, 516 test_bit(i, pbundle->uobj_hw_obj_valid), 517 commit, 518 &pbundle->bundle); 519 } 520 521 i = -1; 522 while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len, 523 i + 1)) < key_bitmap_len) { 524 struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; 525 const struct uverbs_api_attr *attr_uapi; 526 void __rcu **slot; 527 528 slot = uapi_get_attr_for_method( 529 pbundle, 530 pbundle->method_key | uapi_bkey_to_key_attr(i)); 531 if (WARN_ON(!slot)) 532 continue; 533 534 attr_uapi = rcu_dereference_protected(*slot, true); 535 536 if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) { 537 uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr, 538 commit, &pbundle->bundle); 539 } 540 } 541 542 for (memblock = pbundle->allocated_mem; memblock;) { 543 struct bundle_alloc_head *tmp = memblock; 544 545 memblock = memblock->next; 546 kvfree(tmp); 547 } 548 } 549 550 static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, 551 struct ib_uverbs_ioctl_hdr *hdr, 552 struct ib_uverbs_attr __user *user_attrs) 553 { 554 const struct uverbs_api_ioctl_method *method_elm; 555 struct uverbs_api *uapi = ufile->device->uapi; 556 struct radix_tree_iter attrs_iter; 557 struct bundle_priv *pbundle; 558 struct bundle_priv onstack; 559 void __rcu **slot; 560 int ret; 561 562 if (unlikely(hdr->driver_id != uapi->driver_id)) 563 return -EINVAL; 564 565 slot = radix_tree_iter_lookup( 566 &uapi->radix, &attrs_iter, 567 uapi_key_obj(hdr->object_id) | 568 uapi_key_ioctl_method(hdr->method_id)); 569 if (unlikely(!slot)) 570 return -EPROTONOSUPPORT; 571 method_elm = rcu_dereference_protected(*slot, true); 572 573 if (!method_elm->use_stack) { 574 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL); 575 if (!pbundle) 576 return -ENOMEM; 577 pbundle->internal_avail = 578 method_elm->bundle_size - 579 offsetof(struct bundle_priv, internal_buffer); 580 pbundle->alloc_head.next = NULL; 581 pbundle->allocated_mem = &pbundle->alloc_head; 582 } else { 583 pbundle = &onstack; 584 pbundle->internal_avail = sizeof(pbundle->internal_buffer); 585 pbundle->allocated_mem = NULL; 586 } 587 588 /* Space for the pbundle->bundle.attrs flex array */ 589 pbundle->method_elm = method_elm; 590 pbundle->method_key = attrs_iter.index; 591 pbundle->bundle.ufile = ufile; 592 pbundle->bundle.context = NULL; /* only valid if bundle has uobject */ 593 pbundle->radix = &uapi->radix; 594 pbundle->radix_slots = slot; 595 pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter); 596 pbundle->user_attrs = user_attrs; 597 598 pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len * 599 sizeof(*pbundle->bundle.attrs), 600 sizeof(*pbundle->internal_buffer)); 601 memset(pbundle->bundle.attr_present, 0, 602 sizeof(pbundle->bundle.attr_present)); 603 memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize)); 604 memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize)); 605 memset(pbundle->uobj_hw_obj_valid, 0, 606 sizeof(pbundle->uobj_hw_obj_valid)); 607 608 ret = ib_uverbs_run_method(pbundle, hdr->num_attrs); 609 bundle_destroy(pbundle, ret == 0); 610 return ret; 611 } 612 613 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 614 { 615 struct ib_uverbs_file *file = filp->private_data; 616 struct ib_uverbs_ioctl_hdr __user *user_hdr = 617 (struct ib_uverbs_ioctl_hdr __user *)arg; 618 struct ib_uverbs_ioctl_hdr hdr; 619 int srcu_key; 620 int err; 621 622 if (unlikely(cmd != RDMA_VERBS_IOCTL)) 623 return -ENOIOCTLCMD; 624 625 err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); 626 if (err) 627 return -EFAULT; 628 629 if (hdr.length > PAGE_SIZE || 630 hdr.length != struct_size(&hdr, attrs, hdr.num_attrs)) 631 return -EINVAL; 632 633 if (hdr.reserved1 || hdr.reserved2) 634 return -EPROTONOSUPPORT; 635 636 srcu_key = srcu_read_lock(&file->device->disassociate_srcu); 637 err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs); 638 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); 639 return err; 640 } 641 642 int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, 643 size_t idx, u64 allowed_bits) 644 { 645 const struct uverbs_attr *attr; 646 u64 flags; 647 648 attr = uverbs_attr_get(attrs_bundle, idx); 649 /* Missing attribute means 0 flags */ 650 if (IS_ERR(attr)) { 651 *to = 0; 652 return 0; 653 } 654 655 /* 656 * New userspace code should use 8 bytes to pass flags, but we 657 * transparently support old userspaces that were using 4 bytes as 658 * well. 659 */ 660 if (attr->ptr_attr.len == 8) 661 flags = attr->ptr_attr.data; 662 else if (attr->ptr_attr.len == 4) 663 flags = *(u32 *)&attr->ptr_attr.data; 664 else 665 return -EINVAL; 666 667 if (flags & ~allowed_bits) 668 return -EINVAL; 669 670 *to = flags; 671 return 0; 672 } 673 EXPORT_SYMBOL(uverbs_get_flags64); 674 675 int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, 676 size_t idx, u64 allowed_bits) 677 { 678 u64 flags; 679 int ret; 680 681 ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); 682 if (ret) 683 return ret; 684 685 if (flags > U32_MAX) 686 return -EINVAL; 687 *to = flags; 688 689 return 0; 690 } 691 EXPORT_SYMBOL(uverbs_get_flags32); 692 693 /* 694 * Fill a ib_udata struct (core or uhw) using the given attribute IDs. 695 * This is primarily used to convert the UVERBS_ATTR_UHW() into the 696 * ib_udata format used by the drivers. 697 */ 698 void uverbs_fill_udata(struct uverbs_attr_bundle *bundle, 699 struct ib_udata *udata, unsigned int attr_in, 700 unsigned int attr_out) 701 { 702 struct bundle_priv *pbundle = 703 container_of(bundle, struct bundle_priv, bundle); 704 const struct uverbs_attr *in = 705 uverbs_attr_get(&pbundle->bundle, attr_in); 706 const struct uverbs_attr *out = 707 uverbs_attr_get(&pbundle->bundle, attr_out); 708 709 if (!IS_ERR(in)) { 710 udata->inlen = in->ptr_attr.len; 711 if (uverbs_attr_ptr_is_inline(in)) 712 udata->inbuf = 713 &pbundle->user_attrs[in->ptr_attr.uattr_idx] 714 .data; 715 else 716 udata->inbuf = u64_to_user_ptr(in->ptr_attr.data); 717 } else { 718 udata->inbuf = NULL; 719 udata->inlen = 0; 720 } 721 722 if (!IS_ERR(out)) { 723 udata->outbuf = u64_to_user_ptr(out->ptr_attr.data); 724 udata->outlen = out->ptr_attr.len; 725 } else { 726 udata->outbuf = NULL; 727 udata->outlen = 0; 728 } 729 } 730 731 int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, 732 const void *from, size_t size) 733 { 734 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); 735 size_t min_size; 736 737 if (IS_ERR(attr)) 738 return PTR_ERR(attr); 739 740 min_size = min_t(size_t, attr->ptr_attr.len, size); 741 if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) 742 return -EFAULT; 743 744 return uverbs_set_output(bundle, attr); 745 } 746 EXPORT_SYMBOL(uverbs_copy_to); 747 748 749 /* 750 * This is only used if the caller has directly used copy_to_use to write the 751 * data. It signals to user space that the buffer is filled in. 752 */ 753 int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx) 754 { 755 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); 756 757 if (IS_ERR(attr)) 758 return PTR_ERR(attr); 759 760 return uverbs_set_output(bundle, attr); 761 } 762 763 int _uverbs_get_const_signed(s64 *to, 764 const struct uverbs_attr_bundle *attrs_bundle, 765 size_t idx, s64 lower_bound, u64 upper_bound, 766 s64 *def_val) 767 { 768 const struct uverbs_attr *attr; 769 770 attr = uverbs_attr_get(attrs_bundle, idx); 771 if (IS_ERR(attr)) { 772 if ((PTR_ERR(attr) != -ENOENT) || !def_val) 773 return PTR_ERR(attr); 774 775 *to = *def_val; 776 } else { 777 *to = attr->ptr_attr.data; 778 } 779 780 if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound)) 781 return -EINVAL; 782 783 return 0; 784 } 785 EXPORT_SYMBOL(_uverbs_get_const_signed); 786 787 int _uverbs_get_const_unsigned(u64 *to, 788 const struct uverbs_attr_bundle *attrs_bundle, 789 size_t idx, u64 upper_bound, u64 *def_val) 790 { 791 const struct uverbs_attr *attr; 792 793 attr = uverbs_attr_get(attrs_bundle, idx); 794 if (IS_ERR(attr)) { 795 if ((PTR_ERR(attr) != -ENOENT) || !def_val) 796 return PTR_ERR(attr); 797 798 *to = *def_val; 799 } else { 800 *to = attr->ptr_attr.data; 801 } 802 803 if (*to > upper_bound) 804 return -EINVAL; 805 806 return 0; 807 } 808 EXPORT_SYMBOL(_uverbs_get_const_unsigned); 809 810 int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, 811 size_t idx, const void *from, size_t size) 812 { 813 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); 814 815 if (IS_ERR(attr)) 816 return PTR_ERR(attr); 817 818 if (size < attr->ptr_attr.len) { 819 if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size, 820 attr->ptr_attr.len - size)) 821 return -EFAULT; 822 } 823 return uverbs_copy_to(bundle, idx, from, size); 824 } 825 EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); 826 827 /* Once called an abort will call through to the type's destroy_hw() */ 828 void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, 829 u16 idx) 830 { 831 struct bundle_priv *pbundle = 832 container_of(bundle, struct bundle_priv, bundle); 833 834 __set_bit(uapi_bkey_attr(uapi_key_attr(idx)), 835 pbundle->uobj_hw_obj_valid); 836 } 837 EXPORT_SYMBOL(uverbs_finalize_uobj_create); 838