1 /* 2 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/rdma_user_ioctl.h> 34 #include <rdma/uverbs_ioctl.h> 35 #include "rdma_core.h" 36 #include "uverbs.h" 37 38 struct bundle_alloc_head { 39 struct bundle_alloc_head *next; 40 u8 data[]; 41 }; 42 43 struct bundle_priv { 44 /* Must be first */ 45 struct bundle_alloc_head alloc_head; 46 struct bundle_alloc_head *allocated_mem; 47 size_t internal_avail; 48 size_t internal_used; 49 50 struct radix_tree_root *radix; 51 const struct uverbs_api_ioctl_method *method_elm; 52 void __rcu **radix_slots; 53 unsigned long radix_slots_len; 54 u32 method_key; 55 56 struct ib_uverbs_attr __user *user_attrs; 57 struct ib_uverbs_attr *uattrs; 58 59 DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN); 60 61 /* 62 * Must be last. bundle ends in a flex array which overlaps 63 * internal_buffer. 64 */ 65 struct uverbs_attr_bundle bundle; 66 u64 internal_buffer[32]; 67 }; 68 69 /* 70 * Each method has an absolute minimum amount of memory it needs to allocate, 71 * precompute that amount and determine if the onstack memory can be used or 72 * if allocation is need. 73 */ 74 void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, 75 unsigned int num_attrs) 76 { 77 struct bundle_priv *pbundle; 78 size_t bundle_size = 79 offsetof(struct bundle_priv, internal_buffer) + 80 sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len + 81 sizeof(*pbundle->uattrs) * num_attrs; 82 83 method_elm->use_stack = bundle_size <= sizeof(*pbundle); 84 method_elm->bundle_size = 85 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer)); 86 87 /* Do not want order-2 allocations for this. */ 88 WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE); 89 } 90 91 /** 92 * uverbs_alloc() - Quickly allocate memory for use with a bundle 93 * @bundle: The bundle 94 * @size: Number of bytes to allocate 95 * @flags: Allocator flags 96 * 97 * The bundle allocator is intended for allocations that are connected with 98 * processing the system call related to the bundle. The allocated memory is 99 * always freed once the system call completes, and cannot be freed any other 100 * way. 101 * 102 * This tries to use a small pool of pre-allocated memory for performance. 103 */ 104 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, 105 gfp_t flags) 106 { 107 struct bundle_priv *pbundle = 108 container_of(bundle, struct bundle_priv, bundle); 109 size_t new_used; 110 void *res; 111 112 if (check_add_overflow(size, pbundle->internal_used, &new_used)) 113 return ERR_PTR(-EOVERFLOW); 114 115 if (new_used > pbundle->internal_avail) { 116 struct bundle_alloc_head *buf; 117 118 buf = kvmalloc(struct_size(buf, data, size), flags); 119 if (!buf) 120 return ERR_PTR(-ENOMEM); 121 buf->next = pbundle->allocated_mem; 122 pbundle->allocated_mem = buf; 123 return buf->data; 124 } 125 126 res = (void *)pbundle->internal_buffer + pbundle->internal_used; 127 pbundle->internal_used = 128 ALIGN(new_used, sizeof(*pbundle->internal_buffer)); 129 if (flags & __GFP_ZERO) 130 memset(res, 0, size); 131 return res; 132 } 133 EXPORT_SYMBOL(_uverbs_alloc); 134 135 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, 136 u16 len) 137 { 138 if (uattr->len > sizeof(((struct ib_uverbs_attr *)0)->data)) 139 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len, 140 uattr->len - len); 141 142 return !memchr_inv((const void *)&uattr->data + len, 143 0, uattr->len - len); 144 } 145 146 static int uverbs_process_attr(struct bundle_priv *pbundle, 147 const struct uverbs_api_attr *attr_uapi, 148 struct ib_uverbs_attr *uattr, u32 attr_bkey) 149 { 150 const struct uverbs_attr_spec *spec = &attr_uapi->spec; 151 struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey]; 152 const struct uverbs_attr_spec *val_spec = spec; 153 struct uverbs_obj_attr *o_attr; 154 155 switch (spec->type) { 156 case UVERBS_ATTR_TYPE_ENUM_IN: 157 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems) 158 return -EOPNOTSUPP; 159 160 if (uattr->attr_data.enum_data.reserved) 161 return -EINVAL; 162 163 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id]; 164 165 /* Currently we only support PTR_IN based enums */ 166 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN) 167 return -EOPNOTSUPP; 168 169 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id; 170 /* fall through */ 171 case UVERBS_ATTR_TYPE_PTR_IN: 172 /* Ensure that any data provided by userspace beyond the known 173 * struct is zero. Userspace that knows how to use some future 174 * longer struct will fail here if used with an old kernel and 175 * non-zero content, making ABI compat/discovery simpler. 176 */ 177 if (uattr->len > val_spec->u.ptr.len && 178 val_spec->zero_trailing && 179 !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len)) 180 return -EOPNOTSUPP; 181 182 /* fall through */ 183 case UVERBS_ATTR_TYPE_PTR_OUT: 184 if (uattr->len < val_spec->u.ptr.min_len || 185 (!val_spec->zero_trailing && 186 uattr->len > val_spec->u.ptr.len)) 187 return -EINVAL; 188 189 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN && 190 uattr->attr_data.reserved) 191 return -EINVAL; 192 193 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs; 194 e->ptr_attr.len = uattr->len; 195 196 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) { 197 void *p; 198 199 p = uverbs_alloc(&pbundle->bundle, uattr->len); 200 if (IS_ERR(p)) 201 return PTR_ERR(p); 202 203 e->ptr_attr.ptr = p; 204 205 if (copy_from_user(p, u64_to_user_ptr(uattr->data), 206 uattr->len)) 207 return -EFAULT; 208 } else { 209 e->ptr_attr.data = uattr->data; 210 } 211 break; 212 213 case UVERBS_ATTR_TYPE_IDR: 214 case UVERBS_ATTR_TYPE_FD: 215 if (uattr->attr_data.reserved) 216 return -EINVAL; 217 218 if (uattr->len != 0) 219 return -EINVAL; 220 221 o_attr = &e->obj_attr; 222 o_attr->attr_elm = attr_uapi; 223 224 /* 225 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and 226 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64 227 * here without caring about truncation as we know that the 228 * IDR implementation today rejects negative IDs 229 */ 230 o_attr->uobject = uverbs_get_uobject_from_file( 231 spec->u.obj.obj_type, 232 pbundle->bundle.ufile, 233 spec->u.obj.access, 234 uattr->data_s64); 235 if (IS_ERR(o_attr->uobject)) 236 return PTR_ERR(o_attr->uobject); 237 __set_bit(attr_bkey, pbundle->uobj_finalize); 238 239 if (spec->u.obj.access == UVERBS_ACCESS_NEW) { 240 unsigned int uattr_idx = uattr - pbundle->uattrs; 241 s64 id = o_attr->uobject->id; 242 243 /* Copy the allocated id to the user-space */ 244 if (put_user(id, &pbundle->user_attrs[uattr_idx].data)) 245 return -EFAULT; 246 } 247 248 break; 249 default: 250 return -EOPNOTSUPP; 251 } 252 253 return 0; 254 } 255 256 /* 257 * We search the radix tree with the method prefix and now we want to fast 258 * search the suffix bits to get a particular attribute pointer. It is not 259 * totally clear to me if this breaks the radix tree encasulation or not, but 260 * it uses the iter data to determine if the method iter points at the same 261 * chunk that will store the attribute, if so it just derefs it directly. By 262 * construction in most kernel configs the method and attrs will all fit in a 263 * single radix chunk, so in most cases this will have no search. Other cases 264 * this falls back to a full search. 265 */ 266 static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle, 267 u32 attr_key) 268 { 269 void __rcu **slot; 270 271 if (likely(attr_key < pbundle->radix_slots_len)) { 272 void *entry; 273 274 slot = pbundle->radix_slots + attr_key; 275 entry = rcu_dereference_raw(*slot); 276 if (likely(!radix_tree_is_internal_node(entry) && entry)) 277 return slot; 278 } 279 280 return radix_tree_lookup_slot(pbundle->radix, 281 pbundle->method_key | attr_key); 282 } 283 284 static int uverbs_set_attr(struct bundle_priv *pbundle, 285 struct ib_uverbs_attr *uattr) 286 { 287 u32 attr_key = uapi_key_attr(uattr->attr_id); 288 u32 attr_bkey = uapi_bkey_attr(attr_key); 289 const struct uverbs_api_attr *attr; 290 void __rcu **slot; 291 int ret; 292 293 slot = uapi_get_attr_for_method(pbundle, attr_key); 294 if (!slot) { 295 /* 296 * Kernel does not support the attribute but user-space says it 297 * is mandatory 298 */ 299 if (uattr->flags & UVERBS_ATTR_F_MANDATORY) 300 return -EPROTONOSUPPORT; 301 return 0; 302 } 303 attr = srcu_dereference( 304 *slot, &pbundle->bundle.ufile->device->disassociate_srcu); 305 306 /* Reject duplicate attributes from user-space */ 307 if (test_bit(attr_bkey, pbundle->bundle.attr_present)) 308 return -EINVAL; 309 310 ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey); 311 if (ret) 312 return ret; 313 314 __set_bit(attr_bkey, pbundle->bundle.attr_present); 315 316 return 0; 317 } 318 319 static int ib_uverbs_run_method(struct bundle_priv *pbundle, 320 unsigned int num_attrs) 321 { 322 int (*handler)(struct ib_uverbs_file *ufile, 323 struct uverbs_attr_bundle *ctx); 324 size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs); 325 unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey; 326 unsigned int i; 327 int ret; 328 329 /* See uverbs_disassociate_api() */ 330 handler = srcu_dereference( 331 pbundle->method_elm->handler, 332 &pbundle->bundle.ufile->device->disassociate_srcu); 333 if (!handler) 334 return -EIO; 335 336 pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size); 337 if (IS_ERR(pbundle->uattrs)) 338 return PTR_ERR(pbundle->uattrs); 339 if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size)) 340 return -EFAULT; 341 342 for (i = 0; i != num_attrs; i++) { 343 ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]); 344 if (unlikely(ret)) 345 return ret; 346 } 347 348 /* User space did not provide all the mandatory attributes */ 349 if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory, 350 pbundle->bundle.attr_present, 351 pbundle->method_elm->key_bitmap_len))) 352 return -EINVAL; 353 354 if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) { 355 struct uverbs_obj_attr *destroy_attr = 356 &pbundle->bundle.attrs[destroy_bkey].obj_attr; 357 358 ret = uobj_destroy(destroy_attr->uobject); 359 if (ret) 360 return ret; 361 __clear_bit(destroy_bkey, pbundle->uobj_finalize); 362 363 ret = handler(pbundle->bundle.ufile, &pbundle->bundle); 364 uobj_put_destroy(destroy_attr->uobject); 365 } else { 366 ret = handler(pbundle->bundle.ufile, &pbundle->bundle); 367 } 368 369 /* 370 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can 371 * not invoke the method because the request is not supported. No 372 * other cases should return this code. 373 */ 374 if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT)) 375 return -EINVAL; 376 377 return ret; 378 } 379 380 static int bundle_destroy(struct bundle_priv *pbundle, bool commit) 381 { 382 unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len; 383 struct bundle_alloc_head *memblock; 384 unsigned int i; 385 int ret = 0; 386 387 i = -1; 388 while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len, 389 i + 1)) < key_bitmap_len) { 390 struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; 391 int current_ret; 392 393 current_ret = uverbs_finalize_object( 394 attr->obj_attr.uobject, 395 attr->obj_attr.attr_elm->spec.u.obj.access, commit); 396 if (!ret) 397 ret = current_ret; 398 } 399 400 for (memblock = pbundle->allocated_mem; memblock;) { 401 struct bundle_alloc_head *tmp = memblock; 402 403 memblock = memblock->next; 404 kvfree(tmp); 405 } 406 407 return ret; 408 } 409 410 static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, 411 struct ib_uverbs_ioctl_hdr *hdr, 412 struct ib_uverbs_attr __user *user_attrs) 413 { 414 const struct uverbs_api_ioctl_method *method_elm; 415 struct uverbs_api *uapi = ufile->device->uapi; 416 struct radix_tree_iter attrs_iter; 417 struct bundle_priv *pbundle; 418 struct bundle_priv onstack; 419 void __rcu **slot; 420 int destroy_ret; 421 int ret; 422 423 if (unlikely(hdr->driver_id != uapi->driver_id)) 424 return -EINVAL; 425 426 slot = radix_tree_iter_lookup( 427 &uapi->radix, &attrs_iter, 428 uapi_key_obj(hdr->object_id) | 429 uapi_key_ioctl_method(hdr->method_id)); 430 if (unlikely(!slot)) 431 return -EPROTONOSUPPORT; 432 method_elm = srcu_dereference(*slot, &ufile->device->disassociate_srcu); 433 434 if (!method_elm->use_stack) { 435 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL); 436 if (!pbundle) 437 return -ENOMEM; 438 pbundle->internal_avail = 439 method_elm->bundle_size - 440 offsetof(struct bundle_priv, internal_buffer); 441 pbundle->alloc_head.next = NULL; 442 pbundle->allocated_mem = &pbundle->alloc_head; 443 } else { 444 pbundle = &onstack; 445 pbundle->internal_avail = sizeof(pbundle->internal_buffer); 446 pbundle->allocated_mem = NULL; 447 } 448 449 /* Space for the pbundle->bundle.attrs flex array */ 450 pbundle->method_elm = method_elm; 451 pbundle->method_key = attrs_iter.index; 452 pbundle->bundle.ufile = ufile; 453 pbundle->radix = &uapi->radix; 454 pbundle->radix_slots = slot; 455 pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter); 456 pbundle->user_attrs = user_attrs; 457 458 pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len * 459 sizeof(*pbundle->bundle.attrs), 460 sizeof(*pbundle->internal_buffer)); 461 memset(pbundle->bundle.attr_present, 0, 462 sizeof(pbundle->bundle.attr_present)); 463 memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize)); 464 465 ret = ib_uverbs_run_method(pbundle, hdr->num_attrs); 466 destroy_ret = bundle_destroy(pbundle, ret == 0); 467 if (unlikely(destroy_ret && !ret)) 468 return destroy_ret; 469 470 return ret; 471 } 472 473 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 474 { 475 struct ib_uverbs_file *file = filp->private_data; 476 struct ib_uverbs_ioctl_hdr __user *user_hdr = 477 (struct ib_uverbs_ioctl_hdr __user *)arg; 478 struct ib_uverbs_ioctl_hdr hdr; 479 int srcu_key; 480 int err; 481 482 if (unlikely(cmd != RDMA_VERBS_IOCTL)) 483 return -ENOIOCTLCMD; 484 485 err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); 486 if (err) 487 return -EFAULT; 488 489 if (hdr.length > PAGE_SIZE || 490 hdr.length != struct_size(&hdr, attrs, hdr.num_attrs)) 491 return -EINVAL; 492 493 if (hdr.reserved1 || hdr.reserved2) 494 return -EPROTONOSUPPORT; 495 496 srcu_key = srcu_read_lock(&file->device->disassociate_srcu); 497 err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs); 498 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); 499 return err; 500 } 501 502 int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, 503 size_t idx, u64 allowed_bits) 504 { 505 const struct uverbs_attr *attr; 506 u64 flags; 507 508 attr = uverbs_attr_get(attrs_bundle, idx); 509 /* Missing attribute means 0 flags */ 510 if (IS_ERR(attr)) { 511 *to = 0; 512 return 0; 513 } 514 515 /* 516 * New userspace code should use 8 bytes to pass flags, but we 517 * transparently support old userspaces that were using 4 bytes as 518 * well. 519 */ 520 if (attr->ptr_attr.len == 8) 521 flags = attr->ptr_attr.data; 522 else if (attr->ptr_attr.len == 4) 523 flags = *(u32 *)&attr->ptr_attr.data; 524 else 525 return -EINVAL; 526 527 if (flags & ~allowed_bits) 528 return -EINVAL; 529 530 *to = flags; 531 return 0; 532 } 533 EXPORT_SYMBOL(uverbs_get_flags64); 534 535 int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, 536 size_t idx, u64 allowed_bits) 537 { 538 u64 flags; 539 int ret; 540 541 ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); 542 if (ret) 543 return ret; 544 545 if (flags > U32_MAX) 546 return -EINVAL; 547 *to = flags; 548 549 return 0; 550 } 551 EXPORT_SYMBOL(uverbs_get_flags32); 552 553 /* 554 * This is for ease of conversion. The purpose is to convert all drivers to 555 * use uverbs_attr_bundle instead of ib_udata. Assume attr == 0 is input and 556 * attr == 1 is output. 557 */ 558 void create_udata(struct uverbs_attr_bundle *bundle, struct ib_udata *udata) 559 { 560 struct bundle_priv *pbundle = 561 container_of(bundle, struct bundle_priv, bundle); 562 const struct uverbs_attr *uhw_in = 563 uverbs_attr_get(bundle, UVERBS_ATTR_UHW_IN); 564 const struct uverbs_attr *uhw_out = 565 uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT); 566 567 if (!IS_ERR(uhw_in)) { 568 udata->inlen = uhw_in->ptr_attr.len; 569 if (uverbs_attr_ptr_is_inline(uhw_in)) 570 udata->inbuf = 571 &pbundle->user_attrs[uhw_in->ptr_attr.uattr_idx] 572 .data; 573 else 574 udata->inbuf = u64_to_user_ptr(uhw_in->ptr_attr.data); 575 } else { 576 udata->inbuf = NULL; 577 udata->inlen = 0; 578 } 579 580 if (!IS_ERR(uhw_out)) { 581 udata->outbuf = u64_to_user_ptr(uhw_out->ptr_attr.data); 582 udata->outlen = uhw_out->ptr_attr.len; 583 } else { 584 udata->outbuf = NULL; 585 udata->outlen = 0; 586 } 587 } 588 589 int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, 590 const void *from, size_t size) 591 { 592 struct bundle_priv *pbundle = 593 container_of(bundle, struct bundle_priv, bundle); 594 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); 595 u16 flags; 596 size_t min_size; 597 598 if (IS_ERR(attr)) 599 return PTR_ERR(attr); 600 601 min_size = min_t(size_t, attr->ptr_attr.len, size); 602 if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) 603 return -EFAULT; 604 605 flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | 606 UVERBS_ATTR_F_VALID_OUTPUT; 607 if (put_user(flags, 608 &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) 609 return -EFAULT; 610 611 return 0; 612 } 613 EXPORT_SYMBOL(uverbs_copy_to); 614