1 /*
2  * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <rdma/rdma_user_ioctl.h>
34 #include <rdma/uverbs_ioctl.h>
35 #include "rdma_core.h"
36 #include "uverbs.h"
37 
38 struct bundle_alloc_head {
39 	struct bundle_alloc_head *next;
40 	u8 data[];
41 };
42 
43 struct bundle_priv {
44 	/* Must be first */
45 	struct bundle_alloc_head alloc_head;
46 	struct bundle_alloc_head *allocated_mem;
47 	size_t internal_avail;
48 	size_t internal_used;
49 
50 	struct radix_tree_root *radix;
51 	const struct uverbs_api_ioctl_method *method_elm;
52 	void __rcu **radix_slots;
53 	unsigned long radix_slots_len;
54 	u32 method_key;
55 
56 	struct ib_uverbs_attr __user *user_attrs;
57 	struct ib_uverbs_attr *uattrs;
58 
59 	DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
60 	DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
61 	DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
62 
63 	/*
64 	 * Must be last. bundle ends in a flex array which overlaps
65 	 * internal_buffer.
66 	 */
67 	struct uverbs_attr_bundle bundle;
68 	u64 internal_buffer[32];
69 };
70 
71 /*
72  * Each method has an absolute minimum amount of memory it needs to allocate,
73  * precompute that amount and determine if the onstack memory can be used or
74  * if allocation is need.
75  */
76 void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
77 			      unsigned int num_attrs)
78 {
79 	struct bundle_priv *pbundle;
80 	size_t bundle_size =
81 		offsetof(struct bundle_priv, internal_buffer) +
82 		sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
83 		sizeof(*pbundle->uattrs) * num_attrs;
84 
85 	method_elm->use_stack = bundle_size <= sizeof(*pbundle);
86 	method_elm->bundle_size =
87 		ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
88 
89 	/* Do not want order-2 allocations for this. */
90 	WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
91 }
92 
93 /**
94  * _uverbs_alloc() - Quickly allocate memory for use with a bundle
95  * @bundle: The bundle
96  * @size: Number of bytes to allocate
97  * @flags: Allocator flags
98  *
99  * The bundle allocator is intended for allocations that are connected with
100  * processing the system call related to the bundle. The allocated memory is
101  * always freed once the system call completes, and cannot be freed any other
102  * way.
103  *
104  * This tries to use a small pool of pre-allocated memory for performance.
105  */
106 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
107 			     gfp_t flags)
108 {
109 	struct bundle_priv *pbundle =
110 		container_of(bundle, struct bundle_priv, bundle);
111 	size_t new_used;
112 	void *res;
113 
114 	if (check_add_overflow(size, pbundle->internal_used, &new_used))
115 		return ERR_PTR(-EOVERFLOW);
116 
117 	if (new_used > pbundle->internal_avail) {
118 		struct bundle_alloc_head *buf;
119 
120 		buf = kvmalloc(struct_size(buf, data, size), flags);
121 		if (!buf)
122 			return ERR_PTR(-ENOMEM);
123 		buf->next = pbundle->allocated_mem;
124 		pbundle->allocated_mem = buf;
125 		return buf->data;
126 	}
127 
128 	res = (void *)pbundle->internal_buffer + pbundle->internal_used;
129 	pbundle->internal_used =
130 		ALIGN(new_used, sizeof(*pbundle->internal_buffer));
131 	if (want_init_on_alloc(flags))
132 		memset(res, 0, size);
133 	return res;
134 }
135 EXPORT_SYMBOL(_uverbs_alloc);
136 
137 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
138 				   u16 len)
139 {
140 	if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
141 		return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
142 					    uattr->len - len);
143 
144 	return !memchr_inv((const void *)&uattr->data + len,
145 			   0, uattr->len - len);
146 }
147 
148 static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
149 			     const struct uverbs_attr *attr)
150 {
151 	struct bundle_priv *pbundle =
152 		container_of(bundle, struct bundle_priv, bundle);
153 	u16 flags;
154 
155 	flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
156 		UVERBS_ATTR_F_VALID_OUTPUT;
157 	if (put_user(flags,
158 		     &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
159 		return -EFAULT;
160 	return 0;
161 }
162 
163 static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
164 				     const struct uverbs_api_attr *attr_uapi,
165 				     struct uverbs_objs_arr_attr *attr,
166 				     struct ib_uverbs_attr *uattr,
167 				     u32 attr_bkey)
168 {
169 	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
170 	size_t array_len;
171 	u32 *idr_vals;
172 	int ret = 0;
173 	size_t i;
174 
175 	if (uattr->attr_data.reserved)
176 		return -EINVAL;
177 
178 	if (uattr->len % sizeof(u32))
179 		return -EINVAL;
180 
181 	array_len = uattr->len / sizeof(u32);
182 	if (array_len < spec->u2.objs_arr.min_len ||
183 	    array_len > spec->u2.objs_arr.max_len)
184 		return -EINVAL;
185 
186 	attr->uobjects =
187 		uverbs_alloc(&pbundle->bundle,
188 			     array_size(array_len, sizeof(*attr->uobjects)));
189 	if (IS_ERR(attr->uobjects))
190 		return PTR_ERR(attr->uobjects);
191 
192 	/*
193 	 * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
194 	 * to store idrs array and avoid additional memory allocation. The
195 	 * idrs array is offset to the end of the uobjects array so we will be
196 	 * able to read idr and replace with a pointer.
197 	 */
198 	idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
199 
200 	if (uattr->len > sizeof(uattr->data)) {
201 		ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
202 				     uattr->len);
203 		if (ret)
204 			return -EFAULT;
205 	} else {
206 		memcpy(idr_vals, &uattr->data, uattr->len);
207 	}
208 
209 	for (i = 0; i != array_len; i++) {
210 		attr->uobjects[i] = uverbs_get_uobject_from_file(
211 			spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
212 			idr_vals[i], &pbundle->bundle);
213 		if (IS_ERR(attr->uobjects[i])) {
214 			ret = PTR_ERR(attr->uobjects[i]);
215 			break;
216 		}
217 	}
218 
219 	attr->len = i;
220 	__set_bit(attr_bkey, pbundle->spec_finalize);
221 	return ret;
222 }
223 
224 static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
225 				   struct uverbs_objs_arr_attr *attr,
226 				   bool commit,
227 				   struct uverbs_attr_bundle *attrs)
228 {
229 	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
230 	size_t i;
231 
232 	for (i = 0; i != attr->len; i++)
233 		uverbs_finalize_object(attr->uobjects[i],
234 				       spec->u2.objs_arr.access, false, commit,
235 				       attrs);
236 }
237 
238 static int uverbs_process_attr(struct bundle_priv *pbundle,
239 			       const struct uverbs_api_attr *attr_uapi,
240 			       struct ib_uverbs_attr *uattr, u32 attr_bkey)
241 {
242 	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
243 	struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
244 	const struct uverbs_attr_spec *val_spec = spec;
245 	struct uverbs_obj_attr *o_attr;
246 
247 	switch (spec->type) {
248 	case UVERBS_ATTR_TYPE_ENUM_IN:
249 		if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
250 			return -EOPNOTSUPP;
251 
252 		if (uattr->attr_data.enum_data.reserved)
253 			return -EINVAL;
254 
255 		val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
256 
257 		/* Currently we only support PTR_IN based enums */
258 		if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
259 			return -EOPNOTSUPP;
260 
261 		e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
262 		fallthrough;
263 	case UVERBS_ATTR_TYPE_PTR_IN:
264 		/* Ensure that any data provided by userspace beyond the known
265 		 * struct is zero. Userspace that knows how to use some future
266 		 * longer struct will fail here if used with an old kernel and
267 		 * non-zero content, making ABI compat/discovery simpler.
268 		 */
269 		if (uattr->len > val_spec->u.ptr.len &&
270 		    val_spec->zero_trailing &&
271 		    !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
272 			return -EOPNOTSUPP;
273 
274 		fallthrough;
275 	case UVERBS_ATTR_TYPE_PTR_OUT:
276 		if (uattr->len < val_spec->u.ptr.min_len ||
277 		    (!val_spec->zero_trailing &&
278 		     uattr->len > val_spec->u.ptr.len))
279 			return -EINVAL;
280 
281 		if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
282 		    uattr->attr_data.reserved)
283 			return -EINVAL;
284 
285 		e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
286 		e->ptr_attr.len = uattr->len;
287 
288 		if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
289 			void *p;
290 
291 			p = uverbs_alloc(&pbundle->bundle, uattr->len);
292 			if (IS_ERR(p))
293 				return PTR_ERR(p);
294 
295 			e->ptr_attr.ptr = p;
296 
297 			if (copy_from_user(p, u64_to_user_ptr(uattr->data),
298 					   uattr->len))
299 				return -EFAULT;
300 		} else {
301 			e->ptr_attr.data = uattr->data;
302 		}
303 		break;
304 
305 	case UVERBS_ATTR_TYPE_IDR:
306 	case UVERBS_ATTR_TYPE_FD:
307 		if (uattr->attr_data.reserved)
308 			return -EINVAL;
309 
310 		if (uattr->len != 0)
311 			return -EINVAL;
312 
313 		o_attr = &e->obj_attr;
314 		o_attr->attr_elm = attr_uapi;
315 
316 		/*
317 		 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
318 		 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
319 		 * here without caring about truncation as we know that the
320 		 * IDR implementation today rejects negative IDs
321 		 */
322 		o_attr->uobject = uverbs_get_uobject_from_file(
323 			spec->u.obj.obj_type, spec->u.obj.access,
324 			uattr->data_s64, &pbundle->bundle);
325 		if (IS_ERR(o_attr->uobject))
326 			return PTR_ERR(o_attr->uobject);
327 		__set_bit(attr_bkey, pbundle->uobj_finalize);
328 
329 		if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
330 			unsigned int uattr_idx = uattr - pbundle->uattrs;
331 			s64 id = o_attr->uobject->id;
332 
333 			/* Copy the allocated id to the user-space */
334 			if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
335 				return -EFAULT;
336 		}
337 
338 		break;
339 
340 	case UVERBS_ATTR_TYPE_RAW_FD:
341 		if (uattr->attr_data.reserved || uattr->len != 0 ||
342 		    uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)
343 			return -EINVAL;
344 		/* _uverbs_get_const_signed() is the accessor */
345 		e->ptr_attr.data = uattr->data_s64;
346 		break;
347 
348 	case UVERBS_ATTR_TYPE_IDRS_ARRAY:
349 		return uverbs_process_idrs_array(pbundle, attr_uapi,
350 						 &e->objs_arr_attr, uattr,
351 						 attr_bkey);
352 	default:
353 		return -EOPNOTSUPP;
354 	}
355 
356 	return 0;
357 }
358 
359 /*
360  * We search the radix tree with the method prefix and now we want to fast
361  * search the suffix bits to get a particular attribute pointer. It is not
362  * totally clear to me if this breaks the radix tree encasulation or not, but
363  * it uses the iter data to determine if the method iter points at the same
364  * chunk that will store the attribute, if so it just derefs it directly. By
365  * construction in most kernel configs the method and attrs will all fit in a
366  * single radix chunk, so in most cases this will have no search. Other cases
367  * this falls back to a full search.
368  */
369 static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
370 					     u32 attr_key)
371 {
372 	void __rcu **slot;
373 
374 	if (likely(attr_key < pbundle->radix_slots_len)) {
375 		void *entry;
376 
377 		slot = pbundle->radix_slots + attr_key;
378 		entry = rcu_dereference_raw(*slot);
379 		if (likely(!radix_tree_is_internal_node(entry) && entry))
380 			return slot;
381 	}
382 
383 	return radix_tree_lookup_slot(pbundle->radix,
384 				      pbundle->method_key | attr_key);
385 }
386 
387 static int uverbs_set_attr(struct bundle_priv *pbundle,
388 			   struct ib_uverbs_attr *uattr)
389 {
390 	u32 attr_key = uapi_key_attr(uattr->attr_id);
391 	u32 attr_bkey = uapi_bkey_attr(attr_key);
392 	const struct uverbs_api_attr *attr;
393 	void __rcu **slot;
394 	int ret;
395 
396 	slot = uapi_get_attr_for_method(pbundle, attr_key);
397 	if (!slot) {
398 		/*
399 		 * Kernel does not support the attribute but user-space says it
400 		 * is mandatory
401 		 */
402 		if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
403 			return -EPROTONOSUPPORT;
404 		return 0;
405 	}
406 	attr = rcu_dereference_protected(*slot, true);
407 
408 	/* Reject duplicate attributes from user-space */
409 	if (test_bit(attr_bkey, pbundle->bundle.attr_present))
410 		return -EINVAL;
411 
412 	ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
413 	if (ret)
414 		return ret;
415 
416 	__set_bit(attr_bkey, pbundle->bundle.attr_present);
417 
418 	return 0;
419 }
420 
421 static int ib_uverbs_run_method(struct bundle_priv *pbundle,
422 				unsigned int num_attrs)
423 {
424 	int (*handler)(struct uverbs_attr_bundle *attrs);
425 	size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
426 	unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
427 	unsigned int i;
428 	int ret;
429 
430 	/* See uverbs_disassociate_api() */
431 	handler = srcu_dereference(
432 		pbundle->method_elm->handler,
433 		&pbundle->bundle.ufile->device->disassociate_srcu);
434 	if (!handler)
435 		return -EIO;
436 
437 	pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
438 	if (IS_ERR(pbundle->uattrs))
439 		return PTR_ERR(pbundle->uattrs);
440 	if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
441 		return -EFAULT;
442 
443 	for (i = 0; i != num_attrs; i++) {
444 		ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
445 		if (unlikely(ret))
446 			return ret;
447 	}
448 
449 	/* User space did not provide all the mandatory attributes */
450 	if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
451 				    pbundle->bundle.attr_present,
452 				    pbundle->method_elm->key_bitmap_len)))
453 		return -EINVAL;
454 
455 	if (pbundle->method_elm->has_udata)
456 		uverbs_fill_udata(&pbundle->bundle,
457 				  &pbundle->bundle.driver_udata,
458 				  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
459 	else
460 		pbundle->bundle.driver_udata = (struct ib_udata){};
461 
462 	if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
463 		struct uverbs_obj_attr *destroy_attr =
464 			&pbundle->bundle.attrs[destroy_bkey].obj_attr;
465 
466 		ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle);
467 		if (ret)
468 			return ret;
469 		__clear_bit(destroy_bkey, pbundle->uobj_finalize);
470 
471 		ret = handler(&pbundle->bundle);
472 		uobj_put_destroy(destroy_attr->uobject);
473 	} else {
474 		ret = handler(&pbundle->bundle);
475 	}
476 
477 	/*
478 	 * Until the drivers are revised to use the bundle directly we have to
479 	 * assume that the driver wrote to its UHW_OUT and flag userspace
480 	 * appropriately.
481 	 */
482 	if (!ret && pbundle->method_elm->has_udata) {
483 		const struct uverbs_attr *attr =
484 			uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT);
485 
486 		if (!IS_ERR(attr))
487 			ret = uverbs_set_output(&pbundle->bundle, attr);
488 	}
489 
490 	/*
491 	 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
492 	 * not invoke the method because the request is not supported.  No
493 	 * other cases should return this code.
494 	 */
495 	if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
496 		return -EINVAL;
497 
498 	return ret;
499 }
500 
501 static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
502 {
503 	unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
504 	struct bundle_alloc_head *memblock;
505 	unsigned int i;
506 
507 	/* fast path for simple uobjects */
508 	i = -1;
509 	while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
510 				  i + 1)) < key_bitmap_len) {
511 		struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
512 
513 		uverbs_finalize_object(
514 			attr->obj_attr.uobject,
515 			attr->obj_attr.attr_elm->spec.u.obj.access,
516 			test_bit(i, pbundle->uobj_hw_obj_valid),
517 			commit,
518 			&pbundle->bundle);
519 	}
520 
521 	i = -1;
522 	while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
523 				  i + 1)) < key_bitmap_len) {
524 		struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
525 		const struct uverbs_api_attr *attr_uapi;
526 		void __rcu **slot;
527 
528 		slot = uapi_get_attr_for_method(
529 			pbundle,
530 			pbundle->method_key | uapi_bkey_to_key_attr(i));
531 		if (WARN_ON(!slot))
532 			continue;
533 
534 		attr_uapi = rcu_dereference_protected(*slot, true);
535 
536 		if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
537 			uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
538 					       commit, &pbundle->bundle);
539 		}
540 	}
541 
542 	for (memblock = pbundle->allocated_mem; memblock;) {
543 		struct bundle_alloc_head *tmp = memblock;
544 
545 		memblock = memblock->next;
546 		kvfree(tmp);
547 	}
548 }
549 
550 static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
551 			       struct ib_uverbs_ioctl_hdr *hdr,
552 			       struct ib_uverbs_attr __user *user_attrs)
553 {
554 	const struct uverbs_api_ioctl_method *method_elm;
555 	struct uverbs_api *uapi = ufile->device->uapi;
556 	struct radix_tree_iter attrs_iter;
557 	struct bundle_priv *pbundle;
558 	struct bundle_priv onstack;
559 	void __rcu **slot;
560 	int ret;
561 
562 	if (unlikely(hdr->driver_id != uapi->driver_id))
563 		return -EINVAL;
564 
565 	slot = radix_tree_iter_lookup(
566 		&uapi->radix, &attrs_iter,
567 		uapi_key_obj(hdr->object_id) |
568 			uapi_key_ioctl_method(hdr->method_id));
569 	if (unlikely(!slot))
570 		return -EPROTONOSUPPORT;
571 	method_elm = rcu_dereference_protected(*slot, true);
572 
573 	if (!method_elm->use_stack) {
574 		pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
575 		if (!pbundle)
576 			return -ENOMEM;
577 		pbundle->internal_avail =
578 			method_elm->bundle_size -
579 			offsetof(struct bundle_priv, internal_buffer);
580 		pbundle->alloc_head.next = NULL;
581 		pbundle->allocated_mem = &pbundle->alloc_head;
582 	} else {
583 		pbundle = &onstack;
584 		pbundle->internal_avail = sizeof(pbundle->internal_buffer);
585 		pbundle->allocated_mem = NULL;
586 	}
587 
588 	/* Space for the pbundle->bundle.attrs flex array */
589 	pbundle->method_elm = method_elm;
590 	pbundle->method_key = attrs_iter.index;
591 	pbundle->bundle.ufile = ufile;
592 	pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
593 	pbundle->radix = &uapi->radix;
594 	pbundle->radix_slots = slot;
595 	pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
596 	pbundle->user_attrs = user_attrs;
597 
598 	pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
599 					       sizeof(*pbundle->bundle.attrs),
600 				       sizeof(*pbundle->internal_buffer));
601 	memset(pbundle->bundle.attr_present, 0,
602 	       sizeof(pbundle->bundle.attr_present));
603 	memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
604 	memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
605 	memset(pbundle->uobj_hw_obj_valid, 0,
606 	       sizeof(pbundle->uobj_hw_obj_valid));
607 
608 	ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
609 	bundle_destroy(pbundle, ret == 0);
610 	return ret;
611 }
612 
613 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
614 {
615 	struct ib_uverbs_file *file = filp->private_data;
616 	struct ib_uverbs_ioctl_hdr __user *user_hdr =
617 		(struct ib_uverbs_ioctl_hdr __user *)arg;
618 	struct ib_uverbs_ioctl_hdr hdr;
619 	int srcu_key;
620 	int err;
621 
622 	if (unlikely(cmd != RDMA_VERBS_IOCTL))
623 		return -ENOIOCTLCMD;
624 
625 	err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
626 	if (err)
627 		return -EFAULT;
628 
629 	if (hdr.length > PAGE_SIZE ||
630 	    hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
631 		return -EINVAL;
632 
633 	if (hdr.reserved1 || hdr.reserved2)
634 		return -EPROTONOSUPPORT;
635 
636 	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
637 	err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
638 	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
639 	return err;
640 }
641 
642 int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
643 		       size_t idx, u64 allowed_bits)
644 {
645 	const struct uverbs_attr *attr;
646 	u64 flags;
647 
648 	attr = uverbs_attr_get(attrs_bundle, idx);
649 	/* Missing attribute means 0 flags */
650 	if (IS_ERR(attr)) {
651 		*to = 0;
652 		return 0;
653 	}
654 
655 	/*
656 	 * New userspace code should use 8 bytes to pass flags, but we
657 	 * transparently support old userspaces that were using 4 bytes as
658 	 * well.
659 	 */
660 	if (attr->ptr_attr.len == 8)
661 		flags = attr->ptr_attr.data;
662 	else if (attr->ptr_attr.len == 4)
663 		flags = *(u32 *)&attr->ptr_attr.data;
664 	else
665 		return -EINVAL;
666 
667 	if (flags & ~allowed_bits)
668 		return -EINVAL;
669 
670 	*to = flags;
671 	return 0;
672 }
673 EXPORT_SYMBOL(uverbs_get_flags64);
674 
675 int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
676 		       size_t idx, u64 allowed_bits)
677 {
678 	u64 flags;
679 	int ret;
680 
681 	ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
682 	if (ret)
683 		return ret;
684 
685 	if (flags > U32_MAX)
686 		return -EINVAL;
687 	*to = flags;
688 
689 	return 0;
690 }
691 EXPORT_SYMBOL(uverbs_get_flags32);
692 
693 /*
694  * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
695  * This is primarily used to convert the UVERBS_ATTR_UHW() into the
696  * ib_udata format used by the drivers.
697  */
698 void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
699 		       struct ib_udata *udata, unsigned int attr_in,
700 		       unsigned int attr_out)
701 {
702 	struct bundle_priv *pbundle =
703 		container_of(bundle, struct bundle_priv, bundle);
704 	const struct uverbs_attr *in =
705 		uverbs_attr_get(&pbundle->bundle, attr_in);
706 	const struct uverbs_attr *out =
707 		uverbs_attr_get(&pbundle->bundle, attr_out);
708 
709 	if (!IS_ERR(in)) {
710 		udata->inlen = in->ptr_attr.len;
711 		if (uverbs_attr_ptr_is_inline(in))
712 			udata->inbuf =
713 				&pbundle->user_attrs[in->ptr_attr.uattr_idx]
714 					 .data;
715 		else
716 			udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
717 	} else {
718 		udata->inbuf = NULL;
719 		udata->inlen = 0;
720 	}
721 
722 	if (!IS_ERR(out)) {
723 		udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
724 		udata->outlen = out->ptr_attr.len;
725 	} else {
726 		udata->outbuf = NULL;
727 		udata->outlen = 0;
728 	}
729 }
730 
731 int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
732 		   const void *from, size_t size)
733 {
734 	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
735 	size_t min_size;
736 
737 	if (IS_ERR(attr))
738 		return PTR_ERR(attr);
739 
740 	min_size = min_t(size_t, attr->ptr_attr.len, size);
741 	if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
742 		return -EFAULT;
743 
744 	return uverbs_set_output(bundle, attr);
745 }
746 EXPORT_SYMBOL(uverbs_copy_to);
747 
748 
749 /*
750  * This is only used if the caller has directly used copy_to_use to write the
751  * data.  It signals to user space that the buffer is filled in.
752  */
753 int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
754 {
755 	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
756 
757 	if (IS_ERR(attr))
758 		return PTR_ERR(attr);
759 
760 	return uverbs_set_output(bundle, attr);
761 }
762 
763 int _uverbs_get_const_signed(s64 *to,
764 			     const struct uverbs_attr_bundle *attrs_bundle,
765 			     size_t idx, s64 lower_bound, u64 upper_bound,
766 			     s64  *def_val)
767 {
768 	const struct uverbs_attr *attr;
769 
770 	attr = uverbs_attr_get(attrs_bundle, idx);
771 	if (IS_ERR(attr)) {
772 		if ((PTR_ERR(attr) != -ENOENT) || !def_val)
773 			return PTR_ERR(attr);
774 
775 		*to = *def_val;
776 	} else {
777 		*to = attr->ptr_attr.data;
778 	}
779 
780 	if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
781 		return -EINVAL;
782 
783 	return 0;
784 }
785 EXPORT_SYMBOL(_uverbs_get_const_signed);
786 
787 int _uverbs_get_const_unsigned(u64 *to,
788 			       const struct uverbs_attr_bundle *attrs_bundle,
789 			       size_t idx, u64 upper_bound, u64 *def_val)
790 {
791 	const struct uverbs_attr *attr;
792 
793 	attr = uverbs_attr_get(attrs_bundle, idx);
794 	if (IS_ERR(attr)) {
795 		if ((PTR_ERR(attr) != -ENOENT) || !def_val)
796 			return PTR_ERR(attr);
797 
798 		*to = *def_val;
799 	} else {
800 		*to = attr->ptr_attr.data;
801 	}
802 
803 	if (*to > upper_bound)
804 		return -EINVAL;
805 
806 	return 0;
807 }
808 EXPORT_SYMBOL(_uverbs_get_const_unsigned);
809 
810 int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
811 				  size_t idx, const void *from, size_t size)
812 {
813 	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
814 
815 	if (IS_ERR(attr))
816 		return PTR_ERR(attr);
817 
818 	if (size < attr->ptr_attr.len) {
819 		if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
820 			       attr->ptr_attr.len - size))
821 			return -EFAULT;
822 	}
823 	return uverbs_copy_to(bundle, idx, from, size);
824 }
825 EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
826 
827 /* Once called an abort will call through to the type's destroy_hw() */
828 void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
829 				 u16 idx)
830 {
831 	struct bundle_priv *pbundle =
832 		container_of(bundle, struct bundle_priv, bundle);
833 
834 	__set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
835 		  pbundle->uobj_hw_obj_valid);
836 }
837 EXPORT_SYMBOL(uverbs_finalize_uobj_create);
838