1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
4  */
5 #include <rdma/uverbs_ioctl.h>
6 #include <rdma/rdma_user_ioctl.h>
7 #include <linux/bitops.h>
8 #include "rdma_core.h"
9 #include "uverbs.h"
10 
11 static int ib_uverbs_notsupp(struct uverbs_attr_bundle *attrs)
12 {
13 	return -EOPNOTSUPP;
14 }
15 
16 static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
17 {
18 	void *elm;
19 	int rc;
20 
21 	if (key == UVERBS_API_KEY_ERR)
22 		return ERR_PTR(-EOVERFLOW);
23 
24 	elm = kzalloc(alloc_size, GFP_KERNEL);
25 	if (!elm)
26 		return ERR_PTR(-ENOMEM);
27 	rc = radix_tree_insert(&uapi->radix, key, elm);
28 	if (rc) {
29 		kfree(elm);
30 		return ERR_PTR(rc);
31 	}
32 
33 	return elm;
34 }
35 
36 static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,
37 			      size_t alloc_size, bool *exists)
38 {
39 	void *elm;
40 
41 	elm = uapi_add_elm(uapi, key, alloc_size);
42 	if (!IS_ERR(elm)) {
43 		*exists = false;
44 		return elm;
45 	}
46 
47 	if (elm != ERR_PTR(-EEXIST))
48 		return elm;
49 
50 	elm = radix_tree_lookup(&uapi->radix, key);
51 	if (WARN_ON(!elm))
52 		return ERR_PTR(-EINVAL);
53 	*exists = true;
54 	return elm;
55 }
56 
57 static int uapi_create_write(struct uverbs_api *uapi,
58 			     struct ib_device *ibdev,
59 			     const struct uapi_definition *def,
60 			     u32 obj_key,
61 			     u32 *cur_method_key)
62 {
63 	struct uverbs_api_write_method *method_elm;
64 	u32 method_key = obj_key;
65 	bool exists;
66 
67 	if (def->write.is_ex)
68 		method_key |= uapi_key_write_ex_method(def->write.command_num);
69 	else
70 		method_key |= uapi_key_write_method(def->write.command_num);
71 
72 	method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
73 				      &exists);
74 	if (IS_ERR(method_elm))
75 		return PTR_ERR(method_elm);
76 
77 	if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex)))
78 		return -EINVAL;
79 
80 	method_elm->is_ex = def->write.is_ex;
81 	method_elm->handler = def->func_write;
82 	if (!def->write.is_ex)
83 		method_elm->disabled = !(ibdev->uverbs_cmd_mask &
84 					 BIT_ULL(def->write.command_num));
85 
86 	if (!def->write.is_ex && def->func_write) {
87 		method_elm->has_udata = def->write.has_udata;
88 		method_elm->has_resp = def->write.has_resp;
89 		method_elm->req_size = def->write.req_size;
90 		method_elm->resp_size = def->write.resp_size;
91 	}
92 
93 	*cur_method_key = method_key;
94 	return 0;
95 }
96 
97 static int uapi_merge_method(struct uverbs_api *uapi,
98 			     struct uverbs_api_object *obj_elm, u32 obj_key,
99 			     const struct uverbs_method_def *method,
100 			     bool is_driver)
101 {
102 	u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
103 	struct uverbs_api_ioctl_method *method_elm;
104 	unsigned int i;
105 	bool exists;
106 
107 	if (!method->attrs)
108 		return 0;
109 
110 	method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
111 				      &exists);
112 	if (IS_ERR(method_elm))
113 		return PTR_ERR(method_elm);
114 	if (exists) {
115 		/*
116 		 * This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE
117 		 */
118 		if (WARN_ON(method->handler))
119 			return -EINVAL;
120 	} else {
121 		WARN_ON(!method->handler);
122 		rcu_assign_pointer(method_elm->handler, method->handler);
123 		if (method->handler != uverbs_destroy_def_handler)
124 			method_elm->driver_method = is_driver;
125 	}
126 
127 	for (i = 0; i != method->num_attrs; i++) {
128 		const struct uverbs_attr_def *attr = (*method->attrs)[i];
129 		struct uverbs_api_attr *attr_slot;
130 
131 		if (!attr)
132 			continue;
133 
134 		/*
135 		 * ENUM_IN contains the 'ids' pointer to the driver's .rodata,
136 		 * so if it is specified by a driver then it always makes this
137 		 * into a driver method.
138 		 */
139 		if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
140 			method_elm->driver_method |= is_driver;
141 
142 		/*
143 		 * Like other uobject based things we only support a single
144 		 * uobject being NEW'd or DESTROY'd
145 		 */
146 		if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
147 			u8 access = attr->attr.u2.objs_arr.access;
148 
149 			if (WARN_ON(access == UVERBS_ACCESS_NEW ||
150 				    access == UVERBS_ACCESS_DESTROY))
151 				return -EINVAL;
152 		}
153 
154 		attr_slot =
155 			uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
156 				     sizeof(*attr_slot));
157 		/* Attributes are not allowed to be modified by drivers */
158 		if (IS_ERR(attr_slot))
159 			return PTR_ERR(attr_slot);
160 
161 		attr_slot->spec = attr->attr;
162 	}
163 
164 	return 0;
165 }
166 
167 static int uapi_merge_obj_tree(struct uverbs_api *uapi,
168 			       const struct uverbs_object_def *obj,
169 			       bool is_driver)
170 {
171 	struct uverbs_api_object *obj_elm;
172 	unsigned int i;
173 	u32 obj_key;
174 	bool exists;
175 	int rc;
176 
177 	obj_key = uapi_key_obj(obj->id);
178 	obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);
179 	if (IS_ERR(obj_elm))
180 		return PTR_ERR(obj_elm);
181 
182 	if (obj->type_attrs) {
183 		if (WARN_ON(obj_elm->type_attrs))
184 			return -EINVAL;
185 
186 		obj_elm->id = obj->id;
187 		obj_elm->type_attrs = obj->type_attrs;
188 		obj_elm->type_class = obj->type_attrs->type_class;
189 		/*
190 		 * Today drivers are only permitted to use idr_class and
191 		 * fd_class types. We can revoke the IDR types during
192 		 * disassociation, and the FD types require the driver to use
193 		 * struct file_operations.owner to prevent the driver module
194 		 * code from unloading while the file is open. This provides
195 		 * enough safety that uverbs_uobject_fd_release() will
196 		 * continue to work.  Drivers using FD are responsible to
197 		 * handle disassociation of the device on their own.
198 		 */
199 		if (WARN_ON(is_driver &&
200 			    obj->type_attrs->type_class != &uverbs_idr_class &&
201 			    obj->type_attrs->type_class != &uverbs_fd_class))
202 			return -EINVAL;
203 	}
204 
205 	if (!obj->methods)
206 		return 0;
207 
208 	for (i = 0; i != obj->num_methods; i++) {
209 		const struct uverbs_method_def *method = (*obj->methods)[i];
210 
211 		if (!method)
212 			continue;
213 
214 		rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
215 				       is_driver);
216 		if (rc)
217 			return rc;
218 	}
219 
220 	return 0;
221 }
222 
223 static int uapi_disable_elm(struct uverbs_api *uapi,
224 			    const struct uapi_definition *def,
225 			    u32 obj_key,
226 			    u32 method_key)
227 {
228 	bool exists;
229 
230 	if (def->scope == UAPI_SCOPE_OBJECT) {
231 		struct uverbs_api_object *obj_elm;
232 
233 		obj_elm = uapi_add_get_elm(
234 			uapi, obj_key, sizeof(*obj_elm), &exists);
235 		if (IS_ERR(obj_elm))
236 			return PTR_ERR(obj_elm);
237 		obj_elm->disabled = 1;
238 		return 0;
239 	}
240 
241 	if (def->scope == UAPI_SCOPE_METHOD &&
242 	    uapi_key_is_ioctl_method(method_key)) {
243 		struct uverbs_api_ioctl_method *method_elm;
244 
245 		method_elm = uapi_add_get_elm(uapi, method_key,
246 					      sizeof(*method_elm), &exists);
247 		if (IS_ERR(method_elm))
248 			return PTR_ERR(method_elm);
249 		method_elm->disabled = 1;
250 		return 0;
251 	}
252 
253 	if (def->scope == UAPI_SCOPE_METHOD &&
254 	    (uapi_key_is_write_method(method_key) ||
255 	     uapi_key_is_write_ex_method(method_key))) {
256 		struct uverbs_api_write_method *write_elm;
257 
258 		write_elm = uapi_add_get_elm(uapi, method_key,
259 					     sizeof(*write_elm), &exists);
260 		if (IS_ERR(write_elm))
261 			return PTR_ERR(write_elm);
262 		write_elm->disabled = 1;
263 		return 0;
264 	}
265 
266 	WARN_ON(true);
267 	return -EINVAL;
268 }
269 
270 static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,
271 			  const struct uapi_definition *def_list,
272 			  bool is_driver)
273 {
274 	const struct uapi_definition *def = def_list;
275 	u32 cur_obj_key = UVERBS_API_KEY_ERR;
276 	u32 cur_method_key = UVERBS_API_KEY_ERR;
277 	bool exists;
278 	int rc;
279 
280 	if (!def_list)
281 		return 0;
282 
283 	for (;; def++) {
284 		switch ((enum uapi_definition_kind)def->kind) {
285 		case UAPI_DEF_CHAIN:
286 			rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);
287 			if (rc)
288 				return rc;
289 			continue;
290 
291 		case UAPI_DEF_CHAIN_OBJ_TREE:
292 			if (WARN_ON(def->object_start.object_id !=
293 				    def->chain_obj_tree->id))
294 				return -EINVAL;
295 
296 			cur_obj_key = uapi_key_obj(def->object_start.object_id);
297 			rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,
298 						 is_driver);
299 			if (rc)
300 				return rc;
301 			continue;
302 
303 		case UAPI_DEF_END:
304 			return 0;
305 
306 		case UAPI_DEF_IS_SUPPORTED_DEV_FN: {
307 			void **ibdev_fn =
308 				(void *)(&ibdev->ops) + def->needs_fn_offset;
309 
310 			if (*ibdev_fn)
311 				continue;
312 			rc = uapi_disable_elm(
313 				uapi, def, cur_obj_key, cur_method_key);
314 			if (rc)
315 				return rc;
316 			continue;
317 		}
318 
319 		case UAPI_DEF_IS_SUPPORTED_FUNC:
320 			if (def->func_is_supported(ibdev))
321 				continue;
322 			rc = uapi_disable_elm(
323 				uapi, def, cur_obj_key, cur_method_key);
324 			if (rc)
325 				return rc;
326 			continue;
327 
328 		case UAPI_DEF_OBJECT_START: {
329 			struct uverbs_api_object *obj_elm;
330 
331 			cur_obj_key = uapi_key_obj(def->object_start.object_id);
332 			obj_elm = uapi_add_get_elm(uapi, cur_obj_key,
333 						   sizeof(*obj_elm), &exists);
334 			if (IS_ERR(obj_elm))
335 				return PTR_ERR(obj_elm);
336 			continue;
337 		}
338 
339 		case UAPI_DEF_WRITE:
340 			rc = uapi_create_write(
341 				uapi, ibdev, def, cur_obj_key, &cur_method_key);
342 			if (rc)
343 				return rc;
344 			continue;
345 		}
346 		WARN_ON(true);
347 		return -EINVAL;
348 	}
349 }
350 
351 static int
352 uapi_finalize_ioctl_method(struct uverbs_api *uapi,
353 			   struct uverbs_api_ioctl_method *method_elm,
354 			   u32 method_key)
355 {
356 	struct radix_tree_iter iter;
357 	unsigned int num_attrs = 0;
358 	unsigned int max_bkey = 0;
359 	bool single_uobj = false;
360 	void __rcu **slot;
361 
362 	method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
363 	radix_tree_for_each_slot (slot, &uapi->radix, &iter,
364 				  uapi_key_attrs_start(method_key)) {
365 		struct uverbs_api_attr *elm =
366 			rcu_dereference_protected(*slot, true);
367 		u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
368 		u32 attr_bkey = uapi_bkey_attr(attr_key);
369 		u8 type = elm->spec.type;
370 
371 		if (uapi_key_attr_to_ioctl_method(iter.index) !=
372 		    uapi_key_attr_to_ioctl_method(method_key))
373 			break;
374 
375 		if (elm->spec.mandatory)
376 			__set_bit(attr_bkey, method_elm->attr_mandatory);
377 
378 		if (elm->spec.is_udata)
379 			method_elm->has_udata = true;
380 
381 		if (type == UVERBS_ATTR_TYPE_IDR ||
382 		    type == UVERBS_ATTR_TYPE_FD) {
383 			u8 access = elm->spec.u.obj.access;
384 
385 			/*
386 			 * Verbs specs may only have one NEW/DESTROY, we don't
387 			 * have the infrastructure to abort multiple NEW's or
388 			 * cope with multiple DESTROY failure.
389 			 */
390 			if (access == UVERBS_ACCESS_NEW ||
391 			    access == UVERBS_ACCESS_DESTROY) {
392 				if (WARN_ON(single_uobj))
393 					return -EINVAL;
394 
395 				single_uobj = true;
396 				if (WARN_ON(!elm->spec.mandatory))
397 					return -EINVAL;
398 			}
399 
400 			if (access == UVERBS_ACCESS_DESTROY)
401 				method_elm->destroy_bkey = attr_bkey;
402 		}
403 
404 		max_bkey = max(max_bkey, attr_bkey);
405 		num_attrs++;
406 	}
407 
408 	method_elm->key_bitmap_len = max_bkey + 1;
409 	WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);
410 
411 	uapi_compute_bundle_size(method_elm, num_attrs);
412 	return 0;
413 }
414 
415 static int uapi_finalize(struct uverbs_api *uapi)
416 {
417 	const struct uverbs_api_write_method **data;
418 	unsigned long max_write_ex = 0;
419 	unsigned long max_write = 0;
420 	struct radix_tree_iter iter;
421 	void __rcu **slot;
422 	int rc;
423 	int i;
424 
425 	radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
426 		struct uverbs_api_ioctl_method *method_elm =
427 			rcu_dereference_protected(*slot, true);
428 
429 		if (uapi_key_is_ioctl_method(iter.index)) {
430 			rc = uapi_finalize_ioctl_method(uapi, method_elm,
431 							iter.index);
432 			if (rc)
433 				return rc;
434 		}
435 
436 		if (uapi_key_is_write_method(iter.index))
437 			max_write = max(max_write,
438 					iter.index & UVERBS_API_ATTR_KEY_MASK);
439 		if (uapi_key_is_write_ex_method(iter.index))
440 			max_write_ex =
441 				max(max_write_ex,
442 				    iter.index & UVERBS_API_ATTR_KEY_MASK);
443 	}
444 
445 	uapi->notsupp_method.handler = ib_uverbs_notsupp;
446 	uapi->num_write = max_write + 1;
447 	uapi->num_write_ex = max_write_ex + 1;
448 	data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
449 			     sizeof(*uapi->write_methods), GFP_KERNEL);
450 	for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
451 		data[i] = &uapi->notsupp_method;
452 	uapi->write_methods = data;
453 	uapi->write_ex_methods = data + uapi->num_write;
454 
455 	radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
456 		if (uapi_key_is_write_method(iter.index))
457 			uapi->write_methods[iter.index &
458 					    UVERBS_API_ATTR_KEY_MASK] =
459 				rcu_dereference_protected(*slot, true);
460 		if (uapi_key_is_write_ex_method(iter.index))
461 			uapi->write_ex_methods[iter.index &
462 					       UVERBS_API_ATTR_KEY_MASK] =
463 				rcu_dereference_protected(*slot, true);
464 	}
465 
466 	return 0;
467 }
468 
469 static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)
470 {
471 	struct radix_tree_iter iter;
472 	void __rcu **slot;
473 
474 	radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {
475 		if (iter.index > last)
476 			return;
477 		kfree(rcu_dereference_protected(*slot, true));
478 		radix_tree_iter_delete(&uapi->radix, &iter, slot);
479 	}
480 }
481 
482 static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)
483 {
484 	uapi_remove_range(uapi, obj_key,
485 			  obj_key | UVERBS_API_METHOD_KEY_MASK |
486 				  UVERBS_API_ATTR_KEY_MASK);
487 }
488 
489 static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)
490 {
491 	uapi_remove_range(uapi, method_key,
492 			  method_key | UVERBS_API_ATTR_KEY_MASK);
493 }
494 
495 
496 static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)
497 {
498 	if (spec->type == UVERBS_ATTR_TYPE_IDR ||
499 	    spec->type == UVERBS_ATTR_TYPE_FD)
500 		return spec->u.obj.obj_type;
501 	if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)
502 		return spec->u2.objs_arr.obj_type;
503 	return UVERBS_API_KEY_ERR;
504 }
505 
506 static void uapi_key_okay(u32 key)
507 {
508 	unsigned int count = 0;
509 
510 	if (uapi_key_is_object(key))
511 		count++;
512 	if (uapi_key_is_ioctl_method(key))
513 		count++;
514 	if (uapi_key_is_write_method(key))
515 		count++;
516 	if (uapi_key_is_write_ex_method(key))
517 		count++;
518 	if (uapi_key_is_attr(key))
519 		count++;
520 	WARN(count != 1, "Bad count %u key=%x", count, key);
521 }
522 
523 static void uapi_finalize_disable(struct uverbs_api *uapi)
524 {
525 	struct radix_tree_iter iter;
526 	u32 starting_key = 0;
527 	bool scan_again = false;
528 	void __rcu **slot;
529 
530 again:
531 	radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {
532 		uapi_key_okay(iter.index);
533 
534 		if (uapi_key_is_object(iter.index)) {
535 			struct uverbs_api_object *obj_elm =
536 				rcu_dereference_protected(*slot, true);
537 
538 			if (obj_elm->disabled) {
539 				/* Have to check all the attrs again */
540 				scan_again = true;
541 				starting_key = iter.index;
542 				uapi_remove_object(uapi, iter.index);
543 				goto again;
544 			}
545 			continue;
546 		}
547 
548 		if (uapi_key_is_ioctl_method(iter.index)) {
549 			struct uverbs_api_ioctl_method *method_elm =
550 				rcu_dereference_protected(*slot, true);
551 
552 			if (method_elm->disabled) {
553 				starting_key = iter.index;
554 				uapi_remove_method(uapi, iter.index);
555 				goto again;
556 			}
557 			continue;
558 		}
559 
560 		if (uapi_key_is_write_method(iter.index) ||
561 		    uapi_key_is_write_ex_method(iter.index)) {
562 			struct uverbs_api_write_method *method_elm =
563 				rcu_dereference_protected(*slot, true);
564 
565 			if (method_elm->disabled) {
566 				kfree(method_elm);
567 				radix_tree_iter_delete(&uapi->radix, &iter, slot);
568 			}
569 			continue;
570 		}
571 
572 		if (uapi_key_is_attr(iter.index)) {
573 			struct uverbs_api_attr *attr_elm =
574 				rcu_dereference_protected(*slot, true);
575 			const struct uverbs_api_object *tmp_obj;
576 			u32 obj_key;
577 
578 			/*
579 			 * If the method has a mandatory object handle
580 			 * attribute which relies on an object which is not
581 			 * present then the entire method is uncallable.
582 			 */
583 			if (!attr_elm->spec.mandatory)
584 				continue;
585 			obj_key = uapi_get_obj_id(&attr_elm->spec);
586 			if (obj_key == UVERBS_API_KEY_ERR)
587 				continue;
588 			tmp_obj = uapi_get_object(uapi, obj_key);
589 			if (IS_ERR(tmp_obj)) {
590 				if (PTR_ERR(tmp_obj) == -ENOMSG)
591 					continue;
592 			} else {
593 				if (!tmp_obj->disabled)
594 					continue;
595 			}
596 
597 			starting_key = iter.index;
598 			uapi_remove_method(
599 				uapi,
600 				iter.index & (UVERBS_API_OBJ_KEY_MASK |
601 					      UVERBS_API_METHOD_KEY_MASK));
602 			goto again;
603 		}
604 
605 		WARN_ON(false);
606 	}
607 
608 	if (!scan_again)
609 		return;
610 	scan_again = false;
611 	starting_key = 0;
612 	goto again;
613 }
614 
615 void uverbs_destroy_api(struct uverbs_api *uapi)
616 {
617 	if (!uapi)
618 		return;
619 
620 	uapi_remove_range(uapi, 0, U32_MAX);
621 	kfree(uapi->write_methods);
622 	kfree(uapi);
623 }
624 
625 static const struct uapi_definition uverbs_core_api[] = {
626 	UAPI_DEF_CHAIN(uverbs_def_obj_async_fd),
627 	UAPI_DEF_CHAIN(uverbs_def_obj_counters),
628 	UAPI_DEF_CHAIN(uverbs_def_obj_cq),
629 	UAPI_DEF_CHAIN(uverbs_def_obj_device),
630 	UAPI_DEF_CHAIN(uverbs_def_obj_dm),
631 	UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
632 	UAPI_DEF_CHAIN(uverbs_def_obj_intf),
633 	UAPI_DEF_CHAIN(uverbs_def_obj_mr),
634 	UAPI_DEF_CHAIN(uverbs_def_obj_qp),
635 	UAPI_DEF_CHAIN(uverbs_def_obj_srq),
636 	UAPI_DEF_CHAIN(uverbs_def_obj_wq),
637 	UAPI_DEF_CHAIN(uverbs_def_write_intf),
638 	{},
639 };
640 
641 struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
642 {
643 	struct uverbs_api *uapi;
644 	int rc;
645 
646 	uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
647 	if (!uapi)
648 		return ERR_PTR(-ENOMEM);
649 
650 	INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
651 	uapi->driver_id = ibdev->ops.driver_id;
652 
653 	rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
654 	if (rc)
655 		goto err;
656 	rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);
657 	if (rc)
658 		goto err;
659 
660 	uapi_finalize_disable(uapi);
661 	rc = uapi_finalize(uapi);
662 	if (rc)
663 		goto err;
664 
665 	return uapi;
666 err:
667 	if (rc != -ENOMEM)
668 		dev_err(&ibdev->dev,
669 			"Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
670 			rc);
671 
672 	uverbs_destroy_api(uapi);
673 	return ERR_PTR(rc);
674 }
675 
676 /*
677  * The pre version is done before destroying the HW objects, it only blocks
678  * off method access. All methods that require the ib_dev or the module data
679  * must test one of these assignments prior to continuing.
680  */
681 void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
682 {
683 	struct uverbs_api *uapi = uverbs_dev->uapi;
684 	struct radix_tree_iter iter;
685 	void __rcu **slot;
686 
687 	rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
688 
689 	radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
690 		if (uapi_key_is_ioctl_method(iter.index)) {
691 			struct uverbs_api_ioctl_method *method_elm =
692 				rcu_dereference_protected(*slot, true);
693 
694 			if (method_elm->driver_method)
695 				rcu_assign_pointer(method_elm->handler, NULL);
696 		}
697 	}
698 
699 	synchronize_srcu(&uverbs_dev->disassociate_srcu);
700 }
701 
702 /*
703  * Called when a driver disassociates from the ib_uverbs_device. The
704  * assumption is that the driver module will unload after. Replace everything
705  * related to the driver with NULL as a safety measure.
706  */
707 void uverbs_disassociate_api(struct uverbs_api *uapi)
708 {
709 	struct radix_tree_iter iter;
710 	void __rcu **slot;
711 
712 	radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
713 		if (uapi_key_is_object(iter.index)) {
714 			struct uverbs_api_object *object_elm =
715 				rcu_dereference_protected(*slot, true);
716 
717 			/*
718 			 * Some type_attrs are in the driver module. We don't
719 			 * bother to keep track of which since there should be
720 			 * no use of this after disassociate.
721 			 */
722 			object_elm->type_attrs = NULL;
723 		} else if (uapi_key_is_attr(iter.index)) {
724 			struct uverbs_api_attr *elm =
725 				rcu_dereference_protected(*slot, true);
726 
727 			if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
728 				elm->spec.u2.enum_def.ids = NULL;
729 		}
730 	}
731 }
732