1 /*
2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
4  * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
5  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 
36 #include <linux/file.h>
37 #include <linux/fs.h>
38 #include <linux/slab.h>
39 #include <linux/sched.h>
40 
41 #include <linux/uaccess.h>
42 
43 #include <rdma/uverbs_types.h>
44 #include <rdma/uverbs_std_types.h>
45 #include "rdma_core.h"
46 
47 #include "uverbs.h"
48 #include "core_priv.h"
49 
50 /*
51  * Copy a response to userspace. If the provided 'resp' is larger than the
52  * user buffer it is silently truncated. If the user provided a larger buffer
53  * then the trailing portion is zero filled.
54  *
55  * These semantics are intended to support future extension of the output
56  * structures.
57  */
58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,
59 			   size_t resp_len)
60 {
61 	int ret;
62 
63 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
64 		return uverbs_copy_to_struct_or_zero(
65 			attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);
66 
67 	if (copy_to_user(attrs->ucore.outbuf, resp,
68 			 min(attrs->ucore.outlen, resp_len)))
69 		return -EFAULT;
70 
71 	if (resp_len < attrs->ucore.outlen) {
72 		/*
73 		 * Zero fill any extra memory that user
74 		 * space might have provided.
75 		 */
76 		ret = clear_user(attrs->ucore.outbuf + resp_len,
77 				 attrs->ucore.outlen - resp_len);
78 		if (ret)
79 			return -EFAULT;
80 	}
81 
82 	return 0;
83 }
84 
85 /*
86  * Copy a request from userspace. If the provided 'req' is larger than the
87  * user buffer then the user buffer is zero extended into the 'req'. If 'req'
88  * is smaller than the user buffer then the uncopied bytes in the user buffer
89  * must be zero.
90  */
91 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,
92 			  size_t req_len)
93 {
94 	if (copy_from_user(req, attrs->ucore.inbuf,
95 			   min(attrs->ucore.inlen, req_len)))
96 		return -EFAULT;
97 
98 	if (attrs->ucore.inlen < req_len) {
99 		memset(req + attrs->ucore.inlen, 0,
100 		       req_len - attrs->ucore.inlen);
101 	} else if (attrs->ucore.inlen > req_len) {
102 		if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
103 					  attrs->ucore.inlen - req_len))
104 			return -EOPNOTSUPP;
105 	}
106 	return 0;
107 }
108 
109 /*
110  * Generate the value for the 'response_length' protocol used by write_ex.
111  * This is the number of bytes the kernel actually wrote. Userspace can use
112  * this to detect what structure members in the response the kernel
113  * understood.
114  */
115 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
116 				  size_t resp_len)
117 {
118 	return min_t(size_t, attrs->ucore.outlen, resp_len);
119 }
120 
121 /*
122  * The iterator version of the request interface is for handlers that need to
123  * step over a flex array at the end of a command header.
124  */
125 struct uverbs_req_iter {
126 	const void __user *cur;
127 	const void __user *end;
128 };
129 
130 static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
131 				struct uverbs_req_iter *iter,
132 				void *req,
133 				size_t req_len)
134 {
135 	if (attrs->ucore.inlen < req_len)
136 		return -ENOSPC;
137 
138 	if (copy_from_user(req, attrs->ucore.inbuf, req_len))
139 		return -EFAULT;
140 
141 	iter->cur = attrs->ucore.inbuf + req_len;
142 	iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
143 	return 0;
144 }
145 
146 static int uverbs_request_next(struct uverbs_req_iter *iter, void *val,
147 			       size_t len)
148 {
149 	if (iter->cur + len > iter->end)
150 		return -ENOSPC;
151 
152 	if (copy_from_user(val, iter->cur, len))
153 		return -EFAULT;
154 
155 	iter->cur += len;
156 	return 0;
157 }
158 
159 static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,
160 						  size_t len)
161 {
162 	const void __user *res = iter->cur;
163 
164 	if (iter->cur + len > iter->end)
165 		return (void __force __user *)ERR_PTR(-ENOSPC);
166 	iter->cur += len;
167 	return res;
168 }
169 
170 static int uverbs_request_finish(struct uverbs_req_iter *iter)
171 {
172 	if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))
173 		return -EOPNOTSUPP;
174 	return 0;
175 }
176 
177 /*
178  * When calling a destroy function during an error unwind we need to pass in
179  * the udata that is sanitized of all user arguments. Ie from the driver
180  * perspective it looks like no udata was passed.
181  */
182 struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
183 {
184 	attrs->driver_udata = (struct ib_udata){};
185 	return &attrs->driver_udata;
186 }
187 
188 static struct ib_uverbs_completion_event_file *
189 _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
190 {
191 	struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
192 					       fd, attrs);
193 
194 	if (IS_ERR(uobj))
195 		return (void *)uobj;
196 
197 	uverbs_uobject_get(uobj);
198 	uobj_put_read(uobj);
199 
200 	return container_of(uobj, struct ib_uverbs_completion_event_file,
201 			    uobj);
202 }
203 #define ib_uverbs_lookup_comp_file(_fd, _ufile)                                \
204 	_ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
205 
206 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
207 {
208 	struct ib_uverbs_file *file = attrs->ufile;
209 	struct ib_uverbs_get_context      cmd;
210 	struct ib_uverbs_get_context_resp resp;
211 	struct ib_ucontext		 *ucontext;
212 	struct file			 *filp;
213 	struct ib_rdmacg_object		 cg_obj;
214 	struct ib_device *ib_dev;
215 	int ret;
216 
217 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
218 	if (ret)
219 		return ret;
220 
221 	mutex_lock(&file->ucontext_lock);
222 	ib_dev = srcu_dereference(file->device->ib_dev,
223 				  &file->device->disassociate_srcu);
224 	if (!ib_dev) {
225 		ret = -EIO;
226 		goto err;
227 	}
228 
229 	if (file->ucontext) {
230 		ret = -EINVAL;
231 		goto err;
232 	}
233 
234 	ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
235 	if (ret)
236 		goto err;
237 
238 	ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);
239 	if (!ucontext) {
240 		ret = -ENOMEM;
241 		goto err_alloc;
242 	}
243 
244 	attrs->context = ucontext;
245 
246 	ucontext->res.type = RDMA_RESTRACK_CTX;
247 	ucontext->device = ib_dev;
248 	ucontext->cg_obj = cg_obj;
249 	/* ufile is required when some objects are released */
250 	ucontext->ufile = file;
251 
252 	ucontext->closing = false;
253 	ucontext->cleanup_retryable = false;
254 
255 	mutex_init(&ucontext->per_mm_list_lock);
256 	INIT_LIST_HEAD(&ucontext->per_mm_list);
257 
258 	ret = get_unused_fd_flags(O_CLOEXEC);
259 	if (ret < 0)
260 		goto err_free;
261 	resp.async_fd = ret;
262 
263 	filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
264 	if (IS_ERR(filp)) {
265 		ret = PTR_ERR(filp);
266 		goto err_fd;
267 	}
268 
269 	resp.num_comp_vectors = file->device->num_comp_vectors;
270 
271 	ret = uverbs_response(attrs, &resp, sizeof(resp));
272 	if (ret)
273 		goto err_file;
274 
275 	ret = ib_dev->ops.alloc_ucontext(ucontext, &attrs->driver_udata);
276 	if (ret)
277 		goto err_file;
278 	if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
279 		ucontext->invalidate_range = NULL;
280 
281 	rdma_restrack_uadd(&ucontext->res);
282 
283 	fd_install(resp.async_fd, filp);
284 
285 	/*
286 	 * Make sure that ib_uverbs_get_ucontext() sees the pointer update
287 	 * only after all writes to setup the ucontext have completed
288 	 */
289 	smp_store_release(&file->ucontext, ucontext);
290 
291 	mutex_unlock(&file->ucontext_lock);
292 
293 	return 0;
294 
295 err_file:
296 	ib_uverbs_free_async_event_file(file);
297 	fput(filp);
298 
299 err_fd:
300 	put_unused_fd(resp.async_fd);
301 
302 err_free:
303 	kfree(ucontext);
304 
305 err_alloc:
306 	ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
307 
308 err:
309 	mutex_unlock(&file->ucontext_lock);
310 	return ret;
311 }
312 
313 static void copy_query_dev_fields(struct ib_ucontext *ucontext,
314 				  struct ib_uverbs_query_device_resp *resp,
315 				  struct ib_device_attr *attr)
316 {
317 	struct ib_device *ib_dev = ucontext->device;
318 
319 	resp->fw_ver		= attr->fw_ver;
320 	resp->node_guid		= ib_dev->node_guid;
321 	resp->sys_image_guid	= attr->sys_image_guid;
322 	resp->max_mr_size	= attr->max_mr_size;
323 	resp->page_size_cap	= attr->page_size_cap;
324 	resp->vendor_id		= attr->vendor_id;
325 	resp->vendor_part_id	= attr->vendor_part_id;
326 	resp->hw_ver		= attr->hw_ver;
327 	resp->max_qp		= attr->max_qp;
328 	resp->max_qp_wr		= attr->max_qp_wr;
329 	resp->device_cap_flags	= lower_32_bits(attr->device_cap_flags);
330 	resp->max_sge		= min(attr->max_send_sge, attr->max_recv_sge);
331 	resp->max_sge_rd	= attr->max_sge_rd;
332 	resp->max_cq		= attr->max_cq;
333 	resp->max_cqe		= attr->max_cqe;
334 	resp->max_mr		= attr->max_mr;
335 	resp->max_pd		= attr->max_pd;
336 	resp->max_qp_rd_atom	= attr->max_qp_rd_atom;
337 	resp->max_ee_rd_atom	= attr->max_ee_rd_atom;
338 	resp->max_res_rd_atom	= attr->max_res_rd_atom;
339 	resp->max_qp_init_rd_atom	= attr->max_qp_init_rd_atom;
340 	resp->max_ee_init_rd_atom	= attr->max_ee_init_rd_atom;
341 	resp->atomic_cap		= attr->atomic_cap;
342 	resp->max_ee			= attr->max_ee;
343 	resp->max_rdd			= attr->max_rdd;
344 	resp->max_mw			= attr->max_mw;
345 	resp->max_raw_ipv6_qp		= attr->max_raw_ipv6_qp;
346 	resp->max_raw_ethy_qp		= attr->max_raw_ethy_qp;
347 	resp->max_mcast_grp		= attr->max_mcast_grp;
348 	resp->max_mcast_qp_attach	= attr->max_mcast_qp_attach;
349 	resp->max_total_mcast_qp_attach	= attr->max_total_mcast_qp_attach;
350 	resp->max_ah			= attr->max_ah;
351 	resp->max_fmr			= attr->max_fmr;
352 	resp->max_map_per_fmr		= attr->max_map_per_fmr;
353 	resp->max_srq			= attr->max_srq;
354 	resp->max_srq_wr		= attr->max_srq_wr;
355 	resp->max_srq_sge		= attr->max_srq_sge;
356 	resp->max_pkeys			= attr->max_pkeys;
357 	resp->local_ca_ack_delay	= attr->local_ca_ack_delay;
358 	resp->phys_port_cnt		= ib_dev->phys_port_cnt;
359 }
360 
361 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
362 {
363 	struct ib_uverbs_query_device      cmd;
364 	struct ib_uverbs_query_device_resp resp;
365 	struct ib_ucontext *ucontext;
366 	int ret;
367 
368 	ucontext = ib_uverbs_get_ucontext(attrs);
369 	if (IS_ERR(ucontext))
370 		return PTR_ERR(ucontext);
371 
372 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
373 	if (ret)
374 		return ret;
375 
376 	memset(&resp, 0, sizeof resp);
377 	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);
378 
379 	return uverbs_response(attrs, &resp, sizeof(resp));
380 }
381 
382 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
383 {
384 	struct ib_uverbs_query_port      cmd;
385 	struct ib_uverbs_query_port_resp resp;
386 	struct ib_port_attr              attr;
387 	int                              ret;
388 	struct ib_ucontext *ucontext;
389 	struct ib_device *ib_dev;
390 
391 	ucontext = ib_uverbs_get_ucontext(attrs);
392 	if (IS_ERR(ucontext))
393 		return PTR_ERR(ucontext);
394 	ib_dev = ucontext->device;
395 
396 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
397 	if (ret)
398 		return ret;
399 
400 	ret = ib_query_port(ib_dev, cmd.port_num, &attr);
401 	if (ret)
402 		return ret;
403 
404 	memset(&resp, 0, sizeof resp);
405 	copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num);
406 
407 	return uverbs_response(attrs, &resp, sizeof(resp));
408 }
409 
410 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
411 {
412 	struct ib_uverbs_alloc_pd      cmd;
413 	struct ib_uverbs_alloc_pd_resp resp;
414 	struct ib_uobject             *uobj;
415 	struct ib_pd                  *pd;
416 	int                            ret;
417 	struct ib_device *ib_dev;
418 
419 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
420 	if (ret)
421 		return ret;
422 
423 	uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
424 	if (IS_ERR(uobj))
425 		return PTR_ERR(uobj);
426 
427 	pd = rdma_zalloc_drv_obj(ib_dev, ib_pd);
428 	if (!pd) {
429 		ret = -ENOMEM;
430 		goto err;
431 	}
432 
433 	pd->device  = ib_dev;
434 	pd->uobject = uobj;
435 	pd->__internal_mr = NULL;
436 	atomic_set(&pd->usecnt, 0);
437 	pd->res.type = RDMA_RESTRACK_PD;
438 
439 	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
440 	if (ret)
441 		goto err_alloc;
442 
443 	uobj->object = pd;
444 	memset(&resp, 0, sizeof resp);
445 	resp.pd_handle = uobj->id;
446 	rdma_restrack_uadd(&pd->res);
447 
448 	ret = uverbs_response(attrs, &resp, sizeof(resp));
449 	if (ret)
450 		goto err_copy;
451 
452 	return uobj_alloc_commit(uobj, attrs);
453 
454 err_copy:
455 	ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs));
456 	pd = NULL;
457 err_alloc:
458 	kfree(pd);
459 err:
460 	uobj_alloc_abort(uobj, attrs);
461 	return ret;
462 }
463 
464 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
465 {
466 	struct ib_uverbs_dealloc_pd cmd;
467 	int ret;
468 
469 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
470 	if (ret)
471 		return ret;
472 
473 	return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
474 }
475 
476 struct xrcd_table_entry {
477 	struct rb_node  node;
478 	struct ib_xrcd *xrcd;
479 	struct inode   *inode;
480 };
481 
482 static int xrcd_table_insert(struct ib_uverbs_device *dev,
483 			    struct inode *inode,
484 			    struct ib_xrcd *xrcd)
485 {
486 	struct xrcd_table_entry *entry, *scan;
487 	struct rb_node **p = &dev->xrcd_tree.rb_node;
488 	struct rb_node *parent = NULL;
489 
490 	entry = kmalloc(sizeof *entry, GFP_KERNEL);
491 	if (!entry)
492 		return -ENOMEM;
493 
494 	entry->xrcd  = xrcd;
495 	entry->inode = inode;
496 
497 	while (*p) {
498 		parent = *p;
499 		scan = rb_entry(parent, struct xrcd_table_entry, node);
500 
501 		if (inode < scan->inode) {
502 			p = &(*p)->rb_left;
503 		} else if (inode > scan->inode) {
504 			p = &(*p)->rb_right;
505 		} else {
506 			kfree(entry);
507 			return -EEXIST;
508 		}
509 	}
510 
511 	rb_link_node(&entry->node, parent, p);
512 	rb_insert_color(&entry->node, &dev->xrcd_tree);
513 	igrab(inode);
514 	return 0;
515 }
516 
517 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
518 						  struct inode *inode)
519 {
520 	struct xrcd_table_entry *entry;
521 	struct rb_node *p = dev->xrcd_tree.rb_node;
522 
523 	while (p) {
524 		entry = rb_entry(p, struct xrcd_table_entry, node);
525 
526 		if (inode < entry->inode)
527 			p = p->rb_left;
528 		else if (inode > entry->inode)
529 			p = p->rb_right;
530 		else
531 			return entry;
532 	}
533 
534 	return NULL;
535 }
536 
537 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
538 {
539 	struct xrcd_table_entry *entry;
540 
541 	entry = xrcd_table_search(dev, inode);
542 	if (!entry)
543 		return NULL;
544 
545 	return entry->xrcd;
546 }
547 
548 static void xrcd_table_delete(struct ib_uverbs_device *dev,
549 			      struct inode *inode)
550 {
551 	struct xrcd_table_entry *entry;
552 
553 	entry = xrcd_table_search(dev, inode);
554 	if (entry) {
555 		iput(inode);
556 		rb_erase(&entry->node, &dev->xrcd_tree);
557 		kfree(entry);
558 	}
559 }
560 
561 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
562 {
563 	struct ib_uverbs_device *ibudev = attrs->ufile->device;
564 	struct ib_uverbs_open_xrcd	cmd;
565 	struct ib_uverbs_open_xrcd_resp	resp;
566 	struct ib_uxrcd_object         *obj;
567 	struct ib_xrcd                 *xrcd = NULL;
568 	struct fd			f = {NULL, 0};
569 	struct inode                   *inode = NULL;
570 	int				ret = 0;
571 	int				new_xrcd = 0;
572 	struct ib_device *ib_dev;
573 
574 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
575 	if (ret)
576 		return ret;
577 
578 	mutex_lock(&ibudev->xrcd_tree_mutex);
579 
580 	if (cmd.fd != -1) {
581 		/* search for file descriptor */
582 		f = fdget(cmd.fd);
583 		if (!f.file) {
584 			ret = -EBADF;
585 			goto err_tree_mutex_unlock;
586 		}
587 
588 		inode = file_inode(f.file);
589 		xrcd = find_xrcd(ibudev, inode);
590 		if (!xrcd && !(cmd.oflags & O_CREAT)) {
591 			/* no file descriptor. Need CREATE flag */
592 			ret = -EAGAIN;
593 			goto err_tree_mutex_unlock;
594 		}
595 
596 		if (xrcd && cmd.oflags & O_EXCL) {
597 			ret = -EINVAL;
598 			goto err_tree_mutex_unlock;
599 		}
600 	}
601 
602 	obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
603 						   &ib_dev);
604 	if (IS_ERR(obj)) {
605 		ret = PTR_ERR(obj);
606 		goto err_tree_mutex_unlock;
607 	}
608 
609 	if (!xrcd) {
610 		xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata);
611 		if (IS_ERR(xrcd)) {
612 			ret = PTR_ERR(xrcd);
613 			goto err;
614 		}
615 
616 		xrcd->inode   = inode;
617 		xrcd->device  = ib_dev;
618 		atomic_set(&xrcd->usecnt, 0);
619 		mutex_init(&xrcd->tgt_qp_mutex);
620 		INIT_LIST_HEAD(&xrcd->tgt_qp_list);
621 		new_xrcd = 1;
622 	}
623 
624 	atomic_set(&obj->refcnt, 0);
625 	obj->uobject.object = xrcd;
626 	memset(&resp, 0, sizeof resp);
627 	resp.xrcd_handle = obj->uobject.id;
628 
629 	if (inode) {
630 		if (new_xrcd) {
631 			/* create new inode/xrcd table entry */
632 			ret = xrcd_table_insert(ibudev, inode, xrcd);
633 			if (ret)
634 				goto err_dealloc_xrcd;
635 		}
636 		atomic_inc(&xrcd->usecnt);
637 	}
638 
639 	ret = uverbs_response(attrs, &resp, sizeof(resp));
640 	if (ret)
641 		goto err_copy;
642 
643 	if (f.file)
644 		fdput(f);
645 
646 	mutex_unlock(&ibudev->xrcd_tree_mutex);
647 
648 	return uobj_alloc_commit(&obj->uobject, attrs);
649 
650 err_copy:
651 	if (inode) {
652 		if (new_xrcd)
653 			xrcd_table_delete(ibudev, inode);
654 		atomic_dec(&xrcd->usecnt);
655 	}
656 
657 err_dealloc_xrcd:
658 	ib_dealloc_xrcd(xrcd, uverbs_get_cleared_udata(attrs));
659 
660 err:
661 	uobj_alloc_abort(&obj->uobject, attrs);
662 
663 err_tree_mutex_unlock:
664 	if (f.file)
665 		fdput(f);
666 
667 	mutex_unlock(&ibudev->xrcd_tree_mutex);
668 
669 	return ret;
670 }
671 
672 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
673 {
674 	struct ib_uverbs_close_xrcd cmd;
675 	int ret;
676 
677 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
678 	if (ret)
679 		return ret;
680 
681 	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
682 }
683 
684 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
685 			   enum rdma_remove_reason why,
686 			   struct uverbs_attr_bundle *attrs)
687 {
688 	struct inode *inode;
689 	int ret;
690 	struct ib_uverbs_device *dev = attrs->ufile->device;
691 
692 	inode = xrcd->inode;
693 	if (inode && !atomic_dec_and_test(&xrcd->usecnt))
694 		return 0;
695 
696 	ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata);
697 
698 	if (ib_is_destroy_retryable(ret, why, uobject)) {
699 		atomic_inc(&xrcd->usecnt);
700 		return ret;
701 	}
702 
703 	if (inode)
704 		xrcd_table_delete(dev, inode);
705 
706 	return ret;
707 }
708 
709 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
710 {
711 	struct ib_uverbs_reg_mr      cmd;
712 	struct ib_uverbs_reg_mr_resp resp;
713 	struct ib_uobject           *uobj;
714 	struct ib_pd                *pd;
715 	struct ib_mr                *mr;
716 	int                          ret;
717 	struct ib_device *ib_dev;
718 
719 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
720 	if (ret)
721 		return ret;
722 
723 	if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
724 		return -EINVAL;
725 
726 	ret = ib_check_mr_access(cmd.access_flags);
727 	if (ret)
728 		return ret;
729 
730 	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
731 	if (IS_ERR(uobj))
732 		return PTR_ERR(uobj);
733 
734 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
735 	if (!pd) {
736 		ret = -EINVAL;
737 		goto err_free;
738 	}
739 
740 	if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
741 		if (!(pd->device->attrs.device_cap_flags &
742 		      IB_DEVICE_ON_DEMAND_PAGING)) {
743 			pr_debug("ODP support not available\n");
744 			ret = -EINVAL;
745 			goto err_put;
746 		}
747 	}
748 
749 	mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
750 					 cmd.access_flags,
751 					 &attrs->driver_udata);
752 	if (IS_ERR(mr)) {
753 		ret = PTR_ERR(mr);
754 		goto err_put;
755 	}
756 
757 	mr->device  = pd->device;
758 	mr->pd      = pd;
759 	mr->dm	    = NULL;
760 	mr->uobject = uobj;
761 	atomic_inc(&pd->usecnt);
762 	mr->res.type = RDMA_RESTRACK_MR;
763 	rdma_restrack_uadd(&mr->res);
764 
765 	uobj->object = mr;
766 
767 	memset(&resp, 0, sizeof resp);
768 	resp.lkey      = mr->lkey;
769 	resp.rkey      = mr->rkey;
770 	resp.mr_handle = uobj->id;
771 
772 	ret = uverbs_response(attrs, &resp, sizeof(resp));
773 	if (ret)
774 		goto err_copy;
775 
776 	uobj_put_obj_read(pd);
777 
778 	return uobj_alloc_commit(uobj, attrs);
779 
780 err_copy:
781 	ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs));
782 
783 err_put:
784 	uobj_put_obj_read(pd);
785 
786 err_free:
787 	uobj_alloc_abort(uobj, attrs);
788 	return ret;
789 }
790 
791 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
792 {
793 	struct ib_uverbs_rereg_mr      cmd;
794 	struct ib_uverbs_rereg_mr_resp resp;
795 	struct ib_pd                *pd = NULL;
796 	struct ib_mr                *mr;
797 	struct ib_pd		    *old_pd;
798 	int                          ret;
799 	struct ib_uobject	    *uobj;
800 
801 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
802 	if (ret)
803 		return ret;
804 
805 	if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
806 		return -EINVAL;
807 
808 	if ((cmd.flags & IB_MR_REREG_TRANS) &&
809 	    (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
810 	     (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
811 			return -EINVAL;
812 
813 	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
814 	if (IS_ERR(uobj))
815 		return PTR_ERR(uobj);
816 
817 	mr = uobj->object;
818 
819 	if (mr->dm) {
820 		ret = -EINVAL;
821 		goto put_uobjs;
822 	}
823 
824 	if (cmd.flags & IB_MR_REREG_ACCESS) {
825 		ret = ib_check_mr_access(cmd.access_flags);
826 		if (ret)
827 			goto put_uobjs;
828 	}
829 
830 	if (cmd.flags & IB_MR_REREG_PD) {
831 		pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
832 				       attrs);
833 		if (!pd) {
834 			ret = -EINVAL;
835 			goto put_uobjs;
836 		}
837 	}
838 
839 	old_pd = mr->pd;
840 	ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start,
841 					    cmd.length, cmd.hca_va,
842 					    cmd.access_flags, pd,
843 					    &attrs->driver_udata);
844 	if (ret)
845 		goto put_uobj_pd;
846 
847 	if (cmd.flags & IB_MR_REREG_PD) {
848 		atomic_inc(&pd->usecnt);
849 		mr->pd = pd;
850 		atomic_dec(&old_pd->usecnt);
851 	}
852 
853 	memset(&resp, 0, sizeof(resp));
854 	resp.lkey      = mr->lkey;
855 	resp.rkey      = mr->rkey;
856 
857 	ret = uverbs_response(attrs, &resp, sizeof(resp));
858 
859 put_uobj_pd:
860 	if (cmd.flags & IB_MR_REREG_PD)
861 		uobj_put_obj_read(pd);
862 
863 put_uobjs:
864 	uobj_put_write(uobj);
865 
866 	return ret;
867 }
868 
869 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
870 {
871 	struct ib_uverbs_dereg_mr cmd;
872 	int ret;
873 
874 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
875 	if (ret)
876 		return ret;
877 
878 	return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
879 }
880 
881 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
882 {
883 	struct ib_uverbs_alloc_mw      cmd;
884 	struct ib_uverbs_alloc_mw_resp resp;
885 	struct ib_uobject             *uobj;
886 	struct ib_pd                  *pd;
887 	struct ib_mw                  *mw;
888 	int                            ret;
889 	struct ib_device *ib_dev;
890 
891 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
892 	if (ret)
893 		return ret;
894 
895 	uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
896 	if (IS_ERR(uobj))
897 		return PTR_ERR(uobj);
898 
899 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
900 	if (!pd) {
901 		ret = -EINVAL;
902 		goto err_free;
903 	}
904 
905 	if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) {
906 		ret = -EINVAL;
907 		goto err_put;
908 	}
909 
910 	mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata);
911 	if (IS_ERR(mw)) {
912 		ret = PTR_ERR(mw);
913 		goto err_put;
914 	}
915 
916 	mw->device  = pd->device;
917 	mw->pd      = pd;
918 	mw->uobject = uobj;
919 	atomic_inc(&pd->usecnt);
920 
921 	uobj->object = mw;
922 
923 	memset(&resp, 0, sizeof(resp));
924 	resp.rkey      = mw->rkey;
925 	resp.mw_handle = uobj->id;
926 
927 	ret = uverbs_response(attrs, &resp, sizeof(resp));
928 	if (ret)
929 		goto err_copy;
930 
931 	uobj_put_obj_read(pd);
932 	return uobj_alloc_commit(uobj, attrs);
933 
934 err_copy:
935 	uverbs_dealloc_mw(mw);
936 err_put:
937 	uobj_put_obj_read(pd);
938 err_free:
939 	uobj_alloc_abort(uobj, attrs);
940 	return ret;
941 }
942 
943 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
944 {
945 	struct ib_uverbs_dealloc_mw cmd;
946 	int ret;
947 
948 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
949 	if (ret)
950 		return ret;
951 
952 	return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
953 }
954 
955 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
956 {
957 	struct ib_uverbs_create_comp_channel	   cmd;
958 	struct ib_uverbs_create_comp_channel_resp  resp;
959 	struct ib_uobject			  *uobj;
960 	struct ib_uverbs_completion_event_file	  *ev_file;
961 	struct ib_device *ib_dev;
962 	int ret;
963 
964 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
965 	if (ret)
966 		return ret;
967 
968 	uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
969 	if (IS_ERR(uobj))
970 		return PTR_ERR(uobj);
971 
972 	resp.fd = uobj->id;
973 
974 	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
975 			       uobj);
976 	ib_uverbs_init_event_queue(&ev_file->ev_queue);
977 
978 	ret = uverbs_response(attrs, &resp, sizeof(resp));
979 	if (ret) {
980 		uobj_alloc_abort(uobj, attrs);
981 		return ret;
982 	}
983 
984 	return uobj_alloc_commit(uobj, attrs);
985 }
986 
987 static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
988 				       struct ib_uverbs_ex_create_cq *cmd)
989 {
990 	struct ib_ucq_object           *obj;
991 	struct ib_uverbs_completion_event_file    *ev_file = NULL;
992 	struct ib_cq                   *cq;
993 	int                             ret;
994 	struct ib_uverbs_ex_create_cq_resp resp;
995 	struct ib_cq_init_attr attr = {};
996 	struct ib_device *ib_dev;
997 
998 	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
999 		return ERR_PTR(-EINVAL);
1000 
1001 	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
1002 						 &ib_dev);
1003 	if (IS_ERR(obj))
1004 		return obj;
1005 
1006 	if (cmd->comp_channel >= 0) {
1007 		ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
1008 		if (IS_ERR(ev_file)) {
1009 			ret = PTR_ERR(ev_file);
1010 			goto err;
1011 		}
1012 	}
1013 
1014 	obj->uobject.user_handle = cmd->user_handle;
1015 	obj->comp_events_reported  = 0;
1016 	obj->async_events_reported = 0;
1017 	INIT_LIST_HEAD(&obj->comp_list);
1018 	INIT_LIST_HEAD(&obj->async_list);
1019 
1020 	attr.cqe = cmd->cqe;
1021 	attr.comp_vector = cmd->comp_vector;
1022 	attr.flags = cmd->flags;
1023 
1024 	cq = ib_dev->ops.create_cq(ib_dev, &attr, &attrs->driver_udata);
1025 	if (IS_ERR(cq)) {
1026 		ret = PTR_ERR(cq);
1027 		goto err_file;
1028 	}
1029 
1030 	cq->device        = ib_dev;
1031 	cq->uobject       = &obj->uobject;
1032 	cq->comp_handler  = ib_uverbs_comp_handler;
1033 	cq->event_handler = ib_uverbs_cq_event_handler;
1034 	cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
1035 	atomic_set(&cq->usecnt, 0);
1036 
1037 	obj->uobject.object = cq;
1038 	memset(&resp, 0, sizeof resp);
1039 	resp.base.cq_handle = obj->uobject.id;
1040 	resp.base.cqe       = cq->cqe;
1041 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1042 
1043 	cq->res.type = RDMA_RESTRACK_CQ;
1044 	rdma_restrack_uadd(&cq->res);
1045 
1046 	ret = uverbs_response(attrs, &resp, sizeof(resp));
1047 	if (ret)
1048 		goto err_cb;
1049 
1050 	ret = uobj_alloc_commit(&obj->uobject, attrs);
1051 	if (ret)
1052 		return ERR_PTR(ret);
1053 	return obj;
1054 
1055 err_cb:
1056 	ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
1057 
1058 err_file:
1059 	if (ev_file)
1060 		ib_uverbs_release_ucq(attrs->ufile, ev_file, obj);
1061 
1062 err:
1063 	uobj_alloc_abort(&obj->uobject, attrs);
1064 
1065 	return ERR_PTR(ret);
1066 }
1067 
1068 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1069 {
1070 	struct ib_uverbs_create_cq      cmd;
1071 	struct ib_uverbs_ex_create_cq	cmd_ex;
1072 	struct ib_ucq_object           *obj;
1073 	int ret;
1074 
1075 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1076 	if (ret)
1077 		return ret;
1078 
1079 	memset(&cmd_ex, 0, sizeof(cmd_ex));
1080 	cmd_ex.user_handle = cmd.user_handle;
1081 	cmd_ex.cqe = cmd.cqe;
1082 	cmd_ex.comp_vector = cmd.comp_vector;
1083 	cmd_ex.comp_channel = cmd.comp_channel;
1084 
1085 	obj = create_cq(attrs, &cmd_ex);
1086 	return PTR_ERR_OR_ZERO(obj);
1087 }
1088 
1089 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1090 {
1091 	struct ib_uverbs_ex_create_cq  cmd;
1092 	struct ib_ucq_object           *obj;
1093 	int ret;
1094 
1095 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1096 	if (ret)
1097 		return ret;
1098 
1099 	if (cmd.comp_mask)
1100 		return -EINVAL;
1101 
1102 	if (cmd.reserved)
1103 		return -EINVAL;
1104 
1105 	obj = create_cq(attrs, &cmd);
1106 	return PTR_ERR_OR_ZERO(obj);
1107 }
1108 
1109 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
1110 {
1111 	struct ib_uverbs_resize_cq	cmd;
1112 	struct ib_uverbs_resize_cq_resp	resp = {};
1113 	struct ib_cq			*cq;
1114 	int				ret = -EINVAL;
1115 
1116 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1117 	if (ret)
1118 		return ret;
1119 
1120 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1121 	if (!cq)
1122 		return -EINVAL;
1123 
1124 	ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
1125 	if (ret)
1126 		goto out;
1127 
1128 	resp.cqe = cq->cqe;
1129 
1130 	ret = uverbs_response(attrs, &resp, sizeof(resp));
1131 out:
1132 	uobj_put_obj_read(cq);
1133 
1134 	return ret;
1135 }
1136 
1137 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1138 			   struct ib_wc *wc)
1139 {
1140 	struct ib_uverbs_wc tmp;
1141 
1142 	tmp.wr_id		= wc->wr_id;
1143 	tmp.status		= wc->status;
1144 	tmp.opcode		= wc->opcode;
1145 	tmp.vendor_err		= wc->vendor_err;
1146 	tmp.byte_len		= wc->byte_len;
1147 	tmp.ex.imm_data		= wc->ex.imm_data;
1148 	tmp.qp_num		= wc->qp->qp_num;
1149 	tmp.src_qp		= wc->src_qp;
1150 	tmp.wc_flags		= wc->wc_flags;
1151 	tmp.pkey_index		= wc->pkey_index;
1152 	if (rdma_cap_opa_ah(ib_dev, wc->port_num))
1153 		tmp.slid	= OPA_TO_IB_UCAST_LID(wc->slid);
1154 	else
1155 		tmp.slid	= ib_lid_cpu16(wc->slid);
1156 	tmp.sl			= wc->sl;
1157 	tmp.dlid_path_bits	= wc->dlid_path_bits;
1158 	tmp.port_num		= wc->port_num;
1159 	tmp.reserved		= 0;
1160 
1161 	if (copy_to_user(dest, &tmp, sizeof tmp))
1162 		return -EFAULT;
1163 
1164 	return 0;
1165 }
1166 
1167 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
1168 {
1169 	struct ib_uverbs_poll_cq       cmd;
1170 	struct ib_uverbs_poll_cq_resp  resp;
1171 	u8 __user                     *header_ptr;
1172 	u8 __user                     *data_ptr;
1173 	struct ib_cq                  *cq;
1174 	struct ib_wc                   wc;
1175 	int                            ret;
1176 
1177 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1178 	if (ret)
1179 		return ret;
1180 
1181 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1182 	if (!cq)
1183 		return -EINVAL;
1184 
1185 	/* we copy a struct ib_uverbs_poll_cq_resp to user space */
1186 	header_ptr = attrs->ucore.outbuf;
1187 	data_ptr = header_ptr + sizeof resp;
1188 
1189 	memset(&resp, 0, sizeof resp);
1190 	while (resp.count < cmd.ne) {
1191 		ret = ib_poll_cq(cq, 1, &wc);
1192 		if (ret < 0)
1193 			goto out_put;
1194 		if (!ret)
1195 			break;
1196 
1197 		ret = copy_wc_to_user(cq->device, data_ptr, &wc);
1198 		if (ret)
1199 			goto out_put;
1200 
1201 		data_ptr += sizeof(struct ib_uverbs_wc);
1202 		++resp.count;
1203 	}
1204 
1205 	if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1206 		ret = -EFAULT;
1207 		goto out_put;
1208 	}
1209 	ret = 0;
1210 
1211 	if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
1212 		ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);
1213 
1214 out_put:
1215 	uobj_put_obj_read(cq);
1216 	return ret;
1217 }
1218 
1219 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
1220 {
1221 	struct ib_uverbs_req_notify_cq cmd;
1222 	struct ib_cq                  *cq;
1223 	int ret;
1224 
1225 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1226 	if (ret)
1227 		return ret;
1228 
1229 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1230 	if (!cq)
1231 		return -EINVAL;
1232 
1233 	ib_req_notify_cq(cq, cmd.solicited_only ?
1234 			 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1235 
1236 	uobj_put_obj_read(cq);
1237 
1238 	return 0;
1239 }
1240 
1241 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
1242 {
1243 	struct ib_uverbs_destroy_cq      cmd;
1244 	struct ib_uverbs_destroy_cq_resp resp;
1245 	struct ib_uobject		*uobj;
1246 	struct ib_ucq_object        	*obj;
1247 	int ret;
1248 
1249 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1250 	if (ret)
1251 		return ret;
1252 
1253 	uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1254 	if (IS_ERR(uobj))
1255 		return PTR_ERR(uobj);
1256 
1257 	obj = container_of(uobj, struct ib_ucq_object, uobject);
1258 	memset(&resp, 0, sizeof(resp));
1259 	resp.comp_events_reported  = obj->comp_events_reported;
1260 	resp.async_events_reported = obj->async_events_reported;
1261 
1262 	uobj_put_destroy(uobj);
1263 
1264 	return uverbs_response(attrs, &resp, sizeof(resp));
1265 }
1266 
1267 static int create_qp(struct uverbs_attr_bundle *attrs,
1268 		     struct ib_uverbs_ex_create_qp *cmd)
1269 {
1270 	struct ib_uqp_object		*obj;
1271 	struct ib_device		*device;
1272 	struct ib_pd			*pd = NULL;
1273 	struct ib_xrcd			*xrcd = NULL;
1274 	struct ib_uobject		*xrcd_uobj = ERR_PTR(-ENOENT);
1275 	struct ib_cq			*scq = NULL, *rcq = NULL;
1276 	struct ib_srq			*srq = NULL;
1277 	struct ib_qp			*qp;
1278 	struct ib_qp_init_attr		attr = {};
1279 	struct ib_uverbs_ex_create_qp_resp resp;
1280 	int				ret;
1281 	struct ib_rwq_ind_table *ind_tbl = NULL;
1282 	bool has_sq = true;
1283 	struct ib_device *ib_dev;
1284 
1285 	if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1286 		return -EPERM;
1287 
1288 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1289 						 &ib_dev);
1290 	if (IS_ERR(obj))
1291 		return PTR_ERR(obj);
1292 	obj->uxrcd = NULL;
1293 	obj->uevent.uobject.user_handle = cmd->user_handle;
1294 	mutex_init(&obj->mcast_lock);
1295 
1296 	if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
1297 		ind_tbl = uobj_get_obj_read(rwq_ind_table,
1298 					    UVERBS_OBJECT_RWQ_IND_TBL,
1299 					    cmd->rwq_ind_tbl_handle, attrs);
1300 		if (!ind_tbl) {
1301 			ret = -EINVAL;
1302 			goto err_put;
1303 		}
1304 
1305 		attr.rwq_ind_tbl = ind_tbl;
1306 	}
1307 
1308 	if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1309 		ret = -EINVAL;
1310 		goto err_put;
1311 	}
1312 
1313 	if (ind_tbl && !cmd->max_send_wr)
1314 		has_sq = false;
1315 
1316 	if (cmd->qp_type == IB_QPT_XRC_TGT) {
1317 		xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
1318 					  attrs);
1319 
1320 		if (IS_ERR(xrcd_uobj)) {
1321 			ret = -EINVAL;
1322 			goto err_put;
1323 		}
1324 
1325 		xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1326 		if (!xrcd) {
1327 			ret = -EINVAL;
1328 			goto err_put;
1329 		}
1330 		device = xrcd->device;
1331 	} else {
1332 		if (cmd->qp_type == IB_QPT_XRC_INI) {
1333 			cmd->max_recv_wr = 0;
1334 			cmd->max_recv_sge = 0;
1335 		} else {
1336 			if (cmd->is_srq) {
1337 				srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
1338 							cmd->srq_handle, attrs);
1339 				if (!srq || srq->srq_type == IB_SRQT_XRC) {
1340 					ret = -EINVAL;
1341 					goto err_put;
1342 				}
1343 			}
1344 
1345 			if (!ind_tbl) {
1346 				if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1347 					rcq = uobj_get_obj_read(
1348 						cq, UVERBS_OBJECT_CQ,
1349 						cmd->recv_cq_handle, attrs);
1350 					if (!rcq) {
1351 						ret = -EINVAL;
1352 						goto err_put;
1353 					}
1354 				}
1355 			}
1356 		}
1357 
1358 		if (has_sq)
1359 			scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
1360 						cmd->send_cq_handle, attrs);
1361 		if (!ind_tbl)
1362 			rcq = rcq ?: scq;
1363 		pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
1364 				       attrs);
1365 		if (!pd || (!scq && has_sq)) {
1366 			ret = -EINVAL;
1367 			goto err_put;
1368 		}
1369 
1370 		device = pd->device;
1371 	}
1372 
1373 	attr.event_handler = ib_uverbs_qp_event_handler;
1374 	attr.qp_context    = attrs->ufile;
1375 	attr.send_cq       = scq;
1376 	attr.recv_cq       = rcq;
1377 	attr.srq           = srq;
1378 	attr.xrcd	   = xrcd;
1379 	attr.sq_sig_type   = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1380 					      IB_SIGNAL_REQ_WR;
1381 	attr.qp_type       = cmd->qp_type;
1382 	attr.create_flags  = 0;
1383 
1384 	attr.cap.max_send_wr     = cmd->max_send_wr;
1385 	attr.cap.max_recv_wr     = cmd->max_recv_wr;
1386 	attr.cap.max_send_sge    = cmd->max_send_sge;
1387 	attr.cap.max_recv_sge    = cmd->max_recv_sge;
1388 	attr.cap.max_inline_data = cmd->max_inline_data;
1389 
1390 	obj->uevent.events_reported     = 0;
1391 	INIT_LIST_HEAD(&obj->uevent.event_list);
1392 	INIT_LIST_HEAD(&obj->mcast_list);
1393 
1394 	attr.create_flags = cmd->create_flags;
1395 	if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1396 				IB_QP_CREATE_CROSS_CHANNEL |
1397 				IB_QP_CREATE_MANAGED_SEND |
1398 				IB_QP_CREATE_MANAGED_RECV |
1399 				IB_QP_CREATE_SCATTER_FCS |
1400 				IB_QP_CREATE_CVLAN_STRIPPING |
1401 				IB_QP_CREATE_SOURCE_QPN |
1402 				IB_QP_CREATE_PCI_WRITE_END_PADDING)) {
1403 		ret = -EINVAL;
1404 		goto err_put;
1405 	}
1406 
1407 	if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1408 		if (!capable(CAP_NET_RAW)) {
1409 			ret = -EPERM;
1410 			goto err_put;
1411 		}
1412 
1413 		attr.source_qpn = cmd->source_qpn;
1414 	}
1415 
1416 	if (cmd->qp_type == IB_QPT_XRC_TGT)
1417 		qp = ib_create_qp(pd, &attr);
1418 	else
1419 		qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
1420 				   &obj->uevent.uobject);
1421 
1422 	if (IS_ERR(qp)) {
1423 		ret = PTR_ERR(qp);
1424 		goto err_put;
1425 	}
1426 
1427 	if (cmd->qp_type != IB_QPT_XRC_TGT) {
1428 		ret = ib_create_qp_security(qp, device);
1429 		if (ret)
1430 			goto err_cb;
1431 
1432 		qp->pd		  = pd;
1433 		qp->send_cq	  = attr.send_cq;
1434 		qp->recv_cq	  = attr.recv_cq;
1435 		qp->srq		  = attr.srq;
1436 		qp->rwq_ind_tbl	  = ind_tbl;
1437 		qp->event_handler = attr.event_handler;
1438 		qp->qp_context	  = attr.qp_context;
1439 		qp->qp_type	  = attr.qp_type;
1440 		atomic_set(&qp->usecnt, 0);
1441 		atomic_inc(&pd->usecnt);
1442 		qp->port = 0;
1443 		if (attr.send_cq)
1444 			atomic_inc(&attr.send_cq->usecnt);
1445 		if (attr.recv_cq)
1446 			atomic_inc(&attr.recv_cq->usecnt);
1447 		if (attr.srq)
1448 			atomic_inc(&attr.srq->usecnt);
1449 		if (ind_tbl)
1450 			atomic_inc(&ind_tbl->usecnt);
1451 	} else {
1452 		/* It is done in _ib_create_qp for other QP types */
1453 		qp->uobject = &obj->uevent.uobject;
1454 	}
1455 
1456 	obj->uevent.uobject.object = qp;
1457 
1458 	memset(&resp, 0, sizeof resp);
1459 	resp.base.qpn             = qp->qp_num;
1460 	resp.base.qp_handle       = obj->uevent.uobject.id;
1461 	resp.base.max_recv_sge    = attr.cap.max_recv_sge;
1462 	resp.base.max_send_sge    = attr.cap.max_send_sge;
1463 	resp.base.max_recv_wr     = attr.cap.max_recv_wr;
1464 	resp.base.max_send_wr     = attr.cap.max_send_wr;
1465 	resp.base.max_inline_data = attr.cap.max_inline_data;
1466 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
1467 
1468 	ret = uverbs_response(attrs, &resp, sizeof(resp));
1469 	if (ret)
1470 		goto err_cb;
1471 
1472 	if (xrcd) {
1473 		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1474 					  uobject);
1475 		atomic_inc(&obj->uxrcd->refcnt);
1476 		uobj_put_read(xrcd_uobj);
1477 	}
1478 
1479 	if (pd)
1480 		uobj_put_obj_read(pd);
1481 	if (scq)
1482 		uobj_put_obj_read(scq);
1483 	if (rcq && rcq != scq)
1484 		uobj_put_obj_read(rcq);
1485 	if (srq)
1486 		uobj_put_obj_read(srq);
1487 	if (ind_tbl)
1488 		uobj_put_obj_read(ind_tbl);
1489 
1490 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
1491 err_cb:
1492 	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
1493 
1494 err_put:
1495 	if (!IS_ERR(xrcd_uobj))
1496 		uobj_put_read(xrcd_uobj);
1497 	if (pd)
1498 		uobj_put_obj_read(pd);
1499 	if (scq)
1500 		uobj_put_obj_read(scq);
1501 	if (rcq && rcq != scq)
1502 		uobj_put_obj_read(rcq);
1503 	if (srq)
1504 		uobj_put_obj_read(srq);
1505 	if (ind_tbl)
1506 		uobj_put_obj_read(ind_tbl);
1507 
1508 	uobj_alloc_abort(&obj->uevent.uobject, attrs);
1509 	return ret;
1510 }
1511 
1512 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
1513 {
1514 	struct ib_uverbs_create_qp      cmd;
1515 	struct ib_uverbs_ex_create_qp	cmd_ex;
1516 	int ret;
1517 
1518 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1519 	if (ret)
1520 		return ret;
1521 
1522 	memset(&cmd_ex, 0, sizeof(cmd_ex));
1523 	cmd_ex.user_handle = cmd.user_handle;
1524 	cmd_ex.pd_handle = cmd.pd_handle;
1525 	cmd_ex.send_cq_handle = cmd.send_cq_handle;
1526 	cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1527 	cmd_ex.srq_handle = cmd.srq_handle;
1528 	cmd_ex.max_send_wr = cmd.max_send_wr;
1529 	cmd_ex.max_recv_wr = cmd.max_recv_wr;
1530 	cmd_ex.max_send_sge = cmd.max_send_sge;
1531 	cmd_ex.max_recv_sge = cmd.max_recv_sge;
1532 	cmd_ex.max_inline_data = cmd.max_inline_data;
1533 	cmd_ex.sq_sig_all = cmd.sq_sig_all;
1534 	cmd_ex.qp_type = cmd.qp_type;
1535 	cmd_ex.is_srq = cmd.is_srq;
1536 
1537 	return create_qp(attrs, &cmd_ex);
1538 }
1539 
1540 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
1541 {
1542 	struct ib_uverbs_ex_create_qp cmd;
1543 	int ret;
1544 
1545 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1546 	if (ret)
1547 		return ret;
1548 
1549 	if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1550 		return -EINVAL;
1551 
1552 	if (cmd.reserved)
1553 		return -EINVAL;
1554 
1555 	return create_qp(attrs, &cmd);
1556 }
1557 
1558 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1559 {
1560 	struct ib_uverbs_open_qp        cmd;
1561 	struct ib_uverbs_create_qp_resp resp;
1562 	struct ib_uqp_object           *obj;
1563 	struct ib_xrcd		       *xrcd;
1564 	struct ib_uobject	       *uninitialized_var(xrcd_uobj);
1565 	struct ib_qp                   *qp;
1566 	struct ib_qp_open_attr          attr;
1567 	int ret;
1568 	struct ib_device *ib_dev;
1569 
1570 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1571 	if (ret)
1572 		return ret;
1573 
1574 	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1575 						 &ib_dev);
1576 	if (IS_ERR(obj))
1577 		return PTR_ERR(obj);
1578 
1579 	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
1580 	if (IS_ERR(xrcd_uobj)) {
1581 		ret = -EINVAL;
1582 		goto err_put;
1583 	}
1584 
1585 	xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1586 	if (!xrcd) {
1587 		ret = -EINVAL;
1588 		goto err_xrcd;
1589 	}
1590 
1591 	attr.event_handler = ib_uverbs_qp_event_handler;
1592 	attr.qp_context    = attrs->ufile;
1593 	attr.qp_num        = cmd.qpn;
1594 	attr.qp_type       = cmd.qp_type;
1595 
1596 	obj->uevent.events_reported = 0;
1597 	INIT_LIST_HEAD(&obj->uevent.event_list);
1598 	INIT_LIST_HEAD(&obj->mcast_list);
1599 
1600 	qp = ib_open_qp(xrcd, &attr);
1601 	if (IS_ERR(qp)) {
1602 		ret = PTR_ERR(qp);
1603 		goto err_xrcd;
1604 	}
1605 
1606 	obj->uevent.uobject.object = qp;
1607 	obj->uevent.uobject.user_handle = cmd.user_handle;
1608 
1609 	memset(&resp, 0, sizeof resp);
1610 	resp.qpn       = qp->qp_num;
1611 	resp.qp_handle = obj->uevent.uobject.id;
1612 
1613 	ret = uverbs_response(attrs, &resp, sizeof(resp));
1614 	if (ret)
1615 		goto err_destroy;
1616 
1617 	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1618 	atomic_inc(&obj->uxrcd->refcnt);
1619 	qp->uobject = &obj->uevent.uobject;
1620 	uobj_put_read(xrcd_uobj);
1621 
1622 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
1623 
1624 err_destroy:
1625 	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
1626 err_xrcd:
1627 	uobj_put_read(xrcd_uobj);
1628 err_put:
1629 	uobj_alloc_abort(&obj->uevent.uobject, attrs);
1630 	return ret;
1631 }
1632 
1633 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1634 				   struct rdma_ah_attr *rdma_attr)
1635 {
1636 	const struct ib_global_route   *grh;
1637 
1638 	uverb_attr->dlid              = rdma_ah_get_dlid(rdma_attr);
1639 	uverb_attr->sl                = rdma_ah_get_sl(rdma_attr);
1640 	uverb_attr->src_path_bits     = rdma_ah_get_path_bits(rdma_attr);
1641 	uverb_attr->static_rate       = rdma_ah_get_static_rate(rdma_attr);
1642 	uverb_attr->is_global         = !!(rdma_ah_get_ah_flags(rdma_attr) &
1643 					 IB_AH_GRH);
1644 	if (uverb_attr->is_global) {
1645 		grh = rdma_ah_read_grh(rdma_attr);
1646 		memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1647 		uverb_attr->flow_label        = grh->flow_label;
1648 		uverb_attr->sgid_index        = grh->sgid_index;
1649 		uverb_attr->hop_limit         = grh->hop_limit;
1650 		uverb_attr->traffic_class     = grh->traffic_class;
1651 	}
1652 	uverb_attr->port_num          = rdma_ah_get_port_num(rdma_attr);
1653 }
1654 
1655 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
1656 {
1657 	struct ib_uverbs_query_qp      cmd;
1658 	struct ib_uverbs_query_qp_resp resp;
1659 	struct ib_qp                   *qp;
1660 	struct ib_qp_attr              *attr;
1661 	struct ib_qp_init_attr         *init_attr;
1662 	int                            ret;
1663 
1664 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1665 	if (ret)
1666 		return ret;
1667 
1668 	attr      = kmalloc(sizeof *attr, GFP_KERNEL);
1669 	init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1670 	if (!attr || !init_attr) {
1671 		ret = -ENOMEM;
1672 		goto out;
1673 	}
1674 
1675 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1676 	if (!qp) {
1677 		ret = -EINVAL;
1678 		goto out;
1679 	}
1680 
1681 	ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1682 
1683 	uobj_put_obj_read(qp);
1684 
1685 	if (ret)
1686 		goto out;
1687 
1688 	memset(&resp, 0, sizeof resp);
1689 
1690 	resp.qp_state               = attr->qp_state;
1691 	resp.cur_qp_state           = attr->cur_qp_state;
1692 	resp.path_mtu               = attr->path_mtu;
1693 	resp.path_mig_state         = attr->path_mig_state;
1694 	resp.qkey                   = attr->qkey;
1695 	resp.rq_psn                 = attr->rq_psn;
1696 	resp.sq_psn                 = attr->sq_psn;
1697 	resp.dest_qp_num            = attr->dest_qp_num;
1698 	resp.qp_access_flags        = attr->qp_access_flags;
1699 	resp.pkey_index             = attr->pkey_index;
1700 	resp.alt_pkey_index         = attr->alt_pkey_index;
1701 	resp.sq_draining            = attr->sq_draining;
1702 	resp.max_rd_atomic          = attr->max_rd_atomic;
1703 	resp.max_dest_rd_atomic     = attr->max_dest_rd_atomic;
1704 	resp.min_rnr_timer          = attr->min_rnr_timer;
1705 	resp.port_num               = attr->port_num;
1706 	resp.timeout                = attr->timeout;
1707 	resp.retry_cnt              = attr->retry_cnt;
1708 	resp.rnr_retry              = attr->rnr_retry;
1709 	resp.alt_port_num           = attr->alt_port_num;
1710 	resp.alt_timeout            = attr->alt_timeout;
1711 
1712 	copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
1713 	copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
1714 
1715 	resp.max_send_wr            = init_attr->cap.max_send_wr;
1716 	resp.max_recv_wr            = init_attr->cap.max_recv_wr;
1717 	resp.max_send_sge           = init_attr->cap.max_send_sge;
1718 	resp.max_recv_sge           = init_attr->cap.max_recv_sge;
1719 	resp.max_inline_data        = init_attr->cap.max_inline_data;
1720 	resp.sq_sig_all             = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1721 
1722 	ret = uverbs_response(attrs, &resp, sizeof(resp));
1723 
1724 out:
1725 	kfree(attr);
1726 	kfree(init_attr);
1727 
1728 	return ret;
1729 }
1730 
1731 /* Remove ignored fields set in the attribute mask */
1732 static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1733 {
1734 	switch (qp_type) {
1735 	case IB_QPT_XRC_INI:
1736 		return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1737 	case IB_QPT_XRC_TGT:
1738 		return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1739 				IB_QP_RNR_RETRY);
1740 	default:
1741 		return mask;
1742 	}
1743 }
1744 
1745 static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1746 				     struct rdma_ah_attr *rdma_attr,
1747 				     struct ib_uverbs_qp_dest *uverb_attr)
1748 {
1749 	rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
1750 	if (uverb_attr->is_global) {
1751 		rdma_ah_set_grh(rdma_attr, NULL,
1752 				uverb_attr->flow_label,
1753 				uverb_attr->sgid_index,
1754 				uverb_attr->hop_limit,
1755 				uverb_attr->traffic_class);
1756 		rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
1757 	} else {
1758 		rdma_ah_set_ah_flags(rdma_attr, 0);
1759 	}
1760 	rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
1761 	rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
1762 	rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
1763 	rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
1764 	rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
1765 	rdma_ah_set_make_grd(rdma_attr, false);
1766 }
1767 
1768 static int modify_qp(struct uverbs_attr_bundle *attrs,
1769 		     struct ib_uverbs_ex_modify_qp *cmd)
1770 {
1771 	struct ib_qp_attr *attr;
1772 	struct ib_qp *qp;
1773 	int ret;
1774 
1775 	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1776 	if (!attr)
1777 		return -ENOMEM;
1778 
1779 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
1780 			       attrs);
1781 	if (!qp) {
1782 		ret = -EINVAL;
1783 		goto out;
1784 	}
1785 
1786 	if ((cmd->base.attr_mask & IB_QP_PORT) &&
1787 	    !rdma_is_port_valid(qp->device, cmd->base.port_num)) {
1788 		ret = -EINVAL;
1789 		goto release_qp;
1790 	}
1791 
1792 	if ((cmd->base.attr_mask & IB_QP_AV)) {
1793 		if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
1794 			ret = -EINVAL;
1795 			goto release_qp;
1796 		}
1797 
1798 		if (cmd->base.attr_mask & IB_QP_STATE &&
1799 		    cmd->base.qp_state == IB_QPS_RTR) {
1800 		/* We are in INIT->RTR TRANSITION (if we are not,
1801 		 * this transition will be rejected in subsequent checks).
1802 		 * In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1803 		 * but the IB_QP_STATE flag is required.
1804 		 *
1805 		 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1806 		 * when IB_QP_AV is set, has required inclusion of a valid
1807 		 * port number in the primary AV. (AVs are created and handled
1808 		 * differently for infiniband and ethernet (RoCE) ports).
1809 		 *
1810 		 * Check the port number included in the primary AV against
1811 		 * the port number in the qp struct, which was set (and saved)
1812 		 * in the RST->INIT transition.
1813 		 */
1814 			if (cmd->base.dest.port_num != qp->real_qp->port) {
1815 				ret = -EINVAL;
1816 				goto release_qp;
1817 			}
1818 		} else {
1819 		/* We are in SQD->SQD. (If we are not, this transition will
1820 		 * be rejected later in the verbs layer checks).
1821 		 * Check for both IB_QP_PORT and IB_QP_AV, these can be set
1822 		 * together in the SQD->SQD transition.
1823 		 *
1824 		 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the
1825 		 * verbs layer driver does not track primary port changes
1826 		 * resulting from path migration. Thus, in SQD, if the primary
1827 		 * AV is modified, the primary port should also be modified).
1828 		 *
1829 		 * Note that in this transition, the IB_QP_STATE flag
1830 		 * is not allowed.
1831 		 */
1832 			if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1833 			     == (IB_QP_AV | IB_QP_PORT)) &&
1834 			    cmd->base.port_num != cmd->base.dest.port_num) {
1835 				ret = -EINVAL;
1836 				goto release_qp;
1837 			}
1838 			if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1839 			    == IB_QP_AV) {
1840 				cmd->base.attr_mask |= IB_QP_PORT;
1841 				cmd->base.port_num = cmd->base.dest.port_num;
1842 			}
1843 		}
1844 	}
1845 
1846 	if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
1847 	    (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
1848 	    !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) ||
1849 	    cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {
1850 		ret = -EINVAL;
1851 		goto release_qp;
1852 	}
1853 
1854 	if ((cmd->base.attr_mask & IB_QP_CUR_STATE &&
1855 	    cmd->base.cur_qp_state > IB_QPS_ERR) ||
1856 	    (cmd->base.attr_mask & IB_QP_STATE &&
1857 	    cmd->base.qp_state > IB_QPS_ERR)) {
1858 		ret = -EINVAL;
1859 		goto release_qp;
1860 	}
1861 
1862 	if (cmd->base.attr_mask & IB_QP_STATE)
1863 		attr->qp_state = cmd->base.qp_state;
1864 	if (cmd->base.attr_mask & IB_QP_CUR_STATE)
1865 		attr->cur_qp_state = cmd->base.cur_qp_state;
1866 	if (cmd->base.attr_mask & IB_QP_PATH_MTU)
1867 		attr->path_mtu = cmd->base.path_mtu;
1868 	if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)
1869 		attr->path_mig_state = cmd->base.path_mig_state;
1870 	if (cmd->base.attr_mask & IB_QP_QKEY)
1871 		attr->qkey = cmd->base.qkey;
1872 	if (cmd->base.attr_mask & IB_QP_RQ_PSN)
1873 		attr->rq_psn = cmd->base.rq_psn;
1874 	if (cmd->base.attr_mask & IB_QP_SQ_PSN)
1875 		attr->sq_psn = cmd->base.sq_psn;
1876 	if (cmd->base.attr_mask & IB_QP_DEST_QPN)
1877 		attr->dest_qp_num = cmd->base.dest_qp_num;
1878 	if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS)
1879 		attr->qp_access_flags = cmd->base.qp_access_flags;
1880 	if (cmd->base.attr_mask & IB_QP_PKEY_INDEX)
1881 		attr->pkey_index = cmd->base.pkey_index;
1882 	if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1883 		attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
1884 	if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
1885 		attr->max_rd_atomic = cmd->base.max_rd_atomic;
1886 	if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1887 		attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic;
1888 	if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER)
1889 		attr->min_rnr_timer = cmd->base.min_rnr_timer;
1890 	if (cmd->base.attr_mask & IB_QP_PORT)
1891 		attr->port_num = cmd->base.port_num;
1892 	if (cmd->base.attr_mask & IB_QP_TIMEOUT)
1893 		attr->timeout = cmd->base.timeout;
1894 	if (cmd->base.attr_mask & IB_QP_RETRY_CNT)
1895 		attr->retry_cnt = cmd->base.retry_cnt;
1896 	if (cmd->base.attr_mask & IB_QP_RNR_RETRY)
1897 		attr->rnr_retry = cmd->base.rnr_retry;
1898 	if (cmd->base.attr_mask & IB_QP_ALT_PATH) {
1899 		attr->alt_port_num = cmd->base.alt_port_num;
1900 		attr->alt_timeout = cmd->base.alt_timeout;
1901 		attr->alt_pkey_index = cmd->base.alt_pkey_index;
1902 	}
1903 	if (cmd->base.attr_mask & IB_QP_RATE_LIMIT)
1904 		attr->rate_limit = cmd->rate_limit;
1905 
1906 	if (cmd->base.attr_mask & IB_QP_AV)
1907 		copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
1908 					 &cmd->base.dest);
1909 
1910 	if (cmd->base.attr_mask & IB_QP_ALT_PATH)
1911 		copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
1912 					 &cmd->base.alt_dest);
1913 
1914 	ret = ib_modify_qp_with_udata(qp, attr,
1915 				      modify_qp_mask(qp->qp_type,
1916 						     cmd->base.attr_mask),
1917 				      &attrs->driver_udata);
1918 
1919 release_qp:
1920 	uobj_put_obj_read(qp);
1921 out:
1922 	kfree(attr);
1923 
1924 	return ret;
1925 }
1926 
1927 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
1928 {
1929 	struct ib_uverbs_ex_modify_qp cmd;
1930 	int ret;
1931 
1932 	ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));
1933 	if (ret)
1934 		return ret;
1935 
1936 	if (cmd.base.attr_mask &
1937 	    ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
1938 		return -EOPNOTSUPP;
1939 
1940 	return modify_qp(attrs, &cmd);
1941 }
1942 
1943 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
1944 {
1945 	struct ib_uverbs_ex_modify_qp cmd;
1946 	struct ib_uverbs_ex_modify_qp_resp resp = {
1947 		.response_length = uverbs_response_length(attrs, sizeof(resp))
1948 	};
1949 	int ret;
1950 
1951 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1952 	if (ret)
1953 		return ret;
1954 
1955 	/*
1956 	 * Last bit is reserved for extending the attr_mask by
1957 	 * using another field.
1958 	 */
1959 	BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
1960 
1961 	if (cmd.base.attr_mask &
1962 	    ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
1963 		return -EOPNOTSUPP;
1964 
1965 	ret = modify_qp(attrs, &cmd);
1966 	if (ret)
1967 		return ret;
1968 
1969 	return uverbs_response(attrs, &resp, sizeof(resp));
1970 }
1971 
1972 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
1973 {
1974 	struct ib_uverbs_destroy_qp      cmd;
1975 	struct ib_uverbs_destroy_qp_resp resp;
1976 	struct ib_uobject		*uobj;
1977 	struct ib_uqp_object        	*obj;
1978 	int ret;
1979 
1980 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
1981 	if (ret)
1982 		return ret;
1983 
1984 	uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1985 	if (IS_ERR(uobj))
1986 		return PTR_ERR(uobj);
1987 
1988 	obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
1989 	memset(&resp, 0, sizeof(resp));
1990 	resp.events_reported = obj->uevent.events_reported;
1991 
1992 	uobj_put_destroy(uobj);
1993 
1994 	return uverbs_response(attrs, &resp, sizeof(resp));
1995 }
1996 
1997 static void *alloc_wr(size_t wr_size, __u32 num_sge)
1998 {
1999 	if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) /
2000 		       sizeof (struct ib_sge))
2001 		return NULL;
2002 
2003 	return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +
2004 			 num_sge * sizeof (struct ib_sge), GFP_KERNEL);
2005 }
2006 
2007 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
2008 {
2009 	struct ib_uverbs_post_send      cmd;
2010 	struct ib_uverbs_post_send_resp resp;
2011 	struct ib_uverbs_send_wr       *user_wr;
2012 	struct ib_send_wr              *wr = NULL, *last, *next;
2013 	const struct ib_send_wr	       *bad_wr;
2014 	struct ib_qp                   *qp;
2015 	int                             i, sg_ind;
2016 	int				is_ud;
2017 	int ret, ret2;
2018 	size_t                          next_size;
2019 	const struct ib_sge __user *sgls;
2020 	const void __user *wqes;
2021 	struct uverbs_req_iter iter;
2022 
2023 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2024 	if (ret)
2025 		return ret;
2026 	wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);
2027 	if (IS_ERR(wqes))
2028 		return PTR_ERR(wqes);
2029 	sgls = uverbs_request_next_ptr(
2030 		&iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));
2031 	if (IS_ERR(sgls))
2032 		return PTR_ERR(sgls);
2033 	ret = uverbs_request_finish(&iter);
2034 	if (ret)
2035 		return ret;
2036 
2037 	user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2038 	if (!user_wr)
2039 		return -ENOMEM;
2040 
2041 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2042 	if (!qp) {
2043 		ret = -EINVAL;
2044 		goto out;
2045 	}
2046 
2047 	is_ud = qp->qp_type == IB_QPT_UD;
2048 	sg_ind = 0;
2049 	last = NULL;
2050 	for (i = 0; i < cmd.wr_count; ++i) {
2051 		if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,
2052 				   cmd.wqe_size)) {
2053 			ret = -EFAULT;
2054 			goto out_put;
2055 		}
2056 
2057 		if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2058 			ret = -EINVAL;
2059 			goto out_put;
2060 		}
2061 
2062 		if (is_ud) {
2063 			struct ib_ud_wr *ud;
2064 
2065 			if (user_wr->opcode != IB_WR_SEND &&
2066 			    user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2067 				ret = -EINVAL;
2068 				goto out_put;
2069 			}
2070 
2071 			next_size = sizeof(*ud);
2072 			ud = alloc_wr(next_size, user_wr->num_sge);
2073 			if (!ud) {
2074 				ret = -ENOMEM;
2075 				goto out_put;
2076 			}
2077 
2078 			ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
2079 						   user_wr->wr.ud.ah, attrs);
2080 			if (!ud->ah) {
2081 				kfree(ud);
2082 				ret = -EINVAL;
2083 				goto out_put;
2084 			}
2085 			ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2086 			ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2087 
2088 			next = &ud->wr;
2089 		} else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2090 			   user_wr->opcode == IB_WR_RDMA_WRITE ||
2091 			   user_wr->opcode == IB_WR_RDMA_READ) {
2092 			struct ib_rdma_wr *rdma;
2093 
2094 			next_size = sizeof(*rdma);
2095 			rdma = alloc_wr(next_size, user_wr->num_sge);
2096 			if (!rdma) {
2097 				ret = -ENOMEM;
2098 				goto out_put;
2099 			}
2100 
2101 			rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2102 			rdma->rkey = user_wr->wr.rdma.rkey;
2103 
2104 			next = &rdma->wr;
2105 		} else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2106 			   user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2107 			struct ib_atomic_wr *atomic;
2108 
2109 			next_size = sizeof(*atomic);
2110 			atomic = alloc_wr(next_size, user_wr->num_sge);
2111 			if (!atomic) {
2112 				ret = -ENOMEM;
2113 				goto out_put;
2114 			}
2115 
2116 			atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2117 			atomic->compare_add = user_wr->wr.atomic.compare_add;
2118 			atomic->swap = user_wr->wr.atomic.swap;
2119 			atomic->rkey = user_wr->wr.atomic.rkey;
2120 
2121 			next = &atomic->wr;
2122 		} else if (user_wr->opcode == IB_WR_SEND ||
2123 			   user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2124 			   user_wr->opcode == IB_WR_SEND_WITH_INV) {
2125 			next_size = sizeof(*next);
2126 			next = alloc_wr(next_size, user_wr->num_sge);
2127 			if (!next) {
2128 				ret = -ENOMEM;
2129 				goto out_put;
2130 			}
2131 		} else {
2132 			ret = -EINVAL;
2133 			goto out_put;
2134 		}
2135 
2136 		if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2137 		    user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2138 			next->ex.imm_data =
2139 					(__be32 __force) user_wr->ex.imm_data;
2140 		} else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2141 			next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2142 		}
2143 
2144 		if (!last)
2145 			wr = next;
2146 		else
2147 			last->next = next;
2148 		last = next;
2149 
2150 		next->next       = NULL;
2151 		next->wr_id      = user_wr->wr_id;
2152 		next->num_sge    = user_wr->num_sge;
2153 		next->opcode     = user_wr->opcode;
2154 		next->send_flags = user_wr->send_flags;
2155 
2156 		if (next->num_sge) {
2157 			next->sg_list = (void *) next +
2158 				ALIGN(next_size, sizeof(struct ib_sge));
2159 			if (copy_from_user(next->sg_list, sgls + sg_ind,
2160 					   next->num_sge *
2161 						   sizeof(struct ib_sge))) {
2162 				ret = -EFAULT;
2163 				goto out_put;
2164 			}
2165 			sg_ind += next->num_sge;
2166 		} else
2167 			next->sg_list = NULL;
2168 	}
2169 
2170 	resp.bad_wr = 0;
2171 	ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
2172 	if (ret)
2173 		for (next = wr; next; next = next->next) {
2174 			++resp.bad_wr;
2175 			if (next == bad_wr)
2176 				break;
2177 		}
2178 
2179 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2180 	if (ret2)
2181 		ret = ret2;
2182 
2183 out_put:
2184 	uobj_put_obj_read(qp);
2185 
2186 	while (wr) {
2187 		if (is_ud && ud_wr(wr)->ah)
2188 			uobj_put_obj_read(ud_wr(wr)->ah);
2189 		next = wr->next;
2190 		kfree(wr);
2191 		wr = next;
2192 	}
2193 
2194 out:
2195 	kfree(user_wr);
2196 
2197 	return ret;
2198 }
2199 
2200 static struct ib_recv_wr *
2201 ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
2202 			  u32 wqe_size, u32 sge_count)
2203 {
2204 	struct ib_uverbs_recv_wr *user_wr;
2205 	struct ib_recv_wr        *wr = NULL, *last, *next;
2206 	int                       sg_ind;
2207 	int                       i;
2208 	int                       ret;
2209 	const struct ib_sge __user *sgls;
2210 	const void __user *wqes;
2211 
2212 	if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2213 		return ERR_PTR(-EINVAL);
2214 
2215 	wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);
2216 	if (IS_ERR(wqes))
2217 		return ERR_CAST(wqes);
2218 	sgls = uverbs_request_next_ptr(
2219 		iter, sge_count * sizeof(struct ib_uverbs_sge));
2220 	if (IS_ERR(sgls))
2221 		return ERR_CAST(sgls);
2222 	ret = uverbs_request_finish(iter);
2223 	if (ret)
2224 		return ERR_PTR(ret);
2225 
2226 	user_wr = kmalloc(wqe_size, GFP_KERNEL);
2227 	if (!user_wr)
2228 		return ERR_PTR(-ENOMEM);
2229 
2230 	sg_ind = 0;
2231 	last = NULL;
2232 	for (i = 0; i < wr_count; ++i) {
2233 		if (copy_from_user(user_wr, wqes + i * wqe_size,
2234 				   wqe_size)) {
2235 			ret = -EFAULT;
2236 			goto err;
2237 		}
2238 
2239 		if (user_wr->num_sge + sg_ind > sge_count) {
2240 			ret = -EINVAL;
2241 			goto err;
2242 		}
2243 
2244 		if (user_wr->num_sge >=
2245 		    (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) /
2246 		    sizeof (struct ib_sge)) {
2247 			ret = -EINVAL;
2248 			goto err;
2249 		}
2250 
2251 		next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2252 			       user_wr->num_sge * sizeof (struct ib_sge),
2253 			       GFP_KERNEL);
2254 		if (!next) {
2255 			ret = -ENOMEM;
2256 			goto err;
2257 		}
2258 
2259 		if (!last)
2260 			wr = next;
2261 		else
2262 			last->next = next;
2263 		last = next;
2264 
2265 		next->next       = NULL;
2266 		next->wr_id      = user_wr->wr_id;
2267 		next->num_sge    = user_wr->num_sge;
2268 
2269 		if (next->num_sge) {
2270 			next->sg_list = (void *) next +
2271 				ALIGN(sizeof *next, sizeof (struct ib_sge));
2272 			if (copy_from_user(next->sg_list, sgls + sg_ind,
2273 					   next->num_sge *
2274 						   sizeof(struct ib_sge))) {
2275 				ret = -EFAULT;
2276 				goto err;
2277 			}
2278 			sg_ind += next->num_sge;
2279 		} else
2280 			next->sg_list = NULL;
2281 	}
2282 
2283 	kfree(user_wr);
2284 	return wr;
2285 
2286 err:
2287 	kfree(user_wr);
2288 
2289 	while (wr) {
2290 		next = wr->next;
2291 		kfree(wr);
2292 		wr = next;
2293 	}
2294 
2295 	return ERR_PTR(ret);
2296 }
2297 
2298 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
2299 {
2300 	struct ib_uverbs_post_recv      cmd;
2301 	struct ib_uverbs_post_recv_resp resp;
2302 	struct ib_recv_wr              *wr, *next;
2303 	const struct ib_recv_wr	       *bad_wr;
2304 	struct ib_qp                   *qp;
2305 	int ret, ret2;
2306 	struct uverbs_req_iter iter;
2307 
2308 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2309 	if (ret)
2310 		return ret;
2311 
2312 	wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2313 				       cmd.sge_count);
2314 	if (IS_ERR(wr))
2315 		return PTR_ERR(wr);
2316 
2317 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2318 	if (!qp) {
2319 		ret = -EINVAL;
2320 		goto out;
2321 	}
2322 
2323 	resp.bad_wr = 0;
2324 	ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
2325 
2326 	uobj_put_obj_read(qp);
2327 	if (ret) {
2328 		for (next = wr; next; next = next->next) {
2329 			++resp.bad_wr;
2330 			if (next == bad_wr)
2331 				break;
2332 		}
2333 	}
2334 
2335 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2336 	if (ret2)
2337 		ret = ret2;
2338 out:
2339 	while (wr) {
2340 		next = wr->next;
2341 		kfree(wr);
2342 		wr = next;
2343 	}
2344 
2345 	return ret;
2346 }
2347 
2348 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
2349 {
2350 	struct ib_uverbs_post_srq_recv      cmd;
2351 	struct ib_uverbs_post_srq_recv_resp resp;
2352 	struct ib_recv_wr                  *wr, *next;
2353 	const struct ib_recv_wr		   *bad_wr;
2354 	struct ib_srq                      *srq;
2355 	int ret, ret2;
2356 	struct uverbs_req_iter iter;
2357 
2358 	ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
2359 	if (ret)
2360 		return ret;
2361 
2362 	wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
2363 				       cmd.sge_count);
2364 	if (IS_ERR(wr))
2365 		return PTR_ERR(wr);
2366 
2367 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
2368 	if (!srq) {
2369 		ret = -EINVAL;
2370 		goto out;
2371 	}
2372 
2373 	resp.bad_wr = 0;
2374 	ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
2375 
2376 	uobj_put_obj_read(srq);
2377 
2378 	if (ret)
2379 		for (next = wr; next; next = next->next) {
2380 			++resp.bad_wr;
2381 			if (next == bad_wr)
2382 				break;
2383 		}
2384 
2385 	ret2 = uverbs_response(attrs, &resp, sizeof(resp));
2386 	if (ret2)
2387 		ret = ret2;
2388 
2389 out:
2390 	while (wr) {
2391 		next = wr->next;
2392 		kfree(wr);
2393 		wr = next;
2394 	}
2395 
2396 	return ret;
2397 }
2398 
2399 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
2400 {
2401 	struct ib_uverbs_create_ah	 cmd;
2402 	struct ib_uverbs_create_ah_resp	 resp;
2403 	struct ib_uobject		*uobj;
2404 	struct ib_pd			*pd;
2405 	struct ib_ah			*ah;
2406 	struct rdma_ah_attr		attr = {};
2407 	int ret;
2408 	struct ib_device *ib_dev;
2409 
2410 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2411 	if (ret)
2412 		return ret;
2413 
2414 	uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
2415 	if (IS_ERR(uobj))
2416 		return PTR_ERR(uobj);
2417 
2418 	if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) {
2419 		ret = -EINVAL;
2420 		goto err;
2421 	}
2422 
2423 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2424 	if (!pd) {
2425 		ret = -EINVAL;
2426 		goto err;
2427 	}
2428 
2429 	attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
2430 	rdma_ah_set_make_grd(&attr, false);
2431 	rdma_ah_set_dlid(&attr, cmd.attr.dlid);
2432 	rdma_ah_set_sl(&attr, cmd.attr.sl);
2433 	rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
2434 	rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);
2435 	rdma_ah_set_port_num(&attr, cmd.attr.port_num);
2436 
2437 	if (cmd.attr.is_global) {
2438 		rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,
2439 				cmd.attr.grh.sgid_index,
2440 				cmd.attr.grh.hop_limit,
2441 				cmd.attr.grh.traffic_class);
2442 		rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);
2443 	} else {
2444 		rdma_ah_set_ah_flags(&attr, 0);
2445 	}
2446 
2447 	ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);
2448 	if (IS_ERR(ah)) {
2449 		ret = PTR_ERR(ah);
2450 		goto err_put;
2451 	}
2452 
2453 	ah->uobject  = uobj;
2454 	uobj->user_handle = cmd.user_handle;
2455 	uobj->object = ah;
2456 
2457 	resp.ah_handle = uobj->id;
2458 
2459 	ret = uverbs_response(attrs, &resp, sizeof(resp));
2460 	if (ret)
2461 		goto err_copy;
2462 
2463 	uobj_put_obj_read(pd);
2464 	return uobj_alloc_commit(uobj, attrs);
2465 
2466 err_copy:
2467 	rdma_destroy_ah_user(ah, RDMA_DESTROY_AH_SLEEPABLE,
2468 			     uverbs_get_cleared_udata(attrs));
2469 
2470 err_put:
2471 	uobj_put_obj_read(pd);
2472 
2473 err:
2474 	uobj_alloc_abort(uobj, attrs);
2475 	return ret;
2476 }
2477 
2478 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
2479 {
2480 	struct ib_uverbs_destroy_ah cmd;
2481 	int ret;
2482 
2483 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2484 	if (ret)
2485 		return ret;
2486 
2487 	return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
2488 }
2489 
2490 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
2491 {
2492 	struct ib_uverbs_attach_mcast cmd;
2493 	struct ib_qp                 *qp;
2494 	struct ib_uqp_object         *obj;
2495 	struct ib_uverbs_mcast_entry *mcast;
2496 	int                           ret;
2497 
2498 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2499 	if (ret)
2500 		return ret;
2501 
2502 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2503 	if (!qp)
2504 		return -EINVAL;
2505 
2506 	obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2507 
2508 	mutex_lock(&obj->mcast_lock);
2509 	list_for_each_entry(mcast, &obj->mcast_list, list)
2510 		if (cmd.mlid == mcast->lid &&
2511 		    !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2512 			ret = 0;
2513 			goto out_put;
2514 		}
2515 
2516 	mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2517 	if (!mcast) {
2518 		ret = -ENOMEM;
2519 		goto out_put;
2520 	}
2521 
2522 	mcast->lid = cmd.mlid;
2523 	memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2524 
2525 	ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2526 	if (!ret)
2527 		list_add_tail(&mcast->list, &obj->mcast_list);
2528 	else
2529 		kfree(mcast);
2530 
2531 out_put:
2532 	mutex_unlock(&obj->mcast_lock);
2533 	uobj_put_obj_read(qp);
2534 
2535 	return ret;
2536 }
2537 
2538 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
2539 {
2540 	struct ib_uverbs_detach_mcast cmd;
2541 	struct ib_uqp_object         *obj;
2542 	struct ib_qp                 *qp;
2543 	struct ib_uverbs_mcast_entry *mcast;
2544 	int                           ret = -EINVAL;
2545 	bool                          found = false;
2546 
2547 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2548 	if (ret)
2549 		return ret;
2550 
2551 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2552 	if (!qp)
2553 		return -EINVAL;
2554 
2555 	obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2556 	mutex_lock(&obj->mcast_lock);
2557 
2558 	list_for_each_entry(mcast, &obj->mcast_list, list)
2559 		if (cmd.mlid == mcast->lid &&
2560 		    !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2561 			list_del(&mcast->list);
2562 			kfree(mcast);
2563 			found = true;
2564 			break;
2565 		}
2566 
2567 	if (!found) {
2568 		ret = -EINVAL;
2569 		goto out_put;
2570 	}
2571 
2572 	ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);
2573 
2574 out_put:
2575 	mutex_unlock(&obj->mcast_lock);
2576 	uobj_put_obj_read(qp);
2577 	return ret;
2578 }
2579 
2580 struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
2581 {
2582 	struct ib_uflow_resources *resources;
2583 
2584 	resources = kzalloc(sizeof(*resources), GFP_KERNEL);
2585 
2586 	if (!resources)
2587 		return NULL;
2588 
2589 	if (!num_specs)
2590 		goto out;
2591 
2592 	resources->counters =
2593 		kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL);
2594 	resources->collection =
2595 		kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL);
2596 
2597 	if (!resources->counters || !resources->collection)
2598 		goto err;
2599 
2600 out:
2601 	resources->max = num_specs;
2602 	return resources;
2603 
2604 err:
2605 	kfree(resources->counters);
2606 	kfree(resources);
2607 
2608 	return NULL;
2609 }
2610 EXPORT_SYMBOL(flow_resources_alloc);
2611 
2612 void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2613 {
2614 	unsigned int i;
2615 
2616 	if (!uflow_res)
2617 		return;
2618 
2619 	for (i = 0; i < uflow_res->collection_num; i++)
2620 		atomic_dec(&uflow_res->collection[i]->usecnt);
2621 
2622 	for (i = 0; i < uflow_res->counters_num; i++)
2623 		atomic_dec(&uflow_res->counters[i]->usecnt);
2624 
2625 	kfree(uflow_res->collection);
2626 	kfree(uflow_res->counters);
2627 	kfree(uflow_res);
2628 }
2629 EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
2630 
2631 void flow_resources_add(struct ib_uflow_resources *uflow_res,
2632 			enum ib_flow_spec_type type,
2633 			void *ibobj)
2634 {
2635 	WARN_ON(uflow_res->num >= uflow_res->max);
2636 
2637 	switch (type) {
2638 	case IB_FLOW_SPEC_ACTION_HANDLE:
2639 		atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt);
2640 		uflow_res->collection[uflow_res->collection_num++] =
2641 			(struct ib_flow_action *)ibobj;
2642 		break;
2643 	case IB_FLOW_SPEC_ACTION_COUNT:
2644 		atomic_inc(&((struct ib_counters *)ibobj)->usecnt);
2645 		uflow_res->counters[uflow_res->counters_num++] =
2646 			(struct ib_counters *)ibobj;
2647 		break;
2648 	default:
2649 		WARN_ON(1);
2650 	}
2651 
2652 	uflow_res->num++;
2653 }
2654 EXPORT_SYMBOL(flow_resources_add);
2655 
2656 static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,
2657 				       struct ib_uverbs_flow_spec *kern_spec,
2658 				       union ib_flow_spec *ib_spec,
2659 				       struct ib_uflow_resources *uflow_res)
2660 {
2661 	ib_spec->type = kern_spec->type;
2662 	switch (ib_spec->type) {
2663 	case IB_FLOW_SPEC_ACTION_TAG:
2664 		if (kern_spec->flow_tag.size !=
2665 		    sizeof(struct ib_uverbs_flow_spec_action_tag))
2666 			return -EINVAL;
2667 
2668 		ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2669 		ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2670 		break;
2671 	case IB_FLOW_SPEC_ACTION_DROP:
2672 		if (kern_spec->drop.size !=
2673 		    sizeof(struct ib_uverbs_flow_spec_action_drop))
2674 			return -EINVAL;
2675 
2676 		ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2677 		break;
2678 	case IB_FLOW_SPEC_ACTION_HANDLE:
2679 		if (kern_spec->action.size !=
2680 		    sizeof(struct ib_uverbs_flow_spec_action_handle))
2681 			return -EOPNOTSUPP;
2682 		ib_spec->action.act = uobj_get_obj_read(flow_action,
2683 							UVERBS_OBJECT_FLOW_ACTION,
2684 							kern_spec->action.handle,
2685 							attrs);
2686 		if (!ib_spec->action.act)
2687 			return -EINVAL;
2688 		ib_spec->action.size =
2689 			sizeof(struct ib_flow_spec_action_handle);
2690 		flow_resources_add(uflow_res,
2691 				   IB_FLOW_SPEC_ACTION_HANDLE,
2692 				   ib_spec->action.act);
2693 		uobj_put_obj_read(ib_spec->action.act);
2694 		break;
2695 	case IB_FLOW_SPEC_ACTION_COUNT:
2696 		if (kern_spec->flow_count.size !=
2697 			sizeof(struct ib_uverbs_flow_spec_action_count))
2698 			return -EINVAL;
2699 		ib_spec->flow_count.counters =
2700 			uobj_get_obj_read(counters,
2701 					  UVERBS_OBJECT_COUNTERS,
2702 					  kern_spec->flow_count.handle,
2703 					  attrs);
2704 		if (!ib_spec->flow_count.counters)
2705 			return -EINVAL;
2706 		ib_spec->flow_count.size =
2707 				sizeof(struct ib_flow_spec_action_count);
2708 		flow_resources_add(uflow_res,
2709 				   IB_FLOW_SPEC_ACTION_COUNT,
2710 				   ib_spec->flow_count.counters);
2711 		uobj_put_obj_read(ib_spec->flow_count.counters);
2712 		break;
2713 	default:
2714 		return -EINVAL;
2715 	}
2716 	return 0;
2717 }
2718 
2719 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec)
2720 {
2721 	/* Returns user space filter size, includes padding */
2722 	return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;
2723 }
2724 
2725 static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,
2726 				u16 ib_real_filter_sz)
2727 {
2728 	/*
2729 	 * User space filter structures must be 64 bit aligned, otherwise this
2730 	 * may pass, but we won't handle additional new attributes.
2731 	 */
2732 
2733 	if (kern_filter_size > ib_real_filter_sz) {
2734 		if (memchr_inv(kern_spec_filter +
2735 			       ib_real_filter_sz, 0,
2736 			       kern_filter_size - ib_real_filter_sz))
2737 			return -EINVAL;
2738 		return ib_real_filter_sz;
2739 	}
2740 	return kern_filter_size;
2741 }
2742 
2743 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
2744 					  const void *kern_spec_mask,
2745 					  const void *kern_spec_val,
2746 					  size_t kern_filter_sz,
2747 					  union ib_flow_spec *ib_spec)
2748 {
2749 	ssize_t actual_filter_sz;
2750 	ssize_t ib_filter_sz;
2751 
2752 	/* User flow spec size must be aligned to 4 bytes */
2753 	if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2754 		return -EINVAL;
2755 
2756 	ib_spec->type = type;
2757 
2758 	if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2759 		return -EINVAL;
2760 
2761 	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2762 	case IB_FLOW_SPEC_ETH:
2763 		ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
2764 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2765 						    kern_filter_sz,
2766 						    ib_filter_sz);
2767 		if (actual_filter_sz <= 0)
2768 			return -EINVAL;
2769 		ib_spec->size = sizeof(struct ib_flow_spec_eth);
2770 		memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2771 		memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
2772 		break;
2773 	case IB_FLOW_SPEC_IPV4:
2774 		ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);
2775 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2776 						    kern_filter_sz,
2777 						    ib_filter_sz);
2778 		if (actual_filter_sz <= 0)
2779 			return -EINVAL;
2780 		ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2781 		memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2782 		memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
2783 		break;
2784 	case IB_FLOW_SPEC_IPV6:
2785 		ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);
2786 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2787 						    kern_filter_sz,
2788 						    ib_filter_sz);
2789 		if (actual_filter_sz <= 0)
2790 			return -EINVAL;
2791 		ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2792 		memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2793 		memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
2794 
2795 		if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2796 		    (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2797 			return -EINVAL;
2798 		break;
2799 	case IB_FLOW_SPEC_TCP:
2800 	case IB_FLOW_SPEC_UDP:
2801 		ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);
2802 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2803 						    kern_filter_sz,
2804 						    ib_filter_sz);
2805 		if (actual_filter_sz <= 0)
2806 			return -EINVAL;
2807 		ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2808 		memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2809 		memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
2810 		break;
2811 	case IB_FLOW_SPEC_VXLAN_TUNNEL:
2812 		ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);
2813 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2814 						    kern_filter_sz,
2815 						    ib_filter_sz);
2816 		if (actual_filter_sz <= 0)
2817 			return -EINVAL;
2818 		ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2819 		memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2820 		memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2821 
2822 		if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2823 		    (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2824 			return -EINVAL;
2825 		break;
2826 	case IB_FLOW_SPEC_ESP:
2827 		ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz);
2828 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2829 						    kern_filter_sz,
2830 						    ib_filter_sz);
2831 		if (actual_filter_sz <= 0)
2832 			return -EINVAL;
2833 		ib_spec->esp.size = sizeof(struct ib_flow_spec_esp);
2834 		memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);
2835 		memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);
2836 		break;
2837 	case IB_FLOW_SPEC_GRE:
2838 		ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz);
2839 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2840 						    kern_filter_sz,
2841 						    ib_filter_sz);
2842 		if (actual_filter_sz <= 0)
2843 			return -EINVAL;
2844 		ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);
2845 		memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);
2846 		memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);
2847 		break;
2848 	case IB_FLOW_SPEC_MPLS:
2849 		ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz);
2850 		actual_filter_sz = spec_filter_size(kern_spec_mask,
2851 						    kern_filter_sz,
2852 						    ib_filter_sz);
2853 		if (actual_filter_sz <= 0)
2854 			return -EINVAL;
2855 		ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);
2856 		memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);
2857 		memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);
2858 		break;
2859 	default:
2860 		return -EINVAL;
2861 	}
2862 	return 0;
2863 }
2864 
2865 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2866 				       union ib_flow_spec *ib_spec)
2867 {
2868 	ssize_t kern_filter_sz;
2869 	void *kern_spec_mask;
2870 	void *kern_spec_val;
2871 
2872 	kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr);
2873 
2874 	kern_spec_val = (void *)kern_spec +
2875 		sizeof(struct ib_uverbs_flow_spec_hdr);
2876 	kern_spec_mask = kern_spec_val + kern_filter_sz;
2877 
2878 	return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type,
2879 						     kern_spec_mask,
2880 						     kern_spec_val,
2881 						     kern_filter_sz, ib_spec);
2882 }
2883 
2884 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
2885 				struct ib_uverbs_flow_spec *kern_spec,
2886 				union ib_flow_spec *ib_spec,
2887 				struct ib_uflow_resources *uflow_res)
2888 {
2889 	if (kern_spec->reserved)
2890 		return -EINVAL;
2891 
2892 	if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
2893 		return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
2894 						   uflow_res);
2895 	else
2896 		return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2897 }
2898 
2899 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
2900 {
2901 	struct ib_uverbs_ex_create_wq cmd;
2902 	struct ib_uverbs_ex_create_wq_resp resp = {};
2903 	struct ib_uwq_object           *obj;
2904 	int err = 0;
2905 	struct ib_cq *cq;
2906 	struct ib_pd *pd;
2907 	struct ib_wq *wq;
2908 	struct ib_wq_init_attr wq_init_attr = {};
2909 	struct ib_device *ib_dev;
2910 
2911 	err = uverbs_request(attrs, &cmd, sizeof(cmd));
2912 	if (err)
2913 		return err;
2914 
2915 	if (cmd.comp_mask)
2916 		return -EOPNOTSUPP;
2917 
2918 	obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
2919 						 &ib_dev);
2920 	if (IS_ERR(obj))
2921 		return PTR_ERR(obj);
2922 
2923 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2924 	if (!pd) {
2925 		err = -EINVAL;
2926 		goto err_uobj;
2927 	}
2928 
2929 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
2930 	if (!cq) {
2931 		err = -EINVAL;
2932 		goto err_put_pd;
2933 	}
2934 
2935 	wq_init_attr.cq = cq;
2936 	wq_init_attr.max_sge = cmd.max_sge;
2937 	wq_init_attr.max_wr = cmd.max_wr;
2938 	wq_init_attr.wq_context = attrs->ufile;
2939 	wq_init_attr.wq_type = cmd.wq_type;
2940 	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
2941 	wq_init_attr.create_flags = cmd.create_flags;
2942 	obj->uevent.events_reported = 0;
2943 	INIT_LIST_HEAD(&obj->uevent.event_list);
2944 
2945 	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
2946 	if (IS_ERR(wq)) {
2947 		err = PTR_ERR(wq);
2948 		goto err_put_cq;
2949 	}
2950 
2951 	wq->uobject = &obj->uevent.uobject;
2952 	obj->uevent.uobject.object = wq;
2953 	wq->wq_type = wq_init_attr.wq_type;
2954 	wq->cq = cq;
2955 	wq->pd = pd;
2956 	wq->device = pd->device;
2957 	wq->wq_context = wq_init_attr.wq_context;
2958 	atomic_set(&wq->usecnt, 0);
2959 	atomic_inc(&pd->usecnt);
2960 	atomic_inc(&cq->usecnt);
2961 	wq->uobject = &obj->uevent.uobject;
2962 	obj->uevent.uobject.object = wq;
2963 
2964 	memset(&resp, 0, sizeof(resp));
2965 	resp.wq_handle = obj->uevent.uobject.id;
2966 	resp.max_sge = wq_init_attr.max_sge;
2967 	resp.max_wr = wq_init_attr.max_wr;
2968 	resp.wqn = wq->wq_num;
2969 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
2970 	err = uverbs_response(attrs, &resp, sizeof(resp));
2971 	if (err)
2972 		goto err_copy;
2973 
2974 	uobj_put_obj_read(pd);
2975 	uobj_put_obj_read(cq);
2976 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
2977 
2978 err_copy:
2979 	ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs));
2980 err_put_cq:
2981 	uobj_put_obj_read(cq);
2982 err_put_pd:
2983 	uobj_put_obj_read(pd);
2984 err_uobj:
2985 	uobj_alloc_abort(&obj->uevent.uobject, attrs);
2986 
2987 	return err;
2988 }
2989 
2990 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
2991 {
2992 	struct ib_uverbs_ex_destroy_wq	cmd;
2993 	struct ib_uverbs_ex_destroy_wq_resp	resp = {};
2994 	struct ib_uobject		*uobj;
2995 	struct ib_uwq_object		*obj;
2996 	int				ret;
2997 
2998 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
2999 	if (ret)
3000 		return ret;
3001 
3002 	if (cmd.comp_mask)
3003 		return -EOPNOTSUPP;
3004 
3005 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3006 	uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3007 	if (IS_ERR(uobj))
3008 		return PTR_ERR(uobj);
3009 
3010 	obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
3011 	resp.events_reported = obj->uevent.events_reported;
3012 
3013 	uobj_put_destroy(uobj);
3014 
3015 	return uverbs_response(attrs, &resp, sizeof(resp));
3016 }
3017 
3018 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
3019 {
3020 	struct ib_uverbs_ex_modify_wq cmd;
3021 	struct ib_wq *wq;
3022 	struct ib_wq_attr wq_attr = {};
3023 	int ret;
3024 
3025 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3026 	if (ret)
3027 		return ret;
3028 
3029 	if (!cmd.attr_mask)
3030 		return -EINVAL;
3031 
3032 	if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
3033 		return -EINVAL;
3034 
3035 	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3036 	if (!wq)
3037 		return -EINVAL;
3038 
3039 	wq_attr.curr_wq_state = cmd.curr_wq_state;
3040 	wq_attr.wq_state = cmd.wq_state;
3041 	if (cmd.attr_mask & IB_WQ_FLAGS) {
3042 		wq_attr.flags = cmd.flags;
3043 		wq_attr.flags_mask = cmd.flags_mask;
3044 	}
3045 	ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
3046 					&attrs->driver_udata);
3047 	uobj_put_obj_read(wq);
3048 	return ret;
3049 }
3050 
3051 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3052 {
3053 	struct ib_uverbs_ex_create_rwq_ind_table cmd;
3054 	struct ib_uverbs_ex_create_rwq_ind_table_resp  resp = {};
3055 	struct ib_uobject		  *uobj;
3056 	int err;
3057 	struct ib_rwq_ind_table_init_attr init_attr = {};
3058 	struct ib_rwq_ind_table *rwq_ind_tbl;
3059 	struct ib_wq	**wqs = NULL;
3060 	u32 *wqs_handles = NULL;
3061 	struct ib_wq	*wq = NULL;
3062 	int i, j, num_read_wqs;
3063 	u32 num_wq_handles;
3064 	struct uverbs_req_iter iter;
3065 	struct ib_device *ib_dev;
3066 
3067 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3068 	if (err)
3069 		return err;
3070 
3071 	if (cmd.comp_mask)
3072 		return -EOPNOTSUPP;
3073 
3074 	if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3075 		return -EINVAL;
3076 
3077 	num_wq_handles = 1 << cmd.log_ind_tbl_size;
3078 	wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3079 			      GFP_KERNEL);
3080 	if (!wqs_handles)
3081 		return -ENOMEM;
3082 
3083 	err = uverbs_request_next(&iter, wqs_handles,
3084 				  num_wq_handles * sizeof(__u32));
3085 	if (err)
3086 		goto err_free;
3087 
3088 	err = uverbs_request_finish(&iter);
3089 	if (err)
3090 		goto err_free;
3091 
3092 	wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);
3093 	if (!wqs) {
3094 		err = -ENOMEM;
3095 		goto  err_free;
3096 	}
3097 
3098 	for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3099 			num_read_wqs++) {
3100 		wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
3101 				       wqs_handles[num_read_wqs], attrs);
3102 		if (!wq) {
3103 			err = -EINVAL;
3104 			goto put_wqs;
3105 		}
3106 
3107 		wqs[num_read_wqs] = wq;
3108 	}
3109 
3110 	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
3111 	if (IS_ERR(uobj)) {
3112 		err = PTR_ERR(uobj);
3113 		goto put_wqs;
3114 	}
3115 
3116 	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3117 	init_attr.ind_tbl = wqs;
3118 
3119 	rwq_ind_tbl = ib_dev->ops.create_rwq_ind_table(ib_dev, &init_attr,
3120 						       &attrs->driver_udata);
3121 
3122 	if (IS_ERR(rwq_ind_tbl)) {
3123 		err = PTR_ERR(rwq_ind_tbl);
3124 		goto err_uobj;
3125 	}
3126 
3127 	rwq_ind_tbl->ind_tbl = wqs;
3128 	rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3129 	rwq_ind_tbl->uobject = uobj;
3130 	uobj->object = rwq_ind_tbl;
3131 	rwq_ind_tbl->device = ib_dev;
3132 	atomic_set(&rwq_ind_tbl->usecnt, 0);
3133 
3134 	for (i = 0; i < num_wq_handles; i++)
3135 		atomic_inc(&wqs[i]->usecnt);
3136 
3137 	resp.ind_tbl_handle = uobj->id;
3138 	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3139 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3140 
3141 	err = uverbs_response(attrs, &resp, sizeof(resp));
3142 	if (err)
3143 		goto err_copy;
3144 
3145 	kfree(wqs_handles);
3146 
3147 	for (j = 0; j < num_read_wqs; j++)
3148 		uobj_put_obj_read(wqs[j]);
3149 
3150 	return uobj_alloc_commit(uobj, attrs);
3151 
3152 err_copy:
3153 	ib_destroy_rwq_ind_table(rwq_ind_tbl);
3154 err_uobj:
3155 	uobj_alloc_abort(uobj, attrs);
3156 put_wqs:
3157 	for (j = 0; j < num_read_wqs; j++)
3158 		uobj_put_obj_read(wqs[j]);
3159 err_free:
3160 	kfree(wqs_handles);
3161 	kfree(wqs);
3162 	return err;
3163 }
3164 
3165 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3166 {
3167 	struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
3168 	int ret;
3169 
3170 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3171 	if (ret)
3172 		return ret;
3173 
3174 	if (cmd.comp_mask)
3175 		return -EOPNOTSUPP;
3176 
3177 	return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
3178 				    cmd.ind_tbl_handle, attrs);
3179 }
3180 
3181 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3182 {
3183 	struct ib_uverbs_create_flow	  cmd;
3184 	struct ib_uverbs_create_flow_resp resp;
3185 	struct ib_uobject		  *uobj;
3186 	struct ib_flow			  *flow_id;
3187 	struct ib_uverbs_flow_attr	  *kern_flow_attr;
3188 	struct ib_flow_attr		  *flow_attr;
3189 	struct ib_qp			  *qp;
3190 	struct ib_uflow_resources	  *uflow_res;
3191 	struct ib_uverbs_flow_spec_hdr	  *kern_spec;
3192 	struct uverbs_req_iter iter;
3193 	int err;
3194 	void *ib_spec;
3195 	int i;
3196 	struct ib_device *ib_dev;
3197 
3198 	err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3199 	if (err)
3200 		return err;
3201 
3202 	if (cmd.comp_mask)
3203 		return -EINVAL;
3204 
3205 	if (!capable(CAP_NET_RAW))
3206 		return -EPERM;
3207 
3208 	if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3209 		return -EINVAL;
3210 
3211 	if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3212 	    ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3213 	     (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3214 		return -EINVAL;
3215 
3216 	if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3217 		return -EINVAL;
3218 
3219 	if (cmd.flow_attr.size >
3220 	    (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3221 		return -EINVAL;
3222 
3223 	if (cmd.flow_attr.reserved[0] ||
3224 	    cmd.flow_attr.reserved[1])
3225 		return -EINVAL;
3226 
3227 	if (cmd.flow_attr.num_of_specs) {
3228 		kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3229 					 GFP_KERNEL);
3230 		if (!kern_flow_attr)
3231 			return -ENOMEM;
3232 
3233 		*kern_flow_attr = cmd.flow_attr;
3234 		err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,
3235 					  cmd.flow_attr.size);
3236 		if (err)
3237 			goto err_free_attr;
3238 	} else {
3239 		kern_flow_attr = &cmd.flow_attr;
3240 	}
3241 
3242 	err = uverbs_request_finish(&iter);
3243 	if (err)
3244 		goto err_free_attr;
3245 
3246 	uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
3247 	if (IS_ERR(uobj)) {
3248 		err = PTR_ERR(uobj);
3249 		goto err_free_attr;
3250 	}
3251 
3252 	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
3253 	if (!qp) {
3254 		err = -EINVAL;
3255 		goto err_uobj;
3256 	}
3257 
3258 	if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3259 		err = -EINVAL;
3260 		goto err_put;
3261 	}
3262 
3263 	flow_attr = kzalloc(struct_size(flow_attr, flows,
3264 				cmd.flow_attr.num_of_specs), GFP_KERNEL);
3265 	if (!flow_attr) {
3266 		err = -ENOMEM;
3267 		goto err_put;
3268 	}
3269 	uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs);
3270 	if (!uflow_res) {
3271 		err = -ENOMEM;
3272 		goto err_free_flow_attr;
3273 	}
3274 
3275 	flow_attr->type = kern_flow_attr->type;
3276 	flow_attr->priority = kern_flow_attr->priority;
3277 	flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3278 	flow_attr->port = kern_flow_attr->port;
3279 	flow_attr->flags = kern_flow_attr->flags;
3280 	flow_attr->size = sizeof(*flow_attr);
3281 
3282 	kern_spec = kern_flow_attr->flow_specs;
3283 	ib_spec = flow_attr + 1;
3284 	for (i = 0; i < flow_attr->num_of_specs &&
3285 			cmd.flow_attr.size >= sizeof(*kern_spec) &&
3286 			cmd.flow_attr.size >= kern_spec->size;
3287 	     i++) {
3288 		err = kern_spec_to_ib_spec(
3289 				attrs, (struct ib_uverbs_flow_spec *)kern_spec,
3290 				ib_spec, uflow_res);
3291 		if (err)
3292 			goto err_free;
3293 
3294 		flow_attr->size +=
3295 			((union ib_flow_spec *) ib_spec)->size;
3296 		cmd.flow_attr.size -= kern_spec->size;
3297 		kern_spec = ((void *)kern_spec) + kern_spec->size;
3298 		ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3299 	}
3300 	if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3301 		pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3302 			i, cmd.flow_attr.size);
3303 		err = -EINVAL;
3304 		goto err_free;
3305 	}
3306 
3307 	flow_id = qp->device->ops.create_flow(
3308 		qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata);
3309 
3310 	if (IS_ERR(flow_id)) {
3311 		err = PTR_ERR(flow_id);
3312 		goto err_free;
3313 	}
3314 
3315 	ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
3316 
3317 	memset(&resp, 0, sizeof(resp));
3318 	resp.flow_handle = uobj->id;
3319 
3320 	err = uverbs_response(attrs, &resp, sizeof(resp));
3321 	if (err)
3322 		goto err_copy;
3323 
3324 	uobj_put_obj_read(qp);
3325 	kfree(flow_attr);
3326 	if (cmd.flow_attr.num_of_specs)
3327 		kfree(kern_flow_attr);
3328 	return uobj_alloc_commit(uobj, attrs);
3329 err_copy:
3330 	if (!qp->device->ops.destroy_flow(flow_id))
3331 		atomic_dec(&qp->usecnt);
3332 err_free:
3333 	ib_uverbs_flow_resources_free(uflow_res);
3334 err_free_flow_attr:
3335 	kfree(flow_attr);
3336 err_put:
3337 	uobj_put_obj_read(qp);
3338 err_uobj:
3339 	uobj_alloc_abort(uobj, attrs);
3340 err_free_attr:
3341 	if (cmd.flow_attr.num_of_specs)
3342 		kfree(kern_flow_attr);
3343 	return err;
3344 }
3345 
3346 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
3347 {
3348 	struct ib_uverbs_destroy_flow	cmd;
3349 	int				ret;
3350 
3351 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3352 	if (ret)
3353 		return ret;
3354 
3355 	if (cmd.comp_mask)
3356 		return -EINVAL;
3357 
3358 	return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
3359 }
3360 
3361 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
3362 				struct ib_uverbs_create_xsrq *cmd,
3363 				struct ib_udata *udata)
3364 {
3365 	struct ib_uverbs_create_srq_resp resp;
3366 	struct ib_usrq_object           *obj;
3367 	struct ib_pd                    *pd;
3368 	struct ib_srq                   *srq;
3369 	struct ib_uobject               *uninitialized_var(xrcd_uobj);
3370 	struct ib_srq_init_attr          attr;
3371 	int ret;
3372 	struct ib_device *ib_dev;
3373 
3374 	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
3375 						  &ib_dev);
3376 	if (IS_ERR(obj))
3377 		return PTR_ERR(obj);
3378 
3379 	if (cmd->srq_type == IB_SRQT_TM)
3380 		attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3381 
3382 	if (cmd->srq_type == IB_SRQT_XRC) {
3383 		xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
3384 					  attrs);
3385 		if (IS_ERR(xrcd_uobj)) {
3386 			ret = -EINVAL;
3387 			goto err;
3388 		}
3389 
3390 		attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3391 		if (!attr.ext.xrc.xrcd) {
3392 			ret = -EINVAL;
3393 			goto err_put_xrcd;
3394 		}
3395 
3396 		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3397 		atomic_inc(&obj->uxrcd->refcnt);
3398 	}
3399 
3400 	if (ib_srq_has_cq(cmd->srq_type)) {
3401 		attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
3402 						cmd->cq_handle, attrs);
3403 		if (!attr.ext.cq) {
3404 			ret = -EINVAL;
3405 			goto err_put_xrcd;
3406 		}
3407 	}
3408 
3409 	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
3410 	if (!pd) {
3411 		ret = -EINVAL;
3412 		goto err_put_cq;
3413 	}
3414 
3415 	attr.event_handler  = ib_uverbs_srq_event_handler;
3416 	attr.srq_context    = attrs->ufile;
3417 	attr.srq_type       = cmd->srq_type;
3418 	attr.attr.max_wr    = cmd->max_wr;
3419 	attr.attr.max_sge   = cmd->max_sge;
3420 	attr.attr.srq_limit = cmd->srq_limit;
3421 
3422 	obj->uevent.events_reported = 0;
3423 	INIT_LIST_HEAD(&obj->uevent.event_list);
3424 
3425 	srq = rdma_zalloc_drv_obj(ib_dev, ib_srq);
3426 	if (!srq) {
3427 		ret = -ENOMEM;
3428 		goto err_put;
3429 	}
3430 
3431 	srq->device        = pd->device;
3432 	srq->pd            = pd;
3433 	srq->srq_type	   = cmd->srq_type;
3434 	srq->uobject       = &obj->uevent.uobject;
3435 	srq->event_handler = attr.event_handler;
3436 	srq->srq_context   = attr.srq_context;
3437 
3438 	ret = pd->device->ops.create_srq(srq, &attr, udata);
3439 	if (ret)
3440 		goto err_free;
3441 
3442 	if (ib_srq_has_cq(cmd->srq_type)) {
3443 		srq->ext.cq       = attr.ext.cq;
3444 		atomic_inc(&attr.ext.cq->usecnt);
3445 	}
3446 
3447 	if (cmd->srq_type == IB_SRQT_XRC) {
3448 		srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3449 		atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3450 	}
3451 
3452 	atomic_inc(&pd->usecnt);
3453 	atomic_set(&srq->usecnt, 0);
3454 
3455 	obj->uevent.uobject.object = srq;
3456 	obj->uevent.uobject.user_handle = cmd->user_handle;
3457 
3458 	memset(&resp, 0, sizeof resp);
3459 	resp.srq_handle = obj->uevent.uobject.id;
3460 	resp.max_wr     = attr.attr.max_wr;
3461 	resp.max_sge    = attr.attr.max_sge;
3462 	if (cmd->srq_type == IB_SRQT_XRC)
3463 		resp.srqn = srq->ext.xrc.srq_num;
3464 
3465 	ret = uverbs_response(attrs, &resp, sizeof(resp));
3466 	if (ret)
3467 		goto err_copy;
3468 
3469 	if (cmd->srq_type == IB_SRQT_XRC)
3470 		uobj_put_read(xrcd_uobj);
3471 
3472 	if (ib_srq_has_cq(cmd->srq_type))
3473 		uobj_put_obj_read(attr.ext.cq);
3474 
3475 	uobj_put_obj_read(pd);
3476 	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
3477 
3478 err_copy:
3479 	ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs));
3480 
3481 err_free:
3482 	kfree(srq);
3483 err_put:
3484 	uobj_put_obj_read(pd);
3485 
3486 err_put_cq:
3487 	if (ib_srq_has_cq(cmd->srq_type))
3488 		uobj_put_obj_read(attr.ext.cq);
3489 
3490 err_put_xrcd:
3491 	if (cmd->srq_type == IB_SRQT_XRC) {
3492 		atomic_dec(&obj->uxrcd->refcnt);
3493 		uobj_put_read(xrcd_uobj);
3494 	}
3495 
3496 err:
3497 	uobj_alloc_abort(&obj->uevent.uobject, attrs);
3498 	return ret;
3499 }
3500 
3501 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
3502 {
3503 	struct ib_uverbs_create_srq      cmd;
3504 	struct ib_uverbs_create_xsrq     xcmd;
3505 	int ret;
3506 
3507 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3508 	if (ret)
3509 		return ret;
3510 
3511 	memset(&xcmd, 0, sizeof(xcmd));
3512 	xcmd.response	 = cmd.response;
3513 	xcmd.user_handle = cmd.user_handle;
3514 	xcmd.srq_type	 = IB_SRQT_BASIC;
3515 	xcmd.pd_handle	 = cmd.pd_handle;
3516 	xcmd.max_wr	 = cmd.max_wr;
3517 	xcmd.max_sge	 = cmd.max_sge;
3518 	xcmd.srq_limit	 = cmd.srq_limit;
3519 
3520 	return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);
3521 }
3522 
3523 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
3524 {
3525 	struct ib_uverbs_create_xsrq     cmd;
3526 	int ret;
3527 
3528 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3529 	if (ret)
3530 		return ret;
3531 
3532 	return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);
3533 }
3534 
3535 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
3536 {
3537 	struct ib_uverbs_modify_srq cmd;
3538 	struct ib_srq              *srq;
3539 	struct ib_srq_attr          attr;
3540 	int                         ret;
3541 
3542 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3543 	if (ret)
3544 		return ret;
3545 
3546 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3547 	if (!srq)
3548 		return -EINVAL;
3549 
3550 	attr.max_wr    = cmd.max_wr;
3551 	attr.srq_limit = cmd.srq_limit;
3552 
3553 	ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
3554 					  &attrs->driver_udata);
3555 
3556 	uobj_put_obj_read(srq);
3557 
3558 	return ret;
3559 }
3560 
3561 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
3562 {
3563 	struct ib_uverbs_query_srq      cmd;
3564 	struct ib_uverbs_query_srq_resp resp;
3565 	struct ib_srq_attr              attr;
3566 	struct ib_srq                   *srq;
3567 	int                             ret;
3568 
3569 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3570 	if (ret)
3571 		return ret;
3572 
3573 	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3574 	if (!srq)
3575 		return -EINVAL;
3576 
3577 	ret = ib_query_srq(srq, &attr);
3578 
3579 	uobj_put_obj_read(srq);
3580 
3581 	if (ret)
3582 		return ret;
3583 
3584 	memset(&resp, 0, sizeof resp);
3585 
3586 	resp.max_wr    = attr.max_wr;
3587 	resp.max_sge   = attr.max_sge;
3588 	resp.srq_limit = attr.srq_limit;
3589 
3590 	return uverbs_response(attrs, &resp, sizeof(resp));
3591 }
3592 
3593 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
3594 {
3595 	struct ib_uverbs_destroy_srq      cmd;
3596 	struct ib_uverbs_destroy_srq_resp resp;
3597 	struct ib_uobject		 *uobj;
3598 	struct ib_uevent_object        	 *obj;
3599 	int ret;
3600 
3601 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3602 	if (ret)
3603 		return ret;
3604 
3605 	uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3606 	if (IS_ERR(uobj))
3607 		return PTR_ERR(uobj);
3608 
3609 	obj = container_of(uobj, struct ib_uevent_object, uobject);
3610 	memset(&resp, 0, sizeof(resp));
3611 	resp.events_reported = obj->events_reported;
3612 
3613 	uobj_put_destroy(uobj);
3614 
3615 	return uverbs_response(attrs, &resp, sizeof(resp));
3616 }
3617 
3618 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
3619 {
3620 	struct ib_uverbs_ex_query_device_resp resp = {};
3621 	struct ib_uverbs_ex_query_device  cmd;
3622 	struct ib_device_attr attr = {0};
3623 	struct ib_ucontext *ucontext;
3624 	struct ib_device *ib_dev;
3625 	int err;
3626 
3627 	ucontext = ib_uverbs_get_ucontext(attrs);
3628 	if (IS_ERR(ucontext))
3629 		return PTR_ERR(ucontext);
3630 	ib_dev = ucontext->device;
3631 
3632 	err = uverbs_request(attrs, &cmd, sizeof(cmd));
3633 	if (err)
3634 		return err;
3635 
3636 	if (cmd.comp_mask)
3637 		return -EINVAL;
3638 
3639 	if (cmd.reserved)
3640 		return -EINVAL;
3641 
3642 	err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
3643 	if (err)
3644 		return err;
3645 
3646 	copy_query_dev_fields(ucontext, &resp.base, &attr);
3647 
3648 	resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3649 	resp.odp_caps.per_transport_caps.rc_odp_caps =
3650 		attr.odp_caps.per_transport_caps.rc_odp_caps;
3651 	resp.odp_caps.per_transport_caps.uc_odp_caps =
3652 		attr.odp_caps.per_transport_caps.uc_odp_caps;
3653 	resp.odp_caps.per_transport_caps.ud_odp_caps =
3654 		attr.odp_caps.per_transport_caps.ud_odp_caps;
3655 	resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps;
3656 
3657 	resp.timestamp_mask = attr.timestamp_mask;
3658 	resp.hca_core_clock = attr.hca_core_clock;
3659 	resp.device_cap_flags_ex = attr.device_cap_flags;
3660 	resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3661 	resp.rss_caps.max_rwq_indirection_tables =
3662 		attr.rss_caps.max_rwq_indirection_tables;
3663 	resp.rss_caps.max_rwq_indirection_table_size =
3664 		attr.rss_caps.max_rwq_indirection_table_size;
3665 	resp.max_wq_type_rq = attr.max_wq_type_rq;
3666 	resp.raw_packet_caps = attr.raw_packet_caps;
3667 	resp.tm_caps.max_rndv_hdr_size	= attr.tm_caps.max_rndv_hdr_size;
3668 	resp.tm_caps.max_num_tags	= attr.tm_caps.max_num_tags;
3669 	resp.tm_caps.max_ops		= attr.tm_caps.max_ops;
3670 	resp.tm_caps.max_sge		= attr.tm_caps.max_sge;
3671 	resp.tm_caps.flags		= attr.tm_caps.flags;
3672 	resp.cq_moderation_caps.max_cq_moderation_count  =
3673 		attr.cq_caps.max_cq_moderation_count;
3674 	resp.cq_moderation_caps.max_cq_moderation_period =
3675 		attr.cq_caps.max_cq_moderation_period;
3676 	resp.max_dm_size = attr.max_dm_size;
3677 	resp.response_length = uverbs_response_length(attrs, sizeof(resp));
3678 
3679 	return uverbs_response(attrs, &resp, sizeof(resp));
3680 }
3681 
3682 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
3683 {
3684 	struct ib_uverbs_ex_modify_cq cmd;
3685 	struct ib_cq *cq;
3686 	int ret;
3687 
3688 	ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3689 	if (ret)
3690 		return ret;
3691 
3692 	if (!cmd.attr_mask || cmd.reserved)
3693 		return -EINVAL;
3694 
3695 	if (cmd.attr_mask > IB_CQ_MODERATE)
3696 		return -EOPNOTSUPP;
3697 
3698 	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
3699 	if (!cq)
3700 		return -EINVAL;
3701 
3702 	ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
3703 
3704 	uobj_put_obj_read(cq);
3705 
3706 	return ret;
3707 }
3708 
3709 /*
3710  * Describe the input structs for write(). Some write methods have an input
3711  * only struct, most have an input and output. If the struct has an output then
3712  * the 'response' u64 must be the first field in the request structure.
3713  *
3714  * If udata is present then both the request and response structs have a
3715  * trailing driver_data flex array. In this case the size of the base struct
3716  * cannot be changed.
3717  */
3718 #define offsetof_after(_struct, _member)                                       \
3719 	(offsetof(_struct, _member) + sizeof(((_struct *)NULL)->_member))
3720 
3721 #define UAPI_DEF_WRITE_IO(req, resp)                                           \
3722 	.write.has_resp = 1 +                                                  \
3723 			  BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) +    \
3724 			  BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) !=    \
3725 					    sizeof(u64)),                      \
3726 	.write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
3727 
3728 #define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
3729 
3730 #define UAPI_DEF_WRITE_UDATA_IO(req, resp)                                     \
3731 	UAPI_DEF_WRITE_IO(req, resp),                                          \
3732 		.write.has_udata =                                             \
3733 			1 +                                                    \
3734 			BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=        \
3735 					  sizeof(req)) +                       \
3736 			BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) !=       \
3737 					  sizeof(resp))
3738 
3739 #define UAPI_DEF_WRITE_UDATA_I(req)                                            \
3740 	UAPI_DEF_WRITE_I(req),                                                 \
3741 		.write.has_udata =                                             \
3742 			1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) !=    \
3743 					      sizeof(req))
3744 
3745 /*
3746  * The _EX versions are for use with WRITE_EX and allow the last struct member
3747  * to be specified. Buffers that do not include that member will be rejected.
3748  */
3749 #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member)     \
3750 	.write.has_resp = 1,                                                   \
3751 	.write.req_size = offsetof_after(req, req_last_member),                \
3752 	.write.resp_size = offsetof_after(resp, resp_last_member)
3753 
3754 #define UAPI_DEF_WRITE_I_EX(req, req_last_member)                              \
3755 	.write.req_size = offsetof_after(req, req_last_member)
3756 
3757 const struct uapi_definition uverbs_def_write_intf[] = {
3758 	DECLARE_UVERBS_OBJECT(
3759 		UVERBS_OBJECT_AH,
3760 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
3761 				     ib_uverbs_create_ah,
3762 				     UAPI_DEF_WRITE_UDATA_IO(
3763 					     struct ib_uverbs_create_ah,
3764 					     struct ib_uverbs_create_ah_resp),
3765 				     UAPI_DEF_METHOD_NEEDS_FN(create_ah)),
3766 		DECLARE_UVERBS_WRITE(
3767 			IB_USER_VERBS_CMD_DESTROY_AH,
3768 			ib_uverbs_destroy_ah,
3769 			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),
3770 			UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),
3771 
3772 	DECLARE_UVERBS_OBJECT(
3773 		UVERBS_OBJECT_COMP_CHANNEL,
3774 		DECLARE_UVERBS_WRITE(
3775 			IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
3776 			ib_uverbs_create_comp_channel,
3777 			UAPI_DEF_WRITE_IO(
3778 				struct ib_uverbs_create_comp_channel,
3779 				struct ib_uverbs_create_comp_channel_resp))),
3780 
3781 	DECLARE_UVERBS_OBJECT(
3782 		UVERBS_OBJECT_CQ,
3783 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
3784 				     ib_uverbs_create_cq,
3785 				     UAPI_DEF_WRITE_UDATA_IO(
3786 					     struct ib_uverbs_create_cq,
3787 					     struct ib_uverbs_create_cq_resp),
3788 				     UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3789 		DECLARE_UVERBS_WRITE(
3790 			IB_USER_VERBS_CMD_DESTROY_CQ,
3791 			ib_uverbs_destroy_cq,
3792 			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
3793 					  struct ib_uverbs_destroy_cq_resp),
3794 			UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
3795 		DECLARE_UVERBS_WRITE(
3796 			IB_USER_VERBS_CMD_POLL_CQ,
3797 			ib_uverbs_poll_cq,
3798 			UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
3799 					  struct ib_uverbs_poll_cq_resp),
3800 			UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
3801 		DECLARE_UVERBS_WRITE(
3802 			IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
3803 			ib_uverbs_req_notify_cq,
3804 			UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
3805 			UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
3806 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
3807 				     ib_uverbs_resize_cq,
3808 				     UAPI_DEF_WRITE_UDATA_IO(
3809 					     struct ib_uverbs_resize_cq,
3810 					     struct ib_uverbs_resize_cq_resp),
3811 				     UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
3812 		DECLARE_UVERBS_WRITE_EX(
3813 			IB_USER_VERBS_EX_CMD_CREATE_CQ,
3814 			ib_uverbs_ex_create_cq,
3815 			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
3816 					     reserved,
3817 					     struct ib_uverbs_ex_create_cq_resp,
3818 					     response_length),
3819 			UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3820 		DECLARE_UVERBS_WRITE_EX(
3821 			IB_USER_VERBS_EX_CMD_MODIFY_CQ,
3822 			ib_uverbs_ex_modify_cq,
3823 			UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
3824 			UAPI_DEF_METHOD_NEEDS_FN(create_cq))),
3825 
3826 	DECLARE_UVERBS_OBJECT(
3827 		UVERBS_OBJECT_DEVICE,
3828 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
3829 				     ib_uverbs_get_context,
3830 				     UAPI_DEF_WRITE_UDATA_IO(
3831 					     struct ib_uverbs_get_context,
3832 					     struct ib_uverbs_get_context_resp)),
3833 		DECLARE_UVERBS_WRITE(
3834 			IB_USER_VERBS_CMD_QUERY_DEVICE,
3835 			ib_uverbs_query_device,
3836 			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
3837 					  struct ib_uverbs_query_device_resp)),
3838 		DECLARE_UVERBS_WRITE(
3839 			IB_USER_VERBS_CMD_QUERY_PORT,
3840 			ib_uverbs_query_port,
3841 			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
3842 					  struct ib_uverbs_query_port_resp),
3843 			UAPI_DEF_METHOD_NEEDS_FN(query_port)),
3844 		DECLARE_UVERBS_WRITE_EX(
3845 			IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
3846 			ib_uverbs_ex_query_device,
3847 			UAPI_DEF_WRITE_IO_EX(
3848 				struct ib_uverbs_ex_query_device,
3849 				reserved,
3850 				struct ib_uverbs_ex_query_device_resp,
3851 				response_length),
3852 			UAPI_DEF_METHOD_NEEDS_FN(query_device)),
3853 		UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
3854 		UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
3855 
3856 	DECLARE_UVERBS_OBJECT(
3857 		UVERBS_OBJECT_FLOW,
3858 		DECLARE_UVERBS_WRITE_EX(
3859 			IB_USER_VERBS_EX_CMD_CREATE_FLOW,
3860 			ib_uverbs_ex_create_flow,
3861 			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
3862 					     flow_attr,
3863 					     struct ib_uverbs_create_flow_resp,
3864 					     flow_handle),
3865 			UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
3866 		DECLARE_UVERBS_WRITE_EX(
3867 			IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
3868 			ib_uverbs_ex_destroy_flow,
3869 			UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
3870 			UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
3871 
3872 	DECLARE_UVERBS_OBJECT(
3873 		UVERBS_OBJECT_MR,
3874 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
3875 				     ib_uverbs_dereg_mr,
3876 				     UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
3877 				     UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
3878 		DECLARE_UVERBS_WRITE(
3879 			IB_USER_VERBS_CMD_REG_MR,
3880 			ib_uverbs_reg_mr,
3881 			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
3882 						struct ib_uverbs_reg_mr_resp),
3883 			UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
3884 		DECLARE_UVERBS_WRITE(
3885 			IB_USER_VERBS_CMD_REREG_MR,
3886 			ib_uverbs_rereg_mr,
3887 			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
3888 						struct ib_uverbs_rereg_mr_resp),
3889 			UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
3890 
3891 	DECLARE_UVERBS_OBJECT(
3892 		UVERBS_OBJECT_MW,
3893 		DECLARE_UVERBS_WRITE(
3894 			IB_USER_VERBS_CMD_ALLOC_MW,
3895 			ib_uverbs_alloc_mw,
3896 			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
3897 						struct ib_uverbs_alloc_mw_resp),
3898 			UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
3899 		DECLARE_UVERBS_WRITE(
3900 			IB_USER_VERBS_CMD_DEALLOC_MW,
3901 			ib_uverbs_dealloc_mw,
3902 			UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
3903 			UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
3904 
3905 	DECLARE_UVERBS_OBJECT(
3906 		UVERBS_OBJECT_PD,
3907 		DECLARE_UVERBS_WRITE(
3908 			IB_USER_VERBS_CMD_ALLOC_PD,
3909 			ib_uverbs_alloc_pd,
3910 			UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
3911 						struct ib_uverbs_alloc_pd_resp),
3912 			UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
3913 		DECLARE_UVERBS_WRITE(
3914 			IB_USER_VERBS_CMD_DEALLOC_PD,
3915 			ib_uverbs_dealloc_pd,
3916 			UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
3917 			UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
3918 
3919 	DECLARE_UVERBS_OBJECT(
3920 		UVERBS_OBJECT_QP,
3921 		DECLARE_UVERBS_WRITE(
3922 			IB_USER_VERBS_CMD_ATTACH_MCAST,
3923 			ib_uverbs_attach_mcast,
3924 			UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
3925 			UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
3926 			UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3927 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
3928 				     ib_uverbs_create_qp,
3929 				     UAPI_DEF_WRITE_UDATA_IO(
3930 					     struct ib_uverbs_create_qp,
3931 					     struct ib_uverbs_create_qp_resp),
3932 				     UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3933 		DECLARE_UVERBS_WRITE(
3934 			IB_USER_VERBS_CMD_DESTROY_QP,
3935 			ib_uverbs_destroy_qp,
3936 			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
3937 					  struct ib_uverbs_destroy_qp_resp),
3938 			UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
3939 		DECLARE_UVERBS_WRITE(
3940 			IB_USER_VERBS_CMD_DETACH_MCAST,
3941 			ib_uverbs_detach_mcast,
3942 			UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
3943 			UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3944 		DECLARE_UVERBS_WRITE(
3945 			IB_USER_VERBS_CMD_MODIFY_QP,
3946 			ib_uverbs_modify_qp,
3947 			UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
3948 			UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
3949 		DECLARE_UVERBS_WRITE(
3950 			IB_USER_VERBS_CMD_POST_RECV,
3951 			ib_uverbs_post_recv,
3952 			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
3953 					  struct ib_uverbs_post_recv_resp),
3954 			UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
3955 		DECLARE_UVERBS_WRITE(
3956 			IB_USER_VERBS_CMD_POST_SEND,
3957 			ib_uverbs_post_send,
3958 			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
3959 					  struct ib_uverbs_post_send_resp),
3960 			UAPI_DEF_METHOD_NEEDS_FN(post_send)),
3961 		DECLARE_UVERBS_WRITE(
3962 			IB_USER_VERBS_CMD_QUERY_QP,
3963 			ib_uverbs_query_qp,
3964 			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
3965 					  struct ib_uverbs_query_qp_resp),
3966 			UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
3967 		DECLARE_UVERBS_WRITE_EX(
3968 			IB_USER_VERBS_EX_CMD_CREATE_QP,
3969 			ib_uverbs_ex_create_qp,
3970 			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
3971 					     comp_mask,
3972 					     struct ib_uverbs_ex_create_qp_resp,
3973 					     response_length),
3974 			UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3975 		DECLARE_UVERBS_WRITE_EX(
3976 			IB_USER_VERBS_EX_CMD_MODIFY_QP,
3977 			ib_uverbs_ex_modify_qp,
3978 			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
3979 					     base,
3980 					     struct ib_uverbs_ex_modify_qp_resp,
3981 					     response_length),
3982 			UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
3983 
3984 	DECLARE_UVERBS_OBJECT(
3985 		UVERBS_OBJECT_RWQ_IND_TBL,
3986 		DECLARE_UVERBS_WRITE_EX(
3987 			IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
3988 			ib_uverbs_ex_create_rwq_ind_table,
3989 			UAPI_DEF_WRITE_IO_EX(
3990 				struct ib_uverbs_ex_create_rwq_ind_table,
3991 				log_ind_tbl_size,
3992 				struct ib_uverbs_ex_create_rwq_ind_table_resp,
3993 				ind_tbl_num),
3994 			UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
3995 		DECLARE_UVERBS_WRITE_EX(
3996 			IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
3997 			ib_uverbs_ex_destroy_rwq_ind_table,
3998 			UAPI_DEF_WRITE_I(
3999 				struct ib_uverbs_ex_destroy_rwq_ind_table),
4000 			UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
4001 
4002 	DECLARE_UVERBS_OBJECT(
4003 		UVERBS_OBJECT_WQ,
4004 		DECLARE_UVERBS_WRITE_EX(
4005 			IB_USER_VERBS_EX_CMD_CREATE_WQ,
4006 			ib_uverbs_ex_create_wq,
4007 			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
4008 					     max_sge,
4009 					     struct ib_uverbs_ex_create_wq_resp,
4010 					     wqn),
4011 			UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
4012 		DECLARE_UVERBS_WRITE_EX(
4013 			IB_USER_VERBS_EX_CMD_DESTROY_WQ,
4014 			ib_uverbs_ex_destroy_wq,
4015 			UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
4016 					     wq_handle,
4017 					     struct ib_uverbs_ex_destroy_wq_resp,
4018 					     reserved),
4019 			UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
4020 		DECLARE_UVERBS_WRITE_EX(
4021 			IB_USER_VERBS_EX_CMD_MODIFY_WQ,
4022 			ib_uverbs_ex_modify_wq,
4023 			UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
4024 					    curr_wq_state),
4025 			UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
4026 
4027 	DECLARE_UVERBS_OBJECT(
4028 		UVERBS_OBJECT_SRQ,
4029 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
4030 				     ib_uverbs_create_srq,
4031 				     UAPI_DEF_WRITE_UDATA_IO(
4032 					     struct ib_uverbs_create_srq,
4033 					     struct ib_uverbs_create_srq_resp),
4034 				     UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4035 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
4036 				     ib_uverbs_create_xsrq,
4037 				     UAPI_DEF_WRITE_UDATA_IO(
4038 					     struct ib_uverbs_create_xsrq,
4039 					     struct ib_uverbs_create_srq_resp),
4040 				     UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4041 		DECLARE_UVERBS_WRITE(
4042 			IB_USER_VERBS_CMD_DESTROY_SRQ,
4043 			ib_uverbs_destroy_srq,
4044 			UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
4045 					  struct ib_uverbs_destroy_srq_resp),
4046 			UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
4047 		DECLARE_UVERBS_WRITE(
4048 			IB_USER_VERBS_CMD_MODIFY_SRQ,
4049 			ib_uverbs_modify_srq,
4050 			UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
4051 			UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
4052 		DECLARE_UVERBS_WRITE(
4053 			IB_USER_VERBS_CMD_POST_SRQ_RECV,
4054 			ib_uverbs_post_srq_recv,
4055 			UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
4056 					  struct ib_uverbs_post_srq_recv_resp),
4057 			UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
4058 		DECLARE_UVERBS_WRITE(
4059 			IB_USER_VERBS_CMD_QUERY_SRQ,
4060 			ib_uverbs_query_srq,
4061 			UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
4062 					  struct ib_uverbs_query_srq_resp),
4063 			UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
4064 
4065 	DECLARE_UVERBS_OBJECT(
4066 		UVERBS_OBJECT_XRCD,
4067 		DECLARE_UVERBS_WRITE(
4068 			IB_USER_VERBS_CMD_CLOSE_XRCD,
4069 			ib_uverbs_close_xrcd,
4070 			UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),
4071 			UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),
4072 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
4073 				     ib_uverbs_open_qp,
4074 				     UAPI_DEF_WRITE_UDATA_IO(
4075 					     struct ib_uverbs_open_qp,
4076 					     struct ib_uverbs_create_qp_resp)),
4077 		DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
4078 				     ib_uverbs_open_xrcd,
4079 				     UAPI_DEF_WRITE_UDATA_IO(
4080 					     struct ib_uverbs_open_xrcd,
4081 					     struct ib_uverbs_open_xrcd_resp),
4082 				     UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),
4083 
4084 	{},
4085 };
4086