1 /*
2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
6  * Copyright (c) 2005 PathScale, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  */
36 
37 #include <linux/module.h>
38 #include <linux/init.h>
39 #include <linux/device.h>
40 #include <linux/err.h>
41 #include <linux/fs.h>
42 #include <linux/poll.h>
43 #include <linux/sched.h>
44 #include <linux/file.h>
45 #include <linux/cdev.h>
46 #include <linux/anon_inodes.h>
47 #include <linux/slab.h>
48 
49 #include <asm/uaccess.h>
50 
51 #include "uverbs.h"
52 
53 MODULE_AUTHOR("Roland Dreier");
54 MODULE_DESCRIPTION("InfiniBand userspace verbs access");
55 MODULE_LICENSE("Dual BSD/GPL");
56 
57 enum {
58 	IB_UVERBS_MAJOR       = 231,
59 	IB_UVERBS_BASE_MINOR  = 192,
60 	IB_UVERBS_MAX_DEVICES = 32
61 };
62 
63 #define IB_UVERBS_BASE_DEV	MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)
64 
65 static struct class *uverbs_class;
66 
67 DEFINE_SPINLOCK(ib_uverbs_idr_lock);
68 DEFINE_IDR(ib_uverbs_pd_idr);
69 DEFINE_IDR(ib_uverbs_mr_idr);
70 DEFINE_IDR(ib_uverbs_mw_idr);
71 DEFINE_IDR(ib_uverbs_ah_idr);
72 DEFINE_IDR(ib_uverbs_cq_idr);
73 DEFINE_IDR(ib_uverbs_qp_idr);
74 DEFINE_IDR(ib_uverbs_srq_idr);
75 
76 static DEFINE_SPINLOCK(map_lock);
77 static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES);
78 
79 static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file,
80 				     const char __user *buf, int in_len,
81 				     int out_len) = {
82 	[IB_USER_VERBS_CMD_GET_CONTEXT]		= ib_uverbs_get_context,
83 	[IB_USER_VERBS_CMD_QUERY_DEVICE]	= ib_uverbs_query_device,
84 	[IB_USER_VERBS_CMD_QUERY_PORT]		= ib_uverbs_query_port,
85 	[IB_USER_VERBS_CMD_ALLOC_PD]		= ib_uverbs_alloc_pd,
86 	[IB_USER_VERBS_CMD_DEALLOC_PD]		= ib_uverbs_dealloc_pd,
87 	[IB_USER_VERBS_CMD_REG_MR]		= ib_uverbs_reg_mr,
88 	[IB_USER_VERBS_CMD_DEREG_MR]		= ib_uverbs_dereg_mr,
89 	[IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL] = ib_uverbs_create_comp_channel,
90 	[IB_USER_VERBS_CMD_CREATE_CQ]		= ib_uverbs_create_cq,
91 	[IB_USER_VERBS_CMD_RESIZE_CQ]		= ib_uverbs_resize_cq,
92 	[IB_USER_VERBS_CMD_POLL_CQ]		= ib_uverbs_poll_cq,
93 	[IB_USER_VERBS_CMD_REQ_NOTIFY_CQ]	= ib_uverbs_req_notify_cq,
94 	[IB_USER_VERBS_CMD_DESTROY_CQ]		= ib_uverbs_destroy_cq,
95 	[IB_USER_VERBS_CMD_CREATE_QP]		= ib_uverbs_create_qp,
96 	[IB_USER_VERBS_CMD_QUERY_QP]		= ib_uverbs_query_qp,
97 	[IB_USER_VERBS_CMD_MODIFY_QP]		= ib_uverbs_modify_qp,
98 	[IB_USER_VERBS_CMD_DESTROY_QP]		= ib_uverbs_destroy_qp,
99 	[IB_USER_VERBS_CMD_POST_SEND]		= ib_uverbs_post_send,
100 	[IB_USER_VERBS_CMD_POST_RECV]		= ib_uverbs_post_recv,
101 	[IB_USER_VERBS_CMD_POST_SRQ_RECV]	= ib_uverbs_post_srq_recv,
102 	[IB_USER_VERBS_CMD_CREATE_AH]		= ib_uverbs_create_ah,
103 	[IB_USER_VERBS_CMD_DESTROY_AH]		= ib_uverbs_destroy_ah,
104 	[IB_USER_VERBS_CMD_ATTACH_MCAST]	= ib_uverbs_attach_mcast,
105 	[IB_USER_VERBS_CMD_DETACH_MCAST]	= ib_uverbs_detach_mcast,
106 	[IB_USER_VERBS_CMD_CREATE_SRQ]		= ib_uverbs_create_srq,
107 	[IB_USER_VERBS_CMD_MODIFY_SRQ]		= ib_uverbs_modify_srq,
108 	[IB_USER_VERBS_CMD_QUERY_SRQ]		= ib_uverbs_query_srq,
109 	[IB_USER_VERBS_CMD_DESTROY_SRQ]		= ib_uverbs_destroy_srq,
110 };
111 
112 static void ib_uverbs_add_one(struct ib_device *device);
113 static void ib_uverbs_remove_one(struct ib_device *device);
114 
115 static void ib_uverbs_release_dev(struct kref *ref)
116 {
117 	struct ib_uverbs_device *dev =
118 		container_of(ref, struct ib_uverbs_device, ref);
119 
120 	complete(&dev->comp);
121 }
122 
123 static void ib_uverbs_release_event_file(struct kref *ref)
124 {
125 	struct ib_uverbs_event_file *file =
126 		container_of(ref, struct ib_uverbs_event_file, ref);
127 
128 	kfree(file);
129 }
130 
131 void ib_uverbs_release_ucq(struct ib_uverbs_file *file,
132 			  struct ib_uverbs_event_file *ev_file,
133 			  struct ib_ucq_object *uobj)
134 {
135 	struct ib_uverbs_event *evt, *tmp;
136 
137 	if (ev_file) {
138 		spin_lock_irq(&ev_file->lock);
139 		list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) {
140 			list_del(&evt->list);
141 			kfree(evt);
142 		}
143 		spin_unlock_irq(&ev_file->lock);
144 
145 		kref_put(&ev_file->ref, ib_uverbs_release_event_file);
146 	}
147 
148 	spin_lock_irq(&file->async_file->lock);
149 	list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) {
150 		list_del(&evt->list);
151 		kfree(evt);
152 	}
153 	spin_unlock_irq(&file->async_file->lock);
154 }
155 
156 void ib_uverbs_release_uevent(struct ib_uverbs_file *file,
157 			      struct ib_uevent_object *uobj)
158 {
159 	struct ib_uverbs_event *evt, *tmp;
160 
161 	spin_lock_irq(&file->async_file->lock);
162 	list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
163 		list_del(&evt->list);
164 		kfree(evt);
165 	}
166 	spin_unlock_irq(&file->async_file->lock);
167 }
168 
169 static void ib_uverbs_detach_umcast(struct ib_qp *qp,
170 				    struct ib_uqp_object *uobj)
171 {
172 	struct ib_uverbs_mcast_entry *mcast, *tmp;
173 
174 	list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) {
175 		ib_detach_mcast(qp, &mcast->gid, mcast->lid);
176 		list_del(&mcast->list);
177 		kfree(mcast);
178 	}
179 }
180 
181 static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file,
182 				      struct ib_ucontext *context)
183 {
184 	struct ib_uobject *uobj, *tmp;
185 
186 	if (!context)
187 		return 0;
188 
189 	context->closing = 1;
190 
191 	list_for_each_entry_safe(uobj, tmp, &context->ah_list, list) {
192 		struct ib_ah *ah = uobj->object;
193 
194 		idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
195 		ib_destroy_ah(ah);
196 		kfree(uobj);
197 	}
198 
199 	list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) {
200 		struct ib_qp *qp = uobj->object;
201 		struct ib_uqp_object *uqp =
202 			container_of(uobj, struct ib_uqp_object, uevent.uobject);
203 
204 		idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
205 		ib_uverbs_detach_umcast(qp, uqp);
206 		ib_destroy_qp(qp);
207 		ib_uverbs_release_uevent(file, &uqp->uevent);
208 		kfree(uqp);
209 	}
210 
211 	list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) {
212 		struct ib_cq *cq = uobj->object;
213 		struct ib_uverbs_event_file *ev_file = cq->cq_context;
214 		struct ib_ucq_object *ucq =
215 			container_of(uobj, struct ib_ucq_object, uobject);
216 
217 		idr_remove_uobj(&ib_uverbs_cq_idr, uobj);
218 		ib_destroy_cq(cq);
219 		ib_uverbs_release_ucq(file, ev_file, ucq);
220 		kfree(ucq);
221 	}
222 
223 	list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) {
224 		struct ib_srq *srq = uobj->object;
225 		struct ib_uevent_object *uevent =
226 			container_of(uobj, struct ib_uevent_object, uobject);
227 
228 		idr_remove_uobj(&ib_uverbs_srq_idr, uobj);
229 		ib_destroy_srq(srq);
230 		ib_uverbs_release_uevent(file, uevent);
231 		kfree(uevent);
232 	}
233 
234 	/* XXX Free MWs */
235 
236 	list_for_each_entry_safe(uobj, tmp, &context->mr_list, list) {
237 		struct ib_mr *mr = uobj->object;
238 
239 		idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
240 		ib_dereg_mr(mr);
241 		kfree(uobj);
242 	}
243 
244 	list_for_each_entry_safe(uobj, tmp, &context->pd_list, list) {
245 		struct ib_pd *pd = uobj->object;
246 
247 		idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
248 		ib_dealloc_pd(pd);
249 		kfree(uobj);
250 	}
251 
252 	return context->device->dealloc_ucontext(context);
253 }
254 
255 static void ib_uverbs_release_file(struct kref *ref)
256 {
257 	struct ib_uverbs_file *file =
258 		container_of(ref, struct ib_uverbs_file, ref);
259 
260 	module_put(file->device->ib_dev->owner);
261 	kref_put(&file->device->ref, ib_uverbs_release_dev);
262 
263 	kfree(file);
264 }
265 
266 static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf,
267 				    size_t count, loff_t *pos)
268 {
269 	struct ib_uverbs_event_file *file = filp->private_data;
270 	struct ib_uverbs_event *event;
271 	int eventsz;
272 	int ret = 0;
273 
274 	spin_lock_irq(&file->lock);
275 
276 	while (list_empty(&file->event_list)) {
277 		spin_unlock_irq(&file->lock);
278 
279 		if (filp->f_flags & O_NONBLOCK)
280 			return -EAGAIN;
281 
282 		if (wait_event_interruptible(file->poll_wait,
283 					     !list_empty(&file->event_list)))
284 			return -ERESTARTSYS;
285 
286 		spin_lock_irq(&file->lock);
287 	}
288 
289 	event = list_entry(file->event_list.next, struct ib_uverbs_event, list);
290 
291 	if (file->is_async)
292 		eventsz = sizeof (struct ib_uverbs_async_event_desc);
293 	else
294 		eventsz = sizeof (struct ib_uverbs_comp_event_desc);
295 
296 	if (eventsz > count) {
297 		ret   = -EINVAL;
298 		event = NULL;
299 	} else {
300 		list_del(file->event_list.next);
301 		if (event->counter) {
302 			++(*event->counter);
303 			list_del(&event->obj_list);
304 		}
305 	}
306 
307 	spin_unlock_irq(&file->lock);
308 
309 	if (event) {
310 		if (copy_to_user(buf, event, eventsz))
311 			ret = -EFAULT;
312 		else
313 			ret = eventsz;
314 	}
315 
316 	kfree(event);
317 
318 	return ret;
319 }
320 
321 static unsigned int ib_uverbs_event_poll(struct file *filp,
322 					 struct poll_table_struct *wait)
323 {
324 	unsigned int pollflags = 0;
325 	struct ib_uverbs_event_file *file = filp->private_data;
326 
327 	poll_wait(filp, &file->poll_wait, wait);
328 
329 	spin_lock_irq(&file->lock);
330 	if (!list_empty(&file->event_list))
331 		pollflags = POLLIN | POLLRDNORM;
332 	spin_unlock_irq(&file->lock);
333 
334 	return pollflags;
335 }
336 
337 static int ib_uverbs_event_fasync(int fd, struct file *filp, int on)
338 {
339 	struct ib_uverbs_event_file *file = filp->private_data;
340 
341 	return fasync_helper(fd, filp, on, &file->async_queue);
342 }
343 
344 static int ib_uverbs_event_close(struct inode *inode, struct file *filp)
345 {
346 	struct ib_uverbs_event_file *file = filp->private_data;
347 	struct ib_uverbs_event *entry, *tmp;
348 
349 	spin_lock_irq(&file->lock);
350 	file->is_closed = 1;
351 	list_for_each_entry_safe(entry, tmp, &file->event_list, list) {
352 		if (entry->counter)
353 			list_del(&entry->obj_list);
354 		kfree(entry);
355 	}
356 	spin_unlock_irq(&file->lock);
357 
358 	if (file->is_async) {
359 		ib_unregister_event_handler(&file->uverbs_file->event_handler);
360 		kref_put(&file->uverbs_file->ref, ib_uverbs_release_file);
361 	}
362 	kref_put(&file->ref, ib_uverbs_release_event_file);
363 
364 	return 0;
365 }
366 
367 static const struct file_operations uverbs_event_fops = {
368 	.owner	 = THIS_MODULE,
369 	.read	 = ib_uverbs_event_read,
370 	.poll    = ib_uverbs_event_poll,
371 	.release = ib_uverbs_event_close,
372 	.fasync  = ib_uverbs_event_fasync,
373 	.llseek	 = no_llseek,
374 };
375 
376 void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)
377 {
378 	struct ib_uverbs_event_file    *file = cq_context;
379 	struct ib_ucq_object	       *uobj;
380 	struct ib_uverbs_event	       *entry;
381 	unsigned long			flags;
382 
383 	if (!file)
384 		return;
385 
386 	spin_lock_irqsave(&file->lock, flags);
387 	if (file->is_closed) {
388 		spin_unlock_irqrestore(&file->lock, flags);
389 		return;
390 	}
391 
392 	entry = kmalloc(sizeof *entry, GFP_ATOMIC);
393 	if (!entry) {
394 		spin_unlock_irqrestore(&file->lock, flags);
395 		return;
396 	}
397 
398 	uobj = container_of(cq->uobject, struct ib_ucq_object, uobject);
399 
400 	entry->desc.comp.cq_handle = cq->uobject->user_handle;
401 	entry->counter		   = &uobj->comp_events_reported;
402 
403 	list_add_tail(&entry->list, &file->event_list);
404 	list_add_tail(&entry->obj_list, &uobj->comp_list);
405 	spin_unlock_irqrestore(&file->lock, flags);
406 
407 	wake_up_interruptible(&file->poll_wait);
408 	kill_fasync(&file->async_queue, SIGIO, POLL_IN);
409 }
410 
411 static void ib_uverbs_async_handler(struct ib_uverbs_file *file,
412 				    __u64 element, __u64 event,
413 				    struct list_head *obj_list,
414 				    u32 *counter)
415 {
416 	struct ib_uverbs_event *entry;
417 	unsigned long flags;
418 
419 	spin_lock_irqsave(&file->async_file->lock, flags);
420 	if (file->async_file->is_closed) {
421 		spin_unlock_irqrestore(&file->async_file->lock, flags);
422 		return;
423 	}
424 
425 	entry = kmalloc(sizeof *entry, GFP_ATOMIC);
426 	if (!entry) {
427 		spin_unlock_irqrestore(&file->async_file->lock, flags);
428 		return;
429 	}
430 
431 	entry->desc.async.element    = element;
432 	entry->desc.async.event_type = event;
433 	entry->counter               = counter;
434 
435 	list_add_tail(&entry->list, &file->async_file->event_list);
436 	if (obj_list)
437 		list_add_tail(&entry->obj_list, obj_list);
438 	spin_unlock_irqrestore(&file->async_file->lock, flags);
439 
440 	wake_up_interruptible(&file->async_file->poll_wait);
441 	kill_fasync(&file->async_file->async_queue, SIGIO, POLL_IN);
442 }
443 
444 void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)
445 {
446 	struct ib_ucq_object *uobj = container_of(event->element.cq->uobject,
447 						  struct ib_ucq_object, uobject);
448 
449 	ib_uverbs_async_handler(uobj->uverbs_file, uobj->uobject.user_handle,
450 				event->event, &uobj->async_list,
451 				&uobj->async_events_reported);
452 }
453 
454 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
455 {
456 	struct ib_uevent_object *uobj;
457 
458 	uobj = container_of(event->element.qp->uobject,
459 			    struct ib_uevent_object, uobject);
460 
461 	ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
462 				event->event, &uobj->event_list,
463 				&uobj->events_reported);
464 }
465 
466 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
467 {
468 	struct ib_uevent_object *uobj;
469 
470 	uobj = container_of(event->element.srq->uobject,
471 			    struct ib_uevent_object, uobject);
472 
473 	ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
474 				event->event, &uobj->event_list,
475 				&uobj->events_reported);
476 }
477 
478 void ib_uverbs_event_handler(struct ib_event_handler *handler,
479 			     struct ib_event *event)
480 {
481 	struct ib_uverbs_file *file =
482 		container_of(handler, struct ib_uverbs_file, event_handler);
483 
484 	ib_uverbs_async_handler(file, event->element.port_num, event->event,
485 				NULL, NULL);
486 }
487 
488 struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file,
489 					int is_async)
490 {
491 	struct ib_uverbs_event_file *ev_file;
492 	struct file *filp;
493 
494 	ev_file = kmalloc(sizeof *ev_file, GFP_KERNEL);
495 	if (!ev_file)
496 		return ERR_PTR(-ENOMEM);
497 
498 	kref_init(&ev_file->ref);
499 	spin_lock_init(&ev_file->lock);
500 	INIT_LIST_HEAD(&ev_file->event_list);
501 	init_waitqueue_head(&ev_file->poll_wait);
502 	ev_file->uverbs_file = uverbs_file;
503 	ev_file->async_queue = NULL;
504 	ev_file->is_async    = is_async;
505 	ev_file->is_closed   = 0;
506 
507 	filp = anon_inode_getfile("[infinibandevent]", &uverbs_event_fops,
508 				  ev_file, O_RDONLY);
509 	if (IS_ERR(filp))
510 		kfree(ev_file);
511 
512 	return filp;
513 }
514 
515 /*
516  * Look up a completion event file by FD.  If lookup is successful,
517  * takes a ref to the event file struct that it returns; if
518  * unsuccessful, returns NULL.
519  */
520 struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd)
521 {
522 	struct ib_uverbs_event_file *ev_file = NULL;
523 	struct file *filp;
524 
525 	filp = fget(fd);
526 	if (!filp)
527 		return NULL;
528 
529 	if (filp->f_op != &uverbs_event_fops)
530 		goto out;
531 
532 	ev_file = filp->private_data;
533 	if (ev_file->is_async) {
534 		ev_file = NULL;
535 		goto out;
536 	}
537 
538 	kref_get(&ev_file->ref);
539 
540 out:
541 	fput(filp);
542 	return ev_file;
543 }
544 
545 static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
546 			     size_t count, loff_t *pos)
547 {
548 	struct ib_uverbs_file *file = filp->private_data;
549 	struct ib_uverbs_cmd_hdr hdr;
550 
551 	if (count < sizeof hdr)
552 		return -EINVAL;
553 
554 	if (copy_from_user(&hdr, buf, sizeof hdr))
555 		return -EFAULT;
556 
557 	if (hdr.in_words * 4 != count)
558 		return -EINVAL;
559 
560 	if (hdr.command < 0				||
561 	    hdr.command >= ARRAY_SIZE(uverbs_cmd_table) ||
562 	    !uverbs_cmd_table[hdr.command])
563 		return -EINVAL;
564 
565 	if (!file->ucontext &&
566 	    hdr.command != IB_USER_VERBS_CMD_GET_CONTEXT)
567 		return -EINVAL;
568 
569 	if (!(file->device->ib_dev->uverbs_cmd_mask & (1ull << hdr.command)))
570 		return -ENOSYS;
571 
572 	return uverbs_cmd_table[hdr.command](file, buf + sizeof hdr,
573 					     hdr.in_words * 4, hdr.out_words * 4);
574 }
575 
576 static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
577 {
578 	struct ib_uverbs_file *file = filp->private_data;
579 
580 	if (!file->ucontext)
581 		return -ENODEV;
582 	else
583 		return file->device->ib_dev->mmap(file->ucontext, vma);
584 }
585 
586 /*
587  * ib_uverbs_open() does not need the BKL:
588  *
589  *  - the ib_uverbs_device structures are properly reference counted and
590  *    everything else is purely local to the file being created, so
591  *    races against other open calls are not a problem;
592  *  - there is no ioctl method to race against;
593  *  - the open method will either immediately run -ENXIO, or all
594  *    required initialization will be done.
595  */
596 static int ib_uverbs_open(struct inode *inode, struct file *filp)
597 {
598 	struct ib_uverbs_device *dev;
599 	struct ib_uverbs_file *file;
600 	int ret;
601 
602 	dev = container_of(inode->i_cdev, struct ib_uverbs_device, cdev);
603 	if (dev)
604 		kref_get(&dev->ref);
605 	else
606 		return -ENXIO;
607 
608 	if (!try_module_get(dev->ib_dev->owner)) {
609 		ret = -ENODEV;
610 		goto err;
611 	}
612 
613 	file = kmalloc(sizeof *file, GFP_KERNEL);
614 	if (!file) {
615 		ret = -ENOMEM;
616 		goto err_module;
617 	}
618 
619 	file->device	 = dev;
620 	file->ucontext	 = NULL;
621 	file->async_file = NULL;
622 	kref_init(&file->ref);
623 	mutex_init(&file->mutex);
624 
625 	filp->private_data = file;
626 
627 	return nonseekable_open(inode, filp);
628 
629 err_module:
630 	module_put(dev->ib_dev->owner);
631 
632 err:
633 	kref_put(&dev->ref, ib_uverbs_release_dev);
634 	return ret;
635 }
636 
637 static int ib_uverbs_close(struct inode *inode, struct file *filp)
638 {
639 	struct ib_uverbs_file *file = filp->private_data;
640 
641 	ib_uverbs_cleanup_ucontext(file, file->ucontext);
642 
643 	if (file->async_file)
644 		kref_put(&file->async_file->ref, ib_uverbs_release_event_file);
645 
646 	kref_put(&file->ref, ib_uverbs_release_file);
647 
648 	return 0;
649 }
650 
651 static const struct file_operations uverbs_fops = {
652 	.owner	 = THIS_MODULE,
653 	.write	 = ib_uverbs_write,
654 	.open	 = ib_uverbs_open,
655 	.release = ib_uverbs_close,
656 	.llseek	 = no_llseek,
657 };
658 
659 static const struct file_operations uverbs_mmap_fops = {
660 	.owner	 = THIS_MODULE,
661 	.write	 = ib_uverbs_write,
662 	.mmap    = ib_uverbs_mmap,
663 	.open	 = ib_uverbs_open,
664 	.release = ib_uverbs_close,
665 	.llseek	 = no_llseek,
666 };
667 
668 static struct ib_client uverbs_client = {
669 	.name   = "uverbs",
670 	.add    = ib_uverbs_add_one,
671 	.remove = ib_uverbs_remove_one
672 };
673 
674 static ssize_t show_ibdev(struct device *device, struct device_attribute *attr,
675 			  char *buf)
676 {
677 	struct ib_uverbs_device *dev = dev_get_drvdata(device);
678 
679 	if (!dev)
680 		return -ENODEV;
681 
682 	return sprintf(buf, "%s\n", dev->ib_dev->name);
683 }
684 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
685 
686 static ssize_t show_dev_abi_version(struct device *device,
687 				    struct device_attribute *attr, char *buf)
688 {
689 	struct ib_uverbs_device *dev = dev_get_drvdata(device);
690 
691 	if (!dev)
692 		return -ENODEV;
693 
694 	return sprintf(buf, "%d\n", dev->ib_dev->uverbs_abi_ver);
695 }
696 static DEVICE_ATTR(abi_version, S_IRUGO, show_dev_abi_version, NULL);
697 
698 static CLASS_ATTR_STRING(abi_version, S_IRUGO,
699 			 __stringify(IB_USER_VERBS_ABI_VERSION));
700 
701 static dev_t overflow_maj;
702 static DECLARE_BITMAP(overflow_map, IB_UVERBS_MAX_DEVICES);
703 
704 /*
705  * If we have more than IB_UVERBS_MAX_DEVICES, dynamically overflow by
706  * requesting a new major number and doubling the number of max devices we
707  * support. It's stupid, but simple.
708  */
709 static int find_overflow_devnum(void)
710 {
711 	int ret;
712 
713 	if (!overflow_maj) {
714 		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES,
715 					  "infiniband_verbs");
716 		if (ret) {
717 			printk(KERN_ERR "user_verbs: couldn't register dynamic device number\n");
718 			return ret;
719 		}
720 	}
721 
722 	ret = find_first_zero_bit(overflow_map, IB_UVERBS_MAX_DEVICES);
723 	if (ret >= IB_UVERBS_MAX_DEVICES)
724 		return -1;
725 
726 	return ret;
727 }
728 
729 static void ib_uverbs_add_one(struct ib_device *device)
730 {
731 	int devnum;
732 	dev_t base;
733 	struct ib_uverbs_device *uverbs_dev;
734 
735 	if (!device->alloc_ucontext)
736 		return;
737 
738 	uverbs_dev = kzalloc(sizeof *uverbs_dev, GFP_KERNEL);
739 	if (!uverbs_dev)
740 		return;
741 
742 	kref_init(&uverbs_dev->ref);
743 	init_completion(&uverbs_dev->comp);
744 
745 	spin_lock(&map_lock);
746 	devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES);
747 	if (devnum >= IB_UVERBS_MAX_DEVICES) {
748 		spin_unlock(&map_lock);
749 		devnum = find_overflow_devnum();
750 		if (devnum < 0)
751 			goto err;
752 
753 		spin_lock(&map_lock);
754 		uverbs_dev->devnum = devnum + IB_UVERBS_MAX_DEVICES;
755 		base = devnum + overflow_maj;
756 		set_bit(devnum, overflow_map);
757 	} else {
758 		uverbs_dev->devnum = devnum;
759 		base = devnum + IB_UVERBS_BASE_DEV;
760 		set_bit(devnum, dev_map);
761 	}
762 	spin_unlock(&map_lock);
763 
764 	uverbs_dev->ib_dev           = device;
765 	uverbs_dev->num_comp_vectors = device->num_comp_vectors;
766 
767 	cdev_init(&uverbs_dev->cdev, NULL);
768 	uverbs_dev->cdev.owner = THIS_MODULE;
769 	uverbs_dev->cdev.ops = device->mmap ? &uverbs_mmap_fops : &uverbs_fops;
770 	kobject_set_name(&uverbs_dev->cdev.kobj, "uverbs%d", uverbs_dev->devnum);
771 	if (cdev_add(&uverbs_dev->cdev, base, 1))
772 		goto err_cdev;
773 
774 	uverbs_dev->dev = device_create(uverbs_class, device->dma_device,
775 					uverbs_dev->cdev.dev, uverbs_dev,
776 					"uverbs%d", uverbs_dev->devnum);
777 	if (IS_ERR(uverbs_dev->dev))
778 		goto err_cdev;
779 
780 	if (device_create_file(uverbs_dev->dev, &dev_attr_ibdev))
781 		goto err_class;
782 	if (device_create_file(uverbs_dev->dev, &dev_attr_abi_version))
783 		goto err_class;
784 
785 	ib_set_client_data(device, &uverbs_client, uverbs_dev);
786 
787 	return;
788 
789 err_class:
790 	device_destroy(uverbs_class, uverbs_dev->cdev.dev);
791 
792 err_cdev:
793 	cdev_del(&uverbs_dev->cdev);
794 	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
795 		clear_bit(devnum, dev_map);
796 	else
797 		clear_bit(devnum, overflow_map);
798 
799 err:
800 	kref_put(&uverbs_dev->ref, ib_uverbs_release_dev);
801 	wait_for_completion(&uverbs_dev->comp);
802 	kfree(uverbs_dev);
803 	return;
804 }
805 
806 static void ib_uverbs_remove_one(struct ib_device *device)
807 {
808 	struct ib_uverbs_device *uverbs_dev = ib_get_client_data(device, &uverbs_client);
809 
810 	if (!uverbs_dev)
811 		return;
812 
813 	dev_set_drvdata(uverbs_dev->dev, NULL);
814 	device_destroy(uverbs_class, uverbs_dev->cdev.dev);
815 	cdev_del(&uverbs_dev->cdev);
816 
817 	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
818 		clear_bit(uverbs_dev->devnum, dev_map);
819 	else
820 		clear_bit(uverbs_dev->devnum - IB_UVERBS_MAX_DEVICES, overflow_map);
821 
822 	kref_put(&uverbs_dev->ref, ib_uverbs_release_dev);
823 	wait_for_completion(&uverbs_dev->comp);
824 	kfree(uverbs_dev);
825 }
826 
827 static int __init ib_uverbs_init(void)
828 {
829 	int ret;
830 
831 	ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES,
832 				     "infiniband_verbs");
833 	if (ret) {
834 		printk(KERN_ERR "user_verbs: couldn't register device number\n");
835 		goto out;
836 	}
837 
838 	uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
839 	if (IS_ERR(uverbs_class)) {
840 		ret = PTR_ERR(uverbs_class);
841 		printk(KERN_ERR "user_verbs: couldn't create class infiniband_verbs\n");
842 		goto out_chrdev;
843 	}
844 
845 	ret = class_create_file(uverbs_class, &class_attr_abi_version.attr);
846 	if (ret) {
847 		printk(KERN_ERR "user_verbs: couldn't create abi_version attribute\n");
848 		goto out_class;
849 	}
850 
851 	ret = ib_register_client(&uverbs_client);
852 	if (ret) {
853 		printk(KERN_ERR "user_verbs: couldn't register client\n");
854 		goto out_class;
855 	}
856 
857 	return 0;
858 
859 out_class:
860 	class_destroy(uverbs_class);
861 
862 out_chrdev:
863 	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
864 
865 out:
866 	return ret;
867 }
868 
869 static void __exit ib_uverbs_cleanup(void)
870 {
871 	ib_unregister_client(&uverbs_client);
872 	class_destroy(uverbs_class);
873 	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
874 	if (overflow_maj)
875 		unregister_chrdev_region(overflow_maj, IB_UVERBS_MAX_DEVICES);
876 	idr_destroy(&ib_uverbs_pd_idr);
877 	idr_destroy(&ib_uverbs_mr_idr);
878 	idr_destroy(&ib_uverbs_mw_idr);
879 	idr_destroy(&ib_uverbs_ah_idr);
880 	idr_destroy(&ib_uverbs_cq_idr);
881 	idr_destroy(&ib_uverbs_qp_idr);
882 	idr_destroy(&ib_uverbs_srq_idr);
883 }
884 
885 module_init(ib_uverbs_init);
886 module_exit(ib_uverbs_cleanup);
887