1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
5  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
6  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  */
36 
37 #include <rdma/ib_smi.h>
38 #include <rdma/ib_umem.h>
39 #include <rdma/ib_user_verbs.h>
40 #include <rdma/uverbs_ioctl.h>
41 
42 #include <linux/sched.h>
43 #include <linux/slab.h>
44 #include <linux/stat.h>
45 #include <linux/mm.h>
46 #include <linux/export.h>
47 
48 #include "mthca_dev.h"
49 #include "mthca_cmd.h"
50 #include <rdma/mthca-abi.h>
51 #include "mthca_memfree.h"
52 
53 static void init_query_mad(struct ib_smp *mad)
54 {
55 	mad->base_version  = 1;
56 	mad->mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;
57 	mad->class_version = 1;
58 	mad->method    	   = IB_MGMT_METHOD_GET;
59 }
60 
61 static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
62 			      struct ib_udata *uhw)
63 {
64 	struct ib_smp *in_mad  = NULL;
65 	struct ib_smp *out_mad = NULL;
66 	int err = -ENOMEM;
67 	struct mthca_dev *mdev = to_mdev(ibdev);
68 
69 	if (uhw->inlen || uhw->outlen)
70 		return -EINVAL;
71 
72 	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
73 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
74 	if (!in_mad || !out_mad)
75 		goto out;
76 
77 	memset(props, 0, sizeof *props);
78 
79 	props->fw_ver              = mdev->fw_ver;
80 
81 	init_query_mad(in_mad);
82 	in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
83 
84 	err = mthca_MAD_IFC(mdev, 1, 1,
85 			    1, NULL, NULL, in_mad, out_mad);
86 	if (err)
87 		goto out;
88 
89 	props->device_cap_flags    = mdev->device_cap_flags;
90 	props->vendor_id           = be32_to_cpup((__be32 *) (out_mad->data + 36)) &
91 		0xffffff;
92 	props->vendor_part_id      = be16_to_cpup((__be16 *) (out_mad->data + 30));
93 	props->hw_ver              = be32_to_cpup((__be32 *) (out_mad->data + 32));
94 	memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
95 
96 	props->max_mr_size         = ~0ull;
97 	props->page_size_cap       = mdev->limits.page_size_cap;
98 	props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
99 	props->max_qp_wr           = mdev->limits.max_wqes;
100 	props->max_send_sge        = mdev->limits.max_sg;
101 	props->max_recv_sge        = mdev->limits.max_sg;
102 	props->max_sge_rd          = mdev->limits.max_sg;
103 	props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
104 	props->max_cqe             = mdev->limits.max_cqes;
105 	props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
106 	props->max_pd              = mdev->limits.num_pds - mdev->limits.reserved_pds;
107 	props->max_qp_rd_atom      = 1 << mdev->qp_table.rdb_shift;
108 	props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
109 	props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
110 	props->max_srq             = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
111 	props->max_srq_wr          = mdev->limits.max_srq_wqes;
112 	props->max_srq_sge         = mdev->limits.max_srq_sge;
113 	props->local_ca_ack_delay  = mdev->limits.local_ca_ack_delay;
114 	props->atomic_cap          = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ?
115 					IB_ATOMIC_HCA : IB_ATOMIC_NONE;
116 	props->max_pkeys           = mdev->limits.pkey_table_len;
117 	props->max_mcast_grp       = mdev->limits.num_mgms + mdev->limits.num_amgms;
118 	props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
119 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
120 					   props->max_mcast_grp;
121 
122 	err = 0;
123  out:
124 	kfree(in_mad);
125 	kfree(out_mad);
126 	return err;
127 }
128 
129 static int mthca_query_port(struct ib_device *ibdev,
130 			    u8 port, struct ib_port_attr *props)
131 {
132 	struct ib_smp *in_mad  = NULL;
133 	struct ib_smp *out_mad = NULL;
134 	int err = -ENOMEM;
135 
136 	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
137 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
138 	if (!in_mad || !out_mad)
139 		goto out;
140 
141 	/* props being zeroed by the caller, avoid zeroing it here */
142 
143 	init_query_mad(in_mad);
144 	in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
145 	in_mad->attr_mod = cpu_to_be32(port);
146 
147 	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
148 			    port, NULL, NULL, in_mad, out_mad);
149 	if (err)
150 		goto out;
151 
152 	props->lid               = be16_to_cpup((__be16 *) (out_mad->data + 16));
153 	props->lmc               = out_mad->data[34] & 0x7;
154 	props->sm_lid            = be16_to_cpup((__be16 *) (out_mad->data + 18));
155 	props->sm_sl             = out_mad->data[36] & 0xf;
156 	props->state             = out_mad->data[32] & 0xf;
157 	props->phys_state        = out_mad->data[33] >> 4;
158 	props->port_cap_flags    = be32_to_cpup((__be32 *) (out_mad->data + 20));
159 	props->gid_tbl_len       = to_mdev(ibdev)->limits.gid_table_len;
160 	props->max_msg_sz        = 0x80000000;
161 	props->pkey_tbl_len      = to_mdev(ibdev)->limits.pkey_table_len;
162 	props->bad_pkey_cntr     = be16_to_cpup((__be16 *) (out_mad->data + 46));
163 	props->qkey_viol_cntr    = be16_to_cpup((__be16 *) (out_mad->data + 48));
164 	props->active_width      = out_mad->data[31] & 0xf;
165 	props->active_speed      = out_mad->data[35] >> 4;
166 	props->max_mtu           = out_mad->data[41] & 0xf;
167 	props->active_mtu        = out_mad->data[36] >> 4;
168 	props->subnet_timeout    = out_mad->data[51] & 0x1f;
169 	props->max_vl_num        = out_mad->data[37] >> 4;
170 	props->init_type_reply   = out_mad->data[41] >> 4;
171 
172  out:
173 	kfree(in_mad);
174 	kfree(out_mad);
175 	return err;
176 }
177 
178 static int mthca_modify_device(struct ib_device *ibdev,
179 			       int mask,
180 			       struct ib_device_modify *props)
181 {
182 	if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
183 		return -EOPNOTSUPP;
184 
185 	if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
186 		if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
187 			return -ERESTARTSYS;
188 		memcpy(ibdev->node_desc, props->node_desc,
189 		       IB_DEVICE_NODE_DESC_MAX);
190 		mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
191 	}
192 
193 	return 0;
194 }
195 
196 static int mthca_modify_port(struct ib_device *ibdev,
197 			     u8 port, int port_modify_mask,
198 			     struct ib_port_modify *props)
199 {
200 	struct mthca_set_ib_param set_ib;
201 	struct ib_port_attr attr;
202 	int err;
203 
204 	if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
205 		return -ERESTARTSYS;
206 
207 	err = ib_query_port(ibdev, port, &attr);
208 	if (err)
209 		goto out;
210 
211 	set_ib.set_si_guid     = 0;
212 	set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
213 
214 	set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
215 		~props->clr_port_cap_mask;
216 
217 	err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port);
218 	if (err)
219 		goto out;
220 out:
221 	mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
222 	return err;
223 }
224 
225 static int mthca_query_pkey(struct ib_device *ibdev,
226 			    u8 port, u16 index, u16 *pkey)
227 {
228 	struct ib_smp *in_mad  = NULL;
229 	struct ib_smp *out_mad = NULL;
230 	int err = -ENOMEM;
231 
232 	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
233 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
234 	if (!in_mad || !out_mad)
235 		goto out;
236 
237 	init_query_mad(in_mad);
238 	in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
239 	in_mad->attr_mod = cpu_to_be32(index / 32);
240 
241 	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
242 			    port, NULL, NULL, in_mad, out_mad);
243 	if (err)
244 		goto out;
245 
246 	*pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]);
247 
248  out:
249 	kfree(in_mad);
250 	kfree(out_mad);
251 	return err;
252 }
253 
254 static int mthca_query_gid(struct ib_device *ibdev, u8 port,
255 			   int index, union ib_gid *gid)
256 {
257 	struct ib_smp *in_mad  = NULL;
258 	struct ib_smp *out_mad = NULL;
259 	int err = -ENOMEM;
260 
261 	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
262 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
263 	if (!in_mad || !out_mad)
264 		goto out;
265 
266 	init_query_mad(in_mad);
267 	in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
268 	in_mad->attr_mod = cpu_to_be32(port);
269 
270 	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
271 			    port, NULL, NULL, in_mad, out_mad);
272 	if (err)
273 		goto out;
274 
275 	memcpy(gid->raw, out_mad->data + 8, 8);
276 
277 	init_query_mad(in_mad);
278 	in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
279 	in_mad->attr_mod = cpu_to_be32(index / 8);
280 
281 	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
282 			    port, NULL, NULL, in_mad, out_mad);
283 	if (err)
284 		goto out;
285 
286 	memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8);
287 
288  out:
289 	kfree(in_mad);
290 	kfree(out_mad);
291 	return err;
292 }
293 
294 static int mthca_alloc_ucontext(struct ib_ucontext *uctx,
295 				struct ib_udata *udata)
296 {
297 	struct ib_device *ibdev = uctx->device;
298 	struct mthca_alloc_ucontext_resp uresp = {};
299 	struct mthca_ucontext *context = to_mucontext(uctx);
300 	int                              err;
301 
302 	if (!(to_mdev(ibdev)->active))
303 		return -EAGAIN;
304 
305 	uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
306 	if (mthca_is_memfree(to_mdev(ibdev)))
307 		uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
308 	else
309 		uresp.uarc_size = 0;
310 
311 	err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
312 	if (err)
313 		return err;
314 
315 	context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
316 	if (IS_ERR(context->db_tab)) {
317 		err = PTR_ERR(context->db_tab);
318 		mthca_uar_free(to_mdev(ibdev), &context->uar);
319 		return err;
320 	}
321 
322 	if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
323 		mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab);
324 		mthca_uar_free(to_mdev(ibdev), &context->uar);
325 		return -EFAULT;
326 	}
327 
328 	context->reg_mr_warned = 0;
329 
330 	return 0;
331 }
332 
333 static void mthca_dealloc_ucontext(struct ib_ucontext *context)
334 {
335 	mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar,
336 				  to_mucontext(context)->db_tab);
337 	mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar);
338 }
339 
340 static int mthca_mmap_uar(struct ib_ucontext *context,
341 			  struct vm_area_struct *vma)
342 {
343 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
344 		return -EINVAL;
345 
346 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
347 
348 	if (io_remap_pfn_range(vma, vma->vm_start,
349 			       to_mucontext(context)->uar.pfn,
350 			       PAGE_SIZE, vma->vm_page_prot))
351 		return -EAGAIN;
352 
353 	return 0;
354 }
355 
356 static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
357 {
358 	struct ib_device *ibdev = ibpd->device;
359 	struct mthca_pd *pd = to_mpd(ibpd);
360 	int err;
361 
362 	err = mthca_pd_alloc(to_mdev(ibdev), !udata, pd);
363 	if (err)
364 		return err;
365 
366 	if (udata) {
367 		if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) {
368 			mthca_pd_free(to_mdev(ibdev), pd);
369 			return -EFAULT;
370 		}
371 	}
372 
373 	return 0;
374 }
375 
376 static int mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
377 {
378 	mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
379 	return 0;
380 }
381 
382 static int mthca_ah_create(struct ib_ah *ibah,
383 			   struct rdma_ah_init_attr *init_attr,
384 			   struct ib_udata *udata)
385 
386 {
387 	struct mthca_ah *ah = to_mah(ibah);
388 
389 	return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd),
390 			       init_attr->ah_attr, ah);
391 }
392 
393 static int mthca_ah_destroy(struct ib_ah *ah, u32 flags)
394 {
395 	mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
396 	return 0;
397 }
398 
399 static int mthca_create_srq(struct ib_srq *ibsrq,
400 			    struct ib_srq_init_attr *init_attr,
401 			    struct ib_udata *udata)
402 {
403 	struct mthca_create_srq ucmd;
404 	struct mthca_ucontext *context = rdma_udata_to_drv_context(
405 		udata, struct mthca_ucontext, ibucontext);
406 	struct mthca_srq *srq = to_msrq(ibsrq);
407 	int err;
408 
409 	if (init_attr->srq_type != IB_SRQT_BASIC)
410 		return -EOPNOTSUPP;
411 
412 	if (udata) {
413 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
414 			return -EFAULT;
415 
416 		err = mthca_map_user_db(to_mdev(ibsrq->device), &context->uar,
417 					context->db_tab, ucmd.db_index,
418 					ucmd.db_page);
419 
420 		if (err)
421 			return err;
422 
423 		srq->mr.ibmr.lkey = ucmd.lkey;
424 		srq->db_index     = ucmd.db_index;
425 	}
426 
427 	err = mthca_alloc_srq(to_mdev(ibsrq->device), to_mpd(ibsrq->pd),
428 			      &init_attr->attr, srq, udata);
429 
430 	if (err && udata)
431 		mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar,
432 				    context->db_tab, ucmd.db_index);
433 
434 	if (err)
435 		return err;
436 
437 	if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
438 		mthca_free_srq(to_mdev(ibsrq->device), srq);
439 		return -EFAULT;
440 	}
441 
442 	return 0;
443 }
444 
445 static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
446 {
447 	if (udata) {
448 		struct mthca_ucontext *context =
449 			rdma_udata_to_drv_context(
450 				udata,
451 				struct mthca_ucontext,
452 				ibucontext);
453 
454 		mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
455 				    context->db_tab, to_msrq(srq)->db_index);
456 	}
457 
458 	mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
459 	return 0;
460 }
461 
462 static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
463 				     struct ib_qp_init_attr *init_attr,
464 				     struct ib_udata *udata)
465 {
466 	struct mthca_ucontext *context = rdma_udata_to_drv_context(
467 		udata, struct mthca_ucontext, ibucontext);
468 	struct mthca_create_qp ucmd;
469 	struct mthca_qp *qp;
470 	int err;
471 
472 	if (init_attr->create_flags)
473 		return ERR_PTR(-EOPNOTSUPP);
474 
475 	switch (init_attr->qp_type) {
476 	case IB_QPT_RC:
477 	case IB_QPT_UC:
478 	case IB_QPT_UD:
479 	{
480 		qp = kzalloc(sizeof(*qp), GFP_KERNEL);
481 		if (!qp)
482 			return ERR_PTR(-ENOMEM);
483 
484 		if (udata) {
485 			if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
486 				kfree(qp);
487 				return ERR_PTR(-EFAULT);
488 			}
489 
490 			err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
491 						context->db_tab,
492 						ucmd.sq_db_index, ucmd.sq_db_page);
493 			if (err) {
494 				kfree(qp);
495 				return ERR_PTR(err);
496 			}
497 
498 			err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
499 						context->db_tab,
500 						ucmd.rq_db_index, ucmd.rq_db_page);
501 			if (err) {
502 				mthca_unmap_user_db(to_mdev(pd->device),
503 						    &context->uar,
504 						    context->db_tab,
505 						    ucmd.sq_db_index);
506 				kfree(qp);
507 				return ERR_PTR(err);
508 			}
509 
510 			qp->mr.ibmr.lkey = ucmd.lkey;
511 			qp->sq.db_index  = ucmd.sq_db_index;
512 			qp->rq.db_index  = ucmd.rq_db_index;
513 		}
514 
515 		err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
516 				     to_mcq(init_attr->send_cq),
517 				     to_mcq(init_attr->recv_cq),
518 				     init_attr->qp_type, init_attr->sq_sig_type,
519 				     &init_attr->cap, qp, udata);
520 
521 		if (err && udata) {
522 			mthca_unmap_user_db(to_mdev(pd->device),
523 					    &context->uar,
524 					    context->db_tab,
525 					    ucmd.sq_db_index);
526 			mthca_unmap_user_db(to_mdev(pd->device),
527 					    &context->uar,
528 					    context->db_tab,
529 					    ucmd.rq_db_index);
530 		}
531 
532 		qp->ibqp.qp_num = qp->qpn;
533 		break;
534 	}
535 	case IB_QPT_SMI:
536 	case IB_QPT_GSI:
537 	{
538 		qp = kzalloc(sizeof(*qp), GFP_KERNEL);
539 		if (!qp)
540 			return ERR_PTR(-ENOMEM);
541 		qp->sqp = kzalloc(sizeof(struct mthca_sqp), GFP_KERNEL);
542 		if (!qp->sqp) {
543 			kfree(qp);
544 			return ERR_PTR(-ENOMEM);
545 		}
546 
547 		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
548 
549 		err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
550 				      to_mcq(init_attr->send_cq),
551 				      to_mcq(init_attr->recv_cq),
552 				      init_attr->sq_sig_type, &init_attr->cap,
553 				      qp->ibqp.qp_num, init_attr->port_num,
554 				      qp, udata);
555 		break;
556 	}
557 	default:
558 		/* Don't support raw QPs */
559 		return ERR_PTR(-EOPNOTSUPP);
560 	}
561 
562 	if (err) {
563 		kfree(qp->sqp);
564 		kfree(qp);
565 		return ERR_PTR(err);
566 	}
567 
568 	init_attr->cap.max_send_wr     = qp->sq.max;
569 	init_attr->cap.max_recv_wr     = qp->rq.max;
570 	init_attr->cap.max_send_sge    = qp->sq.max_gs;
571 	init_attr->cap.max_recv_sge    = qp->rq.max_gs;
572 	init_attr->cap.max_inline_data = qp->max_inline_data;
573 
574 	return &qp->ibqp;
575 }
576 
577 static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
578 {
579 	if (udata) {
580 		struct mthca_ucontext *context =
581 			rdma_udata_to_drv_context(
582 				udata,
583 				struct mthca_ucontext,
584 				ibucontext);
585 
586 		mthca_unmap_user_db(to_mdev(qp->device),
587 				    &context->uar,
588 				    context->db_tab,
589 				    to_mqp(qp)->sq.db_index);
590 		mthca_unmap_user_db(to_mdev(qp->device),
591 				    &context->uar,
592 				    context->db_tab,
593 				    to_mqp(qp)->rq.db_index);
594 	}
595 	mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
596 	kfree(to_mqp(qp)->sqp);
597 	kfree(to_mqp(qp));
598 	return 0;
599 }
600 
601 static int mthca_create_cq(struct ib_cq *ibcq,
602 			   const struct ib_cq_init_attr *attr,
603 			   struct ib_udata *udata)
604 {
605 	struct ib_device *ibdev = ibcq->device;
606 	int entries = attr->cqe;
607 	struct mthca_create_cq ucmd;
608 	struct mthca_cq *cq;
609 	int nent;
610 	int err;
611 	struct mthca_ucontext *context = rdma_udata_to_drv_context(
612 		udata, struct mthca_ucontext, ibucontext);
613 
614 	if (attr->flags)
615 		return -EOPNOTSUPP;
616 
617 	if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
618 		return -EINVAL;
619 
620 	if (udata) {
621 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
622 			return -EFAULT;
623 
624 		err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
625 					context->db_tab, ucmd.set_db_index,
626 					ucmd.set_db_page);
627 		if (err)
628 			return err;
629 
630 		err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
631 					context->db_tab, ucmd.arm_db_index,
632 					ucmd.arm_db_page);
633 		if (err)
634 			goto err_unmap_set;
635 	}
636 
637 	cq = to_mcq(ibcq);
638 
639 	if (udata) {
640 		cq->buf.mr.ibmr.lkey = ucmd.lkey;
641 		cq->set_ci_db_index  = ucmd.set_db_index;
642 		cq->arm_db_index     = ucmd.arm_db_index;
643 	}
644 
645 	for (nent = 1; nent <= entries; nent <<= 1)
646 		; /* nothing */
647 
648 	err = mthca_init_cq(to_mdev(ibdev), nent, context,
649 			    udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
650 			    cq);
651 	if (err)
652 		goto err_unmap_arm;
653 
654 	if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) {
655 		mthca_free_cq(to_mdev(ibdev), cq);
656 		err = -EFAULT;
657 		goto err_unmap_arm;
658 	}
659 
660 	cq->resize_buf = NULL;
661 
662 	return 0;
663 
664 err_unmap_arm:
665 	if (udata)
666 		mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
667 				    context->db_tab, ucmd.arm_db_index);
668 
669 err_unmap_set:
670 	if (udata)
671 		mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
672 				    context->db_tab, ucmd.set_db_index);
673 
674 	return err;
675 }
676 
677 static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq,
678 				  int entries)
679 {
680 	int ret;
681 
682 	spin_lock_irq(&cq->lock);
683 	if (cq->resize_buf) {
684 		ret = -EBUSY;
685 		goto unlock;
686 	}
687 
688 	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC);
689 	if (!cq->resize_buf) {
690 		ret = -ENOMEM;
691 		goto unlock;
692 	}
693 
694 	cq->resize_buf->state = CQ_RESIZE_ALLOC;
695 
696 	ret = 0;
697 
698 unlock:
699 	spin_unlock_irq(&cq->lock);
700 
701 	if (ret)
702 		return ret;
703 
704 	ret = mthca_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);
705 	if (ret) {
706 		spin_lock_irq(&cq->lock);
707 		kfree(cq->resize_buf);
708 		cq->resize_buf = NULL;
709 		spin_unlock_irq(&cq->lock);
710 		return ret;
711 	}
712 
713 	cq->resize_buf->cqe = entries - 1;
714 
715 	spin_lock_irq(&cq->lock);
716 	cq->resize_buf->state = CQ_RESIZE_READY;
717 	spin_unlock_irq(&cq->lock);
718 
719 	return 0;
720 }
721 
722 static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
723 {
724 	struct mthca_dev *dev = to_mdev(ibcq->device);
725 	struct mthca_cq *cq = to_mcq(ibcq);
726 	struct mthca_resize_cq ucmd;
727 	u32 lkey;
728 	int ret;
729 
730 	if (entries < 1 || entries > dev->limits.max_cqes)
731 		return -EINVAL;
732 
733 	mutex_lock(&cq->mutex);
734 
735 	entries = roundup_pow_of_two(entries + 1);
736 	if (entries == ibcq->cqe + 1) {
737 		ret = 0;
738 		goto out;
739 	}
740 
741 	if (cq->is_kernel) {
742 		ret = mthca_alloc_resize_buf(dev, cq, entries);
743 		if (ret)
744 			goto out;
745 		lkey = cq->resize_buf->buf.mr.ibmr.lkey;
746 	} else {
747 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
748 			ret = -EFAULT;
749 			goto out;
750 		}
751 		lkey = ucmd.lkey;
752 	}
753 
754 	ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries));
755 
756 	if (ret) {
757 		if (cq->resize_buf) {
758 			mthca_free_cq_buf(dev, &cq->resize_buf->buf,
759 					  cq->resize_buf->cqe);
760 			kfree(cq->resize_buf);
761 			spin_lock_irq(&cq->lock);
762 			cq->resize_buf = NULL;
763 			spin_unlock_irq(&cq->lock);
764 		}
765 		goto out;
766 	}
767 
768 	if (cq->is_kernel) {
769 		struct mthca_cq_buf tbuf;
770 		int tcqe;
771 
772 		spin_lock_irq(&cq->lock);
773 		if (cq->resize_buf->state == CQ_RESIZE_READY) {
774 			mthca_cq_resize_copy_cqes(cq);
775 			tbuf         = cq->buf;
776 			tcqe         = cq->ibcq.cqe;
777 			cq->buf      = cq->resize_buf->buf;
778 			cq->ibcq.cqe = cq->resize_buf->cqe;
779 		} else {
780 			tbuf = cq->resize_buf->buf;
781 			tcqe = cq->resize_buf->cqe;
782 		}
783 
784 		kfree(cq->resize_buf);
785 		cq->resize_buf = NULL;
786 		spin_unlock_irq(&cq->lock);
787 
788 		mthca_free_cq_buf(dev, &tbuf, tcqe);
789 	} else
790 		ibcq->cqe = entries - 1;
791 
792 out:
793 	mutex_unlock(&cq->mutex);
794 
795 	return ret;
796 }
797 
798 static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
799 {
800 	if (udata) {
801 		struct mthca_ucontext *context =
802 			rdma_udata_to_drv_context(
803 				udata,
804 				struct mthca_ucontext,
805 				ibucontext);
806 
807 		mthca_unmap_user_db(to_mdev(cq->device),
808 				    &context->uar,
809 				    context->db_tab,
810 				    to_mcq(cq)->arm_db_index);
811 		mthca_unmap_user_db(to_mdev(cq->device),
812 				    &context->uar,
813 				    context->db_tab,
814 				    to_mcq(cq)->set_ci_db_index);
815 	}
816 	mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
817 	return 0;
818 }
819 
820 static inline u32 convert_access(int acc)
821 {
822 	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC       : 0) |
823 	       (acc & IB_ACCESS_REMOTE_WRITE  ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) |
824 	       (acc & IB_ACCESS_REMOTE_READ   ? MTHCA_MPT_FLAG_REMOTE_READ  : 0) |
825 	       (acc & IB_ACCESS_LOCAL_WRITE   ? MTHCA_MPT_FLAG_LOCAL_WRITE  : 0) |
826 	       MTHCA_MPT_FLAG_LOCAL_READ;
827 }
828 
829 static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc)
830 {
831 	struct mthca_mr *mr;
832 	int err;
833 
834 	mr = kmalloc(sizeof *mr, GFP_KERNEL);
835 	if (!mr)
836 		return ERR_PTR(-ENOMEM);
837 
838 	err = mthca_mr_alloc_notrans(to_mdev(pd->device),
839 				     to_mpd(pd)->pd_num,
840 				     convert_access(acc), mr);
841 
842 	if (err) {
843 		kfree(mr);
844 		return ERR_PTR(err);
845 	}
846 
847 	mr->umem = NULL;
848 
849 	return &mr->ibmr;
850 }
851 
852 static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
853 				       u64 virt, int acc, struct ib_udata *udata)
854 {
855 	struct mthca_dev *dev = to_mdev(pd->device);
856 	struct ib_block_iter biter;
857 	struct mthca_ucontext *context = rdma_udata_to_drv_context(
858 		udata, struct mthca_ucontext, ibucontext);
859 	struct mthca_mr *mr;
860 	struct mthca_reg_mr ucmd;
861 	u64 *pages;
862 	int n, i;
863 	int err = 0;
864 	int write_mtt_size;
865 
866 	if (udata->inlen < sizeof ucmd) {
867 		if (!context->reg_mr_warned) {
868 			mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n",
869 				   current->comm);
870 			mthca_warn(dev, "  Update libmthca to fix this.\n");
871 		}
872 		++context->reg_mr_warned;
873 		ucmd.mr_attrs = 0;
874 	} else if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
875 		return ERR_PTR(-EFAULT);
876 
877 	mr = kmalloc(sizeof *mr, GFP_KERNEL);
878 	if (!mr)
879 		return ERR_PTR(-ENOMEM);
880 
881 	mr->umem = ib_umem_get(pd->device, start, length, acc);
882 	if (IS_ERR(mr->umem)) {
883 		err = PTR_ERR(mr->umem);
884 		goto err;
885 	}
886 
887 	n = ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE);
888 
889 	mr->mtt = mthca_alloc_mtt(dev, n);
890 	if (IS_ERR(mr->mtt)) {
891 		err = PTR_ERR(mr->mtt);
892 		goto err_umem;
893 	}
894 
895 	pages = (u64 *) __get_free_page(GFP_KERNEL);
896 	if (!pages) {
897 		err = -ENOMEM;
898 		goto err_mtt;
899 	}
900 
901 	i = n = 0;
902 
903 	write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages));
904 
905 	rdma_umem_for_each_dma_block(mr->umem, &biter, PAGE_SIZE) {
906 		pages[i++] = rdma_block_iter_dma_address(&biter);
907 
908 		/*
909 		 * Be friendly to write_mtt and pass it chunks
910 		 * of appropriate size.
911 		 */
912 		if (i == write_mtt_size) {
913 			err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
914 			if (err)
915 				goto mtt_done;
916 			n += i;
917 			i = 0;
918 		}
919 	}
920 
921 	if (i)
922 		err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
923 mtt_done:
924 	free_page((unsigned long) pages);
925 	if (err)
926 		goto err_mtt;
927 
928 	err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, PAGE_SHIFT, virt, length,
929 			     convert_access(acc), mr);
930 
931 	if (err)
932 		goto err_mtt;
933 
934 	return &mr->ibmr;
935 
936 err_mtt:
937 	mthca_free_mtt(dev, mr->mtt);
938 
939 err_umem:
940 	ib_umem_release(mr->umem);
941 
942 err:
943 	kfree(mr);
944 	return ERR_PTR(err);
945 }
946 
947 static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata)
948 {
949 	struct mthca_mr *mmr = to_mmr(mr);
950 
951 	mthca_free_mr(to_mdev(mr->device), mmr);
952 	ib_umem_release(mmr->umem);
953 	kfree(mmr);
954 
955 	return 0;
956 }
957 
958 static ssize_t hw_rev_show(struct device *device,
959 			   struct device_attribute *attr, char *buf)
960 {
961 	struct mthca_dev *dev =
962 		rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
963 
964 	return sysfs_emit(buf, "%x\n", dev->rev_id);
965 }
966 static DEVICE_ATTR_RO(hw_rev);
967 
968 static const char *hca_type_string(int hca_type)
969 {
970 	switch (hca_type) {
971 	case PCI_DEVICE_ID_MELLANOX_TAVOR:
972 		return "MT23108";
973 	case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT:
974 		return "MT25208 (MT23108 compat mode)";
975 	case PCI_DEVICE_ID_MELLANOX_ARBEL:
976 		return "MT25208";
977 	case PCI_DEVICE_ID_MELLANOX_SINAI:
978 	case PCI_DEVICE_ID_MELLANOX_SINAI_OLD:
979 		return "MT25204";
980 	}
981 
982 	return "unknown";
983 }
984 
985 static ssize_t hca_type_show(struct device *device,
986 			     struct device_attribute *attr, char *buf)
987 {
988 	struct mthca_dev *dev =
989 		rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
990 
991 	return sysfs_emit(buf, "%s\n", hca_type_string(dev->pdev->device));
992 }
993 static DEVICE_ATTR_RO(hca_type);
994 
995 static ssize_t board_id_show(struct device *device,
996 			     struct device_attribute *attr, char *buf)
997 {
998 	struct mthca_dev *dev =
999 		rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
1000 
1001 	return sysfs_emit(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id);
1002 }
1003 static DEVICE_ATTR_RO(board_id);
1004 
1005 static struct attribute *mthca_dev_attributes[] = {
1006 	&dev_attr_hw_rev.attr,
1007 	&dev_attr_hca_type.attr,
1008 	&dev_attr_board_id.attr,
1009 	NULL
1010 };
1011 
1012 static const struct attribute_group mthca_attr_group = {
1013 	.attrs = mthca_dev_attributes,
1014 };
1015 
1016 static int mthca_init_node_data(struct mthca_dev *dev)
1017 {
1018 	struct ib_smp *in_mad  = NULL;
1019 	struct ib_smp *out_mad = NULL;
1020 	int err = -ENOMEM;
1021 
1022 	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1023 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1024 	if (!in_mad || !out_mad)
1025 		goto out;
1026 
1027 	init_query_mad(in_mad);
1028 	in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
1029 
1030 	err = mthca_MAD_IFC(dev, 1, 1,
1031 			    1, NULL, NULL, in_mad, out_mad);
1032 	if (err)
1033 		goto out;
1034 
1035 	memcpy(dev->ib_dev.node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX);
1036 
1037 	in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1038 
1039 	err = mthca_MAD_IFC(dev, 1, 1,
1040 			    1, NULL, NULL, in_mad, out_mad);
1041 	if (err)
1042 		goto out;
1043 
1044 	if (mthca_is_memfree(dev))
1045 		dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32));
1046 	memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1047 
1048 out:
1049 	kfree(in_mad);
1050 	kfree(out_mad);
1051 	return err;
1052 }
1053 
1054 static int mthca_port_immutable(struct ib_device *ibdev, u8 port_num,
1055 			        struct ib_port_immutable *immutable)
1056 {
1057 	struct ib_port_attr attr;
1058 	int err;
1059 
1060 	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
1061 
1062 	err = ib_query_port(ibdev, port_num, &attr);
1063 	if (err)
1064 		return err;
1065 
1066 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
1067 	immutable->gid_tbl_len = attr.gid_tbl_len;
1068 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
1069 
1070 	return 0;
1071 }
1072 
1073 static void get_dev_fw_str(struct ib_device *device, char *str)
1074 {
1075 	struct mthca_dev *dev =
1076 		container_of(device, struct mthca_dev, ib_dev);
1077 	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d",
1078 		 (int) (dev->fw_ver >> 32),
1079 		 (int) (dev->fw_ver >> 16) & 0xffff,
1080 		 (int) dev->fw_ver & 0xffff);
1081 }
1082 
1083 static const struct ib_device_ops mthca_dev_ops = {
1084 	.owner = THIS_MODULE,
1085 	.driver_id = RDMA_DRIVER_MTHCA,
1086 	.uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION,
1087 	.uverbs_no_driver_id_binding = 1,
1088 
1089 	.alloc_pd = mthca_alloc_pd,
1090 	.alloc_ucontext = mthca_alloc_ucontext,
1091 	.attach_mcast = mthca_multicast_attach,
1092 	.create_ah = mthca_ah_create,
1093 	.create_cq = mthca_create_cq,
1094 	.create_qp = mthca_create_qp,
1095 	.dealloc_pd = mthca_dealloc_pd,
1096 	.dealloc_ucontext = mthca_dealloc_ucontext,
1097 	.dereg_mr = mthca_dereg_mr,
1098 	.destroy_ah = mthca_ah_destroy,
1099 	.destroy_cq = mthca_destroy_cq,
1100 	.destroy_qp = mthca_destroy_qp,
1101 	.detach_mcast = mthca_multicast_detach,
1102 	.get_dev_fw_str = get_dev_fw_str,
1103 	.get_dma_mr = mthca_get_dma_mr,
1104 	.get_port_immutable = mthca_port_immutable,
1105 	.mmap = mthca_mmap_uar,
1106 	.modify_device = mthca_modify_device,
1107 	.modify_port = mthca_modify_port,
1108 	.modify_qp = mthca_modify_qp,
1109 	.poll_cq = mthca_poll_cq,
1110 	.process_mad = mthca_process_mad,
1111 	.query_ah = mthca_ah_query,
1112 	.query_device = mthca_query_device,
1113 	.query_gid = mthca_query_gid,
1114 	.query_pkey = mthca_query_pkey,
1115 	.query_port = mthca_query_port,
1116 	.query_qp = mthca_query_qp,
1117 	.reg_user_mr = mthca_reg_user_mr,
1118 	.resize_cq = mthca_resize_cq,
1119 
1120 	INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah),
1121 	INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq),
1122 	INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd),
1123 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext),
1124 };
1125 
1126 static const struct ib_device_ops mthca_dev_arbel_srq_ops = {
1127 	.create_srq = mthca_create_srq,
1128 	.destroy_srq = mthca_destroy_srq,
1129 	.modify_srq = mthca_modify_srq,
1130 	.post_srq_recv = mthca_arbel_post_srq_recv,
1131 	.query_srq = mthca_query_srq,
1132 
1133 	INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq),
1134 };
1135 
1136 static const struct ib_device_ops mthca_dev_tavor_srq_ops = {
1137 	.create_srq = mthca_create_srq,
1138 	.destroy_srq = mthca_destroy_srq,
1139 	.modify_srq = mthca_modify_srq,
1140 	.post_srq_recv = mthca_tavor_post_srq_recv,
1141 	.query_srq = mthca_query_srq,
1142 
1143 	INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq),
1144 };
1145 
1146 static const struct ib_device_ops mthca_dev_arbel_ops = {
1147 	.post_recv = mthca_arbel_post_receive,
1148 	.post_send = mthca_arbel_post_send,
1149 	.req_notify_cq = mthca_arbel_arm_cq,
1150 };
1151 
1152 static const struct ib_device_ops mthca_dev_tavor_ops = {
1153 	.post_recv = mthca_tavor_post_receive,
1154 	.post_send = mthca_tavor_post_send,
1155 	.req_notify_cq = mthca_tavor_arm_cq,
1156 };
1157 
1158 int mthca_register_device(struct mthca_dev *dev)
1159 {
1160 	int ret;
1161 
1162 	ret = mthca_init_node_data(dev);
1163 	if (ret)
1164 		return ret;
1165 
1166 	dev->ib_dev.node_type            = RDMA_NODE_IB_CA;
1167 	dev->ib_dev.phys_port_cnt        = dev->limits.num_ports;
1168 	dev->ib_dev.num_comp_vectors     = 1;
1169 	dev->ib_dev.dev.parent           = &dev->pdev->dev;
1170 
1171 	if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
1172 		if (mthca_is_memfree(dev))
1173 			ib_set_device_ops(&dev->ib_dev,
1174 					  &mthca_dev_arbel_srq_ops);
1175 		else
1176 			ib_set_device_ops(&dev->ib_dev,
1177 					  &mthca_dev_tavor_srq_ops);
1178 	}
1179 
1180 	ib_set_device_ops(&dev->ib_dev, &mthca_dev_ops);
1181 
1182 	if (mthca_is_memfree(dev))
1183 		ib_set_device_ops(&dev->ib_dev, &mthca_dev_arbel_ops);
1184 	else
1185 		ib_set_device_ops(&dev->ib_dev, &mthca_dev_tavor_ops);
1186 
1187 	mutex_init(&dev->cap_mask_mutex);
1188 
1189 	rdma_set_device_sysfs_group(&dev->ib_dev, &mthca_attr_group);
1190 	ret = ib_register_device(&dev->ib_dev, "mthca%d", &dev->pdev->dev);
1191 	if (ret)
1192 		return ret;
1193 
1194 	mthca_start_catas_poll(dev);
1195 
1196 	return 0;
1197 }
1198 
1199 void mthca_unregister_device(struct mthca_dev *dev)
1200 {
1201 	mthca_stop_catas_poll(dev);
1202 	ib_unregister_device(&dev->ib_dev);
1203 }
1204