1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  * $Id: mthca_main.c 1396 2004-12-28 04:10:27Z roland $
35  */
36 
37 #include <linux/module.h>
38 #include <linux/init.h>
39 #include <linux/errno.h>
40 #include <linux/pci.h>
41 #include <linux/interrupt.h>
42 
43 #include "mthca_dev.h"
44 #include "mthca_config_reg.h"
45 #include "mthca_cmd.h"
46 #include "mthca_profile.h"
47 #include "mthca_memfree.h"
48 
49 MODULE_AUTHOR("Roland Dreier");
50 MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
51 MODULE_LICENSE("Dual BSD/GPL");
52 MODULE_VERSION(DRV_VERSION);
53 
54 #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG
55 
56 int mthca_debug_level = 0;
57 module_param_named(debug_level, mthca_debug_level, int, 0644);
58 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
59 
60 #endif /* CONFIG_INFINIBAND_MTHCA_DEBUG */
61 
62 #ifdef CONFIG_PCI_MSI
63 
64 static int msi_x = 1;
65 module_param(msi_x, int, 0444);
66 MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
67 
68 #else /* CONFIG_PCI_MSI */
69 
70 #define msi_x (0)
71 
72 #endif /* CONFIG_PCI_MSI */
73 
74 static int tune_pci = 0;
75 module_param(tune_pci, int, 0444);
76 MODULE_PARM_DESC(tune_pci, "increase PCI burst from the default set by BIOS if nonzero");
77 
78 DEFINE_MUTEX(mthca_device_mutex);
79 
80 #define MTHCA_DEFAULT_NUM_QP            (1 << 16)
81 #define MTHCA_DEFAULT_RDB_PER_QP        (1 << 2)
82 #define MTHCA_DEFAULT_NUM_CQ            (1 << 16)
83 #define MTHCA_DEFAULT_NUM_MCG           (1 << 13)
84 #define MTHCA_DEFAULT_NUM_MPT           (1 << 17)
85 #define MTHCA_DEFAULT_NUM_MTT           (1 << 20)
86 #define MTHCA_DEFAULT_NUM_UDAV          (1 << 15)
87 #define MTHCA_DEFAULT_NUM_RESERVED_MTTS (1 << 18)
88 #define MTHCA_DEFAULT_NUM_UARC_SIZE     (1 << 18)
89 
90 static struct mthca_profile hca_profile = {
91 	.num_qp             = MTHCA_DEFAULT_NUM_QP,
92 	.rdb_per_qp         = MTHCA_DEFAULT_RDB_PER_QP,
93 	.num_cq             = MTHCA_DEFAULT_NUM_CQ,
94 	.num_mcg            = MTHCA_DEFAULT_NUM_MCG,
95 	.num_mpt            = MTHCA_DEFAULT_NUM_MPT,
96 	.num_mtt            = MTHCA_DEFAULT_NUM_MTT,
97 	.num_udav           = MTHCA_DEFAULT_NUM_UDAV,          /* Tavor only */
98 	.fmr_reserved_mtts  = MTHCA_DEFAULT_NUM_RESERVED_MTTS, /* Tavor only */
99 	.uarc_size          = MTHCA_DEFAULT_NUM_UARC_SIZE,     /* Arbel only */
100 };
101 
102 module_param_named(num_qp, hca_profile.num_qp, int, 0444);
103 MODULE_PARM_DESC(num_qp, "maximum number of QPs per HCA");
104 
105 module_param_named(rdb_per_qp, hca_profile.rdb_per_qp, int, 0444);
106 MODULE_PARM_DESC(rdb_per_qp, "number of RDB buffers per QP");
107 
108 module_param_named(num_cq, hca_profile.num_cq, int, 0444);
109 MODULE_PARM_DESC(num_cq, "maximum number of CQs per HCA");
110 
111 module_param_named(num_mcg, hca_profile.num_mcg, int, 0444);
112 MODULE_PARM_DESC(num_mcg, "maximum number of multicast groups per HCA");
113 
114 module_param_named(num_mpt, hca_profile.num_mpt, int, 0444);
115 MODULE_PARM_DESC(num_mpt,
116 		"maximum number of memory protection table entries per HCA");
117 
118 module_param_named(num_mtt, hca_profile.num_mtt, int, 0444);
119 MODULE_PARM_DESC(num_mtt,
120 		 "maximum number of memory translation table segments per HCA");
121 
122 module_param_named(num_udav, hca_profile.num_udav, int, 0444);
123 MODULE_PARM_DESC(num_udav, "maximum number of UD address vectors per HCA");
124 
125 module_param_named(fmr_reserved_mtts, hca_profile.fmr_reserved_mtts, int, 0444);
126 MODULE_PARM_DESC(fmr_reserved_mtts,
127 		 "number of memory translation table segments reserved for FMR");
128 
129 static char mthca_version[] __devinitdata =
130 	DRV_NAME ": Mellanox InfiniBand HCA driver v"
131 	DRV_VERSION " (" DRV_RELDATE ")\n";
132 
133 static int mthca_tune_pci(struct mthca_dev *mdev)
134 {
135 	if (!tune_pci)
136 		return 0;
137 
138 	/* First try to max out Read Byte Count */
139 	if (pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX)) {
140 		if (pcix_set_mmrbc(mdev->pdev, pcix_get_max_mmrbc(mdev->pdev))) {
141 			mthca_err(mdev, "Couldn't set PCI-X max read count, "
142 				"aborting.\n");
143 			return -ENODEV;
144 		}
145 	} else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE))
146 		mthca_info(mdev, "No PCI-X capability, not setting RBC.\n");
147 
148 	if (pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP)) {
149 		if (pcie_set_readrq(mdev->pdev, 4096)) {
150 			mthca_err(mdev, "Couldn't write PCI Express read request, "
151 				"aborting.\n");
152 			return -ENODEV;
153 		}
154 	} else if (mdev->mthca_flags & MTHCA_FLAG_PCIE)
155 		mthca_info(mdev, "No PCI Express capability, "
156 			   "not setting Max Read Request Size.\n");
157 
158 	return 0;
159 }
160 
161 static int mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim)
162 {
163 	int err;
164 	u8 status;
165 
166 	err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status);
167 	if (err) {
168 		mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
169 		return err;
170 	}
171 	if (status) {
172 		mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, "
173 			  "aborting.\n", status);
174 		return -EINVAL;
175 	}
176 	if (dev_lim->min_page_sz > PAGE_SIZE) {
177 		mthca_err(mdev, "HCA minimum page size of %d bigger than "
178 			  "kernel PAGE_SIZE of %ld, aborting.\n",
179 			  dev_lim->min_page_sz, PAGE_SIZE);
180 		return -ENODEV;
181 	}
182 	if (dev_lim->num_ports > MTHCA_MAX_PORTS) {
183 		mthca_err(mdev, "HCA has %d ports, but we only support %d, "
184 			  "aborting.\n",
185 			  dev_lim->num_ports, MTHCA_MAX_PORTS);
186 		return -ENODEV;
187 	}
188 
189 	if (dev_lim->uar_size > pci_resource_len(mdev->pdev, 2)) {
190 		mthca_err(mdev, "HCA reported UAR size of 0x%x bigger than "
191 			  "PCI resource 2 size of 0x%llx, aborting.\n",
192 			  dev_lim->uar_size,
193 			  (unsigned long long)pci_resource_len(mdev->pdev, 2));
194 		return -ENODEV;
195 	}
196 
197 	mdev->limits.num_ports      	= dev_lim->num_ports;
198 	mdev->limits.vl_cap             = dev_lim->max_vl;
199 	mdev->limits.mtu_cap            = dev_lim->max_mtu;
200 	mdev->limits.gid_table_len  	= dev_lim->max_gids;
201 	mdev->limits.pkey_table_len 	= dev_lim->max_pkeys;
202 	mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay;
203 	mdev->limits.max_sg             = dev_lim->max_sg;
204 	mdev->limits.max_wqes           = dev_lim->max_qp_sz;
205 	mdev->limits.max_qp_init_rdma   = dev_lim->max_requester_per_qp;
206 	mdev->limits.reserved_qps       = dev_lim->reserved_qps;
207 	mdev->limits.max_srq_wqes       = dev_lim->max_srq_sz;
208 	mdev->limits.reserved_srqs      = dev_lim->reserved_srqs;
209 	mdev->limits.reserved_eecs      = dev_lim->reserved_eecs;
210 	mdev->limits.max_desc_sz        = dev_lim->max_desc_sz;
211 	mdev->limits.max_srq_sge	= mthca_max_srq_sge(mdev);
212 	/*
213 	 * Subtract 1 from the limit because we need to allocate a
214 	 * spare CQE so the HCA HW can tell the difference between an
215 	 * empty CQ and a full CQ.
216 	 */
217 	mdev->limits.max_cqes           = dev_lim->max_cq_sz - 1;
218 	mdev->limits.reserved_cqs       = dev_lim->reserved_cqs;
219 	mdev->limits.reserved_eqs       = dev_lim->reserved_eqs;
220 	mdev->limits.reserved_mtts      = dev_lim->reserved_mtts;
221 	mdev->limits.reserved_mrws      = dev_lim->reserved_mrws;
222 	mdev->limits.reserved_uars      = dev_lim->reserved_uars;
223 	mdev->limits.reserved_pds       = dev_lim->reserved_pds;
224 	mdev->limits.port_width_cap     = dev_lim->max_port_width;
225 	mdev->limits.page_size_cap      = ~(u32) (dev_lim->min_page_sz - 1);
226 	mdev->limits.flags              = dev_lim->flags;
227 	/*
228 	 * For old FW that doesn't return static rate support, use a
229 	 * value of 0x3 (only static rate values of 0 or 1 are handled),
230 	 * except on Sinai, where even old FW can handle static rate
231 	 * values of 2 and 3.
232 	 */
233 	if (dev_lim->stat_rate_support)
234 		mdev->limits.stat_rate_support = dev_lim->stat_rate_support;
235 	else if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
236 		mdev->limits.stat_rate_support = 0xf;
237 	else
238 		mdev->limits.stat_rate_support = 0x3;
239 
240 	/* IB_DEVICE_RESIZE_MAX_WR not supported by driver.
241 	   May be doable since hardware supports it for SRQ.
242 
243 	   IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver.
244 
245 	   IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not
246 	   supported by driver. */
247 	mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
248 		IB_DEVICE_PORT_ACTIVE_EVENT |
249 		IB_DEVICE_SYS_IMAGE_GUID |
250 		IB_DEVICE_RC_RNR_NAK_GEN;
251 
252 	if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR)
253 		mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
254 
255 	if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR)
256 		mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
257 
258 	if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI)
259 		mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI;
260 
261 	if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG)
262 		mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
263 
264 	if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE)
265 		mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
266 
267 	if (dev_lim->flags & DEV_LIM_FLAG_SRQ)
268 		mdev->mthca_flags |= MTHCA_FLAG_SRQ;
269 
270 	if (mthca_is_memfree(mdev))
271 		if (dev_lim->flags & DEV_LIM_FLAG_IPOIB_CSUM)
272 			mdev->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
273 
274 	return 0;
275 }
276 
277 static int mthca_init_tavor(struct mthca_dev *mdev)
278 {
279 	s64 size;
280 	u8 status;
281 	int err;
282 	struct mthca_dev_lim        dev_lim;
283 	struct mthca_profile        profile;
284 	struct mthca_init_hca_param init_hca;
285 
286 	err = mthca_SYS_EN(mdev, &status);
287 	if (err) {
288 		mthca_err(mdev, "SYS_EN command failed, aborting.\n");
289 		return err;
290 	}
291 	if (status) {
292 		mthca_err(mdev, "SYS_EN returned status 0x%02x, "
293 			  "aborting.\n", status);
294 		return -EINVAL;
295 	}
296 
297 	err = mthca_QUERY_FW(mdev, &status);
298 	if (err) {
299 		mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
300 		goto err_disable;
301 	}
302 	if (status) {
303 		mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
304 			  "aborting.\n", status);
305 		err = -EINVAL;
306 		goto err_disable;
307 	}
308 	err = mthca_QUERY_DDR(mdev, &status);
309 	if (err) {
310 		mthca_err(mdev, "QUERY_DDR command failed, aborting.\n");
311 		goto err_disable;
312 	}
313 	if (status) {
314 		mthca_err(mdev, "QUERY_DDR returned status 0x%02x, "
315 			  "aborting.\n", status);
316 		err = -EINVAL;
317 		goto err_disable;
318 	}
319 
320 	err = mthca_dev_lim(mdev, &dev_lim);
321 	if (err) {
322 		mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
323 		goto err_disable;
324 	}
325 
326 	profile = hca_profile;
327 	profile.num_uar   = dev_lim.uar_size / PAGE_SIZE;
328 	profile.uarc_size = 0;
329 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
330 		profile.num_srq = dev_lim.max_srqs;
331 
332 	size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
333 	if (size < 0) {
334 		err = size;
335 		goto err_disable;
336 	}
337 
338 	err = mthca_INIT_HCA(mdev, &init_hca, &status);
339 	if (err) {
340 		mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
341 		goto err_disable;
342 	}
343 	if (status) {
344 		mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
345 			  "aborting.\n", status);
346 		err = -EINVAL;
347 		goto err_disable;
348 	}
349 
350 	return 0;
351 
352 err_disable:
353 	mthca_SYS_DIS(mdev, &status);
354 
355 	return err;
356 }
357 
358 static int mthca_load_fw(struct mthca_dev *mdev)
359 {
360 	u8 status;
361 	int err;
362 
363 	/* FIXME: use HCA-attached memory for FW if present */
364 
365 	mdev->fw.arbel.fw_icm =
366 		mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages,
367 				GFP_HIGHUSER | __GFP_NOWARN, 0);
368 	if (!mdev->fw.arbel.fw_icm) {
369 		mthca_err(mdev, "Couldn't allocate FW area, aborting.\n");
370 		return -ENOMEM;
371 	}
372 
373 	err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status);
374 	if (err) {
375 		mthca_err(mdev, "MAP_FA command failed, aborting.\n");
376 		goto err_free;
377 	}
378 	if (status) {
379 		mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status);
380 		err = -EINVAL;
381 		goto err_free;
382 	}
383 	err = mthca_RUN_FW(mdev, &status);
384 	if (err) {
385 		mthca_err(mdev, "RUN_FW command failed, aborting.\n");
386 		goto err_unmap_fa;
387 	}
388 	if (status) {
389 		mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status);
390 		err = -EINVAL;
391 		goto err_unmap_fa;
392 	}
393 
394 	return 0;
395 
396 err_unmap_fa:
397 	mthca_UNMAP_FA(mdev, &status);
398 
399 err_free:
400 	mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0);
401 	return err;
402 }
403 
404 static int mthca_init_icm(struct mthca_dev *mdev,
405 			  struct mthca_dev_lim *dev_lim,
406 			  struct mthca_init_hca_param *init_hca,
407 			  u64 icm_size)
408 {
409 	u64 aux_pages;
410 	u8 status;
411 	int err;
412 
413 	err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status);
414 	if (err) {
415 		mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n");
416 		return err;
417 	}
418 	if (status) {
419 		mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, "
420 			  "aborting.\n", status);
421 		return -EINVAL;
422 	}
423 
424 	mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n",
425 		  (unsigned long long) icm_size >> 10,
426 		  (unsigned long long) aux_pages << 2);
427 
428 	mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages,
429 						 GFP_HIGHUSER | __GFP_NOWARN, 0);
430 	if (!mdev->fw.arbel.aux_icm) {
431 		mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n");
432 		return -ENOMEM;
433 	}
434 
435 	err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status);
436 	if (err) {
437 		mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n");
438 		goto err_free_aux;
439 	}
440 	if (status) {
441 		mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status);
442 		err = -EINVAL;
443 		goto err_free_aux;
444 	}
445 
446 	err = mthca_map_eq_icm(mdev, init_hca->eqc_base);
447 	if (err) {
448 		mthca_err(mdev, "Failed to map EQ context memory, aborting.\n");
449 		goto err_unmap_aux;
450 	}
451 
452 	/* CPU writes to non-reserved MTTs, while HCA might DMA to reserved mtts */
453 	mdev->limits.reserved_mtts = ALIGN(mdev->limits.reserved_mtts * MTHCA_MTT_SEG_SIZE,
454 					   dma_get_cache_alignment()) / MTHCA_MTT_SEG_SIZE;
455 
456 	mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
457 							 MTHCA_MTT_SEG_SIZE,
458 							 mdev->limits.num_mtt_segs,
459 							 mdev->limits.reserved_mtts,
460 							 1, 0);
461 	if (!mdev->mr_table.mtt_table) {
462 		mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
463 		err = -ENOMEM;
464 		goto err_unmap_eq;
465 	}
466 
467 	mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
468 							 dev_lim->mpt_entry_sz,
469 							 mdev->limits.num_mpts,
470 							 mdev->limits.reserved_mrws,
471 							 1, 1);
472 	if (!mdev->mr_table.mpt_table) {
473 		mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
474 		err = -ENOMEM;
475 		goto err_unmap_mtt;
476 	}
477 
478 	mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
479 							dev_lim->qpc_entry_sz,
480 							mdev->limits.num_qps,
481 							mdev->limits.reserved_qps,
482 							0, 0);
483 	if (!mdev->qp_table.qp_table) {
484 		mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
485 		err = -ENOMEM;
486 		goto err_unmap_mpt;
487 	}
488 
489 	mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
490 							 dev_lim->eqpc_entry_sz,
491 							 mdev->limits.num_qps,
492 							 mdev->limits.reserved_qps,
493 							 0, 0);
494 	if (!mdev->qp_table.eqp_table) {
495 		mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
496 		err = -ENOMEM;
497 		goto err_unmap_qp;
498 	}
499 
500 	mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base,
501 							 MTHCA_RDB_ENTRY_SIZE,
502 							 mdev->limits.num_qps <<
503 							 mdev->qp_table.rdb_shift, 0,
504 							 0, 0);
505 	if (!mdev->qp_table.rdb_table) {
506 		mthca_err(mdev, "Failed to map RDB context memory, aborting\n");
507 		err = -ENOMEM;
508 		goto err_unmap_eqp;
509 	}
510 
511        mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
512 						    dev_lim->cqc_entry_sz,
513 						    mdev->limits.num_cqs,
514 						    mdev->limits.reserved_cqs,
515 						    0, 0);
516 	if (!mdev->cq_table.table) {
517 		mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
518 		err = -ENOMEM;
519 		goto err_unmap_rdb;
520 	}
521 
522 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ) {
523 		mdev->srq_table.table =
524 			mthca_alloc_icm_table(mdev, init_hca->srqc_base,
525 					      dev_lim->srq_entry_sz,
526 					      mdev->limits.num_srqs,
527 					      mdev->limits.reserved_srqs,
528 					      0, 0);
529 		if (!mdev->srq_table.table) {
530 			mthca_err(mdev, "Failed to map SRQ context memory, "
531 				  "aborting.\n");
532 			err = -ENOMEM;
533 			goto err_unmap_cq;
534 		}
535 	}
536 
537 	/*
538 	 * It's not strictly required, but for simplicity just map the
539 	 * whole multicast group table now.  The table isn't very big
540 	 * and it's a lot easier than trying to track ref counts.
541 	 */
542 	mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
543 						      MTHCA_MGM_ENTRY_SIZE,
544 						      mdev->limits.num_mgms +
545 						      mdev->limits.num_amgms,
546 						      mdev->limits.num_mgms +
547 						      mdev->limits.num_amgms,
548 						      0, 0);
549 	if (!mdev->mcg_table.table) {
550 		mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
551 		err = -ENOMEM;
552 		goto err_unmap_srq;
553 	}
554 
555 	return 0;
556 
557 err_unmap_srq:
558 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
559 		mthca_free_icm_table(mdev, mdev->srq_table.table);
560 
561 err_unmap_cq:
562 	mthca_free_icm_table(mdev, mdev->cq_table.table);
563 
564 err_unmap_rdb:
565 	mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
566 
567 err_unmap_eqp:
568 	mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
569 
570 err_unmap_qp:
571 	mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
572 
573 err_unmap_mpt:
574 	mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
575 
576 err_unmap_mtt:
577 	mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
578 
579 err_unmap_eq:
580 	mthca_unmap_eq_icm(mdev);
581 
582 err_unmap_aux:
583 	mthca_UNMAP_ICM_AUX(mdev, &status);
584 
585 err_free_aux:
586 	mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0);
587 
588 	return err;
589 }
590 
591 static void mthca_free_icms(struct mthca_dev *mdev)
592 {
593 	u8 status;
594 
595 	mthca_free_icm_table(mdev, mdev->mcg_table.table);
596 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
597 		mthca_free_icm_table(mdev, mdev->srq_table.table);
598 	mthca_free_icm_table(mdev, mdev->cq_table.table);
599 	mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
600 	mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
601 	mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
602 	mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
603 	mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
604 	mthca_unmap_eq_icm(mdev);
605 
606 	mthca_UNMAP_ICM_AUX(mdev, &status);
607 	mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0);
608 }
609 
610 static int mthca_init_arbel(struct mthca_dev *mdev)
611 {
612 	struct mthca_dev_lim        dev_lim;
613 	struct mthca_profile        profile;
614 	struct mthca_init_hca_param init_hca;
615 	s64 icm_size;
616 	u8 status;
617 	int err;
618 
619 	err = mthca_QUERY_FW(mdev, &status);
620 	if (err) {
621 		mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
622 		return err;
623 	}
624 	if (status) {
625 		mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
626 			  "aborting.\n", status);
627 		return -EINVAL;
628 	}
629 
630 	err = mthca_ENABLE_LAM(mdev, &status);
631 	if (err) {
632 		mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n");
633 		return err;
634 	}
635 	if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) {
636 		mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n");
637 		mdev->mthca_flags |= MTHCA_FLAG_NO_LAM;
638 	} else if (status) {
639 		mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, "
640 			  "aborting.\n", status);
641 		return -EINVAL;
642 	}
643 
644 	err = mthca_load_fw(mdev);
645 	if (err) {
646 		mthca_err(mdev, "Failed to start FW, aborting.\n");
647 		goto err_disable;
648 	}
649 
650 	err = mthca_dev_lim(mdev, &dev_lim);
651 	if (err) {
652 		mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
653 		goto err_stop_fw;
654 	}
655 
656 	profile = hca_profile;
657 	profile.num_uar  = dev_lim.uar_size / PAGE_SIZE;
658 	profile.num_udav = 0;
659 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
660 		profile.num_srq = dev_lim.max_srqs;
661 
662 	icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
663 	if (icm_size < 0) {
664 		err = icm_size;
665 		goto err_stop_fw;
666 	}
667 
668 	err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size);
669 	if (err)
670 		goto err_stop_fw;
671 
672 	err = mthca_INIT_HCA(mdev, &init_hca, &status);
673 	if (err) {
674 		mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
675 		goto err_free_icm;
676 	}
677 	if (status) {
678 		mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
679 			  "aborting.\n", status);
680 		err = -EINVAL;
681 		goto err_free_icm;
682 	}
683 
684 	return 0;
685 
686 err_free_icm:
687 	mthca_free_icms(mdev);
688 
689 err_stop_fw:
690 	mthca_UNMAP_FA(mdev, &status);
691 	mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0);
692 
693 err_disable:
694 	if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
695 		mthca_DISABLE_LAM(mdev, &status);
696 
697 	return err;
698 }
699 
700 static void mthca_close_hca(struct mthca_dev *mdev)
701 {
702 	u8 status;
703 
704 	mthca_CLOSE_HCA(mdev, 0, &status);
705 
706 	if (mthca_is_memfree(mdev)) {
707 		mthca_free_icms(mdev);
708 
709 		mthca_UNMAP_FA(mdev, &status);
710 		mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0);
711 
712 		if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
713 			mthca_DISABLE_LAM(mdev, &status);
714 	} else
715 		mthca_SYS_DIS(mdev, &status);
716 }
717 
718 static int mthca_init_hca(struct mthca_dev *mdev)
719 {
720 	u8 status;
721 	int err;
722 	struct mthca_adapter adapter;
723 
724 	if (mthca_is_memfree(mdev))
725 		err = mthca_init_arbel(mdev);
726 	else
727 		err = mthca_init_tavor(mdev);
728 
729 	if (err)
730 		return err;
731 
732 	err = mthca_QUERY_ADAPTER(mdev, &adapter, &status);
733 	if (err) {
734 		mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n");
735 		goto err_close;
736 	}
737 	if (status) {
738 		mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, "
739 			  "aborting.\n", status);
740 		err = -EINVAL;
741 		goto err_close;
742 	}
743 
744 	mdev->eq_table.inta_pin = adapter.inta_pin;
745 	if (!mthca_is_memfree(mdev))
746 		mdev->rev_id = adapter.revision_id;
747 	memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id);
748 
749 	return 0;
750 
751 err_close:
752 	mthca_close_hca(mdev);
753 	return err;
754 }
755 
756 static int mthca_setup_hca(struct mthca_dev *dev)
757 {
758 	int err;
759 	u8 status;
760 
761 	MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock);
762 
763 	err = mthca_init_uar_table(dev);
764 	if (err) {
765 		mthca_err(dev, "Failed to initialize "
766 			  "user access region table, aborting.\n");
767 		return err;
768 	}
769 
770 	err = mthca_uar_alloc(dev, &dev->driver_uar);
771 	if (err) {
772 		mthca_err(dev, "Failed to allocate driver access region, "
773 			  "aborting.\n");
774 		goto err_uar_table_free;
775 	}
776 
777 	dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
778 	if (!dev->kar) {
779 		mthca_err(dev, "Couldn't map kernel access region, "
780 			  "aborting.\n");
781 		err = -ENOMEM;
782 		goto err_uar_free;
783 	}
784 
785 	err = mthca_init_pd_table(dev);
786 	if (err) {
787 		mthca_err(dev, "Failed to initialize "
788 			  "protection domain table, aborting.\n");
789 		goto err_kar_unmap;
790 	}
791 
792 	err = mthca_init_mr_table(dev);
793 	if (err) {
794 		mthca_err(dev, "Failed to initialize "
795 			  "memory region table, aborting.\n");
796 		goto err_pd_table_free;
797 	}
798 
799 	err = mthca_pd_alloc(dev, 1, &dev->driver_pd);
800 	if (err) {
801 		mthca_err(dev, "Failed to create driver PD, "
802 			  "aborting.\n");
803 		goto err_mr_table_free;
804 	}
805 
806 	err = mthca_init_eq_table(dev);
807 	if (err) {
808 		mthca_err(dev, "Failed to initialize "
809 			  "event queue table, aborting.\n");
810 		goto err_pd_free;
811 	}
812 
813 	err = mthca_cmd_use_events(dev);
814 	if (err) {
815 		mthca_err(dev, "Failed to switch to event-driven "
816 			  "firmware commands, aborting.\n");
817 		goto err_eq_table_free;
818 	}
819 
820 	err = mthca_NOP(dev, &status);
821 	if (err || status) {
822 		if (dev->mthca_flags & MTHCA_FLAG_MSI_X) {
823 			mthca_warn(dev, "NOP command failed to generate interrupt "
824 				   "(IRQ %d).\n",
825 				   dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector);
826 			mthca_warn(dev, "Trying again with MSI-X disabled.\n");
827 		} else {
828 			mthca_err(dev, "NOP command failed to generate interrupt "
829 				  "(IRQ %d), aborting.\n",
830 				  dev->pdev->irq);
831 			mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
832 		}
833 
834 		goto err_cmd_poll;
835 	}
836 
837 	mthca_dbg(dev, "NOP command IRQ test passed\n");
838 
839 	err = mthca_init_cq_table(dev);
840 	if (err) {
841 		mthca_err(dev, "Failed to initialize "
842 			  "completion queue table, aborting.\n");
843 		goto err_cmd_poll;
844 	}
845 
846 	err = mthca_init_srq_table(dev);
847 	if (err) {
848 		mthca_err(dev, "Failed to initialize "
849 			  "shared receive queue table, aborting.\n");
850 		goto err_cq_table_free;
851 	}
852 
853 	err = mthca_init_qp_table(dev);
854 	if (err) {
855 		mthca_err(dev, "Failed to initialize "
856 			  "queue pair table, aborting.\n");
857 		goto err_srq_table_free;
858 	}
859 
860 	err = mthca_init_av_table(dev);
861 	if (err) {
862 		mthca_err(dev, "Failed to initialize "
863 			  "address vector table, aborting.\n");
864 		goto err_qp_table_free;
865 	}
866 
867 	err = mthca_init_mcg_table(dev);
868 	if (err) {
869 		mthca_err(dev, "Failed to initialize "
870 			  "multicast group table, aborting.\n");
871 		goto err_av_table_free;
872 	}
873 
874 	return 0;
875 
876 err_av_table_free:
877 	mthca_cleanup_av_table(dev);
878 
879 err_qp_table_free:
880 	mthca_cleanup_qp_table(dev);
881 
882 err_srq_table_free:
883 	mthca_cleanup_srq_table(dev);
884 
885 err_cq_table_free:
886 	mthca_cleanup_cq_table(dev);
887 
888 err_cmd_poll:
889 	mthca_cmd_use_polling(dev);
890 
891 err_eq_table_free:
892 	mthca_cleanup_eq_table(dev);
893 
894 err_pd_free:
895 	mthca_pd_free(dev, &dev->driver_pd);
896 
897 err_mr_table_free:
898 	mthca_cleanup_mr_table(dev);
899 
900 err_pd_table_free:
901 	mthca_cleanup_pd_table(dev);
902 
903 err_kar_unmap:
904 	iounmap(dev->kar);
905 
906 err_uar_free:
907 	mthca_uar_free(dev, &dev->driver_uar);
908 
909 err_uar_table_free:
910 	mthca_cleanup_uar_table(dev);
911 	return err;
912 }
913 
914 static int mthca_request_regions(struct pci_dev *pdev, int ddr_hidden)
915 {
916 	int err;
917 
918 	/*
919 	 * We can't just use pci_request_regions() because the MSI-X
920 	 * table is right in the middle of the first BAR.  If we did
921 	 * pci_request_region and grab all of the first BAR, then
922 	 * setting up MSI-X would fail, since the PCI core wants to do
923 	 * request_mem_region on the MSI-X vector table.
924 	 *
925 	 * So just request what we need right now, and request any
926 	 * other regions we need when setting up EQs.
927 	 */
928 	if (!request_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
929 				MTHCA_HCR_SIZE, DRV_NAME))
930 		return -EBUSY;
931 
932 	err = pci_request_region(pdev, 2, DRV_NAME);
933 	if (err)
934 		goto err_bar2_failed;
935 
936 	if (!ddr_hidden) {
937 		err = pci_request_region(pdev, 4, DRV_NAME);
938 		if (err)
939 			goto err_bar4_failed;
940 	}
941 
942 	return 0;
943 
944 err_bar4_failed:
945 	pci_release_region(pdev, 2);
946 
947 err_bar2_failed:
948 	release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
949 			   MTHCA_HCR_SIZE);
950 
951 	return err;
952 }
953 
954 static void mthca_release_regions(struct pci_dev *pdev,
955 				  int ddr_hidden)
956 {
957 	if (!ddr_hidden)
958 		pci_release_region(pdev, 4);
959 
960 	pci_release_region(pdev, 2);
961 
962 	release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
963 			   MTHCA_HCR_SIZE);
964 }
965 
966 static int mthca_enable_msi_x(struct mthca_dev *mdev)
967 {
968 	struct msix_entry entries[3];
969 	int err;
970 
971 	entries[0].entry = 0;
972 	entries[1].entry = 1;
973 	entries[2].entry = 2;
974 
975 	err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries));
976 	if (err) {
977 		if (err > 0)
978 			mthca_info(mdev, "Only %d MSI-X vectors available, "
979 				   "not using MSI-X\n", err);
980 		return err;
981 	}
982 
983 	mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector;
984 	mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector;
985 	mdev->eq_table.eq[MTHCA_EQ_CMD  ].msi_x_vector = entries[2].vector;
986 
987 	return 0;
988 }
989 
990 /* Types of supported HCA */
991 enum {
992 	TAVOR,			/* MT23108                        */
993 	ARBEL_COMPAT,		/* MT25208 in Tavor compat mode   */
994 	ARBEL_NATIVE,		/* MT25208 with extended features */
995 	SINAI			/* MT25204 */
996 };
997 
998 #define MTHCA_FW_VER(major, minor, subminor) \
999 	(((u64) (major) << 32) | ((u64) (minor) << 16) | (u64) (subminor))
1000 
1001 static struct {
1002 	u64 latest_fw;
1003 	u32 flags;
1004 } mthca_hca_table[] = {
1005 	[TAVOR]        = { .latest_fw = MTHCA_FW_VER(3, 5, 0),
1006 			   .flags     = 0 },
1007 	[ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 8, 200),
1008 			   .flags     = MTHCA_FLAG_PCIE },
1009 	[ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 3, 0),
1010 			   .flags     = MTHCA_FLAG_MEMFREE |
1011 					MTHCA_FLAG_PCIE },
1012 	[SINAI]        = { .latest_fw = MTHCA_FW_VER(1, 2, 0),
1013 			   .flags     = MTHCA_FLAG_MEMFREE |
1014 					MTHCA_FLAG_PCIE    |
1015 					MTHCA_FLAG_SINAI_OPT }
1016 };
1017 
1018 static int __mthca_init_one(struct pci_dev *pdev, int hca_type)
1019 {
1020 	int ddr_hidden = 0;
1021 	int err;
1022 	struct mthca_dev *mdev;
1023 
1024 	printk(KERN_INFO PFX "Initializing %s\n",
1025 	       pci_name(pdev));
1026 
1027 	err = pci_enable_device(pdev);
1028 	if (err) {
1029 		dev_err(&pdev->dev, "Cannot enable PCI device, "
1030 			"aborting.\n");
1031 		return err;
1032 	}
1033 
1034 	/*
1035 	 * Check for BARs.  We expect 0: 1MB, 2: 8MB, 4: DDR (may not
1036 	 * be present)
1037 	 */
1038 	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
1039 	    pci_resource_len(pdev, 0) != 1 << 20) {
1040 		dev_err(&pdev->dev, "Missing DCS, aborting.\n");
1041 		err = -ENODEV;
1042 		goto err_disable_pdev;
1043 	}
1044 	if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
1045 		dev_err(&pdev->dev, "Missing UAR, aborting.\n");
1046 		err = -ENODEV;
1047 		goto err_disable_pdev;
1048 	}
1049 	if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM))
1050 		ddr_hidden = 1;
1051 
1052 	err = mthca_request_regions(pdev, ddr_hidden);
1053 	if (err) {
1054 		dev_err(&pdev->dev, "Cannot obtain PCI resources, "
1055 			"aborting.\n");
1056 		goto err_disable_pdev;
1057 	}
1058 
1059 	pci_set_master(pdev);
1060 
1061 	err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
1062 	if (err) {
1063 		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
1064 		err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
1065 		if (err) {
1066 			dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
1067 			goto err_free_res;
1068 		}
1069 	}
1070 	err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1071 	if (err) {
1072 		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
1073 			 "consistent PCI DMA mask.\n");
1074 		err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1075 		if (err) {
1076 			dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
1077 				"aborting.\n");
1078 			goto err_free_res;
1079 		}
1080 	}
1081 
1082 	mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev);
1083 	if (!mdev) {
1084 		dev_err(&pdev->dev, "Device struct alloc failed, "
1085 			"aborting.\n");
1086 		err = -ENOMEM;
1087 		goto err_free_res;
1088 	}
1089 
1090 	mdev->pdev = pdev;
1091 
1092 	mdev->mthca_flags = mthca_hca_table[hca_type].flags;
1093 	if (ddr_hidden)
1094 		mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;
1095 
1096 	/*
1097 	 * Now reset the HCA before we touch the PCI capabilities or
1098 	 * attempt a firmware command, since a boot ROM may have left
1099 	 * the HCA in an undefined state.
1100 	 */
1101 	err = mthca_reset(mdev);
1102 	if (err) {
1103 		mthca_err(mdev, "Failed to reset HCA, aborting.\n");
1104 		goto err_free_dev;
1105 	}
1106 
1107 	if (mthca_cmd_init(mdev)) {
1108 		mthca_err(mdev, "Failed to init command interface, aborting.\n");
1109 		goto err_free_dev;
1110 	}
1111 
1112 	err = mthca_tune_pci(mdev);
1113 	if (err)
1114 		goto err_cmd;
1115 
1116 	err = mthca_init_hca(mdev);
1117 	if (err)
1118 		goto err_cmd;
1119 
1120 	if (mdev->fw_ver < mthca_hca_table[hca_type].latest_fw) {
1121 		mthca_warn(mdev, "HCA FW version %d.%d.%03d is old (%d.%d.%03d is current).\n",
1122 			   (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff,
1123 			   (int) (mdev->fw_ver & 0xffff),
1124 			   (int) (mthca_hca_table[hca_type].latest_fw >> 32),
1125 			   (int) (mthca_hca_table[hca_type].latest_fw >> 16) & 0xffff,
1126 			   (int) (mthca_hca_table[hca_type].latest_fw & 0xffff));
1127 		mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n");
1128 	}
1129 
1130 	if (msi_x && !mthca_enable_msi_x(mdev))
1131 		mdev->mthca_flags |= MTHCA_FLAG_MSI_X;
1132 
1133 	err = mthca_setup_hca(mdev);
1134 	if (err == -EBUSY && (mdev->mthca_flags & MTHCA_FLAG_MSI_X)) {
1135 		if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1136 			pci_disable_msix(pdev);
1137 		mdev->mthca_flags &= ~MTHCA_FLAG_MSI_X;
1138 
1139 		err = mthca_setup_hca(mdev);
1140 	}
1141 
1142 	if (err)
1143 		goto err_close;
1144 
1145 	err = mthca_register_device(mdev);
1146 	if (err)
1147 		goto err_cleanup;
1148 
1149 	err = mthca_create_agents(mdev);
1150 	if (err)
1151 		goto err_unregister;
1152 
1153 	pci_set_drvdata(pdev, mdev);
1154 	mdev->hca_type = hca_type;
1155 
1156 	return 0;
1157 
1158 err_unregister:
1159 	mthca_unregister_device(mdev);
1160 
1161 err_cleanup:
1162 	mthca_cleanup_mcg_table(mdev);
1163 	mthca_cleanup_av_table(mdev);
1164 	mthca_cleanup_qp_table(mdev);
1165 	mthca_cleanup_srq_table(mdev);
1166 	mthca_cleanup_cq_table(mdev);
1167 	mthca_cmd_use_polling(mdev);
1168 	mthca_cleanup_eq_table(mdev);
1169 
1170 	mthca_pd_free(mdev, &mdev->driver_pd);
1171 
1172 	mthca_cleanup_mr_table(mdev);
1173 	mthca_cleanup_pd_table(mdev);
1174 	mthca_cleanup_uar_table(mdev);
1175 
1176 err_close:
1177 	if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1178 		pci_disable_msix(pdev);
1179 
1180 	mthca_close_hca(mdev);
1181 
1182 err_cmd:
1183 	mthca_cmd_cleanup(mdev);
1184 
1185 err_free_dev:
1186 	ib_dealloc_device(&mdev->ib_dev);
1187 
1188 err_free_res:
1189 	mthca_release_regions(pdev, ddr_hidden);
1190 
1191 err_disable_pdev:
1192 	pci_disable_device(pdev);
1193 	pci_set_drvdata(pdev, NULL);
1194 	return err;
1195 }
1196 
1197 static void __mthca_remove_one(struct pci_dev *pdev)
1198 {
1199 	struct mthca_dev *mdev = pci_get_drvdata(pdev);
1200 	u8 status;
1201 	int p;
1202 
1203 	if (mdev) {
1204 		mthca_free_agents(mdev);
1205 		mthca_unregister_device(mdev);
1206 
1207 		for (p = 1; p <= mdev->limits.num_ports; ++p)
1208 			mthca_CLOSE_IB(mdev, p, &status);
1209 
1210 		mthca_cleanup_mcg_table(mdev);
1211 		mthca_cleanup_av_table(mdev);
1212 		mthca_cleanup_qp_table(mdev);
1213 		mthca_cleanup_srq_table(mdev);
1214 		mthca_cleanup_cq_table(mdev);
1215 		mthca_cmd_use_polling(mdev);
1216 		mthca_cleanup_eq_table(mdev);
1217 
1218 		mthca_pd_free(mdev, &mdev->driver_pd);
1219 
1220 		mthca_cleanup_mr_table(mdev);
1221 		mthca_cleanup_pd_table(mdev);
1222 
1223 		iounmap(mdev->kar);
1224 		mthca_uar_free(mdev, &mdev->driver_uar);
1225 		mthca_cleanup_uar_table(mdev);
1226 		mthca_close_hca(mdev);
1227 		mthca_cmd_cleanup(mdev);
1228 
1229 		if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1230 			pci_disable_msix(pdev);
1231 
1232 		ib_dealloc_device(&mdev->ib_dev);
1233 		mthca_release_regions(pdev, mdev->mthca_flags &
1234 				      MTHCA_FLAG_DDR_HIDDEN);
1235 		pci_disable_device(pdev);
1236 		pci_set_drvdata(pdev, NULL);
1237 	}
1238 }
1239 
1240 int __mthca_restart_one(struct pci_dev *pdev)
1241 {
1242 	struct mthca_dev *mdev;
1243 	int hca_type;
1244 
1245 	mdev = pci_get_drvdata(pdev);
1246 	if (!mdev)
1247 		return -ENODEV;
1248 	hca_type = mdev->hca_type;
1249 	__mthca_remove_one(pdev);
1250 	return __mthca_init_one(pdev, hca_type);
1251 }
1252 
1253 static int __devinit mthca_init_one(struct pci_dev *pdev,
1254 				    const struct pci_device_id *id)
1255 {
1256 	static int mthca_version_printed = 0;
1257 	int ret;
1258 
1259 	mutex_lock(&mthca_device_mutex);
1260 
1261 	if (!mthca_version_printed) {
1262 		printk(KERN_INFO "%s", mthca_version);
1263 		++mthca_version_printed;
1264 	}
1265 
1266 	if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) {
1267 		printk(KERN_ERR PFX "%s has invalid driver data %lx\n",
1268 		       pci_name(pdev), id->driver_data);
1269 		mutex_unlock(&mthca_device_mutex);
1270 		return -ENODEV;
1271 	}
1272 
1273 	ret = __mthca_init_one(pdev, id->driver_data);
1274 
1275 	mutex_unlock(&mthca_device_mutex);
1276 
1277 	return ret;
1278 }
1279 
1280 static void __devexit mthca_remove_one(struct pci_dev *pdev)
1281 {
1282 	mutex_lock(&mthca_device_mutex);
1283 	__mthca_remove_one(pdev);
1284 	mutex_unlock(&mthca_device_mutex);
1285 }
1286 
1287 static struct pci_device_id mthca_pci_table[] = {
1288 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
1289 	  .driver_data = TAVOR },
1290 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR),
1291 	  .driver_data = TAVOR },
1292 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1293 	  .driver_data = ARBEL_COMPAT },
1294 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1295 	  .driver_data = ARBEL_COMPAT },
1296 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL),
1297 	  .driver_data = ARBEL_NATIVE },
1298 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL),
1299 	  .driver_data = ARBEL_NATIVE },
1300 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI),
1301 	  .driver_data = SINAI },
1302 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI),
1303 	  .driver_data = SINAI },
1304 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1305 	  .driver_data = SINAI },
1306 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1307 	  .driver_data = SINAI },
1308 	{ 0, }
1309 };
1310 
1311 MODULE_DEVICE_TABLE(pci, mthca_pci_table);
1312 
1313 static struct pci_driver mthca_driver = {
1314 	.name		= DRV_NAME,
1315 	.id_table	= mthca_pci_table,
1316 	.probe		= mthca_init_one,
1317 	.remove		= __devexit_p(mthca_remove_one)
1318 };
1319 
1320 static void __init __mthca_check_profile_val(const char *name, int *pval,
1321 					     int pval_default)
1322 {
1323 	/* value must be positive and power of 2 */
1324 	int old_pval = *pval;
1325 
1326 	if (old_pval <= 0)
1327 		*pval = pval_default;
1328 	else
1329 		*pval = roundup_pow_of_two(old_pval);
1330 
1331 	if (old_pval != *pval) {
1332 		printk(KERN_WARNING PFX "Invalid value %d for %s in module parameter.\n",
1333 		       old_pval, name);
1334 		printk(KERN_WARNING PFX "Corrected %s to %d.\n", name, *pval);
1335 	}
1336 }
1337 
1338 #define mthca_check_profile_val(name, default)				\
1339 	__mthca_check_profile_val(#name, &hca_profile.name, default)
1340 
1341 static void __init mthca_validate_profile(void)
1342 {
1343 	mthca_check_profile_val(num_qp,            MTHCA_DEFAULT_NUM_QP);
1344 	mthca_check_profile_val(rdb_per_qp,        MTHCA_DEFAULT_RDB_PER_QP);
1345 	mthca_check_profile_val(num_cq,            MTHCA_DEFAULT_NUM_CQ);
1346 	mthca_check_profile_val(num_mcg, 	   MTHCA_DEFAULT_NUM_MCG);
1347 	mthca_check_profile_val(num_mpt, 	   MTHCA_DEFAULT_NUM_MPT);
1348 	mthca_check_profile_val(num_mtt, 	   MTHCA_DEFAULT_NUM_MTT);
1349 	mthca_check_profile_val(num_udav,          MTHCA_DEFAULT_NUM_UDAV);
1350 	mthca_check_profile_val(fmr_reserved_mtts, MTHCA_DEFAULT_NUM_RESERVED_MTTS);
1351 
1352 	if (hca_profile.fmr_reserved_mtts >= hca_profile.num_mtt) {
1353 		printk(KERN_WARNING PFX "Invalid fmr_reserved_mtts module parameter %d.\n",
1354 		       hca_profile.fmr_reserved_mtts);
1355 		printk(KERN_WARNING PFX "(Must be smaller than num_mtt %d)\n",
1356 		       hca_profile.num_mtt);
1357 		hca_profile.fmr_reserved_mtts = hca_profile.num_mtt / 2;
1358 		printk(KERN_WARNING PFX "Corrected fmr_reserved_mtts to %d.\n",
1359 		       hca_profile.fmr_reserved_mtts);
1360 	}
1361 }
1362 
1363 static int __init mthca_init(void)
1364 {
1365 	int ret;
1366 
1367 	mthca_validate_profile();
1368 
1369 	ret = mthca_catas_init();
1370 	if (ret)
1371 		return ret;
1372 
1373 	ret = pci_register_driver(&mthca_driver);
1374 	if (ret < 0) {
1375 		mthca_catas_cleanup();
1376 		return ret;
1377 	}
1378 
1379 	return 0;
1380 }
1381 
1382 static void __exit mthca_cleanup(void)
1383 {
1384 	pci_unregister_driver(&mthca_driver);
1385 	mthca_catas_cleanup();
1386 }
1387 
1388 module_init(mthca_init);
1389 module_exit(mthca_cleanup);
1390