1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
4  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
5  */
6 
7 #include "efct_driver.h"
8 
9 #include "efct_hw.h"
10 #include "efct_unsol.h"
11 #include "efct_scsi.h"
12 
13 LIST_HEAD(efct_devices);
14 
15 static int logmask;
16 module_param(logmask, int, 0444);
17 MODULE_PARM_DESC(logmask, "logging bitmask (default 0)");
18 
19 static struct libefc_function_template efct_libefc_templ = {
20 	.issue_mbox_rqst = efct_issue_mbox_rqst,
21 	.send_els = efct_els_hw_srrs_send,
22 	.send_bls = efct_efc_bls_send,
23 
24 	.new_nport = efct_scsi_tgt_new_nport,
25 	.del_nport = efct_scsi_tgt_del_nport,
26 	.scsi_new_node = efct_scsi_new_initiator,
27 	.scsi_del_node = efct_scsi_del_initiator,
28 	.hw_seq_free = efct_efc_hw_sequence_free,
29 };
30 
31 static int
32 efct_device_init(void)
33 {
34 	int rc;
35 
36 	/* driver-wide init for target-server */
37 	rc = efct_scsi_tgt_driver_init();
38 	if (rc) {
39 		pr_err("efct_scsi_tgt_init failed rc=%d\n", rc);
40 		return rc;
41 	}
42 
43 	rc = efct_scsi_reg_fc_transport();
44 	if (rc) {
45 		pr_err("failed to register to FC host\n");
46 		return rc;
47 	}
48 
49 	return 0;
50 }
51 
52 static void
53 efct_device_shutdown(void)
54 {
55 	efct_scsi_release_fc_transport();
56 
57 	efct_scsi_tgt_driver_exit();
58 }
59 
60 static void *
61 efct_device_alloc(u32 nid)
62 {
63 	struct efct *efct = NULL;
64 
65 	efct = kzalloc_node(sizeof(*efct), GFP_KERNEL, nid);
66 	if (!efct)
67 		return efct;
68 
69 	INIT_LIST_HEAD(&efct->list_entry);
70 	list_add_tail(&efct->list_entry, &efct_devices);
71 
72 	return efct;
73 }
74 
75 static void
76 efct_teardown_msix(struct efct *efct)
77 {
78 	u32 i;
79 
80 	for (i = 0; i < efct->n_msix_vec; i++) {
81 		free_irq(pci_irq_vector(efct->pci, i),
82 			 &efct->intr_context[i]);
83 	}
84 
85 	pci_free_irq_vectors(efct->pci);
86 }
87 
88 static int
89 efct_efclib_config(struct efct *efct, struct libefc_function_template *tt)
90 {
91 	struct efc *efc;
92 	struct sli4 *sli;
93 	int rc = 0;
94 
95 	efc = kzalloc(sizeof(*efc), GFP_KERNEL);
96 	if (!efc)
97 		return -ENOMEM;
98 
99 	efct->efcport = efc;
100 
101 	memcpy(&efc->tt, tt, sizeof(*tt));
102 	efc->base = efct;
103 	efc->pci = efct->pci;
104 
105 	efc->def_wwnn = efct_get_wwnn(&efct->hw);
106 	efc->def_wwpn = efct_get_wwpn(&efct->hw);
107 	efc->enable_tgt = 1;
108 	efc->log_level = EFC_LOG_LIB;
109 
110 	sli = &efct->hw.sli;
111 	efc->max_xfer_size = sli->sge_supported_length *
112 			     sli_get_max_sgl(&efct->hw.sli);
113 	efc->sli = sli;
114 	efc->fcfi = efct->hw.fcf_indicator;
115 
116 	rc = efcport_init(efc);
117 	if (rc)
118 		efc_log_err(efc, "efcport_init failed\n");
119 
120 	return rc;
121 }
122 
123 static int efct_request_firmware_update(struct efct *efct);
124 
125 static const char*
126 efct_pci_model(u16 device)
127 {
128 	switch (device) {
129 	case EFCT_DEVICE_LANCER_G6:	return "LPE31004";
130 	case EFCT_DEVICE_LANCER_G7:	return "LPE36000";
131 	default:			return "unknown";
132 	}
133 }
134 
135 static int
136 efct_device_attach(struct efct *efct)
137 {
138 	u32 rc = 0, i = 0;
139 
140 	if (efct->attached) {
141 		efc_log_err(efct, "Device is already attached\n");
142 		return -EIO;
143 	}
144 
145 	snprintf(efct->name, sizeof(efct->name), "[%s%d] ", "fc",
146 		 efct->instance_index);
147 
148 	efct->logmask = logmask;
149 	efct->filter_def = EFCT_DEFAULT_FILTER;
150 	efct->max_isr_time_msec = EFCT_OS_MAX_ISR_TIME_MSEC;
151 
152 	efct->model = efct_pci_model(efct->pci->device);
153 
154 	efct->efct_req_fw_upgrade = true;
155 
156 	/* Allocate transport object and bring online */
157 	efct->xport = efct_xport_alloc(efct);
158 	if (!efct->xport) {
159 		efc_log_err(efct, "failed to allocate transport object\n");
160 		rc = -ENOMEM;
161 		goto out;
162 	}
163 
164 	rc = efct_xport_attach(efct->xport);
165 	if (rc) {
166 		efc_log_err(efct, "failed to attach transport object\n");
167 		goto xport_out;
168 	}
169 
170 	rc = efct_xport_initialize(efct->xport);
171 	if (rc) {
172 		efc_log_err(efct, "failed to initialize transport object\n");
173 		goto xport_out;
174 	}
175 
176 	rc = efct_efclib_config(efct, &efct_libefc_templ);
177 	if (rc) {
178 		efc_log_err(efct, "failed to init efclib\n");
179 		goto efclib_out;
180 	}
181 
182 	for (i = 0; i < efct->n_msix_vec; i++) {
183 		efc_log_debug(efct, "irq %d enabled\n", i);
184 		enable_irq(pci_irq_vector(efct->pci, i));
185 	}
186 
187 	efct->attached = true;
188 
189 	if (efct->efct_req_fw_upgrade)
190 		efct_request_firmware_update(efct);
191 
192 	return rc;
193 
194 efclib_out:
195 	efct_xport_detach(efct->xport);
196 xport_out:
197 	efct_xport_free(efct->xport);
198 	efct->xport = NULL;
199 out:
200 	return rc;
201 }
202 
203 static int
204 efct_device_detach(struct efct *efct)
205 {
206 	int i;
207 
208 	if (!efct || !efct->attached) {
209 		pr_err("Device is not attached\n");
210 		return -EIO;
211 	}
212 
213 	if (efct_xport_control(efct->xport, EFCT_XPORT_SHUTDOWN))
214 		efc_log_err(efct, "Transport Shutdown timed out\n");
215 
216 	for (i = 0; i < efct->n_msix_vec; i++)
217 		disable_irq(pci_irq_vector(efct->pci, i));
218 
219 	efct_xport_detach(efct->xport);
220 
221 	efct_xport_free(efct->xport);
222 	efct->xport = NULL;
223 
224 	efcport_destroy(efct->efcport);
225 	kfree(efct->efcport);
226 
227 	efct->attached = false;
228 
229 	return 0;
230 }
231 
232 static void
233 efct_fw_write_cb(int status, u32 actual_write_length,
234 		 u32 change_status, void *arg)
235 {
236 	struct efct_fw_write_result *result = arg;
237 
238 	result->status = status;
239 	result->actual_xfer = actual_write_length;
240 	result->change_status = change_status;
241 
242 	complete(&result->done);
243 }
244 
245 static int
246 efct_firmware_write(struct efct *efct, const u8 *buf, size_t buf_len,
247 		    u8 *change_status)
248 {
249 	int rc = 0;
250 	u32 bytes_left;
251 	u32 xfer_size;
252 	u32 offset;
253 	struct efc_dma dma;
254 	int last = 0;
255 	struct efct_fw_write_result result;
256 
257 	init_completion(&result.done);
258 
259 	bytes_left = buf_len;
260 	offset = 0;
261 
262 	dma.size = FW_WRITE_BUFSIZE;
263 	dma.virt = dma_alloc_coherent(&efct->pci->dev,
264 				      dma.size, &dma.phys, GFP_DMA);
265 	if (!dma.virt)
266 		return -ENOMEM;
267 
268 	while (bytes_left > 0) {
269 		if (bytes_left > FW_WRITE_BUFSIZE)
270 			xfer_size = FW_WRITE_BUFSIZE;
271 		else
272 			xfer_size = bytes_left;
273 
274 		memcpy(dma.virt, buf + offset, xfer_size);
275 
276 		if (bytes_left == xfer_size)
277 			last = 1;
278 
279 		efct_hw_firmware_write(&efct->hw, &dma, xfer_size, offset,
280 				       last, efct_fw_write_cb, &result);
281 
282 		if (wait_for_completion_interruptible(&result.done) != 0) {
283 			rc = -ENXIO;
284 			break;
285 		}
286 
287 		if (result.actual_xfer == 0 || result.status != 0) {
288 			rc = -EFAULT;
289 			break;
290 		}
291 
292 		if (last)
293 			*change_status = result.change_status;
294 
295 		bytes_left -= result.actual_xfer;
296 		offset += result.actual_xfer;
297 	}
298 
299 	dma_free_coherent(&efct->pci->dev, dma.size, dma.virt, dma.phys);
300 	return rc;
301 }
302 
303 static int
304 efct_fw_reset(struct efct *efct)
305 {
306 	/*
307 	 * Firmware reset to activate the new firmware.
308 	 * Function 0 will update and load the new firmware
309 	 * during attach.
310 	 */
311 	if (timer_pending(&efct->xport->stats_timer))
312 		del_timer(&efct->xport->stats_timer);
313 
314 	if (efct_hw_reset(&efct->hw, EFCT_HW_RESET_FIRMWARE)) {
315 		efc_log_info(efct, "failed to reset firmware\n");
316 		return -EIO;
317 	}
318 
319 	efc_log_info(efct, "successfully reset firmware.Now resetting port\n");
320 
321 	efct_device_detach(efct);
322 	return efct_device_attach(efct);
323 }
324 
325 static int
326 efct_request_firmware_update(struct efct *efct)
327 {
328 	int rc = 0;
329 	u8 file_name[256], fw_change_status = 0;
330 	const struct firmware *fw;
331 	struct efct_hw_grp_hdr *fw_image;
332 
333 	snprintf(file_name, 256, "%s.grp", efct->model);
334 
335 	rc = request_firmware(&fw, file_name, &efct->pci->dev);
336 	if (rc) {
337 		efc_log_debug(efct, "Firmware file(%s) not found.\n", file_name);
338 		return rc;
339 	}
340 
341 	fw_image = (struct efct_hw_grp_hdr *)fw->data;
342 
343 	if (!strncmp(efct->hw.sli.fw_name[0], fw_image->revision,
344 		     strnlen(fw_image->revision, 16))) {
345 		efc_log_debug(efct,
346 			      "Skip update. Firmware is already up to date.\n");
347 		goto exit;
348 	}
349 
350 	efc_log_info(efct, "Firmware update is initiated. %s -> %s\n",
351 		     efct->hw.sli.fw_name[0], fw_image->revision);
352 
353 	rc = efct_firmware_write(efct, fw->data, fw->size, &fw_change_status);
354 	if (rc) {
355 		efc_log_err(efct, "Firmware update failed. rc = %d\n", rc);
356 		goto exit;
357 	}
358 
359 	efc_log_info(efct, "Firmware updated successfully\n");
360 	switch (fw_change_status) {
361 	case 0x00:
362 		efc_log_info(efct, "New firmware is active.\n");
363 		break;
364 	case 0x01:
365 		efc_log_info(efct,
366 			"System reboot needed to activate the new firmware\n");
367 		break;
368 	case 0x02:
369 	case 0x03:
370 		efc_log_info(efct,
371 			     "firmware reset to activate the new firmware\n");
372 		efct_fw_reset(efct);
373 		break;
374 	default:
375 		efc_log_info(efct, "Unexpected value change_status:%d\n",
376 			     fw_change_status);
377 		break;
378 	}
379 
380 exit:
381 	release_firmware(fw);
382 
383 	return rc;
384 }
385 
386 static void
387 efct_device_free(struct efct *efct)
388 {
389 	if (efct) {
390 		list_del(&efct->list_entry);
391 		kfree(efct);
392 	}
393 }
394 
395 static int
396 efct_device_interrupts_required(struct efct *efct)
397 {
398 	int rc;
399 
400 	rc = efct_hw_setup(&efct->hw, efct, efct->pci);
401 	if (rc < 0)
402 		return rc;
403 
404 	return efct->hw.config.n_eq;
405 }
406 
407 static irqreturn_t
408 efct_intr_thread(int irq, void *handle)
409 {
410 	struct efct_intr_context *intr_ctx = handle;
411 	struct efct *efct = intr_ctx->efct;
412 
413 	efct_hw_process(&efct->hw, intr_ctx->index, efct->max_isr_time_msec);
414 	return IRQ_HANDLED;
415 }
416 
417 static irqreturn_t
418 efct_intr_msix(int irq, void *handle)
419 {
420 	return IRQ_WAKE_THREAD;
421 }
422 
423 static int
424 efct_setup_msix(struct efct *efct, u32 num_intrs)
425 {
426 	int rc = 0, i;
427 
428 	if (!pci_find_capability(efct->pci, PCI_CAP_ID_MSIX)) {
429 		dev_err(&efct->pci->dev,
430 			"%s : MSI-X not available\n", __func__);
431 		return -EIO;
432 	}
433 
434 	efct->n_msix_vec = num_intrs;
435 
436 	rc = pci_alloc_irq_vectors(efct->pci, num_intrs, num_intrs,
437 				   PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
438 
439 	if (rc < 0) {
440 		dev_err(&efct->pci->dev, "Failed to alloc irq : %d\n", rc);
441 		return rc;
442 	}
443 
444 	for (i = 0; i < num_intrs; i++) {
445 		struct efct_intr_context *intr_ctx = NULL;
446 
447 		intr_ctx = &efct->intr_context[i];
448 		intr_ctx->efct = efct;
449 		intr_ctx->index = i;
450 
451 		rc = request_threaded_irq(pci_irq_vector(efct->pci, i),
452 					  efct_intr_msix, efct_intr_thread, 0,
453 					  EFCT_DRIVER_NAME, intr_ctx);
454 		if (rc) {
455 			dev_err(&efct->pci->dev,
456 				"Failed to register %d vector: %d\n", i, rc);
457 			goto out;
458 		}
459 	}
460 
461 	return rc;
462 
463 out:
464 	while (--i >= 0)
465 		free_irq(pci_irq_vector(efct->pci, i),
466 			 &efct->intr_context[i]);
467 
468 	pci_free_irq_vectors(efct->pci);
469 	return rc;
470 }
471 
472 static struct pci_device_id efct_pci_table[] = {
473 	{PCI_DEVICE(EFCT_VENDOR_ID, EFCT_DEVICE_LANCER_G6), 0},
474 	{PCI_DEVICE(EFCT_VENDOR_ID, EFCT_DEVICE_LANCER_G7), 0},
475 	{}	/* terminate list */
476 };
477 
478 static int
479 efct_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
480 {
481 	struct efct *efct = NULL;
482 	int rc;
483 	u32 i, r;
484 	int num_interrupts = 0;
485 	int nid;
486 
487 	dev_info(&pdev->dev, "%s\n", EFCT_DRIVER_NAME);
488 
489 	rc = pci_enable_device_mem(pdev);
490 	if (rc)
491 		return rc;
492 
493 	pci_set_master(pdev);
494 
495 	rc = pci_set_mwi(pdev);
496 	if (rc) {
497 		dev_info(&pdev->dev, "pci_set_mwi returned %d\n", rc);
498 		goto mwi_out;
499 	}
500 
501 	rc = pci_request_regions(pdev, EFCT_DRIVER_NAME);
502 	if (rc) {
503 		dev_err(&pdev->dev, "pci_request_regions failed %d\n", rc);
504 		goto req_regions_out;
505 	}
506 
507 	/* Fetch the Numa node id for this device */
508 	nid = dev_to_node(&pdev->dev);
509 	if (nid < 0) {
510 		dev_err(&pdev->dev, "Warning Numa node ID is %d\n", nid);
511 		nid = 0;
512 	}
513 
514 	/* Allocate efct */
515 	efct = efct_device_alloc(nid);
516 	if (!efct) {
517 		dev_err(&pdev->dev, "Failed to allocate efct\n");
518 		rc = -ENOMEM;
519 		goto alloc_out;
520 	}
521 
522 	efct->pci = pdev;
523 	efct->numa_node = nid;
524 
525 	/* Map all memory BARs */
526 	for (i = 0, r = 0; i < EFCT_PCI_MAX_REGS; i++) {
527 		if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
528 			efct->reg[r] = ioremap(pci_resource_start(pdev, i),
529 					       pci_resource_len(pdev, i));
530 			r++;
531 		}
532 
533 		/*
534 		 * If the 64-bit attribute is set, both this BAR and the
535 		 * next form the complete address. Skip processing the
536 		 * next BAR.
537 		 */
538 		if (pci_resource_flags(pdev, i) & IORESOURCE_MEM_64)
539 			i++;
540 	}
541 
542 	pci_set_drvdata(pdev, efct);
543 
544 	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) != 0 ||
545 	    pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
546 		dev_warn(&pdev->dev, "trying DMA_BIT_MASK(32)\n");
547 		if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0 ||
548 		    pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
549 			dev_err(&pdev->dev, "setting DMA_BIT_MASK failed\n");
550 			rc = -1;
551 			goto dma_mask_out;
552 		}
553 	}
554 
555 	num_interrupts = efct_device_interrupts_required(efct);
556 	if (num_interrupts < 0) {
557 		efc_log_err(efct, "efct_device_interrupts_required failed\n");
558 		rc = -1;
559 		goto dma_mask_out;
560 	}
561 
562 	/*
563 	 * Initialize MSIX interrupts, note,
564 	 * efct_setup_msix() enables the interrupt
565 	 */
566 	rc = efct_setup_msix(efct, num_interrupts);
567 	if (rc) {
568 		dev_err(&pdev->dev, "Can't setup msix\n");
569 		goto dma_mask_out;
570 	}
571 	/* Disable interrupt for now */
572 	for (i = 0; i < efct->n_msix_vec; i++) {
573 		efc_log_debug(efct, "irq %d disabled\n", i);
574 		disable_irq(pci_irq_vector(efct->pci, i));
575 	}
576 
577 	rc = efct_device_attach(efct);
578 	if (rc)
579 		goto attach_out;
580 
581 	return 0;
582 
583 attach_out:
584 	efct_teardown_msix(efct);
585 dma_mask_out:
586 	pci_set_drvdata(pdev, NULL);
587 
588 	for (i = 0; i < EFCT_PCI_MAX_REGS; i++) {
589 		if (efct->reg[i])
590 			iounmap(efct->reg[i]);
591 	}
592 	efct_device_free(efct);
593 alloc_out:
594 	pci_release_regions(pdev);
595 req_regions_out:
596 	pci_clear_mwi(pdev);
597 mwi_out:
598 	pci_disable_device(pdev);
599 	return rc;
600 }
601 
602 static void
603 efct_pci_remove(struct pci_dev *pdev)
604 {
605 	struct efct *efct = pci_get_drvdata(pdev);
606 	u32 i;
607 
608 	if (!efct)
609 		return;
610 
611 	efct_device_detach(efct);
612 
613 	efct_teardown_msix(efct);
614 
615 	for (i = 0; i < EFCT_PCI_MAX_REGS; i++) {
616 		if (efct->reg[i])
617 			iounmap(efct->reg[i]);
618 	}
619 
620 	pci_set_drvdata(pdev, NULL);
621 
622 	efct_device_free(efct);
623 
624 	pci_release_regions(pdev);
625 
626 	pci_disable_device(pdev);
627 }
628 
629 static void
630 efct_device_prep_for_reset(struct efct *efct, struct pci_dev *pdev)
631 {
632 	if (efct) {
633 		efc_log_debug(efct,
634 			      "PCI channel disable preparing for reset\n");
635 		efct_device_detach(efct);
636 		/* Disable interrupt and pci device */
637 		efct_teardown_msix(efct);
638 	}
639 	pci_disable_device(pdev);
640 }
641 
642 static void
643 efct_device_prep_for_recover(struct efct *efct)
644 {
645 	if (efct) {
646 		efc_log_debug(efct, "PCI channel preparing for recovery\n");
647 		efct_hw_io_abort_all(&efct->hw);
648 	}
649 }
650 
651 /**
652  * efct_pci_io_error_detected - method for handling PCI I/O error
653  * @pdev: pointer to PCI device.
654  * @state: the current PCI connection state.
655  *
656  * This routine is registered to the PCI subsystem for error handling. This
657  * function is called by the PCI subsystem after a PCI bus error affecting
658  * this device has been detected. When this routine is invoked, it dispatches
659  * device error detected handling routine, which will perform the proper
660  * error detected operation.
661  *
662  * Return codes
663  * PCI_ERS_RESULT_NEED_RESET - need to reset before recovery
664  * PCI_ERS_RESULT_DISCONNECT - device could not be recovered
665  */
666 static pci_ers_result_t
667 efct_pci_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
668 {
669 	struct efct *efct = pci_get_drvdata(pdev);
670 	pci_ers_result_t rc;
671 
672 	switch (state) {
673 	case pci_channel_io_normal:
674 		efct_device_prep_for_recover(efct);
675 		rc = PCI_ERS_RESULT_CAN_RECOVER;
676 		break;
677 	case pci_channel_io_frozen:
678 		efct_device_prep_for_reset(efct, pdev);
679 		rc = PCI_ERS_RESULT_NEED_RESET;
680 		break;
681 	case pci_channel_io_perm_failure:
682 		efct_device_detach(efct);
683 		rc = PCI_ERS_RESULT_DISCONNECT;
684 		break;
685 	default:
686 		efc_log_debug(efct, "Unknown PCI error state:0x%x\n", state);
687 		efct_device_prep_for_reset(efct, pdev);
688 		rc = PCI_ERS_RESULT_NEED_RESET;
689 		break;
690 	}
691 
692 	return rc;
693 }
694 
695 static pci_ers_result_t
696 efct_pci_io_slot_reset(struct pci_dev *pdev)
697 {
698 	int rc;
699 	struct efct *efct = pci_get_drvdata(pdev);
700 
701 	rc = pci_enable_device_mem(pdev);
702 	if (rc) {
703 		efc_log_err(efct, "failed to enable PCI device after reset\n");
704 		return PCI_ERS_RESULT_DISCONNECT;
705 	}
706 
707 	/*
708 	 * As the new kernel behavior of pci_restore_state() API call clears
709 	 * device saved_state flag, need to save the restored state again.
710 	 */
711 
712 	pci_save_state(pdev);
713 
714 	pci_set_master(pdev);
715 
716 	rc = efct_setup_msix(efct, efct->n_msix_vec);
717 	if (rc)
718 		efc_log_err(efct, "rc %d returned, IRQ allocation failed\n",
719 			    rc);
720 
721 	/* Perform device reset */
722 	efct_device_detach(efct);
723 	/* Bring device to online*/
724 	efct_device_attach(efct);
725 
726 	return PCI_ERS_RESULT_RECOVERED;
727 }
728 
729 static void
730 efct_pci_io_resume(struct pci_dev *pdev)
731 {
732 	struct efct *efct = pci_get_drvdata(pdev);
733 
734 	/* Perform device reset */
735 	efct_device_detach(efct);
736 	/* Bring device to online*/
737 	efct_device_attach(efct);
738 }
739 
740 MODULE_DEVICE_TABLE(pci, efct_pci_table);
741 
742 static struct pci_error_handlers efct_pci_err_handler = {
743 	.error_detected = efct_pci_io_error_detected,
744 	.slot_reset = efct_pci_io_slot_reset,
745 	.resume = efct_pci_io_resume,
746 };
747 
748 static struct pci_driver efct_pci_driver = {
749 	.name		= EFCT_DRIVER_NAME,
750 	.id_table	= efct_pci_table,
751 	.probe		= efct_pci_probe,
752 	.remove		= efct_pci_remove,
753 	.err_handler	= &efct_pci_err_handler,
754 };
755 
756 static
757 int __init efct_init(void)
758 {
759 	int rc;
760 
761 	rc = efct_device_init();
762 	if (rc) {
763 		pr_err("efct_device_init failed rc=%d\n", rc);
764 		return rc;
765 	}
766 
767 	rc = pci_register_driver(&efct_pci_driver);
768 	if (rc) {
769 		pr_err("pci_register_driver failed rc=%d\n", rc);
770 		efct_device_shutdown();
771 	}
772 
773 	return rc;
774 }
775 
776 static void __exit efct_exit(void)
777 {
778 	pci_unregister_driver(&efct_pci_driver);
779 	efct_device_shutdown();
780 }
781 
782 module_init(efct_init);
783 module_exit(efct_exit);
784 MODULE_VERSION(EFCT_DRIVER_VERSION);
785 MODULE_LICENSE("GPL");
786 MODULE_AUTHOR("Broadcom");
787