1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2016 Cavium, Inc.
4  */
5 
6 #include <linux/device.h>
7 #include <linux/firmware.h>
8 #include <linux/interrupt.h>
9 #include <linux/module.h>
10 #include <linux/moduleparam.h>
11 #include <linux/pci.h>
12 #include <linux/printk.h>
13 #include <linux/version.h>
14 
15 #include "cptpf.h"
16 
17 #define DRV_NAME	"thunder-cpt"
18 #define DRV_VERSION	"1.0"
19 
20 static u32 num_vfs = 4; /* Default 4 VF enabled */
21 module_param(num_vfs, uint, 0444);
22 MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)");
23 
24 /*
25  * Disable cores specified by coremask
26  */
27 static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask,
28 			      u8 type, u8 grp)
29 {
30 	u64 pf_exe_ctl;
31 	u32 timeout = 100;
32 	u64 grpmask = 0;
33 	struct device *dev = &cpt->pdev->dev;
34 
35 	if (type == AE_TYPES)
36 		coremask = (coremask << cpt->max_se_cores);
37 
38 	/* Disengage the cores from groups */
39 	grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
40 	cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
41 			(grpmask & ~coremask));
42 	udelay(CSR_DELAY);
43 	grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
44 	while (grp & coremask) {
45 		dev_err(dev, "Cores still busy %llx", coremask);
46 		grp = cpt_read_csr64(cpt->reg_base,
47 				     CPTX_PF_EXEC_BUSY(0));
48 		if (timeout--)
49 			break;
50 
51 		udelay(CSR_DELAY);
52 	}
53 
54 	/* Disable the cores */
55 	pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
56 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
57 			(pf_exe_ctl & ~coremask));
58 	udelay(CSR_DELAY);
59 }
60 
61 /*
62  * Enable cores specified by coremask
63  */
64 static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask,
65 			     u8 type)
66 {
67 	u64 pf_exe_ctl;
68 
69 	if (type == AE_TYPES)
70 		coremask = (coremask << cpt->max_se_cores);
71 
72 	pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
73 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
74 			(pf_exe_ctl | coremask));
75 	udelay(CSR_DELAY);
76 }
77 
78 static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
79 				u64 coremask, u8 type)
80 {
81 	u64 pf_gx_en = 0;
82 
83 	if (type == AE_TYPES)
84 		coremask = (coremask << cpt->max_se_cores);
85 
86 	pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
87 	cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
88 			(pf_gx_en | coremask));
89 	udelay(CSR_DELAY);
90 }
91 
92 static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
93 {
94 	/* Clear mbox(0) interupts for all vfs */
95 	cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
96 }
97 
98 static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
99 {
100 	/* Clear ecc(0) interupts for all vfs */
101 	cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
102 }
103 
104 static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
105 {
106 	/* Clear exec interupts for all vfs */
107 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
108 }
109 
110 static void cpt_disable_all_interrupts(struct cpt_device *cpt)
111 {
112 	cpt_disable_mbox_interrupts(cpt);
113 	cpt_disable_ecc_interrupts(cpt);
114 	cpt_disable_exec_interrupts(cpt);
115 }
116 
117 static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
118 {
119 	/* Set mbox(0) interupts for all vfs */
120 	cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
121 }
122 
123 static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
124 {
125 	int ret = 0, core = 0, shift = 0;
126 	u32 total_cores = 0;
127 	struct device *dev = &cpt->pdev->dev;
128 
129 	if (!mcode || !mcode->code) {
130 		dev_err(dev, "Either the mcode is null or data is NULL\n");
131 		return -EINVAL;
132 	}
133 
134 	if (mcode->code_size == 0) {
135 		dev_err(dev, "microcode size is 0\n");
136 		return -EINVAL;
137 	}
138 
139 	/* Assumes 0-9 are SE cores for UCODE_BASE registers and
140 	 * AE core bases follow
141 	 */
142 	if (mcode->is_ae) {
143 		core = CPT_MAX_SE_CORES; /* start couting from 10 */
144 		total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
145 	} else {
146 		core = 0; /* start couting from 0 */
147 		total_cores = CPT_MAX_SE_CORES; /* upto 9 */
148 	}
149 
150 	/* Point to microcode for each core of the group */
151 	for (; core < total_cores ; core++, shift++) {
152 		if (mcode->core_mask & (1 << shift)) {
153 			cpt_write_csr64(cpt->reg_base,
154 					CPTX_PF_ENGX_UCODE_BASE(0, core),
155 					(u64)mcode->phys_base);
156 		}
157 	}
158 	return ret;
159 }
160 
161 static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode)
162 {
163 	int ret = 0;
164 	struct device *dev = &cpt->pdev->dev;
165 
166 	/* Make device not ready */
167 	cpt->flags &= ~CPT_FLAG_DEVICE_READY;
168 	/* Disable All PF interrupts */
169 	cpt_disable_all_interrupts(cpt);
170 	/* Calculate mcode group and coremasks */
171 	if (mcode->is_ae) {
172 		if (mcode->num_cores > cpt->max_ae_cores) {
173 			dev_err(dev, "Requested for more cores than available AE cores\n");
174 			ret = -EINVAL;
175 			goto cpt_init_fail;
176 		}
177 
178 		if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
179 			dev_err(dev, "Can't load, all eight microcode groups in use");
180 			return -ENFILE;
181 		}
182 
183 		mcode->group = cpt->next_group;
184 		/* Convert requested cores to mask */
185 		mcode->core_mask = GENMASK(mcode->num_cores, 0);
186 		cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES,
187 				  mcode->group);
188 		/* Load microcode for AE engines */
189 		ret = cpt_load_microcode(cpt, mcode);
190 		if (ret) {
191 			dev_err(dev, "Microcode load Failed for %s\n",
192 				mcode->version);
193 			goto cpt_init_fail;
194 		}
195 		cpt->next_group++;
196 		/* Configure group mask for the mcode */
197 		cpt_configure_group(cpt, mcode->group, mcode->core_mask,
198 				    AE_TYPES);
199 		/* Enable AE cores for the group mask */
200 		cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES);
201 	} else {
202 		if (mcode->num_cores > cpt->max_se_cores) {
203 			dev_err(dev, "Requested for more cores than available SE cores\n");
204 			ret = -EINVAL;
205 			goto cpt_init_fail;
206 		}
207 		if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
208 			dev_err(dev, "Can't load, all eight microcode groups in use");
209 			return -ENFILE;
210 		}
211 
212 		mcode->group = cpt->next_group;
213 		/* Covert requested cores to mask */
214 		mcode->core_mask = GENMASK(mcode->num_cores, 0);
215 		cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES,
216 				  mcode->group);
217 		/* Load microcode for SE engines */
218 		ret = cpt_load_microcode(cpt, mcode);
219 		if (ret) {
220 			dev_err(dev, "Microcode load Failed for %s\n",
221 				mcode->version);
222 			goto cpt_init_fail;
223 		}
224 		cpt->next_group++;
225 		/* Configure group mask for the mcode */
226 		cpt_configure_group(cpt, mcode->group, mcode->core_mask,
227 				    SE_TYPES);
228 		/* Enable SE cores for the group mask */
229 		cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES);
230 	}
231 
232 	/* Enabled PF mailbox interrupts */
233 	cpt_enable_mbox_interrupts(cpt);
234 	cpt->flags |= CPT_FLAG_DEVICE_READY;
235 
236 	return ret;
237 
238 cpt_init_fail:
239 	/* Enabled PF mailbox interrupts */
240 	cpt_enable_mbox_interrupts(cpt);
241 
242 	return ret;
243 }
244 
245 struct ucode_header {
246 	u8 version[CPT_UCODE_VERSION_SZ];
247 	u32 code_length;
248 	u32 data_length;
249 	u64 sram_address;
250 };
251 
252 static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
253 {
254 	const struct firmware *fw_entry;
255 	struct device *dev = &cpt->pdev->dev;
256 	struct ucode_header *ucode;
257 	struct microcode *mcode;
258 	int j, ret = 0;
259 
260 	ret = request_firmware(&fw_entry, fw, dev);
261 	if (ret)
262 		return ret;
263 
264 	ucode = (struct ucode_header *)fw_entry->data;
265 	mcode = &cpt->mcode[cpt->next_mc_idx];
266 	memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
267 	mcode->code_size = ntohl(ucode->code_length) * 2;
268 	if (!mcode->code_size) {
269 		ret = -EINVAL;
270 		goto fw_release;
271 	}
272 
273 	mcode->is_ae = is_ae;
274 	mcode->core_mask = 0ULL;
275 	mcode->num_cores = is_ae ? 6 : 10;
276 
277 	/*  Allocate DMAable space */
278 	mcode->code = dma_alloc_coherent(&cpt->pdev->dev, mcode->code_size,
279 					 &mcode->phys_base, GFP_KERNEL);
280 	if (!mcode->code) {
281 		dev_err(dev, "Unable to allocate space for microcode");
282 		ret = -ENOMEM;
283 		goto fw_release;
284 	}
285 
286 	memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
287 	       mcode->code_size);
288 
289 	/* Byte swap 64-bit */
290 	for (j = 0; j < (mcode->code_size / 8); j++)
291 		((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]);
292 	/*  MC needs 16-bit swap */
293 	for (j = 0; j < (mcode->code_size / 2); j++)
294 		((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]);
295 
296 	dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size);
297 	dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae);
298 	dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores);
299 	dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code);
300 	dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base);
301 
302 	ret = do_cpt_init(cpt, mcode);
303 	if (ret) {
304 		dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
305 		goto fw_release;
306 	}
307 
308 	dev_info(dev, "Microcode Loaded %s\n", mcode->version);
309 	mcode->is_mc_valid = 1;
310 	cpt->next_mc_idx++;
311 
312 fw_release:
313 	release_firmware(fw_entry);
314 
315 	return ret;
316 }
317 
318 static int cpt_ucode_load(struct cpt_device *cpt)
319 {
320 	int ret = 0;
321 	struct device *dev = &cpt->pdev->dev;
322 
323 	ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-ae.out", true);
324 	if (ret) {
325 		dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret);
326 		return ret;
327 	}
328 	ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-se.out", false);
329 	if (ret) {
330 		dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret);
331 		return ret;
332 	}
333 
334 	return ret;
335 }
336 
337 static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq)
338 {
339 	struct cpt_device *cpt = (struct cpt_device *)cpt_irq;
340 
341 	cpt_mbox_intr_handler(cpt, 0);
342 
343 	return IRQ_HANDLED;
344 }
345 
346 static void cpt_reset(struct cpt_device *cpt)
347 {
348 	cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1);
349 }
350 
351 static void cpt_find_max_enabled_cores(struct cpt_device *cpt)
352 {
353 	union cptx_pf_constants pf_cnsts = {0};
354 
355 	pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0));
356 	cpt->max_se_cores = pf_cnsts.s.se;
357 	cpt->max_ae_cores = pf_cnsts.s.ae;
358 }
359 
360 static u32 cpt_check_bist_status(struct cpt_device *cpt)
361 {
362 	union cptx_pf_bist_status bist_sts = {0};
363 
364 	bist_sts.u = cpt_read_csr64(cpt->reg_base,
365 				    CPTX_PF_BIST_STATUS(0));
366 
367 	return bist_sts.u;
368 }
369 
370 static u64 cpt_check_exe_bist_status(struct cpt_device *cpt)
371 {
372 	union cptx_pf_exe_bist_status bist_sts = {0};
373 
374 	bist_sts.u = cpt_read_csr64(cpt->reg_base,
375 				    CPTX_PF_EXE_BIST_STATUS(0));
376 
377 	return bist_sts.u;
378 }
379 
380 static void cpt_disable_all_cores(struct cpt_device *cpt)
381 {
382 	u32 grp, timeout = 100;
383 	struct device *dev = &cpt->pdev->dev;
384 
385 	/* Disengage the cores from groups */
386 	for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
387 		cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0);
388 		udelay(CSR_DELAY);
389 	}
390 
391 	grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
392 	while (grp) {
393 		dev_err(dev, "Cores still busy");
394 		grp = cpt_read_csr64(cpt->reg_base,
395 				     CPTX_PF_EXEC_BUSY(0));
396 		if (timeout--)
397 			break;
398 
399 		udelay(CSR_DELAY);
400 	}
401 	/* Disable the cores */
402 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
403 }
404 
405 /**
406  * Ensure all cores are disengaged from all groups by
407  * calling cpt_disable_all_cores() before calling this
408  * function.
409  */
410 static void cpt_unload_microcode(struct cpt_device *cpt)
411 {
412 	u32 grp = 0, core;
413 
414 	/* Free microcode bases and reset group masks */
415 	for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
416 		struct microcode *mcode = &cpt->mcode[grp];
417 
418 		if (cpt->mcode[grp].code)
419 			dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
420 					  mcode->code, mcode->phys_base);
421 		mcode->code = NULL;
422 	}
423 	/* Clear UCODE_BASE registers for all engines */
424 	for (core = 0; core < CPT_MAX_TOTAL_CORES; core++)
425 		cpt_write_csr64(cpt->reg_base,
426 				CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull);
427 }
428 
429 static int cpt_device_init(struct cpt_device *cpt)
430 {
431 	u64 bist;
432 	struct device *dev = &cpt->pdev->dev;
433 
434 	/* Reset the PF when probed first */
435 	cpt_reset(cpt);
436 	msleep(100);
437 
438 	/*Check BIST status*/
439 	bist = (u64)cpt_check_bist_status(cpt);
440 	if (bist) {
441 		dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
442 		return -ENODEV;
443 	}
444 
445 	bist = cpt_check_exe_bist_status(cpt);
446 	if (bist) {
447 		dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
448 		return -ENODEV;
449 	}
450 
451 	/*Get CLK frequency*/
452 	/*Get max enabled cores */
453 	cpt_find_max_enabled_cores(cpt);
454 	/*Disable all cores*/
455 	cpt_disable_all_cores(cpt);
456 	/*Reset device parameters*/
457 	cpt->next_mc_idx   = 0;
458 	cpt->next_group = 0;
459 	/* PF is ready */
460 	cpt->flags |= CPT_FLAG_DEVICE_READY;
461 
462 	return 0;
463 }
464 
465 static int cpt_register_interrupts(struct cpt_device *cpt)
466 {
467 	int ret;
468 	struct device *dev = &cpt->pdev->dev;
469 
470 	/* Enable MSI-X */
471 	ret = pci_alloc_irq_vectors(cpt->pdev, CPT_PF_MSIX_VECTORS,
472 			CPT_PF_MSIX_VECTORS, PCI_IRQ_MSIX);
473 	if (ret < 0) {
474 		dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n",
475 			CPT_PF_MSIX_VECTORS);
476 		return ret;
477 	}
478 
479 	/* Register mailbox interrupt handlers */
480 	ret = request_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)),
481 			  cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt);
482 	if (ret)
483 		goto fail;
484 
485 	/* Enable mailbox interrupt */
486 	cpt_enable_mbox_interrupts(cpt);
487 	return 0;
488 
489 fail:
490 	dev_err(dev, "Request irq failed\n");
491 	pci_disable_msix(cpt->pdev);
492 	return ret;
493 }
494 
495 static void cpt_unregister_interrupts(struct cpt_device *cpt)
496 {
497 	free_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), cpt);
498 	pci_disable_msix(cpt->pdev);
499 }
500 
501 static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs)
502 {
503 	int pos = 0;
504 	int err;
505 	u16 total_vf_cnt;
506 	struct pci_dev *pdev = cpt->pdev;
507 
508 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
509 	if (!pos) {
510 		dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
511 		return -ENODEV;
512 	}
513 
514 	cpt->num_vf_en = num_vfs; /* User requested VFs */
515 	pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
516 	if (total_vf_cnt < cpt->num_vf_en)
517 		cpt->num_vf_en = total_vf_cnt;
518 
519 	if (!total_vf_cnt)
520 		return 0;
521 
522 	/*Enabled the available VFs */
523 	err = pci_enable_sriov(pdev, cpt->num_vf_en);
524 	if (err) {
525 		dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
526 			cpt->num_vf_en);
527 		cpt->num_vf_en = 0;
528 		return err;
529 	}
530 
531 	/* TODO: Optionally enable static VQ priorities feature */
532 
533 	dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
534 		 cpt->num_vf_en);
535 
536 	cpt->flags |= CPT_FLAG_SRIOV_ENABLED;
537 
538 	return 0;
539 }
540 
541 static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
542 {
543 	struct device *dev = &pdev->dev;
544 	struct cpt_device *cpt;
545 	int err;
546 
547 	if (num_vfs > 16 || num_vfs < 4) {
548 		dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n",
549 			 num_vfs);
550 		num_vfs = 4;
551 	}
552 
553 	cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL);
554 	if (!cpt)
555 		return -ENOMEM;
556 
557 	pci_set_drvdata(pdev, cpt);
558 	cpt->pdev = pdev;
559 	err = pci_enable_device(pdev);
560 	if (err) {
561 		dev_err(dev, "Failed to enable PCI device\n");
562 		pci_set_drvdata(pdev, NULL);
563 		return err;
564 	}
565 
566 	err = pci_request_regions(pdev, DRV_NAME);
567 	if (err) {
568 		dev_err(dev, "PCI request regions failed 0x%x\n", err);
569 		goto cpt_err_disable_device;
570 	}
571 
572 	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
573 	if (err) {
574 		dev_err(dev, "Unable to get usable DMA configuration\n");
575 		goto cpt_err_release_regions;
576 	}
577 
578 	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
579 	if (err) {
580 		dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
581 		goto cpt_err_release_regions;
582 	}
583 
584 	/* MAP PF's configuration registers */
585 	cpt->reg_base = pcim_iomap(pdev, 0, 0);
586 	if (!cpt->reg_base) {
587 		dev_err(dev, "Cannot map config register space, aborting\n");
588 		err = -ENOMEM;
589 		goto cpt_err_release_regions;
590 	}
591 
592 	/* CPT device HW initialization */
593 	cpt_device_init(cpt);
594 
595 	/* Register interrupts */
596 	err = cpt_register_interrupts(cpt);
597 	if (err)
598 		goto cpt_err_release_regions;
599 
600 	err = cpt_ucode_load(cpt);
601 	if (err)
602 		goto cpt_err_unregister_interrupts;
603 
604 	/* Configure SRIOV */
605 	err = cpt_sriov_init(cpt, num_vfs);
606 	if (err)
607 		goto cpt_err_unregister_interrupts;
608 
609 	return 0;
610 
611 cpt_err_unregister_interrupts:
612 	cpt_unregister_interrupts(cpt);
613 cpt_err_release_regions:
614 	pci_release_regions(pdev);
615 cpt_err_disable_device:
616 	pci_disable_device(pdev);
617 	pci_set_drvdata(pdev, NULL);
618 	return err;
619 }
620 
621 static void cpt_remove(struct pci_dev *pdev)
622 {
623 	struct cpt_device *cpt = pci_get_drvdata(pdev);
624 
625 	/* Disengage SE and AE cores from all groups*/
626 	cpt_disable_all_cores(cpt);
627 	/* Unload microcodes */
628 	cpt_unload_microcode(cpt);
629 	cpt_unregister_interrupts(cpt);
630 	pci_disable_sriov(pdev);
631 	pci_release_regions(pdev);
632 	pci_disable_device(pdev);
633 	pci_set_drvdata(pdev, NULL);
634 }
635 
636 static void cpt_shutdown(struct pci_dev *pdev)
637 {
638 	struct cpt_device *cpt = pci_get_drvdata(pdev);
639 
640 	if (!cpt)
641 		return;
642 
643 	dev_info(&pdev->dev, "Shutdown device %x:%x.\n",
644 		 (u32)pdev->vendor, (u32)pdev->device);
645 
646 	cpt_unregister_interrupts(cpt);
647 	pci_release_regions(pdev);
648 	pci_disable_device(pdev);
649 	pci_set_drvdata(pdev, NULL);
650 }
651 
652 /* Supported devices */
653 static const struct pci_device_id cpt_id_table[] = {
654 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) },
655 	{ 0, }  /* end of table */
656 };
657 
658 static struct pci_driver cpt_pci_driver = {
659 	.name = DRV_NAME,
660 	.id_table = cpt_id_table,
661 	.probe = cpt_probe,
662 	.remove = cpt_remove,
663 	.shutdown = cpt_shutdown,
664 };
665 
666 module_pci_driver(cpt_pci_driver);
667 
668 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
669 MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver");
670 MODULE_LICENSE("GPL v2");
671 MODULE_VERSION(DRV_VERSION);
672 MODULE_DEVICE_TABLE(pci, cpt_id_table);
673