1 /*
2  * Copyright (C) 2016 Cavium, Inc.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of version 2 of the GNU General Public License
6  * as published by the Free Software Foundation.
7  */
8 
9 #include <linux/device.h>
10 #include <linux/firmware.h>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/pci.h>
15 #include <linux/printk.h>
16 #include <linux/version.h>
17 
18 #include "cptpf.h"
19 
20 #define DRV_NAME	"thunder-cpt"
21 #define DRV_VERSION	"1.0"
22 
23 static u32 num_vfs = 4; /* Default 4 VF enabled */
24 module_param(num_vfs, uint, 0444);
25 MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)");
26 
27 /*
28  * Disable cores specified by coremask
29  */
30 static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask,
31 			      u8 type, u8 grp)
32 {
33 	u64 pf_exe_ctl;
34 	u32 timeout = 100;
35 	u64 grpmask = 0;
36 	struct device *dev = &cpt->pdev->dev;
37 
38 	if (type == AE_TYPES)
39 		coremask = (coremask << cpt->max_se_cores);
40 
41 	/* Disengage the cores from groups */
42 	grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
43 	cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
44 			(grpmask & ~coremask));
45 	udelay(CSR_DELAY);
46 	grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
47 	while (grp & coremask) {
48 		dev_err(dev, "Cores still busy %llx", coremask);
49 		grp = cpt_read_csr64(cpt->reg_base,
50 				     CPTX_PF_EXEC_BUSY(0));
51 		if (timeout--)
52 			break;
53 
54 		udelay(CSR_DELAY);
55 	}
56 
57 	/* Disable the cores */
58 	pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
59 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
60 			(pf_exe_ctl & ~coremask));
61 	udelay(CSR_DELAY);
62 }
63 
64 /*
65  * Enable cores specified by coremask
66  */
67 static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask,
68 			     u8 type)
69 {
70 	u64 pf_exe_ctl;
71 
72 	if (type == AE_TYPES)
73 		coremask = (coremask << cpt->max_se_cores);
74 
75 	pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
76 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
77 			(pf_exe_ctl | coremask));
78 	udelay(CSR_DELAY);
79 }
80 
81 static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
82 				u64 coremask, u8 type)
83 {
84 	u64 pf_gx_en = 0;
85 
86 	if (type == AE_TYPES)
87 		coremask = (coremask << cpt->max_se_cores);
88 
89 	pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
90 	cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
91 			(pf_gx_en | coremask));
92 	udelay(CSR_DELAY);
93 }
94 
95 static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
96 {
97 	/* Clear mbox(0) interupts for all vfs */
98 	cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
99 }
100 
101 static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
102 {
103 	/* Clear ecc(0) interupts for all vfs */
104 	cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
105 }
106 
107 static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
108 {
109 	/* Clear exec interupts for all vfs */
110 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
111 }
112 
113 static void cpt_disable_all_interrupts(struct cpt_device *cpt)
114 {
115 	cpt_disable_mbox_interrupts(cpt);
116 	cpt_disable_ecc_interrupts(cpt);
117 	cpt_disable_exec_interrupts(cpt);
118 }
119 
120 static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
121 {
122 	/* Set mbox(0) interupts for all vfs */
123 	cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
124 }
125 
126 static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
127 {
128 	int ret = 0, core = 0, shift = 0;
129 	u32 total_cores = 0;
130 	struct device *dev = &cpt->pdev->dev;
131 
132 	if (!mcode || !mcode->code) {
133 		dev_err(dev, "Either the mcode is null or data is NULL\n");
134 		return -EINVAL;
135 	}
136 
137 	if (mcode->code_size == 0) {
138 		dev_err(dev, "microcode size is 0\n");
139 		return -EINVAL;
140 	}
141 
142 	/* Assumes 0-9 are SE cores for UCODE_BASE registers and
143 	 * AE core bases follow
144 	 */
145 	if (mcode->is_ae) {
146 		core = CPT_MAX_SE_CORES; /* start couting from 10 */
147 		total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
148 	} else {
149 		core = 0; /* start couting from 0 */
150 		total_cores = CPT_MAX_SE_CORES; /* upto 9 */
151 	}
152 
153 	/* Point to microcode for each core of the group */
154 	for (; core < total_cores ; core++, shift++) {
155 		if (mcode->core_mask & (1 << shift)) {
156 			cpt_write_csr64(cpt->reg_base,
157 					CPTX_PF_ENGX_UCODE_BASE(0, core),
158 					(u64)mcode->phys_base);
159 		}
160 	}
161 	return ret;
162 }
163 
164 static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode)
165 {
166 	int ret = 0;
167 	struct device *dev = &cpt->pdev->dev;
168 
169 	/* Make device not ready */
170 	cpt->flags &= ~CPT_FLAG_DEVICE_READY;
171 	/* Disable All PF interrupts */
172 	cpt_disable_all_interrupts(cpt);
173 	/* Calculate mcode group and coremasks */
174 	if (mcode->is_ae) {
175 		if (mcode->num_cores > cpt->max_ae_cores) {
176 			dev_err(dev, "Requested for more cores than available AE cores\n");
177 			ret = -EINVAL;
178 			goto cpt_init_fail;
179 		}
180 
181 		if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
182 			dev_err(dev, "Can't load, all eight microcode groups in use");
183 			return -ENFILE;
184 		}
185 
186 		mcode->group = cpt->next_group;
187 		/* Convert requested cores to mask */
188 		mcode->core_mask = GENMASK(mcode->num_cores, 0);
189 		cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES,
190 				  mcode->group);
191 		/* Load microcode for AE engines */
192 		ret = cpt_load_microcode(cpt, mcode);
193 		if (ret) {
194 			dev_err(dev, "Microcode load Failed for %s\n",
195 				mcode->version);
196 			goto cpt_init_fail;
197 		}
198 		cpt->next_group++;
199 		/* Configure group mask for the mcode */
200 		cpt_configure_group(cpt, mcode->group, mcode->core_mask,
201 				    AE_TYPES);
202 		/* Enable AE cores for the group mask */
203 		cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES);
204 	} else {
205 		if (mcode->num_cores > cpt->max_se_cores) {
206 			dev_err(dev, "Requested for more cores than available SE cores\n");
207 			ret = -EINVAL;
208 			goto cpt_init_fail;
209 		}
210 		if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
211 			dev_err(dev, "Can't load, all eight microcode groups in use");
212 			return -ENFILE;
213 		}
214 
215 		mcode->group = cpt->next_group;
216 		/* Covert requested cores to mask */
217 		mcode->core_mask = GENMASK(mcode->num_cores, 0);
218 		cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES,
219 				  mcode->group);
220 		/* Load microcode for SE engines */
221 		ret = cpt_load_microcode(cpt, mcode);
222 		if (ret) {
223 			dev_err(dev, "Microcode load Failed for %s\n",
224 				mcode->version);
225 			goto cpt_init_fail;
226 		}
227 		cpt->next_group++;
228 		/* Configure group mask for the mcode */
229 		cpt_configure_group(cpt, mcode->group, mcode->core_mask,
230 				    SE_TYPES);
231 		/* Enable SE cores for the group mask */
232 		cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES);
233 	}
234 
235 	/* Enabled PF mailbox interrupts */
236 	cpt_enable_mbox_interrupts(cpt);
237 	cpt->flags |= CPT_FLAG_DEVICE_READY;
238 
239 	return ret;
240 
241 cpt_init_fail:
242 	/* Enabled PF mailbox interrupts */
243 	cpt_enable_mbox_interrupts(cpt);
244 
245 	return ret;
246 }
247 
248 struct ucode_header {
249 	u8 version[CPT_UCODE_VERSION_SZ];
250 	u32 code_length;
251 	u32 data_length;
252 	u64 sram_address;
253 };
254 
255 static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
256 {
257 	const struct firmware *fw_entry;
258 	struct device *dev = &cpt->pdev->dev;
259 	struct ucode_header *ucode;
260 	struct microcode *mcode;
261 	int j, ret = 0;
262 
263 	ret = request_firmware(&fw_entry, fw, dev);
264 	if (ret)
265 		return ret;
266 
267 	ucode = (struct ucode_header *)fw_entry->data;
268 	mcode = &cpt->mcode[cpt->next_mc_idx];
269 	memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
270 	mcode->code_size = ntohl(ucode->code_length) * 2;
271 	if (!mcode->code_size)
272 		return -EINVAL;
273 
274 	mcode->is_ae = is_ae;
275 	mcode->core_mask = 0ULL;
276 	mcode->num_cores = is_ae ? 6 : 10;
277 
278 	/*  Allocate DMAable space */
279 	mcode->code = dma_zalloc_coherent(&cpt->pdev->dev, mcode->code_size,
280 					  &mcode->phys_base, GFP_KERNEL);
281 	if (!mcode->code) {
282 		dev_err(dev, "Unable to allocate space for microcode");
283 		return -ENOMEM;
284 	}
285 
286 	memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
287 	       mcode->code_size);
288 
289 	/* Byte swap 64-bit */
290 	for (j = 0; j < (mcode->code_size / 8); j++)
291 		((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]);
292 	/*  MC needs 16-bit swap */
293 	for (j = 0; j < (mcode->code_size / 2); j++)
294 		((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]);
295 
296 	dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size);
297 	dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae);
298 	dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores);
299 	dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code);
300 	dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base);
301 
302 	ret = do_cpt_init(cpt, mcode);
303 	if (ret) {
304 		dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
305 		return ret;
306 	}
307 
308 	dev_info(dev, "Microcode Loaded %s\n", mcode->version);
309 	mcode->is_mc_valid = 1;
310 	cpt->next_mc_idx++;
311 	release_firmware(fw_entry);
312 
313 	return ret;
314 }
315 
316 static int cpt_ucode_load(struct cpt_device *cpt)
317 {
318 	int ret = 0;
319 	struct device *dev = &cpt->pdev->dev;
320 
321 	ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-ae.out", true);
322 	if (ret) {
323 		dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret);
324 		return ret;
325 	}
326 	ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-se.out", false);
327 	if (ret) {
328 		dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret);
329 		return ret;
330 	}
331 
332 	return ret;
333 }
334 
335 static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq)
336 {
337 	struct cpt_device *cpt = (struct cpt_device *)cpt_irq;
338 
339 	cpt_mbox_intr_handler(cpt, 0);
340 
341 	return IRQ_HANDLED;
342 }
343 
344 static void cpt_reset(struct cpt_device *cpt)
345 {
346 	cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1);
347 }
348 
349 static void cpt_find_max_enabled_cores(struct cpt_device *cpt)
350 {
351 	union cptx_pf_constants pf_cnsts = {0};
352 
353 	pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0));
354 	cpt->max_se_cores = pf_cnsts.s.se;
355 	cpt->max_ae_cores = pf_cnsts.s.ae;
356 }
357 
358 static u32 cpt_check_bist_status(struct cpt_device *cpt)
359 {
360 	union cptx_pf_bist_status bist_sts = {0};
361 
362 	bist_sts.u = cpt_read_csr64(cpt->reg_base,
363 				    CPTX_PF_BIST_STATUS(0));
364 
365 	return bist_sts.u;
366 }
367 
368 static u64 cpt_check_exe_bist_status(struct cpt_device *cpt)
369 {
370 	union cptx_pf_exe_bist_status bist_sts = {0};
371 
372 	bist_sts.u = cpt_read_csr64(cpt->reg_base,
373 				    CPTX_PF_EXE_BIST_STATUS(0));
374 
375 	return bist_sts.u;
376 }
377 
378 static void cpt_disable_all_cores(struct cpt_device *cpt)
379 {
380 	u32 grp, timeout = 100;
381 	struct device *dev = &cpt->pdev->dev;
382 
383 	/* Disengage the cores from groups */
384 	for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
385 		cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0);
386 		udelay(CSR_DELAY);
387 	}
388 
389 	grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
390 	while (grp) {
391 		dev_err(dev, "Cores still busy");
392 		grp = cpt_read_csr64(cpt->reg_base,
393 				     CPTX_PF_EXEC_BUSY(0));
394 		if (timeout--)
395 			break;
396 
397 		udelay(CSR_DELAY);
398 	}
399 	/* Disable the cores */
400 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
401 }
402 
403 /**
404  * Ensure all cores are disengaged from all groups by
405  * calling cpt_disable_all_cores() before calling this
406  * function.
407  */
408 static void cpt_unload_microcode(struct cpt_device *cpt)
409 {
410 	u32 grp = 0, core;
411 
412 	/* Free microcode bases and reset group masks */
413 	for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
414 		struct microcode *mcode = &cpt->mcode[grp];
415 
416 		if (cpt->mcode[grp].code)
417 			dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
418 					  mcode->code, mcode->phys_base);
419 		mcode->code = NULL;
420 	}
421 	/* Clear UCODE_BASE registers for all engines */
422 	for (core = 0; core < CPT_MAX_TOTAL_CORES; core++)
423 		cpt_write_csr64(cpt->reg_base,
424 				CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull);
425 }
426 
427 static int cpt_device_init(struct cpt_device *cpt)
428 {
429 	u64 bist;
430 	struct device *dev = &cpt->pdev->dev;
431 
432 	/* Reset the PF when probed first */
433 	cpt_reset(cpt);
434 	mdelay(100);
435 
436 	/*Check BIST status*/
437 	bist = (u64)cpt_check_bist_status(cpt);
438 	if (bist) {
439 		dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
440 		return -ENODEV;
441 	}
442 
443 	bist = cpt_check_exe_bist_status(cpt);
444 	if (bist) {
445 		dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
446 		return -ENODEV;
447 	}
448 
449 	/*Get CLK frequency*/
450 	/*Get max enabled cores */
451 	cpt_find_max_enabled_cores(cpt);
452 	/*Disable all cores*/
453 	cpt_disable_all_cores(cpt);
454 	/*Reset device parameters*/
455 	cpt->next_mc_idx   = 0;
456 	cpt->next_group = 0;
457 	/* PF is ready */
458 	cpt->flags |= CPT_FLAG_DEVICE_READY;
459 
460 	return 0;
461 }
462 
463 static int cpt_register_interrupts(struct cpt_device *cpt)
464 {
465 	int ret;
466 	struct device *dev = &cpt->pdev->dev;
467 
468 	/* Enable MSI-X */
469 	ret = pci_alloc_irq_vectors(cpt->pdev, CPT_PF_MSIX_VECTORS,
470 			CPT_PF_MSIX_VECTORS, PCI_IRQ_MSIX);
471 	if (ret < 0) {
472 		dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n",
473 			CPT_PF_MSIX_VECTORS);
474 		return ret;
475 	}
476 
477 	/* Register mailbox interrupt handlers */
478 	ret = request_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)),
479 			  cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt);
480 	if (ret)
481 		goto fail;
482 
483 	/* Enable mailbox interrupt */
484 	cpt_enable_mbox_interrupts(cpt);
485 	return 0;
486 
487 fail:
488 	dev_err(dev, "Request irq failed\n");
489 	pci_disable_msix(cpt->pdev);
490 	return ret;
491 }
492 
493 static void cpt_unregister_interrupts(struct cpt_device *cpt)
494 {
495 	free_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), cpt);
496 	pci_disable_msix(cpt->pdev);
497 }
498 
499 static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs)
500 {
501 	int pos = 0;
502 	int err;
503 	u16 total_vf_cnt;
504 	struct pci_dev *pdev = cpt->pdev;
505 
506 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
507 	if (!pos) {
508 		dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
509 		return -ENODEV;
510 	}
511 
512 	cpt->num_vf_en = num_vfs; /* User requested VFs */
513 	pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
514 	if (total_vf_cnt < cpt->num_vf_en)
515 		cpt->num_vf_en = total_vf_cnt;
516 
517 	if (!total_vf_cnt)
518 		return 0;
519 
520 	/*Enabled the available VFs */
521 	err = pci_enable_sriov(pdev, cpt->num_vf_en);
522 	if (err) {
523 		dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
524 			cpt->num_vf_en);
525 		cpt->num_vf_en = 0;
526 		return err;
527 	}
528 
529 	/* TODO: Optionally enable static VQ priorities feature */
530 
531 	dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
532 		 cpt->num_vf_en);
533 
534 	cpt->flags |= CPT_FLAG_SRIOV_ENABLED;
535 
536 	return 0;
537 }
538 
539 static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
540 {
541 	struct device *dev = &pdev->dev;
542 	struct cpt_device *cpt;
543 	int err;
544 
545 	if (num_vfs > 16 || num_vfs < 4) {
546 		dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n",
547 			 num_vfs);
548 		num_vfs = 4;
549 	}
550 
551 	cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL);
552 	if (!cpt)
553 		return -ENOMEM;
554 
555 	pci_set_drvdata(pdev, cpt);
556 	cpt->pdev = pdev;
557 	err = pci_enable_device(pdev);
558 	if (err) {
559 		dev_err(dev, "Failed to enable PCI device\n");
560 		pci_set_drvdata(pdev, NULL);
561 		return err;
562 	}
563 
564 	err = pci_request_regions(pdev, DRV_NAME);
565 	if (err) {
566 		dev_err(dev, "PCI request regions failed 0x%x\n", err);
567 		goto cpt_err_disable_device;
568 	}
569 
570 	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
571 	if (err) {
572 		dev_err(dev, "Unable to get usable DMA configuration\n");
573 		goto cpt_err_release_regions;
574 	}
575 
576 	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
577 	if (err) {
578 		dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
579 		goto cpt_err_release_regions;
580 	}
581 
582 	/* MAP PF's configuration registers */
583 	cpt->reg_base = pcim_iomap(pdev, 0, 0);
584 	if (!cpt->reg_base) {
585 		dev_err(dev, "Cannot map config register space, aborting\n");
586 		err = -ENOMEM;
587 		goto cpt_err_release_regions;
588 	}
589 
590 	/* CPT device HW initialization */
591 	cpt_device_init(cpt);
592 
593 	/* Register interrupts */
594 	err = cpt_register_interrupts(cpt);
595 	if (err)
596 		goto cpt_err_release_regions;
597 
598 	err = cpt_ucode_load(cpt);
599 	if (err)
600 		goto cpt_err_unregister_interrupts;
601 
602 	/* Configure SRIOV */
603 	err = cpt_sriov_init(cpt, num_vfs);
604 	if (err)
605 		goto cpt_err_unregister_interrupts;
606 
607 	return 0;
608 
609 cpt_err_unregister_interrupts:
610 	cpt_unregister_interrupts(cpt);
611 cpt_err_release_regions:
612 	pci_release_regions(pdev);
613 cpt_err_disable_device:
614 	pci_disable_device(pdev);
615 	pci_set_drvdata(pdev, NULL);
616 	return err;
617 }
618 
619 static void cpt_remove(struct pci_dev *pdev)
620 {
621 	struct cpt_device *cpt = pci_get_drvdata(pdev);
622 
623 	/* Disengage SE and AE cores from all groups*/
624 	cpt_disable_all_cores(cpt);
625 	/* Unload microcodes */
626 	cpt_unload_microcode(cpt);
627 	cpt_unregister_interrupts(cpt);
628 	pci_disable_sriov(pdev);
629 	pci_release_regions(pdev);
630 	pci_disable_device(pdev);
631 	pci_set_drvdata(pdev, NULL);
632 }
633 
634 static void cpt_shutdown(struct pci_dev *pdev)
635 {
636 	struct cpt_device *cpt = pci_get_drvdata(pdev);
637 
638 	if (!cpt)
639 		return;
640 
641 	dev_info(&pdev->dev, "Shutdown device %x:%x.\n",
642 		 (u32)pdev->vendor, (u32)pdev->device);
643 
644 	cpt_unregister_interrupts(cpt);
645 	pci_release_regions(pdev);
646 	pci_disable_device(pdev);
647 	pci_set_drvdata(pdev, NULL);
648 }
649 
650 /* Supported devices */
651 static const struct pci_device_id cpt_id_table[] = {
652 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) },
653 	{ 0, }  /* end of table */
654 };
655 
656 static struct pci_driver cpt_pci_driver = {
657 	.name = DRV_NAME,
658 	.id_table = cpt_id_table,
659 	.probe = cpt_probe,
660 	.remove = cpt_remove,
661 	.shutdown = cpt_shutdown,
662 };
663 
664 module_pci_driver(cpt_pci_driver);
665 
666 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
667 MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver");
668 MODULE_LICENSE("GPL v2");
669 MODULE_VERSION(DRV_VERSION);
670 MODULE_DEVICE_TABLE(pci, cpt_id_table);
671