1 #include <linux/aer.h>
2 #include <linux/delay.h>
3 #include <linux/debugfs.h>
4 #include <linux/firmware.h>
5 #include <linux/list.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
8 #include <linux/pci.h>
9 #include <linux/pci_ids.h>
10 
11 #include "nitrox_dev.h"
12 #include "nitrox_common.h"
13 #include "nitrox_csr.h"
14 
15 #define CNN55XX_DEV_ID	0x12
16 #define MAX_PF_QUEUES	64
17 #define UCODE_HLEN 48
18 #define SE_GROUP 0
19 
20 #define DRIVER_VERSION "1.0"
21 #define FW_DIR "cavium/"
22 /* SE microcode */
23 #define SE_FW	FW_DIR "cnn55xx_se.fw"
24 
25 static const char nitrox_driver_name[] = "CNN55XX";
26 
27 static LIST_HEAD(ndevlist);
28 static DEFINE_MUTEX(devlist_lock);
29 static unsigned int num_devices;
30 
31 /**
32  * nitrox_pci_tbl - PCI Device ID Table
33  */
34 static const struct pci_device_id nitrox_pci_tbl[] = {
35 	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
36 	/* required last entry */
37 	{0, }
38 };
39 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
40 
41 static unsigned int qlen = DEFAULT_CMD_QLEN;
42 module_param(qlen, uint, 0644);
43 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
44 
45 /**
46  * struct ucode - Firmware Header
47  * @id: microcode ID
48  * @version: firmware version
49  * @code_size: code section size
50  * @raz: alignment
51  * @code: code section
52  */
53 struct ucode {
54 	u8 id;
55 	char version[VERSION_LEN - 1];
56 	__be32 code_size;
57 	u8 raz[12];
58 	u64 code[0];
59 };
60 
61 /**
62  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
63  */
64 static void write_to_ucd_unit(struct nitrox_device *ndev,
65 			      struct ucode *ucode)
66 {
67 	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
68 	u64 offset, data;
69 	int i = 0;
70 
71 	/*
72 	 * UCD structure
73 	 *
74 	 *  -------------
75 	 *  |    BLK 7  |
76 	 *  -------------
77 	 *  |    BLK 6  |
78 	 *  -------------
79 	 *  |    ...    |
80 	 *  -------------
81 	 *  |    BLK 0  |
82 	 *  -------------
83 	 *  Total of 8 blocks, each size 32KB
84 	 */
85 
86 	/* set the block number */
87 	offset = UCD_UCODE_LOAD_BLOCK_NUM;
88 	nitrox_write_csr(ndev, offset, 0);
89 
90 	code_size = roundup(code_size, 8);
91 	while (code_size) {
92 		data = ucode->code[i];
93 		/* write 8 bytes at a time */
94 		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
95 		nitrox_write_csr(ndev, offset, data);
96 		code_size -= 8;
97 		i++;
98 	}
99 
100 	/* put all SE cores in group 0 */
101 	offset = POM_GRP_EXECMASKX(SE_GROUP);
102 	nitrox_write_csr(ndev, offset, (~0ULL));
103 
104 	for (i = 0; i < ndev->hw.se_cores; i++) {
105 		/*
106 		 * write block number and firware length
107 		 * bit:<2:0> block number
108 		 * bit:3 is set SE uses 32KB microcode
109 		 * bit:3 is clear SE uses 64KB microcode
110 		 */
111 		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
112 		nitrox_write_csr(ndev, offset, 0x8);
113 	}
114 	usleep_range(300, 400);
115 }
116 
117 static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
118 {
119 	const struct firmware *fw;
120 	struct ucode *ucode;
121 	int ret;
122 
123 	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
124 
125 	ret = request_firmware(&fw, fw_name, DEV(ndev));
126 	if (ret < 0) {
127 		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
128 		return ret;
129 	}
130 
131 	ucode = (struct ucode *)fw->data;
132 	/* copy the firmware version */
133 	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
134 	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
135 
136 	write_to_ucd_unit(ndev, ucode);
137 	release_firmware(fw);
138 
139 	set_bit(NITROX_UCODE_LOADED, &ndev->status);
140 	/* barrier to sync with other cpus */
141 	smp_mb__after_atomic();
142 	return 0;
143 }
144 
145 /**
146  * nitrox_add_to_devlist - add NITROX device to global device list
147  * @ndev: NITROX device
148  */
149 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
150 {
151 	struct nitrox_device *dev;
152 	int ret = 0;
153 
154 	INIT_LIST_HEAD(&ndev->list);
155 	refcount_set(&ndev->refcnt, 1);
156 
157 	mutex_lock(&devlist_lock);
158 	list_for_each_entry(dev, &ndevlist, list) {
159 		if (dev == ndev) {
160 			ret = -EEXIST;
161 			goto unlock;
162 		}
163 	}
164 	ndev->idx = num_devices++;
165 	list_add_tail(&ndev->list, &ndevlist);
166 unlock:
167 	mutex_unlock(&devlist_lock);
168 	return ret;
169 }
170 
171 /**
172  * nitrox_remove_from_devlist - remove NITROX device from
173  *   global device list
174  * @ndev: NITROX device
175  */
176 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
177 {
178 	mutex_lock(&devlist_lock);
179 	list_del(&ndev->list);
180 	num_devices--;
181 	mutex_unlock(&devlist_lock);
182 }
183 
184 struct nitrox_device *nitrox_get_first_device(void)
185 {
186 	struct nitrox_device *ndev = NULL;
187 
188 	mutex_lock(&devlist_lock);
189 	list_for_each_entry(ndev, &ndevlist, list) {
190 		if (nitrox_ready(ndev))
191 			break;
192 	}
193 	mutex_unlock(&devlist_lock);
194 	if (!ndev)
195 		return NULL;
196 
197 	refcount_inc(&ndev->refcnt);
198 	/* barrier to sync with other cpus */
199 	smp_mb__after_atomic();
200 	return ndev;
201 }
202 
203 void nitrox_put_device(struct nitrox_device *ndev)
204 {
205 	if (!ndev)
206 		return;
207 
208 	refcount_dec(&ndev->refcnt);
209 	/* barrier to sync with other cpus */
210 	smp_mb__after_atomic();
211 }
212 
213 static int nitrox_reset_device(struct pci_dev *pdev)
214 {
215 	int pos = 0;
216 
217 	pos = pci_save_state(pdev);
218 	if (pos) {
219 		dev_err(&pdev->dev, "Failed to save pci state\n");
220 		return -ENOMEM;
221 	}
222 
223 	pos = pci_pcie_cap(pdev);
224 	if (!pos)
225 		return -ENOTTY;
226 
227 	if (!pci_wait_for_pending_transaction(pdev))
228 		dev_err(&pdev->dev, "waiting for pending transaction\n");
229 
230 	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
231 	msleep(100);
232 	pci_restore_state(pdev);
233 
234 	return 0;
235 }
236 
237 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
238 {
239 	int err;
240 
241 	err = nitrox_common_sw_init(ndev);
242 	if (err)
243 		return err;
244 
245 	err = nitrox_pf_init_isr(ndev);
246 	if (err)
247 		nitrox_common_sw_cleanup(ndev);
248 
249 	return err;
250 }
251 
252 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
253 {
254 	nitrox_pf_cleanup_isr(ndev);
255 	nitrox_common_sw_cleanup(ndev);
256 }
257 
258 /**
259  * nitrox_bist_check - Check NITORX BIST registers status
260  * @ndev: NITROX device
261  */
262 static int nitrox_bist_check(struct nitrox_device *ndev)
263 {
264 	u64 value = 0;
265 	int i;
266 
267 	for (i = 0; i < NR_CLUSTERS; i++) {
268 		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
269 		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
270 	}
271 	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
272 	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
273 	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
274 	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
275 	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
276 	value += nitrox_read_csr(ndev, POM_BIST_REG);
277 	value += nitrox_read_csr(ndev, BMI_BIST_REG);
278 	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
279 	value += nitrox_read_csr(ndev, BMO_BIST_REG);
280 	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
281 	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
282 	if (value)
283 		return -EIO;
284 	return 0;
285 }
286 
287 static void nitrox_get_hwinfo(struct nitrox_device *ndev)
288 {
289 	union emu_fuse_map emu_fuse;
290 	u64 offset;
291 	int i;
292 
293 	for (i = 0; i < NR_CLUSTERS; i++) {
294 		u8 dead_cores;
295 
296 		offset = EMU_FUSE_MAPX(i);
297 		emu_fuse.value = nitrox_read_csr(ndev, offset);
298 		if (emu_fuse.s.valid) {
299 			dead_cores = hweight32(emu_fuse.s.ae_fuse);
300 			ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
301 			dead_cores = hweight16(emu_fuse.s.se_fuse);
302 			ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
303 		}
304 	}
305 }
306 
307 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
308 {
309 	int err;
310 
311 	err = nitrox_bist_check(ndev);
312 	if (err) {
313 		dev_err(&ndev->pdev->dev, "BIST check failed\n");
314 		return err;
315 	}
316 	/* get cores information */
317 	nitrox_get_hwinfo(ndev);
318 
319 	nitrox_config_nps_unit(ndev);
320 	nitrox_config_pom_unit(ndev);
321 	nitrox_config_efl_unit(ndev);
322 	/* configure IO units */
323 	nitrox_config_bmi_unit(ndev);
324 	nitrox_config_bmo_unit(ndev);
325 	/* configure Local Buffer Cache */
326 	nitrox_config_lbc_unit(ndev);
327 	nitrox_config_rand_unit(ndev);
328 
329 	/* load firmware on SE cores */
330 	err = nitrox_load_fw(ndev, SE_FW);
331 	if (err)
332 		return err;
333 
334 	nitrox_config_emu_unit(ndev);
335 
336 	return 0;
337 }
338 
339 #if IS_ENABLED(CONFIG_DEBUG_FS)
340 static int registers_show(struct seq_file *s, void *v)
341 {
342 	struct nitrox_device *ndev = s->private;
343 	u64 offset;
344 
345 	/* NPS DMA stats */
346 	offset = NPS_STATS_PKT_DMA_RD_CNT;
347 	seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
348 		   nitrox_read_csr(ndev, offset));
349 	offset = NPS_STATS_PKT_DMA_WR_CNT;
350 	seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
351 		   nitrox_read_csr(ndev, offset));
352 
353 	/* BMI/BMO stats */
354 	offset = BMI_NPS_PKT_CNT;
355 	seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
356 		   nitrox_read_csr(ndev, offset));
357 	offset = BMO_NPS_SLC_PKT_CNT;
358 	seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
359 		   nitrox_read_csr(ndev, offset));
360 
361 	return 0;
362 }
363 
364 static int registers_open(struct inode *inode, struct file *file)
365 {
366 	return single_open(file, registers_show, inode->i_private);
367 }
368 
369 static const struct file_operations register_fops = {
370 	.owner = THIS_MODULE,
371 	.open = registers_open,
372 	.read = seq_read,
373 	.llseek = seq_lseek,
374 	.release = single_release,
375 };
376 
377 static int firmware_show(struct seq_file *s, void *v)
378 {
379 	struct nitrox_device *ndev = s->private;
380 
381 	seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
382 	return 0;
383 }
384 
385 static int firmware_open(struct inode *inode, struct file *file)
386 {
387 	return single_open(file, firmware_show, inode->i_private);
388 }
389 
390 static const struct file_operations firmware_fops = {
391 	.owner = THIS_MODULE,
392 	.open = firmware_open,
393 	.read = seq_read,
394 	.llseek = seq_lseek,
395 	.release = single_release,
396 };
397 
398 static int nitrox_show(struct seq_file *s, void *v)
399 {
400 	struct nitrox_device *ndev = s->private;
401 
402 	seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
403 	seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
404 	seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
405 		   ndev->hw.ae_cores, ndev->hw.se_cores);
406 	seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
407 	seq_printf(s, "  Queue length: %u\n", ndev->qlen);
408 	seq_printf(s, "  Node: %u\n", ndev->node);
409 
410 	return 0;
411 }
412 
413 static int nitrox_open(struct inode *inode, struct file *file)
414 {
415 	return single_open(file, nitrox_show, inode->i_private);
416 }
417 
418 static const struct file_operations nitrox_fops = {
419 	.owner = THIS_MODULE,
420 	.open = nitrox_open,
421 	.read = seq_read,
422 	.llseek = seq_lseek,
423 	.release = single_release,
424 };
425 
426 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
427 {
428 	debugfs_remove_recursive(ndev->debugfs_dir);
429 	ndev->debugfs_dir = NULL;
430 }
431 
432 static int nitrox_debugfs_init(struct nitrox_device *ndev)
433 {
434 	struct dentry *dir, *f;
435 
436 	dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
437 	if (!dir)
438 		return -ENOMEM;
439 
440 	ndev->debugfs_dir = dir;
441 	f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
442 	if (!f)
443 		goto err;
444 	f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
445 	if (!f)
446 		goto err;
447 	f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
448 	if (!f)
449 		goto err;
450 
451 	return 0;
452 
453 err:
454 	nitrox_debugfs_exit(ndev);
455 	return -ENODEV;
456 }
457 #else
458 static int nitrox_debugfs_init(struct nitrox_device *ndev)
459 {
460 	return 0;
461 }
462 
463 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
464 {
465 }
466 #endif
467 
468 /**
469  * nitrox_probe - NITROX Initialization function.
470  * @pdev: PCI device information struct
471  * @id: entry in nitrox_pci_tbl
472  *
473  * Return: 0, if the driver is bound to the device, or
474  *         a negative error if there is failure.
475  */
476 static int nitrox_probe(struct pci_dev *pdev,
477 			const struct pci_device_id *id)
478 {
479 	struct nitrox_device *ndev;
480 	int err;
481 
482 	dev_info_once(&pdev->dev, "%s driver version %s\n",
483 		      nitrox_driver_name, DRIVER_VERSION);
484 
485 	err = pci_enable_device_mem(pdev);
486 	if (err)
487 		return err;
488 
489 	/* do FLR */
490 	err = nitrox_reset_device(pdev);
491 	if (err) {
492 		dev_err(&pdev->dev, "FLR failed\n");
493 		pci_disable_device(pdev);
494 		return err;
495 	}
496 
497 	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
498 		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
499 	} else {
500 		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
501 		if (err) {
502 			dev_err(&pdev->dev, "DMA configuration failed\n");
503 			pci_disable_device(pdev);
504 			return err;
505 		}
506 	}
507 
508 	err = pci_request_mem_regions(pdev, nitrox_driver_name);
509 	if (err) {
510 		pci_disable_device(pdev);
511 		return err;
512 	}
513 	pci_set_master(pdev);
514 
515 	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
516 	if (!ndev) {
517 		err = -ENOMEM;
518 		goto ndev_fail;
519 	}
520 
521 	pci_set_drvdata(pdev, ndev);
522 	ndev->pdev = pdev;
523 
524 	/* add to device list */
525 	nitrox_add_to_devlist(ndev);
526 
527 	ndev->hw.vendor_id = pdev->vendor;
528 	ndev->hw.device_id = pdev->device;
529 	ndev->hw.revision_id = pdev->revision;
530 	/* command timeout in jiffies */
531 	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
532 	ndev->node = dev_to_node(&pdev->dev);
533 	if (ndev->node == NUMA_NO_NODE)
534 		ndev->node = 0;
535 
536 	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
537 				 pci_resource_len(pdev, 0));
538 	if (!ndev->bar_addr) {
539 		err = -EIO;
540 		goto ioremap_err;
541 	}
542 	/* allocate command queus based on cpus, max queues are 64 */
543 	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
544 	ndev->qlen = qlen;
545 
546 	err = nitrox_pf_sw_init(ndev);
547 	if (err)
548 		goto ioremap_err;
549 
550 	err = nitrox_pf_hw_init(ndev);
551 	if (err)
552 		goto pf_hw_fail;
553 
554 	err = nitrox_debugfs_init(ndev);
555 	if (err)
556 		goto pf_hw_fail;
557 
558 	set_bit(NITROX_READY, &ndev->status);
559 	/* barrier to sync with other cpus */
560 	smp_mb__after_atomic();
561 
562 	err = nitrox_crypto_register();
563 	if (err)
564 		goto crypto_fail;
565 
566 	return 0;
567 
568 crypto_fail:
569 	nitrox_debugfs_exit(ndev);
570 	clear_bit(NITROX_READY, &ndev->status);
571 	/* barrier to sync with other cpus */
572 	smp_mb__after_atomic();
573 pf_hw_fail:
574 	nitrox_pf_sw_cleanup(ndev);
575 ioremap_err:
576 	nitrox_remove_from_devlist(ndev);
577 	kfree(ndev);
578 	pci_set_drvdata(pdev, NULL);
579 ndev_fail:
580 	pci_release_mem_regions(pdev);
581 	pci_disable_device(pdev);
582 	return err;
583 }
584 
585 /**
586  * nitrox_remove - Unbind the driver from the device.
587  * @pdev: PCI device information struct
588  */
589 static void nitrox_remove(struct pci_dev *pdev)
590 {
591 	struct nitrox_device *ndev = pci_get_drvdata(pdev);
592 
593 	if (!ndev)
594 		return;
595 
596 	if (!refcount_dec_and_test(&ndev->refcnt)) {
597 		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
598 			refcount_read(&ndev->refcnt));
599 		return;
600 	}
601 
602 	dev_info(DEV(ndev), "Removing Device %x:%x\n",
603 		 ndev->hw.vendor_id, ndev->hw.device_id);
604 
605 	clear_bit(NITROX_READY, &ndev->status);
606 	/* barrier to sync with other cpus */
607 	smp_mb__after_atomic();
608 
609 	nitrox_remove_from_devlist(ndev);
610 	nitrox_crypto_unregister();
611 	nitrox_debugfs_exit(ndev);
612 	nitrox_pf_sw_cleanup(ndev);
613 
614 	iounmap(ndev->bar_addr);
615 	kfree(ndev);
616 
617 	pci_set_drvdata(pdev, NULL);
618 	pci_release_mem_regions(pdev);
619 	pci_disable_device(pdev);
620 }
621 
622 static void nitrox_shutdown(struct pci_dev *pdev)
623 {
624 	pci_set_drvdata(pdev, NULL);
625 	pci_release_mem_regions(pdev);
626 	pci_disable_device(pdev);
627 }
628 
629 static struct pci_driver nitrox_driver = {
630 	.name = nitrox_driver_name,
631 	.id_table = nitrox_pci_tbl,
632 	.probe = nitrox_probe,
633 	.remove	= nitrox_remove,
634 	.shutdown = nitrox_shutdown,
635 };
636 
637 module_pci_driver(nitrox_driver);
638 
639 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
640 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
641 MODULE_LICENSE("GPL");
642 MODULE_VERSION(DRIVER_VERSION);
643 MODULE_FIRMWARE(SE_FW);
644