1 #include <linux/aer.h>
2 #include <linux/delay.h>
3 #include <linux/debugfs.h>
4 #include <linux/firmware.h>
5 #include <linux/list.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
8 #include <linux/pci.h>
9 #include <linux/pci_ids.h>
10 
11 #include "nitrox_dev.h"
12 #include "nitrox_common.h"
13 #include "nitrox_csr.h"
14 #include "nitrox_hal.h"
15 #include "nitrox_isr.h"
16 
17 #define CNN55XX_DEV_ID	0x12
18 #define MAX_PF_QUEUES	64
19 #define UCODE_HLEN 48
20 #define SE_GROUP 0
21 
22 #define DRIVER_VERSION "1.1"
23 #define FW_DIR "cavium/"
24 /* SE microcode */
25 #define SE_FW	FW_DIR "cnn55xx_se.fw"
26 
27 static const char nitrox_driver_name[] = "CNN55XX";
28 
29 static LIST_HEAD(ndevlist);
30 static DEFINE_MUTEX(devlist_lock);
31 static unsigned int num_devices;
32 
33 /**
34  * nitrox_pci_tbl - PCI Device ID Table
35  */
36 static const struct pci_device_id nitrox_pci_tbl[] = {
37 	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
38 	/* required last entry */
39 	{0, }
40 };
41 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
42 
43 static unsigned int qlen = DEFAULT_CMD_QLEN;
44 module_param(qlen, uint, 0644);
45 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
46 
47 #ifdef CONFIG_PCI_IOV
48 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
49 #else
50 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
51 {
52 	return 0;
53 }
54 #endif
55 
56 /**
57  * struct ucode - Firmware Header
58  * @id: microcode ID
59  * @version: firmware version
60  * @code_size: code section size
61  * @raz: alignment
62  * @code: code section
63  */
64 struct ucode {
65 	u8 id;
66 	char version[VERSION_LEN - 1];
67 	__be32 code_size;
68 	u8 raz[12];
69 	u64 code[0];
70 };
71 
72 /**
73  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
74  */
75 static void write_to_ucd_unit(struct nitrox_device *ndev,
76 			      struct ucode *ucode)
77 {
78 	u32 code_size = be32_to_cpu(ucode->code_size) * 2;
79 	u64 offset, data;
80 	int i = 0;
81 
82 	/*
83 	 * UCD structure
84 	 *
85 	 *  -------------
86 	 *  |    BLK 7  |
87 	 *  -------------
88 	 *  |    BLK 6  |
89 	 *  -------------
90 	 *  |    ...    |
91 	 *  -------------
92 	 *  |    BLK 0  |
93 	 *  -------------
94 	 *  Total of 8 blocks, each size 32KB
95 	 */
96 
97 	/* set the block number */
98 	offset = UCD_UCODE_LOAD_BLOCK_NUM;
99 	nitrox_write_csr(ndev, offset, 0);
100 
101 	code_size = roundup(code_size, 8);
102 	while (code_size) {
103 		data = ucode->code[i];
104 		/* write 8 bytes at a time */
105 		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
106 		nitrox_write_csr(ndev, offset, data);
107 		code_size -= 8;
108 		i++;
109 	}
110 
111 	/* put all SE cores in group 0 */
112 	offset = POM_GRP_EXECMASKX(SE_GROUP);
113 	nitrox_write_csr(ndev, offset, (~0ULL));
114 
115 	for (i = 0; i < ndev->hw.se_cores; i++) {
116 		/*
117 		 * write block number and firware length
118 		 * bit:<2:0> block number
119 		 * bit:3 is set SE uses 32KB microcode
120 		 * bit:3 is clear SE uses 64KB microcode
121 		 */
122 		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
123 		nitrox_write_csr(ndev, offset, 0x8);
124 	}
125 	usleep_range(300, 400);
126 }
127 
128 static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
129 {
130 	const struct firmware *fw;
131 	struct ucode *ucode;
132 	int ret;
133 
134 	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
135 
136 	ret = request_firmware(&fw, fw_name, DEV(ndev));
137 	if (ret < 0) {
138 		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
139 		return ret;
140 	}
141 
142 	ucode = (struct ucode *)fw->data;
143 	/* copy the firmware version */
144 	memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
145 	ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
146 
147 	write_to_ucd_unit(ndev, ucode);
148 	release_firmware(fw);
149 
150 	return 0;
151 }
152 
153 /**
154  * nitrox_add_to_devlist - add NITROX device to global device list
155  * @ndev: NITROX device
156  */
157 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
158 {
159 	struct nitrox_device *dev;
160 	int ret = 0;
161 
162 	INIT_LIST_HEAD(&ndev->list);
163 	refcount_set(&ndev->refcnt, 1);
164 
165 	mutex_lock(&devlist_lock);
166 	list_for_each_entry(dev, &ndevlist, list) {
167 		if (dev == ndev) {
168 			ret = -EEXIST;
169 			goto unlock;
170 		}
171 	}
172 	ndev->idx = num_devices++;
173 	list_add_tail(&ndev->list, &ndevlist);
174 unlock:
175 	mutex_unlock(&devlist_lock);
176 	return ret;
177 }
178 
179 /**
180  * nitrox_remove_from_devlist - remove NITROX device from
181  *   global device list
182  * @ndev: NITROX device
183  */
184 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
185 {
186 	mutex_lock(&devlist_lock);
187 	list_del(&ndev->list);
188 	num_devices--;
189 	mutex_unlock(&devlist_lock);
190 }
191 
192 struct nitrox_device *nitrox_get_first_device(void)
193 {
194 	struct nitrox_device *ndev = NULL;
195 
196 	mutex_lock(&devlist_lock);
197 	list_for_each_entry(ndev, &ndevlist, list) {
198 		if (nitrox_ready(ndev))
199 			break;
200 	}
201 	mutex_unlock(&devlist_lock);
202 	if (!ndev)
203 		return NULL;
204 
205 	refcount_inc(&ndev->refcnt);
206 	/* barrier to sync with other cpus */
207 	smp_mb__after_atomic();
208 	return ndev;
209 }
210 
211 void nitrox_put_device(struct nitrox_device *ndev)
212 {
213 	if (!ndev)
214 		return;
215 
216 	refcount_dec(&ndev->refcnt);
217 	/* barrier to sync with other cpus */
218 	smp_mb__after_atomic();
219 }
220 
221 static int nitrox_device_flr(struct pci_dev *pdev)
222 {
223 	int pos = 0;
224 
225 	pos = pci_save_state(pdev);
226 	if (pos) {
227 		dev_err(&pdev->dev, "Failed to save pci state\n");
228 		return -ENOMEM;
229 	}
230 
231 	/* check flr support */
232 	if (pcie_has_flr(pdev))
233 		pcie_flr(pdev);
234 
235 	pci_restore_state(pdev);
236 
237 	return 0;
238 }
239 
240 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
241 {
242 	int err;
243 
244 	err = nitrox_common_sw_init(ndev);
245 	if (err)
246 		return err;
247 
248 	err = nitrox_register_interrupts(ndev);
249 	if (err)
250 		nitrox_common_sw_cleanup(ndev);
251 
252 	return err;
253 }
254 
255 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
256 {
257 	nitrox_unregister_interrupts(ndev);
258 	nitrox_common_sw_cleanup(ndev);
259 }
260 
261 /**
262  * nitrox_bist_check - Check NITORX BIST registers status
263  * @ndev: NITROX device
264  */
265 static int nitrox_bist_check(struct nitrox_device *ndev)
266 {
267 	u64 value = 0;
268 	int i;
269 
270 	for (i = 0; i < NR_CLUSTERS; i++) {
271 		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
272 		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
273 	}
274 	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
275 	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
276 	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
277 	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
278 	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
279 	value += nitrox_read_csr(ndev, POM_BIST_REG);
280 	value += nitrox_read_csr(ndev, BMI_BIST_REG);
281 	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
282 	value += nitrox_read_csr(ndev, BMO_BIST_REG);
283 	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
284 	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
285 	if (value)
286 		return -EIO;
287 	return 0;
288 }
289 
290 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
291 {
292 	int err;
293 
294 	err = nitrox_bist_check(ndev);
295 	if (err) {
296 		dev_err(&ndev->pdev->dev, "BIST check failed\n");
297 		return err;
298 	}
299 	/* get cores information */
300 	nitrox_get_hwinfo(ndev);
301 
302 	nitrox_config_nps_unit(ndev);
303 	nitrox_config_pom_unit(ndev);
304 	nitrox_config_efl_unit(ndev);
305 	/* configure IO units */
306 	nitrox_config_bmi_unit(ndev);
307 	nitrox_config_bmo_unit(ndev);
308 	/* configure Local Buffer Cache */
309 	nitrox_config_lbc_unit(ndev);
310 	nitrox_config_rand_unit(ndev);
311 
312 	/* load firmware on SE cores */
313 	err = nitrox_load_fw(ndev, SE_FW);
314 	if (err)
315 		return err;
316 
317 	nitrox_config_emu_unit(ndev);
318 
319 	return 0;
320 }
321 
322 /**
323  * nitrox_probe - NITROX Initialization function.
324  * @pdev: PCI device information struct
325  * @id: entry in nitrox_pci_tbl
326  *
327  * Return: 0, if the driver is bound to the device, or
328  *         a negative error if there is failure.
329  */
330 static int nitrox_probe(struct pci_dev *pdev,
331 			const struct pci_device_id *id)
332 {
333 	struct nitrox_device *ndev;
334 	int err;
335 
336 	dev_info_once(&pdev->dev, "%s driver version %s\n",
337 		      nitrox_driver_name, DRIVER_VERSION);
338 
339 	err = pci_enable_device_mem(pdev);
340 	if (err)
341 		return err;
342 
343 	/* do FLR */
344 	err = nitrox_device_flr(pdev);
345 	if (err) {
346 		dev_err(&pdev->dev, "FLR failed\n");
347 		pci_disable_device(pdev);
348 		return err;
349 	}
350 
351 	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
352 		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
353 	} else {
354 		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
355 		if (err) {
356 			dev_err(&pdev->dev, "DMA configuration failed\n");
357 			pci_disable_device(pdev);
358 			return err;
359 		}
360 	}
361 
362 	err = pci_request_mem_regions(pdev, nitrox_driver_name);
363 	if (err) {
364 		pci_disable_device(pdev);
365 		return err;
366 	}
367 	pci_set_master(pdev);
368 
369 	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
370 	if (!ndev) {
371 		err = -ENOMEM;
372 		goto ndev_fail;
373 	}
374 
375 	pci_set_drvdata(pdev, ndev);
376 	ndev->pdev = pdev;
377 
378 	/* add to device list */
379 	nitrox_add_to_devlist(ndev);
380 
381 	ndev->hw.vendor_id = pdev->vendor;
382 	ndev->hw.device_id = pdev->device;
383 	ndev->hw.revision_id = pdev->revision;
384 	/* command timeout in jiffies */
385 	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
386 	ndev->node = dev_to_node(&pdev->dev);
387 	if (ndev->node == NUMA_NO_NODE)
388 		ndev->node = 0;
389 
390 	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
391 				 pci_resource_len(pdev, 0));
392 	if (!ndev->bar_addr) {
393 		err = -EIO;
394 		goto ioremap_err;
395 	}
396 	/* allocate command queus based on cpus, max queues are 64 */
397 	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
398 	ndev->qlen = qlen;
399 
400 	err = nitrox_pf_sw_init(ndev);
401 	if (err)
402 		goto ioremap_err;
403 
404 	err = nitrox_pf_hw_init(ndev);
405 	if (err)
406 		goto pf_hw_fail;
407 
408 	err = nitrox_debugfs_init(ndev);
409 	if (err)
410 		goto pf_hw_fail;
411 
412 	/* clear the statistics */
413 	atomic64_set(&ndev->stats.posted, 0);
414 	atomic64_set(&ndev->stats.completed, 0);
415 	atomic64_set(&ndev->stats.dropped, 0);
416 
417 	atomic_set(&ndev->state, __NDEV_READY);
418 	/* barrier to sync with other cpus */
419 	smp_mb__after_atomic();
420 
421 	err = nitrox_crypto_register();
422 	if (err)
423 		goto crypto_fail;
424 
425 	return 0;
426 
427 crypto_fail:
428 	nitrox_debugfs_exit(ndev);
429 	atomic_set(&ndev->state, __NDEV_NOT_READY);
430 	/* barrier to sync with other cpus */
431 	smp_mb__after_atomic();
432 pf_hw_fail:
433 	nitrox_pf_sw_cleanup(ndev);
434 ioremap_err:
435 	nitrox_remove_from_devlist(ndev);
436 	kfree(ndev);
437 	pci_set_drvdata(pdev, NULL);
438 ndev_fail:
439 	pci_release_mem_regions(pdev);
440 	pci_disable_device(pdev);
441 	return err;
442 }
443 
444 /**
445  * nitrox_remove - Unbind the driver from the device.
446  * @pdev: PCI device information struct
447  */
448 static void nitrox_remove(struct pci_dev *pdev)
449 {
450 	struct nitrox_device *ndev = pci_get_drvdata(pdev);
451 
452 	if (!ndev)
453 		return;
454 
455 	if (!refcount_dec_and_test(&ndev->refcnt)) {
456 		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
457 			refcount_read(&ndev->refcnt));
458 		return;
459 	}
460 
461 	dev_info(DEV(ndev), "Removing Device %x:%x\n",
462 		 ndev->hw.vendor_id, ndev->hw.device_id);
463 
464 	atomic_set(&ndev->state, __NDEV_NOT_READY);
465 	/* barrier to sync with other cpus */
466 	smp_mb__after_atomic();
467 
468 	nitrox_remove_from_devlist(ndev);
469 
470 #ifdef CONFIG_PCI_IOV
471 	/* disable SR-IOV */
472 	nitrox_sriov_configure(pdev, 0);
473 #endif
474 	nitrox_crypto_unregister();
475 	nitrox_debugfs_exit(ndev);
476 	nitrox_pf_sw_cleanup(ndev);
477 
478 	iounmap(ndev->bar_addr);
479 	kfree(ndev);
480 
481 	pci_set_drvdata(pdev, NULL);
482 	pci_release_mem_regions(pdev);
483 	pci_disable_device(pdev);
484 }
485 
486 static void nitrox_shutdown(struct pci_dev *pdev)
487 {
488 	pci_set_drvdata(pdev, NULL);
489 	pci_release_mem_regions(pdev);
490 	pci_disable_device(pdev);
491 }
492 
493 static struct pci_driver nitrox_driver = {
494 	.name = nitrox_driver_name,
495 	.id_table = nitrox_pci_tbl,
496 	.probe = nitrox_probe,
497 	.remove	= nitrox_remove,
498 	.shutdown = nitrox_shutdown,
499 #ifdef CONFIG_PCI_IOV
500 	.sriov_configure = nitrox_sriov_configure,
501 #endif
502 };
503 
504 module_pci_driver(nitrox_driver);
505 
506 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
507 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
508 MODULE_LICENSE("GPL");
509 MODULE_VERSION(DRIVER_VERSION);
510 MODULE_FIRMWARE(SE_FW);
511