1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/delay.h>
3 #include <linux/firmware.h>
4 #include <linux/list.h>
5 #include <linux/module.h>
6 #include <linux/mutex.h>
7 #include <linux/pci.h>
8 #include <linux/pci_ids.h>
9
10 #include "nitrox_dev.h"
11 #include "nitrox_common.h"
12 #include "nitrox_csr.h"
13 #include "nitrox_hal.h"
14 #include "nitrox_isr.h"
15 #include "nitrox_debugfs.h"
16
17 #define CNN55XX_DEV_ID 0x12
18 #define UCODE_HLEN 48
19 #define DEFAULT_SE_GROUP 0
20 #define DEFAULT_AE_GROUP 0
21
22 #define DRIVER_VERSION "1.2"
23 #define CNN55XX_UCD_BLOCK_SIZE 32768
24 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
25 #define FW_DIR "cavium/"
26 /* SE microcode */
27 #define SE_FW FW_DIR "cnn55xx_se.fw"
28 /* AE microcode */
29 #define AE_FW FW_DIR "cnn55xx_ae.fw"
30
31 static const char nitrox_driver_name[] = "CNN55XX";
32
33 static LIST_HEAD(ndevlist);
34 static DEFINE_MUTEX(devlist_lock);
35 static unsigned int num_devices;
36
37 /*
38 * nitrox_pci_tbl - PCI Device ID Table
39 */
40 static const struct pci_device_id nitrox_pci_tbl[] = {
41 {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
42 /* required last entry */
43 {0, }
44 };
45 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
46
47 static unsigned int qlen = DEFAULT_CMD_QLEN;
48 module_param(qlen, uint, 0644);
49 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
50
51 /**
52 * struct ucode - Firmware Header
53 * @id: microcode ID
54 * @version: firmware version
55 * @code_size: code section size
56 * @raz: alignment
57 * @code: code section
58 */
59 struct ucode {
60 u8 id;
61 char version[VERSION_LEN - 1];
62 __be32 code_size;
63 u8 raz[12];
64 u64 code[];
65 };
66
67 /*
68 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
69 */
write_to_ucd_unit(struct nitrox_device * ndev,u32 ucode_size,u64 * ucode_data,int block_num)70 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
71 u64 *ucode_data, int block_num)
72 {
73 u32 code_size;
74 u64 offset, data;
75 int i = 0;
76
77 /*
78 * UCD structure
79 *
80 * -------------
81 * | BLK 7 |
82 * -------------
83 * | BLK 6 |
84 * -------------
85 * | ... |
86 * -------------
87 * | BLK 0 |
88 * -------------
89 * Total of 8 blocks, each size 32KB
90 */
91
92 /* set the block number */
93 offset = UCD_UCODE_LOAD_BLOCK_NUM;
94 nitrox_write_csr(ndev, offset, block_num);
95
96 code_size = roundup(ucode_size, 16);
97 while (code_size) {
98 data = ucode_data[i];
99 /* write 8 bytes at a time */
100 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
101 nitrox_write_csr(ndev, offset, data);
102 code_size -= 8;
103 i++;
104 }
105
106 usleep_range(300, 400);
107 }
108
nitrox_load_fw(struct nitrox_device * ndev)109 static int nitrox_load_fw(struct nitrox_device *ndev)
110 {
111 const struct firmware *fw;
112 const char *fw_name;
113 struct ucode *ucode;
114 u64 *ucode_data;
115 u64 offset;
116 union ucd_core_eid_ucode_block_num core_2_eid_val;
117 union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
118 union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
119 u32 ucode_size;
120 int ret, i = 0;
121
122 fw_name = SE_FW;
123 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
124
125 ret = request_firmware(&fw, fw_name, DEV(ndev));
126 if (ret < 0) {
127 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
128 return ret;
129 }
130
131 ucode = (struct ucode *)fw->data;
132
133 ucode_size = be32_to_cpu(ucode->code_size) * 2;
134 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
135 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
136 ucode_size, fw_name);
137 release_firmware(fw);
138 return -EINVAL;
139 }
140 ucode_data = ucode->code;
141
142 /* copy the firmware version */
143 memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
144 ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
145
146 /* Load SE Firmware on UCD Block 0 */
147 write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
148
149 release_firmware(fw);
150
151 /* put all SE cores in DEFAULT_SE_GROUP */
152 offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
153 nitrox_write_csr(ndev, offset, (~0ULL));
154
155 /* write block number and firmware length
156 * bit:<2:0> block number
157 * bit:3 is set SE uses 32KB microcode
158 * bit:3 is clear SE uses 64KB microcode
159 */
160 core_2_eid_val.value = 0ULL;
161 core_2_eid_val.ucode_blk = 0;
162 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
163 core_2_eid_val.ucode_len = 1;
164 else
165 core_2_eid_val.ucode_len = 0;
166
167 for (i = 0; i < ndev->hw.se_cores; i++) {
168 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
169 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
170 }
171
172
173 fw_name = AE_FW;
174 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
175
176 ret = request_firmware(&fw, fw_name, DEV(ndev));
177 if (ret < 0) {
178 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
179 return ret;
180 }
181
182 ucode = (struct ucode *)fw->data;
183
184 ucode_size = be32_to_cpu(ucode->code_size) * 2;
185 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
186 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
187 ucode_size, fw_name);
188 release_firmware(fw);
189 return -EINVAL;
190 }
191 ucode_data = ucode->code;
192
193 /* copy the firmware version */
194 memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
195 ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
196
197 /* Load AE Firmware on UCD Block 2 */
198 write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
199
200 release_firmware(fw);
201
202 /* put all AE cores in DEFAULT_AE_GROUP */
203 offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
204 aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
205 nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
206 offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
207 aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
208 nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
209
210 /* write block number and firmware length
211 * bit:<2:0> block number
212 * bit:3 is set AE uses 32KB microcode
213 * bit:3 is clear AE uses 64KB microcode
214 */
215 core_2_eid_val.value = 0ULL;
216 core_2_eid_val.ucode_blk = 2;
217 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
218 core_2_eid_val.ucode_len = 1;
219 else
220 core_2_eid_val.ucode_len = 0;
221
222 for (i = 0; i < ndev->hw.ae_cores; i++) {
223 offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
224 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
225 }
226
227 return 0;
228 }
229
230 /**
231 * nitrox_add_to_devlist - add NITROX device to global device list
232 * @ndev: NITROX device
233 */
nitrox_add_to_devlist(struct nitrox_device * ndev)234 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
235 {
236 struct nitrox_device *dev;
237 int ret = 0;
238
239 INIT_LIST_HEAD(&ndev->list);
240 refcount_set(&ndev->refcnt, 1);
241
242 mutex_lock(&devlist_lock);
243 list_for_each_entry(dev, &ndevlist, list) {
244 if (dev == ndev) {
245 ret = -EEXIST;
246 goto unlock;
247 }
248 }
249 ndev->idx = num_devices++;
250 list_add_tail(&ndev->list, &ndevlist);
251 unlock:
252 mutex_unlock(&devlist_lock);
253 return ret;
254 }
255
256 /**
257 * nitrox_remove_from_devlist - remove NITROX device from
258 * global device list
259 * @ndev: NITROX device
260 */
nitrox_remove_from_devlist(struct nitrox_device * ndev)261 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
262 {
263 mutex_lock(&devlist_lock);
264 list_del(&ndev->list);
265 num_devices--;
266 mutex_unlock(&devlist_lock);
267 }
268
nitrox_get_first_device(void)269 struct nitrox_device *nitrox_get_first_device(void)
270 {
271 struct nitrox_device *ndev = NULL, *iter;
272
273 mutex_lock(&devlist_lock);
274 list_for_each_entry(iter, &ndevlist, list) {
275 if (nitrox_ready(iter)) {
276 ndev = iter;
277 break;
278 }
279 }
280 mutex_unlock(&devlist_lock);
281 if (!ndev)
282 return NULL;
283
284 refcount_inc(&ndev->refcnt);
285 /* barrier to sync with other cpus */
286 smp_mb__after_atomic();
287 return ndev;
288 }
289
nitrox_put_device(struct nitrox_device * ndev)290 void nitrox_put_device(struct nitrox_device *ndev)
291 {
292 if (!ndev)
293 return;
294
295 refcount_dec(&ndev->refcnt);
296 /* barrier to sync with other cpus */
297 smp_mb__after_atomic();
298 }
299
nitrox_device_flr(struct pci_dev * pdev)300 static int nitrox_device_flr(struct pci_dev *pdev)
301 {
302 int pos = 0;
303
304 pos = pci_save_state(pdev);
305 if (pos) {
306 dev_err(&pdev->dev, "Failed to save pci state\n");
307 return -ENOMEM;
308 }
309
310 pcie_reset_flr(pdev, PCI_RESET_DO_RESET);
311
312 pci_restore_state(pdev);
313
314 return 0;
315 }
316
nitrox_pf_sw_init(struct nitrox_device * ndev)317 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
318 {
319 int err;
320
321 err = nitrox_common_sw_init(ndev);
322 if (err)
323 return err;
324
325 err = nitrox_register_interrupts(ndev);
326 if (err)
327 nitrox_common_sw_cleanup(ndev);
328
329 return err;
330 }
331
nitrox_pf_sw_cleanup(struct nitrox_device * ndev)332 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
333 {
334 nitrox_unregister_interrupts(ndev);
335 nitrox_common_sw_cleanup(ndev);
336 }
337
338 /**
339 * nitrox_bist_check - Check NITROX BIST registers status
340 * @ndev: NITROX device
341 */
nitrox_bist_check(struct nitrox_device * ndev)342 static int nitrox_bist_check(struct nitrox_device *ndev)
343 {
344 u64 value = 0;
345 int i;
346
347 for (i = 0; i < NR_CLUSTERS; i++) {
348 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
349 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
350 }
351 value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
352 value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
353 value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
354 value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
355 value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
356 value += nitrox_read_csr(ndev, POM_BIST_REG);
357 value += nitrox_read_csr(ndev, BMI_BIST_REG);
358 value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
359 value += nitrox_read_csr(ndev, BMO_BIST_REG);
360 value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
361 value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
362 if (value)
363 return -EIO;
364 return 0;
365 }
366
nitrox_pf_hw_init(struct nitrox_device * ndev)367 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
368 {
369 int err;
370
371 err = nitrox_bist_check(ndev);
372 if (err) {
373 dev_err(&ndev->pdev->dev, "BIST check failed\n");
374 return err;
375 }
376 /* get cores information */
377 nitrox_get_hwinfo(ndev);
378
379 nitrox_config_nps_core_unit(ndev);
380 nitrox_config_aqm_unit(ndev);
381 nitrox_config_nps_pkt_unit(ndev);
382 nitrox_config_pom_unit(ndev);
383 nitrox_config_efl_unit(ndev);
384 /* configure IO units */
385 nitrox_config_bmi_unit(ndev);
386 nitrox_config_bmo_unit(ndev);
387 /* configure Local Buffer Cache */
388 nitrox_config_lbc_unit(ndev);
389 nitrox_config_rand_unit(ndev);
390
391 /* load firmware on cores */
392 err = nitrox_load_fw(ndev);
393 if (err)
394 return err;
395
396 nitrox_config_emu_unit(ndev);
397
398 return 0;
399 }
400
401 /**
402 * nitrox_probe - NITROX Initialization function.
403 * @pdev: PCI device information struct
404 * @id: entry in nitrox_pci_tbl
405 *
406 * Return: 0, if the driver is bound to the device, or
407 * a negative error if there is failure.
408 */
nitrox_probe(struct pci_dev * pdev,const struct pci_device_id * id)409 static int nitrox_probe(struct pci_dev *pdev,
410 const struct pci_device_id *id)
411 {
412 struct nitrox_device *ndev;
413 int err;
414
415 dev_info_once(&pdev->dev, "%s driver version %s\n",
416 nitrox_driver_name, DRIVER_VERSION);
417
418 err = pci_enable_device_mem(pdev);
419 if (err)
420 return err;
421
422 /* do FLR */
423 err = nitrox_device_flr(pdev);
424 if (err) {
425 dev_err(&pdev->dev, "FLR failed\n");
426 goto flr_fail;
427 }
428
429 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
430 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
431 } else {
432 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
433 if (err) {
434 dev_err(&pdev->dev, "DMA configuration failed\n");
435 goto flr_fail;
436 }
437 }
438
439 err = pci_request_mem_regions(pdev, nitrox_driver_name);
440 if (err)
441 goto flr_fail;
442 pci_set_master(pdev);
443
444 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
445 if (!ndev) {
446 err = -ENOMEM;
447 goto ndev_fail;
448 }
449
450 pci_set_drvdata(pdev, ndev);
451 ndev->pdev = pdev;
452
453 /* add to device list */
454 nitrox_add_to_devlist(ndev);
455
456 ndev->hw.vendor_id = pdev->vendor;
457 ndev->hw.device_id = pdev->device;
458 ndev->hw.revision_id = pdev->revision;
459 /* command timeout in jiffies */
460 ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
461 ndev->node = dev_to_node(&pdev->dev);
462 if (ndev->node == NUMA_NO_NODE)
463 ndev->node = 0;
464
465 ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
466 pci_resource_len(pdev, 0));
467 if (!ndev->bar_addr) {
468 err = -EIO;
469 goto ioremap_err;
470 }
471 /* allocate command queus based on cpus, max queues are 64 */
472 ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
473 ndev->qlen = qlen;
474
475 err = nitrox_pf_sw_init(ndev);
476 if (err)
477 goto pf_sw_fail;
478
479 err = nitrox_pf_hw_init(ndev);
480 if (err)
481 goto pf_hw_fail;
482
483 nitrox_debugfs_init(ndev);
484
485 /* clear the statistics */
486 atomic64_set(&ndev->stats.posted, 0);
487 atomic64_set(&ndev->stats.completed, 0);
488 atomic64_set(&ndev->stats.dropped, 0);
489
490 atomic_set(&ndev->state, __NDEV_READY);
491 /* barrier to sync with other cpus */
492 smp_mb__after_atomic();
493
494 err = nitrox_crypto_register();
495 if (err)
496 goto crypto_fail;
497
498 return 0;
499
500 crypto_fail:
501 nitrox_debugfs_exit(ndev);
502 atomic_set(&ndev->state, __NDEV_NOT_READY);
503 /* barrier to sync with other cpus */
504 smp_mb__after_atomic();
505 pf_hw_fail:
506 nitrox_pf_sw_cleanup(ndev);
507 pf_sw_fail:
508 iounmap(ndev->bar_addr);
509 ioremap_err:
510 nitrox_remove_from_devlist(ndev);
511 kfree(ndev);
512 pci_set_drvdata(pdev, NULL);
513 ndev_fail:
514 pci_release_mem_regions(pdev);
515 flr_fail:
516 pci_disable_device(pdev);
517 return err;
518 }
519
520 /**
521 * nitrox_remove - Unbind the driver from the device.
522 * @pdev: PCI device information struct
523 */
nitrox_remove(struct pci_dev * pdev)524 static void nitrox_remove(struct pci_dev *pdev)
525 {
526 struct nitrox_device *ndev = pci_get_drvdata(pdev);
527
528 if (!ndev)
529 return;
530
531 if (!refcount_dec_and_test(&ndev->refcnt)) {
532 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
533 refcount_read(&ndev->refcnt));
534 return;
535 }
536
537 dev_info(DEV(ndev), "Removing Device %x:%x\n",
538 ndev->hw.vendor_id, ndev->hw.device_id);
539
540 atomic_set(&ndev->state, __NDEV_NOT_READY);
541 /* barrier to sync with other cpus */
542 smp_mb__after_atomic();
543
544 nitrox_remove_from_devlist(ndev);
545
546 /* disable SR-IOV */
547 nitrox_sriov_configure(pdev, 0);
548 nitrox_crypto_unregister();
549 nitrox_debugfs_exit(ndev);
550 nitrox_pf_sw_cleanup(ndev);
551
552 iounmap(ndev->bar_addr);
553 kfree(ndev);
554
555 pci_set_drvdata(pdev, NULL);
556 pci_release_mem_regions(pdev);
557 pci_disable_device(pdev);
558 }
559
nitrox_shutdown(struct pci_dev * pdev)560 static void nitrox_shutdown(struct pci_dev *pdev)
561 {
562 pci_set_drvdata(pdev, NULL);
563 pci_release_mem_regions(pdev);
564 pci_disable_device(pdev);
565 }
566
567 static struct pci_driver nitrox_driver = {
568 .name = nitrox_driver_name,
569 .id_table = nitrox_pci_tbl,
570 .probe = nitrox_probe,
571 .remove = nitrox_remove,
572 .shutdown = nitrox_shutdown,
573 .sriov_configure = nitrox_sriov_configure,
574 };
575
576 module_pci_driver(nitrox_driver);
577
578 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
579 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
580 MODULE_LICENSE("GPL");
581 MODULE_VERSION(DRIVER_VERSION);
582 MODULE_FIRMWARE(SE_FW);
583