1 #include <linux/aer.h> 2 #include <linux/delay.h> 3 #include <linux/debugfs.h> 4 #include <linux/firmware.h> 5 #include <linux/list.h> 6 #include <linux/module.h> 7 #include <linux/mutex.h> 8 #include <linux/pci.h> 9 #include <linux/pci_ids.h> 10 11 #include "nitrox_dev.h" 12 #include "nitrox_common.h" 13 #include "nitrox_csr.h" 14 15 #define CNN55XX_DEV_ID 0x12 16 #define MAX_PF_QUEUES 64 17 #define UCODE_HLEN 48 18 #define SE_GROUP 0 19 20 #define DRIVER_VERSION "1.0" 21 /* SE microcode */ 22 #define SE_FW "cnn55xx_se.fw" 23 24 static const char nitrox_driver_name[] = "CNN55XX"; 25 26 static LIST_HEAD(ndevlist); 27 static DEFINE_MUTEX(devlist_lock); 28 static unsigned int num_devices; 29 30 /** 31 * nitrox_pci_tbl - PCI Device ID Table 32 */ 33 static const struct pci_device_id nitrox_pci_tbl[] = { 34 {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, 35 /* required last entry */ 36 {0, } 37 }; 38 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); 39 40 static unsigned int qlen = DEFAULT_CMD_QLEN; 41 module_param(qlen, uint, 0644); 42 MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); 43 44 /** 45 * struct ucode - Firmware Header 46 * @id: microcode ID 47 * @version: firmware version 48 * @code_size: code section size 49 * @raz: alignment 50 * @code: code section 51 */ 52 struct ucode { 53 u8 id; 54 char version[VERSION_LEN - 1]; 55 __be32 code_size; 56 u8 raz[12]; 57 u64 code[0]; 58 }; 59 60 /** 61 * write_to_ucd_unit - Write Firmware to NITROX UCD unit 62 */ 63 static void write_to_ucd_unit(struct nitrox_device *ndev, 64 struct ucode *ucode) 65 { 66 u32 code_size = be32_to_cpu(ucode->code_size) * 2; 67 u64 offset, data; 68 int i = 0; 69 70 /* 71 * UCD structure 72 * 73 * ------------- 74 * | BLK 7 | 75 * ------------- 76 * | BLK 6 | 77 * ------------- 78 * | ... | 79 * ------------- 80 * | BLK 0 | 81 * ------------- 82 * Total of 8 blocks, each size 32KB 83 */ 84 85 /* set the block number */ 86 offset = UCD_UCODE_LOAD_BLOCK_NUM; 87 nitrox_write_csr(ndev, offset, 0); 88 89 code_size = roundup(code_size, 8); 90 while (code_size) { 91 data = ucode->code[i]; 92 /* write 8 bytes at a time */ 93 offset = UCD_UCODE_LOAD_IDX_DATAX(i); 94 nitrox_write_csr(ndev, offset, data); 95 code_size -= 8; 96 i++; 97 } 98 99 /* put all SE cores in group 0 */ 100 offset = POM_GRP_EXECMASKX(SE_GROUP); 101 nitrox_write_csr(ndev, offset, (~0ULL)); 102 103 for (i = 0; i < ndev->hw.se_cores; i++) { 104 /* 105 * write block number and firware length 106 * bit:<2:0> block number 107 * bit:3 is set SE uses 32KB microcode 108 * bit:3 is clear SE uses 64KB microcode 109 */ 110 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); 111 nitrox_write_csr(ndev, offset, 0x8); 112 } 113 usleep_range(300, 400); 114 } 115 116 static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name) 117 { 118 const struct firmware *fw; 119 struct ucode *ucode; 120 int ret; 121 122 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); 123 124 ret = request_firmware(&fw, fw_name, DEV(ndev)); 125 if (ret < 0) { 126 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); 127 return ret; 128 } 129 130 ucode = (struct ucode *)fw->data; 131 /* copy the firmware version */ 132 memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2)); 133 ndev->hw.fw_name[VERSION_LEN - 1] = '\0'; 134 135 write_to_ucd_unit(ndev, ucode); 136 release_firmware(fw); 137 138 set_bit(NITROX_UCODE_LOADED, &ndev->status); 139 /* barrier to sync with other cpus */ 140 smp_mb__after_atomic(); 141 return 0; 142 } 143 144 /** 145 * nitrox_add_to_devlist - add NITROX device to global device list 146 * @ndev: NITROX device 147 */ 148 static int nitrox_add_to_devlist(struct nitrox_device *ndev) 149 { 150 struct nitrox_device *dev; 151 int ret = 0; 152 153 INIT_LIST_HEAD(&ndev->list); 154 refcount_set(&ndev->refcnt, 1); 155 156 mutex_lock(&devlist_lock); 157 list_for_each_entry(dev, &ndevlist, list) { 158 if (dev == ndev) { 159 ret = -EEXIST; 160 goto unlock; 161 } 162 } 163 ndev->idx = num_devices++; 164 list_add_tail(&ndev->list, &ndevlist); 165 unlock: 166 mutex_unlock(&devlist_lock); 167 return ret; 168 } 169 170 /** 171 * nitrox_remove_from_devlist - remove NITROX device from 172 * global device list 173 * @ndev: NITROX device 174 */ 175 static void nitrox_remove_from_devlist(struct nitrox_device *ndev) 176 { 177 mutex_lock(&devlist_lock); 178 list_del(&ndev->list); 179 num_devices--; 180 mutex_unlock(&devlist_lock); 181 } 182 183 struct nitrox_device *nitrox_get_first_device(void) 184 { 185 struct nitrox_device *ndev = NULL; 186 187 mutex_lock(&devlist_lock); 188 list_for_each_entry(ndev, &ndevlist, list) { 189 if (nitrox_ready(ndev)) 190 break; 191 } 192 mutex_unlock(&devlist_lock); 193 if (!ndev) 194 return NULL; 195 196 refcount_inc(&ndev->refcnt); 197 /* barrier to sync with other cpus */ 198 smp_mb__after_atomic(); 199 return ndev; 200 } 201 202 void nitrox_put_device(struct nitrox_device *ndev) 203 { 204 if (!ndev) 205 return; 206 207 refcount_dec(&ndev->refcnt); 208 /* barrier to sync with other cpus */ 209 smp_mb__after_atomic(); 210 } 211 212 static int nitrox_reset_device(struct pci_dev *pdev) 213 { 214 int pos = 0; 215 216 pos = pci_save_state(pdev); 217 if (pos) { 218 dev_err(&pdev->dev, "Failed to save pci state\n"); 219 return -ENOMEM; 220 } 221 222 pos = pci_pcie_cap(pdev); 223 if (!pos) 224 return -ENOTTY; 225 226 if (!pci_wait_for_pending_transaction(pdev)) 227 dev_err(&pdev->dev, "waiting for pending transaction\n"); 228 229 pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); 230 msleep(100); 231 pci_restore_state(pdev); 232 233 return 0; 234 } 235 236 static int nitrox_pf_sw_init(struct nitrox_device *ndev) 237 { 238 int err; 239 240 err = nitrox_common_sw_init(ndev); 241 if (err) 242 return err; 243 244 err = nitrox_pf_init_isr(ndev); 245 if (err) 246 nitrox_common_sw_cleanup(ndev); 247 248 return err; 249 } 250 251 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) 252 { 253 nitrox_pf_cleanup_isr(ndev); 254 nitrox_common_sw_cleanup(ndev); 255 } 256 257 /** 258 * nitrox_bist_check - Check NITORX BIST registers status 259 * @ndev: NITROX device 260 */ 261 static int nitrox_bist_check(struct nitrox_device *ndev) 262 { 263 u64 value = 0; 264 int i; 265 266 for (i = 0; i < NR_CLUSTERS; i++) { 267 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); 268 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); 269 } 270 value += nitrox_read_csr(ndev, UCD_BIST_STATUS); 271 value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); 272 value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); 273 value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); 274 value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); 275 value += nitrox_read_csr(ndev, POM_BIST_REG); 276 value += nitrox_read_csr(ndev, BMI_BIST_REG); 277 value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); 278 value += nitrox_read_csr(ndev, BMO_BIST_REG); 279 value += nitrox_read_csr(ndev, LBC_BIST_STATUS); 280 value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); 281 if (value) 282 return -EIO; 283 return 0; 284 } 285 286 static void nitrox_get_hwinfo(struct nitrox_device *ndev) 287 { 288 union emu_fuse_map emu_fuse; 289 u64 offset; 290 int i; 291 292 for (i = 0; i < NR_CLUSTERS; i++) { 293 u8 dead_cores; 294 295 offset = EMU_FUSE_MAPX(i); 296 emu_fuse.value = nitrox_read_csr(ndev, offset); 297 if (emu_fuse.s.valid) { 298 dead_cores = hweight32(emu_fuse.s.ae_fuse); 299 ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores; 300 dead_cores = hweight16(emu_fuse.s.se_fuse); 301 ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores; 302 } 303 } 304 } 305 306 static int nitrox_pf_hw_init(struct nitrox_device *ndev) 307 { 308 int err; 309 310 err = nitrox_bist_check(ndev); 311 if (err) { 312 dev_err(&ndev->pdev->dev, "BIST check failed\n"); 313 return err; 314 } 315 /* get cores information */ 316 nitrox_get_hwinfo(ndev); 317 318 nitrox_config_nps_unit(ndev); 319 nitrox_config_pom_unit(ndev); 320 nitrox_config_efl_unit(ndev); 321 /* configure IO units */ 322 nitrox_config_bmi_unit(ndev); 323 nitrox_config_bmo_unit(ndev); 324 /* configure Local Buffer Cache */ 325 nitrox_config_lbc_unit(ndev); 326 nitrox_config_rand_unit(ndev); 327 328 /* load firmware on SE cores */ 329 err = nitrox_load_fw(ndev, SE_FW); 330 if (err) 331 return err; 332 333 nitrox_config_emu_unit(ndev); 334 335 return 0; 336 } 337 338 #if IS_ENABLED(CONFIG_DEBUG_FS) 339 static int registers_show(struct seq_file *s, void *v) 340 { 341 struct nitrox_device *ndev = s->private; 342 u64 offset; 343 344 /* NPS DMA stats */ 345 offset = NPS_STATS_PKT_DMA_RD_CNT; 346 seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n", 347 nitrox_read_csr(ndev, offset)); 348 offset = NPS_STATS_PKT_DMA_WR_CNT; 349 seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n", 350 nitrox_read_csr(ndev, offset)); 351 352 /* BMI/BMO stats */ 353 offset = BMI_NPS_PKT_CNT; 354 seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n", 355 nitrox_read_csr(ndev, offset)); 356 offset = BMO_NPS_SLC_PKT_CNT; 357 seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n", 358 nitrox_read_csr(ndev, offset)); 359 360 return 0; 361 } 362 363 static int registers_open(struct inode *inode, struct file *file) 364 { 365 return single_open(file, registers_show, inode->i_private); 366 } 367 368 static const struct file_operations register_fops = { 369 .owner = THIS_MODULE, 370 .open = registers_open, 371 .read = seq_read, 372 .llseek = seq_lseek, 373 .release = single_release, 374 }; 375 376 static int firmware_show(struct seq_file *s, void *v) 377 { 378 struct nitrox_device *ndev = s->private; 379 380 seq_printf(s, "Version: %s\n", ndev->hw.fw_name); 381 return 0; 382 } 383 384 static int firmware_open(struct inode *inode, struct file *file) 385 { 386 return single_open(file, firmware_show, inode->i_private); 387 } 388 389 static const struct file_operations firmware_fops = { 390 .owner = THIS_MODULE, 391 .open = firmware_open, 392 .read = seq_read, 393 .llseek = seq_lseek, 394 .release = single_release, 395 }; 396 397 static int nitrox_show(struct seq_file *s, void *v) 398 { 399 struct nitrox_device *ndev = s->private; 400 401 seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx); 402 seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id); 403 seq_printf(s, " Cores [AE: %u SE: %u]\n", 404 ndev->hw.ae_cores, ndev->hw.se_cores); 405 seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues); 406 seq_printf(s, " Queue length: %u\n", ndev->qlen); 407 seq_printf(s, " Node: %u\n", ndev->node); 408 409 return 0; 410 } 411 412 static int nitrox_open(struct inode *inode, struct file *file) 413 { 414 return single_open(file, nitrox_show, inode->i_private); 415 } 416 417 static const struct file_operations nitrox_fops = { 418 .owner = THIS_MODULE, 419 .open = nitrox_open, 420 .read = seq_read, 421 .llseek = seq_lseek, 422 .release = single_release, 423 }; 424 425 static void nitrox_debugfs_exit(struct nitrox_device *ndev) 426 { 427 debugfs_remove_recursive(ndev->debugfs_dir); 428 ndev->debugfs_dir = NULL; 429 } 430 431 static int nitrox_debugfs_init(struct nitrox_device *ndev) 432 { 433 struct dentry *dir, *f; 434 435 dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 436 if (!dir) 437 return -ENOMEM; 438 439 ndev->debugfs_dir = dir; 440 f = debugfs_create_file("counters", 0400, dir, ndev, ®ister_fops); 441 if (!f) 442 goto err; 443 f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops); 444 if (!f) 445 goto err; 446 f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops); 447 if (!f) 448 goto err; 449 450 return 0; 451 452 err: 453 nitrox_debugfs_exit(ndev); 454 return -ENODEV; 455 } 456 #else 457 static int nitrox_debugfs_init(struct nitrox_device *ndev) 458 { 459 return 0; 460 } 461 462 static void nitrox_debugfs_exit(struct nitrox_device *ndev) 463 { 464 } 465 #endif 466 467 /** 468 * nitrox_probe - NITROX Initialization function. 469 * @pdev: PCI device information struct 470 * @id: entry in nitrox_pci_tbl 471 * 472 * Return: 0, if the driver is bound to the device, or 473 * a negative error if there is failure. 474 */ 475 static int nitrox_probe(struct pci_dev *pdev, 476 const struct pci_device_id *id) 477 { 478 struct nitrox_device *ndev; 479 int err; 480 481 dev_info_once(&pdev->dev, "%s driver version %s\n", 482 nitrox_driver_name, DRIVER_VERSION); 483 484 err = pci_enable_device_mem(pdev); 485 if (err) 486 return err; 487 488 /* do FLR */ 489 err = nitrox_reset_device(pdev); 490 if (err) { 491 dev_err(&pdev->dev, "FLR failed\n"); 492 pci_disable_device(pdev); 493 return err; 494 } 495 496 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 497 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n"); 498 } else { 499 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 500 if (err) { 501 dev_err(&pdev->dev, "DMA configuration failed\n"); 502 pci_disable_device(pdev); 503 return err; 504 } 505 } 506 507 err = pci_request_mem_regions(pdev, nitrox_driver_name); 508 if (err) { 509 pci_disable_device(pdev); 510 return err; 511 } 512 pci_set_master(pdev); 513 514 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); 515 if (!ndev) 516 goto ndev_fail; 517 518 pci_set_drvdata(pdev, ndev); 519 ndev->pdev = pdev; 520 521 /* add to device list */ 522 nitrox_add_to_devlist(ndev); 523 524 ndev->hw.vendor_id = pdev->vendor; 525 ndev->hw.device_id = pdev->device; 526 ndev->hw.revision_id = pdev->revision; 527 /* command timeout in jiffies */ 528 ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); 529 ndev->node = dev_to_node(&pdev->dev); 530 if (ndev->node == NUMA_NO_NODE) 531 ndev->node = 0; 532 533 ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), 534 pci_resource_len(pdev, 0)); 535 if (!ndev->bar_addr) { 536 err = -EIO; 537 goto ioremap_err; 538 } 539 /* allocate command queus based on cpus, max queues are 64 */ 540 ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); 541 ndev->qlen = qlen; 542 543 err = nitrox_pf_sw_init(ndev); 544 if (err) 545 goto ioremap_err; 546 547 err = nitrox_pf_hw_init(ndev); 548 if (err) 549 goto pf_hw_fail; 550 551 err = nitrox_debugfs_init(ndev); 552 if (err) 553 goto pf_hw_fail; 554 555 set_bit(NITROX_READY, &ndev->status); 556 /* barrier to sync with other cpus */ 557 smp_mb__after_atomic(); 558 559 err = nitrox_crypto_register(); 560 if (err) 561 goto crypto_fail; 562 563 return 0; 564 565 crypto_fail: 566 nitrox_debugfs_exit(ndev); 567 clear_bit(NITROX_READY, &ndev->status); 568 /* barrier to sync with other cpus */ 569 smp_mb__after_atomic(); 570 pf_hw_fail: 571 nitrox_pf_sw_cleanup(ndev); 572 ioremap_err: 573 nitrox_remove_from_devlist(ndev); 574 kfree(ndev); 575 pci_set_drvdata(pdev, NULL); 576 ndev_fail: 577 pci_release_mem_regions(pdev); 578 pci_disable_device(pdev); 579 return err; 580 } 581 582 /** 583 * nitrox_remove - Unbind the driver from the device. 584 * @pdev: PCI device information struct 585 */ 586 static void nitrox_remove(struct pci_dev *pdev) 587 { 588 struct nitrox_device *ndev = pci_get_drvdata(pdev); 589 590 if (!ndev) 591 return; 592 593 if (!refcount_dec_and_test(&ndev->refcnt)) { 594 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n", 595 refcount_read(&ndev->refcnt)); 596 return; 597 } 598 599 dev_info(DEV(ndev), "Removing Device %x:%x\n", 600 ndev->hw.vendor_id, ndev->hw.device_id); 601 602 clear_bit(NITROX_READY, &ndev->status); 603 /* barrier to sync with other cpus */ 604 smp_mb__after_atomic(); 605 606 nitrox_remove_from_devlist(ndev); 607 nitrox_crypto_unregister(); 608 nitrox_debugfs_exit(ndev); 609 nitrox_pf_sw_cleanup(ndev); 610 611 iounmap(ndev->bar_addr); 612 kfree(ndev); 613 614 pci_set_drvdata(pdev, NULL); 615 pci_release_mem_regions(pdev); 616 pci_disable_device(pdev); 617 } 618 619 static void nitrox_shutdown(struct pci_dev *pdev) 620 { 621 pci_set_drvdata(pdev, NULL); 622 pci_release_mem_regions(pdev); 623 pci_disable_device(pdev); 624 } 625 626 static struct pci_driver nitrox_driver = { 627 .name = nitrox_driver_name, 628 .id_table = nitrox_pci_tbl, 629 .probe = nitrox_probe, 630 .remove = nitrox_remove, 631 .shutdown = nitrox_shutdown, 632 }; 633 634 module_pci_driver(nitrox_driver); 635 636 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>"); 637 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " "); 638 MODULE_LICENSE("GPL"); 639 MODULE_VERSION(DRIVER_VERSION); 640 MODULE_FIRMWARE(SE_FW); 641