1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/aer.h> 3 #include <linux/delay.h> 4 #include <linux/firmware.h> 5 #include <linux/list.h> 6 #include <linux/module.h> 7 #include <linux/mutex.h> 8 #include <linux/pci.h> 9 #include <linux/pci_ids.h> 10 11 #include "nitrox_dev.h" 12 #include "nitrox_common.h" 13 #include "nitrox_csr.h" 14 #include "nitrox_hal.h" 15 #include "nitrox_isr.h" 16 #include "nitrox_debugfs.h" 17 18 #define CNN55XX_DEV_ID 0x12 19 #define UCODE_HLEN 48 20 #define DEFAULT_SE_GROUP 0 21 #define DEFAULT_AE_GROUP 0 22 23 #define DRIVER_VERSION "1.2" 24 #define CNN55XX_UCD_BLOCK_SIZE 32768 25 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2) 26 #define FW_DIR "cavium/" 27 /* SE microcode */ 28 #define SE_FW FW_DIR "cnn55xx_se.fw" 29 /* AE microcode */ 30 #define AE_FW FW_DIR "cnn55xx_ae.fw" 31 32 static const char nitrox_driver_name[] = "CNN55XX"; 33 34 static LIST_HEAD(ndevlist); 35 static DEFINE_MUTEX(devlist_lock); 36 static unsigned int num_devices; 37 38 /** 39 * nitrox_pci_tbl - PCI Device ID Table 40 */ 41 static const struct pci_device_id nitrox_pci_tbl[] = { 42 {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, 43 /* required last entry */ 44 {0, } 45 }; 46 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); 47 48 static unsigned int qlen = DEFAULT_CMD_QLEN; 49 module_param(qlen, uint, 0644); 50 MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); 51 52 /** 53 * struct ucode - Firmware Header 54 * @id: microcode ID 55 * @version: firmware version 56 * @code_size: code section size 57 * @raz: alignment 58 * @code: code section 59 */ 60 struct ucode { 61 u8 id; 62 char version[VERSION_LEN - 1]; 63 __be32 code_size; 64 u8 raz[12]; 65 u64 code[]; 66 }; 67 68 /** 69 * write_to_ucd_unit - Write Firmware to NITROX UCD unit 70 */ 71 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size, 72 u64 *ucode_data, int block_num) 73 { 74 u32 code_size; 75 u64 offset, data; 76 int i = 0; 77 78 /* 79 * UCD structure 80 * 81 * ------------- 82 * | BLK 7 | 83 * ------------- 84 * | BLK 6 | 85 * ------------- 86 * | ... | 87 * ------------- 88 * | BLK 0 | 89 * ------------- 90 * Total of 8 blocks, each size 32KB 91 */ 92 93 /* set the block number */ 94 offset = UCD_UCODE_LOAD_BLOCK_NUM; 95 nitrox_write_csr(ndev, offset, block_num); 96 97 code_size = roundup(ucode_size, 16); 98 while (code_size) { 99 data = ucode_data[i]; 100 /* write 8 bytes at a time */ 101 offset = UCD_UCODE_LOAD_IDX_DATAX(i); 102 nitrox_write_csr(ndev, offset, data); 103 code_size -= 8; 104 i++; 105 } 106 107 usleep_range(300, 400); 108 } 109 110 static int nitrox_load_fw(struct nitrox_device *ndev) 111 { 112 const struct firmware *fw; 113 const char *fw_name; 114 struct ucode *ucode; 115 u64 *ucode_data; 116 u64 offset; 117 union ucd_core_eid_ucode_block_num core_2_eid_val; 118 union aqm_grp_execmsk_lo aqm_grp_execmask_lo; 119 union aqm_grp_execmsk_hi aqm_grp_execmask_hi; 120 u32 ucode_size; 121 int ret, i = 0; 122 123 fw_name = SE_FW; 124 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); 125 126 ret = request_firmware(&fw, fw_name, DEV(ndev)); 127 if (ret < 0) { 128 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); 129 return ret; 130 } 131 132 ucode = (struct ucode *)fw->data; 133 134 ucode_size = be32_to_cpu(ucode->code_size) * 2; 135 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { 136 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n", 137 ucode_size, fw_name); 138 release_firmware(fw); 139 return -EINVAL; 140 } 141 ucode_data = ucode->code; 142 143 /* copy the firmware version */ 144 memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2)); 145 ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0'; 146 147 /* Load SE Firmware on UCD Block 0 */ 148 write_to_ucd_unit(ndev, ucode_size, ucode_data, 0); 149 150 release_firmware(fw); 151 152 /* put all SE cores in DEFAULT_SE_GROUP */ 153 offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP); 154 nitrox_write_csr(ndev, offset, (~0ULL)); 155 156 /* write block number and firmware length 157 * bit:<2:0> block number 158 * bit:3 is set SE uses 32KB microcode 159 * bit:3 is clear SE uses 64KB microcode 160 */ 161 core_2_eid_val.value = 0ULL; 162 core_2_eid_val.ucode_blk = 0; 163 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) 164 core_2_eid_val.ucode_len = 1; 165 else 166 core_2_eid_val.ucode_len = 0; 167 168 for (i = 0; i < ndev->hw.se_cores; i++) { 169 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); 170 nitrox_write_csr(ndev, offset, core_2_eid_val.value); 171 } 172 173 174 fw_name = AE_FW; 175 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); 176 177 ret = request_firmware(&fw, fw_name, DEV(ndev)); 178 if (ret < 0) { 179 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); 180 return ret; 181 } 182 183 ucode = (struct ucode *)fw->data; 184 185 ucode_size = be32_to_cpu(ucode->code_size) * 2; 186 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) { 187 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n", 188 ucode_size, fw_name); 189 release_firmware(fw); 190 return -EINVAL; 191 } 192 ucode_data = ucode->code; 193 194 /* copy the firmware version */ 195 memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2)); 196 ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0'; 197 198 /* Load AE Firmware on UCD Block 2 */ 199 write_to_ucd_unit(ndev, ucode_size, ucode_data, 2); 200 201 release_firmware(fw); 202 203 /* put all AE cores in DEFAULT_AE_GROUP */ 204 offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP); 205 aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL; 206 nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value); 207 offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP); 208 aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL; 209 nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value); 210 211 /* write block number and firmware length 212 * bit:<2:0> block number 213 * bit:3 is set AE uses 32KB microcode 214 * bit:3 is clear AE uses 64KB microcode 215 */ 216 core_2_eid_val.value = 0ULL; 217 core_2_eid_val.ucode_blk = 2; 218 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE) 219 core_2_eid_val.ucode_len = 1; 220 else 221 core_2_eid_val.ucode_len = 0; 222 223 for (i = 0; i < ndev->hw.ae_cores; i++) { 224 offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i); 225 nitrox_write_csr(ndev, offset, core_2_eid_val.value); 226 } 227 228 return 0; 229 } 230 231 /** 232 * nitrox_add_to_devlist - add NITROX device to global device list 233 * @ndev: NITROX device 234 */ 235 static int nitrox_add_to_devlist(struct nitrox_device *ndev) 236 { 237 struct nitrox_device *dev; 238 int ret = 0; 239 240 INIT_LIST_HEAD(&ndev->list); 241 refcount_set(&ndev->refcnt, 1); 242 243 mutex_lock(&devlist_lock); 244 list_for_each_entry(dev, &ndevlist, list) { 245 if (dev == ndev) { 246 ret = -EEXIST; 247 goto unlock; 248 } 249 } 250 ndev->idx = num_devices++; 251 list_add_tail(&ndev->list, &ndevlist); 252 unlock: 253 mutex_unlock(&devlist_lock); 254 return ret; 255 } 256 257 /** 258 * nitrox_remove_from_devlist - remove NITROX device from 259 * global device list 260 * @ndev: NITROX device 261 */ 262 static void nitrox_remove_from_devlist(struct nitrox_device *ndev) 263 { 264 mutex_lock(&devlist_lock); 265 list_del(&ndev->list); 266 num_devices--; 267 mutex_unlock(&devlist_lock); 268 } 269 270 struct nitrox_device *nitrox_get_first_device(void) 271 { 272 struct nitrox_device *ndev; 273 274 mutex_lock(&devlist_lock); 275 list_for_each_entry(ndev, &ndevlist, list) { 276 if (nitrox_ready(ndev)) 277 break; 278 } 279 mutex_unlock(&devlist_lock); 280 if (&ndev->list == &ndevlist) 281 return NULL; 282 283 refcount_inc(&ndev->refcnt); 284 /* barrier to sync with other cpus */ 285 smp_mb__after_atomic(); 286 return ndev; 287 } 288 289 void nitrox_put_device(struct nitrox_device *ndev) 290 { 291 if (!ndev) 292 return; 293 294 refcount_dec(&ndev->refcnt); 295 /* barrier to sync with other cpus */ 296 smp_mb__after_atomic(); 297 } 298 299 static int nitrox_device_flr(struct pci_dev *pdev) 300 { 301 int pos = 0; 302 303 pos = pci_save_state(pdev); 304 if (pos) { 305 dev_err(&pdev->dev, "Failed to save pci state\n"); 306 return -ENOMEM; 307 } 308 309 /* check flr support */ 310 if (pcie_has_flr(pdev)) 311 pcie_flr(pdev); 312 313 pci_restore_state(pdev); 314 315 return 0; 316 } 317 318 static int nitrox_pf_sw_init(struct nitrox_device *ndev) 319 { 320 int err; 321 322 err = nitrox_common_sw_init(ndev); 323 if (err) 324 return err; 325 326 err = nitrox_register_interrupts(ndev); 327 if (err) 328 nitrox_common_sw_cleanup(ndev); 329 330 return err; 331 } 332 333 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) 334 { 335 nitrox_unregister_interrupts(ndev); 336 nitrox_common_sw_cleanup(ndev); 337 } 338 339 /** 340 * nitrox_bist_check - Check NITROX BIST registers status 341 * @ndev: NITROX device 342 */ 343 static int nitrox_bist_check(struct nitrox_device *ndev) 344 { 345 u64 value = 0; 346 int i; 347 348 for (i = 0; i < NR_CLUSTERS; i++) { 349 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); 350 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); 351 } 352 value += nitrox_read_csr(ndev, UCD_BIST_STATUS); 353 value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); 354 value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); 355 value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); 356 value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); 357 value += nitrox_read_csr(ndev, POM_BIST_REG); 358 value += nitrox_read_csr(ndev, BMI_BIST_REG); 359 value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); 360 value += nitrox_read_csr(ndev, BMO_BIST_REG); 361 value += nitrox_read_csr(ndev, LBC_BIST_STATUS); 362 value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); 363 if (value) 364 return -EIO; 365 return 0; 366 } 367 368 static int nitrox_pf_hw_init(struct nitrox_device *ndev) 369 { 370 int err; 371 372 err = nitrox_bist_check(ndev); 373 if (err) { 374 dev_err(&ndev->pdev->dev, "BIST check failed\n"); 375 return err; 376 } 377 /* get cores information */ 378 nitrox_get_hwinfo(ndev); 379 380 nitrox_config_nps_core_unit(ndev); 381 nitrox_config_aqm_unit(ndev); 382 nitrox_config_nps_pkt_unit(ndev); 383 nitrox_config_pom_unit(ndev); 384 nitrox_config_efl_unit(ndev); 385 /* configure IO units */ 386 nitrox_config_bmi_unit(ndev); 387 nitrox_config_bmo_unit(ndev); 388 /* configure Local Buffer Cache */ 389 nitrox_config_lbc_unit(ndev); 390 nitrox_config_rand_unit(ndev); 391 392 /* load firmware on cores */ 393 err = nitrox_load_fw(ndev); 394 if (err) 395 return err; 396 397 nitrox_config_emu_unit(ndev); 398 399 return 0; 400 } 401 402 /** 403 * nitrox_probe - NITROX Initialization function. 404 * @pdev: PCI device information struct 405 * @id: entry in nitrox_pci_tbl 406 * 407 * Return: 0, if the driver is bound to the device, or 408 * a negative error if there is failure. 409 */ 410 static int nitrox_probe(struct pci_dev *pdev, 411 const struct pci_device_id *id) 412 { 413 struct nitrox_device *ndev; 414 int err; 415 416 dev_info_once(&pdev->dev, "%s driver version %s\n", 417 nitrox_driver_name, DRIVER_VERSION); 418 419 err = pci_enable_device_mem(pdev); 420 if (err) 421 return err; 422 423 /* do FLR */ 424 err = nitrox_device_flr(pdev); 425 if (err) { 426 dev_err(&pdev->dev, "FLR failed\n"); 427 pci_disable_device(pdev); 428 return err; 429 } 430 431 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 432 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n"); 433 } else { 434 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 435 if (err) { 436 dev_err(&pdev->dev, "DMA configuration failed\n"); 437 pci_disable_device(pdev); 438 return err; 439 } 440 } 441 442 err = pci_request_mem_regions(pdev, nitrox_driver_name); 443 if (err) { 444 pci_disable_device(pdev); 445 dev_err(&pdev->dev, "Failed to request mem regions!\n"); 446 return err; 447 } 448 pci_set_master(pdev); 449 450 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); 451 if (!ndev) { 452 err = -ENOMEM; 453 goto ndev_fail; 454 } 455 456 pci_set_drvdata(pdev, ndev); 457 ndev->pdev = pdev; 458 459 /* add to device list */ 460 nitrox_add_to_devlist(ndev); 461 462 ndev->hw.vendor_id = pdev->vendor; 463 ndev->hw.device_id = pdev->device; 464 ndev->hw.revision_id = pdev->revision; 465 /* command timeout in jiffies */ 466 ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); 467 ndev->node = dev_to_node(&pdev->dev); 468 if (ndev->node == NUMA_NO_NODE) 469 ndev->node = 0; 470 471 ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), 472 pci_resource_len(pdev, 0)); 473 if (!ndev->bar_addr) { 474 err = -EIO; 475 goto ioremap_err; 476 } 477 /* allocate command queus based on cpus, max queues are 64 */ 478 ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); 479 ndev->qlen = qlen; 480 481 err = nitrox_pf_sw_init(ndev); 482 if (err) 483 goto ioremap_err; 484 485 err = nitrox_pf_hw_init(ndev); 486 if (err) 487 goto pf_hw_fail; 488 489 nitrox_debugfs_init(ndev); 490 491 /* clear the statistics */ 492 atomic64_set(&ndev->stats.posted, 0); 493 atomic64_set(&ndev->stats.completed, 0); 494 atomic64_set(&ndev->stats.dropped, 0); 495 496 atomic_set(&ndev->state, __NDEV_READY); 497 /* barrier to sync with other cpus */ 498 smp_mb__after_atomic(); 499 500 err = nitrox_crypto_register(); 501 if (err) 502 goto crypto_fail; 503 504 return 0; 505 506 crypto_fail: 507 nitrox_debugfs_exit(ndev); 508 atomic_set(&ndev->state, __NDEV_NOT_READY); 509 /* barrier to sync with other cpus */ 510 smp_mb__after_atomic(); 511 pf_hw_fail: 512 nitrox_pf_sw_cleanup(ndev); 513 ioremap_err: 514 nitrox_remove_from_devlist(ndev); 515 kfree(ndev); 516 pci_set_drvdata(pdev, NULL); 517 ndev_fail: 518 pci_release_mem_regions(pdev); 519 pci_disable_device(pdev); 520 return err; 521 } 522 523 /** 524 * nitrox_remove - Unbind the driver from the device. 525 * @pdev: PCI device information struct 526 */ 527 static void nitrox_remove(struct pci_dev *pdev) 528 { 529 struct nitrox_device *ndev = pci_get_drvdata(pdev); 530 531 if (!ndev) 532 return; 533 534 if (!refcount_dec_and_test(&ndev->refcnt)) { 535 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n", 536 refcount_read(&ndev->refcnt)); 537 return; 538 } 539 540 dev_info(DEV(ndev), "Removing Device %x:%x\n", 541 ndev->hw.vendor_id, ndev->hw.device_id); 542 543 atomic_set(&ndev->state, __NDEV_NOT_READY); 544 /* barrier to sync with other cpus */ 545 smp_mb__after_atomic(); 546 547 nitrox_remove_from_devlist(ndev); 548 549 /* disable SR-IOV */ 550 nitrox_sriov_configure(pdev, 0); 551 nitrox_crypto_unregister(); 552 nitrox_debugfs_exit(ndev); 553 nitrox_pf_sw_cleanup(ndev); 554 555 iounmap(ndev->bar_addr); 556 kfree(ndev); 557 558 pci_set_drvdata(pdev, NULL); 559 pci_release_mem_regions(pdev); 560 pci_disable_device(pdev); 561 } 562 563 static void nitrox_shutdown(struct pci_dev *pdev) 564 { 565 pci_set_drvdata(pdev, NULL); 566 pci_release_mem_regions(pdev); 567 pci_disable_device(pdev); 568 } 569 570 static struct pci_driver nitrox_driver = { 571 .name = nitrox_driver_name, 572 .id_table = nitrox_pci_tbl, 573 .probe = nitrox_probe, 574 .remove = nitrox_remove, 575 .shutdown = nitrox_shutdown, 576 .sriov_configure = nitrox_sriov_configure, 577 }; 578 579 module_pci_driver(nitrox_driver); 580 581 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>"); 582 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " "); 583 MODULE_LICENSE("GPL"); 584 MODULE_VERSION(DRIVER_VERSION); 585 MODULE_FIRMWARE(SE_FW); 586