1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <linux/acpi.h> 4 #include <linux/aer.h> 5 #include <linux/bitops.h> 6 #include <linux/debugfs.h> 7 #include <linux/init.h> 8 #include <linux/io.h> 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/pci.h> 12 #include <linux/seq_file.h> 13 #include <linux/topology.h> 14 #include "zip.h" 15 16 #define PCI_DEVICE_ID_ZIP_PF 0xa250 17 #define PCI_DEVICE_ID_ZIP_VF 0xa251 18 19 #define HZIP_VF_NUM 63 20 #define HZIP_QUEUE_NUM_V1 4096 21 #define HZIP_QUEUE_NUM_V2 1024 22 23 #define HZIP_CLOCK_GATE_CTRL 0x301004 24 #define COMP0_ENABLE BIT(0) 25 #define COMP1_ENABLE BIT(1) 26 #define DECOMP0_ENABLE BIT(2) 27 #define DECOMP1_ENABLE BIT(3) 28 #define DECOMP2_ENABLE BIT(4) 29 #define DECOMP3_ENABLE BIT(5) 30 #define DECOMP4_ENABLE BIT(6) 31 #define DECOMP5_ENABLE BIT(7) 32 #define ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ 33 DECOMP0_ENABLE | DECOMP1_ENABLE | \ 34 DECOMP2_ENABLE | DECOMP3_ENABLE | \ 35 DECOMP4_ENABLE | DECOMP5_ENABLE) 36 #define DECOMP_CHECK_ENABLE BIT(16) 37 #define HZIP_FSM_MAX_CNT 0x301008 38 39 #define HZIP_PORT_ARCA_CHE_0 0x301040 40 #define HZIP_PORT_ARCA_CHE_1 0x301044 41 #define HZIP_PORT_AWCA_CHE_0 0x301060 42 #define HZIP_PORT_AWCA_CHE_1 0x301064 43 #define CACHE_ALL_EN 0xffffffff 44 45 #define HZIP_BD_RUSER_32_63 0x301110 46 #define HZIP_SGL_RUSER_32_63 0x30111c 47 #define HZIP_DATA_RUSER_32_63 0x301128 48 #define HZIP_DATA_WUSER_32_63 0x301134 49 #define HZIP_BD_WUSER_32_63 0x301140 50 51 #define HZIP_QM_IDEL_STATUS 0x3040e4 52 53 #define HZIP_CORE_DEBUG_COMP_0 0x302000 54 #define HZIP_CORE_DEBUG_COMP_1 0x303000 55 #define HZIP_CORE_DEBUG_DECOMP_0 0x304000 56 #define HZIP_CORE_DEBUG_DECOMP_1 0x305000 57 #define HZIP_CORE_DEBUG_DECOMP_2 0x306000 58 #define HZIP_CORE_DEBUG_DECOMP_3 0x307000 59 #define HZIP_CORE_DEBUG_DECOMP_4 0x308000 60 #define HZIP_CORE_DEBUG_DECOMP_5 0x309000 61 62 #define HZIP_CORE_INT_SOURCE 0x3010A0 63 #define HZIP_CORE_INT_MASK 0x3010A4 64 #define HZIP_CORE_INT_STATUS 0x3010AC 65 #define HZIP_CORE_INT_STATUS_M_ECC BIT(1) 66 #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 67 #define SRAM_ECC_ERR_NUM_SHIFT 16 68 #define SRAM_ECC_ERR_ADDR_SHIFT 24 69 #define HZIP_CORE_INT_DISABLE 0x000007FF 70 #define HZIP_COMP_CORE_NUM 2 71 #define HZIP_DECOMP_CORE_NUM 6 72 #define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \ 73 HZIP_DECOMP_CORE_NUM) 74 #define HZIP_SQE_SIZE 128 75 #define HZIP_SQ_SIZE (HZIP_SQE_SIZE * QM_Q_DEPTH) 76 #define HZIP_PF_DEF_Q_NUM 64 77 #define HZIP_PF_DEF_Q_BASE 0 78 79 #define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000 80 #define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) 81 82 #define HZIP_NUMA_DISTANCE 100 83 #define HZIP_BUF_SIZE 22 84 85 static const char hisi_zip_name[] = "hisi_zip"; 86 static struct dentry *hzip_debugfs_root; 87 LIST_HEAD(hisi_zip_list); 88 DEFINE_MUTEX(hisi_zip_list_lock); 89 90 #ifdef CONFIG_NUMA 91 static struct hisi_zip *find_zip_device_numa(int node) 92 { 93 struct hisi_zip *zip = NULL; 94 struct hisi_zip *hisi_zip; 95 int min_distance = HZIP_NUMA_DISTANCE; 96 struct device *dev; 97 98 list_for_each_entry(hisi_zip, &hisi_zip_list, list) { 99 dev = &hisi_zip->qm.pdev->dev; 100 if (node_distance(dev->numa_node, node) < min_distance) { 101 zip = hisi_zip; 102 min_distance = node_distance(dev->numa_node, node); 103 } 104 } 105 106 return zip; 107 } 108 #endif 109 110 struct hisi_zip *find_zip_device(int node) 111 { 112 struct hisi_zip *zip = NULL; 113 114 mutex_lock(&hisi_zip_list_lock); 115 #ifdef CONFIG_NUMA 116 zip = find_zip_device_numa(node); 117 #else 118 zip = list_first_entry(&hisi_zip_list, struct hisi_zip, list); 119 #endif 120 mutex_unlock(&hisi_zip_list_lock); 121 122 return zip; 123 } 124 125 struct hisi_zip_hw_error { 126 u32 int_msk; 127 const char *msg; 128 }; 129 130 static const struct hisi_zip_hw_error zip_hw_error[] = { 131 { .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" }, 132 { .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" }, 133 { .int_msk = BIT(2), .msg = "zip_axi_rresp_err" }, 134 { .int_msk = BIT(3), .msg = "zip_axi_bresp_err" }, 135 { .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" }, 136 { .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" }, 137 { .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" }, 138 { .int_msk = BIT(7), .msg = "zip_pre_in_data_err" }, 139 { .int_msk = BIT(8), .msg = "zip_com_inf_err" }, 140 { .int_msk = BIT(9), .msg = "zip_enc_inf_err" }, 141 { .int_msk = BIT(10), .msg = "zip_pre_out_err" }, 142 { /* sentinel */ } 143 }; 144 145 enum ctrl_debug_file_index { 146 HZIP_CURRENT_QM, 147 HZIP_CLEAR_ENABLE, 148 HZIP_DEBUG_FILE_NUM, 149 }; 150 151 static const char * const ctrl_debug_file_name[] = { 152 [HZIP_CURRENT_QM] = "current_qm", 153 [HZIP_CLEAR_ENABLE] = "clear_enable", 154 }; 155 156 struct ctrl_debug_file { 157 enum ctrl_debug_file_index index; 158 spinlock_t lock; 159 struct hisi_zip_ctrl *ctrl; 160 }; 161 162 /* 163 * One ZIP controller has one PF and multiple VFs, some global configurations 164 * which PF has need this structure. 165 * 166 * Just relevant for PF. 167 */ 168 struct hisi_zip_ctrl { 169 u32 num_vfs; 170 struct hisi_zip *hisi_zip; 171 struct dentry *debug_root; 172 struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; 173 }; 174 175 enum { 176 HZIP_COMP_CORE0, 177 HZIP_COMP_CORE1, 178 HZIP_DECOMP_CORE0, 179 HZIP_DECOMP_CORE1, 180 HZIP_DECOMP_CORE2, 181 HZIP_DECOMP_CORE3, 182 HZIP_DECOMP_CORE4, 183 HZIP_DECOMP_CORE5, 184 }; 185 186 static const u64 core_offsets[] = { 187 [HZIP_COMP_CORE0] = 0x302000, 188 [HZIP_COMP_CORE1] = 0x303000, 189 [HZIP_DECOMP_CORE0] = 0x304000, 190 [HZIP_DECOMP_CORE1] = 0x305000, 191 [HZIP_DECOMP_CORE2] = 0x306000, 192 [HZIP_DECOMP_CORE3] = 0x307000, 193 [HZIP_DECOMP_CORE4] = 0x308000, 194 [HZIP_DECOMP_CORE5] = 0x309000, 195 }; 196 197 static struct debugfs_reg32 hzip_dfx_regs[] = { 198 {"HZIP_GET_BD_NUM ", 0x00ull}, 199 {"HZIP_GET_RIGHT_BD ", 0x04ull}, 200 {"HZIP_GET_ERROR_BD ", 0x08ull}, 201 {"HZIP_DONE_BD_NUM ", 0x0cull}, 202 {"HZIP_WORK_CYCLE ", 0x10ull}, 203 {"HZIP_IDLE_CYCLE ", 0x18ull}, 204 {"HZIP_MAX_DELAY ", 0x20ull}, 205 {"HZIP_MIN_DELAY ", 0x24ull}, 206 {"HZIP_AVG_DELAY ", 0x28ull}, 207 {"HZIP_MEM_VISIBLE_DATA ", 0x30ull}, 208 {"HZIP_MEM_VISIBLE_ADDR ", 0x34ull}, 209 {"HZIP_COMSUMED_BYTE ", 0x38ull}, 210 {"HZIP_PRODUCED_BYTE ", 0x40ull}, 211 {"HZIP_COMP_INF ", 0x70ull}, 212 {"HZIP_PRE_OUT ", 0x78ull}, 213 {"HZIP_BD_RD ", 0x7cull}, 214 {"HZIP_BD_WR ", 0x80ull}, 215 {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84ull}, 216 {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88ull}, 217 {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8cull}, 218 {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94ull}, 219 {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9cull}, 220 }; 221 222 static int pf_q_num_set(const char *val, const struct kernel_param *kp) 223 { 224 struct pci_dev *pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, 225 PCI_DEVICE_ID_ZIP_PF, NULL); 226 u32 n, q_num; 227 u8 rev_id; 228 int ret; 229 230 if (!val) 231 return -EINVAL; 232 233 if (!pdev) { 234 q_num = min_t(u32, HZIP_QUEUE_NUM_V1, HZIP_QUEUE_NUM_V2); 235 pr_info("No device found currently, suppose queue number is %d\n", 236 q_num); 237 } else { 238 rev_id = pdev->revision; 239 switch (rev_id) { 240 case QM_HW_V1: 241 q_num = HZIP_QUEUE_NUM_V1; 242 break; 243 case QM_HW_V2: 244 q_num = HZIP_QUEUE_NUM_V2; 245 break; 246 default: 247 return -EINVAL; 248 } 249 } 250 251 ret = kstrtou32(val, 10, &n); 252 if (ret != 0 || n > q_num || n == 0) 253 return -EINVAL; 254 255 return param_set_int(val, kp); 256 } 257 258 static const struct kernel_param_ops pf_q_num_ops = { 259 .set = pf_q_num_set, 260 .get = param_get_int, 261 }; 262 263 static u32 pf_q_num = HZIP_PF_DEF_Q_NUM; 264 module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444); 265 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)"); 266 267 static int uacce_mode; 268 module_param(uacce_mode, int, 0); 269 270 static const struct pci_device_id hisi_zip_dev_ids[] = { 271 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_ZIP_PF) }, 272 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_ZIP_VF) }, 273 { 0, } 274 }; 275 MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); 276 277 static inline void hisi_zip_add_to_list(struct hisi_zip *hisi_zip) 278 { 279 mutex_lock(&hisi_zip_list_lock); 280 list_add_tail(&hisi_zip->list, &hisi_zip_list); 281 mutex_unlock(&hisi_zip_list_lock); 282 } 283 284 static inline void hisi_zip_remove_from_list(struct hisi_zip *hisi_zip) 285 { 286 mutex_lock(&hisi_zip_list_lock); 287 list_del(&hisi_zip->list); 288 mutex_unlock(&hisi_zip_list_lock); 289 } 290 291 static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip) 292 { 293 void __iomem *base = hisi_zip->qm.io_base; 294 295 /* qm user domain */ 296 writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); 297 writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE); 298 writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1); 299 writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE); 300 writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE); 301 302 /* qm cache */ 303 writel(AXI_M_CFG, base + QM_AXI_M_CFG); 304 writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE); 305 /* disable FLR triggered by BME(bus master enable) */ 306 writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG); 307 writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE); 308 309 /* cache */ 310 writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); 311 writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); 312 writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); 313 writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); 314 315 /* user domain configurations */ 316 writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); 317 writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); 318 writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63); 319 writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63); 320 writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); 321 322 /* let's open all compression/decompression cores */ 323 writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN, 324 base + HZIP_CLOCK_GATE_CTRL); 325 326 /* enable sqc writeback */ 327 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | 328 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | 329 FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); 330 } 331 332 static void hisi_zip_hw_error_set_state(struct hisi_zip *hisi_zip, bool state) 333 { 334 struct hisi_qm *qm = &hisi_zip->qm; 335 336 if (qm->ver == QM_HW_V1) { 337 writel(HZIP_CORE_INT_DISABLE, qm->io_base + HZIP_CORE_INT_MASK); 338 dev_info(&qm->pdev->dev, "ZIP v%d does not support hw error handle\n", 339 qm->ver); 340 return; 341 } 342 343 if (state) { 344 /* clear ZIP hw error source if having */ 345 writel(HZIP_CORE_INT_DISABLE, hisi_zip->qm.io_base + 346 HZIP_CORE_INT_SOURCE); 347 /* enable ZIP hw error interrupts */ 348 writel(0, hisi_zip->qm.io_base + HZIP_CORE_INT_MASK); 349 } else { 350 /* disable ZIP hw error interrupts */ 351 writel(HZIP_CORE_INT_DISABLE, 352 hisi_zip->qm.io_base + HZIP_CORE_INT_MASK); 353 } 354 } 355 356 static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) 357 { 358 struct hisi_zip *hisi_zip = file->ctrl->hisi_zip; 359 360 return &hisi_zip->qm; 361 } 362 363 static u32 current_qm_read(struct ctrl_debug_file *file) 364 { 365 struct hisi_qm *qm = file_to_qm(file); 366 367 return readl(qm->io_base + QM_DFX_MB_CNT_VF); 368 } 369 370 static int current_qm_write(struct ctrl_debug_file *file, u32 val) 371 { 372 struct hisi_qm *qm = file_to_qm(file); 373 struct hisi_zip_ctrl *ctrl = file->ctrl; 374 u32 vfq_num; 375 u32 tmp; 376 377 if (val > ctrl->num_vfs) 378 return -EINVAL; 379 380 /* Calculate curr_qm_qp_num and store */ 381 if (val == 0) { 382 qm->debug.curr_qm_qp_num = qm->qp_num; 383 } else { 384 vfq_num = (qm->ctrl_qp_num - qm->qp_num) / ctrl->num_vfs; 385 if (val == ctrl->num_vfs) 386 qm->debug.curr_qm_qp_num = qm->ctrl_qp_num - 387 qm->qp_num - (ctrl->num_vfs - 1) * vfq_num; 388 else 389 qm->debug.curr_qm_qp_num = vfq_num; 390 } 391 392 writel(val, qm->io_base + QM_DFX_MB_CNT_VF); 393 writel(val, qm->io_base + QM_DFX_DB_CNT_VF); 394 395 tmp = val | 396 (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); 397 writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); 398 399 tmp = val | 400 (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); 401 writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); 402 403 return 0; 404 } 405 406 static u32 clear_enable_read(struct ctrl_debug_file *file) 407 { 408 struct hisi_qm *qm = file_to_qm(file); 409 410 return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & 411 SOFT_CTRL_CNT_CLR_CE_BIT; 412 } 413 414 static int clear_enable_write(struct ctrl_debug_file *file, u32 val) 415 { 416 struct hisi_qm *qm = file_to_qm(file); 417 u32 tmp; 418 419 if (val != 1 && val != 0) 420 return -EINVAL; 421 422 tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & 423 ~SOFT_CTRL_CNT_CLR_CE_BIT) | val; 424 writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); 425 426 return 0; 427 } 428 429 static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, 430 size_t count, loff_t *pos) 431 { 432 struct ctrl_debug_file *file = filp->private_data; 433 char tbuf[HZIP_BUF_SIZE]; 434 u32 val; 435 int ret; 436 437 spin_lock_irq(&file->lock); 438 switch (file->index) { 439 case HZIP_CURRENT_QM: 440 val = current_qm_read(file); 441 break; 442 case HZIP_CLEAR_ENABLE: 443 val = clear_enable_read(file); 444 break; 445 default: 446 spin_unlock_irq(&file->lock); 447 return -EINVAL; 448 } 449 spin_unlock_irq(&file->lock); 450 ret = sprintf(tbuf, "%u\n", val); 451 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 452 } 453 454 static ssize_t ctrl_debug_write(struct file *filp, const char __user *buf, 455 size_t count, loff_t *pos) 456 { 457 struct ctrl_debug_file *file = filp->private_data; 458 char tbuf[HZIP_BUF_SIZE]; 459 unsigned long val; 460 int len, ret; 461 462 if (*pos != 0) 463 return 0; 464 465 if (count >= HZIP_BUF_SIZE) 466 return -ENOSPC; 467 468 len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count); 469 if (len < 0) 470 return len; 471 472 tbuf[len] = '\0'; 473 if (kstrtoul(tbuf, 0, &val)) 474 return -EFAULT; 475 476 spin_lock_irq(&file->lock); 477 switch (file->index) { 478 case HZIP_CURRENT_QM: 479 ret = current_qm_write(file, val); 480 if (ret) 481 goto err_input; 482 break; 483 case HZIP_CLEAR_ENABLE: 484 ret = clear_enable_write(file, val); 485 if (ret) 486 goto err_input; 487 break; 488 default: 489 ret = -EINVAL; 490 goto err_input; 491 } 492 spin_unlock_irq(&file->lock); 493 494 return count; 495 496 err_input: 497 spin_unlock_irq(&file->lock); 498 return ret; 499 } 500 501 static const struct file_operations ctrl_debug_fops = { 502 .owner = THIS_MODULE, 503 .open = simple_open, 504 .read = ctrl_debug_read, 505 .write = ctrl_debug_write, 506 }; 507 508 static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) 509 { 510 struct hisi_zip *hisi_zip = ctrl->hisi_zip; 511 struct hisi_qm *qm = &hisi_zip->qm; 512 struct device *dev = &qm->pdev->dev; 513 struct debugfs_regset32 *regset; 514 struct dentry *tmp_d, *tmp; 515 char buf[HZIP_BUF_SIZE]; 516 int i; 517 518 for (i = 0; i < HZIP_CORE_NUM; i++) { 519 if (i < HZIP_COMP_CORE_NUM) 520 sprintf(buf, "comp_core%d", i); 521 else 522 sprintf(buf, "decomp_core%d", i - HZIP_COMP_CORE_NUM); 523 524 tmp_d = debugfs_create_dir(buf, ctrl->debug_root); 525 if (!tmp_d) 526 return -ENOENT; 527 528 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 529 if (!regset) 530 return -ENOENT; 531 532 regset->regs = hzip_dfx_regs; 533 regset->nregs = ARRAY_SIZE(hzip_dfx_regs); 534 regset->base = qm->io_base + core_offsets[i]; 535 536 tmp = debugfs_create_regset32("regs", 0444, tmp_d, regset); 537 if (!tmp) 538 return -ENOENT; 539 } 540 541 return 0; 542 } 543 544 static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl) 545 { 546 struct dentry *tmp; 547 int i; 548 549 for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) { 550 spin_lock_init(&ctrl->files[i].lock); 551 ctrl->files[i].ctrl = ctrl; 552 ctrl->files[i].index = i; 553 554 tmp = debugfs_create_file(ctrl_debug_file_name[i], 0600, 555 ctrl->debug_root, ctrl->files + i, 556 &ctrl_debug_fops); 557 if (!tmp) 558 return -ENOENT; 559 } 560 561 return hisi_zip_core_debug_init(ctrl); 562 } 563 564 static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) 565 { 566 struct hisi_qm *qm = &hisi_zip->qm; 567 struct device *dev = &qm->pdev->dev; 568 struct dentry *dev_d; 569 int ret; 570 571 dev_d = debugfs_create_dir(dev_name(dev), hzip_debugfs_root); 572 if (!dev_d) 573 return -ENOENT; 574 575 qm->debug.debug_root = dev_d; 576 ret = hisi_qm_debug_init(qm); 577 if (ret) 578 goto failed_to_create; 579 580 if (qm->fun_type == QM_HW_PF) { 581 hisi_zip->ctrl->debug_root = dev_d; 582 ret = hisi_zip_ctrl_debug_init(hisi_zip->ctrl); 583 if (ret) 584 goto failed_to_create; 585 } 586 587 return 0; 588 589 failed_to_create: 590 debugfs_remove_recursive(hzip_debugfs_root); 591 return ret; 592 } 593 594 static void hisi_zip_debug_regs_clear(struct hisi_zip *hisi_zip) 595 { 596 struct hisi_qm *qm = &hisi_zip->qm; 597 598 writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); 599 writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); 600 writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); 601 602 hisi_qm_debug_regs_clear(qm); 603 } 604 605 static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip) 606 { 607 struct hisi_qm *qm = &hisi_zip->qm; 608 609 debugfs_remove_recursive(qm->debug.debug_root); 610 611 if (qm->fun_type == QM_HW_PF) 612 hisi_zip_debug_regs_clear(hisi_zip); 613 } 614 615 static void hisi_zip_hw_error_init(struct hisi_zip *hisi_zip) 616 { 617 hisi_qm_hw_error_init(&hisi_zip->qm, QM_BASE_CE, 618 QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT, 0, 619 QM_DB_RANDOM_INVALID); 620 hisi_zip_hw_error_set_state(hisi_zip, true); 621 } 622 623 static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) 624 { 625 struct hisi_qm *qm = &hisi_zip->qm; 626 struct hisi_zip_ctrl *ctrl; 627 628 ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL); 629 if (!ctrl) 630 return -ENOMEM; 631 632 hisi_zip->ctrl = ctrl; 633 ctrl->hisi_zip = hisi_zip; 634 635 switch (qm->ver) { 636 case QM_HW_V1: 637 qm->ctrl_qp_num = HZIP_QUEUE_NUM_V1; 638 break; 639 640 case QM_HW_V2: 641 qm->ctrl_qp_num = HZIP_QUEUE_NUM_V2; 642 break; 643 644 default: 645 return -EINVAL; 646 } 647 648 hisi_zip_set_user_domain_and_cache(hisi_zip); 649 hisi_zip_hw_error_init(hisi_zip); 650 hisi_zip_debug_regs_clear(hisi_zip); 651 652 return 0; 653 } 654 655 static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) 656 { 657 struct hisi_zip *hisi_zip; 658 enum qm_hw_ver rev_id; 659 struct hisi_qm *qm; 660 int ret; 661 662 rev_id = hisi_qm_get_hw_version(pdev); 663 if (rev_id == QM_HW_UNKNOWN) 664 return -EINVAL; 665 666 hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL); 667 if (!hisi_zip) 668 return -ENOMEM; 669 pci_set_drvdata(pdev, hisi_zip); 670 671 qm = &hisi_zip->qm; 672 qm->pdev = pdev; 673 qm->ver = rev_id; 674 675 qm->sqe_size = HZIP_SQE_SIZE; 676 qm->dev_name = hisi_zip_name; 677 qm->fun_type = (pdev->device == PCI_DEVICE_ID_ZIP_PF) ? QM_HW_PF : 678 QM_HW_VF; 679 switch (uacce_mode) { 680 case 0: 681 qm->use_dma_api = true; 682 break; 683 case 1: 684 qm->use_dma_api = false; 685 break; 686 case 2: 687 qm->use_dma_api = true; 688 break; 689 default: 690 return -EINVAL; 691 } 692 693 ret = hisi_qm_init(qm); 694 if (ret) { 695 dev_err(&pdev->dev, "Failed to init qm!\n"); 696 return ret; 697 } 698 699 if (qm->fun_type == QM_HW_PF) { 700 ret = hisi_zip_pf_probe_init(hisi_zip); 701 if (ret) 702 return ret; 703 704 qm->qp_base = HZIP_PF_DEF_Q_BASE; 705 qm->qp_num = pf_q_num; 706 } else if (qm->fun_type == QM_HW_VF) { 707 /* 708 * have no way to get qm configure in VM in v1 hardware, 709 * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force 710 * to trigger only one VF in v1 hardware. 711 * 712 * v2 hardware has no such problem. 713 */ 714 if (qm->ver == QM_HW_V1) { 715 qm->qp_base = HZIP_PF_DEF_Q_NUM; 716 qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; 717 } else if (qm->ver == QM_HW_V2) 718 /* v2 starts to support get vft by mailbox */ 719 hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); 720 } 721 722 ret = hisi_qm_start(qm); 723 if (ret) 724 goto err_qm_uninit; 725 726 ret = hisi_zip_debugfs_init(hisi_zip); 727 if (ret) 728 dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret); 729 730 hisi_zip_add_to_list(hisi_zip); 731 732 return 0; 733 734 err_qm_uninit: 735 hisi_qm_uninit(qm); 736 return ret; 737 } 738 739 /* Currently we only support equal assignment */ 740 static int hisi_zip_vf_q_assign(struct hisi_zip *hisi_zip, int num_vfs) 741 { 742 struct hisi_qm *qm = &hisi_zip->qm; 743 u32 qp_num = qm->qp_num; 744 u32 q_base = qp_num; 745 u32 q_num, remain_q_num, i; 746 int ret; 747 748 if (!num_vfs) 749 return -EINVAL; 750 751 remain_q_num = qm->ctrl_qp_num - qp_num; 752 if (remain_q_num < num_vfs) 753 return -EINVAL; 754 755 q_num = remain_q_num / num_vfs; 756 for (i = 1; i <= num_vfs; i++) { 757 if (i == num_vfs) 758 q_num += remain_q_num % num_vfs; 759 ret = hisi_qm_set_vft(qm, i, q_base, q_num); 760 if (ret) 761 return ret; 762 q_base += q_num; 763 } 764 765 return 0; 766 } 767 768 static int hisi_zip_clear_vft_config(struct hisi_zip *hisi_zip) 769 { 770 struct hisi_zip_ctrl *ctrl = hisi_zip->ctrl; 771 struct hisi_qm *qm = &hisi_zip->qm; 772 u32 i, num_vfs = ctrl->num_vfs; 773 int ret; 774 775 for (i = 1; i <= num_vfs; i++) { 776 ret = hisi_qm_set_vft(qm, i, 0, 0); 777 if (ret) 778 return ret; 779 } 780 781 ctrl->num_vfs = 0; 782 783 return 0; 784 } 785 786 static int hisi_zip_sriov_enable(struct pci_dev *pdev, int max_vfs) 787 { 788 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 789 int pre_existing_vfs, num_vfs, ret; 790 791 pre_existing_vfs = pci_num_vf(pdev); 792 793 if (pre_existing_vfs) { 794 dev_err(&pdev->dev, 795 "Can't enable VF. Please disable pre-enabled VFs!\n"); 796 return 0; 797 } 798 799 num_vfs = min_t(int, max_vfs, HZIP_VF_NUM); 800 801 ret = hisi_zip_vf_q_assign(hisi_zip, num_vfs); 802 if (ret) { 803 dev_err(&pdev->dev, "Can't assign queues for VF!\n"); 804 return ret; 805 } 806 807 hisi_zip->ctrl->num_vfs = num_vfs; 808 809 ret = pci_enable_sriov(pdev, num_vfs); 810 if (ret) { 811 dev_err(&pdev->dev, "Can't enable VF!\n"); 812 hisi_zip_clear_vft_config(hisi_zip); 813 return ret; 814 } 815 816 return num_vfs; 817 } 818 819 static int hisi_zip_sriov_disable(struct pci_dev *pdev) 820 { 821 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 822 823 if (pci_vfs_assigned(pdev)) { 824 dev_err(&pdev->dev, 825 "Can't disable VFs while VFs are assigned!\n"); 826 return -EPERM; 827 } 828 829 /* remove in hisi_zip_pci_driver will be called to free VF resources */ 830 pci_disable_sriov(pdev); 831 832 return hisi_zip_clear_vft_config(hisi_zip); 833 } 834 835 static int hisi_zip_sriov_configure(struct pci_dev *pdev, int num_vfs) 836 { 837 if (num_vfs == 0) 838 return hisi_zip_sriov_disable(pdev); 839 else 840 return hisi_zip_sriov_enable(pdev, num_vfs); 841 } 842 843 static void hisi_zip_remove(struct pci_dev *pdev) 844 { 845 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 846 struct hisi_qm *qm = &hisi_zip->qm; 847 848 if (qm->fun_type == QM_HW_PF && hisi_zip->ctrl->num_vfs != 0) 849 hisi_zip_sriov_disable(pdev); 850 851 hisi_zip_debugfs_exit(hisi_zip); 852 hisi_qm_stop(qm); 853 854 if (qm->fun_type == QM_HW_PF) 855 hisi_zip_hw_error_set_state(hisi_zip, false); 856 857 hisi_qm_uninit(qm); 858 hisi_zip_remove_from_list(hisi_zip); 859 } 860 861 static void hisi_zip_log_hw_error(struct hisi_zip *hisi_zip, u32 err_sts) 862 { 863 const struct hisi_zip_hw_error *err = zip_hw_error; 864 struct device *dev = &hisi_zip->qm.pdev->dev; 865 u32 err_val; 866 867 while (err->msg) { 868 if (err->int_msk & err_sts) { 869 dev_warn(dev, "%s [error status=0x%x] found\n", 870 err->msg, err->int_msk); 871 872 if (HZIP_CORE_INT_STATUS_M_ECC & err->int_msk) { 873 err_val = readl(hisi_zip->qm.io_base + 874 HZIP_CORE_SRAM_ECC_ERR_INFO); 875 dev_warn(dev, "hisi-zip multi ecc sram num=0x%x\n", 876 ((err_val >> SRAM_ECC_ERR_NUM_SHIFT) & 877 0xFF)); 878 dev_warn(dev, "hisi-zip multi ecc sram addr=0x%x\n", 879 (err_val >> SRAM_ECC_ERR_ADDR_SHIFT)); 880 } 881 } 882 err++; 883 } 884 } 885 886 static pci_ers_result_t hisi_zip_hw_error_handle(struct hisi_zip *hisi_zip) 887 { 888 u32 err_sts; 889 890 /* read err sts */ 891 err_sts = readl(hisi_zip->qm.io_base + HZIP_CORE_INT_STATUS); 892 893 if (err_sts) { 894 hisi_zip_log_hw_error(hisi_zip, err_sts); 895 /* clear error interrupts */ 896 writel(err_sts, hisi_zip->qm.io_base + HZIP_CORE_INT_SOURCE); 897 898 return PCI_ERS_RESULT_NEED_RESET; 899 } 900 901 return PCI_ERS_RESULT_RECOVERED; 902 } 903 904 static pci_ers_result_t hisi_zip_process_hw_error(struct pci_dev *pdev) 905 { 906 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 907 struct device *dev = &pdev->dev; 908 pci_ers_result_t qm_ret, zip_ret; 909 910 if (!hisi_zip) { 911 dev_err(dev, 912 "Can't recover ZIP-error occurred during device init\n"); 913 return PCI_ERS_RESULT_NONE; 914 } 915 916 qm_ret = hisi_qm_hw_error_handle(&hisi_zip->qm); 917 918 zip_ret = hisi_zip_hw_error_handle(hisi_zip); 919 920 return (qm_ret == PCI_ERS_RESULT_NEED_RESET || 921 zip_ret == PCI_ERS_RESULT_NEED_RESET) ? 922 PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED; 923 } 924 925 static pci_ers_result_t hisi_zip_error_detected(struct pci_dev *pdev, 926 pci_channel_state_t state) 927 { 928 if (pdev->is_virtfn) 929 return PCI_ERS_RESULT_NONE; 930 931 dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state); 932 if (state == pci_channel_io_perm_failure) 933 return PCI_ERS_RESULT_DISCONNECT; 934 935 return hisi_zip_process_hw_error(pdev); 936 } 937 938 static const struct pci_error_handlers hisi_zip_err_handler = { 939 .error_detected = hisi_zip_error_detected, 940 }; 941 942 static struct pci_driver hisi_zip_pci_driver = { 943 .name = "hisi_zip", 944 .id_table = hisi_zip_dev_ids, 945 .probe = hisi_zip_probe, 946 .remove = hisi_zip_remove, 947 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? 948 hisi_zip_sriov_configure : 0, 949 .err_handler = &hisi_zip_err_handler, 950 }; 951 952 static void hisi_zip_register_debugfs(void) 953 { 954 if (!debugfs_initialized()) 955 return; 956 957 hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL); 958 if (IS_ERR_OR_NULL(hzip_debugfs_root)) 959 hzip_debugfs_root = NULL; 960 } 961 962 static void hisi_zip_unregister_debugfs(void) 963 { 964 debugfs_remove_recursive(hzip_debugfs_root); 965 } 966 967 static int __init hisi_zip_init(void) 968 { 969 int ret; 970 971 hisi_zip_register_debugfs(); 972 973 ret = pci_register_driver(&hisi_zip_pci_driver); 974 if (ret < 0) { 975 pr_err("Failed to register pci driver.\n"); 976 goto err_pci; 977 } 978 979 if (uacce_mode == 0 || uacce_mode == 2) { 980 ret = hisi_zip_register_to_crypto(); 981 if (ret < 0) { 982 pr_err("Failed to register driver to crypto.\n"); 983 goto err_crypto; 984 } 985 } 986 987 return 0; 988 989 err_crypto: 990 pci_unregister_driver(&hisi_zip_pci_driver); 991 err_pci: 992 hisi_zip_unregister_debugfs(); 993 994 return ret; 995 } 996 997 static void __exit hisi_zip_exit(void) 998 { 999 if (uacce_mode == 0 || uacce_mode == 2) 1000 hisi_zip_unregister_from_crypto(); 1001 pci_unregister_driver(&hisi_zip_pci_driver); 1002 hisi_zip_unregister_debugfs(); 1003 } 1004 1005 module_init(hisi_zip_init); 1006 module_exit(hisi_zip_exit); 1007 1008 MODULE_LICENSE("GPL v2"); 1009 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 1010 MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator"); 1011