1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <linux/acpi.h> 4 #include <linux/aer.h> 5 #include <linux/bitops.h> 6 #include <linux/debugfs.h> 7 #include <linux/init.h> 8 #include <linux/io.h> 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/pci.h> 12 #include <linux/seq_file.h> 13 #include <linux/topology.h> 14 #include "zip.h" 15 16 #define PCI_DEVICE_ID_ZIP_PF 0xa250 17 #define PCI_DEVICE_ID_ZIP_VF 0xa251 18 19 #define HZIP_VF_NUM 63 20 #define HZIP_QUEUE_NUM_V1 4096 21 #define HZIP_QUEUE_NUM_V2 1024 22 23 #define HZIP_CLOCK_GATE_CTRL 0x301004 24 #define COMP0_ENABLE BIT(0) 25 #define COMP1_ENABLE BIT(1) 26 #define DECOMP0_ENABLE BIT(2) 27 #define DECOMP1_ENABLE BIT(3) 28 #define DECOMP2_ENABLE BIT(4) 29 #define DECOMP3_ENABLE BIT(5) 30 #define DECOMP4_ENABLE BIT(6) 31 #define DECOMP5_ENABLE BIT(7) 32 #define ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ 33 DECOMP0_ENABLE | DECOMP1_ENABLE | \ 34 DECOMP2_ENABLE | DECOMP3_ENABLE | \ 35 DECOMP4_ENABLE | DECOMP5_ENABLE) 36 #define DECOMP_CHECK_ENABLE BIT(16) 37 #define HZIP_FSM_MAX_CNT 0x301008 38 39 #define HZIP_PORT_ARCA_CHE_0 0x301040 40 #define HZIP_PORT_ARCA_CHE_1 0x301044 41 #define HZIP_PORT_AWCA_CHE_0 0x301060 42 #define HZIP_PORT_AWCA_CHE_1 0x301064 43 #define CACHE_ALL_EN 0xffffffff 44 45 #define HZIP_BD_RUSER_32_63 0x301110 46 #define HZIP_SGL_RUSER_32_63 0x30111c 47 #define HZIP_DATA_RUSER_32_63 0x301128 48 #define HZIP_DATA_WUSER_32_63 0x301134 49 #define HZIP_BD_WUSER_32_63 0x301140 50 51 #define HZIP_QM_IDEL_STATUS 0x3040e4 52 53 #define HZIP_CORE_DEBUG_COMP_0 0x302000 54 #define HZIP_CORE_DEBUG_COMP_1 0x303000 55 #define HZIP_CORE_DEBUG_DECOMP_0 0x304000 56 #define HZIP_CORE_DEBUG_DECOMP_1 0x305000 57 #define HZIP_CORE_DEBUG_DECOMP_2 0x306000 58 #define HZIP_CORE_DEBUG_DECOMP_3 0x307000 59 #define HZIP_CORE_DEBUG_DECOMP_4 0x308000 60 #define HZIP_CORE_DEBUG_DECOMP_5 0x309000 61 62 #define HZIP_CORE_INT_SOURCE 0x3010A0 63 #define HZIP_CORE_INT_MASK 0x3010A4 64 #define HZIP_CORE_INT_STATUS 0x3010AC 65 #define HZIP_CORE_INT_STATUS_M_ECC BIT(1) 66 #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 67 #define SRAM_ECC_ERR_NUM_SHIFT 16 68 #define SRAM_ECC_ERR_ADDR_SHIFT 24 69 #define HZIP_CORE_INT_DISABLE 0x000007FF 70 #define HZIP_COMP_CORE_NUM 2 71 #define HZIP_DECOMP_CORE_NUM 6 72 #define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \ 73 HZIP_DECOMP_CORE_NUM) 74 #define HZIP_SQE_SIZE 128 75 #define HZIP_SQ_SIZE (HZIP_SQE_SIZE * QM_Q_DEPTH) 76 #define HZIP_PF_DEF_Q_NUM 64 77 #define HZIP_PF_DEF_Q_BASE 0 78 79 #define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000 80 #define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0) 81 82 #define HZIP_BUF_SIZE 22 83 84 static const char hisi_zip_name[] = "hisi_zip"; 85 static struct dentry *hzip_debugfs_root; 86 static LIST_HEAD(hisi_zip_list); 87 static DEFINE_MUTEX(hisi_zip_list_lock); 88 89 struct hisi_zip_resource { 90 struct hisi_zip *hzip; 91 int distance; 92 struct list_head list; 93 }; 94 95 static void free_list(struct list_head *head) 96 { 97 struct hisi_zip_resource *res, *tmp; 98 99 list_for_each_entry_safe(res, tmp, head, list) { 100 list_del(&res->list); 101 kfree(res); 102 } 103 } 104 105 struct hisi_zip *find_zip_device(int node) 106 { 107 struct hisi_zip_resource *res, *tmp; 108 struct hisi_zip *ret = NULL; 109 struct hisi_zip *hisi_zip; 110 struct list_head *n; 111 struct device *dev; 112 LIST_HEAD(head); 113 114 mutex_lock(&hisi_zip_list_lock); 115 116 if (IS_ENABLED(CONFIG_NUMA)) { 117 list_for_each_entry(hisi_zip, &hisi_zip_list, list) { 118 res = kzalloc(sizeof(*res), GFP_KERNEL); 119 if (!res) 120 goto err; 121 122 dev = &hisi_zip->qm.pdev->dev; 123 res->hzip = hisi_zip; 124 res->distance = node_distance(dev_to_node(dev), node); 125 126 n = &head; 127 list_for_each_entry(tmp, &head, list) { 128 if (res->distance < tmp->distance) { 129 n = &tmp->list; 130 break; 131 } 132 } 133 list_add_tail(&res->list, n); 134 } 135 136 list_for_each_entry(tmp, &head, list) { 137 if (hisi_qm_get_free_qp_num(&tmp->hzip->qm)) { 138 ret = tmp->hzip; 139 break; 140 } 141 } 142 143 free_list(&head); 144 } else { 145 ret = list_first_entry(&hisi_zip_list, struct hisi_zip, list); 146 } 147 148 mutex_unlock(&hisi_zip_list_lock); 149 150 return ret; 151 152 err: 153 free_list(&head); 154 mutex_unlock(&hisi_zip_list_lock); 155 return NULL; 156 } 157 158 struct hisi_zip_hw_error { 159 u32 int_msk; 160 const char *msg; 161 }; 162 163 static const struct hisi_zip_hw_error zip_hw_error[] = { 164 { .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" }, 165 { .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" }, 166 { .int_msk = BIT(2), .msg = "zip_axi_rresp_err" }, 167 { .int_msk = BIT(3), .msg = "zip_axi_bresp_err" }, 168 { .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" }, 169 { .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" }, 170 { .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" }, 171 { .int_msk = BIT(7), .msg = "zip_pre_in_data_err" }, 172 { .int_msk = BIT(8), .msg = "zip_com_inf_err" }, 173 { .int_msk = BIT(9), .msg = "zip_enc_inf_err" }, 174 { .int_msk = BIT(10), .msg = "zip_pre_out_err" }, 175 { /* sentinel */ } 176 }; 177 178 enum ctrl_debug_file_index { 179 HZIP_CURRENT_QM, 180 HZIP_CLEAR_ENABLE, 181 HZIP_DEBUG_FILE_NUM, 182 }; 183 184 static const char * const ctrl_debug_file_name[] = { 185 [HZIP_CURRENT_QM] = "current_qm", 186 [HZIP_CLEAR_ENABLE] = "clear_enable", 187 }; 188 189 struct ctrl_debug_file { 190 enum ctrl_debug_file_index index; 191 spinlock_t lock; 192 struct hisi_zip_ctrl *ctrl; 193 }; 194 195 /* 196 * One ZIP controller has one PF and multiple VFs, some global configurations 197 * which PF has need this structure. 198 * 199 * Just relevant for PF. 200 */ 201 struct hisi_zip_ctrl { 202 u32 num_vfs; 203 struct hisi_zip *hisi_zip; 204 struct dentry *debug_root; 205 struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; 206 }; 207 208 enum { 209 HZIP_COMP_CORE0, 210 HZIP_COMP_CORE1, 211 HZIP_DECOMP_CORE0, 212 HZIP_DECOMP_CORE1, 213 HZIP_DECOMP_CORE2, 214 HZIP_DECOMP_CORE3, 215 HZIP_DECOMP_CORE4, 216 HZIP_DECOMP_CORE5, 217 }; 218 219 static const u64 core_offsets[] = { 220 [HZIP_COMP_CORE0] = 0x302000, 221 [HZIP_COMP_CORE1] = 0x303000, 222 [HZIP_DECOMP_CORE0] = 0x304000, 223 [HZIP_DECOMP_CORE1] = 0x305000, 224 [HZIP_DECOMP_CORE2] = 0x306000, 225 [HZIP_DECOMP_CORE3] = 0x307000, 226 [HZIP_DECOMP_CORE4] = 0x308000, 227 [HZIP_DECOMP_CORE5] = 0x309000, 228 }; 229 230 static struct debugfs_reg32 hzip_dfx_regs[] = { 231 {"HZIP_GET_BD_NUM ", 0x00ull}, 232 {"HZIP_GET_RIGHT_BD ", 0x04ull}, 233 {"HZIP_GET_ERROR_BD ", 0x08ull}, 234 {"HZIP_DONE_BD_NUM ", 0x0cull}, 235 {"HZIP_WORK_CYCLE ", 0x10ull}, 236 {"HZIP_IDLE_CYCLE ", 0x18ull}, 237 {"HZIP_MAX_DELAY ", 0x20ull}, 238 {"HZIP_MIN_DELAY ", 0x24ull}, 239 {"HZIP_AVG_DELAY ", 0x28ull}, 240 {"HZIP_MEM_VISIBLE_DATA ", 0x30ull}, 241 {"HZIP_MEM_VISIBLE_ADDR ", 0x34ull}, 242 {"HZIP_COMSUMED_BYTE ", 0x38ull}, 243 {"HZIP_PRODUCED_BYTE ", 0x40ull}, 244 {"HZIP_COMP_INF ", 0x70ull}, 245 {"HZIP_PRE_OUT ", 0x78ull}, 246 {"HZIP_BD_RD ", 0x7cull}, 247 {"HZIP_BD_WR ", 0x80ull}, 248 {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84ull}, 249 {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88ull}, 250 {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8cull}, 251 {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94ull}, 252 {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9cull}, 253 }; 254 255 static int pf_q_num_set(const char *val, const struct kernel_param *kp) 256 { 257 struct pci_dev *pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, 258 PCI_DEVICE_ID_ZIP_PF, NULL); 259 u32 n, q_num; 260 u8 rev_id; 261 int ret; 262 263 if (!val) 264 return -EINVAL; 265 266 if (!pdev) { 267 q_num = min_t(u32, HZIP_QUEUE_NUM_V1, HZIP_QUEUE_NUM_V2); 268 pr_info("No device found currently, suppose queue number is %d\n", 269 q_num); 270 } else { 271 rev_id = pdev->revision; 272 switch (rev_id) { 273 case QM_HW_V1: 274 q_num = HZIP_QUEUE_NUM_V1; 275 break; 276 case QM_HW_V2: 277 q_num = HZIP_QUEUE_NUM_V2; 278 break; 279 default: 280 return -EINVAL; 281 } 282 } 283 284 ret = kstrtou32(val, 10, &n); 285 if (ret != 0 || n > q_num || n == 0) 286 return -EINVAL; 287 288 return param_set_int(val, kp); 289 } 290 291 static const struct kernel_param_ops pf_q_num_ops = { 292 .set = pf_q_num_set, 293 .get = param_get_int, 294 }; 295 296 static u32 pf_q_num = HZIP_PF_DEF_Q_NUM; 297 module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444); 298 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)"); 299 300 static int uacce_mode; 301 module_param(uacce_mode, int, 0); 302 303 static u32 vfs_num; 304 module_param(vfs_num, uint, 0444); 305 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63)"); 306 307 static const struct pci_device_id hisi_zip_dev_ids[] = { 308 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_ZIP_PF) }, 309 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_ZIP_VF) }, 310 { 0, } 311 }; 312 MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); 313 314 static inline void hisi_zip_add_to_list(struct hisi_zip *hisi_zip) 315 { 316 mutex_lock(&hisi_zip_list_lock); 317 list_add_tail(&hisi_zip->list, &hisi_zip_list); 318 mutex_unlock(&hisi_zip_list_lock); 319 } 320 321 static inline void hisi_zip_remove_from_list(struct hisi_zip *hisi_zip) 322 { 323 mutex_lock(&hisi_zip_list_lock); 324 list_del(&hisi_zip->list); 325 mutex_unlock(&hisi_zip_list_lock); 326 } 327 328 static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip) 329 { 330 void __iomem *base = hisi_zip->qm.io_base; 331 332 /* qm user domain */ 333 writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); 334 writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE); 335 writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1); 336 writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE); 337 writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE); 338 339 /* qm cache */ 340 writel(AXI_M_CFG, base + QM_AXI_M_CFG); 341 writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE); 342 /* disable FLR triggered by BME(bus master enable) */ 343 writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG); 344 writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE); 345 346 /* cache */ 347 writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0); 348 writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1); 349 writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0); 350 writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1); 351 352 /* user domain configurations */ 353 writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63); 354 writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); 355 writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63); 356 writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63); 357 writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63); 358 359 /* let's open all compression/decompression cores */ 360 writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN, 361 base + HZIP_CLOCK_GATE_CTRL); 362 363 /* enable sqc writeback */ 364 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | 365 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | 366 FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL); 367 } 368 369 static void hisi_zip_hw_error_set_state(struct hisi_zip *hisi_zip, bool state) 370 { 371 struct hisi_qm *qm = &hisi_zip->qm; 372 373 if (qm->ver == QM_HW_V1) { 374 writel(HZIP_CORE_INT_DISABLE, qm->io_base + HZIP_CORE_INT_MASK); 375 dev_info(&qm->pdev->dev, "Does not support hw error handle\n"); 376 return; 377 } 378 379 if (state) { 380 /* clear ZIP hw error source if having */ 381 writel(HZIP_CORE_INT_DISABLE, hisi_zip->qm.io_base + 382 HZIP_CORE_INT_SOURCE); 383 /* enable ZIP hw error interrupts */ 384 writel(0, hisi_zip->qm.io_base + HZIP_CORE_INT_MASK); 385 } else { 386 /* disable ZIP hw error interrupts */ 387 writel(HZIP_CORE_INT_DISABLE, 388 hisi_zip->qm.io_base + HZIP_CORE_INT_MASK); 389 } 390 } 391 392 static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) 393 { 394 struct hisi_zip *hisi_zip = file->ctrl->hisi_zip; 395 396 return &hisi_zip->qm; 397 } 398 399 static u32 current_qm_read(struct ctrl_debug_file *file) 400 { 401 struct hisi_qm *qm = file_to_qm(file); 402 403 return readl(qm->io_base + QM_DFX_MB_CNT_VF); 404 } 405 406 static int current_qm_write(struct ctrl_debug_file *file, u32 val) 407 { 408 struct hisi_qm *qm = file_to_qm(file); 409 struct hisi_zip_ctrl *ctrl = file->ctrl; 410 u32 vfq_num; 411 u32 tmp; 412 413 if (val > ctrl->num_vfs) 414 return -EINVAL; 415 416 /* Calculate curr_qm_qp_num and store */ 417 if (val == 0) { 418 qm->debug.curr_qm_qp_num = qm->qp_num; 419 } else { 420 vfq_num = (qm->ctrl_qp_num - qm->qp_num) / ctrl->num_vfs; 421 if (val == ctrl->num_vfs) 422 qm->debug.curr_qm_qp_num = qm->ctrl_qp_num - 423 qm->qp_num - (ctrl->num_vfs - 1) * vfq_num; 424 else 425 qm->debug.curr_qm_qp_num = vfq_num; 426 } 427 428 writel(val, qm->io_base + QM_DFX_MB_CNT_VF); 429 writel(val, qm->io_base + QM_DFX_DB_CNT_VF); 430 431 tmp = val | 432 (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); 433 writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); 434 435 tmp = val | 436 (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); 437 writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); 438 439 return 0; 440 } 441 442 static u32 clear_enable_read(struct ctrl_debug_file *file) 443 { 444 struct hisi_qm *qm = file_to_qm(file); 445 446 return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & 447 SOFT_CTRL_CNT_CLR_CE_BIT; 448 } 449 450 static int clear_enable_write(struct ctrl_debug_file *file, u32 val) 451 { 452 struct hisi_qm *qm = file_to_qm(file); 453 u32 tmp; 454 455 if (val != 1 && val != 0) 456 return -EINVAL; 457 458 tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) & 459 ~SOFT_CTRL_CNT_CLR_CE_BIT) | val; 460 writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); 461 462 return 0; 463 } 464 465 static ssize_t ctrl_debug_read(struct file *filp, char __user *buf, 466 size_t count, loff_t *pos) 467 { 468 struct ctrl_debug_file *file = filp->private_data; 469 char tbuf[HZIP_BUF_SIZE]; 470 u32 val; 471 int ret; 472 473 spin_lock_irq(&file->lock); 474 switch (file->index) { 475 case HZIP_CURRENT_QM: 476 val = current_qm_read(file); 477 break; 478 case HZIP_CLEAR_ENABLE: 479 val = clear_enable_read(file); 480 break; 481 default: 482 spin_unlock_irq(&file->lock); 483 return -EINVAL; 484 } 485 spin_unlock_irq(&file->lock); 486 ret = sprintf(tbuf, "%u\n", val); 487 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 488 } 489 490 static ssize_t ctrl_debug_write(struct file *filp, const char __user *buf, 491 size_t count, loff_t *pos) 492 { 493 struct ctrl_debug_file *file = filp->private_data; 494 char tbuf[HZIP_BUF_SIZE]; 495 unsigned long val; 496 int len, ret; 497 498 if (*pos != 0) 499 return 0; 500 501 if (count >= HZIP_BUF_SIZE) 502 return -ENOSPC; 503 504 len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count); 505 if (len < 0) 506 return len; 507 508 tbuf[len] = '\0'; 509 if (kstrtoul(tbuf, 0, &val)) 510 return -EFAULT; 511 512 spin_lock_irq(&file->lock); 513 switch (file->index) { 514 case HZIP_CURRENT_QM: 515 ret = current_qm_write(file, val); 516 if (ret) 517 goto err_input; 518 break; 519 case HZIP_CLEAR_ENABLE: 520 ret = clear_enable_write(file, val); 521 if (ret) 522 goto err_input; 523 break; 524 default: 525 ret = -EINVAL; 526 goto err_input; 527 } 528 spin_unlock_irq(&file->lock); 529 530 return count; 531 532 err_input: 533 spin_unlock_irq(&file->lock); 534 return ret; 535 } 536 537 static const struct file_operations ctrl_debug_fops = { 538 .owner = THIS_MODULE, 539 .open = simple_open, 540 .read = ctrl_debug_read, 541 .write = ctrl_debug_write, 542 }; 543 544 static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl) 545 { 546 struct hisi_zip *hisi_zip = ctrl->hisi_zip; 547 struct hisi_qm *qm = &hisi_zip->qm; 548 struct device *dev = &qm->pdev->dev; 549 struct debugfs_regset32 *regset; 550 struct dentry *tmp_d; 551 char buf[HZIP_BUF_SIZE]; 552 int i; 553 554 for (i = 0; i < HZIP_CORE_NUM; i++) { 555 if (i < HZIP_COMP_CORE_NUM) 556 sprintf(buf, "comp_core%d", i); 557 else 558 sprintf(buf, "decomp_core%d", i - HZIP_COMP_CORE_NUM); 559 560 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 561 if (!regset) 562 return -ENOENT; 563 564 regset->regs = hzip_dfx_regs; 565 regset->nregs = ARRAY_SIZE(hzip_dfx_regs); 566 regset->base = qm->io_base + core_offsets[i]; 567 568 tmp_d = debugfs_create_dir(buf, ctrl->debug_root); 569 debugfs_create_regset32("regs", 0444, tmp_d, regset); 570 } 571 572 return 0; 573 } 574 575 static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl) 576 { 577 int i; 578 579 for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) { 580 spin_lock_init(&ctrl->files[i].lock); 581 ctrl->files[i].ctrl = ctrl; 582 ctrl->files[i].index = i; 583 584 debugfs_create_file(ctrl_debug_file_name[i], 0600, 585 ctrl->debug_root, ctrl->files + i, 586 &ctrl_debug_fops); 587 } 588 589 return hisi_zip_core_debug_init(ctrl); 590 } 591 592 static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip) 593 { 594 struct hisi_qm *qm = &hisi_zip->qm; 595 struct device *dev = &qm->pdev->dev; 596 struct dentry *dev_d; 597 int ret; 598 599 dev_d = debugfs_create_dir(dev_name(dev), hzip_debugfs_root); 600 601 qm->debug.debug_root = dev_d; 602 ret = hisi_qm_debug_init(qm); 603 if (ret) 604 goto failed_to_create; 605 606 if (qm->fun_type == QM_HW_PF) { 607 hisi_zip->ctrl->debug_root = dev_d; 608 ret = hisi_zip_ctrl_debug_init(hisi_zip->ctrl); 609 if (ret) 610 goto failed_to_create; 611 } 612 613 return 0; 614 615 failed_to_create: 616 debugfs_remove_recursive(hzip_debugfs_root); 617 return ret; 618 } 619 620 static void hisi_zip_debug_regs_clear(struct hisi_zip *hisi_zip) 621 { 622 struct hisi_qm *qm = &hisi_zip->qm; 623 624 writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); 625 writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); 626 writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); 627 628 hisi_qm_debug_regs_clear(qm); 629 } 630 631 static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip) 632 { 633 struct hisi_qm *qm = &hisi_zip->qm; 634 635 debugfs_remove_recursive(qm->debug.debug_root); 636 637 if (qm->fun_type == QM_HW_PF) 638 hisi_zip_debug_regs_clear(hisi_zip); 639 } 640 641 static void hisi_zip_hw_error_init(struct hisi_zip *hisi_zip) 642 { 643 hisi_qm_hw_error_init(&hisi_zip->qm, QM_BASE_CE, 644 QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT, 0, 645 QM_DB_RANDOM_INVALID); 646 hisi_zip_hw_error_set_state(hisi_zip, true); 647 } 648 649 static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) 650 { 651 struct hisi_qm *qm = &hisi_zip->qm; 652 struct hisi_zip_ctrl *ctrl; 653 654 ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL); 655 if (!ctrl) 656 return -ENOMEM; 657 658 hisi_zip->ctrl = ctrl; 659 ctrl->hisi_zip = hisi_zip; 660 661 switch (qm->ver) { 662 case QM_HW_V1: 663 qm->ctrl_qp_num = HZIP_QUEUE_NUM_V1; 664 break; 665 666 case QM_HW_V2: 667 qm->ctrl_qp_num = HZIP_QUEUE_NUM_V2; 668 break; 669 670 default: 671 return -EINVAL; 672 } 673 674 hisi_zip_set_user_domain_and_cache(hisi_zip); 675 hisi_zip_hw_error_init(hisi_zip); 676 hisi_zip_debug_regs_clear(hisi_zip); 677 678 return 0; 679 } 680 681 /* Currently we only support equal assignment */ 682 static int hisi_zip_vf_q_assign(struct hisi_zip *hisi_zip, int num_vfs) 683 { 684 struct hisi_qm *qm = &hisi_zip->qm; 685 u32 qp_num = qm->qp_num; 686 u32 q_base = qp_num; 687 u32 q_num, remain_q_num, i; 688 int ret; 689 690 if (!num_vfs) 691 return -EINVAL; 692 693 remain_q_num = qm->ctrl_qp_num - qp_num; 694 if (remain_q_num < num_vfs) 695 return -EINVAL; 696 697 q_num = remain_q_num / num_vfs; 698 for (i = 1; i <= num_vfs; i++) { 699 if (i == num_vfs) 700 q_num += remain_q_num % num_vfs; 701 ret = hisi_qm_set_vft(qm, i, q_base, q_num); 702 if (ret) 703 return ret; 704 q_base += q_num; 705 } 706 707 return 0; 708 } 709 710 static int hisi_zip_clear_vft_config(struct hisi_zip *hisi_zip) 711 { 712 struct hisi_zip_ctrl *ctrl = hisi_zip->ctrl; 713 struct hisi_qm *qm = &hisi_zip->qm; 714 u32 i, num_vfs = ctrl->num_vfs; 715 int ret; 716 717 for (i = 1; i <= num_vfs; i++) { 718 ret = hisi_qm_set_vft(qm, i, 0, 0); 719 if (ret) 720 return ret; 721 } 722 723 ctrl->num_vfs = 0; 724 725 return 0; 726 } 727 728 static int hisi_zip_sriov_enable(struct pci_dev *pdev, int max_vfs) 729 { 730 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 731 int pre_existing_vfs, num_vfs, ret; 732 733 pre_existing_vfs = pci_num_vf(pdev); 734 735 if (pre_existing_vfs) { 736 dev_err(&pdev->dev, 737 "Can't enable VF. Please disable pre-enabled VFs!\n"); 738 return 0; 739 } 740 741 num_vfs = min_t(int, max_vfs, HZIP_VF_NUM); 742 743 ret = hisi_zip_vf_q_assign(hisi_zip, num_vfs); 744 if (ret) { 745 dev_err(&pdev->dev, "Can't assign queues for VF!\n"); 746 return ret; 747 } 748 749 hisi_zip->ctrl->num_vfs = num_vfs; 750 751 ret = pci_enable_sriov(pdev, num_vfs); 752 if (ret) { 753 dev_err(&pdev->dev, "Can't enable VF!\n"); 754 hisi_zip_clear_vft_config(hisi_zip); 755 return ret; 756 } 757 758 return num_vfs; 759 } 760 761 static int hisi_zip_sriov_disable(struct pci_dev *pdev) 762 { 763 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 764 765 if (pci_vfs_assigned(pdev)) { 766 dev_err(&pdev->dev, 767 "Can't disable VFs while VFs are assigned!\n"); 768 return -EPERM; 769 } 770 771 /* remove in hisi_zip_pci_driver will be called to free VF resources */ 772 pci_disable_sriov(pdev); 773 774 return hisi_zip_clear_vft_config(hisi_zip); 775 } 776 777 static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) 778 { 779 struct hisi_zip *hisi_zip; 780 enum qm_hw_ver rev_id; 781 struct hisi_qm *qm; 782 int ret; 783 784 rev_id = hisi_qm_get_hw_version(pdev); 785 if (rev_id == QM_HW_UNKNOWN) 786 return -EINVAL; 787 788 hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL); 789 if (!hisi_zip) 790 return -ENOMEM; 791 pci_set_drvdata(pdev, hisi_zip); 792 793 qm = &hisi_zip->qm; 794 qm->pdev = pdev; 795 qm->ver = rev_id; 796 797 qm->sqe_size = HZIP_SQE_SIZE; 798 qm->dev_name = hisi_zip_name; 799 qm->fun_type = (pdev->device == PCI_DEVICE_ID_ZIP_PF) ? QM_HW_PF : 800 QM_HW_VF; 801 switch (uacce_mode) { 802 case 0: 803 qm->use_dma_api = true; 804 break; 805 case 1: 806 qm->use_dma_api = false; 807 break; 808 case 2: 809 qm->use_dma_api = true; 810 break; 811 default: 812 return -EINVAL; 813 } 814 815 ret = hisi_qm_init(qm); 816 if (ret) { 817 dev_err(&pdev->dev, "Failed to init qm!\n"); 818 return ret; 819 } 820 821 if (qm->fun_type == QM_HW_PF) { 822 ret = hisi_zip_pf_probe_init(hisi_zip); 823 if (ret) 824 return ret; 825 826 qm->qp_base = HZIP_PF_DEF_Q_BASE; 827 qm->qp_num = pf_q_num; 828 } else if (qm->fun_type == QM_HW_VF) { 829 /* 830 * have no way to get qm configure in VM in v1 hardware, 831 * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force 832 * to trigger only one VF in v1 hardware. 833 * 834 * v2 hardware has no such problem. 835 */ 836 if (qm->ver == QM_HW_V1) { 837 qm->qp_base = HZIP_PF_DEF_Q_NUM; 838 qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; 839 } else if (qm->ver == QM_HW_V2) 840 /* v2 starts to support get vft by mailbox */ 841 hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); 842 } 843 844 ret = hisi_qm_start(qm); 845 if (ret) 846 goto err_qm_uninit; 847 848 ret = hisi_zip_debugfs_init(hisi_zip); 849 if (ret) 850 dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret); 851 852 hisi_zip_add_to_list(hisi_zip); 853 854 if (qm->fun_type == QM_HW_PF && vfs_num > 0) { 855 ret = hisi_zip_sriov_enable(pdev, vfs_num); 856 if (ret < 0) 857 goto err_remove_from_list; 858 } 859 860 return 0; 861 862 err_remove_from_list: 863 hisi_zip_remove_from_list(hisi_zip); 864 hisi_zip_debugfs_exit(hisi_zip); 865 hisi_qm_stop(qm); 866 err_qm_uninit: 867 hisi_qm_uninit(qm); 868 return ret; 869 } 870 871 static int hisi_zip_sriov_configure(struct pci_dev *pdev, int num_vfs) 872 { 873 if (num_vfs == 0) 874 return hisi_zip_sriov_disable(pdev); 875 else 876 return hisi_zip_sriov_enable(pdev, num_vfs); 877 } 878 879 static void hisi_zip_remove(struct pci_dev *pdev) 880 { 881 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 882 struct hisi_qm *qm = &hisi_zip->qm; 883 884 if (qm->fun_type == QM_HW_PF && hisi_zip->ctrl->num_vfs != 0) 885 hisi_zip_sriov_disable(pdev); 886 887 hisi_zip_debugfs_exit(hisi_zip); 888 hisi_qm_stop(qm); 889 890 if (qm->fun_type == QM_HW_PF) 891 hisi_zip_hw_error_set_state(hisi_zip, false); 892 893 hisi_qm_uninit(qm); 894 hisi_zip_remove_from_list(hisi_zip); 895 } 896 897 static void hisi_zip_log_hw_error(struct hisi_zip *hisi_zip, u32 err_sts) 898 { 899 const struct hisi_zip_hw_error *err = zip_hw_error; 900 struct device *dev = &hisi_zip->qm.pdev->dev; 901 u32 err_val; 902 903 while (err->msg) { 904 if (err->int_msk & err_sts) { 905 dev_warn(dev, "%s [error status=0x%x] found\n", 906 err->msg, err->int_msk); 907 908 if (HZIP_CORE_INT_STATUS_M_ECC & err->int_msk) { 909 err_val = readl(hisi_zip->qm.io_base + 910 HZIP_CORE_SRAM_ECC_ERR_INFO); 911 dev_warn(dev, "hisi-zip multi ecc sram num=0x%x\n", 912 ((err_val >> SRAM_ECC_ERR_NUM_SHIFT) & 913 0xFF)); 914 dev_warn(dev, "hisi-zip multi ecc sram addr=0x%x\n", 915 (err_val >> SRAM_ECC_ERR_ADDR_SHIFT)); 916 } 917 } 918 err++; 919 } 920 } 921 922 static pci_ers_result_t hisi_zip_hw_error_handle(struct hisi_zip *hisi_zip) 923 { 924 u32 err_sts; 925 926 /* read err sts */ 927 err_sts = readl(hisi_zip->qm.io_base + HZIP_CORE_INT_STATUS); 928 929 if (err_sts) { 930 hisi_zip_log_hw_error(hisi_zip, err_sts); 931 /* clear error interrupts */ 932 writel(err_sts, hisi_zip->qm.io_base + HZIP_CORE_INT_SOURCE); 933 934 return PCI_ERS_RESULT_NEED_RESET; 935 } 936 937 return PCI_ERS_RESULT_RECOVERED; 938 } 939 940 static pci_ers_result_t hisi_zip_process_hw_error(struct pci_dev *pdev) 941 { 942 struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); 943 struct device *dev = &pdev->dev; 944 pci_ers_result_t qm_ret, zip_ret; 945 946 if (!hisi_zip) { 947 dev_err(dev, 948 "Can't recover ZIP-error occurred during device init\n"); 949 return PCI_ERS_RESULT_NONE; 950 } 951 952 qm_ret = hisi_qm_hw_error_handle(&hisi_zip->qm); 953 954 zip_ret = hisi_zip_hw_error_handle(hisi_zip); 955 956 return (qm_ret == PCI_ERS_RESULT_NEED_RESET || 957 zip_ret == PCI_ERS_RESULT_NEED_RESET) ? 958 PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED; 959 } 960 961 static pci_ers_result_t hisi_zip_error_detected(struct pci_dev *pdev, 962 pci_channel_state_t state) 963 { 964 if (pdev->is_virtfn) 965 return PCI_ERS_RESULT_NONE; 966 967 dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state); 968 if (state == pci_channel_io_perm_failure) 969 return PCI_ERS_RESULT_DISCONNECT; 970 971 return hisi_zip_process_hw_error(pdev); 972 } 973 974 static const struct pci_error_handlers hisi_zip_err_handler = { 975 .error_detected = hisi_zip_error_detected, 976 }; 977 978 static struct pci_driver hisi_zip_pci_driver = { 979 .name = "hisi_zip", 980 .id_table = hisi_zip_dev_ids, 981 .probe = hisi_zip_probe, 982 .remove = hisi_zip_remove, 983 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? 984 hisi_zip_sriov_configure : NULL, 985 .err_handler = &hisi_zip_err_handler, 986 }; 987 988 static void hisi_zip_register_debugfs(void) 989 { 990 if (!debugfs_initialized()) 991 return; 992 993 hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL); 994 } 995 996 static void hisi_zip_unregister_debugfs(void) 997 { 998 debugfs_remove_recursive(hzip_debugfs_root); 999 } 1000 1001 static int __init hisi_zip_init(void) 1002 { 1003 int ret; 1004 1005 hisi_zip_register_debugfs(); 1006 1007 ret = pci_register_driver(&hisi_zip_pci_driver); 1008 if (ret < 0) { 1009 pr_err("Failed to register pci driver.\n"); 1010 goto err_pci; 1011 } 1012 1013 if (uacce_mode == 0 || uacce_mode == 2) { 1014 ret = hisi_zip_register_to_crypto(); 1015 if (ret < 0) { 1016 pr_err("Failed to register driver to crypto.\n"); 1017 goto err_crypto; 1018 } 1019 } 1020 1021 return 0; 1022 1023 err_crypto: 1024 pci_unregister_driver(&hisi_zip_pci_driver); 1025 err_pci: 1026 hisi_zip_unregister_debugfs(); 1027 1028 return ret; 1029 } 1030 1031 static void __exit hisi_zip_exit(void) 1032 { 1033 if (uacce_mode == 0 || uacce_mode == 2) 1034 hisi_zip_unregister_from_crypto(); 1035 pci_unregister_driver(&hisi_zip_pci_driver); 1036 hisi_zip_unregister_debugfs(); 1037 } 1038 1039 module_init(hisi_zip_init); 1040 module_exit(hisi_zip_exit); 1041 1042 MODULE_LICENSE("GPL v2"); 1043 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 1044 MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator"); 1045