1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018-2019 HiSilicon Limited. */ 3 #include <linux/acpi.h> 4 #include <linux/aer.h> 5 #include <linux/bitops.h> 6 #include <linux/debugfs.h> 7 #include <linux/init.h> 8 #include <linux/io.h> 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/pci.h> 12 #include <linux/topology.h> 13 #include "hpre.h" 14 15 #define HPRE_QUEUE_NUM_V2 1024 16 #define HPRE_QM_ABNML_INT_MASK 0x100004 17 #define HPRE_CTRL_CNT_CLR_CE_BIT BIT(0) 18 #define HPRE_COMM_CNT_CLR_CE 0x0 19 #define HPRE_CTRL_CNT_CLR_CE 0x301000 20 #define HPRE_FSM_MAX_CNT 0x301008 21 #define HPRE_VFG_AXQOS 0x30100c 22 #define HPRE_VFG_AXCACHE 0x301010 23 #define HPRE_RDCHN_INI_CFG 0x301014 24 #define HPRE_AWUSR_FP_CFG 0x301018 25 #define HPRE_BD_ENDIAN 0x301020 26 #define HPRE_ECC_BYPASS 0x301024 27 #define HPRE_RAS_WIDTH_CFG 0x301028 28 #define HPRE_POISON_BYPASS 0x30102c 29 #define HPRE_BD_ARUSR_CFG 0x301030 30 #define HPRE_BD_AWUSR_CFG 0x301034 31 #define HPRE_TYPES_ENB 0x301038 32 #define HPRE_DATA_RUSER_CFG 0x30103c 33 #define HPRE_DATA_WUSER_CFG 0x301040 34 #define HPRE_INT_MASK 0x301400 35 #define HPRE_INT_STATUS 0x301800 36 #define HPRE_CORE_INT_ENABLE 0 37 #define HPRE_CORE_INT_DISABLE 0x003fffff 38 #define HPRE_RAS_ECC_1BIT_TH 0x30140c 39 #define HPRE_RDCHN_INI_ST 0x301a00 40 #define HPRE_CLSTR_BASE 0x302000 41 #define HPRE_CORE_EN_OFFSET 0x04 42 #define HPRE_CORE_INI_CFG_OFFSET 0x20 43 #define HPRE_CORE_INI_STATUS_OFFSET 0x80 44 #define HPRE_CORE_HTBT_WARN_OFFSET 0x8c 45 #define HPRE_CORE_IS_SCHD_OFFSET 0x90 46 47 #define HPRE_RAS_CE_ENB 0x301410 48 #define HPRE_HAC_RAS_CE_ENABLE 0x1 49 #define HPRE_RAS_NFE_ENB 0x301414 50 #define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe 51 #define HPRE_RAS_FE_ENB 0x301418 52 #define HPRE_HAC_RAS_FE_ENABLE 0 53 54 #define HPRE_CORE_ENB (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET) 55 #define HPRE_CORE_INI_CFG (HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET) 56 #define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET) 57 #define HPRE_HAC_ECC1_CNT 0x301a04 58 #define HPRE_HAC_ECC2_CNT 0x301a08 59 #define HPRE_HAC_INT_STATUS 0x301800 60 #define HPRE_HAC_SOURCE_INT 0x301600 61 #define HPRE_CLSTR_ADDR_INTRVL 0x1000 62 #define HPRE_CLUSTER_INQURY 0x100 63 #define HPRE_CLSTR_ADDR_INQRY_RSLT 0x104 64 #define HPRE_TIMEOUT_ABNML_BIT 6 65 #define HPRE_PASID_EN_BIT 9 66 #define HPRE_REG_RD_INTVRL_US 10 67 #define HPRE_REG_RD_TMOUT_US 1000 68 #define HPRE_DBGFS_VAL_MAX_LEN 20 69 #define HPRE_PCI_DEVICE_ID 0xa258 70 #define HPRE_PCI_VF_DEVICE_ID 0xa259 71 #define HPRE_ADDR(qm, offset) ((qm)->io_base + (offset)) 72 #define HPRE_QM_USR_CFG_MASK 0xfffffffe 73 #define HPRE_QM_AXI_CFG_MASK 0xffff 74 #define HPRE_QM_VFG_AX_MASK 0xff 75 #define HPRE_BD_USR_MASK 0x3 76 #define HPRE_CLUSTER_CORE_MASK 0xf 77 78 #define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044 79 #define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0) 80 #define HPRE_WR_MSI_PORT BIT(2) 81 82 #define HPRE_CORE_ECC_2BIT_ERR BIT(1) 83 #define HPRE_OOO_ECC_2BIT_ERR BIT(5) 84 85 #define HPRE_QM_BME_FLR BIT(7) 86 #define HPRE_QM_PM_FLR BIT(11) 87 #define HPRE_QM_SRIOV_FLR BIT(12) 88 89 #define HPRE_VIA_MSI_DSM 1 90 #define HPRE_SQE_MASK_OFFSET 8 91 #define HPRE_SQE_MASK_LEN 24 92 93 static struct hisi_qm_list hpre_devices; 94 static const char hpre_name[] = "hisi_hpre"; 95 static struct dentry *hpre_debugfs_root; 96 static const struct pci_device_id hpre_dev_ids[] = { 97 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_DEVICE_ID) }, 98 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_VF_DEVICE_ID) }, 99 { 0, } 100 }; 101 102 MODULE_DEVICE_TABLE(pci, hpre_dev_ids); 103 104 struct hpre_hw_error { 105 u32 int_msk; 106 const char *msg; 107 }; 108 109 static const char * const hpre_debug_file_name[] = { 110 [HPRE_CURRENT_QM] = "current_qm", 111 [HPRE_CLEAR_ENABLE] = "rdclr_en", 112 [HPRE_CLUSTER_CTRL] = "cluster_ctrl", 113 }; 114 115 static const struct hpre_hw_error hpre_hw_errors[] = { 116 { .int_msk = BIT(0), .msg = "core_ecc_1bit_err_int_set" }, 117 { .int_msk = BIT(1), .msg = "core_ecc_2bit_err_int_set" }, 118 { .int_msk = BIT(2), .msg = "dat_wb_poison_int_set" }, 119 { .int_msk = BIT(3), .msg = "dat_rd_poison_int_set" }, 120 { .int_msk = BIT(4), .msg = "bd_rd_poison_int_set" }, 121 { .int_msk = BIT(5), .msg = "ooo_ecc_2bit_err_int_set" }, 122 { .int_msk = BIT(6), .msg = "cluster1_shb_timeout_int_set" }, 123 { .int_msk = BIT(7), .msg = "cluster2_shb_timeout_int_set" }, 124 { .int_msk = BIT(8), .msg = "cluster3_shb_timeout_int_set" }, 125 { .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" }, 126 { .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" }, 127 { .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" }, 128 { /* sentinel */ } 129 }; 130 131 static const u64 hpre_cluster_offsets[] = { 132 [HPRE_CLUSTER0] = 133 HPRE_CLSTR_BASE + HPRE_CLUSTER0 * HPRE_CLSTR_ADDR_INTRVL, 134 [HPRE_CLUSTER1] = 135 HPRE_CLSTR_BASE + HPRE_CLUSTER1 * HPRE_CLSTR_ADDR_INTRVL, 136 [HPRE_CLUSTER2] = 137 HPRE_CLSTR_BASE + HPRE_CLUSTER2 * HPRE_CLSTR_ADDR_INTRVL, 138 [HPRE_CLUSTER3] = 139 HPRE_CLSTR_BASE + HPRE_CLUSTER3 * HPRE_CLSTR_ADDR_INTRVL, 140 }; 141 142 static const struct debugfs_reg32 hpre_cluster_dfx_regs[] = { 143 {"CORES_EN_STATUS ", HPRE_CORE_EN_OFFSET}, 144 {"CORES_INI_CFG ", HPRE_CORE_INI_CFG_OFFSET}, 145 {"CORES_INI_STATUS ", HPRE_CORE_INI_STATUS_OFFSET}, 146 {"CORES_HTBT_WARN ", HPRE_CORE_HTBT_WARN_OFFSET}, 147 {"CORES_IS_SCHD ", HPRE_CORE_IS_SCHD_OFFSET}, 148 }; 149 150 static const struct debugfs_reg32 hpre_com_dfx_regs[] = { 151 {"READ_CLR_EN ", HPRE_CTRL_CNT_CLR_CE}, 152 {"AXQOS ", HPRE_VFG_AXQOS}, 153 {"AWUSR_CFG ", HPRE_AWUSR_FP_CFG}, 154 {"QM_ARUSR_MCFG1 ", QM_ARUSER_M_CFG_1}, 155 {"QM_AWUSR_MCFG1 ", QM_AWUSER_M_CFG_1}, 156 {"BD_ENDIAN ", HPRE_BD_ENDIAN}, 157 {"ECC_CHECK_CTRL ", HPRE_ECC_BYPASS}, 158 {"RAS_INT_WIDTH ", HPRE_RAS_WIDTH_CFG}, 159 {"POISON_BYPASS ", HPRE_POISON_BYPASS}, 160 {"BD_ARUSER ", HPRE_BD_ARUSR_CFG}, 161 {"BD_AWUSER ", HPRE_BD_AWUSR_CFG}, 162 {"DATA_ARUSER ", HPRE_DATA_RUSER_CFG}, 163 {"DATA_AWUSER ", HPRE_DATA_WUSER_CFG}, 164 {"INT_STATUS ", HPRE_INT_STATUS}, 165 }; 166 167 static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = { 168 "send_cnt", 169 "recv_cnt", 170 "send_fail_cnt", 171 "send_busy_cnt", 172 "over_thrhld_cnt", 173 "overtime_thrhld", 174 "invalid_req_cnt" 175 }; 176 177 static int pf_q_num_set(const char *val, const struct kernel_param *kp) 178 { 179 return q_num_set(val, kp, HPRE_PCI_DEVICE_ID); 180 } 181 182 static const struct kernel_param_ops hpre_pf_q_num_ops = { 183 .set = pf_q_num_set, 184 .get = param_get_int, 185 }; 186 187 static u32 pf_q_num = HPRE_PF_DEF_Q_NUM; 188 module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444); 189 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(1-1024)"); 190 191 static const struct kernel_param_ops vfs_num_ops = { 192 .set = vfs_num_set, 193 .get = param_get_int, 194 }; 195 196 static u32 vfs_num; 197 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); 198 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); 199 200 struct hisi_qp *hpre_create_qp(void) 201 { 202 int node = cpu_to_node(smp_processor_id()); 203 struct hisi_qp *qp = NULL; 204 int ret; 205 206 ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, 0, node, &qp); 207 if (!ret) 208 return qp; 209 210 return NULL; 211 } 212 213 static int hpre_cfg_by_dsm(struct hisi_qm *qm) 214 { 215 struct device *dev = &qm->pdev->dev; 216 union acpi_object *obj; 217 guid_t guid; 218 219 if (guid_parse("b06b81ab-0134-4a45-9b0c-483447b95fa7", &guid)) { 220 dev_err(dev, "Hpre GUID failed\n"); 221 return -EINVAL; 222 } 223 224 /* Switch over to MSI handling due to non-standard PCI implementation */ 225 obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid, 226 0, HPRE_VIA_MSI_DSM, NULL); 227 if (!obj) { 228 dev_err(dev, "ACPI handle failed!\n"); 229 return -EIO; 230 } 231 232 ACPI_FREE(obj); 233 234 return 0; 235 } 236 237 /* 238 * For Hi1620, we shoul disable FLR triggered by hardware (BME/PM/SRIOV). 239 * Or it may stay in D3 state when we bind and unbind hpre quickly, 240 * as it does FLR triggered by hardware. 241 */ 242 static void disable_flr_of_bme(struct hisi_qm *qm) 243 { 244 u32 val; 245 246 val = readl(HPRE_ADDR(qm, QM_PEH_AXUSER_CFG)); 247 val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR); 248 val |= HPRE_QM_PM_FLR; 249 writel(val, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG)); 250 writel(PEH_AXUSER_CFG_ENABLE, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG_ENABLE)); 251 } 252 253 static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) 254 { 255 struct device *dev = &qm->pdev->dev; 256 unsigned long offset; 257 int ret, i; 258 u32 val; 259 260 writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE)); 261 writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE)); 262 writel_relaxed(HPRE_QM_AXI_CFG_MASK, HPRE_ADDR(qm, QM_AXI_M_CFG)); 263 264 /* HPRE need more time, we close this interrupt */ 265 val = readl_relaxed(HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); 266 val |= BIT(HPRE_TIMEOUT_ABNML_BIT); 267 writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); 268 269 writel(0x1, HPRE_ADDR(qm, HPRE_TYPES_ENB)); 270 writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE)); 271 writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN)); 272 writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK)); 273 writel(0x0, HPRE_ADDR(qm, HPRE_RAS_ECC_1BIT_TH)); 274 writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS)); 275 writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE)); 276 writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS)); 277 278 writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_ARUSR_CFG)); 279 writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_AWUSR_CFG)); 280 writel(0x1, HPRE_ADDR(qm, HPRE_RDCHN_INI_CFG)); 281 ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, HPRE_RDCHN_INI_ST), val, 282 val & BIT(0), 283 HPRE_REG_RD_INTVRL_US, 284 HPRE_REG_RD_TMOUT_US); 285 if (ret) { 286 dev_err(dev, "read rd channel timeout fail!\n"); 287 return -ETIMEDOUT; 288 } 289 290 for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { 291 offset = i * HPRE_CLSTR_ADDR_INTRVL; 292 293 /* clusters initiating */ 294 writel(HPRE_CLUSTER_CORE_MASK, 295 HPRE_ADDR(qm, offset + HPRE_CORE_ENB)); 296 writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG)); 297 ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset + 298 HPRE_CORE_INI_STATUS), val, 299 ((val & HPRE_CLUSTER_CORE_MASK) == 300 HPRE_CLUSTER_CORE_MASK), 301 HPRE_REG_RD_INTVRL_US, 302 HPRE_REG_RD_TMOUT_US); 303 if (ret) { 304 dev_err(dev, 305 "cluster %d int st status timeout!\n", i); 306 return -ETIMEDOUT; 307 } 308 } 309 310 ret = hpre_cfg_by_dsm(qm); 311 if (ret) 312 dev_err(dev, "acpi_evaluate_dsm err.\n"); 313 314 disable_flr_of_bme(qm); 315 316 return ret; 317 } 318 319 static void hpre_cnt_regs_clear(struct hisi_qm *qm) 320 { 321 unsigned long offset; 322 int i; 323 324 /* clear current_qm */ 325 writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); 326 writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); 327 328 /* clear clusterX/cluster_ctrl */ 329 for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { 330 offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; 331 writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY); 332 } 333 334 /* clear rdclr_en */ 335 writel(0x0, qm->io_base + HPRE_CTRL_CNT_CLR_CE); 336 337 hisi_qm_debug_regs_clear(qm); 338 } 339 340 static void hpre_hw_error_disable(struct hisi_qm *qm) 341 { 342 u32 val; 343 344 /* disable hpre hw error interrupts */ 345 writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK); 346 347 /* disable HPRE block master OOO when m-bit error occur */ 348 val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); 349 val &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; 350 writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); 351 } 352 353 static void hpre_hw_error_enable(struct hisi_qm *qm) 354 { 355 u32 val; 356 357 /* clear HPRE hw error source if having */ 358 writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT); 359 360 /* enable hpre hw error interrupts */ 361 writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK); 362 writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB); 363 writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB); 364 writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB); 365 366 /* enable HPRE block master OOO when m-bit error occur */ 367 val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); 368 val |= HPRE_AM_OOO_SHUTDOWN_ENABLE; 369 writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); 370 } 371 372 static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) 373 { 374 struct hpre *hpre = container_of(file->debug, struct hpre, debug); 375 376 return &hpre->qm; 377 } 378 379 static u32 hpre_current_qm_read(struct hpre_debugfs_file *file) 380 { 381 struct hisi_qm *qm = hpre_file_to_qm(file); 382 383 return readl(qm->io_base + QM_DFX_MB_CNT_VF); 384 } 385 386 static int hpre_current_qm_write(struct hpre_debugfs_file *file, u32 val) 387 { 388 struct hisi_qm *qm = hpre_file_to_qm(file); 389 u32 num_vfs = qm->vfs_num; 390 u32 vfq_num, tmp; 391 392 if (val > num_vfs) 393 return -EINVAL; 394 395 /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ 396 if (val == 0) { 397 qm->debug.curr_qm_qp_num = qm->qp_num; 398 } else { 399 vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs; 400 if (val == num_vfs) { 401 qm->debug.curr_qm_qp_num = 402 qm->ctrl_qp_num - qm->qp_num - (num_vfs - 1) * vfq_num; 403 } else { 404 qm->debug.curr_qm_qp_num = vfq_num; 405 } 406 } 407 408 writel(val, qm->io_base + QM_DFX_MB_CNT_VF); 409 writel(val, qm->io_base + QM_DFX_DB_CNT_VF); 410 411 tmp = val | 412 (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); 413 writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); 414 415 tmp = val | 416 (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); 417 writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); 418 419 return 0; 420 } 421 422 static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file) 423 { 424 struct hisi_qm *qm = hpre_file_to_qm(file); 425 426 return readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) & 427 HPRE_CTRL_CNT_CLR_CE_BIT; 428 } 429 430 static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val) 431 { 432 struct hisi_qm *qm = hpre_file_to_qm(file); 433 u32 tmp; 434 435 if (val != 1 && val != 0) 436 return -EINVAL; 437 438 tmp = (readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) & 439 ~HPRE_CTRL_CNT_CLR_CE_BIT) | val; 440 writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE); 441 442 return 0; 443 } 444 445 static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file) 446 { 447 struct hisi_qm *qm = hpre_file_to_qm(file); 448 int cluster_index = file->index - HPRE_CLUSTER_CTRL; 449 unsigned long offset = HPRE_CLSTR_BASE + 450 cluster_index * HPRE_CLSTR_ADDR_INTRVL; 451 452 return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT); 453 } 454 455 static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) 456 { 457 struct hisi_qm *qm = hpre_file_to_qm(file); 458 int cluster_index = file->index - HPRE_CLUSTER_CTRL; 459 unsigned long offset = HPRE_CLSTR_BASE + cluster_index * 460 HPRE_CLSTR_ADDR_INTRVL; 461 462 writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY); 463 464 return 0; 465 } 466 467 static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf, 468 size_t count, loff_t *pos) 469 { 470 struct hpre_debugfs_file *file = filp->private_data; 471 char tbuf[HPRE_DBGFS_VAL_MAX_LEN]; 472 u32 val; 473 int ret; 474 475 spin_lock_irq(&file->lock); 476 switch (file->type) { 477 case HPRE_CURRENT_QM: 478 val = hpre_current_qm_read(file); 479 break; 480 case HPRE_CLEAR_ENABLE: 481 val = hpre_clear_enable_read(file); 482 break; 483 case HPRE_CLUSTER_CTRL: 484 val = hpre_cluster_inqry_read(file); 485 break; 486 default: 487 spin_unlock_irq(&file->lock); 488 return -EINVAL; 489 } 490 spin_unlock_irq(&file->lock); 491 ret = snprintf(tbuf, HPRE_DBGFS_VAL_MAX_LEN, "%u\n", val); 492 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 493 } 494 495 static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf, 496 size_t count, loff_t *pos) 497 { 498 struct hpre_debugfs_file *file = filp->private_data; 499 char tbuf[HPRE_DBGFS_VAL_MAX_LEN]; 500 unsigned long val; 501 int len, ret; 502 503 if (*pos != 0) 504 return 0; 505 506 if (count >= HPRE_DBGFS_VAL_MAX_LEN) 507 return -ENOSPC; 508 509 len = simple_write_to_buffer(tbuf, HPRE_DBGFS_VAL_MAX_LEN - 1, 510 pos, buf, count); 511 if (len < 0) 512 return len; 513 514 tbuf[len] = '\0'; 515 if (kstrtoul(tbuf, 0, &val)) 516 return -EFAULT; 517 518 spin_lock_irq(&file->lock); 519 switch (file->type) { 520 case HPRE_CURRENT_QM: 521 ret = hpre_current_qm_write(file, val); 522 if (ret) 523 goto err_input; 524 break; 525 case HPRE_CLEAR_ENABLE: 526 ret = hpre_clear_enable_write(file, val); 527 if (ret) 528 goto err_input; 529 break; 530 case HPRE_CLUSTER_CTRL: 531 ret = hpre_cluster_inqry_write(file, val); 532 if (ret) 533 goto err_input; 534 break; 535 default: 536 ret = -EINVAL; 537 goto err_input; 538 } 539 spin_unlock_irq(&file->lock); 540 541 return count; 542 543 err_input: 544 spin_unlock_irq(&file->lock); 545 return ret; 546 } 547 548 static const struct file_operations hpre_ctrl_debug_fops = { 549 .owner = THIS_MODULE, 550 .open = simple_open, 551 .read = hpre_ctrl_debug_read, 552 .write = hpre_ctrl_debug_write, 553 }; 554 555 static int hpre_debugfs_atomic64_get(void *data, u64 *val) 556 { 557 struct hpre_dfx *dfx_item = data; 558 559 *val = atomic64_read(&dfx_item->value); 560 561 return 0; 562 } 563 564 static int hpre_debugfs_atomic64_set(void *data, u64 val) 565 { 566 struct hpre_dfx *dfx_item = data; 567 struct hpre_dfx *hpre_dfx = NULL; 568 569 if (dfx_item->type == HPRE_OVERTIME_THRHLD) { 570 hpre_dfx = dfx_item - HPRE_OVERTIME_THRHLD; 571 atomic64_set(&hpre_dfx[HPRE_OVER_THRHLD_CNT].value, 0); 572 } else if (val) { 573 return -EINVAL; 574 } 575 576 atomic64_set(&dfx_item->value, val); 577 578 return 0; 579 } 580 581 DEFINE_DEBUGFS_ATTRIBUTE(hpre_atomic64_ops, hpre_debugfs_atomic64_get, 582 hpre_debugfs_atomic64_set, "%llu\n"); 583 584 static int hpre_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir, 585 enum hpre_ctrl_dbgfs_file type, int indx) 586 { 587 struct hpre *hpre = container_of(qm, struct hpre, qm); 588 struct hpre_debug *dbg = &hpre->debug; 589 struct dentry *file_dir; 590 591 if (dir) 592 file_dir = dir; 593 else 594 file_dir = qm->debug.debug_root; 595 596 if (type >= HPRE_DEBUG_FILE_NUM) 597 return -EINVAL; 598 599 spin_lock_init(&dbg->files[indx].lock); 600 dbg->files[indx].debug = dbg; 601 dbg->files[indx].type = type; 602 dbg->files[indx].index = indx; 603 debugfs_create_file(hpre_debug_file_name[type], 0600, file_dir, 604 dbg->files + indx, &hpre_ctrl_debug_fops); 605 606 return 0; 607 } 608 609 static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm) 610 { 611 struct device *dev = &qm->pdev->dev; 612 struct debugfs_regset32 *regset; 613 614 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 615 if (!regset) 616 return -ENOMEM; 617 618 regset->regs = hpre_com_dfx_regs; 619 regset->nregs = ARRAY_SIZE(hpre_com_dfx_regs); 620 regset->base = qm->io_base; 621 622 debugfs_create_regset32("regs", 0444, qm->debug.debug_root, regset); 623 return 0; 624 } 625 626 static int hpre_cluster_debugfs_init(struct hisi_qm *qm) 627 { 628 struct device *dev = &qm->pdev->dev; 629 char buf[HPRE_DBGFS_VAL_MAX_LEN]; 630 struct debugfs_regset32 *regset; 631 struct dentry *tmp_d; 632 int i, ret; 633 634 for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { 635 ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i); 636 if (ret < 0) 637 return -EINVAL; 638 tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); 639 640 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 641 if (!regset) 642 return -ENOMEM; 643 644 regset->regs = hpre_cluster_dfx_regs; 645 regset->nregs = ARRAY_SIZE(hpre_cluster_dfx_regs); 646 regset->base = qm->io_base + hpre_cluster_offsets[i]; 647 648 debugfs_create_regset32("regs", 0444, tmp_d, regset); 649 ret = hpre_create_debugfs_file(qm, tmp_d, HPRE_CLUSTER_CTRL, 650 i + HPRE_CLUSTER_CTRL); 651 if (ret) 652 return ret; 653 } 654 655 return 0; 656 } 657 658 static int hpre_ctrl_debug_init(struct hisi_qm *qm) 659 { 660 int ret; 661 662 ret = hpre_create_debugfs_file(qm, NULL, HPRE_CURRENT_QM, 663 HPRE_CURRENT_QM); 664 if (ret) 665 return ret; 666 667 ret = hpre_create_debugfs_file(qm, NULL, HPRE_CLEAR_ENABLE, 668 HPRE_CLEAR_ENABLE); 669 if (ret) 670 return ret; 671 672 ret = hpre_pf_comm_regs_debugfs_init(qm); 673 if (ret) 674 return ret; 675 676 return hpre_cluster_debugfs_init(qm); 677 } 678 679 static void hpre_dfx_debug_init(struct hisi_qm *qm) 680 { 681 struct hpre *hpre = container_of(qm, struct hpre, qm); 682 struct hpre_dfx *dfx = hpre->debug.dfx; 683 struct dentry *parent; 684 int i; 685 686 parent = debugfs_create_dir("hpre_dfx", qm->debug.debug_root); 687 for (i = 0; i < HPRE_DFX_FILE_NUM; i++) { 688 dfx[i].type = i; 689 debugfs_create_file(hpre_dfx_files[i], 0644, parent, &dfx[i], 690 &hpre_atomic64_ops); 691 } 692 } 693 694 static int hpre_debugfs_init(struct hisi_qm *qm) 695 { 696 struct device *dev = &qm->pdev->dev; 697 int ret; 698 699 qm->debug.debug_root = debugfs_create_dir(dev_name(dev), 700 hpre_debugfs_root); 701 702 qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET; 703 qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN; 704 ret = hisi_qm_debug_init(qm); 705 if (ret) 706 goto failed_to_create; 707 708 if (qm->pdev->device == HPRE_PCI_DEVICE_ID) { 709 ret = hpre_ctrl_debug_init(qm); 710 if (ret) 711 goto failed_to_create; 712 } 713 714 hpre_dfx_debug_init(qm); 715 716 return 0; 717 718 failed_to_create: 719 debugfs_remove_recursive(qm->debug.debug_root); 720 return ret; 721 } 722 723 static void hpre_debugfs_exit(struct hisi_qm *qm) 724 { 725 debugfs_remove_recursive(qm->debug.debug_root); 726 } 727 728 static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) 729 { 730 if (pdev->revision == QM_HW_V1) { 731 pci_warn(pdev, "HPRE version 1 is not supported!\n"); 732 return -EINVAL; 733 } 734 735 qm->pdev = pdev; 736 qm->ver = pdev->revision; 737 qm->sqe_size = HPRE_SQE_SIZE; 738 qm->dev_name = hpre_name; 739 740 qm->fun_type = (pdev->device == HPRE_PCI_DEVICE_ID) ? 741 QM_HW_PF : QM_HW_VF; 742 if (qm->fun_type == QM_HW_PF) { 743 qm->qp_base = HPRE_PF_DEF_Q_BASE; 744 qm->qp_num = pf_q_num; 745 qm->debug.curr_qm_qp_num = pf_q_num; 746 qm->qm_list = &hpre_devices; 747 } 748 749 return hisi_qm_init(qm); 750 } 751 752 static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts) 753 { 754 const struct hpre_hw_error *err = hpre_hw_errors; 755 struct device *dev = &qm->pdev->dev; 756 757 while (err->msg) { 758 if (err->int_msk & err_sts) 759 dev_warn(dev, "%s [error status=0x%x] found\n", 760 err->msg, err->int_msk); 761 err++; 762 } 763 } 764 765 static u32 hpre_get_hw_err_status(struct hisi_qm *qm) 766 { 767 return readl(qm->io_base + HPRE_HAC_INT_STATUS); 768 } 769 770 static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) 771 { 772 writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT); 773 } 774 775 static void hpre_open_axi_master_ooo(struct hisi_qm *qm) 776 { 777 u32 value; 778 779 value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); 780 writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE, 781 HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB)); 782 writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE, 783 HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB)); 784 } 785 786 static const struct hisi_qm_err_ini hpre_err_ini = { 787 .hw_init = hpre_set_user_domain_and_cache, 788 .hw_err_enable = hpre_hw_error_enable, 789 .hw_err_disable = hpre_hw_error_disable, 790 .get_dev_hw_err_status = hpre_get_hw_err_status, 791 .clear_dev_hw_err_status = hpre_clear_hw_err_status, 792 .log_dev_hw_err = hpre_log_hw_error, 793 .open_axi_master_ooo = hpre_open_axi_master_ooo, 794 .err_info = { 795 .ce = QM_BASE_CE, 796 .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT, 797 .fe = 0, 798 .ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | 799 HPRE_OOO_ECC_2BIT_ERR, 800 .msi_wr_port = HPRE_WR_MSI_PORT, 801 .acpi_rst = "HRST", 802 } 803 }; 804 805 static int hpre_pf_probe_init(struct hpre *hpre) 806 { 807 struct hisi_qm *qm = &hpre->qm; 808 int ret; 809 810 qm->ctrl_qp_num = HPRE_QUEUE_NUM_V2; 811 812 ret = hpre_set_user_domain_and_cache(qm); 813 if (ret) 814 return ret; 815 816 qm->err_ini = &hpre_err_ini; 817 hisi_qm_dev_err_init(qm); 818 819 return 0; 820 } 821 822 static int hpre_probe_init(struct hpre *hpre) 823 { 824 struct hisi_qm *qm = &hpre->qm; 825 int ret; 826 827 if (qm->fun_type == QM_HW_PF) { 828 ret = hpre_pf_probe_init(hpre); 829 if (ret) 830 return ret; 831 } 832 833 return 0; 834 } 835 836 static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) 837 { 838 struct hisi_qm *qm; 839 struct hpre *hpre; 840 int ret; 841 842 hpre = devm_kzalloc(&pdev->dev, sizeof(*hpre), GFP_KERNEL); 843 if (!hpre) 844 return -ENOMEM; 845 846 qm = &hpre->qm; 847 ret = hpre_qm_init(qm, pdev); 848 if (ret) { 849 pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret); 850 return ret; 851 } 852 853 ret = hpre_probe_init(hpre); 854 if (ret) { 855 pci_err(pdev, "Failed to probe (%d)!\n", ret); 856 goto err_with_qm_init; 857 } 858 859 ret = hisi_qm_start(qm); 860 if (ret) 861 goto err_with_err_init; 862 863 ret = hpre_debugfs_init(qm); 864 if (ret) 865 dev_warn(&pdev->dev, "init debugfs fail!\n"); 866 867 hisi_qm_add_to_list(qm, &hpre_devices); 868 869 ret = hpre_algs_register(); 870 if (ret < 0) { 871 pci_err(pdev, "fail to register algs to crypto!\n"); 872 goto err_with_qm_start; 873 } 874 875 if (qm->fun_type == QM_HW_PF && vfs_num) { 876 ret = hisi_qm_sriov_enable(pdev, vfs_num); 877 if (ret < 0) 878 goto err_with_crypto_register; 879 } 880 881 return 0; 882 883 err_with_crypto_register: 884 hpre_algs_unregister(); 885 886 err_with_qm_start: 887 hisi_qm_del_from_list(qm, &hpre_devices); 888 hpre_debugfs_exit(qm); 889 hisi_qm_stop(qm); 890 891 err_with_err_init: 892 hisi_qm_dev_err_uninit(qm); 893 894 err_with_qm_init: 895 hisi_qm_uninit(qm); 896 897 return ret; 898 } 899 900 static void hpre_remove(struct pci_dev *pdev) 901 { 902 struct hpre *hpre = pci_get_drvdata(pdev); 903 struct hisi_qm *qm = &hpre->qm; 904 int ret; 905 906 hpre_algs_unregister(); 907 hisi_qm_del_from_list(qm, &hpre_devices); 908 if (qm->fun_type == QM_HW_PF && qm->vfs_num) { 909 ret = hisi_qm_sriov_disable(pdev); 910 if (ret) { 911 pci_err(pdev, "Disable SRIOV fail!\n"); 912 return; 913 } 914 } 915 if (qm->fun_type == QM_HW_PF) { 916 hpre_cnt_regs_clear(qm); 917 qm->debug.curr_qm_qp_num = 0; 918 } 919 920 hpre_debugfs_exit(qm); 921 hisi_qm_stop(qm); 922 hisi_qm_dev_err_uninit(qm); 923 hisi_qm_uninit(qm); 924 } 925 926 927 static const struct pci_error_handlers hpre_err_handler = { 928 .error_detected = hisi_qm_dev_err_detected, 929 .slot_reset = hisi_qm_dev_slot_reset, 930 .reset_prepare = hisi_qm_reset_prepare, 931 .reset_done = hisi_qm_reset_done, 932 }; 933 934 static struct pci_driver hpre_pci_driver = { 935 .name = hpre_name, 936 .id_table = hpre_dev_ids, 937 .probe = hpre_probe, 938 .remove = hpre_remove, 939 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ? 940 hisi_qm_sriov_configure : NULL, 941 .err_handler = &hpre_err_handler, 942 }; 943 944 static void hpre_register_debugfs(void) 945 { 946 if (!debugfs_initialized()) 947 return; 948 949 hpre_debugfs_root = debugfs_create_dir(hpre_name, NULL); 950 } 951 952 static void hpre_unregister_debugfs(void) 953 { 954 debugfs_remove_recursive(hpre_debugfs_root); 955 } 956 957 static int __init hpre_init(void) 958 { 959 int ret; 960 961 hisi_qm_init_list(&hpre_devices); 962 hpre_register_debugfs(); 963 964 ret = pci_register_driver(&hpre_pci_driver); 965 if (ret) { 966 hpre_unregister_debugfs(); 967 pr_err("hpre: can't register hisi hpre driver.\n"); 968 } 969 970 return ret; 971 } 972 973 static void __exit hpre_exit(void) 974 { 975 pci_unregister_driver(&hpre_pci_driver); 976 hpre_unregister_debugfs(); 977 } 978 979 module_init(hpre_init); 980 module_exit(hpre_exit); 981 982 MODULE_LICENSE("GPL v2"); 983 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>"); 984 MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator"); 985