1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 4 #include <linux/acpi.h> 5 #include <linux/aer.h> 6 #include <linux/bitops.h> 7 #include <linux/debugfs.h> 8 #include <linux/init.h> 9 #include <linux/io.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/seq_file.h> 14 #include <linux/topology.h> 15 16 #include "sec.h" 17 18 #define SEC_VF_NUM 63 19 #define SEC_QUEUE_NUM_V1 4096 20 #define SEC_QUEUE_NUM_V2 1024 21 #define SEC_PF_PCI_DEVICE_ID 0xa255 22 #define SEC_VF_PCI_DEVICE_ID 0xa256 23 24 #define SEC_XTS_MIV_ENABLE_REG 0x301384 25 #define SEC_XTS_MIV_ENABLE_MSK 0x7FFFFFFF 26 #define SEC_XTS_MIV_DISABLE_MSK 0xFFFFFFFF 27 #define SEC_BD_ERR_CHK_EN1 0xfffff7fd 28 #define SEC_BD_ERR_CHK_EN2 0xffffbfff 29 30 #define SEC_SQE_SIZE 128 31 #define SEC_SQ_SIZE (SEC_SQE_SIZE * QM_Q_DEPTH) 32 #define SEC_PF_DEF_Q_NUM 64 33 #define SEC_PF_DEF_Q_BASE 0 34 #define SEC_CTX_Q_NUM_DEF 24 35 36 #define SEC_CTRL_CNT_CLR_CE 0x301120 37 #define SEC_CTRL_CNT_CLR_CE_BIT BIT(0) 38 #define SEC_ENGINE_PF_CFG_OFF 0x300000 39 #define SEC_ACC_COMMON_REG_OFF 0x1000 40 #define SEC_CORE_INT_SOURCE 0x301010 41 #define SEC_CORE_INT_MASK 0x301000 42 #define SEC_CORE_INT_STATUS 0x301008 43 #define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C14 44 #define SEC_ECC_NUM(err) (((err) >> 16) & 0xFF) 45 #define SEC_ECC_ADDR(err) ((err) >> 0) 46 #define SEC_CORE_INT_DISABLE 0x0 47 #define SEC_CORE_INT_ENABLE 0x1ff 48 49 #define SEC_RAS_CE_REG 0x50 50 #define SEC_RAS_FE_REG 0x54 51 #define SEC_RAS_NFE_REG 0x58 52 #define SEC_RAS_CE_ENB_MSK 0x88 53 #define SEC_RAS_FE_ENB_MSK 0x0 54 #define SEC_RAS_NFE_ENB_MSK 0x177 55 #define SEC_RAS_DISABLE 0x0 56 #define SEC_MEM_START_INIT_REG 0x0100 57 #define SEC_MEM_INIT_DONE_REG 0x0104 58 #define SEC_QM_ABNORMAL_INT_MASK 0x100004 59 60 #define SEC_CONTROL_REG 0x0200 61 #define SEC_TRNG_EN_SHIFT 8 62 #define SEC_CLK_GATE_ENABLE BIT(3) 63 #define SEC_CLK_GATE_DISABLE (~BIT(3)) 64 #define SEC_AXI_SHUTDOWN_ENABLE BIT(12) 65 #define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF 66 67 #define SEC_INTERFACE_USER_CTRL0_REG 0x0220 68 #define SEC_INTERFACE_USER_CTRL1_REG 0x0224 69 #define SEC_BD_ERR_CHK_EN_REG1 0x0384 70 #define SEC_BD_ERR_CHK_EN_REG2 0x038c 71 72 #define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) 73 #define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) 74 #define SEC_CORE_INT_STATUS_M_ECC BIT(2) 75 76 #define SEC_DELAY_10_US 10 77 #define SEC_POLL_TIMEOUT_US 1000 78 #define SEC_VF_CNT_MASK 0xffffffc0 79 #define SEC_DBGFS_VAL_MAX_LEN 20 80 81 #define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \ 82 SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF) 83 84 struct sec_hw_error { 85 u32 int_msk; 86 const char *msg; 87 }; 88 89 static const char sec_name[] = "hisi_sec2"; 90 static struct dentry *sec_debugfs_root; 91 static LIST_HEAD(sec_list); 92 static DEFINE_MUTEX(sec_list_lock); 93 94 static const struct sec_hw_error sec_hw_errors[] = { 95 {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"}, 96 {.int_msk = BIT(1), .msg = "sec_axi_bresp_err_rint"}, 97 {.int_msk = BIT(2), .msg = "sec_ecc_2bit_err_rint"}, 98 {.int_msk = BIT(3), .msg = "sec_ecc_1bit_err_rint"}, 99 {.int_msk = BIT(4), .msg = "sec_req_trng_timeout_rint"}, 100 {.int_msk = BIT(5), .msg = "sec_fsm_hbeat_rint"}, 101 {.int_msk = BIT(6), .msg = "sec_channel_req_rng_timeout_rint"}, 102 {.int_msk = BIT(7), .msg = "sec_bd_err_rint"}, 103 {.int_msk = BIT(8), .msg = "sec_chain_buff_err_rint"}, 104 { /* sentinel */ } 105 }; 106 107 struct sec_dev *sec_find_device(int node) 108 { 109 #define SEC_NUMA_MAX_DISTANCE 100 110 int min_distance = SEC_NUMA_MAX_DISTANCE; 111 int dev_node = 0, free_qp_num = 0; 112 struct sec_dev *sec, *ret = NULL; 113 struct hisi_qm *qm; 114 struct device *dev; 115 116 mutex_lock(&sec_list_lock); 117 list_for_each_entry(sec, &sec_list, list) { 118 qm = &sec->qm; 119 dev = &qm->pdev->dev; 120 #ifdef CONFIG_NUMA 121 dev_node = dev->numa_node; 122 if (dev_node < 0) 123 dev_node = 0; 124 #endif 125 if (node_distance(dev_node, node) < min_distance) { 126 free_qp_num = hisi_qm_get_free_qp_num(qm); 127 if (free_qp_num >= sec->ctx_q_num) { 128 ret = sec; 129 min_distance = node_distance(dev_node, node); 130 } 131 } 132 } 133 mutex_unlock(&sec_list_lock); 134 135 return ret; 136 } 137 138 static const char * const sec_dbg_file_name[] = { 139 [SEC_CURRENT_QM] = "current_qm", 140 [SEC_CLEAR_ENABLE] = "clear_enable", 141 }; 142 143 static struct debugfs_reg32 sec_dfx_regs[] = { 144 {"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010}, 145 {"SEC_SAA_EN ", 0x301270}, 146 {"SEC_BD_LATENCY_MIN ", 0x301600}, 147 {"SEC_BD_LATENCY_MAX ", 0x301608}, 148 {"SEC_BD_LATENCY_AVG ", 0x30160C}, 149 {"SEC_BD_NUM_IN_SAA0 ", 0x301670}, 150 {"SEC_BD_NUM_IN_SAA1 ", 0x301674}, 151 {"SEC_BD_NUM_IN_SEC ", 0x301680}, 152 {"SEC_ECC_1BIT_CNT ", 0x301C00}, 153 {"SEC_ECC_1BIT_INFO ", 0x301C04}, 154 {"SEC_ECC_2BIT_CNT ", 0x301C10}, 155 {"SEC_ECC_2BIT_INFO ", 0x301C14}, 156 {"SEC_BD_SAA0 ", 0x301C20}, 157 {"SEC_BD_SAA1 ", 0x301C24}, 158 {"SEC_BD_SAA2 ", 0x301C28}, 159 {"SEC_BD_SAA3 ", 0x301C2C}, 160 {"SEC_BD_SAA4 ", 0x301C30}, 161 {"SEC_BD_SAA5 ", 0x301C34}, 162 {"SEC_BD_SAA6 ", 0x301C38}, 163 {"SEC_BD_SAA7 ", 0x301C3C}, 164 {"SEC_BD_SAA8 ", 0x301C40}, 165 }; 166 167 static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp) 168 { 169 struct pci_dev *pdev; 170 u32 n, q_num; 171 u8 rev_id; 172 int ret; 173 174 if (!val) 175 return -EINVAL; 176 177 pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, 178 SEC_PF_PCI_DEVICE_ID, NULL); 179 if (!pdev) { 180 q_num = min_t(u32, SEC_QUEUE_NUM_V1, SEC_QUEUE_NUM_V2); 181 pr_info("No device, suppose queue number is %d!\n", q_num); 182 } else { 183 rev_id = pdev->revision; 184 185 switch (rev_id) { 186 case QM_HW_V1: 187 q_num = SEC_QUEUE_NUM_V1; 188 break; 189 case QM_HW_V2: 190 q_num = SEC_QUEUE_NUM_V2; 191 break; 192 default: 193 return -EINVAL; 194 } 195 } 196 197 ret = kstrtou32(val, 10, &n); 198 if (ret || !n || n > q_num) 199 return -EINVAL; 200 201 return param_set_int(val, kp); 202 } 203 204 static const struct kernel_param_ops sec_pf_q_num_ops = { 205 .set = sec_pf_q_num_set, 206 .get = param_get_int, 207 }; 208 static u32 pf_q_num = SEC_PF_DEF_Q_NUM; 209 module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444); 210 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)"); 211 212 static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp) 213 { 214 u32 ctx_q_num; 215 int ret; 216 217 if (!val) 218 return -EINVAL; 219 220 ret = kstrtou32(val, 10, &ctx_q_num); 221 if (ret) 222 return -EINVAL; 223 224 if (!ctx_q_num || ctx_q_num > QM_Q_DEPTH || ctx_q_num & 0x1) { 225 pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num); 226 return -EINVAL; 227 } 228 229 return param_set_int(val, kp); 230 } 231 232 static const struct kernel_param_ops sec_ctx_q_num_ops = { 233 .set = sec_ctx_q_num_set, 234 .get = param_get_int, 235 }; 236 static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF; 237 module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444); 238 MODULE_PARM_DESC(ctx_q_num, "Number of queue in ctx (2, 4, 6, ..., 1024)"); 239 240 static const struct pci_device_id sec_dev_ids[] = { 241 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_PF_PCI_DEVICE_ID) }, 242 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_VF_PCI_DEVICE_ID) }, 243 { 0, } 244 }; 245 MODULE_DEVICE_TABLE(pci, sec_dev_ids); 246 247 static inline void sec_add_to_list(struct sec_dev *sec) 248 { 249 mutex_lock(&sec_list_lock); 250 list_add_tail(&sec->list, &sec_list); 251 mutex_unlock(&sec_list_lock); 252 } 253 254 static inline void sec_remove_from_list(struct sec_dev *sec) 255 { 256 mutex_lock(&sec_list_lock); 257 list_del(&sec->list); 258 mutex_unlock(&sec_list_lock); 259 } 260 261 static u8 sec_get_endian(struct sec_dev *sec) 262 { 263 struct hisi_qm *qm = &sec->qm; 264 u32 reg; 265 266 /* 267 * As for VF, it is a wrong way to get endian setting by 268 * reading a register of the engine 269 */ 270 if (qm->pdev->is_virtfn) { 271 dev_err_ratelimited(&qm->pdev->dev, 272 "cannot access a register in VF!\n"); 273 return SEC_LE; 274 } 275 reg = readl_relaxed(qm->io_base + SEC_ENGINE_PF_CFG_OFF + 276 SEC_ACC_COMMON_REG_OFF + SEC_CONTROL_REG); 277 278 /* BD little endian mode */ 279 if (!(reg & BIT(0))) 280 return SEC_LE; 281 282 /* BD 32-bits big endian mode */ 283 else if (!(reg & BIT(1))) 284 return SEC_32BE; 285 286 /* BD 64-bits big endian mode */ 287 else 288 return SEC_64BE; 289 } 290 291 static int sec_engine_init(struct sec_dev *sec) 292 { 293 struct hisi_qm *qm = &sec->qm; 294 int ret; 295 u32 reg; 296 297 /* disable clock gate control */ 298 reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); 299 reg &= SEC_CLK_GATE_DISABLE; 300 writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); 301 302 writel_relaxed(0x1, SEC_ADDR(qm, SEC_MEM_START_INIT_REG)); 303 304 ret = readl_relaxed_poll_timeout(SEC_ADDR(qm, SEC_MEM_INIT_DONE_REG), 305 reg, reg & 0x1, SEC_DELAY_10_US, 306 SEC_POLL_TIMEOUT_US); 307 if (ret) { 308 dev_err(&qm->pdev->dev, "fail to init sec mem\n"); 309 return ret; 310 } 311 312 reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); 313 reg |= (0x1 << SEC_TRNG_EN_SHIFT); 314 writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); 315 316 reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); 317 reg |= SEC_USER0_SMMU_NORMAL; 318 writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); 319 320 reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); 321 reg |= SEC_USER1_SMMU_NORMAL; 322 writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); 323 324 writel_relaxed(SEC_BD_ERR_CHK_EN1, 325 SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG1)); 326 writel_relaxed(SEC_BD_ERR_CHK_EN2, 327 SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG2)); 328 329 /* enable clock gate control */ 330 reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); 331 reg |= SEC_CLK_GATE_ENABLE; 332 writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); 333 334 /* config endian */ 335 reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG)); 336 reg |= sec_get_endian(sec); 337 writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG)); 338 339 /* Enable sm4 xts mode multiple iv */ 340 writel_relaxed(SEC_XTS_MIV_ENABLE_MSK, 341 qm->io_base + SEC_XTS_MIV_ENABLE_REG); 342 343 return 0; 344 } 345 346 static int sec_set_user_domain_and_cache(struct sec_dev *sec) 347 { 348 struct hisi_qm *qm = &sec->qm; 349 350 /* qm user domain */ 351 writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1); 352 writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE); 353 writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1); 354 writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE); 355 writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE); 356 357 /* qm cache */ 358 writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG); 359 writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE); 360 361 /* disable FLR triggered by BME(bus master enable) */ 362 writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG); 363 writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); 364 365 /* enable sqc,cqc writeback */ 366 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | 367 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) | 368 FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL); 369 370 return sec_engine_init(sec); 371 } 372 373 /* sec_debug_regs_clear() - clear the sec debug regs */ 374 static void sec_debug_regs_clear(struct hisi_qm *qm) 375 { 376 /* clear current_qm */ 377 writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); 378 writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); 379 380 /* clear rdclr_en */ 381 writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE); 382 383 hisi_qm_debug_regs_clear(qm); 384 } 385 386 static void sec_hw_error_enable(struct sec_dev *sec) 387 { 388 struct hisi_qm *qm = &sec->qm; 389 u32 val; 390 391 if (qm->ver == QM_HW_V1) { 392 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); 393 dev_info(&qm->pdev->dev, "V1 not support hw error handle\n"); 394 return; 395 } 396 397 val = readl(qm->io_base + SEC_CONTROL_REG); 398 399 /* clear SEC hw error source if having */ 400 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_SOURCE); 401 402 /* enable SEC hw error interrupts */ 403 writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); 404 405 /* enable RAS int */ 406 writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG); 407 writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); 408 writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG); 409 410 /* enable SEC block master OOO when m-bit error occur */ 411 val = val | SEC_AXI_SHUTDOWN_ENABLE; 412 413 writel(val, qm->io_base + SEC_CONTROL_REG); 414 } 415 416 static void sec_hw_error_disable(struct sec_dev *sec) 417 { 418 struct hisi_qm *qm = &sec->qm; 419 u32 val; 420 421 val = readl(qm->io_base + SEC_CONTROL_REG); 422 423 /* disable RAS int */ 424 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); 425 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG); 426 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG); 427 428 /* disable SEC hw error interrupts */ 429 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); 430 431 /* disable SEC block master OOO when m-bit error occur */ 432 val = val & SEC_AXI_SHUTDOWN_DISABLE; 433 434 writel(val, qm->io_base + SEC_CONTROL_REG); 435 } 436 437 static void sec_hw_error_init(struct sec_dev *sec) 438 { 439 if (sec->qm.fun_type == QM_HW_VF) 440 return; 441 442 hisi_qm_hw_error_init(&sec->qm, QM_BASE_CE, 443 QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT 444 | QM_ACC_WB_NOT_READY_TIMEOUT, 0, 445 QM_DB_RANDOM_INVALID); 446 sec_hw_error_enable(sec); 447 } 448 449 static void sec_hw_error_uninit(struct sec_dev *sec) 450 { 451 if (sec->qm.fun_type == QM_HW_VF) 452 return; 453 454 sec_hw_error_disable(sec); 455 writel(GENMASK(12, 0), sec->qm.io_base + SEC_QM_ABNORMAL_INT_MASK); 456 } 457 458 static u32 sec_current_qm_read(struct sec_debug_file *file) 459 { 460 struct hisi_qm *qm = file->qm; 461 462 return readl(qm->io_base + QM_DFX_MB_CNT_VF); 463 } 464 465 static int sec_current_qm_write(struct sec_debug_file *file, u32 val) 466 { 467 struct hisi_qm *qm = file->qm; 468 struct sec_dev *sec = container_of(qm, struct sec_dev, qm); 469 u32 vfq_num; 470 u32 tmp; 471 472 if (val > sec->num_vfs) 473 return -EINVAL; 474 475 /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */ 476 if (!val) { 477 qm->debug.curr_qm_qp_num = qm->qp_num; 478 } else { 479 vfq_num = (qm->ctrl_qp_num - qm->qp_num) / sec->num_vfs; 480 481 if (val == sec->num_vfs) 482 qm->debug.curr_qm_qp_num = 483 qm->ctrl_qp_num - qm->qp_num - 484 (sec->num_vfs - 1) * vfq_num; 485 else 486 qm->debug.curr_qm_qp_num = vfq_num; 487 } 488 489 writel(val, qm->io_base + QM_DFX_MB_CNT_VF); 490 writel(val, qm->io_base + QM_DFX_DB_CNT_VF); 491 492 tmp = val | 493 (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK); 494 writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); 495 496 tmp = val | 497 (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK); 498 writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); 499 500 return 0; 501 } 502 503 static u32 sec_clear_enable_read(struct sec_debug_file *file) 504 { 505 struct hisi_qm *qm = file->qm; 506 507 return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & 508 SEC_CTRL_CNT_CLR_CE_BIT; 509 } 510 511 static int sec_clear_enable_write(struct sec_debug_file *file, u32 val) 512 { 513 struct hisi_qm *qm = file->qm; 514 u32 tmp; 515 516 if (val != 1 && val) 517 return -EINVAL; 518 519 tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) & 520 ~SEC_CTRL_CNT_CLR_CE_BIT) | val; 521 writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE); 522 523 return 0; 524 } 525 526 static ssize_t sec_debug_read(struct file *filp, char __user *buf, 527 size_t count, loff_t *pos) 528 { 529 struct sec_debug_file *file = filp->private_data; 530 char tbuf[SEC_DBGFS_VAL_MAX_LEN]; 531 u32 val; 532 int ret; 533 534 spin_lock_irq(&file->lock); 535 536 switch (file->index) { 537 case SEC_CURRENT_QM: 538 val = sec_current_qm_read(file); 539 break; 540 case SEC_CLEAR_ENABLE: 541 val = sec_clear_enable_read(file); 542 break; 543 default: 544 spin_unlock_irq(&file->lock); 545 return -EINVAL; 546 } 547 548 spin_unlock_irq(&file->lock); 549 ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val); 550 551 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 552 } 553 554 static ssize_t sec_debug_write(struct file *filp, const char __user *buf, 555 size_t count, loff_t *pos) 556 { 557 struct sec_debug_file *file = filp->private_data; 558 char tbuf[SEC_DBGFS_VAL_MAX_LEN]; 559 unsigned long val; 560 int len, ret; 561 562 if (*pos != 0) 563 return 0; 564 565 if (count >= SEC_DBGFS_VAL_MAX_LEN) 566 return -ENOSPC; 567 568 len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1, 569 pos, buf, count); 570 if (len < 0) 571 return len; 572 573 tbuf[len] = '\0'; 574 if (kstrtoul(tbuf, 0, &val)) 575 return -EFAULT; 576 577 spin_lock_irq(&file->lock); 578 579 switch (file->index) { 580 case SEC_CURRENT_QM: 581 ret = sec_current_qm_write(file, val); 582 if (ret) 583 goto err_input; 584 break; 585 case SEC_CLEAR_ENABLE: 586 ret = sec_clear_enable_write(file, val); 587 if (ret) 588 goto err_input; 589 break; 590 default: 591 ret = -EINVAL; 592 goto err_input; 593 } 594 595 spin_unlock_irq(&file->lock); 596 597 return count; 598 599 err_input: 600 spin_unlock_irq(&file->lock); 601 return ret; 602 } 603 604 static const struct file_operations sec_dbg_fops = { 605 .owner = THIS_MODULE, 606 .open = simple_open, 607 .read = sec_debug_read, 608 .write = sec_debug_write, 609 }; 610 611 static int debugfs_atomic64_t_get(void *data, u64 *val) 612 { 613 *val = atomic64_read((atomic64_t *)data); 614 return 0; 615 } 616 DEFINE_DEBUGFS_ATTRIBUTE(fops_atomic64_t_ro, debugfs_atomic64_t_get, NULL, 617 "%lld\n"); 618 619 static int sec_core_debug_init(struct sec_dev *sec) 620 { 621 struct hisi_qm *qm = &sec->qm; 622 struct device *dev = &qm->pdev->dev; 623 struct sec_dfx *dfx = &sec->debug.dfx; 624 struct debugfs_regset32 *regset; 625 struct dentry *tmp_d; 626 627 tmp_d = debugfs_create_dir("sec_dfx", sec->qm.debug.debug_root); 628 629 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); 630 if (!regset) 631 return -ENOENT; 632 633 regset->regs = sec_dfx_regs; 634 regset->nregs = ARRAY_SIZE(sec_dfx_regs); 635 regset->base = qm->io_base; 636 637 debugfs_create_regset32("regs", 0444, tmp_d, regset); 638 639 debugfs_create_file("send_cnt", 0444, tmp_d, &dfx->send_cnt, 640 &fops_atomic64_t_ro); 641 642 debugfs_create_file("recv_cnt", 0444, tmp_d, &dfx->recv_cnt, 643 &fops_atomic64_t_ro); 644 645 return 0; 646 } 647 648 static int sec_debug_init(struct sec_dev *sec) 649 { 650 int i; 651 652 for (i = SEC_CURRENT_QM; i < SEC_DEBUG_FILE_NUM; i++) { 653 spin_lock_init(&sec->debug.files[i].lock); 654 sec->debug.files[i].index = i; 655 sec->debug.files[i].qm = &sec->qm; 656 657 debugfs_create_file(sec_dbg_file_name[i], 0600, 658 sec->qm.debug.debug_root, 659 sec->debug.files + i, 660 &sec_dbg_fops); 661 } 662 663 return sec_core_debug_init(sec); 664 } 665 666 static int sec_debugfs_init(struct sec_dev *sec) 667 { 668 struct hisi_qm *qm = &sec->qm; 669 struct device *dev = &qm->pdev->dev; 670 int ret; 671 672 qm->debug.debug_root = debugfs_create_dir(dev_name(dev), 673 sec_debugfs_root); 674 ret = hisi_qm_debug_init(qm); 675 if (ret) 676 goto failed_to_create; 677 678 if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) { 679 ret = sec_debug_init(sec); 680 if (ret) 681 goto failed_to_create; 682 } 683 684 return 0; 685 686 failed_to_create: 687 debugfs_remove_recursive(sec_debugfs_root); 688 689 return ret; 690 } 691 692 static void sec_debugfs_exit(struct sec_dev *sec) 693 { 694 debugfs_remove_recursive(sec->qm.debug.debug_root); 695 } 696 697 static int sec_pf_probe_init(struct sec_dev *sec) 698 { 699 struct hisi_qm *qm = &sec->qm; 700 int ret; 701 702 switch (qm->ver) { 703 case QM_HW_V1: 704 qm->ctrl_qp_num = SEC_QUEUE_NUM_V1; 705 break; 706 707 case QM_HW_V2: 708 qm->ctrl_qp_num = SEC_QUEUE_NUM_V2; 709 break; 710 711 default: 712 return -EINVAL; 713 } 714 715 ret = sec_set_user_domain_and_cache(sec); 716 if (ret) 717 return ret; 718 719 sec_hw_error_init(sec); 720 sec_debug_regs_clear(qm); 721 722 return 0; 723 } 724 725 static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) 726 { 727 enum qm_hw_ver rev_id; 728 729 rev_id = hisi_qm_get_hw_version(pdev); 730 if (rev_id == QM_HW_UNKNOWN) 731 return -ENODEV; 732 733 qm->pdev = pdev; 734 qm->ver = rev_id; 735 736 qm->sqe_size = SEC_SQE_SIZE; 737 qm->dev_name = sec_name; 738 qm->fun_type = (pdev->device == SEC_PF_PCI_DEVICE_ID) ? 739 QM_HW_PF : QM_HW_VF; 740 qm->use_dma_api = true; 741 742 return hisi_qm_init(qm); 743 } 744 745 static void sec_qm_uninit(struct hisi_qm *qm) 746 { 747 hisi_qm_uninit(qm); 748 } 749 750 static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec) 751 { 752 if (qm->fun_type == QM_HW_PF) { 753 qm->qp_base = SEC_PF_DEF_Q_BASE; 754 qm->qp_num = pf_q_num; 755 qm->debug.curr_qm_qp_num = pf_q_num; 756 757 return sec_pf_probe_init(sec); 758 } else if (qm->fun_type == QM_HW_VF) { 759 /* 760 * have no way to get qm configure in VM in v1 hardware, 761 * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force 762 * to trigger only one VF in v1 hardware. 763 * v2 hardware has no such problem. 764 */ 765 if (qm->ver == QM_HW_V1) { 766 qm->qp_base = SEC_PF_DEF_Q_NUM; 767 qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; 768 } else if (qm->ver == QM_HW_V2) { 769 /* v2 starts to support get vft by mailbox */ 770 return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); 771 } 772 } else { 773 return -ENODEV; 774 } 775 776 return 0; 777 } 778 779 static void sec_probe_uninit(struct sec_dev *sec) 780 { 781 sec_hw_error_uninit(sec); 782 } 783 784 static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) 785 { 786 struct sec_dev *sec; 787 struct hisi_qm *qm; 788 int ret; 789 790 sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL); 791 if (!sec) 792 return -ENOMEM; 793 794 pci_set_drvdata(pdev, sec); 795 796 sec->ctx_q_num = ctx_q_num; 797 798 qm = &sec->qm; 799 800 ret = sec_qm_init(qm, pdev); 801 if (ret) { 802 pci_err(pdev, "Failed to pre init qm!\n"); 803 return ret; 804 } 805 806 ret = sec_probe_init(qm, sec); 807 if (ret) { 808 pci_err(pdev, "Failed to probe!\n"); 809 goto err_qm_uninit; 810 } 811 812 ret = hisi_qm_start(qm); 813 if (ret) { 814 pci_err(pdev, "Failed to start sec qm!\n"); 815 goto err_probe_uninit; 816 } 817 818 ret = sec_debugfs_init(sec); 819 if (ret) 820 pci_warn(pdev, "Failed to init debugfs!\n"); 821 822 sec_add_to_list(sec); 823 824 ret = sec_register_to_crypto(); 825 if (ret < 0) { 826 pr_err("Failed to register driver to crypto.\n"); 827 goto err_remove_from_list; 828 } 829 830 return 0; 831 832 err_remove_from_list: 833 sec_remove_from_list(sec); 834 sec_debugfs_exit(sec); 835 hisi_qm_stop(qm); 836 837 err_probe_uninit: 838 sec_probe_uninit(sec); 839 840 err_qm_uninit: 841 sec_qm_uninit(qm); 842 843 return ret; 844 } 845 846 /* now we only support equal assignment */ 847 static int sec_vf_q_assign(struct sec_dev *sec, u32 num_vfs) 848 { 849 struct hisi_qm *qm = &sec->qm; 850 u32 qp_num = qm->qp_num; 851 u32 q_base = qp_num; 852 u32 q_num, remain_q_num; 853 int i, j, ret; 854 855 if (!num_vfs) 856 return -EINVAL; 857 858 remain_q_num = qm->ctrl_qp_num - qp_num; 859 q_num = remain_q_num / num_vfs; 860 861 for (i = 1; i <= num_vfs; i++) { 862 if (i == num_vfs) 863 q_num += remain_q_num % num_vfs; 864 ret = hisi_qm_set_vft(qm, i, q_base, q_num); 865 if (ret) { 866 for (j = i; j > 0; j--) 867 hisi_qm_set_vft(qm, j, 0, 0); 868 return ret; 869 } 870 q_base += q_num; 871 } 872 873 return 0; 874 } 875 876 static int sec_clear_vft_config(struct sec_dev *sec) 877 { 878 struct hisi_qm *qm = &sec->qm; 879 u32 num_vfs = sec->num_vfs; 880 int ret; 881 u32 i; 882 883 for (i = 1; i <= num_vfs; i++) { 884 ret = hisi_qm_set_vft(qm, i, 0, 0); 885 if (ret) 886 return ret; 887 } 888 889 sec->num_vfs = 0; 890 891 return 0; 892 } 893 894 static int sec_sriov_enable(struct pci_dev *pdev, int max_vfs) 895 { 896 struct sec_dev *sec = pci_get_drvdata(pdev); 897 int pre_existing_vfs, ret; 898 u32 num_vfs; 899 900 pre_existing_vfs = pci_num_vf(pdev); 901 902 if (pre_existing_vfs) { 903 pci_err(pdev, "Can't enable VF. Please disable at first!\n"); 904 return 0; 905 } 906 907 num_vfs = min_t(u32, max_vfs, SEC_VF_NUM); 908 909 ret = sec_vf_q_assign(sec, num_vfs); 910 if (ret) { 911 pci_err(pdev, "Can't assign queues for VF!\n"); 912 return ret; 913 } 914 915 sec->num_vfs = num_vfs; 916 917 ret = pci_enable_sriov(pdev, num_vfs); 918 if (ret) { 919 pci_err(pdev, "Can't enable VF!\n"); 920 sec_clear_vft_config(sec); 921 return ret; 922 } 923 924 return num_vfs; 925 } 926 927 static int sec_sriov_disable(struct pci_dev *pdev) 928 { 929 struct sec_dev *sec = pci_get_drvdata(pdev); 930 931 if (pci_vfs_assigned(pdev)) { 932 pci_err(pdev, "Can't disable VFs while VFs are assigned!\n"); 933 return -EPERM; 934 } 935 936 /* remove in sec_pci_driver will be called to free VF resources */ 937 pci_disable_sriov(pdev); 938 939 return sec_clear_vft_config(sec); 940 } 941 942 static int sec_sriov_configure(struct pci_dev *pdev, int num_vfs) 943 { 944 if (num_vfs) 945 return sec_sriov_enable(pdev, num_vfs); 946 else 947 return sec_sriov_disable(pdev); 948 } 949 950 static void sec_remove(struct pci_dev *pdev) 951 { 952 struct sec_dev *sec = pci_get_drvdata(pdev); 953 struct hisi_qm *qm = &sec->qm; 954 955 sec_unregister_from_crypto(); 956 957 sec_remove_from_list(sec); 958 959 if (qm->fun_type == QM_HW_PF && sec->num_vfs) 960 (void)sec_sriov_disable(pdev); 961 962 sec_debugfs_exit(sec); 963 964 (void)hisi_qm_stop(qm); 965 966 if (qm->fun_type == QM_HW_PF) 967 sec_debug_regs_clear(qm); 968 969 sec_probe_uninit(sec); 970 971 sec_qm_uninit(qm); 972 } 973 974 static void sec_log_hw_error(struct sec_dev *sec, u32 err_sts) 975 { 976 const struct sec_hw_error *errs = sec_hw_errors; 977 struct device *dev = &sec->qm.pdev->dev; 978 u32 err_val; 979 980 while (errs->msg) { 981 if (errs->int_msk & err_sts) { 982 dev_err(dev, "%s [error status=0x%x] found\n", 983 errs->msg, errs->int_msk); 984 985 if (SEC_CORE_INT_STATUS_M_ECC & err_sts) { 986 err_val = readl(sec->qm.io_base + 987 SEC_CORE_SRAM_ECC_ERR_INFO); 988 dev_err(dev, "multi ecc sram num=0x%x\n", 989 SEC_ECC_NUM(err_val)); 990 dev_err(dev, "multi ecc sram addr=0x%x\n", 991 SEC_ECC_ADDR(err_val)); 992 } 993 } 994 errs++; 995 } 996 } 997 998 static pci_ers_result_t sec_hw_error_handle(struct sec_dev *sec) 999 { 1000 u32 err_sts; 1001 1002 /* read err sts */ 1003 err_sts = readl(sec->qm.io_base + SEC_CORE_INT_STATUS); 1004 if (err_sts) { 1005 sec_log_hw_error(sec, err_sts); 1006 1007 /* clear error interrupts */ 1008 writel(err_sts, sec->qm.io_base + SEC_CORE_INT_SOURCE); 1009 1010 return PCI_ERS_RESULT_NEED_RESET; 1011 } 1012 1013 return PCI_ERS_RESULT_RECOVERED; 1014 } 1015 1016 static pci_ers_result_t sec_process_hw_error(struct pci_dev *pdev) 1017 { 1018 struct sec_dev *sec = pci_get_drvdata(pdev); 1019 pci_ers_result_t qm_ret, sec_ret; 1020 1021 if (!sec) { 1022 pci_err(pdev, "Can't recover error during device init\n"); 1023 return PCI_ERS_RESULT_NONE; 1024 } 1025 1026 /* log qm error */ 1027 qm_ret = hisi_qm_hw_error_handle(&sec->qm); 1028 1029 /* log sec error */ 1030 sec_ret = sec_hw_error_handle(sec); 1031 1032 return (qm_ret == PCI_ERS_RESULT_NEED_RESET || 1033 sec_ret == PCI_ERS_RESULT_NEED_RESET) ? 1034 PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED; 1035 } 1036 1037 static pci_ers_result_t sec_error_detected(struct pci_dev *pdev, 1038 pci_channel_state_t state) 1039 { 1040 if (pdev->is_virtfn) 1041 return PCI_ERS_RESULT_NONE; 1042 1043 pci_info(pdev, "PCI error detected, state(=%d)!!\n", state); 1044 if (state == pci_channel_io_perm_failure) 1045 return PCI_ERS_RESULT_DISCONNECT; 1046 1047 return sec_process_hw_error(pdev); 1048 } 1049 1050 static const struct pci_error_handlers sec_err_handler = { 1051 .error_detected = sec_error_detected, 1052 }; 1053 1054 static struct pci_driver sec_pci_driver = { 1055 .name = "hisi_sec2", 1056 .id_table = sec_dev_ids, 1057 .probe = sec_probe, 1058 .remove = sec_remove, 1059 .err_handler = &sec_err_handler, 1060 .sriov_configure = sec_sriov_configure, 1061 }; 1062 1063 static void sec_register_debugfs(void) 1064 { 1065 if (!debugfs_initialized()) 1066 return; 1067 1068 sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL); 1069 } 1070 1071 static void sec_unregister_debugfs(void) 1072 { 1073 debugfs_remove_recursive(sec_debugfs_root); 1074 } 1075 1076 static int __init sec_init(void) 1077 { 1078 int ret; 1079 1080 sec_register_debugfs(); 1081 1082 ret = pci_register_driver(&sec_pci_driver); 1083 if (ret < 0) { 1084 sec_unregister_debugfs(); 1085 pr_err("Failed to register pci driver.\n"); 1086 return ret; 1087 } 1088 1089 return 0; 1090 } 1091 1092 static void __exit sec_exit(void) 1093 { 1094 pci_unregister_driver(&sec_pci_driver); 1095 sec_unregister_debugfs(); 1096 } 1097 1098 module_init(sec_init); 1099 module_exit(sec_exit); 1100 1101 MODULE_LICENSE("GPL v2"); 1102 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>"); 1103 MODULE_AUTHOR("Longfang Liu <liulongfang@huawei.com>"); 1104 MODULE_AUTHOR("Wei Zhang <zhangwei375@huawei.com>"); 1105 MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator"); 1106