1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <asm/page.h> 4 #include <linux/bitmap.h> 5 #include <linux/debugfs.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/io.h> 8 #include <linux/irqreturn.h> 9 #include <linux/log2.h> 10 #include <linux/seq_file.h> 11 #include <linux/slab.h> 12 #include "qm.h" 13 14 /* eq/aeq irq enable */ 15 #define QM_VF_AEQ_INT_SOURCE 0x0 16 #define QM_VF_AEQ_INT_MASK 0x4 17 #define QM_VF_EQ_INT_SOURCE 0x8 18 #define QM_VF_EQ_INT_MASK 0xc 19 #define QM_IRQ_NUM_V1 1 20 #define QM_IRQ_NUM_PF_V2 4 21 #define QM_IRQ_NUM_VF_V2 2 22 23 #define QM_EQ_EVENT_IRQ_VECTOR 0 24 #define QM_AEQ_EVENT_IRQ_VECTOR 1 25 #define QM_ABNORMAL_EVENT_IRQ_VECTOR 3 26 27 /* mailbox */ 28 #define QM_MB_CMD_SQC 0x0 29 #define QM_MB_CMD_CQC 0x1 30 #define QM_MB_CMD_EQC 0x2 31 #define QM_MB_CMD_AEQC 0x3 32 #define QM_MB_CMD_SQC_BT 0x4 33 #define QM_MB_CMD_CQC_BT 0x5 34 #define QM_MB_CMD_SQC_VFT_V2 0x6 35 36 #define QM_MB_CMD_SEND_BASE 0x300 37 #define QM_MB_EVENT_SHIFT 8 38 #define QM_MB_BUSY_SHIFT 13 39 #define QM_MB_OP_SHIFT 14 40 #define QM_MB_CMD_DATA_ADDR_L 0x304 41 #define QM_MB_CMD_DATA_ADDR_H 0x308 42 43 /* sqc shift */ 44 #define QM_SQ_HOP_NUM_SHIFT 0 45 #define QM_SQ_PAGE_SIZE_SHIFT 4 46 #define QM_SQ_BUF_SIZE_SHIFT 8 47 #define QM_SQ_SQE_SIZE_SHIFT 12 48 #define QM_SQ_PRIORITY_SHIFT 0 49 #define QM_SQ_ORDERS_SHIFT 4 50 #define QM_SQ_TYPE_SHIFT 8 51 52 #define QM_SQ_TYPE_MASK GENMASK(3, 0) 53 54 /* cqc shift */ 55 #define QM_CQ_HOP_NUM_SHIFT 0 56 #define QM_CQ_PAGE_SIZE_SHIFT 4 57 #define QM_CQ_BUF_SIZE_SHIFT 8 58 #define QM_CQ_CQE_SIZE_SHIFT 12 59 #define QM_CQ_PHASE_SHIFT 0 60 #define QM_CQ_FLAG_SHIFT 1 61 62 #define QM_CQE_PHASE(cqe) ((cqe)->w7 & 0x1) 63 #define QM_QC_CQE_SIZE 4 64 65 /* eqc shift */ 66 #define QM_EQE_AEQE_SIZE (2UL << 12) 67 #define QM_EQC_PHASE_SHIFT 16 68 69 #define QM_EQE_PHASE(eqe) (((eqe)->dw0 >> 16) & 0x1) 70 #define QM_EQE_CQN_MASK GENMASK(15, 0) 71 72 #define QM_AEQE_PHASE(aeqe) (((aeqe)->dw0 >> 16) & 0x1) 73 #define QM_AEQE_TYPE_SHIFT 17 74 75 #define QM_DOORBELL_CMD_SQ 0 76 #define QM_DOORBELL_CMD_CQ 1 77 #define QM_DOORBELL_CMD_EQ 2 78 #define QM_DOORBELL_CMD_AEQ 3 79 80 #define QM_DOORBELL_BASE_V1 0x340 81 #define QM_DB_CMD_SHIFT_V1 16 82 #define QM_DB_INDEX_SHIFT_V1 32 83 #define QM_DB_PRIORITY_SHIFT_V1 48 84 #define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000 85 #define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000 86 #define QM_DB_CMD_SHIFT_V2 12 87 #define QM_DB_RAND_SHIFT_V2 16 88 #define QM_DB_INDEX_SHIFT_V2 32 89 #define QM_DB_PRIORITY_SHIFT_V2 48 90 91 #define QM_MEM_START_INIT 0x100040 92 #define QM_MEM_INIT_DONE 0x100044 93 #define QM_VFT_CFG_RDY 0x10006c 94 #define QM_VFT_CFG_OP_WR 0x100058 95 #define QM_VFT_CFG_TYPE 0x10005c 96 #define QM_SQC_VFT 0x0 97 #define QM_CQC_VFT 0x1 98 #define QM_VFT_CFG 0x100060 99 #define QM_VFT_CFG_OP_ENABLE 0x100054 100 101 #define QM_VFT_CFG_DATA_L 0x100064 102 #define QM_VFT_CFG_DATA_H 0x100068 103 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8) 104 #define QM_SQC_VFT_SQC_SIZE (5ULL << 12) 105 #define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16) 106 #define QM_SQC_VFT_START_SQN_SHIFT 28 107 #define QM_SQC_VFT_VALID (1ULL << 44) 108 #define QM_SQC_VFT_SQN_SHIFT 45 109 #define QM_CQC_VFT_BUF_SIZE (7ULL << 8) 110 #define QM_CQC_VFT_SQC_SIZE (5ULL << 12) 111 #define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16) 112 #define QM_CQC_VFT_VALID (1ULL << 28) 113 114 #define QM_SQC_VFT_BASE_SHIFT_V2 28 115 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(5, 0) 116 #define QM_SQC_VFT_NUM_SHIFT_V2 45 117 #define QM_SQC_VFT_NUM_MASK_v2 GENMASK(9, 0) 118 119 #define QM_DFX_CNT_CLR_CE 0x100118 120 121 #define QM_ABNORMAL_INT_SOURCE 0x100000 122 #define QM_ABNORMAL_INT_MASK 0x100004 123 #define QM_ABNORMAL_INT_MASK_VALUE 0x1fff 124 #define QM_ABNORMAL_INT_STATUS 0x100008 125 #define QM_ABNORMAL_INF00 0x100010 126 #define QM_FIFO_OVERFLOW_TYPE 0xc0 127 #define QM_FIFO_OVERFLOW_TYPE_SHIFT 6 128 #define QM_FIFO_OVERFLOW_VF 0x3f 129 #define QM_ABNORMAL_INF01 0x100014 130 #define QM_DB_TIMEOUT_TYPE 0xc0 131 #define QM_DB_TIMEOUT_TYPE_SHIFT 6 132 #define QM_DB_TIMEOUT_VF 0x3f 133 #define QM_RAS_CE_ENABLE 0x1000ec 134 #define QM_RAS_FE_ENABLE 0x1000f0 135 #define QM_RAS_NFE_ENABLE 0x1000f4 136 #define QM_RAS_CE_THRESHOLD 0x1000f8 137 #define QM_RAS_CE_TIMES_PER_IRQ 1 138 #define QM_RAS_MSI_INT_SEL 0x1040f4 139 140 #define QM_CACHE_WB_START 0x204 141 #define QM_CACHE_WB_DONE 0x208 142 143 #define PCI_BAR_2 2 144 #define QM_SQE_DATA_ALIGN_MASK GENMASK(6, 0) 145 #define QMC_ALIGN(sz) ALIGN(sz, 32) 146 147 #define QM_DBG_TMP_BUF_LEN 22 148 149 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ 150 (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ 151 ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ 152 ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \ 153 ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) 154 155 #define QM_MK_CQC_DW3_V2(cqe_sz) \ 156 ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) 157 158 #define QM_MK_SQC_W13(priority, orders, alg_type) \ 159 (((priority) << QM_SQ_PRIORITY_SHIFT) | \ 160 ((orders) << QM_SQ_ORDERS_SHIFT) | \ 161 (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT)) 162 163 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \ 164 (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \ 165 ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \ 166 ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \ 167 ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) 168 169 #define QM_MK_SQC_DW3_V2(sqe_sz) \ 170 ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) 171 172 #define INIT_QC_COMMON(qc, base, pasid) do { \ 173 (qc)->head = 0; \ 174 (qc)->tail = 0; \ 175 (qc)->base_l = lower_32_bits(base); \ 176 (qc)->base_h = upper_32_bits(base); \ 177 (qc)->dw3 = 0; \ 178 (qc)->w8 = 0; \ 179 (qc)->rsvd0 = 0; \ 180 (qc)->pasid = pasid; \ 181 (qc)->w11 = 0; \ 182 (qc)->rsvd1 = 0; \ 183 } while (0) 184 185 enum vft_type { 186 SQC_VFT = 0, 187 CQC_VFT, 188 }; 189 190 struct qm_cqe { 191 __le32 rsvd0; 192 __le16 cmd_id; 193 __le16 rsvd1; 194 __le16 sq_head; 195 __le16 sq_num; 196 __le16 rsvd2; 197 __le16 w7; 198 }; 199 200 struct qm_eqe { 201 __le32 dw0; 202 }; 203 204 struct qm_aeqe { 205 __le32 dw0; 206 }; 207 208 struct qm_sqc { 209 __le16 head; 210 __le16 tail; 211 __le32 base_l; 212 __le32 base_h; 213 __le32 dw3; 214 __le16 w8; 215 __le16 rsvd0; 216 __le16 pasid; 217 __le16 w11; 218 __le16 cq_num; 219 __le16 w13; 220 __le32 rsvd1; 221 }; 222 223 struct qm_cqc { 224 __le16 head; 225 __le16 tail; 226 __le32 base_l; 227 __le32 base_h; 228 __le32 dw3; 229 __le16 w8; 230 __le16 rsvd0; 231 __le16 pasid; 232 __le16 w11; 233 __le32 dw6; 234 __le32 rsvd1; 235 }; 236 237 struct qm_eqc { 238 __le16 head; 239 __le16 tail; 240 __le32 base_l; 241 __le32 base_h; 242 __le32 dw3; 243 __le32 rsvd[2]; 244 __le32 dw6; 245 }; 246 247 struct qm_aeqc { 248 __le16 head; 249 __le16 tail; 250 __le32 base_l; 251 __le32 base_h; 252 __le32 dw3; 253 __le32 rsvd[2]; 254 __le32 dw6; 255 }; 256 257 struct qm_mailbox { 258 __le16 w0; 259 __le16 queue_num; 260 __le32 base_l; 261 __le32 base_h; 262 __le32 rsvd; 263 }; 264 265 struct qm_doorbell { 266 __le16 queue_num; 267 __le16 cmd; 268 __le16 index; 269 __le16 priority; 270 }; 271 272 struct hisi_qm_hw_ops { 273 int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number); 274 void (*qm_db)(struct hisi_qm *qm, u16 qn, 275 u8 cmd, u16 index, u8 priority); 276 u32 (*get_irq_num)(struct hisi_qm *qm); 277 int (*debug_init)(struct hisi_qm *qm); 278 void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, 279 u32 msi); 280 pci_ers_result_t (*hw_error_handle)(struct hisi_qm *qm); 281 }; 282 283 static const char * const qm_debug_file_name[] = { 284 [CURRENT_Q] = "current_q", 285 [CLEAR_ENABLE] = "clear_enable", 286 }; 287 288 struct hisi_qm_hw_error { 289 u32 int_msk; 290 const char *msg; 291 }; 292 293 static const struct hisi_qm_hw_error qm_hw_error[] = { 294 { .int_msk = BIT(0), .msg = "qm_axi_rresp" }, 295 { .int_msk = BIT(1), .msg = "qm_axi_bresp" }, 296 { .int_msk = BIT(2), .msg = "qm_ecc_mbit" }, 297 { .int_msk = BIT(3), .msg = "qm_ecc_1bit" }, 298 { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" }, 299 { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" }, 300 { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" }, 301 { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" }, 302 { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" }, 303 { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" }, 304 { .int_msk = BIT(10), .msg = "qm_db_timeout" }, 305 { .int_msk = BIT(11), .msg = "qm_of_fifo_of" }, 306 { .int_msk = BIT(12), .msg = "qm_db_random_invalid" }, 307 { /* sentinel */ } 308 }; 309 310 static const char * const qm_db_timeout[] = { 311 "sq", "cq", "eq", "aeq", 312 }; 313 314 static const char * const qm_fifo_overflow[] = { 315 "cq", "eq", "aeq", 316 }; 317 318 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */ 319 static int qm_wait_mb_ready(struct hisi_qm *qm) 320 { 321 u32 val; 322 323 return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE, 324 val, !((val >> QM_MB_BUSY_SHIFT) & 325 0x1), 10, 1000); 326 } 327 328 /* 128 bit should be written to hardware at one time to trigger a mailbox */ 329 static void qm_mb_write(struct hisi_qm *qm, const void *src) 330 { 331 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; 332 unsigned long tmp0 = 0, tmp1 = 0; 333 334 asm volatile("ldp %0, %1, %3\n" 335 "stp %0, %1, %2\n" 336 "dsb sy\n" 337 : "=&r" (tmp0), 338 "=&r" (tmp1), 339 "+Q" (*((char *)fun_base)) 340 : "Q" (*((char *)src)) 341 : "memory"); 342 } 343 344 static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, 345 bool op) 346 { 347 struct qm_mailbox mailbox; 348 int ret = 0; 349 350 dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n", 351 queue, cmd, (unsigned long long)dma_addr); 352 353 mailbox.w0 = cmd | 354 (op ? 0x1 << QM_MB_OP_SHIFT : 0) | 355 (0x1 << QM_MB_BUSY_SHIFT); 356 mailbox.queue_num = queue; 357 mailbox.base_l = lower_32_bits(dma_addr); 358 mailbox.base_h = upper_32_bits(dma_addr); 359 mailbox.rsvd = 0; 360 361 mutex_lock(&qm->mailbox_lock); 362 363 if (unlikely(qm_wait_mb_ready(qm))) { 364 ret = -EBUSY; 365 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); 366 goto busy_unlock; 367 } 368 369 qm_mb_write(qm, &mailbox); 370 371 if (unlikely(qm_wait_mb_ready(qm))) { 372 ret = -EBUSY; 373 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); 374 goto busy_unlock; 375 } 376 377 busy_unlock: 378 mutex_unlock(&qm->mailbox_lock); 379 380 return ret; 381 } 382 383 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 384 { 385 u64 doorbell; 386 387 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) | 388 ((u64)index << QM_DB_INDEX_SHIFT_V1) | 389 ((u64)priority << QM_DB_PRIORITY_SHIFT_V1); 390 391 writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1); 392 } 393 394 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 395 { 396 u64 doorbell; 397 u64 dbase; 398 u16 randata = 0; 399 400 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) 401 dbase = QM_DOORBELL_SQ_CQ_BASE_V2; 402 else 403 dbase = QM_DOORBELL_EQ_AEQ_BASE_V2; 404 405 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | 406 ((u64)randata << QM_DB_RAND_SHIFT_V2) | 407 ((u64)index << QM_DB_INDEX_SHIFT_V2) | 408 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); 409 410 writeq(doorbell, qm->io_base + dbase); 411 } 412 413 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) 414 { 415 dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n", 416 qn, cmd, index); 417 418 qm->ops->qm_db(qm, qn, cmd, index, priority); 419 } 420 421 static int qm_dev_mem_reset(struct hisi_qm *qm) 422 { 423 u32 val; 424 425 writel(0x1, qm->io_base + QM_MEM_START_INIT); 426 return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, 427 val & BIT(0), 10, 1000); 428 } 429 430 static u32 qm_get_irq_num_v1(struct hisi_qm *qm) 431 { 432 return QM_IRQ_NUM_V1; 433 } 434 435 static u32 qm_get_irq_num_v2(struct hisi_qm *qm) 436 { 437 if (qm->fun_type == QM_HW_PF) 438 return QM_IRQ_NUM_PF_V2; 439 else 440 return QM_IRQ_NUM_VF_V2; 441 } 442 443 static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe) 444 { 445 u16 cqn = eqe->dw0 & QM_EQE_CQN_MASK; 446 447 return qm->qp_array[cqn]; 448 } 449 450 static void qm_cq_head_update(struct hisi_qp *qp) 451 { 452 if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) { 453 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase; 454 qp->qp_status.cq_head = 0; 455 } else { 456 qp->qp_status.cq_head++; 457 } 458 } 459 460 static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) 461 { 462 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; 463 464 if (qp->req_cb) { 465 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { 466 dma_rmb(); 467 qp->req_cb(qp, qp->sqe + qm->sqe_size * cqe->sq_head); 468 qm_cq_head_update(qp); 469 cqe = qp->cqe + qp->qp_status.cq_head; 470 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, 471 qp->qp_status.cq_head, 0); 472 atomic_dec(&qp->qp_status.used); 473 } 474 475 /* set c_flag */ 476 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, 477 qp->qp_status.cq_head, 1); 478 } 479 } 480 481 static void qm_qp_work_func(struct work_struct *work) 482 { 483 struct hisi_qp *qp; 484 485 qp = container_of(work, struct hisi_qp, work); 486 qm_poll_qp(qp, qp->qm); 487 } 488 489 static irqreturn_t qm_irq_handler(int irq, void *data) 490 { 491 struct hisi_qm *qm = data; 492 struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; 493 struct hisi_qp *qp; 494 int eqe_num = 0; 495 496 while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { 497 eqe_num++; 498 qp = qm_to_hisi_qp(qm, eqe); 499 if (qp) 500 queue_work(qp->wq, &qp->work); 501 502 if (qm->status.eq_head == QM_Q_DEPTH - 1) { 503 qm->status.eqc_phase = !qm->status.eqc_phase; 504 eqe = qm->eqe; 505 qm->status.eq_head = 0; 506 } else { 507 eqe++; 508 qm->status.eq_head++; 509 } 510 511 if (eqe_num == QM_Q_DEPTH / 2 - 1) { 512 eqe_num = 0; 513 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 514 } 515 } 516 517 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 518 519 return IRQ_HANDLED; 520 } 521 522 static irqreturn_t qm_irq(int irq, void *data) 523 { 524 struct hisi_qm *qm = data; 525 526 if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE)) 527 return qm_irq_handler(irq, data); 528 529 dev_err(&qm->pdev->dev, "invalid int source\n"); 530 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); 531 532 return IRQ_NONE; 533 } 534 535 static irqreturn_t qm_aeq_irq(int irq, void *data) 536 { 537 struct hisi_qm *qm = data; 538 struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; 539 u32 type; 540 541 if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE)) 542 return IRQ_NONE; 543 544 while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { 545 type = aeqe->dw0 >> QM_AEQE_TYPE_SHIFT; 546 if (type < ARRAY_SIZE(qm_fifo_overflow)) 547 dev_err(&qm->pdev->dev, "%s overflow\n", 548 qm_fifo_overflow[type]); 549 else 550 dev_err(&qm->pdev->dev, "unknown error type %d\n", 551 type); 552 553 if (qm->status.aeq_head == QM_Q_DEPTH - 1) { 554 qm->status.aeqc_phase = !qm->status.aeqc_phase; 555 aeqe = qm->aeqe; 556 qm->status.aeq_head = 0; 557 } else { 558 aeqe++; 559 qm->status.aeq_head++; 560 } 561 562 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); 563 } 564 565 return IRQ_HANDLED; 566 } 567 568 static irqreturn_t qm_abnormal_irq(int irq, void *data) 569 { 570 const struct hisi_qm_hw_error *err = qm_hw_error; 571 struct hisi_qm *qm = data; 572 struct device *dev = &qm->pdev->dev; 573 u32 error_status, tmp; 574 575 /* read err sts */ 576 tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS); 577 error_status = qm->msi_mask & tmp; 578 579 while (err->msg) { 580 if (err->int_msk & error_status) 581 dev_err(dev, "%s [error status=0x%x] found\n", 582 err->msg, err->int_msk); 583 584 err++; 585 } 586 587 /* clear err sts */ 588 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); 589 590 return IRQ_HANDLED; 591 } 592 593 static int qm_irq_register(struct hisi_qm *qm) 594 { 595 struct pci_dev *pdev = qm->pdev; 596 int ret; 597 598 ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), 599 qm_irq, IRQF_SHARED, qm->dev_name, qm); 600 if (ret) 601 return ret; 602 603 if (qm->ver == QM_HW_V2) { 604 ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), 605 qm_aeq_irq, IRQF_SHARED, qm->dev_name, qm); 606 if (ret) 607 goto err_aeq_irq; 608 609 if (qm->fun_type == QM_HW_PF) { 610 ret = request_irq(pci_irq_vector(pdev, 611 QM_ABNORMAL_EVENT_IRQ_VECTOR), 612 qm_abnormal_irq, IRQF_SHARED, 613 qm->dev_name, qm); 614 if (ret) 615 goto err_abonormal_irq; 616 } 617 } 618 619 return 0; 620 621 err_abonormal_irq: 622 free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); 623 err_aeq_irq: 624 free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); 625 return ret; 626 } 627 628 static void qm_irq_unregister(struct hisi_qm *qm) 629 { 630 struct pci_dev *pdev = qm->pdev; 631 632 free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); 633 634 if (qm->ver == QM_HW_V2) { 635 free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); 636 637 if (qm->fun_type == QM_HW_PF) 638 free_irq(pci_irq_vector(pdev, 639 QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); 640 } 641 } 642 643 static void qm_init_qp_status(struct hisi_qp *qp) 644 { 645 struct hisi_qp_status *qp_status = &qp->qp_status; 646 647 qp_status->sq_tail = 0; 648 qp_status->cq_head = 0; 649 qp_status->cqc_phase = 1; 650 qp_status->flags = 0; 651 } 652 653 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, 654 u32 number) 655 { 656 u64 tmp = 0; 657 658 if (number > 0) { 659 switch (type) { 660 case SQC_VFT: 661 switch (qm->ver) { 662 case QM_HW_V1: 663 tmp = QM_SQC_VFT_BUF_SIZE | 664 QM_SQC_VFT_SQC_SIZE | 665 QM_SQC_VFT_INDEX_NUMBER | 666 QM_SQC_VFT_VALID | 667 (u64)base << QM_SQC_VFT_START_SQN_SHIFT; 668 break; 669 case QM_HW_V2: 670 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT | 671 QM_SQC_VFT_VALID | 672 (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT; 673 break; 674 case QM_HW_UNKNOWN: 675 break; 676 } 677 break; 678 case CQC_VFT: 679 switch (qm->ver) { 680 case QM_HW_V1: 681 tmp = QM_CQC_VFT_BUF_SIZE | 682 QM_CQC_VFT_SQC_SIZE | 683 QM_CQC_VFT_INDEX_NUMBER | 684 QM_CQC_VFT_VALID; 685 break; 686 case QM_HW_V2: 687 tmp = QM_CQC_VFT_VALID; 688 break; 689 case QM_HW_UNKNOWN: 690 break; 691 } 692 break; 693 } 694 } 695 696 writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L); 697 writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H); 698 } 699 700 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, 701 u32 fun_num, u32 base, u32 number) 702 { 703 unsigned int val; 704 int ret; 705 706 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 707 val & BIT(0), 10, 1000); 708 if (ret) 709 return ret; 710 711 writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR); 712 writel(type, qm->io_base + QM_VFT_CFG_TYPE); 713 writel(fun_num, qm->io_base + QM_VFT_CFG); 714 715 qm_vft_data_cfg(qm, type, base, number); 716 717 writel(0x0, qm->io_base + QM_VFT_CFG_RDY); 718 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); 719 720 return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 721 val & BIT(0), 10, 1000); 722 } 723 724 /* The config should be conducted after qm_dev_mem_reset() */ 725 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, 726 u32 number) 727 { 728 int ret, i; 729 730 for (i = SQC_VFT; i <= CQC_VFT; i++) { 731 ret = qm_set_vft_common(qm, i, fun_num, base, number); 732 if (ret) 733 return ret; 734 } 735 736 return 0; 737 } 738 739 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) 740 { 741 u64 sqc_vft; 742 int ret; 743 744 ret = qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1); 745 if (ret) 746 return ret; 747 748 sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 749 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); 750 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); 751 *number = (QM_SQC_VFT_NUM_MASK_v2 & 752 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; 753 754 return 0; 755 } 756 757 static struct hisi_qm *file_to_qm(struct debugfs_file *file) 758 { 759 struct qm_debug *debug = file->debug; 760 761 return container_of(debug, struct hisi_qm, debug); 762 } 763 764 static u32 current_q_read(struct debugfs_file *file) 765 { 766 struct hisi_qm *qm = file_to_qm(file); 767 768 return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT; 769 } 770 771 static int current_q_write(struct debugfs_file *file, u32 val) 772 { 773 struct hisi_qm *qm = file_to_qm(file); 774 u32 tmp; 775 776 if (val >= qm->debug.curr_qm_qp_num) 777 return -EINVAL; 778 779 tmp = val << QM_DFX_QN_SHIFT | 780 (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK); 781 writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); 782 783 tmp = val << QM_DFX_QN_SHIFT | 784 (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK); 785 writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); 786 787 return 0; 788 } 789 790 static u32 clear_enable_read(struct debugfs_file *file) 791 { 792 struct hisi_qm *qm = file_to_qm(file); 793 794 return readl(qm->io_base + QM_DFX_CNT_CLR_CE); 795 } 796 797 /* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */ 798 static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl) 799 { 800 struct hisi_qm *qm = file_to_qm(file); 801 802 if (rd_clr_ctrl > 1) 803 return -EINVAL; 804 805 writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE); 806 807 return 0; 808 } 809 810 static ssize_t qm_debug_read(struct file *filp, char __user *buf, 811 size_t count, loff_t *pos) 812 { 813 struct debugfs_file *file = filp->private_data; 814 enum qm_debug_file index = file->index; 815 char tbuf[QM_DBG_TMP_BUF_LEN]; 816 u32 val; 817 int ret; 818 819 mutex_lock(&file->lock); 820 switch (index) { 821 case CURRENT_Q: 822 val = current_q_read(file); 823 break; 824 case CLEAR_ENABLE: 825 val = clear_enable_read(file); 826 break; 827 default: 828 mutex_unlock(&file->lock); 829 return -EINVAL; 830 } 831 mutex_unlock(&file->lock); 832 ret = sprintf(tbuf, "%u\n", val); 833 return simple_read_from_buffer(buf, count, pos, tbuf, ret); 834 } 835 836 static ssize_t qm_debug_write(struct file *filp, const char __user *buf, 837 size_t count, loff_t *pos) 838 { 839 struct debugfs_file *file = filp->private_data; 840 enum qm_debug_file index = file->index; 841 unsigned long val; 842 char tbuf[QM_DBG_TMP_BUF_LEN]; 843 int len, ret; 844 845 if (*pos != 0) 846 return 0; 847 848 if (count >= QM_DBG_TMP_BUF_LEN) 849 return -ENOSPC; 850 851 len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf, 852 count); 853 if (len < 0) 854 return len; 855 856 tbuf[len] = '\0'; 857 if (kstrtoul(tbuf, 0, &val)) 858 return -EFAULT; 859 860 mutex_lock(&file->lock); 861 switch (index) { 862 case CURRENT_Q: 863 ret = current_q_write(file, val); 864 if (ret) 865 goto err_input; 866 break; 867 case CLEAR_ENABLE: 868 ret = clear_enable_write(file, val); 869 if (ret) 870 goto err_input; 871 break; 872 default: 873 ret = -EINVAL; 874 goto err_input; 875 } 876 mutex_unlock(&file->lock); 877 878 return count; 879 880 err_input: 881 mutex_unlock(&file->lock); 882 return ret; 883 } 884 885 static const struct file_operations qm_debug_fops = { 886 .owner = THIS_MODULE, 887 .open = simple_open, 888 .read = qm_debug_read, 889 .write = qm_debug_write, 890 }; 891 892 struct qm_dfx_registers { 893 char *reg_name; 894 u64 reg_offset; 895 }; 896 897 #define CNT_CYC_REGS_NUM 10 898 static struct qm_dfx_registers qm_dfx_regs[] = { 899 /* XXX_CNT are reading clear register */ 900 {"QM_ECC_1BIT_CNT ", 0x104000ull}, 901 {"QM_ECC_MBIT_CNT ", 0x104008ull}, 902 {"QM_DFX_MB_CNT ", 0x104018ull}, 903 {"QM_DFX_DB_CNT ", 0x104028ull}, 904 {"QM_DFX_SQE_CNT ", 0x104038ull}, 905 {"QM_DFX_CQE_CNT ", 0x104048ull}, 906 {"QM_DFX_SEND_SQE_TO_ACC_CNT ", 0x104050ull}, 907 {"QM_DFX_WB_SQE_FROM_ACC_CNT ", 0x104058ull}, 908 {"QM_DFX_ACC_FINISH_CNT ", 0x104060ull}, 909 {"QM_DFX_CQE_ERR_CNT ", 0x1040b4ull}, 910 {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull}, 911 {"QM_ECC_1BIT_INF ", 0x104004ull}, 912 {"QM_ECC_MBIT_INF ", 0x10400cull}, 913 {"QM_DFX_ACC_RDY_VLD0 ", 0x1040a0ull}, 914 {"QM_DFX_ACC_RDY_VLD1 ", 0x1040a4ull}, 915 {"QM_DFX_AXI_RDY_VLD ", 0x1040a8ull}, 916 {"QM_DFX_FF_ST0 ", 0x1040c8ull}, 917 {"QM_DFX_FF_ST1 ", 0x1040ccull}, 918 {"QM_DFX_FF_ST2 ", 0x1040d0ull}, 919 {"QM_DFX_FF_ST3 ", 0x1040d4ull}, 920 {"QM_DFX_FF_ST4 ", 0x1040d8ull}, 921 {"QM_DFX_FF_ST5 ", 0x1040dcull}, 922 {"QM_DFX_FF_ST6 ", 0x1040e0ull}, 923 {"QM_IN_IDLE_ST ", 0x1040e4ull}, 924 { NULL, 0} 925 }; 926 927 static struct qm_dfx_registers qm_vf_dfx_regs[] = { 928 {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull}, 929 { NULL, 0} 930 }; 931 932 static int qm_regs_show(struct seq_file *s, void *unused) 933 { 934 struct hisi_qm *qm = s->private; 935 struct qm_dfx_registers *regs; 936 u32 val; 937 938 if (qm->fun_type == QM_HW_PF) 939 regs = qm_dfx_regs; 940 else 941 regs = qm_vf_dfx_regs; 942 943 while (regs->reg_name) { 944 val = readl(qm->io_base + regs->reg_offset); 945 seq_printf(s, "%s= 0x%08x\n", regs->reg_name, val); 946 regs++; 947 } 948 949 return 0; 950 } 951 952 static int qm_regs_open(struct inode *inode, struct file *file) 953 { 954 return single_open(file, qm_regs_show, inode->i_private); 955 } 956 957 static const struct file_operations qm_regs_fops = { 958 .owner = THIS_MODULE, 959 .open = qm_regs_open, 960 .read = seq_read, 961 .release = single_release, 962 }; 963 964 static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) 965 { 966 struct dentry *qm_d = qm->debug.qm_d, *tmp; 967 struct debugfs_file *file = qm->debug.files + index; 968 969 tmp = debugfs_create_file(qm_debug_file_name[index], 0600, qm_d, file, 970 &qm_debug_fops); 971 if (IS_ERR(tmp)) 972 return -ENOENT; 973 974 file->index = index; 975 mutex_init(&file->lock); 976 file->debug = &qm->debug; 977 978 return 0; 979 } 980 981 static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, 982 u32 msi) 983 { 984 dev_info(&qm->pdev->dev, 985 "QM v%d does not support hw error handle\n", qm->ver); 986 987 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); 988 } 989 990 static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, 991 u32 msi) 992 { 993 u32 irq_enable = ce | nfe | fe | msi; 994 u32 irq_unmask = ~irq_enable; 995 996 qm->error_mask = ce | nfe | fe; 997 qm->msi_mask = msi; 998 999 /* configure error type */ 1000 writel(ce, qm->io_base + QM_RAS_CE_ENABLE); 1001 writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); 1002 writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE); 1003 writel(fe, qm->io_base + QM_RAS_FE_ENABLE); 1004 1005 /* use RAS irq default, so only set QM_RAS_MSI_INT_SEL for MSI */ 1006 writel(msi, qm->io_base + QM_RAS_MSI_INT_SEL); 1007 1008 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); 1009 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); 1010 } 1011 1012 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) 1013 { 1014 const struct hisi_qm_hw_error *err = qm_hw_error; 1015 struct device *dev = &qm->pdev->dev; 1016 u32 reg_val, type, vf_num; 1017 1018 while (err->msg) { 1019 if (err->int_msk & error_status) { 1020 dev_err(dev, "%s [error status=0x%x] found\n", 1021 err->msg, err->int_msk); 1022 1023 if (error_status & QM_DB_TIMEOUT) { 1024 reg_val = readl(qm->io_base + 1025 QM_ABNORMAL_INF01); 1026 type = (reg_val & QM_DB_TIMEOUT_TYPE) >> 1027 QM_DB_TIMEOUT_TYPE_SHIFT; 1028 vf_num = reg_val & QM_DB_TIMEOUT_VF; 1029 dev_err(dev, "qm %s doorbell timeout in function %u\n", 1030 qm_db_timeout[type], vf_num); 1031 } 1032 1033 if (error_status & QM_OF_FIFO_OF) { 1034 reg_val = readl(qm->io_base + 1035 QM_ABNORMAL_INF00); 1036 type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >> 1037 QM_FIFO_OVERFLOW_TYPE_SHIFT; 1038 vf_num = reg_val & QM_FIFO_OVERFLOW_VF; 1039 1040 if (type < ARRAY_SIZE(qm_fifo_overflow)) 1041 dev_err(dev, "qm %s fifo overflow in function %u\n", 1042 qm_fifo_overflow[type], 1043 vf_num); 1044 else 1045 dev_err(dev, "unknown error type\n"); 1046 } 1047 } 1048 err++; 1049 } 1050 } 1051 1052 static pci_ers_result_t qm_hw_error_handle_v2(struct hisi_qm *qm) 1053 { 1054 u32 error_status, tmp; 1055 1056 /* read err sts */ 1057 tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS); 1058 error_status = qm->error_mask & tmp; 1059 1060 if (error_status) { 1061 qm_log_hw_error(qm, error_status); 1062 1063 /* clear err sts */ 1064 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); 1065 1066 return PCI_ERS_RESULT_NEED_RESET; 1067 } 1068 1069 return PCI_ERS_RESULT_RECOVERED; 1070 } 1071 1072 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { 1073 .qm_db = qm_db_v1, 1074 .get_irq_num = qm_get_irq_num_v1, 1075 .hw_error_init = qm_hw_error_init_v1, 1076 }; 1077 1078 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { 1079 .get_vft = qm_get_vft_v2, 1080 .qm_db = qm_db_v2, 1081 .get_irq_num = qm_get_irq_num_v2, 1082 .hw_error_init = qm_hw_error_init_v2, 1083 .hw_error_handle = qm_hw_error_handle_v2, 1084 }; 1085 1086 static void *qm_get_avail_sqe(struct hisi_qp *qp) 1087 { 1088 struct hisi_qp_status *qp_status = &qp->qp_status; 1089 u16 sq_tail = qp_status->sq_tail; 1090 1091 if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH)) 1092 return NULL; 1093 1094 return qp->sqe + sq_tail * qp->qm->sqe_size; 1095 } 1096 1097 /** 1098 * hisi_qm_create_qp() - Create a queue pair from qm. 1099 * @qm: The qm we create a qp from. 1100 * @alg_type: Accelerator specific algorithm type in sqc. 1101 * 1102 * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating 1103 * qp memory fails. 1104 */ 1105 struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type) 1106 { 1107 struct device *dev = &qm->pdev->dev; 1108 struct hisi_qp *qp; 1109 int qp_id, ret; 1110 1111 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1112 if (!qp) 1113 return ERR_PTR(-ENOMEM); 1114 1115 write_lock(&qm->qps_lock); 1116 1117 qp_id = find_first_zero_bit(qm->qp_bitmap, qm->qp_num); 1118 if (qp_id >= qm->qp_num) { 1119 write_unlock(&qm->qps_lock); 1120 dev_info(&qm->pdev->dev, "QM all queues are busy!\n"); 1121 ret = -EBUSY; 1122 goto err_free_qp; 1123 } 1124 set_bit(qp_id, qm->qp_bitmap); 1125 qm->qp_array[qp_id] = qp; 1126 1127 write_unlock(&qm->qps_lock); 1128 1129 qp->qm = qm; 1130 1131 if (qm->use_dma_api) { 1132 qp->qdma.size = qm->sqe_size * QM_Q_DEPTH + 1133 sizeof(struct qm_cqe) * QM_Q_DEPTH; 1134 qp->qdma.va = dma_alloc_coherent(dev, qp->qdma.size, 1135 &qp->qdma.dma, GFP_KERNEL); 1136 if (!qp->qdma.va) { 1137 ret = -ENOMEM; 1138 goto err_clear_bit; 1139 } 1140 1141 dev_dbg(dev, "allocate qp dma buf(va=%pK, dma=%pad, size=%zx)\n", 1142 qp->qdma.va, &qp->qdma.dma, qp->qdma.size); 1143 } 1144 1145 qp->qp_id = qp_id; 1146 qp->alg_type = alg_type; 1147 INIT_WORK(&qp->work, qm_qp_work_func); 1148 qp->wq = alloc_workqueue("hisi_qm", WQ_UNBOUND | WQ_HIGHPRI | 1149 WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0); 1150 if (!qp->wq) { 1151 ret = -EFAULT; 1152 goto err_free_qp_mem; 1153 } 1154 1155 return qp; 1156 1157 err_free_qp_mem: 1158 if (qm->use_dma_api) 1159 dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, 1160 qp->qdma.dma); 1161 err_clear_bit: 1162 write_lock(&qm->qps_lock); 1163 qm->qp_array[qp_id] = NULL; 1164 clear_bit(qp_id, qm->qp_bitmap); 1165 write_unlock(&qm->qps_lock); 1166 err_free_qp: 1167 kfree(qp); 1168 return ERR_PTR(ret); 1169 } 1170 EXPORT_SYMBOL_GPL(hisi_qm_create_qp); 1171 1172 /** 1173 * hisi_qm_release_qp() - Release a qp back to its qm. 1174 * @qp: The qp we want to release. 1175 * 1176 * This function releases the resource of a qp. 1177 */ 1178 void hisi_qm_release_qp(struct hisi_qp *qp) 1179 { 1180 struct hisi_qm *qm = qp->qm; 1181 struct qm_dma *qdma = &qp->qdma; 1182 struct device *dev = &qm->pdev->dev; 1183 1184 if (qm->use_dma_api && qdma->va) 1185 dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); 1186 1187 write_lock(&qm->qps_lock); 1188 qm->qp_array[qp->qp_id] = NULL; 1189 clear_bit(qp->qp_id, qm->qp_bitmap); 1190 write_unlock(&qm->qps_lock); 1191 1192 kfree(qp); 1193 } 1194 EXPORT_SYMBOL_GPL(hisi_qm_release_qp); 1195 1196 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) 1197 { 1198 struct hisi_qm *qm = qp->qm; 1199 struct device *dev = &qm->pdev->dev; 1200 enum qm_hw_ver ver = qm->ver; 1201 struct qm_sqc *sqc; 1202 struct qm_cqc *cqc; 1203 dma_addr_t sqc_dma; 1204 dma_addr_t cqc_dma; 1205 int ret; 1206 1207 qm_init_qp_status(qp); 1208 1209 sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL); 1210 if (!sqc) 1211 return -ENOMEM; 1212 sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc), 1213 DMA_TO_DEVICE); 1214 if (dma_mapping_error(dev, sqc_dma)) { 1215 kfree(sqc); 1216 return -ENOMEM; 1217 } 1218 1219 INIT_QC_COMMON(sqc, qp->sqe_dma, pasid); 1220 if (ver == QM_HW_V1) { 1221 sqc->dw3 = QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size); 1222 sqc->w8 = QM_Q_DEPTH - 1; 1223 } else if (ver == QM_HW_V2) { 1224 sqc->dw3 = QM_MK_SQC_DW3_V2(qm->sqe_size); 1225 sqc->w8 = 0; /* rand_qc */ 1226 } 1227 sqc->cq_num = qp_id; 1228 sqc->w13 = QM_MK_SQC_W13(0, 1, qp->alg_type); 1229 1230 ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0); 1231 dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE); 1232 kfree(sqc); 1233 if (ret) 1234 return ret; 1235 1236 cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); 1237 if (!cqc) 1238 return -ENOMEM; 1239 cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), 1240 DMA_TO_DEVICE); 1241 if (dma_mapping_error(dev, cqc_dma)) { 1242 kfree(cqc); 1243 return -ENOMEM; 1244 } 1245 1246 INIT_QC_COMMON(cqc, qp->cqe_dma, pasid); 1247 if (ver == QM_HW_V1) { 1248 cqc->dw3 = QM_MK_CQC_DW3_V1(0, 0, 0, 4); 1249 cqc->w8 = QM_Q_DEPTH - 1; 1250 } else if (ver == QM_HW_V2) { 1251 cqc->dw3 = QM_MK_CQC_DW3_V2(4); 1252 cqc->w8 = 0; 1253 } 1254 cqc->dw6 = 1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT; 1255 1256 ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0); 1257 dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE); 1258 kfree(cqc); 1259 1260 return ret; 1261 } 1262 1263 /** 1264 * hisi_qm_start_qp() - Start a qp into running. 1265 * @qp: The qp we want to start to run. 1266 * @arg: Accelerator specific argument. 1267 * 1268 * After this function, qp can receive request from user. Return qp_id if 1269 * successful, Return -EBUSY if failed. 1270 */ 1271 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) 1272 { 1273 struct hisi_qm *qm = qp->qm; 1274 struct device *dev = &qm->pdev->dev; 1275 enum qm_hw_ver ver = qm->ver; 1276 int qp_id = qp->qp_id; 1277 int pasid = arg; 1278 size_t off = 0; 1279 int ret; 1280 1281 #define QP_INIT_BUF(qp, type, size) do { \ 1282 (qp)->type = ((qp)->qdma.va + (off)); \ 1283 (qp)->type##_dma = (qp)->qdma.dma + (off); \ 1284 off += (size); \ 1285 } while (0) 1286 1287 if (!qp->qdma.dma) { 1288 dev_err(dev, "cannot get qm dma buffer\n"); 1289 return -EINVAL; 1290 } 1291 1292 /* sq need 128 bytes alignment */ 1293 if (qp->qdma.dma & QM_SQE_DATA_ALIGN_MASK) { 1294 dev_err(dev, "qm sq is not aligned to 128 byte\n"); 1295 return -EINVAL; 1296 } 1297 1298 QP_INIT_BUF(qp, sqe, qm->sqe_size * QM_Q_DEPTH); 1299 QP_INIT_BUF(qp, cqe, sizeof(struct qm_cqe) * QM_Q_DEPTH); 1300 1301 dev_dbg(dev, "init qp buffer(v%d):\n" 1302 " sqe (%pK, %lx)\n" 1303 " cqe (%pK, %lx)\n", 1304 ver, qp->sqe, (unsigned long)qp->sqe_dma, 1305 qp->cqe, (unsigned long)qp->cqe_dma); 1306 1307 ret = qm_qp_ctx_cfg(qp, qp_id, pasid); 1308 if (ret) 1309 return ret; 1310 1311 dev_dbg(dev, "queue %d started\n", qp_id); 1312 1313 return qp_id; 1314 } 1315 EXPORT_SYMBOL_GPL(hisi_qm_start_qp); 1316 1317 /** 1318 * hisi_qm_stop_qp() - Stop a qp in qm. 1319 * @qp: The qp we want to stop. 1320 * 1321 * This function is reverse of hisi_qm_start_qp. Return 0 if successful. 1322 */ 1323 int hisi_qm_stop_qp(struct hisi_qp *qp) 1324 { 1325 struct device *dev = &qp->qm->pdev->dev; 1326 int i = 0; 1327 1328 /* it is stopped */ 1329 if (test_bit(QP_STOP, &qp->qp_status.flags)) 1330 return 0; 1331 1332 while (atomic_read(&qp->qp_status.used)) { 1333 i++; 1334 msleep(20); 1335 if (i == 10) { 1336 dev_err(dev, "Cannot drain out data for stopping, Force to stop!\n"); 1337 return 0; 1338 } 1339 } 1340 1341 set_bit(QP_STOP, &qp->qp_status.flags); 1342 1343 dev_dbg(dev, "stop queue %u!", qp->qp_id); 1344 1345 return 0; 1346 } 1347 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); 1348 1349 /** 1350 * hisi_qp_send() - Queue up a task in the hardware queue. 1351 * @qp: The qp in which to put the message. 1352 * @msg: The message. 1353 * 1354 * This function will return -EBUSY if qp is currently full, and -EAGAIN 1355 * if qp related qm is resetting. 1356 */ 1357 int hisi_qp_send(struct hisi_qp *qp, const void *msg) 1358 { 1359 struct hisi_qp_status *qp_status = &qp->qp_status; 1360 u16 sq_tail = qp_status->sq_tail; 1361 u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH; 1362 void *sqe = qm_get_avail_sqe(qp); 1363 1364 if (unlikely(test_bit(QP_STOP, &qp->qp_status.flags))) { 1365 dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); 1366 return -EAGAIN; 1367 } 1368 1369 if (!sqe) 1370 return -EBUSY; 1371 1372 memcpy(sqe, msg, qp->qm->sqe_size); 1373 1374 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); 1375 atomic_inc(&qp->qp_status.used); 1376 qp_status->sq_tail = sq_tail_next; 1377 1378 return 0; 1379 } 1380 EXPORT_SYMBOL_GPL(hisi_qp_send); 1381 1382 static void hisi_qm_cache_wb(struct hisi_qm *qm) 1383 { 1384 unsigned int val; 1385 1386 if (qm->ver == QM_HW_V2) { 1387 writel(0x1, qm->io_base + QM_CACHE_WB_START); 1388 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, 1389 val, val & BIT(0), 10, 1000)) 1390 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); 1391 } 1392 } 1393 1394 /** 1395 * hisi_qm_init() - Initialize configures about qm. 1396 * @qm: The qm needing init. 1397 * 1398 * This function init qm, then we can call hisi_qm_start to put qm into work. 1399 */ 1400 int hisi_qm_init(struct hisi_qm *qm) 1401 { 1402 struct pci_dev *pdev = qm->pdev; 1403 struct device *dev = &pdev->dev; 1404 unsigned int num_vec; 1405 int ret; 1406 1407 switch (qm->ver) { 1408 case QM_HW_V1: 1409 qm->ops = &qm_hw_ops_v1; 1410 break; 1411 case QM_HW_V2: 1412 qm->ops = &qm_hw_ops_v2; 1413 break; 1414 default: 1415 return -EINVAL; 1416 } 1417 1418 ret = pci_enable_device_mem(pdev); 1419 if (ret < 0) { 1420 dev_err(&pdev->dev, "Failed to enable device mem!\n"); 1421 return ret; 1422 } 1423 1424 ret = pci_request_mem_regions(pdev, qm->dev_name); 1425 if (ret < 0) { 1426 dev_err(&pdev->dev, "Failed to request mem regions!\n"); 1427 goto err_disable_pcidev; 1428 } 1429 1430 qm->io_base = ioremap(pci_resource_start(pdev, PCI_BAR_2), 1431 pci_resource_len(qm->pdev, PCI_BAR_2)); 1432 if (!qm->io_base) { 1433 ret = -EIO; 1434 goto err_release_mem_regions; 1435 } 1436 1437 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 1438 if (ret < 0) 1439 goto err_iounmap; 1440 pci_set_master(pdev); 1441 1442 if (!qm->ops->get_irq_num) { 1443 ret = -EOPNOTSUPP; 1444 goto err_iounmap; 1445 } 1446 num_vec = qm->ops->get_irq_num(qm); 1447 ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); 1448 if (ret < 0) { 1449 dev_err(dev, "Failed to enable MSI vectors!\n"); 1450 goto err_iounmap; 1451 } 1452 1453 ret = qm_irq_register(qm); 1454 if (ret) 1455 goto err_free_irq_vectors; 1456 1457 mutex_init(&qm->mailbox_lock); 1458 rwlock_init(&qm->qps_lock); 1459 1460 dev_dbg(dev, "init qm %s with %s\n", pdev->is_physfn ? "pf" : "vf", 1461 qm->use_dma_api ? "dma api" : "iommu api"); 1462 1463 return 0; 1464 1465 err_free_irq_vectors: 1466 pci_free_irq_vectors(pdev); 1467 err_iounmap: 1468 iounmap(qm->io_base); 1469 err_release_mem_regions: 1470 pci_release_mem_regions(pdev); 1471 err_disable_pcidev: 1472 pci_disable_device(pdev); 1473 1474 return ret; 1475 } 1476 EXPORT_SYMBOL_GPL(hisi_qm_init); 1477 1478 /** 1479 * hisi_qm_uninit() - Uninitialize qm. 1480 * @qm: The qm needed uninit. 1481 * 1482 * This function uninits qm related device resources. 1483 */ 1484 void hisi_qm_uninit(struct hisi_qm *qm) 1485 { 1486 struct pci_dev *pdev = qm->pdev; 1487 struct device *dev = &pdev->dev; 1488 1489 if (qm->use_dma_api && qm->qdma.va) { 1490 hisi_qm_cache_wb(qm); 1491 dma_free_coherent(dev, qm->qdma.size, 1492 qm->qdma.va, qm->qdma.dma); 1493 memset(&qm->qdma, 0, sizeof(qm->qdma)); 1494 } 1495 1496 qm_irq_unregister(qm); 1497 pci_free_irq_vectors(pdev); 1498 iounmap(qm->io_base); 1499 pci_release_mem_regions(pdev); 1500 pci_disable_device(pdev); 1501 } 1502 EXPORT_SYMBOL_GPL(hisi_qm_uninit); 1503 1504 /** 1505 * hisi_qm_get_vft() - Get vft from a qm. 1506 * @qm: The qm we want to get its vft. 1507 * @base: The base number of queue in vft. 1508 * @number: The number of queues in vft. 1509 * 1510 * We can allocate multiple queues to a qm by configuring virtual function 1511 * table. We get related configures by this function. Normally, we call this 1512 * function in VF driver to get the queue information. 1513 * 1514 * qm hw v1 does not support this interface. 1515 */ 1516 int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) 1517 { 1518 if (!base || !number) 1519 return -EINVAL; 1520 1521 if (!qm->ops->get_vft) { 1522 dev_err(&qm->pdev->dev, "Don't support vft read!\n"); 1523 return -EINVAL; 1524 } 1525 1526 return qm->ops->get_vft(qm, base, number); 1527 } 1528 EXPORT_SYMBOL_GPL(hisi_qm_get_vft); 1529 1530 /** 1531 * hisi_qm_set_vft() - Set "virtual function table" for a qm. 1532 * @fun_num: Number of operated function. 1533 * @qm: The qm in which to set vft, alway in a PF. 1534 * @base: The base number of queue in vft. 1535 * @number: The number of queues in vft. 0 means invalid vft. 1536 * 1537 * This function is alway called in PF driver, it is used to assign queues 1538 * among PF and VFs. 1539 * 1540 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1) 1541 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1) 1542 * (VF function number 0x2) 1543 */ 1544 int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, 1545 u32 number) 1546 { 1547 u32 max_q_num = qm->ctrl_qp_num; 1548 1549 if (base >= max_q_num || number > max_q_num || 1550 (base + number) > max_q_num) 1551 return -EINVAL; 1552 1553 return qm_set_sqc_cqc_vft(qm, fun_num, base, number); 1554 } 1555 EXPORT_SYMBOL_GPL(hisi_qm_set_vft); 1556 1557 static void qm_init_eq_aeq_status(struct hisi_qm *qm) 1558 { 1559 struct hisi_qm_status *status = &qm->status; 1560 1561 status->eq_head = 0; 1562 status->aeq_head = 0; 1563 status->eqc_phase = 1; 1564 status->aeqc_phase = 1; 1565 } 1566 1567 static int qm_eq_ctx_cfg(struct hisi_qm *qm) 1568 { 1569 struct device *dev = &qm->pdev->dev; 1570 struct qm_eqc *eqc; 1571 struct qm_aeqc *aeqc; 1572 dma_addr_t eqc_dma; 1573 dma_addr_t aeqc_dma; 1574 int ret; 1575 1576 qm_init_eq_aeq_status(qm); 1577 1578 eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); 1579 if (!eqc) 1580 return -ENOMEM; 1581 eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), 1582 DMA_TO_DEVICE); 1583 if (dma_mapping_error(dev, eqc_dma)) { 1584 kfree(eqc); 1585 return -ENOMEM; 1586 } 1587 1588 eqc->base_l = lower_32_bits(qm->eqe_dma); 1589 eqc->base_h = upper_32_bits(qm->eqe_dma); 1590 if (qm->ver == QM_HW_V1) 1591 eqc->dw3 = QM_EQE_AEQE_SIZE; 1592 eqc->dw6 = (QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT); 1593 ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); 1594 dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); 1595 kfree(eqc); 1596 if (ret) 1597 return ret; 1598 1599 aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL); 1600 if (!aeqc) 1601 return -ENOMEM; 1602 aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc), 1603 DMA_TO_DEVICE); 1604 if (dma_mapping_error(dev, aeqc_dma)) { 1605 kfree(aeqc); 1606 return -ENOMEM; 1607 } 1608 1609 aeqc->base_l = lower_32_bits(qm->aeqe_dma); 1610 aeqc->base_h = upper_32_bits(qm->aeqe_dma); 1611 aeqc->dw6 = (QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT); 1612 1613 ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0); 1614 dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE); 1615 kfree(aeqc); 1616 1617 return ret; 1618 } 1619 1620 static int __hisi_qm_start(struct hisi_qm *qm) 1621 { 1622 struct pci_dev *pdev = qm->pdev; 1623 struct device *dev = &pdev->dev; 1624 size_t off = 0; 1625 int ret; 1626 1627 #define QM_INIT_BUF(qm, type, num) do { \ 1628 (qm)->type = ((qm)->qdma.va + (off)); \ 1629 (qm)->type##_dma = (qm)->qdma.dma + (off); \ 1630 off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ 1631 } while (0) 1632 1633 WARN_ON(!qm->qdma.dma); 1634 1635 if (qm->qp_num == 0) 1636 return -EINVAL; 1637 1638 if (qm->fun_type == QM_HW_PF) { 1639 ret = qm_dev_mem_reset(qm); 1640 if (ret) 1641 return ret; 1642 1643 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); 1644 if (ret) 1645 return ret; 1646 } 1647 1648 QM_INIT_BUF(qm, eqe, QM_Q_DEPTH); 1649 QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); 1650 QM_INIT_BUF(qm, sqc, qm->qp_num); 1651 QM_INIT_BUF(qm, cqc, qm->qp_num); 1652 1653 dev_dbg(dev, "init qm buffer:\n" 1654 " eqe (%pK, %lx)\n" 1655 " aeqe (%pK, %lx)\n" 1656 " sqc (%pK, %lx)\n" 1657 " cqc (%pK, %lx)\n", 1658 qm->eqe, (unsigned long)qm->eqe_dma, 1659 qm->aeqe, (unsigned long)qm->aeqe_dma, 1660 qm->sqc, (unsigned long)qm->sqc_dma, 1661 qm->cqc, (unsigned long)qm->cqc_dma); 1662 1663 ret = qm_eq_ctx_cfg(qm); 1664 if (ret) 1665 return ret; 1666 1667 ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0); 1668 if (ret) 1669 return ret; 1670 1671 ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0); 1672 if (ret) 1673 return ret; 1674 1675 writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); 1676 writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); 1677 1678 return 0; 1679 } 1680 1681 /** 1682 * hisi_qm_start() - start qm 1683 * @qm: The qm to be started. 1684 * 1685 * This function starts a qm, then we can allocate qp from this qm. 1686 */ 1687 int hisi_qm_start(struct hisi_qm *qm) 1688 { 1689 struct device *dev = &qm->pdev->dev; 1690 1691 dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num); 1692 1693 if (!qm->qp_num) { 1694 dev_err(dev, "qp_num should not be 0\n"); 1695 return -EINVAL; 1696 } 1697 1698 if (!qm->qp_bitmap) { 1699 qm->qp_bitmap = devm_kcalloc(dev, BITS_TO_LONGS(qm->qp_num), 1700 sizeof(long), GFP_KERNEL); 1701 qm->qp_array = devm_kcalloc(dev, qm->qp_num, 1702 sizeof(struct hisi_qp *), 1703 GFP_KERNEL); 1704 if (!qm->qp_bitmap || !qm->qp_array) 1705 return -ENOMEM; 1706 } 1707 1708 if (!qm->use_dma_api) { 1709 dev_dbg(&qm->pdev->dev, "qm delay start\n"); 1710 return 0; 1711 } else if (!qm->qdma.va) { 1712 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) + 1713 QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + 1714 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + 1715 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); 1716 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, 1717 &qm->qdma.dma, GFP_KERNEL); 1718 dev_dbg(dev, "allocate qm dma buf(va=%pK, dma=%pad, size=%zx)\n", 1719 qm->qdma.va, &qm->qdma.dma, qm->qdma.size); 1720 if (!qm->qdma.va) 1721 return -ENOMEM; 1722 } 1723 1724 return __hisi_qm_start(qm); 1725 } 1726 EXPORT_SYMBOL_GPL(hisi_qm_start); 1727 1728 /** 1729 * hisi_qm_stop() - Stop a qm. 1730 * @qm: The qm which will be stopped. 1731 * 1732 * This function stops qm and its qps, then qm can not accept request. 1733 * Related resources are not released at this state, we can use hisi_qm_start 1734 * to let qm start again. 1735 */ 1736 int hisi_qm_stop(struct hisi_qm *qm) 1737 { 1738 struct device *dev; 1739 struct hisi_qp *qp; 1740 int ret = 0, i; 1741 1742 if (!qm || !qm->pdev) { 1743 WARN_ON(1); 1744 return -EINVAL; 1745 } 1746 1747 dev = &qm->pdev->dev; 1748 1749 /* Mask eq and aeq irq */ 1750 writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK); 1751 writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK); 1752 1753 /* Stop all qps belong to this qm */ 1754 for (i = 0; i < qm->qp_num; i++) { 1755 qp = qm->qp_array[i]; 1756 if (qp) { 1757 ret = hisi_qm_stop_qp(qp); 1758 if (ret < 0) { 1759 dev_err(dev, "Failed to stop qp%d!\n", i); 1760 return -EBUSY; 1761 } 1762 } 1763 } 1764 1765 if (qm->fun_type == QM_HW_PF) { 1766 ret = hisi_qm_set_vft(qm, 0, 0, 0); 1767 if (ret < 0) 1768 dev_err(dev, "Failed to set vft!\n"); 1769 } 1770 1771 return ret; 1772 } 1773 EXPORT_SYMBOL_GPL(hisi_qm_stop); 1774 1775 /** 1776 * hisi_qm_debug_init() - Initialize qm related debugfs files. 1777 * @qm: The qm for which we want to add debugfs files. 1778 * 1779 * Create qm related debugfs files. 1780 */ 1781 int hisi_qm_debug_init(struct hisi_qm *qm) 1782 { 1783 struct dentry *qm_d, *qm_regs; 1784 int i, ret; 1785 1786 qm_d = debugfs_create_dir("qm", qm->debug.debug_root); 1787 if (IS_ERR(qm_d)) 1788 return -ENOENT; 1789 qm->debug.qm_d = qm_d; 1790 1791 /* only show this in PF */ 1792 if (qm->fun_type == QM_HW_PF) 1793 for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) 1794 if (qm_create_debugfs_file(qm, i)) { 1795 ret = -ENOENT; 1796 goto failed_to_create; 1797 } 1798 1799 qm_regs = debugfs_create_file("qm_regs", 0444, qm->debug.qm_d, qm, 1800 &qm_regs_fops); 1801 if (IS_ERR(qm_regs)) { 1802 ret = -ENOENT; 1803 goto failed_to_create; 1804 } 1805 1806 return 0; 1807 1808 failed_to_create: 1809 debugfs_remove_recursive(qm_d); 1810 return ret; 1811 } 1812 EXPORT_SYMBOL_GPL(hisi_qm_debug_init); 1813 1814 /** 1815 * hisi_qm_debug_regs_clear() - clear qm debug related registers. 1816 * @qm: The qm for which we want to clear its debug registers. 1817 */ 1818 void hisi_qm_debug_regs_clear(struct hisi_qm *qm) 1819 { 1820 struct qm_dfx_registers *regs; 1821 int i; 1822 1823 /* clear current_q */ 1824 writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); 1825 writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); 1826 1827 /* 1828 * these registers are reading and clearing, so clear them after 1829 * reading them. 1830 */ 1831 writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE); 1832 1833 regs = qm_dfx_regs; 1834 for (i = 0; i < CNT_CYC_REGS_NUM; i++) { 1835 readl(qm->io_base + regs->reg_offset); 1836 regs++; 1837 } 1838 1839 writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE); 1840 } 1841 EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); 1842 1843 /** 1844 * hisi_qm_hw_error_init() - Configure qm hardware error report method. 1845 * @qm: The qm which we want to configure. 1846 * @ce: Bit mask of correctable error configure. 1847 * @nfe: Bit mask of non-fatal error configure. 1848 * @fe: Bit mask of fatal error configure. 1849 * @msi: Bit mask of error reported by message signal interrupt. 1850 * 1851 * Hardware errors of qm can be reported either by RAS interrupts which will 1852 * be handled by UEFI and then PCIe AER or by device MSI. User can configure 1853 * each error to use either of above two methods. For RAS interrupts, we can 1854 * configure an error as one of correctable error, non-fatal error or 1855 * fatal error. 1856 * 1857 * Bits indicating errors can be configured to ce, nfe, fe and msi to enable 1858 * related report methods. Error report will be masked if related error bit 1859 * does not configure. 1860 */ 1861 void hisi_qm_hw_error_init(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe, 1862 u32 msi) 1863 { 1864 if (!qm->ops->hw_error_init) { 1865 dev_err(&qm->pdev->dev, "QM version %d doesn't support hw error handling!\n", 1866 qm->ver); 1867 return; 1868 } 1869 1870 qm->ops->hw_error_init(qm, ce, nfe, fe, msi); 1871 } 1872 EXPORT_SYMBOL_GPL(hisi_qm_hw_error_init); 1873 1874 /** 1875 * hisi_qm_hw_error_handle() - Handle qm non-fatal hardware errors. 1876 * @qm: The qm which has non-fatal hardware errors. 1877 * 1878 * Accelerators use this function to handle qm non-fatal hardware errors. 1879 */ 1880 int hisi_qm_hw_error_handle(struct hisi_qm *qm) 1881 { 1882 if (!qm->ops->hw_error_handle) { 1883 dev_err(&qm->pdev->dev, "QM version %d doesn't support hw error report!\n", 1884 qm->ver); 1885 return PCI_ERS_RESULT_NONE; 1886 } 1887 1888 return qm->ops->hw_error_handle(qm); 1889 } 1890 EXPORT_SYMBOL_GPL(hisi_qm_hw_error_handle); 1891 1892 /** 1893 * hisi_qm_get_hw_version() - Get hardware version of a qm. 1894 * @pdev: The device which hardware version we want to get. 1895 * 1896 * This function gets the hardware version of a qm. Return QM_HW_UNKNOWN 1897 * if the hardware version is not supported. 1898 */ 1899 enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev) 1900 { 1901 switch (pdev->revision) { 1902 case QM_HW_V1: 1903 case QM_HW_V2: 1904 return pdev->revision; 1905 default: 1906 return QM_HW_UNKNOWN; 1907 } 1908 } 1909 EXPORT_SYMBOL_GPL(hisi_qm_get_hw_version); 1910 1911 MODULE_LICENSE("GPL v2"); 1912 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 1913 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); 1914