1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 
4 #include <linux/acpi.h>
5 #include <linux/aer.h>
6 #include <linux/bitops.h>
7 #include <linux/debugfs.h>
8 #include <linux/init.h>
9 #include <linux/io.h>
10 #include <linux/iommu.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/seq_file.h>
15 #include <linux/topology.h>
16 
17 #include "sec.h"
18 
19 #define SEC_VF_NUM			63
20 #define SEC_QUEUE_NUM_V1		4096
21 #define SEC_QUEUE_NUM_V2		1024
22 #define SEC_PF_PCI_DEVICE_ID		0xa255
23 #define SEC_VF_PCI_DEVICE_ID		0xa256
24 
25 #define SEC_BD_ERR_CHK_EN0		0xEFFFFFFF
26 #define SEC_BD_ERR_CHK_EN1		0x7ffff7fd
27 #define SEC_BD_ERR_CHK_EN3		0xffffbfff
28 
29 #define SEC_SQE_SIZE			128
30 #define SEC_SQ_SIZE			(SEC_SQE_SIZE * QM_Q_DEPTH)
31 #define SEC_PF_DEF_Q_NUM		256
32 #define SEC_PF_DEF_Q_BASE		0
33 #define SEC_CTX_Q_NUM_DEF		2
34 #define SEC_CTX_Q_NUM_MAX		32
35 
36 #define SEC_CTRL_CNT_CLR_CE		0x301120
37 #define SEC_CTRL_CNT_CLR_CE_BIT		BIT(0)
38 #define SEC_ENGINE_PF_CFG_OFF		0x300000
39 #define SEC_ACC_COMMON_REG_OFF		0x1000
40 #define SEC_CORE_INT_SOURCE		0x301010
41 #define SEC_CORE_INT_MASK		0x301000
42 #define SEC_CORE_INT_STATUS		0x301008
43 #define SEC_CORE_SRAM_ECC_ERR_INFO	0x301C14
44 #define SEC_ECC_NUM(err)			(((err) >> 16) & 0xFF)
45 #define SEC_ECC_ADDR(err)			((err) >> 0)
46 #define SEC_CORE_INT_DISABLE		0x0
47 #define SEC_CORE_INT_ENABLE		0x1ff
48 #define SEC_CORE_INT_CLEAR		0x1ff
49 #define SEC_SAA_ENABLE			0x17f
50 
51 #define SEC_RAS_CE_REG			0x301050
52 #define SEC_RAS_FE_REG			0x301054
53 #define SEC_RAS_NFE_REG			0x301058
54 #define SEC_RAS_CE_ENB_MSK		0x88
55 #define SEC_RAS_FE_ENB_MSK		0x0
56 #define SEC_RAS_NFE_ENB_MSK		0x177
57 #define SEC_RAS_DISABLE			0x0
58 #define SEC_MEM_START_INIT_REG		0x0100
59 #define SEC_MEM_INIT_DONE_REG		0x0104
60 
61 #define SEC_CONTROL_REG			0x0200
62 #define SEC_TRNG_EN_SHIFT		8
63 #define SEC_CLK_GATE_ENABLE		BIT(3)
64 #define SEC_CLK_GATE_DISABLE		(~BIT(3))
65 #define SEC_AXI_SHUTDOWN_ENABLE	BIT(12)
66 #define SEC_AXI_SHUTDOWN_DISABLE	0xFFFFEFFF
67 
68 #define SEC_INTERFACE_USER_CTRL0_REG	0x0220
69 #define SEC_INTERFACE_USER_CTRL1_REG	0x0224
70 #define SEC_SAA_EN_REG					0x0270
71 #define SEC_BD_ERR_CHK_EN_REG0		0x0380
72 #define SEC_BD_ERR_CHK_EN_REG1		0x0384
73 #define SEC_BD_ERR_CHK_EN_REG3		0x038c
74 
75 #define SEC_USER0_SMMU_NORMAL		(BIT(23) | BIT(15))
76 #define SEC_USER1_SMMU_NORMAL		(BIT(31) | BIT(23) | BIT(15) | BIT(7))
77 #define SEC_CORE_INT_STATUS_M_ECC	BIT(2)
78 
79 #define SEC_DELAY_10_US			10
80 #define SEC_POLL_TIMEOUT_US		1000
81 #define SEC_DBGFS_VAL_MAX_LEN		20
82 #define SEC_SINGLE_PORT_MAX_TRANS	0x2060
83 
84 #define SEC_SQE_MASK_OFFSET		64
85 #define SEC_SQE_MASK_LEN		48
86 
87 #define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \
88 			     SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF)
89 
90 struct sec_hw_error {
91 	u32 int_msk;
92 	const char *msg;
93 };
94 
95 struct sec_dfx_item {
96 	const char *name;
97 	u32 offset;
98 };
99 
100 static const char sec_name[] = "hisi_sec2";
101 static struct dentry *sec_debugfs_root;
102 
103 static struct hisi_qm_list sec_devices = {
104 	.register_to_crypto	= sec_register_to_crypto,
105 	.unregister_from_crypto	= sec_unregister_from_crypto,
106 };
107 
108 static const struct sec_hw_error sec_hw_errors[] = {
109 	{.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"},
110 	{.int_msk = BIT(1), .msg = "sec_axi_bresp_err_rint"},
111 	{.int_msk = BIT(2), .msg = "sec_ecc_2bit_err_rint"},
112 	{.int_msk = BIT(3), .msg = "sec_ecc_1bit_err_rint"},
113 	{.int_msk = BIT(4), .msg = "sec_req_trng_timeout_rint"},
114 	{.int_msk = BIT(5), .msg = "sec_fsm_hbeat_rint"},
115 	{.int_msk = BIT(6), .msg = "sec_channel_req_rng_timeout_rint"},
116 	{.int_msk = BIT(7), .msg = "sec_bd_err_rint"},
117 	{.int_msk = BIT(8), .msg = "sec_chain_buff_err_rint"},
118 	{ /* sentinel */ }
119 };
120 
121 static const char * const sec_dbg_file_name[] = {
122 	[SEC_CURRENT_QM] = "current_qm",
123 	[SEC_CLEAR_ENABLE] = "clear_enable",
124 };
125 
126 static struct sec_dfx_item sec_dfx_labels[] = {
127 	{"send_cnt", offsetof(struct sec_dfx, send_cnt)},
128 	{"recv_cnt", offsetof(struct sec_dfx, recv_cnt)},
129 	{"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)},
130 	{"recv_busy_cnt", offsetof(struct sec_dfx, recv_busy_cnt)},
131 	{"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)},
132 	{"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)},
133 	{"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)},
134 };
135 
136 static const struct debugfs_reg32 sec_dfx_regs[] = {
137 	{"SEC_PF_ABNORMAL_INT_SOURCE    ",  0x301010},
138 	{"SEC_SAA_EN                    ",  0x301270},
139 	{"SEC_BD_LATENCY_MIN            ",  0x301600},
140 	{"SEC_BD_LATENCY_MAX            ",  0x301608},
141 	{"SEC_BD_LATENCY_AVG            ",  0x30160C},
142 	{"SEC_BD_NUM_IN_SAA0            ",  0x301670},
143 	{"SEC_BD_NUM_IN_SAA1            ",  0x301674},
144 	{"SEC_BD_NUM_IN_SEC             ",  0x301680},
145 	{"SEC_ECC_1BIT_CNT              ",  0x301C00},
146 	{"SEC_ECC_1BIT_INFO             ",  0x301C04},
147 	{"SEC_ECC_2BIT_CNT              ",  0x301C10},
148 	{"SEC_ECC_2BIT_INFO             ",  0x301C14},
149 	{"SEC_BD_SAA0                   ",  0x301C20},
150 	{"SEC_BD_SAA1                   ",  0x301C24},
151 	{"SEC_BD_SAA2                   ",  0x301C28},
152 	{"SEC_BD_SAA3                   ",  0x301C2C},
153 	{"SEC_BD_SAA4                   ",  0x301C30},
154 	{"SEC_BD_SAA5                   ",  0x301C34},
155 	{"SEC_BD_SAA6                   ",  0x301C38},
156 	{"SEC_BD_SAA7                   ",  0x301C3C},
157 	{"SEC_BD_SAA8                   ",  0x301C40},
158 };
159 
160 static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp)
161 {
162 	return q_num_set(val, kp, SEC_PF_PCI_DEVICE_ID);
163 }
164 
165 static const struct kernel_param_ops sec_pf_q_num_ops = {
166 	.set = sec_pf_q_num_set,
167 	.get = param_get_int,
168 };
169 
170 static u32 pf_q_num = SEC_PF_DEF_Q_NUM;
171 module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444);
172 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
173 
174 static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp)
175 {
176 	u32 ctx_q_num;
177 	int ret;
178 
179 	if (!val)
180 		return -EINVAL;
181 
182 	ret = kstrtou32(val, 10, &ctx_q_num);
183 	if (ret)
184 		return -EINVAL;
185 
186 	if (!ctx_q_num || ctx_q_num > SEC_CTX_Q_NUM_MAX || ctx_q_num & 0x1) {
187 		pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num);
188 		return -EINVAL;
189 	}
190 
191 	return param_set_int(val, kp);
192 }
193 
194 static const struct kernel_param_ops sec_ctx_q_num_ops = {
195 	.set = sec_ctx_q_num_set,
196 	.get = param_get_int,
197 };
198 static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF;
199 module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444);
200 MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (2 default, 2, 4, ..., 32)");
201 
202 static const struct kernel_param_ops vfs_num_ops = {
203 	.set = vfs_num_set,
204 	.get = param_get_int,
205 };
206 
207 static u32 vfs_num;
208 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
209 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
210 
211 void sec_destroy_qps(struct hisi_qp **qps, int qp_num)
212 {
213 	hisi_qm_free_qps(qps, qp_num);
214 	kfree(qps);
215 }
216 
217 struct hisi_qp **sec_create_qps(void)
218 {
219 	int node = cpu_to_node(smp_processor_id());
220 	u32 ctx_num = ctx_q_num;
221 	struct hisi_qp **qps;
222 	int ret;
223 
224 	qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL);
225 	if (!qps)
226 		return NULL;
227 
228 	ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps);
229 	if (!ret)
230 		return qps;
231 
232 	kfree(qps);
233 	return NULL;
234 }
235 
236 
237 static const struct pci_device_id sec_dev_ids[] = {
238 	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_PF_PCI_DEVICE_ID) },
239 	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_VF_PCI_DEVICE_ID) },
240 	{ 0, }
241 };
242 MODULE_DEVICE_TABLE(pci, sec_dev_ids);
243 
244 static u8 sec_get_endian(struct hisi_qm *qm)
245 {
246 	u32 reg;
247 
248 	/*
249 	 * As for VF, it is a wrong way to get endian setting by
250 	 * reading a register of the engine
251 	 */
252 	if (qm->pdev->is_virtfn) {
253 		dev_err_ratelimited(&qm->pdev->dev,
254 				    "cannot access a register in VF!\n");
255 		return SEC_LE;
256 	}
257 	reg = readl_relaxed(qm->io_base + SEC_ENGINE_PF_CFG_OFF +
258 			    SEC_ACC_COMMON_REG_OFF + SEC_CONTROL_REG);
259 
260 	/* BD little endian mode */
261 	if (!(reg & BIT(0)))
262 		return SEC_LE;
263 
264 	/* BD 32-bits big endian mode */
265 	else if (!(reg & BIT(1)))
266 		return SEC_32BE;
267 
268 	/* BD 64-bits big endian mode */
269 	else
270 		return SEC_64BE;
271 }
272 
273 static int sec_engine_init(struct hisi_qm *qm)
274 {
275 	int ret;
276 	u32 reg;
277 
278 	/* disable clock gate control */
279 	reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
280 	reg &= SEC_CLK_GATE_DISABLE;
281 	writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
282 
283 	writel_relaxed(0x1, SEC_ADDR(qm, SEC_MEM_START_INIT_REG));
284 
285 	ret = readl_relaxed_poll_timeout(SEC_ADDR(qm, SEC_MEM_INIT_DONE_REG),
286 					 reg, reg & 0x1, SEC_DELAY_10_US,
287 					 SEC_POLL_TIMEOUT_US);
288 	if (ret) {
289 		pci_err(qm->pdev, "fail to init sec mem\n");
290 		return ret;
291 	}
292 
293 	reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
294 	reg |= (0x1 << SEC_TRNG_EN_SHIFT);
295 	writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
296 
297 	reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG));
298 	reg |= SEC_USER0_SMMU_NORMAL;
299 	writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG));
300 
301 	reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG));
302 	reg |= SEC_USER1_SMMU_NORMAL;
303 	writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG));
304 
305 	writel(SEC_SINGLE_PORT_MAX_TRANS,
306 	       qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS);
307 
308 	writel(SEC_SAA_ENABLE, SEC_ADDR(qm, SEC_SAA_EN_REG));
309 
310 	/* Enable sm4 extra mode, as ctr/ecb */
311 	writel_relaxed(SEC_BD_ERR_CHK_EN0,
312 		       SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG0));
313 	/* Enable sm4 xts mode multiple iv */
314 	writel_relaxed(SEC_BD_ERR_CHK_EN1,
315 		       SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG1));
316 	writel_relaxed(SEC_BD_ERR_CHK_EN3,
317 		       SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG3));
318 
319 	/* config endian */
320 	reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
321 	reg |= sec_get_endian(qm);
322 	writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
323 
324 	return 0;
325 }
326 
327 static int sec_set_user_domain_and_cache(struct hisi_qm *qm)
328 {
329 	/* qm user domain */
330 	writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1);
331 	writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE);
332 	writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1);
333 	writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE);
334 	writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE);
335 
336 	/* qm cache */
337 	writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG);
338 	writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE);
339 
340 	/* disable FLR triggered by BME(bus master enable) */
341 	writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG);
342 	writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);
343 
344 	/* enable sqc,cqc writeback */
345 	writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
346 	       CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
347 	       FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL);
348 
349 	return sec_engine_init(qm);
350 }
351 
352 /* sec_debug_regs_clear() - clear the sec debug regs */
353 static void sec_debug_regs_clear(struct hisi_qm *qm)
354 {
355 	int i;
356 
357 	/* clear current_qm */
358 	writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
359 	writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
360 
361 	/* clear sec dfx regs */
362 	writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE);
363 	for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++)
364 		readl(qm->io_base + sec_dfx_regs[i].offset);
365 
366 	/* clear rdclr_en */
367 	writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE);
368 
369 	hisi_qm_debug_regs_clear(qm);
370 }
371 
372 static void sec_hw_error_enable(struct hisi_qm *qm)
373 {
374 	u32 val;
375 
376 	if (qm->ver == QM_HW_V1) {
377 		writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
378 		pci_info(qm->pdev, "V1 not support hw error handle\n");
379 		return;
380 	}
381 
382 	val = readl(SEC_ADDR(qm, SEC_CONTROL_REG));
383 
384 	/* clear SEC hw error source if having */
385 	writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE);
386 
387 	/* enable SEC hw error interrupts */
388 	writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK);
389 
390 	/* enable RAS int */
391 	writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG);
392 	writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG);
393 	writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG);
394 
395 	/* enable SEC block master OOO when m-bit error occur */
396 	val = val | SEC_AXI_SHUTDOWN_ENABLE;
397 
398 	writel(val, SEC_ADDR(qm, SEC_CONTROL_REG));
399 }
400 
401 static void sec_hw_error_disable(struct hisi_qm *qm)
402 {
403 	u32 val;
404 
405 	val = readl(SEC_ADDR(qm, SEC_CONTROL_REG));
406 
407 	/* disable RAS int */
408 	writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG);
409 	writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG);
410 	writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG);
411 
412 	/* disable SEC hw error interrupts */
413 	writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
414 
415 	/* disable SEC block master OOO when m-bit error occur */
416 	val = val & SEC_AXI_SHUTDOWN_DISABLE;
417 
418 	writel(val, SEC_ADDR(qm, SEC_CONTROL_REG));
419 }
420 
421 static u32 sec_current_qm_read(struct sec_debug_file *file)
422 {
423 	struct hisi_qm *qm = file->qm;
424 
425 	return readl(qm->io_base + QM_DFX_MB_CNT_VF);
426 }
427 
428 static int sec_current_qm_write(struct sec_debug_file *file, u32 val)
429 {
430 	struct hisi_qm *qm = file->qm;
431 	u32 vfq_num;
432 	u32 tmp;
433 
434 	if (val > qm->vfs_num)
435 		return -EINVAL;
436 
437 	/* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
438 	if (!val) {
439 		qm->debug.curr_qm_qp_num = qm->qp_num;
440 	} else {
441 		vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num;
442 
443 		if (val == qm->vfs_num)
444 			qm->debug.curr_qm_qp_num =
445 				qm->ctrl_qp_num - qm->qp_num -
446 				(qm->vfs_num - 1) * vfq_num;
447 		else
448 			qm->debug.curr_qm_qp_num = vfq_num;
449 	}
450 
451 	writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
452 	writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
453 
454 	tmp = val |
455 	      (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
456 	writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
457 
458 	tmp = val |
459 	      (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
460 	writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
461 
462 	return 0;
463 }
464 
465 static u32 sec_clear_enable_read(struct sec_debug_file *file)
466 {
467 	struct hisi_qm *qm = file->qm;
468 
469 	return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &
470 			SEC_CTRL_CNT_CLR_CE_BIT;
471 }
472 
473 static int sec_clear_enable_write(struct sec_debug_file *file, u32 val)
474 {
475 	struct hisi_qm *qm = file->qm;
476 	u32 tmp;
477 
478 	if (val != 1 && val)
479 		return -EINVAL;
480 
481 	tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &
482 	       ~SEC_CTRL_CNT_CLR_CE_BIT) | val;
483 	writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE);
484 
485 	return 0;
486 }
487 
488 static ssize_t sec_debug_read(struct file *filp, char __user *buf,
489 			       size_t count, loff_t *pos)
490 {
491 	struct sec_debug_file *file = filp->private_data;
492 	char tbuf[SEC_DBGFS_VAL_MAX_LEN];
493 	u32 val;
494 	int ret;
495 
496 	spin_lock_irq(&file->lock);
497 
498 	switch (file->index) {
499 	case SEC_CURRENT_QM:
500 		val = sec_current_qm_read(file);
501 		break;
502 	case SEC_CLEAR_ENABLE:
503 		val = sec_clear_enable_read(file);
504 		break;
505 	default:
506 		spin_unlock_irq(&file->lock);
507 		return -EINVAL;
508 	}
509 
510 	spin_unlock_irq(&file->lock);
511 	ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val);
512 
513 	return simple_read_from_buffer(buf, count, pos, tbuf, ret);
514 }
515 
516 static ssize_t sec_debug_write(struct file *filp, const char __user *buf,
517 			       size_t count, loff_t *pos)
518 {
519 	struct sec_debug_file *file = filp->private_data;
520 	char tbuf[SEC_DBGFS_VAL_MAX_LEN];
521 	unsigned long val;
522 	int len, ret;
523 
524 	if (*pos != 0)
525 		return 0;
526 
527 	if (count >= SEC_DBGFS_VAL_MAX_LEN)
528 		return -ENOSPC;
529 
530 	len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1,
531 				     pos, buf, count);
532 	if (len < 0)
533 		return len;
534 
535 	tbuf[len] = '\0';
536 	if (kstrtoul(tbuf, 0, &val))
537 		return -EFAULT;
538 
539 	spin_lock_irq(&file->lock);
540 
541 	switch (file->index) {
542 	case SEC_CURRENT_QM:
543 		ret = sec_current_qm_write(file, val);
544 		if (ret)
545 			goto err_input;
546 		break;
547 	case SEC_CLEAR_ENABLE:
548 		ret = sec_clear_enable_write(file, val);
549 		if (ret)
550 			goto err_input;
551 		break;
552 	default:
553 		ret = -EINVAL;
554 		goto err_input;
555 	}
556 
557 	spin_unlock_irq(&file->lock);
558 
559 	return count;
560 
561  err_input:
562 	spin_unlock_irq(&file->lock);
563 	return ret;
564 }
565 
566 static const struct file_operations sec_dbg_fops = {
567 	.owner = THIS_MODULE,
568 	.open = simple_open,
569 	.read = sec_debug_read,
570 	.write = sec_debug_write,
571 };
572 
573 static int sec_debugfs_atomic64_get(void *data, u64 *val)
574 {
575 	*val = atomic64_read((atomic64_t *)data);
576 
577 	return 0;
578 }
579 
580 static int sec_debugfs_atomic64_set(void *data, u64 val)
581 {
582 	if (val)
583 		return -EINVAL;
584 
585 	atomic64_set((atomic64_t *)data, 0);
586 
587 	return 0;
588 }
589 
590 DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get,
591 			 sec_debugfs_atomic64_set, "%lld\n");
592 
593 static int sec_core_debug_init(struct hisi_qm *qm)
594 {
595 	struct sec_dev *sec = container_of(qm, struct sec_dev, qm);
596 	struct device *dev = &qm->pdev->dev;
597 	struct sec_dfx *dfx = &sec->debug.dfx;
598 	struct debugfs_regset32 *regset;
599 	struct dentry *tmp_d;
600 	int i;
601 
602 	tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root);
603 
604 	regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
605 	if (!regset)
606 		return -ENOMEM;
607 
608 	regset->regs = sec_dfx_regs;
609 	regset->nregs = ARRAY_SIZE(sec_dfx_regs);
610 	regset->base = qm->io_base;
611 
612 	if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID)
613 		debugfs_create_regset32("regs", 0444, tmp_d, regset);
614 
615 	for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) {
616 		atomic64_t *data = (atomic64_t *)((uintptr_t)dfx +
617 					sec_dfx_labels[i].offset);
618 		debugfs_create_file(sec_dfx_labels[i].name, 0644,
619 				   tmp_d, data, &sec_atomic64_ops);
620 	}
621 
622 	return 0;
623 }
624 
625 static int sec_debug_init(struct hisi_qm *qm)
626 {
627 	struct sec_dev *sec = container_of(qm, struct sec_dev, qm);
628 	int i;
629 
630 	if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) {
631 		for (i = SEC_CURRENT_QM; i < SEC_DEBUG_FILE_NUM; i++) {
632 			spin_lock_init(&sec->debug.files[i].lock);
633 			sec->debug.files[i].index = i;
634 			sec->debug.files[i].qm = qm;
635 
636 			debugfs_create_file(sec_dbg_file_name[i], 0600,
637 						  qm->debug.debug_root,
638 						  sec->debug.files + i,
639 						  &sec_dbg_fops);
640 		}
641 	}
642 
643 	return sec_core_debug_init(qm);
644 }
645 
646 static int sec_debugfs_init(struct hisi_qm *qm)
647 {
648 	struct device *dev = &qm->pdev->dev;
649 	int ret;
650 
651 	qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
652 						  sec_debugfs_root);
653 	qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET;
654 	qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN;
655 	hisi_qm_debug_init(qm);
656 
657 	ret = sec_debug_init(qm);
658 	if (ret)
659 		goto failed_to_create;
660 
661 	return 0;
662 
663 failed_to_create:
664 	debugfs_remove_recursive(sec_debugfs_root);
665 	return ret;
666 }
667 
668 static void sec_debugfs_exit(struct hisi_qm *qm)
669 {
670 	debugfs_remove_recursive(qm->debug.debug_root);
671 }
672 
673 static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts)
674 {
675 	const struct sec_hw_error *errs = sec_hw_errors;
676 	struct device *dev = &qm->pdev->dev;
677 	u32 err_val;
678 
679 	while (errs->msg) {
680 		if (errs->int_msk & err_sts) {
681 			dev_err(dev, "%s [error status=0x%x] found\n",
682 					errs->msg, errs->int_msk);
683 
684 			if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) {
685 				err_val = readl(qm->io_base +
686 						SEC_CORE_SRAM_ECC_ERR_INFO);
687 				dev_err(dev, "multi ecc sram num=0x%x\n",
688 						SEC_ECC_NUM(err_val));
689 			}
690 		}
691 		errs++;
692 	}
693 }
694 
695 static u32 sec_get_hw_err_status(struct hisi_qm *qm)
696 {
697 	return readl(qm->io_base + SEC_CORE_INT_STATUS);
698 }
699 
700 static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
701 {
702 	writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE);
703 }
704 
705 static void sec_open_axi_master_ooo(struct hisi_qm *qm)
706 {
707 	u32 val;
708 
709 	val = readl(SEC_ADDR(qm, SEC_CONTROL_REG));
710 	writel(val & SEC_AXI_SHUTDOWN_DISABLE, SEC_ADDR(qm, SEC_CONTROL_REG));
711 	writel(val | SEC_AXI_SHUTDOWN_ENABLE, SEC_ADDR(qm, SEC_CONTROL_REG));
712 }
713 
714 static const struct hisi_qm_err_ini sec_err_ini = {
715 	.hw_init		= sec_set_user_domain_and_cache,
716 	.hw_err_enable		= sec_hw_error_enable,
717 	.hw_err_disable		= sec_hw_error_disable,
718 	.get_dev_hw_err_status	= sec_get_hw_err_status,
719 	.clear_dev_hw_err_status = sec_clear_hw_err_status,
720 	.log_dev_hw_err		= sec_log_hw_error,
721 	.open_axi_master_ooo	= sec_open_axi_master_ooo,
722 	.err_info		= {
723 		.ce		= QM_BASE_CE,
724 		.nfe		= QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT |
725 				  QM_ACC_WB_NOT_READY_TIMEOUT,
726 		.fe		= 0,
727 		.ecc_2bits_mask	= SEC_CORE_INT_STATUS_M_ECC,
728 		.msi_wr_port	= BIT(0),
729 		.acpi_rst	= "SRST",
730 	}
731 };
732 
733 static int sec_pf_probe_init(struct sec_dev *sec)
734 {
735 	struct hisi_qm *qm = &sec->qm;
736 	int ret;
737 
738 	if (qm->ver == QM_HW_V1)
739 		qm->ctrl_qp_num = SEC_QUEUE_NUM_V1;
740 	else
741 		qm->ctrl_qp_num = SEC_QUEUE_NUM_V2;
742 
743 	qm->err_ini = &sec_err_ini;
744 
745 	ret = sec_set_user_domain_and_cache(qm);
746 	if (ret)
747 		return ret;
748 
749 	hisi_qm_dev_err_init(qm);
750 	sec_debug_regs_clear(qm);
751 
752 	return 0;
753 }
754 
755 static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
756 {
757 	int ret;
758 
759 	qm->pdev = pdev;
760 	qm->ver = pdev->revision;
761 	qm->sqe_size = SEC_SQE_SIZE;
762 	qm->dev_name = sec_name;
763 
764 	qm->fun_type = (pdev->device == SEC_PF_PCI_DEVICE_ID) ?
765 			QM_HW_PF : QM_HW_VF;
766 	if (qm->fun_type == QM_HW_PF) {
767 		qm->qp_base = SEC_PF_DEF_Q_BASE;
768 		qm->qp_num = pf_q_num;
769 		qm->debug.curr_qm_qp_num = pf_q_num;
770 		qm->qm_list = &sec_devices;
771 	} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
772 		/*
773 		 * have no way to get qm configure in VM in v1 hardware,
774 		 * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force
775 		 * to trigger only one VF in v1 hardware.
776 		 * v2 hardware has no such problem.
777 		 */
778 		qm->qp_base = SEC_PF_DEF_Q_NUM;
779 		qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM;
780 	}
781 
782 	/*
783 	 * WQ_HIGHPRI: SEC request must be low delayed,
784 	 * so need a high priority workqueue.
785 	 * WQ_UNBOUND: SEC task is likely with long
786 	 * running CPU intensive workloads.
787 	 */
788 	qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM |
789 				 WQ_UNBOUND, num_online_cpus(),
790 				 pci_name(qm->pdev));
791 	if (!qm->wq) {
792 		pci_err(qm->pdev, "fail to alloc workqueue\n");
793 		return -ENOMEM;
794 	}
795 
796 	ret = hisi_qm_init(qm);
797 	if (ret)
798 		destroy_workqueue(qm->wq);
799 
800 	return ret;
801 }
802 
803 static void sec_qm_uninit(struct hisi_qm *qm)
804 {
805 	hisi_qm_uninit(qm);
806 }
807 
808 static int sec_probe_init(struct sec_dev *sec)
809 {
810 	struct hisi_qm *qm = &sec->qm;
811 	int ret;
812 
813 	if (qm->fun_type == QM_HW_PF) {
814 		ret = sec_pf_probe_init(sec);
815 		if (ret)
816 			return ret;
817 	}
818 
819 	return 0;
820 }
821 
822 static void sec_probe_uninit(struct hisi_qm *qm)
823 {
824 	hisi_qm_dev_err_uninit(qm);
825 
826 	destroy_workqueue(qm->wq);
827 }
828 
829 static void sec_iommu_used_check(struct sec_dev *sec)
830 {
831 	struct iommu_domain *domain;
832 	struct device *dev = &sec->qm.pdev->dev;
833 
834 	domain = iommu_get_domain_for_dev(dev);
835 
836 	/* Check if iommu is used */
837 	sec->iommu_used = false;
838 	if (domain) {
839 		if (domain->type & __IOMMU_DOMAIN_PAGING)
840 			sec->iommu_used = true;
841 		dev_info(dev, "SMMU Opened, the iommu type = %u\n",
842 			domain->type);
843 	}
844 }
845 
846 static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)
847 {
848 	struct sec_dev *sec;
849 	struct hisi_qm *qm;
850 	int ret;
851 
852 	sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL);
853 	if (!sec)
854 		return -ENOMEM;
855 
856 	qm = &sec->qm;
857 	ret = sec_qm_init(qm, pdev);
858 	if (ret) {
859 		pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret);
860 		return ret;
861 	}
862 
863 	sec->ctx_q_num = ctx_q_num;
864 	sec_iommu_used_check(sec);
865 
866 	ret = sec_probe_init(sec);
867 	if (ret) {
868 		pci_err(pdev, "Failed to probe!\n");
869 		goto err_qm_uninit;
870 	}
871 
872 	ret = hisi_qm_start(qm);
873 	if (ret) {
874 		pci_err(pdev, "Failed to start sec qm!\n");
875 		goto err_probe_uninit;
876 	}
877 
878 	ret = sec_debugfs_init(qm);
879 	if (ret)
880 		pci_warn(pdev, "Failed to init debugfs!\n");
881 
882 	ret = hisi_qm_alg_register(qm, &sec_devices);
883 	if (ret < 0) {
884 		pr_err("Failed to register driver to crypto.\n");
885 		goto err_qm_stop;
886 	}
887 
888 	if (qm->fun_type == QM_HW_PF && vfs_num) {
889 		ret = hisi_qm_sriov_enable(pdev, vfs_num);
890 		if (ret < 0)
891 			goto err_alg_unregister;
892 	}
893 
894 	return 0;
895 
896 err_alg_unregister:
897 	hisi_qm_alg_unregister(qm, &sec_devices);
898 err_qm_stop:
899 	sec_debugfs_exit(qm);
900 	hisi_qm_stop(qm, QM_NORMAL);
901 err_probe_uninit:
902 	sec_probe_uninit(qm);
903 err_qm_uninit:
904 	sec_qm_uninit(qm);
905 	return ret;
906 }
907 
908 static void sec_remove(struct pci_dev *pdev)
909 {
910 	struct hisi_qm *qm = pci_get_drvdata(pdev);
911 
912 	hisi_qm_wait_task_finish(qm, &sec_devices);
913 	hisi_qm_alg_unregister(qm, &sec_devices);
914 	if (qm->fun_type == QM_HW_PF && qm->vfs_num)
915 		hisi_qm_sriov_disable(pdev, qm->is_frozen);
916 
917 	sec_debugfs_exit(qm);
918 
919 	(void)hisi_qm_stop(qm, QM_NORMAL);
920 
921 	if (qm->fun_type == QM_HW_PF)
922 		sec_debug_regs_clear(qm);
923 
924 	sec_probe_uninit(qm);
925 
926 	sec_qm_uninit(qm);
927 }
928 
929 static const struct pci_error_handlers sec_err_handler = {
930 	.error_detected = hisi_qm_dev_err_detected,
931 	.slot_reset	= hisi_qm_dev_slot_reset,
932 	.reset_prepare	= hisi_qm_reset_prepare,
933 	.reset_done	= hisi_qm_reset_done,
934 };
935 
936 static struct pci_driver sec_pci_driver = {
937 	.name = "hisi_sec2",
938 	.id_table = sec_dev_ids,
939 	.probe = sec_probe,
940 	.remove = sec_remove,
941 	.err_handler = &sec_err_handler,
942 	.sriov_configure = hisi_qm_sriov_configure,
943 	.shutdown = hisi_qm_dev_shutdown,
944 };
945 
946 static void sec_register_debugfs(void)
947 {
948 	if (!debugfs_initialized())
949 		return;
950 
951 	sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL);
952 }
953 
954 static void sec_unregister_debugfs(void)
955 {
956 	debugfs_remove_recursive(sec_debugfs_root);
957 }
958 
959 static int __init sec_init(void)
960 {
961 	int ret;
962 
963 	hisi_qm_init_list(&sec_devices);
964 	sec_register_debugfs();
965 
966 	ret = pci_register_driver(&sec_pci_driver);
967 	if (ret < 0) {
968 		sec_unregister_debugfs();
969 		pr_err("Failed to register pci driver.\n");
970 		return ret;
971 	}
972 
973 	return 0;
974 }
975 
976 static void __exit sec_exit(void)
977 {
978 	pci_unregister_driver(&sec_pci_driver);
979 	sec_unregister_debugfs();
980 }
981 
982 module_init(sec_init);
983 module_exit(sec_exit);
984 
985 MODULE_LICENSE("GPL v2");
986 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
987 MODULE_AUTHOR("Longfang Liu <liulongfang@huawei.com>");
988 MODULE_AUTHOR("Kai Ye <yekai13@huawei.com>");
989 MODULE_AUTHOR("Wei Zhang <zhangwei375@huawei.com>");
990 MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator");
991