1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <linux/acpi.h>
4 #include <linux/bitops.h>
5 #include <linux/debugfs.h>
6 #include <linux/init.h>
7 #include <linux/io.h>
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/seq_file.h>
13 #include <linux/topology.h>
14 #include <linux/uacce.h>
15 #include "zip.h"
16 
17 #define PCI_DEVICE_ID_HUAWEI_ZIP_PF	0xa250
18 
19 #define HZIP_QUEUE_NUM_V1		4096
20 
21 #define HZIP_CLOCK_GATE_CTRL		0x301004
22 #define HZIP_DECOMP_CHECK_ENABLE	BIT(16)
23 #define HZIP_FSM_MAX_CNT		0x301008
24 
25 #define HZIP_PORT_ARCA_CHE_0		0x301040
26 #define HZIP_PORT_ARCA_CHE_1		0x301044
27 #define HZIP_PORT_AWCA_CHE_0		0x301060
28 #define HZIP_PORT_AWCA_CHE_1		0x301064
29 #define HZIP_CACHE_ALL_EN		0xffffffff
30 
31 #define HZIP_BD_RUSER_32_63		0x301110
32 #define HZIP_SGL_RUSER_32_63		0x30111c
33 #define HZIP_DATA_RUSER_32_63		0x301128
34 #define HZIP_DATA_WUSER_32_63		0x301134
35 #define HZIP_BD_WUSER_32_63		0x301140
36 
37 #define HZIP_QM_IDEL_STATUS		0x3040e4
38 
39 #define HZIP_CORE_DFX_BASE		0x301000
40 #define HZIP_CLOCK_GATED_CONTL		0X301004
41 #define HZIP_CORE_DFX_COMP_0		0x302000
42 #define HZIP_CORE_DFX_COMP_1		0x303000
43 #define HZIP_CORE_DFX_DECOMP_0		0x304000
44 #define HZIP_CORE_DFX_DECOMP_1		0x305000
45 #define HZIP_CORE_DFX_DECOMP_2		0x306000
46 #define HZIP_CORE_DFX_DECOMP_3		0x307000
47 #define HZIP_CORE_DFX_DECOMP_4		0x308000
48 #define HZIP_CORE_DFX_DECOMP_5		0x309000
49 #define HZIP_CORE_REGS_BASE_LEN		0xB0
50 #define HZIP_CORE_REGS_DFX_LEN		0x28
51 
52 #define HZIP_CORE_INT_SOURCE		0x3010A0
53 #define HZIP_CORE_INT_MASK_REG		0x3010A4
54 #define HZIP_CORE_INT_SET		0x3010A8
55 #define HZIP_CORE_INT_STATUS		0x3010AC
56 #define HZIP_CORE_INT_STATUS_M_ECC	BIT(1)
57 #define HZIP_CORE_SRAM_ECC_ERR_INFO	0x301148
58 #define HZIP_CORE_INT_RAS_CE_ENB	0x301160
59 #define HZIP_CORE_INT_RAS_NFE_ENB	0x301164
60 #define HZIP_CORE_INT_RAS_FE_ENB        0x301168
61 #define HZIP_CORE_INT_RAS_FE_ENB_MASK	0x0
62 #define HZIP_OOO_SHUTDOWN_SEL		0x30120C
63 #define HZIP_SRAM_ECC_ERR_NUM_SHIFT	16
64 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT	24
65 #define HZIP_CORE_INT_MASK_ALL		GENMASK(12, 0)
66 #define HZIP_SQE_SIZE			128
67 #define HZIP_PF_DEF_Q_NUM		64
68 #define HZIP_PF_DEF_Q_BASE		0
69 
70 #define HZIP_SOFT_CTRL_CNT_CLR_CE	0x301000
71 #define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT	BIT(0)
72 #define HZIP_SOFT_CTRL_ZIP_CONTROL	0x30100C
73 #define HZIP_AXI_SHUTDOWN_ENABLE	BIT(14)
74 #define HZIP_WR_PORT			BIT(11)
75 
76 #define HZIP_ALG_ZLIB_BIT		GENMASK(1, 0)
77 #define HZIP_ALG_GZIP_BIT		GENMASK(3, 2)
78 #define HZIP_ALG_DEFLATE_BIT		GENMASK(5, 4)
79 #define HZIP_ALG_LZ77_BIT		GENMASK(7, 6)
80 
81 #define HZIP_BUF_SIZE			22
82 #define HZIP_SQE_MASK_OFFSET		64
83 #define HZIP_SQE_MASK_LEN		48
84 
85 #define HZIP_CNT_CLR_CE_EN		BIT(0)
86 #define HZIP_RO_CNT_CLR_CE_EN		BIT(2)
87 #define HZIP_RD_CNT_CLR_CE_EN		(HZIP_CNT_CLR_CE_EN | \
88 					 HZIP_RO_CNT_CLR_CE_EN)
89 
90 #define HZIP_PREFETCH_CFG		0x3011B0
91 #define HZIP_SVA_TRANS			0x3011C4
92 #define HZIP_PREFETCH_ENABLE		(~(BIT(26) | BIT(17) | BIT(0)))
93 #define HZIP_SVA_PREFETCH_DISABLE	BIT(26)
94 #define HZIP_SVA_DISABLE_READY		(BIT(26) | BIT(30))
95 #define HZIP_SHAPER_RATE_COMPRESS	750
96 #define HZIP_SHAPER_RATE_DECOMPRESS	140
97 #define HZIP_DELAY_1_US		1
98 #define HZIP_POLL_TIMEOUT_US	1000
99 
100 /* clock gating */
101 #define HZIP_PEH_CFG_AUTO_GATE		0x3011A8
102 #define HZIP_PEH_CFG_AUTO_GATE_EN	BIT(0)
103 #define HZIP_CORE_GATED_EN		GENMASK(15, 8)
104 #define HZIP_CORE_GATED_OOO_EN		BIT(29)
105 #define HZIP_CLOCK_GATED_EN		(HZIP_CORE_GATED_EN | \
106 					 HZIP_CORE_GATED_OOO_EN)
107 
108 /* zip comp high performance */
109 #define HZIP_HIGH_PERF_OFFSET		0x301208
110 
111 enum {
112 	HZIP_HIGH_COMP_RATE,
113 	HZIP_HIGH_COMP_PERF,
114 };
115 
116 static const char hisi_zip_name[] = "hisi_zip";
117 static struct dentry *hzip_debugfs_root;
118 
119 struct hisi_zip_hw_error {
120 	u32 int_msk;
121 	const char *msg;
122 };
123 
124 struct zip_dfx_item {
125 	const char *name;
126 	u32 offset;
127 };
128 
129 static const struct qm_dev_alg zip_dev_algs[] = { {
130 		.alg_msk = HZIP_ALG_ZLIB_BIT,
131 		.alg = "zlib\n",
132 	}, {
133 		.alg_msk = HZIP_ALG_GZIP_BIT,
134 		.alg = "gzip\n",
135 	}, {
136 		.alg_msk = HZIP_ALG_DEFLATE_BIT,
137 		.alg = "deflate\n",
138 	}, {
139 		.alg_msk = HZIP_ALG_LZ77_BIT,
140 		.alg = "lz77_zstd\n",
141 	},
142 };
143 
144 static struct hisi_qm_list zip_devices = {
145 	.register_to_crypto	= hisi_zip_register_to_crypto,
146 	.unregister_from_crypto	= hisi_zip_unregister_from_crypto,
147 };
148 
149 static struct zip_dfx_item zip_dfx_files[] = {
150 	{"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)},
151 	{"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)},
152 	{"send_busy_cnt", offsetof(struct hisi_zip_dfx, send_busy_cnt)},
153 	{"err_bd_cnt", offsetof(struct hisi_zip_dfx, err_bd_cnt)},
154 };
155 
156 static const struct hisi_zip_hw_error zip_hw_error[] = {
157 	{ .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" },
158 	{ .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" },
159 	{ .int_msk = BIT(2), .msg = "zip_axi_rresp_err" },
160 	{ .int_msk = BIT(3), .msg = "zip_axi_bresp_err" },
161 	{ .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" },
162 	{ .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" },
163 	{ .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" },
164 	{ .int_msk = BIT(7), .msg = "zip_pre_in_data_err" },
165 	{ .int_msk = BIT(8), .msg = "zip_com_inf_err" },
166 	{ .int_msk = BIT(9), .msg = "zip_enc_inf_err" },
167 	{ .int_msk = BIT(10), .msg = "zip_pre_out_err" },
168 	{ .int_msk = BIT(11), .msg = "zip_axi_poison_err" },
169 	{ .int_msk = BIT(12), .msg = "zip_sva_err" },
170 	{ /* sentinel */ }
171 };
172 
173 enum ctrl_debug_file_index {
174 	HZIP_CLEAR_ENABLE,
175 	HZIP_DEBUG_FILE_NUM,
176 };
177 
178 static const char * const ctrl_debug_file_name[] = {
179 	[HZIP_CLEAR_ENABLE] = "clear_enable",
180 };
181 
182 struct ctrl_debug_file {
183 	enum ctrl_debug_file_index index;
184 	spinlock_t lock;
185 	struct hisi_zip_ctrl *ctrl;
186 };
187 
188 /*
189  * One ZIP controller has one PF and multiple VFs, some global configurations
190  * which PF has need this structure.
191  *
192  * Just relevant for PF.
193  */
194 struct hisi_zip_ctrl {
195 	struct hisi_zip *hisi_zip;
196 	struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM];
197 };
198 
199 enum zip_cap_type {
200 	ZIP_QM_NFE_MASK_CAP = 0x0,
201 	ZIP_QM_RESET_MASK_CAP,
202 	ZIP_QM_OOO_SHUTDOWN_MASK_CAP,
203 	ZIP_QM_CE_MASK_CAP,
204 	ZIP_NFE_MASK_CAP,
205 	ZIP_RESET_MASK_CAP,
206 	ZIP_OOO_SHUTDOWN_MASK_CAP,
207 	ZIP_CE_MASK_CAP,
208 	ZIP_CLUSTER_NUM_CAP,
209 	ZIP_CORE_TYPE_NUM_CAP,
210 	ZIP_CORE_NUM_CAP,
211 	ZIP_CLUSTER_COMP_NUM_CAP,
212 	ZIP_CLUSTER_DECOMP_NUM_CAP,
213 	ZIP_DECOMP_ENABLE_BITMAP,
214 	ZIP_COMP_ENABLE_BITMAP,
215 	ZIP_DRV_ALG_BITMAP,
216 	ZIP_DEV_ALG_BITMAP,
217 	ZIP_CORE1_ALG_BITMAP,
218 	ZIP_CORE2_ALG_BITMAP,
219 	ZIP_CORE3_ALG_BITMAP,
220 	ZIP_CORE4_ALG_BITMAP,
221 	ZIP_CORE5_ALG_BITMAP,
222 	ZIP_CAP_MAX
223 };
224 
225 static struct hisi_qm_cap_info zip_basic_cap_info[] = {
226 	{ZIP_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C57, 0x7C77},
227 	{ZIP_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC57, 0x6C77},
228 	{ZIP_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77},
229 	{ZIP_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8},
230 	{ZIP_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x1FFE},
231 	{ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE},
232 	{ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE},
233 	{ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1},
234 	{ZIP_CLUSTER_NUM_CAP, 0x313C, 28, GENMASK(3, 0), 0x1, 0x1, 0x1},
235 	{ZIP_CORE_TYPE_NUM_CAP, 0x313C, 24, GENMASK(3, 0), 0x2, 0x2, 0x2},
236 	{ZIP_CORE_NUM_CAP, 0x313C, 16, GENMASK(7, 0), 0x8, 0x8, 0x5},
237 	{ZIP_CLUSTER_COMP_NUM_CAP, 0x313C, 8, GENMASK(7, 0), 0x2, 0x2, 0x2},
238 	{ZIP_CLUSTER_DECOMP_NUM_CAP, 0x313C, 0, GENMASK(7, 0), 0x6, 0x6, 0x3},
239 	{ZIP_DECOMP_ENABLE_BITMAP, 0x3140, 16, GENMASK(15, 0), 0xFC, 0xFC, 0x1C},
240 	{ZIP_COMP_ENABLE_BITMAP, 0x3140, 0, GENMASK(15, 0), 0x3, 0x3, 0x3},
241 	{ZIP_DRV_ALG_BITMAP, 0x3144, 0, GENMASK(31, 0), 0xF, 0xF, 0xF},
242 	{ZIP_DEV_ALG_BITMAP, 0x3148, 0, GENMASK(31, 0), 0xF, 0xF, 0xFF},
243 	{ZIP_CORE1_ALG_BITMAP, 0x314C, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},
244 	{ZIP_CORE2_ALG_BITMAP, 0x3150, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},
245 	{ZIP_CORE3_ALG_BITMAP, 0x3154, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
246 	{ZIP_CORE4_ALG_BITMAP, 0x3158, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
247 	{ZIP_CORE5_ALG_BITMAP, 0x315C, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},
248 	{ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0}
249 };
250 
251 enum zip_pre_store_cap_idx {
252 	ZIP_CORE_NUM_CAP_IDX = 0x0,
253 	ZIP_CLUSTER_COMP_NUM_CAP_IDX,
254 	ZIP_CLUSTER_DECOMP_NUM_CAP_IDX,
255 	ZIP_DECOMP_ENABLE_BITMAP_IDX,
256 	ZIP_COMP_ENABLE_BITMAP_IDX,
257 	ZIP_DRV_ALG_BITMAP_IDX,
258 	ZIP_DEV_ALG_BITMAP_IDX,
259 };
260 
261 static const u32 zip_pre_store_caps[] = {
262 	ZIP_CORE_NUM_CAP,
263 	ZIP_CLUSTER_COMP_NUM_CAP,
264 	ZIP_CLUSTER_DECOMP_NUM_CAP,
265 	ZIP_DECOMP_ENABLE_BITMAP,
266 	ZIP_COMP_ENABLE_BITMAP,
267 	ZIP_DRV_ALG_BITMAP,
268 	ZIP_DEV_ALG_BITMAP,
269 };
270 
271 enum {
272 	HZIP_COMP_CORE0,
273 	HZIP_COMP_CORE1,
274 	HZIP_DECOMP_CORE0,
275 	HZIP_DECOMP_CORE1,
276 	HZIP_DECOMP_CORE2,
277 	HZIP_DECOMP_CORE3,
278 	HZIP_DECOMP_CORE4,
279 	HZIP_DECOMP_CORE5,
280 };
281 
282 static const u64 core_offsets[] = {
283 	[HZIP_COMP_CORE0]   = 0x302000,
284 	[HZIP_COMP_CORE1]   = 0x303000,
285 	[HZIP_DECOMP_CORE0] = 0x304000,
286 	[HZIP_DECOMP_CORE1] = 0x305000,
287 	[HZIP_DECOMP_CORE2] = 0x306000,
288 	[HZIP_DECOMP_CORE3] = 0x307000,
289 	[HZIP_DECOMP_CORE4] = 0x308000,
290 	[HZIP_DECOMP_CORE5] = 0x309000,
291 };
292 
293 static const struct debugfs_reg32 hzip_dfx_regs[] = {
294 	{"HZIP_GET_BD_NUM                ",  0x00ull},
295 	{"HZIP_GET_RIGHT_BD              ",  0x04ull},
296 	{"HZIP_GET_ERROR_BD              ",  0x08ull},
297 	{"HZIP_DONE_BD_NUM               ",  0x0cull},
298 	{"HZIP_WORK_CYCLE                ",  0x10ull},
299 	{"HZIP_IDLE_CYCLE                ",  0x18ull},
300 	{"HZIP_MAX_DELAY                 ",  0x20ull},
301 	{"HZIP_MIN_DELAY                 ",  0x24ull},
302 	{"HZIP_AVG_DELAY                 ",  0x28ull},
303 	{"HZIP_MEM_VISIBLE_DATA          ",  0x30ull},
304 	{"HZIP_MEM_VISIBLE_ADDR          ",  0x34ull},
305 	{"HZIP_CONSUMED_BYTE             ",  0x38ull},
306 	{"HZIP_PRODUCED_BYTE             ",  0x40ull},
307 	{"HZIP_COMP_INF                  ",  0x70ull},
308 	{"HZIP_PRE_OUT                   ",  0x78ull},
309 	{"HZIP_BD_RD                     ",  0x7cull},
310 	{"HZIP_BD_WR                     ",  0x80ull},
311 	{"HZIP_GET_BD_AXI_ERR_NUM        ",  0x84ull},
312 	{"HZIP_GET_BD_PARSE_ERR_NUM      ",  0x88ull},
313 	{"HZIP_ADD_BD_AXI_ERR_NUM        ",  0x8cull},
314 	{"HZIP_DECOMP_STF_RELOAD_CURR_ST ",  0x94ull},
315 	{"HZIP_DECOMP_LZ77_CURR_ST       ",  0x9cull},
316 };
317 
318 static const struct debugfs_reg32 hzip_com_dfx_regs[] = {
319 	{"HZIP_CLOCK_GATE_CTRL           ",  0x301004},
320 	{"HZIP_CORE_INT_RAS_CE_ENB       ",  0x301160},
321 	{"HZIP_CORE_INT_RAS_NFE_ENB      ",  0x301164},
322 	{"HZIP_CORE_INT_RAS_FE_ENB       ",  0x301168},
323 	{"HZIP_UNCOM_ERR_RAS_CTRL        ",  0x30116C},
324 };
325 
326 static const struct debugfs_reg32 hzip_dump_dfx_regs[] = {
327 	{"HZIP_GET_BD_NUM                ",  0x00ull},
328 	{"HZIP_GET_RIGHT_BD              ",  0x04ull},
329 	{"HZIP_GET_ERROR_BD              ",  0x08ull},
330 	{"HZIP_DONE_BD_NUM               ",  0x0cull},
331 	{"HZIP_MAX_DELAY                 ",  0x20ull},
332 };
333 
334 /* define the ZIP's dfx regs region and region length */
335 static struct dfx_diff_registers hzip_diff_regs[] = {
336 	{
337 		.reg_offset = HZIP_CORE_DFX_BASE,
338 		.reg_len = HZIP_CORE_REGS_BASE_LEN,
339 	}, {
340 		.reg_offset = HZIP_CORE_DFX_COMP_0,
341 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
342 	}, {
343 		.reg_offset = HZIP_CORE_DFX_COMP_1,
344 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
345 	}, {
346 		.reg_offset = HZIP_CORE_DFX_DECOMP_0,
347 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
348 	}, {
349 		.reg_offset = HZIP_CORE_DFX_DECOMP_1,
350 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
351 	}, {
352 		.reg_offset = HZIP_CORE_DFX_DECOMP_2,
353 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
354 	}, {
355 		.reg_offset = HZIP_CORE_DFX_DECOMP_3,
356 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
357 	}, {
358 		.reg_offset = HZIP_CORE_DFX_DECOMP_4,
359 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
360 	}, {
361 		.reg_offset = HZIP_CORE_DFX_DECOMP_5,
362 		.reg_len = HZIP_CORE_REGS_DFX_LEN,
363 	},
364 };
365 
hzip_diff_regs_show(struct seq_file * s,void * unused)366 static int hzip_diff_regs_show(struct seq_file *s, void *unused)
367 {
368 	struct hisi_qm *qm = s->private;
369 
370 	hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,
371 					ARRAY_SIZE(hzip_diff_regs));
372 
373 	return 0;
374 }
375 DEFINE_SHOW_ATTRIBUTE(hzip_diff_regs);
376 
perf_mode_set(const char * val,const struct kernel_param * kp)377 static int perf_mode_set(const char *val, const struct kernel_param *kp)
378 {
379 	int ret;
380 	u32 n;
381 
382 	if (!val)
383 		return -EINVAL;
384 
385 	ret = kstrtou32(val, 10, &n);
386 	if (ret != 0 || (n != HZIP_HIGH_COMP_PERF &&
387 			 n != HZIP_HIGH_COMP_RATE))
388 		return -EINVAL;
389 
390 	return param_set_int(val, kp);
391 }
392 
393 static const struct kernel_param_ops zip_com_perf_ops = {
394 	.set = perf_mode_set,
395 	.get = param_get_int,
396 };
397 
398 /*
399  * perf_mode = 0 means enable high compression rate mode,
400  * perf_mode = 1 means enable high compression performance mode.
401  * These two modes only apply to the compression direction.
402  */
403 static u32 perf_mode = HZIP_HIGH_COMP_RATE;
404 module_param_cb(perf_mode, &zip_com_perf_ops, &perf_mode, 0444);
405 MODULE_PARM_DESC(perf_mode, "ZIP high perf mode 0(default), 1(enable)");
406 
407 static const struct kernel_param_ops zip_uacce_mode_ops = {
408 	.set = uacce_mode_set,
409 	.get = param_get_int,
410 };
411 
412 /*
413  * uacce_mode = 0 means zip only register to crypto,
414  * uacce_mode = 1 means zip both register to crypto and uacce.
415  */
416 static u32 uacce_mode = UACCE_MODE_NOUACCE;
417 module_param_cb(uacce_mode, &zip_uacce_mode_ops, &uacce_mode, 0444);
418 MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);
419 
420 static bool pf_q_num_flag;
pf_q_num_set(const char * val,const struct kernel_param * kp)421 static int pf_q_num_set(const char *val, const struct kernel_param *kp)
422 {
423 	pf_q_num_flag = true;
424 
425 	return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_ZIP_PF);
426 }
427 
428 static const struct kernel_param_ops pf_q_num_ops = {
429 	.set = pf_q_num_set,
430 	.get = param_get_int,
431 };
432 
433 static u32 pf_q_num = HZIP_PF_DEF_Q_NUM;
434 module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444);
435 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
436 
437 static const struct kernel_param_ops vfs_num_ops = {
438 	.set = vfs_num_set,
439 	.get = param_get_int,
440 };
441 
442 static u32 vfs_num;
443 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
444 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
445 
446 static const struct pci_device_id hisi_zip_dev_ids[] = {
447 	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_PF) },
448 	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },
449 	{ 0, }
450 };
451 MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids);
452 
zip_create_qps(struct hisi_qp ** qps,int qp_num,int node)453 int zip_create_qps(struct hisi_qp **qps, int qp_num, int node)
454 {
455 	if (node == NUMA_NO_NODE)
456 		node = cpu_to_node(smp_processor_id());
457 
458 	return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps);
459 }
460 
hisi_zip_alg_support(struct hisi_qm * qm,u32 alg)461 bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg)
462 {
463 	u32 cap_val;
464 
465 	cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_IDX].cap_val;
466 	if ((alg & cap_val) == alg)
467 		return true;
468 
469 	return false;
470 }
471 
hisi_zip_set_high_perf(struct hisi_qm * qm)472 static int hisi_zip_set_high_perf(struct hisi_qm *qm)
473 {
474 	u32 val;
475 	int ret;
476 
477 	val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET);
478 	if (perf_mode == HZIP_HIGH_COMP_PERF)
479 		val |= HZIP_HIGH_COMP_PERF;
480 	else
481 		val &= ~HZIP_HIGH_COMP_PERF;
482 
483 	/* Set perf mode */
484 	writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET);
485 	ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_HIGH_PERF_OFFSET,
486 					 val, val == perf_mode, HZIP_DELAY_1_US,
487 					 HZIP_POLL_TIMEOUT_US);
488 	if (ret)
489 		pci_err(qm->pdev, "failed to set perf mode\n");
490 
491 	return ret;
492 }
493 
hisi_zip_open_sva_prefetch(struct hisi_qm * qm)494 static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm)
495 {
496 	u32 val;
497 	int ret;
498 
499 	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
500 		return;
501 
502 	/* Enable prefetch */
503 	val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
504 	val &= HZIP_PREFETCH_ENABLE;
505 	writel(val, qm->io_base + HZIP_PREFETCH_CFG);
506 
507 	ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG,
508 					 val, !(val & HZIP_SVA_PREFETCH_DISABLE),
509 					 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
510 	if (ret)
511 		pci_err(qm->pdev, "failed to open sva prefetch\n");
512 }
513 
hisi_zip_close_sva_prefetch(struct hisi_qm * qm)514 static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm)
515 {
516 	u32 val;
517 	int ret;
518 
519 	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
520 		return;
521 
522 	val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);
523 	val |= HZIP_SVA_PREFETCH_DISABLE;
524 	writel(val, qm->io_base + HZIP_PREFETCH_CFG);
525 
526 	ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS,
527 					 val, !(val & HZIP_SVA_DISABLE_READY),
528 					 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);
529 	if (ret)
530 		pci_err(qm->pdev, "failed to close sva prefetch\n");
531 }
532 
hisi_zip_enable_clock_gate(struct hisi_qm * qm)533 static void hisi_zip_enable_clock_gate(struct hisi_qm *qm)
534 {
535 	u32 val;
536 
537 	if (qm->ver < QM_HW_V3)
538 		return;
539 
540 	val = readl(qm->io_base + HZIP_CLOCK_GATE_CTRL);
541 	val |= HZIP_CLOCK_GATED_EN;
542 	writel(val, qm->io_base + HZIP_CLOCK_GATE_CTRL);
543 
544 	val = readl(qm->io_base + HZIP_PEH_CFG_AUTO_GATE);
545 	val |= HZIP_PEH_CFG_AUTO_GATE_EN;
546 	writel(val, qm->io_base + HZIP_PEH_CFG_AUTO_GATE);
547 }
548 
hisi_zip_set_user_domain_and_cache(struct hisi_qm * qm)549 static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
550 {
551 	void __iomem *base = qm->io_base;
552 	u32 dcomp_bm, comp_bm;
553 
554 	/* qm user domain */
555 	writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1);
556 	writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE);
557 	writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1);
558 	writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE);
559 	writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE);
560 
561 	/* qm cache */
562 	writel(AXI_M_CFG, base + QM_AXI_M_CFG);
563 	writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE);
564 
565 	/* disable FLR triggered by BME(bus master enable) */
566 	writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG);
567 	writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE);
568 
569 	/* cache */
570 	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0);
571 	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1);
572 	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0);
573 	writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1);
574 
575 	/* user domain configurations */
576 	writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
577 	writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
578 
579 	if (qm->use_sva && qm->ver == QM_HW_V2) {
580 		writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);
581 		writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);
582 		writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63);
583 	} else {
584 		writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);
585 		writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);
586 		writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
587 	}
588 
589 	/* let's open all compression/decompression cores */
590 	dcomp_bm = qm->cap_tables.dev_cap_table[ZIP_DECOMP_ENABLE_BITMAP_IDX].cap_val;
591 	comp_bm = qm->cap_tables.dev_cap_table[ZIP_COMP_ENABLE_BITMAP_IDX].cap_val;
592 	writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL);
593 
594 	/* enable sqc,cqc writeback */
595 	writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
596 	       CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
597 	       FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL);
598 
599 	hisi_zip_enable_clock_gate(qm);
600 
601 	return 0;
602 }
603 
hisi_zip_master_ooo_ctrl(struct hisi_qm * qm,bool enable)604 static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
605 {
606 	u32 val1, val2;
607 
608 	val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
609 	if (enable) {
610 		val1 |= HZIP_AXI_SHUTDOWN_ENABLE;
611 		val2 = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
612 				ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
613 	} else {
614 		val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE;
615 		val2 = 0x0;
616 	}
617 
618 	if (qm->ver > QM_HW_V2)
619 		writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL);
620 
621 	writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
622 }
623 
hisi_zip_hw_error_enable(struct hisi_qm * qm)624 static void hisi_zip_hw_error_enable(struct hisi_qm *qm)
625 {
626 	u32 nfe, ce;
627 
628 	if (qm->ver == QM_HW_V1) {
629 		writel(HZIP_CORE_INT_MASK_ALL,
630 		       qm->io_base + HZIP_CORE_INT_MASK_REG);
631 		dev_info(&qm->pdev->dev, "Does not support hw error handle\n");
632 		return;
633 	}
634 
635 	nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);
636 	ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver);
637 
638 	/* clear ZIP hw error source if having */
639 	writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_SOURCE);
640 
641 	/* configure error type */
642 	writel(ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB);
643 	writel(HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB);
644 	writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
645 
646 	hisi_zip_master_ooo_ctrl(qm, true);
647 
648 	/* enable ZIP hw error interrupts */
649 	writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG);
650 }
651 
hisi_zip_hw_error_disable(struct hisi_qm * qm)652 static void hisi_zip_hw_error_disable(struct hisi_qm *qm)
653 {
654 	u32 nfe, ce;
655 
656 	/* disable ZIP hw error interrupts */
657 	nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);
658 	ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver);
659 	writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG);
660 
661 	hisi_zip_master_ooo_ctrl(qm, false);
662 }
663 
file_to_qm(struct ctrl_debug_file * file)664 static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)
665 {
666 	struct hisi_zip *hisi_zip = file->ctrl->hisi_zip;
667 
668 	return &hisi_zip->qm;
669 }
670 
clear_enable_read(struct hisi_qm * qm)671 static u32 clear_enable_read(struct hisi_qm *qm)
672 {
673 	return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
674 		     HZIP_SOFT_CTRL_CNT_CLR_CE_BIT;
675 }
676 
clear_enable_write(struct hisi_qm * qm,u32 val)677 static int clear_enable_write(struct hisi_qm *qm, u32 val)
678 {
679 	u32 tmp;
680 
681 	if (val != 1 && val != 0)
682 		return -EINVAL;
683 
684 	tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
685 	       ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val;
686 	writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
687 
688 	return  0;
689 }
690 
hisi_zip_ctrl_debug_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)691 static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf,
692 					size_t count, loff_t *pos)
693 {
694 	struct ctrl_debug_file *file = filp->private_data;
695 	struct hisi_qm *qm = file_to_qm(file);
696 	char tbuf[HZIP_BUF_SIZE];
697 	u32 val;
698 	int ret;
699 
700 	ret = hisi_qm_get_dfx_access(qm);
701 	if (ret)
702 		return ret;
703 
704 	spin_lock_irq(&file->lock);
705 	switch (file->index) {
706 	case HZIP_CLEAR_ENABLE:
707 		val = clear_enable_read(qm);
708 		break;
709 	default:
710 		goto err_input;
711 	}
712 	spin_unlock_irq(&file->lock);
713 
714 	hisi_qm_put_dfx_access(qm);
715 	ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val);
716 	return simple_read_from_buffer(buf, count, pos, tbuf, ret);
717 
718 err_input:
719 	spin_unlock_irq(&file->lock);
720 	hisi_qm_put_dfx_access(qm);
721 	return -EINVAL;
722 }
723 
hisi_zip_ctrl_debug_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)724 static ssize_t hisi_zip_ctrl_debug_write(struct file *filp,
725 					 const char __user *buf,
726 					 size_t count, loff_t *pos)
727 {
728 	struct ctrl_debug_file *file = filp->private_data;
729 	struct hisi_qm *qm = file_to_qm(file);
730 	char tbuf[HZIP_BUF_SIZE];
731 	unsigned long val;
732 	int len, ret;
733 
734 	if (*pos != 0)
735 		return 0;
736 
737 	if (count >= HZIP_BUF_SIZE)
738 		return -ENOSPC;
739 
740 	len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count);
741 	if (len < 0)
742 		return len;
743 
744 	tbuf[len] = '\0';
745 	ret = kstrtoul(tbuf, 0, &val);
746 	if (ret)
747 		return ret;
748 
749 	ret = hisi_qm_get_dfx_access(qm);
750 	if (ret)
751 		return ret;
752 
753 	spin_lock_irq(&file->lock);
754 	switch (file->index) {
755 	case HZIP_CLEAR_ENABLE:
756 		ret = clear_enable_write(qm, val);
757 		if (ret)
758 			goto err_input;
759 		break;
760 	default:
761 		ret = -EINVAL;
762 		goto err_input;
763 	}
764 
765 	ret = count;
766 
767 err_input:
768 	spin_unlock_irq(&file->lock);
769 	hisi_qm_put_dfx_access(qm);
770 	return ret;
771 }
772 
773 static const struct file_operations ctrl_debug_fops = {
774 	.owner = THIS_MODULE,
775 	.open = simple_open,
776 	.read = hisi_zip_ctrl_debug_read,
777 	.write = hisi_zip_ctrl_debug_write,
778 };
779 
zip_debugfs_atomic64_set(void * data,u64 val)780 static int zip_debugfs_atomic64_set(void *data, u64 val)
781 {
782 	if (val)
783 		return -EINVAL;
784 
785 	atomic64_set((atomic64_t *)data, 0);
786 
787 	return 0;
788 }
789 
zip_debugfs_atomic64_get(void * data,u64 * val)790 static int zip_debugfs_atomic64_get(void *data, u64 *val)
791 {
792 	*val = atomic64_read((atomic64_t *)data);
793 
794 	return 0;
795 }
796 
797 DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get,
798 			 zip_debugfs_atomic64_set, "%llu\n");
799 
hisi_zip_regs_show(struct seq_file * s,void * unused)800 static int hisi_zip_regs_show(struct seq_file *s, void *unused)
801 {
802 	hisi_qm_regs_dump(s, s->private);
803 
804 	return 0;
805 }
806 
807 DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs);
808 
hisi_zip_core_debug_init(struct hisi_qm * qm)809 static int hisi_zip_core_debug_init(struct hisi_qm *qm)
810 {
811 	u32 zip_core_num, zip_comp_core_num;
812 	struct device *dev = &qm->pdev->dev;
813 	struct debugfs_regset32 *regset;
814 	struct dentry *tmp_d;
815 	char buf[HZIP_BUF_SIZE];
816 	int i;
817 
818 	zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;
819 	zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val;
820 
821 	for (i = 0; i < zip_core_num; i++) {
822 		if (i < zip_comp_core_num)
823 			scnprintf(buf, sizeof(buf), "comp_core%d", i);
824 		else
825 			scnprintf(buf, sizeof(buf), "decomp_core%d",
826 				  i - zip_comp_core_num);
827 
828 		regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
829 		if (!regset)
830 			return -ENOENT;
831 
832 		regset->regs = hzip_dfx_regs;
833 		regset->nregs = ARRAY_SIZE(hzip_dfx_regs);
834 		regset->base = qm->io_base + core_offsets[i];
835 		regset->dev = dev;
836 
837 		tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);
838 		debugfs_create_file("regs", 0444, tmp_d, regset,
839 				    &hisi_zip_regs_fops);
840 	}
841 
842 	return 0;
843 }
844 
hisi_zip_dfx_debug_init(struct hisi_qm * qm)845 static void hisi_zip_dfx_debug_init(struct hisi_qm *qm)
846 {
847 	struct dfx_diff_registers *hzip_regs = qm->debug.acc_diff_regs;
848 	struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
849 	struct hisi_zip_dfx *dfx = &zip->dfx;
850 	struct dentry *tmp_dir;
851 	void *data;
852 	int i;
853 
854 	tmp_dir = debugfs_create_dir("zip_dfx", qm->debug.debug_root);
855 	for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) {
856 		data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset);
857 		debugfs_create_file(zip_dfx_files[i].name,
858 				    0644, tmp_dir, data,
859 				    &zip_atomic64_ops);
860 	}
861 
862 	if (qm->fun_type == QM_HW_PF && hzip_regs)
863 		debugfs_create_file("diff_regs", 0444, tmp_dir,
864 				      qm, &hzip_diff_regs_fops);
865 }
866 
hisi_zip_ctrl_debug_init(struct hisi_qm * qm)867 static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm)
868 {
869 	struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
870 	int i;
871 
872 	for (i = HZIP_CLEAR_ENABLE; i < HZIP_DEBUG_FILE_NUM; i++) {
873 		spin_lock_init(&zip->ctrl->files[i].lock);
874 		zip->ctrl->files[i].ctrl = zip->ctrl;
875 		zip->ctrl->files[i].index = i;
876 
877 		debugfs_create_file(ctrl_debug_file_name[i], 0600,
878 				    qm->debug.debug_root,
879 				    zip->ctrl->files + i,
880 				    &ctrl_debug_fops);
881 	}
882 
883 	return hisi_zip_core_debug_init(qm);
884 }
885 
hisi_zip_debugfs_init(struct hisi_qm * qm)886 static int hisi_zip_debugfs_init(struct hisi_qm *qm)
887 {
888 	struct device *dev = &qm->pdev->dev;
889 	struct dentry *dev_d;
890 	int ret;
891 
892 	dev_d = debugfs_create_dir(dev_name(dev), hzip_debugfs_root);
893 
894 	qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET;
895 	qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN;
896 	qm->debug.debug_root = dev_d;
897 	ret = hisi_qm_regs_debugfs_init(qm, hzip_diff_regs, ARRAY_SIZE(hzip_diff_regs));
898 	if (ret) {
899 		dev_warn(dev, "Failed to init ZIP diff regs!\n");
900 		goto debugfs_remove;
901 	}
902 
903 	hisi_qm_debug_init(qm);
904 
905 	if (qm->fun_type == QM_HW_PF) {
906 		ret = hisi_zip_ctrl_debug_init(qm);
907 		if (ret)
908 			goto failed_to_create;
909 	}
910 
911 	hisi_zip_dfx_debug_init(qm);
912 
913 	return 0;
914 
915 failed_to_create:
916 	hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));
917 debugfs_remove:
918 	debugfs_remove_recursive(hzip_debugfs_root);
919 	return ret;
920 }
921 
922 /* hisi_zip_debug_regs_clear() - clear the zip debug regs */
hisi_zip_debug_regs_clear(struct hisi_qm * qm)923 static void hisi_zip_debug_regs_clear(struct hisi_qm *qm)
924 {
925 	int i, j;
926 
927 	/* enable register read_clear bit */
928 	writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
929 	for (i = 0; i < ARRAY_SIZE(core_offsets); i++)
930 		for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++)
931 			readl(qm->io_base + core_offsets[i] +
932 			      hzip_dfx_regs[j].offset);
933 
934 	/* disable register read_clear bit */
935 	writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
936 
937 	hisi_qm_debug_regs_clear(qm);
938 }
939 
hisi_zip_debugfs_exit(struct hisi_qm * qm)940 static void hisi_zip_debugfs_exit(struct hisi_qm *qm)
941 {
942 	hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));
943 
944 	debugfs_remove_recursive(qm->debug.debug_root);
945 
946 	if (qm->fun_type == QM_HW_PF) {
947 		hisi_zip_debug_regs_clear(qm);
948 		qm->debug.curr_qm_qp_num = 0;
949 	}
950 }
951 
hisi_zip_show_last_regs_init(struct hisi_qm * qm)952 static int hisi_zip_show_last_regs_init(struct hisi_qm *qm)
953 {
954 	int core_dfx_regs_num =  ARRAY_SIZE(hzip_dump_dfx_regs);
955 	int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);
956 	struct qm_debug *debug = &qm->debug;
957 	void __iomem *io_base;
958 	u32 zip_core_num;
959 	int i, j, idx;
960 
961 	zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;
962 
963 	debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num,
964 				    sizeof(unsigned int), GFP_KERNEL);
965 	if (!debug->last_words)
966 		return -ENOMEM;
967 
968 	for (i = 0; i < com_dfx_regs_num; i++) {
969 		io_base = qm->io_base + hzip_com_dfx_regs[i].offset;
970 		debug->last_words[i] = readl_relaxed(io_base);
971 	}
972 
973 	for (i = 0; i < zip_core_num; i++) {
974 		io_base = qm->io_base + core_offsets[i];
975 		for (j = 0; j < core_dfx_regs_num; j++) {
976 			idx = com_dfx_regs_num + i * core_dfx_regs_num + j;
977 			debug->last_words[idx] = readl_relaxed(
978 				io_base + hzip_dump_dfx_regs[j].offset);
979 		}
980 	}
981 
982 	return 0;
983 }
984 
hisi_zip_show_last_regs_uninit(struct hisi_qm * qm)985 static void hisi_zip_show_last_regs_uninit(struct hisi_qm *qm)
986 {
987 	struct qm_debug *debug = &qm->debug;
988 
989 	if (qm->fun_type == QM_HW_VF || !debug->last_words)
990 		return;
991 
992 	kfree(debug->last_words);
993 	debug->last_words = NULL;
994 }
995 
hisi_zip_show_last_dfx_regs(struct hisi_qm * qm)996 static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm)
997 {
998 	int core_dfx_regs_num =  ARRAY_SIZE(hzip_dump_dfx_regs);
999 	int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);
1000 	u32 zip_core_num, zip_comp_core_num;
1001 	struct qm_debug *debug = &qm->debug;
1002 	char buf[HZIP_BUF_SIZE];
1003 	void __iomem *base;
1004 	int i, j, idx;
1005 	u32 val;
1006 
1007 	if (qm->fun_type == QM_HW_VF || !debug->last_words)
1008 		return;
1009 
1010 	for (i = 0; i < com_dfx_regs_num; i++) {
1011 		val = readl_relaxed(qm->io_base + hzip_com_dfx_regs[i].offset);
1012 		if (debug->last_words[i] != val)
1013 			pci_info(qm->pdev, "com_dfx: %s \t= 0x%08x => 0x%08x\n",
1014 				 hzip_com_dfx_regs[i].name, debug->last_words[i], val);
1015 	}
1016 
1017 	zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;
1018 	zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val;
1019 
1020 	for (i = 0; i < zip_core_num; i++) {
1021 		if (i < zip_comp_core_num)
1022 			scnprintf(buf, sizeof(buf), "Comp_core-%d", i);
1023 		else
1024 			scnprintf(buf, sizeof(buf), "Decomp_core-%d",
1025 				  i - zip_comp_core_num);
1026 		base = qm->io_base + core_offsets[i];
1027 
1028 		pci_info(qm->pdev, "==>%s:\n", buf);
1029 		/* dump last word for dfx regs during control resetting */
1030 		for (j = 0; j < core_dfx_regs_num; j++) {
1031 			idx = com_dfx_regs_num + i * core_dfx_regs_num + j;
1032 			val = readl_relaxed(base + hzip_dump_dfx_regs[j].offset);
1033 			if (debug->last_words[idx] != val)
1034 				pci_info(qm->pdev, "%s \t= 0x%08x => 0x%08x\n",
1035 					 hzip_dump_dfx_regs[j].name,
1036 					 debug->last_words[idx], val);
1037 		}
1038 	}
1039 }
1040 
hisi_zip_log_hw_error(struct hisi_qm * qm,u32 err_sts)1041 static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts)
1042 {
1043 	const struct hisi_zip_hw_error *err = zip_hw_error;
1044 	struct device *dev = &qm->pdev->dev;
1045 	u32 err_val;
1046 
1047 	while (err->msg) {
1048 		if (err->int_msk & err_sts) {
1049 			dev_err(dev, "%s [error status=0x%x] found\n",
1050 				err->msg, err->int_msk);
1051 
1052 			if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) {
1053 				err_val = readl(qm->io_base +
1054 						HZIP_CORE_SRAM_ECC_ERR_INFO);
1055 				dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n",
1056 					((err_val >>
1057 					HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF));
1058 			}
1059 		}
1060 		err++;
1061 	}
1062 }
1063 
hisi_zip_get_hw_err_status(struct hisi_qm * qm)1064 static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm)
1065 {
1066 	return readl(qm->io_base + HZIP_CORE_INT_STATUS);
1067 }
1068 
hisi_zip_clear_hw_err_status(struct hisi_qm * qm,u32 err_sts)1069 static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
1070 {
1071 	u32 nfe;
1072 
1073 	writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE);
1074 	nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);
1075 	writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1076 }
1077 
hisi_zip_open_axi_master_ooo(struct hisi_qm * qm)1078 static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm)
1079 {
1080 	u32 val;
1081 
1082 	val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1083 
1084 	writel(val & ~HZIP_AXI_SHUTDOWN_ENABLE,
1085 	       qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1086 
1087 	writel(val | HZIP_AXI_SHUTDOWN_ENABLE,
1088 	       qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);
1089 }
1090 
hisi_zip_close_axi_master_ooo(struct hisi_qm * qm)1091 static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm)
1092 {
1093 	u32 nfe_enb;
1094 
1095 	/* Disable ECC Mbit error report. */
1096 	nfe_enb = readl(qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1097 	writel(nfe_enb & ~HZIP_CORE_INT_STATUS_M_ECC,
1098 	       qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
1099 
1100 	/* Inject zip ECC Mbit error to block master ooo. */
1101 	writel(HZIP_CORE_INT_STATUS_M_ECC,
1102 	       qm->io_base + HZIP_CORE_INT_SET);
1103 }
1104 
hisi_zip_err_info_init(struct hisi_qm * qm)1105 static void hisi_zip_err_info_init(struct hisi_qm *qm)
1106 {
1107 	struct hisi_qm_err_info *err_info = &qm->err_info;
1108 
1109 	err_info->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK;
1110 	err_info->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver);
1111 	err_info->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1112 					    ZIP_QM_NFE_MASK_CAP, qm->cap_ver);
1113 	err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC;
1114 	err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1115 							 ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1116 	err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1117 							  ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1118 	err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1119 						      ZIP_QM_RESET_MASK_CAP, qm->cap_ver);
1120 	err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1121 						       ZIP_RESET_MASK_CAP, qm->cap_ver);
1122 	err_info->msi_wr_port = HZIP_WR_PORT;
1123 	err_info->acpi_rst = "ZRST";
1124 }
1125 
1126 static const struct hisi_qm_err_ini hisi_zip_err_ini = {
1127 	.hw_init		= hisi_zip_set_user_domain_and_cache,
1128 	.hw_err_enable		= hisi_zip_hw_error_enable,
1129 	.hw_err_disable		= hisi_zip_hw_error_disable,
1130 	.get_dev_hw_err_status	= hisi_zip_get_hw_err_status,
1131 	.clear_dev_hw_err_status = hisi_zip_clear_hw_err_status,
1132 	.log_dev_hw_err		= hisi_zip_log_hw_error,
1133 	.open_axi_master_ooo	= hisi_zip_open_axi_master_ooo,
1134 	.close_axi_master_ooo	= hisi_zip_close_axi_master_ooo,
1135 	.open_sva_prefetch	= hisi_zip_open_sva_prefetch,
1136 	.close_sva_prefetch	= hisi_zip_close_sva_prefetch,
1137 	.show_last_dfx_regs	= hisi_zip_show_last_dfx_regs,
1138 	.err_info_init		= hisi_zip_err_info_init,
1139 };
1140 
hisi_zip_pf_probe_init(struct hisi_zip * hisi_zip)1141 static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
1142 {
1143 	struct hisi_qm *qm = &hisi_zip->qm;
1144 	struct hisi_zip_ctrl *ctrl;
1145 	int ret;
1146 
1147 	ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL);
1148 	if (!ctrl)
1149 		return -ENOMEM;
1150 
1151 	hisi_zip->ctrl = ctrl;
1152 	ctrl->hisi_zip = hisi_zip;
1153 
1154 	ret = hisi_zip_set_user_domain_and_cache(qm);
1155 	if (ret)
1156 		return ret;
1157 
1158 	ret = hisi_zip_set_high_perf(qm);
1159 	if (ret)
1160 		return ret;
1161 
1162 	hisi_zip_open_sva_prefetch(qm);
1163 	hisi_qm_dev_err_init(qm);
1164 	hisi_zip_debug_regs_clear(qm);
1165 
1166 	ret = hisi_zip_show_last_regs_init(qm);
1167 	if (ret)
1168 		pci_err(qm->pdev, "Failed to init last word regs!\n");
1169 
1170 	return ret;
1171 }
1172 
zip_pre_store_cap_reg(struct hisi_qm * qm)1173 static int zip_pre_store_cap_reg(struct hisi_qm *qm)
1174 {
1175 	struct hisi_qm_cap_record *zip_cap;
1176 	struct pci_dev *pdev = qm->pdev;
1177 	size_t i, size;
1178 
1179 	size = ARRAY_SIZE(zip_pre_store_caps);
1180 	zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL);
1181 	if (!zip_cap)
1182 		return -ENOMEM;
1183 
1184 	for (i = 0; i < size; i++) {
1185 		zip_cap[i].type = zip_pre_store_caps[i];
1186 		zip_cap[i].cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info,
1187 				     zip_pre_store_caps[i], qm->cap_ver);
1188 	}
1189 
1190 	qm->cap_tables.dev_cap_table = zip_cap;
1191 
1192 	return 0;
1193 }
1194 
hisi_zip_qm_init(struct hisi_qm * qm,struct pci_dev * pdev)1195 static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
1196 {
1197 	u64 alg_msk;
1198 	int ret;
1199 
1200 	qm->pdev = pdev;
1201 	qm->ver = pdev->revision;
1202 	qm->mode = uacce_mode;
1203 	qm->sqe_size = HZIP_SQE_SIZE;
1204 	qm->dev_name = hisi_zip_name;
1205 
1206 	qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_ZIP_PF) ?
1207 			QM_HW_PF : QM_HW_VF;
1208 	if (qm->fun_type == QM_HW_PF) {
1209 		qm->qp_base = HZIP_PF_DEF_Q_BASE;
1210 		qm->qp_num = pf_q_num;
1211 		qm->debug.curr_qm_qp_num = pf_q_num;
1212 		qm->qm_list = &zip_devices;
1213 		qm->err_ini = &hisi_zip_err_ini;
1214 		if (pf_q_num_flag)
1215 			set_bit(QM_MODULE_PARAM, &qm->misc_ctl);
1216 	} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
1217 		/*
1218 		 * have no way to get qm configure in VM in v1 hardware,
1219 		 * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force
1220 		 * to trigger only one VF in v1 hardware.
1221 		 *
1222 		 * v2 hardware has no such problem.
1223 		 */
1224 		qm->qp_base = HZIP_PF_DEF_Q_NUM;
1225 		qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM;
1226 	}
1227 
1228 	ret = hisi_qm_init(qm);
1229 	if (ret) {
1230 		pci_err(qm->pdev, "Failed to init zip qm configures!\n");
1231 		return ret;
1232 	}
1233 
1234 	/* Fetch and save the value of capability registers */
1235 	ret = zip_pre_store_cap_reg(qm);
1236 	if (ret) {
1237 		pci_err(qm->pdev, "Failed to pre-store capability registers!\n");
1238 		hisi_qm_uninit(qm);
1239 		return ret;
1240 	}
1241 
1242 	alg_msk = qm->cap_tables.dev_cap_table[ZIP_DEV_ALG_BITMAP_IDX].cap_val;
1243 	ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs));
1244 	if (ret) {
1245 		pci_err(qm->pdev, "Failed to set zip algs!\n");
1246 		hisi_qm_uninit(qm);
1247 	}
1248 
1249 	return ret;
1250 }
1251 
hisi_zip_qm_uninit(struct hisi_qm * qm)1252 static void hisi_zip_qm_uninit(struct hisi_qm *qm)
1253 {
1254 	hisi_qm_uninit(qm);
1255 }
1256 
hisi_zip_probe_init(struct hisi_zip * hisi_zip)1257 static int hisi_zip_probe_init(struct hisi_zip *hisi_zip)
1258 {
1259 	u32 type_rate = HZIP_SHAPER_RATE_COMPRESS;
1260 	struct hisi_qm *qm = &hisi_zip->qm;
1261 	int ret;
1262 
1263 	if (qm->fun_type == QM_HW_PF) {
1264 		ret = hisi_zip_pf_probe_init(hisi_zip);
1265 		if (ret)
1266 			return ret;
1267 		/* enable shaper type 0 */
1268 		if (qm->ver >= QM_HW_V3) {
1269 			type_rate |= QM_SHAPER_ENABLE;
1270 
1271 			/* ZIP need to enable shaper type 1 */
1272 			type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET;
1273 			qm->type_rate = type_rate;
1274 		}
1275 	}
1276 
1277 	return 0;
1278 }
1279 
hisi_zip_probe_uninit(struct hisi_qm * qm)1280 static void hisi_zip_probe_uninit(struct hisi_qm *qm)
1281 {
1282 	if (qm->fun_type == QM_HW_VF)
1283 		return;
1284 
1285 	hisi_zip_show_last_regs_uninit(qm);
1286 	hisi_zip_close_sva_prefetch(qm);
1287 	hisi_qm_dev_err_uninit(qm);
1288 }
1289 
hisi_zip_probe(struct pci_dev * pdev,const struct pci_device_id * id)1290 static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1291 {
1292 	struct hisi_zip *hisi_zip;
1293 	struct hisi_qm *qm;
1294 	int ret;
1295 
1296 	hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);
1297 	if (!hisi_zip)
1298 		return -ENOMEM;
1299 
1300 	qm = &hisi_zip->qm;
1301 
1302 	ret = hisi_zip_qm_init(qm, pdev);
1303 	if (ret) {
1304 		pci_err(pdev, "Failed to init ZIP QM (%d)!\n", ret);
1305 		return ret;
1306 	}
1307 
1308 	ret = hisi_zip_probe_init(hisi_zip);
1309 	if (ret) {
1310 		pci_err(pdev, "Failed to probe (%d)!\n", ret);
1311 		goto err_qm_uninit;
1312 	}
1313 
1314 	ret = hisi_qm_start(qm);
1315 	if (ret)
1316 		goto err_probe_uninit;
1317 
1318 	ret = hisi_zip_debugfs_init(qm);
1319 	if (ret)
1320 		pci_err(pdev, "failed to init debugfs (%d)!\n", ret);
1321 
1322 	ret = hisi_qm_alg_register(qm, &zip_devices);
1323 	if (ret < 0) {
1324 		pci_err(pdev, "failed to register driver to crypto!\n");
1325 		goto err_qm_stop;
1326 	}
1327 
1328 	if (qm->uacce) {
1329 		ret = uacce_register(qm->uacce);
1330 		if (ret) {
1331 			pci_err(pdev, "failed to register uacce (%d)!\n", ret);
1332 			goto err_qm_alg_unregister;
1333 		}
1334 	}
1335 
1336 	if (qm->fun_type == QM_HW_PF && vfs_num > 0) {
1337 		ret = hisi_qm_sriov_enable(pdev, vfs_num);
1338 		if (ret < 0)
1339 			goto err_qm_alg_unregister;
1340 	}
1341 
1342 	hisi_qm_pm_init(qm);
1343 
1344 	return 0;
1345 
1346 err_qm_alg_unregister:
1347 	hisi_qm_alg_unregister(qm, &zip_devices);
1348 
1349 err_qm_stop:
1350 	hisi_zip_debugfs_exit(qm);
1351 	hisi_qm_stop(qm, QM_NORMAL);
1352 
1353 err_probe_uninit:
1354 	hisi_zip_probe_uninit(qm);
1355 
1356 err_qm_uninit:
1357 	hisi_zip_qm_uninit(qm);
1358 
1359 	return ret;
1360 }
1361 
hisi_zip_remove(struct pci_dev * pdev)1362 static void hisi_zip_remove(struct pci_dev *pdev)
1363 {
1364 	struct hisi_qm *qm = pci_get_drvdata(pdev);
1365 
1366 	hisi_qm_pm_uninit(qm);
1367 	hisi_qm_wait_task_finish(qm, &zip_devices);
1368 	hisi_qm_alg_unregister(qm, &zip_devices);
1369 
1370 	if (qm->fun_type == QM_HW_PF && qm->vfs_num)
1371 		hisi_qm_sriov_disable(pdev, true);
1372 
1373 	hisi_zip_debugfs_exit(qm);
1374 	hisi_qm_stop(qm, QM_NORMAL);
1375 	hisi_zip_probe_uninit(qm);
1376 	hisi_zip_qm_uninit(qm);
1377 }
1378 
1379 static const struct dev_pm_ops hisi_zip_pm_ops = {
1380 	SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL)
1381 };
1382 
1383 static const struct pci_error_handlers hisi_zip_err_handler = {
1384 	.error_detected	= hisi_qm_dev_err_detected,
1385 	.slot_reset	= hisi_qm_dev_slot_reset,
1386 	.reset_prepare	= hisi_qm_reset_prepare,
1387 	.reset_done	= hisi_qm_reset_done,
1388 };
1389 
1390 static struct pci_driver hisi_zip_pci_driver = {
1391 	.name			= "hisi_zip",
1392 	.id_table		= hisi_zip_dev_ids,
1393 	.probe			= hisi_zip_probe,
1394 	.remove			= hisi_zip_remove,
1395 	.sriov_configure	= IS_ENABLED(CONFIG_PCI_IOV) ?
1396 					hisi_qm_sriov_configure : NULL,
1397 	.err_handler		= &hisi_zip_err_handler,
1398 	.shutdown		= hisi_qm_dev_shutdown,
1399 	.driver.pm		= &hisi_zip_pm_ops,
1400 };
1401 
hisi_zip_get_pf_driver(void)1402 struct pci_driver *hisi_zip_get_pf_driver(void)
1403 {
1404 	return &hisi_zip_pci_driver;
1405 }
1406 EXPORT_SYMBOL_GPL(hisi_zip_get_pf_driver);
1407 
hisi_zip_register_debugfs(void)1408 static void hisi_zip_register_debugfs(void)
1409 {
1410 	if (!debugfs_initialized())
1411 		return;
1412 
1413 	hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL);
1414 }
1415 
hisi_zip_unregister_debugfs(void)1416 static void hisi_zip_unregister_debugfs(void)
1417 {
1418 	debugfs_remove_recursive(hzip_debugfs_root);
1419 }
1420 
hisi_zip_init(void)1421 static int __init hisi_zip_init(void)
1422 {
1423 	int ret;
1424 
1425 	hisi_qm_init_list(&zip_devices);
1426 	hisi_zip_register_debugfs();
1427 
1428 	ret = pci_register_driver(&hisi_zip_pci_driver);
1429 	if (ret < 0) {
1430 		hisi_zip_unregister_debugfs();
1431 		pr_err("Failed to register pci driver.\n");
1432 	}
1433 
1434 	return ret;
1435 }
1436 
hisi_zip_exit(void)1437 static void __exit hisi_zip_exit(void)
1438 {
1439 	pci_unregister_driver(&hisi_zip_pci_driver);
1440 	hisi_zip_unregister_debugfs();
1441 }
1442 
1443 module_init(hisi_zip_init);
1444 module_exit(hisi_zip_exit);
1445 
1446 MODULE_LICENSE("GPL v2");
1447 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
1448 MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator");
1449