1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <asm/page.h>
4 #include <linux/acpi.h>
5 #include <linux/bitmap.h>
6 #include <linux/dma-mapping.h>
7 #include <linux/idr.h>
8 #include <linux/io.h>
9 #include <linux/irqreturn.h>
10 #include <linux/log2.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/seq_file.h>
13 #include <linux/slab.h>
14 #include <linux/uacce.h>
15 #include <linux/uaccess.h>
16 #include <uapi/misc/uacce/hisi_qm.h>
17 #include <linux/hisi_acc_qm.h>
18 #include "qm_common.h"
19
20 /* eq/aeq irq enable */
21 #define QM_VF_AEQ_INT_SOURCE 0x0
22 #define QM_VF_AEQ_INT_MASK 0x4
23 #define QM_VF_EQ_INT_SOURCE 0x8
24 #define QM_VF_EQ_INT_MASK 0xc
25
26 #define QM_IRQ_VECTOR_MASK GENMASK(15, 0)
27 #define QM_IRQ_TYPE_MASK GENMASK(15, 0)
28 #define QM_IRQ_TYPE_SHIFT 16
29 #define QM_ABN_IRQ_TYPE_MASK GENMASK(7, 0)
30
31 /* mailbox */
32 #define QM_MB_PING_ALL_VFS 0xffff
33 #define QM_MB_CMD_DATA_SHIFT 32
34 #define QM_MB_CMD_DATA_MASK GENMASK(31, 0)
35 #define QM_MB_STATUS_MASK GENMASK(12, 9)
36
37 /* sqc shift */
38 #define QM_SQ_HOP_NUM_SHIFT 0
39 #define QM_SQ_PAGE_SIZE_SHIFT 4
40 #define QM_SQ_BUF_SIZE_SHIFT 8
41 #define QM_SQ_SQE_SIZE_SHIFT 12
42 #define QM_SQ_PRIORITY_SHIFT 0
43 #define QM_SQ_ORDERS_SHIFT 4
44 #define QM_SQ_TYPE_SHIFT 8
45 #define QM_QC_PASID_ENABLE 0x1
46 #define QM_QC_PASID_ENABLE_SHIFT 7
47
48 #define QM_SQ_TYPE_MASK GENMASK(3, 0)
49 #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc)->w11) >> 6) & 0x1)
50
51 /* cqc shift */
52 #define QM_CQ_HOP_NUM_SHIFT 0
53 #define QM_CQ_PAGE_SIZE_SHIFT 4
54 #define QM_CQ_BUF_SIZE_SHIFT 8
55 #define QM_CQ_CQE_SIZE_SHIFT 12
56 #define QM_CQ_PHASE_SHIFT 0
57 #define QM_CQ_FLAG_SHIFT 1
58
59 #define QM_CQE_PHASE(cqe) (le16_to_cpu((cqe)->w7) & 0x1)
60 #define QM_QC_CQE_SIZE 4
61 #define QM_CQ_TAIL_IDX(cqc) ((le16_to_cpu((cqc)->w11) >> 6) & 0x1)
62
63 /* eqc shift */
64 #define QM_EQE_AEQE_SIZE (2UL << 12)
65 #define QM_EQC_PHASE_SHIFT 16
66
67 #define QM_EQE_PHASE(eqe) ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)
68 #define QM_EQE_CQN_MASK GENMASK(15, 0)
69
70 #define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
71 #define QM_AEQE_TYPE_SHIFT 17
72 #define QM_AEQE_CQN_MASK GENMASK(15, 0)
73 #define QM_CQ_OVERFLOW 0
74 #define QM_EQ_OVERFLOW 1
75 #define QM_CQE_ERROR 2
76
77 #define QM_XQ_DEPTH_SHIFT 16
78 #define QM_XQ_DEPTH_MASK GENMASK(15, 0)
79
80 #define QM_DOORBELL_CMD_SQ 0
81 #define QM_DOORBELL_CMD_CQ 1
82 #define QM_DOORBELL_CMD_EQ 2
83 #define QM_DOORBELL_CMD_AEQ 3
84
85 #define QM_DOORBELL_BASE_V1 0x340
86 #define QM_DB_CMD_SHIFT_V1 16
87 #define QM_DB_INDEX_SHIFT_V1 32
88 #define QM_DB_PRIORITY_SHIFT_V1 48
89 #define QM_PAGE_SIZE 0x0034
90 #define QM_QP_DB_INTERVAL 0x10000
91 #define QM_DB_TIMEOUT_CFG 0x100074
92 #define QM_DB_TIMEOUT_SET 0x1fffff
93
94 #define QM_MEM_START_INIT 0x100040
95 #define QM_MEM_INIT_DONE 0x100044
96 #define QM_VFT_CFG_RDY 0x10006c
97 #define QM_VFT_CFG_OP_WR 0x100058
98 #define QM_VFT_CFG_TYPE 0x10005c
99 #define QM_VFT_CFG 0x100060
100 #define QM_VFT_CFG_OP_ENABLE 0x100054
101 #define QM_PM_CTRL 0x100148
102 #define QM_IDLE_DISABLE BIT(9)
103
104 #define QM_VFT_CFG_DATA_L 0x100064
105 #define QM_VFT_CFG_DATA_H 0x100068
106 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8)
107 #define QM_SQC_VFT_SQC_SIZE (5ULL << 12)
108 #define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16)
109 #define QM_SQC_VFT_START_SQN_SHIFT 28
110 #define QM_SQC_VFT_VALID (1ULL << 44)
111 #define QM_SQC_VFT_SQN_SHIFT 45
112 #define QM_CQC_VFT_BUF_SIZE (7ULL << 8)
113 #define QM_CQC_VFT_SQC_SIZE (5ULL << 12)
114 #define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16)
115 #define QM_CQC_VFT_VALID (1ULL << 28)
116
117 #define QM_SQC_VFT_BASE_SHIFT_V2 28
118 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0)
119 #define QM_SQC_VFT_NUM_SHIFT_V2 45
120 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0)
121
122 #define QM_ABNORMAL_INT_SOURCE 0x100000
123 #define QM_ABNORMAL_INT_MASK 0x100004
124 #define QM_ABNORMAL_INT_MASK_VALUE 0x7fff
125 #define QM_ABNORMAL_INT_STATUS 0x100008
126 #define QM_ABNORMAL_INT_SET 0x10000c
127 #define QM_ABNORMAL_INF00 0x100010
128 #define QM_FIFO_OVERFLOW_TYPE 0xc0
129 #define QM_FIFO_OVERFLOW_TYPE_SHIFT 6
130 #define QM_FIFO_OVERFLOW_VF 0x3f
131 #define QM_ABNORMAL_INF01 0x100014
132 #define QM_DB_TIMEOUT_TYPE 0xc0
133 #define QM_DB_TIMEOUT_TYPE_SHIFT 6
134 #define QM_DB_TIMEOUT_VF 0x3f
135 #define QM_RAS_CE_ENABLE 0x1000ec
136 #define QM_RAS_FE_ENABLE 0x1000f0
137 #define QM_RAS_NFE_ENABLE 0x1000f4
138 #define QM_RAS_CE_THRESHOLD 0x1000f8
139 #define QM_RAS_CE_TIMES_PER_IRQ 1
140 #define QM_OOO_SHUTDOWN_SEL 0x1040f8
141 #define QM_ECC_MBIT BIT(2)
142 #define QM_DB_TIMEOUT BIT(10)
143 #define QM_OF_FIFO_OF BIT(11)
144
145 #define QM_RESET_WAIT_TIMEOUT 400
146 #define QM_PEH_VENDOR_ID 0x1000d8
147 #define ACC_VENDOR_ID_VALUE 0x5a5a
148 #define QM_PEH_DFX_INFO0 0x1000fc
149 #define QM_PEH_DFX_INFO1 0x100100
150 #define QM_PEH_DFX_MASK (BIT(0) | BIT(2))
151 #define QM_PEH_MSI_FINISH_MASK GENMASK(19, 16)
152 #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3
153 #define ACC_PEH_MSI_DISABLE GENMASK(31, 0)
154 #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1
155 #define ACC_MASTER_TRANS_RETURN_RW 3
156 #define ACC_MASTER_TRANS_RETURN 0x300150
157 #define ACC_MASTER_GLOBAL_CTRL 0x300000
158 #define ACC_AM_CFG_PORT_WR_EN 0x30001c
159 #define QM_RAS_NFE_MBIT_DISABLE ~QM_ECC_MBIT
160 #define ACC_AM_ROB_ECC_INT_STS 0x300104
161 #define ACC_ROB_ECC_ERR_MULTPL BIT(1)
162 #define QM_MSI_CAP_ENABLE BIT(16)
163
164 /* interfunction communication */
165 #define QM_IFC_READY_STATUS 0x100128
166 #define QM_IFC_INT_SET_P 0x100130
167 #define QM_IFC_INT_CFG 0x100134
168 #define QM_IFC_INT_SOURCE_P 0x100138
169 #define QM_IFC_INT_SOURCE_V 0x0020
170 #define QM_IFC_INT_MASK 0x0024
171 #define QM_IFC_INT_STATUS 0x0028
172 #define QM_IFC_INT_SET_V 0x002C
173 #define QM_IFC_SEND_ALL_VFS GENMASK(6, 0)
174 #define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0)
175 #define QM_IFC_INT_SOURCE_MASK BIT(0)
176 #define QM_IFC_INT_DISABLE BIT(0)
177 #define QM_IFC_INT_STATUS_MASK BIT(0)
178 #define QM_IFC_INT_SET_MASK BIT(0)
179 #define QM_WAIT_DST_ACK 10
180 #define QM_MAX_PF_WAIT_COUNT 10
181 #define QM_MAX_VF_WAIT_COUNT 40
182 #define QM_VF_RESET_WAIT_US 20000
183 #define QM_VF_RESET_WAIT_CNT 3000
184 #define QM_VF_RESET_WAIT_TIMEOUT_US \
185 (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT)
186
187 #define POLL_PERIOD 10
188 #define POLL_TIMEOUT 1000
189 #define WAIT_PERIOD_US_MAX 200
190 #define WAIT_PERIOD_US_MIN 100
191 #define MAX_WAIT_COUNTS 1000
192 #define QM_CACHE_WB_START 0x204
193 #define QM_CACHE_WB_DONE 0x208
194 #define QM_FUNC_CAPS_REG 0x3100
195 #define QM_CAPBILITY_VERSION GENMASK(7, 0)
196
197 #define PCI_BAR_2 2
198 #define PCI_BAR_4 4
199 #define QMC_ALIGN(sz) ALIGN(sz, 32)
200
201 #define QM_DBG_READ_LEN 256
202 #define QM_PCI_COMMAND_INVALID ~0
203 #define QM_RESET_STOP_TX_OFFSET 1
204 #define QM_RESET_STOP_RX_OFFSET 2
205
206 #define WAIT_PERIOD 20
207 #define REMOVE_WAIT_DELAY 10
208
209 #define QM_QOS_PARAM_NUM 2
210 #define QM_QOS_MAX_VAL 1000
211 #define QM_QOS_RATE 100
212 #define QM_QOS_EXPAND_RATE 1000
213 #define QM_SHAPER_CIR_B_MASK GENMASK(7, 0)
214 #define QM_SHAPER_CIR_U_MASK GENMASK(10, 8)
215 #define QM_SHAPER_CIR_S_MASK GENMASK(14, 11)
216 #define QM_SHAPER_FACTOR_CIR_U_SHIFT 8
217 #define QM_SHAPER_FACTOR_CIR_S_SHIFT 11
218 #define QM_SHAPER_FACTOR_CBS_B_SHIFT 15
219 #define QM_SHAPER_FACTOR_CBS_S_SHIFT 19
220 #define QM_SHAPER_CBS_B 1
221 #define QM_SHAPER_VFT_OFFSET 6
222 #define QM_QOS_MIN_ERROR_RATE 5
223 #define QM_SHAPER_MIN_CBS_S 8
224 #define QM_QOS_TICK 0x300U
225 #define QM_QOS_DIVISOR_CLK 0x1f40U
226 #define QM_QOS_MAX_CIR_B 200
227 #define QM_QOS_MIN_CIR_B 100
228 #define QM_QOS_MAX_CIR_U 6
229 #define QM_AUTOSUSPEND_DELAY 3000
230
231 #define QM_DEV_ALG_MAX_LEN 256
232
233 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
234 (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \
235 ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \
236 ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \
237 ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
238
239 #define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \
240 ((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
241
242 #define QM_MK_SQC_W13(priority, orders, alg_type) \
243 (((priority) << QM_SQ_PRIORITY_SHIFT) | \
244 ((orders) << QM_SQ_ORDERS_SHIFT) | \
245 (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
246
247 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
248 (((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \
249 ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \
250 ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \
251 ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
252
253 #define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \
254 ((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
255
256 #define INIT_QC_COMMON(qc, base, pasid) do { \
257 (qc)->head = 0; \
258 (qc)->tail = 0; \
259 (qc)->base_l = cpu_to_le32(lower_32_bits(base)); \
260 (qc)->base_h = cpu_to_le32(upper_32_bits(base)); \
261 (qc)->dw3 = 0; \
262 (qc)->w8 = 0; \
263 (qc)->rsvd0 = 0; \
264 (qc)->pasid = cpu_to_le16(pasid); \
265 (qc)->w11 = 0; \
266 (qc)->rsvd1 = 0; \
267 } while (0)
268
269 enum vft_type {
270 SQC_VFT = 0,
271 CQC_VFT,
272 SHAPER_VFT,
273 };
274
275 enum acc_err_result {
276 ACC_ERR_NONE,
277 ACC_ERR_NEED_RESET,
278 ACC_ERR_RECOVERED,
279 };
280
281 enum qm_alg_type {
282 ALG_TYPE_0,
283 ALG_TYPE_1,
284 };
285
286 enum qm_mb_cmd {
287 QM_PF_FLR_PREPARE = 0x01,
288 QM_PF_SRST_PREPARE,
289 QM_PF_RESET_DONE,
290 QM_VF_PREPARE_DONE,
291 QM_VF_PREPARE_FAIL,
292 QM_VF_START_DONE,
293 QM_VF_START_FAIL,
294 QM_PF_SET_QOS,
295 QM_VF_GET_QOS,
296 };
297
298 enum qm_basic_type {
299 QM_TOTAL_QP_NUM_CAP = 0x0,
300 QM_FUNC_MAX_QP_CAP,
301 QM_XEQ_DEPTH_CAP,
302 QM_QP_DEPTH_CAP,
303 QM_EQ_IRQ_TYPE_CAP,
304 QM_AEQ_IRQ_TYPE_CAP,
305 QM_ABN_IRQ_TYPE_CAP,
306 QM_PF2VF_IRQ_TYPE_CAP,
307 QM_PF_IRQ_NUM_CAP,
308 QM_VF_IRQ_NUM_CAP,
309 };
310
311 enum qm_pre_store_cap_idx {
312 QM_EQ_IRQ_TYPE_CAP_IDX = 0x0,
313 QM_AEQ_IRQ_TYPE_CAP_IDX,
314 QM_ABN_IRQ_TYPE_CAP_IDX,
315 QM_PF2VF_IRQ_TYPE_CAP_IDX,
316 };
317
318 static const struct hisi_qm_cap_info qm_cap_info_comm[] = {
319 {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0},
320 {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1},
321 {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1},
322 {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1},
323 {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1},
324 };
325
326 static const struct hisi_qm_cap_info qm_cap_info_pf[] = {
327 {QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1},
328 };
329
330 static const struct hisi_qm_cap_info qm_cap_info_vf[] = {
331 {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0},
332 };
333
334 static const struct hisi_qm_cap_info qm_basic_info[] = {
335 {QM_TOTAL_QP_NUM_CAP, 0x100158, 0, GENMASK(10, 0), 0x1000, 0x400, 0x400},
336 {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400},
337 {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(31, 0), 0x800, 0x4000800, 0x4000800},
338 {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400},
339 {QM_EQ_IRQ_TYPE_CAP, 0x310c, 0, GENMASK(31, 0), 0x10000, 0x10000, 0x10000},
340 {QM_AEQ_IRQ_TYPE_CAP, 0x3110, 0, GENMASK(31, 0), 0x0, 0x10001, 0x10001},
341 {QM_ABN_IRQ_TYPE_CAP, 0x3114, 0, GENMASK(31, 0), 0x0, 0x10003, 0x10003},
342 {QM_PF2VF_IRQ_TYPE_CAP, 0x3118, 0, GENMASK(31, 0), 0x0, 0x0, 0x10002},
343 {QM_PF_IRQ_NUM_CAP, 0x311c, 16, GENMASK(15, 0), 0x1, 0x4, 0x4},
344 {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3},
345 };
346
347 static const u32 qm_pre_store_caps[] = {
348 QM_EQ_IRQ_TYPE_CAP,
349 QM_AEQ_IRQ_TYPE_CAP,
350 QM_ABN_IRQ_TYPE_CAP,
351 QM_PF2VF_IRQ_TYPE_CAP,
352 };
353
354 struct qm_mailbox {
355 __le16 w0;
356 __le16 queue_num;
357 __le32 base_l;
358 __le32 base_h;
359 __le32 rsvd;
360 };
361
362 struct qm_doorbell {
363 __le16 queue_num;
364 __le16 cmd;
365 __le16 index;
366 __le16 priority;
367 };
368
369 struct hisi_qm_resource {
370 struct hisi_qm *qm;
371 int distance;
372 struct list_head list;
373 };
374
375 /**
376 * struct qm_hw_err - Structure describing the device errors
377 * @list: hardware error list
378 * @timestamp: timestamp when the error occurred
379 */
380 struct qm_hw_err {
381 struct list_head list;
382 unsigned long long timestamp;
383 };
384
385 struct hisi_qm_hw_ops {
386 int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
387 void (*qm_db)(struct hisi_qm *qm, u16 qn,
388 u8 cmd, u16 index, u8 priority);
389 int (*debug_init)(struct hisi_qm *qm);
390 void (*hw_error_init)(struct hisi_qm *qm);
391 void (*hw_error_uninit)(struct hisi_qm *qm);
392 enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
393 int (*set_msi)(struct hisi_qm *qm, bool set);
394 };
395
396 struct hisi_qm_hw_error {
397 u32 int_msk;
398 const char *msg;
399 };
400
401 static const struct hisi_qm_hw_error qm_hw_error[] = {
402 { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
403 { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
404 { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
405 { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
406 { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
407 { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
408 { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
409 { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
410 { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
411 { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
412 { .int_msk = BIT(10), .msg = "qm_db_timeout" },
413 { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
414 { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
415 { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
416 { .int_msk = BIT(14), .msg = "qm_flr_timeout" },
417 { /* sentinel */ }
418 };
419
420 static const char * const qm_db_timeout[] = {
421 "sq", "cq", "eq", "aeq",
422 };
423
424 static const char * const qm_fifo_overflow[] = {
425 "cq", "eq", "aeq",
426 };
427
428 static const char * const qp_s[] = {
429 "none", "init", "start", "stop", "close",
430 };
431
432 struct qm_typical_qos_table {
433 u32 start;
434 u32 end;
435 u32 val;
436 };
437
438 /* the qos step is 100 */
439 static struct qm_typical_qos_table shaper_cir_s[] = {
440 {100, 100, 4},
441 {200, 200, 3},
442 {300, 500, 2},
443 {600, 1000, 1},
444 {1100, 100000, 0},
445 };
446
447 static struct qm_typical_qos_table shaper_cbs_s[] = {
448 {100, 200, 9},
449 {300, 500, 11},
450 {600, 1000, 12},
451 {1100, 10000, 16},
452 {10100, 25000, 17},
453 {25100, 50000, 18},
454 {50100, 100000, 19}
455 };
456
457 static void qm_irqs_unregister(struct hisi_qm *qm);
458
qm_avail_state(struct hisi_qm * qm,enum qm_state new)459 static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
460 {
461 enum qm_state curr = atomic_read(&qm->status.flags);
462 bool avail = false;
463
464 switch (curr) {
465 case QM_INIT:
466 if (new == QM_START || new == QM_CLOSE)
467 avail = true;
468 break;
469 case QM_START:
470 if (new == QM_STOP)
471 avail = true;
472 break;
473 case QM_STOP:
474 if (new == QM_CLOSE || new == QM_START)
475 avail = true;
476 break;
477 default:
478 break;
479 }
480
481 dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n",
482 qm_s[curr], qm_s[new]);
483
484 if (!avail)
485 dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n",
486 qm_s[curr], qm_s[new]);
487
488 return avail;
489 }
490
qm_qp_avail_state(struct hisi_qm * qm,struct hisi_qp * qp,enum qp_state new)491 static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
492 enum qp_state new)
493 {
494 enum qm_state qm_curr = atomic_read(&qm->status.flags);
495 enum qp_state qp_curr = 0;
496 bool avail = false;
497
498 if (qp)
499 qp_curr = atomic_read(&qp->qp_status.flags);
500
501 switch (new) {
502 case QP_INIT:
503 if (qm_curr == QM_START || qm_curr == QM_INIT)
504 avail = true;
505 break;
506 case QP_START:
507 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
508 (qm_curr == QM_START && qp_curr == QP_STOP))
509 avail = true;
510 break;
511 case QP_STOP:
512 if ((qm_curr == QM_START && qp_curr == QP_START) ||
513 (qp_curr == QP_INIT))
514 avail = true;
515 break;
516 case QP_CLOSE:
517 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
518 (qm_curr == QM_START && qp_curr == QP_STOP) ||
519 (qm_curr == QM_STOP && qp_curr == QP_STOP) ||
520 (qm_curr == QM_STOP && qp_curr == QP_INIT))
521 avail = true;
522 break;
523 default:
524 break;
525 }
526
527 dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n",
528 qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
529
530 if (!avail)
531 dev_warn(&qm->pdev->dev,
532 "Can not change qp state from %s to %s in QM %s\n",
533 qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
534
535 return avail;
536 }
537
qm_get_hw_error_status(struct hisi_qm * qm)538 static u32 qm_get_hw_error_status(struct hisi_qm *qm)
539 {
540 return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
541 }
542
qm_get_dev_err_status(struct hisi_qm * qm)543 static u32 qm_get_dev_err_status(struct hisi_qm *qm)
544 {
545 return qm->err_ini->get_dev_hw_err_status(qm);
546 }
547
548 /* Check if the error causes the master ooo block */
qm_check_dev_error(struct hisi_qm * qm)549 static bool qm_check_dev_error(struct hisi_qm *qm)
550 {
551 u32 val, dev_val;
552
553 if (qm->fun_type == QM_HW_VF)
554 return false;
555
556 val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask;
557 dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask;
558
559 return val || dev_val;
560 }
561
qm_wait_reset_finish(struct hisi_qm * qm)562 static int qm_wait_reset_finish(struct hisi_qm *qm)
563 {
564 int delay = 0;
565
566 /* All reset requests need to be queued for processing */
567 while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
568 msleep(++delay);
569 if (delay > QM_RESET_WAIT_TIMEOUT)
570 return -EBUSY;
571 }
572
573 return 0;
574 }
575
qm_reset_prepare_ready(struct hisi_qm * qm)576 static int qm_reset_prepare_ready(struct hisi_qm *qm)
577 {
578 struct pci_dev *pdev = qm->pdev;
579 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
580
581 /*
582 * PF and VF on host doesnot support resetting at the
583 * same time on Kunpeng920.
584 */
585 if (qm->ver < QM_HW_V3)
586 return qm_wait_reset_finish(pf_qm);
587
588 return qm_wait_reset_finish(qm);
589 }
590
qm_reset_bit_clear(struct hisi_qm * qm)591 static void qm_reset_bit_clear(struct hisi_qm *qm)
592 {
593 struct pci_dev *pdev = qm->pdev;
594 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
595
596 if (qm->ver < QM_HW_V3)
597 clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
598
599 clear_bit(QM_RESETTING, &qm->misc_ctl);
600 }
601
qm_mb_pre_init(struct qm_mailbox * mailbox,u8 cmd,u64 base,u16 queue,bool op)602 static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,
603 u64 base, u16 queue, bool op)
604 {
605 mailbox->w0 = cpu_to_le16((cmd) |
606 ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) |
607 (0x1 << QM_MB_BUSY_SHIFT));
608 mailbox->queue_num = cpu_to_le16(queue);
609 mailbox->base_l = cpu_to_le32(lower_32_bits(base));
610 mailbox->base_h = cpu_to_le32(upper_32_bits(base));
611 mailbox->rsvd = 0;
612 }
613
614 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
hisi_qm_wait_mb_ready(struct hisi_qm * qm)615 int hisi_qm_wait_mb_ready(struct hisi_qm *qm)
616 {
617 u32 val;
618
619 return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
620 val, !((val >> QM_MB_BUSY_SHIFT) &
621 0x1), POLL_PERIOD, POLL_TIMEOUT);
622 }
623 EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready);
624
625 /* 128 bit should be written to hardware at one time to trigger a mailbox */
qm_mb_write(struct hisi_qm * qm,const void * src)626 static void qm_mb_write(struct hisi_qm *qm, const void *src)
627 {
628 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
629
630 #if IS_ENABLED(CONFIG_ARM64)
631 unsigned long tmp0 = 0, tmp1 = 0;
632 #endif
633
634 if (!IS_ENABLED(CONFIG_ARM64)) {
635 memcpy_toio(fun_base, src, 16);
636 dma_wmb();
637 return;
638 }
639
640 #if IS_ENABLED(CONFIG_ARM64)
641 asm volatile("ldp %0, %1, %3\n"
642 "stp %0, %1, %2\n"
643 "dmb oshst\n"
644 : "=&r" (tmp0),
645 "=&r" (tmp1),
646 "+Q" (*((char __iomem *)fun_base))
647 : "Q" (*((char *)src))
648 : "memory");
649 #endif
650 }
651
qm_mb_nolock(struct hisi_qm * qm,struct qm_mailbox * mailbox)652 static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox)
653 {
654 int ret;
655 u32 val;
656
657 if (unlikely(hisi_qm_wait_mb_ready(qm))) {
658 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
659 ret = -EBUSY;
660 goto mb_busy;
661 }
662
663 qm_mb_write(qm, mailbox);
664
665 if (unlikely(hisi_qm_wait_mb_ready(qm))) {
666 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
667 ret = -ETIMEDOUT;
668 goto mb_busy;
669 }
670
671 val = readl(qm->io_base + QM_MB_CMD_SEND_BASE);
672 if (val & QM_MB_STATUS_MASK) {
673 dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n");
674 ret = -EIO;
675 goto mb_busy;
676 }
677
678 return 0;
679
680 mb_busy:
681 atomic64_inc(&qm->debug.dfx.mb_err_cnt);
682 return ret;
683 }
684
hisi_qm_mb(struct hisi_qm * qm,u8 cmd,dma_addr_t dma_addr,u16 queue,bool op)685 int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
686 bool op)
687 {
688 struct qm_mailbox mailbox;
689 int ret;
690
691 dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
692 queue, cmd, (unsigned long long)dma_addr);
693
694 qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op);
695
696 mutex_lock(&qm->mailbox_lock);
697 ret = qm_mb_nolock(qm, &mailbox);
698 mutex_unlock(&qm->mailbox_lock);
699
700 return ret;
701 }
702 EXPORT_SYMBOL_GPL(hisi_qm_mb);
703
qm_db_v1(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)704 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
705 {
706 u64 doorbell;
707
708 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
709 ((u64)index << QM_DB_INDEX_SHIFT_V1) |
710 ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
711
712 writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
713 }
714
qm_db_v2(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)715 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
716 {
717 void __iomem *io_base = qm->io_base;
718 u16 randata = 0;
719 u64 doorbell;
720
721 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
722 io_base = qm->db_io_base + (u64)qn * qm->db_interval +
723 QM_DOORBELL_SQ_CQ_BASE_V2;
724 else
725 io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;
726
727 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
728 ((u64)randata << QM_DB_RAND_SHIFT_V2) |
729 ((u64)index << QM_DB_INDEX_SHIFT_V2) |
730 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
731
732 writeq(doorbell, io_base);
733 }
734
qm_db(struct hisi_qm * qm,u16 qn,u8 cmd,u16 index,u8 priority)735 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
736 {
737 dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
738 qn, cmd, index);
739
740 qm->ops->qm_db(qm, qn, cmd, index, priority);
741 }
742
qm_disable_clock_gate(struct hisi_qm * qm)743 static void qm_disable_clock_gate(struct hisi_qm *qm)
744 {
745 u32 val;
746
747 /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */
748 if (qm->ver < QM_HW_V3)
749 return;
750
751 val = readl(qm->io_base + QM_PM_CTRL);
752 val |= QM_IDLE_DISABLE;
753 writel(val, qm->io_base + QM_PM_CTRL);
754 }
755
qm_dev_mem_reset(struct hisi_qm * qm)756 static int qm_dev_mem_reset(struct hisi_qm *qm)
757 {
758 u32 val;
759
760 writel(0x1, qm->io_base + QM_MEM_START_INIT);
761 return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
762 val & BIT(0), POLL_PERIOD,
763 POLL_TIMEOUT);
764 }
765
766 /**
767 * hisi_qm_get_hw_info() - Get device information.
768 * @qm: The qm which want to get information.
769 * @info_table: Array for storing device information.
770 * @index: Index in info_table.
771 * @is_read: Whether read from reg, 0: not support read from reg.
772 *
773 * This function returns device information the caller needs.
774 */
hisi_qm_get_hw_info(struct hisi_qm * qm,const struct hisi_qm_cap_info * info_table,u32 index,bool is_read)775 u32 hisi_qm_get_hw_info(struct hisi_qm *qm,
776 const struct hisi_qm_cap_info *info_table,
777 u32 index, bool is_read)
778 {
779 u32 val;
780
781 switch (qm->ver) {
782 case QM_HW_V1:
783 return info_table[index].v1_val;
784 case QM_HW_V2:
785 return info_table[index].v2_val;
786 default:
787 if (!is_read)
788 return info_table[index].v3_val;
789
790 val = readl(qm->io_base + info_table[index].offset);
791 return (val >> info_table[index].shift) & info_table[index].mask;
792 }
793 }
794 EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info);
795
qm_get_xqc_depth(struct hisi_qm * qm,u16 * low_bits,u16 * high_bits,enum qm_basic_type type)796 static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits,
797 u16 *high_bits, enum qm_basic_type type)
798 {
799 u32 depth;
800
801 depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver);
802 *low_bits = depth & QM_XQ_DEPTH_MASK;
803 *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK;
804 }
805
hisi_qm_set_algs(struct hisi_qm * qm,u64 alg_msk,const struct qm_dev_alg * dev_algs,u32 dev_algs_size)806 int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs,
807 u32 dev_algs_size)
808 {
809 struct device *dev = &qm->pdev->dev;
810 char *algs, *ptr;
811 int i;
812
813 if (!qm->uacce)
814 return 0;
815
816 if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) {
817 dev_err(dev, "algs size %u is equal or larger than %d.\n",
818 dev_algs_size, QM_DEV_ALG_MAX_LEN);
819 return -EINVAL;
820 }
821
822 algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL);
823 if (!algs)
824 return -ENOMEM;
825
826 for (i = 0; i < dev_algs_size; i++)
827 if (alg_msk & dev_algs[i].alg_msk)
828 strcat(algs, dev_algs[i].alg);
829
830 ptr = strrchr(algs, '\n');
831 if (ptr) {
832 *ptr = '\0';
833 qm->uacce->algs = algs;
834 }
835
836 return 0;
837 }
838 EXPORT_SYMBOL_GPL(hisi_qm_set_algs);
839
qm_get_irq_num(struct hisi_qm * qm)840 static u32 qm_get_irq_num(struct hisi_qm *qm)
841 {
842 if (qm->fun_type == QM_HW_PF)
843 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver);
844
845 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver);
846 }
847
qm_pm_get_sync(struct hisi_qm * qm)848 static int qm_pm_get_sync(struct hisi_qm *qm)
849 {
850 struct device *dev = &qm->pdev->dev;
851 int ret;
852
853 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
854 return 0;
855
856 ret = pm_runtime_resume_and_get(dev);
857 if (ret < 0) {
858 dev_err(dev, "failed to get_sync(%d).\n", ret);
859 return ret;
860 }
861
862 return 0;
863 }
864
qm_pm_put_sync(struct hisi_qm * qm)865 static void qm_pm_put_sync(struct hisi_qm *qm)
866 {
867 struct device *dev = &qm->pdev->dev;
868
869 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
870 return;
871
872 pm_runtime_mark_last_busy(dev);
873 pm_runtime_put_autosuspend(dev);
874 }
875
qm_cq_head_update(struct hisi_qp * qp)876 static void qm_cq_head_update(struct hisi_qp *qp)
877 {
878 if (qp->qp_status.cq_head == qp->cq_depth - 1) {
879 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
880 qp->qp_status.cq_head = 0;
881 } else {
882 qp->qp_status.cq_head++;
883 }
884 }
885
qm_poll_req_cb(struct hisi_qp * qp)886 static void qm_poll_req_cb(struct hisi_qp *qp)
887 {
888 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
889 struct hisi_qm *qm = qp->qm;
890
891 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
892 dma_rmb();
893 qp->req_cb(qp, qp->sqe + qm->sqe_size *
894 le16_to_cpu(cqe->sq_head));
895 qm_cq_head_update(qp);
896 cqe = qp->cqe + qp->qp_status.cq_head;
897 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
898 qp->qp_status.cq_head, 0);
899 atomic_dec(&qp->qp_status.used);
900
901 cond_resched();
902 }
903
904 /* set c_flag */
905 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1);
906 }
907
qm_work_process(struct work_struct * work)908 static void qm_work_process(struct work_struct *work)
909 {
910 struct hisi_qm_poll_data *poll_data =
911 container_of(work, struct hisi_qm_poll_data, work);
912 struct hisi_qm *qm = poll_data->qm;
913 u16 eqe_num = poll_data->eqe_num;
914 struct hisi_qp *qp;
915 int i;
916
917 for (i = eqe_num - 1; i >= 0; i--) {
918 qp = &qm->qp_array[poll_data->qp_finish_id[i]];
919 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP))
920 continue;
921
922 if (qp->event_cb) {
923 qp->event_cb(qp);
924 continue;
925 }
926
927 if (likely(qp->req_cb))
928 qm_poll_req_cb(qp);
929 }
930 }
931
qm_get_complete_eqe_num(struct hisi_qm * qm)932 static void qm_get_complete_eqe_num(struct hisi_qm *qm)
933 {
934 struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
935 struct hisi_qm_poll_data *poll_data = NULL;
936 u16 eq_depth = qm->eq_depth;
937 u16 cqn, eqe_num = 0;
938
939 if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) {
940 atomic64_inc(&qm->debug.dfx.err_irq_cnt);
941 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
942 return;
943 }
944
945 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
946 if (unlikely(cqn >= qm->qp_num))
947 return;
948 poll_data = &qm->poll_data[cqn];
949
950 while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
951 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
952 poll_data->qp_finish_id[eqe_num] = cqn;
953 eqe_num++;
954
955 if (qm->status.eq_head == eq_depth - 1) {
956 qm->status.eqc_phase = !qm->status.eqc_phase;
957 eqe = qm->eqe;
958 qm->status.eq_head = 0;
959 } else {
960 eqe++;
961 qm->status.eq_head++;
962 }
963
964 if (eqe_num == (eq_depth >> 1) - 1)
965 break;
966 }
967
968 poll_data->eqe_num = eqe_num;
969 queue_work(qm->wq, &poll_data->work);
970 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
971 }
972
qm_eq_irq(int irq,void * data)973 static irqreturn_t qm_eq_irq(int irq, void *data)
974 {
975 struct hisi_qm *qm = data;
976
977 /* Get qp id of completed tasks and re-enable the interrupt */
978 qm_get_complete_eqe_num(qm);
979
980 return IRQ_HANDLED;
981 }
982
qm_mb_cmd_irq(int irq,void * data)983 static irqreturn_t qm_mb_cmd_irq(int irq, void *data)
984 {
985 struct hisi_qm *qm = data;
986 u32 val;
987
988 val = readl(qm->io_base + QM_IFC_INT_STATUS);
989 val &= QM_IFC_INT_STATUS_MASK;
990 if (!val)
991 return IRQ_NONE;
992
993 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) {
994 dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n");
995 return IRQ_HANDLED;
996 }
997
998 schedule_work(&qm->cmd_process);
999
1000 return IRQ_HANDLED;
1001 }
1002
qm_set_qp_disable(struct hisi_qp * qp,int offset)1003 static void qm_set_qp_disable(struct hisi_qp *qp, int offset)
1004 {
1005 u32 *addr;
1006
1007 if (qp->is_in_kernel)
1008 return;
1009
1010 addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset;
1011 *addr = 1;
1012
1013 /* make sure setup is completed */
1014 smp_wmb();
1015 }
1016
qm_disable_qp(struct hisi_qm * qm,u32 qp_id)1017 static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id)
1018 {
1019 struct hisi_qp *qp = &qm->qp_array[qp_id];
1020
1021 qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET);
1022 hisi_qm_stop_qp(qp);
1023 qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET);
1024 }
1025
qm_reset_function(struct hisi_qm * qm)1026 static void qm_reset_function(struct hisi_qm *qm)
1027 {
1028 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
1029 struct device *dev = &qm->pdev->dev;
1030 int ret;
1031
1032 if (qm_check_dev_error(pf_qm))
1033 return;
1034
1035 ret = qm_reset_prepare_ready(qm);
1036 if (ret) {
1037 dev_err(dev, "reset function not ready\n");
1038 return;
1039 }
1040
1041 ret = hisi_qm_stop(qm, QM_DOWN);
1042 if (ret) {
1043 dev_err(dev, "failed to stop qm when reset function\n");
1044 goto clear_bit;
1045 }
1046
1047 ret = hisi_qm_start(qm);
1048 if (ret)
1049 dev_err(dev, "failed to start qm when reset function\n");
1050
1051 clear_bit:
1052 qm_reset_bit_clear(qm);
1053 }
1054
qm_aeq_thread(int irq,void * data)1055 static irqreturn_t qm_aeq_thread(int irq, void *data)
1056 {
1057 struct hisi_qm *qm = data;
1058 struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
1059 u16 aeq_depth = qm->aeq_depth;
1060 u32 type, qp_id;
1061
1062 atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
1063
1064 while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
1065 type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
1066 qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK;
1067
1068 switch (type) {
1069 case QM_EQ_OVERFLOW:
1070 dev_err(&qm->pdev->dev, "eq overflow, reset function\n");
1071 qm_reset_function(qm);
1072 return IRQ_HANDLED;
1073 case QM_CQ_OVERFLOW:
1074 dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n",
1075 qp_id);
1076 fallthrough;
1077 case QM_CQE_ERROR:
1078 qm_disable_qp(qm, qp_id);
1079 break;
1080 default:
1081 dev_err(&qm->pdev->dev, "unknown error type %u\n",
1082 type);
1083 break;
1084 }
1085
1086 if (qm->status.aeq_head == aeq_depth - 1) {
1087 qm->status.aeqc_phase = !qm->status.aeqc_phase;
1088 aeqe = qm->aeqe;
1089 qm->status.aeq_head = 0;
1090 } else {
1091 aeqe++;
1092 qm->status.aeq_head++;
1093 }
1094 }
1095
1096 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
1097
1098 return IRQ_HANDLED;
1099 }
1100
qm_init_qp_status(struct hisi_qp * qp)1101 static void qm_init_qp_status(struct hisi_qp *qp)
1102 {
1103 struct hisi_qp_status *qp_status = &qp->qp_status;
1104
1105 qp_status->sq_tail = 0;
1106 qp_status->cq_head = 0;
1107 qp_status->cqc_phase = true;
1108 atomic_set(&qp_status->used, 0);
1109 }
1110
qm_init_prefetch(struct hisi_qm * qm)1111 static void qm_init_prefetch(struct hisi_qm *qm)
1112 {
1113 struct device *dev = &qm->pdev->dev;
1114 u32 page_type = 0x0;
1115
1116 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
1117 return;
1118
1119 switch (PAGE_SIZE) {
1120 case SZ_4K:
1121 page_type = 0x0;
1122 break;
1123 case SZ_16K:
1124 page_type = 0x1;
1125 break;
1126 case SZ_64K:
1127 page_type = 0x2;
1128 break;
1129 default:
1130 dev_err(dev, "system page size is not support: %lu, default set to 4KB",
1131 PAGE_SIZE);
1132 }
1133
1134 writel(page_type, qm->io_base + QM_PAGE_SIZE);
1135 }
1136
1137 /*
1138 * acc_shaper_para_calc() Get the IR value by the qos formula, the return value
1139 * is the expected qos calculated.
1140 * the formula:
1141 * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps
1142 *
1143 * IR_b * (2 ^ IR_u) * 8000
1144 * IR(Mbps) = -------------------------
1145 * Tick * (2 ^ IR_s)
1146 */
acc_shaper_para_calc(u64 cir_b,u64 cir_u,u64 cir_s)1147 static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
1148 {
1149 return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) /
1150 (QM_QOS_TICK * (1 << cir_s));
1151 }
1152
acc_shaper_calc_cbs_s(u32 ir)1153 static u32 acc_shaper_calc_cbs_s(u32 ir)
1154 {
1155 int table_size = ARRAY_SIZE(shaper_cbs_s);
1156 int i;
1157
1158 for (i = 0; i < table_size; i++) {
1159 if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end)
1160 return shaper_cbs_s[i].val;
1161 }
1162
1163 return QM_SHAPER_MIN_CBS_S;
1164 }
1165
acc_shaper_calc_cir_s(u32 ir)1166 static u32 acc_shaper_calc_cir_s(u32 ir)
1167 {
1168 int table_size = ARRAY_SIZE(shaper_cir_s);
1169 int i;
1170
1171 for (i = 0; i < table_size; i++) {
1172 if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end)
1173 return shaper_cir_s[i].val;
1174 }
1175
1176 return 0;
1177 }
1178
qm_get_shaper_para(u32 ir,struct qm_shaper_factor * factor)1179 static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
1180 {
1181 u32 cir_b, cir_u, cir_s, ir_calc;
1182 u32 error_rate;
1183
1184 factor->cbs_s = acc_shaper_calc_cbs_s(ir);
1185 cir_s = acc_shaper_calc_cir_s(ir);
1186
1187 for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {
1188 for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {
1189 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
1190
1191 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
1192 if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
1193 factor->cir_b = cir_b;
1194 factor->cir_u = cir_u;
1195 factor->cir_s = cir_s;
1196 return 0;
1197 }
1198 }
1199 }
1200
1201 return -EINVAL;
1202 }
1203
qm_vft_data_cfg(struct hisi_qm * qm,enum vft_type type,u32 base,u32 number,struct qm_shaper_factor * factor)1204 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
1205 u32 number, struct qm_shaper_factor *factor)
1206 {
1207 u64 tmp = 0;
1208
1209 if (number > 0) {
1210 switch (type) {
1211 case SQC_VFT:
1212 if (qm->ver == QM_HW_V1) {
1213 tmp = QM_SQC_VFT_BUF_SIZE |
1214 QM_SQC_VFT_SQC_SIZE |
1215 QM_SQC_VFT_INDEX_NUMBER |
1216 QM_SQC_VFT_VALID |
1217 (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
1218 } else {
1219 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
1220 QM_SQC_VFT_VALID |
1221 (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
1222 }
1223 break;
1224 case CQC_VFT:
1225 if (qm->ver == QM_HW_V1) {
1226 tmp = QM_CQC_VFT_BUF_SIZE |
1227 QM_CQC_VFT_SQC_SIZE |
1228 QM_CQC_VFT_INDEX_NUMBER |
1229 QM_CQC_VFT_VALID;
1230 } else {
1231 tmp = QM_CQC_VFT_VALID;
1232 }
1233 break;
1234 case SHAPER_VFT:
1235 if (factor) {
1236 tmp = factor->cir_b |
1237 (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) |
1238 (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) |
1239 (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) |
1240 (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT);
1241 }
1242 break;
1243 }
1244 }
1245
1246 writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
1247 writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
1248 }
1249
qm_set_vft_common(struct hisi_qm * qm,enum vft_type type,u32 fun_num,u32 base,u32 number)1250 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
1251 u32 fun_num, u32 base, u32 number)
1252 {
1253 struct qm_shaper_factor *factor = NULL;
1254 unsigned int val;
1255 int ret;
1256
1257 if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
1258 factor = &qm->factor[fun_num];
1259
1260 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1261 val & BIT(0), POLL_PERIOD,
1262 POLL_TIMEOUT);
1263 if (ret)
1264 return ret;
1265
1266 writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
1267 writel(type, qm->io_base + QM_VFT_CFG_TYPE);
1268 if (type == SHAPER_VFT)
1269 fun_num |= base << QM_SHAPER_VFT_OFFSET;
1270
1271 writel(fun_num, qm->io_base + QM_VFT_CFG);
1272
1273 qm_vft_data_cfg(qm, type, base, number, factor);
1274
1275 writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
1276 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
1277
1278 return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1279 val & BIT(0), POLL_PERIOD,
1280 POLL_TIMEOUT);
1281 }
1282
qm_shaper_init_vft(struct hisi_qm * qm,u32 fun_num)1283 static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)
1284 {
1285 u32 qos = qm->factor[fun_num].func_qos;
1286 int ret, i;
1287
1288 ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]);
1289 if (ret) {
1290 dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");
1291 return ret;
1292 }
1293 writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG);
1294 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
1295 /* The base number of queue reuse for different alg type */
1296 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1);
1297 if (ret)
1298 return ret;
1299 }
1300
1301 return 0;
1302 }
1303
1304 /* The config should be conducted after qm_dev_mem_reset() */
qm_set_sqc_cqc_vft(struct hisi_qm * qm,u32 fun_num,u32 base,u32 number)1305 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
1306 u32 number)
1307 {
1308 int ret, i;
1309
1310 for (i = SQC_VFT; i <= CQC_VFT; i++) {
1311 ret = qm_set_vft_common(qm, i, fun_num, base, number);
1312 if (ret)
1313 return ret;
1314 }
1315
1316 /* init default shaper qos val */
1317 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {
1318 ret = qm_shaper_init_vft(qm, fun_num);
1319 if (ret)
1320 goto back_sqc_cqc;
1321 }
1322
1323 return 0;
1324 back_sqc_cqc:
1325 for (i = SQC_VFT; i <= CQC_VFT; i++)
1326 qm_set_vft_common(qm, i, fun_num, 0, 0);
1327
1328 return ret;
1329 }
1330
qm_get_vft_v2(struct hisi_qm * qm,u32 * base,u32 * number)1331 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
1332 {
1333 u64 sqc_vft;
1334 int ret;
1335
1336 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
1337 if (ret)
1338 return ret;
1339
1340 sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
1341 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
1342 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
1343 *number = (QM_SQC_VFT_NUM_MASK_V2 &
1344 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
1345
1346 return 0;
1347 }
1348
hisi_qm_ctx_alloc(struct hisi_qm * qm,size_t ctx_size,dma_addr_t * dma_addr)1349 void *hisi_qm_ctx_alloc(struct hisi_qm *qm, size_t ctx_size,
1350 dma_addr_t *dma_addr)
1351 {
1352 struct device *dev = &qm->pdev->dev;
1353 void *ctx_addr;
1354
1355 ctx_addr = kzalloc(ctx_size, GFP_KERNEL);
1356 if (!ctx_addr)
1357 return ERR_PTR(-ENOMEM);
1358
1359 *dma_addr = dma_map_single(dev, ctx_addr, ctx_size, DMA_FROM_DEVICE);
1360 if (dma_mapping_error(dev, *dma_addr)) {
1361 dev_err(dev, "DMA mapping error!\n");
1362 kfree(ctx_addr);
1363 return ERR_PTR(-ENOMEM);
1364 }
1365
1366 return ctx_addr;
1367 }
1368
hisi_qm_ctx_free(struct hisi_qm * qm,size_t ctx_size,const void * ctx_addr,dma_addr_t * dma_addr)1369 void hisi_qm_ctx_free(struct hisi_qm *qm, size_t ctx_size,
1370 const void *ctx_addr, dma_addr_t *dma_addr)
1371 {
1372 struct device *dev = &qm->pdev->dev;
1373
1374 dma_unmap_single(dev, *dma_addr, ctx_size, DMA_FROM_DEVICE);
1375 kfree(ctx_addr);
1376 }
1377
qm_dump_sqc_raw(struct hisi_qm * qm,dma_addr_t dma_addr,u16 qp_id)1378 static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1379 {
1380 return hisi_qm_mb(qm, QM_MB_CMD_SQC, dma_addr, qp_id, 1);
1381 }
1382
qm_dump_cqc_raw(struct hisi_qm * qm,dma_addr_t dma_addr,u16 qp_id)1383 static int qm_dump_cqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1384 {
1385 return hisi_qm_mb(qm, QM_MB_CMD_CQC, dma_addr, qp_id, 1);
1386 }
1387
qm_hw_error_init_v1(struct hisi_qm * qm)1388 static void qm_hw_error_init_v1(struct hisi_qm *qm)
1389 {
1390 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1391 }
1392
qm_hw_error_cfg(struct hisi_qm * qm)1393 static void qm_hw_error_cfg(struct hisi_qm *qm)
1394 {
1395 struct hisi_qm_err_info *err_info = &qm->err_info;
1396
1397 qm->error_mask = err_info->nfe | err_info->ce | err_info->fe;
1398 /* clear QM hw residual error source */
1399 writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE);
1400
1401 /* configure error type */
1402 writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE);
1403 writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1404 writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1405 writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE);
1406 }
1407
qm_hw_error_init_v2(struct hisi_qm * qm)1408 static void qm_hw_error_init_v2(struct hisi_qm *qm)
1409 {
1410 u32 irq_unmask;
1411
1412 qm_hw_error_cfg(qm);
1413
1414 irq_unmask = ~qm->error_mask;
1415 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1416 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1417 }
1418
qm_hw_error_uninit_v2(struct hisi_qm * qm)1419 static void qm_hw_error_uninit_v2(struct hisi_qm *qm)
1420 {
1421 u32 irq_mask = qm->error_mask;
1422
1423 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1424 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);
1425 }
1426
qm_hw_error_init_v3(struct hisi_qm * qm)1427 static void qm_hw_error_init_v3(struct hisi_qm *qm)
1428 {
1429 u32 irq_unmask;
1430
1431 qm_hw_error_cfg(qm);
1432
1433 /* enable close master ooo when hardware error happened */
1434 writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL);
1435
1436 irq_unmask = ~qm->error_mask;
1437 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1438 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1439 }
1440
qm_hw_error_uninit_v3(struct hisi_qm * qm)1441 static void qm_hw_error_uninit_v3(struct hisi_qm *qm)
1442 {
1443 u32 irq_mask = qm->error_mask;
1444
1445 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1446 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);
1447
1448 /* disable close master ooo when hardware error happened */
1449 writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL);
1450 }
1451
qm_log_hw_error(struct hisi_qm * qm,u32 error_status)1452 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
1453 {
1454 const struct hisi_qm_hw_error *err;
1455 struct device *dev = &qm->pdev->dev;
1456 u32 reg_val, type, vf_num;
1457 int i;
1458
1459 for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
1460 err = &qm_hw_error[i];
1461 if (!(err->int_msk & error_status))
1462 continue;
1463
1464 dev_err(dev, "%s [error status=0x%x] found\n",
1465 err->msg, err->int_msk);
1466
1467 if (err->int_msk & QM_DB_TIMEOUT) {
1468 reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);
1469 type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
1470 QM_DB_TIMEOUT_TYPE_SHIFT;
1471 vf_num = reg_val & QM_DB_TIMEOUT_VF;
1472 dev_err(dev, "qm %s doorbell timeout in function %u\n",
1473 qm_db_timeout[type], vf_num);
1474 } else if (err->int_msk & QM_OF_FIFO_OF) {
1475 reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
1476 type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
1477 QM_FIFO_OVERFLOW_TYPE_SHIFT;
1478 vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
1479
1480 if (type < ARRAY_SIZE(qm_fifo_overflow))
1481 dev_err(dev, "qm %s fifo overflow in function %u\n",
1482 qm_fifo_overflow[type], vf_num);
1483 else
1484 dev_err(dev, "unknown error type\n");
1485 }
1486 }
1487 }
1488
qm_hw_error_handle_v2(struct hisi_qm * qm)1489 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
1490 {
1491 u32 error_status, tmp;
1492
1493 /* read err sts */
1494 tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
1495 error_status = qm->error_mask & tmp;
1496
1497 if (error_status) {
1498 if (error_status & QM_ECC_MBIT)
1499 qm->err_status.is_qm_ecc_mbit = true;
1500
1501 qm_log_hw_error(qm, error_status);
1502 if (error_status & qm->err_info.qm_reset_mask)
1503 return ACC_ERR_NEED_RESET;
1504
1505 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);
1506 writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1507 }
1508
1509 return ACC_ERR_RECOVERED;
1510 }
1511
qm_get_mb_cmd(struct hisi_qm * qm,u64 * msg,u16 fun_num)1512 static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)
1513 {
1514 struct qm_mailbox mailbox;
1515 int ret;
1516
1517 qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0);
1518 mutex_lock(&qm->mailbox_lock);
1519 ret = qm_mb_nolock(qm, &mailbox);
1520 if (ret)
1521 goto err_unlock;
1522
1523 *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
1524 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
1525
1526 err_unlock:
1527 mutex_unlock(&qm->mailbox_lock);
1528 return ret;
1529 }
1530
qm_clear_cmd_interrupt(struct hisi_qm * qm,u64 vf_mask)1531 static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask)
1532 {
1533 u32 val;
1534
1535 if (qm->fun_type == QM_HW_PF)
1536 writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P);
1537
1538 val = readl(qm->io_base + QM_IFC_INT_SOURCE_V);
1539 val |= QM_IFC_INT_SOURCE_MASK;
1540 writel(val, qm->io_base + QM_IFC_INT_SOURCE_V);
1541 }
1542
qm_handle_vf_msg(struct hisi_qm * qm,u32 vf_id)1543 static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id)
1544 {
1545 struct device *dev = &qm->pdev->dev;
1546 u32 cmd;
1547 u64 msg;
1548 int ret;
1549
1550 ret = qm_get_mb_cmd(qm, &msg, vf_id);
1551 if (ret) {
1552 dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id);
1553 return;
1554 }
1555
1556 cmd = msg & QM_MB_CMD_DATA_MASK;
1557 switch (cmd) {
1558 case QM_VF_PREPARE_FAIL:
1559 dev_err(dev, "failed to stop VF(%u)!\n", vf_id);
1560 break;
1561 case QM_VF_START_FAIL:
1562 dev_err(dev, "failed to start VF(%u)!\n", vf_id);
1563 break;
1564 case QM_VF_PREPARE_DONE:
1565 case QM_VF_START_DONE:
1566 break;
1567 default:
1568 dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id);
1569 break;
1570 }
1571 }
1572
qm_wait_vf_prepare_finish(struct hisi_qm * qm)1573 static int qm_wait_vf_prepare_finish(struct hisi_qm *qm)
1574 {
1575 struct device *dev = &qm->pdev->dev;
1576 u32 vfs_num = qm->vfs_num;
1577 int cnt = 0;
1578 int ret = 0;
1579 u64 val;
1580 u32 i;
1581
1582 if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
1583 return 0;
1584
1585 while (true) {
1586 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
1587 /* All VFs send command to PF, break */
1588 if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1))
1589 break;
1590
1591 if (++cnt > QM_MAX_PF_WAIT_COUNT) {
1592 ret = -EBUSY;
1593 break;
1594 }
1595
1596 msleep(QM_WAIT_DST_ACK);
1597 }
1598
1599 /* PF check VFs msg */
1600 for (i = 1; i <= vfs_num; i++) {
1601 if (val & BIT(i))
1602 qm_handle_vf_msg(qm, i);
1603 else
1604 dev_err(dev, "VF(%u) not ping PF!\n", i);
1605 }
1606
1607 /* PF clear interrupt to ack VFs */
1608 qm_clear_cmd_interrupt(qm, val);
1609
1610 return ret;
1611 }
1612
qm_trigger_vf_interrupt(struct hisi_qm * qm,u32 fun_num)1613 static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num)
1614 {
1615 u32 val;
1616
1617 val = readl(qm->io_base + QM_IFC_INT_CFG);
1618 val &= ~QM_IFC_SEND_ALL_VFS;
1619 val |= fun_num;
1620 writel(val, qm->io_base + QM_IFC_INT_CFG);
1621
1622 val = readl(qm->io_base + QM_IFC_INT_SET_P);
1623 val |= QM_IFC_INT_SET_MASK;
1624 writel(val, qm->io_base + QM_IFC_INT_SET_P);
1625 }
1626
qm_trigger_pf_interrupt(struct hisi_qm * qm)1627 static void qm_trigger_pf_interrupt(struct hisi_qm *qm)
1628 {
1629 u32 val;
1630
1631 val = readl(qm->io_base + QM_IFC_INT_SET_V);
1632 val |= QM_IFC_INT_SET_MASK;
1633 writel(val, qm->io_base + QM_IFC_INT_SET_V);
1634 }
1635
qm_ping_single_vf(struct hisi_qm * qm,u64 cmd,u32 fun_num)1636 static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num)
1637 {
1638 struct device *dev = &qm->pdev->dev;
1639 struct qm_mailbox mailbox;
1640 int cnt = 0;
1641 u64 val;
1642 int ret;
1643
1644 qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0);
1645 mutex_lock(&qm->mailbox_lock);
1646 ret = qm_mb_nolock(qm, &mailbox);
1647 if (ret) {
1648 dev_err(dev, "failed to send command to vf(%u)!\n", fun_num);
1649 goto err_unlock;
1650 }
1651
1652 qm_trigger_vf_interrupt(qm, fun_num);
1653 while (true) {
1654 msleep(QM_WAIT_DST_ACK);
1655 val = readq(qm->io_base + QM_IFC_READY_STATUS);
1656 /* if VF respond, PF notifies VF successfully. */
1657 if (!(val & BIT(fun_num)))
1658 goto err_unlock;
1659
1660 if (++cnt > QM_MAX_PF_WAIT_COUNT) {
1661 dev_err(dev, "failed to get response from VF(%u)!\n", fun_num);
1662 ret = -ETIMEDOUT;
1663 break;
1664 }
1665 }
1666
1667 err_unlock:
1668 mutex_unlock(&qm->mailbox_lock);
1669 return ret;
1670 }
1671
qm_ping_all_vfs(struct hisi_qm * qm,u64 cmd)1672 static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd)
1673 {
1674 struct device *dev = &qm->pdev->dev;
1675 u32 vfs_num = qm->vfs_num;
1676 struct qm_mailbox mailbox;
1677 u64 val = 0;
1678 int cnt = 0;
1679 int ret;
1680 u32 i;
1681
1682 qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0);
1683 mutex_lock(&qm->mailbox_lock);
1684 /* PF sends command to all VFs by mailbox */
1685 ret = qm_mb_nolock(qm, &mailbox);
1686 if (ret) {
1687 dev_err(dev, "failed to send command to VFs!\n");
1688 mutex_unlock(&qm->mailbox_lock);
1689 return ret;
1690 }
1691
1692 qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS);
1693 while (true) {
1694 msleep(QM_WAIT_DST_ACK);
1695 val = readq(qm->io_base + QM_IFC_READY_STATUS);
1696 /* If all VFs acked, PF notifies VFs successfully. */
1697 if (!(val & GENMASK(vfs_num, 1))) {
1698 mutex_unlock(&qm->mailbox_lock);
1699 return 0;
1700 }
1701
1702 if (++cnt > QM_MAX_PF_WAIT_COUNT)
1703 break;
1704 }
1705
1706 mutex_unlock(&qm->mailbox_lock);
1707
1708 /* Check which vf respond timeout. */
1709 for (i = 1; i <= vfs_num; i++) {
1710 if (val & BIT(i))
1711 dev_err(dev, "failed to get response from VF(%u)!\n", i);
1712 }
1713
1714 return -ETIMEDOUT;
1715 }
1716
qm_ping_pf(struct hisi_qm * qm,u64 cmd)1717 static int qm_ping_pf(struct hisi_qm *qm, u64 cmd)
1718 {
1719 struct qm_mailbox mailbox;
1720 int cnt = 0;
1721 u32 val;
1722 int ret;
1723
1724 qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0);
1725 mutex_lock(&qm->mailbox_lock);
1726 ret = qm_mb_nolock(qm, &mailbox);
1727 if (ret) {
1728 dev_err(&qm->pdev->dev, "failed to send command to PF!\n");
1729 goto unlock;
1730 }
1731
1732 qm_trigger_pf_interrupt(qm);
1733 /* Waiting for PF response */
1734 while (true) {
1735 msleep(QM_WAIT_DST_ACK);
1736 val = readl(qm->io_base + QM_IFC_INT_SET_V);
1737 if (!(val & QM_IFC_INT_STATUS_MASK))
1738 break;
1739
1740 if (++cnt > QM_MAX_VF_WAIT_COUNT) {
1741 ret = -ETIMEDOUT;
1742 break;
1743 }
1744 }
1745
1746 unlock:
1747 mutex_unlock(&qm->mailbox_lock);
1748 return ret;
1749 }
1750
qm_stop_qp(struct hisi_qp * qp)1751 static int qm_stop_qp(struct hisi_qp *qp)
1752 {
1753 return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);
1754 }
1755
qm_set_msi(struct hisi_qm * qm,bool set)1756 static int qm_set_msi(struct hisi_qm *qm, bool set)
1757 {
1758 struct pci_dev *pdev = qm->pdev;
1759
1760 if (set) {
1761 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
1762 0);
1763 } else {
1764 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
1765 ACC_PEH_MSI_DISABLE);
1766 if (qm->err_status.is_qm_ecc_mbit ||
1767 qm->err_status.is_dev_ecc_mbit)
1768 return 0;
1769
1770 mdelay(1);
1771 if (readl(qm->io_base + QM_PEH_DFX_INFO0))
1772 return -EFAULT;
1773 }
1774
1775 return 0;
1776 }
1777
qm_wait_msi_finish(struct hisi_qm * qm)1778 static void qm_wait_msi_finish(struct hisi_qm *qm)
1779 {
1780 struct pci_dev *pdev = qm->pdev;
1781 u32 cmd = ~0;
1782 int cnt = 0;
1783 u32 val;
1784 int ret;
1785
1786 while (true) {
1787 pci_read_config_dword(pdev, pdev->msi_cap +
1788 PCI_MSI_PENDING_64, &cmd);
1789 if (!cmd)
1790 break;
1791
1792 if (++cnt > MAX_WAIT_COUNTS) {
1793 pci_warn(pdev, "failed to empty MSI PENDING!\n");
1794 break;
1795 }
1796
1797 udelay(1);
1798 }
1799
1800 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0,
1801 val, !(val & QM_PEH_DFX_MASK),
1802 POLL_PERIOD, POLL_TIMEOUT);
1803 if (ret)
1804 pci_warn(pdev, "failed to empty PEH MSI!\n");
1805
1806 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1,
1807 val, !(val & QM_PEH_MSI_FINISH_MASK),
1808 POLL_PERIOD, POLL_TIMEOUT);
1809 if (ret)
1810 pci_warn(pdev, "failed to finish MSI operation!\n");
1811 }
1812
qm_set_msi_v3(struct hisi_qm * qm,bool set)1813 static int qm_set_msi_v3(struct hisi_qm *qm, bool set)
1814 {
1815 struct pci_dev *pdev = qm->pdev;
1816 int ret = -ETIMEDOUT;
1817 u32 cmd, i;
1818
1819 pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
1820 if (set)
1821 cmd |= QM_MSI_CAP_ENABLE;
1822 else
1823 cmd &= ~QM_MSI_CAP_ENABLE;
1824
1825 pci_write_config_dword(pdev, pdev->msi_cap, cmd);
1826 if (set) {
1827 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
1828 pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
1829 if (cmd & QM_MSI_CAP_ENABLE)
1830 return 0;
1831
1832 udelay(1);
1833 }
1834 } else {
1835 udelay(WAIT_PERIOD_US_MIN);
1836 qm_wait_msi_finish(qm);
1837 ret = 0;
1838 }
1839
1840 return ret;
1841 }
1842
1843 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
1844 .qm_db = qm_db_v1,
1845 .hw_error_init = qm_hw_error_init_v1,
1846 .set_msi = qm_set_msi,
1847 };
1848
1849 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
1850 .get_vft = qm_get_vft_v2,
1851 .qm_db = qm_db_v2,
1852 .hw_error_init = qm_hw_error_init_v2,
1853 .hw_error_uninit = qm_hw_error_uninit_v2,
1854 .hw_error_handle = qm_hw_error_handle_v2,
1855 .set_msi = qm_set_msi,
1856 };
1857
1858 static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {
1859 .get_vft = qm_get_vft_v2,
1860 .qm_db = qm_db_v2,
1861 .hw_error_init = qm_hw_error_init_v3,
1862 .hw_error_uninit = qm_hw_error_uninit_v3,
1863 .hw_error_handle = qm_hw_error_handle_v2,
1864 .set_msi = qm_set_msi_v3,
1865 };
1866
qm_get_avail_sqe(struct hisi_qp * qp)1867 static void *qm_get_avail_sqe(struct hisi_qp *qp)
1868 {
1869 struct hisi_qp_status *qp_status = &qp->qp_status;
1870 u16 sq_tail = qp_status->sq_tail;
1871
1872 if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1))
1873 return NULL;
1874
1875 return qp->sqe + sq_tail * qp->qm->sqe_size;
1876 }
1877
hisi_qm_unset_hw_reset(struct hisi_qp * qp)1878 static void hisi_qm_unset_hw_reset(struct hisi_qp *qp)
1879 {
1880 u64 *addr;
1881
1882 /* Use last 64 bits of DUS to reset status. */
1883 addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET;
1884 *addr = 0;
1885 }
1886
qm_create_qp_nolock(struct hisi_qm * qm,u8 alg_type)1887 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
1888 {
1889 struct device *dev = &qm->pdev->dev;
1890 struct hisi_qp *qp;
1891 int qp_id;
1892
1893 if (!qm_qp_avail_state(qm, NULL, QP_INIT))
1894 return ERR_PTR(-EPERM);
1895
1896 if (qm->qp_in_used == qm->qp_num) {
1897 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1898 qm->qp_num);
1899 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1900 return ERR_PTR(-EBUSY);
1901 }
1902
1903 qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);
1904 if (qp_id < 0) {
1905 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1906 qm->qp_num);
1907 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1908 return ERR_PTR(-EBUSY);
1909 }
1910
1911 qp = &qm->qp_array[qp_id];
1912 hisi_qm_unset_hw_reset(qp);
1913 memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth);
1914
1915 qp->event_cb = NULL;
1916 qp->req_cb = NULL;
1917 qp->qp_id = qp_id;
1918 qp->alg_type = alg_type;
1919 qp->is_in_kernel = true;
1920 qm->qp_in_used++;
1921 atomic_set(&qp->qp_status.flags, QP_INIT);
1922
1923 return qp;
1924 }
1925
1926 /**
1927 * hisi_qm_create_qp() - Create a queue pair from qm.
1928 * @qm: The qm we create a qp from.
1929 * @alg_type: Accelerator specific algorithm type in sqc.
1930 *
1931 * Return created qp, negative error code if failed.
1932 */
hisi_qm_create_qp(struct hisi_qm * qm,u8 alg_type)1933 static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
1934 {
1935 struct hisi_qp *qp;
1936 int ret;
1937
1938 ret = qm_pm_get_sync(qm);
1939 if (ret)
1940 return ERR_PTR(ret);
1941
1942 down_write(&qm->qps_lock);
1943 qp = qm_create_qp_nolock(qm, alg_type);
1944 up_write(&qm->qps_lock);
1945
1946 if (IS_ERR(qp))
1947 qm_pm_put_sync(qm);
1948
1949 return qp;
1950 }
1951
1952 /**
1953 * hisi_qm_release_qp() - Release a qp back to its qm.
1954 * @qp: The qp we want to release.
1955 *
1956 * This function releases the resource of a qp.
1957 */
hisi_qm_release_qp(struct hisi_qp * qp)1958 static void hisi_qm_release_qp(struct hisi_qp *qp)
1959 {
1960 struct hisi_qm *qm = qp->qm;
1961
1962 down_write(&qm->qps_lock);
1963
1964 if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) {
1965 up_write(&qm->qps_lock);
1966 return;
1967 }
1968
1969 qm->qp_in_used--;
1970 idr_remove(&qm->qp_idr, qp->qp_id);
1971
1972 up_write(&qm->qps_lock);
1973
1974 qm_pm_put_sync(qm);
1975 }
1976
qm_sq_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)1977 static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1978 {
1979 struct hisi_qm *qm = qp->qm;
1980 struct device *dev = &qm->pdev->dev;
1981 enum qm_hw_ver ver = qm->ver;
1982 struct qm_sqc *sqc;
1983 dma_addr_t sqc_dma;
1984 int ret;
1985
1986 sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL);
1987 if (!sqc)
1988 return -ENOMEM;
1989
1990 INIT_QC_COMMON(sqc, qp->sqe_dma, pasid);
1991 if (ver == QM_HW_V1) {
1992 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));
1993 sqc->w8 = cpu_to_le16(qp->sq_depth - 1);
1994 } else {
1995 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth));
1996 sqc->w8 = 0; /* rand_qc */
1997 }
1998 sqc->cq_num = cpu_to_le16(qp_id);
1999 sqc->w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));
2000
2001 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2002 sqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE <<
2003 QM_QC_PASID_ENABLE_SHIFT);
2004
2005 sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc),
2006 DMA_TO_DEVICE);
2007 if (dma_mapping_error(dev, sqc_dma)) {
2008 kfree(sqc);
2009 return -ENOMEM;
2010 }
2011
2012 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0);
2013 dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE);
2014 kfree(sqc);
2015
2016 return ret;
2017 }
2018
qm_cq_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2019 static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2020 {
2021 struct hisi_qm *qm = qp->qm;
2022 struct device *dev = &qm->pdev->dev;
2023 enum qm_hw_ver ver = qm->ver;
2024 struct qm_cqc *cqc;
2025 dma_addr_t cqc_dma;
2026 int ret;
2027
2028 cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL);
2029 if (!cqc)
2030 return -ENOMEM;
2031
2032 INIT_QC_COMMON(cqc, qp->cqe_dma, pasid);
2033 if (ver == QM_HW_V1) {
2034 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0,
2035 QM_QC_CQE_SIZE));
2036 cqc->w8 = cpu_to_le16(qp->cq_depth - 1);
2037 } else {
2038 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth));
2039 cqc->w8 = 0; /* rand_qc */
2040 }
2041 cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
2042
2043 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2044 cqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE);
2045
2046 cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc),
2047 DMA_TO_DEVICE);
2048 if (dma_mapping_error(dev, cqc_dma)) {
2049 kfree(cqc);
2050 return -ENOMEM;
2051 }
2052
2053 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0);
2054 dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE);
2055 kfree(cqc);
2056
2057 return ret;
2058 }
2059
qm_qp_ctx_cfg(struct hisi_qp * qp,int qp_id,u32 pasid)2060 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2061 {
2062 int ret;
2063
2064 qm_init_qp_status(qp);
2065
2066 ret = qm_sq_ctx_cfg(qp, qp_id, pasid);
2067 if (ret)
2068 return ret;
2069
2070 return qm_cq_ctx_cfg(qp, qp_id, pasid);
2071 }
2072
qm_start_qp_nolock(struct hisi_qp * qp,unsigned long arg)2073 static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
2074 {
2075 struct hisi_qm *qm = qp->qm;
2076 struct device *dev = &qm->pdev->dev;
2077 int qp_id = qp->qp_id;
2078 u32 pasid = arg;
2079 int ret;
2080
2081 if (!qm_qp_avail_state(qm, qp, QP_START))
2082 return -EPERM;
2083
2084 ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
2085 if (ret)
2086 return ret;
2087
2088 atomic_set(&qp->qp_status.flags, QP_START);
2089 dev_dbg(dev, "queue %d started\n", qp_id);
2090
2091 return 0;
2092 }
2093
2094 /**
2095 * hisi_qm_start_qp() - Start a qp into running.
2096 * @qp: The qp we want to start to run.
2097 * @arg: Accelerator specific argument.
2098 *
2099 * After this function, qp can receive request from user. Return 0 if
2100 * successful, negative error code if failed.
2101 */
hisi_qm_start_qp(struct hisi_qp * qp,unsigned long arg)2102 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
2103 {
2104 struct hisi_qm *qm = qp->qm;
2105 int ret;
2106
2107 down_write(&qm->qps_lock);
2108 ret = qm_start_qp_nolock(qp, arg);
2109 up_write(&qm->qps_lock);
2110
2111 return ret;
2112 }
2113 EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
2114
2115 /**
2116 * qp_stop_fail_cb() - call request cb.
2117 * @qp: stopped failed qp.
2118 *
2119 * Callback function should be called whether task completed or not.
2120 */
qp_stop_fail_cb(struct hisi_qp * qp)2121 static void qp_stop_fail_cb(struct hisi_qp *qp)
2122 {
2123 int qp_used = atomic_read(&qp->qp_status.used);
2124 u16 cur_tail = qp->qp_status.sq_tail;
2125 u16 sq_depth = qp->sq_depth;
2126 u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth;
2127 struct hisi_qm *qm = qp->qm;
2128 u16 pos;
2129 int i;
2130
2131 for (i = 0; i < qp_used; i++) {
2132 pos = (i + cur_head) % sq_depth;
2133 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));
2134 atomic_dec(&qp->qp_status.used);
2135 }
2136 }
2137
2138 /**
2139 * qm_drain_qp() - Drain a qp.
2140 * @qp: The qp we want to drain.
2141 *
2142 * Determine whether the queue is cleared by judging the tail pointers of
2143 * sq and cq.
2144 */
qm_drain_qp(struct hisi_qp * qp)2145 static int qm_drain_qp(struct hisi_qp *qp)
2146 {
2147 size_t size = sizeof(struct qm_sqc) + sizeof(struct qm_cqc);
2148 struct hisi_qm *qm = qp->qm;
2149 struct device *dev = &qm->pdev->dev;
2150 struct qm_sqc *sqc;
2151 struct qm_cqc *cqc;
2152 dma_addr_t dma_addr;
2153 int ret = 0, i = 0;
2154 void *addr;
2155
2156 /* No need to judge if master OOO is blocked. */
2157 if (qm_check_dev_error(qm))
2158 return 0;
2159
2160 /* Kunpeng930 supports drain qp by device */
2161 if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) {
2162 ret = qm_stop_qp(qp);
2163 if (ret)
2164 dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id);
2165 return ret;
2166 }
2167
2168 addr = hisi_qm_ctx_alloc(qm, size, &dma_addr);
2169 if (IS_ERR(addr)) {
2170 dev_err(dev, "Failed to alloc ctx for sqc and cqc!\n");
2171 return -ENOMEM;
2172 }
2173
2174 while (++i) {
2175 ret = qm_dump_sqc_raw(qm, dma_addr, qp->qp_id);
2176 if (ret) {
2177 dev_err_ratelimited(dev, "Failed to dump sqc!\n");
2178 break;
2179 }
2180 sqc = addr;
2181
2182 ret = qm_dump_cqc_raw(qm, (dma_addr + sizeof(struct qm_sqc)),
2183 qp->qp_id);
2184 if (ret) {
2185 dev_err_ratelimited(dev, "Failed to dump cqc!\n");
2186 break;
2187 }
2188 cqc = addr + sizeof(struct qm_sqc);
2189
2190 if ((sqc->tail == cqc->tail) &&
2191 (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))
2192 break;
2193
2194 if (i == MAX_WAIT_COUNTS) {
2195 dev_err(dev, "Fail to empty queue %u!\n", qp->qp_id);
2196 ret = -EBUSY;
2197 break;
2198 }
2199
2200 usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);
2201 }
2202
2203 hisi_qm_ctx_free(qm, size, addr, &dma_addr);
2204
2205 return ret;
2206 }
2207
qm_stop_qp_nolock(struct hisi_qp * qp)2208 static int qm_stop_qp_nolock(struct hisi_qp *qp)
2209 {
2210 struct device *dev = &qp->qm->pdev->dev;
2211 int ret;
2212
2213 /*
2214 * It is allowed to stop and release qp when reset, If the qp is
2215 * stopped when reset but still want to be released then, the
2216 * is_resetting flag should be set negative so that this qp will not
2217 * be restarted after reset.
2218 */
2219 if (atomic_read(&qp->qp_status.flags) == QP_STOP) {
2220 qp->is_resetting = false;
2221 return 0;
2222 }
2223
2224 if (!qm_qp_avail_state(qp->qm, qp, QP_STOP))
2225 return -EPERM;
2226
2227 atomic_set(&qp->qp_status.flags, QP_STOP);
2228
2229 ret = qm_drain_qp(qp);
2230 if (ret)
2231 dev_err(dev, "Failed to drain out data for stopping!\n");
2232
2233
2234 flush_workqueue(qp->qm->wq);
2235 if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))
2236 qp_stop_fail_cb(qp);
2237
2238 dev_dbg(dev, "stop queue %u!", qp->qp_id);
2239
2240 return 0;
2241 }
2242
2243 /**
2244 * hisi_qm_stop_qp() - Stop a qp in qm.
2245 * @qp: The qp we want to stop.
2246 *
2247 * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
2248 */
hisi_qm_stop_qp(struct hisi_qp * qp)2249 int hisi_qm_stop_qp(struct hisi_qp *qp)
2250 {
2251 int ret;
2252
2253 down_write(&qp->qm->qps_lock);
2254 ret = qm_stop_qp_nolock(qp);
2255 up_write(&qp->qm->qps_lock);
2256
2257 return ret;
2258 }
2259 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
2260
2261 /**
2262 * hisi_qp_send() - Queue up a task in the hardware queue.
2263 * @qp: The qp in which to put the message.
2264 * @msg: The message.
2265 *
2266 * This function will return -EBUSY if qp is currently full, and -EAGAIN
2267 * if qp related qm is resetting.
2268 *
2269 * Note: This function may run with qm_irq_thread and ACC reset at same time.
2270 * It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
2271 * reset may happen, we have no lock here considering performance. This
2272 * causes current qm_db sending fail or can not receive sended sqe. QM
2273 * sync/async receive function should handle the error sqe. ACC reset
2274 * done function should clear used sqe to 0.
2275 */
hisi_qp_send(struct hisi_qp * qp,const void * msg)2276 int hisi_qp_send(struct hisi_qp *qp, const void *msg)
2277 {
2278 struct hisi_qp_status *qp_status = &qp->qp_status;
2279 u16 sq_tail = qp_status->sq_tail;
2280 u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth;
2281 void *sqe = qm_get_avail_sqe(qp);
2282
2283 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
2284 atomic_read(&qp->qm->status.flags) == QM_STOP ||
2285 qp->is_resetting)) {
2286 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
2287 return -EAGAIN;
2288 }
2289
2290 if (!sqe)
2291 return -EBUSY;
2292
2293 memcpy(sqe, msg, qp->qm->sqe_size);
2294
2295 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
2296 atomic_inc(&qp->qp_status.used);
2297 qp_status->sq_tail = sq_tail_next;
2298
2299 return 0;
2300 }
2301 EXPORT_SYMBOL_GPL(hisi_qp_send);
2302
hisi_qm_cache_wb(struct hisi_qm * qm)2303 static void hisi_qm_cache_wb(struct hisi_qm *qm)
2304 {
2305 unsigned int val;
2306
2307 if (qm->ver == QM_HW_V1)
2308 return;
2309
2310 writel(0x1, qm->io_base + QM_CACHE_WB_START);
2311 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
2312 val, val & BIT(0), POLL_PERIOD,
2313 POLL_TIMEOUT))
2314 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
2315 }
2316
qm_qp_event_notifier(struct hisi_qp * qp)2317 static void qm_qp_event_notifier(struct hisi_qp *qp)
2318 {
2319 wake_up_interruptible(&qp->uacce_q->wait);
2320 }
2321
2322 /* This function returns free number of qp in qm. */
hisi_qm_get_available_instances(struct uacce_device * uacce)2323 static int hisi_qm_get_available_instances(struct uacce_device *uacce)
2324 {
2325 struct hisi_qm *qm = uacce->priv;
2326 int ret;
2327
2328 down_read(&qm->qps_lock);
2329 ret = qm->qp_num - qm->qp_in_used;
2330 up_read(&qm->qps_lock);
2331
2332 return ret;
2333 }
2334
hisi_qm_set_hw_reset(struct hisi_qm * qm,int offset)2335 static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset)
2336 {
2337 int i;
2338
2339 for (i = 0; i < qm->qp_num; i++)
2340 qm_set_qp_disable(&qm->qp_array[i], offset);
2341 }
2342
hisi_qm_uacce_get_queue(struct uacce_device * uacce,unsigned long arg,struct uacce_queue * q)2343 static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,
2344 unsigned long arg,
2345 struct uacce_queue *q)
2346 {
2347 struct hisi_qm *qm = uacce->priv;
2348 struct hisi_qp *qp;
2349 u8 alg_type = 0;
2350
2351 qp = hisi_qm_create_qp(qm, alg_type);
2352 if (IS_ERR(qp))
2353 return PTR_ERR(qp);
2354
2355 q->priv = qp;
2356 q->uacce = uacce;
2357 qp->uacce_q = q;
2358 qp->event_cb = qm_qp_event_notifier;
2359 qp->pasid = arg;
2360 qp->is_in_kernel = false;
2361
2362 return 0;
2363 }
2364
hisi_qm_uacce_put_queue(struct uacce_queue * q)2365 static void hisi_qm_uacce_put_queue(struct uacce_queue *q)
2366 {
2367 struct hisi_qp *qp = q->priv;
2368
2369 hisi_qm_release_qp(qp);
2370 }
2371
2372 /* map sq/cq/doorbell to user space */
hisi_qm_uacce_mmap(struct uacce_queue * q,struct vm_area_struct * vma,struct uacce_qfile_region * qfr)2373 static int hisi_qm_uacce_mmap(struct uacce_queue *q,
2374 struct vm_area_struct *vma,
2375 struct uacce_qfile_region *qfr)
2376 {
2377 struct hisi_qp *qp = q->priv;
2378 struct hisi_qm *qm = qp->qm;
2379 resource_size_t phys_base = qm->db_phys_base +
2380 qp->qp_id * qm->db_interval;
2381 size_t sz = vma->vm_end - vma->vm_start;
2382 struct pci_dev *pdev = qm->pdev;
2383 struct device *dev = &pdev->dev;
2384 unsigned long vm_pgoff;
2385 int ret;
2386
2387 switch (qfr->type) {
2388 case UACCE_QFRT_MMIO:
2389 if (qm->ver == QM_HW_V1) {
2390 if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
2391 return -EINVAL;
2392 } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {
2393 if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
2394 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
2395 return -EINVAL;
2396 } else {
2397 if (sz > qm->db_interval)
2398 return -EINVAL;
2399 }
2400
2401 vm_flags_set(vma, VM_IO);
2402
2403 return remap_pfn_range(vma, vma->vm_start,
2404 phys_base >> PAGE_SHIFT,
2405 sz, pgprot_noncached(vma->vm_page_prot));
2406 case UACCE_QFRT_DUS:
2407 if (sz != qp->qdma.size)
2408 return -EINVAL;
2409
2410 /*
2411 * dma_mmap_coherent() requires vm_pgoff as 0
2412 * restore vm_pfoff to initial value for mmap()
2413 */
2414 vm_pgoff = vma->vm_pgoff;
2415 vma->vm_pgoff = 0;
2416 ret = dma_mmap_coherent(dev, vma, qp->qdma.va,
2417 qp->qdma.dma, sz);
2418 vma->vm_pgoff = vm_pgoff;
2419 return ret;
2420
2421 default:
2422 return -EINVAL;
2423 }
2424 }
2425
hisi_qm_uacce_start_queue(struct uacce_queue * q)2426 static int hisi_qm_uacce_start_queue(struct uacce_queue *q)
2427 {
2428 struct hisi_qp *qp = q->priv;
2429
2430 return hisi_qm_start_qp(qp, qp->pasid);
2431 }
2432
hisi_qm_uacce_stop_queue(struct uacce_queue * q)2433 static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)
2434 {
2435 hisi_qm_stop_qp(q->priv);
2436 }
2437
hisi_qm_is_q_updated(struct uacce_queue * q)2438 static int hisi_qm_is_q_updated(struct uacce_queue *q)
2439 {
2440 struct hisi_qp *qp = q->priv;
2441 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
2442 int updated = 0;
2443
2444 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
2445 /* make sure to read data from memory */
2446 dma_rmb();
2447 qm_cq_head_update(qp);
2448 cqe = qp->cqe + qp->qp_status.cq_head;
2449 updated = 1;
2450 }
2451
2452 return updated;
2453 }
2454
qm_set_sqctype(struct uacce_queue * q,u16 type)2455 static void qm_set_sqctype(struct uacce_queue *q, u16 type)
2456 {
2457 struct hisi_qm *qm = q->uacce->priv;
2458 struct hisi_qp *qp = q->priv;
2459
2460 down_write(&qm->qps_lock);
2461 qp->alg_type = type;
2462 up_write(&qm->qps_lock);
2463 }
2464
hisi_qm_uacce_ioctl(struct uacce_queue * q,unsigned int cmd,unsigned long arg)2465 static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,
2466 unsigned long arg)
2467 {
2468 struct hisi_qp *qp = q->priv;
2469 struct hisi_qp_info qp_info;
2470 struct hisi_qp_ctx qp_ctx;
2471
2472 if (cmd == UACCE_CMD_QM_SET_QP_CTX) {
2473 if (copy_from_user(&qp_ctx, (void __user *)arg,
2474 sizeof(struct hisi_qp_ctx)))
2475 return -EFAULT;
2476
2477 if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1)
2478 return -EINVAL;
2479
2480 qm_set_sqctype(q, qp_ctx.qc_type);
2481 qp_ctx.id = qp->qp_id;
2482
2483 if (copy_to_user((void __user *)arg, &qp_ctx,
2484 sizeof(struct hisi_qp_ctx)))
2485 return -EFAULT;
2486
2487 return 0;
2488 } else if (cmd == UACCE_CMD_QM_SET_QP_INFO) {
2489 if (copy_from_user(&qp_info, (void __user *)arg,
2490 sizeof(struct hisi_qp_info)))
2491 return -EFAULT;
2492
2493 qp_info.sqe_size = qp->qm->sqe_size;
2494 qp_info.sq_depth = qp->sq_depth;
2495 qp_info.cq_depth = qp->cq_depth;
2496
2497 if (copy_to_user((void __user *)arg, &qp_info,
2498 sizeof(struct hisi_qp_info)))
2499 return -EFAULT;
2500
2501 return 0;
2502 }
2503
2504 return -EINVAL;
2505 }
2506
2507 /**
2508 * qm_hw_err_isolate() - Try to set the isolation status of the uacce device
2509 * according to user's configuration of error threshold.
2510 * @qm: the uacce device
2511 */
qm_hw_err_isolate(struct hisi_qm * qm)2512 static int qm_hw_err_isolate(struct hisi_qm *qm)
2513 {
2514 struct qm_hw_err *err, *tmp, *hw_err;
2515 struct qm_err_isolate *isolate;
2516 u32 count = 0;
2517
2518 isolate = &qm->isolate_data;
2519
2520 #define SECONDS_PER_HOUR 3600
2521
2522 /* All the hw errs are processed by PF driver */
2523 if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold)
2524 return 0;
2525
2526 hw_err = kzalloc(sizeof(*hw_err), GFP_KERNEL);
2527 if (!hw_err)
2528 return -ENOMEM;
2529
2530 /*
2531 * Time-stamp every slot AER error. Then check the AER error log when the
2532 * next device AER error occurred. if the device slot AER error count exceeds
2533 * the setting error threshold in one hour, the isolated state will be set
2534 * to true. And the AER error logs that exceed one hour will be cleared.
2535 */
2536 mutex_lock(&isolate->isolate_lock);
2537 hw_err->timestamp = jiffies;
2538 list_for_each_entry_safe(err, tmp, &isolate->qm_hw_errs, list) {
2539 if ((hw_err->timestamp - err->timestamp) / HZ >
2540 SECONDS_PER_HOUR) {
2541 list_del(&err->list);
2542 kfree(err);
2543 } else {
2544 count++;
2545 }
2546 }
2547 list_add(&hw_err->list, &isolate->qm_hw_errs);
2548 mutex_unlock(&isolate->isolate_lock);
2549
2550 if (count >= isolate->err_threshold)
2551 isolate->is_isolate = true;
2552
2553 return 0;
2554 }
2555
qm_hw_err_destroy(struct hisi_qm * qm)2556 static void qm_hw_err_destroy(struct hisi_qm *qm)
2557 {
2558 struct qm_hw_err *err, *tmp;
2559
2560 mutex_lock(&qm->isolate_data.isolate_lock);
2561 list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) {
2562 list_del(&err->list);
2563 kfree(err);
2564 }
2565 mutex_unlock(&qm->isolate_data.isolate_lock);
2566 }
2567
hisi_qm_get_isolate_state(struct uacce_device * uacce)2568 static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce)
2569 {
2570 struct hisi_qm *qm = uacce->priv;
2571 struct hisi_qm *pf_qm;
2572
2573 if (uacce->is_vf)
2574 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
2575 else
2576 pf_qm = qm;
2577
2578 return pf_qm->isolate_data.is_isolate ?
2579 UACCE_DEV_ISOLATE : UACCE_DEV_NORMAL;
2580 }
2581
hisi_qm_isolate_threshold_write(struct uacce_device * uacce,u32 num)2582 static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num)
2583 {
2584 struct hisi_qm *qm = uacce->priv;
2585
2586 /* Must be set by PF */
2587 if (uacce->is_vf)
2588 return -EPERM;
2589
2590 if (qm->isolate_data.is_isolate)
2591 return -EPERM;
2592
2593 qm->isolate_data.err_threshold = num;
2594
2595 /* After the policy is updated, need to reset the hardware err list */
2596 qm_hw_err_destroy(qm);
2597
2598 return 0;
2599 }
2600
hisi_qm_isolate_threshold_read(struct uacce_device * uacce)2601 static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce)
2602 {
2603 struct hisi_qm *qm = uacce->priv;
2604 struct hisi_qm *pf_qm;
2605
2606 if (uacce->is_vf) {
2607 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
2608 return pf_qm->isolate_data.err_threshold;
2609 }
2610
2611 return qm->isolate_data.err_threshold;
2612 }
2613
2614 static const struct uacce_ops uacce_qm_ops = {
2615 .get_available_instances = hisi_qm_get_available_instances,
2616 .get_queue = hisi_qm_uacce_get_queue,
2617 .put_queue = hisi_qm_uacce_put_queue,
2618 .start_queue = hisi_qm_uacce_start_queue,
2619 .stop_queue = hisi_qm_uacce_stop_queue,
2620 .mmap = hisi_qm_uacce_mmap,
2621 .ioctl = hisi_qm_uacce_ioctl,
2622 .is_q_updated = hisi_qm_is_q_updated,
2623 .get_isolate_state = hisi_qm_get_isolate_state,
2624 .isolate_err_threshold_write = hisi_qm_isolate_threshold_write,
2625 .isolate_err_threshold_read = hisi_qm_isolate_threshold_read,
2626 };
2627
qm_remove_uacce(struct hisi_qm * qm)2628 static void qm_remove_uacce(struct hisi_qm *qm)
2629 {
2630 struct uacce_device *uacce = qm->uacce;
2631
2632 if (qm->use_sva) {
2633 qm_hw_err_destroy(qm);
2634 uacce_remove(uacce);
2635 qm->uacce = NULL;
2636 }
2637 }
2638
qm_alloc_uacce(struct hisi_qm * qm)2639 static int qm_alloc_uacce(struct hisi_qm *qm)
2640 {
2641 struct pci_dev *pdev = qm->pdev;
2642 struct uacce_device *uacce;
2643 unsigned long mmio_page_nr;
2644 unsigned long dus_page_nr;
2645 u16 sq_depth, cq_depth;
2646 struct uacce_interface interface = {
2647 .flags = UACCE_DEV_SVA,
2648 .ops = &uacce_qm_ops,
2649 };
2650 int ret;
2651
2652 ret = strscpy(interface.name, dev_driver_string(&pdev->dev),
2653 sizeof(interface.name));
2654 if (ret < 0)
2655 return -ENAMETOOLONG;
2656
2657 uacce = uacce_alloc(&pdev->dev, &interface);
2658 if (IS_ERR(uacce))
2659 return PTR_ERR(uacce);
2660
2661 if (uacce->flags & UACCE_DEV_SVA) {
2662 qm->use_sva = true;
2663 } else {
2664 /* only consider sva case */
2665 qm_remove_uacce(qm);
2666 return -EINVAL;
2667 }
2668
2669 uacce->is_vf = pdev->is_virtfn;
2670 uacce->priv = qm;
2671
2672 if (qm->ver == QM_HW_V1)
2673 uacce->api_ver = HISI_QM_API_VER_BASE;
2674 else if (qm->ver == QM_HW_V2)
2675 uacce->api_ver = HISI_QM_API_VER2_BASE;
2676 else
2677 uacce->api_ver = HISI_QM_API_VER3_BASE;
2678
2679 if (qm->ver == QM_HW_V1)
2680 mmio_page_nr = QM_DOORBELL_PAGE_NR;
2681 else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
2682 mmio_page_nr = QM_DOORBELL_PAGE_NR +
2683 QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
2684 else
2685 mmio_page_nr = qm->db_interval / PAGE_SIZE;
2686
2687 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);
2688
2689 /* Add one more page for device or qp status */
2690 dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth +
2691 sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >>
2692 PAGE_SHIFT;
2693
2694 uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;
2695 uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr;
2696
2697 qm->uacce = uacce;
2698 INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs);
2699 mutex_init(&qm->isolate_data.isolate_lock);
2700
2701 return 0;
2702 }
2703
2704 /**
2705 * qm_frozen() - Try to froze QM to cut continuous queue request. If
2706 * there is user on the QM, return failure without doing anything.
2707 * @qm: The qm needed to be fronzen.
2708 *
2709 * This function frozes QM, then we can do SRIOV disabling.
2710 */
qm_frozen(struct hisi_qm * qm)2711 static int qm_frozen(struct hisi_qm *qm)
2712 {
2713 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl))
2714 return 0;
2715
2716 down_write(&qm->qps_lock);
2717
2718 if (!qm->qp_in_used) {
2719 qm->qp_in_used = qm->qp_num;
2720 up_write(&qm->qps_lock);
2721 set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl);
2722 return 0;
2723 }
2724
2725 up_write(&qm->qps_lock);
2726
2727 return -EBUSY;
2728 }
2729
qm_try_frozen_vfs(struct pci_dev * pdev,struct hisi_qm_list * qm_list)2730 static int qm_try_frozen_vfs(struct pci_dev *pdev,
2731 struct hisi_qm_list *qm_list)
2732 {
2733 struct hisi_qm *qm, *vf_qm;
2734 struct pci_dev *dev;
2735 int ret = 0;
2736
2737 if (!qm_list || !pdev)
2738 return -EINVAL;
2739
2740 /* Try to frozen all the VFs as disable SRIOV */
2741 mutex_lock(&qm_list->lock);
2742 list_for_each_entry(qm, &qm_list->list, list) {
2743 dev = qm->pdev;
2744 if (dev == pdev)
2745 continue;
2746 if (pci_physfn(dev) == pdev) {
2747 vf_qm = pci_get_drvdata(dev);
2748 ret = qm_frozen(vf_qm);
2749 if (ret)
2750 goto frozen_fail;
2751 }
2752 }
2753
2754 frozen_fail:
2755 mutex_unlock(&qm_list->lock);
2756
2757 return ret;
2758 }
2759
2760 /**
2761 * hisi_qm_wait_task_finish() - Wait until the task is finished
2762 * when removing the driver.
2763 * @qm: The qm needed to wait for the task to finish.
2764 * @qm_list: The list of all available devices.
2765 */
hisi_qm_wait_task_finish(struct hisi_qm * qm,struct hisi_qm_list * qm_list)2766 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
2767 {
2768 while (qm_frozen(qm) ||
2769 ((qm->fun_type == QM_HW_PF) &&
2770 qm_try_frozen_vfs(qm->pdev, qm_list))) {
2771 msleep(WAIT_PERIOD);
2772 }
2773
2774 while (test_bit(QM_RST_SCHED, &qm->misc_ctl) ||
2775 test_bit(QM_RESETTING, &qm->misc_ctl))
2776 msleep(WAIT_PERIOD);
2777
2778 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2779 flush_work(&qm->cmd_process);
2780
2781 udelay(REMOVE_WAIT_DELAY);
2782 }
2783 EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
2784
hisi_qp_memory_uninit(struct hisi_qm * qm,int num)2785 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
2786 {
2787 struct device *dev = &qm->pdev->dev;
2788 struct qm_dma *qdma;
2789 int i;
2790
2791 for (i = num - 1; i >= 0; i--) {
2792 qdma = &qm->qp_array[i].qdma;
2793 dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
2794 kfree(qm->poll_data[i].qp_finish_id);
2795 }
2796
2797 kfree(qm->poll_data);
2798 kfree(qm->qp_array);
2799 }
2800
hisi_qp_memory_init(struct hisi_qm * qm,size_t dma_size,int id,u16 sq_depth,u16 cq_depth)2801 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id,
2802 u16 sq_depth, u16 cq_depth)
2803 {
2804 struct device *dev = &qm->pdev->dev;
2805 size_t off = qm->sqe_size * sq_depth;
2806 struct hisi_qp *qp;
2807 int ret = -ENOMEM;
2808
2809 qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16),
2810 GFP_KERNEL);
2811 if (!qm->poll_data[id].qp_finish_id)
2812 return -ENOMEM;
2813
2814 qp = &qm->qp_array[id];
2815 qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
2816 GFP_KERNEL);
2817 if (!qp->qdma.va)
2818 goto err_free_qp_finish_id;
2819
2820 qp->sqe = qp->qdma.va;
2821 qp->sqe_dma = qp->qdma.dma;
2822 qp->cqe = qp->qdma.va + off;
2823 qp->cqe_dma = qp->qdma.dma + off;
2824 qp->qdma.size = dma_size;
2825 qp->sq_depth = sq_depth;
2826 qp->cq_depth = cq_depth;
2827 qp->qm = qm;
2828 qp->qp_id = id;
2829
2830 return 0;
2831
2832 err_free_qp_finish_id:
2833 kfree(qm->poll_data[id].qp_finish_id);
2834 return ret;
2835 }
2836
hisi_qm_pre_init(struct hisi_qm * qm)2837 static void hisi_qm_pre_init(struct hisi_qm *qm)
2838 {
2839 struct pci_dev *pdev = qm->pdev;
2840
2841 if (qm->ver == QM_HW_V1)
2842 qm->ops = &qm_hw_ops_v1;
2843 else if (qm->ver == QM_HW_V2)
2844 qm->ops = &qm_hw_ops_v2;
2845 else
2846 qm->ops = &qm_hw_ops_v3;
2847
2848 pci_set_drvdata(pdev, qm);
2849 mutex_init(&qm->mailbox_lock);
2850 init_rwsem(&qm->qps_lock);
2851 qm->qp_in_used = 0;
2852 if (test_bit(QM_SUPPORT_RPM, &qm->caps)) {
2853 if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev)))
2854 dev_info(&pdev->dev, "_PS0 and _PR0 are not defined");
2855 }
2856 }
2857
qm_cmd_uninit(struct hisi_qm * qm)2858 static void qm_cmd_uninit(struct hisi_qm *qm)
2859 {
2860 u32 val;
2861
2862 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2863 return;
2864
2865 val = readl(qm->io_base + QM_IFC_INT_MASK);
2866 val |= QM_IFC_INT_DISABLE;
2867 writel(val, qm->io_base + QM_IFC_INT_MASK);
2868 }
2869
qm_cmd_init(struct hisi_qm * qm)2870 static void qm_cmd_init(struct hisi_qm *qm)
2871 {
2872 u32 val;
2873
2874 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
2875 return;
2876
2877 /* Clear communication interrupt source */
2878 qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR);
2879
2880 /* Enable pf to vf communication reg. */
2881 val = readl(qm->io_base + QM_IFC_INT_MASK);
2882 val &= ~QM_IFC_INT_DISABLE;
2883 writel(val, qm->io_base + QM_IFC_INT_MASK);
2884 }
2885
qm_put_pci_res(struct hisi_qm * qm)2886 static void qm_put_pci_res(struct hisi_qm *qm)
2887 {
2888 struct pci_dev *pdev = qm->pdev;
2889
2890 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
2891 iounmap(qm->db_io_base);
2892
2893 iounmap(qm->io_base);
2894 pci_release_mem_regions(pdev);
2895 }
2896
hisi_qm_pci_uninit(struct hisi_qm * qm)2897 static void hisi_qm_pci_uninit(struct hisi_qm *qm)
2898 {
2899 struct pci_dev *pdev = qm->pdev;
2900
2901 pci_free_irq_vectors(pdev);
2902 qm_put_pci_res(qm);
2903 pci_disable_device(pdev);
2904 }
2905
hisi_qm_set_state(struct hisi_qm * qm,u8 state)2906 static void hisi_qm_set_state(struct hisi_qm *qm, u8 state)
2907 {
2908 if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF)
2909 writel(state, qm->io_base + QM_VF_STATE);
2910 }
2911
hisi_qm_unint_work(struct hisi_qm * qm)2912 static void hisi_qm_unint_work(struct hisi_qm *qm)
2913 {
2914 destroy_workqueue(qm->wq);
2915 }
2916
hisi_qm_memory_uninit(struct hisi_qm * qm)2917 static void hisi_qm_memory_uninit(struct hisi_qm *qm)
2918 {
2919 struct device *dev = &qm->pdev->dev;
2920
2921 hisi_qp_memory_uninit(qm, qm->qp_num);
2922 if (qm->qdma.va) {
2923 hisi_qm_cache_wb(qm);
2924 dma_free_coherent(dev, qm->qdma.size,
2925 qm->qdma.va, qm->qdma.dma);
2926 }
2927
2928 idr_destroy(&qm->qp_idr);
2929
2930 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
2931 kfree(qm->factor);
2932 }
2933
2934 /**
2935 * hisi_qm_uninit() - Uninitialize qm.
2936 * @qm: The qm needed uninit.
2937 *
2938 * This function uninits qm related device resources.
2939 */
hisi_qm_uninit(struct hisi_qm * qm)2940 void hisi_qm_uninit(struct hisi_qm *qm)
2941 {
2942 qm_cmd_uninit(qm);
2943 hisi_qm_unint_work(qm);
2944 down_write(&qm->qps_lock);
2945
2946 if (!qm_avail_state(qm, QM_CLOSE)) {
2947 up_write(&qm->qps_lock);
2948 return;
2949 }
2950
2951 hisi_qm_memory_uninit(qm);
2952 hisi_qm_set_state(qm, QM_NOT_READY);
2953 up_write(&qm->qps_lock);
2954
2955 qm_remove_uacce(qm);
2956 qm_irqs_unregister(qm);
2957 hisi_qm_pci_uninit(qm);
2958 }
2959 EXPORT_SYMBOL_GPL(hisi_qm_uninit);
2960
2961 /**
2962 * hisi_qm_get_vft() - Get vft from a qm.
2963 * @qm: The qm we want to get its vft.
2964 * @base: The base number of queue in vft.
2965 * @number: The number of queues in vft.
2966 *
2967 * We can allocate multiple queues to a qm by configuring virtual function
2968 * table. We get related configures by this function. Normally, we call this
2969 * function in VF driver to get the queue information.
2970 *
2971 * qm hw v1 does not support this interface.
2972 */
hisi_qm_get_vft(struct hisi_qm * qm,u32 * base,u32 * number)2973 static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
2974 {
2975 if (!base || !number)
2976 return -EINVAL;
2977
2978 if (!qm->ops->get_vft) {
2979 dev_err(&qm->pdev->dev, "Don't support vft read!\n");
2980 return -EINVAL;
2981 }
2982
2983 return qm->ops->get_vft(qm, base, number);
2984 }
2985
2986 /**
2987 * hisi_qm_set_vft() - Set vft to a qm.
2988 * @qm: The qm we want to set its vft.
2989 * @fun_num: The function number.
2990 * @base: The base number of queue in vft.
2991 * @number: The number of queues in vft.
2992 *
2993 * This function is alway called in PF driver, it is used to assign queues
2994 * among PF and VFs.
2995 *
2996 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
2997 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
2998 * (VF function number 0x2)
2999 */
hisi_qm_set_vft(struct hisi_qm * qm,u32 fun_num,u32 base,u32 number)3000 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
3001 u32 number)
3002 {
3003 u32 max_q_num = qm->ctrl_qp_num;
3004
3005 if (base >= max_q_num || number > max_q_num ||
3006 (base + number) > max_q_num)
3007 return -EINVAL;
3008
3009 return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
3010 }
3011
qm_init_eq_aeq_status(struct hisi_qm * qm)3012 static void qm_init_eq_aeq_status(struct hisi_qm *qm)
3013 {
3014 struct hisi_qm_status *status = &qm->status;
3015
3016 status->eq_head = 0;
3017 status->aeq_head = 0;
3018 status->eqc_phase = true;
3019 status->aeqc_phase = true;
3020 }
3021
qm_enable_eq_aeq_interrupts(struct hisi_qm * qm)3022 static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm)
3023 {
3024 /* Clear eq/aeq interrupt source */
3025 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
3026 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
3027
3028 writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
3029 writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
3030 }
3031
qm_disable_eq_aeq_interrupts(struct hisi_qm * qm)3032 static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm)
3033 {
3034 writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
3035 writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
3036 }
3037
qm_eq_ctx_cfg(struct hisi_qm * qm)3038 static int qm_eq_ctx_cfg(struct hisi_qm *qm)
3039 {
3040 struct device *dev = &qm->pdev->dev;
3041 struct qm_eqc *eqc;
3042 dma_addr_t eqc_dma;
3043 int ret;
3044
3045 eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL);
3046 if (!eqc)
3047 return -ENOMEM;
3048
3049 eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));
3050 eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
3051 if (qm->ver == QM_HW_V1)
3052 eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
3053 eqc->dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));
3054
3055 eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc),
3056 DMA_TO_DEVICE);
3057 if (dma_mapping_error(dev, eqc_dma)) {
3058 kfree(eqc);
3059 return -ENOMEM;
3060 }
3061
3062 ret = hisi_qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
3063 dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
3064 kfree(eqc);
3065
3066 return ret;
3067 }
3068
qm_aeq_ctx_cfg(struct hisi_qm * qm)3069 static int qm_aeq_ctx_cfg(struct hisi_qm *qm)
3070 {
3071 struct device *dev = &qm->pdev->dev;
3072 struct qm_aeqc *aeqc;
3073 dma_addr_t aeqc_dma;
3074 int ret;
3075
3076 aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL);
3077 if (!aeqc)
3078 return -ENOMEM;
3079
3080 aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));
3081 aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));
3082 aeqc->dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));
3083
3084 aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc),
3085 DMA_TO_DEVICE);
3086 if (dma_mapping_error(dev, aeqc_dma)) {
3087 kfree(aeqc);
3088 return -ENOMEM;
3089 }
3090
3091 ret = hisi_qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0);
3092 dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE);
3093 kfree(aeqc);
3094
3095 return ret;
3096 }
3097
qm_eq_aeq_ctx_cfg(struct hisi_qm * qm)3098 static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)
3099 {
3100 struct device *dev = &qm->pdev->dev;
3101 int ret;
3102
3103 qm_init_eq_aeq_status(qm);
3104
3105 ret = qm_eq_ctx_cfg(qm);
3106 if (ret) {
3107 dev_err(dev, "Set eqc failed!\n");
3108 return ret;
3109 }
3110
3111 return qm_aeq_ctx_cfg(qm);
3112 }
3113
__hisi_qm_start(struct hisi_qm * qm)3114 static int __hisi_qm_start(struct hisi_qm *qm)
3115 {
3116 int ret;
3117
3118 WARN_ON(!qm->qdma.va);
3119
3120 if (qm->fun_type == QM_HW_PF) {
3121 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
3122 if (ret)
3123 return ret;
3124 }
3125
3126 ret = qm_eq_aeq_ctx_cfg(qm);
3127 if (ret)
3128 return ret;
3129
3130 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
3131 if (ret)
3132 return ret;
3133
3134 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
3135 if (ret)
3136 return ret;
3137
3138 qm_init_prefetch(qm);
3139 qm_enable_eq_aeq_interrupts(qm);
3140
3141 return 0;
3142 }
3143
3144 /**
3145 * hisi_qm_start() - start qm
3146 * @qm: The qm to be started.
3147 *
3148 * This function starts a qm, then we can allocate qp from this qm.
3149 */
hisi_qm_start(struct hisi_qm * qm)3150 int hisi_qm_start(struct hisi_qm *qm)
3151 {
3152 struct device *dev = &qm->pdev->dev;
3153 int ret = 0;
3154
3155 down_write(&qm->qps_lock);
3156
3157 if (!qm_avail_state(qm, QM_START)) {
3158 up_write(&qm->qps_lock);
3159 return -EPERM;
3160 }
3161
3162 dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);
3163
3164 if (!qm->qp_num) {
3165 dev_err(dev, "qp_num should not be 0\n");
3166 ret = -EINVAL;
3167 goto err_unlock;
3168 }
3169
3170 ret = __hisi_qm_start(qm);
3171 if (!ret)
3172 atomic_set(&qm->status.flags, QM_START);
3173
3174 hisi_qm_set_state(qm, QM_READY);
3175 err_unlock:
3176 up_write(&qm->qps_lock);
3177 return ret;
3178 }
3179 EXPORT_SYMBOL_GPL(hisi_qm_start);
3180
qm_restart(struct hisi_qm * qm)3181 static int qm_restart(struct hisi_qm *qm)
3182 {
3183 struct device *dev = &qm->pdev->dev;
3184 struct hisi_qp *qp;
3185 int ret, i;
3186
3187 ret = hisi_qm_start(qm);
3188 if (ret < 0)
3189 return ret;
3190
3191 down_write(&qm->qps_lock);
3192 for (i = 0; i < qm->qp_num; i++) {
3193 qp = &qm->qp_array[i];
3194 if (atomic_read(&qp->qp_status.flags) == QP_STOP &&
3195 qp->is_resetting == true) {
3196 ret = qm_start_qp_nolock(qp, 0);
3197 if (ret < 0) {
3198 dev_err(dev, "Failed to start qp%d!\n", i);
3199
3200 up_write(&qm->qps_lock);
3201 return ret;
3202 }
3203 qp->is_resetting = false;
3204 }
3205 }
3206 up_write(&qm->qps_lock);
3207
3208 return 0;
3209 }
3210
3211 /* Stop started qps in reset flow */
qm_stop_started_qp(struct hisi_qm * qm)3212 static int qm_stop_started_qp(struct hisi_qm *qm)
3213 {
3214 struct device *dev = &qm->pdev->dev;
3215 struct hisi_qp *qp;
3216 int i, ret;
3217
3218 for (i = 0; i < qm->qp_num; i++) {
3219 qp = &qm->qp_array[i];
3220 if (qp && atomic_read(&qp->qp_status.flags) == QP_START) {
3221 qp->is_resetting = true;
3222 ret = qm_stop_qp_nolock(qp);
3223 if (ret < 0) {
3224 dev_err(dev, "Failed to stop qp%d!\n", i);
3225 return ret;
3226 }
3227 }
3228 }
3229
3230 return 0;
3231 }
3232
3233 /**
3234 * qm_clear_queues() - Clear all queues memory in a qm.
3235 * @qm: The qm in which the queues will be cleared.
3236 *
3237 * This function clears all queues memory in a qm. Reset of accelerator can
3238 * use this to clear queues.
3239 */
qm_clear_queues(struct hisi_qm * qm)3240 static void qm_clear_queues(struct hisi_qm *qm)
3241 {
3242 struct hisi_qp *qp;
3243 int i;
3244
3245 for (i = 0; i < qm->qp_num; i++) {
3246 qp = &qm->qp_array[i];
3247 if (qp->is_in_kernel && qp->is_resetting)
3248 memset(qp->qdma.va, 0, qp->qdma.size);
3249 }
3250
3251 memset(qm->qdma.va, 0, qm->qdma.size);
3252 }
3253
3254 /**
3255 * hisi_qm_stop() - Stop a qm.
3256 * @qm: The qm which will be stopped.
3257 * @r: The reason to stop qm.
3258 *
3259 * This function stops qm and its qps, then qm can not accept request.
3260 * Related resources are not released at this state, we can use hisi_qm_start
3261 * to let qm start again.
3262 */
hisi_qm_stop(struct hisi_qm * qm,enum qm_stop_reason r)3263 int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
3264 {
3265 struct device *dev = &qm->pdev->dev;
3266 int ret = 0;
3267
3268 down_write(&qm->qps_lock);
3269
3270 qm->status.stop_reason = r;
3271 if (!qm_avail_state(qm, QM_STOP)) {
3272 ret = -EPERM;
3273 goto err_unlock;
3274 }
3275
3276 if (qm->status.stop_reason == QM_SOFT_RESET ||
3277 qm->status.stop_reason == QM_DOWN) {
3278 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
3279 ret = qm_stop_started_qp(qm);
3280 if (ret < 0) {
3281 dev_err(dev, "Failed to stop started qp!\n");
3282 goto err_unlock;
3283 }
3284 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
3285 }
3286
3287 qm_disable_eq_aeq_interrupts(qm);
3288 if (qm->fun_type == QM_HW_PF) {
3289 ret = hisi_qm_set_vft(qm, 0, 0, 0);
3290 if (ret < 0) {
3291 dev_err(dev, "Failed to set vft!\n");
3292 ret = -EBUSY;
3293 goto err_unlock;
3294 }
3295 }
3296
3297 qm_clear_queues(qm);
3298 atomic_set(&qm->status.flags, QM_STOP);
3299
3300 err_unlock:
3301 up_write(&qm->qps_lock);
3302 return ret;
3303 }
3304 EXPORT_SYMBOL_GPL(hisi_qm_stop);
3305
qm_hw_error_init(struct hisi_qm * qm)3306 static void qm_hw_error_init(struct hisi_qm *qm)
3307 {
3308 if (!qm->ops->hw_error_init) {
3309 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
3310 return;
3311 }
3312
3313 qm->ops->hw_error_init(qm);
3314 }
3315
qm_hw_error_uninit(struct hisi_qm * qm)3316 static void qm_hw_error_uninit(struct hisi_qm *qm)
3317 {
3318 if (!qm->ops->hw_error_uninit) {
3319 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");
3320 return;
3321 }
3322
3323 qm->ops->hw_error_uninit(qm);
3324 }
3325
qm_hw_error_handle(struct hisi_qm * qm)3326 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)
3327 {
3328 if (!qm->ops->hw_error_handle) {
3329 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");
3330 return ACC_ERR_NONE;
3331 }
3332
3333 return qm->ops->hw_error_handle(qm);
3334 }
3335
3336 /**
3337 * hisi_qm_dev_err_init() - Initialize device error configuration.
3338 * @qm: The qm for which we want to do error initialization.
3339 *
3340 * Initialize QM and device error related configuration.
3341 */
hisi_qm_dev_err_init(struct hisi_qm * qm)3342 void hisi_qm_dev_err_init(struct hisi_qm *qm)
3343 {
3344 if (qm->fun_type == QM_HW_VF)
3345 return;
3346
3347 qm_hw_error_init(qm);
3348
3349 if (!qm->err_ini->hw_err_enable) {
3350 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");
3351 return;
3352 }
3353 qm->err_ini->hw_err_enable(qm);
3354 }
3355 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);
3356
3357 /**
3358 * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.
3359 * @qm: The qm for which we want to do error uninitialization.
3360 *
3361 * Uninitialize QM and device error related configuration.
3362 */
hisi_qm_dev_err_uninit(struct hisi_qm * qm)3363 void hisi_qm_dev_err_uninit(struct hisi_qm *qm)
3364 {
3365 if (qm->fun_type == QM_HW_VF)
3366 return;
3367
3368 qm_hw_error_uninit(qm);
3369
3370 if (!qm->err_ini->hw_err_disable) {
3371 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");
3372 return;
3373 }
3374 qm->err_ini->hw_err_disable(qm);
3375 }
3376 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);
3377
3378 /**
3379 * hisi_qm_free_qps() - free multiple queue pairs.
3380 * @qps: The queue pairs need to be freed.
3381 * @qp_num: The num of queue pairs.
3382 */
hisi_qm_free_qps(struct hisi_qp ** qps,int qp_num)3383 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)
3384 {
3385 int i;
3386
3387 if (!qps || qp_num <= 0)
3388 return;
3389
3390 for (i = qp_num - 1; i >= 0; i--)
3391 hisi_qm_release_qp(qps[i]);
3392 }
3393 EXPORT_SYMBOL_GPL(hisi_qm_free_qps);
3394
free_list(struct list_head * head)3395 static void free_list(struct list_head *head)
3396 {
3397 struct hisi_qm_resource *res, *tmp;
3398
3399 list_for_each_entry_safe(res, tmp, head, list) {
3400 list_del(&res->list);
3401 kfree(res);
3402 }
3403 }
3404
hisi_qm_sort_devices(int node,struct list_head * head,struct hisi_qm_list * qm_list)3405 static int hisi_qm_sort_devices(int node, struct list_head *head,
3406 struct hisi_qm_list *qm_list)
3407 {
3408 struct hisi_qm_resource *res, *tmp;
3409 struct hisi_qm *qm;
3410 struct list_head *n;
3411 struct device *dev;
3412 int dev_node;
3413
3414 list_for_each_entry(qm, &qm_list->list, list) {
3415 dev = &qm->pdev->dev;
3416
3417 dev_node = dev_to_node(dev);
3418 if (dev_node < 0)
3419 dev_node = 0;
3420
3421 res = kzalloc(sizeof(*res), GFP_KERNEL);
3422 if (!res)
3423 return -ENOMEM;
3424
3425 res->qm = qm;
3426 res->distance = node_distance(dev_node, node);
3427 n = head;
3428 list_for_each_entry(tmp, head, list) {
3429 if (res->distance < tmp->distance) {
3430 n = &tmp->list;
3431 break;
3432 }
3433 }
3434 list_add_tail(&res->list, n);
3435 }
3436
3437 return 0;
3438 }
3439
3440 /**
3441 * hisi_qm_alloc_qps_node() - Create multiple queue pairs.
3442 * @qm_list: The list of all available devices.
3443 * @qp_num: The number of queue pairs need created.
3444 * @alg_type: The algorithm type.
3445 * @node: The numa node.
3446 * @qps: The queue pairs need created.
3447 *
3448 * This function will sort all available device according to numa distance.
3449 * Then try to create all queue pairs from one device, if all devices do
3450 * not meet the requirements will return error.
3451 */
hisi_qm_alloc_qps_node(struct hisi_qm_list * qm_list,int qp_num,u8 alg_type,int node,struct hisi_qp ** qps)3452 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
3453 u8 alg_type, int node, struct hisi_qp **qps)
3454 {
3455 struct hisi_qm_resource *tmp;
3456 int ret = -ENODEV;
3457 LIST_HEAD(head);
3458 int i;
3459
3460 if (!qps || !qm_list || qp_num <= 0)
3461 return -EINVAL;
3462
3463 mutex_lock(&qm_list->lock);
3464 if (hisi_qm_sort_devices(node, &head, qm_list)) {
3465 mutex_unlock(&qm_list->lock);
3466 goto err;
3467 }
3468
3469 list_for_each_entry(tmp, &head, list) {
3470 for (i = 0; i < qp_num; i++) {
3471 qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);
3472 if (IS_ERR(qps[i])) {
3473 hisi_qm_free_qps(qps, i);
3474 break;
3475 }
3476 }
3477
3478 if (i == qp_num) {
3479 ret = 0;
3480 break;
3481 }
3482 }
3483
3484 mutex_unlock(&qm_list->lock);
3485 if (ret)
3486 pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n",
3487 node, alg_type, qp_num);
3488
3489 err:
3490 free_list(&head);
3491 return ret;
3492 }
3493 EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
3494
qm_vf_q_assign(struct hisi_qm * qm,u32 num_vfs)3495 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
3496 {
3497 u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;
3498 u32 max_qp_num = qm->max_qp_num;
3499 u32 q_base = qm->qp_num;
3500 int ret;
3501
3502 if (!num_vfs)
3503 return -EINVAL;
3504
3505 vfs_q_num = qm->ctrl_qp_num - qm->qp_num;
3506
3507 /* If vfs_q_num is less than num_vfs, return error. */
3508 if (vfs_q_num < num_vfs)
3509 return -EINVAL;
3510
3511 q_num = vfs_q_num / num_vfs;
3512 remain_q_num = vfs_q_num % num_vfs;
3513
3514 for (i = num_vfs; i > 0; i--) {
3515 /*
3516 * if q_num + remain_q_num > max_qp_num in last vf, divide the
3517 * remaining queues equally.
3518 */
3519 if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {
3520 act_q_num = q_num + remain_q_num;
3521 remain_q_num = 0;
3522 } else if (remain_q_num > 0) {
3523 act_q_num = q_num + 1;
3524 remain_q_num--;
3525 } else {
3526 act_q_num = q_num;
3527 }
3528
3529 act_q_num = min(act_q_num, max_qp_num);
3530 ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);
3531 if (ret) {
3532 for (j = num_vfs; j > i; j--)
3533 hisi_qm_set_vft(qm, j, 0, 0);
3534 return ret;
3535 }
3536 q_base += act_q_num;
3537 }
3538
3539 return 0;
3540 }
3541
qm_clear_vft_config(struct hisi_qm * qm)3542 static int qm_clear_vft_config(struct hisi_qm *qm)
3543 {
3544 int ret;
3545 u32 i;
3546
3547 for (i = 1; i <= qm->vfs_num; i++) {
3548 ret = hisi_qm_set_vft(qm, i, 0, 0);
3549 if (ret)
3550 return ret;
3551 }
3552 qm->vfs_num = 0;
3553
3554 return 0;
3555 }
3556
qm_func_shaper_enable(struct hisi_qm * qm,u32 fun_index,u32 qos)3557 static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos)
3558 {
3559 struct device *dev = &qm->pdev->dev;
3560 u32 ir = qos * QM_QOS_RATE;
3561 int ret, total_vfs, i;
3562
3563 total_vfs = pci_sriov_get_totalvfs(qm->pdev);
3564 if (fun_index > total_vfs)
3565 return -EINVAL;
3566
3567 qm->factor[fun_index].func_qos = qos;
3568
3569 ret = qm_get_shaper_para(ir, &qm->factor[fun_index]);
3570 if (ret) {
3571 dev_err(dev, "failed to calculate shaper parameter!\n");
3572 return -EINVAL;
3573 }
3574
3575 for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
3576 /* The base number of queue reuse for different alg type */
3577 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1);
3578 if (ret) {
3579 dev_err(dev, "type: %d, failed to set shaper vft!\n", i);
3580 return -EINVAL;
3581 }
3582 }
3583
3584 return 0;
3585 }
3586
qm_get_shaper_vft_qos(struct hisi_qm * qm,u32 fun_index)3587 static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index)
3588 {
3589 u64 cir_u = 0, cir_b = 0, cir_s = 0;
3590 u64 shaper_vft, ir_calc, ir;
3591 unsigned int val;
3592 u32 error_rate;
3593 int ret;
3594
3595 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
3596 val & BIT(0), POLL_PERIOD,
3597 POLL_TIMEOUT);
3598 if (ret)
3599 return 0;
3600
3601 writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);
3602 writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE);
3603 writel(fun_index, qm->io_base + QM_VFT_CFG);
3604
3605 writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
3606 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
3607
3608 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
3609 val & BIT(0), POLL_PERIOD,
3610 POLL_TIMEOUT);
3611 if (ret)
3612 return 0;
3613
3614 shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |
3615 ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32);
3616
3617 cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK;
3618 cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK;
3619 cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT;
3620
3621 cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK;
3622 cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT;
3623
3624 ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
3625
3626 ir = qm->factor[fun_index].func_qos * QM_QOS_RATE;
3627
3628 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
3629 if (error_rate > QM_QOS_MIN_ERROR_RATE) {
3630 pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate);
3631 return 0;
3632 }
3633
3634 return ir;
3635 }
3636
qm_vf_get_qos(struct hisi_qm * qm,u32 fun_num)3637 static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num)
3638 {
3639 struct device *dev = &qm->pdev->dev;
3640 u64 mb_cmd;
3641 u32 qos;
3642 int ret;
3643
3644 qos = qm_get_shaper_vft_qos(qm, fun_num);
3645 if (!qos) {
3646 dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num);
3647 return;
3648 }
3649
3650 mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT;
3651 ret = qm_ping_single_vf(qm, mb_cmd, fun_num);
3652 if (ret)
3653 dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num);
3654 }
3655
qm_vf_read_qos(struct hisi_qm * qm)3656 static int qm_vf_read_qos(struct hisi_qm *qm)
3657 {
3658 int cnt = 0;
3659 int ret = -EINVAL;
3660
3661 /* reset mailbox qos val */
3662 qm->mb_qos = 0;
3663
3664 /* vf ping pf to get function qos */
3665 ret = qm_ping_pf(qm, QM_VF_GET_QOS);
3666 if (ret) {
3667 pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n");
3668 return ret;
3669 }
3670
3671 while (true) {
3672 msleep(QM_WAIT_DST_ACK);
3673 if (qm->mb_qos)
3674 break;
3675
3676 if (++cnt > QM_MAX_VF_WAIT_COUNT) {
3677 pci_err(qm->pdev, "PF ping VF timeout!\n");
3678 return -ETIMEDOUT;
3679 }
3680 }
3681
3682 return ret;
3683 }
3684
qm_algqos_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)3685 static ssize_t qm_algqos_read(struct file *filp, char __user *buf,
3686 size_t count, loff_t *pos)
3687 {
3688 struct hisi_qm *qm = filp->private_data;
3689 char tbuf[QM_DBG_READ_LEN];
3690 u32 qos_val, ir;
3691 int ret;
3692
3693 ret = hisi_qm_get_dfx_access(qm);
3694 if (ret)
3695 return ret;
3696
3697 /* Mailbox and reset cannot be operated at the same time */
3698 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
3699 pci_err(qm->pdev, "dev resetting, read alg qos failed!\n");
3700 ret = -EAGAIN;
3701 goto err_put_dfx_access;
3702 }
3703
3704 if (qm->fun_type == QM_HW_PF) {
3705 ir = qm_get_shaper_vft_qos(qm, 0);
3706 } else {
3707 ret = qm_vf_read_qos(qm);
3708 if (ret)
3709 goto err_get_status;
3710 ir = qm->mb_qos;
3711 }
3712
3713 qos_val = ir / QM_QOS_RATE;
3714 ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val);
3715
3716 ret = simple_read_from_buffer(buf, count, pos, tbuf, ret);
3717
3718 err_get_status:
3719 clear_bit(QM_RESETTING, &qm->misc_ctl);
3720 err_put_dfx_access:
3721 hisi_qm_put_dfx_access(qm);
3722 return ret;
3723 }
3724
qm_get_qos_value(struct hisi_qm * qm,const char * buf,unsigned long * val,unsigned int * fun_index)3725 static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,
3726 unsigned long *val,
3727 unsigned int *fun_index)
3728 {
3729 const struct bus_type *bus_type = qm->pdev->dev.bus;
3730 char tbuf_bdf[QM_DBG_READ_LEN] = {0};
3731 char val_buf[QM_DBG_READ_LEN] = {0};
3732 struct pci_dev *pdev;
3733 struct device *dev;
3734 int ret;
3735
3736 ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf);
3737 if (ret != QM_QOS_PARAM_NUM)
3738 return -EINVAL;
3739
3740 ret = kstrtoul(val_buf, 10, val);
3741 if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) {
3742 pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
3743 return -EINVAL;
3744 }
3745
3746 dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf);
3747 if (!dev) {
3748 pci_err(qm->pdev, "input pci bdf number is error!\n");
3749 return -ENODEV;
3750 }
3751
3752 pdev = container_of(dev, struct pci_dev, dev);
3753
3754 *fun_index = pdev->devfn;
3755
3756 return 0;
3757 }
3758
qm_algqos_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)3759 static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
3760 size_t count, loff_t *pos)
3761 {
3762 struct hisi_qm *qm = filp->private_data;
3763 char tbuf[QM_DBG_READ_LEN];
3764 unsigned int fun_index;
3765 unsigned long val;
3766 int len, ret;
3767
3768 if (*pos != 0)
3769 return 0;
3770
3771 if (count >= QM_DBG_READ_LEN)
3772 return -ENOSPC;
3773
3774 len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
3775 if (len < 0)
3776 return len;
3777
3778 tbuf[len] = '\0';
3779 ret = qm_get_qos_value(qm, tbuf, &val, &fun_index);
3780 if (ret)
3781 return ret;
3782
3783 /* Mailbox and reset cannot be operated at the same time */
3784 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
3785 pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
3786 return -EAGAIN;
3787 }
3788
3789 ret = qm_pm_get_sync(qm);
3790 if (ret) {
3791 ret = -EINVAL;
3792 goto err_get_status;
3793 }
3794
3795 ret = qm_func_shaper_enable(qm, fun_index, val);
3796 if (ret) {
3797 pci_err(qm->pdev, "failed to enable function shaper!\n");
3798 ret = -EINVAL;
3799 goto err_put_sync;
3800 }
3801
3802 pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n",
3803 fun_index, val);
3804 ret = count;
3805
3806 err_put_sync:
3807 qm_pm_put_sync(qm);
3808 err_get_status:
3809 clear_bit(QM_RESETTING, &qm->misc_ctl);
3810 return ret;
3811 }
3812
3813 static const struct file_operations qm_algqos_fops = {
3814 .owner = THIS_MODULE,
3815 .open = simple_open,
3816 .read = qm_algqos_read,
3817 .write = qm_algqos_write,
3818 };
3819
3820 /**
3821 * hisi_qm_set_algqos_init() - Initialize function qos debugfs files.
3822 * @qm: The qm for which we want to add debugfs files.
3823 *
3824 * Create function qos debugfs files, VF ping PF to get function qos.
3825 */
hisi_qm_set_algqos_init(struct hisi_qm * qm)3826 void hisi_qm_set_algqos_init(struct hisi_qm *qm)
3827 {
3828 if (qm->fun_type == QM_HW_PF)
3829 debugfs_create_file("alg_qos", 0644, qm->debug.debug_root,
3830 qm, &qm_algqos_fops);
3831 else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))
3832 debugfs_create_file("alg_qos", 0444, qm->debug.debug_root,
3833 qm, &qm_algqos_fops);
3834 }
3835
hisi_qm_init_vf_qos(struct hisi_qm * qm,int total_func)3836 static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func)
3837 {
3838 int i;
3839
3840 for (i = 1; i <= total_func; i++)
3841 qm->factor[i].func_qos = QM_QOS_MAX_VAL;
3842 }
3843
3844 /**
3845 * hisi_qm_sriov_enable() - enable virtual functions
3846 * @pdev: the PCIe device
3847 * @max_vfs: the number of virtual functions to enable
3848 *
3849 * Returns the number of enabled VFs. If there are VFs enabled already or
3850 * max_vfs is more than the total number of device can be enabled, returns
3851 * failure.
3852 */
hisi_qm_sriov_enable(struct pci_dev * pdev,int max_vfs)3853 int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)
3854 {
3855 struct hisi_qm *qm = pci_get_drvdata(pdev);
3856 int pre_existing_vfs, num_vfs, total_vfs, ret;
3857
3858 ret = qm_pm_get_sync(qm);
3859 if (ret)
3860 return ret;
3861
3862 total_vfs = pci_sriov_get_totalvfs(pdev);
3863 pre_existing_vfs = pci_num_vf(pdev);
3864 if (pre_existing_vfs) {
3865 pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",
3866 pre_existing_vfs);
3867 goto err_put_sync;
3868 }
3869
3870 if (max_vfs > total_vfs) {
3871 pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs);
3872 ret = -ERANGE;
3873 goto err_put_sync;
3874 }
3875
3876 num_vfs = max_vfs;
3877
3878 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
3879 hisi_qm_init_vf_qos(qm, num_vfs);
3880
3881 ret = qm_vf_q_assign(qm, num_vfs);
3882 if (ret) {
3883 pci_err(pdev, "Can't assign queues for VF!\n");
3884 goto err_put_sync;
3885 }
3886
3887 qm->vfs_num = num_vfs;
3888
3889 ret = pci_enable_sriov(pdev, num_vfs);
3890 if (ret) {
3891 pci_err(pdev, "Can't enable VF!\n");
3892 qm_clear_vft_config(qm);
3893 goto err_put_sync;
3894 }
3895
3896 pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);
3897
3898 return num_vfs;
3899
3900 err_put_sync:
3901 qm_pm_put_sync(qm);
3902 return ret;
3903 }
3904 EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);
3905
3906 /**
3907 * hisi_qm_sriov_disable - disable virtual functions
3908 * @pdev: the PCI device.
3909 * @is_frozen: true when all the VFs are frozen.
3910 *
3911 * Return failure if there are VFs assigned already or VF is in used.
3912 */
hisi_qm_sriov_disable(struct pci_dev * pdev,bool is_frozen)3913 int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)
3914 {
3915 struct hisi_qm *qm = pci_get_drvdata(pdev);
3916 int ret;
3917
3918 if (pci_vfs_assigned(pdev)) {
3919 pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");
3920 return -EPERM;
3921 }
3922
3923 /* While VF is in used, SRIOV cannot be disabled. */
3924 if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {
3925 pci_err(pdev, "Task is using its VF!\n");
3926 return -EBUSY;
3927 }
3928
3929 pci_disable_sriov(pdev);
3930
3931 ret = qm_clear_vft_config(qm);
3932 if (ret)
3933 return ret;
3934
3935 qm_pm_put_sync(qm);
3936
3937 return 0;
3938 }
3939 EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);
3940
3941 /**
3942 * hisi_qm_sriov_configure - configure the number of VFs
3943 * @pdev: The PCI device
3944 * @num_vfs: The number of VFs need enabled
3945 *
3946 * Enable SR-IOV according to num_vfs, 0 means disable.
3947 */
hisi_qm_sriov_configure(struct pci_dev * pdev,int num_vfs)3948 int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
3949 {
3950 if (num_vfs == 0)
3951 return hisi_qm_sriov_disable(pdev, false);
3952 else
3953 return hisi_qm_sriov_enable(pdev, num_vfs);
3954 }
3955 EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);
3956
qm_dev_err_handle(struct hisi_qm * qm)3957 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
3958 {
3959 u32 err_sts;
3960
3961 if (!qm->err_ini->get_dev_hw_err_status) {
3962 dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n");
3963 return ACC_ERR_NONE;
3964 }
3965
3966 /* get device hardware error status */
3967 err_sts = qm->err_ini->get_dev_hw_err_status(qm);
3968 if (err_sts) {
3969 if (err_sts & qm->err_info.ecc_2bits_mask)
3970 qm->err_status.is_dev_ecc_mbit = true;
3971
3972 if (qm->err_ini->log_dev_hw_err)
3973 qm->err_ini->log_dev_hw_err(qm, err_sts);
3974
3975 if (err_sts & qm->err_info.dev_reset_mask)
3976 return ACC_ERR_NEED_RESET;
3977
3978 if (qm->err_ini->clear_dev_hw_err_status)
3979 qm->err_ini->clear_dev_hw_err_status(qm, err_sts);
3980 }
3981
3982 return ACC_ERR_RECOVERED;
3983 }
3984
qm_process_dev_error(struct hisi_qm * qm)3985 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)
3986 {
3987 enum acc_err_result qm_ret, dev_ret;
3988
3989 /* log qm error */
3990 qm_ret = qm_hw_error_handle(qm);
3991
3992 /* log device error */
3993 dev_ret = qm_dev_err_handle(qm);
3994
3995 return (qm_ret == ACC_ERR_NEED_RESET ||
3996 dev_ret == ACC_ERR_NEED_RESET) ?
3997 ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
3998 }
3999
4000 /**
4001 * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
4002 * @pdev: The PCI device which need report error.
4003 * @state: The connectivity between CPU and device.
4004 *
4005 * We register this function into PCIe AER handlers, It will report device or
4006 * qm hardware error status when error occur.
4007 */
hisi_qm_dev_err_detected(struct pci_dev * pdev,pci_channel_state_t state)4008 pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
4009 pci_channel_state_t state)
4010 {
4011 struct hisi_qm *qm = pci_get_drvdata(pdev);
4012 enum acc_err_result ret;
4013
4014 if (pdev->is_virtfn)
4015 return PCI_ERS_RESULT_NONE;
4016
4017 pci_info(pdev, "PCI error detected, state(=%u)!!\n", state);
4018 if (state == pci_channel_io_perm_failure)
4019 return PCI_ERS_RESULT_DISCONNECT;
4020
4021 ret = qm_process_dev_error(qm);
4022 if (ret == ACC_ERR_NEED_RESET)
4023 return PCI_ERS_RESULT_NEED_RESET;
4024
4025 return PCI_ERS_RESULT_RECOVERED;
4026 }
4027 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);
4028
qm_check_req_recv(struct hisi_qm * qm)4029 static int qm_check_req_recv(struct hisi_qm *qm)
4030 {
4031 struct pci_dev *pdev = qm->pdev;
4032 int ret;
4033 u32 val;
4034
4035 if (qm->ver >= QM_HW_V3)
4036 return 0;
4037
4038 writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);
4039 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4040 (val == ACC_VENDOR_ID_VALUE),
4041 POLL_PERIOD, POLL_TIMEOUT);
4042 if (ret) {
4043 dev_err(&pdev->dev, "Fails to read QM reg!\n");
4044 return ret;
4045 }
4046
4047 writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);
4048 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4049 (val == PCI_VENDOR_ID_HUAWEI),
4050 POLL_PERIOD, POLL_TIMEOUT);
4051 if (ret)
4052 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");
4053
4054 return ret;
4055 }
4056
qm_set_pf_mse(struct hisi_qm * qm,bool set)4057 static int qm_set_pf_mse(struct hisi_qm *qm, bool set)
4058 {
4059 struct pci_dev *pdev = qm->pdev;
4060 u16 cmd;
4061 int i;
4062
4063 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4064 if (set)
4065 cmd |= PCI_COMMAND_MEMORY;
4066 else
4067 cmd &= ~PCI_COMMAND_MEMORY;
4068
4069 pci_write_config_word(pdev, PCI_COMMAND, cmd);
4070 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4071 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4072 if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))
4073 return 0;
4074
4075 udelay(1);
4076 }
4077
4078 return -ETIMEDOUT;
4079 }
4080
qm_set_vf_mse(struct hisi_qm * qm,bool set)4081 static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
4082 {
4083 struct pci_dev *pdev = qm->pdev;
4084 u16 sriov_ctrl;
4085 int pos;
4086 int i;
4087
4088 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
4089 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4090 if (set)
4091 sriov_ctrl |= PCI_SRIOV_CTRL_MSE;
4092 else
4093 sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;
4094 pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);
4095
4096 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4097 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4098 if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>
4099 ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)
4100 return 0;
4101
4102 udelay(1);
4103 }
4104
4105 return -ETIMEDOUT;
4106 }
4107
qm_vf_reset_prepare(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4108 static int qm_vf_reset_prepare(struct hisi_qm *qm,
4109 enum qm_stop_reason stop_reason)
4110 {
4111 struct hisi_qm_list *qm_list = qm->qm_list;
4112 struct pci_dev *pdev = qm->pdev;
4113 struct pci_dev *virtfn;
4114 struct hisi_qm *vf_qm;
4115 int ret = 0;
4116
4117 mutex_lock(&qm_list->lock);
4118 list_for_each_entry(vf_qm, &qm_list->list, list) {
4119 virtfn = vf_qm->pdev;
4120 if (virtfn == pdev)
4121 continue;
4122
4123 if (pci_physfn(virtfn) == pdev) {
4124 /* save VFs PCIE BAR configuration */
4125 pci_save_state(virtfn);
4126
4127 ret = hisi_qm_stop(vf_qm, stop_reason);
4128 if (ret)
4129 goto stop_fail;
4130 }
4131 }
4132
4133 stop_fail:
4134 mutex_unlock(&qm_list->lock);
4135 return ret;
4136 }
4137
qm_try_stop_vfs(struct hisi_qm * qm,u64 cmd,enum qm_stop_reason stop_reason)4138 static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd,
4139 enum qm_stop_reason stop_reason)
4140 {
4141 struct pci_dev *pdev = qm->pdev;
4142 int ret;
4143
4144 if (!qm->vfs_num)
4145 return 0;
4146
4147 /* Kunpeng930 supports to notify VFs to stop before PF reset */
4148 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {
4149 ret = qm_ping_all_vfs(qm, cmd);
4150 if (ret)
4151 pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n");
4152 } else {
4153 ret = qm_vf_reset_prepare(qm, stop_reason);
4154 if (ret)
4155 pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret);
4156 }
4157
4158 return ret;
4159 }
4160
qm_controller_reset_prepare(struct hisi_qm * qm)4161 static int qm_controller_reset_prepare(struct hisi_qm *qm)
4162 {
4163 struct pci_dev *pdev = qm->pdev;
4164 int ret;
4165
4166 ret = qm_reset_prepare_ready(qm);
4167 if (ret) {
4168 pci_err(pdev, "Controller reset not ready!\n");
4169 return ret;
4170 }
4171
4172 /* PF obtains the information of VF by querying the register. */
4173 qm_cmd_uninit(qm);
4174
4175 /* Whether VFs stop successfully, soft reset will continue. */
4176 ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET);
4177 if (ret)
4178 pci_err(pdev, "failed to stop vfs by pf in soft reset.\n");
4179
4180 ret = hisi_qm_stop(qm, QM_SOFT_RESET);
4181 if (ret) {
4182 pci_err(pdev, "Fails to stop QM!\n");
4183 qm_reset_bit_clear(qm);
4184 return ret;
4185 }
4186
4187 if (qm->use_sva) {
4188 ret = qm_hw_err_isolate(qm);
4189 if (ret)
4190 pci_err(pdev, "failed to isolate hw err!\n");
4191 }
4192
4193 ret = qm_wait_vf_prepare_finish(qm);
4194 if (ret)
4195 pci_err(pdev, "failed to stop by vfs in soft reset!\n");
4196
4197 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4198
4199 return 0;
4200 }
4201
qm_dev_ecc_mbit_handle(struct hisi_qm * qm)4202 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)
4203 {
4204 u32 nfe_enb = 0;
4205
4206 /* Kunpeng930 hardware automatically close master ooo when NFE occurs */
4207 if (qm->ver >= QM_HW_V3)
4208 return;
4209
4210 if (!qm->err_status.is_dev_ecc_mbit &&
4211 qm->err_status.is_qm_ecc_mbit &&
4212 qm->err_ini->close_axi_master_ooo) {
4213 qm->err_ini->close_axi_master_ooo(qm);
4214 } else if (qm->err_status.is_dev_ecc_mbit &&
4215 !qm->err_status.is_qm_ecc_mbit &&
4216 !qm->err_ini->close_axi_master_ooo) {
4217 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);
4218 writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,
4219 qm->io_base + QM_RAS_NFE_ENABLE);
4220 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);
4221 }
4222 }
4223
qm_soft_reset(struct hisi_qm * qm)4224 static int qm_soft_reset(struct hisi_qm *qm)
4225 {
4226 struct pci_dev *pdev = qm->pdev;
4227 int ret;
4228 u32 val;
4229
4230 /* Ensure all doorbells and mailboxes received by QM */
4231 ret = qm_check_req_recv(qm);
4232 if (ret)
4233 return ret;
4234
4235 if (qm->vfs_num) {
4236 ret = qm_set_vf_mse(qm, false);
4237 if (ret) {
4238 pci_err(pdev, "Fails to disable vf MSE bit.\n");
4239 return ret;
4240 }
4241 }
4242
4243 ret = qm->ops->set_msi(qm, false);
4244 if (ret) {
4245 pci_err(pdev, "Fails to disable PEH MSI bit.\n");
4246 return ret;
4247 }
4248
4249 qm_dev_ecc_mbit_handle(qm);
4250
4251 /* OOO register set and check */
4252 writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN,
4253 qm->io_base + ACC_MASTER_GLOBAL_CTRL);
4254
4255 /* If bus lock, reset chip */
4256 ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
4257 val,
4258 (val == ACC_MASTER_TRANS_RETURN_RW),
4259 POLL_PERIOD, POLL_TIMEOUT);
4260 if (ret) {
4261 pci_emerg(pdev, "Bus lock! Please reset system.\n");
4262 return ret;
4263 }
4264
4265 if (qm->err_ini->close_sva_prefetch)
4266 qm->err_ini->close_sva_prefetch(qm);
4267
4268 ret = qm_set_pf_mse(qm, false);
4269 if (ret) {
4270 pci_err(pdev, "Fails to disable pf MSE bit.\n");
4271 return ret;
4272 }
4273
4274 /* The reset related sub-control registers are not in PCI BAR */
4275 if (ACPI_HANDLE(&pdev->dev)) {
4276 unsigned long long value = 0;
4277 acpi_status s;
4278
4279 s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
4280 qm->err_info.acpi_rst,
4281 NULL, &value);
4282 if (ACPI_FAILURE(s)) {
4283 pci_err(pdev, "NO controller reset method!\n");
4284 return -EIO;
4285 }
4286
4287 if (value) {
4288 pci_err(pdev, "Reset step %llu failed!\n", value);
4289 return -EIO;
4290 }
4291 } else {
4292 pci_err(pdev, "No reset method!\n");
4293 return -EINVAL;
4294 }
4295
4296 return 0;
4297 }
4298
qm_vf_reset_done(struct hisi_qm * qm)4299 static int qm_vf_reset_done(struct hisi_qm *qm)
4300 {
4301 struct hisi_qm_list *qm_list = qm->qm_list;
4302 struct pci_dev *pdev = qm->pdev;
4303 struct pci_dev *virtfn;
4304 struct hisi_qm *vf_qm;
4305 int ret = 0;
4306
4307 mutex_lock(&qm_list->lock);
4308 list_for_each_entry(vf_qm, &qm_list->list, list) {
4309 virtfn = vf_qm->pdev;
4310 if (virtfn == pdev)
4311 continue;
4312
4313 if (pci_physfn(virtfn) == pdev) {
4314 /* enable VFs PCIE BAR configuration */
4315 pci_restore_state(virtfn);
4316
4317 ret = qm_restart(vf_qm);
4318 if (ret)
4319 goto restart_fail;
4320 }
4321 }
4322
4323 restart_fail:
4324 mutex_unlock(&qm_list->lock);
4325 return ret;
4326 }
4327
qm_try_start_vfs(struct hisi_qm * qm,enum qm_mb_cmd cmd)4328 static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd)
4329 {
4330 struct pci_dev *pdev = qm->pdev;
4331 int ret;
4332
4333 if (!qm->vfs_num)
4334 return 0;
4335
4336 ret = qm_vf_q_assign(qm, qm->vfs_num);
4337 if (ret) {
4338 pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret);
4339 return ret;
4340 }
4341
4342 /* Kunpeng930 supports to notify VFs to start after PF reset. */
4343 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {
4344 ret = qm_ping_all_vfs(qm, cmd);
4345 if (ret)
4346 pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n");
4347 } else {
4348 ret = qm_vf_reset_done(qm);
4349 if (ret)
4350 pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret);
4351 }
4352
4353 return ret;
4354 }
4355
qm_dev_hw_init(struct hisi_qm * qm)4356 static int qm_dev_hw_init(struct hisi_qm *qm)
4357 {
4358 return qm->err_ini->hw_init(qm);
4359 }
4360
qm_restart_prepare(struct hisi_qm * qm)4361 static void qm_restart_prepare(struct hisi_qm *qm)
4362 {
4363 u32 value;
4364
4365 if (qm->err_ini->open_sva_prefetch)
4366 qm->err_ini->open_sva_prefetch(qm);
4367
4368 if (qm->ver >= QM_HW_V3)
4369 return;
4370
4371 if (!qm->err_status.is_qm_ecc_mbit &&
4372 !qm->err_status.is_dev_ecc_mbit)
4373 return;
4374
4375 /* temporarily close the OOO port used for PEH to write out MSI */
4376 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4377 writel(value & ~qm->err_info.msi_wr_port,
4378 qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4379
4380 /* clear dev ecc 2bit error source if having */
4381 value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;
4382 if (value && qm->err_ini->clear_dev_hw_err_status)
4383 qm->err_ini->clear_dev_hw_err_status(qm, value);
4384
4385 /* clear QM ecc mbit error source */
4386 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);
4387
4388 /* clear AM Reorder Buffer ecc mbit source */
4389 writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);
4390 }
4391
qm_restart_done(struct hisi_qm * qm)4392 static void qm_restart_done(struct hisi_qm *qm)
4393 {
4394 u32 value;
4395
4396 if (qm->ver >= QM_HW_V3)
4397 goto clear_flags;
4398
4399 if (!qm->err_status.is_qm_ecc_mbit &&
4400 !qm->err_status.is_dev_ecc_mbit)
4401 return;
4402
4403 /* open the OOO port for PEH to write out MSI */
4404 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4405 value |= qm->err_info.msi_wr_port;
4406 writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4407
4408 clear_flags:
4409 qm->err_status.is_qm_ecc_mbit = false;
4410 qm->err_status.is_dev_ecc_mbit = false;
4411 }
4412
qm_controller_reset_done(struct hisi_qm * qm)4413 static int qm_controller_reset_done(struct hisi_qm *qm)
4414 {
4415 struct pci_dev *pdev = qm->pdev;
4416 int ret;
4417
4418 ret = qm->ops->set_msi(qm, true);
4419 if (ret) {
4420 pci_err(pdev, "Fails to enable PEH MSI bit!\n");
4421 return ret;
4422 }
4423
4424 ret = qm_set_pf_mse(qm, true);
4425 if (ret) {
4426 pci_err(pdev, "Fails to enable pf MSE bit!\n");
4427 return ret;
4428 }
4429
4430 if (qm->vfs_num) {
4431 ret = qm_set_vf_mse(qm, true);
4432 if (ret) {
4433 pci_err(pdev, "Fails to enable vf MSE bit!\n");
4434 return ret;
4435 }
4436 }
4437
4438 ret = qm_dev_hw_init(qm);
4439 if (ret) {
4440 pci_err(pdev, "Failed to init device\n");
4441 return ret;
4442 }
4443
4444 qm_restart_prepare(qm);
4445 hisi_qm_dev_err_init(qm);
4446 if (qm->err_ini->open_axi_master_ooo)
4447 qm->err_ini->open_axi_master_ooo(qm);
4448
4449 ret = qm_dev_mem_reset(qm);
4450 if (ret) {
4451 pci_err(pdev, "failed to reset device memory\n");
4452 return ret;
4453 }
4454
4455 ret = qm_restart(qm);
4456 if (ret) {
4457 pci_err(pdev, "Failed to start QM!\n");
4458 return ret;
4459 }
4460
4461 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
4462 if (ret)
4463 pci_err(pdev, "failed to start vfs by pf in soft reset.\n");
4464
4465 ret = qm_wait_vf_prepare_finish(qm);
4466 if (ret)
4467 pci_err(pdev, "failed to start by vfs in soft reset!\n");
4468
4469 qm_cmd_init(qm);
4470 qm_restart_done(qm);
4471
4472 qm_reset_bit_clear(qm);
4473
4474 return 0;
4475 }
4476
qm_controller_reset(struct hisi_qm * qm)4477 static int qm_controller_reset(struct hisi_qm *qm)
4478 {
4479 struct pci_dev *pdev = qm->pdev;
4480 int ret;
4481
4482 pci_info(pdev, "Controller resetting...\n");
4483
4484 ret = qm_controller_reset_prepare(qm);
4485 if (ret) {
4486 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4487 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4488 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4489 return ret;
4490 }
4491
4492 hisi_qm_show_last_dfx_regs(qm);
4493 if (qm->err_ini->show_last_dfx_regs)
4494 qm->err_ini->show_last_dfx_regs(qm);
4495
4496 ret = qm_soft_reset(qm);
4497 if (ret)
4498 goto err_reset;
4499
4500 ret = qm_controller_reset_done(qm);
4501 if (ret)
4502 goto err_reset;
4503
4504 pci_info(pdev, "Controller reset complete\n");
4505
4506 return 0;
4507
4508 err_reset:
4509 pci_err(pdev, "Controller reset failed (%d)\n", ret);
4510 qm_reset_bit_clear(qm);
4511
4512 /* if resetting fails, isolate the device */
4513 if (qm->use_sva)
4514 qm->isolate_data.is_isolate = true;
4515 return ret;
4516 }
4517
4518 /**
4519 * hisi_qm_dev_slot_reset() - slot reset
4520 * @pdev: the PCIe device
4521 *
4522 * This function offers QM relate PCIe device reset interface. Drivers which
4523 * use QM can use this function as slot_reset in its struct pci_error_handlers.
4524 */
hisi_qm_dev_slot_reset(struct pci_dev * pdev)4525 pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)
4526 {
4527 struct hisi_qm *qm = pci_get_drvdata(pdev);
4528 int ret;
4529
4530 if (pdev->is_virtfn)
4531 return PCI_ERS_RESULT_RECOVERED;
4532
4533 /* reset pcie device controller */
4534 ret = qm_controller_reset(qm);
4535 if (ret) {
4536 pci_err(pdev, "Controller reset failed (%d)\n", ret);
4537 return PCI_ERS_RESULT_DISCONNECT;
4538 }
4539
4540 return PCI_ERS_RESULT_RECOVERED;
4541 }
4542 EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);
4543
hisi_qm_reset_prepare(struct pci_dev * pdev)4544 void hisi_qm_reset_prepare(struct pci_dev *pdev)
4545 {
4546 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4547 struct hisi_qm *qm = pci_get_drvdata(pdev);
4548 u32 delay = 0;
4549 int ret;
4550
4551 hisi_qm_dev_err_uninit(pf_qm);
4552
4553 /*
4554 * Check whether there is an ECC mbit error, If it occurs, need to
4555 * wait for soft reset to fix it.
4556 */
4557 while (qm_check_dev_error(pf_qm)) {
4558 msleep(++delay);
4559 if (delay > QM_RESET_WAIT_TIMEOUT)
4560 return;
4561 }
4562
4563 ret = qm_reset_prepare_ready(qm);
4564 if (ret) {
4565 pci_err(pdev, "FLR not ready!\n");
4566 return;
4567 }
4568
4569 /* PF obtains the information of VF by querying the register. */
4570 if (qm->fun_type == QM_HW_PF)
4571 qm_cmd_uninit(qm);
4572
4573 ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN);
4574 if (ret)
4575 pci_err(pdev, "failed to stop vfs by pf in FLR.\n");
4576
4577 ret = hisi_qm_stop(qm, QM_DOWN);
4578 if (ret) {
4579 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
4580 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4581 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4582 return;
4583 }
4584
4585 ret = qm_wait_vf_prepare_finish(qm);
4586 if (ret)
4587 pci_err(pdev, "failed to stop by vfs in FLR!\n");
4588
4589 pci_info(pdev, "FLR resetting...\n");
4590 }
4591 EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);
4592
qm_flr_reset_complete(struct pci_dev * pdev)4593 static bool qm_flr_reset_complete(struct pci_dev *pdev)
4594 {
4595 struct pci_dev *pf_pdev = pci_physfn(pdev);
4596 struct hisi_qm *qm = pci_get_drvdata(pf_pdev);
4597 u32 id;
4598
4599 pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);
4600 if (id == QM_PCI_COMMAND_INVALID) {
4601 pci_err(pdev, "Device can not be used!\n");
4602 return false;
4603 }
4604
4605 return true;
4606 }
4607
hisi_qm_reset_done(struct pci_dev * pdev)4608 void hisi_qm_reset_done(struct pci_dev *pdev)
4609 {
4610 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4611 struct hisi_qm *qm = pci_get_drvdata(pdev);
4612 int ret;
4613
4614 if (qm->fun_type == QM_HW_PF) {
4615 ret = qm_dev_hw_init(qm);
4616 if (ret) {
4617 pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);
4618 goto flr_done;
4619 }
4620 }
4621
4622 hisi_qm_dev_err_init(pf_qm);
4623
4624 ret = qm_restart(qm);
4625 if (ret) {
4626 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);
4627 goto flr_done;
4628 }
4629
4630 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
4631 if (ret)
4632 pci_err(pdev, "failed to start vfs by pf in FLR.\n");
4633
4634 ret = qm_wait_vf_prepare_finish(qm);
4635 if (ret)
4636 pci_err(pdev, "failed to start by vfs in FLR!\n");
4637
4638 flr_done:
4639 if (qm->fun_type == QM_HW_PF)
4640 qm_cmd_init(qm);
4641
4642 if (qm_flr_reset_complete(pdev))
4643 pci_info(pdev, "FLR reset complete\n");
4644
4645 qm_reset_bit_clear(qm);
4646 }
4647 EXPORT_SYMBOL_GPL(hisi_qm_reset_done);
4648
qm_abnormal_irq(int irq,void * data)4649 static irqreturn_t qm_abnormal_irq(int irq, void *data)
4650 {
4651 struct hisi_qm *qm = data;
4652 enum acc_err_result ret;
4653
4654 atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);
4655 ret = qm_process_dev_error(qm);
4656 if (ret == ACC_ERR_NEED_RESET &&
4657 !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) &&
4658 !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl))
4659 schedule_work(&qm->rst_work);
4660
4661 return IRQ_HANDLED;
4662 }
4663
4664 /**
4665 * hisi_qm_dev_shutdown() - Shutdown device.
4666 * @pdev: The device will be shutdown.
4667 *
4668 * This function will stop qm when OS shutdown or rebooting.
4669 */
hisi_qm_dev_shutdown(struct pci_dev * pdev)4670 void hisi_qm_dev_shutdown(struct pci_dev *pdev)
4671 {
4672 struct hisi_qm *qm = pci_get_drvdata(pdev);
4673 int ret;
4674
4675 ret = hisi_qm_stop(qm, QM_DOWN);
4676 if (ret)
4677 dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");
4678
4679 hisi_qm_cache_wb(qm);
4680 }
4681 EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);
4682
hisi_qm_controller_reset(struct work_struct * rst_work)4683 static void hisi_qm_controller_reset(struct work_struct *rst_work)
4684 {
4685 struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
4686 int ret;
4687
4688 ret = qm_pm_get_sync(qm);
4689 if (ret) {
4690 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4691 return;
4692 }
4693
4694 /* reset pcie device controller */
4695 ret = qm_controller_reset(qm);
4696 if (ret)
4697 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);
4698
4699 qm_pm_put_sync(qm);
4700 }
4701
qm_pf_reset_vf_prepare(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4702 static void qm_pf_reset_vf_prepare(struct hisi_qm *qm,
4703 enum qm_stop_reason stop_reason)
4704 {
4705 enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE;
4706 struct pci_dev *pdev = qm->pdev;
4707 int ret;
4708
4709 ret = qm_reset_prepare_ready(qm);
4710 if (ret) {
4711 dev_err(&pdev->dev, "reset prepare not ready!\n");
4712 atomic_set(&qm->status.flags, QM_STOP);
4713 cmd = QM_VF_PREPARE_FAIL;
4714 goto err_prepare;
4715 }
4716
4717 ret = hisi_qm_stop(qm, stop_reason);
4718 if (ret) {
4719 dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret);
4720 atomic_set(&qm->status.flags, QM_STOP);
4721 cmd = QM_VF_PREPARE_FAIL;
4722 goto err_prepare;
4723 } else {
4724 goto out;
4725 }
4726
4727 err_prepare:
4728 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);
4729 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
4730 out:
4731 pci_save_state(pdev);
4732 ret = qm_ping_pf(qm, cmd);
4733 if (ret)
4734 dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n");
4735 }
4736
qm_pf_reset_vf_done(struct hisi_qm * qm)4737 static void qm_pf_reset_vf_done(struct hisi_qm *qm)
4738 {
4739 enum qm_mb_cmd cmd = QM_VF_START_DONE;
4740 struct pci_dev *pdev = qm->pdev;
4741 int ret;
4742
4743 pci_restore_state(pdev);
4744 ret = hisi_qm_start(qm);
4745 if (ret) {
4746 dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret);
4747 cmd = QM_VF_START_FAIL;
4748 }
4749
4750 qm_cmd_init(qm);
4751 ret = qm_ping_pf(qm, cmd);
4752 if (ret)
4753 dev_warn(&pdev->dev, "PF responds timeout in reset done!\n");
4754
4755 qm_reset_bit_clear(qm);
4756 }
4757
qm_wait_pf_reset_finish(struct hisi_qm * qm)4758 static int qm_wait_pf_reset_finish(struct hisi_qm *qm)
4759 {
4760 struct device *dev = &qm->pdev->dev;
4761 u32 val, cmd;
4762 u64 msg;
4763 int ret;
4764
4765 /* Wait for reset to finish */
4766 ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val,
4767 val == BIT(0), QM_VF_RESET_WAIT_US,
4768 QM_VF_RESET_WAIT_TIMEOUT_US);
4769 /* hardware completion status should be available by this time */
4770 if (ret) {
4771 dev_err(dev, "couldn't get reset done status from PF, timeout!\n");
4772 return -ETIMEDOUT;
4773 }
4774
4775 /*
4776 * Whether message is got successfully,
4777 * VF needs to ack PF by clearing the interrupt.
4778 */
4779 ret = qm_get_mb_cmd(qm, &msg, 0);
4780 qm_clear_cmd_interrupt(qm, 0);
4781 if (ret) {
4782 dev_err(dev, "failed to get msg from PF in reset done!\n");
4783 return ret;
4784 }
4785
4786 cmd = msg & QM_MB_CMD_DATA_MASK;
4787 if (cmd != QM_PF_RESET_DONE) {
4788 dev_err(dev, "the cmd(%u) is not reset done!\n", cmd);
4789 ret = -EINVAL;
4790 }
4791
4792 return ret;
4793 }
4794
qm_pf_reset_vf_process(struct hisi_qm * qm,enum qm_stop_reason stop_reason)4795 static void qm_pf_reset_vf_process(struct hisi_qm *qm,
4796 enum qm_stop_reason stop_reason)
4797 {
4798 struct device *dev = &qm->pdev->dev;
4799 int ret;
4800
4801 dev_info(dev, "device reset start...\n");
4802
4803 /* The message is obtained by querying the register during resetting */
4804 qm_cmd_uninit(qm);
4805 qm_pf_reset_vf_prepare(qm, stop_reason);
4806
4807 ret = qm_wait_pf_reset_finish(qm);
4808 if (ret)
4809 goto err_get_status;
4810
4811 qm_pf_reset_vf_done(qm);
4812
4813 dev_info(dev, "device reset done.\n");
4814
4815 return;
4816
4817 err_get_status:
4818 qm_cmd_init(qm);
4819 qm_reset_bit_clear(qm);
4820 }
4821
qm_handle_cmd_msg(struct hisi_qm * qm,u32 fun_num)4822 static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num)
4823 {
4824 struct device *dev = &qm->pdev->dev;
4825 u64 msg;
4826 u32 cmd;
4827 int ret;
4828
4829 /*
4830 * Get the msg from source by sending mailbox. Whether message is got
4831 * successfully, destination needs to ack source by clearing the interrupt.
4832 */
4833 ret = qm_get_mb_cmd(qm, &msg, fun_num);
4834 qm_clear_cmd_interrupt(qm, BIT(fun_num));
4835 if (ret) {
4836 dev_err(dev, "failed to get msg from source!\n");
4837 return;
4838 }
4839
4840 cmd = msg & QM_MB_CMD_DATA_MASK;
4841 switch (cmd) {
4842 case QM_PF_FLR_PREPARE:
4843 qm_pf_reset_vf_process(qm, QM_DOWN);
4844 break;
4845 case QM_PF_SRST_PREPARE:
4846 qm_pf_reset_vf_process(qm, QM_SOFT_RESET);
4847 break;
4848 case QM_VF_GET_QOS:
4849 qm_vf_get_qos(qm, fun_num);
4850 break;
4851 case QM_PF_SET_QOS:
4852 qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT;
4853 break;
4854 default:
4855 dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num);
4856 break;
4857 }
4858 }
4859
qm_cmd_process(struct work_struct * cmd_process)4860 static void qm_cmd_process(struct work_struct *cmd_process)
4861 {
4862 struct hisi_qm *qm = container_of(cmd_process,
4863 struct hisi_qm, cmd_process);
4864 u32 vfs_num = qm->vfs_num;
4865 u64 val;
4866 u32 i;
4867
4868 if (qm->fun_type == QM_HW_PF) {
4869 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
4870 if (!val)
4871 return;
4872
4873 for (i = 1; i <= vfs_num; i++) {
4874 if (val & BIT(i))
4875 qm_handle_cmd_msg(qm, i);
4876 }
4877
4878 return;
4879 }
4880
4881 qm_handle_cmd_msg(qm, 0);
4882 }
4883
4884 /**
4885 * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list.
4886 * @qm: The qm needs add.
4887 * @qm_list: The qm list.
4888 *
4889 * This function adds qm to qm list, and will register algorithm to
4890 * crypto when the qm list is empty.
4891 */
hisi_qm_alg_register(struct hisi_qm * qm,struct hisi_qm_list * qm_list)4892 int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
4893 {
4894 struct device *dev = &qm->pdev->dev;
4895 int flag = 0;
4896 int ret = 0;
4897
4898 mutex_lock(&qm_list->lock);
4899 if (list_empty(&qm_list->list))
4900 flag = 1;
4901 list_add_tail(&qm->list, &qm_list->list);
4902 mutex_unlock(&qm_list->lock);
4903
4904 if (qm->ver <= QM_HW_V2 && qm->use_sva) {
4905 dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n");
4906 return 0;
4907 }
4908
4909 if (flag) {
4910 ret = qm_list->register_to_crypto(qm);
4911 if (ret) {
4912 mutex_lock(&qm_list->lock);
4913 list_del(&qm->list);
4914 mutex_unlock(&qm_list->lock);
4915 }
4916 }
4917
4918 return ret;
4919 }
4920 EXPORT_SYMBOL_GPL(hisi_qm_alg_register);
4921
4922 /**
4923 * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from
4924 * qm list.
4925 * @qm: The qm needs delete.
4926 * @qm_list: The qm list.
4927 *
4928 * This function deletes qm from qm list, and will unregister algorithm
4929 * from crypto when the qm list is empty.
4930 */
hisi_qm_alg_unregister(struct hisi_qm * qm,struct hisi_qm_list * qm_list)4931 void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
4932 {
4933 mutex_lock(&qm_list->lock);
4934 list_del(&qm->list);
4935 mutex_unlock(&qm_list->lock);
4936
4937 if (qm->ver <= QM_HW_V2 && qm->use_sva)
4938 return;
4939
4940 if (list_empty(&qm_list->list))
4941 qm_list->unregister_from_crypto(qm);
4942 }
4943 EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);
4944
qm_unregister_abnormal_irq(struct hisi_qm * qm)4945 static void qm_unregister_abnormal_irq(struct hisi_qm *qm)
4946 {
4947 struct pci_dev *pdev = qm->pdev;
4948 u32 irq_vector, val;
4949
4950 if (qm->fun_type == QM_HW_VF)
4951 return;
4952
4953 val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val;
4954 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
4955 return;
4956
4957 irq_vector = val & QM_IRQ_VECTOR_MASK;
4958 free_irq(pci_irq_vector(pdev, irq_vector), qm);
4959 }
4960
qm_register_abnormal_irq(struct hisi_qm * qm)4961 static int qm_register_abnormal_irq(struct hisi_qm *qm)
4962 {
4963 struct pci_dev *pdev = qm->pdev;
4964 u32 irq_vector, val;
4965 int ret;
4966
4967 if (qm->fun_type == QM_HW_VF)
4968 return 0;
4969
4970 val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val;
4971 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))
4972 return 0;
4973
4974 irq_vector = val & QM_IRQ_VECTOR_MASK;
4975 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm);
4976 if (ret)
4977 dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret);
4978
4979 return ret;
4980 }
4981
qm_unregister_mb_cmd_irq(struct hisi_qm * qm)4982 static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm)
4983 {
4984 struct pci_dev *pdev = qm->pdev;
4985 u32 irq_vector, val;
4986
4987 val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val;
4988 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
4989 return;
4990
4991 irq_vector = val & QM_IRQ_VECTOR_MASK;
4992 free_irq(pci_irq_vector(pdev, irq_vector), qm);
4993 }
4994
qm_register_mb_cmd_irq(struct hisi_qm * qm)4995 static int qm_register_mb_cmd_irq(struct hisi_qm *qm)
4996 {
4997 struct pci_dev *pdev = qm->pdev;
4998 u32 irq_vector, val;
4999 int ret;
5000
5001 val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val;
5002 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5003 return 0;
5004
5005 irq_vector = val & QM_IRQ_VECTOR_MASK;
5006 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm);
5007 if (ret)
5008 dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret);
5009
5010 return ret;
5011 }
5012
qm_unregister_aeq_irq(struct hisi_qm * qm)5013 static void qm_unregister_aeq_irq(struct hisi_qm *qm)
5014 {
5015 struct pci_dev *pdev = qm->pdev;
5016 u32 irq_vector, val;
5017
5018 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val;
5019 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5020 return;
5021
5022 irq_vector = val & QM_IRQ_VECTOR_MASK;
5023 free_irq(pci_irq_vector(pdev, irq_vector), qm);
5024 }
5025
qm_register_aeq_irq(struct hisi_qm * qm)5026 static int qm_register_aeq_irq(struct hisi_qm *qm)
5027 {
5028 struct pci_dev *pdev = qm->pdev;
5029 u32 irq_vector, val;
5030 int ret;
5031
5032 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val;
5033 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5034 return 0;
5035
5036 irq_vector = val & QM_IRQ_VECTOR_MASK;
5037 ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL,
5038 qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm);
5039 if (ret)
5040 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);
5041
5042 return ret;
5043 }
5044
qm_unregister_eq_irq(struct hisi_qm * qm)5045 static void qm_unregister_eq_irq(struct hisi_qm *qm)
5046 {
5047 struct pci_dev *pdev = qm->pdev;
5048 u32 irq_vector, val;
5049
5050 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val;
5051 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5052 return;
5053
5054 irq_vector = val & QM_IRQ_VECTOR_MASK;
5055 free_irq(pci_irq_vector(pdev, irq_vector), qm);
5056 }
5057
qm_register_eq_irq(struct hisi_qm * qm)5058 static int qm_register_eq_irq(struct hisi_qm *qm)
5059 {
5060 struct pci_dev *pdev = qm->pdev;
5061 u32 irq_vector, val;
5062 int ret;
5063
5064 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val;
5065 if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))
5066 return 0;
5067
5068 irq_vector = val & QM_IRQ_VECTOR_MASK;
5069 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm);
5070 if (ret)
5071 dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);
5072
5073 return ret;
5074 }
5075
qm_irqs_unregister(struct hisi_qm * qm)5076 static void qm_irqs_unregister(struct hisi_qm *qm)
5077 {
5078 qm_unregister_mb_cmd_irq(qm);
5079 qm_unregister_abnormal_irq(qm);
5080 qm_unregister_aeq_irq(qm);
5081 qm_unregister_eq_irq(qm);
5082 }
5083
qm_irqs_register(struct hisi_qm * qm)5084 static int qm_irqs_register(struct hisi_qm *qm)
5085 {
5086 int ret;
5087
5088 ret = qm_register_eq_irq(qm);
5089 if (ret)
5090 return ret;
5091
5092 ret = qm_register_aeq_irq(qm);
5093 if (ret)
5094 goto free_eq_irq;
5095
5096 ret = qm_register_abnormal_irq(qm);
5097 if (ret)
5098 goto free_aeq_irq;
5099
5100 ret = qm_register_mb_cmd_irq(qm);
5101 if (ret)
5102 goto free_abnormal_irq;
5103
5104 return 0;
5105
5106 free_abnormal_irq:
5107 qm_unregister_abnormal_irq(qm);
5108 free_aeq_irq:
5109 qm_unregister_aeq_irq(qm);
5110 free_eq_irq:
5111 qm_unregister_eq_irq(qm);
5112 return ret;
5113 }
5114
qm_get_qp_num(struct hisi_qm * qm)5115 static int qm_get_qp_num(struct hisi_qm *qm)
5116 {
5117 struct device *dev = &qm->pdev->dev;
5118 bool is_db_isolation;
5119
5120 /* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */
5121 if (qm->fun_type == QM_HW_VF) {
5122 if (qm->ver != QM_HW_V1)
5123 /* v2 starts to support get vft by mailbox */
5124 return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
5125
5126 return 0;
5127 }
5128
5129 is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);
5130 qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true);
5131 qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info,
5132 QM_FUNC_MAX_QP_CAP, is_db_isolation);
5133
5134 if (qm->qp_num <= qm->max_qp_num)
5135 return 0;
5136
5137 if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) {
5138 /* Check whether the set qp number is valid */
5139 dev_err(dev, "qp num(%u) is more than max qp num(%u)!\n",
5140 qm->qp_num, qm->max_qp_num);
5141 return -EINVAL;
5142 }
5143
5144 dev_info(dev, "Default qp num(%u) is too big, reset it to Function's max qp num(%u)!\n",
5145 qm->qp_num, qm->max_qp_num);
5146 qm->qp_num = qm->max_qp_num;
5147 qm->debug.curr_qm_qp_num = qm->qp_num;
5148
5149 return 0;
5150 }
5151
qm_pre_store_irq_type_caps(struct hisi_qm * qm)5152 static int qm_pre_store_irq_type_caps(struct hisi_qm *qm)
5153 {
5154 struct hisi_qm_cap_record *qm_cap;
5155 struct pci_dev *pdev = qm->pdev;
5156 size_t i, size;
5157
5158 size = ARRAY_SIZE(qm_pre_store_caps);
5159 qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL);
5160 if (!qm_cap)
5161 return -ENOMEM;
5162
5163 for (i = 0; i < size; i++) {
5164 qm_cap[i].type = qm_pre_store_caps[i];
5165 qm_cap[i].cap_val = hisi_qm_get_hw_info(qm, qm_basic_info,
5166 qm_pre_store_caps[i], qm->cap_ver);
5167 }
5168
5169 qm->cap_tables.qm_cap_table = qm_cap;
5170
5171 return 0;
5172 }
5173
qm_get_hw_caps(struct hisi_qm * qm)5174 static int qm_get_hw_caps(struct hisi_qm *qm)
5175 {
5176 const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ?
5177 qm_cap_info_pf : qm_cap_info_vf;
5178 u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) :
5179 ARRAY_SIZE(qm_cap_info_vf);
5180 u32 val, i;
5181
5182 /* Doorbell isolate register is a independent register. */
5183 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true);
5184 if (val)
5185 set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);
5186
5187 if (qm->ver >= QM_HW_V3) {
5188 val = readl(qm->io_base + QM_FUNC_CAPS_REG);
5189 qm->cap_ver = val & QM_CAPBILITY_VERSION;
5190 }
5191
5192 /* Get PF/VF common capbility */
5193 for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) {
5194 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver);
5195 if (val)
5196 set_bit(qm_cap_info_comm[i].type, &qm->caps);
5197 }
5198
5199 /* Get PF/VF different capbility */
5200 for (i = 0; i < size; i++) {
5201 val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver);
5202 if (val)
5203 set_bit(cap_info[i].type, &qm->caps);
5204 }
5205
5206 /* Fetch and save the value of irq type related capability registers */
5207 return qm_pre_store_irq_type_caps(qm);
5208 }
5209
qm_get_pci_res(struct hisi_qm * qm)5210 static int qm_get_pci_res(struct hisi_qm *qm)
5211 {
5212 struct pci_dev *pdev = qm->pdev;
5213 struct device *dev = &pdev->dev;
5214 int ret;
5215
5216 ret = pci_request_mem_regions(pdev, qm->dev_name);
5217 if (ret < 0) {
5218 dev_err(dev, "Failed to request mem regions!\n");
5219 return ret;
5220 }
5221
5222 qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);
5223 qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2));
5224 if (!qm->io_base) {
5225 ret = -EIO;
5226 goto err_request_mem_regions;
5227 }
5228
5229 ret = qm_get_hw_caps(qm);
5230 if (ret)
5231 goto err_ioremap;
5232
5233 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {
5234 qm->db_interval = QM_QP_DB_INTERVAL;
5235 qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);
5236 qm->db_io_base = ioremap(qm->db_phys_base,
5237 pci_resource_len(pdev, PCI_BAR_4));
5238 if (!qm->db_io_base) {
5239 ret = -EIO;
5240 goto err_ioremap;
5241 }
5242 } else {
5243 qm->db_phys_base = qm->phys_base;
5244 qm->db_io_base = qm->io_base;
5245 qm->db_interval = 0;
5246 }
5247
5248 ret = qm_get_qp_num(qm);
5249 if (ret)
5250 goto err_db_ioremap;
5251
5252 return 0;
5253
5254 err_db_ioremap:
5255 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))
5256 iounmap(qm->db_io_base);
5257 err_ioremap:
5258 iounmap(qm->io_base);
5259 err_request_mem_regions:
5260 pci_release_mem_regions(pdev);
5261 return ret;
5262 }
5263
hisi_qm_pci_init(struct hisi_qm * qm)5264 static int hisi_qm_pci_init(struct hisi_qm *qm)
5265 {
5266 struct pci_dev *pdev = qm->pdev;
5267 struct device *dev = &pdev->dev;
5268 unsigned int num_vec;
5269 int ret;
5270
5271 ret = pci_enable_device_mem(pdev);
5272 if (ret < 0) {
5273 dev_err(dev, "Failed to enable device mem!\n");
5274 return ret;
5275 }
5276
5277 ret = qm_get_pci_res(qm);
5278 if (ret)
5279 goto err_disable_pcidev;
5280
5281 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
5282 if (ret < 0)
5283 goto err_get_pci_res;
5284 pci_set_master(pdev);
5285
5286 num_vec = qm_get_irq_num(qm);
5287 ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
5288 if (ret < 0) {
5289 dev_err(dev, "Failed to enable MSI vectors!\n");
5290 goto err_get_pci_res;
5291 }
5292
5293 return 0;
5294
5295 err_get_pci_res:
5296 qm_put_pci_res(qm);
5297 err_disable_pcidev:
5298 pci_disable_device(pdev);
5299 return ret;
5300 }
5301
hisi_qm_init_work(struct hisi_qm * qm)5302 static int hisi_qm_init_work(struct hisi_qm *qm)
5303 {
5304 int i;
5305
5306 for (i = 0; i < qm->qp_num; i++)
5307 INIT_WORK(&qm->poll_data[i].work, qm_work_process);
5308
5309 if (qm->fun_type == QM_HW_PF)
5310 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset);
5311
5312 if (qm->ver > QM_HW_V2)
5313 INIT_WORK(&qm->cmd_process, qm_cmd_process);
5314
5315 qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM |
5316 WQ_UNBOUND, num_online_cpus(),
5317 pci_name(qm->pdev));
5318 if (!qm->wq) {
5319 pci_err(qm->pdev, "failed to alloc workqueue!\n");
5320 return -ENOMEM;
5321 }
5322
5323 return 0;
5324 }
5325
hisi_qp_alloc_memory(struct hisi_qm * qm)5326 static int hisi_qp_alloc_memory(struct hisi_qm *qm)
5327 {
5328 struct device *dev = &qm->pdev->dev;
5329 u16 sq_depth, cq_depth;
5330 size_t qp_dma_size;
5331 int i, ret;
5332
5333 qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);
5334 if (!qm->qp_array)
5335 return -ENOMEM;
5336
5337 qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL);
5338 if (!qm->poll_data) {
5339 kfree(qm->qp_array);
5340 return -ENOMEM;
5341 }
5342
5343 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);
5344
5345 /* one more page for device or qp statuses */
5346 qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth;
5347 qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE;
5348 for (i = 0; i < qm->qp_num; i++) {
5349 qm->poll_data[i].qm = qm;
5350 ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth);
5351 if (ret)
5352 goto err_init_qp_mem;
5353
5354 dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);
5355 }
5356
5357 return 0;
5358 err_init_qp_mem:
5359 hisi_qp_memory_uninit(qm, i);
5360
5361 return ret;
5362 }
5363
hisi_qm_memory_init(struct hisi_qm * qm)5364 static int hisi_qm_memory_init(struct hisi_qm *qm)
5365 {
5366 struct device *dev = &qm->pdev->dev;
5367 int ret, total_func;
5368 size_t off = 0;
5369
5370 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {
5371 total_func = pci_sriov_get_totalvfs(qm->pdev) + 1;
5372 qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL);
5373 if (!qm->factor)
5374 return -ENOMEM;
5375
5376 /* Only the PF value needs to be initialized */
5377 qm->factor[0].func_qos = QM_QOS_MAX_VAL;
5378 }
5379
5380 #define QM_INIT_BUF(qm, type, num) do { \
5381 (qm)->type = ((qm)->qdma.va + (off)); \
5382 (qm)->type##_dma = (qm)->qdma.dma + (off); \
5383 off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
5384 } while (0)
5385
5386 idr_init(&qm->qp_idr);
5387 qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP);
5388 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) +
5389 QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) +
5390 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
5391 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
5392 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,
5393 GFP_ATOMIC);
5394 dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);
5395 if (!qm->qdma.va) {
5396 ret = -ENOMEM;
5397 goto err_destroy_idr;
5398 }
5399
5400 QM_INIT_BUF(qm, eqe, qm->eq_depth);
5401 QM_INIT_BUF(qm, aeqe, qm->aeq_depth);
5402 QM_INIT_BUF(qm, sqc, qm->qp_num);
5403 QM_INIT_BUF(qm, cqc, qm->qp_num);
5404
5405 ret = hisi_qp_alloc_memory(qm);
5406 if (ret)
5407 goto err_alloc_qp_array;
5408
5409 return 0;
5410
5411 err_alloc_qp_array:
5412 dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
5413 err_destroy_idr:
5414 idr_destroy(&qm->qp_idr);
5415 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))
5416 kfree(qm->factor);
5417
5418 return ret;
5419 }
5420
5421 /**
5422 * hisi_qm_init() - Initialize configures about qm.
5423 * @qm: The qm needing init.
5424 *
5425 * This function init qm, then we can call hisi_qm_start to put qm into work.
5426 */
hisi_qm_init(struct hisi_qm * qm)5427 int hisi_qm_init(struct hisi_qm *qm)
5428 {
5429 struct pci_dev *pdev = qm->pdev;
5430 struct device *dev = &pdev->dev;
5431 int ret;
5432
5433 hisi_qm_pre_init(qm);
5434
5435 ret = hisi_qm_pci_init(qm);
5436 if (ret)
5437 return ret;
5438
5439 ret = qm_irqs_register(qm);
5440 if (ret)
5441 goto err_pci_init;
5442
5443 if (qm->fun_type == QM_HW_PF) {
5444 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */
5445 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);
5446 qm_disable_clock_gate(qm);
5447 ret = qm_dev_mem_reset(qm);
5448 if (ret) {
5449 dev_err(dev, "failed to reset device memory\n");
5450 goto err_irq_register;
5451 }
5452 }
5453
5454 if (qm->mode == UACCE_MODE_SVA) {
5455 ret = qm_alloc_uacce(qm);
5456 if (ret < 0)
5457 dev_warn(dev, "fail to alloc uacce (%d)\n", ret);
5458 }
5459
5460 ret = hisi_qm_memory_init(qm);
5461 if (ret)
5462 goto err_alloc_uacce;
5463
5464 ret = hisi_qm_init_work(qm);
5465 if (ret)
5466 goto err_free_qm_memory;
5467
5468 qm_cmd_init(qm);
5469 atomic_set(&qm->status.flags, QM_INIT);
5470
5471 return 0;
5472
5473 err_free_qm_memory:
5474 hisi_qm_memory_uninit(qm);
5475 err_alloc_uacce:
5476 qm_remove_uacce(qm);
5477 err_irq_register:
5478 qm_irqs_unregister(qm);
5479 err_pci_init:
5480 hisi_qm_pci_uninit(qm);
5481 return ret;
5482 }
5483 EXPORT_SYMBOL_GPL(hisi_qm_init);
5484
5485 /**
5486 * hisi_qm_get_dfx_access() - Try to get dfx access.
5487 * @qm: pointer to accelerator device.
5488 *
5489 * Try to get dfx access, then user can get message.
5490 *
5491 * If device is in suspended, return failure, otherwise
5492 * bump up the runtime PM usage counter.
5493 */
hisi_qm_get_dfx_access(struct hisi_qm * qm)5494 int hisi_qm_get_dfx_access(struct hisi_qm *qm)
5495 {
5496 struct device *dev = &qm->pdev->dev;
5497
5498 if (pm_runtime_suspended(dev)) {
5499 dev_info(dev, "can not read/write - device in suspended.\n");
5500 return -EAGAIN;
5501 }
5502
5503 return qm_pm_get_sync(qm);
5504 }
5505 EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access);
5506
5507 /**
5508 * hisi_qm_put_dfx_access() - Put dfx access.
5509 * @qm: pointer to accelerator device.
5510 *
5511 * Put dfx access, drop runtime PM usage counter.
5512 */
hisi_qm_put_dfx_access(struct hisi_qm * qm)5513 void hisi_qm_put_dfx_access(struct hisi_qm *qm)
5514 {
5515 qm_pm_put_sync(qm);
5516 }
5517 EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access);
5518
5519 /**
5520 * hisi_qm_pm_init() - Initialize qm runtime PM.
5521 * @qm: pointer to accelerator device.
5522 *
5523 * Function that initialize qm runtime PM.
5524 */
hisi_qm_pm_init(struct hisi_qm * qm)5525 void hisi_qm_pm_init(struct hisi_qm *qm)
5526 {
5527 struct device *dev = &qm->pdev->dev;
5528
5529 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
5530 return;
5531
5532 pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY);
5533 pm_runtime_use_autosuspend(dev);
5534 pm_runtime_put_noidle(dev);
5535 }
5536 EXPORT_SYMBOL_GPL(hisi_qm_pm_init);
5537
5538 /**
5539 * hisi_qm_pm_uninit() - Uninitialize qm runtime PM.
5540 * @qm: pointer to accelerator device.
5541 *
5542 * Function that uninitialize qm runtime PM.
5543 */
hisi_qm_pm_uninit(struct hisi_qm * qm)5544 void hisi_qm_pm_uninit(struct hisi_qm *qm)
5545 {
5546 struct device *dev = &qm->pdev->dev;
5547
5548 if (!test_bit(QM_SUPPORT_RPM, &qm->caps))
5549 return;
5550
5551 pm_runtime_get_noresume(dev);
5552 pm_runtime_dont_use_autosuspend(dev);
5553 }
5554 EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit);
5555
qm_prepare_for_suspend(struct hisi_qm * qm)5556 static int qm_prepare_for_suspend(struct hisi_qm *qm)
5557 {
5558 struct pci_dev *pdev = qm->pdev;
5559 int ret;
5560 u32 val;
5561
5562 ret = qm->ops->set_msi(qm, false);
5563 if (ret) {
5564 pci_err(pdev, "failed to disable MSI before suspending!\n");
5565 return ret;
5566 }
5567
5568 /* shutdown OOO register */
5569 writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN,
5570 qm->io_base + ACC_MASTER_GLOBAL_CTRL);
5571
5572 ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
5573 val,
5574 (val == ACC_MASTER_TRANS_RETURN_RW),
5575 POLL_PERIOD, POLL_TIMEOUT);
5576 if (ret) {
5577 pci_emerg(pdev, "Bus lock! Please reset system.\n");
5578 return ret;
5579 }
5580
5581 ret = qm_set_pf_mse(qm, false);
5582 if (ret)
5583 pci_err(pdev, "failed to disable MSE before suspending!\n");
5584
5585 return ret;
5586 }
5587
qm_rebuild_for_resume(struct hisi_qm * qm)5588 static int qm_rebuild_for_resume(struct hisi_qm *qm)
5589 {
5590 struct pci_dev *pdev = qm->pdev;
5591 int ret;
5592
5593 ret = qm_set_pf_mse(qm, true);
5594 if (ret) {
5595 pci_err(pdev, "failed to enable MSE after resuming!\n");
5596 return ret;
5597 }
5598
5599 ret = qm->ops->set_msi(qm, true);
5600 if (ret) {
5601 pci_err(pdev, "failed to enable MSI after resuming!\n");
5602 return ret;
5603 }
5604
5605 ret = qm_dev_hw_init(qm);
5606 if (ret) {
5607 pci_err(pdev, "failed to init device after resuming\n");
5608 return ret;
5609 }
5610
5611 qm_cmd_init(qm);
5612 hisi_qm_dev_err_init(qm);
5613 /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */
5614 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);
5615 qm_disable_clock_gate(qm);
5616 ret = qm_dev_mem_reset(qm);
5617 if (ret)
5618 pci_err(pdev, "failed to reset device memory\n");
5619
5620 return ret;
5621 }
5622
5623 /**
5624 * hisi_qm_suspend() - Runtime suspend of given device.
5625 * @dev: device to suspend.
5626 *
5627 * Function that suspend the device.
5628 */
hisi_qm_suspend(struct device * dev)5629 int hisi_qm_suspend(struct device *dev)
5630 {
5631 struct pci_dev *pdev = to_pci_dev(dev);
5632 struct hisi_qm *qm = pci_get_drvdata(pdev);
5633 int ret;
5634
5635 pci_info(pdev, "entering suspended state\n");
5636
5637 ret = hisi_qm_stop(qm, QM_NORMAL);
5638 if (ret) {
5639 pci_err(pdev, "failed to stop qm(%d)\n", ret);
5640 return ret;
5641 }
5642
5643 ret = qm_prepare_for_suspend(qm);
5644 if (ret)
5645 pci_err(pdev, "failed to prepare suspended(%d)\n", ret);
5646
5647 return ret;
5648 }
5649 EXPORT_SYMBOL_GPL(hisi_qm_suspend);
5650
5651 /**
5652 * hisi_qm_resume() - Runtime resume of given device.
5653 * @dev: device to resume.
5654 *
5655 * Function that resume the device.
5656 */
hisi_qm_resume(struct device * dev)5657 int hisi_qm_resume(struct device *dev)
5658 {
5659 struct pci_dev *pdev = to_pci_dev(dev);
5660 struct hisi_qm *qm = pci_get_drvdata(pdev);
5661 int ret;
5662
5663 pci_info(pdev, "resuming from suspend state\n");
5664
5665 ret = qm_rebuild_for_resume(qm);
5666 if (ret) {
5667 pci_err(pdev, "failed to rebuild resume(%d)\n", ret);
5668 return ret;
5669 }
5670
5671 ret = hisi_qm_start(qm);
5672 if (ret) {
5673 if (qm_check_dev_error(qm)) {
5674 pci_info(pdev, "failed to start qm due to device error, device will be reset!\n");
5675 return 0;
5676 }
5677
5678 pci_err(pdev, "failed to start qm(%d)!\n", ret);
5679 }
5680
5681 return ret;
5682 }
5683 EXPORT_SYMBOL_GPL(hisi_qm_resume);
5684
5685 MODULE_LICENSE("GPL v2");
5686 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
5687 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");
5688