xref: /openbmc/linux/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c (revision 9144f784f852f9a125cabe9927b986d909bfa439)
1  // SPDX-License-Identifier: GPL-2.0-or-later
2  /*
3   * Copyright (c) 2017 Hisilicon Limited.
4   */
5  
6  #include <linux/sched/clock.h>
7  #include "hisi_sas.h"
8  #define DRV_NAME "hisi_sas_v3_hw"
9  
10  /* global registers need init */
11  #define DLVRY_QUEUE_ENABLE		0x0
12  #define IOST_BASE_ADDR_LO		0x8
13  #define IOST_BASE_ADDR_HI		0xc
14  #define ITCT_BASE_ADDR_LO		0x10
15  #define ITCT_BASE_ADDR_HI		0x14
16  #define IO_BROKEN_MSG_ADDR_LO		0x18
17  #define IO_BROKEN_MSG_ADDR_HI		0x1c
18  #define PHY_CONTEXT			0x20
19  #define PHY_STATE			0x24
20  #define PHY_PORT_NUM_MA			0x28
21  #define PHY_CONN_RATE			0x30
22  #define ITCT_CLR			0x44
23  #define ITCT_CLR_EN_OFF			16
24  #define ITCT_CLR_EN_MSK			(0x1 << ITCT_CLR_EN_OFF)
25  #define ITCT_DEV_OFF			0
26  #define ITCT_DEV_MSK			(0x7ff << ITCT_DEV_OFF)
27  #define SAS_AXI_USER3			0x50
28  #define IO_SATA_BROKEN_MSG_ADDR_LO	0x58
29  #define IO_SATA_BROKEN_MSG_ADDR_HI	0x5c
30  #define SATA_INITI_D2H_STORE_ADDR_LO	0x60
31  #define SATA_INITI_D2H_STORE_ADDR_HI	0x64
32  #define CFG_MAX_TAG			0x68
33  #define TRANS_LOCK_ICT_TIME		0X70
34  #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL	0x84
35  #define HGC_SAS_TXFAIL_RETRY_CTRL	0x88
36  #define HGC_GET_ITV_TIME		0x90
37  #define DEVICE_MSG_WORK_MODE		0x94
38  #define OPENA_WT_CONTI_TIME		0x9c
39  #define I_T_NEXUS_LOSS_TIME		0xa0
40  #define MAX_CON_TIME_LIMIT_TIME		0xa4
41  #define BUS_INACTIVE_LIMIT_TIME		0xa8
42  #define REJECT_TO_OPEN_LIMIT_TIME	0xac
43  #define CQ_INT_CONVERGE_EN		0xb0
44  #define CFG_AGING_TIME			0xbc
45  #define HGC_DFX_CFG2			0xc0
46  #define CFG_ABT_SET_QUERY_IPTT	0xd4
47  #define CFG_SET_ABORTED_IPTT_OFF	0
48  #define CFG_SET_ABORTED_IPTT_MSK	(0xfff << CFG_SET_ABORTED_IPTT_OFF)
49  #define CFG_SET_ABORTED_EN_OFF	12
50  #define CFG_ABT_SET_IPTT_DONE	0xd8
51  #define CFG_ABT_SET_IPTT_DONE_OFF	0
52  #define HGC_IOMB_PROC1_STATUS	0x104
53  #define HGC_LM_DFX_STATUS2		0x128
54  #define HGC_LM_DFX_STATUS2_IOSTLIST_OFF		0
55  #define HGC_LM_DFX_STATUS2_IOSTLIST_MSK	(0xfff << \
56  					 HGC_LM_DFX_STATUS2_IOSTLIST_OFF)
57  #define HGC_LM_DFX_STATUS2_ITCTLIST_OFF		12
58  #define HGC_LM_DFX_STATUS2_ITCTLIST_MSK	(0x7ff << \
59  					 HGC_LM_DFX_STATUS2_ITCTLIST_OFF)
60  #define HGC_CQE_ECC_ADDR		0x13c
61  #define HGC_CQE_ECC_1B_ADDR_OFF	0
62  #define HGC_CQE_ECC_1B_ADDR_MSK	(0x3f << HGC_CQE_ECC_1B_ADDR_OFF)
63  #define HGC_CQE_ECC_MB_ADDR_OFF	8
64  #define HGC_CQE_ECC_MB_ADDR_MSK (0x3f << HGC_CQE_ECC_MB_ADDR_OFF)
65  #define HGC_IOST_ECC_ADDR		0x140
66  #define HGC_IOST_ECC_1B_ADDR_OFF	0
67  #define HGC_IOST_ECC_1B_ADDR_MSK	(0x3ff << HGC_IOST_ECC_1B_ADDR_OFF)
68  #define HGC_IOST_ECC_MB_ADDR_OFF	16
69  #define HGC_IOST_ECC_MB_ADDR_MSK	(0x3ff << HGC_IOST_ECC_MB_ADDR_OFF)
70  #define HGC_DQE_ECC_ADDR		0x144
71  #define HGC_DQE_ECC_1B_ADDR_OFF	0
72  #define HGC_DQE_ECC_1B_ADDR_MSK	(0xfff << HGC_DQE_ECC_1B_ADDR_OFF)
73  #define HGC_DQE_ECC_MB_ADDR_OFF	16
74  #define HGC_DQE_ECC_MB_ADDR_MSK (0xfff << HGC_DQE_ECC_MB_ADDR_OFF)
75  #define CHNL_INT_STATUS			0x148
76  #define TAB_DFX				0x14c
77  #define HGC_ITCT_ECC_ADDR		0x150
78  #define HGC_ITCT_ECC_1B_ADDR_OFF		0
79  #define HGC_ITCT_ECC_1B_ADDR_MSK		(0x3ff << \
80  						 HGC_ITCT_ECC_1B_ADDR_OFF)
81  #define HGC_ITCT_ECC_MB_ADDR_OFF		16
82  #define HGC_ITCT_ECC_MB_ADDR_MSK		(0x3ff << \
83  						 HGC_ITCT_ECC_MB_ADDR_OFF)
84  #define HGC_AXI_FIFO_ERR_INFO  0x154
85  #define AXI_ERR_INFO_OFF               0
86  #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
87  #define FIFO_ERR_INFO_OFF              8
88  #define FIFO_ERR_INFO_MSK              (0xff << FIFO_ERR_INFO_OFF)
89  #define TAB_RD_TYPE			0x15c
90  #define INT_COAL_EN			0x19c
91  #define OQ_INT_COAL_TIME		0x1a0
92  #define OQ_INT_COAL_CNT			0x1a4
93  #define ENT_INT_COAL_TIME		0x1a8
94  #define ENT_INT_COAL_CNT		0x1ac
95  #define OQ_INT_SRC			0x1b0
96  #define OQ_INT_SRC_MSK			0x1b4
97  #define ENT_INT_SRC1			0x1b8
98  #define ENT_INT_SRC1_D2H_FIS_CH0_OFF	0
99  #define ENT_INT_SRC1_D2H_FIS_CH0_MSK	(0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
100  #define ENT_INT_SRC1_D2H_FIS_CH1_OFF	8
101  #define ENT_INT_SRC1_D2H_FIS_CH1_MSK	(0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
102  #define ENT_INT_SRC2			0x1bc
103  #define ENT_INT_SRC3			0x1c0
104  #define ENT_INT_SRC3_WP_DEPTH_OFF		8
105  #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF	9
106  #define ENT_INT_SRC3_RP_DEPTH_OFF		10
107  #define ENT_INT_SRC3_AXI_OFF			11
108  #define ENT_INT_SRC3_FIFO_OFF			12
109  #define ENT_INT_SRC3_LM_OFF				14
110  #define ENT_INT_SRC3_ITC_INT_OFF	15
111  #define ENT_INT_SRC3_ITC_INT_MSK	(0x1 << ENT_INT_SRC3_ITC_INT_OFF)
112  #define ENT_INT_SRC3_ABT_OFF		16
113  #define ENT_INT_SRC3_DQE_POISON_OFF	18
114  #define ENT_INT_SRC3_IOST_POISON_OFF	19
115  #define ENT_INT_SRC3_ITCT_POISON_OFF	20
116  #define ENT_INT_SRC3_ITCT_NCQ_POISON_OFF	21
117  #define ENT_INT_SRC_MSK1		0x1c4
118  #define ENT_INT_SRC_MSK2		0x1c8
119  #define ENT_INT_SRC_MSK3		0x1cc
120  #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF	31
121  #define CHNL_PHYUPDOWN_INT_MSK		0x1d0
122  #define CHNL_ENT_INT_MSK			0x1d4
123  #define HGC_COM_INT_MSK				0x1d8
124  #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK	(0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
125  #define SAS_ECC_INTR			0x1e8
126  #define SAS_ECC_INTR_DQE_ECC_1B_OFF		0
127  #define SAS_ECC_INTR_DQE_ECC_MB_OFF		1
128  #define SAS_ECC_INTR_IOST_ECC_1B_OFF	2
129  #define SAS_ECC_INTR_IOST_ECC_MB_OFF	3
130  #define SAS_ECC_INTR_ITCT_ECC_1B_OFF	4
131  #define SAS_ECC_INTR_ITCT_ECC_MB_OFF	5
132  #define SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF	6
133  #define SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF	7
134  #define SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF	8
135  #define SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF	9
136  #define SAS_ECC_INTR_CQE_ECC_1B_OFF		10
137  #define SAS_ECC_INTR_CQE_ECC_MB_OFF		11
138  #define SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF	12
139  #define SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF	13
140  #define SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF	14
141  #define SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF	15
142  #define SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF	16
143  #define SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF	17
144  #define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF	18
145  #define SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF	19
146  #define SAS_ECC_INTR_OOO_RAM_ECC_1B_OFF		20
147  #define SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF		21
148  #define SAS_ECC_INTR_MSK		0x1ec
149  #define HGC_ERR_STAT_EN			0x238
150  #define CQE_SEND_CNT			0x248
151  #define DLVRY_Q_0_BASE_ADDR_LO		0x260
152  #define DLVRY_Q_0_BASE_ADDR_HI		0x264
153  #define DLVRY_Q_0_DEPTH			0x268
154  #define DLVRY_Q_0_WR_PTR		0x26c
155  #define DLVRY_Q_0_RD_PTR		0x270
156  #define HYPER_STREAM_ID_EN_CFG		0xc80
157  #define OQ0_INT_SRC_MSK			0xc90
158  #define COMPL_Q_0_BASE_ADDR_LO		0x4e0
159  #define COMPL_Q_0_BASE_ADDR_HI		0x4e4
160  #define COMPL_Q_0_DEPTH			0x4e8
161  #define COMPL_Q_0_WR_PTR		0x4ec
162  #define COMPL_Q_0_RD_PTR		0x4f0
163  #define HGC_RXM_DFX_STATUS14		0xae8
164  #define HGC_RXM_DFX_STATUS14_MEM0_OFF	0
165  #define HGC_RXM_DFX_STATUS14_MEM0_MSK	(0x1ff << \
166  					 HGC_RXM_DFX_STATUS14_MEM0_OFF)
167  #define HGC_RXM_DFX_STATUS14_MEM1_OFF	9
168  #define HGC_RXM_DFX_STATUS14_MEM1_MSK	(0x1ff << \
169  					 HGC_RXM_DFX_STATUS14_MEM1_OFF)
170  #define HGC_RXM_DFX_STATUS14_MEM2_OFF	18
171  #define HGC_RXM_DFX_STATUS14_MEM2_MSK	(0x1ff << \
172  					 HGC_RXM_DFX_STATUS14_MEM2_OFF)
173  #define HGC_RXM_DFX_STATUS15		0xaec
174  #define HGC_RXM_DFX_STATUS15_MEM3_OFF	0
175  #define HGC_RXM_DFX_STATUS15_MEM3_MSK	(0x1ff << \
176  					 HGC_RXM_DFX_STATUS15_MEM3_OFF)
177  #define AWQOS_AWCACHE_CFG	0xc84
178  #define ARQOS_ARCACHE_CFG	0xc88
179  #define HILINK_ERR_DFX		0xe04
180  #define SAS_GPIO_CFG_0		0x1000
181  #define SAS_GPIO_CFG_1		0x1004
182  #define SAS_GPIO_TX_0_1	0x1040
183  #define SAS_CFG_DRIVE_VLD	0x1070
184  
185  /* phy registers requiring init */
186  #define PORT_BASE			(0x2000)
187  #define PHY_CFG				(PORT_BASE + 0x0)
188  #define HARD_PHY_LINKRATE		(PORT_BASE + 0x4)
189  #define PHY_CFG_ENA_OFF			0
190  #define PHY_CFG_ENA_MSK			(0x1 << PHY_CFG_ENA_OFF)
191  #define PHY_CFG_DC_OPT_OFF		2
192  #define PHY_CFG_DC_OPT_MSK		(0x1 << PHY_CFG_DC_OPT_OFF)
193  #define PHY_CFG_PHY_RST_OFF		3
194  #define PHY_CFG_PHY_RST_MSK		(0x1 << PHY_CFG_PHY_RST_OFF)
195  #define PROG_PHY_LINK_RATE		(PORT_BASE + 0x8)
196  #define CFG_PROG_PHY_LINK_RATE_OFF	0
197  #define CFG_PROG_PHY_LINK_RATE_MSK	(0xff << CFG_PROG_PHY_LINK_RATE_OFF)
198  #define CFG_PROG_OOB_PHY_LINK_RATE_OFF	8
199  #define CFG_PROG_OOB_PHY_LINK_RATE_MSK	(0xf << CFG_PROG_OOB_PHY_LINK_RATE_OFF)
200  #define PHY_CTRL			(PORT_BASE + 0x14)
201  #define PHY_CTRL_RESET_OFF		0
202  #define PHY_CTRL_RESET_MSK		(0x1 << PHY_CTRL_RESET_OFF)
203  #define CMD_HDR_PIR_OFF			8
204  #define CMD_HDR_PIR_MSK			(0x1 << CMD_HDR_PIR_OFF)
205  #define SERDES_CFG			(PORT_BASE + 0x1c)
206  #define CFG_ALOS_CHK_DISABLE_OFF	9
207  #define CFG_ALOS_CHK_DISABLE_MSK	(0x1 << CFG_ALOS_CHK_DISABLE_OFF)
208  #define SAS_PHY_BIST_CTRL		(PORT_BASE + 0x2c)
209  #define CFG_BIST_MODE_SEL_OFF		0
210  #define CFG_BIST_MODE_SEL_MSK		(0xf << CFG_BIST_MODE_SEL_OFF)
211  #define CFG_LOOP_TEST_MODE_OFF		14
212  #define CFG_LOOP_TEST_MODE_MSK		(0x3 << CFG_LOOP_TEST_MODE_OFF)
213  #define CFG_RX_BIST_EN_OFF		16
214  #define CFG_RX_BIST_EN_MSK		(0x1 << CFG_RX_BIST_EN_OFF)
215  #define CFG_TX_BIST_EN_OFF		17
216  #define CFG_TX_BIST_EN_MSK		(0x1 << CFG_TX_BIST_EN_OFF)
217  #define CFG_BIST_TEST_OFF		18
218  #define CFG_BIST_TEST_MSK		(0x1 << CFG_BIST_TEST_OFF)
219  #define SAS_PHY_BIST_CODE		(PORT_BASE + 0x30)
220  #define SAS_PHY_BIST_CODE1		(PORT_BASE + 0x34)
221  #define SAS_BIST_ERR_CNT		(PORT_BASE + 0x38)
222  #define SL_CFG				(PORT_BASE + 0x84)
223  #define AIP_LIMIT			(PORT_BASE + 0x90)
224  #define SL_CONTROL			(PORT_BASE + 0x94)
225  #define SL_CONTROL_NOTIFY_EN_OFF	0
226  #define SL_CONTROL_NOTIFY_EN_MSK	(0x1 << SL_CONTROL_NOTIFY_EN_OFF)
227  #define SL_CTA_OFF		17
228  #define SL_CTA_MSK		(0x1 << SL_CTA_OFF)
229  #define RX_PRIMS_STATUS			(PORT_BASE + 0x98)
230  #define RX_BCAST_CHG_OFF		1
231  #define RX_BCAST_CHG_MSK		(0x1 << RX_BCAST_CHG_OFF)
232  #define TX_ID_DWORD0			(PORT_BASE + 0x9c)
233  #define TX_ID_DWORD1			(PORT_BASE + 0xa0)
234  #define TX_ID_DWORD2			(PORT_BASE + 0xa4)
235  #define TX_ID_DWORD3			(PORT_BASE + 0xa8)
236  #define TX_ID_DWORD4			(PORT_BASE + 0xaC)
237  #define TX_ID_DWORD5			(PORT_BASE + 0xb0)
238  #define TX_ID_DWORD6			(PORT_BASE + 0xb4)
239  #define TXID_AUTO				(PORT_BASE + 0xb8)
240  #define CT3_OFF		1
241  #define CT3_MSK		(0x1 << CT3_OFF)
242  #define TX_HARDRST_OFF          2
243  #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
244  #define RX_IDAF_DWORD0			(PORT_BASE + 0xc4)
245  #define RXOP_CHECK_CFG_H		(PORT_BASE + 0xfc)
246  #define STP_LINK_TIMER			(PORT_BASE + 0x120)
247  #define STP_LINK_TIMEOUT_STATE		(PORT_BASE + 0x124)
248  #define CON_CFG_DRIVER			(PORT_BASE + 0x130)
249  #define SAS_SSP_CON_TIMER_CFG		(PORT_BASE + 0x134)
250  #define SAS_SMP_CON_TIMER_CFG		(PORT_BASE + 0x138)
251  #define SAS_STP_CON_TIMER_CFG		(PORT_BASE + 0x13c)
252  #define CHL_INT0			(PORT_BASE + 0x1b4)
253  #define CHL_INT0_HOTPLUG_TOUT_OFF	0
254  #define CHL_INT0_HOTPLUG_TOUT_MSK	(0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
255  #define CHL_INT0_SL_RX_BCST_ACK_OFF	1
256  #define CHL_INT0_SL_RX_BCST_ACK_MSK	(0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
257  #define CHL_INT0_SL_PHY_ENABLE_OFF	2
258  #define CHL_INT0_SL_PHY_ENABLE_MSK	(0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
259  #define CHL_INT0_NOT_RDY_OFF		4
260  #define CHL_INT0_NOT_RDY_MSK		(0x1 << CHL_INT0_NOT_RDY_OFF)
261  #define CHL_INT0_PHY_RDY_OFF		5
262  #define CHL_INT0_PHY_RDY_MSK		(0x1 << CHL_INT0_PHY_RDY_OFF)
263  #define CHL_INT1			(PORT_BASE + 0x1b8)
264  #define CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF	15
265  #define CHL_INT1_DMAC_TX_ECC_1B_ERR_OFF	16
266  #define CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF	17
267  #define CHL_INT1_DMAC_RX_ECC_1B_ERR_OFF	18
268  #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF	19
269  #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF	20
270  #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF	21
271  #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF	22
272  #define CHL_INT1_DMAC_TX_FIFO_ERR_OFF	23
273  #define CHL_INT1_DMAC_RX_FIFO_ERR_OFF	24
274  #define CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF	26
275  #define CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF	27
276  #define CHL_INT2			(PORT_BASE + 0x1bc)
277  #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF	0
278  #define CHL_INT2_RX_DISP_ERR_OFF	28
279  #define CHL_INT2_RX_CODE_ERR_OFF	29
280  #define CHL_INT2_RX_INVLD_DW_OFF	30
281  #define CHL_INT2_STP_LINK_TIMEOUT_OFF	31
282  #define CHL_INT0_MSK			(PORT_BASE + 0x1c0)
283  #define CHL_INT1_MSK			(PORT_BASE + 0x1c4)
284  #define CHL_INT2_MSK			(PORT_BASE + 0x1c8)
285  #define SAS_EC_INT_COAL_TIME		(PORT_BASE + 0x1cc)
286  #define CHL_INT_COAL_EN			(PORT_BASE + 0x1d0)
287  #define SAS_RX_TRAIN_TIMER		(PORT_BASE + 0x2a4)
288  #define PHY_CTRL_RDY_MSK		(PORT_BASE + 0x2b0)
289  #define PHYCTRL_NOT_RDY_MSK		(PORT_BASE + 0x2b4)
290  #define PHYCTRL_DWS_RESET_MSK		(PORT_BASE + 0x2b8)
291  #define PHYCTRL_PHY_ENA_MSK		(PORT_BASE + 0x2bc)
292  #define SL_RX_BCAST_CHK_MSK		(PORT_BASE + 0x2c0)
293  #define PHYCTRL_OOB_RESTART_MSK		(PORT_BASE + 0x2c4)
294  #define DMA_TX_STATUS			(PORT_BASE + 0x2d0)
295  #define DMA_TX_STATUS_BUSY_OFF		0
296  #define DMA_TX_STATUS_BUSY_MSK		(0x1 << DMA_TX_STATUS_BUSY_OFF)
297  #define DMA_RX_STATUS			(PORT_BASE + 0x2e8)
298  #define DMA_RX_STATUS_BUSY_OFF		0
299  #define DMA_RX_STATUS_BUSY_MSK		(0x1 << DMA_RX_STATUS_BUSY_OFF)
300  
301  #define COARSETUNE_TIME			(PORT_BASE + 0x304)
302  #define TXDEEMPH_G1			(PORT_BASE + 0x350)
303  #define ERR_CNT_DWS_LOST		(PORT_BASE + 0x380)
304  #define ERR_CNT_RESET_PROB		(PORT_BASE + 0x384)
305  #define ERR_CNT_INVLD_DW		(PORT_BASE + 0x390)
306  #define ERR_CNT_CODE_ERR		(PORT_BASE + 0x394)
307  #define ERR_CNT_DISP_ERR		(PORT_BASE + 0x398)
308  #define DFX_FIFO_CTRL			(PORT_BASE + 0x3a0)
309  #define DFX_FIFO_CTRL_TRIGGER_MODE_OFF	0
310  #define DFX_FIFO_CTRL_TRIGGER_MODE_MSK	(0x7 << DFX_FIFO_CTRL_TRIGGER_MODE_OFF)
311  #define DFX_FIFO_CTRL_DUMP_MODE_OFF	3
312  #define DFX_FIFO_CTRL_DUMP_MODE_MSK	(0x7 << DFX_FIFO_CTRL_DUMP_MODE_OFF)
313  #define DFX_FIFO_CTRL_SIGNAL_SEL_OFF	6
314  #define DFX_FIFO_CTRL_SIGNAL_SEL_MSK	(0xF << DFX_FIFO_CTRL_SIGNAL_SEL_OFF)
315  #define DFX_FIFO_CTRL_DUMP_DISABLE_OFF	10
316  #define DFX_FIFO_CTRL_DUMP_DISABLE_MSK	(0x1 << DFX_FIFO_CTRL_DUMP_DISABLE_OFF)
317  #define DFX_FIFO_TRIGGER		(PORT_BASE + 0x3a4)
318  #define DFX_FIFO_TRIGGER_MSK		(PORT_BASE + 0x3a8)
319  #define DFX_FIFO_DUMP_MSK		(PORT_BASE + 0x3aC)
320  #define DFX_FIFO_RD_DATA		(PORT_BASE + 0x3b0)
321  
322  #define DEFAULT_ITCT_HW		2048 /* reset value, not reprogrammed */
323  #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
324  #error Max ITCT exceeded
325  #endif
326  
327  #define AXI_MASTER_CFG_BASE		(0x5000)
328  #define AM_CTRL_GLOBAL			(0x0)
329  #define AM_CTRL_SHUTDOWN_REQ_OFF	0
330  #define AM_CTRL_SHUTDOWN_REQ_MSK	(0x1 << AM_CTRL_SHUTDOWN_REQ_OFF)
331  #define AM_CURR_TRANS_RETURN	(0x150)
332  
333  #define AM_CFG_MAX_TRANS		(0x5010)
334  #define AM_CFG_SINGLE_PORT_MAX_TRANS	(0x5014)
335  #define AXI_CFG					(0x5100)
336  #define AM_ROB_ECC_ERR_ADDR		(0x510c)
337  #define AM_ROB_ECC_ERR_ADDR_OFF	0
338  #define AM_ROB_ECC_ERR_ADDR_MSK	0xffffffff
339  
340  /* RAS registers need init */
341  #define RAS_BASE		(0x6000)
342  #define SAS_RAS_INTR0			(RAS_BASE)
343  #define SAS_RAS_INTR1			(RAS_BASE + 0x04)
344  #define SAS_RAS_INTR0_MASK		(RAS_BASE + 0x08)
345  #define SAS_RAS_INTR1_MASK		(RAS_BASE + 0x0c)
346  #define CFG_SAS_RAS_INTR_MASK		(RAS_BASE + 0x1c)
347  #define SAS_RAS_INTR2			(RAS_BASE + 0x20)
348  #define SAS_RAS_INTR2_MASK		(RAS_BASE + 0x24)
349  
350  /* HW dma structures */
351  /* Delivery queue header */
352  /* dw0 */
353  #define CMD_HDR_ABORT_FLAG_OFF		0
354  #define CMD_HDR_ABORT_FLAG_MSK		(0x3 << CMD_HDR_ABORT_FLAG_OFF)
355  #define CMD_HDR_ABORT_DEVICE_TYPE_OFF	2
356  #define CMD_HDR_ABORT_DEVICE_TYPE_MSK	(0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
357  #define CMD_HDR_RESP_REPORT_OFF		5
358  #define CMD_HDR_RESP_REPORT_MSK		(0x1 << CMD_HDR_RESP_REPORT_OFF)
359  #define CMD_HDR_TLR_CTRL_OFF		6
360  #define CMD_HDR_TLR_CTRL_MSK		(0x3 << CMD_HDR_TLR_CTRL_OFF)
361  #define CMD_HDR_PORT_OFF		18
362  #define CMD_HDR_PORT_MSK		(0xf << CMD_HDR_PORT_OFF)
363  #define CMD_HDR_PRIORITY_OFF		27
364  #define CMD_HDR_PRIORITY_MSK		(0x1 << CMD_HDR_PRIORITY_OFF)
365  #define CMD_HDR_CMD_OFF			29
366  #define CMD_HDR_CMD_MSK			(0x7 << CMD_HDR_CMD_OFF)
367  /* dw1 */
368  #define CMD_HDR_UNCON_CMD_OFF	3
369  #define CMD_HDR_DIR_OFF			5
370  #define CMD_HDR_DIR_MSK			(0x3 << CMD_HDR_DIR_OFF)
371  #define CMD_HDR_RESET_OFF		7
372  #define CMD_HDR_RESET_MSK		(0x1 << CMD_HDR_RESET_OFF)
373  #define CMD_HDR_VDTL_OFF		10
374  #define CMD_HDR_VDTL_MSK		(0x1 << CMD_HDR_VDTL_OFF)
375  #define CMD_HDR_FRAME_TYPE_OFF		11
376  #define CMD_HDR_FRAME_TYPE_MSK		(0x1f << CMD_HDR_FRAME_TYPE_OFF)
377  #define CMD_HDR_DEV_ID_OFF		16
378  #define CMD_HDR_DEV_ID_MSK		(0xffff << CMD_HDR_DEV_ID_OFF)
379  /* dw2 */
380  #define CMD_HDR_CFL_OFF			0
381  #define CMD_HDR_CFL_MSK			(0x1ff << CMD_HDR_CFL_OFF)
382  #define CMD_HDR_NCQ_TAG_OFF		10
383  #define CMD_HDR_NCQ_TAG_MSK		(0x1f << CMD_HDR_NCQ_TAG_OFF)
384  #define CMD_HDR_MRFL_OFF		15
385  #define CMD_HDR_MRFL_MSK		(0x1ff << CMD_HDR_MRFL_OFF)
386  #define CMD_HDR_SG_MOD_OFF		24
387  #define CMD_HDR_SG_MOD_MSK		(0x3 << CMD_HDR_SG_MOD_OFF)
388  /* dw3 */
389  #define CMD_HDR_IPTT_OFF		0
390  #define CMD_HDR_IPTT_MSK		(0xffff << CMD_HDR_IPTT_OFF)
391  /* dw6 */
392  #define CMD_HDR_DIF_SGL_LEN_OFF		0
393  #define CMD_HDR_DIF_SGL_LEN_MSK		(0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
394  #define CMD_HDR_DATA_SGL_LEN_OFF	16
395  #define CMD_HDR_DATA_SGL_LEN_MSK	(0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
396  /* dw7 */
397  #define CMD_HDR_ADDR_MODE_SEL_OFF		15
398  #define CMD_HDR_ADDR_MODE_SEL_MSK		(1 << CMD_HDR_ADDR_MODE_SEL_OFF)
399  #define CMD_HDR_ABORT_IPTT_OFF		16
400  #define CMD_HDR_ABORT_IPTT_MSK		(0xffff << CMD_HDR_ABORT_IPTT_OFF)
401  
402  /* Completion header */
403  /* dw0 */
404  #define CMPLT_HDR_CMPLT_OFF		0
405  #define CMPLT_HDR_CMPLT_MSK		(0x3 << CMPLT_HDR_CMPLT_OFF)
406  #define CMPLT_HDR_ERROR_PHASE_OFF   2
407  #define CMPLT_HDR_ERROR_PHASE_MSK   (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
408  /* bit[9:2] Error Phase */
409  #define ERR_PHASE_RESPONSE_FRAME_REV_STAGE_OFF	\
410  					8
411  #define ERR_PHASE_RESPONSE_FRAME_REV_STAGE_MSK	\
412  	(0x1 << ERR_PHASE_RESPONSE_FRAME_REV_STAGE_OFF)
413  #define CMPLT_HDR_RSPNS_XFRD_OFF	10
414  #define CMPLT_HDR_RSPNS_XFRD_MSK	(0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
415  #define CMPLT_HDR_RSPNS_GOOD_OFF	11
416  #define CMPLT_HDR_RSPNS_GOOD_MSK	(0x1 << CMPLT_HDR_RSPNS_GOOD_OFF)
417  #define CMPLT_HDR_ERX_OFF		12
418  #define CMPLT_HDR_ERX_MSK		(0x1 << CMPLT_HDR_ERX_OFF)
419  #define CMPLT_HDR_ABORT_STAT_OFF	13
420  #define CMPLT_HDR_ABORT_STAT_MSK	(0x7 << CMPLT_HDR_ABORT_STAT_OFF)
421  /* abort_stat */
422  #define STAT_IO_NOT_VALID		0x1
423  #define STAT_IO_NO_DEVICE		0x2
424  #define STAT_IO_COMPLETE		0x3
425  #define STAT_IO_ABORTED			0x4
426  /* dw1 */
427  #define CMPLT_HDR_IPTT_OFF		0
428  #define CMPLT_HDR_IPTT_MSK		(0xffff << CMPLT_HDR_IPTT_OFF)
429  #define CMPLT_HDR_DEV_ID_OFF		16
430  #define CMPLT_HDR_DEV_ID_MSK		(0xffff << CMPLT_HDR_DEV_ID_OFF)
431  /* dw3 */
432  #define SATA_DISK_IN_ERROR_STATUS_OFF	8
433  #define SATA_DISK_IN_ERROR_STATUS_MSK	(0x1 << SATA_DISK_IN_ERROR_STATUS_OFF)
434  #define CMPLT_HDR_SATA_DISK_ERR_OFF	16
435  #define CMPLT_HDR_SATA_DISK_ERR_MSK	(0x1 << CMPLT_HDR_SATA_DISK_ERR_OFF)
436  #define CMPLT_HDR_IO_IN_TARGET_OFF	17
437  #define CMPLT_HDR_IO_IN_TARGET_MSK	(0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
438  /* bit[23:18] ERR_FIS_ATA_STATUS */
439  #define FIS_ATA_STATUS_ERR_OFF		18
440  #define FIS_ATA_STATUS_ERR_MSK		(0x1 << FIS_ATA_STATUS_ERR_OFF)
441  #define FIS_TYPE_SDB_OFF		31
442  #define FIS_TYPE_SDB_MSK		(0x1 << FIS_TYPE_SDB_OFF)
443  
444  /* ITCT header */
445  /* qw0 */
446  #define ITCT_HDR_DEV_TYPE_OFF		0
447  #define ITCT_HDR_DEV_TYPE_MSK		(0x3 << ITCT_HDR_DEV_TYPE_OFF)
448  #define ITCT_HDR_VALID_OFF		2
449  #define ITCT_HDR_VALID_MSK		(0x1 << ITCT_HDR_VALID_OFF)
450  #define ITCT_HDR_MCR_OFF		5
451  #define ITCT_HDR_MCR_MSK		(0xf << ITCT_HDR_MCR_OFF)
452  #define ITCT_HDR_VLN_OFF		9
453  #define ITCT_HDR_VLN_MSK		(0xf << ITCT_HDR_VLN_OFF)
454  #define ITCT_HDR_SMP_TIMEOUT_OFF	16
455  #define ITCT_HDR_AWT_CONTINUE_OFF	25
456  #define ITCT_HDR_PORT_ID_OFF		28
457  #define ITCT_HDR_PORT_ID_MSK		(0xf << ITCT_HDR_PORT_ID_OFF)
458  /* qw2 */
459  #define ITCT_HDR_INLT_OFF		0
460  #define ITCT_HDR_INLT_MSK		(0xffffULL << ITCT_HDR_INLT_OFF)
461  #define ITCT_HDR_RTOLT_OFF		48
462  #define ITCT_HDR_RTOLT_MSK		(0xffffULL << ITCT_HDR_RTOLT_OFF)
463  
464  struct hisi_sas_protect_iu_v3_hw {
465  	u32 dw0;
466  	u32 lbrtcv;
467  	u32 lbrtgv;
468  	u32 dw3;
469  	u32 dw4;
470  	u32 dw5;
471  	u32 rsv;
472  };
473  
474  struct hisi_sas_complete_v3_hdr {
475  	__le32 dw0;
476  	__le32 dw1;
477  	__le32 act;
478  	__le32 dw3;
479  };
480  
481  struct hisi_sas_err_record_v3 {
482  	/* dw0 */
483  	__le32 trans_tx_fail_type;
484  
485  	/* dw1 */
486  	__le32 trans_rx_fail_type;
487  
488  	/* dw2 */
489  	__le16 dma_tx_err_type;
490  	__le16 sipc_rx_err_type;
491  
492  	/* dw3 */
493  	__le32 dma_rx_err_type;
494  };
495  
496  #define RX_DATA_LEN_UNDERFLOW_OFF	6
497  #define RX_DATA_LEN_UNDERFLOW_MSK	(1 << RX_DATA_LEN_UNDERFLOW_OFF)
498  
499  #define RX_FIS_STATUS_ERR_OFF		0
500  #define RX_FIS_STATUS_ERR_MSK		(1 << RX_FIS_STATUS_ERR_OFF)
501  
502  #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
503  #define HISI_SAS_MSI_COUNT_V3_HW 32
504  
505  #define DIR_NO_DATA 0
506  #define DIR_TO_INI 1
507  #define DIR_TO_DEVICE 2
508  #define DIR_RESERVED 3
509  
510  #define FIS_CMD_IS_UNCONSTRAINED(fis) \
511  	((fis.command == ATA_CMD_READ_LOG_EXT) || \
512  	(fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
513  	((fis.command == ATA_CMD_DEV_RESET) && \
514  	((fis.control & ATA_SRST) != 0)))
515  
516  #define T10_INSRT_EN_OFF    0
517  #define T10_INSRT_EN_MSK    (1 << T10_INSRT_EN_OFF)
518  #define T10_RMV_EN_OFF	    1
519  #define T10_RMV_EN_MSK	    (1 << T10_RMV_EN_OFF)
520  #define T10_RPLC_EN_OFF	    2
521  #define T10_RPLC_EN_MSK	    (1 << T10_RPLC_EN_OFF)
522  #define T10_CHK_EN_OFF	    3
523  #define T10_CHK_EN_MSK	    (1 << T10_CHK_EN_OFF)
524  #define INCR_LBRT_OFF	    5
525  #define INCR_LBRT_MSK	    (1 << INCR_LBRT_OFF)
526  #define USR_DATA_BLOCK_SZ_OFF	20
527  #define USR_DATA_BLOCK_SZ_MSK	(0x3 << USR_DATA_BLOCK_SZ_OFF)
528  #define T10_CHK_MSK_OFF	    16
529  #define T10_CHK_REF_TAG_MSK (0xf0 << T10_CHK_MSK_OFF)
530  #define T10_CHK_APP_TAG_MSK (0xc << T10_CHK_MSK_OFF)
531  
532  #define BASE_VECTORS_V3_HW  16
533  #define MIN_AFFINE_VECTORS_V3_HW  (BASE_VECTORS_V3_HW + 1)
534  
535  #define CHNL_INT_STS_MSK	0xeeeeeeee
536  #define CHNL_INT_STS_PHY_MSK	0xe
537  #define CHNL_INT_STS_INT0_MSK BIT(1)
538  #define CHNL_INT_STS_INT1_MSK BIT(2)
539  #define CHNL_INT_STS_INT2_MSK BIT(3)
540  #define CHNL_WIDTH 4
541  
542  #define BAR_NO_V3_HW	5
543  
544  enum {
545  	DSM_FUNC_ERR_HANDLE_MSI = 0,
546  };
547  
548  static bool hisi_sas_intr_conv;
549  MODULE_PARM_DESC(intr_conv, "interrupt converge enable (0-1)");
550  
551  /* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
552  static int prot_mask;
553  module_param(prot_mask, int, 0444);
554  MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=0x0 ");
555  
556  /* the index of iopoll queues are bigger than interrupt queues' */
557  static int experimental_iopoll_q_cnt;
558  module_param(experimental_iopoll_q_cnt, int, 0444);
559  MODULE_PARM_DESC(experimental_iopoll_q_cnt, "number of queues to be used as poll mode, def=0");
560  
561  static int debugfs_snapshot_regs_v3_hw(struct hisi_hba *hisi_hba);
562  
hisi_sas_read32(struct hisi_hba * hisi_hba,u32 off)563  static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
564  {
565  	void __iomem *regs = hisi_hba->regs + off;
566  
567  	return readl(regs);
568  }
569  
hisi_sas_write32(struct hisi_hba * hisi_hba,u32 off,u32 val)570  static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
571  {
572  	void __iomem *regs = hisi_hba->regs + off;
573  
574  	writel(val, regs);
575  }
576  
hisi_sas_phy_write32(struct hisi_hba * hisi_hba,int phy_no,u32 off,u32 val)577  static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
578  				 u32 off, u32 val)
579  {
580  	void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
581  
582  	writel(val, regs);
583  }
584  
hisi_sas_phy_read32(struct hisi_hba * hisi_hba,int phy_no,u32 off)585  static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
586  				      int phy_no, u32 off)
587  {
588  	void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
589  
590  	return readl(regs);
591  }
592  
593  #define hisi_sas_read32_poll_timeout(off, val, cond, delay_us,		\
594  				     timeout_us)			\
595  ({									\
596  	void __iomem *regs = hisi_hba->regs + off;			\
597  	readl_poll_timeout(regs, val, cond, delay_us, timeout_us);	\
598  })
599  
600  #define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us,	\
601  					    timeout_us)			\
602  ({									\
603  	void __iomem *regs = hisi_hba->regs + off;			\
604  	readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
605  })
606  
interrupt_enable_v3_hw(struct hisi_hba * hisi_hba)607  static void interrupt_enable_v3_hw(struct hisi_hba *hisi_hba)
608  {
609  	int i;
610  
611  	for (i = 0; i < hisi_hba->queue_count; i++)
612  		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0);
613  
614  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
615  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
616  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffc220ff);
617  	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x155555);
618  
619  	for (i = 0; i < hisi_hba->n_phy; i++) {
620  		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xf2057fff);
621  		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
622  		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
623  		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
624  		hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
625  	}
626  }
627  
init_reg_v3_hw(struct hisi_hba * hisi_hba)628  static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
629  {
630  	struct pci_dev *pdev = hisi_hba->pci_dev;
631  	int i, j;
632  
633  	/* Global registers init */
634  	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
635  			 (u32)((1ULL << hisi_hba->queue_count) - 1));
636  	hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400);
637  	/* time / CLK_AHB = 2.5s / 2ns = 0x4A817C80 */
638  	hisi_sas_write32(hisi_hba, TRANS_LOCK_ICT_TIME, 0x4A817C80);
639  	hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
640  	hisi_sas_write32(hisi_hba, CFG_AGING_TIME, 0x1);
641  	hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
642  	hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
643  	hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
644  	hisi_sas_write32(hisi_hba, CQ_INT_CONVERGE_EN,
645  			 hisi_sas_intr_conv);
646  	hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff);
647  	hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
648  	hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
649  	hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
650  	hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
651  	hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
652  	hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
653  	hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
654  	hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
655  	hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
656  
657  	if (pdev->revision < 0x30)
658  		hisi_sas_write32(hisi_hba, SAS_AXI_USER3, 0);
659  
660  	interrupt_enable_v3_hw(hisi_hba);
661  	for (i = 0; i < hisi_hba->n_phy; i++) {
662  		enum sas_linkrate max;
663  		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
664  		struct asd_sas_phy *sas_phy = &phy->sas_phy;
665  		u32 prog_phy_link_rate = hisi_sas_phy_read32(hisi_hba, i,
666  							   PROG_PHY_LINK_RATE);
667  
668  		prog_phy_link_rate &= ~CFG_PROG_PHY_LINK_RATE_MSK;
669  		if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
670  				SAS_LINK_RATE_1_5_GBPS))
671  			max = SAS_LINK_RATE_12_0_GBPS;
672  		else
673  			max = sas_phy->phy->maximum_linkrate;
674  		prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
675  		hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
676  			prog_phy_link_rate);
677  		hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
678  		hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
679  		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
680  		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
681  		hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
682  		hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
683  		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
684  		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
685  		hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
686  		hisi_sas_phy_write32(hisi_hba, i, CON_CFG_DRIVER, 0x2a0a01);
687  		hisi_sas_phy_write32(hisi_hba, i, SAS_EC_INT_COAL_TIME,
688  				     0x30f4240);
689  		hisi_sas_phy_write32(hisi_hba, i, AIP_LIMIT, 0x2ffff);
690  
691  		/* set value through firmware for 920B and later version */
692  		if (pdev->revision < 0x30) {
693  			hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG, 0x32);
694  			hisi_sas_phy_write32(hisi_hba, i, SERDES_CFG, 0xffc00);
695  			/* used for 12G negotiate */
696  			hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e);
697  		}
698  
699  		/* get default FFE configuration for BIST */
700  		for (j = 0; j < FFE_CFG_MAX; j++) {
701  			u32 val = hisi_sas_phy_read32(hisi_hba, i,
702  						      TXDEEMPH_G1 + (j * 0x4));
703  			hisi_hba->debugfs_bist_ffe[i][j] = val;
704  		}
705  	}
706  
707  	for (i = 0; i < hisi_hba->queue_count; i++) {
708  		/* Delivery queue */
709  		hisi_sas_write32(hisi_hba,
710  				 DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
711  				 upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
712  
713  		hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
714  				 lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
715  
716  		hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
717  				 HISI_SAS_QUEUE_SLOTS);
718  
719  		/* Completion queue */
720  		hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
721  				 upper_32_bits(hisi_hba->complete_hdr_dma[i]));
722  
723  		hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
724  				 lower_32_bits(hisi_hba->complete_hdr_dma[i]));
725  
726  		hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
727  				 HISI_SAS_QUEUE_SLOTS);
728  	}
729  
730  	/* itct */
731  	hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
732  			 lower_32_bits(hisi_hba->itct_dma));
733  
734  	hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
735  			 upper_32_bits(hisi_hba->itct_dma));
736  
737  	/* iost */
738  	hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
739  			 lower_32_bits(hisi_hba->iost_dma));
740  
741  	hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
742  			 upper_32_bits(hisi_hba->iost_dma));
743  
744  	/* breakpoint */
745  	hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
746  			 lower_32_bits(hisi_hba->breakpoint_dma));
747  
748  	hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
749  			 upper_32_bits(hisi_hba->breakpoint_dma));
750  
751  	/* SATA broken msg */
752  	hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
753  			 lower_32_bits(hisi_hba->sata_breakpoint_dma));
754  
755  	hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
756  			 upper_32_bits(hisi_hba->sata_breakpoint_dma));
757  
758  	/* SATA initial fis */
759  	hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
760  			 lower_32_bits(hisi_hba->initial_fis_dma));
761  
762  	hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
763  			 upper_32_bits(hisi_hba->initial_fis_dma));
764  
765  	/* RAS registers init */
766  	hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
767  	hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
768  	hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
769  	hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
770  
771  	/* LED registers init */
772  	hisi_sas_write32(hisi_hba, SAS_CFG_DRIVE_VLD, 0x80000ff);
773  	hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1, 0x80808080);
774  	hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1 + 0x4, 0x80808080);
775  	/* Configure blink generator rate A to 1Hz and B to 4Hz */
776  	hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_1, 0x121700);
777  	hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_0, 0x800000);
778  }
779  
config_phy_opt_mode_v3_hw(struct hisi_hba * hisi_hba,int phy_no)780  static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
781  {
782  	u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
783  
784  	cfg &= ~PHY_CFG_DC_OPT_MSK;
785  	cfg |= 1 << PHY_CFG_DC_OPT_OFF;
786  	hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
787  }
788  
config_id_frame_v3_hw(struct hisi_hba * hisi_hba,int phy_no)789  static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
790  {
791  	struct sas_identify_frame identify_frame;
792  	u32 *identify_buffer;
793  
794  	memset(&identify_frame, 0, sizeof(identify_frame));
795  	identify_frame.dev_type = SAS_END_DEVICE;
796  	identify_frame.frame_type = 0;
797  	identify_frame._un1 = 1;
798  	identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
799  	identify_frame.target_bits = SAS_PROTOCOL_NONE;
800  	memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
801  	memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr,	SAS_ADDR_SIZE);
802  	identify_frame.phy_id = phy_no;
803  	identify_buffer = (u32 *)(&identify_frame);
804  
805  	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
806  			__swab32(identify_buffer[0]));
807  	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
808  			__swab32(identify_buffer[1]));
809  	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
810  			__swab32(identify_buffer[2]));
811  	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
812  			__swab32(identify_buffer[3]));
813  	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
814  			__swab32(identify_buffer[4]));
815  	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
816  			__swab32(identify_buffer[5]));
817  }
818  
setup_itct_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_device * sas_dev)819  static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
820  			     struct hisi_sas_device *sas_dev)
821  {
822  	struct domain_device *device = sas_dev->sas_device;
823  	struct device *dev = hisi_hba->dev;
824  	u64 qw0, device_id = sas_dev->device_id;
825  	struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
826  	struct domain_device *parent_dev = device->parent;
827  	struct asd_sas_port *sas_port = device->port;
828  	struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
829  	u64 sas_addr;
830  
831  	memset(itct, 0, sizeof(*itct));
832  
833  	/* qw0 */
834  	qw0 = 0;
835  	switch (sas_dev->dev_type) {
836  	case SAS_END_DEVICE:
837  	case SAS_EDGE_EXPANDER_DEVICE:
838  	case SAS_FANOUT_EXPANDER_DEVICE:
839  		qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
840  		break;
841  	case SAS_SATA_DEV:
842  	case SAS_SATA_PENDING:
843  		if (parent_dev && dev_is_expander(parent_dev->dev_type))
844  			qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
845  		else
846  			qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
847  		break;
848  	default:
849  		dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
850  			 sas_dev->dev_type);
851  	}
852  
853  	qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
854  		(device->linkrate << ITCT_HDR_MCR_OFF) |
855  		(1 << ITCT_HDR_VLN_OFF) |
856  		(0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
857  		(1 << ITCT_HDR_AWT_CONTINUE_OFF) |
858  		(port->id << ITCT_HDR_PORT_ID_OFF));
859  	itct->qw0 = cpu_to_le64(qw0);
860  
861  	/* qw1 */
862  	memcpy(&sas_addr, device->sas_addr, SAS_ADDR_SIZE);
863  	itct->sas_addr = cpu_to_le64(__swab64(sas_addr));
864  
865  	/* qw2 */
866  	if (!dev_is_sata(device))
867  		itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
868  					(0x1ULL << ITCT_HDR_RTOLT_OFF));
869  }
870  
clear_itct_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_device * sas_dev)871  static int clear_itct_v3_hw(struct hisi_hba *hisi_hba,
872  			    struct hisi_sas_device *sas_dev)
873  {
874  	DECLARE_COMPLETION_ONSTACK(completion);
875  	u64 dev_id = sas_dev->device_id;
876  	struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
877  	u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
878  	struct device *dev = hisi_hba->dev;
879  
880  	sas_dev->completion = &completion;
881  
882  	/* clear the itct interrupt state */
883  	if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
884  		hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
885  				 ENT_INT_SRC3_ITC_INT_MSK);
886  
887  	/* clear the itct table */
888  	reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
889  	hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
890  
891  	if (!wait_for_completion_timeout(sas_dev->completion,
892  					 HISI_SAS_CLEAR_ITCT_TIMEOUT)) {
893  		dev_warn(dev, "failed to clear ITCT\n");
894  		return -ETIMEDOUT;
895  	}
896  
897  	memset(itct, 0, sizeof(struct hisi_sas_itct));
898  	return 0;
899  }
900  
dereg_device_v3_hw(struct hisi_hba * hisi_hba,struct domain_device * device)901  static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
902  				struct domain_device *device)
903  {
904  	struct hisi_sas_slot *slot, *slot2;
905  	struct hisi_sas_device *sas_dev = device->lldd_dev;
906  	u32 cfg_abt_set_query_iptt;
907  
908  	cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
909  		CFG_ABT_SET_QUERY_IPTT);
910  	spin_lock(&sas_dev->lock);
911  	list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
912  		cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
913  		cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
914  			(slot->idx << CFG_SET_ABORTED_IPTT_OFF);
915  		hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
916  			cfg_abt_set_query_iptt);
917  	}
918  	spin_unlock(&sas_dev->lock);
919  	cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
920  	hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
921  		cfg_abt_set_query_iptt);
922  	hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
923  					1 << CFG_ABT_SET_IPTT_DONE_OFF);
924  }
925  
reset_hw_v3_hw(struct hisi_hba * hisi_hba)926  static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
927  {
928  	struct device *dev = hisi_hba->dev;
929  	int ret;
930  	u32 val;
931  
932  	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
933  
934  	/* Disable all of the PHYs */
935  	hisi_sas_stop_phys(hisi_hba);
936  	udelay(50);
937  
938  	/* Ensure axi bus idle */
939  	ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
940  					   20000, 1000000);
941  	if (ret) {
942  		dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
943  		return -EIO;
944  	}
945  
946  	if (ACPI_HANDLE(dev)) {
947  		acpi_status s;
948  
949  		s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
950  		if (ACPI_FAILURE(s)) {
951  			dev_err(dev, "Reset failed\n");
952  			return -EIO;
953  		}
954  	} else {
955  		dev_err(dev, "no reset method!\n");
956  		return -EINVAL;
957  	}
958  
959  	return 0;
960  }
961  
hw_init_v3_hw(struct hisi_hba * hisi_hba)962  static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
963  {
964  	struct device *dev = hisi_hba->dev;
965  	struct acpi_device *acpi_dev;
966  	union acpi_object *obj;
967  	guid_t guid;
968  	int rc;
969  
970  	rc = reset_hw_v3_hw(hisi_hba);
971  	if (rc) {
972  		dev_err(dev, "hisi_sas_reset_hw failed, rc=%d\n", rc);
973  		return rc;
974  	}
975  
976  	msleep(100);
977  	init_reg_v3_hw(hisi_hba);
978  
979  	if (guid_parse("D5918B4B-37AE-4E10-A99F-E5E8A6EF4C1F", &guid)) {
980  		dev_err(dev, "Parse GUID failed\n");
981  		return -EINVAL;
982  	}
983  
984  	/*
985  	 * This DSM handles some hardware-related configurations:
986  	 * 1. Switch over to MSI error handling in kernel
987  	 * 2. BIOS *may* reset some register values through this method
988  	 */
989  	obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid, 0,
990  				DSM_FUNC_ERR_HANDLE_MSI, NULL);
991  	if (!obj)
992  		dev_warn(dev, "can not find DSM method, ignore\n");
993  	else
994  		ACPI_FREE(obj);
995  
996  	acpi_dev = ACPI_COMPANION(dev);
997  	if (!acpi_device_power_manageable(acpi_dev))
998  		dev_notice(dev, "neither _PS0 nor _PR0 is defined\n");
999  	return 0;
1000  }
1001  
enable_phy_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1002  static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1003  {
1004  	u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
1005  
1006  	cfg |= PHY_CFG_ENA_MSK;
1007  	cfg &= ~PHY_CFG_PHY_RST_MSK;
1008  	hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
1009  }
1010  
disable_phy_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1011  static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1012  {
1013  	u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
1014  	u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
1015  	static const u32 msk = BIT(CHL_INT2_RX_DISP_ERR_OFF) |
1016  			       BIT(CHL_INT2_RX_CODE_ERR_OFF) |
1017  			       BIT(CHL_INT2_RX_INVLD_DW_OFF);
1018  	u32 state;
1019  
1020  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2_MSK, msk | irq_msk);
1021  
1022  	cfg &= ~PHY_CFG_ENA_MSK;
1023  	hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
1024  
1025  	mdelay(50);
1026  
1027  	state = hisi_sas_read32(hisi_hba, PHY_STATE);
1028  	if (state & BIT(phy_no)) {
1029  		cfg |= PHY_CFG_PHY_RST_MSK;
1030  		hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
1031  	}
1032  
1033  	udelay(1);
1034  
1035  	hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1036  	hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1037  	hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_CODE_ERR);
1038  
1039  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, msk);
1040  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2_MSK, irq_msk);
1041  }
1042  
start_phy_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1043  static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1044  {
1045  	config_id_frame_v3_hw(hisi_hba, phy_no);
1046  	config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
1047  	enable_phy_v3_hw(hisi_hba, phy_no);
1048  }
1049  
phy_hard_reset_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1050  static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1051  {
1052  	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1053  	u32 txid_auto;
1054  
1055  	hisi_sas_phy_enable(hisi_hba, phy_no, 0);
1056  	if (phy->identify.device_type == SAS_END_DEVICE) {
1057  		txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1058  		hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1059  					txid_auto | TX_HARDRST_MSK);
1060  	}
1061  	msleep(100);
1062  	hisi_sas_phy_enable(hisi_hba, phy_no, 1);
1063  }
1064  
phy_get_max_linkrate_v3_hw(void)1065  static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
1066  {
1067  	return SAS_LINK_RATE_12_0_GBPS;
1068  }
1069  
phys_init_v3_hw(struct hisi_hba * hisi_hba)1070  static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
1071  {
1072  	int i;
1073  
1074  	for (i = 0; i < hisi_hba->n_phy; i++) {
1075  		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
1076  		struct asd_sas_phy *sas_phy = &phy->sas_phy;
1077  
1078  		if (!sas_phy->phy->enabled)
1079  			continue;
1080  
1081  		hisi_sas_phy_enable(hisi_hba, i, 1);
1082  	}
1083  }
1084  
sl_notify_ssp_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1085  static void sl_notify_ssp_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1086  {
1087  	u32 sl_control;
1088  
1089  	sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1090  	sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
1091  	hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
1092  	msleep(1);
1093  	sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1094  	sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
1095  	hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
1096  }
1097  
get_wideport_bitmap_v3_hw(struct hisi_hba * hisi_hba,int port_id)1098  static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
1099  {
1100  	int i, bitmap = 0;
1101  	u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1102  	u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1103  
1104  	for (i = 0; i < hisi_hba->n_phy; i++)
1105  		if (phy_state & BIT(i))
1106  			if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
1107  				bitmap |= BIT(i);
1108  
1109  	return bitmap;
1110  }
1111  
start_delivery_v3_hw(struct hisi_sas_dq * dq)1112  static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
1113  {
1114  	struct hisi_hba *hisi_hba = dq->hisi_hba;
1115  	struct hisi_sas_slot *s, *s1, *s2 = NULL;
1116  	int dlvry_queue = dq->id;
1117  	int wp;
1118  
1119  	list_for_each_entry_safe(s, s1, &dq->list, delivery) {
1120  		if (!s->ready)
1121  			break;
1122  		s2 = s;
1123  		list_del(&s->delivery);
1124  	}
1125  
1126  	if (!s2)
1127  		return;
1128  
1129  	/*
1130  	 * Ensure that memories for slots built on other CPUs is observed.
1131  	 */
1132  	smp_rmb();
1133  	wp = (s2->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
1134  
1135  	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
1136  }
1137  
prep_prd_sge_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_slot * slot,struct hisi_sas_cmd_hdr * hdr,struct scatterlist * scatter,int n_elem)1138  static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
1139  			      struct hisi_sas_slot *slot,
1140  			      struct hisi_sas_cmd_hdr *hdr,
1141  			      struct scatterlist *scatter,
1142  			      int n_elem)
1143  {
1144  	struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
1145  	struct scatterlist *sg;
1146  	int i;
1147  
1148  	for_each_sg(scatter, sg, n_elem, i) {
1149  		struct hisi_sas_sge *entry = &sge_page->sge[i];
1150  
1151  		entry->addr = cpu_to_le64(sg_dma_address(sg));
1152  		entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
1153  		entry->data_len = cpu_to_le32(sg_dma_len(sg));
1154  		entry->data_off = 0;
1155  	}
1156  
1157  	hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
1158  
1159  	hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
1160  }
1161  
prep_prd_sge_dif_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_slot * slot,struct hisi_sas_cmd_hdr * hdr,struct scatterlist * scatter,int n_elem)1162  static void prep_prd_sge_dif_v3_hw(struct hisi_hba *hisi_hba,
1163  				   struct hisi_sas_slot *slot,
1164  				   struct hisi_sas_cmd_hdr *hdr,
1165  				   struct scatterlist *scatter,
1166  				   int n_elem)
1167  {
1168  	struct hisi_sas_sge_dif_page *sge_dif_page;
1169  	struct scatterlist *sg;
1170  	int i;
1171  
1172  	sge_dif_page = hisi_sas_sge_dif_addr_mem(slot);
1173  
1174  	for_each_sg(scatter, sg, n_elem, i) {
1175  		struct hisi_sas_sge *entry = &sge_dif_page->sge[i];
1176  
1177  		entry->addr = cpu_to_le64(sg_dma_address(sg));
1178  		entry->page_ctrl_0 = 0;
1179  		entry->page_ctrl_1 = 0;
1180  		entry->data_len = cpu_to_le32(sg_dma_len(sg));
1181  		entry->data_off = 0;
1182  	}
1183  
1184  	hdr->dif_prd_table_addr =
1185  		cpu_to_le64(hisi_sas_sge_dif_addr_dma(slot));
1186  
1187  	hdr->sg_len |= cpu_to_le32(n_elem << CMD_HDR_DIF_SGL_LEN_OFF);
1188  }
1189  
get_prot_chk_msk_v3_hw(struct scsi_cmnd * scsi_cmnd)1190  static u32 get_prot_chk_msk_v3_hw(struct scsi_cmnd *scsi_cmnd)
1191  {
1192  	unsigned char prot_flags = scsi_cmnd->prot_flags;
1193  
1194  	if (prot_flags & SCSI_PROT_REF_CHECK)
1195  		return T10_CHK_APP_TAG_MSK;
1196  	return T10_CHK_REF_TAG_MSK | T10_CHK_APP_TAG_MSK;
1197  }
1198  
fill_prot_v3_hw(struct scsi_cmnd * scsi_cmnd,struct hisi_sas_protect_iu_v3_hw * prot)1199  static void fill_prot_v3_hw(struct scsi_cmnd *scsi_cmnd,
1200  			    struct hisi_sas_protect_iu_v3_hw *prot)
1201  {
1202  	unsigned char prot_op = scsi_get_prot_op(scsi_cmnd);
1203  	unsigned int interval = scsi_prot_interval(scsi_cmnd);
1204  	u32 lbrt_chk_val = t10_pi_ref_tag(scsi_cmd_to_rq(scsi_cmnd));
1205  
1206  	switch (prot_op) {
1207  	case SCSI_PROT_READ_INSERT:
1208  		prot->dw0 |= T10_INSRT_EN_MSK;
1209  		prot->lbrtgv = lbrt_chk_val;
1210  		break;
1211  	case SCSI_PROT_READ_STRIP:
1212  		prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK);
1213  		prot->lbrtcv = lbrt_chk_val;
1214  		prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1215  		break;
1216  	case SCSI_PROT_READ_PASS:
1217  		prot->dw0 |= T10_CHK_EN_MSK;
1218  		prot->lbrtcv = lbrt_chk_val;
1219  		prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1220  		break;
1221  	case SCSI_PROT_WRITE_INSERT:
1222  		prot->dw0 |= T10_INSRT_EN_MSK;
1223  		prot->lbrtgv = lbrt_chk_val;
1224  		break;
1225  	case SCSI_PROT_WRITE_STRIP:
1226  		prot->dw0 |= (T10_RMV_EN_MSK | T10_CHK_EN_MSK);
1227  		prot->lbrtcv = lbrt_chk_val;
1228  		break;
1229  	case SCSI_PROT_WRITE_PASS:
1230  		prot->dw0 |= T10_CHK_EN_MSK;
1231  		prot->lbrtcv = lbrt_chk_val;
1232  		prot->dw4 |= get_prot_chk_msk_v3_hw(scsi_cmnd);
1233  		break;
1234  	default:
1235  		WARN(1, "prot_op(0x%x) is not valid\n", prot_op);
1236  		break;
1237  	}
1238  
1239  	switch (interval) {
1240  	case 512:
1241  		break;
1242  	case 4096:
1243  		prot->dw0 |= (0x1 << USR_DATA_BLOCK_SZ_OFF);
1244  		break;
1245  	case 520:
1246  		prot->dw0 |= (0x2 << USR_DATA_BLOCK_SZ_OFF);
1247  		break;
1248  	default:
1249  		WARN(1, "protection interval (0x%x) invalid\n",
1250  		     interval);
1251  		break;
1252  	}
1253  
1254  	prot->dw0 |= INCR_LBRT_MSK;
1255  }
1256  
prep_ssp_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_slot * slot)1257  static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
1258  			  struct hisi_sas_slot *slot)
1259  {
1260  	struct sas_task *task = slot->task;
1261  	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1262  	struct domain_device *device = task->dev;
1263  	struct hisi_sas_device *sas_dev = device->lldd_dev;
1264  	struct hisi_sas_port *port = slot->port;
1265  	struct sas_ssp_task *ssp_task = &task->ssp_task;
1266  	struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
1267  	struct sas_tmf_task *tmf = slot->tmf;
1268  	int has_data = 0, priority = !!tmf;
1269  	unsigned char prot_op;
1270  	u8 *buf_cmd;
1271  	u32 dw1 = 0, dw2 = 0, len = 0;
1272  
1273  	hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
1274  			       (2 << CMD_HDR_TLR_CTRL_OFF) |
1275  			       (port->id << CMD_HDR_PORT_OFF) |
1276  			       (priority << CMD_HDR_PRIORITY_OFF) |
1277  			       (1 << CMD_HDR_CMD_OFF)); /* ssp */
1278  
1279  	dw1 = 1 << CMD_HDR_VDTL_OFF;
1280  	if (tmf) {
1281  		dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
1282  		dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
1283  	} else {
1284  		prot_op = scsi_get_prot_op(scsi_cmnd);
1285  		dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
1286  		switch (scsi_cmnd->sc_data_direction) {
1287  		case DMA_TO_DEVICE:
1288  			has_data = 1;
1289  			dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1290  			break;
1291  		case DMA_FROM_DEVICE:
1292  			has_data = 1;
1293  			dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1294  			break;
1295  		default:
1296  			dw1 &= ~CMD_HDR_DIR_MSK;
1297  		}
1298  	}
1299  
1300  	/* map itct entry */
1301  	dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1302  
1303  	dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
1304  	      + 3) / 4) << CMD_HDR_CFL_OFF) |
1305  	      ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
1306  	      (2 << CMD_HDR_SG_MOD_OFF);
1307  	hdr->dw2 = cpu_to_le32(dw2);
1308  	hdr->transfer_tags = cpu_to_le32(slot->idx);
1309  
1310  	if (has_data) {
1311  		prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1312  				   slot->n_elem);
1313  
1314  		if (scsi_prot_sg_count(scsi_cmnd))
1315  			prep_prd_sge_dif_v3_hw(hisi_hba, slot, hdr,
1316  					       scsi_prot_sglist(scsi_cmnd),
1317  					       slot->n_elem_dif);
1318  	}
1319  
1320  	hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1321  	hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1322  
1323  	buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
1324  		sizeof(struct ssp_frame_hdr);
1325  
1326  	memcpy(buf_cmd, &task->ssp_task.LUN, 8);
1327  	if (!tmf) {
1328  		buf_cmd[9] = ssp_task->task_attr;
1329  		memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
1330  	} else {
1331  		buf_cmd[10] = tmf->tmf;
1332  		switch (tmf->tmf) {
1333  		case TMF_ABORT_TASK:
1334  		case TMF_QUERY_TASK:
1335  			buf_cmd[12] =
1336  				(tmf->tag_of_task_to_be_managed >> 8) & 0xff;
1337  			buf_cmd[13] =
1338  				tmf->tag_of_task_to_be_managed & 0xff;
1339  			break;
1340  		default:
1341  			break;
1342  		}
1343  	}
1344  
1345  	if (has_data && (prot_op != SCSI_PROT_NORMAL)) {
1346  		struct hisi_sas_protect_iu_v3_hw prot;
1347  		u8 *buf_cmd_prot;
1348  
1349  		hdr->dw7 |= cpu_to_le32(1 << CMD_HDR_ADDR_MODE_SEL_OFF);
1350  		dw1 |= CMD_HDR_PIR_MSK;
1351  		buf_cmd_prot = hisi_sas_cmd_hdr_addr_mem(slot) +
1352  			       sizeof(struct ssp_frame_hdr) +
1353  			       sizeof(struct ssp_command_iu);
1354  
1355  		memset(&prot, 0, sizeof(struct hisi_sas_protect_iu_v3_hw));
1356  		fill_prot_v3_hw(scsi_cmnd, &prot);
1357  		memcpy(buf_cmd_prot, &prot,
1358  		       sizeof(struct hisi_sas_protect_iu_v3_hw));
1359  		/*
1360  		 * For READ, we need length of info read to memory, while for
1361  		 * WRITE we need length of data written to the disk.
1362  		 */
1363  		if (prot_op == SCSI_PROT_WRITE_INSERT ||
1364  		    prot_op == SCSI_PROT_READ_INSERT ||
1365  		    prot_op == SCSI_PROT_WRITE_PASS ||
1366  		    prot_op == SCSI_PROT_READ_PASS) {
1367  			unsigned int interval = scsi_prot_interval(scsi_cmnd);
1368  			unsigned int ilog2_interval = ilog2(interval);
1369  
1370  			len = (task->total_xfer_len >> ilog2_interval) * 8;
1371  		}
1372  	}
1373  
1374  	hdr->dw1 = cpu_to_le32(dw1);
1375  
1376  	hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len + len);
1377  }
1378  
prep_smp_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_slot * slot)1379  static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
1380  			  struct hisi_sas_slot *slot)
1381  {
1382  	struct sas_task *task = slot->task;
1383  	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1384  	struct domain_device *device = task->dev;
1385  	struct hisi_sas_port *port = slot->port;
1386  	struct scatterlist *sg_req;
1387  	struct hisi_sas_device *sas_dev = device->lldd_dev;
1388  	dma_addr_t req_dma_addr;
1389  	unsigned int req_len;
1390  
1391  	/* req */
1392  	sg_req = &task->smp_task.smp_req;
1393  	req_len = sg_dma_len(sg_req);
1394  	req_dma_addr = sg_dma_address(sg_req);
1395  
1396  	/* create header */
1397  	/* dw0 */
1398  	hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
1399  			       (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
1400  			       (2 << CMD_HDR_CMD_OFF)); /* smp */
1401  
1402  	/* map itct entry */
1403  	hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
1404  			       (1 << CMD_HDR_FRAME_TYPE_OFF) |
1405  			       (DIR_NO_DATA << CMD_HDR_DIR_OFF));
1406  
1407  	/* dw2 */
1408  	hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1409  			       (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1410  			       CMD_HDR_MRFL_OFF));
1411  
1412  	hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1413  
1414  	hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1415  	hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1416  }
1417  
prep_ata_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_slot * slot)1418  static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1419  			  struct hisi_sas_slot *slot)
1420  {
1421  	struct sas_task *task = slot->task;
1422  	struct domain_device *device = task->dev;
1423  	struct domain_device *parent_dev = device->parent;
1424  	struct hisi_sas_device *sas_dev = device->lldd_dev;
1425  	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1426  	struct asd_sas_port *sas_port = device->port;
1427  	struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1428  	u8 *buf_cmd;
1429  	int has_data = 0, hdr_tag = 0;
1430  	u32 dw1 = 0, dw2 = 0;
1431  
1432  	hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1433  	if (parent_dev && dev_is_expander(parent_dev->dev_type))
1434  		hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1435  	else
1436  		hdr->dw0 |= cpu_to_le32(4U << CMD_HDR_CMD_OFF);
1437  
1438  	switch (task->data_dir) {
1439  	case DMA_TO_DEVICE:
1440  		has_data = 1;
1441  		dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1442  		break;
1443  	case DMA_FROM_DEVICE:
1444  		has_data = 1;
1445  		dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1446  		break;
1447  	default:
1448  		dw1 &= ~CMD_HDR_DIR_MSK;
1449  	}
1450  
1451  	if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1452  			(task->ata_task.fis.control & ATA_SRST))
1453  		dw1 |= 1 << CMD_HDR_RESET_OFF;
1454  
1455  	dw1 |= (hisi_sas_get_ata_protocol(
1456  		&task->ata_task.fis, task->data_dir))
1457  		<< CMD_HDR_FRAME_TYPE_OFF;
1458  	dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1459  
1460  	if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
1461  		dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1462  
1463  	hdr->dw1 = cpu_to_le32(dw1);
1464  
1465  	/* dw2 */
1466  	if (task->ata_task.use_ncq) {
1467  		struct ata_queued_cmd *qc = task->uldd_task;
1468  
1469  		hdr_tag = qc->tag;
1470  		task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1471  		dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1472  	}
1473  
1474  	dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1475  			2 << CMD_HDR_SG_MOD_OFF;
1476  	hdr->dw2 = cpu_to_le32(dw2);
1477  
1478  	/* dw3 */
1479  	hdr->transfer_tags = cpu_to_le32(slot->idx);
1480  
1481  	if (has_data)
1482  		prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1483  					slot->n_elem);
1484  
1485  	hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1486  	hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1487  	hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1488  
1489  	buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1490  
1491  	if (likely(!task->ata_task.device_control_reg_update))
1492  		task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1493  	/* fill in command FIS */
1494  	memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1495  }
1496  
prep_abort_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_slot * slot)1497  static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1498  			     struct hisi_sas_slot *slot)
1499  {
1500  	struct sas_task *task = slot->task;
1501  	struct sas_internal_abort_task *abort = &task->abort_task;
1502  	struct domain_device *dev = task->dev;
1503  	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1504  	struct hisi_sas_port *port = slot->port;
1505  	struct hisi_sas_device *sas_dev = dev->lldd_dev;
1506  	bool sata = dev_is_sata(dev);
1507  
1508  	/* dw0 */
1509  	hdr->dw0 = cpu_to_le32((5U << CMD_HDR_CMD_OFF) | /* abort */
1510  			       (port->id << CMD_HDR_PORT_OFF) |
1511  				(sata << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1512  				(abort->type << CMD_HDR_ABORT_FLAG_OFF));
1513  
1514  	/* dw1 */
1515  	hdr->dw1 = cpu_to_le32(sas_dev->device_id
1516  			<< CMD_HDR_DEV_ID_OFF);
1517  
1518  	/* dw7 */
1519  	hdr->dw7 = cpu_to_le32(abort->tag << CMD_HDR_ABORT_IPTT_OFF);
1520  	hdr->transfer_tags = cpu_to_le32(slot->idx);
1521  }
1522  
phy_up_v3_hw(int phy_no,struct hisi_hba * hisi_hba)1523  static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1524  {
1525  	int i;
1526  	irqreturn_t res;
1527  	u32 context, port_id, link_rate;
1528  	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1529  	struct asd_sas_phy *sas_phy = &phy->sas_phy;
1530  	struct device *dev = hisi_hba->dev;
1531  
1532  	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
1533  
1534  	port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1535  	port_id = (port_id >> (4 * phy_no)) & 0xf;
1536  	link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1537  	link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1538  
1539  	if (port_id == 0xf) {
1540  		dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1541  		res = IRQ_NONE;
1542  		goto end;
1543  	}
1544  	sas_phy->linkrate = link_rate;
1545  	phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1546  
1547  	/* Check for SATA dev */
1548  	context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1549  	if (context & (1 << phy_no)) {
1550  		struct hisi_sas_initial_fis *initial_fis;
1551  		struct dev_to_host_fis *fis;
1552  		u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1553  		struct Scsi_Host *shost = hisi_hba->shost;
1554  
1555  		dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1556  		initial_fis = &hisi_hba->initial_fis[phy_no];
1557  		fis = &initial_fis->fis;
1558  
1559  		/* check ERR bit of Status Register */
1560  		if (fis->status & ATA_ERR) {
1561  			dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n",
1562  				 phy_no, fis->status);
1563  			hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1564  			res = IRQ_NONE;
1565  			goto end;
1566  		}
1567  
1568  		sas_phy->oob_mode = SATA_OOB_MODE;
1569  		attached_sas_addr[0] = 0x50;
1570  		attached_sas_addr[6] = shost->host_no;
1571  		attached_sas_addr[7] = phy_no;
1572  		memcpy(sas_phy->attached_sas_addr,
1573  		       attached_sas_addr,
1574  		       SAS_ADDR_SIZE);
1575  		memcpy(sas_phy->frame_rcvd, fis,
1576  		       sizeof(struct dev_to_host_fis));
1577  		phy->phy_type |= PORT_TYPE_SATA;
1578  		phy->identify.device_type = SAS_SATA_DEV;
1579  		phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1580  		phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1581  	} else {
1582  		u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1583  		struct sas_identify_frame *id =
1584  			(struct sas_identify_frame *)frame_rcvd;
1585  
1586  		dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1587  		for (i = 0; i < 6; i++) {
1588  			u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1589  					       RX_IDAF_DWORD0 + (i * 4));
1590  			frame_rcvd[i] = __swab32(idaf);
1591  		}
1592  		sas_phy->oob_mode = SAS_OOB_MODE;
1593  		memcpy(sas_phy->attached_sas_addr,
1594  		       &id->sas_addr,
1595  		       SAS_ADDR_SIZE);
1596  		phy->phy_type |= PORT_TYPE_SAS;
1597  		phy->identify.device_type = id->dev_type;
1598  		phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1599  		if (phy->identify.device_type == SAS_END_DEVICE)
1600  			phy->identify.target_port_protocols =
1601  				SAS_PROTOCOL_SSP;
1602  		else if (phy->identify.device_type != SAS_PHY_UNUSED)
1603  			phy->identify.target_port_protocols =
1604  				SAS_PROTOCOL_SMP;
1605  	}
1606  
1607  	phy->port_id = port_id;
1608  	spin_lock(&phy->lock);
1609  	/* Delete timer and set phy_attached atomically */
1610  	del_timer(&phy->timer);
1611  	phy->phy_attached = 1;
1612  	spin_unlock(&phy->lock);
1613  
1614  	/*
1615  	 * Call pm_runtime_get_noresume() which pairs with
1616  	 * hisi_sas_phyup_pm_work() -> pm_runtime_put_sync().
1617  	 * For failure call pm_runtime_put() as we are in a hardirq context.
1618  	 */
1619  	pm_runtime_get_noresume(dev);
1620  	res = hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP_PM);
1621  	if (!res)
1622  		pm_runtime_put(dev);
1623  
1624  	res = IRQ_HANDLED;
1625  
1626  end:
1627  	if (phy->reset_completion)
1628  		complete(phy->reset_completion);
1629  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1630  			     CHL_INT0_SL_PHY_ENABLE_MSK);
1631  	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0);
1632  
1633  	return res;
1634  }
1635  
phy_down_v3_hw(int phy_no,struct hisi_hba * hisi_hba)1636  static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1637  {
1638  	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1639  	u32 phy_state, sl_ctrl, txid_auto;
1640  	struct device *dev = hisi_hba->dev;
1641  
1642  	atomic_inc(&phy->down_cnt);
1643  
1644  	del_timer(&phy->timer);
1645  	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
1646  
1647  	phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1648  	dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1649  	hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0,
1650  			  GFP_ATOMIC);
1651  
1652  	sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1653  	hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1654  						sl_ctrl&(~SL_CTA_MSK));
1655  
1656  	txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1657  	hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1658  						txid_auto | CT3_MSK);
1659  
1660  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1661  	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
1662  
1663  	return IRQ_HANDLED;
1664  }
1665  
phy_bcast_v3_hw(int phy_no,struct hisi_hba * hisi_hba)1666  static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1667  {
1668  	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1669  	u32 bcast_status;
1670  
1671  	hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
1672  	bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS);
1673  	if (bcast_status & RX_BCAST_CHG_MSK)
1674  		hisi_sas_phy_bcast(phy);
1675  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1676  			     CHL_INT0_SL_RX_BCST_ACK_MSK);
1677  	hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
1678  
1679  	return IRQ_HANDLED;
1680  }
1681  
int_phy_up_down_bcast_v3_hw(int irq_no,void * p)1682  static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1683  {
1684  	struct hisi_hba *hisi_hba = p;
1685  	u32 irq_msk;
1686  	int phy_no = 0;
1687  	irqreturn_t res = IRQ_NONE;
1688  
1689  	irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1690  				& 0x11111111;
1691  	while (irq_msk) {
1692  		if (irq_msk  & 1) {
1693  			u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1694  							    CHL_INT0);
1695  			u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1696  			int rdy = phy_state & (1 << phy_no);
1697  
1698  			if (rdy) {
1699  				if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1700  					/* phy up */
1701  					if (phy_up_v3_hw(phy_no, hisi_hba)
1702  							== IRQ_HANDLED)
1703  						res = IRQ_HANDLED;
1704  				if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1705  					/* phy bcast */
1706  					if (phy_bcast_v3_hw(phy_no, hisi_hba)
1707  							== IRQ_HANDLED)
1708  						res = IRQ_HANDLED;
1709  			} else {
1710  				if (irq_value & CHL_INT0_NOT_RDY_MSK)
1711  					/* phy down */
1712  					if (phy_down_v3_hw(phy_no, hisi_hba)
1713  							== IRQ_HANDLED)
1714  						res = IRQ_HANDLED;
1715  			}
1716  		}
1717  		irq_msk >>= 4;
1718  		phy_no++;
1719  	}
1720  
1721  	return res;
1722  }
1723  
1724  static const struct hisi_sas_hw_error port_axi_error[] = {
1725  	{
1726  		.irq_msk = BIT(CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF),
1727  		.msg = "dmac_tx_ecc_bad_err",
1728  	},
1729  	{
1730  		.irq_msk = BIT(CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF),
1731  		.msg = "dmac_rx_ecc_bad_err",
1732  	},
1733  	{
1734  		.irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1735  		.msg = "dma_tx_axi_wr_err",
1736  	},
1737  	{
1738  		.irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1739  		.msg = "dma_tx_axi_rd_err",
1740  	},
1741  	{
1742  		.irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1743  		.msg = "dma_rx_axi_wr_err",
1744  	},
1745  	{
1746  		.irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1747  		.msg = "dma_rx_axi_rd_err",
1748  	},
1749  	{
1750  		.irq_msk = BIT(CHL_INT1_DMAC_TX_FIFO_ERR_OFF),
1751  		.msg = "dma_tx_fifo_err",
1752  	},
1753  	{
1754  		.irq_msk = BIT(CHL_INT1_DMAC_RX_FIFO_ERR_OFF),
1755  		.msg = "dma_rx_fifo_err",
1756  	},
1757  	{
1758  		.irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF),
1759  		.msg = "dma_tx_axi_ruser_err",
1760  	},
1761  	{
1762  		.irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF),
1763  		.msg = "dma_rx_axi_ruser_err",
1764  	},
1765  };
1766  
handle_chl_int1_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1767  static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1768  {
1769  	u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1);
1770  	u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1_MSK);
1771  	struct device *dev = hisi_hba->dev;
1772  	int i;
1773  
1774  	irq_value &= ~irq_msk;
1775  	if (!irq_value) {
1776  		dev_warn(dev, "phy%d channel int 1 received with status bits cleared\n",
1777  			 phy_no);
1778  		return;
1779  	}
1780  
1781  	for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1782  		const struct hisi_sas_hw_error *error = &port_axi_error[i];
1783  
1784  		if (!(irq_value & error->irq_msk))
1785  			continue;
1786  
1787  		dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1788  			error->msg, phy_no, irq_value);
1789  		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1790  	}
1791  
1792  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT1, irq_value);
1793  }
1794  
phy_get_events_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1795  static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1796  {
1797  	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1798  	struct asd_sas_phy *sas_phy = &phy->sas_phy;
1799  	struct sas_phy *sphy = sas_phy->phy;
1800  	unsigned long flags;
1801  	u32 reg_value;
1802  
1803  	spin_lock_irqsave(&phy->lock, flags);
1804  
1805  	/* loss dword sync */
1806  	reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1807  	sphy->loss_of_dword_sync_count += reg_value;
1808  
1809  	/* phy reset problem */
1810  	reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1811  	sphy->phy_reset_problem_count += reg_value;
1812  
1813  	/* invalid dword */
1814  	reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1815  	sphy->invalid_dword_count += reg_value;
1816  
1817  	/* disparity err */
1818  	reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1819  	sphy->running_disparity_error_count += reg_value;
1820  
1821  	/* code violation error */
1822  	reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_CODE_ERR);
1823  	phy->code_violation_err_count += reg_value;
1824  
1825  	spin_unlock_irqrestore(&phy->lock, flags);
1826  }
1827  
handle_chl_int2_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1828  static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1829  {
1830  	u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
1831  	u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2);
1832  	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1833  	struct pci_dev *pci_dev = hisi_hba->pci_dev;
1834  	struct device *dev = hisi_hba->dev;
1835  	static const u32 msk = BIT(CHL_INT2_RX_DISP_ERR_OFF) |
1836  			BIT(CHL_INT2_RX_CODE_ERR_OFF) |
1837  			BIT(CHL_INT2_RX_INVLD_DW_OFF);
1838  
1839  	irq_value &= ~irq_msk;
1840  	if (!irq_value) {
1841  		dev_warn(dev, "phy%d channel int 2 received with status bits cleared\n",
1842  			 phy_no);
1843  		return;
1844  	}
1845  
1846  	if (irq_value & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1847  		dev_warn(dev, "phy%d identify timeout\n", phy_no);
1848  		hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1849  	}
1850  
1851  	if (irq_value & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1852  		u32 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1853  				STP_LINK_TIMEOUT_STATE);
1854  
1855  		dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1856  			 phy_no, reg_value);
1857  		if (reg_value & BIT(4))
1858  			hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1859  	}
1860  
1861  	if (pci_dev->revision > 0x20 && (irq_value & msk)) {
1862  		struct asd_sas_phy *sas_phy = &phy->sas_phy;
1863  		struct sas_phy *sphy = sas_phy->phy;
1864  
1865  		phy_get_events_v3_hw(hisi_hba, phy_no);
1866  
1867  		if (irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF))
1868  			dev_info(dev, "phy%d invalid dword cnt:   %u\n", phy_no,
1869  				 sphy->invalid_dword_count);
1870  
1871  		if (irq_value & BIT(CHL_INT2_RX_CODE_ERR_OFF))
1872  			dev_info(dev, "phy%d code violation cnt:  %u\n", phy_no,
1873  				 phy->code_violation_err_count);
1874  
1875  		if (irq_value & BIT(CHL_INT2_RX_DISP_ERR_OFF))
1876  			dev_info(dev, "phy%d disparity error cnt: %u\n", phy_no,
1877  				 sphy->running_disparity_error_count);
1878  	}
1879  
1880  	if ((irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
1881  	    (pci_dev->revision == 0x20)) {
1882  		u32 reg_value;
1883  		int rc;
1884  
1885  		rc = hisi_sas_read32_poll_timeout_atomic(
1886  				HILINK_ERR_DFX, reg_value,
1887  				!((reg_value >> 8) & BIT(phy_no)),
1888  				1000, 10000);
1889  		if (rc)
1890  			hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1891  	}
1892  
1893  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, irq_value);
1894  }
1895  
handle_chl_int0_v3_hw(struct hisi_hba * hisi_hba,int phy_no)1896  static void handle_chl_int0_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1897  {
1898  	u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT0);
1899  
1900  	if (irq_value0 & CHL_INT0_PHY_RDY_MSK)
1901  		hisi_sas_phy_oob_ready(hisi_hba, phy_no);
1902  
1903  	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1904  			     irq_value0 & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1905  			     & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1906  			     & (~CHL_INT0_NOT_RDY_MSK));
1907  }
1908  
int_chnl_int_v3_hw(int irq_no,void * p)1909  static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1910  {
1911  	struct hisi_hba *hisi_hba = p;
1912  	u32 irq_msk;
1913  	int phy_no = 0;
1914  
1915  	irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1916  		  & CHNL_INT_STS_MSK;
1917  
1918  	while (irq_msk) {
1919  		if (irq_msk & (CHNL_INT_STS_INT0_MSK << (phy_no * CHNL_WIDTH)))
1920  			handle_chl_int0_v3_hw(hisi_hba, phy_no);
1921  
1922  		if (irq_msk & (CHNL_INT_STS_INT1_MSK << (phy_no * CHNL_WIDTH)))
1923  			handle_chl_int1_v3_hw(hisi_hba, phy_no);
1924  
1925  		if (irq_msk & (CHNL_INT_STS_INT2_MSK << (phy_no * CHNL_WIDTH)))
1926  			handle_chl_int2_v3_hw(hisi_hba, phy_no);
1927  
1928  		irq_msk &= ~(CHNL_INT_STS_PHY_MSK << (phy_no * CHNL_WIDTH));
1929  		phy_no++;
1930  	}
1931  
1932  	return IRQ_HANDLED;
1933  }
1934  
1935  static const struct hisi_sas_hw_error multi_bit_ecc_errors[] = {
1936  	{
1937  		.irq_msk = BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF),
1938  		.msk = HGC_DQE_ECC_MB_ADDR_MSK,
1939  		.shift = HGC_DQE_ECC_MB_ADDR_OFF,
1940  		.msg = "hgc_dqe_eccbad_intr",
1941  		.reg = HGC_DQE_ECC_ADDR,
1942  	},
1943  	{
1944  		.irq_msk = BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF),
1945  		.msk = HGC_IOST_ECC_MB_ADDR_MSK,
1946  		.shift = HGC_IOST_ECC_MB_ADDR_OFF,
1947  		.msg = "hgc_iost_eccbad_intr",
1948  		.reg = HGC_IOST_ECC_ADDR,
1949  	},
1950  	{
1951  		.irq_msk = BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF),
1952  		.msk = HGC_ITCT_ECC_MB_ADDR_MSK,
1953  		.shift = HGC_ITCT_ECC_MB_ADDR_OFF,
1954  		.msg = "hgc_itct_eccbad_intr",
1955  		.reg = HGC_ITCT_ECC_ADDR,
1956  	},
1957  	{
1958  		.irq_msk = BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF),
1959  		.msk = HGC_LM_DFX_STATUS2_IOSTLIST_MSK,
1960  		.shift = HGC_LM_DFX_STATUS2_IOSTLIST_OFF,
1961  		.msg = "hgc_iostl_eccbad_intr",
1962  		.reg = HGC_LM_DFX_STATUS2,
1963  	},
1964  	{
1965  		.irq_msk = BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF),
1966  		.msk = HGC_LM_DFX_STATUS2_ITCTLIST_MSK,
1967  		.shift = HGC_LM_DFX_STATUS2_ITCTLIST_OFF,
1968  		.msg = "hgc_itctl_eccbad_intr",
1969  		.reg = HGC_LM_DFX_STATUS2,
1970  	},
1971  	{
1972  		.irq_msk = BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF),
1973  		.msk = HGC_CQE_ECC_MB_ADDR_MSK,
1974  		.shift = HGC_CQE_ECC_MB_ADDR_OFF,
1975  		.msg = "hgc_cqe_eccbad_intr",
1976  		.reg = HGC_CQE_ECC_ADDR,
1977  	},
1978  	{
1979  		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF),
1980  		.msk = HGC_RXM_DFX_STATUS14_MEM0_MSK,
1981  		.shift = HGC_RXM_DFX_STATUS14_MEM0_OFF,
1982  		.msg = "rxm_mem0_eccbad_intr",
1983  		.reg = HGC_RXM_DFX_STATUS14,
1984  	},
1985  	{
1986  		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF),
1987  		.msk = HGC_RXM_DFX_STATUS14_MEM1_MSK,
1988  		.shift = HGC_RXM_DFX_STATUS14_MEM1_OFF,
1989  		.msg = "rxm_mem1_eccbad_intr",
1990  		.reg = HGC_RXM_DFX_STATUS14,
1991  	},
1992  	{
1993  		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF),
1994  		.msk = HGC_RXM_DFX_STATUS14_MEM2_MSK,
1995  		.shift = HGC_RXM_DFX_STATUS14_MEM2_OFF,
1996  		.msg = "rxm_mem2_eccbad_intr",
1997  		.reg = HGC_RXM_DFX_STATUS14,
1998  	},
1999  	{
2000  		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF),
2001  		.msk = HGC_RXM_DFX_STATUS15_MEM3_MSK,
2002  		.shift = HGC_RXM_DFX_STATUS15_MEM3_OFF,
2003  		.msg = "rxm_mem3_eccbad_intr",
2004  		.reg = HGC_RXM_DFX_STATUS15,
2005  	},
2006  	{
2007  		.irq_msk = BIT(SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF),
2008  		.msk = AM_ROB_ECC_ERR_ADDR_MSK,
2009  		.shift = AM_ROB_ECC_ERR_ADDR_OFF,
2010  		.msg = "ooo_ram_eccbad_intr",
2011  		.reg = AM_ROB_ECC_ERR_ADDR,
2012  	},
2013  };
2014  
multi_bit_ecc_error_process_v3_hw(struct hisi_hba * hisi_hba,u32 irq_value)2015  static void multi_bit_ecc_error_process_v3_hw(struct hisi_hba *hisi_hba,
2016  					      u32 irq_value)
2017  {
2018  	struct device *dev = hisi_hba->dev;
2019  	const struct hisi_sas_hw_error *ecc_error;
2020  	u32 val;
2021  	int i;
2022  
2023  	for (i = 0; i < ARRAY_SIZE(multi_bit_ecc_errors); i++) {
2024  		ecc_error = &multi_bit_ecc_errors[i];
2025  		if (irq_value & ecc_error->irq_msk) {
2026  			val = hisi_sas_read32(hisi_hba, ecc_error->reg);
2027  			val &= ecc_error->msk;
2028  			val >>= ecc_error->shift;
2029  			dev_err(dev, "%s (0x%x) found: mem addr is 0x%08X\n",
2030  				ecc_error->msg, irq_value, val);
2031  			queue_work(hisi_hba->wq, &hisi_hba->rst_work);
2032  		}
2033  	}
2034  }
2035  
fatal_ecc_int_v3_hw(struct hisi_hba * hisi_hba)2036  static void fatal_ecc_int_v3_hw(struct hisi_hba *hisi_hba)
2037  {
2038  	u32 irq_value, irq_msk;
2039  
2040  	irq_msk = hisi_sas_read32(hisi_hba, SAS_ECC_INTR_MSK);
2041  	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
2042  
2043  	irq_value = hisi_sas_read32(hisi_hba, SAS_ECC_INTR);
2044  	if (irq_value)
2045  		multi_bit_ecc_error_process_v3_hw(hisi_hba, irq_value);
2046  
2047  	hisi_sas_write32(hisi_hba, SAS_ECC_INTR, irq_value);
2048  	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk);
2049  }
2050  
2051  static const struct hisi_sas_hw_error axi_error[] = {
2052  	{ .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
2053  	{ .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
2054  	{ .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
2055  	{ .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
2056  	{ .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
2057  	{ .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
2058  	{ .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
2059  	{ .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
2060  	{}
2061  };
2062  
2063  static const struct hisi_sas_hw_error fifo_error[] = {
2064  	{ .msk = BIT(8),  .msg = "CQE_WINFO_FIFO" },
2065  	{ .msk = BIT(9),  .msg = "CQE_MSG_FIFIO" },
2066  	{ .msk = BIT(10), .msg = "GETDQE_FIFO" },
2067  	{ .msk = BIT(11), .msg = "CMDP_FIFO" },
2068  	{ .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
2069  	{}
2070  };
2071  
2072  static const struct hisi_sas_hw_error fatal_axi_error[] = {
2073  	{
2074  		.irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
2075  		.msg = "write pointer and depth",
2076  	},
2077  	{
2078  		.irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
2079  		.msg = "iptt no match slot",
2080  	},
2081  	{
2082  		.irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
2083  		.msg = "read pointer and depth",
2084  	},
2085  	{
2086  		.irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
2087  		.reg = HGC_AXI_FIFO_ERR_INFO,
2088  		.sub = axi_error,
2089  	},
2090  	{
2091  		.irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
2092  		.reg = HGC_AXI_FIFO_ERR_INFO,
2093  		.sub = fifo_error,
2094  	},
2095  	{
2096  		.irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
2097  		.msg = "LM add/fetch list",
2098  	},
2099  	{
2100  		.irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
2101  		.msg = "SAS_HGC_ABT fetch LM list",
2102  	},
2103  	{
2104  		.irq_msk = BIT(ENT_INT_SRC3_DQE_POISON_OFF),
2105  		.msg = "read dqe poison",
2106  	},
2107  	{
2108  		.irq_msk = BIT(ENT_INT_SRC3_IOST_POISON_OFF),
2109  		.msg = "read iost poison",
2110  	},
2111  	{
2112  		.irq_msk = BIT(ENT_INT_SRC3_ITCT_POISON_OFF),
2113  		.msg = "read itct poison",
2114  	},
2115  	{
2116  		.irq_msk = BIT(ENT_INT_SRC3_ITCT_NCQ_POISON_OFF),
2117  		.msg = "read itct ncq poison",
2118  	},
2119  
2120  };
2121  
fatal_axi_int_v3_hw(int irq_no,void * p)2122  static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
2123  {
2124  	u32 irq_value, irq_msk;
2125  	struct hisi_hba *hisi_hba = p;
2126  	struct device *dev = hisi_hba->dev;
2127  	struct pci_dev *pdev = hisi_hba->pci_dev;
2128  	int i;
2129  
2130  	irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
2131  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
2132  
2133  	irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
2134  	irq_value &= ~irq_msk;
2135  
2136  	for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
2137  		const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
2138  
2139  		if (!(irq_value & error->irq_msk))
2140  			continue;
2141  
2142  		if (error->sub) {
2143  			const struct hisi_sas_hw_error *sub = error->sub;
2144  			u32 err_value = hisi_sas_read32(hisi_hba, error->reg);
2145  
2146  			for (; sub->msk || sub->msg; sub++) {
2147  				if (!(err_value & sub->msk))
2148  					continue;
2149  
2150  				dev_err(dev, "%s error (0x%x) found!\n",
2151  					sub->msg, irq_value);
2152  				queue_work(hisi_hba->wq, &hisi_hba->rst_work);
2153  			}
2154  		} else {
2155  			dev_err(dev, "%s error (0x%x) found!\n",
2156  				error->msg, irq_value);
2157  			queue_work(hisi_hba->wq, &hisi_hba->rst_work);
2158  		}
2159  
2160  		if (pdev->revision < 0x21) {
2161  			u32 reg_val;
2162  
2163  			reg_val = hisi_sas_read32(hisi_hba,
2164  						  AXI_MASTER_CFG_BASE +
2165  						  AM_CTRL_GLOBAL);
2166  			reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2167  			hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2168  					 AM_CTRL_GLOBAL, reg_val);
2169  		}
2170  	}
2171  
2172  	fatal_ecc_int_v3_hw(hisi_hba);
2173  
2174  	if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
2175  		u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
2176  		u32 dev_id = reg_val & ITCT_DEV_MSK;
2177  		struct hisi_sas_device *sas_dev =
2178  				&hisi_hba->devices[dev_id];
2179  
2180  		hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
2181  		dev_dbg(dev, "clear ITCT ok\n");
2182  		complete(sas_dev->completion);
2183  	}
2184  
2185  	hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value & 0x1df00);
2186  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
2187  
2188  	return IRQ_HANDLED;
2189  }
2190  
is_ncq_err_v3_hw(struct hisi_sas_complete_v3_hdr * complete_hdr)2191  static bool is_ncq_err_v3_hw(struct hisi_sas_complete_v3_hdr *complete_hdr)
2192  {
2193  	u32 dw0, dw3;
2194  
2195  	dw0 = le32_to_cpu(complete_hdr->dw0);
2196  	dw3 = le32_to_cpu(complete_hdr->dw3);
2197  
2198  	return (dw0 & ERR_PHASE_RESPONSE_FRAME_REV_STAGE_MSK) &&
2199  	       (dw3 & FIS_TYPE_SDB_MSK) &&
2200  	       (dw3 & FIS_ATA_STATUS_ERR_MSK);
2201  }
2202  
2203  static bool
slot_err_v3_hw(struct hisi_hba * hisi_hba,struct sas_task * task,struct hisi_sas_slot * slot)2204  slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
2205  	       struct hisi_sas_slot *slot)
2206  {
2207  	struct task_status_struct *ts = &task->task_status;
2208  	struct hisi_sas_complete_v3_hdr *complete_queue =
2209  			hisi_hba->complete_hdr[slot->cmplt_queue];
2210  	struct hisi_sas_complete_v3_hdr *complete_hdr =
2211  			&complete_queue[slot->cmplt_queue_slot];
2212  	struct hisi_sas_err_record_v3 *record =
2213  			hisi_sas_status_buf_addr_mem(slot);
2214  	u32 dma_rx_err_type = le32_to_cpu(record->dma_rx_err_type);
2215  	u32 trans_tx_fail_type = le32_to_cpu(record->trans_tx_fail_type);
2216  	u16 sipc_rx_err_type = le16_to_cpu(record->sipc_rx_err_type);
2217  	u32 dw3 = le32_to_cpu(complete_hdr->dw3);
2218  	u32 dw0 = le32_to_cpu(complete_hdr->dw0);
2219  
2220  	switch (task->task_proto) {
2221  	case SAS_PROTOCOL_SSP:
2222  		if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
2223  			/*
2224  			 * If returned response frame is incorrect because of data underflow,
2225  			 * but I/O information has been written to the host memory, we examine
2226  			 * response IU.
2227  			 */
2228  			if (!(dw0 & CMPLT_HDR_RSPNS_GOOD_MSK) &&
2229  			    (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK))
2230  				return false;
2231  
2232  			ts->residual = trans_tx_fail_type;
2233  			ts->stat = SAS_DATA_UNDERRUN;
2234  		} else if (dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
2235  			ts->stat = SAS_QUEUE_FULL;
2236  			slot->abort = 1;
2237  		} else {
2238  			ts->stat = SAS_OPEN_REJECT;
2239  			ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
2240  		}
2241  		break;
2242  	case SAS_PROTOCOL_SATA:
2243  	case SAS_PROTOCOL_STP:
2244  	case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
2245  		if ((dw0 & CMPLT_HDR_RSPNS_XFRD_MSK) &&
2246  		    (sipc_rx_err_type & RX_FIS_STATUS_ERR_MSK)) {
2247  			if (task->ata_task.use_ncq) {
2248  				struct domain_device *device = task->dev;
2249  				struct hisi_sas_device *sas_dev = device->lldd_dev;
2250  
2251  				sas_dev->dev_status = HISI_SAS_DEV_NCQ_ERR;
2252  				slot->abort = 1;
2253  			} else {
2254  				ts->stat = SAS_PROTO_RESPONSE;
2255  			}
2256  		} else if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
2257  			ts->residual = trans_tx_fail_type;
2258  			ts->stat = SAS_DATA_UNDERRUN;
2259  		} else if ((dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) ||
2260  			   (dw3 & SATA_DISK_IN_ERROR_STATUS_MSK)) {
2261  			ts->stat = SAS_PHY_DOWN;
2262  			slot->abort = 1;
2263  		} else {
2264  			ts->stat = SAS_OPEN_REJECT;
2265  			ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
2266  		}
2267  		if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK)
2268  			hisi_sas_sata_done(task, slot);
2269  		break;
2270  	case SAS_PROTOCOL_SMP:
2271  		ts->stat = SAS_SAM_STAT_CHECK_CONDITION;
2272  		break;
2273  	default:
2274  		break;
2275  	}
2276  	return true;
2277  }
2278  
slot_complete_v3_hw(struct hisi_hba * hisi_hba,struct hisi_sas_slot * slot)2279  static void slot_complete_v3_hw(struct hisi_hba *hisi_hba,
2280  				struct hisi_sas_slot *slot)
2281  {
2282  	struct sas_task *task = slot->task;
2283  	struct hisi_sas_device *sas_dev;
2284  	struct device *dev = hisi_hba->dev;
2285  	struct task_status_struct *ts;
2286  	struct domain_device *device;
2287  	struct sas_ha_struct *ha;
2288  	struct hisi_sas_complete_v3_hdr *complete_queue =
2289  			hisi_hba->complete_hdr[slot->cmplt_queue];
2290  	struct hisi_sas_complete_v3_hdr *complete_hdr =
2291  			&complete_queue[slot->cmplt_queue_slot];
2292  	unsigned long flags;
2293  	bool is_internal = slot->is_internal;
2294  	u32 dw0, dw1, dw3;
2295  
2296  	if (unlikely(!task || !task->lldd_task || !task->dev))
2297  		return;
2298  
2299  	ts = &task->task_status;
2300  	device = task->dev;
2301  	ha = device->port->ha;
2302  	sas_dev = device->lldd_dev;
2303  
2304  	spin_lock_irqsave(&task->task_state_lock, flags);
2305  	task->task_state_flags &= ~SAS_TASK_STATE_PENDING;
2306  	spin_unlock_irqrestore(&task->task_state_lock, flags);
2307  
2308  	memset(ts, 0, sizeof(*ts));
2309  	ts->resp = SAS_TASK_COMPLETE;
2310  
2311  	if (unlikely(!sas_dev)) {
2312  		dev_dbg(dev, "slot complete: port has not device\n");
2313  		ts->stat = SAS_PHY_DOWN;
2314  		goto out;
2315  	}
2316  
2317  	dw0 = le32_to_cpu(complete_hdr->dw0);
2318  	dw1 = le32_to_cpu(complete_hdr->dw1);
2319  	dw3 = le32_to_cpu(complete_hdr->dw3);
2320  
2321  	/*
2322  	 * Use SAS+TMF status codes
2323  	 */
2324  	switch ((dw0 & CMPLT_HDR_ABORT_STAT_MSK) >> CMPLT_HDR_ABORT_STAT_OFF) {
2325  	case STAT_IO_ABORTED:
2326  		/* this IO has been aborted by abort command */
2327  		ts->stat = SAS_ABORTED_TASK;
2328  		goto out;
2329  	case STAT_IO_COMPLETE:
2330  		/* internal abort command complete */
2331  		ts->stat = TMF_RESP_FUNC_SUCC;
2332  		goto out;
2333  	case STAT_IO_NO_DEVICE:
2334  		ts->stat = TMF_RESP_FUNC_COMPLETE;
2335  		goto out;
2336  	case STAT_IO_NOT_VALID:
2337  		/*
2338  		 * abort single IO, the controller can't find the IO
2339  		 */
2340  		ts->stat = TMF_RESP_FUNC_FAILED;
2341  		goto out;
2342  	default:
2343  		break;
2344  	}
2345  
2346  	/* check for erroneous completion */
2347  	if ((dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
2348  		u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
2349  
2350  		if (slot_err_v3_hw(hisi_hba, task, slot)) {
2351  			if (ts->stat != SAS_DATA_UNDERRUN)
2352  				dev_info(dev, "erroneous completion iptt=%d task=%pK dev id=%d addr=%016llx CQ hdr: 0x%x 0x%x 0x%x 0x%x Error info: 0x%x 0x%x 0x%x 0x%x\n",
2353  					slot->idx, task, sas_dev->device_id,
2354  					SAS_ADDR(device->sas_addr),
2355  					dw0, dw1, complete_hdr->act, dw3,
2356  					error_info[0], error_info[1],
2357  					error_info[2], error_info[3]);
2358  			if (unlikely(slot->abort)) {
2359  				if (dev_is_sata(device) && task->ata_task.use_ncq)
2360  					sas_ata_device_link_abort(device, true);
2361  				else
2362  					sas_task_abort(task);
2363  
2364  				return;
2365  			}
2366  			goto out;
2367  		}
2368  	}
2369  
2370  	switch (task->task_proto) {
2371  	case SAS_PROTOCOL_SSP: {
2372  		struct ssp_response_iu *iu =
2373  			hisi_sas_status_buf_addr_mem(slot) +
2374  			sizeof(struct hisi_sas_err_record);
2375  
2376  		sas_ssp_task_response(dev, task, iu);
2377  		break;
2378  	}
2379  	case SAS_PROTOCOL_SMP: {
2380  		struct scatterlist *sg_resp = &task->smp_task.smp_resp;
2381  		void *to = page_address(sg_page(sg_resp));
2382  
2383  		ts->stat = SAS_SAM_STAT_GOOD;
2384  
2385  		memcpy(to + sg_resp->offset,
2386  			hisi_sas_status_buf_addr_mem(slot) +
2387  		       sizeof(struct hisi_sas_err_record),
2388  		       sg_resp->length);
2389  		break;
2390  	}
2391  	case SAS_PROTOCOL_SATA:
2392  	case SAS_PROTOCOL_STP:
2393  	case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
2394  		ts->stat = SAS_SAM_STAT_GOOD;
2395  		if (dw0 & CMPLT_HDR_RSPNS_XFRD_MSK)
2396  			hisi_sas_sata_done(task, slot);
2397  		break;
2398  	default:
2399  		ts->stat = SAS_SAM_STAT_CHECK_CONDITION;
2400  		break;
2401  	}
2402  
2403  	if (!slot->port->port_attached) {
2404  		dev_warn(dev, "slot complete: port %d has removed\n",
2405  			slot->port->sas_port.id);
2406  		ts->stat = SAS_PHY_DOWN;
2407  	}
2408  
2409  out:
2410  	spin_lock_irqsave(&task->task_state_lock, flags);
2411  	if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
2412  		spin_unlock_irqrestore(&task->task_state_lock, flags);
2413  		dev_info(dev, "slot complete: task(%pK) aborted\n", task);
2414  		return;
2415  	}
2416  	task->task_state_flags |= SAS_TASK_STATE_DONE;
2417  	spin_unlock_irqrestore(&task->task_state_lock, flags);
2418  	hisi_sas_slot_task_free(hisi_hba, task, slot, true);
2419  
2420  	if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
2421  		spin_lock_irqsave(&device->done_lock, flags);
2422  		if (test_bit(SAS_HA_FROZEN, &ha->state)) {
2423  			spin_unlock_irqrestore(&device->done_lock, flags);
2424  			dev_info(dev, "slot complete: task(%pK) ignored\n ",
2425  				 task);
2426  			return;
2427  		}
2428  		spin_unlock_irqrestore(&device->done_lock, flags);
2429  	}
2430  
2431  	if (task->task_done)
2432  		task->task_done(task);
2433  }
2434  
complete_v3_hw(struct hisi_sas_cq * cq)2435  static int complete_v3_hw(struct hisi_sas_cq *cq)
2436  {
2437  	struct hisi_sas_complete_v3_hdr *complete_queue;
2438  	struct hisi_hba *hisi_hba = cq->hisi_hba;
2439  	u32 rd_point, wr_point;
2440  	int queue = cq->id;
2441  	int completed;
2442  
2443  	rd_point = cq->rd_point;
2444  	complete_queue = hisi_hba->complete_hdr[queue];
2445  
2446  	wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
2447  				   (0x14 * queue));
2448  	completed = (wr_point + HISI_SAS_QUEUE_SLOTS - rd_point) % HISI_SAS_QUEUE_SLOTS;
2449  
2450  	while (rd_point != wr_point) {
2451  		struct hisi_sas_complete_v3_hdr *complete_hdr;
2452  		struct device *dev = hisi_hba->dev;
2453  		struct hisi_sas_slot *slot;
2454  		u32 dw0, dw1, dw3;
2455  		int iptt;
2456  
2457  		complete_hdr = &complete_queue[rd_point];
2458  		dw0 = le32_to_cpu(complete_hdr->dw0);
2459  		dw1 = le32_to_cpu(complete_hdr->dw1);
2460  		dw3 = le32_to_cpu(complete_hdr->dw3);
2461  
2462  		iptt = dw1 & CMPLT_HDR_IPTT_MSK;
2463  		if (unlikely((dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) &&
2464  			     (dw3 & CMPLT_HDR_SATA_DISK_ERR_MSK)) {
2465  			int device_id = (dw1 & CMPLT_HDR_DEV_ID_MSK) >>
2466  					CMPLT_HDR_DEV_ID_OFF;
2467  			struct hisi_sas_itct *itct =
2468  				&hisi_hba->itct[device_id];
2469  			struct hisi_sas_device *sas_dev =
2470  				&hisi_hba->devices[device_id];
2471  			struct domain_device *device = sas_dev->sas_device;
2472  
2473  			dev_err(dev, "erroneous completion disk err dev id=%d sas_addr=0x%llx CQ hdr: 0x%x 0x%x 0x%x 0x%x\n",
2474  				device_id, itct->sas_addr, dw0, dw1,
2475  				complete_hdr->act, dw3);
2476  
2477  			if (is_ncq_err_v3_hw(complete_hdr))
2478  				sas_dev->dev_status = HISI_SAS_DEV_NCQ_ERR;
2479  
2480  			sas_ata_device_link_abort(device, true);
2481  		} else if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
2482  			slot = &hisi_hba->slot_info[iptt];
2483  			slot->cmplt_queue_slot = rd_point;
2484  			slot->cmplt_queue = queue;
2485  			slot_complete_v3_hw(hisi_hba, slot);
2486  		} else
2487  			dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
2488  
2489  		if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
2490  			rd_point = 0;
2491  	}
2492  
2493  	/* update rd_point */
2494  	cq->rd_point = rd_point;
2495  	hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
2496  	cond_resched();
2497  
2498  	return completed;
2499  }
2500  
queue_complete_v3_hw(struct Scsi_Host * shost,unsigned int queue)2501  static int queue_complete_v3_hw(struct Scsi_Host *shost, unsigned int queue)
2502  {
2503  	struct hisi_hba *hisi_hba = shost_priv(shost);
2504  	struct hisi_sas_cq *cq = &hisi_hba->cq[queue];
2505  	int completed;
2506  
2507  	spin_lock(&cq->poll_lock);
2508  	completed = complete_v3_hw(cq);
2509  	spin_unlock(&cq->poll_lock);
2510  
2511  	return completed;
2512  }
2513  
cq_thread_v3_hw(int irq_no,void * p)2514  static irqreturn_t cq_thread_v3_hw(int irq_no, void *p)
2515  {
2516  	struct hisi_sas_cq *cq = p;
2517  
2518  	complete_v3_hw(cq);
2519  
2520  	return IRQ_HANDLED;
2521  }
2522  
cq_interrupt_v3_hw(int irq_no,void * p)2523  static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
2524  {
2525  	struct hisi_sas_cq *cq = p;
2526  	struct hisi_hba *hisi_hba = cq->hisi_hba;
2527  	int queue = cq->id;
2528  
2529  	hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
2530  
2531  	return IRQ_WAKE_THREAD;
2532  }
2533  
hisi_sas_v3_free_vectors(void * data)2534  static void hisi_sas_v3_free_vectors(void *data)
2535  {
2536  	struct pci_dev *pdev = data;
2537  
2538  	pci_free_irq_vectors(pdev);
2539  }
2540  
interrupt_preinit_v3_hw(struct hisi_hba * hisi_hba)2541  static int interrupt_preinit_v3_hw(struct hisi_hba *hisi_hba)
2542  {
2543  	/* Allocate all MSI vectors to avoid re-insertion issue */
2544  	int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
2545  	int vectors, min_msi;
2546  	struct Scsi_Host *shost = hisi_hba->shost;
2547  	struct pci_dev *pdev = hisi_hba->pci_dev;
2548  	struct irq_affinity desc = {
2549  		.pre_vectors = BASE_VECTORS_V3_HW,
2550  	};
2551  
2552  	min_msi = MIN_AFFINE_VECTORS_V3_HW;
2553  	vectors = pci_alloc_irq_vectors_affinity(pdev,
2554  						 min_msi, max_msi,
2555  						 PCI_IRQ_MSI |
2556  						 PCI_IRQ_AFFINITY,
2557  						 &desc);
2558  	if (vectors < 0)
2559  		return -ENOENT;
2560  
2561  
2562  	hisi_hba->cq_nvecs = vectors - BASE_VECTORS_V3_HW - hisi_hba->iopoll_q_cnt;
2563  	shost->nr_hw_queues = hisi_hba->cq_nvecs + hisi_hba->iopoll_q_cnt;
2564  
2565  	return devm_add_action(&pdev->dev, hisi_sas_v3_free_vectors, pdev);
2566  }
2567  
interrupt_init_v3_hw(struct hisi_hba * hisi_hba)2568  static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
2569  {
2570  	struct device *dev = hisi_hba->dev;
2571  	struct pci_dev *pdev = hisi_hba->pci_dev;
2572  	int rc, i;
2573  
2574  	rc = devm_request_irq(dev, pci_irq_vector(pdev, 1),
2575  			      int_phy_up_down_bcast_v3_hw, 0,
2576  			      DRV_NAME " phy", hisi_hba);
2577  	if (rc) {
2578  		dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
2579  		return -ENOENT;
2580  	}
2581  
2582  	rc = devm_request_irq(dev, pci_irq_vector(pdev, 2),
2583  			      int_chnl_int_v3_hw, 0,
2584  			      DRV_NAME " channel", hisi_hba);
2585  	if (rc) {
2586  		dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
2587  		return -ENOENT;
2588  	}
2589  
2590  	rc = devm_request_irq(dev, pci_irq_vector(pdev, 11),
2591  			      fatal_axi_int_v3_hw, 0,
2592  			      DRV_NAME " fatal", hisi_hba);
2593  	if (rc) {
2594  		dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
2595  		return -ENOENT;
2596  	}
2597  
2598  	if (hisi_sas_intr_conv)
2599  		dev_info(dev, "Enable interrupt converge\n");
2600  
2601  	for (i = 0; i < hisi_hba->cq_nvecs; i++) {
2602  		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2603  		int nr = hisi_sas_intr_conv ? 16 : 16 + i;
2604  		unsigned long irqflags = hisi_sas_intr_conv ? IRQF_SHARED :
2605  							      IRQF_ONESHOT;
2606  
2607  		cq->irq_no = pci_irq_vector(pdev, nr);
2608  		rc = devm_request_threaded_irq(dev, cq->irq_no,
2609  				      cq_interrupt_v3_hw,
2610  				      cq_thread_v3_hw,
2611  				      irqflags,
2612  				      DRV_NAME " cq", cq);
2613  		if (rc) {
2614  			dev_err(dev, "could not request cq%d interrupt, rc=%d\n",
2615  				i, rc);
2616  			return -ENOENT;
2617  		}
2618  		cq->irq_mask = pci_irq_get_affinity(pdev, i + BASE_VECTORS_V3_HW);
2619  		if (!cq->irq_mask) {
2620  			dev_err(dev, "could not get cq%d irq affinity!\n", i);
2621  			return -ENOENT;
2622  		}
2623  	}
2624  
2625  	return 0;
2626  }
2627  
hisi_sas_v3_init(struct hisi_hba * hisi_hba)2628  static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
2629  {
2630  	int rc;
2631  
2632  	rc = hw_init_v3_hw(hisi_hba);
2633  	if (rc)
2634  		return rc;
2635  
2636  	rc = interrupt_init_v3_hw(hisi_hba);
2637  	if (rc)
2638  		return rc;
2639  
2640  	return 0;
2641  }
2642  
phy_set_linkrate_v3_hw(struct hisi_hba * hisi_hba,int phy_no,struct sas_phy_linkrates * r)2643  static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
2644  		struct sas_phy_linkrates *r)
2645  {
2646  	enum sas_linkrate max = r->maximum_linkrate;
2647  	u32 prog_phy_link_rate = hisi_sas_phy_read32(hisi_hba, phy_no,
2648  						     PROG_PHY_LINK_RATE);
2649  
2650  	prog_phy_link_rate &= ~CFG_PROG_PHY_LINK_RATE_MSK;
2651  	prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
2652  	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
2653  			     prog_phy_link_rate);
2654  }
2655  
interrupt_disable_v3_hw(struct hisi_hba * hisi_hba)2656  static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
2657  {
2658  	struct pci_dev *pdev = hisi_hba->pci_dev;
2659  	int i;
2660  
2661  	synchronize_irq(pci_irq_vector(pdev, 1));
2662  	synchronize_irq(pci_irq_vector(pdev, 2));
2663  	synchronize_irq(pci_irq_vector(pdev, 11));
2664  	for (i = 0; i < hisi_hba->queue_count; i++)
2665  		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
2666  
2667  	for (i = 0; i < hisi_hba->cq_nvecs; i++)
2668  		synchronize_irq(pci_irq_vector(pdev, i + 16));
2669  
2670  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
2671  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
2672  	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
2673  	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
2674  
2675  	for (i = 0; i < hisi_hba->n_phy; i++) {
2676  		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
2677  		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
2678  		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
2679  		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
2680  		hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
2681  	}
2682  }
2683  
get_phys_state_v3_hw(struct hisi_hba * hisi_hba)2684  static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
2685  {
2686  	return hisi_sas_read32(hisi_hba, PHY_STATE);
2687  }
2688  
disable_host_v3_hw(struct hisi_hba * hisi_hba)2689  static int disable_host_v3_hw(struct hisi_hba *hisi_hba)
2690  {
2691  	struct device *dev = hisi_hba->dev;
2692  	u32 status, reg_val;
2693  	int rc;
2694  
2695  	hisi_sas_sync_poll_cqs(hisi_hba);
2696  	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
2697  
2698  	hisi_sas_stop_phys(hisi_hba);
2699  
2700  	mdelay(10);
2701  
2702  	reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2703  				  AM_CTRL_GLOBAL);
2704  	reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2705  	hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2706  			 AM_CTRL_GLOBAL, reg_val);
2707  
2708  	/* wait until bus idle */
2709  	rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
2710  					  AM_CURR_TRANS_RETURN, status,
2711  					  status == 0x3, 10, 100);
2712  	if (rc) {
2713  		dev_err(dev, "axi bus is not idle, rc=%d\n", rc);
2714  		return rc;
2715  	}
2716  
2717  	return 0;
2718  }
2719  
soft_reset_v3_hw(struct hisi_hba * hisi_hba)2720  static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
2721  {
2722  	struct device *dev = hisi_hba->dev;
2723  	int rc;
2724  
2725  	interrupt_disable_v3_hw(hisi_hba);
2726  	rc = disable_host_v3_hw(hisi_hba);
2727  	if (rc) {
2728  		dev_err(dev, "soft reset: disable host failed rc=%d\n", rc);
2729  		return rc;
2730  	}
2731  
2732  	hisi_sas_init_mem(hisi_hba);
2733  
2734  	return hw_init_v3_hw(hisi_hba);
2735  }
2736  
write_gpio_v3_hw(struct hisi_hba * hisi_hba,u8 reg_type,u8 reg_index,u8 reg_count,u8 * write_data)2737  static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
2738  			u8 reg_index, u8 reg_count, u8 *write_data)
2739  {
2740  	struct device *dev = hisi_hba->dev;
2741  	u32 *data = (u32 *)write_data;
2742  	int i;
2743  
2744  	switch (reg_type) {
2745  	case SAS_GPIO_REG_TX:
2746  		if ((reg_index + reg_count) > ((hisi_hba->n_phy + 3) / 4)) {
2747  			dev_err(dev, "write gpio: invalid reg range[%d, %d]\n",
2748  				reg_index, reg_index + reg_count - 1);
2749  			return -EINVAL;
2750  		}
2751  
2752  		for (i = 0; i < reg_count; i++)
2753  			hisi_sas_write32(hisi_hba,
2754  					 SAS_GPIO_TX_0_1 + (reg_index + i) * 4,
2755  					 data[i]);
2756  		break;
2757  	default:
2758  		dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
2759  			reg_type);
2760  		return -EINVAL;
2761  	}
2762  
2763  	return 0;
2764  }
2765  
wait_cmds_complete_timeout_v3_hw(struct hisi_hba * hisi_hba,int delay_ms,int timeout_ms)2766  static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
2767  					     int delay_ms, int timeout_ms)
2768  {
2769  	struct device *dev = hisi_hba->dev;
2770  	int entries, entries_old = 0, time;
2771  
2772  	for (time = 0; time < timeout_ms; time += delay_ms) {
2773  		entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
2774  		if (entries == entries_old)
2775  			break;
2776  
2777  		entries_old = entries;
2778  		msleep(delay_ms);
2779  	}
2780  
2781  	if (time >= timeout_ms) {
2782  		dev_dbg(dev, "Wait commands complete timeout!\n");
2783  		return;
2784  	}
2785  
2786  	dev_dbg(dev, "wait commands complete %dms\n", time);
2787  }
2788  
intr_conv_v3_hw_show(struct device * dev,struct device_attribute * attr,char * buf)2789  static ssize_t intr_conv_v3_hw_show(struct device *dev,
2790  				    struct device_attribute *attr, char *buf)
2791  {
2792  	return scnprintf(buf, PAGE_SIZE, "%u\n", hisi_sas_intr_conv);
2793  }
2794  static DEVICE_ATTR_RO(intr_conv_v3_hw);
2795  
config_intr_coal_v3_hw(struct hisi_hba * hisi_hba)2796  static void config_intr_coal_v3_hw(struct hisi_hba *hisi_hba)
2797  {
2798  	/* config those registers between enable and disable PHYs */
2799  	hisi_sas_stop_phys(hisi_hba);
2800  
2801  	if (hisi_hba->intr_coal_ticks == 0 ||
2802  	    hisi_hba->intr_coal_count == 0) {
2803  		hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
2804  		hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
2805  		hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
2806  	} else {
2807  		hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x3);
2808  		hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME,
2809  				 hisi_hba->intr_coal_ticks);
2810  		hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT,
2811  				 hisi_hba->intr_coal_count);
2812  	}
2813  	phys_init_v3_hw(hisi_hba);
2814  }
2815  
intr_coal_ticks_v3_hw_show(struct device * dev,struct device_attribute * attr,char * buf)2816  static ssize_t intr_coal_ticks_v3_hw_show(struct device *dev,
2817  					  struct device_attribute *attr,
2818  					  char *buf)
2819  {
2820  	struct Scsi_Host *shost = class_to_shost(dev);
2821  	struct hisi_hba *hisi_hba = shost_priv(shost);
2822  
2823  	return scnprintf(buf, PAGE_SIZE, "%u\n",
2824  			 hisi_hba->intr_coal_ticks);
2825  }
2826  
intr_coal_ticks_v3_hw_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2827  static ssize_t intr_coal_ticks_v3_hw_store(struct device *dev,
2828  					   struct device_attribute *attr,
2829  					   const char *buf, size_t count)
2830  {
2831  	struct Scsi_Host *shost = class_to_shost(dev);
2832  	struct hisi_hba *hisi_hba = shost_priv(shost);
2833  	u32 intr_coal_ticks;
2834  	int ret;
2835  
2836  	ret = kstrtou32(buf, 10, &intr_coal_ticks);
2837  	if (ret) {
2838  		dev_err(dev, "Input data of interrupt coalesce unmatch\n");
2839  		return -EINVAL;
2840  	}
2841  
2842  	if (intr_coal_ticks >= BIT(24)) {
2843  		dev_err(dev, "intr_coal_ticks must be less than 2^24!\n");
2844  		return -EINVAL;
2845  	}
2846  
2847  	hisi_hba->intr_coal_ticks = intr_coal_ticks;
2848  
2849  	config_intr_coal_v3_hw(hisi_hba);
2850  
2851  	return count;
2852  }
2853  static DEVICE_ATTR_RW(intr_coal_ticks_v3_hw);
2854  
intr_coal_count_v3_hw_show(struct device * dev,struct device_attribute * attr,char * buf)2855  static ssize_t intr_coal_count_v3_hw_show(struct device *dev,
2856  					  struct device_attribute
2857  					  *attr, char *buf)
2858  {
2859  	struct Scsi_Host *shost = class_to_shost(dev);
2860  	struct hisi_hba *hisi_hba = shost_priv(shost);
2861  
2862  	return scnprintf(buf, PAGE_SIZE, "%u\n",
2863  			 hisi_hba->intr_coal_count);
2864  }
2865  
intr_coal_count_v3_hw_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)2866  static ssize_t intr_coal_count_v3_hw_store(struct device *dev,
2867  		struct device_attribute
2868  		*attr, const char *buf, size_t count)
2869  {
2870  	struct Scsi_Host *shost = class_to_shost(dev);
2871  	struct hisi_hba *hisi_hba = shost_priv(shost);
2872  	u32 intr_coal_count;
2873  	int ret;
2874  
2875  	ret = kstrtou32(buf, 10, &intr_coal_count);
2876  	if (ret) {
2877  		dev_err(dev, "Input data of interrupt coalesce unmatch\n");
2878  		return -EINVAL;
2879  	}
2880  
2881  	if (intr_coal_count >= BIT(8)) {
2882  		dev_err(dev, "intr_coal_count must be less than 2^8!\n");
2883  		return -EINVAL;
2884  	}
2885  
2886  	hisi_hba->intr_coal_count = intr_coal_count;
2887  
2888  	config_intr_coal_v3_hw(hisi_hba);
2889  
2890  	return count;
2891  }
2892  static DEVICE_ATTR_RW(intr_coal_count_v3_hw);
2893  
iopoll_q_cnt_v3_hw_show(struct device * dev,struct device_attribute * attr,char * buf)2894  static ssize_t iopoll_q_cnt_v3_hw_show(struct device *dev,
2895  					  struct device_attribute
2896  					  *attr, char *buf)
2897  {
2898  	struct Scsi_Host *shost = class_to_shost(dev);
2899  	struct hisi_hba *hisi_hba = shost_priv(shost);
2900  
2901  	return scnprintf(buf, PAGE_SIZE, "%u\n",
2902  			 hisi_hba->iopoll_q_cnt);
2903  }
2904  static DEVICE_ATTR_RO(iopoll_q_cnt_v3_hw);
2905  
slave_configure_v3_hw(struct scsi_device * sdev)2906  static int slave_configure_v3_hw(struct scsi_device *sdev)
2907  {
2908  	struct Scsi_Host *shost = dev_to_shost(&sdev->sdev_gendev);
2909  	struct hisi_hba *hisi_hba = shost_priv(shost);
2910  	int ret = hisi_sas_slave_configure(sdev);
2911  	struct device *dev = hisi_hba->dev;
2912  
2913  	if (ret)
2914  		return ret;
2915  
2916  	if (sdev->type == TYPE_ENCLOSURE)
2917  		return 0;
2918  
2919  	if (!device_link_add(&sdev->sdev_gendev, dev,
2920  			     DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)) {
2921  		if (pm_runtime_enabled(dev)) {
2922  			dev_info(dev, "add device link failed, disable runtime PM for the host\n");
2923  			pm_runtime_disable(dev);
2924  		}
2925  	}
2926  
2927  	return 0;
2928  }
2929  
2930  static struct attribute *host_v3_hw_attrs[] = {
2931  	&dev_attr_phy_event_threshold.attr,
2932  	&dev_attr_intr_conv_v3_hw.attr,
2933  	&dev_attr_intr_coal_ticks_v3_hw.attr,
2934  	&dev_attr_intr_coal_count_v3_hw.attr,
2935  	&dev_attr_iopoll_q_cnt_v3_hw.attr,
2936  	NULL
2937  };
2938  
2939  ATTRIBUTE_GROUPS(host_v3_hw);
2940  
2941  #define HISI_SAS_DEBUGFS_REG(x) {#x, x}
2942  
2943  struct hisi_sas_debugfs_reg_lu {
2944  	char *name;
2945  	int off;
2946  };
2947  
2948  struct hisi_sas_debugfs_reg {
2949  	const struct hisi_sas_debugfs_reg_lu *lu;
2950  	int count;
2951  	int base_off;
2952  };
2953  
2954  static const struct hisi_sas_debugfs_reg_lu debugfs_port_reg_lu[] = {
2955  	HISI_SAS_DEBUGFS_REG(PHY_CFG),
2956  	HISI_SAS_DEBUGFS_REG(HARD_PHY_LINKRATE),
2957  	HISI_SAS_DEBUGFS_REG(PROG_PHY_LINK_RATE),
2958  	HISI_SAS_DEBUGFS_REG(PHY_CTRL),
2959  	HISI_SAS_DEBUGFS_REG(SL_CFG),
2960  	HISI_SAS_DEBUGFS_REG(AIP_LIMIT),
2961  	HISI_SAS_DEBUGFS_REG(SL_CONTROL),
2962  	HISI_SAS_DEBUGFS_REG(RX_PRIMS_STATUS),
2963  	HISI_SAS_DEBUGFS_REG(TX_ID_DWORD0),
2964  	HISI_SAS_DEBUGFS_REG(TX_ID_DWORD1),
2965  	HISI_SAS_DEBUGFS_REG(TX_ID_DWORD2),
2966  	HISI_SAS_DEBUGFS_REG(TX_ID_DWORD3),
2967  	HISI_SAS_DEBUGFS_REG(TX_ID_DWORD4),
2968  	HISI_SAS_DEBUGFS_REG(TX_ID_DWORD5),
2969  	HISI_SAS_DEBUGFS_REG(TX_ID_DWORD6),
2970  	HISI_SAS_DEBUGFS_REG(TXID_AUTO),
2971  	HISI_SAS_DEBUGFS_REG(RX_IDAF_DWORD0),
2972  	HISI_SAS_DEBUGFS_REG(RXOP_CHECK_CFG_H),
2973  	HISI_SAS_DEBUGFS_REG(STP_LINK_TIMER),
2974  	HISI_SAS_DEBUGFS_REG(STP_LINK_TIMEOUT_STATE),
2975  	HISI_SAS_DEBUGFS_REG(CON_CFG_DRIVER),
2976  	HISI_SAS_DEBUGFS_REG(SAS_SSP_CON_TIMER_CFG),
2977  	HISI_SAS_DEBUGFS_REG(SAS_SMP_CON_TIMER_CFG),
2978  	HISI_SAS_DEBUGFS_REG(SAS_STP_CON_TIMER_CFG),
2979  	HISI_SAS_DEBUGFS_REG(CHL_INT0),
2980  	HISI_SAS_DEBUGFS_REG(CHL_INT1),
2981  	HISI_SAS_DEBUGFS_REG(CHL_INT2),
2982  	HISI_SAS_DEBUGFS_REG(CHL_INT0_MSK),
2983  	HISI_SAS_DEBUGFS_REG(CHL_INT1_MSK),
2984  	HISI_SAS_DEBUGFS_REG(CHL_INT2_MSK),
2985  	HISI_SAS_DEBUGFS_REG(SAS_EC_INT_COAL_TIME),
2986  	HISI_SAS_DEBUGFS_REG(CHL_INT_COAL_EN),
2987  	HISI_SAS_DEBUGFS_REG(SAS_RX_TRAIN_TIMER),
2988  	HISI_SAS_DEBUGFS_REG(PHY_CTRL_RDY_MSK),
2989  	HISI_SAS_DEBUGFS_REG(PHYCTRL_NOT_RDY_MSK),
2990  	HISI_SAS_DEBUGFS_REG(PHYCTRL_DWS_RESET_MSK),
2991  	HISI_SAS_DEBUGFS_REG(PHYCTRL_PHY_ENA_MSK),
2992  	HISI_SAS_DEBUGFS_REG(SL_RX_BCAST_CHK_MSK),
2993  	HISI_SAS_DEBUGFS_REG(PHYCTRL_OOB_RESTART_MSK),
2994  	HISI_SAS_DEBUGFS_REG(DMA_TX_STATUS),
2995  	HISI_SAS_DEBUGFS_REG(DMA_RX_STATUS),
2996  	HISI_SAS_DEBUGFS_REG(COARSETUNE_TIME),
2997  	HISI_SAS_DEBUGFS_REG(ERR_CNT_DWS_LOST),
2998  	HISI_SAS_DEBUGFS_REG(ERR_CNT_RESET_PROB),
2999  	HISI_SAS_DEBUGFS_REG(ERR_CNT_INVLD_DW),
3000  	HISI_SAS_DEBUGFS_REG(ERR_CNT_CODE_ERR),
3001  	HISI_SAS_DEBUGFS_REG(ERR_CNT_DISP_ERR),
3002  	{}
3003  };
3004  
3005  static const struct hisi_sas_debugfs_reg debugfs_port_reg = {
3006  	.lu = debugfs_port_reg_lu,
3007  	.count = 0x100,
3008  	.base_off = PORT_BASE,
3009  };
3010  
3011  static const struct hisi_sas_debugfs_reg_lu debugfs_global_reg_lu[] = {
3012  	HISI_SAS_DEBUGFS_REG(DLVRY_QUEUE_ENABLE),
3013  	HISI_SAS_DEBUGFS_REG(PHY_CONTEXT),
3014  	HISI_SAS_DEBUGFS_REG(PHY_STATE),
3015  	HISI_SAS_DEBUGFS_REG(PHY_PORT_NUM_MA),
3016  	HISI_SAS_DEBUGFS_REG(PHY_CONN_RATE),
3017  	HISI_SAS_DEBUGFS_REG(ITCT_CLR),
3018  	HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_LO),
3019  	HISI_SAS_DEBUGFS_REG(IO_SATA_BROKEN_MSG_ADDR_HI),
3020  	HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_LO),
3021  	HISI_SAS_DEBUGFS_REG(SATA_INITI_D2H_STORE_ADDR_HI),
3022  	HISI_SAS_DEBUGFS_REG(CFG_MAX_TAG),
3023  	HISI_SAS_DEBUGFS_REG(TRANS_LOCK_ICT_TIME),
3024  	HISI_SAS_DEBUGFS_REG(HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL),
3025  	HISI_SAS_DEBUGFS_REG(HGC_SAS_TXFAIL_RETRY_CTRL),
3026  	HISI_SAS_DEBUGFS_REG(HGC_GET_ITV_TIME),
3027  	HISI_SAS_DEBUGFS_REG(DEVICE_MSG_WORK_MODE),
3028  	HISI_SAS_DEBUGFS_REG(OPENA_WT_CONTI_TIME),
3029  	HISI_SAS_DEBUGFS_REG(I_T_NEXUS_LOSS_TIME),
3030  	HISI_SAS_DEBUGFS_REG(MAX_CON_TIME_LIMIT_TIME),
3031  	HISI_SAS_DEBUGFS_REG(BUS_INACTIVE_LIMIT_TIME),
3032  	HISI_SAS_DEBUGFS_REG(REJECT_TO_OPEN_LIMIT_TIME),
3033  	HISI_SAS_DEBUGFS_REG(CQ_INT_CONVERGE_EN),
3034  	HISI_SAS_DEBUGFS_REG(CFG_AGING_TIME),
3035  	HISI_SAS_DEBUGFS_REG(HGC_DFX_CFG2),
3036  	HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_QUERY_IPTT),
3037  	HISI_SAS_DEBUGFS_REG(CFG_ABT_SET_IPTT_DONE),
3038  	HISI_SAS_DEBUGFS_REG(HGC_IOMB_PROC1_STATUS),
3039  	HISI_SAS_DEBUGFS_REG(CHNL_INT_STATUS),
3040  	HISI_SAS_DEBUGFS_REG(HGC_AXI_FIFO_ERR_INFO),
3041  	HISI_SAS_DEBUGFS_REG(INT_COAL_EN),
3042  	HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_TIME),
3043  	HISI_SAS_DEBUGFS_REG(OQ_INT_COAL_CNT),
3044  	HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_TIME),
3045  	HISI_SAS_DEBUGFS_REG(ENT_INT_COAL_CNT),
3046  	HISI_SAS_DEBUGFS_REG(OQ_INT_SRC),
3047  	HISI_SAS_DEBUGFS_REG(OQ_INT_SRC_MSK),
3048  	HISI_SAS_DEBUGFS_REG(ENT_INT_SRC1),
3049  	HISI_SAS_DEBUGFS_REG(ENT_INT_SRC2),
3050  	HISI_SAS_DEBUGFS_REG(ENT_INT_SRC3),
3051  	HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK1),
3052  	HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK2),
3053  	HISI_SAS_DEBUGFS_REG(ENT_INT_SRC_MSK3),
3054  	HISI_SAS_DEBUGFS_REG(CHNL_PHYUPDOWN_INT_MSK),
3055  	HISI_SAS_DEBUGFS_REG(CHNL_ENT_INT_MSK),
3056  	HISI_SAS_DEBUGFS_REG(HGC_COM_INT_MSK),
3057  	HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR),
3058  	HISI_SAS_DEBUGFS_REG(SAS_ECC_INTR_MSK),
3059  	HISI_SAS_DEBUGFS_REG(HGC_ERR_STAT_EN),
3060  	HISI_SAS_DEBUGFS_REG(CQE_SEND_CNT),
3061  	HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_DEPTH),
3062  	HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_WR_PTR),
3063  	HISI_SAS_DEBUGFS_REG(DLVRY_Q_0_RD_PTR),
3064  	HISI_SAS_DEBUGFS_REG(HYPER_STREAM_ID_EN_CFG),
3065  	HISI_SAS_DEBUGFS_REG(OQ0_INT_SRC_MSK),
3066  	HISI_SAS_DEBUGFS_REG(COMPL_Q_0_DEPTH),
3067  	HISI_SAS_DEBUGFS_REG(COMPL_Q_0_WR_PTR),
3068  	HISI_SAS_DEBUGFS_REG(COMPL_Q_0_RD_PTR),
3069  	HISI_SAS_DEBUGFS_REG(AWQOS_AWCACHE_CFG),
3070  	HISI_SAS_DEBUGFS_REG(ARQOS_ARCACHE_CFG),
3071  	HISI_SAS_DEBUGFS_REG(HILINK_ERR_DFX),
3072  	HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_0),
3073  	HISI_SAS_DEBUGFS_REG(SAS_GPIO_CFG_1),
3074  	HISI_SAS_DEBUGFS_REG(SAS_GPIO_TX_0_1),
3075  	HISI_SAS_DEBUGFS_REG(SAS_CFG_DRIVE_VLD),
3076  	{}
3077  };
3078  
3079  static const struct hisi_sas_debugfs_reg debugfs_global_reg = {
3080  	.lu = debugfs_global_reg_lu,
3081  	.count = 0x800,
3082  };
3083  
3084  static const struct hisi_sas_debugfs_reg_lu debugfs_axi_reg_lu[] = {
3085  	HISI_SAS_DEBUGFS_REG(AM_CFG_MAX_TRANS),
3086  	HISI_SAS_DEBUGFS_REG(AM_CFG_SINGLE_PORT_MAX_TRANS),
3087  	HISI_SAS_DEBUGFS_REG(AXI_CFG),
3088  	HISI_SAS_DEBUGFS_REG(AM_ROB_ECC_ERR_ADDR),
3089  	{}
3090  };
3091  
3092  static const struct hisi_sas_debugfs_reg debugfs_axi_reg = {
3093  	.lu = debugfs_axi_reg_lu,
3094  	.count = 0x61,
3095  	.base_off = AXI_MASTER_CFG_BASE,
3096  };
3097  
3098  static const struct hisi_sas_debugfs_reg_lu debugfs_ras_reg_lu[] = {
3099  	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0),
3100  	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1),
3101  	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0_MASK),
3102  	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1_MASK),
3103  	HISI_SAS_DEBUGFS_REG(CFG_SAS_RAS_INTR_MASK),
3104  	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR2),
3105  	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR2_MASK),
3106  	{}
3107  };
3108  
3109  static const struct hisi_sas_debugfs_reg debugfs_ras_reg = {
3110  	.lu = debugfs_ras_reg_lu,
3111  	.count = 0x10,
3112  	.base_off = RAS_BASE,
3113  };
3114  
debugfs_snapshot_prepare_v3_hw(struct hisi_hba * hisi_hba)3115  static void debugfs_snapshot_prepare_v3_hw(struct hisi_hba *hisi_hba)
3116  {
3117  	struct Scsi_Host *shost = hisi_hba->shost;
3118  
3119  	scsi_block_requests(shost);
3120  	wait_cmds_complete_timeout_v3_hw(hisi_hba, 100, 5000);
3121  
3122  	set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
3123  	hisi_sas_sync_cqs(hisi_hba);
3124  	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
3125  }
3126  
debugfs_snapshot_restore_v3_hw(struct hisi_hba * hisi_hba)3127  static void debugfs_snapshot_restore_v3_hw(struct hisi_hba *hisi_hba)
3128  {
3129  	struct Scsi_Host *shost = hisi_hba->shost;
3130  
3131  	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
3132  			 (u32)((1ULL << hisi_hba->queue_count) - 1));
3133  
3134  	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
3135  	scsi_unblock_requests(shost);
3136  }
3137  
read_iost_itct_cache_v3_hw(struct hisi_hba * hisi_hba,enum hisi_sas_debugfs_cache_type type,u32 * cache)3138  static void read_iost_itct_cache_v3_hw(struct hisi_hba *hisi_hba,
3139  				       enum hisi_sas_debugfs_cache_type type,
3140  				       u32 *cache)
3141  {
3142  	u32 cache_dw_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ *
3143  			    HISI_SAS_IOST_ITCT_CACHE_NUM;
3144  	struct device *dev = hisi_hba->dev;
3145  	u32 *buf = cache;
3146  	u32 i, val;
3147  
3148  	hisi_sas_write32(hisi_hba, TAB_RD_TYPE, type);
3149  
3150  	for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_DW_SZ; i++) {
3151  		val = hisi_sas_read32(hisi_hba, TAB_DFX);
3152  		if (val == 0xffffffff)
3153  			break;
3154  	}
3155  
3156  	if (val != 0xffffffff) {
3157  		dev_err(dev, "Issue occurred in reading IOST/ITCT cache!\n");
3158  		return;
3159  	}
3160  
3161  	memset(buf, 0, cache_dw_size * 4);
3162  	buf[0] = val;
3163  
3164  	for (i = 1; i < cache_dw_size; i++)
3165  		buf[i] = hisi_sas_read32(hisi_hba, TAB_DFX);
3166  }
3167  
hisi_sas_bist_test_prep_v3_hw(struct hisi_hba * hisi_hba)3168  static void hisi_sas_bist_test_prep_v3_hw(struct hisi_hba *hisi_hba)
3169  {
3170  	u32 reg_val;
3171  	int phy_no = hisi_hba->debugfs_bist_phy_no;
3172  	int i;
3173  
3174  	/* disable PHY */
3175  	hisi_sas_phy_enable(hisi_hba, phy_no, 0);
3176  
3177  	/* update FFE */
3178  	for (i = 0; i < FFE_CFG_MAX; i++)
3179  		hisi_sas_phy_write32(hisi_hba, phy_no, TXDEEMPH_G1 + (i * 0x4),
3180  				     hisi_hba->debugfs_bist_ffe[phy_no][i]);
3181  
3182  	/* disable ALOS */
3183  	reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SERDES_CFG);
3184  	reg_val |= CFG_ALOS_CHK_DISABLE_MSK;
3185  	hisi_sas_phy_write32(hisi_hba, phy_no, SERDES_CFG, reg_val);
3186  }
3187  
hisi_sas_bist_test_restore_v3_hw(struct hisi_hba * hisi_hba)3188  static void hisi_sas_bist_test_restore_v3_hw(struct hisi_hba *hisi_hba)
3189  {
3190  	u32 reg_val;
3191  	int phy_no = hisi_hba->debugfs_bist_phy_no;
3192  
3193  	/* disable loopback */
3194  	reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL);
3195  	reg_val &= ~(CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK |
3196  		     CFG_BIST_TEST_MSK);
3197  	hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL, reg_val);
3198  
3199  	/* enable ALOS */
3200  	reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SERDES_CFG);
3201  	reg_val &= ~CFG_ALOS_CHK_DISABLE_MSK;
3202  	hisi_sas_phy_write32(hisi_hba, phy_no, SERDES_CFG, reg_val);
3203  
3204  	/* restore the linkrate */
3205  	reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
3206  	/* init OOB link rate as 1.5 Gbits */
3207  	reg_val &= ~CFG_PROG_OOB_PHY_LINK_RATE_MSK;
3208  	reg_val |= (0x8 << CFG_PROG_OOB_PHY_LINK_RATE_OFF);
3209  	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE, reg_val);
3210  
3211  	/* enable PHY */
3212  	hisi_sas_phy_enable(hisi_hba, phy_no, 1);
3213  }
3214  
3215  #define SAS_PHY_BIST_CODE_INIT	0x1
3216  #define SAS_PHY_BIST_CODE1_INIT	0X80
debugfs_set_bist_v3_hw(struct hisi_hba * hisi_hba,bool enable)3217  static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable)
3218  {
3219  	u32 reg_val, mode_tmp;
3220  	u32 linkrate = hisi_hba->debugfs_bist_linkrate;
3221  	u32 phy_no = hisi_hba->debugfs_bist_phy_no;
3222  	u32 *ffe = hisi_hba->debugfs_bist_ffe[phy_no];
3223  	u32 code_mode = hisi_hba->debugfs_bist_code_mode;
3224  	u32 path_mode = hisi_hba->debugfs_bist_mode;
3225  	u32 *fix_code = &hisi_hba->debugfs_bist_fixed_code[0];
3226  	struct device *dev = hisi_hba->dev;
3227  
3228  	dev_info(dev, "BIST info:phy%d link_rate=%d code_mode=%d path_mode=%d ffe={0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x} fixed_code={0x%x, 0x%x}\n",
3229  		 phy_no, linkrate, code_mode, path_mode,
3230  		 ffe[FFE_SAS_1_5_GBPS], ffe[FFE_SAS_3_0_GBPS],
3231  		 ffe[FFE_SAS_6_0_GBPS], ffe[FFE_SAS_12_0_GBPS],
3232  		 ffe[FFE_SATA_1_5_GBPS], ffe[FFE_SATA_3_0_GBPS],
3233  		 ffe[FFE_SATA_6_0_GBPS], fix_code[FIXED_CODE],
3234  		 fix_code[FIXED_CODE_1]);
3235  	mode_tmp = path_mode ? 2 : 1;
3236  	if (enable) {
3237  		/* some preparations before bist test */
3238  		hisi_sas_bist_test_prep_v3_hw(hisi_hba);
3239  
3240  		/* set linkrate of bit test*/
3241  		reg_val = hisi_sas_phy_read32(hisi_hba, phy_no,
3242  					      PROG_PHY_LINK_RATE);
3243  		reg_val &= ~CFG_PROG_OOB_PHY_LINK_RATE_MSK;
3244  		reg_val |= (linkrate << CFG_PROG_OOB_PHY_LINK_RATE_OFF);
3245  		hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
3246  				     reg_val);
3247  
3248  		/* set code mode of bit test */
3249  		reg_val = hisi_sas_phy_read32(hisi_hba, phy_no,
3250  					      SAS_PHY_BIST_CTRL);
3251  		reg_val &= ~(CFG_BIST_MODE_SEL_MSK | CFG_LOOP_TEST_MODE_MSK |
3252  			     CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK |
3253  			     CFG_BIST_TEST_MSK);
3254  		reg_val |= ((code_mode << CFG_BIST_MODE_SEL_OFF) |
3255  			    (mode_tmp << CFG_LOOP_TEST_MODE_OFF) |
3256  			    CFG_BIST_TEST_MSK);
3257  		hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL,
3258  				     reg_val);
3259  
3260  		/* set the bist init value */
3261  		if (code_mode == HISI_SAS_BIST_CODE_MODE_FIXED_DATA) {
3262  			reg_val = hisi_hba->debugfs_bist_fixed_code[0];
3263  			hisi_sas_phy_write32(hisi_hba, phy_no,
3264  					     SAS_PHY_BIST_CODE, reg_val);
3265  
3266  			reg_val = hisi_hba->debugfs_bist_fixed_code[1];
3267  			hisi_sas_phy_write32(hisi_hba, phy_no,
3268  					     SAS_PHY_BIST_CODE1, reg_val);
3269  		} else {
3270  			hisi_sas_phy_write32(hisi_hba, phy_no,
3271  					     SAS_PHY_BIST_CODE,
3272  					     SAS_PHY_BIST_CODE_INIT);
3273  			hisi_sas_phy_write32(hisi_hba, phy_no,
3274  					     SAS_PHY_BIST_CODE1,
3275  					     SAS_PHY_BIST_CODE1_INIT);
3276  		}
3277  
3278  		mdelay(100);
3279  		reg_val |= (CFG_RX_BIST_EN_MSK | CFG_TX_BIST_EN_MSK);
3280  		hisi_sas_phy_write32(hisi_hba, phy_no, SAS_PHY_BIST_CTRL,
3281  				     reg_val);
3282  
3283  		/* clear error bit */
3284  		mdelay(100);
3285  		hisi_sas_phy_read32(hisi_hba, phy_no, SAS_BIST_ERR_CNT);
3286  	} else {
3287  		/* disable bist test and recover it */
3288  		hisi_hba->debugfs_bist_cnt += hisi_sas_phy_read32(hisi_hba,
3289  				phy_no, SAS_BIST_ERR_CNT);
3290  		hisi_sas_bist_test_restore_v3_hw(hisi_hba);
3291  	}
3292  
3293  	return 0;
3294  }
3295  
hisi_sas_map_queues(struct Scsi_Host * shost)3296  static void hisi_sas_map_queues(struct Scsi_Host *shost)
3297  {
3298  	struct hisi_hba *hisi_hba = shost_priv(shost);
3299  	struct blk_mq_queue_map *qmap;
3300  	int i, qoff;
3301  
3302  	for (i = 0, qoff = 0; i < shost->nr_maps; i++) {
3303  		qmap = &shost->tag_set.map[i];
3304  		if (i == HCTX_TYPE_DEFAULT) {
3305  			qmap->nr_queues = hisi_hba->cq_nvecs;
3306  		} else if (i == HCTX_TYPE_POLL) {
3307  			qmap->nr_queues = hisi_hba->iopoll_q_cnt;
3308  		} else {
3309  			qmap->nr_queues = 0;
3310  			continue;
3311  		}
3312  
3313  		/* At least one interrupt hardware queue */
3314  		if (!qmap->nr_queues)
3315  			WARN_ON(i == HCTX_TYPE_DEFAULT);
3316  		qmap->queue_offset = qoff;
3317  		if (i == HCTX_TYPE_POLL)
3318  			blk_mq_map_queues(qmap);
3319  		else
3320  			blk_mq_pci_map_queues(qmap, hisi_hba->pci_dev,
3321  					      BASE_VECTORS_V3_HW);
3322  		qoff += qmap->nr_queues;
3323  	}
3324  }
3325  
3326  static const struct scsi_host_template sht_v3_hw = {
3327  	.name			= DRV_NAME,
3328  	.proc_name		= DRV_NAME,
3329  	.module			= THIS_MODULE,
3330  	.queuecommand		= sas_queuecommand,
3331  	.dma_need_drain		= ata_scsi_dma_need_drain,
3332  	.target_alloc		= sas_target_alloc,
3333  	.slave_configure	= slave_configure_v3_hw,
3334  	.scan_finished		= hisi_sas_scan_finished,
3335  	.scan_start		= hisi_sas_scan_start,
3336  	.map_queues		= hisi_sas_map_queues,
3337  	.change_queue_depth	= sas_change_queue_depth,
3338  	.bios_param		= sas_bios_param,
3339  	.this_id		= -1,
3340  	.sg_tablesize		= HISI_SAS_SGE_PAGE_CNT,
3341  	.sg_prot_tablesize	= HISI_SAS_SGE_PAGE_CNT,
3342  	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
3343  	.eh_device_reset_handler = sas_eh_device_reset_handler,
3344  	.eh_target_reset_handler = sas_eh_target_reset_handler,
3345  	.slave_alloc		= hisi_sas_slave_alloc,
3346  	.target_destroy		= sas_target_destroy,
3347  	.ioctl			= sas_ioctl,
3348  #ifdef CONFIG_COMPAT
3349  	.compat_ioctl		= sas_ioctl,
3350  #endif
3351  	.shost_groups		= host_v3_hw_groups,
3352  	.tag_alloc_policy	= BLK_TAG_ALLOC_RR,
3353  	.host_reset             = hisi_sas_host_reset,
3354  	.host_tagset		= 1,
3355  	.mq_poll		= queue_complete_v3_hw,
3356  };
3357  
3358  static const struct hisi_sas_hw hisi_sas_v3_hw = {
3359  	.setup_itct = setup_itct_v3_hw,
3360  	.get_wideport_bitmap = get_wideport_bitmap_v3_hw,
3361  	.complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
3362  	.clear_itct = clear_itct_v3_hw,
3363  	.sl_notify_ssp = sl_notify_ssp_v3_hw,
3364  	.prep_ssp = prep_ssp_v3_hw,
3365  	.prep_smp = prep_smp_v3_hw,
3366  	.prep_stp = prep_ata_v3_hw,
3367  	.prep_abort = prep_abort_v3_hw,
3368  	.start_delivery = start_delivery_v3_hw,
3369  	.phys_init = phys_init_v3_hw,
3370  	.phy_start = start_phy_v3_hw,
3371  	.phy_disable = disable_phy_v3_hw,
3372  	.phy_hard_reset = phy_hard_reset_v3_hw,
3373  	.phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
3374  	.phy_set_linkrate = phy_set_linkrate_v3_hw,
3375  	.dereg_device = dereg_device_v3_hw,
3376  	.soft_reset = soft_reset_v3_hw,
3377  	.get_phys_state = get_phys_state_v3_hw,
3378  	.get_events = phy_get_events_v3_hw,
3379  	.write_gpio = write_gpio_v3_hw,
3380  	.wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw,
3381  	.debugfs_snapshot_regs = debugfs_snapshot_regs_v3_hw,
3382  };
3383  
3384  static struct Scsi_Host *
hisi_sas_shost_alloc_pci(struct pci_dev * pdev)3385  hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
3386  {
3387  	struct Scsi_Host *shost;
3388  	struct hisi_hba *hisi_hba;
3389  	struct device *dev = &pdev->dev;
3390  
3391  	shost = scsi_host_alloc(&sht_v3_hw, sizeof(*hisi_hba));
3392  	if (!shost) {
3393  		dev_err(dev, "shost alloc failed\n");
3394  		return NULL;
3395  	}
3396  	hisi_hba = shost_priv(shost);
3397  
3398  	INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
3399  	hisi_hba->hw = &hisi_sas_v3_hw;
3400  	hisi_hba->pci_dev = pdev;
3401  	hisi_hba->dev = dev;
3402  	hisi_hba->shost = shost;
3403  	SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
3404  
3405  	if (prot_mask & ~HISI_SAS_PROT_MASK)
3406  		dev_err(dev, "unsupported protection mask 0x%x, using default (0x0)\n",
3407  			prot_mask);
3408  	else
3409  		hisi_hba->prot_mask = prot_mask;
3410  
3411  	if (hisi_sas_get_fw_info(hisi_hba) < 0)
3412  		goto err_out;
3413  
3414  	if (experimental_iopoll_q_cnt < 0 ||
3415  		experimental_iopoll_q_cnt >= hisi_hba->queue_count)
3416  		dev_err(dev, "iopoll queue count %d cannot exceed or equal 16, using default 0\n",
3417  			experimental_iopoll_q_cnt);
3418  	else
3419  		hisi_hba->iopoll_q_cnt = experimental_iopoll_q_cnt;
3420  
3421  	if (hisi_sas_alloc(hisi_hba)) {
3422  		hisi_sas_free(hisi_hba);
3423  		goto err_out;
3424  	}
3425  
3426  	return shost;
3427  err_out:
3428  	scsi_host_put(shost);
3429  	dev_err(dev, "shost alloc failed\n");
3430  	return NULL;
3431  }
3432  
debugfs_snapshot_cq_reg_v3_hw(struct hisi_hba * hisi_hba)3433  static void debugfs_snapshot_cq_reg_v3_hw(struct hisi_hba *hisi_hba)
3434  {
3435  	int queue_entry_size = hisi_hba->hw->complete_hdr_size;
3436  	int dump_index = hisi_hba->debugfs_dump_index;
3437  	int i;
3438  
3439  	for (i = 0; i < hisi_hba->queue_count; i++)
3440  		memcpy(hisi_hba->debugfs_cq[dump_index][i].complete_hdr,
3441  		       hisi_hba->complete_hdr[i],
3442  		       HISI_SAS_QUEUE_SLOTS * queue_entry_size);
3443  }
3444  
debugfs_snapshot_dq_reg_v3_hw(struct hisi_hba * hisi_hba)3445  static void debugfs_snapshot_dq_reg_v3_hw(struct hisi_hba *hisi_hba)
3446  {
3447  	int queue_entry_size = sizeof(struct hisi_sas_cmd_hdr);
3448  	int dump_index = hisi_hba->debugfs_dump_index;
3449  	int i;
3450  
3451  	for (i = 0; i < hisi_hba->queue_count; i++) {
3452  		struct hisi_sas_cmd_hdr *debugfs_cmd_hdr, *cmd_hdr;
3453  		int j;
3454  
3455  		debugfs_cmd_hdr = hisi_hba->debugfs_dq[dump_index][i].hdr;
3456  		cmd_hdr = hisi_hba->cmd_hdr[i];
3457  
3458  		for (j = 0; j < HISI_SAS_QUEUE_SLOTS; j++)
3459  			memcpy(&debugfs_cmd_hdr[j], &cmd_hdr[j],
3460  			       queue_entry_size);
3461  	}
3462  }
3463  
debugfs_snapshot_port_reg_v3_hw(struct hisi_hba * hisi_hba)3464  static void debugfs_snapshot_port_reg_v3_hw(struct hisi_hba *hisi_hba)
3465  {
3466  	int dump_index = hisi_hba->debugfs_dump_index;
3467  	const struct hisi_sas_debugfs_reg *port = &debugfs_port_reg;
3468  	int i, phy_cnt;
3469  	u32 offset;
3470  	u32 *databuf;
3471  
3472  	for (phy_cnt = 0; phy_cnt < hisi_hba->n_phy; phy_cnt++) {
3473  		databuf = hisi_hba->debugfs_port_reg[dump_index][phy_cnt].data;
3474  		for (i = 0; i < port->count; i++, databuf++) {
3475  			offset = port->base_off + 4 * i;
3476  			*databuf = hisi_sas_phy_read32(hisi_hba, phy_cnt,
3477  						       offset);
3478  		}
3479  	}
3480  }
3481  
debugfs_snapshot_global_reg_v3_hw(struct hisi_hba * hisi_hba)3482  static void debugfs_snapshot_global_reg_v3_hw(struct hisi_hba *hisi_hba)
3483  {
3484  	int dump_index = hisi_hba->debugfs_dump_index;
3485  	u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_GLOBAL].data;
3486  	int i;
3487  
3488  	for (i = 0; i < debugfs_global_reg.count; i++, databuf++)
3489  		*databuf = hisi_sas_read32(hisi_hba, 4 * i);
3490  }
3491  
debugfs_snapshot_axi_reg_v3_hw(struct hisi_hba * hisi_hba)3492  static void debugfs_snapshot_axi_reg_v3_hw(struct hisi_hba *hisi_hba)
3493  {
3494  	int dump_index = hisi_hba->debugfs_dump_index;
3495  	u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_AXI].data;
3496  	const struct hisi_sas_debugfs_reg *axi = &debugfs_axi_reg;
3497  	int i;
3498  
3499  	for (i = 0; i < axi->count; i++, databuf++)
3500  		*databuf = hisi_sas_read32(hisi_hba, 4 * i + axi->base_off);
3501  }
3502  
debugfs_snapshot_ras_reg_v3_hw(struct hisi_hba * hisi_hba)3503  static void debugfs_snapshot_ras_reg_v3_hw(struct hisi_hba *hisi_hba)
3504  {
3505  	int dump_index = hisi_hba->debugfs_dump_index;
3506  	u32 *databuf = hisi_hba->debugfs_regs[dump_index][DEBUGFS_RAS].data;
3507  	const struct hisi_sas_debugfs_reg *ras = &debugfs_ras_reg;
3508  	int i;
3509  
3510  	for (i = 0; i < ras->count; i++, databuf++)
3511  		*databuf = hisi_sas_read32(hisi_hba, 4 * i + ras->base_off);
3512  }
3513  
debugfs_snapshot_itct_reg_v3_hw(struct hisi_hba * hisi_hba)3514  static void debugfs_snapshot_itct_reg_v3_hw(struct hisi_hba *hisi_hba)
3515  {
3516  	int dump_index = hisi_hba->debugfs_dump_index;
3517  	void *cachebuf = hisi_hba->debugfs_itct_cache[dump_index].cache;
3518  	void *databuf = hisi_hba->debugfs_itct[dump_index].itct;
3519  	struct hisi_sas_itct *itct;
3520  	int i;
3521  
3522  	read_iost_itct_cache_v3_hw(hisi_hba, HISI_SAS_ITCT_CACHE, cachebuf);
3523  
3524  	itct = hisi_hba->itct;
3525  
3526  	for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) {
3527  		memcpy(databuf, itct, sizeof(struct hisi_sas_itct));
3528  		databuf += sizeof(struct hisi_sas_itct);
3529  	}
3530  }
3531  
debugfs_snapshot_iost_reg_v3_hw(struct hisi_hba * hisi_hba)3532  static void debugfs_snapshot_iost_reg_v3_hw(struct hisi_hba *hisi_hba)
3533  {
3534  	int dump_index = hisi_hba->debugfs_dump_index;
3535  	int max_command_entries = HISI_SAS_MAX_COMMANDS;
3536  	void *cachebuf = hisi_hba->debugfs_iost_cache[dump_index].cache;
3537  	void *databuf = hisi_hba->debugfs_iost[dump_index].iost;
3538  	struct hisi_sas_iost *iost;
3539  	int i;
3540  
3541  	read_iost_itct_cache_v3_hw(hisi_hba, HISI_SAS_IOST_CACHE, cachebuf);
3542  
3543  	iost = hisi_hba->iost;
3544  
3545  	for (i = 0; i < max_command_entries; i++, iost++) {
3546  		memcpy(databuf, iost, sizeof(struct hisi_sas_iost));
3547  		databuf += sizeof(struct hisi_sas_iost);
3548  	}
3549  }
3550  
3551  static const char *
debugfs_to_reg_name_v3_hw(int off,int base_off,const struct hisi_sas_debugfs_reg_lu * lu)3552  debugfs_to_reg_name_v3_hw(int off, int base_off,
3553  			  const struct hisi_sas_debugfs_reg_lu *lu)
3554  {
3555  	for (; lu->name; lu++) {
3556  		if (off == lu->off - base_off)
3557  			return lu->name;
3558  	}
3559  
3560  	return NULL;
3561  }
3562  
debugfs_dump_is_generated_v3_hw(void * p)3563  static bool debugfs_dump_is_generated_v3_hw(void *p)
3564  {
3565  	return p ? true : false;
3566  }
3567  
debugfs_print_reg_v3_hw(u32 * regs_val,struct seq_file * s,const struct hisi_sas_debugfs_reg * reg)3568  static void debugfs_print_reg_v3_hw(u32 *regs_val, struct seq_file *s,
3569  				    const struct hisi_sas_debugfs_reg *reg)
3570  {
3571  	int i;
3572  
3573  	for (i = 0; i < reg->count; i++) {
3574  		int off = i * 4;
3575  		const char *name;
3576  
3577  		name = debugfs_to_reg_name_v3_hw(off, reg->base_off,
3578  						 reg->lu);
3579  
3580  		if (name)
3581  			seq_printf(s, "0x%08x 0x%08x %s\n", off,
3582  				   regs_val[i], name);
3583  		else
3584  			seq_printf(s, "0x%08x 0x%08x\n", off,
3585  				   regs_val[i]);
3586  	}
3587  }
3588  
debugfs_global_v3_hw_show(struct seq_file * s,void * p)3589  static int debugfs_global_v3_hw_show(struct seq_file *s, void *p)
3590  {
3591  	struct hisi_sas_debugfs_regs *global = s->private;
3592  
3593  	if (!debugfs_dump_is_generated_v3_hw(global->data))
3594  		return -EPERM;
3595  
3596  	debugfs_print_reg_v3_hw(global->data, s,
3597  				&debugfs_global_reg);
3598  
3599  	return 0;
3600  }
3601  DEFINE_SHOW_ATTRIBUTE(debugfs_global_v3_hw);
3602  
debugfs_axi_v3_hw_show(struct seq_file * s,void * p)3603  static int debugfs_axi_v3_hw_show(struct seq_file *s, void *p)
3604  {
3605  	struct hisi_sas_debugfs_regs *axi = s->private;
3606  
3607  	if (!debugfs_dump_is_generated_v3_hw(axi->data))
3608  		return -EPERM;
3609  
3610  	debugfs_print_reg_v3_hw(axi->data, s,
3611  				&debugfs_axi_reg);
3612  
3613  	return 0;
3614  }
3615  DEFINE_SHOW_ATTRIBUTE(debugfs_axi_v3_hw);
3616  
debugfs_ras_v3_hw_show(struct seq_file * s,void * p)3617  static int debugfs_ras_v3_hw_show(struct seq_file *s, void *p)
3618  {
3619  	struct hisi_sas_debugfs_regs *ras = s->private;
3620  
3621  	if (!debugfs_dump_is_generated_v3_hw(ras->data))
3622  		return -EPERM;
3623  
3624  	debugfs_print_reg_v3_hw(ras->data, s,
3625  				&debugfs_ras_reg);
3626  
3627  	return 0;
3628  }
3629  DEFINE_SHOW_ATTRIBUTE(debugfs_ras_v3_hw);
3630  
debugfs_port_v3_hw_show(struct seq_file * s,void * p)3631  static int debugfs_port_v3_hw_show(struct seq_file *s, void *p)
3632  {
3633  	struct hisi_sas_debugfs_port *port = s->private;
3634  	const struct hisi_sas_debugfs_reg *reg_port = &debugfs_port_reg;
3635  
3636  	if (!debugfs_dump_is_generated_v3_hw(port->data))
3637  		return -EPERM;
3638  
3639  	debugfs_print_reg_v3_hw(port->data, s, reg_port);
3640  
3641  	return 0;
3642  }
3643  DEFINE_SHOW_ATTRIBUTE(debugfs_port_v3_hw);
3644  
debugfs_show_row_64_v3_hw(struct seq_file * s,int index,int sz,__le64 * ptr)3645  static void debugfs_show_row_64_v3_hw(struct seq_file *s, int index,
3646  				      int sz, __le64 *ptr)
3647  {
3648  	int i;
3649  
3650  	/* completion header size not fixed per HW version */
3651  	seq_printf(s, "index %04d:\n\t", index);
3652  	for (i = 1; i <= sz / 8; i++, ptr++) {
3653  		seq_printf(s, " 0x%016llx", le64_to_cpu(*ptr));
3654  		if (!(i % 2))
3655  			seq_puts(s, "\n\t");
3656  	}
3657  
3658  	seq_puts(s, "\n");
3659  }
3660  
debugfs_show_row_32_v3_hw(struct seq_file * s,int index,int sz,__le32 * ptr)3661  static void debugfs_show_row_32_v3_hw(struct seq_file *s, int index,
3662  				      int sz, __le32 *ptr)
3663  {
3664  	int i;
3665  
3666  	/* completion header size not fixed per HW version */
3667  	seq_printf(s, "index %04d:\n\t", index);
3668  	for (i = 1; i <= sz / 4; i++, ptr++) {
3669  		seq_printf(s, " 0x%08x", le32_to_cpu(*ptr));
3670  		if (!(i % 4))
3671  			seq_puts(s, "\n\t");
3672  	}
3673  	seq_puts(s, "\n");
3674  }
3675  
debugfs_cq_show_slot_v3_hw(struct seq_file * s,int slot,struct hisi_sas_debugfs_cq * debugfs_cq)3676  static void debugfs_cq_show_slot_v3_hw(struct seq_file *s, int slot,
3677  				       struct hisi_sas_debugfs_cq *debugfs_cq)
3678  {
3679  	struct hisi_sas_cq *cq = debugfs_cq->cq;
3680  	struct hisi_hba *hisi_hba = cq->hisi_hba;
3681  	__le32 *complete_hdr = debugfs_cq->complete_hdr +
3682  			       (hisi_hba->hw->complete_hdr_size * slot);
3683  
3684  	debugfs_show_row_32_v3_hw(s, slot,
3685  				  hisi_hba->hw->complete_hdr_size,
3686  				  complete_hdr);
3687  }
3688  
debugfs_cq_v3_hw_show(struct seq_file * s,void * p)3689  static int debugfs_cq_v3_hw_show(struct seq_file *s, void *p)
3690  {
3691  	struct hisi_sas_debugfs_cq *debugfs_cq = s->private;
3692  	int slot;
3693  
3694  	if (!debugfs_dump_is_generated_v3_hw(debugfs_cq->complete_hdr))
3695  		return -EPERM;
3696  
3697  	for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++)
3698  		debugfs_cq_show_slot_v3_hw(s, slot, debugfs_cq);
3699  
3700  	return 0;
3701  }
3702  DEFINE_SHOW_ATTRIBUTE(debugfs_cq_v3_hw);
3703  
debugfs_dq_show_slot_v3_hw(struct seq_file * s,int slot,void * dq_ptr)3704  static void debugfs_dq_show_slot_v3_hw(struct seq_file *s, int slot,
3705  				       void *dq_ptr)
3706  {
3707  	struct hisi_sas_debugfs_dq *debugfs_dq = dq_ptr;
3708  	void *cmd_queue = debugfs_dq->hdr;
3709  	__le32 *cmd_hdr = cmd_queue +
3710  		sizeof(struct hisi_sas_cmd_hdr) * slot;
3711  
3712  	debugfs_show_row_32_v3_hw(s, slot, sizeof(struct hisi_sas_cmd_hdr),
3713  				  cmd_hdr);
3714  }
3715  
debugfs_dq_v3_hw_show(struct seq_file * s,void * p)3716  static int debugfs_dq_v3_hw_show(struct seq_file *s, void *p)
3717  {
3718  	struct hisi_sas_debugfs_dq *debugfs_dq = s->private;
3719  	int slot;
3720  
3721  	if (!debugfs_dump_is_generated_v3_hw(debugfs_dq->hdr))
3722  		return -EPERM;
3723  
3724  	for (slot = 0; slot < HISI_SAS_QUEUE_SLOTS; slot++)
3725  		debugfs_dq_show_slot_v3_hw(s, slot, s->private);
3726  
3727  	return 0;
3728  }
3729  DEFINE_SHOW_ATTRIBUTE(debugfs_dq_v3_hw);
3730  
debugfs_iost_v3_hw_show(struct seq_file * s,void * p)3731  static int debugfs_iost_v3_hw_show(struct seq_file *s, void *p)
3732  {
3733  	struct hisi_sas_debugfs_iost *debugfs_iost = s->private;
3734  	struct hisi_sas_iost *iost = debugfs_iost->iost;
3735  	int i, max_command_entries = HISI_SAS_MAX_COMMANDS;
3736  
3737  	if (!debugfs_dump_is_generated_v3_hw(iost))
3738  		return -EPERM;
3739  
3740  	for (i = 0; i < max_command_entries; i++, iost++) {
3741  		__le64 *data = &iost->qw0;
3742  
3743  		debugfs_show_row_64_v3_hw(s, i, sizeof(*iost), data);
3744  	}
3745  
3746  	return 0;
3747  }
3748  DEFINE_SHOW_ATTRIBUTE(debugfs_iost_v3_hw);
3749  
debugfs_iost_cache_v3_hw_show(struct seq_file * s,void * p)3750  static int debugfs_iost_cache_v3_hw_show(struct seq_file *s, void *p)
3751  {
3752  	struct hisi_sas_debugfs_iost_cache *debugfs_iost_cache = s->private;
3753  	struct hisi_sas_iost_itct_cache *iost_cache =
3754  						debugfs_iost_cache->cache;
3755  	u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4;
3756  	int i, tab_idx;
3757  	__le64 *iost;
3758  
3759  	if (!debugfs_dump_is_generated_v3_hw(iost_cache))
3760  		return -EPERM;
3761  
3762  	for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, iost_cache++) {
3763  		/*
3764  		 * Data struct of IOST cache:
3765  		 * Data[1]: BIT0~15: Table index
3766  		 *	    Bit16:   Valid mask
3767  		 * Data[2]~[9]: IOST table
3768  		 */
3769  		tab_idx = (iost_cache->data[1] & 0xffff);
3770  		iost = (__le64 *)iost_cache;
3771  
3772  		debugfs_show_row_64_v3_hw(s, tab_idx, cache_size, iost);
3773  	}
3774  
3775  	return 0;
3776  }
3777  DEFINE_SHOW_ATTRIBUTE(debugfs_iost_cache_v3_hw);
3778  
debugfs_itct_v3_hw_show(struct seq_file * s,void * p)3779  static int debugfs_itct_v3_hw_show(struct seq_file *s, void *p)
3780  {
3781  	int i;
3782  	struct hisi_sas_debugfs_itct *debugfs_itct = s->private;
3783  	struct hisi_sas_itct *itct = debugfs_itct->itct;
3784  
3785  	if (!debugfs_dump_is_generated_v3_hw(itct))
3786  		return -EPERM;
3787  
3788  	for (i = 0; i < HISI_SAS_MAX_ITCT_ENTRIES; i++, itct++) {
3789  		__le64 *data = &itct->qw0;
3790  
3791  		debugfs_show_row_64_v3_hw(s, i, sizeof(*itct), data);
3792  	}
3793  
3794  	return 0;
3795  }
3796  DEFINE_SHOW_ATTRIBUTE(debugfs_itct_v3_hw);
3797  
debugfs_itct_cache_v3_hw_show(struct seq_file * s,void * p)3798  static int debugfs_itct_cache_v3_hw_show(struct seq_file *s, void *p)
3799  {
3800  	struct hisi_sas_debugfs_itct_cache *debugfs_itct_cache = s->private;
3801  	struct hisi_sas_iost_itct_cache *itct_cache =
3802  						debugfs_itct_cache->cache;
3803  	u32 cache_size = HISI_SAS_IOST_ITCT_CACHE_DW_SZ * 4;
3804  	int i, tab_idx;
3805  	__le64 *itct;
3806  
3807  	if (!debugfs_dump_is_generated_v3_hw(itct_cache))
3808  		return -EPERM;
3809  
3810  	for (i = 0; i < HISI_SAS_IOST_ITCT_CACHE_NUM; i++, itct_cache++) {
3811  		/*
3812  		 * Data struct of ITCT cache:
3813  		 * Data[1]: BIT0~15: Table index
3814  		 *	    Bit16:   Valid mask
3815  		 * Data[2]~[9]: ITCT table
3816  		 */
3817  		tab_idx = itct_cache->data[1] & 0xffff;
3818  		itct = (__le64 *)itct_cache;
3819  
3820  		debugfs_show_row_64_v3_hw(s, tab_idx, cache_size, itct);
3821  	}
3822  
3823  	return 0;
3824  }
3825  DEFINE_SHOW_ATTRIBUTE(debugfs_itct_cache_v3_hw);
3826  
debugfs_create_files_v3_hw(struct hisi_hba * hisi_hba,int index)3827  static void debugfs_create_files_v3_hw(struct hisi_hba *hisi_hba, int index)
3828  {
3829  	u64 *debugfs_timestamp;
3830  	struct dentry *dump_dentry;
3831  	struct dentry *dentry;
3832  	char name[256];
3833  	int p;
3834  	int c;
3835  	int d;
3836  
3837  	snprintf(name, 256, "%d", index);
3838  
3839  	dump_dentry = debugfs_create_dir(name, hisi_hba->debugfs_dump_dentry);
3840  
3841  	debugfs_timestamp = &hisi_hba->debugfs_timestamp[index];
3842  
3843  	debugfs_create_u64("timestamp", 0400, dump_dentry,
3844  			   debugfs_timestamp);
3845  
3846  	debugfs_create_file("global", 0400, dump_dentry,
3847  			    &hisi_hba->debugfs_regs[index][DEBUGFS_GLOBAL],
3848  			    &debugfs_global_v3_hw_fops);
3849  
3850  	/* Create port dir and files */
3851  	dentry = debugfs_create_dir("port", dump_dentry);
3852  	for (p = 0; p < hisi_hba->n_phy; p++) {
3853  		snprintf(name, 256, "%d", p);
3854  
3855  		debugfs_create_file(name, 0400, dentry,
3856  				    &hisi_hba->debugfs_port_reg[index][p],
3857  				    &debugfs_port_v3_hw_fops);
3858  	}
3859  
3860  	/* Create CQ dir and files */
3861  	dentry = debugfs_create_dir("cq", dump_dentry);
3862  	for (c = 0; c < hisi_hba->queue_count; c++) {
3863  		snprintf(name, 256, "%d", c);
3864  
3865  		debugfs_create_file(name, 0400, dentry,
3866  				    &hisi_hba->debugfs_cq[index][c],
3867  				    &debugfs_cq_v3_hw_fops);
3868  	}
3869  
3870  	/* Create DQ dir and files */
3871  	dentry = debugfs_create_dir("dq", dump_dentry);
3872  	for (d = 0; d < hisi_hba->queue_count; d++) {
3873  		snprintf(name, 256, "%d", d);
3874  
3875  		debugfs_create_file(name, 0400, dentry,
3876  				    &hisi_hba->debugfs_dq[index][d],
3877  				    &debugfs_dq_v3_hw_fops);
3878  	}
3879  
3880  	debugfs_create_file("iost", 0400, dump_dentry,
3881  			    &hisi_hba->debugfs_iost[index],
3882  			    &debugfs_iost_v3_hw_fops);
3883  
3884  	debugfs_create_file("iost_cache", 0400, dump_dentry,
3885  			    &hisi_hba->debugfs_iost_cache[index],
3886  			    &debugfs_iost_cache_v3_hw_fops);
3887  
3888  	debugfs_create_file("itct", 0400, dump_dentry,
3889  			    &hisi_hba->debugfs_itct[index],
3890  			    &debugfs_itct_v3_hw_fops);
3891  
3892  	debugfs_create_file("itct_cache", 0400, dump_dentry,
3893  			    &hisi_hba->debugfs_itct_cache[index],
3894  			    &debugfs_itct_cache_v3_hw_fops);
3895  
3896  	debugfs_create_file("axi", 0400, dump_dentry,
3897  			    &hisi_hba->debugfs_regs[index][DEBUGFS_AXI],
3898  			    &debugfs_axi_v3_hw_fops);
3899  
3900  	debugfs_create_file("ras", 0400, dump_dentry,
3901  			    &hisi_hba->debugfs_regs[index][DEBUGFS_RAS],
3902  			    &debugfs_ras_v3_hw_fops);
3903  }
3904  
debugfs_trigger_dump_v3_hw_write(struct file * file,const char __user * user_buf,size_t count,loff_t * ppos)3905  static ssize_t debugfs_trigger_dump_v3_hw_write(struct file *file,
3906  						const char __user *user_buf,
3907  						size_t count, loff_t *ppos)
3908  {
3909  	struct hisi_hba *hisi_hba = file->f_inode->i_private;
3910  	char buf[8];
3911  
3912  	if (count > 8)
3913  		return -EFAULT;
3914  
3915  	if (copy_from_user(buf, user_buf, count))
3916  		return -EFAULT;
3917  
3918  	if (buf[0] != '1')
3919  		return -EFAULT;
3920  
3921  	down(&hisi_hba->sem);
3922  	if (debugfs_snapshot_regs_v3_hw(hisi_hba)) {
3923  		up(&hisi_hba->sem);
3924  		return -EFAULT;
3925  	}
3926  	up(&hisi_hba->sem);
3927  
3928  	return count;
3929  }
3930  
3931  static const struct file_operations debugfs_trigger_dump_v3_hw_fops = {
3932  	.write = &debugfs_trigger_dump_v3_hw_write,
3933  	.owner = THIS_MODULE,
3934  };
3935  
3936  enum {
3937  	HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL = 0,
3938  	HISI_SAS_BIST_LOOPBACK_MODE_SERDES,
3939  	HISI_SAS_BIST_LOOPBACK_MODE_REMOTE,
3940  };
3941  
3942  static const struct {
3943  	int		value;
3944  	char		*name;
3945  } debugfs_loop_linkrate_v3_hw[] = {
3946  	{ SAS_LINK_RATE_1_5_GBPS, "1.5 Gbit" },
3947  	{ SAS_LINK_RATE_3_0_GBPS, "3.0 Gbit" },
3948  	{ SAS_LINK_RATE_6_0_GBPS, "6.0 Gbit" },
3949  	{ SAS_LINK_RATE_12_0_GBPS, "12.0 Gbit" },
3950  };
3951  
debugfs_bist_linkrate_v3_hw_show(struct seq_file * s,void * p)3952  static int debugfs_bist_linkrate_v3_hw_show(struct seq_file *s, void *p)
3953  {
3954  	struct hisi_hba *hisi_hba = s->private;
3955  	int i;
3956  
3957  	for (i = 0; i < ARRAY_SIZE(debugfs_loop_linkrate_v3_hw); i++) {
3958  		int match = (hisi_hba->debugfs_bist_linkrate ==
3959  			     debugfs_loop_linkrate_v3_hw[i].value);
3960  
3961  		seq_printf(s, "%s%s%s ", match ? "[" : "",
3962  			   debugfs_loop_linkrate_v3_hw[i].name,
3963  			   match ? "]" : "");
3964  	}
3965  	seq_puts(s, "\n");
3966  
3967  	return 0;
3968  }
3969  
debugfs_bist_linkrate_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)3970  static ssize_t debugfs_bist_linkrate_v3_hw_write(struct file *filp,
3971  						 const char __user *buf,
3972  						 size_t count, loff_t *ppos)
3973  {
3974  	struct seq_file *m = filp->private_data;
3975  	struct hisi_hba *hisi_hba = m->private;
3976  	char kbuf[16] = {}, *pkbuf;
3977  	bool found = false;
3978  	int i;
3979  
3980  	if (hisi_hba->debugfs_bist_enable)
3981  		return -EPERM;
3982  
3983  	if (count >= sizeof(kbuf))
3984  		return -EOVERFLOW;
3985  
3986  	if (copy_from_user(kbuf, buf, count))
3987  		return -EINVAL;
3988  
3989  	pkbuf = strstrip(kbuf);
3990  
3991  	for (i = 0; i < ARRAY_SIZE(debugfs_loop_linkrate_v3_hw); i++) {
3992  		if (!strncmp(debugfs_loop_linkrate_v3_hw[i].name,
3993  			     pkbuf, 16)) {
3994  			hisi_hba->debugfs_bist_linkrate =
3995  				debugfs_loop_linkrate_v3_hw[i].value;
3996  			found = true;
3997  			break;
3998  		}
3999  	}
4000  
4001  	if (!found)
4002  		return -EINVAL;
4003  
4004  	return count;
4005  }
4006  
debugfs_bist_linkrate_v3_hw_open(struct inode * inode,struct file * filp)4007  static int debugfs_bist_linkrate_v3_hw_open(struct inode *inode,
4008  					    struct file *filp)
4009  {
4010  	return single_open(filp, debugfs_bist_linkrate_v3_hw_show,
4011  			   inode->i_private);
4012  }
4013  
4014  static const struct file_operations debugfs_bist_linkrate_v3_hw_fops = {
4015  	.open = debugfs_bist_linkrate_v3_hw_open,
4016  	.read = seq_read,
4017  	.write = debugfs_bist_linkrate_v3_hw_write,
4018  	.llseek = seq_lseek,
4019  	.release = single_release,
4020  	.owner = THIS_MODULE,
4021  };
4022  
4023  static const struct {
4024  	int		value;
4025  	char		*name;
4026  } debugfs_loop_code_mode_v3_hw[] = {
4027  	{ HISI_SAS_BIST_CODE_MODE_PRBS7, "PRBS7" },
4028  	{ HISI_SAS_BIST_CODE_MODE_PRBS23, "PRBS23" },
4029  	{ HISI_SAS_BIST_CODE_MODE_PRBS31, "PRBS31" },
4030  	{ HISI_SAS_BIST_CODE_MODE_JTPAT, "JTPAT" },
4031  	{ HISI_SAS_BIST_CODE_MODE_CJTPAT, "CJTPAT" },
4032  	{ HISI_SAS_BIST_CODE_MODE_SCRAMBED_0, "SCRAMBED_0" },
4033  	{ HISI_SAS_BIST_CODE_MODE_TRAIN, "TRAIN" },
4034  	{ HISI_SAS_BIST_CODE_MODE_TRAIN_DONE, "TRAIN_DONE" },
4035  	{ HISI_SAS_BIST_CODE_MODE_HFTP, "HFTP" },
4036  	{ HISI_SAS_BIST_CODE_MODE_MFTP, "MFTP" },
4037  	{ HISI_SAS_BIST_CODE_MODE_LFTP, "LFTP" },
4038  	{ HISI_SAS_BIST_CODE_MODE_FIXED_DATA, "FIXED_DATA" },
4039  };
4040  
debugfs_bist_code_mode_v3_hw_show(struct seq_file * s,void * p)4041  static int debugfs_bist_code_mode_v3_hw_show(struct seq_file *s, void *p)
4042  {
4043  	struct hisi_hba *hisi_hba = s->private;
4044  	int i;
4045  
4046  	for (i = 0; i < ARRAY_SIZE(debugfs_loop_code_mode_v3_hw); i++) {
4047  		int match = (hisi_hba->debugfs_bist_code_mode ==
4048  			     debugfs_loop_code_mode_v3_hw[i].value);
4049  
4050  		seq_printf(s, "%s%s%s ", match ? "[" : "",
4051  			   debugfs_loop_code_mode_v3_hw[i].name,
4052  			   match ? "]" : "");
4053  	}
4054  	seq_puts(s, "\n");
4055  
4056  	return 0;
4057  }
4058  
debugfs_bist_code_mode_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4059  static ssize_t debugfs_bist_code_mode_v3_hw_write(struct file *filp,
4060  						  const char __user *buf,
4061  						  size_t count,
4062  						  loff_t *ppos)
4063  {
4064  	struct seq_file *m = filp->private_data;
4065  	struct hisi_hba *hisi_hba = m->private;
4066  	char kbuf[16] = {}, *pkbuf;
4067  	bool found = false;
4068  	int i;
4069  
4070  	if (hisi_hba->debugfs_bist_enable)
4071  		return -EPERM;
4072  
4073  	if (count >= sizeof(kbuf))
4074  		return -EINVAL;
4075  
4076  	if (copy_from_user(kbuf, buf, count))
4077  		return -EOVERFLOW;
4078  
4079  	pkbuf = strstrip(kbuf);
4080  
4081  	for (i = 0; i < ARRAY_SIZE(debugfs_loop_code_mode_v3_hw); i++) {
4082  		if (!strncmp(debugfs_loop_code_mode_v3_hw[i].name,
4083  			     pkbuf, 16)) {
4084  			hisi_hba->debugfs_bist_code_mode =
4085  				debugfs_loop_code_mode_v3_hw[i].value;
4086  			found = true;
4087  			break;
4088  		}
4089  	}
4090  
4091  	if (!found)
4092  		return -EINVAL;
4093  
4094  	return count;
4095  }
4096  
debugfs_bist_code_mode_v3_hw_open(struct inode * inode,struct file * filp)4097  static int debugfs_bist_code_mode_v3_hw_open(struct inode *inode,
4098  					     struct file *filp)
4099  {
4100  	return single_open(filp, debugfs_bist_code_mode_v3_hw_show,
4101  			   inode->i_private);
4102  }
4103  
4104  static const struct file_operations debugfs_bist_code_mode_v3_hw_fops = {
4105  	.open = debugfs_bist_code_mode_v3_hw_open,
4106  	.read = seq_read,
4107  	.write = debugfs_bist_code_mode_v3_hw_write,
4108  	.llseek = seq_lseek,
4109  	.release = single_release,
4110  	.owner = THIS_MODULE,
4111  };
4112  
debugfs_bist_phy_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4113  static ssize_t debugfs_bist_phy_v3_hw_write(struct file *filp,
4114  					    const char __user *buf,
4115  					    size_t count, loff_t *ppos)
4116  {
4117  	struct seq_file *m = filp->private_data;
4118  	struct hisi_hba *hisi_hba = m->private;
4119  	unsigned int phy_no;
4120  	int val;
4121  
4122  	if (hisi_hba->debugfs_bist_enable)
4123  		return -EPERM;
4124  
4125  	val = kstrtouint_from_user(buf, count, 0, &phy_no);
4126  	if (val)
4127  		return val;
4128  
4129  	if (phy_no >= hisi_hba->n_phy)
4130  		return -EINVAL;
4131  
4132  	hisi_hba->debugfs_bist_phy_no = phy_no;
4133  
4134  	return count;
4135  }
4136  
debugfs_bist_phy_v3_hw_show(struct seq_file * s,void * p)4137  static int debugfs_bist_phy_v3_hw_show(struct seq_file *s, void *p)
4138  {
4139  	struct hisi_hba *hisi_hba = s->private;
4140  
4141  	seq_printf(s, "%d\n", hisi_hba->debugfs_bist_phy_no);
4142  
4143  	return 0;
4144  }
4145  
debugfs_bist_phy_v3_hw_open(struct inode * inode,struct file * filp)4146  static int debugfs_bist_phy_v3_hw_open(struct inode *inode,
4147  				       struct file *filp)
4148  {
4149  	return single_open(filp, debugfs_bist_phy_v3_hw_show,
4150  			   inode->i_private);
4151  }
4152  
4153  static const struct file_operations debugfs_bist_phy_v3_hw_fops = {
4154  	.open = debugfs_bist_phy_v3_hw_open,
4155  	.read = seq_read,
4156  	.write = debugfs_bist_phy_v3_hw_write,
4157  	.llseek = seq_lseek,
4158  	.release = single_release,
4159  	.owner = THIS_MODULE,
4160  };
4161  
debugfs_bist_cnt_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4162  static ssize_t debugfs_bist_cnt_v3_hw_write(struct file *filp,
4163  					const char __user *buf,
4164  					size_t count, loff_t *ppos)
4165  {
4166  	struct seq_file *m = filp->private_data;
4167  	struct hisi_hba *hisi_hba = m->private;
4168  	unsigned int cnt;
4169  	int val;
4170  
4171  	if (hisi_hba->debugfs_bist_enable)
4172  		return -EPERM;
4173  
4174  	val = kstrtouint_from_user(buf, count, 0, &cnt);
4175  	if (val)
4176  		return val;
4177  
4178  	if (cnt)
4179  		return -EINVAL;
4180  
4181  	hisi_hba->debugfs_bist_cnt = 0;
4182  	return count;
4183  }
4184  
debugfs_bist_cnt_v3_hw_show(struct seq_file * s,void * p)4185  static int debugfs_bist_cnt_v3_hw_show(struct seq_file *s, void *p)
4186  {
4187  	struct hisi_hba *hisi_hba = s->private;
4188  
4189  	seq_printf(s, "%u\n", hisi_hba->debugfs_bist_cnt);
4190  
4191  	return 0;
4192  }
4193  
debugfs_bist_cnt_v3_hw_open(struct inode * inode,struct file * filp)4194  static int debugfs_bist_cnt_v3_hw_open(struct inode *inode,
4195  					  struct file *filp)
4196  {
4197  	return single_open(filp, debugfs_bist_cnt_v3_hw_show,
4198  			   inode->i_private);
4199  }
4200  
4201  static const struct file_operations debugfs_bist_cnt_v3_hw_ops = {
4202  	.open = debugfs_bist_cnt_v3_hw_open,
4203  	.read = seq_read,
4204  	.write = debugfs_bist_cnt_v3_hw_write,
4205  	.llseek = seq_lseek,
4206  	.release = single_release,
4207  	.owner = THIS_MODULE,
4208  };
4209  
4210  static const struct {
4211  	int		value;
4212  	char		*name;
4213  } debugfs_loop_modes_v3_hw[] = {
4214  	{ HISI_SAS_BIST_LOOPBACK_MODE_DIGITAL, "digital" },
4215  	{ HISI_SAS_BIST_LOOPBACK_MODE_SERDES, "serdes" },
4216  	{ HISI_SAS_BIST_LOOPBACK_MODE_REMOTE, "remote" },
4217  };
4218  
debugfs_bist_mode_v3_hw_show(struct seq_file * s,void * p)4219  static int debugfs_bist_mode_v3_hw_show(struct seq_file *s, void *p)
4220  {
4221  	struct hisi_hba *hisi_hba = s->private;
4222  	int i;
4223  
4224  	for (i = 0; i < ARRAY_SIZE(debugfs_loop_modes_v3_hw); i++) {
4225  		int match = (hisi_hba->debugfs_bist_mode ==
4226  			     debugfs_loop_modes_v3_hw[i].value);
4227  
4228  		seq_printf(s, "%s%s%s ", match ? "[" : "",
4229  			   debugfs_loop_modes_v3_hw[i].name,
4230  			   match ? "]" : "");
4231  	}
4232  	seq_puts(s, "\n");
4233  
4234  	return 0;
4235  }
4236  
debugfs_bist_mode_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4237  static ssize_t debugfs_bist_mode_v3_hw_write(struct file *filp,
4238  					     const char __user *buf,
4239  					     size_t count, loff_t *ppos)
4240  {
4241  	struct seq_file *m = filp->private_data;
4242  	struct hisi_hba *hisi_hba = m->private;
4243  	char kbuf[16] = {}, *pkbuf;
4244  	bool found = false;
4245  	int i;
4246  
4247  	if (hisi_hba->debugfs_bist_enable)
4248  		return -EPERM;
4249  
4250  	if (count >= sizeof(kbuf))
4251  		return -EINVAL;
4252  
4253  	if (copy_from_user(kbuf, buf, count))
4254  		return -EOVERFLOW;
4255  
4256  	pkbuf = strstrip(kbuf);
4257  
4258  	for (i = 0; i < ARRAY_SIZE(debugfs_loop_modes_v3_hw); i++) {
4259  		if (!strncmp(debugfs_loop_modes_v3_hw[i].name, pkbuf, 16)) {
4260  			hisi_hba->debugfs_bist_mode =
4261  				debugfs_loop_modes_v3_hw[i].value;
4262  			found = true;
4263  			break;
4264  		}
4265  	}
4266  
4267  	if (!found)
4268  		return -EINVAL;
4269  
4270  	return count;
4271  }
4272  
debugfs_bist_mode_v3_hw_open(struct inode * inode,struct file * filp)4273  static int debugfs_bist_mode_v3_hw_open(struct inode *inode,
4274  					struct file *filp)
4275  {
4276  	return single_open(filp, debugfs_bist_mode_v3_hw_show,
4277  			   inode->i_private);
4278  }
4279  
4280  static const struct file_operations debugfs_bist_mode_v3_hw_fops = {
4281  	.open = debugfs_bist_mode_v3_hw_open,
4282  	.read = seq_read,
4283  	.write = debugfs_bist_mode_v3_hw_write,
4284  	.llseek = seq_lseek,
4285  	.release = single_release,
4286  	.owner = THIS_MODULE,
4287  };
4288  
debugfs_bist_enable_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4289  static ssize_t debugfs_bist_enable_v3_hw_write(struct file *filp,
4290  					       const char __user *buf,
4291  					       size_t count, loff_t *ppos)
4292  {
4293  	struct seq_file *m = filp->private_data;
4294  	struct hisi_hba *hisi_hba = m->private;
4295  	unsigned int enable;
4296  	int val;
4297  
4298  	val = kstrtouint_from_user(buf, count, 0, &enable);
4299  	if (val)
4300  		return val;
4301  
4302  	if (enable > 1)
4303  		return -EINVAL;
4304  
4305  	if (enable == hisi_hba->debugfs_bist_enable)
4306  		return count;
4307  
4308  	val = debugfs_set_bist_v3_hw(hisi_hba, enable);
4309  	if (val < 0)
4310  		return val;
4311  
4312  	hisi_hba->debugfs_bist_enable = enable;
4313  
4314  	return count;
4315  }
4316  
debugfs_bist_enable_v3_hw_show(struct seq_file * s,void * p)4317  static int debugfs_bist_enable_v3_hw_show(struct seq_file *s, void *p)
4318  {
4319  	struct hisi_hba *hisi_hba = s->private;
4320  
4321  	seq_printf(s, "%d\n", hisi_hba->debugfs_bist_enable);
4322  
4323  	return 0;
4324  }
4325  
debugfs_bist_enable_v3_hw_open(struct inode * inode,struct file * filp)4326  static int debugfs_bist_enable_v3_hw_open(struct inode *inode,
4327  					  struct file *filp)
4328  {
4329  	return single_open(filp, debugfs_bist_enable_v3_hw_show,
4330  			   inode->i_private);
4331  }
4332  
4333  static const struct file_operations debugfs_bist_enable_v3_hw_fops = {
4334  	.open = debugfs_bist_enable_v3_hw_open,
4335  	.read = seq_read,
4336  	.write = debugfs_bist_enable_v3_hw_write,
4337  	.llseek = seq_lseek,
4338  	.release = single_release,
4339  	.owner = THIS_MODULE,
4340  };
4341  
4342  static const struct {
4343  	char *name;
4344  } debugfs_ffe_name_v3_hw[FFE_CFG_MAX] = {
4345  	{ "SAS_1_5_GBPS" },
4346  	{ "SAS_3_0_GBPS" },
4347  	{ "SAS_6_0_GBPS" },
4348  	{ "SAS_12_0_GBPS" },
4349  	{ "FFE_RESV" },
4350  	{ "SATA_1_5_GBPS" },
4351  	{ "SATA_3_0_GBPS" },
4352  	{ "SATA_6_0_GBPS" },
4353  };
4354  
debugfs_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4355  static ssize_t debugfs_v3_hw_write(struct file *filp,
4356  				   const char __user *buf,
4357  				   size_t count, loff_t *ppos)
4358  {
4359  	struct seq_file *m = filp->private_data;
4360  	u32 *val = m->private;
4361  	int res;
4362  
4363  	res = kstrtouint_from_user(buf, count, 0, val);
4364  	if (res)
4365  		return res;
4366  
4367  	return count;
4368  }
4369  
debugfs_v3_hw_show(struct seq_file * s,void * p)4370  static int debugfs_v3_hw_show(struct seq_file *s, void *p)
4371  {
4372  	u32 *val = s->private;
4373  
4374  	seq_printf(s, "0x%x\n", *val);
4375  
4376  	return 0;
4377  }
4378  
debugfs_v3_hw_open(struct inode * inode,struct file * filp)4379  static int debugfs_v3_hw_open(struct inode *inode, struct file *filp)
4380  {
4381  	return single_open(filp, debugfs_v3_hw_show,
4382  			   inode->i_private);
4383  }
4384  
4385  static const struct file_operations debugfs_v3_hw_fops = {
4386  	.open = debugfs_v3_hw_open,
4387  	.read = seq_read,
4388  	.write = debugfs_v3_hw_write,
4389  	.llseek = seq_lseek,
4390  	.release = single_release,
4391  	.owner = THIS_MODULE,
4392  };
4393  
debugfs_phy_down_cnt_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4394  static ssize_t debugfs_phy_down_cnt_v3_hw_write(struct file *filp,
4395  						const char __user *buf,
4396  						size_t count, loff_t *ppos)
4397  {
4398  	struct seq_file *s = filp->private_data;
4399  	struct hisi_sas_phy *phy = s->private;
4400  	unsigned int set_val;
4401  	int res;
4402  
4403  	res = kstrtouint_from_user(buf, count, 0, &set_val);
4404  	if (res)
4405  		return res;
4406  
4407  	if (set_val > 0)
4408  		return -EINVAL;
4409  
4410  	atomic_set(&phy->down_cnt, 0);
4411  
4412  	return count;
4413  }
4414  
debugfs_phy_down_cnt_v3_hw_show(struct seq_file * s,void * p)4415  static int debugfs_phy_down_cnt_v3_hw_show(struct seq_file *s, void *p)
4416  {
4417  	struct hisi_sas_phy *phy = s->private;
4418  
4419  	seq_printf(s, "%d\n", atomic_read(&phy->down_cnt));
4420  
4421  	return 0;
4422  }
4423  
debugfs_phy_down_cnt_v3_hw_open(struct inode * inode,struct file * filp)4424  static int debugfs_phy_down_cnt_v3_hw_open(struct inode *inode,
4425  					   struct file *filp)
4426  {
4427  	return single_open(filp, debugfs_phy_down_cnt_v3_hw_show,
4428  			   inode->i_private);
4429  }
4430  
4431  static const struct file_operations debugfs_phy_down_cnt_v3_hw_fops = {
4432  	.open = debugfs_phy_down_cnt_v3_hw_open,
4433  	.read = seq_read,
4434  	.write = debugfs_phy_down_cnt_v3_hw_write,
4435  	.llseek = seq_lseek,
4436  	.release = single_release,
4437  	.owner = THIS_MODULE,
4438  };
4439  
4440  enum fifo_dump_mode_v3_hw {
4441  	FIFO_DUMP_FORVER =		(1U << 0),
4442  	FIFO_DUMP_AFTER_TRIGGER =	(1U << 1),
4443  	FIFO_DUMP_UNTILL_TRIGGER =	(1U << 2),
4444  };
4445  
4446  enum fifo_trigger_mode_v3_hw {
4447  	FIFO_TRIGGER_EDGE =		(1U << 0),
4448  	FIFO_TRIGGER_SAME_LEVEL =	(1U << 1),
4449  	FIFO_TRIGGER_DIFF_LEVEL =	(1U << 2),
4450  };
4451  
debugfs_is_fifo_config_valid_v3_hw(struct hisi_sas_phy * phy)4452  static int debugfs_is_fifo_config_valid_v3_hw(struct hisi_sas_phy *phy)
4453  {
4454  	struct hisi_hba *hisi_hba = phy->hisi_hba;
4455  
4456  	if (phy->fifo.signal_sel > 0xf) {
4457  		dev_info(hisi_hba->dev, "Invalid signal select: %u\n",
4458  			 phy->fifo.signal_sel);
4459  		return -EINVAL;
4460  	}
4461  
4462  	switch (phy->fifo.dump_mode) {
4463  	case FIFO_DUMP_FORVER:
4464  	case FIFO_DUMP_AFTER_TRIGGER:
4465  	case FIFO_DUMP_UNTILL_TRIGGER:
4466  		break;
4467  	default:
4468  		dev_info(hisi_hba->dev, "Invalid dump mode: %u\n",
4469  			 phy->fifo.dump_mode);
4470  		return -EINVAL;
4471  	}
4472  
4473  	/* when FIFO_DUMP_FORVER, no need to check trigger_mode */
4474  	if (phy->fifo.dump_mode == FIFO_DUMP_FORVER)
4475  		return 0;
4476  
4477  	switch (phy->fifo.trigger_mode) {
4478  	case FIFO_TRIGGER_EDGE:
4479  	case FIFO_TRIGGER_SAME_LEVEL:
4480  	case FIFO_TRIGGER_DIFF_LEVEL:
4481  		break;
4482  	default:
4483  		dev_info(hisi_hba->dev, "Invalid trigger mode: %u\n",
4484  			 phy->fifo.trigger_mode);
4485  		return -EINVAL;
4486  	}
4487  	return 0;
4488  }
4489  
debugfs_update_fifo_config_v3_hw(struct hisi_sas_phy * phy)4490  static int debugfs_update_fifo_config_v3_hw(struct hisi_sas_phy *phy)
4491  {
4492  	u32 trigger_mode = phy->fifo.trigger_mode;
4493  	u32 signal_sel = phy->fifo.signal_sel;
4494  	u32 dump_mode = phy->fifo.dump_mode;
4495  	struct hisi_hba *hisi_hba = phy->hisi_hba;
4496  	int phy_no = phy->sas_phy.id;
4497  	u32 reg_val;
4498  	int res;
4499  
4500  	/* Check the validity of trace FIFO configuration */
4501  	res = debugfs_is_fifo_config_valid_v3_hw(phy);
4502  	if (res)
4503  		return res;
4504  
4505  	reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4506  	/* Disable trace FIFO before update configuration */
4507  	reg_val |= DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4508  
4509  	/* Update trace FIFO configuration */
4510  	reg_val &= ~(DFX_FIFO_CTRL_DUMP_MODE_MSK |
4511  		     DFX_FIFO_CTRL_SIGNAL_SEL_MSK |
4512  		     DFX_FIFO_CTRL_TRIGGER_MODE_MSK);
4513  
4514  	reg_val |= ((trigger_mode << DFX_FIFO_CTRL_TRIGGER_MODE_OFF) |
4515  		    (dump_mode << DFX_FIFO_CTRL_DUMP_MODE_OFF) |
4516  		    (signal_sel << DFX_FIFO_CTRL_SIGNAL_SEL_OFF));
4517  	hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, reg_val);
4518  
4519  	hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_DUMP_MSK,
4520  			     phy->fifo.dump_msk);
4521  
4522  	hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_TRIGGER,
4523  			     phy->fifo.trigger);
4524  
4525  	hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_TRIGGER_MSK,
4526  			     phy->fifo.trigger_msk);
4527  
4528  	/* Enable trace FIFO after updated configuration */
4529  	reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4530  	reg_val &= ~DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4531  	hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, reg_val);
4532  
4533  	return 0;
4534  }
4535  
debugfs_fifo_update_cfg_v3_hw_write(struct file * filp,const char __user * buf,size_t count,loff_t * ppos)4536  static ssize_t debugfs_fifo_update_cfg_v3_hw_write(struct file *filp,
4537  						   const char __user *buf,
4538  						   size_t count, loff_t *ppos)
4539  {
4540  	struct hisi_sas_phy *phy = filp->private_data;
4541  	bool update;
4542  	int val;
4543  
4544  	val = kstrtobool_from_user(buf, count, &update);
4545  	if (val)
4546  		return val;
4547  
4548  	if (update != 1)
4549  		return -EINVAL;
4550  
4551  	val = debugfs_update_fifo_config_v3_hw(phy);
4552  	if (val)
4553  		return val;
4554  
4555  	return count;
4556  }
4557  
4558  static const struct file_operations debugfs_fifo_update_cfg_v3_hw_fops = {
4559  	.open = simple_open,
4560  	.write = debugfs_fifo_update_cfg_v3_hw_write,
4561  	.owner = THIS_MODULE,
4562  };
4563  
debugfs_read_fifo_data_v3_hw(struct hisi_sas_phy * phy)4564  static void debugfs_read_fifo_data_v3_hw(struct hisi_sas_phy *phy)
4565  {
4566  	struct hisi_hba *hisi_hba = phy->hisi_hba;
4567  	u32 *buf = phy->fifo.rd_data;
4568  	int phy_no = phy->sas_phy.id;
4569  	u32 val;
4570  	int i;
4571  
4572  	memset(buf, 0, sizeof(phy->fifo.rd_data));
4573  
4574  	/* Disable trace FIFO before read data */
4575  	val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4576  	val |= DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4577  	hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val);
4578  
4579  	for (i = 0; i < HISI_SAS_FIFO_DATA_DW_SIZE; i++) {
4580  		val = hisi_sas_phy_read32(hisi_hba, phy_no,
4581  					  DFX_FIFO_RD_DATA);
4582  		buf[i] = val;
4583  	}
4584  
4585  	/* Enable trace FIFO after read data */
4586  	val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4587  	val &= ~DFX_FIFO_CTRL_DUMP_DISABLE_MSK;
4588  	hisi_sas_phy_write32(hisi_hba, phy_no, DFX_FIFO_CTRL, val);
4589  }
4590  
debugfs_fifo_data_v3_hw_show(struct seq_file * s,void * p)4591  static int debugfs_fifo_data_v3_hw_show(struct seq_file *s, void *p)
4592  {
4593  	struct hisi_sas_phy *phy = s->private;
4594  
4595  	debugfs_read_fifo_data_v3_hw(phy);
4596  
4597  	debugfs_show_row_32_v3_hw(s, 0, HISI_SAS_FIFO_DATA_DW_SIZE * 4,
4598  				  (__le32 *)phy->fifo.rd_data);
4599  
4600  	return 0;
4601  }
4602  DEFINE_SHOW_ATTRIBUTE(debugfs_fifo_data_v3_hw);
4603  
debugfs_fifo_init_v3_hw(struct hisi_hba * hisi_hba)4604  static void debugfs_fifo_init_v3_hw(struct hisi_hba *hisi_hba)
4605  {
4606  	int phy_no;
4607  
4608  	hisi_hba->debugfs_fifo_dentry =
4609  			debugfs_create_dir("fifo", hisi_hba->debugfs_dir);
4610  
4611  	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
4612  		struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
4613  		struct dentry *port_dentry;
4614  		char name[256];
4615  		u32 val;
4616  
4617  		/* get default configuration for trace FIFO */
4618  		val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4619  		val &= DFX_FIFO_CTRL_DUMP_MODE_MSK;
4620  		val >>= DFX_FIFO_CTRL_DUMP_MODE_OFF;
4621  		phy->fifo.dump_mode = val;
4622  
4623  		val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4624  		val &= DFX_FIFO_CTRL_TRIGGER_MODE_MSK;
4625  		val >>= DFX_FIFO_CTRL_TRIGGER_MODE_OFF;
4626  		phy->fifo.trigger_mode = val;
4627  
4628  		val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_CTRL);
4629  		val &= DFX_FIFO_CTRL_SIGNAL_SEL_MSK;
4630  		val >>= DFX_FIFO_CTRL_SIGNAL_SEL_OFF;
4631  		phy->fifo.signal_sel = val;
4632  
4633  		val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_DUMP_MSK);
4634  		phy->fifo.dump_msk = val;
4635  
4636  		val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_TRIGGER);
4637  		phy->fifo.trigger = val;
4638  		val = hisi_sas_phy_read32(hisi_hba, phy_no, DFX_FIFO_TRIGGER_MSK);
4639  		phy->fifo.trigger_msk = val;
4640  
4641  		snprintf(name, 256, "%d", phy_no);
4642  		port_dentry = debugfs_create_dir(name,
4643  						 hisi_hba->debugfs_fifo_dentry);
4644  
4645  		debugfs_create_file("update_config", 0200, port_dentry, phy,
4646  				    &debugfs_fifo_update_cfg_v3_hw_fops);
4647  
4648  		debugfs_create_file("signal_sel", 0600, port_dentry,
4649  				    &phy->fifo.signal_sel,
4650  				    &debugfs_v3_hw_fops);
4651  
4652  		debugfs_create_file("dump_msk", 0600, port_dentry,
4653  				    &phy->fifo.dump_msk,
4654  				    &debugfs_v3_hw_fops);
4655  
4656  		debugfs_create_file("dump_mode", 0600, port_dentry,
4657  				    &phy->fifo.dump_mode,
4658  				    &debugfs_v3_hw_fops);
4659  
4660  		debugfs_create_file("trigger_mode", 0600, port_dentry,
4661  				    &phy->fifo.trigger_mode,
4662  				    &debugfs_v3_hw_fops);
4663  
4664  		debugfs_create_file("trigger", 0600, port_dentry,
4665  				    &phy->fifo.trigger,
4666  				    &debugfs_v3_hw_fops);
4667  
4668  		debugfs_create_file("trigger_msk", 0600, port_dentry,
4669  				    &phy->fifo.trigger_msk,
4670  				    &debugfs_v3_hw_fops);
4671  
4672  		debugfs_create_file("fifo_data", 0400, port_dentry, phy,
4673  				    &debugfs_fifo_data_v3_hw_fops);
4674  	}
4675  }
4676  
debugfs_release_v3_hw(struct hisi_hba * hisi_hba,int dump_index)4677  static void debugfs_release_v3_hw(struct hisi_hba *hisi_hba, int dump_index)
4678  {
4679  	struct device *dev = hisi_hba->dev;
4680  	int i;
4681  
4682  	devm_kfree(dev, hisi_hba->debugfs_iost_cache[dump_index].cache);
4683  	hisi_hba->debugfs_iost_cache[dump_index].cache = NULL;
4684  	devm_kfree(dev, hisi_hba->debugfs_itct_cache[dump_index].cache);
4685  	hisi_hba->debugfs_itct_cache[dump_index].cache = NULL;
4686  	devm_kfree(dev, hisi_hba->debugfs_iost[dump_index].iost);
4687  	hisi_hba->debugfs_iost[dump_index].iost = NULL;
4688  	devm_kfree(dev, hisi_hba->debugfs_itct[dump_index].itct);
4689  	hisi_hba->debugfs_itct[dump_index].itct = NULL;
4690  
4691  	for (i = 0; i < hisi_hba->queue_count; i++) {
4692  		devm_kfree(dev, hisi_hba->debugfs_dq[dump_index][i].hdr);
4693  		hisi_hba->debugfs_dq[dump_index][i].hdr = NULL;
4694  	}
4695  
4696  	for (i = 0; i < hisi_hba->queue_count; i++) {
4697  		devm_kfree(dev,
4698  			   hisi_hba->debugfs_cq[dump_index][i].complete_hdr);
4699  		hisi_hba->debugfs_cq[dump_index][i].complete_hdr = NULL;
4700  	}
4701  
4702  	for (i = 0; i < DEBUGFS_REGS_NUM; i++) {
4703  		devm_kfree(dev, hisi_hba->debugfs_regs[dump_index][i].data);
4704  		hisi_hba->debugfs_regs[dump_index][i].data = NULL;
4705  	}
4706  
4707  	for (i = 0; i < hisi_hba->n_phy; i++) {
4708  		devm_kfree(dev, hisi_hba->debugfs_port_reg[dump_index][i].data);
4709  		hisi_hba->debugfs_port_reg[dump_index][i].data = NULL;
4710  	}
4711  }
4712  
4713  static const struct hisi_sas_debugfs_reg *debugfs_reg_array_v3_hw[DEBUGFS_REGS_NUM] = {
4714  	[DEBUGFS_GLOBAL] = &debugfs_global_reg,
4715  	[DEBUGFS_AXI] = &debugfs_axi_reg,
4716  	[DEBUGFS_RAS] = &debugfs_ras_reg,
4717  };
4718  
debugfs_alloc_v3_hw(struct hisi_hba * hisi_hba,int dump_index)4719  static int debugfs_alloc_v3_hw(struct hisi_hba *hisi_hba, int dump_index)
4720  {
4721  	const struct hisi_sas_hw *hw = hisi_hba->hw;
4722  	struct device *dev = hisi_hba->dev;
4723  	int p, c, d, r;
4724  	size_t sz;
4725  
4726  	for (r = 0; r < DEBUGFS_REGS_NUM; r++) {
4727  		struct hisi_sas_debugfs_regs *regs =
4728  				&hisi_hba->debugfs_regs[dump_index][r];
4729  
4730  		sz = debugfs_reg_array_v3_hw[r]->count * 4;
4731  		regs->data = devm_kmalloc(dev, sz, GFP_KERNEL);
4732  		if (!regs->data)
4733  			goto fail;
4734  		regs->hisi_hba = hisi_hba;
4735  	}
4736  
4737  	sz = debugfs_port_reg.count * 4;
4738  	for (p = 0; p < hisi_hba->n_phy; p++) {
4739  		struct hisi_sas_debugfs_port *port =
4740  				&hisi_hba->debugfs_port_reg[dump_index][p];
4741  
4742  		port->data = devm_kmalloc(dev, sz, GFP_KERNEL);
4743  		if (!port->data)
4744  			goto fail;
4745  		port->phy = &hisi_hba->phy[p];
4746  	}
4747  
4748  	sz = hw->complete_hdr_size * HISI_SAS_QUEUE_SLOTS;
4749  	for (c = 0; c < hisi_hba->queue_count; c++) {
4750  		struct hisi_sas_debugfs_cq *cq =
4751  				&hisi_hba->debugfs_cq[dump_index][c];
4752  
4753  		cq->complete_hdr = devm_kmalloc(dev, sz, GFP_KERNEL);
4754  		if (!cq->complete_hdr)
4755  			goto fail;
4756  		cq->cq = &hisi_hba->cq[c];
4757  	}
4758  
4759  	sz = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS;
4760  	for (d = 0; d < hisi_hba->queue_count; d++) {
4761  		struct hisi_sas_debugfs_dq *dq =
4762  				&hisi_hba->debugfs_dq[dump_index][d];
4763  
4764  		dq->hdr = devm_kmalloc(dev, sz, GFP_KERNEL);
4765  		if (!dq->hdr)
4766  			goto fail;
4767  		dq->dq = &hisi_hba->dq[d];
4768  	}
4769  
4770  	sz = HISI_SAS_MAX_COMMANDS * sizeof(struct hisi_sas_iost);
4771  
4772  	hisi_hba->debugfs_iost[dump_index].iost =
4773  				devm_kmalloc(dev, sz, GFP_KERNEL);
4774  	if (!hisi_hba->debugfs_iost[dump_index].iost)
4775  		goto fail;
4776  
4777  	sz = HISI_SAS_IOST_ITCT_CACHE_NUM *
4778  	     sizeof(struct hisi_sas_iost_itct_cache);
4779  
4780  	hisi_hba->debugfs_iost_cache[dump_index].cache =
4781  				devm_kmalloc(dev, sz, GFP_KERNEL);
4782  	if (!hisi_hba->debugfs_iost_cache[dump_index].cache)
4783  		goto fail;
4784  
4785  	sz = HISI_SAS_IOST_ITCT_CACHE_NUM *
4786  	     sizeof(struct hisi_sas_iost_itct_cache);
4787  
4788  	hisi_hba->debugfs_itct_cache[dump_index].cache =
4789  				devm_kmalloc(dev, sz, GFP_KERNEL);
4790  	if (!hisi_hba->debugfs_itct_cache[dump_index].cache)
4791  		goto fail;
4792  
4793  	/* New memory allocation must be locate before itct */
4794  	sz = HISI_SAS_MAX_ITCT_ENTRIES * sizeof(struct hisi_sas_itct);
4795  
4796  	hisi_hba->debugfs_itct[dump_index].itct =
4797  				devm_kmalloc(dev, sz, GFP_KERNEL);
4798  	if (!hisi_hba->debugfs_itct[dump_index].itct)
4799  		goto fail;
4800  
4801  	return 0;
4802  fail:
4803  	debugfs_release_v3_hw(hisi_hba, dump_index);
4804  	return -ENOMEM;
4805  }
4806  
debugfs_snapshot_regs_v3_hw(struct hisi_hba * hisi_hba)4807  static int debugfs_snapshot_regs_v3_hw(struct hisi_hba *hisi_hba)
4808  {
4809  	int debugfs_dump_index = hisi_hba->debugfs_dump_index;
4810  	struct device *dev = hisi_hba->dev;
4811  	u64 timestamp = local_clock();
4812  
4813  	if (debugfs_dump_index >= hisi_sas_debugfs_dump_count) {
4814  		dev_warn(dev, "dump count exceeded!\n");
4815  		return -EINVAL;
4816  	}
4817  
4818  	if (debugfs_alloc_v3_hw(hisi_hba, debugfs_dump_index)) {
4819  		dev_warn(dev, "failed to alloc memory\n");
4820  		return -ENOMEM;
4821  	}
4822  
4823  	do_div(timestamp, NSEC_PER_MSEC);
4824  	hisi_hba->debugfs_timestamp[debugfs_dump_index] = timestamp;
4825  
4826  	debugfs_snapshot_prepare_v3_hw(hisi_hba);
4827  
4828  	debugfs_snapshot_global_reg_v3_hw(hisi_hba);
4829  	debugfs_snapshot_port_reg_v3_hw(hisi_hba);
4830  	debugfs_snapshot_axi_reg_v3_hw(hisi_hba);
4831  	debugfs_snapshot_ras_reg_v3_hw(hisi_hba);
4832  	debugfs_snapshot_cq_reg_v3_hw(hisi_hba);
4833  	debugfs_snapshot_dq_reg_v3_hw(hisi_hba);
4834  	debugfs_snapshot_itct_reg_v3_hw(hisi_hba);
4835  	debugfs_snapshot_iost_reg_v3_hw(hisi_hba);
4836  
4837  	debugfs_snapshot_restore_v3_hw(hisi_hba);
4838  	hisi_hba->debugfs_dump_index++;
4839  
4840  	return 0;
4841  }
4842  
debugfs_phy_down_cnt_init_v3_hw(struct hisi_hba * hisi_hba)4843  static void debugfs_phy_down_cnt_init_v3_hw(struct hisi_hba *hisi_hba)
4844  {
4845  	struct dentry *dir = debugfs_create_dir("phy_down_cnt",
4846  						hisi_hba->debugfs_dir);
4847  	char name[16];
4848  	int phy_no;
4849  
4850  	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
4851  		snprintf(name, 16, "%d", phy_no);
4852  		debugfs_create_file(name, 0600, dir,
4853  				    &hisi_hba->phy[phy_no],
4854  				    &debugfs_phy_down_cnt_v3_hw_fops);
4855  	}
4856  }
4857  
debugfs_bist_init_v3_hw(struct hisi_hba * hisi_hba)4858  static void debugfs_bist_init_v3_hw(struct hisi_hba *hisi_hba)
4859  {
4860  	struct dentry *ports_dentry;
4861  	int phy_no;
4862  
4863  	hisi_hba->debugfs_bist_dentry =
4864  			debugfs_create_dir("bist", hisi_hba->debugfs_dir);
4865  	debugfs_create_file("link_rate", 0600,
4866  			    hisi_hba->debugfs_bist_dentry, hisi_hba,
4867  			    &debugfs_bist_linkrate_v3_hw_fops);
4868  
4869  	debugfs_create_file("code_mode", 0600,
4870  			    hisi_hba->debugfs_bist_dentry, hisi_hba,
4871  			    &debugfs_bist_code_mode_v3_hw_fops);
4872  
4873  	debugfs_create_file("fixed_code", 0600,
4874  			    hisi_hba->debugfs_bist_dentry,
4875  			    &hisi_hba->debugfs_bist_fixed_code[0],
4876  			    &debugfs_v3_hw_fops);
4877  
4878  	debugfs_create_file("fixed_code_1", 0600,
4879  			    hisi_hba->debugfs_bist_dentry,
4880  			    &hisi_hba->debugfs_bist_fixed_code[1],
4881  			    &debugfs_v3_hw_fops);
4882  
4883  	debugfs_create_file("phy_id", 0600, hisi_hba->debugfs_bist_dentry,
4884  			    hisi_hba, &debugfs_bist_phy_v3_hw_fops);
4885  
4886  	debugfs_create_file("cnt", 0600, hisi_hba->debugfs_bist_dentry,
4887  			    hisi_hba, &debugfs_bist_cnt_v3_hw_ops);
4888  
4889  	debugfs_create_file("loopback_mode", 0600,
4890  			    hisi_hba->debugfs_bist_dentry,
4891  			    hisi_hba, &debugfs_bist_mode_v3_hw_fops);
4892  
4893  	debugfs_create_file("enable", 0600, hisi_hba->debugfs_bist_dentry,
4894  			    hisi_hba, &debugfs_bist_enable_v3_hw_fops);
4895  
4896  	ports_dentry = debugfs_create_dir("port", hisi_hba->debugfs_bist_dentry);
4897  
4898  	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
4899  		struct dentry *port_dentry;
4900  		struct dentry *ffe_dentry;
4901  		char name[256];
4902  		int i;
4903  
4904  		snprintf(name, 256, "%d", phy_no);
4905  		port_dentry = debugfs_create_dir(name, ports_dentry);
4906  		ffe_dentry = debugfs_create_dir("ffe", port_dentry);
4907  		for (i = 0; i < FFE_CFG_MAX; i++) {
4908  			if (i == FFE_RESV)
4909  				continue;
4910  			debugfs_create_file(debugfs_ffe_name_v3_hw[i].name,
4911  					    0600, ffe_dentry,
4912  					    &hisi_hba->debugfs_bist_ffe[phy_no][i],
4913  					    &debugfs_v3_hw_fops);
4914  		}
4915  	}
4916  
4917  	hisi_hba->debugfs_bist_linkrate = SAS_LINK_RATE_1_5_GBPS;
4918  }
4919  
debugfs_dump_init_v3_hw(struct hisi_hba * hisi_hba)4920  static void debugfs_dump_init_v3_hw(struct hisi_hba *hisi_hba)
4921  {
4922  	int i;
4923  
4924  	hisi_hba->debugfs_dump_dentry =
4925  			debugfs_create_dir("dump", hisi_hba->debugfs_dir);
4926  
4927  	for (i = 0; i < hisi_sas_debugfs_dump_count; i++)
4928  		debugfs_create_files_v3_hw(hisi_hba, i);
4929  }
4930  
debugfs_exit_v3_hw(struct hisi_hba * hisi_hba)4931  static void debugfs_exit_v3_hw(struct hisi_hba *hisi_hba)
4932  {
4933  	debugfs_remove_recursive(hisi_hba->debugfs_dir);
4934  	hisi_hba->debugfs_dir = NULL;
4935  }
4936  
debugfs_init_v3_hw(struct hisi_hba * hisi_hba)4937  static void debugfs_init_v3_hw(struct hisi_hba *hisi_hba)
4938  {
4939  	struct device *dev = hisi_hba->dev;
4940  
4941  	hisi_hba->debugfs_dir = debugfs_create_dir(dev_name(dev),
4942  						   hisi_sas_debugfs_dir);
4943  	debugfs_create_file("trigger_dump", 0200,
4944  			    hisi_hba->debugfs_dir,
4945  			    hisi_hba,
4946  			    &debugfs_trigger_dump_v3_hw_fops);
4947  
4948  	/* create bist structures */
4949  	debugfs_bist_init_v3_hw(hisi_hba);
4950  
4951  	debugfs_dump_init_v3_hw(hisi_hba);
4952  
4953  	debugfs_phy_down_cnt_init_v3_hw(hisi_hba);
4954  	debugfs_fifo_init_v3_hw(hisi_hba);
4955  }
4956  
4957  static int
hisi_sas_v3_probe(struct pci_dev * pdev,const struct pci_device_id * id)4958  hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
4959  {
4960  	struct Scsi_Host *shost;
4961  	struct hisi_hba *hisi_hba;
4962  	struct device *dev = &pdev->dev;
4963  	struct asd_sas_phy **arr_phy;
4964  	struct asd_sas_port **arr_port;
4965  	struct sas_ha_struct *sha;
4966  	int rc, phy_nr, port_nr, i;
4967  
4968  	rc = pcim_enable_device(pdev);
4969  	if (rc)
4970  		goto err_out;
4971  
4972  	pci_set_master(pdev);
4973  
4974  	rc = pcim_iomap_regions(pdev, 1 << BAR_NO_V3_HW, DRV_NAME);
4975  	if (rc)
4976  		goto err_out;
4977  
4978  	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4979  	if (rc) {
4980  		dev_err(dev, "No usable DMA addressing method\n");
4981  		rc = -ENODEV;
4982  		goto err_out;
4983  	}
4984  
4985  	shost = hisi_sas_shost_alloc_pci(pdev);
4986  	if (!shost) {
4987  		rc = -ENOMEM;
4988  		goto err_out;
4989  	}
4990  
4991  	sha = SHOST_TO_SAS_HA(shost);
4992  	hisi_hba = shost_priv(shost);
4993  	dev_set_drvdata(dev, sha);
4994  
4995  	hisi_hba->regs = pcim_iomap_table(pdev)[BAR_NO_V3_HW];
4996  	if (!hisi_hba->regs) {
4997  		dev_err(dev, "cannot map register\n");
4998  		rc = -ENOMEM;
4999  		goto err_out_free_host;
5000  	}
5001  
5002  	phy_nr = port_nr = hisi_hba->n_phy;
5003  
5004  	arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
5005  	arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
5006  	if (!arr_phy || !arr_port) {
5007  		rc = -ENOMEM;
5008  		goto err_out_free_host;
5009  	}
5010  
5011  	sha->sas_phy = arr_phy;
5012  	sha->sas_port = arr_port;
5013  	sha->shost = shost;
5014  	sha->lldd_ha = hisi_hba;
5015  
5016  	shost->transportt = hisi_sas_stt;
5017  	shost->max_id = HISI_SAS_MAX_DEVICES;
5018  	shost->max_lun = ~0;
5019  	shost->max_channel = 1;
5020  	shost->max_cmd_len = 16;
5021  	shost->can_queue = HISI_SAS_UNRESERVED_IPTT;
5022  	shost->cmd_per_lun = HISI_SAS_UNRESERVED_IPTT;
5023  	if (hisi_hba->iopoll_q_cnt)
5024  		shost->nr_maps = 3;
5025  	else
5026  		shost->nr_maps = 1;
5027  
5028  	sha->sas_ha_name = DRV_NAME;
5029  	sha->dev = dev;
5030  	sha->sas_addr = &hisi_hba->sas_addr[0];
5031  	sha->num_phys = hisi_hba->n_phy;
5032  
5033  	for (i = 0; i < hisi_hba->n_phy; i++) {
5034  		sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
5035  		sha->sas_port[i] = &hisi_hba->port[i].sas_port;
5036  	}
5037  
5038  	if (hisi_hba->prot_mask) {
5039  		dev_info(dev, "Registering for DIF/DIX prot_mask=0x%x\n",
5040  			 prot_mask);
5041  		scsi_host_set_prot(hisi_hba->shost, prot_mask);
5042  		if (hisi_hba->prot_mask & HISI_SAS_DIX_PROT_MASK)
5043  			scsi_host_set_guard(hisi_hba->shost,
5044  					    SHOST_DIX_GUARD_CRC);
5045  	}
5046  
5047  	if (hisi_sas_debugfs_enable)
5048  		debugfs_init_v3_hw(hisi_hba);
5049  
5050  	rc = interrupt_preinit_v3_hw(hisi_hba);
5051  	if (rc)
5052  		goto err_out_undo_debugfs;
5053  
5054  	rc = scsi_add_host(shost, dev);
5055  	if (rc)
5056  		goto err_out_undo_debugfs;
5057  
5058  	rc = sas_register_ha(sha);
5059  	if (rc)
5060  		goto err_out_remove_host;
5061  
5062  	rc = hisi_sas_v3_init(hisi_hba);
5063  	if (rc)
5064  		goto err_out_unregister_ha;
5065  
5066  	scsi_scan_host(shost);
5067  
5068  	pm_runtime_set_autosuspend_delay(dev, 5000);
5069  	pm_runtime_use_autosuspend(dev);
5070  	/*
5071  	 * For the situation that there are ATA disks connected with SAS
5072  	 * controller, it additionally creates ata_port which will affect the
5073  	 * child_count of hisi_hba->dev. Even if suspended all the disks,
5074  	 * ata_port is still and the child_count of hisi_hba->dev is not 0.
5075  	 * So use pm_suspend_ignore_children() to ignore the effect to
5076  	 * hisi_hba->dev.
5077  	 */
5078  	pm_suspend_ignore_children(dev, true);
5079  	pm_runtime_put_noidle(&pdev->dev);
5080  
5081  	return 0;
5082  
5083  err_out_unregister_ha:
5084  	sas_unregister_ha(sha);
5085  err_out_remove_host:
5086  	scsi_remove_host(shost);
5087  err_out_undo_debugfs:
5088  	debugfs_exit_v3_hw(hisi_hba);
5089  err_out_free_host:
5090  	hisi_sas_free(hisi_hba);
5091  	scsi_host_put(shost);
5092  err_out:
5093  	return rc;
5094  }
5095  
5096  static void
hisi_sas_v3_destroy_irqs(struct pci_dev * pdev,struct hisi_hba * hisi_hba)5097  hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
5098  {
5099  	int i;
5100  
5101  	devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 1), hisi_hba);
5102  	devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 2), hisi_hba);
5103  	devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 11), hisi_hba);
5104  	for (i = 0; i < hisi_hba->cq_nvecs; i++) {
5105  		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
5106  		int nr = hisi_sas_intr_conv ? 16 : 16 + i;
5107  
5108  		devm_free_irq(&pdev->dev, pci_irq_vector(pdev, nr), cq);
5109  	}
5110  }
5111  
hisi_sas_v3_remove(struct pci_dev * pdev)5112  static void hisi_sas_v3_remove(struct pci_dev *pdev)
5113  {
5114  	struct device *dev = &pdev->dev;
5115  	struct sas_ha_struct *sha = dev_get_drvdata(dev);
5116  	struct hisi_hba *hisi_hba = sha->lldd_ha;
5117  	struct Scsi_Host *shost = sha->shost;
5118  
5119  	pm_runtime_get_noresume(dev);
5120  	del_timer_sync(&hisi_hba->timer);
5121  
5122  	sas_unregister_ha(sha);
5123  	flush_workqueue(hisi_hba->wq);
5124  	sas_remove_host(shost);
5125  
5126  	hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
5127  	hisi_sas_free(hisi_hba);
5128  	debugfs_exit_v3_hw(hisi_hba);
5129  	scsi_host_put(shost);
5130  }
5131  
hisi_sas_reset_prepare_v3_hw(struct pci_dev * pdev)5132  static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev)
5133  {
5134  	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5135  	struct hisi_hba *hisi_hba = sha->lldd_ha;
5136  	struct device *dev = hisi_hba->dev;
5137  	int rc;
5138  
5139  	dev_info(dev, "FLR prepare\n");
5140  	down(&hisi_hba->sem);
5141  	set_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags);
5142  	hisi_sas_controller_reset_prepare(hisi_hba);
5143  
5144  	interrupt_disable_v3_hw(hisi_hba);
5145  	rc = disable_host_v3_hw(hisi_hba);
5146  	if (rc)
5147  		dev_err(dev, "FLR: disable host failed rc=%d\n", rc);
5148  }
5149  
hisi_sas_reset_done_v3_hw(struct pci_dev * pdev)5150  static void hisi_sas_reset_done_v3_hw(struct pci_dev *pdev)
5151  {
5152  	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5153  	struct hisi_hba *hisi_hba = sha->lldd_ha;
5154  	struct Scsi_Host *shost = hisi_hba->shost;
5155  	struct device *dev = hisi_hba->dev;
5156  	int rc;
5157  
5158  	hisi_sas_init_mem(hisi_hba);
5159  
5160  	rc = hw_init_v3_hw(hisi_hba);
5161  	if (rc) {
5162  		dev_err(dev, "FLR: hw init failed rc=%d\n", rc);
5163  		clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
5164  		scsi_unblock_requests(shost);
5165  		clear_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags);
5166  		up(&hisi_hba->sem);
5167  		return;
5168  	}
5169  
5170  	hisi_sas_controller_reset_done(hisi_hba);
5171  	dev_info(dev, "FLR done\n");
5172  }
5173  
5174  enum {
5175  	/* instances of the controller */
5176  	hip08,
5177  };
5178  
enable_host_v3_hw(struct hisi_hba * hisi_hba)5179  static void enable_host_v3_hw(struct hisi_hba *hisi_hba)
5180  {
5181  	u32 reg_val;
5182  
5183  	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
5184  			 (u32)((1ULL << hisi_hba->queue_count) - 1));
5185  
5186  	phys_init_v3_hw(hisi_hba);
5187  	reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
5188  				  AM_CTRL_GLOBAL);
5189  	reg_val &= ~AM_CTRL_SHUTDOWN_REQ_MSK;
5190  	hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
5191  			 AM_CTRL_GLOBAL, reg_val);
5192  }
5193  
_suspend_v3_hw(struct device * device)5194  static int _suspend_v3_hw(struct device *device)
5195  {
5196  	struct pci_dev *pdev = to_pci_dev(device);
5197  	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5198  	struct hisi_hba *hisi_hba = sha->lldd_ha;
5199  	struct device *dev = hisi_hba->dev;
5200  	struct Scsi_Host *shost = hisi_hba->shost;
5201  	int rc;
5202  
5203  	if (!pdev->pm_cap) {
5204  		dev_err(dev, "PCI PM not supported\n");
5205  		return -ENODEV;
5206  	}
5207  
5208  	if (test_and_set_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags))
5209  		return -EPERM;
5210  
5211  	dev_warn(dev, "entering suspend state\n");
5212  
5213  	scsi_block_requests(shost);
5214  	set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
5215  	flush_workqueue(hisi_hba->wq);
5216  	interrupt_disable_v3_hw(hisi_hba);
5217  
5218  #ifdef CONFIG_PM
5219  	if (atomic_read(&device->power.usage_count)) {
5220  		dev_err(dev, "PM suspend: host status cannot be suspended\n");
5221  		rc = -EBUSY;
5222  		goto err_out;
5223  	}
5224  #endif
5225  
5226  	rc = disable_host_v3_hw(hisi_hba);
5227  	if (rc) {
5228  		dev_err(dev, "PM suspend: disable host failed rc=%d\n", rc);
5229  		goto err_out_recover_host;
5230  	}
5231  
5232  	hisi_sas_init_mem(hisi_hba);
5233  
5234  	hisi_sas_release_tasks(hisi_hba);
5235  
5236  	sas_suspend_ha(sha);
5237  
5238  	dev_warn(dev, "end of suspending controller\n");
5239  	return 0;
5240  
5241  err_out_recover_host:
5242  	enable_host_v3_hw(hisi_hba);
5243  #ifdef CONFIG_PM
5244  err_out:
5245  #endif
5246  	interrupt_enable_v3_hw(hisi_hba);
5247  	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
5248  	clear_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags);
5249  	scsi_unblock_requests(shost);
5250  	return rc;
5251  }
5252  
_resume_v3_hw(struct device * device)5253  static int _resume_v3_hw(struct device *device)
5254  {
5255  	struct pci_dev *pdev = to_pci_dev(device);
5256  	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5257  	struct hisi_hba *hisi_hba = sha->lldd_ha;
5258  	struct Scsi_Host *shost = hisi_hba->shost;
5259  	struct device *dev = hisi_hba->dev;
5260  	unsigned int rc;
5261  	pci_power_t device_state = pdev->current_state;
5262  
5263  	dev_warn(dev, "resuming from operating state [D%d]\n",
5264  		 device_state);
5265  
5266  	scsi_unblock_requests(shost);
5267  	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
5268  
5269  	sas_prep_resume_ha(sha);
5270  	rc = hw_init_v3_hw(hisi_hba);
5271  	if (rc) {
5272  		scsi_remove_host(shost);
5273  		return rc;
5274  	}
5275  	phys_init_v3_hw(hisi_hba);
5276  
5277  	/*
5278  	 * If a directly-attached disk is removed during suspend, a deadlock
5279  	 * may occur, as the PHYE_RESUME_TIMEOUT processing will require the
5280  	 * hisi_hba->device to be active, which can only happen when resume
5281  	 * completes. So don't wait for the HA event workqueue to drain upon
5282  	 * resume.
5283  	 */
5284  	sas_resume_ha_no_sync(sha);
5285  	clear_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags);
5286  
5287  	dev_warn(dev, "end of resuming controller\n");
5288  
5289  	return 0;
5290  }
5291  
suspend_v3_hw(struct device * device)5292  static int __maybe_unused suspend_v3_hw(struct device *device)
5293  {
5294  	struct pci_dev *pdev = to_pci_dev(device);
5295  	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5296  	struct hisi_hba *hisi_hba = sha->lldd_ha;
5297  	int rc;
5298  
5299  	set_bit(HISI_SAS_PM_BIT, &hisi_hba->flags);
5300  
5301  	rc = _suspend_v3_hw(device);
5302  	if (rc)
5303  		clear_bit(HISI_SAS_PM_BIT, &hisi_hba->flags);
5304  
5305  	return rc;
5306  }
5307  
resume_v3_hw(struct device * device)5308  static int __maybe_unused resume_v3_hw(struct device *device)
5309  {
5310  	struct pci_dev *pdev = to_pci_dev(device);
5311  	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
5312  	struct hisi_hba *hisi_hba = sha->lldd_ha;
5313  	int rc = _resume_v3_hw(device);
5314  
5315  	clear_bit(HISI_SAS_PM_BIT, &hisi_hba->flags);
5316  
5317  	return rc;
5318  }
5319  
5320  static const struct pci_device_id sas_v3_pci_table[] = {
5321  	{ PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
5322  	{}
5323  };
5324  MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
5325  
5326  static const struct pci_error_handlers hisi_sas_err_handler = {
5327  	.reset_prepare	= hisi_sas_reset_prepare_v3_hw,
5328  	.reset_done	= hisi_sas_reset_done_v3_hw,
5329  };
5330  
5331  static UNIVERSAL_DEV_PM_OPS(hisi_sas_v3_pm_ops,
5332  			    suspend_v3_hw,
5333  			    resume_v3_hw,
5334  			    NULL);
5335  
5336  static struct pci_driver sas_v3_pci_driver = {
5337  	.name		= DRV_NAME,
5338  	.id_table	= sas_v3_pci_table,
5339  	.probe		= hisi_sas_v3_probe,
5340  	.remove		= hisi_sas_v3_remove,
5341  	.err_handler	= &hisi_sas_err_handler,
5342  	.driver.pm	= &hisi_sas_v3_pm_ops,
5343  };
5344  
5345  module_pci_driver(sas_v3_pci_driver);
5346  module_param_named(intr_conv, hisi_sas_intr_conv, bool, 0444);
5347  
5348  MODULE_LICENSE("GPL");
5349  MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
5350  MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
5351  MODULE_ALIAS("pci:" DRV_NAME);
5352