1 /* 2 * Copyright (c) 2017 Hisilicon Limited. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 */ 10 11 #include "hisi_sas.h" 12 #define DRV_NAME "hisi_sas_v3_hw" 13 14 /* global registers need init*/ 15 #define DLVRY_QUEUE_ENABLE 0x0 16 #define IOST_BASE_ADDR_LO 0x8 17 #define IOST_BASE_ADDR_HI 0xc 18 #define ITCT_BASE_ADDR_LO 0x10 19 #define ITCT_BASE_ADDR_HI 0x14 20 #define IO_BROKEN_MSG_ADDR_LO 0x18 21 #define IO_BROKEN_MSG_ADDR_HI 0x1c 22 #define PHY_CONTEXT 0x20 23 #define PHY_STATE 0x24 24 #define PHY_PORT_NUM_MA 0x28 25 #define PHY_CONN_RATE 0x30 26 #define AXI_AHB_CLK_CFG 0x3c 27 #define ITCT_CLR 0x44 28 #define ITCT_CLR_EN_OFF 16 29 #define ITCT_CLR_EN_MSK (0x1 << ITCT_CLR_EN_OFF) 30 #define ITCT_DEV_OFF 0 31 #define ITCT_DEV_MSK (0x7ff << ITCT_DEV_OFF) 32 #define AXI_USER1 0x48 33 #define AXI_USER2 0x4c 34 #define IO_SATA_BROKEN_MSG_ADDR_LO 0x58 35 #define IO_SATA_BROKEN_MSG_ADDR_HI 0x5c 36 #define SATA_INITI_D2H_STORE_ADDR_LO 0x60 37 #define SATA_INITI_D2H_STORE_ADDR_HI 0x64 38 #define CFG_MAX_TAG 0x68 39 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84 40 #define HGC_SAS_TXFAIL_RETRY_CTRL 0x88 41 #define HGC_GET_ITV_TIME 0x90 42 #define DEVICE_MSG_WORK_MODE 0x94 43 #define OPENA_WT_CONTI_TIME 0x9c 44 #define I_T_NEXUS_LOSS_TIME 0xa0 45 #define MAX_CON_TIME_LIMIT_TIME 0xa4 46 #define BUS_INACTIVE_LIMIT_TIME 0xa8 47 #define REJECT_TO_OPEN_LIMIT_TIME 0xac 48 #define CFG_AGING_TIME 0xbc 49 #define HGC_DFX_CFG2 0xc0 50 #define CFG_ABT_SET_QUERY_IPTT 0xd4 51 #define CFG_SET_ABORTED_IPTT_OFF 0 52 #define CFG_SET_ABORTED_IPTT_MSK (0xfff << CFG_SET_ABORTED_IPTT_OFF) 53 #define CFG_SET_ABORTED_EN_OFF 12 54 #define CFG_ABT_SET_IPTT_DONE 0xd8 55 #define CFG_ABT_SET_IPTT_DONE_OFF 0 56 #define HGC_IOMB_PROC1_STATUS 0x104 57 #define CFG_1US_TIMER_TRSH 0xcc 58 #define CHNL_INT_STATUS 0x148 59 #define INT_COAL_EN 0x19c 60 #define OQ_INT_COAL_TIME 0x1a0 61 #define OQ_INT_COAL_CNT 0x1a4 62 #define ENT_INT_COAL_TIME 0x1a8 63 #define ENT_INT_COAL_CNT 0x1ac 64 #define OQ_INT_SRC 0x1b0 65 #define OQ_INT_SRC_MSK 0x1b4 66 #define ENT_INT_SRC1 0x1b8 67 #define ENT_INT_SRC1_D2H_FIS_CH0_OFF 0 68 #define ENT_INT_SRC1_D2H_FIS_CH0_MSK (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF) 69 #define ENT_INT_SRC1_D2H_FIS_CH1_OFF 8 70 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF) 71 #define ENT_INT_SRC2 0x1bc 72 #define ENT_INT_SRC3 0x1c0 73 #define ENT_INT_SRC3_WP_DEPTH_OFF 8 74 #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF 9 75 #define ENT_INT_SRC3_RP_DEPTH_OFF 10 76 #define ENT_INT_SRC3_AXI_OFF 11 77 #define ENT_INT_SRC3_FIFO_OFF 12 78 #define ENT_INT_SRC3_LM_OFF 14 79 #define ENT_INT_SRC3_ITC_INT_OFF 15 80 #define ENT_INT_SRC3_ITC_INT_MSK (0x1 << ENT_INT_SRC3_ITC_INT_OFF) 81 #define ENT_INT_SRC3_ABT_OFF 16 82 #define ENT_INT_SRC_MSK1 0x1c4 83 #define ENT_INT_SRC_MSK2 0x1c8 84 #define ENT_INT_SRC_MSK3 0x1cc 85 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF 31 86 #define CHNL_PHYUPDOWN_INT_MSK 0x1d0 87 #define CHNL_ENT_INT_MSK 0x1d4 88 #define HGC_COM_INT_MSK 0x1d8 89 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF) 90 #define SAS_ECC_INTR 0x1e8 91 #define SAS_ECC_INTR_MSK 0x1ec 92 #define HGC_ERR_STAT_EN 0x238 93 #define DLVRY_Q_0_BASE_ADDR_LO 0x260 94 #define DLVRY_Q_0_BASE_ADDR_HI 0x264 95 #define DLVRY_Q_0_DEPTH 0x268 96 #define DLVRY_Q_0_WR_PTR 0x26c 97 #define DLVRY_Q_0_RD_PTR 0x270 98 #define HYPER_STREAM_ID_EN_CFG 0xc80 99 #define OQ0_INT_SRC_MSK 0xc90 100 #define COMPL_Q_0_BASE_ADDR_LO 0x4e0 101 #define COMPL_Q_0_BASE_ADDR_HI 0x4e4 102 #define COMPL_Q_0_DEPTH 0x4e8 103 #define COMPL_Q_0_WR_PTR 0x4ec 104 #define COMPL_Q_0_RD_PTR 0x4f0 105 #define AWQOS_AWCACHE_CFG 0xc84 106 #define ARQOS_ARCACHE_CFG 0xc88 107 108 /* phy registers requiring init */ 109 #define PORT_BASE (0x2000) 110 #define PHY_CFG (PORT_BASE + 0x0) 111 #define HARD_PHY_LINKRATE (PORT_BASE + 0x4) 112 #define PHY_CFG_ENA_OFF 0 113 #define PHY_CFG_ENA_MSK (0x1 << PHY_CFG_ENA_OFF) 114 #define PHY_CFG_DC_OPT_OFF 2 115 #define PHY_CFG_DC_OPT_MSK (0x1 << PHY_CFG_DC_OPT_OFF) 116 #define PROG_PHY_LINK_RATE (PORT_BASE + 0x8) 117 #define PHY_CTRL (PORT_BASE + 0x14) 118 #define PHY_CTRL_RESET_OFF 0 119 #define PHY_CTRL_RESET_MSK (0x1 << PHY_CTRL_RESET_OFF) 120 #define SL_CFG (PORT_BASE + 0x84) 121 #define SL_CONTROL (PORT_BASE + 0x94) 122 #define SL_CONTROL_NOTIFY_EN_OFF 0 123 #define SL_CONTROL_NOTIFY_EN_MSK (0x1 << SL_CONTROL_NOTIFY_EN_OFF) 124 #define SL_CTA_OFF 17 125 #define SL_CTA_MSK (0x1 << SL_CTA_OFF) 126 #define TX_ID_DWORD0 (PORT_BASE + 0x9c) 127 #define TX_ID_DWORD1 (PORT_BASE + 0xa0) 128 #define TX_ID_DWORD2 (PORT_BASE + 0xa4) 129 #define TX_ID_DWORD3 (PORT_BASE + 0xa8) 130 #define TX_ID_DWORD4 (PORT_BASE + 0xaC) 131 #define TX_ID_DWORD5 (PORT_BASE + 0xb0) 132 #define TX_ID_DWORD6 (PORT_BASE + 0xb4) 133 #define TXID_AUTO (PORT_BASE + 0xb8) 134 #define CT3_OFF 1 135 #define CT3_MSK (0x1 << CT3_OFF) 136 #define TX_HARDRST_OFF 2 137 #define TX_HARDRST_MSK (0x1 << TX_HARDRST_OFF) 138 #define RX_IDAF_DWORD0 (PORT_BASE + 0xc4) 139 #define RXOP_CHECK_CFG_H (PORT_BASE + 0xfc) 140 #define SAS_SSP_CON_TIMER_CFG (PORT_BASE + 0x134) 141 #define SAS_SMP_CON_TIMER_CFG (PORT_BASE + 0x138) 142 #define SAS_STP_CON_TIMER_CFG (PORT_BASE + 0x13c) 143 #define CHL_INT0 (PORT_BASE + 0x1b4) 144 #define CHL_INT0_HOTPLUG_TOUT_OFF 0 145 #define CHL_INT0_HOTPLUG_TOUT_MSK (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF) 146 #define CHL_INT0_SL_RX_BCST_ACK_OFF 1 147 #define CHL_INT0_SL_RX_BCST_ACK_MSK (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF) 148 #define CHL_INT0_SL_PHY_ENABLE_OFF 2 149 #define CHL_INT0_SL_PHY_ENABLE_MSK (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF) 150 #define CHL_INT0_NOT_RDY_OFF 4 151 #define CHL_INT0_NOT_RDY_MSK (0x1 << CHL_INT0_NOT_RDY_OFF) 152 #define CHL_INT0_PHY_RDY_OFF 5 153 #define CHL_INT0_PHY_RDY_MSK (0x1 << CHL_INT0_PHY_RDY_OFF) 154 #define CHL_INT1 (PORT_BASE + 0x1b8) 155 #define CHL_INT1_DMAC_TX_ECC_ERR_OFF 15 156 #define CHL_INT1_DMAC_TX_ECC_ERR_MSK (0x1 << CHL_INT1_DMAC_TX_ECC_ERR_OFF) 157 #define CHL_INT1_DMAC_RX_ECC_ERR_OFF 17 158 #define CHL_INT1_DMAC_RX_ECC_ERR_MSK (0x1 << CHL_INT1_DMAC_RX_ECC_ERR_OFF) 159 #define CHL_INT2 (PORT_BASE + 0x1bc) 160 #define CHL_INT0_MSK (PORT_BASE + 0x1c0) 161 #define CHL_INT1_MSK (PORT_BASE + 0x1c4) 162 #define CHL_INT2_MSK (PORT_BASE + 0x1c8) 163 #define CHL_INT_COAL_EN (PORT_BASE + 0x1d0) 164 #define PHY_CTRL_RDY_MSK (PORT_BASE + 0x2b0) 165 #define PHYCTRL_NOT_RDY_MSK (PORT_BASE + 0x2b4) 166 #define PHYCTRL_DWS_RESET_MSK (PORT_BASE + 0x2b8) 167 #define PHYCTRL_PHY_ENA_MSK (PORT_BASE + 0x2bc) 168 #define SL_RX_BCAST_CHK_MSK (PORT_BASE + 0x2c0) 169 #define PHYCTRL_OOB_RESTART_MSK (PORT_BASE + 0x2c4) 170 171 /* HW dma structures */ 172 /* Delivery queue header */ 173 /* dw0 */ 174 #define CMD_HDR_ABORT_FLAG_OFF 0 175 #define CMD_HDR_ABORT_FLAG_MSK (0x3 << CMD_HDR_ABORT_FLAG_OFF) 176 #define CMD_HDR_ABORT_DEVICE_TYPE_OFF 2 177 #define CMD_HDR_ABORT_DEVICE_TYPE_MSK (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF) 178 #define CMD_HDR_RESP_REPORT_OFF 5 179 #define CMD_HDR_RESP_REPORT_MSK (0x1 << CMD_HDR_RESP_REPORT_OFF) 180 #define CMD_HDR_TLR_CTRL_OFF 6 181 #define CMD_HDR_TLR_CTRL_MSK (0x3 << CMD_HDR_TLR_CTRL_OFF) 182 #define CMD_HDR_PORT_OFF 18 183 #define CMD_HDR_PORT_MSK (0xf << CMD_HDR_PORT_OFF) 184 #define CMD_HDR_PRIORITY_OFF 27 185 #define CMD_HDR_PRIORITY_MSK (0x1 << CMD_HDR_PRIORITY_OFF) 186 #define CMD_HDR_CMD_OFF 29 187 #define CMD_HDR_CMD_MSK (0x7 << CMD_HDR_CMD_OFF) 188 /* dw1 */ 189 #define CMD_HDR_UNCON_CMD_OFF 3 190 #define CMD_HDR_DIR_OFF 5 191 #define CMD_HDR_DIR_MSK (0x3 << CMD_HDR_DIR_OFF) 192 #define CMD_HDR_RESET_OFF 7 193 #define CMD_HDR_RESET_MSK (0x1 << CMD_HDR_RESET_OFF) 194 #define CMD_HDR_VDTL_OFF 10 195 #define CMD_HDR_VDTL_MSK (0x1 << CMD_HDR_VDTL_OFF) 196 #define CMD_HDR_FRAME_TYPE_OFF 11 197 #define CMD_HDR_FRAME_TYPE_MSK (0x1f << CMD_HDR_FRAME_TYPE_OFF) 198 #define CMD_HDR_DEV_ID_OFF 16 199 #define CMD_HDR_DEV_ID_MSK (0xffff << CMD_HDR_DEV_ID_OFF) 200 /* dw2 */ 201 #define CMD_HDR_CFL_OFF 0 202 #define CMD_HDR_CFL_MSK (0x1ff << CMD_HDR_CFL_OFF) 203 #define CMD_HDR_NCQ_TAG_OFF 10 204 #define CMD_HDR_NCQ_TAG_MSK (0x1f << CMD_HDR_NCQ_TAG_OFF) 205 #define CMD_HDR_MRFL_OFF 15 206 #define CMD_HDR_MRFL_MSK (0x1ff << CMD_HDR_MRFL_OFF) 207 #define CMD_HDR_SG_MOD_OFF 24 208 #define CMD_HDR_SG_MOD_MSK (0x3 << CMD_HDR_SG_MOD_OFF) 209 /* dw3 */ 210 #define CMD_HDR_IPTT_OFF 0 211 #define CMD_HDR_IPTT_MSK (0xffff << CMD_HDR_IPTT_OFF) 212 /* dw6 */ 213 #define CMD_HDR_DIF_SGL_LEN_OFF 0 214 #define CMD_HDR_DIF_SGL_LEN_MSK (0xffff << CMD_HDR_DIF_SGL_LEN_OFF) 215 #define CMD_HDR_DATA_SGL_LEN_OFF 16 216 #define CMD_HDR_DATA_SGL_LEN_MSK (0xffff << CMD_HDR_DATA_SGL_LEN_OFF) 217 /* dw7 */ 218 #define CMD_HDR_ADDR_MODE_SEL_OFF 15 219 #define CMD_HDR_ADDR_MODE_SEL_MSK (1 << CMD_HDR_ADDR_MODE_SEL_OFF) 220 #define CMD_HDR_ABORT_IPTT_OFF 16 221 #define CMD_HDR_ABORT_IPTT_MSK (0xffff << CMD_HDR_ABORT_IPTT_OFF) 222 223 /* Completion header */ 224 /* dw0 */ 225 #define CMPLT_HDR_CMPLT_OFF 0 226 #define CMPLT_HDR_CMPLT_MSK (0x3 << CMPLT_HDR_CMPLT_OFF) 227 #define CMPLT_HDR_ERROR_PHASE_OFF 2 228 #define CMPLT_HDR_ERROR_PHASE_MSK (0xff << CMPLT_HDR_ERROR_PHASE_OFF) 229 #define CMPLT_HDR_RSPNS_XFRD_OFF 10 230 #define CMPLT_HDR_RSPNS_XFRD_MSK (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF) 231 #define CMPLT_HDR_ERX_OFF 12 232 #define CMPLT_HDR_ERX_MSK (0x1 << CMPLT_HDR_ERX_OFF) 233 #define CMPLT_HDR_ABORT_STAT_OFF 13 234 #define CMPLT_HDR_ABORT_STAT_MSK (0x7 << CMPLT_HDR_ABORT_STAT_OFF) 235 /* abort_stat */ 236 #define STAT_IO_NOT_VALID 0x1 237 #define STAT_IO_NO_DEVICE 0x2 238 #define STAT_IO_COMPLETE 0x3 239 #define STAT_IO_ABORTED 0x4 240 /* dw1 */ 241 #define CMPLT_HDR_IPTT_OFF 0 242 #define CMPLT_HDR_IPTT_MSK (0xffff << CMPLT_HDR_IPTT_OFF) 243 #define CMPLT_HDR_DEV_ID_OFF 16 244 #define CMPLT_HDR_DEV_ID_MSK (0xffff << CMPLT_HDR_DEV_ID_OFF) 245 /* dw3 */ 246 #define CMPLT_HDR_IO_IN_TARGET_OFF 17 247 #define CMPLT_HDR_IO_IN_TARGET_MSK (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF) 248 249 /* ITCT header */ 250 /* qw0 */ 251 #define ITCT_HDR_DEV_TYPE_OFF 0 252 #define ITCT_HDR_DEV_TYPE_MSK (0x3 << ITCT_HDR_DEV_TYPE_OFF) 253 #define ITCT_HDR_VALID_OFF 2 254 #define ITCT_HDR_VALID_MSK (0x1 << ITCT_HDR_VALID_OFF) 255 #define ITCT_HDR_MCR_OFF 5 256 #define ITCT_HDR_MCR_MSK (0xf << ITCT_HDR_MCR_OFF) 257 #define ITCT_HDR_VLN_OFF 9 258 #define ITCT_HDR_VLN_MSK (0xf << ITCT_HDR_VLN_OFF) 259 #define ITCT_HDR_SMP_TIMEOUT_OFF 16 260 #define ITCT_HDR_AWT_CONTINUE_OFF 25 261 #define ITCT_HDR_PORT_ID_OFF 28 262 #define ITCT_HDR_PORT_ID_MSK (0xf << ITCT_HDR_PORT_ID_OFF) 263 /* qw2 */ 264 #define ITCT_HDR_INLT_OFF 0 265 #define ITCT_HDR_INLT_MSK (0xffffULL << ITCT_HDR_INLT_OFF) 266 #define ITCT_HDR_RTOLT_OFF 48 267 #define ITCT_HDR_RTOLT_MSK (0xffffULL << ITCT_HDR_RTOLT_OFF) 268 269 struct hisi_sas_complete_v3_hdr { 270 __le32 dw0; 271 __le32 dw1; 272 __le32 act; 273 __le32 dw3; 274 }; 275 276 struct hisi_sas_err_record_v3 { 277 /* dw0 */ 278 __le32 trans_tx_fail_type; 279 280 /* dw1 */ 281 __le32 trans_rx_fail_type; 282 283 /* dw2 */ 284 __le16 dma_tx_err_type; 285 __le16 sipc_rx_err_type; 286 287 /* dw3 */ 288 __le32 dma_rx_err_type; 289 }; 290 291 #define RX_DATA_LEN_UNDERFLOW_OFF 6 292 #define RX_DATA_LEN_UNDERFLOW_MSK (1 << RX_DATA_LEN_UNDERFLOW_OFF) 293 294 #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096 295 #define HISI_SAS_MSI_COUNT_V3_HW 32 296 297 enum { 298 HISI_SAS_PHY_PHY_UPDOWN, 299 HISI_SAS_PHY_CHNL_INT, 300 HISI_SAS_PHY_INT_NR 301 }; 302 303 #define DIR_NO_DATA 0 304 #define DIR_TO_INI 1 305 #define DIR_TO_DEVICE 2 306 #define DIR_RESERVED 3 307 308 #define CMD_IS_UNCONSTRAINT(cmd) \ 309 ((cmd == ATA_CMD_READ_LOG_EXT) || \ 310 (cmd == ATA_CMD_READ_LOG_DMA_EXT) || \ 311 (cmd == ATA_CMD_DEV_RESET)) 312 313 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off) 314 { 315 void __iomem *regs = hisi_hba->regs + off; 316 317 return readl(regs); 318 } 319 320 static u32 hisi_sas_read32_relaxed(struct hisi_hba *hisi_hba, u32 off) 321 { 322 void __iomem *regs = hisi_hba->regs + off; 323 324 return readl_relaxed(regs); 325 } 326 327 static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val) 328 { 329 void __iomem *regs = hisi_hba->regs + off; 330 331 writel(val, regs); 332 } 333 334 static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no, 335 u32 off, u32 val) 336 { 337 void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off; 338 339 writel(val, regs); 340 } 341 342 static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba, 343 int phy_no, u32 off) 344 { 345 void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off; 346 347 return readl(regs); 348 } 349 350 static void init_reg_v3_hw(struct hisi_hba *hisi_hba) 351 { 352 int i; 353 354 /* Global registers init */ 355 hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 356 (u32)((1ULL << hisi_hba->queue_count) - 1)); 357 hisi_sas_write32(hisi_hba, AXI_USER1, 0x0); 358 hisi_sas_write32(hisi_hba, AXI_USER2, 0x40000060); 359 hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108); 360 hisi_sas_write32(hisi_hba, CFG_1US_TIMER_TRSH, 0xd); 361 hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1); 362 hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1); 363 hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1); 364 hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff); 365 hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff); 366 hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff); 367 hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff); 368 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe); 369 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe); 370 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff); 371 hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0); 372 hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0); 373 hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0); 374 hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfff00c30); 375 hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0); 376 hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0); 377 for (i = 0; i < hisi_hba->queue_count; i++) 378 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0); 379 380 hisi_sas_write32(hisi_hba, AXI_AHB_CLK_CFG, 1); 381 hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1); 382 hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff07fff); 383 384 for (i = 0; i < hisi_hba->n_phy; i++) { 385 hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x801); 386 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff); 387 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff); 388 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff); 389 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000); 390 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff); 391 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0x8ffffbff); 392 hisi_sas_phy_write32(hisi_hba, i, SL_CFG, 0x83f801fc); 393 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0); 394 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0); 395 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0); 396 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0); 397 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0); 398 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x0); 399 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL, 0x199b4fa); 400 hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG, 401 0xa0064); 402 hisi_sas_phy_write32(hisi_hba, i, SAS_STP_CON_TIMER_CFG, 403 0xa0064); 404 } 405 for (i = 0; i < hisi_hba->queue_count; i++) { 406 /* Delivery queue */ 407 hisi_sas_write32(hisi_hba, 408 DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14), 409 upper_32_bits(hisi_hba->cmd_hdr_dma[i])); 410 411 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14), 412 lower_32_bits(hisi_hba->cmd_hdr_dma[i])); 413 414 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14), 415 HISI_SAS_QUEUE_SLOTS); 416 417 /* Completion queue */ 418 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14), 419 upper_32_bits(hisi_hba->complete_hdr_dma[i])); 420 421 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14), 422 lower_32_bits(hisi_hba->complete_hdr_dma[i])); 423 424 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14), 425 HISI_SAS_QUEUE_SLOTS); 426 } 427 428 /* itct */ 429 hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO, 430 lower_32_bits(hisi_hba->itct_dma)); 431 432 hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI, 433 upper_32_bits(hisi_hba->itct_dma)); 434 435 /* iost */ 436 hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO, 437 lower_32_bits(hisi_hba->iost_dma)); 438 439 hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI, 440 upper_32_bits(hisi_hba->iost_dma)); 441 442 /* breakpoint */ 443 hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO, 444 lower_32_bits(hisi_hba->breakpoint_dma)); 445 446 hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI, 447 upper_32_bits(hisi_hba->breakpoint_dma)); 448 449 /* SATA broken msg */ 450 hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO, 451 lower_32_bits(hisi_hba->sata_breakpoint_dma)); 452 453 hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI, 454 upper_32_bits(hisi_hba->sata_breakpoint_dma)); 455 456 /* SATA initial fis */ 457 hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO, 458 lower_32_bits(hisi_hba->initial_fis_dma)); 459 460 hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI, 461 upper_32_bits(hisi_hba->initial_fis_dma)); 462 } 463 464 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 465 { 466 u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG); 467 468 cfg &= ~PHY_CFG_DC_OPT_MSK; 469 cfg |= 1 << PHY_CFG_DC_OPT_OFF; 470 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg); 471 } 472 473 static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 474 { 475 struct sas_identify_frame identify_frame; 476 u32 *identify_buffer; 477 478 memset(&identify_frame, 0, sizeof(identify_frame)); 479 identify_frame.dev_type = SAS_END_DEVICE; 480 identify_frame.frame_type = 0; 481 identify_frame._un1 = 1; 482 identify_frame.initiator_bits = SAS_PROTOCOL_ALL; 483 identify_frame.target_bits = SAS_PROTOCOL_NONE; 484 memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE); 485 memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE); 486 identify_frame.phy_id = phy_no; 487 identify_buffer = (u32 *)(&identify_frame); 488 489 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0, 490 __swab32(identify_buffer[0])); 491 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1, 492 __swab32(identify_buffer[1])); 493 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2, 494 __swab32(identify_buffer[2])); 495 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3, 496 __swab32(identify_buffer[3])); 497 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4, 498 __swab32(identify_buffer[4])); 499 hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5, 500 __swab32(identify_buffer[5])); 501 } 502 503 static void setup_itct_v3_hw(struct hisi_hba *hisi_hba, 504 struct hisi_sas_device *sas_dev) 505 { 506 struct domain_device *device = sas_dev->sas_device; 507 struct device *dev = hisi_hba->dev; 508 u64 qw0, device_id = sas_dev->device_id; 509 struct hisi_sas_itct *itct = &hisi_hba->itct[device_id]; 510 struct domain_device *parent_dev = device->parent; 511 struct asd_sas_port *sas_port = device->port; 512 struct hisi_sas_port *port = to_hisi_sas_port(sas_port); 513 514 memset(itct, 0, sizeof(*itct)); 515 516 /* qw0 */ 517 qw0 = 0; 518 switch (sas_dev->dev_type) { 519 case SAS_END_DEVICE: 520 case SAS_EDGE_EXPANDER_DEVICE: 521 case SAS_FANOUT_EXPANDER_DEVICE: 522 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF; 523 break; 524 case SAS_SATA_DEV: 525 case SAS_SATA_PENDING: 526 if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type)) 527 qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF; 528 else 529 qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF; 530 break; 531 default: 532 dev_warn(dev, "setup itct: unsupported dev type (%d)\n", 533 sas_dev->dev_type); 534 } 535 536 qw0 |= ((1 << ITCT_HDR_VALID_OFF) | 537 (device->linkrate << ITCT_HDR_MCR_OFF) | 538 (1 << ITCT_HDR_VLN_OFF) | 539 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) | 540 (1 << ITCT_HDR_AWT_CONTINUE_OFF) | 541 (port->id << ITCT_HDR_PORT_ID_OFF)); 542 itct->qw0 = cpu_to_le64(qw0); 543 544 /* qw1 */ 545 memcpy(&itct->sas_addr, device->sas_addr, SAS_ADDR_SIZE); 546 itct->sas_addr = __swab64(itct->sas_addr); 547 548 /* qw2 */ 549 if (!dev_is_sata(device)) 550 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) | 551 (0x1ULL << ITCT_HDR_RTOLT_OFF)); 552 } 553 554 static void free_device_v3_hw(struct hisi_hba *hisi_hba, 555 struct hisi_sas_device *sas_dev) 556 { 557 u64 dev_id = sas_dev->device_id; 558 struct device *dev = hisi_hba->dev; 559 struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id]; 560 u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3); 561 562 /* clear the itct interrupt state */ 563 if (ENT_INT_SRC3_ITC_INT_MSK & reg_val) 564 hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 565 ENT_INT_SRC3_ITC_INT_MSK); 566 567 /* clear the itct table*/ 568 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR); 569 reg_val |= ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK); 570 hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val); 571 572 udelay(10); 573 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3); 574 if (ENT_INT_SRC3_ITC_INT_MSK & reg_val) { 575 dev_dbg(dev, "got clear ITCT done interrupt\n"); 576 577 /* invalid the itct state*/ 578 memset(itct, 0, sizeof(struct hisi_sas_itct)); 579 hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 580 ENT_INT_SRC3_ITC_INT_MSK); 581 hisi_hba->devices[dev_id].dev_type = SAS_PHY_UNUSED; 582 hisi_hba->devices[dev_id].dev_status = HISI_SAS_DEV_NORMAL; 583 584 /* clear the itct */ 585 hisi_sas_write32(hisi_hba, ITCT_CLR, 0); 586 dev_dbg(dev, "clear ITCT ok\n"); 587 } 588 } 589 590 static void dereg_device_v3_hw(struct hisi_hba *hisi_hba, 591 struct domain_device *device) 592 { 593 struct hisi_sas_slot *slot, *slot2; 594 struct hisi_sas_device *sas_dev = device->lldd_dev; 595 u32 cfg_abt_set_query_iptt; 596 597 cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba, 598 CFG_ABT_SET_QUERY_IPTT); 599 list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) { 600 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK; 601 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) | 602 (slot->idx << CFG_SET_ABORTED_IPTT_OFF); 603 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT, 604 cfg_abt_set_query_iptt); 605 } 606 cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF); 607 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT, 608 cfg_abt_set_query_iptt); 609 hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE, 610 1 << CFG_ABT_SET_IPTT_DONE_OFF); 611 } 612 613 static int hw_init_v3_hw(struct hisi_hba *hisi_hba) 614 { 615 init_reg_v3_hw(hisi_hba); 616 617 return 0; 618 } 619 620 static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 621 { 622 u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG); 623 624 cfg |= PHY_CFG_ENA_MSK; 625 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg); 626 } 627 628 static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 629 { 630 u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG); 631 632 cfg &= ~PHY_CFG_ENA_MSK; 633 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg); 634 } 635 636 static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 637 { 638 config_id_frame_v3_hw(hisi_hba, phy_no); 639 config_phy_opt_mode_v3_hw(hisi_hba, phy_no); 640 enable_phy_v3_hw(hisi_hba, phy_no); 641 } 642 643 static void stop_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 644 { 645 disable_phy_v3_hw(hisi_hba, phy_no); 646 } 647 648 static void start_phys_v3_hw(struct hisi_hba *hisi_hba) 649 { 650 int i; 651 652 for (i = 0; i < hisi_hba->n_phy; i++) 653 start_phy_v3_hw(hisi_hba, i); 654 } 655 656 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 657 { 658 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; 659 u32 txid_auto; 660 661 stop_phy_v3_hw(hisi_hba, phy_no); 662 if (phy->identify.device_type == SAS_END_DEVICE) { 663 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO); 664 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO, 665 txid_auto | TX_HARDRST_MSK); 666 } 667 msleep(100); 668 start_phy_v3_hw(hisi_hba, phy_no); 669 } 670 671 enum sas_linkrate phy_get_max_linkrate_v3_hw(void) 672 { 673 return SAS_LINK_RATE_12_0_GBPS; 674 } 675 676 static void phys_init_v3_hw(struct hisi_hba *hisi_hba) 677 { 678 start_phys_v3_hw(hisi_hba); 679 } 680 681 static void sl_notify_v3_hw(struct hisi_hba *hisi_hba, int phy_no) 682 { 683 u32 sl_control; 684 685 sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL); 686 sl_control |= SL_CONTROL_NOTIFY_EN_MSK; 687 hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control); 688 msleep(1); 689 sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL); 690 sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK; 691 hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control); 692 } 693 694 static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id) 695 { 696 int i, bitmap = 0; 697 u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA); 698 699 for (i = 0; i < hisi_hba->n_phy; i++) 700 if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id) 701 bitmap |= 1 << i; 702 703 return bitmap; 704 } 705 706 /** 707 * The callpath to this function and upto writing the write 708 * queue pointer should be safe from interruption. 709 */ 710 static int 711 get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq) 712 { 713 struct device *dev = hisi_hba->dev; 714 int queue = dq->id; 715 u32 r, w; 716 717 w = dq->wr_point; 718 r = hisi_sas_read32_relaxed(hisi_hba, 719 DLVRY_Q_0_RD_PTR + (queue * 0x14)); 720 if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) { 721 dev_warn(dev, "full queue=%d r=%d w=%d\n\n", 722 queue, r, w); 723 return -EAGAIN; 724 } 725 726 return 0; 727 } 728 729 static void start_delivery_v3_hw(struct hisi_sas_dq *dq) 730 { 731 struct hisi_hba *hisi_hba = dq->hisi_hba; 732 int dlvry_queue = dq->slot_prep->dlvry_queue; 733 int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot; 734 735 dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS; 736 hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), 737 dq->wr_point); 738 } 739 740 static int prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba, 741 struct hisi_sas_slot *slot, 742 struct hisi_sas_cmd_hdr *hdr, 743 struct scatterlist *scatter, 744 int n_elem) 745 { 746 struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot); 747 struct device *dev = hisi_hba->dev; 748 struct scatterlist *sg; 749 int i; 750 751 if (n_elem > HISI_SAS_SGE_PAGE_CNT) { 752 dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT", 753 n_elem); 754 return -EINVAL; 755 } 756 757 for_each_sg(scatter, sg, n_elem, i) { 758 struct hisi_sas_sge *entry = &sge_page->sge[i]; 759 760 entry->addr = cpu_to_le64(sg_dma_address(sg)); 761 entry->page_ctrl_0 = entry->page_ctrl_1 = 0; 762 entry->data_len = cpu_to_le32(sg_dma_len(sg)); 763 entry->data_off = 0; 764 } 765 766 hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot)); 767 768 hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF); 769 770 return 0; 771 } 772 773 static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba, 774 struct hisi_sas_slot *slot, int is_tmf, 775 struct hisi_sas_tmf_task *tmf) 776 { 777 struct sas_task *task = slot->task; 778 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr; 779 struct domain_device *device = task->dev; 780 struct hisi_sas_device *sas_dev = device->lldd_dev; 781 struct hisi_sas_port *port = slot->port; 782 struct sas_ssp_task *ssp_task = &task->ssp_task; 783 struct scsi_cmnd *scsi_cmnd = ssp_task->cmd; 784 int has_data = 0, rc, priority = is_tmf; 785 u8 *buf_cmd; 786 u32 dw1 = 0, dw2 = 0; 787 788 hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) | 789 (2 << CMD_HDR_TLR_CTRL_OFF) | 790 (port->id << CMD_HDR_PORT_OFF) | 791 (priority << CMD_HDR_PRIORITY_OFF) | 792 (1 << CMD_HDR_CMD_OFF)); /* ssp */ 793 794 dw1 = 1 << CMD_HDR_VDTL_OFF; 795 if (is_tmf) { 796 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF; 797 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF; 798 } else { 799 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF; 800 switch (scsi_cmnd->sc_data_direction) { 801 case DMA_TO_DEVICE: 802 has_data = 1; 803 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF; 804 break; 805 case DMA_FROM_DEVICE: 806 has_data = 1; 807 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF; 808 break; 809 default: 810 dw1 &= ~CMD_HDR_DIR_MSK; 811 } 812 } 813 814 /* map itct entry */ 815 dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF; 816 hdr->dw1 = cpu_to_le32(dw1); 817 818 dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr) 819 + 3) / 4) << CMD_HDR_CFL_OFF) | 820 ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) | 821 (2 << CMD_HDR_SG_MOD_OFF); 822 hdr->dw2 = cpu_to_le32(dw2); 823 hdr->transfer_tags = cpu_to_le32(slot->idx); 824 825 if (has_data) { 826 rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter, 827 slot->n_elem); 828 if (rc) 829 return rc; 830 } 831 832 hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len); 833 hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot)); 834 hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot)); 835 836 buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) + 837 sizeof(struct ssp_frame_hdr); 838 839 memcpy(buf_cmd, &task->ssp_task.LUN, 8); 840 if (!is_tmf) { 841 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3); 842 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len); 843 } else { 844 buf_cmd[10] = tmf->tmf; 845 switch (tmf->tmf) { 846 case TMF_ABORT_TASK: 847 case TMF_QUERY_TASK: 848 buf_cmd[12] = 849 (tmf->tag_of_task_to_be_managed >> 8) & 0xff; 850 buf_cmd[13] = 851 tmf->tag_of_task_to_be_managed & 0xff; 852 break; 853 default: 854 break; 855 } 856 } 857 858 return 0; 859 } 860 861 static int prep_smp_v3_hw(struct hisi_hba *hisi_hba, 862 struct hisi_sas_slot *slot) 863 { 864 struct sas_task *task = slot->task; 865 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr; 866 struct domain_device *device = task->dev; 867 struct device *dev = hisi_hba->dev; 868 struct hisi_sas_port *port = slot->port; 869 struct scatterlist *sg_req, *sg_resp; 870 struct hisi_sas_device *sas_dev = device->lldd_dev; 871 dma_addr_t req_dma_addr; 872 unsigned int req_len, resp_len; 873 int elem, rc; 874 875 /* 876 * DMA-map SMP request, response buffers 877 */ 878 /* req */ 879 sg_req = &task->smp_task.smp_req; 880 elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE); 881 if (!elem) 882 return -ENOMEM; 883 req_len = sg_dma_len(sg_req); 884 req_dma_addr = sg_dma_address(sg_req); 885 886 /* resp */ 887 sg_resp = &task->smp_task.smp_resp; 888 elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE); 889 if (!elem) { 890 rc = -ENOMEM; 891 goto err_out_req; 892 } 893 resp_len = sg_dma_len(sg_resp); 894 if ((req_len & 0x3) || (resp_len & 0x3)) { 895 rc = -EINVAL; 896 goto err_out_resp; 897 } 898 899 /* create header */ 900 /* dw0 */ 901 hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) | 902 (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */ 903 (2 << CMD_HDR_CMD_OFF)); /* smp */ 904 905 /* map itct entry */ 906 hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) | 907 (1 << CMD_HDR_FRAME_TYPE_OFF) | 908 (DIR_NO_DATA << CMD_HDR_DIR_OFF)); 909 910 /* dw2 */ 911 hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) | 912 (HISI_SAS_MAX_SMP_RESP_SZ / 4 << 913 CMD_HDR_MRFL_OFF)); 914 915 hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF); 916 917 hdr->cmd_table_addr = cpu_to_le64(req_dma_addr); 918 hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot)); 919 920 return 0; 921 922 err_out_resp: 923 dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1, 924 DMA_FROM_DEVICE); 925 err_out_req: 926 dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1, 927 DMA_TO_DEVICE); 928 return rc; 929 } 930 931 static int get_ncq_tag_v3_hw(struct sas_task *task, u32 *tag) 932 { 933 struct ata_queued_cmd *qc = task->uldd_task; 934 935 if (qc) { 936 if (qc->tf.command == ATA_CMD_FPDMA_WRITE || 937 qc->tf.command == ATA_CMD_FPDMA_READ) { 938 *tag = qc->tag; 939 return 1; 940 } 941 } 942 return 0; 943 } 944 945 static int prep_ata_v3_hw(struct hisi_hba *hisi_hba, 946 struct hisi_sas_slot *slot) 947 { 948 struct sas_task *task = slot->task; 949 struct domain_device *device = task->dev; 950 struct domain_device *parent_dev = device->parent; 951 struct hisi_sas_device *sas_dev = device->lldd_dev; 952 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr; 953 struct asd_sas_port *sas_port = device->port; 954 struct hisi_sas_port *port = to_hisi_sas_port(sas_port); 955 u8 *buf_cmd; 956 int has_data = 0, rc = 0, hdr_tag = 0; 957 u32 dw1 = 0, dw2 = 0; 958 959 hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF); 960 if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type)) 961 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF); 962 else 963 hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF); 964 965 switch (task->data_dir) { 966 case DMA_TO_DEVICE: 967 has_data = 1; 968 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF; 969 break; 970 case DMA_FROM_DEVICE: 971 has_data = 1; 972 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF; 973 break; 974 default: 975 dw1 &= ~CMD_HDR_DIR_MSK; 976 } 977 978 if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) && 979 (task->ata_task.fis.control & ATA_SRST)) 980 dw1 |= 1 << CMD_HDR_RESET_OFF; 981 982 dw1 |= (hisi_sas_get_ata_protocol( 983 task->ata_task.fis.command, task->data_dir)) 984 << CMD_HDR_FRAME_TYPE_OFF; 985 dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF; 986 987 if (CMD_IS_UNCONSTRAINT(task->ata_task.fis.command)) 988 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF; 989 990 hdr->dw1 = cpu_to_le32(dw1); 991 992 /* dw2 */ 993 if (task->ata_task.use_ncq && get_ncq_tag_v3_hw(task, &hdr_tag)) { 994 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3); 995 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF; 996 } 997 998 dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF | 999 2 << CMD_HDR_SG_MOD_OFF; 1000 hdr->dw2 = cpu_to_le32(dw2); 1001 1002 /* dw3 */ 1003 hdr->transfer_tags = cpu_to_le32(slot->idx); 1004 1005 if (has_data) { 1006 rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter, 1007 slot->n_elem); 1008 if (rc) 1009 return rc; 1010 } 1011 1012 hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len); 1013 hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot)); 1014 hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot)); 1015 1016 buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot); 1017 1018 if (likely(!task->ata_task.device_control_reg_update)) 1019 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */ 1020 /* fill in command FIS */ 1021 memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis)); 1022 1023 return 0; 1024 } 1025 1026 static int prep_abort_v3_hw(struct hisi_hba *hisi_hba, 1027 struct hisi_sas_slot *slot, 1028 int device_id, int abort_flag, int tag_to_abort) 1029 { 1030 struct sas_task *task = slot->task; 1031 struct domain_device *dev = task->dev; 1032 struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr; 1033 struct hisi_sas_port *port = slot->port; 1034 1035 /* dw0 */ 1036 hdr->dw0 = cpu_to_le32((5 << CMD_HDR_CMD_OFF) | /*abort*/ 1037 (port->id << CMD_HDR_PORT_OFF) | 1038 ((dev_is_sata(dev) ? 1:0) 1039 << CMD_HDR_ABORT_DEVICE_TYPE_OFF) | 1040 (abort_flag 1041 << CMD_HDR_ABORT_FLAG_OFF)); 1042 1043 /* dw1 */ 1044 hdr->dw1 = cpu_to_le32(device_id 1045 << CMD_HDR_DEV_ID_OFF); 1046 1047 /* dw7 */ 1048 hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF); 1049 hdr->transfer_tags = cpu_to_le32(slot->idx); 1050 1051 return 0; 1052 } 1053 1054 static int phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba) 1055 { 1056 int i, res = 0; 1057 u32 context, port_id, link_rate, hard_phy_linkrate; 1058 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; 1059 struct asd_sas_phy *sas_phy = &phy->sas_phy; 1060 struct device *dev = hisi_hba->dev; 1061 1062 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1); 1063 1064 port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA); 1065 port_id = (port_id >> (4 * phy_no)) & 0xf; 1066 link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE); 1067 link_rate = (link_rate >> (phy_no * 4)) & 0xf; 1068 1069 if (port_id == 0xf) { 1070 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no); 1071 res = IRQ_NONE; 1072 goto end; 1073 } 1074 sas_phy->linkrate = link_rate; 1075 hard_phy_linkrate = hisi_sas_phy_read32(hisi_hba, phy_no, 1076 HARD_PHY_LINKRATE); 1077 phy->maximum_linkrate = hard_phy_linkrate & 0xf; 1078 phy->minimum_linkrate = (hard_phy_linkrate >> 4) & 0xf; 1079 phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA); 1080 1081 /* Check for SATA dev */ 1082 context = hisi_sas_read32(hisi_hba, PHY_CONTEXT); 1083 if (context & (1 << phy_no)) { 1084 struct hisi_sas_initial_fis *initial_fis; 1085 struct dev_to_host_fis *fis; 1086 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0}; 1087 1088 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate); 1089 initial_fis = &hisi_hba->initial_fis[phy_no]; 1090 fis = &initial_fis->fis; 1091 sas_phy->oob_mode = SATA_OOB_MODE; 1092 attached_sas_addr[0] = 0x50; 1093 attached_sas_addr[7] = phy_no; 1094 memcpy(sas_phy->attached_sas_addr, 1095 attached_sas_addr, 1096 SAS_ADDR_SIZE); 1097 memcpy(sas_phy->frame_rcvd, fis, 1098 sizeof(struct dev_to_host_fis)); 1099 phy->phy_type |= PORT_TYPE_SATA; 1100 phy->identify.device_type = SAS_SATA_DEV; 1101 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis); 1102 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA; 1103 } else { 1104 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd; 1105 struct sas_identify_frame *id = 1106 (struct sas_identify_frame *)frame_rcvd; 1107 1108 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate); 1109 for (i = 0; i < 6; i++) { 1110 u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no, 1111 RX_IDAF_DWORD0 + (i * 4)); 1112 frame_rcvd[i] = __swab32(idaf); 1113 } 1114 sas_phy->oob_mode = SAS_OOB_MODE; 1115 memcpy(sas_phy->attached_sas_addr, 1116 &id->sas_addr, 1117 SAS_ADDR_SIZE); 1118 phy->phy_type |= PORT_TYPE_SAS; 1119 phy->identify.device_type = id->dev_type; 1120 phy->frame_rcvd_size = sizeof(struct sas_identify_frame); 1121 if (phy->identify.device_type == SAS_END_DEVICE) 1122 phy->identify.target_port_protocols = 1123 SAS_PROTOCOL_SSP; 1124 else if (phy->identify.device_type != SAS_PHY_UNUSED) 1125 phy->identify.target_port_protocols = 1126 SAS_PROTOCOL_SMP; 1127 } 1128 1129 phy->port_id = port_id; 1130 phy->phy_attached = 1; 1131 queue_work(hisi_hba->wq, &phy->phyup_ws); 1132 1133 end: 1134 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, 1135 CHL_INT0_SL_PHY_ENABLE_MSK); 1136 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0); 1137 1138 return res; 1139 } 1140 1141 static int phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba) 1142 { 1143 int res = 0; 1144 u32 phy_state, sl_ctrl, txid_auto; 1145 struct device *dev = hisi_hba->dev; 1146 1147 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1); 1148 1149 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE); 1150 dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state); 1151 hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0); 1152 1153 sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL); 1154 hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, 1155 sl_ctrl&(~SL_CTA_MSK)); 1156 1157 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO); 1158 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO, 1159 txid_auto | CT3_MSK); 1160 1161 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK); 1162 hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0); 1163 1164 return res; 1165 } 1166 1167 static void phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba) 1168 { 1169 struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; 1170 struct asd_sas_phy *sas_phy = &phy->sas_phy; 1171 struct sas_ha_struct *sas_ha = &hisi_hba->sha; 1172 1173 hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1); 1174 sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD); 1175 hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, 1176 CHL_INT0_SL_RX_BCST_ACK_MSK); 1177 hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0); 1178 } 1179 1180 static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p) 1181 { 1182 struct hisi_hba *hisi_hba = p; 1183 u32 irq_msk; 1184 int phy_no = 0; 1185 irqreturn_t res = IRQ_NONE; 1186 1187 irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS) 1188 & 0x11111111; 1189 while (irq_msk) { 1190 if (irq_msk & 1) { 1191 u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, 1192 CHL_INT0); 1193 u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE); 1194 int rdy = phy_state & (1 << phy_no); 1195 1196 if (rdy) { 1197 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK) 1198 /* phy up */ 1199 if (phy_up_v3_hw(phy_no, hisi_hba) 1200 == IRQ_HANDLED) 1201 res = IRQ_HANDLED; 1202 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK) 1203 /* phy bcast */ 1204 phy_bcast_v3_hw(phy_no, hisi_hba); 1205 } else { 1206 if (irq_value & CHL_INT0_NOT_RDY_MSK) 1207 /* phy down */ 1208 if (phy_down_v3_hw(phy_no, hisi_hba) 1209 == IRQ_HANDLED) 1210 res = IRQ_HANDLED; 1211 } 1212 } 1213 irq_msk >>= 4; 1214 phy_no++; 1215 } 1216 1217 return res; 1218 } 1219 1220 static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p) 1221 { 1222 struct hisi_hba *hisi_hba = p; 1223 struct device *dev = hisi_hba->dev; 1224 u32 ent_msk, ent_tmp, irq_msk; 1225 int phy_no = 0; 1226 1227 ent_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3); 1228 ent_tmp = ent_msk; 1229 ent_msk |= ENT_INT_SRC_MSK3_ENT95_MSK_MSK; 1230 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_msk); 1231 1232 irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS) 1233 & 0xeeeeeeee; 1234 1235 while (irq_msk) { 1236 u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no, 1237 CHL_INT0); 1238 u32 irq_value1 = hisi_sas_phy_read32(hisi_hba, phy_no, 1239 CHL_INT1); 1240 u32 irq_value2 = hisi_sas_phy_read32(hisi_hba, phy_no, 1241 CHL_INT2); 1242 1243 if ((irq_msk & (4 << (phy_no * 4))) && 1244 irq_value1) { 1245 if (irq_value1 & (CHL_INT1_DMAC_RX_ECC_ERR_MSK | 1246 CHL_INT1_DMAC_TX_ECC_ERR_MSK)) 1247 panic("%s: DMAC RX/TX ecc bad error! (0x%x)", 1248 dev_name(dev), irq_value1); 1249 1250 hisi_sas_phy_write32(hisi_hba, phy_no, 1251 CHL_INT1, irq_value1); 1252 } 1253 1254 if (irq_msk & (8 << (phy_no * 4)) && irq_value2) 1255 hisi_sas_phy_write32(hisi_hba, phy_no, 1256 CHL_INT2, irq_value2); 1257 1258 1259 if (irq_msk & (2 << (phy_no * 4)) && irq_value0) { 1260 hisi_sas_phy_write32(hisi_hba, phy_no, 1261 CHL_INT0, irq_value0 1262 & (~CHL_INT0_HOTPLUG_TOUT_MSK) 1263 & (~CHL_INT0_SL_PHY_ENABLE_MSK) 1264 & (~CHL_INT0_NOT_RDY_MSK)); 1265 } 1266 irq_msk &= ~(0xe << (phy_no * 4)); 1267 phy_no++; 1268 } 1269 1270 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_tmp); 1271 1272 return IRQ_HANDLED; 1273 } 1274 1275 static void 1276 slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task, 1277 struct hisi_sas_slot *slot) 1278 { 1279 struct task_status_struct *ts = &task->task_status; 1280 struct hisi_sas_complete_v3_hdr *complete_queue = 1281 hisi_hba->complete_hdr[slot->cmplt_queue]; 1282 struct hisi_sas_complete_v3_hdr *complete_hdr = 1283 &complete_queue[slot->cmplt_queue_slot]; 1284 struct hisi_sas_err_record_v3 *record = 1285 hisi_sas_status_buf_addr_mem(slot); 1286 u32 dma_rx_err_type = record->dma_rx_err_type; 1287 u32 trans_tx_fail_type = record->trans_tx_fail_type; 1288 1289 switch (task->task_proto) { 1290 case SAS_PROTOCOL_SSP: 1291 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) { 1292 ts->residual = trans_tx_fail_type; 1293 ts->stat = SAS_DATA_UNDERRUN; 1294 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) { 1295 ts->stat = SAS_QUEUE_FULL; 1296 slot->abort = 1; 1297 } else { 1298 ts->stat = SAS_OPEN_REJECT; 1299 ts->open_rej_reason = SAS_OREJ_RSVD_RETRY; 1300 } 1301 break; 1302 case SAS_PROTOCOL_SATA: 1303 case SAS_PROTOCOL_STP: 1304 case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: 1305 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) { 1306 ts->residual = trans_tx_fail_type; 1307 ts->stat = SAS_DATA_UNDERRUN; 1308 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) { 1309 ts->stat = SAS_PHY_DOWN; 1310 slot->abort = 1; 1311 } else { 1312 ts->stat = SAS_OPEN_REJECT; 1313 ts->open_rej_reason = SAS_OREJ_RSVD_RETRY; 1314 } 1315 hisi_sas_sata_done(task, slot); 1316 break; 1317 case SAS_PROTOCOL_SMP: 1318 ts->stat = SAM_STAT_CHECK_CONDITION; 1319 break; 1320 default: 1321 break; 1322 } 1323 } 1324 1325 static int 1326 slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot) 1327 { 1328 struct sas_task *task = slot->task; 1329 struct hisi_sas_device *sas_dev; 1330 struct device *dev = hisi_hba->dev; 1331 struct task_status_struct *ts; 1332 struct domain_device *device; 1333 enum exec_status sts; 1334 struct hisi_sas_complete_v3_hdr *complete_queue = 1335 hisi_hba->complete_hdr[slot->cmplt_queue]; 1336 struct hisi_sas_complete_v3_hdr *complete_hdr = 1337 &complete_queue[slot->cmplt_queue_slot]; 1338 int aborted; 1339 unsigned long flags; 1340 1341 if (unlikely(!task || !task->lldd_task || !task->dev)) 1342 return -EINVAL; 1343 1344 ts = &task->task_status; 1345 device = task->dev; 1346 sas_dev = device->lldd_dev; 1347 1348 spin_lock_irqsave(&task->task_state_lock, flags); 1349 aborted = task->task_state_flags & SAS_TASK_STATE_ABORTED; 1350 task->task_state_flags &= 1351 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR); 1352 spin_unlock_irqrestore(&task->task_state_lock, flags); 1353 1354 memset(ts, 0, sizeof(*ts)); 1355 ts->resp = SAS_TASK_COMPLETE; 1356 if (unlikely(aborted)) { 1357 ts->stat = SAS_ABORTED_TASK; 1358 hisi_sas_slot_task_free(hisi_hba, task, slot); 1359 return -1; 1360 } 1361 1362 if (unlikely(!sas_dev)) { 1363 dev_dbg(dev, "slot complete: port has not device\n"); 1364 ts->stat = SAS_PHY_DOWN; 1365 goto out; 1366 } 1367 1368 /* 1369 * Use SAS+TMF status codes 1370 */ 1371 switch ((complete_hdr->dw0 & CMPLT_HDR_ABORT_STAT_MSK) 1372 >> CMPLT_HDR_ABORT_STAT_OFF) { 1373 case STAT_IO_ABORTED: 1374 /* this IO has been aborted by abort command */ 1375 ts->stat = SAS_ABORTED_TASK; 1376 goto out; 1377 case STAT_IO_COMPLETE: 1378 /* internal abort command complete */ 1379 ts->stat = TMF_RESP_FUNC_SUCC; 1380 goto out; 1381 case STAT_IO_NO_DEVICE: 1382 ts->stat = TMF_RESP_FUNC_COMPLETE; 1383 goto out; 1384 case STAT_IO_NOT_VALID: 1385 /* 1386 * abort single IO, the controller can't find the IO 1387 */ 1388 ts->stat = TMF_RESP_FUNC_FAILED; 1389 goto out; 1390 default: 1391 break; 1392 } 1393 1394 /* check for erroneous completion */ 1395 if ((complete_hdr->dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) { 1396 slot_err_v3_hw(hisi_hba, task, slot); 1397 if (unlikely(slot->abort)) 1398 return ts->stat; 1399 goto out; 1400 } 1401 1402 switch (task->task_proto) { 1403 case SAS_PROTOCOL_SSP: { 1404 struct ssp_response_iu *iu = 1405 hisi_sas_status_buf_addr_mem(slot) + 1406 sizeof(struct hisi_sas_err_record); 1407 1408 sas_ssp_task_response(dev, task, iu); 1409 break; 1410 } 1411 case SAS_PROTOCOL_SMP: { 1412 struct scatterlist *sg_resp = &task->smp_task.smp_resp; 1413 void *to; 1414 1415 ts->stat = SAM_STAT_GOOD; 1416 to = kmap_atomic(sg_page(sg_resp)); 1417 1418 dma_unmap_sg(dev, &task->smp_task.smp_resp, 1, 1419 DMA_FROM_DEVICE); 1420 dma_unmap_sg(dev, &task->smp_task.smp_req, 1, 1421 DMA_TO_DEVICE); 1422 memcpy(to + sg_resp->offset, 1423 hisi_sas_status_buf_addr_mem(slot) + 1424 sizeof(struct hisi_sas_err_record), 1425 sg_dma_len(sg_resp)); 1426 kunmap_atomic(to); 1427 break; 1428 } 1429 case SAS_PROTOCOL_SATA: 1430 case SAS_PROTOCOL_STP: 1431 case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP: 1432 ts->stat = SAM_STAT_GOOD; 1433 hisi_sas_sata_done(task, slot); 1434 break; 1435 default: 1436 ts->stat = SAM_STAT_CHECK_CONDITION; 1437 break; 1438 } 1439 1440 if (!slot->port->port_attached) { 1441 dev_err(dev, "slot complete: port %d has removed\n", 1442 slot->port->sas_port.id); 1443 ts->stat = SAS_PHY_DOWN; 1444 } 1445 1446 out: 1447 spin_lock_irqsave(&task->task_state_lock, flags); 1448 task->task_state_flags |= SAS_TASK_STATE_DONE; 1449 spin_unlock_irqrestore(&task->task_state_lock, flags); 1450 spin_lock_irqsave(&hisi_hba->lock, flags); 1451 hisi_sas_slot_task_free(hisi_hba, task, slot); 1452 spin_unlock_irqrestore(&hisi_hba->lock, flags); 1453 sts = ts->stat; 1454 1455 if (task->task_done) 1456 task->task_done(task); 1457 1458 return sts; 1459 } 1460 1461 static void cq_tasklet_v3_hw(unsigned long val) 1462 { 1463 struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val; 1464 struct hisi_hba *hisi_hba = cq->hisi_hba; 1465 struct hisi_sas_slot *slot; 1466 struct hisi_sas_itct *itct; 1467 struct hisi_sas_complete_v3_hdr *complete_queue; 1468 u32 rd_point = cq->rd_point, wr_point, dev_id; 1469 int queue = cq->id; 1470 struct hisi_sas_dq *dq = &hisi_hba->dq[queue]; 1471 1472 complete_queue = hisi_hba->complete_hdr[queue]; 1473 1474 spin_lock(&dq->lock); 1475 wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR + 1476 (0x14 * queue)); 1477 1478 while (rd_point != wr_point) { 1479 struct hisi_sas_complete_v3_hdr *complete_hdr; 1480 int iptt; 1481 1482 complete_hdr = &complete_queue[rd_point]; 1483 1484 /* Check for NCQ completion */ 1485 if (complete_hdr->act) { 1486 u32 act_tmp = complete_hdr->act; 1487 int ncq_tag_count = ffs(act_tmp); 1488 1489 dev_id = (complete_hdr->dw1 & CMPLT_HDR_DEV_ID_MSK) >> 1490 CMPLT_HDR_DEV_ID_OFF; 1491 itct = &hisi_hba->itct[dev_id]; 1492 1493 /* The NCQ tags are held in the itct header */ 1494 while (ncq_tag_count) { 1495 __le64 *ncq_tag = &itct->qw4_15[0]; 1496 1497 ncq_tag_count -= 1; 1498 iptt = (ncq_tag[ncq_tag_count / 5] 1499 >> (ncq_tag_count % 5) * 12) & 0xfff; 1500 1501 slot = &hisi_hba->slot_info[iptt]; 1502 slot->cmplt_queue_slot = rd_point; 1503 slot->cmplt_queue = queue; 1504 slot_complete_v3_hw(hisi_hba, slot); 1505 1506 act_tmp &= ~(1 << ncq_tag_count); 1507 ncq_tag_count = ffs(act_tmp); 1508 } 1509 } else { 1510 iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK; 1511 slot = &hisi_hba->slot_info[iptt]; 1512 slot->cmplt_queue_slot = rd_point; 1513 slot->cmplt_queue = queue; 1514 slot_complete_v3_hw(hisi_hba, slot); 1515 } 1516 1517 if (++rd_point >= HISI_SAS_QUEUE_SLOTS) 1518 rd_point = 0; 1519 } 1520 1521 /* update rd_point */ 1522 cq->rd_point = rd_point; 1523 hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point); 1524 spin_unlock(&dq->lock); 1525 } 1526 1527 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p) 1528 { 1529 struct hisi_sas_cq *cq = p; 1530 struct hisi_hba *hisi_hba = cq->hisi_hba; 1531 int queue = cq->id; 1532 1533 hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue); 1534 1535 tasklet_schedule(&cq->tasklet); 1536 1537 return IRQ_HANDLED; 1538 } 1539 1540 static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba) 1541 { 1542 struct device *dev = hisi_hba->dev; 1543 struct pci_dev *pdev = hisi_hba->pci_dev; 1544 int vectors, rc; 1545 int i, k; 1546 int max_msi = HISI_SAS_MSI_COUNT_V3_HW; 1547 1548 vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, 1, 1549 max_msi, PCI_IRQ_MSI); 1550 if (vectors < max_msi) { 1551 dev_err(dev, "could not allocate all msi (%d)\n", vectors); 1552 return -ENOENT; 1553 } 1554 1555 rc = devm_request_irq(dev, pci_irq_vector(pdev, 1), 1556 int_phy_up_down_bcast_v3_hw, 0, 1557 DRV_NAME " phy", hisi_hba); 1558 if (rc) { 1559 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc); 1560 rc = -ENOENT; 1561 goto free_irq_vectors; 1562 } 1563 1564 rc = devm_request_irq(dev, pci_irq_vector(pdev, 2), 1565 int_chnl_int_v3_hw, 0, 1566 DRV_NAME " channel", hisi_hba); 1567 if (rc) { 1568 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc); 1569 rc = -ENOENT; 1570 goto free_phy_irq; 1571 } 1572 1573 /* Init tasklets for cq only */ 1574 for (i = 0; i < hisi_hba->queue_count; i++) { 1575 struct hisi_sas_cq *cq = &hisi_hba->cq[i]; 1576 struct tasklet_struct *t = &cq->tasklet; 1577 1578 rc = devm_request_irq(dev, pci_irq_vector(pdev, i+16), 1579 cq_interrupt_v3_hw, 0, 1580 DRV_NAME " cq", cq); 1581 if (rc) { 1582 dev_err(dev, 1583 "could not request cq%d interrupt, rc=%d\n", 1584 i, rc); 1585 rc = -ENOENT; 1586 goto free_cq_irqs; 1587 } 1588 1589 tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq); 1590 } 1591 1592 return 0; 1593 1594 free_cq_irqs: 1595 for (k = 0; k < i; k++) { 1596 struct hisi_sas_cq *cq = &hisi_hba->cq[k]; 1597 1598 free_irq(pci_irq_vector(pdev, k+16), cq); 1599 } 1600 free_irq(pci_irq_vector(pdev, 2), hisi_hba); 1601 free_phy_irq: 1602 free_irq(pci_irq_vector(pdev, 1), hisi_hba); 1603 free_irq_vectors: 1604 pci_free_irq_vectors(pdev); 1605 return rc; 1606 } 1607 1608 static int hisi_sas_v3_init(struct hisi_hba *hisi_hba) 1609 { 1610 int rc; 1611 1612 rc = hw_init_v3_hw(hisi_hba); 1613 if (rc) 1614 return rc; 1615 1616 rc = interrupt_init_v3_hw(hisi_hba); 1617 if (rc) 1618 return rc; 1619 1620 return 0; 1621 } 1622 1623 static const struct hisi_sas_hw hisi_sas_v3_hw = { 1624 .hw_init = hisi_sas_v3_init, 1625 .setup_itct = setup_itct_v3_hw, 1626 .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V3_HW, 1627 .get_wideport_bitmap = get_wideport_bitmap_v3_hw, 1628 .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr), 1629 .free_device = free_device_v3_hw, 1630 .sl_notify = sl_notify_v3_hw, 1631 .prep_ssp = prep_ssp_v3_hw, 1632 .prep_smp = prep_smp_v3_hw, 1633 .prep_stp = prep_ata_v3_hw, 1634 .prep_abort = prep_abort_v3_hw, 1635 .get_free_slot = get_free_slot_v3_hw, 1636 .start_delivery = start_delivery_v3_hw, 1637 .slot_complete = slot_complete_v3_hw, 1638 .phys_init = phys_init_v3_hw, 1639 .phy_enable = enable_phy_v3_hw, 1640 .phy_disable = disable_phy_v3_hw, 1641 .phy_hard_reset = phy_hard_reset_v3_hw, 1642 .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw, 1643 .dereg_device = dereg_device_v3_hw, 1644 }; 1645 1646 static struct Scsi_Host * 1647 hisi_sas_shost_alloc_pci(struct pci_dev *pdev) 1648 { 1649 struct Scsi_Host *shost; 1650 struct hisi_hba *hisi_hba; 1651 struct device *dev = &pdev->dev; 1652 1653 shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba)); 1654 if (!shost) 1655 goto err_out; 1656 hisi_hba = shost_priv(shost); 1657 1658 hisi_hba->hw = &hisi_sas_v3_hw; 1659 hisi_hba->pci_dev = pdev; 1660 hisi_hba->dev = dev; 1661 hisi_hba->shost = shost; 1662 SHOST_TO_SAS_HA(shost) = &hisi_hba->sha; 1663 1664 init_timer(&hisi_hba->timer); 1665 1666 if (hisi_sas_get_fw_info(hisi_hba) < 0) 1667 goto err_out; 1668 1669 if (hisi_sas_alloc(hisi_hba, shost)) { 1670 hisi_sas_free(hisi_hba); 1671 goto err_out; 1672 } 1673 1674 return shost; 1675 err_out: 1676 dev_err(dev, "shost alloc failed\n"); 1677 return NULL; 1678 } 1679 1680 static int 1681 hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1682 { 1683 struct Scsi_Host *shost; 1684 struct hisi_hba *hisi_hba; 1685 struct device *dev = &pdev->dev; 1686 struct asd_sas_phy **arr_phy; 1687 struct asd_sas_port **arr_port; 1688 struct sas_ha_struct *sha; 1689 int rc, phy_nr, port_nr, i; 1690 1691 rc = pci_enable_device(pdev); 1692 if (rc) 1693 goto err_out; 1694 1695 pci_set_master(pdev); 1696 1697 rc = pci_request_regions(pdev, DRV_NAME); 1698 if (rc) 1699 goto err_out_disable_device; 1700 1701 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) || 1702 (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0)) { 1703 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) || 1704 (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) != 0)) { 1705 dev_err(dev, "No usable DMA addressing method\n"); 1706 rc = -EIO; 1707 goto err_out_regions; 1708 } 1709 } 1710 1711 shost = hisi_sas_shost_alloc_pci(pdev); 1712 if (!shost) { 1713 rc = -ENOMEM; 1714 goto err_out_regions; 1715 } 1716 1717 sha = SHOST_TO_SAS_HA(shost); 1718 hisi_hba = shost_priv(shost); 1719 dev_set_drvdata(dev, sha); 1720 1721 hisi_hba->regs = pcim_iomap(pdev, 5, 0); 1722 if (!hisi_hba->regs) { 1723 dev_err(dev, "cannot map register.\n"); 1724 rc = -ENOMEM; 1725 goto err_out_ha; 1726 } 1727 1728 phy_nr = port_nr = hisi_hba->n_phy; 1729 1730 arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL); 1731 arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL); 1732 if (!arr_phy || !arr_port) { 1733 rc = -ENOMEM; 1734 goto err_out_ha; 1735 } 1736 1737 sha->sas_phy = arr_phy; 1738 sha->sas_port = arr_port; 1739 sha->core.shost = shost; 1740 sha->lldd_ha = hisi_hba; 1741 1742 shost->transportt = hisi_sas_stt; 1743 shost->max_id = HISI_SAS_MAX_DEVICES; 1744 shost->max_lun = ~0; 1745 shost->max_channel = 1; 1746 shost->max_cmd_len = 16; 1747 shost->sg_tablesize = min_t(u16, SG_ALL, HISI_SAS_SGE_PAGE_CNT); 1748 shost->can_queue = hisi_hba->hw->max_command_entries; 1749 shost->cmd_per_lun = hisi_hba->hw->max_command_entries; 1750 1751 sha->sas_ha_name = DRV_NAME; 1752 sha->dev = dev; 1753 sha->lldd_module = THIS_MODULE; 1754 sha->sas_addr = &hisi_hba->sas_addr[0]; 1755 sha->num_phys = hisi_hba->n_phy; 1756 sha->core.shost = hisi_hba->shost; 1757 1758 for (i = 0; i < hisi_hba->n_phy; i++) { 1759 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy; 1760 sha->sas_port[i] = &hisi_hba->port[i].sas_port; 1761 } 1762 1763 hisi_sas_init_add(hisi_hba); 1764 1765 rc = scsi_add_host(shost, dev); 1766 if (rc) 1767 goto err_out_ha; 1768 1769 rc = sas_register_ha(sha); 1770 if (rc) 1771 goto err_out_register_ha; 1772 1773 rc = hisi_hba->hw->hw_init(hisi_hba); 1774 if (rc) 1775 goto err_out_register_ha; 1776 1777 scsi_scan_host(shost); 1778 1779 return 0; 1780 1781 err_out_register_ha: 1782 scsi_remove_host(shost); 1783 err_out_ha: 1784 kfree(shost); 1785 err_out_regions: 1786 pci_release_regions(pdev); 1787 err_out_disable_device: 1788 pci_disable_device(pdev); 1789 err_out: 1790 return rc; 1791 } 1792 1793 static void 1794 hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba) 1795 { 1796 int i; 1797 1798 free_irq(pci_irq_vector(pdev, 1), hisi_hba); 1799 free_irq(pci_irq_vector(pdev, 2), hisi_hba); 1800 for (i = 0; i < hisi_hba->queue_count; i++) { 1801 struct hisi_sas_cq *cq = &hisi_hba->cq[i]; 1802 1803 free_irq(pci_irq_vector(pdev, i+16), cq); 1804 } 1805 pci_free_irq_vectors(pdev); 1806 } 1807 1808 static void hisi_sas_v3_remove(struct pci_dev *pdev) 1809 { 1810 struct device *dev = &pdev->dev; 1811 struct sas_ha_struct *sha = dev_get_drvdata(dev); 1812 struct hisi_hba *hisi_hba = sha->lldd_ha; 1813 1814 sas_unregister_ha(sha); 1815 sas_remove_host(sha->core.shost); 1816 1817 hisi_sas_free(hisi_hba); 1818 hisi_sas_v3_destroy_irqs(pdev, hisi_hba); 1819 pci_release_regions(pdev); 1820 pci_disable_device(pdev); 1821 } 1822 1823 enum { 1824 /* instances of the controller */ 1825 hip08, 1826 }; 1827 1828 static const struct pci_device_id sas_v3_pci_table[] = { 1829 { PCI_VDEVICE(HUAWEI, 0xa230), hip08 }, 1830 {} 1831 }; 1832 1833 static struct pci_driver sas_v3_pci_driver = { 1834 .name = DRV_NAME, 1835 .id_table = sas_v3_pci_table, 1836 .probe = hisi_sas_v3_probe, 1837 .remove = hisi_sas_v3_remove, 1838 }; 1839 1840 module_pci_driver(sas_v3_pci_driver); 1841 1842 MODULE_VERSION(DRV_VERSION); 1843 MODULE_LICENSE("GPL"); 1844 MODULE_AUTHOR("John Garry <john.garry@huawei.com>"); 1845 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device"); 1846 MODULE_ALIAS("platform:" DRV_NAME); 1847