1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Universal Flash Storage Host controller driver Core 4 * Copyright (C) 2011-2013 Samsung India Software Operations 5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. 6 * 7 * Authors: 8 * Santosh Yaraganavi <santosh.sy@samsung.com> 9 * Vinayak Holikatti <h.vinayak@samsung.com> 10 */ 11 12 #include <linux/async.h> 13 #include <linux/devfreq.h> 14 #include <linux/nls.h> 15 #include <linux/of.h> 16 #include <linux/bitfield.h> 17 #include <linux/blk-pm.h> 18 #include <linux/blkdev.h> 19 #include <linux/clk.h> 20 #include <linux/delay.h> 21 #include <linux/interrupt.h> 22 #include <linux/module.h> 23 #include <linux/regulator/consumer.h> 24 #include <linux/sched/clock.h> 25 #include <scsi/scsi_cmnd.h> 26 #include <scsi/scsi_dbg.h> 27 #include <scsi/scsi_driver.h> 28 #include <scsi/scsi_eh.h> 29 #include "ufshcd-priv.h" 30 #include <ufs/ufs_quirks.h> 31 #include <ufs/unipro.h> 32 #include "ufs-sysfs.h" 33 #include "ufs-debugfs.h" 34 #include "ufs-fault-injection.h" 35 #include "ufs_bsg.h" 36 #include "ufshcd-crypto.h" 37 #include "ufshpb.h" 38 #include <asm/unaligned.h> 39 40 #define CREATE_TRACE_POINTS 41 #include <trace/events/ufs.h> 42 43 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ 44 UTP_TASK_REQ_COMPL |\ 45 UFSHCD_ERROR_MASK) 46 47 #define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\ 48 UFSHCD_ERROR_MASK |\ 49 MCQ_CQ_EVENT_STATUS) 50 51 52 /* UIC command timeout, unit: ms */ 53 #define UIC_CMD_TIMEOUT 500 54 55 /* NOP OUT retries waiting for NOP IN response */ 56 #define NOP_OUT_RETRIES 10 57 /* Timeout after 50 msecs if NOP OUT hangs without response */ 58 #define NOP_OUT_TIMEOUT 50 /* msecs */ 59 60 /* Query request retries */ 61 #define QUERY_REQ_RETRIES 3 62 /* Query request timeout */ 63 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */ 64 65 /* Advanced RPMB request timeout */ 66 #define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */ 67 68 /* Task management command timeout */ 69 #define TM_CMD_TIMEOUT 100 /* msecs */ 70 71 /* maximum number of retries for a general UIC command */ 72 #define UFS_UIC_COMMAND_RETRIES 3 73 74 /* maximum number of link-startup retries */ 75 #define DME_LINKSTARTUP_RETRIES 3 76 77 /* maximum number of reset retries before giving up */ 78 #define MAX_HOST_RESET_RETRIES 5 79 80 /* Maximum number of error handler retries before giving up */ 81 #define MAX_ERR_HANDLER_RETRIES 5 82 83 /* Expose the flag value from utp_upiu_query.value */ 84 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF 85 86 /* Interrupt aggregation default timeout, unit: 40us */ 87 #define INT_AGGR_DEF_TO 0x02 88 89 /* default delay of autosuspend: 2000 ms */ 90 #define RPM_AUTOSUSPEND_DELAY_MS 2000 91 92 /* Default delay of RPM device flush delayed work */ 93 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000 94 95 /* Default value of wait time before gating device ref clock */ 96 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */ 97 98 /* Polling time to wait for fDeviceInit */ 99 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */ 100 101 /* UFSHC 4.0 compliant HC support this mode, refer param_set_mcq_mode() */ 102 static bool use_mcq_mode = true; 103 104 static bool is_mcq_supported(struct ufs_hba *hba) 105 { 106 return hba->mcq_sup && use_mcq_mode; 107 } 108 109 static int param_set_mcq_mode(const char *val, const struct kernel_param *kp) 110 { 111 int ret; 112 113 ret = param_set_bool(val, kp); 114 if (ret) 115 return ret; 116 117 return 0; 118 } 119 120 static const struct kernel_param_ops mcq_mode_ops = { 121 .set = param_set_mcq_mode, 122 .get = param_get_bool, 123 }; 124 125 module_param_cb(use_mcq_mode, &mcq_mode_ops, &use_mcq_mode, 0644); 126 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); 127 128 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \ 129 ({ \ 130 int _ret; \ 131 if (_on) \ 132 _ret = ufshcd_enable_vreg(_dev, _vreg); \ 133 else \ 134 _ret = ufshcd_disable_vreg(_dev, _vreg); \ 135 _ret; \ 136 }) 137 138 #define ufshcd_hex_dump(prefix_str, buf, len) do { \ 139 size_t __len = (len); \ 140 print_hex_dump(KERN_ERR, prefix_str, \ 141 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\ 142 16, 4, buf, __len, false); \ 143 } while (0) 144 145 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len, 146 const char *prefix) 147 { 148 u32 *regs; 149 size_t pos; 150 151 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */ 152 return -EINVAL; 153 154 regs = kzalloc(len, GFP_ATOMIC); 155 if (!regs) 156 return -ENOMEM; 157 158 for (pos = 0; pos < len; pos += 4) { 159 if (offset == 0 && 160 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER && 161 pos <= REG_UIC_ERROR_CODE_DME) 162 continue; 163 regs[pos / 4] = ufshcd_readl(hba, offset + pos); 164 } 165 166 ufshcd_hex_dump(prefix, regs, len); 167 kfree(regs); 168 169 return 0; 170 } 171 EXPORT_SYMBOL_GPL(ufshcd_dump_regs); 172 173 enum { 174 UFSHCD_MAX_CHANNEL = 0, 175 UFSHCD_MAX_ID = 1, 176 UFSHCD_NUM_RESERVED = 1, 177 UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED, 178 UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED, 179 }; 180 181 static const char *const ufshcd_state_name[] = { 182 [UFSHCD_STATE_RESET] = "reset", 183 [UFSHCD_STATE_OPERATIONAL] = "operational", 184 [UFSHCD_STATE_ERROR] = "error", 185 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal", 186 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal", 187 }; 188 189 /* UFSHCD error handling flags */ 190 enum { 191 UFSHCD_EH_IN_PROGRESS = (1 << 0), 192 }; 193 194 /* UFSHCD UIC layer error flags */ 195 enum { 196 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ 197 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */ 198 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */ 199 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */ 200 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */ 201 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */ 202 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */ 203 }; 204 205 #define ufshcd_set_eh_in_progress(h) \ 206 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS) 207 #define ufshcd_eh_in_progress(h) \ 208 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS) 209 #define ufshcd_clear_eh_in_progress(h) \ 210 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) 211 212 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { 213 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 214 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 215 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 216 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 217 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 218 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, 219 /* 220 * For DeepSleep, the link is first put in hibern8 and then off. 221 * Leaving the link in hibern8 is not supported. 222 */ 223 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE}, 224 }; 225 226 static inline enum ufs_dev_pwr_mode 227 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) 228 { 229 return ufs_pm_lvl_states[lvl].dev_state; 230 } 231 232 static inline enum uic_link_state 233 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) 234 { 235 return ufs_pm_lvl_states[lvl].link_state; 236 } 237 238 static inline enum ufs_pm_level 239 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, 240 enum uic_link_state link_state) 241 { 242 enum ufs_pm_level lvl; 243 244 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) { 245 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) && 246 (ufs_pm_lvl_states[lvl].link_state == link_state)) 247 return lvl; 248 } 249 250 /* if no match found, return the level 0 */ 251 return UFS_PM_LVL_0; 252 } 253 254 static const struct ufs_dev_quirk ufs_fixups[] = { 255 /* UFS cards deviations table */ 256 { .wmanufacturerid = UFS_VENDOR_MICRON, 257 .model = UFS_ANY_MODEL, 258 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | 259 UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ }, 260 { .wmanufacturerid = UFS_VENDOR_SAMSUNG, 261 .model = UFS_ANY_MODEL, 262 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | 263 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | 264 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, 265 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 266 .model = UFS_ANY_MODEL, 267 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME }, 268 { .wmanufacturerid = UFS_VENDOR_SKHYNIX, 269 .model = "hB8aL1" /*H28U62301AMR*/, 270 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME }, 271 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 272 .model = UFS_ANY_MODEL, 273 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, 274 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 275 .model = "THGLF2G9C8KBADG", 276 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 277 { .wmanufacturerid = UFS_VENDOR_TOSHIBA, 278 .model = "THGLF2G9D8KBADG", 279 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, 280 {} 281 }; 282 283 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba); 284 static void ufshcd_async_scan(void *data, async_cookie_t cookie); 285 static int ufshcd_reset_and_restore(struct ufs_hba *hba); 286 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); 287 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); 288 static void ufshcd_hba_exit(struct ufs_hba *hba); 289 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params); 290 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); 291 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); 292 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); 293 static void ufshcd_resume_clkscaling(struct ufs_hba *hba); 294 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); 295 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba); 296 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up); 297 static irqreturn_t ufshcd_intr(int irq, void *__hba); 298 static int ufshcd_change_power_mode(struct ufs_hba *hba, 299 struct ufs_pa_layer_attr *pwr_mode); 300 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on); 301 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on); 302 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 303 struct ufs_vreg *vreg); 304 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); 305 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 306 bool enable); 307 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); 308 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); 309 310 static inline void ufshcd_enable_irq(struct ufs_hba *hba) 311 { 312 if (!hba->is_irq_enabled) { 313 enable_irq(hba->irq); 314 hba->is_irq_enabled = true; 315 } 316 } 317 318 static inline void ufshcd_disable_irq(struct ufs_hba *hba) 319 { 320 if (hba->is_irq_enabled) { 321 disable_irq(hba->irq); 322 hba->is_irq_enabled = false; 323 } 324 } 325 326 static void ufshcd_configure_wb(struct ufs_hba *hba) 327 { 328 if (!ufshcd_is_wb_allowed(hba)) 329 return; 330 331 ufshcd_wb_toggle(hba, true); 332 333 ufshcd_wb_toggle_buf_flush_during_h8(hba, true); 334 335 if (ufshcd_is_wb_buf_flush_allowed(hba)) 336 ufshcd_wb_toggle_buf_flush(hba, true); 337 } 338 339 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba) 340 { 341 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt)) 342 scsi_unblock_requests(hba->host); 343 } 344 345 static void ufshcd_scsi_block_requests(struct ufs_hba *hba) 346 { 347 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1) 348 scsi_block_requests(hba->host); 349 } 350 351 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, 352 enum ufs_trace_str_t str_t) 353 { 354 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr; 355 struct utp_upiu_header *header; 356 357 if (!trace_ufshcd_upiu_enabled()) 358 return; 359 360 if (str_t == UFS_CMD_SEND) 361 header = &rq->header; 362 else 363 header = &hba->lrb[tag].ucd_rsp_ptr->header; 364 365 trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb, 366 UFS_TSF_CDB); 367 } 368 369 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, 370 enum ufs_trace_str_t str_t, 371 struct utp_upiu_req *rq_rsp) 372 { 373 if (!trace_ufshcd_upiu_enabled()) 374 return; 375 376 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header, 377 &rq_rsp->qr, UFS_TSF_OSF); 378 } 379 380 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, 381 enum ufs_trace_str_t str_t) 382 { 383 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag]; 384 385 if (!trace_ufshcd_upiu_enabled()) 386 return; 387 388 if (str_t == UFS_TM_SEND) 389 trace_ufshcd_upiu(dev_name(hba->dev), str_t, 390 &descp->upiu_req.req_header, 391 &descp->upiu_req.input_param1, 392 UFS_TSF_TM_INPUT); 393 else 394 trace_ufshcd_upiu(dev_name(hba->dev), str_t, 395 &descp->upiu_rsp.rsp_header, 396 &descp->upiu_rsp.output_param1, 397 UFS_TSF_TM_OUTPUT); 398 } 399 400 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, 401 const struct uic_command *ucmd, 402 enum ufs_trace_str_t str_t) 403 { 404 u32 cmd; 405 406 if (!trace_ufshcd_uic_command_enabled()) 407 return; 408 409 if (str_t == UFS_CMD_SEND) 410 cmd = ucmd->command; 411 else 412 cmd = ufshcd_readl(hba, REG_UIC_COMMAND); 413 414 trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd, 415 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), 416 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), 417 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); 418 } 419 420 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, 421 enum ufs_trace_str_t str_t) 422 { 423 u64 lba = 0; 424 u8 opcode = 0, group_id = 0; 425 u32 intr, doorbell; 426 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 427 struct scsi_cmnd *cmd = lrbp->cmd; 428 struct request *rq = scsi_cmd_to_rq(cmd); 429 int transfer_len = -1; 430 431 if (!cmd) 432 return; 433 434 /* trace UPIU also */ 435 ufshcd_add_cmd_upiu_trace(hba, tag, str_t); 436 if (!trace_ufshcd_command_enabled()) 437 return; 438 439 opcode = cmd->cmnd[0]; 440 441 if (opcode == READ_10 || opcode == WRITE_10) { 442 /* 443 * Currently we only fully trace read(10) and write(10) commands 444 */ 445 transfer_len = 446 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); 447 lba = scsi_get_lba(cmd); 448 if (opcode == WRITE_10) 449 group_id = lrbp->cmd->cmnd[6]; 450 } else if (opcode == UNMAP) { 451 /* 452 * The number of Bytes to be unmapped beginning with the lba. 453 */ 454 transfer_len = blk_rq_bytes(rq); 455 lba = scsi_get_lba(cmd); 456 } 457 458 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 459 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 460 trace_ufshcd_command(dev_name(hba->dev), str_t, tag, 461 doorbell, transfer_len, intr, lba, opcode, group_id); 462 } 463 464 static void ufshcd_print_clk_freqs(struct ufs_hba *hba) 465 { 466 struct ufs_clk_info *clki; 467 struct list_head *head = &hba->clk_list_head; 468 469 if (list_empty(head)) 470 return; 471 472 list_for_each_entry(clki, head, list) { 473 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq && 474 clki->max_freq) 475 dev_err(hba->dev, "clk: %s, rate: %u\n", 476 clki->name, clki->curr_freq); 477 } 478 } 479 480 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, 481 const char *err_name) 482 { 483 int i; 484 bool found = false; 485 const struct ufs_event_hist *e; 486 487 if (id >= UFS_EVT_CNT) 488 return; 489 490 e = &hba->ufs_stats.event[id]; 491 492 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) { 493 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH; 494 495 if (e->tstamp[p] == 0) 496 continue; 497 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, 498 e->val[p], div_u64(e->tstamp[p], 1000)); 499 found = true; 500 } 501 502 if (!found) 503 dev_err(hba->dev, "No record of %s\n", err_name); 504 else 505 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt); 506 } 507 508 static void ufshcd_print_evt_hist(struct ufs_hba *hba) 509 { 510 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 511 512 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err"); 513 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err"); 514 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err"); 515 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err"); 516 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err"); 517 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR, 518 "auto_hibern8_err"); 519 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err"); 520 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL, 521 "link_startup_fail"); 522 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail"); 523 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR, 524 "suspend_fail"); 525 ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail"); 526 ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR, 527 "wlun suspend_fail"); 528 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset"); 529 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset"); 530 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort"); 531 532 ufshcd_vops_dbg_register_dump(hba); 533 } 534 535 static 536 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt) 537 { 538 const struct ufshcd_lrb *lrbp; 539 int prdt_length; 540 int tag; 541 542 for_each_set_bit(tag, &bitmap, hba->nutrs) { 543 lrbp = &hba->lrb[tag]; 544 545 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", 546 tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000)); 547 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", 548 tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000)); 549 dev_err(hba->dev, 550 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", 551 tag, (u64)lrbp->utrd_dma_addr); 552 553 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, 554 sizeof(struct utp_transfer_req_desc)); 555 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, 556 (u64)lrbp->ucd_req_dma_addr); 557 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, 558 sizeof(struct utp_upiu_req)); 559 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, 560 (u64)lrbp->ucd_rsp_dma_addr); 561 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, 562 sizeof(struct utp_upiu_rsp)); 563 564 prdt_length = le16_to_cpu( 565 lrbp->utr_descriptor_ptr->prd_table_length); 566 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 567 prdt_length /= ufshcd_sg_entry_size(hba); 568 569 dev_err(hba->dev, 570 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", 571 tag, prdt_length, 572 (u64)lrbp->ucd_prdt_dma_addr); 573 574 if (pr_prdt) 575 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, 576 ufshcd_sg_entry_size(hba) * prdt_length); 577 } 578 } 579 580 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap) 581 { 582 int tag; 583 584 for_each_set_bit(tag, &bitmap, hba->nutmrs) { 585 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag]; 586 587 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag); 588 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp)); 589 } 590 } 591 592 static void ufshcd_print_host_state(struct ufs_hba *hba) 593 { 594 const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; 595 596 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); 597 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n", 598 hba->outstanding_reqs, hba->outstanding_tasks); 599 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", 600 hba->saved_err, hba->saved_uic_err); 601 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", 602 hba->curr_dev_pwr_mode, hba->uic_link_state); 603 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n", 604 hba->pm_op_in_progress, hba->is_sys_suspended); 605 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n", 606 hba->auto_bkops_enabled, hba->host->host_self_blocked); 607 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); 608 dev_err(hba->dev, 609 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", 610 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), 611 hba->ufs_stats.hibern8_exit_cnt); 612 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n", 613 div_u64(hba->ufs_stats.last_intr_ts, 1000), 614 hba->ufs_stats.last_intr_status); 615 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", 616 hba->eh_flags, hba->req_abort_count); 617 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", 618 hba->ufs_version, hba->capabilities, hba->caps); 619 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks, 620 hba->dev_quirks); 621 if (sdev_ufs) 622 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n", 623 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev); 624 625 ufshcd_print_clk_freqs(hba); 626 } 627 628 /** 629 * ufshcd_print_pwr_info - print power params as saved in hba 630 * power info 631 * @hba: per-adapter instance 632 */ 633 static void ufshcd_print_pwr_info(struct ufs_hba *hba) 634 { 635 static const char * const names[] = { 636 "INVALID MODE", 637 "FAST MODE", 638 "SLOW_MODE", 639 "INVALID MODE", 640 "FASTAUTO_MODE", 641 "SLOWAUTO_MODE", 642 "INVALID MODE", 643 }; 644 645 /* 646 * Using dev_dbg to avoid messages during runtime PM to avoid 647 * never-ending cycles of messages written back to storage by user space 648 * causing runtime resume, causing more messages and so on. 649 */ 650 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n", 651 __func__, 652 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, 653 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, 654 names[hba->pwr_info.pwr_rx], 655 names[hba->pwr_info.pwr_tx], 656 hba->pwr_info.hs_rate); 657 } 658 659 static void ufshcd_device_reset(struct ufs_hba *hba) 660 { 661 int err; 662 663 err = ufshcd_vops_device_reset(hba); 664 665 if (!err) { 666 ufshcd_set_ufs_dev_active(hba); 667 if (ufshcd_is_wb_allowed(hba)) { 668 hba->dev_info.wb_enabled = false; 669 hba->dev_info.wb_buf_flush_enabled = false; 670 } 671 } 672 if (err != -EOPNOTSUPP) 673 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); 674 } 675 676 void ufshcd_delay_us(unsigned long us, unsigned long tolerance) 677 { 678 if (!us) 679 return; 680 681 if (us < 10) 682 udelay(us); 683 else 684 usleep_range(us, us + tolerance); 685 } 686 EXPORT_SYMBOL_GPL(ufshcd_delay_us); 687 688 /** 689 * ufshcd_wait_for_register - wait for register value to change 690 * @hba: per-adapter interface 691 * @reg: mmio register offset 692 * @mask: mask to apply to the read register value 693 * @val: value to wait for 694 * @interval_us: polling interval in microseconds 695 * @timeout_ms: timeout in milliseconds 696 * 697 * Return: 698 * -ETIMEDOUT on error, zero on success. 699 */ 700 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, 701 u32 val, unsigned long interval_us, 702 unsigned long timeout_ms) 703 { 704 int err = 0; 705 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); 706 707 /* ignore bits that we don't intend to wait on */ 708 val = val & mask; 709 710 while ((ufshcd_readl(hba, reg) & mask) != val) { 711 usleep_range(interval_us, interval_us + 50); 712 if (time_after(jiffies, timeout)) { 713 if ((ufshcd_readl(hba, reg) & mask) != val) 714 err = -ETIMEDOUT; 715 break; 716 } 717 } 718 719 return err; 720 } 721 722 /** 723 * ufshcd_get_intr_mask - Get the interrupt bit mask 724 * @hba: Pointer to adapter instance 725 * 726 * Returns interrupt bit mask per version 727 */ 728 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) 729 { 730 if (hba->ufs_version == ufshci_version(1, 0)) 731 return INTERRUPT_MASK_ALL_VER_10; 732 if (hba->ufs_version <= ufshci_version(2, 0)) 733 return INTERRUPT_MASK_ALL_VER_11; 734 735 return INTERRUPT_MASK_ALL_VER_21; 736 } 737 738 /** 739 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA 740 * @hba: Pointer to adapter instance 741 * 742 * Returns UFSHCI version supported by the controller 743 */ 744 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) 745 { 746 u32 ufshci_ver; 747 748 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION) 749 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba); 750 else 751 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION); 752 753 /* 754 * UFSHCI v1.x uses a different version scheme, in order 755 * to allow the use of comparisons with the ufshci_version 756 * function, we convert it to the same scheme as ufs 2.0+. 757 */ 758 if (ufshci_ver & 0x00010000) 759 return ufshci_version(1, ufshci_ver & 0x00000100); 760 761 return ufshci_ver; 762 } 763 764 /** 765 * ufshcd_is_device_present - Check if any device connected to 766 * the host controller 767 * @hba: pointer to adapter instance 768 * 769 * Returns true if device present, false if no device detected 770 */ 771 static inline bool ufshcd_is_device_present(struct ufs_hba *hba) 772 { 773 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT; 774 } 775 776 /** 777 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status 778 * @lrbp: pointer to local command reference block 779 * @cqe: pointer to the completion queue entry 780 * 781 * This function is used to get the OCS field from UTRD 782 * Returns the OCS field in the UTRD 783 */ 784 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, 785 struct cq_entry *cqe) 786 { 787 if (cqe) 788 return le32_to_cpu(cqe->status) & MASK_OCS; 789 790 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; 791 } 792 793 /** 794 * ufshcd_utrl_clear() - Clear requests from the controller request list. 795 * @hba: per adapter instance 796 * @mask: mask with one bit set for each request to be cleared 797 */ 798 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask) 799 { 800 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 801 mask = ~mask; 802 /* 803 * From the UFSHCI specification: "UTP Transfer Request List CLear 804 * Register (UTRLCLR): This field is bit significant. Each bit 805 * corresponds to a slot in the UTP Transfer Request List, where bit 0 806 * corresponds to request slot 0. A bit in this field is set to ‘0’ 807 * by host software to indicate to the host controller that a transfer 808 * request slot is cleared. The host controller 809 * shall free up any resources associated to the request slot 810 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The 811 * host software indicates no change to request slots by setting the 812 * associated bits in this field to ‘1’. Bits in this field shall only 813 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’." 814 */ 815 ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR); 816 } 817 818 /** 819 * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register 820 * @hba: per adapter instance 821 * @pos: position of the bit to be cleared 822 */ 823 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) 824 { 825 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR) 826 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 827 else 828 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR); 829 } 830 831 /** 832 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY 833 * @reg: Register value of host controller status 834 * 835 * Returns integer, 0 on Success and positive value if failed 836 */ 837 static inline int ufshcd_get_lists_status(u32 reg) 838 { 839 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY); 840 } 841 842 /** 843 * ufshcd_get_uic_cmd_result - Get the UIC command result 844 * @hba: Pointer to adapter instance 845 * 846 * This function gets the result of UIC command completion 847 * Returns 0 on success, non zero value on error 848 */ 849 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) 850 { 851 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & 852 MASK_UIC_COMMAND_RESULT; 853 } 854 855 /** 856 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command 857 * @hba: Pointer to adapter instance 858 * 859 * This function gets UIC command argument3 860 * Returns 0 on success, non zero value on error 861 */ 862 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) 863 { 864 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); 865 } 866 867 /** 868 * ufshcd_get_req_rsp - returns the TR response transaction type 869 * @ucd_rsp_ptr: pointer to response UPIU 870 */ 871 static inline int 872 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) 873 { 874 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; 875 } 876 877 /** 878 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU 879 * @ucd_rsp_ptr: pointer to response UPIU 880 * 881 * This function gets the response status and scsi_status from response UPIU 882 * Returns the response result code. 883 */ 884 static inline int 885 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) 886 { 887 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; 888 } 889 890 /* 891 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length 892 * from response UPIU 893 * @ucd_rsp_ptr: pointer to response UPIU 894 * 895 * Return the data segment length. 896 */ 897 static inline unsigned int 898 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) 899 { 900 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & 901 MASK_RSP_UPIU_DATA_SEG_LEN; 902 } 903 904 /** 905 * ufshcd_is_exception_event - Check if the device raised an exception event 906 * @ucd_rsp_ptr: pointer to response UPIU 907 * 908 * The function checks if the device raised an exception event indicated in 909 * the Device Information field of response UPIU. 910 * 911 * Returns true if exception is raised, false otherwise. 912 */ 913 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) 914 { 915 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & 916 MASK_RSP_EXCEPTION_EVENT; 917 } 918 919 /** 920 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. 921 * @hba: per adapter instance 922 */ 923 static inline void 924 ufshcd_reset_intr_aggr(struct ufs_hba *hba) 925 { 926 ufshcd_writel(hba, INT_AGGR_ENABLE | 927 INT_AGGR_COUNTER_AND_TIMER_RESET, 928 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 929 } 930 931 /** 932 * ufshcd_config_intr_aggr - Configure interrupt aggregation values. 933 * @hba: per adapter instance 934 * @cnt: Interrupt aggregation counter threshold 935 * @tmout: Interrupt aggregation timeout value 936 */ 937 static inline void 938 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) 939 { 940 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | 941 INT_AGGR_COUNTER_THLD_VAL(cnt) | 942 INT_AGGR_TIMEOUT_VAL(tmout), 943 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 944 } 945 946 /** 947 * ufshcd_disable_intr_aggr - Disables interrupt aggregation. 948 * @hba: per adapter instance 949 */ 950 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba) 951 { 952 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 953 } 954 955 /** 956 * ufshcd_enable_run_stop_reg - Enable run-stop registers, 957 * When run-stop registers are set to 1, it indicates the 958 * host controller that it can process the requests 959 * @hba: per adapter instance 960 */ 961 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) 962 { 963 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, 964 REG_UTP_TASK_REQ_LIST_RUN_STOP); 965 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, 966 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); 967 } 968 969 /** 970 * ufshcd_hba_start - Start controller initialization sequence 971 * @hba: per adapter instance 972 */ 973 static inline void ufshcd_hba_start(struct ufs_hba *hba) 974 { 975 u32 val = CONTROLLER_ENABLE; 976 977 if (ufshcd_crypto_enable(hba)) 978 val |= CRYPTO_GENERAL_ENABLE; 979 980 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE); 981 } 982 983 /** 984 * ufshcd_is_hba_active - Get controller state 985 * @hba: per adapter instance 986 * 987 * Returns true if and only if the controller is active. 988 */ 989 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba) 990 { 991 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; 992 } 993 994 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba) 995 { 996 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */ 997 if (hba->ufs_version <= ufshci_version(1, 1)) 998 return UFS_UNIPRO_VER_1_41; 999 else 1000 return UFS_UNIPRO_VER_1_6; 1001 } 1002 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver); 1003 1004 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba) 1005 { 1006 /* 1007 * If both host and device support UniPro ver1.6 or later, PA layer 1008 * parameters tuning happens during link startup itself. 1009 * 1010 * We can manually tune PA layer parameters if either host or device 1011 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning 1012 * logic simple, we will only do manual tuning if local unipro version 1013 * doesn't support ver1.6 or later. 1014 */ 1015 return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6; 1016 } 1017 1018 /** 1019 * ufshcd_set_clk_freq - set UFS controller clock frequencies 1020 * @hba: per adapter instance 1021 * @scale_up: If True, set max possible frequency othewise set low frequency 1022 * 1023 * Returns 0 if successful 1024 * Returns < 0 for any other errors 1025 */ 1026 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) 1027 { 1028 int ret = 0; 1029 struct ufs_clk_info *clki; 1030 struct list_head *head = &hba->clk_list_head; 1031 1032 if (list_empty(head)) 1033 goto out; 1034 1035 list_for_each_entry(clki, head, list) { 1036 if (!IS_ERR_OR_NULL(clki->clk)) { 1037 if (scale_up && clki->max_freq) { 1038 if (clki->curr_freq == clki->max_freq) 1039 continue; 1040 1041 ret = clk_set_rate(clki->clk, clki->max_freq); 1042 if (ret) { 1043 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1044 __func__, clki->name, 1045 clki->max_freq, ret); 1046 break; 1047 } 1048 trace_ufshcd_clk_scaling(dev_name(hba->dev), 1049 "scaled up", clki->name, 1050 clki->curr_freq, 1051 clki->max_freq); 1052 1053 clki->curr_freq = clki->max_freq; 1054 1055 } else if (!scale_up && clki->min_freq) { 1056 if (clki->curr_freq == clki->min_freq) 1057 continue; 1058 1059 ret = clk_set_rate(clki->clk, clki->min_freq); 1060 if (ret) { 1061 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 1062 __func__, clki->name, 1063 clki->min_freq, ret); 1064 break; 1065 } 1066 trace_ufshcd_clk_scaling(dev_name(hba->dev), 1067 "scaled down", clki->name, 1068 clki->curr_freq, 1069 clki->min_freq); 1070 clki->curr_freq = clki->min_freq; 1071 } 1072 } 1073 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, 1074 clki->name, clk_get_rate(clki->clk)); 1075 } 1076 1077 out: 1078 return ret; 1079 } 1080 1081 /** 1082 * ufshcd_scale_clks - scale up or scale down UFS controller clocks 1083 * @hba: per adapter instance 1084 * @scale_up: True if scaling up and false if scaling down 1085 * 1086 * Returns 0 if successful 1087 * Returns < 0 for any other errors 1088 */ 1089 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) 1090 { 1091 int ret = 0; 1092 ktime_t start = ktime_get(); 1093 1094 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE); 1095 if (ret) 1096 goto out; 1097 1098 ret = ufshcd_set_clk_freq(hba, scale_up); 1099 if (ret) 1100 goto out; 1101 1102 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE); 1103 if (ret) 1104 ufshcd_set_clk_freq(hba, !scale_up); 1105 1106 out: 1107 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), 1108 (scale_up ? "up" : "down"), 1109 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1110 return ret; 1111 } 1112 1113 /** 1114 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not 1115 * @hba: per adapter instance 1116 * @scale_up: True if scaling up and false if scaling down 1117 * 1118 * Returns true if scaling is required, false otherwise. 1119 */ 1120 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, 1121 bool scale_up) 1122 { 1123 struct ufs_clk_info *clki; 1124 struct list_head *head = &hba->clk_list_head; 1125 1126 if (list_empty(head)) 1127 return false; 1128 1129 list_for_each_entry(clki, head, list) { 1130 if (!IS_ERR_OR_NULL(clki->clk)) { 1131 if (scale_up && clki->max_freq) { 1132 if (clki->curr_freq == clki->max_freq) 1133 continue; 1134 return true; 1135 } else if (!scale_up && clki->min_freq) { 1136 if (clki->curr_freq == clki->min_freq) 1137 continue; 1138 return true; 1139 } 1140 } 1141 } 1142 1143 return false; 1144 } 1145 1146 /* 1147 * Determine the number of pending commands by counting the bits in the SCSI 1148 * device budget maps. This approach has been selected because a bit is set in 1149 * the budget map before scsi_host_queue_ready() checks the host_self_blocked 1150 * flag. The host_self_blocked flag can be modified by calling 1151 * scsi_block_requests() or scsi_unblock_requests(). 1152 */ 1153 static u32 ufshcd_pending_cmds(struct ufs_hba *hba) 1154 { 1155 const struct scsi_device *sdev; 1156 u32 pending = 0; 1157 1158 lockdep_assert_held(hba->host->host_lock); 1159 __shost_for_each_device(sdev, hba->host) 1160 pending += sbitmap_weight(&sdev->budget_map); 1161 1162 return pending; 1163 } 1164 1165 /* 1166 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1167 * has expired. 1168 * 1169 * Return: 0 upon success; -EBUSY upon timeout. 1170 */ 1171 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, 1172 u64 wait_timeout_us) 1173 { 1174 unsigned long flags; 1175 int ret = 0; 1176 u32 tm_doorbell; 1177 u32 tr_pending; 1178 bool timeout = false, do_last_check = false; 1179 ktime_t start; 1180 1181 ufshcd_hold(hba, false); 1182 spin_lock_irqsave(hba->host->host_lock, flags); 1183 /* 1184 * Wait for all the outstanding tasks/transfer requests. 1185 * Verify by checking the doorbell registers are clear. 1186 */ 1187 start = ktime_get(); 1188 do { 1189 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) { 1190 ret = -EBUSY; 1191 goto out; 1192 } 1193 1194 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 1195 tr_pending = ufshcd_pending_cmds(hba); 1196 if (!tm_doorbell && !tr_pending) { 1197 timeout = false; 1198 break; 1199 } else if (do_last_check) { 1200 break; 1201 } 1202 1203 spin_unlock_irqrestore(hba->host->host_lock, flags); 1204 io_schedule_timeout(msecs_to_jiffies(20)); 1205 if (ktime_to_us(ktime_sub(ktime_get(), start)) > 1206 wait_timeout_us) { 1207 timeout = true; 1208 /* 1209 * We might have scheduled out for long time so make 1210 * sure to check if doorbells are cleared by this time 1211 * or not. 1212 */ 1213 do_last_check = true; 1214 } 1215 spin_lock_irqsave(hba->host->host_lock, flags); 1216 } while (tm_doorbell || tr_pending); 1217 1218 if (timeout) { 1219 dev_err(hba->dev, 1220 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n", 1221 __func__, tm_doorbell, tr_pending); 1222 ret = -EBUSY; 1223 } 1224 out: 1225 spin_unlock_irqrestore(hba->host->host_lock, flags); 1226 ufshcd_release(hba); 1227 return ret; 1228 } 1229 1230 /** 1231 * ufshcd_scale_gear - scale up/down UFS gear 1232 * @hba: per adapter instance 1233 * @scale_up: True for scaling up gear and false for scaling down 1234 * 1235 * Returns 0 for success, 1236 * Returns -EBUSY if scaling can't happen at this time 1237 * Returns non-zero for any other errors 1238 */ 1239 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) 1240 { 1241 int ret = 0; 1242 struct ufs_pa_layer_attr new_pwr_info; 1243 1244 if (scale_up) { 1245 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info, 1246 sizeof(struct ufs_pa_layer_attr)); 1247 } else { 1248 memcpy(&new_pwr_info, &hba->pwr_info, 1249 sizeof(struct ufs_pa_layer_attr)); 1250 1251 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear || 1252 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) { 1253 /* save the current power mode */ 1254 memcpy(&hba->clk_scaling.saved_pwr_info.info, 1255 &hba->pwr_info, 1256 sizeof(struct ufs_pa_layer_attr)); 1257 1258 /* scale down gear */ 1259 new_pwr_info.gear_tx = hba->clk_scaling.min_gear; 1260 new_pwr_info.gear_rx = hba->clk_scaling.min_gear; 1261 } 1262 } 1263 1264 /* check if the power mode needs to be changed or not? */ 1265 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info); 1266 if (ret) 1267 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)", 1268 __func__, ret, 1269 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx, 1270 new_pwr_info.gear_tx, new_pwr_info.gear_rx); 1271 1272 return ret; 1273 } 1274 1275 /* 1276 * Wait until all pending SCSI commands and TMFs have finished or the timeout 1277 * has expired. 1278 * 1279 * Return: 0 upon success; -EBUSY upon timeout. 1280 */ 1281 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) 1282 { 1283 int ret = 0; 1284 /* 1285 * make sure that there are no outstanding requests when 1286 * clock scaling is in progress 1287 */ 1288 ufshcd_scsi_block_requests(hba); 1289 mutex_lock(&hba->wb_mutex); 1290 down_write(&hba->clk_scaling_lock); 1291 1292 if (!hba->clk_scaling.is_allowed || 1293 ufshcd_wait_for_doorbell_clr(hba, timeout_us)) { 1294 ret = -EBUSY; 1295 up_write(&hba->clk_scaling_lock); 1296 mutex_unlock(&hba->wb_mutex); 1297 ufshcd_scsi_unblock_requests(hba); 1298 goto out; 1299 } 1300 1301 /* let's not get into low power until clock scaling is completed */ 1302 ufshcd_hold(hba, false); 1303 1304 out: 1305 return ret; 1306 } 1307 1308 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up) 1309 { 1310 up_write(&hba->clk_scaling_lock); 1311 1312 /* Enable Write Booster if we have scaled up else disable it */ 1313 if (ufshcd_enable_wb_if_scaling_up(hba) && !err) 1314 ufshcd_wb_toggle(hba, scale_up); 1315 1316 mutex_unlock(&hba->wb_mutex); 1317 1318 ufshcd_scsi_unblock_requests(hba); 1319 ufshcd_release(hba); 1320 } 1321 1322 /** 1323 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear 1324 * @hba: per adapter instance 1325 * @scale_up: True for scaling up and false for scalin down 1326 * 1327 * Returns 0 for success, 1328 * Returns -EBUSY if scaling can't happen at this time 1329 * Returns non-zero for any other errors 1330 */ 1331 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up) 1332 { 1333 int ret = 0; 1334 1335 ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); 1336 if (ret) 1337 return ret; 1338 1339 /* scale down the gear before scaling down clocks */ 1340 if (!scale_up) { 1341 ret = ufshcd_scale_gear(hba, false); 1342 if (ret) 1343 goto out_unprepare; 1344 } 1345 1346 ret = ufshcd_scale_clks(hba, scale_up); 1347 if (ret) { 1348 if (!scale_up) 1349 ufshcd_scale_gear(hba, true); 1350 goto out_unprepare; 1351 } 1352 1353 /* scale up the gear after scaling up clocks */ 1354 if (scale_up) { 1355 ret = ufshcd_scale_gear(hba, true); 1356 if (ret) { 1357 ufshcd_scale_clks(hba, false); 1358 goto out_unprepare; 1359 } 1360 } 1361 1362 out_unprepare: 1363 ufshcd_clock_scaling_unprepare(hba, ret, scale_up); 1364 return ret; 1365 } 1366 1367 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) 1368 { 1369 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1370 clk_scaling.suspend_work); 1371 unsigned long irq_flags; 1372 1373 spin_lock_irqsave(hba->host->host_lock, irq_flags); 1374 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) { 1375 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1376 return; 1377 } 1378 hba->clk_scaling.is_suspended = true; 1379 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1380 1381 __ufshcd_suspend_clkscaling(hba); 1382 } 1383 1384 static void ufshcd_clk_scaling_resume_work(struct work_struct *work) 1385 { 1386 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1387 clk_scaling.resume_work); 1388 unsigned long irq_flags; 1389 1390 spin_lock_irqsave(hba->host->host_lock, irq_flags); 1391 if (!hba->clk_scaling.is_suspended) { 1392 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1393 return; 1394 } 1395 hba->clk_scaling.is_suspended = false; 1396 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1397 1398 devfreq_resume_device(hba->devfreq); 1399 } 1400 1401 static int ufshcd_devfreq_target(struct device *dev, 1402 unsigned long *freq, u32 flags) 1403 { 1404 int ret = 0; 1405 struct ufs_hba *hba = dev_get_drvdata(dev); 1406 ktime_t start; 1407 bool scale_up, sched_clk_scaling_suspend_work = false; 1408 struct list_head *clk_list = &hba->clk_list_head; 1409 struct ufs_clk_info *clki; 1410 unsigned long irq_flags; 1411 1412 if (!ufshcd_is_clkscaling_supported(hba)) 1413 return -EINVAL; 1414 1415 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); 1416 /* Override with the closest supported frequency */ 1417 *freq = (unsigned long) clk_round_rate(clki->clk, *freq); 1418 spin_lock_irqsave(hba->host->host_lock, irq_flags); 1419 if (ufshcd_eh_in_progress(hba)) { 1420 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1421 return 0; 1422 } 1423 1424 if (!hba->clk_scaling.active_reqs) 1425 sched_clk_scaling_suspend_work = true; 1426 1427 if (list_empty(clk_list)) { 1428 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1429 goto out; 1430 } 1431 1432 /* Decide based on the rounded-off frequency and update */ 1433 scale_up = *freq == clki->max_freq; 1434 if (!scale_up) 1435 *freq = clki->min_freq; 1436 /* Update the frequency */ 1437 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) { 1438 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1439 ret = 0; 1440 goto out; /* no state change required */ 1441 } 1442 spin_unlock_irqrestore(hba->host->host_lock, irq_flags); 1443 1444 start = ktime_get(); 1445 ret = ufshcd_devfreq_scale(hba, scale_up); 1446 1447 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), 1448 (scale_up ? "up" : "down"), 1449 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 1450 1451 out: 1452 if (sched_clk_scaling_suspend_work) 1453 queue_work(hba->clk_scaling.workq, 1454 &hba->clk_scaling.suspend_work); 1455 1456 return ret; 1457 } 1458 1459 static int ufshcd_devfreq_get_dev_status(struct device *dev, 1460 struct devfreq_dev_status *stat) 1461 { 1462 struct ufs_hba *hba = dev_get_drvdata(dev); 1463 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 1464 unsigned long flags; 1465 struct list_head *clk_list = &hba->clk_list_head; 1466 struct ufs_clk_info *clki; 1467 ktime_t curr_t; 1468 1469 if (!ufshcd_is_clkscaling_supported(hba)) 1470 return -EINVAL; 1471 1472 memset(stat, 0, sizeof(*stat)); 1473 1474 spin_lock_irqsave(hba->host->host_lock, flags); 1475 curr_t = ktime_get(); 1476 if (!scaling->window_start_t) 1477 goto start_window; 1478 1479 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1480 /* 1481 * If current frequency is 0, then the ondemand governor considers 1482 * there's no initial frequency set. And it always requests to set 1483 * to max. frequency. 1484 */ 1485 stat->current_frequency = clki->curr_freq; 1486 if (scaling->is_busy_started) 1487 scaling->tot_busy_t += ktime_us_delta(curr_t, 1488 scaling->busy_start_t); 1489 1490 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); 1491 stat->busy_time = scaling->tot_busy_t; 1492 start_window: 1493 scaling->window_start_t = curr_t; 1494 scaling->tot_busy_t = 0; 1495 1496 if (scaling->active_reqs) { 1497 scaling->busy_start_t = curr_t; 1498 scaling->is_busy_started = true; 1499 } else { 1500 scaling->busy_start_t = 0; 1501 scaling->is_busy_started = false; 1502 } 1503 spin_unlock_irqrestore(hba->host->host_lock, flags); 1504 return 0; 1505 } 1506 1507 static int ufshcd_devfreq_init(struct ufs_hba *hba) 1508 { 1509 struct list_head *clk_list = &hba->clk_list_head; 1510 struct ufs_clk_info *clki; 1511 struct devfreq *devfreq; 1512 int ret; 1513 1514 /* Skip devfreq if we don't have any clocks in the list */ 1515 if (list_empty(clk_list)) 1516 return 0; 1517 1518 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1519 dev_pm_opp_add(hba->dev, clki->min_freq, 0); 1520 dev_pm_opp_add(hba->dev, clki->max_freq, 0); 1521 1522 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile, 1523 &hba->vps->ondemand_data); 1524 devfreq = devfreq_add_device(hba->dev, 1525 &hba->vps->devfreq_profile, 1526 DEVFREQ_GOV_SIMPLE_ONDEMAND, 1527 &hba->vps->ondemand_data); 1528 if (IS_ERR(devfreq)) { 1529 ret = PTR_ERR(devfreq); 1530 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret); 1531 1532 dev_pm_opp_remove(hba->dev, clki->min_freq); 1533 dev_pm_opp_remove(hba->dev, clki->max_freq); 1534 return ret; 1535 } 1536 1537 hba->devfreq = devfreq; 1538 1539 return 0; 1540 } 1541 1542 static void ufshcd_devfreq_remove(struct ufs_hba *hba) 1543 { 1544 struct list_head *clk_list = &hba->clk_list_head; 1545 struct ufs_clk_info *clki; 1546 1547 if (!hba->devfreq) 1548 return; 1549 1550 devfreq_remove_device(hba->devfreq); 1551 hba->devfreq = NULL; 1552 1553 clki = list_first_entry(clk_list, struct ufs_clk_info, list); 1554 dev_pm_opp_remove(hba->dev, clki->min_freq); 1555 dev_pm_opp_remove(hba->dev, clki->max_freq); 1556 } 1557 1558 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba) 1559 { 1560 unsigned long flags; 1561 1562 devfreq_suspend_device(hba->devfreq); 1563 spin_lock_irqsave(hba->host->host_lock, flags); 1564 hba->clk_scaling.window_start_t = 0; 1565 spin_unlock_irqrestore(hba->host->host_lock, flags); 1566 } 1567 1568 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) 1569 { 1570 unsigned long flags; 1571 bool suspend = false; 1572 1573 cancel_work_sync(&hba->clk_scaling.suspend_work); 1574 cancel_work_sync(&hba->clk_scaling.resume_work); 1575 1576 spin_lock_irqsave(hba->host->host_lock, flags); 1577 if (!hba->clk_scaling.is_suspended) { 1578 suspend = true; 1579 hba->clk_scaling.is_suspended = true; 1580 } 1581 spin_unlock_irqrestore(hba->host->host_lock, flags); 1582 1583 if (suspend) 1584 __ufshcd_suspend_clkscaling(hba); 1585 } 1586 1587 static void ufshcd_resume_clkscaling(struct ufs_hba *hba) 1588 { 1589 unsigned long flags; 1590 bool resume = false; 1591 1592 spin_lock_irqsave(hba->host->host_lock, flags); 1593 if (hba->clk_scaling.is_suspended) { 1594 resume = true; 1595 hba->clk_scaling.is_suspended = false; 1596 } 1597 spin_unlock_irqrestore(hba->host->host_lock, flags); 1598 1599 if (resume) 1600 devfreq_resume_device(hba->devfreq); 1601 } 1602 1603 static ssize_t ufshcd_clkscale_enable_show(struct device *dev, 1604 struct device_attribute *attr, char *buf) 1605 { 1606 struct ufs_hba *hba = dev_get_drvdata(dev); 1607 1608 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled); 1609 } 1610 1611 static ssize_t ufshcd_clkscale_enable_store(struct device *dev, 1612 struct device_attribute *attr, const char *buf, size_t count) 1613 { 1614 struct ufs_hba *hba = dev_get_drvdata(dev); 1615 u32 value; 1616 int err = 0; 1617 1618 if (kstrtou32(buf, 0, &value)) 1619 return -EINVAL; 1620 1621 down(&hba->host_sem); 1622 if (!ufshcd_is_user_access_allowed(hba)) { 1623 err = -EBUSY; 1624 goto out; 1625 } 1626 1627 value = !!value; 1628 if (value == hba->clk_scaling.is_enabled) 1629 goto out; 1630 1631 ufshcd_rpm_get_sync(hba); 1632 ufshcd_hold(hba, false); 1633 1634 hba->clk_scaling.is_enabled = value; 1635 1636 if (value) { 1637 ufshcd_resume_clkscaling(hba); 1638 } else { 1639 ufshcd_suspend_clkscaling(hba); 1640 err = ufshcd_devfreq_scale(hba, true); 1641 if (err) 1642 dev_err(hba->dev, "%s: failed to scale clocks up %d\n", 1643 __func__, err); 1644 } 1645 1646 ufshcd_release(hba); 1647 ufshcd_rpm_put_sync(hba); 1648 out: 1649 up(&hba->host_sem); 1650 return err ? err : count; 1651 } 1652 1653 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba) 1654 { 1655 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show; 1656 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store; 1657 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr); 1658 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable"; 1659 hba->clk_scaling.enable_attr.attr.mode = 0644; 1660 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr)) 1661 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n"); 1662 } 1663 1664 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba) 1665 { 1666 if (hba->clk_scaling.enable_attr.attr.name) 1667 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr); 1668 } 1669 1670 static void ufshcd_init_clk_scaling(struct ufs_hba *hba) 1671 { 1672 char wq_name[sizeof("ufs_clkscaling_00")]; 1673 1674 if (!ufshcd_is_clkscaling_supported(hba)) 1675 return; 1676 1677 if (!hba->clk_scaling.min_gear) 1678 hba->clk_scaling.min_gear = UFS_HS_G1; 1679 1680 INIT_WORK(&hba->clk_scaling.suspend_work, 1681 ufshcd_clk_scaling_suspend_work); 1682 INIT_WORK(&hba->clk_scaling.resume_work, 1683 ufshcd_clk_scaling_resume_work); 1684 1685 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d", 1686 hba->host->host_no); 1687 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name); 1688 1689 hba->clk_scaling.is_initialized = true; 1690 } 1691 1692 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) 1693 { 1694 if (!hba->clk_scaling.is_initialized) 1695 return; 1696 1697 ufshcd_remove_clk_scaling_sysfs(hba); 1698 destroy_workqueue(hba->clk_scaling.workq); 1699 ufshcd_devfreq_remove(hba); 1700 hba->clk_scaling.is_initialized = false; 1701 } 1702 1703 static void ufshcd_ungate_work(struct work_struct *work) 1704 { 1705 int ret; 1706 unsigned long flags; 1707 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1708 clk_gating.ungate_work); 1709 1710 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 1711 1712 spin_lock_irqsave(hba->host->host_lock, flags); 1713 if (hba->clk_gating.state == CLKS_ON) { 1714 spin_unlock_irqrestore(hba->host->host_lock, flags); 1715 goto unblock_reqs; 1716 } 1717 1718 spin_unlock_irqrestore(hba->host->host_lock, flags); 1719 ufshcd_hba_vreg_set_hpm(hba); 1720 ufshcd_setup_clocks(hba, true); 1721 1722 ufshcd_enable_irq(hba); 1723 1724 /* Exit from hibern8 */ 1725 if (ufshcd_can_hibern8_during_gating(hba)) { 1726 /* Prevent gating in this path */ 1727 hba->clk_gating.is_suspended = true; 1728 if (ufshcd_is_link_hibern8(hba)) { 1729 ret = ufshcd_uic_hibern8_exit(hba); 1730 if (ret) 1731 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 1732 __func__, ret); 1733 else 1734 ufshcd_set_link_active(hba); 1735 } 1736 hba->clk_gating.is_suspended = false; 1737 } 1738 unblock_reqs: 1739 ufshcd_scsi_unblock_requests(hba); 1740 } 1741 1742 /** 1743 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. 1744 * Also, exit from hibern8 mode and set the link as active. 1745 * @hba: per adapter instance 1746 * @async: This indicates whether caller should ungate clocks asynchronously. 1747 */ 1748 int ufshcd_hold(struct ufs_hba *hba, bool async) 1749 { 1750 int rc = 0; 1751 bool flush_result; 1752 unsigned long flags; 1753 1754 if (!ufshcd_is_clkgating_allowed(hba) || 1755 !hba->clk_gating.is_initialized) 1756 goto out; 1757 spin_lock_irqsave(hba->host->host_lock, flags); 1758 hba->clk_gating.active_reqs++; 1759 1760 start: 1761 switch (hba->clk_gating.state) { 1762 case CLKS_ON: 1763 /* 1764 * Wait for the ungate work to complete if in progress. 1765 * Though the clocks may be in ON state, the link could 1766 * still be in hibner8 state if hibern8 is allowed 1767 * during clock gating. 1768 * Make sure we exit hibern8 state also in addition to 1769 * clocks being ON. 1770 */ 1771 if (ufshcd_can_hibern8_during_gating(hba) && 1772 ufshcd_is_link_hibern8(hba)) { 1773 if (async) { 1774 rc = -EAGAIN; 1775 hba->clk_gating.active_reqs--; 1776 break; 1777 } 1778 spin_unlock_irqrestore(hba->host->host_lock, flags); 1779 flush_result = flush_work(&hba->clk_gating.ungate_work); 1780 if (hba->clk_gating.is_suspended && !flush_result) 1781 goto out; 1782 spin_lock_irqsave(hba->host->host_lock, flags); 1783 goto start; 1784 } 1785 break; 1786 case REQ_CLKS_OFF: 1787 if (cancel_delayed_work(&hba->clk_gating.gate_work)) { 1788 hba->clk_gating.state = CLKS_ON; 1789 trace_ufshcd_clk_gating(dev_name(hba->dev), 1790 hba->clk_gating.state); 1791 break; 1792 } 1793 /* 1794 * If we are here, it means gating work is either done or 1795 * currently running. Hence, fall through to cancel gating 1796 * work and to enable clocks. 1797 */ 1798 fallthrough; 1799 case CLKS_OFF: 1800 hba->clk_gating.state = REQ_CLKS_ON; 1801 trace_ufshcd_clk_gating(dev_name(hba->dev), 1802 hba->clk_gating.state); 1803 if (queue_work(hba->clk_gating.clk_gating_workq, 1804 &hba->clk_gating.ungate_work)) 1805 ufshcd_scsi_block_requests(hba); 1806 /* 1807 * fall through to check if we should wait for this 1808 * work to be done or not. 1809 */ 1810 fallthrough; 1811 case REQ_CLKS_ON: 1812 if (async) { 1813 rc = -EAGAIN; 1814 hba->clk_gating.active_reqs--; 1815 break; 1816 } 1817 1818 spin_unlock_irqrestore(hba->host->host_lock, flags); 1819 flush_work(&hba->clk_gating.ungate_work); 1820 /* Make sure state is CLKS_ON before returning */ 1821 spin_lock_irqsave(hba->host->host_lock, flags); 1822 goto start; 1823 default: 1824 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", 1825 __func__, hba->clk_gating.state); 1826 break; 1827 } 1828 spin_unlock_irqrestore(hba->host->host_lock, flags); 1829 out: 1830 return rc; 1831 } 1832 EXPORT_SYMBOL_GPL(ufshcd_hold); 1833 1834 static void ufshcd_gate_work(struct work_struct *work) 1835 { 1836 struct ufs_hba *hba = container_of(work, struct ufs_hba, 1837 clk_gating.gate_work.work); 1838 unsigned long flags; 1839 int ret; 1840 1841 spin_lock_irqsave(hba->host->host_lock, flags); 1842 /* 1843 * In case you are here to cancel this work the gating state 1844 * would be marked as REQ_CLKS_ON. In this case save time by 1845 * skipping the gating work and exit after changing the clock 1846 * state to CLKS_ON. 1847 */ 1848 if (hba->clk_gating.is_suspended || 1849 (hba->clk_gating.state != REQ_CLKS_OFF)) { 1850 hba->clk_gating.state = CLKS_ON; 1851 trace_ufshcd_clk_gating(dev_name(hba->dev), 1852 hba->clk_gating.state); 1853 goto rel_lock; 1854 } 1855 1856 if (hba->clk_gating.active_reqs 1857 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL 1858 || hba->outstanding_reqs || hba->outstanding_tasks 1859 || hba->active_uic_cmd || hba->uic_async_done) 1860 goto rel_lock; 1861 1862 spin_unlock_irqrestore(hba->host->host_lock, flags); 1863 1864 /* put the link into hibern8 mode before turning off clocks */ 1865 if (ufshcd_can_hibern8_during_gating(hba)) { 1866 ret = ufshcd_uic_hibern8_enter(hba); 1867 if (ret) { 1868 hba->clk_gating.state = CLKS_ON; 1869 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 1870 __func__, ret); 1871 trace_ufshcd_clk_gating(dev_name(hba->dev), 1872 hba->clk_gating.state); 1873 goto out; 1874 } 1875 ufshcd_set_link_hibern8(hba); 1876 } 1877 1878 ufshcd_disable_irq(hba); 1879 1880 ufshcd_setup_clocks(hba, false); 1881 1882 /* Put the host controller in low power mode if possible */ 1883 ufshcd_hba_vreg_set_lpm(hba); 1884 /* 1885 * In case you are here to cancel this work the gating state 1886 * would be marked as REQ_CLKS_ON. In this case keep the state 1887 * as REQ_CLKS_ON which would anyway imply that clocks are off 1888 * and a request to turn them on is pending. By doing this way, 1889 * we keep the state machine in tact and this would ultimately 1890 * prevent from doing cancel work multiple times when there are 1891 * new requests arriving before the current cancel work is done. 1892 */ 1893 spin_lock_irqsave(hba->host->host_lock, flags); 1894 if (hba->clk_gating.state == REQ_CLKS_OFF) { 1895 hba->clk_gating.state = CLKS_OFF; 1896 trace_ufshcd_clk_gating(dev_name(hba->dev), 1897 hba->clk_gating.state); 1898 } 1899 rel_lock: 1900 spin_unlock_irqrestore(hba->host->host_lock, flags); 1901 out: 1902 return; 1903 } 1904 1905 /* host lock must be held before calling this variant */ 1906 static void __ufshcd_release(struct ufs_hba *hba) 1907 { 1908 if (!ufshcd_is_clkgating_allowed(hba)) 1909 return; 1910 1911 hba->clk_gating.active_reqs--; 1912 1913 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || 1914 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL || 1915 hba->outstanding_tasks || !hba->clk_gating.is_initialized || 1916 hba->active_uic_cmd || hba->uic_async_done || 1917 hba->clk_gating.state == CLKS_OFF) 1918 return; 1919 1920 hba->clk_gating.state = REQ_CLKS_OFF; 1921 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state); 1922 queue_delayed_work(hba->clk_gating.clk_gating_workq, 1923 &hba->clk_gating.gate_work, 1924 msecs_to_jiffies(hba->clk_gating.delay_ms)); 1925 } 1926 1927 void ufshcd_release(struct ufs_hba *hba) 1928 { 1929 unsigned long flags; 1930 1931 spin_lock_irqsave(hba->host->host_lock, flags); 1932 __ufshcd_release(hba); 1933 spin_unlock_irqrestore(hba->host->host_lock, flags); 1934 } 1935 EXPORT_SYMBOL_GPL(ufshcd_release); 1936 1937 static ssize_t ufshcd_clkgate_delay_show(struct device *dev, 1938 struct device_attribute *attr, char *buf) 1939 { 1940 struct ufs_hba *hba = dev_get_drvdata(dev); 1941 1942 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms); 1943 } 1944 1945 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) 1946 { 1947 struct ufs_hba *hba = dev_get_drvdata(dev); 1948 unsigned long flags; 1949 1950 spin_lock_irqsave(hba->host->host_lock, flags); 1951 hba->clk_gating.delay_ms = value; 1952 spin_unlock_irqrestore(hba->host->host_lock, flags); 1953 } 1954 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); 1955 1956 static ssize_t ufshcd_clkgate_delay_store(struct device *dev, 1957 struct device_attribute *attr, const char *buf, size_t count) 1958 { 1959 unsigned long value; 1960 1961 if (kstrtoul(buf, 0, &value)) 1962 return -EINVAL; 1963 1964 ufshcd_clkgate_delay_set(dev, value); 1965 return count; 1966 } 1967 1968 static ssize_t ufshcd_clkgate_enable_show(struct device *dev, 1969 struct device_attribute *attr, char *buf) 1970 { 1971 struct ufs_hba *hba = dev_get_drvdata(dev); 1972 1973 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled); 1974 } 1975 1976 static ssize_t ufshcd_clkgate_enable_store(struct device *dev, 1977 struct device_attribute *attr, const char *buf, size_t count) 1978 { 1979 struct ufs_hba *hba = dev_get_drvdata(dev); 1980 unsigned long flags; 1981 u32 value; 1982 1983 if (kstrtou32(buf, 0, &value)) 1984 return -EINVAL; 1985 1986 value = !!value; 1987 1988 spin_lock_irqsave(hba->host->host_lock, flags); 1989 if (value == hba->clk_gating.is_enabled) 1990 goto out; 1991 1992 if (value) 1993 __ufshcd_release(hba); 1994 else 1995 hba->clk_gating.active_reqs++; 1996 1997 hba->clk_gating.is_enabled = value; 1998 out: 1999 spin_unlock_irqrestore(hba->host->host_lock, flags); 2000 return count; 2001 } 2002 2003 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba) 2004 { 2005 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; 2006 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; 2007 sysfs_attr_init(&hba->clk_gating.delay_attr.attr); 2008 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; 2009 hba->clk_gating.delay_attr.attr.mode = 0644; 2010 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) 2011 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); 2012 2013 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show; 2014 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store; 2015 sysfs_attr_init(&hba->clk_gating.enable_attr.attr); 2016 hba->clk_gating.enable_attr.attr.name = "clkgate_enable"; 2017 hba->clk_gating.enable_attr.attr.mode = 0644; 2018 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr)) 2019 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n"); 2020 } 2021 2022 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba) 2023 { 2024 if (hba->clk_gating.delay_attr.attr.name) 2025 device_remove_file(hba->dev, &hba->clk_gating.delay_attr); 2026 if (hba->clk_gating.enable_attr.attr.name) 2027 device_remove_file(hba->dev, &hba->clk_gating.enable_attr); 2028 } 2029 2030 static void ufshcd_init_clk_gating(struct ufs_hba *hba) 2031 { 2032 char wq_name[sizeof("ufs_clk_gating_00")]; 2033 2034 if (!ufshcd_is_clkgating_allowed(hba)) 2035 return; 2036 2037 hba->clk_gating.state = CLKS_ON; 2038 2039 hba->clk_gating.delay_ms = 150; 2040 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); 2041 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); 2042 2043 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d", 2044 hba->host->host_no); 2045 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name, 2046 WQ_MEM_RECLAIM | WQ_HIGHPRI); 2047 2048 ufshcd_init_clk_gating_sysfs(hba); 2049 2050 hba->clk_gating.is_enabled = true; 2051 hba->clk_gating.is_initialized = true; 2052 } 2053 2054 static void ufshcd_exit_clk_gating(struct ufs_hba *hba) 2055 { 2056 if (!hba->clk_gating.is_initialized) 2057 return; 2058 2059 ufshcd_remove_clk_gating_sysfs(hba); 2060 2061 /* Ungate the clock if necessary. */ 2062 ufshcd_hold(hba, false); 2063 hba->clk_gating.is_initialized = false; 2064 ufshcd_release(hba); 2065 2066 destroy_workqueue(hba->clk_gating.clk_gating_workq); 2067 } 2068 2069 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) 2070 { 2071 bool queue_resume_work = false; 2072 ktime_t curr_t = ktime_get(); 2073 unsigned long flags; 2074 2075 if (!ufshcd_is_clkscaling_supported(hba)) 2076 return; 2077 2078 spin_lock_irqsave(hba->host->host_lock, flags); 2079 if (!hba->clk_scaling.active_reqs++) 2080 queue_resume_work = true; 2081 2082 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) { 2083 spin_unlock_irqrestore(hba->host->host_lock, flags); 2084 return; 2085 } 2086 2087 if (queue_resume_work) 2088 queue_work(hba->clk_scaling.workq, 2089 &hba->clk_scaling.resume_work); 2090 2091 if (!hba->clk_scaling.window_start_t) { 2092 hba->clk_scaling.window_start_t = curr_t; 2093 hba->clk_scaling.tot_busy_t = 0; 2094 hba->clk_scaling.is_busy_started = false; 2095 } 2096 2097 if (!hba->clk_scaling.is_busy_started) { 2098 hba->clk_scaling.busy_start_t = curr_t; 2099 hba->clk_scaling.is_busy_started = true; 2100 } 2101 spin_unlock_irqrestore(hba->host->host_lock, flags); 2102 } 2103 2104 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) 2105 { 2106 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 2107 unsigned long flags; 2108 2109 if (!ufshcd_is_clkscaling_supported(hba)) 2110 return; 2111 2112 spin_lock_irqsave(hba->host->host_lock, flags); 2113 hba->clk_scaling.active_reqs--; 2114 if (!scaling->active_reqs && scaling->is_busy_started) { 2115 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 2116 scaling->busy_start_t)); 2117 scaling->busy_start_t = 0; 2118 scaling->is_busy_started = false; 2119 } 2120 spin_unlock_irqrestore(hba->host->host_lock, flags); 2121 } 2122 2123 static inline int ufshcd_monitor_opcode2dir(u8 opcode) 2124 { 2125 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16) 2126 return READ; 2127 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16) 2128 return WRITE; 2129 else 2130 return -EINVAL; 2131 } 2132 2133 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, 2134 struct ufshcd_lrb *lrbp) 2135 { 2136 const struct ufs_hba_monitor *m = &hba->monitor; 2137 2138 return (m->enabled && lrbp && lrbp->cmd && 2139 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) && 2140 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp)); 2141 } 2142 2143 static void ufshcd_start_monitor(struct ufs_hba *hba, 2144 const struct ufshcd_lrb *lrbp) 2145 { 2146 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); 2147 unsigned long flags; 2148 2149 spin_lock_irqsave(hba->host->host_lock, flags); 2150 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0) 2151 hba->monitor.busy_start_ts[dir] = ktime_get(); 2152 spin_unlock_irqrestore(hba->host->host_lock, flags); 2153 } 2154 2155 static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp) 2156 { 2157 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); 2158 unsigned long flags; 2159 2160 spin_lock_irqsave(hba->host->host_lock, flags); 2161 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { 2162 const struct request *req = scsi_cmd_to_rq(lrbp->cmd); 2163 struct ufs_hba_monitor *m = &hba->monitor; 2164 ktime_t now, inc, lat; 2165 2166 now = lrbp->compl_time_stamp; 2167 inc = ktime_sub(now, m->busy_start_ts[dir]); 2168 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc); 2169 m->nr_sec_rw[dir] += blk_rq_sectors(req); 2170 2171 /* Update latencies */ 2172 m->nr_req[dir]++; 2173 lat = ktime_sub(now, lrbp->issue_time_stamp); 2174 m->lat_sum[dir] += lat; 2175 if (m->lat_max[dir] < lat || !m->lat_max[dir]) 2176 m->lat_max[dir] = lat; 2177 if (m->lat_min[dir] > lat || !m->lat_min[dir]) 2178 m->lat_min[dir] = lat; 2179 2180 m->nr_queued[dir]--; 2181 /* Push forward the busy start of monitor */ 2182 m->busy_start_ts[dir] = now; 2183 } 2184 spin_unlock_irqrestore(hba->host->host_lock, flags); 2185 } 2186 2187 /** 2188 * ufshcd_send_command - Send SCSI or device management commands 2189 * @hba: per adapter instance 2190 * @task_tag: Task tag of the command 2191 * @hwq: pointer to hardware queue instance 2192 */ 2193 static inline 2194 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, 2195 struct ufs_hw_queue *hwq) 2196 { 2197 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 2198 unsigned long flags; 2199 2200 lrbp->issue_time_stamp = ktime_get(); 2201 lrbp->issue_time_stamp_local_clock = local_clock(); 2202 lrbp->compl_time_stamp = ktime_set(0, 0); 2203 lrbp->compl_time_stamp_local_clock = 0; 2204 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); 2205 ufshcd_clk_scaling_start_busy(hba); 2206 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) 2207 ufshcd_start_monitor(hba, lrbp); 2208 2209 if (is_mcq_enabled(hba)) { 2210 int utrd_size = sizeof(struct utp_transfer_req_desc); 2211 2212 spin_lock(&hwq->sq_lock); 2213 memcpy(hwq->sqe_base_addr + (hwq->sq_tail_slot * utrd_size), 2214 lrbp->utr_descriptor_ptr, utrd_size); 2215 ufshcd_inc_sq_tail(hwq); 2216 spin_unlock(&hwq->sq_lock); 2217 } else { 2218 spin_lock_irqsave(&hba->outstanding_lock, flags); 2219 if (hba->vops && hba->vops->setup_xfer_req) 2220 hba->vops->setup_xfer_req(hba, lrbp->task_tag, 2221 !!lrbp->cmd); 2222 __set_bit(lrbp->task_tag, &hba->outstanding_reqs); 2223 ufshcd_writel(hba, 1 << lrbp->task_tag, 2224 REG_UTP_TRANSFER_REQ_DOOR_BELL); 2225 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 2226 } 2227 } 2228 2229 /** 2230 * ufshcd_copy_sense_data - Copy sense data in case of check condition 2231 * @lrbp: pointer to local reference block 2232 */ 2233 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) 2234 { 2235 u8 *const sense_buffer = lrbp->cmd->sense_buffer; 2236 int len; 2237 2238 if (sense_buffer && 2239 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { 2240 int len_to_copy; 2241 2242 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); 2243 len_to_copy = min_t(int, UFS_SENSE_SIZE, len); 2244 2245 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data, 2246 len_to_copy); 2247 } 2248 } 2249 2250 /** 2251 * ufshcd_copy_query_response() - Copy the Query Response and the data 2252 * descriptor 2253 * @hba: per adapter instance 2254 * @lrbp: pointer to local reference block 2255 */ 2256 static 2257 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2258 { 2259 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 2260 2261 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); 2262 2263 /* Get the descriptor */ 2264 if (hba->dev_cmd.query.descriptor && 2265 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { 2266 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + 2267 GENERAL_UPIU_REQUEST_SIZE; 2268 u16 resp_len; 2269 u16 buf_len; 2270 2271 /* data segment length */ 2272 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & 2273 MASK_QUERY_DATA_SEG_LEN; 2274 buf_len = be16_to_cpu( 2275 hba->dev_cmd.query.request.upiu_req.length); 2276 if (likely(buf_len >= resp_len)) { 2277 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); 2278 } else { 2279 dev_warn(hba->dev, 2280 "%s: rsp size %d is bigger than buffer size %d", 2281 __func__, resp_len, buf_len); 2282 return -EINVAL; 2283 } 2284 } 2285 2286 return 0; 2287 } 2288 2289 /** 2290 * ufshcd_hba_capabilities - Read controller capabilities 2291 * @hba: per adapter instance 2292 * 2293 * Return: 0 on success, negative on error. 2294 */ 2295 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) 2296 { 2297 int err; 2298 2299 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 2300 if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS) 2301 hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT; 2302 2303 /* nutrs and nutmrs are 0 based values */ 2304 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1; 2305 hba->nutmrs = 2306 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 2307 hba->reserved_slot = hba->nutrs - 1; 2308 2309 /* Read crypto capabilities */ 2310 err = ufshcd_hba_init_crypto_capabilities(hba); 2311 if (err) 2312 dev_err(hba->dev, "crypto setup failed\n"); 2313 2314 hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities); 2315 if (!hba->mcq_sup) 2316 return err; 2317 2318 hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); 2319 hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT, 2320 hba->mcq_capabilities); 2321 2322 return err; 2323 } 2324 2325 /** 2326 * ufshcd_ready_for_uic_cmd - Check if controller is ready 2327 * to accept UIC commands 2328 * @hba: per adapter instance 2329 * Return true on success, else false 2330 */ 2331 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) 2332 { 2333 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY; 2334 } 2335 2336 /** 2337 * ufshcd_get_upmcrs - Get the power mode change request status 2338 * @hba: Pointer to adapter instance 2339 * 2340 * This function gets the UPMCRS field of HCS register 2341 * Returns value of UPMCRS field 2342 */ 2343 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) 2344 { 2345 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; 2346 } 2347 2348 /** 2349 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer 2350 * @hba: per adapter instance 2351 * @uic_cmd: UIC command 2352 */ 2353 static inline void 2354 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2355 { 2356 lockdep_assert_held(&hba->uic_cmd_mutex); 2357 2358 WARN_ON(hba->active_uic_cmd); 2359 2360 hba->active_uic_cmd = uic_cmd; 2361 2362 /* Write Args */ 2363 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); 2364 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); 2365 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); 2366 2367 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND); 2368 2369 /* Write UIC Cmd */ 2370 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, 2371 REG_UIC_COMMAND); 2372 } 2373 2374 /** 2375 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command 2376 * @hba: per adapter instance 2377 * @uic_cmd: UIC command 2378 * 2379 * Returns 0 only if success. 2380 */ 2381 static int 2382 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2383 { 2384 int ret; 2385 unsigned long flags; 2386 2387 lockdep_assert_held(&hba->uic_cmd_mutex); 2388 2389 if (wait_for_completion_timeout(&uic_cmd->done, 2390 msecs_to_jiffies(UIC_CMD_TIMEOUT))) { 2391 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2392 } else { 2393 ret = -ETIMEDOUT; 2394 dev_err(hba->dev, 2395 "uic cmd 0x%x with arg3 0x%x completion timeout\n", 2396 uic_cmd->command, uic_cmd->argument3); 2397 2398 if (!uic_cmd->cmd_active) { 2399 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n", 2400 __func__); 2401 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 2402 } 2403 } 2404 2405 spin_lock_irqsave(hba->host->host_lock, flags); 2406 hba->active_uic_cmd = NULL; 2407 spin_unlock_irqrestore(hba->host->host_lock, flags); 2408 2409 return ret; 2410 } 2411 2412 /** 2413 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2414 * @hba: per adapter instance 2415 * @uic_cmd: UIC command 2416 * @completion: initialize the completion only if this is set to true 2417 * 2418 * Returns 0 only if success. 2419 */ 2420 static int 2421 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, 2422 bool completion) 2423 { 2424 lockdep_assert_held(&hba->uic_cmd_mutex); 2425 lockdep_assert_held(hba->host->host_lock); 2426 2427 if (!ufshcd_ready_for_uic_cmd(hba)) { 2428 dev_err(hba->dev, 2429 "Controller not ready to accept UIC commands\n"); 2430 return -EIO; 2431 } 2432 2433 if (completion) 2434 init_completion(&uic_cmd->done); 2435 2436 uic_cmd->cmd_active = 1; 2437 ufshcd_dispatch_uic_cmd(hba, uic_cmd); 2438 2439 return 0; 2440 } 2441 2442 /** 2443 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 2444 * @hba: per adapter instance 2445 * @uic_cmd: UIC command 2446 * 2447 * Returns 0 only if success. 2448 */ 2449 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 2450 { 2451 int ret; 2452 unsigned long flags; 2453 2454 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) 2455 return 0; 2456 2457 ufshcd_hold(hba, false); 2458 mutex_lock(&hba->uic_cmd_mutex); 2459 ufshcd_add_delay_before_dme_cmd(hba); 2460 2461 spin_lock_irqsave(hba->host->host_lock, flags); 2462 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true); 2463 spin_unlock_irqrestore(hba->host->host_lock, flags); 2464 if (!ret) 2465 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); 2466 2467 mutex_unlock(&hba->uic_cmd_mutex); 2468 2469 ufshcd_release(hba); 2470 return ret; 2471 } 2472 2473 /** 2474 * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format) 2475 * @hba: per-adapter instance 2476 * @lrbp: pointer to local reference block 2477 * @sg_entries: The number of sg lists actually used 2478 * @sg_list: Pointer to SG list 2479 */ 2480 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries, 2481 struct scatterlist *sg_list) 2482 { 2483 struct ufshcd_sg_entry *prd; 2484 struct scatterlist *sg; 2485 int i; 2486 2487 if (sg_entries) { 2488 2489 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) 2490 lrbp->utr_descriptor_ptr->prd_table_length = 2491 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba)); 2492 else 2493 lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries); 2494 2495 prd = lrbp->ucd_prdt_ptr; 2496 2497 for_each_sg(sg_list, sg, sg_entries, i) { 2498 const unsigned int len = sg_dma_len(sg); 2499 2500 /* 2501 * From the UFSHCI spec: "Data Byte Count (DBC): A '0' 2502 * based value that indicates the length, in bytes, of 2503 * the data block. A maximum of length of 256KB may 2504 * exist for any entry. Bits 1:0 of this field shall be 2505 * 11b to indicate Dword granularity. A value of '3' 2506 * indicates 4 bytes, '7' indicates 8 bytes, etc." 2507 */ 2508 WARN_ONCE(len > 256 * 1024, "len = %#x\n", len); 2509 prd->size = cpu_to_le32(len - 1); 2510 prd->addr = cpu_to_le64(sg->dma_address); 2511 prd->reserved = 0; 2512 prd = (void *)prd + ufshcd_sg_entry_size(hba); 2513 } 2514 } else { 2515 lrbp->utr_descriptor_ptr->prd_table_length = 0; 2516 } 2517 } 2518 2519 /** 2520 * ufshcd_map_sg - Map scatter-gather list to prdt 2521 * @hba: per adapter instance 2522 * @lrbp: pointer to local reference block 2523 * 2524 * Returns 0 in case of success, non-zero value in case of failure 2525 */ 2526 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2527 { 2528 struct scsi_cmnd *cmd = lrbp->cmd; 2529 int sg_segments = scsi_dma_map(cmd); 2530 2531 if (sg_segments < 0) 2532 return sg_segments; 2533 2534 ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); 2535 2536 return 0; 2537 } 2538 2539 /** 2540 * ufshcd_enable_intr - enable interrupts 2541 * @hba: per adapter instance 2542 * @intrs: interrupt bits 2543 */ 2544 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) 2545 { 2546 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 2547 2548 if (hba->ufs_version == ufshci_version(1, 0)) { 2549 u32 rw; 2550 rw = set & INTERRUPT_MASK_RW_VER_10; 2551 set = rw | ((set ^ intrs) & intrs); 2552 } else { 2553 set |= intrs; 2554 } 2555 2556 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); 2557 } 2558 2559 /** 2560 * ufshcd_disable_intr - disable interrupts 2561 * @hba: per adapter instance 2562 * @intrs: interrupt bits 2563 */ 2564 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) 2565 { 2566 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 2567 2568 if (hba->ufs_version == ufshci_version(1, 0)) { 2569 u32 rw; 2570 rw = (set & INTERRUPT_MASK_RW_VER_10) & 2571 ~(intrs & INTERRUPT_MASK_RW_VER_10); 2572 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10); 2573 2574 } else { 2575 set &= ~intrs; 2576 } 2577 2578 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); 2579 } 2580 2581 /** 2582 * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request 2583 * descriptor according to request 2584 * @lrbp: pointer to local reference block 2585 * @upiu_flags: flags required in the header 2586 * @cmd_dir: requests data direction 2587 * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments) 2588 */ 2589 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u8 *upiu_flags, 2590 enum dma_data_direction cmd_dir, int ehs_length) 2591 { 2592 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; 2593 u32 data_direction; 2594 u32 dword_0; 2595 u32 dword_1 = 0; 2596 u32 dword_3 = 0; 2597 2598 if (cmd_dir == DMA_FROM_DEVICE) { 2599 data_direction = UTP_DEVICE_TO_HOST; 2600 *upiu_flags = UPIU_CMD_FLAGS_READ; 2601 } else if (cmd_dir == DMA_TO_DEVICE) { 2602 data_direction = UTP_HOST_TO_DEVICE; 2603 *upiu_flags = UPIU_CMD_FLAGS_WRITE; 2604 } else { 2605 data_direction = UTP_NO_DATA_TRANSFER; 2606 *upiu_flags = UPIU_CMD_FLAGS_NONE; 2607 } 2608 2609 dword_0 = data_direction | (lrbp->command_type << UPIU_COMMAND_TYPE_OFFSET) | 2610 ehs_length << 8; 2611 if (lrbp->intr_cmd) 2612 dword_0 |= UTP_REQ_DESC_INT_CMD; 2613 2614 /* Prepare crypto related dwords */ 2615 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3); 2616 2617 /* Transfer request descriptor header fields */ 2618 req_desc->header.dword_0 = cpu_to_le32(dword_0); 2619 req_desc->header.dword_1 = cpu_to_le32(dword_1); 2620 /* 2621 * assigning invalid value for command status. Controller 2622 * updates OCS on command completion, with the command 2623 * status 2624 */ 2625 req_desc->header.dword_2 = 2626 cpu_to_le32(OCS_INVALID_COMMAND_STATUS); 2627 req_desc->header.dword_3 = cpu_to_le32(dword_3); 2628 2629 req_desc->prd_table_length = 0; 2630 } 2631 2632 /** 2633 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 2634 * for scsi commands 2635 * @lrbp: local reference block pointer 2636 * @upiu_flags: flags 2637 */ 2638 static 2639 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) 2640 { 2641 struct scsi_cmnd *cmd = lrbp->cmd; 2642 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2643 unsigned short cdb_len; 2644 2645 /* command descriptor fields */ 2646 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( 2647 UPIU_TRANSACTION_COMMAND, upiu_flags, 2648 lrbp->lun, lrbp->task_tag); 2649 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( 2650 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); 2651 2652 /* Total EHS length and Data segment length will be zero */ 2653 ucd_req_ptr->header.dword_2 = 0; 2654 2655 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); 2656 2657 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE); 2658 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE); 2659 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len); 2660 2661 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2662 } 2663 2664 /** 2665 * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request 2666 * @hba: UFS hba 2667 * @lrbp: local reference block pointer 2668 * @upiu_flags: flags 2669 */ 2670 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 2671 struct ufshcd_lrb *lrbp, u8 upiu_flags) 2672 { 2673 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2674 struct ufs_query *query = &hba->dev_cmd.query; 2675 u16 len = be16_to_cpu(query->request.upiu_req.length); 2676 2677 /* Query request header */ 2678 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( 2679 UPIU_TRANSACTION_QUERY_REQ, upiu_flags, 2680 lrbp->lun, lrbp->task_tag); 2681 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( 2682 0, query->request.query_func, 0, 0); 2683 2684 /* Data segment length only need for WRITE_DESC */ 2685 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 2686 ucd_req_ptr->header.dword_2 = 2687 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len); 2688 else 2689 ucd_req_ptr->header.dword_2 = 0; 2690 2691 /* Copy the Query Request buffer as is */ 2692 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, 2693 QUERY_OSF_SIZE); 2694 2695 /* Copy the Descriptor */ 2696 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 2697 memcpy(ucd_req_ptr + 1, query->descriptor, len); 2698 2699 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2700 } 2701 2702 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) 2703 { 2704 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 2705 2706 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 2707 2708 /* command descriptor fields */ 2709 ucd_req_ptr->header.dword_0 = 2710 UPIU_HEADER_DWORD( 2711 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); 2712 /* clear rest of the fields of basic header */ 2713 ucd_req_ptr->header.dword_1 = 0; 2714 ucd_req_ptr->header.dword_2 = 0; 2715 2716 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 2717 } 2718 2719 /** 2720 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) 2721 * for Device Management Purposes 2722 * @hba: per adapter instance 2723 * @lrbp: pointer to local reference block 2724 */ 2725 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, 2726 struct ufshcd_lrb *lrbp) 2727 { 2728 u8 upiu_flags; 2729 int ret = 0; 2730 2731 if (hba->ufs_version <= ufshci_version(1, 1)) 2732 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; 2733 else 2734 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 2735 2736 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE, 0); 2737 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 2738 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags); 2739 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 2740 ufshcd_prepare_utp_nop_upiu(lrbp); 2741 else 2742 ret = -EINVAL; 2743 2744 return ret; 2745 } 2746 2747 /** 2748 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) 2749 * for SCSI Purposes 2750 * @hba: per adapter instance 2751 * @lrbp: pointer to local reference block 2752 */ 2753 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2754 { 2755 u8 upiu_flags; 2756 int ret = 0; 2757 2758 if (hba->ufs_version <= ufshci_version(1, 1)) 2759 lrbp->command_type = UTP_CMD_TYPE_SCSI; 2760 else 2761 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 2762 2763 if (likely(lrbp->cmd)) { 2764 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0); 2765 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); 2766 } else { 2767 ret = -EINVAL; 2768 } 2769 2770 return ret; 2771 } 2772 2773 /** 2774 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID 2775 * @upiu_wlun_id: UPIU W-LUN id 2776 * 2777 * Returns SCSI W-LUN id 2778 */ 2779 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) 2780 { 2781 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE; 2782 } 2783 2784 static inline bool is_device_wlun(struct scsi_device *sdev) 2785 { 2786 return sdev->lun == 2787 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN); 2788 } 2789 2790 /* 2791 * Associate the UFS controller queue with the default and poll HCTX types. 2792 * Initialize the mq_map[] arrays. 2793 */ 2794 static void ufshcd_map_queues(struct Scsi_Host *shost) 2795 { 2796 struct ufs_hba *hba = shost_priv(shost); 2797 int i, queue_offset = 0; 2798 2799 if (!is_mcq_supported(hba)) { 2800 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1; 2801 hba->nr_queues[HCTX_TYPE_READ] = 0; 2802 hba->nr_queues[HCTX_TYPE_POLL] = 1; 2803 hba->nr_hw_queues = 1; 2804 } 2805 2806 for (i = 0; i < shost->nr_maps; i++) { 2807 struct blk_mq_queue_map *map = &shost->tag_set.map[i]; 2808 2809 map->nr_queues = hba->nr_queues[i]; 2810 if (!map->nr_queues) 2811 continue; 2812 map->queue_offset = queue_offset; 2813 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba)) 2814 map->queue_offset = 0; 2815 2816 blk_mq_map_queues(map); 2817 queue_offset += map->nr_queues; 2818 } 2819 } 2820 2821 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) 2822 { 2823 struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr + 2824 i * sizeof_utp_transfer_cmd_desc(hba); 2825 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; 2826 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr + 2827 i * sizeof_utp_transfer_cmd_desc(hba); 2828 u16 response_offset = offsetof(struct utp_transfer_cmd_desc, 2829 response_upiu); 2830 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table); 2831 2832 lrb->utr_descriptor_ptr = utrdlp + i; 2833 lrb->utrd_dma_addr = hba->utrdl_dma_addr + 2834 i * sizeof(struct utp_transfer_req_desc); 2835 lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; 2836 lrb->ucd_req_dma_addr = cmd_desc_element_addr; 2837 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; 2838 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; 2839 lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; 2840 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; 2841 } 2842 2843 /** 2844 * ufshcd_queuecommand - main entry point for SCSI requests 2845 * @host: SCSI host pointer 2846 * @cmd: command from SCSI Midlayer 2847 * 2848 * Returns 0 for success, non-zero in case of failure 2849 */ 2850 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) 2851 { 2852 struct ufs_hba *hba = shost_priv(host); 2853 int tag = scsi_cmd_to_rq(cmd)->tag; 2854 struct ufshcd_lrb *lrbp; 2855 int err = 0; 2856 struct ufs_hw_queue *hwq = NULL; 2857 2858 WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag); 2859 2860 /* 2861 * Allows the UFS error handler to wait for prior ufshcd_queuecommand() 2862 * calls. 2863 */ 2864 rcu_read_lock(); 2865 2866 switch (hba->ufshcd_state) { 2867 case UFSHCD_STATE_OPERATIONAL: 2868 break; 2869 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL: 2870 /* 2871 * SCSI error handler can call ->queuecommand() while UFS error 2872 * handler is in progress. Error interrupts could change the 2873 * state from UFSHCD_STATE_RESET to 2874 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests 2875 * being issued in that case. 2876 */ 2877 if (ufshcd_eh_in_progress(hba)) { 2878 err = SCSI_MLQUEUE_HOST_BUSY; 2879 goto out; 2880 } 2881 break; 2882 case UFSHCD_STATE_EH_SCHEDULED_FATAL: 2883 /* 2884 * pm_runtime_get_sync() is used at error handling preparation 2885 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's 2886 * PM ops, it can never be finished if we let SCSI layer keep 2887 * retrying it, which gets err handler stuck forever. Neither 2888 * can we let the scsi cmd pass through, because UFS is in bad 2889 * state, the scsi cmd may eventually time out, which will get 2890 * err handler blocked for too long. So, just fail the scsi cmd 2891 * sent from PM ops, err handler can recover PM error anyways. 2892 */ 2893 if (hba->pm_op_in_progress) { 2894 hba->force_reset = true; 2895 set_host_byte(cmd, DID_BAD_TARGET); 2896 scsi_done(cmd); 2897 goto out; 2898 } 2899 fallthrough; 2900 case UFSHCD_STATE_RESET: 2901 err = SCSI_MLQUEUE_HOST_BUSY; 2902 goto out; 2903 case UFSHCD_STATE_ERROR: 2904 set_host_byte(cmd, DID_ERROR); 2905 scsi_done(cmd); 2906 goto out; 2907 } 2908 2909 hba->req_abort_count = 0; 2910 2911 err = ufshcd_hold(hba, true); 2912 if (err) { 2913 err = SCSI_MLQUEUE_HOST_BUSY; 2914 goto out; 2915 } 2916 WARN_ON(ufshcd_is_clkgating_allowed(hba) && 2917 (hba->clk_gating.state != CLKS_ON)); 2918 2919 lrbp = &hba->lrb[tag]; 2920 WARN_ON(lrbp->cmd); 2921 lrbp->cmd = cmd; 2922 lrbp->task_tag = tag; 2923 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 2924 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); 2925 2926 ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp); 2927 2928 lrbp->req_abort_skip = false; 2929 2930 ufshpb_prep(hba, lrbp); 2931 2932 ufshcd_comp_scsi_upiu(hba, lrbp); 2933 2934 err = ufshcd_map_sg(hba, lrbp); 2935 if (err) { 2936 lrbp->cmd = NULL; 2937 ufshcd_release(hba); 2938 goto out; 2939 } 2940 2941 if (is_mcq_enabled(hba)) 2942 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 2943 2944 ufshcd_send_command(hba, tag, hwq); 2945 2946 out: 2947 rcu_read_unlock(); 2948 2949 if (ufs_trigger_eh()) { 2950 unsigned long flags; 2951 2952 spin_lock_irqsave(hba->host->host_lock, flags); 2953 ufshcd_schedule_eh_work(hba); 2954 spin_unlock_irqrestore(hba->host->host_lock, flags); 2955 } 2956 2957 return err; 2958 } 2959 2960 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, 2961 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) 2962 { 2963 lrbp->cmd = NULL; 2964 lrbp->task_tag = tag; 2965 lrbp->lun = 0; /* device management cmd is not specific to any LUN */ 2966 lrbp->intr_cmd = true; /* No interrupt aggregation */ 2967 ufshcd_prepare_lrbp_crypto(NULL, lrbp); 2968 hba->dev_cmd.type = cmd_type; 2969 2970 return ufshcd_compose_devman_upiu(hba, lrbp); 2971 } 2972 2973 /* 2974 * Clear all the requests from the controller for which a bit has been set in 2975 * @mask and wait until the controller confirms that these requests have been 2976 * cleared. 2977 */ 2978 static int ufshcd_clear_cmds(struct ufs_hba *hba, u32 mask) 2979 { 2980 unsigned long flags; 2981 2982 /* clear outstanding transaction before retry */ 2983 spin_lock_irqsave(hba->host->host_lock, flags); 2984 ufshcd_utrl_clear(hba, mask); 2985 spin_unlock_irqrestore(hba->host->host_lock, flags); 2986 2987 /* 2988 * wait for h/w to clear corresponding bit in door-bell. 2989 * max. wait is 1 sec. 2990 */ 2991 return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL, 2992 mask, ~mask, 1000, 1000); 2993 } 2994 2995 static int 2996 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2997 { 2998 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 2999 3000 /* Get the UPIU response */ 3001 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> 3002 UPIU_RSP_CODE_OFFSET; 3003 return query_res->response; 3004 } 3005 3006 /** 3007 * ufshcd_dev_cmd_completion() - handles device management command responses 3008 * @hba: per adapter instance 3009 * @lrbp: pointer to local reference block 3010 */ 3011 static int 3012 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 3013 { 3014 int resp; 3015 int err = 0; 3016 3017 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 3018 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 3019 3020 switch (resp) { 3021 case UPIU_TRANSACTION_NOP_IN: 3022 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { 3023 err = -EINVAL; 3024 dev_err(hba->dev, "%s: unexpected response %x\n", 3025 __func__, resp); 3026 } 3027 break; 3028 case UPIU_TRANSACTION_QUERY_RSP: 3029 err = ufshcd_check_query_response(hba, lrbp); 3030 if (!err) 3031 err = ufshcd_copy_query_response(hba, lrbp); 3032 break; 3033 case UPIU_TRANSACTION_REJECT_UPIU: 3034 /* TODO: handle Reject UPIU Response */ 3035 err = -EPERM; 3036 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", 3037 __func__); 3038 break; 3039 case UPIU_TRANSACTION_RESPONSE: 3040 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) { 3041 err = -EINVAL; 3042 dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp); 3043 } 3044 break; 3045 default: 3046 err = -EINVAL; 3047 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", 3048 __func__, resp); 3049 break; 3050 } 3051 3052 return err; 3053 } 3054 3055 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, 3056 struct ufshcd_lrb *lrbp, int max_timeout) 3057 { 3058 unsigned long time_left = msecs_to_jiffies(max_timeout); 3059 unsigned long flags; 3060 bool pending; 3061 int err; 3062 3063 retry: 3064 time_left = wait_for_completion_timeout(hba->dev_cmd.complete, 3065 time_left); 3066 3067 if (likely(time_left)) { 3068 /* 3069 * The completion handler called complete() and the caller of 3070 * this function still owns the @lrbp tag so the code below does 3071 * not trigger any race conditions. 3072 */ 3073 hba->dev_cmd.complete = NULL; 3074 err = ufshcd_get_tr_ocs(lrbp, hba->dev_cmd.cqe); 3075 if (!err) 3076 err = ufshcd_dev_cmd_completion(hba, lrbp); 3077 } else { 3078 err = -ETIMEDOUT; 3079 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", 3080 __func__, lrbp->task_tag); 3081 if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) { 3082 /* successfully cleared the command, retry if needed */ 3083 err = -EAGAIN; 3084 /* 3085 * Since clearing the command succeeded we also need to 3086 * clear the task tag bit from the outstanding_reqs 3087 * variable. 3088 */ 3089 spin_lock_irqsave(&hba->outstanding_lock, flags); 3090 pending = test_bit(lrbp->task_tag, 3091 &hba->outstanding_reqs); 3092 if (pending) { 3093 hba->dev_cmd.complete = NULL; 3094 __clear_bit(lrbp->task_tag, 3095 &hba->outstanding_reqs); 3096 } 3097 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 3098 3099 if (!pending) { 3100 /* 3101 * The completion handler ran while we tried to 3102 * clear the command. 3103 */ 3104 time_left = 1; 3105 goto retry; 3106 } 3107 } else { 3108 dev_err(hba->dev, "%s: failed to clear tag %d\n", 3109 __func__, lrbp->task_tag); 3110 3111 spin_lock_irqsave(&hba->outstanding_lock, flags); 3112 pending = test_bit(lrbp->task_tag, 3113 &hba->outstanding_reqs); 3114 if (pending) 3115 hba->dev_cmd.complete = NULL; 3116 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 3117 3118 if (!pending) { 3119 /* 3120 * The completion handler ran while we tried to 3121 * clear the command. 3122 */ 3123 time_left = 1; 3124 goto retry; 3125 } 3126 } 3127 } 3128 3129 return err; 3130 } 3131 3132 /** 3133 * ufshcd_exec_dev_cmd - API for sending device management requests 3134 * @hba: UFS hba 3135 * @cmd_type: specifies the type (NOP, Query...) 3136 * @timeout: timeout in milliseconds 3137 * 3138 * NOTE: Since there is only one available tag for device management commands, 3139 * it is expected you hold the hba->dev_cmd.lock mutex. 3140 */ 3141 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 3142 enum dev_cmd_type cmd_type, int timeout) 3143 { 3144 DECLARE_COMPLETION_ONSTACK(wait); 3145 const u32 tag = hba->reserved_slot; 3146 struct ufshcd_lrb *lrbp; 3147 int err; 3148 3149 /* Protects use of hba->reserved_slot. */ 3150 lockdep_assert_held(&hba->dev_cmd.lock); 3151 3152 down_read(&hba->clk_scaling_lock); 3153 3154 lrbp = &hba->lrb[tag]; 3155 WARN_ON(lrbp->cmd); 3156 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); 3157 if (unlikely(err)) 3158 goto out; 3159 3160 hba->dev_cmd.complete = &wait; 3161 hba->dev_cmd.cqe = NULL; 3162 3163 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 3164 3165 ufshcd_send_command(hba, tag, hba->dev_cmd_queue); 3166 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); 3167 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, 3168 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 3169 3170 out: 3171 up_read(&hba->clk_scaling_lock); 3172 return err; 3173 } 3174 3175 /** 3176 * ufshcd_init_query() - init the query response and request parameters 3177 * @hba: per-adapter instance 3178 * @request: address of the request pointer to be initialized 3179 * @response: address of the response pointer to be initialized 3180 * @opcode: operation to perform 3181 * @idn: flag idn to access 3182 * @index: LU number to access 3183 * @selector: query/flag/descriptor further identification 3184 */ 3185 static inline void ufshcd_init_query(struct ufs_hba *hba, 3186 struct ufs_query_req **request, struct ufs_query_res **response, 3187 enum query_opcode opcode, u8 idn, u8 index, u8 selector) 3188 { 3189 *request = &hba->dev_cmd.query.request; 3190 *response = &hba->dev_cmd.query.response; 3191 memset(*request, 0, sizeof(struct ufs_query_req)); 3192 memset(*response, 0, sizeof(struct ufs_query_res)); 3193 (*request)->upiu_req.opcode = opcode; 3194 (*request)->upiu_req.idn = idn; 3195 (*request)->upiu_req.index = index; 3196 (*request)->upiu_req.selector = selector; 3197 } 3198 3199 static int ufshcd_query_flag_retry(struct ufs_hba *hba, 3200 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) 3201 { 3202 int ret; 3203 int retries; 3204 3205 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) { 3206 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res); 3207 if (ret) 3208 dev_dbg(hba->dev, 3209 "%s: failed with error %d, retries %d\n", 3210 __func__, ret, retries); 3211 else 3212 break; 3213 } 3214 3215 if (ret) 3216 dev_err(hba->dev, 3217 "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n", 3218 __func__, opcode, idn, ret, retries); 3219 return ret; 3220 } 3221 3222 /** 3223 * ufshcd_query_flag() - API function for sending flag query requests 3224 * @hba: per-adapter instance 3225 * @opcode: flag query to perform 3226 * @idn: flag idn to access 3227 * @index: flag index to access 3228 * @flag_res: the flag value after the query request completes 3229 * 3230 * Returns 0 for success, non-zero in case of failure 3231 */ 3232 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 3233 enum flag_idn idn, u8 index, bool *flag_res) 3234 { 3235 struct ufs_query_req *request = NULL; 3236 struct ufs_query_res *response = NULL; 3237 int err, selector = 0; 3238 int timeout = QUERY_REQ_TIMEOUT; 3239 3240 BUG_ON(!hba); 3241 3242 ufshcd_hold(hba, false); 3243 mutex_lock(&hba->dev_cmd.lock); 3244 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3245 selector); 3246 3247 switch (opcode) { 3248 case UPIU_QUERY_OPCODE_SET_FLAG: 3249 case UPIU_QUERY_OPCODE_CLEAR_FLAG: 3250 case UPIU_QUERY_OPCODE_TOGGLE_FLAG: 3251 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3252 break; 3253 case UPIU_QUERY_OPCODE_READ_FLAG: 3254 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3255 if (!flag_res) { 3256 /* No dummy reads */ 3257 dev_err(hba->dev, "%s: Invalid argument for read request\n", 3258 __func__); 3259 err = -EINVAL; 3260 goto out_unlock; 3261 } 3262 break; 3263 default: 3264 dev_err(hba->dev, 3265 "%s: Expected query flag opcode but got = %d\n", 3266 __func__, opcode); 3267 err = -EINVAL; 3268 goto out_unlock; 3269 } 3270 3271 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout); 3272 3273 if (err) { 3274 dev_err(hba->dev, 3275 "%s: Sending flag query for idn %d failed, err = %d\n", 3276 __func__, idn, err); 3277 goto out_unlock; 3278 } 3279 3280 if (flag_res) 3281 *flag_res = (be32_to_cpu(response->upiu_res.value) & 3282 MASK_QUERY_UPIU_FLAG_LOC) & 0x1; 3283 3284 out_unlock: 3285 mutex_unlock(&hba->dev_cmd.lock); 3286 ufshcd_release(hba); 3287 return err; 3288 } 3289 3290 /** 3291 * ufshcd_query_attr - API function for sending attribute requests 3292 * @hba: per-adapter instance 3293 * @opcode: attribute opcode 3294 * @idn: attribute idn to access 3295 * @index: index field 3296 * @selector: selector field 3297 * @attr_val: the attribute value after the query request completes 3298 * 3299 * Returns 0 for success, non-zero in case of failure 3300 */ 3301 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 3302 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) 3303 { 3304 struct ufs_query_req *request = NULL; 3305 struct ufs_query_res *response = NULL; 3306 int err; 3307 3308 BUG_ON(!hba); 3309 3310 if (!attr_val) { 3311 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", 3312 __func__, opcode); 3313 return -EINVAL; 3314 } 3315 3316 ufshcd_hold(hba, false); 3317 3318 mutex_lock(&hba->dev_cmd.lock); 3319 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3320 selector); 3321 3322 switch (opcode) { 3323 case UPIU_QUERY_OPCODE_WRITE_ATTR: 3324 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3325 request->upiu_req.value = cpu_to_be32(*attr_val); 3326 break; 3327 case UPIU_QUERY_OPCODE_READ_ATTR: 3328 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3329 break; 3330 default: 3331 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", 3332 __func__, opcode); 3333 err = -EINVAL; 3334 goto out_unlock; 3335 } 3336 3337 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 3338 3339 if (err) { 3340 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3341 __func__, opcode, idn, index, err); 3342 goto out_unlock; 3343 } 3344 3345 *attr_val = be32_to_cpu(response->upiu_res.value); 3346 3347 out_unlock: 3348 mutex_unlock(&hba->dev_cmd.lock); 3349 ufshcd_release(hba); 3350 return err; 3351 } 3352 3353 /** 3354 * ufshcd_query_attr_retry() - API function for sending query 3355 * attribute with retries 3356 * @hba: per-adapter instance 3357 * @opcode: attribute opcode 3358 * @idn: attribute idn to access 3359 * @index: index field 3360 * @selector: selector field 3361 * @attr_val: the attribute value after the query request 3362 * completes 3363 * 3364 * Returns 0 for success, non-zero in case of failure 3365 */ 3366 int ufshcd_query_attr_retry(struct ufs_hba *hba, 3367 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, 3368 u32 *attr_val) 3369 { 3370 int ret = 0; 3371 u32 retries; 3372 3373 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3374 ret = ufshcd_query_attr(hba, opcode, idn, index, 3375 selector, attr_val); 3376 if (ret) 3377 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n", 3378 __func__, ret, retries); 3379 else 3380 break; 3381 } 3382 3383 if (ret) 3384 dev_err(hba->dev, 3385 "%s: query attribute, idn %d, failed with error %d after %d retries\n", 3386 __func__, idn, ret, QUERY_REQ_RETRIES); 3387 return ret; 3388 } 3389 3390 static int __ufshcd_query_descriptor(struct ufs_hba *hba, 3391 enum query_opcode opcode, enum desc_idn idn, u8 index, 3392 u8 selector, u8 *desc_buf, int *buf_len) 3393 { 3394 struct ufs_query_req *request = NULL; 3395 struct ufs_query_res *response = NULL; 3396 int err; 3397 3398 BUG_ON(!hba); 3399 3400 if (!desc_buf) { 3401 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", 3402 __func__, opcode); 3403 return -EINVAL; 3404 } 3405 3406 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { 3407 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", 3408 __func__, *buf_len); 3409 return -EINVAL; 3410 } 3411 3412 ufshcd_hold(hba, false); 3413 3414 mutex_lock(&hba->dev_cmd.lock); 3415 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 3416 selector); 3417 hba->dev_cmd.query.descriptor = desc_buf; 3418 request->upiu_req.length = cpu_to_be16(*buf_len); 3419 3420 switch (opcode) { 3421 case UPIU_QUERY_OPCODE_WRITE_DESC: 3422 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 3423 break; 3424 case UPIU_QUERY_OPCODE_READ_DESC: 3425 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 3426 break; 3427 default: 3428 dev_err(hba->dev, 3429 "%s: Expected query descriptor opcode but got = 0x%.2x\n", 3430 __func__, opcode); 3431 err = -EINVAL; 3432 goto out_unlock; 3433 } 3434 3435 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 3436 3437 if (err) { 3438 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", 3439 __func__, opcode, idn, index, err); 3440 goto out_unlock; 3441 } 3442 3443 *buf_len = be16_to_cpu(response->upiu_res.length); 3444 3445 out_unlock: 3446 hba->dev_cmd.query.descriptor = NULL; 3447 mutex_unlock(&hba->dev_cmd.lock); 3448 ufshcd_release(hba); 3449 return err; 3450 } 3451 3452 /** 3453 * ufshcd_query_descriptor_retry - API function for sending descriptor requests 3454 * @hba: per-adapter instance 3455 * @opcode: attribute opcode 3456 * @idn: attribute idn to access 3457 * @index: index field 3458 * @selector: selector field 3459 * @desc_buf: the buffer that contains the descriptor 3460 * @buf_len: length parameter passed to the device 3461 * 3462 * Returns 0 for success, non-zero in case of failure. 3463 * The buf_len parameter will contain, on return, the length parameter 3464 * received on the response. 3465 */ 3466 int ufshcd_query_descriptor_retry(struct ufs_hba *hba, 3467 enum query_opcode opcode, 3468 enum desc_idn idn, u8 index, 3469 u8 selector, 3470 u8 *desc_buf, int *buf_len) 3471 { 3472 int err; 3473 int retries; 3474 3475 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 3476 err = __ufshcd_query_descriptor(hba, opcode, idn, index, 3477 selector, desc_buf, buf_len); 3478 if (!err || err == -EINVAL) 3479 break; 3480 } 3481 3482 return err; 3483 } 3484 3485 /** 3486 * ufshcd_read_desc_param - read the specified descriptor parameter 3487 * @hba: Pointer to adapter instance 3488 * @desc_id: descriptor idn value 3489 * @desc_index: descriptor index 3490 * @param_offset: offset of the parameter to read 3491 * @param_read_buf: pointer to buffer where parameter would be read 3492 * @param_size: sizeof(param_read_buf) 3493 * 3494 * Return 0 in case of success, non-zero otherwise 3495 */ 3496 int ufshcd_read_desc_param(struct ufs_hba *hba, 3497 enum desc_idn desc_id, 3498 int desc_index, 3499 u8 param_offset, 3500 u8 *param_read_buf, 3501 u8 param_size) 3502 { 3503 int ret; 3504 u8 *desc_buf; 3505 int buff_len = QUERY_DESC_MAX_SIZE; 3506 bool is_kmalloc = true; 3507 3508 /* Safety check */ 3509 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) 3510 return -EINVAL; 3511 3512 /* Check whether we need temp memory */ 3513 if (param_offset != 0 || param_size < buff_len) { 3514 desc_buf = kzalloc(buff_len, GFP_KERNEL); 3515 if (!desc_buf) 3516 return -ENOMEM; 3517 } else { 3518 desc_buf = param_read_buf; 3519 is_kmalloc = false; 3520 } 3521 3522 /* Request for full descriptor */ 3523 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, 3524 desc_id, desc_index, 0, 3525 desc_buf, &buff_len); 3526 if (ret) { 3527 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n", 3528 __func__, desc_id, desc_index, param_offset, ret); 3529 goto out; 3530 } 3531 3532 /* Update descriptor length */ 3533 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; 3534 3535 if (param_offset >= buff_len) { 3536 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n", 3537 __func__, param_offset, desc_id, buff_len); 3538 ret = -EINVAL; 3539 goto out; 3540 } 3541 3542 /* Sanity check */ 3543 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { 3544 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n", 3545 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]); 3546 ret = -EINVAL; 3547 goto out; 3548 } 3549 3550 if (is_kmalloc) { 3551 /* Make sure we don't copy more data than available */ 3552 if (param_offset >= buff_len) 3553 ret = -EINVAL; 3554 else 3555 memcpy(param_read_buf, &desc_buf[param_offset], 3556 min_t(u32, param_size, buff_len - param_offset)); 3557 } 3558 out: 3559 if (is_kmalloc) 3560 kfree(desc_buf); 3561 return ret; 3562 } 3563 3564 /** 3565 * struct uc_string_id - unicode string 3566 * 3567 * @len: size of this descriptor inclusive 3568 * @type: descriptor type 3569 * @uc: unicode string character 3570 */ 3571 struct uc_string_id { 3572 u8 len; 3573 u8 type; 3574 wchar_t uc[]; 3575 } __packed; 3576 3577 /* replace non-printable or non-ASCII characters with spaces */ 3578 static inline char ufshcd_remove_non_printable(u8 ch) 3579 { 3580 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' '; 3581 } 3582 3583 /** 3584 * ufshcd_read_string_desc - read string descriptor 3585 * @hba: pointer to adapter instance 3586 * @desc_index: descriptor index 3587 * @buf: pointer to buffer where descriptor would be read, 3588 * the caller should free the memory. 3589 * @ascii: if true convert from unicode to ascii characters 3590 * null terminated string. 3591 * 3592 * Return: 3593 * * string size on success. 3594 * * -ENOMEM: on allocation failure 3595 * * -EINVAL: on a wrong parameter 3596 */ 3597 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, 3598 u8 **buf, bool ascii) 3599 { 3600 struct uc_string_id *uc_str; 3601 u8 *str; 3602 int ret; 3603 3604 if (!buf) 3605 return -EINVAL; 3606 3607 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 3608 if (!uc_str) 3609 return -ENOMEM; 3610 3611 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0, 3612 (u8 *)uc_str, QUERY_DESC_MAX_SIZE); 3613 if (ret < 0) { 3614 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n", 3615 QUERY_REQ_RETRIES, ret); 3616 str = NULL; 3617 goto out; 3618 } 3619 3620 if (uc_str->len <= QUERY_DESC_HDR_SIZE) { 3621 dev_dbg(hba->dev, "String Desc is of zero length\n"); 3622 str = NULL; 3623 ret = 0; 3624 goto out; 3625 } 3626 3627 if (ascii) { 3628 ssize_t ascii_len; 3629 int i; 3630 /* remove header and divide by 2 to move from UTF16 to UTF8 */ 3631 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1; 3632 str = kzalloc(ascii_len, GFP_KERNEL); 3633 if (!str) { 3634 ret = -ENOMEM; 3635 goto out; 3636 } 3637 3638 /* 3639 * the descriptor contains string in UTF16 format 3640 * we need to convert to utf-8 so it can be displayed 3641 */ 3642 ret = utf16s_to_utf8s(uc_str->uc, 3643 uc_str->len - QUERY_DESC_HDR_SIZE, 3644 UTF16_BIG_ENDIAN, str, ascii_len); 3645 3646 /* replace non-printable or non-ASCII characters with spaces */ 3647 for (i = 0; i < ret; i++) 3648 str[i] = ufshcd_remove_non_printable(str[i]); 3649 3650 str[ret++] = '\0'; 3651 3652 } else { 3653 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL); 3654 if (!str) { 3655 ret = -ENOMEM; 3656 goto out; 3657 } 3658 ret = uc_str->len; 3659 } 3660 out: 3661 *buf = str; 3662 kfree(uc_str); 3663 return ret; 3664 } 3665 3666 /** 3667 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter 3668 * @hba: Pointer to adapter instance 3669 * @lun: lun id 3670 * @param_offset: offset of the parameter to read 3671 * @param_read_buf: pointer to buffer where parameter would be read 3672 * @param_size: sizeof(param_read_buf) 3673 * 3674 * Return 0 in case of success, non-zero otherwise 3675 */ 3676 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, 3677 int lun, 3678 enum unit_desc_param param_offset, 3679 u8 *param_read_buf, 3680 u32 param_size) 3681 { 3682 /* 3683 * Unit descriptors are only available for general purpose LUs (LUN id 3684 * from 0 to 7) and RPMB Well known LU. 3685 */ 3686 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun)) 3687 return -EOPNOTSUPP; 3688 3689 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun, 3690 param_offset, param_read_buf, param_size); 3691 } 3692 3693 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) 3694 { 3695 int err = 0; 3696 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3697 3698 if (hba->dev_info.wspecversion >= 0x300) { 3699 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3700 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0, 3701 &gating_wait); 3702 if (err) 3703 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n", 3704 err, gating_wait); 3705 3706 if (gating_wait == 0) { 3707 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US; 3708 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n", 3709 gating_wait); 3710 } 3711 3712 hba->dev_info.clk_gating_wait_us = gating_wait; 3713 } 3714 3715 return err; 3716 } 3717 3718 /** 3719 * ufshcd_memory_alloc - allocate memory for host memory space data structures 3720 * @hba: per adapter instance 3721 * 3722 * 1. Allocate DMA memory for Command Descriptor array 3723 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT 3724 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). 3725 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List 3726 * (UTMRDL) 3727 * 4. Allocate memory for local reference block(lrb). 3728 * 3729 * Returns 0 for success, non-zero in case of failure 3730 */ 3731 static int ufshcd_memory_alloc(struct ufs_hba *hba) 3732 { 3733 size_t utmrdl_size, utrdl_size, ucdl_size; 3734 3735 /* Allocate memory for UTP command descriptors */ 3736 ucdl_size = sizeof_utp_transfer_cmd_desc(hba) * hba->nutrs; 3737 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, 3738 ucdl_size, 3739 &hba->ucdl_dma_addr, 3740 GFP_KERNEL); 3741 3742 /* 3743 * UFSHCI requires UTP command descriptor to be 128 byte aligned. 3744 */ 3745 if (!hba->ucdl_base_addr || 3746 WARN_ON(hba->ucdl_dma_addr & (128 - 1))) { 3747 dev_err(hba->dev, 3748 "Command Descriptor Memory allocation failed\n"); 3749 goto out; 3750 } 3751 3752 /* 3753 * Allocate memory for UTP Transfer descriptors 3754 * UFSHCI requires 1024 byte alignment of UTRD 3755 */ 3756 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); 3757 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, 3758 utrdl_size, 3759 &hba->utrdl_dma_addr, 3760 GFP_KERNEL); 3761 if (!hba->utrdl_base_addr || 3762 WARN_ON(hba->utrdl_dma_addr & (1024 - 1))) { 3763 dev_err(hba->dev, 3764 "Transfer Descriptor Memory allocation failed\n"); 3765 goto out; 3766 } 3767 3768 /* 3769 * Skip utmrdl allocation; it may have been 3770 * allocated during first pass and not released during 3771 * MCQ memory allocation. 3772 * See ufshcd_release_sdb_queue() and ufshcd_config_mcq() 3773 */ 3774 if (hba->utmrdl_base_addr) 3775 goto skip_utmrdl; 3776 /* 3777 * Allocate memory for UTP Task Management descriptors 3778 * UFSHCI requires 1024 byte alignment of UTMRD 3779 */ 3780 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; 3781 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, 3782 utmrdl_size, 3783 &hba->utmrdl_dma_addr, 3784 GFP_KERNEL); 3785 if (!hba->utmrdl_base_addr || 3786 WARN_ON(hba->utmrdl_dma_addr & (1024 - 1))) { 3787 dev_err(hba->dev, 3788 "Task Management Descriptor Memory allocation failed\n"); 3789 goto out; 3790 } 3791 3792 skip_utmrdl: 3793 /* Allocate memory for local reference block */ 3794 hba->lrb = devm_kcalloc(hba->dev, 3795 hba->nutrs, sizeof(struct ufshcd_lrb), 3796 GFP_KERNEL); 3797 if (!hba->lrb) { 3798 dev_err(hba->dev, "LRB Memory allocation failed\n"); 3799 goto out; 3800 } 3801 return 0; 3802 out: 3803 return -ENOMEM; 3804 } 3805 3806 /** 3807 * ufshcd_host_memory_configure - configure local reference block with 3808 * memory offsets 3809 * @hba: per adapter instance 3810 * 3811 * Configure Host memory space 3812 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA 3813 * address. 3814 * 2. Update each UTRD with Response UPIU offset, Response UPIU length 3815 * and PRDT offset. 3816 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT 3817 * into local reference block. 3818 */ 3819 static void ufshcd_host_memory_configure(struct ufs_hba *hba) 3820 { 3821 struct utp_transfer_req_desc *utrdlp; 3822 dma_addr_t cmd_desc_dma_addr; 3823 dma_addr_t cmd_desc_element_addr; 3824 u16 response_offset; 3825 u16 prdt_offset; 3826 int cmd_desc_size; 3827 int i; 3828 3829 utrdlp = hba->utrdl_base_addr; 3830 3831 response_offset = 3832 offsetof(struct utp_transfer_cmd_desc, response_upiu); 3833 prdt_offset = 3834 offsetof(struct utp_transfer_cmd_desc, prd_table); 3835 3836 cmd_desc_size = sizeof_utp_transfer_cmd_desc(hba); 3837 cmd_desc_dma_addr = hba->ucdl_dma_addr; 3838 3839 for (i = 0; i < hba->nutrs; i++) { 3840 /* Configure UTRD with command descriptor base address */ 3841 cmd_desc_element_addr = 3842 (cmd_desc_dma_addr + (cmd_desc_size * i)); 3843 utrdlp[i].command_desc_base_addr_lo = 3844 cpu_to_le32(lower_32_bits(cmd_desc_element_addr)); 3845 utrdlp[i].command_desc_base_addr_hi = 3846 cpu_to_le32(upper_32_bits(cmd_desc_element_addr)); 3847 3848 /* Response upiu and prdt offset should be in double words */ 3849 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { 3850 utrdlp[i].response_upiu_offset = 3851 cpu_to_le16(response_offset); 3852 utrdlp[i].prd_table_offset = 3853 cpu_to_le16(prdt_offset); 3854 utrdlp[i].response_upiu_length = 3855 cpu_to_le16(ALIGNED_UPIU_SIZE); 3856 } else { 3857 utrdlp[i].response_upiu_offset = 3858 cpu_to_le16(response_offset >> 2); 3859 utrdlp[i].prd_table_offset = 3860 cpu_to_le16(prdt_offset >> 2); 3861 utrdlp[i].response_upiu_length = 3862 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 3863 } 3864 3865 ufshcd_init_lrb(hba, &hba->lrb[i], i); 3866 } 3867 } 3868 3869 /** 3870 * ufshcd_dme_link_startup - Notify Unipro to perform link startup 3871 * @hba: per adapter instance 3872 * 3873 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, 3874 * in order to initialize the Unipro link startup procedure. 3875 * Once the Unipro links are up, the device connected to the controller 3876 * is detected. 3877 * 3878 * Returns 0 on success, non-zero value on failure 3879 */ 3880 static int ufshcd_dme_link_startup(struct ufs_hba *hba) 3881 { 3882 struct uic_command uic_cmd = {0}; 3883 int ret; 3884 3885 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP; 3886 3887 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 3888 if (ret) 3889 dev_dbg(hba->dev, 3890 "dme-link-startup: error code %d\n", ret); 3891 return ret; 3892 } 3893 /** 3894 * ufshcd_dme_reset - UIC command for DME_RESET 3895 * @hba: per adapter instance 3896 * 3897 * DME_RESET command is issued in order to reset UniPro stack. 3898 * This function now deals with cold reset. 3899 * 3900 * Returns 0 on success, non-zero value on failure 3901 */ 3902 static int ufshcd_dme_reset(struct ufs_hba *hba) 3903 { 3904 struct uic_command uic_cmd = {0}; 3905 int ret; 3906 3907 uic_cmd.command = UIC_CMD_DME_RESET; 3908 3909 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 3910 if (ret) 3911 dev_err(hba->dev, 3912 "dme-reset: error code %d\n", ret); 3913 3914 return ret; 3915 } 3916 3917 int ufshcd_dme_configure_adapt(struct ufs_hba *hba, 3918 int agreed_gear, 3919 int adapt_val) 3920 { 3921 int ret; 3922 3923 if (agreed_gear < UFS_HS_G4) 3924 adapt_val = PA_NO_ADAPT; 3925 3926 ret = ufshcd_dme_set(hba, 3927 UIC_ARG_MIB(PA_TXHSADAPTTYPE), 3928 adapt_val); 3929 return ret; 3930 } 3931 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); 3932 3933 /** 3934 * ufshcd_dme_enable - UIC command for DME_ENABLE 3935 * @hba: per adapter instance 3936 * 3937 * DME_ENABLE command is issued in order to enable UniPro stack. 3938 * 3939 * Returns 0 on success, non-zero value on failure 3940 */ 3941 static int ufshcd_dme_enable(struct ufs_hba *hba) 3942 { 3943 struct uic_command uic_cmd = {0}; 3944 int ret; 3945 3946 uic_cmd.command = UIC_CMD_DME_ENABLE; 3947 3948 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 3949 if (ret) 3950 dev_err(hba->dev, 3951 "dme-enable: error code %d\n", ret); 3952 3953 return ret; 3954 } 3955 3956 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) 3957 { 3958 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000 3959 unsigned long min_sleep_time_us; 3960 3961 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS)) 3962 return; 3963 3964 /* 3965 * last_dme_cmd_tstamp will be 0 only for 1st call to 3966 * this function 3967 */ 3968 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) { 3969 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US; 3970 } else { 3971 unsigned long delta = 3972 (unsigned long) ktime_to_us( 3973 ktime_sub(ktime_get(), 3974 hba->last_dme_cmd_tstamp)); 3975 3976 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US) 3977 min_sleep_time_us = 3978 MIN_DELAY_BEFORE_DME_CMDS_US - delta; 3979 else 3980 return; /* no more delay required */ 3981 } 3982 3983 /* allow sleep for extra 50us if needed */ 3984 usleep_range(min_sleep_time_us, min_sleep_time_us + 50); 3985 } 3986 3987 /** 3988 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET 3989 * @hba: per adapter instance 3990 * @attr_sel: uic command argument1 3991 * @attr_set: attribute set type as uic command argument2 3992 * @mib_val: setting value as uic command argument3 3993 * @peer: indicate whether peer or local 3994 * 3995 * Returns 0 on success, non-zero value on failure 3996 */ 3997 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, 3998 u8 attr_set, u32 mib_val, u8 peer) 3999 { 4000 struct uic_command uic_cmd = {0}; 4001 static const char *const action[] = { 4002 "dme-set", 4003 "dme-peer-set" 4004 }; 4005 const char *set = action[!!peer]; 4006 int ret; 4007 int retries = UFS_UIC_COMMAND_RETRIES; 4008 4009 uic_cmd.command = peer ? 4010 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET; 4011 uic_cmd.argument1 = attr_sel; 4012 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set); 4013 uic_cmd.argument3 = mib_val; 4014 4015 do { 4016 /* for peer attributes we retry upon failure */ 4017 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4018 if (ret) 4019 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", 4020 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); 4021 } while (ret && peer && --retries); 4022 4023 if (ret) 4024 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n", 4025 set, UIC_GET_ATTR_ID(attr_sel), mib_val, 4026 UFS_UIC_COMMAND_RETRIES - retries); 4027 4028 return ret; 4029 } 4030 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); 4031 4032 /** 4033 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET 4034 * @hba: per adapter instance 4035 * @attr_sel: uic command argument1 4036 * @mib_val: the value of the attribute as returned by the UIC command 4037 * @peer: indicate whether peer or local 4038 * 4039 * Returns 0 on success, non-zero value on failure 4040 */ 4041 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, 4042 u32 *mib_val, u8 peer) 4043 { 4044 struct uic_command uic_cmd = {0}; 4045 static const char *const action[] = { 4046 "dme-get", 4047 "dme-peer-get" 4048 }; 4049 const char *get = action[!!peer]; 4050 int ret; 4051 int retries = UFS_UIC_COMMAND_RETRIES; 4052 struct ufs_pa_layer_attr orig_pwr_info; 4053 struct ufs_pa_layer_attr temp_pwr_info; 4054 bool pwr_mode_change = false; 4055 4056 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) { 4057 orig_pwr_info = hba->pwr_info; 4058 temp_pwr_info = orig_pwr_info; 4059 4060 if (orig_pwr_info.pwr_tx == FAST_MODE || 4061 orig_pwr_info.pwr_rx == FAST_MODE) { 4062 temp_pwr_info.pwr_tx = FASTAUTO_MODE; 4063 temp_pwr_info.pwr_rx = FASTAUTO_MODE; 4064 pwr_mode_change = true; 4065 } else if (orig_pwr_info.pwr_tx == SLOW_MODE || 4066 orig_pwr_info.pwr_rx == SLOW_MODE) { 4067 temp_pwr_info.pwr_tx = SLOWAUTO_MODE; 4068 temp_pwr_info.pwr_rx = SLOWAUTO_MODE; 4069 pwr_mode_change = true; 4070 } 4071 if (pwr_mode_change) { 4072 ret = ufshcd_change_power_mode(hba, &temp_pwr_info); 4073 if (ret) 4074 goto out; 4075 } 4076 } 4077 4078 uic_cmd.command = peer ? 4079 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET; 4080 uic_cmd.argument1 = attr_sel; 4081 4082 do { 4083 /* for peer attributes we retry upon failure */ 4084 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 4085 if (ret) 4086 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n", 4087 get, UIC_GET_ATTR_ID(attr_sel), ret); 4088 } while (ret && peer && --retries); 4089 4090 if (ret) 4091 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n", 4092 get, UIC_GET_ATTR_ID(attr_sel), 4093 UFS_UIC_COMMAND_RETRIES - retries); 4094 4095 if (mib_val && !ret) 4096 *mib_val = uic_cmd.argument3; 4097 4098 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) 4099 && pwr_mode_change) 4100 ufshcd_change_power_mode(hba, &orig_pwr_info); 4101 out: 4102 return ret; 4103 } 4104 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); 4105 4106 /** 4107 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power 4108 * state) and waits for it to take effect. 4109 * 4110 * @hba: per adapter instance 4111 * @cmd: UIC command to execute 4112 * 4113 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER & 4114 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host 4115 * and device UniPro link and hence it's final completion would be indicated by 4116 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in 4117 * addition to normal UIC command completion Status (UCCS). This function only 4118 * returns after the relevant status bits indicate the completion. 4119 * 4120 * Returns 0 on success, non-zero value on failure 4121 */ 4122 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) 4123 { 4124 DECLARE_COMPLETION_ONSTACK(uic_async_done); 4125 unsigned long flags; 4126 u8 status; 4127 int ret; 4128 bool reenable_intr = false; 4129 4130 mutex_lock(&hba->uic_cmd_mutex); 4131 ufshcd_add_delay_before_dme_cmd(hba); 4132 4133 spin_lock_irqsave(hba->host->host_lock, flags); 4134 if (ufshcd_is_link_broken(hba)) { 4135 ret = -ENOLINK; 4136 goto out_unlock; 4137 } 4138 hba->uic_async_done = &uic_async_done; 4139 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) { 4140 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); 4141 /* 4142 * Make sure UIC command completion interrupt is disabled before 4143 * issuing UIC command. 4144 */ 4145 wmb(); 4146 reenable_intr = true; 4147 } 4148 ret = __ufshcd_send_uic_cmd(hba, cmd, false); 4149 spin_unlock_irqrestore(hba->host->host_lock, flags); 4150 if (ret) { 4151 dev_err(hba->dev, 4152 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 4153 cmd->command, cmd->argument3, ret); 4154 goto out; 4155 } 4156 4157 if (!wait_for_completion_timeout(hba->uic_async_done, 4158 msecs_to_jiffies(UIC_CMD_TIMEOUT))) { 4159 dev_err(hba->dev, 4160 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", 4161 cmd->command, cmd->argument3); 4162 4163 if (!cmd->cmd_active) { 4164 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n", 4165 __func__); 4166 goto check_upmcrs; 4167 } 4168 4169 ret = -ETIMEDOUT; 4170 goto out; 4171 } 4172 4173 check_upmcrs: 4174 status = ufshcd_get_upmcrs(hba); 4175 if (status != PWR_LOCAL) { 4176 dev_err(hba->dev, 4177 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n", 4178 cmd->command, status); 4179 ret = (status != PWR_OK) ? status : -1; 4180 } 4181 out: 4182 if (ret) { 4183 ufshcd_print_host_state(hba); 4184 ufshcd_print_pwr_info(hba); 4185 ufshcd_print_evt_hist(hba); 4186 } 4187 4188 spin_lock_irqsave(hba->host->host_lock, flags); 4189 hba->active_uic_cmd = NULL; 4190 hba->uic_async_done = NULL; 4191 if (reenable_intr) 4192 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); 4193 if (ret) { 4194 ufshcd_set_link_broken(hba); 4195 ufshcd_schedule_eh_work(hba); 4196 } 4197 out_unlock: 4198 spin_unlock_irqrestore(hba->host->host_lock, flags); 4199 mutex_unlock(&hba->uic_cmd_mutex); 4200 4201 return ret; 4202 } 4203 4204 /** 4205 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage 4206 * using DME_SET primitives. 4207 * @hba: per adapter instance 4208 * @mode: powr mode value 4209 * 4210 * Returns 0 on success, non-zero value on failure 4211 */ 4212 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) 4213 { 4214 struct uic_command uic_cmd = {0}; 4215 int ret; 4216 4217 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) { 4218 ret = ufshcd_dme_set(hba, 4219 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1); 4220 if (ret) { 4221 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n", 4222 __func__, ret); 4223 goto out; 4224 } 4225 } 4226 4227 uic_cmd.command = UIC_CMD_DME_SET; 4228 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE); 4229 uic_cmd.argument3 = mode; 4230 ufshcd_hold(hba, false); 4231 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4232 ufshcd_release(hba); 4233 4234 out: 4235 return ret; 4236 } 4237 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode); 4238 4239 int ufshcd_link_recovery(struct ufs_hba *hba) 4240 { 4241 int ret; 4242 unsigned long flags; 4243 4244 spin_lock_irqsave(hba->host->host_lock, flags); 4245 hba->ufshcd_state = UFSHCD_STATE_RESET; 4246 ufshcd_set_eh_in_progress(hba); 4247 spin_unlock_irqrestore(hba->host->host_lock, flags); 4248 4249 /* Reset the attached device */ 4250 ufshcd_device_reset(hba); 4251 4252 ret = ufshcd_host_reset_and_restore(hba); 4253 4254 spin_lock_irqsave(hba->host->host_lock, flags); 4255 if (ret) 4256 hba->ufshcd_state = UFSHCD_STATE_ERROR; 4257 ufshcd_clear_eh_in_progress(hba); 4258 spin_unlock_irqrestore(hba->host->host_lock, flags); 4259 4260 if (ret) 4261 dev_err(hba->dev, "%s: link recovery failed, err %d", 4262 __func__, ret); 4263 4264 return ret; 4265 } 4266 EXPORT_SYMBOL_GPL(ufshcd_link_recovery); 4267 4268 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) 4269 { 4270 int ret; 4271 struct uic_command uic_cmd = {0}; 4272 ktime_t start = ktime_get(); 4273 4274 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); 4275 4276 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER; 4277 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4278 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter", 4279 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4280 4281 if (ret) 4282 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n", 4283 __func__, ret); 4284 else 4285 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, 4286 POST_CHANGE); 4287 4288 return ret; 4289 } 4290 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter); 4291 4292 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) 4293 { 4294 struct uic_command uic_cmd = {0}; 4295 int ret; 4296 ktime_t start = ktime_get(); 4297 4298 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); 4299 4300 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT; 4301 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 4302 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit", 4303 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 4304 4305 if (ret) { 4306 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n", 4307 __func__, ret); 4308 } else { 4309 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, 4310 POST_CHANGE); 4311 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock(); 4312 hba->ufs_stats.hibern8_exit_cnt++; 4313 } 4314 4315 return ret; 4316 } 4317 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit); 4318 4319 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) 4320 { 4321 unsigned long flags; 4322 bool update = false; 4323 4324 if (!ufshcd_is_auto_hibern8_supported(hba)) 4325 return; 4326 4327 spin_lock_irqsave(hba->host->host_lock, flags); 4328 if (hba->ahit != ahit) { 4329 hba->ahit = ahit; 4330 update = true; 4331 } 4332 spin_unlock_irqrestore(hba->host->host_lock, flags); 4333 4334 if (update && 4335 !pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { 4336 ufshcd_rpm_get_sync(hba); 4337 ufshcd_hold(hba, false); 4338 ufshcd_auto_hibern8_enable(hba); 4339 ufshcd_release(hba); 4340 ufshcd_rpm_put_sync(hba); 4341 } 4342 } 4343 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update); 4344 4345 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba) 4346 { 4347 if (!ufshcd_is_auto_hibern8_supported(hba)) 4348 return; 4349 4350 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); 4351 } 4352 4353 /** 4354 * ufshcd_init_pwr_info - setting the POR (power on reset) 4355 * values in hba power info 4356 * @hba: per-adapter instance 4357 */ 4358 static void ufshcd_init_pwr_info(struct ufs_hba *hba) 4359 { 4360 hba->pwr_info.gear_rx = UFS_PWM_G1; 4361 hba->pwr_info.gear_tx = UFS_PWM_G1; 4362 hba->pwr_info.lane_rx = 1; 4363 hba->pwr_info.lane_tx = 1; 4364 hba->pwr_info.pwr_rx = SLOWAUTO_MODE; 4365 hba->pwr_info.pwr_tx = SLOWAUTO_MODE; 4366 hba->pwr_info.hs_rate = 0; 4367 } 4368 4369 /** 4370 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device 4371 * @hba: per-adapter instance 4372 */ 4373 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) 4374 { 4375 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 4376 4377 if (hba->max_pwr_info.is_valid) 4378 return 0; 4379 4380 if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) { 4381 pwr_info->pwr_tx = FASTAUTO_MODE; 4382 pwr_info->pwr_rx = FASTAUTO_MODE; 4383 } else { 4384 pwr_info->pwr_tx = FAST_MODE; 4385 pwr_info->pwr_rx = FAST_MODE; 4386 } 4387 pwr_info->hs_rate = PA_HS_MODE_B; 4388 4389 /* Get the connected lane count */ 4390 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), 4391 &pwr_info->lane_rx); 4392 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4393 &pwr_info->lane_tx); 4394 4395 if (!pwr_info->lane_rx || !pwr_info->lane_tx) { 4396 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", 4397 __func__, 4398 pwr_info->lane_rx, 4399 pwr_info->lane_tx); 4400 return -EINVAL; 4401 } 4402 4403 /* 4404 * First, get the maximum gears of HS speed. 4405 * If a zero value, it means there is no HSGEAR capability. 4406 * Then, get the maximum gears of PWM speed. 4407 */ 4408 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); 4409 if (!pwr_info->gear_rx) { 4410 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4411 &pwr_info->gear_rx); 4412 if (!pwr_info->gear_rx) { 4413 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", 4414 __func__, pwr_info->gear_rx); 4415 return -EINVAL; 4416 } 4417 pwr_info->pwr_rx = SLOW_MODE; 4418 } 4419 4420 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), 4421 &pwr_info->gear_tx); 4422 if (!pwr_info->gear_tx) { 4423 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 4424 &pwr_info->gear_tx); 4425 if (!pwr_info->gear_tx) { 4426 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", 4427 __func__, pwr_info->gear_tx); 4428 return -EINVAL; 4429 } 4430 pwr_info->pwr_tx = SLOW_MODE; 4431 } 4432 4433 hba->max_pwr_info.is_valid = true; 4434 return 0; 4435 } 4436 4437 static int ufshcd_change_power_mode(struct ufs_hba *hba, 4438 struct ufs_pa_layer_attr *pwr_mode) 4439 { 4440 int ret; 4441 4442 /* if already configured to the requested pwr_mode */ 4443 if (!hba->force_pmc && 4444 pwr_mode->gear_rx == hba->pwr_info.gear_rx && 4445 pwr_mode->gear_tx == hba->pwr_info.gear_tx && 4446 pwr_mode->lane_rx == hba->pwr_info.lane_rx && 4447 pwr_mode->lane_tx == hba->pwr_info.lane_tx && 4448 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && 4449 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && 4450 pwr_mode->hs_rate == hba->pwr_info.hs_rate) { 4451 dev_dbg(hba->dev, "%s: power already configured\n", __func__); 4452 return 0; 4453 } 4454 4455 /* 4456 * Configure attributes for power mode change with below. 4457 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, 4458 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, 4459 * - PA_HSSERIES 4460 */ 4461 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); 4462 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), 4463 pwr_mode->lane_rx); 4464 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4465 pwr_mode->pwr_rx == FAST_MODE) 4466 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true); 4467 else 4468 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false); 4469 4470 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); 4471 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), 4472 pwr_mode->lane_tx); 4473 if (pwr_mode->pwr_tx == FASTAUTO_MODE || 4474 pwr_mode->pwr_tx == FAST_MODE) 4475 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true); 4476 else 4477 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false); 4478 4479 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 4480 pwr_mode->pwr_tx == FASTAUTO_MODE || 4481 pwr_mode->pwr_rx == FAST_MODE || 4482 pwr_mode->pwr_tx == FAST_MODE) 4483 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), 4484 pwr_mode->hs_rate); 4485 4486 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) { 4487 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 4488 DL_FC0ProtectionTimeOutVal_Default); 4489 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 4490 DL_TC0ReplayTimeOutVal_Default); 4491 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 4492 DL_AFC0ReqTimeOutVal_Default); 4493 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), 4494 DL_FC1ProtectionTimeOutVal_Default); 4495 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), 4496 DL_TC1ReplayTimeOutVal_Default); 4497 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), 4498 DL_AFC1ReqTimeOutVal_Default); 4499 4500 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal), 4501 DL_FC0ProtectionTimeOutVal_Default); 4502 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal), 4503 DL_TC0ReplayTimeOutVal_Default); 4504 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal), 4505 DL_AFC0ReqTimeOutVal_Default); 4506 } 4507 4508 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 4509 | pwr_mode->pwr_tx); 4510 4511 if (ret) { 4512 dev_err(hba->dev, 4513 "%s: power mode change failed %d\n", __func__, ret); 4514 } else { 4515 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, 4516 pwr_mode); 4517 4518 memcpy(&hba->pwr_info, pwr_mode, 4519 sizeof(struct ufs_pa_layer_attr)); 4520 } 4521 4522 return ret; 4523 } 4524 4525 /** 4526 * ufshcd_config_pwr_mode - configure a new power mode 4527 * @hba: per-adapter instance 4528 * @desired_pwr_mode: desired power configuration 4529 */ 4530 int ufshcd_config_pwr_mode(struct ufs_hba *hba, 4531 struct ufs_pa_layer_attr *desired_pwr_mode) 4532 { 4533 struct ufs_pa_layer_attr final_params = { 0 }; 4534 int ret; 4535 4536 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE, 4537 desired_pwr_mode, &final_params); 4538 4539 if (ret) 4540 memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); 4541 4542 ret = ufshcd_change_power_mode(hba, &final_params); 4543 4544 return ret; 4545 } 4546 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); 4547 4548 /** 4549 * ufshcd_complete_dev_init() - checks device readiness 4550 * @hba: per-adapter instance 4551 * 4552 * Set fDeviceInit flag and poll until device toggles it. 4553 */ 4554 static int ufshcd_complete_dev_init(struct ufs_hba *hba) 4555 { 4556 int err; 4557 bool flag_res = true; 4558 ktime_t timeout; 4559 4560 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 4561 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL); 4562 if (err) { 4563 dev_err(hba->dev, 4564 "%s: setting fDeviceInit flag failed with error %d\n", 4565 __func__, err); 4566 goto out; 4567 } 4568 4569 /* Poll fDeviceInit flag to be cleared */ 4570 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT); 4571 do { 4572 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, 4573 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res); 4574 if (!flag_res) 4575 break; 4576 usleep_range(500, 1000); 4577 } while (ktime_before(ktime_get(), timeout)); 4578 4579 if (err) { 4580 dev_err(hba->dev, 4581 "%s: reading fDeviceInit flag failed with error %d\n", 4582 __func__, err); 4583 } else if (flag_res) { 4584 dev_err(hba->dev, 4585 "%s: fDeviceInit was not cleared by the device\n", 4586 __func__); 4587 err = -EBUSY; 4588 } 4589 out: 4590 return err; 4591 } 4592 4593 /** 4594 * ufshcd_make_hba_operational - Make UFS controller operational 4595 * @hba: per adapter instance 4596 * 4597 * To bring UFS host controller to operational state, 4598 * 1. Enable required interrupts 4599 * 2. Configure interrupt aggregation 4600 * 3. Program UTRL and UTMRL base address 4601 * 4. Configure run-stop-registers 4602 * 4603 * Returns 0 on success, non-zero value on failure 4604 */ 4605 int ufshcd_make_hba_operational(struct ufs_hba *hba) 4606 { 4607 int err = 0; 4608 u32 reg; 4609 4610 /* Enable required interrupts */ 4611 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); 4612 4613 /* Configure interrupt aggregation */ 4614 if (ufshcd_is_intr_aggr_allowed(hba)) 4615 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); 4616 else 4617 ufshcd_disable_intr_aggr(hba); 4618 4619 /* Configure UTRL and UTMRL base address registers */ 4620 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 4621 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 4622 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 4623 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 4624 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 4625 REG_UTP_TASK_REQ_LIST_BASE_L); 4626 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 4627 REG_UTP_TASK_REQ_LIST_BASE_H); 4628 4629 /* 4630 * Make sure base address and interrupt setup are updated before 4631 * enabling the run/stop registers below. 4632 */ 4633 wmb(); 4634 4635 /* 4636 * UCRDY, UTMRLDY and UTRLRDY bits must be 1 4637 */ 4638 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); 4639 if (!(ufshcd_get_lists_status(reg))) { 4640 ufshcd_enable_run_stop_reg(hba); 4641 } else { 4642 dev_err(hba->dev, 4643 "Host controller not ready to process requests"); 4644 err = -EIO; 4645 } 4646 4647 return err; 4648 } 4649 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); 4650 4651 /** 4652 * ufshcd_hba_stop - Send controller to reset state 4653 * @hba: per adapter instance 4654 */ 4655 void ufshcd_hba_stop(struct ufs_hba *hba) 4656 { 4657 unsigned long flags; 4658 int err; 4659 4660 /* 4661 * Obtain the host lock to prevent that the controller is disabled 4662 * while the UFS interrupt handler is active on another CPU. 4663 */ 4664 spin_lock_irqsave(hba->host->host_lock, flags); 4665 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); 4666 spin_unlock_irqrestore(hba->host->host_lock, flags); 4667 4668 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, 4669 CONTROLLER_ENABLE, CONTROLLER_DISABLE, 4670 10, 1); 4671 if (err) 4672 dev_err(hba->dev, "%s: Controller disable failed\n", __func__); 4673 } 4674 EXPORT_SYMBOL_GPL(ufshcd_hba_stop); 4675 4676 /** 4677 * ufshcd_hba_execute_hce - initialize the controller 4678 * @hba: per adapter instance 4679 * 4680 * The controller resets itself and controller firmware initialization 4681 * sequence kicks off. When controller is ready it will set 4682 * the Host Controller Enable bit to 1. 4683 * 4684 * Returns 0 on success, non-zero value on failure 4685 */ 4686 static int ufshcd_hba_execute_hce(struct ufs_hba *hba) 4687 { 4688 int retry_outer = 3; 4689 int retry_inner; 4690 4691 start: 4692 if (ufshcd_is_hba_active(hba)) 4693 /* change controller state to "reset state" */ 4694 ufshcd_hba_stop(hba); 4695 4696 /* UniPro link is disabled at this point */ 4697 ufshcd_set_link_off(hba); 4698 4699 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4700 4701 /* start controller initialization sequence */ 4702 ufshcd_hba_start(hba); 4703 4704 /* 4705 * To initialize a UFS host controller HCE bit must be set to 1. 4706 * During initialization the HCE bit value changes from 1->0->1. 4707 * When the host controller completes initialization sequence 4708 * it sets the value of HCE bit to 1. The same HCE bit is read back 4709 * to check if the controller has completed initialization sequence. 4710 * So without this delay the value HCE = 1, set in the previous 4711 * instruction might be read back. 4712 * This delay can be changed based on the controller. 4713 */ 4714 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); 4715 4716 /* wait for the host controller to complete initialization */ 4717 retry_inner = 50; 4718 while (!ufshcd_is_hba_active(hba)) { 4719 if (retry_inner) { 4720 retry_inner--; 4721 } else { 4722 dev_err(hba->dev, 4723 "Controller enable failed\n"); 4724 if (retry_outer) { 4725 retry_outer--; 4726 goto start; 4727 } 4728 return -EIO; 4729 } 4730 usleep_range(1000, 1100); 4731 } 4732 4733 /* enable UIC related interrupts */ 4734 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4735 4736 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4737 4738 return 0; 4739 } 4740 4741 int ufshcd_hba_enable(struct ufs_hba *hba) 4742 { 4743 int ret; 4744 4745 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) { 4746 ufshcd_set_link_off(hba); 4747 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); 4748 4749 /* enable UIC related interrupts */ 4750 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 4751 ret = ufshcd_dme_reset(hba); 4752 if (ret) { 4753 dev_err(hba->dev, "DME_RESET failed\n"); 4754 return ret; 4755 } 4756 4757 ret = ufshcd_dme_enable(hba); 4758 if (ret) { 4759 dev_err(hba->dev, "Enabling DME failed\n"); 4760 return ret; 4761 } 4762 4763 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE); 4764 } else { 4765 ret = ufshcd_hba_execute_hce(hba); 4766 } 4767 4768 return ret; 4769 } 4770 EXPORT_SYMBOL_GPL(ufshcd_hba_enable); 4771 4772 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) 4773 { 4774 int tx_lanes = 0, i, err = 0; 4775 4776 if (!peer) 4777 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4778 &tx_lanes); 4779 else 4780 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 4781 &tx_lanes); 4782 for (i = 0; i < tx_lanes; i++) { 4783 if (!peer) 4784 err = ufshcd_dme_set(hba, 4785 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 4786 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 4787 0); 4788 else 4789 err = ufshcd_dme_peer_set(hba, 4790 UIC_ARG_MIB_SEL(TX_LCC_ENABLE, 4791 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), 4792 0); 4793 if (err) { 4794 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d", 4795 __func__, peer, i, err); 4796 break; 4797 } 4798 } 4799 4800 return err; 4801 } 4802 4803 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba) 4804 { 4805 return ufshcd_disable_tx_lcc(hba, true); 4806 } 4807 4808 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val) 4809 { 4810 struct ufs_event_hist *e; 4811 4812 if (id >= UFS_EVT_CNT) 4813 return; 4814 4815 e = &hba->ufs_stats.event[id]; 4816 e->val[e->pos] = val; 4817 e->tstamp[e->pos] = local_clock(); 4818 e->cnt += 1; 4819 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH; 4820 4821 ufshcd_vops_event_notify(hba, id, &val); 4822 } 4823 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); 4824 4825 /** 4826 * ufshcd_link_startup - Initialize unipro link startup 4827 * @hba: per adapter instance 4828 * 4829 * Returns 0 for success, non-zero in case of failure 4830 */ 4831 static int ufshcd_link_startup(struct ufs_hba *hba) 4832 { 4833 int ret; 4834 int retries = DME_LINKSTARTUP_RETRIES; 4835 bool link_startup_again = false; 4836 4837 /* 4838 * If UFS device isn't active then we will have to issue link startup 4839 * 2 times to make sure the device state move to active. 4840 */ 4841 if (!ufshcd_is_ufs_dev_active(hba)) 4842 link_startup_again = true; 4843 4844 link_startup: 4845 do { 4846 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE); 4847 4848 ret = ufshcd_dme_link_startup(hba); 4849 4850 /* check if device is detected by inter-connect layer */ 4851 if (!ret && !ufshcd_is_device_present(hba)) { 4852 ufshcd_update_evt_hist(hba, 4853 UFS_EVT_LINK_STARTUP_FAIL, 4854 0); 4855 dev_err(hba->dev, "%s: Device not present\n", __func__); 4856 ret = -ENXIO; 4857 goto out; 4858 } 4859 4860 /* 4861 * DME link lost indication is only received when link is up, 4862 * but we can't be sure if the link is up until link startup 4863 * succeeds. So reset the local Uni-Pro and try again. 4864 */ 4865 if (ret && retries && ufshcd_hba_enable(hba)) { 4866 ufshcd_update_evt_hist(hba, 4867 UFS_EVT_LINK_STARTUP_FAIL, 4868 (u32)ret); 4869 goto out; 4870 } 4871 } while (ret && retries--); 4872 4873 if (ret) { 4874 /* failed to get the link up... retire */ 4875 ufshcd_update_evt_hist(hba, 4876 UFS_EVT_LINK_STARTUP_FAIL, 4877 (u32)ret); 4878 goto out; 4879 } 4880 4881 if (link_startup_again) { 4882 link_startup_again = false; 4883 retries = DME_LINKSTARTUP_RETRIES; 4884 goto link_startup; 4885 } 4886 4887 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ 4888 ufshcd_init_pwr_info(hba); 4889 ufshcd_print_pwr_info(hba); 4890 4891 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { 4892 ret = ufshcd_disable_device_tx_lcc(hba); 4893 if (ret) 4894 goto out; 4895 } 4896 4897 /* Include any host controller configuration via UIC commands */ 4898 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE); 4899 if (ret) 4900 goto out; 4901 4902 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */ 4903 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 4904 ret = ufshcd_make_hba_operational(hba); 4905 out: 4906 if (ret) { 4907 dev_err(hba->dev, "link startup failed %d\n", ret); 4908 ufshcd_print_host_state(hba); 4909 ufshcd_print_pwr_info(hba); 4910 ufshcd_print_evt_hist(hba); 4911 } 4912 return ret; 4913 } 4914 4915 /** 4916 * ufshcd_verify_dev_init() - Verify device initialization 4917 * @hba: per-adapter instance 4918 * 4919 * Send NOP OUT UPIU and wait for NOP IN response to check whether the 4920 * device Transport Protocol (UTP) layer is ready after a reset. 4921 * If the UTP layer at the device side is not initialized, it may 4922 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT 4923 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. 4924 */ 4925 static int ufshcd_verify_dev_init(struct ufs_hba *hba) 4926 { 4927 int err = 0; 4928 int retries; 4929 4930 ufshcd_hold(hba, false); 4931 mutex_lock(&hba->dev_cmd.lock); 4932 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { 4933 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, 4934 hba->nop_out_timeout); 4935 4936 if (!err || err == -ETIMEDOUT) 4937 break; 4938 4939 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 4940 } 4941 mutex_unlock(&hba->dev_cmd.lock); 4942 ufshcd_release(hba); 4943 4944 if (err) 4945 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); 4946 return err; 4947 } 4948 4949 /** 4950 * ufshcd_setup_links - associate link b/w device wlun and other luns 4951 * @sdev: pointer to SCSI device 4952 * @hba: pointer to ufs hba 4953 */ 4954 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev) 4955 { 4956 struct device_link *link; 4957 4958 /* 4959 * Device wlun is the supplier & rest of the luns are consumers. 4960 * This ensures that device wlun suspends after all other luns. 4961 */ 4962 if (hba->ufs_device_wlun) { 4963 link = device_link_add(&sdev->sdev_gendev, 4964 &hba->ufs_device_wlun->sdev_gendev, 4965 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); 4966 if (!link) { 4967 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n", 4968 dev_name(&hba->ufs_device_wlun->sdev_gendev)); 4969 return; 4970 } 4971 hba->luns_avail--; 4972 /* Ignore REPORT_LUN wlun probing */ 4973 if (hba->luns_avail == 1) { 4974 ufshcd_rpm_put(hba); 4975 return; 4976 } 4977 } else { 4978 /* 4979 * Device wlun is probed. The assumption is that WLUNs are 4980 * scanned before other LUNs. 4981 */ 4982 hba->luns_avail--; 4983 } 4984 } 4985 4986 /** 4987 * ufshcd_lu_init - Initialize the relevant parameters of the LU 4988 * @hba: per-adapter instance 4989 * @sdev: pointer to SCSI device 4990 */ 4991 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) 4992 { 4993 int len = QUERY_DESC_MAX_SIZE; 4994 u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun); 4995 u8 lun_qdepth = hba->nutrs; 4996 u8 *desc_buf; 4997 int ret; 4998 4999 desc_buf = kzalloc(len, GFP_KERNEL); 5000 if (!desc_buf) 5001 goto set_qdepth; 5002 5003 ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len); 5004 if (ret < 0) { 5005 if (ret == -EOPNOTSUPP) 5006 /* If LU doesn't support unit descriptor, its queue depth is set to 1 */ 5007 lun_qdepth = 1; 5008 kfree(desc_buf); 5009 goto set_qdepth; 5010 } 5011 5012 if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) { 5013 /* 5014 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will 5015 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth 5016 */ 5017 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs); 5018 } 5019 /* 5020 * According to UFS device specification, the write protection mode is only supported by 5021 * normal LU, not supported by WLUN. 5022 */ 5023 if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported && 5024 !hba->dev_info.is_lu_power_on_wp && 5025 desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP) 5026 hba->dev_info.is_lu_power_on_wp = true; 5027 5028 /* In case of RPMB LU, check if advanced RPMB mode is enabled */ 5029 if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN && 5030 desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) 5031 hba->dev_info.b_advanced_rpmb_en = true; 5032 5033 5034 kfree(desc_buf); 5035 set_qdepth: 5036 /* 5037 * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose 5038 * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue. 5039 */ 5040 dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth); 5041 scsi_change_queue_depth(sdev, lun_qdepth); 5042 } 5043 5044 /** 5045 * ufshcd_slave_alloc - handle initial SCSI device configurations 5046 * @sdev: pointer to SCSI device 5047 * 5048 * Returns success 5049 */ 5050 static int ufshcd_slave_alloc(struct scsi_device *sdev) 5051 { 5052 struct ufs_hba *hba; 5053 5054 hba = shost_priv(sdev->host); 5055 5056 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ 5057 sdev->use_10_for_ms = 1; 5058 5059 /* DBD field should be set to 1 in mode sense(10) */ 5060 sdev->set_dbd_for_ms = 1; 5061 5062 /* allow SCSI layer to restart the device in case of errors */ 5063 sdev->allow_restart = 1; 5064 5065 /* REPORT SUPPORTED OPERATION CODES is not supported */ 5066 sdev->no_report_opcodes = 1; 5067 5068 /* WRITE_SAME command is not supported */ 5069 sdev->no_write_same = 1; 5070 5071 ufshcd_lu_init(hba, sdev); 5072 5073 ufshcd_setup_links(hba, sdev); 5074 5075 return 0; 5076 } 5077 5078 /** 5079 * ufshcd_change_queue_depth - change queue depth 5080 * @sdev: pointer to SCSI device 5081 * @depth: required depth to set 5082 * 5083 * Change queue depth and make sure the max. limits are not crossed. 5084 */ 5085 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) 5086 { 5087 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); 5088 } 5089 5090 static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev) 5091 { 5092 /* skip well-known LU */ 5093 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || 5094 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) 5095 return; 5096 5097 ufshpb_destroy_lu(hba, sdev); 5098 } 5099 5100 static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev) 5101 { 5102 /* skip well-known LU */ 5103 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || 5104 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) 5105 return; 5106 5107 ufshpb_init_hpb_lu(hba, sdev); 5108 } 5109 5110 /** 5111 * ufshcd_slave_configure - adjust SCSI device configurations 5112 * @sdev: pointer to SCSI device 5113 */ 5114 static int ufshcd_slave_configure(struct scsi_device *sdev) 5115 { 5116 struct ufs_hba *hba = shost_priv(sdev->host); 5117 struct request_queue *q = sdev->request_queue; 5118 5119 ufshcd_hpb_configure(hba, sdev); 5120 5121 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); 5122 if (hba->quirks & UFSHCD_QUIRK_4KB_DMA_ALIGNMENT) 5123 blk_queue_update_dma_alignment(q, 4096 - 1); 5124 /* 5125 * Block runtime-pm until all consumers are added. 5126 * Refer ufshcd_setup_links(). 5127 */ 5128 if (is_device_wlun(sdev)) 5129 pm_runtime_get_noresume(&sdev->sdev_gendev); 5130 else if (ufshcd_is_rpm_autosuspend_allowed(hba)) 5131 sdev->rpm_autosuspend = 1; 5132 /* 5133 * Do not print messages during runtime PM to avoid never-ending cycles 5134 * of messages written back to storage by user space causing runtime 5135 * resume, causing more messages and so on. 5136 */ 5137 sdev->silence_suspend = 1; 5138 5139 ufshcd_crypto_register(hba, q); 5140 5141 return 0; 5142 } 5143 5144 /** 5145 * ufshcd_slave_destroy - remove SCSI device configurations 5146 * @sdev: pointer to SCSI device 5147 */ 5148 static void ufshcd_slave_destroy(struct scsi_device *sdev) 5149 { 5150 struct ufs_hba *hba; 5151 unsigned long flags; 5152 5153 hba = shost_priv(sdev->host); 5154 5155 ufshcd_hpb_destroy(hba, sdev); 5156 5157 /* Drop the reference as it won't be needed anymore */ 5158 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { 5159 spin_lock_irqsave(hba->host->host_lock, flags); 5160 hba->ufs_device_wlun = NULL; 5161 spin_unlock_irqrestore(hba->host->host_lock, flags); 5162 } else if (hba->ufs_device_wlun) { 5163 struct device *supplier = NULL; 5164 5165 /* Ensure UFS Device WLUN exists and does not disappear */ 5166 spin_lock_irqsave(hba->host->host_lock, flags); 5167 if (hba->ufs_device_wlun) { 5168 supplier = &hba->ufs_device_wlun->sdev_gendev; 5169 get_device(supplier); 5170 } 5171 spin_unlock_irqrestore(hba->host->host_lock, flags); 5172 5173 if (supplier) { 5174 /* 5175 * If a LUN fails to probe (e.g. absent BOOT WLUN), the 5176 * device will not have been registered but can still 5177 * have a device link holding a reference to the device. 5178 */ 5179 device_link_remove(&sdev->sdev_gendev, supplier); 5180 put_device(supplier); 5181 } 5182 } 5183 } 5184 5185 /** 5186 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 5187 * @lrbp: pointer to local reference block of completed command 5188 * @scsi_status: SCSI command status 5189 * 5190 * Returns value base on SCSI command status 5191 */ 5192 static inline int 5193 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) 5194 { 5195 int result = 0; 5196 5197 switch (scsi_status) { 5198 case SAM_STAT_CHECK_CONDITION: 5199 ufshcd_copy_sense_data(lrbp); 5200 fallthrough; 5201 case SAM_STAT_GOOD: 5202 result |= DID_OK << 16 | scsi_status; 5203 break; 5204 case SAM_STAT_TASK_SET_FULL: 5205 case SAM_STAT_BUSY: 5206 case SAM_STAT_TASK_ABORTED: 5207 ufshcd_copy_sense_data(lrbp); 5208 result |= scsi_status; 5209 break; 5210 default: 5211 result |= DID_ERROR << 16; 5212 break; 5213 } /* end of switch */ 5214 5215 return result; 5216 } 5217 5218 /** 5219 * ufshcd_transfer_rsp_status - Get overall status of the response 5220 * @hba: per adapter instance 5221 * @lrbp: pointer to local reference block of completed command 5222 * @cqe: pointer to the completion queue entry 5223 * 5224 * Returns result of the command to notify SCSI midlayer 5225 */ 5226 static inline int 5227 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, 5228 struct cq_entry *cqe) 5229 { 5230 int result = 0; 5231 int scsi_status; 5232 enum utp_ocs ocs; 5233 5234 /* overall command status of utrd */ 5235 ocs = ufshcd_get_tr_ocs(lrbp, cqe); 5236 5237 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { 5238 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) & 5239 MASK_RSP_UPIU_RESULT) 5240 ocs = OCS_SUCCESS; 5241 } 5242 5243 switch (ocs) { 5244 case OCS_SUCCESS: 5245 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 5246 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 5247 switch (result) { 5248 case UPIU_TRANSACTION_RESPONSE: 5249 /* 5250 * get the response UPIU result to extract 5251 * the SCSI command status 5252 */ 5253 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); 5254 5255 /* 5256 * get the result based on SCSI status response 5257 * to notify the SCSI midlayer of the command status 5258 */ 5259 scsi_status = result & MASK_SCSI_STATUS; 5260 result = ufshcd_scsi_cmd_status(lrbp, scsi_status); 5261 5262 /* 5263 * Currently we are only supporting BKOPs exception 5264 * events hence we can ignore BKOPs exception event 5265 * during power management callbacks. BKOPs exception 5266 * event is not expected to be raised in runtime suspend 5267 * callback as it allows the urgent bkops. 5268 * During system suspend, we are anyway forcefully 5269 * disabling the bkops and if urgent bkops is needed 5270 * it will be enabled on system resume. Long term 5271 * solution could be to abort the system suspend if 5272 * UFS device needs urgent BKOPs. 5273 */ 5274 if (!hba->pm_op_in_progress && 5275 !ufshcd_eh_in_progress(hba) && 5276 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) 5277 /* Flushed in suspend */ 5278 schedule_work(&hba->eeh_work); 5279 5280 if (scsi_status == SAM_STAT_GOOD) 5281 ufshpb_rsp_upiu(hba, lrbp); 5282 break; 5283 case UPIU_TRANSACTION_REJECT_UPIU: 5284 /* TODO: handle Reject UPIU Response */ 5285 result = DID_ERROR << 16; 5286 dev_err(hba->dev, 5287 "Reject UPIU not fully implemented\n"); 5288 break; 5289 default: 5290 dev_err(hba->dev, 5291 "Unexpected request response code = %x\n", 5292 result); 5293 result = DID_ERROR << 16; 5294 break; 5295 } 5296 break; 5297 case OCS_ABORTED: 5298 result |= DID_ABORT << 16; 5299 break; 5300 case OCS_INVALID_COMMAND_STATUS: 5301 result |= DID_REQUEUE << 16; 5302 break; 5303 case OCS_INVALID_CMD_TABLE_ATTR: 5304 case OCS_INVALID_PRDT_ATTR: 5305 case OCS_MISMATCH_DATA_BUF_SIZE: 5306 case OCS_MISMATCH_RESP_UPIU_SIZE: 5307 case OCS_PEER_COMM_FAILURE: 5308 case OCS_FATAL_ERROR: 5309 case OCS_DEVICE_FATAL_ERROR: 5310 case OCS_INVALID_CRYPTO_CONFIG: 5311 case OCS_GENERAL_CRYPTO_ERROR: 5312 default: 5313 result |= DID_ERROR << 16; 5314 dev_err(hba->dev, 5315 "OCS error from controller = %x for tag %d\n", 5316 ocs, lrbp->task_tag); 5317 ufshcd_print_evt_hist(hba); 5318 ufshcd_print_host_state(hba); 5319 break; 5320 } /* end of switch */ 5321 5322 if ((host_byte(result) != DID_OK) && 5323 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) 5324 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true); 5325 return result; 5326 } 5327 5328 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, 5329 u32 intr_mask) 5330 { 5331 if (!ufshcd_is_auto_hibern8_supported(hba) || 5332 !ufshcd_is_auto_hibern8_enabled(hba)) 5333 return false; 5334 5335 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK)) 5336 return false; 5337 5338 if (hba->active_uic_cmd && 5339 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER || 5340 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT)) 5341 return false; 5342 5343 return true; 5344 } 5345 5346 /** 5347 * ufshcd_uic_cmd_compl - handle completion of uic command 5348 * @hba: per adapter instance 5349 * @intr_status: interrupt status generated by the controller 5350 * 5351 * Returns 5352 * IRQ_HANDLED - If interrupt is valid 5353 * IRQ_NONE - If invalid interrupt 5354 */ 5355 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) 5356 { 5357 irqreturn_t retval = IRQ_NONE; 5358 5359 spin_lock(hba->host->host_lock); 5360 if (ufshcd_is_auto_hibern8_error(hba, intr_status)) 5361 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status); 5362 5363 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) { 5364 hba->active_uic_cmd->argument2 |= 5365 ufshcd_get_uic_cmd_result(hba); 5366 hba->active_uic_cmd->argument3 = 5367 ufshcd_get_dme_attr_val(hba); 5368 if (!hba->uic_async_done) 5369 hba->active_uic_cmd->cmd_active = 0; 5370 complete(&hba->active_uic_cmd->done); 5371 retval = IRQ_HANDLED; 5372 } 5373 5374 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) { 5375 hba->active_uic_cmd->cmd_active = 0; 5376 complete(hba->uic_async_done); 5377 retval = IRQ_HANDLED; 5378 } 5379 5380 if (retval == IRQ_HANDLED) 5381 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd, 5382 UFS_CMD_COMP); 5383 spin_unlock(hba->host->host_lock); 5384 return retval; 5385 } 5386 5387 /* Release the resources allocated for processing a SCSI command. */ 5388 static void ufshcd_release_scsi_cmd(struct ufs_hba *hba, 5389 struct ufshcd_lrb *lrbp) 5390 { 5391 struct scsi_cmnd *cmd = lrbp->cmd; 5392 5393 scsi_dma_unmap(cmd); 5394 lrbp->cmd = NULL; /* Mark the command as completed. */ 5395 ufshcd_release(hba); 5396 ufshcd_clk_scaling_update_busy(hba); 5397 } 5398 5399 /** 5400 * ufshcd_compl_one_cqe - handle a completion queue entry 5401 * @hba: per adapter instance 5402 * @task_tag: the task tag of the request to be completed 5403 * @cqe: pointer to the completion queue entry 5404 */ 5405 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, 5406 struct cq_entry *cqe) 5407 { 5408 struct ufshcd_lrb *lrbp; 5409 struct scsi_cmnd *cmd; 5410 5411 lrbp = &hba->lrb[task_tag]; 5412 lrbp->compl_time_stamp = ktime_get(); 5413 cmd = lrbp->cmd; 5414 if (cmd) { 5415 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) 5416 ufshcd_update_monitor(hba, lrbp); 5417 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP); 5418 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); 5419 ufshcd_release_scsi_cmd(hba, lrbp); 5420 /* Do not touch lrbp after scsi done */ 5421 scsi_done(cmd); 5422 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE || 5423 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) { 5424 if (hba->dev_cmd.complete) { 5425 hba->dev_cmd.cqe = cqe; 5426 ufshcd_add_command_trace(hba, task_tag, UFS_DEV_COMP); 5427 complete(hba->dev_cmd.complete); 5428 ufshcd_clk_scaling_update_busy(hba); 5429 } 5430 } 5431 } 5432 5433 /** 5434 * __ufshcd_transfer_req_compl - handle SCSI and query command completion 5435 * @hba: per adapter instance 5436 * @completed_reqs: bitmask that indicates which requests to complete 5437 */ 5438 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, 5439 unsigned long completed_reqs) 5440 { 5441 int tag; 5442 5443 for_each_set_bit(tag, &completed_reqs, hba->nutrs) 5444 ufshcd_compl_one_cqe(hba, tag, NULL); 5445 } 5446 5447 /* Any value that is not an existing queue number is fine for this constant. */ 5448 enum { 5449 UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1 5450 }; 5451 5452 static void ufshcd_clear_polled(struct ufs_hba *hba, 5453 unsigned long *completed_reqs) 5454 { 5455 int tag; 5456 5457 for_each_set_bit(tag, completed_reqs, hba->nutrs) { 5458 struct scsi_cmnd *cmd = hba->lrb[tag].cmd; 5459 5460 if (!cmd) 5461 continue; 5462 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED) 5463 __clear_bit(tag, completed_reqs); 5464 } 5465 } 5466 5467 /* 5468 * Returns > 0 if one or more commands have been completed or 0 if no 5469 * requests have been completed. 5470 */ 5471 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) 5472 { 5473 struct ufs_hba *hba = shost_priv(shost); 5474 unsigned long completed_reqs, flags; 5475 u32 tr_doorbell; 5476 struct ufs_hw_queue *hwq; 5477 5478 if (is_mcq_enabled(hba)) { 5479 hwq = &hba->uhq[queue_num + UFSHCD_MCQ_IO_QUEUE_OFFSET]; 5480 5481 return ufshcd_mcq_poll_cqe_lock(hba, hwq); 5482 } 5483 5484 spin_lock_irqsave(&hba->outstanding_lock, flags); 5485 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 5486 completed_reqs = ~tr_doorbell & hba->outstanding_reqs; 5487 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs, 5488 "completed: %#lx; outstanding: %#lx\n", completed_reqs, 5489 hba->outstanding_reqs); 5490 if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) { 5491 /* Do not complete polled requests from interrupt context. */ 5492 ufshcd_clear_polled(hba, &completed_reqs); 5493 } 5494 hba->outstanding_reqs &= ~completed_reqs; 5495 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 5496 5497 if (completed_reqs) 5498 __ufshcd_transfer_req_compl(hba, completed_reqs); 5499 5500 return completed_reqs != 0; 5501 } 5502 5503 /** 5504 * ufshcd_transfer_req_compl - handle SCSI and query command completion 5505 * @hba: per adapter instance 5506 * 5507 * Returns 5508 * IRQ_HANDLED - If interrupt is valid 5509 * IRQ_NONE - If invalid interrupt 5510 */ 5511 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) 5512 { 5513 /* Resetting interrupt aggregation counters first and reading the 5514 * DOOR_BELL afterward allows us to handle all the completed requests. 5515 * In order to prevent other interrupts starvation the DB is read once 5516 * after reset. The down side of this solution is the possibility of 5517 * false interrupt if device completes another request after resetting 5518 * aggregation and before reading the DB. 5519 */ 5520 if (ufshcd_is_intr_aggr_allowed(hba) && 5521 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) 5522 ufshcd_reset_intr_aggr(hba); 5523 5524 if (ufs_fail_completion()) 5525 return IRQ_HANDLED; 5526 5527 /* 5528 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we 5529 * do not want polling to trigger spurious interrupt complaints. 5530 */ 5531 ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT); 5532 5533 return IRQ_HANDLED; 5534 } 5535 5536 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask) 5537 { 5538 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 5539 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, 5540 &ee_ctrl_mask); 5541 } 5542 5543 int ufshcd_write_ee_control(struct ufs_hba *hba) 5544 { 5545 int err; 5546 5547 mutex_lock(&hba->ee_ctrl_mutex); 5548 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask); 5549 mutex_unlock(&hba->ee_ctrl_mutex); 5550 if (err) 5551 dev_err(hba->dev, "%s: failed to write ee control %d\n", 5552 __func__, err); 5553 return err; 5554 } 5555 5556 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, 5557 const u16 *other_mask, u16 set, u16 clr) 5558 { 5559 u16 new_mask, ee_ctrl_mask; 5560 int err = 0; 5561 5562 mutex_lock(&hba->ee_ctrl_mutex); 5563 new_mask = (*mask & ~clr) | set; 5564 ee_ctrl_mask = new_mask | *other_mask; 5565 if (ee_ctrl_mask != hba->ee_ctrl_mask) 5566 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask); 5567 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */ 5568 if (!err) { 5569 hba->ee_ctrl_mask = ee_ctrl_mask; 5570 *mask = new_mask; 5571 } 5572 mutex_unlock(&hba->ee_ctrl_mutex); 5573 return err; 5574 } 5575 5576 /** 5577 * ufshcd_disable_ee - disable exception event 5578 * @hba: per-adapter instance 5579 * @mask: exception event to disable 5580 * 5581 * Disables exception event in the device so that the EVENT_ALERT 5582 * bit is not set. 5583 * 5584 * Returns zero on success, non-zero error value on failure. 5585 */ 5586 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) 5587 { 5588 return ufshcd_update_ee_drv_mask(hba, 0, mask); 5589 } 5590 5591 /** 5592 * ufshcd_enable_ee - enable exception event 5593 * @hba: per-adapter instance 5594 * @mask: exception event to enable 5595 * 5596 * Enable corresponding exception event in the device to allow 5597 * device to alert host in critical scenarios. 5598 * 5599 * Returns zero on success, non-zero error value on failure. 5600 */ 5601 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) 5602 { 5603 return ufshcd_update_ee_drv_mask(hba, mask, 0); 5604 } 5605 5606 /** 5607 * ufshcd_enable_auto_bkops - Allow device managed BKOPS 5608 * @hba: per-adapter instance 5609 * 5610 * Allow device to manage background operations on its own. Enabling 5611 * this might lead to inconsistent latencies during normal data transfers 5612 * as the device is allowed to manage its own way of handling background 5613 * operations. 5614 * 5615 * Returns zero on success, non-zero on failure. 5616 */ 5617 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) 5618 { 5619 int err = 0; 5620 5621 if (hba->auto_bkops_enabled) 5622 goto out; 5623 5624 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, 5625 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5626 if (err) { 5627 dev_err(hba->dev, "%s: failed to enable bkops %d\n", 5628 __func__, err); 5629 goto out; 5630 } 5631 5632 hba->auto_bkops_enabled = true; 5633 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled"); 5634 5635 /* No need of URGENT_BKOPS exception from the device */ 5636 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5637 if (err) 5638 dev_err(hba->dev, "%s: failed to disable exception event %d\n", 5639 __func__, err); 5640 out: 5641 return err; 5642 } 5643 5644 /** 5645 * ufshcd_disable_auto_bkops - block device in doing background operations 5646 * @hba: per-adapter instance 5647 * 5648 * Disabling background operations improves command response latency but 5649 * has drawback of device moving into critical state where the device is 5650 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the 5651 * host is idle so that BKOPS are managed effectively without any negative 5652 * impacts. 5653 * 5654 * Returns zero on success, non-zero on failure. 5655 */ 5656 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) 5657 { 5658 int err = 0; 5659 5660 if (!hba->auto_bkops_enabled) 5661 goto out; 5662 5663 /* 5664 * If host assisted BKOPs is to be enabled, make sure 5665 * urgent bkops exception is allowed. 5666 */ 5667 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); 5668 if (err) { 5669 dev_err(hba->dev, "%s: failed to enable exception event %d\n", 5670 __func__, err); 5671 goto out; 5672 } 5673 5674 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, 5675 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL); 5676 if (err) { 5677 dev_err(hba->dev, "%s: failed to disable bkops %d\n", 5678 __func__, err); 5679 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 5680 goto out; 5681 } 5682 5683 hba->auto_bkops_enabled = false; 5684 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled"); 5685 hba->is_urgent_bkops_lvl_checked = false; 5686 out: 5687 return err; 5688 } 5689 5690 /** 5691 * ufshcd_force_reset_auto_bkops - force reset auto bkops state 5692 * @hba: per adapter instance 5693 * 5694 * After a device reset the device may toggle the BKOPS_EN flag 5695 * to default value. The s/w tracking variables should be updated 5696 * as well. This function would change the auto-bkops state based on 5697 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND. 5698 */ 5699 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) 5700 { 5701 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) { 5702 hba->auto_bkops_enabled = false; 5703 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; 5704 ufshcd_enable_auto_bkops(hba); 5705 } else { 5706 hba->auto_bkops_enabled = true; 5707 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS; 5708 ufshcd_disable_auto_bkops(hba); 5709 } 5710 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; 5711 hba->is_urgent_bkops_lvl_checked = false; 5712 } 5713 5714 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) 5715 { 5716 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 5717 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); 5718 } 5719 5720 /** 5721 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status 5722 * @hba: per-adapter instance 5723 * @status: bkops_status value 5724 * 5725 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn 5726 * flag in the device to permit background operations if the device 5727 * bkops_status is greater than or equal to "status" argument passed to 5728 * this function, disable otherwise. 5729 * 5730 * Returns 0 for success, non-zero in case of failure. 5731 * 5732 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag 5733 * to know whether auto bkops is enabled or disabled after this function 5734 * returns control to it. 5735 */ 5736 static int ufshcd_bkops_ctrl(struct ufs_hba *hba, 5737 enum bkops_status status) 5738 { 5739 int err; 5740 u32 curr_status = 0; 5741 5742 err = ufshcd_get_bkops_status(hba, &curr_status); 5743 if (err) { 5744 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 5745 __func__, err); 5746 goto out; 5747 } else if (curr_status > BKOPS_STATUS_MAX) { 5748 dev_err(hba->dev, "%s: invalid BKOPS status %d\n", 5749 __func__, curr_status); 5750 err = -EINVAL; 5751 goto out; 5752 } 5753 5754 if (curr_status >= status) 5755 err = ufshcd_enable_auto_bkops(hba); 5756 else 5757 err = ufshcd_disable_auto_bkops(hba); 5758 out: 5759 return err; 5760 } 5761 5762 /** 5763 * ufshcd_urgent_bkops - handle urgent bkops exception event 5764 * @hba: per-adapter instance 5765 * 5766 * Enable fBackgroundOpsEn flag in the device to permit background 5767 * operations. 5768 * 5769 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled 5770 * and negative error value for any other failure. 5771 */ 5772 static int ufshcd_urgent_bkops(struct ufs_hba *hba) 5773 { 5774 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl); 5775 } 5776 5777 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) 5778 { 5779 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 5780 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); 5781 } 5782 5783 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba) 5784 { 5785 int err; 5786 u32 curr_status = 0; 5787 5788 if (hba->is_urgent_bkops_lvl_checked) 5789 goto enable_auto_bkops; 5790 5791 err = ufshcd_get_bkops_status(hba, &curr_status); 5792 if (err) { 5793 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 5794 __func__, err); 5795 goto out; 5796 } 5797 5798 /* 5799 * We are seeing that some devices are raising the urgent bkops 5800 * exception events even when BKOPS status doesn't indicate performace 5801 * impacted or critical. Handle these device by determining their urgent 5802 * bkops status at runtime. 5803 */ 5804 if (curr_status < BKOPS_STATUS_PERF_IMPACT) { 5805 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n", 5806 __func__, curr_status); 5807 /* update the current status as the urgent bkops level */ 5808 hba->urgent_bkops_lvl = curr_status; 5809 hba->is_urgent_bkops_lvl_checked = true; 5810 } 5811 5812 enable_auto_bkops: 5813 err = ufshcd_enable_auto_bkops(hba); 5814 out: 5815 if (err < 0) 5816 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", 5817 __func__, err); 5818 } 5819 5820 static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status) 5821 { 5822 u32 value; 5823 5824 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 5825 QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value)) 5826 return; 5827 5828 dev_info(hba->dev, "exception Tcase %d\n", value - 80); 5829 5830 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); 5831 5832 /* 5833 * A placeholder for the platform vendors to add whatever additional 5834 * steps required 5835 */ 5836 } 5837 5838 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) 5839 { 5840 u8 index; 5841 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG : 5842 UPIU_QUERY_OPCODE_CLEAR_FLAG; 5843 5844 index = ufshcd_wb_get_query_index(hba); 5845 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL); 5846 } 5847 5848 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable) 5849 { 5850 int ret; 5851 5852 if (!ufshcd_is_wb_allowed(hba) || 5853 hba->dev_info.wb_enabled == enable) 5854 return 0; 5855 5856 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN); 5857 if (ret) { 5858 dev_err(hba->dev, "%s: Write Booster %s failed %d\n", 5859 __func__, enable ? "enabling" : "disabling", ret); 5860 return ret; 5861 } 5862 5863 hba->dev_info.wb_enabled = enable; 5864 dev_dbg(hba->dev, "%s: Write Booster %s\n", 5865 __func__, enable ? "enabled" : "disabled"); 5866 5867 return ret; 5868 } 5869 5870 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, 5871 bool enable) 5872 { 5873 int ret; 5874 5875 ret = __ufshcd_wb_toggle(hba, enable, 5876 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8); 5877 if (ret) { 5878 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n", 5879 __func__, enable ? "enabling" : "disabling", ret); 5880 return; 5881 } 5882 dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n", 5883 __func__, enable ? "enabled" : "disabled"); 5884 } 5885 5886 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) 5887 { 5888 int ret; 5889 5890 if (!ufshcd_is_wb_allowed(hba) || 5891 hba->dev_info.wb_buf_flush_enabled == enable) 5892 return 0; 5893 5894 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN); 5895 if (ret) { 5896 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n", 5897 __func__, enable ? "enabling" : "disabling", ret); 5898 return ret; 5899 } 5900 5901 hba->dev_info.wb_buf_flush_enabled = enable; 5902 dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n", 5903 __func__, enable ? "enabled" : "disabled"); 5904 5905 return ret; 5906 } 5907 5908 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba, 5909 u32 avail_buf) 5910 { 5911 u32 cur_buf; 5912 int ret; 5913 u8 index; 5914 5915 index = ufshcd_wb_get_query_index(hba); 5916 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 5917 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE, 5918 index, 0, &cur_buf); 5919 if (ret) { 5920 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n", 5921 __func__, ret); 5922 return false; 5923 } 5924 5925 if (!cur_buf) { 5926 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n", 5927 cur_buf); 5928 return false; 5929 } 5930 /* Let it continue to flush when available buffer exceeds threshold */ 5931 return avail_buf < hba->vps->wb_flush_threshold; 5932 } 5933 5934 static void ufshcd_wb_force_disable(struct ufs_hba *hba) 5935 { 5936 if (ufshcd_is_wb_buf_flush_allowed(hba)) 5937 ufshcd_wb_toggle_buf_flush(hba, false); 5938 5939 ufshcd_wb_toggle_buf_flush_during_h8(hba, false); 5940 ufshcd_wb_toggle(hba, false); 5941 hba->caps &= ~UFSHCD_CAP_WB_EN; 5942 5943 dev_info(hba->dev, "%s: WB force disabled\n", __func__); 5944 } 5945 5946 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba) 5947 { 5948 u32 lifetime; 5949 int ret; 5950 u8 index; 5951 5952 index = ufshcd_wb_get_query_index(hba); 5953 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 5954 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST, 5955 index, 0, &lifetime); 5956 if (ret) { 5957 dev_err(hba->dev, 5958 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n", 5959 __func__, ret); 5960 return false; 5961 } 5962 5963 if (lifetime == UFS_WB_EXCEED_LIFETIME) { 5964 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n", 5965 __func__, lifetime); 5966 return false; 5967 } 5968 5969 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n", 5970 __func__, lifetime); 5971 5972 return true; 5973 } 5974 5975 static bool ufshcd_wb_need_flush(struct ufs_hba *hba) 5976 { 5977 int ret; 5978 u32 avail_buf; 5979 u8 index; 5980 5981 if (!ufshcd_is_wb_allowed(hba)) 5982 return false; 5983 5984 if (!ufshcd_is_wb_buf_lifetime_available(hba)) { 5985 ufshcd_wb_force_disable(hba); 5986 return false; 5987 } 5988 5989 /* 5990 * The ufs device needs the vcc to be ON to flush. 5991 * With user-space reduction enabled, it's enough to enable flush 5992 * by checking only the available buffer. The threshold 5993 * defined here is > 90% full. 5994 * With user-space preserved enabled, the current-buffer 5995 * should be checked too because the wb buffer size can reduce 5996 * when disk tends to be full. This info is provided by current 5997 * buffer (dCurrentWriteBoosterBufferSize). There's no point in 5998 * keeping vcc on when current buffer is empty. 5999 */ 6000 index = ufshcd_wb_get_query_index(hba); 6001 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 6002 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE, 6003 index, 0, &avail_buf); 6004 if (ret) { 6005 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n", 6006 __func__, ret); 6007 return false; 6008 } 6009 6010 if (!hba->dev_info.b_presrv_uspc_en) 6011 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); 6012 6013 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf); 6014 } 6015 6016 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) 6017 { 6018 struct ufs_hba *hba = container_of(to_delayed_work(work), 6019 struct ufs_hba, 6020 rpm_dev_flush_recheck_work); 6021 /* 6022 * To prevent unnecessary VCC power drain after device finishes 6023 * WriteBooster buffer flush or Auto BKOPs, force runtime resume 6024 * after a certain delay to recheck the threshold by next runtime 6025 * suspend. 6026 */ 6027 ufshcd_rpm_get_sync(hba); 6028 ufshcd_rpm_put_sync(hba); 6029 } 6030 6031 /** 6032 * ufshcd_exception_event_handler - handle exceptions raised by device 6033 * @work: pointer to work data 6034 * 6035 * Read bExceptionEventStatus attribute from the device and handle the 6036 * exception event accordingly. 6037 */ 6038 static void ufshcd_exception_event_handler(struct work_struct *work) 6039 { 6040 struct ufs_hba *hba; 6041 int err; 6042 u32 status = 0; 6043 hba = container_of(work, struct ufs_hba, eeh_work); 6044 6045 ufshcd_scsi_block_requests(hba); 6046 err = ufshcd_get_ee_status(hba, &status); 6047 if (err) { 6048 dev_err(hba->dev, "%s: failed to get exception status %d\n", 6049 __func__, err); 6050 goto out; 6051 } 6052 6053 trace_ufshcd_exception_event(dev_name(hba->dev), status); 6054 6055 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) 6056 ufshcd_bkops_exception_event_handler(hba); 6057 6058 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) 6059 ufshcd_temp_exception_event_handler(hba, status); 6060 6061 ufs_debugfs_exception_event(hba, status); 6062 out: 6063 ufshcd_scsi_unblock_requests(hba); 6064 } 6065 6066 /* Complete requests that have door-bell cleared */ 6067 static void ufshcd_complete_requests(struct ufs_hba *hba) 6068 { 6069 ufshcd_transfer_req_compl(hba); 6070 ufshcd_tmc_handler(hba); 6071 } 6072 6073 /** 6074 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is 6075 * to recover from the DL NAC errors or not. 6076 * @hba: per-adapter instance 6077 * 6078 * Returns true if error handling is required, false otherwise 6079 */ 6080 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) 6081 { 6082 unsigned long flags; 6083 bool err_handling = true; 6084 6085 spin_lock_irqsave(hba->host->host_lock, flags); 6086 /* 6087 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the 6088 * device fatal error and/or DL NAC & REPLAY timeout errors. 6089 */ 6090 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR)) 6091 goto out; 6092 6093 if ((hba->saved_err & DEVICE_FATAL_ERROR) || 6094 ((hba->saved_err & UIC_ERROR) && 6095 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR))) 6096 goto out; 6097 6098 if ((hba->saved_err & UIC_ERROR) && 6099 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) { 6100 int err; 6101 /* 6102 * wait for 50ms to see if we can get any other errors or not. 6103 */ 6104 spin_unlock_irqrestore(hba->host->host_lock, flags); 6105 msleep(50); 6106 spin_lock_irqsave(hba->host->host_lock, flags); 6107 6108 /* 6109 * now check if we have got any other severe errors other than 6110 * DL NAC error? 6111 */ 6112 if ((hba->saved_err & INT_FATAL_ERRORS) || 6113 ((hba->saved_err & UIC_ERROR) && 6114 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR))) 6115 goto out; 6116 6117 /* 6118 * As DL NAC is the only error received so far, send out NOP 6119 * command to confirm if link is still active or not. 6120 * - If we don't get any response then do error recovery. 6121 * - If we get response then clear the DL NAC error bit. 6122 */ 6123 6124 spin_unlock_irqrestore(hba->host->host_lock, flags); 6125 err = ufshcd_verify_dev_init(hba); 6126 spin_lock_irqsave(hba->host->host_lock, flags); 6127 6128 if (err) 6129 goto out; 6130 6131 /* Link seems to be alive hence ignore the DL NAC errors */ 6132 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR) 6133 hba->saved_err &= ~UIC_ERROR; 6134 /* clear NAC error */ 6135 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6136 if (!hba->saved_uic_err) 6137 err_handling = false; 6138 } 6139 out: 6140 spin_unlock_irqrestore(hba->host->host_lock, flags); 6141 return err_handling; 6142 } 6143 6144 /* host lock must be held before calling this func */ 6145 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba) 6146 { 6147 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) || 6148 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)); 6149 } 6150 6151 void ufshcd_schedule_eh_work(struct ufs_hba *hba) 6152 { 6153 lockdep_assert_held(hba->host->host_lock); 6154 6155 /* handle fatal errors only when link is not in error state */ 6156 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6157 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6158 ufshcd_is_saved_err_fatal(hba)) 6159 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL; 6160 else 6161 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL; 6162 queue_work(hba->eh_wq, &hba->eh_work); 6163 } 6164 } 6165 6166 static void ufshcd_force_error_recovery(struct ufs_hba *hba) 6167 { 6168 spin_lock_irq(hba->host->host_lock); 6169 hba->force_reset = true; 6170 ufshcd_schedule_eh_work(hba); 6171 spin_unlock_irq(hba->host->host_lock); 6172 } 6173 6174 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) 6175 { 6176 mutex_lock(&hba->wb_mutex); 6177 down_write(&hba->clk_scaling_lock); 6178 hba->clk_scaling.is_allowed = allow; 6179 up_write(&hba->clk_scaling_lock); 6180 mutex_unlock(&hba->wb_mutex); 6181 } 6182 6183 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend) 6184 { 6185 if (suspend) { 6186 if (hba->clk_scaling.is_enabled) 6187 ufshcd_suspend_clkscaling(hba); 6188 ufshcd_clk_scaling_allow(hba, false); 6189 } else { 6190 ufshcd_clk_scaling_allow(hba, true); 6191 if (hba->clk_scaling.is_enabled) 6192 ufshcd_resume_clkscaling(hba); 6193 } 6194 } 6195 6196 static void ufshcd_err_handling_prepare(struct ufs_hba *hba) 6197 { 6198 ufshcd_rpm_get_sync(hba); 6199 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) || 6200 hba->is_sys_suspended) { 6201 enum ufs_pm_op pm_op; 6202 6203 /* 6204 * Don't assume anything of resume, if 6205 * resume fails, irq and clocks can be OFF, and powers 6206 * can be OFF or in LPM. 6207 */ 6208 ufshcd_setup_hba_vreg(hba, true); 6209 ufshcd_enable_irq(hba); 6210 ufshcd_setup_vreg(hba, true); 6211 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 6212 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 6213 ufshcd_hold(hba, false); 6214 if (!ufshcd_is_clkgating_allowed(hba)) 6215 ufshcd_setup_clocks(hba, true); 6216 ufshcd_release(hba); 6217 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; 6218 ufshcd_vops_resume(hba, pm_op); 6219 } else { 6220 ufshcd_hold(hba, false); 6221 if (ufshcd_is_clkscaling_supported(hba) && 6222 hba->clk_scaling.is_enabled) 6223 ufshcd_suspend_clkscaling(hba); 6224 ufshcd_clk_scaling_allow(hba, false); 6225 } 6226 ufshcd_scsi_block_requests(hba); 6227 /* Drain ufshcd_queuecommand() */ 6228 synchronize_rcu(); 6229 cancel_work_sync(&hba->eeh_work); 6230 } 6231 6232 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) 6233 { 6234 ufshcd_scsi_unblock_requests(hba); 6235 ufshcd_release(hba); 6236 if (ufshcd_is_clkscaling_supported(hba)) 6237 ufshcd_clk_scaling_suspend(hba, false); 6238 ufshcd_rpm_put(hba); 6239 } 6240 6241 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba) 6242 { 6243 return (!hba->is_powered || hba->shutting_down || 6244 !hba->ufs_device_wlun || 6245 hba->ufshcd_state == UFSHCD_STATE_ERROR || 6246 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset || 6247 ufshcd_is_link_broken(hba)))); 6248 } 6249 6250 #ifdef CONFIG_PM 6251 static void ufshcd_recover_pm_error(struct ufs_hba *hba) 6252 { 6253 struct Scsi_Host *shost = hba->host; 6254 struct scsi_device *sdev; 6255 struct request_queue *q; 6256 int ret; 6257 6258 hba->is_sys_suspended = false; 6259 /* 6260 * Set RPM status of wlun device to RPM_ACTIVE, 6261 * this also clears its runtime error. 6262 */ 6263 ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev); 6264 6265 /* hba device might have a runtime error otherwise */ 6266 if (ret) 6267 ret = pm_runtime_set_active(hba->dev); 6268 /* 6269 * If wlun device had runtime error, we also need to resume those 6270 * consumer scsi devices in case any of them has failed to be 6271 * resumed due to supplier runtime resume failure. This is to unblock 6272 * blk_queue_enter in case there are bios waiting inside it. 6273 */ 6274 if (!ret) { 6275 shost_for_each_device(sdev, shost) { 6276 q = sdev->request_queue; 6277 if (q->dev && (q->rpm_status == RPM_SUSPENDED || 6278 q->rpm_status == RPM_SUSPENDING)) 6279 pm_request_resume(q->dev); 6280 } 6281 } 6282 } 6283 #else 6284 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba) 6285 { 6286 } 6287 #endif 6288 6289 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) 6290 { 6291 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info; 6292 u32 mode; 6293 6294 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode); 6295 6296 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK)) 6297 return true; 6298 6299 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK)) 6300 return true; 6301 6302 return false; 6303 } 6304 6305 static bool ufshcd_abort_all(struct ufs_hba *hba) 6306 { 6307 bool needs_reset = false; 6308 int tag, ret; 6309 6310 /* Clear pending transfer requests */ 6311 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) { 6312 ret = ufshcd_try_to_abort_task(hba, tag); 6313 dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, 6314 hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, 6315 ret ? "failed" : "succeeded"); 6316 if (ret) { 6317 needs_reset = true; 6318 goto out; 6319 } 6320 } 6321 6322 /* Clear pending task management requests */ 6323 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { 6324 if (ufshcd_clear_tm_cmd(hba, tag)) { 6325 needs_reset = true; 6326 goto out; 6327 } 6328 } 6329 6330 out: 6331 /* Complete the requests that are cleared by s/w */ 6332 ufshcd_complete_requests(hba); 6333 6334 return needs_reset; 6335 } 6336 6337 /** 6338 * ufshcd_err_handler - handle UFS errors that require s/w attention 6339 * @work: pointer to work structure 6340 */ 6341 static void ufshcd_err_handler(struct work_struct *work) 6342 { 6343 int retries = MAX_ERR_HANDLER_RETRIES; 6344 struct ufs_hba *hba; 6345 unsigned long flags; 6346 bool needs_restore; 6347 bool needs_reset; 6348 int pmc_err; 6349 6350 hba = container_of(work, struct ufs_hba, eh_work); 6351 6352 dev_info(hba->dev, 6353 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n", 6354 __func__, ufshcd_state_name[hba->ufshcd_state], 6355 hba->is_powered, hba->shutting_down, hba->saved_err, 6356 hba->saved_uic_err, hba->force_reset, 6357 ufshcd_is_link_broken(hba) ? "; link is broken" : ""); 6358 6359 down(&hba->host_sem); 6360 spin_lock_irqsave(hba->host->host_lock, flags); 6361 if (ufshcd_err_handling_should_stop(hba)) { 6362 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6363 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6364 spin_unlock_irqrestore(hba->host->host_lock, flags); 6365 up(&hba->host_sem); 6366 return; 6367 } 6368 ufshcd_set_eh_in_progress(hba); 6369 spin_unlock_irqrestore(hba->host->host_lock, flags); 6370 ufshcd_err_handling_prepare(hba); 6371 /* Complete requests that have door-bell cleared by h/w */ 6372 ufshcd_complete_requests(hba); 6373 spin_lock_irqsave(hba->host->host_lock, flags); 6374 again: 6375 needs_restore = false; 6376 needs_reset = false; 6377 6378 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) 6379 hba->ufshcd_state = UFSHCD_STATE_RESET; 6380 /* 6381 * A full reset and restore might have happened after preparation 6382 * is finished, double check whether we should stop. 6383 */ 6384 if (ufshcd_err_handling_should_stop(hba)) 6385 goto skip_err_handling; 6386 6387 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { 6388 bool ret; 6389 6390 spin_unlock_irqrestore(hba->host->host_lock, flags); 6391 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */ 6392 ret = ufshcd_quirk_dl_nac_errors(hba); 6393 spin_lock_irqsave(hba->host->host_lock, flags); 6394 if (!ret && ufshcd_err_handling_should_stop(hba)) 6395 goto skip_err_handling; 6396 } 6397 6398 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 6399 (hba->saved_uic_err && 6400 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 6401 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR); 6402 6403 spin_unlock_irqrestore(hba->host->host_lock, flags); 6404 ufshcd_print_host_state(hba); 6405 ufshcd_print_pwr_info(hba); 6406 ufshcd_print_evt_hist(hba); 6407 ufshcd_print_tmrs(hba, hba->outstanding_tasks); 6408 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt); 6409 spin_lock_irqsave(hba->host->host_lock, flags); 6410 } 6411 6412 /* 6413 * if host reset is required then skip clearing the pending 6414 * transfers forcefully because they will get cleared during 6415 * host reset and restore 6416 */ 6417 if (hba->force_reset || ufshcd_is_link_broken(hba) || 6418 ufshcd_is_saved_err_fatal(hba) || 6419 ((hba->saved_err & UIC_ERROR) && 6420 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR | 6421 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) { 6422 needs_reset = true; 6423 goto do_reset; 6424 } 6425 6426 /* 6427 * If LINERESET was caught, UFS might have been put to PWM mode, 6428 * check if power mode restore is needed. 6429 */ 6430 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) { 6431 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6432 if (!hba->saved_uic_err) 6433 hba->saved_err &= ~UIC_ERROR; 6434 spin_unlock_irqrestore(hba->host->host_lock, flags); 6435 if (ufshcd_is_pwr_mode_restore_needed(hba)) 6436 needs_restore = true; 6437 spin_lock_irqsave(hba->host->host_lock, flags); 6438 if (!hba->saved_err && !needs_restore) 6439 goto skip_err_handling; 6440 } 6441 6442 hba->silence_err_logs = true; 6443 /* release lock as clear command might sleep */ 6444 spin_unlock_irqrestore(hba->host->host_lock, flags); 6445 6446 needs_reset = ufshcd_abort_all(hba); 6447 6448 spin_lock_irqsave(hba->host->host_lock, flags); 6449 hba->silence_err_logs = false; 6450 if (needs_reset) 6451 goto do_reset; 6452 6453 /* 6454 * After all reqs and tasks are cleared from doorbell, 6455 * now it is safe to retore power mode. 6456 */ 6457 if (needs_restore) { 6458 spin_unlock_irqrestore(hba->host->host_lock, flags); 6459 /* 6460 * Hold the scaling lock just in case dev cmds 6461 * are sent via bsg and/or sysfs. 6462 */ 6463 down_write(&hba->clk_scaling_lock); 6464 hba->force_pmc = true; 6465 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info)); 6466 if (pmc_err) { 6467 needs_reset = true; 6468 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n", 6469 __func__, pmc_err); 6470 } 6471 hba->force_pmc = false; 6472 ufshcd_print_pwr_info(hba); 6473 up_write(&hba->clk_scaling_lock); 6474 spin_lock_irqsave(hba->host->host_lock, flags); 6475 } 6476 6477 do_reset: 6478 /* Fatal errors need reset */ 6479 if (needs_reset) { 6480 int err; 6481 6482 hba->force_reset = false; 6483 spin_unlock_irqrestore(hba->host->host_lock, flags); 6484 err = ufshcd_reset_and_restore(hba); 6485 if (err) 6486 dev_err(hba->dev, "%s: reset and restore failed with err %d\n", 6487 __func__, err); 6488 else 6489 ufshcd_recover_pm_error(hba); 6490 spin_lock_irqsave(hba->host->host_lock, flags); 6491 } 6492 6493 skip_err_handling: 6494 if (!needs_reset) { 6495 if (hba->ufshcd_state == UFSHCD_STATE_RESET) 6496 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 6497 if (hba->saved_err || hba->saved_uic_err) 6498 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x", 6499 __func__, hba->saved_err, hba->saved_uic_err); 6500 } 6501 /* Exit in an operational state or dead */ 6502 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 6503 hba->ufshcd_state != UFSHCD_STATE_ERROR) { 6504 if (--retries) 6505 goto again; 6506 hba->ufshcd_state = UFSHCD_STATE_ERROR; 6507 } 6508 ufshcd_clear_eh_in_progress(hba); 6509 spin_unlock_irqrestore(hba->host->host_lock, flags); 6510 ufshcd_err_handling_unprepare(hba); 6511 up(&hba->host_sem); 6512 6513 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__, 6514 ufshcd_state_name[hba->ufshcd_state]); 6515 } 6516 6517 /** 6518 * ufshcd_update_uic_error - check and set fatal UIC error flags. 6519 * @hba: per-adapter instance 6520 * 6521 * Returns 6522 * IRQ_HANDLED - If interrupt is valid 6523 * IRQ_NONE - If invalid interrupt 6524 */ 6525 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) 6526 { 6527 u32 reg; 6528 irqreturn_t retval = IRQ_NONE; 6529 6530 /* PHY layer error */ 6531 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); 6532 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && 6533 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) { 6534 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg); 6535 /* 6536 * To know whether this error is fatal or not, DB timeout 6537 * must be checked but this error is handled separately. 6538 */ 6539 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK) 6540 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", 6541 __func__); 6542 6543 /* Got a LINERESET indication. */ 6544 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) { 6545 struct uic_command *cmd = NULL; 6546 6547 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR; 6548 if (hba->uic_async_done && hba->active_uic_cmd) 6549 cmd = hba->active_uic_cmd; 6550 /* 6551 * Ignore the LINERESET during power mode change 6552 * operation via DME_SET command. 6553 */ 6554 if (cmd && (cmd->command == UIC_CMD_DME_SET)) 6555 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR; 6556 } 6557 retval |= IRQ_HANDLED; 6558 } 6559 6560 /* PA_INIT_ERROR is fatal and needs UIC reset */ 6561 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); 6562 if ((reg & UIC_DATA_LINK_LAYER_ERROR) && 6563 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) { 6564 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg); 6565 6566 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) 6567 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; 6568 else if (hba->dev_quirks & 6569 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { 6570 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED) 6571 hba->uic_error |= 6572 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR; 6573 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT) 6574 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR; 6575 } 6576 retval |= IRQ_HANDLED; 6577 } 6578 6579 /* UIC NL/TL/DME errors needs software retry */ 6580 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); 6581 if ((reg & UIC_NETWORK_LAYER_ERROR) && 6582 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) { 6583 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg); 6584 hba->uic_error |= UFSHCD_UIC_NL_ERROR; 6585 retval |= IRQ_HANDLED; 6586 } 6587 6588 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); 6589 if ((reg & UIC_TRANSPORT_LAYER_ERROR) && 6590 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) { 6591 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg); 6592 hba->uic_error |= UFSHCD_UIC_TL_ERROR; 6593 retval |= IRQ_HANDLED; 6594 } 6595 6596 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); 6597 if ((reg & UIC_DME_ERROR) && 6598 (reg & UIC_DME_ERROR_CODE_MASK)) { 6599 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg); 6600 hba->uic_error |= UFSHCD_UIC_DME_ERROR; 6601 retval |= IRQ_HANDLED; 6602 } 6603 6604 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", 6605 __func__, hba->uic_error); 6606 return retval; 6607 } 6608 6609 /** 6610 * ufshcd_check_errors - Check for errors that need s/w attention 6611 * @hba: per-adapter instance 6612 * @intr_status: interrupt status generated by the controller 6613 * 6614 * Returns 6615 * IRQ_HANDLED - If interrupt is valid 6616 * IRQ_NONE - If invalid interrupt 6617 */ 6618 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) 6619 { 6620 bool queue_eh_work = false; 6621 irqreturn_t retval = IRQ_NONE; 6622 6623 spin_lock(hba->host->host_lock); 6624 hba->errors |= UFSHCD_ERROR_MASK & intr_status; 6625 6626 if (hba->errors & INT_FATAL_ERRORS) { 6627 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR, 6628 hba->errors); 6629 queue_eh_work = true; 6630 } 6631 6632 if (hba->errors & UIC_ERROR) { 6633 hba->uic_error = 0; 6634 retval = ufshcd_update_uic_error(hba); 6635 if (hba->uic_error) 6636 queue_eh_work = true; 6637 } 6638 6639 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) { 6640 dev_err(hba->dev, 6641 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n", 6642 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ? 6643 "Enter" : "Exit", 6644 hba->errors, ufshcd_get_upmcrs(hba)); 6645 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR, 6646 hba->errors); 6647 ufshcd_set_link_broken(hba); 6648 queue_eh_work = true; 6649 } 6650 6651 if (queue_eh_work) { 6652 /* 6653 * update the transfer error masks to sticky bits, let's do this 6654 * irrespective of current ufshcd_state. 6655 */ 6656 hba->saved_err |= hba->errors; 6657 hba->saved_uic_err |= hba->uic_error; 6658 6659 /* dump controller state before resetting */ 6660 if ((hba->saved_err & 6661 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) || 6662 (hba->saved_uic_err && 6663 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) { 6664 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n", 6665 __func__, hba->saved_err, 6666 hba->saved_uic_err); 6667 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, 6668 "host_regs: "); 6669 ufshcd_print_pwr_info(hba); 6670 } 6671 ufshcd_schedule_eh_work(hba); 6672 retval |= IRQ_HANDLED; 6673 } 6674 /* 6675 * if (!queue_eh_work) - 6676 * Other errors are either non-fatal where host recovers 6677 * itself without s/w intervention or errors that will be 6678 * handled by the SCSI core layer. 6679 */ 6680 hba->errors = 0; 6681 hba->uic_error = 0; 6682 spin_unlock(hba->host->host_lock); 6683 return retval; 6684 } 6685 6686 /** 6687 * ufshcd_tmc_handler - handle task management function completion 6688 * @hba: per adapter instance 6689 * 6690 * Returns 6691 * IRQ_HANDLED - If interrupt is valid 6692 * IRQ_NONE - If invalid interrupt 6693 */ 6694 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) 6695 { 6696 unsigned long flags, pending, issued; 6697 irqreturn_t ret = IRQ_NONE; 6698 int tag; 6699 6700 spin_lock_irqsave(hba->host->host_lock, flags); 6701 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 6702 issued = hba->outstanding_tasks & ~pending; 6703 for_each_set_bit(tag, &issued, hba->nutmrs) { 6704 struct request *req = hba->tmf_rqs[tag]; 6705 struct completion *c = req->end_io_data; 6706 6707 complete(c); 6708 ret = IRQ_HANDLED; 6709 } 6710 spin_unlock_irqrestore(hba->host->host_lock, flags); 6711 6712 return ret; 6713 } 6714 6715 /** 6716 * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events 6717 * @hba: per adapter instance 6718 * 6719 * Returns IRQ_HANDLED if interrupt is handled 6720 */ 6721 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) 6722 { 6723 struct ufs_hw_queue *hwq; 6724 unsigned long outstanding_cqs; 6725 unsigned int nr_queues; 6726 int i, ret; 6727 u32 events; 6728 6729 ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); 6730 if (ret) 6731 outstanding_cqs = (1U << hba->nr_hw_queues) - 1; 6732 6733 /* Exclude the poll queues */ 6734 nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; 6735 for_each_set_bit(i, &outstanding_cqs, nr_queues) { 6736 hwq = &hba->uhq[i]; 6737 6738 events = ufshcd_mcq_read_cqis(hba, i); 6739 if (events) 6740 ufshcd_mcq_write_cqis(hba, events, i); 6741 6742 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) 6743 ufshcd_mcq_poll_cqe_nolock(hba, hwq); 6744 } 6745 6746 return IRQ_HANDLED; 6747 } 6748 6749 /** 6750 * ufshcd_sl_intr - Interrupt service routine 6751 * @hba: per adapter instance 6752 * @intr_status: contains interrupts generated by the controller 6753 * 6754 * Returns 6755 * IRQ_HANDLED - If interrupt is valid 6756 * IRQ_NONE - If invalid interrupt 6757 */ 6758 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) 6759 { 6760 irqreturn_t retval = IRQ_NONE; 6761 6762 if (intr_status & UFSHCD_UIC_MASK) 6763 retval |= ufshcd_uic_cmd_compl(hba, intr_status); 6764 6765 if (intr_status & UFSHCD_ERROR_MASK || hba->errors) 6766 retval |= ufshcd_check_errors(hba, intr_status); 6767 6768 if (intr_status & UTP_TASK_REQ_COMPL) 6769 retval |= ufshcd_tmc_handler(hba); 6770 6771 if (intr_status & UTP_TRANSFER_REQ_COMPL) 6772 retval |= ufshcd_transfer_req_compl(hba); 6773 6774 if (intr_status & MCQ_CQ_EVENT_STATUS) 6775 retval |= ufshcd_handle_mcq_cq_events(hba); 6776 6777 return retval; 6778 } 6779 6780 /** 6781 * ufshcd_intr - Main interrupt service routine 6782 * @irq: irq number 6783 * @__hba: pointer to adapter instance 6784 * 6785 * Returns 6786 * IRQ_HANDLED - If interrupt is valid 6787 * IRQ_NONE - If invalid interrupt 6788 */ 6789 static irqreturn_t ufshcd_intr(int irq, void *__hba) 6790 { 6791 u32 intr_status, enabled_intr_status = 0; 6792 irqreturn_t retval = IRQ_NONE; 6793 struct ufs_hba *hba = __hba; 6794 int retries = hba->nutrs; 6795 6796 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 6797 hba->ufs_stats.last_intr_status = intr_status; 6798 hba->ufs_stats.last_intr_ts = local_clock(); 6799 6800 /* 6801 * There could be max of hba->nutrs reqs in flight and in worst case 6802 * if the reqs get finished 1 by 1 after the interrupt status is 6803 * read, make sure we handle them by checking the interrupt status 6804 * again in a loop until we process all of the reqs before returning. 6805 */ 6806 while (intr_status && retries--) { 6807 enabled_intr_status = 6808 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 6809 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 6810 if (enabled_intr_status) 6811 retval |= ufshcd_sl_intr(hba, enabled_intr_status); 6812 6813 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 6814 } 6815 6816 if (enabled_intr_status && retval == IRQ_NONE && 6817 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) || 6818 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) { 6819 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n", 6820 __func__, 6821 intr_status, 6822 hba->ufs_stats.last_intr_status, 6823 enabled_intr_status); 6824 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); 6825 } 6826 6827 return retval; 6828 } 6829 6830 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) 6831 { 6832 int err = 0; 6833 u32 mask = 1 << tag; 6834 unsigned long flags; 6835 6836 if (!test_bit(tag, &hba->outstanding_tasks)) 6837 goto out; 6838 6839 spin_lock_irqsave(hba->host->host_lock, flags); 6840 ufshcd_utmrl_clear(hba, tag); 6841 spin_unlock_irqrestore(hba->host->host_lock, flags); 6842 6843 /* poll for max. 1 sec to clear door bell register by h/w */ 6844 err = ufshcd_wait_for_register(hba, 6845 REG_UTP_TASK_REQ_DOOR_BELL, 6846 mask, 0, 1000, 1000); 6847 6848 dev_err(hba->dev, "Clearing task management function with tag %d %s\n", 6849 tag, err ? "succeeded" : "failed"); 6850 6851 out: 6852 return err; 6853 } 6854 6855 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, 6856 struct utp_task_req_desc *treq, u8 tm_function) 6857 { 6858 struct request_queue *q = hba->tmf_queue; 6859 struct Scsi_Host *host = hba->host; 6860 DECLARE_COMPLETION_ONSTACK(wait); 6861 struct request *req; 6862 unsigned long flags; 6863 int task_tag, err; 6864 6865 /* 6866 * blk_mq_alloc_request() is used here only to get a free tag. 6867 */ 6868 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0); 6869 if (IS_ERR(req)) 6870 return PTR_ERR(req); 6871 6872 req->end_io_data = &wait; 6873 ufshcd_hold(hba, false); 6874 6875 spin_lock_irqsave(host->host_lock, flags); 6876 6877 task_tag = req->tag; 6878 WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n", 6879 task_tag); 6880 hba->tmf_rqs[req->tag] = req; 6881 treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag); 6882 6883 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); 6884 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); 6885 6886 /* send command to the controller */ 6887 __set_bit(task_tag, &hba->outstanding_tasks); 6888 6889 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); 6890 /* Make sure that doorbell is committed immediately */ 6891 wmb(); 6892 6893 spin_unlock_irqrestore(host->host_lock, flags); 6894 6895 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND); 6896 6897 /* wait until the task management command is completed */ 6898 err = wait_for_completion_io_timeout(&wait, 6899 msecs_to_jiffies(TM_CMD_TIMEOUT)); 6900 if (!err) { 6901 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR); 6902 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", 6903 __func__, tm_function); 6904 if (ufshcd_clear_tm_cmd(hba, task_tag)) 6905 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n", 6906 __func__, task_tag); 6907 err = -ETIMEDOUT; 6908 } else { 6909 err = 0; 6910 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq)); 6911 6912 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP); 6913 } 6914 6915 spin_lock_irqsave(hba->host->host_lock, flags); 6916 hba->tmf_rqs[req->tag] = NULL; 6917 __clear_bit(task_tag, &hba->outstanding_tasks); 6918 spin_unlock_irqrestore(hba->host->host_lock, flags); 6919 6920 ufshcd_release(hba); 6921 blk_mq_free_request(req); 6922 6923 return err; 6924 } 6925 6926 /** 6927 * ufshcd_issue_tm_cmd - issues task management commands to controller 6928 * @hba: per adapter instance 6929 * @lun_id: LUN ID to which TM command is sent 6930 * @task_id: task ID to which the TM command is applicable 6931 * @tm_function: task management function opcode 6932 * @tm_response: task management service response return value 6933 * 6934 * Returns non-zero value on error, zero on success. 6935 */ 6936 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, 6937 u8 tm_function, u8 *tm_response) 6938 { 6939 struct utp_task_req_desc treq = { { 0 }, }; 6940 enum utp_ocs ocs_value; 6941 int err; 6942 6943 /* Configure task request descriptor */ 6944 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); 6945 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); 6946 6947 /* Configure task request UPIU */ 6948 treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) | 6949 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24); 6950 treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16); 6951 6952 /* 6953 * The host shall provide the same value for LUN field in the basic 6954 * header and for Input Parameter. 6955 */ 6956 treq.upiu_req.input_param1 = cpu_to_be32(lun_id); 6957 treq.upiu_req.input_param2 = cpu_to_be32(task_id); 6958 6959 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function); 6960 if (err == -ETIMEDOUT) 6961 return err; 6962 6963 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; 6964 if (ocs_value != OCS_SUCCESS) 6965 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", 6966 __func__, ocs_value); 6967 else if (tm_response) 6968 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) & 6969 MASK_TM_SERVICE_RESP; 6970 return err; 6971 } 6972 6973 /** 6974 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests 6975 * @hba: per-adapter instance 6976 * @req_upiu: upiu request 6977 * @rsp_upiu: upiu reply 6978 * @desc_buff: pointer to descriptor buffer, NULL if NA 6979 * @buff_len: descriptor size, 0 if NA 6980 * @cmd_type: specifies the type (NOP, Query...) 6981 * @desc_op: descriptor operation 6982 * 6983 * Those type of requests uses UTP Transfer Request Descriptor - utrd. 6984 * Therefore, it "rides" the device management infrastructure: uses its tag and 6985 * tasks work queues. 6986 * 6987 * Since there is only one available tag for device management commands, 6988 * the caller is expected to hold the hba->dev_cmd.lock mutex. 6989 */ 6990 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, 6991 struct utp_upiu_req *req_upiu, 6992 struct utp_upiu_req *rsp_upiu, 6993 u8 *desc_buff, int *buff_len, 6994 enum dev_cmd_type cmd_type, 6995 enum query_opcode desc_op) 6996 { 6997 DECLARE_COMPLETION_ONSTACK(wait); 6998 const u32 tag = hba->reserved_slot; 6999 struct ufshcd_lrb *lrbp; 7000 int err = 0; 7001 u8 upiu_flags; 7002 7003 /* Protects use of hba->reserved_slot. */ 7004 lockdep_assert_held(&hba->dev_cmd.lock); 7005 7006 down_read(&hba->clk_scaling_lock); 7007 7008 lrbp = &hba->lrb[tag]; 7009 WARN_ON(lrbp->cmd); 7010 lrbp->cmd = NULL; 7011 lrbp->task_tag = tag; 7012 lrbp->lun = 0; 7013 lrbp->intr_cmd = true; 7014 ufshcd_prepare_lrbp_crypto(NULL, lrbp); 7015 hba->dev_cmd.type = cmd_type; 7016 7017 if (hba->ufs_version <= ufshci_version(1, 1)) 7018 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; 7019 else 7020 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 7021 7022 /* update the task tag in the request upiu */ 7023 req_upiu->header.dword_0 |= cpu_to_be32(tag); 7024 7025 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE, 0); 7026 7027 /* just copy the upiu request as it is */ 7028 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7029 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) { 7030 /* The Data Segment Area is optional depending upon the query 7031 * function value. for WRITE DESCRIPTOR, the data segment 7032 * follows right after the tsf. 7033 */ 7034 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len); 7035 *buff_len = 0; 7036 } 7037 7038 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7039 7040 hba->dev_cmd.complete = &wait; 7041 7042 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); 7043 7044 ufshcd_send_command(hba, tag, hba->dev_cmd_queue); 7045 /* 7046 * ignore the returning value here - ufshcd_check_query_response is 7047 * bound to fail since dev_cmd.query and dev_cmd.type were left empty. 7048 * read the response directly ignoring all errors. 7049 */ 7050 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT); 7051 7052 /* just copy the upiu response as it is */ 7053 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7054 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { 7055 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); 7056 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & 7057 MASK_QUERY_DATA_SEG_LEN; 7058 7059 if (*buff_len >= resp_len) { 7060 memcpy(desc_buff, descp, resp_len); 7061 *buff_len = resp_len; 7062 } else { 7063 dev_warn(hba->dev, 7064 "%s: rsp size %d is bigger than buffer size %d", 7065 __func__, resp_len, *buff_len); 7066 *buff_len = 0; 7067 err = -EINVAL; 7068 } 7069 } 7070 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, 7071 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); 7072 7073 up_read(&hba->clk_scaling_lock); 7074 return err; 7075 } 7076 7077 /** 7078 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands 7079 * @hba: per-adapter instance 7080 * @req_upiu: upiu request 7081 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands 7082 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target 7083 * @desc_buff: pointer to descriptor buffer, NULL if NA 7084 * @buff_len: descriptor size, 0 if NA 7085 * @desc_op: descriptor operation 7086 * 7087 * Supports UTP Transfer requests (nop and query), and UTP Task 7088 * Management requests. 7089 * It is up to the caller to fill the upiu conent properly, as it will 7090 * be copied without any further input validations. 7091 */ 7092 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, 7093 struct utp_upiu_req *req_upiu, 7094 struct utp_upiu_req *rsp_upiu, 7095 int msgcode, 7096 u8 *desc_buff, int *buff_len, 7097 enum query_opcode desc_op) 7098 { 7099 int err; 7100 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; 7101 struct utp_task_req_desc treq = { { 0 }, }; 7102 enum utp_ocs ocs_value; 7103 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC; 7104 7105 switch (msgcode) { 7106 case UPIU_TRANSACTION_NOP_OUT: 7107 cmd_type = DEV_CMD_TYPE_NOP; 7108 fallthrough; 7109 case UPIU_TRANSACTION_QUERY_REQ: 7110 ufshcd_hold(hba, false); 7111 mutex_lock(&hba->dev_cmd.lock); 7112 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu, 7113 desc_buff, buff_len, 7114 cmd_type, desc_op); 7115 mutex_unlock(&hba->dev_cmd.lock); 7116 ufshcd_release(hba); 7117 7118 break; 7119 case UPIU_TRANSACTION_TASK_REQ: 7120 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); 7121 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); 7122 7123 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); 7124 7125 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f); 7126 if (err == -ETIMEDOUT) 7127 break; 7128 7129 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; 7130 if (ocs_value != OCS_SUCCESS) { 7131 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, 7132 ocs_value); 7133 break; 7134 } 7135 7136 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu)); 7137 7138 break; 7139 default: 7140 err = -EINVAL; 7141 7142 break; 7143 } 7144 7145 return err; 7146 } 7147 7148 /** 7149 * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request 7150 * @hba: per adapter instance 7151 * @req_upiu: upiu request 7152 * @rsp_upiu: upiu reply 7153 * @req_ehs: EHS field which contains Advanced RPMB Request Message 7154 * @rsp_ehs: EHS field which returns Advanced RPMB Response Message 7155 * @sg_cnt: The number of sg lists actually used 7156 * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation 7157 * @dir: DMA direction 7158 * 7159 * Returns zero on success, non-zero on failure 7160 */ 7161 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, 7162 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, 7163 struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, 7164 enum dma_data_direction dir) 7165 { 7166 DECLARE_COMPLETION_ONSTACK(wait); 7167 const u32 tag = hba->reserved_slot; 7168 struct ufshcd_lrb *lrbp; 7169 int err = 0; 7170 int result; 7171 u8 upiu_flags; 7172 u8 *ehs_data; 7173 u16 ehs_len; 7174 7175 /* Protects use of hba->reserved_slot. */ 7176 ufshcd_hold(hba, false); 7177 mutex_lock(&hba->dev_cmd.lock); 7178 down_read(&hba->clk_scaling_lock); 7179 7180 lrbp = &hba->lrb[tag]; 7181 WARN_ON(lrbp->cmd); 7182 lrbp->cmd = NULL; 7183 lrbp->task_tag = tag; 7184 lrbp->lun = UFS_UPIU_RPMB_WLUN; 7185 7186 lrbp->intr_cmd = true; 7187 ufshcd_prepare_lrbp_crypto(NULL, lrbp); 7188 hba->dev_cmd.type = DEV_CMD_TYPE_RPMB; 7189 7190 /* Advanced RPMB starts from UFS 4.0, so its command type is UTP_CMD_TYPE_UFS_STORAGE */ 7191 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; 7192 7193 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, dir, 2); 7194 7195 /* update the task tag and LUN in the request upiu */ 7196 req_upiu->header.dword_0 |= cpu_to_be32(upiu_flags << 16 | UFS_UPIU_RPMB_WLUN << 8 | tag); 7197 7198 /* copy the UPIU(contains CDB) request as it is */ 7199 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); 7200 /* Copy EHS, starting with byte32, immediately after the CDB package */ 7201 memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs)); 7202 7203 if (dir != DMA_NONE && sg_list) 7204 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list); 7205 7206 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); 7207 7208 hba->dev_cmd.complete = &wait; 7209 7210 ufshcd_send_command(hba, tag, hba->dev_cmd_queue); 7211 7212 err = ufshcd_wait_for_dev_cmd(hba, lrbp, ADVANCED_RPMB_REQ_TIMEOUT); 7213 7214 if (!err) { 7215 /* Just copy the upiu response as it is */ 7216 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); 7217 /* Get the response UPIU result */ 7218 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); 7219 7220 ehs_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) >> 24; 7221 /* 7222 * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data 7223 * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB 7224 * Message is 02h 7225 */ 7226 if (ehs_len == 2 && rsp_ehs) { 7227 /* 7228 * ucd_rsp_ptr points to a buffer with a length of 512 bytes 7229 * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32 7230 */ 7231 ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE; 7232 memcpy(rsp_ehs, ehs_data, ehs_len * 32); 7233 } 7234 } 7235 7236 up_read(&hba->clk_scaling_lock); 7237 mutex_unlock(&hba->dev_cmd.lock); 7238 ufshcd_release(hba); 7239 return err ? : result; 7240 } 7241 7242 /** 7243 * ufshcd_eh_device_reset_handler() - Reset a single logical unit. 7244 * @cmd: SCSI command pointer 7245 * 7246 * Returns SUCCESS/FAILED 7247 */ 7248 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 7249 { 7250 unsigned long flags, pending_reqs = 0, not_cleared = 0; 7251 struct Scsi_Host *host; 7252 struct ufs_hba *hba; 7253 u32 pos; 7254 int err; 7255 u8 resp = 0xF, lun; 7256 7257 host = cmd->device->host; 7258 hba = shost_priv(host); 7259 7260 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 7261 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); 7262 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7263 if (!err) 7264 err = resp; 7265 goto out; 7266 } 7267 7268 /* clear the commands that were pending for corresponding LUN */ 7269 spin_lock_irqsave(&hba->outstanding_lock, flags); 7270 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) 7271 if (hba->lrb[pos].lun == lun) 7272 __set_bit(pos, &pending_reqs); 7273 hba->outstanding_reqs &= ~pending_reqs; 7274 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7275 7276 if (ufshcd_clear_cmds(hba, pending_reqs) < 0) { 7277 spin_lock_irqsave(&hba->outstanding_lock, flags); 7278 not_cleared = pending_reqs & 7279 ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7280 hba->outstanding_reqs |= not_cleared; 7281 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7282 7283 dev_err(hba->dev, "%s: failed to clear requests %#lx\n", 7284 __func__, not_cleared); 7285 } 7286 __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared); 7287 7288 out: 7289 hba->req_abort_count = 0; 7290 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); 7291 if (!err) { 7292 err = SUCCESS; 7293 } else { 7294 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7295 err = FAILED; 7296 } 7297 return err; 7298 } 7299 7300 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) 7301 { 7302 struct ufshcd_lrb *lrbp; 7303 int tag; 7304 7305 for_each_set_bit(tag, &bitmap, hba->nutrs) { 7306 lrbp = &hba->lrb[tag]; 7307 lrbp->req_abort_skip = true; 7308 } 7309 } 7310 7311 /** 7312 * ufshcd_try_to_abort_task - abort a specific task 7313 * @hba: Pointer to adapter instance 7314 * @tag: Task tag/index to be aborted 7315 * 7316 * Abort the pending command in device by sending UFS_ABORT_TASK task management 7317 * command, and in host controller by clearing the door-bell register. There can 7318 * be race between controller sending the command to the device while abort is 7319 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is 7320 * really issued and then try to abort it. 7321 * 7322 * Returns zero on success, non-zero on failure 7323 */ 7324 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) 7325 { 7326 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7327 int err = 0; 7328 int poll_cnt; 7329 u8 resp = 0xF; 7330 u32 reg; 7331 7332 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 7333 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 7334 UFS_QUERY_TASK, &resp); 7335 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 7336 /* cmd pending in the device */ 7337 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", 7338 __func__, tag); 7339 break; 7340 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7341 /* 7342 * cmd not pending in the device, check if it is 7343 * in transition. 7344 */ 7345 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n", 7346 __func__, tag); 7347 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7348 if (reg & (1 << tag)) { 7349 /* sleep for max. 200us to stabilize */ 7350 usleep_range(100, 200); 7351 continue; 7352 } 7353 /* command completed already */ 7354 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n", 7355 __func__, tag); 7356 goto out; 7357 } else { 7358 dev_err(hba->dev, 7359 "%s: no response from device. tag = %d, err %d\n", 7360 __func__, tag, err); 7361 if (!err) 7362 err = resp; /* service response error */ 7363 goto out; 7364 } 7365 } 7366 7367 if (!poll_cnt) { 7368 err = -EBUSY; 7369 goto out; 7370 } 7371 7372 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 7373 UFS_ABORT_TASK, &resp); 7374 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 7375 if (!err) { 7376 err = resp; /* service response error */ 7377 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", 7378 __func__, tag, err); 7379 } 7380 goto out; 7381 } 7382 7383 err = ufshcd_clear_cmds(hba, 1U << tag); 7384 if (err) 7385 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", 7386 __func__, tag, err); 7387 7388 out: 7389 return err; 7390 } 7391 7392 /** 7393 * ufshcd_abort - scsi host template eh_abort_handler callback 7394 * @cmd: SCSI command pointer 7395 * 7396 * Returns SUCCESS/FAILED 7397 */ 7398 static int ufshcd_abort(struct scsi_cmnd *cmd) 7399 { 7400 struct Scsi_Host *host = cmd->device->host; 7401 struct ufs_hba *hba = shost_priv(host); 7402 int tag = scsi_cmd_to_rq(cmd)->tag; 7403 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 7404 unsigned long flags; 7405 int err = FAILED; 7406 bool outstanding; 7407 u32 reg; 7408 7409 WARN_ONCE(tag < 0, "Invalid tag %d\n", tag); 7410 7411 ufshcd_hold(hba, false); 7412 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 7413 /* If command is already aborted/completed, return FAILED. */ 7414 if (!(test_bit(tag, &hba->outstanding_reqs))) { 7415 dev_err(hba->dev, 7416 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", 7417 __func__, tag, hba->outstanding_reqs, reg); 7418 goto release; 7419 } 7420 7421 /* Print Transfer Request of aborted task */ 7422 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag); 7423 7424 /* 7425 * Print detailed info about aborted request. 7426 * As more than one request might get aborted at the same time, 7427 * print full information only for the first aborted request in order 7428 * to reduce repeated printouts. For other aborted requests only print 7429 * basic details. 7430 */ 7431 scsi_print_command(cmd); 7432 if (!hba->req_abort_count) { 7433 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); 7434 ufshcd_print_evt_hist(hba); 7435 ufshcd_print_host_state(hba); 7436 ufshcd_print_pwr_info(hba); 7437 ufshcd_print_trs(hba, 1 << tag, true); 7438 } else { 7439 ufshcd_print_trs(hba, 1 << tag, false); 7440 } 7441 hba->req_abort_count++; 7442 7443 if (!(reg & (1 << tag))) { 7444 dev_err(hba->dev, 7445 "%s: cmd was completed, but without a notifying intr, tag = %d", 7446 __func__, tag); 7447 __ufshcd_transfer_req_compl(hba, 1UL << tag); 7448 goto release; 7449 } 7450 7451 /* 7452 * Task abort to the device W-LUN is illegal. When this command 7453 * will fail, due to spec violation, scsi err handling next step 7454 * will be to send LU reset which, again, is a spec violation. 7455 * To avoid these unnecessary/illegal steps, first we clean up 7456 * the lrb taken by this cmd and re-set it in outstanding_reqs, 7457 * then queue the eh_work and bail. 7458 */ 7459 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) { 7460 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun); 7461 7462 spin_lock_irqsave(host->host_lock, flags); 7463 hba->force_reset = true; 7464 ufshcd_schedule_eh_work(hba); 7465 spin_unlock_irqrestore(host->host_lock, flags); 7466 goto release; 7467 } 7468 7469 /* Skip task abort in case previous aborts failed and report failure */ 7470 if (lrbp->req_abort_skip) { 7471 dev_err(hba->dev, "%s: skipping abort\n", __func__); 7472 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7473 goto release; 7474 } 7475 7476 err = ufshcd_try_to_abort_task(hba, tag); 7477 if (err) { 7478 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 7479 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); 7480 err = FAILED; 7481 goto release; 7482 } 7483 7484 /* 7485 * Clear the corresponding bit from outstanding_reqs since the command 7486 * has been aborted successfully. 7487 */ 7488 spin_lock_irqsave(&hba->outstanding_lock, flags); 7489 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs); 7490 spin_unlock_irqrestore(&hba->outstanding_lock, flags); 7491 7492 if (outstanding) 7493 ufshcd_release_scsi_cmd(hba, lrbp); 7494 7495 err = SUCCESS; 7496 7497 release: 7498 /* Matches the ufshcd_hold() call at the start of this function. */ 7499 ufshcd_release(hba); 7500 return err; 7501 } 7502 7503 /** 7504 * ufshcd_host_reset_and_restore - reset and restore host controller 7505 * @hba: per-adapter instance 7506 * 7507 * Note that host controller reset may issue DME_RESET to 7508 * local and remote (device) Uni-Pro stack and the attributes 7509 * are reset to default state. 7510 * 7511 * Returns zero on success, non-zero on failure 7512 */ 7513 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) 7514 { 7515 int err; 7516 7517 /* 7518 * Stop the host controller and complete the requests 7519 * cleared by h/w 7520 */ 7521 ufshpb_toggle_state(hba, HPB_PRESENT, HPB_RESET); 7522 ufshcd_hba_stop(hba); 7523 hba->silence_err_logs = true; 7524 ufshcd_complete_requests(hba); 7525 hba->silence_err_logs = false; 7526 7527 /* scale up clocks to max frequency before full reinitialization */ 7528 ufshcd_scale_clks(hba, true); 7529 7530 err = ufshcd_hba_enable(hba); 7531 7532 /* Establish the link again and restore the device */ 7533 if (!err) 7534 err = ufshcd_probe_hba(hba, false); 7535 7536 if (err) 7537 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); 7538 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err); 7539 return err; 7540 } 7541 7542 /** 7543 * ufshcd_reset_and_restore - reset and re-initialize host/device 7544 * @hba: per-adapter instance 7545 * 7546 * Reset and recover device, host and re-establish link. This 7547 * is helpful to recover the communication in fatal error conditions. 7548 * 7549 * Returns zero on success, non-zero on failure 7550 */ 7551 static int ufshcd_reset_and_restore(struct ufs_hba *hba) 7552 { 7553 u32 saved_err = 0; 7554 u32 saved_uic_err = 0; 7555 int err = 0; 7556 unsigned long flags; 7557 int retries = MAX_HOST_RESET_RETRIES; 7558 7559 spin_lock_irqsave(hba->host->host_lock, flags); 7560 do { 7561 /* 7562 * This is a fresh start, cache and clear saved error first, 7563 * in case new error generated during reset and restore. 7564 */ 7565 saved_err |= hba->saved_err; 7566 saved_uic_err |= hba->saved_uic_err; 7567 hba->saved_err = 0; 7568 hba->saved_uic_err = 0; 7569 hba->force_reset = false; 7570 hba->ufshcd_state = UFSHCD_STATE_RESET; 7571 spin_unlock_irqrestore(hba->host->host_lock, flags); 7572 7573 /* Reset the attached device */ 7574 ufshcd_device_reset(hba); 7575 7576 err = ufshcd_host_reset_and_restore(hba); 7577 7578 spin_lock_irqsave(hba->host->host_lock, flags); 7579 if (err) 7580 continue; 7581 /* Do not exit unless operational or dead */ 7582 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL && 7583 hba->ufshcd_state != UFSHCD_STATE_ERROR && 7584 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL) 7585 err = -EAGAIN; 7586 } while (err && --retries); 7587 7588 /* 7589 * Inform scsi mid-layer that we did reset and allow to handle 7590 * Unit Attention properly. 7591 */ 7592 scsi_report_bus_reset(hba->host, 0); 7593 if (err) { 7594 hba->ufshcd_state = UFSHCD_STATE_ERROR; 7595 hba->saved_err |= saved_err; 7596 hba->saved_uic_err |= saved_uic_err; 7597 } 7598 spin_unlock_irqrestore(hba->host->host_lock, flags); 7599 7600 return err; 7601 } 7602 7603 /** 7604 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer 7605 * @cmd: SCSI command pointer 7606 * 7607 * Returns SUCCESS/FAILED 7608 */ 7609 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) 7610 { 7611 int err = SUCCESS; 7612 unsigned long flags; 7613 struct ufs_hba *hba; 7614 7615 hba = shost_priv(cmd->device->host); 7616 7617 spin_lock_irqsave(hba->host->host_lock, flags); 7618 hba->force_reset = true; 7619 ufshcd_schedule_eh_work(hba); 7620 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__); 7621 spin_unlock_irqrestore(hba->host->host_lock, flags); 7622 7623 flush_work(&hba->eh_work); 7624 7625 spin_lock_irqsave(hba->host->host_lock, flags); 7626 if (hba->ufshcd_state == UFSHCD_STATE_ERROR) 7627 err = FAILED; 7628 spin_unlock_irqrestore(hba->host->host_lock, flags); 7629 7630 return err; 7631 } 7632 7633 /** 7634 * ufshcd_get_max_icc_level - calculate the ICC level 7635 * @sup_curr_uA: max. current supported by the regulator 7636 * @start_scan: row at the desc table to start scan from 7637 * @buff: power descriptor buffer 7638 * 7639 * Returns calculated max ICC level for specific regulator 7640 */ 7641 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, 7642 const char *buff) 7643 { 7644 int i; 7645 int curr_uA; 7646 u16 data; 7647 u16 unit; 7648 7649 for (i = start_scan; i >= 0; i--) { 7650 data = get_unaligned_be16(&buff[2 * i]); 7651 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> 7652 ATTR_ICC_LVL_UNIT_OFFSET; 7653 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; 7654 switch (unit) { 7655 case UFSHCD_NANO_AMP: 7656 curr_uA = curr_uA / 1000; 7657 break; 7658 case UFSHCD_MILI_AMP: 7659 curr_uA = curr_uA * 1000; 7660 break; 7661 case UFSHCD_AMP: 7662 curr_uA = curr_uA * 1000 * 1000; 7663 break; 7664 case UFSHCD_MICRO_AMP: 7665 default: 7666 break; 7667 } 7668 if (sup_curr_uA >= curr_uA) 7669 break; 7670 } 7671 if (i < 0) { 7672 i = 0; 7673 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i); 7674 } 7675 7676 return (u32)i; 7677 } 7678 7679 /** 7680 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level 7681 * In case regulators are not initialized we'll return 0 7682 * @hba: per-adapter instance 7683 * @desc_buf: power descriptor buffer to extract ICC levels from. 7684 * 7685 * Returns calculated ICC level 7686 */ 7687 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, 7688 const u8 *desc_buf) 7689 { 7690 u32 icc_level = 0; 7691 7692 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq || 7693 !hba->vreg_info.vccq2) { 7694 /* 7695 * Using dev_dbg to avoid messages during runtime PM to avoid 7696 * never-ending cycles of messages written back to storage by 7697 * user space causing runtime resume, causing more messages and 7698 * so on. 7699 */ 7700 dev_dbg(hba->dev, 7701 "%s: Regulator capability was not set, actvIccLevel=%d", 7702 __func__, icc_level); 7703 goto out; 7704 } 7705 7706 if (hba->vreg_info.vcc->max_uA) 7707 icc_level = ufshcd_get_max_icc_level( 7708 hba->vreg_info.vcc->max_uA, 7709 POWER_DESC_MAX_ACTV_ICC_LVLS - 1, 7710 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]); 7711 7712 if (hba->vreg_info.vccq->max_uA) 7713 icc_level = ufshcd_get_max_icc_level( 7714 hba->vreg_info.vccq->max_uA, 7715 icc_level, 7716 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]); 7717 7718 if (hba->vreg_info.vccq2->max_uA) 7719 icc_level = ufshcd_get_max_icc_level( 7720 hba->vreg_info.vccq2->max_uA, 7721 icc_level, 7722 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]); 7723 out: 7724 return icc_level; 7725 } 7726 7727 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) 7728 { 7729 int ret; 7730 u8 *desc_buf; 7731 u32 icc_level; 7732 7733 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 7734 if (!desc_buf) 7735 return; 7736 7737 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0, 7738 desc_buf, QUERY_DESC_MAX_SIZE); 7739 if (ret) { 7740 dev_err(hba->dev, 7741 "%s: Failed reading power descriptor ret = %d", 7742 __func__, ret); 7743 goto out; 7744 } 7745 7746 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf); 7747 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level); 7748 7749 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 7750 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level); 7751 7752 if (ret) 7753 dev_err(hba->dev, 7754 "%s: Failed configuring bActiveICCLevel = %d ret = %d", 7755 __func__, icc_level, ret); 7756 7757 out: 7758 kfree(desc_buf); 7759 } 7760 7761 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) 7762 { 7763 scsi_autopm_get_device(sdev); 7764 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); 7765 if (sdev->rpm_autosuspend) 7766 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, 7767 RPM_AUTOSUSPEND_DELAY_MS); 7768 scsi_autopm_put_device(sdev); 7769 } 7770 7771 /** 7772 * ufshcd_scsi_add_wlus - Adds required W-LUs 7773 * @hba: per-adapter instance 7774 * 7775 * UFS device specification requires the UFS devices to support 4 well known 7776 * logical units: 7777 * "REPORT_LUNS" (address: 01h) 7778 * "UFS Device" (address: 50h) 7779 * "RPMB" (address: 44h) 7780 * "BOOT" (address: 30h) 7781 * UFS device's power management needs to be controlled by "POWER CONDITION" 7782 * field of SSU (START STOP UNIT) command. But this "power condition" field 7783 * will take effect only when its sent to "UFS device" well known logical unit 7784 * hence we require the scsi_device instance to represent this logical unit in 7785 * order for the UFS host driver to send the SSU command for power management. 7786 * 7787 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory 7788 * Block) LU so user space process can control this LU. User space may also 7789 * want to have access to BOOT LU. 7790 * 7791 * This function adds scsi device instances for each of all well known LUs 7792 * (except "REPORT LUNS" LU). 7793 * 7794 * Returns zero on success (all required W-LUs are added successfully), 7795 * non-zero error value on failure (if failed to add any of the required W-LU). 7796 */ 7797 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) 7798 { 7799 int ret = 0; 7800 struct scsi_device *sdev_boot, *sdev_rpmb; 7801 7802 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0, 7803 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL); 7804 if (IS_ERR(hba->ufs_device_wlun)) { 7805 ret = PTR_ERR(hba->ufs_device_wlun); 7806 hba->ufs_device_wlun = NULL; 7807 goto out; 7808 } 7809 scsi_device_put(hba->ufs_device_wlun); 7810 7811 sdev_rpmb = __scsi_add_device(hba->host, 0, 0, 7812 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); 7813 if (IS_ERR(sdev_rpmb)) { 7814 ret = PTR_ERR(sdev_rpmb); 7815 goto remove_ufs_device_wlun; 7816 } 7817 ufshcd_blk_pm_runtime_init(sdev_rpmb); 7818 scsi_device_put(sdev_rpmb); 7819 7820 sdev_boot = __scsi_add_device(hba->host, 0, 0, 7821 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); 7822 if (IS_ERR(sdev_boot)) { 7823 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__); 7824 } else { 7825 ufshcd_blk_pm_runtime_init(sdev_boot); 7826 scsi_device_put(sdev_boot); 7827 } 7828 goto out; 7829 7830 remove_ufs_device_wlun: 7831 scsi_remove_device(hba->ufs_device_wlun); 7832 out: 7833 return ret; 7834 } 7835 7836 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) 7837 { 7838 struct ufs_dev_info *dev_info = &hba->dev_info; 7839 u8 lun; 7840 u32 d_lu_wb_buf_alloc; 7841 u32 ext_ufs_feature; 7842 7843 if (!ufshcd_is_wb_allowed(hba)) 7844 return; 7845 7846 /* 7847 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or 7848 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES 7849 * enabled 7850 */ 7851 if (!(dev_info->wspecversion >= 0x310 || 7852 dev_info->wspecversion == 0x220 || 7853 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) 7854 goto wb_disabled; 7855 7856 ext_ufs_feature = get_unaligned_be32(desc_buf + 7857 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 7858 7859 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP)) 7860 goto wb_disabled; 7861 7862 /* 7863 * WB may be supported but not configured while provisioning. The spec 7864 * says, in dedicated wb buffer mode, a max of 1 lun would have wb 7865 * buffer configured. 7866 */ 7867 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; 7868 7869 dev_info->b_presrv_uspc_en = 7870 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; 7871 7872 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) { 7873 if (!get_unaligned_be32(desc_buf + 7874 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS)) 7875 goto wb_disabled; 7876 } else { 7877 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) { 7878 d_lu_wb_buf_alloc = 0; 7879 ufshcd_read_unit_desc_param(hba, 7880 lun, 7881 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS, 7882 (u8 *)&d_lu_wb_buf_alloc, 7883 sizeof(d_lu_wb_buf_alloc)); 7884 if (d_lu_wb_buf_alloc) { 7885 dev_info->wb_dedicated_lu = lun; 7886 break; 7887 } 7888 } 7889 7890 if (!d_lu_wb_buf_alloc) 7891 goto wb_disabled; 7892 } 7893 7894 if (!ufshcd_is_wb_buf_lifetime_available(hba)) 7895 goto wb_disabled; 7896 7897 return; 7898 7899 wb_disabled: 7900 hba->caps &= ~UFSHCD_CAP_WB_EN; 7901 } 7902 7903 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) 7904 { 7905 struct ufs_dev_info *dev_info = &hba->dev_info; 7906 u32 ext_ufs_feature; 7907 u8 mask = 0; 7908 7909 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300) 7910 return; 7911 7912 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 7913 7914 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF) 7915 mask |= MASK_EE_TOO_LOW_TEMP; 7916 7917 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF) 7918 mask |= MASK_EE_TOO_HIGH_TEMP; 7919 7920 if (mask) { 7921 ufshcd_enable_ee(hba, mask); 7922 ufs_hwmon_probe(hba, mask); 7923 } 7924 } 7925 7926 static void ufshcd_ext_iid_probe(struct ufs_hba *hba, u8 *desc_buf) 7927 { 7928 struct ufs_dev_info *dev_info = &hba->dev_info; 7929 u32 ext_ufs_feature; 7930 u32 ext_iid_en = 0; 7931 int err; 7932 7933 /* Only UFS-4.0 and above may support EXT_IID */ 7934 if (dev_info->wspecversion < 0x400) 7935 goto out; 7936 7937 ext_ufs_feature = get_unaligned_be32(desc_buf + 7938 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); 7939 if (!(ext_ufs_feature & UFS_DEV_EXT_IID_SUP)) 7940 goto out; 7941 7942 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 7943 QUERY_ATTR_IDN_EXT_IID_EN, 0, 0, &ext_iid_en); 7944 if (err) 7945 dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err); 7946 7947 out: 7948 dev_info->b_ext_iid_en = ext_iid_en; 7949 } 7950 7951 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, 7952 const struct ufs_dev_quirk *fixups) 7953 { 7954 const struct ufs_dev_quirk *f; 7955 struct ufs_dev_info *dev_info = &hba->dev_info; 7956 7957 if (!fixups) 7958 return; 7959 7960 for (f = fixups; f->quirk; f++) { 7961 if ((f->wmanufacturerid == dev_info->wmanufacturerid || 7962 f->wmanufacturerid == UFS_ANY_VENDOR) && 7963 ((dev_info->model && 7964 STR_PRFX_EQUAL(f->model, dev_info->model)) || 7965 !strcmp(f->model, UFS_ANY_MODEL))) 7966 hba->dev_quirks |= f->quirk; 7967 } 7968 } 7969 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks); 7970 7971 static void ufs_fixup_device_setup(struct ufs_hba *hba) 7972 { 7973 /* fix by general quirk table */ 7974 ufshcd_fixup_dev_quirks(hba, ufs_fixups); 7975 7976 /* allow vendors to fix quirks */ 7977 ufshcd_vops_fixup_dev_quirks(hba); 7978 } 7979 7980 static int ufs_get_device_desc(struct ufs_hba *hba) 7981 { 7982 int err; 7983 u8 model_index; 7984 u8 b_ufs_feature_sup; 7985 u8 *desc_buf; 7986 struct ufs_dev_info *dev_info = &hba->dev_info; 7987 7988 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 7989 if (!desc_buf) { 7990 err = -ENOMEM; 7991 goto out; 7992 } 7993 7994 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf, 7995 QUERY_DESC_MAX_SIZE); 7996 if (err) { 7997 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", 7998 __func__, err); 7999 goto out; 8000 } 8001 8002 /* 8003 * getting vendor (manufacturerID) and Bank Index in big endian 8004 * format 8005 */ 8006 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | 8007 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1]; 8008 8009 /* getting Specification Version in big endian format */ 8010 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | 8011 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; 8012 dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; 8013 b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT]; 8014 8015 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; 8016 8017 if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION && 8018 (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) { 8019 bool hpb_en = false; 8020 8021 ufshpb_get_dev_info(hba, desc_buf); 8022 8023 if (!ufshpb_is_legacy(hba)) 8024 err = ufshcd_query_flag_retry(hba, 8025 UPIU_QUERY_OPCODE_READ_FLAG, 8026 QUERY_FLAG_IDN_HPB_EN, 0, 8027 &hpb_en); 8028 8029 if (ufshpb_is_legacy(hba) || (!err && hpb_en)) 8030 dev_info->hpb_enabled = true; 8031 } 8032 8033 err = ufshcd_read_string_desc(hba, model_index, 8034 &dev_info->model, SD_ASCII_STD); 8035 if (err < 0) { 8036 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n", 8037 __func__, err); 8038 goto out; 8039 } 8040 8041 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] + 8042 desc_buf[DEVICE_DESC_PARAM_NUM_WLU]; 8043 8044 ufs_fixup_device_setup(hba); 8045 8046 ufshcd_wb_probe(hba, desc_buf); 8047 8048 ufshcd_temp_notif_probe(hba, desc_buf); 8049 8050 if (hba->ext_iid_sup) 8051 ufshcd_ext_iid_probe(hba, desc_buf); 8052 8053 /* 8054 * ufshcd_read_string_desc returns size of the string 8055 * reset the error value 8056 */ 8057 err = 0; 8058 8059 out: 8060 kfree(desc_buf); 8061 return err; 8062 } 8063 8064 static void ufs_put_device_desc(struct ufs_hba *hba) 8065 { 8066 struct ufs_dev_info *dev_info = &hba->dev_info; 8067 8068 kfree(dev_info->model); 8069 dev_info->model = NULL; 8070 } 8071 8072 /** 8073 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro 8074 * @hba: per-adapter instance 8075 * 8076 * PA_TActivate parameter can be tuned manually if UniPro version is less than 8077 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's 8078 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce 8079 * the hibern8 exit latency. 8080 * 8081 * Returns zero on success, non-zero error value on failure. 8082 */ 8083 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba) 8084 { 8085 int ret = 0; 8086 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate; 8087 8088 ret = ufshcd_dme_peer_get(hba, 8089 UIC_ARG_MIB_SEL( 8090 RX_MIN_ACTIVATETIME_CAPABILITY, 8091 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 8092 &peer_rx_min_activatetime); 8093 if (ret) 8094 goto out; 8095 8096 /* make sure proper unit conversion is applied */ 8097 tuned_pa_tactivate = 8098 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US) 8099 / PA_TACTIVATE_TIME_UNIT_US); 8100 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 8101 tuned_pa_tactivate); 8102 8103 out: 8104 return ret; 8105 } 8106 8107 /** 8108 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro 8109 * @hba: per-adapter instance 8110 * 8111 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than 8112 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's 8113 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY. 8114 * This optimal value can help reduce the hibern8 exit latency. 8115 * 8116 * Returns zero on success, non-zero error value on failure. 8117 */ 8118 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba) 8119 { 8120 int ret = 0; 8121 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0; 8122 u32 max_hibern8_time, tuned_pa_hibern8time; 8123 8124 ret = ufshcd_dme_get(hba, 8125 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY, 8126 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)), 8127 &local_tx_hibern8_time_cap); 8128 if (ret) 8129 goto out; 8130 8131 ret = ufshcd_dme_peer_get(hba, 8132 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY, 8133 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 8134 &peer_rx_hibern8_time_cap); 8135 if (ret) 8136 goto out; 8137 8138 max_hibern8_time = max(local_tx_hibern8_time_cap, 8139 peer_rx_hibern8_time_cap); 8140 /* make sure proper unit conversion is applied */ 8141 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US) 8142 / PA_HIBERN8_TIME_UNIT_US); 8143 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), 8144 tuned_pa_hibern8time); 8145 out: 8146 return ret; 8147 } 8148 8149 /** 8150 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is 8151 * less than device PA_TACTIVATE time. 8152 * @hba: per-adapter instance 8153 * 8154 * Some UFS devices require host PA_TACTIVATE to be lower than device 8155 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk 8156 * for such devices. 8157 * 8158 * Returns zero on success, non-zero error value on failure. 8159 */ 8160 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) 8161 { 8162 int ret = 0; 8163 u32 granularity, peer_granularity; 8164 u32 pa_tactivate, peer_pa_tactivate; 8165 u32 pa_tactivate_us, peer_pa_tactivate_us; 8166 static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100}; 8167 8168 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8169 &granularity); 8170 if (ret) 8171 goto out; 8172 8173 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), 8174 &peer_granularity); 8175 if (ret) 8176 goto out; 8177 8178 if ((granularity < PA_GRANULARITY_MIN_VAL) || 8179 (granularity > PA_GRANULARITY_MAX_VAL)) { 8180 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d", 8181 __func__, granularity); 8182 return -EINVAL; 8183 } 8184 8185 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) || 8186 (peer_granularity > PA_GRANULARITY_MAX_VAL)) { 8187 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d", 8188 __func__, peer_granularity); 8189 return -EINVAL; 8190 } 8191 8192 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate); 8193 if (ret) 8194 goto out; 8195 8196 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE), 8197 &peer_pa_tactivate); 8198 if (ret) 8199 goto out; 8200 8201 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1]; 8202 peer_pa_tactivate_us = peer_pa_tactivate * 8203 gran_to_us_table[peer_granularity - 1]; 8204 8205 if (pa_tactivate_us >= peer_pa_tactivate_us) { 8206 u32 new_peer_pa_tactivate; 8207 8208 new_peer_pa_tactivate = pa_tactivate_us / 8209 gran_to_us_table[peer_granularity - 1]; 8210 new_peer_pa_tactivate++; 8211 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 8212 new_peer_pa_tactivate); 8213 } 8214 8215 out: 8216 return ret; 8217 } 8218 8219 static void ufshcd_tune_unipro_params(struct ufs_hba *hba) 8220 { 8221 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) { 8222 ufshcd_tune_pa_tactivate(hba); 8223 ufshcd_tune_pa_hibern8time(hba); 8224 } 8225 8226 ufshcd_vops_apply_dev_quirks(hba); 8227 8228 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) 8229 /* set 1ms timeout for PA_TACTIVATE */ 8230 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10); 8231 8232 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) 8233 ufshcd_quirk_tune_host_pa_tactivate(hba); 8234 } 8235 8236 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) 8237 { 8238 hba->ufs_stats.hibern8_exit_cnt = 0; 8239 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); 8240 hba->req_abort_count = 0; 8241 } 8242 8243 static int ufshcd_device_geo_params_init(struct ufs_hba *hba) 8244 { 8245 int err; 8246 u8 *desc_buf; 8247 8248 desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); 8249 if (!desc_buf) { 8250 err = -ENOMEM; 8251 goto out; 8252 } 8253 8254 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0, 8255 desc_buf, QUERY_DESC_MAX_SIZE); 8256 if (err) { 8257 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n", 8258 __func__, err); 8259 goto out; 8260 } 8261 8262 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1) 8263 hba->dev_info.max_lu_supported = 32; 8264 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) 8265 hba->dev_info.max_lu_supported = 8; 8266 8267 if (desc_buf[QUERY_DESC_LENGTH_OFFSET] >= 8268 GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS) 8269 ufshpb_get_geo_info(hba, desc_buf); 8270 8271 out: 8272 kfree(desc_buf); 8273 return err; 8274 } 8275 8276 struct ufs_ref_clk { 8277 unsigned long freq_hz; 8278 enum ufs_ref_clk_freq val; 8279 }; 8280 8281 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = { 8282 {19200000, REF_CLK_FREQ_19_2_MHZ}, 8283 {26000000, REF_CLK_FREQ_26_MHZ}, 8284 {38400000, REF_CLK_FREQ_38_4_MHZ}, 8285 {52000000, REF_CLK_FREQ_52_MHZ}, 8286 {0, REF_CLK_FREQ_INVAL}, 8287 }; 8288 8289 static enum ufs_ref_clk_freq 8290 ufs_get_bref_clk_from_hz(unsigned long freq) 8291 { 8292 int i; 8293 8294 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++) 8295 if (ufs_ref_clk_freqs[i].freq_hz == freq) 8296 return ufs_ref_clk_freqs[i].val; 8297 8298 return REF_CLK_FREQ_INVAL; 8299 } 8300 8301 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk) 8302 { 8303 unsigned long freq; 8304 8305 freq = clk_get_rate(refclk); 8306 8307 hba->dev_ref_clk_freq = 8308 ufs_get_bref_clk_from_hz(freq); 8309 8310 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 8311 dev_err(hba->dev, 8312 "invalid ref_clk setting = %ld\n", freq); 8313 } 8314 8315 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba) 8316 { 8317 int err; 8318 u32 ref_clk; 8319 u32 freq = hba->dev_ref_clk_freq; 8320 8321 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, 8322 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk); 8323 8324 if (err) { 8325 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n", 8326 err); 8327 goto out; 8328 } 8329 8330 if (ref_clk == freq) 8331 goto out; /* nothing to update */ 8332 8333 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 8334 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq); 8335 8336 if (err) { 8337 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n", 8338 ufs_ref_clk_freqs[freq].freq_hz); 8339 goto out; 8340 } 8341 8342 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n", 8343 ufs_ref_clk_freqs[freq].freq_hz); 8344 8345 out: 8346 return err; 8347 } 8348 8349 static int ufshcd_device_params_init(struct ufs_hba *hba) 8350 { 8351 bool flag; 8352 int ret; 8353 8354 /* Init UFS geometry descriptor related parameters */ 8355 ret = ufshcd_device_geo_params_init(hba); 8356 if (ret) 8357 goto out; 8358 8359 /* Check and apply UFS device quirks */ 8360 ret = ufs_get_device_desc(hba); 8361 if (ret) { 8362 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", 8363 __func__, ret); 8364 goto out; 8365 } 8366 8367 ufshcd_get_ref_clk_gating_wait(hba); 8368 8369 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, 8370 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag)) 8371 hba->dev_info.f_power_on_wp_en = flag; 8372 8373 /* Probe maximum power mode co-supported by both UFS host and device */ 8374 if (ufshcd_get_max_pwr_mode(hba)) 8375 dev_err(hba->dev, 8376 "%s: Failed getting max supported power mode\n", 8377 __func__); 8378 out: 8379 return ret; 8380 } 8381 8382 /** 8383 * ufshcd_add_lus - probe and add UFS logical units 8384 * @hba: per-adapter instance 8385 */ 8386 static int ufshcd_add_lus(struct ufs_hba *hba) 8387 { 8388 int ret; 8389 8390 /* Add required well known logical units to scsi mid layer */ 8391 ret = ufshcd_scsi_add_wlus(hba); 8392 if (ret) 8393 goto out; 8394 8395 /* Initialize devfreq after UFS device is detected */ 8396 if (ufshcd_is_clkscaling_supported(hba)) { 8397 memcpy(&hba->clk_scaling.saved_pwr_info.info, 8398 &hba->pwr_info, 8399 sizeof(struct ufs_pa_layer_attr)); 8400 hba->clk_scaling.saved_pwr_info.is_valid = true; 8401 hba->clk_scaling.is_allowed = true; 8402 8403 ret = ufshcd_devfreq_init(hba); 8404 if (ret) 8405 goto out; 8406 8407 hba->clk_scaling.is_enabled = true; 8408 ufshcd_init_clk_scaling_sysfs(hba); 8409 } 8410 8411 ufs_bsg_probe(hba); 8412 ufshpb_init(hba); 8413 scsi_scan_host(hba->host); 8414 pm_runtime_put_sync(hba->dev); 8415 8416 out: 8417 return ret; 8418 } 8419 8420 /* SDB - Single Doorbell */ 8421 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) 8422 { 8423 size_t ucdl_size, utrdl_size; 8424 8425 ucdl_size = sizeof(struct utp_transfer_cmd_desc) * nutrs; 8426 dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr, 8427 hba->ucdl_dma_addr); 8428 8429 utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; 8430 dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, 8431 hba->utrdl_dma_addr); 8432 8433 devm_kfree(hba->dev, hba->lrb); 8434 } 8435 8436 static int ufshcd_alloc_mcq(struct ufs_hba *hba) 8437 { 8438 int ret; 8439 int old_nutrs = hba->nutrs; 8440 8441 ret = ufshcd_mcq_decide_queue_depth(hba); 8442 if (ret < 0) 8443 return ret; 8444 8445 hba->nutrs = ret; 8446 ret = ufshcd_mcq_init(hba); 8447 if (ret) 8448 goto err; 8449 8450 /* 8451 * Previously allocated memory for nutrs may not be enough in MCQ mode. 8452 * Number of supported tags in MCQ mode may be larger than SDB mode. 8453 */ 8454 if (hba->nutrs != old_nutrs) { 8455 ufshcd_release_sdb_queue(hba, old_nutrs); 8456 ret = ufshcd_memory_alloc(hba); 8457 if (ret) 8458 goto err; 8459 ufshcd_host_memory_configure(hba); 8460 } 8461 8462 ret = ufshcd_mcq_memory_alloc(hba); 8463 if (ret) 8464 goto err; 8465 8466 return 0; 8467 err: 8468 hba->nutrs = old_nutrs; 8469 return ret; 8470 } 8471 8472 static void ufshcd_config_mcq(struct ufs_hba *hba) 8473 { 8474 int ret; 8475 8476 ret = ufshcd_mcq_vops_config_esi(hba); 8477 dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); 8478 8479 ufshcd_enable_intr(hba, UFSHCD_ENABLE_MCQ_INTRS); 8480 ufshcd_mcq_make_queues_operational(hba); 8481 ufshcd_mcq_config_mac(hba, hba->nutrs); 8482 8483 hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 8484 hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED; 8485 8486 /* Select MCQ mode */ 8487 ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1, 8488 REG_UFS_MEM_CFG); 8489 hba->mcq_enabled = true; 8490 8491 dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", 8492 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT], 8493 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL], 8494 hba->nutrs); 8495 } 8496 8497 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) 8498 { 8499 int ret; 8500 struct Scsi_Host *host = hba->host; 8501 8502 hba->ufshcd_state = UFSHCD_STATE_RESET; 8503 8504 ret = ufshcd_link_startup(hba); 8505 if (ret) 8506 return ret; 8507 8508 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) 8509 return ret; 8510 8511 /* Debug counters initialization */ 8512 ufshcd_clear_dbg_ufs_stats(hba); 8513 8514 /* UniPro link is active now */ 8515 ufshcd_set_link_active(hba); 8516 8517 /* Reconfigure MCQ upon reset */ 8518 if (is_mcq_enabled(hba) && !init_dev_params) 8519 ufshcd_config_mcq(hba); 8520 8521 /* Verify device initialization by sending NOP OUT UPIU */ 8522 ret = ufshcd_verify_dev_init(hba); 8523 if (ret) 8524 return ret; 8525 8526 /* Initiate UFS initialization, and waiting until completion */ 8527 ret = ufshcd_complete_dev_init(hba); 8528 if (ret) 8529 return ret; 8530 8531 /* 8532 * Initialize UFS device parameters used by driver, these 8533 * parameters are associated with UFS descriptors. 8534 */ 8535 if (init_dev_params) { 8536 ret = ufshcd_device_params_init(hba); 8537 if (ret) 8538 return ret; 8539 if (is_mcq_supported(hba) && !hba->scsi_host_added) { 8540 ret = ufshcd_alloc_mcq(hba); 8541 if (!ret) { 8542 ufshcd_config_mcq(hba); 8543 } else { 8544 /* Continue with SDB mode */ 8545 use_mcq_mode = false; 8546 dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", 8547 ret); 8548 } 8549 ret = scsi_add_host(host, hba->dev); 8550 if (ret) { 8551 dev_err(hba->dev, "scsi_add_host failed\n"); 8552 return ret; 8553 } 8554 hba->scsi_host_added = true; 8555 } else if (is_mcq_supported(hba)) { 8556 /* UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is set */ 8557 ufshcd_config_mcq(hba); 8558 } 8559 } 8560 8561 ufshcd_tune_unipro_params(hba); 8562 8563 /* UFS device is also active now */ 8564 ufshcd_set_ufs_dev_active(hba); 8565 ufshcd_force_reset_auto_bkops(hba); 8566 8567 /* Gear up to HS gear if supported */ 8568 if (hba->max_pwr_info.is_valid) { 8569 /* 8570 * Set the right value to bRefClkFreq before attempting to 8571 * switch to HS gears. 8572 */ 8573 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) 8574 ufshcd_set_dev_ref_clk(hba); 8575 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); 8576 if (ret) { 8577 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", 8578 __func__, ret); 8579 return ret; 8580 } 8581 } 8582 8583 return 0; 8584 } 8585 8586 /** 8587 * ufshcd_probe_hba - probe hba to detect device and initialize it 8588 * @hba: per-adapter instance 8589 * @init_dev_params: whether or not to call ufshcd_device_params_init(). 8590 * 8591 * Execute link-startup and verify device initialization 8592 */ 8593 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) 8594 { 8595 ktime_t start = ktime_get(); 8596 unsigned long flags; 8597 int ret; 8598 8599 ret = ufshcd_device_init(hba, init_dev_params); 8600 if (ret) 8601 goto out; 8602 8603 if (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { 8604 /* Reset the device and controller before doing reinit */ 8605 ufshcd_device_reset(hba); 8606 ufshcd_hba_stop(hba); 8607 ufshcd_vops_reinit_notify(hba); 8608 ret = ufshcd_hba_enable(hba); 8609 if (ret) { 8610 dev_err(hba->dev, "Host controller enable failed\n"); 8611 ufshcd_print_evt_hist(hba); 8612 ufshcd_print_host_state(hba); 8613 goto out; 8614 } 8615 8616 /* Reinit the device */ 8617 ret = ufshcd_device_init(hba, init_dev_params); 8618 if (ret) 8619 goto out; 8620 } 8621 8622 ufshcd_print_pwr_info(hba); 8623 8624 /* 8625 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec) 8626 * and for removable UFS card as well, hence always set the parameter. 8627 * Note: Error handler may issue the device reset hence resetting 8628 * bActiveICCLevel as well so it is always safe to set this here. 8629 */ 8630 ufshcd_set_active_icc_lvl(hba); 8631 8632 /* Enable UFS Write Booster if supported */ 8633 ufshcd_configure_wb(hba); 8634 8635 if (hba->ee_usr_mask) 8636 ufshcd_write_ee_control(hba); 8637 /* Enable Auto-Hibernate if configured */ 8638 ufshcd_auto_hibern8_enable(hba); 8639 8640 ufshpb_toggle_state(hba, HPB_RESET, HPB_PRESENT); 8641 out: 8642 spin_lock_irqsave(hba->host->host_lock, flags); 8643 if (ret) 8644 hba->ufshcd_state = UFSHCD_STATE_ERROR; 8645 else if (hba->ufshcd_state == UFSHCD_STATE_RESET) 8646 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 8647 spin_unlock_irqrestore(hba->host->host_lock, flags); 8648 8649 trace_ufshcd_init(dev_name(hba->dev), ret, 8650 ktime_to_us(ktime_sub(ktime_get(), start)), 8651 hba->curr_dev_pwr_mode, hba->uic_link_state); 8652 return ret; 8653 } 8654 8655 /** 8656 * ufshcd_async_scan - asynchronous execution for probing hba 8657 * @data: data pointer to pass to this function 8658 * @cookie: cookie data 8659 */ 8660 static void ufshcd_async_scan(void *data, async_cookie_t cookie) 8661 { 8662 struct ufs_hba *hba = (struct ufs_hba *)data; 8663 int ret; 8664 8665 down(&hba->host_sem); 8666 /* Initialize hba, detect and initialize UFS device */ 8667 ret = ufshcd_probe_hba(hba, true); 8668 up(&hba->host_sem); 8669 if (ret) 8670 goto out; 8671 8672 /* Probe and add UFS logical units */ 8673 ret = ufshcd_add_lus(hba); 8674 out: 8675 /* 8676 * If we failed to initialize the device or the device is not 8677 * present, turn off the power/clocks etc. 8678 */ 8679 if (ret) { 8680 pm_runtime_put_sync(hba->dev); 8681 ufshcd_hba_exit(hba); 8682 } 8683 } 8684 8685 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) 8686 { 8687 struct ufs_hba *hba = shost_priv(scmd->device->host); 8688 8689 if (!hba->system_suspending) { 8690 /* Activate the error handler in the SCSI core. */ 8691 return SCSI_EH_NOT_HANDLED; 8692 } 8693 8694 /* 8695 * If we get here we know that no TMFs are outstanding and also that 8696 * the only pending command is a START STOP UNIT command. Handle the 8697 * timeout of that command directly to prevent a deadlock between 8698 * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler(). 8699 */ 8700 ufshcd_link_recovery(hba); 8701 dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", 8702 __func__, hba->outstanding_tasks); 8703 8704 return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; 8705 } 8706 8707 static const struct attribute_group *ufshcd_driver_groups[] = { 8708 &ufs_sysfs_unit_descriptor_group, 8709 &ufs_sysfs_lun_attributes_group, 8710 #ifdef CONFIG_SCSI_UFS_HPB 8711 &ufs_sysfs_hpb_stat_group, 8712 &ufs_sysfs_hpb_param_group, 8713 #endif 8714 NULL, 8715 }; 8716 8717 static struct ufs_hba_variant_params ufs_hba_vps = { 8718 .hba_enable_delay_us = 1000, 8719 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40), 8720 .devfreq_profile.polling_ms = 100, 8721 .devfreq_profile.target = ufshcd_devfreq_target, 8722 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status, 8723 .ondemand_data.upthreshold = 70, 8724 .ondemand_data.downdifferential = 5, 8725 }; 8726 8727 static struct scsi_host_template ufshcd_driver_template = { 8728 .module = THIS_MODULE, 8729 .name = UFSHCD, 8730 .proc_name = UFSHCD, 8731 .map_queues = ufshcd_map_queues, 8732 .queuecommand = ufshcd_queuecommand, 8733 .mq_poll = ufshcd_poll, 8734 .slave_alloc = ufshcd_slave_alloc, 8735 .slave_configure = ufshcd_slave_configure, 8736 .slave_destroy = ufshcd_slave_destroy, 8737 .change_queue_depth = ufshcd_change_queue_depth, 8738 .eh_abort_handler = ufshcd_abort, 8739 .eh_device_reset_handler = ufshcd_eh_device_reset_handler, 8740 .eh_host_reset_handler = ufshcd_eh_host_reset_handler, 8741 .eh_timed_out = ufshcd_eh_timed_out, 8742 .this_id = -1, 8743 .sg_tablesize = SG_ALL, 8744 .cmd_per_lun = UFSHCD_CMD_PER_LUN, 8745 .can_queue = UFSHCD_CAN_QUEUE, 8746 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, 8747 .max_sectors = (1 << 20) / SECTOR_SIZE, /* 1 MiB */ 8748 .max_host_blocked = 1, 8749 .track_queue_depth = 1, 8750 .sdev_groups = ufshcd_driver_groups, 8751 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS, 8752 }; 8753 8754 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, 8755 int ua) 8756 { 8757 int ret; 8758 8759 if (!vreg) 8760 return 0; 8761 8762 /* 8763 * "set_load" operation shall be required on those regulators 8764 * which specifically configured current limitation. Otherwise 8765 * zero max_uA may cause unexpected behavior when regulator is 8766 * enabled or set as high power mode. 8767 */ 8768 if (!vreg->max_uA) 8769 return 0; 8770 8771 ret = regulator_set_load(vreg->reg, ua); 8772 if (ret < 0) { 8773 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n", 8774 __func__, vreg->name, ua, ret); 8775 } 8776 8777 return ret; 8778 } 8779 8780 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, 8781 struct ufs_vreg *vreg) 8782 { 8783 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); 8784 } 8785 8786 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 8787 struct ufs_vreg *vreg) 8788 { 8789 if (!vreg) 8790 return 0; 8791 8792 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); 8793 } 8794 8795 static int ufshcd_config_vreg(struct device *dev, 8796 struct ufs_vreg *vreg, bool on) 8797 { 8798 if (regulator_count_voltages(vreg->reg) <= 0) 8799 return 0; 8800 8801 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0); 8802 } 8803 8804 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) 8805 { 8806 int ret = 0; 8807 8808 if (!vreg || vreg->enabled) 8809 goto out; 8810 8811 ret = ufshcd_config_vreg(dev, vreg, true); 8812 if (!ret) 8813 ret = regulator_enable(vreg->reg); 8814 8815 if (!ret) 8816 vreg->enabled = true; 8817 else 8818 dev_err(dev, "%s: %s enable failed, err=%d\n", 8819 __func__, vreg->name, ret); 8820 out: 8821 return ret; 8822 } 8823 8824 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) 8825 { 8826 int ret = 0; 8827 8828 if (!vreg || !vreg->enabled || vreg->always_on) 8829 goto out; 8830 8831 ret = regulator_disable(vreg->reg); 8832 8833 if (!ret) { 8834 /* ignore errors on applying disable config */ 8835 ufshcd_config_vreg(dev, vreg, false); 8836 vreg->enabled = false; 8837 } else { 8838 dev_err(dev, "%s: %s disable failed, err=%d\n", 8839 __func__, vreg->name, ret); 8840 } 8841 out: 8842 return ret; 8843 } 8844 8845 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on) 8846 { 8847 int ret = 0; 8848 struct device *dev = hba->dev; 8849 struct ufs_vreg_info *info = &hba->vreg_info; 8850 8851 ret = ufshcd_toggle_vreg(dev, info->vcc, on); 8852 if (ret) 8853 goto out; 8854 8855 ret = ufshcd_toggle_vreg(dev, info->vccq, on); 8856 if (ret) 8857 goto out; 8858 8859 ret = ufshcd_toggle_vreg(dev, info->vccq2, on); 8860 8861 out: 8862 if (ret) { 8863 ufshcd_toggle_vreg(dev, info->vccq2, false); 8864 ufshcd_toggle_vreg(dev, info->vccq, false); 8865 ufshcd_toggle_vreg(dev, info->vcc, false); 8866 } 8867 return ret; 8868 } 8869 8870 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on) 8871 { 8872 struct ufs_vreg_info *info = &hba->vreg_info; 8873 8874 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on); 8875 } 8876 8877 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg) 8878 { 8879 int ret = 0; 8880 8881 if (!vreg) 8882 goto out; 8883 8884 vreg->reg = devm_regulator_get(dev, vreg->name); 8885 if (IS_ERR(vreg->reg)) { 8886 ret = PTR_ERR(vreg->reg); 8887 dev_err(dev, "%s: %s get failed, err=%d\n", 8888 __func__, vreg->name, ret); 8889 } 8890 out: 8891 return ret; 8892 } 8893 EXPORT_SYMBOL_GPL(ufshcd_get_vreg); 8894 8895 static int ufshcd_init_vreg(struct ufs_hba *hba) 8896 { 8897 int ret = 0; 8898 struct device *dev = hba->dev; 8899 struct ufs_vreg_info *info = &hba->vreg_info; 8900 8901 ret = ufshcd_get_vreg(dev, info->vcc); 8902 if (ret) 8903 goto out; 8904 8905 ret = ufshcd_get_vreg(dev, info->vccq); 8906 if (!ret) 8907 ret = ufshcd_get_vreg(dev, info->vccq2); 8908 out: 8909 return ret; 8910 } 8911 8912 static int ufshcd_init_hba_vreg(struct ufs_hba *hba) 8913 { 8914 struct ufs_vreg_info *info = &hba->vreg_info; 8915 8916 return ufshcd_get_vreg(hba->dev, info->vdd_hba); 8917 } 8918 8919 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) 8920 { 8921 int ret = 0; 8922 struct ufs_clk_info *clki; 8923 struct list_head *head = &hba->clk_list_head; 8924 unsigned long flags; 8925 ktime_t start = ktime_get(); 8926 bool clk_state_changed = false; 8927 8928 if (list_empty(head)) 8929 goto out; 8930 8931 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE); 8932 if (ret) 8933 return ret; 8934 8935 list_for_each_entry(clki, head, list) { 8936 if (!IS_ERR_OR_NULL(clki->clk)) { 8937 /* 8938 * Don't disable clocks which are needed 8939 * to keep the link active. 8940 */ 8941 if (ufshcd_is_link_active(hba) && 8942 clki->keep_link_active) 8943 continue; 8944 8945 clk_state_changed = on ^ clki->enabled; 8946 if (on && !clki->enabled) { 8947 ret = clk_prepare_enable(clki->clk); 8948 if (ret) { 8949 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n", 8950 __func__, clki->name, ret); 8951 goto out; 8952 } 8953 } else if (!on && clki->enabled) { 8954 clk_disable_unprepare(clki->clk); 8955 } 8956 clki->enabled = on; 8957 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__, 8958 clki->name, on ? "en" : "dis"); 8959 } 8960 } 8961 8962 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE); 8963 if (ret) 8964 return ret; 8965 8966 out: 8967 if (ret) { 8968 list_for_each_entry(clki, head, list) { 8969 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) 8970 clk_disable_unprepare(clki->clk); 8971 } 8972 } else if (!ret && on) { 8973 spin_lock_irqsave(hba->host->host_lock, flags); 8974 hba->clk_gating.state = CLKS_ON; 8975 trace_ufshcd_clk_gating(dev_name(hba->dev), 8976 hba->clk_gating.state); 8977 spin_unlock_irqrestore(hba->host->host_lock, flags); 8978 } 8979 8980 if (clk_state_changed) 8981 trace_ufshcd_profile_clk_gating(dev_name(hba->dev), 8982 (on ? "on" : "off"), 8983 ktime_to_us(ktime_sub(ktime_get(), start)), ret); 8984 return ret; 8985 } 8986 8987 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba) 8988 { 8989 u32 freq; 8990 int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq); 8991 8992 if (ret) { 8993 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret); 8994 return REF_CLK_FREQ_INVAL; 8995 } 8996 8997 return ufs_get_bref_clk_from_hz(freq); 8998 } 8999 9000 static int ufshcd_init_clocks(struct ufs_hba *hba) 9001 { 9002 int ret = 0; 9003 struct ufs_clk_info *clki; 9004 struct device *dev = hba->dev; 9005 struct list_head *head = &hba->clk_list_head; 9006 9007 if (list_empty(head)) 9008 goto out; 9009 9010 list_for_each_entry(clki, head, list) { 9011 if (!clki->name) 9012 continue; 9013 9014 clki->clk = devm_clk_get(dev, clki->name); 9015 if (IS_ERR(clki->clk)) { 9016 ret = PTR_ERR(clki->clk); 9017 dev_err(dev, "%s: %s clk get failed, %d\n", 9018 __func__, clki->name, ret); 9019 goto out; 9020 } 9021 9022 /* 9023 * Parse device ref clk freq as per device tree "ref_clk". 9024 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL 9025 * in ufshcd_alloc_host(). 9026 */ 9027 if (!strcmp(clki->name, "ref_clk")) 9028 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk); 9029 9030 if (clki->max_freq) { 9031 ret = clk_set_rate(clki->clk, clki->max_freq); 9032 if (ret) { 9033 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 9034 __func__, clki->name, 9035 clki->max_freq, ret); 9036 goto out; 9037 } 9038 clki->curr_freq = clki->max_freq; 9039 } 9040 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, 9041 clki->name, clk_get_rate(clki->clk)); 9042 } 9043 out: 9044 return ret; 9045 } 9046 9047 static int ufshcd_variant_hba_init(struct ufs_hba *hba) 9048 { 9049 int err = 0; 9050 9051 if (!hba->vops) 9052 goto out; 9053 9054 err = ufshcd_vops_init(hba); 9055 if (err) 9056 dev_err(hba->dev, "%s: variant %s init failed err %d\n", 9057 __func__, ufshcd_get_var_name(hba), err); 9058 out: 9059 return err; 9060 } 9061 9062 static void ufshcd_variant_hba_exit(struct ufs_hba *hba) 9063 { 9064 if (!hba->vops) 9065 return; 9066 9067 ufshcd_vops_exit(hba); 9068 } 9069 9070 static int ufshcd_hba_init(struct ufs_hba *hba) 9071 { 9072 int err; 9073 9074 /* 9075 * Handle host controller power separately from the UFS device power 9076 * rails as it will help controlling the UFS host controller power 9077 * collapse easily which is different than UFS device power collapse. 9078 * Also, enable the host controller power before we go ahead with rest 9079 * of the initialization here. 9080 */ 9081 err = ufshcd_init_hba_vreg(hba); 9082 if (err) 9083 goto out; 9084 9085 err = ufshcd_setup_hba_vreg(hba, true); 9086 if (err) 9087 goto out; 9088 9089 err = ufshcd_init_clocks(hba); 9090 if (err) 9091 goto out_disable_hba_vreg; 9092 9093 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL) 9094 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba); 9095 9096 err = ufshcd_setup_clocks(hba, true); 9097 if (err) 9098 goto out_disable_hba_vreg; 9099 9100 err = ufshcd_init_vreg(hba); 9101 if (err) 9102 goto out_disable_clks; 9103 9104 err = ufshcd_setup_vreg(hba, true); 9105 if (err) 9106 goto out_disable_clks; 9107 9108 err = ufshcd_variant_hba_init(hba); 9109 if (err) 9110 goto out_disable_vreg; 9111 9112 ufs_debugfs_hba_init(hba); 9113 9114 hba->is_powered = true; 9115 goto out; 9116 9117 out_disable_vreg: 9118 ufshcd_setup_vreg(hba, false); 9119 out_disable_clks: 9120 ufshcd_setup_clocks(hba, false); 9121 out_disable_hba_vreg: 9122 ufshcd_setup_hba_vreg(hba, false); 9123 out: 9124 return err; 9125 } 9126 9127 static void ufshcd_hba_exit(struct ufs_hba *hba) 9128 { 9129 if (hba->is_powered) { 9130 ufshcd_exit_clk_scaling(hba); 9131 ufshcd_exit_clk_gating(hba); 9132 if (hba->eh_wq) 9133 destroy_workqueue(hba->eh_wq); 9134 ufs_debugfs_hba_exit(hba); 9135 ufshcd_variant_hba_exit(hba); 9136 ufshcd_setup_vreg(hba, false); 9137 ufshcd_setup_clocks(hba, false); 9138 ufshcd_setup_hba_vreg(hba, false); 9139 hba->is_powered = false; 9140 ufs_put_device_desc(hba); 9141 } 9142 } 9143 9144 static int ufshcd_execute_start_stop(struct scsi_device *sdev, 9145 enum ufs_dev_pwr_mode pwr_mode, 9146 struct scsi_sense_hdr *sshdr) 9147 { 9148 const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; 9149 const struct scsi_exec_args args = { 9150 .sshdr = sshdr, 9151 .req_flags = BLK_MQ_REQ_PM, 9152 .scmd_flags = SCMD_FAIL_IF_RECOVERING, 9153 }; 9154 9155 return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL, 9156 /*bufflen=*/0, /*timeout=*/HZ, /*retries=*/0, &args); 9157 } 9158 9159 /** 9160 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device 9161 * power mode 9162 * @hba: per adapter instance 9163 * @pwr_mode: device power mode to set 9164 * 9165 * Returns 0 if requested power mode is set successfully 9166 * Returns < 0 if failed to set the requested power mode 9167 */ 9168 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, 9169 enum ufs_dev_pwr_mode pwr_mode) 9170 { 9171 struct scsi_sense_hdr sshdr; 9172 struct scsi_device *sdp; 9173 unsigned long flags; 9174 int ret, retries; 9175 9176 spin_lock_irqsave(hba->host->host_lock, flags); 9177 sdp = hba->ufs_device_wlun; 9178 if (sdp && scsi_device_online(sdp)) 9179 ret = scsi_device_get(sdp); 9180 else 9181 ret = -ENODEV; 9182 spin_unlock_irqrestore(hba->host->host_lock, flags); 9183 9184 if (ret) 9185 return ret; 9186 9187 /* 9188 * If scsi commands fail, the scsi mid-layer schedules scsi error- 9189 * handling, which would wait for host to be resumed. Since we know 9190 * we are functional while we are here, skip host resume in error 9191 * handling context. 9192 */ 9193 hba->host->eh_noresume = 1; 9194 9195 /* 9196 * Current function would be generally called from the power management 9197 * callbacks hence set the RQF_PM flag so that it doesn't resume the 9198 * already suspended childs. 9199 */ 9200 for (retries = 3; retries > 0; --retries) { 9201 ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); 9202 /* 9203 * scsi_execute() only returns a negative value if the request 9204 * queue is dying. 9205 */ 9206 if (ret <= 0) 9207 break; 9208 } 9209 if (ret) { 9210 sdev_printk(KERN_WARNING, sdp, 9211 "START_STOP failed for power mode: %d, result %x\n", 9212 pwr_mode, ret); 9213 if (ret > 0) { 9214 if (scsi_sense_valid(&sshdr)) 9215 scsi_print_sense_hdr(sdp, NULL, &sshdr); 9216 ret = -EIO; 9217 } 9218 } else { 9219 hba->curr_dev_pwr_mode = pwr_mode; 9220 } 9221 9222 scsi_device_put(sdp); 9223 hba->host->eh_noresume = 0; 9224 return ret; 9225 } 9226 9227 static int ufshcd_link_state_transition(struct ufs_hba *hba, 9228 enum uic_link_state req_link_state, 9229 bool check_for_bkops) 9230 { 9231 int ret = 0; 9232 9233 if (req_link_state == hba->uic_link_state) 9234 return 0; 9235 9236 if (req_link_state == UIC_LINK_HIBERN8_STATE) { 9237 ret = ufshcd_uic_hibern8_enter(hba); 9238 if (!ret) { 9239 ufshcd_set_link_hibern8(hba); 9240 } else { 9241 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9242 __func__, ret); 9243 goto out; 9244 } 9245 } 9246 /* 9247 * If autobkops is enabled, link can't be turned off because 9248 * turning off the link would also turn off the device, except in the 9249 * case of DeepSleep where the device is expected to remain powered. 9250 */ 9251 else if ((req_link_state == UIC_LINK_OFF_STATE) && 9252 (!check_for_bkops || !hba->auto_bkops_enabled)) { 9253 /* 9254 * Let's make sure that link is in low power mode, we are doing 9255 * this currently by putting the link in Hibern8. Otherway to 9256 * put the link in low power mode is to send the DME end point 9257 * to device and then send the DME reset command to local 9258 * unipro. But putting the link in hibern8 is much faster. 9259 * 9260 * Note also that putting the link in Hibern8 is a requirement 9261 * for entering DeepSleep. 9262 */ 9263 ret = ufshcd_uic_hibern8_enter(hba); 9264 if (ret) { 9265 dev_err(hba->dev, "%s: hibern8 enter failed %d\n", 9266 __func__, ret); 9267 goto out; 9268 } 9269 /* 9270 * Change controller state to "reset state" which 9271 * should also put the link in off/reset state 9272 */ 9273 ufshcd_hba_stop(hba); 9274 /* 9275 * TODO: Check if we need any delay to make sure that 9276 * controller is reset 9277 */ 9278 ufshcd_set_link_off(hba); 9279 } 9280 9281 out: 9282 return ret; 9283 } 9284 9285 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) 9286 { 9287 bool vcc_off = false; 9288 9289 /* 9290 * It seems some UFS devices may keep drawing more than sleep current 9291 * (atleast for 500us) from UFS rails (especially from VCCQ rail). 9292 * To avoid this situation, add 2ms delay before putting these UFS 9293 * rails in LPM mode. 9294 */ 9295 if (!ufshcd_is_link_active(hba) && 9296 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM) 9297 usleep_range(2000, 2100); 9298 9299 /* 9300 * If UFS device is either in UFS_Sleep turn off VCC rail to save some 9301 * power. 9302 * 9303 * If UFS device and link is in OFF state, all power supplies (VCC, 9304 * VCCQ, VCCQ2) can be turned off if power on write protect is not 9305 * required. If UFS link is inactive (Hibern8 or OFF state) and device 9306 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode. 9307 * 9308 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway 9309 * in low power state which would save some power. 9310 * 9311 * If Write Booster is enabled and the device needs to flush the WB 9312 * buffer OR if bkops status is urgent for WB, keep Vcc on. 9313 */ 9314 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9315 !hba->dev_info.is_lu_power_on_wp) { 9316 ufshcd_setup_vreg(hba, false); 9317 vcc_off = true; 9318 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9319 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9320 vcc_off = true; 9321 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) { 9322 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9323 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); 9324 } 9325 } 9326 9327 /* 9328 * Some UFS devices require delay after VCC power rail is turned-off. 9329 */ 9330 if (vcc_off && hba->vreg_info.vcc && 9331 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM) 9332 usleep_range(5000, 5100); 9333 } 9334 9335 #ifdef CONFIG_PM 9336 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) 9337 { 9338 int ret = 0; 9339 9340 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 9341 !hba->dev_info.is_lu_power_on_wp) { 9342 ret = ufshcd_setup_vreg(hba, true); 9343 } else if (!ufshcd_is_ufs_dev_active(hba)) { 9344 if (!ufshcd_is_link_active(hba)) { 9345 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 9346 if (ret) 9347 goto vcc_disable; 9348 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 9349 if (ret) 9350 goto vccq_lpm; 9351 } 9352 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); 9353 } 9354 goto out; 9355 9356 vccq_lpm: 9357 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 9358 vcc_disable: 9359 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 9360 out: 9361 return ret; 9362 } 9363 #endif /* CONFIG_PM */ 9364 9365 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) 9366 { 9367 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9368 ufshcd_setup_hba_vreg(hba, false); 9369 } 9370 9371 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) 9372 { 9373 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba)) 9374 ufshcd_setup_hba_vreg(hba, true); 9375 } 9376 9377 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) 9378 { 9379 int ret = 0; 9380 bool check_for_bkops; 9381 enum ufs_pm_level pm_lvl; 9382 enum ufs_dev_pwr_mode req_dev_pwr_mode; 9383 enum uic_link_state req_link_state; 9384 9385 hba->pm_op_in_progress = true; 9386 if (pm_op != UFS_SHUTDOWN_PM) { 9387 pm_lvl = pm_op == UFS_RUNTIME_PM ? 9388 hba->rpm_lvl : hba->spm_lvl; 9389 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl); 9390 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl); 9391 } else { 9392 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE; 9393 req_link_state = UIC_LINK_OFF_STATE; 9394 } 9395 9396 ufshpb_suspend(hba); 9397 9398 /* 9399 * If we can't transition into any of the low power modes 9400 * just gate the clocks. 9401 */ 9402 ufshcd_hold(hba, false); 9403 hba->clk_gating.is_suspended = true; 9404 9405 if (ufshcd_is_clkscaling_supported(hba)) 9406 ufshcd_clk_scaling_suspend(hba, true); 9407 9408 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && 9409 req_link_state == UIC_LINK_ACTIVE_STATE) { 9410 goto vops_suspend; 9411 } 9412 9413 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && 9414 (req_link_state == hba->uic_link_state)) 9415 goto enable_scaling; 9416 9417 /* UFS device & link must be active before we enter in this function */ 9418 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { 9419 ret = -EINVAL; 9420 goto enable_scaling; 9421 } 9422 9423 if (pm_op == UFS_RUNTIME_PM) { 9424 if (ufshcd_can_autobkops_during_suspend(hba)) { 9425 /* 9426 * The device is idle with no requests in the queue, 9427 * allow background operations if bkops status shows 9428 * that performance might be impacted. 9429 */ 9430 ret = ufshcd_urgent_bkops(hba); 9431 if (ret) 9432 goto enable_scaling; 9433 } else { 9434 /* make sure that auto bkops is disabled */ 9435 ufshcd_disable_auto_bkops(hba); 9436 } 9437 /* 9438 * If device needs to do BKOP or WB buffer flush during 9439 * Hibern8, keep device power mode as "active power mode" 9440 * and VCC supply. 9441 */ 9442 hba->dev_info.b_rpm_dev_flush_capable = 9443 hba->auto_bkops_enabled || 9444 (((req_link_state == UIC_LINK_HIBERN8_STATE) || 9445 ((req_link_state == UIC_LINK_ACTIVE_STATE) && 9446 ufshcd_is_auto_hibern8_enabled(hba))) && 9447 ufshcd_wb_need_flush(hba)); 9448 } 9449 9450 flush_work(&hba->eeh_work); 9451 9452 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 9453 if (ret) 9454 goto enable_scaling; 9455 9456 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) { 9457 if (pm_op != UFS_RUNTIME_PM) 9458 /* ensure that bkops is disabled */ 9459 ufshcd_disable_auto_bkops(hba); 9460 9461 if (!hba->dev_info.b_rpm_dev_flush_capable) { 9462 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); 9463 if (ret && pm_op != UFS_SHUTDOWN_PM) { 9464 /* 9465 * If return err in suspend flow, IO will hang. 9466 * Trigger error handler and break suspend for 9467 * error recovery. 9468 */ 9469 ufshcd_force_error_recovery(hba); 9470 ret = -EBUSY; 9471 } 9472 if (ret) 9473 goto enable_scaling; 9474 } 9475 } 9476 9477 /* 9478 * In the case of DeepSleep, the device is expected to remain powered 9479 * with the link off, so do not check for bkops. 9480 */ 9481 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba); 9482 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops); 9483 if (ret && pm_op != UFS_SHUTDOWN_PM) { 9484 /* 9485 * If return err in suspend flow, IO will hang. 9486 * Trigger error handler and break suspend for 9487 * error recovery. 9488 */ 9489 ufshcd_force_error_recovery(hba); 9490 ret = -EBUSY; 9491 } 9492 if (ret) 9493 goto set_dev_active; 9494 9495 vops_suspend: 9496 /* 9497 * Call vendor specific suspend callback. As these callbacks may access 9498 * vendor specific host controller register space call them before the 9499 * host clocks are ON. 9500 */ 9501 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 9502 if (ret) 9503 goto set_link_active; 9504 goto out; 9505 9506 set_link_active: 9507 /* 9508 * Device hardware reset is required to exit DeepSleep. Also, for 9509 * DeepSleep, the link is off so host reset and restore will be done 9510 * further below. 9511 */ 9512 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 9513 ufshcd_device_reset(hba); 9514 WARN_ON(!ufshcd_is_link_off(hba)); 9515 } 9516 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) 9517 ufshcd_set_link_active(hba); 9518 else if (ufshcd_is_link_off(hba)) 9519 ufshcd_host_reset_and_restore(hba); 9520 set_dev_active: 9521 /* Can also get here needing to exit DeepSleep */ 9522 if (ufshcd_is_ufs_dev_deepsleep(hba)) { 9523 ufshcd_device_reset(hba); 9524 ufshcd_host_reset_and_restore(hba); 9525 } 9526 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) 9527 ufshcd_disable_auto_bkops(hba); 9528 enable_scaling: 9529 if (ufshcd_is_clkscaling_supported(hba)) 9530 ufshcd_clk_scaling_suspend(hba, false); 9531 9532 hba->dev_info.b_rpm_dev_flush_capable = false; 9533 out: 9534 if (hba->dev_info.b_rpm_dev_flush_capable) { 9535 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work, 9536 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS)); 9537 } 9538 9539 if (ret) { 9540 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); 9541 hba->clk_gating.is_suspended = false; 9542 ufshcd_release(hba); 9543 ufshpb_resume(hba); 9544 } 9545 hba->pm_op_in_progress = false; 9546 return ret; 9547 } 9548 9549 #ifdef CONFIG_PM 9550 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 9551 { 9552 int ret; 9553 enum uic_link_state old_link_state = hba->uic_link_state; 9554 9555 hba->pm_op_in_progress = true; 9556 9557 /* 9558 * Call vendor specific resume callback. As these callbacks may access 9559 * vendor specific host controller register space call them when the 9560 * host clocks are ON. 9561 */ 9562 ret = ufshcd_vops_resume(hba, pm_op); 9563 if (ret) 9564 goto out; 9565 9566 /* For DeepSleep, the only supported option is to have the link off */ 9567 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba)); 9568 9569 if (ufshcd_is_link_hibern8(hba)) { 9570 ret = ufshcd_uic_hibern8_exit(hba); 9571 if (!ret) { 9572 ufshcd_set_link_active(hba); 9573 } else { 9574 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 9575 __func__, ret); 9576 goto vendor_suspend; 9577 } 9578 } else if (ufshcd_is_link_off(hba)) { 9579 /* 9580 * A full initialization of the host and the device is 9581 * required since the link was put to off during suspend. 9582 * Note, in the case of DeepSleep, the device will exit 9583 * DeepSleep due to device reset. 9584 */ 9585 ret = ufshcd_reset_and_restore(hba); 9586 /* 9587 * ufshcd_reset_and_restore() should have already 9588 * set the link state as active 9589 */ 9590 if (ret || !ufshcd_is_link_active(hba)) 9591 goto vendor_suspend; 9592 } 9593 9594 if (!ufshcd_is_ufs_dev_active(hba)) { 9595 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); 9596 if (ret) 9597 goto set_old_link_state; 9598 } 9599 9600 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) 9601 ufshcd_enable_auto_bkops(hba); 9602 else 9603 /* 9604 * If BKOPs operations are urgently needed at this moment then 9605 * keep auto-bkops enabled or else disable it. 9606 */ 9607 ufshcd_urgent_bkops(hba); 9608 9609 if (hba->ee_usr_mask) 9610 ufshcd_write_ee_control(hba); 9611 9612 if (ufshcd_is_clkscaling_supported(hba)) 9613 ufshcd_clk_scaling_suspend(hba, false); 9614 9615 if (hba->dev_info.b_rpm_dev_flush_capable) { 9616 hba->dev_info.b_rpm_dev_flush_capable = false; 9617 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); 9618 } 9619 9620 /* Enable Auto-Hibernate if configured */ 9621 ufshcd_auto_hibern8_enable(hba); 9622 9623 ufshpb_resume(hba); 9624 goto out; 9625 9626 set_old_link_state: 9627 ufshcd_link_state_transition(hba, old_link_state, 0); 9628 vendor_suspend: 9629 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE); 9630 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); 9631 out: 9632 if (ret) 9633 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret); 9634 hba->clk_gating.is_suspended = false; 9635 ufshcd_release(hba); 9636 hba->pm_op_in_progress = false; 9637 return ret; 9638 } 9639 9640 static int ufshcd_wl_runtime_suspend(struct device *dev) 9641 { 9642 struct scsi_device *sdev = to_scsi_device(dev); 9643 struct ufs_hba *hba; 9644 int ret; 9645 ktime_t start = ktime_get(); 9646 9647 hba = shost_priv(sdev->host); 9648 9649 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM); 9650 if (ret) 9651 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 9652 9653 trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret, 9654 ktime_to_us(ktime_sub(ktime_get(), start)), 9655 hba->curr_dev_pwr_mode, hba->uic_link_state); 9656 9657 return ret; 9658 } 9659 9660 static int ufshcd_wl_runtime_resume(struct device *dev) 9661 { 9662 struct scsi_device *sdev = to_scsi_device(dev); 9663 struct ufs_hba *hba; 9664 int ret = 0; 9665 ktime_t start = ktime_get(); 9666 9667 hba = shost_priv(sdev->host); 9668 9669 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM); 9670 if (ret) 9671 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 9672 9673 trace_ufshcd_wl_runtime_resume(dev_name(dev), ret, 9674 ktime_to_us(ktime_sub(ktime_get(), start)), 9675 hba->curr_dev_pwr_mode, hba->uic_link_state); 9676 9677 return ret; 9678 } 9679 #endif 9680 9681 #ifdef CONFIG_PM_SLEEP 9682 static int ufshcd_wl_suspend(struct device *dev) 9683 { 9684 struct scsi_device *sdev = to_scsi_device(dev); 9685 struct ufs_hba *hba; 9686 int ret = 0; 9687 ktime_t start = ktime_get(); 9688 9689 hba = shost_priv(sdev->host); 9690 down(&hba->host_sem); 9691 hba->system_suspending = true; 9692 9693 if (pm_runtime_suspended(dev)) 9694 goto out; 9695 9696 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM); 9697 if (ret) { 9698 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 9699 up(&hba->host_sem); 9700 } 9701 9702 out: 9703 if (!ret) 9704 hba->is_sys_suspended = true; 9705 trace_ufshcd_wl_suspend(dev_name(dev), ret, 9706 ktime_to_us(ktime_sub(ktime_get(), start)), 9707 hba->curr_dev_pwr_mode, hba->uic_link_state); 9708 9709 return ret; 9710 } 9711 9712 static int ufshcd_wl_resume(struct device *dev) 9713 { 9714 struct scsi_device *sdev = to_scsi_device(dev); 9715 struct ufs_hba *hba; 9716 int ret = 0; 9717 ktime_t start = ktime_get(); 9718 9719 hba = shost_priv(sdev->host); 9720 9721 if (pm_runtime_suspended(dev)) 9722 goto out; 9723 9724 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM); 9725 if (ret) 9726 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); 9727 out: 9728 trace_ufshcd_wl_resume(dev_name(dev), ret, 9729 ktime_to_us(ktime_sub(ktime_get(), start)), 9730 hba->curr_dev_pwr_mode, hba->uic_link_state); 9731 if (!ret) 9732 hba->is_sys_suspended = false; 9733 hba->system_suspending = false; 9734 up(&hba->host_sem); 9735 return ret; 9736 } 9737 #endif 9738 9739 static void ufshcd_wl_shutdown(struct device *dev) 9740 { 9741 struct scsi_device *sdev = to_scsi_device(dev); 9742 struct ufs_hba *hba; 9743 9744 hba = shost_priv(sdev->host); 9745 9746 down(&hba->host_sem); 9747 hba->shutting_down = true; 9748 up(&hba->host_sem); 9749 9750 /* Turn on everything while shutting down */ 9751 ufshcd_rpm_get_sync(hba); 9752 scsi_device_quiesce(sdev); 9753 shost_for_each_device(sdev, hba->host) { 9754 if (sdev == hba->ufs_device_wlun) 9755 continue; 9756 scsi_device_quiesce(sdev); 9757 } 9758 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 9759 } 9760 9761 /** 9762 * ufshcd_suspend - helper function for suspend operations 9763 * @hba: per adapter instance 9764 * 9765 * This function will put disable irqs, turn off clocks 9766 * and set vreg and hba-vreg in lpm mode. 9767 */ 9768 static int ufshcd_suspend(struct ufs_hba *hba) 9769 { 9770 int ret; 9771 9772 if (!hba->is_powered) 9773 return 0; 9774 /* 9775 * Disable the host irq as host controller as there won't be any 9776 * host controller transaction expected till resume. 9777 */ 9778 ufshcd_disable_irq(hba); 9779 ret = ufshcd_setup_clocks(hba, false); 9780 if (ret) { 9781 ufshcd_enable_irq(hba); 9782 return ret; 9783 } 9784 if (ufshcd_is_clkgating_allowed(hba)) { 9785 hba->clk_gating.state = CLKS_OFF; 9786 trace_ufshcd_clk_gating(dev_name(hba->dev), 9787 hba->clk_gating.state); 9788 } 9789 9790 ufshcd_vreg_set_lpm(hba); 9791 /* Put the host controller in low power mode if possible */ 9792 ufshcd_hba_vreg_set_lpm(hba); 9793 return ret; 9794 } 9795 9796 #ifdef CONFIG_PM 9797 /** 9798 * ufshcd_resume - helper function for resume operations 9799 * @hba: per adapter instance 9800 * 9801 * This function basically turns on the regulators, clocks and 9802 * irqs of the hba. 9803 * 9804 * Returns 0 for success and non-zero for failure 9805 */ 9806 static int ufshcd_resume(struct ufs_hba *hba) 9807 { 9808 int ret; 9809 9810 if (!hba->is_powered) 9811 return 0; 9812 9813 ufshcd_hba_vreg_set_hpm(hba); 9814 ret = ufshcd_vreg_set_hpm(hba); 9815 if (ret) 9816 goto out; 9817 9818 /* Make sure clocks are enabled before accessing controller */ 9819 ret = ufshcd_setup_clocks(hba, true); 9820 if (ret) 9821 goto disable_vreg; 9822 9823 /* enable the host irq as host controller would be active soon */ 9824 ufshcd_enable_irq(hba); 9825 9826 goto out; 9827 9828 disable_vreg: 9829 ufshcd_vreg_set_lpm(hba); 9830 out: 9831 if (ret) 9832 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret); 9833 return ret; 9834 } 9835 #endif /* CONFIG_PM */ 9836 9837 #ifdef CONFIG_PM_SLEEP 9838 /** 9839 * ufshcd_system_suspend - system suspend callback 9840 * @dev: Device associated with the UFS controller. 9841 * 9842 * Executed before putting the system into a sleep state in which the contents 9843 * of main memory are preserved. 9844 * 9845 * Returns 0 for success and non-zero for failure 9846 */ 9847 int ufshcd_system_suspend(struct device *dev) 9848 { 9849 struct ufs_hba *hba = dev_get_drvdata(dev); 9850 int ret = 0; 9851 ktime_t start = ktime_get(); 9852 9853 if (pm_runtime_suspended(hba->dev)) 9854 goto out; 9855 9856 ret = ufshcd_suspend(hba); 9857 out: 9858 trace_ufshcd_system_suspend(dev_name(hba->dev), ret, 9859 ktime_to_us(ktime_sub(ktime_get(), start)), 9860 hba->curr_dev_pwr_mode, hba->uic_link_state); 9861 return ret; 9862 } 9863 EXPORT_SYMBOL(ufshcd_system_suspend); 9864 9865 /** 9866 * ufshcd_system_resume - system resume callback 9867 * @dev: Device associated with the UFS controller. 9868 * 9869 * Executed after waking the system up from a sleep state in which the contents 9870 * of main memory were preserved. 9871 * 9872 * Returns 0 for success and non-zero for failure 9873 */ 9874 int ufshcd_system_resume(struct device *dev) 9875 { 9876 struct ufs_hba *hba = dev_get_drvdata(dev); 9877 ktime_t start = ktime_get(); 9878 int ret = 0; 9879 9880 if (pm_runtime_suspended(hba->dev)) 9881 goto out; 9882 9883 ret = ufshcd_resume(hba); 9884 9885 out: 9886 trace_ufshcd_system_resume(dev_name(hba->dev), ret, 9887 ktime_to_us(ktime_sub(ktime_get(), start)), 9888 hba->curr_dev_pwr_mode, hba->uic_link_state); 9889 9890 return ret; 9891 } 9892 EXPORT_SYMBOL(ufshcd_system_resume); 9893 #endif /* CONFIG_PM_SLEEP */ 9894 9895 #ifdef CONFIG_PM 9896 /** 9897 * ufshcd_runtime_suspend - runtime suspend callback 9898 * @dev: Device associated with the UFS controller. 9899 * 9900 * Check the description of ufshcd_suspend() function for more details. 9901 * 9902 * Returns 0 for success and non-zero for failure 9903 */ 9904 int ufshcd_runtime_suspend(struct device *dev) 9905 { 9906 struct ufs_hba *hba = dev_get_drvdata(dev); 9907 int ret; 9908 ktime_t start = ktime_get(); 9909 9910 ret = ufshcd_suspend(hba); 9911 9912 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret, 9913 ktime_to_us(ktime_sub(ktime_get(), start)), 9914 hba->curr_dev_pwr_mode, hba->uic_link_state); 9915 return ret; 9916 } 9917 EXPORT_SYMBOL(ufshcd_runtime_suspend); 9918 9919 /** 9920 * ufshcd_runtime_resume - runtime resume routine 9921 * @dev: Device associated with the UFS controller. 9922 * 9923 * This function basically brings controller 9924 * to active state. Following operations are done in this function: 9925 * 9926 * 1. Turn on all the controller related clocks 9927 * 2. Turn ON VCC rail 9928 */ 9929 int ufshcd_runtime_resume(struct device *dev) 9930 { 9931 struct ufs_hba *hba = dev_get_drvdata(dev); 9932 int ret; 9933 ktime_t start = ktime_get(); 9934 9935 ret = ufshcd_resume(hba); 9936 9937 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret, 9938 ktime_to_us(ktime_sub(ktime_get(), start)), 9939 hba->curr_dev_pwr_mode, hba->uic_link_state); 9940 return ret; 9941 } 9942 EXPORT_SYMBOL(ufshcd_runtime_resume); 9943 #endif /* CONFIG_PM */ 9944 9945 /** 9946 * ufshcd_shutdown - shutdown routine 9947 * @hba: per adapter instance 9948 * 9949 * This function would turn off both UFS device and UFS hba 9950 * regulators. It would also disable clocks. 9951 * 9952 * Returns 0 always to allow force shutdown even in case of errors. 9953 */ 9954 int ufshcd_shutdown(struct ufs_hba *hba) 9955 { 9956 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) 9957 ufshcd_suspend(hba); 9958 9959 hba->is_powered = false; 9960 /* allow force shutdown even in case of errors */ 9961 return 0; 9962 } 9963 EXPORT_SYMBOL(ufshcd_shutdown); 9964 9965 /** 9966 * ufshcd_remove - de-allocate SCSI host and host memory space 9967 * data structure memory 9968 * @hba: per adapter instance 9969 */ 9970 void ufshcd_remove(struct ufs_hba *hba) 9971 { 9972 if (hba->ufs_device_wlun) 9973 ufshcd_rpm_get_sync(hba); 9974 ufs_hwmon_remove(hba); 9975 ufs_bsg_remove(hba); 9976 ufshpb_remove(hba); 9977 ufs_sysfs_remove_nodes(hba->dev); 9978 blk_mq_destroy_queue(hba->tmf_queue); 9979 blk_put_queue(hba->tmf_queue); 9980 blk_mq_free_tag_set(&hba->tmf_tag_set); 9981 scsi_remove_host(hba->host); 9982 /* disable interrupts */ 9983 ufshcd_disable_intr(hba, hba->intr_mask); 9984 ufshcd_hba_stop(hba); 9985 ufshcd_hba_exit(hba); 9986 } 9987 EXPORT_SYMBOL_GPL(ufshcd_remove); 9988 9989 #ifdef CONFIG_PM_SLEEP 9990 int ufshcd_system_freeze(struct device *dev) 9991 { 9992 9993 return ufshcd_system_suspend(dev); 9994 9995 } 9996 EXPORT_SYMBOL_GPL(ufshcd_system_freeze); 9997 9998 int ufshcd_system_restore(struct device *dev) 9999 { 10000 10001 struct ufs_hba *hba = dev_get_drvdata(dev); 10002 int ret; 10003 10004 ret = ufshcd_system_resume(dev); 10005 if (ret) 10006 return ret; 10007 10008 /* Configure UTRL and UTMRL base address registers */ 10009 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 10010 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 10011 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 10012 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 10013 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 10014 REG_UTP_TASK_REQ_LIST_BASE_L); 10015 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 10016 REG_UTP_TASK_REQ_LIST_BASE_H); 10017 /* 10018 * Make sure that UTRL and UTMRL base address registers 10019 * are updated with the latest queue addresses. Only after 10020 * updating these addresses, we can queue the new commands. 10021 */ 10022 mb(); 10023 10024 /* Resuming from hibernate, assume that link was OFF */ 10025 ufshcd_set_link_off(hba); 10026 10027 return 0; 10028 10029 } 10030 EXPORT_SYMBOL_GPL(ufshcd_system_restore); 10031 10032 int ufshcd_system_thaw(struct device *dev) 10033 { 10034 return ufshcd_system_resume(dev); 10035 } 10036 EXPORT_SYMBOL_GPL(ufshcd_system_thaw); 10037 #endif /* CONFIG_PM_SLEEP */ 10038 10039 /** 10040 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA) 10041 * @hba: pointer to Host Bus Adapter (HBA) 10042 */ 10043 void ufshcd_dealloc_host(struct ufs_hba *hba) 10044 { 10045 scsi_host_put(hba->host); 10046 } 10047 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host); 10048 10049 /** 10050 * ufshcd_set_dma_mask - Set dma mask based on the controller 10051 * addressing capability 10052 * @hba: per adapter instance 10053 * 10054 * Returns 0 for success, non-zero for failure 10055 */ 10056 static int ufshcd_set_dma_mask(struct ufs_hba *hba) 10057 { 10058 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { 10059 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) 10060 return 0; 10061 } 10062 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); 10063 } 10064 10065 /** 10066 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) 10067 * @dev: pointer to device handle 10068 * @hba_handle: driver private handle 10069 * Returns 0 on success, non-zero value on failure 10070 */ 10071 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) 10072 { 10073 struct Scsi_Host *host; 10074 struct ufs_hba *hba; 10075 int err = 0; 10076 10077 if (!dev) { 10078 dev_err(dev, 10079 "Invalid memory reference for dev is NULL\n"); 10080 err = -ENODEV; 10081 goto out_error; 10082 } 10083 10084 host = scsi_host_alloc(&ufshcd_driver_template, 10085 sizeof(struct ufs_hba)); 10086 if (!host) { 10087 dev_err(dev, "scsi_host_alloc failed\n"); 10088 err = -ENOMEM; 10089 goto out_error; 10090 } 10091 host->nr_maps = HCTX_TYPE_POLL + 1; 10092 hba = shost_priv(host); 10093 hba->host = host; 10094 hba->dev = dev; 10095 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL; 10096 hba->nop_out_timeout = NOP_OUT_TIMEOUT; 10097 ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry)); 10098 INIT_LIST_HEAD(&hba->clk_list_head); 10099 spin_lock_init(&hba->outstanding_lock); 10100 10101 *hba_handle = hba; 10102 10103 out_error: 10104 return err; 10105 } 10106 EXPORT_SYMBOL(ufshcd_alloc_host); 10107 10108 /* This function exists because blk_mq_alloc_tag_set() requires this. */ 10109 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx, 10110 const struct blk_mq_queue_data *qd) 10111 { 10112 WARN_ON_ONCE(true); 10113 return BLK_STS_NOTSUPP; 10114 } 10115 10116 static const struct blk_mq_ops ufshcd_tmf_ops = { 10117 .queue_rq = ufshcd_queue_tmf, 10118 }; 10119 10120 /** 10121 * ufshcd_init - Driver initialization routine 10122 * @hba: per-adapter instance 10123 * @mmio_base: base register address 10124 * @irq: Interrupt line of device 10125 * Returns 0 on success, non-zero value on failure 10126 */ 10127 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) 10128 { 10129 int err; 10130 struct Scsi_Host *host = hba->host; 10131 struct device *dev = hba->dev; 10132 char eh_wq_name[sizeof("ufs_eh_wq_00")]; 10133 10134 /* 10135 * dev_set_drvdata() must be called before any callbacks are registered 10136 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon, 10137 * sysfs). 10138 */ 10139 dev_set_drvdata(dev, hba); 10140 10141 if (!mmio_base) { 10142 dev_err(hba->dev, 10143 "Invalid memory reference for mmio_base is NULL\n"); 10144 err = -ENODEV; 10145 goto out_error; 10146 } 10147 10148 hba->mmio_base = mmio_base; 10149 hba->irq = irq; 10150 hba->vps = &ufs_hba_vps; 10151 10152 err = ufshcd_hba_init(hba); 10153 if (err) 10154 goto out_error; 10155 10156 /* Read capabilities registers */ 10157 err = ufshcd_hba_capabilities(hba); 10158 if (err) 10159 goto out_disable; 10160 10161 /* Get UFS version supported by the controller */ 10162 hba->ufs_version = ufshcd_get_ufs_version(hba); 10163 10164 /* Get Interrupt bit mask per version */ 10165 hba->intr_mask = ufshcd_get_intr_mask(hba); 10166 10167 err = ufshcd_set_dma_mask(hba); 10168 if (err) { 10169 dev_err(hba->dev, "set dma mask failed\n"); 10170 goto out_disable; 10171 } 10172 10173 /* Allocate memory for host memory space */ 10174 err = ufshcd_memory_alloc(hba); 10175 if (err) { 10176 dev_err(hba->dev, "Memory allocation failed\n"); 10177 goto out_disable; 10178 } 10179 10180 /* Configure LRB */ 10181 ufshcd_host_memory_configure(hba); 10182 10183 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; 10184 host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED; 10185 host->max_id = UFSHCD_MAX_ID; 10186 host->max_lun = UFS_MAX_LUNS; 10187 host->max_channel = UFSHCD_MAX_CHANNEL; 10188 host->unique_id = host->host_no; 10189 host->max_cmd_len = UFS_CDB_SIZE; 10190 10191 hba->max_pwr_info.is_valid = false; 10192 10193 /* Initialize work queues */ 10194 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d", 10195 hba->host->host_no); 10196 hba->eh_wq = create_singlethread_workqueue(eh_wq_name); 10197 if (!hba->eh_wq) { 10198 dev_err(hba->dev, "%s: failed to create eh workqueue\n", 10199 __func__); 10200 err = -ENOMEM; 10201 goto out_disable; 10202 } 10203 INIT_WORK(&hba->eh_work, ufshcd_err_handler); 10204 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); 10205 10206 sema_init(&hba->host_sem, 1); 10207 10208 /* Initialize UIC command mutex */ 10209 mutex_init(&hba->uic_cmd_mutex); 10210 10211 /* Initialize mutex for device management commands */ 10212 mutex_init(&hba->dev_cmd.lock); 10213 10214 /* Initialize mutex for exception event control */ 10215 mutex_init(&hba->ee_ctrl_mutex); 10216 10217 mutex_init(&hba->wb_mutex); 10218 init_rwsem(&hba->clk_scaling_lock); 10219 10220 ufshcd_init_clk_gating(hba); 10221 10222 ufshcd_init_clk_scaling(hba); 10223 10224 /* 10225 * In order to avoid any spurious interrupt immediately after 10226 * registering UFS controller interrupt handler, clear any pending UFS 10227 * interrupt status and disable all the UFS interrupts. 10228 */ 10229 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS), 10230 REG_INTERRUPT_STATUS); 10231 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE); 10232 /* 10233 * Make sure that UFS interrupts are disabled and any pending interrupt 10234 * status is cleared before registering UFS interrupt handler. 10235 */ 10236 mb(); 10237 10238 /* IRQ registration */ 10239 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); 10240 if (err) { 10241 dev_err(hba->dev, "request irq failed\n"); 10242 goto out_disable; 10243 } else { 10244 hba->is_irq_enabled = true; 10245 } 10246 10247 if (!is_mcq_supported(hba)) { 10248 err = scsi_add_host(host, hba->dev); 10249 if (err) { 10250 dev_err(hba->dev, "scsi_add_host failed\n"); 10251 goto out_disable; 10252 } 10253 } 10254 10255 hba->tmf_tag_set = (struct blk_mq_tag_set) { 10256 .nr_hw_queues = 1, 10257 .queue_depth = hba->nutmrs, 10258 .ops = &ufshcd_tmf_ops, 10259 .flags = BLK_MQ_F_NO_SCHED, 10260 }; 10261 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); 10262 if (err < 0) 10263 goto out_remove_scsi_host; 10264 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set); 10265 if (IS_ERR(hba->tmf_queue)) { 10266 err = PTR_ERR(hba->tmf_queue); 10267 goto free_tmf_tag_set; 10268 } 10269 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, 10270 sizeof(*hba->tmf_rqs), GFP_KERNEL); 10271 if (!hba->tmf_rqs) { 10272 err = -ENOMEM; 10273 goto free_tmf_queue; 10274 } 10275 10276 /* Reset the attached device */ 10277 ufshcd_device_reset(hba); 10278 10279 ufshcd_init_crypto(hba); 10280 10281 /* Host controller enable */ 10282 err = ufshcd_hba_enable(hba); 10283 if (err) { 10284 dev_err(hba->dev, "Host controller enable failed\n"); 10285 ufshcd_print_evt_hist(hba); 10286 ufshcd_print_host_state(hba); 10287 goto free_tmf_queue; 10288 } 10289 10290 /* 10291 * Set the default power management level for runtime and system PM. 10292 * Default power saving mode is to keep UFS link in Hibern8 state 10293 * and UFS device in sleep state. 10294 */ 10295 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10296 UFS_SLEEP_PWR_MODE, 10297 UIC_LINK_HIBERN8_STATE); 10298 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( 10299 UFS_SLEEP_PWR_MODE, 10300 UIC_LINK_HIBERN8_STATE); 10301 10302 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, 10303 ufshcd_rpm_dev_flush_recheck_work); 10304 10305 /* Set the default auto-hiberate idle timer value to 150 ms */ 10306 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) { 10307 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) | 10308 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); 10309 } 10310 10311 /* Hold auto suspend until async scan completes */ 10312 pm_runtime_get_sync(dev); 10313 atomic_set(&hba->scsi_block_reqs_cnt, 0); 10314 /* 10315 * We are assuming that device wasn't put in sleep/power-down 10316 * state exclusively during the boot stage before kernel. 10317 * This assumption helps avoid doing link startup twice during 10318 * ufshcd_probe_hba(). 10319 */ 10320 ufshcd_set_ufs_dev_active(hba); 10321 10322 async_schedule(ufshcd_async_scan, hba); 10323 ufs_sysfs_add_nodes(hba->dev); 10324 10325 device_enable_async_suspend(dev); 10326 return 0; 10327 10328 free_tmf_queue: 10329 blk_mq_destroy_queue(hba->tmf_queue); 10330 blk_put_queue(hba->tmf_queue); 10331 free_tmf_tag_set: 10332 blk_mq_free_tag_set(&hba->tmf_tag_set); 10333 out_remove_scsi_host: 10334 scsi_remove_host(hba->host); 10335 out_disable: 10336 hba->is_irq_enabled = false; 10337 ufshcd_hba_exit(hba); 10338 out_error: 10339 return err; 10340 } 10341 EXPORT_SYMBOL_GPL(ufshcd_init); 10342 10343 void ufshcd_resume_complete(struct device *dev) 10344 { 10345 struct ufs_hba *hba = dev_get_drvdata(dev); 10346 10347 if (hba->complete_put) { 10348 ufshcd_rpm_put(hba); 10349 hba->complete_put = false; 10350 } 10351 } 10352 EXPORT_SYMBOL_GPL(ufshcd_resume_complete); 10353 10354 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba) 10355 { 10356 struct device *dev = &hba->ufs_device_wlun->sdev_gendev; 10357 enum ufs_dev_pwr_mode dev_pwr_mode; 10358 enum uic_link_state link_state; 10359 unsigned long flags; 10360 bool res; 10361 10362 spin_lock_irqsave(&dev->power.lock, flags); 10363 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl); 10364 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl); 10365 res = pm_runtime_suspended(dev) && 10366 hba->curr_dev_pwr_mode == dev_pwr_mode && 10367 hba->uic_link_state == link_state && 10368 !hba->dev_info.b_rpm_dev_flush_capable; 10369 spin_unlock_irqrestore(&dev->power.lock, flags); 10370 10371 return res; 10372 } 10373 10374 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm) 10375 { 10376 struct ufs_hba *hba = dev_get_drvdata(dev); 10377 int ret; 10378 10379 /* 10380 * SCSI assumes that runtime-pm and system-pm for scsi drivers 10381 * are same. And it doesn't wake up the device for system-suspend 10382 * if it's runtime suspended. But ufs doesn't follow that. 10383 * Refer ufshcd_resume_complete() 10384 */ 10385 if (hba->ufs_device_wlun) { 10386 /* Prevent runtime suspend */ 10387 ufshcd_rpm_get_noresume(hba); 10388 /* 10389 * Check if already runtime suspended in same state as system 10390 * suspend would be. 10391 */ 10392 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) { 10393 /* RPM state is not ok for SPM, so runtime resume */ 10394 ret = ufshcd_rpm_resume(hba); 10395 if (ret < 0 && ret != -EACCES) { 10396 ufshcd_rpm_put(hba); 10397 return ret; 10398 } 10399 } 10400 hba->complete_put = true; 10401 } 10402 return 0; 10403 } 10404 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare); 10405 10406 int ufshcd_suspend_prepare(struct device *dev) 10407 { 10408 return __ufshcd_suspend_prepare(dev, true); 10409 } 10410 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare); 10411 10412 #ifdef CONFIG_PM_SLEEP 10413 static int ufshcd_wl_poweroff(struct device *dev) 10414 { 10415 struct scsi_device *sdev = to_scsi_device(dev); 10416 struct ufs_hba *hba = shost_priv(sdev->host); 10417 10418 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); 10419 return 0; 10420 } 10421 #endif 10422 10423 static int ufshcd_wl_probe(struct device *dev) 10424 { 10425 struct scsi_device *sdev = to_scsi_device(dev); 10426 10427 if (!is_device_wlun(sdev)) 10428 return -ENODEV; 10429 10430 blk_pm_runtime_init(sdev->request_queue, dev); 10431 pm_runtime_set_autosuspend_delay(dev, 0); 10432 pm_runtime_allow(dev); 10433 10434 return 0; 10435 } 10436 10437 static int ufshcd_wl_remove(struct device *dev) 10438 { 10439 pm_runtime_forbid(dev); 10440 return 0; 10441 } 10442 10443 static const struct dev_pm_ops ufshcd_wl_pm_ops = { 10444 #ifdef CONFIG_PM_SLEEP 10445 .suspend = ufshcd_wl_suspend, 10446 .resume = ufshcd_wl_resume, 10447 .freeze = ufshcd_wl_suspend, 10448 .thaw = ufshcd_wl_resume, 10449 .poweroff = ufshcd_wl_poweroff, 10450 .restore = ufshcd_wl_resume, 10451 #endif 10452 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) 10453 }; 10454 10455 /* 10456 * ufs_dev_wlun_template - describes ufs device wlun 10457 * ufs-device wlun - used to send pm commands 10458 * All luns are consumers of ufs-device wlun. 10459 * 10460 * Currently, no sd driver is present for wluns. 10461 * Hence the no specific pm operations are performed. 10462 * With ufs design, SSU should be sent to ufs-device wlun. 10463 * Hence register a scsi driver for ufs wluns only. 10464 */ 10465 static struct scsi_driver ufs_dev_wlun_template = { 10466 .gendrv = { 10467 .name = "ufs_device_wlun", 10468 .owner = THIS_MODULE, 10469 .probe = ufshcd_wl_probe, 10470 .remove = ufshcd_wl_remove, 10471 .pm = &ufshcd_wl_pm_ops, 10472 .shutdown = ufshcd_wl_shutdown, 10473 }, 10474 }; 10475 10476 static int __init ufshcd_core_init(void) 10477 { 10478 int ret; 10479 10480 ufs_debugfs_init(); 10481 10482 ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv); 10483 if (ret) 10484 ufs_debugfs_exit(); 10485 return ret; 10486 } 10487 10488 static void __exit ufshcd_core_exit(void) 10489 { 10490 ufs_debugfs_exit(); 10491 scsi_unregister_driver(&ufs_dev_wlun_template.gendrv); 10492 } 10493 10494 module_init(ufshcd_core_init); 10495 module_exit(ufshcd_core_exit); 10496 10497 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); 10498 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); 10499 MODULE_DESCRIPTION("Generic UFS host controller driver Core"); 10500 MODULE_SOFTDEP("pre: governor_simpleondemand"); 10501 MODULE_LICENSE("GPL"); 10502