Lines Matching +full:cpu +full:- +full:ufs

1 // SPDX-License-Identifier: GPL-2.0-or-later
4  * Copyright (C) 2011-2013 Samsung India Software Operations
5  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
17 #include <linux/blk-pm.h>
30 #include "ufshcd-priv.h"
31 #include <ufs/ufs_quirks.h>
32 #include <ufs/unipro.h>
33 #include "ufs-sysfs.h"
34 #include "ufs-debugfs.h"
35 #include "ufs-fault-injection.h"
37 #include "ufshcd-crypto.h"
41 #include <trace/events/ufs.h>
74 /* maximum number of link-startup retries */
106 	return hba->mcq_sup && use_mcq_mode;
110 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default");
136 		return -EINVAL;
140 		return -ENOMEM;
160 	UFSHCD_CMD_PER_LUN	= 32 - UFSHCD_NUM_RESERVED,
161 	UFSHCD_CAN_QUEUE	= 32 - UFSHCD_NUM_RESERVED,
189 	((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
191 	((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
193 	((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
238 	/* UFS cards deviations table */
293 	if (!hba->is_irq_enabled) {
294 		enable_irq(hba->irq);
295 		hba->is_irq_enabled = true;
301 	if (hba->is_irq_enabled) {
302 		disable_irq(hba->irq);
303 		hba->is_irq_enabled = false;
322 	if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
323 		scsi_unblock_requests(hba->host);
328 	if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
329 		scsi_block_requests(hba->host);
335 	struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
342 		header = &rq->header;
344 		header = &hba->lrb[tag].ucd_rsp_ptr->header;
346 	trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
357 	trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
358 			  &rq_rsp->qr, UFS_TSF_OSF);
364 	struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
370 		trace_ufshcd_upiu(dev_name(hba->dev), str_t,
371 				  &descp->upiu_req.req_header,
372 				  &descp->upiu_req.input_param1,
375 		trace_ufshcd_upiu(dev_name(hba->dev), str_t,
376 				  &descp->upiu_rsp.rsp_header,
377 				  &descp->upiu_rsp.output_param1,
391 		cmd = ucmd->command;
395 	trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
408 	int hwq_id = -1;
409 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
410 	struct scsi_cmnd *cmd = lrbp->cmd;
412 	int transfer_len = -1;
422 	opcode = cmd->cmnd[0];
429 		       be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
432 			group_id = lrbp->cmd->cmnd[6];
446 		hwq_id = hwq->id;
450 	trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
457 	struct list_head *head = &hba->clk_list_head;
463 		if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
464 				clki->max_freq)
465 			dev_err(hba->dev, "clk: %s, rate: %u\n",
466 					clki->name, clki->curr_freq);
480 	e = &hba->ufs_stats.event[id];
483 		int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
485 		if (e->tstamp[p] == 0)
487 		dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
488 			e->val[p], div_u64(e->tstamp[p], 1000));
493 		dev_err(hba->dev, "No record of %s\n", err_name);
495 		dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
531 	lrbp = &hba->lrb[tag];
533 	dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
534 			tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
535 	dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
536 			tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
537 	dev_err(hba->dev,
538 		"UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
539 		tag, (u64)lrbp->utrd_dma_addr);
541 	ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
543 	dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
544 		(u64)lrbp->ucd_req_dma_addr);
545 	ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
547 	dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
548 		(u64)lrbp->ucd_rsp_dma_addr);
549 	ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
553 		lrbp->utr_descriptor_ptr->prd_table_length);
554 	if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
557 	dev_err(hba->dev,
558 		"UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
560 		(u64)lrbp->ucd_prdt_dma_addr);
563 		ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
569 	struct scsi_device *sdev = req->q->queuedata;
570 	struct Scsi_Host *shost = sdev->host;
573 	ufshcd_print_tr(hba, req->tag, *(bool *)priv);
579  * ufshcd_print_trs_all - print trs for all started requests.
580  * @hba: per-adapter instance.
585 	blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt);
592 	for_each_set_bit(tag, &bitmap, hba->nutmrs) {
593 		struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
595 		dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
602 	const struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
604 	dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
605 	dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
606 		hba->outstanding_reqs, hba->outstanding_tasks);
607 	dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
608 		hba->saved_err, hba->saved_uic_err);
609 	dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
610 		hba->curr_dev_pwr_mode, hba->uic_link_state);
611 	dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
612 		hba->pm_op_in_progress, hba->is_sys_suspended);
613 	dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
614 		hba->auto_bkops_enabled, hba->host->host_self_blocked);
615 	dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
616 	dev_err(hba->dev,
618 		div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000),
619 		hba->ufs_stats.hibern8_exit_cnt);
620 	dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
621 		div_u64(hba->ufs_stats.last_intr_ts, 1000),
622 		hba->ufs_stats.last_intr_status);
623 	dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
624 		hba->eh_flags, hba->req_abort_count);
625 	dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
626 		hba->ufs_version, hba->capabilities, hba->caps);
627 	dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
628 		hba->dev_quirks);
630 		dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
631 			sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
637  * ufshcd_print_pwr_info - print power params as saved in hba
639  * @hba: per-adapter instance
655 	 * never-ending cycles of messages written back to storage by user space
658 	dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
660 		 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
661 		 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
662 		 names[hba->pwr_info.pwr_rx],
663 		 names[hba->pwr_info.pwr_tx],
664 		 hba->pwr_info.hs_rate);
676 			hba->dev_info.wb_enabled = false;
677 			hba->dev_info.wb_buf_flush_enabled = false;
680 	if (err != -EOPNOTSUPP)
697  * ufshcd_wait_for_register - wait for register value to change
698  * @hba: per-adapter interface
705  * Return: -ETIMEDOUT on error, zero on success.
721 				err = -ETIMEDOUT;
730  * ufshcd_get_intr_mask - Get the interrupt bit mask
737 	if (hba->ufs_version == ufshci_version(1, 0))
739 	if (hba->ufs_version <= ufshci_version(2, 0))
746  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
755 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
763 	 * function, we convert it to the same scheme as ufs 2.0+.
772  * ufshcd_is_device_present - Check if any device connected to
784  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
796 		return le32_to_cpu(cqe->status) & MASK_OCS;
798 	return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS;
802  * ufshcd_utrl_clear() - Clear requests from the controller request list.
808 	if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
827  * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register
833 	if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
840  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
851  * ufshcd_get_uic_cmd_result - Get the UIC command result
856  * Return: 0 on success; non-zero value on error.
865  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
870  * Return: 0 on success; non-zero value on error.
878  * ufshcd_get_req_rsp - returns the TR response transaction type
886 	return ucd_rsp_ptr->header.transaction_code;
890  * ufshcd_is_exception_event - Check if the device raised an exception event
900 	return ucd_rsp_ptr->header.device_information & 1;
904  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
916  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
931  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
940  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
941  *			When run-stop registers are set to 1, it indicates the
954  * ufshcd_hba_start - Start controller initialization sequence
968  * ufshcd_is_hba_active - Get controller state
982 	if (hba->ufs_version <= ufshci_version(1, 1))
1004  * ufshcd_set_clk_freq - set UFS controller clock frequencies
1014 	struct list_head *head = &hba->clk_list_head;
1020 		if (!IS_ERR_OR_NULL(clki->clk)) {
1021 			if (scale_up && clki->max_freq) {
1022 				if (clki->curr_freq == clki->max_freq)
1025 				ret = clk_set_rate(clki->clk, clki->max_freq);
1027 					dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1028 						__func__, clki->name,
1029 						clki->max_freq, ret);
1032 				trace_ufshcd_clk_scaling(dev_name(hba->dev),
1033 						"scaled up", clki->name,
1034 						clki->curr_freq,
1035 						clki->max_freq);
1037 				clki->curr_freq = clki->max_freq;
1039 			} else if (!scale_up && clki->min_freq) {
1040 				if (clki->curr_freq == clki->min_freq)
1043 				ret = clk_set_rate(clki->clk, clki->min_freq);
1045 					dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1046 						__func__, clki->name,
1047 						clki->min_freq, ret);
1050 				trace_ufshcd_clk_scaling(dev_name(hba->dev),
1051 						"scaled down", clki->name,
1052 						clki->curr_freq,
1053 						clki->min_freq);
1054 				clki->curr_freq = clki->min_freq;
1057 		dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1058 				clki->name, clk_get_rate(clki->clk));
1066  * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1090 	trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1097  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1107 	struct list_head *head = &hba->clk_list_head;
1113 		if (!IS_ERR_OR_NULL(clki->clk)) {
1114 			if (scale_up && clki->max_freq) {
1115 				if (clki->curr_freq == clki->max_freq)
1118 			} else if (!scale_up && clki->min_freq) {
1119 				if (clki->curr_freq == clki->min_freq)
1141 	lockdep_assert_held(hba->host->host_lock);
1142 	__shost_for_each_device(sdev, hba->host)
1143 		pending += sbitmap_weight(&sdev->budget_map);
1152  * Return: 0 upon success; -EBUSY upon timeout.
1165 	spin_lock_irqsave(hba->host->host_lock, flags);
1172 		if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1173 			ret = -EBUSY;
1186 		spin_unlock_irqrestore(hba->host->host_lock, flags);
1198 		spin_lock_irqsave(hba->host->host_lock, flags);
1202 		dev_err(hba->dev,
1205 		ret = -EBUSY;
1208 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1214  * ufshcd_scale_gear - scale up/down UFS gear
1218  * Return: 0 for success; -EBUSY if scaling can't happen at this time;
1219  * non-zero for any other errors.
1227 		memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info,
1230 		memcpy(&new_pwr_info, &hba->pwr_info,
1233 		if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1234 		    hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1236 			memcpy(&hba->clk_scaling.saved_pwr_info,
1237 				&hba->pwr_info,
1241 			new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1242 			new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1249 		dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1251 			hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1261  * Return: 0 upon success; -EBUSY upon timeout.
1270 	blk_mq_quiesce_tagset(&hba->host->tag_set);
1271 	mutex_lock(&hba->wb_mutex);
1272 	down_write(&hba->clk_scaling_lock);
1274 	if (!hba->clk_scaling.is_allowed ||
1276 		ret = -EBUSY;
1277 		up_write(&hba->clk_scaling_lock);
1278 		mutex_unlock(&hba->wb_mutex);
1279 		blk_mq_unquiesce_tagset(&hba->host->tag_set);
1292 	up_write(&hba->clk_scaling_lock);
1298 	mutex_unlock(&hba->wb_mutex);
1300 	blk_mq_unquiesce_tagset(&hba->host->tag_set);
1305  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1309  * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero
1354 	spin_lock_irqsave(hba->host->host_lock, irq_flags);
1355 	if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1356 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1359 	hba->clk_scaling.is_suspended = true;
1360 	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1371 	spin_lock_irqsave(hba->host->host_lock, irq_flags);
1372 	if (!hba->clk_scaling.is_suspended) {
1373 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1376 	hba->clk_scaling.is_suspended = false;
1377 	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1379 	devfreq_resume_device(hba->devfreq);
1389 	struct list_head *clk_list = &hba->clk_list_head;
1394 		return -EINVAL;
1396 	clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1398 	*freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1399 	spin_lock_irqsave(hba->host->host_lock, irq_flags);
1401 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1405 	if (!hba->clk_scaling.active_reqs)
1409 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1413 	/* Decide based on the rounded-off frequency and update */
1414 	scale_up = *freq == clki->max_freq;
1416 		*freq = clki->min_freq;
1419 		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1423 	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1428 	trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1434 		queue_work(hba->clk_scaling.workq,
1435 			   &hba->clk_scaling.suspend_work);
1444 	struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1446 	struct list_head *clk_list = &hba->clk_list_head;
1451 		return -EINVAL;
1455 	spin_lock_irqsave(hba->host->host_lock, flags);
1457 	if (!scaling->window_start_t)
1466 	stat->current_frequency = clki->curr_freq;
1467 	if (scaling->is_busy_started)
1468 		scaling->tot_busy_t += ktime_us_delta(curr_t,
1469 				scaling->busy_start_t);
1471 	stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1472 	stat->busy_time = scaling->tot_busy_t;
1474 	scaling->window_start_t = curr_t;
1475 	scaling->tot_busy_t = 0;
1477 	if (scaling->active_reqs) {
1478 		scaling->busy_start_t = curr_t;
1479 		scaling->is_busy_started = true;
1481 		scaling->busy_start_t = 0;
1482 		scaling->is_busy_started = false;
1484 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1490 	struct list_head *clk_list = &hba->clk_list_head;
1500 	dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1501 	dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1503 	ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1504 					 &hba->vps->ondemand_data);
1505 	devfreq = devfreq_add_device(hba->dev,
1506 			&hba->vps->devfreq_profile,
1508 			&hba->vps->ondemand_data);
1511 		dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1513 		dev_pm_opp_remove(hba->dev, clki->min_freq);
1514 		dev_pm_opp_remove(hba->dev, clki->max_freq);
1518 	hba->devfreq = devfreq;
1525 	struct list_head *clk_list = &hba->clk_list_head;
1528 	if (!hba->devfreq)
1531 	devfreq_remove_device(hba->devfreq);
1532 	hba->devfreq = NULL;
1535 	dev_pm_opp_remove(hba->dev, clki->min_freq);
1536 	dev_pm_opp_remove(hba->dev, clki->max_freq);
1543 	devfreq_suspend_device(hba->devfreq);
1544 	spin_lock_irqsave(hba->host->host_lock, flags);
1545 	hba->clk_scaling.window_start_t = 0;
1546 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1554 	cancel_work_sync(&hba->clk_scaling.suspend_work);
1555 	cancel_work_sync(&hba->clk_scaling.resume_work);
1557 	spin_lock_irqsave(hba->host->host_lock, flags);
1558 	if (!hba->clk_scaling.is_suspended) {
1560 		hba->clk_scaling.is_suspended = true;
1562 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1573 	spin_lock_irqsave(hba->host->host_lock, flags);
1574 	if (hba->clk_scaling.is_suspended) {
1576 		hba->clk_scaling.is_suspended = false;
1578 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1581 		devfreq_resume_device(hba->devfreq);
1589 	return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1600 		return -EINVAL;
1602 	down(&hba->host_sem);
1604 		err = -EBUSY;
1609 	if (value == hba->clk_scaling.is_enabled)
1615 	hba->clk_scaling.is_enabled = value;
1623 			dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1630 	up(&hba->host_sem);
1636 	hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1637 	hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1638 	sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1639 	hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1640 	hba->clk_scaling.enable_attr.attr.mode = 0644;
1641 	if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1642 		dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1647 	if (hba->clk_scaling.enable_attr.attr.name)
1648 		device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1658 	if (!hba->clk_scaling.min_gear)
1659 		hba->clk_scaling.min_gear = UFS_HS_G1;
1661 	INIT_WORK(&hba->clk_scaling.suspend_work,
1663 	INIT_WORK(&hba->clk_scaling.resume_work,
1667 		 hba->host->host_no);
1668 	hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1670 	hba->clk_scaling.is_initialized = true;
1675 	if (!hba->clk_scaling.is_initialized)
1679 	destroy_workqueue(hba->clk_scaling.workq);
1681 	hba->clk_scaling.is_initialized = false;
1691 	cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1693 	spin_lock_irqsave(hba->host->host_lock, flags);
1694 	if (hba->clk_gating.state == CLKS_ON) {
1695 		spin_unlock_irqrestore(hba->host->host_lock, flags);
1699 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1708 		hba->clk_gating.is_suspended = true;
1712 				dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1717 		hba->clk_gating.is_suspended = false;
1722  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1732 	    !hba->clk_gating.is_initialized)
1734 	spin_lock_irqsave(hba->host->host_lock, flags);
1735 	hba->clk_gating.active_reqs++;
1738 	switch (hba->clk_gating.state) {
1750 			spin_unlock_irqrestore(hba->host->host_lock, flags);
1751 			flush_result = flush_work(&hba->clk_gating.ungate_work);
1752 			if (hba->clk_gating.is_suspended && !flush_result)
1754 			spin_lock_irqsave(hba->host->host_lock, flags);
1759 		if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1760 			hba->clk_gating.state = CLKS_ON;
1761 			trace_ufshcd_clk_gating(dev_name(hba->dev),
1762 						hba->clk_gating.state);
1772 		hba->clk_gating.state = REQ_CLKS_ON;
1773 		trace_ufshcd_clk_gating(dev_name(hba->dev),
1774 					hba->clk_gating.state);
1775 		queue_work(hba->clk_gating.clk_gating_workq,
1776 			   &hba->clk_gating.ungate_work);
1783 		spin_unlock_irqrestore(hba->host->host_lock, flags);
1784 		flush_work(&hba->clk_gating.ungate_work);
1786 		spin_lock_irqsave(hba->host->host_lock, flags);
1789 		dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1790 				__func__, hba->clk_gating.state);
1793 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1804 	spin_lock_irqsave(hba->host->host_lock, flags);
1811 	if (hba->clk_gating.is_suspended ||
1812 		(hba->clk_gating.state != REQ_CLKS_OFF)) {
1813 		hba->clk_gating.state = CLKS_ON;
1814 		trace_ufshcd_clk_gating(dev_name(hba->dev),
1815 					hba->clk_gating.state);
1819 	if (hba->clk_gating.active_reqs
1820 		|| hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1821 		|| hba->outstanding_reqs || hba->outstanding_tasks
1822 		|| hba->active_uic_cmd || hba->uic_async_done)
1825 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1831 			hba->clk_gating.state = CLKS_ON;
1832 			dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1834 			trace_ufshcd_clk_gating(dev_name(hba->dev),
1835 						hba->clk_gating.state);
1856 	spin_lock_irqsave(hba->host->host_lock, flags);
1857 	if (hba->clk_gating.state == REQ_CLKS_OFF) {
1858 		hba->clk_gating.state = CLKS_OFF;
1859 		trace_ufshcd_clk_gating(dev_name(hba->dev),
1860 					hba->clk_gating.state);
1863 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1874 	hba->clk_gating.active_reqs--;
1876 	if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1877 	    hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1878 	    hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
1879 	    hba->active_uic_cmd || hba->uic_async_done ||
1880 	    hba->clk_gating.state == CLKS_OFF)
1883 	hba->clk_gating.state = REQ_CLKS_OFF;
1884 	trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1885 	queue_delayed_work(hba->clk_gating.clk_gating_workq,
1886 			   &hba->clk_gating.gate_work,
1887 			   msecs_to_jiffies(hba->clk_gating.delay_ms));
1894 	spin_lock_irqsave(hba->host->host_lock, flags);
1896 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1905 	return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
1913 	spin_lock_irqsave(hba->host->host_lock, flags);
1914 	hba->clk_gating.delay_ms = value;
1915 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1925 		return -EINVAL;
1936 	return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
1947 		return -EINVAL;
1951 	spin_lock_irqsave(hba->host->host_lock, flags);
1952 	if (value == hba->clk_gating.is_enabled)
1958 		hba->clk_gating.active_reqs++;
1960 	hba->clk_gating.is_enabled = value;
1962 	spin_unlock_irqrestore(hba->host->host_lock, flags);
1968 	hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1969 	hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1970 	sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1971 	hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1972 	hba->clk_gating.delay_attr.attr.mode = 0644;
1973 	if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1974 		dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1976 	hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1977 	hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1978 	sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1979 	hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1980 	hba->clk_gating.enable_attr.attr.mode = 0644;
1981 	if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1982 		dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1987 	if (hba->clk_gating.delay_attr.attr.name)
1988 		device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1989 	if (hba->clk_gating.enable_attr.attr.name)
1990 		device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
2000 	hba->clk_gating.state = CLKS_ON;
2002 	hba->clk_gating.delay_ms = 150;
2003 	INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
2004 	INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
2007 		 hba->host->host_no);
2008 	hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
2013 	hba->clk_gating.is_enabled = true;
2014 	hba->clk_gating.is_initialized = true;
2019 	if (!hba->clk_gating.is_initialized)
2026 	hba->clk_gating.is_initialized = false;
2029 	destroy_workqueue(hba->clk_gating.clk_gating_workq);
2041 	spin_lock_irqsave(hba->host->host_lock, flags);
2042 	if (!hba->clk_scaling.active_reqs++)
2045 	if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2046 		spin_unlock_irqrestore(hba->host->host_lock, flags);
2051 		queue_work(hba->clk_scaling.workq,
2052 			   &hba->clk_scaling.resume_work);
2054 	if (!hba->clk_scaling.window_start_t) {
2055 		hba->clk_scaling.window_start_t = curr_t;
2056 		hba->clk_scaling.tot_busy_t = 0;
2057 		hba->clk_scaling.is_busy_started = false;
2060 	if (!hba->clk_scaling.is_busy_started) {
2061 		hba->clk_scaling.busy_start_t = curr_t;
2062 		hba->clk_scaling.is_busy_started = true;
2064 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2069 	struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2075 	spin_lock_irqsave(hba->host->host_lock, flags);
2076 	hba->clk_scaling.active_reqs--;
2077 	if (!scaling->active_reqs && scaling->is_busy_started) {
2078 		scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2079 					scaling->busy_start_t));
2080 		scaling->busy_start_t = 0;
2081 		scaling->is_busy_started = false;
2083 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2093 		return -EINVAL;
2099 	const struct ufs_hba_monitor *m = &hba->monitor;
2101 	return (m->enabled && lrbp && lrbp->cmd &&
2102 		(!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2103 		ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2109 	int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2112 	spin_lock_irqsave(hba->host->host_lock, flags);
2113 	if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2114 		hba->monitor.busy_start_ts[dir] = ktime_get();
2115 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2120 	int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2123 	spin_lock_irqsave(hba->host->host_lock, flags);
2124 	if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2125 		const struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2126 		struct ufs_hba_monitor *m = &hba->monitor;
2129 		now = lrbp->compl_time_stamp;
2130 		inc = ktime_sub(now, m->busy_start_ts[dir]);
2131 		m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2132 		m->nr_sec_rw[dir] += blk_rq_sectors(req);
2135 		m->nr_req[dir]++;
2136 		lat = ktime_sub(now, lrbp->issue_time_stamp);
2137 		m->lat_sum[dir] += lat;
2138 		if (m->lat_max[dir] < lat || !m->lat_max[dir])
2139 			m->lat_max[dir] = lat;
2140 		if (m->lat_min[dir] > lat || !m->lat_min[dir])
2141 			m->lat_min[dir] = lat;
2143 		m->nr_queued[dir]--;
2145 		m->busy_start_ts[dir] = now;
2147 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2151  * ufshcd_send_command - Send SCSI or device management commands
2160 	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2163 	lrbp->issue_time_stamp = ktime_get();
2164 	lrbp->issue_time_stamp_local_clock = local_clock();
2165 	lrbp->compl_time_stamp = ktime_set(0, 0);
2166 	lrbp->compl_time_stamp_local_clock = 0;
2174 		struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr;
2177 		spin_lock(&hwq->sq_lock);
2178 		dest = hwq->sqe_base_addr + hwq->sq_tail_slot;
2181 		spin_unlock(&hwq->sq_lock);
2183 		spin_lock_irqsave(&hba->outstanding_lock, flags);
2184 		if (hba->vops && hba->vops->setup_xfer_req)
2185 			hba->vops->setup_xfer_req(hba, lrbp->task_tag,
2186 						  !!lrbp->cmd);
2187 		__set_bit(lrbp->task_tag, &hba->outstanding_reqs);
2188 		ufshcd_writel(hba, 1 << lrbp->task_tag,
2190 		spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2195  * ufshcd_copy_sense_data - Copy sense data in case of check condition
2200 	u8 *const sense_buffer = lrbp->cmd->sense_buffer;
2204 	resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length);
2208 		len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2211 		memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2217  * ufshcd_copy_query_response() - Copy the Query Response and the data
2227 	struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2229 	memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2232 	if (hba->dev_cmd.query.descriptor &&
2233 	    lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2234 		u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2240 		resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
2243 				hba->dev_cmd.query.request.upiu_req.length);
2245 			memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2247 			dev_warn(hba->dev,
2250 			return -EINVAL;
2258  * ufshcd_hba_capabilities - Read controller capabilities
2267 	hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2270 	hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2271 	hba->nutmrs =
2272 	((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2273 	hba->reserved_slot = hba->nutrs - 1;
2278 		dev_err(hba->dev, "crypto setup failed\n");
2287 	hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities);
2292 	hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities);
2293 	if (!hba->mcq_sup)
2296 	hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP);
2297 	hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT,
2298 				     hba->mcq_capabilities);
2304  * ufshcd_ready_for_uic_cmd - Check if controller is ready
2320  * ufshcd_get_upmcrs - Get the power mode change request status
2333  * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2340 	lockdep_assert_held(&hba->uic_cmd_mutex);
2342 	WARN_ON(hba->active_uic_cmd);
2344 	hba->active_uic_cmd = uic_cmd;
2347 	ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2348 	ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2349 	ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2354 	ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2359  * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2371 	lockdep_assert_held(&hba->uic_cmd_mutex);
2373 	if (wait_for_completion_timeout(&uic_cmd->done,
2375 		ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2377 		ret = -ETIMEDOUT;
2378 		dev_err(hba->dev,
2380 			uic_cmd->command, uic_cmd->argument3);
2382 		if (!uic_cmd->cmd_active) {
2383 			dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2385 			ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2389 	spin_lock_irqsave(hba->host->host_lock, flags);
2390 	hba->active_uic_cmd = NULL;
2391 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2397  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2406 	lockdep_assert_held(&hba->uic_cmd_mutex);
2409 		dev_err(hba->dev,
2411 		return -EIO;
2414 	init_completion(&uic_cmd->done);
2416 	uic_cmd->cmd_active = 1;
2423  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2433 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2437 	mutex_lock(&hba->uic_cmd_mutex);
2444 	mutex_unlock(&hba->uic_cmd_mutex);
2451  * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format)
2452  * @hba:	per-adapter instance
2466 		if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2467 			lrbp->utr_descriptor_ptr->prd_table_length =
2470 			lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries);
2472 		prd = lrbp->ucd_prdt_ptr;
2486 			prd->size = cpu_to_le32(len - 1);
2487 			prd->addr = cpu_to_le64(sg->dma_address);
2488 			prd->reserved = 0;
2492 		lrbp->utr_descriptor_ptr->prd_table_length = 0;
2497  * ufshcd_map_sg - Map scatter-gather list to prdt
2501  * Return: 0 in case of success, non-zero value in case of failure.
2505 	struct scsi_cmnd *cmd = lrbp->cmd;
2517  * ufshcd_enable_intr - enable interrupts
2525 	if (hba->ufs_version == ufshci_version(1, 0)) {
2537  * ufshcd_disable_intr - disable interrupts
2545 	if (hba->ufs_version == ufshci_version(1, 0)) {
2559  * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request
2569 	struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2570 	struct request_desc_header *h = &req_desc->header;
2586 	h->command_type = lrbp->command_type;
2587 	h->data_direction = data_direction;
2588 	h->ehs_length = ehs_length;
2590 	if (lrbp->intr_cmd)
2591 		h->interrupt = 1;
2601 	h->ocs = OCS_INVALID_COMMAND_STATUS;
2603 	req_desc->prd_table_length = 0;
2607  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2615 	struct scsi_cmnd *cmd = lrbp->cmd;
2616 	struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2619 	ucd_req_ptr->header = (struct utp_upiu_header){
2622 		.lun = lrbp->lun,
2623 		.task_tag = lrbp->task_tag,
2627 	ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2629 	cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2630 	memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2631 	memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2633 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2637  * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request
2638  * @hba: UFS hba
2645 	struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2646 	struct ufs_query *query = &hba->dev_cmd.query;
2647 	u16 len = be16_to_cpu(query->request.upiu_req.length);
2650 	ucd_req_ptr->header = (struct utp_upiu_header){
2653 		.lun = lrbp->lun,
2654 		.task_tag = lrbp->task_tag,
2655 		.query_function = query->request.query_func,
2658 			query->request.upiu_req.opcode ==
2665 	memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2669 	if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2670 		memcpy(ucd_req_ptr + 1, query->descriptor, len);
2672 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2677 	struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2681 	ucd_req_ptr->header = (struct utp_upiu_header){
2683 		.task_tag = lrbp->task_tag,
2686 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2690  * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2703 	if (hba->ufs_version <= ufshci_version(1, 1))
2704 		lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2706 		lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2709 	if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2711 	else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2714 		ret = -EINVAL;
2720  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2732 	if (hba->ufs_version <= ufshci_version(1, 1))
2733 		lrbp->command_type = UTP_CMD_TYPE_SCSI;
2735 		lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2737 	if (likely(lrbp->cmd)) {
2738 		ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
2741 		ret = -EINVAL;
2748  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2749  * @upiu_wlun_id: UPIU W-LUN id
2751  * Return: SCSI W-LUN id.
2760 	return sdev->lun ==
2765  * Associate the UFS controller queue with the default and poll HCTX types.
2774 		hba->nr_queues[HCTX_TYPE_DEFAULT] = 1;
2775 		hba->nr_queues[HCTX_TYPE_READ] = 0;
2776 		hba->nr_queues[HCTX_TYPE_POLL] = 1;
2777 		hba->nr_hw_queues = 1;
2780 	for (i = 0; i < shost->nr_maps; i++) {
2781 		struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2783 		map->nr_queues = hba->nr_queues[i];
2784 		if (!map->nr_queues)
2786 		map->queue_offset = queue_offset;
2788 			map->queue_offset = 0;
2791 		queue_offset += map->nr_queues;
2797 	struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
2799 	struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2800 	dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2805 	lrb->utr_descriptor_ptr = utrdlp + i;
2806 	lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2808 	lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu;
2809 	lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2810 	lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
2811 	lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2812 	lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
2813 	lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2817  * ufshcd_queuecommand - main entry point for SCSI requests
2821  * Return: 0 for success, non-zero in case of failure.
2826 	int tag = scsi_cmd_to_rq(cmd)->tag;
2831 	WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag);
2833 	switch (hba->ufshcd_state) {
2838 		 * SCSI error handler can call ->queuecommand() while UFS error
2855 		 * can we let the scsi cmd pass through, because UFS is in bad
2860 		if (hba->pm_op_in_progress) {
2861 			hba->force_reset = true;
2876 	hba->req_abort_count = 0;
2880 	lrbp = &hba->lrb[tag];
2881 	lrbp->cmd = cmd;
2882 	lrbp->task_tag = tag;
2883 	lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2884 	lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
2888 	lrbp->req_abort_skip = false;
2907 		spin_lock_irqsave(hba->host->host_lock, flags);
2909 		spin_unlock_irqrestore(hba->host->host_lock, flags);
2918 	lrbp->cmd = NULL;
2919 	lrbp->task_tag = tag;
2920 	lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2921 	lrbp->intr_cmd = true; /* No interrupt aggregation */
2923 	hba->dev_cmd.type = cmd_type;
2967 			dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
2977 	spin_lock_irqsave(hba->host->host_lock, flags);
2979 	spin_unlock_irqrestore(hba->host->host_lock, flags);
2982 	 * wait for h/w to clear corresponding bit in door-bell.
2990  * ufshcd_dev_cmd_completion() - handles device management command responses
3002 	hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
3003 	resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3007 		if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
3008 			err = -EINVAL;
3009 			dev_err(hba->dev, "%s: unexpected response %x\n",
3014 		u8 response = lrbp->ucd_rsp_ptr->header.response;
3022 		err = -EPERM;
3023 		dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
3027 		if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) {
3028 			err = -EINVAL;
3029 			dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp);
3033 		err = -EINVAL;
3034 		dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
3051 	time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
3060 		hba->dev_cmd.complete = NULL;
3065 		err = -ETIMEDOUT;
3066 		dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
3067 			__func__, lrbp->task_tag);
3072 			if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0)
3073 				err = -EAGAIN;
3074 			hba->dev_cmd.complete = NULL;
3079 		if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) {
3081 			err = -EAGAIN;
3087 			spin_lock_irqsave(&hba->outstanding_lock, flags);
3088 			pending = test_bit(lrbp->task_tag,
3089 					   &hba->outstanding_reqs);
3091 				hba->dev_cmd.complete = NULL;
3092 				__clear_bit(lrbp->task_tag,
3093 					    &hba->outstanding_reqs);
3095 			spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3106 			dev_err(hba->dev, "%s: failed to clear tag %d\n",
3107 				__func__, lrbp->task_tag);
3109 			spin_lock_irqsave(&hba->outstanding_lock, flags);
3110 			pending = test_bit(lrbp->task_tag,
3111 					   &hba->outstanding_reqs);
3113 				hba->dev_cmd.complete = NULL;
3114 			spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3131  * ufshcd_exec_dev_cmd - API for sending device management requests
3132  * @hba: UFS hba
3139  * it is expected you hold the hba->dev_cmd.lock mutex.
3145 	const u32 tag = hba->reserved_slot;
3149 	/* Protects use of hba->reserved_slot. */
3150 	lockdep_assert_held(&hba->dev_cmd.lock);
3152 	down_read(&hba->clk_scaling_lock);
3154 	lrbp = &hba->lrb[tag];
3155 	lrbp->cmd = NULL;
3160 	hba->dev_cmd.complete = &wait;
3162 	ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
3164 	ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
3167 				    (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
3170 	up_read(&hba->clk_scaling_lock);
3175  * ufshcd_init_query() - init the query response and request parameters
3176  * @hba: per-adapter instance
3188 	*request = &hba->dev_cmd.query.request;
3189 	*response = &hba->dev_cmd.query.response;
3192 	(*request)->upiu_req.opcode = opcode;
3193 	(*request)->upiu_req.idn = idn;
3194 	(*request)->upiu_req.index = index;
3195 	(*request)->upiu_req.selector = selector;
3207 			dev_dbg(hba->dev,
3215 		dev_err(hba->dev,
3222  * ufshcd_query_flag() - API function for sending flag query requests
3223  * @hba: per-adapter instance
3229  * Return: 0 for success, non-zero in case of failure.
3242 	mutex_lock(&hba->dev_cmd.lock);
3250 		request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3253 		request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3256 			dev_err(hba->dev, "%s: Invalid argument for read request\n",
3258 			err = -EINVAL;
3263 		dev_err(hba->dev,
3266 		err = -EINVAL;
3273 		dev_err(hba->dev,
3280 		*flag_res = (be32_to_cpu(response->upiu_res.value) &
3284 	mutex_unlock(&hba->dev_cmd.lock);
3290  * ufshcd_query_attr - API function for sending attribute requests
3291  * @hba: per-adapter instance
3298  * Return: 0 for success, non-zero in case of failure.
3310 		dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3312 		return -EINVAL;
3317 	mutex_lock(&hba->dev_cmd.lock);
3323 		request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3324 		request->upiu_req.value = cpu_to_be32(*attr_val);
3327 		request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3330 		dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3332 		err = -EINVAL;
3339 		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3344 	*attr_val = be32_to_cpu(response->upiu_res.value);
3347 	mutex_unlock(&hba->dev_cmd.lock);
3353  * ufshcd_query_attr_retry() - API function for sending query
3355  * @hba: per-adapter instance
3363  * Return: 0 for success, non-zero in case of failure.
3372 	for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3376 			dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3383 		dev_err(hba->dev,
3400 		dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3402 		return -EINVAL;
3406 		dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3408 		return -EINVAL;
3413 	mutex_lock(&hba->dev_cmd.lock);
3416 	hba->dev_cmd.query.descriptor = desc_buf;
3417 	request->upiu_req.length = cpu_to_be16(*buf_len);
3421 		request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3424 		request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3427 		dev_err(hba->dev,
3430 		err = -EINVAL;
3437 		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3442 	*buf_len = be16_to_cpu(response->upiu_res.length);
3445 	hba->dev_cmd.query.descriptor = NULL;
3446 	mutex_unlock(&hba->dev_cmd.lock);
3452  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3453  * @hba: per-adapter instance
3464  * Return: 0 for success, non-zero in case of failure.
3475 	for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3478 		if (!err || err == -EINVAL)
3486  * ufshcd_read_desc_param - read the specified descriptor parameter
3494  * Return: 0 in case of success, non-zero otherwise.
3510 		return -EINVAL;
3516 			return -ENOMEM;
3527 		dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3536 		dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3538 		ret = -EINVAL;
3544 		dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3546 		ret = -EINVAL;
3553 			ret = -EINVAL;
3556 			       min_t(u32, param_size, buff_len - param_offset));
3565  * struct uc_string_id - unicode string
3577 /* replace non-printable or non-ASCII characters with spaces */
3584  * ufshcd_read_string_desc - read string descriptor
3594  * *      -ENOMEM: on allocation failure
3595  * *      -EINVAL: on a wrong parameter
3605 		return -EINVAL;
3609 		return -ENOMEM;
3614 		dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3620 	if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3621 		dev_dbg(hba->dev, "String Desc is of zero length\n");
3631 		ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3634 			ret = -ENOMEM;
3640 		 * we need to convert to utf-8 so it can be displayed
3642 		ret = utf16s_to_utf8s(uc_str->uc,
3643 				      uc_str->len - QUERY_DESC_HDR_SIZE,
3644 				      UTF16_BIG_ENDIAN, str, ascii_len - 1);
3646 		/* replace non-printable or non-ASCII characters with spaces */
3653 		str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3655 			ret = -ENOMEM;
3658 		ret = uc_str->len;
3667  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3674  * Return: 0 in case of success, non-zero otherwise.
3686 	if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
3687 		return -EOPNOTSUPP;
3698 	if (hba->dev_info.wspecversion >= 0x300) {
3703 			dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3708 			dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3712 		hba->dev_info.clk_gating_wait_us = gating_wait;
3719  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3729  * Return: 0 for success, non-zero in case of failure.
3736 	ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs;
3737 	hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3739 						  &hba->ucdl_dma_addr,
3745 	if (!hba->ucdl_base_addr ||
3746 	    WARN_ON(hba->ucdl_dma_addr & (128 - 1))) {
3747 		dev_err(hba->dev,
3756 	utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3757 	hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3759 						   &hba->utrdl_dma_addr,
3761 	if (!hba->utrdl_base_addr ||
3762 	    WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) {
3763 		dev_err(hba->dev,
3774 	if (hba->utmrdl_base_addr)
3780 	utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3781 	hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3783 						    &hba->utmrdl_dma_addr,
3785 	if (!hba->utmrdl_base_addr ||
3786 	    WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) {
3787 		dev_err(hba->dev,
3794 	hba->lrb = devm_kcalloc(hba->dev,
3795 				hba->nutrs, sizeof(struct ufshcd_lrb),
3797 	if (!hba->lrb) {
3798 		dev_err(hba->dev, "LRB Memory allocation failed\n");
3803 	return -ENOMEM;
3807  * ufshcd_host_memory_configure - configure local reference block with
3829 	utrdlp = hba->utrdl_base_addr;
3837 	cmd_desc_dma_addr = hba->ucdl_dma_addr;
3839 	for (i = 0; i < hba->nutrs; i++) {
3847 		if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3863 		ufshcd_init_lrb(hba, &hba->lrb[i], i);
3868  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3876  * Return: 0 on success, non-zero value on failure.
3887 		dev_dbg(hba->dev,
3888 			"dme-link-startup: error code %d\n", ret);
3892  * ufshcd_dme_reset - UIC command for DME_RESET
3898  * Return: 0 on success, non-zero value on failure.
3909 		dev_err(hba->dev,
3910 			"dme-reset: error code %d\n", ret);
3932  * ufshcd_dme_enable - UIC command for DME_ENABLE
3937  * Return: 0 on success, non-zero value on failure.
3948 		dev_err(hba->dev,
3949 			"dme-enable: error code %d\n", ret);
3959 	if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3966 	if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3972 				hba->last_dme_cmd_tstamp));
3976 				MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3987 	hba->last_dme_cmd_tstamp = ktime_get();
3991  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3998  * Return: 0 on success, non-zero value on failure.
4005 		"dme-set",
4006 		"dme-peer-set"
4022 			dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
4024 	} while (ret && peer && --retries);
4027 		dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
4029 			UFS_UIC_COMMAND_RETRIES - retries);
4036  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
4042  * Return: 0 on success, non-zero value on failure.
4049 		"dme-get",
4050 		"dme-peer-get"
4059 	if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
4060 		orig_pwr_info = hba->pwr_info;
4089 			dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
4091 	} while (ret && peer && --retries);
4094 		dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
4096 			UFS_UIC_COMMAND_RETRIES - retries);
4101 	if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
4110  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
4123  * Return: 0 on success, non-zero value on failure.
4133 	mutex_lock(&hba->uic_cmd_mutex);
4136 	spin_lock_irqsave(hba->host->host_lock, flags);
4138 		ret = -ENOLINK;
4141 	hba->uic_async_done = &uic_async_done;
4151 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4154 		dev_err(hba->dev,
4156 			cmd->command, cmd->argument3, ret);
4160 	if (!wait_for_completion_timeout(hba->uic_async_done,
4162 		dev_err(hba->dev,
4164 			cmd->command, cmd->argument3);
4166 		if (!cmd->cmd_active) {
4167 			dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4172 		ret = -ETIMEDOUT;
4179 		dev_err(hba->dev,
4181 			cmd->command, status);
4182 		ret = (status != PWR_OK) ? status : -1;
4191 	spin_lock_irqsave(hba->host->host_lock, flags);
4192 	hba->active_uic_cmd = NULL;
4193 	hba->uic_async_done = NULL;
4201 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4202 	mutex_unlock(&hba->uic_cmd_mutex);
4208  * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result
4218 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
4223 	if (uic_cmd->argument1 == UIC_ARG_MIB(PA_PWRMODE) &&
4224 	    uic_cmd->command == UIC_CMD_DME_SET) {
4229 	mutex_lock(&hba->uic_cmd_mutex);
4236 	mutex_unlock(&hba->uic_cmd_mutex);
4244  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4249  * Return: 0 on success, non-zero value on failure.
4256 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4260 			dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4283 	spin_lock_irqsave(hba->host->host_lock, flags);
4284 	hba->ufshcd_state = UFSHCD_STATE_RESET;
4286 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4293 	spin_lock_irqsave(hba->host->host_lock, flags);
4295 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
4297 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4300 		dev_err(hba->dev, "%s: link recovery failed, err %d",
4317 	trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4321 		dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4341 	trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4345 		dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4350 		hba->ufs_stats.last_hibern8_exit_tstamp = local_clock();
4351 		hba->ufs_stats.hibern8_exit_cnt++;
4366 	spin_lock_irqsave(hba->host->host_lock, flags);
4367 	if (hba->ahit != ahit) {
4368 		hba->ahit = ahit;
4371 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4374 	    !pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
4389 	ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4393  * ufshcd_init_pwr_info - setting the POR (power on reset)
4395  * @hba: per-adapter instance
4399 	hba->pwr_info.gear_rx = UFS_PWM_G1;
4400 	hba->pwr_info.gear_tx = UFS_PWM_G1;
4401 	hba->pwr_info.lane_rx = UFS_LANE_1;
4402 	hba->pwr_info.lane_tx = UFS_LANE_1;
4403 	hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4404 	hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4405 	hba->pwr_info.hs_rate = 0;
4409  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4410  * @hba: per-adapter instance
4416 	struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4418 	if (hba->max_pwr_info.is_valid)
4421 	if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) {
4422 		pwr_info->pwr_tx = FASTAUTO_MODE;
4423 		pwr_info->pwr_rx = FASTAUTO_MODE;
4425 		pwr_info->pwr_tx = FAST_MODE;
4426 		pwr_info->pwr_rx = FAST_MODE;
4428 	pwr_info->hs_rate = PA_HS_MODE_B;
4432 			&pwr_info->lane_rx);
4434 			&pwr_info->lane_tx);
4436 	if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4437 		dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4439 				pwr_info->lane_rx,
4440 				pwr_info->lane_tx);
4441 		return -EINVAL;
4449 	ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4450 	if (!pwr_info->gear_rx) {
4452 				&pwr_info->gear_rx);
4453 		if (!pwr_info->gear_rx) {
4454 			dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4455 				__func__, pwr_info->gear_rx);
4456 			return -EINVAL;
4458 		pwr_info->pwr_rx = SLOW_MODE;
4462 			&pwr_info->gear_tx);
4463 	if (!pwr_info->gear_tx) {
4465 				&pwr_info->gear_tx);
4466 		if (!pwr_info->gear_tx) {
4467 			dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4468 				__func__, pwr_info->gear_tx);
4469 			return -EINVAL;
4471 		pwr_info->pwr_tx = SLOW_MODE;
4474 	hba->max_pwr_info.is_valid = true;
4484 	if (!hba->force_pmc &&
4485 	    pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4486 	    pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4487 	    pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4488 	    pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4489 	    pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4490 	    pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4491 	    pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4492 		dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4498 	 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4499 	 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4500 	 * - PA_HSSERIES
4502 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4504 			pwr_mode->lane_rx);
4505 	if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4506 			pwr_mode->pwr_rx == FAST_MODE)
4511 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4513 			pwr_mode->lane_tx);
4514 	if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4515 			pwr_mode->pwr_tx == FAST_MODE)
4520 	if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4521 	    pwr_mode->pwr_tx == FASTAUTO_MODE ||
4522 	    pwr_mode->pwr_rx == FAST_MODE ||
4523 	    pwr_mode->pwr_tx == FAST_MODE)
4525 						pwr_mode->hs_rate);
4527 	if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4549 	ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4550 			| pwr_mode->pwr_tx);
4553 		dev_err(hba->dev,
4556 		memcpy(&hba->pwr_info, pwr_mode,
4564  * ufshcd_config_pwr_mode - configure a new power mode
4565  * @hba: per-adapter instance
4593  * ufshcd_complete_dev_init() - checks device readiness
4594  * @hba: per-adapter instance
4609 		dev_err(hba->dev,
4626 		dev_err(hba->dev,
4630 		dev_err(hba->dev,
4633 		err = -EBUSY;
4640  * ufshcd_make_hba_operational - Make UFS controller operational
4643  * To bring UFS host controller to operational state,
4647  * 4. Configure run-stop-registers
4649  * Return: 0 on success, non-zero value on failure.
4661 		ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4666 	ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4668 	ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4670 	ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4672 	ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4688 		dev_err(hba->dev,
4690 		err = -EIO;
4698  * ufshcd_hba_stop - Send controller to reset state
4708 	 * while the UFS interrupt handler is active on another CPU.
4710 	spin_lock_irqsave(hba->host->host_lock, flags);
4712 	spin_unlock_irqrestore(hba->host->host_lock, flags);
4718 		dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4723  * ufshcd_hba_execute_hce - initialize the controller
4730  * Return: 0 on success, non-zero value on failure.
4751 	 * To initialize a UFS host controller HCE bit must be set to 1.
4752 	 * During initialization the HCE bit value changes from 1->0->1.
4760 	ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4766 			retry_inner--;
4768 			dev_err(hba->dev,
4771 				retry_outer--;
4774 			return -EIO;
4791 	if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4799 			dev_err(hba->dev, "DME_RESET failed\n");
4805 			dev_err(hba->dev, "Enabling DME failed\n");
4840 			dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4861 	e = &hba->ufs_stats.event[id];
4862 	e->val[e->pos] = val;
4863 	e->tstamp[e->pos] = local_clock();
4864 	e->cnt += 1;
4865 	e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4872  * ufshcd_link_startup - Initialize unipro link startup
4875  * Return: 0 for success, non-zero in case of failure.
4884 	 * If UFS device isn't active then we will have to issue link startup
4896 		/* check if device is detected by inter-connect layer */
4901 			dev_err(hba->dev, "%s: Device not present\n", __func__);
4902 			ret = -ENXIO;
4909 		 * succeeds. So reset the local Uni-Pro and try again.
4917 	} while (ret && retries--);
4933 	/* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4937 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4953 		dev_err(hba->dev, "link startup failed %d\n", ret);
4962  * ufshcd_verify_dev_init() - Verify device initialization
4963  * @hba: per-adapter instance
4979 	mutex_lock(&hba->dev_cmd.lock);
4980 	for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4982 					  hba->nop_out_timeout);
4984 		if (!err || err == -ETIMEDOUT)
4987 		dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4989 	mutex_unlock(&hba->dev_cmd.lock);
4993 		dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4998  * ufshcd_setup_links - associate link b/w device wlun and other luns
5000  * @hba: pointer to ufs hba
5010 	if (hba->ufs_device_wlun) {
5011 		link = device_link_add(&sdev->sdev_gendev,
5012 				       &hba->ufs_device_wlun->sdev_gendev,
5015 			dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
5016 				dev_name(&hba->ufs_device_wlun->sdev_gendev));
5019 		hba->luns_avail--;
5021 		if (hba->luns_avail == 1) {
5030 		hba->luns_avail--;
5035  * ufshcd_lu_init - Initialize the relevant parameters of the LU
5036  * @hba: per-adapter instance
5042 	u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun);
5043 	u8 lun_qdepth = hba->nutrs;
5053 		if (ret == -EOPNOTSUPP)
5062 		 * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will
5063 		 * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth
5065 		lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs);
5068 	 * According to UFS device specification, the write protection mode is only supported by
5071 	if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported &&
5072 	    !hba->dev_info.is_lu_power_on_wp &&
5074 		hba->dev_info.is_lu_power_on_wp = true;
5079 		hba->dev_info.b_advanced_rpmb_en = true;
5088 	dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth);
5093  * ufshcd_slave_alloc - handle initial SCSI device configurations
5102 	hba = shost_priv(sdev->host);
5104 	/* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
5105 	sdev->use_10_for_ms = 1;
5108 	sdev->set_dbd_for_ms = 1;
5111 	sdev->allow_restart = 1;
5114 	sdev->no_report_opcodes = 1;
5117 	sdev->no_write_same = 1;
5127  * ufshcd_change_queue_depth - change queue depth
5137 	return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
5141  * ufshcd_slave_configure - adjust SCSI device configurations
5148 	struct ufs_hba *hba = shost_priv(sdev->host);
5149 	struct request_queue *q = sdev->request_queue;
5151 	blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
5152 	if (hba->quirks & UFSHCD_QUIRK_4KB_DMA_ALIGNMENT)
5153 		blk_queue_update_dma_alignment(q, SZ_4K - 1);
5155 	 * Block runtime-pm until all consumers are added.
5159 		pm_runtime_get_noresume(&sdev->sdev_gendev);
5161 		sdev->rpm_autosuspend = 1;
5163 	 * Do not print messages during runtime PM to avoid never-ending cycles
5167 	sdev->silence_suspend = 1;
5175  * ufshcd_slave_destroy - remove SCSI device configurations
5183 	hba = shost_priv(sdev->host);
5186 	if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5187 		spin_lock_irqsave(hba->host->host_lock, flags);
5188 		hba->ufs_device_wlun = NULL;
5189 		spin_unlock_irqrestore(hba->host->host_lock, flags);
5190 	} else if (hba->ufs_device_wlun) {
5193 		/* Ensure UFS Device WLUN exists and does not disappear */
5194 		spin_lock_irqsave(hba->host->host_lock, flags);
5195 		if (hba->ufs_device_wlun) {
5196 			supplier = &hba->ufs_device_wlun->sdev_gendev;
5199 		spin_unlock_irqrestore(hba->host->host_lock, flags);
5207 			device_link_remove(&sdev->sdev_gendev, supplier);
5214  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5247  * ufshcd_transfer_rsp_status - Get overall status of the response
5264 	upiu_flags = lrbp->ucd_rsp_ptr->header.flags;
5265 	resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count);
5267 	 * Test !overflow instead of underflow to support UFS devices that do
5271 		scsi_set_resid(lrbp->cmd, resid);
5276 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5277 		if (lrbp->ucd_rsp_ptr->header.response ||
5278 		    lrbp->ucd_rsp_ptr->header.status)
5284 		hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5285 		switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) {
5291 			scsi_status = lrbp->ucd_rsp_ptr->header.status;
5304 			 * UFS device needs urgent BKOPs.
5306 			if (!hba->pm_op_in_progress &&
5308 			    ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5310 				schedule_work(&hba->eeh_work);
5315 			dev_err(hba->dev,
5319 			dev_err(hba->dev,
5343 		dev_err(hba->dev,
5345 				ocs, lrbp->task_tag);
5352 	    (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5353 		ufshcd_print_tr(hba, lrbp->task_tag, true);
5367 	if (hba->active_uic_cmd &&
5368 	    (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5369 	    hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5376  * ufshcd_uic_cmd_compl - handle completion of uic command
5381  *  IRQ_HANDLED - If interrupt is valid
5382  *  IRQ_NONE    - If invalid interrupt
5388 	spin_lock(hba->host->host_lock);
5390 		hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5392 	if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5393 		hba->active_uic_cmd->argument2 |=
5395 		hba->active_uic_cmd->argument3 =
5397 		if (!hba->uic_async_done)
5398 			hba->active_uic_cmd->cmd_active = 0;
5399 		complete(&hba->active_uic_cmd->done);
5403 	if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5404 		hba->active_uic_cmd->cmd_active = 0;
5405 		complete(hba->uic_async_done);
5410 		ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5412 	spin_unlock(hba->host->host_lock);
5420 	struct scsi_cmnd *cmd = lrbp->cmd;
5428  * ufshcd_compl_one_cqe - handle a completion queue entry
5440 	lrbp = &hba->lrb[task_tag];
5441 	lrbp->compl_time_stamp = ktime_get();
5442 	lrbp->compl_time_stamp_local_clock = local_clock();
5443 	cmd = lrbp->cmd;
5448 		cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
5452 	} else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5453 		   lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
5454 		if (hba->dev_cmd.complete) {
5456 				ocs = le32_to_cpu(cqe->status) & MASK_OCS;
5457 				lrbp->utr_descriptor_ptr->header.ocs = ocs;
5459 			complete(hba->dev_cmd.complete);
5466  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5475 	for_each_set_bit(tag, &completed_reqs, hba->nutrs)
5481 	UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1
5489 	for_each_set_bit(tag, completed_reqs, hba->nutrs) {
5490 		struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
5494 		if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED)
5511 		hwq = &hba->uhq[queue_num];
5516 	spin_lock_irqsave(&hba->outstanding_lock, flags);
5518 	completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5519 	WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5521 		  hba->outstanding_reqs);
5526 	hba->outstanding_reqs &= ~completed_reqs;
5527 	spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5536  * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is
5556 	for (tag = 0; tag < hba->nutrs; tag++) {
5557 		lrbp = &hba->lrb[tag];
5558 		cmd = lrbp->cmd;
5560 		    test_bit(SCMD_STATE_COMPLETE, &cmd->state))
5565 		hwq = &hba->uhq[hwq_num];
5573 			if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
5574 				spin_lock_irqsave(&hwq->cq_lock, flags);
5578 				spin_unlock_irqrestore(&hwq->cq_lock, flags);
5587  * ufshcd_transfer_req_compl - handle SCSI and query command completion
5591  *  IRQ_HANDLED - If interrupt is valid
5592  *  IRQ_NONE    - If invalid interrupt
5604 	    !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5614 	ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
5630 	mutex_lock(&hba->ee_ctrl_mutex);
5631 	err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5632 	mutex_unlock(&hba->ee_ctrl_mutex);
5634 		dev_err(hba->dev, "%s: failed to write ee control %d\n",
5645 	mutex_lock(&hba->ee_ctrl_mutex);
5648 	if (ee_ctrl_mask != hba->ee_ctrl_mask)
5652 		hba->ee_ctrl_mask = ee_ctrl_mask;
5655 	mutex_unlock(&hba->ee_ctrl_mutex);
5660  * ufshcd_disable_ee - disable exception event
5661  * @hba: per-adapter instance
5667  * Return: zero on success, non-zero error value on failure.
5675  * ufshcd_enable_ee - enable exception event
5676  * @hba: per-adapter instance
5682  * Return: zero on success, non-zero error value on failure.
5690  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5691  * @hba: per-adapter instance
5698  * Return: zero on success, non-zero on failure.
5704 	if (hba->auto_bkops_enabled)
5710 		dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5715 	hba->auto_bkops_enabled = true;
5716 	trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5721 		dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5728  * ufshcd_disable_auto_bkops - block device in doing background operations
5729  * @hba: per-adapter instance
5733  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5737  * Return: zero on success, non-zero on failure.
5743 	if (!hba->auto_bkops_enabled)
5752 		dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5760 		dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5766 	hba->auto_bkops_enabled = false;
5767 	trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5768 	hba->is_urgent_bkops_lvl_checked = false;
5774  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5779  * as well. This function would change the auto-bkops state based on
5785 		hba->auto_bkops_enabled = false;
5786 		hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5789 		hba->auto_bkops_enabled = true;
5790 		hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5793 	hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5794 	hba->is_urgent_bkops_lvl_checked = false;
5804  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5805  * @hba: per-adapter instance
5808  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5813  * Return: 0 for success, non-zero in case of failure.
5815  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5827 		dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5831 		dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5833 		err = -EINVAL;
5846  * ufshcd_urgent_bkops - handle urgent bkops exception event
5847  * @hba: per-adapter instance
5859 	return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5873 	if (hba->is_urgent_bkops_lvl_checked)
5878 		dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5890 		dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5893 		hba->urgent_bkops_lvl = curr_status;
5894 		hba->is_urgent_bkops_lvl_checked = true;
5901 		dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5913 	dev_info(hba->dev, "exception Tcase %d\n", value - 80);
5938 	    hba->dev_info.wb_enabled == enable)
5943 		dev_err(hba->dev, "%s: Write Booster %s failed %d\n",
5948 	hba->dev_info.wb_enabled = enable;
5949 	dev_dbg(hba->dev, "%s: Write Booster %s\n",
5963 		dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n",
5967 	dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n",
5976 	    hba->dev_info.wb_buf_flush_enabled == enable)
5981 		dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n",
5986 	hba->dev_info.wb_buf_flush_enabled = enable;
5987 	dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n",
6005 		dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n",
6011 		dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
6016 	return avail_buf < hba->vps->wb_flush_threshold;
6026 	hba->caps &= ~UFSHCD_CAP_WB_EN;
6028 	dev_info(hba->dev, "%s: WB force disabled\n", __func__);
6042 		dev_err(hba->dev,
6049 		dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
6054 	dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
6075 	 * The ufs device needs the vcc to be ON to flush.
6076 	 * With user-space reduction enabled, it's enough to enable flush
6079 	 * With user-space preserved enabled, the current-buffer
6090 		dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n",
6095 	if (!hba->dev_info.b_presrv_uspc_en)
6117  * ufshcd_exception_event_handler - handle exceptions raised by device
6133 		dev_err(hba->dev, "%s: failed to get exception status %d\n",
6138 	trace_ufshcd_exception_event(dev_name(hba->dev), status);
6140 	if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
6143 	if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
6151 /* Complete requests that have door-bell cleared */
6163  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
6165  * @hba: per-adapter instance
6174 	spin_lock_irqsave(hba->host->host_lock, flags);
6179 	if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
6182 	if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
6183 	    ((hba->saved_err & UIC_ERROR) &&
6184 	     (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
6187 	if ((hba->saved_err & UIC_ERROR) &&
6188 	    (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
6193 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6195 		spin_lock_irqsave(hba->host->host_lock, flags);
6201 		if ((hba->saved_err & INT_FATAL_ERRORS) ||
6202 		    ((hba->saved_err & UIC_ERROR) &&
6203 		    (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
6209 		 *   - If we don't get any response then do error recovery.
6210 		 *   - If we get response then clear the DL NAC error bit.
6213 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6215 		spin_lock_irqsave(hba->host->host_lock, flags);
6221 		if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
6222 			hba->saved_err &= ~UIC_ERROR;
6224 		hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6225 		if (!hba->saved_uic_err)
6229 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6236 	return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6237 	       (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6242 	lockdep_assert_held(hba->host->host_lock);
6245 	if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6246 		if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6248 			hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6250 			hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6251 		queue_work(hba->eh_wq, &hba->eh_work);
6257 	spin_lock_irq(hba->host->host_lock);
6258 	hba->force_reset = true;
6260 	spin_unlock_irq(hba->host->host_lock);
6265 	mutex_lock(&hba->wb_mutex);
6266 	down_write(&hba->clk_scaling_lock);
6267 	hba->clk_scaling.is_allowed = allow;
6268 	up_write(&hba->clk_scaling_lock);
6269 	mutex_unlock(&hba->wb_mutex);
6275 		if (hba->clk_scaling.is_enabled)
6280 		if (hba->clk_scaling.is_enabled)
6288 	if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
6289 	    hba->is_sys_suspended) {
6300 		ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6301 		ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6305 		pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6310 		    hba->clk_scaling.is_enabled)
6316 	blk_mq_wait_quiesce_done(&hba->host->tag_set);
6317 	cancel_work_sync(&hba->eeh_work);
6331 	return (!hba->is_powered || hba->shutting_down ||
6332 		!hba->ufs_device_wlun ||
6333 		hba->ufshcd_state == UFSHCD_STATE_ERROR ||
6334 		(!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
6341 	struct Scsi_Host *shost = hba->host;
6346 	hba->is_sys_suspended = false;
6351 	ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
6355 		ret = pm_runtime_set_active(hba->dev);
6364 			q = sdev->request_queue;
6365 			if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6366 				       q->rpm_status == RPM_SUSPENDING))
6367 				pm_request_resume(q->dev);
6379 	struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6384 	if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6387 	if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6396 	u32 tag = rq->tag;
6398 	struct scsi_device *sdev = cmd->device;
6399 	struct Scsi_Host *shost = sdev->host;
6401 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6406 	dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
6407 		hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
6412 		hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
6415 		spin_lock_irqsave(&hwq->cq_lock, flags);
6416 		if (ufshcd_cmd_inflight(lrbp->cmd))
6418 		spin_unlock_irqrestore(&hwq->cq_lock, flags);
6425  * ufshcd_abort_all - Abort all pending commands.
6434 	blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret);
6439 	for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6453  * ufshcd_err_handler - handle UFS errors that require s/w attention
6467 	dev_info(hba->dev,
6469 		 __func__, ufshcd_state_name[hba->ufshcd_state],
6470 		 hba->is_powered, hba->shutting_down, hba->saved_err,
6471 		 hba->saved_uic_err, hba->force_reset,
6474 	down(&hba->host_sem);
6475 	spin_lock_irqsave(hba->host->host_lock, flags);
6477 		if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6478 			hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6479 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6480 		up(&hba->host_sem);
6484 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6486 	/* Complete requests that have door-bell cleared by h/w */
6488 	spin_lock_irqsave(hba->host->host_lock, flags);
6493 	if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6494 		hba->ufshcd_state = UFSHCD_STATE_RESET;
6502 	if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) &&
6503 	    !hba->force_reset) {
6506 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6509 		spin_lock_irqsave(hba->host->host_lock, flags);
6514 	if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6515 	    (hba->saved_uic_err &&
6516 	     (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6517 		bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6519 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6523 		ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6525 		spin_lock_irqsave(hba->host->host_lock, flags);
6533 	if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6535 	    ((hba->saved_err & UIC_ERROR) &&
6536 	     (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6543 	 * If LINERESET was caught, UFS might have been put to PWM mode,
6546 	if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6547 		hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6548 		if (!hba->saved_uic_err)
6549 			hba->saved_err &= ~UIC_ERROR;
6550 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6553 		spin_lock_irqsave(hba->host->host_lock, flags);
6554 		if (!hba->saved_err && !needs_restore)
6558 	hba->silence_err_logs = true;
6560 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6564 	spin_lock_irqsave(hba->host->host_lock, flags);
6565 	hba->silence_err_logs = false;
6574 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6579 		down_write(&hba->clk_scaling_lock);
6580 		hba->force_pmc = true;
6581 		pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6584 			dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6587 		hba->force_pmc = false;
6589 		up_write(&hba->clk_scaling_lock);
6590 		spin_lock_irqsave(hba->host->host_lock, flags);
6598 		hba->force_reset = false;
6599 		spin_unlock_irqrestore(hba->host->host_lock, flags);
6602 			dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6606 		spin_lock_irqsave(hba->host->host_lock, flags);
6611 		if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6612 			hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6613 		if (hba->saved_err || hba->saved_uic_err)
6614 			dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6615 			    __func__, hba->saved_err, hba->saved_uic_err);
6618 	if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6619 	    hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6620 		if (--retries)
6622 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
6625 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6627 	up(&hba->host_sem);
6629 	dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6630 		 ufshcd_state_name[hba->ufshcd_state]);
6634  * ufshcd_update_uic_error - check and set fatal UIC error flags.
6635  * @hba: per-adapter instance
6638  *  IRQ_HANDLED - If interrupt is valid
6639  *  IRQ_NONE    - If invalid interrupt
6656 			dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6663 			hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6664 			if (hba->uic_async_done && hba->active_uic_cmd)
6665 				cmd = hba->active_uic_cmd;
6670 			if (cmd && (cmd->command == UIC_CMD_DME_SET))
6671 				hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6683 			hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6684 		else if (hba->dev_quirks &
6687 				hba->uic_error |=
6690 				hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6700 		hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6708 		hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6716 		hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6720 	dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6721 			__func__, hba->uic_error);
6726  * ufshcd_check_errors - Check for errors that need s/w attention
6727  * @hba: per-adapter instance
6731  *  IRQ_HANDLED - If interrupt is valid
6732  *  IRQ_NONE    - If invalid interrupt
6739 	spin_lock(hba->host->host_lock);
6740 	hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6742 	if (hba->errors & INT_FATAL_ERRORS) {
6744 				       hba->errors);
6748 	if (hba->errors & UIC_ERROR) {
6749 		hba->uic_error = 0;
6751 		if (hba->uic_error)
6755 	if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6756 		dev_err(hba->dev,
6757 			"%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6758 			__func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6760 			hba->errors, ufshcd_get_upmcrs(hba));
6762 				       hba->errors);
6772 		hba->saved_err |= hba->errors;
6773 		hba->saved_uic_err |= hba->uic_error;
6776 		if ((hba->saved_err &
6778 		    (hba->saved_uic_err &&
6779 		     (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6780 			dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6781 					__func__, hba->saved_err,
6782 					hba->saved_uic_err);
6791 	 * if (!queue_eh_work) -
6792 	 * Other errors are either non-fatal where host recovers
6796 	hba->errors = 0;
6797 	hba->uic_error = 0;
6798 	spin_unlock(hba->host->host_lock);
6803  * ufshcd_tmc_handler - handle task management function completion
6807  *  IRQ_HANDLED - If interrupt is valid
6808  *  IRQ_NONE    - If invalid interrupt
6816 	spin_lock_irqsave(hba->host->host_lock, flags);
6818 	issued = hba->outstanding_tasks & ~pending;
6819 	for_each_set_bit(tag, &issued, hba->nutmrs) {
6820 		struct request *req = hba->tmf_rqs[tag];
6821 		struct completion *c = req->end_io_data;
6826 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6832  * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events
6847 		outstanding_cqs = (1U << hba->nr_hw_queues) - 1;
6850 	nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
6852 		hwq = &hba->uhq[i];
6866  * ufshcd_sl_intr - Interrupt service routine
6871  *  IRQ_HANDLED - If interrupt is valid
6872  *  IRQ_NONE    - If invalid interrupt
6881 	if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6897  * ufshcd_intr - Main interrupt service routine
6902  *  IRQ_HANDLED - If interrupt is valid
6903  *  IRQ_NONE    - If invalid interrupt
6910 	int retries = hba->nutrs;
6913 	hba->ufs_stats.last_intr_status = intr_status;
6914 	hba->ufs_stats.last_intr_ts = local_clock();
6917 	 * There could be max of hba->nutrs reqs in flight and in worst case
6922 	while (intr_status && retries--) {
6934 	     hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
6935 		dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6938 					hba->ufs_stats.last_intr_status,
6952 	if (!test_bit(tag, &hba->outstanding_tasks))
6955 	spin_lock_irqsave(hba->host->host_lock, flags);
6957 	spin_unlock_irqrestore(hba->host->host_lock, flags);
6964 	dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
6974 	struct request_queue *q = hba->tmf_queue;
6975 	struct Scsi_Host *host = hba->host;
6988 	req->end_io_data = &wait;
6991 	spin_lock_irqsave(host->host_lock, flags);
6993 	task_tag = req->tag;
6994 	WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n",
6996 	hba->tmf_rqs[req->tag] = req;
6997 	treq->upiu_req.req_header.task_tag = task_tag;
6999 	memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
7003 	__set_bit(task_tag, &hba->outstanding_tasks);
7009 	spin_unlock_irqrestore(host->host_lock, flags);
7018 		dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
7021 			dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
7023 		err = -ETIMEDOUT;
7026 		memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
7031 	spin_lock_irqsave(hba->host->host_lock, flags);
7032 	hba->tmf_rqs[req->tag] = NULL;
7033 	__clear_bit(task_tag, &hba->outstanding_tasks);
7034 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7043  * ufshcd_issue_tm_cmd - issues task management commands to controller
7050  * Return: non-zero value on error, zero on success.
7076 	if (err == -ETIMEDOUT)
7081 		dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
7090  * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
7091  * @hba:	per-adapter instance
7099  * Those type of requests uses UTP Transfer Request Descriptor - utrd.
7104  * the caller is expected to hold the hba->dev_cmd.lock mutex.
7116 	const u32 tag = hba->reserved_slot;
7121 	/* Protects use of hba->reserved_slot. */
7122 	lockdep_assert_held(&hba->dev_cmd.lock);
7124 	down_read(&hba->clk_scaling_lock);
7126 	lrbp = &hba->lrb[tag];
7127 	lrbp->cmd = NULL;
7128 	lrbp->task_tag = tag;
7129 	lrbp->lun = 0;
7130 	lrbp->intr_cmd = true;
7132 	hba->dev_cmd.type = cmd_type;
7134 	if (hba->ufs_version <= ufshci_version(1, 1))
7135 		lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
7137 		lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
7140 	req_upiu->header.task_tag = tag;
7145 	memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7151 		memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
7155 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7157 	hba->dev_cmd.complete = &wait;
7159 	ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
7161 	ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
7163 	 * ignore the returning value here - ufshcd_check_query_response is
7170 	memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7172 		u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
7173 		u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
7180 			dev_warn(hba->dev,
7184 			err = -EINVAL;
7188 				    (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
7190 	up_read(&hba->clk_scaling_lock);
7195  * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
7196  * @hba:	per-adapter instance
7198  * @rsp_upiu:	upiu reply - only 8 DW as we do not support scsi commands
7222 	u8 tm_f = req_upiu->header.tm_function;
7230 		mutex_lock(&hba->dev_cmd.lock);
7234 		mutex_unlock(&hba->dev_cmd.lock);
7245 		if (err == -ETIMEDOUT)
7250 			dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
7259 		err = -EINVAL;
7268  * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request
7278  * Return: zero on success, non-zero on failure.
7286 	const u32 tag = hba->reserved_slot;
7294 	/* Protects use of hba->reserved_slot. */
7296 	mutex_lock(&hba->dev_cmd.lock);
7297 	down_read(&hba->clk_scaling_lock);
7299 	lrbp = &hba->lrb[tag];
7300 	lrbp->cmd = NULL;
7301 	lrbp->task_tag = tag;
7302 	lrbp->lun = UFS_UPIU_RPMB_WLUN;
7304 	lrbp->intr_cmd = true;
7306 	hba->dev_cmd.type = DEV_CMD_TYPE_RPMB;
7308 	/* Advanced RPMB starts from UFS 4.0, so its command type is UTP_CMD_TYPE_UFS_STORAGE */
7309 	lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
7316 	if (hba->capabilities & MASK_EHSLUTRD_SUPPORTED)
7322 	req_upiu->header.task_tag = tag;
7325 	memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7327 	memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs));
7332 	memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7334 	hba->dev_cmd.complete = &wait;
7336 	ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
7342 		memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7344 		result = (lrbp->ucd_rsp_ptr->header.response << 8) |
7345 			lrbp->ucd_rsp_ptr->header.status;
7347 		ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length;
7358 			ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE;
7363 	up_read(&hba->clk_scaling_lock);
7364 	mutex_unlock(&hba->dev_cmd.lock);
7370  * ufshcd_eh_device_reset_handler() - Reset a single logical unit.
7386 	host = cmd->device->host;
7389 	lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
7398 		for (pos = 0; pos < hba->nutrs; pos++) {
7399 			lrbp = &hba->lrb[pos];
7400 			if (ufshcd_cmd_inflight(lrbp->cmd) &&
7401 			    lrbp->lun == lun) {
7403 				hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
7412 	spin_lock_irqsave(&hba->outstanding_lock, flags);
7413 	for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
7414 		if (hba->lrb[pos].lun == lun)
7416 	hba->outstanding_reqs &= ~pending_reqs;
7417 	spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7419 	for_each_set_bit(pos, &pending_reqs, hba->nutrs) {
7421 			spin_lock_irqsave(&hba->outstanding_lock, flags);
7424 			hba->outstanding_reqs |= not_cleared;
7426 			spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7428 			dev_err(hba->dev, "%s: failed to clear request %d\n",
7435 	hba->req_abort_count = 0;
7440 		dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7451 	for_each_set_bit(tag, &bitmap, hba->nutrs) {
7452 		lrbp = &hba->lrb[tag];
7453 		lrbp->req_abort_skip = true;
7458  * ufshcd_try_to_abort_task - abort a specific task
7463  * command, and in host controller by clearing the door-bell register. There can
7468  * Return: zero on success, non-zero on failure.
7472 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7478 	for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7479 		err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7483 			dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7491 			dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7495 				if (ufshcd_cmd_inflight(lrbp->cmd)) {
7501 				dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n",
7514 			dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7518 			dev_err(hba->dev,
7528 		err = -EBUSY;
7532 	err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7537 			dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7545 		dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7553  * ufshcd_abort - scsi host template eh_abort_handler callback
7560 	struct Scsi_Host *host = cmd->device->host;
7562 	int tag = scsi_cmd_to_rq(cmd)->tag;
7563 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7575 		if (!test_bit(tag, &hba->outstanding_reqs)) {
7577 			dev_err(hba->dev,
7579 				__func__, tag, hba->outstanding_reqs, reg);
7585 	dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7595 	if (!hba->req_abort_count) {
7604 	hba->req_abort_count++;
7608 		dev_err(hba->dev,
7616 	 * Task abort to the device W-LUN is illegal. When this command
7620 	 * the lrb taken by this cmd and re-set it in outstanding_reqs,
7623 	if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7624 		ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7626 		spin_lock_irqsave(host->host_lock, flags);
7627 		hba->force_reset = true;
7629 		spin_unlock_irqrestore(host->host_lock, flags);
7640 	if (lrbp->req_abort_skip) {
7641 		dev_err(hba->dev, "%s: skipping abort\n", __func__);
7642 		ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7648 		dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7649 		ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7658 	spin_lock_irqsave(&hba->outstanding_lock, flags);
7659 	outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7660 	spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7674  * ufshcd_host_reset_and_restore - reset and restore host controller
7675  * @hba: per-adapter instance
7678  * local and remote (device) Uni-Pro stack and the attributes
7681  * Return: zero on success, non-zero on failure.
7692 	hba->silence_err_logs = true;
7694 	hba->silence_err_logs = false;
7706 		dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7712  * ufshcd_reset_and_restore - reset and re-initialize host/device
7713  * @hba: per-adapter instance
7715  * Reset and recover device, host and re-establish link. This
7718  * Return: zero on success, non-zero on failure.
7728 	spin_lock_irqsave(hba->host->host_lock, flags);
7734 		saved_err |= hba->saved_err;
7735 		saved_uic_err |= hba->saved_uic_err;
7736 		hba->saved_err = 0;
7737 		hba->saved_uic_err = 0;
7738 		hba->force_reset = false;
7739 		hba->ufshcd_state = UFSHCD_STATE_RESET;
7740 		spin_unlock_irqrestore(hba->host->host_lock, flags);
7747 		spin_lock_irqsave(hba->host->host_lock, flags);
7751 		if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7752 		    hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7753 		    hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7754 			err = -EAGAIN;
7755 	} while (err && --retries);
7758 	 * Inform scsi mid-layer that we did reset and allow to handle
7761 	scsi_report_bus_reset(hba->host, 0);
7763 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
7764 		hba->saved_err |= saved_err;
7765 		hba->saved_uic_err |= saved_uic_err;
7767 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7773  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7784 	hba = shost_priv(cmd->device->host);
7786 	spin_lock_irqsave(hba->host->host_lock, flags);
7787 	hba->force_reset = true;
7789 	dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7790 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7792 	flush_work(&hba->eh_work);
7794 	spin_lock_irqsave(hba->host->host_lock, flags);
7795 	if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7797 	spin_unlock_irqrestore(hba->host->host_lock, flags);
7803  * ufshcd_get_max_icc_level - calculate the ICC level
7818 	for (i = start_scan; i >= 0; i--) {
7849  * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7851  * @hba: per-adapter instance
7861 	if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7862 						!hba->vreg_info.vccq2) {
7865 		 * never-ending cycles of messages written back to storage by
7869 		dev_dbg(hba->dev,
7875 	if (hba->vreg_info.vcc->max_uA)
7877 				hba->vreg_info.vcc->max_uA,
7878 				POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7881 	if (hba->vreg_info.vccq->max_uA)
7883 				hba->vreg_info.vccq->max_uA,
7887 	if (hba->vreg_info.vccq2->max_uA)
7889 				hba->vreg_info.vccq2->max_uA,
7909 		dev_err(hba->dev,
7916 	dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7922 		dev_err(hba->dev,
7933 	blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7934 	if (sdev->rpm_autosuspend)
7935 		pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7941  * ufshcd_scsi_add_wlus - Adds required W-LUs
7942  * @hba: per-adapter instance
7944  * UFS device specification requires the UFS devices to support 4 well known
7947  *	"UFS Device" (address: 50h)
7950  * UFS device's power management needs to be controlled by "POWER CONDITION"
7952  * will take effect only when its sent to "UFS device" well known logical unit
7954  * order for the UFS host driver to send the SSU command for power management.
7963  * Return: zero on success (all required W-LUs are added successfully),
7964  * non-zero error value on failure (if failed to add any of the required W-LU).
7971 	hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
7973 	if (IS_ERR(hba->ufs_device_wlun)) {
7974 		ret = PTR_ERR(hba->ufs_device_wlun);
7975 		hba->ufs_device_wlun = NULL;
7978 	scsi_device_put(hba->ufs_device_wlun);
7980 	sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7989 	sdev_boot = __scsi_add_device(hba->host, 0, 0,
7992 		dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
8000 	scsi_remove_device(hba->ufs_device_wlun);
8007 	struct ufs_dev_info *dev_info = &hba->dev_info;
8016 	 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
8017 	 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
8020 	if (!(dev_info->wspecversion >= 0x310 ||
8021 	      dev_info->wspecversion == 0x220 ||
8022 	     (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
8036 	dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
8038 	dev_info->b_presrv_uspc_en =
8041 	if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
8054 				dev_info->wb_dedicated_lu = lun;
8069 	hba->caps &= ~UFSHCD_CAP_WB_EN;
8074 	struct ufs_dev_info *dev_info = &hba->dev_info;
8078 	if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
8097 	struct ufs_dev_info *dev_info = &hba->dev_info;
8102 	/* Only UFS-4.0 and above may support EXT_IID */
8103 	if (dev_info->wspecversion < 0x400)
8114 		dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err);
8117 	dev_info->b_ext_iid_en = ext_iid_en;
8124 	struct ufs_dev_info *dev_info = &hba->dev_info;
8129 	for (f = fixups; f->quirk; f++) {
8130 		if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
8131 		     f->wmanufacturerid == UFS_ANY_VENDOR) &&
8132 		     ((dev_info->model &&
8133 		       STR_PRFX_EQUAL(f->model, dev_info->model)) ||
8134 		      !strcmp(f->model, UFS_ANY_MODEL)))
8135 			hba->dev_quirks |= f->quirk;
8154 	struct ufs_dev_info *dev_info = &hba->dev_info;
8158 		err = -ENOMEM;
8165 		dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
8174 	dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
8178 	dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
8180 	dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH];
8185 				      &dev_info->model, SD_ASCII_STD);
8187 		dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
8192 	hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
8201 	if (hba->ext_iid_sup)
8217 	struct ufs_dev_info *dev_info = &hba->dev_info;
8219 	kfree(dev_info->model);
8220 	dev_info->model = NULL;
8224  * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
8225  * @hba: per-adapter instance
8228  * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
8232  * Return: zero on success, non-zero error value on failure.
8259  * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
8260  * @hba: per-adapter instance
8263  * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
8264  * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
8267  * Return: zero on success, non-zero error value on failure.
8301  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
8303  * @hba: per-adapter instance
8305  * Some UFS devices require host PA_TACTIVATE to be lower than device
8309  * Return: zero on success, non-zero error value on failure.
8331 		dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
8333 		return -EINVAL;
8338 		dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
8340 		return -EINVAL;
8352 	pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
8354 			     gran_to_us_table[peer_granularity - 1];
8360 				      gran_to_us_table[peer_granularity - 1];
8379 	if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
8383 	if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
8389 	hba->ufs_stats.hibern8_exit_cnt = 0;
8390 	hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
8391 	hba->req_abort_count = 0;
8401 		err = -ENOMEM;
8408 		dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
8414 		hba->dev_info.max_lu_supported = 32;
8416 		hba->dev_info.max_lu_supported = 8;
8454 	hba->dev_ref_clk_freq =
8457 	if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8458 		dev_err(hba->dev,
8466 	u32 freq = hba->dev_ref_clk_freq;
8472 		dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8484 		dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8489 	dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8501 	/* Init UFS geometry descriptor related parameters */
8506 	/* Check and apply UFS device quirks */
8509 		dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8518 		hba->dev_info.f_power_on_wp_en = flag;
8520 	/* Probe maximum power mode co-supported by both UFS host and device */
8522 		dev_err(hba->dev,
8534 	struct ufs_dev_info *dev_info = &hba->dev_info;
8537 	if (dev_info->wspecversion < 0x400)
8542 	mutex_lock(&hba->dev_cmd.lock);
8548 	request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
8550 	upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req;
8552 	put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3);
8557 		dev_err(hba->dev, "%s: failed to set timestamp %d\n",
8560 	mutex_unlock(&hba->dev_cmd.lock);
8565  * ufshcd_add_lus - probe and add UFS logical units
8566  * @hba: per-adapter instance
8579 	/* Initialize devfreq after UFS device is detected */
8581 		memcpy(&hba->clk_scaling.saved_pwr_info,
8582 			&hba->pwr_info,
8584 		hba->clk_scaling.is_allowed = true;
8590 		hba->clk_scaling.is_enabled = true;
8595 	scsi_scan_host(hba->host);
8601 /* SDB - Single Doorbell */
8607 	dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr,
8608 			   hba->ucdl_dma_addr);
8611 	dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
8612 			   hba->utrdl_dma_addr);
8614 	devm_kfree(hba->dev, hba->lrb);
8620 	int old_nutrs = hba->nutrs;
8626 	hba->nutrs = ret;
8635 	if (hba->nutrs != old_nutrs) {
8649 	hba->nutrs = old_nutrs;
8659 	dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : "");
8662 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR)
8666 	ufshcd_mcq_config_mac(hba, hba->nutrs);
8668 	hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
8669 	hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED;
8674 	hba->mcq_enabled = true;
8676 	dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n",
8677 		 hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT],
8678 		 hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL],
8679 		 hba->nutrs);
8685 	struct Scsi_Host *host = hba->host;
8687 	hba->ufshcd_state = UFSHCD_STATE_RESET;
8693 	if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8711 	/* Initiate UFS initialization, and waiting until completion */
8717 	 * Initialize UFS device parameters used by driver, these
8718 	 * parameters are associated with UFS descriptors.
8724 		if (is_mcq_supported(hba) && !hba->scsi_host_added) {
8731 				dev_err(hba->dev, "MCQ mode is disabled, err=%d\n",
8734 			ret = scsi_add_host(host, hba->dev);
8736 				dev_err(hba->dev, "scsi_add_host failed\n");
8739 			hba->scsi_host_added = true;
8748 	/* UFS device is also active now */
8755 	if (hba->max_pwr_info.is_valid) {
8760 		if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8762 		ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8764 			dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8774  * ufshcd_probe_hba - probe hba to detect device and initialize it
8775  * @hba: per-adapter instance
8778  * Execute link-startup and verify device initialization
8792 	if (!hba->pm_op_in_progress &&
8793 	    (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) {
8800 			dev_err(hba->dev, "Host controller enable failed\n");
8815 	 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8816 	 * and for removable UFS card as well, hence always set the parameter.
8822 	/* Enable UFS Write Booster if supported */
8825 	if (hba->ee_usr_mask)
8827 	/* Enable Auto-Hibernate if configured */
8831 	spin_lock_irqsave(hba->host->host_lock, flags);
8833 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
8834 	else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8835 		hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8836 	spin_unlock_irqrestore(hba->host->host_lock, flags);
8838 	trace_ufshcd_init(dev_name(hba->dev), ret,
8840 		hba->curr_dev_pwr_mode, hba->uic_link_state);
8845  * ufshcd_async_scan - asynchronous execution for probing hba
8854 	down(&hba->host_sem);
8855 	/* Initialize hba, detect and initialize UFS device */
8857 	up(&hba->host_sem);
8861 	/* Probe and add UFS logical units  */
8865 	pm_runtime_put_sync(hba->dev);
8868 		dev_err(hba->dev, "%s failed: %d\n", __func__, ret);
8873 	struct ufs_hba *hba = shost_priv(scmd->device->host);
8875 	if (!hba->system_suspending) {
8887 	dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n",
8888 		 __func__, hba->outstanding_tasks);
8890 	return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE;
8924 	.this_id		= -1,
8951 	if (!vreg->max_uA)
8954 	ret = regulator_set_load(vreg->reg, ua);
8957 				__func__, vreg->name, ua, ret);
8966 	return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
8975 	return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
8981 	if (regulator_count_voltages(vreg->reg) <= 0)
8984 	return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
8991 	if (!vreg || vreg->enabled)
8996 		ret = regulator_enable(vreg->reg);
8999 		vreg->enabled = true;
9002 				__func__, vreg->name, ret);
9011 	if (!vreg || !vreg->enabled || vreg->always_on)
9014 	ret = regulator_disable(vreg->reg);
9019 		vreg->enabled = false;
9022 				__func__, vreg->name, ret);
9031 	struct device *dev = hba->dev;
9032 	struct ufs_vreg_info *info = &hba->vreg_info;
9034 	ret = ufshcd_toggle_vreg(dev, info->vcc, on);
9038 	ret = ufshcd_toggle_vreg(dev, info->vccq, on);
9042 	ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
9046 		ufshcd_toggle_vreg(dev, info->vccq2, false);
9047 		ufshcd_toggle_vreg(dev, info->vccq, false);
9048 		ufshcd_toggle_vreg(dev, info->vcc, false);
9055 	struct ufs_vreg_info *info = &hba->vreg_info;
9057 	return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
9067 	vreg->reg = devm_regulator_get(dev, vreg->name);
9068 	if (IS_ERR(vreg->reg)) {
9069 		ret = PTR_ERR(vreg->reg);
9071 				__func__, vreg->name, ret);
9081 	struct device *dev = hba->dev;
9082 	struct ufs_vreg_info *info = &hba->vreg_info;
9084 	ret = ufshcd_get_vreg(dev, info->vcc);
9088 	ret = ufshcd_get_vreg(dev, info->vccq);
9090 		ret = ufshcd_get_vreg(dev, info->vccq2);
9097 	struct ufs_vreg_info *info = &hba->vreg_info;
9099 	return ufshcd_get_vreg(hba->dev, info->vdd_hba);
9106 	struct list_head *head = &hba->clk_list_head;
9119 		if (!IS_ERR_OR_NULL(clki->clk)) {
9125 			    clki->keep_link_active)
9128 			clk_state_changed = on ^ clki->enabled;
9129 			if (on && !clki->enabled) {
9130 				ret = clk_prepare_enable(clki->clk);
9132 					dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
9133 						__func__, clki->name, ret);
9136 			} else if (!on && clki->enabled) {
9137 				clk_disable_unprepare(clki->clk);
9139 			clki->enabled = on;
9140 			dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
9141 					clki->name, on ? "en" : "dis");
9152 			if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
9153 				clk_disable_unprepare(clki->clk);
9156 		spin_lock_irqsave(hba->host->host_lock, flags);
9157 		hba->clk_gating.state = CLKS_ON;
9158 		trace_ufshcd_clk_gating(dev_name(hba->dev),
9159 					hba->clk_gating.state);
9160 		spin_unlock_irqrestore(hba->host->host_lock, flags);
9164 		trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
9173 	int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq);
9176 		dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret);
9187 	struct device *dev = hba->dev;
9188 	struct list_head *head = &hba->clk_list_head;
9194 		if (!clki->name)
9197 		clki->clk = devm_clk_get(dev, clki->name);
9198 		if (IS_ERR(clki->clk)) {
9199 			ret = PTR_ERR(clki->clk);
9201 					__func__, clki->name, ret);
9210 		if (!strcmp(clki->name, "ref_clk"))
9211 			ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
9213 		if (clki->max_freq) {
9214 			ret = clk_set_rate(clki->clk, clki->max_freq);
9216 				dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
9217 					__func__, clki->name,
9218 					clki->max_freq, ret);
9221 			clki->curr_freq = clki->max_freq;
9224 				clki->name, clk_get_rate(clki->clk));
9234 	if (!hba->vops)
9239 		dev_err_probe(hba->dev, err,
9248 	if (!hba->vops)
9259 	 * Handle host controller power separately from the UFS device power
9260 	 * rails as it will help controlling the UFS host controller power
9261 	 * collapse easily which is different than UFS device power collapse.
9277 	if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
9278 		hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba);
9298 	hba->is_powered = true;
9313 	if (hba->is_powered) {
9316 		if (hba->eh_wq)
9317 			destroy_workqueue(hba->eh_wq);
9323 		hba->is_powered = false;
9345  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
9361 	spin_lock_irqsave(hba->host->host_lock, flags);
9362 	sdp = hba->ufs_device_wlun;
9366 		ret = -ENODEV;
9367 	spin_unlock_irqrestore(hba->host->host_lock, flags);
9373 	 * If scsi commands fail, the scsi mid-layer schedules scsi error-
9378 	hba->host->eh_noresume = 1;
9385 	for (retries = 3; retries > 0; --retries) {
9401 			ret = -EIO;
9404 		hba->curr_dev_pwr_mode = pwr_mode;
9408 	hba->host->eh_noresume = 0;
9418 	if (req_link_state == hba->uic_link_state)
9426 			dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9437 		 (!check_for_bkops || !hba->auto_bkops_enabled)) {
9450 			dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9475 	 * It seems some UFS devices may keep drawing more than sleep current
9476 	 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
9477 	 * To avoid this situation, add 2ms delay before putting these UFS
9481 	    hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
9485 	 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
9488 	 * If UFS device and link is in OFF state, all power supplies (VCC,
9490 	 * required. If UFS link is inactive (Hibern8 or OFF state) and device
9500 	    !hba->dev_info.is_lu_power_on_wp) {
9504 		ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9507 			ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9508 			ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
9513 	 * Some UFS devices require delay after VCC power rail is turned-off.
9515 	if (vcc_off && hba->vreg_info.vcc &&
9516 		hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
9526 	    !hba->dev_info.is_lu_power_on_wp) {
9530 			ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
9533 			ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
9537 		ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
9542 	ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9544 	ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9570 	hba->pm_op_in_progress = true;
9573 			 hba->rpm_lvl : hba->spm_lvl;
9586 	hba->clk_gating.is_suspended = true;
9596 	if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
9597 	    (req_link_state == hba->uic_link_state))
9600 	/* UFS device & link must be active before we enter in this function */
9605 		ret = -EBUSY;
9624 				ret = -EBUSY;
9636 		hba->dev_info.b_rpm_dev_flush_capable =
9637 			hba->auto_bkops_enabled ||
9644 	flush_work(&hba->eeh_work);
9650 	if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
9655 		if (!hba->dev_info.b_rpm_dev_flush_capable) {
9664 				ret = -EBUSY;
9684 		ret = -EBUSY;
9726 	hba->dev_info.b_rpm_dev_flush_capable = false;
9728 	if (hba->dev_info.b_rpm_dev_flush_capable) {
9729 		schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9735 		hba->clk_gating.is_suspended = false;
9738 	hba->pm_op_in_progress = false;
9746 	enum uic_link_state old_link_state = hba->uic_link_state;
9748 	hba->pm_op_in_progress = true;
9767 			dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9799 		 * keep auto-bkops enabled or else disable it.
9803 	if (hba->ee_usr_mask)
9809 	if (hba->dev_info.b_rpm_dev_flush_capable) {
9810 		hba->dev_info.b_rpm_dev_flush_capable = false;
9811 		cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9814 	/* Enable Auto-Hibernate if configured */
9827 	hba->clk_gating.is_suspended = false;
9829 	hba->pm_op_in_progress = false;
9840 	hba = shost_priv(sdev->host);
9844 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9848 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9860 	hba = shost_priv(sdev->host);
9864 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9868 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9882 	hba = shost_priv(sdev->host);
9883 	down(&hba->host_sem);
9884 	hba->system_suspending = true;
9891 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__,  ret);
9892 		up(&hba->host_sem);
9897 		hba->is_sys_suspended = true;
9900 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9912 	hba = shost_priv(sdev->host);
9919 		dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9923 		hba->curr_dev_pwr_mode, hba->uic_link_state);
9925 		hba->is_sys_suspended = false;
9926 	hba->system_suspending = false;
9927 	up(&hba->host_sem);
9933  * ufshcd_suspend - helper function for suspend operations
9937  * and set vreg and hba-vreg in lpm mode.
9945 	if (!hba->is_powered)
9958 		hba->clk_gating.state = CLKS_OFF;
9959 		trace_ufshcd_clk_gating(dev_name(hba->dev),
9960 					hba->clk_gating.state);
9971  * ufshcd_resume - helper function for resume operations
9977  * Return: 0 for success and non-zero for failure.
9983 	if (!hba->is_powered)
10012  * ufshcd_system_suspend - system suspend callback
10013  * @dev: Device associated with the UFS controller.
10018  * Return: 0 for success and non-zero for failure.
10026 	if (pm_runtime_suspended(hba->dev))
10031 	trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
10033 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10039  * ufshcd_system_resume - system resume callback
10040  * @dev: Device associated with the UFS controller.
10045  * Return: 0 for success and non-zero for failure.
10053 	if (pm_runtime_suspended(hba->dev))
10059 	trace_ufshcd_system_resume(dev_name(hba->dev), ret,
10061 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10070  * ufshcd_runtime_suspend - runtime suspend callback
10071  * @dev: Device associated with the UFS controller.
10075  * Return: 0 for success and non-zero for failure.
10085 	trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
10087 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10093  * ufshcd_runtime_resume - runtime resume routine
10094  * @dev: Device associated with the UFS controller.
10112 	trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
10114 		hba->curr_dev_pwr_mode, hba->uic_link_state);
10123 	struct ufs_hba *hba = shost_priv(sdev->host);
10125 	down(&hba->host_sem);
10126 	hba->shutting_down = true;
10127 	up(&hba->host_sem);
10132 	shost_for_each_device(sdev, hba->host) {
10133 		if (sdev == hba->ufs_device_wlun)
10135 		mutex_lock(&sdev->state_mutex);
10137 		mutex_unlock(&sdev->state_mutex);
10142 	 * Next, turn off the UFS controller and the UFS regulators. Disable
10148 	hba->is_powered = false;
10152  * ufshcd_remove - de-allocate SCSI host and host memory space
10158 	if (hba->ufs_device_wlun)
10162 	ufs_sysfs_remove_nodes(hba->dev);
10163 	blk_mq_destroy_queue(hba->tmf_queue);
10164 	blk_put_queue(hba->tmf_queue);
10165 	blk_mq_free_tag_set(&hba->tmf_tag_set);
10166 	if (hba->scsi_host_added)
10167 		scsi_remove_host(hba->host);
10169 	ufshcd_disable_intr(hba, hba->intr_mask);
10195 	ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
10197 	ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
10199 	ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
10201 	ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
10223  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
10228 	scsi_host_put(hba->host);
10233  * ufshcd_set_dma_mask - Set dma mask based on the controller
10237  * Return: 0 for success, non-zero for failure.
10241 	if (hba->vops && hba->vops->set_dma_mask)
10242 		return hba->vops->set_dma_mask(hba);
10243 	if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
10244 		if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
10247 	return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
10251  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
10255  * Return: 0 on success, non-zero value on failure.
10266 		err = -ENODEV;
10274 		err = -ENOMEM;
10277 	host->nr_maps = HCTX_TYPE_POLL + 1;
10279 	hba->host = host;
10280 	hba->dev = dev;
10281 	hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
10282 	hba->nop_out_timeout = NOP_OUT_TIMEOUT;
10284 	INIT_LIST_HEAD(&hba->clk_list_head);
10285 	spin_lock_init(&hba->outstanding_lock);
10307  * ufshcd_init - Driver initialization routine
10308  * @hba: per-adapter instance
10312  * Return: 0 on success, non-zero value on failure.
10317 	struct Scsi_Host *host = hba->host;
10318 	struct device *dev = hba->dev;
10329 		dev_err(hba->dev,
10331 		err = -ENODEV;
10335 	hba->mmio_base = mmio_base;
10336 	hba->irq = irq;
10337 	hba->vps = &ufs_hba_vps;
10348 	/* Get UFS version supported by the controller */
10349 	hba->ufs_version = ufshcd_get_ufs_version(hba);
10352 	hba->intr_mask = ufshcd_get_intr_mask(hba);
10356 		dev_err(hba->dev, "set dma mask failed\n");
10363 		dev_err(hba->dev, "Memory allocation failed\n");
10370 	host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
10371 	host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
10372 	host->max_id = UFSHCD_MAX_ID;
10373 	host->max_lun = UFS_MAX_LUNS;
10374 	host->max_channel = UFSHCD_MAX_CHANNEL;
10375 	host->unique_id = host->host_no;
10376 	host->max_cmd_len = UFS_CDB_SIZE;
10377 	host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING);
10379 	hba->max_pwr_info.is_valid = false;
10383 		 hba->host->host_no);
10384 	hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
10385 	if (!hba->eh_wq) {
10386 		dev_err(hba->dev, "%s: failed to create eh workqueue\n",
10388 		err = -ENOMEM;
10391 	INIT_WORK(&hba->eh_work, ufshcd_err_handler);
10392 	INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
10394 	sema_init(&hba->host_sem, 1);
10397 	mutex_init(&hba->uic_cmd_mutex);
10400 	mutex_init(&hba->dev_cmd.lock);
10403 	mutex_init(&hba->ee_ctrl_mutex);
10405 	mutex_init(&hba->wb_mutex);
10406 	init_rwsem(&hba->clk_scaling_lock);
10414 	 * registering UFS controller interrupt handler, clear any pending UFS
10415 	 * interrupt status and disable all the UFS interrupts.
10421 	 * Make sure that UFS interrupts are disabled and any pending interrupt
10422 	 * status is cleared before registering UFS interrupt handler.
10429 		dev_err(hba->dev, "request irq failed\n");
10432 		hba->is_irq_enabled = true;
10436 		if (!hba->lsdb_sup) {
10437 			dev_err(hba->dev, "%s: failed to initialize (legacy doorbell mode not supported)\n",
10439 			err = -EINVAL;
10442 		err = scsi_add_host(host, hba->dev);
10444 			dev_err(hba->dev, "scsi_add_host failed\n");
10447 		hba->scsi_host_added = true;
10450 	hba->tmf_tag_set = (struct blk_mq_tag_set) {
10452 		.queue_depth	= hba->nutmrs,
10456 	err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
10459 	hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
10460 	if (IS_ERR(hba->tmf_queue)) {
10461 		err = PTR_ERR(hba->tmf_queue);
10464 	hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
10465 				    sizeof(*hba->tmf_rqs), GFP_KERNEL);
10466 	if (!hba->tmf_rqs) {
10467 		err = -ENOMEM;
10479 		dev_err(hba->dev, "Host controller enable failed\n");
10488 	 * Default power saving mode is to keep UFS link in Hibern8 state
10489 	 * and UFS device in sleep state.
10491 	if (!hba->rpm_lvl)
10492 		hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10495 	if (!hba->spm_lvl)
10496 		hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10500 	INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
10503 	/* Set the default auto-hiberate idle timer value to 150 ms */
10504 	if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
10505 		hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
10511 	atomic_set(&hba->scsi_block_reqs_cnt, 0);
10513 	 * We are assuming that device wasn't put in sleep/power-down
10521 	ufs_sysfs_add_nodes(hba->dev);
10527 	blk_mq_destroy_queue(hba->tmf_queue);
10528 	blk_put_queue(hba->tmf_queue);
10530 	blk_mq_free_tag_set(&hba->tmf_tag_set);
10532 	if (hba->scsi_host_added)
10533 		scsi_remove_host(hba->host);
10535 	hba->is_irq_enabled = false;
10546 	if (hba->complete_put) {
10548 		hba->complete_put = false;
10555 	struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
10561 	spin_lock_irqsave(&dev->power.lock, flags);
10562 	dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
10563 	link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
10565 	      hba->curr_dev_pwr_mode == dev_pwr_mode &&
10566 	      hba->uic_link_state == link_state &&
10567 	      !hba->dev_info.b_rpm_dev_flush_capable;
10568 	spin_unlock_irqrestore(&dev->power.lock, flags);
10579 	 * SCSI assumes that runtime-pm and system-pm for scsi drivers
10580 	 * are same. And it doesn't wake up the device for system-suspend
10581 	 * if it's runtime suspended. But ufs doesn't follow that.
10584 	if (hba->ufs_device_wlun) {
10594 			if (ret < 0 && ret != -EACCES) {
10599 		hba->complete_put = true;
10615 	struct ufs_hba *hba = shost_priv(sdev->host);
10627 		return -ENODEV;
10629 	blk_pm_runtime_init(sdev->request_queue, dev);
10657 	 * gcc compilers before version 10 cannot do constant-folding for
10658 	 * sub-byte bitfields. Hence skip the layout checks for gcc 9 and
10702  * ufs_dev_wlun_template - describes ufs device wlun
10703  * ufs-device wlun - used to send pm commands
10704  * All luns are consumers of ufs-device wlun.
10708  * With ufs design, SSU should be sent to ufs-device wlun.
10709  * Hence register a scsi driver for ufs wluns only.
10747 MODULE_DESCRIPTION("Generic UFS host controller driver Core");