xref: /openbmc/linux/drivers/ufs/core/ufs-mcq.c (revision d088d6b6)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved.
4  *
5  * Authors:
6  *	Asutosh Das <quic_asutoshd@quicinc.com>
7  *	Can Guo <quic_cang@quicinc.com>
8  */
9 
10 #include <asm/unaligned.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/module.h>
13 #include <linux/platform_device.h>
14 #include "ufshcd-priv.h"
15 #include <linux/delay.h>
16 #include <scsi/scsi_cmnd.h>
17 #include <linux/bitfield.h>
18 #include <linux/iopoll.h>
19 
20 #define MAX_QUEUE_SUP GENMASK(7, 0)
21 #define UFS_MCQ_MIN_RW_QUEUES 2
22 #define UFS_MCQ_MIN_READ_QUEUES 0
23 #define UFS_MCQ_MIN_POLL_QUEUES 0
24 #define QUEUE_EN_OFFSET 31
25 #define QUEUE_ID_OFFSET 16
26 
27 #define MCQ_CFG_MAC_MASK	GENMASK(16, 8)
28 #define MCQ_QCFG_SIZE		0x40
29 #define MCQ_ENTRY_SIZE_IN_DWORD	8
30 #define CQE_UCD_BA GENMASK_ULL(63, 7)
31 
32 /* Max mcq register polling time in microseconds */
33 #define MCQ_POLL_US 500000
34 
35 static int rw_queue_count_set(const char *val, const struct kernel_param *kp)
36 {
37 	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES,
38 				     num_possible_cpus());
39 }
40 
41 static const struct kernel_param_ops rw_queue_count_ops = {
42 	.set = rw_queue_count_set,
43 	.get = param_get_uint,
44 };
45 
46 static unsigned int rw_queues;
47 module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644);
48 MODULE_PARM_DESC(rw_queues,
49 		 "Number of interrupt driven I/O queues used for rw. Default value is nr_cpus");
50 
51 static int read_queue_count_set(const char *val, const struct kernel_param *kp)
52 {
53 	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES,
54 				     num_possible_cpus());
55 }
56 
57 static const struct kernel_param_ops read_queue_count_ops = {
58 	.set = read_queue_count_set,
59 	.get = param_get_uint,
60 };
61 
62 static unsigned int read_queues;
63 module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644);
64 MODULE_PARM_DESC(read_queues,
65 		 "Number of interrupt driven read queues used for read. Default value is 0");
66 
67 static int poll_queue_count_set(const char *val, const struct kernel_param *kp)
68 {
69 	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES,
70 				     num_possible_cpus());
71 }
72 
73 static const struct kernel_param_ops poll_queue_count_ops = {
74 	.set = poll_queue_count_set,
75 	.get = param_get_uint,
76 };
77 
78 static unsigned int poll_queues = 1;
79 module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644);
80 MODULE_PARM_DESC(poll_queues,
81 		 "Number of poll queues used for r/w. Default value is 1");
82 
83 /**
84  * ufshcd_mcq_config_mac - Set the #Max Activ Cmds.
85  * @hba: per adapter instance
86  * @max_active_cmds: maximum # of active commands to the device at any time.
87  *
88  * The controller won't send more than the max_active_cmds to the device at
89  * any time.
90  */
91 void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds)
92 {
93 	u32 val;
94 
95 	val = ufshcd_readl(hba, REG_UFS_MCQ_CFG);
96 	val &= ~MCQ_CFG_MAC_MASK;
97 	val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds);
98 	ufshcd_writel(hba, val, REG_UFS_MCQ_CFG);
99 }
100 
101 /**
102  * ufshcd_mcq_req_to_hwq - find the hardware queue on which the
103  * request would be issued.
104  * @hba: per adapter instance
105  * @req: pointer to the request to be issued
106  *
107  * Returns the hardware queue instance on which the request would
108  * be queued.
109  */
110 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
111 					 struct request *req)
112 {
113 	u32 utag = blk_mq_unique_tag(req);
114 	u32 hwq = blk_mq_unique_tag_to_hwq(utag);
115 
116 	return &hba->uhq[hwq];
117 }
118 
119 /**
120  * ufshcd_mcq_decide_queue_depth - decide the queue depth
121  * @hba: per adapter instance
122  *
123  * Returns queue-depth on success, non-zero on error
124  *
125  * MAC - Max. Active Command of the Host Controller (HC)
126  * HC wouldn't send more than this commands to the device.
127  * It is mandatory to implement get_hba_mac() to enable MCQ mode.
128  * Calculates and adjusts the queue depth based on the depth
129  * supported by the HC and ufs device.
130  */
131 int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)
132 {
133 	int mac;
134 
135 	/* Mandatory to implement get_hba_mac() */
136 	mac = ufshcd_mcq_vops_get_hba_mac(hba);
137 	if (mac < 0) {
138 		dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);
139 		return mac;
140 	}
141 
142 	WARN_ON_ONCE(!hba->dev_info.bqueuedepth);
143 	/*
144 	 * max. value of bqueuedepth = 256, mac is host dependent.
145 	 * It is mandatory for UFS device to define bQueueDepth if
146 	 * shared queuing architecture is enabled.
147 	 */
148 	return min_t(int, mac, hba->dev_info.bqueuedepth);
149 }
150 
151 static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
152 {
153 	int i;
154 	u32 hba_maxq, rem, tot_queues;
155 	struct Scsi_Host *host = hba->host;
156 
157 	/* maxq is 0 based value */
158 	hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1;
159 
160 	tot_queues = read_queues + poll_queues + rw_queues;
161 
162 	if (hba_maxq < tot_queues) {
163 		dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n",
164 			tot_queues, hba_maxq);
165 		return -EOPNOTSUPP;
166 	}
167 
168 	rem = hba_maxq;
169 
170 	if (rw_queues) {
171 		hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues;
172 		rem -= hba->nr_queues[HCTX_TYPE_DEFAULT];
173 	} else {
174 		rw_queues = num_possible_cpus();
175 	}
176 
177 	if (poll_queues) {
178 		hba->nr_queues[HCTX_TYPE_POLL] = poll_queues;
179 		rem -= hba->nr_queues[HCTX_TYPE_POLL];
180 	}
181 
182 	if (read_queues) {
183 		hba->nr_queues[HCTX_TYPE_READ] = read_queues;
184 		rem -= hba->nr_queues[HCTX_TYPE_READ];
185 	}
186 
187 	if (!hba->nr_queues[HCTX_TYPE_DEFAULT])
188 		hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues,
189 							 num_possible_cpus());
190 
191 	for (i = 0; i < HCTX_MAX_TYPES; i++)
192 		host->nr_hw_queues += hba->nr_queues[i];
193 
194 	hba->nr_hw_queues = host->nr_hw_queues;
195 	return 0;
196 }
197 
198 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba)
199 {
200 	struct ufs_hw_queue *hwq;
201 	size_t utrdl_size, cqe_size;
202 	int i;
203 
204 	for (i = 0; i < hba->nr_hw_queues; i++) {
205 		hwq = &hba->uhq[i];
206 
207 		utrdl_size = sizeof(struct utp_transfer_req_desc) *
208 			     hwq->max_entries;
209 		hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size,
210 							 &hwq->sqe_dma_addr,
211 							 GFP_KERNEL);
212 		if (!hwq->sqe_dma_addr) {
213 			dev_err(hba->dev, "SQE allocation failed\n");
214 			return -ENOMEM;
215 		}
216 
217 		cqe_size = sizeof(struct cq_entry) * hwq->max_entries;
218 		hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size,
219 							 &hwq->cqe_dma_addr,
220 							 GFP_KERNEL);
221 		if (!hwq->cqe_dma_addr) {
222 			dev_err(hba->dev, "CQE allocation failed\n");
223 			return -ENOMEM;
224 		}
225 	}
226 
227 	return 0;
228 }
229 
230 
231 /* Operation and runtime registers configuration */
232 #define MCQ_CFG_n(r, i)	((r) + MCQ_QCFG_SIZE * (i))
233 #define MCQ_OPR_OFFSET_n(p, i) \
234 	(hba->mcq_opr[(p)].offset + hba->mcq_opr[(p)].stride * (i))
235 
236 static void __iomem *mcq_opr_base(struct ufs_hba *hba,
237 					 enum ufshcd_mcq_opr n, int i)
238 {
239 	struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n];
240 
241 	return opr->base + opr->stride * i;
242 }
243 
244 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i)
245 {
246 	return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);
247 }
248 
249 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i)
250 {
251 	writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);
252 }
253 EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis);
254 
255 /*
256  * Current MCQ specification doesn't provide a Task Tag or its equivalent in
257  * the Completion Queue Entry. Find the Task Tag using an indirect method.
258  */
259 static int ufshcd_mcq_get_tag(struct ufs_hba *hba,
260 				     struct ufs_hw_queue *hwq,
261 				     struct cq_entry *cqe)
262 {
263 	u64 addr;
264 
265 	/* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */
266 	BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0));
267 
268 	/* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */
269 	addr = (le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA) -
270 		hba->ucdl_dma_addr;
271 
272 	return div_u64(addr, ufshcd_get_ucd_size(hba));
273 }
274 
275 static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,
276 				   struct ufs_hw_queue *hwq)
277 {
278 	struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq);
279 	int tag = ufshcd_mcq_get_tag(hba, hwq, cqe);
280 
281 	if (cqe->command_desc_base_addr) {
282 		ufshcd_compl_one_cqe(hba, tag, cqe);
283 		/* After processed the cqe, mark it empty (invalid) entry */
284 		cqe->command_desc_base_addr = 0;
285 	}
286 }
287 
288 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba,
289 				    struct ufs_hw_queue *hwq)
290 {
291 	unsigned long flags;
292 	u32 entries = hwq->max_entries;
293 
294 	spin_lock_irqsave(&hwq->cq_lock, flags);
295 	while (entries > 0) {
296 		ufshcd_mcq_process_cqe(hba, hwq);
297 		ufshcd_mcq_inc_cq_head_slot(hwq);
298 		entries--;
299 	}
300 
301 	ufshcd_mcq_update_cq_tail_slot(hwq);
302 	hwq->cq_head_slot = hwq->cq_tail_slot;
303 	spin_unlock_irqrestore(&hwq->cq_lock, flags);
304 }
305 
306 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
307 				       struct ufs_hw_queue *hwq)
308 {
309 	unsigned long completed_reqs = 0;
310 	unsigned long flags;
311 
312 	spin_lock_irqsave(&hwq->cq_lock, flags);
313 	ufshcd_mcq_update_cq_tail_slot(hwq);
314 	while (!ufshcd_mcq_is_cq_empty(hwq)) {
315 		ufshcd_mcq_process_cqe(hba, hwq);
316 		ufshcd_mcq_inc_cq_head_slot(hwq);
317 		completed_reqs++;
318 	}
319 
320 	if (completed_reqs)
321 		ufshcd_mcq_update_cq_head(hwq);
322 	spin_unlock_irqrestore(&hwq->cq_lock, flags);
323 
324 	return completed_reqs;
325 }
326 EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock);
327 
328 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)
329 {
330 	struct ufs_hw_queue *hwq;
331 	u16 qsize;
332 	int i;
333 
334 	for (i = 0; i < hba->nr_hw_queues; i++) {
335 		hwq = &hba->uhq[i];
336 		hwq->id = i;
337 		qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1;
338 
339 		/* Submission Queue Lower Base Address */
340 		ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr),
341 			      MCQ_CFG_n(REG_SQLBA, i));
342 		/* Submission Queue Upper Base Address */
343 		ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr),
344 			      MCQ_CFG_n(REG_SQUBA, i));
345 		/* Submission Queue Doorbell Address Offset */
346 		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQD, i),
347 			      MCQ_CFG_n(REG_SQDAO, i));
348 		/* Submission Queue Interrupt Status Address Offset */
349 		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQIS, i),
350 			      MCQ_CFG_n(REG_SQISAO, i));
351 
352 		/* Completion Queue Lower Base Address */
353 		ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr),
354 			      MCQ_CFG_n(REG_CQLBA, i));
355 		/* Completion Queue Upper Base Address */
356 		ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr),
357 			      MCQ_CFG_n(REG_CQUBA, i));
358 		/* Completion Queue Doorbell Address Offset */
359 		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQD, i),
360 			      MCQ_CFG_n(REG_CQDAO, i));
361 		/* Completion Queue Interrupt Status Address Offset */
362 		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQIS, i),
363 			      MCQ_CFG_n(REG_CQISAO, i));
364 
365 		/* Save the base addresses for quicker access */
366 		hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP;
367 		hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP;
368 		hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP;
369 		hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP;
370 
371 		/* Reinitializing is needed upon HC reset */
372 		hwq->sq_tail_slot = hwq->cq_tail_slot = hwq->cq_head_slot = 0;
373 
374 		/* Enable Tail Entry Push Status interrupt only for non-poll queues */
375 		if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL])
376 			writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE);
377 
378 		/* Completion Queue Enable|Size to Completion Queue Attribute */
379 		ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize,
380 			      MCQ_CFG_n(REG_CQATTR, i));
381 
382 		/*
383 		 * Submission Qeueue Enable|Size|Completion Queue ID to
384 		 * Submission Queue Attribute
385 		 */
386 		ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize |
387 			      (i << QUEUE_ID_OFFSET),
388 			      MCQ_CFG_n(REG_SQATTR, i));
389 	}
390 }
391 
392 void ufshcd_mcq_enable_esi(struct ufs_hba *hba)
393 {
394 	ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x2,
395 		      REG_UFS_MEM_CFG);
396 }
397 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi);
398 
399 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg)
400 {
401 	ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA);
402 	ufshcd_writel(hba, msg->address_hi, REG_UFS_ESIUBA);
403 }
404 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_esi);
405 
406 int ufshcd_mcq_init(struct ufs_hba *hba)
407 {
408 	struct Scsi_Host *host = hba->host;
409 	struct ufs_hw_queue *hwq;
410 	int ret, i;
411 
412 	ret = ufshcd_mcq_config_nr_queues(hba);
413 	if (ret)
414 		return ret;
415 
416 	ret = ufshcd_vops_mcq_config_resource(hba);
417 	if (ret)
418 		return ret;
419 
420 	ret = ufshcd_mcq_vops_op_runtime_config(hba);
421 	if (ret) {
422 		dev_err(hba->dev, "Operation runtime config failed, ret=%d\n",
423 			ret);
424 		return ret;
425 	}
426 	hba->uhq = devm_kzalloc(hba->dev,
427 				hba->nr_hw_queues * sizeof(struct ufs_hw_queue),
428 				GFP_KERNEL);
429 	if (!hba->uhq) {
430 		dev_err(hba->dev, "ufs hw queue memory allocation failed\n");
431 		return -ENOMEM;
432 	}
433 
434 	for (i = 0; i < hba->nr_hw_queues; i++) {
435 		hwq = &hba->uhq[i];
436 		hwq->max_entries = hba->nutrs;
437 		spin_lock_init(&hwq->sq_lock);
438 		spin_lock_init(&hwq->cq_lock);
439 		mutex_init(&hwq->sq_mutex);
440 	}
441 
442 	/* The very first HW queue serves device commands */
443 	hba->dev_cmd_queue = &hba->uhq[0];
444 
445 	host->host_tagset = 1;
446 	return 0;
447 }
448 
449 static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
450 {
451 	void __iomem *reg;
452 	u32 id = hwq->id, val;
453 	int err;
454 
455 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
456 		return -ETIMEDOUT;
457 
458 	writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
459 	reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
460 	err = read_poll_timeout(readl, val, val & SQ_STS, 20,
461 				MCQ_POLL_US, false, reg);
462 	if (err)
463 		dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
464 			__func__, id, err);
465 	return err;
466 }
467 
468 static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
469 {
470 	void __iomem *reg;
471 	u32 id = hwq->id, val;
472 	int err;
473 
474 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
475 		return -ETIMEDOUT;
476 
477 	writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
478 	reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
479 	err = read_poll_timeout(readl, val, !(val & SQ_STS), 20,
480 				MCQ_POLL_US, false, reg);
481 	if (err)
482 		dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
483 			__func__, id, err);
484 	return err;
485 }
486 
487 /**
488  * ufshcd_mcq_sq_cleanup - Clean up submission queue resources
489  * associated with the pending command.
490  * @hba - per adapter instance.
491  * @task_tag - The command's task tag.
492  *
493  * Returns 0 for success; error code otherwise.
494  */
495 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
496 {
497 	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
498 	struct scsi_cmnd *cmd = lrbp->cmd;
499 	struct ufs_hw_queue *hwq;
500 	void __iomem *reg, *opr_sqd_base;
501 	u32 nexus, id, val;
502 	int err;
503 
504 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
505 		return -ETIMEDOUT;
506 
507 	if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) {
508 		if (!cmd)
509 			return -EINVAL;
510 		hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
511 	} else {
512 		hwq = hba->dev_cmd_queue;
513 	}
514 
515 	id = hwq->id;
516 
517 	mutex_lock(&hwq->sq_mutex);
518 
519 	/* stop the SQ fetching before working on it */
520 	err = ufshcd_mcq_sq_stop(hba, hwq);
521 	if (err)
522 		goto unlock;
523 
524 	/* SQCTI = EXT_IID, IID, LUN, Task Tag */
525 	nexus = lrbp->lun << 8 | task_tag;
526 	opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id);
527 	writel(nexus, opr_sqd_base + REG_SQCTI);
528 
529 	/* SQRTCy.ICU = 1 */
530 	writel(SQ_ICU, opr_sqd_base + REG_SQRTC);
531 
532 	/* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */
533 	reg = opr_sqd_base + REG_SQRTS;
534 	err = read_poll_timeout(readl, val, val & SQ_CUS, 20,
535 				MCQ_POLL_US, false, reg);
536 	if (err)
537 		dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%ld\n",
538 			__func__, id, task_tag,
539 			FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg)));
540 
541 	if (ufshcd_mcq_sq_start(hba, hwq))
542 		err = -ETIMEDOUT;
543 
544 unlock:
545 	mutex_unlock(&hwq->sq_mutex);
546 	return err;
547 }
548 
549 /**
550  * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry.
551  * Write the sqe's Command Type to 0xF. The host controller will not
552  * fetch any sqe with Command Type = 0xF.
553  *
554  * @utrd - UTP Transfer Request Descriptor to be nullified.
555  */
556 static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
557 {
558 	u32 dword_0;
559 
560 	dword_0 = le32_to_cpu(utrd->header.dword_0);
561 	dword_0 &= ~UPIU_COMMAND_TYPE_MASK;
562 	dword_0 |= FIELD_PREP(UPIU_COMMAND_TYPE_MASK, 0xF);
563 	utrd->header.dword_0 = cpu_to_le32(dword_0);
564 }
565 
566 /**
567  * ufshcd_mcq_sqe_search - Search for the command in the submission queue
568  * If the command is in the submission queue and not issued to the device yet,
569  * nullify the sqe so the host controller will skip fetching the sqe.
570  *
571  * @hba - per adapter instance.
572  * @hwq - Hardware Queue to be searched.
573  * @task_tag - The command's task tag.
574  *
575  * Returns true if the SQE containing the command is present in the SQ
576  * (not fetched by the controller); returns false if the SQE is not in the SQ.
577  */
578 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
579 				  struct ufs_hw_queue *hwq, int task_tag)
580 {
581 	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
582 	struct utp_transfer_req_desc *utrd;
583 	u32 mask = hwq->max_entries - 1;
584 	__le64  cmd_desc_base_addr;
585 	bool ret = false;
586 	u64 addr, match;
587 	u32 sq_head_slot;
588 
589 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
590 		return true;
591 
592 	mutex_lock(&hwq->sq_mutex);
593 
594 	ufshcd_mcq_sq_stop(hba, hwq);
595 	sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq);
596 	if (sq_head_slot == hwq->sq_tail_slot)
597 		goto out;
598 
599 	cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr;
600 	addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA;
601 
602 	while (sq_head_slot != hwq->sq_tail_slot) {
603 		utrd = hwq->sqe_base_addr +
604 				sq_head_slot * sizeof(struct utp_transfer_req_desc);
605 		match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA;
606 		if (addr == match) {
607 			ufshcd_mcq_nullify_sqe(utrd);
608 			ret = true;
609 			goto out;
610 		}
611 		sq_head_slot = (sq_head_slot + 1) & mask;
612 	}
613 
614 out:
615 	ufshcd_mcq_sq_start(hba, hwq);
616 	mutex_unlock(&hwq->sq_mutex);
617 	return ret;
618 }
619 
620 /**
621  * ufshcd_mcq_abort - Abort the command in MCQ.
622  * @cmd - The command to be aborted.
623  *
624  * Returns SUCCESS or FAILED error codes
625  */
626 int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
627 {
628 	struct Scsi_Host *host = cmd->device->host;
629 	struct ufs_hba *hba = shost_priv(host);
630 	int tag = scsi_cmd_to_rq(cmd)->tag;
631 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];
632 	struct ufs_hw_queue *hwq;
633 	int err = FAILED;
634 
635 	if (!ufshcd_cmd_inflight(lrbp->cmd)) {
636 		dev_err(hba->dev,
637 			"%s: skip abort. cmd at tag %d already completed.\n",
638 			__func__, tag);
639 		goto out;
640 	}
641 
642 	/* Skip task abort in case previous aborts failed and report failure */
643 	if (lrbp->req_abort_skip) {
644 		dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n",
645 			__func__, tag);
646 		goto out;
647 	}
648 
649 	hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
650 
651 	if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {
652 		/*
653 		 * Failure. The command should not be "stuck" in SQ for
654 		 * a long time which resulted in command being aborted.
655 		 */
656 		dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",
657 			__func__, hwq->id, tag);
658 		goto out;
659 	}
660 
661 	/*
662 	 * The command is not in the submission queue, and it is not
663 	 * in the completion queue either. Query the device to see if
664 	 * the command is being processed in the device.
665 	 */
666 	if (ufshcd_try_to_abort_task(hba, tag)) {
667 		dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err);
668 		lrbp->req_abort_skip = true;
669 		goto out;
670 	}
671 
672 	err = SUCCESS;
673 	if (ufshcd_cmd_inflight(lrbp->cmd))
674 		ufshcd_release_scsi_cmd(hba, lrbp);
675 
676 out:
677 	return err;
678 }
679