1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved.
4 *
5 * Authors:
6 * Asutosh Das <quic_asutoshd@quicinc.com>
7 * Can Guo <quic_cang@quicinc.com>
8 */
9
10 #include <asm/unaligned.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/module.h>
13 #include <linux/platform_device.h>
14 #include "ufshcd-priv.h"
15 #include <linux/delay.h>
16 #include <scsi/scsi_cmnd.h>
17 #include <linux/bitfield.h>
18 #include <linux/iopoll.h>
19
20 #define MAX_QUEUE_SUP GENMASK(7, 0)
21 #define UFS_MCQ_MIN_RW_QUEUES 2
22 #define UFS_MCQ_MIN_READ_QUEUES 0
23 #define UFS_MCQ_MIN_POLL_QUEUES 0
24 #define QUEUE_EN_OFFSET 31
25 #define QUEUE_ID_OFFSET 16
26
27 #define MCQ_CFG_MAC_MASK GENMASK(16, 8)
28 #define MCQ_QCFG_SIZE 0x40
29 #define MCQ_ENTRY_SIZE_IN_DWORD 8
30 #define CQE_UCD_BA GENMASK_ULL(63, 7)
31
32 /* Max mcq register polling time in microseconds */
33 #define MCQ_POLL_US 500000
34
rw_queue_count_set(const char * val,const struct kernel_param * kp)35 static int rw_queue_count_set(const char *val, const struct kernel_param *kp)
36 {
37 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES,
38 num_possible_cpus());
39 }
40
41 static const struct kernel_param_ops rw_queue_count_ops = {
42 .set = rw_queue_count_set,
43 .get = param_get_uint,
44 };
45
46 static unsigned int rw_queues;
47 module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644);
48 MODULE_PARM_DESC(rw_queues,
49 "Number of interrupt driven I/O queues used for rw. Default value is nr_cpus");
50
read_queue_count_set(const char * val,const struct kernel_param * kp)51 static int read_queue_count_set(const char *val, const struct kernel_param *kp)
52 {
53 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES,
54 num_possible_cpus());
55 }
56
57 static const struct kernel_param_ops read_queue_count_ops = {
58 .set = read_queue_count_set,
59 .get = param_get_uint,
60 };
61
62 static unsigned int read_queues;
63 module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644);
64 MODULE_PARM_DESC(read_queues,
65 "Number of interrupt driven read queues used for read. Default value is 0");
66
poll_queue_count_set(const char * val,const struct kernel_param * kp)67 static int poll_queue_count_set(const char *val, const struct kernel_param *kp)
68 {
69 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES,
70 num_possible_cpus());
71 }
72
73 static const struct kernel_param_ops poll_queue_count_ops = {
74 .set = poll_queue_count_set,
75 .get = param_get_uint,
76 };
77
78 static unsigned int poll_queues = 1;
79 module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644);
80 MODULE_PARM_DESC(poll_queues,
81 "Number of poll queues used for r/w. Default value is 1");
82
83 /**
84 * ufshcd_mcq_config_mac - Set the #Max Activ Cmds.
85 * @hba: per adapter instance
86 * @max_active_cmds: maximum # of active commands to the device at any time.
87 *
88 * The controller won't send more than the max_active_cmds to the device at
89 * any time.
90 */
ufshcd_mcq_config_mac(struct ufs_hba * hba,u32 max_active_cmds)91 void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds)
92 {
93 u32 val;
94
95 val = ufshcd_readl(hba, REG_UFS_MCQ_CFG);
96 val &= ~MCQ_CFG_MAC_MASK;
97 val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds - 1);
98 ufshcd_writel(hba, val, REG_UFS_MCQ_CFG);
99 }
100 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_mac);
101
102 /**
103 * ufshcd_mcq_req_to_hwq - find the hardware queue on which the
104 * request would be issued.
105 * @hba: per adapter instance
106 * @req: pointer to the request to be issued
107 *
108 * Return: the hardware queue instance on which the request will be or has
109 * been queued. %NULL if the request has already been freed.
110 */
ufshcd_mcq_req_to_hwq(struct ufs_hba * hba,struct request * req)111 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,
112 struct request *req)
113 {
114 struct blk_mq_hw_ctx *hctx = READ_ONCE(req->mq_hctx);
115
116 return hctx ? &hba->uhq[hctx->queue_num] : NULL;
117 }
118
119 /**
120 * ufshcd_mcq_decide_queue_depth - decide the queue depth
121 * @hba: per adapter instance
122 *
123 * Return: queue-depth on success, non-zero on error
124 *
125 * MAC - Max. Active Command of the Host Controller (HC)
126 * HC wouldn't send more than this commands to the device.
127 * It is mandatory to implement get_hba_mac() to enable MCQ mode.
128 * Calculates and adjusts the queue depth based on the depth
129 * supported by the HC and ufs device.
130 */
ufshcd_mcq_decide_queue_depth(struct ufs_hba * hba)131 int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)
132 {
133 int mac;
134
135 /* Mandatory to implement get_hba_mac() */
136 mac = ufshcd_mcq_vops_get_hba_mac(hba);
137 if (mac < 0) {
138 dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);
139 return mac;
140 }
141
142 WARN_ON_ONCE(!hba->dev_info.bqueuedepth);
143 /*
144 * max. value of bqueuedepth = 256, mac is host dependent.
145 * It is mandatory for UFS device to define bQueueDepth if
146 * shared queuing architecture is enabled.
147 */
148 return min_t(int, mac, hba->dev_info.bqueuedepth);
149 }
150
ufshcd_mcq_config_nr_queues(struct ufs_hba * hba)151 static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
152 {
153 int i;
154 u32 hba_maxq, rem, tot_queues;
155 struct Scsi_Host *host = hba->host;
156
157 /* maxq is 0 based value */
158 hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1;
159
160 tot_queues = read_queues + poll_queues + rw_queues;
161
162 if (hba_maxq < tot_queues) {
163 dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n",
164 tot_queues, hba_maxq);
165 return -EOPNOTSUPP;
166 }
167
168 rem = hba_maxq;
169
170 if (rw_queues) {
171 hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues;
172 rem -= hba->nr_queues[HCTX_TYPE_DEFAULT];
173 } else {
174 rw_queues = num_possible_cpus();
175 }
176
177 if (poll_queues) {
178 hba->nr_queues[HCTX_TYPE_POLL] = poll_queues;
179 rem -= hba->nr_queues[HCTX_TYPE_POLL];
180 }
181
182 if (read_queues) {
183 hba->nr_queues[HCTX_TYPE_READ] = read_queues;
184 rem -= hba->nr_queues[HCTX_TYPE_READ];
185 }
186
187 if (!hba->nr_queues[HCTX_TYPE_DEFAULT])
188 hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues,
189 num_possible_cpus());
190
191 for (i = 0; i < HCTX_MAX_TYPES; i++)
192 host->nr_hw_queues += hba->nr_queues[i];
193
194 hba->nr_hw_queues = host->nr_hw_queues;
195 return 0;
196 }
197
ufshcd_mcq_memory_alloc(struct ufs_hba * hba)198 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba)
199 {
200 struct ufs_hw_queue *hwq;
201 size_t utrdl_size, cqe_size;
202 int i;
203
204 for (i = 0; i < hba->nr_hw_queues; i++) {
205 hwq = &hba->uhq[i];
206
207 utrdl_size = sizeof(struct utp_transfer_req_desc) *
208 hwq->max_entries;
209 hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size,
210 &hwq->sqe_dma_addr,
211 GFP_KERNEL);
212 if (!hwq->sqe_dma_addr) {
213 dev_err(hba->dev, "SQE allocation failed\n");
214 return -ENOMEM;
215 }
216
217 cqe_size = sizeof(struct cq_entry) * hwq->max_entries;
218 hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size,
219 &hwq->cqe_dma_addr,
220 GFP_KERNEL);
221 if (!hwq->cqe_dma_addr) {
222 dev_err(hba->dev, "CQE allocation failed\n");
223 return -ENOMEM;
224 }
225 }
226
227 return 0;
228 }
229
230
231 /* Operation and runtime registers configuration */
232 #define MCQ_CFG_n(r, i) ((r) + MCQ_QCFG_SIZE * (i))
233
mcq_opr_base(struct ufs_hba * hba,enum ufshcd_mcq_opr n,int i)234 static void __iomem *mcq_opr_base(struct ufs_hba *hba,
235 enum ufshcd_mcq_opr n, int i)
236 {
237 struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n];
238
239 return opr->base + opr->stride * i;
240 }
241
ufshcd_mcq_read_cqis(struct ufs_hba * hba,int i)242 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i)
243 {
244 return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);
245 }
246 EXPORT_SYMBOL_GPL(ufshcd_mcq_read_cqis);
247
ufshcd_mcq_write_cqis(struct ufs_hba * hba,u32 val,int i)248 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i)
249 {
250 writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);
251 }
252 EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis);
253
254 /*
255 * Current MCQ specification doesn't provide a Task Tag or its equivalent in
256 * the Completion Queue Entry. Find the Task Tag using an indirect method.
257 */
ufshcd_mcq_get_tag(struct ufs_hba * hba,struct ufs_hw_queue * hwq,struct cq_entry * cqe)258 static int ufshcd_mcq_get_tag(struct ufs_hba *hba,
259 struct ufs_hw_queue *hwq,
260 struct cq_entry *cqe)
261 {
262 u64 addr;
263
264 /* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */
265 BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0));
266
267 /* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */
268 addr = (le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA) -
269 hba->ucdl_dma_addr;
270
271 return div_u64(addr, ufshcd_get_ucd_size(hba));
272 }
273
ufshcd_mcq_process_cqe(struct ufs_hba * hba,struct ufs_hw_queue * hwq)274 static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,
275 struct ufs_hw_queue *hwq)
276 {
277 struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq);
278 int tag = ufshcd_mcq_get_tag(hba, hwq, cqe);
279
280 if (cqe->command_desc_base_addr) {
281 ufshcd_compl_one_cqe(hba, tag, cqe);
282 /* After processed the cqe, mark it empty (invalid) entry */
283 cqe->command_desc_base_addr = 0;
284 }
285 }
286
ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba * hba,struct ufs_hw_queue * hwq)287 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba,
288 struct ufs_hw_queue *hwq)
289 {
290 unsigned long flags;
291 u32 entries = hwq->max_entries;
292
293 spin_lock_irqsave(&hwq->cq_lock, flags);
294 while (entries > 0) {
295 ufshcd_mcq_process_cqe(hba, hwq);
296 ufshcd_mcq_inc_cq_head_slot(hwq);
297 entries--;
298 }
299
300 ufshcd_mcq_update_cq_tail_slot(hwq);
301 hwq->cq_head_slot = hwq->cq_tail_slot;
302 spin_unlock_irqrestore(&hwq->cq_lock, flags);
303 }
304
ufshcd_mcq_poll_cqe_lock(struct ufs_hba * hba,struct ufs_hw_queue * hwq)305 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
306 struct ufs_hw_queue *hwq)
307 {
308 unsigned long completed_reqs = 0;
309 unsigned long flags;
310
311 spin_lock_irqsave(&hwq->cq_lock, flags);
312 ufshcd_mcq_update_cq_tail_slot(hwq);
313 while (!ufshcd_mcq_is_cq_empty(hwq)) {
314 ufshcd_mcq_process_cqe(hba, hwq);
315 ufshcd_mcq_inc_cq_head_slot(hwq);
316 completed_reqs++;
317 }
318
319 if (completed_reqs)
320 ufshcd_mcq_update_cq_head(hwq);
321 spin_unlock_irqrestore(&hwq->cq_lock, flags);
322
323 return completed_reqs;
324 }
325 EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock);
326
ufshcd_mcq_make_queues_operational(struct ufs_hba * hba)327 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)
328 {
329 struct ufs_hw_queue *hwq;
330 u16 qsize;
331 int i;
332
333 for (i = 0; i < hba->nr_hw_queues; i++) {
334 hwq = &hba->uhq[i];
335 hwq->id = i;
336 qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1;
337
338 /* Submission Queue Lower Base Address */
339 ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr),
340 MCQ_CFG_n(REG_SQLBA, i));
341 /* Submission Queue Upper Base Address */
342 ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr),
343 MCQ_CFG_n(REG_SQUBA, i));
344 /* Submission Queue Doorbell Address Offset */
345 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_SQD, i),
346 MCQ_CFG_n(REG_SQDAO, i));
347 /* Submission Queue Interrupt Status Address Offset */
348 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_SQIS, i),
349 MCQ_CFG_n(REG_SQISAO, i));
350
351 /* Completion Queue Lower Base Address */
352 ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr),
353 MCQ_CFG_n(REG_CQLBA, i));
354 /* Completion Queue Upper Base Address */
355 ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr),
356 MCQ_CFG_n(REG_CQUBA, i));
357 /* Completion Queue Doorbell Address Offset */
358 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_CQD, i),
359 MCQ_CFG_n(REG_CQDAO, i));
360 /* Completion Queue Interrupt Status Address Offset */
361 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_CQIS, i),
362 MCQ_CFG_n(REG_CQISAO, i));
363
364 /* Save the base addresses for quicker access */
365 hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP;
366 hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP;
367 hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP;
368 hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP;
369
370 /* Reinitializing is needed upon HC reset */
371 hwq->sq_tail_slot = hwq->cq_tail_slot = hwq->cq_head_slot = 0;
372
373 /* Enable Tail Entry Push Status interrupt only for non-poll queues */
374 if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL])
375 writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE);
376
377 /* Completion Queue Enable|Size to Completion Queue Attribute */
378 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize,
379 MCQ_CFG_n(REG_CQATTR, i));
380
381 /*
382 * Submission Qeueue Enable|Size|Completion Queue ID to
383 * Submission Queue Attribute
384 */
385 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize |
386 (i << QUEUE_ID_OFFSET),
387 MCQ_CFG_n(REG_SQATTR, i));
388 }
389 }
390 EXPORT_SYMBOL_GPL(ufshcd_mcq_make_queues_operational);
391
ufshcd_mcq_enable_esi(struct ufs_hba * hba)392 void ufshcd_mcq_enable_esi(struct ufs_hba *hba)
393 {
394 ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x2,
395 REG_UFS_MEM_CFG);
396 }
397 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi);
398
ufshcd_mcq_config_esi(struct ufs_hba * hba,struct msi_msg * msg)399 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg)
400 {
401 ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA);
402 ufshcd_writel(hba, msg->address_hi, REG_UFS_ESIUBA);
403 }
404 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_esi);
405
ufshcd_mcq_init(struct ufs_hba * hba)406 int ufshcd_mcq_init(struct ufs_hba *hba)
407 {
408 struct Scsi_Host *host = hba->host;
409 struct ufs_hw_queue *hwq;
410 int ret, i;
411
412 ret = ufshcd_mcq_config_nr_queues(hba);
413 if (ret)
414 return ret;
415
416 ret = ufshcd_vops_mcq_config_resource(hba);
417 if (ret)
418 return ret;
419
420 ret = ufshcd_mcq_vops_op_runtime_config(hba);
421 if (ret) {
422 dev_err(hba->dev, "Operation runtime config failed, ret=%d\n",
423 ret);
424 return ret;
425 }
426 hba->uhq = devm_kzalloc(hba->dev,
427 hba->nr_hw_queues * sizeof(struct ufs_hw_queue),
428 GFP_KERNEL);
429 if (!hba->uhq) {
430 dev_err(hba->dev, "ufs hw queue memory allocation failed\n");
431 return -ENOMEM;
432 }
433
434 for (i = 0; i < hba->nr_hw_queues; i++) {
435 hwq = &hba->uhq[i];
436 hwq->max_entries = hba->nutrs + 1;
437 spin_lock_init(&hwq->sq_lock);
438 spin_lock_init(&hwq->cq_lock);
439 mutex_init(&hwq->sq_mutex);
440 }
441
442 /* The very first HW queue serves device commands */
443 hba->dev_cmd_queue = &hba->uhq[0];
444
445 host->host_tagset = 1;
446 return 0;
447 }
448
ufshcd_mcq_sq_stop(struct ufs_hba * hba,struct ufs_hw_queue * hwq)449 static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
450 {
451 void __iomem *reg;
452 u32 id = hwq->id, val;
453 int err;
454
455 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
456 return -ETIMEDOUT;
457
458 writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
459 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
460 err = read_poll_timeout(readl, val, val & SQ_STS, 20,
461 MCQ_POLL_US, false, reg);
462 if (err)
463 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
464 __func__, id, err);
465 return err;
466 }
467
ufshcd_mcq_sq_start(struct ufs_hba * hba,struct ufs_hw_queue * hwq)468 static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
469 {
470 void __iomem *reg;
471 u32 id = hwq->id, val;
472 int err;
473
474 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
475 return -ETIMEDOUT;
476
477 writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);
478 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;
479 err = read_poll_timeout(readl, val, !(val & SQ_STS), 20,
480 MCQ_POLL_US, false, reg);
481 if (err)
482 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",
483 __func__, id, err);
484 return err;
485 }
486
487 /**
488 * ufshcd_mcq_sq_cleanup - Clean up submission queue resources
489 * associated with the pending command.
490 * @hba: per adapter instance.
491 * @task_tag: The command's task tag.
492 *
493 * Return: 0 for success; error code otherwise.
494 */
ufshcd_mcq_sq_cleanup(struct ufs_hba * hba,int task_tag)495 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
496 {
497 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
498 struct scsi_cmnd *cmd = lrbp->cmd;
499 struct ufs_hw_queue *hwq;
500 void __iomem *reg, *opr_sqd_base;
501 u32 nexus, id, val;
502 int err;
503
504 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
505 return -ETIMEDOUT;
506
507 if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) {
508 if (!cmd)
509 return -EINVAL;
510 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
511 if (!hwq)
512 return 0;
513 } else {
514 hwq = hba->dev_cmd_queue;
515 }
516
517 id = hwq->id;
518
519 mutex_lock(&hwq->sq_mutex);
520
521 /* stop the SQ fetching before working on it */
522 err = ufshcd_mcq_sq_stop(hba, hwq);
523 if (err)
524 goto unlock;
525
526 /* SQCTI = EXT_IID, IID, LUN, Task Tag */
527 nexus = lrbp->lun << 8 | task_tag;
528 opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id);
529 writel(nexus, opr_sqd_base + REG_SQCTI);
530
531 /* SQRTCy.ICU = 1 */
532 writel(SQ_ICU, opr_sqd_base + REG_SQRTC);
533
534 /* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */
535 reg = opr_sqd_base + REG_SQRTS;
536 err = read_poll_timeout(readl, val, val & SQ_CUS, 20,
537 MCQ_POLL_US, false, reg);
538 if (err)
539 dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%ld\n",
540 __func__, id, task_tag,
541 FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg)));
542
543 if (ufshcd_mcq_sq_start(hba, hwq))
544 err = -ETIMEDOUT;
545
546 unlock:
547 mutex_unlock(&hwq->sq_mutex);
548 return err;
549 }
550
551 /**
552 * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry.
553 * Write the sqe's Command Type to 0xF. The host controller will not
554 * fetch any sqe with Command Type = 0xF.
555 *
556 * @utrd: UTP Transfer Request Descriptor to be nullified.
557 */
ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc * utrd)558 static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
559 {
560 utrd->header.command_type = 0xf;
561 }
562
563 /**
564 * ufshcd_mcq_sqe_search - Search for the command in the submission queue
565 * If the command is in the submission queue and not issued to the device yet,
566 * nullify the sqe so the host controller will skip fetching the sqe.
567 *
568 * @hba: per adapter instance.
569 * @hwq: Hardware Queue to be searched.
570 * @task_tag: The command's task tag.
571 *
572 * Return: true if the SQE containing the command is present in the SQ
573 * (not fetched by the controller); returns false if the SQE is not in the SQ.
574 */
ufshcd_mcq_sqe_search(struct ufs_hba * hba,struct ufs_hw_queue * hwq,int task_tag)575 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
576 struct ufs_hw_queue *hwq, int task_tag)
577 {
578 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
579 struct utp_transfer_req_desc *utrd;
580 __le64 cmd_desc_base_addr;
581 bool ret = false;
582 u64 addr, match;
583 u32 sq_head_slot;
584
585 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)
586 return true;
587
588 mutex_lock(&hwq->sq_mutex);
589
590 ufshcd_mcq_sq_stop(hba, hwq);
591 sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq);
592 if (sq_head_slot == hwq->sq_tail_slot)
593 goto out;
594
595 cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr;
596 addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA;
597
598 while (sq_head_slot != hwq->sq_tail_slot) {
599 utrd = hwq->sqe_base_addr + sq_head_slot;
600 match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA;
601 if (addr == match) {
602 ufshcd_mcq_nullify_sqe(utrd);
603 ret = true;
604 goto out;
605 }
606
607 sq_head_slot++;
608 if (sq_head_slot == hwq->max_entries)
609 sq_head_slot = 0;
610 }
611
612 out:
613 ufshcd_mcq_sq_start(hba, hwq);
614 mutex_unlock(&hwq->sq_mutex);
615 return ret;
616 }
617
618 /**
619 * ufshcd_mcq_abort - Abort the command in MCQ.
620 * @cmd: The command to be aborted.
621 *
622 * Return: SUCCESS or FAILED error codes
623 */
ufshcd_mcq_abort(struct scsi_cmnd * cmd)624 int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
625 {
626 struct Scsi_Host *host = cmd->device->host;
627 struct ufs_hba *hba = shost_priv(host);
628 int tag = scsi_cmd_to_rq(cmd)->tag;
629 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
630 struct ufs_hw_queue *hwq;
631 unsigned long flags;
632 int err;
633
634 if (!ufshcd_cmd_inflight(lrbp->cmd)) {
635 dev_err(hba->dev,
636 "%s: skip abort. cmd at tag %d already completed.\n",
637 __func__, tag);
638 return FAILED;
639 }
640
641 /* Skip task abort in case previous aborts failed and report failure */
642 if (lrbp->req_abort_skip) {
643 dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n",
644 __func__, tag);
645 return FAILED;
646 }
647
648 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
649
650 if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {
651 /*
652 * Failure. The command should not be "stuck" in SQ for
653 * a long time which resulted in command being aborted.
654 */
655 dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",
656 __func__, hwq->id, tag);
657 return FAILED;
658 }
659
660 /*
661 * The command is not in the submission queue, and it is not
662 * in the completion queue either. Query the device to see if
663 * the command is being processed in the device.
664 */
665 err = ufshcd_try_to_abort_task(hba, tag);
666 if (err) {
667 dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err);
668 lrbp->req_abort_skip = true;
669 return FAILED;
670 }
671
672 spin_lock_irqsave(&hwq->cq_lock, flags);
673 if (ufshcd_cmd_inflight(lrbp->cmd))
674 ufshcd_release_scsi_cmd(hba, lrbp);
675 spin_unlock_irqrestore(&hwq->cq_lock, flags);
676
677 return SUCCESS;
678 }
679