1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved. 4 * 5 * Authors: 6 * Asutosh Das <quic_asutoshd@quicinc.com> 7 * Can Guo <quic_cang@quicinc.com> 8 */ 9 10 #include <asm/unaligned.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/module.h> 13 #include <linux/platform_device.h> 14 #include "ufshcd-priv.h" 15 #include <linux/delay.h> 16 #include <scsi/scsi_cmnd.h> 17 #include <linux/bitfield.h> 18 #include <linux/iopoll.h> 19 20 #define MAX_QUEUE_SUP GENMASK(7, 0) 21 #define UFS_MCQ_MIN_RW_QUEUES 2 22 #define UFS_MCQ_MIN_READ_QUEUES 0 23 #define UFS_MCQ_NUM_DEV_CMD_QUEUES 1 24 #define UFS_MCQ_MIN_POLL_QUEUES 0 25 #define QUEUE_EN_OFFSET 31 26 #define QUEUE_ID_OFFSET 16 27 28 #define MAX_DEV_CMD_ENTRIES 2 29 #define MCQ_CFG_MAC_MASK GENMASK(16, 8) 30 #define MCQ_QCFG_SIZE 0x40 31 #define MCQ_ENTRY_SIZE_IN_DWORD 8 32 #define CQE_UCD_BA GENMASK_ULL(63, 7) 33 34 /* Max mcq register polling time in microseconds */ 35 #define MCQ_POLL_US 500000 36 37 static int rw_queue_count_set(const char *val, const struct kernel_param *kp) 38 { 39 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES, 40 num_possible_cpus()); 41 } 42 43 static const struct kernel_param_ops rw_queue_count_ops = { 44 .set = rw_queue_count_set, 45 .get = param_get_uint, 46 }; 47 48 static unsigned int rw_queues; 49 module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644); 50 MODULE_PARM_DESC(rw_queues, 51 "Number of interrupt driven I/O queues used for rw. Default value is nr_cpus"); 52 53 static int read_queue_count_set(const char *val, const struct kernel_param *kp) 54 { 55 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES, 56 num_possible_cpus()); 57 } 58 59 static const struct kernel_param_ops read_queue_count_ops = { 60 .set = read_queue_count_set, 61 .get = param_get_uint, 62 }; 63 64 static unsigned int read_queues; 65 module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644); 66 MODULE_PARM_DESC(read_queues, 67 "Number of interrupt driven read queues used for read. Default value is 0"); 68 69 static int poll_queue_count_set(const char *val, const struct kernel_param *kp) 70 { 71 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES, 72 num_possible_cpus()); 73 } 74 75 static const struct kernel_param_ops poll_queue_count_ops = { 76 .set = poll_queue_count_set, 77 .get = param_get_uint, 78 }; 79 80 static unsigned int poll_queues = 1; 81 module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644); 82 MODULE_PARM_DESC(poll_queues, 83 "Number of poll queues used for r/w. Default value is 1"); 84 85 /** 86 * ufshcd_mcq_config_mac - Set the #Max Activ Cmds. 87 * @hba: per adapter instance 88 * @max_active_cmds: maximum # of active commands to the device at any time. 89 * 90 * The controller won't send more than the max_active_cmds to the device at 91 * any time. 92 */ 93 void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds) 94 { 95 u32 val; 96 97 val = ufshcd_readl(hba, REG_UFS_MCQ_CFG); 98 val &= ~MCQ_CFG_MAC_MASK; 99 val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds); 100 ufshcd_writel(hba, val, REG_UFS_MCQ_CFG); 101 } 102 103 /** 104 * ufshcd_mcq_req_to_hwq - find the hardware queue on which the 105 * request would be issued. 106 * @hba: per adapter instance 107 * @req: pointer to the request to be issued 108 * 109 * Returns the hardware queue instance on which the request would 110 * be queued. 111 */ 112 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, 113 struct request *req) 114 { 115 u32 utag = blk_mq_unique_tag(req); 116 u32 hwq = blk_mq_unique_tag_to_hwq(utag); 117 118 /* uhq[0] is used to serve device commands */ 119 return &hba->uhq[hwq + UFSHCD_MCQ_IO_QUEUE_OFFSET]; 120 } 121 122 /** 123 * ufshcd_mcq_decide_queue_depth - decide the queue depth 124 * @hba: per adapter instance 125 * 126 * Returns queue-depth on success, non-zero on error 127 * 128 * MAC - Max. Active Command of the Host Controller (HC) 129 * HC wouldn't send more than this commands to the device. 130 * It is mandatory to implement get_hba_mac() to enable MCQ mode. 131 * Calculates and adjusts the queue depth based on the depth 132 * supported by the HC and ufs device. 133 */ 134 int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba) 135 { 136 int mac; 137 138 /* Mandatory to implement get_hba_mac() */ 139 mac = ufshcd_mcq_vops_get_hba_mac(hba); 140 if (mac < 0) { 141 dev_err(hba->dev, "Failed to get mac, err=%d\n", mac); 142 return mac; 143 } 144 145 WARN_ON_ONCE(!hba->dev_info.bqueuedepth); 146 /* 147 * max. value of bqueuedepth = 256, mac is host dependent. 148 * It is mandatory for UFS device to define bQueueDepth if 149 * shared queuing architecture is enabled. 150 */ 151 return min_t(int, mac, hba->dev_info.bqueuedepth); 152 } 153 154 static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba) 155 { 156 int i; 157 u32 hba_maxq, rem, tot_queues; 158 struct Scsi_Host *host = hba->host; 159 160 hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities); 161 162 tot_queues = UFS_MCQ_NUM_DEV_CMD_QUEUES + read_queues + poll_queues + 163 rw_queues; 164 165 if (hba_maxq < tot_queues) { 166 dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n", 167 tot_queues, hba_maxq); 168 return -EOPNOTSUPP; 169 } 170 171 rem = hba_maxq - UFS_MCQ_NUM_DEV_CMD_QUEUES; 172 173 if (rw_queues) { 174 hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues; 175 rem -= hba->nr_queues[HCTX_TYPE_DEFAULT]; 176 } else { 177 rw_queues = num_possible_cpus(); 178 } 179 180 if (poll_queues) { 181 hba->nr_queues[HCTX_TYPE_POLL] = poll_queues; 182 rem -= hba->nr_queues[HCTX_TYPE_POLL]; 183 } 184 185 if (read_queues) { 186 hba->nr_queues[HCTX_TYPE_READ] = read_queues; 187 rem -= hba->nr_queues[HCTX_TYPE_READ]; 188 } 189 190 if (!hba->nr_queues[HCTX_TYPE_DEFAULT]) 191 hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues, 192 num_possible_cpus()); 193 194 for (i = 0; i < HCTX_MAX_TYPES; i++) 195 host->nr_hw_queues += hba->nr_queues[i]; 196 197 hba->nr_hw_queues = host->nr_hw_queues + UFS_MCQ_NUM_DEV_CMD_QUEUES; 198 return 0; 199 } 200 201 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba) 202 { 203 struct ufs_hw_queue *hwq; 204 size_t utrdl_size, cqe_size; 205 int i; 206 207 for (i = 0; i < hba->nr_hw_queues; i++) { 208 hwq = &hba->uhq[i]; 209 210 utrdl_size = sizeof(struct utp_transfer_req_desc) * 211 hwq->max_entries; 212 hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size, 213 &hwq->sqe_dma_addr, 214 GFP_KERNEL); 215 if (!hwq->sqe_dma_addr) { 216 dev_err(hba->dev, "SQE allocation failed\n"); 217 return -ENOMEM; 218 } 219 220 cqe_size = sizeof(struct cq_entry) * hwq->max_entries; 221 hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size, 222 &hwq->cqe_dma_addr, 223 GFP_KERNEL); 224 if (!hwq->cqe_dma_addr) { 225 dev_err(hba->dev, "CQE allocation failed\n"); 226 return -ENOMEM; 227 } 228 } 229 230 return 0; 231 } 232 233 234 /* Operation and runtime registers configuration */ 235 #define MCQ_CFG_n(r, i) ((r) + MCQ_QCFG_SIZE * (i)) 236 #define MCQ_OPR_OFFSET_n(p, i) \ 237 (hba->mcq_opr[(p)].offset + hba->mcq_opr[(p)].stride * (i)) 238 239 static void __iomem *mcq_opr_base(struct ufs_hba *hba, 240 enum ufshcd_mcq_opr n, int i) 241 { 242 struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n]; 243 244 return opr->base + opr->stride * i; 245 } 246 247 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i) 248 { 249 return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); 250 } 251 252 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i) 253 { 254 writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); 255 } 256 EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis); 257 258 /* 259 * Current MCQ specification doesn't provide a Task Tag or its equivalent in 260 * the Completion Queue Entry. Find the Task Tag using an indirect method. 261 */ 262 static int ufshcd_mcq_get_tag(struct ufs_hba *hba, 263 struct ufs_hw_queue *hwq, 264 struct cq_entry *cqe) 265 { 266 u64 addr; 267 268 /* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */ 269 BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0)); 270 271 /* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */ 272 addr = (le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA) - 273 hba->ucdl_dma_addr; 274 275 return div_u64(addr, sizeof(struct utp_transfer_cmd_desc)); 276 } 277 278 static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, 279 struct ufs_hw_queue *hwq) 280 { 281 struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); 282 int tag = ufshcd_mcq_get_tag(hba, hwq, cqe); 283 284 if (cqe->command_desc_base_addr) { 285 ufshcd_compl_one_cqe(hba, tag, cqe); 286 /* After processed the cqe, mark it empty (invalid) entry */ 287 cqe->command_desc_base_addr = 0; 288 } 289 } 290 291 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, 292 struct ufs_hw_queue *hwq) 293 { 294 unsigned long flags; 295 u32 entries = hwq->max_entries; 296 297 spin_lock_irqsave(&hwq->cq_lock, flags); 298 while (entries > 0) { 299 ufshcd_mcq_process_cqe(hba, hwq); 300 ufshcd_mcq_inc_cq_head_slot(hwq); 301 entries--; 302 } 303 304 ufshcd_mcq_update_cq_tail_slot(hwq); 305 hwq->cq_head_slot = hwq->cq_tail_slot; 306 spin_unlock_irqrestore(&hwq->cq_lock, flags); 307 } 308 309 static unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba, 310 struct ufs_hw_queue *hwq) 311 { 312 unsigned long completed_reqs = 0; 313 314 ufshcd_mcq_update_cq_tail_slot(hwq); 315 while (!ufshcd_mcq_is_cq_empty(hwq)) { 316 ufshcd_mcq_process_cqe(hba, hwq); 317 ufshcd_mcq_inc_cq_head_slot(hwq); 318 completed_reqs++; 319 } 320 321 if (completed_reqs) 322 ufshcd_mcq_update_cq_head(hwq); 323 324 return completed_reqs; 325 } 326 327 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba, 328 struct ufs_hw_queue *hwq) 329 { 330 unsigned long completed_reqs, flags; 331 332 spin_lock_irqsave(&hwq->cq_lock, flags); 333 completed_reqs = ufshcd_mcq_poll_cqe_nolock(hba, hwq); 334 spin_unlock_irqrestore(&hwq->cq_lock, flags); 335 336 return completed_reqs; 337 } 338 EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock); 339 340 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba) 341 { 342 struct ufs_hw_queue *hwq; 343 u16 qsize; 344 int i; 345 346 for (i = 0; i < hba->nr_hw_queues; i++) { 347 hwq = &hba->uhq[i]; 348 hwq->id = i; 349 qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1; 350 351 /* Submission Queue Lower Base Address */ 352 ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr), 353 MCQ_CFG_n(REG_SQLBA, i)); 354 /* Submission Queue Upper Base Address */ 355 ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr), 356 MCQ_CFG_n(REG_SQUBA, i)); 357 /* Submission Queue Doorbell Address Offset */ 358 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQD, i), 359 MCQ_CFG_n(REG_SQDAO, i)); 360 /* Submission Queue Interrupt Status Address Offset */ 361 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQIS, i), 362 MCQ_CFG_n(REG_SQISAO, i)); 363 364 /* Completion Queue Lower Base Address */ 365 ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr), 366 MCQ_CFG_n(REG_CQLBA, i)); 367 /* Completion Queue Upper Base Address */ 368 ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr), 369 MCQ_CFG_n(REG_CQUBA, i)); 370 /* Completion Queue Doorbell Address Offset */ 371 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQD, i), 372 MCQ_CFG_n(REG_CQDAO, i)); 373 /* Completion Queue Interrupt Status Address Offset */ 374 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQIS, i), 375 MCQ_CFG_n(REG_CQISAO, i)); 376 377 /* Save the base addresses for quicker access */ 378 hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP; 379 hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP; 380 hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP; 381 hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP; 382 383 /* Reinitializing is needed upon HC reset */ 384 hwq->sq_tail_slot = hwq->cq_tail_slot = hwq->cq_head_slot = 0; 385 386 /* Enable Tail Entry Push Status interrupt only for non-poll queues */ 387 if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]) 388 writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE); 389 390 /* Completion Queue Enable|Size to Completion Queue Attribute */ 391 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize, 392 MCQ_CFG_n(REG_CQATTR, i)); 393 394 /* 395 * Submission Qeueue Enable|Size|Completion Queue ID to 396 * Submission Queue Attribute 397 */ 398 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize | 399 (i << QUEUE_ID_OFFSET), 400 MCQ_CFG_n(REG_SQATTR, i)); 401 } 402 } 403 404 void ufshcd_mcq_enable_esi(struct ufs_hba *hba) 405 { 406 ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x2, 407 REG_UFS_MEM_CFG); 408 } 409 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi); 410 411 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg) 412 { 413 ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA); 414 ufshcd_writel(hba, msg->address_hi, REG_UFS_ESIUBA); 415 } 416 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_esi); 417 418 int ufshcd_mcq_init(struct ufs_hba *hba) 419 { 420 struct Scsi_Host *host = hba->host; 421 struct ufs_hw_queue *hwq; 422 int ret, i; 423 424 ret = ufshcd_mcq_config_nr_queues(hba); 425 if (ret) 426 return ret; 427 428 ret = ufshcd_vops_mcq_config_resource(hba); 429 if (ret) 430 return ret; 431 432 ret = ufshcd_mcq_vops_op_runtime_config(hba); 433 if (ret) { 434 dev_err(hba->dev, "Operation runtime config failed, ret=%d\n", 435 ret); 436 return ret; 437 } 438 hba->uhq = devm_kzalloc(hba->dev, 439 hba->nr_hw_queues * sizeof(struct ufs_hw_queue), 440 GFP_KERNEL); 441 if (!hba->uhq) { 442 dev_err(hba->dev, "ufs hw queue memory allocation failed\n"); 443 return -ENOMEM; 444 } 445 446 for (i = 0; i < hba->nr_hw_queues; i++) { 447 hwq = &hba->uhq[i]; 448 hwq->max_entries = hba->nutrs; 449 spin_lock_init(&hwq->sq_lock); 450 spin_lock_init(&hwq->cq_lock); 451 mutex_init(&hwq->sq_mutex); 452 } 453 454 /* The very first HW queue serves device commands */ 455 hba->dev_cmd_queue = &hba->uhq[0]; 456 /* Give dev_cmd_queue the minimal number of entries */ 457 hba->dev_cmd_queue->max_entries = MAX_DEV_CMD_ENTRIES; 458 459 host->host_tagset = 1; 460 return 0; 461 } 462 463 static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq) 464 { 465 void __iomem *reg; 466 u32 id = hwq->id, val; 467 int err; 468 469 writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); 470 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; 471 err = read_poll_timeout(readl, val, val & SQ_STS, 20, 472 MCQ_POLL_US, false, reg); 473 if (err) 474 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", 475 __func__, id, err); 476 return err; 477 } 478 479 static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq) 480 { 481 void __iomem *reg; 482 u32 id = hwq->id, val; 483 int err; 484 485 writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); 486 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; 487 err = read_poll_timeout(readl, val, !(val & SQ_STS), 20, 488 MCQ_POLL_US, false, reg); 489 if (err) 490 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", 491 __func__, id, err); 492 return err; 493 } 494 495 /** 496 * ufshcd_mcq_sq_cleanup - Clean up submission queue resources 497 * associated with the pending command. 498 * @hba - per adapter instance. 499 * @task_tag - The command's task tag. 500 * 501 * Returns 0 for success; error code otherwise. 502 */ 503 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) 504 { 505 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 506 struct scsi_cmnd *cmd = lrbp->cmd; 507 struct ufs_hw_queue *hwq; 508 void __iomem *reg, *opr_sqd_base; 509 u32 nexus, id, val; 510 int err; 511 512 if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) { 513 if (!cmd) 514 return -EINVAL; 515 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 516 } else { 517 hwq = hba->dev_cmd_queue; 518 } 519 520 id = hwq->id; 521 522 mutex_lock(&hwq->sq_mutex); 523 524 /* stop the SQ fetching before working on it */ 525 err = ufshcd_mcq_sq_stop(hba, hwq); 526 if (err) 527 goto unlock; 528 529 /* SQCTI = EXT_IID, IID, LUN, Task Tag */ 530 nexus = lrbp->lun << 8 | task_tag; 531 opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id); 532 writel(nexus, opr_sqd_base + REG_SQCTI); 533 534 /* SQRTCy.ICU = 1 */ 535 writel(SQ_ICU, opr_sqd_base + REG_SQRTC); 536 537 /* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */ 538 reg = opr_sqd_base + REG_SQRTS; 539 err = read_poll_timeout(readl, val, val & SQ_CUS, 20, 540 MCQ_POLL_US, false, reg); 541 if (err) 542 dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%ld\n", 543 __func__, id, task_tag, 544 FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg))); 545 546 if (ufshcd_mcq_sq_start(hba, hwq)) 547 err = -ETIMEDOUT; 548 549 unlock: 550 mutex_unlock(&hwq->sq_mutex); 551 return err; 552 } 553 554 /** 555 * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry. 556 * Write the sqe's Command Type to 0xF. The host controller will not 557 * fetch any sqe with Command Type = 0xF. 558 * 559 * @utrd - UTP Transfer Request Descriptor to be nullified. 560 */ 561 static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) 562 { 563 u32 dword_0; 564 565 dword_0 = le32_to_cpu(utrd->header.dword_0); 566 dword_0 &= ~UPIU_COMMAND_TYPE_MASK; 567 dword_0 |= FIELD_PREP(UPIU_COMMAND_TYPE_MASK, 0xF); 568 utrd->header.dword_0 = cpu_to_le32(dword_0); 569 } 570 571 /** 572 * ufshcd_mcq_sqe_search - Search for the command in the submission queue 573 * If the command is in the submission queue and not issued to the device yet, 574 * nullify the sqe so the host controller will skip fetching the sqe. 575 * 576 * @hba - per adapter instance. 577 * @hwq - Hardware Queue to be searched. 578 * @task_tag - The command's task tag. 579 * 580 * Returns true if the SQE containing the command is present in the SQ 581 * (not fetched by the controller); returns false if the SQE is not in the SQ. 582 */ 583 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, 584 struct ufs_hw_queue *hwq, int task_tag) 585 { 586 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 587 struct utp_transfer_req_desc *utrd; 588 u32 mask = hwq->max_entries - 1; 589 __le64 cmd_desc_base_addr; 590 bool ret = false; 591 u64 addr, match; 592 u32 sq_head_slot; 593 594 mutex_lock(&hwq->sq_mutex); 595 596 ufshcd_mcq_sq_stop(hba, hwq); 597 sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq); 598 if (sq_head_slot == hwq->sq_tail_slot) 599 goto out; 600 601 cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr; 602 addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA; 603 604 while (sq_head_slot != hwq->sq_tail_slot) { 605 utrd = hwq->sqe_base_addr + 606 sq_head_slot * sizeof(struct utp_transfer_req_desc); 607 match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA; 608 if (addr == match) { 609 ufshcd_mcq_nullify_sqe(utrd); 610 ret = true; 611 goto out; 612 } 613 sq_head_slot = (sq_head_slot + 1) & mask; 614 } 615 616 out: 617 ufshcd_mcq_sq_start(hba, hwq); 618 mutex_unlock(&hwq->sq_mutex); 619 return ret; 620 } 621 622 /** 623 * ufshcd_mcq_abort - Abort the command in MCQ. 624 * @cmd - The command to be aborted. 625 * 626 * Returns SUCCESS or FAILED error codes 627 */ 628 int ufshcd_mcq_abort(struct scsi_cmnd *cmd) 629 { 630 struct Scsi_Host *host = cmd->device->host; 631 struct ufs_hba *hba = shost_priv(host); 632 int tag = scsi_cmd_to_rq(cmd)->tag; 633 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 634 struct ufs_hw_queue *hwq; 635 int err = FAILED; 636 637 if (!ufshcd_cmd_inflight(lrbp->cmd)) { 638 dev_err(hba->dev, 639 "%s: skip abort. cmd at tag %d already completed.\n", 640 __func__, tag); 641 goto out; 642 } 643 644 /* Skip task abort in case previous aborts failed and report failure */ 645 if (lrbp->req_abort_skip) { 646 dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n", 647 __func__, tag); 648 goto out; 649 } 650 651 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 652 653 if (ufshcd_mcq_sqe_search(hba, hwq, tag)) { 654 /* 655 * Failure. The command should not be "stuck" in SQ for 656 * a long time which resulted in command being aborted. 657 */ 658 dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n", 659 __func__, hwq->id, tag); 660 goto out; 661 } 662 663 /* 664 * The command is not in the submission queue, and it is not 665 * in the completion queue either. Query the device to see if 666 * the command is being processed in the device. 667 */ 668 if (ufshcd_try_to_abort_task(hba, tag)) { 669 dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err); 670 lrbp->req_abort_skip = true; 671 goto out; 672 } 673 674 err = SUCCESS; 675 if (ufshcd_cmd_inflight(lrbp->cmd)) 676 ufshcd_release_scsi_cmd(hba, lrbp); 677 678 out: 679 return err; 680 } 681