1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ 3 4 #include <linux/kernel.h> 5 #include "cc_driver.h" 6 #include "cc_buffer_mgr.h" 7 #include "cc_request_mgr.h" 8 #include "cc_ivgen.h" 9 #include "cc_pm.h" 10 11 #define CC_MAX_POLL_ITER 10 12 /* The highest descriptor count in used */ 13 #define CC_MAX_DESC_SEQ_LEN 23 14 15 struct cc_req_mgr_handle { 16 /* Request manager resources */ 17 unsigned int hw_queue_size; /* HW capability */ 18 unsigned int min_free_hw_slots; 19 unsigned int max_used_sw_slots; 20 struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE]; 21 u32 req_queue_head; 22 u32 req_queue_tail; 23 u32 axi_completed; 24 u32 q_free_slots; 25 /* This lock protects access to HW register 26 * that must be single request at a time 27 */ 28 spinlock_t hw_lock; 29 struct cc_hw_desc compl_desc; 30 u8 *dummy_comp_buff; 31 dma_addr_t dummy_comp_buff_dma; 32 33 /* backlog queue */ 34 struct list_head backlog; 35 unsigned int bl_len; 36 spinlock_t bl_lock; /* protect backlog queue */ 37 38 #ifdef COMP_IN_WQ 39 struct workqueue_struct *workq; 40 struct delayed_work compwork; 41 #else 42 struct tasklet_struct comptask; 43 #endif 44 bool is_runtime_suspended; 45 }; 46 47 struct cc_bl_item { 48 struct cc_crypto_req creq; 49 struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN]; 50 unsigned int len; 51 struct list_head list; 52 bool notif; 53 }; 54 55 static void comp_handler(unsigned long devarg); 56 #ifdef COMP_IN_WQ 57 static void comp_work_handler(struct work_struct *work); 58 #endif 59 60 void cc_req_mgr_fini(struct cc_drvdata *drvdata) 61 { 62 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 63 struct device *dev = drvdata_to_dev(drvdata); 64 65 if (!req_mgr_h) 66 return; /* Not allocated */ 67 68 if (req_mgr_h->dummy_comp_buff_dma) { 69 dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff, 70 req_mgr_h->dummy_comp_buff_dma); 71 } 72 73 dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size - 74 req_mgr_h->min_free_hw_slots)); 75 dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots); 76 77 #ifdef COMP_IN_WQ 78 flush_workqueue(req_mgr_h->workq); 79 destroy_workqueue(req_mgr_h->workq); 80 #else 81 /* Kill tasklet */ 82 tasklet_kill(&req_mgr_h->comptask); 83 #endif 84 kzfree(req_mgr_h); 85 drvdata->request_mgr_handle = NULL; 86 } 87 88 int cc_req_mgr_init(struct cc_drvdata *drvdata) 89 { 90 struct cc_req_mgr_handle *req_mgr_h; 91 struct device *dev = drvdata_to_dev(drvdata); 92 int rc = 0; 93 94 req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL); 95 if (!req_mgr_h) { 96 rc = -ENOMEM; 97 goto req_mgr_init_err; 98 } 99 100 drvdata->request_mgr_handle = req_mgr_h; 101 102 spin_lock_init(&req_mgr_h->hw_lock); 103 spin_lock_init(&req_mgr_h->bl_lock); 104 INIT_LIST_HEAD(&req_mgr_h->backlog); 105 106 #ifdef COMP_IN_WQ 107 dev_dbg(dev, "Initializing completion workqueue\n"); 108 req_mgr_h->workq = create_singlethread_workqueue("ccree"); 109 if (!req_mgr_h->workq) { 110 dev_err(dev, "Failed creating work queue\n"); 111 rc = -ENOMEM; 112 goto req_mgr_init_err; 113 } 114 INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler); 115 #else 116 dev_dbg(dev, "Initializing completion tasklet\n"); 117 tasklet_init(&req_mgr_h->comptask, comp_handler, 118 (unsigned long)drvdata); 119 #endif 120 req_mgr_h->hw_queue_size = cc_ioread(drvdata, 121 CC_REG(DSCRPTR_QUEUE_SRAM_SIZE)); 122 dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size); 123 if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) { 124 dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n", 125 req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE); 126 rc = -ENOMEM; 127 goto req_mgr_init_err; 128 } 129 req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size; 130 req_mgr_h->max_used_sw_slots = 0; 131 132 /* Allocate DMA word for "dummy" completion descriptor use */ 133 req_mgr_h->dummy_comp_buff = 134 dma_alloc_coherent(dev, sizeof(u32), 135 &req_mgr_h->dummy_comp_buff_dma, 136 GFP_KERNEL); 137 if (!req_mgr_h->dummy_comp_buff) { 138 dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n", 139 sizeof(u32)); 140 rc = -ENOMEM; 141 goto req_mgr_init_err; 142 } 143 144 /* Init. "dummy" completion descriptor */ 145 hw_desc_init(&req_mgr_h->compl_desc); 146 set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32)); 147 set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma, 148 sizeof(u32), NS_BIT, 1); 149 set_flow_mode(&req_mgr_h->compl_desc, BYPASS); 150 set_queue_last_ind(drvdata, &req_mgr_h->compl_desc); 151 152 return 0; 153 154 req_mgr_init_err: 155 cc_req_mgr_fini(drvdata); 156 return rc; 157 } 158 159 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[], 160 unsigned int seq_len) 161 { 162 int i, w; 163 void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0); 164 struct device *dev = drvdata_to_dev(drvdata); 165 166 /* 167 * We do indeed write all 6 command words to the same 168 * register. The HW supports this. 169 */ 170 171 for (i = 0; i < seq_len; i++) { 172 for (w = 0; w <= 5; w++) 173 writel_relaxed(seq[i].word[w], reg); 174 175 if (cc_dump_desc) 176 dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", 177 i, seq[i].word[0], seq[i].word[1], 178 seq[i].word[2], seq[i].word[3], 179 seq[i].word[4], seq[i].word[5]); 180 } 181 } 182 183 /*! 184 * Completion will take place if and only if user requested completion 185 * by cc_send_sync_request(). 186 * 187 * \param dev 188 * \param dx_compl_h The completion event to signal 189 */ 190 static void request_mgr_complete(struct device *dev, void *dx_compl_h, 191 int dummy) 192 { 193 struct completion *this_compl = dx_compl_h; 194 195 complete(this_compl); 196 } 197 198 static int cc_queues_status(struct cc_drvdata *drvdata, 199 struct cc_req_mgr_handle *req_mgr_h, 200 unsigned int total_seq_len) 201 { 202 unsigned long poll_queue; 203 struct device *dev = drvdata_to_dev(drvdata); 204 205 /* SW queue is checked only once as it will not 206 * be chaned during the poll because the spinlock_bh 207 * is held by the thread 208 */ 209 if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) == 210 req_mgr_h->req_queue_tail) { 211 dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n", 212 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE); 213 return -ENOSPC; 214 } 215 216 if (req_mgr_h->q_free_slots >= total_seq_len) 217 return 0; 218 219 /* Wait for space in HW queue. Poll constant num of iterations. */ 220 for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) { 221 req_mgr_h->q_free_slots = 222 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 223 if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots) 224 req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots; 225 226 if (req_mgr_h->q_free_slots >= total_seq_len) { 227 /* If there is enough place return */ 228 return 0; 229 } 230 231 dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n", 232 req_mgr_h->q_free_slots, total_seq_len); 233 } 234 /* No room in the HW queue try again later */ 235 dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n", 236 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE, 237 req_mgr_h->q_free_slots, total_seq_len); 238 return -ENOSPC; 239 } 240 241 /*! 242 * Enqueue caller request to crypto hardware. 243 * Need to be called with HW lock held and PM running 244 * 245 * \param drvdata 246 * \param cc_req The request to enqueue 247 * \param desc The crypto sequence 248 * \param len The crypto sequence length 249 * \param add_comp If "true": add an artificial dout DMA to mark completion 250 * 251 * \return int Returns -EINPROGRESS or error code 252 */ 253 static int cc_do_send_request(struct cc_drvdata *drvdata, 254 struct cc_crypto_req *cc_req, 255 struct cc_hw_desc *desc, unsigned int len, 256 bool add_comp, bool ivgen) 257 { 258 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 259 unsigned int used_sw_slots; 260 unsigned int iv_seq_len = 0; 261 unsigned int total_seq_len = len; /*initial sequence length*/ 262 struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN]; 263 struct device *dev = drvdata_to_dev(drvdata); 264 int rc; 265 266 if (ivgen) { 267 dev_dbg(dev, "Acquire IV from pool into %d DMA addresses %pad, %pad, %pad, IV-size=%u\n", 268 cc_req->ivgen_dma_addr_len, 269 &cc_req->ivgen_dma_addr[0], 270 &cc_req->ivgen_dma_addr[1], 271 &cc_req->ivgen_dma_addr[2], 272 cc_req->ivgen_size); 273 274 /* Acquire IV from pool */ 275 rc = cc_get_iv(drvdata, cc_req->ivgen_dma_addr, 276 cc_req->ivgen_dma_addr_len, 277 cc_req->ivgen_size, iv_seq, &iv_seq_len); 278 279 if (rc) { 280 dev_err(dev, "Failed to generate IV (rc=%d)\n", rc); 281 return rc; 282 } 283 284 total_seq_len += iv_seq_len; 285 } 286 287 used_sw_slots = ((req_mgr_h->req_queue_head - 288 req_mgr_h->req_queue_tail) & 289 (MAX_REQUEST_QUEUE_SIZE - 1)); 290 if (used_sw_slots > req_mgr_h->max_used_sw_slots) 291 req_mgr_h->max_used_sw_slots = used_sw_slots; 292 293 /* Enqueue request - must be locked with HW lock*/ 294 req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req; 295 req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) & 296 (MAX_REQUEST_QUEUE_SIZE - 1); 297 /* TODO: Use circ_buf.h ? */ 298 299 dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head); 300 301 /* 302 * We are about to push command to the HW via the command registers 303 * that may refernece hsot memory. We need to issue a memory barrier 304 * to make sure there are no outstnading memory writes 305 */ 306 wmb(); 307 308 /* STAT_PHASE_4: Push sequence */ 309 if (ivgen) 310 enqueue_seq(drvdata, iv_seq, iv_seq_len); 311 312 enqueue_seq(drvdata, desc, len); 313 314 if (add_comp) { 315 enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1); 316 total_seq_len++; 317 } 318 319 if (req_mgr_h->q_free_slots < total_seq_len) { 320 /* This situation should never occur. Maybe indicating problem 321 * with resuming power. Set the free slot count to 0 and hope 322 * for the best. 323 */ 324 dev_err(dev, "HW free slot count mismatch."); 325 req_mgr_h->q_free_slots = 0; 326 } else { 327 /* Update the free slots in HW queue */ 328 req_mgr_h->q_free_slots -= total_seq_len; 329 } 330 331 /* Operation still in process */ 332 return -EINPROGRESS; 333 } 334 335 static void cc_enqueue_backlog(struct cc_drvdata *drvdata, 336 struct cc_bl_item *bli) 337 { 338 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 339 340 spin_lock_bh(&mgr->bl_lock); 341 list_add_tail(&bli->list, &mgr->backlog); 342 ++mgr->bl_len; 343 spin_unlock_bh(&mgr->bl_lock); 344 tasklet_schedule(&mgr->comptask); 345 } 346 347 static void cc_proc_backlog(struct cc_drvdata *drvdata) 348 { 349 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 350 struct cc_bl_item *bli; 351 struct cc_crypto_req *creq; 352 struct crypto_async_request *req; 353 bool ivgen; 354 unsigned int total_len; 355 struct device *dev = drvdata_to_dev(drvdata); 356 int rc; 357 358 spin_lock(&mgr->bl_lock); 359 360 while (mgr->bl_len) { 361 bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list); 362 spin_unlock(&mgr->bl_lock); 363 364 creq = &bli->creq; 365 req = (struct crypto_async_request *)creq->user_arg; 366 367 /* 368 * Notify the request we're moving out of the backlog 369 * but only if we haven't done so already. 370 */ 371 if (!bli->notif) { 372 req->complete(req, -EINPROGRESS); 373 bli->notif = true; 374 } 375 376 ivgen = !!creq->ivgen_dma_addr_len; 377 total_len = bli->len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0); 378 379 spin_lock(&mgr->hw_lock); 380 381 rc = cc_queues_status(drvdata, mgr, total_len); 382 if (rc) { 383 /* 384 * There is still not room in the FIFO for 385 * this request. Bail out. We'll return here 386 * on the next completion irq. 387 */ 388 spin_unlock(&mgr->hw_lock); 389 return; 390 } 391 392 rc = cc_do_send_request(drvdata, &bli->creq, bli->desc, 393 bli->len, false, ivgen); 394 395 spin_unlock(&mgr->hw_lock); 396 397 if (rc != -EINPROGRESS) { 398 cc_pm_put_suspend(dev); 399 creq->user_cb(dev, req, rc); 400 } 401 402 /* Remove ourselves from the backlog list */ 403 spin_lock(&mgr->bl_lock); 404 list_del(&bli->list); 405 --mgr->bl_len; 406 } 407 408 spin_unlock(&mgr->bl_lock); 409 } 410 411 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, 412 struct cc_hw_desc *desc, unsigned int len, 413 struct crypto_async_request *req) 414 { 415 int rc; 416 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 417 bool ivgen = !!cc_req->ivgen_dma_addr_len; 418 unsigned int total_len = len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0); 419 struct device *dev = drvdata_to_dev(drvdata); 420 bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; 421 gfp_t flags = cc_gfp_flags(req); 422 struct cc_bl_item *bli; 423 424 rc = cc_pm_get(dev); 425 if (rc) { 426 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc); 427 return rc; 428 } 429 430 spin_lock_bh(&mgr->hw_lock); 431 rc = cc_queues_status(drvdata, mgr, total_len); 432 433 #ifdef CC_DEBUG_FORCE_BACKLOG 434 if (backlog_ok) 435 rc = -ENOSPC; 436 #endif /* CC_DEBUG_FORCE_BACKLOG */ 437 438 if (rc == -ENOSPC && backlog_ok) { 439 spin_unlock_bh(&mgr->hw_lock); 440 441 bli = kmalloc(sizeof(*bli), flags); 442 if (!bli) { 443 cc_pm_put_suspend(dev); 444 return -ENOMEM; 445 } 446 447 memcpy(&bli->creq, cc_req, sizeof(*cc_req)); 448 memcpy(&bli->desc, desc, len * sizeof(*desc)); 449 bli->len = len; 450 bli->notif = false; 451 cc_enqueue_backlog(drvdata, bli); 452 return -EBUSY; 453 } 454 455 if (!rc) 456 rc = cc_do_send_request(drvdata, cc_req, desc, len, false, 457 ivgen); 458 459 spin_unlock_bh(&mgr->hw_lock); 460 return rc; 461 } 462 463 int cc_send_sync_request(struct cc_drvdata *drvdata, 464 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc, 465 unsigned int len) 466 { 467 int rc; 468 struct device *dev = drvdata_to_dev(drvdata); 469 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 470 471 init_completion(&cc_req->seq_compl); 472 cc_req->user_cb = request_mgr_complete; 473 cc_req->user_arg = &cc_req->seq_compl; 474 475 rc = cc_pm_get(dev); 476 if (rc) { 477 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc); 478 return rc; 479 } 480 481 while (true) { 482 spin_lock_bh(&mgr->hw_lock); 483 rc = cc_queues_status(drvdata, mgr, len + 1); 484 485 if (!rc) 486 break; 487 488 spin_unlock_bh(&mgr->hw_lock); 489 if (rc != -EAGAIN) { 490 cc_pm_put_suspend(dev); 491 return rc; 492 } 493 wait_for_completion_interruptible(&drvdata->hw_queue_avail); 494 reinit_completion(&drvdata->hw_queue_avail); 495 } 496 497 rc = cc_do_send_request(drvdata, cc_req, desc, len, true, false); 498 spin_unlock_bh(&mgr->hw_lock); 499 500 if (rc != -EINPROGRESS) { 501 cc_pm_put_suspend(dev); 502 return rc; 503 } 504 505 wait_for_completion(&cc_req->seq_compl); 506 return 0; 507 } 508 509 /*! 510 * Enqueue caller request to crypto hardware during init process. 511 * assume this function is not called in middle of a flow, 512 * since we set QUEUE_LAST_IND flag in the last descriptor. 513 * 514 * \param drvdata 515 * \param desc The crypto sequence 516 * \param len The crypto sequence length 517 * 518 * \return int Returns "0" upon success 519 */ 520 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc, 521 unsigned int len) 522 { 523 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 524 unsigned int total_seq_len = len; /*initial sequence length*/ 525 int rc = 0; 526 527 /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT. 528 */ 529 rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len); 530 if (rc) 531 return rc; 532 533 set_queue_last_ind(drvdata, &desc[(len - 1)]); 534 535 /* 536 * We are about to push command to the HW via the command registers 537 * that may refernece hsot memory. We need to issue a memory barrier 538 * to make sure there are no outstnading memory writes 539 */ 540 wmb(); 541 enqueue_seq(drvdata, desc, len); 542 543 /* Update the free slots in HW queue */ 544 req_mgr_h->q_free_slots = 545 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 546 547 return 0; 548 } 549 550 void complete_request(struct cc_drvdata *drvdata) 551 { 552 struct cc_req_mgr_handle *request_mgr_handle = 553 drvdata->request_mgr_handle; 554 555 complete(&drvdata->hw_queue_avail); 556 #ifdef COMP_IN_WQ 557 queue_delayed_work(request_mgr_handle->workq, 558 &request_mgr_handle->compwork, 0); 559 #else 560 tasklet_schedule(&request_mgr_handle->comptask); 561 #endif 562 } 563 564 #ifdef COMP_IN_WQ 565 static void comp_work_handler(struct work_struct *work) 566 { 567 struct cc_drvdata *drvdata = 568 container_of(work, struct cc_drvdata, compwork.work); 569 570 comp_handler((unsigned long)drvdata); 571 } 572 #endif 573 574 static void proc_completions(struct cc_drvdata *drvdata) 575 { 576 struct cc_crypto_req *cc_req; 577 struct device *dev = drvdata_to_dev(drvdata); 578 struct cc_req_mgr_handle *request_mgr_handle = 579 drvdata->request_mgr_handle; 580 unsigned int *tail = &request_mgr_handle->req_queue_tail; 581 unsigned int *head = &request_mgr_handle->req_queue_head; 582 583 while (request_mgr_handle->axi_completed) { 584 request_mgr_handle->axi_completed--; 585 586 /* Dequeue request */ 587 if (*head == *tail) { 588 /* We are supposed to handle a completion but our 589 * queue is empty. This is not normal. Return and 590 * hope for the best. 591 */ 592 dev_err(dev, "Request queue is empty head == tail %u\n", 593 *head); 594 break; 595 } 596 597 cc_req = &request_mgr_handle->req_queue[*tail]; 598 599 if (cc_req->user_cb) 600 cc_req->user_cb(dev, cc_req->user_arg, 0); 601 *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); 602 dev_dbg(dev, "Dequeue request tail=%u\n", *tail); 603 dev_dbg(dev, "Request completed. axi_completed=%d\n", 604 request_mgr_handle->axi_completed); 605 cc_pm_put_suspend(dev); 606 } 607 } 608 609 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata) 610 { 611 return FIELD_GET(AXIM_MON_COMP_VALUE, 612 cc_ioread(drvdata, drvdata->axim_mon_offset)); 613 } 614 615 /* Deferred service handler, run as interrupt-fired tasklet */ 616 static void comp_handler(unsigned long devarg) 617 { 618 struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; 619 struct cc_req_mgr_handle *request_mgr_handle = 620 drvdata->request_mgr_handle; 621 622 u32 irq; 623 624 irq = (drvdata->irq & CC_COMP_IRQ_MASK); 625 626 if (irq & CC_COMP_IRQ_MASK) { 627 /* To avoid the interrupt from firing as we unmask it, 628 * we clear it now 629 */ 630 cc_iowrite(drvdata, CC_REG(HOST_ICR), CC_COMP_IRQ_MASK); 631 632 /* Avoid race with above clear: Test completion counter 633 * once more 634 */ 635 request_mgr_handle->axi_completed += 636 cc_axi_comp_count(drvdata); 637 638 while (request_mgr_handle->axi_completed) { 639 do { 640 proc_completions(drvdata); 641 /* At this point (after proc_completions()), 642 * request_mgr_handle->axi_completed is 0. 643 */ 644 request_mgr_handle->axi_completed = 645 cc_axi_comp_count(drvdata); 646 } while (request_mgr_handle->axi_completed > 0); 647 648 cc_iowrite(drvdata, CC_REG(HOST_ICR), 649 CC_COMP_IRQ_MASK); 650 651 request_mgr_handle->axi_completed += 652 cc_axi_comp_count(drvdata); 653 } 654 } 655 /* after verifing that there is nothing to do, 656 * unmask AXI completion interrupt 657 */ 658 cc_iowrite(drvdata, CC_REG(HOST_IMR), 659 cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~irq); 660 661 cc_proc_backlog(drvdata); 662 } 663 664 /* 665 * resume the queue configuration - no need to take the lock as this happens 666 * inside the spin lock protection 667 */ 668 #if defined(CONFIG_PM) 669 int cc_resume_req_queue(struct cc_drvdata *drvdata) 670 { 671 struct cc_req_mgr_handle *request_mgr_handle = 672 drvdata->request_mgr_handle; 673 674 spin_lock_bh(&request_mgr_handle->hw_lock); 675 request_mgr_handle->is_runtime_suspended = false; 676 spin_unlock_bh(&request_mgr_handle->hw_lock); 677 678 return 0; 679 } 680 681 /* 682 * suspend the queue configuration. Since it is used for the runtime suspend 683 * only verify that the queue can be suspended. 684 */ 685 int cc_suspend_req_queue(struct cc_drvdata *drvdata) 686 { 687 struct cc_req_mgr_handle *request_mgr_handle = 688 drvdata->request_mgr_handle; 689 690 /* lock the send_request */ 691 spin_lock_bh(&request_mgr_handle->hw_lock); 692 if (request_mgr_handle->req_queue_head != 693 request_mgr_handle->req_queue_tail) { 694 spin_unlock_bh(&request_mgr_handle->hw_lock); 695 return -EBUSY; 696 } 697 request_mgr_handle->is_runtime_suspended = true; 698 spin_unlock_bh(&request_mgr_handle->hw_lock); 699 700 return 0; 701 } 702 703 bool cc_req_queue_suspended(struct cc_drvdata *drvdata) 704 { 705 struct cc_req_mgr_handle *request_mgr_handle = 706 drvdata->request_mgr_handle; 707 708 return request_mgr_handle->is_runtime_suspended; 709 } 710 711 #endif 712