1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/nospec.h> 6 #include "cc_driver.h" 7 #include "cc_buffer_mgr.h" 8 #include "cc_request_mgr.h" 9 #include "cc_pm.h" 10 11 #define CC_MAX_POLL_ITER 10 12 /* The highest descriptor count in used */ 13 #define CC_MAX_DESC_SEQ_LEN 23 14 15 struct cc_req_mgr_handle { 16 /* Request manager resources */ 17 unsigned int hw_queue_size; /* HW capability */ 18 unsigned int min_free_hw_slots; 19 unsigned int max_used_sw_slots; 20 struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE]; 21 u32 req_queue_head; 22 u32 req_queue_tail; 23 u32 axi_completed; 24 u32 q_free_slots; 25 /* This lock protects access to HW register 26 * that must be single request at a time 27 */ 28 spinlock_t hw_lock; 29 struct cc_hw_desc compl_desc; 30 u8 *dummy_comp_buff; 31 dma_addr_t dummy_comp_buff_dma; 32 33 /* backlog queue */ 34 struct list_head backlog; 35 unsigned int bl_len; 36 spinlock_t bl_lock; /* protect backlog queue */ 37 38 #ifdef COMP_IN_WQ 39 struct workqueue_struct *workq; 40 struct delayed_work compwork; 41 #else 42 struct tasklet_struct comptask; 43 #endif 44 bool is_runtime_suspended; 45 }; 46 47 struct cc_bl_item { 48 struct cc_crypto_req creq; 49 struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN]; 50 unsigned int len; 51 struct list_head list; 52 bool notif; 53 }; 54 55 static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = { 56 { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT), 57 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT), 58 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT), 59 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT), 60 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT), 61 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT), 62 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT), 63 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) }, 64 { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT), 65 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT), 66 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT), 67 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT), 68 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT), 69 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT), 70 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT), 71 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) } 72 }; 73 74 static void comp_handler(unsigned long devarg); 75 #ifdef COMP_IN_WQ 76 static void comp_work_handler(struct work_struct *work); 77 #endif 78 79 static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot) 80 { 81 alg = array_index_nospec(alg, CC_CPP_NUM_ALGS); 82 slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS); 83 84 return cc_cpp_int_masks[alg][slot]; 85 } 86 87 void cc_req_mgr_fini(struct cc_drvdata *drvdata) 88 { 89 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 90 struct device *dev = drvdata_to_dev(drvdata); 91 92 if (!req_mgr_h) 93 return; /* Not allocated */ 94 95 if (req_mgr_h->dummy_comp_buff_dma) { 96 dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff, 97 req_mgr_h->dummy_comp_buff_dma); 98 } 99 100 dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size - 101 req_mgr_h->min_free_hw_slots)); 102 dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots); 103 104 #ifdef COMP_IN_WQ 105 flush_workqueue(req_mgr_h->workq); 106 destroy_workqueue(req_mgr_h->workq); 107 #else 108 /* Kill tasklet */ 109 tasklet_kill(&req_mgr_h->comptask); 110 #endif 111 kzfree(req_mgr_h); 112 drvdata->request_mgr_handle = NULL; 113 } 114 115 int cc_req_mgr_init(struct cc_drvdata *drvdata) 116 { 117 struct cc_req_mgr_handle *req_mgr_h; 118 struct device *dev = drvdata_to_dev(drvdata); 119 int rc = 0; 120 121 req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL); 122 if (!req_mgr_h) { 123 rc = -ENOMEM; 124 goto req_mgr_init_err; 125 } 126 127 drvdata->request_mgr_handle = req_mgr_h; 128 129 spin_lock_init(&req_mgr_h->hw_lock); 130 spin_lock_init(&req_mgr_h->bl_lock); 131 INIT_LIST_HEAD(&req_mgr_h->backlog); 132 133 #ifdef COMP_IN_WQ 134 dev_dbg(dev, "Initializing completion workqueue\n"); 135 req_mgr_h->workq = create_singlethread_workqueue("ccree"); 136 if (!req_mgr_h->workq) { 137 dev_err(dev, "Failed creating work queue\n"); 138 rc = -ENOMEM; 139 goto req_mgr_init_err; 140 } 141 INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler); 142 #else 143 dev_dbg(dev, "Initializing completion tasklet\n"); 144 tasklet_init(&req_mgr_h->comptask, comp_handler, 145 (unsigned long)drvdata); 146 #endif 147 req_mgr_h->hw_queue_size = cc_ioread(drvdata, 148 CC_REG(DSCRPTR_QUEUE_SRAM_SIZE)); 149 dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size); 150 if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) { 151 dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n", 152 req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE); 153 rc = -ENOMEM; 154 goto req_mgr_init_err; 155 } 156 req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size; 157 req_mgr_h->max_used_sw_slots = 0; 158 159 /* Allocate DMA word for "dummy" completion descriptor use */ 160 req_mgr_h->dummy_comp_buff = 161 dma_alloc_coherent(dev, sizeof(u32), 162 &req_mgr_h->dummy_comp_buff_dma, 163 GFP_KERNEL); 164 if (!req_mgr_h->dummy_comp_buff) { 165 dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n", 166 sizeof(u32)); 167 rc = -ENOMEM; 168 goto req_mgr_init_err; 169 } 170 171 /* Init. "dummy" completion descriptor */ 172 hw_desc_init(&req_mgr_h->compl_desc); 173 set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32)); 174 set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma, 175 sizeof(u32), NS_BIT, 1); 176 set_flow_mode(&req_mgr_h->compl_desc, BYPASS); 177 set_queue_last_ind(drvdata, &req_mgr_h->compl_desc); 178 179 return 0; 180 181 req_mgr_init_err: 182 cc_req_mgr_fini(drvdata); 183 return rc; 184 } 185 186 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[], 187 unsigned int seq_len) 188 { 189 int i, w; 190 void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0); 191 struct device *dev = drvdata_to_dev(drvdata); 192 193 /* 194 * We do indeed write all 6 command words to the same 195 * register. The HW supports this. 196 */ 197 198 for (i = 0; i < seq_len; i++) { 199 for (w = 0; w <= 5; w++) 200 writel_relaxed(seq[i].word[w], reg); 201 202 if (cc_dump_desc) 203 dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", 204 i, seq[i].word[0], seq[i].word[1], 205 seq[i].word[2], seq[i].word[3], 206 seq[i].word[4], seq[i].word[5]); 207 } 208 } 209 210 /*! 211 * Completion will take place if and only if user requested completion 212 * by cc_send_sync_request(). 213 * 214 * \param dev 215 * \param dx_compl_h The completion event to signal 216 */ 217 static void request_mgr_complete(struct device *dev, void *dx_compl_h, 218 int dummy) 219 { 220 struct completion *this_compl = dx_compl_h; 221 222 complete(this_compl); 223 } 224 225 static int cc_queues_status(struct cc_drvdata *drvdata, 226 struct cc_req_mgr_handle *req_mgr_h, 227 unsigned int total_seq_len) 228 { 229 unsigned long poll_queue; 230 struct device *dev = drvdata_to_dev(drvdata); 231 232 /* SW queue is checked only once as it will not 233 * be chaned during the poll because the spinlock_bh 234 * is held by the thread 235 */ 236 if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) == 237 req_mgr_h->req_queue_tail) { 238 dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n", 239 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE); 240 return -ENOSPC; 241 } 242 243 if (req_mgr_h->q_free_slots >= total_seq_len) 244 return 0; 245 246 /* Wait for space in HW queue. Poll constant num of iterations. */ 247 for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) { 248 req_mgr_h->q_free_slots = 249 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 250 if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots) 251 req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots; 252 253 if (req_mgr_h->q_free_slots >= total_seq_len) { 254 /* If there is enough place return */ 255 return 0; 256 } 257 258 dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n", 259 req_mgr_h->q_free_slots, total_seq_len); 260 } 261 /* No room in the HW queue try again later */ 262 dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n", 263 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE, 264 req_mgr_h->q_free_slots, total_seq_len); 265 return -ENOSPC; 266 } 267 268 /*! 269 * Enqueue caller request to crypto hardware. 270 * Need to be called with HW lock held and PM running 271 * 272 * \param drvdata 273 * \param cc_req The request to enqueue 274 * \param desc The crypto sequence 275 * \param len The crypto sequence length 276 * \param add_comp If "true": add an artificial dout DMA to mark completion 277 * 278 * \return int Returns -EINPROGRESS or error code 279 */ 280 static int cc_do_send_request(struct cc_drvdata *drvdata, 281 struct cc_crypto_req *cc_req, 282 struct cc_hw_desc *desc, unsigned int len, 283 bool add_comp) 284 { 285 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 286 unsigned int used_sw_slots; 287 unsigned int total_seq_len = len; /*initial sequence length*/ 288 struct device *dev = drvdata_to_dev(drvdata); 289 290 used_sw_slots = ((req_mgr_h->req_queue_head - 291 req_mgr_h->req_queue_tail) & 292 (MAX_REQUEST_QUEUE_SIZE - 1)); 293 if (used_sw_slots > req_mgr_h->max_used_sw_slots) 294 req_mgr_h->max_used_sw_slots = used_sw_slots; 295 296 /* Enqueue request - must be locked with HW lock*/ 297 req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req; 298 req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) & 299 (MAX_REQUEST_QUEUE_SIZE - 1); 300 /* TODO: Use circ_buf.h ? */ 301 302 dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head); 303 304 /* 305 * We are about to push command to the HW via the command registers 306 * that may refernece hsot memory. We need to issue a memory barrier 307 * to make sure there are no outstnading memory writes 308 */ 309 wmb(); 310 311 /* STAT_PHASE_4: Push sequence */ 312 313 enqueue_seq(drvdata, desc, len); 314 315 if (add_comp) { 316 enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1); 317 total_seq_len++; 318 } 319 320 if (req_mgr_h->q_free_slots < total_seq_len) { 321 /* This situation should never occur. Maybe indicating problem 322 * with resuming power. Set the free slot count to 0 and hope 323 * for the best. 324 */ 325 dev_err(dev, "HW free slot count mismatch."); 326 req_mgr_h->q_free_slots = 0; 327 } else { 328 /* Update the free slots in HW queue */ 329 req_mgr_h->q_free_slots -= total_seq_len; 330 } 331 332 /* Operation still in process */ 333 return -EINPROGRESS; 334 } 335 336 static void cc_enqueue_backlog(struct cc_drvdata *drvdata, 337 struct cc_bl_item *bli) 338 { 339 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 340 struct device *dev = drvdata_to_dev(drvdata); 341 342 spin_lock_bh(&mgr->bl_lock); 343 list_add_tail(&bli->list, &mgr->backlog); 344 ++mgr->bl_len; 345 dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len); 346 spin_unlock_bh(&mgr->bl_lock); 347 tasklet_schedule(&mgr->comptask); 348 } 349 350 static void cc_proc_backlog(struct cc_drvdata *drvdata) 351 { 352 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 353 struct cc_bl_item *bli; 354 struct cc_crypto_req *creq; 355 void *req; 356 struct device *dev = drvdata_to_dev(drvdata); 357 int rc; 358 359 spin_lock(&mgr->bl_lock); 360 361 while (mgr->bl_len) { 362 bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list); 363 dev_dbg(dev, "---bl len: %d\n", mgr->bl_len); 364 365 spin_unlock(&mgr->bl_lock); 366 367 368 creq = &bli->creq; 369 req = creq->user_arg; 370 371 /* 372 * Notify the request we're moving out of the backlog 373 * but only if we haven't done so already. 374 */ 375 if (!bli->notif) { 376 creq->user_cb(dev, req, -EINPROGRESS); 377 bli->notif = true; 378 } 379 380 spin_lock(&mgr->hw_lock); 381 382 rc = cc_queues_status(drvdata, mgr, bli->len); 383 if (rc) { 384 /* 385 * There is still not room in the FIFO for 386 * this request. Bail out. We'll return here 387 * on the next completion irq. 388 */ 389 spin_unlock(&mgr->hw_lock); 390 return; 391 } 392 393 rc = cc_do_send_request(drvdata, &bli->creq, bli->desc, 394 bli->len, false); 395 396 spin_unlock(&mgr->hw_lock); 397 398 if (rc != -EINPROGRESS) { 399 cc_pm_put_suspend(dev); 400 creq->user_cb(dev, req, rc); 401 } 402 403 /* Remove ourselves from the backlog list */ 404 spin_lock(&mgr->bl_lock); 405 list_del(&bli->list); 406 --mgr->bl_len; 407 } 408 409 spin_unlock(&mgr->bl_lock); 410 } 411 412 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, 413 struct cc_hw_desc *desc, unsigned int len, 414 struct crypto_async_request *req) 415 { 416 int rc; 417 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 418 struct device *dev = drvdata_to_dev(drvdata); 419 bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; 420 gfp_t flags = cc_gfp_flags(req); 421 struct cc_bl_item *bli; 422 423 rc = cc_pm_get(dev); 424 if (rc) { 425 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc); 426 return rc; 427 } 428 429 spin_lock_bh(&mgr->hw_lock); 430 rc = cc_queues_status(drvdata, mgr, len); 431 432 #ifdef CC_DEBUG_FORCE_BACKLOG 433 if (backlog_ok) 434 rc = -ENOSPC; 435 #endif /* CC_DEBUG_FORCE_BACKLOG */ 436 437 if (rc == -ENOSPC && backlog_ok) { 438 spin_unlock_bh(&mgr->hw_lock); 439 440 bli = kmalloc(sizeof(*bli), flags); 441 if (!bli) { 442 cc_pm_put_suspend(dev); 443 return -ENOMEM; 444 } 445 446 memcpy(&bli->creq, cc_req, sizeof(*cc_req)); 447 memcpy(&bli->desc, desc, len * sizeof(*desc)); 448 bli->len = len; 449 bli->notif = false; 450 cc_enqueue_backlog(drvdata, bli); 451 return -EBUSY; 452 } 453 454 if (!rc) 455 rc = cc_do_send_request(drvdata, cc_req, desc, len, false); 456 457 spin_unlock_bh(&mgr->hw_lock); 458 return rc; 459 } 460 461 int cc_send_sync_request(struct cc_drvdata *drvdata, 462 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc, 463 unsigned int len) 464 { 465 int rc; 466 struct device *dev = drvdata_to_dev(drvdata); 467 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 468 469 init_completion(&cc_req->seq_compl); 470 cc_req->user_cb = request_mgr_complete; 471 cc_req->user_arg = &cc_req->seq_compl; 472 473 rc = cc_pm_get(dev); 474 if (rc) { 475 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc); 476 return rc; 477 } 478 479 while (true) { 480 spin_lock_bh(&mgr->hw_lock); 481 rc = cc_queues_status(drvdata, mgr, len + 1); 482 483 if (!rc) 484 break; 485 486 spin_unlock_bh(&mgr->hw_lock); 487 if (rc != -EAGAIN) { 488 cc_pm_put_suspend(dev); 489 return rc; 490 } 491 wait_for_completion_interruptible(&drvdata->hw_queue_avail); 492 reinit_completion(&drvdata->hw_queue_avail); 493 } 494 495 rc = cc_do_send_request(drvdata, cc_req, desc, len, true); 496 spin_unlock_bh(&mgr->hw_lock); 497 498 if (rc != -EINPROGRESS) { 499 cc_pm_put_suspend(dev); 500 return rc; 501 } 502 503 wait_for_completion(&cc_req->seq_compl); 504 return 0; 505 } 506 507 /*! 508 * Enqueue caller request to crypto hardware during init process. 509 * assume this function is not called in middle of a flow, 510 * since we set QUEUE_LAST_IND flag in the last descriptor. 511 * 512 * \param drvdata 513 * \param desc The crypto sequence 514 * \param len The crypto sequence length 515 * 516 * \return int Returns "0" upon success 517 */ 518 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc, 519 unsigned int len) 520 { 521 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 522 unsigned int total_seq_len = len; /*initial sequence length*/ 523 int rc = 0; 524 525 /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT. 526 */ 527 rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len); 528 if (rc) 529 return rc; 530 531 set_queue_last_ind(drvdata, &desc[(len - 1)]); 532 533 /* 534 * We are about to push command to the HW via the command registers 535 * that may refernece hsot memory. We need to issue a memory barrier 536 * to make sure there are no outstnading memory writes 537 */ 538 wmb(); 539 enqueue_seq(drvdata, desc, len); 540 541 /* Update the free slots in HW queue */ 542 req_mgr_h->q_free_slots = 543 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 544 545 return 0; 546 } 547 548 void complete_request(struct cc_drvdata *drvdata) 549 { 550 struct cc_req_mgr_handle *request_mgr_handle = 551 drvdata->request_mgr_handle; 552 553 complete(&drvdata->hw_queue_avail); 554 #ifdef COMP_IN_WQ 555 queue_delayed_work(request_mgr_handle->workq, 556 &request_mgr_handle->compwork, 0); 557 #else 558 tasklet_schedule(&request_mgr_handle->comptask); 559 #endif 560 } 561 562 #ifdef COMP_IN_WQ 563 static void comp_work_handler(struct work_struct *work) 564 { 565 struct cc_drvdata *drvdata = 566 container_of(work, struct cc_drvdata, compwork.work); 567 568 comp_handler((unsigned long)drvdata); 569 } 570 #endif 571 572 static void proc_completions(struct cc_drvdata *drvdata) 573 { 574 struct cc_crypto_req *cc_req; 575 struct device *dev = drvdata_to_dev(drvdata); 576 struct cc_req_mgr_handle *request_mgr_handle = 577 drvdata->request_mgr_handle; 578 unsigned int *tail = &request_mgr_handle->req_queue_tail; 579 unsigned int *head = &request_mgr_handle->req_queue_head; 580 int rc; 581 u32 mask; 582 583 while (request_mgr_handle->axi_completed) { 584 request_mgr_handle->axi_completed--; 585 586 /* Dequeue request */ 587 if (*head == *tail) { 588 /* We are supposed to handle a completion but our 589 * queue is empty. This is not normal. Return and 590 * hope for the best. 591 */ 592 dev_err(dev, "Request queue is empty head == tail %u\n", 593 *head); 594 break; 595 } 596 597 cc_req = &request_mgr_handle->req_queue[*tail]; 598 599 if (cc_req->cpp.is_cpp) { 600 601 dev_dbg(dev, "CPP request completion slot: %d alg:%d\n", 602 cc_req->cpp.slot, cc_req->cpp.alg); 603 mask = cc_cpp_int_mask(cc_req->cpp.alg, 604 cc_req->cpp.slot); 605 rc = (drvdata->irq & mask ? -EPERM : 0); 606 dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask, 607 drvdata->irq, rc); 608 } else { 609 dev_dbg(dev, "None CPP request completion\n"); 610 rc = 0; 611 } 612 613 if (cc_req->user_cb) 614 cc_req->user_cb(dev, cc_req->user_arg, rc); 615 *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); 616 dev_dbg(dev, "Dequeue request tail=%u\n", *tail); 617 dev_dbg(dev, "Request completed. axi_completed=%d\n", 618 request_mgr_handle->axi_completed); 619 cc_pm_put_suspend(dev); 620 } 621 } 622 623 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata) 624 { 625 return FIELD_GET(AXIM_MON_COMP_VALUE, 626 cc_ioread(drvdata, drvdata->axim_mon_offset)); 627 } 628 629 /* Deferred service handler, run as interrupt-fired tasklet */ 630 static void comp_handler(unsigned long devarg) 631 { 632 struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; 633 struct cc_req_mgr_handle *request_mgr_handle = 634 drvdata->request_mgr_handle; 635 struct device *dev = drvdata_to_dev(drvdata); 636 u32 irq; 637 638 dev_dbg(dev, "Completion handler called!\n"); 639 irq = (drvdata->irq & drvdata->comp_mask); 640 641 /* To avoid the interrupt from firing as we unmask it, 642 * we clear it now 643 */ 644 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); 645 646 /* Avoid race with above clear: Test completion counter once more */ 647 648 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); 649 650 dev_dbg(dev, "AXI completion after updated: %d\n", 651 request_mgr_handle->axi_completed); 652 653 while (request_mgr_handle->axi_completed) { 654 do { 655 drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR)); 656 irq = (drvdata->irq & drvdata->comp_mask); 657 proc_completions(drvdata); 658 659 /* At this point (after proc_completions()), 660 * request_mgr_handle->axi_completed is 0. 661 */ 662 request_mgr_handle->axi_completed += 663 cc_axi_comp_count(drvdata); 664 } while (request_mgr_handle->axi_completed > 0); 665 666 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); 667 668 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); 669 } 670 671 /* after verifing that there is nothing to do, 672 * unmask AXI completion interrupt 673 */ 674 cc_iowrite(drvdata, CC_REG(HOST_IMR), 675 cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask); 676 677 cc_proc_backlog(drvdata); 678 dev_dbg(dev, "Comp. handler done.\n"); 679 } 680 681 /* 682 * resume the queue configuration - no need to take the lock as this happens 683 * inside the spin lock protection 684 */ 685 #if defined(CONFIG_PM) 686 int cc_resume_req_queue(struct cc_drvdata *drvdata) 687 { 688 struct cc_req_mgr_handle *request_mgr_handle = 689 drvdata->request_mgr_handle; 690 691 spin_lock_bh(&request_mgr_handle->hw_lock); 692 request_mgr_handle->is_runtime_suspended = false; 693 spin_unlock_bh(&request_mgr_handle->hw_lock); 694 695 return 0; 696 } 697 698 /* 699 * suspend the queue configuration. Since it is used for the runtime suspend 700 * only verify that the queue can be suspended. 701 */ 702 int cc_suspend_req_queue(struct cc_drvdata *drvdata) 703 { 704 struct cc_req_mgr_handle *request_mgr_handle = 705 drvdata->request_mgr_handle; 706 707 /* lock the send_request */ 708 spin_lock_bh(&request_mgr_handle->hw_lock); 709 if (request_mgr_handle->req_queue_head != 710 request_mgr_handle->req_queue_tail) { 711 spin_unlock_bh(&request_mgr_handle->hw_lock); 712 return -EBUSY; 713 } 714 request_mgr_handle->is_runtime_suspended = true; 715 spin_unlock_bh(&request_mgr_handle->hw_lock); 716 717 return 0; 718 } 719 720 bool cc_req_queue_suspended(struct cc_drvdata *drvdata) 721 { 722 struct cc_req_mgr_handle *request_mgr_handle = 723 drvdata->request_mgr_handle; 724 725 return request_mgr_handle->is_runtime_suspended; 726 } 727 728 #endif 729