1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/nospec.h> 6 #include "cc_driver.h" 7 #include "cc_buffer_mgr.h" 8 #include "cc_request_mgr.h" 9 #include "cc_ivgen.h" 10 #include "cc_pm.h" 11 12 #define CC_MAX_POLL_ITER 10 13 /* The highest descriptor count in used */ 14 #define CC_MAX_DESC_SEQ_LEN 23 15 16 struct cc_req_mgr_handle { 17 /* Request manager resources */ 18 unsigned int hw_queue_size; /* HW capability */ 19 unsigned int min_free_hw_slots; 20 unsigned int max_used_sw_slots; 21 struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE]; 22 u32 req_queue_head; 23 u32 req_queue_tail; 24 u32 axi_completed; 25 u32 q_free_slots; 26 /* This lock protects access to HW register 27 * that must be single request at a time 28 */ 29 spinlock_t hw_lock; 30 struct cc_hw_desc compl_desc; 31 u8 *dummy_comp_buff; 32 dma_addr_t dummy_comp_buff_dma; 33 34 /* backlog queue */ 35 struct list_head backlog; 36 unsigned int bl_len; 37 spinlock_t bl_lock; /* protect backlog queue */ 38 39 #ifdef COMP_IN_WQ 40 struct workqueue_struct *workq; 41 struct delayed_work compwork; 42 #else 43 struct tasklet_struct comptask; 44 #endif 45 bool is_runtime_suspended; 46 }; 47 48 struct cc_bl_item { 49 struct cc_crypto_req creq; 50 struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN]; 51 unsigned int len; 52 struct list_head list; 53 bool notif; 54 }; 55 56 static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = { 57 { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT), 58 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT), 59 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT), 60 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT), 61 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT), 62 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT), 63 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT), 64 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) }, 65 { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT), 66 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT), 67 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT), 68 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT), 69 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT), 70 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT), 71 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT), 72 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) } 73 }; 74 75 static void comp_handler(unsigned long devarg); 76 #ifdef COMP_IN_WQ 77 static void comp_work_handler(struct work_struct *work); 78 #endif 79 80 static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot) 81 { 82 alg = array_index_nospec(alg, CC_CPP_NUM_ALGS); 83 slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS); 84 85 return cc_cpp_int_masks[alg][slot]; 86 } 87 88 void cc_req_mgr_fini(struct cc_drvdata *drvdata) 89 { 90 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 91 struct device *dev = drvdata_to_dev(drvdata); 92 93 if (!req_mgr_h) 94 return; /* Not allocated */ 95 96 if (req_mgr_h->dummy_comp_buff_dma) { 97 dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff, 98 req_mgr_h->dummy_comp_buff_dma); 99 } 100 101 dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size - 102 req_mgr_h->min_free_hw_slots)); 103 dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots); 104 105 #ifdef COMP_IN_WQ 106 flush_workqueue(req_mgr_h->workq); 107 destroy_workqueue(req_mgr_h->workq); 108 #else 109 /* Kill tasklet */ 110 tasklet_kill(&req_mgr_h->comptask); 111 #endif 112 kzfree(req_mgr_h); 113 drvdata->request_mgr_handle = NULL; 114 } 115 116 int cc_req_mgr_init(struct cc_drvdata *drvdata) 117 { 118 struct cc_req_mgr_handle *req_mgr_h; 119 struct device *dev = drvdata_to_dev(drvdata); 120 int rc = 0; 121 122 req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL); 123 if (!req_mgr_h) { 124 rc = -ENOMEM; 125 goto req_mgr_init_err; 126 } 127 128 drvdata->request_mgr_handle = req_mgr_h; 129 130 spin_lock_init(&req_mgr_h->hw_lock); 131 spin_lock_init(&req_mgr_h->bl_lock); 132 INIT_LIST_HEAD(&req_mgr_h->backlog); 133 134 #ifdef COMP_IN_WQ 135 dev_dbg(dev, "Initializing completion workqueue\n"); 136 req_mgr_h->workq = create_singlethread_workqueue("ccree"); 137 if (!req_mgr_h->workq) { 138 dev_err(dev, "Failed creating work queue\n"); 139 rc = -ENOMEM; 140 goto req_mgr_init_err; 141 } 142 INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler); 143 #else 144 dev_dbg(dev, "Initializing completion tasklet\n"); 145 tasklet_init(&req_mgr_h->comptask, comp_handler, 146 (unsigned long)drvdata); 147 #endif 148 req_mgr_h->hw_queue_size = cc_ioread(drvdata, 149 CC_REG(DSCRPTR_QUEUE_SRAM_SIZE)); 150 dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size); 151 if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) { 152 dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n", 153 req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE); 154 rc = -ENOMEM; 155 goto req_mgr_init_err; 156 } 157 req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size; 158 req_mgr_h->max_used_sw_slots = 0; 159 160 /* Allocate DMA word for "dummy" completion descriptor use */ 161 req_mgr_h->dummy_comp_buff = 162 dma_alloc_coherent(dev, sizeof(u32), 163 &req_mgr_h->dummy_comp_buff_dma, 164 GFP_KERNEL); 165 if (!req_mgr_h->dummy_comp_buff) { 166 dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n", 167 sizeof(u32)); 168 rc = -ENOMEM; 169 goto req_mgr_init_err; 170 } 171 172 /* Init. "dummy" completion descriptor */ 173 hw_desc_init(&req_mgr_h->compl_desc); 174 set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32)); 175 set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma, 176 sizeof(u32), NS_BIT, 1); 177 set_flow_mode(&req_mgr_h->compl_desc, BYPASS); 178 set_queue_last_ind(drvdata, &req_mgr_h->compl_desc); 179 180 return 0; 181 182 req_mgr_init_err: 183 cc_req_mgr_fini(drvdata); 184 return rc; 185 } 186 187 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[], 188 unsigned int seq_len) 189 { 190 int i, w; 191 void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0); 192 struct device *dev = drvdata_to_dev(drvdata); 193 194 /* 195 * We do indeed write all 6 command words to the same 196 * register. The HW supports this. 197 */ 198 199 for (i = 0; i < seq_len; i++) { 200 for (w = 0; w <= 5; w++) 201 writel_relaxed(seq[i].word[w], reg); 202 203 if (cc_dump_desc) 204 dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", 205 i, seq[i].word[0], seq[i].word[1], 206 seq[i].word[2], seq[i].word[3], 207 seq[i].word[4], seq[i].word[5]); 208 } 209 } 210 211 /*! 212 * Completion will take place if and only if user requested completion 213 * by cc_send_sync_request(). 214 * 215 * \param dev 216 * \param dx_compl_h The completion event to signal 217 */ 218 static void request_mgr_complete(struct device *dev, void *dx_compl_h, 219 int dummy) 220 { 221 struct completion *this_compl = dx_compl_h; 222 223 complete(this_compl); 224 } 225 226 static int cc_queues_status(struct cc_drvdata *drvdata, 227 struct cc_req_mgr_handle *req_mgr_h, 228 unsigned int total_seq_len) 229 { 230 unsigned long poll_queue; 231 struct device *dev = drvdata_to_dev(drvdata); 232 233 /* SW queue is checked only once as it will not 234 * be chaned during the poll because the spinlock_bh 235 * is held by the thread 236 */ 237 if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) == 238 req_mgr_h->req_queue_tail) { 239 dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n", 240 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE); 241 return -ENOSPC; 242 } 243 244 if (req_mgr_h->q_free_slots >= total_seq_len) 245 return 0; 246 247 /* Wait for space in HW queue. Poll constant num of iterations. */ 248 for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) { 249 req_mgr_h->q_free_slots = 250 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 251 if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots) 252 req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots; 253 254 if (req_mgr_h->q_free_slots >= total_seq_len) { 255 /* If there is enough place return */ 256 return 0; 257 } 258 259 dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n", 260 req_mgr_h->q_free_slots, total_seq_len); 261 } 262 /* No room in the HW queue try again later */ 263 dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n", 264 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE, 265 req_mgr_h->q_free_slots, total_seq_len); 266 return -ENOSPC; 267 } 268 269 /*! 270 * Enqueue caller request to crypto hardware. 271 * Need to be called with HW lock held and PM running 272 * 273 * \param drvdata 274 * \param cc_req The request to enqueue 275 * \param desc The crypto sequence 276 * \param len The crypto sequence length 277 * \param add_comp If "true": add an artificial dout DMA to mark completion 278 * 279 * \return int Returns -EINPROGRESS or error code 280 */ 281 static int cc_do_send_request(struct cc_drvdata *drvdata, 282 struct cc_crypto_req *cc_req, 283 struct cc_hw_desc *desc, unsigned int len, 284 bool add_comp, bool ivgen) 285 { 286 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 287 unsigned int used_sw_slots; 288 unsigned int iv_seq_len = 0; 289 unsigned int total_seq_len = len; /*initial sequence length*/ 290 struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN]; 291 struct device *dev = drvdata_to_dev(drvdata); 292 int rc; 293 294 if (ivgen) { 295 dev_dbg(dev, "Acquire IV from pool into %d DMA addresses %pad, %pad, %pad, IV-size=%u\n", 296 cc_req->ivgen_dma_addr_len, 297 &cc_req->ivgen_dma_addr[0], 298 &cc_req->ivgen_dma_addr[1], 299 &cc_req->ivgen_dma_addr[2], 300 cc_req->ivgen_size); 301 302 /* Acquire IV from pool */ 303 rc = cc_get_iv(drvdata, cc_req->ivgen_dma_addr, 304 cc_req->ivgen_dma_addr_len, 305 cc_req->ivgen_size, iv_seq, &iv_seq_len); 306 307 if (rc) { 308 dev_err(dev, "Failed to generate IV (rc=%d)\n", rc); 309 return rc; 310 } 311 312 total_seq_len += iv_seq_len; 313 } 314 315 used_sw_slots = ((req_mgr_h->req_queue_head - 316 req_mgr_h->req_queue_tail) & 317 (MAX_REQUEST_QUEUE_SIZE - 1)); 318 if (used_sw_slots > req_mgr_h->max_used_sw_slots) 319 req_mgr_h->max_used_sw_slots = used_sw_slots; 320 321 /* Enqueue request - must be locked with HW lock*/ 322 req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req; 323 req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) & 324 (MAX_REQUEST_QUEUE_SIZE - 1); 325 /* TODO: Use circ_buf.h ? */ 326 327 dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head); 328 329 /* 330 * We are about to push command to the HW via the command registers 331 * that may refernece hsot memory. We need to issue a memory barrier 332 * to make sure there are no outstnading memory writes 333 */ 334 wmb(); 335 336 /* STAT_PHASE_4: Push sequence */ 337 if (ivgen) 338 enqueue_seq(drvdata, iv_seq, iv_seq_len); 339 340 enqueue_seq(drvdata, desc, len); 341 342 if (add_comp) { 343 enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1); 344 total_seq_len++; 345 } 346 347 if (req_mgr_h->q_free_slots < total_seq_len) { 348 /* This situation should never occur. Maybe indicating problem 349 * with resuming power. Set the free slot count to 0 and hope 350 * for the best. 351 */ 352 dev_err(dev, "HW free slot count mismatch."); 353 req_mgr_h->q_free_slots = 0; 354 } else { 355 /* Update the free slots in HW queue */ 356 req_mgr_h->q_free_slots -= total_seq_len; 357 } 358 359 /* Operation still in process */ 360 return -EINPROGRESS; 361 } 362 363 static void cc_enqueue_backlog(struct cc_drvdata *drvdata, 364 struct cc_bl_item *bli) 365 { 366 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 367 struct device *dev = drvdata_to_dev(drvdata); 368 369 spin_lock_bh(&mgr->bl_lock); 370 list_add_tail(&bli->list, &mgr->backlog); 371 ++mgr->bl_len; 372 dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len); 373 spin_unlock_bh(&mgr->bl_lock); 374 tasklet_schedule(&mgr->comptask); 375 } 376 377 static void cc_proc_backlog(struct cc_drvdata *drvdata) 378 { 379 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 380 struct cc_bl_item *bli; 381 struct cc_crypto_req *creq; 382 void *req; 383 bool ivgen; 384 unsigned int total_len; 385 struct device *dev = drvdata_to_dev(drvdata); 386 int rc; 387 388 spin_lock(&mgr->bl_lock); 389 390 while (mgr->bl_len) { 391 bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list); 392 dev_dbg(dev, "---bl len: %d\n", mgr->bl_len); 393 394 spin_unlock(&mgr->bl_lock); 395 396 397 creq = &bli->creq; 398 req = creq->user_arg; 399 400 /* 401 * Notify the request we're moving out of the backlog 402 * but only if we haven't done so already. 403 */ 404 if (!bli->notif) { 405 creq->user_cb(dev, req, -EINPROGRESS); 406 bli->notif = true; 407 } 408 409 ivgen = !!creq->ivgen_dma_addr_len; 410 total_len = bli->len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0); 411 412 spin_lock(&mgr->hw_lock); 413 414 rc = cc_queues_status(drvdata, mgr, total_len); 415 if (rc) { 416 /* 417 * There is still not room in the FIFO for 418 * this request. Bail out. We'll return here 419 * on the next completion irq. 420 */ 421 spin_unlock(&mgr->hw_lock); 422 return; 423 } 424 425 rc = cc_do_send_request(drvdata, &bli->creq, bli->desc, 426 bli->len, false, ivgen); 427 428 spin_unlock(&mgr->hw_lock); 429 430 if (rc != -EINPROGRESS) { 431 cc_pm_put_suspend(dev); 432 creq->user_cb(dev, req, rc); 433 } 434 435 /* Remove ourselves from the backlog list */ 436 spin_lock(&mgr->bl_lock); 437 list_del(&bli->list); 438 --mgr->bl_len; 439 } 440 441 spin_unlock(&mgr->bl_lock); 442 } 443 444 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, 445 struct cc_hw_desc *desc, unsigned int len, 446 struct crypto_async_request *req) 447 { 448 int rc; 449 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 450 bool ivgen = !!cc_req->ivgen_dma_addr_len; 451 unsigned int total_len = len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0); 452 struct device *dev = drvdata_to_dev(drvdata); 453 bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; 454 gfp_t flags = cc_gfp_flags(req); 455 struct cc_bl_item *bli; 456 457 rc = cc_pm_get(dev); 458 if (rc) { 459 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc); 460 return rc; 461 } 462 463 spin_lock_bh(&mgr->hw_lock); 464 rc = cc_queues_status(drvdata, mgr, total_len); 465 466 #ifdef CC_DEBUG_FORCE_BACKLOG 467 if (backlog_ok) 468 rc = -ENOSPC; 469 #endif /* CC_DEBUG_FORCE_BACKLOG */ 470 471 if (rc == -ENOSPC && backlog_ok) { 472 spin_unlock_bh(&mgr->hw_lock); 473 474 bli = kmalloc(sizeof(*bli), flags); 475 if (!bli) { 476 cc_pm_put_suspend(dev); 477 return -ENOMEM; 478 } 479 480 memcpy(&bli->creq, cc_req, sizeof(*cc_req)); 481 memcpy(&bli->desc, desc, len * sizeof(*desc)); 482 bli->len = len; 483 bli->notif = false; 484 cc_enqueue_backlog(drvdata, bli); 485 return -EBUSY; 486 } 487 488 if (!rc) 489 rc = cc_do_send_request(drvdata, cc_req, desc, len, false, 490 ivgen); 491 492 spin_unlock_bh(&mgr->hw_lock); 493 return rc; 494 } 495 496 int cc_send_sync_request(struct cc_drvdata *drvdata, 497 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc, 498 unsigned int len) 499 { 500 int rc; 501 struct device *dev = drvdata_to_dev(drvdata); 502 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 503 504 init_completion(&cc_req->seq_compl); 505 cc_req->user_cb = request_mgr_complete; 506 cc_req->user_arg = &cc_req->seq_compl; 507 508 rc = cc_pm_get(dev); 509 if (rc) { 510 dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc); 511 return rc; 512 } 513 514 while (true) { 515 spin_lock_bh(&mgr->hw_lock); 516 rc = cc_queues_status(drvdata, mgr, len + 1); 517 518 if (!rc) 519 break; 520 521 spin_unlock_bh(&mgr->hw_lock); 522 if (rc != -EAGAIN) { 523 cc_pm_put_suspend(dev); 524 return rc; 525 } 526 wait_for_completion_interruptible(&drvdata->hw_queue_avail); 527 reinit_completion(&drvdata->hw_queue_avail); 528 } 529 530 rc = cc_do_send_request(drvdata, cc_req, desc, len, true, false); 531 spin_unlock_bh(&mgr->hw_lock); 532 533 if (rc != -EINPROGRESS) { 534 cc_pm_put_suspend(dev); 535 return rc; 536 } 537 538 wait_for_completion(&cc_req->seq_compl); 539 return 0; 540 } 541 542 /*! 543 * Enqueue caller request to crypto hardware during init process. 544 * assume this function is not called in middle of a flow, 545 * since we set QUEUE_LAST_IND flag in the last descriptor. 546 * 547 * \param drvdata 548 * \param desc The crypto sequence 549 * \param len The crypto sequence length 550 * 551 * \return int Returns "0" upon success 552 */ 553 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc, 554 unsigned int len) 555 { 556 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 557 unsigned int total_seq_len = len; /*initial sequence length*/ 558 int rc = 0; 559 560 /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT. 561 */ 562 rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len); 563 if (rc) 564 return rc; 565 566 set_queue_last_ind(drvdata, &desc[(len - 1)]); 567 568 /* 569 * We are about to push command to the HW via the command registers 570 * that may refernece hsot memory. We need to issue a memory barrier 571 * to make sure there are no outstnading memory writes 572 */ 573 wmb(); 574 enqueue_seq(drvdata, desc, len); 575 576 /* Update the free slots in HW queue */ 577 req_mgr_h->q_free_slots = 578 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 579 580 return 0; 581 } 582 583 void complete_request(struct cc_drvdata *drvdata) 584 { 585 struct cc_req_mgr_handle *request_mgr_handle = 586 drvdata->request_mgr_handle; 587 588 complete(&drvdata->hw_queue_avail); 589 #ifdef COMP_IN_WQ 590 queue_delayed_work(request_mgr_handle->workq, 591 &request_mgr_handle->compwork, 0); 592 #else 593 tasklet_schedule(&request_mgr_handle->comptask); 594 #endif 595 } 596 597 #ifdef COMP_IN_WQ 598 static void comp_work_handler(struct work_struct *work) 599 { 600 struct cc_drvdata *drvdata = 601 container_of(work, struct cc_drvdata, compwork.work); 602 603 comp_handler((unsigned long)drvdata); 604 } 605 #endif 606 607 static void proc_completions(struct cc_drvdata *drvdata) 608 { 609 struct cc_crypto_req *cc_req; 610 struct device *dev = drvdata_to_dev(drvdata); 611 struct cc_req_mgr_handle *request_mgr_handle = 612 drvdata->request_mgr_handle; 613 unsigned int *tail = &request_mgr_handle->req_queue_tail; 614 unsigned int *head = &request_mgr_handle->req_queue_head; 615 int rc; 616 u32 mask; 617 618 while (request_mgr_handle->axi_completed) { 619 request_mgr_handle->axi_completed--; 620 621 /* Dequeue request */ 622 if (*head == *tail) { 623 /* We are supposed to handle a completion but our 624 * queue is empty. This is not normal. Return and 625 * hope for the best. 626 */ 627 dev_err(dev, "Request queue is empty head == tail %u\n", 628 *head); 629 break; 630 } 631 632 cc_req = &request_mgr_handle->req_queue[*tail]; 633 634 if (cc_req->cpp.is_cpp) { 635 636 dev_dbg(dev, "CPP request completion slot: %d alg:%d\n", 637 cc_req->cpp.slot, cc_req->cpp.alg); 638 mask = cc_cpp_int_mask(cc_req->cpp.alg, 639 cc_req->cpp.slot); 640 rc = (drvdata->irq & mask ? -EPERM : 0); 641 dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask, 642 drvdata->irq, rc); 643 } else { 644 dev_dbg(dev, "None CPP request completion\n"); 645 rc = 0; 646 } 647 648 if (cc_req->user_cb) 649 cc_req->user_cb(dev, cc_req->user_arg, rc); 650 *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); 651 dev_dbg(dev, "Dequeue request tail=%u\n", *tail); 652 dev_dbg(dev, "Request completed. axi_completed=%d\n", 653 request_mgr_handle->axi_completed); 654 cc_pm_put_suspend(dev); 655 } 656 } 657 658 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata) 659 { 660 return FIELD_GET(AXIM_MON_COMP_VALUE, 661 cc_ioread(drvdata, drvdata->axim_mon_offset)); 662 } 663 664 /* Deferred service handler, run as interrupt-fired tasklet */ 665 static void comp_handler(unsigned long devarg) 666 { 667 struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; 668 struct cc_req_mgr_handle *request_mgr_handle = 669 drvdata->request_mgr_handle; 670 struct device *dev = drvdata_to_dev(drvdata); 671 u32 irq; 672 673 dev_dbg(dev, "Completion handler called!\n"); 674 irq = (drvdata->irq & drvdata->comp_mask); 675 676 /* To avoid the interrupt from firing as we unmask it, 677 * we clear it now 678 */ 679 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); 680 681 /* Avoid race with above clear: Test completion counter once more */ 682 683 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); 684 685 dev_dbg(dev, "AXI completion after updated: %d\n", 686 request_mgr_handle->axi_completed); 687 688 while (request_mgr_handle->axi_completed) { 689 do { 690 drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR)); 691 irq = (drvdata->irq & drvdata->comp_mask); 692 proc_completions(drvdata); 693 694 /* At this point (after proc_completions()), 695 * request_mgr_handle->axi_completed is 0. 696 */ 697 request_mgr_handle->axi_completed += 698 cc_axi_comp_count(drvdata); 699 } while (request_mgr_handle->axi_completed > 0); 700 701 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); 702 703 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); 704 } 705 706 /* after verifing that there is nothing to do, 707 * unmask AXI completion interrupt 708 */ 709 cc_iowrite(drvdata, CC_REG(HOST_IMR), 710 cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask); 711 712 cc_proc_backlog(drvdata); 713 dev_dbg(dev, "Comp. handler done.\n"); 714 } 715 716 /* 717 * resume the queue configuration - no need to take the lock as this happens 718 * inside the spin lock protection 719 */ 720 #if defined(CONFIG_PM) 721 int cc_resume_req_queue(struct cc_drvdata *drvdata) 722 { 723 struct cc_req_mgr_handle *request_mgr_handle = 724 drvdata->request_mgr_handle; 725 726 spin_lock_bh(&request_mgr_handle->hw_lock); 727 request_mgr_handle->is_runtime_suspended = false; 728 spin_unlock_bh(&request_mgr_handle->hw_lock); 729 730 return 0; 731 } 732 733 /* 734 * suspend the queue configuration. Since it is used for the runtime suspend 735 * only verify that the queue can be suspended. 736 */ 737 int cc_suspend_req_queue(struct cc_drvdata *drvdata) 738 { 739 struct cc_req_mgr_handle *request_mgr_handle = 740 drvdata->request_mgr_handle; 741 742 /* lock the send_request */ 743 spin_lock_bh(&request_mgr_handle->hw_lock); 744 if (request_mgr_handle->req_queue_head != 745 request_mgr_handle->req_queue_tail) { 746 spin_unlock_bh(&request_mgr_handle->hw_lock); 747 return -EBUSY; 748 } 749 request_mgr_handle->is_runtime_suspended = true; 750 spin_unlock_bh(&request_mgr_handle->hw_lock); 751 752 return 0; 753 } 754 755 bool cc_req_queue_suspended(struct cc_drvdata *drvdata) 756 { 757 struct cc_req_mgr_handle *request_mgr_handle = 758 drvdata->request_mgr_handle; 759 760 return request_mgr_handle->is_runtime_suspended; 761 } 762 763 #endif 764