1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTX CPT driver 3 * 4 * Copyright (C) 2019 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include "otx_cptvf.h" 12 #include "otx_cptvf_algs.h" 13 14 /* Completion code size and initial value */ 15 #define COMPLETION_CODE_SIZE 8 16 #define COMPLETION_CODE_INIT 0 17 18 /* SG list header size in bytes */ 19 #define SG_LIST_HDR_SIZE 8 20 21 /* Default timeout when waiting for free pending entry in us */ 22 #define CPT_PENTRY_TIMEOUT 1000 23 #define CPT_PENTRY_STEP 50 24 25 /* Default threshold for stopping and resuming sender requests */ 26 #define CPT_IQ_STOP_MARGIN 128 27 #define CPT_IQ_RESUME_MARGIN 512 28 29 #define CPT_DMA_ALIGN 128 30 31 void otx_cpt_dump_sg_list(struct pci_dev *pdev, struct otx_cpt_req_info *req) 32 { 33 int i; 34 35 pr_debug("Gather list size %d\n", req->incnt); 36 for (i = 0; i < req->incnt; i++) { 37 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, 38 req->in[i].size, req->in[i].vptr, 39 (void *) req->in[i].dma_addr); 40 pr_debug("Buffer hexdump (%d bytes)\n", 41 req->in[i].size); 42 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, 43 req->in[i].vptr, req->in[i].size, false); 44 } 45 46 pr_debug("Scatter list size %d\n", req->outcnt); 47 for (i = 0; i < req->outcnt; i++) { 48 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, 49 req->out[i].size, req->out[i].vptr, 50 (void *) req->out[i].dma_addr); 51 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); 52 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, 53 req->out[i].vptr, req->out[i].size, false); 54 } 55 } 56 57 static inline struct otx_cpt_pending_entry *get_free_pending_entry( 58 struct otx_cpt_pending_queue *q, 59 int qlen) 60 { 61 struct otx_cpt_pending_entry *ent = NULL; 62 63 ent = &q->head[q->rear]; 64 if (unlikely(ent->busy)) 65 return NULL; 66 67 q->rear++; 68 if (unlikely(q->rear == qlen)) 69 q->rear = 0; 70 71 return ent; 72 } 73 74 static inline u32 modulo_inc(u32 index, u32 length, u32 inc) 75 { 76 if (WARN_ON(inc > length)) 77 inc = length; 78 79 index += inc; 80 if (unlikely(index >= length)) 81 index -= length; 82 83 return index; 84 } 85 86 static inline void free_pentry(struct otx_cpt_pending_entry *pentry) 87 { 88 pentry->completion_addr = NULL; 89 pentry->info = NULL; 90 pentry->callback = NULL; 91 pentry->areq = NULL; 92 pentry->resume_sender = false; 93 pentry->busy = false; 94 } 95 96 static inline int setup_sgio_components(struct pci_dev *pdev, 97 struct otx_cpt_buf_ptr *list, 98 int buf_count, u8 *buffer) 99 { 100 struct otx_cpt_sglist_component *sg_ptr = NULL; 101 int ret = 0, i, j; 102 int components; 103 104 if (unlikely(!list)) { 105 dev_err(&pdev->dev, "Input list pointer is NULL\n"); 106 return -EFAULT; 107 } 108 109 for (i = 0; i < buf_count; i++) { 110 if (likely(list[i].vptr)) { 111 list[i].dma_addr = dma_map_single(&pdev->dev, 112 list[i].vptr, 113 list[i].size, 114 DMA_BIDIRECTIONAL); 115 if (unlikely(dma_mapping_error(&pdev->dev, 116 list[i].dma_addr))) { 117 dev_err(&pdev->dev, "Dma mapping failed\n"); 118 ret = -EIO; 119 goto sg_cleanup; 120 } 121 } 122 } 123 124 components = buf_count / 4; 125 sg_ptr = (struct otx_cpt_sglist_component *)buffer; 126 for (i = 0; i < components; i++) { 127 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); 128 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); 129 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); 130 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size); 131 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 132 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 133 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 134 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); 135 sg_ptr++; 136 } 137 components = buf_count % 4; 138 139 switch (components) { 140 case 3: 141 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size); 142 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); 143 /* Fall through */ 144 case 2: 145 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size); 146 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); 147 /* Fall through */ 148 case 1: 149 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size); 150 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); 151 break; 152 default: 153 break; 154 } 155 return ret; 156 157 sg_cleanup: 158 for (j = 0; j < i; j++) { 159 if (list[j].dma_addr) { 160 dma_unmap_single(&pdev->dev, list[i].dma_addr, 161 list[i].size, DMA_BIDIRECTIONAL); 162 } 163 164 list[j].dma_addr = 0; 165 } 166 return ret; 167 } 168 169 static inline int setup_sgio_list(struct pci_dev *pdev, 170 struct otx_cpt_info_buffer **pinfo, 171 struct otx_cpt_req_info *req, gfp_t gfp) 172 { 173 u32 dlen, align_dlen, info_len, rlen; 174 struct otx_cpt_info_buffer *info; 175 u16 g_sz_bytes, s_sz_bytes; 176 int align = CPT_DMA_ALIGN; 177 u32 total_mem_len; 178 179 if (unlikely(req->incnt > OTX_CPT_MAX_SG_IN_CNT || 180 req->outcnt > OTX_CPT_MAX_SG_OUT_CNT)) { 181 dev_err(&pdev->dev, "Error too many sg components\n"); 182 return -EINVAL; 183 } 184 185 g_sz_bytes = ((req->incnt + 3) / 4) * 186 sizeof(struct otx_cpt_sglist_component); 187 s_sz_bytes = ((req->outcnt + 3) / 4) * 188 sizeof(struct otx_cpt_sglist_component); 189 190 dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; 191 align_dlen = ALIGN(dlen, align); 192 info_len = ALIGN(sizeof(*info), align); 193 rlen = ALIGN(sizeof(union otx_cpt_res_s), align); 194 total_mem_len = align_dlen + info_len + rlen + COMPLETION_CODE_SIZE; 195 196 info = kzalloc(total_mem_len, gfp); 197 if (unlikely(!info)) { 198 dev_err(&pdev->dev, "Memory allocation failed\n"); 199 return -ENOMEM; 200 } 201 *pinfo = info; 202 info->dlen = dlen; 203 info->in_buffer = (u8 *)info + info_len; 204 205 ((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt); 206 ((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt); 207 ((u16 *)info->in_buffer)[2] = 0; 208 ((u16 *)info->in_buffer)[3] = 0; 209 210 /* Setup gather (input) components */ 211 if (setup_sgio_components(pdev, req->in, req->incnt, 212 &info->in_buffer[8])) { 213 dev_err(&pdev->dev, "Failed to setup gather list\n"); 214 return -EFAULT; 215 } 216 217 if (setup_sgio_components(pdev, req->out, req->outcnt, 218 &info->in_buffer[8 + g_sz_bytes])) { 219 dev_err(&pdev->dev, "Failed to setup scatter list\n"); 220 return -EFAULT; 221 } 222 223 info->dma_len = total_mem_len - info_len; 224 info->dptr_baddr = dma_map_single(&pdev->dev, (void *)info->in_buffer, 225 info->dma_len, DMA_BIDIRECTIONAL); 226 if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { 227 dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); 228 return -EIO; 229 } 230 /* 231 * Get buffer for union otx_cpt_res_s response 232 * structure and its physical address 233 */ 234 info->completion_addr = (u64 *)(info->in_buffer + align_dlen); 235 info->comp_baddr = info->dptr_baddr + align_dlen; 236 237 /* Create and initialize RPTR */ 238 info->out_buffer = (u8 *)info->completion_addr + rlen; 239 info->rptr_baddr = info->comp_baddr + rlen; 240 241 *((u64 *) info->out_buffer) = ~((u64) COMPLETION_CODE_INIT); 242 243 return 0; 244 } 245 246 247 static void cpt_fill_inst(union otx_cpt_inst_s *inst, 248 struct otx_cpt_info_buffer *info, 249 struct otx_cpt_iq_cmd *cmd) 250 { 251 inst->u[0] = 0x0; 252 inst->s.doneint = true; 253 inst->s.res_addr = (u64)info->comp_baddr; 254 inst->u[2] = 0x0; 255 inst->s.wq_ptr = 0; 256 inst->s.ei0 = cmd->cmd.u64; 257 inst->s.ei1 = cmd->dptr; 258 inst->s.ei2 = cmd->rptr; 259 inst->s.ei3 = cmd->cptr.u64; 260 } 261 262 /* 263 * On OcteonTX platform the parameter db_count is used as a count for ringing 264 * door bell. The valid values for db_count are: 265 * 0 - 1 CPT instruction will be enqueued however CPT will not be informed 266 * 1 - 1 CPT instruction will be enqueued and CPT will be informed 267 */ 268 static void cpt_send_cmd(union otx_cpt_inst_s *cptinst, struct otx_cptvf *cptvf) 269 { 270 struct otx_cpt_cmd_qinfo *qinfo = &cptvf->cqinfo; 271 struct otx_cpt_cmd_queue *queue; 272 struct otx_cpt_cmd_chunk *curr; 273 u8 *ent; 274 275 queue = &qinfo->queue[0]; 276 /* 277 * cpt_send_cmd is currently called only from critical section 278 * therefore no locking is required for accessing instruction queue 279 */ 280 ent = &queue->qhead->head[queue->idx * OTX_CPT_INST_SIZE]; 281 memcpy(ent, (void *) cptinst, OTX_CPT_INST_SIZE); 282 283 if (++queue->idx >= queue->qhead->size / 64) { 284 curr = queue->qhead; 285 286 if (list_is_last(&curr->nextchunk, &queue->chead)) 287 queue->qhead = queue->base; 288 else 289 queue->qhead = list_next_entry(queue->qhead, nextchunk); 290 queue->idx = 0; 291 } 292 /* make sure all memory stores are done before ringing doorbell */ 293 smp_wmb(); 294 otx_cptvf_write_vq_doorbell(cptvf, 1); 295 } 296 297 static int process_request(struct pci_dev *pdev, struct otx_cpt_req_info *req, 298 struct otx_cpt_pending_queue *pqueue, 299 struct otx_cptvf *cptvf) 300 { 301 struct otx_cptvf_request *cpt_req = &req->req; 302 struct otx_cpt_pending_entry *pentry = NULL; 303 union otx_cpt_ctrl_info *ctrl = &req->ctrl; 304 struct otx_cpt_info_buffer *info = NULL; 305 union otx_cpt_res_s *result = NULL; 306 struct otx_cpt_iq_cmd iq_cmd; 307 union otx_cpt_inst_s cptinst; 308 int retry, ret = 0; 309 u8 resume_sender; 310 gfp_t gfp; 311 312 gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : 313 GFP_ATOMIC; 314 ret = setup_sgio_list(pdev, &info, req, gfp); 315 if (unlikely(ret)) { 316 dev_err(&pdev->dev, "Setting up SG list failed\n"); 317 goto request_cleanup; 318 } 319 cpt_req->dlen = info->dlen; 320 321 result = (union otx_cpt_res_s *) info->completion_addr; 322 result->s.compcode = COMPLETION_CODE_INIT; 323 324 spin_lock_bh(&pqueue->lock); 325 pentry = get_free_pending_entry(pqueue, pqueue->qlen); 326 retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; 327 while (unlikely(!pentry) && retry--) { 328 spin_unlock_bh(&pqueue->lock); 329 udelay(CPT_PENTRY_STEP); 330 spin_lock_bh(&pqueue->lock); 331 pentry = get_free_pending_entry(pqueue, pqueue->qlen); 332 } 333 334 if (unlikely(!pentry)) { 335 ret = -ENOSPC; 336 spin_unlock_bh(&pqueue->lock); 337 goto request_cleanup; 338 } 339 340 /* 341 * Check if we are close to filling in entire pending queue, 342 * if so then tell the sender to stop/sleep by returning -EBUSY 343 * We do it only for context which can sleep (GFP_KERNEL) 344 */ 345 if (gfp == GFP_KERNEL && 346 pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { 347 pentry->resume_sender = true; 348 } else 349 pentry->resume_sender = false; 350 resume_sender = pentry->resume_sender; 351 pqueue->pending_count++; 352 353 pentry->completion_addr = info->completion_addr; 354 pentry->info = info; 355 pentry->callback = req->callback; 356 pentry->areq = req->areq; 357 pentry->busy = true; 358 info->pentry = pentry; 359 info->time_in = jiffies; 360 info->req = req; 361 362 /* Fill in the command */ 363 iq_cmd.cmd.u64 = 0; 364 iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); 365 iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); 366 iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); 367 iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); 368 369 iq_cmd.dptr = info->dptr_baddr; 370 iq_cmd.rptr = info->rptr_baddr; 371 iq_cmd.cptr.u64 = 0; 372 iq_cmd.cptr.s.grp = ctrl->s.grp; 373 374 /* Fill in the CPT_INST_S type command for HW interpretation */ 375 cpt_fill_inst(&cptinst, info, &iq_cmd); 376 377 /* Print debug info if enabled */ 378 otx_cpt_dump_sg_list(pdev, req); 379 pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX_CPT_INST_SIZE); 380 print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX_CPT_INST_SIZE, false); 381 pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); 382 print_hex_dump_debug("", 0, 16, 1, info->in_buffer, 383 cpt_req->dlen, false); 384 385 /* Send CPT command */ 386 cpt_send_cmd(&cptinst, cptvf); 387 388 /* 389 * We allocate and prepare pending queue entry in critical section 390 * together with submitting CPT instruction to CPT instruction queue 391 * to make sure that order of CPT requests is the same in both 392 * pending and instruction queues 393 */ 394 spin_unlock_bh(&pqueue->lock); 395 396 ret = resume_sender ? -EBUSY : -EINPROGRESS; 397 return ret; 398 399 request_cleanup: 400 do_request_cleanup(pdev, info); 401 return ret; 402 } 403 404 int otx_cpt_do_request(struct pci_dev *pdev, struct otx_cpt_req_info *req, 405 int cpu_num) 406 { 407 struct otx_cptvf *cptvf = pci_get_drvdata(pdev); 408 409 if (!otx_cpt_device_ready(cptvf)) { 410 dev_err(&pdev->dev, "CPT Device is not ready\n"); 411 return -ENODEV; 412 } 413 414 if ((cptvf->vftype == OTX_CPT_SE_TYPES) && (!req->ctrl.s.se_req)) { 415 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request\n", 416 cptvf->vfid); 417 return -EINVAL; 418 } else if ((cptvf->vftype == OTX_CPT_AE_TYPES) && 419 (req->ctrl.s.se_req)) { 420 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request\n", 421 cptvf->vfid); 422 return -EINVAL; 423 } 424 425 return process_request(pdev, req, &cptvf->pqinfo.queue[0], cptvf); 426 } 427 428 static int cpt_process_ccode(struct pci_dev *pdev, 429 union otx_cpt_res_s *cpt_status, 430 struct otx_cpt_info_buffer *cpt_info, 431 struct otx_cpt_req_info *req, u32 *res_code) 432 { 433 u8 ccode = cpt_status->s.compcode; 434 union otx_cpt_error_code ecode; 435 436 ecode.u = be64_to_cpup((__be64 *)cpt_info->out_buffer); 437 switch (ccode) { 438 case CPT_COMP_E_FAULT: 439 dev_err(&pdev->dev, 440 "Request failed with DMA fault\n"); 441 otx_cpt_dump_sg_list(pdev, req); 442 break; 443 444 case CPT_COMP_E_SWERR: 445 dev_err(&pdev->dev, 446 "Request failed with software error code %d\n", 447 ecode.s.ccode); 448 otx_cpt_dump_sg_list(pdev, req); 449 break; 450 451 case CPT_COMP_E_HWERR: 452 dev_err(&pdev->dev, 453 "Request failed with hardware error\n"); 454 otx_cpt_dump_sg_list(pdev, req); 455 break; 456 457 case COMPLETION_CODE_INIT: 458 /* check for timeout */ 459 if (time_after_eq(jiffies, cpt_info->time_in + 460 OTX_CPT_COMMAND_TIMEOUT * HZ)) 461 dev_warn(&pdev->dev, "Request timed out 0x%p\n", req); 462 else if (cpt_info->extra_time < OTX_CPT_TIME_IN_RESET_COUNT) { 463 cpt_info->time_in = jiffies; 464 cpt_info->extra_time++; 465 } 466 return 1; 467 468 case CPT_COMP_E_GOOD: 469 /* Check microcode completion code */ 470 if (ecode.s.ccode) { 471 /* 472 * If requested hmac is truncated and ucode returns 473 * s/g write length error then we report success 474 * because ucode writes as many bytes of calculated 475 * hmac as available in gather buffer and reports 476 * s/g write length error if number of bytes in gather 477 * buffer is less than full hmac size. 478 */ 479 if (req->is_trunc_hmac && 480 ecode.s.ccode == ERR_SCATTER_GATHER_WRITE_LENGTH) { 481 *res_code = 0; 482 break; 483 } 484 485 dev_err(&pdev->dev, 486 "Request failed with software error code 0x%x\n", 487 ecode.s.ccode); 488 otx_cpt_dump_sg_list(pdev, req); 489 break; 490 } 491 492 /* Request has been processed with success */ 493 *res_code = 0; 494 break; 495 496 default: 497 dev_err(&pdev->dev, "Request returned invalid status\n"); 498 break; 499 } 500 501 return 0; 502 } 503 504 static inline void process_pending_queue(struct pci_dev *pdev, 505 struct otx_cpt_pending_queue *pqueue) 506 { 507 void (*callback)(int status, void *arg1, void *arg2); 508 struct otx_cpt_pending_entry *resume_pentry = NULL; 509 struct otx_cpt_pending_entry *pentry = NULL; 510 struct otx_cpt_info_buffer *cpt_info = NULL; 511 union otx_cpt_res_s *cpt_status = NULL; 512 struct otx_cpt_req_info *req = NULL; 513 struct crypto_async_request *areq; 514 u32 res_code, resume_index; 515 516 while (1) { 517 spin_lock_bh(&pqueue->lock); 518 pentry = &pqueue->head[pqueue->front]; 519 520 if (WARN_ON(!pentry)) { 521 spin_unlock_bh(&pqueue->lock); 522 break; 523 } 524 525 res_code = -EINVAL; 526 if (unlikely(!pentry->busy)) { 527 spin_unlock_bh(&pqueue->lock); 528 break; 529 } 530 531 if (unlikely(!pentry->callback)) { 532 dev_err(&pdev->dev, "Callback NULL\n"); 533 goto process_pentry; 534 } 535 536 cpt_info = pentry->info; 537 if (unlikely(!cpt_info)) { 538 dev_err(&pdev->dev, "Pending entry post arg NULL\n"); 539 goto process_pentry; 540 } 541 542 req = cpt_info->req; 543 if (unlikely(!req)) { 544 dev_err(&pdev->dev, "Request NULL\n"); 545 goto process_pentry; 546 } 547 548 cpt_status = (union otx_cpt_res_s *) pentry->completion_addr; 549 if (unlikely(!cpt_status)) { 550 dev_err(&pdev->dev, "Completion address NULL\n"); 551 goto process_pentry; 552 } 553 554 if (cpt_process_ccode(pdev, cpt_status, cpt_info, req, 555 &res_code)) { 556 spin_unlock_bh(&pqueue->lock); 557 return; 558 } 559 cpt_info->pdev = pdev; 560 561 process_pentry: 562 /* 563 * Check if we should inform sending side to resume 564 * We do it CPT_IQ_RESUME_MARGIN elements in advance before 565 * pending queue becomes empty 566 */ 567 resume_index = modulo_inc(pqueue->front, pqueue->qlen, 568 CPT_IQ_RESUME_MARGIN); 569 resume_pentry = &pqueue->head[resume_index]; 570 if (resume_pentry && 571 resume_pentry->resume_sender) { 572 resume_pentry->resume_sender = false; 573 callback = resume_pentry->callback; 574 areq = resume_pentry->areq; 575 576 if (callback) { 577 spin_unlock_bh(&pqueue->lock); 578 579 /* 580 * EINPROGRESS is an indication for sending 581 * side that it can resume sending requests 582 */ 583 callback(-EINPROGRESS, areq, cpt_info); 584 spin_lock_bh(&pqueue->lock); 585 } 586 } 587 588 callback = pentry->callback; 589 areq = pentry->areq; 590 free_pentry(pentry); 591 592 pqueue->pending_count--; 593 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); 594 spin_unlock_bh(&pqueue->lock); 595 596 /* 597 * Call callback after current pending entry has been 598 * processed, we don't do it if the callback pointer is 599 * invalid. 600 */ 601 if (callback) 602 callback(res_code, areq, cpt_info); 603 } 604 } 605 606 void otx_cpt_post_process(struct otx_cptvf_wqe *wqe) 607 { 608 process_pending_queue(wqe->cptvf->pdev, &wqe->cptvf->pqinfo.queue[0]); 609 } 610