1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/gfp.h> 3 #include <linux/workqueue.h> 4 #include <crypto/internal/skcipher.h> 5 6 #include "nitrox_common.h" 7 #include "nitrox_dev.h" 8 #include "nitrox_req.h" 9 #include "nitrox_csr.h" 10 11 /* SLC_STORE_INFO */ 12 #define MIN_UDD_LEN 16 13 /* PKT_IN_HDR + SLC_STORE_INFO */ 14 #define FDATA_SIZE 32 15 /* Base destination port for the solicited requests */ 16 #define SOLICIT_BASE_DPORT 256 17 18 #define REQ_NOT_POSTED 1 19 #define REQ_BACKLOG 2 20 #define REQ_POSTED 3 21 22 /** 23 * Response codes from SE microcode 24 * 0x00 - Success 25 * Completion with no error 26 * 0x43 - ERR_GC_DATA_LEN_INVALID 27 * Invalid Data length if Encryption Data length is 28 * less than 16 bytes for AES-XTS and AES-CTS. 29 * 0x45 - ERR_GC_CTX_LEN_INVALID 30 * Invalid context length: CTXL != 23 words. 31 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID 32 * DOCSIS support is enabled with other than 33 * AES/DES-CBC mode encryption. 34 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID 35 * Authentication offset is other than 0 with 36 * Encryption IV source = 0. 37 * Authentication offset is other than 8 (DES)/16 (AES) 38 * with Encryption IV source = 1 39 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION 40 * CRC32 is enabled for other than DOCSIS encryption. 41 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID 42 * Invalid flag options in AES-CCM IV. 43 */ 44 45 static inline int incr_index(int index, int count, int max) 46 { 47 if ((index + count) >= max) 48 index = index + count - max; 49 else 50 index += count; 51 52 return index; 53 } 54 55 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) 56 { 57 struct nitrox_device *ndev = sr->ndev; 58 struct device *dev = DEV(ndev); 59 60 61 dma_unmap_sg(dev, sr->in.sg, sr->in.sgmap_cnt, DMA_BIDIRECTIONAL); 62 dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len, 63 DMA_TO_DEVICE); 64 kfree(sr->in.sgcomp); 65 sr->in.sg = NULL; 66 sr->in.sgmap_cnt = 0; 67 68 dma_unmap_sg(dev, sr->out.sg, sr->out.sgmap_cnt, 69 DMA_BIDIRECTIONAL); 70 dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len, 71 DMA_TO_DEVICE); 72 kfree(sr->out.sgcomp); 73 sr->out.sg = NULL; 74 sr->out.sgmap_cnt = 0; 75 } 76 77 static void softreq_destroy(struct nitrox_softreq *sr) 78 { 79 softreq_unmap_sgbufs(sr); 80 kfree(sr); 81 } 82 83 /** 84 * create_sg_component - create SG componets for N5 device. 85 * @sr: Request structure 86 * @sgtbl: SG table 87 * @map_nents: number of dma mapped entries 88 * 89 * Component structure 90 * 91 * 63 48 47 32 31 16 15 0 92 * -------------------------------------- 93 * | LEN0 | LEN1 | LEN2 | LEN3 | 94 * |------------------------------------- 95 * | PTR0 | 96 * -------------------------------------- 97 * | PTR1 | 98 * -------------------------------------- 99 * | PTR2 | 100 * -------------------------------------- 101 * | PTR3 | 102 * -------------------------------------- 103 * 104 * Returns 0 if success or a negative errno code on error. 105 */ 106 static int create_sg_component(struct nitrox_softreq *sr, 107 struct nitrox_sgtable *sgtbl, int map_nents) 108 { 109 struct nitrox_device *ndev = sr->ndev; 110 struct nitrox_sgcomp *sgcomp; 111 struct scatterlist *sg; 112 dma_addr_t dma; 113 size_t sz_comp; 114 int i, j, nr_sgcomp; 115 116 nr_sgcomp = roundup(map_nents, 4) / 4; 117 118 /* each component holds 4 dma pointers */ 119 sz_comp = nr_sgcomp * sizeof(*sgcomp); 120 sgcomp = kzalloc(sz_comp, sr->gfp); 121 if (!sgcomp) 122 return -ENOMEM; 123 124 sgtbl->sgcomp = sgcomp; 125 126 sg = sgtbl->sg; 127 /* populate device sg component */ 128 for (i = 0; i < nr_sgcomp; i++) { 129 for (j = 0; j < 4 && sg; j++) { 130 sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg)); 131 sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg)); 132 sg = sg_next(sg); 133 } 134 } 135 /* map the device sg component */ 136 dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); 137 if (dma_mapping_error(DEV(ndev), dma)) { 138 kfree(sgtbl->sgcomp); 139 sgtbl->sgcomp = NULL; 140 return -ENOMEM; 141 } 142 143 sgtbl->sgcomp_dma = dma; 144 sgtbl->sgcomp_len = sz_comp; 145 146 return 0; 147 } 148 149 /** 150 * dma_map_inbufs - DMA map input sglist and creates sglist component 151 * for N5 device. 152 * @sr: Request structure 153 * @req: Crypto request structre 154 * 155 * Returns 0 if successful or a negative errno code on error. 156 */ 157 static int dma_map_inbufs(struct nitrox_softreq *sr, 158 struct se_crypto_request *req) 159 { 160 struct device *dev = DEV(sr->ndev); 161 struct scatterlist *sg = req->src; 162 int i, nents, ret = 0; 163 164 nents = dma_map_sg(dev, req->src, sg_nents(req->src), 165 DMA_BIDIRECTIONAL); 166 if (!nents) 167 return -EINVAL; 168 169 for_each_sg(req->src, sg, nents, i) 170 sr->in.total_bytes += sg_dma_len(sg); 171 172 sr->in.sg = req->src; 173 sr->in.sgmap_cnt = nents; 174 ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt); 175 if (ret) 176 goto incomp_err; 177 178 return 0; 179 180 incomp_err: 181 dma_unmap_sg(dev, req->src, nents, DMA_BIDIRECTIONAL); 182 sr->in.sgmap_cnt = 0; 183 return ret; 184 } 185 186 static int dma_map_outbufs(struct nitrox_softreq *sr, 187 struct se_crypto_request *req) 188 { 189 struct device *dev = DEV(sr->ndev); 190 int nents, ret = 0; 191 192 nents = dma_map_sg(dev, req->dst, sg_nents(req->dst), 193 DMA_BIDIRECTIONAL); 194 if (!nents) 195 return -EINVAL; 196 197 sr->out.sg = req->dst; 198 sr->out.sgmap_cnt = nents; 199 ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt); 200 if (ret) 201 goto outcomp_map_err; 202 203 return 0; 204 205 outcomp_map_err: 206 dma_unmap_sg(dev, req->dst, nents, DMA_BIDIRECTIONAL); 207 sr->out.sgmap_cnt = 0; 208 sr->out.sg = NULL; 209 return ret; 210 } 211 212 static inline int softreq_map_iobuf(struct nitrox_softreq *sr, 213 struct se_crypto_request *creq) 214 { 215 int ret; 216 217 ret = dma_map_inbufs(sr, creq); 218 if (ret) 219 return ret; 220 221 ret = dma_map_outbufs(sr, creq); 222 if (ret) 223 softreq_unmap_sgbufs(sr); 224 225 return ret; 226 } 227 228 static inline void backlog_list_add(struct nitrox_softreq *sr, 229 struct nitrox_cmdq *cmdq) 230 { 231 INIT_LIST_HEAD(&sr->backlog); 232 233 spin_lock_bh(&cmdq->backlog_qlock); 234 list_add_tail(&sr->backlog, &cmdq->backlog_head); 235 atomic_inc(&cmdq->backlog_count); 236 atomic_set(&sr->status, REQ_BACKLOG); 237 spin_unlock_bh(&cmdq->backlog_qlock); 238 } 239 240 static inline void response_list_add(struct nitrox_softreq *sr, 241 struct nitrox_cmdq *cmdq) 242 { 243 INIT_LIST_HEAD(&sr->response); 244 245 spin_lock_bh(&cmdq->resp_qlock); 246 list_add_tail(&sr->response, &cmdq->response_head); 247 spin_unlock_bh(&cmdq->resp_qlock); 248 } 249 250 static inline void response_list_del(struct nitrox_softreq *sr, 251 struct nitrox_cmdq *cmdq) 252 { 253 spin_lock_bh(&cmdq->resp_qlock); 254 list_del(&sr->response); 255 spin_unlock_bh(&cmdq->resp_qlock); 256 } 257 258 static struct nitrox_softreq * 259 get_first_response_entry(struct nitrox_cmdq *cmdq) 260 { 261 return list_first_entry_or_null(&cmdq->response_head, 262 struct nitrox_softreq, response); 263 } 264 265 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) 266 { 267 if (atomic_inc_return(&cmdq->pending_count) > qlen) { 268 atomic_dec(&cmdq->pending_count); 269 /* sync with other cpus */ 270 smp_mb__after_atomic(); 271 return true; 272 } 273 /* sync with other cpus */ 274 smp_mb__after_atomic(); 275 return false; 276 } 277 278 /** 279 * post_se_instr - Post SE instruction to Packet Input ring 280 * @sr: Request structure 281 * 282 * Returns 0 if successful or a negative error code, 283 * if no space in ring. 284 */ 285 static void post_se_instr(struct nitrox_softreq *sr, 286 struct nitrox_cmdq *cmdq) 287 { 288 struct nitrox_device *ndev = sr->ndev; 289 int idx; 290 u8 *ent; 291 292 spin_lock_bh(&cmdq->cmd_qlock); 293 294 idx = cmdq->write_idx; 295 /* copy the instruction */ 296 ent = cmdq->base + (idx * cmdq->instr_size); 297 memcpy(ent, &sr->instr, cmdq->instr_size); 298 299 atomic_set(&sr->status, REQ_POSTED); 300 response_list_add(sr, cmdq); 301 sr->tstamp = jiffies; 302 /* flush the command queue updates */ 303 dma_wmb(); 304 305 /* Ring doorbell with count 1 */ 306 writeq(1, cmdq->dbell_csr_addr); 307 308 cmdq->write_idx = incr_index(idx, 1, ndev->qlen); 309 310 spin_unlock_bh(&cmdq->cmd_qlock); 311 312 /* increment the posted command count */ 313 atomic64_inc(&ndev->stats.posted); 314 } 315 316 static int post_backlog_cmds(struct nitrox_cmdq *cmdq) 317 { 318 struct nitrox_device *ndev = cmdq->ndev; 319 struct nitrox_softreq *sr, *tmp; 320 int ret = 0; 321 322 if (!atomic_read(&cmdq->backlog_count)) 323 return 0; 324 325 spin_lock_bh(&cmdq->backlog_qlock); 326 327 list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { 328 /* submit until space available */ 329 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { 330 ret = -ENOSPC; 331 break; 332 } 333 /* delete from backlog list */ 334 list_del(&sr->backlog); 335 atomic_dec(&cmdq->backlog_count); 336 /* sync with other cpus */ 337 smp_mb__after_atomic(); 338 339 /* post the command */ 340 post_se_instr(sr, cmdq); 341 } 342 spin_unlock_bh(&cmdq->backlog_qlock); 343 344 return ret; 345 } 346 347 static int nitrox_enqueue_request(struct nitrox_softreq *sr) 348 { 349 struct nitrox_cmdq *cmdq = sr->cmdq; 350 struct nitrox_device *ndev = sr->ndev; 351 352 /* try to post backlog requests */ 353 post_backlog_cmds(cmdq); 354 355 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { 356 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 357 /* increment drop count */ 358 atomic64_inc(&ndev->stats.dropped); 359 return -ENOSPC; 360 } 361 /* add to backlog list */ 362 backlog_list_add(sr, cmdq); 363 return -EINPROGRESS; 364 } 365 post_se_instr(sr, cmdq); 366 367 return -EINPROGRESS; 368 } 369 370 /** 371 * nitrox_se_request - Send request to SE core 372 * @ndev: NITROX device 373 * @req: Crypto request 374 * 375 * Returns 0 on success, or a negative error code. 376 */ 377 int nitrox_process_se_request(struct nitrox_device *ndev, 378 struct se_crypto_request *req, 379 completion_t callback, 380 void *cb_arg) 381 { 382 struct nitrox_softreq *sr; 383 dma_addr_t ctx_handle = 0; 384 int qno, ret = 0; 385 386 if (!nitrox_ready(ndev)) 387 return -ENODEV; 388 389 sr = kzalloc(sizeof(*sr), req->gfp); 390 if (!sr) 391 return -ENOMEM; 392 393 sr->ndev = ndev; 394 sr->flags = req->flags; 395 sr->gfp = req->gfp; 396 sr->callback = callback; 397 sr->cb_arg = cb_arg; 398 399 atomic_set(&sr->status, REQ_NOT_POSTED); 400 401 sr->resp.orh = req->orh; 402 sr->resp.completion = req->comp; 403 404 ret = softreq_map_iobuf(sr, req); 405 if (ret) { 406 kfree(sr); 407 return ret; 408 } 409 410 /* get the context handle */ 411 if (req->ctx_handle) { 412 struct ctx_hdr *hdr; 413 u8 *ctx_ptr; 414 415 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; 416 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); 417 ctx_handle = hdr->ctx_dma; 418 } 419 420 /* select the queue */ 421 qno = smp_processor_id() % ndev->nr_queues; 422 423 sr->cmdq = &ndev->pkt_inq[qno]; 424 425 /* 426 * 64-Byte Instruction Format 427 * 428 * ---------------------- 429 * | DPTR0 | 8 bytes 430 * ---------------------- 431 * | PKT_IN_INSTR_HDR | 8 bytes 432 * ---------------------- 433 * | PKT_IN_HDR | 16 bytes 434 * ---------------------- 435 * | SLC_INFO | 16 bytes 436 * ---------------------- 437 * | Front data | 16 bytes 438 * ---------------------- 439 */ 440 441 /* fill the packet instruction */ 442 /* word 0 */ 443 sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma); 444 445 /* word 1 */ 446 sr->instr.ih.value = 0; 447 sr->instr.ih.s.g = 1; 448 sr->instr.ih.s.gsz = sr->in.sgmap_cnt; 449 sr->instr.ih.s.ssz = sr->out.sgmap_cnt; 450 sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); 451 sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; 452 sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); 453 454 /* word 2 */ 455 sr->instr.irh.value[0] = 0; 456 sr->instr.irh.s.uddl = MIN_UDD_LEN; 457 /* context length in 64-bit words */ 458 sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); 459 /* offset from solicit base port 256 */ 460 sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; 461 sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; 462 sr->instr.irh.s.arg = req->ctrl.s.arg; 463 sr->instr.irh.s.opcode = req->opcode; 464 sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); 465 466 /* word 3 */ 467 sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); 468 469 /* word 4 */ 470 sr->instr.slc.value[0] = 0; 471 sr->instr.slc.s.ssz = sr->out.sgmap_cnt; 472 sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); 473 474 /* word 5 */ 475 sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); 476 477 /* 478 * No conversion for front data, 479 * It goes into payload 480 * put GP Header in front data 481 */ 482 sr->instr.fdata[0] = *((u64 *)&req->gph); 483 sr->instr.fdata[1] = 0; 484 485 ret = nitrox_enqueue_request(sr); 486 if (ret == -ENOSPC) 487 goto send_fail; 488 489 return ret; 490 491 send_fail: 492 softreq_destroy(sr); 493 return ret; 494 } 495 496 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) 497 { 498 return time_after_eq(jiffies, (tstamp + timeout)); 499 } 500 501 void backlog_qflush_work(struct work_struct *work) 502 { 503 struct nitrox_cmdq *cmdq; 504 505 cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); 506 post_backlog_cmds(cmdq); 507 } 508 509 static bool sr_completed(struct nitrox_softreq *sr) 510 { 511 u64 orh = READ_ONCE(*sr->resp.orh); 512 unsigned long timeout = jiffies + msecs_to_jiffies(1); 513 514 if ((orh != PENDING_SIG) && (orh & 0xff)) 515 return true; 516 517 while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) { 518 if (time_after(jiffies, timeout)) { 519 pr_err("comp not done\n"); 520 return false; 521 } 522 } 523 524 return true; 525 } 526 527 /** 528 * process_request_list - process completed requests 529 * @ndev: N5 device 530 * @qno: queue to operate 531 * 532 * Returns the number of responses processed. 533 */ 534 static void process_response_list(struct nitrox_cmdq *cmdq) 535 { 536 struct nitrox_device *ndev = cmdq->ndev; 537 struct nitrox_softreq *sr; 538 int req_completed = 0, err = 0, budget; 539 completion_t callback; 540 void *cb_arg; 541 542 /* check all pending requests */ 543 budget = atomic_read(&cmdq->pending_count); 544 545 while (req_completed < budget) { 546 sr = get_first_response_entry(cmdq); 547 if (!sr) 548 break; 549 550 if (atomic_read(&sr->status) != REQ_POSTED) 551 break; 552 553 /* check orh and completion bytes updates */ 554 if (!sr_completed(sr)) { 555 /* request not completed, check for timeout */ 556 if (!cmd_timeout(sr->tstamp, ndev->timeout)) 557 break; 558 dev_err_ratelimited(DEV(ndev), 559 "Request timeout, orh 0x%016llx\n", 560 READ_ONCE(*sr->resp.orh)); 561 } 562 atomic_dec(&cmdq->pending_count); 563 atomic64_inc(&ndev->stats.completed); 564 /* sync with other cpus */ 565 smp_mb__after_atomic(); 566 /* remove from response list */ 567 response_list_del(sr, cmdq); 568 /* ORH error code */ 569 err = READ_ONCE(*sr->resp.orh) & 0xff; 570 callback = sr->callback; 571 cb_arg = sr->cb_arg; 572 softreq_destroy(sr); 573 if (callback) 574 callback(cb_arg, err); 575 576 req_completed++; 577 } 578 } 579 580 /** 581 * pkt_slc_resp_tasklet - post processing of SE responses 582 */ 583 void pkt_slc_resp_tasklet(unsigned long data) 584 { 585 struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); 586 struct nitrox_cmdq *cmdq = qvec->cmdq; 587 union nps_pkt_slc_cnts slc_cnts; 588 589 /* read completion count */ 590 slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); 591 /* resend the interrupt if more work to do */ 592 slc_cnts.s.resend = 1; 593 594 process_response_list(cmdq); 595 596 /* 597 * clear the interrupt with resend bit enabled, 598 * MSI-X interrupt generates if Completion count > Threshold 599 */ 600 writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr); 601 602 if (atomic_read(&cmdq->backlog_count)) 603 schedule_work(&cmdq->backlog_qflush); 604 } 605