1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/gfp.h> 3 #include <linux/workqueue.h> 4 #include <crypto/internal/skcipher.h> 5 6 #include "nitrox_dev.h" 7 #include "nitrox_req.h" 8 #include "nitrox_csr.h" 9 10 /* SLC_STORE_INFO */ 11 #define MIN_UDD_LEN 16 12 /* PKT_IN_HDR + SLC_STORE_INFO */ 13 #define FDATA_SIZE 32 14 /* Base destination port for the solicited requests */ 15 #define SOLICIT_BASE_DPORT 256 16 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL 17 18 #define REQ_NOT_POSTED 1 19 #define REQ_BACKLOG 2 20 #define REQ_POSTED 3 21 22 /** 23 * Response codes from SE microcode 24 * 0x00 - Success 25 * Completion with no error 26 * 0x43 - ERR_GC_DATA_LEN_INVALID 27 * Invalid Data length if Encryption Data length is 28 * less than 16 bytes for AES-XTS and AES-CTS. 29 * 0x45 - ERR_GC_CTX_LEN_INVALID 30 * Invalid context length: CTXL != 23 words. 31 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID 32 * DOCSIS support is enabled with other than 33 * AES/DES-CBC mode encryption. 34 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID 35 * Authentication offset is other than 0 with 36 * Encryption IV source = 0. 37 * Authentication offset is other than 8 (DES)/16 (AES) 38 * with Encryption IV source = 1 39 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION 40 * CRC32 is enabled for other than DOCSIS encryption. 41 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID 42 * Invalid flag options in AES-CCM IV. 43 */ 44 45 static inline int incr_index(int index, int count, int max) 46 { 47 if ((index + count) >= max) 48 index = index + count - max; 49 else 50 index += count; 51 52 return index; 53 } 54 55 /** 56 * dma_free_sglist - unmap and free the sg lists. 57 * @ndev: N5 device 58 * @sgtbl: SG table 59 */ 60 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) 61 { 62 struct nitrox_device *ndev = sr->ndev; 63 struct device *dev = DEV(ndev); 64 struct nitrox_sglist *sglist; 65 66 /* unmap in sgbuf */ 67 sglist = sr->in.sglist; 68 if (!sglist) 69 goto out_unmap; 70 71 /* unmap iv */ 72 dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL); 73 /* unmpa src sglist */ 74 dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir); 75 /* unamp gather component */ 76 dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE); 77 kfree(sr->in.sglist); 78 kfree(sr->in.sgcomp); 79 sr->in.sglist = NULL; 80 sr->in.buf = NULL; 81 sr->in.map_bufs_cnt = 0; 82 83 out_unmap: 84 /* unmap out sgbuf */ 85 sglist = sr->out.sglist; 86 if (!sglist) 87 return; 88 89 /* unmap orh */ 90 dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); 91 92 /* unmap dst sglist */ 93 if (!sr->inplace) { 94 dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3), 95 sr->out.dir); 96 } 97 /* unmap completion */ 98 dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); 99 100 /* unmap scatter component */ 101 dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE); 102 kfree(sr->out.sglist); 103 kfree(sr->out.sgcomp); 104 sr->out.sglist = NULL; 105 sr->out.buf = NULL; 106 sr->out.map_bufs_cnt = 0; 107 } 108 109 static void softreq_destroy(struct nitrox_softreq *sr) 110 { 111 softreq_unmap_sgbufs(sr); 112 kfree(sr); 113 } 114 115 /** 116 * create_sg_component - create SG componets for N5 device. 117 * @sr: Request structure 118 * @sgtbl: SG table 119 * @nr_comp: total number of components required 120 * 121 * Component structure 122 * 123 * 63 48 47 32 31 16 15 0 124 * -------------------------------------- 125 * | LEN0 | LEN1 | LEN2 | LEN3 | 126 * |------------------------------------- 127 * | PTR0 | 128 * -------------------------------------- 129 * | PTR1 | 130 * -------------------------------------- 131 * | PTR2 | 132 * -------------------------------------- 133 * | PTR3 | 134 * -------------------------------------- 135 * 136 * Returns 0 if success or a negative errno code on error. 137 */ 138 static int create_sg_component(struct nitrox_softreq *sr, 139 struct nitrox_sgtable *sgtbl, int map_nents) 140 { 141 struct nitrox_device *ndev = sr->ndev; 142 struct nitrox_sgcomp *sgcomp; 143 struct nitrox_sglist *sglist; 144 dma_addr_t dma; 145 size_t sz_comp; 146 int i, j, nr_sgcomp; 147 148 nr_sgcomp = roundup(map_nents, 4) / 4; 149 150 /* each component holds 4 dma pointers */ 151 sz_comp = nr_sgcomp * sizeof(*sgcomp); 152 sgcomp = kzalloc(sz_comp, sr->gfp); 153 if (!sgcomp) 154 return -ENOMEM; 155 156 sgtbl->sgcomp = sgcomp; 157 sgtbl->nr_sgcomp = nr_sgcomp; 158 159 sglist = sgtbl->sglist; 160 /* populate device sg component */ 161 for (i = 0; i < nr_sgcomp; i++) { 162 for (j = 0; j < 4; j++) { 163 sgcomp->len[j] = cpu_to_be16(sglist->len); 164 sgcomp->dma[j] = cpu_to_be64(sglist->dma); 165 sglist++; 166 } 167 sgcomp++; 168 } 169 /* map the device sg component */ 170 dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); 171 if (dma_mapping_error(DEV(ndev), dma)) { 172 kfree(sgtbl->sgcomp); 173 sgtbl->sgcomp = NULL; 174 return -ENOMEM; 175 } 176 177 sgtbl->dma = dma; 178 sgtbl->len = sz_comp; 179 180 return 0; 181 } 182 183 /** 184 * dma_map_inbufs - DMA map input sglist and creates sglist component 185 * for N5 device. 186 * @sr: Request structure 187 * @req: Crypto request structre 188 * 189 * Returns 0 if successful or a negative errno code on error. 190 */ 191 static int dma_map_inbufs(struct nitrox_softreq *sr, 192 struct se_crypto_request *req) 193 { 194 struct device *dev = DEV(sr->ndev); 195 struct scatterlist *sg = req->src; 196 struct nitrox_sglist *glist; 197 int i, nents, ret = 0; 198 dma_addr_t dma; 199 size_t sz; 200 201 nents = sg_nents(req->src); 202 203 /* creater gather list IV and src entries */ 204 sz = roundup((1 + nents), 4) * sizeof(*glist); 205 glist = kzalloc(sz, sr->gfp); 206 if (!glist) 207 return -ENOMEM; 208 209 sr->in.sglist = glist; 210 /* map IV */ 211 dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL); 212 if (dma_mapping_error(dev, dma)) { 213 ret = -EINVAL; 214 goto iv_map_err; 215 } 216 217 sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 218 /* map src entries */ 219 nents = dma_map_sg(dev, req->src, nents, sr->in.dir); 220 if (!nents) { 221 ret = -EINVAL; 222 goto src_map_err; 223 } 224 sr->in.buf = req->src; 225 226 /* store the mappings */ 227 glist->len = req->ivsize; 228 glist->dma = dma; 229 glist++; 230 sr->in.total_bytes += req->ivsize; 231 232 for_each_sg(req->src, sg, nents, i) { 233 glist->len = sg_dma_len(sg); 234 glist->dma = sg_dma_address(sg); 235 sr->in.total_bytes += glist->len; 236 glist++; 237 } 238 /* roundup map count to align with entires in sg component */ 239 sr->in.map_bufs_cnt = (1 + nents); 240 241 /* create NITROX gather component */ 242 ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt); 243 if (ret) 244 goto incomp_err; 245 246 return 0; 247 248 incomp_err: 249 dma_unmap_sg(dev, req->src, nents, sr->in.dir); 250 sr->in.map_bufs_cnt = 0; 251 src_map_err: 252 dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL); 253 iv_map_err: 254 kfree(sr->in.sglist); 255 sr->in.sglist = NULL; 256 return ret; 257 } 258 259 static int dma_map_outbufs(struct nitrox_softreq *sr, 260 struct se_crypto_request *req) 261 { 262 struct device *dev = DEV(sr->ndev); 263 struct nitrox_sglist *glist = sr->in.sglist; 264 struct nitrox_sglist *slist; 265 struct scatterlist *sg; 266 int i, nents, map_bufs_cnt, ret = 0; 267 size_t sz; 268 269 nents = sg_nents(req->dst); 270 271 /* create scatter list ORH, IV, dst entries and Completion header */ 272 sz = roundup((3 + nents), 4) * sizeof(*slist); 273 slist = kzalloc(sz, sr->gfp); 274 if (!slist) 275 return -ENOMEM; 276 277 sr->out.sglist = slist; 278 sr->out.dir = DMA_BIDIRECTIONAL; 279 /* map ORH */ 280 sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN, 281 sr->out.dir); 282 if (dma_mapping_error(dev, sr->resp.orh_dma)) { 283 ret = -EINVAL; 284 goto orh_map_err; 285 } 286 287 /* map completion */ 288 sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion, 289 COMP_HLEN, sr->out.dir); 290 if (dma_mapping_error(dev, sr->resp.completion_dma)) { 291 ret = -EINVAL; 292 goto compl_map_err; 293 } 294 295 sr->inplace = (req->src == req->dst) ? true : false; 296 /* out place */ 297 if (!sr->inplace) { 298 nents = dma_map_sg(dev, req->dst, nents, sr->out.dir); 299 if (!nents) { 300 ret = -EINVAL; 301 goto dst_map_err; 302 } 303 } 304 sr->out.buf = req->dst; 305 306 /* store the mappings */ 307 /* orh */ 308 slist->len = ORH_HLEN; 309 slist->dma = sr->resp.orh_dma; 310 slist++; 311 312 /* copy the glist mappings */ 313 if (sr->inplace) { 314 nents = sr->in.map_bufs_cnt - 1; 315 map_bufs_cnt = sr->in.map_bufs_cnt; 316 while (map_bufs_cnt--) { 317 slist->len = glist->len; 318 slist->dma = glist->dma; 319 slist++; 320 glist++; 321 } 322 } else { 323 /* copy iv mapping */ 324 slist->len = glist->len; 325 slist->dma = glist->dma; 326 slist++; 327 /* copy remaining maps */ 328 for_each_sg(req->dst, sg, nents, i) { 329 slist->len = sg_dma_len(sg); 330 slist->dma = sg_dma_address(sg); 331 slist++; 332 } 333 } 334 335 /* completion */ 336 slist->len = COMP_HLEN; 337 slist->dma = sr->resp.completion_dma; 338 339 sr->out.map_bufs_cnt = (3 + nents); 340 341 ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt); 342 if (ret) 343 goto outcomp_map_err; 344 345 return 0; 346 347 outcomp_map_err: 348 if (!sr->inplace) 349 dma_unmap_sg(dev, req->dst, nents, sr->out.dir); 350 sr->out.map_bufs_cnt = 0; 351 sr->out.buf = NULL; 352 dst_map_err: 353 dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); 354 sr->resp.completion_dma = 0; 355 compl_map_err: 356 dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); 357 sr->resp.orh_dma = 0; 358 orh_map_err: 359 kfree(sr->out.sglist); 360 sr->out.sglist = NULL; 361 return ret; 362 } 363 364 static inline int softreq_map_iobuf(struct nitrox_softreq *sr, 365 struct se_crypto_request *creq) 366 { 367 int ret; 368 369 ret = dma_map_inbufs(sr, creq); 370 if (ret) 371 return ret; 372 373 ret = dma_map_outbufs(sr, creq); 374 if (ret) 375 softreq_unmap_sgbufs(sr); 376 377 return ret; 378 } 379 380 static inline void backlog_list_add(struct nitrox_softreq *sr, 381 struct nitrox_cmdq *cmdq) 382 { 383 INIT_LIST_HEAD(&sr->backlog); 384 385 spin_lock_bh(&cmdq->backlog_lock); 386 list_add_tail(&sr->backlog, &cmdq->backlog_head); 387 atomic_inc(&cmdq->backlog_count); 388 atomic_set(&sr->status, REQ_BACKLOG); 389 spin_unlock_bh(&cmdq->backlog_lock); 390 } 391 392 static inline void response_list_add(struct nitrox_softreq *sr, 393 struct nitrox_cmdq *cmdq) 394 { 395 INIT_LIST_HEAD(&sr->response); 396 397 spin_lock_bh(&cmdq->response_lock); 398 list_add_tail(&sr->response, &cmdq->response_head); 399 spin_unlock_bh(&cmdq->response_lock); 400 } 401 402 static inline void response_list_del(struct nitrox_softreq *sr, 403 struct nitrox_cmdq *cmdq) 404 { 405 spin_lock_bh(&cmdq->response_lock); 406 list_del(&sr->response); 407 spin_unlock_bh(&cmdq->response_lock); 408 } 409 410 static struct nitrox_softreq * 411 get_first_response_entry(struct nitrox_cmdq *cmdq) 412 { 413 return list_first_entry_or_null(&cmdq->response_head, 414 struct nitrox_softreq, response); 415 } 416 417 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) 418 { 419 if (atomic_inc_return(&cmdq->pending_count) > qlen) { 420 atomic_dec(&cmdq->pending_count); 421 /* sync with other cpus */ 422 smp_mb__after_atomic(); 423 return true; 424 } 425 return false; 426 } 427 428 /** 429 * post_se_instr - Post SE instruction to Packet Input ring 430 * @sr: Request structure 431 * 432 * Returns 0 if successful or a negative error code, 433 * if no space in ring. 434 */ 435 static void post_se_instr(struct nitrox_softreq *sr, 436 struct nitrox_cmdq *cmdq) 437 { 438 struct nitrox_device *ndev = sr->ndev; 439 int idx; 440 u8 *ent; 441 442 spin_lock_bh(&cmdq->cmdq_lock); 443 444 idx = cmdq->write_idx; 445 /* copy the instruction */ 446 ent = cmdq->head + (idx * cmdq->instr_size); 447 memcpy(ent, &sr->instr, cmdq->instr_size); 448 449 atomic_set(&sr->status, REQ_POSTED); 450 response_list_add(sr, cmdq); 451 sr->tstamp = jiffies; 452 /* flush the command queue updates */ 453 dma_wmb(); 454 455 /* Ring doorbell with count 1 */ 456 writeq(1, cmdq->dbell_csr_addr); 457 /* orders the doorbell rings */ 458 mmiowb(); 459 460 cmdq->write_idx = incr_index(idx, 1, ndev->qlen); 461 462 spin_unlock_bh(&cmdq->cmdq_lock); 463 } 464 465 static int post_backlog_cmds(struct nitrox_cmdq *cmdq) 466 { 467 struct nitrox_device *ndev = cmdq->ndev; 468 struct nitrox_softreq *sr, *tmp; 469 int ret = 0; 470 471 if (!atomic_read(&cmdq->backlog_count)) 472 return 0; 473 474 spin_lock_bh(&cmdq->backlog_lock); 475 476 list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { 477 struct skcipher_request *skreq; 478 479 /* submit until space available */ 480 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { 481 ret = -ENOSPC; 482 break; 483 } 484 /* delete from backlog list */ 485 list_del(&sr->backlog); 486 atomic_dec(&cmdq->backlog_count); 487 /* sync with other cpus */ 488 smp_mb__after_atomic(); 489 490 skreq = sr->skreq; 491 /* post the command */ 492 post_se_instr(sr, cmdq); 493 494 /* backlog requests are posted, wakeup with -EINPROGRESS */ 495 skcipher_request_complete(skreq, -EINPROGRESS); 496 } 497 spin_unlock_bh(&cmdq->backlog_lock); 498 499 return ret; 500 } 501 502 static int nitrox_enqueue_request(struct nitrox_softreq *sr) 503 { 504 struct nitrox_cmdq *cmdq = sr->cmdq; 505 struct nitrox_device *ndev = sr->ndev; 506 507 /* try to post backlog requests */ 508 post_backlog_cmds(cmdq); 509 510 if (unlikely(cmdq_full(cmdq, ndev->qlen))) { 511 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 512 return -ENOSPC; 513 /* add to backlog list */ 514 backlog_list_add(sr, cmdq); 515 return -EBUSY; 516 } 517 post_se_instr(sr, cmdq); 518 519 return -EINPROGRESS; 520 } 521 522 /** 523 * nitrox_se_request - Send request to SE core 524 * @ndev: NITROX device 525 * @req: Crypto request 526 * 527 * Returns 0 on success, or a negative error code. 528 */ 529 int nitrox_process_se_request(struct nitrox_device *ndev, 530 struct se_crypto_request *req, 531 completion_t callback, 532 struct skcipher_request *skreq) 533 { 534 struct nitrox_softreq *sr; 535 dma_addr_t ctx_handle = 0; 536 int qno, ret = 0; 537 538 if (!nitrox_ready(ndev)) 539 return -ENODEV; 540 541 sr = kzalloc(sizeof(*sr), req->gfp); 542 if (!sr) 543 return -ENOMEM; 544 545 sr->ndev = ndev; 546 sr->flags = req->flags; 547 sr->gfp = req->gfp; 548 sr->callback = callback; 549 sr->skreq = skreq; 550 551 atomic_set(&sr->status, REQ_NOT_POSTED); 552 553 WRITE_ONCE(sr->resp.orh, PENDING_SIG); 554 WRITE_ONCE(sr->resp.completion, PENDING_SIG); 555 556 ret = softreq_map_iobuf(sr, req); 557 if (ret) { 558 kfree(sr); 559 return ret; 560 } 561 562 /* get the context handle */ 563 if (req->ctx_handle) { 564 struct ctx_hdr *hdr; 565 u8 *ctx_ptr; 566 567 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; 568 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); 569 ctx_handle = hdr->ctx_dma; 570 } 571 572 /* select the queue */ 573 qno = smp_processor_id() % ndev->nr_queues; 574 575 sr->cmdq = &ndev->pkt_cmdqs[qno]; 576 577 /* 578 * 64-Byte Instruction Format 579 * 580 * ---------------------- 581 * | DPTR0 | 8 bytes 582 * ---------------------- 583 * | PKT_IN_INSTR_HDR | 8 bytes 584 * ---------------------- 585 * | PKT_IN_HDR | 16 bytes 586 * ---------------------- 587 * | SLC_INFO | 16 bytes 588 * ---------------------- 589 * | Front data | 16 bytes 590 * ---------------------- 591 */ 592 593 /* fill the packet instruction */ 594 /* word 0 */ 595 sr->instr.dptr0 = cpu_to_be64(sr->in.dma); 596 597 /* word 1 */ 598 sr->instr.ih.value = 0; 599 sr->instr.ih.s.g = 1; 600 sr->instr.ih.s.gsz = sr->in.map_bufs_cnt; 601 sr->instr.ih.s.ssz = sr->out.map_bufs_cnt; 602 sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); 603 sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; 604 sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value); 605 606 /* word 2 */ 607 sr->instr.irh.value[0] = 0; 608 sr->instr.irh.s.uddl = MIN_UDD_LEN; 609 /* context length in 64-bit words */ 610 sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); 611 /* offset from solicit base port 256 */ 612 sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; 613 sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; 614 sr->instr.irh.s.arg = req->ctrl.s.arg; 615 sr->instr.irh.s.opcode = req->opcode; 616 sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]); 617 618 /* word 3 */ 619 sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); 620 621 /* word 4 */ 622 sr->instr.slc.value[0] = 0; 623 sr->instr.slc.s.ssz = sr->out.map_bufs_cnt; 624 sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]); 625 626 /* word 5 */ 627 sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma); 628 629 /* 630 * No conversion for front data, 631 * It goes into payload 632 * put GP Header in front data 633 */ 634 sr->instr.fdata[0] = *((u64 *)&req->gph); 635 sr->instr.fdata[1] = 0; 636 637 ret = nitrox_enqueue_request(sr); 638 if (ret == -ENOSPC) 639 goto send_fail; 640 641 return ret; 642 643 send_fail: 644 softreq_destroy(sr); 645 return ret; 646 } 647 648 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) 649 { 650 return time_after_eq(jiffies, (tstamp + timeout)); 651 } 652 653 void backlog_qflush_work(struct work_struct *work) 654 { 655 struct nitrox_cmdq *cmdq; 656 657 cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); 658 post_backlog_cmds(cmdq); 659 } 660 661 /** 662 * process_request_list - process completed requests 663 * @ndev: N5 device 664 * @qno: queue to operate 665 * 666 * Returns the number of responses processed. 667 */ 668 static void process_response_list(struct nitrox_cmdq *cmdq) 669 { 670 struct nitrox_device *ndev = cmdq->ndev; 671 struct nitrox_softreq *sr; 672 struct skcipher_request *skreq; 673 completion_t callback; 674 int req_completed = 0, err = 0, budget; 675 676 /* check all pending requests */ 677 budget = atomic_read(&cmdq->pending_count); 678 679 while (req_completed < budget) { 680 sr = get_first_response_entry(cmdq); 681 if (!sr) 682 break; 683 684 if (atomic_read(&sr->status) != REQ_POSTED) 685 break; 686 687 /* check orh and completion bytes updates */ 688 if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) { 689 /* request not completed, check for timeout */ 690 if (!cmd_timeout(sr->tstamp, ndev->timeout)) 691 break; 692 dev_err_ratelimited(DEV(ndev), 693 "Request timeout, orh 0x%016llx\n", 694 READ_ONCE(sr->resp.orh)); 695 } 696 atomic_dec(&cmdq->pending_count); 697 /* sync with other cpus */ 698 smp_mb__after_atomic(); 699 /* remove from response list */ 700 response_list_del(sr, cmdq); 701 702 callback = sr->callback; 703 skreq = sr->skreq; 704 705 /* ORH error code */ 706 err = READ_ONCE(sr->resp.orh) & 0xff; 707 softreq_destroy(sr); 708 709 if (callback) 710 callback(skreq, err); 711 712 req_completed++; 713 } 714 } 715 716 /** 717 * pkt_slc_resp_handler - post processing of SE responses 718 */ 719 void pkt_slc_resp_handler(unsigned long data) 720 { 721 struct bh_data *bh = (void *)(uintptr_t)(data); 722 struct nitrox_cmdq *cmdq = bh->cmdq; 723 union nps_pkt_slc_cnts pkt_slc_cnts; 724 725 /* read completion count */ 726 pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr); 727 /* resend the interrupt if more work to do */ 728 pkt_slc_cnts.s.resend = 1; 729 730 process_response_list(cmdq); 731 732 /* 733 * clear the interrupt with resend bit enabled, 734 * MSI-X interrupt generates if Completion count > Threshold 735 */ 736 writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr); 737 /* order the writes */ 738 mmiowb(); 739 740 if (atomic_read(&cmdq->backlog_count)) 741 schedule_work(&cmdq->backlog_qflush); 742 } 743