1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/internal/acompress.h> 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/scatterlist.h> 8 #include "zip.h" 9 10 /* hisi_zip_sqe dw3 */ 11 #define HZIP_BD_STATUS_M GENMASK(7, 0) 12 /* hisi_zip_sqe dw7 */ 13 #define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) 14 #define HZIP_SQE_TYPE_M GENMASK(31, 28) 15 /* hisi_zip_sqe dw8 */ 16 #define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) 17 /* hisi_zip_sqe dw9 */ 18 #define HZIP_REQ_TYPE_M GENMASK(7, 0) 19 #define HZIP_ALG_TYPE_ZLIB 0x02 20 #define HZIP_ALG_TYPE_GZIP 0x03 21 #define HZIP_BUF_TYPE_M GENMASK(11, 8) 22 #define HZIP_PBUFFER 0x0 23 #define HZIP_SGL 0x1 24 25 #define HZIP_ZLIB_HEAD_SIZE 2 26 #define HZIP_GZIP_HEAD_SIZE 10 27 28 #define GZIP_HEAD_FHCRC_BIT BIT(1) 29 #define GZIP_HEAD_FEXTRA_BIT BIT(2) 30 #define GZIP_HEAD_FNAME_BIT BIT(3) 31 #define GZIP_HEAD_FCOMMENT_BIT BIT(4) 32 33 #define GZIP_HEAD_FLG_SHIFT 3 34 #define GZIP_HEAD_FEXTRA_SHIFT 10 35 #define GZIP_HEAD_FEXTRA_XLEN 2UL 36 #define GZIP_HEAD_FHCRC_SIZE 2 37 38 #define HZIP_GZIP_HEAD_BUF 256 39 #define HZIP_ALG_PRIORITY 300 40 #define HZIP_SGL_SGE_NR 10 41 42 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; 43 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { 44 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 45 }; 46 47 enum hisi_zip_alg_type { 48 HZIP_ALG_TYPE_COMP = 0, 49 HZIP_ALG_TYPE_DECOMP = 1, 50 }; 51 52 enum { 53 HZIP_QPC_COMP, 54 HZIP_QPC_DECOMP, 55 HZIP_CTX_Q_NUM 56 }; 57 58 #define COMP_NAME_TO_TYPE(alg_name) \ 59 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ 60 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ 61 62 #define TO_HEAD_SIZE(req_type) \ 63 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \ 64 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \ 65 66 #define TO_HEAD(req_type) \ 67 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \ 68 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL) \ 69 70 struct hisi_zip_req { 71 struct acomp_req *req; 72 u32 sskip; 73 u32 dskip; 74 struct hisi_acc_hw_sgl *hw_src; 75 struct hisi_acc_hw_sgl *hw_dst; 76 dma_addr_t dma_src; 77 dma_addr_t dma_dst; 78 u16 req_id; 79 }; 80 81 struct hisi_zip_req_q { 82 struct hisi_zip_req *q; 83 unsigned long *req_bitmap; 84 rwlock_t req_lock; 85 u16 size; 86 }; 87 88 struct hisi_zip_qp_ctx { 89 struct hisi_qp *qp; 90 struct hisi_zip_req_q req_q; 91 struct hisi_acc_sgl_pool *sgl_pool; 92 struct hisi_zip *zip_dev; 93 struct hisi_zip_ctx *ctx; 94 }; 95 96 struct hisi_zip_sqe_ops { 97 u8 sqe_type; 98 void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 99 void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 100 void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type); 101 void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); 102 void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 103 void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); 104 u32 (*get_tag)(struct hisi_zip_sqe *sqe); 105 u32 (*get_status)(struct hisi_zip_sqe *sqe); 106 u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); 107 }; 108 109 struct hisi_zip_ctx { 110 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; 111 const struct hisi_zip_sqe_ops *ops; 112 }; 113 114 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) 115 { 116 int ret; 117 u16 n; 118 119 if (!val) 120 return -EINVAL; 121 122 ret = kstrtou16(val, 10, &n); 123 if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) 124 return -EINVAL; 125 126 return param_set_int(val, kp); 127 } 128 129 static const struct kernel_param_ops sgl_sge_nr_ops = { 130 .set = sgl_sge_nr_set, 131 .get = param_get_int, 132 }; 133 134 static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; 135 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); 136 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); 137 138 static u16 get_extra_field_size(const u8 *start) 139 { 140 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; 141 } 142 143 static u32 get_name_field_size(const u8 *start) 144 { 145 return strlen(start) + 1; 146 } 147 148 static u32 get_comment_field_size(const u8 *start) 149 { 150 return strlen(start) + 1; 151 } 152 153 static u32 __get_gzip_head_size(const u8 *src) 154 { 155 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); 156 u32 size = GZIP_HEAD_FEXTRA_SHIFT; 157 158 if (head_flg & GZIP_HEAD_FEXTRA_BIT) 159 size += get_extra_field_size(src + size); 160 if (head_flg & GZIP_HEAD_FNAME_BIT) 161 size += get_name_field_size(src + size); 162 if (head_flg & GZIP_HEAD_FCOMMENT_BIT) 163 size += get_comment_field_size(src + size); 164 if (head_flg & GZIP_HEAD_FHCRC_BIT) 165 size += GZIP_HEAD_FHCRC_SIZE; 166 167 return size; 168 } 169 170 static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) 171 { 172 char buf[HZIP_GZIP_HEAD_BUF]; 173 174 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); 175 176 return __get_gzip_head_size(buf); 177 } 178 179 static int add_comp_head(struct scatterlist *dst, u8 req_type) 180 { 181 int head_size = TO_HEAD_SIZE(req_type); 182 const u8 *head = TO_HEAD(req_type); 183 int ret; 184 185 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); 186 if (ret != head_size) { 187 pr_err("the head size of buffer is wrong (%d)!\n", ret); 188 return -ENOMEM; 189 } 190 191 return head_size; 192 } 193 194 static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) 195 { 196 if (!acomp_req->src || !acomp_req->slen) 197 return -EINVAL; 198 199 if (req_type == HZIP_ALG_TYPE_GZIP && 200 acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT) 201 return -EINVAL; 202 203 switch (req_type) { 204 case HZIP_ALG_TYPE_ZLIB: 205 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); 206 case HZIP_ALG_TYPE_GZIP: 207 return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); 208 default: 209 pr_err("request type does not support!\n"); 210 return -EINVAL; 211 } 212 } 213 214 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, 215 struct hisi_zip_qp_ctx *qp_ctx, 216 size_t head_size, bool is_comp) 217 { 218 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 219 struct hisi_zip_req *q = req_q->q; 220 struct hisi_zip_req *req_cache; 221 int req_id; 222 223 write_lock(&req_q->req_lock); 224 225 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); 226 if (req_id >= req_q->size) { 227 write_unlock(&req_q->req_lock); 228 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); 229 return ERR_PTR(-EAGAIN); 230 } 231 set_bit(req_id, req_q->req_bitmap); 232 233 req_cache = q + req_id; 234 req_cache->req_id = req_id; 235 req_cache->req = req; 236 237 if (is_comp) { 238 req_cache->sskip = 0; 239 req_cache->dskip = head_size; 240 } else { 241 req_cache->sskip = head_size; 242 req_cache->dskip = 0; 243 } 244 245 write_unlock(&req_q->req_lock); 246 247 return req_cache; 248 } 249 250 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, 251 struct hisi_zip_req *req) 252 { 253 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 254 255 write_lock(&req_q->req_lock); 256 clear_bit(req->req_id, req_q->req_bitmap); 257 memset(req, 0, sizeof(struct hisi_zip_req)); 258 write_unlock(&req_q->req_lock); 259 } 260 261 static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 262 { 263 sqe->source_addr_l = lower_32_bits(req->dma_src); 264 sqe->source_addr_h = upper_32_bits(req->dma_src); 265 sqe->dest_addr_l = lower_32_bits(req->dma_dst); 266 sqe->dest_addr_h = upper_32_bits(req->dma_dst); 267 } 268 269 static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 270 { 271 struct acomp_req *a_req = req->req; 272 273 sqe->input_data_length = a_req->slen - req->sskip; 274 sqe->dest_avail_out = a_req->dlen - req->dskip; 275 sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, req->sskip); 276 sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, req->dskip); 277 } 278 279 static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) 280 { 281 u32 val; 282 283 val = sqe->dw9 & ~HZIP_BUF_TYPE_M; 284 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); 285 sqe->dw9 = val; 286 } 287 288 static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) 289 { 290 u32 val; 291 292 val = sqe->dw9 & ~HZIP_REQ_TYPE_M; 293 val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type); 294 sqe->dw9 = val; 295 } 296 297 static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 298 { 299 sqe->dw13 = req->req_id; 300 } 301 302 static void hisi_zip_fill_tag_v2(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 303 { 304 sqe->dw26 = req->req_id; 305 } 306 307 static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) 308 { 309 u32 val; 310 311 val = sqe->dw7 & ~HZIP_SQE_TYPE_M; 312 val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type); 313 sqe->dw7 = val; 314 } 315 316 static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe, 317 u8 req_type, struct hisi_zip_req *req) 318 { 319 const struct hisi_zip_sqe_ops *ops = ctx->ops; 320 321 memset(sqe, 0, sizeof(struct hisi_zip_sqe)); 322 323 ops->fill_addr(sqe, req); 324 ops->fill_buf_size(sqe, req); 325 ops->fill_buf_type(sqe, HZIP_SGL); 326 ops->fill_req_type(sqe, req_type); 327 ops->fill_tag(sqe, req); 328 ops->fill_sqe_type(sqe, ops->sqe_type); 329 } 330 331 static int hisi_zip_do_work(struct hisi_zip_req *req, 332 struct hisi_zip_qp_ctx *qp_ctx) 333 { 334 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; 335 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 336 struct acomp_req *a_req = req->req; 337 struct hisi_qp *qp = qp_ctx->qp; 338 struct device *dev = &qp->qm->pdev->dev; 339 struct hisi_zip_sqe zip_sqe; 340 int ret; 341 342 if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) 343 return -EINVAL; 344 345 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, 346 req->req_id << 1, &req->dma_src); 347 if (IS_ERR(req->hw_src)) { 348 dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", 349 PTR_ERR(req->hw_src)); 350 return PTR_ERR(req->hw_src); 351 } 352 353 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, 354 (req->req_id << 1) + 1, 355 &req->dma_dst); 356 if (IS_ERR(req->hw_dst)) { 357 ret = PTR_ERR(req->hw_dst); 358 dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", 359 ret); 360 goto err_unmap_input; 361 } 362 363 hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); 364 365 /* send command to start a task */ 366 atomic64_inc(&dfx->send_cnt); 367 ret = hisi_qp_send(qp, &zip_sqe); 368 if (ret < 0) { 369 atomic64_inc(&dfx->send_busy_cnt); 370 ret = -EAGAIN; 371 dev_dbg_ratelimited(dev, "failed to send request!\n"); 372 goto err_unmap_output; 373 } 374 375 return -EINPROGRESS; 376 377 err_unmap_output: 378 hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); 379 err_unmap_input: 380 hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); 381 return ret; 382 } 383 384 static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe) 385 { 386 return sqe->dw13; 387 } 388 389 static u32 hisi_zip_get_tag_v2(struct hisi_zip_sqe *sqe) 390 { 391 return sqe->dw26; 392 } 393 394 static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) 395 { 396 return sqe->dw3 & HZIP_BD_STATUS_M; 397 } 398 399 static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) 400 { 401 return sqe->produced; 402 } 403 404 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) 405 { 406 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; 407 const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; 408 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 409 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 410 struct device *dev = &qp->qm->pdev->dev; 411 struct hisi_zip_sqe *sqe = data; 412 u32 tag = ops->get_tag(sqe); 413 struct hisi_zip_req *req = req_q->q + tag; 414 struct acomp_req *acomp_req = req->req; 415 u32 status, dlen, head_size; 416 int err = 0; 417 418 atomic64_inc(&dfx->recv_cnt); 419 status = ops->get_status(sqe); 420 if (status != 0 && status != HZIP_NC_ERR) { 421 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", 422 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, 423 sqe->produced); 424 atomic64_inc(&dfx->err_bd_cnt); 425 err = -EIO; 426 } 427 428 dlen = ops->get_dstlen(sqe); 429 430 hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); 431 hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); 432 433 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; 434 acomp_req->dlen = dlen + head_size; 435 436 if (acomp_req->base.complete) 437 acomp_request_complete(acomp_req, err); 438 439 hisi_zip_remove_req(qp_ctx, req); 440 } 441 442 static int hisi_zip_acompress(struct acomp_req *acomp_req) 443 { 444 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 445 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; 446 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 447 struct hisi_zip_req *req; 448 int head_size; 449 int ret; 450 451 /* let's output compression head now */ 452 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); 453 if (head_size < 0) { 454 dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", 455 head_size); 456 return head_size; 457 } 458 459 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); 460 if (IS_ERR(req)) 461 return PTR_ERR(req); 462 463 ret = hisi_zip_do_work(req, qp_ctx); 464 if (ret != -EINPROGRESS) { 465 dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); 466 hisi_zip_remove_req(qp_ctx, req); 467 } 468 469 return ret; 470 } 471 472 static int hisi_zip_adecompress(struct acomp_req *acomp_req) 473 { 474 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 475 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; 476 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 477 struct hisi_zip_req *req; 478 int head_size, ret; 479 480 head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); 481 if (head_size < 0) { 482 dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", 483 head_size); 484 return head_size; 485 } 486 487 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); 488 if (IS_ERR(req)) 489 return PTR_ERR(req); 490 491 ret = hisi_zip_do_work(req, qp_ctx); 492 if (ret != -EINPROGRESS) { 493 dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", 494 ret); 495 hisi_zip_remove_req(qp_ctx, req); 496 } 497 498 return ret; 499 } 500 501 static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, 502 int alg_type, int req_type) 503 { 504 struct device *dev = &qp->qm->pdev->dev; 505 int ret; 506 507 qp->req_type = req_type; 508 qp->alg_type = alg_type; 509 qp->qp_ctx = ctx; 510 511 ret = hisi_qm_start_qp(qp, 0); 512 if (ret < 0) { 513 dev_err(dev, "failed to start qp (%d)!\n", ret); 514 return ret; 515 } 516 517 ctx->qp = qp; 518 519 return 0; 520 } 521 522 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) 523 { 524 hisi_qm_stop_qp(ctx->qp); 525 hisi_qm_free_qps(&ctx->qp, 1); 526 } 527 528 static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { 529 .sqe_type = 0, 530 .fill_addr = hisi_zip_fill_addr, 531 .fill_buf_size = hisi_zip_fill_buf_size, 532 .fill_buf_type = hisi_zip_fill_buf_type, 533 .fill_req_type = hisi_zip_fill_req_type, 534 .fill_tag = hisi_zip_fill_tag_v1, 535 .fill_sqe_type = hisi_zip_fill_sqe_type, 536 .get_tag = hisi_zip_get_tag_v1, 537 .get_status = hisi_zip_get_status, 538 .get_dstlen = hisi_zip_get_dstlen, 539 }; 540 541 static const struct hisi_zip_sqe_ops hisi_zip_ops_v2 = { 542 .sqe_type = 0x3, 543 .fill_addr = hisi_zip_fill_addr, 544 .fill_buf_size = hisi_zip_fill_buf_size, 545 .fill_buf_type = hisi_zip_fill_buf_type, 546 .fill_req_type = hisi_zip_fill_req_type, 547 .fill_tag = hisi_zip_fill_tag_v2, 548 .fill_sqe_type = hisi_zip_fill_sqe_type, 549 .get_tag = hisi_zip_get_tag_v2, 550 .get_status = hisi_zip_get_status, 551 .get_dstlen = hisi_zip_get_dstlen, 552 }; 553 554 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node) 555 { 556 struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; 557 struct hisi_zip_qp_ctx *qp_ctx; 558 struct hisi_zip *hisi_zip; 559 int ret, i, j; 560 561 ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); 562 if (ret) { 563 pr_err("failed to create zip qps (%d)!\n", ret); 564 return -ENODEV; 565 } 566 567 hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); 568 569 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 570 /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 571 qp_ctx = &hisi_zip_ctx->qp_ctx[i]; 572 qp_ctx->ctx = hisi_zip_ctx; 573 ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); 574 if (ret) { 575 for (j = i - 1; j >= 0; j--) 576 hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); 577 578 hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); 579 return ret; 580 } 581 582 qp_ctx->zip_dev = hisi_zip; 583 } 584 585 if (hisi_zip->qm.ver < QM_HW_V3) 586 hisi_zip_ctx->ops = &hisi_zip_ops_v1; 587 else 588 hisi_zip_ctx->ops = &hisi_zip_ops_v2; 589 590 return 0; 591 } 592 593 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) 594 { 595 int i; 596 597 for (i = 1; i >= 0; i--) 598 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); 599 } 600 601 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) 602 { 603 struct hisi_zip_req_q *req_q; 604 int i, ret; 605 606 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 607 req_q = &ctx->qp_ctx[i].req_q; 608 req_q->size = QM_Q_DEPTH; 609 610 req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); 611 if (!req_q->req_bitmap) { 612 ret = -ENOMEM; 613 if (i == 0) 614 return ret; 615 616 goto err_free_loop0; 617 } 618 rwlock_init(&req_q->req_lock); 619 620 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req), 621 GFP_KERNEL); 622 if (!req_q->q) { 623 ret = -ENOMEM; 624 if (i == 0) 625 goto err_free_bitmap; 626 else 627 goto err_free_loop1; 628 } 629 } 630 631 return 0; 632 633 err_free_loop1: 634 bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); 635 err_free_loop0: 636 kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); 637 err_free_bitmap: 638 bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); 639 return ret; 640 } 641 642 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) 643 { 644 int i; 645 646 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 647 kfree(ctx->qp_ctx[i].req_q.q); 648 bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap); 649 } 650 } 651 652 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) 653 { 654 struct hisi_zip_qp_ctx *tmp; 655 struct device *dev; 656 int i; 657 658 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 659 tmp = &ctx->qp_ctx[i]; 660 dev = &tmp->qp->qm->pdev->dev; 661 tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH << 1, 662 sgl_sge_nr); 663 if (IS_ERR(tmp->sgl_pool)) { 664 if (i == 1) 665 goto err_free_sgl_pool0; 666 return -ENOMEM; 667 } 668 } 669 670 return 0; 671 672 err_free_sgl_pool0: 673 hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, 674 ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); 675 return -ENOMEM; 676 } 677 678 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) 679 { 680 int i; 681 682 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 683 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, 684 ctx->qp_ctx[i].sgl_pool); 685 } 686 687 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, 688 void (*fn)(struct hisi_qp *, void *)) 689 { 690 int i; 691 692 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 693 ctx->qp_ctx[i].qp->req_cb = fn; 694 } 695 696 static int hisi_zip_acomp_init(struct crypto_acomp *tfm) 697 { 698 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 699 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 700 struct device *dev; 701 int ret; 702 703 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); 704 if (ret) { 705 pr_err("failed to init ctx (%d)!\n", ret); 706 return ret; 707 } 708 709 dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; 710 711 ret = hisi_zip_create_req_q(ctx); 712 if (ret) { 713 dev_err(dev, "failed to create request queue (%d)!\n", ret); 714 goto err_ctx_exit; 715 } 716 717 ret = hisi_zip_create_sgl_pool(ctx); 718 if (ret) { 719 dev_err(dev, "failed to create sgl pool (%d)!\n", ret); 720 goto err_release_req_q; 721 } 722 723 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); 724 725 return 0; 726 727 err_release_req_q: 728 hisi_zip_release_req_q(ctx); 729 err_ctx_exit: 730 hisi_zip_ctx_exit(ctx); 731 return ret; 732 } 733 734 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) 735 { 736 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 737 738 hisi_zip_set_acomp_cb(ctx, NULL); 739 hisi_zip_release_sgl_pool(ctx); 740 hisi_zip_release_req_q(ctx); 741 hisi_zip_ctx_exit(ctx); 742 } 743 744 static struct acomp_alg hisi_zip_acomp_zlib = { 745 .init = hisi_zip_acomp_init, 746 .exit = hisi_zip_acomp_exit, 747 .compress = hisi_zip_acompress, 748 .decompress = hisi_zip_adecompress, 749 .base = { 750 .cra_name = "zlib-deflate", 751 .cra_driver_name = "hisi-zlib-acomp", 752 .cra_module = THIS_MODULE, 753 .cra_priority = HZIP_ALG_PRIORITY, 754 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 755 } 756 }; 757 758 static struct acomp_alg hisi_zip_acomp_gzip = { 759 .init = hisi_zip_acomp_init, 760 .exit = hisi_zip_acomp_exit, 761 .compress = hisi_zip_acompress, 762 .decompress = hisi_zip_adecompress, 763 .base = { 764 .cra_name = "gzip", 765 .cra_driver_name = "hisi-gzip-acomp", 766 .cra_module = THIS_MODULE, 767 .cra_priority = HZIP_ALG_PRIORITY, 768 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 769 } 770 }; 771 772 int hisi_zip_register_to_crypto(struct hisi_qm *qm) 773 { 774 int ret; 775 776 ret = crypto_register_acomp(&hisi_zip_acomp_zlib); 777 if (ret) { 778 pr_err("failed to register to zlib (%d)!\n", ret); 779 return ret; 780 } 781 782 ret = crypto_register_acomp(&hisi_zip_acomp_gzip); 783 if (ret) { 784 pr_err("failed to register to gzip (%d)!\n", ret); 785 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 786 } 787 788 return ret; 789 } 790 791 void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) 792 { 793 crypto_unregister_acomp(&hisi_zip_acomp_gzip); 794 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 795 } 796