1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/internal/acompress.h> 4 #include <linux/bitfield.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/scatterlist.h> 7 #include "zip.h" 8 9 #define HZIP_ZLIB_HEAD_SIZE 2 10 #define HZIP_GZIP_HEAD_SIZE 10 11 12 #define GZIP_HEAD_FHCRC_BIT BIT(1) 13 #define GZIP_HEAD_FEXTRA_BIT BIT(2) 14 #define GZIP_HEAD_FNAME_BIT BIT(3) 15 #define GZIP_HEAD_FCOMMENT_BIT BIT(4) 16 17 #define GZIP_HEAD_FLG_SHIFT 3 18 #define GZIP_HEAD_FEXTRA_SHIFT 10 19 #define GZIP_HEAD_FEXTRA_XLEN 2 20 #define GZIP_HEAD_FHCRC_SIZE 2 21 22 #define HZIP_CTX_Q_NUM 2 23 #define HZIP_GZIP_HEAD_BUF 256 24 #define HZIP_ALG_PRIORITY 300 25 #define HZIP_SGL_SGE_NR 10 26 27 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; 28 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0, 29 0x0, 0x0, 0x0, 0x0, 0x03}; 30 enum hisi_zip_alg_type { 31 HZIP_ALG_TYPE_COMP = 0, 32 HZIP_ALG_TYPE_DECOMP = 1, 33 }; 34 35 #define COMP_NAME_TO_TYPE(alg_name) \ 36 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ 37 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ 38 39 #define TO_HEAD_SIZE(req_type) \ 40 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \ 41 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \ 42 43 #define TO_HEAD(req_type) \ 44 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \ 45 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL) \ 46 47 struct hisi_zip_req { 48 struct acomp_req *req; 49 int sskip; 50 int dskip; 51 struct hisi_acc_hw_sgl *hw_src; 52 struct hisi_acc_hw_sgl *hw_dst; 53 dma_addr_t dma_src; 54 dma_addr_t dma_dst; 55 int req_id; 56 }; 57 58 struct hisi_zip_req_q { 59 struct hisi_zip_req *q; 60 unsigned long *req_bitmap; 61 rwlock_t req_lock; 62 u16 size; 63 }; 64 65 struct hisi_zip_qp_ctx { 66 struct hisi_qp *qp; 67 struct hisi_zip_sqe zip_sqe; 68 struct hisi_zip_req_q req_q; 69 struct hisi_acc_sgl_pool *sgl_pool; 70 struct hisi_zip *zip_dev; 71 struct hisi_zip_ctx *ctx; 72 }; 73 74 struct hisi_zip_ctx { 75 #define QPC_COMP 0 76 #define QPC_DECOMP 1 77 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; 78 }; 79 80 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) 81 { 82 int ret; 83 u16 n; 84 85 if (!val) 86 return -EINVAL; 87 88 ret = kstrtou16(val, 10, &n); 89 if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) 90 return -EINVAL; 91 92 return param_set_int(val, kp); 93 } 94 95 static const struct kernel_param_ops sgl_sge_nr_ops = { 96 .set = sgl_sge_nr_set, 97 .get = param_get_int, 98 }; 99 100 static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; 101 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); 102 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); 103 104 static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) 105 { 106 u32 val; 107 108 val = (sqe->dw9) & ~HZIP_BUF_TYPE_M; 109 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); 110 sqe->dw9 = val; 111 } 112 113 static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag) 114 { 115 sqe->tag = tag; 116 } 117 118 static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, 119 dma_addr_t s_addr, dma_addr_t d_addr, u32 slen, 120 u32 dlen, int sskip, int dskip) 121 { 122 memset(sqe, 0, sizeof(struct hisi_zip_sqe)); 123 124 sqe->input_data_length = slen - sskip; 125 sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, sskip); 126 sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, dskip); 127 sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type); 128 sqe->dest_avail_out = dlen - dskip; 129 sqe->source_addr_l = lower_32_bits(s_addr); 130 sqe->source_addr_h = upper_32_bits(s_addr); 131 sqe->dest_addr_l = lower_32_bits(d_addr); 132 sqe->dest_addr_h = upper_32_bits(d_addr); 133 } 134 135 static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, 136 int alg_type, int req_type) 137 { 138 struct device *dev = &qp->qm->pdev->dev; 139 int ret; 140 141 qp->req_type = req_type; 142 qp->alg_type = alg_type; 143 qp->qp_ctx = ctx; 144 145 ret = hisi_qm_start_qp(qp, 0); 146 if (ret < 0) { 147 dev_err(dev, "start qp failed!\n"); 148 return ret; 149 } 150 151 ctx->qp = qp; 152 153 return 0; 154 } 155 156 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) 157 { 158 hisi_qm_stop_qp(ctx->qp); 159 hisi_qm_release_qp(ctx->qp); 160 } 161 162 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type) 163 { 164 struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; 165 struct hisi_zip *hisi_zip; 166 int ret, i, j; 167 168 ret = zip_create_qps(qps, HZIP_CTX_Q_NUM); 169 if (ret) { 170 pr_err("Can not create zip qps!\n"); 171 return -ENODEV; 172 } 173 174 hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); 175 176 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 177 /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 178 ret = hisi_zip_start_qp(qps[i], &hisi_zip_ctx->qp_ctx[i], i, 179 req_type); 180 if (ret) { 181 for (j = i - 1; j >= 0; j--) 182 hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); 183 184 hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); 185 return ret; 186 } 187 188 hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip; 189 } 190 191 return 0; 192 } 193 194 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) 195 { 196 int i; 197 198 for (i = 1; i >= 0; i--) 199 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); 200 } 201 202 static u16 get_extra_field_size(const u8 *start) 203 { 204 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; 205 } 206 207 static u32 get_name_field_size(const u8 *start) 208 { 209 return strlen(start) + 1; 210 } 211 212 static u32 get_comment_field_size(const u8 *start) 213 { 214 return strlen(start) + 1; 215 } 216 217 static u32 __get_gzip_head_size(const u8 *src) 218 { 219 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); 220 u32 size = GZIP_HEAD_FEXTRA_SHIFT; 221 222 if (head_flg & GZIP_HEAD_FEXTRA_BIT) 223 size += get_extra_field_size(src + size); 224 if (head_flg & GZIP_HEAD_FNAME_BIT) 225 size += get_name_field_size(src + size); 226 if (head_flg & GZIP_HEAD_FCOMMENT_BIT) 227 size += get_comment_field_size(src + size); 228 if (head_flg & GZIP_HEAD_FHCRC_BIT) 229 size += GZIP_HEAD_FHCRC_SIZE; 230 231 return size; 232 } 233 234 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) 235 { 236 struct hisi_zip_req_q *req_q; 237 int i, ret; 238 239 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 240 req_q = &ctx->qp_ctx[i].req_q; 241 req_q->size = QM_Q_DEPTH; 242 243 req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size), 244 sizeof(long), GFP_KERNEL); 245 if (!req_q->req_bitmap) { 246 ret = -ENOMEM; 247 if (i == 0) 248 return ret; 249 250 goto err_free_loop0; 251 } 252 rwlock_init(&req_q->req_lock); 253 254 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req), 255 GFP_KERNEL); 256 if (!req_q->q) { 257 ret = -ENOMEM; 258 if (i == 0) 259 goto err_free_bitmap; 260 else 261 goto err_free_loop1; 262 } 263 } 264 265 return 0; 266 267 err_free_loop1: 268 kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap); 269 err_free_loop0: 270 kfree(ctx->qp_ctx[QPC_COMP].req_q.q); 271 err_free_bitmap: 272 kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap); 273 return ret; 274 } 275 276 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) 277 { 278 int i; 279 280 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 281 kfree(ctx->qp_ctx[i].req_q.q); 282 kfree(ctx->qp_ctx[i].req_q.req_bitmap); 283 } 284 } 285 286 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) 287 { 288 struct hisi_zip_qp_ctx *tmp; 289 struct device *dev; 290 int i; 291 292 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 293 tmp = &ctx->qp_ctx[i]; 294 dev = &tmp->qp->qm->pdev->dev; 295 tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH << 1, 296 sgl_sge_nr); 297 if (IS_ERR(tmp->sgl_pool)) { 298 if (i == 1) 299 goto err_free_sgl_pool0; 300 return -ENOMEM; 301 } 302 } 303 304 return 0; 305 306 err_free_sgl_pool0: 307 hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev, 308 ctx->qp_ctx[QPC_COMP].sgl_pool); 309 return -ENOMEM; 310 } 311 312 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) 313 { 314 int i; 315 316 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 317 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, 318 ctx->qp_ctx[i].sgl_pool); 319 } 320 321 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, 322 struct hisi_zip_req *req) 323 { 324 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 325 326 write_lock(&req_q->req_lock); 327 clear_bit(req->req_id, req_q->req_bitmap); 328 memset(req, 0, sizeof(struct hisi_zip_req)); 329 write_unlock(&req_q->req_lock); 330 } 331 332 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) 333 { 334 struct hisi_zip_sqe *sqe = data; 335 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; 336 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 337 struct hisi_zip_req *req = req_q->q + sqe->tag; 338 struct acomp_req *acomp_req = req->req; 339 struct device *dev = &qp->qm->pdev->dev; 340 u32 status, dlen, head_size; 341 int err = 0; 342 343 status = sqe->dw3 & HZIP_BD_STATUS_M; 344 345 if (status != 0 && status != HZIP_NC_ERR) { 346 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", 347 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, 348 sqe->produced); 349 err = -EIO; 350 } 351 dlen = sqe->produced; 352 353 hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); 354 hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); 355 356 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; 357 acomp_req->dlen = dlen + head_size; 358 359 if (acomp_req->base.complete) 360 acomp_request_complete(acomp_req, err); 361 362 hisi_zip_remove_req(qp_ctx, req); 363 } 364 365 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, 366 void (*fn)(struct hisi_qp *, void *)) 367 { 368 int i; 369 370 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 371 ctx->qp_ctx[i].qp->req_cb = fn; 372 } 373 374 static int hisi_zip_acomp_init(struct crypto_acomp *tfm) 375 { 376 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 377 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 378 int ret; 379 380 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name)); 381 if (ret) 382 return ret; 383 384 ret = hisi_zip_create_req_q(ctx); 385 if (ret) 386 goto err_ctx_exit; 387 388 ret = hisi_zip_create_sgl_pool(ctx); 389 if (ret) 390 goto err_release_req_q; 391 392 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); 393 394 return 0; 395 396 err_release_req_q: 397 hisi_zip_release_req_q(ctx); 398 err_ctx_exit: 399 hisi_zip_ctx_exit(ctx); 400 return ret; 401 } 402 403 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) 404 { 405 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 406 407 hisi_zip_set_acomp_cb(ctx, NULL); 408 hisi_zip_release_sgl_pool(ctx); 409 hisi_zip_release_req_q(ctx); 410 hisi_zip_ctx_exit(ctx); 411 } 412 413 static int add_comp_head(struct scatterlist *dst, u8 req_type) 414 { 415 int head_size = TO_HEAD_SIZE(req_type); 416 const u8 *head = TO_HEAD(req_type); 417 int ret; 418 419 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); 420 if (ret != head_size) 421 return -ENOMEM; 422 423 return head_size; 424 } 425 426 static size_t get_gzip_head_size(struct scatterlist *sgl) 427 { 428 char buf[HZIP_GZIP_HEAD_BUF]; 429 430 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); 431 432 return __get_gzip_head_size(buf); 433 } 434 435 static size_t get_comp_head_size(struct scatterlist *src, u8 req_type) 436 { 437 switch (req_type) { 438 case HZIP_ALG_TYPE_ZLIB: 439 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); 440 case HZIP_ALG_TYPE_GZIP: 441 return get_gzip_head_size(src); 442 default: 443 pr_err("request type does not support!\n"); 444 return -EINVAL; 445 } 446 } 447 448 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, 449 struct hisi_zip_qp_ctx *qp_ctx, 450 size_t head_size, bool is_comp) 451 { 452 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 453 struct hisi_zip_req *q = req_q->q; 454 struct hisi_zip_req *req_cache; 455 int req_id; 456 457 write_lock(&req_q->req_lock); 458 459 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); 460 if (req_id >= req_q->size) { 461 write_unlock(&req_q->req_lock); 462 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); 463 return ERR_PTR(-EBUSY); 464 } 465 set_bit(req_id, req_q->req_bitmap); 466 467 req_cache = q + req_id; 468 req_cache->req_id = req_id; 469 req_cache->req = req; 470 471 if (is_comp) { 472 req_cache->sskip = 0; 473 req_cache->dskip = head_size; 474 } else { 475 req_cache->sskip = head_size; 476 req_cache->dskip = 0; 477 } 478 479 write_unlock(&req_q->req_lock); 480 481 return req_cache; 482 } 483 484 static int hisi_zip_do_work(struct hisi_zip_req *req, 485 struct hisi_zip_qp_ctx *qp_ctx) 486 { 487 struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe; 488 struct acomp_req *a_req = req->req; 489 struct hisi_qp *qp = qp_ctx->qp; 490 struct device *dev = &qp->qm->pdev->dev; 491 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; 492 dma_addr_t input; 493 dma_addr_t output; 494 int ret; 495 496 if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) 497 return -EINVAL; 498 499 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, 500 req->req_id << 1, &input); 501 if (IS_ERR(req->hw_src)) 502 return PTR_ERR(req->hw_src); 503 req->dma_src = input; 504 505 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, 506 (req->req_id << 1) + 1, 507 &output); 508 if (IS_ERR(req->hw_dst)) { 509 ret = PTR_ERR(req->hw_dst); 510 goto err_unmap_input; 511 } 512 req->dma_dst = output; 513 514 hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, a_req->slen, 515 a_req->dlen, req->sskip, req->dskip); 516 hisi_zip_config_buf_type(zip_sqe, HZIP_SGL); 517 hisi_zip_config_tag(zip_sqe, req->req_id); 518 519 /* send command to start a task */ 520 ret = hisi_qp_send(qp, zip_sqe); 521 if (ret < 0) 522 goto err_unmap_output; 523 524 return -EINPROGRESS; 525 526 err_unmap_output: 527 hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); 528 err_unmap_input: 529 hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); 530 return ret; 531 } 532 533 static int hisi_zip_acompress(struct acomp_req *acomp_req) 534 { 535 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 536 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP]; 537 struct hisi_zip_req *req; 538 int head_size; 539 int ret; 540 541 /* let's output compression head now */ 542 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); 543 if (head_size < 0) 544 return -ENOMEM; 545 546 req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true); 547 if (IS_ERR(req)) 548 return PTR_ERR(req); 549 550 ret = hisi_zip_do_work(req, qp_ctx); 551 if (ret != -EINPROGRESS) 552 hisi_zip_remove_req(qp_ctx, req); 553 554 return ret; 555 } 556 557 static int hisi_zip_adecompress(struct acomp_req *acomp_req) 558 { 559 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 560 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP]; 561 struct hisi_zip_req *req; 562 size_t head_size; 563 int ret; 564 565 head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type); 566 567 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); 568 if (IS_ERR(req)) 569 return PTR_ERR(req); 570 571 ret = hisi_zip_do_work(req, qp_ctx); 572 if (ret != -EINPROGRESS) 573 hisi_zip_remove_req(qp_ctx, req); 574 575 return ret; 576 } 577 578 static struct acomp_alg hisi_zip_acomp_zlib = { 579 .init = hisi_zip_acomp_init, 580 .exit = hisi_zip_acomp_exit, 581 .compress = hisi_zip_acompress, 582 .decompress = hisi_zip_adecompress, 583 .base = { 584 .cra_name = "zlib-deflate", 585 .cra_driver_name = "hisi-zlib-acomp", 586 .cra_module = THIS_MODULE, 587 .cra_priority = HZIP_ALG_PRIORITY, 588 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 589 } 590 }; 591 592 static struct acomp_alg hisi_zip_acomp_gzip = { 593 .init = hisi_zip_acomp_init, 594 .exit = hisi_zip_acomp_exit, 595 .compress = hisi_zip_acompress, 596 .decompress = hisi_zip_adecompress, 597 .base = { 598 .cra_name = "gzip", 599 .cra_driver_name = "hisi-gzip-acomp", 600 .cra_module = THIS_MODULE, 601 .cra_priority = HZIP_ALG_PRIORITY, 602 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 603 } 604 }; 605 606 int hisi_zip_register_to_crypto(void) 607 { 608 int ret = 0; 609 610 ret = crypto_register_acomp(&hisi_zip_acomp_zlib); 611 if (ret) { 612 pr_err("Zlib acomp algorithm registration failed\n"); 613 return ret; 614 } 615 616 ret = crypto_register_acomp(&hisi_zip_acomp_gzip); 617 if (ret) { 618 pr_err("Gzip acomp algorithm registration failed\n"); 619 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 620 } 621 622 return ret; 623 } 624 625 void hisi_zip_unregister_from_crypto(void) 626 { 627 crypto_unregister_acomp(&hisi_zip_acomp_gzip); 628 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 629 } 630