1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/internal/acompress.h> 4 #include <linux/bitfield.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/scatterlist.h> 7 #include "zip.h" 8 9 #define HZIP_ZLIB_HEAD_SIZE 2 10 #define HZIP_GZIP_HEAD_SIZE 10 11 12 #define GZIP_HEAD_FHCRC_BIT BIT(1) 13 #define GZIP_HEAD_FEXTRA_BIT BIT(2) 14 #define GZIP_HEAD_FNAME_BIT BIT(3) 15 #define GZIP_HEAD_FCOMMENT_BIT BIT(4) 16 17 #define GZIP_HEAD_FLG_SHIFT 3 18 #define GZIP_HEAD_FEXTRA_SHIFT 10 19 #define GZIP_HEAD_FEXTRA_XLEN 2 20 #define GZIP_HEAD_FHCRC_SIZE 2 21 22 #define HZIP_CTX_Q_NUM 2 23 #define HZIP_GZIP_HEAD_BUF 256 24 #define HZIP_ALG_PRIORITY 300 25 #define HZIP_SGL_SGE_NR 10 26 27 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; 28 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0, 29 0x0, 0x0, 0x0, 0x0, 0x03}; 30 enum hisi_zip_alg_type { 31 HZIP_ALG_TYPE_COMP = 0, 32 HZIP_ALG_TYPE_DECOMP = 1, 33 }; 34 35 #define COMP_NAME_TO_TYPE(alg_name) \ 36 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ 37 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ 38 39 #define TO_HEAD_SIZE(req_type) \ 40 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \ 41 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \ 42 43 #define TO_HEAD(req_type) \ 44 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \ 45 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL) \ 46 47 struct hisi_zip_req { 48 struct acomp_req *req; 49 int sskip; 50 int dskip; 51 struct hisi_acc_hw_sgl *hw_src; 52 struct hisi_acc_hw_sgl *hw_dst; 53 dma_addr_t dma_src; 54 dma_addr_t dma_dst; 55 int req_id; 56 }; 57 58 struct hisi_zip_req_q { 59 struct hisi_zip_req *q; 60 unsigned long *req_bitmap; 61 rwlock_t req_lock; 62 u16 size; 63 }; 64 65 struct hisi_zip_qp_ctx { 66 struct hisi_qp *qp; 67 struct hisi_zip_sqe zip_sqe; 68 struct hisi_zip_req_q req_q; 69 struct hisi_acc_sgl_pool *sgl_pool; 70 struct hisi_zip *zip_dev; 71 struct hisi_zip_ctx *ctx; 72 }; 73 74 struct hisi_zip_ctx { 75 #define QPC_COMP 0 76 #define QPC_DECOMP 1 77 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; 78 }; 79 80 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) 81 { 82 int ret; 83 u16 n; 84 85 if (!val) 86 return -EINVAL; 87 88 ret = kstrtou16(val, 10, &n); 89 if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) 90 return -EINVAL; 91 92 return param_set_int(val, kp); 93 } 94 95 static const struct kernel_param_ops sgl_sge_nr_ops = { 96 .set = sgl_sge_nr_set, 97 .get = param_get_int, 98 }; 99 100 static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; 101 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); 102 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); 103 104 static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) 105 { 106 u32 val; 107 108 val = (sqe->dw9) & ~HZIP_BUF_TYPE_M; 109 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); 110 sqe->dw9 = val; 111 } 112 113 static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag) 114 { 115 sqe->tag = tag; 116 } 117 118 static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type, 119 dma_addr_t s_addr, dma_addr_t d_addr, u32 slen, 120 u32 dlen, int sskip, int dskip) 121 { 122 memset(sqe, 0, sizeof(struct hisi_zip_sqe)); 123 124 sqe->input_data_length = slen - sskip; 125 sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, sskip); 126 sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, dskip); 127 sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type); 128 sqe->dest_avail_out = dlen - dskip; 129 sqe->source_addr_l = lower_32_bits(s_addr); 130 sqe->source_addr_h = upper_32_bits(s_addr); 131 sqe->dest_addr_l = lower_32_bits(d_addr); 132 sqe->dest_addr_h = upper_32_bits(d_addr); 133 } 134 135 static int hisi_zip_create_qp(struct hisi_qm *qm, struct hisi_zip_qp_ctx *ctx, 136 int alg_type, int req_type) 137 { 138 struct hisi_qp *qp; 139 int ret; 140 141 qp = hisi_qm_create_qp(qm, alg_type); 142 if (IS_ERR(qp)) 143 return PTR_ERR(qp); 144 145 qp->req_type = req_type; 146 qp->qp_ctx = ctx; 147 ctx->qp = qp; 148 149 ret = hisi_qm_start_qp(qp, 0); 150 if (ret < 0) 151 goto err_release_qp; 152 153 return 0; 154 155 err_release_qp: 156 hisi_qm_release_qp(qp); 157 return ret; 158 } 159 160 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) 161 { 162 hisi_qm_stop_qp(ctx->qp); 163 hisi_qm_release_qp(ctx->qp); 164 } 165 166 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type) 167 { 168 struct hisi_zip *hisi_zip; 169 struct hisi_qm *qm; 170 int ret, i, j; 171 172 /* find the proper zip device */ 173 hisi_zip = find_zip_device(cpu_to_node(smp_processor_id())); 174 if (!hisi_zip) { 175 pr_err("Failed to find a proper ZIP device!\n"); 176 return -ENODEV; 177 } 178 qm = &hisi_zip->qm; 179 180 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 181 /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 182 ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i, 183 req_type); 184 if (ret) 185 goto err; 186 187 hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip; 188 } 189 190 return 0; 191 err: 192 for (j = i - 1; j >= 0; j--) 193 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]); 194 195 return ret; 196 } 197 198 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) 199 { 200 int i; 201 202 for (i = 1; i >= 0; i--) 203 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); 204 } 205 206 static u16 get_extra_field_size(const u8 *start) 207 { 208 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; 209 } 210 211 static u32 get_name_field_size(const u8 *start) 212 { 213 return strlen(start) + 1; 214 } 215 216 static u32 get_comment_field_size(const u8 *start) 217 { 218 return strlen(start) + 1; 219 } 220 221 static u32 __get_gzip_head_size(const u8 *src) 222 { 223 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); 224 u32 size = GZIP_HEAD_FEXTRA_SHIFT; 225 226 if (head_flg & GZIP_HEAD_FEXTRA_BIT) 227 size += get_extra_field_size(src + size); 228 if (head_flg & GZIP_HEAD_FNAME_BIT) 229 size += get_name_field_size(src + size); 230 if (head_flg & GZIP_HEAD_FCOMMENT_BIT) 231 size += get_comment_field_size(src + size); 232 if (head_flg & GZIP_HEAD_FHCRC_BIT) 233 size += GZIP_HEAD_FHCRC_SIZE; 234 235 return size; 236 } 237 238 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) 239 { 240 struct hisi_zip_req_q *req_q; 241 int i, ret; 242 243 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 244 req_q = &ctx->qp_ctx[i].req_q; 245 req_q->size = QM_Q_DEPTH; 246 247 req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size), 248 sizeof(long), GFP_KERNEL); 249 if (!req_q->req_bitmap) { 250 ret = -ENOMEM; 251 if (i == 0) 252 return ret; 253 254 goto err_free_loop0; 255 } 256 rwlock_init(&req_q->req_lock); 257 258 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req), 259 GFP_KERNEL); 260 if (!req_q->q) { 261 ret = -ENOMEM; 262 if (i == 0) 263 goto err_free_bitmap; 264 else 265 goto err_free_loop1; 266 } 267 } 268 269 return 0; 270 271 err_free_loop1: 272 kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap); 273 err_free_loop0: 274 kfree(ctx->qp_ctx[QPC_COMP].req_q.q); 275 err_free_bitmap: 276 kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap); 277 return ret; 278 } 279 280 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) 281 { 282 int i; 283 284 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 285 kfree(ctx->qp_ctx[i].req_q.q); 286 kfree(ctx->qp_ctx[i].req_q.req_bitmap); 287 } 288 } 289 290 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) 291 { 292 struct hisi_zip_qp_ctx *tmp; 293 struct device *dev; 294 int i; 295 296 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 297 tmp = &ctx->qp_ctx[i]; 298 dev = &tmp->qp->qm->pdev->dev; 299 tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH << 1, 300 sgl_sge_nr); 301 if (IS_ERR(tmp->sgl_pool)) { 302 if (i == 1) 303 goto err_free_sgl_pool0; 304 return -ENOMEM; 305 } 306 } 307 308 return 0; 309 310 err_free_sgl_pool0: 311 hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev, 312 ctx->qp_ctx[QPC_COMP].sgl_pool); 313 return -ENOMEM; 314 } 315 316 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) 317 { 318 int i; 319 320 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 321 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, 322 ctx->qp_ctx[i].sgl_pool); 323 } 324 325 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, 326 struct hisi_zip_req *req) 327 { 328 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 329 330 write_lock(&req_q->req_lock); 331 clear_bit(req->req_id, req_q->req_bitmap); 332 memset(req, 0, sizeof(struct hisi_zip_req)); 333 write_unlock(&req_q->req_lock); 334 } 335 336 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) 337 { 338 struct hisi_zip_sqe *sqe = data; 339 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; 340 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 341 struct hisi_zip_req *req = req_q->q + sqe->tag; 342 struct acomp_req *acomp_req = req->req; 343 struct device *dev = &qp->qm->pdev->dev; 344 u32 status, dlen, head_size; 345 int err = 0; 346 347 status = sqe->dw3 & HZIP_BD_STATUS_M; 348 349 if (status != 0 && status != HZIP_NC_ERR) { 350 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", 351 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, 352 sqe->produced); 353 err = -EIO; 354 } 355 dlen = sqe->produced; 356 357 hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); 358 hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); 359 360 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; 361 acomp_req->dlen = dlen + head_size; 362 363 if (acomp_req->base.complete) 364 acomp_request_complete(acomp_req, err); 365 366 hisi_zip_remove_req(qp_ctx, req); 367 } 368 369 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, 370 void (*fn)(struct hisi_qp *, void *)) 371 { 372 int i; 373 374 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 375 ctx->qp_ctx[i].qp->req_cb = fn; 376 } 377 378 static int hisi_zip_acomp_init(struct crypto_acomp *tfm) 379 { 380 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 381 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 382 int ret; 383 384 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name)); 385 if (ret) 386 return ret; 387 388 ret = hisi_zip_create_req_q(ctx); 389 if (ret) 390 goto err_ctx_exit; 391 392 ret = hisi_zip_create_sgl_pool(ctx); 393 if (ret) 394 goto err_release_req_q; 395 396 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); 397 398 return 0; 399 400 err_release_req_q: 401 hisi_zip_release_req_q(ctx); 402 err_ctx_exit: 403 hisi_zip_ctx_exit(ctx); 404 return ret; 405 } 406 407 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) 408 { 409 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 410 411 hisi_zip_set_acomp_cb(ctx, NULL); 412 hisi_zip_release_sgl_pool(ctx); 413 hisi_zip_release_req_q(ctx); 414 hisi_zip_ctx_exit(ctx); 415 } 416 417 static int add_comp_head(struct scatterlist *dst, u8 req_type) 418 { 419 int head_size = TO_HEAD_SIZE(req_type); 420 const u8 *head = TO_HEAD(req_type); 421 int ret; 422 423 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); 424 if (ret != head_size) 425 return -ENOMEM; 426 427 return head_size; 428 } 429 430 static size_t get_gzip_head_size(struct scatterlist *sgl) 431 { 432 char buf[HZIP_GZIP_HEAD_BUF]; 433 434 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); 435 436 return __get_gzip_head_size(buf); 437 } 438 439 static size_t get_comp_head_size(struct scatterlist *src, u8 req_type) 440 { 441 switch (req_type) { 442 case HZIP_ALG_TYPE_ZLIB: 443 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); 444 case HZIP_ALG_TYPE_GZIP: 445 return get_gzip_head_size(src); 446 default: 447 pr_err("request type does not support!\n"); 448 return -EINVAL; 449 } 450 } 451 452 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, 453 struct hisi_zip_qp_ctx *qp_ctx, 454 size_t head_size, bool is_comp) 455 { 456 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 457 struct hisi_zip_req *q = req_q->q; 458 struct hisi_zip_req *req_cache; 459 int req_id; 460 461 write_lock(&req_q->req_lock); 462 463 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); 464 if (req_id >= req_q->size) { 465 write_unlock(&req_q->req_lock); 466 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); 467 return ERR_PTR(-EBUSY); 468 } 469 set_bit(req_id, req_q->req_bitmap); 470 471 req_cache = q + req_id; 472 req_cache->req_id = req_id; 473 req_cache->req = req; 474 475 if (is_comp) { 476 req_cache->sskip = 0; 477 req_cache->dskip = head_size; 478 } else { 479 req_cache->sskip = head_size; 480 req_cache->dskip = 0; 481 } 482 483 write_unlock(&req_q->req_lock); 484 485 return req_cache; 486 } 487 488 static int hisi_zip_do_work(struct hisi_zip_req *req, 489 struct hisi_zip_qp_ctx *qp_ctx) 490 { 491 struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe; 492 struct acomp_req *a_req = req->req; 493 struct hisi_qp *qp = qp_ctx->qp; 494 struct device *dev = &qp->qm->pdev->dev; 495 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; 496 dma_addr_t input; 497 dma_addr_t output; 498 int ret; 499 500 if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) 501 return -EINVAL; 502 503 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, 504 req->req_id << 1, &input); 505 if (IS_ERR(req->hw_src)) 506 return PTR_ERR(req->hw_src); 507 req->dma_src = input; 508 509 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, 510 (req->req_id << 1) + 1, 511 &output); 512 if (IS_ERR(req->hw_dst)) { 513 ret = PTR_ERR(req->hw_dst); 514 goto err_unmap_input; 515 } 516 req->dma_dst = output; 517 518 hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, a_req->slen, 519 a_req->dlen, req->sskip, req->dskip); 520 hisi_zip_config_buf_type(zip_sqe, HZIP_SGL); 521 hisi_zip_config_tag(zip_sqe, req->req_id); 522 523 /* send command to start a task */ 524 ret = hisi_qp_send(qp, zip_sqe); 525 if (ret < 0) 526 goto err_unmap_output; 527 528 return -EINPROGRESS; 529 530 err_unmap_output: 531 hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); 532 err_unmap_input: 533 hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); 534 return ret; 535 } 536 537 static int hisi_zip_acompress(struct acomp_req *acomp_req) 538 { 539 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 540 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP]; 541 struct hisi_zip_req *req; 542 int head_size; 543 int ret; 544 545 /* let's output compression head now */ 546 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); 547 if (head_size < 0) 548 return -ENOMEM; 549 550 req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true); 551 if (IS_ERR(req)) 552 return PTR_ERR(req); 553 554 ret = hisi_zip_do_work(req, qp_ctx); 555 if (ret != -EINPROGRESS) 556 hisi_zip_remove_req(qp_ctx, req); 557 558 return ret; 559 } 560 561 static int hisi_zip_adecompress(struct acomp_req *acomp_req) 562 { 563 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 564 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP]; 565 struct hisi_zip_req *req; 566 size_t head_size; 567 int ret; 568 569 head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type); 570 571 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); 572 if (IS_ERR(req)) 573 return PTR_ERR(req); 574 575 ret = hisi_zip_do_work(req, qp_ctx); 576 if (ret != -EINPROGRESS) 577 hisi_zip_remove_req(qp_ctx, req); 578 579 return ret; 580 } 581 582 static struct acomp_alg hisi_zip_acomp_zlib = { 583 .init = hisi_zip_acomp_init, 584 .exit = hisi_zip_acomp_exit, 585 .compress = hisi_zip_acompress, 586 .decompress = hisi_zip_adecompress, 587 .base = { 588 .cra_name = "zlib-deflate", 589 .cra_driver_name = "hisi-zlib-acomp", 590 .cra_module = THIS_MODULE, 591 .cra_priority = HZIP_ALG_PRIORITY, 592 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 593 } 594 }; 595 596 static struct acomp_alg hisi_zip_acomp_gzip = { 597 .init = hisi_zip_acomp_init, 598 .exit = hisi_zip_acomp_exit, 599 .compress = hisi_zip_acompress, 600 .decompress = hisi_zip_adecompress, 601 .base = { 602 .cra_name = "gzip", 603 .cra_driver_name = "hisi-gzip-acomp", 604 .cra_module = THIS_MODULE, 605 .cra_priority = HZIP_ALG_PRIORITY, 606 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 607 } 608 }; 609 610 int hisi_zip_register_to_crypto(void) 611 { 612 int ret = 0; 613 614 ret = crypto_register_acomp(&hisi_zip_acomp_zlib); 615 if (ret) { 616 pr_err("Zlib acomp algorithm registration failed\n"); 617 return ret; 618 } 619 620 ret = crypto_register_acomp(&hisi_zip_acomp_gzip); 621 if (ret) { 622 pr_err("Gzip acomp algorithm registration failed\n"); 623 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 624 } 625 626 return ret; 627 } 628 629 void hisi_zip_unregister_from_crypto(void) 630 { 631 crypto_unregister_acomp(&hisi_zip_acomp_gzip); 632 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 633 } 634