1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/cryptohash.h> 48 #include <linux/skbuff.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/highmem.h> 51 #include <linux/scatterlist.h> 52 53 #include <crypto/aes.h> 54 #include <crypto/algapi.h> 55 #include <crypto/hash.h> 56 #include <crypto/gcm.h> 57 #include <crypto/sha.h> 58 #include <crypto/authenc.h> 59 #include <crypto/ctr.h> 60 #include <crypto/gf128mul.h> 61 #include <crypto/internal/aead.h> 62 #include <crypto/null.h> 63 #include <crypto/internal/skcipher.h> 64 #include <crypto/aead.h> 65 #include <crypto/scatterwalk.h> 66 #include <crypto/internal/hash.h> 67 68 #include "t4fw_api.h" 69 #include "t4_msg.h" 70 #include "chcr_core.h" 71 #include "chcr_algo.h" 72 #include "chcr_crypto.h" 73 74 #define IV AES_BLOCK_SIZE 75 76 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 77 { 78 return ctx->crypto_ctx->aeadctx; 79 } 80 81 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 82 { 83 return ctx->crypto_ctx->ablkctx; 84 } 85 86 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 87 { 88 return ctx->crypto_ctx->hmacctx; 89 } 90 91 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 92 { 93 return gctx->ctx->gcm; 94 } 95 96 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 97 { 98 return gctx->ctx->authenc; 99 } 100 101 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 102 { 103 return ctx->dev->u_ctx; 104 } 105 106 static inline int is_ofld_imm(const struct sk_buff *skb) 107 { 108 return (skb->len <= SGE_MAX_WR_LEN); 109 } 110 111 /* 112 * sgl_len - calculates the size of an SGL of the given capacity 113 * @n: the number of SGL entries 114 * Calculates the number of flits needed for a scatter/gather list that 115 * can hold the given number of entries. 116 */ 117 static inline unsigned int sgl_len(unsigned int n) 118 { 119 n--; 120 return (3 * n) / 2 + (n & 1) + 2; 121 } 122 123 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, 124 unsigned int entlen, 125 unsigned int skip) 126 { 127 int nents = 0; 128 unsigned int less; 129 unsigned int skip_len = 0; 130 131 while (sg && skip) { 132 if (sg_dma_len(sg) <= skip) { 133 skip -= sg_dma_len(sg); 134 skip_len = 0; 135 sg = sg_next(sg); 136 } else { 137 skip_len = skip; 138 skip = 0; 139 } 140 } 141 142 while (sg && reqlen) { 143 less = min(reqlen, sg_dma_len(sg) - skip_len); 144 nents += DIV_ROUND_UP(less, entlen); 145 reqlen -= less; 146 skip_len = 0; 147 sg = sg_next(sg); 148 } 149 return nents; 150 } 151 152 static inline void chcr_handle_ahash_resp(struct ahash_request *req, 153 unsigned char *input, 154 int err) 155 { 156 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 157 int digestsize, updated_digestsize; 158 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 159 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 160 161 if (input == NULL) 162 goto out; 163 reqctx = ahash_request_ctx(req); 164 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); 165 if (reqctx->is_sg_map) 166 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 167 if (reqctx->dma_addr) 168 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr, 169 reqctx->dma_len, DMA_TO_DEVICE); 170 reqctx->dma_addr = 0; 171 updated_digestsize = digestsize; 172 if (digestsize == SHA224_DIGEST_SIZE) 173 updated_digestsize = SHA256_DIGEST_SIZE; 174 else if (digestsize == SHA384_DIGEST_SIZE) 175 updated_digestsize = SHA512_DIGEST_SIZE; 176 if (reqctx->result == 1) { 177 reqctx->result = 0; 178 memcpy(req->result, input + sizeof(struct cpl_fw6_pld), 179 digestsize); 180 } else { 181 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), 182 updated_digestsize); 183 } 184 out: 185 req->base.complete(&req->base, err); 186 187 } 188 189 static inline void chcr_handle_aead_resp(struct aead_request *req, 190 unsigned char *input, 191 int err) 192 { 193 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 194 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 195 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); 196 197 198 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); 199 if (reqctx->b0_dma) 200 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, 201 reqctx->b0_len, DMA_BIDIRECTIONAL); 202 if (reqctx->verify == VERIFY_SW) { 203 chcr_verify_tag(req, input, &err); 204 reqctx->verify = VERIFY_HW; 205 } 206 req->base.complete(&req->base, err); 207 208 } 209 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 210 { 211 u8 temp[SHA512_DIGEST_SIZE]; 212 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 213 int authsize = crypto_aead_authsize(tfm); 214 struct cpl_fw6_pld *fw6_pld; 215 int cmp = 0; 216 217 fw6_pld = (struct cpl_fw6_pld *)input; 218 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 219 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 220 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); 221 } else { 222 223 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 224 authsize, req->assoclen + 225 req->cryptlen - authsize); 226 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); 227 } 228 if (cmp) 229 *err = -EBADMSG; 230 else 231 *err = 0; 232 } 233 234 /* 235 * chcr_handle_resp - Unmap the DMA buffers associated with the request 236 * @req: crypto request 237 */ 238 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 239 int err) 240 { 241 struct crypto_tfm *tfm = req->tfm; 242 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 243 struct adapter *adap = padap(ctx->dev); 244 245 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 246 case CRYPTO_ALG_TYPE_AEAD: 247 chcr_handle_aead_resp(aead_request_cast(req), input, err); 248 break; 249 250 case CRYPTO_ALG_TYPE_ABLKCIPHER: 251 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req), 252 input, err); 253 break; 254 255 case CRYPTO_ALG_TYPE_AHASH: 256 chcr_handle_ahash_resp(ahash_request_cast(req), input, err); 257 } 258 atomic_inc(&adap->chcr_stats.complete); 259 return err; 260 } 261 262 static void get_aes_decrypt_key(unsigned char *dec_key, 263 const unsigned char *key, 264 unsigned int keylength) 265 { 266 u32 temp; 267 u32 w_ring[MAX_NK]; 268 int i, j, k; 269 u8 nr, nk; 270 271 switch (keylength) { 272 case AES_KEYLENGTH_128BIT: 273 nk = KEYLENGTH_4BYTES; 274 nr = NUMBER_OF_ROUNDS_10; 275 break; 276 case AES_KEYLENGTH_192BIT: 277 nk = KEYLENGTH_6BYTES; 278 nr = NUMBER_OF_ROUNDS_12; 279 break; 280 case AES_KEYLENGTH_256BIT: 281 nk = KEYLENGTH_8BYTES; 282 nr = NUMBER_OF_ROUNDS_14; 283 break; 284 default: 285 return; 286 } 287 for (i = 0; i < nk; i++) 288 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]); 289 290 i = 0; 291 temp = w_ring[nk - 1]; 292 while (i + nk < (nr + 1) * 4) { 293 if (!(i % nk)) { 294 /* RotWord(temp) */ 295 temp = (temp << 8) | (temp >> 24); 296 temp = aes_ks_subword(temp); 297 temp ^= round_constant[i / nk]; 298 } else if (nk == 8 && (i % 4 == 0)) { 299 temp = aes_ks_subword(temp); 300 } 301 w_ring[i % nk] ^= temp; 302 temp = w_ring[i % nk]; 303 i++; 304 } 305 i--; 306 for (k = 0, j = i % nk; k < nk; k++) { 307 *((u32 *)dec_key + k) = htonl(w_ring[j]); 308 j--; 309 if (j < 0) 310 j += nk; 311 } 312 } 313 314 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 315 { 316 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 317 318 switch (ds) { 319 case SHA1_DIGEST_SIZE: 320 base_hash = crypto_alloc_shash("sha1", 0, 0); 321 break; 322 case SHA224_DIGEST_SIZE: 323 base_hash = crypto_alloc_shash("sha224", 0, 0); 324 break; 325 case SHA256_DIGEST_SIZE: 326 base_hash = crypto_alloc_shash("sha256", 0, 0); 327 break; 328 case SHA384_DIGEST_SIZE: 329 base_hash = crypto_alloc_shash("sha384", 0, 0); 330 break; 331 case SHA512_DIGEST_SIZE: 332 base_hash = crypto_alloc_shash("sha512", 0, 0); 333 break; 334 } 335 336 return base_hash; 337 } 338 339 static int chcr_compute_partial_hash(struct shash_desc *desc, 340 char *iopad, char *result_hash, 341 int digest_size) 342 { 343 struct sha1_state sha1_st; 344 struct sha256_state sha256_st; 345 struct sha512_state sha512_st; 346 int error; 347 348 if (digest_size == SHA1_DIGEST_SIZE) { 349 error = crypto_shash_init(desc) ?: 350 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 351 crypto_shash_export(desc, (void *)&sha1_st); 352 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 353 } else if (digest_size == SHA224_DIGEST_SIZE) { 354 error = crypto_shash_init(desc) ?: 355 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 356 crypto_shash_export(desc, (void *)&sha256_st); 357 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 358 359 } else if (digest_size == SHA256_DIGEST_SIZE) { 360 error = crypto_shash_init(desc) ?: 361 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 362 crypto_shash_export(desc, (void *)&sha256_st); 363 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 364 365 } else if (digest_size == SHA384_DIGEST_SIZE) { 366 error = crypto_shash_init(desc) ?: 367 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 368 crypto_shash_export(desc, (void *)&sha512_st); 369 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 370 371 } else if (digest_size == SHA512_DIGEST_SIZE) { 372 error = crypto_shash_init(desc) ?: 373 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 374 crypto_shash_export(desc, (void *)&sha512_st); 375 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 376 } else { 377 error = -EINVAL; 378 pr_err("Unknown digest size %d\n", digest_size); 379 } 380 return error; 381 } 382 383 static void chcr_change_order(char *buf, int ds) 384 { 385 int i; 386 387 if (ds == SHA512_DIGEST_SIZE) { 388 for (i = 0; i < (ds / sizeof(u64)); i++) 389 *((__be64 *)buf + i) = 390 cpu_to_be64(*((u64 *)buf + i)); 391 } else { 392 for (i = 0; i < (ds / sizeof(u32)); i++) 393 *((__be32 *)buf + i) = 394 cpu_to_be32(*((u32 *)buf + i)); 395 } 396 } 397 398 static inline int is_hmac(struct crypto_tfm *tfm) 399 { 400 struct crypto_alg *alg = tfm->__crt_alg; 401 struct chcr_alg_template *chcr_crypto_alg = 402 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 403 alg.hash); 404 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 405 return 1; 406 return 0; 407 } 408 409 static inline void dsgl_walk_init(struct dsgl_walk *walk, 410 struct cpl_rx_phys_dsgl *dsgl) 411 { 412 walk->dsgl = dsgl; 413 walk->nents = 0; 414 walk->to = (struct phys_sge_pairs *)(dsgl + 1); 415 } 416 417 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid) 418 { 419 struct cpl_rx_phys_dsgl *phys_cpl; 420 421 phys_cpl = walk->dsgl; 422 423 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 424 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 425 phys_cpl->pcirlxorder_to_noofsgentr = 426 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 427 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 428 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 429 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 430 CPL_RX_PHYS_DSGL_DCAID_V(0) | 431 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); 432 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 433 phys_cpl->rss_hdr_int.qid = htons(qid); 434 phys_cpl->rss_hdr_int.hash_val = 0; 435 } 436 437 static inline void dsgl_walk_add_page(struct dsgl_walk *walk, 438 size_t size, 439 dma_addr_t *addr) 440 { 441 int j; 442 443 if (!size) 444 return; 445 j = walk->nents; 446 walk->to->len[j % 8] = htons(size); 447 walk->to->addr[j % 8] = cpu_to_be64(*addr); 448 j++; 449 if ((j % 8) == 0) 450 walk->to++; 451 walk->nents = j; 452 } 453 454 static void dsgl_walk_add_sg(struct dsgl_walk *walk, 455 struct scatterlist *sg, 456 unsigned int slen, 457 unsigned int skip) 458 { 459 int skip_len = 0; 460 unsigned int left_size = slen, len = 0; 461 unsigned int j = walk->nents; 462 int offset, ent_len; 463 464 if (!slen) 465 return; 466 while (sg && skip) { 467 if (sg_dma_len(sg) <= skip) { 468 skip -= sg_dma_len(sg); 469 skip_len = 0; 470 sg = sg_next(sg); 471 } else { 472 skip_len = skip; 473 skip = 0; 474 } 475 } 476 477 while (left_size && sg) { 478 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); 479 offset = 0; 480 while (len) { 481 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); 482 walk->to->len[j % 8] = htons(ent_len); 483 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + 484 offset + skip_len); 485 offset += ent_len; 486 len -= ent_len; 487 j++; 488 if ((j % 8) == 0) 489 walk->to++; 490 } 491 walk->last_sg = sg; 492 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - 493 skip_len) + skip_len; 494 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); 495 skip_len = 0; 496 sg = sg_next(sg); 497 } 498 walk->nents = j; 499 } 500 501 static inline void ulptx_walk_init(struct ulptx_walk *walk, 502 struct ulptx_sgl *ulp) 503 { 504 walk->sgl = ulp; 505 walk->nents = 0; 506 walk->pair_idx = 0; 507 walk->pair = ulp->sge; 508 walk->last_sg = NULL; 509 walk->last_sg_len = 0; 510 } 511 512 static inline void ulptx_walk_end(struct ulptx_walk *walk) 513 { 514 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | 515 ULPTX_NSGE_V(walk->nents)); 516 } 517 518 519 static inline void ulptx_walk_add_page(struct ulptx_walk *walk, 520 size_t size, 521 dma_addr_t *addr) 522 { 523 if (!size) 524 return; 525 526 if (walk->nents == 0) { 527 walk->sgl->len0 = cpu_to_be32(size); 528 walk->sgl->addr0 = cpu_to_be64(*addr); 529 } else { 530 walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr); 531 walk->pair->len[walk->pair_idx] = cpu_to_be32(size); 532 walk->pair_idx = !walk->pair_idx; 533 if (!walk->pair_idx) 534 walk->pair++; 535 } 536 walk->nents++; 537 } 538 539 static void ulptx_walk_add_sg(struct ulptx_walk *walk, 540 struct scatterlist *sg, 541 unsigned int len, 542 unsigned int skip) 543 { 544 int small; 545 int skip_len = 0; 546 unsigned int sgmin; 547 548 if (!len) 549 return; 550 551 while (sg && skip) { 552 if (sg_dma_len(sg) <= skip) { 553 skip -= sg_dma_len(sg); 554 skip_len = 0; 555 sg = sg_next(sg); 556 } else { 557 skip_len = skip; 558 skip = 0; 559 } 560 } 561 if (walk->nents == 0) { 562 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); 563 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 564 walk->sgl->len0 = cpu_to_be32(sgmin); 565 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); 566 walk->nents++; 567 len -= sgmin; 568 walk->last_sg = sg; 569 walk->last_sg_len = sgmin + skip_len; 570 skip_len += sgmin; 571 if (sg_dma_len(sg) == skip_len) { 572 sg = sg_next(sg); 573 skip_len = 0; 574 } 575 } 576 577 while (sg && len) { 578 small = min(sg_dma_len(sg) - skip_len, len); 579 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 580 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); 581 walk->pair->addr[walk->pair_idx] = 582 cpu_to_be64(sg_dma_address(sg) + skip_len); 583 walk->pair_idx = !walk->pair_idx; 584 walk->nents++; 585 if (!walk->pair_idx) 586 walk->pair++; 587 len -= sgmin; 588 skip_len += sgmin; 589 walk->last_sg = sg; 590 walk->last_sg_len = skip_len; 591 if (sg_dma_len(sg) == skip_len) { 592 sg = sg_next(sg); 593 skip_len = 0; 594 } 595 } 596 } 597 598 static inline int get_aead_subtype(struct crypto_aead *aead) 599 { 600 struct aead_alg *alg = crypto_aead_alg(aead); 601 struct chcr_alg_template *chcr_crypto_alg = 602 container_of(alg, struct chcr_alg_template, alg.aead); 603 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 604 } 605 606 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) 607 { 608 struct crypto_alg *alg = tfm->__crt_alg; 609 struct chcr_alg_template *chcr_crypto_alg = 610 container_of(alg, struct chcr_alg_template, alg.crypto); 611 612 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 613 } 614 615 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 616 { 617 struct adapter *adap = netdev2adap(dev); 618 struct sge_uld_txq_info *txq_info = 619 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 620 struct sge_uld_txq *txq; 621 int ret = 0; 622 623 local_bh_disable(); 624 txq = &txq_info->uldtxq[idx]; 625 spin_lock(&txq->sendq.lock); 626 if (txq->full) 627 ret = -1; 628 spin_unlock(&txq->sendq.lock); 629 local_bh_enable(); 630 return ret; 631 } 632 633 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 634 struct _key_ctx *key_ctx) 635 { 636 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 637 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 638 } else { 639 memcpy(key_ctx->key, 640 ablkctx->key + (ablkctx->enckey_len >> 1), 641 ablkctx->enckey_len >> 1); 642 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 643 ablkctx->rrkey, ablkctx->enckey_len >> 1); 644 } 645 return 0; 646 } 647 static int chcr_sg_ent_in_wr(struct scatterlist *src, 648 struct scatterlist *dst, 649 unsigned int minsg, 650 unsigned int space, 651 unsigned int srcskip, 652 unsigned int dstskip) 653 { 654 int srclen = 0, dstlen = 0; 655 int srcsg = minsg, dstsg = minsg; 656 int offset = 0, less; 657 658 if (sg_dma_len(src) == srcskip) { 659 src = sg_next(src); 660 srcskip = 0; 661 } 662 663 if (sg_dma_len(dst) == dstskip) { 664 dst = sg_next(dst); 665 dstskip = 0; 666 } 667 668 while (src && dst && 669 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { 670 srclen += (sg_dma_len(src) - srcskip); 671 srcsg++; 672 offset = 0; 673 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && 674 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { 675 if (srclen <= dstlen) 676 break; 677 less = min_t(unsigned int, sg_dma_len(dst) - offset - 678 dstskip, CHCR_DST_SG_SIZE); 679 dstlen += less; 680 offset += less; 681 if (offset == sg_dma_len(dst)) { 682 dst = sg_next(dst); 683 offset = 0; 684 } 685 dstsg++; 686 dstskip = 0; 687 } 688 src = sg_next(src); 689 srcskip = 0; 690 } 691 return min(srclen, dstlen); 692 } 693 694 static int chcr_cipher_fallback(struct crypto_skcipher *cipher, 695 u32 flags, 696 struct scatterlist *src, 697 struct scatterlist *dst, 698 unsigned int nbytes, 699 u8 *iv, 700 unsigned short op_type) 701 { 702 int err; 703 704 SKCIPHER_REQUEST_ON_STACK(subreq, cipher); 705 skcipher_request_set_tfm(subreq, cipher); 706 skcipher_request_set_callback(subreq, flags, NULL, NULL); 707 skcipher_request_set_crypt(subreq, src, dst, 708 nbytes, iv); 709 710 err = op_type ? crypto_skcipher_decrypt(subreq) : 711 crypto_skcipher_encrypt(subreq); 712 skcipher_request_zero(subreq); 713 714 return err; 715 716 } 717 static inline void create_wreq(struct chcr_context *ctx, 718 struct chcr_wr *chcr_req, 719 struct crypto_async_request *req, 720 unsigned int imm, 721 int hash_sz, 722 unsigned int len16, 723 unsigned int sc_len, 724 unsigned int lcb) 725 { 726 struct uld_ctx *u_ctx = ULD_CTX(ctx); 727 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; 728 729 730 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; 731 chcr_req->wreq.pld_size_hash_size = 732 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 733 chcr_req->wreq.len16_pkd = 734 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); 735 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 736 chcr_req->wreq.rx_chid_to_rx_q_id = 737 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, 738 !!lcb, ctx->tx_qidx); 739 740 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, 741 qid); 742 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - 743 ((sizeof(chcr_req->wreq)) >> 4))); 744 745 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); 746 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 747 sizeof(chcr_req->key_ctx) + sc_len); 748 } 749 750 /** 751 * create_cipher_wr - form the WR for cipher operations 752 * @req: cipher req. 753 * @ctx: crypto driver context of the request. 754 * @qid: ingress qid where response of this WR should be received. 755 * @op_type: encryption or decryption 756 */ 757 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) 758 { 759 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); 760 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 761 struct sk_buff *skb = NULL; 762 struct chcr_wr *chcr_req; 763 struct cpl_rx_phys_dsgl *phys_cpl; 764 struct ulptx_sgl *ulptx; 765 struct chcr_blkcipher_req_ctx *reqctx = 766 ablkcipher_request_ctx(wrparam->req); 767 unsigned int temp = 0, transhdr_len, dst_size; 768 int error; 769 int nents; 770 unsigned int kctx_len; 771 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 772 GFP_KERNEL : GFP_ATOMIC; 773 struct adapter *adap = padap(c_ctx(tfm)->dev); 774 775 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, 776 reqctx->dst_ofst); 777 dst_size = get_space_for_phys_dsgl(nents + 1); 778 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); 779 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 780 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, 781 CHCR_SRC_SG_SIZE, reqctx->src_ofst); 782 temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16) 783 * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8); 784 transhdr_len += temp; 785 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 786 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 787 if (!skb) { 788 error = -ENOMEM; 789 goto err; 790 } 791 chcr_req = __skb_put_zero(skb, transhdr_len); 792 chcr_req->sec_cpl.op_ivinsrtofst = 793 FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1); 794 795 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); 796 chcr_req->sec_cpl.aadstart_cipherstop_hi = 797 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); 798 799 chcr_req->sec_cpl.cipherstop_lo_authinsert = 800 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 801 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, 802 ablkctx->ciph_mode, 803 0, 0, IV >> 1); 804 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 805 0, 0, dst_size); 806 807 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 808 if ((reqctx->op == CHCR_DECRYPT_OP) && 809 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 810 CRYPTO_ALG_SUB_TYPE_CTR)) && 811 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 812 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { 813 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 814 } else { 815 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || 816 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { 817 memcpy(chcr_req->key_ctx.key, ablkctx->key, 818 ablkctx->enckey_len); 819 } else { 820 memcpy(chcr_req->key_ctx.key, ablkctx->key + 821 (ablkctx->enckey_len >> 1), 822 ablkctx->enckey_len >> 1); 823 memcpy(chcr_req->key_ctx.key + 824 (ablkctx->enckey_len >> 1), 825 ablkctx->key, 826 ablkctx->enckey_len >> 1); 827 } 828 } 829 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 830 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 831 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); 832 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); 833 834 atomic_inc(&adap->chcr_stats.cipher_rqst); 835 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len 836 +(reqctx->imm ? (IV + wrparam->bytes) : 0); 837 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, 838 transhdr_len, temp, 839 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); 840 reqctx->skb = skb; 841 return skb; 842 err: 843 return ERR_PTR(error); 844 } 845 846 static inline int chcr_keyctx_ck_size(unsigned int keylen) 847 { 848 int ck_size = 0; 849 850 if (keylen == AES_KEYSIZE_128) 851 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 852 else if (keylen == AES_KEYSIZE_192) 853 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 854 else if (keylen == AES_KEYSIZE_256) 855 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 856 else 857 ck_size = 0; 858 859 return ck_size; 860 } 861 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, 862 const u8 *key, 863 unsigned int keylen) 864 { 865 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 866 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 867 int err = 0; 868 869 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 870 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags & 871 CRYPTO_TFM_REQ_MASK); 872 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen); 873 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; 874 tfm->crt_flags |= 875 crypto_skcipher_get_flags(ablkctx->sw_cipher) & 876 CRYPTO_TFM_RES_MASK; 877 return err; 878 } 879 880 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, 881 const u8 *key, 882 unsigned int keylen) 883 { 884 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 885 unsigned int ck_size, context_size; 886 u16 alignment = 0; 887 int err; 888 889 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 890 if (err) 891 goto badkey_err; 892 893 ck_size = chcr_keyctx_ck_size(keylen); 894 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; 895 memcpy(ablkctx->key, key, keylen); 896 ablkctx->enckey_len = keylen; 897 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 898 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 899 keylen + alignment) >> 4; 900 901 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 902 0, 0, context_size); 903 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 904 return 0; 905 badkey_err: 906 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 907 ablkctx->enckey_len = 0; 908 909 return err; 910 } 911 912 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, 913 const u8 *key, 914 unsigned int keylen) 915 { 916 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 917 unsigned int ck_size, context_size; 918 u16 alignment = 0; 919 int err; 920 921 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 922 if (err) 923 goto badkey_err; 924 ck_size = chcr_keyctx_ck_size(keylen); 925 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 926 memcpy(ablkctx->key, key, keylen); 927 ablkctx->enckey_len = keylen; 928 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 929 keylen + alignment) >> 4; 930 931 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 932 0, 0, context_size); 933 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 934 935 return 0; 936 badkey_err: 937 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 938 ablkctx->enckey_len = 0; 939 940 return err; 941 } 942 943 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, 944 const u8 *key, 945 unsigned int keylen) 946 { 947 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 948 unsigned int ck_size, context_size; 949 u16 alignment = 0; 950 int err; 951 952 if (keylen < CTR_RFC3686_NONCE_SIZE) 953 return -EINVAL; 954 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), 955 CTR_RFC3686_NONCE_SIZE); 956 957 keylen -= CTR_RFC3686_NONCE_SIZE; 958 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 959 if (err) 960 goto badkey_err; 961 962 ck_size = chcr_keyctx_ck_size(keylen); 963 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 964 memcpy(ablkctx->key, key, keylen); 965 ablkctx->enckey_len = keylen; 966 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 967 keylen + alignment) >> 4; 968 969 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 970 0, 0, context_size); 971 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 972 973 return 0; 974 badkey_err: 975 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 976 ablkctx->enckey_len = 0; 977 978 return err; 979 } 980 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) 981 { 982 unsigned int size = AES_BLOCK_SIZE; 983 __be32 *b = (__be32 *)(dstiv + size); 984 u32 c, prev; 985 986 memcpy(dstiv, srciv, AES_BLOCK_SIZE); 987 for (; size >= 4; size -= 4) { 988 prev = be32_to_cpu(*--b); 989 c = prev + add; 990 *b = cpu_to_be32(c); 991 if (prev < c) 992 break; 993 add = 1; 994 } 995 996 } 997 998 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) 999 { 1000 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); 1001 u64 c; 1002 u32 temp = be32_to_cpu(*--b); 1003 1004 temp = ~temp; 1005 c = (u64)temp + 1; // No of block can processed withou overflow 1006 if ((bytes / AES_BLOCK_SIZE) > c) 1007 bytes = c * AES_BLOCK_SIZE; 1008 return bytes; 1009 } 1010 1011 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) 1012 { 1013 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1014 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1015 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1016 struct crypto_cipher *cipher; 1017 int ret, i; 1018 u8 *key; 1019 unsigned int keylen; 1020 int round = reqctx->last_req_len / AES_BLOCK_SIZE; 1021 int round8 = round / 8; 1022 1023 cipher = ablkctx->aes_generic; 1024 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); 1025 1026 keylen = ablkctx->enckey_len / 2; 1027 key = ablkctx->key + keylen; 1028 ret = crypto_cipher_setkey(cipher, key, keylen); 1029 if (ret) 1030 goto out; 1031 /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/ 1032 for (i = 0; i < round8; i++) 1033 gf128mul_x8_ble((le128 *)iv, (le128 *)iv); 1034 1035 for (i = 0; i < (round % 8); i++) 1036 gf128mul_x_ble((le128 *)iv, (le128 *)iv); 1037 1038 crypto_cipher_decrypt_one(cipher, iv, iv); 1039 out: 1040 return ret; 1041 } 1042 1043 static int chcr_update_cipher_iv(struct ablkcipher_request *req, 1044 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1045 { 1046 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1047 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1048 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 1049 int ret = 0; 1050 1051 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1052 ctr_add_iv(iv, req->info, (reqctx->processed / 1053 AES_BLOCK_SIZE)); 1054 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) 1055 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1056 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / 1057 AES_BLOCK_SIZE) + 1); 1058 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1059 ret = chcr_update_tweak(req, iv); 1060 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1061 if (reqctx->op) 1062 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, 1063 16, 1064 reqctx->processed - AES_BLOCK_SIZE); 1065 else 1066 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1067 } 1068 1069 return ret; 1070 1071 } 1072 1073 /* We need separate function for final iv because in rfc3686 Initial counter 1074 * starts from 1 and buffer size of iv is 8 byte only which remains constant 1075 * for subsequent update requests 1076 */ 1077 1078 static int chcr_final_cipher_iv(struct ablkcipher_request *req, 1079 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1080 { 1081 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1082 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1083 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 1084 int ret = 0; 1085 1086 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1087 ctr_add_iv(iv, req->info, (reqctx->processed / 1088 AES_BLOCK_SIZE)); 1089 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1090 ret = chcr_update_tweak(req, iv); 1091 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1092 if (reqctx->op) 1093 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, 1094 16, 1095 reqctx->processed - AES_BLOCK_SIZE); 1096 else 1097 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1098 1099 } 1100 return ret; 1101 1102 } 1103 1104 1105 static int chcr_handle_cipher_resp(struct ablkcipher_request *req, 1106 unsigned char *input, int err) 1107 { 1108 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1109 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1110 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1111 struct sk_buff *skb; 1112 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; 1113 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1114 struct cipher_wr_param wrparam; 1115 int bytes; 1116 1117 if (err) 1118 goto unmap; 1119 if (req->nbytes == reqctx->processed) { 1120 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1121 req); 1122 err = chcr_final_cipher_iv(req, fw6_pld, req->info); 1123 goto complete; 1124 } 1125 1126 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1127 c_ctx(tfm)->tx_qidx))) { 1128 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1129 err = -EBUSY; 1130 goto unmap; 1131 } 1132 1133 } 1134 if (!reqctx->imm) { 1135 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1, 1136 SPACE_LEFT(ablkctx->enckey_len), 1137 reqctx->src_ofst, reqctx->dst_ofst); 1138 if ((bytes + reqctx->processed) >= req->nbytes) 1139 bytes = req->nbytes - reqctx->processed; 1140 else 1141 bytes = ROUND_16(bytes); 1142 } else { 1143 /*CTR mode counter overfloa*/ 1144 bytes = req->nbytes - reqctx->processed; 1145 } 1146 dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1147 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 1148 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); 1149 dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1150 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 1151 if (err) 1152 goto unmap; 1153 1154 if (unlikely(bytes == 0)) { 1155 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1156 req); 1157 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1158 req->base.flags, 1159 req->src, 1160 req->dst, 1161 req->nbytes, 1162 req->info, 1163 reqctx->op); 1164 goto complete; 1165 } 1166 1167 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1168 CRYPTO_ALG_SUB_TYPE_CTR) 1169 bytes = adjust_ctr_overflow(reqctx->iv, bytes); 1170 wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx]; 1171 wrparam.req = req; 1172 wrparam.bytes = bytes; 1173 skb = create_cipher_wr(&wrparam); 1174 if (IS_ERR(skb)) { 1175 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 1176 err = PTR_ERR(skb); 1177 goto unmap; 1178 } 1179 skb->dev = u_ctx->lldi.ports[0]; 1180 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1181 chcr_send_wr(skb); 1182 reqctx->last_req_len = bytes; 1183 reqctx->processed += bytes; 1184 return 0; 1185 unmap: 1186 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1187 complete: 1188 req->base.complete(&req->base, err); 1189 return err; 1190 } 1191 1192 static int process_cipher(struct ablkcipher_request *req, 1193 unsigned short qid, 1194 struct sk_buff **skb, 1195 unsigned short op_type) 1196 { 1197 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1198 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 1199 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1200 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1201 struct cipher_wr_param wrparam; 1202 int bytes, err = -EINVAL; 1203 1204 reqctx->processed = 0; 1205 if (!req->info) 1206 goto error; 1207 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 1208 (req->nbytes == 0) || 1209 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) { 1210 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 1211 ablkctx->enckey_len, req->nbytes, ivsize); 1212 goto error; 1213 } 1214 chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1215 if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + 1216 AES_MIN_KEY_SIZE + 1217 sizeof(struct cpl_rx_phys_dsgl) + 1218 /*Min dsgl size*/ 1219 32))) { 1220 /* Can be sent as Imm*/ 1221 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; 1222 1223 dnents = sg_nents_xlen(req->dst, req->nbytes, 1224 CHCR_DST_SG_SIZE, 0); 1225 dnents += 1; // IV 1226 phys_dsgl = get_space_for_phys_dsgl(dnents); 1227 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); 1228 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 1229 reqctx->imm = (transhdr_len + IV + req->nbytes) <= 1230 SGE_MAX_WR_LEN; 1231 bytes = IV + req->nbytes; 1232 1233 } else { 1234 reqctx->imm = 0; 1235 } 1236 1237 if (!reqctx->imm) { 1238 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 1239 MIN_CIPHER_SG, 1240 SPACE_LEFT(ablkctx->enckey_len), 1241 0, 0); 1242 if ((bytes + reqctx->processed) >= req->nbytes) 1243 bytes = req->nbytes - reqctx->processed; 1244 else 1245 bytes = ROUND_16(bytes); 1246 } else { 1247 bytes = req->nbytes; 1248 } 1249 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1250 CRYPTO_ALG_SUB_TYPE_CTR) { 1251 bytes = adjust_ctr_overflow(req->info, bytes); 1252 } 1253 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1254 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { 1255 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); 1256 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info, 1257 CTR_RFC3686_IV_SIZE); 1258 1259 /* initialize counter portion of counter block */ 1260 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1261 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1262 1263 } else { 1264 1265 memcpy(reqctx->iv, req->info, IV); 1266 } 1267 if (unlikely(bytes == 0)) { 1268 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1269 req); 1270 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1271 req->base.flags, 1272 req->src, 1273 req->dst, 1274 req->nbytes, 1275 req->info, 1276 op_type); 1277 goto error; 1278 } 1279 reqctx->op = op_type; 1280 reqctx->srcsg = req->src; 1281 reqctx->dstsg = req->dst; 1282 reqctx->src_ofst = 0; 1283 reqctx->dst_ofst = 0; 1284 wrparam.qid = qid; 1285 wrparam.req = req; 1286 wrparam.bytes = bytes; 1287 *skb = create_cipher_wr(&wrparam); 1288 if (IS_ERR(*skb)) { 1289 err = PTR_ERR(*skb); 1290 goto unmap; 1291 } 1292 reqctx->processed = bytes; 1293 reqctx->last_req_len = bytes; 1294 1295 return 0; 1296 unmap: 1297 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1298 error: 1299 return err; 1300 } 1301 1302 static int chcr_aes_encrypt(struct ablkcipher_request *req) 1303 { 1304 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1305 struct sk_buff *skb = NULL; 1306 int err; 1307 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1308 1309 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1310 c_ctx(tfm)->tx_qidx))) { 1311 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1312 return -EBUSY; 1313 } 1314 1315 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], 1316 &skb, CHCR_ENCRYPT_OP); 1317 if (err || !skb) 1318 return err; 1319 skb->dev = u_ctx->lldi.ports[0]; 1320 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1321 chcr_send_wr(skb); 1322 return -EINPROGRESS; 1323 } 1324 1325 static int chcr_aes_decrypt(struct ablkcipher_request *req) 1326 { 1327 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1328 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1329 struct sk_buff *skb = NULL; 1330 int err; 1331 1332 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1333 c_ctx(tfm)->tx_qidx))) { 1334 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1335 return -EBUSY; 1336 } 1337 1338 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], 1339 &skb, CHCR_DECRYPT_OP); 1340 if (err || !skb) 1341 return err; 1342 skb->dev = u_ctx->lldi.ports[0]; 1343 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1344 chcr_send_wr(skb); 1345 return -EINPROGRESS; 1346 } 1347 1348 static int chcr_device_init(struct chcr_context *ctx) 1349 { 1350 struct uld_ctx *u_ctx = NULL; 1351 struct adapter *adap; 1352 unsigned int id; 1353 int txq_perchan, txq_idx, ntxq; 1354 int err = 0, rxq_perchan, rxq_idx; 1355 1356 id = smp_processor_id(); 1357 if (!ctx->dev) { 1358 u_ctx = assign_chcr_device(); 1359 if (!u_ctx) { 1360 pr_err("chcr device assignment fails\n"); 1361 goto out; 1362 } 1363 ctx->dev = u_ctx->dev; 1364 adap = padap(ctx->dev); 1365 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq, 1366 adap->vres.ncrypto_fc); 1367 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 1368 txq_perchan = ntxq / u_ctx->lldi.nchan; 1369 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan; 1370 rxq_idx += id % rxq_perchan; 1371 txq_idx = ctx->dev->tx_channel_id * txq_perchan; 1372 txq_idx += id % txq_perchan; 1373 spin_lock(&ctx->dev->lock_chcr_dev); 1374 ctx->rx_qidx = rxq_idx; 1375 ctx->tx_qidx = txq_idx; 1376 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; 1377 ctx->dev->rx_channel_id = 0; 1378 spin_unlock(&ctx->dev->lock_chcr_dev); 1379 } 1380 out: 1381 return err; 1382 } 1383 1384 static int chcr_cra_init(struct crypto_tfm *tfm) 1385 { 1386 struct crypto_alg *alg = tfm->__crt_alg; 1387 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1388 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1389 1390 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0, 1391 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 1392 if (IS_ERR(ablkctx->sw_cipher)) { 1393 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1394 return PTR_ERR(ablkctx->sw_cipher); 1395 } 1396 1397 if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) { 1398 /* To update tweak*/ 1399 ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0); 1400 if (IS_ERR(ablkctx->aes_generic)) { 1401 pr_err("failed to allocate aes cipher for tweak\n"); 1402 return PTR_ERR(ablkctx->aes_generic); 1403 } 1404 } else 1405 ablkctx->aes_generic = NULL; 1406 1407 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1408 return chcr_device_init(crypto_tfm_ctx(tfm)); 1409 } 1410 1411 static int chcr_rfc3686_init(struct crypto_tfm *tfm) 1412 { 1413 struct crypto_alg *alg = tfm->__crt_alg; 1414 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1415 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1416 1417 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) 1418 * cannot be used as fallback in chcr_handle_cipher_response 1419 */ 1420 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0, 1421 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 1422 if (IS_ERR(ablkctx->sw_cipher)) { 1423 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1424 return PTR_ERR(ablkctx->sw_cipher); 1425 } 1426 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1427 return chcr_device_init(crypto_tfm_ctx(tfm)); 1428 } 1429 1430 1431 static void chcr_cra_exit(struct crypto_tfm *tfm) 1432 { 1433 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1434 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1435 1436 crypto_free_skcipher(ablkctx->sw_cipher); 1437 if (ablkctx->aes_generic) 1438 crypto_free_cipher(ablkctx->aes_generic); 1439 } 1440 1441 static int get_alg_config(struct algo_param *params, 1442 unsigned int auth_size) 1443 { 1444 switch (auth_size) { 1445 case SHA1_DIGEST_SIZE: 1446 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 1447 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 1448 params->result_size = SHA1_DIGEST_SIZE; 1449 break; 1450 case SHA224_DIGEST_SIZE: 1451 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1452 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 1453 params->result_size = SHA256_DIGEST_SIZE; 1454 break; 1455 case SHA256_DIGEST_SIZE: 1456 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1457 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 1458 params->result_size = SHA256_DIGEST_SIZE; 1459 break; 1460 case SHA384_DIGEST_SIZE: 1461 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1462 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 1463 params->result_size = SHA512_DIGEST_SIZE; 1464 break; 1465 case SHA512_DIGEST_SIZE: 1466 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1467 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 1468 params->result_size = SHA512_DIGEST_SIZE; 1469 break; 1470 default: 1471 pr_err("chcr : ERROR, unsupported digest size\n"); 1472 return -EINVAL; 1473 } 1474 return 0; 1475 } 1476 1477 static inline void chcr_free_shash(struct crypto_shash *base_hash) 1478 { 1479 crypto_free_shash(base_hash); 1480 } 1481 1482 /** 1483 * create_hash_wr - Create hash work request 1484 * @req - Cipher req base 1485 */ 1486 static struct sk_buff *create_hash_wr(struct ahash_request *req, 1487 struct hash_wr_param *param) 1488 { 1489 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1490 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1491 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 1492 struct sk_buff *skb = NULL; 1493 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 1494 struct chcr_wr *chcr_req; 1495 struct ulptx_sgl *ulptx; 1496 unsigned int nents = 0, transhdr_len, iopad_alignment = 0; 1497 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1498 unsigned int kctx_len = 0, temp = 0; 1499 u8 hash_size_in_response = 0; 1500 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1501 GFP_ATOMIC; 1502 struct adapter *adap = padap(h_ctx(tfm)->dev); 1503 int error = 0; 1504 1505 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); 1506 kctx_len = param->alg_prm.result_size + iopad_alignment; 1507 if (param->opad_needed) 1508 kctx_len += param->alg_prm.result_size + iopad_alignment; 1509 1510 if (req_ctx->result) 1511 hash_size_in_response = digestsize; 1512 else 1513 hash_size_in_response = param->alg_prm.result_size; 1514 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); 1515 req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <= 1516 SGE_MAX_WR_LEN; 1517 nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0); 1518 nents += param->bfr_len ? 1 : 0; 1519 transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len + 1520 param->sg_len), 16) * 16) : 1521 (sgl_len(nents) * 8); 1522 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 1523 1524 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 1525 if (!skb) 1526 return ERR_PTR(-ENOMEM); 1527 chcr_req = __skb_put_zero(skb, transhdr_len); 1528 1529 chcr_req->sec_cpl.op_ivinsrtofst = 1530 FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0); 1531 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 1532 1533 chcr_req->sec_cpl.aadstart_cipherstop_hi = 1534 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 1535 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1536 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 1537 chcr_req->sec_cpl.seqno_numivs = 1538 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 1539 param->opad_needed, 0); 1540 1541 chcr_req->sec_cpl.ivgen_hdrlen = 1542 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 1543 1544 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 1545 param->alg_prm.result_size); 1546 1547 if (param->opad_needed) 1548 memcpy(chcr_req->key_ctx.key + 1549 ((param->alg_prm.result_size <= 32) ? 32 : 1550 CHCR_HASH_MAX_DIGEST_SIZE), 1551 hmacctx->opad, param->alg_prm.result_size); 1552 1553 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 1554 param->alg_prm.mk_size, 0, 1555 param->opad_needed, 1556 ((kctx_len + 1557 sizeof(chcr_req->key_ctx)) >> 4)); 1558 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 1559 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len + 1560 DUMMY_BYTES); 1561 if (param->bfr_len != 0) { 1562 req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev, 1563 req_ctx->reqbfr, param->bfr_len, 1564 DMA_TO_DEVICE); 1565 if (dma_mapping_error(&u_ctx->lldi.pdev->dev, 1566 req_ctx->dma_addr)) { 1567 error = -ENOMEM; 1568 goto err; 1569 } 1570 req_ctx->dma_len = param->bfr_len; 1571 } else { 1572 req_ctx->dma_addr = 0; 1573 } 1574 chcr_add_hash_src_ent(req, ulptx, param); 1575 /* Request upto max wr size */ 1576 temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len 1577 + param->bfr_len) : 0); 1578 atomic_inc(&adap->chcr_stats.digest_rqst); 1579 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm, 1580 hash_size_in_response, transhdr_len, 1581 temp, 0); 1582 req_ctx->skb = skb; 1583 return skb; 1584 err: 1585 kfree_skb(skb); 1586 return ERR_PTR(error); 1587 } 1588 1589 static int chcr_ahash_update(struct ahash_request *req) 1590 { 1591 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1592 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1593 struct uld_ctx *u_ctx = NULL; 1594 struct sk_buff *skb; 1595 u8 remainder = 0, bs; 1596 unsigned int nbytes = req->nbytes; 1597 struct hash_wr_param params; 1598 int error; 1599 1600 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1601 1602 u_ctx = ULD_CTX(h_ctx(rtfm)); 1603 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1604 h_ctx(rtfm)->tx_qidx))) { 1605 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1606 return -EBUSY; 1607 } 1608 1609 if (nbytes + req_ctx->reqlen >= bs) { 1610 remainder = (nbytes + req_ctx->reqlen) % bs; 1611 nbytes = nbytes + req_ctx->reqlen - remainder; 1612 } else { 1613 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 1614 + req_ctx->reqlen, nbytes, 0); 1615 req_ctx->reqlen += nbytes; 1616 return 0; 1617 } 1618 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1619 if (error) 1620 return -ENOMEM; 1621 params.opad_needed = 0; 1622 params.more = 1; 1623 params.last = 0; 1624 params.sg_len = nbytes - req_ctx->reqlen; 1625 params.bfr_len = req_ctx->reqlen; 1626 params.scmd1 = 0; 1627 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1628 req_ctx->result = 0; 1629 req_ctx->data_len += params.sg_len + params.bfr_len; 1630 skb = create_hash_wr(req, ¶ms); 1631 if (IS_ERR(skb)) { 1632 error = PTR_ERR(skb); 1633 goto unmap; 1634 } 1635 1636 if (remainder) { 1637 /* Swap buffers */ 1638 swap(req_ctx->reqbfr, req_ctx->skbfr); 1639 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 1640 req_ctx->reqbfr, remainder, req->nbytes - 1641 remainder); 1642 } 1643 req_ctx->reqlen = remainder; 1644 skb->dev = u_ctx->lldi.ports[0]; 1645 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1646 chcr_send_wr(skb); 1647 1648 return -EINPROGRESS; 1649 unmap: 1650 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1651 return error; 1652 } 1653 1654 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 1655 { 1656 memset(bfr_ptr, 0, bs); 1657 *bfr_ptr = 0x80; 1658 if (bs == 64) 1659 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 1660 else 1661 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 1662 } 1663 1664 static int chcr_ahash_final(struct ahash_request *req) 1665 { 1666 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1667 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1668 struct hash_wr_param params; 1669 struct sk_buff *skb; 1670 struct uld_ctx *u_ctx = NULL; 1671 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1672 1673 u_ctx = ULD_CTX(h_ctx(rtfm)); 1674 if (is_hmac(crypto_ahash_tfm(rtfm))) 1675 params.opad_needed = 1; 1676 else 1677 params.opad_needed = 0; 1678 params.sg_len = 0; 1679 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1680 req_ctx->result = 1; 1681 params.bfr_len = req_ctx->reqlen; 1682 req_ctx->data_len += params.bfr_len + params.sg_len; 1683 if (req_ctx->reqlen == 0) { 1684 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1685 params.last = 0; 1686 params.more = 1; 1687 params.scmd1 = 0; 1688 params.bfr_len = bs; 1689 1690 } else { 1691 params.scmd1 = req_ctx->data_len; 1692 params.last = 1; 1693 params.more = 0; 1694 } 1695 skb = create_hash_wr(req, ¶ms); 1696 if (IS_ERR(skb)) 1697 return PTR_ERR(skb); 1698 1699 skb->dev = u_ctx->lldi.ports[0]; 1700 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1701 chcr_send_wr(skb); 1702 return -EINPROGRESS; 1703 } 1704 1705 static int chcr_ahash_finup(struct ahash_request *req) 1706 { 1707 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1708 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1709 struct uld_ctx *u_ctx = NULL; 1710 struct sk_buff *skb; 1711 struct hash_wr_param params; 1712 u8 bs; 1713 int error; 1714 1715 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1716 u_ctx = ULD_CTX(h_ctx(rtfm)); 1717 1718 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1719 h_ctx(rtfm)->tx_qidx))) { 1720 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1721 return -EBUSY; 1722 } 1723 1724 if (is_hmac(crypto_ahash_tfm(rtfm))) 1725 params.opad_needed = 1; 1726 else 1727 params.opad_needed = 0; 1728 1729 params.sg_len = req->nbytes; 1730 params.bfr_len = req_ctx->reqlen; 1731 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1732 req_ctx->data_len += params.bfr_len + params.sg_len; 1733 req_ctx->result = 1; 1734 if ((req_ctx->reqlen + req->nbytes) == 0) { 1735 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1736 params.last = 0; 1737 params.more = 1; 1738 params.scmd1 = 0; 1739 params.bfr_len = bs; 1740 } else { 1741 params.scmd1 = req_ctx->data_len; 1742 params.last = 1; 1743 params.more = 0; 1744 } 1745 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1746 if (error) 1747 return -ENOMEM; 1748 1749 skb = create_hash_wr(req, ¶ms); 1750 if (IS_ERR(skb)) { 1751 error = PTR_ERR(skb); 1752 goto unmap; 1753 } 1754 skb->dev = u_ctx->lldi.ports[0]; 1755 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1756 chcr_send_wr(skb); 1757 1758 return -EINPROGRESS; 1759 unmap: 1760 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1761 return error; 1762 } 1763 1764 static int chcr_ahash_digest(struct ahash_request *req) 1765 { 1766 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1767 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1768 struct uld_ctx *u_ctx = NULL; 1769 struct sk_buff *skb; 1770 struct hash_wr_param params; 1771 u8 bs; 1772 int error; 1773 1774 rtfm->init(req); 1775 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1776 1777 u_ctx = ULD_CTX(h_ctx(rtfm)); 1778 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1779 h_ctx(rtfm)->tx_qidx))) { 1780 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1781 return -EBUSY; 1782 } 1783 1784 if (is_hmac(crypto_ahash_tfm(rtfm))) 1785 params.opad_needed = 1; 1786 else 1787 params.opad_needed = 0; 1788 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1789 if (error) 1790 return -ENOMEM; 1791 1792 params.last = 0; 1793 params.more = 0; 1794 params.sg_len = req->nbytes; 1795 params.bfr_len = 0; 1796 params.scmd1 = 0; 1797 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1798 req_ctx->result = 1; 1799 req_ctx->data_len += params.bfr_len + params.sg_len; 1800 1801 if (req->nbytes == 0) { 1802 create_last_hash_block(req_ctx->reqbfr, bs, 0); 1803 params.more = 1; 1804 params.bfr_len = bs; 1805 } 1806 1807 skb = create_hash_wr(req, ¶ms); 1808 if (IS_ERR(skb)) { 1809 error = PTR_ERR(skb); 1810 goto unmap; 1811 } 1812 skb->dev = u_ctx->lldi.ports[0]; 1813 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1814 chcr_send_wr(skb); 1815 return -EINPROGRESS; 1816 unmap: 1817 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1818 return error; 1819 } 1820 1821 static int chcr_ahash_export(struct ahash_request *areq, void *out) 1822 { 1823 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1824 struct chcr_ahash_req_ctx *state = out; 1825 1826 state->reqlen = req_ctx->reqlen; 1827 state->data_len = req_ctx->data_len; 1828 state->is_sg_map = 0; 1829 state->result = 0; 1830 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 1831 memcpy(state->partial_hash, req_ctx->partial_hash, 1832 CHCR_HASH_MAX_DIGEST_SIZE); 1833 return 0; 1834 } 1835 1836 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 1837 { 1838 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1839 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 1840 1841 req_ctx->reqlen = state->reqlen; 1842 req_ctx->data_len = state->data_len; 1843 req_ctx->reqbfr = req_ctx->bfr1; 1844 req_ctx->skbfr = req_ctx->bfr2; 1845 req_ctx->is_sg_map = 0; 1846 req_ctx->result = 0; 1847 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 1848 memcpy(req_ctx->partial_hash, state->partial_hash, 1849 CHCR_HASH_MAX_DIGEST_SIZE); 1850 return 0; 1851 } 1852 1853 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 1854 unsigned int keylen) 1855 { 1856 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 1857 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1858 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1859 unsigned int i, err = 0, updated_digestsize; 1860 1861 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 1862 1863 /* use the key to calculate the ipad and opad. ipad will sent with the 1864 * first request's data. opad will be sent with the final hash result 1865 * ipad in hmacctx->ipad and opad in hmacctx->opad location 1866 */ 1867 shash->tfm = hmacctx->base_hash; 1868 shash->flags = crypto_shash_get_flags(hmacctx->base_hash); 1869 if (keylen > bs) { 1870 err = crypto_shash_digest(shash, key, keylen, 1871 hmacctx->ipad); 1872 if (err) 1873 goto out; 1874 keylen = digestsize; 1875 } else { 1876 memcpy(hmacctx->ipad, key, keylen); 1877 } 1878 memset(hmacctx->ipad + keylen, 0, bs - keylen); 1879 memcpy(hmacctx->opad, hmacctx->ipad, bs); 1880 1881 for (i = 0; i < bs / sizeof(int); i++) { 1882 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 1883 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 1884 } 1885 1886 updated_digestsize = digestsize; 1887 if (digestsize == SHA224_DIGEST_SIZE) 1888 updated_digestsize = SHA256_DIGEST_SIZE; 1889 else if (digestsize == SHA384_DIGEST_SIZE) 1890 updated_digestsize = SHA512_DIGEST_SIZE; 1891 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 1892 hmacctx->ipad, digestsize); 1893 if (err) 1894 goto out; 1895 chcr_change_order(hmacctx->ipad, updated_digestsize); 1896 1897 err = chcr_compute_partial_hash(shash, hmacctx->opad, 1898 hmacctx->opad, digestsize); 1899 if (err) 1900 goto out; 1901 chcr_change_order(hmacctx->opad, updated_digestsize); 1902 out: 1903 return err; 1904 } 1905 1906 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 1907 unsigned int key_len) 1908 { 1909 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 1910 unsigned short context_size = 0; 1911 int err; 1912 1913 err = chcr_cipher_fallback_setkey(cipher, key, key_len); 1914 if (err) 1915 goto badkey_err; 1916 1917 memcpy(ablkctx->key, key, key_len); 1918 ablkctx->enckey_len = key_len; 1919 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 1920 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 1921 ablkctx->key_ctx_hdr = 1922 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 1923 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 1924 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 1925 CHCR_KEYCTX_NO_KEY, 1, 1926 0, context_size); 1927 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 1928 return 0; 1929 badkey_err: 1930 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 1931 ablkctx->enckey_len = 0; 1932 1933 return err; 1934 } 1935 1936 static int chcr_sha_init(struct ahash_request *areq) 1937 { 1938 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1939 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1940 int digestsize = crypto_ahash_digestsize(tfm); 1941 1942 req_ctx->data_len = 0; 1943 req_ctx->reqlen = 0; 1944 req_ctx->reqbfr = req_ctx->bfr1; 1945 req_ctx->skbfr = req_ctx->bfr2; 1946 req_ctx->skb = NULL; 1947 req_ctx->result = 0; 1948 req_ctx->is_sg_map = 0; 1949 copy_hash_init_values(req_ctx->partial_hash, digestsize); 1950 return 0; 1951 } 1952 1953 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 1954 { 1955 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1956 sizeof(struct chcr_ahash_req_ctx)); 1957 return chcr_device_init(crypto_tfm_ctx(tfm)); 1958 } 1959 1960 static int chcr_hmac_init(struct ahash_request *areq) 1961 { 1962 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1963 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 1964 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); 1965 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 1966 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1967 1968 chcr_sha_init(areq); 1969 req_ctx->data_len = bs; 1970 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1971 if (digestsize == SHA224_DIGEST_SIZE) 1972 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1973 SHA256_DIGEST_SIZE); 1974 else if (digestsize == SHA384_DIGEST_SIZE) 1975 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1976 SHA512_DIGEST_SIZE); 1977 else 1978 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1979 digestsize); 1980 } 1981 return 0; 1982 } 1983 1984 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 1985 { 1986 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1987 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1988 unsigned int digestsize = 1989 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 1990 1991 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1992 sizeof(struct chcr_ahash_req_ctx)); 1993 hmacctx->base_hash = chcr_alloc_shash(digestsize); 1994 if (IS_ERR(hmacctx->base_hash)) 1995 return PTR_ERR(hmacctx->base_hash); 1996 return chcr_device_init(crypto_tfm_ctx(tfm)); 1997 } 1998 1999 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 2000 { 2001 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2002 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2003 2004 if (hmacctx->base_hash) { 2005 chcr_free_shash(hmacctx->base_hash); 2006 hmacctx->base_hash = NULL; 2007 } 2008 } 2009 2010 static int chcr_aead_common_init(struct aead_request *req, 2011 unsigned short op_type) 2012 { 2013 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2014 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2015 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2016 int error = -EINVAL; 2017 unsigned int dst_size; 2018 unsigned int authsize = crypto_aead_authsize(tfm); 2019 2020 dst_size = req->assoclen + req->cryptlen + (op_type ? 2021 -authsize : authsize); 2022 /* validate key size */ 2023 if (aeadctx->enckey_len == 0) 2024 goto err; 2025 if (op_type && req->cryptlen < authsize) 2026 goto err; 2027 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2028 op_type); 2029 if (error) { 2030 error = -ENOMEM; 2031 goto err; 2032 } 2033 reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen, 2034 CHCR_SRC_SG_SIZE, 0); 2035 reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen, 2036 CHCR_SRC_SG_SIZE, req->assoclen); 2037 return 0; 2038 err: 2039 return error; 2040 } 2041 2042 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, 2043 int aadmax, int wrlen, 2044 unsigned short op_type) 2045 { 2046 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 2047 2048 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || 2049 dst_nents > MAX_DSGL_ENT || 2050 (req->assoclen > aadmax) || 2051 (wrlen > SGE_MAX_WR_LEN)) 2052 return 1; 2053 return 0; 2054 } 2055 2056 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) 2057 { 2058 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2059 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2060 struct aead_request *subreq = aead_request_ctx(req); 2061 2062 aead_request_set_tfm(subreq, aeadctx->sw_cipher); 2063 aead_request_set_callback(subreq, req->base.flags, 2064 req->base.complete, req->base.data); 2065 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2066 req->iv); 2067 aead_request_set_ad(subreq, req->assoclen); 2068 return op_type ? crypto_aead_decrypt(subreq) : 2069 crypto_aead_encrypt(subreq); 2070 } 2071 2072 static struct sk_buff *create_authenc_wr(struct aead_request *req, 2073 unsigned short qid, 2074 int size, 2075 unsigned short op_type) 2076 { 2077 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2078 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2079 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2080 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2081 struct sk_buff *skb = NULL; 2082 struct chcr_wr *chcr_req; 2083 struct cpl_rx_phys_dsgl *phys_cpl; 2084 struct ulptx_sgl *ulptx; 2085 unsigned int transhdr_len; 2086 unsigned int dst_size = 0, temp; 2087 unsigned int kctx_len = 0, dnents; 2088 unsigned int assoclen = req->assoclen; 2089 unsigned int authsize = crypto_aead_authsize(tfm); 2090 int error = -EINVAL; 2091 int null = 0; 2092 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2093 GFP_ATOMIC; 2094 struct adapter *adap = padap(a_ctx(tfm)->dev); 2095 2096 if (req->cryptlen == 0) 2097 return NULL; 2098 2099 reqctx->b0_dma = 0; 2100 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { 2101 null = 1; 2102 assoclen = 0; 2103 } 2104 dst_size = assoclen + req->cryptlen + (op_type ? -authsize : 2105 authsize); 2106 error = chcr_aead_common_init(req, op_type); 2107 if (error) 2108 return ERR_PTR(error); 2109 if (dst_size) { 2110 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); 2111 dnents += sg_nents_xlen(req->dst, req->cryptlen + 2112 (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, 2113 req->assoclen); 2114 dnents += MIN_AUTH_SG; // For IV 2115 } else { 2116 dnents = 0; 2117 } 2118 2119 dst_size = get_space_for_phys_dsgl(dnents); 2120 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) 2121 - sizeof(chcr_req->key_ctx); 2122 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2123 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) < 2124 SGE_MAX_WR_LEN; 2125 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16) 2126 * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents 2127 + MIN_GCM_SG) * 8); 2128 transhdr_len += temp; 2129 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 2130 2131 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 2132 transhdr_len, op_type)) { 2133 atomic_inc(&adap->chcr_stats.fallback); 2134 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2135 op_type); 2136 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2137 } 2138 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 2139 if (!skb) { 2140 error = -ENOMEM; 2141 goto err; 2142 } 2143 2144 chcr_req = __skb_put_zero(skb, transhdr_len); 2145 2146 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 2147 2148 /* 2149 * Input order is AAD,IV and Payload. where IV should be included as 2150 * the part of authdata. All other fields should be filled according 2151 * to the hardware spec 2152 */ 2153 chcr_req->sec_cpl.op_ivinsrtofst = 2154 FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2, 2155 assoclen + 1); 2156 chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen); 2157 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2158 assoclen ? 1 : 0, assoclen, 2159 assoclen + IV + 1, 2160 (temp & 0x1F0) >> 4); 2161 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 2162 temp & 0xF, 2163 null ? 0 : assoclen + IV + 1, 2164 temp, temp); 2165 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2166 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, 2167 CHCR_SCMD_CIPHER_MODE_AES_CBC, 2168 actx->auth_mode, aeadctx->hmac_ctrl, 2169 IV >> 1); 2170 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2171 0, 0, dst_size); 2172 2173 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2174 if (op_type == CHCR_ENCRYPT_OP) 2175 memcpy(chcr_req->key_ctx.key, aeadctx->key, 2176 aeadctx->enckey_len); 2177 else 2178 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 2179 aeadctx->enckey_len); 2180 2181 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 2182 4), actx->h_iopad, kctx_len - 2183 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); 2184 memcpy(reqctx->iv, req->iv, IV); 2185 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2186 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 2187 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); 2188 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); 2189 atomic_inc(&adap->chcr_stats.cipher_rqst); 2190 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + 2191 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); 2192 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 2193 transhdr_len, temp, 0); 2194 reqctx->skb = skb; 2195 reqctx->op = op_type; 2196 2197 return skb; 2198 err: 2199 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2200 op_type); 2201 2202 return ERR_PTR(error); 2203 } 2204 2205 static int chcr_aead_dma_map(struct device *dev, 2206 struct aead_request *req, 2207 unsigned short op_type) 2208 { 2209 int error; 2210 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2211 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2212 unsigned int authsize = crypto_aead_authsize(tfm); 2213 int dst_size; 2214 2215 dst_size = req->assoclen + req->cryptlen + (op_type ? 2216 -authsize : authsize); 2217 if (!req->cryptlen || !dst_size) 2218 return 0; 2219 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, 2220 DMA_BIDIRECTIONAL); 2221 if (dma_mapping_error(dev, reqctx->iv_dma)) 2222 return -ENOMEM; 2223 2224 if (req->src == req->dst) { 2225 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2226 DMA_BIDIRECTIONAL); 2227 if (!error) 2228 goto err; 2229 } else { 2230 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2231 DMA_TO_DEVICE); 2232 if (!error) 2233 goto err; 2234 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2235 DMA_FROM_DEVICE); 2236 if (!error) { 2237 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2238 DMA_TO_DEVICE); 2239 goto err; 2240 } 2241 } 2242 2243 return 0; 2244 err: 2245 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 2246 return -ENOMEM; 2247 } 2248 2249 static void chcr_aead_dma_unmap(struct device *dev, 2250 struct aead_request *req, 2251 unsigned short op_type) 2252 { 2253 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2254 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2255 unsigned int authsize = crypto_aead_authsize(tfm); 2256 int dst_size; 2257 2258 dst_size = req->assoclen + req->cryptlen + (op_type ? 2259 -authsize : authsize); 2260 if (!req->cryptlen || !dst_size) 2261 return; 2262 2263 dma_unmap_single(dev, reqctx->iv_dma, IV, 2264 DMA_BIDIRECTIONAL); 2265 if (req->src == req->dst) { 2266 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2267 DMA_BIDIRECTIONAL); 2268 } else { 2269 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2270 DMA_TO_DEVICE); 2271 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2272 DMA_FROM_DEVICE); 2273 } 2274 } 2275 2276 static inline void chcr_add_aead_src_ent(struct aead_request *req, 2277 struct ulptx_sgl *ulptx, 2278 unsigned int assoclen, 2279 unsigned short op_type) 2280 { 2281 struct ulptx_walk ulp_walk; 2282 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2283 2284 if (reqctx->imm) { 2285 u8 *buf = (u8 *)ulptx; 2286 2287 if (reqctx->b0_dma) { 2288 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); 2289 buf += reqctx->b0_len; 2290 } 2291 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2292 buf, assoclen, 0); 2293 buf += assoclen; 2294 memcpy(buf, reqctx->iv, IV); 2295 buf += IV; 2296 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2297 buf, req->cryptlen, req->assoclen); 2298 } else { 2299 ulptx_walk_init(&ulp_walk, ulptx); 2300 if (reqctx->b0_dma) 2301 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, 2302 &reqctx->b0_dma); 2303 ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0); 2304 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); 2305 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen, 2306 req->assoclen); 2307 ulptx_walk_end(&ulp_walk); 2308 } 2309 } 2310 2311 static inline void chcr_add_aead_dst_ent(struct aead_request *req, 2312 struct cpl_rx_phys_dsgl *phys_cpl, 2313 unsigned int assoclen, 2314 unsigned short op_type, 2315 unsigned short qid) 2316 { 2317 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2318 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2319 struct dsgl_walk dsgl_walk; 2320 unsigned int authsize = crypto_aead_authsize(tfm); 2321 u32 temp; 2322 2323 dsgl_walk_init(&dsgl_walk, phys_cpl); 2324 if (reqctx->b0_dma) 2325 dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma); 2326 dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0); 2327 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); 2328 temp = req->cryptlen + (op_type ? -authsize : authsize); 2329 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen); 2330 dsgl_walk_end(&dsgl_walk, qid); 2331 } 2332 2333 static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, 2334 struct ulptx_sgl *ulptx, 2335 struct cipher_wr_param *wrparam) 2336 { 2337 struct ulptx_walk ulp_walk; 2338 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2339 2340 if (reqctx->imm) { 2341 u8 *buf = (u8 *)ulptx; 2342 2343 memcpy(buf, reqctx->iv, IV); 2344 buf += IV; 2345 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2346 buf, wrparam->bytes, reqctx->processed); 2347 } else { 2348 ulptx_walk_init(&ulp_walk, ulptx); 2349 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); 2350 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, 2351 reqctx->src_ofst); 2352 reqctx->srcsg = ulp_walk.last_sg; 2353 reqctx->src_ofst = ulp_walk.last_sg_len; 2354 ulptx_walk_end(&ulp_walk); 2355 } 2356 } 2357 2358 static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, 2359 struct cpl_rx_phys_dsgl *phys_cpl, 2360 struct cipher_wr_param *wrparam, 2361 unsigned short qid) 2362 { 2363 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2364 struct dsgl_walk dsgl_walk; 2365 2366 dsgl_walk_init(&dsgl_walk, phys_cpl); 2367 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); 2368 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, 2369 reqctx->dst_ofst); 2370 reqctx->dstsg = dsgl_walk.last_sg; 2371 reqctx->dst_ofst = dsgl_walk.last_sg_len; 2372 2373 dsgl_walk_end(&dsgl_walk, qid); 2374 } 2375 2376 static inline void chcr_add_hash_src_ent(struct ahash_request *req, 2377 struct ulptx_sgl *ulptx, 2378 struct hash_wr_param *param) 2379 { 2380 struct ulptx_walk ulp_walk; 2381 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2382 2383 if (reqctx->imm) { 2384 u8 *buf = (u8 *)ulptx; 2385 2386 if (param->bfr_len) { 2387 memcpy(buf, reqctx->reqbfr, param->bfr_len); 2388 buf += param->bfr_len; 2389 } 2390 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2391 buf, param->sg_len, 0); 2392 } else { 2393 ulptx_walk_init(&ulp_walk, ulptx); 2394 if (param->bfr_len) 2395 ulptx_walk_add_page(&ulp_walk, param->bfr_len, 2396 &reqctx->dma_addr); 2397 ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len, 2398 0); 2399 // reqctx->srcsg = ulp_walk.last_sg; 2400 // reqctx->src_ofst = ulp_walk.last_sg_len; 2401 ulptx_walk_end(&ulp_walk); 2402 } 2403 } 2404 2405 2406 static inline int chcr_hash_dma_map(struct device *dev, 2407 struct ahash_request *req) 2408 { 2409 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2410 int error = 0; 2411 2412 if (!req->nbytes) 2413 return 0; 2414 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2415 DMA_TO_DEVICE); 2416 if (!error) 2417 return error; 2418 req_ctx->is_sg_map = 1; 2419 return 0; 2420 } 2421 2422 static inline void chcr_hash_dma_unmap(struct device *dev, 2423 struct ahash_request *req) 2424 { 2425 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2426 2427 if (!req->nbytes) 2428 return; 2429 2430 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2431 DMA_TO_DEVICE); 2432 req_ctx->is_sg_map = 0; 2433 2434 } 2435 2436 2437 static int chcr_cipher_dma_map(struct device *dev, 2438 struct ablkcipher_request *req) 2439 { 2440 int error; 2441 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2442 2443 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, 2444 DMA_BIDIRECTIONAL); 2445 if (dma_mapping_error(dev, reqctx->iv_dma)) 2446 return -ENOMEM; 2447 2448 if (req->src == req->dst) { 2449 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2450 DMA_BIDIRECTIONAL); 2451 if (!error) 2452 goto err; 2453 } else { 2454 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2455 DMA_TO_DEVICE); 2456 if (!error) 2457 goto err; 2458 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2459 DMA_FROM_DEVICE); 2460 if (!error) { 2461 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2462 DMA_TO_DEVICE); 2463 goto err; 2464 } 2465 } 2466 2467 return 0; 2468 err: 2469 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 2470 return -ENOMEM; 2471 } 2472 static void chcr_cipher_dma_unmap(struct device *dev, 2473 struct ablkcipher_request *req) 2474 { 2475 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2476 2477 dma_unmap_single(dev, reqctx->iv_dma, IV, 2478 DMA_BIDIRECTIONAL); 2479 if (req->src == req->dst) { 2480 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2481 DMA_BIDIRECTIONAL); 2482 } else { 2483 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2484 DMA_TO_DEVICE); 2485 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2486 DMA_FROM_DEVICE); 2487 } 2488 } 2489 2490 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 2491 { 2492 __be32 data; 2493 2494 memset(block, 0, csize); 2495 block += csize; 2496 2497 if (csize >= 4) 2498 csize = 4; 2499 else if (msglen > (unsigned int)(1 << (8 * csize))) 2500 return -EOVERFLOW; 2501 2502 data = cpu_to_be32(msglen); 2503 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 2504 2505 return 0; 2506 } 2507 2508 static void generate_b0(struct aead_request *req, 2509 struct chcr_aead_ctx *aeadctx, 2510 unsigned short op_type) 2511 { 2512 unsigned int l, lp, m; 2513 int rc; 2514 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2515 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2516 u8 *b0 = reqctx->scratch_pad; 2517 2518 m = crypto_aead_authsize(aead); 2519 2520 memcpy(b0, reqctx->iv, 16); 2521 2522 lp = b0[0]; 2523 l = lp + 1; 2524 2525 /* set m, bits 3-5 */ 2526 *b0 |= (8 * ((m - 2) / 2)); 2527 2528 /* set adata, bit 6, if associated data is used */ 2529 if (req->assoclen) 2530 *b0 |= 64; 2531 rc = set_msg_len(b0 + 16 - l, 2532 (op_type == CHCR_DECRYPT_OP) ? 2533 req->cryptlen - m : req->cryptlen, l); 2534 } 2535 2536 static inline int crypto_ccm_check_iv(const u8 *iv) 2537 { 2538 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 2539 if (iv[0] < 1 || iv[0] > 7) 2540 return -EINVAL; 2541 2542 return 0; 2543 } 2544 2545 static int ccm_format_packet(struct aead_request *req, 2546 struct chcr_aead_ctx *aeadctx, 2547 unsigned int sub_type, 2548 unsigned short op_type) 2549 { 2550 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2551 int rc = 0; 2552 2553 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2554 reqctx->iv[0] = 3; 2555 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3); 2556 memcpy(reqctx->iv + 4, req->iv, 8); 2557 memset(reqctx->iv + 12, 0, 4); 2558 *((unsigned short *)(reqctx->scratch_pad + 16)) = 2559 htons(req->assoclen - 8); 2560 } else { 2561 memcpy(reqctx->iv, req->iv, 16); 2562 *((unsigned short *)(reqctx->scratch_pad + 16)) = 2563 htons(req->assoclen); 2564 } 2565 generate_b0(req, aeadctx, op_type); 2566 /* zero the ctr value */ 2567 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1); 2568 return rc; 2569 } 2570 2571 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 2572 unsigned int dst_size, 2573 struct aead_request *req, 2574 unsigned short op_type) 2575 { 2576 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2577 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2578 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 2579 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 2580 unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id; 2581 unsigned int ccm_xtra; 2582 unsigned char tag_offset = 0, auth_offset = 0; 2583 unsigned int assoclen; 2584 2585 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2586 assoclen = req->assoclen - 8; 2587 else 2588 assoclen = req->assoclen; 2589 ccm_xtra = CCM_B0_SIZE + 2590 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 2591 2592 auth_offset = req->cryptlen ? 2593 (assoclen + IV + 1 + ccm_xtra) : 0; 2594 if (op_type == CHCR_DECRYPT_OP) { 2595 if (crypto_aead_authsize(tfm) != req->cryptlen) 2596 tag_offset = crypto_aead_authsize(tfm); 2597 else 2598 auth_offset = 0; 2599 } 2600 2601 2602 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, 2603 2, assoclen + 1 + ccm_xtra); 2604 sec_cpl->pldlen = 2605 htonl(assoclen + IV + req->cryptlen + ccm_xtra); 2606 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 2607 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2608 1, assoclen + ccm_xtra, assoclen 2609 + IV + 1 + ccm_xtra, 0); 2610 2611 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 2612 auth_offset, tag_offset, 2613 (op_type == CHCR_ENCRYPT_OP) ? 0 : 2614 crypto_aead_authsize(tfm)); 2615 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2616 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 2617 cipher_mode, mac_mode, 2618 aeadctx->hmac_ctrl, IV >> 1); 2619 2620 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 2621 0, dst_size); 2622 } 2623 2624 int aead_ccm_validate_input(unsigned short op_type, 2625 struct aead_request *req, 2626 struct chcr_aead_ctx *aeadctx, 2627 unsigned int sub_type) 2628 { 2629 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2630 if (crypto_ccm_check_iv(req->iv)) { 2631 pr_err("CCM: IV check fails\n"); 2632 return -EINVAL; 2633 } 2634 } else { 2635 if (req->assoclen != 16 && req->assoclen != 20) { 2636 pr_err("RFC4309: Invalid AAD length %d\n", 2637 req->assoclen); 2638 return -EINVAL; 2639 } 2640 } 2641 return 0; 2642 } 2643 2644 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 2645 unsigned short qid, 2646 int size, 2647 unsigned short op_type) 2648 { 2649 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2650 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2651 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2652 struct sk_buff *skb = NULL; 2653 struct chcr_wr *chcr_req; 2654 struct cpl_rx_phys_dsgl *phys_cpl; 2655 struct ulptx_sgl *ulptx; 2656 unsigned int transhdr_len; 2657 unsigned int dst_size = 0, kctx_len, dnents, temp; 2658 unsigned int sub_type, assoclen = req->assoclen; 2659 unsigned int authsize = crypto_aead_authsize(tfm); 2660 int error = -EINVAL; 2661 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2662 GFP_ATOMIC; 2663 struct adapter *adap = padap(a_ctx(tfm)->dev); 2664 2665 reqctx->b0_dma = 0; 2666 sub_type = get_aead_subtype(tfm); 2667 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2668 assoclen -= 8; 2669 dst_size = assoclen + req->cryptlen + (op_type ? -authsize : 2670 authsize); 2671 error = chcr_aead_common_init(req, op_type); 2672 if (error) 2673 return ERR_PTR(error); 2674 2675 2676 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); 2677 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); 2678 if (error) 2679 goto err; 2680 if (dst_size) { 2681 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); 2682 dnents += sg_nents_xlen(req->dst, req->cryptlen 2683 + (op_type ? -authsize : authsize), 2684 CHCR_DST_SG_SIZE, req->assoclen); 2685 dnents += MIN_CCM_SG; // For IV and B0 2686 } else { 2687 dnents = 0; 2688 } 2689 dst_size = get_space_for_phys_dsgl(dnents); 2690 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; 2691 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2692 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen + 2693 reqctx->b0_len) <= SGE_MAX_WR_LEN; 2694 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen + 2695 reqctx->b0_len), 16) * 16) : 2696 (sgl_len(reqctx->src_nents + reqctx->aad_nents + 2697 MIN_CCM_SG) * 8); 2698 transhdr_len += temp; 2699 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 2700 2701 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - 2702 reqctx->b0_len, transhdr_len, op_type)) { 2703 atomic_inc(&adap->chcr_stats.fallback); 2704 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2705 op_type); 2706 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2707 } 2708 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 2709 2710 if (!skb) { 2711 error = -ENOMEM; 2712 goto err; 2713 } 2714 2715 chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len); 2716 2717 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type); 2718 2719 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2720 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 2721 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 2722 16), aeadctx->key, aeadctx->enckey_len); 2723 2724 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2725 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 2726 error = ccm_format_packet(req, aeadctx, sub_type, op_type); 2727 if (error) 2728 goto dstmap_fail; 2729 2730 reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, 2731 &reqctx->scratch_pad, reqctx->b0_len, 2732 DMA_BIDIRECTIONAL); 2733 if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, 2734 reqctx->b0_dma)) { 2735 error = -ENOMEM; 2736 goto dstmap_fail; 2737 } 2738 2739 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); 2740 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); 2741 2742 atomic_inc(&adap->chcr_stats.aead_rqst); 2743 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + 2744 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen + 2745 reqctx->b0_len) : 0); 2746 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, 2747 transhdr_len, temp, 0); 2748 reqctx->skb = skb; 2749 reqctx->op = op_type; 2750 2751 return skb; 2752 dstmap_fail: 2753 kfree_skb(skb); 2754 err: 2755 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); 2756 return ERR_PTR(error); 2757 } 2758 2759 static struct sk_buff *create_gcm_wr(struct aead_request *req, 2760 unsigned short qid, 2761 int size, 2762 unsigned short op_type) 2763 { 2764 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2765 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2766 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2767 struct sk_buff *skb = NULL; 2768 struct chcr_wr *chcr_req; 2769 struct cpl_rx_phys_dsgl *phys_cpl; 2770 struct ulptx_sgl *ulptx; 2771 unsigned int transhdr_len, dnents = 0; 2772 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; 2773 unsigned int authsize = crypto_aead_authsize(tfm); 2774 int error = -EINVAL; 2775 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2776 GFP_ATOMIC; 2777 struct adapter *adap = padap(a_ctx(tfm)->dev); 2778 2779 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 2780 assoclen = req->assoclen - 8; 2781 2782 reqctx->b0_dma = 0; 2783 dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); 2784 error = chcr_aead_common_init(req, op_type); 2785 if (error) 2786 return ERR_PTR(error); 2787 if (dst_size) { 2788 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); 2789 dnents += sg_nents_xlen(req->dst, 2790 req->cryptlen + (op_type ? -authsize : authsize), 2791 CHCR_DST_SG_SIZE, req->assoclen); 2792 dnents += MIN_GCM_SG; // For IV 2793 } else { 2794 dnents = 0; 2795 } 2796 dst_size = get_space_for_phys_dsgl(dnents); 2797 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + 2798 AEAD_H_SIZE; 2799 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2800 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <= 2801 SGE_MAX_WR_LEN; 2802 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + 2803 req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents + 2804 reqctx->aad_nents + MIN_GCM_SG) * 8); 2805 transhdr_len += temp; 2806 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 2807 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 2808 transhdr_len, op_type)) { 2809 atomic_inc(&adap->chcr_stats.fallback); 2810 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2811 op_type); 2812 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2813 } 2814 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 2815 if (!skb) { 2816 error = -ENOMEM; 2817 goto err; 2818 } 2819 2820 chcr_req = __skb_put_zero(skb, transhdr_len); 2821 2822 //Offset of tag from end 2823 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 2824 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 2825 a_ctx(tfm)->dev->rx_channel_id, 2, 2826 (assoclen + 1)); 2827 chcr_req->sec_cpl.pldlen = 2828 htonl(assoclen + IV + req->cryptlen); 2829 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2830 assoclen ? 1 : 0, assoclen, 2831 assoclen + IV + 1, 0); 2832 chcr_req->sec_cpl.cipherstop_lo_authinsert = 2833 FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1, 2834 temp, temp); 2835 chcr_req->sec_cpl.seqno_numivs = 2836 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == 2837 CHCR_ENCRYPT_OP) ? 1 : 0, 2838 CHCR_SCMD_CIPHER_MODE_AES_GCM, 2839 CHCR_SCMD_AUTH_MODE_GHASH, 2840 aeadctx->hmac_ctrl, IV >> 1); 2841 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2842 0, 0, dst_size); 2843 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2844 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 2845 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 2846 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 2847 2848 /* prepare a 16 byte iv */ 2849 /* S A L T | IV | 0x00000001 */ 2850 if (get_aead_subtype(tfm) == 2851 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 2852 memcpy(reqctx->iv, aeadctx->salt, 4); 2853 memcpy(reqctx->iv + 4, req->iv, GCM_RFC4106_IV_SIZE); 2854 } else { 2855 memcpy(reqctx->iv, req->iv, GCM_AES_IV_SIZE); 2856 } 2857 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); 2858 2859 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2860 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 2861 2862 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); 2863 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); 2864 atomic_inc(&adap->chcr_stats.aead_rqst); 2865 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + 2866 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); 2867 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 2868 transhdr_len, temp, reqctx->verify); 2869 reqctx->skb = skb; 2870 reqctx->op = op_type; 2871 return skb; 2872 2873 err: 2874 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); 2875 return ERR_PTR(error); 2876 } 2877 2878 2879 2880 static int chcr_aead_cra_init(struct crypto_aead *tfm) 2881 { 2882 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2883 struct aead_alg *alg = crypto_aead_alg(tfm); 2884 2885 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, 2886 CRYPTO_ALG_NEED_FALLBACK | 2887 CRYPTO_ALG_ASYNC); 2888 if (IS_ERR(aeadctx->sw_cipher)) 2889 return PTR_ERR(aeadctx->sw_cipher); 2890 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), 2891 sizeof(struct aead_request) + 2892 crypto_aead_reqsize(aeadctx->sw_cipher))); 2893 return chcr_device_init(a_ctx(tfm)); 2894 } 2895 2896 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 2897 { 2898 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2899 2900 crypto_free_aead(aeadctx->sw_cipher); 2901 } 2902 2903 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 2904 unsigned int authsize) 2905 { 2906 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2907 2908 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 2909 aeadctx->mayverify = VERIFY_HW; 2910 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2911 } 2912 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 2913 unsigned int authsize) 2914 { 2915 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2916 u32 maxauth = crypto_aead_maxauthsize(tfm); 2917 2918 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 2919 * true for sha1. authsize == 12 condition should be before 2920 * authsize == (maxauth >> 1) 2921 */ 2922 if (authsize == ICV_4) { 2923 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2924 aeadctx->mayverify = VERIFY_HW; 2925 } else if (authsize == ICV_6) { 2926 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2927 aeadctx->mayverify = VERIFY_HW; 2928 } else if (authsize == ICV_10) { 2929 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2930 aeadctx->mayverify = VERIFY_HW; 2931 } else if (authsize == ICV_12) { 2932 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2933 aeadctx->mayverify = VERIFY_HW; 2934 } else if (authsize == ICV_14) { 2935 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2936 aeadctx->mayverify = VERIFY_HW; 2937 } else if (authsize == (maxauth >> 1)) { 2938 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2939 aeadctx->mayverify = VERIFY_HW; 2940 } else if (authsize == maxauth) { 2941 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2942 aeadctx->mayverify = VERIFY_HW; 2943 } else { 2944 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2945 aeadctx->mayverify = VERIFY_SW; 2946 } 2947 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2948 } 2949 2950 2951 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 2952 { 2953 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2954 2955 switch (authsize) { 2956 case ICV_4: 2957 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2958 aeadctx->mayverify = VERIFY_HW; 2959 break; 2960 case ICV_8: 2961 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2962 aeadctx->mayverify = VERIFY_HW; 2963 break; 2964 case ICV_12: 2965 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2966 aeadctx->mayverify = VERIFY_HW; 2967 break; 2968 case ICV_14: 2969 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2970 aeadctx->mayverify = VERIFY_HW; 2971 break; 2972 case ICV_16: 2973 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2974 aeadctx->mayverify = VERIFY_HW; 2975 break; 2976 case ICV_13: 2977 case ICV_15: 2978 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2979 aeadctx->mayverify = VERIFY_SW; 2980 break; 2981 default: 2982 2983 crypto_tfm_set_flags((struct crypto_tfm *) tfm, 2984 CRYPTO_TFM_RES_BAD_KEY_LEN); 2985 return -EINVAL; 2986 } 2987 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2988 } 2989 2990 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 2991 unsigned int authsize) 2992 { 2993 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2994 2995 switch (authsize) { 2996 case ICV_8: 2997 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2998 aeadctx->mayverify = VERIFY_HW; 2999 break; 3000 case ICV_12: 3001 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3002 aeadctx->mayverify = VERIFY_HW; 3003 break; 3004 case ICV_16: 3005 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3006 aeadctx->mayverify = VERIFY_HW; 3007 break; 3008 default: 3009 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 3010 CRYPTO_TFM_RES_BAD_KEY_LEN); 3011 return -EINVAL; 3012 } 3013 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3014 } 3015 3016 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 3017 unsigned int authsize) 3018 { 3019 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3020 3021 switch (authsize) { 3022 case ICV_4: 3023 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3024 aeadctx->mayverify = VERIFY_HW; 3025 break; 3026 case ICV_6: 3027 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3028 aeadctx->mayverify = VERIFY_HW; 3029 break; 3030 case ICV_8: 3031 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3032 aeadctx->mayverify = VERIFY_HW; 3033 break; 3034 case ICV_10: 3035 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3036 aeadctx->mayverify = VERIFY_HW; 3037 break; 3038 case ICV_12: 3039 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3040 aeadctx->mayverify = VERIFY_HW; 3041 break; 3042 case ICV_14: 3043 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3044 aeadctx->mayverify = VERIFY_HW; 3045 break; 3046 case ICV_16: 3047 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3048 aeadctx->mayverify = VERIFY_HW; 3049 break; 3050 default: 3051 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 3052 CRYPTO_TFM_RES_BAD_KEY_LEN); 3053 return -EINVAL; 3054 } 3055 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3056 } 3057 3058 static int chcr_ccm_common_setkey(struct crypto_aead *aead, 3059 const u8 *key, 3060 unsigned int keylen) 3061 { 3062 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3063 unsigned char ck_size, mk_size; 3064 int key_ctx_size = 0; 3065 3066 key_ctx_size = sizeof(struct _key_ctx) + 3067 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2; 3068 if (keylen == AES_KEYSIZE_128) { 3069 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3070 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3071 } else if (keylen == AES_KEYSIZE_192) { 3072 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3073 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 3074 } else if (keylen == AES_KEYSIZE_256) { 3075 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3076 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 3077 } else { 3078 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3079 CRYPTO_TFM_RES_BAD_KEY_LEN); 3080 aeadctx->enckey_len = 0; 3081 return -EINVAL; 3082 } 3083 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 3084 key_ctx_size >> 4); 3085 memcpy(aeadctx->key, key, keylen); 3086 aeadctx->enckey_len = keylen; 3087 3088 return 0; 3089 } 3090 3091 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 3092 const u8 *key, 3093 unsigned int keylen) 3094 { 3095 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3096 int error; 3097 3098 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3099 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3100 CRYPTO_TFM_REQ_MASK); 3101 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3102 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3103 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3104 CRYPTO_TFM_RES_MASK); 3105 if (error) 3106 return error; 3107 return chcr_ccm_common_setkey(aead, key, keylen); 3108 } 3109 3110 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 3111 unsigned int keylen) 3112 { 3113 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3114 int error; 3115 3116 if (keylen < 3) { 3117 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3118 CRYPTO_TFM_RES_BAD_KEY_LEN); 3119 aeadctx->enckey_len = 0; 3120 return -EINVAL; 3121 } 3122 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3123 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3124 CRYPTO_TFM_REQ_MASK); 3125 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3126 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3127 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3128 CRYPTO_TFM_RES_MASK); 3129 if (error) 3130 return error; 3131 keylen -= 3; 3132 memcpy(aeadctx->salt, key + keylen, 3); 3133 return chcr_ccm_common_setkey(aead, key, keylen); 3134 } 3135 3136 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 3137 unsigned int keylen) 3138 { 3139 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3140 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 3141 struct crypto_cipher *cipher; 3142 unsigned int ck_size; 3143 int ret = 0, key_ctx_size = 0; 3144 3145 aeadctx->enckey_len = 0; 3146 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3147 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) 3148 & CRYPTO_TFM_REQ_MASK); 3149 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3150 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3151 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3152 CRYPTO_TFM_RES_MASK); 3153 if (ret) 3154 goto out; 3155 3156 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 3157 keylen > 3) { 3158 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 3159 memcpy(aeadctx->salt, key + keylen, 4); 3160 } 3161 if (keylen == AES_KEYSIZE_128) { 3162 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3163 } else if (keylen == AES_KEYSIZE_192) { 3164 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3165 } else if (keylen == AES_KEYSIZE_256) { 3166 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3167 } else { 3168 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3169 CRYPTO_TFM_RES_BAD_KEY_LEN); 3170 pr_err("GCM: Invalid key length %d\n", keylen); 3171 ret = -EINVAL; 3172 goto out; 3173 } 3174 3175 memcpy(aeadctx->key, key, keylen); 3176 aeadctx->enckey_len = keylen; 3177 key_ctx_size = sizeof(struct _key_ctx) + 3178 ((DIV_ROUND_UP(keylen, 16)) << 4) + 3179 AEAD_H_SIZE; 3180 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 3181 CHCR_KEYCTX_MAC_KEY_SIZE_128, 3182 0, 0, 3183 key_ctx_size >> 4); 3184 /* Calculate the H = CIPH(K, 0 repeated 16 times). 3185 * It will go in key context 3186 */ 3187 cipher = crypto_alloc_cipher("aes-generic", 0, 0); 3188 if (IS_ERR(cipher)) { 3189 aeadctx->enckey_len = 0; 3190 ret = -ENOMEM; 3191 goto out; 3192 } 3193 3194 ret = crypto_cipher_setkey(cipher, key, keylen); 3195 if (ret) { 3196 aeadctx->enckey_len = 0; 3197 goto out1; 3198 } 3199 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 3200 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h); 3201 3202 out1: 3203 crypto_free_cipher(cipher); 3204 out: 3205 return ret; 3206 } 3207 3208 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 3209 unsigned int keylen) 3210 { 3211 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3212 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3213 /* it contains auth and cipher key both*/ 3214 struct crypto_authenc_keys keys; 3215 unsigned int bs; 3216 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 3217 int err = 0, i, key_ctx_len = 0; 3218 unsigned char ck_size = 0; 3219 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 3220 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 3221 struct algo_param param; 3222 int align; 3223 u8 *o_ptr = NULL; 3224 3225 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3226 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3227 & CRYPTO_TFM_REQ_MASK); 3228 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3229 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 3230 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 3231 & CRYPTO_TFM_RES_MASK); 3232 if (err) 3233 goto out; 3234 3235 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 3236 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 3237 goto out; 3238 } 3239 3240 if (get_alg_config(¶m, max_authsize)) { 3241 pr_err("chcr : Unsupported digest size\n"); 3242 goto out; 3243 } 3244 if (keys.enckeylen == AES_KEYSIZE_128) { 3245 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3246 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3247 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3248 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3249 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3250 } else { 3251 pr_err("chcr : Unsupported cipher key\n"); 3252 goto out; 3253 } 3254 3255 /* Copy only encryption key. We use authkey to generate h(ipad) and 3256 * h(opad) so authkey is not needed again. authkeylen size have the 3257 * size of the hash digest size. 3258 */ 3259 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3260 aeadctx->enckey_len = keys.enckeylen; 3261 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3262 aeadctx->enckey_len << 3); 3263 3264 base_hash = chcr_alloc_shash(max_authsize); 3265 if (IS_ERR(base_hash)) { 3266 pr_err("chcr : Base driver cannot be loaded\n"); 3267 aeadctx->enckey_len = 0; 3268 return -EINVAL; 3269 } 3270 { 3271 SHASH_DESC_ON_STACK(shash, base_hash); 3272 shash->tfm = base_hash; 3273 shash->flags = crypto_shash_get_flags(base_hash); 3274 bs = crypto_shash_blocksize(base_hash); 3275 align = KEYCTX_ALIGN_PAD(max_authsize); 3276 o_ptr = actx->h_iopad + param.result_size + align; 3277 3278 if (keys.authkeylen > bs) { 3279 err = crypto_shash_digest(shash, keys.authkey, 3280 keys.authkeylen, 3281 o_ptr); 3282 if (err) { 3283 pr_err("chcr : Base driver cannot be loaded\n"); 3284 goto out; 3285 } 3286 keys.authkeylen = max_authsize; 3287 } else 3288 memcpy(o_ptr, keys.authkey, keys.authkeylen); 3289 3290 /* Compute the ipad-digest*/ 3291 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3292 memcpy(pad, o_ptr, keys.authkeylen); 3293 for (i = 0; i < bs >> 2; i++) 3294 *((unsigned int *)pad + i) ^= IPAD_DATA; 3295 3296 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 3297 max_authsize)) 3298 goto out; 3299 /* Compute the opad-digest */ 3300 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3301 memcpy(pad, o_ptr, keys.authkeylen); 3302 for (i = 0; i < bs >> 2; i++) 3303 *((unsigned int *)pad + i) ^= OPAD_DATA; 3304 3305 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 3306 goto out; 3307 3308 /* convert the ipad and opad digest to network order */ 3309 chcr_change_order(actx->h_iopad, param.result_size); 3310 chcr_change_order(o_ptr, param.result_size); 3311 key_ctx_len = sizeof(struct _key_ctx) + 3312 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) + 3313 (param.result_size + align) * 2; 3314 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 3315 0, 1, key_ctx_len >> 4); 3316 actx->auth_mode = param.auth_mode; 3317 chcr_free_shash(base_hash); 3318 3319 return 0; 3320 } 3321 out: 3322 aeadctx->enckey_len = 0; 3323 if (!IS_ERR(base_hash)) 3324 chcr_free_shash(base_hash); 3325 return -EINVAL; 3326 } 3327 3328 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 3329 const u8 *key, unsigned int keylen) 3330 { 3331 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3332 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3333 struct crypto_authenc_keys keys; 3334 int err; 3335 /* it contains auth and cipher key both*/ 3336 int key_ctx_len = 0; 3337 unsigned char ck_size = 0; 3338 3339 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3340 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3341 & CRYPTO_TFM_REQ_MASK); 3342 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3343 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 3344 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 3345 & CRYPTO_TFM_RES_MASK); 3346 if (err) 3347 goto out; 3348 3349 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 3350 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 3351 goto out; 3352 } 3353 if (keys.enckeylen == AES_KEYSIZE_128) { 3354 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3355 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3356 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3357 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3358 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3359 } else { 3360 pr_err("chcr : Unsupported cipher key\n"); 3361 goto out; 3362 } 3363 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3364 aeadctx->enckey_len = keys.enckeylen; 3365 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3366 aeadctx->enckey_len << 3); 3367 key_ctx_len = sizeof(struct _key_ctx) 3368 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4); 3369 3370 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 3371 0, key_ctx_len >> 4); 3372 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 3373 return 0; 3374 out: 3375 aeadctx->enckey_len = 0; 3376 return -EINVAL; 3377 } 3378 static int chcr_aead_encrypt(struct aead_request *req) 3379 { 3380 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3381 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3382 3383 reqctx->verify = VERIFY_HW; 3384 3385 switch (get_aead_subtype(tfm)) { 3386 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 3387 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 3388 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3389 create_authenc_wr); 3390 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3391 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3392 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3393 create_aead_ccm_wr); 3394 default: 3395 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3396 create_gcm_wr); 3397 } 3398 } 3399 3400 static int chcr_aead_decrypt(struct aead_request *req) 3401 { 3402 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3403 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3404 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3405 int size; 3406 3407 if (aeadctx->mayverify == VERIFY_SW) { 3408 size = crypto_aead_maxauthsize(tfm); 3409 reqctx->verify = VERIFY_SW; 3410 } else { 3411 size = 0; 3412 reqctx->verify = VERIFY_HW; 3413 } 3414 3415 switch (get_aead_subtype(tfm)) { 3416 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 3417 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 3418 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3419 create_authenc_wr); 3420 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3421 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3422 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3423 create_aead_ccm_wr); 3424 default: 3425 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3426 create_gcm_wr); 3427 } 3428 } 3429 3430 static int chcr_aead_op(struct aead_request *req, 3431 unsigned short op_type, 3432 int size, 3433 create_wr_t create_wr_fn) 3434 { 3435 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3436 struct uld_ctx *u_ctx; 3437 struct sk_buff *skb; 3438 3439 if (!a_ctx(tfm)->dev) { 3440 pr_err("chcr : %s : No crypto device.\n", __func__); 3441 return -ENXIO; 3442 } 3443 u_ctx = ULD_CTX(a_ctx(tfm)); 3444 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 3445 a_ctx(tfm)->tx_qidx)) { 3446 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3447 return -EBUSY; 3448 } 3449 3450 /* Form a WR from req */ 3451 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, 3452 op_type); 3453 3454 if (IS_ERR(skb) || !skb) 3455 return PTR_ERR(skb); 3456 3457 skb->dev = u_ctx->lldi.ports[0]; 3458 set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); 3459 chcr_send_wr(skb); 3460 return -EINPROGRESS; 3461 } 3462 static struct chcr_alg_template driver_algs[] = { 3463 /* AES-CBC */ 3464 { 3465 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, 3466 .is_registered = 0, 3467 .alg.crypto = { 3468 .cra_name = "cbc(aes)", 3469 .cra_driver_name = "cbc-aes-chcr", 3470 .cra_blocksize = AES_BLOCK_SIZE, 3471 .cra_init = chcr_cra_init, 3472 .cra_exit = chcr_cra_exit, 3473 .cra_u.ablkcipher = { 3474 .min_keysize = AES_MIN_KEY_SIZE, 3475 .max_keysize = AES_MAX_KEY_SIZE, 3476 .ivsize = AES_BLOCK_SIZE, 3477 .setkey = chcr_aes_cbc_setkey, 3478 .encrypt = chcr_aes_encrypt, 3479 .decrypt = chcr_aes_decrypt, 3480 } 3481 } 3482 }, 3483 { 3484 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, 3485 .is_registered = 0, 3486 .alg.crypto = { 3487 .cra_name = "xts(aes)", 3488 .cra_driver_name = "xts-aes-chcr", 3489 .cra_blocksize = AES_BLOCK_SIZE, 3490 .cra_init = chcr_cra_init, 3491 .cra_exit = NULL, 3492 .cra_u .ablkcipher = { 3493 .min_keysize = 2 * AES_MIN_KEY_SIZE, 3494 .max_keysize = 2 * AES_MAX_KEY_SIZE, 3495 .ivsize = AES_BLOCK_SIZE, 3496 .setkey = chcr_aes_xts_setkey, 3497 .encrypt = chcr_aes_encrypt, 3498 .decrypt = chcr_aes_decrypt, 3499 } 3500 } 3501 }, 3502 { 3503 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, 3504 .is_registered = 0, 3505 .alg.crypto = { 3506 .cra_name = "ctr(aes)", 3507 .cra_driver_name = "ctr-aes-chcr", 3508 .cra_blocksize = 1, 3509 .cra_init = chcr_cra_init, 3510 .cra_exit = chcr_cra_exit, 3511 .cra_u.ablkcipher = { 3512 .min_keysize = AES_MIN_KEY_SIZE, 3513 .max_keysize = AES_MAX_KEY_SIZE, 3514 .ivsize = AES_BLOCK_SIZE, 3515 .setkey = chcr_aes_ctr_setkey, 3516 .encrypt = chcr_aes_encrypt, 3517 .decrypt = chcr_aes_decrypt, 3518 } 3519 } 3520 }, 3521 { 3522 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | 3523 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, 3524 .is_registered = 0, 3525 .alg.crypto = { 3526 .cra_name = "rfc3686(ctr(aes))", 3527 .cra_driver_name = "rfc3686-ctr-aes-chcr", 3528 .cra_blocksize = 1, 3529 .cra_init = chcr_rfc3686_init, 3530 .cra_exit = chcr_cra_exit, 3531 .cra_u.ablkcipher = { 3532 .min_keysize = AES_MIN_KEY_SIZE + 3533 CTR_RFC3686_NONCE_SIZE, 3534 .max_keysize = AES_MAX_KEY_SIZE + 3535 CTR_RFC3686_NONCE_SIZE, 3536 .ivsize = CTR_RFC3686_IV_SIZE, 3537 .setkey = chcr_aes_rfc3686_setkey, 3538 .encrypt = chcr_aes_encrypt, 3539 .decrypt = chcr_aes_decrypt, 3540 .geniv = "seqiv", 3541 } 3542 } 3543 }, 3544 /* SHA */ 3545 { 3546 .type = CRYPTO_ALG_TYPE_AHASH, 3547 .is_registered = 0, 3548 .alg.hash = { 3549 .halg.digestsize = SHA1_DIGEST_SIZE, 3550 .halg.base = { 3551 .cra_name = "sha1", 3552 .cra_driver_name = "sha1-chcr", 3553 .cra_blocksize = SHA1_BLOCK_SIZE, 3554 } 3555 } 3556 }, 3557 { 3558 .type = CRYPTO_ALG_TYPE_AHASH, 3559 .is_registered = 0, 3560 .alg.hash = { 3561 .halg.digestsize = SHA256_DIGEST_SIZE, 3562 .halg.base = { 3563 .cra_name = "sha256", 3564 .cra_driver_name = "sha256-chcr", 3565 .cra_blocksize = SHA256_BLOCK_SIZE, 3566 } 3567 } 3568 }, 3569 { 3570 .type = CRYPTO_ALG_TYPE_AHASH, 3571 .is_registered = 0, 3572 .alg.hash = { 3573 .halg.digestsize = SHA224_DIGEST_SIZE, 3574 .halg.base = { 3575 .cra_name = "sha224", 3576 .cra_driver_name = "sha224-chcr", 3577 .cra_blocksize = SHA224_BLOCK_SIZE, 3578 } 3579 } 3580 }, 3581 { 3582 .type = CRYPTO_ALG_TYPE_AHASH, 3583 .is_registered = 0, 3584 .alg.hash = { 3585 .halg.digestsize = SHA384_DIGEST_SIZE, 3586 .halg.base = { 3587 .cra_name = "sha384", 3588 .cra_driver_name = "sha384-chcr", 3589 .cra_blocksize = SHA384_BLOCK_SIZE, 3590 } 3591 } 3592 }, 3593 { 3594 .type = CRYPTO_ALG_TYPE_AHASH, 3595 .is_registered = 0, 3596 .alg.hash = { 3597 .halg.digestsize = SHA512_DIGEST_SIZE, 3598 .halg.base = { 3599 .cra_name = "sha512", 3600 .cra_driver_name = "sha512-chcr", 3601 .cra_blocksize = SHA512_BLOCK_SIZE, 3602 } 3603 } 3604 }, 3605 /* HMAC */ 3606 { 3607 .type = CRYPTO_ALG_TYPE_HMAC, 3608 .is_registered = 0, 3609 .alg.hash = { 3610 .halg.digestsize = SHA1_DIGEST_SIZE, 3611 .halg.base = { 3612 .cra_name = "hmac(sha1)", 3613 .cra_driver_name = "hmac-sha1-chcr", 3614 .cra_blocksize = SHA1_BLOCK_SIZE, 3615 } 3616 } 3617 }, 3618 { 3619 .type = CRYPTO_ALG_TYPE_HMAC, 3620 .is_registered = 0, 3621 .alg.hash = { 3622 .halg.digestsize = SHA224_DIGEST_SIZE, 3623 .halg.base = { 3624 .cra_name = "hmac(sha224)", 3625 .cra_driver_name = "hmac-sha224-chcr", 3626 .cra_blocksize = SHA224_BLOCK_SIZE, 3627 } 3628 } 3629 }, 3630 { 3631 .type = CRYPTO_ALG_TYPE_HMAC, 3632 .is_registered = 0, 3633 .alg.hash = { 3634 .halg.digestsize = SHA256_DIGEST_SIZE, 3635 .halg.base = { 3636 .cra_name = "hmac(sha256)", 3637 .cra_driver_name = "hmac-sha256-chcr", 3638 .cra_blocksize = SHA256_BLOCK_SIZE, 3639 } 3640 } 3641 }, 3642 { 3643 .type = CRYPTO_ALG_TYPE_HMAC, 3644 .is_registered = 0, 3645 .alg.hash = { 3646 .halg.digestsize = SHA384_DIGEST_SIZE, 3647 .halg.base = { 3648 .cra_name = "hmac(sha384)", 3649 .cra_driver_name = "hmac-sha384-chcr", 3650 .cra_blocksize = SHA384_BLOCK_SIZE, 3651 } 3652 } 3653 }, 3654 { 3655 .type = CRYPTO_ALG_TYPE_HMAC, 3656 .is_registered = 0, 3657 .alg.hash = { 3658 .halg.digestsize = SHA512_DIGEST_SIZE, 3659 .halg.base = { 3660 .cra_name = "hmac(sha512)", 3661 .cra_driver_name = "hmac-sha512-chcr", 3662 .cra_blocksize = SHA512_BLOCK_SIZE, 3663 } 3664 } 3665 }, 3666 /* Add AEAD Algorithms */ 3667 { 3668 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 3669 .is_registered = 0, 3670 .alg.aead = { 3671 .base = { 3672 .cra_name = "gcm(aes)", 3673 .cra_driver_name = "gcm-aes-chcr", 3674 .cra_blocksize = 1, 3675 .cra_priority = CHCR_AEAD_PRIORITY, 3676 .cra_ctxsize = sizeof(struct chcr_context) + 3677 sizeof(struct chcr_aead_ctx) + 3678 sizeof(struct chcr_gcm_ctx), 3679 }, 3680 .ivsize = GCM_AES_IV_SIZE, 3681 .maxauthsize = GHASH_DIGEST_SIZE, 3682 .setkey = chcr_gcm_setkey, 3683 .setauthsize = chcr_gcm_setauthsize, 3684 } 3685 }, 3686 { 3687 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 3688 .is_registered = 0, 3689 .alg.aead = { 3690 .base = { 3691 .cra_name = "rfc4106(gcm(aes))", 3692 .cra_driver_name = "rfc4106-gcm-aes-chcr", 3693 .cra_blocksize = 1, 3694 .cra_priority = CHCR_AEAD_PRIORITY + 1, 3695 .cra_ctxsize = sizeof(struct chcr_context) + 3696 sizeof(struct chcr_aead_ctx) + 3697 sizeof(struct chcr_gcm_ctx), 3698 3699 }, 3700 .ivsize = GCM_RFC4106_IV_SIZE, 3701 .maxauthsize = GHASH_DIGEST_SIZE, 3702 .setkey = chcr_gcm_setkey, 3703 .setauthsize = chcr_4106_4309_setauthsize, 3704 } 3705 }, 3706 { 3707 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 3708 .is_registered = 0, 3709 .alg.aead = { 3710 .base = { 3711 .cra_name = "ccm(aes)", 3712 .cra_driver_name = "ccm-aes-chcr", 3713 .cra_blocksize = 1, 3714 .cra_priority = CHCR_AEAD_PRIORITY, 3715 .cra_ctxsize = sizeof(struct chcr_context) + 3716 sizeof(struct chcr_aead_ctx), 3717 3718 }, 3719 .ivsize = AES_BLOCK_SIZE, 3720 .maxauthsize = GHASH_DIGEST_SIZE, 3721 .setkey = chcr_aead_ccm_setkey, 3722 .setauthsize = chcr_ccm_setauthsize, 3723 } 3724 }, 3725 { 3726 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 3727 .is_registered = 0, 3728 .alg.aead = { 3729 .base = { 3730 .cra_name = "rfc4309(ccm(aes))", 3731 .cra_driver_name = "rfc4309-ccm-aes-chcr", 3732 .cra_blocksize = 1, 3733 .cra_priority = CHCR_AEAD_PRIORITY + 1, 3734 .cra_ctxsize = sizeof(struct chcr_context) + 3735 sizeof(struct chcr_aead_ctx), 3736 3737 }, 3738 .ivsize = 8, 3739 .maxauthsize = GHASH_DIGEST_SIZE, 3740 .setkey = chcr_aead_rfc4309_setkey, 3741 .setauthsize = chcr_4106_4309_setauthsize, 3742 } 3743 }, 3744 { 3745 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3746 .is_registered = 0, 3747 .alg.aead = { 3748 .base = { 3749 .cra_name = "authenc(hmac(sha1),cbc(aes))", 3750 .cra_driver_name = 3751 "authenc-hmac-sha1-cbc-aes-chcr", 3752 .cra_blocksize = AES_BLOCK_SIZE, 3753 .cra_priority = CHCR_AEAD_PRIORITY, 3754 .cra_ctxsize = sizeof(struct chcr_context) + 3755 sizeof(struct chcr_aead_ctx) + 3756 sizeof(struct chcr_authenc_ctx), 3757 3758 }, 3759 .ivsize = AES_BLOCK_SIZE, 3760 .maxauthsize = SHA1_DIGEST_SIZE, 3761 .setkey = chcr_authenc_setkey, 3762 .setauthsize = chcr_authenc_setauthsize, 3763 } 3764 }, 3765 { 3766 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3767 .is_registered = 0, 3768 .alg.aead = { 3769 .base = { 3770 3771 .cra_name = "authenc(hmac(sha256),cbc(aes))", 3772 .cra_driver_name = 3773 "authenc-hmac-sha256-cbc-aes-chcr", 3774 .cra_blocksize = AES_BLOCK_SIZE, 3775 .cra_priority = CHCR_AEAD_PRIORITY, 3776 .cra_ctxsize = sizeof(struct chcr_context) + 3777 sizeof(struct chcr_aead_ctx) + 3778 sizeof(struct chcr_authenc_ctx), 3779 3780 }, 3781 .ivsize = AES_BLOCK_SIZE, 3782 .maxauthsize = SHA256_DIGEST_SIZE, 3783 .setkey = chcr_authenc_setkey, 3784 .setauthsize = chcr_authenc_setauthsize, 3785 } 3786 }, 3787 { 3788 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3789 .is_registered = 0, 3790 .alg.aead = { 3791 .base = { 3792 .cra_name = "authenc(hmac(sha224),cbc(aes))", 3793 .cra_driver_name = 3794 "authenc-hmac-sha224-cbc-aes-chcr", 3795 .cra_blocksize = AES_BLOCK_SIZE, 3796 .cra_priority = CHCR_AEAD_PRIORITY, 3797 .cra_ctxsize = sizeof(struct chcr_context) + 3798 sizeof(struct chcr_aead_ctx) + 3799 sizeof(struct chcr_authenc_ctx), 3800 }, 3801 .ivsize = AES_BLOCK_SIZE, 3802 .maxauthsize = SHA224_DIGEST_SIZE, 3803 .setkey = chcr_authenc_setkey, 3804 .setauthsize = chcr_authenc_setauthsize, 3805 } 3806 }, 3807 { 3808 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3809 .is_registered = 0, 3810 .alg.aead = { 3811 .base = { 3812 .cra_name = "authenc(hmac(sha384),cbc(aes))", 3813 .cra_driver_name = 3814 "authenc-hmac-sha384-cbc-aes-chcr", 3815 .cra_blocksize = AES_BLOCK_SIZE, 3816 .cra_priority = CHCR_AEAD_PRIORITY, 3817 .cra_ctxsize = sizeof(struct chcr_context) + 3818 sizeof(struct chcr_aead_ctx) + 3819 sizeof(struct chcr_authenc_ctx), 3820 3821 }, 3822 .ivsize = AES_BLOCK_SIZE, 3823 .maxauthsize = SHA384_DIGEST_SIZE, 3824 .setkey = chcr_authenc_setkey, 3825 .setauthsize = chcr_authenc_setauthsize, 3826 } 3827 }, 3828 { 3829 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3830 .is_registered = 0, 3831 .alg.aead = { 3832 .base = { 3833 .cra_name = "authenc(hmac(sha512),cbc(aes))", 3834 .cra_driver_name = 3835 "authenc-hmac-sha512-cbc-aes-chcr", 3836 .cra_blocksize = AES_BLOCK_SIZE, 3837 .cra_priority = CHCR_AEAD_PRIORITY, 3838 .cra_ctxsize = sizeof(struct chcr_context) + 3839 sizeof(struct chcr_aead_ctx) + 3840 sizeof(struct chcr_authenc_ctx), 3841 3842 }, 3843 .ivsize = AES_BLOCK_SIZE, 3844 .maxauthsize = SHA512_DIGEST_SIZE, 3845 .setkey = chcr_authenc_setkey, 3846 .setauthsize = chcr_authenc_setauthsize, 3847 } 3848 }, 3849 { 3850 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL, 3851 .is_registered = 0, 3852 .alg.aead = { 3853 .base = { 3854 .cra_name = "authenc(digest_null,cbc(aes))", 3855 .cra_driver_name = 3856 "authenc-digest_null-cbc-aes-chcr", 3857 .cra_blocksize = AES_BLOCK_SIZE, 3858 .cra_priority = CHCR_AEAD_PRIORITY, 3859 .cra_ctxsize = sizeof(struct chcr_context) + 3860 sizeof(struct chcr_aead_ctx) + 3861 sizeof(struct chcr_authenc_ctx), 3862 3863 }, 3864 .ivsize = AES_BLOCK_SIZE, 3865 .maxauthsize = 0, 3866 .setkey = chcr_aead_digest_null_setkey, 3867 .setauthsize = chcr_authenc_null_setauthsize, 3868 } 3869 }, 3870 }; 3871 3872 /* 3873 * chcr_unregister_alg - Deregister crypto algorithms with 3874 * kernel framework. 3875 */ 3876 static int chcr_unregister_alg(void) 3877 { 3878 int i; 3879 3880 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3881 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 3882 case CRYPTO_ALG_TYPE_ABLKCIPHER: 3883 if (driver_algs[i].is_registered) 3884 crypto_unregister_alg( 3885 &driver_algs[i].alg.crypto); 3886 break; 3887 case CRYPTO_ALG_TYPE_AEAD: 3888 if (driver_algs[i].is_registered) 3889 crypto_unregister_aead( 3890 &driver_algs[i].alg.aead); 3891 break; 3892 case CRYPTO_ALG_TYPE_AHASH: 3893 if (driver_algs[i].is_registered) 3894 crypto_unregister_ahash( 3895 &driver_algs[i].alg.hash); 3896 break; 3897 } 3898 driver_algs[i].is_registered = 0; 3899 } 3900 return 0; 3901 } 3902 3903 #define SZ_AHASH_CTX sizeof(struct chcr_context) 3904 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 3905 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 3906 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC) 3907 3908 /* 3909 * chcr_register_alg - Register crypto algorithms with kernel framework. 3910 */ 3911 static int chcr_register_alg(void) 3912 { 3913 struct crypto_alg ai; 3914 struct ahash_alg *a_hash; 3915 int err = 0, i; 3916 char *name = NULL; 3917 3918 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3919 if (driver_algs[i].is_registered) 3920 continue; 3921 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 3922 case CRYPTO_ALG_TYPE_ABLKCIPHER: 3923 driver_algs[i].alg.crypto.cra_priority = 3924 CHCR_CRA_PRIORITY; 3925 driver_algs[i].alg.crypto.cra_module = THIS_MODULE; 3926 driver_algs[i].alg.crypto.cra_flags = 3927 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | 3928 CRYPTO_ALG_NEED_FALLBACK; 3929 driver_algs[i].alg.crypto.cra_ctxsize = 3930 sizeof(struct chcr_context) + 3931 sizeof(struct ablk_ctx); 3932 driver_algs[i].alg.crypto.cra_alignmask = 0; 3933 driver_algs[i].alg.crypto.cra_type = 3934 &crypto_ablkcipher_type; 3935 err = crypto_register_alg(&driver_algs[i].alg.crypto); 3936 name = driver_algs[i].alg.crypto.cra_driver_name; 3937 break; 3938 case CRYPTO_ALG_TYPE_AEAD: 3939 driver_algs[i].alg.aead.base.cra_flags = 3940 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC | 3941 CRYPTO_ALG_NEED_FALLBACK; 3942 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 3943 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 3944 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 3945 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 3946 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 3947 err = crypto_register_aead(&driver_algs[i].alg.aead); 3948 name = driver_algs[i].alg.aead.base.cra_driver_name; 3949 break; 3950 case CRYPTO_ALG_TYPE_AHASH: 3951 a_hash = &driver_algs[i].alg.hash; 3952 a_hash->update = chcr_ahash_update; 3953 a_hash->final = chcr_ahash_final; 3954 a_hash->finup = chcr_ahash_finup; 3955 a_hash->digest = chcr_ahash_digest; 3956 a_hash->export = chcr_ahash_export; 3957 a_hash->import = chcr_ahash_import; 3958 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 3959 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 3960 a_hash->halg.base.cra_module = THIS_MODULE; 3961 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS; 3962 a_hash->halg.base.cra_alignmask = 0; 3963 a_hash->halg.base.cra_exit = NULL; 3964 a_hash->halg.base.cra_type = &crypto_ahash_type; 3965 3966 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 3967 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 3968 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 3969 a_hash->init = chcr_hmac_init; 3970 a_hash->setkey = chcr_ahash_setkey; 3971 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 3972 } else { 3973 a_hash->init = chcr_sha_init; 3974 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 3975 a_hash->halg.base.cra_init = chcr_sha_cra_init; 3976 } 3977 err = crypto_register_ahash(&driver_algs[i].alg.hash); 3978 ai = driver_algs[i].alg.hash.halg.base; 3979 name = ai.cra_driver_name; 3980 break; 3981 } 3982 if (err) { 3983 pr_err("chcr : %s : Algorithm registration failed\n", 3984 name); 3985 goto register_err; 3986 } else { 3987 driver_algs[i].is_registered = 1; 3988 } 3989 } 3990 return 0; 3991 3992 register_err: 3993 chcr_unregister_alg(); 3994 return err; 3995 } 3996 3997 /* 3998 * start_crypto - Register the crypto algorithms. 3999 * This should called once when the first device comesup. After this 4000 * kernel will start calling driver APIs for crypto operations. 4001 */ 4002 int start_crypto(void) 4003 { 4004 return chcr_register_alg(); 4005 } 4006 4007 /* 4008 * stop_crypto - Deregister all the crypto algorithms with kernel. 4009 * This should be called once when the last device goes down. After this 4010 * kernel will not call the driver API for crypto operations. 4011 */ 4012 int stop_crypto(void) 4013 { 4014 chcr_unregister_alg(); 4015 return 0; 4016 } 4017