1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/cryptohash.h> 48 #include <linux/skbuff.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/highmem.h> 51 #include <linux/scatterlist.h> 52 53 #include <crypto/aes.h> 54 #include <crypto/algapi.h> 55 #include <crypto/hash.h> 56 #include <crypto/gcm.h> 57 #include <crypto/sha.h> 58 #include <crypto/authenc.h> 59 #include <crypto/ctr.h> 60 #include <crypto/gf128mul.h> 61 #include <crypto/internal/aead.h> 62 #include <crypto/null.h> 63 #include <crypto/internal/skcipher.h> 64 #include <crypto/aead.h> 65 #include <crypto/scatterwalk.h> 66 #include <crypto/internal/hash.h> 67 68 #include "t4fw_api.h" 69 #include "t4_msg.h" 70 #include "chcr_core.h" 71 #include "chcr_algo.h" 72 #include "chcr_crypto.h" 73 74 #define IV AES_BLOCK_SIZE 75 76 static unsigned int sgl_ent_len[] = { 77 0, 0, 16, 24, 40, 48, 64, 72, 88, 78 96, 112, 120, 136, 144, 160, 168, 184, 79 192, 208, 216, 232, 240, 256, 264, 280, 80 288, 304, 312, 328, 336, 352, 360, 376 81 }; 82 83 static unsigned int dsgl_ent_len[] = { 84 0, 32, 32, 48, 48, 64, 64, 80, 80, 85 112, 112, 128, 128, 144, 144, 160, 160, 86 192, 192, 208, 208, 224, 224, 240, 240, 87 272, 272, 288, 288, 304, 304, 320, 320 88 }; 89 90 static u32 round_constant[11] = { 91 0x01000000, 0x02000000, 0x04000000, 0x08000000, 92 0x10000000, 0x20000000, 0x40000000, 0x80000000, 93 0x1B000000, 0x36000000, 0x6C000000 94 }; 95 96 static int chcr_handle_cipher_resp(struct ablkcipher_request *req, 97 unsigned char *input, int err); 98 99 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 100 { 101 return ctx->crypto_ctx->aeadctx; 102 } 103 104 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 105 { 106 return ctx->crypto_ctx->ablkctx; 107 } 108 109 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 110 { 111 return ctx->crypto_ctx->hmacctx; 112 } 113 114 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 115 { 116 return gctx->ctx->gcm; 117 } 118 119 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 120 { 121 return gctx->ctx->authenc; 122 } 123 124 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 125 { 126 return container_of(ctx->dev, struct uld_ctx, dev); 127 } 128 129 static inline int is_ofld_imm(const struct sk_buff *skb) 130 { 131 return (skb->len <= SGE_MAX_WR_LEN); 132 } 133 134 static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx) 135 { 136 memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr)); 137 } 138 139 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, 140 unsigned int entlen, 141 unsigned int skip) 142 { 143 int nents = 0; 144 unsigned int less; 145 unsigned int skip_len = 0; 146 147 while (sg && skip) { 148 if (sg_dma_len(sg) <= skip) { 149 skip -= sg_dma_len(sg); 150 skip_len = 0; 151 sg = sg_next(sg); 152 } else { 153 skip_len = skip; 154 skip = 0; 155 } 156 } 157 158 while (sg && reqlen) { 159 less = min(reqlen, sg_dma_len(sg) - skip_len); 160 nents += DIV_ROUND_UP(less, entlen); 161 reqlen -= less; 162 skip_len = 0; 163 sg = sg_next(sg); 164 } 165 return nents; 166 } 167 168 static inline int get_aead_subtype(struct crypto_aead *aead) 169 { 170 struct aead_alg *alg = crypto_aead_alg(aead); 171 struct chcr_alg_template *chcr_crypto_alg = 172 container_of(alg, struct chcr_alg_template, alg.aead); 173 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 174 } 175 176 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 177 { 178 u8 temp[SHA512_DIGEST_SIZE]; 179 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 180 int authsize = crypto_aead_authsize(tfm); 181 struct cpl_fw6_pld *fw6_pld; 182 int cmp = 0; 183 184 fw6_pld = (struct cpl_fw6_pld *)input; 185 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 186 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 187 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); 188 } else { 189 190 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 191 authsize, req->assoclen + 192 req->cryptlen - authsize); 193 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); 194 } 195 if (cmp) 196 *err = -EBADMSG; 197 else 198 *err = 0; 199 } 200 201 static int chcr_inc_wrcount(struct chcr_dev *dev) 202 { 203 if (dev->state == CHCR_DETACH) 204 return 1; 205 atomic_inc(&dev->inflight); 206 return 0; 207 } 208 209 static inline void chcr_dec_wrcount(struct chcr_dev *dev) 210 { 211 atomic_dec(&dev->inflight); 212 } 213 214 static inline int chcr_handle_aead_resp(struct aead_request *req, 215 unsigned char *input, 216 int err) 217 { 218 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 219 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 220 struct chcr_dev *dev = a_ctx(tfm)->dev; 221 222 chcr_aead_common_exit(req); 223 if (reqctx->verify == VERIFY_SW) { 224 chcr_verify_tag(req, input, &err); 225 reqctx->verify = VERIFY_HW; 226 } 227 chcr_dec_wrcount(dev); 228 req->base.complete(&req->base, err); 229 230 return err; 231 } 232 233 static void get_aes_decrypt_key(unsigned char *dec_key, 234 const unsigned char *key, 235 unsigned int keylength) 236 { 237 u32 temp; 238 u32 w_ring[MAX_NK]; 239 int i, j, k; 240 u8 nr, nk; 241 242 switch (keylength) { 243 case AES_KEYLENGTH_128BIT: 244 nk = KEYLENGTH_4BYTES; 245 nr = NUMBER_OF_ROUNDS_10; 246 break; 247 case AES_KEYLENGTH_192BIT: 248 nk = KEYLENGTH_6BYTES; 249 nr = NUMBER_OF_ROUNDS_12; 250 break; 251 case AES_KEYLENGTH_256BIT: 252 nk = KEYLENGTH_8BYTES; 253 nr = NUMBER_OF_ROUNDS_14; 254 break; 255 default: 256 return; 257 } 258 for (i = 0; i < nk; i++) 259 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]); 260 261 i = 0; 262 temp = w_ring[nk - 1]; 263 while (i + nk < (nr + 1) * 4) { 264 if (!(i % nk)) { 265 /* RotWord(temp) */ 266 temp = (temp << 8) | (temp >> 24); 267 temp = aes_ks_subword(temp); 268 temp ^= round_constant[i / nk]; 269 } else if (nk == 8 && (i % 4 == 0)) { 270 temp = aes_ks_subword(temp); 271 } 272 w_ring[i % nk] ^= temp; 273 temp = w_ring[i % nk]; 274 i++; 275 } 276 i--; 277 for (k = 0, j = i % nk; k < nk; k++) { 278 *((u32 *)dec_key + k) = htonl(w_ring[j]); 279 j--; 280 if (j < 0) 281 j += nk; 282 } 283 } 284 285 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 286 { 287 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 288 289 switch (ds) { 290 case SHA1_DIGEST_SIZE: 291 base_hash = crypto_alloc_shash("sha1", 0, 0); 292 break; 293 case SHA224_DIGEST_SIZE: 294 base_hash = crypto_alloc_shash("sha224", 0, 0); 295 break; 296 case SHA256_DIGEST_SIZE: 297 base_hash = crypto_alloc_shash("sha256", 0, 0); 298 break; 299 case SHA384_DIGEST_SIZE: 300 base_hash = crypto_alloc_shash("sha384", 0, 0); 301 break; 302 case SHA512_DIGEST_SIZE: 303 base_hash = crypto_alloc_shash("sha512", 0, 0); 304 break; 305 } 306 307 return base_hash; 308 } 309 310 static int chcr_compute_partial_hash(struct shash_desc *desc, 311 char *iopad, char *result_hash, 312 int digest_size) 313 { 314 struct sha1_state sha1_st; 315 struct sha256_state sha256_st; 316 struct sha512_state sha512_st; 317 int error; 318 319 if (digest_size == SHA1_DIGEST_SIZE) { 320 error = crypto_shash_init(desc) ?: 321 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 322 crypto_shash_export(desc, (void *)&sha1_st); 323 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 324 } else if (digest_size == SHA224_DIGEST_SIZE) { 325 error = crypto_shash_init(desc) ?: 326 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 327 crypto_shash_export(desc, (void *)&sha256_st); 328 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 329 330 } else if (digest_size == SHA256_DIGEST_SIZE) { 331 error = crypto_shash_init(desc) ?: 332 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 333 crypto_shash_export(desc, (void *)&sha256_st); 334 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 335 336 } else if (digest_size == SHA384_DIGEST_SIZE) { 337 error = crypto_shash_init(desc) ?: 338 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 339 crypto_shash_export(desc, (void *)&sha512_st); 340 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 341 342 } else if (digest_size == SHA512_DIGEST_SIZE) { 343 error = crypto_shash_init(desc) ?: 344 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 345 crypto_shash_export(desc, (void *)&sha512_st); 346 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 347 } else { 348 error = -EINVAL; 349 pr_err("Unknown digest size %d\n", digest_size); 350 } 351 return error; 352 } 353 354 static void chcr_change_order(char *buf, int ds) 355 { 356 int i; 357 358 if (ds == SHA512_DIGEST_SIZE) { 359 for (i = 0; i < (ds / sizeof(u64)); i++) 360 *((__be64 *)buf + i) = 361 cpu_to_be64(*((u64 *)buf + i)); 362 } else { 363 for (i = 0; i < (ds / sizeof(u32)); i++) 364 *((__be32 *)buf + i) = 365 cpu_to_be32(*((u32 *)buf + i)); 366 } 367 } 368 369 static inline int is_hmac(struct crypto_tfm *tfm) 370 { 371 struct crypto_alg *alg = tfm->__crt_alg; 372 struct chcr_alg_template *chcr_crypto_alg = 373 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 374 alg.hash); 375 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 376 return 1; 377 return 0; 378 } 379 380 static inline void dsgl_walk_init(struct dsgl_walk *walk, 381 struct cpl_rx_phys_dsgl *dsgl) 382 { 383 walk->dsgl = dsgl; 384 walk->nents = 0; 385 walk->to = (struct phys_sge_pairs *)(dsgl + 1); 386 } 387 388 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid, 389 int pci_chan_id) 390 { 391 struct cpl_rx_phys_dsgl *phys_cpl; 392 393 phys_cpl = walk->dsgl; 394 395 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 396 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 397 phys_cpl->pcirlxorder_to_noofsgentr = 398 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 399 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 400 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 401 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 402 CPL_RX_PHYS_DSGL_DCAID_V(0) | 403 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); 404 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 405 phys_cpl->rss_hdr_int.qid = htons(qid); 406 phys_cpl->rss_hdr_int.hash_val = 0; 407 phys_cpl->rss_hdr_int.channel = pci_chan_id; 408 } 409 410 static inline void dsgl_walk_add_page(struct dsgl_walk *walk, 411 size_t size, 412 dma_addr_t addr) 413 { 414 int j; 415 416 if (!size) 417 return; 418 j = walk->nents; 419 walk->to->len[j % 8] = htons(size); 420 walk->to->addr[j % 8] = cpu_to_be64(addr); 421 j++; 422 if ((j % 8) == 0) 423 walk->to++; 424 walk->nents = j; 425 } 426 427 static void dsgl_walk_add_sg(struct dsgl_walk *walk, 428 struct scatterlist *sg, 429 unsigned int slen, 430 unsigned int skip) 431 { 432 int skip_len = 0; 433 unsigned int left_size = slen, len = 0; 434 unsigned int j = walk->nents; 435 int offset, ent_len; 436 437 if (!slen) 438 return; 439 while (sg && skip) { 440 if (sg_dma_len(sg) <= skip) { 441 skip -= sg_dma_len(sg); 442 skip_len = 0; 443 sg = sg_next(sg); 444 } else { 445 skip_len = skip; 446 skip = 0; 447 } 448 } 449 450 while (left_size && sg) { 451 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); 452 offset = 0; 453 while (len) { 454 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); 455 walk->to->len[j % 8] = htons(ent_len); 456 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + 457 offset + skip_len); 458 offset += ent_len; 459 len -= ent_len; 460 j++; 461 if ((j % 8) == 0) 462 walk->to++; 463 } 464 walk->last_sg = sg; 465 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - 466 skip_len) + skip_len; 467 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); 468 skip_len = 0; 469 sg = sg_next(sg); 470 } 471 walk->nents = j; 472 } 473 474 static inline void ulptx_walk_init(struct ulptx_walk *walk, 475 struct ulptx_sgl *ulp) 476 { 477 walk->sgl = ulp; 478 walk->nents = 0; 479 walk->pair_idx = 0; 480 walk->pair = ulp->sge; 481 walk->last_sg = NULL; 482 walk->last_sg_len = 0; 483 } 484 485 static inline void ulptx_walk_end(struct ulptx_walk *walk) 486 { 487 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | 488 ULPTX_NSGE_V(walk->nents)); 489 } 490 491 492 static inline void ulptx_walk_add_page(struct ulptx_walk *walk, 493 size_t size, 494 dma_addr_t addr) 495 { 496 if (!size) 497 return; 498 499 if (walk->nents == 0) { 500 walk->sgl->len0 = cpu_to_be32(size); 501 walk->sgl->addr0 = cpu_to_be64(addr); 502 } else { 503 walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr); 504 walk->pair->len[walk->pair_idx] = cpu_to_be32(size); 505 walk->pair_idx = !walk->pair_idx; 506 if (!walk->pair_idx) 507 walk->pair++; 508 } 509 walk->nents++; 510 } 511 512 static void ulptx_walk_add_sg(struct ulptx_walk *walk, 513 struct scatterlist *sg, 514 unsigned int len, 515 unsigned int skip) 516 { 517 int small; 518 int skip_len = 0; 519 unsigned int sgmin; 520 521 if (!len) 522 return; 523 while (sg && skip) { 524 if (sg_dma_len(sg) <= skip) { 525 skip -= sg_dma_len(sg); 526 skip_len = 0; 527 sg = sg_next(sg); 528 } else { 529 skip_len = skip; 530 skip = 0; 531 } 532 } 533 WARN(!sg, "SG should not be null here\n"); 534 if (sg && (walk->nents == 0)) { 535 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); 536 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 537 walk->sgl->len0 = cpu_to_be32(sgmin); 538 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); 539 walk->nents++; 540 len -= sgmin; 541 walk->last_sg = sg; 542 walk->last_sg_len = sgmin + skip_len; 543 skip_len += sgmin; 544 if (sg_dma_len(sg) == skip_len) { 545 sg = sg_next(sg); 546 skip_len = 0; 547 } 548 } 549 550 while (sg && len) { 551 small = min(sg_dma_len(sg) - skip_len, len); 552 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 553 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); 554 walk->pair->addr[walk->pair_idx] = 555 cpu_to_be64(sg_dma_address(sg) + skip_len); 556 walk->pair_idx = !walk->pair_idx; 557 walk->nents++; 558 if (!walk->pair_idx) 559 walk->pair++; 560 len -= sgmin; 561 skip_len += sgmin; 562 walk->last_sg = sg; 563 walk->last_sg_len = skip_len; 564 if (sg_dma_len(sg) == skip_len) { 565 sg = sg_next(sg); 566 skip_len = 0; 567 } 568 } 569 } 570 571 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) 572 { 573 struct crypto_alg *alg = tfm->__crt_alg; 574 struct chcr_alg_template *chcr_crypto_alg = 575 container_of(alg, struct chcr_alg_template, alg.crypto); 576 577 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 578 } 579 580 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 581 { 582 struct adapter *adap = netdev2adap(dev); 583 struct sge_uld_txq_info *txq_info = 584 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 585 struct sge_uld_txq *txq; 586 int ret = 0; 587 588 local_bh_disable(); 589 txq = &txq_info->uldtxq[idx]; 590 spin_lock(&txq->sendq.lock); 591 if (txq->full) 592 ret = -1; 593 spin_unlock(&txq->sendq.lock); 594 local_bh_enable(); 595 return ret; 596 } 597 598 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 599 struct _key_ctx *key_ctx) 600 { 601 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 602 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 603 } else { 604 memcpy(key_ctx->key, 605 ablkctx->key + (ablkctx->enckey_len >> 1), 606 ablkctx->enckey_len >> 1); 607 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 608 ablkctx->rrkey, ablkctx->enckey_len >> 1); 609 } 610 return 0; 611 } 612 613 static int chcr_hash_ent_in_wr(struct scatterlist *src, 614 unsigned int minsg, 615 unsigned int space, 616 unsigned int srcskip) 617 { 618 int srclen = 0; 619 int srcsg = minsg; 620 int soffset = 0, sless; 621 622 if (sg_dma_len(src) == srcskip) { 623 src = sg_next(src); 624 srcskip = 0; 625 } 626 while (src && space > (sgl_ent_len[srcsg + 1])) { 627 sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip, 628 CHCR_SRC_SG_SIZE); 629 srclen += sless; 630 soffset += sless; 631 srcsg++; 632 if (sg_dma_len(src) == (soffset + srcskip)) { 633 src = sg_next(src); 634 soffset = 0; 635 srcskip = 0; 636 } 637 } 638 return srclen; 639 } 640 641 static int chcr_sg_ent_in_wr(struct scatterlist *src, 642 struct scatterlist *dst, 643 unsigned int minsg, 644 unsigned int space, 645 unsigned int srcskip, 646 unsigned int dstskip) 647 { 648 int srclen = 0, dstlen = 0; 649 int srcsg = minsg, dstsg = minsg; 650 int offset = 0, soffset = 0, less, sless = 0; 651 652 if (sg_dma_len(src) == srcskip) { 653 src = sg_next(src); 654 srcskip = 0; 655 } 656 if (sg_dma_len(dst) == dstskip) { 657 dst = sg_next(dst); 658 dstskip = 0; 659 } 660 661 while (src && dst && 662 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { 663 sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset, 664 CHCR_SRC_SG_SIZE); 665 srclen += sless; 666 srcsg++; 667 offset = 0; 668 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && 669 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { 670 if (srclen <= dstlen) 671 break; 672 less = min_t(unsigned int, sg_dma_len(dst) - offset - 673 dstskip, CHCR_DST_SG_SIZE); 674 dstlen += less; 675 offset += less; 676 if ((offset + dstskip) == sg_dma_len(dst)) { 677 dst = sg_next(dst); 678 offset = 0; 679 } 680 dstsg++; 681 dstskip = 0; 682 } 683 soffset += sless; 684 if ((soffset + srcskip) == sg_dma_len(src)) { 685 src = sg_next(src); 686 srcskip = 0; 687 soffset = 0; 688 } 689 690 } 691 return min(srclen, dstlen); 692 } 693 694 static int chcr_cipher_fallback(struct crypto_sync_skcipher *cipher, 695 u32 flags, 696 struct scatterlist *src, 697 struct scatterlist *dst, 698 unsigned int nbytes, 699 u8 *iv, 700 unsigned short op_type) 701 { 702 int err; 703 704 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, cipher); 705 706 skcipher_request_set_sync_tfm(subreq, cipher); 707 skcipher_request_set_callback(subreq, flags, NULL, NULL); 708 skcipher_request_set_crypt(subreq, src, dst, 709 nbytes, iv); 710 711 err = op_type ? crypto_skcipher_decrypt(subreq) : 712 crypto_skcipher_encrypt(subreq); 713 skcipher_request_zero(subreq); 714 715 return err; 716 717 } 718 static inline void create_wreq(struct chcr_context *ctx, 719 struct chcr_wr *chcr_req, 720 struct crypto_async_request *req, 721 unsigned int imm, 722 int hash_sz, 723 unsigned int len16, 724 unsigned int sc_len, 725 unsigned int lcb) 726 { 727 struct uld_ctx *u_ctx = ULD_CTX(ctx); 728 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; 729 730 731 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; 732 chcr_req->wreq.pld_size_hash_size = 733 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 734 chcr_req->wreq.len16_pkd = 735 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); 736 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 737 chcr_req->wreq.rx_chid_to_rx_q_id = 738 FILL_WR_RX_Q_ID(ctx->tx_chan_id, qid, 739 !!lcb, ctx->tx_qidx); 740 741 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->tx_chan_id, 742 qid); 743 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - 744 ((sizeof(chcr_req->wreq)) >> 4))); 745 746 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); 747 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 748 sizeof(chcr_req->key_ctx) + sc_len); 749 } 750 751 /** 752 * create_cipher_wr - form the WR for cipher operations 753 * @req: cipher req. 754 * @ctx: crypto driver context of the request. 755 * @qid: ingress qid where response of this WR should be received. 756 * @op_type: encryption or decryption 757 */ 758 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) 759 { 760 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); 761 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 762 struct sk_buff *skb = NULL; 763 struct chcr_wr *chcr_req; 764 struct cpl_rx_phys_dsgl *phys_cpl; 765 struct ulptx_sgl *ulptx; 766 struct chcr_blkcipher_req_ctx *reqctx = 767 ablkcipher_request_ctx(wrparam->req); 768 unsigned int temp = 0, transhdr_len, dst_size; 769 int error; 770 int nents; 771 unsigned int kctx_len; 772 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 773 GFP_KERNEL : GFP_ATOMIC; 774 struct adapter *adap = padap(c_ctx(tfm)->dev); 775 776 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, 777 reqctx->dst_ofst); 778 dst_size = get_space_for_phys_dsgl(nents); 779 kctx_len = roundup(ablkctx->enckey_len, 16); 780 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 781 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, 782 CHCR_SRC_SG_SIZE, reqctx->src_ofst); 783 temp = reqctx->imm ? roundup(wrparam->bytes, 16) : 784 (sgl_len(nents) * 8); 785 transhdr_len += temp; 786 transhdr_len = roundup(transhdr_len, 16); 787 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 788 if (!skb) { 789 error = -ENOMEM; 790 goto err; 791 } 792 chcr_req = __skb_put_zero(skb, transhdr_len); 793 chcr_req->sec_cpl.op_ivinsrtofst = 794 FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->tx_chan_id, 2, 1); 795 796 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); 797 chcr_req->sec_cpl.aadstart_cipherstop_hi = 798 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); 799 800 chcr_req->sec_cpl.cipherstop_lo_authinsert = 801 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 802 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, 803 ablkctx->ciph_mode, 804 0, 0, IV >> 1); 805 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 806 0, 1, dst_size); 807 808 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 809 if ((reqctx->op == CHCR_DECRYPT_OP) && 810 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 811 CRYPTO_ALG_SUB_TYPE_CTR)) && 812 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 813 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { 814 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 815 } else { 816 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || 817 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { 818 memcpy(chcr_req->key_ctx.key, ablkctx->key, 819 ablkctx->enckey_len); 820 } else { 821 memcpy(chcr_req->key_ctx.key, ablkctx->key + 822 (ablkctx->enckey_len >> 1), 823 ablkctx->enckey_len >> 1); 824 memcpy(chcr_req->key_ctx.key + 825 (ablkctx->enckey_len >> 1), 826 ablkctx->key, 827 ablkctx->enckey_len >> 1); 828 } 829 } 830 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 831 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 832 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); 833 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); 834 835 atomic_inc(&adap->chcr_stats.cipher_rqst); 836 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV 837 + (reqctx->imm ? (wrparam->bytes) : 0); 838 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, 839 transhdr_len, temp, 840 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); 841 reqctx->skb = skb; 842 843 if (reqctx->op && (ablkctx->ciph_mode == 844 CHCR_SCMD_CIPHER_MODE_AES_CBC)) 845 sg_pcopy_to_buffer(wrparam->req->src, 846 sg_nents(wrparam->req->src), wrparam->req->info, 16, 847 reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE); 848 849 return skb; 850 err: 851 return ERR_PTR(error); 852 } 853 854 static inline int chcr_keyctx_ck_size(unsigned int keylen) 855 { 856 int ck_size = 0; 857 858 if (keylen == AES_KEYSIZE_128) 859 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 860 else if (keylen == AES_KEYSIZE_192) 861 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 862 else if (keylen == AES_KEYSIZE_256) 863 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 864 else 865 ck_size = 0; 866 867 return ck_size; 868 } 869 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, 870 const u8 *key, 871 unsigned int keylen) 872 { 873 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 874 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 875 int err = 0; 876 877 crypto_sync_skcipher_clear_flags(ablkctx->sw_cipher, 878 CRYPTO_TFM_REQ_MASK); 879 crypto_sync_skcipher_set_flags(ablkctx->sw_cipher, 880 cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK); 881 err = crypto_sync_skcipher_setkey(ablkctx->sw_cipher, key, keylen); 882 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; 883 tfm->crt_flags |= 884 crypto_sync_skcipher_get_flags(ablkctx->sw_cipher) & 885 CRYPTO_TFM_RES_MASK; 886 return err; 887 } 888 889 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, 890 const u8 *key, 891 unsigned int keylen) 892 { 893 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 894 unsigned int ck_size, context_size; 895 u16 alignment = 0; 896 int err; 897 898 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 899 if (err) 900 goto badkey_err; 901 902 ck_size = chcr_keyctx_ck_size(keylen); 903 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; 904 memcpy(ablkctx->key, key, keylen); 905 ablkctx->enckey_len = keylen; 906 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 907 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 908 keylen + alignment) >> 4; 909 910 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 911 0, 0, context_size); 912 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 913 return 0; 914 badkey_err: 915 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 916 ablkctx->enckey_len = 0; 917 918 return err; 919 } 920 921 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, 922 const u8 *key, 923 unsigned int keylen) 924 { 925 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 926 unsigned int ck_size, context_size; 927 u16 alignment = 0; 928 int err; 929 930 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 931 if (err) 932 goto badkey_err; 933 ck_size = chcr_keyctx_ck_size(keylen); 934 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 935 memcpy(ablkctx->key, key, keylen); 936 ablkctx->enckey_len = keylen; 937 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 938 keylen + alignment) >> 4; 939 940 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 941 0, 0, context_size); 942 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 943 944 return 0; 945 badkey_err: 946 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 947 ablkctx->enckey_len = 0; 948 949 return err; 950 } 951 952 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, 953 const u8 *key, 954 unsigned int keylen) 955 { 956 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 957 unsigned int ck_size, context_size; 958 u16 alignment = 0; 959 int err; 960 961 if (keylen < CTR_RFC3686_NONCE_SIZE) 962 return -EINVAL; 963 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), 964 CTR_RFC3686_NONCE_SIZE); 965 966 keylen -= CTR_RFC3686_NONCE_SIZE; 967 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 968 if (err) 969 goto badkey_err; 970 971 ck_size = chcr_keyctx_ck_size(keylen); 972 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 973 memcpy(ablkctx->key, key, keylen); 974 ablkctx->enckey_len = keylen; 975 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 976 keylen + alignment) >> 4; 977 978 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 979 0, 0, context_size); 980 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 981 982 return 0; 983 badkey_err: 984 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 985 ablkctx->enckey_len = 0; 986 987 return err; 988 } 989 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) 990 { 991 unsigned int size = AES_BLOCK_SIZE; 992 __be32 *b = (__be32 *)(dstiv + size); 993 u32 c, prev; 994 995 memcpy(dstiv, srciv, AES_BLOCK_SIZE); 996 for (; size >= 4; size -= 4) { 997 prev = be32_to_cpu(*--b); 998 c = prev + add; 999 *b = cpu_to_be32(c); 1000 if (prev < c) 1001 break; 1002 add = 1; 1003 } 1004 1005 } 1006 1007 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) 1008 { 1009 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); 1010 u64 c; 1011 u32 temp = be32_to_cpu(*--b); 1012 1013 temp = ~temp; 1014 c = (u64)temp + 1; // No of block can processed withou overflow 1015 if ((bytes / AES_BLOCK_SIZE) > c) 1016 bytes = c * AES_BLOCK_SIZE; 1017 return bytes; 1018 } 1019 1020 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv, 1021 u32 isfinal) 1022 { 1023 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1024 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1025 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1026 struct crypto_cipher *cipher; 1027 int ret, i; 1028 u8 *key; 1029 unsigned int keylen; 1030 int round = reqctx->last_req_len / AES_BLOCK_SIZE; 1031 int round8 = round / 8; 1032 1033 cipher = ablkctx->aes_generic; 1034 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); 1035 1036 keylen = ablkctx->enckey_len / 2; 1037 key = ablkctx->key + keylen; 1038 ret = crypto_cipher_setkey(cipher, key, keylen); 1039 if (ret) 1040 goto out; 1041 crypto_cipher_encrypt_one(cipher, iv, iv); 1042 for (i = 0; i < round8; i++) 1043 gf128mul_x8_ble((le128 *)iv, (le128 *)iv); 1044 1045 for (i = 0; i < (round % 8); i++) 1046 gf128mul_x_ble((le128 *)iv, (le128 *)iv); 1047 1048 if (!isfinal) 1049 crypto_cipher_decrypt_one(cipher, iv, iv); 1050 out: 1051 return ret; 1052 } 1053 1054 static int chcr_update_cipher_iv(struct ablkcipher_request *req, 1055 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1056 { 1057 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1058 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1059 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 1060 int ret = 0; 1061 1062 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1063 ctr_add_iv(iv, req->info, (reqctx->processed / 1064 AES_BLOCK_SIZE)); 1065 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) 1066 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1067 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / 1068 AES_BLOCK_SIZE) + 1); 1069 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1070 ret = chcr_update_tweak(req, iv, 0); 1071 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1072 if (reqctx->op) 1073 /*Updated before sending last WR*/ 1074 memcpy(iv, req->info, AES_BLOCK_SIZE); 1075 else 1076 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1077 } 1078 1079 return ret; 1080 1081 } 1082 1083 /* We need separate function for final iv because in rfc3686 Initial counter 1084 * starts from 1 and buffer size of iv is 8 byte only which remains constant 1085 * for subsequent update requests 1086 */ 1087 1088 static int chcr_final_cipher_iv(struct ablkcipher_request *req, 1089 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1090 { 1091 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1092 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1093 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 1094 int ret = 0; 1095 1096 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1097 ctr_add_iv(iv, req->info, DIV_ROUND_UP(reqctx->processed, 1098 AES_BLOCK_SIZE)); 1099 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1100 ret = chcr_update_tweak(req, iv, 1); 1101 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1102 /*Already updated for Decrypt*/ 1103 if (!reqctx->op) 1104 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1105 1106 } 1107 return ret; 1108 1109 } 1110 1111 static int chcr_handle_cipher_resp(struct ablkcipher_request *req, 1112 unsigned char *input, int err) 1113 { 1114 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1115 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1116 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1117 struct sk_buff *skb; 1118 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; 1119 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1120 struct cipher_wr_param wrparam; 1121 struct chcr_dev *dev = c_ctx(tfm)->dev; 1122 int bytes; 1123 1124 if (err) 1125 goto unmap; 1126 if (req->nbytes == reqctx->processed) { 1127 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1128 req); 1129 err = chcr_final_cipher_iv(req, fw6_pld, req->info); 1130 goto complete; 1131 } 1132 1133 if (!reqctx->imm) { 1134 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0, 1135 CIP_SPACE_LEFT(ablkctx->enckey_len), 1136 reqctx->src_ofst, reqctx->dst_ofst); 1137 if ((bytes + reqctx->processed) >= req->nbytes) 1138 bytes = req->nbytes - reqctx->processed; 1139 else 1140 bytes = rounddown(bytes, 16); 1141 } else { 1142 /*CTR mode counter overfloa*/ 1143 bytes = req->nbytes - reqctx->processed; 1144 } 1145 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); 1146 if (err) 1147 goto unmap; 1148 1149 if (unlikely(bytes == 0)) { 1150 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1151 req); 1152 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1153 req->base.flags, 1154 req->src, 1155 req->dst, 1156 req->nbytes, 1157 req->info, 1158 reqctx->op); 1159 goto complete; 1160 } 1161 1162 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1163 CRYPTO_ALG_SUB_TYPE_CTR) 1164 bytes = adjust_ctr_overflow(reqctx->iv, bytes); 1165 wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx]; 1166 wrparam.req = req; 1167 wrparam.bytes = bytes; 1168 skb = create_cipher_wr(&wrparam); 1169 if (IS_ERR(skb)) { 1170 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 1171 err = PTR_ERR(skb); 1172 goto unmap; 1173 } 1174 skb->dev = u_ctx->lldi.ports[0]; 1175 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1176 chcr_send_wr(skb); 1177 reqctx->last_req_len = bytes; 1178 reqctx->processed += bytes; 1179 return 0; 1180 unmap: 1181 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1182 complete: 1183 chcr_dec_wrcount(dev); 1184 req->base.complete(&req->base, err); 1185 return err; 1186 } 1187 1188 static int process_cipher(struct ablkcipher_request *req, 1189 unsigned short qid, 1190 struct sk_buff **skb, 1191 unsigned short op_type) 1192 { 1193 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1194 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 1195 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1196 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1197 struct cipher_wr_param wrparam; 1198 int bytes, err = -EINVAL; 1199 1200 reqctx->processed = 0; 1201 if (!req->info) 1202 goto error; 1203 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 1204 (req->nbytes == 0) || 1205 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) { 1206 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 1207 ablkctx->enckey_len, req->nbytes, ivsize); 1208 goto error; 1209 } 1210 1211 err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1212 if (err) 1213 goto error; 1214 if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + 1215 AES_MIN_KEY_SIZE + 1216 sizeof(struct cpl_rx_phys_dsgl) + 1217 /*Min dsgl size*/ 1218 32))) { 1219 /* Can be sent as Imm*/ 1220 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; 1221 1222 dnents = sg_nents_xlen(req->dst, req->nbytes, 1223 CHCR_DST_SG_SIZE, 0); 1224 phys_dsgl = get_space_for_phys_dsgl(dnents); 1225 kctx_len = roundup(ablkctx->enckey_len, 16); 1226 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 1227 reqctx->imm = (transhdr_len + IV + req->nbytes) <= 1228 SGE_MAX_WR_LEN; 1229 bytes = IV + req->nbytes; 1230 1231 } else { 1232 reqctx->imm = 0; 1233 } 1234 1235 if (!reqctx->imm) { 1236 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0, 1237 CIP_SPACE_LEFT(ablkctx->enckey_len), 1238 0, 0); 1239 if ((bytes + reqctx->processed) >= req->nbytes) 1240 bytes = req->nbytes - reqctx->processed; 1241 else 1242 bytes = rounddown(bytes, 16); 1243 } else { 1244 bytes = req->nbytes; 1245 } 1246 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1247 CRYPTO_ALG_SUB_TYPE_CTR) { 1248 bytes = adjust_ctr_overflow(req->info, bytes); 1249 } 1250 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1251 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { 1252 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); 1253 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info, 1254 CTR_RFC3686_IV_SIZE); 1255 1256 /* initialize counter portion of counter block */ 1257 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1258 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1259 1260 } else { 1261 1262 memcpy(reqctx->iv, req->info, IV); 1263 } 1264 if (unlikely(bytes == 0)) { 1265 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1266 req); 1267 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1268 req->base.flags, 1269 req->src, 1270 req->dst, 1271 req->nbytes, 1272 reqctx->iv, 1273 op_type); 1274 goto error; 1275 } 1276 reqctx->op = op_type; 1277 reqctx->srcsg = req->src; 1278 reqctx->dstsg = req->dst; 1279 reqctx->src_ofst = 0; 1280 reqctx->dst_ofst = 0; 1281 wrparam.qid = qid; 1282 wrparam.req = req; 1283 wrparam.bytes = bytes; 1284 *skb = create_cipher_wr(&wrparam); 1285 if (IS_ERR(*skb)) { 1286 err = PTR_ERR(*skb); 1287 goto unmap; 1288 } 1289 reqctx->processed = bytes; 1290 reqctx->last_req_len = bytes; 1291 1292 return 0; 1293 unmap: 1294 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1295 error: 1296 return err; 1297 } 1298 1299 static int chcr_aes_encrypt(struct ablkcipher_request *req) 1300 { 1301 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1302 struct chcr_dev *dev = c_ctx(tfm)->dev; 1303 struct sk_buff *skb = NULL; 1304 int err, isfull = 0; 1305 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1306 1307 err = chcr_inc_wrcount(dev); 1308 if (err) 1309 return -ENXIO; 1310 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1311 c_ctx(tfm)->tx_qidx))) { 1312 isfull = 1; 1313 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1314 err = -ENOSPC; 1315 goto error; 1316 } 1317 } 1318 1319 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], 1320 &skb, CHCR_ENCRYPT_OP); 1321 if (err || !skb) 1322 return err; 1323 skb->dev = u_ctx->lldi.ports[0]; 1324 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1325 chcr_send_wr(skb); 1326 return isfull ? -EBUSY : -EINPROGRESS; 1327 error: 1328 chcr_dec_wrcount(dev); 1329 return err; 1330 } 1331 1332 static int chcr_aes_decrypt(struct ablkcipher_request *req) 1333 { 1334 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1335 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1336 struct chcr_dev *dev = c_ctx(tfm)->dev; 1337 struct sk_buff *skb = NULL; 1338 int err, isfull = 0; 1339 1340 err = chcr_inc_wrcount(dev); 1341 if (err) 1342 return -ENXIO; 1343 1344 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1345 c_ctx(tfm)->tx_qidx))) { 1346 isfull = 1; 1347 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1348 return -ENOSPC; 1349 } 1350 1351 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], 1352 &skb, CHCR_DECRYPT_OP); 1353 if (err || !skb) 1354 return err; 1355 skb->dev = u_ctx->lldi.ports[0]; 1356 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1357 chcr_send_wr(skb); 1358 return isfull ? -EBUSY : -EINPROGRESS; 1359 } 1360 1361 static int chcr_device_init(struct chcr_context *ctx) 1362 { 1363 struct uld_ctx *u_ctx = NULL; 1364 unsigned int id; 1365 int txq_perchan, txq_idx, ntxq; 1366 int err = 0, rxq_perchan, rxq_idx; 1367 1368 id = smp_processor_id(); 1369 if (!ctx->dev) { 1370 u_ctx = assign_chcr_device(); 1371 if (!u_ctx) { 1372 err = -ENXIO; 1373 pr_err("chcr device assignment fails\n"); 1374 goto out; 1375 } 1376 ctx->dev = &u_ctx->dev; 1377 ntxq = u_ctx->lldi.ntxq; 1378 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 1379 txq_perchan = ntxq / u_ctx->lldi.nchan; 1380 spin_lock(&ctx->dev->lock_chcr_dev); 1381 ctx->tx_chan_id = ctx->dev->tx_channel_id; 1382 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; 1383 spin_unlock(&ctx->dev->lock_chcr_dev); 1384 rxq_idx = ctx->tx_chan_id * rxq_perchan; 1385 rxq_idx += id % rxq_perchan; 1386 txq_idx = ctx->tx_chan_id * txq_perchan; 1387 txq_idx += id % txq_perchan; 1388 ctx->rx_qidx = rxq_idx; 1389 ctx->tx_qidx = txq_idx; 1390 /* Channel Id used by SGE to forward packet to Host. 1391 * Same value should be used in cpl_fw6_pld RSS_CH field 1392 * by FW. Driver programs PCI channel ID to be used in fw 1393 * at the time of queue allocation with value "pi->tx_chan" 1394 */ 1395 ctx->pci_chan_id = txq_idx / txq_perchan; 1396 } 1397 out: 1398 return err; 1399 } 1400 1401 static int chcr_cra_init(struct crypto_tfm *tfm) 1402 { 1403 struct crypto_alg *alg = tfm->__crt_alg; 1404 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1405 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1406 1407 ablkctx->sw_cipher = crypto_alloc_sync_skcipher(alg->cra_name, 0, 1408 CRYPTO_ALG_NEED_FALLBACK); 1409 if (IS_ERR(ablkctx->sw_cipher)) { 1410 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1411 return PTR_ERR(ablkctx->sw_cipher); 1412 } 1413 1414 if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) { 1415 /* To update tweak*/ 1416 ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0); 1417 if (IS_ERR(ablkctx->aes_generic)) { 1418 pr_err("failed to allocate aes cipher for tweak\n"); 1419 return PTR_ERR(ablkctx->aes_generic); 1420 } 1421 } else 1422 ablkctx->aes_generic = NULL; 1423 1424 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1425 return chcr_device_init(crypto_tfm_ctx(tfm)); 1426 } 1427 1428 static int chcr_rfc3686_init(struct crypto_tfm *tfm) 1429 { 1430 struct crypto_alg *alg = tfm->__crt_alg; 1431 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1432 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1433 1434 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) 1435 * cannot be used as fallback in chcr_handle_cipher_response 1436 */ 1437 ablkctx->sw_cipher = crypto_alloc_sync_skcipher("ctr(aes)", 0, 1438 CRYPTO_ALG_NEED_FALLBACK); 1439 if (IS_ERR(ablkctx->sw_cipher)) { 1440 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1441 return PTR_ERR(ablkctx->sw_cipher); 1442 } 1443 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1444 return chcr_device_init(crypto_tfm_ctx(tfm)); 1445 } 1446 1447 1448 static void chcr_cra_exit(struct crypto_tfm *tfm) 1449 { 1450 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1451 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1452 1453 crypto_free_sync_skcipher(ablkctx->sw_cipher); 1454 if (ablkctx->aes_generic) 1455 crypto_free_cipher(ablkctx->aes_generic); 1456 } 1457 1458 static int get_alg_config(struct algo_param *params, 1459 unsigned int auth_size) 1460 { 1461 switch (auth_size) { 1462 case SHA1_DIGEST_SIZE: 1463 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 1464 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 1465 params->result_size = SHA1_DIGEST_SIZE; 1466 break; 1467 case SHA224_DIGEST_SIZE: 1468 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1469 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 1470 params->result_size = SHA256_DIGEST_SIZE; 1471 break; 1472 case SHA256_DIGEST_SIZE: 1473 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1474 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 1475 params->result_size = SHA256_DIGEST_SIZE; 1476 break; 1477 case SHA384_DIGEST_SIZE: 1478 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1479 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 1480 params->result_size = SHA512_DIGEST_SIZE; 1481 break; 1482 case SHA512_DIGEST_SIZE: 1483 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1484 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 1485 params->result_size = SHA512_DIGEST_SIZE; 1486 break; 1487 default: 1488 pr_err("chcr : ERROR, unsupported digest size\n"); 1489 return -EINVAL; 1490 } 1491 return 0; 1492 } 1493 1494 static inline void chcr_free_shash(struct crypto_shash *base_hash) 1495 { 1496 crypto_free_shash(base_hash); 1497 } 1498 1499 /** 1500 * create_hash_wr - Create hash work request 1501 * @req - Cipher req base 1502 */ 1503 static struct sk_buff *create_hash_wr(struct ahash_request *req, 1504 struct hash_wr_param *param) 1505 { 1506 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1507 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1508 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 1509 struct sk_buff *skb = NULL; 1510 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 1511 struct chcr_wr *chcr_req; 1512 struct ulptx_sgl *ulptx; 1513 unsigned int nents = 0, transhdr_len; 1514 unsigned int temp = 0; 1515 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1516 GFP_ATOMIC; 1517 struct adapter *adap = padap(h_ctx(tfm)->dev); 1518 int error = 0; 1519 1520 transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len); 1521 req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len + 1522 param->sg_len) <= SGE_MAX_WR_LEN; 1523 nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len, 1524 CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst); 1525 nents += param->bfr_len ? 1 : 0; 1526 transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len + 1527 param->sg_len, 16) : (sgl_len(nents) * 8); 1528 transhdr_len = roundup(transhdr_len, 16); 1529 1530 skb = alloc_skb(transhdr_len, flags); 1531 if (!skb) 1532 return ERR_PTR(-ENOMEM); 1533 chcr_req = __skb_put_zero(skb, transhdr_len); 1534 1535 chcr_req->sec_cpl.op_ivinsrtofst = 1536 FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->tx_chan_id, 2, 0); 1537 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 1538 1539 chcr_req->sec_cpl.aadstart_cipherstop_hi = 1540 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 1541 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1542 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 1543 chcr_req->sec_cpl.seqno_numivs = 1544 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 1545 param->opad_needed, 0); 1546 1547 chcr_req->sec_cpl.ivgen_hdrlen = 1548 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 1549 1550 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 1551 param->alg_prm.result_size); 1552 1553 if (param->opad_needed) 1554 memcpy(chcr_req->key_ctx.key + 1555 ((param->alg_prm.result_size <= 32) ? 32 : 1556 CHCR_HASH_MAX_DIGEST_SIZE), 1557 hmacctx->opad, param->alg_prm.result_size); 1558 1559 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 1560 param->alg_prm.mk_size, 0, 1561 param->opad_needed, 1562 ((param->kctx_len + 1563 sizeof(chcr_req->key_ctx)) >> 4)); 1564 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 1565 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len + 1566 DUMMY_BYTES); 1567 if (param->bfr_len != 0) { 1568 req_ctx->hctx_wr.dma_addr = 1569 dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr, 1570 param->bfr_len, DMA_TO_DEVICE); 1571 if (dma_mapping_error(&u_ctx->lldi.pdev->dev, 1572 req_ctx->hctx_wr. dma_addr)) { 1573 error = -ENOMEM; 1574 goto err; 1575 } 1576 req_ctx->hctx_wr.dma_len = param->bfr_len; 1577 } else { 1578 req_ctx->hctx_wr.dma_addr = 0; 1579 } 1580 chcr_add_hash_src_ent(req, ulptx, param); 1581 /* Request upto max wr size */ 1582 temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ? 1583 (param->sg_len + param->bfr_len) : 0); 1584 atomic_inc(&adap->chcr_stats.digest_rqst); 1585 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm, 1586 param->hash_size, transhdr_len, 1587 temp, 0); 1588 req_ctx->hctx_wr.skb = skb; 1589 return skb; 1590 err: 1591 kfree_skb(skb); 1592 return ERR_PTR(error); 1593 } 1594 1595 static int chcr_ahash_update(struct ahash_request *req) 1596 { 1597 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1598 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1599 struct uld_ctx *u_ctx = NULL; 1600 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1601 struct sk_buff *skb; 1602 u8 remainder = 0, bs; 1603 unsigned int nbytes = req->nbytes; 1604 struct hash_wr_param params; 1605 int error, isfull = 0; 1606 1607 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1608 u_ctx = ULD_CTX(h_ctx(rtfm)); 1609 1610 if (nbytes + req_ctx->reqlen >= bs) { 1611 remainder = (nbytes + req_ctx->reqlen) % bs; 1612 nbytes = nbytes + req_ctx->reqlen - remainder; 1613 } else { 1614 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 1615 + req_ctx->reqlen, nbytes, 0); 1616 req_ctx->reqlen += nbytes; 1617 return 0; 1618 } 1619 error = chcr_inc_wrcount(dev); 1620 if (error) 1621 return -ENXIO; 1622 /* Detach state for CHCR means lldi or padap is freed. Increasing 1623 * inflight count for dev guarantees that lldi and padap is valid 1624 */ 1625 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1626 h_ctx(rtfm)->tx_qidx))) { 1627 isfull = 1; 1628 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1629 error = -ENOSPC; 1630 goto err; 1631 } 1632 } 1633 1634 chcr_init_hctx_per_wr(req_ctx); 1635 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1636 if (error) { 1637 error = -ENOMEM; 1638 goto err; 1639 } 1640 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1641 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1642 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1643 HASH_SPACE_LEFT(params.kctx_len), 0); 1644 if (params.sg_len > req->nbytes) 1645 params.sg_len = req->nbytes; 1646 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) - 1647 req_ctx->reqlen; 1648 params.opad_needed = 0; 1649 params.more = 1; 1650 params.last = 0; 1651 params.bfr_len = req_ctx->reqlen; 1652 params.scmd1 = 0; 1653 req_ctx->hctx_wr.srcsg = req->src; 1654 1655 params.hash_size = params.alg_prm.result_size; 1656 req_ctx->data_len += params.sg_len + params.bfr_len; 1657 skb = create_hash_wr(req, ¶ms); 1658 if (IS_ERR(skb)) { 1659 error = PTR_ERR(skb); 1660 goto unmap; 1661 } 1662 1663 req_ctx->hctx_wr.processed += params.sg_len; 1664 if (remainder) { 1665 /* Swap buffers */ 1666 swap(req_ctx->reqbfr, req_ctx->skbfr); 1667 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 1668 req_ctx->reqbfr, remainder, req->nbytes - 1669 remainder); 1670 } 1671 req_ctx->reqlen = remainder; 1672 skb->dev = u_ctx->lldi.ports[0]; 1673 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1674 chcr_send_wr(skb); 1675 1676 return isfull ? -EBUSY : -EINPROGRESS; 1677 unmap: 1678 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1679 err: 1680 chcr_dec_wrcount(dev); 1681 return error; 1682 } 1683 1684 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 1685 { 1686 memset(bfr_ptr, 0, bs); 1687 *bfr_ptr = 0x80; 1688 if (bs == 64) 1689 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 1690 else 1691 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 1692 } 1693 1694 static int chcr_ahash_final(struct ahash_request *req) 1695 { 1696 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1697 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1698 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1699 struct hash_wr_param params; 1700 struct sk_buff *skb; 1701 struct uld_ctx *u_ctx = NULL; 1702 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1703 int error = -EINVAL; 1704 1705 error = chcr_inc_wrcount(dev); 1706 if (error) 1707 return -ENXIO; 1708 1709 chcr_init_hctx_per_wr(req_ctx); 1710 u_ctx = ULD_CTX(h_ctx(rtfm)); 1711 if (is_hmac(crypto_ahash_tfm(rtfm))) 1712 params.opad_needed = 1; 1713 else 1714 params.opad_needed = 0; 1715 params.sg_len = 0; 1716 req_ctx->hctx_wr.isfinal = 1; 1717 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1718 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1719 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1720 params.opad_needed = 1; 1721 params.kctx_len *= 2; 1722 } else { 1723 params.opad_needed = 0; 1724 } 1725 1726 req_ctx->hctx_wr.result = 1; 1727 params.bfr_len = req_ctx->reqlen; 1728 req_ctx->data_len += params.bfr_len + params.sg_len; 1729 req_ctx->hctx_wr.srcsg = req->src; 1730 if (req_ctx->reqlen == 0) { 1731 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1732 params.last = 0; 1733 params.more = 1; 1734 params.scmd1 = 0; 1735 params.bfr_len = bs; 1736 1737 } else { 1738 params.scmd1 = req_ctx->data_len; 1739 params.last = 1; 1740 params.more = 0; 1741 } 1742 params.hash_size = crypto_ahash_digestsize(rtfm); 1743 skb = create_hash_wr(req, ¶ms); 1744 if (IS_ERR(skb)) { 1745 error = PTR_ERR(skb); 1746 goto err; 1747 } 1748 req_ctx->reqlen = 0; 1749 skb->dev = u_ctx->lldi.ports[0]; 1750 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1751 chcr_send_wr(skb); 1752 return -EINPROGRESS; 1753 err: 1754 chcr_dec_wrcount(dev); 1755 return error; 1756 } 1757 1758 static int chcr_ahash_finup(struct ahash_request *req) 1759 { 1760 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1761 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1762 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1763 struct uld_ctx *u_ctx = NULL; 1764 struct sk_buff *skb; 1765 struct hash_wr_param params; 1766 u8 bs; 1767 int error, isfull = 0; 1768 1769 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1770 u_ctx = ULD_CTX(h_ctx(rtfm)); 1771 error = chcr_inc_wrcount(dev); 1772 if (error) 1773 return -ENXIO; 1774 1775 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1776 h_ctx(rtfm)->tx_qidx))) { 1777 isfull = 1; 1778 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1779 error = -ENOSPC; 1780 goto err; 1781 } 1782 } 1783 chcr_init_hctx_per_wr(req_ctx); 1784 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1785 if (error) { 1786 error = -ENOMEM; 1787 goto err; 1788 } 1789 1790 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1791 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1792 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1793 params.kctx_len *= 2; 1794 params.opad_needed = 1; 1795 } else { 1796 params.opad_needed = 0; 1797 } 1798 1799 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1800 HASH_SPACE_LEFT(params.kctx_len), 0); 1801 if (params.sg_len < req->nbytes) { 1802 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1803 params.kctx_len /= 2; 1804 params.opad_needed = 0; 1805 } 1806 params.last = 0; 1807 params.more = 1; 1808 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) 1809 - req_ctx->reqlen; 1810 params.hash_size = params.alg_prm.result_size; 1811 params.scmd1 = 0; 1812 } else { 1813 params.last = 1; 1814 params.more = 0; 1815 params.sg_len = req->nbytes; 1816 params.hash_size = crypto_ahash_digestsize(rtfm); 1817 params.scmd1 = req_ctx->data_len + req_ctx->reqlen + 1818 params.sg_len; 1819 } 1820 params.bfr_len = req_ctx->reqlen; 1821 req_ctx->data_len += params.bfr_len + params.sg_len; 1822 req_ctx->hctx_wr.result = 1; 1823 req_ctx->hctx_wr.srcsg = req->src; 1824 if ((req_ctx->reqlen + req->nbytes) == 0) { 1825 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1826 params.last = 0; 1827 params.more = 1; 1828 params.scmd1 = 0; 1829 params.bfr_len = bs; 1830 } 1831 skb = create_hash_wr(req, ¶ms); 1832 if (IS_ERR(skb)) { 1833 error = PTR_ERR(skb); 1834 goto unmap; 1835 } 1836 req_ctx->reqlen = 0; 1837 req_ctx->hctx_wr.processed += params.sg_len; 1838 skb->dev = u_ctx->lldi.ports[0]; 1839 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1840 chcr_send_wr(skb); 1841 1842 return isfull ? -EBUSY : -EINPROGRESS; 1843 unmap: 1844 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1845 err: 1846 chcr_dec_wrcount(dev); 1847 return error; 1848 } 1849 1850 static int chcr_ahash_digest(struct ahash_request *req) 1851 { 1852 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1853 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1854 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1855 struct uld_ctx *u_ctx = NULL; 1856 struct sk_buff *skb; 1857 struct hash_wr_param params; 1858 u8 bs; 1859 int error, isfull = 0; 1860 1861 rtfm->init(req); 1862 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1863 error = chcr_inc_wrcount(dev); 1864 if (error) 1865 return -ENXIO; 1866 1867 u_ctx = ULD_CTX(h_ctx(rtfm)); 1868 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1869 h_ctx(rtfm)->tx_qidx))) { 1870 isfull = 1; 1871 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1872 error = -ENOSPC; 1873 goto err; 1874 } 1875 } 1876 1877 chcr_init_hctx_per_wr(req_ctx); 1878 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1879 if (error) { 1880 error = -ENOMEM; 1881 goto err; 1882 } 1883 1884 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1885 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1886 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1887 params.kctx_len *= 2; 1888 params.opad_needed = 1; 1889 } else { 1890 params.opad_needed = 0; 1891 } 1892 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1893 HASH_SPACE_LEFT(params.kctx_len), 0); 1894 if (params.sg_len < req->nbytes) { 1895 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1896 params.kctx_len /= 2; 1897 params.opad_needed = 0; 1898 } 1899 params.last = 0; 1900 params.more = 1; 1901 params.scmd1 = 0; 1902 params.sg_len = rounddown(params.sg_len, bs); 1903 params.hash_size = params.alg_prm.result_size; 1904 } else { 1905 params.sg_len = req->nbytes; 1906 params.hash_size = crypto_ahash_digestsize(rtfm); 1907 params.last = 1; 1908 params.more = 0; 1909 params.scmd1 = req->nbytes + req_ctx->data_len; 1910 1911 } 1912 params.bfr_len = 0; 1913 req_ctx->hctx_wr.result = 1; 1914 req_ctx->hctx_wr.srcsg = req->src; 1915 req_ctx->data_len += params.bfr_len + params.sg_len; 1916 1917 if (req->nbytes == 0) { 1918 create_last_hash_block(req_ctx->reqbfr, bs, 0); 1919 params.more = 1; 1920 params.bfr_len = bs; 1921 } 1922 1923 skb = create_hash_wr(req, ¶ms); 1924 if (IS_ERR(skb)) { 1925 error = PTR_ERR(skb); 1926 goto unmap; 1927 } 1928 req_ctx->hctx_wr.processed += params.sg_len; 1929 skb->dev = u_ctx->lldi.ports[0]; 1930 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1931 chcr_send_wr(skb); 1932 return isfull ? -EBUSY : -EINPROGRESS; 1933 unmap: 1934 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1935 err: 1936 chcr_dec_wrcount(dev); 1937 return error; 1938 } 1939 1940 static int chcr_ahash_continue(struct ahash_request *req) 1941 { 1942 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 1943 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; 1944 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1945 struct uld_ctx *u_ctx = NULL; 1946 struct sk_buff *skb; 1947 struct hash_wr_param params; 1948 u8 bs; 1949 int error; 1950 1951 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1952 u_ctx = ULD_CTX(h_ctx(rtfm)); 1953 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1954 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1955 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1956 params.kctx_len *= 2; 1957 params.opad_needed = 1; 1958 } else { 1959 params.opad_needed = 0; 1960 } 1961 params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0, 1962 HASH_SPACE_LEFT(params.kctx_len), 1963 hctx_wr->src_ofst); 1964 if ((params.sg_len + hctx_wr->processed) > req->nbytes) 1965 params.sg_len = req->nbytes - hctx_wr->processed; 1966 if (!hctx_wr->result || 1967 ((params.sg_len + hctx_wr->processed) < req->nbytes)) { 1968 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1969 params.kctx_len /= 2; 1970 params.opad_needed = 0; 1971 } 1972 params.last = 0; 1973 params.more = 1; 1974 params.sg_len = rounddown(params.sg_len, bs); 1975 params.hash_size = params.alg_prm.result_size; 1976 params.scmd1 = 0; 1977 } else { 1978 params.last = 1; 1979 params.more = 0; 1980 params.hash_size = crypto_ahash_digestsize(rtfm); 1981 params.scmd1 = reqctx->data_len + params.sg_len; 1982 } 1983 params.bfr_len = 0; 1984 reqctx->data_len += params.sg_len; 1985 skb = create_hash_wr(req, ¶ms); 1986 if (IS_ERR(skb)) { 1987 error = PTR_ERR(skb); 1988 goto err; 1989 } 1990 hctx_wr->processed += params.sg_len; 1991 skb->dev = u_ctx->lldi.ports[0]; 1992 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1993 chcr_send_wr(skb); 1994 return 0; 1995 err: 1996 return error; 1997 } 1998 1999 static inline void chcr_handle_ahash_resp(struct ahash_request *req, 2000 unsigned char *input, 2001 int err) 2002 { 2003 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2004 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; 2005 int digestsize, updated_digestsize; 2006 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2007 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 2008 struct chcr_dev *dev = h_ctx(tfm)->dev; 2009 2010 if (input == NULL) 2011 goto out; 2012 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); 2013 updated_digestsize = digestsize; 2014 if (digestsize == SHA224_DIGEST_SIZE) 2015 updated_digestsize = SHA256_DIGEST_SIZE; 2016 else if (digestsize == SHA384_DIGEST_SIZE) 2017 updated_digestsize = SHA512_DIGEST_SIZE; 2018 2019 if (hctx_wr->dma_addr) { 2020 dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr, 2021 hctx_wr->dma_len, DMA_TO_DEVICE); 2022 hctx_wr->dma_addr = 0; 2023 } 2024 if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) == 2025 req->nbytes)) { 2026 if (hctx_wr->result == 1) { 2027 hctx_wr->result = 0; 2028 memcpy(req->result, input + sizeof(struct cpl_fw6_pld), 2029 digestsize); 2030 } else { 2031 memcpy(reqctx->partial_hash, 2032 input + sizeof(struct cpl_fw6_pld), 2033 updated_digestsize); 2034 2035 } 2036 goto unmap; 2037 } 2038 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), 2039 updated_digestsize); 2040 2041 err = chcr_ahash_continue(req); 2042 if (err) 2043 goto unmap; 2044 return; 2045 unmap: 2046 if (hctx_wr->is_sg_map) 2047 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 2048 2049 2050 out: 2051 chcr_dec_wrcount(dev); 2052 req->base.complete(&req->base, err); 2053 } 2054 2055 /* 2056 * chcr_handle_resp - Unmap the DMA buffers associated with the request 2057 * @req: crypto request 2058 */ 2059 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 2060 int err) 2061 { 2062 struct crypto_tfm *tfm = req->tfm; 2063 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2064 struct adapter *adap = padap(ctx->dev); 2065 2066 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 2067 case CRYPTO_ALG_TYPE_AEAD: 2068 err = chcr_handle_aead_resp(aead_request_cast(req), input, err); 2069 break; 2070 2071 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2072 chcr_handle_cipher_resp(ablkcipher_request_cast(req), 2073 input, err); 2074 break; 2075 case CRYPTO_ALG_TYPE_AHASH: 2076 chcr_handle_ahash_resp(ahash_request_cast(req), input, err); 2077 } 2078 atomic_inc(&adap->chcr_stats.complete); 2079 return err; 2080 } 2081 static int chcr_ahash_export(struct ahash_request *areq, void *out) 2082 { 2083 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2084 struct chcr_ahash_req_ctx *state = out; 2085 2086 state->reqlen = req_ctx->reqlen; 2087 state->data_len = req_ctx->data_len; 2088 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 2089 memcpy(state->partial_hash, req_ctx->partial_hash, 2090 CHCR_HASH_MAX_DIGEST_SIZE); 2091 chcr_init_hctx_per_wr(state); 2092 return 0; 2093 } 2094 2095 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 2096 { 2097 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2098 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 2099 2100 req_ctx->reqlen = state->reqlen; 2101 req_ctx->data_len = state->data_len; 2102 req_ctx->reqbfr = req_ctx->bfr1; 2103 req_ctx->skbfr = req_ctx->bfr2; 2104 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 2105 memcpy(req_ctx->partial_hash, state->partial_hash, 2106 CHCR_HASH_MAX_DIGEST_SIZE); 2107 chcr_init_hctx_per_wr(req_ctx); 2108 return 0; 2109 } 2110 2111 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2112 unsigned int keylen) 2113 { 2114 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 2115 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2116 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2117 unsigned int i, err = 0, updated_digestsize; 2118 2119 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 2120 2121 /* use the key to calculate the ipad and opad. ipad will sent with the 2122 * first request's data. opad will be sent with the final hash result 2123 * ipad in hmacctx->ipad and opad in hmacctx->opad location 2124 */ 2125 shash->tfm = hmacctx->base_hash; 2126 if (keylen > bs) { 2127 err = crypto_shash_digest(shash, key, keylen, 2128 hmacctx->ipad); 2129 if (err) 2130 goto out; 2131 keylen = digestsize; 2132 } else { 2133 memcpy(hmacctx->ipad, key, keylen); 2134 } 2135 memset(hmacctx->ipad + keylen, 0, bs - keylen); 2136 memcpy(hmacctx->opad, hmacctx->ipad, bs); 2137 2138 for (i = 0; i < bs / sizeof(int); i++) { 2139 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 2140 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 2141 } 2142 2143 updated_digestsize = digestsize; 2144 if (digestsize == SHA224_DIGEST_SIZE) 2145 updated_digestsize = SHA256_DIGEST_SIZE; 2146 else if (digestsize == SHA384_DIGEST_SIZE) 2147 updated_digestsize = SHA512_DIGEST_SIZE; 2148 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 2149 hmacctx->ipad, digestsize); 2150 if (err) 2151 goto out; 2152 chcr_change_order(hmacctx->ipad, updated_digestsize); 2153 2154 err = chcr_compute_partial_hash(shash, hmacctx->opad, 2155 hmacctx->opad, digestsize); 2156 if (err) 2157 goto out; 2158 chcr_change_order(hmacctx->opad, updated_digestsize); 2159 out: 2160 return err; 2161 } 2162 2163 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 2164 unsigned int key_len) 2165 { 2166 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 2167 unsigned short context_size = 0; 2168 int err; 2169 2170 err = chcr_cipher_fallback_setkey(cipher, key, key_len); 2171 if (err) 2172 goto badkey_err; 2173 2174 memcpy(ablkctx->key, key, key_len); 2175 ablkctx->enckey_len = key_len; 2176 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 2177 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 2178 ablkctx->key_ctx_hdr = 2179 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 2180 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 2181 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 2182 CHCR_KEYCTX_NO_KEY, 1, 2183 0, context_size); 2184 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 2185 return 0; 2186 badkey_err: 2187 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 2188 ablkctx->enckey_len = 0; 2189 2190 return err; 2191 } 2192 2193 static int chcr_sha_init(struct ahash_request *areq) 2194 { 2195 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2196 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2197 int digestsize = crypto_ahash_digestsize(tfm); 2198 2199 req_ctx->data_len = 0; 2200 req_ctx->reqlen = 0; 2201 req_ctx->reqbfr = req_ctx->bfr1; 2202 req_ctx->skbfr = req_ctx->bfr2; 2203 copy_hash_init_values(req_ctx->partial_hash, digestsize); 2204 2205 return 0; 2206 } 2207 2208 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 2209 { 2210 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2211 sizeof(struct chcr_ahash_req_ctx)); 2212 return chcr_device_init(crypto_tfm_ctx(tfm)); 2213 } 2214 2215 static int chcr_hmac_init(struct ahash_request *areq) 2216 { 2217 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2218 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 2219 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); 2220 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 2221 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 2222 2223 chcr_sha_init(areq); 2224 req_ctx->data_len = bs; 2225 if (is_hmac(crypto_ahash_tfm(rtfm))) { 2226 if (digestsize == SHA224_DIGEST_SIZE) 2227 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2228 SHA256_DIGEST_SIZE); 2229 else if (digestsize == SHA384_DIGEST_SIZE) 2230 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2231 SHA512_DIGEST_SIZE); 2232 else 2233 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2234 digestsize); 2235 } 2236 return 0; 2237 } 2238 2239 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 2240 { 2241 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2242 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2243 unsigned int digestsize = 2244 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 2245 2246 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2247 sizeof(struct chcr_ahash_req_ctx)); 2248 hmacctx->base_hash = chcr_alloc_shash(digestsize); 2249 if (IS_ERR(hmacctx->base_hash)) 2250 return PTR_ERR(hmacctx->base_hash); 2251 return chcr_device_init(crypto_tfm_ctx(tfm)); 2252 } 2253 2254 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 2255 { 2256 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2257 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2258 2259 if (hmacctx->base_hash) { 2260 chcr_free_shash(hmacctx->base_hash); 2261 hmacctx->base_hash = NULL; 2262 } 2263 } 2264 2265 inline void chcr_aead_common_exit(struct aead_request *req) 2266 { 2267 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2268 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2269 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); 2270 2271 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); 2272 } 2273 2274 static int chcr_aead_common_init(struct aead_request *req) 2275 { 2276 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2277 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2278 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2279 unsigned int authsize = crypto_aead_authsize(tfm); 2280 int error = -EINVAL; 2281 2282 /* validate key size */ 2283 if (aeadctx->enckey_len == 0) 2284 goto err; 2285 if (reqctx->op && req->cryptlen < authsize) 2286 goto err; 2287 if (reqctx->b0_len) 2288 reqctx->scratch_pad = reqctx->iv + IV; 2289 else 2290 reqctx->scratch_pad = NULL; 2291 2292 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2293 reqctx->op); 2294 if (error) { 2295 error = -ENOMEM; 2296 goto err; 2297 } 2298 2299 return 0; 2300 err: 2301 return error; 2302 } 2303 2304 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, 2305 int aadmax, int wrlen, 2306 unsigned short op_type) 2307 { 2308 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 2309 2310 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || 2311 dst_nents > MAX_DSGL_ENT || 2312 (req->assoclen > aadmax) || 2313 (wrlen > SGE_MAX_WR_LEN)) 2314 return 1; 2315 return 0; 2316 } 2317 2318 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) 2319 { 2320 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2321 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2322 struct aead_request *subreq = aead_request_ctx(req); 2323 2324 aead_request_set_tfm(subreq, aeadctx->sw_cipher); 2325 aead_request_set_callback(subreq, req->base.flags, 2326 req->base.complete, req->base.data); 2327 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2328 req->iv); 2329 aead_request_set_ad(subreq, req->assoclen); 2330 return op_type ? crypto_aead_decrypt(subreq) : 2331 crypto_aead_encrypt(subreq); 2332 } 2333 2334 static struct sk_buff *create_authenc_wr(struct aead_request *req, 2335 unsigned short qid, 2336 int size) 2337 { 2338 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2339 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2340 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2341 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2342 struct sk_buff *skb = NULL; 2343 struct chcr_wr *chcr_req; 2344 struct cpl_rx_phys_dsgl *phys_cpl; 2345 struct ulptx_sgl *ulptx; 2346 unsigned int transhdr_len; 2347 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm); 2348 unsigned int kctx_len = 0, dnents, snents; 2349 unsigned int authsize = crypto_aead_authsize(tfm); 2350 int error = -EINVAL; 2351 u8 *ivptr; 2352 int null = 0; 2353 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2354 GFP_ATOMIC; 2355 struct adapter *adap = padap(a_ctx(tfm)->dev); 2356 2357 if (req->cryptlen == 0) 2358 return NULL; 2359 2360 reqctx->b0_len = 0; 2361 error = chcr_aead_common_init(req); 2362 if (error) 2363 return ERR_PTR(error); 2364 2365 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL || 2366 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2367 null = 1; 2368 } 2369 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + 2370 (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0); 2371 dnents += MIN_AUTH_SG; // For IV 2372 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 2373 CHCR_SRC_SG_SIZE, 0); 2374 dst_size = get_space_for_phys_dsgl(dnents); 2375 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) 2376 - sizeof(chcr_req->key_ctx); 2377 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2378 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) < 2379 SGE_MAX_WR_LEN; 2380 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) 2381 : (sgl_len(snents) * 8); 2382 transhdr_len += temp; 2383 transhdr_len = roundup(transhdr_len, 16); 2384 2385 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 2386 transhdr_len, reqctx->op)) { 2387 atomic_inc(&adap->chcr_stats.fallback); 2388 chcr_aead_common_exit(req); 2389 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 2390 } 2391 skb = alloc_skb(transhdr_len, flags); 2392 if (!skb) { 2393 error = -ENOMEM; 2394 goto err; 2395 } 2396 2397 chcr_req = __skb_put_zero(skb, transhdr_len); 2398 2399 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; 2400 2401 /* 2402 * Input order is AAD,IV and Payload. where IV should be included as 2403 * the part of authdata. All other fields should be filled according 2404 * to the hardware spec 2405 */ 2406 chcr_req->sec_cpl.op_ivinsrtofst = 2407 FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->tx_chan_id, 2, 1); 2408 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen); 2409 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2410 null ? 0 : 1 + IV, 2411 null ? 0 : IV + req->assoclen, 2412 req->assoclen + IV + 1, 2413 (temp & 0x1F0) >> 4); 2414 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 2415 temp & 0xF, 2416 null ? 0 : req->assoclen + IV + 1, 2417 temp, temp); 2418 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL || 2419 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA) 2420 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR; 2421 else 2422 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC; 2423 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 2424 (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0, 2425 temp, 2426 actx->auth_mode, aeadctx->hmac_ctrl, 2427 IV >> 1); 2428 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2429 0, 0, dst_size); 2430 2431 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2432 if (reqctx->op == CHCR_ENCRYPT_OP || 2433 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2434 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) 2435 memcpy(chcr_req->key_ctx.key, aeadctx->key, 2436 aeadctx->enckey_len); 2437 else 2438 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 2439 aeadctx->enckey_len); 2440 2441 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 2442 actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16)); 2443 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2444 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 2445 ulptx = (struct ulptx_sgl *)(ivptr + IV); 2446 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2447 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2448 memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE); 2449 memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv, 2450 CTR_RFC3686_IV_SIZE); 2451 *(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE + 2452 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 2453 } else { 2454 memcpy(ivptr, req->iv, IV); 2455 } 2456 chcr_add_aead_dst_ent(req, phys_cpl, qid); 2457 chcr_add_aead_src_ent(req, ulptx); 2458 atomic_inc(&adap->chcr_stats.cipher_rqst); 2459 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 2460 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); 2461 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 2462 transhdr_len, temp, 0); 2463 reqctx->skb = skb; 2464 2465 return skb; 2466 err: 2467 chcr_aead_common_exit(req); 2468 2469 return ERR_PTR(error); 2470 } 2471 2472 int chcr_aead_dma_map(struct device *dev, 2473 struct aead_request *req, 2474 unsigned short op_type) 2475 { 2476 int error; 2477 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2478 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2479 unsigned int authsize = crypto_aead_authsize(tfm); 2480 int dst_size; 2481 2482 dst_size = req->assoclen + req->cryptlen + (op_type ? 2483 -authsize : authsize); 2484 if (!req->cryptlen || !dst_size) 2485 return 0; 2486 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len), 2487 DMA_BIDIRECTIONAL); 2488 if (dma_mapping_error(dev, reqctx->iv_dma)) 2489 return -ENOMEM; 2490 if (reqctx->b0_len) 2491 reqctx->b0_dma = reqctx->iv_dma + IV; 2492 else 2493 reqctx->b0_dma = 0; 2494 if (req->src == req->dst) { 2495 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2496 DMA_BIDIRECTIONAL); 2497 if (!error) 2498 goto err; 2499 } else { 2500 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2501 DMA_TO_DEVICE); 2502 if (!error) 2503 goto err; 2504 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2505 DMA_FROM_DEVICE); 2506 if (!error) { 2507 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2508 DMA_TO_DEVICE); 2509 goto err; 2510 } 2511 } 2512 2513 return 0; 2514 err: 2515 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 2516 return -ENOMEM; 2517 } 2518 2519 void chcr_aead_dma_unmap(struct device *dev, 2520 struct aead_request *req, 2521 unsigned short op_type) 2522 { 2523 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2524 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2525 unsigned int authsize = crypto_aead_authsize(tfm); 2526 int dst_size; 2527 2528 dst_size = req->assoclen + req->cryptlen + (op_type ? 2529 -authsize : authsize); 2530 if (!req->cryptlen || !dst_size) 2531 return; 2532 2533 dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len), 2534 DMA_BIDIRECTIONAL); 2535 if (req->src == req->dst) { 2536 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2537 DMA_BIDIRECTIONAL); 2538 } else { 2539 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2540 DMA_TO_DEVICE); 2541 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2542 DMA_FROM_DEVICE); 2543 } 2544 } 2545 2546 void chcr_add_aead_src_ent(struct aead_request *req, 2547 struct ulptx_sgl *ulptx) 2548 { 2549 struct ulptx_walk ulp_walk; 2550 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2551 2552 if (reqctx->imm) { 2553 u8 *buf = (u8 *)ulptx; 2554 2555 if (reqctx->b0_len) { 2556 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); 2557 buf += reqctx->b0_len; 2558 } 2559 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2560 buf, req->cryptlen + req->assoclen, 0); 2561 } else { 2562 ulptx_walk_init(&ulp_walk, ulptx); 2563 if (reqctx->b0_len) 2564 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, 2565 reqctx->b0_dma); 2566 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen + 2567 req->assoclen, 0); 2568 ulptx_walk_end(&ulp_walk); 2569 } 2570 } 2571 2572 void chcr_add_aead_dst_ent(struct aead_request *req, 2573 struct cpl_rx_phys_dsgl *phys_cpl, 2574 unsigned short qid) 2575 { 2576 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2577 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2578 struct dsgl_walk dsgl_walk; 2579 unsigned int authsize = crypto_aead_authsize(tfm); 2580 struct chcr_context *ctx = a_ctx(tfm); 2581 u32 temp; 2582 2583 dsgl_walk_init(&dsgl_walk, phys_cpl); 2584 dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma); 2585 temp = req->assoclen + req->cryptlen + 2586 (reqctx->op ? -authsize : authsize); 2587 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0); 2588 dsgl_walk_end(&dsgl_walk, qid, ctx->pci_chan_id); 2589 } 2590 2591 void chcr_add_cipher_src_ent(struct ablkcipher_request *req, 2592 void *ulptx, 2593 struct cipher_wr_param *wrparam) 2594 { 2595 struct ulptx_walk ulp_walk; 2596 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2597 u8 *buf = ulptx; 2598 2599 memcpy(buf, reqctx->iv, IV); 2600 buf += IV; 2601 if (reqctx->imm) { 2602 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2603 buf, wrparam->bytes, reqctx->processed); 2604 } else { 2605 ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf); 2606 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, 2607 reqctx->src_ofst); 2608 reqctx->srcsg = ulp_walk.last_sg; 2609 reqctx->src_ofst = ulp_walk.last_sg_len; 2610 ulptx_walk_end(&ulp_walk); 2611 } 2612 } 2613 2614 void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, 2615 struct cpl_rx_phys_dsgl *phys_cpl, 2616 struct cipher_wr_param *wrparam, 2617 unsigned short qid) 2618 { 2619 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2620 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); 2621 struct chcr_context *ctx = c_ctx(tfm); 2622 struct dsgl_walk dsgl_walk; 2623 2624 dsgl_walk_init(&dsgl_walk, phys_cpl); 2625 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, 2626 reqctx->dst_ofst); 2627 reqctx->dstsg = dsgl_walk.last_sg; 2628 reqctx->dst_ofst = dsgl_walk.last_sg_len; 2629 2630 dsgl_walk_end(&dsgl_walk, qid, ctx->pci_chan_id); 2631 } 2632 2633 void chcr_add_hash_src_ent(struct ahash_request *req, 2634 struct ulptx_sgl *ulptx, 2635 struct hash_wr_param *param) 2636 { 2637 struct ulptx_walk ulp_walk; 2638 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2639 2640 if (reqctx->hctx_wr.imm) { 2641 u8 *buf = (u8 *)ulptx; 2642 2643 if (param->bfr_len) { 2644 memcpy(buf, reqctx->reqbfr, param->bfr_len); 2645 buf += param->bfr_len; 2646 } 2647 2648 sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg, 2649 sg_nents(reqctx->hctx_wr.srcsg), buf, 2650 param->sg_len, 0); 2651 } else { 2652 ulptx_walk_init(&ulp_walk, ulptx); 2653 if (param->bfr_len) 2654 ulptx_walk_add_page(&ulp_walk, param->bfr_len, 2655 reqctx->hctx_wr.dma_addr); 2656 ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg, 2657 param->sg_len, reqctx->hctx_wr.src_ofst); 2658 reqctx->hctx_wr.srcsg = ulp_walk.last_sg; 2659 reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len; 2660 ulptx_walk_end(&ulp_walk); 2661 } 2662 } 2663 2664 int chcr_hash_dma_map(struct device *dev, 2665 struct ahash_request *req) 2666 { 2667 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2668 int error = 0; 2669 2670 if (!req->nbytes) 2671 return 0; 2672 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2673 DMA_TO_DEVICE); 2674 if (!error) 2675 return -ENOMEM; 2676 req_ctx->hctx_wr.is_sg_map = 1; 2677 return 0; 2678 } 2679 2680 void chcr_hash_dma_unmap(struct device *dev, 2681 struct ahash_request *req) 2682 { 2683 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2684 2685 if (!req->nbytes) 2686 return; 2687 2688 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2689 DMA_TO_DEVICE); 2690 req_ctx->hctx_wr.is_sg_map = 0; 2691 2692 } 2693 2694 int chcr_cipher_dma_map(struct device *dev, 2695 struct ablkcipher_request *req) 2696 { 2697 int error; 2698 2699 if (req->src == req->dst) { 2700 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2701 DMA_BIDIRECTIONAL); 2702 if (!error) 2703 goto err; 2704 } else { 2705 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2706 DMA_TO_DEVICE); 2707 if (!error) 2708 goto err; 2709 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2710 DMA_FROM_DEVICE); 2711 if (!error) { 2712 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2713 DMA_TO_DEVICE); 2714 goto err; 2715 } 2716 } 2717 2718 return 0; 2719 err: 2720 return -ENOMEM; 2721 } 2722 2723 void chcr_cipher_dma_unmap(struct device *dev, 2724 struct ablkcipher_request *req) 2725 { 2726 if (req->src == req->dst) { 2727 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2728 DMA_BIDIRECTIONAL); 2729 } else { 2730 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2731 DMA_TO_DEVICE); 2732 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2733 DMA_FROM_DEVICE); 2734 } 2735 } 2736 2737 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 2738 { 2739 __be32 data; 2740 2741 memset(block, 0, csize); 2742 block += csize; 2743 2744 if (csize >= 4) 2745 csize = 4; 2746 else if (msglen > (unsigned int)(1 << (8 * csize))) 2747 return -EOVERFLOW; 2748 2749 data = cpu_to_be32(msglen); 2750 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 2751 2752 return 0; 2753 } 2754 2755 static int generate_b0(struct aead_request *req, u8 *ivptr, 2756 unsigned short op_type) 2757 { 2758 unsigned int l, lp, m; 2759 int rc; 2760 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2761 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2762 u8 *b0 = reqctx->scratch_pad; 2763 2764 m = crypto_aead_authsize(aead); 2765 2766 memcpy(b0, ivptr, 16); 2767 2768 lp = b0[0]; 2769 l = lp + 1; 2770 2771 /* set m, bits 3-5 */ 2772 *b0 |= (8 * ((m - 2) / 2)); 2773 2774 /* set adata, bit 6, if associated data is used */ 2775 if (req->assoclen) 2776 *b0 |= 64; 2777 rc = set_msg_len(b0 + 16 - l, 2778 (op_type == CHCR_DECRYPT_OP) ? 2779 req->cryptlen - m : req->cryptlen, l); 2780 2781 return rc; 2782 } 2783 2784 static inline int crypto_ccm_check_iv(const u8 *iv) 2785 { 2786 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 2787 if (iv[0] < 1 || iv[0] > 7) 2788 return -EINVAL; 2789 2790 return 0; 2791 } 2792 2793 static int ccm_format_packet(struct aead_request *req, 2794 u8 *ivptr, 2795 unsigned int sub_type, 2796 unsigned short op_type, 2797 unsigned int assoclen) 2798 { 2799 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2800 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2801 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2802 int rc = 0; 2803 2804 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2805 ivptr[0] = 3; 2806 memcpy(ivptr + 1, &aeadctx->salt[0], 3); 2807 memcpy(ivptr + 4, req->iv, 8); 2808 memset(ivptr + 12, 0, 4); 2809 } else { 2810 memcpy(ivptr, req->iv, 16); 2811 } 2812 if (assoclen) 2813 *((unsigned short *)(reqctx->scratch_pad + 16)) = 2814 htons(assoclen); 2815 2816 rc = generate_b0(req, ivptr, op_type); 2817 /* zero the ctr value */ 2818 memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1); 2819 return rc; 2820 } 2821 2822 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 2823 unsigned int dst_size, 2824 struct aead_request *req, 2825 unsigned short op_type) 2826 { 2827 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2828 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2829 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 2830 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 2831 unsigned int c_id = a_ctx(tfm)->tx_chan_id; 2832 unsigned int ccm_xtra; 2833 unsigned char tag_offset = 0, auth_offset = 0; 2834 unsigned int assoclen; 2835 2836 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2837 assoclen = req->assoclen - 8; 2838 else 2839 assoclen = req->assoclen; 2840 ccm_xtra = CCM_B0_SIZE + 2841 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 2842 2843 auth_offset = req->cryptlen ? 2844 (req->assoclen + IV + 1 + ccm_xtra) : 0; 2845 if (op_type == CHCR_DECRYPT_OP) { 2846 if (crypto_aead_authsize(tfm) != req->cryptlen) 2847 tag_offset = crypto_aead_authsize(tfm); 2848 else 2849 auth_offset = 0; 2850 } 2851 2852 2853 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, 2854 2, 1); 2855 sec_cpl->pldlen = 2856 htonl(req->assoclen + IV + req->cryptlen + ccm_xtra); 2857 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 2858 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2859 1 + IV, IV + assoclen + ccm_xtra, 2860 req->assoclen + IV + 1 + ccm_xtra, 0); 2861 2862 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 2863 auth_offset, tag_offset, 2864 (op_type == CHCR_ENCRYPT_OP) ? 0 : 2865 crypto_aead_authsize(tfm)); 2866 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2867 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 2868 cipher_mode, mac_mode, 2869 aeadctx->hmac_ctrl, IV >> 1); 2870 2871 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 2872 0, dst_size); 2873 } 2874 2875 static int aead_ccm_validate_input(unsigned short op_type, 2876 struct aead_request *req, 2877 struct chcr_aead_ctx *aeadctx, 2878 unsigned int sub_type) 2879 { 2880 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2881 if (crypto_ccm_check_iv(req->iv)) { 2882 pr_err("CCM: IV check fails\n"); 2883 return -EINVAL; 2884 } 2885 } else { 2886 if (req->assoclen != 16 && req->assoclen != 20) { 2887 pr_err("RFC4309: Invalid AAD length %d\n", 2888 req->assoclen); 2889 return -EINVAL; 2890 } 2891 } 2892 return 0; 2893 } 2894 2895 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 2896 unsigned short qid, 2897 int size) 2898 { 2899 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2900 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2901 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2902 struct sk_buff *skb = NULL; 2903 struct chcr_wr *chcr_req; 2904 struct cpl_rx_phys_dsgl *phys_cpl; 2905 struct ulptx_sgl *ulptx; 2906 unsigned int transhdr_len; 2907 unsigned int dst_size = 0, kctx_len, dnents, temp, snents; 2908 unsigned int sub_type, assoclen = req->assoclen; 2909 unsigned int authsize = crypto_aead_authsize(tfm); 2910 int error = -EINVAL; 2911 u8 *ivptr; 2912 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2913 GFP_ATOMIC; 2914 struct adapter *adap = padap(a_ctx(tfm)->dev); 2915 2916 sub_type = get_aead_subtype(tfm); 2917 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2918 assoclen -= 8; 2919 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); 2920 error = chcr_aead_common_init(req); 2921 if (error) 2922 return ERR_PTR(error); 2923 2924 error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type); 2925 if (error) 2926 goto err; 2927 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen 2928 + (reqctx->op ? -authsize : authsize), 2929 CHCR_DST_SG_SIZE, 0); 2930 dnents += MIN_CCM_SG; // For IV and B0 2931 dst_size = get_space_for_phys_dsgl(dnents); 2932 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 2933 CHCR_SRC_SG_SIZE, 0); 2934 snents += MIN_CCM_SG; //For B0 2935 kctx_len = roundup(aeadctx->enckey_len, 16) * 2; 2936 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2937 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen + 2938 reqctx->b0_len) <= SGE_MAX_WR_LEN; 2939 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen + 2940 reqctx->b0_len, 16) : 2941 (sgl_len(snents) * 8); 2942 transhdr_len += temp; 2943 transhdr_len = roundup(transhdr_len, 16); 2944 2945 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - 2946 reqctx->b0_len, transhdr_len, reqctx->op)) { 2947 atomic_inc(&adap->chcr_stats.fallback); 2948 chcr_aead_common_exit(req); 2949 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 2950 } 2951 skb = alloc_skb(transhdr_len, flags); 2952 2953 if (!skb) { 2954 error = -ENOMEM; 2955 goto err; 2956 } 2957 2958 chcr_req = __skb_put_zero(skb, transhdr_len); 2959 2960 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op); 2961 2962 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2963 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 2964 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 2965 aeadctx->key, aeadctx->enckey_len); 2966 2967 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2968 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 2969 ulptx = (struct ulptx_sgl *)(ivptr + IV); 2970 error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen); 2971 if (error) 2972 goto dstmap_fail; 2973 chcr_add_aead_dst_ent(req, phys_cpl, qid); 2974 chcr_add_aead_src_ent(req, ulptx); 2975 2976 atomic_inc(&adap->chcr_stats.aead_rqst); 2977 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 2978 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen + 2979 reqctx->b0_len) : 0); 2980 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, 2981 transhdr_len, temp, 0); 2982 reqctx->skb = skb; 2983 2984 return skb; 2985 dstmap_fail: 2986 kfree_skb(skb); 2987 err: 2988 chcr_aead_common_exit(req); 2989 return ERR_PTR(error); 2990 } 2991 2992 static struct sk_buff *create_gcm_wr(struct aead_request *req, 2993 unsigned short qid, 2994 int size) 2995 { 2996 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2997 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2998 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2999 struct sk_buff *skb = NULL; 3000 struct chcr_wr *chcr_req; 3001 struct cpl_rx_phys_dsgl *phys_cpl; 3002 struct ulptx_sgl *ulptx; 3003 unsigned int transhdr_len, dnents = 0, snents; 3004 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; 3005 unsigned int authsize = crypto_aead_authsize(tfm); 3006 int error = -EINVAL; 3007 u8 *ivptr; 3008 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 3009 GFP_ATOMIC; 3010 struct adapter *adap = padap(a_ctx(tfm)->dev); 3011 3012 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 3013 assoclen = req->assoclen - 8; 3014 3015 reqctx->b0_len = 0; 3016 error = chcr_aead_common_init(req); 3017 if (error) 3018 return ERR_PTR(error); 3019 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + 3020 (reqctx->op ? -authsize : authsize), 3021 CHCR_DST_SG_SIZE, 0); 3022 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 3023 CHCR_SRC_SG_SIZE, 0); 3024 dnents += MIN_GCM_SG; // For IV 3025 dst_size = get_space_for_phys_dsgl(dnents); 3026 kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE; 3027 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 3028 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <= 3029 SGE_MAX_WR_LEN; 3030 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) : 3031 (sgl_len(snents) * 8); 3032 transhdr_len += temp; 3033 transhdr_len = roundup(transhdr_len, 16); 3034 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 3035 transhdr_len, reqctx->op)) { 3036 3037 atomic_inc(&adap->chcr_stats.fallback); 3038 chcr_aead_common_exit(req); 3039 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 3040 } 3041 skb = alloc_skb(transhdr_len, flags); 3042 if (!skb) { 3043 error = -ENOMEM; 3044 goto err; 3045 } 3046 3047 chcr_req = __skb_put_zero(skb, transhdr_len); 3048 3049 //Offset of tag from end 3050 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; 3051 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 3052 a_ctx(tfm)->tx_chan_id, 2, 1); 3053 chcr_req->sec_cpl.pldlen = 3054 htonl(req->assoclen + IV + req->cryptlen); 3055 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 3056 assoclen ? 1 + IV : 0, 3057 assoclen ? IV + assoclen : 0, 3058 req->assoclen + IV + 1, 0); 3059 chcr_req->sec_cpl.cipherstop_lo_authinsert = 3060 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1, 3061 temp, temp); 3062 chcr_req->sec_cpl.seqno_numivs = 3063 FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op == 3064 CHCR_ENCRYPT_OP) ? 1 : 0, 3065 CHCR_SCMD_CIPHER_MODE_AES_GCM, 3066 CHCR_SCMD_AUTH_MODE_GHASH, 3067 aeadctx->hmac_ctrl, IV >> 1); 3068 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 3069 0, 0, dst_size); 3070 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 3071 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 3072 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 3073 GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 3074 3075 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 3076 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 3077 /* prepare a 16 byte iv */ 3078 /* S A L T | IV | 0x00000001 */ 3079 if (get_aead_subtype(tfm) == 3080 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 3081 memcpy(ivptr, aeadctx->salt, 4); 3082 memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE); 3083 } else { 3084 memcpy(ivptr, req->iv, GCM_AES_IV_SIZE); 3085 } 3086 *((unsigned int *)(ivptr + 12)) = htonl(0x01); 3087 3088 ulptx = (struct ulptx_sgl *)(ivptr + 16); 3089 3090 chcr_add_aead_dst_ent(req, phys_cpl, qid); 3091 chcr_add_aead_src_ent(req, ulptx); 3092 atomic_inc(&adap->chcr_stats.aead_rqst); 3093 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 3094 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); 3095 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 3096 transhdr_len, temp, reqctx->verify); 3097 reqctx->skb = skb; 3098 return skb; 3099 3100 err: 3101 chcr_aead_common_exit(req); 3102 return ERR_PTR(error); 3103 } 3104 3105 3106 3107 static int chcr_aead_cra_init(struct crypto_aead *tfm) 3108 { 3109 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3110 struct aead_alg *alg = crypto_aead_alg(tfm); 3111 3112 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, 3113 CRYPTO_ALG_NEED_FALLBACK | 3114 CRYPTO_ALG_ASYNC); 3115 if (IS_ERR(aeadctx->sw_cipher)) 3116 return PTR_ERR(aeadctx->sw_cipher); 3117 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), 3118 sizeof(struct aead_request) + 3119 crypto_aead_reqsize(aeadctx->sw_cipher))); 3120 return chcr_device_init(a_ctx(tfm)); 3121 } 3122 3123 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 3124 { 3125 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3126 3127 crypto_free_aead(aeadctx->sw_cipher); 3128 } 3129 3130 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 3131 unsigned int authsize) 3132 { 3133 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3134 3135 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 3136 aeadctx->mayverify = VERIFY_HW; 3137 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3138 } 3139 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 3140 unsigned int authsize) 3141 { 3142 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3143 u32 maxauth = crypto_aead_maxauthsize(tfm); 3144 3145 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 3146 * true for sha1. authsize == 12 condition should be before 3147 * authsize == (maxauth >> 1) 3148 */ 3149 if (authsize == ICV_4) { 3150 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3151 aeadctx->mayverify = VERIFY_HW; 3152 } else if (authsize == ICV_6) { 3153 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3154 aeadctx->mayverify = VERIFY_HW; 3155 } else if (authsize == ICV_10) { 3156 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3157 aeadctx->mayverify = VERIFY_HW; 3158 } else if (authsize == ICV_12) { 3159 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3160 aeadctx->mayverify = VERIFY_HW; 3161 } else if (authsize == ICV_14) { 3162 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3163 aeadctx->mayverify = VERIFY_HW; 3164 } else if (authsize == (maxauth >> 1)) { 3165 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3166 aeadctx->mayverify = VERIFY_HW; 3167 } else if (authsize == maxauth) { 3168 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3169 aeadctx->mayverify = VERIFY_HW; 3170 } else { 3171 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3172 aeadctx->mayverify = VERIFY_SW; 3173 } 3174 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3175 } 3176 3177 3178 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 3179 { 3180 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3181 3182 switch (authsize) { 3183 case ICV_4: 3184 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3185 aeadctx->mayverify = VERIFY_HW; 3186 break; 3187 case ICV_8: 3188 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3189 aeadctx->mayverify = VERIFY_HW; 3190 break; 3191 case ICV_12: 3192 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3193 aeadctx->mayverify = VERIFY_HW; 3194 break; 3195 case ICV_14: 3196 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3197 aeadctx->mayverify = VERIFY_HW; 3198 break; 3199 case ICV_16: 3200 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3201 aeadctx->mayverify = VERIFY_HW; 3202 break; 3203 case ICV_13: 3204 case ICV_15: 3205 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3206 aeadctx->mayverify = VERIFY_SW; 3207 break; 3208 default: 3209 3210 crypto_tfm_set_flags((struct crypto_tfm *) tfm, 3211 CRYPTO_TFM_RES_BAD_KEY_LEN); 3212 return -EINVAL; 3213 } 3214 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3215 } 3216 3217 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 3218 unsigned int authsize) 3219 { 3220 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3221 3222 switch (authsize) { 3223 case ICV_8: 3224 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3225 aeadctx->mayverify = VERIFY_HW; 3226 break; 3227 case ICV_12: 3228 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3229 aeadctx->mayverify = VERIFY_HW; 3230 break; 3231 case ICV_16: 3232 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3233 aeadctx->mayverify = VERIFY_HW; 3234 break; 3235 default: 3236 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 3237 CRYPTO_TFM_RES_BAD_KEY_LEN); 3238 return -EINVAL; 3239 } 3240 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3241 } 3242 3243 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 3244 unsigned int authsize) 3245 { 3246 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3247 3248 switch (authsize) { 3249 case ICV_4: 3250 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3251 aeadctx->mayverify = VERIFY_HW; 3252 break; 3253 case ICV_6: 3254 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3255 aeadctx->mayverify = VERIFY_HW; 3256 break; 3257 case ICV_8: 3258 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3259 aeadctx->mayverify = VERIFY_HW; 3260 break; 3261 case ICV_10: 3262 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3263 aeadctx->mayverify = VERIFY_HW; 3264 break; 3265 case ICV_12: 3266 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3267 aeadctx->mayverify = VERIFY_HW; 3268 break; 3269 case ICV_14: 3270 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3271 aeadctx->mayverify = VERIFY_HW; 3272 break; 3273 case ICV_16: 3274 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3275 aeadctx->mayverify = VERIFY_HW; 3276 break; 3277 default: 3278 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 3279 CRYPTO_TFM_RES_BAD_KEY_LEN); 3280 return -EINVAL; 3281 } 3282 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3283 } 3284 3285 static int chcr_ccm_common_setkey(struct crypto_aead *aead, 3286 const u8 *key, 3287 unsigned int keylen) 3288 { 3289 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3290 unsigned char ck_size, mk_size; 3291 int key_ctx_size = 0; 3292 3293 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2; 3294 if (keylen == AES_KEYSIZE_128) { 3295 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3296 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; 3297 } else if (keylen == AES_KEYSIZE_192) { 3298 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3299 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 3300 } else if (keylen == AES_KEYSIZE_256) { 3301 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3302 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 3303 } else { 3304 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3305 CRYPTO_TFM_RES_BAD_KEY_LEN); 3306 aeadctx->enckey_len = 0; 3307 return -EINVAL; 3308 } 3309 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 3310 key_ctx_size >> 4); 3311 memcpy(aeadctx->key, key, keylen); 3312 aeadctx->enckey_len = keylen; 3313 3314 return 0; 3315 } 3316 3317 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 3318 const u8 *key, 3319 unsigned int keylen) 3320 { 3321 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3322 int error; 3323 3324 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3325 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3326 CRYPTO_TFM_REQ_MASK); 3327 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3328 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3329 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3330 CRYPTO_TFM_RES_MASK); 3331 if (error) 3332 return error; 3333 return chcr_ccm_common_setkey(aead, key, keylen); 3334 } 3335 3336 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 3337 unsigned int keylen) 3338 { 3339 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3340 int error; 3341 3342 if (keylen < 3) { 3343 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3344 CRYPTO_TFM_RES_BAD_KEY_LEN); 3345 aeadctx->enckey_len = 0; 3346 return -EINVAL; 3347 } 3348 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3349 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3350 CRYPTO_TFM_REQ_MASK); 3351 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3352 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3353 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3354 CRYPTO_TFM_RES_MASK); 3355 if (error) 3356 return error; 3357 keylen -= 3; 3358 memcpy(aeadctx->salt, key + keylen, 3); 3359 return chcr_ccm_common_setkey(aead, key, keylen); 3360 } 3361 3362 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 3363 unsigned int keylen) 3364 { 3365 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3366 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 3367 struct crypto_cipher *cipher; 3368 unsigned int ck_size; 3369 int ret = 0, key_ctx_size = 0; 3370 3371 aeadctx->enckey_len = 0; 3372 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3373 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) 3374 & CRYPTO_TFM_REQ_MASK); 3375 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3376 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3377 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3378 CRYPTO_TFM_RES_MASK); 3379 if (ret) 3380 goto out; 3381 3382 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 3383 keylen > 3) { 3384 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 3385 memcpy(aeadctx->salt, key + keylen, 4); 3386 } 3387 if (keylen == AES_KEYSIZE_128) { 3388 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3389 } else if (keylen == AES_KEYSIZE_192) { 3390 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3391 } else if (keylen == AES_KEYSIZE_256) { 3392 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3393 } else { 3394 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3395 CRYPTO_TFM_RES_BAD_KEY_LEN); 3396 pr_err("GCM: Invalid key length %d\n", keylen); 3397 ret = -EINVAL; 3398 goto out; 3399 } 3400 3401 memcpy(aeadctx->key, key, keylen); 3402 aeadctx->enckey_len = keylen; 3403 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) + 3404 AEAD_H_SIZE; 3405 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 3406 CHCR_KEYCTX_MAC_KEY_SIZE_128, 3407 0, 0, 3408 key_ctx_size >> 4); 3409 /* Calculate the H = CIPH(K, 0 repeated 16 times). 3410 * It will go in key context 3411 */ 3412 cipher = crypto_alloc_cipher("aes-generic", 0, 0); 3413 if (IS_ERR(cipher)) { 3414 aeadctx->enckey_len = 0; 3415 ret = -ENOMEM; 3416 goto out; 3417 } 3418 3419 ret = crypto_cipher_setkey(cipher, key, keylen); 3420 if (ret) { 3421 aeadctx->enckey_len = 0; 3422 goto out1; 3423 } 3424 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 3425 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h); 3426 3427 out1: 3428 crypto_free_cipher(cipher); 3429 out: 3430 return ret; 3431 } 3432 3433 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 3434 unsigned int keylen) 3435 { 3436 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3437 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3438 /* it contains auth and cipher key both*/ 3439 struct crypto_authenc_keys keys; 3440 unsigned int bs, subtype; 3441 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 3442 int err = 0, i, key_ctx_len = 0; 3443 unsigned char ck_size = 0; 3444 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 3445 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 3446 struct algo_param param; 3447 int align; 3448 u8 *o_ptr = NULL; 3449 3450 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3451 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3452 & CRYPTO_TFM_REQ_MASK); 3453 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3454 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 3455 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 3456 & CRYPTO_TFM_RES_MASK); 3457 if (err) 3458 goto out; 3459 3460 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 3461 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 3462 goto out; 3463 } 3464 3465 if (get_alg_config(¶m, max_authsize)) { 3466 pr_err("chcr : Unsupported digest size\n"); 3467 goto out; 3468 } 3469 subtype = get_aead_subtype(authenc); 3470 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3471 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3472 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3473 goto out; 3474 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3475 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3476 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3477 } 3478 if (keys.enckeylen == AES_KEYSIZE_128) { 3479 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3480 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3481 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3482 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3483 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3484 } else { 3485 pr_err("chcr : Unsupported cipher key\n"); 3486 goto out; 3487 } 3488 3489 /* Copy only encryption key. We use authkey to generate h(ipad) and 3490 * h(opad) so authkey is not needed again. authkeylen size have the 3491 * size of the hash digest size. 3492 */ 3493 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3494 aeadctx->enckey_len = keys.enckeylen; 3495 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3496 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3497 3498 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3499 aeadctx->enckey_len << 3); 3500 } 3501 base_hash = chcr_alloc_shash(max_authsize); 3502 if (IS_ERR(base_hash)) { 3503 pr_err("chcr : Base driver cannot be loaded\n"); 3504 aeadctx->enckey_len = 0; 3505 memzero_explicit(&keys, sizeof(keys)); 3506 return -EINVAL; 3507 } 3508 { 3509 SHASH_DESC_ON_STACK(shash, base_hash); 3510 3511 shash->tfm = base_hash; 3512 bs = crypto_shash_blocksize(base_hash); 3513 align = KEYCTX_ALIGN_PAD(max_authsize); 3514 o_ptr = actx->h_iopad + param.result_size + align; 3515 3516 if (keys.authkeylen > bs) { 3517 err = crypto_shash_digest(shash, keys.authkey, 3518 keys.authkeylen, 3519 o_ptr); 3520 if (err) { 3521 pr_err("chcr : Base driver cannot be loaded\n"); 3522 goto out; 3523 } 3524 keys.authkeylen = max_authsize; 3525 } else 3526 memcpy(o_ptr, keys.authkey, keys.authkeylen); 3527 3528 /* Compute the ipad-digest*/ 3529 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3530 memcpy(pad, o_ptr, keys.authkeylen); 3531 for (i = 0; i < bs >> 2; i++) 3532 *((unsigned int *)pad + i) ^= IPAD_DATA; 3533 3534 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 3535 max_authsize)) 3536 goto out; 3537 /* Compute the opad-digest */ 3538 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3539 memcpy(pad, o_ptr, keys.authkeylen); 3540 for (i = 0; i < bs >> 2; i++) 3541 *((unsigned int *)pad + i) ^= OPAD_DATA; 3542 3543 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 3544 goto out; 3545 3546 /* convert the ipad and opad digest to network order */ 3547 chcr_change_order(actx->h_iopad, param.result_size); 3548 chcr_change_order(o_ptr, param.result_size); 3549 key_ctx_len = sizeof(struct _key_ctx) + 3550 roundup(keys.enckeylen, 16) + 3551 (param.result_size + align) * 2; 3552 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 3553 0, 1, key_ctx_len >> 4); 3554 actx->auth_mode = param.auth_mode; 3555 chcr_free_shash(base_hash); 3556 3557 memzero_explicit(&keys, sizeof(keys)); 3558 return 0; 3559 } 3560 out: 3561 aeadctx->enckey_len = 0; 3562 memzero_explicit(&keys, sizeof(keys)); 3563 if (!IS_ERR(base_hash)) 3564 chcr_free_shash(base_hash); 3565 return -EINVAL; 3566 } 3567 3568 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 3569 const u8 *key, unsigned int keylen) 3570 { 3571 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3572 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3573 struct crypto_authenc_keys keys; 3574 int err; 3575 /* it contains auth and cipher key both*/ 3576 unsigned int subtype; 3577 int key_ctx_len = 0; 3578 unsigned char ck_size = 0; 3579 3580 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3581 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3582 & CRYPTO_TFM_REQ_MASK); 3583 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3584 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 3585 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 3586 & CRYPTO_TFM_RES_MASK); 3587 if (err) 3588 goto out; 3589 3590 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 3591 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 3592 goto out; 3593 } 3594 subtype = get_aead_subtype(authenc); 3595 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3596 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3597 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3598 goto out; 3599 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3600 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3601 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3602 } 3603 if (keys.enckeylen == AES_KEYSIZE_128) { 3604 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3605 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3606 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3607 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3608 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3609 } else { 3610 pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen); 3611 goto out; 3612 } 3613 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3614 aeadctx->enckey_len = keys.enckeylen; 3615 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3616 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3617 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3618 aeadctx->enckey_len << 3); 3619 } 3620 key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16); 3621 3622 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 3623 0, key_ctx_len >> 4); 3624 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 3625 memzero_explicit(&keys, sizeof(keys)); 3626 return 0; 3627 out: 3628 aeadctx->enckey_len = 0; 3629 memzero_explicit(&keys, sizeof(keys)); 3630 return -EINVAL; 3631 } 3632 3633 static int chcr_aead_op(struct aead_request *req, 3634 int size, 3635 create_wr_t create_wr_fn) 3636 { 3637 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3638 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3639 struct uld_ctx *u_ctx; 3640 struct sk_buff *skb; 3641 int isfull = 0; 3642 struct chcr_dev *cdev; 3643 3644 cdev = a_ctx(tfm)->dev; 3645 if (!cdev) { 3646 pr_err("chcr : %s : No crypto device.\n", __func__); 3647 return -ENXIO; 3648 } 3649 3650 if (chcr_inc_wrcount(cdev)) { 3651 /* Detach state for CHCR means lldi or padap is freed. 3652 * We cannot increment fallback here. 3653 */ 3654 return chcr_aead_fallback(req, reqctx->op); 3655 } 3656 3657 u_ctx = ULD_CTX(a_ctx(tfm)); 3658 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 3659 a_ctx(tfm)->tx_qidx)) { 3660 isfull = 1; 3661 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 3662 chcr_dec_wrcount(cdev); 3663 return -ENOSPC; 3664 } 3665 } 3666 3667 /* Form a WR from req */ 3668 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size); 3669 3670 if (IS_ERR_OR_NULL(skb)) { 3671 chcr_dec_wrcount(cdev); 3672 return PTR_ERR_OR_ZERO(skb); 3673 } 3674 3675 skb->dev = u_ctx->lldi.ports[0]; 3676 set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); 3677 chcr_send_wr(skb); 3678 return isfull ? -EBUSY : -EINPROGRESS; 3679 } 3680 3681 static int chcr_aead_encrypt(struct aead_request *req) 3682 { 3683 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3684 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3685 3686 reqctx->verify = VERIFY_HW; 3687 reqctx->op = CHCR_ENCRYPT_OP; 3688 3689 switch (get_aead_subtype(tfm)) { 3690 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3691 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3692 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3693 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3694 return chcr_aead_op(req, 0, create_authenc_wr); 3695 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3696 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3697 return chcr_aead_op(req, 0, create_aead_ccm_wr); 3698 default: 3699 return chcr_aead_op(req, 0, create_gcm_wr); 3700 } 3701 } 3702 3703 static int chcr_aead_decrypt(struct aead_request *req) 3704 { 3705 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3706 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3707 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3708 int size; 3709 3710 if (aeadctx->mayverify == VERIFY_SW) { 3711 size = crypto_aead_maxauthsize(tfm); 3712 reqctx->verify = VERIFY_SW; 3713 } else { 3714 size = 0; 3715 reqctx->verify = VERIFY_HW; 3716 } 3717 reqctx->op = CHCR_DECRYPT_OP; 3718 switch (get_aead_subtype(tfm)) { 3719 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3720 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3721 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3722 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3723 return chcr_aead_op(req, size, create_authenc_wr); 3724 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3725 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3726 return chcr_aead_op(req, size, create_aead_ccm_wr); 3727 default: 3728 return chcr_aead_op(req, size, create_gcm_wr); 3729 } 3730 } 3731 3732 static struct chcr_alg_template driver_algs[] = { 3733 /* AES-CBC */ 3734 { 3735 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, 3736 .is_registered = 0, 3737 .alg.crypto = { 3738 .cra_name = "cbc(aes)", 3739 .cra_driver_name = "cbc-aes-chcr", 3740 .cra_blocksize = AES_BLOCK_SIZE, 3741 .cra_init = chcr_cra_init, 3742 .cra_exit = chcr_cra_exit, 3743 .cra_u.ablkcipher = { 3744 .min_keysize = AES_MIN_KEY_SIZE, 3745 .max_keysize = AES_MAX_KEY_SIZE, 3746 .ivsize = AES_BLOCK_SIZE, 3747 .setkey = chcr_aes_cbc_setkey, 3748 .encrypt = chcr_aes_encrypt, 3749 .decrypt = chcr_aes_decrypt, 3750 } 3751 } 3752 }, 3753 { 3754 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, 3755 .is_registered = 0, 3756 .alg.crypto = { 3757 .cra_name = "xts(aes)", 3758 .cra_driver_name = "xts-aes-chcr", 3759 .cra_blocksize = AES_BLOCK_SIZE, 3760 .cra_init = chcr_cra_init, 3761 .cra_exit = NULL, 3762 .cra_u .ablkcipher = { 3763 .min_keysize = 2 * AES_MIN_KEY_SIZE, 3764 .max_keysize = 2 * AES_MAX_KEY_SIZE, 3765 .ivsize = AES_BLOCK_SIZE, 3766 .setkey = chcr_aes_xts_setkey, 3767 .encrypt = chcr_aes_encrypt, 3768 .decrypt = chcr_aes_decrypt, 3769 } 3770 } 3771 }, 3772 { 3773 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, 3774 .is_registered = 0, 3775 .alg.crypto = { 3776 .cra_name = "ctr(aes)", 3777 .cra_driver_name = "ctr-aes-chcr", 3778 .cra_blocksize = 1, 3779 .cra_init = chcr_cra_init, 3780 .cra_exit = chcr_cra_exit, 3781 .cra_u.ablkcipher = { 3782 .min_keysize = AES_MIN_KEY_SIZE, 3783 .max_keysize = AES_MAX_KEY_SIZE, 3784 .ivsize = AES_BLOCK_SIZE, 3785 .setkey = chcr_aes_ctr_setkey, 3786 .encrypt = chcr_aes_encrypt, 3787 .decrypt = chcr_aes_decrypt, 3788 } 3789 } 3790 }, 3791 { 3792 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | 3793 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, 3794 .is_registered = 0, 3795 .alg.crypto = { 3796 .cra_name = "rfc3686(ctr(aes))", 3797 .cra_driver_name = "rfc3686-ctr-aes-chcr", 3798 .cra_blocksize = 1, 3799 .cra_init = chcr_rfc3686_init, 3800 .cra_exit = chcr_cra_exit, 3801 .cra_u.ablkcipher = { 3802 .min_keysize = AES_MIN_KEY_SIZE + 3803 CTR_RFC3686_NONCE_SIZE, 3804 .max_keysize = AES_MAX_KEY_SIZE + 3805 CTR_RFC3686_NONCE_SIZE, 3806 .ivsize = CTR_RFC3686_IV_SIZE, 3807 .setkey = chcr_aes_rfc3686_setkey, 3808 .encrypt = chcr_aes_encrypt, 3809 .decrypt = chcr_aes_decrypt, 3810 } 3811 } 3812 }, 3813 /* SHA */ 3814 { 3815 .type = CRYPTO_ALG_TYPE_AHASH, 3816 .is_registered = 0, 3817 .alg.hash = { 3818 .halg.digestsize = SHA1_DIGEST_SIZE, 3819 .halg.base = { 3820 .cra_name = "sha1", 3821 .cra_driver_name = "sha1-chcr", 3822 .cra_blocksize = SHA1_BLOCK_SIZE, 3823 } 3824 } 3825 }, 3826 { 3827 .type = CRYPTO_ALG_TYPE_AHASH, 3828 .is_registered = 0, 3829 .alg.hash = { 3830 .halg.digestsize = SHA256_DIGEST_SIZE, 3831 .halg.base = { 3832 .cra_name = "sha256", 3833 .cra_driver_name = "sha256-chcr", 3834 .cra_blocksize = SHA256_BLOCK_SIZE, 3835 } 3836 } 3837 }, 3838 { 3839 .type = CRYPTO_ALG_TYPE_AHASH, 3840 .is_registered = 0, 3841 .alg.hash = { 3842 .halg.digestsize = SHA224_DIGEST_SIZE, 3843 .halg.base = { 3844 .cra_name = "sha224", 3845 .cra_driver_name = "sha224-chcr", 3846 .cra_blocksize = SHA224_BLOCK_SIZE, 3847 } 3848 } 3849 }, 3850 { 3851 .type = CRYPTO_ALG_TYPE_AHASH, 3852 .is_registered = 0, 3853 .alg.hash = { 3854 .halg.digestsize = SHA384_DIGEST_SIZE, 3855 .halg.base = { 3856 .cra_name = "sha384", 3857 .cra_driver_name = "sha384-chcr", 3858 .cra_blocksize = SHA384_BLOCK_SIZE, 3859 } 3860 } 3861 }, 3862 { 3863 .type = CRYPTO_ALG_TYPE_AHASH, 3864 .is_registered = 0, 3865 .alg.hash = { 3866 .halg.digestsize = SHA512_DIGEST_SIZE, 3867 .halg.base = { 3868 .cra_name = "sha512", 3869 .cra_driver_name = "sha512-chcr", 3870 .cra_blocksize = SHA512_BLOCK_SIZE, 3871 } 3872 } 3873 }, 3874 /* HMAC */ 3875 { 3876 .type = CRYPTO_ALG_TYPE_HMAC, 3877 .is_registered = 0, 3878 .alg.hash = { 3879 .halg.digestsize = SHA1_DIGEST_SIZE, 3880 .halg.base = { 3881 .cra_name = "hmac(sha1)", 3882 .cra_driver_name = "hmac-sha1-chcr", 3883 .cra_blocksize = SHA1_BLOCK_SIZE, 3884 } 3885 } 3886 }, 3887 { 3888 .type = CRYPTO_ALG_TYPE_HMAC, 3889 .is_registered = 0, 3890 .alg.hash = { 3891 .halg.digestsize = SHA224_DIGEST_SIZE, 3892 .halg.base = { 3893 .cra_name = "hmac(sha224)", 3894 .cra_driver_name = "hmac-sha224-chcr", 3895 .cra_blocksize = SHA224_BLOCK_SIZE, 3896 } 3897 } 3898 }, 3899 { 3900 .type = CRYPTO_ALG_TYPE_HMAC, 3901 .is_registered = 0, 3902 .alg.hash = { 3903 .halg.digestsize = SHA256_DIGEST_SIZE, 3904 .halg.base = { 3905 .cra_name = "hmac(sha256)", 3906 .cra_driver_name = "hmac-sha256-chcr", 3907 .cra_blocksize = SHA256_BLOCK_SIZE, 3908 } 3909 } 3910 }, 3911 { 3912 .type = CRYPTO_ALG_TYPE_HMAC, 3913 .is_registered = 0, 3914 .alg.hash = { 3915 .halg.digestsize = SHA384_DIGEST_SIZE, 3916 .halg.base = { 3917 .cra_name = "hmac(sha384)", 3918 .cra_driver_name = "hmac-sha384-chcr", 3919 .cra_blocksize = SHA384_BLOCK_SIZE, 3920 } 3921 } 3922 }, 3923 { 3924 .type = CRYPTO_ALG_TYPE_HMAC, 3925 .is_registered = 0, 3926 .alg.hash = { 3927 .halg.digestsize = SHA512_DIGEST_SIZE, 3928 .halg.base = { 3929 .cra_name = "hmac(sha512)", 3930 .cra_driver_name = "hmac-sha512-chcr", 3931 .cra_blocksize = SHA512_BLOCK_SIZE, 3932 } 3933 } 3934 }, 3935 /* Add AEAD Algorithms */ 3936 { 3937 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 3938 .is_registered = 0, 3939 .alg.aead = { 3940 .base = { 3941 .cra_name = "gcm(aes)", 3942 .cra_driver_name = "gcm-aes-chcr", 3943 .cra_blocksize = 1, 3944 .cra_priority = CHCR_AEAD_PRIORITY, 3945 .cra_ctxsize = sizeof(struct chcr_context) + 3946 sizeof(struct chcr_aead_ctx) + 3947 sizeof(struct chcr_gcm_ctx), 3948 }, 3949 .ivsize = GCM_AES_IV_SIZE, 3950 .maxauthsize = GHASH_DIGEST_SIZE, 3951 .setkey = chcr_gcm_setkey, 3952 .setauthsize = chcr_gcm_setauthsize, 3953 } 3954 }, 3955 { 3956 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 3957 .is_registered = 0, 3958 .alg.aead = { 3959 .base = { 3960 .cra_name = "rfc4106(gcm(aes))", 3961 .cra_driver_name = "rfc4106-gcm-aes-chcr", 3962 .cra_blocksize = 1, 3963 .cra_priority = CHCR_AEAD_PRIORITY + 1, 3964 .cra_ctxsize = sizeof(struct chcr_context) + 3965 sizeof(struct chcr_aead_ctx) + 3966 sizeof(struct chcr_gcm_ctx), 3967 3968 }, 3969 .ivsize = GCM_RFC4106_IV_SIZE, 3970 .maxauthsize = GHASH_DIGEST_SIZE, 3971 .setkey = chcr_gcm_setkey, 3972 .setauthsize = chcr_4106_4309_setauthsize, 3973 } 3974 }, 3975 { 3976 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 3977 .is_registered = 0, 3978 .alg.aead = { 3979 .base = { 3980 .cra_name = "ccm(aes)", 3981 .cra_driver_name = "ccm-aes-chcr", 3982 .cra_blocksize = 1, 3983 .cra_priority = CHCR_AEAD_PRIORITY, 3984 .cra_ctxsize = sizeof(struct chcr_context) + 3985 sizeof(struct chcr_aead_ctx), 3986 3987 }, 3988 .ivsize = AES_BLOCK_SIZE, 3989 .maxauthsize = GHASH_DIGEST_SIZE, 3990 .setkey = chcr_aead_ccm_setkey, 3991 .setauthsize = chcr_ccm_setauthsize, 3992 } 3993 }, 3994 { 3995 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 3996 .is_registered = 0, 3997 .alg.aead = { 3998 .base = { 3999 .cra_name = "rfc4309(ccm(aes))", 4000 .cra_driver_name = "rfc4309-ccm-aes-chcr", 4001 .cra_blocksize = 1, 4002 .cra_priority = CHCR_AEAD_PRIORITY + 1, 4003 .cra_ctxsize = sizeof(struct chcr_context) + 4004 sizeof(struct chcr_aead_ctx), 4005 4006 }, 4007 .ivsize = 8, 4008 .maxauthsize = GHASH_DIGEST_SIZE, 4009 .setkey = chcr_aead_rfc4309_setkey, 4010 .setauthsize = chcr_4106_4309_setauthsize, 4011 } 4012 }, 4013 { 4014 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4015 .is_registered = 0, 4016 .alg.aead = { 4017 .base = { 4018 .cra_name = "authenc(hmac(sha1),cbc(aes))", 4019 .cra_driver_name = 4020 "authenc-hmac-sha1-cbc-aes-chcr", 4021 .cra_blocksize = AES_BLOCK_SIZE, 4022 .cra_priority = CHCR_AEAD_PRIORITY, 4023 .cra_ctxsize = sizeof(struct chcr_context) + 4024 sizeof(struct chcr_aead_ctx) + 4025 sizeof(struct chcr_authenc_ctx), 4026 4027 }, 4028 .ivsize = AES_BLOCK_SIZE, 4029 .maxauthsize = SHA1_DIGEST_SIZE, 4030 .setkey = chcr_authenc_setkey, 4031 .setauthsize = chcr_authenc_setauthsize, 4032 } 4033 }, 4034 { 4035 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4036 .is_registered = 0, 4037 .alg.aead = { 4038 .base = { 4039 4040 .cra_name = "authenc(hmac(sha256),cbc(aes))", 4041 .cra_driver_name = 4042 "authenc-hmac-sha256-cbc-aes-chcr", 4043 .cra_blocksize = AES_BLOCK_SIZE, 4044 .cra_priority = CHCR_AEAD_PRIORITY, 4045 .cra_ctxsize = sizeof(struct chcr_context) + 4046 sizeof(struct chcr_aead_ctx) + 4047 sizeof(struct chcr_authenc_ctx), 4048 4049 }, 4050 .ivsize = AES_BLOCK_SIZE, 4051 .maxauthsize = SHA256_DIGEST_SIZE, 4052 .setkey = chcr_authenc_setkey, 4053 .setauthsize = chcr_authenc_setauthsize, 4054 } 4055 }, 4056 { 4057 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4058 .is_registered = 0, 4059 .alg.aead = { 4060 .base = { 4061 .cra_name = "authenc(hmac(sha224),cbc(aes))", 4062 .cra_driver_name = 4063 "authenc-hmac-sha224-cbc-aes-chcr", 4064 .cra_blocksize = AES_BLOCK_SIZE, 4065 .cra_priority = CHCR_AEAD_PRIORITY, 4066 .cra_ctxsize = sizeof(struct chcr_context) + 4067 sizeof(struct chcr_aead_ctx) + 4068 sizeof(struct chcr_authenc_ctx), 4069 }, 4070 .ivsize = AES_BLOCK_SIZE, 4071 .maxauthsize = SHA224_DIGEST_SIZE, 4072 .setkey = chcr_authenc_setkey, 4073 .setauthsize = chcr_authenc_setauthsize, 4074 } 4075 }, 4076 { 4077 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4078 .is_registered = 0, 4079 .alg.aead = { 4080 .base = { 4081 .cra_name = "authenc(hmac(sha384),cbc(aes))", 4082 .cra_driver_name = 4083 "authenc-hmac-sha384-cbc-aes-chcr", 4084 .cra_blocksize = AES_BLOCK_SIZE, 4085 .cra_priority = CHCR_AEAD_PRIORITY, 4086 .cra_ctxsize = sizeof(struct chcr_context) + 4087 sizeof(struct chcr_aead_ctx) + 4088 sizeof(struct chcr_authenc_ctx), 4089 4090 }, 4091 .ivsize = AES_BLOCK_SIZE, 4092 .maxauthsize = SHA384_DIGEST_SIZE, 4093 .setkey = chcr_authenc_setkey, 4094 .setauthsize = chcr_authenc_setauthsize, 4095 } 4096 }, 4097 { 4098 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4099 .is_registered = 0, 4100 .alg.aead = { 4101 .base = { 4102 .cra_name = "authenc(hmac(sha512),cbc(aes))", 4103 .cra_driver_name = 4104 "authenc-hmac-sha512-cbc-aes-chcr", 4105 .cra_blocksize = AES_BLOCK_SIZE, 4106 .cra_priority = CHCR_AEAD_PRIORITY, 4107 .cra_ctxsize = sizeof(struct chcr_context) + 4108 sizeof(struct chcr_aead_ctx) + 4109 sizeof(struct chcr_authenc_ctx), 4110 4111 }, 4112 .ivsize = AES_BLOCK_SIZE, 4113 .maxauthsize = SHA512_DIGEST_SIZE, 4114 .setkey = chcr_authenc_setkey, 4115 .setauthsize = chcr_authenc_setauthsize, 4116 } 4117 }, 4118 { 4119 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL, 4120 .is_registered = 0, 4121 .alg.aead = { 4122 .base = { 4123 .cra_name = "authenc(digest_null,cbc(aes))", 4124 .cra_driver_name = 4125 "authenc-digest_null-cbc-aes-chcr", 4126 .cra_blocksize = AES_BLOCK_SIZE, 4127 .cra_priority = CHCR_AEAD_PRIORITY, 4128 .cra_ctxsize = sizeof(struct chcr_context) + 4129 sizeof(struct chcr_aead_ctx) + 4130 sizeof(struct chcr_authenc_ctx), 4131 4132 }, 4133 .ivsize = AES_BLOCK_SIZE, 4134 .maxauthsize = 0, 4135 .setkey = chcr_aead_digest_null_setkey, 4136 .setauthsize = chcr_authenc_null_setauthsize, 4137 } 4138 }, 4139 { 4140 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4141 .is_registered = 0, 4142 .alg.aead = { 4143 .base = { 4144 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 4145 .cra_driver_name = 4146 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr", 4147 .cra_blocksize = 1, 4148 .cra_priority = CHCR_AEAD_PRIORITY, 4149 .cra_ctxsize = sizeof(struct chcr_context) + 4150 sizeof(struct chcr_aead_ctx) + 4151 sizeof(struct chcr_authenc_ctx), 4152 4153 }, 4154 .ivsize = CTR_RFC3686_IV_SIZE, 4155 .maxauthsize = SHA1_DIGEST_SIZE, 4156 .setkey = chcr_authenc_setkey, 4157 .setauthsize = chcr_authenc_setauthsize, 4158 } 4159 }, 4160 { 4161 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4162 .is_registered = 0, 4163 .alg.aead = { 4164 .base = { 4165 4166 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 4167 .cra_driver_name = 4168 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr", 4169 .cra_blocksize = 1, 4170 .cra_priority = CHCR_AEAD_PRIORITY, 4171 .cra_ctxsize = sizeof(struct chcr_context) + 4172 sizeof(struct chcr_aead_ctx) + 4173 sizeof(struct chcr_authenc_ctx), 4174 4175 }, 4176 .ivsize = CTR_RFC3686_IV_SIZE, 4177 .maxauthsize = SHA256_DIGEST_SIZE, 4178 .setkey = chcr_authenc_setkey, 4179 .setauthsize = chcr_authenc_setauthsize, 4180 } 4181 }, 4182 { 4183 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4184 .is_registered = 0, 4185 .alg.aead = { 4186 .base = { 4187 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 4188 .cra_driver_name = 4189 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr", 4190 .cra_blocksize = 1, 4191 .cra_priority = CHCR_AEAD_PRIORITY, 4192 .cra_ctxsize = sizeof(struct chcr_context) + 4193 sizeof(struct chcr_aead_ctx) + 4194 sizeof(struct chcr_authenc_ctx), 4195 }, 4196 .ivsize = CTR_RFC3686_IV_SIZE, 4197 .maxauthsize = SHA224_DIGEST_SIZE, 4198 .setkey = chcr_authenc_setkey, 4199 .setauthsize = chcr_authenc_setauthsize, 4200 } 4201 }, 4202 { 4203 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4204 .is_registered = 0, 4205 .alg.aead = { 4206 .base = { 4207 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 4208 .cra_driver_name = 4209 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr", 4210 .cra_blocksize = 1, 4211 .cra_priority = CHCR_AEAD_PRIORITY, 4212 .cra_ctxsize = sizeof(struct chcr_context) + 4213 sizeof(struct chcr_aead_ctx) + 4214 sizeof(struct chcr_authenc_ctx), 4215 4216 }, 4217 .ivsize = CTR_RFC3686_IV_SIZE, 4218 .maxauthsize = SHA384_DIGEST_SIZE, 4219 .setkey = chcr_authenc_setkey, 4220 .setauthsize = chcr_authenc_setauthsize, 4221 } 4222 }, 4223 { 4224 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4225 .is_registered = 0, 4226 .alg.aead = { 4227 .base = { 4228 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 4229 .cra_driver_name = 4230 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr", 4231 .cra_blocksize = 1, 4232 .cra_priority = CHCR_AEAD_PRIORITY, 4233 .cra_ctxsize = sizeof(struct chcr_context) + 4234 sizeof(struct chcr_aead_ctx) + 4235 sizeof(struct chcr_authenc_ctx), 4236 4237 }, 4238 .ivsize = CTR_RFC3686_IV_SIZE, 4239 .maxauthsize = SHA512_DIGEST_SIZE, 4240 .setkey = chcr_authenc_setkey, 4241 .setauthsize = chcr_authenc_setauthsize, 4242 } 4243 }, 4244 { 4245 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL, 4246 .is_registered = 0, 4247 .alg.aead = { 4248 .base = { 4249 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))", 4250 .cra_driver_name = 4251 "authenc-digest_null-rfc3686-ctr-aes-chcr", 4252 .cra_blocksize = 1, 4253 .cra_priority = CHCR_AEAD_PRIORITY, 4254 .cra_ctxsize = sizeof(struct chcr_context) + 4255 sizeof(struct chcr_aead_ctx) + 4256 sizeof(struct chcr_authenc_ctx), 4257 4258 }, 4259 .ivsize = CTR_RFC3686_IV_SIZE, 4260 .maxauthsize = 0, 4261 .setkey = chcr_aead_digest_null_setkey, 4262 .setauthsize = chcr_authenc_null_setauthsize, 4263 } 4264 }, 4265 }; 4266 4267 /* 4268 * chcr_unregister_alg - Deregister crypto algorithms with 4269 * kernel framework. 4270 */ 4271 static int chcr_unregister_alg(void) 4272 { 4273 int i; 4274 4275 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4276 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4277 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4278 if (driver_algs[i].is_registered) 4279 crypto_unregister_alg( 4280 &driver_algs[i].alg.crypto); 4281 break; 4282 case CRYPTO_ALG_TYPE_AEAD: 4283 if (driver_algs[i].is_registered) 4284 crypto_unregister_aead( 4285 &driver_algs[i].alg.aead); 4286 break; 4287 case CRYPTO_ALG_TYPE_AHASH: 4288 if (driver_algs[i].is_registered) 4289 crypto_unregister_ahash( 4290 &driver_algs[i].alg.hash); 4291 break; 4292 } 4293 driver_algs[i].is_registered = 0; 4294 } 4295 return 0; 4296 } 4297 4298 #define SZ_AHASH_CTX sizeof(struct chcr_context) 4299 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 4300 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 4301 4302 /* 4303 * chcr_register_alg - Register crypto algorithms with kernel framework. 4304 */ 4305 static int chcr_register_alg(void) 4306 { 4307 struct crypto_alg ai; 4308 struct ahash_alg *a_hash; 4309 int err = 0, i; 4310 char *name = NULL; 4311 4312 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4313 if (driver_algs[i].is_registered) 4314 continue; 4315 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4316 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4317 driver_algs[i].alg.crypto.cra_priority = 4318 CHCR_CRA_PRIORITY; 4319 driver_algs[i].alg.crypto.cra_module = THIS_MODULE; 4320 driver_algs[i].alg.crypto.cra_flags = 4321 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | 4322 CRYPTO_ALG_NEED_FALLBACK; 4323 driver_algs[i].alg.crypto.cra_ctxsize = 4324 sizeof(struct chcr_context) + 4325 sizeof(struct ablk_ctx); 4326 driver_algs[i].alg.crypto.cra_alignmask = 0; 4327 driver_algs[i].alg.crypto.cra_type = 4328 &crypto_ablkcipher_type; 4329 err = crypto_register_alg(&driver_algs[i].alg.crypto); 4330 name = driver_algs[i].alg.crypto.cra_driver_name; 4331 break; 4332 case CRYPTO_ALG_TYPE_AEAD: 4333 driver_algs[i].alg.aead.base.cra_flags = 4334 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK; 4335 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 4336 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 4337 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 4338 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 4339 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 4340 err = crypto_register_aead(&driver_algs[i].alg.aead); 4341 name = driver_algs[i].alg.aead.base.cra_driver_name; 4342 break; 4343 case CRYPTO_ALG_TYPE_AHASH: 4344 a_hash = &driver_algs[i].alg.hash; 4345 a_hash->update = chcr_ahash_update; 4346 a_hash->final = chcr_ahash_final; 4347 a_hash->finup = chcr_ahash_finup; 4348 a_hash->digest = chcr_ahash_digest; 4349 a_hash->export = chcr_ahash_export; 4350 a_hash->import = chcr_ahash_import; 4351 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 4352 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 4353 a_hash->halg.base.cra_module = THIS_MODULE; 4354 a_hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC; 4355 a_hash->halg.base.cra_alignmask = 0; 4356 a_hash->halg.base.cra_exit = NULL; 4357 4358 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 4359 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 4360 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 4361 a_hash->init = chcr_hmac_init; 4362 a_hash->setkey = chcr_ahash_setkey; 4363 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 4364 } else { 4365 a_hash->init = chcr_sha_init; 4366 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 4367 a_hash->halg.base.cra_init = chcr_sha_cra_init; 4368 } 4369 err = crypto_register_ahash(&driver_algs[i].alg.hash); 4370 ai = driver_algs[i].alg.hash.halg.base; 4371 name = ai.cra_driver_name; 4372 break; 4373 } 4374 if (err) { 4375 pr_err("chcr : %s : Algorithm registration failed\n", 4376 name); 4377 goto register_err; 4378 } else { 4379 driver_algs[i].is_registered = 1; 4380 } 4381 } 4382 return 0; 4383 4384 register_err: 4385 chcr_unregister_alg(); 4386 return err; 4387 } 4388 4389 /* 4390 * start_crypto - Register the crypto algorithms. 4391 * This should called once when the first device comesup. After this 4392 * kernel will start calling driver APIs for crypto operations. 4393 */ 4394 int start_crypto(void) 4395 { 4396 return chcr_register_alg(); 4397 } 4398 4399 /* 4400 * stop_crypto - Deregister all the crypto algorithms with kernel. 4401 * This should be called once when the last device goes down. After this 4402 * kernel will not call the driver API for crypto operations. 4403 */ 4404 int stop_crypto(void) 4405 { 4406 chcr_unregister_alg(); 4407 return 0; 4408 } 4409