1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/skbuff.h> 48 #include <linux/rtnetlink.h> 49 #include <linux/highmem.h> 50 #include <linux/scatterlist.h> 51 52 #include <crypto/aes.h> 53 #include <crypto/algapi.h> 54 #include <crypto/hash.h> 55 #include <crypto/gcm.h> 56 #include <crypto/sha1.h> 57 #include <crypto/sha2.h> 58 #include <crypto/authenc.h> 59 #include <crypto/ctr.h> 60 #include <crypto/gf128mul.h> 61 #include <crypto/internal/aead.h> 62 #include <crypto/null.h> 63 #include <crypto/internal/skcipher.h> 64 #include <crypto/aead.h> 65 #include <crypto/scatterwalk.h> 66 #include <crypto/internal/hash.h> 67 68 #include "t4fw_api.h" 69 #include "t4_msg.h" 70 #include "chcr_core.h" 71 #include "chcr_algo.h" 72 #include "chcr_crypto.h" 73 74 #define IV AES_BLOCK_SIZE 75 76 static unsigned int sgl_ent_len[] = { 77 0, 0, 16, 24, 40, 48, 64, 72, 88, 78 96, 112, 120, 136, 144, 160, 168, 184, 79 192, 208, 216, 232, 240, 256, 264, 280, 80 288, 304, 312, 328, 336, 352, 360, 376 81 }; 82 83 static unsigned int dsgl_ent_len[] = { 84 0, 32, 32, 48, 48, 64, 64, 80, 80, 85 112, 112, 128, 128, 144, 144, 160, 160, 86 192, 192, 208, 208, 224, 224, 240, 240, 87 272, 272, 288, 288, 304, 304, 320, 320 88 }; 89 90 static u32 round_constant[11] = { 91 0x01000000, 0x02000000, 0x04000000, 0x08000000, 92 0x10000000, 0x20000000, 0x40000000, 0x80000000, 93 0x1B000000, 0x36000000, 0x6C000000 94 }; 95 96 static int chcr_handle_cipher_resp(struct skcipher_request *req, 97 unsigned char *input, int err); 98 99 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 100 { 101 return ctx->crypto_ctx->aeadctx; 102 } 103 104 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 105 { 106 return ctx->crypto_ctx->ablkctx; 107 } 108 109 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 110 { 111 return ctx->crypto_ctx->hmacctx; 112 } 113 114 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 115 { 116 return gctx->ctx->gcm; 117 } 118 119 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 120 { 121 return gctx->ctx->authenc; 122 } 123 124 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 125 { 126 return container_of(ctx->dev, struct uld_ctx, dev); 127 } 128 129 static inline int is_ofld_imm(const struct sk_buff *skb) 130 { 131 return (skb->len <= SGE_MAX_WR_LEN); 132 } 133 134 static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx) 135 { 136 memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr)); 137 } 138 139 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, 140 unsigned int entlen, 141 unsigned int skip) 142 { 143 int nents = 0; 144 unsigned int less; 145 unsigned int skip_len = 0; 146 147 while (sg && skip) { 148 if (sg_dma_len(sg) <= skip) { 149 skip -= sg_dma_len(sg); 150 skip_len = 0; 151 sg = sg_next(sg); 152 } else { 153 skip_len = skip; 154 skip = 0; 155 } 156 } 157 158 while (sg && reqlen) { 159 less = min(reqlen, sg_dma_len(sg) - skip_len); 160 nents += DIV_ROUND_UP(less, entlen); 161 reqlen -= less; 162 skip_len = 0; 163 sg = sg_next(sg); 164 } 165 return nents; 166 } 167 168 static inline int get_aead_subtype(struct crypto_aead *aead) 169 { 170 struct aead_alg *alg = crypto_aead_alg(aead); 171 struct chcr_alg_template *chcr_crypto_alg = 172 container_of(alg, struct chcr_alg_template, alg.aead); 173 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 174 } 175 176 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 177 { 178 u8 temp[SHA512_DIGEST_SIZE]; 179 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 180 int authsize = crypto_aead_authsize(tfm); 181 struct cpl_fw6_pld *fw6_pld; 182 int cmp = 0; 183 184 fw6_pld = (struct cpl_fw6_pld *)input; 185 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 186 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 187 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); 188 } else { 189 190 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 191 authsize, req->assoclen + 192 req->cryptlen - authsize); 193 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); 194 } 195 if (cmp) 196 *err = -EBADMSG; 197 else 198 *err = 0; 199 } 200 201 static int chcr_inc_wrcount(struct chcr_dev *dev) 202 { 203 if (dev->state == CHCR_DETACH) 204 return 1; 205 atomic_inc(&dev->inflight); 206 return 0; 207 } 208 209 static inline void chcr_dec_wrcount(struct chcr_dev *dev) 210 { 211 atomic_dec(&dev->inflight); 212 } 213 214 static inline int chcr_handle_aead_resp(struct aead_request *req, 215 unsigned char *input, 216 int err) 217 { 218 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 219 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 220 struct chcr_dev *dev = a_ctx(tfm)->dev; 221 222 chcr_aead_common_exit(req); 223 if (reqctx->verify == VERIFY_SW) { 224 chcr_verify_tag(req, input, &err); 225 reqctx->verify = VERIFY_HW; 226 } 227 chcr_dec_wrcount(dev); 228 req->base.complete(&req->base, err); 229 230 return err; 231 } 232 233 static void get_aes_decrypt_key(unsigned char *dec_key, 234 const unsigned char *key, 235 unsigned int keylength) 236 { 237 u32 temp; 238 u32 w_ring[MAX_NK]; 239 int i, j, k; 240 u8 nr, nk; 241 242 switch (keylength) { 243 case AES_KEYLENGTH_128BIT: 244 nk = KEYLENGTH_4BYTES; 245 nr = NUMBER_OF_ROUNDS_10; 246 break; 247 case AES_KEYLENGTH_192BIT: 248 nk = KEYLENGTH_6BYTES; 249 nr = NUMBER_OF_ROUNDS_12; 250 break; 251 case AES_KEYLENGTH_256BIT: 252 nk = KEYLENGTH_8BYTES; 253 nr = NUMBER_OF_ROUNDS_14; 254 break; 255 default: 256 return; 257 } 258 for (i = 0; i < nk; i++) 259 w_ring[i] = get_unaligned_be32(&key[i * 4]); 260 261 i = 0; 262 temp = w_ring[nk - 1]; 263 while (i + nk < (nr + 1) * 4) { 264 if (!(i % nk)) { 265 /* RotWord(temp) */ 266 temp = (temp << 8) | (temp >> 24); 267 temp = aes_ks_subword(temp); 268 temp ^= round_constant[i / nk]; 269 } else if (nk == 8 && (i % 4 == 0)) { 270 temp = aes_ks_subword(temp); 271 } 272 w_ring[i % nk] ^= temp; 273 temp = w_ring[i % nk]; 274 i++; 275 } 276 i--; 277 for (k = 0, j = i % nk; k < nk; k++) { 278 put_unaligned_be32(w_ring[j], &dec_key[k * 4]); 279 j--; 280 if (j < 0) 281 j += nk; 282 } 283 } 284 285 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 286 { 287 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 288 289 switch (ds) { 290 case SHA1_DIGEST_SIZE: 291 base_hash = crypto_alloc_shash("sha1", 0, 0); 292 break; 293 case SHA224_DIGEST_SIZE: 294 base_hash = crypto_alloc_shash("sha224", 0, 0); 295 break; 296 case SHA256_DIGEST_SIZE: 297 base_hash = crypto_alloc_shash("sha256", 0, 0); 298 break; 299 case SHA384_DIGEST_SIZE: 300 base_hash = crypto_alloc_shash("sha384", 0, 0); 301 break; 302 case SHA512_DIGEST_SIZE: 303 base_hash = crypto_alloc_shash("sha512", 0, 0); 304 break; 305 } 306 307 return base_hash; 308 } 309 310 static int chcr_compute_partial_hash(struct shash_desc *desc, 311 char *iopad, char *result_hash, 312 int digest_size) 313 { 314 struct sha1_state sha1_st; 315 struct sha256_state sha256_st; 316 struct sha512_state sha512_st; 317 int error; 318 319 if (digest_size == SHA1_DIGEST_SIZE) { 320 error = crypto_shash_init(desc) ?: 321 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 322 crypto_shash_export(desc, (void *)&sha1_st); 323 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 324 } else if (digest_size == SHA224_DIGEST_SIZE) { 325 error = crypto_shash_init(desc) ?: 326 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 327 crypto_shash_export(desc, (void *)&sha256_st); 328 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 329 330 } else if (digest_size == SHA256_DIGEST_SIZE) { 331 error = crypto_shash_init(desc) ?: 332 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 333 crypto_shash_export(desc, (void *)&sha256_st); 334 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 335 336 } else if (digest_size == SHA384_DIGEST_SIZE) { 337 error = crypto_shash_init(desc) ?: 338 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 339 crypto_shash_export(desc, (void *)&sha512_st); 340 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 341 342 } else if (digest_size == SHA512_DIGEST_SIZE) { 343 error = crypto_shash_init(desc) ?: 344 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 345 crypto_shash_export(desc, (void *)&sha512_st); 346 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 347 } else { 348 error = -EINVAL; 349 pr_err("Unknown digest size %d\n", digest_size); 350 } 351 return error; 352 } 353 354 static void chcr_change_order(char *buf, int ds) 355 { 356 int i; 357 358 if (ds == SHA512_DIGEST_SIZE) { 359 for (i = 0; i < (ds / sizeof(u64)); i++) 360 *((__be64 *)buf + i) = 361 cpu_to_be64(*((u64 *)buf + i)); 362 } else { 363 for (i = 0; i < (ds / sizeof(u32)); i++) 364 *((__be32 *)buf + i) = 365 cpu_to_be32(*((u32 *)buf + i)); 366 } 367 } 368 369 static inline int is_hmac(struct crypto_tfm *tfm) 370 { 371 struct crypto_alg *alg = tfm->__crt_alg; 372 struct chcr_alg_template *chcr_crypto_alg = 373 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 374 alg.hash); 375 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 376 return 1; 377 return 0; 378 } 379 380 static inline void dsgl_walk_init(struct dsgl_walk *walk, 381 struct cpl_rx_phys_dsgl *dsgl) 382 { 383 walk->dsgl = dsgl; 384 walk->nents = 0; 385 walk->to = (struct phys_sge_pairs *)(dsgl + 1); 386 } 387 388 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid, 389 int pci_chan_id) 390 { 391 struct cpl_rx_phys_dsgl *phys_cpl; 392 393 phys_cpl = walk->dsgl; 394 395 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 396 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 397 phys_cpl->pcirlxorder_to_noofsgentr = 398 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 399 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 400 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 401 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 402 CPL_RX_PHYS_DSGL_DCAID_V(0) | 403 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); 404 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 405 phys_cpl->rss_hdr_int.qid = htons(qid); 406 phys_cpl->rss_hdr_int.hash_val = 0; 407 phys_cpl->rss_hdr_int.channel = pci_chan_id; 408 } 409 410 static inline void dsgl_walk_add_page(struct dsgl_walk *walk, 411 size_t size, 412 dma_addr_t addr) 413 { 414 int j; 415 416 if (!size) 417 return; 418 j = walk->nents; 419 walk->to->len[j % 8] = htons(size); 420 walk->to->addr[j % 8] = cpu_to_be64(addr); 421 j++; 422 if ((j % 8) == 0) 423 walk->to++; 424 walk->nents = j; 425 } 426 427 static void dsgl_walk_add_sg(struct dsgl_walk *walk, 428 struct scatterlist *sg, 429 unsigned int slen, 430 unsigned int skip) 431 { 432 int skip_len = 0; 433 unsigned int left_size = slen, len = 0; 434 unsigned int j = walk->nents; 435 int offset, ent_len; 436 437 if (!slen) 438 return; 439 while (sg && skip) { 440 if (sg_dma_len(sg) <= skip) { 441 skip -= sg_dma_len(sg); 442 skip_len = 0; 443 sg = sg_next(sg); 444 } else { 445 skip_len = skip; 446 skip = 0; 447 } 448 } 449 450 while (left_size && sg) { 451 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); 452 offset = 0; 453 while (len) { 454 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); 455 walk->to->len[j % 8] = htons(ent_len); 456 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + 457 offset + skip_len); 458 offset += ent_len; 459 len -= ent_len; 460 j++; 461 if ((j % 8) == 0) 462 walk->to++; 463 } 464 walk->last_sg = sg; 465 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - 466 skip_len) + skip_len; 467 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); 468 skip_len = 0; 469 sg = sg_next(sg); 470 } 471 walk->nents = j; 472 } 473 474 static inline void ulptx_walk_init(struct ulptx_walk *walk, 475 struct ulptx_sgl *ulp) 476 { 477 walk->sgl = ulp; 478 walk->nents = 0; 479 walk->pair_idx = 0; 480 walk->pair = ulp->sge; 481 walk->last_sg = NULL; 482 walk->last_sg_len = 0; 483 } 484 485 static inline void ulptx_walk_end(struct ulptx_walk *walk) 486 { 487 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | 488 ULPTX_NSGE_V(walk->nents)); 489 } 490 491 492 static inline void ulptx_walk_add_page(struct ulptx_walk *walk, 493 size_t size, 494 dma_addr_t addr) 495 { 496 if (!size) 497 return; 498 499 if (walk->nents == 0) { 500 walk->sgl->len0 = cpu_to_be32(size); 501 walk->sgl->addr0 = cpu_to_be64(addr); 502 } else { 503 walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr); 504 walk->pair->len[walk->pair_idx] = cpu_to_be32(size); 505 walk->pair_idx = !walk->pair_idx; 506 if (!walk->pair_idx) 507 walk->pair++; 508 } 509 walk->nents++; 510 } 511 512 static void ulptx_walk_add_sg(struct ulptx_walk *walk, 513 struct scatterlist *sg, 514 unsigned int len, 515 unsigned int skip) 516 { 517 int small; 518 int skip_len = 0; 519 unsigned int sgmin; 520 521 if (!len) 522 return; 523 while (sg && skip) { 524 if (sg_dma_len(sg) <= skip) { 525 skip -= sg_dma_len(sg); 526 skip_len = 0; 527 sg = sg_next(sg); 528 } else { 529 skip_len = skip; 530 skip = 0; 531 } 532 } 533 WARN(!sg, "SG should not be null here\n"); 534 if (sg && (walk->nents == 0)) { 535 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); 536 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 537 walk->sgl->len0 = cpu_to_be32(sgmin); 538 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); 539 walk->nents++; 540 len -= sgmin; 541 walk->last_sg = sg; 542 walk->last_sg_len = sgmin + skip_len; 543 skip_len += sgmin; 544 if (sg_dma_len(sg) == skip_len) { 545 sg = sg_next(sg); 546 skip_len = 0; 547 } 548 } 549 550 while (sg && len) { 551 small = min(sg_dma_len(sg) - skip_len, len); 552 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 553 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); 554 walk->pair->addr[walk->pair_idx] = 555 cpu_to_be64(sg_dma_address(sg) + skip_len); 556 walk->pair_idx = !walk->pair_idx; 557 walk->nents++; 558 if (!walk->pair_idx) 559 walk->pair++; 560 len -= sgmin; 561 skip_len += sgmin; 562 walk->last_sg = sg; 563 walk->last_sg_len = skip_len; 564 if (sg_dma_len(sg) == skip_len) { 565 sg = sg_next(sg); 566 skip_len = 0; 567 } 568 } 569 } 570 571 static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm) 572 { 573 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 574 struct chcr_alg_template *chcr_crypto_alg = 575 container_of(alg, struct chcr_alg_template, alg.skcipher); 576 577 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 578 } 579 580 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 581 { 582 struct adapter *adap = netdev2adap(dev); 583 struct sge_uld_txq_info *txq_info = 584 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 585 struct sge_uld_txq *txq; 586 int ret = 0; 587 588 local_bh_disable(); 589 txq = &txq_info->uldtxq[idx]; 590 spin_lock(&txq->sendq.lock); 591 if (txq->full) 592 ret = -1; 593 spin_unlock(&txq->sendq.lock); 594 local_bh_enable(); 595 return ret; 596 } 597 598 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 599 struct _key_ctx *key_ctx) 600 { 601 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 602 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 603 } else { 604 memcpy(key_ctx->key, 605 ablkctx->key + (ablkctx->enckey_len >> 1), 606 ablkctx->enckey_len >> 1); 607 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 608 ablkctx->rrkey, ablkctx->enckey_len >> 1); 609 } 610 return 0; 611 } 612 613 static int chcr_hash_ent_in_wr(struct scatterlist *src, 614 unsigned int minsg, 615 unsigned int space, 616 unsigned int srcskip) 617 { 618 int srclen = 0; 619 int srcsg = minsg; 620 int soffset = 0, sless; 621 622 if (sg_dma_len(src) == srcskip) { 623 src = sg_next(src); 624 srcskip = 0; 625 } 626 while (src && space > (sgl_ent_len[srcsg + 1])) { 627 sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip, 628 CHCR_SRC_SG_SIZE); 629 srclen += sless; 630 soffset += sless; 631 srcsg++; 632 if (sg_dma_len(src) == (soffset + srcskip)) { 633 src = sg_next(src); 634 soffset = 0; 635 srcskip = 0; 636 } 637 } 638 return srclen; 639 } 640 641 static int chcr_sg_ent_in_wr(struct scatterlist *src, 642 struct scatterlist *dst, 643 unsigned int minsg, 644 unsigned int space, 645 unsigned int srcskip, 646 unsigned int dstskip) 647 { 648 int srclen = 0, dstlen = 0; 649 int srcsg = minsg, dstsg = minsg; 650 int offset = 0, soffset = 0, less, sless = 0; 651 652 if (sg_dma_len(src) == srcskip) { 653 src = sg_next(src); 654 srcskip = 0; 655 } 656 if (sg_dma_len(dst) == dstskip) { 657 dst = sg_next(dst); 658 dstskip = 0; 659 } 660 661 while (src && dst && 662 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { 663 sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset, 664 CHCR_SRC_SG_SIZE); 665 srclen += sless; 666 srcsg++; 667 offset = 0; 668 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && 669 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { 670 if (srclen <= dstlen) 671 break; 672 less = min_t(unsigned int, sg_dma_len(dst) - offset - 673 dstskip, CHCR_DST_SG_SIZE); 674 dstlen += less; 675 offset += less; 676 if ((offset + dstskip) == sg_dma_len(dst)) { 677 dst = sg_next(dst); 678 offset = 0; 679 } 680 dstsg++; 681 dstskip = 0; 682 } 683 soffset += sless; 684 if ((soffset + srcskip) == sg_dma_len(src)) { 685 src = sg_next(src); 686 srcskip = 0; 687 soffset = 0; 688 } 689 690 } 691 return min(srclen, dstlen); 692 } 693 694 static int chcr_cipher_fallback(struct crypto_skcipher *cipher, 695 struct skcipher_request *req, 696 u8 *iv, 697 unsigned short op_type) 698 { 699 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 700 int err; 701 702 skcipher_request_set_tfm(&reqctx->fallback_req, cipher); 703 skcipher_request_set_callback(&reqctx->fallback_req, req->base.flags, 704 req->base.complete, req->base.data); 705 skcipher_request_set_crypt(&reqctx->fallback_req, req->src, req->dst, 706 req->cryptlen, iv); 707 708 err = op_type ? crypto_skcipher_decrypt(&reqctx->fallback_req) : 709 crypto_skcipher_encrypt(&reqctx->fallback_req); 710 711 return err; 712 713 } 714 715 static inline int get_qidxs(struct crypto_async_request *req, 716 unsigned int *txqidx, unsigned int *rxqidx) 717 { 718 struct crypto_tfm *tfm = req->tfm; 719 int ret = 0; 720 721 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 722 case CRYPTO_ALG_TYPE_AEAD: 723 { 724 struct aead_request *aead_req = 725 container_of(req, struct aead_request, base); 726 struct chcr_aead_reqctx *reqctx = aead_request_ctx(aead_req); 727 *txqidx = reqctx->txqidx; 728 *rxqidx = reqctx->rxqidx; 729 break; 730 } 731 case CRYPTO_ALG_TYPE_SKCIPHER: 732 { 733 struct skcipher_request *sk_req = 734 container_of(req, struct skcipher_request, base); 735 struct chcr_skcipher_req_ctx *reqctx = 736 skcipher_request_ctx(sk_req); 737 *txqidx = reqctx->txqidx; 738 *rxqidx = reqctx->rxqidx; 739 break; 740 } 741 case CRYPTO_ALG_TYPE_AHASH: 742 { 743 struct ahash_request *ahash_req = 744 container_of(req, struct ahash_request, base); 745 struct chcr_ahash_req_ctx *reqctx = 746 ahash_request_ctx(ahash_req); 747 *txqidx = reqctx->txqidx; 748 *rxqidx = reqctx->rxqidx; 749 break; 750 } 751 default: 752 ret = -EINVAL; 753 /* should never get here */ 754 BUG(); 755 break; 756 } 757 return ret; 758 } 759 760 static inline void create_wreq(struct chcr_context *ctx, 761 struct chcr_wr *chcr_req, 762 struct crypto_async_request *req, 763 unsigned int imm, 764 int hash_sz, 765 unsigned int len16, 766 unsigned int sc_len, 767 unsigned int lcb) 768 { 769 struct uld_ctx *u_ctx = ULD_CTX(ctx); 770 unsigned int tx_channel_id, rx_channel_id; 771 unsigned int txqidx = 0, rxqidx = 0; 772 unsigned int qid, fid; 773 774 get_qidxs(req, &txqidx, &rxqidx); 775 qid = u_ctx->lldi.rxq_ids[rxqidx]; 776 fid = u_ctx->lldi.rxq_ids[0]; 777 tx_channel_id = txqidx / ctx->txq_perchan; 778 rx_channel_id = rxqidx / ctx->rxq_perchan; 779 780 781 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; 782 chcr_req->wreq.pld_size_hash_size = 783 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 784 chcr_req->wreq.len16_pkd = 785 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); 786 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 787 chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid, 788 !!lcb, txqidx); 789 790 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid); 791 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - 792 ((sizeof(chcr_req->wreq)) >> 4))); 793 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); 794 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 795 sizeof(chcr_req->key_ctx) + sc_len); 796 } 797 798 /** 799 * create_cipher_wr - form the WR for cipher operations 800 * @req: cipher req. 801 * @ctx: crypto driver context of the request. 802 * @qid: ingress qid where response of this WR should be received. 803 * @op_type: encryption or decryption 804 */ 805 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) 806 { 807 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); 808 struct chcr_context *ctx = c_ctx(tfm); 809 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 810 struct sk_buff *skb = NULL; 811 struct chcr_wr *chcr_req; 812 struct cpl_rx_phys_dsgl *phys_cpl; 813 struct ulptx_sgl *ulptx; 814 struct chcr_skcipher_req_ctx *reqctx = 815 skcipher_request_ctx(wrparam->req); 816 unsigned int temp = 0, transhdr_len, dst_size; 817 int error; 818 int nents; 819 unsigned int kctx_len; 820 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 821 GFP_KERNEL : GFP_ATOMIC; 822 struct adapter *adap = padap(ctx->dev); 823 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 824 825 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, 826 reqctx->dst_ofst); 827 dst_size = get_space_for_phys_dsgl(nents); 828 kctx_len = roundup(ablkctx->enckey_len, 16); 829 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 830 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, 831 CHCR_SRC_SG_SIZE, reqctx->src_ofst); 832 temp = reqctx->imm ? roundup(wrparam->bytes, 16) : 833 (sgl_len(nents) * 8); 834 transhdr_len += temp; 835 transhdr_len = roundup(transhdr_len, 16); 836 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 837 if (!skb) { 838 error = -ENOMEM; 839 goto err; 840 } 841 chcr_req = __skb_put_zero(skb, transhdr_len); 842 chcr_req->sec_cpl.op_ivinsrtofst = 843 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); 844 845 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); 846 chcr_req->sec_cpl.aadstart_cipherstop_hi = 847 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); 848 849 chcr_req->sec_cpl.cipherstop_lo_authinsert = 850 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 851 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, 852 ablkctx->ciph_mode, 853 0, 0, IV >> 1); 854 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 855 0, 1, dst_size); 856 857 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 858 if ((reqctx->op == CHCR_DECRYPT_OP) && 859 (!(get_cryptoalg_subtype(tfm) == 860 CRYPTO_ALG_SUB_TYPE_CTR)) && 861 (!(get_cryptoalg_subtype(tfm) == 862 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { 863 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 864 } else { 865 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || 866 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { 867 memcpy(chcr_req->key_ctx.key, ablkctx->key, 868 ablkctx->enckey_len); 869 } else { 870 memcpy(chcr_req->key_ctx.key, ablkctx->key + 871 (ablkctx->enckey_len >> 1), 872 ablkctx->enckey_len >> 1); 873 memcpy(chcr_req->key_ctx.key + 874 (ablkctx->enckey_len >> 1), 875 ablkctx->key, 876 ablkctx->enckey_len >> 1); 877 } 878 } 879 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 880 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 881 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); 882 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); 883 884 atomic_inc(&adap->chcr_stats.cipher_rqst); 885 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV 886 + (reqctx->imm ? (wrparam->bytes) : 0); 887 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, 888 transhdr_len, temp, 889 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); 890 reqctx->skb = skb; 891 892 if (reqctx->op && (ablkctx->ciph_mode == 893 CHCR_SCMD_CIPHER_MODE_AES_CBC)) 894 sg_pcopy_to_buffer(wrparam->req->src, 895 sg_nents(wrparam->req->src), wrparam->req->iv, 16, 896 reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE); 897 898 return skb; 899 err: 900 return ERR_PTR(error); 901 } 902 903 static inline int chcr_keyctx_ck_size(unsigned int keylen) 904 { 905 int ck_size = 0; 906 907 if (keylen == AES_KEYSIZE_128) 908 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 909 else if (keylen == AES_KEYSIZE_192) 910 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 911 else if (keylen == AES_KEYSIZE_256) 912 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 913 else 914 ck_size = 0; 915 916 return ck_size; 917 } 918 static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher, 919 const u8 *key, 920 unsigned int keylen) 921 { 922 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 923 924 crypto_skcipher_clear_flags(ablkctx->sw_cipher, 925 CRYPTO_TFM_REQ_MASK); 926 crypto_skcipher_set_flags(ablkctx->sw_cipher, 927 cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK); 928 return crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen); 929 } 930 931 static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher, 932 const u8 *key, 933 unsigned int keylen) 934 { 935 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 936 unsigned int ck_size, context_size; 937 u16 alignment = 0; 938 int err; 939 940 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 941 if (err) 942 goto badkey_err; 943 944 ck_size = chcr_keyctx_ck_size(keylen); 945 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; 946 memcpy(ablkctx->key, key, keylen); 947 ablkctx->enckey_len = keylen; 948 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 949 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 950 keylen + alignment) >> 4; 951 952 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 953 0, 0, context_size); 954 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 955 return 0; 956 badkey_err: 957 ablkctx->enckey_len = 0; 958 959 return err; 960 } 961 962 static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher, 963 const u8 *key, 964 unsigned int keylen) 965 { 966 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 967 unsigned int ck_size, context_size; 968 u16 alignment = 0; 969 int err; 970 971 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 972 if (err) 973 goto badkey_err; 974 ck_size = chcr_keyctx_ck_size(keylen); 975 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 976 memcpy(ablkctx->key, key, keylen); 977 ablkctx->enckey_len = keylen; 978 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 979 keylen + alignment) >> 4; 980 981 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 982 0, 0, context_size); 983 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 984 985 return 0; 986 badkey_err: 987 ablkctx->enckey_len = 0; 988 989 return err; 990 } 991 992 static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher, 993 const u8 *key, 994 unsigned int keylen) 995 { 996 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 997 unsigned int ck_size, context_size; 998 u16 alignment = 0; 999 int err; 1000 1001 if (keylen < CTR_RFC3686_NONCE_SIZE) 1002 return -EINVAL; 1003 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), 1004 CTR_RFC3686_NONCE_SIZE); 1005 1006 keylen -= CTR_RFC3686_NONCE_SIZE; 1007 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 1008 if (err) 1009 goto badkey_err; 1010 1011 ck_size = chcr_keyctx_ck_size(keylen); 1012 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 1013 memcpy(ablkctx->key, key, keylen); 1014 ablkctx->enckey_len = keylen; 1015 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 1016 keylen + alignment) >> 4; 1017 1018 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 1019 0, 0, context_size); 1020 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 1021 1022 return 0; 1023 badkey_err: 1024 ablkctx->enckey_len = 0; 1025 1026 return err; 1027 } 1028 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) 1029 { 1030 unsigned int size = AES_BLOCK_SIZE; 1031 __be32 *b = (__be32 *)(dstiv + size); 1032 u32 c, prev; 1033 1034 memcpy(dstiv, srciv, AES_BLOCK_SIZE); 1035 for (; size >= 4; size -= 4) { 1036 prev = be32_to_cpu(*--b); 1037 c = prev + add; 1038 *b = cpu_to_be32(c); 1039 if (prev < c) 1040 break; 1041 add = 1; 1042 } 1043 1044 } 1045 1046 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) 1047 { 1048 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); 1049 u64 c; 1050 u32 temp = be32_to_cpu(*--b); 1051 1052 temp = ~temp; 1053 c = (u64)temp + 1; // No of block can processed without overflow 1054 if ((bytes / AES_BLOCK_SIZE) >= c) 1055 bytes = c * AES_BLOCK_SIZE; 1056 return bytes; 1057 } 1058 1059 static int chcr_update_tweak(struct skcipher_request *req, u8 *iv, 1060 u32 isfinal) 1061 { 1062 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1063 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1064 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1065 struct crypto_aes_ctx aes; 1066 int ret, i; 1067 u8 *key; 1068 unsigned int keylen; 1069 int round = reqctx->last_req_len / AES_BLOCK_SIZE; 1070 int round8 = round / 8; 1071 1072 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); 1073 1074 keylen = ablkctx->enckey_len / 2; 1075 key = ablkctx->key + keylen; 1076 /* For a 192 bit key remove the padded zeroes which was 1077 * added in chcr_xts_setkey 1078 */ 1079 if (KEY_CONTEXT_CK_SIZE_G(ntohl(ablkctx->key_ctx_hdr)) 1080 == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) 1081 ret = aes_expandkey(&aes, key, keylen - 8); 1082 else 1083 ret = aes_expandkey(&aes, key, keylen); 1084 if (ret) 1085 return ret; 1086 aes_encrypt(&aes, iv, iv); 1087 for (i = 0; i < round8; i++) 1088 gf128mul_x8_ble((le128 *)iv, (le128 *)iv); 1089 1090 for (i = 0; i < (round % 8); i++) 1091 gf128mul_x_ble((le128 *)iv, (le128 *)iv); 1092 1093 if (!isfinal) 1094 aes_decrypt(&aes, iv, iv); 1095 1096 memzero_explicit(&aes, sizeof(aes)); 1097 return 0; 1098 } 1099 1100 static int chcr_update_cipher_iv(struct skcipher_request *req, 1101 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1102 { 1103 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1104 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1105 int subtype = get_cryptoalg_subtype(tfm); 1106 int ret = 0; 1107 1108 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1109 ctr_add_iv(iv, req->iv, (reqctx->processed / 1110 AES_BLOCK_SIZE)); 1111 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) 1112 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1113 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / 1114 AES_BLOCK_SIZE) + 1); 1115 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1116 ret = chcr_update_tweak(req, iv, 0); 1117 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1118 if (reqctx->op) 1119 /*Updated before sending last WR*/ 1120 memcpy(iv, req->iv, AES_BLOCK_SIZE); 1121 else 1122 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1123 } 1124 1125 return ret; 1126 1127 } 1128 1129 /* We need separate function for final iv because in rfc3686 Initial counter 1130 * starts from 1 and buffer size of iv is 8 byte only which remains constant 1131 * for subsequent update requests 1132 */ 1133 1134 static int chcr_final_cipher_iv(struct skcipher_request *req, 1135 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1136 { 1137 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1138 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1139 int subtype = get_cryptoalg_subtype(tfm); 1140 int ret = 0; 1141 1142 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1143 ctr_add_iv(iv, req->iv, DIV_ROUND_UP(reqctx->processed, 1144 AES_BLOCK_SIZE)); 1145 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) { 1146 if (!reqctx->partial_req) 1147 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); 1148 else 1149 ret = chcr_update_tweak(req, iv, 1); 1150 } 1151 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1152 /*Already updated for Decrypt*/ 1153 if (!reqctx->op) 1154 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1155 1156 } 1157 return ret; 1158 1159 } 1160 1161 static int chcr_handle_cipher_resp(struct skcipher_request *req, 1162 unsigned char *input, int err) 1163 { 1164 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1165 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1166 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; 1167 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1168 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1169 struct chcr_dev *dev = c_ctx(tfm)->dev; 1170 struct chcr_context *ctx = c_ctx(tfm); 1171 struct adapter *adap = padap(ctx->dev); 1172 struct cipher_wr_param wrparam; 1173 struct sk_buff *skb; 1174 int bytes; 1175 1176 if (err) 1177 goto unmap; 1178 if (req->cryptlen == reqctx->processed) { 1179 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1180 req); 1181 err = chcr_final_cipher_iv(req, fw6_pld, req->iv); 1182 goto complete; 1183 } 1184 1185 if (!reqctx->imm) { 1186 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0, 1187 CIP_SPACE_LEFT(ablkctx->enckey_len), 1188 reqctx->src_ofst, reqctx->dst_ofst); 1189 if ((bytes + reqctx->processed) >= req->cryptlen) 1190 bytes = req->cryptlen - reqctx->processed; 1191 else 1192 bytes = rounddown(bytes, 16); 1193 } else { 1194 /*CTR mode counter overfloa*/ 1195 bytes = req->cryptlen - reqctx->processed; 1196 } 1197 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); 1198 if (err) 1199 goto unmap; 1200 1201 if (unlikely(bytes == 0)) { 1202 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1203 req); 1204 memcpy(req->iv, reqctx->init_iv, IV); 1205 atomic_inc(&adap->chcr_stats.fallback); 1206 err = chcr_cipher_fallback(ablkctx->sw_cipher, req, req->iv, 1207 reqctx->op); 1208 goto complete; 1209 } 1210 1211 if (get_cryptoalg_subtype(tfm) == 1212 CRYPTO_ALG_SUB_TYPE_CTR) 1213 bytes = adjust_ctr_overflow(reqctx->iv, bytes); 1214 wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx]; 1215 wrparam.req = req; 1216 wrparam.bytes = bytes; 1217 skb = create_cipher_wr(&wrparam); 1218 if (IS_ERR(skb)) { 1219 pr_err("%s : Failed to form WR. No memory\n", __func__); 1220 err = PTR_ERR(skb); 1221 goto unmap; 1222 } 1223 skb->dev = u_ctx->lldi.ports[0]; 1224 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 1225 chcr_send_wr(skb); 1226 reqctx->last_req_len = bytes; 1227 reqctx->processed += bytes; 1228 if (get_cryptoalg_subtype(tfm) == 1229 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == 1230 CRYPTO_TFM_REQ_MAY_SLEEP ) { 1231 complete(&ctx->cbc_aes_aio_done); 1232 } 1233 return 0; 1234 unmap: 1235 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1236 complete: 1237 if (get_cryptoalg_subtype(tfm) == 1238 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == 1239 CRYPTO_TFM_REQ_MAY_SLEEP ) { 1240 complete(&ctx->cbc_aes_aio_done); 1241 } 1242 chcr_dec_wrcount(dev); 1243 req->base.complete(&req->base, err); 1244 return err; 1245 } 1246 1247 static int process_cipher(struct skcipher_request *req, 1248 unsigned short qid, 1249 struct sk_buff **skb, 1250 unsigned short op_type) 1251 { 1252 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1253 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1254 unsigned int ivsize = crypto_skcipher_ivsize(tfm); 1255 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1256 struct adapter *adap = padap(c_ctx(tfm)->dev); 1257 struct cipher_wr_param wrparam; 1258 int bytes, err = -EINVAL; 1259 int subtype; 1260 1261 reqctx->processed = 0; 1262 reqctx->partial_req = 0; 1263 if (!req->iv) 1264 goto error; 1265 subtype = get_cryptoalg_subtype(tfm); 1266 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 1267 (req->cryptlen == 0) || 1268 (req->cryptlen % crypto_skcipher_blocksize(tfm))) { 1269 if (req->cryptlen == 0 && subtype != CRYPTO_ALG_SUB_TYPE_XTS) 1270 goto fallback; 1271 else if (req->cryptlen % crypto_skcipher_blocksize(tfm) && 1272 subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1273 goto fallback; 1274 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 1275 ablkctx->enckey_len, req->cryptlen, ivsize); 1276 goto error; 1277 } 1278 1279 err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1280 if (err) 1281 goto error; 1282 if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + 1283 AES_MIN_KEY_SIZE + 1284 sizeof(struct cpl_rx_phys_dsgl) + 1285 /*Min dsgl size*/ 1286 32))) { 1287 /* Can be sent as Imm*/ 1288 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; 1289 1290 dnents = sg_nents_xlen(req->dst, req->cryptlen, 1291 CHCR_DST_SG_SIZE, 0); 1292 phys_dsgl = get_space_for_phys_dsgl(dnents); 1293 kctx_len = roundup(ablkctx->enckey_len, 16); 1294 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 1295 reqctx->imm = (transhdr_len + IV + req->cryptlen) <= 1296 SGE_MAX_WR_LEN; 1297 bytes = IV + req->cryptlen; 1298 1299 } else { 1300 reqctx->imm = 0; 1301 } 1302 1303 if (!reqctx->imm) { 1304 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0, 1305 CIP_SPACE_LEFT(ablkctx->enckey_len), 1306 0, 0); 1307 if ((bytes + reqctx->processed) >= req->cryptlen) 1308 bytes = req->cryptlen - reqctx->processed; 1309 else 1310 bytes = rounddown(bytes, 16); 1311 } else { 1312 bytes = req->cryptlen; 1313 } 1314 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) { 1315 bytes = adjust_ctr_overflow(req->iv, bytes); 1316 } 1317 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { 1318 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); 1319 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv, 1320 CTR_RFC3686_IV_SIZE); 1321 1322 /* initialize counter portion of counter block */ 1323 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1324 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1325 memcpy(reqctx->init_iv, reqctx->iv, IV); 1326 1327 } else { 1328 1329 memcpy(reqctx->iv, req->iv, IV); 1330 memcpy(reqctx->init_iv, req->iv, IV); 1331 } 1332 if (unlikely(bytes == 0)) { 1333 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1334 req); 1335 fallback: atomic_inc(&adap->chcr_stats.fallback); 1336 err = chcr_cipher_fallback(ablkctx->sw_cipher, req, 1337 subtype == 1338 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 ? 1339 reqctx->iv : req->iv, 1340 op_type); 1341 goto error; 1342 } 1343 reqctx->op = op_type; 1344 reqctx->srcsg = req->src; 1345 reqctx->dstsg = req->dst; 1346 reqctx->src_ofst = 0; 1347 reqctx->dst_ofst = 0; 1348 wrparam.qid = qid; 1349 wrparam.req = req; 1350 wrparam.bytes = bytes; 1351 *skb = create_cipher_wr(&wrparam); 1352 if (IS_ERR(*skb)) { 1353 err = PTR_ERR(*skb); 1354 goto unmap; 1355 } 1356 reqctx->processed = bytes; 1357 reqctx->last_req_len = bytes; 1358 reqctx->partial_req = !!(req->cryptlen - reqctx->processed); 1359 1360 return 0; 1361 unmap: 1362 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1363 error: 1364 return err; 1365 } 1366 1367 static int chcr_aes_encrypt(struct skcipher_request *req) 1368 { 1369 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1370 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1371 struct chcr_dev *dev = c_ctx(tfm)->dev; 1372 struct sk_buff *skb = NULL; 1373 int err; 1374 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1375 struct chcr_context *ctx = c_ctx(tfm); 1376 unsigned int cpu; 1377 1378 cpu = get_cpu(); 1379 reqctx->txqidx = cpu % ctx->ntxq; 1380 reqctx->rxqidx = cpu % ctx->nrxq; 1381 put_cpu(); 1382 1383 err = chcr_inc_wrcount(dev); 1384 if (err) 1385 return -ENXIO; 1386 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1387 reqctx->txqidx) && 1388 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1389 err = -ENOSPC; 1390 goto error; 1391 } 1392 1393 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], 1394 &skb, CHCR_ENCRYPT_OP); 1395 if (err || !skb) 1396 return err; 1397 skb->dev = u_ctx->lldi.ports[0]; 1398 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 1399 chcr_send_wr(skb); 1400 if (get_cryptoalg_subtype(tfm) == 1401 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == 1402 CRYPTO_TFM_REQ_MAY_SLEEP ) { 1403 reqctx->partial_req = 1; 1404 wait_for_completion(&ctx->cbc_aes_aio_done); 1405 } 1406 return -EINPROGRESS; 1407 error: 1408 chcr_dec_wrcount(dev); 1409 return err; 1410 } 1411 1412 static int chcr_aes_decrypt(struct skcipher_request *req) 1413 { 1414 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1415 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1416 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1417 struct chcr_dev *dev = c_ctx(tfm)->dev; 1418 struct sk_buff *skb = NULL; 1419 int err; 1420 struct chcr_context *ctx = c_ctx(tfm); 1421 unsigned int cpu; 1422 1423 cpu = get_cpu(); 1424 reqctx->txqidx = cpu % ctx->ntxq; 1425 reqctx->rxqidx = cpu % ctx->nrxq; 1426 put_cpu(); 1427 1428 err = chcr_inc_wrcount(dev); 1429 if (err) 1430 return -ENXIO; 1431 1432 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1433 reqctx->txqidx) && 1434 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) 1435 return -ENOSPC; 1436 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], 1437 &skb, CHCR_DECRYPT_OP); 1438 if (err || !skb) 1439 return err; 1440 skb->dev = u_ctx->lldi.ports[0]; 1441 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 1442 chcr_send_wr(skb); 1443 return -EINPROGRESS; 1444 } 1445 static int chcr_device_init(struct chcr_context *ctx) 1446 { 1447 struct uld_ctx *u_ctx = NULL; 1448 int txq_perchan, ntxq; 1449 int err = 0, rxq_perchan; 1450 1451 if (!ctx->dev) { 1452 u_ctx = assign_chcr_device(); 1453 if (!u_ctx) { 1454 err = -ENXIO; 1455 pr_err("chcr device assignment fails\n"); 1456 goto out; 1457 } 1458 ctx->dev = &u_ctx->dev; 1459 ntxq = u_ctx->lldi.ntxq; 1460 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 1461 txq_perchan = ntxq / u_ctx->lldi.nchan; 1462 ctx->ntxq = ntxq; 1463 ctx->nrxq = u_ctx->lldi.nrxq; 1464 ctx->rxq_perchan = rxq_perchan; 1465 ctx->txq_perchan = txq_perchan; 1466 } 1467 out: 1468 return err; 1469 } 1470 1471 static int chcr_init_tfm(struct crypto_skcipher *tfm) 1472 { 1473 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1474 struct chcr_context *ctx = crypto_skcipher_ctx(tfm); 1475 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1476 1477 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0, 1478 CRYPTO_ALG_NEED_FALLBACK); 1479 if (IS_ERR(ablkctx->sw_cipher)) { 1480 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name); 1481 return PTR_ERR(ablkctx->sw_cipher); 1482 } 1483 init_completion(&ctx->cbc_aes_aio_done); 1484 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) + 1485 crypto_skcipher_reqsize(ablkctx->sw_cipher)); 1486 1487 return chcr_device_init(ctx); 1488 } 1489 1490 static int chcr_rfc3686_init(struct crypto_skcipher *tfm) 1491 { 1492 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1493 struct chcr_context *ctx = crypto_skcipher_ctx(tfm); 1494 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1495 1496 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) 1497 * cannot be used as fallback in chcr_handle_cipher_response 1498 */ 1499 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0, 1500 CRYPTO_ALG_NEED_FALLBACK); 1501 if (IS_ERR(ablkctx->sw_cipher)) { 1502 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name); 1503 return PTR_ERR(ablkctx->sw_cipher); 1504 } 1505 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) + 1506 crypto_skcipher_reqsize(ablkctx->sw_cipher)); 1507 return chcr_device_init(ctx); 1508 } 1509 1510 1511 static void chcr_exit_tfm(struct crypto_skcipher *tfm) 1512 { 1513 struct chcr_context *ctx = crypto_skcipher_ctx(tfm); 1514 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1515 1516 crypto_free_skcipher(ablkctx->sw_cipher); 1517 } 1518 1519 static int get_alg_config(struct algo_param *params, 1520 unsigned int auth_size) 1521 { 1522 switch (auth_size) { 1523 case SHA1_DIGEST_SIZE: 1524 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 1525 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 1526 params->result_size = SHA1_DIGEST_SIZE; 1527 break; 1528 case SHA224_DIGEST_SIZE: 1529 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1530 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 1531 params->result_size = SHA256_DIGEST_SIZE; 1532 break; 1533 case SHA256_DIGEST_SIZE: 1534 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1535 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 1536 params->result_size = SHA256_DIGEST_SIZE; 1537 break; 1538 case SHA384_DIGEST_SIZE: 1539 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1540 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 1541 params->result_size = SHA512_DIGEST_SIZE; 1542 break; 1543 case SHA512_DIGEST_SIZE: 1544 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1545 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 1546 params->result_size = SHA512_DIGEST_SIZE; 1547 break; 1548 default: 1549 pr_err("ERROR, unsupported digest size\n"); 1550 return -EINVAL; 1551 } 1552 return 0; 1553 } 1554 1555 static inline void chcr_free_shash(struct crypto_shash *base_hash) 1556 { 1557 crypto_free_shash(base_hash); 1558 } 1559 1560 /** 1561 * create_hash_wr - Create hash work request 1562 * @req - Cipher req base 1563 */ 1564 static struct sk_buff *create_hash_wr(struct ahash_request *req, 1565 struct hash_wr_param *param) 1566 { 1567 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1568 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1569 struct chcr_context *ctx = h_ctx(tfm); 1570 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1571 struct sk_buff *skb = NULL; 1572 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1573 struct chcr_wr *chcr_req; 1574 struct ulptx_sgl *ulptx; 1575 unsigned int nents = 0, transhdr_len; 1576 unsigned int temp = 0; 1577 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1578 GFP_ATOMIC; 1579 struct adapter *adap = padap(h_ctx(tfm)->dev); 1580 int error = 0; 1581 unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan; 1582 1583 transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len); 1584 req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len + 1585 param->sg_len) <= SGE_MAX_WR_LEN; 1586 nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len, 1587 CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst); 1588 nents += param->bfr_len ? 1 : 0; 1589 transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len + 1590 param->sg_len, 16) : (sgl_len(nents) * 8); 1591 transhdr_len = roundup(transhdr_len, 16); 1592 1593 skb = alloc_skb(transhdr_len, flags); 1594 if (!skb) 1595 return ERR_PTR(-ENOMEM); 1596 chcr_req = __skb_put_zero(skb, transhdr_len); 1597 1598 chcr_req->sec_cpl.op_ivinsrtofst = 1599 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0); 1600 1601 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 1602 1603 chcr_req->sec_cpl.aadstart_cipherstop_hi = 1604 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 1605 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1606 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 1607 chcr_req->sec_cpl.seqno_numivs = 1608 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 1609 param->opad_needed, 0); 1610 1611 chcr_req->sec_cpl.ivgen_hdrlen = 1612 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 1613 1614 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 1615 param->alg_prm.result_size); 1616 1617 if (param->opad_needed) 1618 memcpy(chcr_req->key_ctx.key + 1619 ((param->alg_prm.result_size <= 32) ? 32 : 1620 CHCR_HASH_MAX_DIGEST_SIZE), 1621 hmacctx->opad, param->alg_prm.result_size); 1622 1623 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 1624 param->alg_prm.mk_size, 0, 1625 param->opad_needed, 1626 ((param->kctx_len + 1627 sizeof(chcr_req->key_ctx)) >> 4)); 1628 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 1629 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len + 1630 DUMMY_BYTES); 1631 if (param->bfr_len != 0) { 1632 req_ctx->hctx_wr.dma_addr = 1633 dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr, 1634 param->bfr_len, DMA_TO_DEVICE); 1635 if (dma_mapping_error(&u_ctx->lldi.pdev->dev, 1636 req_ctx->hctx_wr. dma_addr)) { 1637 error = -ENOMEM; 1638 goto err; 1639 } 1640 req_ctx->hctx_wr.dma_len = param->bfr_len; 1641 } else { 1642 req_ctx->hctx_wr.dma_addr = 0; 1643 } 1644 chcr_add_hash_src_ent(req, ulptx, param); 1645 /* Request upto max wr size */ 1646 temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ? 1647 (param->sg_len + param->bfr_len) : 0); 1648 atomic_inc(&adap->chcr_stats.digest_rqst); 1649 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm, 1650 param->hash_size, transhdr_len, 1651 temp, 0); 1652 req_ctx->hctx_wr.skb = skb; 1653 return skb; 1654 err: 1655 kfree_skb(skb); 1656 return ERR_PTR(error); 1657 } 1658 1659 static int chcr_ahash_update(struct ahash_request *req) 1660 { 1661 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1662 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1663 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1664 struct chcr_context *ctx = h_ctx(rtfm); 1665 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1666 struct sk_buff *skb; 1667 u8 remainder = 0, bs; 1668 unsigned int nbytes = req->nbytes; 1669 struct hash_wr_param params; 1670 int error; 1671 unsigned int cpu; 1672 1673 cpu = get_cpu(); 1674 req_ctx->txqidx = cpu % ctx->ntxq; 1675 req_ctx->rxqidx = cpu % ctx->nrxq; 1676 put_cpu(); 1677 1678 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1679 1680 if (nbytes + req_ctx->reqlen >= bs) { 1681 remainder = (nbytes + req_ctx->reqlen) % bs; 1682 nbytes = nbytes + req_ctx->reqlen - remainder; 1683 } else { 1684 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 1685 + req_ctx->reqlen, nbytes, 0); 1686 req_ctx->reqlen += nbytes; 1687 return 0; 1688 } 1689 error = chcr_inc_wrcount(dev); 1690 if (error) 1691 return -ENXIO; 1692 /* Detach state for CHCR means lldi or padap is freed. Increasing 1693 * inflight count for dev guarantees that lldi and padap is valid 1694 */ 1695 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1696 req_ctx->txqidx) && 1697 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1698 error = -ENOSPC; 1699 goto err; 1700 } 1701 1702 chcr_init_hctx_per_wr(req_ctx); 1703 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1704 if (error) { 1705 error = -ENOMEM; 1706 goto err; 1707 } 1708 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1709 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1710 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1711 HASH_SPACE_LEFT(params.kctx_len), 0); 1712 if (params.sg_len > req->nbytes) 1713 params.sg_len = req->nbytes; 1714 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) - 1715 req_ctx->reqlen; 1716 params.opad_needed = 0; 1717 params.more = 1; 1718 params.last = 0; 1719 params.bfr_len = req_ctx->reqlen; 1720 params.scmd1 = 0; 1721 req_ctx->hctx_wr.srcsg = req->src; 1722 1723 params.hash_size = params.alg_prm.result_size; 1724 req_ctx->data_len += params.sg_len + params.bfr_len; 1725 skb = create_hash_wr(req, ¶ms); 1726 if (IS_ERR(skb)) { 1727 error = PTR_ERR(skb); 1728 goto unmap; 1729 } 1730 1731 req_ctx->hctx_wr.processed += params.sg_len; 1732 if (remainder) { 1733 /* Swap buffers */ 1734 swap(req_ctx->reqbfr, req_ctx->skbfr); 1735 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 1736 req_ctx->reqbfr, remainder, req->nbytes - 1737 remainder); 1738 } 1739 req_ctx->reqlen = remainder; 1740 skb->dev = u_ctx->lldi.ports[0]; 1741 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 1742 chcr_send_wr(skb); 1743 return -EINPROGRESS; 1744 unmap: 1745 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1746 err: 1747 chcr_dec_wrcount(dev); 1748 return error; 1749 } 1750 1751 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 1752 { 1753 memset(bfr_ptr, 0, bs); 1754 *bfr_ptr = 0x80; 1755 if (bs == 64) 1756 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 1757 else 1758 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 1759 } 1760 1761 static int chcr_ahash_final(struct ahash_request *req) 1762 { 1763 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1764 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1765 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1766 struct hash_wr_param params; 1767 struct sk_buff *skb; 1768 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1769 struct chcr_context *ctx = h_ctx(rtfm); 1770 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1771 int error; 1772 unsigned int cpu; 1773 1774 cpu = get_cpu(); 1775 req_ctx->txqidx = cpu % ctx->ntxq; 1776 req_ctx->rxqidx = cpu % ctx->nrxq; 1777 put_cpu(); 1778 1779 error = chcr_inc_wrcount(dev); 1780 if (error) 1781 return -ENXIO; 1782 1783 chcr_init_hctx_per_wr(req_ctx); 1784 if (is_hmac(crypto_ahash_tfm(rtfm))) 1785 params.opad_needed = 1; 1786 else 1787 params.opad_needed = 0; 1788 params.sg_len = 0; 1789 req_ctx->hctx_wr.isfinal = 1; 1790 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1791 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1792 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1793 params.opad_needed = 1; 1794 params.kctx_len *= 2; 1795 } else { 1796 params.opad_needed = 0; 1797 } 1798 1799 req_ctx->hctx_wr.result = 1; 1800 params.bfr_len = req_ctx->reqlen; 1801 req_ctx->data_len += params.bfr_len + params.sg_len; 1802 req_ctx->hctx_wr.srcsg = req->src; 1803 if (req_ctx->reqlen == 0) { 1804 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1805 params.last = 0; 1806 params.more = 1; 1807 params.scmd1 = 0; 1808 params.bfr_len = bs; 1809 1810 } else { 1811 params.scmd1 = req_ctx->data_len; 1812 params.last = 1; 1813 params.more = 0; 1814 } 1815 params.hash_size = crypto_ahash_digestsize(rtfm); 1816 skb = create_hash_wr(req, ¶ms); 1817 if (IS_ERR(skb)) { 1818 error = PTR_ERR(skb); 1819 goto err; 1820 } 1821 req_ctx->reqlen = 0; 1822 skb->dev = u_ctx->lldi.ports[0]; 1823 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 1824 chcr_send_wr(skb); 1825 return -EINPROGRESS; 1826 err: 1827 chcr_dec_wrcount(dev); 1828 return error; 1829 } 1830 1831 static int chcr_ahash_finup(struct ahash_request *req) 1832 { 1833 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1834 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1835 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1836 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1837 struct chcr_context *ctx = h_ctx(rtfm); 1838 struct sk_buff *skb; 1839 struct hash_wr_param params; 1840 u8 bs; 1841 int error; 1842 unsigned int cpu; 1843 1844 cpu = get_cpu(); 1845 req_ctx->txqidx = cpu % ctx->ntxq; 1846 req_ctx->rxqidx = cpu % ctx->nrxq; 1847 put_cpu(); 1848 1849 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1850 error = chcr_inc_wrcount(dev); 1851 if (error) 1852 return -ENXIO; 1853 1854 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1855 req_ctx->txqidx) && 1856 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1857 error = -ENOSPC; 1858 goto err; 1859 } 1860 chcr_init_hctx_per_wr(req_ctx); 1861 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1862 if (error) { 1863 error = -ENOMEM; 1864 goto err; 1865 } 1866 1867 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1868 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1869 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1870 params.kctx_len *= 2; 1871 params.opad_needed = 1; 1872 } else { 1873 params.opad_needed = 0; 1874 } 1875 1876 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1877 HASH_SPACE_LEFT(params.kctx_len), 0); 1878 if (params.sg_len < req->nbytes) { 1879 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1880 params.kctx_len /= 2; 1881 params.opad_needed = 0; 1882 } 1883 params.last = 0; 1884 params.more = 1; 1885 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) 1886 - req_ctx->reqlen; 1887 params.hash_size = params.alg_prm.result_size; 1888 params.scmd1 = 0; 1889 } else { 1890 params.last = 1; 1891 params.more = 0; 1892 params.sg_len = req->nbytes; 1893 params.hash_size = crypto_ahash_digestsize(rtfm); 1894 params.scmd1 = req_ctx->data_len + req_ctx->reqlen + 1895 params.sg_len; 1896 } 1897 params.bfr_len = req_ctx->reqlen; 1898 req_ctx->data_len += params.bfr_len + params.sg_len; 1899 req_ctx->hctx_wr.result = 1; 1900 req_ctx->hctx_wr.srcsg = req->src; 1901 if ((req_ctx->reqlen + req->nbytes) == 0) { 1902 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1903 params.last = 0; 1904 params.more = 1; 1905 params.scmd1 = 0; 1906 params.bfr_len = bs; 1907 } 1908 skb = create_hash_wr(req, ¶ms); 1909 if (IS_ERR(skb)) { 1910 error = PTR_ERR(skb); 1911 goto unmap; 1912 } 1913 req_ctx->reqlen = 0; 1914 req_ctx->hctx_wr.processed += params.sg_len; 1915 skb->dev = u_ctx->lldi.ports[0]; 1916 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 1917 chcr_send_wr(skb); 1918 return -EINPROGRESS; 1919 unmap: 1920 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1921 err: 1922 chcr_dec_wrcount(dev); 1923 return error; 1924 } 1925 1926 static int chcr_ahash_digest(struct ahash_request *req) 1927 { 1928 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1929 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1930 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1931 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1932 struct chcr_context *ctx = h_ctx(rtfm); 1933 struct sk_buff *skb; 1934 struct hash_wr_param params; 1935 u8 bs; 1936 int error; 1937 unsigned int cpu; 1938 1939 cpu = get_cpu(); 1940 req_ctx->txqidx = cpu % ctx->ntxq; 1941 req_ctx->rxqidx = cpu % ctx->nrxq; 1942 put_cpu(); 1943 1944 rtfm->init(req); 1945 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1946 error = chcr_inc_wrcount(dev); 1947 if (error) 1948 return -ENXIO; 1949 1950 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1951 req_ctx->txqidx) && 1952 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1953 error = -ENOSPC; 1954 goto err; 1955 } 1956 1957 chcr_init_hctx_per_wr(req_ctx); 1958 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1959 if (error) { 1960 error = -ENOMEM; 1961 goto err; 1962 } 1963 1964 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1965 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1966 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1967 params.kctx_len *= 2; 1968 params.opad_needed = 1; 1969 } else { 1970 params.opad_needed = 0; 1971 } 1972 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1973 HASH_SPACE_LEFT(params.kctx_len), 0); 1974 if (params.sg_len < req->nbytes) { 1975 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1976 params.kctx_len /= 2; 1977 params.opad_needed = 0; 1978 } 1979 params.last = 0; 1980 params.more = 1; 1981 params.scmd1 = 0; 1982 params.sg_len = rounddown(params.sg_len, bs); 1983 params.hash_size = params.alg_prm.result_size; 1984 } else { 1985 params.sg_len = req->nbytes; 1986 params.hash_size = crypto_ahash_digestsize(rtfm); 1987 params.last = 1; 1988 params.more = 0; 1989 params.scmd1 = req->nbytes + req_ctx->data_len; 1990 1991 } 1992 params.bfr_len = 0; 1993 req_ctx->hctx_wr.result = 1; 1994 req_ctx->hctx_wr.srcsg = req->src; 1995 req_ctx->data_len += params.bfr_len + params.sg_len; 1996 1997 if (req->nbytes == 0) { 1998 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1999 params.more = 1; 2000 params.bfr_len = bs; 2001 } 2002 2003 skb = create_hash_wr(req, ¶ms); 2004 if (IS_ERR(skb)) { 2005 error = PTR_ERR(skb); 2006 goto unmap; 2007 } 2008 req_ctx->hctx_wr.processed += params.sg_len; 2009 skb->dev = u_ctx->lldi.ports[0]; 2010 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 2011 chcr_send_wr(skb); 2012 return -EINPROGRESS; 2013 unmap: 2014 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 2015 err: 2016 chcr_dec_wrcount(dev); 2017 return error; 2018 } 2019 2020 static int chcr_ahash_continue(struct ahash_request *req) 2021 { 2022 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2023 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; 2024 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 2025 struct chcr_context *ctx = h_ctx(rtfm); 2026 struct uld_ctx *u_ctx = ULD_CTX(ctx); 2027 struct sk_buff *skb; 2028 struct hash_wr_param params; 2029 u8 bs; 2030 int error; 2031 unsigned int cpu; 2032 2033 cpu = get_cpu(); 2034 reqctx->txqidx = cpu % ctx->ntxq; 2035 reqctx->rxqidx = cpu % ctx->nrxq; 2036 put_cpu(); 2037 2038 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 2039 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 2040 params.kctx_len = roundup(params.alg_prm.result_size, 16); 2041 if (is_hmac(crypto_ahash_tfm(rtfm))) { 2042 params.kctx_len *= 2; 2043 params.opad_needed = 1; 2044 } else { 2045 params.opad_needed = 0; 2046 } 2047 params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0, 2048 HASH_SPACE_LEFT(params.kctx_len), 2049 hctx_wr->src_ofst); 2050 if ((params.sg_len + hctx_wr->processed) > req->nbytes) 2051 params.sg_len = req->nbytes - hctx_wr->processed; 2052 if (!hctx_wr->result || 2053 ((params.sg_len + hctx_wr->processed) < req->nbytes)) { 2054 if (is_hmac(crypto_ahash_tfm(rtfm))) { 2055 params.kctx_len /= 2; 2056 params.opad_needed = 0; 2057 } 2058 params.last = 0; 2059 params.more = 1; 2060 params.sg_len = rounddown(params.sg_len, bs); 2061 params.hash_size = params.alg_prm.result_size; 2062 params.scmd1 = 0; 2063 } else { 2064 params.last = 1; 2065 params.more = 0; 2066 params.hash_size = crypto_ahash_digestsize(rtfm); 2067 params.scmd1 = reqctx->data_len + params.sg_len; 2068 } 2069 params.bfr_len = 0; 2070 reqctx->data_len += params.sg_len; 2071 skb = create_hash_wr(req, ¶ms); 2072 if (IS_ERR(skb)) { 2073 error = PTR_ERR(skb); 2074 goto err; 2075 } 2076 hctx_wr->processed += params.sg_len; 2077 skb->dev = u_ctx->lldi.ports[0]; 2078 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 2079 chcr_send_wr(skb); 2080 return 0; 2081 err: 2082 return error; 2083 } 2084 2085 static inline void chcr_handle_ahash_resp(struct ahash_request *req, 2086 unsigned char *input, 2087 int err) 2088 { 2089 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2090 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; 2091 int digestsize, updated_digestsize; 2092 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2093 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 2094 struct chcr_dev *dev = h_ctx(tfm)->dev; 2095 2096 if (input == NULL) 2097 goto out; 2098 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); 2099 updated_digestsize = digestsize; 2100 if (digestsize == SHA224_DIGEST_SIZE) 2101 updated_digestsize = SHA256_DIGEST_SIZE; 2102 else if (digestsize == SHA384_DIGEST_SIZE) 2103 updated_digestsize = SHA512_DIGEST_SIZE; 2104 2105 if (hctx_wr->dma_addr) { 2106 dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr, 2107 hctx_wr->dma_len, DMA_TO_DEVICE); 2108 hctx_wr->dma_addr = 0; 2109 } 2110 if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) == 2111 req->nbytes)) { 2112 if (hctx_wr->result == 1) { 2113 hctx_wr->result = 0; 2114 memcpy(req->result, input + sizeof(struct cpl_fw6_pld), 2115 digestsize); 2116 } else { 2117 memcpy(reqctx->partial_hash, 2118 input + sizeof(struct cpl_fw6_pld), 2119 updated_digestsize); 2120 2121 } 2122 goto unmap; 2123 } 2124 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), 2125 updated_digestsize); 2126 2127 err = chcr_ahash_continue(req); 2128 if (err) 2129 goto unmap; 2130 return; 2131 unmap: 2132 if (hctx_wr->is_sg_map) 2133 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 2134 2135 2136 out: 2137 chcr_dec_wrcount(dev); 2138 req->base.complete(&req->base, err); 2139 } 2140 2141 /* 2142 * chcr_handle_resp - Unmap the DMA buffers associated with the request 2143 * @req: crypto request 2144 */ 2145 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 2146 int err) 2147 { 2148 struct crypto_tfm *tfm = req->tfm; 2149 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2150 struct adapter *adap = padap(ctx->dev); 2151 2152 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 2153 case CRYPTO_ALG_TYPE_AEAD: 2154 err = chcr_handle_aead_resp(aead_request_cast(req), input, err); 2155 break; 2156 2157 case CRYPTO_ALG_TYPE_SKCIPHER: 2158 chcr_handle_cipher_resp(skcipher_request_cast(req), 2159 input, err); 2160 break; 2161 case CRYPTO_ALG_TYPE_AHASH: 2162 chcr_handle_ahash_resp(ahash_request_cast(req), input, err); 2163 } 2164 atomic_inc(&adap->chcr_stats.complete); 2165 return err; 2166 } 2167 static int chcr_ahash_export(struct ahash_request *areq, void *out) 2168 { 2169 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2170 struct chcr_ahash_req_ctx *state = out; 2171 2172 state->reqlen = req_ctx->reqlen; 2173 state->data_len = req_ctx->data_len; 2174 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 2175 memcpy(state->partial_hash, req_ctx->partial_hash, 2176 CHCR_HASH_MAX_DIGEST_SIZE); 2177 chcr_init_hctx_per_wr(state); 2178 return 0; 2179 } 2180 2181 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 2182 { 2183 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2184 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 2185 2186 req_ctx->reqlen = state->reqlen; 2187 req_ctx->data_len = state->data_len; 2188 req_ctx->reqbfr = req_ctx->bfr1; 2189 req_ctx->skbfr = req_ctx->bfr2; 2190 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 2191 memcpy(req_ctx->partial_hash, state->partial_hash, 2192 CHCR_HASH_MAX_DIGEST_SIZE); 2193 chcr_init_hctx_per_wr(req_ctx); 2194 return 0; 2195 } 2196 2197 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2198 unsigned int keylen) 2199 { 2200 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 2201 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2202 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2203 unsigned int i, err = 0, updated_digestsize; 2204 2205 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 2206 2207 /* use the key to calculate the ipad and opad. ipad will sent with the 2208 * first request's data. opad will be sent with the final hash result 2209 * ipad in hmacctx->ipad and opad in hmacctx->opad location 2210 */ 2211 shash->tfm = hmacctx->base_hash; 2212 if (keylen > bs) { 2213 err = crypto_shash_digest(shash, key, keylen, 2214 hmacctx->ipad); 2215 if (err) 2216 goto out; 2217 keylen = digestsize; 2218 } else { 2219 memcpy(hmacctx->ipad, key, keylen); 2220 } 2221 memset(hmacctx->ipad + keylen, 0, bs - keylen); 2222 memcpy(hmacctx->opad, hmacctx->ipad, bs); 2223 2224 for (i = 0; i < bs / sizeof(int); i++) { 2225 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 2226 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 2227 } 2228 2229 updated_digestsize = digestsize; 2230 if (digestsize == SHA224_DIGEST_SIZE) 2231 updated_digestsize = SHA256_DIGEST_SIZE; 2232 else if (digestsize == SHA384_DIGEST_SIZE) 2233 updated_digestsize = SHA512_DIGEST_SIZE; 2234 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 2235 hmacctx->ipad, digestsize); 2236 if (err) 2237 goto out; 2238 chcr_change_order(hmacctx->ipad, updated_digestsize); 2239 2240 err = chcr_compute_partial_hash(shash, hmacctx->opad, 2241 hmacctx->opad, digestsize); 2242 if (err) 2243 goto out; 2244 chcr_change_order(hmacctx->opad, updated_digestsize); 2245 out: 2246 return err; 2247 } 2248 2249 static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, 2250 unsigned int key_len) 2251 { 2252 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 2253 unsigned short context_size = 0; 2254 int err; 2255 2256 err = chcr_cipher_fallback_setkey(cipher, key, key_len); 2257 if (err) 2258 goto badkey_err; 2259 2260 memcpy(ablkctx->key, key, key_len); 2261 ablkctx->enckey_len = key_len; 2262 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 2263 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 2264 /* Both keys for xts must be aligned to 16 byte boundary 2265 * by padding with zeros. So for 24 byte keys padding 8 zeroes. 2266 */ 2267 if (key_len == 48) { 2268 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len 2269 + 16) >> 4; 2270 memmove(ablkctx->key + 32, ablkctx->key + 24, 24); 2271 memset(ablkctx->key + 24, 0, 8); 2272 memset(ablkctx->key + 56, 0, 8); 2273 ablkctx->enckey_len = 64; 2274 ablkctx->key_ctx_hdr = 2275 FILL_KEY_CTX_HDR(CHCR_KEYCTX_CIPHER_KEY_SIZE_192, 2276 CHCR_KEYCTX_NO_KEY, 1, 2277 0, context_size); 2278 } else { 2279 ablkctx->key_ctx_hdr = 2280 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 2281 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 2282 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 2283 CHCR_KEYCTX_NO_KEY, 1, 2284 0, context_size); 2285 } 2286 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 2287 return 0; 2288 badkey_err: 2289 ablkctx->enckey_len = 0; 2290 2291 return err; 2292 } 2293 2294 static int chcr_sha_init(struct ahash_request *areq) 2295 { 2296 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2297 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2298 int digestsize = crypto_ahash_digestsize(tfm); 2299 2300 req_ctx->data_len = 0; 2301 req_ctx->reqlen = 0; 2302 req_ctx->reqbfr = req_ctx->bfr1; 2303 req_ctx->skbfr = req_ctx->bfr2; 2304 copy_hash_init_values(req_ctx->partial_hash, digestsize); 2305 2306 return 0; 2307 } 2308 2309 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 2310 { 2311 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2312 sizeof(struct chcr_ahash_req_ctx)); 2313 return chcr_device_init(crypto_tfm_ctx(tfm)); 2314 } 2315 2316 static int chcr_hmac_init(struct ahash_request *areq) 2317 { 2318 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2319 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 2320 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); 2321 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 2322 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 2323 2324 chcr_sha_init(areq); 2325 req_ctx->data_len = bs; 2326 if (is_hmac(crypto_ahash_tfm(rtfm))) { 2327 if (digestsize == SHA224_DIGEST_SIZE) 2328 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2329 SHA256_DIGEST_SIZE); 2330 else if (digestsize == SHA384_DIGEST_SIZE) 2331 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2332 SHA512_DIGEST_SIZE); 2333 else 2334 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2335 digestsize); 2336 } 2337 return 0; 2338 } 2339 2340 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 2341 { 2342 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2343 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2344 unsigned int digestsize = 2345 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 2346 2347 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2348 sizeof(struct chcr_ahash_req_ctx)); 2349 hmacctx->base_hash = chcr_alloc_shash(digestsize); 2350 if (IS_ERR(hmacctx->base_hash)) 2351 return PTR_ERR(hmacctx->base_hash); 2352 return chcr_device_init(crypto_tfm_ctx(tfm)); 2353 } 2354 2355 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 2356 { 2357 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2358 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2359 2360 if (hmacctx->base_hash) { 2361 chcr_free_shash(hmacctx->base_hash); 2362 hmacctx->base_hash = NULL; 2363 } 2364 } 2365 2366 inline void chcr_aead_common_exit(struct aead_request *req) 2367 { 2368 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2369 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2370 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); 2371 2372 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); 2373 } 2374 2375 static int chcr_aead_common_init(struct aead_request *req) 2376 { 2377 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2378 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2379 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2380 unsigned int authsize = crypto_aead_authsize(tfm); 2381 int error = -EINVAL; 2382 2383 /* validate key size */ 2384 if (aeadctx->enckey_len == 0) 2385 goto err; 2386 if (reqctx->op && req->cryptlen < authsize) 2387 goto err; 2388 if (reqctx->b0_len) 2389 reqctx->scratch_pad = reqctx->iv + IV; 2390 else 2391 reqctx->scratch_pad = NULL; 2392 2393 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2394 reqctx->op); 2395 if (error) { 2396 error = -ENOMEM; 2397 goto err; 2398 } 2399 2400 return 0; 2401 err: 2402 return error; 2403 } 2404 2405 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, 2406 int aadmax, int wrlen, 2407 unsigned short op_type) 2408 { 2409 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 2410 2411 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || 2412 dst_nents > MAX_DSGL_ENT || 2413 (req->assoclen > aadmax) || 2414 (wrlen > SGE_MAX_WR_LEN)) 2415 return 1; 2416 return 0; 2417 } 2418 2419 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) 2420 { 2421 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2422 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2423 struct aead_request *subreq = aead_request_ctx(req); 2424 2425 aead_request_set_tfm(subreq, aeadctx->sw_cipher); 2426 aead_request_set_callback(subreq, req->base.flags, 2427 req->base.complete, req->base.data); 2428 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2429 req->iv); 2430 aead_request_set_ad(subreq, req->assoclen); 2431 return op_type ? crypto_aead_decrypt(subreq) : 2432 crypto_aead_encrypt(subreq); 2433 } 2434 2435 static struct sk_buff *create_authenc_wr(struct aead_request *req, 2436 unsigned short qid, 2437 int size) 2438 { 2439 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2440 struct chcr_context *ctx = a_ctx(tfm); 2441 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2442 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2443 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2444 struct sk_buff *skb = NULL; 2445 struct chcr_wr *chcr_req; 2446 struct cpl_rx_phys_dsgl *phys_cpl; 2447 struct ulptx_sgl *ulptx; 2448 unsigned int transhdr_len; 2449 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm); 2450 unsigned int kctx_len = 0, dnents, snents; 2451 unsigned int authsize = crypto_aead_authsize(tfm); 2452 int error = -EINVAL; 2453 u8 *ivptr; 2454 int null = 0; 2455 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2456 GFP_ATOMIC; 2457 struct adapter *adap = padap(ctx->dev); 2458 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2459 2460 if (req->cryptlen == 0) 2461 return NULL; 2462 2463 reqctx->b0_len = 0; 2464 error = chcr_aead_common_init(req); 2465 if (error) 2466 return ERR_PTR(error); 2467 2468 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL || 2469 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2470 null = 1; 2471 } 2472 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + 2473 (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0); 2474 dnents += MIN_AUTH_SG; // For IV 2475 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 2476 CHCR_SRC_SG_SIZE, 0); 2477 dst_size = get_space_for_phys_dsgl(dnents); 2478 kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4) 2479 - sizeof(chcr_req->key_ctx); 2480 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2481 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) < 2482 SGE_MAX_WR_LEN; 2483 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) 2484 : (sgl_len(snents) * 8); 2485 transhdr_len += temp; 2486 transhdr_len = roundup(transhdr_len, 16); 2487 2488 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 2489 transhdr_len, reqctx->op)) { 2490 atomic_inc(&adap->chcr_stats.fallback); 2491 chcr_aead_common_exit(req); 2492 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 2493 } 2494 skb = alloc_skb(transhdr_len, flags); 2495 if (!skb) { 2496 error = -ENOMEM; 2497 goto err; 2498 } 2499 2500 chcr_req = __skb_put_zero(skb, transhdr_len); 2501 2502 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; 2503 2504 /* 2505 * Input order is AAD,IV and Payload. where IV should be included as 2506 * the part of authdata. All other fields should be filled according 2507 * to the hardware spec 2508 */ 2509 chcr_req->sec_cpl.op_ivinsrtofst = 2510 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); 2511 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen); 2512 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2513 null ? 0 : 1 + IV, 2514 null ? 0 : IV + req->assoclen, 2515 req->assoclen + IV + 1, 2516 (temp & 0x1F0) >> 4); 2517 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 2518 temp & 0xF, 2519 null ? 0 : req->assoclen + IV + 1, 2520 temp, temp); 2521 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL || 2522 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA) 2523 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR; 2524 else 2525 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC; 2526 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 2527 (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0, 2528 temp, 2529 actx->auth_mode, aeadctx->hmac_ctrl, 2530 IV >> 1); 2531 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2532 0, 0, dst_size); 2533 2534 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2535 if (reqctx->op == CHCR_ENCRYPT_OP || 2536 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2537 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) 2538 memcpy(chcr_req->key_ctx.key, aeadctx->key, 2539 aeadctx->enckey_len); 2540 else 2541 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 2542 aeadctx->enckey_len); 2543 2544 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 2545 actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16)); 2546 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2547 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 2548 ulptx = (struct ulptx_sgl *)(ivptr + IV); 2549 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2550 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2551 memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE); 2552 memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv, 2553 CTR_RFC3686_IV_SIZE); 2554 *(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE + 2555 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 2556 } else { 2557 memcpy(ivptr, req->iv, IV); 2558 } 2559 chcr_add_aead_dst_ent(req, phys_cpl, qid); 2560 chcr_add_aead_src_ent(req, ulptx); 2561 atomic_inc(&adap->chcr_stats.cipher_rqst); 2562 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 2563 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); 2564 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 2565 transhdr_len, temp, 0); 2566 reqctx->skb = skb; 2567 2568 return skb; 2569 err: 2570 chcr_aead_common_exit(req); 2571 2572 return ERR_PTR(error); 2573 } 2574 2575 int chcr_aead_dma_map(struct device *dev, 2576 struct aead_request *req, 2577 unsigned short op_type) 2578 { 2579 int error; 2580 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2581 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2582 unsigned int authsize = crypto_aead_authsize(tfm); 2583 int src_len, dst_len; 2584 2585 /* calculate and handle src and dst sg length separately 2586 * for inplace and out-of place operations 2587 */ 2588 if (req->src == req->dst) { 2589 src_len = req->assoclen + req->cryptlen + (op_type ? 2590 0 : authsize); 2591 dst_len = src_len; 2592 } else { 2593 src_len = req->assoclen + req->cryptlen; 2594 dst_len = req->assoclen + req->cryptlen + (op_type ? 2595 -authsize : authsize); 2596 } 2597 2598 if (!req->cryptlen || !src_len || !dst_len) 2599 return 0; 2600 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len), 2601 DMA_BIDIRECTIONAL); 2602 if (dma_mapping_error(dev, reqctx->iv_dma)) 2603 return -ENOMEM; 2604 if (reqctx->b0_len) 2605 reqctx->b0_dma = reqctx->iv_dma + IV; 2606 else 2607 reqctx->b0_dma = 0; 2608 if (req->src == req->dst) { 2609 error = dma_map_sg(dev, req->src, 2610 sg_nents_for_len(req->src, src_len), 2611 DMA_BIDIRECTIONAL); 2612 if (!error) 2613 goto err; 2614 } else { 2615 error = dma_map_sg(dev, req->src, 2616 sg_nents_for_len(req->src, src_len), 2617 DMA_TO_DEVICE); 2618 if (!error) 2619 goto err; 2620 error = dma_map_sg(dev, req->dst, 2621 sg_nents_for_len(req->dst, dst_len), 2622 DMA_FROM_DEVICE); 2623 if (!error) { 2624 dma_unmap_sg(dev, req->src, 2625 sg_nents_for_len(req->src, src_len), 2626 DMA_TO_DEVICE); 2627 goto err; 2628 } 2629 } 2630 2631 return 0; 2632 err: 2633 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 2634 return -ENOMEM; 2635 } 2636 2637 void chcr_aead_dma_unmap(struct device *dev, 2638 struct aead_request *req, 2639 unsigned short op_type) 2640 { 2641 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2642 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2643 unsigned int authsize = crypto_aead_authsize(tfm); 2644 int src_len, dst_len; 2645 2646 /* calculate and handle src and dst sg length separately 2647 * for inplace and out-of place operations 2648 */ 2649 if (req->src == req->dst) { 2650 src_len = req->assoclen + req->cryptlen + (op_type ? 2651 0 : authsize); 2652 dst_len = src_len; 2653 } else { 2654 src_len = req->assoclen + req->cryptlen; 2655 dst_len = req->assoclen + req->cryptlen + (op_type ? 2656 -authsize : authsize); 2657 } 2658 2659 if (!req->cryptlen || !src_len || !dst_len) 2660 return; 2661 2662 dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len), 2663 DMA_BIDIRECTIONAL); 2664 if (req->src == req->dst) { 2665 dma_unmap_sg(dev, req->src, 2666 sg_nents_for_len(req->src, src_len), 2667 DMA_BIDIRECTIONAL); 2668 } else { 2669 dma_unmap_sg(dev, req->src, 2670 sg_nents_for_len(req->src, src_len), 2671 DMA_TO_DEVICE); 2672 dma_unmap_sg(dev, req->dst, 2673 sg_nents_for_len(req->dst, dst_len), 2674 DMA_FROM_DEVICE); 2675 } 2676 } 2677 2678 void chcr_add_aead_src_ent(struct aead_request *req, 2679 struct ulptx_sgl *ulptx) 2680 { 2681 struct ulptx_walk ulp_walk; 2682 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2683 2684 if (reqctx->imm) { 2685 u8 *buf = (u8 *)ulptx; 2686 2687 if (reqctx->b0_len) { 2688 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); 2689 buf += reqctx->b0_len; 2690 } 2691 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2692 buf, req->cryptlen + req->assoclen, 0); 2693 } else { 2694 ulptx_walk_init(&ulp_walk, ulptx); 2695 if (reqctx->b0_len) 2696 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, 2697 reqctx->b0_dma); 2698 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen + 2699 req->assoclen, 0); 2700 ulptx_walk_end(&ulp_walk); 2701 } 2702 } 2703 2704 void chcr_add_aead_dst_ent(struct aead_request *req, 2705 struct cpl_rx_phys_dsgl *phys_cpl, 2706 unsigned short qid) 2707 { 2708 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2709 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2710 struct dsgl_walk dsgl_walk; 2711 unsigned int authsize = crypto_aead_authsize(tfm); 2712 struct chcr_context *ctx = a_ctx(tfm); 2713 u32 temp; 2714 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2715 2716 dsgl_walk_init(&dsgl_walk, phys_cpl); 2717 dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma); 2718 temp = req->assoclen + req->cryptlen + 2719 (reqctx->op ? -authsize : authsize); 2720 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0); 2721 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id); 2722 } 2723 2724 void chcr_add_cipher_src_ent(struct skcipher_request *req, 2725 void *ulptx, 2726 struct cipher_wr_param *wrparam) 2727 { 2728 struct ulptx_walk ulp_walk; 2729 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 2730 u8 *buf = ulptx; 2731 2732 memcpy(buf, reqctx->iv, IV); 2733 buf += IV; 2734 if (reqctx->imm) { 2735 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2736 buf, wrparam->bytes, reqctx->processed); 2737 } else { 2738 ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf); 2739 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, 2740 reqctx->src_ofst); 2741 reqctx->srcsg = ulp_walk.last_sg; 2742 reqctx->src_ofst = ulp_walk.last_sg_len; 2743 ulptx_walk_end(&ulp_walk); 2744 } 2745 } 2746 2747 void chcr_add_cipher_dst_ent(struct skcipher_request *req, 2748 struct cpl_rx_phys_dsgl *phys_cpl, 2749 struct cipher_wr_param *wrparam, 2750 unsigned short qid) 2751 { 2752 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 2753 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); 2754 struct chcr_context *ctx = c_ctx(tfm); 2755 struct dsgl_walk dsgl_walk; 2756 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2757 2758 dsgl_walk_init(&dsgl_walk, phys_cpl); 2759 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, 2760 reqctx->dst_ofst); 2761 reqctx->dstsg = dsgl_walk.last_sg; 2762 reqctx->dst_ofst = dsgl_walk.last_sg_len; 2763 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id); 2764 } 2765 2766 void chcr_add_hash_src_ent(struct ahash_request *req, 2767 struct ulptx_sgl *ulptx, 2768 struct hash_wr_param *param) 2769 { 2770 struct ulptx_walk ulp_walk; 2771 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2772 2773 if (reqctx->hctx_wr.imm) { 2774 u8 *buf = (u8 *)ulptx; 2775 2776 if (param->bfr_len) { 2777 memcpy(buf, reqctx->reqbfr, param->bfr_len); 2778 buf += param->bfr_len; 2779 } 2780 2781 sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg, 2782 sg_nents(reqctx->hctx_wr.srcsg), buf, 2783 param->sg_len, 0); 2784 } else { 2785 ulptx_walk_init(&ulp_walk, ulptx); 2786 if (param->bfr_len) 2787 ulptx_walk_add_page(&ulp_walk, param->bfr_len, 2788 reqctx->hctx_wr.dma_addr); 2789 ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg, 2790 param->sg_len, reqctx->hctx_wr.src_ofst); 2791 reqctx->hctx_wr.srcsg = ulp_walk.last_sg; 2792 reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len; 2793 ulptx_walk_end(&ulp_walk); 2794 } 2795 } 2796 2797 int chcr_hash_dma_map(struct device *dev, 2798 struct ahash_request *req) 2799 { 2800 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2801 int error = 0; 2802 2803 if (!req->nbytes) 2804 return 0; 2805 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2806 DMA_TO_DEVICE); 2807 if (!error) 2808 return -ENOMEM; 2809 req_ctx->hctx_wr.is_sg_map = 1; 2810 return 0; 2811 } 2812 2813 void chcr_hash_dma_unmap(struct device *dev, 2814 struct ahash_request *req) 2815 { 2816 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2817 2818 if (!req->nbytes) 2819 return; 2820 2821 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2822 DMA_TO_DEVICE); 2823 req_ctx->hctx_wr.is_sg_map = 0; 2824 2825 } 2826 2827 int chcr_cipher_dma_map(struct device *dev, 2828 struct skcipher_request *req) 2829 { 2830 int error; 2831 2832 if (req->src == req->dst) { 2833 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2834 DMA_BIDIRECTIONAL); 2835 if (!error) 2836 goto err; 2837 } else { 2838 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2839 DMA_TO_DEVICE); 2840 if (!error) 2841 goto err; 2842 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2843 DMA_FROM_DEVICE); 2844 if (!error) { 2845 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2846 DMA_TO_DEVICE); 2847 goto err; 2848 } 2849 } 2850 2851 return 0; 2852 err: 2853 return -ENOMEM; 2854 } 2855 2856 void chcr_cipher_dma_unmap(struct device *dev, 2857 struct skcipher_request *req) 2858 { 2859 if (req->src == req->dst) { 2860 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2861 DMA_BIDIRECTIONAL); 2862 } else { 2863 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2864 DMA_TO_DEVICE); 2865 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2866 DMA_FROM_DEVICE); 2867 } 2868 } 2869 2870 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 2871 { 2872 __be32 data; 2873 2874 memset(block, 0, csize); 2875 block += csize; 2876 2877 if (csize >= 4) 2878 csize = 4; 2879 else if (msglen > (unsigned int)(1 << (8 * csize))) 2880 return -EOVERFLOW; 2881 2882 data = cpu_to_be32(msglen); 2883 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 2884 2885 return 0; 2886 } 2887 2888 static int generate_b0(struct aead_request *req, u8 *ivptr, 2889 unsigned short op_type) 2890 { 2891 unsigned int l, lp, m; 2892 int rc; 2893 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2894 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2895 u8 *b0 = reqctx->scratch_pad; 2896 2897 m = crypto_aead_authsize(aead); 2898 2899 memcpy(b0, ivptr, 16); 2900 2901 lp = b0[0]; 2902 l = lp + 1; 2903 2904 /* set m, bits 3-5 */ 2905 *b0 |= (8 * ((m - 2) / 2)); 2906 2907 /* set adata, bit 6, if associated data is used */ 2908 if (req->assoclen) 2909 *b0 |= 64; 2910 rc = set_msg_len(b0 + 16 - l, 2911 (op_type == CHCR_DECRYPT_OP) ? 2912 req->cryptlen - m : req->cryptlen, l); 2913 2914 return rc; 2915 } 2916 2917 static inline int crypto_ccm_check_iv(const u8 *iv) 2918 { 2919 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 2920 if (iv[0] < 1 || iv[0] > 7) 2921 return -EINVAL; 2922 2923 return 0; 2924 } 2925 2926 static int ccm_format_packet(struct aead_request *req, 2927 u8 *ivptr, 2928 unsigned int sub_type, 2929 unsigned short op_type, 2930 unsigned int assoclen) 2931 { 2932 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2933 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2934 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2935 int rc = 0; 2936 2937 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2938 ivptr[0] = 3; 2939 memcpy(ivptr + 1, &aeadctx->salt[0], 3); 2940 memcpy(ivptr + 4, req->iv, 8); 2941 memset(ivptr + 12, 0, 4); 2942 } else { 2943 memcpy(ivptr, req->iv, 16); 2944 } 2945 if (assoclen) 2946 put_unaligned_be16(assoclen, &reqctx->scratch_pad[16]); 2947 2948 rc = generate_b0(req, ivptr, op_type); 2949 /* zero the ctr value */ 2950 memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1); 2951 return rc; 2952 } 2953 2954 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 2955 unsigned int dst_size, 2956 struct aead_request *req, 2957 unsigned short op_type) 2958 { 2959 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2960 struct chcr_context *ctx = a_ctx(tfm); 2961 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2962 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2963 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 2964 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 2965 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2966 unsigned int ccm_xtra; 2967 unsigned int tag_offset = 0, auth_offset = 0; 2968 unsigned int assoclen; 2969 2970 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2971 assoclen = req->assoclen - 8; 2972 else 2973 assoclen = req->assoclen; 2974 ccm_xtra = CCM_B0_SIZE + 2975 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 2976 2977 auth_offset = req->cryptlen ? 2978 (req->assoclen + IV + 1 + ccm_xtra) : 0; 2979 if (op_type == CHCR_DECRYPT_OP) { 2980 if (crypto_aead_authsize(tfm) != req->cryptlen) 2981 tag_offset = crypto_aead_authsize(tfm); 2982 else 2983 auth_offset = 0; 2984 } 2985 2986 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); 2987 sec_cpl->pldlen = 2988 htonl(req->assoclen + IV + req->cryptlen + ccm_xtra); 2989 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 2990 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2991 1 + IV, IV + assoclen + ccm_xtra, 2992 req->assoclen + IV + 1 + ccm_xtra, 0); 2993 2994 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 2995 auth_offset, tag_offset, 2996 (op_type == CHCR_ENCRYPT_OP) ? 0 : 2997 crypto_aead_authsize(tfm)); 2998 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2999 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 3000 cipher_mode, mac_mode, 3001 aeadctx->hmac_ctrl, IV >> 1); 3002 3003 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 3004 0, dst_size); 3005 } 3006 3007 static int aead_ccm_validate_input(unsigned short op_type, 3008 struct aead_request *req, 3009 struct chcr_aead_ctx *aeadctx, 3010 unsigned int sub_type) 3011 { 3012 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 3013 if (crypto_ccm_check_iv(req->iv)) { 3014 pr_err("CCM: IV check fails\n"); 3015 return -EINVAL; 3016 } 3017 } else { 3018 if (req->assoclen != 16 && req->assoclen != 20) { 3019 pr_err("RFC4309: Invalid AAD length %d\n", 3020 req->assoclen); 3021 return -EINVAL; 3022 } 3023 } 3024 return 0; 3025 } 3026 3027 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 3028 unsigned short qid, 3029 int size) 3030 { 3031 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3032 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3033 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3034 struct sk_buff *skb = NULL; 3035 struct chcr_wr *chcr_req; 3036 struct cpl_rx_phys_dsgl *phys_cpl; 3037 struct ulptx_sgl *ulptx; 3038 unsigned int transhdr_len; 3039 unsigned int dst_size = 0, kctx_len, dnents, temp, snents; 3040 unsigned int sub_type, assoclen = req->assoclen; 3041 unsigned int authsize = crypto_aead_authsize(tfm); 3042 int error = -EINVAL; 3043 u8 *ivptr; 3044 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 3045 GFP_ATOMIC; 3046 struct adapter *adap = padap(a_ctx(tfm)->dev); 3047 3048 sub_type = get_aead_subtype(tfm); 3049 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 3050 assoclen -= 8; 3051 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); 3052 error = chcr_aead_common_init(req); 3053 if (error) 3054 return ERR_PTR(error); 3055 3056 error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type); 3057 if (error) 3058 goto err; 3059 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen 3060 + (reqctx->op ? -authsize : authsize), 3061 CHCR_DST_SG_SIZE, 0); 3062 dnents += MIN_CCM_SG; // For IV and B0 3063 dst_size = get_space_for_phys_dsgl(dnents); 3064 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 3065 CHCR_SRC_SG_SIZE, 0); 3066 snents += MIN_CCM_SG; //For B0 3067 kctx_len = roundup(aeadctx->enckey_len, 16) * 2; 3068 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 3069 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen + 3070 reqctx->b0_len) <= SGE_MAX_WR_LEN; 3071 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen + 3072 reqctx->b0_len, 16) : 3073 (sgl_len(snents) * 8); 3074 transhdr_len += temp; 3075 transhdr_len = roundup(transhdr_len, 16); 3076 3077 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - 3078 reqctx->b0_len, transhdr_len, reqctx->op)) { 3079 atomic_inc(&adap->chcr_stats.fallback); 3080 chcr_aead_common_exit(req); 3081 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 3082 } 3083 skb = alloc_skb(transhdr_len, flags); 3084 3085 if (!skb) { 3086 error = -ENOMEM; 3087 goto err; 3088 } 3089 3090 chcr_req = __skb_put_zero(skb, transhdr_len); 3091 3092 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op); 3093 3094 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 3095 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 3096 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 3097 aeadctx->key, aeadctx->enckey_len); 3098 3099 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 3100 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 3101 ulptx = (struct ulptx_sgl *)(ivptr + IV); 3102 error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen); 3103 if (error) 3104 goto dstmap_fail; 3105 chcr_add_aead_dst_ent(req, phys_cpl, qid); 3106 chcr_add_aead_src_ent(req, ulptx); 3107 3108 atomic_inc(&adap->chcr_stats.aead_rqst); 3109 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 3110 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen + 3111 reqctx->b0_len) : 0); 3112 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, 3113 transhdr_len, temp, 0); 3114 reqctx->skb = skb; 3115 3116 return skb; 3117 dstmap_fail: 3118 kfree_skb(skb); 3119 err: 3120 chcr_aead_common_exit(req); 3121 return ERR_PTR(error); 3122 } 3123 3124 static struct sk_buff *create_gcm_wr(struct aead_request *req, 3125 unsigned short qid, 3126 int size) 3127 { 3128 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3129 struct chcr_context *ctx = a_ctx(tfm); 3130 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 3131 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3132 struct sk_buff *skb = NULL; 3133 struct chcr_wr *chcr_req; 3134 struct cpl_rx_phys_dsgl *phys_cpl; 3135 struct ulptx_sgl *ulptx; 3136 unsigned int transhdr_len, dnents = 0, snents; 3137 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; 3138 unsigned int authsize = crypto_aead_authsize(tfm); 3139 int error = -EINVAL; 3140 u8 *ivptr; 3141 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 3142 GFP_ATOMIC; 3143 struct adapter *adap = padap(ctx->dev); 3144 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 3145 3146 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 3147 assoclen = req->assoclen - 8; 3148 3149 reqctx->b0_len = 0; 3150 error = chcr_aead_common_init(req); 3151 if (error) 3152 return ERR_PTR(error); 3153 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + 3154 (reqctx->op ? -authsize : authsize), 3155 CHCR_DST_SG_SIZE, 0); 3156 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 3157 CHCR_SRC_SG_SIZE, 0); 3158 dnents += MIN_GCM_SG; // For IV 3159 dst_size = get_space_for_phys_dsgl(dnents); 3160 kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE; 3161 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 3162 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <= 3163 SGE_MAX_WR_LEN; 3164 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) : 3165 (sgl_len(snents) * 8); 3166 transhdr_len += temp; 3167 transhdr_len = roundup(transhdr_len, 16); 3168 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 3169 transhdr_len, reqctx->op)) { 3170 3171 atomic_inc(&adap->chcr_stats.fallback); 3172 chcr_aead_common_exit(req); 3173 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 3174 } 3175 skb = alloc_skb(transhdr_len, flags); 3176 if (!skb) { 3177 error = -ENOMEM; 3178 goto err; 3179 } 3180 3181 chcr_req = __skb_put_zero(skb, transhdr_len); 3182 3183 //Offset of tag from end 3184 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; 3185 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 3186 rx_channel_id, 2, 1); 3187 chcr_req->sec_cpl.pldlen = 3188 htonl(req->assoclen + IV + req->cryptlen); 3189 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 3190 assoclen ? 1 + IV : 0, 3191 assoclen ? IV + assoclen : 0, 3192 req->assoclen + IV + 1, 0); 3193 chcr_req->sec_cpl.cipherstop_lo_authinsert = 3194 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1, 3195 temp, temp); 3196 chcr_req->sec_cpl.seqno_numivs = 3197 FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op == 3198 CHCR_ENCRYPT_OP) ? 1 : 0, 3199 CHCR_SCMD_CIPHER_MODE_AES_GCM, 3200 CHCR_SCMD_AUTH_MODE_GHASH, 3201 aeadctx->hmac_ctrl, IV >> 1); 3202 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 3203 0, 0, dst_size); 3204 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 3205 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 3206 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 3207 GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 3208 3209 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 3210 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 3211 /* prepare a 16 byte iv */ 3212 /* S A L T | IV | 0x00000001 */ 3213 if (get_aead_subtype(tfm) == 3214 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 3215 memcpy(ivptr, aeadctx->salt, 4); 3216 memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE); 3217 } else { 3218 memcpy(ivptr, req->iv, GCM_AES_IV_SIZE); 3219 } 3220 put_unaligned_be32(0x01, &ivptr[12]); 3221 ulptx = (struct ulptx_sgl *)(ivptr + 16); 3222 3223 chcr_add_aead_dst_ent(req, phys_cpl, qid); 3224 chcr_add_aead_src_ent(req, ulptx); 3225 atomic_inc(&adap->chcr_stats.aead_rqst); 3226 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 3227 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); 3228 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 3229 transhdr_len, temp, reqctx->verify); 3230 reqctx->skb = skb; 3231 return skb; 3232 3233 err: 3234 chcr_aead_common_exit(req); 3235 return ERR_PTR(error); 3236 } 3237 3238 3239 3240 static int chcr_aead_cra_init(struct crypto_aead *tfm) 3241 { 3242 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3243 struct aead_alg *alg = crypto_aead_alg(tfm); 3244 3245 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, 3246 CRYPTO_ALG_NEED_FALLBACK | 3247 CRYPTO_ALG_ASYNC); 3248 if (IS_ERR(aeadctx->sw_cipher)) 3249 return PTR_ERR(aeadctx->sw_cipher); 3250 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), 3251 sizeof(struct aead_request) + 3252 crypto_aead_reqsize(aeadctx->sw_cipher))); 3253 return chcr_device_init(a_ctx(tfm)); 3254 } 3255 3256 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 3257 { 3258 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3259 3260 crypto_free_aead(aeadctx->sw_cipher); 3261 } 3262 3263 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 3264 unsigned int authsize) 3265 { 3266 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3267 3268 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 3269 aeadctx->mayverify = VERIFY_HW; 3270 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3271 } 3272 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 3273 unsigned int authsize) 3274 { 3275 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3276 u32 maxauth = crypto_aead_maxauthsize(tfm); 3277 3278 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 3279 * true for sha1. authsize == 12 condition should be before 3280 * authsize == (maxauth >> 1) 3281 */ 3282 if (authsize == ICV_4) { 3283 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3284 aeadctx->mayverify = VERIFY_HW; 3285 } else if (authsize == ICV_6) { 3286 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3287 aeadctx->mayverify = VERIFY_HW; 3288 } else if (authsize == ICV_10) { 3289 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3290 aeadctx->mayverify = VERIFY_HW; 3291 } else if (authsize == ICV_12) { 3292 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3293 aeadctx->mayverify = VERIFY_HW; 3294 } else if (authsize == ICV_14) { 3295 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3296 aeadctx->mayverify = VERIFY_HW; 3297 } else if (authsize == (maxauth >> 1)) { 3298 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3299 aeadctx->mayverify = VERIFY_HW; 3300 } else if (authsize == maxauth) { 3301 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3302 aeadctx->mayverify = VERIFY_HW; 3303 } else { 3304 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3305 aeadctx->mayverify = VERIFY_SW; 3306 } 3307 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3308 } 3309 3310 3311 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 3312 { 3313 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3314 3315 switch (authsize) { 3316 case ICV_4: 3317 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3318 aeadctx->mayverify = VERIFY_HW; 3319 break; 3320 case ICV_8: 3321 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3322 aeadctx->mayverify = VERIFY_HW; 3323 break; 3324 case ICV_12: 3325 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3326 aeadctx->mayverify = VERIFY_HW; 3327 break; 3328 case ICV_14: 3329 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3330 aeadctx->mayverify = VERIFY_HW; 3331 break; 3332 case ICV_16: 3333 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3334 aeadctx->mayverify = VERIFY_HW; 3335 break; 3336 case ICV_13: 3337 case ICV_15: 3338 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3339 aeadctx->mayverify = VERIFY_SW; 3340 break; 3341 default: 3342 return -EINVAL; 3343 } 3344 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3345 } 3346 3347 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 3348 unsigned int authsize) 3349 { 3350 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3351 3352 switch (authsize) { 3353 case ICV_8: 3354 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3355 aeadctx->mayverify = VERIFY_HW; 3356 break; 3357 case ICV_12: 3358 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3359 aeadctx->mayverify = VERIFY_HW; 3360 break; 3361 case ICV_16: 3362 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3363 aeadctx->mayverify = VERIFY_HW; 3364 break; 3365 default: 3366 return -EINVAL; 3367 } 3368 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3369 } 3370 3371 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 3372 unsigned int authsize) 3373 { 3374 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3375 3376 switch (authsize) { 3377 case ICV_4: 3378 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3379 aeadctx->mayverify = VERIFY_HW; 3380 break; 3381 case ICV_6: 3382 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3383 aeadctx->mayverify = VERIFY_HW; 3384 break; 3385 case ICV_8: 3386 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3387 aeadctx->mayverify = VERIFY_HW; 3388 break; 3389 case ICV_10: 3390 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3391 aeadctx->mayverify = VERIFY_HW; 3392 break; 3393 case ICV_12: 3394 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3395 aeadctx->mayverify = VERIFY_HW; 3396 break; 3397 case ICV_14: 3398 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3399 aeadctx->mayverify = VERIFY_HW; 3400 break; 3401 case ICV_16: 3402 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3403 aeadctx->mayverify = VERIFY_HW; 3404 break; 3405 default: 3406 return -EINVAL; 3407 } 3408 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3409 } 3410 3411 static int chcr_ccm_common_setkey(struct crypto_aead *aead, 3412 const u8 *key, 3413 unsigned int keylen) 3414 { 3415 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3416 unsigned char ck_size, mk_size; 3417 int key_ctx_size = 0; 3418 3419 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2; 3420 if (keylen == AES_KEYSIZE_128) { 3421 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3422 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; 3423 } else if (keylen == AES_KEYSIZE_192) { 3424 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3425 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 3426 } else if (keylen == AES_KEYSIZE_256) { 3427 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3428 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 3429 } else { 3430 aeadctx->enckey_len = 0; 3431 return -EINVAL; 3432 } 3433 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 3434 key_ctx_size >> 4); 3435 memcpy(aeadctx->key, key, keylen); 3436 aeadctx->enckey_len = keylen; 3437 3438 return 0; 3439 } 3440 3441 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 3442 const u8 *key, 3443 unsigned int keylen) 3444 { 3445 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3446 int error; 3447 3448 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3449 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3450 CRYPTO_TFM_REQ_MASK); 3451 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3452 if (error) 3453 return error; 3454 return chcr_ccm_common_setkey(aead, key, keylen); 3455 } 3456 3457 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 3458 unsigned int keylen) 3459 { 3460 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3461 int error; 3462 3463 if (keylen < 3) { 3464 aeadctx->enckey_len = 0; 3465 return -EINVAL; 3466 } 3467 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3468 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3469 CRYPTO_TFM_REQ_MASK); 3470 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3471 if (error) 3472 return error; 3473 keylen -= 3; 3474 memcpy(aeadctx->salt, key + keylen, 3); 3475 return chcr_ccm_common_setkey(aead, key, keylen); 3476 } 3477 3478 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 3479 unsigned int keylen) 3480 { 3481 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3482 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 3483 unsigned int ck_size; 3484 int ret = 0, key_ctx_size = 0; 3485 struct crypto_aes_ctx aes; 3486 3487 aeadctx->enckey_len = 0; 3488 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3489 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) 3490 & CRYPTO_TFM_REQ_MASK); 3491 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3492 if (ret) 3493 goto out; 3494 3495 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 3496 keylen > 3) { 3497 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 3498 memcpy(aeadctx->salt, key + keylen, 4); 3499 } 3500 if (keylen == AES_KEYSIZE_128) { 3501 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3502 } else if (keylen == AES_KEYSIZE_192) { 3503 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3504 } else if (keylen == AES_KEYSIZE_256) { 3505 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3506 } else { 3507 pr_err("GCM: Invalid key length %d\n", keylen); 3508 ret = -EINVAL; 3509 goto out; 3510 } 3511 3512 memcpy(aeadctx->key, key, keylen); 3513 aeadctx->enckey_len = keylen; 3514 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) + 3515 AEAD_H_SIZE; 3516 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 3517 CHCR_KEYCTX_MAC_KEY_SIZE_128, 3518 0, 0, 3519 key_ctx_size >> 4); 3520 /* Calculate the H = CIPH(K, 0 repeated 16 times). 3521 * It will go in key context 3522 */ 3523 ret = aes_expandkey(&aes, key, keylen); 3524 if (ret) { 3525 aeadctx->enckey_len = 0; 3526 goto out; 3527 } 3528 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 3529 aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h); 3530 memzero_explicit(&aes, sizeof(aes)); 3531 3532 out: 3533 return ret; 3534 } 3535 3536 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 3537 unsigned int keylen) 3538 { 3539 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3540 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3541 /* it contains auth and cipher key both*/ 3542 struct crypto_authenc_keys keys; 3543 unsigned int bs, subtype; 3544 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 3545 int err = 0, i, key_ctx_len = 0; 3546 unsigned char ck_size = 0; 3547 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 3548 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 3549 struct algo_param param; 3550 int align; 3551 u8 *o_ptr = NULL; 3552 3553 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3554 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3555 & CRYPTO_TFM_REQ_MASK); 3556 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3557 if (err) 3558 goto out; 3559 3560 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 3561 goto out; 3562 3563 if (get_alg_config(¶m, max_authsize)) { 3564 pr_err("Unsupported digest size\n"); 3565 goto out; 3566 } 3567 subtype = get_aead_subtype(authenc); 3568 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3569 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3570 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3571 goto out; 3572 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3573 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3574 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3575 } 3576 if (keys.enckeylen == AES_KEYSIZE_128) { 3577 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3578 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3579 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3580 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3581 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3582 } else { 3583 pr_err("Unsupported cipher key\n"); 3584 goto out; 3585 } 3586 3587 /* Copy only encryption key. We use authkey to generate h(ipad) and 3588 * h(opad) so authkey is not needed again. authkeylen size have the 3589 * size of the hash digest size. 3590 */ 3591 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3592 aeadctx->enckey_len = keys.enckeylen; 3593 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3594 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3595 3596 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3597 aeadctx->enckey_len << 3); 3598 } 3599 base_hash = chcr_alloc_shash(max_authsize); 3600 if (IS_ERR(base_hash)) { 3601 pr_err("Base driver cannot be loaded\n"); 3602 goto out; 3603 } 3604 { 3605 SHASH_DESC_ON_STACK(shash, base_hash); 3606 3607 shash->tfm = base_hash; 3608 bs = crypto_shash_blocksize(base_hash); 3609 align = KEYCTX_ALIGN_PAD(max_authsize); 3610 o_ptr = actx->h_iopad + param.result_size + align; 3611 3612 if (keys.authkeylen > bs) { 3613 err = crypto_shash_digest(shash, keys.authkey, 3614 keys.authkeylen, 3615 o_ptr); 3616 if (err) { 3617 pr_err("Base driver cannot be loaded\n"); 3618 goto out; 3619 } 3620 keys.authkeylen = max_authsize; 3621 } else 3622 memcpy(o_ptr, keys.authkey, keys.authkeylen); 3623 3624 /* Compute the ipad-digest*/ 3625 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3626 memcpy(pad, o_ptr, keys.authkeylen); 3627 for (i = 0; i < bs >> 2; i++) 3628 *((unsigned int *)pad + i) ^= IPAD_DATA; 3629 3630 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 3631 max_authsize)) 3632 goto out; 3633 /* Compute the opad-digest */ 3634 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3635 memcpy(pad, o_ptr, keys.authkeylen); 3636 for (i = 0; i < bs >> 2; i++) 3637 *((unsigned int *)pad + i) ^= OPAD_DATA; 3638 3639 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 3640 goto out; 3641 3642 /* convert the ipad and opad digest to network order */ 3643 chcr_change_order(actx->h_iopad, param.result_size); 3644 chcr_change_order(o_ptr, param.result_size); 3645 key_ctx_len = sizeof(struct _key_ctx) + 3646 roundup(keys.enckeylen, 16) + 3647 (param.result_size + align) * 2; 3648 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 3649 0, 1, key_ctx_len >> 4); 3650 actx->auth_mode = param.auth_mode; 3651 chcr_free_shash(base_hash); 3652 3653 memzero_explicit(&keys, sizeof(keys)); 3654 return 0; 3655 } 3656 out: 3657 aeadctx->enckey_len = 0; 3658 memzero_explicit(&keys, sizeof(keys)); 3659 if (!IS_ERR(base_hash)) 3660 chcr_free_shash(base_hash); 3661 return -EINVAL; 3662 } 3663 3664 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 3665 const u8 *key, unsigned int keylen) 3666 { 3667 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3668 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3669 struct crypto_authenc_keys keys; 3670 int err; 3671 /* it contains auth and cipher key both*/ 3672 unsigned int subtype; 3673 int key_ctx_len = 0; 3674 unsigned char ck_size = 0; 3675 3676 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3677 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3678 & CRYPTO_TFM_REQ_MASK); 3679 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3680 if (err) 3681 goto out; 3682 3683 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 3684 goto out; 3685 3686 subtype = get_aead_subtype(authenc); 3687 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3688 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3689 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3690 goto out; 3691 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3692 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3693 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3694 } 3695 if (keys.enckeylen == AES_KEYSIZE_128) { 3696 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3697 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3698 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3699 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3700 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3701 } else { 3702 pr_err("Unsupported cipher key %d\n", keys.enckeylen); 3703 goto out; 3704 } 3705 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3706 aeadctx->enckey_len = keys.enckeylen; 3707 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3708 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3709 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3710 aeadctx->enckey_len << 3); 3711 } 3712 key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16); 3713 3714 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 3715 0, key_ctx_len >> 4); 3716 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 3717 memzero_explicit(&keys, sizeof(keys)); 3718 return 0; 3719 out: 3720 aeadctx->enckey_len = 0; 3721 memzero_explicit(&keys, sizeof(keys)); 3722 return -EINVAL; 3723 } 3724 3725 static int chcr_aead_op(struct aead_request *req, 3726 int size, 3727 create_wr_t create_wr_fn) 3728 { 3729 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3730 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3731 struct chcr_context *ctx = a_ctx(tfm); 3732 struct uld_ctx *u_ctx = ULD_CTX(ctx); 3733 struct sk_buff *skb; 3734 struct chcr_dev *cdev; 3735 3736 cdev = a_ctx(tfm)->dev; 3737 if (!cdev) { 3738 pr_err("%s : No crypto device.\n", __func__); 3739 return -ENXIO; 3740 } 3741 3742 if (chcr_inc_wrcount(cdev)) { 3743 /* Detach state for CHCR means lldi or padap is freed. 3744 * We cannot increment fallback here. 3745 */ 3746 return chcr_aead_fallback(req, reqctx->op); 3747 } 3748 3749 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 3750 reqctx->txqidx) && 3751 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) { 3752 chcr_dec_wrcount(cdev); 3753 return -ENOSPC; 3754 } 3755 3756 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 3757 crypto_ipsec_check_assoclen(req->assoclen) != 0) { 3758 pr_err("RFC4106: Invalid value of assoclen %d\n", 3759 req->assoclen); 3760 return -EINVAL; 3761 } 3762 3763 /* Form a WR from req */ 3764 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size); 3765 3766 if (IS_ERR_OR_NULL(skb)) { 3767 chcr_dec_wrcount(cdev); 3768 return PTR_ERR_OR_ZERO(skb); 3769 } 3770 3771 skb->dev = u_ctx->lldi.ports[0]; 3772 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 3773 chcr_send_wr(skb); 3774 return -EINPROGRESS; 3775 } 3776 3777 static int chcr_aead_encrypt(struct aead_request *req) 3778 { 3779 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3780 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3781 struct chcr_context *ctx = a_ctx(tfm); 3782 unsigned int cpu; 3783 3784 cpu = get_cpu(); 3785 reqctx->txqidx = cpu % ctx->ntxq; 3786 reqctx->rxqidx = cpu % ctx->nrxq; 3787 put_cpu(); 3788 3789 reqctx->verify = VERIFY_HW; 3790 reqctx->op = CHCR_ENCRYPT_OP; 3791 3792 switch (get_aead_subtype(tfm)) { 3793 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3794 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3795 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3796 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3797 return chcr_aead_op(req, 0, create_authenc_wr); 3798 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3799 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3800 return chcr_aead_op(req, 0, create_aead_ccm_wr); 3801 default: 3802 return chcr_aead_op(req, 0, create_gcm_wr); 3803 } 3804 } 3805 3806 static int chcr_aead_decrypt(struct aead_request *req) 3807 { 3808 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3809 struct chcr_context *ctx = a_ctx(tfm); 3810 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 3811 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3812 int size; 3813 unsigned int cpu; 3814 3815 cpu = get_cpu(); 3816 reqctx->txqidx = cpu % ctx->ntxq; 3817 reqctx->rxqidx = cpu % ctx->nrxq; 3818 put_cpu(); 3819 3820 if (aeadctx->mayverify == VERIFY_SW) { 3821 size = crypto_aead_maxauthsize(tfm); 3822 reqctx->verify = VERIFY_SW; 3823 } else { 3824 size = 0; 3825 reqctx->verify = VERIFY_HW; 3826 } 3827 reqctx->op = CHCR_DECRYPT_OP; 3828 switch (get_aead_subtype(tfm)) { 3829 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3830 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3831 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3832 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3833 return chcr_aead_op(req, size, create_authenc_wr); 3834 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3835 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3836 return chcr_aead_op(req, size, create_aead_ccm_wr); 3837 default: 3838 return chcr_aead_op(req, size, create_gcm_wr); 3839 } 3840 } 3841 3842 static struct chcr_alg_template driver_algs[] = { 3843 /* AES-CBC */ 3844 { 3845 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, 3846 .is_registered = 0, 3847 .alg.skcipher = { 3848 .base.cra_name = "cbc(aes)", 3849 .base.cra_driver_name = "cbc-aes-chcr", 3850 .base.cra_blocksize = AES_BLOCK_SIZE, 3851 3852 .init = chcr_init_tfm, 3853 .exit = chcr_exit_tfm, 3854 .min_keysize = AES_MIN_KEY_SIZE, 3855 .max_keysize = AES_MAX_KEY_SIZE, 3856 .ivsize = AES_BLOCK_SIZE, 3857 .setkey = chcr_aes_cbc_setkey, 3858 .encrypt = chcr_aes_encrypt, 3859 .decrypt = chcr_aes_decrypt, 3860 } 3861 }, 3862 { 3863 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, 3864 .is_registered = 0, 3865 .alg.skcipher = { 3866 .base.cra_name = "xts(aes)", 3867 .base.cra_driver_name = "xts-aes-chcr", 3868 .base.cra_blocksize = AES_BLOCK_SIZE, 3869 3870 .init = chcr_init_tfm, 3871 .exit = chcr_exit_tfm, 3872 .min_keysize = 2 * AES_MIN_KEY_SIZE, 3873 .max_keysize = 2 * AES_MAX_KEY_SIZE, 3874 .ivsize = AES_BLOCK_SIZE, 3875 .setkey = chcr_aes_xts_setkey, 3876 .encrypt = chcr_aes_encrypt, 3877 .decrypt = chcr_aes_decrypt, 3878 } 3879 }, 3880 { 3881 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, 3882 .is_registered = 0, 3883 .alg.skcipher = { 3884 .base.cra_name = "ctr(aes)", 3885 .base.cra_driver_name = "ctr-aes-chcr", 3886 .base.cra_blocksize = 1, 3887 3888 .init = chcr_init_tfm, 3889 .exit = chcr_exit_tfm, 3890 .min_keysize = AES_MIN_KEY_SIZE, 3891 .max_keysize = AES_MAX_KEY_SIZE, 3892 .ivsize = AES_BLOCK_SIZE, 3893 .setkey = chcr_aes_ctr_setkey, 3894 .encrypt = chcr_aes_encrypt, 3895 .decrypt = chcr_aes_decrypt, 3896 } 3897 }, 3898 { 3899 .type = CRYPTO_ALG_TYPE_SKCIPHER | 3900 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, 3901 .is_registered = 0, 3902 .alg.skcipher = { 3903 .base.cra_name = "rfc3686(ctr(aes))", 3904 .base.cra_driver_name = "rfc3686-ctr-aes-chcr", 3905 .base.cra_blocksize = 1, 3906 3907 .init = chcr_rfc3686_init, 3908 .exit = chcr_exit_tfm, 3909 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3910 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3911 .ivsize = CTR_RFC3686_IV_SIZE, 3912 .setkey = chcr_aes_rfc3686_setkey, 3913 .encrypt = chcr_aes_encrypt, 3914 .decrypt = chcr_aes_decrypt, 3915 } 3916 }, 3917 /* SHA */ 3918 { 3919 .type = CRYPTO_ALG_TYPE_AHASH, 3920 .is_registered = 0, 3921 .alg.hash = { 3922 .halg.digestsize = SHA1_DIGEST_SIZE, 3923 .halg.base = { 3924 .cra_name = "sha1", 3925 .cra_driver_name = "sha1-chcr", 3926 .cra_blocksize = SHA1_BLOCK_SIZE, 3927 } 3928 } 3929 }, 3930 { 3931 .type = CRYPTO_ALG_TYPE_AHASH, 3932 .is_registered = 0, 3933 .alg.hash = { 3934 .halg.digestsize = SHA256_DIGEST_SIZE, 3935 .halg.base = { 3936 .cra_name = "sha256", 3937 .cra_driver_name = "sha256-chcr", 3938 .cra_blocksize = SHA256_BLOCK_SIZE, 3939 } 3940 } 3941 }, 3942 { 3943 .type = CRYPTO_ALG_TYPE_AHASH, 3944 .is_registered = 0, 3945 .alg.hash = { 3946 .halg.digestsize = SHA224_DIGEST_SIZE, 3947 .halg.base = { 3948 .cra_name = "sha224", 3949 .cra_driver_name = "sha224-chcr", 3950 .cra_blocksize = SHA224_BLOCK_SIZE, 3951 } 3952 } 3953 }, 3954 { 3955 .type = CRYPTO_ALG_TYPE_AHASH, 3956 .is_registered = 0, 3957 .alg.hash = { 3958 .halg.digestsize = SHA384_DIGEST_SIZE, 3959 .halg.base = { 3960 .cra_name = "sha384", 3961 .cra_driver_name = "sha384-chcr", 3962 .cra_blocksize = SHA384_BLOCK_SIZE, 3963 } 3964 } 3965 }, 3966 { 3967 .type = CRYPTO_ALG_TYPE_AHASH, 3968 .is_registered = 0, 3969 .alg.hash = { 3970 .halg.digestsize = SHA512_DIGEST_SIZE, 3971 .halg.base = { 3972 .cra_name = "sha512", 3973 .cra_driver_name = "sha512-chcr", 3974 .cra_blocksize = SHA512_BLOCK_SIZE, 3975 } 3976 } 3977 }, 3978 /* HMAC */ 3979 { 3980 .type = CRYPTO_ALG_TYPE_HMAC, 3981 .is_registered = 0, 3982 .alg.hash = { 3983 .halg.digestsize = SHA1_DIGEST_SIZE, 3984 .halg.base = { 3985 .cra_name = "hmac(sha1)", 3986 .cra_driver_name = "hmac-sha1-chcr", 3987 .cra_blocksize = SHA1_BLOCK_SIZE, 3988 } 3989 } 3990 }, 3991 { 3992 .type = CRYPTO_ALG_TYPE_HMAC, 3993 .is_registered = 0, 3994 .alg.hash = { 3995 .halg.digestsize = SHA224_DIGEST_SIZE, 3996 .halg.base = { 3997 .cra_name = "hmac(sha224)", 3998 .cra_driver_name = "hmac-sha224-chcr", 3999 .cra_blocksize = SHA224_BLOCK_SIZE, 4000 } 4001 } 4002 }, 4003 { 4004 .type = CRYPTO_ALG_TYPE_HMAC, 4005 .is_registered = 0, 4006 .alg.hash = { 4007 .halg.digestsize = SHA256_DIGEST_SIZE, 4008 .halg.base = { 4009 .cra_name = "hmac(sha256)", 4010 .cra_driver_name = "hmac-sha256-chcr", 4011 .cra_blocksize = SHA256_BLOCK_SIZE, 4012 } 4013 } 4014 }, 4015 { 4016 .type = CRYPTO_ALG_TYPE_HMAC, 4017 .is_registered = 0, 4018 .alg.hash = { 4019 .halg.digestsize = SHA384_DIGEST_SIZE, 4020 .halg.base = { 4021 .cra_name = "hmac(sha384)", 4022 .cra_driver_name = "hmac-sha384-chcr", 4023 .cra_blocksize = SHA384_BLOCK_SIZE, 4024 } 4025 } 4026 }, 4027 { 4028 .type = CRYPTO_ALG_TYPE_HMAC, 4029 .is_registered = 0, 4030 .alg.hash = { 4031 .halg.digestsize = SHA512_DIGEST_SIZE, 4032 .halg.base = { 4033 .cra_name = "hmac(sha512)", 4034 .cra_driver_name = "hmac-sha512-chcr", 4035 .cra_blocksize = SHA512_BLOCK_SIZE, 4036 } 4037 } 4038 }, 4039 /* Add AEAD Algorithms */ 4040 { 4041 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 4042 .is_registered = 0, 4043 .alg.aead = { 4044 .base = { 4045 .cra_name = "gcm(aes)", 4046 .cra_driver_name = "gcm-aes-chcr", 4047 .cra_blocksize = 1, 4048 .cra_priority = CHCR_AEAD_PRIORITY, 4049 .cra_ctxsize = sizeof(struct chcr_context) + 4050 sizeof(struct chcr_aead_ctx) + 4051 sizeof(struct chcr_gcm_ctx), 4052 }, 4053 .ivsize = GCM_AES_IV_SIZE, 4054 .maxauthsize = GHASH_DIGEST_SIZE, 4055 .setkey = chcr_gcm_setkey, 4056 .setauthsize = chcr_gcm_setauthsize, 4057 } 4058 }, 4059 { 4060 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 4061 .is_registered = 0, 4062 .alg.aead = { 4063 .base = { 4064 .cra_name = "rfc4106(gcm(aes))", 4065 .cra_driver_name = "rfc4106-gcm-aes-chcr", 4066 .cra_blocksize = 1, 4067 .cra_priority = CHCR_AEAD_PRIORITY + 1, 4068 .cra_ctxsize = sizeof(struct chcr_context) + 4069 sizeof(struct chcr_aead_ctx) + 4070 sizeof(struct chcr_gcm_ctx), 4071 4072 }, 4073 .ivsize = GCM_RFC4106_IV_SIZE, 4074 .maxauthsize = GHASH_DIGEST_SIZE, 4075 .setkey = chcr_gcm_setkey, 4076 .setauthsize = chcr_4106_4309_setauthsize, 4077 } 4078 }, 4079 { 4080 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 4081 .is_registered = 0, 4082 .alg.aead = { 4083 .base = { 4084 .cra_name = "ccm(aes)", 4085 .cra_driver_name = "ccm-aes-chcr", 4086 .cra_blocksize = 1, 4087 .cra_priority = CHCR_AEAD_PRIORITY, 4088 .cra_ctxsize = sizeof(struct chcr_context) + 4089 sizeof(struct chcr_aead_ctx), 4090 4091 }, 4092 .ivsize = AES_BLOCK_SIZE, 4093 .maxauthsize = GHASH_DIGEST_SIZE, 4094 .setkey = chcr_aead_ccm_setkey, 4095 .setauthsize = chcr_ccm_setauthsize, 4096 } 4097 }, 4098 { 4099 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 4100 .is_registered = 0, 4101 .alg.aead = { 4102 .base = { 4103 .cra_name = "rfc4309(ccm(aes))", 4104 .cra_driver_name = "rfc4309-ccm-aes-chcr", 4105 .cra_blocksize = 1, 4106 .cra_priority = CHCR_AEAD_PRIORITY + 1, 4107 .cra_ctxsize = sizeof(struct chcr_context) + 4108 sizeof(struct chcr_aead_ctx), 4109 4110 }, 4111 .ivsize = 8, 4112 .maxauthsize = GHASH_DIGEST_SIZE, 4113 .setkey = chcr_aead_rfc4309_setkey, 4114 .setauthsize = chcr_4106_4309_setauthsize, 4115 } 4116 }, 4117 { 4118 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4119 .is_registered = 0, 4120 .alg.aead = { 4121 .base = { 4122 .cra_name = "authenc(hmac(sha1),cbc(aes))", 4123 .cra_driver_name = 4124 "authenc-hmac-sha1-cbc-aes-chcr", 4125 .cra_blocksize = AES_BLOCK_SIZE, 4126 .cra_priority = CHCR_AEAD_PRIORITY, 4127 .cra_ctxsize = sizeof(struct chcr_context) + 4128 sizeof(struct chcr_aead_ctx) + 4129 sizeof(struct chcr_authenc_ctx), 4130 4131 }, 4132 .ivsize = AES_BLOCK_SIZE, 4133 .maxauthsize = SHA1_DIGEST_SIZE, 4134 .setkey = chcr_authenc_setkey, 4135 .setauthsize = chcr_authenc_setauthsize, 4136 } 4137 }, 4138 { 4139 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4140 .is_registered = 0, 4141 .alg.aead = { 4142 .base = { 4143 4144 .cra_name = "authenc(hmac(sha256),cbc(aes))", 4145 .cra_driver_name = 4146 "authenc-hmac-sha256-cbc-aes-chcr", 4147 .cra_blocksize = AES_BLOCK_SIZE, 4148 .cra_priority = CHCR_AEAD_PRIORITY, 4149 .cra_ctxsize = sizeof(struct chcr_context) + 4150 sizeof(struct chcr_aead_ctx) + 4151 sizeof(struct chcr_authenc_ctx), 4152 4153 }, 4154 .ivsize = AES_BLOCK_SIZE, 4155 .maxauthsize = SHA256_DIGEST_SIZE, 4156 .setkey = chcr_authenc_setkey, 4157 .setauthsize = chcr_authenc_setauthsize, 4158 } 4159 }, 4160 { 4161 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4162 .is_registered = 0, 4163 .alg.aead = { 4164 .base = { 4165 .cra_name = "authenc(hmac(sha224),cbc(aes))", 4166 .cra_driver_name = 4167 "authenc-hmac-sha224-cbc-aes-chcr", 4168 .cra_blocksize = AES_BLOCK_SIZE, 4169 .cra_priority = CHCR_AEAD_PRIORITY, 4170 .cra_ctxsize = sizeof(struct chcr_context) + 4171 sizeof(struct chcr_aead_ctx) + 4172 sizeof(struct chcr_authenc_ctx), 4173 }, 4174 .ivsize = AES_BLOCK_SIZE, 4175 .maxauthsize = SHA224_DIGEST_SIZE, 4176 .setkey = chcr_authenc_setkey, 4177 .setauthsize = chcr_authenc_setauthsize, 4178 } 4179 }, 4180 { 4181 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4182 .is_registered = 0, 4183 .alg.aead = { 4184 .base = { 4185 .cra_name = "authenc(hmac(sha384),cbc(aes))", 4186 .cra_driver_name = 4187 "authenc-hmac-sha384-cbc-aes-chcr", 4188 .cra_blocksize = AES_BLOCK_SIZE, 4189 .cra_priority = CHCR_AEAD_PRIORITY, 4190 .cra_ctxsize = sizeof(struct chcr_context) + 4191 sizeof(struct chcr_aead_ctx) + 4192 sizeof(struct chcr_authenc_ctx), 4193 4194 }, 4195 .ivsize = AES_BLOCK_SIZE, 4196 .maxauthsize = SHA384_DIGEST_SIZE, 4197 .setkey = chcr_authenc_setkey, 4198 .setauthsize = chcr_authenc_setauthsize, 4199 } 4200 }, 4201 { 4202 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4203 .is_registered = 0, 4204 .alg.aead = { 4205 .base = { 4206 .cra_name = "authenc(hmac(sha512),cbc(aes))", 4207 .cra_driver_name = 4208 "authenc-hmac-sha512-cbc-aes-chcr", 4209 .cra_blocksize = AES_BLOCK_SIZE, 4210 .cra_priority = CHCR_AEAD_PRIORITY, 4211 .cra_ctxsize = sizeof(struct chcr_context) + 4212 sizeof(struct chcr_aead_ctx) + 4213 sizeof(struct chcr_authenc_ctx), 4214 4215 }, 4216 .ivsize = AES_BLOCK_SIZE, 4217 .maxauthsize = SHA512_DIGEST_SIZE, 4218 .setkey = chcr_authenc_setkey, 4219 .setauthsize = chcr_authenc_setauthsize, 4220 } 4221 }, 4222 { 4223 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL, 4224 .is_registered = 0, 4225 .alg.aead = { 4226 .base = { 4227 .cra_name = "authenc(digest_null,cbc(aes))", 4228 .cra_driver_name = 4229 "authenc-digest_null-cbc-aes-chcr", 4230 .cra_blocksize = AES_BLOCK_SIZE, 4231 .cra_priority = CHCR_AEAD_PRIORITY, 4232 .cra_ctxsize = sizeof(struct chcr_context) + 4233 sizeof(struct chcr_aead_ctx) + 4234 sizeof(struct chcr_authenc_ctx), 4235 4236 }, 4237 .ivsize = AES_BLOCK_SIZE, 4238 .maxauthsize = 0, 4239 .setkey = chcr_aead_digest_null_setkey, 4240 .setauthsize = chcr_authenc_null_setauthsize, 4241 } 4242 }, 4243 { 4244 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4245 .is_registered = 0, 4246 .alg.aead = { 4247 .base = { 4248 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 4249 .cra_driver_name = 4250 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr", 4251 .cra_blocksize = 1, 4252 .cra_priority = CHCR_AEAD_PRIORITY, 4253 .cra_ctxsize = sizeof(struct chcr_context) + 4254 sizeof(struct chcr_aead_ctx) + 4255 sizeof(struct chcr_authenc_ctx), 4256 4257 }, 4258 .ivsize = CTR_RFC3686_IV_SIZE, 4259 .maxauthsize = SHA1_DIGEST_SIZE, 4260 .setkey = chcr_authenc_setkey, 4261 .setauthsize = chcr_authenc_setauthsize, 4262 } 4263 }, 4264 { 4265 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4266 .is_registered = 0, 4267 .alg.aead = { 4268 .base = { 4269 4270 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 4271 .cra_driver_name = 4272 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr", 4273 .cra_blocksize = 1, 4274 .cra_priority = CHCR_AEAD_PRIORITY, 4275 .cra_ctxsize = sizeof(struct chcr_context) + 4276 sizeof(struct chcr_aead_ctx) + 4277 sizeof(struct chcr_authenc_ctx), 4278 4279 }, 4280 .ivsize = CTR_RFC3686_IV_SIZE, 4281 .maxauthsize = SHA256_DIGEST_SIZE, 4282 .setkey = chcr_authenc_setkey, 4283 .setauthsize = chcr_authenc_setauthsize, 4284 } 4285 }, 4286 { 4287 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4288 .is_registered = 0, 4289 .alg.aead = { 4290 .base = { 4291 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 4292 .cra_driver_name = 4293 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr", 4294 .cra_blocksize = 1, 4295 .cra_priority = CHCR_AEAD_PRIORITY, 4296 .cra_ctxsize = sizeof(struct chcr_context) + 4297 sizeof(struct chcr_aead_ctx) + 4298 sizeof(struct chcr_authenc_ctx), 4299 }, 4300 .ivsize = CTR_RFC3686_IV_SIZE, 4301 .maxauthsize = SHA224_DIGEST_SIZE, 4302 .setkey = chcr_authenc_setkey, 4303 .setauthsize = chcr_authenc_setauthsize, 4304 } 4305 }, 4306 { 4307 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4308 .is_registered = 0, 4309 .alg.aead = { 4310 .base = { 4311 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 4312 .cra_driver_name = 4313 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr", 4314 .cra_blocksize = 1, 4315 .cra_priority = CHCR_AEAD_PRIORITY, 4316 .cra_ctxsize = sizeof(struct chcr_context) + 4317 sizeof(struct chcr_aead_ctx) + 4318 sizeof(struct chcr_authenc_ctx), 4319 4320 }, 4321 .ivsize = CTR_RFC3686_IV_SIZE, 4322 .maxauthsize = SHA384_DIGEST_SIZE, 4323 .setkey = chcr_authenc_setkey, 4324 .setauthsize = chcr_authenc_setauthsize, 4325 } 4326 }, 4327 { 4328 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4329 .is_registered = 0, 4330 .alg.aead = { 4331 .base = { 4332 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 4333 .cra_driver_name = 4334 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr", 4335 .cra_blocksize = 1, 4336 .cra_priority = CHCR_AEAD_PRIORITY, 4337 .cra_ctxsize = sizeof(struct chcr_context) + 4338 sizeof(struct chcr_aead_ctx) + 4339 sizeof(struct chcr_authenc_ctx), 4340 4341 }, 4342 .ivsize = CTR_RFC3686_IV_SIZE, 4343 .maxauthsize = SHA512_DIGEST_SIZE, 4344 .setkey = chcr_authenc_setkey, 4345 .setauthsize = chcr_authenc_setauthsize, 4346 } 4347 }, 4348 { 4349 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL, 4350 .is_registered = 0, 4351 .alg.aead = { 4352 .base = { 4353 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))", 4354 .cra_driver_name = 4355 "authenc-digest_null-rfc3686-ctr-aes-chcr", 4356 .cra_blocksize = 1, 4357 .cra_priority = CHCR_AEAD_PRIORITY, 4358 .cra_ctxsize = sizeof(struct chcr_context) + 4359 sizeof(struct chcr_aead_ctx) + 4360 sizeof(struct chcr_authenc_ctx), 4361 4362 }, 4363 .ivsize = CTR_RFC3686_IV_SIZE, 4364 .maxauthsize = 0, 4365 .setkey = chcr_aead_digest_null_setkey, 4366 .setauthsize = chcr_authenc_null_setauthsize, 4367 } 4368 }, 4369 }; 4370 4371 /* 4372 * chcr_unregister_alg - Deregister crypto algorithms with 4373 * kernel framework. 4374 */ 4375 static int chcr_unregister_alg(void) 4376 { 4377 int i; 4378 4379 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4380 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4381 case CRYPTO_ALG_TYPE_SKCIPHER: 4382 if (driver_algs[i].is_registered && refcount_read( 4383 &driver_algs[i].alg.skcipher.base.cra_refcnt) 4384 == 1) { 4385 crypto_unregister_skcipher( 4386 &driver_algs[i].alg.skcipher); 4387 driver_algs[i].is_registered = 0; 4388 } 4389 break; 4390 case CRYPTO_ALG_TYPE_AEAD: 4391 if (driver_algs[i].is_registered && refcount_read( 4392 &driver_algs[i].alg.aead.base.cra_refcnt) == 1) { 4393 crypto_unregister_aead( 4394 &driver_algs[i].alg.aead); 4395 driver_algs[i].is_registered = 0; 4396 } 4397 break; 4398 case CRYPTO_ALG_TYPE_AHASH: 4399 if (driver_algs[i].is_registered && refcount_read( 4400 &driver_algs[i].alg.hash.halg.base.cra_refcnt) 4401 == 1) { 4402 crypto_unregister_ahash( 4403 &driver_algs[i].alg.hash); 4404 driver_algs[i].is_registered = 0; 4405 } 4406 break; 4407 } 4408 } 4409 return 0; 4410 } 4411 4412 #define SZ_AHASH_CTX sizeof(struct chcr_context) 4413 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 4414 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 4415 4416 /* 4417 * chcr_register_alg - Register crypto algorithms with kernel framework. 4418 */ 4419 static int chcr_register_alg(void) 4420 { 4421 struct crypto_alg ai; 4422 struct ahash_alg *a_hash; 4423 int err = 0, i; 4424 char *name = NULL; 4425 4426 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4427 if (driver_algs[i].is_registered) 4428 continue; 4429 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4430 case CRYPTO_ALG_TYPE_SKCIPHER: 4431 driver_algs[i].alg.skcipher.base.cra_priority = 4432 CHCR_CRA_PRIORITY; 4433 driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE; 4434 driver_algs[i].alg.skcipher.base.cra_flags = 4435 CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC | 4436 CRYPTO_ALG_ALLOCATES_MEMORY | 4437 CRYPTO_ALG_NEED_FALLBACK; 4438 driver_algs[i].alg.skcipher.base.cra_ctxsize = 4439 sizeof(struct chcr_context) + 4440 sizeof(struct ablk_ctx); 4441 driver_algs[i].alg.skcipher.base.cra_alignmask = 0; 4442 4443 err = crypto_register_skcipher(&driver_algs[i].alg.skcipher); 4444 name = driver_algs[i].alg.skcipher.base.cra_driver_name; 4445 break; 4446 case CRYPTO_ALG_TYPE_AEAD: 4447 driver_algs[i].alg.aead.base.cra_flags = 4448 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK | 4449 CRYPTO_ALG_ALLOCATES_MEMORY; 4450 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 4451 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 4452 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 4453 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 4454 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 4455 err = crypto_register_aead(&driver_algs[i].alg.aead); 4456 name = driver_algs[i].alg.aead.base.cra_driver_name; 4457 break; 4458 case CRYPTO_ALG_TYPE_AHASH: 4459 a_hash = &driver_algs[i].alg.hash; 4460 a_hash->update = chcr_ahash_update; 4461 a_hash->final = chcr_ahash_final; 4462 a_hash->finup = chcr_ahash_finup; 4463 a_hash->digest = chcr_ahash_digest; 4464 a_hash->export = chcr_ahash_export; 4465 a_hash->import = chcr_ahash_import; 4466 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 4467 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 4468 a_hash->halg.base.cra_module = THIS_MODULE; 4469 a_hash->halg.base.cra_flags = 4470 CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4471 a_hash->halg.base.cra_alignmask = 0; 4472 a_hash->halg.base.cra_exit = NULL; 4473 4474 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 4475 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 4476 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 4477 a_hash->init = chcr_hmac_init; 4478 a_hash->setkey = chcr_ahash_setkey; 4479 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 4480 } else { 4481 a_hash->init = chcr_sha_init; 4482 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 4483 a_hash->halg.base.cra_init = chcr_sha_cra_init; 4484 } 4485 err = crypto_register_ahash(&driver_algs[i].alg.hash); 4486 ai = driver_algs[i].alg.hash.halg.base; 4487 name = ai.cra_driver_name; 4488 break; 4489 } 4490 if (err) { 4491 pr_err("%s : Algorithm registration failed\n", name); 4492 goto register_err; 4493 } else { 4494 driver_algs[i].is_registered = 1; 4495 } 4496 } 4497 return 0; 4498 4499 register_err: 4500 chcr_unregister_alg(); 4501 return err; 4502 } 4503 4504 /* 4505 * start_crypto - Register the crypto algorithms. 4506 * This should called once when the first device comesup. After this 4507 * kernel will start calling driver APIs for crypto operations. 4508 */ 4509 int start_crypto(void) 4510 { 4511 return chcr_register_alg(); 4512 } 4513 4514 /* 4515 * stop_crypto - Deregister all the crypto algorithms with kernel. 4516 * This should be called once when the last device goes down. After this 4517 * kernel will not call the driver API for crypto operations. 4518 */ 4519 int stop_crypto(void) 4520 { 4521 chcr_unregister_alg(); 4522 return 0; 4523 } 4524