1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/skbuff.h> 48 #include <linux/rtnetlink.h> 49 #include <linux/highmem.h> 50 #include <linux/scatterlist.h> 51 52 #include <crypto/aes.h> 53 #include <crypto/algapi.h> 54 #include <crypto/hash.h> 55 #include <crypto/gcm.h> 56 #include <crypto/sha1.h> 57 #include <crypto/sha2.h> 58 #include <crypto/authenc.h> 59 #include <crypto/ctr.h> 60 #include <crypto/gf128mul.h> 61 #include <crypto/internal/aead.h> 62 #include <crypto/null.h> 63 #include <crypto/internal/skcipher.h> 64 #include <crypto/aead.h> 65 #include <crypto/scatterwalk.h> 66 #include <crypto/internal/hash.h> 67 68 #include "t4fw_api.h" 69 #include "t4_msg.h" 70 #include "chcr_core.h" 71 #include "chcr_algo.h" 72 #include "chcr_crypto.h" 73 74 #define IV AES_BLOCK_SIZE 75 76 static unsigned int sgl_ent_len[] = { 77 0, 0, 16, 24, 40, 48, 64, 72, 88, 78 96, 112, 120, 136, 144, 160, 168, 184, 79 192, 208, 216, 232, 240, 256, 264, 280, 80 288, 304, 312, 328, 336, 352, 360, 376 81 }; 82 83 static unsigned int dsgl_ent_len[] = { 84 0, 32, 32, 48, 48, 64, 64, 80, 80, 85 112, 112, 128, 128, 144, 144, 160, 160, 86 192, 192, 208, 208, 224, 224, 240, 240, 87 272, 272, 288, 288, 304, 304, 320, 320 88 }; 89 90 static u32 round_constant[11] = { 91 0x01000000, 0x02000000, 0x04000000, 0x08000000, 92 0x10000000, 0x20000000, 0x40000000, 0x80000000, 93 0x1B000000, 0x36000000, 0x6C000000 94 }; 95 96 static int chcr_handle_cipher_resp(struct skcipher_request *req, 97 unsigned char *input, int err); 98 99 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 100 { 101 return ctx->crypto_ctx->aeadctx; 102 } 103 104 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 105 { 106 return ctx->crypto_ctx->ablkctx; 107 } 108 109 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 110 { 111 return ctx->crypto_ctx->hmacctx; 112 } 113 114 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 115 { 116 return gctx->ctx->gcm; 117 } 118 119 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 120 { 121 return gctx->ctx->authenc; 122 } 123 124 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 125 { 126 return container_of(ctx->dev, struct uld_ctx, dev); 127 } 128 129 static inline int is_ofld_imm(const struct sk_buff *skb) 130 { 131 return (skb->len <= SGE_MAX_WR_LEN); 132 } 133 134 static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx) 135 { 136 memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr)); 137 } 138 139 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, 140 unsigned int entlen, 141 unsigned int skip) 142 { 143 int nents = 0; 144 unsigned int less; 145 unsigned int skip_len = 0; 146 147 while (sg && skip) { 148 if (sg_dma_len(sg) <= skip) { 149 skip -= sg_dma_len(sg); 150 skip_len = 0; 151 sg = sg_next(sg); 152 } else { 153 skip_len = skip; 154 skip = 0; 155 } 156 } 157 158 while (sg && reqlen) { 159 less = min(reqlen, sg_dma_len(sg) - skip_len); 160 nents += DIV_ROUND_UP(less, entlen); 161 reqlen -= less; 162 skip_len = 0; 163 sg = sg_next(sg); 164 } 165 return nents; 166 } 167 168 static inline int get_aead_subtype(struct crypto_aead *aead) 169 { 170 struct aead_alg *alg = crypto_aead_alg(aead); 171 struct chcr_alg_template *chcr_crypto_alg = 172 container_of(alg, struct chcr_alg_template, alg.aead); 173 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 174 } 175 176 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 177 { 178 u8 temp[SHA512_DIGEST_SIZE]; 179 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 180 int authsize = crypto_aead_authsize(tfm); 181 struct cpl_fw6_pld *fw6_pld; 182 int cmp = 0; 183 184 fw6_pld = (struct cpl_fw6_pld *)input; 185 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 186 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 187 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); 188 } else { 189 190 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 191 authsize, req->assoclen + 192 req->cryptlen - authsize); 193 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); 194 } 195 if (cmp) 196 *err = -EBADMSG; 197 else 198 *err = 0; 199 } 200 201 static int chcr_inc_wrcount(struct chcr_dev *dev) 202 { 203 if (dev->state == CHCR_DETACH) 204 return 1; 205 atomic_inc(&dev->inflight); 206 return 0; 207 } 208 209 static inline void chcr_dec_wrcount(struct chcr_dev *dev) 210 { 211 atomic_dec(&dev->inflight); 212 } 213 214 static inline int chcr_handle_aead_resp(struct aead_request *req, 215 unsigned char *input, 216 int err) 217 { 218 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 219 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 220 struct chcr_dev *dev = a_ctx(tfm)->dev; 221 222 chcr_aead_common_exit(req); 223 if (reqctx->verify == VERIFY_SW) { 224 chcr_verify_tag(req, input, &err); 225 reqctx->verify = VERIFY_HW; 226 } 227 chcr_dec_wrcount(dev); 228 req->base.complete(&req->base, err); 229 230 return err; 231 } 232 233 static void get_aes_decrypt_key(unsigned char *dec_key, 234 const unsigned char *key, 235 unsigned int keylength) 236 { 237 u32 temp; 238 u32 w_ring[MAX_NK]; 239 int i, j, k; 240 u8 nr, nk; 241 242 switch (keylength) { 243 case AES_KEYLENGTH_128BIT: 244 nk = KEYLENGTH_4BYTES; 245 nr = NUMBER_OF_ROUNDS_10; 246 break; 247 case AES_KEYLENGTH_192BIT: 248 nk = KEYLENGTH_6BYTES; 249 nr = NUMBER_OF_ROUNDS_12; 250 break; 251 case AES_KEYLENGTH_256BIT: 252 nk = KEYLENGTH_8BYTES; 253 nr = NUMBER_OF_ROUNDS_14; 254 break; 255 default: 256 return; 257 } 258 for (i = 0; i < nk; i++) 259 w_ring[i] = get_unaligned_be32(&key[i * 4]); 260 261 i = 0; 262 temp = w_ring[nk - 1]; 263 while (i + nk < (nr + 1) * 4) { 264 if (!(i % nk)) { 265 /* RotWord(temp) */ 266 temp = (temp << 8) | (temp >> 24); 267 temp = aes_ks_subword(temp); 268 temp ^= round_constant[i / nk]; 269 } else if (nk == 8 && (i % 4 == 0)) { 270 temp = aes_ks_subword(temp); 271 } 272 w_ring[i % nk] ^= temp; 273 temp = w_ring[i % nk]; 274 i++; 275 } 276 i--; 277 for (k = 0, j = i % nk; k < nk; k++) { 278 put_unaligned_be32(w_ring[j], &dec_key[k * 4]); 279 j--; 280 if (j < 0) 281 j += nk; 282 } 283 } 284 285 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 286 { 287 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 288 289 switch (ds) { 290 case SHA1_DIGEST_SIZE: 291 base_hash = crypto_alloc_shash("sha1", 0, 0); 292 break; 293 case SHA224_DIGEST_SIZE: 294 base_hash = crypto_alloc_shash("sha224", 0, 0); 295 break; 296 case SHA256_DIGEST_SIZE: 297 base_hash = crypto_alloc_shash("sha256", 0, 0); 298 break; 299 case SHA384_DIGEST_SIZE: 300 base_hash = crypto_alloc_shash("sha384", 0, 0); 301 break; 302 case SHA512_DIGEST_SIZE: 303 base_hash = crypto_alloc_shash("sha512", 0, 0); 304 break; 305 } 306 307 return base_hash; 308 } 309 310 static int chcr_compute_partial_hash(struct shash_desc *desc, 311 char *iopad, char *result_hash, 312 int digest_size) 313 { 314 struct sha1_state sha1_st; 315 struct sha256_state sha256_st; 316 struct sha512_state sha512_st; 317 int error; 318 319 if (digest_size == SHA1_DIGEST_SIZE) { 320 error = crypto_shash_init(desc) ?: 321 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 322 crypto_shash_export(desc, (void *)&sha1_st); 323 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 324 } else if (digest_size == SHA224_DIGEST_SIZE) { 325 error = crypto_shash_init(desc) ?: 326 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 327 crypto_shash_export(desc, (void *)&sha256_st); 328 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 329 330 } else if (digest_size == SHA256_DIGEST_SIZE) { 331 error = crypto_shash_init(desc) ?: 332 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 333 crypto_shash_export(desc, (void *)&sha256_st); 334 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 335 336 } else if (digest_size == SHA384_DIGEST_SIZE) { 337 error = crypto_shash_init(desc) ?: 338 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 339 crypto_shash_export(desc, (void *)&sha512_st); 340 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 341 342 } else if (digest_size == SHA512_DIGEST_SIZE) { 343 error = crypto_shash_init(desc) ?: 344 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 345 crypto_shash_export(desc, (void *)&sha512_st); 346 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 347 } else { 348 error = -EINVAL; 349 pr_err("Unknown digest size %d\n", digest_size); 350 } 351 return error; 352 } 353 354 static void chcr_change_order(char *buf, int ds) 355 { 356 int i; 357 358 if (ds == SHA512_DIGEST_SIZE) { 359 for (i = 0; i < (ds / sizeof(u64)); i++) 360 *((__be64 *)buf + i) = 361 cpu_to_be64(*((u64 *)buf + i)); 362 } else { 363 for (i = 0; i < (ds / sizeof(u32)); i++) 364 *((__be32 *)buf + i) = 365 cpu_to_be32(*((u32 *)buf + i)); 366 } 367 } 368 369 static inline int is_hmac(struct crypto_tfm *tfm) 370 { 371 struct crypto_alg *alg = tfm->__crt_alg; 372 struct chcr_alg_template *chcr_crypto_alg = 373 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 374 alg.hash); 375 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 376 return 1; 377 return 0; 378 } 379 380 static inline void dsgl_walk_init(struct dsgl_walk *walk, 381 struct cpl_rx_phys_dsgl *dsgl) 382 { 383 walk->dsgl = dsgl; 384 walk->nents = 0; 385 walk->to = (struct phys_sge_pairs *)(dsgl + 1); 386 } 387 388 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid, 389 int pci_chan_id) 390 { 391 struct cpl_rx_phys_dsgl *phys_cpl; 392 393 phys_cpl = walk->dsgl; 394 395 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 396 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 397 phys_cpl->pcirlxorder_to_noofsgentr = 398 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 399 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 400 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 401 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 402 CPL_RX_PHYS_DSGL_DCAID_V(0) | 403 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); 404 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 405 phys_cpl->rss_hdr_int.qid = htons(qid); 406 phys_cpl->rss_hdr_int.hash_val = 0; 407 phys_cpl->rss_hdr_int.channel = pci_chan_id; 408 } 409 410 static inline void dsgl_walk_add_page(struct dsgl_walk *walk, 411 size_t size, 412 dma_addr_t addr) 413 { 414 int j; 415 416 if (!size) 417 return; 418 j = walk->nents; 419 walk->to->len[j % 8] = htons(size); 420 walk->to->addr[j % 8] = cpu_to_be64(addr); 421 j++; 422 if ((j % 8) == 0) 423 walk->to++; 424 walk->nents = j; 425 } 426 427 static void dsgl_walk_add_sg(struct dsgl_walk *walk, 428 struct scatterlist *sg, 429 unsigned int slen, 430 unsigned int skip) 431 { 432 int skip_len = 0; 433 unsigned int left_size = slen, len = 0; 434 unsigned int j = walk->nents; 435 int offset, ent_len; 436 437 if (!slen) 438 return; 439 while (sg && skip) { 440 if (sg_dma_len(sg) <= skip) { 441 skip -= sg_dma_len(sg); 442 skip_len = 0; 443 sg = sg_next(sg); 444 } else { 445 skip_len = skip; 446 skip = 0; 447 } 448 } 449 450 while (left_size && sg) { 451 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); 452 offset = 0; 453 while (len) { 454 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); 455 walk->to->len[j % 8] = htons(ent_len); 456 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + 457 offset + skip_len); 458 offset += ent_len; 459 len -= ent_len; 460 j++; 461 if ((j % 8) == 0) 462 walk->to++; 463 } 464 walk->last_sg = sg; 465 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - 466 skip_len) + skip_len; 467 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); 468 skip_len = 0; 469 sg = sg_next(sg); 470 } 471 walk->nents = j; 472 } 473 474 static inline void ulptx_walk_init(struct ulptx_walk *walk, 475 struct ulptx_sgl *ulp) 476 { 477 walk->sgl = ulp; 478 walk->nents = 0; 479 walk->pair_idx = 0; 480 walk->pair = ulp->sge; 481 walk->last_sg = NULL; 482 walk->last_sg_len = 0; 483 } 484 485 static inline void ulptx_walk_end(struct ulptx_walk *walk) 486 { 487 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | 488 ULPTX_NSGE_V(walk->nents)); 489 } 490 491 492 static inline void ulptx_walk_add_page(struct ulptx_walk *walk, 493 size_t size, 494 dma_addr_t addr) 495 { 496 if (!size) 497 return; 498 499 if (walk->nents == 0) { 500 walk->sgl->len0 = cpu_to_be32(size); 501 walk->sgl->addr0 = cpu_to_be64(addr); 502 } else { 503 walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr); 504 walk->pair->len[walk->pair_idx] = cpu_to_be32(size); 505 walk->pair_idx = !walk->pair_idx; 506 if (!walk->pair_idx) 507 walk->pair++; 508 } 509 walk->nents++; 510 } 511 512 static void ulptx_walk_add_sg(struct ulptx_walk *walk, 513 struct scatterlist *sg, 514 unsigned int len, 515 unsigned int skip) 516 { 517 int small; 518 int skip_len = 0; 519 unsigned int sgmin; 520 521 if (!len) 522 return; 523 while (sg && skip) { 524 if (sg_dma_len(sg) <= skip) { 525 skip -= sg_dma_len(sg); 526 skip_len = 0; 527 sg = sg_next(sg); 528 } else { 529 skip_len = skip; 530 skip = 0; 531 } 532 } 533 WARN(!sg, "SG should not be null here\n"); 534 if (sg && (walk->nents == 0)) { 535 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); 536 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 537 walk->sgl->len0 = cpu_to_be32(sgmin); 538 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); 539 walk->nents++; 540 len -= sgmin; 541 walk->last_sg = sg; 542 walk->last_sg_len = sgmin + skip_len; 543 skip_len += sgmin; 544 if (sg_dma_len(sg) == skip_len) { 545 sg = sg_next(sg); 546 skip_len = 0; 547 } 548 } 549 550 while (sg && len) { 551 small = min(sg_dma_len(sg) - skip_len, len); 552 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 553 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); 554 walk->pair->addr[walk->pair_idx] = 555 cpu_to_be64(sg_dma_address(sg) + skip_len); 556 walk->pair_idx = !walk->pair_idx; 557 walk->nents++; 558 if (!walk->pair_idx) 559 walk->pair++; 560 len -= sgmin; 561 skip_len += sgmin; 562 walk->last_sg = sg; 563 walk->last_sg_len = skip_len; 564 if (sg_dma_len(sg) == skip_len) { 565 sg = sg_next(sg); 566 skip_len = 0; 567 } 568 } 569 } 570 571 static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm) 572 { 573 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 574 struct chcr_alg_template *chcr_crypto_alg = 575 container_of(alg, struct chcr_alg_template, alg.skcipher); 576 577 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 578 } 579 580 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 581 { 582 struct adapter *adap = netdev2adap(dev); 583 struct sge_uld_txq_info *txq_info = 584 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 585 struct sge_uld_txq *txq; 586 int ret = 0; 587 588 local_bh_disable(); 589 txq = &txq_info->uldtxq[idx]; 590 spin_lock(&txq->sendq.lock); 591 if (txq->full) 592 ret = -1; 593 spin_unlock(&txq->sendq.lock); 594 local_bh_enable(); 595 return ret; 596 } 597 598 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 599 struct _key_ctx *key_ctx) 600 { 601 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 602 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 603 } else { 604 memcpy(key_ctx->key, 605 ablkctx->key + (ablkctx->enckey_len >> 1), 606 ablkctx->enckey_len >> 1); 607 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 608 ablkctx->rrkey, ablkctx->enckey_len >> 1); 609 } 610 return 0; 611 } 612 613 static int chcr_hash_ent_in_wr(struct scatterlist *src, 614 unsigned int minsg, 615 unsigned int space, 616 unsigned int srcskip) 617 { 618 int srclen = 0; 619 int srcsg = minsg; 620 int soffset = 0, sless; 621 622 if (sg_dma_len(src) == srcskip) { 623 src = sg_next(src); 624 srcskip = 0; 625 } 626 while (src && space > (sgl_ent_len[srcsg + 1])) { 627 sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip, 628 CHCR_SRC_SG_SIZE); 629 srclen += sless; 630 soffset += sless; 631 srcsg++; 632 if (sg_dma_len(src) == (soffset + srcskip)) { 633 src = sg_next(src); 634 soffset = 0; 635 srcskip = 0; 636 } 637 } 638 return srclen; 639 } 640 641 static int chcr_sg_ent_in_wr(struct scatterlist *src, 642 struct scatterlist *dst, 643 unsigned int minsg, 644 unsigned int space, 645 unsigned int srcskip, 646 unsigned int dstskip) 647 { 648 int srclen = 0, dstlen = 0; 649 int srcsg = minsg, dstsg = minsg; 650 int offset = 0, soffset = 0, less, sless = 0; 651 652 if (sg_dma_len(src) == srcskip) { 653 src = sg_next(src); 654 srcskip = 0; 655 } 656 if (sg_dma_len(dst) == dstskip) { 657 dst = sg_next(dst); 658 dstskip = 0; 659 } 660 661 while (src && dst && 662 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { 663 sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset, 664 CHCR_SRC_SG_SIZE); 665 srclen += sless; 666 srcsg++; 667 offset = 0; 668 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && 669 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { 670 if (srclen <= dstlen) 671 break; 672 less = min_t(unsigned int, sg_dma_len(dst) - offset - 673 dstskip, CHCR_DST_SG_SIZE); 674 dstlen += less; 675 offset += less; 676 if ((offset + dstskip) == sg_dma_len(dst)) { 677 dst = sg_next(dst); 678 offset = 0; 679 } 680 dstsg++; 681 dstskip = 0; 682 } 683 soffset += sless; 684 if ((soffset + srcskip) == sg_dma_len(src)) { 685 src = sg_next(src); 686 srcskip = 0; 687 soffset = 0; 688 } 689 690 } 691 return min(srclen, dstlen); 692 } 693 694 static int chcr_cipher_fallback(struct crypto_skcipher *cipher, 695 struct skcipher_request *req, 696 u8 *iv, 697 unsigned short op_type) 698 { 699 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 700 int err; 701 702 skcipher_request_set_tfm(&reqctx->fallback_req, cipher); 703 skcipher_request_set_callback(&reqctx->fallback_req, req->base.flags, 704 req->base.complete, req->base.data); 705 skcipher_request_set_crypt(&reqctx->fallback_req, req->src, req->dst, 706 req->cryptlen, iv); 707 708 err = op_type ? crypto_skcipher_decrypt(&reqctx->fallback_req) : 709 crypto_skcipher_encrypt(&reqctx->fallback_req); 710 711 return err; 712 713 } 714 715 static inline int get_qidxs(struct crypto_async_request *req, 716 unsigned int *txqidx, unsigned int *rxqidx) 717 { 718 struct crypto_tfm *tfm = req->tfm; 719 int ret = 0; 720 721 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 722 case CRYPTO_ALG_TYPE_AEAD: 723 { 724 struct aead_request *aead_req = 725 container_of(req, struct aead_request, base); 726 struct chcr_aead_reqctx *reqctx = aead_request_ctx(aead_req); 727 *txqidx = reqctx->txqidx; 728 *rxqidx = reqctx->rxqidx; 729 break; 730 } 731 case CRYPTO_ALG_TYPE_SKCIPHER: 732 { 733 struct skcipher_request *sk_req = 734 container_of(req, struct skcipher_request, base); 735 struct chcr_skcipher_req_ctx *reqctx = 736 skcipher_request_ctx(sk_req); 737 *txqidx = reqctx->txqidx; 738 *rxqidx = reqctx->rxqidx; 739 break; 740 } 741 case CRYPTO_ALG_TYPE_AHASH: 742 { 743 struct ahash_request *ahash_req = 744 container_of(req, struct ahash_request, base); 745 struct chcr_ahash_req_ctx *reqctx = 746 ahash_request_ctx(ahash_req); 747 *txqidx = reqctx->txqidx; 748 *rxqidx = reqctx->rxqidx; 749 break; 750 } 751 default: 752 ret = -EINVAL; 753 /* should never get here */ 754 BUG(); 755 break; 756 } 757 return ret; 758 } 759 760 static inline void create_wreq(struct chcr_context *ctx, 761 struct chcr_wr *chcr_req, 762 struct crypto_async_request *req, 763 unsigned int imm, 764 int hash_sz, 765 unsigned int len16, 766 unsigned int sc_len, 767 unsigned int lcb) 768 { 769 struct uld_ctx *u_ctx = ULD_CTX(ctx); 770 unsigned int tx_channel_id, rx_channel_id; 771 unsigned int txqidx = 0, rxqidx = 0; 772 unsigned int qid, fid; 773 774 get_qidxs(req, &txqidx, &rxqidx); 775 qid = u_ctx->lldi.rxq_ids[rxqidx]; 776 fid = u_ctx->lldi.rxq_ids[0]; 777 tx_channel_id = txqidx / ctx->txq_perchan; 778 rx_channel_id = rxqidx / ctx->rxq_perchan; 779 780 781 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; 782 chcr_req->wreq.pld_size_hash_size = 783 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 784 chcr_req->wreq.len16_pkd = 785 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); 786 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 787 chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid, 788 !!lcb, txqidx); 789 790 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid); 791 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - 792 ((sizeof(chcr_req->wreq)) >> 4))); 793 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); 794 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 795 sizeof(chcr_req->key_ctx) + sc_len); 796 } 797 798 /** 799 * create_cipher_wr - form the WR for cipher operations 800 * @wrparam: Container for create_cipher_wr()'s parameters 801 */ 802 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) 803 { 804 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); 805 struct chcr_context *ctx = c_ctx(tfm); 806 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 807 struct sk_buff *skb = NULL; 808 struct chcr_wr *chcr_req; 809 struct cpl_rx_phys_dsgl *phys_cpl; 810 struct ulptx_sgl *ulptx; 811 struct chcr_skcipher_req_ctx *reqctx = 812 skcipher_request_ctx(wrparam->req); 813 unsigned int temp = 0, transhdr_len, dst_size; 814 int error; 815 int nents; 816 unsigned int kctx_len; 817 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 818 GFP_KERNEL : GFP_ATOMIC; 819 struct adapter *adap = padap(ctx->dev); 820 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 821 822 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, 823 reqctx->dst_ofst); 824 dst_size = get_space_for_phys_dsgl(nents); 825 kctx_len = roundup(ablkctx->enckey_len, 16); 826 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 827 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, 828 CHCR_SRC_SG_SIZE, reqctx->src_ofst); 829 temp = reqctx->imm ? roundup(wrparam->bytes, 16) : 830 (sgl_len(nents) * 8); 831 transhdr_len += temp; 832 transhdr_len = roundup(transhdr_len, 16); 833 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 834 if (!skb) { 835 error = -ENOMEM; 836 goto err; 837 } 838 chcr_req = __skb_put_zero(skb, transhdr_len); 839 chcr_req->sec_cpl.op_ivinsrtofst = 840 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); 841 842 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); 843 chcr_req->sec_cpl.aadstart_cipherstop_hi = 844 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); 845 846 chcr_req->sec_cpl.cipherstop_lo_authinsert = 847 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 848 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, 849 ablkctx->ciph_mode, 850 0, 0, IV >> 1); 851 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 852 0, 1, dst_size); 853 854 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 855 if ((reqctx->op == CHCR_DECRYPT_OP) && 856 (!(get_cryptoalg_subtype(tfm) == 857 CRYPTO_ALG_SUB_TYPE_CTR)) && 858 (!(get_cryptoalg_subtype(tfm) == 859 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { 860 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 861 } else { 862 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || 863 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { 864 memcpy(chcr_req->key_ctx.key, ablkctx->key, 865 ablkctx->enckey_len); 866 } else { 867 memcpy(chcr_req->key_ctx.key, ablkctx->key + 868 (ablkctx->enckey_len >> 1), 869 ablkctx->enckey_len >> 1); 870 memcpy(chcr_req->key_ctx.key + 871 (ablkctx->enckey_len >> 1), 872 ablkctx->key, 873 ablkctx->enckey_len >> 1); 874 } 875 } 876 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 877 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 878 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); 879 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); 880 881 atomic_inc(&adap->chcr_stats.cipher_rqst); 882 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV 883 + (reqctx->imm ? (wrparam->bytes) : 0); 884 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, 885 transhdr_len, temp, 886 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); 887 reqctx->skb = skb; 888 889 if (reqctx->op && (ablkctx->ciph_mode == 890 CHCR_SCMD_CIPHER_MODE_AES_CBC)) 891 sg_pcopy_to_buffer(wrparam->req->src, 892 sg_nents(wrparam->req->src), wrparam->req->iv, 16, 893 reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE); 894 895 return skb; 896 err: 897 return ERR_PTR(error); 898 } 899 900 static inline int chcr_keyctx_ck_size(unsigned int keylen) 901 { 902 int ck_size = 0; 903 904 if (keylen == AES_KEYSIZE_128) 905 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 906 else if (keylen == AES_KEYSIZE_192) 907 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 908 else if (keylen == AES_KEYSIZE_256) 909 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 910 else 911 ck_size = 0; 912 913 return ck_size; 914 } 915 static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher, 916 const u8 *key, 917 unsigned int keylen) 918 { 919 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 920 921 crypto_skcipher_clear_flags(ablkctx->sw_cipher, 922 CRYPTO_TFM_REQ_MASK); 923 crypto_skcipher_set_flags(ablkctx->sw_cipher, 924 cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK); 925 return crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen); 926 } 927 928 static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher, 929 const u8 *key, 930 unsigned int keylen) 931 { 932 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 933 unsigned int ck_size, context_size; 934 u16 alignment = 0; 935 int err; 936 937 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 938 if (err) 939 goto badkey_err; 940 941 ck_size = chcr_keyctx_ck_size(keylen); 942 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; 943 memcpy(ablkctx->key, key, keylen); 944 ablkctx->enckey_len = keylen; 945 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 946 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 947 keylen + alignment) >> 4; 948 949 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 950 0, 0, context_size); 951 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 952 return 0; 953 badkey_err: 954 ablkctx->enckey_len = 0; 955 956 return err; 957 } 958 959 static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher, 960 const u8 *key, 961 unsigned int keylen) 962 { 963 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 964 unsigned int ck_size, context_size; 965 u16 alignment = 0; 966 int err; 967 968 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 969 if (err) 970 goto badkey_err; 971 ck_size = chcr_keyctx_ck_size(keylen); 972 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 973 memcpy(ablkctx->key, key, keylen); 974 ablkctx->enckey_len = keylen; 975 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 976 keylen + alignment) >> 4; 977 978 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 979 0, 0, context_size); 980 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 981 982 return 0; 983 badkey_err: 984 ablkctx->enckey_len = 0; 985 986 return err; 987 } 988 989 static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher, 990 const u8 *key, 991 unsigned int keylen) 992 { 993 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 994 unsigned int ck_size, context_size; 995 u16 alignment = 0; 996 int err; 997 998 if (keylen < CTR_RFC3686_NONCE_SIZE) 999 return -EINVAL; 1000 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), 1001 CTR_RFC3686_NONCE_SIZE); 1002 1003 keylen -= CTR_RFC3686_NONCE_SIZE; 1004 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 1005 if (err) 1006 goto badkey_err; 1007 1008 ck_size = chcr_keyctx_ck_size(keylen); 1009 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 1010 memcpy(ablkctx->key, key, keylen); 1011 ablkctx->enckey_len = keylen; 1012 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 1013 keylen + alignment) >> 4; 1014 1015 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 1016 0, 0, context_size); 1017 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 1018 1019 return 0; 1020 badkey_err: 1021 ablkctx->enckey_len = 0; 1022 1023 return err; 1024 } 1025 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) 1026 { 1027 unsigned int size = AES_BLOCK_SIZE; 1028 __be32 *b = (__be32 *)(dstiv + size); 1029 u32 c, prev; 1030 1031 memcpy(dstiv, srciv, AES_BLOCK_SIZE); 1032 for (; size >= 4; size -= 4) { 1033 prev = be32_to_cpu(*--b); 1034 c = prev + add; 1035 *b = cpu_to_be32(c); 1036 if (prev < c) 1037 break; 1038 add = 1; 1039 } 1040 1041 } 1042 1043 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) 1044 { 1045 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); 1046 u64 c; 1047 u32 temp = be32_to_cpu(*--b); 1048 1049 temp = ~temp; 1050 c = (u64)temp + 1; // No of block can processed without overflow 1051 if ((bytes / AES_BLOCK_SIZE) >= c) 1052 bytes = c * AES_BLOCK_SIZE; 1053 return bytes; 1054 } 1055 1056 static int chcr_update_tweak(struct skcipher_request *req, u8 *iv, 1057 u32 isfinal) 1058 { 1059 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1060 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1061 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1062 struct crypto_aes_ctx aes; 1063 int ret, i; 1064 u8 *key; 1065 unsigned int keylen; 1066 int round = reqctx->last_req_len / AES_BLOCK_SIZE; 1067 int round8 = round / 8; 1068 1069 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); 1070 1071 keylen = ablkctx->enckey_len / 2; 1072 key = ablkctx->key + keylen; 1073 /* For a 192 bit key remove the padded zeroes which was 1074 * added in chcr_xts_setkey 1075 */ 1076 if (KEY_CONTEXT_CK_SIZE_G(ntohl(ablkctx->key_ctx_hdr)) 1077 == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) 1078 ret = aes_expandkey(&aes, key, keylen - 8); 1079 else 1080 ret = aes_expandkey(&aes, key, keylen); 1081 if (ret) 1082 return ret; 1083 aes_encrypt(&aes, iv, iv); 1084 for (i = 0; i < round8; i++) 1085 gf128mul_x8_ble((le128 *)iv, (le128 *)iv); 1086 1087 for (i = 0; i < (round % 8); i++) 1088 gf128mul_x_ble((le128 *)iv, (le128 *)iv); 1089 1090 if (!isfinal) 1091 aes_decrypt(&aes, iv, iv); 1092 1093 memzero_explicit(&aes, sizeof(aes)); 1094 return 0; 1095 } 1096 1097 static int chcr_update_cipher_iv(struct skcipher_request *req, 1098 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1099 { 1100 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1101 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1102 int subtype = get_cryptoalg_subtype(tfm); 1103 int ret = 0; 1104 1105 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1106 ctr_add_iv(iv, req->iv, (reqctx->processed / 1107 AES_BLOCK_SIZE)); 1108 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) 1109 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1110 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / 1111 AES_BLOCK_SIZE) + 1); 1112 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1113 ret = chcr_update_tweak(req, iv, 0); 1114 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1115 if (reqctx->op) 1116 /*Updated before sending last WR*/ 1117 memcpy(iv, req->iv, AES_BLOCK_SIZE); 1118 else 1119 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1120 } 1121 1122 return ret; 1123 1124 } 1125 1126 /* We need separate function for final iv because in rfc3686 Initial counter 1127 * starts from 1 and buffer size of iv is 8 byte only which remains constant 1128 * for subsequent update requests 1129 */ 1130 1131 static int chcr_final_cipher_iv(struct skcipher_request *req, 1132 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1133 { 1134 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1135 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1136 int subtype = get_cryptoalg_subtype(tfm); 1137 int ret = 0; 1138 1139 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1140 ctr_add_iv(iv, req->iv, DIV_ROUND_UP(reqctx->processed, 1141 AES_BLOCK_SIZE)); 1142 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) { 1143 if (!reqctx->partial_req) 1144 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); 1145 else 1146 ret = chcr_update_tweak(req, iv, 1); 1147 } 1148 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1149 /*Already updated for Decrypt*/ 1150 if (!reqctx->op) 1151 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1152 1153 } 1154 return ret; 1155 1156 } 1157 1158 static int chcr_handle_cipher_resp(struct skcipher_request *req, 1159 unsigned char *input, int err) 1160 { 1161 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1162 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1163 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; 1164 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1165 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1166 struct chcr_dev *dev = c_ctx(tfm)->dev; 1167 struct chcr_context *ctx = c_ctx(tfm); 1168 struct adapter *adap = padap(ctx->dev); 1169 struct cipher_wr_param wrparam; 1170 struct sk_buff *skb; 1171 int bytes; 1172 1173 if (err) 1174 goto unmap; 1175 if (req->cryptlen == reqctx->processed) { 1176 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1177 req); 1178 err = chcr_final_cipher_iv(req, fw6_pld, req->iv); 1179 goto complete; 1180 } 1181 1182 if (!reqctx->imm) { 1183 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0, 1184 CIP_SPACE_LEFT(ablkctx->enckey_len), 1185 reqctx->src_ofst, reqctx->dst_ofst); 1186 if ((bytes + reqctx->processed) >= req->cryptlen) 1187 bytes = req->cryptlen - reqctx->processed; 1188 else 1189 bytes = rounddown(bytes, 16); 1190 } else { 1191 /*CTR mode counter overfloa*/ 1192 bytes = req->cryptlen - reqctx->processed; 1193 } 1194 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); 1195 if (err) 1196 goto unmap; 1197 1198 if (unlikely(bytes == 0)) { 1199 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1200 req); 1201 memcpy(req->iv, reqctx->init_iv, IV); 1202 atomic_inc(&adap->chcr_stats.fallback); 1203 err = chcr_cipher_fallback(ablkctx->sw_cipher, req, req->iv, 1204 reqctx->op); 1205 goto complete; 1206 } 1207 1208 if (get_cryptoalg_subtype(tfm) == 1209 CRYPTO_ALG_SUB_TYPE_CTR) 1210 bytes = adjust_ctr_overflow(reqctx->iv, bytes); 1211 wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx]; 1212 wrparam.req = req; 1213 wrparam.bytes = bytes; 1214 skb = create_cipher_wr(&wrparam); 1215 if (IS_ERR(skb)) { 1216 pr_err("%s : Failed to form WR. No memory\n", __func__); 1217 err = PTR_ERR(skb); 1218 goto unmap; 1219 } 1220 skb->dev = u_ctx->lldi.ports[0]; 1221 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 1222 chcr_send_wr(skb); 1223 reqctx->last_req_len = bytes; 1224 reqctx->processed += bytes; 1225 if (get_cryptoalg_subtype(tfm) == 1226 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == 1227 CRYPTO_TFM_REQ_MAY_SLEEP ) { 1228 complete(&ctx->cbc_aes_aio_done); 1229 } 1230 return 0; 1231 unmap: 1232 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1233 complete: 1234 if (get_cryptoalg_subtype(tfm) == 1235 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == 1236 CRYPTO_TFM_REQ_MAY_SLEEP ) { 1237 complete(&ctx->cbc_aes_aio_done); 1238 } 1239 chcr_dec_wrcount(dev); 1240 req->base.complete(&req->base, err); 1241 return err; 1242 } 1243 1244 static int process_cipher(struct skcipher_request *req, 1245 unsigned short qid, 1246 struct sk_buff **skb, 1247 unsigned short op_type) 1248 { 1249 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1250 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1251 unsigned int ivsize = crypto_skcipher_ivsize(tfm); 1252 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1253 struct adapter *adap = padap(c_ctx(tfm)->dev); 1254 struct cipher_wr_param wrparam; 1255 int bytes, err = -EINVAL; 1256 int subtype; 1257 1258 reqctx->processed = 0; 1259 reqctx->partial_req = 0; 1260 if (!req->iv) 1261 goto error; 1262 subtype = get_cryptoalg_subtype(tfm); 1263 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 1264 (req->cryptlen == 0) || 1265 (req->cryptlen % crypto_skcipher_blocksize(tfm))) { 1266 if (req->cryptlen == 0 && subtype != CRYPTO_ALG_SUB_TYPE_XTS) 1267 goto fallback; 1268 else if (req->cryptlen % crypto_skcipher_blocksize(tfm) && 1269 subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1270 goto fallback; 1271 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 1272 ablkctx->enckey_len, req->cryptlen, ivsize); 1273 goto error; 1274 } 1275 1276 err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1277 if (err) 1278 goto error; 1279 if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + 1280 AES_MIN_KEY_SIZE + 1281 sizeof(struct cpl_rx_phys_dsgl) + 1282 /*Min dsgl size*/ 1283 32))) { 1284 /* Can be sent as Imm*/ 1285 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; 1286 1287 dnents = sg_nents_xlen(req->dst, req->cryptlen, 1288 CHCR_DST_SG_SIZE, 0); 1289 phys_dsgl = get_space_for_phys_dsgl(dnents); 1290 kctx_len = roundup(ablkctx->enckey_len, 16); 1291 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 1292 reqctx->imm = (transhdr_len + IV + req->cryptlen) <= 1293 SGE_MAX_WR_LEN; 1294 bytes = IV + req->cryptlen; 1295 1296 } else { 1297 reqctx->imm = 0; 1298 } 1299 1300 if (!reqctx->imm) { 1301 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0, 1302 CIP_SPACE_LEFT(ablkctx->enckey_len), 1303 0, 0); 1304 if ((bytes + reqctx->processed) >= req->cryptlen) 1305 bytes = req->cryptlen - reqctx->processed; 1306 else 1307 bytes = rounddown(bytes, 16); 1308 } else { 1309 bytes = req->cryptlen; 1310 } 1311 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) { 1312 bytes = adjust_ctr_overflow(req->iv, bytes); 1313 } 1314 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { 1315 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); 1316 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv, 1317 CTR_RFC3686_IV_SIZE); 1318 1319 /* initialize counter portion of counter block */ 1320 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1321 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1322 memcpy(reqctx->init_iv, reqctx->iv, IV); 1323 1324 } else { 1325 1326 memcpy(reqctx->iv, req->iv, IV); 1327 memcpy(reqctx->init_iv, req->iv, IV); 1328 } 1329 if (unlikely(bytes == 0)) { 1330 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1331 req); 1332 fallback: atomic_inc(&adap->chcr_stats.fallback); 1333 err = chcr_cipher_fallback(ablkctx->sw_cipher, req, 1334 subtype == 1335 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 ? 1336 reqctx->iv : req->iv, 1337 op_type); 1338 goto error; 1339 } 1340 reqctx->op = op_type; 1341 reqctx->srcsg = req->src; 1342 reqctx->dstsg = req->dst; 1343 reqctx->src_ofst = 0; 1344 reqctx->dst_ofst = 0; 1345 wrparam.qid = qid; 1346 wrparam.req = req; 1347 wrparam.bytes = bytes; 1348 *skb = create_cipher_wr(&wrparam); 1349 if (IS_ERR(*skb)) { 1350 err = PTR_ERR(*skb); 1351 goto unmap; 1352 } 1353 reqctx->processed = bytes; 1354 reqctx->last_req_len = bytes; 1355 reqctx->partial_req = !!(req->cryptlen - reqctx->processed); 1356 1357 return 0; 1358 unmap: 1359 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1360 error: 1361 return err; 1362 } 1363 1364 static int chcr_aes_encrypt(struct skcipher_request *req) 1365 { 1366 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1367 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1368 struct chcr_dev *dev = c_ctx(tfm)->dev; 1369 struct sk_buff *skb = NULL; 1370 int err; 1371 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1372 struct chcr_context *ctx = c_ctx(tfm); 1373 unsigned int cpu; 1374 1375 cpu = get_cpu(); 1376 reqctx->txqidx = cpu % ctx->ntxq; 1377 reqctx->rxqidx = cpu % ctx->nrxq; 1378 put_cpu(); 1379 1380 err = chcr_inc_wrcount(dev); 1381 if (err) 1382 return -ENXIO; 1383 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1384 reqctx->txqidx) && 1385 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1386 err = -ENOSPC; 1387 goto error; 1388 } 1389 1390 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], 1391 &skb, CHCR_ENCRYPT_OP); 1392 if (err || !skb) 1393 return err; 1394 skb->dev = u_ctx->lldi.ports[0]; 1395 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 1396 chcr_send_wr(skb); 1397 if (get_cryptoalg_subtype(tfm) == 1398 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags == 1399 CRYPTO_TFM_REQ_MAY_SLEEP ) { 1400 reqctx->partial_req = 1; 1401 wait_for_completion(&ctx->cbc_aes_aio_done); 1402 } 1403 return -EINPROGRESS; 1404 error: 1405 chcr_dec_wrcount(dev); 1406 return err; 1407 } 1408 1409 static int chcr_aes_decrypt(struct skcipher_request *req) 1410 { 1411 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1412 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 1413 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1414 struct chcr_dev *dev = c_ctx(tfm)->dev; 1415 struct sk_buff *skb = NULL; 1416 int err; 1417 struct chcr_context *ctx = c_ctx(tfm); 1418 unsigned int cpu; 1419 1420 cpu = get_cpu(); 1421 reqctx->txqidx = cpu % ctx->ntxq; 1422 reqctx->rxqidx = cpu % ctx->nrxq; 1423 put_cpu(); 1424 1425 err = chcr_inc_wrcount(dev); 1426 if (err) 1427 return -ENXIO; 1428 1429 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1430 reqctx->txqidx) && 1431 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) 1432 return -ENOSPC; 1433 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], 1434 &skb, CHCR_DECRYPT_OP); 1435 if (err || !skb) 1436 return err; 1437 skb->dev = u_ctx->lldi.ports[0]; 1438 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 1439 chcr_send_wr(skb); 1440 return -EINPROGRESS; 1441 } 1442 static int chcr_device_init(struct chcr_context *ctx) 1443 { 1444 struct uld_ctx *u_ctx = NULL; 1445 int txq_perchan, ntxq; 1446 int err = 0, rxq_perchan; 1447 1448 if (!ctx->dev) { 1449 u_ctx = assign_chcr_device(); 1450 if (!u_ctx) { 1451 err = -ENXIO; 1452 pr_err("chcr device assignment fails\n"); 1453 goto out; 1454 } 1455 ctx->dev = &u_ctx->dev; 1456 ntxq = u_ctx->lldi.ntxq; 1457 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 1458 txq_perchan = ntxq / u_ctx->lldi.nchan; 1459 ctx->ntxq = ntxq; 1460 ctx->nrxq = u_ctx->lldi.nrxq; 1461 ctx->rxq_perchan = rxq_perchan; 1462 ctx->txq_perchan = txq_perchan; 1463 } 1464 out: 1465 return err; 1466 } 1467 1468 static int chcr_init_tfm(struct crypto_skcipher *tfm) 1469 { 1470 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1471 struct chcr_context *ctx = crypto_skcipher_ctx(tfm); 1472 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1473 1474 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0, 1475 CRYPTO_ALG_NEED_FALLBACK); 1476 if (IS_ERR(ablkctx->sw_cipher)) { 1477 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name); 1478 return PTR_ERR(ablkctx->sw_cipher); 1479 } 1480 init_completion(&ctx->cbc_aes_aio_done); 1481 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) + 1482 crypto_skcipher_reqsize(ablkctx->sw_cipher)); 1483 1484 return chcr_device_init(ctx); 1485 } 1486 1487 static int chcr_rfc3686_init(struct crypto_skcipher *tfm) 1488 { 1489 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1490 struct chcr_context *ctx = crypto_skcipher_ctx(tfm); 1491 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1492 1493 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) 1494 * cannot be used as fallback in chcr_handle_cipher_response 1495 */ 1496 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0, 1497 CRYPTO_ALG_NEED_FALLBACK); 1498 if (IS_ERR(ablkctx->sw_cipher)) { 1499 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name); 1500 return PTR_ERR(ablkctx->sw_cipher); 1501 } 1502 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx) + 1503 crypto_skcipher_reqsize(ablkctx->sw_cipher)); 1504 return chcr_device_init(ctx); 1505 } 1506 1507 1508 static void chcr_exit_tfm(struct crypto_skcipher *tfm) 1509 { 1510 struct chcr_context *ctx = crypto_skcipher_ctx(tfm); 1511 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1512 1513 crypto_free_skcipher(ablkctx->sw_cipher); 1514 } 1515 1516 static int get_alg_config(struct algo_param *params, 1517 unsigned int auth_size) 1518 { 1519 switch (auth_size) { 1520 case SHA1_DIGEST_SIZE: 1521 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 1522 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 1523 params->result_size = SHA1_DIGEST_SIZE; 1524 break; 1525 case SHA224_DIGEST_SIZE: 1526 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1527 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 1528 params->result_size = SHA256_DIGEST_SIZE; 1529 break; 1530 case SHA256_DIGEST_SIZE: 1531 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1532 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 1533 params->result_size = SHA256_DIGEST_SIZE; 1534 break; 1535 case SHA384_DIGEST_SIZE: 1536 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1537 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 1538 params->result_size = SHA512_DIGEST_SIZE; 1539 break; 1540 case SHA512_DIGEST_SIZE: 1541 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1542 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 1543 params->result_size = SHA512_DIGEST_SIZE; 1544 break; 1545 default: 1546 pr_err("ERROR, unsupported digest size\n"); 1547 return -EINVAL; 1548 } 1549 return 0; 1550 } 1551 1552 static inline void chcr_free_shash(struct crypto_shash *base_hash) 1553 { 1554 crypto_free_shash(base_hash); 1555 } 1556 1557 /** 1558 * create_hash_wr - Create hash work request 1559 * @req: Cipher req base 1560 * @param: Container for create_hash_wr()'s parameters 1561 */ 1562 static struct sk_buff *create_hash_wr(struct ahash_request *req, 1563 struct hash_wr_param *param) 1564 { 1565 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1566 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1567 struct chcr_context *ctx = h_ctx(tfm); 1568 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1569 struct sk_buff *skb = NULL; 1570 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1571 struct chcr_wr *chcr_req; 1572 struct ulptx_sgl *ulptx; 1573 unsigned int nents = 0, transhdr_len; 1574 unsigned int temp = 0; 1575 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1576 GFP_ATOMIC; 1577 struct adapter *adap = padap(h_ctx(tfm)->dev); 1578 int error = 0; 1579 unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan; 1580 1581 transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len); 1582 req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len + 1583 param->sg_len) <= SGE_MAX_WR_LEN; 1584 nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len, 1585 CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst); 1586 nents += param->bfr_len ? 1 : 0; 1587 transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len + 1588 param->sg_len, 16) : (sgl_len(nents) * 8); 1589 transhdr_len = roundup(transhdr_len, 16); 1590 1591 skb = alloc_skb(transhdr_len, flags); 1592 if (!skb) 1593 return ERR_PTR(-ENOMEM); 1594 chcr_req = __skb_put_zero(skb, transhdr_len); 1595 1596 chcr_req->sec_cpl.op_ivinsrtofst = 1597 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0); 1598 1599 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 1600 1601 chcr_req->sec_cpl.aadstart_cipherstop_hi = 1602 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 1603 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1604 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 1605 chcr_req->sec_cpl.seqno_numivs = 1606 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 1607 param->opad_needed, 0); 1608 1609 chcr_req->sec_cpl.ivgen_hdrlen = 1610 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 1611 1612 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 1613 param->alg_prm.result_size); 1614 1615 if (param->opad_needed) 1616 memcpy(chcr_req->key_ctx.key + 1617 ((param->alg_prm.result_size <= 32) ? 32 : 1618 CHCR_HASH_MAX_DIGEST_SIZE), 1619 hmacctx->opad, param->alg_prm.result_size); 1620 1621 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 1622 param->alg_prm.mk_size, 0, 1623 param->opad_needed, 1624 ((param->kctx_len + 1625 sizeof(chcr_req->key_ctx)) >> 4)); 1626 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 1627 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len + 1628 DUMMY_BYTES); 1629 if (param->bfr_len != 0) { 1630 req_ctx->hctx_wr.dma_addr = 1631 dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr, 1632 param->bfr_len, DMA_TO_DEVICE); 1633 if (dma_mapping_error(&u_ctx->lldi.pdev->dev, 1634 req_ctx->hctx_wr. dma_addr)) { 1635 error = -ENOMEM; 1636 goto err; 1637 } 1638 req_ctx->hctx_wr.dma_len = param->bfr_len; 1639 } else { 1640 req_ctx->hctx_wr.dma_addr = 0; 1641 } 1642 chcr_add_hash_src_ent(req, ulptx, param); 1643 /* Request upto max wr size */ 1644 temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ? 1645 (param->sg_len + param->bfr_len) : 0); 1646 atomic_inc(&adap->chcr_stats.digest_rqst); 1647 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm, 1648 param->hash_size, transhdr_len, 1649 temp, 0); 1650 req_ctx->hctx_wr.skb = skb; 1651 return skb; 1652 err: 1653 kfree_skb(skb); 1654 return ERR_PTR(error); 1655 } 1656 1657 static int chcr_ahash_update(struct ahash_request *req) 1658 { 1659 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1660 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1661 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1662 struct chcr_context *ctx = h_ctx(rtfm); 1663 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1664 struct sk_buff *skb; 1665 u8 remainder = 0, bs; 1666 unsigned int nbytes = req->nbytes; 1667 struct hash_wr_param params; 1668 int error; 1669 unsigned int cpu; 1670 1671 cpu = get_cpu(); 1672 req_ctx->txqidx = cpu % ctx->ntxq; 1673 req_ctx->rxqidx = cpu % ctx->nrxq; 1674 put_cpu(); 1675 1676 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1677 1678 if (nbytes + req_ctx->reqlen >= bs) { 1679 remainder = (nbytes + req_ctx->reqlen) % bs; 1680 nbytes = nbytes + req_ctx->reqlen - remainder; 1681 } else { 1682 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 1683 + req_ctx->reqlen, nbytes, 0); 1684 req_ctx->reqlen += nbytes; 1685 return 0; 1686 } 1687 error = chcr_inc_wrcount(dev); 1688 if (error) 1689 return -ENXIO; 1690 /* Detach state for CHCR means lldi or padap is freed. Increasing 1691 * inflight count for dev guarantees that lldi and padap is valid 1692 */ 1693 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1694 req_ctx->txqidx) && 1695 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1696 error = -ENOSPC; 1697 goto err; 1698 } 1699 1700 chcr_init_hctx_per_wr(req_ctx); 1701 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1702 if (error) { 1703 error = -ENOMEM; 1704 goto err; 1705 } 1706 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1707 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1708 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1709 HASH_SPACE_LEFT(params.kctx_len), 0); 1710 if (params.sg_len > req->nbytes) 1711 params.sg_len = req->nbytes; 1712 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) - 1713 req_ctx->reqlen; 1714 params.opad_needed = 0; 1715 params.more = 1; 1716 params.last = 0; 1717 params.bfr_len = req_ctx->reqlen; 1718 params.scmd1 = 0; 1719 req_ctx->hctx_wr.srcsg = req->src; 1720 1721 params.hash_size = params.alg_prm.result_size; 1722 req_ctx->data_len += params.sg_len + params.bfr_len; 1723 skb = create_hash_wr(req, ¶ms); 1724 if (IS_ERR(skb)) { 1725 error = PTR_ERR(skb); 1726 goto unmap; 1727 } 1728 1729 req_ctx->hctx_wr.processed += params.sg_len; 1730 if (remainder) { 1731 /* Swap buffers */ 1732 swap(req_ctx->reqbfr, req_ctx->skbfr); 1733 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 1734 req_ctx->reqbfr, remainder, req->nbytes - 1735 remainder); 1736 } 1737 req_ctx->reqlen = remainder; 1738 skb->dev = u_ctx->lldi.ports[0]; 1739 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 1740 chcr_send_wr(skb); 1741 return -EINPROGRESS; 1742 unmap: 1743 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1744 err: 1745 chcr_dec_wrcount(dev); 1746 return error; 1747 } 1748 1749 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 1750 { 1751 memset(bfr_ptr, 0, bs); 1752 *bfr_ptr = 0x80; 1753 if (bs == 64) 1754 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 1755 else 1756 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 1757 } 1758 1759 static int chcr_ahash_final(struct ahash_request *req) 1760 { 1761 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1762 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1763 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1764 struct hash_wr_param params; 1765 struct sk_buff *skb; 1766 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1767 struct chcr_context *ctx = h_ctx(rtfm); 1768 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1769 int error; 1770 unsigned int cpu; 1771 1772 cpu = get_cpu(); 1773 req_ctx->txqidx = cpu % ctx->ntxq; 1774 req_ctx->rxqidx = cpu % ctx->nrxq; 1775 put_cpu(); 1776 1777 error = chcr_inc_wrcount(dev); 1778 if (error) 1779 return -ENXIO; 1780 1781 chcr_init_hctx_per_wr(req_ctx); 1782 if (is_hmac(crypto_ahash_tfm(rtfm))) 1783 params.opad_needed = 1; 1784 else 1785 params.opad_needed = 0; 1786 params.sg_len = 0; 1787 req_ctx->hctx_wr.isfinal = 1; 1788 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1789 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1790 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1791 params.opad_needed = 1; 1792 params.kctx_len *= 2; 1793 } else { 1794 params.opad_needed = 0; 1795 } 1796 1797 req_ctx->hctx_wr.result = 1; 1798 params.bfr_len = req_ctx->reqlen; 1799 req_ctx->data_len += params.bfr_len + params.sg_len; 1800 req_ctx->hctx_wr.srcsg = req->src; 1801 if (req_ctx->reqlen == 0) { 1802 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1803 params.last = 0; 1804 params.more = 1; 1805 params.scmd1 = 0; 1806 params.bfr_len = bs; 1807 1808 } else { 1809 params.scmd1 = req_ctx->data_len; 1810 params.last = 1; 1811 params.more = 0; 1812 } 1813 params.hash_size = crypto_ahash_digestsize(rtfm); 1814 skb = create_hash_wr(req, ¶ms); 1815 if (IS_ERR(skb)) { 1816 error = PTR_ERR(skb); 1817 goto err; 1818 } 1819 req_ctx->reqlen = 0; 1820 skb->dev = u_ctx->lldi.ports[0]; 1821 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 1822 chcr_send_wr(skb); 1823 return -EINPROGRESS; 1824 err: 1825 chcr_dec_wrcount(dev); 1826 return error; 1827 } 1828 1829 static int chcr_ahash_finup(struct ahash_request *req) 1830 { 1831 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1832 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1833 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1834 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1835 struct chcr_context *ctx = h_ctx(rtfm); 1836 struct sk_buff *skb; 1837 struct hash_wr_param params; 1838 u8 bs; 1839 int error; 1840 unsigned int cpu; 1841 1842 cpu = get_cpu(); 1843 req_ctx->txqidx = cpu % ctx->ntxq; 1844 req_ctx->rxqidx = cpu % ctx->nrxq; 1845 put_cpu(); 1846 1847 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1848 error = chcr_inc_wrcount(dev); 1849 if (error) 1850 return -ENXIO; 1851 1852 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1853 req_ctx->txqidx) && 1854 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1855 error = -ENOSPC; 1856 goto err; 1857 } 1858 chcr_init_hctx_per_wr(req_ctx); 1859 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1860 if (error) { 1861 error = -ENOMEM; 1862 goto err; 1863 } 1864 1865 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1866 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1867 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1868 params.kctx_len *= 2; 1869 params.opad_needed = 1; 1870 } else { 1871 params.opad_needed = 0; 1872 } 1873 1874 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1875 HASH_SPACE_LEFT(params.kctx_len), 0); 1876 if (params.sg_len < req->nbytes) { 1877 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1878 params.kctx_len /= 2; 1879 params.opad_needed = 0; 1880 } 1881 params.last = 0; 1882 params.more = 1; 1883 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) 1884 - req_ctx->reqlen; 1885 params.hash_size = params.alg_prm.result_size; 1886 params.scmd1 = 0; 1887 } else { 1888 params.last = 1; 1889 params.more = 0; 1890 params.sg_len = req->nbytes; 1891 params.hash_size = crypto_ahash_digestsize(rtfm); 1892 params.scmd1 = req_ctx->data_len + req_ctx->reqlen + 1893 params.sg_len; 1894 } 1895 params.bfr_len = req_ctx->reqlen; 1896 req_ctx->data_len += params.bfr_len + params.sg_len; 1897 req_ctx->hctx_wr.result = 1; 1898 req_ctx->hctx_wr.srcsg = req->src; 1899 if ((req_ctx->reqlen + req->nbytes) == 0) { 1900 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1901 params.last = 0; 1902 params.more = 1; 1903 params.scmd1 = 0; 1904 params.bfr_len = bs; 1905 } 1906 skb = create_hash_wr(req, ¶ms); 1907 if (IS_ERR(skb)) { 1908 error = PTR_ERR(skb); 1909 goto unmap; 1910 } 1911 req_ctx->reqlen = 0; 1912 req_ctx->hctx_wr.processed += params.sg_len; 1913 skb->dev = u_ctx->lldi.ports[0]; 1914 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 1915 chcr_send_wr(skb); 1916 return -EINPROGRESS; 1917 unmap: 1918 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1919 err: 1920 chcr_dec_wrcount(dev); 1921 return error; 1922 } 1923 1924 static int chcr_ahash_digest(struct ahash_request *req) 1925 { 1926 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1927 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1928 struct chcr_dev *dev = h_ctx(rtfm)->dev; 1929 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm)); 1930 struct chcr_context *ctx = h_ctx(rtfm); 1931 struct sk_buff *skb; 1932 struct hash_wr_param params; 1933 u8 bs; 1934 int error; 1935 unsigned int cpu; 1936 1937 cpu = get_cpu(); 1938 req_ctx->txqidx = cpu % ctx->ntxq; 1939 req_ctx->rxqidx = cpu % ctx->nrxq; 1940 put_cpu(); 1941 1942 rtfm->init(req); 1943 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1944 error = chcr_inc_wrcount(dev); 1945 if (error) 1946 return -ENXIO; 1947 1948 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1949 req_ctx->txqidx) && 1950 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1951 error = -ENOSPC; 1952 goto err; 1953 } 1954 1955 chcr_init_hctx_per_wr(req_ctx); 1956 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1957 if (error) { 1958 error = -ENOMEM; 1959 goto err; 1960 } 1961 1962 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1963 params.kctx_len = roundup(params.alg_prm.result_size, 16); 1964 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1965 params.kctx_len *= 2; 1966 params.opad_needed = 1; 1967 } else { 1968 params.opad_needed = 0; 1969 } 1970 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen, 1971 HASH_SPACE_LEFT(params.kctx_len), 0); 1972 if (params.sg_len < req->nbytes) { 1973 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1974 params.kctx_len /= 2; 1975 params.opad_needed = 0; 1976 } 1977 params.last = 0; 1978 params.more = 1; 1979 params.scmd1 = 0; 1980 params.sg_len = rounddown(params.sg_len, bs); 1981 params.hash_size = params.alg_prm.result_size; 1982 } else { 1983 params.sg_len = req->nbytes; 1984 params.hash_size = crypto_ahash_digestsize(rtfm); 1985 params.last = 1; 1986 params.more = 0; 1987 params.scmd1 = req->nbytes + req_ctx->data_len; 1988 1989 } 1990 params.bfr_len = 0; 1991 req_ctx->hctx_wr.result = 1; 1992 req_ctx->hctx_wr.srcsg = req->src; 1993 req_ctx->data_len += params.bfr_len + params.sg_len; 1994 1995 if (req->nbytes == 0) { 1996 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1997 params.more = 1; 1998 params.bfr_len = bs; 1999 } 2000 2001 skb = create_hash_wr(req, ¶ms); 2002 if (IS_ERR(skb)) { 2003 error = PTR_ERR(skb); 2004 goto unmap; 2005 } 2006 req_ctx->hctx_wr.processed += params.sg_len; 2007 skb->dev = u_ctx->lldi.ports[0]; 2008 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx); 2009 chcr_send_wr(skb); 2010 return -EINPROGRESS; 2011 unmap: 2012 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 2013 err: 2014 chcr_dec_wrcount(dev); 2015 return error; 2016 } 2017 2018 static int chcr_ahash_continue(struct ahash_request *req) 2019 { 2020 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2021 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; 2022 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 2023 struct chcr_context *ctx = h_ctx(rtfm); 2024 struct uld_ctx *u_ctx = ULD_CTX(ctx); 2025 struct sk_buff *skb; 2026 struct hash_wr_param params; 2027 u8 bs; 2028 int error; 2029 unsigned int cpu; 2030 2031 cpu = get_cpu(); 2032 reqctx->txqidx = cpu % ctx->ntxq; 2033 reqctx->rxqidx = cpu % ctx->nrxq; 2034 put_cpu(); 2035 2036 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 2037 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 2038 params.kctx_len = roundup(params.alg_prm.result_size, 16); 2039 if (is_hmac(crypto_ahash_tfm(rtfm))) { 2040 params.kctx_len *= 2; 2041 params.opad_needed = 1; 2042 } else { 2043 params.opad_needed = 0; 2044 } 2045 params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0, 2046 HASH_SPACE_LEFT(params.kctx_len), 2047 hctx_wr->src_ofst); 2048 if ((params.sg_len + hctx_wr->processed) > req->nbytes) 2049 params.sg_len = req->nbytes - hctx_wr->processed; 2050 if (!hctx_wr->result || 2051 ((params.sg_len + hctx_wr->processed) < req->nbytes)) { 2052 if (is_hmac(crypto_ahash_tfm(rtfm))) { 2053 params.kctx_len /= 2; 2054 params.opad_needed = 0; 2055 } 2056 params.last = 0; 2057 params.more = 1; 2058 params.sg_len = rounddown(params.sg_len, bs); 2059 params.hash_size = params.alg_prm.result_size; 2060 params.scmd1 = 0; 2061 } else { 2062 params.last = 1; 2063 params.more = 0; 2064 params.hash_size = crypto_ahash_digestsize(rtfm); 2065 params.scmd1 = reqctx->data_len + params.sg_len; 2066 } 2067 params.bfr_len = 0; 2068 reqctx->data_len += params.sg_len; 2069 skb = create_hash_wr(req, ¶ms); 2070 if (IS_ERR(skb)) { 2071 error = PTR_ERR(skb); 2072 goto err; 2073 } 2074 hctx_wr->processed += params.sg_len; 2075 skb->dev = u_ctx->lldi.ports[0]; 2076 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 2077 chcr_send_wr(skb); 2078 return 0; 2079 err: 2080 return error; 2081 } 2082 2083 static inline void chcr_handle_ahash_resp(struct ahash_request *req, 2084 unsigned char *input, 2085 int err) 2086 { 2087 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2088 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr; 2089 int digestsize, updated_digestsize; 2090 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2091 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 2092 struct chcr_dev *dev = h_ctx(tfm)->dev; 2093 2094 if (input == NULL) 2095 goto out; 2096 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); 2097 updated_digestsize = digestsize; 2098 if (digestsize == SHA224_DIGEST_SIZE) 2099 updated_digestsize = SHA256_DIGEST_SIZE; 2100 else if (digestsize == SHA384_DIGEST_SIZE) 2101 updated_digestsize = SHA512_DIGEST_SIZE; 2102 2103 if (hctx_wr->dma_addr) { 2104 dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr, 2105 hctx_wr->dma_len, DMA_TO_DEVICE); 2106 hctx_wr->dma_addr = 0; 2107 } 2108 if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) == 2109 req->nbytes)) { 2110 if (hctx_wr->result == 1) { 2111 hctx_wr->result = 0; 2112 memcpy(req->result, input + sizeof(struct cpl_fw6_pld), 2113 digestsize); 2114 } else { 2115 memcpy(reqctx->partial_hash, 2116 input + sizeof(struct cpl_fw6_pld), 2117 updated_digestsize); 2118 2119 } 2120 goto unmap; 2121 } 2122 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), 2123 updated_digestsize); 2124 2125 err = chcr_ahash_continue(req); 2126 if (err) 2127 goto unmap; 2128 return; 2129 unmap: 2130 if (hctx_wr->is_sg_map) 2131 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 2132 2133 2134 out: 2135 chcr_dec_wrcount(dev); 2136 req->base.complete(&req->base, err); 2137 } 2138 2139 /* 2140 * chcr_handle_resp - Unmap the DMA buffers associated with the request 2141 * @req: crypto request 2142 */ 2143 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 2144 int err) 2145 { 2146 struct crypto_tfm *tfm = req->tfm; 2147 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2148 struct adapter *adap = padap(ctx->dev); 2149 2150 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 2151 case CRYPTO_ALG_TYPE_AEAD: 2152 err = chcr_handle_aead_resp(aead_request_cast(req), input, err); 2153 break; 2154 2155 case CRYPTO_ALG_TYPE_SKCIPHER: 2156 chcr_handle_cipher_resp(skcipher_request_cast(req), 2157 input, err); 2158 break; 2159 case CRYPTO_ALG_TYPE_AHASH: 2160 chcr_handle_ahash_resp(ahash_request_cast(req), input, err); 2161 } 2162 atomic_inc(&adap->chcr_stats.complete); 2163 return err; 2164 } 2165 static int chcr_ahash_export(struct ahash_request *areq, void *out) 2166 { 2167 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2168 struct chcr_ahash_req_ctx *state = out; 2169 2170 state->reqlen = req_ctx->reqlen; 2171 state->data_len = req_ctx->data_len; 2172 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 2173 memcpy(state->partial_hash, req_ctx->partial_hash, 2174 CHCR_HASH_MAX_DIGEST_SIZE); 2175 chcr_init_hctx_per_wr(state); 2176 return 0; 2177 } 2178 2179 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 2180 { 2181 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2182 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 2183 2184 req_ctx->reqlen = state->reqlen; 2185 req_ctx->data_len = state->data_len; 2186 req_ctx->reqbfr = req_ctx->bfr1; 2187 req_ctx->skbfr = req_ctx->bfr2; 2188 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 2189 memcpy(req_ctx->partial_hash, state->partial_hash, 2190 CHCR_HASH_MAX_DIGEST_SIZE); 2191 chcr_init_hctx_per_wr(req_ctx); 2192 return 0; 2193 } 2194 2195 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 2196 unsigned int keylen) 2197 { 2198 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 2199 unsigned int digestsize = crypto_ahash_digestsize(tfm); 2200 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 2201 unsigned int i, err = 0, updated_digestsize; 2202 2203 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 2204 2205 /* use the key to calculate the ipad and opad. ipad will sent with the 2206 * first request's data. opad will be sent with the final hash result 2207 * ipad in hmacctx->ipad and opad in hmacctx->opad location 2208 */ 2209 shash->tfm = hmacctx->base_hash; 2210 if (keylen > bs) { 2211 err = crypto_shash_digest(shash, key, keylen, 2212 hmacctx->ipad); 2213 if (err) 2214 goto out; 2215 keylen = digestsize; 2216 } else { 2217 memcpy(hmacctx->ipad, key, keylen); 2218 } 2219 memset(hmacctx->ipad + keylen, 0, bs - keylen); 2220 memcpy(hmacctx->opad, hmacctx->ipad, bs); 2221 2222 for (i = 0; i < bs / sizeof(int); i++) { 2223 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 2224 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 2225 } 2226 2227 updated_digestsize = digestsize; 2228 if (digestsize == SHA224_DIGEST_SIZE) 2229 updated_digestsize = SHA256_DIGEST_SIZE; 2230 else if (digestsize == SHA384_DIGEST_SIZE) 2231 updated_digestsize = SHA512_DIGEST_SIZE; 2232 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 2233 hmacctx->ipad, digestsize); 2234 if (err) 2235 goto out; 2236 chcr_change_order(hmacctx->ipad, updated_digestsize); 2237 2238 err = chcr_compute_partial_hash(shash, hmacctx->opad, 2239 hmacctx->opad, digestsize); 2240 if (err) 2241 goto out; 2242 chcr_change_order(hmacctx->opad, updated_digestsize); 2243 out: 2244 return err; 2245 } 2246 2247 static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, 2248 unsigned int key_len) 2249 { 2250 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 2251 unsigned short context_size = 0; 2252 int err; 2253 2254 err = chcr_cipher_fallback_setkey(cipher, key, key_len); 2255 if (err) 2256 goto badkey_err; 2257 2258 memcpy(ablkctx->key, key, key_len); 2259 ablkctx->enckey_len = key_len; 2260 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 2261 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 2262 /* Both keys for xts must be aligned to 16 byte boundary 2263 * by padding with zeros. So for 24 byte keys padding 8 zeroes. 2264 */ 2265 if (key_len == 48) { 2266 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len 2267 + 16) >> 4; 2268 memmove(ablkctx->key + 32, ablkctx->key + 24, 24); 2269 memset(ablkctx->key + 24, 0, 8); 2270 memset(ablkctx->key + 56, 0, 8); 2271 ablkctx->enckey_len = 64; 2272 ablkctx->key_ctx_hdr = 2273 FILL_KEY_CTX_HDR(CHCR_KEYCTX_CIPHER_KEY_SIZE_192, 2274 CHCR_KEYCTX_NO_KEY, 1, 2275 0, context_size); 2276 } else { 2277 ablkctx->key_ctx_hdr = 2278 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 2279 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 2280 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 2281 CHCR_KEYCTX_NO_KEY, 1, 2282 0, context_size); 2283 } 2284 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 2285 return 0; 2286 badkey_err: 2287 ablkctx->enckey_len = 0; 2288 2289 return err; 2290 } 2291 2292 static int chcr_sha_init(struct ahash_request *areq) 2293 { 2294 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2295 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2296 int digestsize = crypto_ahash_digestsize(tfm); 2297 2298 req_ctx->data_len = 0; 2299 req_ctx->reqlen = 0; 2300 req_ctx->reqbfr = req_ctx->bfr1; 2301 req_ctx->skbfr = req_ctx->bfr2; 2302 copy_hash_init_values(req_ctx->partial_hash, digestsize); 2303 2304 return 0; 2305 } 2306 2307 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 2308 { 2309 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2310 sizeof(struct chcr_ahash_req_ctx)); 2311 return chcr_device_init(crypto_tfm_ctx(tfm)); 2312 } 2313 2314 static int chcr_hmac_init(struct ahash_request *areq) 2315 { 2316 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 2317 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 2318 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); 2319 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 2320 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 2321 2322 chcr_sha_init(areq); 2323 req_ctx->data_len = bs; 2324 if (is_hmac(crypto_ahash_tfm(rtfm))) { 2325 if (digestsize == SHA224_DIGEST_SIZE) 2326 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2327 SHA256_DIGEST_SIZE); 2328 else if (digestsize == SHA384_DIGEST_SIZE) 2329 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2330 SHA512_DIGEST_SIZE); 2331 else 2332 memcpy(req_ctx->partial_hash, hmacctx->ipad, 2333 digestsize); 2334 } 2335 return 0; 2336 } 2337 2338 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 2339 { 2340 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2341 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2342 unsigned int digestsize = 2343 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 2344 2345 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2346 sizeof(struct chcr_ahash_req_ctx)); 2347 hmacctx->base_hash = chcr_alloc_shash(digestsize); 2348 if (IS_ERR(hmacctx->base_hash)) 2349 return PTR_ERR(hmacctx->base_hash); 2350 return chcr_device_init(crypto_tfm_ctx(tfm)); 2351 } 2352 2353 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 2354 { 2355 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2356 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2357 2358 if (hmacctx->base_hash) { 2359 chcr_free_shash(hmacctx->base_hash); 2360 hmacctx->base_hash = NULL; 2361 } 2362 } 2363 2364 inline void chcr_aead_common_exit(struct aead_request *req) 2365 { 2366 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2367 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2368 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); 2369 2370 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); 2371 } 2372 2373 static int chcr_aead_common_init(struct aead_request *req) 2374 { 2375 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2376 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2377 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2378 unsigned int authsize = crypto_aead_authsize(tfm); 2379 int error = -EINVAL; 2380 2381 /* validate key size */ 2382 if (aeadctx->enckey_len == 0) 2383 goto err; 2384 if (reqctx->op && req->cryptlen < authsize) 2385 goto err; 2386 if (reqctx->b0_len) 2387 reqctx->scratch_pad = reqctx->iv + IV; 2388 else 2389 reqctx->scratch_pad = NULL; 2390 2391 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2392 reqctx->op); 2393 if (error) { 2394 error = -ENOMEM; 2395 goto err; 2396 } 2397 2398 return 0; 2399 err: 2400 return error; 2401 } 2402 2403 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, 2404 int aadmax, int wrlen, 2405 unsigned short op_type) 2406 { 2407 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 2408 2409 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || 2410 dst_nents > MAX_DSGL_ENT || 2411 (req->assoclen > aadmax) || 2412 (wrlen > SGE_MAX_WR_LEN)) 2413 return 1; 2414 return 0; 2415 } 2416 2417 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) 2418 { 2419 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2420 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2421 struct aead_request *subreq = aead_request_ctx(req); 2422 2423 aead_request_set_tfm(subreq, aeadctx->sw_cipher); 2424 aead_request_set_callback(subreq, req->base.flags, 2425 req->base.complete, req->base.data); 2426 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2427 req->iv); 2428 aead_request_set_ad(subreq, req->assoclen); 2429 return op_type ? crypto_aead_decrypt(subreq) : 2430 crypto_aead_encrypt(subreq); 2431 } 2432 2433 static struct sk_buff *create_authenc_wr(struct aead_request *req, 2434 unsigned short qid, 2435 int size) 2436 { 2437 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2438 struct chcr_context *ctx = a_ctx(tfm); 2439 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2440 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2441 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2442 struct sk_buff *skb = NULL; 2443 struct chcr_wr *chcr_req; 2444 struct cpl_rx_phys_dsgl *phys_cpl; 2445 struct ulptx_sgl *ulptx; 2446 unsigned int transhdr_len; 2447 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm); 2448 unsigned int kctx_len = 0, dnents, snents; 2449 unsigned int authsize = crypto_aead_authsize(tfm); 2450 int error = -EINVAL; 2451 u8 *ivptr; 2452 int null = 0; 2453 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2454 GFP_ATOMIC; 2455 struct adapter *adap = padap(ctx->dev); 2456 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2457 2458 if (req->cryptlen == 0) 2459 return NULL; 2460 2461 reqctx->b0_len = 0; 2462 error = chcr_aead_common_init(req); 2463 if (error) 2464 return ERR_PTR(error); 2465 2466 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL || 2467 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2468 null = 1; 2469 } 2470 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + 2471 (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0); 2472 dnents += MIN_AUTH_SG; // For IV 2473 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 2474 CHCR_SRC_SG_SIZE, 0); 2475 dst_size = get_space_for_phys_dsgl(dnents); 2476 kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4) 2477 - sizeof(chcr_req->key_ctx); 2478 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2479 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) < 2480 SGE_MAX_WR_LEN; 2481 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) 2482 : (sgl_len(snents) * 8); 2483 transhdr_len += temp; 2484 transhdr_len = roundup(transhdr_len, 16); 2485 2486 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 2487 transhdr_len, reqctx->op)) { 2488 atomic_inc(&adap->chcr_stats.fallback); 2489 chcr_aead_common_exit(req); 2490 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 2491 } 2492 skb = alloc_skb(transhdr_len, flags); 2493 if (!skb) { 2494 error = -ENOMEM; 2495 goto err; 2496 } 2497 2498 chcr_req = __skb_put_zero(skb, transhdr_len); 2499 2500 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; 2501 2502 /* 2503 * Input order is AAD,IV and Payload. where IV should be included as 2504 * the part of authdata. All other fields should be filled according 2505 * to the hardware spec 2506 */ 2507 chcr_req->sec_cpl.op_ivinsrtofst = 2508 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); 2509 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen); 2510 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2511 null ? 0 : 1 + IV, 2512 null ? 0 : IV + req->assoclen, 2513 req->assoclen + IV + 1, 2514 (temp & 0x1F0) >> 4); 2515 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 2516 temp & 0xF, 2517 null ? 0 : req->assoclen + IV + 1, 2518 temp, temp); 2519 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL || 2520 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA) 2521 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR; 2522 else 2523 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC; 2524 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 2525 (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0, 2526 temp, 2527 actx->auth_mode, aeadctx->hmac_ctrl, 2528 IV >> 1); 2529 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2530 0, 0, dst_size); 2531 2532 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2533 if (reqctx->op == CHCR_ENCRYPT_OP || 2534 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2535 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) 2536 memcpy(chcr_req->key_ctx.key, aeadctx->key, 2537 aeadctx->enckey_len); 2538 else 2539 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 2540 aeadctx->enckey_len); 2541 2542 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 2543 actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16)); 2544 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2545 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 2546 ulptx = (struct ulptx_sgl *)(ivptr + IV); 2547 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2548 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2549 memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE); 2550 memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv, 2551 CTR_RFC3686_IV_SIZE); 2552 *(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE + 2553 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 2554 } else { 2555 memcpy(ivptr, req->iv, IV); 2556 } 2557 chcr_add_aead_dst_ent(req, phys_cpl, qid); 2558 chcr_add_aead_src_ent(req, ulptx); 2559 atomic_inc(&adap->chcr_stats.cipher_rqst); 2560 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 2561 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); 2562 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 2563 transhdr_len, temp, 0); 2564 reqctx->skb = skb; 2565 2566 return skb; 2567 err: 2568 chcr_aead_common_exit(req); 2569 2570 return ERR_PTR(error); 2571 } 2572 2573 int chcr_aead_dma_map(struct device *dev, 2574 struct aead_request *req, 2575 unsigned short op_type) 2576 { 2577 int error; 2578 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2579 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2580 unsigned int authsize = crypto_aead_authsize(tfm); 2581 int src_len, dst_len; 2582 2583 /* calculate and handle src and dst sg length separately 2584 * for inplace and out-of place operations 2585 */ 2586 if (req->src == req->dst) { 2587 src_len = req->assoclen + req->cryptlen + (op_type ? 2588 0 : authsize); 2589 dst_len = src_len; 2590 } else { 2591 src_len = req->assoclen + req->cryptlen; 2592 dst_len = req->assoclen + req->cryptlen + (op_type ? 2593 -authsize : authsize); 2594 } 2595 2596 if (!req->cryptlen || !src_len || !dst_len) 2597 return 0; 2598 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len), 2599 DMA_BIDIRECTIONAL); 2600 if (dma_mapping_error(dev, reqctx->iv_dma)) 2601 return -ENOMEM; 2602 if (reqctx->b0_len) 2603 reqctx->b0_dma = reqctx->iv_dma + IV; 2604 else 2605 reqctx->b0_dma = 0; 2606 if (req->src == req->dst) { 2607 error = dma_map_sg(dev, req->src, 2608 sg_nents_for_len(req->src, src_len), 2609 DMA_BIDIRECTIONAL); 2610 if (!error) 2611 goto err; 2612 } else { 2613 error = dma_map_sg(dev, req->src, 2614 sg_nents_for_len(req->src, src_len), 2615 DMA_TO_DEVICE); 2616 if (!error) 2617 goto err; 2618 error = dma_map_sg(dev, req->dst, 2619 sg_nents_for_len(req->dst, dst_len), 2620 DMA_FROM_DEVICE); 2621 if (!error) { 2622 dma_unmap_sg(dev, req->src, 2623 sg_nents_for_len(req->src, src_len), 2624 DMA_TO_DEVICE); 2625 goto err; 2626 } 2627 } 2628 2629 return 0; 2630 err: 2631 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 2632 return -ENOMEM; 2633 } 2634 2635 void chcr_aead_dma_unmap(struct device *dev, 2636 struct aead_request *req, 2637 unsigned short op_type) 2638 { 2639 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2640 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2641 unsigned int authsize = crypto_aead_authsize(tfm); 2642 int src_len, dst_len; 2643 2644 /* calculate and handle src and dst sg length separately 2645 * for inplace and out-of place operations 2646 */ 2647 if (req->src == req->dst) { 2648 src_len = req->assoclen + req->cryptlen + (op_type ? 2649 0 : authsize); 2650 dst_len = src_len; 2651 } else { 2652 src_len = req->assoclen + req->cryptlen; 2653 dst_len = req->assoclen + req->cryptlen + (op_type ? 2654 -authsize : authsize); 2655 } 2656 2657 if (!req->cryptlen || !src_len || !dst_len) 2658 return; 2659 2660 dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len), 2661 DMA_BIDIRECTIONAL); 2662 if (req->src == req->dst) { 2663 dma_unmap_sg(dev, req->src, 2664 sg_nents_for_len(req->src, src_len), 2665 DMA_BIDIRECTIONAL); 2666 } else { 2667 dma_unmap_sg(dev, req->src, 2668 sg_nents_for_len(req->src, src_len), 2669 DMA_TO_DEVICE); 2670 dma_unmap_sg(dev, req->dst, 2671 sg_nents_for_len(req->dst, dst_len), 2672 DMA_FROM_DEVICE); 2673 } 2674 } 2675 2676 void chcr_add_aead_src_ent(struct aead_request *req, 2677 struct ulptx_sgl *ulptx) 2678 { 2679 struct ulptx_walk ulp_walk; 2680 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2681 2682 if (reqctx->imm) { 2683 u8 *buf = (u8 *)ulptx; 2684 2685 if (reqctx->b0_len) { 2686 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); 2687 buf += reqctx->b0_len; 2688 } 2689 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2690 buf, req->cryptlen + req->assoclen, 0); 2691 } else { 2692 ulptx_walk_init(&ulp_walk, ulptx); 2693 if (reqctx->b0_len) 2694 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, 2695 reqctx->b0_dma); 2696 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen + 2697 req->assoclen, 0); 2698 ulptx_walk_end(&ulp_walk); 2699 } 2700 } 2701 2702 void chcr_add_aead_dst_ent(struct aead_request *req, 2703 struct cpl_rx_phys_dsgl *phys_cpl, 2704 unsigned short qid) 2705 { 2706 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2707 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2708 struct dsgl_walk dsgl_walk; 2709 unsigned int authsize = crypto_aead_authsize(tfm); 2710 struct chcr_context *ctx = a_ctx(tfm); 2711 u32 temp; 2712 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2713 2714 dsgl_walk_init(&dsgl_walk, phys_cpl); 2715 dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma); 2716 temp = req->assoclen + req->cryptlen + 2717 (reqctx->op ? -authsize : authsize); 2718 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0); 2719 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id); 2720 } 2721 2722 void chcr_add_cipher_src_ent(struct skcipher_request *req, 2723 void *ulptx, 2724 struct cipher_wr_param *wrparam) 2725 { 2726 struct ulptx_walk ulp_walk; 2727 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 2728 u8 *buf = ulptx; 2729 2730 memcpy(buf, reqctx->iv, IV); 2731 buf += IV; 2732 if (reqctx->imm) { 2733 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2734 buf, wrparam->bytes, reqctx->processed); 2735 } else { 2736 ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf); 2737 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, 2738 reqctx->src_ofst); 2739 reqctx->srcsg = ulp_walk.last_sg; 2740 reqctx->src_ofst = ulp_walk.last_sg_len; 2741 ulptx_walk_end(&ulp_walk); 2742 } 2743 } 2744 2745 void chcr_add_cipher_dst_ent(struct skcipher_request *req, 2746 struct cpl_rx_phys_dsgl *phys_cpl, 2747 struct cipher_wr_param *wrparam, 2748 unsigned short qid) 2749 { 2750 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req); 2751 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req); 2752 struct chcr_context *ctx = c_ctx(tfm); 2753 struct dsgl_walk dsgl_walk; 2754 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2755 2756 dsgl_walk_init(&dsgl_walk, phys_cpl); 2757 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, 2758 reqctx->dst_ofst); 2759 reqctx->dstsg = dsgl_walk.last_sg; 2760 reqctx->dst_ofst = dsgl_walk.last_sg_len; 2761 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id); 2762 } 2763 2764 void chcr_add_hash_src_ent(struct ahash_request *req, 2765 struct ulptx_sgl *ulptx, 2766 struct hash_wr_param *param) 2767 { 2768 struct ulptx_walk ulp_walk; 2769 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2770 2771 if (reqctx->hctx_wr.imm) { 2772 u8 *buf = (u8 *)ulptx; 2773 2774 if (param->bfr_len) { 2775 memcpy(buf, reqctx->reqbfr, param->bfr_len); 2776 buf += param->bfr_len; 2777 } 2778 2779 sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg, 2780 sg_nents(reqctx->hctx_wr.srcsg), buf, 2781 param->sg_len, 0); 2782 } else { 2783 ulptx_walk_init(&ulp_walk, ulptx); 2784 if (param->bfr_len) 2785 ulptx_walk_add_page(&ulp_walk, param->bfr_len, 2786 reqctx->hctx_wr.dma_addr); 2787 ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg, 2788 param->sg_len, reqctx->hctx_wr.src_ofst); 2789 reqctx->hctx_wr.srcsg = ulp_walk.last_sg; 2790 reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len; 2791 ulptx_walk_end(&ulp_walk); 2792 } 2793 } 2794 2795 int chcr_hash_dma_map(struct device *dev, 2796 struct ahash_request *req) 2797 { 2798 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2799 int error = 0; 2800 2801 if (!req->nbytes) 2802 return 0; 2803 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2804 DMA_TO_DEVICE); 2805 if (!error) 2806 return -ENOMEM; 2807 req_ctx->hctx_wr.is_sg_map = 1; 2808 return 0; 2809 } 2810 2811 void chcr_hash_dma_unmap(struct device *dev, 2812 struct ahash_request *req) 2813 { 2814 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2815 2816 if (!req->nbytes) 2817 return; 2818 2819 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2820 DMA_TO_DEVICE); 2821 req_ctx->hctx_wr.is_sg_map = 0; 2822 2823 } 2824 2825 int chcr_cipher_dma_map(struct device *dev, 2826 struct skcipher_request *req) 2827 { 2828 int error; 2829 2830 if (req->src == req->dst) { 2831 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2832 DMA_BIDIRECTIONAL); 2833 if (!error) 2834 goto err; 2835 } else { 2836 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2837 DMA_TO_DEVICE); 2838 if (!error) 2839 goto err; 2840 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2841 DMA_FROM_DEVICE); 2842 if (!error) { 2843 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2844 DMA_TO_DEVICE); 2845 goto err; 2846 } 2847 } 2848 2849 return 0; 2850 err: 2851 return -ENOMEM; 2852 } 2853 2854 void chcr_cipher_dma_unmap(struct device *dev, 2855 struct skcipher_request *req) 2856 { 2857 if (req->src == req->dst) { 2858 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2859 DMA_BIDIRECTIONAL); 2860 } else { 2861 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2862 DMA_TO_DEVICE); 2863 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2864 DMA_FROM_DEVICE); 2865 } 2866 } 2867 2868 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 2869 { 2870 __be32 data; 2871 2872 memset(block, 0, csize); 2873 block += csize; 2874 2875 if (csize >= 4) 2876 csize = 4; 2877 else if (msglen > (unsigned int)(1 << (8 * csize))) 2878 return -EOVERFLOW; 2879 2880 data = cpu_to_be32(msglen); 2881 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 2882 2883 return 0; 2884 } 2885 2886 static int generate_b0(struct aead_request *req, u8 *ivptr, 2887 unsigned short op_type) 2888 { 2889 unsigned int l, lp, m; 2890 int rc; 2891 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2892 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2893 u8 *b0 = reqctx->scratch_pad; 2894 2895 m = crypto_aead_authsize(aead); 2896 2897 memcpy(b0, ivptr, 16); 2898 2899 lp = b0[0]; 2900 l = lp + 1; 2901 2902 /* set m, bits 3-5 */ 2903 *b0 |= (8 * ((m - 2) / 2)); 2904 2905 /* set adata, bit 6, if associated data is used */ 2906 if (req->assoclen) 2907 *b0 |= 64; 2908 rc = set_msg_len(b0 + 16 - l, 2909 (op_type == CHCR_DECRYPT_OP) ? 2910 req->cryptlen - m : req->cryptlen, l); 2911 2912 return rc; 2913 } 2914 2915 static inline int crypto_ccm_check_iv(const u8 *iv) 2916 { 2917 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 2918 if (iv[0] < 1 || iv[0] > 7) 2919 return -EINVAL; 2920 2921 return 0; 2922 } 2923 2924 static int ccm_format_packet(struct aead_request *req, 2925 u8 *ivptr, 2926 unsigned int sub_type, 2927 unsigned short op_type, 2928 unsigned int assoclen) 2929 { 2930 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2931 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2932 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2933 int rc = 0; 2934 2935 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2936 ivptr[0] = 3; 2937 memcpy(ivptr + 1, &aeadctx->salt[0], 3); 2938 memcpy(ivptr + 4, req->iv, 8); 2939 memset(ivptr + 12, 0, 4); 2940 } else { 2941 memcpy(ivptr, req->iv, 16); 2942 } 2943 if (assoclen) 2944 put_unaligned_be16(assoclen, &reqctx->scratch_pad[16]); 2945 2946 rc = generate_b0(req, ivptr, op_type); 2947 /* zero the ctr value */ 2948 memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1); 2949 return rc; 2950 } 2951 2952 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 2953 unsigned int dst_size, 2954 struct aead_request *req, 2955 unsigned short op_type) 2956 { 2957 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2958 struct chcr_context *ctx = a_ctx(tfm); 2959 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2960 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2961 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 2962 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 2963 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 2964 unsigned int ccm_xtra; 2965 unsigned int tag_offset = 0, auth_offset = 0; 2966 unsigned int assoclen; 2967 2968 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2969 assoclen = req->assoclen - 8; 2970 else 2971 assoclen = req->assoclen; 2972 ccm_xtra = CCM_B0_SIZE + 2973 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 2974 2975 auth_offset = req->cryptlen ? 2976 (req->assoclen + IV + 1 + ccm_xtra) : 0; 2977 if (op_type == CHCR_DECRYPT_OP) { 2978 if (crypto_aead_authsize(tfm) != req->cryptlen) 2979 tag_offset = crypto_aead_authsize(tfm); 2980 else 2981 auth_offset = 0; 2982 } 2983 2984 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1); 2985 sec_cpl->pldlen = 2986 htonl(req->assoclen + IV + req->cryptlen + ccm_xtra); 2987 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 2988 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2989 1 + IV, IV + assoclen + ccm_xtra, 2990 req->assoclen + IV + 1 + ccm_xtra, 0); 2991 2992 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 2993 auth_offset, tag_offset, 2994 (op_type == CHCR_ENCRYPT_OP) ? 0 : 2995 crypto_aead_authsize(tfm)); 2996 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2997 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 2998 cipher_mode, mac_mode, 2999 aeadctx->hmac_ctrl, IV >> 1); 3000 3001 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 3002 0, dst_size); 3003 } 3004 3005 static int aead_ccm_validate_input(unsigned short op_type, 3006 struct aead_request *req, 3007 struct chcr_aead_ctx *aeadctx, 3008 unsigned int sub_type) 3009 { 3010 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 3011 if (crypto_ccm_check_iv(req->iv)) { 3012 pr_err("CCM: IV check fails\n"); 3013 return -EINVAL; 3014 } 3015 } else { 3016 if (req->assoclen != 16 && req->assoclen != 20) { 3017 pr_err("RFC4309: Invalid AAD length %d\n", 3018 req->assoclen); 3019 return -EINVAL; 3020 } 3021 } 3022 return 0; 3023 } 3024 3025 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 3026 unsigned short qid, 3027 int size) 3028 { 3029 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3030 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3031 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3032 struct sk_buff *skb = NULL; 3033 struct chcr_wr *chcr_req; 3034 struct cpl_rx_phys_dsgl *phys_cpl; 3035 struct ulptx_sgl *ulptx; 3036 unsigned int transhdr_len; 3037 unsigned int dst_size = 0, kctx_len, dnents, temp, snents; 3038 unsigned int sub_type, assoclen = req->assoclen; 3039 unsigned int authsize = crypto_aead_authsize(tfm); 3040 int error = -EINVAL; 3041 u8 *ivptr; 3042 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 3043 GFP_ATOMIC; 3044 struct adapter *adap = padap(a_ctx(tfm)->dev); 3045 3046 sub_type = get_aead_subtype(tfm); 3047 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 3048 assoclen -= 8; 3049 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); 3050 error = chcr_aead_common_init(req); 3051 if (error) 3052 return ERR_PTR(error); 3053 3054 error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type); 3055 if (error) 3056 goto err; 3057 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen 3058 + (reqctx->op ? -authsize : authsize), 3059 CHCR_DST_SG_SIZE, 0); 3060 dnents += MIN_CCM_SG; // For IV and B0 3061 dst_size = get_space_for_phys_dsgl(dnents); 3062 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 3063 CHCR_SRC_SG_SIZE, 0); 3064 snents += MIN_CCM_SG; //For B0 3065 kctx_len = roundup(aeadctx->enckey_len, 16) * 2; 3066 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 3067 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen + 3068 reqctx->b0_len) <= SGE_MAX_WR_LEN; 3069 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen + 3070 reqctx->b0_len, 16) : 3071 (sgl_len(snents) * 8); 3072 transhdr_len += temp; 3073 transhdr_len = roundup(transhdr_len, 16); 3074 3075 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - 3076 reqctx->b0_len, transhdr_len, reqctx->op)) { 3077 atomic_inc(&adap->chcr_stats.fallback); 3078 chcr_aead_common_exit(req); 3079 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 3080 } 3081 skb = alloc_skb(transhdr_len, flags); 3082 3083 if (!skb) { 3084 error = -ENOMEM; 3085 goto err; 3086 } 3087 3088 chcr_req = __skb_put_zero(skb, transhdr_len); 3089 3090 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op); 3091 3092 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 3093 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 3094 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 3095 aeadctx->key, aeadctx->enckey_len); 3096 3097 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 3098 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 3099 ulptx = (struct ulptx_sgl *)(ivptr + IV); 3100 error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen); 3101 if (error) 3102 goto dstmap_fail; 3103 chcr_add_aead_dst_ent(req, phys_cpl, qid); 3104 chcr_add_aead_src_ent(req, ulptx); 3105 3106 atomic_inc(&adap->chcr_stats.aead_rqst); 3107 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 3108 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen + 3109 reqctx->b0_len) : 0); 3110 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, 3111 transhdr_len, temp, 0); 3112 reqctx->skb = skb; 3113 3114 return skb; 3115 dstmap_fail: 3116 kfree_skb(skb); 3117 err: 3118 chcr_aead_common_exit(req); 3119 return ERR_PTR(error); 3120 } 3121 3122 static struct sk_buff *create_gcm_wr(struct aead_request *req, 3123 unsigned short qid, 3124 int size) 3125 { 3126 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3127 struct chcr_context *ctx = a_ctx(tfm); 3128 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 3129 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3130 struct sk_buff *skb = NULL; 3131 struct chcr_wr *chcr_req; 3132 struct cpl_rx_phys_dsgl *phys_cpl; 3133 struct ulptx_sgl *ulptx; 3134 unsigned int transhdr_len, dnents = 0, snents; 3135 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; 3136 unsigned int authsize = crypto_aead_authsize(tfm); 3137 int error = -EINVAL; 3138 u8 *ivptr; 3139 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 3140 GFP_ATOMIC; 3141 struct adapter *adap = padap(ctx->dev); 3142 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan; 3143 3144 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 3145 assoclen = req->assoclen - 8; 3146 3147 reqctx->b0_len = 0; 3148 error = chcr_aead_common_init(req); 3149 if (error) 3150 return ERR_PTR(error); 3151 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen + 3152 (reqctx->op ? -authsize : authsize), 3153 CHCR_DST_SG_SIZE, 0); 3154 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen, 3155 CHCR_SRC_SG_SIZE, 0); 3156 dnents += MIN_GCM_SG; // For IV 3157 dst_size = get_space_for_phys_dsgl(dnents); 3158 kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE; 3159 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 3160 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <= 3161 SGE_MAX_WR_LEN; 3162 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) : 3163 (sgl_len(snents) * 8); 3164 transhdr_len += temp; 3165 transhdr_len = roundup(transhdr_len, 16); 3166 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 3167 transhdr_len, reqctx->op)) { 3168 3169 atomic_inc(&adap->chcr_stats.fallback); 3170 chcr_aead_common_exit(req); 3171 return ERR_PTR(chcr_aead_fallback(req, reqctx->op)); 3172 } 3173 skb = alloc_skb(transhdr_len, flags); 3174 if (!skb) { 3175 error = -ENOMEM; 3176 goto err; 3177 } 3178 3179 chcr_req = __skb_put_zero(skb, transhdr_len); 3180 3181 //Offset of tag from end 3182 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize; 3183 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 3184 rx_channel_id, 2, 1); 3185 chcr_req->sec_cpl.pldlen = 3186 htonl(req->assoclen + IV + req->cryptlen); 3187 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 3188 assoclen ? 1 + IV : 0, 3189 assoclen ? IV + assoclen : 0, 3190 req->assoclen + IV + 1, 0); 3191 chcr_req->sec_cpl.cipherstop_lo_authinsert = 3192 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1, 3193 temp, temp); 3194 chcr_req->sec_cpl.seqno_numivs = 3195 FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op == 3196 CHCR_ENCRYPT_OP) ? 1 : 0, 3197 CHCR_SCMD_CIPHER_MODE_AES_GCM, 3198 CHCR_SCMD_AUTH_MODE_GHASH, 3199 aeadctx->hmac_ctrl, IV >> 1); 3200 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 3201 0, 0, dst_size); 3202 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 3203 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 3204 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16), 3205 GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 3206 3207 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 3208 ivptr = (u8 *)(phys_cpl + 1) + dst_size; 3209 /* prepare a 16 byte iv */ 3210 /* S A L T | IV | 0x00000001 */ 3211 if (get_aead_subtype(tfm) == 3212 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 3213 memcpy(ivptr, aeadctx->salt, 4); 3214 memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE); 3215 } else { 3216 memcpy(ivptr, req->iv, GCM_AES_IV_SIZE); 3217 } 3218 put_unaligned_be32(0x01, &ivptr[12]); 3219 ulptx = (struct ulptx_sgl *)(ivptr + 16); 3220 3221 chcr_add_aead_dst_ent(req, phys_cpl, qid); 3222 chcr_add_aead_src_ent(req, ulptx); 3223 atomic_inc(&adap->chcr_stats.aead_rqst); 3224 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV + 3225 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0); 3226 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 3227 transhdr_len, temp, reqctx->verify); 3228 reqctx->skb = skb; 3229 return skb; 3230 3231 err: 3232 chcr_aead_common_exit(req); 3233 return ERR_PTR(error); 3234 } 3235 3236 3237 3238 static int chcr_aead_cra_init(struct crypto_aead *tfm) 3239 { 3240 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3241 struct aead_alg *alg = crypto_aead_alg(tfm); 3242 3243 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, 3244 CRYPTO_ALG_NEED_FALLBACK | 3245 CRYPTO_ALG_ASYNC); 3246 if (IS_ERR(aeadctx->sw_cipher)) 3247 return PTR_ERR(aeadctx->sw_cipher); 3248 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), 3249 sizeof(struct aead_request) + 3250 crypto_aead_reqsize(aeadctx->sw_cipher))); 3251 return chcr_device_init(a_ctx(tfm)); 3252 } 3253 3254 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 3255 { 3256 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3257 3258 crypto_free_aead(aeadctx->sw_cipher); 3259 } 3260 3261 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 3262 unsigned int authsize) 3263 { 3264 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3265 3266 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 3267 aeadctx->mayverify = VERIFY_HW; 3268 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3269 } 3270 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 3271 unsigned int authsize) 3272 { 3273 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3274 u32 maxauth = crypto_aead_maxauthsize(tfm); 3275 3276 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 3277 * true for sha1. authsize == 12 condition should be before 3278 * authsize == (maxauth >> 1) 3279 */ 3280 if (authsize == ICV_4) { 3281 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3282 aeadctx->mayverify = VERIFY_HW; 3283 } else if (authsize == ICV_6) { 3284 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3285 aeadctx->mayverify = VERIFY_HW; 3286 } else if (authsize == ICV_10) { 3287 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3288 aeadctx->mayverify = VERIFY_HW; 3289 } else if (authsize == ICV_12) { 3290 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3291 aeadctx->mayverify = VERIFY_HW; 3292 } else if (authsize == ICV_14) { 3293 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3294 aeadctx->mayverify = VERIFY_HW; 3295 } else if (authsize == (maxauth >> 1)) { 3296 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3297 aeadctx->mayverify = VERIFY_HW; 3298 } else if (authsize == maxauth) { 3299 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3300 aeadctx->mayverify = VERIFY_HW; 3301 } else { 3302 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3303 aeadctx->mayverify = VERIFY_SW; 3304 } 3305 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3306 } 3307 3308 3309 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 3310 { 3311 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3312 3313 switch (authsize) { 3314 case ICV_4: 3315 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3316 aeadctx->mayverify = VERIFY_HW; 3317 break; 3318 case ICV_8: 3319 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3320 aeadctx->mayverify = VERIFY_HW; 3321 break; 3322 case ICV_12: 3323 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3324 aeadctx->mayverify = VERIFY_HW; 3325 break; 3326 case ICV_14: 3327 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3328 aeadctx->mayverify = VERIFY_HW; 3329 break; 3330 case ICV_16: 3331 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3332 aeadctx->mayverify = VERIFY_HW; 3333 break; 3334 case ICV_13: 3335 case ICV_15: 3336 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3337 aeadctx->mayverify = VERIFY_SW; 3338 break; 3339 default: 3340 return -EINVAL; 3341 } 3342 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3343 } 3344 3345 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 3346 unsigned int authsize) 3347 { 3348 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3349 3350 switch (authsize) { 3351 case ICV_8: 3352 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3353 aeadctx->mayverify = VERIFY_HW; 3354 break; 3355 case ICV_12: 3356 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3357 aeadctx->mayverify = VERIFY_HW; 3358 break; 3359 case ICV_16: 3360 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3361 aeadctx->mayverify = VERIFY_HW; 3362 break; 3363 default: 3364 return -EINVAL; 3365 } 3366 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3367 } 3368 3369 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 3370 unsigned int authsize) 3371 { 3372 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3373 3374 switch (authsize) { 3375 case ICV_4: 3376 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3377 aeadctx->mayverify = VERIFY_HW; 3378 break; 3379 case ICV_6: 3380 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3381 aeadctx->mayverify = VERIFY_HW; 3382 break; 3383 case ICV_8: 3384 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3385 aeadctx->mayverify = VERIFY_HW; 3386 break; 3387 case ICV_10: 3388 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3389 aeadctx->mayverify = VERIFY_HW; 3390 break; 3391 case ICV_12: 3392 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3393 aeadctx->mayverify = VERIFY_HW; 3394 break; 3395 case ICV_14: 3396 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3397 aeadctx->mayverify = VERIFY_HW; 3398 break; 3399 case ICV_16: 3400 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3401 aeadctx->mayverify = VERIFY_HW; 3402 break; 3403 default: 3404 return -EINVAL; 3405 } 3406 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3407 } 3408 3409 static int chcr_ccm_common_setkey(struct crypto_aead *aead, 3410 const u8 *key, 3411 unsigned int keylen) 3412 { 3413 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3414 unsigned char ck_size, mk_size; 3415 int key_ctx_size = 0; 3416 3417 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2; 3418 if (keylen == AES_KEYSIZE_128) { 3419 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3420 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; 3421 } else if (keylen == AES_KEYSIZE_192) { 3422 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3423 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 3424 } else if (keylen == AES_KEYSIZE_256) { 3425 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3426 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 3427 } else { 3428 aeadctx->enckey_len = 0; 3429 return -EINVAL; 3430 } 3431 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 3432 key_ctx_size >> 4); 3433 memcpy(aeadctx->key, key, keylen); 3434 aeadctx->enckey_len = keylen; 3435 3436 return 0; 3437 } 3438 3439 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 3440 const u8 *key, 3441 unsigned int keylen) 3442 { 3443 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3444 int error; 3445 3446 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3447 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3448 CRYPTO_TFM_REQ_MASK); 3449 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3450 if (error) 3451 return error; 3452 return chcr_ccm_common_setkey(aead, key, keylen); 3453 } 3454 3455 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 3456 unsigned int keylen) 3457 { 3458 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3459 int error; 3460 3461 if (keylen < 3) { 3462 aeadctx->enckey_len = 0; 3463 return -EINVAL; 3464 } 3465 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3466 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3467 CRYPTO_TFM_REQ_MASK); 3468 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3469 if (error) 3470 return error; 3471 keylen -= 3; 3472 memcpy(aeadctx->salt, key + keylen, 3); 3473 return chcr_ccm_common_setkey(aead, key, keylen); 3474 } 3475 3476 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 3477 unsigned int keylen) 3478 { 3479 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3480 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 3481 unsigned int ck_size; 3482 int ret = 0, key_ctx_size = 0; 3483 struct crypto_aes_ctx aes; 3484 3485 aeadctx->enckey_len = 0; 3486 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3487 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) 3488 & CRYPTO_TFM_REQ_MASK); 3489 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3490 if (ret) 3491 goto out; 3492 3493 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 3494 keylen > 3) { 3495 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 3496 memcpy(aeadctx->salt, key + keylen, 4); 3497 } 3498 if (keylen == AES_KEYSIZE_128) { 3499 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3500 } else if (keylen == AES_KEYSIZE_192) { 3501 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3502 } else if (keylen == AES_KEYSIZE_256) { 3503 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3504 } else { 3505 pr_err("GCM: Invalid key length %d\n", keylen); 3506 ret = -EINVAL; 3507 goto out; 3508 } 3509 3510 memcpy(aeadctx->key, key, keylen); 3511 aeadctx->enckey_len = keylen; 3512 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) + 3513 AEAD_H_SIZE; 3514 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 3515 CHCR_KEYCTX_MAC_KEY_SIZE_128, 3516 0, 0, 3517 key_ctx_size >> 4); 3518 /* Calculate the H = CIPH(K, 0 repeated 16 times). 3519 * It will go in key context 3520 */ 3521 ret = aes_expandkey(&aes, key, keylen); 3522 if (ret) { 3523 aeadctx->enckey_len = 0; 3524 goto out; 3525 } 3526 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 3527 aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h); 3528 memzero_explicit(&aes, sizeof(aes)); 3529 3530 out: 3531 return ret; 3532 } 3533 3534 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 3535 unsigned int keylen) 3536 { 3537 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3538 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3539 /* it contains auth and cipher key both*/ 3540 struct crypto_authenc_keys keys; 3541 unsigned int bs, subtype; 3542 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 3543 int err = 0, i, key_ctx_len = 0; 3544 unsigned char ck_size = 0; 3545 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 3546 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 3547 struct algo_param param; 3548 int align; 3549 u8 *o_ptr = NULL; 3550 3551 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3552 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3553 & CRYPTO_TFM_REQ_MASK); 3554 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3555 if (err) 3556 goto out; 3557 3558 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 3559 goto out; 3560 3561 if (get_alg_config(¶m, max_authsize)) { 3562 pr_err("Unsupported digest size\n"); 3563 goto out; 3564 } 3565 subtype = get_aead_subtype(authenc); 3566 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3567 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3568 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3569 goto out; 3570 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3571 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3572 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3573 } 3574 if (keys.enckeylen == AES_KEYSIZE_128) { 3575 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3576 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3577 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3578 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3579 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3580 } else { 3581 pr_err("Unsupported cipher key\n"); 3582 goto out; 3583 } 3584 3585 /* Copy only encryption key. We use authkey to generate h(ipad) and 3586 * h(opad) so authkey is not needed again. authkeylen size have the 3587 * size of the hash digest size. 3588 */ 3589 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3590 aeadctx->enckey_len = keys.enckeylen; 3591 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3592 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3593 3594 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3595 aeadctx->enckey_len << 3); 3596 } 3597 base_hash = chcr_alloc_shash(max_authsize); 3598 if (IS_ERR(base_hash)) { 3599 pr_err("Base driver cannot be loaded\n"); 3600 goto out; 3601 } 3602 { 3603 SHASH_DESC_ON_STACK(shash, base_hash); 3604 3605 shash->tfm = base_hash; 3606 bs = crypto_shash_blocksize(base_hash); 3607 align = KEYCTX_ALIGN_PAD(max_authsize); 3608 o_ptr = actx->h_iopad + param.result_size + align; 3609 3610 if (keys.authkeylen > bs) { 3611 err = crypto_shash_digest(shash, keys.authkey, 3612 keys.authkeylen, 3613 o_ptr); 3614 if (err) { 3615 pr_err("Base driver cannot be loaded\n"); 3616 goto out; 3617 } 3618 keys.authkeylen = max_authsize; 3619 } else 3620 memcpy(o_ptr, keys.authkey, keys.authkeylen); 3621 3622 /* Compute the ipad-digest*/ 3623 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3624 memcpy(pad, o_ptr, keys.authkeylen); 3625 for (i = 0; i < bs >> 2; i++) 3626 *((unsigned int *)pad + i) ^= IPAD_DATA; 3627 3628 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 3629 max_authsize)) 3630 goto out; 3631 /* Compute the opad-digest */ 3632 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3633 memcpy(pad, o_ptr, keys.authkeylen); 3634 for (i = 0; i < bs >> 2; i++) 3635 *((unsigned int *)pad + i) ^= OPAD_DATA; 3636 3637 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 3638 goto out; 3639 3640 /* convert the ipad and opad digest to network order */ 3641 chcr_change_order(actx->h_iopad, param.result_size); 3642 chcr_change_order(o_ptr, param.result_size); 3643 key_ctx_len = sizeof(struct _key_ctx) + 3644 roundup(keys.enckeylen, 16) + 3645 (param.result_size + align) * 2; 3646 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 3647 0, 1, key_ctx_len >> 4); 3648 actx->auth_mode = param.auth_mode; 3649 chcr_free_shash(base_hash); 3650 3651 memzero_explicit(&keys, sizeof(keys)); 3652 return 0; 3653 } 3654 out: 3655 aeadctx->enckey_len = 0; 3656 memzero_explicit(&keys, sizeof(keys)); 3657 if (!IS_ERR(base_hash)) 3658 chcr_free_shash(base_hash); 3659 return -EINVAL; 3660 } 3661 3662 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 3663 const u8 *key, unsigned int keylen) 3664 { 3665 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3666 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3667 struct crypto_authenc_keys keys; 3668 int err; 3669 /* it contains auth and cipher key both*/ 3670 unsigned int subtype; 3671 int key_ctx_len = 0; 3672 unsigned char ck_size = 0; 3673 3674 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3675 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3676 & CRYPTO_TFM_REQ_MASK); 3677 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3678 if (err) 3679 goto out; 3680 3681 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 3682 goto out; 3683 3684 subtype = get_aead_subtype(authenc); 3685 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3686 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3687 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3688 goto out; 3689 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3690 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3691 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3692 } 3693 if (keys.enckeylen == AES_KEYSIZE_128) { 3694 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3695 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3696 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3697 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3698 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3699 } else { 3700 pr_err("Unsupported cipher key %d\n", keys.enckeylen); 3701 goto out; 3702 } 3703 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3704 aeadctx->enckey_len = keys.enckeylen; 3705 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3706 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3707 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3708 aeadctx->enckey_len << 3); 3709 } 3710 key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16); 3711 3712 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 3713 0, key_ctx_len >> 4); 3714 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 3715 memzero_explicit(&keys, sizeof(keys)); 3716 return 0; 3717 out: 3718 aeadctx->enckey_len = 0; 3719 memzero_explicit(&keys, sizeof(keys)); 3720 return -EINVAL; 3721 } 3722 3723 static int chcr_aead_op(struct aead_request *req, 3724 int size, 3725 create_wr_t create_wr_fn) 3726 { 3727 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3728 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3729 struct chcr_context *ctx = a_ctx(tfm); 3730 struct uld_ctx *u_ctx = ULD_CTX(ctx); 3731 struct sk_buff *skb; 3732 struct chcr_dev *cdev; 3733 3734 cdev = a_ctx(tfm)->dev; 3735 if (!cdev) { 3736 pr_err("%s : No crypto device.\n", __func__); 3737 return -ENXIO; 3738 } 3739 3740 if (chcr_inc_wrcount(cdev)) { 3741 /* Detach state for CHCR means lldi or padap is freed. 3742 * We cannot increment fallback here. 3743 */ 3744 return chcr_aead_fallback(req, reqctx->op); 3745 } 3746 3747 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 3748 reqctx->txqidx) && 3749 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) { 3750 chcr_dec_wrcount(cdev); 3751 return -ENOSPC; 3752 } 3753 3754 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 3755 crypto_ipsec_check_assoclen(req->assoclen) != 0) { 3756 pr_err("RFC4106: Invalid value of assoclen %d\n", 3757 req->assoclen); 3758 return -EINVAL; 3759 } 3760 3761 /* Form a WR from req */ 3762 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size); 3763 3764 if (IS_ERR_OR_NULL(skb)) { 3765 chcr_dec_wrcount(cdev); 3766 return PTR_ERR_OR_ZERO(skb); 3767 } 3768 3769 skb->dev = u_ctx->lldi.ports[0]; 3770 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx); 3771 chcr_send_wr(skb); 3772 return -EINPROGRESS; 3773 } 3774 3775 static int chcr_aead_encrypt(struct aead_request *req) 3776 { 3777 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3778 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3779 struct chcr_context *ctx = a_ctx(tfm); 3780 unsigned int cpu; 3781 3782 cpu = get_cpu(); 3783 reqctx->txqidx = cpu % ctx->ntxq; 3784 reqctx->rxqidx = cpu % ctx->nrxq; 3785 put_cpu(); 3786 3787 reqctx->verify = VERIFY_HW; 3788 reqctx->op = CHCR_ENCRYPT_OP; 3789 3790 switch (get_aead_subtype(tfm)) { 3791 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3792 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3793 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3794 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3795 return chcr_aead_op(req, 0, create_authenc_wr); 3796 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3797 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3798 return chcr_aead_op(req, 0, create_aead_ccm_wr); 3799 default: 3800 return chcr_aead_op(req, 0, create_gcm_wr); 3801 } 3802 } 3803 3804 static int chcr_aead_decrypt(struct aead_request *req) 3805 { 3806 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3807 struct chcr_context *ctx = a_ctx(tfm); 3808 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 3809 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3810 int size; 3811 unsigned int cpu; 3812 3813 cpu = get_cpu(); 3814 reqctx->txqidx = cpu % ctx->ntxq; 3815 reqctx->rxqidx = cpu % ctx->nrxq; 3816 put_cpu(); 3817 3818 if (aeadctx->mayverify == VERIFY_SW) { 3819 size = crypto_aead_maxauthsize(tfm); 3820 reqctx->verify = VERIFY_SW; 3821 } else { 3822 size = 0; 3823 reqctx->verify = VERIFY_HW; 3824 } 3825 reqctx->op = CHCR_DECRYPT_OP; 3826 switch (get_aead_subtype(tfm)) { 3827 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3828 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3829 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3830 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3831 return chcr_aead_op(req, size, create_authenc_wr); 3832 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3833 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3834 return chcr_aead_op(req, size, create_aead_ccm_wr); 3835 default: 3836 return chcr_aead_op(req, size, create_gcm_wr); 3837 } 3838 } 3839 3840 static struct chcr_alg_template driver_algs[] = { 3841 /* AES-CBC */ 3842 { 3843 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, 3844 .is_registered = 0, 3845 .alg.skcipher = { 3846 .base.cra_name = "cbc(aes)", 3847 .base.cra_driver_name = "cbc-aes-chcr", 3848 .base.cra_blocksize = AES_BLOCK_SIZE, 3849 3850 .init = chcr_init_tfm, 3851 .exit = chcr_exit_tfm, 3852 .min_keysize = AES_MIN_KEY_SIZE, 3853 .max_keysize = AES_MAX_KEY_SIZE, 3854 .ivsize = AES_BLOCK_SIZE, 3855 .setkey = chcr_aes_cbc_setkey, 3856 .encrypt = chcr_aes_encrypt, 3857 .decrypt = chcr_aes_decrypt, 3858 } 3859 }, 3860 { 3861 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, 3862 .is_registered = 0, 3863 .alg.skcipher = { 3864 .base.cra_name = "xts(aes)", 3865 .base.cra_driver_name = "xts-aes-chcr", 3866 .base.cra_blocksize = AES_BLOCK_SIZE, 3867 3868 .init = chcr_init_tfm, 3869 .exit = chcr_exit_tfm, 3870 .min_keysize = 2 * AES_MIN_KEY_SIZE, 3871 .max_keysize = 2 * AES_MAX_KEY_SIZE, 3872 .ivsize = AES_BLOCK_SIZE, 3873 .setkey = chcr_aes_xts_setkey, 3874 .encrypt = chcr_aes_encrypt, 3875 .decrypt = chcr_aes_decrypt, 3876 } 3877 }, 3878 { 3879 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, 3880 .is_registered = 0, 3881 .alg.skcipher = { 3882 .base.cra_name = "ctr(aes)", 3883 .base.cra_driver_name = "ctr-aes-chcr", 3884 .base.cra_blocksize = 1, 3885 3886 .init = chcr_init_tfm, 3887 .exit = chcr_exit_tfm, 3888 .min_keysize = AES_MIN_KEY_SIZE, 3889 .max_keysize = AES_MAX_KEY_SIZE, 3890 .ivsize = AES_BLOCK_SIZE, 3891 .setkey = chcr_aes_ctr_setkey, 3892 .encrypt = chcr_aes_encrypt, 3893 .decrypt = chcr_aes_decrypt, 3894 } 3895 }, 3896 { 3897 .type = CRYPTO_ALG_TYPE_SKCIPHER | 3898 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, 3899 .is_registered = 0, 3900 .alg.skcipher = { 3901 .base.cra_name = "rfc3686(ctr(aes))", 3902 .base.cra_driver_name = "rfc3686-ctr-aes-chcr", 3903 .base.cra_blocksize = 1, 3904 3905 .init = chcr_rfc3686_init, 3906 .exit = chcr_exit_tfm, 3907 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3908 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3909 .ivsize = CTR_RFC3686_IV_SIZE, 3910 .setkey = chcr_aes_rfc3686_setkey, 3911 .encrypt = chcr_aes_encrypt, 3912 .decrypt = chcr_aes_decrypt, 3913 } 3914 }, 3915 /* SHA */ 3916 { 3917 .type = CRYPTO_ALG_TYPE_AHASH, 3918 .is_registered = 0, 3919 .alg.hash = { 3920 .halg.digestsize = SHA1_DIGEST_SIZE, 3921 .halg.base = { 3922 .cra_name = "sha1", 3923 .cra_driver_name = "sha1-chcr", 3924 .cra_blocksize = SHA1_BLOCK_SIZE, 3925 } 3926 } 3927 }, 3928 { 3929 .type = CRYPTO_ALG_TYPE_AHASH, 3930 .is_registered = 0, 3931 .alg.hash = { 3932 .halg.digestsize = SHA256_DIGEST_SIZE, 3933 .halg.base = { 3934 .cra_name = "sha256", 3935 .cra_driver_name = "sha256-chcr", 3936 .cra_blocksize = SHA256_BLOCK_SIZE, 3937 } 3938 } 3939 }, 3940 { 3941 .type = CRYPTO_ALG_TYPE_AHASH, 3942 .is_registered = 0, 3943 .alg.hash = { 3944 .halg.digestsize = SHA224_DIGEST_SIZE, 3945 .halg.base = { 3946 .cra_name = "sha224", 3947 .cra_driver_name = "sha224-chcr", 3948 .cra_blocksize = SHA224_BLOCK_SIZE, 3949 } 3950 } 3951 }, 3952 { 3953 .type = CRYPTO_ALG_TYPE_AHASH, 3954 .is_registered = 0, 3955 .alg.hash = { 3956 .halg.digestsize = SHA384_DIGEST_SIZE, 3957 .halg.base = { 3958 .cra_name = "sha384", 3959 .cra_driver_name = "sha384-chcr", 3960 .cra_blocksize = SHA384_BLOCK_SIZE, 3961 } 3962 } 3963 }, 3964 { 3965 .type = CRYPTO_ALG_TYPE_AHASH, 3966 .is_registered = 0, 3967 .alg.hash = { 3968 .halg.digestsize = SHA512_DIGEST_SIZE, 3969 .halg.base = { 3970 .cra_name = "sha512", 3971 .cra_driver_name = "sha512-chcr", 3972 .cra_blocksize = SHA512_BLOCK_SIZE, 3973 } 3974 } 3975 }, 3976 /* HMAC */ 3977 { 3978 .type = CRYPTO_ALG_TYPE_HMAC, 3979 .is_registered = 0, 3980 .alg.hash = { 3981 .halg.digestsize = SHA1_DIGEST_SIZE, 3982 .halg.base = { 3983 .cra_name = "hmac(sha1)", 3984 .cra_driver_name = "hmac-sha1-chcr", 3985 .cra_blocksize = SHA1_BLOCK_SIZE, 3986 } 3987 } 3988 }, 3989 { 3990 .type = CRYPTO_ALG_TYPE_HMAC, 3991 .is_registered = 0, 3992 .alg.hash = { 3993 .halg.digestsize = SHA224_DIGEST_SIZE, 3994 .halg.base = { 3995 .cra_name = "hmac(sha224)", 3996 .cra_driver_name = "hmac-sha224-chcr", 3997 .cra_blocksize = SHA224_BLOCK_SIZE, 3998 } 3999 } 4000 }, 4001 { 4002 .type = CRYPTO_ALG_TYPE_HMAC, 4003 .is_registered = 0, 4004 .alg.hash = { 4005 .halg.digestsize = SHA256_DIGEST_SIZE, 4006 .halg.base = { 4007 .cra_name = "hmac(sha256)", 4008 .cra_driver_name = "hmac-sha256-chcr", 4009 .cra_blocksize = SHA256_BLOCK_SIZE, 4010 } 4011 } 4012 }, 4013 { 4014 .type = CRYPTO_ALG_TYPE_HMAC, 4015 .is_registered = 0, 4016 .alg.hash = { 4017 .halg.digestsize = SHA384_DIGEST_SIZE, 4018 .halg.base = { 4019 .cra_name = "hmac(sha384)", 4020 .cra_driver_name = "hmac-sha384-chcr", 4021 .cra_blocksize = SHA384_BLOCK_SIZE, 4022 } 4023 } 4024 }, 4025 { 4026 .type = CRYPTO_ALG_TYPE_HMAC, 4027 .is_registered = 0, 4028 .alg.hash = { 4029 .halg.digestsize = SHA512_DIGEST_SIZE, 4030 .halg.base = { 4031 .cra_name = "hmac(sha512)", 4032 .cra_driver_name = "hmac-sha512-chcr", 4033 .cra_blocksize = SHA512_BLOCK_SIZE, 4034 } 4035 } 4036 }, 4037 /* Add AEAD Algorithms */ 4038 { 4039 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 4040 .is_registered = 0, 4041 .alg.aead = { 4042 .base = { 4043 .cra_name = "gcm(aes)", 4044 .cra_driver_name = "gcm-aes-chcr", 4045 .cra_blocksize = 1, 4046 .cra_priority = CHCR_AEAD_PRIORITY, 4047 .cra_ctxsize = sizeof(struct chcr_context) + 4048 sizeof(struct chcr_aead_ctx) + 4049 sizeof(struct chcr_gcm_ctx), 4050 }, 4051 .ivsize = GCM_AES_IV_SIZE, 4052 .maxauthsize = GHASH_DIGEST_SIZE, 4053 .setkey = chcr_gcm_setkey, 4054 .setauthsize = chcr_gcm_setauthsize, 4055 } 4056 }, 4057 { 4058 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 4059 .is_registered = 0, 4060 .alg.aead = { 4061 .base = { 4062 .cra_name = "rfc4106(gcm(aes))", 4063 .cra_driver_name = "rfc4106-gcm-aes-chcr", 4064 .cra_blocksize = 1, 4065 .cra_priority = CHCR_AEAD_PRIORITY + 1, 4066 .cra_ctxsize = sizeof(struct chcr_context) + 4067 sizeof(struct chcr_aead_ctx) + 4068 sizeof(struct chcr_gcm_ctx), 4069 4070 }, 4071 .ivsize = GCM_RFC4106_IV_SIZE, 4072 .maxauthsize = GHASH_DIGEST_SIZE, 4073 .setkey = chcr_gcm_setkey, 4074 .setauthsize = chcr_4106_4309_setauthsize, 4075 } 4076 }, 4077 { 4078 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 4079 .is_registered = 0, 4080 .alg.aead = { 4081 .base = { 4082 .cra_name = "ccm(aes)", 4083 .cra_driver_name = "ccm-aes-chcr", 4084 .cra_blocksize = 1, 4085 .cra_priority = CHCR_AEAD_PRIORITY, 4086 .cra_ctxsize = sizeof(struct chcr_context) + 4087 sizeof(struct chcr_aead_ctx), 4088 4089 }, 4090 .ivsize = AES_BLOCK_SIZE, 4091 .maxauthsize = GHASH_DIGEST_SIZE, 4092 .setkey = chcr_aead_ccm_setkey, 4093 .setauthsize = chcr_ccm_setauthsize, 4094 } 4095 }, 4096 { 4097 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 4098 .is_registered = 0, 4099 .alg.aead = { 4100 .base = { 4101 .cra_name = "rfc4309(ccm(aes))", 4102 .cra_driver_name = "rfc4309-ccm-aes-chcr", 4103 .cra_blocksize = 1, 4104 .cra_priority = CHCR_AEAD_PRIORITY + 1, 4105 .cra_ctxsize = sizeof(struct chcr_context) + 4106 sizeof(struct chcr_aead_ctx), 4107 4108 }, 4109 .ivsize = 8, 4110 .maxauthsize = GHASH_DIGEST_SIZE, 4111 .setkey = chcr_aead_rfc4309_setkey, 4112 .setauthsize = chcr_4106_4309_setauthsize, 4113 } 4114 }, 4115 { 4116 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4117 .is_registered = 0, 4118 .alg.aead = { 4119 .base = { 4120 .cra_name = "authenc(hmac(sha1),cbc(aes))", 4121 .cra_driver_name = 4122 "authenc-hmac-sha1-cbc-aes-chcr", 4123 .cra_blocksize = AES_BLOCK_SIZE, 4124 .cra_priority = CHCR_AEAD_PRIORITY, 4125 .cra_ctxsize = sizeof(struct chcr_context) + 4126 sizeof(struct chcr_aead_ctx) + 4127 sizeof(struct chcr_authenc_ctx), 4128 4129 }, 4130 .ivsize = AES_BLOCK_SIZE, 4131 .maxauthsize = SHA1_DIGEST_SIZE, 4132 .setkey = chcr_authenc_setkey, 4133 .setauthsize = chcr_authenc_setauthsize, 4134 } 4135 }, 4136 { 4137 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4138 .is_registered = 0, 4139 .alg.aead = { 4140 .base = { 4141 4142 .cra_name = "authenc(hmac(sha256),cbc(aes))", 4143 .cra_driver_name = 4144 "authenc-hmac-sha256-cbc-aes-chcr", 4145 .cra_blocksize = AES_BLOCK_SIZE, 4146 .cra_priority = CHCR_AEAD_PRIORITY, 4147 .cra_ctxsize = sizeof(struct chcr_context) + 4148 sizeof(struct chcr_aead_ctx) + 4149 sizeof(struct chcr_authenc_ctx), 4150 4151 }, 4152 .ivsize = AES_BLOCK_SIZE, 4153 .maxauthsize = SHA256_DIGEST_SIZE, 4154 .setkey = chcr_authenc_setkey, 4155 .setauthsize = chcr_authenc_setauthsize, 4156 } 4157 }, 4158 { 4159 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4160 .is_registered = 0, 4161 .alg.aead = { 4162 .base = { 4163 .cra_name = "authenc(hmac(sha224),cbc(aes))", 4164 .cra_driver_name = 4165 "authenc-hmac-sha224-cbc-aes-chcr", 4166 .cra_blocksize = AES_BLOCK_SIZE, 4167 .cra_priority = CHCR_AEAD_PRIORITY, 4168 .cra_ctxsize = sizeof(struct chcr_context) + 4169 sizeof(struct chcr_aead_ctx) + 4170 sizeof(struct chcr_authenc_ctx), 4171 }, 4172 .ivsize = AES_BLOCK_SIZE, 4173 .maxauthsize = SHA224_DIGEST_SIZE, 4174 .setkey = chcr_authenc_setkey, 4175 .setauthsize = chcr_authenc_setauthsize, 4176 } 4177 }, 4178 { 4179 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4180 .is_registered = 0, 4181 .alg.aead = { 4182 .base = { 4183 .cra_name = "authenc(hmac(sha384),cbc(aes))", 4184 .cra_driver_name = 4185 "authenc-hmac-sha384-cbc-aes-chcr", 4186 .cra_blocksize = AES_BLOCK_SIZE, 4187 .cra_priority = CHCR_AEAD_PRIORITY, 4188 .cra_ctxsize = sizeof(struct chcr_context) + 4189 sizeof(struct chcr_aead_ctx) + 4190 sizeof(struct chcr_authenc_ctx), 4191 4192 }, 4193 .ivsize = AES_BLOCK_SIZE, 4194 .maxauthsize = SHA384_DIGEST_SIZE, 4195 .setkey = chcr_authenc_setkey, 4196 .setauthsize = chcr_authenc_setauthsize, 4197 } 4198 }, 4199 { 4200 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 4201 .is_registered = 0, 4202 .alg.aead = { 4203 .base = { 4204 .cra_name = "authenc(hmac(sha512),cbc(aes))", 4205 .cra_driver_name = 4206 "authenc-hmac-sha512-cbc-aes-chcr", 4207 .cra_blocksize = AES_BLOCK_SIZE, 4208 .cra_priority = CHCR_AEAD_PRIORITY, 4209 .cra_ctxsize = sizeof(struct chcr_context) + 4210 sizeof(struct chcr_aead_ctx) + 4211 sizeof(struct chcr_authenc_ctx), 4212 4213 }, 4214 .ivsize = AES_BLOCK_SIZE, 4215 .maxauthsize = SHA512_DIGEST_SIZE, 4216 .setkey = chcr_authenc_setkey, 4217 .setauthsize = chcr_authenc_setauthsize, 4218 } 4219 }, 4220 { 4221 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL, 4222 .is_registered = 0, 4223 .alg.aead = { 4224 .base = { 4225 .cra_name = "authenc(digest_null,cbc(aes))", 4226 .cra_driver_name = 4227 "authenc-digest_null-cbc-aes-chcr", 4228 .cra_blocksize = AES_BLOCK_SIZE, 4229 .cra_priority = CHCR_AEAD_PRIORITY, 4230 .cra_ctxsize = sizeof(struct chcr_context) + 4231 sizeof(struct chcr_aead_ctx) + 4232 sizeof(struct chcr_authenc_ctx), 4233 4234 }, 4235 .ivsize = AES_BLOCK_SIZE, 4236 .maxauthsize = 0, 4237 .setkey = chcr_aead_digest_null_setkey, 4238 .setauthsize = chcr_authenc_null_setauthsize, 4239 } 4240 }, 4241 { 4242 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4243 .is_registered = 0, 4244 .alg.aead = { 4245 .base = { 4246 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 4247 .cra_driver_name = 4248 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr", 4249 .cra_blocksize = 1, 4250 .cra_priority = CHCR_AEAD_PRIORITY, 4251 .cra_ctxsize = sizeof(struct chcr_context) + 4252 sizeof(struct chcr_aead_ctx) + 4253 sizeof(struct chcr_authenc_ctx), 4254 4255 }, 4256 .ivsize = CTR_RFC3686_IV_SIZE, 4257 .maxauthsize = SHA1_DIGEST_SIZE, 4258 .setkey = chcr_authenc_setkey, 4259 .setauthsize = chcr_authenc_setauthsize, 4260 } 4261 }, 4262 { 4263 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4264 .is_registered = 0, 4265 .alg.aead = { 4266 .base = { 4267 4268 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 4269 .cra_driver_name = 4270 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr", 4271 .cra_blocksize = 1, 4272 .cra_priority = CHCR_AEAD_PRIORITY, 4273 .cra_ctxsize = sizeof(struct chcr_context) + 4274 sizeof(struct chcr_aead_ctx) + 4275 sizeof(struct chcr_authenc_ctx), 4276 4277 }, 4278 .ivsize = CTR_RFC3686_IV_SIZE, 4279 .maxauthsize = SHA256_DIGEST_SIZE, 4280 .setkey = chcr_authenc_setkey, 4281 .setauthsize = chcr_authenc_setauthsize, 4282 } 4283 }, 4284 { 4285 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4286 .is_registered = 0, 4287 .alg.aead = { 4288 .base = { 4289 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 4290 .cra_driver_name = 4291 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr", 4292 .cra_blocksize = 1, 4293 .cra_priority = CHCR_AEAD_PRIORITY, 4294 .cra_ctxsize = sizeof(struct chcr_context) + 4295 sizeof(struct chcr_aead_ctx) + 4296 sizeof(struct chcr_authenc_ctx), 4297 }, 4298 .ivsize = CTR_RFC3686_IV_SIZE, 4299 .maxauthsize = SHA224_DIGEST_SIZE, 4300 .setkey = chcr_authenc_setkey, 4301 .setauthsize = chcr_authenc_setauthsize, 4302 } 4303 }, 4304 { 4305 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4306 .is_registered = 0, 4307 .alg.aead = { 4308 .base = { 4309 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 4310 .cra_driver_name = 4311 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr", 4312 .cra_blocksize = 1, 4313 .cra_priority = CHCR_AEAD_PRIORITY, 4314 .cra_ctxsize = sizeof(struct chcr_context) + 4315 sizeof(struct chcr_aead_ctx) + 4316 sizeof(struct chcr_authenc_ctx), 4317 4318 }, 4319 .ivsize = CTR_RFC3686_IV_SIZE, 4320 .maxauthsize = SHA384_DIGEST_SIZE, 4321 .setkey = chcr_authenc_setkey, 4322 .setauthsize = chcr_authenc_setauthsize, 4323 } 4324 }, 4325 { 4326 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 4327 .is_registered = 0, 4328 .alg.aead = { 4329 .base = { 4330 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 4331 .cra_driver_name = 4332 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr", 4333 .cra_blocksize = 1, 4334 .cra_priority = CHCR_AEAD_PRIORITY, 4335 .cra_ctxsize = sizeof(struct chcr_context) + 4336 sizeof(struct chcr_aead_ctx) + 4337 sizeof(struct chcr_authenc_ctx), 4338 4339 }, 4340 .ivsize = CTR_RFC3686_IV_SIZE, 4341 .maxauthsize = SHA512_DIGEST_SIZE, 4342 .setkey = chcr_authenc_setkey, 4343 .setauthsize = chcr_authenc_setauthsize, 4344 } 4345 }, 4346 { 4347 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL, 4348 .is_registered = 0, 4349 .alg.aead = { 4350 .base = { 4351 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))", 4352 .cra_driver_name = 4353 "authenc-digest_null-rfc3686-ctr-aes-chcr", 4354 .cra_blocksize = 1, 4355 .cra_priority = CHCR_AEAD_PRIORITY, 4356 .cra_ctxsize = sizeof(struct chcr_context) + 4357 sizeof(struct chcr_aead_ctx) + 4358 sizeof(struct chcr_authenc_ctx), 4359 4360 }, 4361 .ivsize = CTR_RFC3686_IV_SIZE, 4362 .maxauthsize = 0, 4363 .setkey = chcr_aead_digest_null_setkey, 4364 .setauthsize = chcr_authenc_null_setauthsize, 4365 } 4366 }, 4367 }; 4368 4369 /* 4370 * chcr_unregister_alg - Deregister crypto algorithms with 4371 * kernel framework. 4372 */ 4373 static int chcr_unregister_alg(void) 4374 { 4375 int i; 4376 4377 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4378 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4379 case CRYPTO_ALG_TYPE_SKCIPHER: 4380 if (driver_algs[i].is_registered && refcount_read( 4381 &driver_algs[i].alg.skcipher.base.cra_refcnt) 4382 == 1) { 4383 crypto_unregister_skcipher( 4384 &driver_algs[i].alg.skcipher); 4385 driver_algs[i].is_registered = 0; 4386 } 4387 break; 4388 case CRYPTO_ALG_TYPE_AEAD: 4389 if (driver_algs[i].is_registered && refcount_read( 4390 &driver_algs[i].alg.aead.base.cra_refcnt) == 1) { 4391 crypto_unregister_aead( 4392 &driver_algs[i].alg.aead); 4393 driver_algs[i].is_registered = 0; 4394 } 4395 break; 4396 case CRYPTO_ALG_TYPE_AHASH: 4397 if (driver_algs[i].is_registered && refcount_read( 4398 &driver_algs[i].alg.hash.halg.base.cra_refcnt) 4399 == 1) { 4400 crypto_unregister_ahash( 4401 &driver_algs[i].alg.hash); 4402 driver_algs[i].is_registered = 0; 4403 } 4404 break; 4405 } 4406 } 4407 return 0; 4408 } 4409 4410 #define SZ_AHASH_CTX sizeof(struct chcr_context) 4411 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 4412 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 4413 4414 /* 4415 * chcr_register_alg - Register crypto algorithms with kernel framework. 4416 */ 4417 static int chcr_register_alg(void) 4418 { 4419 struct crypto_alg ai; 4420 struct ahash_alg *a_hash; 4421 int err = 0, i; 4422 char *name = NULL; 4423 4424 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4425 if (driver_algs[i].is_registered) 4426 continue; 4427 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4428 case CRYPTO_ALG_TYPE_SKCIPHER: 4429 driver_algs[i].alg.skcipher.base.cra_priority = 4430 CHCR_CRA_PRIORITY; 4431 driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE; 4432 driver_algs[i].alg.skcipher.base.cra_flags = 4433 CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC | 4434 CRYPTO_ALG_ALLOCATES_MEMORY | 4435 CRYPTO_ALG_NEED_FALLBACK; 4436 driver_algs[i].alg.skcipher.base.cra_ctxsize = 4437 sizeof(struct chcr_context) + 4438 sizeof(struct ablk_ctx); 4439 driver_algs[i].alg.skcipher.base.cra_alignmask = 0; 4440 4441 err = crypto_register_skcipher(&driver_algs[i].alg.skcipher); 4442 name = driver_algs[i].alg.skcipher.base.cra_driver_name; 4443 break; 4444 case CRYPTO_ALG_TYPE_AEAD: 4445 driver_algs[i].alg.aead.base.cra_flags = 4446 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK | 4447 CRYPTO_ALG_ALLOCATES_MEMORY; 4448 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 4449 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 4450 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 4451 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 4452 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 4453 err = crypto_register_aead(&driver_algs[i].alg.aead); 4454 name = driver_algs[i].alg.aead.base.cra_driver_name; 4455 break; 4456 case CRYPTO_ALG_TYPE_AHASH: 4457 a_hash = &driver_algs[i].alg.hash; 4458 a_hash->update = chcr_ahash_update; 4459 a_hash->final = chcr_ahash_final; 4460 a_hash->finup = chcr_ahash_finup; 4461 a_hash->digest = chcr_ahash_digest; 4462 a_hash->export = chcr_ahash_export; 4463 a_hash->import = chcr_ahash_import; 4464 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 4465 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 4466 a_hash->halg.base.cra_module = THIS_MODULE; 4467 a_hash->halg.base.cra_flags = 4468 CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4469 a_hash->halg.base.cra_alignmask = 0; 4470 a_hash->halg.base.cra_exit = NULL; 4471 4472 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 4473 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 4474 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 4475 a_hash->init = chcr_hmac_init; 4476 a_hash->setkey = chcr_ahash_setkey; 4477 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 4478 } else { 4479 a_hash->init = chcr_sha_init; 4480 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 4481 a_hash->halg.base.cra_init = chcr_sha_cra_init; 4482 } 4483 err = crypto_register_ahash(&driver_algs[i].alg.hash); 4484 ai = driver_algs[i].alg.hash.halg.base; 4485 name = ai.cra_driver_name; 4486 break; 4487 } 4488 if (err) { 4489 pr_err("%s : Algorithm registration failed\n", name); 4490 goto register_err; 4491 } else { 4492 driver_algs[i].is_registered = 1; 4493 } 4494 } 4495 return 0; 4496 4497 register_err: 4498 chcr_unregister_alg(); 4499 return err; 4500 } 4501 4502 /* 4503 * start_crypto - Register the crypto algorithms. 4504 * This should called once when the first device comesup. After this 4505 * kernel will start calling driver APIs for crypto operations. 4506 */ 4507 int start_crypto(void) 4508 { 4509 return chcr_register_alg(); 4510 } 4511 4512 /* 4513 * stop_crypto - Deregister all the crypto algorithms with kernel. 4514 * This should be called once when the last device goes down. After this 4515 * kernel will not call the driver API for crypto operations. 4516 */ 4517 int stop_crypto(void) 4518 { 4519 chcr_unregister_alg(); 4520 return 0; 4521 } 4522