1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/cryptohash.h> 48 #include <linux/skbuff.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/highmem.h> 51 #include <linux/scatterlist.h> 52 53 #include <crypto/aes.h> 54 #include <crypto/algapi.h> 55 #include <crypto/hash.h> 56 #include <crypto/gcm.h> 57 #include <crypto/sha.h> 58 #include <crypto/authenc.h> 59 #include <crypto/ctr.h> 60 #include <crypto/gf128mul.h> 61 #include <crypto/internal/aead.h> 62 #include <crypto/null.h> 63 #include <crypto/internal/skcipher.h> 64 #include <crypto/aead.h> 65 #include <crypto/scatterwalk.h> 66 #include <crypto/internal/hash.h> 67 68 #include "t4fw_api.h" 69 #include "t4_msg.h" 70 #include "chcr_core.h" 71 #include "chcr_algo.h" 72 #include "chcr_crypto.h" 73 74 #define IV AES_BLOCK_SIZE 75 76 static unsigned int sgl_ent_len[] = { 77 0, 0, 16, 24, 40, 48, 64, 72, 88, 78 96, 112, 120, 136, 144, 160, 168, 184, 79 192, 208, 216, 232, 240, 256, 264, 280, 80 288, 304, 312, 328, 336, 352, 360, 376 81 }; 82 83 static unsigned int dsgl_ent_len[] = { 84 0, 32, 32, 48, 48, 64, 64, 80, 80, 85 112, 112, 128, 128, 144, 144, 160, 160, 86 192, 192, 208, 208, 224, 224, 240, 240, 87 272, 272, 288, 288, 304, 304, 320, 320 88 }; 89 90 static u32 round_constant[11] = { 91 0x01000000, 0x02000000, 0x04000000, 0x08000000, 92 0x10000000, 0x20000000, 0x40000000, 0x80000000, 93 0x1B000000, 0x36000000, 0x6C000000 94 }; 95 96 static int chcr_handle_cipher_resp(struct ablkcipher_request *req, 97 unsigned char *input, int err); 98 99 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 100 { 101 return ctx->crypto_ctx->aeadctx; 102 } 103 104 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 105 { 106 return ctx->crypto_ctx->ablkctx; 107 } 108 109 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 110 { 111 return ctx->crypto_ctx->hmacctx; 112 } 113 114 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 115 { 116 return gctx->ctx->gcm; 117 } 118 119 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 120 { 121 return gctx->ctx->authenc; 122 } 123 124 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 125 { 126 return ctx->dev->u_ctx; 127 } 128 129 static inline int is_ofld_imm(const struct sk_buff *skb) 130 { 131 return (skb->len <= SGE_MAX_WR_LEN); 132 } 133 134 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, 135 unsigned int entlen, 136 unsigned int skip) 137 { 138 int nents = 0; 139 unsigned int less; 140 unsigned int skip_len = 0; 141 142 while (sg && skip) { 143 if (sg_dma_len(sg) <= skip) { 144 skip -= sg_dma_len(sg); 145 skip_len = 0; 146 sg = sg_next(sg); 147 } else { 148 skip_len = skip; 149 skip = 0; 150 } 151 } 152 153 while (sg && reqlen) { 154 less = min(reqlen, sg_dma_len(sg) - skip_len); 155 nents += DIV_ROUND_UP(less, entlen); 156 reqlen -= less; 157 skip_len = 0; 158 sg = sg_next(sg); 159 } 160 return nents; 161 } 162 163 static inline void chcr_handle_ahash_resp(struct ahash_request *req, 164 unsigned char *input, 165 int err) 166 { 167 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 168 int digestsize, updated_digestsize; 169 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 170 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 171 172 if (input == NULL) 173 goto out; 174 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); 175 if (reqctx->is_sg_map) 176 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 177 if (reqctx->dma_addr) 178 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr, 179 reqctx->dma_len, DMA_TO_DEVICE); 180 reqctx->dma_addr = 0; 181 updated_digestsize = digestsize; 182 if (digestsize == SHA224_DIGEST_SIZE) 183 updated_digestsize = SHA256_DIGEST_SIZE; 184 else if (digestsize == SHA384_DIGEST_SIZE) 185 updated_digestsize = SHA512_DIGEST_SIZE; 186 if (reqctx->result == 1) { 187 reqctx->result = 0; 188 memcpy(req->result, input + sizeof(struct cpl_fw6_pld), 189 digestsize); 190 } else { 191 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), 192 updated_digestsize); 193 } 194 out: 195 req->base.complete(&req->base, err); 196 } 197 198 static inline int get_aead_subtype(struct crypto_aead *aead) 199 { 200 struct aead_alg *alg = crypto_aead_alg(aead); 201 struct chcr_alg_template *chcr_crypto_alg = 202 container_of(alg, struct chcr_alg_template, alg.aead); 203 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 204 } 205 206 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 207 { 208 u8 temp[SHA512_DIGEST_SIZE]; 209 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 210 int authsize = crypto_aead_authsize(tfm); 211 struct cpl_fw6_pld *fw6_pld; 212 int cmp = 0; 213 214 fw6_pld = (struct cpl_fw6_pld *)input; 215 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 216 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 217 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); 218 } else { 219 220 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 221 authsize, req->assoclen + 222 req->cryptlen - authsize); 223 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); 224 } 225 if (cmp) 226 *err = -EBADMSG; 227 else 228 *err = 0; 229 } 230 231 static inline void chcr_handle_aead_resp(struct aead_request *req, 232 unsigned char *input, 233 int err) 234 { 235 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 236 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 237 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); 238 239 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); 240 if (reqctx->b0_dma) 241 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, 242 reqctx->b0_len, DMA_BIDIRECTIONAL); 243 if (reqctx->verify == VERIFY_SW) { 244 chcr_verify_tag(req, input, &err); 245 reqctx->verify = VERIFY_HW; 246 } 247 req->base.complete(&req->base, err); 248 } 249 250 /* 251 * chcr_handle_resp - Unmap the DMA buffers associated with the request 252 * @req: crypto request 253 */ 254 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 255 int err) 256 { 257 struct crypto_tfm *tfm = req->tfm; 258 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 259 struct adapter *adap = padap(ctx->dev); 260 261 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 262 case CRYPTO_ALG_TYPE_AEAD: 263 chcr_handle_aead_resp(aead_request_cast(req), input, err); 264 break; 265 266 case CRYPTO_ALG_TYPE_ABLKCIPHER: 267 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req), 268 input, err); 269 break; 270 271 case CRYPTO_ALG_TYPE_AHASH: 272 chcr_handle_ahash_resp(ahash_request_cast(req), input, err); 273 } 274 atomic_inc(&adap->chcr_stats.complete); 275 return err; 276 } 277 278 static void get_aes_decrypt_key(unsigned char *dec_key, 279 const unsigned char *key, 280 unsigned int keylength) 281 { 282 u32 temp; 283 u32 w_ring[MAX_NK]; 284 int i, j, k; 285 u8 nr, nk; 286 287 switch (keylength) { 288 case AES_KEYLENGTH_128BIT: 289 nk = KEYLENGTH_4BYTES; 290 nr = NUMBER_OF_ROUNDS_10; 291 break; 292 case AES_KEYLENGTH_192BIT: 293 nk = KEYLENGTH_6BYTES; 294 nr = NUMBER_OF_ROUNDS_12; 295 break; 296 case AES_KEYLENGTH_256BIT: 297 nk = KEYLENGTH_8BYTES; 298 nr = NUMBER_OF_ROUNDS_14; 299 break; 300 default: 301 return; 302 } 303 for (i = 0; i < nk; i++) 304 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]); 305 306 i = 0; 307 temp = w_ring[nk - 1]; 308 while (i + nk < (nr + 1) * 4) { 309 if (!(i % nk)) { 310 /* RotWord(temp) */ 311 temp = (temp << 8) | (temp >> 24); 312 temp = aes_ks_subword(temp); 313 temp ^= round_constant[i / nk]; 314 } else if (nk == 8 && (i % 4 == 0)) { 315 temp = aes_ks_subword(temp); 316 } 317 w_ring[i % nk] ^= temp; 318 temp = w_ring[i % nk]; 319 i++; 320 } 321 i--; 322 for (k = 0, j = i % nk; k < nk; k++) { 323 *((u32 *)dec_key + k) = htonl(w_ring[j]); 324 j--; 325 if (j < 0) 326 j += nk; 327 } 328 } 329 330 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 331 { 332 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 333 334 switch (ds) { 335 case SHA1_DIGEST_SIZE: 336 base_hash = crypto_alloc_shash("sha1", 0, 0); 337 break; 338 case SHA224_DIGEST_SIZE: 339 base_hash = crypto_alloc_shash("sha224", 0, 0); 340 break; 341 case SHA256_DIGEST_SIZE: 342 base_hash = crypto_alloc_shash("sha256", 0, 0); 343 break; 344 case SHA384_DIGEST_SIZE: 345 base_hash = crypto_alloc_shash("sha384", 0, 0); 346 break; 347 case SHA512_DIGEST_SIZE: 348 base_hash = crypto_alloc_shash("sha512", 0, 0); 349 break; 350 } 351 352 return base_hash; 353 } 354 355 static int chcr_compute_partial_hash(struct shash_desc *desc, 356 char *iopad, char *result_hash, 357 int digest_size) 358 { 359 struct sha1_state sha1_st; 360 struct sha256_state sha256_st; 361 struct sha512_state sha512_st; 362 int error; 363 364 if (digest_size == SHA1_DIGEST_SIZE) { 365 error = crypto_shash_init(desc) ?: 366 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 367 crypto_shash_export(desc, (void *)&sha1_st); 368 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 369 } else if (digest_size == SHA224_DIGEST_SIZE) { 370 error = crypto_shash_init(desc) ?: 371 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 372 crypto_shash_export(desc, (void *)&sha256_st); 373 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 374 375 } else if (digest_size == SHA256_DIGEST_SIZE) { 376 error = crypto_shash_init(desc) ?: 377 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 378 crypto_shash_export(desc, (void *)&sha256_st); 379 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 380 381 } else if (digest_size == SHA384_DIGEST_SIZE) { 382 error = crypto_shash_init(desc) ?: 383 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 384 crypto_shash_export(desc, (void *)&sha512_st); 385 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 386 387 } else if (digest_size == SHA512_DIGEST_SIZE) { 388 error = crypto_shash_init(desc) ?: 389 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 390 crypto_shash_export(desc, (void *)&sha512_st); 391 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 392 } else { 393 error = -EINVAL; 394 pr_err("Unknown digest size %d\n", digest_size); 395 } 396 return error; 397 } 398 399 static void chcr_change_order(char *buf, int ds) 400 { 401 int i; 402 403 if (ds == SHA512_DIGEST_SIZE) { 404 for (i = 0; i < (ds / sizeof(u64)); i++) 405 *((__be64 *)buf + i) = 406 cpu_to_be64(*((u64 *)buf + i)); 407 } else { 408 for (i = 0; i < (ds / sizeof(u32)); i++) 409 *((__be32 *)buf + i) = 410 cpu_to_be32(*((u32 *)buf + i)); 411 } 412 } 413 414 static inline int is_hmac(struct crypto_tfm *tfm) 415 { 416 struct crypto_alg *alg = tfm->__crt_alg; 417 struct chcr_alg_template *chcr_crypto_alg = 418 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 419 alg.hash); 420 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 421 return 1; 422 return 0; 423 } 424 425 static inline void dsgl_walk_init(struct dsgl_walk *walk, 426 struct cpl_rx_phys_dsgl *dsgl) 427 { 428 walk->dsgl = dsgl; 429 walk->nents = 0; 430 walk->to = (struct phys_sge_pairs *)(dsgl + 1); 431 } 432 433 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid) 434 { 435 struct cpl_rx_phys_dsgl *phys_cpl; 436 437 phys_cpl = walk->dsgl; 438 439 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 440 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 441 phys_cpl->pcirlxorder_to_noofsgentr = 442 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 443 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 444 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 445 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 446 CPL_RX_PHYS_DSGL_DCAID_V(0) | 447 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); 448 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 449 phys_cpl->rss_hdr_int.qid = htons(qid); 450 phys_cpl->rss_hdr_int.hash_val = 0; 451 } 452 453 static inline void dsgl_walk_add_page(struct dsgl_walk *walk, 454 size_t size, 455 dma_addr_t *addr) 456 { 457 int j; 458 459 if (!size) 460 return; 461 j = walk->nents; 462 walk->to->len[j % 8] = htons(size); 463 walk->to->addr[j % 8] = cpu_to_be64(*addr); 464 j++; 465 if ((j % 8) == 0) 466 walk->to++; 467 walk->nents = j; 468 } 469 470 static void dsgl_walk_add_sg(struct dsgl_walk *walk, 471 struct scatterlist *sg, 472 unsigned int slen, 473 unsigned int skip) 474 { 475 int skip_len = 0; 476 unsigned int left_size = slen, len = 0; 477 unsigned int j = walk->nents; 478 int offset, ent_len; 479 480 if (!slen) 481 return; 482 while (sg && skip) { 483 if (sg_dma_len(sg) <= skip) { 484 skip -= sg_dma_len(sg); 485 skip_len = 0; 486 sg = sg_next(sg); 487 } else { 488 skip_len = skip; 489 skip = 0; 490 } 491 } 492 493 while (left_size && sg) { 494 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); 495 offset = 0; 496 while (len) { 497 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); 498 walk->to->len[j % 8] = htons(ent_len); 499 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + 500 offset + skip_len); 501 offset += ent_len; 502 len -= ent_len; 503 j++; 504 if ((j % 8) == 0) 505 walk->to++; 506 } 507 walk->last_sg = sg; 508 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - 509 skip_len) + skip_len; 510 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); 511 skip_len = 0; 512 sg = sg_next(sg); 513 } 514 walk->nents = j; 515 } 516 517 static inline void ulptx_walk_init(struct ulptx_walk *walk, 518 struct ulptx_sgl *ulp) 519 { 520 walk->sgl = ulp; 521 walk->nents = 0; 522 walk->pair_idx = 0; 523 walk->pair = ulp->sge; 524 walk->last_sg = NULL; 525 walk->last_sg_len = 0; 526 } 527 528 static inline void ulptx_walk_end(struct ulptx_walk *walk) 529 { 530 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | 531 ULPTX_NSGE_V(walk->nents)); 532 } 533 534 535 static inline void ulptx_walk_add_page(struct ulptx_walk *walk, 536 size_t size, 537 dma_addr_t *addr) 538 { 539 if (!size) 540 return; 541 542 if (walk->nents == 0) { 543 walk->sgl->len0 = cpu_to_be32(size); 544 walk->sgl->addr0 = cpu_to_be64(*addr); 545 } else { 546 walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr); 547 walk->pair->len[walk->pair_idx] = cpu_to_be32(size); 548 walk->pair_idx = !walk->pair_idx; 549 if (!walk->pair_idx) 550 walk->pair++; 551 } 552 walk->nents++; 553 } 554 555 static void ulptx_walk_add_sg(struct ulptx_walk *walk, 556 struct scatterlist *sg, 557 unsigned int len, 558 unsigned int skip) 559 { 560 int small; 561 int skip_len = 0; 562 unsigned int sgmin; 563 564 if (!len) 565 return; 566 567 while (sg && skip) { 568 if (sg_dma_len(sg) <= skip) { 569 skip -= sg_dma_len(sg); 570 skip_len = 0; 571 sg = sg_next(sg); 572 } else { 573 skip_len = skip; 574 skip = 0; 575 } 576 } 577 WARN(!sg, "SG should not be null here\n"); 578 if (sg && (walk->nents == 0)) { 579 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); 580 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 581 walk->sgl->len0 = cpu_to_be32(sgmin); 582 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); 583 walk->nents++; 584 len -= sgmin; 585 walk->last_sg = sg; 586 walk->last_sg_len = sgmin + skip_len; 587 skip_len += sgmin; 588 if (sg_dma_len(sg) == skip_len) { 589 sg = sg_next(sg); 590 skip_len = 0; 591 } 592 } 593 594 while (sg && len) { 595 small = min(sg_dma_len(sg) - skip_len, len); 596 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); 597 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); 598 walk->pair->addr[walk->pair_idx] = 599 cpu_to_be64(sg_dma_address(sg) + skip_len); 600 walk->pair_idx = !walk->pair_idx; 601 walk->nents++; 602 if (!walk->pair_idx) 603 walk->pair++; 604 len -= sgmin; 605 skip_len += sgmin; 606 walk->last_sg = sg; 607 walk->last_sg_len = skip_len; 608 if (sg_dma_len(sg) == skip_len) { 609 sg = sg_next(sg); 610 skip_len = 0; 611 } 612 } 613 } 614 615 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) 616 { 617 struct crypto_alg *alg = tfm->__crt_alg; 618 struct chcr_alg_template *chcr_crypto_alg = 619 container_of(alg, struct chcr_alg_template, alg.crypto); 620 621 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 622 } 623 624 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 625 { 626 struct adapter *adap = netdev2adap(dev); 627 struct sge_uld_txq_info *txq_info = 628 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 629 struct sge_uld_txq *txq; 630 int ret = 0; 631 632 local_bh_disable(); 633 txq = &txq_info->uldtxq[idx]; 634 spin_lock(&txq->sendq.lock); 635 if (txq->full) 636 ret = -1; 637 spin_unlock(&txq->sendq.lock); 638 local_bh_enable(); 639 return ret; 640 } 641 642 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 643 struct _key_ctx *key_ctx) 644 { 645 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 646 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 647 } else { 648 memcpy(key_ctx->key, 649 ablkctx->key + (ablkctx->enckey_len >> 1), 650 ablkctx->enckey_len >> 1); 651 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 652 ablkctx->rrkey, ablkctx->enckey_len >> 1); 653 } 654 return 0; 655 } 656 static int chcr_sg_ent_in_wr(struct scatterlist *src, 657 struct scatterlist *dst, 658 unsigned int minsg, 659 unsigned int space, 660 unsigned int srcskip, 661 unsigned int dstskip) 662 { 663 int srclen = 0, dstlen = 0; 664 int srcsg = minsg, dstsg = minsg; 665 int offset = 0, less; 666 667 if (sg_dma_len(src) == srcskip) { 668 src = sg_next(src); 669 srcskip = 0; 670 } 671 672 if (sg_dma_len(dst) == dstskip) { 673 dst = sg_next(dst); 674 dstskip = 0; 675 } 676 677 while (src && dst && 678 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { 679 srclen += (sg_dma_len(src) - srcskip); 680 srcsg++; 681 offset = 0; 682 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && 683 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { 684 if (srclen <= dstlen) 685 break; 686 less = min_t(unsigned int, sg_dma_len(dst) - offset - 687 dstskip, CHCR_DST_SG_SIZE); 688 dstlen += less; 689 offset += less; 690 if (offset == sg_dma_len(dst)) { 691 dst = sg_next(dst); 692 offset = 0; 693 } 694 dstsg++; 695 dstskip = 0; 696 } 697 src = sg_next(src); 698 srcskip = 0; 699 } 700 return min(srclen, dstlen); 701 } 702 703 static int chcr_cipher_fallback(struct crypto_skcipher *cipher, 704 u32 flags, 705 struct scatterlist *src, 706 struct scatterlist *dst, 707 unsigned int nbytes, 708 u8 *iv, 709 unsigned short op_type) 710 { 711 int err; 712 713 SKCIPHER_REQUEST_ON_STACK(subreq, cipher); 714 skcipher_request_set_tfm(subreq, cipher); 715 skcipher_request_set_callback(subreq, flags, NULL, NULL); 716 skcipher_request_set_crypt(subreq, src, dst, 717 nbytes, iv); 718 719 err = op_type ? crypto_skcipher_decrypt(subreq) : 720 crypto_skcipher_encrypt(subreq); 721 skcipher_request_zero(subreq); 722 723 return err; 724 725 } 726 static inline void create_wreq(struct chcr_context *ctx, 727 struct chcr_wr *chcr_req, 728 struct crypto_async_request *req, 729 unsigned int imm, 730 int hash_sz, 731 unsigned int len16, 732 unsigned int sc_len, 733 unsigned int lcb) 734 { 735 struct uld_ctx *u_ctx = ULD_CTX(ctx); 736 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; 737 738 739 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; 740 chcr_req->wreq.pld_size_hash_size = 741 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 742 chcr_req->wreq.len16_pkd = 743 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); 744 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 745 chcr_req->wreq.rx_chid_to_rx_q_id = 746 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, 747 !!lcb, ctx->tx_qidx); 748 749 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, 750 qid); 751 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - 752 ((sizeof(chcr_req->wreq)) >> 4))); 753 754 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); 755 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 756 sizeof(chcr_req->key_ctx) + sc_len); 757 } 758 759 /** 760 * create_cipher_wr - form the WR for cipher operations 761 * @req: cipher req. 762 * @ctx: crypto driver context of the request. 763 * @qid: ingress qid where response of this WR should be received. 764 * @op_type: encryption or decryption 765 */ 766 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) 767 { 768 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); 769 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 770 struct sk_buff *skb = NULL; 771 struct chcr_wr *chcr_req; 772 struct cpl_rx_phys_dsgl *phys_cpl; 773 struct ulptx_sgl *ulptx; 774 struct chcr_blkcipher_req_ctx *reqctx = 775 ablkcipher_request_ctx(wrparam->req); 776 unsigned int temp = 0, transhdr_len, dst_size; 777 int error; 778 int nents; 779 unsigned int kctx_len; 780 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 781 GFP_KERNEL : GFP_ATOMIC; 782 struct adapter *adap = padap(c_ctx(tfm)->dev); 783 784 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, 785 reqctx->dst_ofst); 786 dst_size = get_space_for_phys_dsgl(nents + 1); 787 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); 788 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 789 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, 790 CHCR_SRC_SG_SIZE, reqctx->src_ofst); 791 temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16) 792 * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8); 793 transhdr_len += temp; 794 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 795 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 796 if (!skb) { 797 error = -ENOMEM; 798 goto err; 799 } 800 chcr_req = __skb_put_zero(skb, transhdr_len); 801 chcr_req->sec_cpl.op_ivinsrtofst = 802 FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1); 803 804 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); 805 chcr_req->sec_cpl.aadstart_cipherstop_hi = 806 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); 807 808 chcr_req->sec_cpl.cipherstop_lo_authinsert = 809 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 810 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, 811 ablkctx->ciph_mode, 812 0, 0, IV >> 1); 813 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 814 0, 0, dst_size); 815 816 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 817 if ((reqctx->op == CHCR_DECRYPT_OP) && 818 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 819 CRYPTO_ALG_SUB_TYPE_CTR)) && 820 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 821 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { 822 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 823 } else { 824 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || 825 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { 826 memcpy(chcr_req->key_ctx.key, ablkctx->key, 827 ablkctx->enckey_len); 828 } else { 829 memcpy(chcr_req->key_ctx.key, ablkctx->key + 830 (ablkctx->enckey_len >> 1), 831 ablkctx->enckey_len >> 1); 832 memcpy(chcr_req->key_ctx.key + 833 (ablkctx->enckey_len >> 1), 834 ablkctx->key, 835 ablkctx->enckey_len >> 1); 836 } 837 } 838 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 839 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 840 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); 841 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); 842 843 atomic_inc(&adap->chcr_stats.cipher_rqst); 844 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len 845 +(reqctx->imm ? (IV + wrparam->bytes) : 0); 846 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, 847 transhdr_len, temp, 848 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); 849 reqctx->skb = skb; 850 return skb; 851 err: 852 return ERR_PTR(error); 853 } 854 855 static inline int chcr_keyctx_ck_size(unsigned int keylen) 856 { 857 int ck_size = 0; 858 859 if (keylen == AES_KEYSIZE_128) 860 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 861 else if (keylen == AES_KEYSIZE_192) 862 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 863 else if (keylen == AES_KEYSIZE_256) 864 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 865 else 866 ck_size = 0; 867 868 return ck_size; 869 } 870 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, 871 const u8 *key, 872 unsigned int keylen) 873 { 874 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 875 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 876 int err = 0; 877 878 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 879 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags & 880 CRYPTO_TFM_REQ_MASK); 881 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen); 882 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; 883 tfm->crt_flags |= 884 crypto_skcipher_get_flags(ablkctx->sw_cipher) & 885 CRYPTO_TFM_RES_MASK; 886 return err; 887 } 888 889 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, 890 const u8 *key, 891 unsigned int keylen) 892 { 893 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 894 unsigned int ck_size, context_size; 895 u16 alignment = 0; 896 int err; 897 898 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 899 if (err) 900 goto badkey_err; 901 902 ck_size = chcr_keyctx_ck_size(keylen); 903 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; 904 memcpy(ablkctx->key, key, keylen); 905 ablkctx->enckey_len = keylen; 906 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 907 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 908 keylen + alignment) >> 4; 909 910 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 911 0, 0, context_size); 912 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 913 return 0; 914 badkey_err: 915 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 916 ablkctx->enckey_len = 0; 917 918 return err; 919 } 920 921 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, 922 const u8 *key, 923 unsigned int keylen) 924 { 925 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 926 unsigned int ck_size, context_size; 927 u16 alignment = 0; 928 int err; 929 930 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 931 if (err) 932 goto badkey_err; 933 ck_size = chcr_keyctx_ck_size(keylen); 934 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 935 memcpy(ablkctx->key, key, keylen); 936 ablkctx->enckey_len = keylen; 937 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 938 keylen + alignment) >> 4; 939 940 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 941 0, 0, context_size); 942 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 943 944 return 0; 945 badkey_err: 946 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 947 ablkctx->enckey_len = 0; 948 949 return err; 950 } 951 952 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, 953 const u8 *key, 954 unsigned int keylen) 955 { 956 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 957 unsigned int ck_size, context_size; 958 u16 alignment = 0; 959 int err; 960 961 if (keylen < CTR_RFC3686_NONCE_SIZE) 962 return -EINVAL; 963 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), 964 CTR_RFC3686_NONCE_SIZE); 965 966 keylen -= CTR_RFC3686_NONCE_SIZE; 967 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 968 if (err) 969 goto badkey_err; 970 971 ck_size = chcr_keyctx_ck_size(keylen); 972 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 973 memcpy(ablkctx->key, key, keylen); 974 ablkctx->enckey_len = keylen; 975 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 976 keylen + alignment) >> 4; 977 978 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 979 0, 0, context_size); 980 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 981 982 return 0; 983 badkey_err: 984 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 985 ablkctx->enckey_len = 0; 986 987 return err; 988 } 989 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) 990 { 991 unsigned int size = AES_BLOCK_SIZE; 992 __be32 *b = (__be32 *)(dstiv + size); 993 u32 c, prev; 994 995 memcpy(dstiv, srciv, AES_BLOCK_SIZE); 996 for (; size >= 4; size -= 4) { 997 prev = be32_to_cpu(*--b); 998 c = prev + add; 999 *b = cpu_to_be32(c); 1000 if (prev < c) 1001 break; 1002 add = 1; 1003 } 1004 1005 } 1006 1007 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) 1008 { 1009 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); 1010 u64 c; 1011 u32 temp = be32_to_cpu(*--b); 1012 1013 temp = ~temp; 1014 c = (u64)temp + 1; // No of block can processed withou overflow 1015 if ((bytes / AES_BLOCK_SIZE) > c) 1016 bytes = c * AES_BLOCK_SIZE; 1017 return bytes; 1018 } 1019 1020 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv, 1021 u32 isfinal) 1022 { 1023 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1024 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1025 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1026 struct crypto_cipher *cipher; 1027 int ret, i; 1028 u8 *key; 1029 unsigned int keylen; 1030 int round = reqctx->last_req_len / AES_BLOCK_SIZE; 1031 int round8 = round / 8; 1032 1033 cipher = ablkctx->aes_generic; 1034 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); 1035 1036 keylen = ablkctx->enckey_len / 2; 1037 key = ablkctx->key + keylen; 1038 ret = crypto_cipher_setkey(cipher, key, keylen); 1039 if (ret) 1040 goto out; 1041 /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/ 1042 for (i = 0; i < round8; i++) 1043 gf128mul_x8_ble((le128 *)iv, (le128 *)iv); 1044 1045 for (i = 0; i < (round % 8); i++) 1046 gf128mul_x_ble((le128 *)iv, (le128 *)iv); 1047 1048 if (!isfinal) 1049 crypto_cipher_decrypt_one(cipher, iv, iv); 1050 out: 1051 return ret; 1052 } 1053 1054 static int chcr_update_cipher_iv(struct ablkcipher_request *req, 1055 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1056 { 1057 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1058 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1059 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 1060 int ret = 0; 1061 1062 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1063 ctr_add_iv(iv, req->info, (reqctx->processed / 1064 AES_BLOCK_SIZE)); 1065 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) 1066 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1067 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / 1068 AES_BLOCK_SIZE) + 1); 1069 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1070 ret = chcr_update_tweak(req, iv, 0); 1071 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1072 if (reqctx->op) 1073 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, 1074 16, 1075 reqctx->processed - AES_BLOCK_SIZE); 1076 else 1077 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1078 } 1079 1080 return ret; 1081 1082 } 1083 1084 /* We need separate function for final iv because in rfc3686 Initial counter 1085 * starts from 1 and buffer size of iv is 8 byte only which remains constant 1086 * for subsequent update requests 1087 */ 1088 1089 static int chcr_final_cipher_iv(struct ablkcipher_request *req, 1090 struct cpl_fw6_pld *fw6_pld, u8 *iv) 1091 { 1092 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1093 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1094 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 1095 int ret = 0; 1096 1097 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 1098 ctr_add_iv(iv, req->info, (reqctx->processed / 1099 AES_BLOCK_SIZE)); 1100 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 1101 ret = chcr_update_tweak(req, iv, 1); 1102 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 1103 if (reqctx->op) 1104 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, 1105 16, 1106 reqctx->processed - AES_BLOCK_SIZE); 1107 else 1108 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 1109 1110 } 1111 return ret; 1112 1113 } 1114 1115 static int chcr_handle_cipher_resp(struct ablkcipher_request *req, 1116 unsigned char *input, int err) 1117 { 1118 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1119 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1120 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1121 struct sk_buff *skb; 1122 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; 1123 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1124 struct cipher_wr_param wrparam; 1125 int bytes; 1126 1127 if (err) 1128 goto unmap; 1129 if (req->nbytes == reqctx->processed) { 1130 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1131 req); 1132 err = chcr_final_cipher_iv(req, fw6_pld, req->info); 1133 goto complete; 1134 } 1135 1136 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1137 c_ctx(tfm)->tx_qidx))) { 1138 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1139 err = -EBUSY; 1140 goto unmap; 1141 } 1142 1143 } 1144 if (!reqctx->imm) { 1145 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1, 1146 SPACE_LEFT(ablkctx->enckey_len), 1147 reqctx->src_ofst, reqctx->dst_ofst); 1148 if ((bytes + reqctx->processed) >= req->nbytes) 1149 bytes = req->nbytes - reqctx->processed; 1150 else 1151 bytes = ROUND_16(bytes); 1152 } else { 1153 /*CTR mode counter overfloa*/ 1154 bytes = req->nbytes - reqctx->processed; 1155 } 1156 dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1157 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 1158 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); 1159 dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1160 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 1161 if (err) 1162 goto unmap; 1163 1164 if (unlikely(bytes == 0)) { 1165 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1166 req); 1167 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1168 req->base.flags, 1169 req->src, 1170 req->dst, 1171 req->nbytes, 1172 req->info, 1173 reqctx->op); 1174 goto complete; 1175 } 1176 1177 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1178 CRYPTO_ALG_SUB_TYPE_CTR) 1179 bytes = adjust_ctr_overflow(reqctx->iv, bytes); 1180 wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx]; 1181 wrparam.req = req; 1182 wrparam.bytes = bytes; 1183 skb = create_cipher_wr(&wrparam); 1184 if (IS_ERR(skb)) { 1185 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 1186 err = PTR_ERR(skb); 1187 goto unmap; 1188 } 1189 skb->dev = u_ctx->lldi.ports[0]; 1190 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1191 chcr_send_wr(skb); 1192 reqctx->last_req_len = bytes; 1193 reqctx->processed += bytes; 1194 return 0; 1195 unmap: 1196 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1197 complete: 1198 req->base.complete(&req->base, err); 1199 return err; 1200 } 1201 1202 static int process_cipher(struct ablkcipher_request *req, 1203 unsigned short qid, 1204 struct sk_buff **skb, 1205 unsigned short op_type) 1206 { 1207 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1208 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 1209 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1210 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); 1211 struct cipher_wr_param wrparam; 1212 int bytes, err = -EINVAL; 1213 1214 reqctx->processed = 0; 1215 if (!req->info) 1216 goto error; 1217 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 1218 (req->nbytes == 0) || 1219 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) { 1220 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 1221 ablkctx->enckey_len, req->nbytes, ivsize); 1222 goto error; 1223 } 1224 chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1225 if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + 1226 AES_MIN_KEY_SIZE + 1227 sizeof(struct cpl_rx_phys_dsgl) + 1228 /*Min dsgl size*/ 1229 32))) { 1230 /* Can be sent as Imm*/ 1231 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; 1232 1233 dnents = sg_nents_xlen(req->dst, req->nbytes, 1234 CHCR_DST_SG_SIZE, 0); 1235 dnents += 1; // IV 1236 phys_dsgl = get_space_for_phys_dsgl(dnents); 1237 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); 1238 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 1239 reqctx->imm = (transhdr_len + IV + req->nbytes) <= 1240 SGE_MAX_WR_LEN; 1241 bytes = IV + req->nbytes; 1242 1243 } else { 1244 reqctx->imm = 0; 1245 } 1246 1247 if (!reqctx->imm) { 1248 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 1249 MIN_CIPHER_SG, 1250 SPACE_LEFT(ablkctx->enckey_len), 1251 0, 0); 1252 if ((bytes + reqctx->processed) >= req->nbytes) 1253 bytes = req->nbytes - reqctx->processed; 1254 else 1255 bytes = ROUND_16(bytes); 1256 } else { 1257 bytes = req->nbytes; 1258 } 1259 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1260 CRYPTO_ALG_SUB_TYPE_CTR) { 1261 bytes = adjust_ctr_overflow(req->info, bytes); 1262 } 1263 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1264 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { 1265 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); 1266 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info, 1267 CTR_RFC3686_IV_SIZE); 1268 1269 /* initialize counter portion of counter block */ 1270 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1271 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1272 1273 } else { 1274 1275 memcpy(reqctx->iv, req->info, IV); 1276 } 1277 if (unlikely(bytes == 0)) { 1278 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, 1279 req); 1280 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1281 req->base.flags, 1282 req->src, 1283 req->dst, 1284 req->nbytes, 1285 req->info, 1286 op_type); 1287 goto error; 1288 } 1289 reqctx->op = op_type; 1290 reqctx->srcsg = req->src; 1291 reqctx->dstsg = req->dst; 1292 reqctx->src_ofst = 0; 1293 reqctx->dst_ofst = 0; 1294 wrparam.qid = qid; 1295 wrparam.req = req; 1296 wrparam.bytes = bytes; 1297 *skb = create_cipher_wr(&wrparam); 1298 if (IS_ERR(*skb)) { 1299 err = PTR_ERR(*skb); 1300 goto unmap; 1301 } 1302 reqctx->processed = bytes; 1303 reqctx->last_req_len = bytes; 1304 1305 return 0; 1306 unmap: 1307 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); 1308 error: 1309 return err; 1310 } 1311 1312 static int chcr_aes_encrypt(struct ablkcipher_request *req) 1313 { 1314 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1315 struct sk_buff *skb = NULL; 1316 int err; 1317 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1318 1319 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1320 c_ctx(tfm)->tx_qidx))) { 1321 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1322 return -EBUSY; 1323 } 1324 1325 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], 1326 &skb, CHCR_ENCRYPT_OP); 1327 if (err || !skb) 1328 return err; 1329 skb->dev = u_ctx->lldi.ports[0]; 1330 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1331 chcr_send_wr(skb); 1332 return -EINPROGRESS; 1333 } 1334 1335 static int chcr_aes_decrypt(struct ablkcipher_request *req) 1336 { 1337 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1338 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); 1339 struct sk_buff *skb = NULL; 1340 int err; 1341 1342 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1343 c_ctx(tfm)->tx_qidx))) { 1344 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1345 return -EBUSY; 1346 } 1347 1348 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], 1349 &skb, CHCR_DECRYPT_OP); 1350 if (err || !skb) 1351 return err; 1352 skb->dev = u_ctx->lldi.ports[0]; 1353 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); 1354 chcr_send_wr(skb); 1355 return -EINPROGRESS; 1356 } 1357 1358 static int chcr_device_init(struct chcr_context *ctx) 1359 { 1360 struct uld_ctx *u_ctx = NULL; 1361 struct adapter *adap; 1362 unsigned int id; 1363 int txq_perchan, txq_idx, ntxq; 1364 int err = 0, rxq_perchan, rxq_idx; 1365 1366 id = smp_processor_id(); 1367 if (!ctx->dev) { 1368 u_ctx = assign_chcr_device(); 1369 if (!u_ctx) { 1370 pr_err("chcr device assignment fails\n"); 1371 goto out; 1372 } 1373 ctx->dev = u_ctx->dev; 1374 adap = padap(ctx->dev); 1375 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq, 1376 adap->vres.ncrypto_fc); 1377 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 1378 txq_perchan = ntxq / u_ctx->lldi.nchan; 1379 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan; 1380 rxq_idx += id % rxq_perchan; 1381 txq_idx = ctx->dev->tx_channel_id * txq_perchan; 1382 txq_idx += id % txq_perchan; 1383 spin_lock(&ctx->dev->lock_chcr_dev); 1384 ctx->rx_qidx = rxq_idx; 1385 ctx->tx_qidx = txq_idx; 1386 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; 1387 ctx->dev->rx_channel_id = 0; 1388 spin_unlock(&ctx->dev->lock_chcr_dev); 1389 } 1390 out: 1391 return err; 1392 } 1393 1394 static int chcr_cra_init(struct crypto_tfm *tfm) 1395 { 1396 struct crypto_alg *alg = tfm->__crt_alg; 1397 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1398 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1399 1400 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0, 1401 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 1402 if (IS_ERR(ablkctx->sw_cipher)) { 1403 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1404 return PTR_ERR(ablkctx->sw_cipher); 1405 } 1406 1407 if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) { 1408 /* To update tweak*/ 1409 ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0); 1410 if (IS_ERR(ablkctx->aes_generic)) { 1411 pr_err("failed to allocate aes cipher for tweak\n"); 1412 return PTR_ERR(ablkctx->aes_generic); 1413 } 1414 } else 1415 ablkctx->aes_generic = NULL; 1416 1417 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1418 return chcr_device_init(crypto_tfm_ctx(tfm)); 1419 } 1420 1421 static int chcr_rfc3686_init(struct crypto_tfm *tfm) 1422 { 1423 struct crypto_alg *alg = tfm->__crt_alg; 1424 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1425 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1426 1427 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) 1428 * cannot be used as fallback in chcr_handle_cipher_response 1429 */ 1430 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0, 1431 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 1432 if (IS_ERR(ablkctx->sw_cipher)) { 1433 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1434 return PTR_ERR(ablkctx->sw_cipher); 1435 } 1436 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1437 return chcr_device_init(crypto_tfm_ctx(tfm)); 1438 } 1439 1440 1441 static void chcr_cra_exit(struct crypto_tfm *tfm) 1442 { 1443 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1444 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1445 1446 crypto_free_skcipher(ablkctx->sw_cipher); 1447 if (ablkctx->aes_generic) 1448 crypto_free_cipher(ablkctx->aes_generic); 1449 } 1450 1451 static int get_alg_config(struct algo_param *params, 1452 unsigned int auth_size) 1453 { 1454 switch (auth_size) { 1455 case SHA1_DIGEST_SIZE: 1456 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 1457 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 1458 params->result_size = SHA1_DIGEST_SIZE; 1459 break; 1460 case SHA224_DIGEST_SIZE: 1461 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1462 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 1463 params->result_size = SHA256_DIGEST_SIZE; 1464 break; 1465 case SHA256_DIGEST_SIZE: 1466 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1467 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 1468 params->result_size = SHA256_DIGEST_SIZE; 1469 break; 1470 case SHA384_DIGEST_SIZE: 1471 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1472 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 1473 params->result_size = SHA512_DIGEST_SIZE; 1474 break; 1475 case SHA512_DIGEST_SIZE: 1476 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1477 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 1478 params->result_size = SHA512_DIGEST_SIZE; 1479 break; 1480 default: 1481 pr_err("chcr : ERROR, unsupported digest size\n"); 1482 return -EINVAL; 1483 } 1484 return 0; 1485 } 1486 1487 static inline void chcr_free_shash(struct crypto_shash *base_hash) 1488 { 1489 crypto_free_shash(base_hash); 1490 } 1491 1492 /** 1493 * create_hash_wr - Create hash work request 1494 * @req - Cipher req base 1495 */ 1496 static struct sk_buff *create_hash_wr(struct ahash_request *req, 1497 struct hash_wr_param *param) 1498 { 1499 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1500 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1501 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 1502 struct sk_buff *skb = NULL; 1503 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); 1504 struct chcr_wr *chcr_req; 1505 struct ulptx_sgl *ulptx; 1506 unsigned int nents = 0, transhdr_len, iopad_alignment = 0; 1507 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1508 unsigned int kctx_len = 0, temp = 0; 1509 u8 hash_size_in_response = 0; 1510 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1511 GFP_ATOMIC; 1512 struct adapter *adap = padap(h_ctx(tfm)->dev); 1513 int error = 0; 1514 1515 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); 1516 kctx_len = param->alg_prm.result_size + iopad_alignment; 1517 if (param->opad_needed) 1518 kctx_len += param->alg_prm.result_size + iopad_alignment; 1519 1520 if (req_ctx->result) 1521 hash_size_in_response = digestsize; 1522 else 1523 hash_size_in_response = param->alg_prm.result_size; 1524 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); 1525 req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <= 1526 SGE_MAX_WR_LEN; 1527 nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0); 1528 nents += param->bfr_len ? 1 : 0; 1529 transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len + 1530 param->sg_len), 16) * 16) : 1531 (sgl_len(nents) * 8); 1532 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 1533 1534 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 1535 if (!skb) 1536 return ERR_PTR(-ENOMEM); 1537 chcr_req = __skb_put_zero(skb, transhdr_len); 1538 1539 chcr_req->sec_cpl.op_ivinsrtofst = 1540 FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0); 1541 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 1542 1543 chcr_req->sec_cpl.aadstart_cipherstop_hi = 1544 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 1545 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1546 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 1547 chcr_req->sec_cpl.seqno_numivs = 1548 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 1549 param->opad_needed, 0); 1550 1551 chcr_req->sec_cpl.ivgen_hdrlen = 1552 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 1553 1554 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 1555 param->alg_prm.result_size); 1556 1557 if (param->opad_needed) 1558 memcpy(chcr_req->key_ctx.key + 1559 ((param->alg_prm.result_size <= 32) ? 32 : 1560 CHCR_HASH_MAX_DIGEST_SIZE), 1561 hmacctx->opad, param->alg_prm.result_size); 1562 1563 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 1564 param->alg_prm.mk_size, 0, 1565 param->opad_needed, 1566 ((kctx_len + 1567 sizeof(chcr_req->key_ctx)) >> 4)); 1568 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 1569 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len + 1570 DUMMY_BYTES); 1571 if (param->bfr_len != 0) { 1572 req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev, 1573 req_ctx->reqbfr, param->bfr_len, 1574 DMA_TO_DEVICE); 1575 if (dma_mapping_error(&u_ctx->lldi.pdev->dev, 1576 req_ctx->dma_addr)) { 1577 error = -ENOMEM; 1578 goto err; 1579 } 1580 req_ctx->dma_len = param->bfr_len; 1581 } else { 1582 req_ctx->dma_addr = 0; 1583 } 1584 chcr_add_hash_src_ent(req, ulptx, param); 1585 /* Request upto max wr size */ 1586 temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len 1587 + param->bfr_len) : 0); 1588 atomic_inc(&adap->chcr_stats.digest_rqst); 1589 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm, 1590 hash_size_in_response, transhdr_len, 1591 temp, 0); 1592 req_ctx->skb = skb; 1593 return skb; 1594 err: 1595 kfree_skb(skb); 1596 return ERR_PTR(error); 1597 } 1598 1599 static int chcr_ahash_update(struct ahash_request *req) 1600 { 1601 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1602 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1603 struct uld_ctx *u_ctx = NULL; 1604 struct sk_buff *skb; 1605 u8 remainder = 0, bs; 1606 unsigned int nbytes = req->nbytes; 1607 struct hash_wr_param params; 1608 int error; 1609 1610 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1611 1612 u_ctx = ULD_CTX(h_ctx(rtfm)); 1613 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1614 h_ctx(rtfm)->tx_qidx))) { 1615 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1616 return -EBUSY; 1617 } 1618 1619 if (nbytes + req_ctx->reqlen >= bs) { 1620 remainder = (nbytes + req_ctx->reqlen) % bs; 1621 nbytes = nbytes + req_ctx->reqlen - remainder; 1622 } else { 1623 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 1624 + req_ctx->reqlen, nbytes, 0); 1625 req_ctx->reqlen += nbytes; 1626 return 0; 1627 } 1628 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1629 if (error) 1630 return -ENOMEM; 1631 params.opad_needed = 0; 1632 params.more = 1; 1633 params.last = 0; 1634 params.sg_len = nbytes - req_ctx->reqlen; 1635 params.bfr_len = req_ctx->reqlen; 1636 params.scmd1 = 0; 1637 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1638 req_ctx->result = 0; 1639 req_ctx->data_len += params.sg_len + params.bfr_len; 1640 skb = create_hash_wr(req, ¶ms); 1641 if (IS_ERR(skb)) { 1642 error = PTR_ERR(skb); 1643 goto unmap; 1644 } 1645 1646 if (remainder) { 1647 /* Swap buffers */ 1648 swap(req_ctx->reqbfr, req_ctx->skbfr); 1649 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 1650 req_ctx->reqbfr, remainder, req->nbytes - 1651 remainder); 1652 } 1653 req_ctx->reqlen = remainder; 1654 skb->dev = u_ctx->lldi.ports[0]; 1655 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1656 chcr_send_wr(skb); 1657 1658 return -EINPROGRESS; 1659 unmap: 1660 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1661 return error; 1662 } 1663 1664 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 1665 { 1666 memset(bfr_ptr, 0, bs); 1667 *bfr_ptr = 0x80; 1668 if (bs == 64) 1669 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 1670 else 1671 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 1672 } 1673 1674 static int chcr_ahash_final(struct ahash_request *req) 1675 { 1676 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1677 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1678 struct hash_wr_param params; 1679 struct sk_buff *skb; 1680 struct uld_ctx *u_ctx = NULL; 1681 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1682 1683 u_ctx = ULD_CTX(h_ctx(rtfm)); 1684 if (is_hmac(crypto_ahash_tfm(rtfm))) 1685 params.opad_needed = 1; 1686 else 1687 params.opad_needed = 0; 1688 params.sg_len = 0; 1689 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1690 req_ctx->result = 1; 1691 params.bfr_len = req_ctx->reqlen; 1692 req_ctx->data_len += params.bfr_len + params.sg_len; 1693 if (req_ctx->reqlen == 0) { 1694 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1695 params.last = 0; 1696 params.more = 1; 1697 params.scmd1 = 0; 1698 params.bfr_len = bs; 1699 1700 } else { 1701 params.scmd1 = req_ctx->data_len; 1702 params.last = 1; 1703 params.more = 0; 1704 } 1705 skb = create_hash_wr(req, ¶ms); 1706 if (IS_ERR(skb)) 1707 return PTR_ERR(skb); 1708 1709 skb->dev = u_ctx->lldi.ports[0]; 1710 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1711 chcr_send_wr(skb); 1712 return -EINPROGRESS; 1713 } 1714 1715 static int chcr_ahash_finup(struct ahash_request *req) 1716 { 1717 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1718 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1719 struct uld_ctx *u_ctx = NULL; 1720 struct sk_buff *skb; 1721 struct hash_wr_param params; 1722 u8 bs; 1723 int error; 1724 1725 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1726 u_ctx = ULD_CTX(h_ctx(rtfm)); 1727 1728 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1729 h_ctx(rtfm)->tx_qidx))) { 1730 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1731 return -EBUSY; 1732 } 1733 1734 if (is_hmac(crypto_ahash_tfm(rtfm))) 1735 params.opad_needed = 1; 1736 else 1737 params.opad_needed = 0; 1738 1739 params.sg_len = req->nbytes; 1740 params.bfr_len = req_ctx->reqlen; 1741 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1742 req_ctx->data_len += params.bfr_len + params.sg_len; 1743 req_ctx->result = 1; 1744 if ((req_ctx->reqlen + req->nbytes) == 0) { 1745 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1746 params.last = 0; 1747 params.more = 1; 1748 params.scmd1 = 0; 1749 params.bfr_len = bs; 1750 } else { 1751 params.scmd1 = req_ctx->data_len; 1752 params.last = 1; 1753 params.more = 0; 1754 } 1755 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1756 if (error) 1757 return -ENOMEM; 1758 1759 skb = create_hash_wr(req, ¶ms); 1760 if (IS_ERR(skb)) { 1761 error = PTR_ERR(skb); 1762 goto unmap; 1763 } 1764 skb->dev = u_ctx->lldi.ports[0]; 1765 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1766 chcr_send_wr(skb); 1767 1768 return -EINPROGRESS; 1769 unmap: 1770 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1771 return error; 1772 } 1773 1774 static int chcr_ahash_digest(struct ahash_request *req) 1775 { 1776 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1777 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1778 struct uld_ctx *u_ctx = NULL; 1779 struct sk_buff *skb; 1780 struct hash_wr_param params; 1781 u8 bs; 1782 int error; 1783 1784 rtfm->init(req); 1785 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1786 1787 u_ctx = ULD_CTX(h_ctx(rtfm)); 1788 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1789 h_ctx(rtfm)->tx_qidx))) { 1790 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1791 return -EBUSY; 1792 } 1793 1794 if (is_hmac(crypto_ahash_tfm(rtfm))) 1795 params.opad_needed = 1; 1796 else 1797 params.opad_needed = 0; 1798 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); 1799 if (error) 1800 return -ENOMEM; 1801 1802 params.last = 0; 1803 params.more = 0; 1804 params.sg_len = req->nbytes; 1805 params.bfr_len = 0; 1806 params.scmd1 = 0; 1807 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1808 req_ctx->result = 1; 1809 req_ctx->data_len += params.bfr_len + params.sg_len; 1810 1811 if (req->nbytes == 0) { 1812 create_last_hash_block(req_ctx->reqbfr, bs, 0); 1813 params.more = 1; 1814 params.bfr_len = bs; 1815 } 1816 1817 skb = create_hash_wr(req, ¶ms); 1818 if (IS_ERR(skb)) { 1819 error = PTR_ERR(skb); 1820 goto unmap; 1821 } 1822 skb->dev = u_ctx->lldi.ports[0]; 1823 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); 1824 chcr_send_wr(skb); 1825 return -EINPROGRESS; 1826 unmap: 1827 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); 1828 return error; 1829 } 1830 1831 static int chcr_ahash_export(struct ahash_request *areq, void *out) 1832 { 1833 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1834 struct chcr_ahash_req_ctx *state = out; 1835 1836 state->reqlen = req_ctx->reqlen; 1837 state->data_len = req_ctx->data_len; 1838 state->is_sg_map = 0; 1839 state->result = 0; 1840 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 1841 memcpy(state->partial_hash, req_ctx->partial_hash, 1842 CHCR_HASH_MAX_DIGEST_SIZE); 1843 return 0; 1844 } 1845 1846 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 1847 { 1848 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1849 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 1850 1851 req_ctx->reqlen = state->reqlen; 1852 req_ctx->data_len = state->data_len; 1853 req_ctx->reqbfr = req_ctx->bfr1; 1854 req_ctx->skbfr = req_ctx->bfr2; 1855 req_ctx->is_sg_map = 0; 1856 req_ctx->result = 0; 1857 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 1858 memcpy(req_ctx->partial_hash, state->partial_hash, 1859 CHCR_HASH_MAX_DIGEST_SIZE); 1860 return 0; 1861 } 1862 1863 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 1864 unsigned int keylen) 1865 { 1866 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); 1867 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1868 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1869 unsigned int i, err = 0, updated_digestsize; 1870 1871 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 1872 1873 /* use the key to calculate the ipad and opad. ipad will sent with the 1874 * first request's data. opad will be sent with the final hash result 1875 * ipad in hmacctx->ipad and opad in hmacctx->opad location 1876 */ 1877 shash->tfm = hmacctx->base_hash; 1878 shash->flags = crypto_shash_get_flags(hmacctx->base_hash); 1879 if (keylen > bs) { 1880 err = crypto_shash_digest(shash, key, keylen, 1881 hmacctx->ipad); 1882 if (err) 1883 goto out; 1884 keylen = digestsize; 1885 } else { 1886 memcpy(hmacctx->ipad, key, keylen); 1887 } 1888 memset(hmacctx->ipad + keylen, 0, bs - keylen); 1889 memcpy(hmacctx->opad, hmacctx->ipad, bs); 1890 1891 for (i = 0; i < bs / sizeof(int); i++) { 1892 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 1893 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 1894 } 1895 1896 updated_digestsize = digestsize; 1897 if (digestsize == SHA224_DIGEST_SIZE) 1898 updated_digestsize = SHA256_DIGEST_SIZE; 1899 else if (digestsize == SHA384_DIGEST_SIZE) 1900 updated_digestsize = SHA512_DIGEST_SIZE; 1901 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 1902 hmacctx->ipad, digestsize); 1903 if (err) 1904 goto out; 1905 chcr_change_order(hmacctx->ipad, updated_digestsize); 1906 1907 err = chcr_compute_partial_hash(shash, hmacctx->opad, 1908 hmacctx->opad, digestsize); 1909 if (err) 1910 goto out; 1911 chcr_change_order(hmacctx->opad, updated_digestsize); 1912 out: 1913 return err; 1914 } 1915 1916 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 1917 unsigned int key_len) 1918 { 1919 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); 1920 unsigned short context_size = 0; 1921 int err; 1922 1923 err = chcr_cipher_fallback_setkey(cipher, key, key_len); 1924 if (err) 1925 goto badkey_err; 1926 1927 memcpy(ablkctx->key, key, key_len); 1928 ablkctx->enckey_len = key_len; 1929 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 1930 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 1931 ablkctx->key_ctx_hdr = 1932 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 1933 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 1934 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 1935 CHCR_KEYCTX_NO_KEY, 1, 1936 0, context_size); 1937 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 1938 return 0; 1939 badkey_err: 1940 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 1941 ablkctx->enckey_len = 0; 1942 1943 return err; 1944 } 1945 1946 static int chcr_sha_init(struct ahash_request *areq) 1947 { 1948 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1949 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1950 int digestsize = crypto_ahash_digestsize(tfm); 1951 1952 req_ctx->data_len = 0; 1953 req_ctx->reqlen = 0; 1954 req_ctx->reqbfr = req_ctx->bfr1; 1955 req_ctx->skbfr = req_ctx->bfr2; 1956 req_ctx->skb = NULL; 1957 req_ctx->result = 0; 1958 req_ctx->is_sg_map = 0; 1959 copy_hash_init_values(req_ctx->partial_hash, digestsize); 1960 return 0; 1961 } 1962 1963 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 1964 { 1965 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1966 sizeof(struct chcr_ahash_req_ctx)); 1967 return chcr_device_init(crypto_tfm_ctx(tfm)); 1968 } 1969 1970 static int chcr_hmac_init(struct ahash_request *areq) 1971 { 1972 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1973 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 1974 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); 1975 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 1976 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1977 1978 chcr_sha_init(areq); 1979 req_ctx->data_len = bs; 1980 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1981 if (digestsize == SHA224_DIGEST_SIZE) 1982 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1983 SHA256_DIGEST_SIZE); 1984 else if (digestsize == SHA384_DIGEST_SIZE) 1985 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1986 SHA512_DIGEST_SIZE); 1987 else 1988 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1989 digestsize); 1990 } 1991 return 0; 1992 } 1993 1994 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 1995 { 1996 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1997 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1998 unsigned int digestsize = 1999 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 2000 2001 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2002 sizeof(struct chcr_ahash_req_ctx)); 2003 hmacctx->base_hash = chcr_alloc_shash(digestsize); 2004 if (IS_ERR(hmacctx->base_hash)) 2005 return PTR_ERR(hmacctx->base_hash); 2006 return chcr_device_init(crypto_tfm_ctx(tfm)); 2007 } 2008 2009 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 2010 { 2011 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 2012 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 2013 2014 if (hmacctx->base_hash) { 2015 chcr_free_shash(hmacctx->base_hash); 2016 hmacctx->base_hash = NULL; 2017 } 2018 } 2019 2020 static int chcr_aead_common_init(struct aead_request *req, 2021 unsigned short op_type) 2022 { 2023 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2024 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2025 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2026 int error = -EINVAL; 2027 unsigned int authsize = crypto_aead_authsize(tfm); 2028 2029 /* validate key size */ 2030 if (aeadctx->enckey_len == 0) 2031 goto err; 2032 if (op_type && req->cryptlen < authsize) 2033 goto err; 2034 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2035 op_type); 2036 if (error) { 2037 error = -ENOMEM; 2038 goto err; 2039 } 2040 reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen, 2041 CHCR_SRC_SG_SIZE, 0); 2042 reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen, 2043 CHCR_SRC_SG_SIZE, req->assoclen); 2044 return 0; 2045 err: 2046 return error; 2047 } 2048 2049 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, 2050 int aadmax, int wrlen, 2051 unsigned short op_type) 2052 { 2053 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 2054 2055 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || 2056 dst_nents > MAX_DSGL_ENT || 2057 (req->assoclen > aadmax) || 2058 (wrlen > SGE_MAX_WR_LEN)) 2059 return 1; 2060 return 0; 2061 } 2062 2063 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) 2064 { 2065 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2066 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2067 struct aead_request *subreq = aead_request_ctx(req); 2068 2069 aead_request_set_tfm(subreq, aeadctx->sw_cipher); 2070 aead_request_set_callback(subreq, req->base.flags, 2071 req->base.complete, req->base.data); 2072 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2073 req->iv); 2074 aead_request_set_ad(subreq, req->assoclen); 2075 return op_type ? crypto_aead_decrypt(subreq) : 2076 crypto_aead_encrypt(subreq); 2077 } 2078 2079 static struct sk_buff *create_authenc_wr(struct aead_request *req, 2080 unsigned short qid, 2081 int size, 2082 unsigned short op_type) 2083 { 2084 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2085 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2086 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2087 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2088 struct sk_buff *skb = NULL; 2089 struct chcr_wr *chcr_req; 2090 struct cpl_rx_phys_dsgl *phys_cpl; 2091 struct ulptx_sgl *ulptx; 2092 unsigned int transhdr_len; 2093 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm); 2094 unsigned int kctx_len = 0, dnents; 2095 unsigned int assoclen = req->assoclen; 2096 unsigned int authsize = crypto_aead_authsize(tfm); 2097 int error = -EINVAL; 2098 int null = 0; 2099 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2100 GFP_ATOMIC; 2101 struct adapter *adap = padap(a_ctx(tfm)->dev); 2102 2103 if (req->cryptlen == 0) 2104 return NULL; 2105 2106 reqctx->b0_dma = 0; 2107 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL || 2108 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2109 null = 1; 2110 assoclen = 0; 2111 } 2112 error = chcr_aead_common_init(req, op_type); 2113 if (error) 2114 return ERR_PTR(error); 2115 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); 2116 dnents += sg_nents_xlen(req->dst, req->cryptlen + 2117 (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, 2118 req->assoclen); 2119 dnents += MIN_AUTH_SG; // For IV 2120 2121 dst_size = get_space_for_phys_dsgl(dnents); 2122 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) 2123 - sizeof(chcr_req->key_ctx); 2124 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2125 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) < 2126 SGE_MAX_WR_LEN; 2127 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16) 2128 * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents 2129 + MIN_GCM_SG) * 8); 2130 transhdr_len += temp; 2131 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 2132 2133 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 2134 transhdr_len, op_type)) { 2135 atomic_inc(&adap->chcr_stats.fallback); 2136 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2137 op_type); 2138 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2139 } 2140 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 2141 if (!skb) { 2142 error = -ENOMEM; 2143 goto err; 2144 } 2145 2146 chcr_req = __skb_put_zero(skb, transhdr_len); 2147 2148 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 2149 2150 /* 2151 * Input order is AAD,IV and Payload. where IV should be included as 2152 * the part of authdata. All other fields should be filled according 2153 * to the hardware spec 2154 */ 2155 chcr_req->sec_cpl.op_ivinsrtofst = 2156 FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2, 2157 assoclen + 1); 2158 chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen); 2159 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2160 assoclen ? 1 : 0, assoclen, 2161 assoclen + IV + 1, 2162 (temp & 0x1F0) >> 4); 2163 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 2164 temp & 0xF, 2165 null ? 0 : assoclen + IV + 1, 2166 temp, temp); 2167 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL || 2168 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA) 2169 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR; 2170 else 2171 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC; 2172 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2173 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, 2174 temp, 2175 actx->auth_mode, aeadctx->hmac_ctrl, 2176 IV >> 1); 2177 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2178 0, 0, dst_size); 2179 2180 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2181 if (op_type == CHCR_ENCRYPT_OP || 2182 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2183 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) 2184 memcpy(chcr_req->key_ctx.key, aeadctx->key, 2185 aeadctx->enckey_len); 2186 else 2187 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 2188 aeadctx->enckey_len); 2189 2190 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 2191 4), actx->h_iopad, kctx_len - 2192 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); 2193 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 2194 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 2195 memcpy(reqctx->iv, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE); 2196 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv, 2197 CTR_RFC3686_IV_SIZE); 2198 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 2199 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 2200 } else { 2201 memcpy(reqctx->iv, req->iv, IV); 2202 } 2203 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2204 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 2205 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); 2206 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); 2207 atomic_inc(&adap->chcr_stats.cipher_rqst); 2208 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + 2209 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); 2210 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 2211 transhdr_len, temp, 0); 2212 reqctx->skb = skb; 2213 reqctx->op = op_type; 2214 2215 return skb; 2216 err: 2217 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2218 op_type); 2219 2220 return ERR_PTR(error); 2221 } 2222 2223 int chcr_aead_dma_map(struct device *dev, 2224 struct aead_request *req, 2225 unsigned short op_type) 2226 { 2227 int error; 2228 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2229 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2230 unsigned int authsize = crypto_aead_authsize(tfm); 2231 int dst_size; 2232 2233 dst_size = req->assoclen + req->cryptlen + (op_type ? 2234 -authsize : authsize); 2235 if (!req->cryptlen || !dst_size) 2236 return 0; 2237 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, 2238 DMA_BIDIRECTIONAL); 2239 if (dma_mapping_error(dev, reqctx->iv_dma)) 2240 return -ENOMEM; 2241 2242 if (req->src == req->dst) { 2243 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2244 DMA_BIDIRECTIONAL); 2245 if (!error) 2246 goto err; 2247 } else { 2248 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2249 DMA_TO_DEVICE); 2250 if (!error) 2251 goto err; 2252 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2253 DMA_FROM_DEVICE); 2254 if (!error) { 2255 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2256 DMA_TO_DEVICE); 2257 goto err; 2258 } 2259 } 2260 2261 return 0; 2262 err: 2263 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 2264 return -ENOMEM; 2265 } 2266 2267 void chcr_aead_dma_unmap(struct device *dev, 2268 struct aead_request *req, 2269 unsigned short op_type) 2270 { 2271 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2272 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2273 unsigned int authsize = crypto_aead_authsize(tfm); 2274 int dst_size; 2275 2276 dst_size = req->assoclen + req->cryptlen + (op_type ? 2277 -authsize : authsize); 2278 if (!req->cryptlen || !dst_size) 2279 return; 2280 2281 dma_unmap_single(dev, reqctx->iv_dma, IV, 2282 DMA_BIDIRECTIONAL); 2283 if (req->src == req->dst) { 2284 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2285 DMA_BIDIRECTIONAL); 2286 } else { 2287 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2288 DMA_TO_DEVICE); 2289 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2290 DMA_FROM_DEVICE); 2291 } 2292 } 2293 2294 void chcr_add_aead_src_ent(struct aead_request *req, 2295 struct ulptx_sgl *ulptx, 2296 unsigned int assoclen, 2297 unsigned short op_type) 2298 { 2299 struct ulptx_walk ulp_walk; 2300 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2301 2302 if (reqctx->imm) { 2303 u8 *buf = (u8 *)ulptx; 2304 2305 if (reqctx->b0_dma) { 2306 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); 2307 buf += reqctx->b0_len; 2308 } 2309 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2310 buf, assoclen, 0); 2311 buf += assoclen; 2312 memcpy(buf, reqctx->iv, IV); 2313 buf += IV; 2314 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2315 buf, req->cryptlen, req->assoclen); 2316 } else { 2317 ulptx_walk_init(&ulp_walk, ulptx); 2318 if (reqctx->b0_dma) 2319 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, 2320 &reqctx->b0_dma); 2321 ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0); 2322 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); 2323 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen, 2324 req->assoclen); 2325 ulptx_walk_end(&ulp_walk); 2326 } 2327 } 2328 2329 void chcr_add_aead_dst_ent(struct aead_request *req, 2330 struct cpl_rx_phys_dsgl *phys_cpl, 2331 unsigned int assoclen, 2332 unsigned short op_type, 2333 unsigned short qid) 2334 { 2335 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2336 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2337 struct dsgl_walk dsgl_walk; 2338 unsigned int authsize = crypto_aead_authsize(tfm); 2339 u32 temp; 2340 2341 dsgl_walk_init(&dsgl_walk, phys_cpl); 2342 if (reqctx->b0_dma) 2343 dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma); 2344 dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0); 2345 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); 2346 temp = req->cryptlen + (op_type ? -authsize : authsize); 2347 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen); 2348 dsgl_walk_end(&dsgl_walk, qid); 2349 } 2350 2351 void chcr_add_cipher_src_ent(struct ablkcipher_request *req, 2352 struct ulptx_sgl *ulptx, 2353 struct cipher_wr_param *wrparam) 2354 { 2355 struct ulptx_walk ulp_walk; 2356 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2357 2358 if (reqctx->imm) { 2359 u8 *buf = (u8 *)ulptx; 2360 2361 memcpy(buf, reqctx->iv, IV); 2362 buf += IV; 2363 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2364 buf, wrparam->bytes, reqctx->processed); 2365 } else { 2366 ulptx_walk_init(&ulp_walk, ulptx); 2367 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); 2368 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, 2369 reqctx->src_ofst); 2370 reqctx->srcsg = ulp_walk.last_sg; 2371 reqctx->src_ofst = ulp_walk.last_sg_len; 2372 ulptx_walk_end(&ulp_walk); 2373 } 2374 } 2375 2376 void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, 2377 struct cpl_rx_phys_dsgl *phys_cpl, 2378 struct cipher_wr_param *wrparam, 2379 unsigned short qid) 2380 { 2381 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2382 struct dsgl_walk dsgl_walk; 2383 2384 dsgl_walk_init(&dsgl_walk, phys_cpl); 2385 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); 2386 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, 2387 reqctx->dst_ofst); 2388 reqctx->dstsg = dsgl_walk.last_sg; 2389 reqctx->dst_ofst = dsgl_walk.last_sg_len; 2390 2391 dsgl_walk_end(&dsgl_walk, qid); 2392 } 2393 2394 void chcr_add_hash_src_ent(struct ahash_request *req, 2395 struct ulptx_sgl *ulptx, 2396 struct hash_wr_param *param) 2397 { 2398 struct ulptx_walk ulp_walk; 2399 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); 2400 2401 if (reqctx->imm) { 2402 u8 *buf = (u8 *)ulptx; 2403 2404 if (param->bfr_len) { 2405 memcpy(buf, reqctx->reqbfr, param->bfr_len); 2406 buf += param->bfr_len; 2407 } 2408 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 2409 buf, param->sg_len, 0); 2410 } else { 2411 ulptx_walk_init(&ulp_walk, ulptx); 2412 if (param->bfr_len) 2413 ulptx_walk_add_page(&ulp_walk, param->bfr_len, 2414 &reqctx->dma_addr); 2415 ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len, 2416 0); 2417 ulptx_walk_end(&ulp_walk); 2418 } 2419 } 2420 2421 int chcr_hash_dma_map(struct device *dev, 2422 struct ahash_request *req) 2423 { 2424 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2425 int error = 0; 2426 2427 if (!req->nbytes) 2428 return 0; 2429 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2430 DMA_TO_DEVICE); 2431 if (!error) 2432 return -ENOMEM; 2433 req_ctx->is_sg_map = 1; 2434 return 0; 2435 } 2436 2437 void chcr_hash_dma_unmap(struct device *dev, 2438 struct ahash_request *req) 2439 { 2440 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 2441 2442 if (!req->nbytes) 2443 return; 2444 2445 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2446 DMA_TO_DEVICE); 2447 req_ctx->is_sg_map = 0; 2448 2449 } 2450 2451 int chcr_cipher_dma_map(struct device *dev, 2452 struct ablkcipher_request *req) 2453 { 2454 int error; 2455 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2456 2457 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, 2458 DMA_BIDIRECTIONAL); 2459 if (dma_mapping_error(dev, reqctx->iv_dma)) 2460 return -ENOMEM; 2461 2462 if (req->src == req->dst) { 2463 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2464 DMA_BIDIRECTIONAL); 2465 if (!error) 2466 goto err; 2467 } else { 2468 error = dma_map_sg(dev, req->src, sg_nents(req->src), 2469 DMA_TO_DEVICE); 2470 if (!error) 2471 goto err; 2472 error = dma_map_sg(dev, req->dst, sg_nents(req->dst), 2473 DMA_FROM_DEVICE); 2474 if (!error) { 2475 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2476 DMA_TO_DEVICE); 2477 goto err; 2478 } 2479 } 2480 2481 return 0; 2482 err: 2483 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); 2484 return -ENOMEM; 2485 } 2486 2487 void chcr_cipher_dma_unmap(struct device *dev, 2488 struct ablkcipher_request *req) 2489 { 2490 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 2491 2492 dma_unmap_single(dev, reqctx->iv_dma, IV, 2493 DMA_BIDIRECTIONAL); 2494 if (req->src == req->dst) { 2495 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2496 DMA_BIDIRECTIONAL); 2497 } else { 2498 dma_unmap_sg(dev, req->src, sg_nents(req->src), 2499 DMA_TO_DEVICE); 2500 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), 2501 DMA_FROM_DEVICE); 2502 } 2503 } 2504 2505 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 2506 { 2507 __be32 data; 2508 2509 memset(block, 0, csize); 2510 block += csize; 2511 2512 if (csize >= 4) 2513 csize = 4; 2514 else if (msglen > (unsigned int)(1 << (8 * csize))) 2515 return -EOVERFLOW; 2516 2517 data = cpu_to_be32(msglen); 2518 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 2519 2520 return 0; 2521 } 2522 2523 static void generate_b0(struct aead_request *req, 2524 struct chcr_aead_ctx *aeadctx, 2525 unsigned short op_type) 2526 { 2527 unsigned int l, lp, m; 2528 int rc; 2529 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2530 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2531 u8 *b0 = reqctx->scratch_pad; 2532 2533 m = crypto_aead_authsize(aead); 2534 2535 memcpy(b0, reqctx->iv, 16); 2536 2537 lp = b0[0]; 2538 l = lp + 1; 2539 2540 /* set m, bits 3-5 */ 2541 *b0 |= (8 * ((m - 2) / 2)); 2542 2543 /* set adata, bit 6, if associated data is used */ 2544 if (req->assoclen) 2545 *b0 |= 64; 2546 rc = set_msg_len(b0 + 16 - l, 2547 (op_type == CHCR_DECRYPT_OP) ? 2548 req->cryptlen - m : req->cryptlen, l); 2549 } 2550 2551 static inline int crypto_ccm_check_iv(const u8 *iv) 2552 { 2553 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 2554 if (iv[0] < 1 || iv[0] > 7) 2555 return -EINVAL; 2556 2557 return 0; 2558 } 2559 2560 static int ccm_format_packet(struct aead_request *req, 2561 struct chcr_aead_ctx *aeadctx, 2562 unsigned int sub_type, 2563 unsigned short op_type) 2564 { 2565 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2566 int rc = 0; 2567 2568 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2569 reqctx->iv[0] = 3; 2570 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3); 2571 memcpy(reqctx->iv + 4, req->iv, 8); 2572 memset(reqctx->iv + 12, 0, 4); 2573 *((unsigned short *)(reqctx->scratch_pad + 16)) = 2574 htons(req->assoclen - 8); 2575 } else { 2576 memcpy(reqctx->iv, req->iv, 16); 2577 *((unsigned short *)(reqctx->scratch_pad + 16)) = 2578 htons(req->assoclen); 2579 } 2580 generate_b0(req, aeadctx, op_type); 2581 /* zero the ctr value */ 2582 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1); 2583 return rc; 2584 } 2585 2586 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 2587 unsigned int dst_size, 2588 struct aead_request *req, 2589 unsigned short op_type) 2590 { 2591 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2592 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2593 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 2594 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 2595 unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id; 2596 unsigned int ccm_xtra; 2597 unsigned char tag_offset = 0, auth_offset = 0; 2598 unsigned int assoclen; 2599 2600 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2601 assoclen = req->assoclen - 8; 2602 else 2603 assoclen = req->assoclen; 2604 ccm_xtra = CCM_B0_SIZE + 2605 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 2606 2607 auth_offset = req->cryptlen ? 2608 (assoclen + IV + 1 + ccm_xtra) : 0; 2609 if (op_type == CHCR_DECRYPT_OP) { 2610 if (crypto_aead_authsize(tfm) != req->cryptlen) 2611 tag_offset = crypto_aead_authsize(tfm); 2612 else 2613 auth_offset = 0; 2614 } 2615 2616 2617 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, 2618 2, assoclen + 1 + ccm_xtra); 2619 sec_cpl->pldlen = 2620 htonl(assoclen + IV + req->cryptlen + ccm_xtra); 2621 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 2622 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2623 1, assoclen + ccm_xtra, assoclen 2624 + IV + 1 + ccm_xtra, 0); 2625 2626 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 2627 auth_offset, tag_offset, 2628 (op_type == CHCR_ENCRYPT_OP) ? 0 : 2629 crypto_aead_authsize(tfm)); 2630 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2631 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 2632 cipher_mode, mac_mode, 2633 aeadctx->hmac_ctrl, IV >> 1); 2634 2635 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 2636 0, dst_size); 2637 } 2638 2639 int aead_ccm_validate_input(unsigned short op_type, 2640 struct aead_request *req, 2641 struct chcr_aead_ctx *aeadctx, 2642 unsigned int sub_type) 2643 { 2644 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2645 if (crypto_ccm_check_iv(req->iv)) { 2646 pr_err("CCM: IV check fails\n"); 2647 return -EINVAL; 2648 } 2649 } else { 2650 if (req->assoclen != 16 && req->assoclen != 20) { 2651 pr_err("RFC4309: Invalid AAD length %d\n", 2652 req->assoclen); 2653 return -EINVAL; 2654 } 2655 } 2656 return 0; 2657 } 2658 2659 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 2660 unsigned short qid, 2661 int size, 2662 unsigned short op_type) 2663 { 2664 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2665 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2666 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2667 struct sk_buff *skb = NULL; 2668 struct chcr_wr *chcr_req; 2669 struct cpl_rx_phys_dsgl *phys_cpl; 2670 struct ulptx_sgl *ulptx; 2671 unsigned int transhdr_len; 2672 unsigned int dst_size = 0, kctx_len, dnents, temp; 2673 unsigned int sub_type, assoclen = req->assoclen; 2674 unsigned int authsize = crypto_aead_authsize(tfm); 2675 int error = -EINVAL; 2676 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2677 GFP_ATOMIC; 2678 struct adapter *adap = padap(a_ctx(tfm)->dev); 2679 2680 reqctx->b0_dma = 0; 2681 sub_type = get_aead_subtype(tfm); 2682 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2683 assoclen -= 8; 2684 error = chcr_aead_common_init(req, op_type); 2685 if (error) 2686 return ERR_PTR(error); 2687 2688 2689 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); 2690 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); 2691 if (error) 2692 goto err; 2693 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); 2694 dnents += sg_nents_xlen(req->dst, req->cryptlen 2695 + (op_type ? -authsize : authsize), 2696 CHCR_DST_SG_SIZE, req->assoclen); 2697 dnents += MIN_CCM_SG; // For IV and B0 2698 dst_size = get_space_for_phys_dsgl(dnents); 2699 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; 2700 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2701 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen + 2702 reqctx->b0_len) <= SGE_MAX_WR_LEN; 2703 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen + 2704 reqctx->b0_len), 16) * 16) : 2705 (sgl_len(reqctx->src_nents + reqctx->aad_nents + 2706 MIN_CCM_SG) * 8); 2707 transhdr_len += temp; 2708 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 2709 2710 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - 2711 reqctx->b0_len, transhdr_len, op_type)) { 2712 atomic_inc(&adap->chcr_stats.fallback); 2713 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2714 op_type); 2715 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2716 } 2717 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 2718 2719 if (!skb) { 2720 error = -ENOMEM; 2721 goto err; 2722 } 2723 2724 chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len); 2725 2726 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type); 2727 2728 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2729 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 2730 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 2731 16), aeadctx->key, aeadctx->enckey_len); 2732 2733 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2734 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 2735 error = ccm_format_packet(req, aeadctx, sub_type, op_type); 2736 if (error) 2737 goto dstmap_fail; 2738 2739 reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, 2740 &reqctx->scratch_pad, reqctx->b0_len, 2741 DMA_BIDIRECTIONAL); 2742 if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, 2743 reqctx->b0_dma)) { 2744 error = -ENOMEM; 2745 goto dstmap_fail; 2746 } 2747 2748 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); 2749 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); 2750 2751 atomic_inc(&adap->chcr_stats.aead_rqst); 2752 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + 2753 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen + 2754 reqctx->b0_len) : 0); 2755 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, 2756 transhdr_len, temp, 0); 2757 reqctx->skb = skb; 2758 reqctx->op = op_type; 2759 2760 return skb; 2761 dstmap_fail: 2762 kfree_skb(skb); 2763 err: 2764 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); 2765 return ERR_PTR(error); 2766 } 2767 2768 static struct sk_buff *create_gcm_wr(struct aead_request *req, 2769 unsigned short qid, 2770 int size, 2771 unsigned short op_type) 2772 { 2773 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2774 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2775 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2776 struct sk_buff *skb = NULL; 2777 struct chcr_wr *chcr_req; 2778 struct cpl_rx_phys_dsgl *phys_cpl; 2779 struct ulptx_sgl *ulptx; 2780 unsigned int transhdr_len, dnents = 0; 2781 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; 2782 unsigned int authsize = crypto_aead_authsize(tfm); 2783 int error = -EINVAL; 2784 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2785 GFP_ATOMIC; 2786 struct adapter *adap = padap(a_ctx(tfm)->dev); 2787 2788 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 2789 assoclen = req->assoclen - 8; 2790 2791 reqctx->b0_dma = 0; 2792 error = chcr_aead_common_init(req, op_type); 2793 if (error) 2794 return ERR_PTR(error); 2795 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); 2796 dnents += sg_nents_xlen(req->dst, req->cryptlen + 2797 (op_type ? -authsize : authsize), 2798 CHCR_DST_SG_SIZE, req->assoclen); 2799 dnents += MIN_GCM_SG; // For IV 2800 dst_size = get_space_for_phys_dsgl(dnents); 2801 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + 2802 AEAD_H_SIZE; 2803 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2804 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <= 2805 SGE_MAX_WR_LEN; 2806 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + 2807 req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents + 2808 reqctx->aad_nents + MIN_GCM_SG) * 8); 2809 transhdr_len += temp; 2810 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; 2811 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, 2812 transhdr_len, op_type)) { 2813 atomic_inc(&adap->chcr_stats.fallback); 2814 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, 2815 op_type); 2816 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2817 } 2818 skb = alloc_skb(SGE_MAX_WR_LEN, flags); 2819 if (!skb) { 2820 error = -ENOMEM; 2821 goto err; 2822 } 2823 2824 chcr_req = __skb_put_zero(skb, transhdr_len); 2825 2826 //Offset of tag from end 2827 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 2828 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 2829 a_ctx(tfm)->dev->rx_channel_id, 2, 2830 (assoclen + 1)); 2831 chcr_req->sec_cpl.pldlen = 2832 htonl(assoclen + IV + req->cryptlen); 2833 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2834 assoclen ? 1 : 0, assoclen, 2835 assoclen + IV + 1, 0); 2836 chcr_req->sec_cpl.cipherstop_lo_authinsert = 2837 FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1, 2838 temp, temp); 2839 chcr_req->sec_cpl.seqno_numivs = 2840 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == 2841 CHCR_ENCRYPT_OP) ? 1 : 0, 2842 CHCR_SCMD_CIPHER_MODE_AES_GCM, 2843 CHCR_SCMD_AUTH_MODE_GHASH, 2844 aeadctx->hmac_ctrl, IV >> 1); 2845 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2846 0, 0, dst_size); 2847 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2848 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 2849 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 2850 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 2851 2852 /* prepare a 16 byte iv */ 2853 /* S A L T | IV | 0x00000001 */ 2854 if (get_aead_subtype(tfm) == 2855 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 2856 memcpy(reqctx->iv, aeadctx->salt, 4); 2857 memcpy(reqctx->iv + 4, req->iv, GCM_RFC4106_IV_SIZE); 2858 } else { 2859 memcpy(reqctx->iv, req->iv, GCM_AES_IV_SIZE); 2860 } 2861 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); 2862 2863 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2864 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); 2865 2866 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); 2867 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); 2868 atomic_inc(&adap->chcr_stats.aead_rqst); 2869 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + 2870 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); 2871 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, 2872 transhdr_len, temp, reqctx->verify); 2873 reqctx->skb = skb; 2874 reqctx->op = op_type; 2875 return skb; 2876 2877 err: 2878 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); 2879 return ERR_PTR(error); 2880 } 2881 2882 2883 2884 static int chcr_aead_cra_init(struct crypto_aead *tfm) 2885 { 2886 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2887 struct aead_alg *alg = crypto_aead_alg(tfm); 2888 2889 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, 2890 CRYPTO_ALG_NEED_FALLBACK | 2891 CRYPTO_ALG_ASYNC); 2892 if (IS_ERR(aeadctx->sw_cipher)) 2893 return PTR_ERR(aeadctx->sw_cipher); 2894 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), 2895 sizeof(struct aead_request) + 2896 crypto_aead_reqsize(aeadctx->sw_cipher))); 2897 return chcr_device_init(a_ctx(tfm)); 2898 } 2899 2900 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 2901 { 2902 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2903 2904 crypto_free_aead(aeadctx->sw_cipher); 2905 } 2906 2907 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 2908 unsigned int authsize) 2909 { 2910 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2911 2912 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 2913 aeadctx->mayverify = VERIFY_HW; 2914 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2915 } 2916 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 2917 unsigned int authsize) 2918 { 2919 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2920 u32 maxauth = crypto_aead_maxauthsize(tfm); 2921 2922 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 2923 * true for sha1. authsize == 12 condition should be before 2924 * authsize == (maxauth >> 1) 2925 */ 2926 if (authsize == ICV_4) { 2927 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2928 aeadctx->mayverify = VERIFY_HW; 2929 } else if (authsize == ICV_6) { 2930 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2931 aeadctx->mayverify = VERIFY_HW; 2932 } else if (authsize == ICV_10) { 2933 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2934 aeadctx->mayverify = VERIFY_HW; 2935 } else if (authsize == ICV_12) { 2936 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2937 aeadctx->mayverify = VERIFY_HW; 2938 } else if (authsize == ICV_14) { 2939 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2940 aeadctx->mayverify = VERIFY_HW; 2941 } else if (authsize == (maxauth >> 1)) { 2942 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2943 aeadctx->mayverify = VERIFY_HW; 2944 } else if (authsize == maxauth) { 2945 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2946 aeadctx->mayverify = VERIFY_HW; 2947 } else { 2948 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2949 aeadctx->mayverify = VERIFY_SW; 2950 } 2951 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2952 } 2953 2954 2955 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 2956 { 2957 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2958 2959 switch (authsize) { 2960 case ICV_4: 2961 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2962 aeadctx->mayverify = VERIFY_HW; 2963 break; 2964 case ICV_8: 2965 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2966 aeadctx->mayverify = VERIFY_HW; 2967 break; 2968 case ICV_12: 2969 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2970 aeadctx->mayverify = VERIFY_HW; 2971 break; 2972 case ICV_14: 2973 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2974 aeadctx->mayverify = VERIFY_HW; 2975 break; 2976 case ICV_16: 2977 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2978 aeadctx->mayverify = VERIFY_HW; 2979 break; 2980 case ICV_13: 2981 case ICV_15: 2982 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2983 aeadctx->mayverify = VERIFY_SW; 2984 break; 2985 default: 2986 2987 crypto_tfm_set_flags((struct crypto_tfm *) tfm, 2988 CRYPTO_TFM_RES_BAD_KEY_LEN); 2989 return -EINVAL; 2990 } 2991 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2992 } 2993 2994 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 2995 unsigned int authsize) 2996 { 2997 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 2998 2999 switch (authsize) { 3000 case ICV_8: 3001 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3002 aeadctx->mayverify = VERIFY_HW; 3003 break; 3004 case ICV_12: 3005 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3006 aeadctx->mayverify = VERIFY_HW; 3007 break; 3008 case ICV_16: 3009 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3010 aeadctx->mayverify = VERIFY_HW; 3011 break; 3012 default: 3013 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 3014 CRYPTO_TFM_RES_BAD_KEY_LEN); 3015 return -EINVAL; 3016 } 3017 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3018 } 3019 3020 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 3021 unsigned int authsize) 3022 { 3023 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3024 3025 switch (authsize) { 3026 case ICV_4: 3027 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 3028 aeadctx->mayverify = VERIFY_HW; 3029 break; 3030 case ICV_6: 3031 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 3032 aeadctx->mayverify = VERIFY_HW; 3033 break; 3034 case ICV_8: 3035 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 3036 aeadctx->mayverify = VERIFY_HW; 3037 break; 3038 case ICV_10: 3039 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 3040 aeadctx->mayverify = VERIFY_HW; 3041 break; 3042 case ICV_12: 3043 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 3044 aeadctx->mayverify = VERIFY_HW; 3045 break; 3046 case ICV_14: 3047 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 3048 aeadctx->mayverify = VERIFY_HW; 3049 break; 3050 case ICV_16: 3051 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 3052 aeadctx->mayverify = VERIFY_HW; 3053 break; 3054 default: 3055 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 3056 CRYPTO_TFM_RES_BAD_KEY_LEN); 3057 return -EINVAL; 3058 } 3059 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 3060 } 3061 3062 static int chcr_ccm_common_setkey(struct crypto_aead *aead, 3063 const u8 *key, 3064 unsigned int keylen) 3065 { 3066 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3067 unsigned char ck_size, mk_size; 3068 int key_ctx_size = 0; 3069 3070 key_ctx_size = sizeof(struct _key_ctx) + 3071 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2; 3072 if (keylen == AES_KEYSIZE_128) { 3073 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3074 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3075 } else if (keylen == AES_KEYSIZE_192) { 3076 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3077 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 3078 } else if (keylen == AES_KEYSIZE_256) { 3079 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3080 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 3081 } else { 3082 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3083 CRYPTO_TFM_RES_BAD_KEY_LEN); 3084 aeadctx->enckey_len = 0; 3085 return -EINVAL; 3086 } 3087 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 3088 key_ctx_size >> 4); 3089 memcpy(aeadctx->key, key, keylen); 3090 aeadctx->enckey_len = keylen; 3091 3092 return 0; 3093 } 3094 3095 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 3096 const u8 *key, 3097 unsigned int keylen) 3098 { 3099 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3100 int error; 3101 3102 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3103 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3104 CRYPTO_TFM_REQ_MASK); 3105 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3106 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3107 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3108 CRYPTO_TFM_RES_MASK); 3109 if (error) 3110 return error; 3111 return chcr_ccm_common_setkey(aead, key, keylen); 3112 } 3113 3114 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 3115 unsigned int keylen) 3116 { 3117 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3118 int error; 3119 3120 if (keylen < 3) { 3121 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3122 CRYPTO_TFM_RES_BAD_KEY_LEN); 3123 aeadctx->enckey_len = 0; 3124 return -EINVAL; 3125 } 3126 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3127 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 3128 CRYPTO_TFM_REQ_MASK); 3129 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3130 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3131 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3132 CRYPTO_TFM_RES_MASK); 3133 if (error) 3134 return error; 3135 keylen -= 3; 3136 memcpy(aeadctx->salt, key + keylen, 3); 3137 return chcr_ccm_common_setkey(aead, key, keylen); 3138 } 3139 3140 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 3141 unsigned int keylen) 3142 { 3143 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); 3144 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 3145 struct crypto_cipher *cipher; 3146 unsigned int ck_size; 3147 int ret = 0, key_ctx_size = 0; 3148 3149 aeadctx->enckey_len = 0; 3150 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3151 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) 3152 & CRYPTO_TFM_REQ_MASK); 3153 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3154 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 3155 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 3156 CRYPTO_TFM_RES_MASK); 3157 if (ret) 3158 goto out; 3159 3160 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 3161 keylen > 3) { 3162 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 3163 memcpy(aeadctx->salt, key + keylen, 4); 3164 } 3165 if (keylen == AES_KEYSIZE_128) { 3166 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3167 } else if (keylen == AES_KEYSIZE_192) { 3168 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3169 } else if (keylen == AES_KEYSIZE_256) { 3170 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3171 } else { 3172 crypto_tfm_set_flags((struct crypto_tfm *)aead, 3173 CRYPTO_TFM_RES_BAD_KEY_LEN); 3174 pr_err("GCM: Invalid key length %d\n", keylen); 3175 ret = -EINVAL; 3176 goto out; 3177 } 3178 3179 memcpy(aeadctx->key, key, keylen); 3180 aeadctx->enckey_len = keylen; 3181 key_ctx_size = sizeof(struct _key_ctx) + 3182 ((DIV_ROUND_UP(keylen, 16)) << 4) + 3183 AEAD_H_SIZE; 3184 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 3185 CHCR_KEYCTX_MAC_KEY_SIZE_128, 3186 0, 0, 3187 key_ctx_size >> 4); 3188 /* Calculate the H = CIPH(K, 0 repeated 16 times). 3189 * It will go in key context 3190 */ 3191 cipher = crypto_alloc_cipher("aes-generic", 0, 0); 3192 if (IS_ERR(cipher)) { 3193 aeadctx->enckey_len = 0; 3194 ret = -ENOMEM; 3195 goto out; 3196 } 3197 3198 ret = crypto_cipher_setkey(cipher, key, keylen); 3199 if (ret) { 3200 aeadctx->enckey_len = 0; 3201 goto out1; 3202 } 3203 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 3204 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h); 3205 3206 out1: 3207 crypto_free_cipher(cipher); 3208 out: 3209 return ret; 3210 } 3211 3212 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 3213 unsigned int keylen) 3214 { 3215 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3216 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3217 /* it contains auth and cipher key both*/ 3218 struct crypto_authenc_keys keys; 3219 unsigned int bs, subtype; 3220 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 3221 int err = 0, i, key_ctx_len = 0; 3222 unsigned char ck_size = 0; 3223 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 3224 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 3225 struct algo_param param; 3226 int align; 3227 u8 *o_ptr = NULL; 3228 3229 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3230 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3231 & CRYPTO_TFM_REQ_MASK); 3232 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3233 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 3234 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 3235 & CRYPTO_TFM_RES_MASK); 3236 if (err) 3237 goto out; 3238 3239 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 3240 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 3241 goto out; 3242 } 3243 3244 if (get_alg_config(¶m, max_authsize)) { 3245 pr_err("chcr : Unsupported digest size\n"); 3246 goto out; 3247 } 3248 subtype = get_aead_subtype(authenc); 3249 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3250 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3251 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3252 goto out; 3253 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3254 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3255 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3256 } 3257 if (keys.enckeylen == AES_KEYSIZE_128) { 3258 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3259 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3260 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3261 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3262 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3263 } else { 3264 pr_err("chcr : Unsupported cipher key\n"); 3265 goto out; 3266 } 3267 3268 /* Copy only encryption key. We use authkey to generate h(ipad) and 3269 * h(opad) so authkey is not needed again. authkeylen size have the 3270 * size of the hash digest size. 3271 */ 3272 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3273 aeadctx->enckey_len = keys.enckeylen; 3274 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3275 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3276 3277 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3278 aeadctx->enckey_len << 3); 3279 } 3280 base_hash = chcr_alloc_shash(max_authsize); 3281 if (IS_ERR(base_hash)) { 3282 pr_err("chcr : Base driver cannot be loaded\n"); 3283 aeadctx->enckey_len = 0; 3284 return -EINVAL; 3285 } 3286 { 3287 SHASH_DESC_ON_STACK(shash, base_hash); 3288 shash->tfm = base_hash; 3289 shash->flags = crypto_shash_get_flags(base_hash); 3290 bs = crypto_shash_blocksize(base_hash); 3291 align = KEYCTX_ALIGN_PAD(max_authsize); 3292 o_ptr = actx->h_iopad + param.result_size + align; 3293 3294 if (keys.authkeylen > bs) { 3295 err = crypto_shash_digest(shash, keys.authkey, 3296 keys.authkeylen, 3297 o_ptr); 3298 if (err) { 3299 pr_err("chcr : Base driver cannot be loaded\n"); 3300 goto out; 3301 } 3302 keys.authkeylen = max_authsize; 3303 } else 3304 memcpy(o_ptr, keys.authkey, keys.authkeylen); 3305 3306 /* Compute the ipad-digest*/ 3307 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3308 memcpy(pad, o_ptr, keys.authkeylen); 3309 for (i = 0; i < bs >> 2; i++) 3310 *((unsigned int *)pad + i) ^= IPAD_DATA; 3311 3312 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 3313 max_authsize)) 3314 goto out; 3315 /* Compute the opad-digest */ 3316 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3317 memcpy(pad, o_ptr, keys.authkeylen); 3318 for (i = 0; i < bs >> 2; i++) 3319 *((unsigned int *)pad + i) ^= OPAD_DATA; 3320 3321 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 3322 goto out; 3323 3324 /* convert the ipad and opad digest to network order */ 3325 chcr_change_order(actx->h_iopad, param.result_size); 3326 chcr_change_order(o_ptr, param.result_size); 3327 key_ctx_len = sizeof(struct _key_ctx) + 3328 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) + 3329 (param.result_size + align) * 2; 3330 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 3331 0, 1, key_ctx_len >> 4); 3332 actx->auth_mode = param.auth_mode; 3333 chcr_free_shash(base_hash); 3334 3335 return 0; 3336 } 3337 out: 3338 aeadctx->enckey_len = 0; 3339 if (!IS_ERR(base_hash)) 3340 chcr_free_shash(base_hash); 3341 return -EINVAL; 3342 } 3343 3344 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 3345 const u8 *key, unsigned int keylen) 3346 { 3347 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); 3348 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3349 struct crypto_authenc_keys keys; 3350 int err; 3351 /* it contains auth and cipher key both*/ 3352 unsigned int subtype; 3353 int key_ctx_len = 0; 3354 unsigned char ck_size = 0; 3355 3356 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3357 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3358 & CRYPTO_TFM_REQ_MASK); 3359 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3360 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 3361 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 3362 & CRYPTO_TFM_RES_MASK); 3363 if (err) 3364 goto out; 3365 3366 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 3367 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 3368 goto out; 3369 } 3370 subtype = get_aead_subtype(authenc); 3371 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA || 3372 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) { 3373 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE) 3374 goto out; 3375 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen 3376 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE); 3377 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 3378 } 3379 if (keys.enckeylen == AES_KEYSIZE_128) { 3380 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3381 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3382 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3383 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3384 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3385 } else { 3386 pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen); 3387 goto out; 3388 } 3389 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3390 aeadctx->enckey_len = keys.enckeylen; 3391 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA || 3392 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) { 3393 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3394 aeadctx->enckey_len << 3); 3395 } 3396 key_ctx_len = sizeof(struct _key_ctx) 3397 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4); 3398 3399 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 3400 0, key_ctx_len >> 4); 3401 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 3402 return 0; 3403 out: 3404 aeadctx->enckey_len = 0; 3405 return -EINVAL; 3406 } 3407 3408 static int chcr_aead_op(struct aead_request *req, 3409 unsigned short op_type, 3410 int size, 3411 create_wr_t create_wr_fn) 3412 { 3413 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3414 struct uld_ctx *u_ctx; 3415 struct sk_buff *skb; 3416 3417 if (!a_ctx(tfm)->dev) { 3418 pr_err("chcr : %s : No crypto device.\n", __func__); 3419 return -ENXIO; 3420 } 3421 u_ctx = ULD_CTX(a_ctx(tfm)); 3422 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 3423 a_ctx(tfm)->tx_qidx)) { 3424 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3425 return -EBUSY; 3426 } 3427 3428 /* Form a WR from req */ 3429 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, 3430 op_type); 3431 3432 if (IS_ERR(skb) || !skb) 3433 return PTR_ERR(skb); 3434 3435 skb->dev = u_ctx->lldi.ports[0]; 3436 set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); 3437 chcr_send_wr(skb); 3438 return -EINPROGRESS; 3439 } 3440 3441 static int chcr_aead_encrypt(struct aead_request *req) 3442 { 3443 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3444 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3445 3446 reqctx->verify = VERIFY_HW; 3447 3448 switch (get_aead_subtype(tfm)) { 3449 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3450 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3451 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3452 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3453 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3454 create_authenc_wr); 3455 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3456 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3457 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3458 create_aead_ccm_wr); 3459 default: 3460 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3461 create_gcm_wr); 3462 } 3463 } 3464 3465 static int chcr_aead_decrypt(struct aead_request *req) 3466 { 3467 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3468 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); 3469 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3470 int size; 3471 3472 if (aeadctx->mayverify == VERIFY_SW) { 3473 size = crypto_aead_maxauthsize(tfm); 3474 reqctx->verify = VERIFY_SW; 3475 } else { 3476 size = 0; 3477 reqctx->verify = VERIFY_HW; 3478 } 3479 3480 switch (get_aead_subtype(tfm)) { 3481 case CRYPTO_ALG_SUB_TYPE_CBC_SHA: 3482 case CRYPTO_ALG_SUB_TYPE_CTR_SHA: 3483 case CRYPTO_ALG_SUB_TYPE_CBC_NULL: 3484 case CRYPTO_ALG_SUB_TYPE_CTR_NULL: 3485 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3486 create_authenc_wr); 3487 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3488 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3489 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3490 create_aead_ccm_wr); 3491 default: 3492 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3493 create_gcm_wr); 3494 } 3495 } 3496 3497 static struct chcr_alg_template driver_algs[] = { 3498 /* AES-CBC */ 3499 { 3500 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, 3501 .is_registered = 0, 3502 .alg.crypto = { 3503 .cra_name = "cbc(aes)", 3504 .cra_driver_name = "cbc-aes-chcr", 3505 .cra_blocksize = AES_BLOCK_SIZE, 3506 .cra_init = chcr_cra_init, 3507 .cra_exit = chcr_cra_exit, 3508 .cra_u.ablkcipher = { 3509 .min_keysize = AES_MIN_KEY_SIZE, 3510 .max_keysize = AES_MAX_KEY_SIZE, 3511 .ivsize = AES_BLOCK_SIZE, 3512 .setkey = chcr_aes_cbc_setkey, 3513 .encrypt = chcr_aes_encrypt, 3514 .decrypt = chcr_aes_decrypt, 3515 } 3516 } 3517 }, 3518 { 3519 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, 3520 .is_registered = 0, 3521 .alg.crypto = { 3522 .cra_name = "xts(aes)", 3523 .cra_driver_name = "xts-aes-chcr", 3524 .cra_blocksize = AES_BLOCK_SIZE, 3525 .cra_init = chcr_cra_init, 3526 .cra_exit = NULL, 3527 .cra_u .ablkcipher = { 3528 .min_keysize = 2 * AES_MIN_KEY_SIZE, 3529 .max_keysize = 2 * AES_MAX_KEY_SIZE, 3530 .ivsize = AES_BLOCK_SIZE, 3531 .setkey = chcr_aes_xts_setkey, 3532 .encrypt = chcr_aes_encrypt, 3533 .decrypt = chcr_aes_decrypt, 3534 } 3535 } 3536 }, 3537 { 3538 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, 3539 .is_registered = 0, 3540 .alg.crypto = { 3541 .cra_name = "ctr(aes)", 3542 .cra_driver_name = "ctr-aes-chcr", 3543 .cra_blocksize = 1, 3544 .cra_init = chcr_cra_init, 3545 .cra_exit = chcr_cra_exit, 3546 .cra_u.ablkcipher = { 3547 .min_keysize = AES_MIN_KEY_SIZE, 3548 .max_keysize = AES_MAX_KEY_SIZE, 3549 .ivsize = AES_BLOCK_SIZE, 3550 .setkey = chcr_aes_ctr_setkey, 3551 .encrypt = chcr_aes_encrypt, 3552 .decrypt = chcr_aes_decrypt, 3553 } 3554 } 3555 }, 3556 { 3557 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | 3558 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, 3559 .is_registered = 0, 3560 .alg.crypto = { 3561 .cra_name = "rfc3686(ctr(aes))", 3562 .cra_driver_name = "rfc3686-ctr-aes-chcr", 3563 .cra_blocksize = 1, 3564 .cra_init = chcr_rfc3686_init, 3565 .cra_exit = chcr_cra_exit, 3566 .cra_u.ablkcipher = { 3567 .min_keysize = AES_MIN_KEY_SIZE + 3568 CTR_RFC3686_NONCE_SIZE, 3569 .max_keysize = AES_MAX_KEY_SIZE + 3570 CTR_RFC3686_NONCE_SIZE, 3571 .ivsize = CTR_RFC3686_IV_SIZE, 3572 .setkey = chcr_aes_rfc3686_setkey, 3573 .encrypt = chcr_aes_encrypt, 3574 .decrypt = chcr_aes_decrypt, 3575 .geniv = "seqiv", 3576 } 3577 } 3578 }, 3579 /* SHA */ 3580 { 3581 .type = CRYPTO_ALG_TYPE_AHASH, 3582 .is_registered = 0, 3583 .alg.hash = { 3584 .halg.digestsize = SHA1_DIGEST_SIZE, 3585 .halg.base = { 3586 .cra_name = "sha1", 3587 .cra_driver_name = "sha1-chcr", 3588 .cra_blocksize = SHA1_BLOCK_SIZE, 3589 } 3590 } 3591 }, 3592 { 3593 .type = CRYPTO_ALG_TYPE_AHASH, 3594 .is_registered = 0, 3595 .alg.hash = { 3596 .halg.digestsize = SHA256_DIGEST_SIZE, 3597 .halg.base = { 3598 .cra_name = "sha256", 3599 .cra_driver_name = "sha256-chcr", 3600 .cra_blocksize = SHA256_BLOCK_SIZE, 3601 } 3602 } 3603 }, 3604 { 3605 .type = CRYPTO_ALG_TYPE_AHASH, 3606 .is_registered = 0, 3607 .alg.hash = { 3608 .halg.digestsize = SHA224_DIGEST_SIZE, 3609 .halg.base = { 3610 .cra_name = "sha224", 3611 .cra_driver_name = "sha224-chcr", 3612 .cra_blocksize = SHA224_BLOCK_SIZE, 3613 } 3614 } 3615 }, 3616 { 3617 .type = CRYPTO_ALG_TYPE_AHASH, 3618 .is_registered = 0, 3619 .alg.hash = { 3620 .halg.digestsize = SHA384_DIGEST_SIZE, 3621 .halg.base = { 3622 .cra_name = "sha384", 3623 .cra_driver_name = "sha384-chcr", 3624 .cra_blocksize = SHA384_BLOCK_SIZE, 3625 } 3626 } 3627 }, 3628 { 3629 .type = CRYPTO_ALG_TYPE_AHASH, 3630 .is_registered = 0, 3631 .alg.hash = { 3632 .halg.digestsize = SHA512_DIGEST_SIZE, 3633 .halg.base = { 3634 .cra_name = "sha512", 3635 .cra_driver_name = "sha512-chcr", 3636 .cra_blocksize = SHA512_BLOCK_SIZE, 3637 } 3638 } 3639 }, 3640 /* HMAC */ 3641 { 3642 .type = CRYPTO_ALG_TYPE_HMAC, 3643 .is_registered = 0, 3644 .alg.hash = { 3645 .halg.digestsize = SHA1_DIGEST_SIZE, 3646 .halg.base = { 3647 .cra_name = "hmac(sha1)", 3648 .cra_driver_name = "hmac-sha1-chcr", 3649 .cra_blocksize = SHA1_BLOCK_SIZE, 3650 } 3651 } 3652 }, 3653 { 3654 .type = CRYPTO_ALG_TYPE_HMAC, 3655 .is_registered = 0, 3656 .alg.hash = { 3657 .halg.digestsize = SHA224_DIGEST_SIZE, 3658 .halg.base = { 3659 .cra_name = "hmac(sha224)", 3660 .cra_driver_name = "hmac-sha224-chcr", 3661 .cra_blocksize = SHA224_BLOCK_SIZE, 3662 } 3663 } 3664 }, 3665 { 3666 .type = CRYPTO_ALG_TYPE_HMAC, 3667 .is_registered = 0, 3668 .alg.hash = { 3669 .halg.digestsize = SHA256_DIGEST_SIZE, 3670 .halg.base = { 3671 .cra_name = "hmac(sha256)", 3672 .cra_driver_name = "hmac-sha256-chcr", 3673 .cra_blocksize = SHA256_BLOCK_SIZE, 3674 } 3675 } 3676 }, 3677 { 3678 .type = CRYPTO_ALG_TYPE_HMAC, 3679 .is_registered = 0, 3680 .alg.hash = { 3681 .halg.digestsize = SHA384_DIGEST_SIZE, 3682 .halg.base = { 3683 .cra_name = "hmac(sha384)", 3684 .cra_driver_name = "hmac-sha384-chcr", 3685 .cra_blocksize = SHA384_BLOCK_SIZE, 3686 } 3687 } 3688 }, 3689 { 3690 .type = CRYPTO_ALG_TYPE_HMAC, 3691 .is_registered = 0, 3692 .alg.hash = { 3693 .halg.digestsize = SHA512_DIGEST_SIZE, 3694 .halg.base = { 3695 .cra_name = "hmac(sha512)", 3696 .cra_driver_name = "hmac-sha512-chcr", 3697 .cra_blocksize = SHA512_BLOCK_SIZE, 3698 } 3699 } 3700 }, 3701 /* Add AEAD Algorithms */ 3702 { 3703 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 3704 .is_registered = 0, 3705 .alg.aead = { 3706 .base = { 3707 .cra_name = "gcm(aes)", 3708 .cra_driver_name = "gcm-aes-chcr", 3709 .cra_blocksize = 1, 3710 .cra_priority = CHCR_AEAD_PRIORITY, 3711 .cra_ctxsize = sizeof(struct chcr_context) + 3712 sizeof(struct chcr_aead_ctx) + 3713 sizeof(struct chcr_gcm_ctx), 3714 }, 3715 .ivsize = GCM_AES_IV_SIZE, 3716 .maxauthsize = GHASH_DIGEST_SIZE, 3717 .setkey = chcr_gcm_setkey, 3718 .setauthsize = chcr_gcm_setauthsize, 3719 } 3720 }, 3721 { 3722 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 3723 .is_registered = 0, 3724 .alg.aead = { 3725 .base = { 3726 .cra_name = "rfc4106(gcm(aes))", 3727 .cra_driver_name = "rfc4106-gcm-aes-chcr", 3728 .cra_blocksize = 1, 3729 .cra_priority = CHCR_AEAD_PRIORITY + 1, 3730 .cra_ctxsize = sizeof(struct chcr_context) + 3731 sizeof(struct chcr_aead_ctx) + 3732 sizeof(struct chcr_gcm_ctx), 3733 3734 }, 3735 .ivsize = GCM_RFC4106_IV_SIZE, 3736 .maxauthsize = GHASH_DIGEST_SIZE, 3737 .setkey = chcr_gcm_setkey, 3738 .setauthsize = chcr_4106_4309_setauthsize, 3739 } 3740 }, 3741 { 3742 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 3743 .is_registered = 0, 3744 .alg.aead = { 3745 .base = { 3746 .cra_name = "ccm(aes)", 3747 .cra_driver_name = "ccm-aes-chcr", 3748 .cra_blocksize = 1, 3749 .cra_priority = CHCR_AEAD_PRIORITY, 3750 .cra_ctxsize = sizeof(struct chcr_context) + 3751 sizeof(struct chcr_aead_ctx), 3752 3753 }, 3754 .ivsize = AES_BLOCK_SIZE, 3755 .maxauthsize = GHASH_DIGEST_SIZE, 3756 .setkey = chcr_aead_ccm_setkey, 3757 .setauthsize = chcr_ccm_setauthsize, 3758 } 3759 }, 3760 { 3761 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 3762 .is_registered = 0, 3763 .alg.aead = { 3764 .base = { 3765 .cra_name = "rfc4309(ccm(aes))", 3766 .cra_driver_name = "rfc4309-ccm-aes-chcr", 3767 .cra_blocksize = 1, 3768 .cra_priority = CHCR_AEAD_PRIORITY + 1, 3769 .cra_ctxsize = sizeof(struct chcr_context) + 3770 sizeof(struct chcr_aead_ctx), 3771 3772 }, 3773 .ivsize = 8, 3774 .maxauthsize = GHASH_DIGEST_SIZE, 3775 .setkey = chcr_aead_rfc4309_setkey, 3776 .setauthsize = chcr_4106_4309_setauthsize, 3777 } 3778 }, 3779 { 3780 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 3781 .is_registered = 0, 3782 .alg.aead = { 3783 .base = { 3784 .cra_name = "authenc(hmac(sha1),cbc(aes))", 3785 .cra_driver_name = 3786 "authenc-hmac-sha1-cbc-aes-chcr", 3787 .cra_blocksize = AES_BLOCK_SIZE, 3788 .cra_priority = CHCR_AEAD_PRIORITY, 3789 .cra_ctxsize = sizeof(struct chcr_context) + 3790 sizeof(struct chcr_aead_ctx) + 3791 sizeof(struct chcr_authenc_ctx), 3792 3793 }, 3794 .ivsize = AES_BLOCK_SIZE, 3795 .maxauthsize = SHA1_DIGEST_SIZE, 3796 .setkey = chcr_authenc_setkey, 3797 .setauthsize = chcr_authenc_setauthsize, 3798 } 3799 }, 3800 { 3801 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 3802 .is_registered = 0, 3803 .alg.aead = { 3804 .base = { 3805 3806 .cra_name = "authenc(hmac(sha256),cbc(aes))", 3807 .cra_driver_name = 3808 "authenc-hmac-sha256-cbc-aes-chcr", 3809 .cra_blocksize = AES_BLOCK_SIZE, 3810 .cra_priority = CHCR_AEAD_PRIORITY, 3811 .cra_ctxsize = sizeof(struct chcr_context) + 3812 sizeof(struct chcr_aead_ctx) + 3813 sizeof(struct chcr_authenc_ctx), 3814 3815 }, 3816 .ivsize = AES_BLOCK_SIZE, 3817 .maxauthsize = SHA256_DIGEST_SIZE, 3818 .setkey = chcr_authenc_setkey, 3819 .setauthsize = chcr_authenc_setauthsize, 3820 } 3821 }, 3822 { 3823 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 3824 .is_registered = 0, 3825 .alg.aead = { 3826 .base = { 3827 .cra_name = "authenc(hmac(sha224),cbc(aes))", 3828 .cra_driver_name = 3829 "authenc-hmac-sha224-cbc-aes-chcr", 3830 .cra_blocksize = AES_BLOCK_SIZE, 3831 .cra_priority = CHCR_AEAD_PRIORITY, 3832 .cra_ctxsize = sizeof(struct chcr_context) + 3833 sizeof(struct chcr_aead_ctx) + 3834 sizeof(struct chcr_authenc_ctx), 3835 }, 3836 .ivsize = AES_BLOCK_SIZE, 3837 .maxauthsize = SHA224_DIGEST_SIZE, 3838 .setkey = chcr_authenc_setkey, 3839 .setauthsize = chcr_authenc_setauthsize, 3840 } 3841 }, 3842 { 3843 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 3844 .is_registered = 0, 3845 .alg.aead = { 3846 .base = { 3847 .cra_name = "authenc(hmac(sha384),cbc(aes))", 3848 .cra_driver_name = 3849 "authenc-hmac-sha384-cbc-aes-chcr", 3850 .cra_blocksize = AES_BLOCK_SIZE, 3851 .cra_priority = CHCR_AEAD_PRIORITY, 3852 .cra_ctxsize = sizeof(struct chcr_context) + 3853 sizeof(struct chcr_aead_ctx) + 3854 sizeof(struct chcr_authenc_ctx), 3855 3856 }, 3857 .ivsize = AES_BLOCK_SIZE, 3858 .maxauthsize = SHA384_DIGEST_SIZE, 3859 .setkey = chcr_authenc_setkey, 3860 .setauthsize = chcr_authenc_setauthsize, 3861 } 3862 }, 3863 { 3864 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA, 3865 .is_registered = 0, 3866 .alg.aead = { 3867 .base = { 3868 .cra_name = "authenc(hmac(sha512),cbc(aes))", 3869 .cra_driver_name = 3870 "authenc-hmac-sha512-cbc-aes-chcr", 3871 .cra_blocksize = AES_BLOCK_SIZE, 3872 .cra_priority = CHCR_AEAD_PRIORITY, 3873 .cra_ctxsize = sizeof(struct chcr_context) + 3874 sizeof(struct chcr_aead_ctx) + 3875 sizeof(struct chcr_authenc_ctx), 3876 3877 }, 3878 .ivsize = AES_BLOCK_SIZE, 3879 .maxauthsize = SHA512_DIGEST_SIZE, 3880 .setkey = chcr_authenc_setkey, 3881 .setauthsize = chcr_authenc_setauthsize, 3882 } 3883 }, 3884 { 3885 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL, 3886 .is_registered = 0, 3887 .alg.aead = { 3888 .base = { 3889 .cra_name = "authenc(digest_null,cbc(aes))", 3890 .cra_driver_name = 3891 "authenc-digest_null-cbc-aes-chcr", 3892 .cra_blocksize = AES_BLOCK_SIZE, 3893 .cra_priority = CHCR_AEAD_PRIORITY, 3894 .cra_ctxsize = sizeof(struct chcr_context) + 3895 sizeof(struct chcr_aead_ctx) + 3896 sizeof(struct chcr_authenc_ctx), 3897 3898 }, 3899 .ivsize = AES_BLOCK_SIZE, 3900 .maxauthsize = 0, 3901 .setkey = chcr_aead_digest_null_setkey, 3902 .setauthsize = chcr_authenc_null_setauthsize, 3903 } 3904 }, 3905 { 3906 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 3907 .is_registered = 0, 3908 .alg.aead = { 3909 .base = { 3910 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 3911 .cra_driver_name = 3912 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr", 3913 .cra_blocksize = 1, 3914 .cra_priority = CHCR_AEAD_PRIORITY, 3915 .cra_ctxsize = sizeof(struct chcr_context) + 3916 sizeof(struct chcr_aead_ctx) + 3917 sizeof(struct chcr_authenc_ctx), 3918 3919 }, 3920 .ivsize = CTR_RFC3686_IV_SIZE, 3921 .maxauthsize = SHA1_DIGEST_SIZE, 3922 .setkey = chcr_authenc_setkey, 3923 .setauthsize = chcr_authenc_setauthsize, 3924 } 3925 }, 3926 { 3927 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 3928 .is_registered = 0, 3929 .alg.aead = { 3930 .base = { 3931 3932 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 3933 .cra_driver_name = 3934 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr", 3935 .cra_blocksize = 1, 3936 .cra_priority = CHCR_AEAD_PRIORITY, 3937 .cra_ctxsize = sizeof(struct chcr_context) + 3938 sizeof(struct chcr_aead_ctx) + 3939 sizeof(struct chcr_authenc_ctx), 3940 3941 }, 3942 .ivsize = CTR_RFC3686_IV_SIZE, 3943 .maxauthsize = SHA256_DIGEST_SIZE, 3944 .setkey = chcr_authenc_setkey, 3945 .setauthsize = chcr_authenc_setauthsize, 3946 } 3947 }, 3948 { 3949 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 3950 .is_registered = 0, 3951 .alg.aead = { 3952 .base = { 3953 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 3954 .cra_driver_name = 3955 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr", 3956 .cra_blocksize = 1, 3957 .cra_priority = CHCR_AEAD_PRIORITY, 3958 .cra_ctxsize = sizeof(struct chcr_context) + 3959 sizeof(struct chcr_aead_ctx) + 3960 sizeof(struct chcr_authenc_ctx), 3961 }, 3962 .ivsize = CTR_RFC3686_IV_SIZE, 3963 .maxauthsize = SHA224_DIGEST_SIZE, 3964 .setkey = chcr_authenc_setkey, 3965 .setauthsize = chcr_authenc_setauthsize, 3966 } 3967 }, 3968 { 3969 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 3970 .is_registered = 0, 3971 .alg.aead = { 3972 .base = { 3973 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 3974 .cra_driver_name = 3975 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr", 3976 .cra_blocksize = 1, 3977 .cra_priority = CHCR_AEAD_PRIORITY, 3978 .cra_ctxsize = sizeof(struct chcr_context) + 3979 sizeof(struct chcr_aead_ctx) + 3980 sizeof(struct chcr_authenc_ctx), 3981 3982 }, 3983 .ivsize = CTR_RFC3686_IV_SIZE, 3984 .maxauthsize = SHA384_DIGEST_SIZE, 3985 .setkey = chcr_authenc_setkey, 3986 .setauthsize = chcr_authenc_setauthsize, 3987 } 3988 }, 3989 { 3990 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA, 3991 .is_registered = 0, 3992 .alg.aead = { 3993 .base = { 3994 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 3995 .cra_driver_name = 3996 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr", 3997 .cra_blocksize = 1, 3998 .cra_priority = CHCR_AEAD_PRIORITY, 3999 .cra_ctxsize = sizeof(struct chcr_context) + 4000 sizeof(struct chcr_aead_ctx) + 4001 sizeof(struct chcr_authenc_ctx), 4002 4003 }, 4004 .ivsize = CTR_RFC3686_IV_SIZE, 4005 .maxauthsize = SHA512_DIGEST_SIZE, 4006 .setkey = chcr_authenc_setkey, 4007 .setauthsize = chcr_authenc_setauthsize, 4008 } 4009 }, 4010 { 4011 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL, 4012 .is_registered = 0, 4013 .alg.aead = { 4014 .base = { 4015 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))", 4016 .cra_driver_name = 4017 "authenc-digest_null-rfc3686-ctr-aes-chcr", 4018 .cra_blocksize = 1, 4019 .cra_priority = CHCR_AEAD_PRIORITY, 4020 .cra_ctxsize = sizeof(struct chcr_context) + 4021 sizeof(struct chcr_aead_ctx) + 4022 sizeof(struct chcr_authenc_ctx), 4023 4024 }, 4025 .ivsize = CTR_RFC3686_IV_SIZE, 4026 .maxauthsize = 0, 4027 .setkey = chcr_aead_digest_null_setkey, 4028 .setauthsize = chcr_authenc_null_setauthsize, 4029 } 4030 }, 4031 4032 }; 4033 4034 /* 4035 * chcr_unregister_alg - Deregister crypto algorithms with 4036 * kernel framework. 4037 */ 4038 static int chcr_unregister_alg(void) 4039 { 4040 int i; 4041 4042 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4043 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4044 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4045 if (driver_algs[i].is_registered) 4046 crypto_unregister_alg( 4047 &driver_algs[i].alg.crypto); 4048 break; 4049 case CRYPTO_ALG_TYPE_AEAD: 4050 if (driver_algs[i].is_registered) 4051 crypto_unregister_aead( 4052 &driver_algs[i].alg.aead); 4053 break; 4054 case CRYPTO_ALG_TYPE_AHASH: 4055 if (driver_algs[i].is_registered) 4056 crypto_unregister_ahash( 4057 &driver_algs[i].alg.hash); 4058 break; 4059 } 4060 driver_algs[i].is_registered = 0; 4061 } 4062 return 0; 4063 } 4064 4065 #define SZ_AHASH_CTX sizeof(struct chcr_context) 4066 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 4067 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 4068 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC) 4069 4070 /* 4071 * chcr_register_alg - Register crypto algorithms with kernel framework. 4072 */ 4073 static int chcr_register_alg(void) 4074 { 4075 struct crypto_alg ai; 4076 struct ahash_alg *a_hash; 4077 int err = 0, i; 4078 char *name = NULL; 4079 4080 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4081 if (driver_algs[i].is_registered) 4082 continue; 4083 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 4084 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4085 driver_algs[i].alg.crypto.cra_priority = 4086 CHCR_CRA_PRIORITY; 4087 driver_algs[i].alg.crypto.cra_module = THIS_MODULE; 4088 driver_algs[i].alg.crypto.cra_flags = 4089 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | 4090 CRYPTO_ALG_NEED_FALLBACK; 4091 driver_algs[i].alg.crypto.cra_ctxsize = 4092 sizeof(struct chcr_context) + 4093 sizeof(struct ablk_ctx); 4094 driver_algs[i].alg.crypto.cra_alignmask = 0; 4095 driver_algs[i].alg.crypto.cra_type = 4096 &crypto_ablkcipher_type; 4097 err = crypto_register_alg(&driver_algs[i].alg.crypto); 4098 name = driver_algs[i].alg.crypto.cra_driver_name; 4099 break; 4100 case CRYPTO_ALG_TYPE_AEAD: 4101 driver_algs[i].alg.aead.base.cra_flags = 4102 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC | 4103 CRYPTO_ALG_NEED_FALLBACK; 4104 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 4105 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 4106 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 4107 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 4108 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 4109 err = crypto_register_aead(&driver_algs[i].alg.aead); 4110 name = driver_algs[i].alg.aead.base.cra_driver_name; 4111 break; 4112 case CRYPTO_ALG_TYPE_AHASH: 4113 a_hash = &driver_algs[i].alg.hash; 4114 a_hash->update = chcr_ahash_update; 4115 a_hash->final = chcr_ahash_final; 4116 a_hash->finup = chcr_ahash_finup; 4117 a_hash->digest = chcr_ahash_digest; 4118 a_hash->export = chcr_ahash_export; 4119 a_hash->import = chcr_ahash_import; 4120 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 4121 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 4122 a_hash->halg.base.cra_module = THIS_MODULE; 4123 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS; 4124 a_hash->halg.base.cra_alignmask = 0; 4125 a_hash->halg.base.cra_exit = NULL; 4126 a_hash->halg.base.cra_type = &crypto_ahash_type; 4127 4128 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 4129 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 4130 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 4131 a_hash->init = chcr_hmac_init; 4132 a_hash->setkey = chcr_ahash_setkey; 4133 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 4134 } else { 4135 a_hash->init = chcr_sha_init; 4136 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 4137 a_hash->halg.base.cra_init = chcr_sha_cra_init; 4138 } 4139 err = crypto_register_ahash(&driver_algs[i].alg.hash); 4140 ai = driver_algs[i].alg.hash.halg.base; 4141 name = ai.cra_driver_name; 4142 break; 4143 } 4144 if (err) { 4145 pr_err("chcr : %s : Algorithm registration failed\n", 4146 name); 4147 goto register_err; 4148 } else { 4149 driver_algs[i].is_registered = 1; 4150 } 4151 } 4152 return 0; 4153 4154 register_err: 4155 chcr_unregister_alg(); 4156 return err; 4157 } 4158 4159 /* 4160 * start_crypto - Register the crypto algorithms. 4161 * This should called once when the first device comesup. After this 4162 * kernel will start calling driver APIs for crypto operations. 4163 */ 4164 int start_crypto(void) 4165 { 4166 return chcr_register_alg(); 4167 } 4168 4169 /* 4170 * stop_crypto - Deregister all the crypto algorithms with kernel. 4171 * This should be called once when the last device goes down. After this 4172 * kernel will not call the driver API for crypto operations. 4173 */ 4174 int stop_crypto(void) 4175 { 4176 chcr_unregister_alg(); 4177 return 0; 4178 } 4179