19bf4852dSDavid S. Miller /* Glue code for AES encryption optimized for sparc64 crypto opcodes. 29bf4852dSDavid S. Miller * 39bf4852dSDavid S. Miller * This is based largely upon arch/x86/crypto/aesni-intel_glue.c 49bf4852dSDavid S. Miller * 59bf4852dSDavid S. Miller * Copyright (C) 2008, Intel Corp. 69bf4852dSDavid S. Miller * Author: Huang Ying <ying.huang@intel.com> 79bf4852dSDavid S. Miller * 89bf4852dSDavid S. Miller * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 99bf4852dSDavid S. Miller * interface for 64-bit kernels. 109bf4852dSDavid S. Miller * Authors: Adrian Hoban <adrian.hoban@intel.com> 119bf4852dSDavid S. Miller * Gabriele Paoloni <gabriele.paoloni@intel.com> 129bf4852dSDavid S. Miller * Tadeusz Struk (tadeusz.struk@intel.com) 139bf4852dSDavid S. Miller * Aidan O'Mahony (aidan.o.mahony@intel.com) 149bf4852dSDavid S. Miller * Copyright (c) 2010, Intel Corporation. 159bf4852dSDavid S. Miller */ 169bf4852dSDavid S. Miller 179bf4852dSDavid S. Miller #include <linux/crypto.h> 189bf4852dSDavid S. Miller #include <linux/init.h> 199bf4852dSDavid S. Miller #include <linux/module.h> 209bf4852dSDavid S. Miller #include <linux/mm.h> 219bf4852dSDavid S. Miller #include <linux/types.h> 229bf4852dSDavid S. Miller #include <crypto/algapi.h> 239bf4852dSDavid S. Miller #include <crypto/aes.h> 249bf4852dSDavid S. Miller 259bf4852dSDavid S. Miller #include <asm/fpumacro.h> 269bf4852dSDavid S. Miller #include <asm/pstate.h> 279bf4852dSDavid S. Miller #include <asm/elf.h> 289bf4852dSDavid S. Miller 2910803624SDavid S. Miller #include "opcodes.h" 3010803624SDavid S. Miller 310bdcaf74SDavid S. Miller struct aes_ops { 320bdcaf74SDavid S. Miller void (*encrypt)(const u64 *key, const u32 *input, u32 *output); 330bdcaf74SDavid S. Miller void (*decrypt)(const u64 *key, const u32 *input, u32 *output); 340bdcaf74SDavid S. Miller void (*load_encrypt_keys)(const u64 *key); 350bdcaf74SDavid S. Miller void (*load_decrypt_keys)(const u64 *key); 360bdcaf74SDavid S. Miller void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output, 370bdcaf74SDavid S. Miller unsigned int len); 380bdcaf74SDavid S. Miller void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output, 390bdcaf74SDavid S. Miller unsigned int len); 400bdcaf74SDavid S. Miller void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output, 410bdcaf74SDavid S. Miller unsigned int len, u64 *iv); 420bdcaf74SDavid S. Miller void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output, 430bdcaf74SDavid S. Miller unsigned int len, u64 *iv); 449fd130ecSDavid S. Miller void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output, 459fd130ecSDavid S. Miller unsigned int len, u64 *iv); 460bdcaf74SDavid S. Miller }; 470bdcaf74SDavid S. Miller 489bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx { 490bdcaf74SDavid S. Miller struct aes_ops *ops; 509bf4852dSDavid S. Miller u64 key[AES_MAX_KEYLENGTH / sizeof(u64)]; 519bf4852dSDavid S. Miller u32 key_length; 529bf4852dSDavid S. Miller u32 expanded_key_length; 539bf4852dSDavid S. Miller }; 549bf4852dSDavid S. Miller 550bdcaf74SDavid S. Miller extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, 560bdcaf74SDavid S. Miller u32 *output); 570bdcaf74SDavid S. Miller extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, 580bdcaf74SDavid S. Miller u32 *output); 590bdcaf74SDavid S. Miller extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, 600bdcaf74SDavid S. Miller u32 *output); 610bdcaf74SDavid S. Miller 620bdcaf74SDavid S. Miller extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, 630bdcaf74SDavid S. Miller u32 *output); 640bdcaf74SDavid S. Miller extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, 650bdcaf74SDavid S. Miller u32 *output); 660bdcaf74SDavid S. Miller extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, 670bdcaf74SDavid S. Miller u32 *output); 680bdcaf74SDavid S. Miller 690bdcaf74SDavid S. Miller extern void aes_sparc64_load_encrypt_keys_128(const u64 *key); 700bdcaf74SDavid S. Miller extern void aes_sparc64_load_encrypt_keys_192(const u64 *key); 710bdcaf74SDavid S. Miller extern void aes_sparc64_load_encrypt_keys_256(const u64 *key); 720bdcaf74SDavid S. Miller 730bdcaf74SDavid S. Miller extern void aes_sparc64_load_decrypt_keys_128(const u64 *key); 740bdcaf74SDavid S. Miller extern void aes_sparc64_load_decrypt_keys_192(const u64 *key); 750bdcaf74SDavid S. Miller extern void aes_sparc64_load_decrypt_keys_256(const u64 *key); 760bdcaf74SDavid S. Miller 770bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, 780bdcaf74SDavid S. Miller u64 *output, unsigned int len); 790bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, 800bdcaf74SDavid S. Miller u64 *output, unsigned int len); 810bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, 820bdcaf74SDavid S. Miller u64 *output, unsigned int len); 830bdcaf74SDavid S. Miller 840bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, 850bdcaf74SDavid S. Miller u64 *output, unsigned int len); 860bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, 870bdcaf74SDavid S. Miller u64 *output, unsigned int len); 880bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, 890bdcaf74SDavid S. Miller u64 *output, unsigned int len); 900bdcaf74SDavid S. Miller 910bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, 920bdcaf74SDavid S. Miller u64 *output, unsigned int len, 930bdcaf74SDavid S. Miller u64 *iv); 940bdcaf74SDavid S. Miller 950bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, 960bdcaf74SDavid S. Miller u64 *output, unsigned int len, 970bdcaf74SDavid S. Miller u64 *iv); 980bdcaf74SDavid S. Miller 990bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, 1000bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1010bdcaf74SDavid S. Miller u64 *iv); 1020bdcaf74SDavid S. Miller 1030bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, 1040bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1050bdcaf74SDavid S. Miller u64 *iv); 1060bdcaf74SDavid S. Miller 1070bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, 1080bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1090bdcaf74SDavid S. Miller u64 *iv); 1100bdcaf74SDavid S. Miller 1110bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, 1120bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1130bdcaf74SDavid S. Miller u64 *iv); 1140bdcaf74SDavid S. Miller 1159fd130ecSDavid S. Miller extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, 1169fd130ecSDavid S. Miller u64 *output, unsigned int len, 1179fd130ecSDavid S. Miller u64 *iv); 1189fd130ecSDavid S. Miller extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, 1199fd130ecSDavid S. Miller u64 *output, unsigned int len, 1209fd130ecSDavid S. Miller u64 *iv); 1219fd130ecSDavid S. Miller extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, 1229fd130ecSDavid S. Miller u64 *output, unsigned int len, 1239fd130ecSDavid S. Miller u64 *iv); 1249fd130ecSDavid S. Miller 1250bdcaf74SDavid S. Miller struct aes_ops aes128_ops = { 1260bdcaf74SDavid S. Miller .encrypt = aes_sparc64_encrypt_128, 1270bdcaf74SDavid S. Miller .decrypt = aes_sparc64_decrypt_128, 1280bdcaf74SDavid S. Miller .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128, 1290bdcaf74SDavid S. Miller .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128, 1300bdcaf74SDavid S. Miller .ecb_encrypt = aes_sparc64_ecb_encrypt_128, 1310bdcaf74SDavid S. Miller .ecb_decrypt = aes_sparc64_ecb_decrypt_128, 1320bdcaf74SDavid S. Miller .cbc_encrypt = aes_sparc64_cbc_encrypt_128, 1330bdcaf74SDavid S. Miller .cbc_decrypt = aes_sparc64_cbc_decrypt_128, 1349fd130ecSDavid S. Miller .ctr_crypt = aes_sparc64_ctr_crypt_128, 1350bdcaf74SDavid S. Miller }; 1360bdcaf74SDavid S. Miller 1370bdcaf74SDavid S. Miller struct aes_ops aes192_ops = { 1380bdcaf74SDavid S. Miller .encrypt = aes_sparc64_encrypt_192, 1390bdcaf74SDavid S. Miller .decrypt = aes_sparc64_decrypt_192, 1400bdcaf74SDavid S. Miller .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192, 1410bdcaf74SDavid S. Miller .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192, 1420bdcaf74SDavid S. Miller .ecb_encrypt = aes_sparc64_ecb_encrypt_192, 1430bdcaf74SDavid S. Miller .ecb_decrypt = aes_sparc64_ecb_decrypt_192, 1440bdcaf74SDavid S. Miller .cbc_encrypt = aes_sparc64_cbc_encrypt_192, 1450bdcaf74SDavid S. Miller .cbc_decrypt = aes_sparc64_cbc_decrypt_192, 1469fd130ecSDavid S. Miller .ctr_crypt = aes_sparc64_ctr_crypt_192, 1470bdcaf74SDavid S. Miller }; 1480bdcaf74SDavid S. Miller 1490bdcaf74SDavid S. Miller struct aes_ops aes256_ops = { 1500bdcaf74SDavid S. Miller .encrypt = aes_sparc64_encrypt_256, 1510bdcaf74SDavid S. Miller .decrypt = aes_sparc64_decrypt_256, 1520bdcaf74SDavid S. Miller .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256, 1530bdcaf74SDavid S. Miller .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256, 1540bdcaf74SDavid S. Miller .ecb_encrypt = aes_sparc64_ecb_encrypt_256, 1550bdcaf74SDavid S. Miller .ecb_decrypt = aes_sparc64_ecb_decrypt_256, 1560bdcaf74SDavid S. Miller .cbc_encrypt = aes_sparc64_cbc_encrypt_256, 1570bdcaf74SDavid S. Miller .cbc_decrypt = aes_sparc64_cbc_decrypt_256, 1589fd130ecSDavid S. Miller .ctr_crypt = aes_sparc64_ctr_crypt_256, 1590bdcaf74SDavid S. Miller }; 1600bdcaf74SDavid S. Miller 1619bf4852dSDavid S. Miller extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key, 1629bf4852dSDavid S. Miller unsigned int key_len); 1639bf4852dSDavid S. Miller 1649bf4852dSDavid S. Miller static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 1659bf4852dSDavid S. Miller unsigned int key_len) 1669bf4852dSDavid S. Miller { 1679bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 1689bf4852dSDavid S. Miller u32 *flags = &tfm->crt_flags; 1699bf4852dSDavid S. Miller 1709bf4852dSDavid S. Miller switch (key_len) { 1719bf4852dSDavid S. Miller case AES_KEYSIZE_128: 1729bf4852dSDavid S. Miller ctx->expanded_key_length = 0xb0; 1730bdcaf74SDavid S. Miller ctx->ops = &aes128_ops; 1749bf4852dSDavid S. Miller break; 1759bf4852dSDavid S. Miller 1769bf4852dSDavid S. Miller case AES_KEYSIZE_192: 1779bf4852dSDavid S. Miller ctx->expanded_key_length = 0xd0; 1780bdcaf74SDavid S. Miller ctx->ops = &aes192_ops; 1799bf4852dSDavid S. Miller break; 1809bf4852dSDavid S. Miller 1819bf4852dSDavid S. Miller case AES_KEYSIZE_256: 1829bf4852dSDavid S. Miller ctx->expanded_key_length = 0xf0; 1830bdcaf74SDavid S. Miller ctx->ops = &aes256_ops; 1849bf4852dSDavid S. Miller break; 1859bf4852dSDavid S. Miller 1869bf4852dSDavid S. Miller default: 1879bf4852dSDavid S. Miller *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 1889bf4852dSDavid S. Miller return -EINVAL; 1899bf4852dSDavid S. Miller } 1909bf4852dSDavid S. Miller 1919bf4852dSDavid S. Miller aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len); 1929bf4852dSDavid S. Miller ctx->key_length = key_len; 1939bf4852dSDavid S. Miller 1949bf4852dSDavid S. Miller return 0; 1959bf4852dSDavid S. Miller } 1969bf4852dSDavid S. Miller 1979bf4852dSDavid S. Miller static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 1989bf4852dSDavid S. Miller { 1999bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 2009bf4852dSDavid S. Miller 2010bdcaf74SDavid S. Miller ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); 2029bf4852dSDavid S. Miller } 2039bf4852dSDavid S. Miller 2049bf4852dSDavid S. Miller static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 2059bf4852dSDavid S. Miller { 2069bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 2079bf4852dSDavid S. Miller 2080bdcaf74SDavid S. Miller ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); 2099bf4852dSDavid S. Miller } 2109bf4852dSDavid S. Miller 2119bf4852dSDavid S. Miller #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 2129bf4852dSDavid S. Miller 2139bf4852dSDavid S. Miller static int ecb_encrypt(struct blkcipher_desc *desc, 2149bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 2159bf4852dSDavid S. Miller unsigned int nbytes) 2169bf4852dSDavid S. Miller { 2179bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 2189bf4852dSDavid S. Miller struct blkcipher_walk walk; 2199bf4852dSDavid S. Miller int err; 2209bf4852dSDavid S. Miller 2219bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 2229bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 2239bf4852dSDavid S. Miller 2240bdcaf74SDavid S. Miller ctx->ops->load_encrypt_keys(&ctx->key[0]); 2259bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 2269bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 2279bf4852dSDavid S. Miller 2289bf4852dSDavid S. Miller if (likely(block_len)) { 2290bdcaf74SDavid S. Miller ctx->ops->ecb_encrypt(&ctx->key[0], 2300bdcaf74SDavid S. Miller (const u64 *)walk.src.virt.addr, 2310bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, 2320bdcaf74SDavid S. Miller block_len); 2339bf4852dSDavid S. Miller } 2349bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 2359bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 2369bf4852dSDavid S. Miller } 2379bf4852dSDavid S. Miller fprs_write(0); 2389bf4852dSDavid S. Miller return err; 2399bf4852dSDavid S. Miller } 2409bf4852dSDavid S. Miller 2419bf4852dSDavid S. Miller static int ecb_decrypt(struct blkcipher_desc *desc, 2429bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 2439bf4852dSDavid S. Miller unsigned int nbytes) 2449bf4852dSDavid S. Miller { 2459bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 2469bf4852dSDavid S. Miller struct blkcipher_walk walk; 2479bf4852dSDavid S. Miller u64 *key_end; 2489bf4852dSDavid S. Miller int err; 2499bf4852dSDavid S. Miller 2509bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 2519bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 2529bf4852dSDavid S. Miller 2530bdcaf74SDavid S. Miller ctx->ops->load_decrypt_keys(&ctx->key[0]); 2549bf4852dSDavid S. Miller key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; 2559bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 2569bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 2579bf4852dSDavid S. Miller 2580bdcaf74SDavid S. Miller if (likely(block_len)) { 2590bdcaf74SDavid S. Miller ctx->ops->ecb_decrypt(key_end, 2600bdcaf74SDavid S. Miller (const u64 *) walk.src.virt.addr, 2610bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, block_len); 2620bdcaf74SDavid S. Miller } 2639bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 2649bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 2659bf4852dSDavid S. Miller } 2669bf4852dSDavid S. Miller fprs_write(0); 2679bf4852dSDavid S. Miller 2689bf4852dSDavid S. Miller return err; 2699bf4852dSDavid S. Miller } 2709bf4852dSDavid S. Miller 2719bf4852dSDavid S. Miller static int cbc_encrypt(struct blkcipher_desc *desc, 2729bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 2739bf4852dSDavid S. Miller unsigned int nbytes) 2749bf4852dSDavid S. Miller { 2759bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 2769bf4852dSDavid S. Miller struct blkcipher_walk walk; 2779bf4852dSDavid S. Miller int err; 2789bf4852dSDavid S. Miller 2799bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 2809bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 2819bf4852dSDavid S. Miller 2820bdcaf74SDavid S. Miller ctx->ops->load_encrypt_keys(&ctx->key[0]); 2839bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 2849bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 2859bf4852dSDavid S. Miller 2869bf4852dSDavid S. Miller if (likely(block_len)) { 2870bdcaf74SDavid S. Miller ctx->ops->cbc_encrypt(&ctx->key[0], 2880bdcaf74SDavid S. Miller (const u64 *)walk.src.virt.addr, 2890bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, 2900bdcaf74SDavid S. Miller block_len, (u64 *) walk.iv); 2919bf4852dSDavid S. Miller } 2929bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 2939bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 2949bf4852dSDavid S. Miller } 2959bf4852dSDavid S. Miller fprs_write(0); 2969bf4852dSDavid S. Miller return err; 2979bf4852dSDavid S. Miller } 2989bf4852dSDavid S. Miller 2999bf4852dSDavid S. Miller static int cbc_decrypt(struct blkcipher_desc *desc, 3009bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 3019bf4852dSDavid S. Miller unsigned int nbytes) 3029bf4852dSDavid S. Miller { 3039bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 3049bf4852dSDavid S. Miller struct blkcipher_walk walk; 3059bf4852dSDavid S. Miller u64 *key_end; 3069bf4852dSDavid S. Miller int err; 3079bf4852dSDavid S. Miller 3089bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 3099bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 3109bf4852dSDavid S. Miller 3110bdcaf74SDavid S. Miller ctx->ops->load_decrypt_keys(&ctx->key[0]); 3129bf4852dSDavid S. Miller key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; 3139bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 3149bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 3159bf4852dSDavid S. Miller 3160bdcaf74SDavid S. Miller if (likely(block_len)) { 3170bdcaf74SDavid S. Miller ctx->ops->cbc_decrypt(key_end, 3180bdcaf74SDavid S. Miller (const u64 *) walk.src.virt.addr, 3190bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, 3209bf4852dSDavid S. Miller block_len, (u64 *) walk.iv); 3210bdcaf74SDavid S. Miller } 3229bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 3239bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 3249bf4852dSDavid S. Miller } 3259bf4852dSDavid S. Miller fprs_write(0); 3269bf4852dSDavid S. Miller 3279bf4852dSDavid S. Miller return err; 3289bf4852dSDavid S. Miller } 3299bf4852dSDavid S. Miller 3309fd130ecSDavid S. Miller static int ctr_crypt(struct blkcipher_desc *desc, 3319fd130ecSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 3329fd130ecSDavid S. Miller unsigned int nbytes) 3339fd130ecSDavid S. Miller { 3349fd130ecSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 3359fd130ecSDavid S. Miller struct blkcipher_walk walk; 3369fd130ecSDavid S. Miller int err; 3379fd130ecSDavid S. Miller 3389fd130ecSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 3399fd130ecSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 3409fd130ecSDavid S. Miller 3419fd130ecSDavid S. Miller ctx->ops->load_encrypt_keys(&ctx->key[0]); 3429fd130ecSDavid S. Miller while ((nbytes = walk.nbytes)) { 3439fd130ecSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 3449fd130ecSDavid S. Miller 3459fd130ecSDavid S. Miller if (likely(block_len)) { 3469fd130ecSDavid S. Miller ctx->ops->ctr_crypt(&ctx->key[0], 3479fd130ecSDavid S. Miller (const u64 *)walk.src.virt.addr, 3489fd130ecSDavid S. Miller (u64 *) walk.dst.virt.addr, 3499fd130ecSDavid S. Miller block_len, (u64 *) walk.iv); 3509fd130ecSDavid S. Miller } 3519fd130ecSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 3529fd130ecSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 3539fd130ecSDavid S. Miller } 3549fd130ecSDavid S. Miller fprs_write(0); 3559fd130ecSDavid S. Miller return err; 3569fd130ecSDavid S. Miller } 3579fd130ecSDavid S. Miller 3589bf4852dSDavid S. Miller static struct crypto_alg algs[] = { { 3599bf4852dSDavid S. Miller .cra_name = "aes", 3609bf4852dSDavid S. Miller .cra_driver_name = "aes-sparc64", 36110803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 3629bf4852dSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 3639bf4852dSDavid S. Miller .cra_blocksize = AES_BLOCK_SIZE, 3649bf4852dSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 3659bf4852dSDavid S. Miller .cra_alignmask = 3, 3669bf4852dSDavid S. Miller .cra_module = THIS_MODULE, 3679bf4852dSDavid S. Miller .cra_u = { 3689bf4852dSDavid S. Miller .cipher = { 3699bf4852dSDavid S. Miller .cia_min_keysize = AES_MIN_KEY_SIZE, 3709bf4852dSDavid S. Miller .cia_max_keysize = AES_MAX_KEY_SIZE, 3719bf4852dSDavid S. Miller .cia_setkey = aes_set_key, 3729bf4852dSDavid S. Miller .cia_encrypt = aes_encrypt, 3739bf4852dSDavid S. Miller .cia_decrypt = aes_decrypt 3749bf4852dSDavid S. Miller } 3759bf4852dSDavid S. Miller } 3769bf4852dSDavid S. Miller }, { 3779bf4852dSDavid S. Miller .cra_name = "ecb(aes)", 3789bf4852dSDavid S. Miller .cra_driver_name = "ecb-aes-sparc64", 37910803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 3809bf4852dSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 3819bf4852dSDavid S. Miller .cra_blocksize = AES_BLOCK_SIZE, 3829bf4852dSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 3839bf4852dSDavid S. Miller .cra_alignmask = 7, 3849bf4852dSDavid S. Miller .cra_type = &crypto_blkcipher_type, 3859bf4852dSDavid S. Miller .cra_module = THIS_MODULE, 3869bf4852dSDavid S. Miller .cra_u = { 3879bf4852dSDavid S. Miller .blkcipher = { 3889bf4852dSDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 3899bf4852dSDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 3909bf4852dSDavid S. Miller .setkey = aes_set_key, 3919bf4852dSDavid S. Miller .encrypt = ecb_encrypt, 3929bf4852dSDavid S. Miller .decrypt = ecb_decrypt, 3939bf4852dSDavid S. Miller }, 3949bf4852dSDavid S. Miller }, 3959bf4852dSDavid S. Miller }, { 3969bf4852dSDavid S. Miller .cra_name = "cbc(aes)", 3979bf4852dSDavid S. Miller .cra_driver_name = "cbc-aes-sparc64", 39810803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 3999bf4852dSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 4009bf4852dSDavid S. Miller .cra_blocksize = AES_BLOCK_SIZE, 4019bf4852dSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 4029bf4852dSDavid S. Miller .cra_alignmask = 7, 4039bf4852dSDavid S. Miller .cra_type = &crypto_blkcipher_type, 4049bf4852dSDavid S. Miller .cra_module = THIS_MODULE, 4059bf4852dSDavid S. Miller .cra_u = { 4069bf4852dSDavid S. Miller .blkcipher = { 4079bf4852dSDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 4089bf4852dSDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 4099bf4852dSDavid S. Miller .setkey = aes_set_key, 4109bf4852dSDavid S. Miller .encrypt = cbc_encrypt, 4119bf4852dSDavid S. Miller .decrypt = cbc_decrypt, 4129bf4852dSDavid S. Miller }, 4139bf4852dSDavid S. Miller }, 4149fd130ecSDavid S. Miller }, { 4159fd130ecSDavid S. Miller .cra_name = "ctr(aes)", 4169fd130ecSDavid S. Miller .cra_driver_name = "ctr-aes-sparc64", 41710803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 4189fd130ecSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 4199fd130ecSDavid S. Miller .cra_blocksize = AES_BLOCK_SIZE, 4209fd130ecSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 4219fd130ecSDavid S. Miller .cra_alignmask = 7, 4229fd130ecSDavid S. Miller .cra_type = &crypto_blkcipher_type, 4239fd130ecSDavid S. Miller .cra_module = THIS_MODULE, 4249fd130ecSDavid S. Miller .cra_u = { 4259fd130ecSDavid S. Miller .blkcipher = { 4269fd130ecSDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 4279fd130ecSDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 4289fd130ecSDavid S. Miller .setkey = aes_set_key, 4299fd130ecSDavid S. Miller .encrypt = ctr_crypt, 4309fd130ecSDavid S. Miller .decrypt = ctr_crypt, 4319fd130ecSDavid S. Miller }, 4329fd130ecSDavid S. Miller }, 4339bf4852dSDavid S. Miller } }; 4349bf4852dSDavid S. Miller 4359bf4852dSDavid S. Miller static bool __init sparc64_has_aes_opcode(void) 4369bf4852dSDavid S. Miller { 4379bf4852dSDavid S. Miller unsigned long cfr; 4389bf4852dSDavid S. Miller 4399bf4852dSDavid S. Miller if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 4409bf4852dSDavid S. Miller return false; 4419bf4852dSDavid S. Miller 4429bf4852dSDavid S. Miller __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 4439bf4852dSDavid S. Miller if (!(cfr & CFR_AES)) 4449bf4852dSDavid S. Miller return false; 4459bf4852dSDavid S. Miller 4469bf4852dSDavid S. Miller return true; 4479bf4852dSDavid S. Miller } 4489bf4852dSDavid S. Miller 4499bf4852dSDavid S. Miller static int __init aes_sparc64_mod_init(void) 4509bf4852dSDavid S. Miller { 4519bf4852dSDavid S. Miller int i; 4529bf4852dSDavid S. Miller 4539bf4852dSDavid S. Miller for (i = 0; i < ARRAY_SIZE(algs); i++) 4549bf4852dSDavid S. Miller INIT_LIST_HEAD(&algs[i].cra_list); 4559bf4852dSDavid S. Miller 4569bf4852dSDavid S. Miller if (sparc64_has_aes_opcode()) { 4579bf4852dSDavid S. Miller pr_info("Using sparc64 aes opcodes optimized AES implementation\n"); 4589bf4852dSDavid S. Miller return crypto_register_algs(algs, ARRAY_SIZE(algs)); 4599bf4852dSDavid S. Miller } 4609bf4852dSDavid S. Miller pr_info("sparc64 aes opcodes not available.\n"); 4619bf4852dSDavid S. Miller return -ENODEV; 4629bf4852dSDavid S. Miller } 4639bf4852dSDavid S. Miller 4649bf4852dSDavid S. Miller static void __exit aes_sparc64_mod_fini(void) 4659bf4852dSDavid S. Miller { 4669bf4852dSDavid S. Miller crypto_unregister_algs(algs, ARRAY_SIZE(algs)); 4679bf4852dSDavid S. Miller } 4689bf4852dSDavid S. Miller 4699bf4852dSDavid S. Miller module_init(aes_sparc64_mod_init); 4709bf4852dSDavid S. Miller module_exit(aes_sparc64_mod_fini); 4719bf4852dSDavid S. Miller 4729bf4852dSDavid S. Miller MODULE_LICENSE("GPL"); 4739bf4852dSDavid S. Miller MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated"); 4749bf4852dSDavid S. Miller 4759bf4852dSDavid S. Miller MODULE_ALIAS("aes"); 476