19bf4852dSDavid S. Miller /* Glue code for AES encryption optimized for sparc64 crypto opcodes. 29bf4852dSDavid S. Miller * 39bf4852dSDavid S. Miller * This is based largely upon arch/x86/crypto/aesni-intel_glue.c 49bf4852dSDavid S. Miller * 59bf4852dSDavid S. Miller * Copyright (C) 2008, Intel Corp. 69bf4852dSDavid S. Miller * Author: Huang Ying <ying.huang@intel.com> 79bf4852dSDavid S. Miller * 89bf4852dSDavid S. Miller * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 99bf4852dSDavid S. Miller * interface for 64-bit kernels. 109bf4852dSDavid S. Miller * Authors: Adrian Hoban <adrian.hoban@intel.com> 119bf4852dSDavid S. Miller * Gabriele Paoloni <gabriele.paoloni@intel.com> 129bf4852dSDavid S. Miller * Tadeusz Struk (tadeusz.struk@intel.com) 139bf4852dSDavid S. Miller * Aidan O'Mahony (aidan.o.mahony@intel.com) 149bf4852dSDavid S. Miller * Copyright (c) 2010, Intel Corporation. 159bf4852dSDavid S. Miller */ 169bf4852dSDavid S. Miller 1771741680SDavid S. Miller #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 1871741680SDavid S. Miller 199bf4852dSDavid S. Miller #include <linux/crypto.h> 209bf4852dSDavid S. Miller #include <linux/init.h> 219bf4852dSDavid S. Miller #include <linux/module.h> 229bf4852dSDavid S. Miller #include <linux/mm.h> 239bf4852dSDavid S. Miller #include <linux/types.h> 249bf4852dSDavid S. Miller #include <crypto/algapi.h> 259bf4852dSDavid S. Miller #include <crypto/aes.h> 269bf4852dSDavid S. Miller 279bf4852dSDavid S. Miller #include <asm/fpumacro.h> 289bf4852dSDavid S. Miller #include <asm/pstate.h> 299bf4852dSDavid S. Miller #include <asm/elf.h> 309bf4852dSDavid S. Miller 3110803624SDavid S. Miller #include "opcodes.h" 3210803624SDavid S. Miller 330bdcaf74SDavid S. Miller struct aes_ops { 340bdcaf74SDavid S. Miller void (*encrypt)(const u64 *key, const u32 *input, u32 *output); 350bdcaf74SDavid S. Miller void (*decrypt)(const u64 *key, const u32 *input, u32 *output); 360bdcaf74SDavid S. Miller void (*load_encrypt_keys)(const u64 *key); 370bdcaf74SDavid S. Miller void (*load_decrypt_keys)(const u64 *key); 380bdcaf74SDavid S. Miller void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output, 390bdcaf74SDavid S. Miller unsigned int len); 400bdcaf74SDavid S. Miller void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output, 410bdcaf74SDavid S. Miller unsigned int len); 420bdcaf74SDavid S. Miller void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output, 430bdcaf74SDavid S. Miller unsigned int len, u64 *iv); 440bdcaf74SDavid S. Miller void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output, 450bdcaf74SDavid S. Miller unsigned int len, u64 *iv); 469fd130ecSDavid S. Miller void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output, 479fd130ecSDavid S. Miller unsigned int len, u64 *iv); 480bdcaf74SDavid S. Miller }; 490bdcaf74SDavid S. Miller 509bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx { 510bdcaf74SDavid S. Miller struct aes_ops *ops; 529bf4852dSDavid S. Miller u64 key[AES_MAX_KEYLENGTH / sizeof(u64)]; 539bf4852dSDavid S. Miller u32 key_length; 549bf4852dSDavid S. Miller u32 expanded_key_length; 559bf4852dSDavid S. Miller }; 569bf4852dSDavid S. Miller 570bdcaf74SDavid S. Miller extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, 580bdcaf74SDavid S. Miller u32 *output); 590bdcaf74SDavid S. Miller extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, 600bdcaf74SDavid S. Miller u32 *output); 610bdcaf74SDavid S. Miller extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, 620bdcaf74SDavid S. Miller u32 *output); 630bdcaf74SDavid S. Miller 640bdcaf74SDavid S. Miller extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, 650bdcaf74SDavid S. Miller u32 *output); 660bdcaf74SDavid S. Miller extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, 670bdcaf74SDavid S. Miller u32 *output); 680bdcaf74SDavid S. Miller extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, 690bdcaf74SDavid S. Miller u32 *output); 700bdcaf74SDavid S. Miller 710bdcaf74SDavid S. Miller extern void aes_sparc64_load_encrypt_keys_128(const u64 *key); 720bdcaf74SDavid S. Miller extern void aes_sparc64_load_encrypt_keys_192(const u64 *key); 730bdcaf74SDavid S. Miller extern void aes_sparc64_load_encrypt_keys_256(const u64 *key); 740bdcaf74SDavid S. Miller 750bdcaf74SDavid S. Miller extern void aes_sparc64_load_decrypt_keys_128(const u64 *key); 760bdcaf74SDavid S. Miller extern void aes_sparc64_load_decrypt_keys_192(const u64 *key); 770bdcaf74SDavid S. Miller extern void aes_sparc64_load_decrypt_keys_256(const u64 *key); 780bdcaf74SDavid S. Miller 790bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, 800bdcaf74SDavid S. Miller u64 *output, unsigned int len); 810bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, 820bdcaf74SDavid S. Miller u64 *output, unsigned int len); 830bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, 840bdcaf74SDavid S. Miller u64 *output, unsigned int len); 850bdcaf74SDavid S. Miller 860bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, 870bdcaf74SDavid S. Miller u64 *output, unsigned int len); 880bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, 890bdcaf74SDavid S. Miller u64 *output, unsigned int len); 900bdcaf74SDavid S. Miller extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, 910bdcaf74SDavid S. Miller u64 *output, unsigned int len); 920bdcaf74SDavid S. Miller 930bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, 940bdcaf74SDavid S. Miller u64 *output, unsigned int len, 950bdcaf74SDavid S. Miller u64 *iv); 960bdcaf74SDavid S. Miller 970bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, 980bdcaf74SDavid S. Miller u64 *output, unsigned int len, 990bdcaf74SDavid S. Miller u64 *iv); 1000bdcaf74SDavid S. Miller 1010bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, 1020bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1030bdcaf74SDavid S. Miller u64 *iv); 1040bdcaf74SDavid S. Miller 1050bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, 1060bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1070bdcaf74SDavid S. Miller u64 *iv); 1080bdcaf74SDavid S. Miller 1090bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, 1100bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1110bdcaf74SDavid S. Miller u64 *iv); 1120bdcaf74SDavid S. Miller 1130bdcaf74SDavid S. Miller extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, 1140bdcaf74SDavid S. Miller u64 *output, unsigned int len, 1150bdcaf74SDavid S. Miller u64 *iv); 1160bdcaf74SDavid S. Miller 1179fd130ecSDavid S. Miller extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, 1189fd130ecSDavid S. Miller u64 *output, unsigned int len, 1199fd130ecSDavid S. Miller u64 *iv); 1209fd130ecSDavid S. Miller extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, 1219fd130ecSDavid S. Miller u64 *output, unsigned int len, 1229fd130ecSDavid S. Miller u64 *iv); 1239fd130ecSDavid S. Miller extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, 1249fd130ecSDavid S. Miller u64 *output, unsigned int len, 1259fd130ecSDavid S. Miller u64 *iv); 1269fd130ecSDavid S. Miller 1270bdcaf74SDavid S. Miller struct aes_ops aes128_ops = { 1280bdcaf74SDavid S. Miller .encrypt = aes_sparc64_encrypt_128, 1290bdcaf74SDavid S. Miller .decrypt = aes_sparc64_decrypt_128, 1300bdcaf74SDavid S. Miller .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128, 1310bdcaf74SDavid S. Miller .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128, 1320bdcaf74SDavid S. Miller .ecb_encrypt = aes_sparc64_ecb_encrypt_128, 1330bdcaf74SDavid S. Miller .ecb_decrypt = aes_sparc64_ecb_decrypt_128, 1340bdcaf74SDavid S. Miller .cbc_encrypt = aes_sparc64_cbc_encrypt_128, 1350bdcaf74SDavid S. Miller .cbc_decrypt = aes_sparc64_cbc_decrypt_128, 1369fd130ecSDavid S. Miller .ctr_crypt = aes_sparc64_ctr_crypt_128, 1370bdcaf74SDavid S. Miller }; 1380bdcaf74SDavid S. Miller 1390bdcaf74SDavid S. Miller struct aes_ops aes192_ops = { 1400bdcaf74SDavid S. Miller .encrypt = aes_sparc64_encrypt_192, 1410bdcaf74SDavid S. Miller .decrypt = aes_sparc64_decrypt_192, 1420bdcaf74SDavid S. Miller .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192, 1430bdcaf74SDavid S. Miller .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192, 1440bdcaf74SDavid S. Miller .ecb_encrypt = aes_sparc64_ecb_encrypt_192, 1450bdcaf74SDavid S. Miller .ecb_decrypt = aes_sparc64_ecb_decrypt_192, 1460bdcaf74SDavid S. Miller .cbc_encrypt = aes_sparc64_cbc_encrypt_192, 1470bdcaf74SDavid S. Miller .cbc_decrypt = aes_sparc64_cbc_decrypt_192, 1489fd130ecSDavid S. Miller .ctr_crypt = aes_sparc64_ctr_crypt_192, 1490bdcaf74SDavid S. Miller }; 1500bdcaf74SDavid S. Miller 1510bdcaf74SDavid S. Miller struct aes_ops aes256_ops = { 1520bdcaf74SDavid S. Miller .encrypt = aes_sparc64_encrypt_256, 1530bdcaf74SDavid S. Miller .decrypt = aes_sparc64_decrypt_256, 1540bdcaf74SDavid S. Miller .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256, 1550bdcaf74SDavid S. Miller .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256, 1560bdcaf74SDavid S. Miller .ecb_encrypt = aes_sparc64_ecb_encrypt_256, 1570bdcaf74SDavid S. Miller .ecb_decrypt = aes_sparc64_ecb_decrypt_256, 1580bdcaf74SDavid S. Miller .cbc_encrypt = aes_sparc64_cbc_encrypt_256, 1590bdcaf74SDavid S. Miller .cbc_decrypt = aes_sparc64_cbc_decrypt_256, 1609fd130ecSDavid S. Miller .ctr_crypt = aes_sparc64_ctr_crypt_256, 1610bdcaf74SDavid S. Miller }; 1620bdcaf74SDavid S. Miller 1639bf4852dSDavid S. Miller extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key, 1649bf4852dSDavid S. Miller unsigned int key_len); 1659bf4852dSDavid S. Miller 1669bf4852dSDavid S. Miller static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 1679bf4852dSDavid S. Miller unsigned int key_len) 1689bf4852dSDavid S. Miller { 1699bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 1709bf4852dSDavid S. Miller u32 *flags = &tfm->crt_flags; 1719bf4852dSDavid S. Miller 1729bf4852dSDavid S. Miller switch (key_len) { 1739bf4852dSDavid S. Miller case AES_KEYSIZE_128: 1749bf4852dSDavid S. Miller ctx->expanded_key_length = 0xb0; 1750bdcaf74SDavid S. Miller ctx->ops = &aes128_ops; 1769bf4852dSDavid S. Miller break; 1779bf4852dSDavid S. Miller 1789bf4852dSDavid S. Miller case AES_KEYSIZE_192: 1799bf4852dSDavid S. Miller ctx->expanded_key_length = 0xd0; 1800bdcaf74SDavid S. Miller ctx->ops = &aes192_ops; 1819bf4852dSDavid S. Miller break; 1829bf4852dSDavid S. Miller 1839bf4852dSDavid S. Miller case AES_KEYSIZE_256: 1849bf4852dSDavid S. Miller ctx->expanded_key_length = 0xf0; 1850bdcaf74SDavid S. Miller ctx->ops = &aes256_ops; 1869bf4852dSDavid S. Miller break; 1879bf4852dSDavid S. Miller 1889bf4852dSDavid S. Miller default: 1899bf4852dSDavid S. Miller *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 1909bf4852dSDavid S. Miller return -EINVAL; 1919bf4852dSDavid S. Miller } 1929bf4852dSDavid S. Miller 1939bf4852dSDavid S. Miller aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len); 1949bf4852dSDavid S. Miller ctx->key_length = key_len; 1959bf4852dSDavid S. Miller 1969bf4852dSDavid S. Miller return 0; 1979bf4852dSDavid S. Miller } 1989bf4852dSDavid S. Miller 1999bf4852dSDavid S. Miller static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 2009bf4852dSDavid S. Miller { 2019bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 2029bf4852dSDavid S. Miller 2030bdcaf74SDavid S. Miller ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); 2049bf4852dSDavid S. Miller } 2059bf4852dSDavid S. Miller 2069bf4852dSDavid S. Miller static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 2079bf4852dSDavid S. Miller { 2089bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 2099bf4852dSDavid S. Miller 2100bdcaf74SDavid S. Miller ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); 2119bf4852dSDavid S. Miller } 2129bf4852dSDavid S. Miller 2139bf4852dSDavid S. Miller #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 2149bf4852dSDavid S. Miller 2159bf4852dSDavid S. Miller static int ecb_encrypt(struct blkcipher_desc *desc, 2169bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 2179bf4852dSDavid S. Miller unsigned int nbytes) 2189bf4852dSDavid S. Miller { 2199bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 2209bf4852dSDavid S. Miller struct blkcipher_walk walk; 2219bf4852dSDavid S. Miller int err; 2229bf4852dSDavid S. Miller 2239bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 2249bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 2259bf4852dSDavid S. Miller 2260bdcaf74SDavid S. Miller ctx->ops->load_encrypt_keys(&ctx->key[0]); 2279bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 2289bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 2299bf4852dSDavid S. Miller 2309bf4852dSDavid S. Miller if (likely(block_len)) { 2310bdcaf74SDavid S. Miller ctx->ops->ecb_encrypt(&ctx->key[0], 2320bdcaf74SDavid S. Miller (const u64 *)walk.src.virt.addr, 2330bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, 2340bdcaf74SDavid S. Miller block_len); 2359bf4852dSDavid S. Miller } 2369bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 2379bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 2389bf4852dSDavid S. Miller } 2399bf4852dSDavid S. Miller fprs_write(0); 2409bf4852dSDavid S. Miller return err; 2419bf4852dSDavid S. Miller } 2429bf4852dSDavid S. Miller 2439bf4852dSDavid S. Miller static int ecb_decrypt(struct blkcipher_desc *desc, 2449bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 2459bf4852dSDavid S. Miller unsigned int nbytes) 2469bf4852dSDavid S. Miller { 2479bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 2489bf4852dSDavid S. Miller struct blkcipher_walk walk; 2499bf4852dSDavid S. Miller u64 *key_end; 2509bf4852dSDavid S. Miller int err; 2519bf4852dSDavid S. Miller 2529bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 2539bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 2549bf4852dSDavid S. Miller 2550bdcaf74SDavid S. Miller ctx->ops->load_decrypt_keys(&ctx->key[0]); 2569bf4852dSDavid S. Miller key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; 2579bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 2589bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 2599bf4852dSDavid S. Miller 2600bdcaf74SDavid S. Miller if (likely(block_len)) { 2610bdcaf74SDavid S. Miller ctx->ops->ecb_decrypt(key_end, 2620bdcaf74SDavid S. Miller (const u64 *) walk.src.virt.addr, 2630bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, block_len); 2640bdcaf74SDavid S. Miller } 2659bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 2669bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 2679bf4852dSDavid S. Miller } 2689bf4852dSDavid S. Miller fprs_write(0); 2699bf4852dSDavid S. Miller 2709bf4852dSDavid S. Miller return err; 2719bf4852dSDavid S. Miller } 2729bf4852dSDavid S. Miller 2739bf4852dSDavid S. Miller static int cbc_encrypt(struct blkcipher_desc *desc, 2749bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 2759bf4852dSDavid S. Miller unsigned int nbytes) 2769bf4852dSDavid S. Miller { 2779bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 2789bf4852dSDavid S. Miller struct blkcipher_walk walk; 2799bf4852dSDavid S. Miller int err; 2809bf4852dSDavid S. Miller 2819bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 2829bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 2839bf4852dSDavid S. Miller 2840bdcaf74SDavid S. Miller ctx->ops->load_encrypt_keys(&ctx->key[0]); 2859bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 2869bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 2879bf4852dSDavid S. Miller 2889bf4852dSDavid S. Miller if (likely(block_len)) { 2890bdcaf74SDavid S. Miller ctx->ops->cbc_encrypt(&ctx->key[0], 2900bdcaf74SDavid S. Miller (const u64 *)walk.src.virt.addr, 2910bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, 2920bdcaf74SDavid S. Miller block_len, (u64 *) walk.iv); 2939bf4852dSDavid S. Miller } 2949bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 2959bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 2969bf4852dSDavid S. Miller } 2979bf4852dSDavid S. Miller fprs_write(0); 2989bf4852dSDavid S. Miller return err; 2999bf4852dSDavid S. Miller } 3009bf4852dSDavid S. Miller 3019bf4852dSDavid S. Miller static int cbc_decrypt(struct blkcipher_desc *desc, 3029bf4852dSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 3039bf4852dSDavid S. Miller unsigned int nbytes) 3049bf4852dSDavid S. Miller { 3059bf4852dSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 3069bf4852dSDavid S. Miller struct blkcipher_walk walk; 3079bf4852dSDavid S. Miller u64 *key_end; 3089bf4852dSDavid S. Miller int err; 3099bf4852dSDavid S. Miller 3109bf4852dSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 3119bf4852dSDavid S. Miller err = blkcipher_walk_virt(desc, &walk); 3129bf4852dSDavid S. Miller 3130bdcaf74SDavid S. Miller ctx->ops->load_decrypt_keys(&ctx->key[0]); 3149bf4852dSDavid S. Miller key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; 3159bf4852dSDavid S. Miller while ((nbytes = walk.nbytes)) { 3169bf4852dSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 3179bf4852dSDavid S. Miller 3180bdcaf74SDavid S. Miller if (likely(block_len)) { 3190bdcaf74SDavid S. Miller ctx->ops->cbc_decrypt(key_end, 3200bdcaf74SDavid S. Miller (const u64 *) walk.src.virt.addr, 3210bdcaf74SDavid S. Miller (u64 *) walk.dst.virt.addr, 3229bf4852dSDavid S. Miller block_len, (u64 *) walk.iv); 3230bdcaf74SDavid S. Miller } 3249bf4852dSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 3259bf4852dSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 3269bf4852dSDavid S. Miller } 3279bf4852dSDavid S. Miller fprs_write(0); 3289bf4852dSDavid S. Miller 3299bf4852dSDavid S. Miller return err; 3309bf4852dSDavid S. Miller } 3319bf4852dSDavid S. Miller 332a8d97cefSDavid S. Miller static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx, 333a8d97cefSDavid S. Miller struct blkcipher_walk *walk) 334a8d97cefSDavid S. Miller { 335a8d97cefSDavid S. Miller u8 *ctrblk = walk->iv; 336a8d97cefSDavid S. Miller u64 keystream[AES_BLOCK_SIZE / sizeof(u64)]; 337a8d97cefSDavid S. Miller u8 *src = walk->src.virt.addr; 338a8d97cefSDavid S. Miller u8 *dst = walk->dst.virt.addr; 339a8d97cefSDavid S. Miller unsigned int nbytes = walk->nbytes; 340a8d97cefSDavid S. Miller 341a8d97cefSDavid S. Miller ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk, 342a8d97cefSDavid S. Miller keystream, AES_BLOCK_SIZE); 343a8d97cefSDavid S. Miller crypto_xor((u8 *) keystream, src, nbytes); 344a8d97cefSDavid S. Miller memcpy(dst, keystream, nbytes); 345a8d97cefSDavid S. Miller crypto_inc(ctrblk, AES_BLOCK_SIZE); 346a8d97cefSDavid S. Miller } 347a8d97cefSDavid S. Miller 3489fd130ecSDavid S. Miller static int ctr_crypt(struct blkcipher_desc *desc, 3499fd130ecSDavid S. Miller struct scatterlist *dst, struct scatterlist *src, 3509fd130ecSDavid S. Miller unsigned int nbytes) 3519fd130ecSDavid S. Miller { 3529fd130ecSDavid S. Miller struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 3539fd130ecSDavid S. Miller struct blkcipher_walk walk; 3549fd130ecSDavid S. Miller int err; 3559fd130ecSDavid S. Miller 3569fd130ecSDavid S. Miller blkcipher_walk_init(&walk, dst, src, nbytes); 357a8d97cefSDavid S. Miller err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 358a8d97cefSDavid S. Miller desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 3599fd130ecSDavid S. Miller 3609fd130ecSDavid S. Miller ctx->ops->load_encrypt_keys(&ctx->key[0]); 361a8d97cefSDavid S. Miller while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 3629fd130ecSDavid S. Miller unsigned int block_len = nbytes & AES_BLOCK_MASK; 3639fd130ecSDavid S. Miller 3649fd130ecSDavid S. Miller if (likely(block_len)) { 3659fd130ecSDavid S. Miller ctx->ops->ctr_crypt(&ctx->key[0], 3669fd130ecSDavid S. Miller (const u64 *)walk.src.virt.addr, 3679fd130ecSDavid S. Miller (u64 *) walk.dst.virt.addr, 3689fd130ecSDavid S. Miller block_len, (u64 *) walk.iv); 3699fd130ecSDavid S. Miller } 3709fd130ecSDavid S. Miller nbytes &= AES_BLOCK_SIZE - 1; 3719fd130ecSDavid S. Miller err = blkcipher_walk_done(desc, &walk, nbytes); 3729fd130ecSDavid S. Miller } 373a8d97cefSDavid S. Miller if (walk.nbytes) { 374a8d97cefSDavid S. Miller ctr_crypt_final(ctx, &walk); 375a8d97cefSDavid S. Miller err = blkcipher_walk_done(desc, &walk, 0); 376a8d97cefSDavid S. Miller } 3779fd130ecSDavid S. Miller fprs_write(0); 3789fd130ecSDavid S. Miller return err; 3799fd130ecSDavid S. Miller } 3809fd130ecSDavid S. Miller 3819bf4852dSDavid S. Miller static struct crypto_alg algs[] = { { 3829bf4852dSDavid S. Miller .cra_name = "aes", 3839bf4852dSDavid S. Miller .cra_driver_name = "aes-sparc64", 38410803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 3859bf4852dSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 3869bf4852dSDavid S. Miller .cra_blocksize = AES_BLOCK_SIZE, 3879bf4852dSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 3889bf4852dSDavid S. Miller .cra_alignmask = 3, 3899bf4852dSDavid S. Miller .cra_module = THIS_MODULE, 3909bf4852dSDavid S. Miller .cra_u = { 3919bf4852dSDavid S. Miller .cipher = { 3929bf4852dSDavid S. Miller .cia_min_keysize = AES_MIN_KEY_SIZE, 3939bf4852dSDavid S. Miller .cia_max_keysize = AES_MAX_KEY_SIZE, 3949bf4852dSDavid S. Miller .cia_setkey = aes_set_key, 3959bf4852dSDavid S. Miller .cia_encrypt = aes_encrypt, 3969bf4852dSDavid S. Miller .cia_decrypt = aes_decrypt 3979bf4852dSDavid S. Miller } 3989bf4852dSDavid S. Miller } 3999bf4852dSDavid S. Miller }, { 4009bf4852dSDavid S. Miller .cra_name = "ecb(aes)", 4019bf4852dSDavid S. Miller .cra_driver_name = "ecb-aes-sparc64", 40210803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 4039bf4852dSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 4049bf4852dSDavid S. Miller .cra_blocksize = AES_BLOCK_SIZE, 4059bf4852dSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 4069bf4852dSDavid S. Miller .cra_alignmask = 7, 4079bf4852dSDavid S. Miller .cra_type = &crypto_blkcipher_type, 4089bf4852dSDavid S. Miller .cra_module = THIS_MODULE, 4099bf4852dSDavid S. Miller .cra_u = { 4109bf4852dSDavid S. Miller .blkcipher = { 4119bf4852dSDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 4129bf4852dSDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 4139bf4852dSDavid S. Miller .setkey = aes_set_key, 4149bf4852dSDavid S. Miller .encrypt = ecb_encrypt, 4159bf4852dSDavid S. Miller .decrypt = ecb_decrypt, 4169bf4852dSDavid S. Miller }, 4179bf4852dSDavid S. Miller }, 4189bf4852dSDavid S. Miller }, { 4199bf4852dSDavid S. Miller .cra_name = "cbc(aes)", 4209bf4852dSDavid S. Miller .cra_driver_name = "cbc-aes-sparc64", 42110803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 4229bf4852dSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 4239bf4852dSDavid S. Miller .cra_blocksize = AES_BLOCK_SIZE, 4249bf4852dSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 4259bf4852dSDavid S. Miller .cra_alignmask = 7, 4269bf4852dSDavid S. Miller .cra_type = &crypto_blkcipher_type, 4279bf4852dSDavid S. Miller .cra_module = THIS_MODULE, 4289bf4852dSDavid S. Miller .cra_u = { 4299bf4852dSDavid S. Miller .blkcipher = { 4309bf4852dSDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 4319bf4852dSDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 4329bf4852dSDavid S. Miller .setkey = aes_set_key, 4339bf4852dSDavid S. Miller .encrypt = cbc_encrypt, 4349bf4852dSDavid S. Miller .decrypt = cbc_decrypt, 4359bf4852dSDavid S. Miller }, 4369bf4852dSDavid S. Miller }, 4379fd130ecSDavid S. Miller }, { 4389fd130ecSDavid S. Miller .cra_name = "ctr(aes)", 4399fd130ecSDavid S. Miller .cra_driver_name = "ctr-aes-sparc64", 44010803624SDavid S. Miller .cra_priority = SPARC_CR_OPCODE_PRIORITY, 4419fd130ecSDavid S. Miller .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 442a8d97cefSDavid S. Miller .cra_blocksize = 1, 4439fd130ecSDavid S. Miller .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), 4449fd130ecSDavid S. Miller .cra_alignmask = 7, 4459fd130ecSDavid S. Miller .cra_type = &crypto_blkcipher_type, 4469fd130ecSDavid S. Miller .cra_module = THIS_MODULE, 4479fd130ecSDavid S. Miller .cra_u = { 4489fd130ecSDavid S. Miller .blkcipher = { 4499fd130ecSDavid S. Miller .min_keysize = AES_MIN_KEY_SIZE, 4509fd130ecSDavid S. Miller .max_keysize = AES_MAX_KEY_SIZE, 4519fd130ecSDavid S. Miller .setkey = aes_set_key, 4529fd130ecSDavid S. Miller .encrypt = ctr_crypt, 4539fd130ecSDavid S. Miller .decrypt = ctr_crypt, 4549fd130ecSDavid S. Miller }, 4559fd130ecSDavid S. Miller }, 4569bf4852dSDavid S. Miller } }; 4579bf4852dSDavid S. Miller 4589bf4852dSDavid S. Miller static bool __init sparc64_has_aes_opcode(void) 4599bf4852dSDavid S. Miller { 4609bf4852dSDavid S. Miller unsigned long cfr; 4619bf4852dSDavid S. Miller 4629bf4852dSDavid S. Miller if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 4639bf4852dSDavid S. Miller return false; 4649bf4852dSDavid S. Miller 4659bf4852dSDavid S. Miller __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 4669bf4852dSDavid S. Miller if (!(cfr & CFR_AES)) 4679bf4852dSDavid S. Miller return false; 4689bf4852dSDavid S. Miller 4699bf4852dSDavid S. Miller return true; 4709bf4852dSDavid S. Miller } 4719bf4852dSDavid S. Miller 4729bf4852dSDavid S. Miller static int __init aes_sparc64_mod_init(void) 4739bf4852dSDavid S. Miller { 4749bf4852dSDavid S. Miller int i; 4759bf4852dSDavid S. Miller 4769bf4852dSDavid S. Miller for (i = 0; i < ARRAY_SIZE(algs); i++) 4779bf4852dSDavid S. Miller INIT_LIST_HEAD(&algs[i].cra_list); 4789bf4852dSDavid S. Miller 4799bf4852dSDavid S. Miller if (sparc64_has_aes_opcode()) { 4809bf4852dSDavid S. Miller pr_info("Using sparc64 aes opcodes optimized AES implementation\n"); 4819bf4852dSDavid S. Miller return crypto_register_algs(algs, ARRAY_SIZE(algs)); 4829bf4852dSDavid S. Miller } 4839bf4852dSDavid S. Miller pr_info("sparc64 aes opcodes not available.\n"); 4849bf4852dSDavid S. Miller return -ENODEV; 4859bf4852dSDavid S. Miller } 4869bf4852dSDavid S. Miller 4879bf4852dSDavid S. Miller static void __exit aes_sparc64_mod_fini(void) 4889bf4852dSDavid S. Miller { 4899bf4852dSDavid S. Miller crypto_unregister_algs(algs, ARRAY_SIZE(algs)); 4909bf4852dSDavid S. Miller } 4919bf4852dSDavid S. Miller 4929bf4852dSDavid S. Miller module_init(aes_sparc64_mod_init); 4939bf4852dSDavid S. Miller module_exit(aes_sparc64_mod_fini); 4949bf4852dSDavid S. Miller 4959bf4852dSDavid S. Miller MODULE_LICENSE("GPL"); 4969bf4852dSDavid S. Miller MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated"); 4979bf4852dSDavid S. Miller 4989bf4852dSDavid S. Miller MODULE_ALIAS("aes"); 499226f7ceaSDavid S. Miller 500226f7ceaSDavid S. Miller #include "crop_devid.c" 501