1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * MIPS accelerated ChaCha and XChaCha stream ciphers, 4 * including ChaCha20 (RFC7539) 5 * 6 * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> 7 */ 8 9 #include <asm/byteorder.h> 10 #include <crypto/algapi.h> 11 #include <crypto/internal/chacha.h> 12 #include <crypto/internal/skcipher.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 16 asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, 17 unsigned int bytes, int nrounds); 18 EXPORT_SYMBOL(chacha_crypt_arch); 19 20 asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds); 21 EXPORT_SYMBOL(hchacha_block_arch); 22 23 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) 24 { 25 chacha_init_generic(state, key, iv); 26 } 27 EXPORT_SYMBOL(chacha_init_arch); 28 29 static int chacha_mips_stream_xor(struct skcipher_request *req, 30 const struct chacha_ctx *ctx, const u8 *iv) 31 { 32 struct skcipher_walk walk; 33 u32 state[16]; 34 int err; 35 36 err = skcipher_walk_virt(&walk, req, false); 37 38 chacha_init_generic(state, ctx->key, iv); 39 40 while (walk.nbytes > 0) { 41 unsigned int nbytes = walk.nbytes; 42 43 if (nbytes < walk.total) 44 nbytes = round_down(nbytes, walk.stride); 45 46 chacha_crypt(state, walk.dst.virt.addr, walk.src.virt.addr, 47 nbytes, ctx->nrounds); 48 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 49 } 50 51 return err; 52 } 53 54 static int chacha_mips(struct skcipher_request *req) 55 { 56 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 57 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); 58 59 return chacha_mips_stream_xor(req, ctx, req->iv); 60 } 61 62 static int xchacha_mips(struct skcipher_request *req) 63 { 64 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 65 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); 66 struct chacha_ctx subctx; 67 u32 state[16]; 68 u8 real_iv[16]; 69 70 chacha_init_generic(state, ctx->key, req->iv); 71 72 hchacha_block(state, subctx.key, ctx->nrounds); 73 subctx.nrounds = ctx->nrounds; 74 75 memcpy(&real_iv[0], req->iv + 24, 8); 76 memcpy(&real_iv[8], req->iv + 16, 8); 77 return chacha_mips_stream_xor(req, &subctx, real_iv); 78 } 79 80 static struct skcipher_alg algs[] = { 81 { 82 .base.cra_name = "chacha20", 83 .base.cra_driver_name = "chacha20-mips", 84 .base.cra_priority = 200, 85 .base.cra_blocksize = 1, 86 .base.cra_ctxsize = sizeof(struct chacha_ctx), 87 .base.cra_module = THIS_MODULE, 88 89 .min_keysize = CHACHA_KEY_SIZE, 90 .max_keysize = CHACHA_KEY_SIZE, 91 .ivsize = CHACHA_IV_SIZE, 92 .chunksize = CHACHA_BLOCK_SIZE, 93 .setkey = chacha20_setkey, 94 .encrypt = chacha_mips, 95 .decrypt = chacha_mips, 96 }, { 97 .base.cra_name = "xchacha20", 98 .base.cra_driver_name = "xchacha20-mips", 99 .base.cra_priority = 200, 100 .base.cra_blocksize = 1, 101 .base.cra_ctxsize = sizeof(struct chacha_ctx), 102 .base.cra_module = THIS_MODULE, 103 104 .min_keysize = CHACHA_KEY_SIZE, 105 .max_keysize = CHACHA_KEY_SIZE, 106 .ivsize = XCHACHA_IV_SIZE, 107 .chunksize = CHACHA_BLOCK_SIZE, 108 .setkey = chacha20_setkey, 109 .encrypt = xchacha_mips, 110 .decrypt = xchacha_mips, 111 }, { 112 .base.cra_name = "xchacha12", 113 .base.cra_driver_name = "xchacha12-mips", 114 .base.cra_priority = 200, 115 .base.cra_blocksize = 1, 116 .base.cra_ctxsize = sizeof(struct chacha_ctx), 117 .base.cra_module = THIS_MODULE, 118 119 .min_keysize = CHACHA_KEY_SIZE, 120 .max_keysize = CHACHA_KEY_SIZE, 121 .ivsize = XCHACHA_IV_SIZE, 122 .chunksize = CHACHA_BLOCK_SIZE, 123 .setkey = chacha12_setkey, 124 .encrypt = xchacha_mips, 125 .decrypt = xchacha_mips, 126 } 127 }; 128 129 static int __init chacha_simd_mod_init(void) 130 { 131 return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? 132 crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; 133 } 134 135 static void __exit chacha_simd_mod_fini(void) 136 { 137 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) 138 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); 139 } 140 141 module_init(chacha_simd_mod_init); 142 module_exit(chacha_simd_mod_fini); 143 144 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)"); 145 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 146 MODULE_LICENSE("GPL v2"); 147 MODULE_ALIAS_CRYPTO("chacha20"); 148 MODULE_ALIAS_CRYPTO("chacha20-mips"); 149 MODULE_ALIAS_CRYPTO("xchacha20"); 150 MODULE_ALIAS_CRYPTO("xchacha20-mips"); 151 MODULE_ALIAS_CRYPTO("xchacha12"); 152 MODULE_ALIAS_CRYPTO("xchacha12-mips"); 153