1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21abee99eSArd Biesheuvel /* 31abee99eSArd Biesheuvel * Bit sliced AES using NEON instructions 41abee99eSArd Biesheuvel * 5ec808bbeSArd Biesheuvel * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 61abee99eSArd Biesheuvel */ 71abee99eSArd Biesheuvel 81abee99eSArd Biesheuvel #include <asm/neon.h> 9ec808bbeSArd Biesheuvel #include <asm/simd.h> 101abee99eSArd Biesheuvel #include <crypto/aes.h> 11ff6f4115SArd Biesheuvel #include <crypto/ctr.h> 121abee99eSArd Biesheuvel #include <crypto/internal/simd.h> 131abee99eSArd Biesheuvel #include <crypto/internal/skcipher.h> 1467cfa5d3SArd Biesheuvel #include <crypto/scatterwalk.h> 151abee99eSArd Biesheuvel #include <crypto/xts.h> 161abee99eSArd Biesheuvel #include <linux/module.h> 171abee99eSArd Biesheuvel 181abee99eSArd Biesheuvel MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 191abee99eSArd Biesheuvel MODULE_LICENSE("GPL v2"); 201abee99eSArd Biesheuvel 211abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("ecb(aes)"); 221abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("cbc(aes)"); 231abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("ctr(aes)"); 241abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("xts(aes)"); 251abee99eSArd Biesheuvel 261abee99eSArd Biesheuvel asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds); 271abee99eSArd Biesheuvel 281abee99eSArd Biesheuvel asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], 291abee99eSArd Biesheuvel int rounds, int blocks); 301abee99eSArd Biesheuvel asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], 311abee99eSArd Biesheuvel int rounds, int blocks); 321abee99eSArd Biesheuvel 331abee99eSArd Biesheuvel asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], 341abee99eSArd Biesheuvel int rounds, int blocks, u8 iv[]); 351abee99eSArd Biesheuvel 361abee99eSArd Biesheuvel asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], 3788a3f582SArd Biesheuvel int rounds, int blocks, u8 iv[], u8 final[]); 381abee99eSArd Biesheuvel 391abee99eSArd Biesheuvel asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], 401abee99eSArd Biesheuvel int rounds, int blocks, u8 iv[]); 411abee99eSArd Biesheuvel asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], 421abee99eSArd Biesheuvel int rounds, int blocks, u8 iv[]); 431abee99eSArd Biesheuvel 4412fcd923SArd Biesheuvel /* borrowed from aes-neon-blk.ko */ 4512fcd923SArd Biesheuvel asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 4668338174SArd Biesheuvel int rounds, int blocks); 4712fcd923SArd Biesheuvel asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 4868338174SArd Biesheuvel int rounds, int blocks, u8 iv[]); 4967cfa5d3SArd Biesheuvel asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], 5067cfa5d3SArd Biesheuvel u32 const rk1[], int rounds, int bytes, 5167cfa5d3SArd Biesheuvel u32 const rk2[], u8 iv[], int first); 5267cfa5d3SArd Biesheuvel asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], 5367cfa5d3SArd Biesheuvel u32 const rk1[], int rounds, int bytes, 5467cfa5d3SArd Biesheuvel u32 const rk2[], u8 iv[], int first); 551abee99eSArd Biesheuvel 561abee99eSArd Biesheuvel struct aesbs_ctx { 571abee99eSArd Biesheuvel u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32]; 581abee99eSArd Biesheuvel int rounds; 591abee99eSArd Biesheuvel } __aligned(AES_BLOCK_SIZE); 601abee99eSArd Biesheuvel 611abee99eSArd Biesheuvel struct aesbs_cbc_ctx { 621abee99eSArd Biesheuvel struct aesbs_ctx key; 631abee99eSArd Biesheuvel u32 enc[AES_MAX_KEYLENGTH_U32]; 641abee99eSArd Biesheuvel }; 651abee99eSArd Biesheuvel 661abee99eSArd Biesheuvel struct aesbs_xts_ctx { 671abee99eSArd Biesheuvel struct aesbs_ctx key; 681abee99eSArd Biesheuvel u32 twkey[AES_MAX_KEYLENGTH_U32]; 6967cfa5d3SArd Biesheuvel struct crypto_aes_ctx cts; 701abee99eSArd Biesheuvel }; 711abee99eSArd Biesheuvel 721abee99eSArd Biesheuvel static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 731abee99eSArd Biesheuvel unsigned int key_len) 741abee99eSArd Biesheuvel { 751abee99eSArd Biesheuvel struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 761abee99eSArd Biesheuvel struct crypto_aes_ctx rk; 771abee99eSArd Biesheuvel int err; 781abee99eSArd Biesheuvel 79f68df543SArd Biesheuvel err = aes_expandkey(&rk, in_key, key_len); 801abee99eSArd Biesheuvel if (err) 811abee99eSArd Biesheuvel return err; 821abee99eSArd Biesheuvel 831abee99eSArd Biesheuvel ctx->rounds = 6 + key_len / 4; 841abee99eSArd Biesheuvel 851abee99eSArd Biesheuvel kernel_neon_begin(); 861abee99eSArd Biesheuvel aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds); 871abee99eSArd Biesheuvel kernel_neon_end(); 881abee99eSArd Biesheuvel 891abee99eSArd Biesheuvel return 0; 901abee99eSArd Biesheuvel } 911abee99eSArd Biesheuvel 921abee99eSArd Biesheuvel static int __ecb_crypt(struct skcipher_request *req, 931abee99eSArd Biesheuvel void (*fn)(u8 out[], u8 const in[], u8 const rk[], 941abee99eSArd Biesheuvel int rounds, int blocks)) 951abee99eSArd Biesheuvel { 961abee99eSArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 971abee99eSArd Biesheuvel struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 981abee99eSArd Biesheuvel struct skcipher_walk walk; 991abee99eSArd Biesheuvel int err; 1001abee99eSArd Biesheuvel 10178ad7b08SArd Biesheuvel err = skcipher_walk_virt(&walk, req, false); 1021abee99eSArd Biesheuvel 1031abee99eSArd Biesheuvel while (walk.nbytes >= AES_BLOCK_SIZE) { 1041abee99eSArd Biesheuvel unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 1051abee99eSArd Biesheuvel 1061abee99eSArd Biesheuvel if (walk.nbytes < walk.total) 1071abee99eSArd Biesheuvel blocks = round_down(blocks, 1081abee99eSArd Biesheuvel walk.stride / AES_BLOCK_SIZE); 1091abee99eSArd Biesheuvel 11078ad7b08SArd Biesheuvel kernel_neon_begin(); 1111abee99eSArd Biesheuvel fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk, 1121abee99eSArd Biesheuvel ctx->rounds, blocks); 11378ad7b08SArd Biesheuvel kernel_neon_end(); 1141abee99eSArd Biesheuvel err = skcipher_walk_done(&walk, 1151abee99eSArd Biesheuvel walk.nbytes - blocks * AES_BLOCK_SIZE); 1161abee99eSArd Biesheuvel } 1171abee99eSArd Biesheuvel 1181abee99eSArd Biesheuvel return err; 1191abee99eSArd Biesheuvel } 1201abee99eSArd Biesheuvel 1211abee99eSArd Biesheuvel static int ecb_encrypt(struct skcipher_request *req) 1221abee99eSArd Biesheuvel { 1231abee99eSArd Biesheuvel return __ecb_crypt(req, aesbs_ecb_encrypt); 1241abee99eSArd Biesheuvel } 1251abee99eSArd Biesheuvel 1261abee99eSArd Biesheuvel static int ecb_decrypt(struct skcipher_request *req) 1271abee99eSArd Biesheuvel { 1281abee99eSArd Biesheuvel return __ecb_crypt(req, aesbs_ecb_decrypt); 1291abee99eSArd Biesheuvel } 1301abee99eSArd Biesheuvel 1311abee99eSArd Biesheuvel static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 1321abee99eSArd Biesheuvel unsigned int key_len) 1331abee99eSArd Biesheuvel { 1341abee99eSArd Biesheuvel struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 1351abee99eSArd Biesheuvel struct crypto_aes_ctx rk; 1361abee99eSArd Biesheuvel int err; 1371abee99eSArd Biesheuvel 138f68df543SArd Biesheuvel err = aes_expandkey(&rk, in_key, key_len); 1391abee99eSArd Biesheuvel if (err) 1401abee99eSArd Biesheuvel return err; 1411abee99eSArd Biesheuvel 1421abee99eSArd Biesheuvel ctx->key.rounds = 6 + key_len / 4; 1431abee99eSArd Biesheuvel 1441abee99eSArd Biesheuvel memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc)); 1451abee99eSArd Biesheuvel 1461abee99eSArd Biesheuvel kernel_neon_begin(); 1471abee99eSArd Biesheuvel aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds); 1481abee99eSArd Biesheuvel kernel_neon_end(); 14982ff493eSTorsten Duwe memzero_explicit(&rk, sizeof(rk)); 1501abee99eSArd Biesheuvel 1511abee99eSArd Biesheuvel return 0; 1521abee99eSArd Biesheuvel } 1531abee99eSArd Biesheuvel 1541abee99eSArd Biesheuvel static int cbc_encrypt(struct skcipher_request *req) 1551abee99eSArd Biesheuvel { 15612fcd923SArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 15712fcd923SArd Biesheuvel struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 15812fcd923SArd Biesheuvel struct skcipher_walk walk; 15968338174SArd Biesheuvel int err; 16012fcd923SArd Biesheuvel 16178ad7b08SArd Biesheuvel err = skcipher_walk_virt(&walk, req, false); 16212fcd923SArd Biesheuvel 16312fcd923SArd Biesheuvel while (walk.nbytes >= AES_BLOCK_SIZE) { 16412fcd923SArd Biesheuvel unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 16512fcd923SArd Biesheuvel 16612fcd923SArd Biesheuvel /* fall back to the non-bitsliced NEON implementation */ 16778ad7b08SArd Biesheuvel kernel_neon_begin(); 16812fcd923SArd Biesheuvel neon_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 16968338174SArd Biesheuvel ctx->enc, ctx->key.rounds, blocks, 17068338174SArd Biesheuvel walk.iv); 17178ad7b08SArd Biesheuvel kernel_neon_end(); 17212fcd923SArd Biesheuvel err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 17312fcd923SArd Biesheuvel } 17412fcd923SArd Biesheuvel return err; 1751abee99eSArd Biesheuvel } 1761abee99eSArd Biesheuvel 1771abee99eSArd Biesheuvel static int cbc_decrypt(struct skcipher_request *req) 1781abee99eSArd Biesheuvel { 1791abee99eSArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1801abee99eSArd Biesheuvel struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 1811abee99eSArd Biesheuvel struct skcipher_walk walk; 1821abee99eSArd Biesheuvel int err; 1831abee99eSArd Biesheuvel 18478ad7b08SArd Biesheuvel err = skcipher_walk_virt(&walk, req, false); 1851abee99eSArd Biesheuvel 1861abee99eSArd Biesheuvel while (walk.nbytes >= AES_BLOCK_SIZE) { 1871abee99eSArd Biesheuvel unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 1881abee99eSArd Biesheuvel 1891abee99eSArd Biesheuvel if (walk.nbytes < walk.total) 1901abee99eSArd Biesheuvel blocks = round_down(blocks, 1911abee99eSArd Biesheuvel walk.stride / AES_BLOCK_SIZE); 1921abee99eSArd Biesheuvel 19378ad7b08SArd Biesheuvel kernel_neon_begin(); 1941abee99eSArd Biesheuvel aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 1951abee99eSArd Biesheuvel ctx->key.rk, ctx->key.rounds, blocks, 1961abee99eSArd Biesheuvel walk.iv); 19778ad7b08SArd Biesheuvel kernel_neon_end(); 1981abee99eSArd Biesheuvel err = skcipher_walk_done(&walk, 1991abee99eSArd Biesheuvel walk.nbytes - blocks * AES_BLOCK_SIZE); 2001abee99eSArd Biesheuvel } 2011abee99eSArd Biesheuvel 2021abee99eSArd Biesheuvel return err; 2031abee99eSArd Biesheuvel } 2041abee99eSArd Biesheuvel 2051abee99eSArd Biesheuvel static int ctr_encrypt(struct skcipher_request *req) 2061abee99eSArd Biesheuvel { 2071abee99eSArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 2081abee99eSArd Biesheuvel struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 2091abee99eSArd Biesheuvel struct skcipher_walk walk; 21088a3f582SArd Biesheuvel u8 buf[AES_BLOCK_SIZE]; 2111abee99eSArd Biesheuvel int err; 2121abee99eSArd Biesheuvel 21378ad7b08SArd Biesheuvel err = skcipher_walk_virt(&walk, req, false); 2141abee99eSArd Biesheuvel 2151abee99eSArd Biesheuvel while (walk.nbytes > 0) { 2161abee99eSArd Biesheuvel unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 21788a3f582SArd Biesheuvel u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL; 2181abee99eSArd Biesheuvel 2191abee99eSArd Biesheuvel if (walk.nbytes < walk.total) { 2201abee99eSArd Biesheuvel blocks = round_down(blocks, 2211abee99eSArd Biesheuvel walk.stride / AES_BLOCK_SIZE); 22288a3f582SArd Biesheuvel final = NULL; 2231abee99eSArd Biesheuvel } 2241abee99eSArd Biesheuvel 22578ad7b08SArd Biesheuvel kernel_neon_begin(); 2261abee99eSArd Biesheuvel aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 2271abee99eSArd Biesheuvel ctx->rk, ctx->rounds, blocks, walk.iv, final); 22878ad7b08SArd Biesheuvel kernel_neon_end(); 2291abee99eSArd Biesheuvel 2301abee99eSArd Biesheuvel if (final) { 2311abee99eSArd Biesheuvel u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; 2321abee99eSArd Biesheuvel u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; 2331abee99eSArd Biesheuvel 23445fe93dfSArd Biesheuvel crypto_xor_cpy(dst, src, final, 23545fe93dfSArd Biesheuvel walk.total % AES_BLOCK_SIZE); 2361abee99eSArd Biesheuvel 2371abee99eSArd Biesheuvel err = skcipher_walk_done(&walk, 0); 2381abee99eSArd Biesheuvel break; 2391abee99eSArd Biesheuvel } 2401abee99eSArd Biesheuvel err = skcipher_walk_done(&walk, 2411abee99eSArd Biesheuvel walk.nbytes - blocks * AES_BLOCK_SIZE); 2421abee99eSArd Biesheuvel } 2431abee99eSArd Biesheuvel return err; 2441abee99eSArd Biesheuvel } 2451abee99eSArd Biesheuvel 2461abee99eSArd Biesheuvel static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 2471abee99eSArd Biesheuvel unsigned int key_len) 2481abee99eSArd Biesheuvel { 2491abee99eSArd Biesheuvel struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 2501abee99eSArd Biesheuvel struct crypto_aes_ctx rk; 2511abee99eSArd Biesheuvel int err; 2521abee99eSArd Biesheuvel 2531abee99eSArd Biesheuvel err = xts_verify_key(tfm, in_key, key_len); 2541abee99eSArd Biesheuvel if (err) 2551abee99eSArd Biesheuvel return err; 2561abee99eSArd Biesheuvel 2571abee99eSArd Biesheuvel key_len /= 2; 25867cfa5d3SArd Biesheuvel err = aes_expandkey(&ctx->cts, in_key, key_len); 25967cfa5d3SArd Biesheuvel if (err) 26067cfa5d3SArd Biesheuvel return err; 26167cfa5d3SArd Biesheuvel 262f68df543SArd Biesheuvel err = aes_expandkey(&rk, in_key + key_len, key_len); 2631abee99eSArd Biesheuvel if (err) 2641abee99eSArd Biesheuvel return err; 2651abee99eSArd Biesheuvel 2661abee99eSArd Biesheuvel memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey)); 2671abee99eSArd Biesheuvel 2681abee99eSArd Biesheuvel return aesbs_setkey(tfm, in_key, key_len); 2691abee99eSArd Biesheuvel } 2701abee99eSArd Biesheuvel 27167cfa5d3SArd Biesheuvel static int __xts_crypt(struct skcipher_request *req, bool encrypt, 2721abee99eSArd Biesheuvel void (*fn)(u8 out[], u8 const in[], u8 const rk[], 2731abee99eSArd Biesheuvel int rounds, int blocks, u8 iv[])) 2741abee99eSArd Biesheuvel { 2751abee99eSArd Biesheuvel struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 2761abee99eSArd Biesheuvel struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 27767cfa5d3SArd Biesheuvel int tail = req->cryptlen % (8 * AES_BLOCK_SIZE); 27867cfa5d3SArd Biesheuvel struct scatterlist sg_src[2], sg_dst[2]; 27967cfa5d3SArd Biesheuvel struct skcipher_request subreq; 28067cfa5d3SArd Biesheuvel struct scatterlist *src, *dst; 2811abee99eSArd Biesheuvel struct skcipher_walk walk; 28267cfa5d3SArd Biesheuvel int nbytes, err; 28367cfa5d3SArd Biesheuvel int first = 1; 28467cfa5d3SArd Biesheuvel u8 *out, *in; 28567cfa5d3SArd Biesheuvel 28667cfa5d3SArd Biesheuvel if (req->cryptlen < AES_BLOCK_SIZE) 28767cfa5d3SArd Biesheuvel return -EINVAL; 28867cfa5d3SArd Biesheuvel 28967cfa5d3SArd Biesheuvel /* ensure that the cts tail is covered by a single step */ 29067cfa5d3SArd Biesheuvel if (unlikely(tail > 0 && tail < AES_BLOCK_SIZE)) { 29167cfa5d3SArd Biesheuvel int xts_blocks = DIV_ROUND_UP(req->cryptlen, 29267cfa5d3SArd Biesheuvel AES_BLOCK_SIZE) - 2; 29367cfa5d3SArd Biesheuvel 29467cfa5d3SArd Biesheuvel skcipher_request_set_tfm(&subreq, tfm); 29567cfa5d3SArd Biesheuvel skcipher_request_set_callback(&subreq, 29667cfa5d3SArd Biesheuvel skcipher_request_flags(req), 29767cfa5d3SArd Biesheuvel NULL, NULL); 29867cfa5d3SArd Biesheuvel skcipher_request_set_crypt(&subreq, req->src, req->dst, 29967cfa5d3SArd Biesheuvel xts_blocks * AES_BLOCK_SIZE, 30067cfa5d3SArd Biesheuvel req->iv); 30167cfa5d3SArd Biesheuvel req = &subreq; 30267cfa5d3SArd Biesheuvel } else { 30367cfa5d3SArd Biesheuvel tail = 0; 30467cfa5d3SArd Biesheuvel } 3051abee99eSArd Biesheuvel 30678ad7b08SArd Biesheuvel err = skcipher_walk_virt(&walk, req, false); 3074a8108b7SEric Biggers if (err) 3084a8108b7SEric Biggers return err; 3091abee99eSArd Biesheuvel 3101abee99eSArd Biesheuvel while (walk.nbytes >= AES_BLOCK_SIZE) { 3111abee99eSArd Biesheuvel unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 3121abee99eSArd Biesheuvel 31367cfa5d3SArd Biesheuvel if (walk.nbytes < walk.total || walk.nbytes % AES_BLOCK_SIZE) 3141abee99eSArd Biesheuvel blocks = round_down(blocks, 3151abee99eSArd Biesheuvel walk.stride / AES_BLOCK_SIZE); 3161abee99eSArd Biesheuvel 31767cfa5d3SArd Biesheuvel out = walk.dst.virt.addr; 31867cfa5d3SArd Biesheuvel in = walk.src.virt.addr; 31967cfa5d3SArd Biesheuvel nbytes = walk.nbytes; 32067cfa5d3SArd Biesheuvel 32178ad7b08SArd Biesheuvel kernel_neon_begin(); 32267cfa5d3SArd Biesheuvel if (likely(blocks > 6)) { /* plain NEON is faster otherwise */ 32367cfa5d3SArd Biesheuvel if (first) 32467cfa5d3SArd Biesheuvel neon_aes_ecb_encrypt(walk.iv, walk.iv, 32567cfa5d3SArd Biesheuvel ctx->twkey, 32667cfa5d3SArd Biesheuvel ctx->key.rounds, 1); 32767cfa5d3SArd Biesheuvel first = 0; 32867cfa5d3SArd Biesheuvel 32967cfa5d3SArd Biesheuvel fn(out, in, ctx->key.rk, ctx->key.rounds, blocks, 33067cfa5d3SArd Biesheuvel walk.iv); 33167cfa5d3SArd Biesheuvel 33267cfa5d3SArd Biesheuvel out += blocks * AES_BLOCK_SIZE; 33367cfa5d3SArd Biesheuvel in += blocks * AES_BLOCK_SIZE; 33467cfa5d3SArd Biesheuvel nbytes -= blocks * AES_BLOCK_SIZE; 3351abee99eSArd Biesheuvel } 33667cfa5d3SArd Biesheuvel 33767cfa5d3SArd Biesheuvel if (walk.nbytes == walk.total && nbytes > 0) 33867cfa5d3SArd Biesheuvel goto xts_tail; 33967cfa5d3SArd Biesheuvel 34067cfa5d3SArd Biesheuvel kernel_neon_end(); 3419b537997SYunfeng Ye err = skcipher_walk_done(&walk, nbytes); 34267cfa5d3SArd Biesheuvel } 34367cfa5d3SArd Biesheuvel 34467cfa5d3SArd Biesheuvel if (err || likely(!tail)) 3451abee99eSArd Biesheuvel return err; 34667cfa5d3SArd Biesheuvel 34767cfa5d3SArd Biesheuvel /* handle ciphertext stealing */ 34867cfa5d3SArd Biesheuvel dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 34967cfa5d3SArd Biesheuvel if (req->dst != req->src) 35067cfa5d3SArd Biesheuvel dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 35167cfa5d3SArd Biesheuvel 35267cfa5d3SArd Biesheuvel skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 35367cfa5d3SArd Biesheuvel req->iv); 35467cfa5d3SArd Biesheuvel 35567cfa5d3SArd Biesheuvel err = skcipher_walk_virt(&walk, req, false); 35667cfa5d3SArd Biesheuvel if (err) 35767cfa5d3SArd Biesheuvel return err; 35867cfa5d3SArd Biesheuvel 35967cfa5d3SArd Biesheuvel out = walk.dst.virt.addr; 36067cfa5d3SArd Biesheuvel in = walk.src.virt.addr; 36167cfa5d3SArd Biesheuvel nbytes = walk.nbytes; 36267cfa5d3SArd Biesheuvel 36367cfa5d3SArd Biesheuvel kernel_neon_begin(); 36467cfa5d3SArd Biesheuvel xts_tail: 36567cfa5d3SArd Biesheuvel if (encrypt) 36667cfa5d3SArd Biesheuvel neon_aes_xts_encrypt(out, in, ctx->cts.key_enc, ctx->key.rounds, 36767cfa5d3SArd Biesheuvel nbytes, ctx->twkey, walk.iv, first ?: 2); 36867cfa5d3SArd Biesheuvel else 36967cfa5d3SArd Biesheuvel neon_aes_xts_decrypt(out, in, ctx->cts.key_dec, ctx->key.rounds, 37067cfa5d3SArd Biesheuvel nbytes, ctx->twkey, walk.iv, first ?: 2); 37167cfa5d3SArd Biesheuvel kernel_neon_end(); 37267cfa5d3SArd Biesheuvel 37367cfa5d3SArd Biesheuvel return skcipher_walk_done(&walk, 0); 3741abee99eSArd Biesheuvel } 3751abee99eSArd Biesheuvel 3761abee99eSArd Biesheuvel static int xts_encrypt(struct skcipher_request *req) 3771abee99eSArd Biesheuvel { 37867cfa5d3SArd Biesheuvel return __xts_crypt(req, true, aesbs_xts_encrypt); 3791abee99eSArd Biesheuvel } 3801abee99eSArd Biesheuvel 3811abee99eSArd Biesheuvel static int xts_decrypt(struct skcipher_request *req) 3821abee99eSArd Biesheuvel { 38367cfa5d3SArd Biesheuvel return __xts_crypt(req, false, aesbs_xts_decrypt); 3841abee99eSArd Biesheuvel } 3851abee99eSArd Biesheuvel 3861abee99eSArd Biesheuvel static struct skcipher_alg aes_algs[] = { { 387*96c34e14SArd Biesheuvel .base.cra_name = "ecb(aes)", 388*96c34e14SArd Biesheuvel .base.cra_driver_name = "ecb-aes-neonbs", 3891abee99eSArd Biesheuvel .base.cra_priority = 250, 3901abee99eSArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE, 3911abee99eSArd Biesheuvel .base.cra_ctxsize = sizeof(struct aesbs_ctx), 3921abee99eSArd Biesheuvel .base.cra_module = THIS_MODULE, 3931abee99eSArd Biesheuvel 3941abee99eSArd Biesheuvel .min_keysize = AES_MIN_KEY_SIZE, 3951abee99eSArd Biesheuvel .max_keysize = AES_MAX_KEY_SIZE, 3961abee99eSArd Biesheuvel .walksize = 8 * AES_BLOCK_SIZE, 3971abee99eSArd Biesheuvel .setkey = aesbs_setkey, 3981abee99eSArd Biesheuvel .encrypt = ecb_encrypt, 3991abee99eSArd Biesheuvel .decrypt = ecb_decrypt, 4001abee99eSArd Biesheuvel }, { 401*96c34e14SArd Biesheuvel .base.cra_name = "cbc(aes)", 402*96c34e14SArd Biesheuvel .base.cra_driver_name = "cbc-aes-neonbs", 4031abee99eSArd Biesheuvel .base.cra_priority = 250, 4041abee99eSArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE, 4051abee99eSArd Biesheuvel .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), 4061abee99eSArd Biesheuvel .base.cra_module = THIS_MODULE, 4071abee99eSArd Biesheuvel 4081abee99eSArd Biesheuvel .min_keysize = AES_MIN_KEY_SIZE, 4091abee99eSArd Biesheuvel .max_keysize = AES_MAX_KEY_SIZE, 4101abee99eSArd Biesheuvel .walksize = 8 * AES_BLOCK_SIZE, 4111abee99eSArd Biesheuvel .ivsize = AES_BLOCK_SIZE, 4121abee99eSArd Biesheuvel .setkey = aesbs_cbc_setkey, 4131abee99eSArd Biesheuvel .encrypt = cbc_encrypt, 4141abee99eSArd Biesheuvel .decrypt = cbc_decrypt, 4151abee99eSArd Biesheuvel }, { 416*96c34e14SArd Biesheuvel .base.cra_name = "ctr(aes)", 417*96c34e14SArd Biesheuvel .base.cra_driver_name = "ctr-aes-neonbs", 4181abee99eSArd Biesheuvel .base.cra_priority = 250, 4191abee99eSArd Biesheuvel .base.cra_blocksize = 1, 4201abee99eSArd Biesheuvel .base.cra_ctxsize = sizeof(struct aesbs_ctx), 4211abee99eSArd Biesheuvel .base.cra_module = THIS_MODULE, 4221abee99eSArd Biesheuvel 4231abee99eSArd Biesheuvel .min_keysize = AES_MIN_KEY_SIZE, 4241abee99eSArd Biesheuvel .max_keysize = AES_MAX_KEY_SIZE, 4251abee99eSArd Biesheuvel .chunksize = AES_BLOCK_SIZE, 4261abee99eSArd Biesheuvel .walksize = 8 * AES_BLOCK_SIZE, 4271abee99eSArd Biesheuvel .ivsize = AES_BLOCK_SIZE, 4281abee99eSArd Biesheuvel .setkey = aesbs_setkey, 4291abee99eSArd Biesheuvel .encrypt = ctr_encrypt, 4301abee99eSArd Biesheuvel .decrypt = ctr_encrypt, 4311abee99eSArd Biesheuvel }, { 432*96c34e14SArd Biesheuvel .base.cra_name = "xts(aes)", 433*96c34e14SArd Biesheuvel .base.cra_driver_name = "xts-aes-neonbs", 4341abee99eSArd Biesheuvel .base.cra_priority = 250, 4351abee99eSArd Biesheuvel .base.cra_blocksize = AES_BLOCK_SIZE, 4361abee99eSArd Biesheuvel .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), 4371abee99eSArd Biesheuvel .base.cra_module = THIS_MODULE, 4381abee99eSArd Biesheuvel 4391abee99eSArd Biesheuvel .min_keysize = 2 * AES_MIN_KEY_SIZE, 4401abee99eSArd Biesheuvel .max_keysize = 2 * AES_MAX_KEY_SIZE, 4411abee99eSArd Biesheuvel .walksize = 8 * AES_BLOCK_SIZE, 4421abee99eSArd Biesheuvel .ivsize = AES_BLOCK_SIZE, 4431abee99eSArd Biesheuvel .setkey = aesbs_xts_setkey, 4441abee99eSArd Biesheuvel .encrypt = xts_encrypt, 4451abee99eSArd Biesheuvel .decrypt = xts_decrypt, 4461abee99eSArd Biesheuvel } }; 4471abee99eSArd Biesheuvel 4481abee99eSArd Biesheuvel static void aes_exit(void) 4491abee99eSArd Biesheuvel { 4501abee99eSArd Biesheuvel crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 4511abee99eSArd Biesheuvel } 4521abee99eSArd Biesheuvel 4531abee99eSArd Biesheuvel static int __init aes_init(void) 4541abee99eSArd Biesheuvel { 455aaba098fSAndrew Murray if (!cpu_have_named_feature(ASIMD)) 4561abee99eSArd Biesheuvel return -ENODEV; 4571abee99eSArd Biesheuvel 458*96c34e14SArd Biesheuvel return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 4591abee99eSArd Biesheuvel } 4601abee99eSArd Biesheuvel 4611abee99eSArd Biesheuvel module_init(aes_init); 4621abee99eSArd Biesheuvel module_exit(aes_exit); 463