120a884f5SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0+
2bf754ae8SJan Glauber /*
3bf754ae8SJan Glauber * Cryptographic API.
4bf754ae8SJan Glauber *
5bf754ae8SJan Glauber * s390 implementation of the AES Cipher Algorithm.
6bf754ae8SJan Glauber *
7bf754ae8SJan Glauber * s390 Version:
8bf7fa038SHarald Freudenberger * Copyright IBM Corp. 2005, 2017
9bf754ae8SJan Glauber * Author(s): Jan Glauber (jang@de.ibm.com)
10b0c3e75dSSebastian Siewior * Sebastian Siewior (sebastian@breakpoint.cc> SW-Fallback
11bf7fa038SHarald Freudenberger * Patrick Steuer <patrick.steuer@de.ibm.com>
12bf7fa038SHarald Freudenberger * Harald Freudenberger <freude@de.ibm.com>
13bf754ae8SJan Glauber *
14f8246af0SSebastian Siewior * Derived from "crypto/aes_generic.c"
15bf754ae8SJan Glauber */
16bf754ae8SJan Glauber
1739f09392SJan Glauber #define KMSG_COMPONENT "aes_s390"
1839f09392SJan Glauber #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
1939f09392SJan Glauber
2089e12654SSebastian Siewior #include <crypto/aes.h>
21a9e62fadSHerbert Xu #include <crypto/algapi.h>
22bf7fa038SHarald Freudenberger #include <crypto/ghash.h>
23bf7fa038SHarald Freudenberger #include <crypto/internal/aead.h>
240eb76ba2SArd Biesheuvel #include <crypto/internal/cipher.h>
2564e26807SHerbert Xu #include <crypto/internal/skcipher.h>
26bf7fa038SHarald Freudenberger #include <crypto/scatterwalk.h>
27b0c3e75dSSebastian Siewior #include <linux/err.h>
28bf754ae8SJan Glauber #include <linux/module.h>
29d05377c1SHendrik Brueckner #include <linux/cpufeature.h>
30bf754ae8SJan Glauber #include <linux/init.h>
311c2c7029SHarald Freudenberger #include <linux/mutex.h>
32a4f2779eSHarald Freudenberger #include <linux/fips.h>
33bf7fa038SHarald Freudenberger #include <linux/string.h>
3449abc0d2SStephan Mueller #include <crypto/xts.h>
35c7d4d259SMartin Schwidefsky #include <asm/cpacf.h>
36bf754ae8SJan Glauber
370200f3ecSGerald Schaefer static u8 *ctrblk;
381c2c7029SHarald Freudenberger static DEFINE_MUTEX(ctrblk_lock);
3969c0e360SMartin Schwidefsky
40bf7fa038SHarald Freudenberger static cpacf_mask_t km_functions, kmc_functions, kmctr_functions,
41bf7fa038SHarald Freudenberger kma_functions;
42bf754ae8SJan Glauber
43bf754ae8SJan Glauber struct s390_aes_ctx {
44bf754ae8SJan Glauber u8 key[AES_MAX_KEY_SIZE];
45bf754ae8SJan Glauber int key_len;
46edc63a37SMartin Schwidefsky unsigned long fc;
47b0c3e75dSSebastian Siewior union {
487988fb2cSEric Biggers struct crypto_skcipher *skcipher;
49b0c3e75dSSebastian Siewior struct crypto_cipher *cip;
50b0c3e75dSSebastian Siewior } fallback;
51bf754ae8SJan Glauber };
52bf754ae8SJan Glauber
5399d97222SGerald Schaefer struct s390_xts_ctx {
5499d97222SGerald Schaefer u8 key[32];
559dda2769SGerald Schaefer u8 pcc_key[32];
5699d97222SGerald Schaefer int key_len;
57edc63a37SMartin Schwidefsky unsigned long fc;
587988fb2cSEric Biggers struct crypto_skcipher *fallback;
5999d97222SGerald Schaefer };
6099d97222SGerald Schaefer
61bf7fa038SHarald Freudenberger struct gcm_sg_walk {
62bf7fa038SHarald Freudenberger struct scatter_walk walk;
63bf7fa038SHarald Freudenberger unsigned int walk_bytes;
64bf7fa038SHarald Freudenberger u8 *walk_ptr;
65bf7fa038SHarald Freudenberger unsigned int walk_bytes_remain;
66bf7fa038SHarald Freudenberger u8 buf[AES_BLOCK_SIZE];
67bf7fa038SHarald Freudenberger unsigned int buf_bytes;
68bf7fa038SHarald Freudenberger u8 *ptr;
69bf7fa038SHarald Freudenberger unsigned int nbytes;
70bf7fa038SHarald Freudenberger };
71bf7fa038SHarald Freudenberger
setkey_fallback_cip(struct crypto_tfm * tfm,const u8 * in_key,unsigned int key_len)72b0c3e75dSSebastian Siewior static int setkey_fallback_cip(struct crypto_tfm *tfm, const u8 *in_key,
73b0c3e75dSSebastian Siewior unsigned int key_len)
74b0c3e75dSSebastian Siewior {
75b0c3e75dSSebastian Siewior struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
76b0c3e75dSSebastian Siewior
77d7ac7690SRoel Kluin sctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
78d7ac7690SRoel Kluin sctx->fallback.cip->base.crt_flags |= (tfm->crt_flags &
79b0c3e75dSSebastian Siewior CRYPTO_TFM_REQ_MASK);
80b0c3e75dSSebastian Siewior
81af5034e8SEric Biggers return crypto_cipher_setkey(sctx->fallback.cip, in_key, key_len);
82b0c3e75dSSebastian Siewior }
83b0c3e75dSSebastian Siewior
aes_set_key(struct crypto_tfm * tfm,const u8 * in_key,unsigned int key_len)846c2bb98bSHerbert Xu static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
85560c06aeSHerbert Xu unsigned int key_len)
86bf754ae8SJan Glauber {
876c2bb98bSHerbert Xu struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
8869c0e360SMartin Schwidefsky unsigned long fc;
89bf754ae8SJan Glauber
9069c0e360SMartin Schwidefsky /* Pick the correct function code based on the key length */
9169c0e360SMartin Schwidefsky fc = (key_len == 16) ? CPACF_KM_AES_128 :
9269c0e360SMartin Schwidefsky (key_len == 24) ? CPACF_KM_AES_192 :
9369c0e360SMartin Schwidefsky (key_len == 32) ? CPACF_KM_AES_256 : 0;
9469c0e360SMartin Schwidefsky
9569c0e360SMartin Schwidefsky /* Check if the function code is available */
9669c0e360SMartin Schwidefsky sctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
9769c0e360SMartin Schwidefsky if (!sctx->fc)
9869c0e360SMartin Schwidefsky return setkey_fallback_cip(tfm, in_key, key_len);
99bf754ae8SJan Glauber
100bf754ae8SJan Glauber sctx->key_len = key_len;
101bf754ae8SJan Glauber memcpy(sctx->key, in_key, key_len);
102bf754ae8SJan Glauber return 0;
103b0c3e75dSSebastian Siewior }
104b0c3e75dSSebastian Siewior
crypto_aes_encrypt(struct crypto_tfm * tfm,u8 * out,const u8 * in)105931c940fSArd Biesheuvel static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
106bf754ae8SJan Glauber {
107e6a67ad0SChen Gang struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
108bf754ae8SJan Glauber
10969c0e360SMartin Schwidefsky if (unlikely(!sctx->fc)) {
110b0c3e75dSSebastian Siewior crypto_cipher_encrypt_one(sctx->fallback.cip, out, in);
111b0c3e75dSSebastian Siewior return;
112b0c3e75dSSebastian Siewior }
11369c0e360SMartin Schwidefsky cpacf_km(sctx->fc, &sctx->key, out, in, AES_BLOCK_SIZE);
114bf754ae8SJan Glauber }
115bf754ae8SJan Glauber
crypto_aes_decrypt(struct crypto_tfm * tfm,u8 * out,const u8 * in)116931c940fSArd Biesheuvel static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
117bf754ae8SJan Glauber {
118e6a67ad0SChen Gang struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
119bf754ae8SJan Glauber
12069c0e360SMartin Schwidefsky if (unlikely(!sctx->fc)) {
121b0c3e75dSSebastian Siewior crypto_cipher_decrypt_one(sctx->fallback.cip, out, in);
122b0c3e75dSSebastian Siewior return;
123b0c3e75dSSebastian Siewior }
12469c0e360SMartin Schwidefsky cpacf_km(sctx->fc | CPACF_DECRYPT,
125edc63a37SMartin Schwidefsky &sctx->key, out, in, AES_BLOCK_SIZE);
126bf754ae8SJan Glauber }
127bf754ae8SJan Glauber
fallback_init_cip(struct crypto_tfm * tfm)128b0c3e75dSSebastian Siewior static int fallback_init_cip(struct crypto_tfm *tfm)
129b0c3e75dSSebastian Siewior {
130b0c3e75dSSebastian Siewior const char *name = tfm->__crt_alg->cra_name;
131b0c3e75dSSebastian Siewior struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
132b0c3e75dSSebastian Siewior
133b0c3e75dSSebastian Siewior sctx->fallback.cip = crypto_alloc_cipher(name, 0,
1341ad0f160SEric Biggers CRYPTO_ALG_NEED_FALLBACK);
135b0c3e75dSSebastian Siewior
136b0c3e75dSSebastian Siewior if (IS_ERR(sctx->fallback.cip)) {
13739f09392SJan Glauber pr_err("Allocating AES fallback algorithm %s failed\n",
13839f09392SJan Glauber name);
139b59cdcb3SRoel Kluin return PTR_ERR(sctx->fallback.cip);
140b0c3e75dSSebastian Siewior }
141b0c3e75dSSebastian Siewior
142b0c3e75dSSebastian Siewior return 0;
143b0c3e75dSSebastian Siewior }
144b0c3e75dSSebastian Siewior
fallback_exit_cip(struct crypto_tfm * tfm)145b0c3e75dSSebastian Siewior static void fallback_exit_cip(struct crypto_tfm *tfm)
146b0c3e75dSSebastian Siewior {
147b0c3e75dSSebastian Siewior struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
148b0c3e75dSSebastian Siewior
149b0c3e75dSSebastian Siewior crypto_free_cipher(sctx->fallback.cip);
150b0c3e75dSSebastian Siewior sctx->fallback.cip = NULL;
151b0c3e75dSSebastian Siewior }
152bf754ae8SJan Glauber
153bf754ae8SJan Glauber static struct crypto_alg aes_alg = {
154bf754ae8SJan Glauber .cra_name = "aes",
15565b75c36SHerbert Xu .cra_driver_name = "aes-s390",
156c7d4d259SMartin Schwidefsky .cra_priority = 300,
157f67d1369SJan Glauber .cra_flags = CRYPTO_ALG_TYPE_CIPHER |
158f67d1369SJan Glauber CRYPTO_ALG_NEED_FALLBACK,
159bf754ae8SJan Glauber .cra_blocksize = AES_BLOCK_SIZE,
160bf754ae8SJan Glauber .cra_ctxsize = sizeof(struct s390_aes_ctx),
161bf754ae8SJan Glauber .cra_module = THIS_MODULE,
162b0c3e75dSSebastian Siewior .cra_init = fallback_init_cip,
163b0c3e75dSSebastian Siewior .cra_exit = fallback_exit_cip,
164bf754ae8SJan Glauber .cra_u = {
165bf754ae8SJan Glauber .cipher = {
166bf754ae8SJan Glauber .cia_min_keysize = AES_MIN_KEY_SIZE,
167bf754ae8SJan Glauber .cia_max_keysize = AES_MAX_KEY_SIZE,
168bf754ae8SJan Glauber .cia_setkey = aes_set_key,
169931c940fSArd Biesheuvel .cia_encrypt = crypto_aes_encrypt,
170931c940fSArd Biesheuvel .cia_decrypt = crypto_aes_decrypt,
171bf754ae8SJan Glauber }
172bf754ae8SJan Glauber }
173bf754ae8SJan Glauber };
174bf754ae8SJan Glauber
setkey_fallback_skcipher(struct crypto_skcipher * tfm,const u8 * key,unsigned int len)1757988fb2cSEric Biggers static int setkey_fallback_skcipher(struct crypto_skcipher *tfm, const u8 *key,
176b0c3e75dSSebastian Siewior unsigned int len)
177b0c3e75dSSebastian Siewior {
1787988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
179b0c3e75dSSebastian Siewior
1807988fb2cSEric Biggers crypto_skcipher_clear_flags(sctx->fallback.skcipher,
181531fa5d6SKees Cook CRYPTO_TFM_REQ_MASK);
1827988fb2cSEric Biggers crypto_skcipher_set_flags(sctx->fallback.skcipher,
1837988fb2cSEric Biggers crypto_skcipher_get_flags(tfm) &
184b0c3e75dSSebastian Siewior CRYPTO_TFM_REQ_MASK);
185af5034e8SEric Biggers return crypto_skcipher_setkey(sctx->fallback.skcipher, key, len);
186b0c3e75dSSebastian Siewior }
187b0c3e75dSSebastian Siewior
fallback_skcipher_crypt(struct s390_aes_ctx * sctx,struct skcipher_request * req,unsigned long modifier)1887988fb2cSEric Biggers static int fallback_skcipher_crypt(struct s390_aes_ctx *sctx,
1897988fb2cSEric Biggers struct skcipher_request *req,
1907988fb2cSEric Biggers unsigned long modifier)
191b0c3e75dSSebastian Siewior {
1927988fb2cSEric Biggers struct skcipher_request *subreq = skcipher_request_ctx(req);
193b0c3e75dSSebastian Siewior
1947988fb2cSEric Biggers *subreq = *req;
1957988fb2cSEric Biggers skcipher_request_set_tfm(subreq, sctx->fallback.skcipher);
1967988fb2cSEric Biggers return (modifier & CPACF_DECRYPT) ?
1977988fb2cSEric Biggers crypto_skcipher_decrypt(subreq) :
1987988fb2cSEric Biggers crypto_skcipher_encrypt(subreq);
199b0c3e75dSSebastian Siewior }
200b0c3e75dSSebastian Siewior
ecb_aes_set_key(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)2017988fb2cSEric Biggers static int ecb_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
202a9e62fadSHerbert Xu unsigned int key_len)
203a9e62fadSHerbert Xu {
2047988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
20569c0e360SMartin Schwidefsky unsigned long fc;
206b0c3e75dSSebastian Siewior
20769c0e360SMartin Schwidefsky /* Pick the correct function code based on the key length */
20869c0e360SMartin Schwidefsky fc = (key_len == 16) ? CPACF_KM_AES_128 :
20969c0e360SMartin Schwidefsky (key_len == 24) ? CPACF_KM_AES_192 :
21069c0e360SMartin Schwidefsky (key_len == 32) ? CPACF_KM_AES_256 : 0;
21169c0e360SMartin Schwidefsky
21269c0e360SMartin Schwidefsky /* Check if the function code is available */
21369c0e360SMartin Schwidefsky sctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
21469c0e360SMartin Schwidefsky if (!sctx->fc)
2157988fb2cSEric Biggers return setkey_fallback_skcipher(tfm, in_key, key_len);
216a9e62fadSHerbert Xu
21769c0e360SMartin Schwidefsky sctx->key_len = key_len;
21869c0e360SMartin Schwidefsky memcpy(sctx->key, in_key, key_len);
21969c0e360SMartin Schwidefsky return 0;
220a9e62fadSHerbert Xu }
221a9e62fadSHerbert Xu
ecb_aes_crypt(struct skcipher_request * req,unsigned long modifier)2227988fb2cSEric Biggers static int ecb_aes_crypt(struct skcipher_request *req, unsigned long modifier)
223a9e62fadSHerbert Xu {
2247988fb2cSEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2257988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
2267988fb2cSEric Biggers struct skcipher_walk walk;
2277bac4f5bSMartin Schwidefsky unsigned int nbytes, n;
2287bac4f5bSMartin Schwidefsky int ret;
229a9e62fadSHerbert Xu
2307988fb2cSEric Biggers if (unlikely(!sctx->fc))
2317988fb2cSEric Biggers return fallback_skcipher_crypt(sctx, req, modifier);
2327988fb2cSEric Biggers
2337988fb2cSEric Biggers ret = skcipher_walk_virt(&walk, req, false);
2347988fb2cSEric Biggers while ((nbytes = walk.nbytes) != 0) {
235a9e62fadSHerbert Xu /* only use complete blocks */
2367bac4f5bSMartin Schwidefsky n = nbytes & ~(AES_BLOCK_SIZE - 1);
2377bac4f5bSMartin Schwidefsky cpacf_km(sctx->fc | modifier, sctx->key,
2387988fb2cSEric Biggers walk.dst.virt.addr, walk.src.virt.addr, n);
2397988fb2cSEric Biggers ret = skcipher_walk_done(&walk, nbytes - n);
240a9e62fadSHerbert Xu }
241a9e62fadSHerbert Xu return ret;
242a9e62fadSHerbert Xu }
243a9e62fadSHerbert Xu
ecb_aes_encrypt(struct skcipher_request * req)2447988fb2cSEric Biggers static int ecb_aes_encrypt(struct skcipher_request *req)
245a9e62fadSHerbert Xu {
2467988fb2cSEric Biggers return ecb_aes_crypt(req, 0);
247a9e62fadSHerbert Xu }
248a9e62fadSHerbert Xu
ecb_aes_decrypt(struct skcipher_request * req)2497988fb2cSEric Biggers static int ecb_aes_decrypt(struct skcipher_request *req)
250a9e62fadSHerbert Xu {
2517988fb2cSEric Biggers return ecb_aes_crypt(req, CPACF_DECRYPT);
252a9e62fadSHerbert Xu }
253a9e62fadSHerbert Xu
fallback_init_skcipher(struct crypto_skcipher * tfm)2547988fb2cSEric Biggers static int fallback_init_skcipher(struct crypto_skcipher *tfm)
255b0c3e75dSSebastian Siewior {
2567988fb2cSEric Biggers const char *name = crypto_tfm_alg_name(&tfm->base);
2577988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
258b0c3e75dSSebastian Siewior
2597988fb2cSEric Biggers sctx->fallback.skcipher = crypto_alloc_skcipher(name, 0,
2607988fb2cSEric Biggers CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC);
261b0c3e75dSSebastian Siewior
2627988fb2cSEric Biggers if (IS_ERR(sctx->fallback.skcipher)) {
26339f09392SJan Glauber pr_err("Allocating AES fallback algorithm %s failed\n",
26439f09392SJan Glauber name);
2657988fb2cSEric Biggers return PTR_ERR(sctx->fallback.skcipher);
266b0c3e75dSSebastian Siewior }
267b0c3e75dSSebastian Siewior
2687988fb2cSEric Biggers crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
2697988fb2cSEric Biggers crypto_skcipher_reqsize(sctx->fallback.skcipher));
270b0c3e75dSSebastian Siewior return 0;
271b0c3e75dSSebastian Siewior }
272b0c3e75dSSebastian Siewior
fallback_exit_skcipher(struct crypto_skcipher * tfm)2737988fb2cSEric Biggers static void fallback_exit_skcipher(struct crypto_skcipher *tfm)
274b0c3e75dSSebastian Siewior {
2757988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
276b0c3e75dSSebastian Siewior
2777988fb2cSEric Biggers crypto_free_skcipher(sctx->fallback.skcipher);
278b0c3e75dSSebastian Siewior }
279b0c3e75dSSebastian Siewior
2807988fb2cSEric Biggers static struct skcipher_alg ecb_aes_alg = {
2817988fb2cSEric Biggers .base.cra_name = "ecb(aes)",
2827988fb2cSEric Biggers .base.cra_driver_name = "ecb-aes-s390",
2837988fb2cSEric Biggers .base.cra_priority = 401, /* combo: aes + ecb + 1 */
2847988fb2cSEric Biggers .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
2857988fb2cSEric Biggers .base.cra_blocksize = AES_BLOCK_SIZE,
2867988fb2cSEric Biggers .base.cra_ctxsize = sizeof(struct s390_aes_ctx),
2877988fb2cSEric Biggers .base.cra_module = THIS_MODULE,
2887988fb2cSEric Biggers .init = fallback_init_skcipher,
2897988fb2cSEric Biggers .exit = fallback_exit_skcipher,
290a9e62fadSHerbert Xu .min_keysize = AES_MIN_KEY_SIZE,
291a9e62fadSHerbert Xu .max_keysize = AES_MAX_KEY_SIZE,
292a9e62fadSHerbert Xu .setkey = ecb_aes_set_key,
293a9e62fadSHerbert Xu .encrypt = ecb_aes_encrypt,
294a9e62fadSHerbert Xu .decrypt = ecb_aes_decrypt,
295a9e62fadSHerbert Xu };
296a9e62fadSHerbert Xu
cbc_aes_set_key(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)2977988fb2cSEric Biggers static int cbc_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
298a9e62fadSHerbert Xu unsigned int key_len)
299a9e62fadSHerbert Xu {
3007988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
30169c0e360SMartin Schwidefsky unsigned long fc;
302b0c3e75dSSebastian Siewior
30369c0e360SMartin Schwidefsky /* Pick the correct function code based on the key length */
30469c0e360SMartin Schwidefsky fc = (key_len == 16) ? CPACF_KMC_AES_128 :
30569c0e360SMartin Schwidefsky (key_len == 24) ? CPACF_KMC_AES_192 :
30669c0e360SMartin Schwidefsky (key_len == 32) ? CPACF_KMC_AES_256 : 0;
30769c0e360SMartin Schwidefsky
30869c0e360SMartin Schwidefsky /* Check if the function code is available */
30969c0e360SMartin Schwidefsky sctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0;
31069c0e360SMartin Schwidefsky if (!sctx->fc)
3117988fb2cSEric Biggers return setkey_fallback_skcipher(tfm, in_key, key_len);
312a9e62fadSHerbert Xu
31369c0e360SMartin Schwidefsky sctx->key_len = key_len;
31469c0e360SMartin Schwidefsky memcpy(sctx->key, in_key, key_len);
31569c0e360SMartin Schwidefsky return 0;
316a9e62fadSHerbert Xu }
317a9e62fadSHerbert Xu
cbc_aes_crypt(struct skcipher_request * req,unsigned long modifier)3187988fb2cSEric Biggers static int cbc_aes_crypt(struct skcipher_request *req, unsigned long modifier)
319a9e62fadSHerbert Xu {
3207988fb2cSEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3217988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
3227988fb2cSEric Biggers struct skcipher_walk walk;
3237bac4f5bSMartin Schwidefsky unsigned int nbytes, n;
3247bac4f5bSMartin Schwidefsky int ret;
325f262f0f5SHerbert Xu struct {
326f262f0f5SHerbert Xu u8 iv[AES_BLOCK_SIZE];
327f262f0f5SHerbert Xu u8 key[AES_MAX_KEY_SIZE];
328f262f0f5SHerbert Xu } param;
329a9e62fadSHerbert Xu
3307988fb2cSEric Biggers if (unlikely(!sctx->fc))
3317988fb2cSEric Biggers return fallback_skcipher_crypt(sctx, req, modifier);
3327988fb2cSEric Biggers
3337988fb2cSEric Biggers ret = skcipher_walk_virt(&walk, req, false);
3347988fb2cSEric Biggers if (ret)
3357988fb2cSEric Biggers return ret;
3367988fb2cSEric Biggers memcpy(param.iv, walk.iv, AES_BLOCK_SIZE);
337f262f0f5SHerbert Xu memcpy(param.key, sctx->key, sctx->key_len);
3387988fb2cSEric Biggers while ((nbytes = walk.nbytes) != 0) {
339a9e62fadSHerbert Xu /* only use complete blocks */
3407bac4f5bSMartin Schwidefsky n = nbytes & ~(AES_BLOCK_SIZE - 1);
3417bac4f5bSMartin Schwidefsky cpacf_kmc(sctx->fc | modifier, ¶m,
3427988fb2cSEric Biggers walk.dst.virt.addr, walk.src.virt.addr, n);
3437988fb2cSEric Biggers memcpy(walk.iv, param.iv, AES_BLOCK_SIZE);
3447988fb2cSEric Biggers ret = skcipher_walk_done(&walk, nbytes - n);
3457bac4f5bSMartin Schwidefsky }
3464a559cd1STorsten Duwe memzero_explicit(¶m, sizeof(param));
347a9e62fadSHerbert Xu return ret;
348a9e62fadSHerbert Xu }
349a9e62fadSHerbert Xu
cbc_aes_encrypt(struct skcipher_request * req)3507988fb2cSEric Biggers static int cbc_aes_encrypt(struct skcipher_request *req)
351a9e62fadSHerbert Xu {
3527988fb2cSEric Biggers return cbc_aes_crypt(req, 0);
353a9e62fadSHerbert Xu }
354a9e62fadSHerbert Xu
cbc_aes_decrypt(struct skcipher_request * req)3557988fb2cSEric Biggers static int cbc_aes_decrypt(struct skcipher_request *req)
356a9e62fadSHerbert Xu {
3577988fb2cSEric Biggers return cbc_aes_crypt(req, CPACF_DECRYPT);
358a9e62fadSHerbert Xu }
359a9e62fadSHerbert Xu
3607988fb2cSEric Biggers static struct skcipher_alg cbc_aes_alg = {
3617988fb2cSEric Biggers .base.cra_name = "cbc(aes)",
3627988fb2cSEric Biggers .base.cra_driver_name = "cbc-aes-s390",
3637988fb2cSEric Biggers .base.cra_priority = 402, /* ecb-aes-s390 + 1 */
3647988fb2cSEric Biggers .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
3657988fb2cSEric Biggers .base.cra_blocksize = AES_BLOCK_SIZE,
3667988fb2cSEric Biggers .base.cra_ctxsize = sizeof(struct s390_aes_ctx),
3677988fb2cSEric Biggers .base.cra_module = THIS_MODULE,
3687988fb2cSEric Biggers .init = fallback_init_skcipher,
3697988fb2cSEric Biggers .exit = fallback_exit_skcipher,
370a9e62fadSHerbert Xu .min_keysize = AES_MIN_KEY_SIZE,
371a9e62fadSHerbert Xu .max_keysize = AES_MAX_KEY_SIZE,
372a9e62fadSHerbert Xu .ivsize = AES_BLOCK_SIZE,
373a9e62fadSHerbert Xu .setkey = cbc_aes_set_key,
374a9e62fadSHerbert Xu .encrypt = cbc_aes_encrypt,
375a9e62fadSHerbert Xu .decrypt = cbc_aes_decrypt,
376a9e62fadSHerbert Xu };
377a9e62fadSHerbert Xu
xts_fallback_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int len)3787988fb2cSEric Biggers static int xts_fallback_setkey(struct crypto_skcipher *tfm, const u8 *key,
37999d97222SGerald Schaefer unsigned int len)
38099d97222SGerald Schaefer {
3817988fb2cSEric Biggers struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm);
38299d97222SGerald Schaefer
3837988fb2cSEric Biggers crypto_skcipher_clear_flags(xts_ctx->fallback, CRYPTO_TFM_REQ_MASK);
3847988fb2cSEric Biggers crypto_skcipher_set_flags(xts_ctx->fallback,
3857988fb2cSEric Biggers crypto_skcipher_get_flags(tfm) &
386531fa5d6SKees Cook CRYPTO_TFM_REQ_MASK);
387af5034e8SEric Biggers return crypto_skcipher_setkey(xts_ctx->fallback, key, len);
38899d97222SGerald Schaefer }
38999d97222SGerald Schaefer
xts_aes_set_key(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)3907988fb2cSEric Biggers static int xts_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
39199d97222SGerald Schaefer unsigned int key_len)
39299d97222SGerald Schaefer {
3937988fb2cSEric Biggers struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm);
39469c0e360SMartin Schwidefsky unsigned long fc;
39528856a9eSStephan Mueller int err;
39628856a9eSStephan Mueller
397ce68acbcSArd Biesheuvel err = xts_fallback_setkey(tfm, in_key, key_len);
39828856a9eSStephan Mueller if (err)
39928856a9eSStephan Mueller return err;
40099d97222SGerald Schaefer
40169c0e360SMartin Schwidefsky /* Pick the correct function code based on the key length */
40269c0e360SMartin Schwidefsky fc = (key_len == 32) ? CPACF_KM_XTS_128 :
40369c0e360SMartin Schwidefsky (key_len == 64) ? CPACF_KM_XTS_256 : 0;
40469c0e360SMartin Schwidefsky
40569c0e360SMartin Schwidefsky /* Check if the function code is available */
40669c0e360SMartin Schwidefsky xts_ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
40769c0e360SMartin Schwidefsky if (!xts_ctx->fc)
408ce68acbcSArd Biesheuvel return 0;
40969c0e360SMartin Schwidefsky
41069c0e360SMartin Schwidefsky /* Split the XTS key into the two subkeys */
41169c0e360SMartin Schwidefsky key_len = key_len / 2;
41299d97222SGerald Schaefer xts_ctx->key_len = key_len;
41369c0e360SMartin Schwidefsky memcpy(xts_ctx->key, in_key, key_len);
41469c0e360SMartin Schwidefsky memcpy(xts_ctx->pcc_key, in_key + key_len, key_len);
41599d97222SGerald Schaefer return 0;
41699d97222SGerald Schaefer }
41799d97222SGerald Schaefer
xts_aes_crypt(struct skcipher_request * req,unsigned long modifier)4187988fb2cSEric Biggers static int xts_aes_crypt(struct skcipher_request *req, unsigned long modifier)
41999d97222SGerald Schaefer {
4207988fb2cSEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
4217988fb2cSEric Biggers struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm);
4227988fb2cSEric Biggers struct skcipher_walk walk;
4237bac4f5bSMartin Schwidefsky unsigned int offset, nbytes, n;
4247bac4f5bSMartin Schwidefsky int ret;
4257bac4f5bSMartin Schwidefsky struct {
4267bac4f5bSMartin Schwidefsky u8 key[32];
4277bac4f5bSMartin Schwidefsky u8 tweak[16];
4287bac4f5bSMartin Schwidefsky u8 block[16];
4297bac4f5bSMartin Schwidefsky u8 bit[16];
4307bac4f5bSMartin Schwidefsky u8 xts[16];
4317bac4f5bSMartin Schwidefsky } pcc_param;
4329dda2769SGerald Schaefer struct {
4339dda2769SGerald Schaefer u8 key[32];
4349dda2769SGerald Schaefer u8 init[16];
4359dda2769SGerald Schaefer } xts_param;
43699d97222SGerald Schaefer
4377988fb2cSEric Biggers if (req->cryptlen < AES_BLOCK_SIZE)
4387988fb2cSEric Biggers return -EINVAL;
4397988fb2cSEric Biggers
4407988fb2cSEric Biggers if (unlikely(!xts_ctx->fc || (req->cryptlen % AES_BLOCK_SIZE) != 0)) {
4417988fb2cSEric Biggers struct skcipher_request *subreq = skcipher_request_ctx(req);
4427988fb2cSEric Biggers
4437988fb2cSEric Biggers *subreq = *req;
4447988fb2cSEric Biggers skcipher_request_set_tfm(subreq, xts_ctx->fallback);
4457988fb2cSEric Biggers return (modifier & CPACF_DECRYPT) ?
4467988fb2cSEric Biggers crypto_skcipher_decrypt(subreq) :
4477988fb2cSEric Biggers crypto_skcipher_encrypt(subreq);
4487988fb2cSEric Biggers }
4497988fb2cSEric Biggers
4507988fb2cSEric Biggers ret = skcipher_walk_virt(&walk, req, false);
4517988fb2cSEric Biggers if (ret)
4527988fb2cSEric Biggers return ret;
4537bac4f5bSMartin Schwidefsky offset = xts_ctx->key_len & 0x10;
4549dda2769SGerald Schaefer memset(pcc_param.block, 0, sizeof(pcc_param.block));
4559dda2769SGerald Schaefer memset(pcc_param.bit, 0, sizeof(pcc_param.bit));
4569dda2769SGerald Schaefer memset(pcc_param.xts, 0, sizeof(pcc_param.xts));
4577988fb2cSEric Biggers memcpy(pcc_param.tweak, walk.iv, sizeof(pcc_param.tweak));
45869c0e360SMartin Schwidefsky memcpy(pcc_param.key + offset, xts_ctx->pcc_key, xts_ctx->key_len);
4597bac4f5bSMartin Schwidefsky cpacf_pcc(xts_ctx->fc, pcc_param.key + offset);
46099d97222SGerald Schaefer
46169c0e360SMartin Schwidefsky memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len);
4629dda2769SGerald Schaefer memcpy(xts_param.init, pcc_param.xts, 16);
4637bac4f5bSMartin Schwidefsky
4647988fb2cSEric Biggers while ((nbytes = walk.nbytes) != 0) {
46599d97222SGerald Schaefer /* only use complete blocks */
46699d97222SGerald Schaefer n = nbytes & ~(AES_BLOCK_SIZE - 1);
4677bac4f5bSMartin Schwidefsky cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset,
4687988fb2cSEric Biggers walk.dst.virt.addr, walk.src.virt.addr, n);
4697988fb2cSEric Biggers ret = skcipher_walk_done(&walk, nbytes - n);
4707bac4f5bSMartin Schwidefsky }
4714a559cd1STorsten Duwe memzero_explicit(&pcc_param, sizeof(pcc_param));
4724a559cd1STorsten Duwe memzero_explicit(&xts_param, sizeof(xts_param));
47399d97222SGerald Schaefer return ret;
47499d97222SGerald Schaefer }
47599d97222SGerald Schaefer
xts_aes_encrypt(struct skcipher_request * req)4767988fb2cSEric Biggers static int xts_aes_encrypt(struct skcipher_request *req)
47799d97222SGerald Schaefer {
4787988fb2cSEric Biggers return xts_aes_crypt(req, 0);
47999d97222SGerald Schaefer }
48099d97222SGerald Schaefer
xts_aes_decrypt(struct skcipher_request * req)4817988fb2cSEric Biggers static int xts_aes_decrypt(struct skcipher_request *req)
48299d97222SGerald Schaefer {
4837988fb2cSEric Biggers return xts_aes_crypt(req, CPACF_DECRYPT);
48499d97222SGerald Schaefer }
48599d97222SGerald Schaefer
xts_fallback_init(struct crypto_skcipher * tfm)4867988fb2cSEric Biggers static int xts_fallback_init(struct crypto_skcipher *tfm)
48799d97222SGerald Schaefer {
4887988fb2cSEric Biggers const char *name = crypto_tfm_alg_name(&tfm->base);
4897988fb2cSEric Biggers struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm);
49099d97222SGerald Schaefer
4917988fb2cSEric Biggers xts_ctx->fallback = crypto_alloc_skcipher(name, 0,
4927988fb2cSEric Biggers CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC);
49399d97222SGerald Schaefer
49499d97222SGerald Schaefer if (IS_ERR(xts_ctx->fallback)) {
49599d97222SGerald Schaefer pr_err("Allocating XTS fallback algorithm %s failed\n",
49699d97222SGerald Schaefer name);
49799d97222SGerald Schaefer return PTR_ERR(xts_ctx->fallback);
49899d97222SGerald Schaefer }
4997988fb2cSEric Biggers crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
5007988fb2cSEric Biggers crypto_skcipher_reqsize(xts_ctx->fallback));
50199d97222SGerald Schaefer return 0;
50299d97222SGerald Schaefer }
50399d97222SGerald Schaefer
xts_fallback_exit(struct crypto_skcipher * tfm)5047988fb2cSEric Biggers static void xts_fallback_exit(struct crypto_skcipher *tfm)
50599d97222SGerald Schaefer {
5067988fb2cSEric Biggers struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm);
50799d97222SGerald Schaefer
5087988fb2cSEric Biggers crypto_free_skcipher(xts_ctx->fallback);
50999d97222SGerald Schaefer }
51099d97222SGerald Schaefer
5117988fb2cSEric Biggers static struct skcipher_alg xts_aes_alg = {
5127988fb2cSEric Biggers .base.cra_name = "xts(aes)",
5137988fb2cSEric Biggers .base.cra_driver_name = "xts-aes-s390",
5147988fb2cSEric Biggers .base.cra_priority = 402, /* ecb-aes-s390 + 1 */
5157988fb2cSEric Biggers .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
5167988fb2cSEric Biggers .base.cra_blocksize = AES_BLOCK_SIZE,
5177988fb2cSEric Biggers .base.cra_ctxsize = sizeof(struct s390_xts_ctx),
5187988fb2cSEric Biggers .base.cra_module = THIS_MODULE,
5197988fb2cSEric Biggers .init = xts_fallback_init,
5207988fb2cSEric Biggers .exit = xts_fallback_exit,
52199d97222SGerald Schaefer .min_keysize = 2 * AES_MIN_KEY_SIZE,
52299d97222SGerald Schaefer .max_keysize = 2 * AES_MAX_KEY_SIZE,
52399d97222SGerald Schaefer .ivsize = AES_BLOCK_SIZE,
52499d97222SGerald Schaefer .setkey = xts_aes_set_key,
52599d97222SGerald Schaefer .encrypt = xts_aes_encrypt,
52699d97222SGerald Schaefer .decrypt = xts_aes_decrypt,
52799d97222SGerald Schaefer };
52899d97222SGerald Schaefer
ctr_aes_set_key(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)5297988fb2cSEric Biggers static int ctr_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
5300200f3ecSGerald Schaefer unsigned int key_len)
5310200f3ecSGerald Schaefer {
5327988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
53369c0e360SMartin Schwidefsky unsigned long fc;
5340200f3ecSGerald Schaefer
53569c0e360SMartin Schwidefsky /* Pick the correct function code based on the key length */
53669c0e360SMartin Schwidefsky fc = (key_len == 16) ? CPACF_KMCTR_AES_128 :
53769c0e360SMartin Schwidefsky (key_len == 24) ? CPACF_KMCTR_AES_192 :
53869c0e360SMartin Schwidefsky (key_len == 32) ? CPACF_KMCTR_AES_256 : 0;
5390200f3ecSGerald Schaefer
54069c0e360SMartin Schwidefsky /* Check if the function code is available */
54169c0e360SMartin Schwidefsky sctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0;
54269c0e360SMartin Schwidefsky if (!sctx->fc)
5437988fb2cSEric Biggers return setkey_fallback_skcipher(tfm, in_key, key_len);
54469c0e360SMartin Schwidefsky
54569c0e360SMartin Schwidefsky sctx->key_len = key_len;
54669c0e360SMartin Schwidefsky memcpy(sctx->key, in_key, key_len);
54769c0e360SMartin Schwidefsky return 0;
5480200f3ecSGerald Schaefer }
5490200f3ecSGerald Schaefer
__ctrblk_init(u8 * ctrptr,u8 * iv,unsigned int nbytes)5507bac4f5bSMartin Schwidefsky static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes)
5510519e9adSHarald Freudenberger {
5520519e9adSHarald Freudenberger unsigned int i, n;
5530519e9adSHarald Freudenberger
5540519e9adSHarald Freudenberger /* only use complete blocks, max. PAGE_SIZE */
5557bac4f5bSMartin Schwidefsky memcpy(ctrptr, iv, AES_BLOCK_SIZE);
5560519e9adSHarald Freudenberger n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1);
5577bac4f5bSMartin Schwidefsky for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) {
5587bac4f5bSMartin Schwidefsky memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE);
5597bac4f5bSMartin Schwidefsky crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE);
5607bac4f5bSMartin Schwidefsky ctrptr += AES_BLOCK_SIZE;
5610519e9adSHarald Freudenberger }
5620519e9adSHarald Freudenberger return n;
5630519e9adSHarald Freudenberger }
5640519e9adSHarald Freudenberger
ctr_aes_crypt(struct skcipher_request * req)5657988fb2cSEric Biggers static int ctr_aes_crypt(struct skcipher_request *req)
5660200f3ecSGerald Schaefer {
5677988fb2cSEric Biggers struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
5687988fb2cSEric Biggers struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
5697bac4f5bSMartin Schwidefsky u8 buf[AES_BLOCK_SIZE], *ctrptr;
5707988fb2cSEric Biggers struct skcipher_walk walk;
5710519e9adSHarald Freudenberger unsigned int n, nbytes;
5727bac4f5bSMartin Schwidefsky int ret, locked;
5730200f3ecSGerald Schaefer
5747988fb2cSEric Biggers if (unlikely(!sctx->fc))
5757988fb2cSEric Biggers return fallback_skcipher_crypt(sctx, req, 0);
5767988fb2cSEric Biggers
5771c2c7029SHarald Freudenberger locked = mutex_trylock(&ctrblk_lock);
5780200f3ecSGerald Schaefer
5797988fb2cSEric Biggers ret = skcipher_walk_virt(&walk, req, false);
5807988fb2cSEric Biggers while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
5810519e9adSHarald Freudenberger n = AES_BLOCK_SIZE;
5827988fb2cSEric Biggers
5837bac4f5bSMartin Schwidefsky if (nbytes >= 2*AES_BLOCK_SIZE && locked)
5847988fb2cSEric Biggers n = __ctrblk_init(ctrblk, walk.iv, nbytes);
5857988fb2cSEric Biggers ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv;
5867988fb2cSEric Biggers cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr,
5877988fb2cSEric Biggers walk.src.virt.addr, n, ctrptr);
5887bac4f5bSMartin Schwidefsky if (ctrptr == ctrblk)
5897988fb2cSEric Biggers memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE,
5900200f3ecSGerald Schaefer AES_BLOCK_SIZE);
5917988fb2cSEric Biggers crypto_inc(walk.iv, AES_BLOCK_SIZE);
5927988fb2cSEric Biggers ret = skcipher_walk_done(&walk, nbytes - n);
5930200f3ecSGerald Schaefer }
5947bac4f5bSMartin Schwidefsky if (locked)
5951c2c7029SHarald Freudenberger mutex_unlock(&ctrblk_lock);
5960200f3ecSGerald Schaefer /*
5970200f3ecSGerald Schaefer * final block may be < AES_BLOCK_SIZE, copy only nbytes
5980200f3ecSGerald Schaefer */
5990200f3ecSGerald Schaefer if (nbytes) {
600*d68ac388SHerbert Xu memset(buf, 0, AES_BLOCK_SIZE);
601*d68ac388SHerbert Xu memcpy(buf, walk.src.virt.addr, nbytes);
602*d68ac388SHerbert Xu cpacf_kmctr(sctx->fc, sctx->key, buf, buf,
6037988fb2cSEric Biggers AES_BLOCK_SIZE, walk.iv);
6047988fb2cSEric Biggers memcpy(walk.dst.virt.addr, buf, nbytes);
6057988fb2cSEric Biggers crypto_inc(walk.iv, AES_BLOCK_SIZE);
6067988fb2cSEric Biggers ret = skcipher_walk_done(&walk, 0);
6070200f3ecSGerald Schaefer }
6080519e9adSHarald Freudenberger
6090200f3ecSGerald Schaefer return ret;
6100200f3ecSGerald Schaefer }
6110200f3ecSGerald Schaefer
6127988fb2cSEric Biggers static struct skcipher_alg ctr_aes_alg = {
6137988fb2cSEric Biggers .base.cra_name = "ctr(aes)",
6147988fb2cSEric Biggers .base.cra_driver_name = "ctr-aes-s390",
6157988fb2cSEric Biggers .base.cra_priority = 402, /* ecb-aes-s390 + 1 */
6167988fb2cSEric Biggers .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
6177988fb2cSEric Biggers .base.cra_blocksize = 1,
6187988fb2cSEric Biggers .base.cra_ctxsize = sizeof(struct s390_aes_ctx),
6197988fb2cSEric Biggers .base.cra_module = THIS_MODULE,
6207988fb2cSEric Biggers .init = fallback_init_skcipher,
6217988fb2cSEric Biggers .exit = fallback_exit_skcipher,
6220200f3ecSGerald Schaefer .min_keysize = AES_MIN_KEY_SIZE,
6230200f3ecSGerald Schaefer .max_keysize = AES_MAX_KEY_SIZE,
6240200f3ecSGerald Schaefer .ivsize = AES_BLOCK_SIZE,
6250200f3ecSGerald Schaefer .setkey = ctr_aes_set_key,
6267988fb2cSEric Biggers .encrypt = ctr_aes_crypt,
6277988fb2cSEric Biggers .decrypt = ctr_aes_crypt,
6287988fb2cSEric Biggers .chunksize = AES_BLOCK_SIZE,
6290200f3ecSGerald Schaefer };
6300200f3ecSGerald Schaefer
gcm_aes_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)631bf7fa038SHarald Freudenberger static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *key,
632bf7fa038SHarald Freudenberger unsigned int keylen)
633bf7fa038SHarald Freudenberger {
634bf7fa038SHarald Freudenberger struct s390_aes_ctx *ctx = crypto_aead_ctx(tfm);
635bf7fa038SHarald Freudenberger
636bf7fa038SHarald Freudenberger switch (keylen) {
637bf7fa038SHarald Freudenberger case AES_KEYSIZE_128:
638bf7fa038SHarald Freudenberger ctx->fc = CPACF_KMA_GCM_AES_128;
639bf7fa038SHarald Freudenberger break;
640bf7fa038SHarald Freudenberger case AES_KEYSIZE_192:
641bf7fa038SHarald Freudenberger ctx->fc = CPACF_KMA_GCM_AES_192;
642bf7fa038SHarald Freudenberger break;
643bf7fa038SHarald Freudenberger case AES_KEYSIZE_256:
644bf7fa038SHarald Freudenberger ctx->fc = CPACF_KMA_GCM_AES_256;
645bf7fa038SHarald Freudenberger break;
646bf7fa038SHarald Freudenberger default:
647bf7fa038SHarald Freudenberger return -EINVAL;
648bf7fa038SHarald Freudenberger }
649bf7fa038SHarald Freudenberger
650bf7fa038SHarald Freudenberger memcpy(ctx->key, key, keylen);
651bf7fa038SHarald Freudenberger ctx->key_len = keylen;
652bf7fa038SHarald Freudenberger return 0;
653bf7fa038SHarald Freudenberger }
654bf7fa038SHarald Freudenberger
gcm_aes_setauthsize(struct crypto_aead * tfm,unsigned int authsize)655bf7fa038SHarald Freudenberger static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
656bf7fa038SHarald Freudenberger {
657bf7fa038SHarald Freudenberger switch (authsize) {
658bf7fa038SHarald Freudenberger case 4:
659bf7fa038SHarald Freudenberger case 8:
660bf7fa038SHarald Freudenberger case 12:
661bf7fa038SHarald Freudenberger case 13:
662bf7fa038SHarald Freudenberger case 14:
663bf7fa038SHarald Freudenberger case 15:
664bf7fa038SHarald Freudenberger case 16:
665bf7fa038SHarald Freudenberger break;
666bf7fa038SHarald Freudenberger default:
667bf7fa038SHarald Freudenberger return -EINVAL;
668bf7fa038SHarald Freudenberger }
669bf7fa038SHarald Freudenberger
670bf7fa038SHarald Freudenberger return 0;
671bf7fa038SHarald Freudenberger }
672bf7fa038SHarald Freudenberger
gcm_walk_start(struct gcm_sg_walk * gw,struct scatterlist * sg,unsigned int len)673bef9f0baSHarald Freudenberger static void gcm_walk_start(struct gcm_sg_walk *gw, struct scatterlist *sg,
674bf7fa038SHarald Freudenberger unsigned int len)
675bf7fa038SHarald Freudenberger {
676bf7fa038SHarald Freudenberger memset(gw, 0, sizeof(*gw));
677bf7fa038SHarald Freudenberger gw->walk_bytes_remain = len;
678bf7fa038SHarald Freudenberger scatterwalk_start(&gw->walk, sg);
679bf7fa038SHarald Freudenberger }
680bf7fa038SHarald Freudenberger
_gcm_sg_clamp_and_map(struct gcm_sg_walk * gw)681bef9f0baSHarald Freudenberger static inline unsigned int _gcm_sg_clamp_and_map(struct gcm_sg_walk *gw)
682bef9f0baSHarald Freudenberger {
683bef9f0baSHarald Freudenberger struct scatterlist *nextsg;
684bef9f0baSHarald Freudenberger
685bef9f0baSHarald Freudenberger gw->walk_bytes = scatterwalk_clamp(&gw->walk, gw->walk_bytes_remain);
686bef9f0baSHarald Freudenberger while (!gw->walk_bytes) {
687bef9f0baSHarald Freudenberger nextsg = sg_next(gw->walk.sg);
688bef9f0baSHarald Freudenberger if (!nextsg)
689bef9f0baSHarald Freudenberger return 0;
690bef9f0baSHarald Freudenberger scatterwalk_start(&gw->walk, nextsg);
691bef9f0baSHarald Freudenberger gw->walk_bytes = scatterwalk_clamp(&gw->walk,
692bef9f0baSHarald Freudenberger gw->walk_bytes_remain);
693bef9f0baSHarald Freudenberger }
694bef9f0baSHarald Freudenberger gw->walk_ptr = scatterwalk_map(&gw->walk);
695bef9f0baSHarald Freudenberger return gw->walk_bytes;
696bef9f0baSHarald Freudenberger }
697bef9f0baSHarald Freudenberger
_gcm_sg_unmap_and_advance(struct gcm_sg_walk * gw,unsigned int nbytes)698bef9f0baSHarald Freudenberger static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw,
699bef9f0baSHarald Freudenberger unsigned int nbytes)
700bef9f0baSHarald Freudenberger {
701bef9f0baSHarald Freudenberger gw->walk_bytes_remain -= nbytes;
702bd52cd5eSJann Horn scatterwalk_unmap(gw->walk_ptr);
703bef9f0baSHarald Freudenberger scatterwalk_advance(&gw->walk, nbytes);
704bef9f0baSHarald Freudenberger scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain);
705bef9f0baSHarald Freudenberger gw->walk_ptr = NULL;
706bef9f0baSHarald Freudenberger }
707bef9f0baSHarald Freudenberger
gcm_in_walk_go(struct gcm_sg_walk * gw,unsigned int minbytesneeded)708bef9f0baSHarald Freudenberger static int gcm_in_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)
709bf7fa038SHarald Freudenberger {
710bf7fa038SHarald Freudenberger int n;
711bf7fa038SHarald Freudenberger
712bf7fa038SHarald Freudenberger if (gw->buf_bytes && gw->buf_bytes >= minbytesneeded) {
713bf7fa038SHarald Freudenberger gw->ptr = gw->buf;
714bf7fa038SHarald Freudenberger gw->nbytes = gw->buf_bytes;
715bf7fa038SHarald Freudenberger goto out;
716bf7fa038SHarald Freudenberger }
717bf7fa038SHarald Freudenberger
718bf7fa038SHarald Freudenberger if (gw->walk_bytes_remain == 0) {
719bf7fa038SHarald Freudenberger gw->ptr = NULL;
720bf7fa038SHarald Freudenberger gw->nbytes = 0;
721bf7fa038SHarald Freudenberger goto out;
722bf7fa038SHarald Freudenberger }
723bf7fa038SHarald Freudenberger
724bef9f0baSHarald Freudenberger if (!_gcm_sg_clamp_and_map(gw)) {
725bef9f0baSHarald Freudenberger gw->ptr = NULL;
726bef9f0baSHarald Freudenberger gw->nbytes = 0;
727bef9f0baSHarald Freudenberger goto out;
728bf7fa038SHarald Freudenberger }
729bf7fa038SHarald Freudenberger
730bf7fa038SHarald Freudenberger if (!gw->buf_bytes && gw->walk_bytes >= minbytesneeded) {
731bf7fa038SHarald Freudenberger gw->ptr = gw->walk_ptr;
732bf7fa038SHarald Freudenberger gw->nbytes = gw->walk_bytes;
733bf7fa038SHarald Freudenberger goto out;
734bf7fa038SHarald Freudenberger }
735bf7fa038SHarald Freudenberger
736bf7fa038SHarald Freudenberger while (1) {
737bf7fa038SHarald Freudenberger n = min(gw->walk_bytes, AES_BLOCK_SIZE - gw->buf_bytes);
738bf7fa038SHarald Freudenberger memcpy(gw->buf + gw->buf_bytes, gw->walk_ptr, n);
739bf7fa038SHarald Freudenberger gw->buf_bytes += n;
740bef9f0baSHarald Freudenberger _gcm_sg_unmap_and_advance(gw, n);
741bf7fa038SHarald Freudenberger if (gw->buf_bytes >= minbytesneeded) {
742bf7fa038SHarald Freudenberger gw->ptr = gw->buf;
743bf7fa038SHarald Freudenberger gw->nbytes = gw->buf_bytes;
744bf7fa038SHarald Freudenberger goto out;
745bf7fa038SHarald Freudenberger }
746bef9f0baSHarald Freudenberger if (!_gcm_sg_clamp_and_map(gw)) {
747bef9f0baSHarald Freudenberger gw->ptr = NULL;
748bef9f0baSHarald Freudenberger gw->nbytes = 0;
749bef9f0baSHarald Freudenberger goto out;
750bf7fa038SHarald Freudenberger }
751bf7fa038SHarald Freudenberger }
752bf7fa038SHarald Freudenberger
753bf7fa038SHarald Freudenberger out:
754bf7fa038SHarald Freudenberger return gw->nbytes;
755bf7fa038SHarald Freudenberger }
756bf7fa038SHarald Freudenberger
gcm_out_walk_go(struct gcm_sg_walk * gw,unsigned int minbytesneeded)757bef9f0baSHarald Freudenberger static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)
758bf7fa038SHarald Freudenberger {
759bef9f0baSHarald Freudenberger if (gw->walk_bytes_remain == 0) {
760bef9f0baSHarald Freudenberger gw->ptr = NULL;
761bef9f0baSHarald Freudenberger gw->nbytes = 0;
762bef9f0baSHarald Freudenberger goto out;
763bef9f0baSHarald Freudenberger }
764bf7fa038SHarald Freudenberger
765bef9f0baSHarald Freudenberger if (!_gcm_sg_clamp_and_map(gw)) {
766bef9f0baSHarald Freudenberger gw->ptr = NULL;
767bef9f0baSHarald Freudenberger gw->nbytes = 0;
768bef9f0baSHarald Freudenberger goto out;
769bef9f0baSHarald Freudenberger }
770bef9f0baSHarald Freudenberger
771bef9f0baSHarald Freudenberger if (gw->walk_bytes >= minbytesneeded) {
772bef9f0baSHarald Freudenberger gw->ptr = gw->walk_ptr;
773bef9f0baSHarald Freudenberger gw->nbytes = gw->walk_bytes;
774bef9f0baSHarald Freudenberger goto out;
775bef9f0baSHarald Freudenberger }
776bef9f0baSHarald Freudenberger
777bd52cd5eSJann Horn scatterwalk_unmap(gw->walk_ptr);
778bef9f0baSHarald Freudenberger gw->walk_ptr = NULL;
779bef9f0baSHarald Freudenberger
780bef9f0baSHarald Freudenberger gw->ptr = gw->buf;
781bef9f0baSHarald Freudenberger gw->nbytes = sizeof(gw->buf);
782bef9f0baSHarald Freudenberger
783bef9f0baSHarald Freudenberger out:
784bef9f0baSHarald Freudenberger return gw->nbytes;
785bef9f0baSHarald Freudenberger }
786bef9f0baSHarald Freudenberger
gcm_in_walk_done(struct gcm_sg_walk * gw,unsigned int bytesdone)787bef9f0baSHarald Freudenberger static int gcm_in_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone)
788bef9f0baSHarald Freudenberger {
789bf7fa038SHarald Freudenberger if (gw->ptr == NULL)
790bef9f0baSHarald Freudenberger return 0;
791bf7fa038SHarald Freudenberger
792bf7fa038SHarald Freudenberger if (gw->ptr == gw->buf) {
793bef9f0baSHarald Freudenberger int n = gw->buf_bytes - bytesdone;
794bf7fa038SHarald Freudenberger if (n > 0) {
795bf7fa038SHarald Freudenberger memmove(gw->buf, gw->buf + bytesdone, n);
796bef9f0baSHarald Freudenberger gw->buf_bytes = n;
797bf7fa038SHarald Freudenberger } else
798bf7fa038SHarald Freudenberger gw->buf_bytes = 0;
799bef9f0baSHarald Freudenberger } else
800bef9f0baSHarald Freudenberger _gcm_sg_unmap_and_advance(gw, bytesdone);
801bef9f0baSHarald Freudenberger
802bef9f0baSHarald Freudenberger return bytesdone;
803bf7fa038SHarald Freudenberger }
804bef9f0baSHarald Freudenberger
gcm_out_walk_done(struct gcm_sg_walk * gw,unsigned int bytesdone)805bef9f0baSHarald Freudenberger static int gcm_out_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone)
806bef9f0baSHarald Freudenberger {
807bef9f0baSHarald Freudenberger int i, n;
808bef9f0baSHarald Freudenberger
809bef9f0baSHarald Freudenberger if (gw->ptr == NULL)
810bef9f0baSHarald Freudenberger return 0;
811bef9f0baSHarald Freudenberger
812bef9f0baSHarald Freudenberger if (gw->ptr == gw->buf) {
813bef9f0baSHarald Freudenberger for (i = 0; i < bytesdone; i += n) {
814bef9f0baSHarald Freudenberger if (!_gcm_sg_clamp_and_map(gw))
815bef9f0baSHarald Freudenberger return i;
816bef9f0baSHarald Freudenberger n = min(gw->walk_bytes, bytesdone - i);
817bef9f0baSHarald Freudenberger memcpy(gw->walk_ptr, gw->buf + i, n);
818bef9f0baSHarald Freudenberger _gcm_sg_unmap_and_advance(gw, n);
819bef9f0baSHarald Freudenberger }
820bef9f0baSHarald Freudenberger } else
821bef9f0baSHarald Freudenberger _gcm_sg_unmap_and_advance(gw, bytesdone);
822bef9f0baSHarald Freudenberger
823bef9f0baSHarald Freudenberger return bytesdone;
824bf7fa038SHarald Freudenberger }
825bf7fa038SHarald Freudenberger
gcm_aes_crypt(struct aead_request * req,unsigned int flags)826bf7fa038SHarald Freudenberger static int gcm_aes_crypt(struct aead_request *req, unsigned int flags)
827bf7fa038SHarald Freudenberger {
828bf7fa038SHarald Freudenberger struct crypto_aead *tfm = crypto_aead_reqtfm(req);
829bf7fa038SHarald Freudenberger struct s390_aes_ctx *ctx = crypto_aead_ctx(tfm);
830bf7fa038SHarald Freudenberger unsigned int ivsize = crypto_aead_ivsize(tfm);
831bf7fa038SHarald Freudenberger unsigned int taglen = crypto_aead_authsize(tfm);
832bf7fa038SHarald Freudenberger unsigned int aadlen = req->assoclen;
833bf7fa038SHarald Freudenberger unsigned int pclen = req->cryptlen;
834bf7fa038SHarald Freudenberger int ret = 0;
835bf7fa038SHarald Freudenberger
836bef9f0baSHarald Freudenberger unsigned int n, len, in_bytes, out_bytes,
837bf7fa038SHarald Freudenberger min_bytes, bytes, aad_bytes, pc_bytes;
838bf7fa038SHarald Freudenberger struct gcm_sg_walk gw_in, gw_out;
839bf7fa038SHarald Freudenberger u8 tag[GHASH_DIGEST_SIZE];
840bf7fa038SHarald Freudenberger
841bf7fa038SHarald Freudenberger struct {
842bf7fa038SHarald Freudenberger u32 _[3]; /* reserved */
843bf7fa038SHarald Freudenberger u32 cv; /* Counter Value */
844bf7fa038SHarald Freudenberger u8 t[GHASH_DIGEST_SIZE];/* Tag */
845bf7fa038SHarald Freudenberger u8 h[AES_BLOCK_SIZE]; /* Hash-subkey */
846bf7fa038SHarald Freudenberger u64 taadl; /* Total AAD Length */
847bf7fa038SHarald Freudenberger u64 tpcl; /* Total Plain-/Cipher-text Length */
848bf7fa038SHarald Freudenberger u8 j0[GHASH_BLOCK_SIZE];/* initial counter value */
849bf7fa038SHarald Freudenberger u8 k[AES_MAX_KEY_SIZE]; /* Key */
850bf7fa038SHarald Freudenberger } param;
851bf7fa038SHarald Freudenberger
852bf7fa038SHarald Freudenberger /*
853bf7fa038SHarald Freudenberger * encrypt
854bf7fa038SHarald Freudenberger * req->src: aad||plaintext
855bf7fa038SHarald Freudenberger * req->dst: aad||ciphertext||tag
856bf7fa038SHarald Freudenberger * decrypt
857bf7fa038SHarald Freudenberger * req->src: aad||ciphertext||tag
858bf7fa038SHarald Freudenberger * req->dst: aad||plaintext, return 0 or -EBADMSG
859bf7fa038SHarald Freudenberger * aad, plaintext and ciphertext may be empty.
860bf7fa038SHarald Freudenberger */
861bf7fa038SHarald Freudenberger if (flags & CPACF_DECRYPT)
862bf7fa038SHarald Freudenberger pclen -= taglen;
863bf7fa038SHarald Freudenberger len = aadlen + pclen;
864bf7fa038SHarald Freudenberger
865bf7fa038SHarald Freudenberger memset(¶m, 0, sizeof(param));
866bf7fa038SHarald Freudenberger param.cv = 1;
867bf7fa038SHarald Freudenberger param.taadl = aadlen * 8;
868bf7fa038SHarald Freudenberger param.tpcl = pclen * 8;
869bf7fa038SHarald Freudenberger memcpy(param.j0, req->iv, ivsize);
870bf7fa038SHarald Freudenberger *(u32 *)(param.j0 + ivsize) = 1;
871bf7fa038SHarald Freudenberger memcpy(param.k, ctx->key, ctx->key_len);
872bf7fa038SHarald Freudenberger
873bef9f0baSHarald Freudenberger gcm_walk_start(&gw_in, req->src, len);
874bef9f0baSHarald Freudenberger gcm_walk_start(&gw_out, req->dst, len);
875bf7fa038SHarald Freudenberger
876bf7fa038SHarald Freudenberger do {
877bf7fa038SHarald Freudenberger min_bytes = min_t(unsigned int,
878bf7fa038SHarald Freudenberger aadlen > 0 ? aadlen : pclen, AES_BLOCK_SIZE);
879bef9f0baSHarald Freudenberger in_bytes = gcm_in_walk_go(&gw_in, min_bytes);
880bef9f0baSHarald Freudenberger out_bytes = gcm_out_walk_go(&gw_out, min_bytes);
881bf7fa038SHarald Freudenberger bytes = min(in_bytes, out_bytes);
882bf7fa038SHarald Freudenberger
883bf7fa038SHarald Freudenberger if (aadlen + pclen <= bytes) {
884bf7fa038SHarald Freudenberger aad_bytes = aadlen;
885bf7fa038SHarald Freudenberger pc_bytes = pclen;
886bf7fa038SHarald Freudenberger flags |= CPACF_KMA_LAAD | CPACF_KMA_LPC;
887bf7fa038SHarald Freudenberger } else {
888bf7fa038SHarald Freudenberger if (aadlen <= bytes) {
889bf7fa038SHarald Freudenberger aad_bytes = aadlen;
890bf7fa038SHarald Freudenberger pc_bytes = (bytes - aadlen) &
891bf7fa038SHarald Freudenberger ~(AES_BLOCK_SIZE - 1);
892bf7fa038SHarald Freudenberger flags |= CPACF_KMA_LAAD;
893bf7fa038SHarald Freudenberger } else {
894bf7fa038SHarald Freudenberger aad_bytes = bytes & ~(AES_BLOCK_SIZE - 1);
895bf7fa038SHarald Freudenberger pc_bytes = 0;
896bf7fa038SHarald Freudenberger }
897bf7fa038SHarald Freudenberger }
898bf7fa038SHarald Freudenberger
899bf7fa038SHarald Freudenberger if (aad_bytes > 0)
900bf7fa038SHarald Freudenberger memcpy(gw_out.ptr, gw_in.ptr, aad_bytes);
901bf7fa038SHarald Freudenberger
902bf7fa038SHarald Freudenberger cpacf_kma(ctx->fc | flags, ¶m,
903bf7fa038SHarald Freudenberger gw_out.ptr + aad_bytes,
904bf7fa038SHarald Freudenberger gw_in.ptr + aad_bytes, pc_bytes,
905bf7fa038SHarald Freudenberger gw_in.ptr, aad_bytes);
906bf7fa038SHarald Freudenberger
907bef9f0baSHarald Freudenberger n = aad_bytes + pc_bytes;
908bef9f0baSHarald Freudenberger if (gcm_in_walk_done(&gw_in, n) != n)
909bef9f0baSHarald Freudenberger return -ENOMEM;
910bef9f0baSHarald Freudenberger if (gcm_out_walk_done(&gw_out, n) != n)
911bef9f0baSHarald Freudenberger return -ENOMEM;
912bf7fa038SHarald Freudenberger aadlen -= aad_bytes;
913bf7fa038SHarald Freudenberger pclen -= pc_bytes;
914bf7fa038SHarald Freudenberger } while (aadlen + pclen > 0);
915bf7fa038SHarald Freudenberger
916bf7fa038SHarald Freudenberger if (flags & CPACF_DECRYPT) {
917bf7fa038SHarald Freudenberger scatterwalk_map_and_copy(tag, req->src, len, taglen, 0);
918bf7fa038SHarald Freudenberger if (crypto_memneq(tag, param.t, taglen))
919bf7fa038SHarald Freudenberger ret = -EBADMSG;
920bf7fa038SHarald Freudenberger } else
921bf7fa038SHarald Freudenberger scatterwalk_map_and_copy(param.t, req->dst, len, taglen, 1);
922bf7fa038SHarald Freudenberger
923bf7fa038SHarald Freudenberger memzero_explicit(¶m, sizeof(param));
924bf7fa038SHarald Freudenberger return ret;
925bf7fa038SHarald Freudenberger }
926bf7fa038SHarald Freudenberger
gcm_aes_encrypt(struct aead_request * req)927bf7fa038SHarald Freudenberger static int gcm_aes_encrypt(struct aead_request *req)
928bf7fa038SHarald Freudenberger {
929bf7fa038SHarald Freudenberger return gcm_aes_crypt(req, CPACF_ENCRYPT);
930bf7fa038SHarald Freudenberger }
931bf7fa038SHarald Freudenberger
gcm_aes_decrypt(struct aead_request * req)932bf7fa038SHarald Freudenberger static int gcm_aes_decrypt(struct aead_request *req)
933bf7fa038SHarald Freudenberger {
934bf7fa038SHarald Freudenberger return gcm_aes_crypt(req, CPACF_DECRYPT);
935bf7fa038SHarald Freudenberger }
936bf7fa038SHarald Freudenberger
937bf7fa038SHarald Freudenberger static struct aead_alg gcm_aes_aead = {
938bf7fa038SHarald Freudenberger .setkey = gcm_aes_setkey,
939bf7fa038SHarald Freudenberger .setauthsize = gcm_aes_setauthsize,
940bf7fa038SHarald Freudenberger .encrypt = gcm_aes_encrypt,
941bf7fa038SHarald Freudenberger .decrypt = gcm_aes_decrypt,
942bf7fa038SHarald Freudenberger
943bf7fa038SHarald Freudenberger .ivsize = GHASH_BLOCK_SIZE - sizeof(u32),
944bf7fa038SHarald Freudenberger .maxauthsize = GHASH_DIGEST_SIZE,
945bf7fa038SHarald Freudenberger .chunksize = AES_BLOCK_SIZE,
946bf7fa038SHarald Freudenberger
947bf7fa038SHarald Freudenberger .base = {
948bf7fa038SHarald Freudenberger .cra_blocksize = 1,
949bf7fa038SHarald Freudenberger .cra_ctxsize = sizeof(struct s390_aes_ctx),
950bf7fa038SHarald Freudenberger .cra_priority = 900,
951bf7fa038SHarald Freudenberger .cra_name = "gcm(aes)",
952bf7fa038SHarald Freudenberger .cra_driver_name = "gcm-aes-s390",
953bf7fa038SHarald Freudenberger .cra_module = THIS_MODULE,
954bf7fa038SHarald Freudenberger },
955bf7fa038SHarald Freudenberger };
956bf7fa038SHarald Freudenberger
9577988fb2cSEric Biggers static struct crypto_alg *aes_s390_alg;
9587988fb2cSEric Biggers static struct skcipher_alg *aes_s390_skcipher_algs[4];
9597988fb2cSEric Biggers static int aes_s390_skciphers_num;
960c7260ca3SHarald Freudenberger static struct aead_alg *aes_s390_aead_alg;
961d863d594SMartin Schwidefsky
aes_s390_register_skcipher(struct skcipher_alg * alg)9627988fb2cSEric Biggers static int aes_s390_register_skcipher(struct skcipher_alg *alg)
963d863d594SMartin Schwidefsky {
964d863d594SMartin Schwidefsky int ret;
965d863d594SMartin Schwidefsky
9667988fb2cSEric Biggers ret = crypto_register_skcipher(alg);
967d863d594SMartin Schwidefsky if (!ret)
9687988fb2cSEric Biggers aes_s390_skcipher_algs[aes_s390_skciphers_num++] = alg;
969d863d594SMartin Schwidefsky return ret;
970d863d594SMartin Schwidefsky }
971d863d594SMartin Schwidefsky
aes_s390_fini(void)972d863d594SMartin Schwidefsky static void aes_s390_fini(void)
973d863d594SMartin Schwidefsky {
9747988fb2cSEric Biggers if (aes_s390_alg)
9757988fb2cSEric Biggers crypto_unregister_alg(aes_s390_alg);
9767988fb2cSEric Biggers while (aes_s390_skciphers_num--)
9777988fb2cSEric Biggers crypto_unregister_skcipher(aes_s390_skcipher_algs[aes_s390_skciphers_num]);
978d863d594SMartin Schwidefsky if (ctrblk)
979d863d594SMartin Schwidefsky free_page((unsigned long) ctrblk);
980bf7fa038SHarald Freudenberger
981c7260ca3SHarald Freudenberger if (aes_s390_aead_alg)
982c7260ca3SHarald Freudenberger crypto_unregister_aead(aes_s390_aead_alg);
983d863d594SMartin Schwidefsky }
9844f57ba71SIngo Tuchscherer
aes_s390_init(void)9859f7819c1SHeiko Carstens static int __init aes_s390_init(void)
986bf754ae8SJan Glauber {
987bf754ae8SJan Glauber int ret;
988bf754ae8SJan Glauber
989bf7fa038SHarald Freudenberger /* Query available functions for KM, KMC, KMCTR and KMA */
99069c0e360SMartin Schwidefsky cpacf_query(CPACF_KM, &km_functions);
99169c0e360SMartin Schwidefsky cpacf_query(CPACF_KMC, &kmc_functions);
99269c0e360SMartin Schwidefsky cpacf_query(CPACF_KMCTR, &kmctr_functions);
993bf7fa038SHarald Freudenberger cpacf_query(CPACF_KMA, &kma_functions);
994bf754ae8SJan Glauber
99569c0e360SMartin Schwidefsky if (cpacf_test_func(&km_functions, CPACF_KM_AES_128) ||
99669c0e360SMartin Schwidefsky cpacf_test_func(&km_functions, CPACF_KM_AES_192) ||
99769c0e360SMartin Schwidefsky cpacf_test_func(&km_functions, CPACF_KM_AES_256)) {
9987988fb2cSEric Biggers ret = crypto_register_alg(&aes_alg);
99986aa9fc2SJan Glauber if (ret)
1000d863d594SMartin Schwidefsky goto out_err;
10017988fb2cSEric Biggers aes_s390_alg = &aes_alg;
10027988fb2cSEric Biggers ret = aes_s390_register_skcipher(&ecb_aes_alg);
100386aa9fc2SJan Glauber if (ret)
1004d863d594SMartin Schwidefsky goto out_err;
100569c0e360SMartin Schwidefsky }
1006a9e62fadSHerbert Xu
100769c0e360SMartin Schwidefsky if (cpacf_test_func(&kmc_functions, CPACF_KMC_AES_128) ||
100869c0e360SMartin Schwidefsky cpacf_test_func(&kmc_functions, CPACF_KMC_AES_192) ||
100969c0e360SMartin Schwidefsky cpacf_test_func(&kmc_functions, CPACF_KMC_AES_256)) {
10107988fb2cSEric Biggers ret = aes_s390_register_skcipher(&cbc_aes_alg);
101186aa9fc2SJan Glauber if (ret)
1012d863d594SMartin Schwidefsky goto out_err;
101369c0e360SMartin Schwidefsky }
1014a9e62fadSHerbert Xu
101569c0e360SMartin Schwidefsky if (cpacf_test_func(&km_functions, CPACF_KM_XTS_128) ||
101669c0e360SMartin Schwidefsky cpacf_test_func(&km_functions, CPACF_KM_XTS_256)) {
10177988fb2cSEric Biggers ret = aes_s390_register_skcipher(&xts_aes_alg);
101899d97222SGerald Schaefer if (ret)
1019d863d594SMartin Schwidefsky goto out_err;
102099d97222SGerald Schaefer }
102199d97222SGerald Schaefer
102269c0e360SMartin Schwidefsky if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_AES_128) ||
102369c0e360SMartin Schwidefsky cpacf_test_func(&kmctr_functions, CPACF_KMCTR_AES_192) ||
102469c0e360SMartin Schwidefsky cpacf_test_func(&kmctr_functions, CPACF_KMCTR_AES_256)) {
10250200f3ecSGerald Schaefer ctrblk = (u8 *) __get_free_page(GFP_KERNEL);
10260200f3ecSGerald Schaefer if (!ctrblk) {
10270200f3ecSGerald Schaefer ret = -ENOMEM;
1028d863d594SMartin Schwidefsky goto out_err;
10290200f3ecSGerald Schaefer }
10307988fb2cSEric Biggers ret = aes_s390_register_skcipher(&ctr_aes_alg);
1031d863d594SMartin Schwidefsky if (ret)
1032d863d594SMartin Schwidefsky goto out_err;
10330200f3ecSGerald Schaefer }
10340200f3ecSGerald Schaefer
1035bf7fa038SHarald Freudenberger if (cpacf_test_func(&kma_functions, CPACF_KMA_GCM_AES_128) ||
1036bf7fa038SHarald Freudenberger cpacf_test_func(&kma_functions, CPACF_KMA_GCM_AES_192) ||
1037bf7fa038SHarald Freudenberger cpacf_test_func(&kma_functions, CPACF_KMA_GCM_AES_256)) {
1038bf7fa038SHarald Freudenberger ret = crypto_register_aead(&gcm_aes_aead);
1039bf7fa038SHarald Freudenberger if (ret)
1040bf7fa038SHarald Freudenberger goto out_err;
1041c7260ca3SHarald Freudenberger aes_s390_aead_alg = &gcm_aes_aead;
1042bf7fa038SHarald Freudenberger }
1043bf7fa038SHarald Freudenberger
1044d863d594SMartin Schwidefsky return 0;
1045d863d594SMartin Schwidefsky out_err:
1046d863d594SMartin Schwidefsky aes_s390_fini();
1047bf754ae8SJan Glauber return ret;
1048bf754ae8SJan Glauber }
1049bf754ae8SJan Glauber
10500a5f9b38SHeiko Carstens module_cpu_feature_match(S390_CPU_FEATURE_MSA, aes_s390_init);
10519f7819c1SHeiko Carstens module_exit(aes_s390_fini);
1052bf754ae8SJan Glauber
10535d26a105SKees Cook MODULE_ALIAS_CRYPTO("aes-all");
1054bf754ae8SJan Glauber
1055bf754ae8SJan Glauber MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
1056bf754ae8SJan Glauber MODULE_LICENSE("GPL");
10570eb76ba2SArd Biesheuvel MODULE_IMPORT_NS(CRYPTO_INTERNAL);
1058