127937843SMartin Schwidefsky /* 227937843SMartin Schwidefsky * Cryptographic API. 327937843SMartin Schwidefsky * 427937843SMartin Schwidefsky * s390 implementation of the AES Cipher Algorithm with protected keys. 527937843SMartin Schwidefsky * 627937843SMartin Schwidefsky * s390 Version: 727937843SMartin Schwidefsky * Copyright IBM Corp. 2017 827937843SMartin Schwidefsky * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 927937843SMartin Schwidefsky * Harald Freudenberger <freude@de.ibm.com> 1027937843SMartin Schwidefsky * 1127937843SMartin Schwidefsky * This program is free software; you can redistribute it and/or modify 1227937843SMartin Schwidefsky * it under the terms of the GNU General Public License (version 2 only) 1327937843SMartin Schwidefsky * as published by the Free Software Foundation. 1427937843SMartin Schwidefsky * 1527937843SMartin Schwidefsky */ 1627937843SMartin Schwidefsky 1727937843SMartin Schwidefsky #define KMSG_COMPONENT "paes_s390" 1827937843SMartin Schwidefsky #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 1927937843SMartin Schwidefsky 2027937843SMartin Schwidefsky #include <crypto/aes.h> 2127937843SMartin Schwidefsky #include <crypto/algapi.h> 2227937843SMartin Schwidefsky #include <linux/bug.h> 2327937843SMartin Schwidefsky #include <linux/err.h> 2427937843SMartin Schwidefsky #include <linux/module.h> 2527937843SMartin Schwidefsky #include <linux/cpufeature.h> 2627937843SMartin Schwidefsky #include <linux/init.h> 2727937843SMartin Schwidefsky #include <linux/spinlock.h> 2827937843SMartin Schwidefsky #include <crypto/xts.h> 2927937843SMartin Schwidefsky #include <asm/cpacf.h> 3027937843SMartin Schwidefsky #include <asm/pkey.h> 3127937843SMartin Schwidefsky 3227937843SMartin Schwidefsky static u8 *ctrblk; 3327937843SMartin Schwidefsky static DEFINE_SPINLOCK(ctrblk_lock); 3427937843SMartin Schwidefsky 3527937843SMartin Schwidefsky static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; 3627937843SMartin Schwidefsky 3727937843SMartin Schwidefsky struct s390_paes_ctx { 3827937843SMartin Schwidefsky struct pkey_seckey sk; 3927937843SMartin Schwidefsky struct pkey_protkey pk; 4027937843SMartin Schwidefsky unsigned long fc; 4127937843SMartin Schwidefsky }; 4227937843SMartin Schwidefsky 4327937843SMartin Schwidefsky struct s390_pxts_ctx { 4427937843SMartin Schwidefsky struct pkey_seckey sk[2]; 4527937843SMartin Schwidefsky struct pkey_protkey pk[2]; 4627937843SMartin Schwidefsky unsigned long fc; 4727937843SMartin Schwidefsky }; 4827937843SMartin Schwidefsky 4927937843SMartin Schwidefsky static inline int __paes_convert_key(struct pkey_seckey *sk, 5027937843SMartin Schwidefsky struct pkey_protkey *pk) 5127937843SMartin Schwidefsky { 5227937843SMartin Schwidefsky int i, ret; 5327937843SMartin Schwidefsky 5427937843SMartin Schwidefsky /* try three times in case of failure */ 5527937843SMartin Schwidefsky for (i = 0; i < 3; i++) { 5627937843SMartin Schwidefsky ret = pkey_skey2pkey(sk, pk); 5727937843SMartin Schwidefsky if (ret == 0) 5827937843SMartin Schwidefsky break; 5927937843SMartin Schwidefsky } 6027937843SMartin Schwidefsky 6127937843SMartin Schwidefsky return ret; 6227937843SMartin Schwidefsky } 6327937843SMartin Schwidefsky 6427937843SMartin Schwidefsky static int __paes_set_key(struct s390_paes_ctx *ctx) 6527937843SMartin Schwidefsky { 6627937843SMartin Schwidefsky unsigned long fc; 6727937843SMartin Schwidefsky 6827937843SMartin Schwidefsky if (__paes_convert_key(&ctx->sk, &ctx->pk)) 6927937843SMartin Schwidefsky return -EINVAL; 7027937843SMartin Schwidefsky 7127937843SMartin Schwidefsky /* Pick the correct function code based on the protected key type */ 7227937843SMartin Schwidefsky fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 : 7327937843SMartin Schwidefsky (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 : 7427937843SMartin Schwidefsky (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0; 7527937843SMartin Schwidefsky 7627937843SMartin Schwidefsky /* Check if the function code is available */ 7727937843SMartin Schwidefsky ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 7827937843SMartin Schwidefsky 7927937843SMartin Schwidefsky return ctx->fc ? 0 : -EINVAL; 8027937843SMartin Schwidefsky } 8127937843SMartin Schwidefsky 8227937843SMartin Schwidefsky static int ecb_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 8327937843SMartin Schwidefsky unsigned int key_len) 8427937843SMartin Schwidefsky { 8527937843SMartin Schwidefsky struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 8627937843SMartin Schwidefsky 8727937843SMartin Schwidefsky if (key_len != SECKEYBLOBSIZE) 8827937843SMartin Schwidefsky return -EINVAL; 8927937843SMartin Schwidefsky 9027937843SMartin Schwidefsky memcpy(ctx->sk.seckey, in_key, SECKEYBLOBSIZE); 9127937843SMartin Schwidefsky if (__paes_set_key(ctx)) { 9227937843SMartin Schwidefsky tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 9327937843SMartin Schwidefsky return -EINVAL; 9427937843SMartin Schwidefsky } 9527937843SMartin Schwidefsky return 0; 9627937843SMartin Schwidefsky } 9727937843SMartin Schwidefsky 9827937843SMartin Schwidefsky static int ecb_paes_crypt(struct blkcipher_desc *desc, 9927937843SMartin Schwidefsky unsigned long modifier, 10027937843SMartin Schwidefsky struct blkcipher_walk *walk) 10127937843SMartin Schwidefsky { 10227937843SMartin Schwidefsky struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 10327937843SMartin Schwidefsky unsigned int nbytes, n, k; 10427937843SMartin Schwidefsky int ret; 10527937843SMartin Schwidefsky 10627937843SMartin Schwidefsky ret = blkcipher_walk_virt(desc, walk); 10727937843SMartin Schwidefsky while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 10827937843SMartin Schwidefsky /* only use complete blocks */ 10927937843SMartin Schwidefsky n = nbytes & ~(AES_BLOCK_SIZE - 1); 11027937843SMartin Schwidefsky k = cpacf_km(ctx->fc | modifier, ctx->pk.protkey, 11127937843SMartin Schwidefsky walk->dst.virt.addr, walk->src.virt.addr, n); 11227937843SMartin Schwidefsky if (k) 11327937843SMartin Schwidefsky ret = blkcipher_walk_done(desc, walk, nbytes - k); 11427937843SMartin Schwidefsky if (k < n) { 11527937843SMartin Schwidefsky if (__paes_set_key(ctx) != 0) 11627937843SMartin Schwidefsky return blkcipher_walk_done(desc, walk, -EIO); 11727937843SMartin Schwidefsky } 11827937843SMartin Schwidefsky } 11927937843SMartin Schwidefsky return ret; 12027937843SMartin Schwidefsky } 12127937843SMartin Schwidefsky 12227937843SMartin Schwidefsky static int ecb_paes_encrypt(struct blkcipher_desc *desc, 12327937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 12427937843SMartin Schwidefsky unsigned int nbytes) 12527937843SMartin Schwidefsky { 12627937843SMartin Schwidefsky struct blkcipher_walk walk; 12727937843SMartin Schwidefsky 12827937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 12927937843SMartin Schwidefsky return ecb_paes_crypt(desc, CPACF_ENCRYPT, &walk); 13027937843SMartin Schwidefsky } 13127937843SMartin Schwidefsky 13227937843SMartin Schwidefsky static int ecb_paes_decrypt(struct blkcipher_desc *desc, 13327937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 13427937843SMartin Schwidefsky unsigned int nbytes) 13527937843SMartin Schwidefsky { 13627937843SMartin Schwidefsky struct blkcipher_walk walk; 13727937843SMartin Schwidefsky 13827937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 13927937843SMartin Schwidefsky return ecb_paes_crypt(desc, CPACF_DECRYPT, &walk); 14027937843SMartin Schwidefsky } 14127937843SMartin Schwidefsky 14227937843SMartin Schwidefsky static struct crypto_alg ecb_paes_alg = { 14327937843SMartin Schwidefsky .cra_name = "ecb(paes)", 14427937843SMartin Schwidefsky .cra_driver_name = "ecb-paes-s390", 14527937843SMartin Schwidefsky .cra_priority = 400, /* combo: aes + ecb */ 14627937843SMartin Schwidefsky .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 14727937843SMartin Schwidefsky .cra_blocksize = AES_BLOCK_SIZE, 14827937843SMartin Schwidefsky .cra_ctxsize = sizeof(struct s390_paes_ctx), 14927937843SMartin Schwidefsky .cra_type = &crypto_blkcipher_type, 15027937843SMartin Schwidefsky .cra_module = THIS_MODULE, 15127937843SMartin Schwidefsky .cra_list = LIST_HEAD_INIT(ecb_paes_alg.cra_list), 15227937843SMartin Schwidefsky .cra_u = { 15327937843SMartin Schwidefsky .blkcipher = { 15427937843SMartin Schwidefsky .min_keysize = SECKEYBLOBSIZE, 15527937843SMartin Schwidefsky .max_keysize = SECKEYBLOBSIZE, 15627937843SMartin Schwidefsky .setkey = ecb_paes_set_key, 15727937843SMartin Schwidefsky .encrypt = ecb_paes_encrypt, 15827937843SMartin Schwidefsky .decrypt = ecb_paes_decrypt, 15927937843SMartin Schwidefsky } 16027937843SMartin Schwidefsky } 16127937843SMartin Schwidefsky }; 16227937843SMartin Schwidefsky 16327937843SMartin Schwidefsky static int __cbc_paes_set_key(struct s390_paes_ctx *ctx) 16427937843SMartin Schwidefsky { 16527937843SMartin Schwidefsky unsigned long fc; 16627937843SMartin Schwidefsky 16727937843SMartin Schwidefsky if (__paes_convert_key(&ctx->sk, &ctx->pk)) 16827937843SMartin Schwidefsky return -EINVAL; 16927937843SMartin Schwidefsky 17027937843SMartin Schwidefsky /* Pick the correct function code based on the protected key type */ 17127937843SMartin Schwidefsky fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMC_PAES_128 : 17227937843SMartin Schwidefsky (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMC_PAES_192 : 17327937843SMartin Schwidefsky (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KMC_PAES_256 : 0; 17427937843SMartin Schwidefsky 17527937843SMartin Schwidefsky /* Check if the function code is available */ 17627937843SMartin Schwidefsky ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0; 17727937843SMartin Schwidefsky 17827937843SMartin Schwidefsky return ctx->fc ? 0 : -EINVAL; 17927937843SMartin Schwidefsky } 18027937843SMartin Schwidefsky 18127937843SMartin Schwidefsky static int cbc_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 18227937843SMartin Schwidefsky unsigned int key_len) 18327937843SMartin Schwidefsky { 18427937843SMartin Schwidefsky struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 18527937843SMartin Schwidefsky 18627937843SMartin Schwidefsky memcpy(ctx->sk.seckey, in_key, SECKEYBLOBSIZE); 18727937843SMartin Schwidefsky if (__cbc_paes_set_key(ctx)) { 18827937843SMartin Schwidefsky tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 18927937843SMartin Schwidefsky return -EINVAL; 19027937843SMartin Schwidefsky } 19127937843SMartin Schwidefsky return 0; 19227937843SMartin Schwidefsky } 19327937843SMartin Schwidefsky 19427937843SMartin Schwidefsky static int cbc_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, 19527937843SMartin Schwidefsky struct blkcipher_walk *walk) 19627937843SMartin Schwidefsky { 19727937843SMartin Schwidefsky struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 19827937843SMartin Schwidefsky unsigned int nbytes, n, k; 19927937843SMartin Schwidefsky int ret; 20027937843SMartin Schwidefsky struct { 20127937843SMartin Schwidefsky u8 iv[AES_BLOCK_SIZE]; 20227937843SMartin Schwidefsky u8 key[MAXPROTKEYSIZE]; 20327937843SMartin Schwidefsky } param; 20427937843SMartin Schwidefsky 20527937843SMartin Schwidefsky ret = blkcipher_walk_virt(desc, walk); 20627937843SMartin Schwidefsky memcpy(param.iv, walk->iv, AES_BLOCK_SIZE); 20727937843SMartin Schwidefsky memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 20827937843SMartin Schwidefsky while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 20927937843SMartin Schwidefsky /* only use complete blocks */ 21027937843SMartin Schwidefsky n = nbytes & ~(AES_BLOCK_SIZE - 1); 21127937843SMartin Schwidefsky k = cpacf_kmc(ctx->fc | modifier, ¶m, 21227937843SMartin Schwidefsky walk->dst.virt.addr, walk->src.virt.addr, n); 21327937843SMartin Schwidefsky if (k) 21427937843SMartin Schwidefsky ret = blkcipher_walk_done(desc, walk, nbytes - k); 21527937843SMartin Schwidefsky if (n < k) { 21627937843SMartin Schwidefsky if (__cbc_paes_set_key(ctx) != 0) 21727937843SMartin Schwidefsky return blkcipher_walk_done(desc, walk, -EIO); 21827937843SMartin Schwidefsky memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); 21927937843SMartin Schwidefsky } 22027937843SMartin Schwidefsky } 22127937843SMartin Schwidefsky memcpy(walk->iv, param.iv, AES_BLOCK_SIZE); 22227937843SMartin Schwidefsky return ret; 22327937843SMartin Schwidefsky } 22427937843SMartin Schwidefsky 22527937843SMartin Schwidefsky static int cbc_paes_encrypt(struct blkcipher_desc *desc, 22627937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 22727937843SMartin Schwidefsky unsigned int nbytes) 22827937843SMartin Schwidefsky { 22927937843SMartin Schwidefsky struct blkcipher_walk walk; 23027937843SMartin Schwidefsky 23127937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 23227937843SMartin Schwidefsky return cbc_paes_crypt(desc, 0, &walk); 23327937843SMartin Schwidefsky } 23427937843SMartin Schwidefsky 23527937843SMartin Schwidefsky static int cbc_paes_decrypt(struct blkcipher_desc *desc, 23627937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 23727937843SMartin Schwidefsky unsigned int nbytes) 23827937843SMartin Schwidefsky { 23927937843SMartin Schwidefsky struct blkcipher_walk walk; 24027937843SMartin Schwidefsky 24127937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 24227937843SMartin Schwidefsky return cbc_paes_crypt(desc, CPACF_DECRYPT, &walk); 24327937843SMartin Schwidefsky } 24427937843SMartin Schwidefsky 24527937843SMartin Schwidefsky static struct crypto_alg cbc_paes_alg = { 24627937843SMartin Schwidefsky .cra_name = "cbc(paes)", 24727937843SMartin Schwidefsky .cra_driver_name = "cbc-paes-s390", 24827937843SMartin Schwidefsky .cra_priority = 400, /* combo: aes + cbc */ 24927937843SMartin Schwidefsky .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 25027937843SMartin Schwidefsky .cra_blocksize = AES_BLOCK_SIZE, 25127937843SMartin Schwidefsky .cra_ctxsize = sizeof(struct s390_paes_ctx), 25227937843SMartin Schwidefsky .cra_type = &crypto_blkcipher_type, 25327937843SMartin Schwidefsky .cra_module = THIS_MODULE, 25427937843SMartin Schwidefsky .cra_list = LIST_HEAD_INIT(cbc_paes_alg.cra_list), 25527937843SMartin Schwidefsky .cra_u = { 25627937843SMartin Schwidefsky .blkcipher = { 25727937843SMartin Schwidefsky .min_keysize = SECKEYBLOBSIZE, 25827937843SMartin Schwidefsky .max_keysize = SECKEYBLOBSIZE, 25927937843SMartin Schwidefsky .ivsize = AES_BLOCK_SIZE, 26027937843SMartin Schwidefsky .setkey = cbc_paes_set_key, 26127937843SMartin Schwidefsky .encrypt = cbc_paes_encrypt, 26227937843SMartin Schwidefsky .decrypt = cbc_paes_decrypt, 26327937843SMartin Schwidefsky } 26427937843SMartin Schwidefsky } 26527937843SMartin Schwidefsky }; 26627937843SMartin Schwidefsky 26727937843SMartin Schwidefsky static int __xts_paes_set_key(struct s390_pxts_ctx *ctx) 26827937843SMartin Schwidefsky { 26927937843SMartin Schwidefsky unsigned long fc; 27027937843SMartin Schwidefsky 27127937843SMartin Schwidefsky if (__paes_convert_key(&ctx->sk[0], &ctx->pk[0]) || 27227937843SMartin Schwidefsky __paes_convert_key(&ctx->sk[1], &ctx->pk[1])) 27327937843SMartin Schwidefsky return -EINVAL; 27427937843SMartin Schwidefsky 27527937843SMartin Schwidefsky if (ctx->pk[0].type != ctx->pk[1].type) 27627937843SMartin Schwidefsky return -EINVAL; 27727937843SMartin Schwidefsky 27827937843SMartin Schwidefsky /* Pick the correct function code based on the protected key type */ 27927937843SMartin Schwidefsky fc = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PXTS_128 : 28027937843SMartin Schwidefsky (ctx->pk[0].type == PKEY_KEYTYPE_AES_256) ? 28127937843SMartin Schwidefsky CPACF_KM_PXTS_256 : 0; 28227937843SMartin Schwidefsky 28327937843SMartin Schwidefsky /* Check if the function code is available */ 28427937843SMartin Schwidefsky ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; 28527937843SMartin Schwidefsky 28627937843SMartin Schwidefsky return ctx->fc ? 0 : -EINVAL; 28727937843SMartin Schwidefsky } 28827937843SMartin Schwidefsky 28927937843SMartin Schwidefsky static int xts_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 29027937843SMartin Schwidefsky unsigned int key_len) 29127937843SMartin Schwidefsky { 29227937843SMartin Schwidefsky struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm); 29327937843SMartin Schwidefsky u8 ckey[2 * AES_MAX_KEY_SIZE]; 29427937843SMartin Schwidefsky unsigned int ckey_len; 29527937843SMartin Schwidefsky 29627937843SMartin Schwidefsky memcpy(ctx->sk[0].seckey, in_key, SECKEYBLOBSIZE); 29727937843SMartin Schwidefsky memcpy(ctx->sk[1].seckey, in_key + SECKEYBLOBSIZE, SECKEYBLOBSIZE); 29827937843SMartin Schwidefsky if (__xts_paes_set_key(ctx)) { 29927937843SMartin Schwidefsky tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 30027937843SMartin Schwidefsky return -EINVAL; 30127937843SMartin Schwidefsky } 30227937843SMartin Schwidefsky 30327937843SMartin Schwidefsky /* 30427937843SMartin Schwidefsky * xts_check_key verifies the key length is not odd and makes 30527937843SMartin Schwidefsky * sure that the two keys are not the same. This can be done 30627937843SMartin Schwidefsky * on the two protected keys as well 30727937843SMartin Schwidefsky */ 30827937843SMartin Schwidefsky ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 30927937843SMartin Schwidefsky AES_KEYSIZE_128 : AES_KEYSIZE_256; 31027937843SMartin Schwidefsky memcpy(ckey, ctx->pk[0].protkey, ckey_len); 31127937843SMartin Schwidefsky memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len); 31227937843SMartin Schwidefsky return xts_check_key(tfm, ckey, 2*ckey_len); 31327937843SMartin Schwidefsky } 31427937843SMartin Schwidefsky 31527937843SMartin Schwidefsky static int xts_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, 31627937843SMartin Schwidefsky struct blkcipher_walk *walk) 31727937843SMartin Schwidefsky { 31827937843SMartin Schwidefsky struct s390_pxts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 31927937843SMartin Schwidefsky unsigned int keylen, offset, nbytes, n, k; 32027937843SMartin Schwidefsky int ret; 32127937843SMartin Schwidefsky struct { 32227937843SMartin Schwidefsky u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 32327937843SMartin Schwidefsky u8 tweak[16]; 32427937843SMartin Schwidefsky u8 block[16]; 32527937843SMartin Schwidefsky u8 bit[16]; 32627937843SMartin Schwidefsky u8 xts[16]; 32727937843SMartin Schwidefsky } pcc_param; 32827937843SMartin Schwidefsky struct { 32927937843SMartin Schwidefsky u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ 33027937843SMartin Schwidefsky u8 init[16]; 33127937843SMartin Schwidefsky } xts_param; 33227937843SMartin Schwidefsky 33327937843SMartin Schwidefsky ret = blkcipher_walk_virt(desc, walk); 33427937843SMartin Schwidefsky keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64; 33527937843SMartin Schwidefsky offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0; 33627937843SMartin Schwidefsky retry: 33727937843SMartin Schwidefsky memset(&pcc_param, 0, sizeof(pcc_param)); 33827937843SMartin Schwidefsky memcpy(pcc_param.tweak, walk->iv, sizeof(pcc_param.tweak)); 33927937843SMartin Schwidefsky memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen); 34027937843SMartin Schwidefsky cpacf_pcc(ctx->fc, pcc_param.key + offset); 34127937843SMartin Schwidefsky 34227937843SMartin Schwidefsky memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen); 34327937843SMartin Schwidefsky memcpy(xts_param.init, pcc_param.xts, 16); 34427937843SMartin Schwidefsky 34527937843SMartin Schwidefsky while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 34627937843SMartin Schwidefsky /* only use complete blocks */ 34727937843SMartin Schwidefsky n = nbytes & ~(AES_BLOCK_SIZE - 1); 34827937843SMartin Schwidefsky k = cpacf_km(ctx->fc | modifier, xts_param.key + offset, 34927937843SMartin Schwidefsky walk->dst.virt.addr, walk->src.virt.addr, n); 35027937843SMartin Schwidefsky if (k) 35127937843SMartin Schwidefsky ret = blkcipher_walk_done(desc, walk, nbytes - k); 35227937843SMartin Schwidefsky if (k < n) { 35327937843SMartin Schwidefsky if (__xts_paes_set_key(ctx) != 0) 35427937843SMartin Schwidefsky return blkcipher_walk_done(desc, walk, -EIO); 35527937843SMartin Schwidefsky goto retry; 35627937843SMartin Schwidefsky } 35727937843SMartin Schwidefsky } 35827937843SMartin Schwidefsky return ret; 35927937843SMartin Schwidefsky } 36027937843SMartin Schwidefsky 36127937843SMartin Schwidefsky static int xts_paes_encrypt(struct blkcipher_desc *desc, 36227937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 36327937843SMartin Schwidefsky unsigned int nbytes) 36427937843SMartin Schwidefsky { 36527937843SMartin Schwidefsky struct blkcipher_walk walk; 36627937843SMartin Schwidefsky 36727937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 36827937843SMartin Schwidefsky return xts_paes_crypt(desc, 0, &walk); 36927937843SMartin Schwidefsky } 37027937843SMartin Schwidefsky 37127937843SMartin Schwidefsky static int xts_paes_decrypt(struct blkcipher_desc *desc, 37227937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 37327937843SMartin Schwidefsky unsigned int nbytes) 37427937843SMartin Schwidefsky { 37527937843SMartin Schwidefsky struct blkcipher_walk walk; 37627937843SMartin Schwidefsky 37727937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 37827937843SMartin Schwidefsky return xts_paes_crypt(desc, CPACF_DECRYPT, &walk); 37927937843SMartin Schwidefsky } 38027937843SMartin Schwidefsky 38127937843SMartin Schwidefsky static struct crypto_alg xts_paes_alg = { 38227937843SMartin Schwidefsky .cra_name = "xts(paes)", 38327937843SMartin Schwidefsky .cra_driver_name = "xts-paes-s390", 38427937843SMartin Schwidefsky .cra_priority = 400, /* combo: aes + xts */ 38527937843SMartin Schwidefsky .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 38627937843SMartin Schwidefsky .cra_blocksize = AES_BLOCK_SIZE, 38727937843SMartin Schwidefsky .cra_ctxsize = sizeof(struct s390_pxts_ctx), 38827937843SMartin Schwidefsky .cra_type = &crypto_blkcipher_type, 38927937843SMartin Schwidefsky .cra_module = THIS_MODULE, 39027937843SMartin Schwidefsky .cra_list = LIST_HEAD_INIT(xts_paes_alg.cra_list), 39127937843SMartin Schwidefsky .cra_u = { 39227937843SMartin Schwidefsky .blkcipher = { 39327937843SMartin Schwidefsky .min_keysize = 2 * SECKEYBLOBSIZE, 39427937843SMartin Schwidefsky .max_keysize = 2 * SECKEYBLOBSIZE, 39527937843SMartin Schwidefsky .ivsize = AES_BLOCK_SIZE, 39627937843SMartin Schwidefsky .setkey = xts_paes_set_key, 39727937843SMartin Schwidefsky .encrypt = xts_paes_encrypt, 39827937843SMartin Schwidefsky .decrypt = xts_paes_decrypt, 39927937843SMartin Schwidefsky } 40027937843SMartin Schwidefsky } 40127937843SMartin Schwidefsky }; 40227937843SMartin Schwidefsky 40327937843SMartin Schwidefsky static int __ctr_paes_set_key(struct s390_paes_ctx *ctx) 40427937843SMartin Schwidefsky { 40527937843SMartin Schwidefsky unsigned long fc; 40627937843SMartin Schwidefsky 40727937843SMartin Schwidefsky if (__paes_convert_key(&ctx->sk, &ctx->pk)) 40827937843SMartin Schwidefsky return -EINVAL; 40927937843SMartin Schwidefsky 41027937843SMartin Schwidefsky /* Pick the correct function code based on the protected key type */ 41127937843SMartin Schwidefsky fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMCTR_PAES_128 : 41227937843SMartin Schwidefsky (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMCTR_PAES_192 : 41327937843SMartin Schwidefsky (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? 41427937843SMartin Schwidefsky CPACF_KMCTR_PAES_256 : 0; 41527937843SMartin Schwidefsky 41627937843SMartin Schwidefsky /* Check if the function code is available */ 41727937843SMartin Schwidefsky ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0; 41827937843SMartin Schwidefsky 41927937843SMartin Schwidefsky return ctx->fc ? 0 : -EINVAL; 42027937843SMartin Schwidefsky } 42127937843SMartin Schwidefsky 42227937843SMartin Schwidefsky static int ctr_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 42327937843SMartin Schwidefsky unsigned int key_len) 42427937843SMartin Schwidefsky { 42527937843SMartin Schwidefsky struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); 42627937843SMartin Schwidefsky 42727937843SMartin Schwidefsky memcpy(ctx->sk.seckey, in_key, key_len); 42827937843SMartin Schwidefsky if (__ctr_paes_set_key(ctx)) { 42927937843SMartin Schwidefsky tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 43027937843SMartin Schwidefsky return -EINVAL; 43127937843SMartin Schwidefsky } 43227937843SMartin Schwidefsky return 0; 43327937843SMartin Schwidefsky } 43427937843SMartin Schwidefsky 43527937843SMartin Schwidefsky static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) 43627937843SMartin Schwidefsky { 43727937843SMartin Schwidefsky unsigned int i, n; 43827937843SMartin Schwidefsky 43927937843SMartin Schwidefsky /* only use complete blocks, max. PAGE_SIZE */ 44027937843SMartin Schwidefsky memcpy(ctrptr, iv, AES_BLOCK_SIZE); 44127937843SMartin Schwidefsky n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1); 44227937843SMartin Schwidefsky for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) { 44327937843SMartin Schwidefsky memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE); 44427937843SMartin Schwidefsky crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE); 44527937843SMartin Schwidefsky ctrptr += AES_BLOCK_SIZE; 44627937843SMartin Schwidefsky } 44727937843SMartin Schwidefsky return n; 44827937843SMartin Schwidefsky } 44927937843SMartin Schwidefsky 45027937843SMartin Schwidefsky static int ctr_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, 45127937843SMartin Schwidefsky struct blkcipher_walk *walk) 45227937843SMartin Schwidefsky { 45327937843SMartin Schwidefsky struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 45427937843SMartin Schwidefsky u8 buf[AES_BLOCK_SIZE], *ctrptr; 45527937843SMartin Schwidefsky unsigned int nbytes, n, k; 45627937843SMartin Schwidefsky int ret, locked; 45727937843SMartin Schwidefsky 45827937843SMartin Schwidefsky locked = spin_trylock(&ctrblk_lock); 45927937843SMartin Schwidefsky 46027937843SMartin Schwidefsky ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE); 46127937843SMartin Schwidefsky while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { 46227937843SMartin Schwidefsky n = AES_BLOCK_SIZE; 46327937843SMartin Schwidefsky if (nbytes >= 2*AES_BLOCK_SIZE && locked) 46427937843SMartin Schwidefsky n = __ctrblk_init(ctrblk, walk->iv, nbytes); 46527937843SMartin Schwidefsky ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; 46627937843SMartin Schwidefsky k = cpacf_kmctr(ctx->fc | modifier, ctx->pk.protkey, 46727937843SMartin Schwidefsky walk->dst.virt.addr, walk->src.virt.addr, 46827937843SMartin Schwidefsky n, ctrptr); 46927937843SMartin Schwidefsky if (k) { 47027937843SMartin Schwidefsky if (ctrptr == ctrblk) 47127937843SMartin Schwidefsky memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE, 47227937843SMartin Schwidefsky AES_BLOCK_SIZE); 47327937843SMartin Schwidefsky crypto_inc(walk->iv, AES_BLOCK_SIZE); 47427937843SMartin Schwidefsky ret = blkcipher_walk_done(desc, walk, nbytes - n); 47527937843SMartin Schwidefsky } 47627937843SMartin Schwidefsky if (k < n) { 477e7c95effSMartin Schwidefsky if (__ctr_paes_set_key(ctx) != 0) { 478e7c95effSMartin Schwidefsky if (locked) 479e7c95effSMartin Schwidefsky spin_unlock(&ctrblk_lock); 48027937843SMartin Schwidefsky return blkcipher_walk_done(desc, walk, -EIO); 48127937843SMartin Schwidefsky } 48227937843SMartin Schwidefsky } 483e7c95effSMartin Schwidefsky } 48427937843SMartin Schwidefsky if (locked) 48527937843SMartin Schwidefsky spin_unlock(&ctrblk_lock); 48627937843SMartin Schwidefsky /* 48727937843SMartin Schwidefsky * final block may be < AES_BLOCK_SIZE, copy only nbytes 48827937843SMartin Schwidefsky */ 48927937843SMartin Schwidefsky if (nbytes) { 49027937843SMartin Schwidefsky while (1) { 49127937843SMartin Schwidefsky if (cpacf_kmctr(ctx->fc | modifier, 49227937843SMartin Schwidefsky ctx->pk.protkey, buf, 49327937843SMartin Schwidefsky walk->src.virt.addr, AES_BLOCK_SIZE, 49427937843SMartin Schwidefsky walk->iv) == AES_BLOCK_SIZE) 49527937843SMartin Schwidefsky break; 49627937843SMartin Schwidefsky if (__ctr_paes_set_key(ctx) != 0) 49727937843SMartin Schwidefsky return blkcipher_walk_done(desc, walk, -EIO); 49827937843SMartin Schwidefsky } 49927937843SMartin Schwidefsky memcpy(walk->dst.virt.addr, buf, nbytes); 50027937843SMartin Schwidefsky crypto_inc(walk->iv, AES_BLOCK_SIZE); 50127937843SMartin Schwidefsky ret = blkcipher_walk_done(desc, walk, 0); 50227937843SMartin Schwidefsky } 50327937843SMartin Schwidefsky 50427937843SMartin Schwidefsky return ret; 50527937843SMartin Schwidefsky } 50627937843SMartin Schwidefsky 50727937843SMartin Schwidefsky static int ctr_paes_encrypt(struct blkcipher_desc *desc, 50827937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 50927937843SMartin Schwidefsky unsigned int nbytes) 51027937843SMartin Schwidefsky { 51127937843SMartin Schwidefsky struct blkcipher_walk walk; 51227937843SMartin Schwidefsky 51327937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 51427937843SMartin Schwidefsky return ctr_paes_crypt(desc, 0, &walk); 51527937843SMartin Schwidefsky } 51627937843SMartin Schwidefsky 51727937843SMartin Schwidefsky static int ctr_paes_decrypt(struct blkcipher_desc *desc, 51827937843SMartin Schwidefsky struct scatterlist *dst, struct scatterlist *src, 51927937843SMartin Schwidefsky unsigned int nbytes) 52027937843SMartin Schwidefsky { 52127937843SMartin Schwidefsky struct blkcipher_walk walk; 52227937843SMartin Schwidefsky 52327937843SMartin Schwidefsky blkcipher_walk_init(&walk, dst, src, nbytes); 52427937843SMartin Schwidefsky return ctr_paes_crypt(desc, CPACF_DECRYPT, &walk); 52527937843SMartin Schwidefsky } 52627937843SMartin Schwidefsky 52727937843SMartin Schwidefsky static struct crypto_alg ctr_paes_alg = { 52827937843SMartin Schwidefsky .cra_name = "ctr(paes)", 52927937843SMartin Schwidefsky .cra_driver_name = "ctr-paes-s390", 53027937843SMartin Schwidefsky .cra_priority = 400, /* combo: aes + ctr */ 53127937843SMartin Schwidefsky .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 53227937843SMartin Schwidefsky .cra_blocksize = 1, 53327937843SMartin Schwidefsky .cra_ctxsize = sizeof(struct s390_paes_ctx), 53427937843SMartin Schwidefsky .cra_type = &crypto_blkcipher_type, 53527937843SMartin Schwidefsky .cra_module = THIS_MODULE, 53627937843SMartin Schwidefsky .cra_list = LIST_HEAD_INIT(ctr_paes_alg.cra_list), 53727937843SMartin Schwidefsky .cra_u = { 53827937843SMartin Schwidefsky .blkcipher = { 53927937843SMartin Schwidefsky .min_keysize = SECKEYBLOBSIZE, 54027937843SMartin Schwidefsky .max_keysize = SECKEYBLOBSIZE, 54127937843SMartin Schwidefsky .ivsize = AES_BLOCK_SIZE, 54227937843SMartin Schwidefsky .setkey = ctr_paes_set_key, 54327937843SMartin Schwidefsky .encrypt = ctr_paes_encrypt, 54427937843SMartin Schwidefsky .decrypt = ctr_paes_decrypt, 54527937843SMartin Schwidefsky } 54627937843SMartin Schwidefsky } 54727937843SMartin Schwidefsky }; 54827937843SMartin Schwidefsky 54927937843SMartin Schwidefsky static inline void __crypto_unregister_alg(struct crypto_alg *alg) 55027937843SMartin Schwidefsky { 55127937843SMartin Schwidefsky if (!list_empty(&alg->cra_list)) 55227937843SMartin Schwidefsky crypto_unregister_alg(alg); 55327937843SMartin Schwidefsky } 55427937843SMartin Schwidefsky 55527937843SMartin Schwidefsky static void paes_s390_fini(void) 55627937843SMartin Schwidefsky { 55727937843SMartin Schwidefsky if (ctrblk) 55827937843SMartin Schwidefsky free_page((unsigned long) ctrblk); 55927937843SMartin Schwidefsky __crypto_unregister_alg(&ctr_paes_alg); 56027937843SMartin Schwidefsky __crypto_unregister_alg(&xts_paes_alg); 56127937843SMartin Schwidefsky __crypto_unregister_alg(&cbc_paes_alg); 56227937843SMartin Schwidefsky __crypto_unregister_alg(&ecb_paes_alg); 56327937843SMartin Schwidefsky } 56427937843SMartin Schwidefsky 56527937843SMartin Schwidefsky static int __init paes_s390_init(void) 56627937843SMartin Schwidefsky { 56727937843SMartin Schwidefsky int ret; 56827937843SMartin Schwidefsky 56927937843SMartin Schwidefsky /* Query available functions for KM, KMC and KMCTR */ 57027937843SMartin Schwidefsky cpacf_query(CPACF_KM, &km_functions); 57127937843SMartin Schwidefsky cpacf_query(CPACF_KMC, &kmc_functions); 57227937843SMartin Schwidefsky cpacf_query(CPACF_KMCTR, &kmctr_functions); 57327937843SMartin Schwidefsky 57427937843SMartin Schwidefsky if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) || 57527937843SMartin Schwidefsky cpacf_test_func(&km_functions, CPACF_KM_PAES_192) || 57627937843SMartin Schwidefsky cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) { 57727937843SMartin Schwidefsky ret = crypto_register_alg(&ecb_paes_alg); 57827937843SMartin Schwidefsky if (ret) 57927937843SMartin Schwidefsky goto out_err; 58027937843SMartin Schwidefsky } 58127937843SMartin Schwidefsky 58227937843SMartin Schwidefsky if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) || 58327937843SMartin Schwidefsky cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) || 58427937843SMartin Schwidefsky cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) { 58527937843SMartin Schwidefsky ret = crypto_register_alg(&cbc_paes_alg); 58627937843SMartin Schwidefsky if (ret) 58727937843SMartin Schwidefsky goto out_err; 58827937843SMartin Schwidefsky } 58927937843SMartin Schwidefsky 59027937843SMartin Schwidefsky if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) || 59127937843SMartin Schwidefsky cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) { 59227937843SMartin Schwidefsky ret = crypto_register_alg(&xts_paes_alg); 59327937843SMartin Schwidefsky if (ret) 59427937843SMartin Schwidefsky goto out_err; 59527937843SMartin Schwidefsky } 59627937843SMartin Schwidefsky 59727937843SMartin Schwidefsky if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) || 59827937843SMartin Schwidefsky cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) || 59927937843SMartin Schwidefsky cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) { 60027937843SMartin Schwidefsky ret = crypto_register_alg(&ctr_paes_alg); 60127937843SMartin Schwidefsky if (ret) 60227937843SMartin Schwidefsky goto out_err; 60327937843SMartin Schwidefsky ctrblk = (u8 *) __get_free_page(GFP_KERNEL); 60427937843SMartin Schwidefsky if (!ctrblk) { 60527937843SMartin Schwidefsky ret = -ENOMEM; 60627937843SMartin Schwidefsky goto out_err; 60727937843SMartin Schwidefsky } 60827937843SMartin Schwidefsky } 60927937843SMartin Schwidefsky 61027937843SMartin Schwidefsky return 0; 61127937843SMartin Schwidefsky out_err: 61227937843SMartin Schwidefsky paes_s390_fini(); 61327937843SMartin Schwidefsky return ret; 61427937843SMartin Schwidefsky } 61527937843SMartin Schwidefsky 61627937843SMartin Schwidefsky module_init(paes_s390_init); 61727937843SMartin Schwidefsky module_exit(paes_s390_fini); 61827937843SMartin Schwidefsky 61927937843SMartin Schwidefsky MODULE_ALIAS_CRYPTO("aes-all"); 62027937843SMartin Schwidefsky 62127937843SMartin Schwidefsky MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys"); 62227937843SMartin Schwidefsky MODULE_LICENSE("GPL"); 623