15068c7a8SSteffen Klassert /* 25068c7a8SSteffen Klassert * pcrypt - Parallel crypto wrapper. 35068c7a8SSteffen Klassert * 45068c7a8SSteffen Klassert * Copyright (C) 2009 secunet Security Networks AG 55068c7a8SSteffen Klassert * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com> 65068c7a8SSteffen Klassert * 75068c7a8SSteffen Klassert * This program is free software; you can redistribute it and/or modify it 85068c7a8SSteffen Klassert * under the terms and conditions of the GNU General Public License, 95068c7a8SSteffen Klassert * version 2, as published by the Free Software Foundation. 105068c7a8SSteffen Klassert * 115068c7a8SSteffen Klassert * This program is distributed in the hope it will be useful, but WITHOUT 125068c7a8SSteffen Klassert * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 135068c7a8SSteffen Klassert * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 145068c7a8SSteffen Klassert * more details. 155068c7a8SSteffen Klassert * 165068c7a8SSteffen Klassert * You should have received a copy of the GNU General Public License along with 175068c7a8SSteffen Klassert * this program; if not, write to the Free Software Foundation, Inc., 185068c7a8SSteffen Klassert * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 195068c7a8SSteffen Klassert */ 205068c7a8SSteffen Klassert 215068c7a8SSteffen Klassert #include <crypto/algapi.h> 225068c7a8SSteffen Klassert #include <crypto/internal/aead.h> 23a5a22e57SHerbert Xu #include <linux/atomic.h> 245068c7a8SSteffen Klassert #include <linux/err.h> 255068c7a8SSteffen Klassert #include <linux/init.h> 265068c7a8SSteffen Klassert #include <linux/module.h> 275068c7a8SSteffen Klassert #include <linux/slab.h> 28e15bacbeSDan Kruchinin #include <linux/notifier.h> 29a3fb1e33SDan Kruchinin #include <linux/kobject.h> 30d3f64e46SSteffen Klassert #include <linux/cpu.h> 315068c7a8SSteffen Klassert #include <crypto/pcrypt.h> 325068c7a8SSteffen Klassert 33c57e842eSSteffen Klassert struct padata_pcrypt { 34e15bacbeSDan Kruchinin struct padata_instance *pinst; 35e15bacbeSDan Kruchinin struct workqueue_struct *wq; 36e15bacbeSDan Kruchinin 37e15bacbeSDan Kruchinin /* 38e15bacbeSDan Kruchinin * Cpumask for callback CPUs. It should be 39e15bacbeSDan Kruchinin * equal to serial cpumask of corresponding padata instance, 40e15bacbeSDan Kruchinin * so it is updated when padata notifies us about serial 41e15bacbeSDan Kruchinin * cpumask change. 42e15bacbeSDan Kruchinin * 43e15bacbeSDan Kruchinin * cb_cpumask is protected by RCU. This fact prevents us from 44e15bacbeSDan Kruchinin * using cpumask_var_t directly because the actual type of 45e15bacbeSDan Kruchinin * cpumsak_var_t depends on kernel configuration(particularly on 46e15bacbeSDan Kruchinin * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration 47e15bacbeSDan Kruchinin * cpumask_var_t may be either a pointer to the struct cpumask 48e15bacbeSDan Kruchinin * or a variable allocated on the stack. Thus we can not safely use 49e15bacbeSDan Kruchinin * cpumask_var_t with RCU operations such as rcu_assign_pointer or 50e15bacbeSDan Kruchinin * rcu_dereference. So cpumask_var_t is wrapped with struct 51e15bacbeSDan Kruchinin * pcrypt_cpumask which makes possible to use it with RCU. 52e15bacbeSDan Kruchinin */ 53e15bacbeSDan Kruchinin struct pcrypt_cpumask { 54e15bacbeSDan Kruchinin cpumask_var_t mask; 55e15bacbeSDan Kruchinin } *cb_cpumask; 56e15bacbeSDan Kruchinin struct notifier_block nblock; 57e15bacbeSDan Kruchinin }; 58e15bacbeSDan Kruchinin 59c57e842eSSteffen Klassert static struct padata_pcrypt pencrypt; 60c57e842eSSteffen Klassert static struct padata_pcrypt pdecrypt; 61a3fb1e33SDan Kruchinin static struct kset *pcrypt_kset; 625068c7a8SSteffen Klassert 635068c7a8SSteffen Klassert struct pcrypt_instance_ctx { 6466d948e7SHerbert Xu struct crypto_aead_spawn spawn; 65a5a22e57SHerbert Xu atomic_t tfm_count; 665068c7a8SSteffen Klassert }; 675068c7a8SSteffen Klassert 685068c7a8SSteffen Klassert struct pcrypt_aead_ctx { 695068c7a8SSteffen Klassert struct crypto_aead *child; 705068c7a8SSteffen Klassert unsigned int cb_cpu; 715068c7a8SSteffen Klassert }; 725068c7a8SSteffen Klassert 735068c7a8SSteffen Klassert static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, 74c57e842eSSteffen Klassert struct padata_pcrypt *pcrypt) 755068c7a8SSteffen Klassert { 765068c7a8SSteffen Klassert unsigned int cpu_index, cpu, i; 77e15bacbeSDan Kruchinin struct pcrypt_cpumask *cpumask; 785068c7a8SSteffen Klassert 795068c7a8SSteffen Klassert cpu = *cb_cpu; 805068c7a8SSteffen Klassert 81e15bacbeSDan Kruchinin rcu_read_lock_bh(); 823110e400SMathias Krause cpumask = rcu_dereference_bh(pcrypt->cb_cpumask); 83e15bacbeSDan Kruchinin if (cpumask_test_cpu(cpu, cpumask->mask)) 845068c7a8SSteffen Klassert goto out; 855068c7a8SSteffen Klassert 86cc74f4bcSSteffen Klassert if (!cpumask_weight(cpumask->mask)) 87cc74f4bcSSteffen Klassert goto out; 88cc74f4bcSSteffen Klassert 89e15bacbeSDan Kruchinin cpu_index = cpu % cpumask_weight(cpumask->mask); 905068c7a8SSteffen Klassert 91e15bacbeSDan Kruchinin cpu = cpumask_first(cpumask->mask); 925068c7a8SSteffen Klassert for (i = 0; i < cpu_index; i++) 93e15bacbeSDan Kruchinin cpu = cpumask_next(cpu, cpumask->mask); 945068c7a8SSteffen Klassert 955068c7a8SSteffen Klassert *cb_cpu = cpu; 965068c7a8SSteffen Klassert 975068c7a8SSteffen Klassert out: 98e15bacbeSDan Kruchinin rcu_read_unlock_bh(); 99e15bacbeSDan Kruchinin return padata_do_parallel(pcrypt->pinst, padata, cpu); 1005068c7a8SSteffen Klassert } 1015068c7a8SSteffen Klassert 1025068c7a8SSteffen Klassert static int pcrypt_aead_setkey(struct crypto_aead *parent, 1035068c7a8SSteffen Klassert const u8 *key, unsigned int keylen) 1045068c7a8SSteffen Klassert { 1055068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); 1065068c7a8SSteffen Klassert 1075068c7a8SSteffen Klassert return crypto_aead_setkey(ctx->child, key, keylen); 1085068c7a8SSteffen Klassert } 1095068c7a8SSteffen Klassert 1105068c7a8SSteffen Klassert static int pcrypt_aead_setauthsize(struct crypto_aead *parent, 1115068c7a8SSteffen Klassert unsigned int authsize) 1125068c7a8SSteffen Klassert { 1135068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); 1145068c7a8SSteffen Klassert 1155068c7a8SSteffen Klassert return crypto_aead_setauthsize(ctx->child, authsize); 1165068c7a8SSteffen Klassert } 1175068c7a8SSteffen Klassert 1185068c7a8SSteffen Klassert static void pcrypt_aead_serial(struct padata_priv *padata) 1195068c7a8SSteffen Klassert { 1205068c7a8SSteffen Klassert struct pcrypt_request *preq = pcrypt_padata_request(padata); 1215068c7a8SSteffen Klassert struct aead_request *req = pcrypt_request_ctx(preq); 1225068c7a8SSteffen Klassert 1235068c7a8SSteffen Klassert aead_request_complete(req->base.data, padata->info); 1245068c7a8SSteffen Klassert } 1255068c7a8SSteffen Klassert 1265068c7a8SSteffen Klassert static void pcrypt_aead_done(struct crypto_async_request *areq, int err) 1275068c7a8SSteffen Klassert { 1285068c7a8SSteffen Klassert struct aead_request *req = areq->data; 1295068c7a8SSteffen Klassert struct pcrypt_request *preq = aead_request_ctx(req); 1305068c7a8SSteffen Klassert struct padata_priv *padata = pcrypt_request_padata(preq); 1315068c7a8SSteffen Klassert 1325068c7a8SSteffen Klassert padata->info = err; 1335068c7a8SSteffen Klassert req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 1345068c7a8SSteffen Klassert 1355068c7a8SSteffen Klassert padata_do_serial(padata); 1365068c7a8SSteffen Klassert } 1375068c7a8SSteffen Klassert 1385068c7a8SSteffen Klassert static void pcrypt_aead_enc(struct padata_priv *padata) 1395068c7a8SSteffen Klassert { 1405068c7a8SSteffen Klassert struct pcrypt_request *preq = pcrypt_padata_request(padata); 1415068c7a8SSteffen Klassert struct aead_request *req = pcrypt_request_ctx(preq); 1425068c7a8SSteffen Klassert 1435068c7a8SSteffen Klassert padata->info = crypto_aead_encrypt(req); 1445068c7a8SSteffen Klassert 1455a1436beSSteffen Klassert if (padata->info == -EINPROGRESS) 1465068c7a8SSteffen Klassert return; 1475068c7a8SSteffen Klassert 1485068c7a8SSteffen Klassert padata_do_serial(padata); 1495068c7a8SSteffen Klassert } 1505068c7a8SSteffen Klassert 1515068c7a8SSteffen Klassert static int pcrypt_aead_encrypt(struct aead_request *req) 1525068c7a8SSteffen Klassert { 1535068c7a8SSteffen Klassert int err; 1545068c7a8SSteffen Klassert struct pcrypt_request *preq = aead_request_ctx(req); 1555068c7a8SSteffen Klassert struct aead_request *creq = pcrypt_request_ctx(preq); 1565068c7a8SSteffen Klassert struct padata_priv *padata = pcrypt_request_padata(preq); 1575068c7a8SSteffen Klassert struct crypto_aead *aead = crypto_aead_reqtfm(req); 1585068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 1595068c7a8SSteffen Klassert u32 flags = aead_request_flags(req); 1605068c7a8SSteffen Klassert 1615068c7a8SSteffen Klassert memset(padata, 0, sizeof(struct padata_priv)); 1625068c7a8SSteffen Klassert 1635068c7a8SSteffen Klassert padata->parallel = pcrypt_aead_enc; 1645068c7a8SSteffen Klassert padata->serial = pcrypt_aead_serial; 1655068c7a8SSteffen Klassert 1665068c7a8SSteffen Klassert aead_request_set_tfm(creq, ctx->child); 1675068c7a8SSteffen Klassert aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, 1685068c7a8SSteffen Klassert pcrypt_aead_done, req); 1695068c7a8SSteffen Klassert aead_request_set_crypt(creq, req->src, req->dst, 1705068c7a8SSteffen Klassert req->cryptlen, req->iv); 1710496f560SHerbert Xu aead_request_set_ad(creq, req->assoclen); 1725068c7a8SSteffen Klassert 173e15bacbeSDan Kruchinin err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); 17483f619f3SSteffen Klassert if (!err) 17583f619f3SSteffen Klassert return -EINPROGRESS; 1765068c7a8SSteffen Klassert 1775068c7a8SSteffen Klassert return err; 1785068c7a8SSteffen Klassert } 1795068c7a8SSteffen Klassert 1805068c7a8SSteffen Klassert static void pcrypt_aead_dec(struct padata_priv *padata) 1815068c7a8SSteffen Klassert { 1825068c7a8SSteffen Klassert struct pcrypt_request *preq = pcrypt_padata_request(padata); 1835068c7a8SSteffen Klassert struct aead_request *req = pcrypt_request_ctx(preq); 1845068c7a8SSteffen Klassert 1855068c7a8SSteffen Klassert padata->info = crypto_aead_decrypt(req); 1865068c7a8SSteffen Klassert 1875a1436beSSteffen Klassert if (padata->info == -EINPROGRESS) 1885068c7a8SSteffen Klassert return; 1895068c7a8SSteffen Klassert 1905068c7a8SSteffen Klassert padata_do_serial(padata); 1915068c7a8SSteffen Klassert } 1925068c7a8SSteffen Klassert 1935068c7a8SSteffen Klassert static int pcrypt_aead_decrypt(struct aead_request *req) 1945068c7a8SSteffen Klassert { 1955068c7a8SSteffen Klassert int err; 1965068c7a8SSteffen Klassert struct pcrypt_request *preq = aead_request_ctx(req); 1975068c7a8SSteffen Klassert struct aead_request *creq = pcrypt_request_ctx(preq); 1985068c7a8SSteffen Klassert struct padata_priv *padata = pcrypt_request_padata(preq); 1995068c7a8SSteffen Klassert struct crypto_aead *aead = crypto_aead_reqtfm(req); 2005068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 2015068c7a8SSteffen Klassert u32 flags = aead_request_flags(req); 2025068c7a8SSteffen Klassert 2035068c7a8SSteffen Klassert memset(padata, 0, sizeof(struct padata_priv)); 2045068c7a8SSteffen Klassert 2055068c7a8SSteffen Klassert padata->parallel = pcrypt_aead_dec; 2065068c7a8SSteffen Klassert padata->serial = pcrypt_aead_serial; 2075068c7a8SSteffen Klassert 2085068c7a8SSteffen Klassert aead_request_set_tfm(creq, ctx->child); 2095068c7a8SSteffen Klassert aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, 2105068c7a8SSteffen Klassert pcrypt_aead_done, req); 2115068c7a8SSteffen Klassert aead_request_set_crypt(creq, req->src, req->dst, 2125068c7a8SSteffen Klassert req->cryptlen, req->iv); 2130496f560SHerbert Xu aead_request_set_ad(creq, req->assoclen); 2145068c7a8SSteffen Klassert 215e15bacbeSDan Kruchinin err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); 21683f619f3SSteffen Klassert if (!err) 21783f619f3SSteffen Klassert return -EINPROGRESS; 2185068c7a8SSteffen Klassert 2195068c7a8SSteffen Klassert return err; 2205068c7a8SSteffen Klassert } 2215068c7a8SSteffen Klassert 2220496f560SHerbert Xu static int pcrypt_aead_init_tfm(struct crypto_aead *tfm) 2235068c7a8SSteffen Klassert { 2245068c7a8SSteffen Klassert int cpu, cpu_index; 2250496f560SHerbert Xu struct aead_instance *inst = aead_alg_instance(tfm); 2260496f560SHerbert Xu struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst); 2270496f560SHerbert Xu struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm); 2285068c7a8SSteffen Klassert struct crypto_aead *cipher; 2295068c7a8SSteffen Klassert 230a5a22e57SHerbert Xu cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) % 231a5a22e57SHerbert Xu cpumask_weight(cpu_online_mask); 2325068c7a8SSteffen Klassert 233fbf0ca1bSSteffen Klassert ctx->cb_cpu = cpumask_first(cpu_online_mask); 2345068c7a8SSteffen Klassert for (cpu = 0; cpu < cpu_index; cpu++) 235fbf0ca1bSSteffen Klassert ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask); 2365068c7a8SSteffen Klassert 2370496f560SHerbert Xu cipher = crypto_spawn_aead(&ictx->spawn); 2385068c7a8SSteffen Klassert 2395068c7a8SSteffen Klassert if (IS_ERR(cipher)) 2405068c7a8SSteffen Klassert return PTR_ERR(cipher); 2415068c7a8SSteffen Klassert 2425068c7a8SSteffen Klassert ctx->child = cipher; 2430496f560SHerbert Xu crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) + 2440496f560SHerbert Xu sizeof(struct aead_request) + 245fd0de978SHerbert Xu crypto_aead_reqsize(cipher)); 2465068c7a8SSteffen Klassert 2475068c7a8SSteffen Klassert return 0; 2485068c7a8SSteffen Klassert } 2495068c7a8SSteffen Klassert 2500496f560SHerbert Xu static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm) 2515068c7a8SSteffen Klassert { 2520496f560SHerbert Xu struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm); 2535068c7a8SSteffen Klassert 2545068c7a8SSteffen Klassert crypto_free_aead(ctx->child); 2555068c7a8SSteffen Klassert } 2565068c7a8SSteffen Klassert 25766d948e7SHerbert Xu static int pcrypt_init_instance(struct crypto_instance *inst, 25866d948e7SHerbert Xu struct crypto_alg *alg) 2595068c7a8SSteffen Klassert { 2605068c7a8SSteffen Klassert if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 2615068c7a8SSteffen Klassert "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 26266d948e7SHerbert Xu return -ENAMETOOLONG; 2635068c7a8SSteffen Klassert 2645068c7a8SSteffen Klassert memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 2655068c7a8SSteffen Klassert 2665068c7a8SSteffen Klassert inst->alg.cra_priority = alg->cra_priority + 100; 2675068c7a8SSteffen Klassert inst->alg.cra_blocksize = alg->cra_blocksize; 2685068c7a8SSteffen Klassert inst->alg.cra_alignmask = alg->cra_alignmask; 2695068c7a8SSteffen Klassert 27066d948e7SHerbert Xu return 0; 2715068c7a8SSteffen Klassert } 2725068c7a8SSteffen Klassert 2730496f560SHerbert Xu static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb, 27480a6d7dbSDan Carpenter u32 type, u32 mask) 2755068c7a8SSteffen Klassert { 27666d948e7SHerbert Xu struct pcrypt_instance_ctx *ctx; 277846f97dfSHerbert Xu struct crypto_attr_type *algt; 2780496f560SHerbert Xu struct aead_instance *inst; 2790496f560SHerbert Xu struct aead_alg *alg; 28066d948e7SHerbert Xu const char *name; 28166d948e7SHerbert Xu int err; 2825068c7a8SSteffen Klassert 283846f97dfSHerbert Xu algt = crypto_get_attr_type(tb); 284846f97dfSHerbert Xu if (IS_ERR(algt)) 285846f97dfSHerbert Xu return PTR_ERR(algt); 286846f97dfSHerbert Xu 28766d948e7SHerbert Xu name = crypto_attr_alg_name(tb[1]); 28866d948e7SHerbert Xu if (IS_ERR(name)) 2890496f560SHerbert Xu return PTR_ERR(name); 2905068c7a8SSteffen Klassert 29166d948e7SHerbert Xu inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 29266d948e7SHerbert Xu if (!inst) 2930496f560SHerbert Xu return -ENOMEM; 29466d948e7SHerbert Xu 2950496f560SHerbert Xu ctx = aead_instance_ctx(inst); 2960496f560SHerbert Xu crypto_set_aead_spawn(&ctx->spawn, aead_crypto_instance(inst)); 29766d948e7SHerbert Xu 298846f97dfSHerbert Xu err = crypto_grab_aead(&ctx->spawn, name, 299846f97dfSHerbert Xu algt->type & CRYPTO_ALG_AEAD_NEW, 300846f97dfSHerbert Xu algt->mask & CRYPTO_ALG_AEAD_NEW); 30166d948e7SHerbert Xu if (err) 30266d948e7SHerbert Xu goto out_free_inst; 30366d948e7SHerbert Xu 3040496f560SHerbert Xu alg = crypto_spawn_aead_alg(&ctx->spawn); 3050496f560SHerbert Xu err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base); 30666d948e7SHerbert Xu if (err) 30766d948e7SHerbert Xu goto out_drop_aead; 3085068c7a8SSteffen Klassert 309846f97dfSHerbert Xu inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC; 310846f97dfSHerbert Xu inst->alg.base.cra_flags |= alg->base.cra_flags & CRYPTO_ALG_AEAD_NEW; 311846f97dfSHerbert Xu 3120496f560SHerbert Xu inst->alg.ivsize = crypto_aead_alg_ivsize(alg); 3130496f560SHerbert Xu inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 3145068c7a8SSteffen Klassert 3150496f560SHerbert Xu inst->alg.base.cra_ctxsize = sizeof(struct pcrypt_aead_ctx); 3165068c7a8SSteffen Klassert 3170496f560SHerbert Xu inst->alg.init = pcrypt_aead_init_tfm; 3180496f560SHerbert Xu inst->alg.exit = pcrypt_aead_exit_tfm; 3195068c7a8SSteffen Klassert 3200496f560SHerbert Xu inst->alg.setkey = pcrypt_aead_setkey; 3210496f560SHerbert Xu inst->alg.setauthsize = pcrypt_aead_setauthsize; 3220496f560SHerbert Xu inst->alg.encrypt = pcrypt_aead_encrypt; 3230496f560SHerbert Xu inst->alg.decrypt = pcrypt_aead_decrypt; 3245068c7a8SSteffen Klassert 3250496f560SHerbert Xu err = aead_register_instance(tmpl, inst); 3260496f560SHerbert Xu if (err) 3270496f560SHerbert Xu goto out_drop_aead; 3285068c7a8SSteffen Klassert 32966d948e7SHerbert Xu out: 3300496f560SHerbert Xu return err; 33166d948e7SHerbert Xu 33266d948e7SHerbert Xu out_drop_aead: 33366d948e7SHerbert Xu crypto_drop_aead(&ctx->spawn); 33466d948e7SHerbert Xu out_free_inst: 33566d948e7SHerbert Xu kfree(inst); 33666d948e7SHerbert Xu goto out; 3375068c7a8SSteffen Klassert } 3385068c7a8SSteffen Klassert 3390496f560SHerbert Xu static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb) 3405068c7a8SSteffen Klassert { 3415068c7a8SSteffen Klassert struct crypto_attr_type *algt; 3425068c7a8SSteffen Klassert 3435068c7a8SSteffen Klassert algt = crypto_get_attr_type(tb); 3445068c7a8SSteffen Klassert if (IS_ERR(algt)) 3450496f560SHerbert Xu return PTR_ERR(algt); 3465068c7a8SSteffen Klassert 3475068c7a8SSteffen Klassert switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 3485068c7a8SSteffen Klassert case CRYPTO_ALG_TYPE_AEAD: 3490496f560SHerbert Xu return pcrypt_create_aead(tmpl, tb, algt->type, algt->mask); 3505068c7a8SSteffen Klassert } 3515068c7a8SSteffen Klassert 3520496f560SHerbert Xu return -EINVAL; 3535068c7a8SSteffen Klassert } 3545068c7a8SSteffen Klassert 3555068c7a8SSteffen Klassert static void pcrypt_free(struct crypto_instance *inst) 3565068c7a8SSteffen Klassert { 3575068c7a8SSteffen Klassert struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst); 3585068c7a8SSteffen Klassert 35966d948e7SHerbert Xu crypto_drop_aead(&ctx->spawn); 3605068c7a8SSteffen Klassert kfree(inst); 3615068c7a8SSteffen Klassert } 3625068c7a8SSteffen Klassert 363e15bacbeSDan Kruchinin static int pcrypt_cpumask_change_notify(struct notifier_block *self, 364e15bacbeSDan Kruchinin unsigned long val, void *data) 365e15bacbeSDan Kruchinin { 366c57e842eSSteffen Klassert struct padata_pcrypt *pcrypt; 367e15bacbeSDan Kruchinin struct pcrypt_cpumask *new_mask, *old_mask; 368d3f64e46SSteffen Klassert struct padata_cpumask *cpumask = (struct padata_cpumask *)data; 369e15bacbeSDan Kruchinin 370e15bacbeSDan Kruchinin if (!(val & PADATA_CPU_SERIAL)) 371e15bacbeSDan Kruchinin return 0; 372e15bacbeSDan Kruchinin 373c57e842eSSteffen Klassert pcrypt = container_of(self, struct padata_pcrypt, nblock); 374e15bacbeSDan Kruchinin new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); 375e15bacbeSDan Kruchinin if (!new_mask) 376e15bacbeSDan Kruchinin return -ENOMEM; 377e15bacbeSDan Kruchinin if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) { 378e15bacbeSDan Kruchinin kfree(new_mask); 379e15bacbeSDan Kruchinin return -ENOMEM; 380e15bacbeSDan Kruchinin } 381e15bacbeSDan Kruchinin 382e15bacbeSDan Kruchinin old_mask = pcrypt->cb_cpumask; 383e15bacbeSDan Kruchinin 384d3f64e46SSteffen Klassert cpumask_copy(new_mask->mask, cpumask->cbcpu); 385e15bacbeSDan Kruchinin rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); 386e15bacbeSDan Kruchinin synchronize_rcu_bh(); 387e15bacbeSDan Kruchinin 388e15bacbeSDan Kruchinin free_cpumask_var(old_mask->mask); 389e15bacbeSDan Kruchinin kfree(old_mask); 390e15bacbeSDan Kruchinin return 0; 391e15bacbeSDan Kruchinin } 392e15bacbeSDan Kruchinin 393a3fb1e33SDan Kruchinin static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) 394a3fb1e33SDan Kruchinin { 395a3fb1e33SDan Kruchinin int ret; 396a3fb1e33SDan Kruchinin 397a3fb1e33SDan Kruchinin pinst->kobj.kset = pcrypt_kset; 398a3fb1e33SDan Kruchinin ret = kobject_add(&pinst->kobj, NULL, name); 399a3fb1e33SDan Kruchinin if (!ret) 400a3fb1e33SDan Kruchinin kobject_uevent(&pinst->kobj, KOBJ_ADD); 401a3fb1e33SDan Kruchinin 402a3fb1e33SDan Kruchinin return ret; 403a3fb1e33SDan Kruchinin } 404a3fb1e33SDan Kruchinin 405c57e842eSSteffen Klassert static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, 406e15bacbeSDan Kruchinin const char *name) 407e15bacbeSDan Kruchinin { 408e15bacbeSDan Kruchinin int ret = -ENOMEM; 409e15bacbeSDan Kruchinin struct pcrypt_cpumask *mask; 410e15bacbeSDan Kruchinin 411d3f64e46SSteffen Klassert get_online_cpus(); 412d3f64e46SSteffen Klassert 413d8537548SKees Cook pcrypt->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 414d8537548SKees Cook 1, name); 415e15bacbeSDan Kruchinin if (!pcrypt->wq) 416e15bacbeSDan Kruchinin goto err; 417e15bacbeSDan Kruchinin 418e6cc1170SSteffen Klassert pcrypt->pinst = padata_alloc_possible(pcrypt->wq); 419e15bacbeSDan Kruchinin if (!pcrypt->pinst) 420e15bacbeSDan Kruchinin goto err_destroy_workqueue; 421e15bacbeSDan Kruchinin 422e15bacbeSDan Kruchinin mask = kmalloc(sizeof(*mask), GFP_KERNEL); 423e15bacbeSDan Kruchinin if (!mask) 424e15bacbeSDan Kruchinin goto err_free_padata; 425e15bacbeSDan Kruchinin if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { 426e15bacbeSDan Kruchinin kfree(mask); 427e15bacbeSDan Kruchinin goto err_free_padata; 428e15bacbeSDan Kruchinin } 429e15bacbeSDan Kruchinin 430fbf0ca1bSSteffen Klassert cpumask_and(mask->mask, cpu_possible_mask, cpu_online_mask); 431e15bacbeSDan Kruchinin rcu_assign_pointer(pcrypt->cb_cpumask, mask); 432e15bacbeSDan Kruchinin 433e15bacbeSDan Kruchinin pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; 434e15bacbeSDan Kruchinin ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 435e15bacbeSDan Kruchinin if (ret) 436e15bacbeSDan Kruchinin goto err_free_cpumask; 437e15bacbeSDan Kruchinin 438a3fb1e33SDan Kruchinin ret = pcrypt_sysfs_add(pcrypt->pinst, name); 439a3fb1e33SDan Kruchinin if (ret) 440a3fb1e33SDan Kruchinin goto err_unregister_notifier; 441a3fb1e33SDan Kruchinin 442d3f64e46SSteffen Klassert put_online_cpus(); 443d3f64e46SSteffen Klassert 444e15bacbeSDan Kruchinin return ret; 445d3f64e46SSteffen Klassert 446a3fb1e33SDan Kruchinin err_unregister_notifier: 447a3fb1e33SDan Kruchinin padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 448e15bacbeSDan Kruchinin err_free_cpumask: 449e15bacbeSDan Kruchinin free_cpumask_var(mask->mask); 450e15bacbeSDan Kruchinin kfree(mask); 451e15bacbeSDan Kruchinin err_free_padata: 452e15bacbeSDan Kruchinin padata_free(pcrypt->pinst); 453e15bacbeSDan Kruchinin err_destroy_workqueue: 454e15bacbeSDan Kruchinin destroy_workqueue(pcrypt->wq); 455e15bacbeSDan Kruchinin err: 456d3f64e46SSteffen Klassert put_online_cpus(); 457d3f64e46SSteffen Klassert 458e15bacbeSDan Kruchinin return ret; 459e15bacbeSDan Kruchinin } 460e15bacbeSDan Kruchinin 461c57e842eSSteffen Klassert static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) 462e15bacbeSDan Kruchinin { 463e15bacbeSDan Kruchinin free_cpumask_var(pcrypt->cb_cpumask->mask); 464e15bacbeSDan Kruchinin kfree(pcrypt->cb_cpumask); 465e15bacbeSDan Kruchinin 466e15bacbeSDan Kruchinin padata_stop(pcrypt->pinst); 467e15bacbeSDan Kruchinin padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 468e15bacbeSDan Kruchinin destroy_workqueue(pcrypt->wq); 469e15bacbeSDan Kruchinin padata_free(pcrypt->pinst); 470e15bacbeSDan Kruchinin } 471e15bacbeSDan Kruchinin 4725068c7a8SSteffen Klassert static struct crypto_template pcrypt_tmpl = { 4735068c7a8SSteffen Klassert .name = "pcrypt", 4740496f560SHerbert Xu .create = pcrypt_create, 4755068c7a8SSteffen Klassert .free = pcrypt_free, 4765068c7a8SSteffen Klassert .module = THIS_MODULE, 4775068c7a8SSteffen Klassert }; 4785068c7a8SSteffen Klassert 4795068c7a8SSteffen Klassert static int __init pcrypt_init(void) 4805068c7a8SSteffen Klassert { 481a3fb1e33SDan Kruchinin int err = -ENOMEM; 482a3fb1e33SDan Kruchinin 483a3fb1e33SDan Kruchinin pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj); 484a3fb1e33SDan Kruchinin if (!pcrypt_kset) 485a3fb1e33SDan Kruchinin goto err; 486e15bacbeSDan Kruchinin 487c57e842eSSteffen Klassert err = pcrypt_init_padata(&pencrypt, "pencrypt"); 488e15bacbeSDan Kruchinin if (err) 489a3fb1e33SDan Kruchinin goto err_unreg_kset; 4905068c7a8SSteffen Klassert 491c57e842eSSteffen Klassert err = pcrypt_init_padata(&pdecrypt, "pdecrypt"); 4924c879170SSteffen Klassert if (err) 493e15bacbeSDan Kruchinin goto err_deinit_pencrypt; 4944c879170SSteffen Klassert 495e15bacbeSDan Kruchinin padata_start(pencrypt.pinst); 496e15bacbeSDan Kruchinin padata_start(pdecrypt.pinst); 4975068c7a8SSteffen Klassert 4985068c7a8SSteffen Klassert return crypto_register_template(&pcrypt_tmpl); 4995068c7a8SSteffen Klassert 500e15bacbeSDan Kruchinin err_deinit_pencrypt: 501c57e842eSSteffen Klassert pcrypt_fini_padata(&pencrypt); 502a3fb1e33SDan Kruchinin err_unreg_kset: 503a3fb1e33SDan Kruchinin kset_unregister(pcrypt_kset); 5045068c7a8SSteffen Klassert err: 5054c879170SSteffen Klassert return err; 5065068c7a8SSteffen Klassert } 5075068c7a8SSteffen Klassert 5085068c7a8SSteffen Klassert static void __exit pcrypt_exit(void) 5095068c7a8SSteffen Klassert { 510c57e842eSSteffen Klassert pcrypt_fini_padata(&pencrypt); 511c57e842eSSteffen Klassert pcrypt_fini_padata(&pdecrypt); 5125068c7a8SSteffen Klassert 513a3fb1e33SDan Kruchinin kset_unregister(pcrypt_kset); 5145068c7a8SSteffen Klassert crypto_unregister_template(&pcrypt_tmpl); 5155068c7a8SSteffen Klassert } 5165068c7a8SSteffen Klassert 5175068c7a8SSteffen Klassert module_init(pcrypt_init); 5185068c7a8SSteffen Klassert module_exit(pcrypt_exit); 5195068c7a8SSteffen Klassert 5205068c7a8SSteffen Klassert MODULE_LICENSE("GPL"); 5215068c7a8SSteffen Klassert MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); 5225068c7a8SSteffen Klassert MODULE_DESCRIPTION("Parallel crypto wrapper"); 5234943ba16SKees Cook MODULE_ALIAS_CRYPTO("pcrypt"); 524