1a61127c2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 25068c7a8SSteffen Klassert /* 35068c7a8SSteffen Klassert * pcrypt - Parallel crypto wrapper. 45068c7a8SSteffen Klassert * 55068c7a8SSteffen Klassert * Copyright (C) 2009 secunet Security Networks AG 65068c7a8SSteffen Klassert * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com> 75068c7a8SSteffen Klassert */ 85068c7a8SSteffen Klassert 95068c7a8SSteffen Klassert #include <crypto/algapi.h> 105068c7a8SSteffen Klassert #include <crypto/internal/aead.h> 11a5a22e57SHerbert Xu #include <linux/atomic.h> 125068c7a8SSteffen Klassert #include <linux/err.h> 135068c7a8SSteffen Klassert #include <linux/init.h> 145068c7a8SSteffen Klassert #include <linux/module.h> 155068c7a8SSteffen Klassert #include <linux/slab.h> 16e15bacbeSDan Kruchinin #include <linux/notifier.h> 17a3fb1e33SDan Kruchinin #include <linux/kobject.h> 18d3f64e46SSteffen Klassert #include <linux/cpu.h> 195068c7a8SSteffen Klassert #include <crypto/pcrypt.h> 205068c7a8SSteffen Klassert 21c57e842eSSteffen Klassert struct padata_pcrypt { 22e15bacbeSDan Kruchinin struct padata_instance *pinst; 23e15bacbeSDan Kruchinin struct workqueue_struct *wq; 24e15bacbeSDan Kruchinin 25e15bacbeSDan Kruchinin /* 26e15bacbeSDan Kruchinin * Cpumask for callback CPUs. It should be 27e15bacbeSDan Kruchinin * equal to serial cpumask of corresponding padata instance, 28e15bacbeSDan Kruchinin * so it is updated when padata notifies us about serial 29e15bacbeSDan Kruchinin * cpumask change. 30e15bacbeSDan Kruchinin * 31e15bacbeSDan Kruchinin * cb_cpumask is protected by RCU. This fact prevents us from 32e15bacbeSDan Kruchinin * using cpumask_var_t directly because the actual type of 33e15bacbeSDan Kruchinin * cpumsak_var_t depends on kernel configuration(particularly on 34e15bacbeSDan Kruchinin * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration 35e15bacbeSDan Kruchinin * cpumask_var_t may be either a pointer to the struct cpumask 36e15bacbeSDan Kruchinin * or a variable allocated on the stack. Thus we can not safely use 37e15bacbeSDan Kruchinin * cpumask_var_t with RCU operations such as rcu_assign_pointer or 38e15bacbeSDan Kruchinin * rcu_dereference. So cpumask_var_t is wrapped with struct 39e15bacbeSDan Kruchinin * pcrypt_cpumask which makes possible to use it with RCU. 40e15bacbeSDan Kruchinin */ 41e15bacbeSDan Kruchinin struct pcrypt_cpumask { 42e15bacbeSDan Kruchinin cpumask_var_t mask; 43e15bacbeSDan Kruchinin } *cb_cpumask; 44e15bacbeSDan Kruchinin struct notifier_block nblock; 45e15bacbeSDan Kruchinin }; 46e15bacbeSDan Kruchinin 47c57e842eSSteffen Klassert static struct padata_pcrypt pencrypt; 48c57e842eSSteffen Klassert static struct padata_pcrypt pdecrypt; 49a3fb1e33SDan Kruchinin static struct kset *pcrypt_kset; 505068c7a8SSteffen Klassert 515068c7a8SSteffen Klassert struct pcrypt_instance_ctx { 5266d948e7SHerbert Xu struct crypto_aead_spawn spawn; 53a5a22e57SHerbert Xu atomic_t tfm_count; 545068c7a8SSteffen Klassert }; 555068c7a8SSteffen Klassert 565068c7a8SSteffen Klassert struct pcrypt_aead_ctx { 575068c7a8SSteffen Klassert struct crypto_aead *child; 585068c7a8SSteffen Klassert unsigned int cb_cpu; 595068c7a8SSteffen Klassert }; 605068c7a8SSteffen Klassert 615068c7a8SSteffen Klassert static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, 62c57e842eSSteffen Klassert struct padata_pcrypt *pcrypt) 635068c7a8SSteffen Klassert { 645068c7a8SSteffen Klassert unsigned int cpu_index, cpu, i; 65e15bacbeSDan Kruchinin struct pcrypt_cpumask *cpumask; 665068c7a8SSteffen Klassert 675068c7a8SSteffen Klassert cpu = *cb_cpu; 685068c7a8SSteffen Klassert 69e15bacbeSDan Kruchinin rcu_read_lock_bh(); 703110e400SMathias Krause cpumask = rcu_dereference_bh(pcrypt->cb_cpumask); 71e15bacbeSDan Kruchinin if (cpumask_test_cpu(cpu, cpumask->mask)) 725068c7a8SSteffen Klassert goto out; 735068c7a8SSteffen Klassert 74cc74f4bcSSteffen Klassert if (!cpumask_weight(cpumask->mask)) 75cc74f4bcSSteffen Klassert goto out; 76cc74f4bcSSteffen Klassert 77e15bacbeSDan Kruchinin cpu_index = cpu % cpumask_weight(cpumask->mask); 785068c7a8SSteffen Klassert 79e15bacbeSDan Kruchinin cpu = cpumask_first(cpumask->mask); 805068c7a8SSteffen Klassert for (i = 0; i < cpu_index; i++) 81e15bacbeSDan Kruchinin cpu = cpumask_next(cpu, cpumask->mask); 825068c7a8SSteffen Klassert 835068c7a8SSteffen Klassert *cb_cpu = cpu; 845068c7a8SSteffen Klassert 855068c7a8SSteffen Klassert out: 86e15bacbeSDan Kruchinin rcu_read_unlock_bh(); 87e15bacbeSDan Kruchinin return padata_do_parallel(pcrypt->pinst, padata, cpu); 885068c7a8SSteffen Klassert } 895068c7a8SSteffen Klassert 905068c7a8SSteffen Klassert static int pcrypt_aead_setkey(struct crypto_aead *parent, 915068c7a8SSteffen Klassert const u8 *key, unsigned int keylen) 925068c7a8SSteffen Klassert { 935068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); 945068c7a8SSteffen Klassert 955068c7a8SSteffen Klassert return crypto_aead_setkey(ctx->child, key, keylen); 965068c7a8SSteffen Klassert } 975068c7a8SSteffen Klassert 985068c7a8SSteffen Klassert static int pcrypt_aead_setauthsize(struct crypto_aead *parent, 995068c7a8SSteffen Klassert unsigned int authsize) 1005068c7a8SSteffen Klassert { 1015068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); 1025068c7a8SSteffen Klassert 1035068c7a8SSteffen Klassert return crypto_aead_setauthsize(ctx->child, authsize); 1045068c7a8SSteffen Klassert } 1055068c7a8SSteffen Klassert 1065068c7a8SSteffen Klassert static void pcrypt_aead_serial(struct padata_priv *padata) 1075068c7a8SSteffen Klassert { 1085068c7a8SSteffen Klassert struct pcrypt_request *preq = pcrypt_padata_request(padata); 1095068c7a8SSteffen Klassert struct aead_request *req = pcrypt_request_ctx(preq); 1105068c7a8SSteffen Klassert 1115068c7a8SSteffen Klassert aead_request_complete(req->base.data, padata->info); 1125068c7a8SSteffen Klassert } 1135068c7a8SSteffen Klassert 1145068c7a8SSteffen Klassert static void pcrypt_aead_done(struct crypto_async_request *areq, int err) 1155068c7a8SSteffen Klassert { 1165068c7a8SSteffen Klassert struct aead_request *req = areq->data; 1175068c7a8SSteffen Klassert struct pcrypt_request *preq = aead_request_ctx(req); 1185068c7a8SSteffen Klassert struct padata_priv *padata = pcrypt_request_padata(preq); 1195068c7a8SSteffen Klassert 1205068c7a8SSteffen Klassert padata->info = err; 1215068c7a8SSteffen Klassert req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 1225068c7a8SSteffen Klassert 1235068c7a8SSteffen Klassert padata_do_serial(padata); 1245068c7a8SSteffen Klassert } 1255068c7a8SSteffen Klassert 1265068c7a8SSteffen Klassert static void pcrypt_aead_enc(struct padata_priv *padata) 1275068c7a8SSteffen Klassert { 1285068c7a8SSteffen Klassert struct pcrypt_request *preq = pcrypt_padata_request(padata); 1295068c7a8SSteffen Klassert struct aead_request *req = pcrypt_request_ctx(preq); 1305068c7a8SSteffen Klassert 1315068c7a8SSteffen Klassert padata->info = crypto_aead_encrypt(req); 1325068c7a8SSteffen Klassert 1335a1436beSSteffen Klassert if (padata->info == -EINPROGRESS) 1345068c7a8SSteffen Klassert return; 1355068c7a8SSteffen Klassert 1365068c7a8SSteffen Klassert padata_do_serial(padata); 1375068c7a8SSteffen Klassert } 1385068c7a8SSteffen Klassert 1395068c7a8SSteffen Klassert static int pcrypt_aead_encrypt(struct aead_request *req) 1405068c7a8SSteffen Klassert { 1415068c7a8SSteffen Klassert int err; 1425068c7a8SSteffen Klassert struct pcrypt_request *preq = aead_request_ctx(req); 1435068c7a8SSteffen Klassert struct aead_request *creq = pcrypt_request_ctx(preq); 1445068c7a8SSteffen Klassert struct padata_priv *padata = pcrypt_request_padata(preq); 1455068c7a8SSteffen Klassert struct crypto_aead *aead = crypto_aead_reqtfm(req); 1465068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 1475068c7a8SSteffen Klassert u32 flags = aead_request_flags(req); 1485068c7a8SSteffen Klassert 1495068c7a8SSteffen Klassert memset(padata, 0, sizeof(struct padata_priv)); 1505068c7a8SSteffen Klassert 1515068c7a8SSteffen Klassert padata->parallel = pcrypt_aead_enc; 1525068c7a8SSteffen Klassert padata->serial = pcrypt_aead_serial; 1535068c7a8SSteffen Klassert 1545068c7a8SSteffen Klassert aead_request_set_tfm(creq, ctx->child); 1555068c7a8SSteffen Klassert aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, 1565068c7a8SSteffen Klassert pcrypt_aead_done, req); 1575068c7a8SSteffen Klassert aead_request_set_crypt(creq, req->src, req->dst, 1585068c7a8SSteffen Klassert req->cryptlen, req->iv); 1590496f560SHerbert Xu aead_request_set_ad(creq, req->assoclen); 1605068c7a8SSteffen Klassert 161e15bacbeSDan Kruchinin err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); 16283f619f3SSteffen Klassert if (!err) 16383f619f3SSteffen Klassert return -EINPROGRESS; 1645068c7a8SSteffen Klassert 1655068c7a8SSteffen Klassert return err; 1665068c7a8SSteffen Klassert } 1675068c7a8SSteffen Klassert 1685068c7a8SSteffen Klassert static void pcrypt_aead_dec(struct padata_priv *padata) 1695068c7a8SSteffen Klassert { 1705068c7a8SSteffen Klassert struct pcrypt_request *preq = pcrypt_padata_request(padata); 1715068c7a8SSteffen Klassert struct aead_request *req = pcrypt_request_ctx(preq); 1725068c7a8SSteffen Klassert 1735068c7a8SSteffen Klassert padata->info = crypto_aead_decrypt(req); 1745068c7a8SSteffen Klassert 1755a1436beSSteffen Klassert if (padata->info == -EINPROGRESS) 1765068c7a8SSteffen Klassert return; 1775068c7a8SSteffen Klassert 1785068c7a8SSteffen Klassert padata_do_serial(padata); 1795068c7a8SSteffen Klassert } 1805068c7a8SSteffen Klassert 1815068c7a8SSteffen Klassert static int pcrypt_aead_decrypt(struct aead_request *req) 1825068c7a8SSteffen Klassert { 1835068c7a8SSteffen Klassert int err; 1845068c7a8SSteffen Klassert struct pcrypt_request *preq = aead_request_ctx(req); 1855068c7a8SSteffen Klassert struct aead_request *creq = pcrypt_request_ctx(preq); 1865068c7a8SSteffen Klassert struct padata_priv *padata = pcrypt_request_padata(preq); 1875068c7a8SSteffen Klassert struct crypto_aead *aead = crypto_aead_reqtfm(req); 1885068c7a8SSteffen Klassert struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 1895068c7a8SSteffen Klassert u32 flags = aead_request_flags(req); 1905068c7a8SSteffen Klassert 1915068c7a8SSteffen Klassert memset(padata, 0, sizeof(struct padata_priv)); 1925068c7a8SSteffen Klassert 1935068c7a8SSteffen Klassert padata->parallel = pcrypt_aead_dec; 1945068c7a8SSteffen Klassert padata->serial = pcrypt_aead_serial; 1955068c7a8SSteffen Klassert 1965068c7a8SSteffen Klassert aead_request_set_tfm(creq, ctx->child); 1975068c7a8SSteffen Klassert aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, 1985068c7a8SSteffen Klassert pcrypt_aead_done, req); 1995068c7a8SSteffen Klassert aead_request_set_crypt(creq, req->src, req->dst, 2005068c7a8SSteffen Klassert req->cryptlen, req->iv); 2010496f560SHerbert Xu aead_request_set_ad(creq, req->assoclen); 2025068c7a8SSteffen Klassert 203e15bacbeSDan Kruchinin err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); 20483f619f3SSteffen Klassert if (!err) 20583f619f3SSteffen Klassert return -EINPROGRESS; 2065068c7a8SSteffen Klassert 2075068c7a8SSteffen Klassert return err; 2085068c7a8SSteffen Klassert } 2095068c7a8SSteffen Klassert 2100496f560SHerbert Xu static int pcrypt_aead_init_tfm(struct crypto_aead *tfm) 2115068c7a8SSteffen Klassert { 2125068c7a8SSteffen Klassert int cpu, cpu_index; 2130496f560SHerbert Xu struct aead_instance *inst = aead_alg_instance(tfm); 2140496f560SHerbert Xu struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst); 2150496f560SHerbert Xu struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm); 2165068c7a8SSteffen Klassert struct crypto_aead *cipher; 2175068c7a8SSteffen Klassert 218a5a22e57SHerbert Xu cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) % 219a5a22e57SHerbert Xu cpumask_weight(cpu_online_mask); 2205068c7a8SSteffen Klassert 221fbf0ca1bSSteffen Klassert ctx->cb_cpu = cpumask_first(cpu_online_mask); 2225068c7a8SSteffen Klassert for (cpu = 0; cpu < cpu_index; cpu++) 223fbf0ca1bSSteffen Klassert ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask); 2245068c7a8SSteffen Klassert 2250496f560SHerbert Xu cipher = crypto_spawn_aead(&ictx->spawn); 2265068c7a8SSteffen Klassert 2275068c7a8SSteffen Klassert if (IS_ERR(cipher)) 2285068c7a8SSteffen Klassert return PTR_ERR(cipher); 2295068c7a8SSteffen Klassert 2305068c7a8SSteffen Klassert ctx->child = cipher; 2310496f560SHerbert Xu crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) + 2320496f560SHerbert Xu sizeof(struct aead_request) + 233fd0de978SHerbert Xu crypto_aead_reqsize(cipher)); 2345068c7a8SSteffen Klassert 2355068c7a8SSteffen Klassert return 0; 2365068c7a8SSteffen Klassert } 2375068c7a8SSteffen Klassert 2380496f560SHerbert Xu static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm) 2395068c7a8SSteffen Klassert { 2400496f560SHerbert Xu struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm); 2415068c7a8SSteffen Klassert 2425068c7a8SSteffen Klassert crypto_free_aead(ctx->child); 2435068c7a8SSteffen Klassert } 2445068c7a8SSteffen Klassert 245d76c6810SEric Biggers static void pcrypt_free(struct aead_instance *inst) 246d76c6810SEric Biggers { 247d76c6810SEric Biggers struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst); 248d76c6810SEric Biggers 249d76c6810SEric Biggers crypto_drop_aead(&ctx->spawn); 250d76c6810SEric Biggers kfree(inst); 251d76c6810SEric Biggers } 252d76c6810SEric Biggers 25366d948e7SHerbert Xu static int pcrypt_init_instance(struct crypto_instance *inst, 25466d948e7SHerbert Xu struct crypto_alg *alg) 2555068c7a8SSteffen Klassert { 2565068c7a8SSteffen Klassert if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 2575068c7a8SSteffen Klassert "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 25866d948e7SHerbert Xu return -ENAMETOOLONG; 2595068c7a8SSteffen Klassert 2605068c7a8SSteffen Klassert memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 2615068c7a8SSteffen Klassert 2625068c7a8SSteffen Klassert inst->alg.cra_priority = alg->cra_priority + 100; 2635068c7a8SSteffen Klassert inst->alg.cra_blocksize = alg->cra_blocksize; 2645068c7a8SSteffen Klassert inst->alg.cra_alignmask = alg->cra_alignmask; 2655068c7a8SSteffen Klassert 26666d948e7SHerbert Xu return 0; 2675068c7a8SSteffen Klassert } 2685068c7a8SSteffen Klassert 2690496f560SHerbert Xu static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb, 27080a6d7dbSDan Carpenter u32 type, u32 mask) 2715068c7a8SSteffen Klassert { 27266d948e7SHerbert Xu struct pcrypt_instance_ctx *ctx; 273846f97dfSHerbert Xu struct crypto_attr_type *algt; 2740496f560SHerbert Xu struct aead_instance *inst; 2750496f560SHerbert Xu struct aead_alg *alg; 27666d948e7SHerbert Xu const char *name; 27766d948e7SHerbert Xu int err; 2785068c7a8SSteffen Klassert 279846f97dfSHerbert Xu algt = crypto_get_attr_type(tb); 280846f97dfSHerbert Xu if (IS_ERR(algt)) 281846f97dfSHerbert Xu return PTR_ERR(algt); 282846f97dfSHerbert Xu 28366d948e7SHerbert Xu name = crypto_attr_alg_name(tb[1]); 28466d948e7SHerbert Xu if (IS_ERR(name)) 2850496f560SHerbert Xu return PTR_ERR(name); 2865068c7a8SSteffen Klassert 28766d948e7SHerbert Xu inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 28866d948e7SHerbert Xu if (!inst) 2890496f560SHerbert Xu return -ENOMEM; 29066d948e7SHerbert Xu 2910496f560SHerbert Xu ctx = aead_instance_ctx(inst); 2920496f560SHerbert Xu crypto_set_aead_spawn(&ctx->spawn, aead_crypto_instance(inst)); 29366d948e7SHerbert Xu 2945e4b8c1fSHerbert Xu err = crypto_grab_aead(&ctx->spawn, name, 0, 0); 29566d948e7SHerbert Xu if (err) 29666d948e7SHerbert Xu goto out_free_inst; 29766d948e7SHerbert Xu 2980496f560SHerbert Xu alg = crypto_spawn_aead_alg(&ctx->spawn); 2990496f560SHerbert Xu err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base); 30066d948e7SHerbert Xu if (err) 30166d948e7SHerbert Xu goto out_drop_aead; 3025068c7a8SSteffen Klassert 303846f97dfSHerbert Xu inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC; 304846f97dfSHerbert Xu 3050496f560SHerbert Xu inst->alg.ivsize = crypto_aead_alg_ivsize(alg); 3060496f560SHerbert Xu inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 3075068c7a8SSteffen Klassert 3080496f560SHerbert Xu inst->alg.base.cra_ctxsize = sizeof(struct pcrypt_aead_ctx); 3095068c7a8SSteffen Klassert 3100496f560SHerbert Xu inst->alg.init = pcrypt_aead_init_tfm; 3110496f560SHerbert Xu inst->alg.exit = pcrypt_aead_exit_tfm; 3125068c7a8SSteffen Klassert 3130496f560SHerbert Xu inst->alg.setkey = pcrypt_aead_setkey; 3140496f560SHerbert Xu inst->alg.setauthsize = pcrypt_aead_setauthsize; 3150496f560SHerbert Xu inst->alg.encrypt = pcrypt_aead_encrypt; 3160496f560SHerbert Xu inst->alg.decrypt = pcrypt_aead_decrypt; 3175068c7a8SSteffen Klassert 318d76c6810SEric Biggers inst->free = pcrypt_free; 319d76c6810SEric Biggers 3200496f560SHerbert Xu err = aead_register_instance(tmpl, inst); 3210496f560SHerbert Xu if (err) 3220496f560SHerbert Xu goto out_drop_aead; 3235068c7a8SSteffen Klassert 32466d948e7SHerbert Xu out: 3250496f560SHerbert Xu return err; 32666d948e7SHerbert Xu 32766d948e7SHerbert Xu out_drop_aead: 32866d948e7SHerbert Xu crypto_drop_aead(&ctx->spawn); 32966d948e7SHerbert Xu out_free_inst: 33066d948e7SHerbert Xu kfree(inst); 33166d948e7SHerbert Xu goto out; 3325068c7a8SSteffen Klassert } 3335068c7a8SSteffen Klassert 3340496f560SHerbert Xu static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb) 3355068c7a8SSteffen Klassert { 3365068c7a8SSteffen Klassert struct crypto_attr_type *algt; 3375068c7a8SSteffen Klassert 3385068c7a8SSteffen Klassert algt = crypto_get_attr_type(tb); 3395068c7a8SSteffen Klassert if (IS_ERR(algt)) 3400496f560SHerbert Xu return PTR_ERR(algt); 3415068c7a8SSteffen Klassert 3425068c7a8SSteffen Klassert switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 3435068c7a8SSteffen Klassert case CRYPTO_ALG_TYPE_AEAD: 3440496f560SHerbert Xu return pcrypt_create_aead(tmpl, tb, algt->type, algt->mask); 3455068c7a8SSteffen Klassert } 3465068c7a8SSteffen Klassert 3470496f560SHerbert Xu return -EINVAL; 3485068c7a8SSteffen Klassert } 3495068c7a8SSteffen Klassert 350e15bacbeSDan Kruchinin static int pcrypt_cpumask_change_notify(struct notifier_block *self, 351e15bacbeSDan Kruchinin unsigned long val, void *data) 352e15bacbeSDan Kruchinin { 353c57e842eSSteffen Klassert struct padata_pcrypt *pcrypt; 354e15bacbeSDan Kruchinin struct pcrypt_cpumask *new_mask, *old_mask; 355d3f64e46SSteffen Klassert struct padata_cpumask *cpumask = (struct padata_cpumask *)data; 356e15bacbeSDan Kruchinin 357e15bacbeSDan Kruchinin if (!(val & PADATA_CPU_SERIAL)) 358e15bacbeSDan Kruchinin return 0; 359e15bacbeSDan Kruchinin 360c57e842eSSteffen Klassert pcrypt = container_of(self, struct padata_pcrypt, nblock); 361e15bacbeSDan Kruchinin new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); 362e15bacbeSDan Kruchinin if (!new_mask) 363e15bacbeSDan Kruchinin return -ENOMEM; 364e15bacbeSDan Kruchinin if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) { 365e15bacbeSDan Kruchinin kfree(new_mask); 366e15bacbeSDan Kruchinin return -ENOMEM; 367e15bacbeSDan Kruchinin } 368e15bacbeSDan Kruchinin 369e15bacbeSDan Kruchinin old_mask = pcrypt->cb_cpumask; 370e15bacbeSDan Kruchinin 371d3f64e46SSteffen Klassert cpumask_copy(new_mask->mask, cpumask->cbcpu); 372e15bacbeSDan Kruchinin rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); 373a0076e17SPaul E. McKenney synchronize_rcu(); 374e15bacbeSDan Kruchinin 375e15bacbeSDan Kruchinin free_cpumask_var(old_mask->mask); 376e15bacbeSDan Kruchinin kfree(old_mask); 377e15bacbeSDan Kruchinin return 0; 378e15bacbeSDan Kruchinin } 379e15bacbeSDan Kruchinin 380a3fb1e33SDan Kruchinin static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) 381a3fb1e33SDan Kruchinin { 382a3fb1e33SDan Kruchinin int ret; 383a3fb1e33SDan Kruchinin 384a3fb1e33SDan Kruchinin pinst->kobj.kset = pcrypt_kset; 385b1e3874cSColin Ian King ret = kobject_add(&pinst->kobj, NULL, "%s", name); 386a3fb1e33SDan Kruchinin if (!ret) 387a3fb1e33SDan Kruchinin kobject_uevent(&pinst->kobj, KOBJ_ADD); 388a3fb1e33SDan Kruchinin 389a3fb1e33SDan Kruchinin return ret; 390a3fb1e33SDan Kruchinin } 391a3fb1e33SDan Kruchinin 392c57e842eSSteffen Klassert static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, 393e15bacbeSDan Kruchinin const char *name) 394e15bacbeSDan Kruchinin { 395e15bacbeSDan Kruchinin int ret = -ENOMEM; 396e15bacbeSDan Kruchinin struct pcrypt_cpumask *mask; 397e15bacbeSDan Kruchinin 398d3f64e46SSteffen Klassert get_online_cpus(); 399d3f64e46SSteffen Klassert 400d8537548SKees Cook pcrypt->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 401d8537548SKees Cook 1, name); 402e15bacbeSDan Kruchinin if (!pcrypt->wq) 403e15bacbeSDan Kruchinin goto err; 404e15bacbeSDan Kruchinin 405e6cc1170SSteffen Klassert pcrypt->pinst = padata_alloc_possible(pcrypt->wq); 406e15bacbeSDan Kruchinin if (!pcrypt->pinst) 407e15bacbeSDan Kruchinin goto err_destroy_workqueue; 408e15bacbeSDan Kruchinin 409e15bacbeSDan Kruchinin mask = kmalloc(sizeof(*mask), GFP_KERNEL); 410e15bacbeSDan Kruchinin if (!mask) 411e15bacbeSDan Kruchinin goto err_free_padata; 412e15bacbeSDan Kruchinin if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { 413e15bacbeSDan Kruchinin kfree(mask); 414e15bacbeSDan Kruchinin goto err_free_padata; 415e15bacbeSDan Kruchinin } 416e15bacbeSDan Kruchinin 417fbf0ca1bSSteffen Klassert cpumask_and(mask->mask, cpu_possible_mask, cpu_online_mask); 418e15bacbeSDan Kruchinin rcu_assign_pointer(pcrypt->cb_cpumask, mask); 419e15bacbeSDan Kruchinin 420e15bacbeSDan Kruchinin pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; 421e15bacbeSDan Kruchinin ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 422e15bacbeSDan Kruchinin if (ret) 423e15bacbeSDan Kruchinin goto err_free_cpumask; 424e15bacbeSDan Kruchinin 425a3fb1e33SDan Kruchinin ret = pcrypt_sysfs_add(pcrypt->pinst, name); 426a3fb1e33SDan Kruchinin if (ret) 427a3fb1e33SDan Kruchinin goto err_unregister_notifier; 428a3fb1e33SDan Kruchinin 429d3f64e46SSteffen Klassert put_online_cpus(); 430d3f64e46SSteffen Klassert 431e15bacbeSDan Kruchinin return ret; 432d3f64e46SSteffen Klassert 433a3fb1e33SDan Kruchinin err_unregister_notifier: 434a3fb1e33SDan Kruchinin padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 435e15bacbeSDan Kruchinin err_free_cpumask: 436e15bacbeSDan Kruchinin free_cpumask_var(mask->mask); 437e15bacbeSDan Kruchinin kfree(mask); 438e15bacbeSDan Kruchinin err_free_padata: 439e15bacbeSDan Kruchinin padata_free(pcrypt->pinst); 440e15bacbeSDan Kruchinin err_destroy_workqueue: 441e15bacbeSDan Kruchinin destroy_workqueue(pcrypt->wq); 442e15bacbeSDan Kruchinin err: 443d3f64e46SSteffen Klassert put_online_cpus(); 444d3f64e46SSteffen Klassert 445e15bacbeSDan Kruchinin return ret; 446e15bacbeSDan Kruchinin } 447e15bacbeSDan Kruchinin 448c57e842eSSteffen Klassert static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) 449e15bacbeSDan Kruchinin { 450e15bacbeSDan Kruchinin free_cpumask_var(pcrypt->cb_cpumask->mask); 451e15bacbeSDan Kruchinin kfree(pcrypt->cb_cpumask); 452e15bacbeSDan Kruchinin 453e15bacbeSDan Kruchinin padata_stop(pcrypt->pinst); 454e15bacbeSDan Kruchinin padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 455e15bacbeSDan Kruchinin destroy_workqueue(pcrypt->wq); 456e15bacbeSDan Kruchinin padata_free(pcrypt->pinst); 457e15bacbeSDan Kruchinin } 458e15bacbeSDan Kruchinin 4595068c7a8SSteffen Klassert static struct crypto_template pcrypt_tmpl = { 4605068c7a8SSteffen Klassert .name = "pcrypt", 4610496f560SHerbert Xu .create = pcrypt_create, 4625068c7a8SSteffen Klassert .module = THIS_MODULE, 4635068c7a8SSteffen Klassert }; 4645068c7a8SSteffen Klassert 4655068c7a8SSteffen Klassert static int __init pcrypt_init(void) 4665068c7a8SSteffen Klassert { 467a3fb1e33SDan Kruchinin int err = -ENOMEM; 468a3fb1e33SDan Kruchinin 469a3fb1e33SDan Kruchinin pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj); 470a3fb1e33SDan Kruchinin if (!pcrypt_kset) 471a3fb1e33SDan Kruchinin goto err; 472e15bacbeSDan Kruchinin 473c57e842eSSteffen Klassert err = pcrypt_init_padata(&pencrypt, "pencrypt"); 474e15bacbeSDan Kruchinin if (err) 475a3fb1e33SDan Kruchinin goto err_unreg_kset; 4765068c7a8SSteffen Klassert 477c57e842eSSteffen Klassert err = pcrypt_init_padata(&pdecrypt, "pdecrypt"); 4784c879170SSteffen Klassert if (err) 479e15bacbeSDan Kruchinin goto err_deinit_pencrypt; 4804c879170SSteffen Klassert 481e15bacbeSDan Kruchinin padata_start(pencrypt.pinst); 482e15bacbeSDan Kruchinin padata_start(pdecrypt.pinst); 4835068c7a8SSteffen Klassert 4845068c7a8SSteffen Klassert return crypto_register_template(&pcrypt_tmpl); 4855068c7a8SSteffen Klassert 486e15bacbeSDan Kruchinin err_deinit_pencrypt: 487c57e842eSSteffen Klassert pcrypt_fini_padata(&pencrypt); 488a3fb1e33SDan Kruchinin err_unreg_kset: 489a3fb1e33SDan Kruchinin kset_unregister(pcrypt_kset); 4905068c7a8SSteffen Klassert err: 4914c879170SSteffen Klassert return err; 4925068c7a8SSteffen Klassert } 4935068c7a8SSteffen Klassert 4945068c7a8SSteffen Klassert static void __exit pcrypt_exit(void) 4955068c7a8SSteffen Klassert { 496c57e842eSSteffen Klassert pcrypt_fini_padata(&pencrypt); 497c57e842eSSteffen Klassert pcrypt_fini_padata(&pdecrypt); 4985068c7a8SSteffen Klassert 499a3fb1e33SDan Kruchinin kset_unregister(pcrypt_kset); 5005068c7a8SSteffen Klassert crypto_unregister_template(&pcrypt_tmpl); 5015068c7a8SSteffen Klassert } 5025068c7a8SSteffen Klassert 503c4741b23SEric Biggers subsys_initcall(pcrypt_init); 5045068c7a8SSteffen Klassert module_exit(pcrypt_exit); 5055068c7a8SSteffen Klassert 5065068c7a8SSteffen Klassert MODULE_LICENSE("GPL"); 5075068c7a8SSteffen Klassert MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); 5085068c7a8SSteffen Klassert MODULE_DESCRIPTION("Parallel crypto wrapper"); 5094943ba16SKees Cook MODULE_ALIAS_CRYPTO("pcrypt"); 510