11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4542da317SMilan Broz * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 943d69034SMilan Broz #include <linux/completion.h> 10d1806f6aSHerbert Xu #include <linux/err.h> 111da177e4SLinus Torvalds #include <linux/module.h> 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/kernel.h> 141da177e4SLinus Torvalds #include <linux/bio.h> 151da177e4SLinus Torvalds #include <linux/blkdev.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/crypto.h> 191da177e4SLinus Torvalds #include <linux/workqueue.h> 203fcfab16SAndrew Morton #include <linux/backing-dev.h> 21c0297721SAndi Kleen #include <linux/percpu.h> 221da177e4SLinus Torvalds #include <asm/atomic.h> 23378f058cSDavid Hardeman #include <linux/scatterlist.h> 241da177e4SLinus Torvalds #include <asm/page.h> 2548527fa7SRik Snel #include <asm/unaligned.h> 261da177e4SLinus Torvalds 27586e80e6SMikulas Patocka #include <linux/device-mapper.h> 281da177e4SLinus Torvalds 2972d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 30e48d4bbfSMilan Broz #define MESG_STR(x) x, sizeof(x) 311da177e4SLinus Torvalds 321da177e4SLinus Torvalds /* 331da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 341da177e4SLinus Torvalds */ 351da177e4SLinus Torvalds struct convert_context { 3643d69034SMilan Broz struct completion restart; 371da177e4SLinus Torvalds struct bio *bio_in; 381da177e4SLinus Torvalds struct bio *bio_out; 391da177e4SLinus Torvalds unsigned int offset_in; 401da177e4SLinus Torvalds unsigned int offset_out; 411da177e4SLinus Torvalds unsigned int idx_in; 421da177e4SLinus Torvalds unsigned int idx_out; 431da177e4SLinus Torvalds sector_t sector; 4443d69034SMilan Broz atomic_t pending; 451da177e4SLinus Torvalds }; 461da177e4SLinus Torvalds 4753017030SMilan Broz /* 4853017030SMilan Broz * per bio private data 4953017030SMilan Broz */ 5053017030SMilan Broz struct dm_crypt_io { 5153017030SMilan Broz struct dm_target *target; 5253017030SMilan Broz struct bio *base_bio; 5353017030SMilan Broz struct work_struct work; 5453017030SMilan Broz 5553017030SMilan Broz struct convert_context ctx; 5653017030SMilan Broz 5753017030SMilan Broz atomic_t pending; 5853017030SMilan Broz int error; 590c395b0fSMilan Broz sector_t sector; 60393b47efSMilan Broz struct dm_crypt_io *base_io; 6153017030SMilan Broz }; 6253017030SMilan Broz 6301482b76SMilan Broz struct dm_crypt_request { 64b2174eebSHuang Ying struct convert_context *ctx; 6501482b76SMilan Broz struct scatterlist sg_in; 6601482b76SMilan Broz struct scatterlist sg_out; 6701482b76SMilan Broz }; 6801482b76SMilan Broz 691da177e4SLinus Torvalds struct crypt_config; 701da177e4SLinus Torvalds 711da177e4SLinus Torvalds struct crypt_iv_operations { 721da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 731da177e4SLinus Torvalds const char *opts); 741da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 75b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 76542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 771da177e4SLinus Torvalds int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector); 781da177e4SLinus Torvalds }; 791da177e4SLinus Torvalds 8060473592SMilan Broz struct iv_essiv_private { 81b95bf2d3SMilan Broz struct crypto_hash *hash_tfm; 82b95bf2d3SMilan Broz u8 *salt; 8360473592SMilan Broz }; 8460473592SMilan Broz 8560473592SMilan Broz struct iv_benbi_private { 8660473592SMilan Broz int shift; 8760473592SMilan Broz }; 8860473592SMilan Broz 891da177e4SLinus Torvalds /* 901da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 911da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 921da177e4SLinus Torvalds */ 93e48d4bbfSMilan Broz enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 94c0297721SAndi Kleen 95c0297721SAndi Kleen /* 96c0297721SAndi Kleen * Duplicated per-CPU state for cipher. 97c0297721SAndi Kleen */ 98c0297721SAndi Kleen struct crypt_cpu { 99c0297721SAndi Kleen struct ablkcipher_request *req; 100c0297721SAndi Kleen struct crypto_ablkcipher *tfm; 101c0297721SAndi Kleen 102c0297721SAndi Kleen /* ESSIV: struct crypto_cipher *essiv_tfm */ 103c0297721SAndi Kleen void *iv_private; 104c0297721SAndi Kleen }; 105c0297721SAndi Kleen 106c0297721SAndi Kleen /* 107c0297721SAndi Kleen * The fields in here must be read only after initialization, 108c0297721SAndi Kleen * changing state should be in crypt_cpu. 109c0297721SAndi Kleen */ 1101da177e4SLinus Torvalds struct crypt_config { 1111da177e4SLinus Torvalds struct dm_dev *dev; 1121da177e4SLinus Torvalds sector_t start; 1131da177e4SLinus Torvalds 1141da177e4SLinus Torvalds /* 115ddd42edfSMilan Broz * pool for per bio private data, crypto requests and 116ddd42edfSMilan Broz * encryption requeusts/buffer pages 1171da177e4SLinus Torvalds */ 1181da177e4SLinus Torvalds mempool_t *io_pool; 119ddd42edfSMilan Broz mempool_t *req_pool; 1201da177e4SLinus Torvalds mempool_t *page_pool; 1216a24c718SMilan Broz struct bio_set *bs; 1221da177e4SLinus Torvalds 123cabf08e4SMilan Broz struct workqueue_struct *io_queue; 124cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1253f1e9070SMilan Broz 1265ebaee6dSMilan Broz char *cipher; 1277dbcd137SMilan Broz char *cipher_string; 1285ebaee6dSMilan Broz 1291da177e4SLinus Torvalds struct crypt_iv_operations *iv_gen_ops; 13079066ad3SHerbert Xu union { 13160473592SMilan Broz struct iv_essiv_private essiv; 13260473592SMilan Broz struct iv_benbi_private benbi; 13379066ad3SHerbert Xu } iv_gen_private; 1341da177e4SLinus Torvalds sector_t iv_offset; 1351da177e4SLinus Torvalds unsigned int iv_size; 1361da177e4SLinus Torvalds 137ddd42edfSMilan Broz /* 138c0297721SAndi Kleen * Duplicated per cpu state. Access through 139c0297721SAndi Kleen * per_cpu_ptr() only. 140c0297721SAndi Kleen */ 141c0297721SAndi Kleen struct crypt_cpu __percpu *cpu; 142c0297721SAndi Kleen 143c0297721SAndi Kleen /* 144ddd42edfSMilan Broz * Layout of each crypto request: 145ddd42edfSMilan Broz * 146ddd42edfSMilan Broz * struct ablkcipher_request 147ddd42edfSMilan Broz * context 148ddd42edfSMilan Broz * padding 149ddd42edfSMilan Broz * struct dm_crypt_request 150ddd42edfSMilan Broz * padding 151ddd42edfSMilan Broz * IV 152ddd42edfSMilan Broz * 153ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 154ddd42edfSMilan Broz * correctly aligned. 155ddd42edfSMilan Broz */ 156ddd42edfSMilan Broz unsigned int dmreq_start; 157ddd42edfSMilan Broz 158e48d4bbfSMilan Broz unsigned long flags; 1591da177e4SLinus Torvalds unsigned int key_size; 1601da177e4SLinus Torvalds u8 key[0]; 1611da177e4SLinus Torvalds }; 1621da177e4SLinus Torvalds 1636a24c718SMilan Broz #define MIN_IOS 16 1641da177e4SLinus Torvalds #define MIN_POOL_PAGES 32 1651da177e4SLinus Torvalds #define MIN_BIO_PAGES 8 1661da177e4SLinus Torvalds 167e18b890bSChristoph Lameter static struct kmem_cache *_crypt_io_pool; 1681da177e4SLinus Torvalds 169028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 170395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 171027581f3SOlaf Kirch 172c0297721SAndi Kleen static struct crypt_cpu *this_crypt_config(struct crypt_config *cc) 173c0297721SAndi Kleen { 174c0297721SAndi Kleen return this_cpu_ptr(cc->cpu); 175c0297721SAndi Kleen } 176c0297721SAndi Kleen 177c0297721SAndi Kleen /* 178c0297721SAndi Kleen * Use this to access cipher attributes that are the same for each CPU. 179c0297721SAndi Kleen */ 180c0297721SAndi Kleen static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc) 181c0297721SAndi Kleen { 182c0297721SAndi Kleen return __this_cpu_ptr(cc->cpu)->tfm; 183c0297721SAndi Kleen } 184c0297721SAndi Kleen 1851da177e4SLinus Torvalds /* 1861da177e4SLinus Torvalds * Different IV generation algorithms: 1871da177e4SLinus Torvalds * 1883c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 1893a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 1901da177e4SLinus Torvalds * 19161afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 19261afef61SMilan Broz * number, padded with zeros if necessary. 19361afef61SMilan Broz * 1943c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 1951da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 1961da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 1971da177e4SLinus Torvalds * 19848527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 19948527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 20048527fa7SRik Snel * 20146b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 20246b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 20346b47730SLudwig Nussel * 2041da177e4SLinus Torvalds * plumb: unimplemented, see: 2051da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 2061da177e4SLinus Torvalds */ 2071da177e4SLinus Torvalds 2081da177e4SLinus Torvalds static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 2091da177e4SLinus Torvalds { 2101da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 2111da177e4SLinus Torvalds *(u32 *)iv = cpu_to_le32(sector & 0xffffffff); 2121da177e4SLinus Torvalds 2131da177e4SLinus Torvalds return 0; 2141da177e4SLinus Torvalds } 2151da177e4SLinus Torvalds 21661afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 21761afef61SMilan Broz sector_t sector) 21861afef61SMilan Broz { 21961afef61SMilan Broz memset(iv, 0, cc->iv_size); 22061afef61SMilan Broz *(u64 *)iv = cpu_to_le64(sector); 22161afef61SMilan Broz 22261afef61SMilan Broz return 0; 22361afef61SMilan Broz } 22461afef61SMilan Broz 225b95bf2d3SMilan Broz /* Initialise ESSIV - compute salt but no local memory allocations */ 226b95bf2d3SMilan Broz static int crypt_iv_essiv_init(struct crypt_config *cc) 227b95bf2d3SMilan Broz { 228b95bf2d3SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 229b95bf2d3SMilan Broz struct hash_desc desc; 230b95bf2d3SMilan Broz struct scatterlist sg; 231c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 232c0297721SAndi Kleen int err, cpu; 233b95bf2d3SMilan Broz 234b95bf2d3SMilan Broz sg_init_one(&sg, cc->key, cc->key_size); 235b95bf2d3SMilan Broz desc.tfm = essiv->hash_tfm; 236b95bf2d3SMilan Broz desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 237b95bf2d3SMilan Broz 238b95bf2d3SMilan Broz err = crypto_hash_digest(&desc, &sg, cc->key_size, essiv->salt); 239b95bf2d3SMilan Broz if (err) 240b95bf2d3SMilan Broz return err; 241b95bf2d3SMilan Broz 242c0297721SAndi Kleen for_each_possible_cpu(cpu) { 243c0297721SAndi Kleen essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private, 244c0297721SAndi Kleen 245c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, essiv->salt, 246b95bf2d3SMilan Broz crypto_hash_digestsize(essiv->hash_tfm)); 247c0297721SAndi Kleen if (err) 248c0297721SAndi Kleen return err; 249c0297721SAndi Kleen } 250c0297721SAndi Kleen 251c0297721SAndi Kleen return 0; 252b95bf2d3SMilan Broz } 253b95bf2d3SMilan Broz 254542da317SMilan Broz /* Wipe salt and reset key derived from volume key */ 255542da317SMilan Broz static int crypt_iv_essiv_wipe(struct crypt_config *cc) 256542da317SMilan Broz { 257542da317SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 258542da317SMilan Broz unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm); 259c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 260c0297721SAndi Kleen int cpu, r, err = 0; 261542da317SMilan Broz 262542da317SMilan Broz memset(essiv->salt, 0, salt_size); 263542da317SMilan Broz 264c0297721SAndi Kleen for_each_possible_cpu(cpu) { 265c0297721SAndi Kleen essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private; 266c0297721SAndi Kleen r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size); 267c0297721SAndi Kleen if (r) 268c0297721SAndi Kleen err = r; 269c0297721SAndi Kleen } 270c0297721SAndi Kleen 271c0297721SAndi Kleen return err; 272c0297721SAndi Kleen } 273c0297721SAndi Kleen 274c0297721SAndi Kleen /* Set up per cpu cipher state */ 275c0297721SAndi Kleen static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc, 276c0297721SAndi Kleen struct dm_target *ti, 277c0297721SAndi Kleen u8 *salt, unsigned saltsize) 278c0297721SAndi Kleen { 279c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 280c0297721SAndi Kleen int err; 281c0297721SAndi Kleen 282c0297721SAndi Kleen /* Setup the essiv_tfm with the given salt */ 283c0297721SAndi Kleen essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 284c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 285c0297721SAndi Kleen ti->error = "Error allocating crypto tfm for ESSIV"; 286c0297721SAndi Kleen return essiv_tfm; 287c0297721SAndi Kleen } 288c0297721SAndi Kleen 289c0297721SAndi Kleen if (crypto_cipher_blocksize(essiv_tfm) != 290c0297721SAndi Kleen crypto_ablkcipher_ivsize(any_tfm(cc))) { 291c0297721SAndi Kleen ti->error = "Block size of ESSIV cipher does " 292c0297721SAndi Kleen "not match IV size of block cipher"; 293c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 294c0297721SAndi Kleen return ERR_PTR(-EINVAL); 295c0297721SAndi Kleen } 296c0297721SAndi Kleen 297c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 298c0297721SAndi Kleen if (err) { 299c0297721SAndi Kleen ti->error = "Failed to set key for ESSIV cipher"; 300c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 301c0297721SAndi Kleen return ERR_PTR(err); 302c0297721SAndi Kleen } 303c0297721SAndi Kleen 304c0297721SAndi Kleen return essiv_tfm; 305542da317SMilan Broz } 306542da317SMilan Broz 30760473592SMilan Broz static void crypt_iv_essiv_dtr(struct crypt_config *cc) 30860473592SMilan Broz { 309c0297721SAndi Kleen int cpu; 310c0297721SAndi Kleen struct crypt_cpu *cpu_cc; 311c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 31260473592SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 31360473592SMilan Broz 314b95bf2d3SMilan Broz crypto_free_hash(essiv->hash_tfm); 315b95bf2d3SMilan Broz essiv->hash_tfm = NULL; 316b95bf2d3SMilan Broz 317b95bf2d3SMilan Broz kzfree(essiv->salt); 318b95bf2d3SMilan Broz essiv->salt = NULL; 319c0297721SAndi Kleen 320c0297721SAndi Kleen for_each_possible_cpu(cpu) { 321c0297721SAndi Kleen cpu_cc = per_cpu_ptr(cc->cpu, cpu); 322c0297721SAndi Kleen essiv_tfm = cpu_cc->iv_private; 323c0297721SAndi Kleen 324c0297721SAndi Kleen if (essiv_tfm) 325c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 326c0297721SAndi Kleen 327c0297721SAndi Kleen cpu_cc->iv_private = NULL; 328c0297721SAndi Kleen } 32960473592SMilan Broz } 33060473592SMilan Broz 3311da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 3321da177e4SLinus Torvalds const char *opts) 3331da177e4SLinus Torvalds { 3345861f1beSMilan Broz struct crypto_cipher *essiv_tfm = NULL; 3355861f1beSMilan Broz struct crypto_hash *hash_tfm = NULL; 3365861f1beSMilan Broz u8 *salt = NULL; 337c0297721SAndi Kleen int err, cpu; 3381da177e4SLinus Torvalds 3395861f1beSMilan Broz if (!opts) { 34072d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 3411da177e4SLinus Torvalds return -EINVAL; 3421da177e4SLinus Torvalds } 3431da177e4SLinus Torvalds 344b95bf2d3SMilan Broz /* Allocate hash algorithm */ 34535058687SHerbert Xu hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 34635058687SHerbert Xu if (IS_ERR(hash_tfm)) { 34772d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 3485861f1beSMilan Broz err = PTR_ERR(hash_tfm); 3495861f1beSMilan Broz goto bad; 3501da177e4SLinus Torvalds } 3511da177e4SLinus Torvalds 352b95bf2d3SMilan Broz salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL); 3535861f1beSMilan Broz if (!salt) { 35472d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 3555861f1beSMilan Broz err = -ENOMEM; 3565861f1beSMilan Broz goto bad; 3571da177e4SLinus Torvalds } 3581da177e4SLinus Torvalds 359b95bf2d3SMilan Broz cc->iv_gen_private.essiv.salt = salt; 360b95bf2d3SMilan Broz cc->iv_gen_private.essiv.hash_tfm = hash_tfm; 361b95bf2d3SMilan Broz 362c0297721SAndi Kleen for_each_possible_cpu(cpu) { 363c0297721SAndi Kleen essiv_tfm = setup_essiv_cpu(cc, ti, salt, 364c0297721SAndi Kleen crypto_hash_digestsize(hash_tfm)); 365c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 366c0297721SAndi Kleen crypt_iv_essiv_dtr(cc); 367c0297721SAndi Kleen return PTR_ERR(essiv_tfm); 368c0297721SAndi Kleen } 369c0297721SAndi Kleen per_cpu_ptr(cc->cpu, cpu)->iv_private = essiv_tfm; 370c0297721SAndi Kleen } 371c0297721SAndi Kleen 3721da177e4SLinus Torvalds return 0; 3735861f1beSMilan Broz 3745861f1beSMilan Broz bad: 3755861f1beSMilan Broz if (hash_tfm && !IS_ERR(hash_tfm)) 3765861f1beSMilan Broz crypto_free_hash(hash_tfm); 377b95bf2d3SMilan Broz kfree(salt); 3785861f1beSMilan Broz return err; 3791da177e4SLinus Torvalds } 3801da177e4SLinus Torvalds 3811da177e4SLinus Torvalds static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 3821da177e4SLinus Torvalds { 383c0297721SAndi Kleen struct crypto_cipher *essiv_tfm = this_crypt_config(cc)->iv_private; 384c0297721SAndi Kleen 3851da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 3861da177e4SLinus Torvalds *(u64 *)iv = cpu_to_le64(sector); 387c0297721SAndi Kleen crypto_cipher_encrypt_one(essiv_tfm, iv, iv); 388c0297721SAndi Kleen 3891da177e4SLinus Torvalds return 0; 3901da177e4SLinus Torvalds } 3911da177e4SLinus Torvalds 39248527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 39348527fa7SRik Snel const char *opts) 39448527fa7SRik Snel { 395c0297721SAndi Kleen unsigned bs = crypto_ablkcipher_blocksize(any_tfm(cc)); 396f0d1b0b3SDavid Howells int log = ilog2(bs); 39748527fa7SRik Snel 39848527fa7SRik Snel /* we need to calculate how far we must shift the sector count 39948527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 40048527fa7SRik Snel 40148527fa7SRik Snel if (1 << log != bs) { 40248527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 40348527fa7SRik Snel return -EINVAL; 40448527fa7SRik Snel } 40548527fa7SRik Snel 40648527fa7SRik Snel if (log > 9) { 40748527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 40848527fa7SRik Snel return -EINVAL; 40948527fa7SRik Snel } 41048527fa7SRik Snel 41160473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 41248527fa7SRik Snel 41348527fa7SRik Snel return 0; 41448527fa7SRik Snel } 41548527fa7SRik Snel 41648527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 41748527fa7SRik Snel { 41848527fa7SRik Snel } 41948527fa7SRik Snel 42048527fa7SRik Snel static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 42148527fa7SRik Snel { 42279066ad3SHerbert Xu __be64 val; 42379066ad3SHerbert Xu 42448527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 42579066ad3SHerbert Xu 42660473592SMilan Broz val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi.shift) + 1); 42779066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 42848527fa7SRik Snel 4291da177e4SLinus Torvalds return 0; 4301da177e4SLinus Torvalds } 4311da177e4SLinus Torvalds 43246b47730SLudwig Nussel static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 43346b47730SLudwig Nussel { 43446b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 43546b47730SLudwig Nussel 43646b47730SLudwig Nussel return 0; 43746b47730SLudwig Nussel } 43846b47730SLudwig Nussel 4391da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_plain_ops = { 4401da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 4411da177e4SLinus Torvalds }; 4421da177e4SLinus Torvalds 44361afef61SMilan Broz static struct crypt_iv_operations crypt_iv_plain64_ops = { 44461afef61SMilan Broz .generator = crypt_iv_plain64_gen 44561afef61SMilan Broz }; 44661afef61SMilan Broz 4471da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_essiv_ops = { 4481da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 4491da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 450b95bf2d3SMilan Broz .init = crypt_iv_essiv_init, 451542da317SMilan Broz .wipe = crypt_iv_essiv_wipe, 4521da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 4531da177e4SLinus Torvalds }; 4541da177e4SLinus Torvalds 45548527fa7SRik Snel static struct crypt_iv_operations crypt_iv_benbi_ops = { 45648527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 45748527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 45848527fa7SRik Snel .generator = crypt_iv_benbi_gen 45948527fa7SRik Snel }; 4601da177e4SLinus Torvalds 46146b47730SLudwig Nussel static struct crypt_iv_operations crypt_iv_null_ops = { 46246b47730SLudwig Nussel .generator = crypt_iv_null_gen 46346b47730SLudwig Nussel }; 46446b47730SLudwig Nussel 465d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 466d469f841SMilan Broz struct convert_context *ctx, 4671da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 468fcd369daSMilan Broz sector_t sector) 4691da177e4SLinus Torvalds { 4701da177e4SLinus Torvalds ctx->bio_in = bio_in; 4711da177e4SLinus Torvalds ctx->bio_out = bio_out; 4721da177e4SLinus Torvalds ctx->offset_in = 0; 4731da177e4SLinus Torvalds ctx->offset_out = 0; 4741da177e4SLinus Torvalds ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 4751da177e4SLinus Torvalds ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 4761da177e4SLinus Torvalds ctx->sector = sector + cc->iv_offset; 47743d69034SMilan Broz init_completion(&ctx->restart); 4781da177e4SLinus Torvalds } 4791da177e4SLinus Torvalds 480b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 481b2174eebSHuang Ying struct ablkcipher_request *req) 482b2174eebSHuang Ying { 483b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 484b2174eebSHuang Ying } 485b2174eebSHuang Ying 486b2174eebSHuang Ying static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc, 487b2174eebSHuang Ying struct dm_crypt_request *dmreq) 488b2174eebSHuang Ying { 489b2174eebSHuang Ying return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start); 490b2174eebSHuang Ying } 491b2174eebSHuang Ying 49201482b76SMilan Broz static int crypt_convert_block(struct crypt_config *cc, 4933a7f6c99SMilan Broz struct convert_context *ctx, 4943a7f6c99SMilan Broz struct ablkcipher_request *req) 49501482b76SMilan Broz { 49601482b76SMilan Broz struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 49701482b76SMilan Broz struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 4983a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 4993a7f6c99SMilan Broz u8 *iv; 5003a7f6c99SMilan Broz int r = 0; 50101482b76SMilan Broz 502b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 5033a7f6c99SMilan Broz iv = (u8 *)ALIGN((unsigned long)(dmreq + 1), 504c0297721SAndi Kleen crypto_ablkcipher_alignmask(any_tfm(cc)) + 1); 5053a7f6c99SMilan Broz 506b2174eebSHuang Ying dmreq->ctx = ctx; 5073a7f6c99SMilan Broz sg_init_table(&dmreq->sg_in, 1); 5083a7f6c99SMilan Broz sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT, 50901482b76SMilan Broz bv_in->bv_offset + ctx->offset_in); 51001482b76SMilan Broz 5113a7f6c99SMilan Broz sg_init_table(&dmreq->sg_out, 1); 5123a7f6c99SMilan Broz sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT, 51301482b76SMilan Broz bv_out->bv_offset + ctx->offset_out); 51401482b76SMilan Broz 51501482b76SMilan Broz ctx->offset_in += 1 << SECTOR_SHIFT; 51601482b76SMilan Broz if (ctx->offset_in >= bv_in->bv_len) { 51701482b76SMilan Broz ctx->offset_in = 0; 51801482b76SMilan Broz ctx->idx_in++; 51901482b76SMilan Broz } 52001482b76SMilan Broz 52101482b76SMilan Broz ctx->offset_out += 1 << SECTOR_SHIFT; 52201482b76SMilan Broz if (ctx->offset_out >= bv_out->bv_len) { 52301482b76SMilan Broz ctx->offset_out = 0; 52401482b76SMilan Broz ctx->idx_out++; 52501482b76SMilan Broz } 52601482b76SMilan Broz 5273a7f6c99SMilan Broz if (cc->iv_gen_ops) { 5283a7f6c99SMilan Broz r = cc->iv_gen_ops->generator(cc, iv, ctx->sector); 5293a7f6c99SMilan Broz if (r < 0) 5303a7f6c99SMilan Broz return r; 5313a7f6c99SMilan Broz } 5323a7f6c99SMilan Broz 5333a7f6c99SMilan Broz ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out, 5343a7f6c99SMilan Broz 1 << SECTOR_SHIFT, iv); 5353a7f6c99SMilan Broz 5363a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 5373a7f6c99SMilan Broz r = crypto_ablkcipher_encrypt(req); 5383a7f6c99SMilan Broz else 5393a7f6c99SMilan Broz r = crypto_ablkcipher_decrypt(req); 5403a7f6c99SMilan Broz 5413a7f6c99SMilan Broz return r; 54201482b76SMilan Broz } 54301482b76SMilan Broz 54495497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 54595497a96SMilan Broz int error); 546c0297721SAndi Kleen 547ddd42edfSMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 548ddd42edfSMilan Broz struct convert_context *ctx) 549ddd42edfSMilan Broz { 550c0297721SAndi Kleen struct crypt_cpu *this_cc = this_crypt_config(cc); 551c0297721SAndi Kleen 552c0297721SAndi Kleen if (!this_cc->req) 553c0297721SAndi Kleen this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO); 554c0297721SAndi Kleen 555c0297721SAndi Kleen ablkcipher_request_set_tfm(this_cc->req, this_cc->tfm); 556c0297721SAndi Kleen ablkcipher_request_set_callback(this_cc->req, 557c0297721SAndi Kleen CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, 558c0297721SAndi Kleen kcryptd_async_done, dmreq_of_req(cc, this_cc->req)); 559ddd42edfSMilan Broz } 560ddd42edfSMilan Broz 5611da177e4SLinus Torvalds /* 5621da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 5631da177e4SLinus Torvalds */ 5641da177e4SLinus Torvalds static int crypt_convert(struct crypt_config *cc, 5651da177e4SLinus Torvalds struct convert_context *ctx) 5661da177e4SLinus Torvalds { 567c0297721SAndi Kleen struct crypt_cpu *this_cc = this_crypt_config(cc); 5683f1e9070SMilan Broz int r; 5691da177e4SLinus Torvalds 570c8081618SMilan Broz atomic_set(&ctx->pending, 1); 571c8081618SMilan Broz 5721da177e4SLinus Torvalds while(ctx->idx_in < ctx->bio_in->bi_vcnt && 5731da177e4SLinus Torvalds ctx->idx_out < ctx->bio_out->bi_vcnt) { 5741da177e4SLinus Torvalds 5753a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 5763a7f6c99SMilan Broz 5773f1e9070SMilan Broz atomic_inc(&ctx->pending); 5783f1e9070SMilan Broz 579c0297721SAndi Kleen r = crypt_convert_block(cc, ctx, this_cc->req); 5803a7f6c99SMilan Broz 5813a7f6c99SMilan Broz switch (r) { 5823f1e9070SMilan Broz /* async */ 5833a7f6c99SMilan Broz case -EBUSY: 5843a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 5853a7f6c99SMilan Broz INIT_COMPLETION(ctx->restart); 5863a7f6c99SMilan Broz /* fall through*/ 5873a7f6c99SMilan Broz case -EINPROGRESS: 588c0297721SAndi Kleen this_cc->req = NULL; 5891da177e4SLinus Torvalds ctx->sector++; 5903a7f6c99SMilan Broz continue; 5913a7f6c99SMilan Broz 5923f1e9070SMilan Broz /* sync */ 5933f1e9070SMilan Broz case 0: 5943f1e9070SMilan Broz atomic_dec(&ctx->pending); 5953f1e9070SMilan Broz ctx->sector++; 596c7f1b204SMilan Broz cond_resched(); 5973f1e9070SMilan Broz continue; 5981da177e4SLinus Torvalds 5993f1e9070SMilan Broz /* error */ 6003f1e9070SMilan Broz default: 6013f1e9070SMilan Broz atomic_dec(&ctx->pending); 6021da177e4SLinus Torvalds return r; 6031da177e4SLinus Torvalds } 6043f1e9070SMilan Broz } 6053f1e9070SMilan Broz 6063f1e9070SMilan Broz return 0; 6073f1e9070SMilan Broz } 6081da177e4SLinus Torvalds 6096a24c718SMilan Broz static void dm_crypt_bio_destructor(struct bio *bio) 6106a24c718SMilan Broz { 611028867acSAlasdair G Kergon struct dm_crypt_io *io = bio->bi_private; 6126a24c718SMilan Broz struct crypt_config *cc = io->target->private; 6136a24c718SMilan Broz 6146a24c718SMilan Broz bio_free(bio, cc->bs); 6156a24c718SMilan Broz } 6166a24c718SMilan Broz 6171da177e4SLinus Torvalds /* 6181da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 6191da177e4SLinus Torvalds * This should never violate the device limitations 620933f01d4SMilan Broz * May return a smaller bio when running out of pages, indicated by 621933f01d4SMilan Broz * *out_of_pages set to 1. 6221da177e4SLinus Torvalds */ 623933f01d4SMilan Broz static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, 624933f01d4SMilan Broz unsigned *out_of_pages) 6251da177e4SLinus Torvalds { 626027581f3SOlaf Kirch struct crypt_config *cc = io->target->private; 6278b004457SMilan Broz struct bio *clone; 6281da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 629b4e3ca1aSAl Viro gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 63091e10625SMilan Broz unsigned i, len; 63191e10625SMilan Broz struct page *page; 6321da177e4SLinus Torvalds 6336a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 6348b004457SMilan Broz if (!clone) 6351da177e4SLinus Torvalds return NULL; 6361da177e4SLinus Torvalds 637027581f3SOlaf Kirch clone_init(io, clone); 638933f01d4SMilan Broz *out_of_pages = 0; 6396a24c718SMilan Broz 640f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 64191e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 642933f01d4SMilan Broz if (!page) { 643933f01d4SMilan Broz *out_of_pages = 1; 6441da177e4SLinus Torvalds break; 645933f01d4SMilan Broz } 6461da177e4SLinus Torvalds 6471da177e4SLinus Torvalds /* 6481da177e4SLinus Torvalds * if additional pages cannot be allocated without waiting, 6491da177e4SLinus Torvalds * return a partially allocated bio, the caller will then try 6501da177e4SLinus Torvalds * to allocate additional bios while submitting this partial bio 6511da177e4SLinus Torvalds */ 652f97380bcSOlaf Kirch if (i == (MIN_BIO_PAGES - 1)) 6531da177e4SLinus Torvalds gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 6541da177e4SLinus Torvalds 65591e10625SMilan Broz len = (size > PAGE_SIZE) ? PAGE_SIZE : size; 6561da177e4SLinus Torvalds 65791e10625SMilan Broz if (!bio_add_page(clone, page, len, 0)) { 65891e10625SMilan Broz mempool_free(page, cc->page_pool); 65991e10625SMilan Broz break; 66091e10625SMilan Broz } 66191e10625SMilan Broz 66291e10625SMilan Broz size -= len; 6631da177e4SLinus Torvalds } 6641da177e4SLinus Torvalds 6658b004457SMilan Broz if (!clone->bi_size) { 6668b004457SMilan Broz bio_put(clone); 6671da177e4SLinus Torvalds return NULL; 6681da177e4SLinus Torvalds } 6691da177e4SLinus Torvalds 6708b004457SMilan Broz return clone; 6711da177e4SLinus Torvalds } 6721da177e4SLinus Torvalds 673644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 6741da177e4SLinus Torvalds { 675644bd2f0SNeil Brown unsigned int i; 6761da177e4SLinus Torvalds struct bio_vec *bv; 6771da177e4SLinus Torvalds 678644bd2f0SNeil Brown for (i = 0; i < clone->bi_vcnt; i++) { 6798b004457SMilan Broz bv = bio_iovec_idx(clone, i); 6801da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 6811da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 6821da177e4SLinus Torvalds bv->bv_page = NULL; 6831da177e4SLinus Torvalds } 6841da177e4SLinus Torvalds } 6851da177e4SLinus Torvalds 686dc440d1eSMilan Broz static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti, 687dc440d1eSMilan Broz struct bio *bio, sector_t sector) 688dc440d1eSMilan Broz { 689dc440d1eSMilan Broz struct crypt_config *cc = ti->private; 690dc440d1eSMilan Broz struct dm_crypt_io *io; 691dc440d1eSMilan Broz 692dc440d1eSMilan Broz io = mempool_alloc(cc->io_pool, GFP_NOIO); 693dc440d1eSMilan Broz io->target = ti; 694dc440d1eSMilan Broz io->base_bio = bio; 695dc440d1eSMilan Broz io->sector = sector; 696dc440d1eSMilan Broz io->error = 0; 697393b47efSMilan Broz io->base_io = NULL; 698dc440d1eSMilan Broz atomic_set(&io->pending, 0); 699dc440d1eSMilan Broz 700dc440d1eSMilan Broz return io; 701dc440d1eSMilan Broz } 702dc440d1eSMilan Broz 7033e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 7043e1a8bddSMilan Broz { 7053e1a8bddSMilan Broz atomic_inc(&io->pending); 7063e1a8bddSMilan Broz } 7073e1a8bddSMilan Broz 7081da177e4SLinus Torvalds /* 7091da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 7101da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 711393b47efSMilan Broz * If base_io is set, wait for the last fragment to complete. 7121da177e4SLinus Torvalds */ 7135742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 7141da177e4SLinus Torvalds { 7155742fd77SMilan Broz struct crypt_config *cc = io->target->private; 716b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 717b35f8caaSMilan Broz struct dm_crypt_io *base_io = io->base_io; 718b35f8caaSMilan Broz int error = io->error; 7191da177e4SLinus Torvalds 7201da177e4SLinus Torvalds if (!atomic_dec_and_test(&io->pending)) 7211da177e4SLinus Torvalds return; 7221da177e4SLinus Torvalds 7231da177e4SLinus Torvalds mempool_free(io, cc->io_pool); 724b35f8caaSMilan Broz 725b35f8caaSMilan Broz if (likely(!base_io)) 726b35f8caaSMilan Broz bio_endio(base_bio, error); 727b35f8caaSMilan Broz else { 728b35f8caaSMilan Broz if (error && !base_io->error) 729b35f8caaSMilan Broz base_io->error = error; 730b35f8caaSMilan Broz crypt_dec_pending(base_io); 731b35f8caaSMilan Broz } 7321da177e4SLinus Torvalds } 7331da177e4SLinus Torvalds 7341da177e4SLinus Torvalds /* 735cabf08e4SMilan Broz * kcryptd/kcryptd_io: 7361da177e4SLinus Torvalds * 7371da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 73823541d2dSMilan Broz * interrupt context. 739cabf08e4SMilan Broz * 740cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 741cabf08e4SMilan Broz * 742cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 743cabf08e4SMilan Broz * 744cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 745cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 746cabf08e4SMilan Broz * to memory allocation. 747c0297721SAndi Kleen * 748c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances. 749c0297721SAndi Kleen * They should not depend on each other and do not block. 7501da177e4SLinus Torvalds */ 7516712ecf8SNeilBrown static void crypt_endio(struct bio *clone, int error) 7528b004457SMilan Broz { 753028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 7548b004457SMilan Broz struct crypt_config *cc = io->target->private; 755ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 7568b004457SMilan Broz 757adfe4770SMilan Broz if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error)) 758adfe4770SMilan Broz error = -EIO; 759adfe4770SMilan Broz 7608b004457SMilan Broz /* 7616712ecf8SNeilBrown * free the processed pages 7628b004457SMilan Broz */ 763ee7a491eSMilan Broz if (rw == WRITE) 764644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 7658b004457SMilan Broz 7668b004457SMilan Broz bio_put(clone); 767ee7a491eSMilan Broz 768ee7a491eSMilan Broz if (rw == READ && !error) { 769cabf08e4SMilan Broz kcryptd_queue_crypt(io); 7706712ecf8SNeilBrown return; 771ee7a491eSMilan Broz } 7725742fd77SMilan Broz 7735742fd77SMilan Broz if (unlikely(error)) 7745742fd77SMilan Broz io->error = error; 7755742fd77SMilan Broz 7765742fd77SMilan Broz crypt_dec_pending(io); 7778b004457SMilan Broz } 7788b004457SMilan Broz 779028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 7808b004457SMilan Broz { 7818b004457SMilan Broz struct crypt_config *cc = io->target->private; 7828b004457SMilan Broz 7838b004457SMilan Broz clone->bi_private = io; 7848b004457SMilan Broz clone->bi_end_io = crypt_endio; 7858b004457SMilan Broz clone->bi_bdev = cc->dev->bdev; 7868b004457SMilan Broz clone->bi_rw = io->base_bio->bi_rw; 787027581f3SOlaf Kirch clone->bi_destructor = dm_crypt_bio_destructor; 7888b004457SMilan Broz } 7898b004457SMilan Broz 7904e4eef64SMilan Broz static void kcryptd_io_read(struct dm_crypt_io *io) 7918b004457SMilan Broz { 7928b004457SMilan Broz struct crypt_config *cc = io->target->private; 7938b004457SMilan Broz struct bio *base_bio = io->base_bio; 7948b004457SMilan Broz struct bio *clone; 79593e605c2SMilan Broz 7963e1a8bddSMilan Broz crypt_inc_pending(io); 7978b004457SMilan Broz 7988b004457SMilan Broz /* 7998b004457SMilan Broz * The block layer might modify the bvec array, so always 8008b004457SMilan Broz * copy the required bvecs because we need the original 8018b004457SMilan Broz * one in order to decrypt the whole bio data *afterwards*. 8028b004457SMilan Broz */ 8036a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs); 80493e605c2SMilan Broz if (unlikely(!clone)) { 8055742fd77SMilan Broz io->error = -ENOMEM; 8065742fd77SMilan Broz crypt_dec_pending(io); 80723541d2dSMilan Broz return; 80893e605c2SMilan Broz } 8098b004457SMilan Broz 8108b004457SMilan Broz clone_init(io, clone); 8118b004457SMilan Broz clone->bi_idx = 0; 8128b004457SMilan Broz clone->bi_vcnt = bio_segments(base_bio); 8138b004457SMilan Broz clone->bi_size = base_bio->bi_size; 8140c395b0fSMilan Broz clone->bi_sector = cc->start + io->sector; 8158b004457SMilan Broz memcpy(clone->bi_io_vec, bio_iovec(base_bio), 8168b004457SMilan Broz sizeof(struct bio_vec) * clone->bi_vcnt); 8178b004457SMilan Broz 81893e605c2SMilan Broz generic_make_request(clone); 8198b004457SMilan Broz } 8208b004457SMilan Broz 8214e4eef64SMilan Broz static void kcryptd_io_write(struct dm_crypt_io *io) 8224e4eef64SMilan Broz { 82395497a96SMilan Broz struct bio *clone = io->ctx.bio_out; 82495497a96SMilan Broz generic_make_request(clone); 8254e4eef64SMilan Broz } 8264e4eef64SMilan Broz 827395b167cSAlasdair G Kergon static void kcryptd_io(struct work_struct *work) 828395b167cSAlasdair G Kergon { 829395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 830395b167cSAlasdair G Kergon 831395b167cSAlasdair G Kergon if (bio_data_dir(io->base_bio) == READ) 832395b167cSAlasdair G Kergon kcryptd_io_read(io); 833395b167cSAlasdair G Kergon else 834395b167cSAlasdair G Kergon kcryptd_io_write(io); 835395b167cSAlasdair G Kergon } 836395b167cSAlasdair G Kergon 837395b167cSAlasdair G Kergon static void kcryptd_queue_io(struct dm_crypt_io *io) 838395b167cSAlasdair G Kergon { 839395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 840395b167cSAlasdair G Kergon 841395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_io); 842395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 843395b167cSAlasdair G Kergon } 844395b167cSAlasdair G Kergon 84595497a96SMilan Broz static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, 84695497a96SMilan Broz int error, int async) 8474e4eef64SMilan Broz { 848dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 849dec1cedfSMilan Broz struct crypt_config *cc = io->target->private; 850dec1cedfSMilan Broz 851dec1cedfSMilan Broz if (unlikely(error < 0)) { 852dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 853dec1cedfSMilan Broz bio_put(clone); 854dec1cedfSMilan Broz io->error = -EIO; 8556c031f41SMilan Broz crypt_dec_pending(io); 856dec1cedfSMilan Broz return; 857dec1cedfSMilan Broz } 858dec1cedfSMilan Broz 859dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 860dec1cedfSMilan Broz BUG_ON(io->ctx.idx_out < clone->bi_vcnt); 861dec1cedfSMilan Broz 862dec1cedfSMilan Broz clone->bi_sector = cc->start + io->sector; 863899c95d3SMilan Broz 86495497a96SMilan Broz if (async) 86595497a96SMilan Broz kcryptd_queue_io(io); 8661e37bb8eSAlasdair G Kergon else 867899c95d3SMilan Broz generic_make_request(clone); 8684e4eef64SMilan Broz } 8694e4eef64SMilan Broz 870fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 8718b004457SMilan Broz { 8728b004457SMilan Broz struct crypt_config *cc = io->target->private; 8738b004457SMilan Broz struct bio *clone; 874393b47efSMilan Broz struct dm_crypt_io *new_io; 875c8081618SMilan Broz int crypt_finished; 876933f01d4SMilan Broz unsigned out_of_pages = 0; 877dec1cedfSMilan Broz unsigned remaining = io->base_bio->bi_size; 878b635b00eSMilan Broz sector_t sector = io->sector; 879dec1cedfSMilan Broz int r; 8808b004457SMilan Broz 88193e605c2SMilan Broz /* 882fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 883fc5a5e9aSMilan Broz */ 884fc5a5e9aSMilan Broz crypt_inc_pending(io); 885b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 886fc5a5e9aSMilan Broz 887fc5a5e9aSMilan Broz /* 88893e605c2SMilan Broz * The allocated buffers can be smaller than the whole bio, 88993e605c2SMilan Broz * so repeat the whole process until all the data can be handled. 89093e605c2SMilan Broz */ 89193e605c2SMilan Broz while (remaining) { 892933f01d4SMilan Broz clone = crypt_alloc_buffer(io, remaining, &out_of_pages); 89323541d2dSMilan Broz if (unlikely(!clone)) { 8945742fd77SMilan Broz io->error = -ENOMEM; 895fc5a5e9aSMilan Broz break; 89623541d2dSMilan Broz } 8978b004457SMilan Broz 89853017030SMilan Broz io->ctx.bio_out = clone; 89953017030SMilan Broz io->ctx.idx_out = 0; 9008b004457SMilan Broz 90193e605c2SMilan Broz remaining -= clone->bi_size; 902b635b00eSMilan Broz sector += bio_sectors(clone); 903dec1cedfSMilan Broz 9044e594098SMilan Broz crypt_inc_pending(io); 905dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 906c8081618SMilan Broz crypt_finished = atomic_dec_and_test(&io->ctx.pending); 907dec1cedfSMilan Broz 908c8081618SMilan Broz /* Encryption was already finished, submit io now */ 909c8081618SMilan Broz if (crypt_finished) { 91095497a96SMilan Broz kcryptd_crypt_write_io_submit(io, r, 0); 911c8081618SMilan Broz 912c8081618SMilan Broz /* 913c8081618SMilan Broz * If there was an error, do not try next fragments. 914c8081618SMilan Broz * For async, error is processed in async handler. 915c8081618SMilan Broz */ 9166c031f41SMilan Broz if (unlikely(r < 0)) 917fc5a5e9aSMilan Broz break; 918b635b00eSMilan Broz 919b635b00eSMilan Broz io->sector = sector; 9204e594098SMilan Broz } 92193e605c2SMilan Broz 922933f01d4SMilan Broz /* 923933f01d4SMilan Broz * Out of memory -> run queues 924933f01d4SMilan Broz * But don't wait if split was due to the io size restriction 925933f01d4SMilan Broz */ 926933f01d4SMilan Broz if (unlikely(out_of_pages)) 9278aa7e847SJens Axboe congestion_wait(BLK_RW_ASYNC, HZ/100); 928933f01d4SMilan Broz 929393b47efSMilan Broz /* 930393b47efSMilan Broz * With async crypto it is unsafe to share the crypto context 931393b47efSMilan Broz * between fragments, so switch to a new dm_crypt_io structure. 932393b47efSMilan Broz */ 933393b47efSMilan Broz if (unlikely(!crypt_finished && remaining)) { 934393b47efSMilan Broz new_io = crypt_io_alloc(io->target, io->base_bio, 935393b47efSMilan Broz sector); 936393b47efSMilan Broz crypt_inc_pending(new_io); 937393b47efSMilan Broz crypt_convert_init(cc, &new_io->ctx, NULL, 938393b47efSMilan Broz io->base_bio, sector); 939393b47efSMilan Broz new_io->ctx.idx_in = io->ctx.idx_in; 940393b47efSMilan Broz new_io->ctx.offset_in = io->ctx.offset_in; 941393b47efSMilan Broz 942393b47efSMilan Broz /* 943393b47efSMilan Broz * Fragments after the first use the base_io 944393b47efSMilan Broz * pending count. 945393b47efSMilan Broz */ 946393b47efSMilan Broz if (!io->base_io) 947393b47efSMilan Broz new_io->base_io = io; 948393b47efSMilan Broz else { 949393b47efSMilan Broz new_io->base_io = io->base_io; 950393b47efSMilan Broz crypt_inc_pending(io->base_io); 951393b47efSMilan Broz crypt_dec_pending(io); 952393b47efSMilan Broz } 953393b47efSMilan Broz 954393b47efSMilan Broz io = new_io; 955393b47efSMilan Broz } 9568b004457SMilan Broz } 957899c95d3SMilan Broz 958899c95d3SMilan Broz crypt_dec_pending(io); 95984131db6SMilan Broz } 96084131db6SMilan Broz 9614e4eef64SMilan Broz static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error) 9625742fd77SMilan Broz { 9635742fd77SMilan Broz if (unlikely(error < 0)) 9645742fd77SMilan Broz io->error = -EIO; 9655742fd77SMilan Broz 9665742fd77SMilan Broz crypt_dec_pending(io); 9675742fd77SMilan Broz } 9685742fd77SMilan Broz 9694e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 9708b004457SMilan Broz { 9718b004457SMilan Broz struct crypt_config *cc = io->target->private; 9725742fd77SMilan Broz int r = 0; 9738b004457SMilan Broz 9743e1a8bddSMilan Broz crypt_inc_pending(io); 9753a7f6c99SMilan Broz 97653017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 9770c395b0fSMilan Broz io->sector); 9788b004457SMilan Broz 9795742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 9805742fd77SMilan Broz 9813f1e9070SMilan Broz if (atomic_dec_and_test(&io->ctx.pending)) 9824e4eef64SMilan Broz kcryptd_crypt_read_done(io, r); 9833a7f6c99SMilan Broz 9843a7f6c99SMilan Broz crypt_dec_pending(io); 9858b004457SMilan Broz } 9868b004457SMilan Broz 98795497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 98895497a96SMilan Broz int error) 98995497a96SMilan Broz { 990b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 991b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 99295497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 99395497a96SMilan Broz struct crypt_config *cc = io->target->private; 99495497a96SMilan Broz 99595497a96SMilan Broz if (error == -EINPROGRESS) { 99695497a96SMilan Broz complete(&ctx->restart); 99795497a96SMilan Broz return; 99895497a96SMilan Broz } 99995497a96SMilan Broz 1000b2174eebSHuang Ying mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool); 100195497a96SMilan Broz 100295497a96SMilan Broz if (!atomic_dec_and_test(&ctx->pending)) 100395497a96SMilan Broz return; 100495497a96SMilan Broz 100595497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 100695497a96SMilan Broz kcryptd_crypt_read_done(io, error); 100795497a96SMilan Broz else 100895497a96SMilan Broz kcryptd_crypt_write_io_submit(io, error, 1); 100995497a96SMilan Broz } 101095497a96SMilan Broz 10114e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 10124e4eef64SMilan Broz { 10134e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 10144e4eef64SMilan Broz 10154e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 10164e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 10174e4eef64SMilan Broz else 10184e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 10198b004457SMilan Broz } 10208b004457SMilan Broz 1021395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 1022395b167cSAlasdair G Kergon { 1023395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 1024395b167cSAlasdair G Kergon 1025395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 1026395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 1027395b167cSAlasdair G Kergon } 1028395b167cSAlasdair G Kergon 10291da177e4SLinus Torvalds /* 10301da177e4SLinus Torvalds * Decode key from its hex representation 10311da177e4SLinus Torvalds */ 10321da177e4SLinus Torvalds static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 10331da177e4SLinus Torvalds { 10341da177e4SLinus Torvalds char buffer[3]; 10351da177e4SLinus Torvalds char *endp; 10361da177e4SLinus Torvalds unsigned int i; 10371da177e4SLinus Torvalds 10381da177e4SLinus Torvalds buffer[2] = '\0'; 10391da177e4SLinus Torvalds 10401da177e4SLinus Torvalds for (i = 0; i < size; i++) { 10411da177e4SLinus Torvalds buffer[0] = *hex++; 10421da177e4SLinus Torvalds buffer[1] = *hex++; 10431da177e4SLinus Torvalds 10441da177e4SLinus Torvalds key[i] = (u8)simple_strtoul(buffer, &endp, 16); 10451da177e4SLinus Torvalds 10461da177e4SLinus Torvalds if (endp != &buffer[2]) 10471da177e4SLinus Torvalds return -EINVAL; 10481da177e4SLinus Torvalds } 10491da177e4SLinus Torvalds 10501da177e4SLinus Torvalds if (*hex != '\0') 10511da177e4SLinus Torvalds return -EINVAL; 10521da177e4SLinus Torvalds 10531da177e4SLinus Torvalds return 0; 10541da177e4SLinus Torvalds } 10551da177e4SLinus Torvalds 10561da177e4SLinus Torvalds /* 10571da177e4SLinus Torvalds * Encode key into its hex representation 10581da177e4SLinus Torvalds */ 10591da177e4SLinus Torvalds static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 10601da177e4SLinus Torvalds { 10611da177e4SLinus Torvalds unsigned int i; 10621da177e4SLinus Torvalds 10631da177e4SLinus Torvalds for (i = 0; i < size; i++) { 10641da177e4SLinus Torvalds sprintf(hex, "%02x", *key); 10651da177e4SLinus Torvalds hex += 2; 10661da177e4SLinus Torvalds key++; 10671da177e4SLinus Torvalds } 10681da177e4SLinus Torvalds } 10691da177e4SLinus Torvalds 1070c0297721SAndi Kleen static int crypt_setkey_allcpus(struct crypt_config *cc) 1071c0297721SAndi Kleen { 1072c0297721SAndi Kleen int cpu, err = 0, r; 1073c0297721SAndi Kleen 1074c0297721SAndi Kleen for_each_possible_cpu(cpu) { 1075c0297721SAndi Kleen r = crypto_ablkcipher_setkey(per_cpu_ptr(cc->cpu, cpu)->tfm, 1076c0297721SAndi Kleen cc->key, cc->key_size); 1077c0297721SAndi Kleen if (r) 1078c0297721SAndi Kleen err = r; 1079c0297721SAndi Kleen } 1080c0297721SAndi Kleen 1081c0297721SAndi Kleen return err; 1082c0297721SAndi Kleen } 1083c0297721SAndi Kleen 1084e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 1085e48d4bbfSMilan Broz { 108669a8cfcdSMilan Broz /* The key size may not be changed. */ 108769a8cfcdSMilan Broz if (cc->key_size != (strlen(key) >> 1)) 1088e48d4bbfSMilan Broz return -EINVAL; 1089e48d4bbfSMilan Broz 109069a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */ 109169a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-")) 109269a8cfcdSMilan Broz return -EINVAL; 1093e48d4bbfSMilan Broz 109469a8cfcdSMilan Broz if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0) 1095e48d4bbfSMilan Broz return -EINVAL; 1096e48d4bbfSMilan Broz 1097e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1098e48d4bbfSMilan Broz 1099c0297721SAndi Kleen return crypt_setkey_allcpus(cc); 1100e48d4bbfSMilan Broz } 1101e48d4bbfSMilan Broz 1102e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 1103e48d4bbfSMilan Broz { 1104e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1105e48d4bbfSMilan Broz memset(&cc->key, 0, cc->key_size * sizeof(u8)); 1106c0297721SAndi Kleen 1107c0297721SAndi Kleen return crypt_setkey_allcpus(cc); 1108e48d4bbfSMilan Broz } 1109e48d4bbfSMilan Broz 111028513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 111128513fccSMilan Broz { 111228513fccSMilan Broz struct crypt_config *cc = ti->private; 1113c0297721SAndi Kleen struct crypt_cpu *cpu_cc; 1114c0297721SAndi Kleen int cpu; 111528513fccSMilan Broz 111628513fccSMilan Broz ti->private = NULL; 111728513fccSMilan Broz 111828513fccSMilan Broz if (!cc) 111928513fccSMilan Broz return; 112028513fccSMilan Broz 112128513fccSMilan Broz if (cc->io_queue) 112228513fccSMilan Broz destroy_workqueue(cc->io_queue); 112328513fccSMilan Broz if (cc->crypt_queue) 112428513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 112528513fccSMilan Broz 1126c0297721SAndi Kleen if (cc->cpu) 1127c0297721SAndi Kleen for_each_possible_cpu(cpu) { 1128c0297721SAndi Kleen cpu_cc = per_cpu_ptr(cc->cpu, cpu); 1129c0297721SAndi Kleen if (cpu_cc->req) 1130c0297721SAndi Kleen mempool_free(cpu_cc->req, cc->req_pool); 1131c0297721SAndi Kleen if (cpu_cc->tfm) 1132c0297721SAndi Kleen crypto_free_ablkcipher(cpu_cc->tfm); 1133c0297721SAndi Kleen } 1134c0297721SAndi Kleen 113528513fccSMilan Broz if (cc->bs) 113628513fccSMilan Broz bioset_free(cc->bs); 113728513fccSMilan Broz 113828513fccSMilan Broz if (cc->page_pool) 113928513fccSMilan Broz mempool_destroy(cc->page_pool); 114028513fccSMilan Broz if (cc->req_pool) 114128513fccSMilan Broz mempool_destroy(cc->req_pool); 114228513fccSMilan Broz if (cc->io_pool) 114328513fccSMilan Broz mempool_destroy(cc->io_pool); 114428513fccSMilan Broz 114528513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 114628513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 114728513fccSMilan Broz 114828513fccSMilan Broz if (cc->dev) 114928513fccSMilan Broz dm_put_device(ti, cc->dev); 115028513fccSMilan Broz 1151c0297721SAndi Kleen if (cc->cpu) 1152c0297721SAndi Kleen free_percpu(cc->cpu); 1153c0297721SAndi Kleen 11545ebaee6dSMilan Broz kzfree(cc->cipher); 11557dbcd137SMilan Broz kzfree(cc->cipher_string); 115628513fccSMilan Broz 115728513fccSMilan Broz /* Must zero key material before freeing */ 115828513fccSMilan Broz kzfree(cc); 115928513fccSMilan Broz } 116028513fccSMilan Broz 11615ebaee6dSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, 11625ebaee6dSMilan Broz char *cipher_in, char *key) 11631da177e4SLinus Torvalds { 11645ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 1165c0297721SAndi Kleen struct crypto_ablkcipher *tfm; 11665ebaee6dSMilan Broz char *tmp, *cipher, *chainmode, *ivmode, *ivopts; 11675ebaee6dSMilan Broz char *cipher_api = NULL; 1168c0297721SAndi Kleen int cpu, ret = -EINVAL; 11691da177e4SLinus Torvalds 11705ebaee6dSMilan Broz /* Convert to crypto api definition? */ 11715ebaee6dSMilan Broz if (strchr(cipher_in, '(')) { 11725ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 11731da177e4SLinus Torvalds return -EINVAL; 11741da177e4SLinus Torvalds } 11751da177e4SLinus Torvalds 11767dbcd137SMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL); 11777dbcd137SMilan Broz if (!cc->cipher_string) 11787dbcd137SMilan Broz goto bad_mem; 11797dbcd137SMilan Broz 11805ebaee6dSMilan Broz /* 11815ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 11825ebaee6dSMilan Broz * cipher-mode-iv:ivopts 11835ebaee6dSMilan Broz */ 11845ebaee6dSMilan Broz tmp = cipher_in; 11851da177e4SLinus Torvalds cipher = strsep(&tmp, "-"); 11865ebaee6dSMilan Broz 11875ebaee6dSMilan Broz cc->cipher = kstrdup(cipher, GFP_KERNEL); 11885ebaee6dSMilan Broz if (!cc->cipher) 11895ebaee6dSMilan Broz goto bad_mem; 11905ebaee6dSMilan Broz 11911da177e4SLinus Torvalds chainmode = strsep(&tmp, "-"); 11921da177e4SLinus Torvalds ivopts = strsep(&tmp, "-"); 11931da177e4SLinus Torvalds ivmode = strsep(&ivopts, ":"); 11941da177e4SLinus Torvalds 11951da177e4SLinus Torvalds if (tmp) 11965ebaee6dSMilan Broz DMWARN("Ignoring unexpected additional cipher options"); 11971da177e4SLinus Torvalds 1198c0297721SAndi Kleen cc->cpu = alloc_percpu(struct crypt_cpu); 1199c0297721SAndi Kleen if (!cc->cpu) { 1200c0297721SAndi Kleen ti->error = "Cannot allocate per cpu state"; 1201c0297721SAndi Kleen goto bad_mem; 1202c0297721SAndi Kleen } 1203c0297721SAndi Kleen 12047dbcd137SMilan Broz /* 12057dbcd137SMilan Broz * For compatibility with the original dm-crypt mapping format, if 12067dbcd137SMilan Broz * only the cipher name is supplied, use cbc-plain. 12077dbcd137SMilan Broz */ 12085ebaee6dSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) { 12091da177e4SLinus Torvalds chainmode = "cbc"; 12101da177e4SLinus Torvalds ivmode = "plain"; 12111da177e4SLinus Torvalds } 12121da177e4SLinus Torvalds 1213d1806f6aSHerbert Xu if (strcmp(chainmode, "ecb") && !ivmode) { 12145ebaee6dSMilan Broz ti->error = "IV mechanism required"; 12155ebaee6dSMilan Broz return -EINVAL; 12161da177e4SLinus Torvalds } 12171da177e4SLinus Torvalds 12185ebaee6dSMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 12195ebaee6dSMilan Broz if (!cipher_api) 12205ebaee6dSMilan Broz goto bad_mem; 12215ebaee6dSMilan Broz 12225ebaee6dSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 12235ebaee6dSMilan Broz "%s(%s)", chainmode, cipher); 12245ebaee6dSMilan Broz if (ret < 0) { 12255ebaee6dSMilan Broz kfree(cipher_api); 12265ebaee6dSMilan Broz goto bad_mem; 1227d1806f6aSHerbert Xu } 1228d1806f6aSHerbert Xu 12295ebaee6dSMilan Broz /* Allocate cipher */ 1230c0297721SAndi Kleen for_each_possible_cpu(cpu) { 1231c0297721SAndi Kleen tfm = crypto_alloc_ablkcipher(cipher_api, 0, 0); 1232c0297721SAndi Kleen if (IS_ERR(tfm)) { 1233c0297721SAndi Kleen ret = PTR_ERR(tfm); 123472d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm"; 123528513fccSMilan Broz goto bad; 12361da177e4SLinus Torvalds } 1237c0297721SAndi Kleen per_cpu_ptr(cc->cpu, cpu)->tfm = tfm; 1238c0297721SAndi Kleen } 12391da177e4SLinus Torvalds 12405ebaee6dSMilan Broz /* Initialize and set key */ 12415ebaee6dSMilan Broz ret = crypt_set_key(cc, key); 124228513fccSMilan Broz if (ret < 0) { 12430b430958SMilan Broz ti->error = "Error decoding and setting key"; 124428513fccSMilan Broz goto bad; 12450b430958SMilan Broz } 12460b430958SMilan Broz 12475ebaee6dSMilan Broz /* Initialize IV */ 1248c0297721SAndi Kleen cc->iv_size = crypto_ablkcipher_ivsize(any_tfm(cc)); 12495ebaee6dSMilan Broz if (cc->iv_size) 12505ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 12515ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 12525ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 12535ebaee6dSMilan Broz else if (ivmode) { 12545ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 12555ebaee6dSMilan Broz ivmode = NULL; 12565ebaee6dSMilan Broz } 12575ebaee6dSMilan Broz 12585ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 12591da177e4SLinus Torvalds if (ivmode == NULL) 12601da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 12611da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 12621da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 126361afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 126461afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 12651da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 12661da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 126748527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 126848527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 126946b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 127046b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 12711da177e4SLinus Torvalds else { 12725ebaee6dSMilan Broz ret = -EINVAL; 127372d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 127428513fccSMilan Broz goto bad; 12751da177e4SLinus Torvalds } 12761da177e4SLinus Torvalds 127728513fccSMilan Broz /* Allocate IV */ 127828513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 127928513fccSMilan Broz ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 128028513fccSMilan Broz if (ret < 0) { 128128513fccSMilan Broz ti->error = "Error creating IV"; 128228513fccSMilan Broz goto bad; 128328513fccSMilan Broz } 128428513fccSMilan Broz } 12851da177e4SLinus Torvalds 128628513fccSMilan Broz /* Initialize IV (set keys for ESSIV etc) */ 128728513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 128828513fccSMilan Broz ret = cc->iv_gen_ops->init(cc); 128928513fccSMilan Broz if (ret < 0) { 1290b95bf2d3SMilan Broz ti->error = "Error initialising IV"; 129128513fccSMilan Broz goto bad; 129228513fccSMilan Broz } 1293b95bf2d3SMilan Broz } 1294b95bf2d3SMilan Broz 12955ebaee6dSMilan Broz ret = 0; 12965ebaee6dSMilan Broz bad: 12975ebaee6dSMilan Broz kfree(cipher_api); 12985ebaee6dSMilan Broz return ret; 12995ebaee6dSMilan Broz 13005ebaee6dSMilan Broz bad_mem: 13015ebaee6dSMilan Broz ti->error = "Cannot allocate cipher strings"; 13025ebaee6dSMilan Broz return -ENOMEM; 13031da177e4SLinus Torvalds } 13045ebaee6dSMilan Broz 13055ebaee6dSMilan Broz /* 13065ebaee6dSMilan Broz * Construct an encryption mapping: 13075ebaee6dSMilan Broz * <cipher> <key> <iv_offset> <dev_path> <start> 13085ebaee6dSMilan Broz */ 13095ebaee6dSMilan Broz static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 13105ebaee6dSMilan Broz { 13115ebaee6dSMilan Broz struct crypt_config *cc; 13125ebaee6dSMilan Broz unsigned int key_size; 13135ebaee6dSMilan Broz unsigned long long tmpll; 13145ebaee6dSMilan Broz int ret; 13155ebaee6dSMilan Broz 13165ebaee6dSMilan Broz if (argc != 5) { 13175ebaee6dSMilan Broz ti->error = "Not enough arguments"; 13185ebaee6dSMilan Broz return -EINVAL; 13191da177e4SLinus Torvalds } 13201da177e4SLinus Torvalds 13215ebaee6dSMilan Broz key_size = strlen(argv[1]) >> 1; 13225ebaee6dSMilan Broz 13235ebaee6dSMilan Broz cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 13245ebaee6dSMilan Broz if (!cc) { 13255ebaee6dSMilan Broz ti->error = "Cannot allocate encryption context"; 13265ebaee6dSMilan Broz return -ENOMEM; 13275ebaee6dSMilan Broz } 132869a8cfcdSMilan Broz cc->key_size = key_size; 13295ebaee6dSMilan Broz 13305ebaee6dSMilan Broz ti->private = cc; 13315ebaee6dSMilan Broz ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 13325ebaee6dSMilan Broz if (ret < 0) 13335ebaee6dSMilan Broz goto bad; 13345ebaee6dSMilan Broz 133528513fccSMilan Broz ret = -ENOMEM; 133693d2341cSMatthew Dobson cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 13371da177e4SLinus Torvalds if (!cc->io_pool) { 133872d94861SAlasdair G Kergon ti->error = "Cannot allocate crypt io mempool"; 133928513fccSMilan Broz goto bad; 13401da177e4SLinus Torvalds } 13411da177e4SLinus Torvalds 1342ddd42edfSMilan Broz cc->dmreq_start = sizeof(struct ablkcipher_request); 1343c0297721SAndi Kleen cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc)); 1344ddd42edfSMilan Broz cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment()); 1345c0297721SAndi Kleen cc->dmreq_start += crypto_ablkcipher_alignmask(any_tfm(cc)) & 13463a7f6c99SMilan Broz ~(crypto_tfm_ctx_alignment() - 1); 1347ddd42edfSMilan Broz 1348ddd42edfSMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1349ddd42edfSMilan Broz sizeof(struct dm_crypt_request) + cc->iv_size); 1350ddd42edfSMilan Broz if (!cc->req_pool) { 1351ddd42edfSMilan Broz ti->error = "Cannot allocate crypt request mempool"; 135228513fccSMilan Broz goto bad; 1353ddd42edfSMilan Broz } 1354ddd42edfSMilan Broz 1355a19b27ceSMatthew Dobson cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 13561da177e4SLinus Torvalds if (!cc->page_pool) { 135772d94861SAlasdair G Kergon ti->error = "Cannot allocate page mempool"; 135828513fccSMilan Broz goto bad; 13591da177e4SLinus Torvalds } 13601da177e4SLinus Torvalds 1361bb799ca0SJens Axboe cc->bs = bioset_create(MIN_IOS, 0); 13626a24c718SMilan Broz if (!cc->bs) { 13636a24c718SMilan Broz ti->error = "Cannot allocate crypt bioset"; 136428513fccSMilan Broz goto bad; 13656a24c718SMilan Broz } 13666a24c718SMilan Broz 136728513fccSMilan Broz ret = -EINVAL; 13684ee218cdSAndrew Morton if (sscanf(argv[2], "%llu", &tmpll) != 1) { 136972d94861SAlasdair G Kergon ti->error = "Invalid iv_offset sector"; 137028513fccSMilan Broz goto bad; 13711da177e4SLinus Torvalds } 13724ee218cdSAndrew Morton cc->iv_offset = tmpll; 13731da177e4SLinus Torvalds 137428513fccSMilan Broz if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev)) { 137528513fccSMilan Broz ti->error = "Device lookup failed"; 137628513fccSMilan Broz goto bad; 137728513fccSMilan Broz } 137828513fccSMilan Broz 13794ee218cdSAndrew Morton if (sscanf(argv[4], "%llu", &tmpll) != 1) { 138072d94861SAlasdair G Kergon ti->error = "Invalid device sector"; 138128513fccSMilan Broz goto bad; 13821da177e4SLinus Torvalds } 13834ee218cdSAndrew Morton cc->start = tmpll; 13841da177e4SLinus Torvalds 138528513fccSMilan Broz ret = -ENOMEM; 1386c0297721SAndi Kleen cc->io_queue = alloc_workqueue("kcryptd_io", 1387c0297721SAndi Kleen WQ_NON_REENTRANT| 1388c0297721SAndi Kleen WQ_MEM_RECLAIM, 1389c0297721SAndi Kleen 1); 1390cabf08e4SMilan Broz if (!cc->io_queue) { 1391cabf08e4SMilan Broz ti->error = "Couldn't create kcryptd io queue"; 139228513fccSMilan Broz goto bad; 1393cabf08e4SMilan Broz } 1394cabf08e4SMilan Broz 1395c0297721SAndi Kleen cc->crypt_queue = alloc_workqueue("kcryptd", 1396c0297721SAndi Kleen WQ_NON_REENTRANT| 1397c0297721SAndi Kleen WQ_CPU_INTENSIVE| 1398c0297721SAndi Kleen WQ_MEM_RECLAIM, 1399c0297721SAndi Kleen 1); 1400cabf08e4SMilan Broz if (!cc->crypt_queue) { 14019934a8beSMilan Broz ti->error = "Couldn't create kcryptd queue"; 140228513fccSMilan Broz goto bad; 14039934a8beSMilan Broz } 14049934a8beSMilan Broz 1405647c7db1SMikulas Patocka ti->num_flush_requests = 1; 14061da177e4SLinus Torvalds return 0; 14071da177e4SLinus Torvalds 140828513fccSMilan Broz bad: 140928513fccSMilan Broz crypt_dtr(ti); 141028513fccSMilan Broz return ret; 14111da177e4SLinus Torvalds } 14121da177e4SLinus Torvalds 14131da177e4SLinus Torvalds static int crypt_map(struct dm_target *ti, struct bio *bio, 14141da177e4SLinus Torvalds union map_info *map_context) 14151da177e4SLinus Torvalds { 1416028867acSAlasdair G Kergon struct dm_crypt_io *io; 1417647c7db1SMikulas Patocka struct crypt_config *cc; 1418647c7db1SMikulas Patocka 1419d87f4c14STejun Heo if (bio->bi_rw & REQ_FLUSH) { 1420647c7db1SMikulas Patocka cc = ti->private; 1421647c7db1SMikulas Patocka bio->bi_bdev = cc->dev->bdev; 1422647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 1423647c7db1SMikulas Patocka } 14241da177e4SLinus Torvalds 1425b441a262SAlasdair G Kergon io = crypt_io_alloc(ti, bio, dm_target_offset(ti, bio->bi_sector)); 1426cabf08e4SMilan Broz 1427cabf08e4SMilan Broz if (bio_data_dir(io->base_bio) == READ) 142823541d2dSMilan Broz kcryptd_queue_io(io); 1429cabf08e4SMilan Broz else 1430cabf08e4SMilan Broz kcryptd_queue_crypt(io); 14311da177e4SLinus Torvalds 1432d2a7ad29SKiyoshi Ueda return DM_MAPIO_SUBMITTED; 14331da177e4SLinus Torvalds } 14341da177e4SLinus Torvalds 14351da177e4SLinus Torvalds static int crypt_status(struct dm_target *ti, status_type_t type, 14361da177e4SLinus Torvalds char *result, unsigned int maxlen) 14371da177e4SLinus Torvalds { 14385ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 14391da177e4SLinus Torvalds unsigned int sz = 0; 14401da177e4SLinus Torvalds 14411da177e4SLinus Torvalds switch (type) { 14421da177e4SLinus Torvalds case STATUSTYPE_INFO: 14431da177e4SLinus Torvalds result[0] = '\0'; 14441da177e4SLinus Torvalds break; 14451da177e4SLinus Torvalds 14461da177e4SLinus Torvalds case STATUSTYPE_TABLE: 14477dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string); 14481da177e4SLinus Torvalds 14491da177e4SLinus Torvalds if (cc->key_size > 0) { 14501da177e4SLinus Torvalds if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 14511da177e4SLinus Torvalds return -ENOMEM; 14521da177e4SLinus Torvalds 14531da177e4SLinus Torvalds crypt_encode_key(result + sz, cc->key, cc->key_size); 14541da177e4SLinus Torvalds sz += cc->key_size << 1; 14551da177e4SLinus Torvalds } else { 14561da177e4SLinus Torvalds if (sz >= maxlen) 14571da177e4SLinus Torvalds return -ENOMEM; 14581da177e4SLinus Torvalds result[sz++] = '-'; 14591da177e4SLinus Torvalds } 14601da177e4SLinus Torvalds 14614ee218cdSAndrew Morton DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 14624ee218cdSAndrew Morton cc->dev->name, (unsigned long long)cc->start); 14631da177e4SLinus Torvalds break; 14641da177e4SLinus Torvalds } 14651da177e4SLinus Torvalds return 0; 14661da177e4SLinus Torvalds } 14671da177e4SLinus Torvalds 1468e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 1469e48d4bbfSMilan Broz { 1470e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1471e48d4bbfSMilan Broz 1472e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1473e48d4bbfSMilan Broz } 1474e48d4bbfSMilan Broz 1475e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 1476e48d4bbfSMilan Broz { 1477e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1478e48d4bbfSMilan Broz 1479e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1480e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 1481e48d4bbfSMilan Broz return -EAGAIN; 1482e48d4bbfSMilan Broz } 1483e48d4bbfSMilan Broz 1484e48d4bbfSMilan Broz return 0; 1485e48d4bbfSMilan Broz } 1486e48d4bbfSMilan Broz 1487e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 1488e48d4bbfSMilan Broz { 1489e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1490e48d4bbfSMilan Broz 1491e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1492e48d4bbfSMilan Broz } 1493e48d4bbfSMilan Broz 1494e48d4bbfSMilan Broz /* Message interface 1495e48d4bbfSMilan Broz * key set <key> 1496e48d4bbfSMilan Broz * key wipe 1497e48d4bbfSMilan Broz */ 1498e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1499e48d4bbfSMilan Broz { 1500e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1501542da317SMilan Broz int ret = -EINVAL; 1502e48d4bbfSMilan Broz 1503e48d4bbfSMilan Broz if (argc < 2) 1504e48d4bbfSMilan Broz goto error; 1505e48d4bbfSMilan Broz 1506e48d4bbfSMilan Broz if (!strnicmp(argv[0], MESG_STR("key"))) { 1507e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1508e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 1509e48d4bbfSMilan Broz return -EINVAL; 1510e48d4bbfSMilan Broz } 1511542da317SMilan Broz if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) { 1512542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 1513542da317SMilan Broz if (ret) 1514542da317SMilan Broz return ret; 1515542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 1516542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 1517542da317SMilan Broz return ret; 1518542da317SMilan Broz } 1519542da317SMilan Broz if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) { 1520542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 1521542da317SMilan Broz ret = cc->iv_gen_ops->wipe(cc); 1522542da317SMilan Broz if (ret) 1523542da317SMilan Broz return ret; 1524542da317SMilan Broz } 1525e48d4bbfSMilan Broz return crypt_wipe_key(cc); 1526e48d4bbfSMilan Broz } 1527542da317SMilan Broz } 1528e48d4bbfSMilan Broz 1529e48d4bbfSMilan Broz error: 1530e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 1531e48d4bbfSMilan Broz return -EINVAL; 1532e48d4bbfSMilan Broz } 1533e48d4bbfSMilan Broz 1534d41e26b9SMilan Broz static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 1535d41e26b9SMilan Broz struct bio_vec *biovec, int max_size) 1536d41e26b9SMilan Broz { 1537d41e26b9SMilan Broz struct crypt_config *cc = ti->private; 1538d41e26b9SMilan Broz struct request_queue *q = bdev_get_queue(cc->dev->bdev); 1539d41e26b9SMilan Broz 1540d41e26b9SMilan Broz if (!q->merge_bvec_fn) 1541d41e26b9SMilan Broz return max_size; 1542d41e26b9SMilan Broz 1543d41e26b9SMilan Broz bvm->bi_bdev = cc->dev->bdev; 1544b441a262SAlasdair G Kergon bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector); 1545d41e26b9SMilan Broz 1546d41e26b9SMilan Broz return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 1547d41e26b9SMilan Broz } 1548d41e26b9SMilan Broz 1549af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 1550af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 1551af4874e0SMike Snitzer { 1552af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 1553af4874e0SMike Snitzer 15545dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 1555af4874e0SMike Snitzer } 1556af4874e0SMike Snitzer 15571da177e4SLinus Torvalds static struct target_type crypt_target = { 15581da177e4SLinus Torvalds .name = "crypt", 1559c0297721SAndi Kleen .version = {1, 9, 0}, 15601da177e4SLinus Torvalds .module = THIS_MODULE, 15611da177e4SLinus Torvalds .ctr = crypt_ctr, 15621da177e4SLinus Torvalds .dtr = crypt_dtr, 15631da177e4SLinus Torvalds .map = crypt_map, 15641da177e4SLinus Torvalds .status = crypt_status, 1565e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 1566e48d4bbfSMilan Broz .preresume = crypt_preresume, 1567e48d4bbfSMilan Broz .resume = crypt_resume, 1568e48d4bbfSMilan Broz .message = crypt_message, 1569d41e26b9SMilan Broz .merge = crypt_merge, 1570af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 15711da177e4SLinus Torvalds }; 15721da177e4SLinus Torvalds 15731da177e4SLinus Torvalds static int __init dm_crypt_init(void) 15741da177e4SLinus Torvalds { 15751da177e4SLinus Torvalds int r; 15761da177e4SLinus Torvalds 1577028867acSAlasdair G Kergon _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 15781da177e4SLinus Torvalds if (!_crypt_io_pool) 15791da177e4SLinus Torvalds return -ENOMEM; 15801da177e4SLinus Torvalds 15811da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 15821da177e4SLinus Torvalds if (r < 0) { 158372d94861SAlasdair G Kergon DMERR("register failed %d", r); 15849934a8beSMilan Broz kmem_cache_destroy(_crypt_io_pool); 15851da177e4SLinus Torvalds } 15861da177e4SLinus Torvalds 15871da177e4SLinus Torvalds return r; 15881da177e4SLinus Torvalds } 15891da177e4SLinus Torvalds 15901da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 15911da177e4SLinus Torvalds { 159210d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 15931da177e4SLinus Torvalds kmem_cache_destroy(_crypt_io_pool); 15941da177e4SLinus Torvalds } 15951da177e4SLinus Torvalds 15961da177e4SLinus Torvalds module_init(dm_crypt_init); 15971da177e4SLinus Torvalds module_exit(dm_crypt_exit); 15981da177e4SLinus Torvalds 15991da177e4SLinus Torvalds MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 16001da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 16011da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1602