11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 43f1e9070SMilan Broz * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 943d69034SMilan Broz #include <linux/completion.h> 10d1806f6aSHerbert Xu #include <linux/err.h> 111da177e4SLinus Torvalds #include <linux/module.h> 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/kernel.h> 141da177e4SLinus Torvalds #include <linux/bio.h> 151da177e4SLinus Torvalds #include <linux/blkdev.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/crypto.h> 191da177e4SLinus Torvalds #include <linux/workqueue.h> 203fcfab16SAndrew Morton #include <linux/backing-dev.h> 211da177e4SLinus Torvalds #include <asm/atomic.h> 22378f058cSDavid Hardeman #include <linux/scatterlist.h> 231da177e4SLinus Torvalds #include <asm/page.h> 2448527fa7SRik Snel #include <asm/unaligned.h> 251da177e4SLinus Torvalds 26586e80e6SMikulas Patocka #include <linux/device-mapper.h> 271da177e4SLinus Torvalds 2872d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 29e48d4bbfSMilan Broz #define MESG_STR(x) x, sizeof(x) 301da177e4SLinus Torvalds 311da177e4SLinus Torvalds /* 321da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 331da177e4SLinus Torvalds */ 341da177e4SLinus Torvalds struct convert_context { 3543d69034SMilan Broz struct completion restart; 361da177e4SLinus Torvalds struct bio *bio_in; 371da177e4SLinus Torvalds struct bio *bio_out; 381da177e4SLinus Torvalds unsigned int offset_in; 391da177e4SLinus Torvalds unsigned int offset_out; 401da177e4SLinus Torvalds unsigned int idx_in; 411da177e4SLinus Torvalds unsigned int idx_out; 421da177e4SLinus Torvalds sector_t sector; 4343d69034SMilan Broz atomic_t pending; 441da177e4SLinus Torvalds }; 451da177e4SLinus Torvalds 4653017030SMilan Broz /* 4753017030SMilan Broz * per bio private data 4853017030SMilan Broz */ 4953017030SMilan Broz struct dm_crypt_io { 5053017030SMilan Broz struct dm_target *target; 5153017030SMilan Broz struct bio *base_bio; 5253017030SMilan Broz struct work_struct work; 5353017030SMilan Broz 5453017030SMilan Broz struct convert_context ctx; 5553017030SMilan Broz 5653017030SMilan Broz atomic_t pending; 5753017030SMilan Broz int error; 580c395b0fSMilan Broz sector_t sector; 59393b47efSMilan Broz struct dm_crypt_io *base_io; 6053017030SMilan Broz }; 6153017030SMilan Broz 6201482b76SMilan Broz struct dm_crypt_request { 63b2174eebSHuang Ying struct convert_context *ctx; 6401482b76SMilan Broz struct scatterlist sg_in; 6501482b76SMilan Broz struct scatterlist sg_out; 6601482b76SMilan Broz }; 6701482b76SMilan Broz 681da177e4SLinus Torvalds struct crypt_config; 691da177e4SLinus Torvalds 701da177e4SLinus Torvalds struct crypt_iv_operations { 711da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 721da177e4SLinus Torvalds const char *opts); 731da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 741da177e4SLinus Torvalds const char *(*status)(struct crypt_config *cc); 751da177e4SLinus Torvalds int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector); 761da177e4SLinus Torvalds }; 771da177e4SLinus Torvalds 781da177e4SLinus Torvalds /* 791da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 801da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 811da177e4SLinus Torvalds */ 82e48d4bbfSMilan Broz enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 831da177e4SLinus Torvalds struct crypt_config { 841da177e4SLinus Torvalds struct dm_dev *dev; 851da177e4SLinus Torvalds sector_t start; 861da177e4SLinus Torvalds 871da177e4SLinus Torvalds /* 88ddd42edfSMilan Broz * pool for per bio private data, crypto requests and 89ddd42edfSMilan Broz * encryption requeusts/buffer pages 901da177e4SLinus Torvalds */ 911da177e4SLinus Torvalds mempool_t *io_pool; 92ddd42edfSMilan Broz mempool_t *req_pool; 931da177e4SLinus Torvalds mempool_t *page_pool; 946a24c718SMilan Broz struct bio_set *bs; 951da177e4SLinus Torvalds 96cabf08e4SMilan Broz struct workqueue_struct *io_queue; 97cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 983f1e9070SMilan Broz 991da177e4SLinus Torvalds /* 1001da177e4SLinus Torvalds * crypto related data 1011da177e4SLinus Torvalds */ 1021da177e4SLinus Torvalds struct crypt_iv_operations *iv_gen_ops; 1031da177e4SLinus Torvalds char *iv_mode; 10479066ad3SHerbert Xu union { 10579066ad3SHerbert Xu struct crypto_cipher *essiv_tfm; 10679066ad3SHerbert Xu int benbi_shift; 10779066ad3SHerbert Xu } iv_gen_private; 1081da177e4SLinus Torvalds sector_t iv_offset; 1091da177e4SLinus Torvalds unsigned int iv_size; 1101da177e4SLinus Torvalds 111ddd42edfSMilan Broz /* 112ddd42edfSMilan Broz * Layout of each crypto request: 113ddd42edfSMilan Broz * 114ddd42edfSMilan Broz * struct ablkcipher_request 115ddd42edfSMilan Broz * context 116ddd42edfSMilan Broz * padding 117ddd42edfSMilan Broz * struct dm_crypt_request 118ddd42edfSMilan Broz * padding 119ddd42edfSMilan Broz * IV 120ddd42edfSMilan Broz * 121ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 122ddd42edfSMilan Broz * correctly aligned. 123ddd42edfSMilan Broz */ 124ddd42edfSMilan Broz unsigned int dmreq_start; 125ddd42edfSMilan Broz struct ablkcipher_request *req; 126ddd42edfSMilan Broz 127d1806f6aSHerbert Xu char cipher[CRYPTO_MAX_ALG_NAME]; 128d1806f6aSHerbert Xu char chainmode[CRYPTO_MAX_ALG_NAME]; 1293a7f6c99SMilan Broz struct crypto_ablkcipher *tfm; 130e48d4bbfSMilan Broz unsigned long flags; 1311da177e4SLinus Torvalds unsigned int key_size; 1321da177e4SLinus Torvalds u8 key[0]; 1331da177e4SLinus Torvalds }; 1341da177e4SLinus Torvalds 1356a24c718SMilan Broz #define MIN_IOS 16 1361da177e4SLinus Torvalds #define MIN_POOL_PAGES 32 1371da177e4SLinus Torvalds #define MIN_BIO_PAGES 8 1381da177e4SLinus Torvalds 139e18b890bSChristoph Lameter static struct kmem_cache *_crypt_io_pool; 1401da177e4SLinus Torvalds 141028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 142395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 143027581f3SOlaf Kirch 1441da177e4SLinus Torvalds /* 1451da177e4SLinus Torvalds * Different IV generation algorithms: 1461da177e4SLinus Torvalds * 1473c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 1483a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 1491da177e4SLinus Torvalds * 1503c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 1511da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 1521da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 1531da177e4SLinus Torvalds * 15448527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 15548527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 15648527fa7SRik Snel * 15746b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 15846b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 15946b47730SLudwig Nussel * 1601da177e4SLinus Torvalds * plumb: unimplemented, see: 1611da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 1621da177e4SLinus Torvalds */ 1631da177e4SLinus Torvalds 1641da177e4SLinus Torvalds static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 1651da177e4SLinus Torvalds { 1661da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 1671da177e4SLinus Torvalds *(u32 *)iv = cpu_to_le32(sector & 0xffffffff); 1681da177e4SLinus Torvalds 1691da177e4SLinus Torvalds return 0; 1701da177e4SLinus Torvalds } 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 1731da177e4SLinus Torvalds const char *opts) 1741da177e4SLinus Torvalds { 175d1806f6aSHerbert Xu struct crypto_cipher *essiv_tfm; 17635058687SHerbert Xu struct crypto_hash *hash_tfm; 17735058687SHerbert Xu struct hash_desc desc; 1781da177e4SLinus Torvalds struct scatterlist sg; 1791da177e4SLinus Torvalds unsigned int saltsize; 1801da177e4SLinus Torvalds u8 *salt; 181d1806f6aSHerbert Xu int err; 1821da177e4SLinus Torvalds 1831da177e4SLinus Torvalds if (opts == NULL) { 18472d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 1851da177e4SLinus Torvalds return -EINVAL; 1861da177e4SLinus Torvalds } 1871da177e4SLinus Torvalds 1881da177e4SLinus Torvalds /* Hash the cipher key with the given hash algorithm */ 18935058687SHerbert Xu hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 19035058687SHerbert Xu if (IS_ERR(hash_tfm)) { 19172d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 19235058687SHerbert Xu return PTR_ERR(hash_tfm); 1931da177e4SLinus Torvalds } 1941da177e4SLinus Torvalds 19535058687SHerbert Xu saltsize = crypto_hash_digestsize(hash_tfm); 1961da177e4SLinus Torvalds salt = kmalloc(saltsize, GFP_KERNEL); 1971da177e4SLinus Torvalds if (salt == NULL) { 19872d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 19935058687SHerbert Xu crypto_free_hash(hash_tfm); 2001da177e4SLinus Torvalds return -ENOMEM; 2011da177e4SLinus Torvalds } 2021da177e4SLinus Torvalds 20368e3f5ddSHerbert Xu sg_init_one(&sg, cc->key, cc->key_size); 20435058687SHerbert Xu desc.tfm = hash_tfm; 20535058687SHerbert Xu desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 20635058687SHerbert Xu err = crypto_hash_digest(&desc, &sg, cc->key_size, salt); 20735058687SHerbert Xu crypto_free_hash(hash_tfm); 20835058687SHerbert Xu 20935058687SHerbert Xu if (err) { 21035058687SHerbert Xu ti->error = "Error calculating hash in ESSIV"; 211815f9e32SDmitry Monakhov kfree(salt); 21235058687SHerbert Xu return err; 21335058687SHerbert Xu } 2141da177e4SLinus Torvalds 2151da177e4SLinus Torvalds /* Setup the essiv_tfm with the given salt */ 216d1806f6aSHerbert Xu essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 217d1806f6aSHerbert Xu if (IS_ERR(essiv_tfm)) { 21872d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm for ESSIV"; 2191da177e4SLinus Torvalds kfree(salt); 220d1806f6aSHerbert Xu return PTR_ERR(essiv_tfm); 2211da177e4SLinus Torvalds } 222d1806f6aSHerbert Xu if (crypto_cipher_blocksize(essiv_tfm) != 2233a7f6c99SMilan Broz crypto_ablkcipher_ivsize(cc->tfm)) { 22472d94861SAlasdair G Kergon ti->error = "Block size of ESSIV cipher does " 2251da177e4SLinus Torvalds "not match IV size of block cipher"; 226d1806f6aSHerbert Xu crypto_free_cipher(essiv_tfm); 2271da177e4SLinus Torvalds kfree(salt); 2281da177e4SLinus Torvalds return -EINVAL; 2291da177e4SLinus Torvalds } 230d1806f6aSHerbert Xu err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 231d1806f6aSHerbert Xu if (err) { 23272d94861SAlasdair G Kergon ti->error = "Failed to set key for ESSIV cipher"; 233d1806f6aSHerbert Xu crypto_free_cipher(essiv_tfm); 2341da177e4SLinus Torvalds kfree(salt); 235d1806f6aSHerbert Xu return err; 2361da177e4SLinus Torvalds } 2371da177e4SLinus Torvalds kfree(salt); 2381da177e4SLinus Torvalds 23979066ad3SHerbert Xu cc->iv_gen_private.essiv_tfm = essiv_tfm; 2401da177e4SLinus Torvalds return 0; 2411da177e4SLinus Torvalds } 2421da177e4SLinus Torvalds 2431da177e4SLinus Torvalds static void crypt_iv_essiv_dtr(struct crypt_config *cc) 2441da177e4SLinus Torvalds { 24579066ad3SHerbert Xu crypto_free_cipher(cc->iv_gen_private.essiv_tfm); 24679066ad3SHerbert Xu cc->iv_gen_private.essiv_tfm = NULL; 2471da177e4SLinus Torvalds } 2481da177e4SLinus Torvalds 2491da177e4SLinus Torvalds static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 2501da177e4SLinus Torvalds { 2511da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 2521da177e4SLinus Torvalds *(u64 *)iv = cpu_to_le64(sector); 25379066ad3SHerbert Xu crypto_cipher_encrypt_one(cc->iv_gen_private.essiv_tfm, iv, iv); 2541da177e4SLinus Torvalds return 0; 2551da177e4SLinus Torvalds } 2561da177e4SLinus Torvalds 25748527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 25848527fa7SRik Snel const char *opts) 25948527fa7SRik Snel { 2603a7f6c99SMilan Broz unsigned bs = crypto_ablkcipher_blocksize(cc->tfm); 261f0d1b0b3SDavid Howells int log = ilog2(bs); 26248527fa7SRik Snel 26348527fa7SRik Snel /* we need to calculate how far we must shift the sector count 26448527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 26548527fa7SRik Snel 26648527fa7SRik Snel if (1 << log != bs) { 26748527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 26848527fa7SRik Snel return -EINVAL; 26948527fa7SRik Snel } 27048527fa7SRik Snel 27148527fa7SRik Snel if (log > 9) { 27248527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 27348527fa7SRik Snel return -EINVAL; 27448527fa7SRik Snel } 27548527fa7SRik Snel 27679066ad3SHerbert Xu cc->iv_gen_private.benbi_shift = 9 - log; 27748527fa7SRik Snel 27848527fa7SRik Snel return 0; 27948527fa7SRik Snel } 28048527fa7SRik Snel 28148527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 28248527fa7SRik Snel { 28348527fa7SRik Snel } 28448527fa7SRik Snel 28548527fa7SRik Snel static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 28648527fa7SRik Snel { 28779066ad3SHerbert Xu __be64 val; 28879066ad3SHerbert Xu 28948527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 29079066ad3SHerbert Xu 29179066ad3SHerbert Xu val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi_shift) + 1); 29279066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 29348527fa7SRik Snel 2941da177e4SLinus Torvalds return 0; 2951da177e4SLinus Torvalds } 2961da177e4SLinus Torvalds 29746b47730SLudwig Nussel static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 29846b47730SLudwig Nussel { 29946b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 30046b47730SLudwig Nussel 30146b47730SLudwig Nussel return 0; 30246b47730SLudwig Nussel } 30346b47730SLudwig Nussel 3041da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_plain_ops = { 3051da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 3061da177e4SLinus Torvalds }; 3071da177e4SLinus Torvalds 3081da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_essiv_ops = { 3091da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 3101da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 3111da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 3121da177e4SLinus Torvalds }; 3131da177e4SLinus Torvalds 31448527fa7SRik Snel static struct crypt_iv_operations crypt_iv_benbi_ops = { 31548527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 31648527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 31748527fa7SRik Snel .generator = crypt_iv_benbi_gen 31848527fa7SRik Snel }; 3191da177e4SLinus Torvalds 32046b47730SLudwig Nussel static struct crypt_iv_operations crypt_iv_null_ops = { 32146b47730SLudwig Nussel .generator = crypt_iv_null_gen 32246b47730SLudwig Nussel }; 32346b47730SLudwig Nussel 324d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 325d469f841SMilan Broz struct convert_context *ctx, 3261da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 327fcd369daSMilan Broz sector_t sector) 3281da177e4SLinus Torvalds { 3291da177e4SLinus Torvalds ctx->bio_in = bio_in; 3301da177e4SLinus Torvalds ctx->bio_out = bio_out; 3311da177e4SLinus Torvalds ctx->offset_in = 0; 3321da177e4SLinus Torvalds ctx->offset_out = 0; 3331da177e4SLinus Torvalds ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 3341da177e4SLinus Torvalds ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 3351da177e4SLinus Torvalds ctx->sector = sector + cc->iv_offset; 33643d69034SMilan Broz init_completion(&ctx->restart); 3371da177e4SLinus Torvalds } 3381da177e4SLinus Torvalds 339b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 340b2174eebSHuang Ying struct ablkcipher_request *req) 341b2174eebSHuang Ying { 342b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 343b2174eebSHuang Ying } 344b2174eebSHuang Ying 345b2174eebSHuang Ying static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc, 346b2174eebSHuang Ying struct dm_crypt_request *dmreq) 347b2174eebSHuang Ying { 348b2174eebSHuang Ying return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start); 349b2174eebSHuang Ying } 350b2174eebSHuang Ying 35101482b76SMilan Broz static int crypt_convert_block(struct crypt_config *cc, 3523a7f6c99SMilan Broz struct convert_context *ctx, 3533a7f6c99SMilan Broz struct ablkcipher_request *req) 35401482b76SMilan Broz { 35501482b76SMilan Broz struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 35601482b76SMilan Broz struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 3573a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 3583a7f6c99SMilan Broz u8 *iv; 3593a7f6c99SMilan Broz int r = 0; 36001482b76SMilan Broz 361b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 3623a7f6c99SMilan Broz iv = (u8 *)ALIGN((unsigned long)(dmreq + 1), 3633a7f6c99SMilan Broz crypto_ablkcipher_alignmask(cc->tfm) + 1); 3643a7f6c99SMilan Broz 365b2174eebSHuang Ying dmreq->ctx = ctx; 3663a7f6c99SMilan Broz sg_init_table(&dmreq->sg_in, 1); 3673a7f6c99SMilan Broz sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT, 36801482b76SMilan Broz bv_in->bv_offset + ctx->offset_in); 36901482b76SMilan Broz 3703a7f6c99SMilan Broz sg_init_table(&dmreq->sg_out, 1); 3713a7f6c99SMilan Broz sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT, 37201482b76SMilan Broz bv_out->bv_offset + ctx->offset_out); 37301482b76SMilan Broz 37401482b76SMilan Broz ctx->offset_in += 1 << SECTOR_SHIFT; 37501482b76SMilan Broz if (ctx->offset_in >= bv_in->bv_len) { 37601482b76SMilan Broz ctx->offset_in = 0; 37701482b76SMilan Broz ctx->idx_in++; 37801482b76SMilan Broz } 37901482b76SMilan Broz 38001482b76SMilan Broz ctx->offset_out += 1 << SECTOR_SHIFT; 38101482b76SMilan Broz if (ctx->offset_out >= bv_out->bv_len) { 38201482b76SMilan Broz ctx->offset_out = 0; 38301482b76SMilan Broz ctx->idx_out++; 38401482b76SMilan Broz } 38501482b76SMilan Broz 3863a7f6c99SMilan Broz if (cc->iv_gen_ops) { 3873a7f6c99SMilan Broz r = cc->iv_gen_ops->generator(cc, iv, ctx->sector); 3883a7f6c99SMilan Broz if (r < 0) 3893a7f6c99SMilan Broz return r; 3903a7f6c99SMilan Broz } 3913a7f6c99SMilan Broz 3923a7f6c99SMilan Broz ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out, 3933a7f6c99SMilan Broz 1 << SECTOR_SHIFT, iv); 3943a7f6c99SMilan Broz 3953a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 3963a7f6c99SMilan Broz r = crypto_ablkcipher_encrypt(req); 3973a7f6c99SMilan Broz else 3983a7f6c99SMilan Broz r = crypto_ablkcipher_decrypt(req); 3993a7f6c99SMilan Broz 4003a7f6c99SMilan Broz return r; 40101482b76SMilan Broz } 40201482b76SMilan Broz 40395497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 40495497a96SMilan Broz int error); 405ddd42edfSMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 406ddd42edfSMilan Broz struct convert_context *ctx) 407ddd42edfSMilan Broz { 408ddd42edfSMilan Broz if (!cc->req) 409ddd42edfSMilan Broz cc->req = mempool_alloc(cc->req_pool, GFP_NOIO); 41095497a96SMilan Broz ablkcipher_request_set_tfm(cc->req, cc->tfm); 41195497a96SMilan Broz ablkcipher_request_set_callback(cc->req, CRYPTO_TFM_REQ_MAY_BACKLOG | 41295497a96SMilan Broz CRYPTO_TFM_REQ_MAY_SLEEP, 413b2174eebSHuang Ying kcryptd_async_done, 414b2174eebSHuang Ying dmreq_of_req(cc, cc->req)); 415ddd42edfSMilan Broz } 416ddd42edfSMilan Broz 4171da177e4SLinus Torvalds /* 4181da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 4191da177e4SLinus Torvalds */ 4201da177e4SLinus Torvalds static int crypt_convert(struct crypt_config *cc, 4211da177e4SLinus Torvalds struct convert_context *ctx) 4221da177e4SLinus Torvalds { 4233f1e9070SMilan Broz int r; 4241da177e4SLinus Torvalds 425c8081618SMilan Broz atomic_set(&ctx->pending, 1); 426c8081618SMilan Broz 4271da177e4SLinus Torvalds while(ctx->idx_in < ctx->bio_in->bi_vcnt && 4281da177e4SLinus Torvalds ctx->idx_out < ctx->bio_out->bi_vcnt) { 4291da177e4SLinus Torvalds 4303a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 4313a7f6c99SMilan Broz 4323f1e9070SMilan Broz atomic_inc(&ctx->pending); 4333f1e9070SMilan Broz 4343a7f6c99SMilan Broz r = crypt_convert_block(cc, ctx, cc->req); 4353a7f6c99SMilan Broz 4363a7f6c99SMilan Broz switch (r) { 4373f1e9070SMilan Broz /* async */ 4383a7f6c99SMilan Broz case -EBUSY: 4393a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 4403a7f6c99SMilan Broz INIT_COMPLETION(ctx->restart); 4413a7f6c99SMilan Broz /* fall through*/ 4423a7f6c99SMilan Broz case -EINPROGRESS: 4433a7f6c99SMilan Broz cc->req = NULL; 4441da177e4SLinus Torvalds ctx->sector++; 4453a7f6c99SMilan Broz continue; 4463a7f6c99SMilan Broz 4473f1e9070SMilan Broz /* sync */ 4483f1e9070SMilan Broz case 0: 4493f1e9070SMilan Broz atomic_dec(&ctx->pending); 4503f1e9070SMilan Broz ctx->sector++; 451c7f1b204SMilan Broz cond_resched(); 4523f1e9070SMilan Broz continue; 4531da177e4SLinus Torvalds 4543f1e9070SMilan Broz /* error */ 4553f1e9070SMilan Broz default: 4563f1e9070SMilan Broz atomic_dec(&ctx->pending); 4571da177e4SLinus Torvalds return r; 4581da177e4SLinus Torvalds } 4593f1e9070SMilan Broz } 4603f1e9070SMilan Broz 4613f1e9070SMilan Broz return 0; 4623f1e9070SMilan Broz } 4631da177e4SLinus Torvalds 4646a24c718SMilan Broz static void dm_crypt_bio_destructor(struct bio *bio) 4656a24c718SMilan Broz { 466028867acSAlasdair G Kergon struct dm_crypt_io *io = bio->bi_private; 4676a24c718SMilan Broz struct crypt_config *cc = io->target->private; 4686a24c718SMilan Broz 4696a24c718SMilan Broz bio_free(bio, cc->bs); 4706a24c718SMilan Broz } 4716a24c718SMilan Broz 4721da177e4SLinus Torvalds /* 4731da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 4741da177e4SLinus Torvalds * This should never violate the device limitations 475933f01d4SMilan Broz * May return a smaller bio when running out of pages, indicated by 476933f01d4SMilan Broz * *out_of_pages set to 1. 4771da177e4SLinus Torvalds */ 478933f01d4SMilan Broz static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, 479933f01d4SMilan Broz unsigned *out_of_pages) 4801da177e4SLinus Torvalds { 481027581f3SOlaf Kirch struct crypt_config *cc = io->target->private; 4828b004457SMilan Broz struct bio *clone; 4831da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 484b4e3ca1aSAl Viro gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 48591e10625SMilan Broz unsigned i, len; 48691e10625SMilan Broz struct page *page; 4871da177e4SLinus Torvalds 4886a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 4898b004457SMilan Broz if (!clone) 4901da177e4SLinus Torvalds return NULL; 4911da177e4SLinus Torvalds 492027581f3SOlaf Kirch clone_init(io, clone); 493933f01d4SMilan Broz *out_of_pages = 0; 4946a24c718SMilan Broz 495f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 49691e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 497933f01d4SMilan Broz if (!page) { 498933f01d4SMilan Broz *out_of_pages = 1; 4991da177e4SLinus Torvalds break; 500933f01d4SMilan Broz } 5011da177e4SLinus Torvalds 5021da177e4SLinus Torvalds /* 5031da177e4SLinus Torvalds * if additional pages cannot be allocated without waiting, 5041da177e4SLinus Torvalds * return a partially allocated bio, the caller will then try 5051da177e4SLinus Torvalds * to allocate additional bios while submitting this partial bio 5061da177e4SLinus Torvalds */ 507f97380bcSOlaf Kirch if (i == (MIN_BIO_PAGES - 1)) 5081da177e4SLinus Torvalds gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 5091da177e4SLinus Torvalds 51091e10625SMilan Broz len = (size > PAGE_SIZE) ? PAGE_SIZE : size; 5111da177e4SLinus Torvalds 51291e10625SMilan Broz if (!bio_add_page(clone, page, len, 0)) { 51391e10625SMilan Broz mempool_free(page, cc->page_pool); 51491e10625SMilan Broz break; 51591e10625SMilan Broz } 51691e10625SMilan Broz 51791e10625SMilan Broz size -= len; 5181da177e4SLinus Torvalds } 5191da177e4SLinus Torvalds 5208b004457SMilan Broz if (!clone->bi_size) { 5218b004457SMilan Broz bio_put(clone); 5221da177e4SLinus Torvalds return NULL; 5231da177e4SLinus Torvalds } 5241da177e4SLinus Torvalds 5258b004457SMilan Broz return clone; 5261da177e4SLinus Torvalds } 5271da177e4SLinus Torvalds 528644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 5291da177e4SLinus Torvalds { 530644bd2f0SNeil Brown unsigned int i; 5311da177e4SLinus Torvalds struct bio_vec *bv; 5321da177e4SLinus Torvalds 533644bd2f0SNeil Brown for (i = 0; i < clone->bi_vcnt; i++) { 5348b004457SMilan Broz bv = bio_iovec_idx(clone, i); 5351da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 5361da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 5371da177e4SLinus Torvalds bv->bv_page = NULL; 5381da177e4SLinus Torvalds } 5391da177e4SLinus Torvalds } 5401da177e4SLinus Torvalds 541dc440d1eSMilan Broz static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti, 542dc440d1eSMilan Broz struct bio *bio, sector_t sector) 543dc440d1eSMilan Broz { 544dc440d1eSMilan Broz struct crypt_config *cc = ti->private; 545dc440d1eSMilan Broz struct dm_crypt_io *io; 546dc440d1eSMilan Broz 547dc440d1eSMilan Broz io = mempool_alloc(cc->io_pool, GFP_NOIO); 548dc440d1eSMilan Broz io->target = ti; 549dc440d1eSMilan Broz io->base_bio = bio; 550dc440d1eSMilan Broz io->sector = sector; 551dc440d1eSMilan Broz io->error = 0; 552393b47efSMilan Broz io->base_io = NULL; 553dc440d1eSMilan Broz atomic_set(&io->pending, 0); 554dc440d1eSMilan Broz 555dc440d1eSMilan Broz return io; 556dc440d1eSMilan Broz } 557dc440d1eSMilan Broz 5583e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 5593e1a8bddSMilan Broz { 5603e1a8bddSMilan Broz atomic_inc(&io->pending); 5613e1a8bddSMilan Broz } 5623e1a8bddSMilan Broz 5631da177e4SLinus Torvalds /* 5641da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 5651da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 566393b47efSMilan Broz * If base_io is set, wait for the last fragment to complete. 5671da177e4SLinus Torvalds */ 5685742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 5691da177e4SLinus Torvalds { 5705742fd77SMilan Broz struct crypt_config *cc = io->target->private; 5711da177e4SLinus Torvalds 5721da177e4SLinus Torvalds if (!atomic_dec_and_test(&io->pending)) 5731da177e4SLinus Torvalds return; 5741da177e4SLinus Torvalds 575393b47efSMilan Broz if (likely(!io->base_io)) 5766712ecf8SNeilBrown bio_endio(io->base_bio, io->error); 577393b47efSMilan Broz else { 578393b47efSMilan Broz if (io->error && !io->base_io->error) 579393b47efSMilan Broz io->base_io->error = io->error; 580393b47efSMilan Broz crypt_dec_pending(io->base_io); 581393b47efSMilan Broz } 582393b47efSMilan Broz 5831da177e4SLinus Torvalds mempool_free(io, cc->io_pool); 5841da177e4SLinus Torvalds } 5851da177e4SLinus Torvalds 5861da177e4SLinus Torvalds /* 587cabf08e4SMilan Broz * kcryptd/kcryptd_io: 5881da177e4SLinus Torvalds * 5891da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 59023541d2dSMilan Broz * interrupt context. 591cabf08e4SMilan Broz * 592cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 593cabf08e4SMilan Broz * 594cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 595cabf08e4SMilan Broz * 596cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 597cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 598cabf08e4SMilan Broz * to memory allocation. 5991da177e4SLinus Torvalds */ 6006712ecf8SNeilBrown static void crypt_endio(struct bio *clone, int error) 6018b004457SMilan Broz { 602028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 6038b004457SMilan Broz struct crypt_config *cc = io->target->private; 604ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 6058b004457SMilan Broz 606adfe4770SMilan Broz if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error)) 607adfe4770SMilan Broz error = -EIO; 608adfe4770SMilan Broz 6098b004457SMilan Broz /* 6106712ecf8SNeilBrown * free the processed pages 6118b004457SMilan Broz */ 612ee7a491eSMilan Broz if (rw == WRITE) 613644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 6148b004457SMilan Broz 6158b004457SMilan Broz bio_put(clone); 616ee7a491eSMilan Broz 617ee7a491eSMilan Broz if (rw == READ && !error) { 618cabf08e4SMilan Broz kcryptd_queue_crypt(io); 6196712ecf8SNeilBrown return; 620ee7a491eSMilan Broz } 6215742fd77SMilan Broz 6225742fd77SMilan Broz if (unlikely(error)) 6235742fd77SMilan Broz io->error = error; 6245742fd77SMilan Broz 6255742fd77SMilan Broz crypt_dec_pending(io); 6268b004457SMilan Broz } 6278b004457SMilan Broz 628028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 6298b004457SMilan Broz { 6308b004457SMilan Broz struct crypt_config *cc = io->target->private; 6318b004457SMilan Broz 6328b004457SMilan Broz clone->bi_private = io; 6338b004457SMilan Broz clone->bi_end_io = crypt_endio; 6348b004457SMilan Broz clone->bi_bdev = cc->dev->bdev; 6358b004457SMilan Broz clone->bi_rw = io->base_bio->bi_rw; 636027581f3SOlaf Kirch clone->bi_destructor = dm_crypt_bio_destructor; 6378b004457SMilan Broz } 6388b004457SMilan Broz 6394e4eef64SMilan Broz static void kcryptd_io_read(struct dm_crypt_io *io) 6408b004457SMilan Broz { 6418b004457SMilan Broz struct crypt_config *cc = io->target->private; 6428b004457SMilan Broz struct bio *base_bio = io->base_bio; 6438b004457SMilan Broz struct bio *clone; 64493e605c2SMilan Broz 6453e1a8bddSMilan Broz crypt_inc_pending(io); 6468b004457SMilan Broz 6478b004457SMilan Broz /* 6488b004457SMilan Broz * The block layer might modify the bvec array, so always 6498b004457SMilan Broz * copy the required bvecs because we need the original 6508b004457SMilan Broz * one in order to decrypt the whole bio data *afterwards*. 6518b004457SMilan Broz */ 6526a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs); 65393e605c2SMilan Broz if (unlikely(!clone)) { 6545742fd77SMilan Broz io->error = -ENOMEM; 6555742fd77SMilan Broz crypt_dec_pending(io); 65623541d2dSMilan Broz return; 65793e605c2SMilan Broz } 6588b004457SMilan Broz 6598b004457SMilan Broz clone_init(io, clone); 6608b004457SMilan Broz clone->bi_idx = 0; 6618b004457SMilan Broz clone->bi_vcnt = bio_segments(base_bio); 6628b004457SMilan Broz clone->bi_size = base_bio->bi_size; 6630c395b0fSMilan Broz clone->bi_sector = cc->start + io->sector; 6648b004457SMilan Broz memcpy(clone->bi_io_vec, bio_iovec(base_bio), 6658b004457SMilan Broz sizeof(struct bio_vec) * clone->bi_vcnt); 6668b004457SMilan Broz 66793e605c2SMilan Broz generic_make_request(clone); 6688b004457SMilan Broz } 6698b004457SMilan Broz 6704e4eef64SMilan Broz static void kcryptd_io_write(struct dm_crypt_io *io) 6714e4eef64SMilan Broz { 67295497a96SMilan Broz struct bio *clone = io->ctx.bio_out; 67395497a96SMilan Broz generic_make_request(clone); 6744e4eef64SMilan Broz } 6754e4eef64SMilan Broz 676395b167cSAlasdair G Kergon static void kcryptd_io(struct work_struct *work) 677395b167cSAlasdair G Kergon { 678395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 679395b167cSAlasdair G Kergon 680395b167cSAlasdair G Kergon if (bio_data_dir(io->base_bio) == READ) 681395b167cSAlasdair G Kergon kcryptd_io_read(io); 682395b167cSAlasdair G Kergon else 683395b167cSAlasdair G Kergon kcryptd_io_write(io); 684395b167cSAlasdair G Kergon } 685395b167cSAlasdair G Kergon 686395b167cSAlasdair G Kergon static void kcryptd_queue_io(struct dm_crypt_io *io) 687395b167cSAlasdair G Kergon { 688395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 689395b167cSAlasdair G Kergon 690395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_io); 691395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 692395b167cSAlasdair G Kergon } 693395b167cSAlasdair G Kergon 69495497a96SMilan Broz static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, 69595497a96SMilan Broz int error, int async) 6964e4eef64SMilan Broz { 697dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 698dec1cedfSMilan Broz struct crypt_config *cc = io->target->private; 699dec1cedfSMilan Broz 700dec1cedfSMilan Broz if (unlikely(error < 0)) { 701dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 702dec1cedfSMilan Broz bio_put(clone); 703dec1cedfSMilan Broz io->error = -EIO; 7046c031f41SMilan Broz crypt_dec_pending(io); 705dec1cedfSMilan Broz return; 706dec1cedfSMilan Broz } 707dec1cedfSMilan Broz 708dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 709dec1cedfSMilan Broz BUG_ON(io->ctx.idx_out < clone->bi_vcnt); 710dec1cedfSMilan Broz 711dec1cedfSMilan Broz clone->bi_sector = cc->start + io->sector; 712899c95d3SMilan Broz 71395497a96SMilan Broz if (async) 71495497a96SMilan Broz kcryptd_queue_io(io); 7151e37bb8eSAlasdair G Kergon else 716899c95d3SMilan Broz generic_make_request(clone); 7174e4eef64SMilan Broz } 7184e4eef64SMilan Broz 719fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 7208b004457SMilan Broz { 7218b004457SMilan Broz struct crypt_config *cc = io->target->private; 7228b004457SMilan Broz struct bio *clone; 723393b47efSMilan Broz struct dm_crypt_io *new_io; 724c8081618SMilan Broz int crypt_finished; 725933f01d4SMilan Broz unsigned out_of_pages = 0; 726dec1cedfSMilan Broz unsigned remaining = io->base_bio->bi_size; 727b635b00eSMilan Broz sector_t sector = io->sector; 728dec1cedfSMilan Broz int r; 7298b004457SMilan Broz 73093e605c2SMilan Broz /* 731fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 732fc5a5e9aSMilan Broz */ 733fc5a5e9aSMilan Broz crypt_inc_pending(io); 734b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 735fc5a5e9aSMilan Broz 736fc5a5e9aSMilan Broz /* 73793e605c2SMilan Broz * The allocated buffers can be smaller than the whole bio, 73893e605c2SMilan Broz * so repeat the whole process until all the data can be handled. 73993e605c2SMilan Broz */ 74093e605c2SMilan Broz while (remaining) { 741933f01d4SMilan Broz clone = crypt_alloc_buffer(io, remaining, &out_of_pages); 74223541d2dSMilan Broz if (unlikely(!clone)) { 7435742fd77SMilan Broz io->error = -ENOMEM; 744fc5a5e9aSMilan Broz break; 74523541d2dSMilan Broz } 7468b004457SMilan Broz 74753017030SMilan Broz io->ctx.bio_out = clone; 74853017030SMilan Broz io->ctx.idx_out = 0; 7498b004457SMilan Broz 75093e605c2SMilan Broz remaining -= clone->bi_size; 751b635b00eSMilan Broz sector += bio_sectors(clone); 752dec1cedfSMilan Broz 7534e594098SMilan Broz crypt_inc_pending(io); 754dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 755c8081618SMilan Broz crypt_finished = atomic_dec_and_test(&io->ctx.pending); 756dec1cedfSMilan Broz 757c8081618SMilan Broz /* Encryption was already finished, submit io now */ 758c8081618SMilan Broz if (crypt_finished) { 75995497a96SMilan Broz kcryptd_crypt_write_io_submit(io, r, 0); 760c8081618SMilan Broz 761c8081618SMilan Broz /* 762c8081618SMilan Broz * If there was an error, do not try next fragments. 763c8081618SMilan Broz * For async, error is processed in async handler. 764c8081618SMilan Broz */ 7656c031f41SMilan Broz if (unlikely(r < 0)) 766fc5a5e9aSMilan Broz break; 767b635b00eSMilan Broz 768b635b00eSMilan Broz io->sector = sector; 7694e594098SMilan Broz } 77093e605c2SMilan Broz 771933f01d4SMilan Broz /* 772933f01d4SMilan Broz * Out of memory -> run queues 773933f01d4SMilan Broz * But don't wait if split was due to the io size restriction 774933f01d4SMilan Broz */ 775933f01d4SMilan Broz if (unlikely(out_of_pages)) 77698221eb7SOlaf Kirch congestion_wait(WRITE, HZ/100); 777933f01d4SMilan Broz 778393b47efSMilan Broz /* 779393b47efSMilan Broz * With async crypto it is unsafe to share the crypto context 780393b47efSMilan Broz * between fragments, so switch to a new dm_crypt_io structure. 781393b47efSMilan Broz */ 782393b47efSMilan Broz if (unlikely(!crypt_finished && remaining)) { 783393b47efSMilan Broz new_io = crypt_io_alloc(io->target, io->base_bio, 784393b47efSMilan Broz sector); 785393b47efSMilan Broz crypt_inc_pending(new_io); 786393b47efSMilan Broz crypt_convert_init(cc, &new_io->ctx, NULL, 787393b47efSMilan Broz io->base_bio, sector); 788393b47efSMilan Broz new_io->ctx.idx_in = io->ctx.idx_in; 789393b47efSMilan Broz new_io->ctx.offset_in = io->ctx.offset_in; 790393b47efSMilan Broz 791393b47efSMilan Broz /* 792393b47efSMilan Broz * Fragments after the first use the base_io 793393b47efSMilan Broz * pending count. 794393b47efSMilan Broz */ 795393b47efSMilan Broz if (!io->base_io) 796393b47efSMilan Broz new_io->base_io = io; 797393b47efSMilan Broz else { 798393b47efSMilan Broz new_io->base_io = io->base_io; 799393b47efSMilan Broz crypt_inc_pending(io->base_io); 800393b47efSMilan Broz crypt_dec_pending(io); 801393b47efSMilan Broz } 802393b47efSMilan Broz 803393b47efSMilan Broz io = new_io; 804393b47efSMilan Broz } 8058b004457SMilan Broz } 806899c95d3SMilan Broz 807899c95d3SMilan Broz crypt_dec_pending(io); 80884131db6SMilan Broz } 80984131db6SMilan Broz 8104e4eef64SMilan Broz static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error) 8115742fd77SMilan Broz { 8125742fd77SMilan Broz if (unlikely(error < 0)) 8135742fd77SMilan Broz io->error = -EIO; 8145742fd77SMilan Broz 8155742fd77SMilan Broz crypt_dec_pending(io); 8165742fd77SMilan Broz } 8175742fd77SMilan Broz 8184e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 8198b004457SMilan Broz { 8208b004457SMilan Broz struct crypt_config *cc = io->target->private; 8215742fd77SMilan Broz int r = 0; 8228b004457SMilan Broz 8233e1a8bddSMilan Broz crypt_inc_pending(io); 8243a7f6c99SMilan Broz 82553017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 8260c395b0fSMilan Broz io->sector); 8278b004457SMilan Broz 8285742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 8295742fd77SMilan Broz 8303f1e9070SMilan Broz if (atomic_dec_and_test(&io->ctx.pending)) 8314e4eef64SMilan Broz kcryptd_crypt_read_done(io, r); 8323a7f6c99SMilan Broz 8333a7f6c99SMilan Broz crypt_dec_pending(io); 8348b004457SMilan Broz } 8358b004457SMilan Broz 83695497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 83795497a96SMilan Broz int error) 83895497a96SMilan Broz { 839b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 840b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 84195497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 84295497a96SMilan Broz struct crypt_config *cc = io->target->private; 84395497a96SMilan Broz 84495497a96SMilan Broz if (error == -EINPROGRESS) { 84595497a96SMilan Broz complete(&ctx->restart); 84695497a96SMilan Broz return; 84795497a96SMilan Broz } 84895497a96SMilan Broz 849b2174eebSHuang Ying mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool); 85095497a96SMilan Broz 85195497a96SMilan Broz if (!atomic_dec_and_test(&ctx->pending)) 85295497a96SMilan Broz return; 85395497a96SMilan Broz 85495497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 85595497a96SMilan Broz kcryptd_crypt_read_done(io, error); 85695497a96SMilan Broz else 85795497a96SMilan Broz kcryptd_crypt_write_io_submit(io, error, 1); 85895497a96SMilan Broz } 85995497a96SMilan Broz 8604e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 8614e4eef64SMilan Broz { 8624e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 8634e4eef64SMilan Broz 8644e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 8654e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 8664e4eef64SMilan Broz else 8674e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 8688b004457SMilan Broz } 8698b004457SMilan Broz 870395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 871395b167cSAlasdair G Kergon { 872395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 873395b167cSAlasdair G Kergon 874395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 875395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 876395b167cSAlasdair G Kergon } 877395b167cSAlasdair G Kergon 8781da177e4SLinus Torvalds /* 8791da177e4SLinus Torvalds * Decode key from its hex representation 8801da177e4SLinus Torvalds */ 8811da177e4SLinus Torvalds static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 8821da177e4SLinus Torvalds { 8831da177e4SLinus Torvalds char buffer[3]; 8841da177e4SLinus Torvalds char *endp; 8851da177e4SLinus Torvalds unsigned int i; 8861da177e4SLinus Torvalds 8871da177e4SLinus Torvalds buffer[2] = '\0'; 8881da177e4SLinus Torvalds 8891da177e4SLinus Torvalds for (i = 0; i < size; i++) { 8901da177e4SLinus Torvalds buffer[0] = *hex++; 8911da177e4SLinus Torvalds buffer[1] = *hex++; 8921da177e4SLinus Torvalds 8931da177e4SLinus Torvalds key[i] = (u8)simple_strtoul(buffer, &endp, 16); 8941da177e4SLinus Torvalds 8951da177e4SLinus Torvalds if (endp != &buffer[2]) 8961da177e4SLinus Torvalds return -EINVAL; 8971da177e4SLinus Torvalds } 8981da177e4SLinus Torvalds 8991da177e4SLinus Torvalds if (*hex != '\0') 9001da177e4SLinus Torvalds return -EINVAL; 9011da177e4SLinus Torvalds 9021da177e4SLinus Torvalds return 0; 9031da177e4SLinus Torvalds } 9041da177e4SLinus Torvalds 9051da177e4SLinus Torvalds /* 9061da177e4SLinus Torvalds * Encode key into its hex representation 9071da177e4SLinus Torvalds */ 9081da177e4SLinus Torvalds static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 9091da177e4SLinus Torvalds { 9101da177e4SLinus Torvalds unsigned int i; 9111da177e4SLinus Torvalds 9121da177e4SLinus Torvalds for (i = 0; i < size; i++) { 9131da177e4SLinus Torvalds sprintf(hex, "%02x", *key); 9141da177e4SLinus Torvalds hex += 2; 9151da177e4SLinus Torvalds key++; 9161da177e4SLinus Torvalds } 9171da177e4SLinus Torvalds } 9181da177e4SLinus Torvalds 919e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 920e48d4bbfSMilan Broz { 921e48d4bbfSMilan Broz unsigned key_size = strlen(key) >> 1; 922e48d4bbfSMilan Broz 923e48d4bbfSMilan Broz if (cc->key_size && cc->key_size != key_size) 924e48d4bbfSMilan Broz return -EINVAL; 925e48d4bbfSMilan Broz 926e48d4bbfSMilan Broz cc->key_size = key_size; /* initial settings */ 927e48d4bbfSMilan Broz 928e48d4bbfSMilan Broz if ((!key_size && strcmp(key, "-")) || 929e48d4bbfSMilan Broz (key_size && crypt_decode_key(cc->key, key, key_size) < 0)) 930e48d4bbfSMilan Broz return -EINVAL; 931e48d4bbfSMilan Broz 932e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 933e48d4bbfSMilan Broz 934e48d4bbfSMilan Broz return 0; 935e48d4bbfSMilan Broz } 936e48d4bbfSMilan Broz 937e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 938e48d4bbfSMilan Broz { 939e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 940e48d4bbfSMilan Broz memset(&cc->key, 0, cc->key_size * sizeof(u8)); 941e48d4bbfSMilan Broz return 0; 942e48d4bbfSMilan Broz } 943e48d4bbfSMilan Broz 9441da177e4SLinus Torvalds /* 9451da177e4SLinus Torvalds * Construct an encryption mapping: 9461da177e4SLinus Torvalds * <cipher> <key> <iv_offset> <dev_path> <start> 9471da177e4SLinus Torvalds */ 9481da177e4SLinus Torvalds static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 9491da177e4SLinus Torvalds { 9501da177e4SLinus Torvalds struct crypt_config *cc; 9513a7f6c99SMilan Broz struct crypto_ablkcipher *tfm; 9521da177e4SLinus Torvalds char *tmp; 9531da177e4SLinus Torvalds char *cipher; 9541da177e4SLinus Torvalds char *chainmode; 9551da177e4SLinus Torvalds char *ivmode; 9561da177e4SLinus Torvalds char *ivopts; 9571da177e4SLinus Torvalds unsigned int key_size; 9584ee218cdSAndrew Morton unsigned long long tmpll; 9591da177e4SLinus Torvalds 9601da177e4SLinus Torvalds if (argc != 5) { 96172d94861SAlasdair G Kergon ti->error = "Not enough arguments"; 9621da177e4SLinus Torvalds return -EINVAL; 9631da177e4SLinus Torvalds } 9641da177e4SLinus Torvalds 9651da177e4SLinus Torvalds tmp = argv[0]; 9661da177e4SLinus Torvalds cipher = strsep(&tmp, "-"); 9671da177e4SLinus Torvalds chainmode = strsep(&tmp, "-"); 9681da177e4SLinus Torvalds ivopts = strsep(&tmp, "-"); 9691da177e4SLinus Torvalds ivmode = strsep(&ivopts, ":"); 9701da177e4SLinus Torvalds 9711da177e4SLinus Torvalds if (tmp) 97272d94861SAlasdair G Kergon DMWARN("Unexpected additional cipher options"); 9731da177e4SLinus Torvalds 9741da177e4SLinus Torvalds key_size = strlen(argv[1]) >> 1; 9751da177e4SLinus Torvalds 976e48d4bbfSMilan Broz cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 9771da177e4SLinus Torvalds if (cc == NULL) { 9781da177e4SLinus Torvalds ti->error = 97972d94861SAlasdair G Kergon "Cannot allocate transparent encryption context"; 9801da177e4SLinus Torvalds return -ENOMEM; 9811da177e4SLinus Torvalds } 9821da177e4SLinus Torvalds 983e48d4bbfSMilan Broz if (crypt_set_key(cc, argv[1])) { 98472d94861SAlasdair G Kergon ti->error = "Error decoding key"; 985636d5786SMilan Broz goto bad_cipher; 9861da177e4SLinus Torvalds } 9871da177e4SLinus Torvalds 9881da177e4SLinus Torvalds /* Compatiblity mode for old dm-crypt cipher strings */ 9891da177e4SLinus Torvalds if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) { 9901da177e4SLinus Torvalds chainmode = "cbc"; 9911da177e4SLinus Torvalds ivmode = "plain"; 9921da177e4SLinus Torvalds } 9931da177e4SLinus Torvalds 994d1806f6aSHerbert Xu if (strcmp(chainmode, "ecb") && !ivmode) { 99572d94861SAlasdair G Kergon ti->error = "This chaining mode requires an IV mechanism"; 996636d5786SMilan Broz goto bad_cipher; 9971da177e4SLinus Torvalds } 9981da177e4SLinus Torvalds 999d469f841SMilan Broz if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)", 1000d469f841SMilan Broz chainmode, cipher) >= CRYPTO_MAX_ALG_NAME) { 1001d1806f6aSHerbert Xu ti->error = "Chain mode + cipher name is too long"; 1002636d5786SMilan Broz goto bad_cipher; 1003d1806f6aSHerbert Xu } 1004d1806f6aSHerbert Xu 10053a7f6c99SMilan Broz tfm = crypto_alloc_ablkcipher(cc->cipher, 0, 0); 1006d1806f6aSHerbert Xu if (IS_ERR(tfm)) { 100772d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm"; 1008636d5786SMilan Broz goto bad_cipher; 10091da177e4SLinus Torvalds } 10101da177e4SLinus Torvalds 1011d1806f6aSHerbert Xu strcpy(cc->cipher, cipher); 1012d1806f6aSHerbert Xu strcpy(cc->chainmode, chainmode); 10131da177e4SLinus Torvalds cc->tfm = tfm; 10141da177e4SLinus Torvalds 10151da177e4SLinus Torvalds /* 101648527fa7SRik Snel * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi". 10171da177e4SLinus Torvalds * See comments at iv code 10181da177e4SLinus Torvalds */ 10191da177e4SLinus Torvalds 10201da177e4SLinus Torvalds if (ivmode == NULL) 10211da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 10221da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 10231da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 10241da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 10251da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 102648527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 102748527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 102846b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 102946b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 10301da177e4SLinus Torvalds else { 103172d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 1032636d5786SMilan Broz goto bad_ivmode; 10331da177e4SLinus Torvalds } 10341da177e4SLinus Torvalds 10351da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->ctr && 10361da177e4SLinus Torvalds cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0) 1037636d5786SMilan Broz goto bad_ivmode; 10381da177e4SLinus Torvalds 10393a7f6c99SMilan Broz cc->iv_size = crypto_ablkcipher_ivsize(tfm); 1040d1806f6aSHerbert Xu if (cc->iv_size) 10411da177e4SLinus Torvalds /* at least a 64 bit sector number should fit in our buffer */ 1042d1806f6aSHerbert Xu cc->iv_size = max(cc->iv_size, 10431da177e4SLinus Torvalds (unsigned int)(sizeof(u64) / sizeof(u8))); 10441da177e4SLinus Torvalds else { 10451da177e4SLinus Torvalds if (cc->iv_gen_ops) { 104672d94861SAlasdair G Kergon DMWARN("Selected cipher does not support IVs"); 10471da177e4SLinus Torvalds if (cc->iv_gen_ops->dtr) 10481da177e4SLinus Torvalds cc->iv_gen_ops->dtr(cc); 10491da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 10501da177e4SLinus Torvalds } 10511da177e4SLinus Torvalds } 10521da177e4SLinus Torvalds 105393d2341cSMatthew Dobson cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 10541da177e4SLinus Torvalds if (!cc->io_pool) { 105572d94861SAlasdair G Kergon ti->error = "Cannot allocate crypt io mempool"; 1056636d5786SMilan Broz goto bad_slab_pool; 10571da177e4SLinus Torvalds } 10581da177e4SLinus Torvalds 1059ddd42edfSMilan Broz cc->dmreq_start = sizeof(struct ablkcipher_request); 10603a7f6c99SMilan Broz cc->dmreq_start += crypto_ablkcipher_reqsize(tfm); 1061ddd42edfSMilan Broz cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment()); 10623a7f6c99SMilan Broz cc->dmreq_start += crypto_ablkcipher_alignmask(tfm) & 10633a7f6c99SMilan Broz ~(crypto_tfm_ctx_alignment() - 1); 1064ddd42edfSMilan Broz 1065ddd42edfSMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1066ddd42edfSMilan Broz sizeof(struct dm_crypt_request) + cc->iv_size); 1067ddd42edfSMilan Broz if (!cc->req_pool) { 1068ddd42edfSMilan Broz ti->error = "Cannot allocate crypt request mempool"; 1069ddd42edfSMilan Broz goto bad_req_pool; 1070ddd42edfSMilan Broz } 1071ddd42edfSMilan Broz cc->req = NULL; 1072ddd42edfSMilan Broz 1073a19b27ceSMatthew Dobson cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 10741da177e4SLinus Torvalds if (!cc->page_pool) { 107572d94861SAlasdair G Kergon ti->error = "Cannot allocate page mempool"; 1076636d5786SMilan Broz goto bad_page_pool; 10771da177e4SLinus Torvalds } 10781da177e4SLinus Torvalds 1079bb799ca0SJens Axboe cc->bs = bioset_create(MIN_IOS, 0); 10806a24c718SMilan Broz if (!cc->bs) { 10816a24c718SMilan Broz ti->error = "Cannot allocate crypt bioset"; 10826a24c718SMilan Broz goto bad_bs; 10836a24c718SMilan Broz } 10846a24c718SMilan Broz 10853a7f6c99SMilan Broz if (crypto_ablkcipher_setkey(tfm, cc->key, key_size) < 0) { 108672d94861SAlasdair G Kergon ti->error = "Error setting key"; 1087636d5786SMilan Broz goto bad_device; 10881da177e4SLinus Torvalds } 10891da177e4SLinus Torvalds 10904ee218cdSAndrew Morton if (sscanf(argv[2], "%llu", &tmpll) != 1) { 109172d94861SAlasdair G Kergon ti->error = "Invalid iv_offset sector"; 1092636d5786SMilan Broz goto bad_device; 10931da177e4SLinus Torvalds } 10944ee218cdSAndrew Morton cc->iv_offset = tmpll; 10951da177e4SLinus Torvalds 10964ee218cdSAndrew Morton if (sscanf(argv[4], "%llu", &tmpll) != 1) { 109772d94861SAlasdair G Kergon ti->error = "Invalid device sector"; 1098636d5786SMilan Broz goto bad_device; 10991da177e4SLinus Torvalds } 11004ee218cdSAndrew Morton cc->start = tmpll; 11011da177e4SLinus Torvalds 11021da177e4SLinus Torvalds if (dm_get_device(ti, argv[3], cc->start, ti->len, 11031da177e4SLinus Torvalds dm_table_get_mode(ti->table), &cc->dev)) { 110472d94861SAlasdair G Kergon ti->error = "Device lookup failed"; 1105636d5786SMilan Broz goto bad_device; 11061da177e4SLinus Torvalds } 11071da177e4SLinus Torvalds 11081da177e4SLinus Torvalds if (ivmode && cc->iv_gen_ops) { 11091da177e4SLinus Torvalds if (ivopts) 11101da177e4SLinus Torvalds *(ivopts - 1) = ':'; 11111da177e4SLinus Torvalds cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL); 11121da177e4SLinus Torvalds if (!cc->iv_mode) { 111372d94861SAlasdair G Kergon ti->error = "Error kmallocing iv_mode string"; 1114636d5786SMilan Broz goto bad_ivmode_string; 11151da177e4SLinus Torvalds } 11161da177e4SLinus Torvalds strcpy(cc->iv_mode, ivmode); 11171da177e4SLinus Torvalds } else 11181da177e4SLinus Torvalds cc->iv_mode = NULL; 11191da177e4SLinus Torvalds 1120cabf08e4SMilan Broz cc->io_queue = create_singlethread_workqueue("kcryptd_io"); 1121cabf08e4SMilan Broz if (!cc->io_queue) { 1122cabf08e4SMilan Broz ti->error = "Couldn't create kcryptd io queue"; 1123cabf08e4SMilan Broz goto bad_io_queue; 1124cabf08e4SMilan Broz } 1125cabf08e4SMilan Broz 1126cabf08e4SMilan Broz cc->crypt_queue = create_singlethread_workqueue("kcryptd"); 1127cabf08e4SMilan Broz if (!cc->crypt_queue) { 11289934a8beSMilan Broz ti->error = "Couldn't create kcryptd queue"; 1129cabf08e4SMilan Broz goto bad_crypt_queue; 11309934a8beSMilan Broz } 11319934a8beSMilan Broz 11321da177e4SLinus Torvalds ti->private = cc; 11331da177e4SLinus Torvalds return 0; 11341da177e4SLinus Torvalds 1135cabf08e4SMilan Broz bad_crypt_queue: 1136cabf08e4SMilan Broz destroy_workqueue(cc->io_queue); 1137cabf08e4SMilan Broz bad_io_queue: 11389934a8beSMilan Broz kfree(cc->iv_mode); 1139636d5786SMilan Broz bad_ivmode_string: 114055b42c5aSDmitry Monakhov dm_put_device(ti, cc->dev); 1141636d5786SMilan Broz bad_device: 11426a24c718SMilan Broz bioset_free(cc->bs); 11436a24c718SMilan Broz bad_bs: 11441da177e4SLinus Torvalds mempool_destroy(cc->page_pool); 1145636d5786SMilan Broz bad_page_pool: 1146ddd42edfSMilan Broz mempool_destroy(cc->req_pool); 1147ddd42edfSMilan Broz bad_req_pool: 11481da177e4SLinus Torvalds mempool_destroy(cc->io_pool); 1149636d5786SMilan Broz bad_slab_pool: 11501da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 11511da177e4SLinus Torvalds cc->iv_gen_ops->dtr(cc); 1152636d5786SMilan Broz bad_ivmode: 11533a7f6c99SMilan Broz crypto_free_ablkcipher(tfm); 1154636d5786SMilan Broz bad_cipher: 11559d3520a3SStefan Rompf /* Must zero key material before freeing */ 11569d3520a3SStefan Rompf memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8)); 11571da177e4SLinus Torvalds kfree(cc); 11581da177e4SLinus Torvalds return -EINVAL; 11591da177e4SLinus Torvalds } 11601da177e4SLinus Torvalds 11611da177e4SLinus Torvalds static void crypt_dtr(struct dm_target *ti) 11621da177e4SLinus Torvalds { 11631da177e4SLinus Torvalds struct crypt_config *cc = (struct crypt_config *) ti->private; 11641da177e4SLinus Torvalds 1165cabf08e4SMilan Broz destroy_workqueue(cc->io_queue); 1166cabf08e4SMilan Broz destroy_workqueue(cc->crypt_queue); 116780b16c19SMilan Broz 1168ddd42edfSMilan Broz if (cc->req) 1169ddd42edfSMilan Broz mempool_free(cc->req, cc->req_pool); 1170ddd42edfSMilan Broz 11716a24c718SMilan Broz bioset_free(cc->bs); 11721da177e4SLinus Torvalds mempool_destroy(cc->page_pool); 1173ddd42edfSMilan Broz mempool_destroy(cc->req_pool); 11741da177e4SLinus Torvalds mempool_destroy(cc->io_pool); 11751da177e4SLinus Torvalds 11761da177e4SLinus Torvalds kfree(cc->iv_mode); 11771da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 11781da177e4SLinus Torvalds cc->iv_gen_ops->dtr(cc); 11793a7f6c99SMilan Broz crypto_free_ablkcipher(cc->tfm); 11801da177e4SLinus Torvalds dm_put_device(ti, cc->dev); 11819d3520a3SStefan Rompf 11829d3520a3SStefan Rompf /* Must zero key material before freeing */ 11839d3520a3SStefan Rompf memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8)); 11841da177e4SLinus Torvalds kfree(cc); 11851da177e4SLinus Torvalds } 11861da177e4SLinus Torvalds 11871da177e4SLinus Torvalds static int crypt_map(struct dm_target *ti, struct bio *bio, 11881da177e4SLinus Torvalds union map_info *map_context) 11891da177e4SLinus Torvalds { 1190028867acSAlasdair G Kergon struct dm_crypt_io *io; 11911da177e4SLinus Torvalds 1192dc440d1eSMilan Broz io = crypt_io_alloc(ti, bio, bio->bi_sector - ti->begin); 1193cabf08e4SMilan Broz 1194cabf08e4SMilan Broz if (bio_data_dir(io->base_bio) == READ) 119523541d2dSMilan Broz kcryptd_queue_io(io); 1196cabf08e4SMilan Broz else 1197cabf08e4SMilan Broz kcryptd_queue_crypt(io); 11981da177e4SLinus Torvalds 1199d2a7ad29SKiyoshi Ueda return DM_MAPIO_SUBMITTED; 12001da177e4SLinus Torvalds } 12011da177e4SLinus Torvalds 12021da177e4SLinus Torvalds static int crypt_status(struct dm_target *ti, status_type_t type, 12031da177e4SLinus Torvalds char *result, unsigned int maxlen) 12041da177e4SLinus Torvalds { 12051da177e4SLinus Torvalds struct crypt_config *cc = (struct crypt_config *) ti->private; 12061da177e4SLinus Torvalds unsigned int sz = 0; 12071da177e4SLinus Torvalds 12081da177e4SLinus Torvalds switch (type) { 12091da177e4SLinus Torvalds case STATUSTYPE_INFO: 12101da177e4SLinus Torvalds result[0] = '\0'; 12111da177e4SLinus Torvalds break; 12121da177e4SLinus Torvalds 12131da177e4SLinus Torvalds case STATUSTYPE_TABLE: 12141da177e4SLinus Torvalds if (cc->iv_mode) 121537af6560SChristophe Saout DMEMIT("%s-%s-%s ", cc->cipher, cc->chainmode, 121637af6560SChristophe Saout cc->iv_mode); 12171da177e4SLinus Torvalds else 121837af6560SChristophe Saout DMEMIT("%s-%s ", cc->cipher, cc->chainmode); 12191da177e4SLinus Torvalds 12201da177e4SLinus Torvalds if (cc->key_size > 0) { 12211da177e4SLinus Torvalds if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 12221da177e4SLinus Torvalds return -ENOMEM; 12231da177e4SLinus Torvalds 12241da177e4SLinus Torvalds crypt_encode_key(result + sz, cc->key, cc->key_size); 12251da177e4SLinus Torvalds sz += cc->key_size << 1; 12261da177e4SLinus Torvalds } else { 12271da177e4SLinus Torvalds if (sz >= maxlen) 12281da177e4SLinus Torvalds return -ENOMEM; 12291da177e4SLinus Torvalds result[sz++] = '-'; 12301da177e4SLinus Torvalds } 12311da177e4SLinus Torvalds 12324ee218cdSAndrew Morton DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 12334ee218cdSAndrew Morton cc->dev->name, (unsigned long long)cc->start); 12341da177e4SLinus Torvalds break; 12351da177e4SLinus Torvalds } 12361da177e4SLinus Torvalds return 0; 12371da177e4SLinus Torvalds } 12381da177e4SLinus Torvalds 1239e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 1240e48d4bbfSMilan Broz { 1241e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1242e48d4bbfSMilan Broz 1243e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1244e48d4bbfSMilan Broz } 1245e48d4bbfSMilan Broz 1246e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 1247e48d4bbfSMilan Broz { 1248e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1249e48d4bbfSMilan Broz 1250e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1251e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 1252e48d4bbfSMilan Broz return -EAGAIN; 1253e48d4bbfSMilan Broz } 1254e48d4bbfSMilan Broz 1255e48d4bbfSMilan Broz return 0; 1256e48d4bbfSMilan Broz } 1257e48d4bbfSMilan Broz 1258e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 1259e48d4bbfSMilan Broz { 1260e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1261e48d4bbfSMilan Broz 1262e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1263e48d4bbfSMilan Broz } 1264e48d4bbfSMilan Broz 1265e48d4bbfSMilan Broz /* Message interface 1266e48d4bbfSMilan Broz * key set <key> 1267e48d4bbfSMilan Broz * key wipe 1268e48d4bbfSMilan Broz */ 1269e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1270e48d4bbfSMilan Broz { 1271e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1272e48d4bbfSMilan Broz 1273e48d4bbfSMilan Broz if (argc < 2) 1274e48d4bbfSMilan Broz goto error; 1275e48d4bbfSMilan Broz 1276e48d4bbfSMilan Broz if (!strnicmp(argv[0], MESG_STR("key"))) { 1277e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1278e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 1279e48d4bbfSMilan Broz return -EINVAL; 1280e48d4bbfSMilan Broz } 1281e48d4bbfSMilan Broz if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) 1282e48d4bbfSMilan Broz return crypt_set_key(cc, argv[2]); 1283e48d4bbfSMilan Broz if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) 1284e48d4bbfSMilan Broz return crypt_wipe_key(cc); 1285e48d4bbfSMilan Broz } 1286e48d4bbfSMilan Broz 1287e48d4bbfSMilan Broz error: 1288e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 1289e48d4bbfSMilan Broz return -EINVAL; 1290e48d4bbfSMilan Broz } 1291e48d4bbfSMilan Broz 1292d41e26b9SMilan Broz static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 1293d41e26b9SMilan Broz struct bio_vec *biovec, int max_size) 1294d41e26b9SMilan Broz { 1295d41e26b9SMilan Broz struct crypt_config *cc = ti->private; 1296d41e26b9SMilan Broz struct request_queue *q = bdev_get_queue(cc->dev->bdev); 1297d41e26b9SMilan Broz 1298d41e26b9SMilan Broz if (!q->merge_bvec_fn) 1299d41e26b9SMilan Broz return max_size; 1300d41e26b9SMilan Broz 1301d41e26b9SMilan Broz bvm->bi_bdev = cc->dev->bdev; 1302d41e26b9SMilan Broz bvm->bi_sector = cc->start + bvm->bi_sector - ti->begin; 1303d41e26b9SMilan Broz 1304d41e26b9SMilan Broz return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 1305d41e26b9SMilan Broz } 1306d41e26b9SMilan Broz 13071da177e4SLinus Torvalds static struct target_type crypt_target = { 13081da177e4SLinus Torvalds .name = "crypt", 1309d41e26b9SMilan Broz .version= {1, 6, 0}, 13101da177e4SLinus Torvalds .module = THIS_MODULE, 13111da177e4SLinus Torvalds .ctr = crypt_ctr, 13121da177e4SLinus Torvalds .dtr = crypt_dtr, 13131da177e4SLinus Torvalds .map = crypt_map, 13141da177e4SLinus Torvalds .status = crypt_status, 1315e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 1316e48d4bbfSMilan Broz .preresume = crypt_preresume, 1317e48d4bbfSMilan Broz .resume = crypt_resume, 1318e48d4bbfSMilan Broz .message = crypt_message, 1319d41e26b9SMilan Broz .merge = crypt_merge, 13201da177e4SLinus Torvalds }; 13211da177e4SLinus Torvalds 13221da177e4SLinus Torvalds static int __init dm_crypt_init(void) 13231da177e4SLinus Torvalds { 13241da177e4SLinus Torvalds int r; 13251da177e4SLinus Torvalds 1326028867acSAlasdair G Kergon _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 13271da177e4SLinus Torvalds if (!_crypt_io_pool) 13281da177e4SLinus Torvalds return -ENOMEM; 13291da177e4SLinus Torvalds 13301da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 13311da177e4SLinus Torvalds if (r < 0) { 133272d94861SAlasdair G Kergon DMERR("register failed %d", r); 13339934a8beSMilan Broz kmem_cache_destroy(_crypt_io_pool); 13341da177e4SLinus Torvalds } 13351da177e4SLinus Torvalds 13361da177e4SLinus Torvalds return r; 13371da177e4SLinus Torvalds } 13381da177e4SLinus Torvalds 13391da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 13401da177e4SLinus Torvalds { 134110d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 13421da177e4SLinus Torvalds kmem_cache_destroy(_crypt_io_pool); 13431da177e4SLinus Torvalds } 13441da177e4SLinus Torvalds 13451da177e4SLinus Torvalds module_init(dm_crypt_init); 13461da177e4SLinus Torvalds module_exit(dm_crypt_exit); 13471da177e4SLinus Torvalds 13481da177e4SLinus Torvalds MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 13491da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 13501da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1351