11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 43f1e9070SMilan Broz * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 943d69034SMilan Broz #include <linux/completion.h> 10d1806f6aSHerbert Xu #include <linux/err.h> 111da177e4SLinus Torvalds #include <linux/module.h> 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/kernel.h> 141da177e4SLinus Torvalds #include <linux/bio.h> 151da177e4SLinus Torvalds #include <linux/blkdev.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/crypto.h> 191da177e4SLinus Torvalds #include <linux/workqueue.h> 203fcfab16SAndrew Morton #include <linux/backing-dev.h> 211da177e4SLinus Torvalds #include <asm/atomic.h> 22378f058cSDavid Hardeman #include <linux/scatterlist.h> 231da177e4SLinus Torvalds #include <asm/page.h> 2448527fa7SRik Snel #include <asm/unaligned.h> 251da177e4SLinus Torvalds 26586e80e6SMikulas Patocka #include <linux/device-mapper.h> 271da177e4SLinus Torvalds 2872d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 29e48d4bbfSMilan Broz #define MESG_STR(x) x, sizeof(x) 301da177e4SLinus Torvalds 311da177e4SLinus Torvalds /* 321da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 331da177e4SLinus Torvalds */ 341da177e4SLinus Torvalds struct convert_context { 3543d69034SMilan Broz struct completion restart; 361da177e4SLinus Torvalds struct bio *bio_in; 371da177e4SLinus Torvalds struct bio *bio_out; 381da177e4SLinus Torvalds unsigned int offset_in; 391da177e4SLinus Torvalds unsigned int offset_out; 401da177e4SLinus Torvalds unsigned int idx_in; 411da177e4SLinus Torvalds unsigned int idx_out; 421da177e4SLinus Torvalds sector_t sector; 4343d69034SMilan Broz atomic_t pending; 441da177e4SLinus Torvalds }; 451da177e4SLinus Torvalds 4653017030SMilan Broz /* 4753017030SMilan Broz * per bio private data 4853017030SMilan Broz */ 4953017030SMilan Broz struct dm_crypt_io { 5053017030SMilan Broz struct dm_target *target; 5153017030SMilan Broz struct bio *base_bio; 5253017030SMilan Broz struct work_struct work; 5353017030SMilan Broz 5453017030SMilan Broz struct convert_context ctx; 5553017030SMilan Broz 5653017030SMilan Broz atomic_t pending; 5753017030SMilan Broz int error; 580c395b0fSMilan Broz sector_t sector; 59393b47efSMilan Broz struct dm_crypt_io *base_io; 6053017030SMilan Broz }; 6153017030SMilan Broz 6201482b76SMilan Broz struct dm_crypt_request { 63b2174eebSHuang Ying struct convert_context *ctx; 6401482b76SMilan Broz struct scatterlist sg_in; 6501482b76SMilan Broz struct scatterlist sg_out; 6601482b76SMilan Broz }; 6701482b76SMilan Broz 681da177e4SLinus Torvalds struct crypt_config; 691da177e4SLinus Torvalds 701da177e4SLinus Torvalds struct crypt_iv_operations { 711da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 721da177e4SLinus Torvalds const char *opts); 731da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 741da177e4SLinus Torvalds int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector); 751da177e4SLinus Torvalds }; 761da177e4SLinus Torvalds 7760473592SMilan Broz struct iv_essiv_private { 7860473592SMilan Broz struct crypto_cipher *tfm; 7960473592SMilan Broz }; 8060473592SMilan Broz 8160473592SMilan Broz struct iv_benbi_private { 8260473592SMilan Broz int shift; 8360473592SMilan Broz }; 8460473592SMilan Broz 851da177e4SLinus Torvalds /* 861da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 871da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 881da177e4SLinus Torvalds */ 89e48d4bbfSMilan Broz enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 901da177e4SLinus Torvalds struct crypt_config { 911da177e4SLinus Torvalds struct dm_dev *dev; 921da177e4SLinus Torvalds sector_t start; 931da177e4SLinus Torvalds 941da177e4SLinus Torvalds /* 95ddd42edfSMilan Broz * pool for per bio private data, crypto requests and 96ddd42edfSMilan Broz * encryption requeusts/buffer pages 971da177e4SLinus Torvalds */ 981da177e4SLinus Torvalds mempool_t *io_pool; 99ddd42edfSMilan Broz mempool_t *req_pool; 1001da177e4SLinus Torvalds mempool_t *page_pool; 1016a24c718SMilan Broz struct bio_set *bs; 1021da177e4SLinus Torvalds 103cabf08e4SMilan Broz struct workqueue_struct *io_queue; 104cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1053f1e9070SMilan Broz 1061da177e4SLinus Torvalds /* 1071da177e4SLinus Torvalds * crypto related data 1081da177e4SLinus Torvalds */ 1091da177e4SLinus Torvalds struct crypt_iv_operations *iv_gen_ops; 1101da177e4SLinus Torvalds char *iv_mode; 11179066ad3SHerbert Xu union { 11260473592SMilan Broz struct iv_essiv_private essiv; 11360473592SMilan Broz struct iv_benbi_private benbi; 11479066ad3SHerbert Xu } iv_gen_private; 1151da177e4SLinus Torvalds sector_t iv_offset; 1161da177e4SLinus Torvalds unsigned int iv_size; 1171da177e4SLinus Torvalds 118ddd42edfSMilan Broz /* 119ddd42edfSMilan Broz * Layout of each crypto request: 120ddd42edfSMilan Broz * 121ddd42edfSMilan Broz * struct ablkcipher_request 122ddd42edfSMilan Broz * context 123ddd42edfSMilan Broz * padding 124ddd42edfSMilan Broz * struct dm_crypt_request 125ddd42edfSMilan Broz * padding 126ddd42edfSMilan Broz * IV 127ddd42edfSMilan Broz * 128ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 129ddd42edfSMilan Broz * correctly aligned. 130ddd42edfSMilan Broz */ 131ddd42edfSMilan Broz unsigned int dmreq_start; 132ddd42edfSMilan Broz struct ablkcipher_request *req; 133ddd42edfSMilan Broz 134d1806f6aSHerbert Xu char cipher[CRYPTO_MAX_ALG_NAME]; 135d1806f6aSHerbert Xu char chainmode[CRYPTO_MAX_ALG_NAME]; 1363a7f6c99SMilan Broz struct crypto_ablkcipher *tfm; 137e48d4bbfSMilan Broz unsigned long flags; 1381da177e4SLinus Torvalds unsigned int key_size; 1391da177e4SLinus Torvalds u8 key[0]; 1401da177e4SLinus Torvalds }; 1411da177e4SLinus Torvalds 1426a24c718SMilan Broz #define MIN_IOS 16 1431da177e4SLinus Torvalds #define MIN_POOL_PAGES 32 1441da177e4SLinus Torvalds #define MIN_BIO_PAGES 8 1451da177e4SLinus Torvalds 146e18b890bSChristoph Lameter static struct kmem_cache *_crypt_io_pool; 1471da177e4SLinus Torvalds 148028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 149395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 150027581f3SOlaf Kirch 1511da177e4SLinus Torvalds /* 1521da177e4SLinus Torvalds * Different IV generation algorithms: 1531da177e4SLinus Torvalds * 1543c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 1553a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 1561da177e4SLinus Torvalds * 1573c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 1581da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 1591da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 1601da177e4SLinus Torvalds * 16148527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 16248527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 16348527fa7SRik Snel * 16446b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 16546b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 16646b47730SLudwig Nussel * 1671da177e4SLinus Torvalds * plumb: unimplemented, see: 1681da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 1691da177e4SLinus Torvalds */ 1701da177e4SLinus Torvalds 1711da177e4SLinus Torvalds static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 1721da177e4SLinus Torvalds { 1731da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 1741da177e4SLinus Torvalds *(u32 *)iv = cpu_to_le32(sector & 0xffffffff); 1751da177e4SLinus Torvalds 1761da177e4SLinus Torvalds return 0; 1771da177e4SLinus Torvalds } 1781da177e4SLinus Torvalds 17960473592SMilan Broz static void crypt_iv_essiv_dtr(struct crypt_config *cc) 18060473592SMilan Broz { 18160473592SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 18260473592SMilan Broz 18360473592SMilan Broz crypto_free_cipher(essiv->tfm); 18460473592SMilan Broz essiv->tfm = NULL; 18560473592SMilan Broz } 18660473592SMilan Broz 1871da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 1881da177e4SLinus Torvalds const char *opts) 1891da177e4SLinus Torvalds { 190d1806f6aSHerbert Xu struct crypto_cipher *essiv_tfm; 19135058687SHerbert Xu struct crypto_hash *hash_tfm; 19235058687SHerbert Xu struct hash_desc desc; 1931da177e4SLinus Torvalds struct scatterlist sg; 1941da177e4SLinus Torvalds unsigned int saltsize; 1951da177e4SLinus Torvalds u8 *salt; 196d1806f6aSHerbert Xu int err; 1971da177e4SLinus Torvalds 1981da177e4SLinus Torvalds if (opts == NULL) { 19972d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 2001da177e4SLinus Torvalds return -EINVAL; 2011da177e4SLinus Torvalds } 2021da177e4SLinus Torvalds 2031da177e4SLinus Torvalds /* Hash the cipher key with the given hash algorithm */ 20435058687SHerbert Xu hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 20535058687SHerbert Xu if (IS_ERR(hash_tfm)) { 20672d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 20735058687SHerbert Xu return PTR_ERR(hash_tfm); 2081da177e4SLinus Torvalds } 2091da177e4SLinus Torvalds 21035058687SHerbert Xu saltsize = crypto_hash_digestsize(hash_tfm); 2111da177e4SLinus Torvalds salt = kmalloc(saltsize, GFP_KERNEL); 2121da177e4SLinus Torvalds if (salt == NULL) { 21372d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 21435058687SHerbert Xu crypto_free_hash(hash_tfm); 2151da177e4SLinus Torvalds return -ENOMEM; 2161da177e4SLinus Torvalds } 2171da177e4SLinus Torvalds 21868e3f5ddSHerbert Xu sg_init_one(&sg, cc->key, cc->key_size); 21935058687SHerbert Xu desc.tfm = hash_tfm; 22035058687SHerbert Xu desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 22135058687SHerbert Xu err = crypto_hash_digest(&desc, &sg, cc->key_size, salt); 22235058687SHerbert Xu crypto_free_hash(hash_tfm); 22335058687SHerbert Xu 22435058687SHerbert Xu if (err) { 22535058687SHerbert Xu ti->error = "Error calculating hash in ESSIV"; 226815f9e32SDmitry Monakhov kfree(salt); 22735058687SHerbert Xu return err; 22835058687SHerbert Xu } 2291da177e4SLinus Torvalds 2301da177e4SLinus Torvalds /* Setup the essiv_tfm with the given salt */ 231d1806f6aSHerbert Xu essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 232d1806f6aSHerbert Xu if (IS_ERR(essiv_tfm)) { 23372d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm for ESSIV"; 2341da177e4SLinus Torvalds kfree(salt); 235d1806f6aSHerbert Xu return PTR_ERR(essiv_tfm); 2361da177e4SLinus Torvalds } 237d1806f6aSHerbert Xu if (crypto_cipher_blocksize(essiv_tfm) != 2383a7f6c99SMilan Broz crypto_ablkcipher_ivsize(cc->tfm)) { 23972d94861SAlasdair G Kergon ti->error = "Block size of ESSIV cipher does " 2401da177e4SLinus Torvalds "not match IV size of block cipher"; 241d1806f6aSHerbert Xu crypto_free_cipher(essiv_tfm); 2421da177e4SLinus Torvalds kfree(salt); 2431da177e4SLinus Torvalds return -EINVAL; 2441da177e4SLinus Torvalds } 245d1806f6aSHerbert Xu err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 246d1806f6aSHerbert Xu if (err) { 24772d94861SAlasdair G Kergon ti->error = "Failed to set key for ESSIV cipher"; 248d1806f6aSHerbert Xu crypto_free_cipher(essiv_tfm); 2491da177e4SLinus Torvalds kfree(salt); 250d1806f6aSHerbert Xu return err; 2511da177e4SLinus Torvalds } 2521da177e4SLinus Torvalds kfree(salt); 2531da177e4SLinus Torvalds 25460473592SMilan Broz cc->iv_gen_private.essiv.tfm = essiv_tfm; 2551da177e4SLinus Torvalds return 0; 2561da177e4SLinus Torvalds } 2571da177e4SLinus Torvalds 2581da177e4SLinus Torvalds static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 2591da177e4SLinus Torvalds { 2601da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 2611da177e4SLinus Torvalds *(u64 *)iv = cpu_to_le64(sector); 26260473592SMilan Broz crypto_cipher_encrypt_one(cc->iv_gen_private.essiv.tfm, iv, iv); 2631da177e4SLinus Torvalds return 0; 2641da177e4SLinus Torvalds } 2651da177e4SLinus Torvalds 26648527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 26748527fa7SRik Snel const char *opts) 26848527fa7SRik Snel { 2693a7f6c99SMilan Broz unsigned bs = crypto_ablkcipher_blocksize(cc->tfm); 270f0d1b0b3SDavid Howells int log = ilog2(bs); 27148527fa7SRik Snel 27248527fa7SRik Snel /* we need to calculate how far we must shift the sector count 27348527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 27448527fa7SRik Snel 27548527fa7SRik Snel if (1 << log != bs) { 27648527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 27748527fa7SRik Snel return -EINVAL; 27848527fa7SRik Snel } 27948527fa7SRik Snel 28048527fa7SRik Snel if (log > 9) { 28148527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 28248527fa7SRik Snel return -EINVAL; 28348527fa7SRik Snel } 28448527fa7SRik Snel 28560473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 28648527fa7SRik Snel 28748527fa7SRik Snel return 0; 28848527fa7SRik Snel } 28948527fa7SRik Snel 29048527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 29148527fa7SRik Snel { 29248527fa7SRik Snel } 29348527fa7SRik Snel 29448527fa7SRik Snel static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 29548527fa7SRik Snel { 29679066ad3SHerbert Xu __be64 val; 29779066ad3SHerbert Xu 29848527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 29979066ad3SHerbert Xu 30060473592SMilan Broz val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi.shift) + 1); 30179066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 30248527fa7SRik Snel 3031da177e4SLinus Torvalds return 0; 3041da177e4SLinus Torvalds } 3051da177e4SLinus Torvalds 30646b47730SLudwig Nussel static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 30746b47730SLudwig Nussel { 30846b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 30946b47730SLudwig Nussel 31046b47730SLudwig Nussel return 0; 31146b47730SLudwig Nussel } 31246b47730SLudwig Nussel 3131da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_plain_ops = { 3141da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 3151da177e4SLinus Torvalds }; 3161da177e4SLinus Torvalds 3171da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_essiv_ops = { 3181da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 3191da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 3201da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 3211da177e4SLinus Torvalds }; 3221da177e4SLinus Torvalds 32348527fa7SRik Snel static struct crypt_iv_operations crypt_iv_benbi_ops = { 32448527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 32548527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 32648527fa7SRik Snel .generator = crypt_iv_benbi_gen 32748527fa7SRik Snel }; 3281da177e4SLinus Torvalds 32946b47730SLudwig Nussel static struct crypt_iv_operations crypt_iv_null_ops = { 33046b47730SLudwig Nussel .generator = crypt_iv_null_gen 33146b47730SLudwig Nussel }; 33246b47730SLudwig Nussel 333d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 334d469f841SMilan Broz struct convert_context *ctx, 3351da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 336fcd369daSMilan Broz sector_t sector) 3371da177e4SLinus Torvalds { 3381da177e4SLinus Torvalds ctx->bio_in = bio_in; 3391da177e4SLinus Torvalds ctx->bio_out = bio_out; 3401da177e4SLinus Torvalds ctx->offset_in = 0; 3411da177e4SLinus Torvalds ctx->offset_out = 0; 3421da177e4SLinus Torvalds ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 3431da177e4SLinus Torvalds ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 3441da177e4SLinus Torvalds ctx->sector = sector + cc->iv_offset; 34543d69034SMilan Broz init_completion(&ctx->restart); 3461da177e4SLinus Torvalds } 3471da177e4SLinus Torvalds 348b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 349b2174eebSHuang Ying struct ablkcipher_request *req) 350b2174eebSHuang Ying { 351b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 352b2174eebSHuang Ying } 353b2174eebSHuang Ying 354b2174eebSHuang Ying static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc, 355b2174eebSHuang Ying struct dm_crypt_request *dmreq) 356b2174eebSHuang Ying { 357b2174eebSHuang Ying return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start); 358b2174eebSHuang Ying } 359b2174eebSHuang Ying 36001482b76SMilan Broz static int crypt_convert_block(struct crypt_config *cc, 3613a7f6c99SMilan Broz struct convert_context *ctx, 3623a7f6c99SMilan Broz struct ablkcipher_request *req) 36301482b76SMilan Broz { 36401482b76SMilan Broz struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 36501482b76SMilan Broz struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 3663a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 3673a7f6c99SMilan Broz u8 *iv; 3683a7f6c99SMilan Broz int r = 0; 36901482b76SMilan Broz 370b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 3713a7f6c99SMilan Broz iv = (u8 *)ALIGN((unsigned long)(dmreq + 1), 3723a7f6c99SMilan Broz crypto_ablkcipher_alignmask(cc->tfm) + 1); 3733a7f6c99SMilan Broz 374b2174eebSHuang Ying dmreq->ctx = ctx; 3753a7f6c99SMilan Broz sg_init_table(&dmreq->sg_in, 1); 3763a7f6c99SMilan Broz sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT, 37701482b76SMilan Broz bv_in->bv_offset + ctx->offset_in); 37801482b76SMilan Broz 3793a7f6c99SMilan Broz sg_init_table(&dmreq->sg_out, 1); 3803a7f6c99SMilan Broz sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT, 38101482b76SMilan Broz bv_out->bv_offset + ctx->offset_out); 38201482b76SMilan Broz 38301482b76SMilan Broz ctx->offset_in += 1 << SECTOR_SHIFT; 38401482b76SMilan Broz if (ctx->offset_in >= bv_in->bv_len) { 38501482b76SMilan Broz ctx->offset_in = 0; 38601482b76SMilan Broz ctx->idx_in++; 38701482b76SMilan Broz } 38801482b76SMilan Broz 38901482b76SMilan Broz ctx->offset_out += 1 << SECTOR_SHIFT; 39001482b76SMilan Broz if (ctx->offset_out >= bv_out->bv_len) { 39101482b76SMilan Broz ctx->offset_out = 0; 39201482b76SMilan Broz ctx->idx_out++; 39301482b76SMilan Broz } 39401482b76SMilan Broz 3953a7f6c99SMilan Broz if (cc->iv_gen_ops) { 3963a7f6c99SMilan Broz r = cc->iv_gen_ops->generator(cc, iv, ctx->sector); 3973a7f6c99SMilan Broz if (r < 0) 3983a7f6c99SMilan Broz return r; 3993a7f6c99SMilan Broz } 4003a7f6c99SMilan Broz 4013a7f6c99SMilan Broz ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out, 4023a7f6c99SMilan Broz 1 << SECTOR_SHIFT, iv); 4033a7f6c99SMilan Broz 4043a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 4053a7f6c99SMilan Broz r = crypto_ablkcipher_encrypt(req); 4063a7f6c99SMilan Broz else 4073a7f6c99SMilan Broz r = crypto_ablkcipher_decrypt(req); 4083a7f6c99SMilan Broz 4093a7f6c99SMilan Broz return r; 41001482b76SMilan Broz } 41101482b76SMilan Broz 41295497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 41395497a96SMilan Broz int error); 414ddd42edfSMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 415ddd42edfSMilan Broz struct convert_context *ctx) 416ddd42edfSMilan Broz { 417ddd42edfSMilan Broz if (!cc->req) 418ddd42edfSMilan Broz cc->req = mempool_alloc(cc->req_pool, GFP_NOIO); 41995497a96SMilan Broz ablkcipher_request_set_tfm(cc->req, cc->tfm); 42095497a96SMilan Broz ablkcipher_request_set_callback(cc->req, CRYPTO_TFM_REQ_MAY_BACKLOG | 42195497a96SMilan Broz CRYPTO_TFM_REQ_MAY_SLEEP, 422b2174eebSHuang Ying kcryptd_async_done, 423b2174eebSHuang Ying dmreq_of_req(cc, cc->req)); 424ddd42edfSMilan Broz } 425ddd42edfSMilan Broz 4261da177e4SLinus Torvalds /* 4271da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 4281da177e4SLinus Torvalds */ 4291da177e4SLinus Torvalds static int crypt_convert(struct crypt_config *cc, 4301da177e4SLinus Torvalds struct convert_context *ctx) 4311da177e4SLinus Torvalds { 4323f1e9070SMilan Broz int r; 4331da177e4SLinus Torvalds 434c8081618SMilan Broz atomic_set(&ctx->pending, 1); 435c8081618SMilan Broz 4361da177e4SLinus Torvalds while(ctx->idx_in < ctx->bio_in->bi_vcnt && 4371da177e4SLinus Torvalds ctx->idx_out < ctx->bio_out->bi_vcnt) { 4381da177e4SLinus Torvalds 4393a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 4403a7f6c99SMilan Broz 4413f1e9070SMilan Broz atomic_inc(&ctx->pending); 4423f1e9070SMilan Broz 4433a7f6c99SMilan Broz r = crypt_convert_block(cc, ctx, cc->req); 4443a7f6c99SMilan Broz 4453a7f6c99SMilan Broz switch (r) { 4463f1e9070SMilan Broz /* async */ 4473a7f6c99SMilan Broz case -EBUSY: 4483a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 4493a7f6c99SMilan Broz INIT_COMPLETION(ctx->restart); 4503a7f6c99SMilan Broz /* fall through*/ 4513a7f6c99SMilan Broz case -EINPROGRESS: 4523a7f6c99SMilan Broz cc->req = NULL; 4531da177e4SLinus Torvalds ctx->sector++; 4543a7f6c99SMilan Broz continue; 4553a7f6c99SMilan Broz 4563f1e9070SMilan Broz /* sync */ 4573f1e9070SMilan Broz case 0: 4583f1e9070SMilan Broz atomic_dec(&ctx->pending); 4593f1e9070SMilan Broz ctx->sector++; 460c7f1b204SMilan Broz cond_resched(); 4613f1e9070SMilan Broz continue; 4621da177e4SLinus Torvalds 4633f1e9070SMilan Broz /* error */ 4643f1e9070SMilan Broz default: 4653f1e9070SMilan Broz atomic_dec(&ctx->pending); 4661da177e4SLinus Torvalds return r; 4671da177e4SLinus Torvalds } 4683f1e9070SMilan Broz } 4693f1e9070SMilan Broz 4703f1e9070SMilan Broz return 0; 4713f1e9070SMilan Broz } 4721da177e4SLinus Torvalds 4736a24c718SMilan Broz static void dm_crypt_bio_destructor(struct bio *bio) 4746a24c718SMilan Broz { 475028867acSAlasdair G Kergon struct dm_crypt_io *io = bio->bi_private; 4766a24c718SMilan Broz struct crypt_config *cc = io->target->private; 4776a24c718SMilan Broz 4786a24c718SMilan Broz bio_free(bio, cc->bs); 4796a24c718SMilan Broz } 4806a24c718SMilan Broz 4811da177e4SLinus Torvalds /* 4821da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 4831da177e4SLinus Torvalds * This should never violate the device limitations 484933f01d4SMilan Broz * May return a smaller bio when running out of pages, indicated by 485933f01d4SMilan Broz * *out_of_pages set to 1. 4861da177e4SLinus Torvalds */ 487933f01d4SMilan Broz static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, 488933f01d4SMilan Broz unsigned *out_of_pages) 4891da177e4SLinus Torvalds { 490027581f3SOlaf Kirch struct crypt_config *cc = io->target->private; 4918b004457SMilan Broz struct bio *clone; 4921da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 493b4e3ca1aSAl Viro gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 49491e10625SMilan Broz unsigned i, len; 49591e10625SMilan Broz struct page *page; 4961da177e4SLinus Torvalds 4976a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 4988b004457SMilan Broz if (!clone) 4991da177e4SLinus Torvalds return NULL; 5001da177e4SLinus Torvalds 501027581f3SOlaf Kirch clone_init(io, clone); 502933f01d4SMilan Broz *out_of_pages = 0; 5036a24c718SMilan Broz 504f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 50591e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 506933f01d4SMilan Broz if (!page) { 507933f01d4SMilan Broz *out_of_pages = 1; 5081da177e4SLinus Torvalds break; 509933f01d4SMilan Broz } 5101da177e4SLinus Torvalds 5111da177e4SLinus Torvalds /* 5121da177e4SLinus Torvalds * if additional pages cannot be allocated without waiting, 5131da177e4SLinus Torvalds * return a partially allocated bio, the caller will then try 5141da177e4SLinus Torvalds * to allocate additional bios while submitting this partial bio 5151da177e4SLinus Torvalds */ 516f97380bcSOlaf Kirch if (i == (MIN_BIO_PAGES - 1)) 5171da177e4SLinus Torvalds gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 5181da177e4SLinus Torvalds 51991e10625SMilan Broz len = (size > PAGE_SIZE) ? PAGE_SIZE : size; 5201da177e4SLinus Torvalds 52191e10625SMilan Broz if (!bio_add_page(clone, page, len, 0)) { 52291e10625SMilan Broz mempool_free(page, cc->page_pool); 52391e10625SMilan Broz break; 52491e10625SMilan Broz } 52591e10625SMilan Broz 52691e10625SMilan Broz size -= len; 5271da177e4SLinus Torvalds } 5281da177e4SLinus Torvalds 5298b004457SMilan Broz if (!clone->bi_size) { 5308b004457SMilan Broz bio_put(clone); 5311da177e4SLinus Torvalds return NULL; 5321da177e4SLinus Torvalds } 5331da177e4SLinus Torvalds 5348b004457SMilan Broz return clone; 5351da177e4SLinus Torvalds } 5361da177e4SLinus Torvalds 537644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 5381da177e4SLinus Torvalds { 539644bd2f0SNeil Brown unsigned int i; 5401da177e4SLinus Torvalds struct bio_vec *bv; 5411da177e4SLinus Torvalds 542644bd2f0SNeil Brown for (i = 0; i < clone->bi_vcnt; i++) { 5438b004457SMilan Broz bv = bio_iovec_idx(clone, i); 5441da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 5451da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 5461da177e4SLinus Torvalds bv->bv_page = NULL; 5471da177e4SLinus Torvalds } 5481da177e4SLinus Torvalds } 5491da177e4SLinus Torvalds 550dc440d1eSMilan Broz static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti, 551dc440d1eSMilan Broz struct bio *bio, sector_t sector) 552dc440d1eSMilan Broz { 553dc440d1eSMilan Broz struct crypt_config *cc = ti->private; 554dc440d1eSMilan Broz struct dm_crypt_io *io; 555dc440d1eSMilan Broz 556dc440d1eSMilan Broz io = mempool_alloc(cc->io_pool, GFP_NOIO); 557dc440d1eSMilan Broz io->target = ti; 558dc440d1eSMilan Broz io->base_bio = bio; 559dc440d1eSMilan Broz io->sector = sector; 560dc440d1eSMilan Broz io->error = 0; 561393b47efSMilan Broz io->base_io = NULL; 562dc440d1eSMilan Broz atomic_set(&io->pending, 0); 563dc440d1eSMilan Broz 564dc440d1eSMilan Broz return io; 565dc440d1eSMilan Broz } 566dc440d1eSMilan Broz 5673e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 5683e1a8bddSMilan Broz { 5693e1a8bddSMilan Broz atomic_inc(&io->pending); 5703e1a8bddSMilan Broz } 5713e1a8bddSMilan Broz 5721da177e4SLinus Torvalds /* 5731da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 5741da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 575393b47efSMilan Broz * If base_io is set, wait for the last fragment to complete. 5761da177e4SLinus Torvalds */ 5775742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 5781da177e4SLinus Torvalds { 5795742fd77SMilan Broz struct crypt_config *cc = io->target->private; 580b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 581b35f8caaSMilan Broz struct dm_crypt_io *base_io = io->base_io; 582b35f8caaSMilan Broz int error = io->error; 5831da177e4SLinus Torvalds 5841da177e4SLinus Torvalds if (!atomic_dec_and_test(&io->pending)) 5851da177e4SLinus Torvalds return; 5861da177e4SLinus Torvalds 5871da177e4SLinus Torvalds mempool_free(io, cc->io_pool); 588b35f8caaSMilan Broz 589b35f8caaSMilan Broz if (likely(!base_io)) 590b35f8caaSMilan Broz bio_endio(base_bio, error); 591b35f8caaSMilan Broz else { 592b35f8caaSMilan Broz if (error && !base_io->error) 593b35f8caaSMilan Broz base_io->error = error; 594b35f8caaSMilan Broz crypt_dec_pending(base_io); 595b35f8caaSMilan Broz } 5961da177e4SLinus Torvalds } 5971da177e4SLinus Torvalds 5981da177e4SLinus Torvalds /* 599cabf08e4SMilan Broz * kcryptd/kcryptd_io: 6001da177e4SLinus Torvalds * 6011da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 60223541d2dSMilan Broz * interrupt context. 603cabf08e4SMilan Broz * 604cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 605cabf08e4SMilan Broz * 606cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 607cabf08e4SMilan Broz * 608cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 609cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 610cabf08e4SMilan Broz * to memory allocation. 6111da177e4SLinus Torvalds */ 6126712ecf8SNeilBrown static void crypt_endio(struct bio *clone, int error) 6138b004457SMilan Broz { 614028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 6158b004457SMilan Broz struct crypt_config *cc = io->target->private; 616ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 6178b004457SMilan Broz 618adfe4770SMilan Broz if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error)) 619adfe4770SMilan Broz error = -EIO; 620adfe4770SMilan Broz 6218b004457SMilan Broz /* 6226712ecf8SNeilBrown * free the processed pages 6238b004457SMilan Broz */ 624ee7a491eSMilan Broz if (rw == WRITE) 625644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 6268b004457SMilan Broz 6278b004457SMilan Broz bio_put(clone); 628ee7a491eSMilan Broz 629ee7a491eSMilan Broz if (rw == READ && !error) { 630cabf08e4SMilan Broz kcryptd_queue_crypt(io); 6316712ecf8SNeilBrown return; 632ee7a491eSMilan Broz } 6335742fd77SMilan Broz 6345742fd77SMilan Broz if (unlikely(error)) 6355742fd77SMilan Broz io->error = error; 6365742fd77SMilan Broz 6375742fd77SMilan Broz crypt_dec_pending(io); 6388b004457SMilan Broz } 6398b004457SMilan Broz 640028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 6418b004457SMilan Broz { 6428b004457SMilan Broz struct crypt_config *cc = io->target->private; 6438b004457SMilan Broz 6448b004457SMilan Broz clone->bi_private = io; 6458b004457SMilan Broz clone->bi_end_io = crypt_endio; 6468b004457SMilan Broz clone->bi_bdev = cc->dev->bdev; 6478b004457SMilan Broz clone->bi_rw = io->base_bio->bi_rw; 648027581f3SOlaf Kirch clone->bi_destructor = dm_crypt_bio_destructor; 6498b004457SMilan Broz } 6508b004457SMilan Broz 6514e4eef64SMilan Broz static void kcryptd_io_read(struct dm_crypt_io *io) 6528b004457SMilan Broz { 6538b004457SMilan Broz struct crypt_config *cc = io->target->private; 6548b004457SMilan Broz struct bio *base_bio = io->base_bio; 6558b004457SMilan Broz struct bio *clone; 65693e605c2SMilan Broz 6573e1a8bddSMilan Broz crypt_inc_pending(io); 6588b004457SMilan Broz 6598b004457SMilan Broz /* 6608b004457SMilan Broz * The block layer might modify the bvec array, so always 6618b004457SMilan Broz * copy the required bvecs because we need the original 6628b004457SMilan Broz * one in order to decrypt the whole bio data *afterwards*. 6638b004457SMilan Broz */ 6646a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs); 66593e605c2SMilan Broz if (unlikely(!clone)) { 6665742fd77SMilan Broz io->error = -ENOMEM; 6675742fd77SMilan Broz crypt_dec_pending(io); 66823541d2dSMilan Broz return; 66993e605c2SMilan Broz } 6708b004457SMilan Broz 6718b004457SMilan Broz clone_init(io, clone); 6728b004457SMilan Broz clone->bi_idx = 0; 6738b004457SMilan Broz clone->bi_vcnt = bio_segments(base_bio); 6748b004457SMilan Broz clone->bi_size = base_bio->bi_size; 6750c395b0fSMilan Broz clone->bi_sector = cc->start + io->sector; 6768b004457SMilan Broz memcpy(clone->bi_io_vec, bio_iovec(base_bio), 6778b004457SMilan Broz sizeof(struct bio_vec) * clone->bi_vcnt); 6788b004457SMilan Broz 67993e605c2SMilan Broz generic_make_request(clone); 6808b004457SMilan Broz } 6818b004457SMilan Broz 6824e4eef64SMilan Broz static void kcryptd_io_write(struct dm_crypt_io *io) 6834e4eef64SMilan Broz { 68495497a96SMilan Broz struct bio *clone = io->ctx.bio_out; 68595497a96SMilan Broz generic_make_request(clone); 6864e4eef64SMilan Broz } 6874e4eef64SMilan Broz 688395b167cSAlasdair G Kergon static void kcryptd_io(struct work_struct *work) 689395b167cSAlasdair G Kergon { 690395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 691395b167cSAlasdair G Kergon 692395b167cSAlasdair G Kergon if (bio_data_dir(io->base_bio) == READ) 693395b167cSAlasdair G Kergon kcryptd_io_read(io); 694395b167cSAlasdair G Kergon else 695395b167cSAlasdair G Kergon kcryptd_io_write(io); 696395b167cSAlasdair G Kergon } 697395b167cSAlasdair G Kergon 698395b167cSAlasdair G Kergon static void kcryptd_queue_io(struct dm_crypt_io *io) 699395b167cSAlasdair G Kergon { 700395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 701395b167cSAlasdair G Kergon 702395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_io); 703395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 704395b167cSAlasdair G Kergon } 705395b167cSAlasdair G Kergon 70695497a96SMilan Broz static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, 70795497a96SMilan Broz int error, int async) 7084e4eef64SMilan Broz { 709dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 710dec1cedfSMilan Broz struct crypt_config *cc = io->target->private; 711dec1cedfSMilan Broz 712dec1cedfSMilan Broz if (unlikely(error < 0)) { 713dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 714dec1cedfSMilan Broz bio_put(clone); 715dec1cedfSMilan Broz io->error = -EIO; 7166c031f41SMilan Broz crypt_dec_pending(io); 717dec1cedfSMilan Broz return; 718dec1cedfSMilan Broz } 719dec1cedfSMilan Broz 720dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 721dec1cedfSMilan Broz BUG_ON(io->ctx.idx_out < clone->bi_vcnt); 722dec1cedfSMilan Broz 723dec1cedfSMilan Broz clone->bi_sector = cc->start + io->sector; 724899c95d3SMilan Broz 72595497a96SMilan Broz if (async) 72695497a96SMilan Broz kcryptd_queue_io(io); 7271e37bb8eSAlasdair G Kergon else 728899c95d3SMilan Broz generic_make_request(clone); 7294e4eef64SMilan Broz } 7304e4eef64SMilan Broz 731fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 7328b004457SMilan Broz { 7338b004457SMilan Broz struct crypt_config *cc = io->target->private; 7348b004457SMilan Broz struct bio *clone; 735393b47efSMilan Broz struct dm_crypt_io *new_io; 736c8081618SMilan Broz int crypt_finished; 737933f01d4SMilan Broz unsigned out_of_pages = 0; 738dec1cedfSMilan Broz unsigned remaining = io->base_bio->bi_size; 739b635b00eSMilan Broz sector_t sector = io->sector; 740dec1cedfSMilan Broz int r; 7418b004457SMilan Broz 74293e605c2SMilan Broz /* 743fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 744fc5a5e9aSMilan Broz */ 745fc5a5e9aSMilan Broz crypt_inc_pending(io); 746b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 747fc5a5e9aSMilan Broz 748fc5a5e9aSMilan Broz /* 74993e605c2SMilan Broz * The allocated buffers can be smaller than the whole bio, 75093e605c2SMilan Broz * so repeat the whole process until all the data can be handled. 75193e605c2SMilan Broz */ 75293e605c2SMilan Broz while (remaining) { 753933f01d4SMilan Broz clone = crypt_alloc_buffer(io, remaining, &out_of_pages); 75423541d2dSMilan Broz if (unlikely(!clone)) { 7555742fd77SMilan Broz io->error = -ENOMEM; 756fc5a5e9aSMilan Broz break; 75723541d2dSMilan Broz } 7588b004457SMilan Broz 75953017030SMilan Broz io->ctx.bio_out = clone; 76053017030SMilan Broz io->ctx.idx_out = 0; 7618b004457SMilan Broz 76293e605c2SMilan Broz remaining -= clone->bi_size; 763b635b00eSMilan Broz sector += bio_sectors(clone); 764dec1cedfSMilan Broz 7654e594098SMilan Broz crypt_inc_pending(io); 766dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 767c8081618SMilan Broz crypt_finished = atomic_dec_and_test(&io->ctx.pending); 768dec1cedfSMilan Broz 769c8081618SMilan Broz /* Encryption was already finished, submit io now */ 770c8081618SMilan Broz if (crypt_finished) { 77195497a96SMilan Broz kcryptd_crypt_write_io_submit(io, r, 0); 772c8081618SMilan Broz 773c8081618SMilan Broz /* 774c8081618SMilan Broz * If there was an error, do not try next fragments. 775c8081618SMilan Broz * For async, error is processed in async handler. 776c8081618SMilan Broz */ 7776c031f41SMilan Broz if (unlikely(r < 0)) 778fc5a5e9aSMilan Broz break; 779b635b00eSMilan Broz 780b635b00eSMilan Broz io->sector = sector; 7814e594098SMilan Broz } 78293e605c2SMilan Broz 783933f01d4SMilan Broz /* 784933f01d4SMilan Broz * Out of memory -> run queues 785933f01d4SMilan Broz * But don't wait if split was due to the io size restriction 786933f01d4SMilan Broz */ 787933f01d4SMilan Broz if (unlikely(out_of_pages)) 7888aa7e847SJens Axboe congestion_wait(BLK_RW_ASYNC, HZ/100); 789933f01d4SMilan Broz 790393b47efSMilan Broz /* 791393b47efSMilan Broz * With async crypto it is unsafe to share the crypto context 792393b47efSMilan Broz * between fragments, so switch to a new dm_crypt_io structure. 793393b47efSMilan Broz */ 794393b47efSMilan Broz if (unlikely(!crypt_finished && remaining)) { 795393b47efSMilan Broz new_io = crypt_io_alloc(io->target, io->base_bio, 796393b47efSMilan Broz sector); 797393b47efSMilan Broz crypt_inc_pending(new_io); 798393b47efSMilan Broz crypt_convert_init(cc, &new_io->ctx, NULL, 799393b47efSMilan Broz io->base_bio, sector); 800393b47efSMilan Broz new_io->ctx.idx_in = io->ctx.idx_in; 801393b47efSMilan Broz new_io->ctx.offset_in = io->ctx.offset_in; 802393b47efSMilan Broz 803393b47efSMilan Broz /* 804393b47efSMilan Broz * Fragments after the first use the base_io 805393b47efSMilan Broz * pending count. 806393b47efSMilan Broz */ 807393b47efSMilan Broz if (!io->base_io) 808393b47efSMilan Broz new_io->base_io = io; 809393b47efSMilan Broz else { 810393b47efSMilan Broz new_io->base_io = io->base_io; 811393b47efSMilan Broz crypt_inc_pending(io->base_io); 812393b47efSMilan Broz crypt_dec_pending(io); 813393b47efSMilan Broz } 814393b47efSMilan Broz 815393b47efSMilan Broz io = new_io; 816393b47efSMilan Broz } 8178b004457SMilan Broz } 818899c95d3SMilan Broz 819899c95d3SMilan Broz crypt_dec_pending(io); 82084131db6SMilan Broz } 82184131db6SMilan Broz 8224e4eef64SMilan Broz static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error) 8235742fd77SMilan Broz { 8245742fd77SMilan Broz if (unlikely(error < 0)) 8255742fd77SMilan Broz io->error = -EIO; 8265742fd77SMilan Broz 8275742fd77SMilan Broz crypt_dec_pending(io); 8285742fd77SMilan Broz } 8295742fd77SMilan Broz 8304e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 8318b004457SMilan Broz { 8328b004457SMilan Broz struct crypt_config *cc = io->target->private; 8335742fd77SMilan Broz int r = 0; 8348b004457SMilan Broz 8353e1a8bddSMilan Broz crypt_inc_pending(io); 8363a7f6c99SMilan Broz 83753017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 8380c395b0fSMilan Broz io->sector); 8398b004457SMilan Broz 8405742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 8415742fd77SMilan Broz 8423f1e9070SMilan Broz if (atomic_dec_and_test(&io->ctx.pending)) 8434e4eef64SMilan Broz kcryptd_crypt_read_done(io, r); 8443a7f6c99SMilan Broz 8453a7f6c99SMilan Broz crypt_dec_pending(io); 8468b004457SMilan Broz } 8478b004457SMilan Broz 84895497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 84995497a96SMilan Broz int error) 85095497a96SMilan Broz { 851b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 852b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 85395497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 85495497a96SMilan Broz struct crypt_config *cc = io->target->private; 85595497a96SMilan Broz 85695497a96SMilan Broz if (error == -EINPROGRESS) { 85795497a96SMilan Broz complete(&ctx->restart); 85895497a96SMilan Broz return; 85995497a96SMilan Broz } 86095497a96SMilan Broz 861b2174eebSHuang Ying mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool); 86295497a96SMilan Broz 86395497a96SMilan Broz if (!atomic_dec_and_test(&ctx->pending)) 86495497a96SMilan Broz return; 86595497a96SMilan Broz 86695497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 86795497a96SMilan Broz kcryptd_crypt_read_done(io, error); 86895497a96SMilan Broz else 86995497a96SMilan Broz kcryptd_crypt_write_io_submit(io, error, 1); 87095497a96SMilan Broz } 87195497a96SMilan Broz 8724e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 8734e4eef64SMilan Broz { 8744e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 8754e4eef64SMilan Broz 8764e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 8774e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 8784e4eef64SMilan Broz else 8794e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 8808b004457SMilan Broz } 8818b004457SMilan Broz 882395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 883395b167cSAlasdair G Kergon { 884395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 885395b167cSAlasdair G Kergon 886395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 887395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 888395b167cSAlasdair G Kergon } 889395b167cSAlasdair G Kergon 8901da177e4SLinus Torvalds /* 8911da177e4SLinus Torvalds * Decode key from its hex representation 8921da177e4SLinus Torvalds */ 8931da177e4SLinus Torvalds static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 8941da177e4SLinus Torvalds { 8951da177e4SLinus Torvalds char buffer[3]; 8961da177e4SLinus Torvalds char *endp; 8971da177e4SLinus Torvalds unsigned int i; 8981da177e4SLinus Torvalds 8991da177e4SLinus Torvalds buffer[2] = '\0'; 9001da177e4SLinus Torvalds 9011da177e4SLinus Torvalds for (i = 0; i < size; i++) { 9021da177e4SLinus Torvalds buffer[0] = *hex++; 9031da177e4SLinus Torvalds buffer[1] = *hex++; 9041da177e4SLinus Torvalds 9051da177e4SLinus Torvalds key[i] = (u8)simple_strtoul(buffer, &endp, 16); 9061da177e4SLinus Torvalds 9071da177e4SLinus Torvalds if (endp != &buffer[2]) 9081da177e4SLinus Torvalds return -EINVAL; 9091da177e4SLinus Torvalds } 9101da177e4SLinus Torvalds 9111da177e4SLinus Torvalds if (*hex != '\0') 9121da177e4SLinus Torvalds return -EINVAL; 9131da177e4SLinus Torvalds 9141da177e4SLinus Torvalds return 0; 9151da177e4SLinus Torvalds } 9161da177e4SLinus Torvalds 9171da177e4SLinus Torvalds /* 9181da177e4SLinus Torvalds * Encode key into its hex representation 9191da177e4SLinus Torvalds */ 9201da177e4SLinus Torvalds static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 9211da177e4SLinus Torvalds { 9221da177e4SLinus Torvalds unsigned int i; 9231da177e4SLinus Torvalds 9241da177e4SLinus Torvalds for (i = 0; i < size; i++) { 9251da177e4SLinus Torvalds sprintf(hex, "%02x", *key); 9261da177e4SLinus Torvalds hex += 2; 9271da177e4SLinus Torvalds key++; 9281da177e4SLinus Torvalds } 9291da177e4SLinus Torvalds } 9301da177e4SLinus Torvalds 931e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 932e48d4bbfSMilan Broz { 933e48d4bbfSMilan Broz unsigned key_size = strlen(key) >> 1; 934e48d4bbfSMilan Broz 935e48d4bbfSMilan Broz if (cc->key_size && cc->key_size != key_size) 936e48d4bbfSMilan Broz return -EINVAL; 937e48d4bbfSMilan Broz 938e48d4bbfSMilan Broz cc->key_size = key_size; /* initial settings */ 939e48d4bbfSMilan Broz 940e48d4bbfSMilan Broz if ((!key_size && strcmp(key, "-")) || 941e48d4bbfSMilan Broz (key_size && crypt_decode_key(cc->key, key, key_size) < 0)) 942e48d4bbfSMilan Broz return -EINVAL; 943e48d4bbfSMilan Broz 944e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 945e48d4bbfSMilan Broz 9460b430958SMilan Broz return crypto_ablkcipher_setkey(cc->tfm, cc->key, cc->key_size); 947e48d4bbfSMilan Broz } 948e48d4bbfSMilan Broz 949e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 950e48d4bbfSMilan Broz { 951e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 952e48d4bbfSMilan Broz memset(&cc->key, 0, cc->key_size * sizeof(u8)); 9530b430958SMilan Broz return crypto_ablkcipher_setkey(cc->tfm, cc->key, cc->key_size); 954e48d4bbfSMilan Broz } 955e48d4bbfSMilan Broz 9561da177e4SLinus Torvalds /* 9571da177e4SLinus Torvalds * Construct an encryption mapping: 9581da177e4SLinus Torvalds * <cipher> <key> <iv_offset> <dev_path> <start> 9591da177e4SLinus Torvalds */ 9601da177e4SLinus Torvalds static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 9611da177e4SLinus Torvalds { 9621da177e4SLinus Torvalds struct crypt_config *cc; 9633a7f6c99SMilan Broz struct crypto_ablkcipher *tfm; 9641da177e4SLinus Torvalds char *tmp; 9651da177e4SLinus Torvalds char *cipher; 9661da177e4SLinus Torvalds char *chainmode; 9671da177e4SLinus Torvalds char *ivmode; 9681da177e4SLinus Torvalds char *ivopts; 9691da177e4SLinus Torvalds unsigned int key_size; 9704ee218cdSAndrew Morton unsigned long long tmpll; 9711da177e4SLinus Torvalds 9721da177e4SLinus Torvalds if (argc != 5) { 97372d94861SAlasdair G Kergon ti->error = "Not enough arguments"; 9741da177e4SLinus Torvalds return -EINVAL; 9751da177e4SLinus Torvalds } 9761da177e4SLinus Torvalds 9771da177e4SLinus Torvalds tmp = argv[0]; 9781da177e4SLinus Torvalds cipher = strsep(&tmp, "-"); 9791da177e4SLinus Torvalds chainmode = strsep(&tmp, "-"); 9801da177e4SLinus Torvalds ivopts = strsep(&tmp, "-"); 9811da177e4SLinus Torvalds ivmode = strsep(&ivopts, ":"); 9821da177e4SLinus Torvalds 9831da177e4SLinus Torvalds if (tmp) 98472d94861SAlasdair G Kergon DMWARN("Unexpected additional cipher options"); 9851da177e4SLinus Torvalds 9861da177e4SLinus Torvalds key_size = strlen(argv[1]) >> 1; 9871da177e4SLinus Torvalds 988e48d4bbfSMilan Broz cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 9891da177e4SLinus Torvalds if (cc == NULL) { 9901da177e4SLinus Torvalds ti->error = 99172d94861SAlasdair G Kergon "Cannot allocate transparent encryption context"; 9921da177e4SLinus Torvalds return -ENOMEM; 9931da177e4SLinus Torvalds } 9941da177e4SLinus Torvalds 99506fe9fb4SDirk Hohndel /* Compatibility mode for old dm-crypt cipher strings */ 9961da177e4SLinus Torvalds if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) { 9971da177e4SLinus Torvalds chainmode = "cbc"; 9981da177e4SLinus Torvalds ivmode = "plain"; 9991da177e4SLinus Torvalds } 10001da177e4SLinus Torvalds 1001d1806f6aSHerbert Xu if (strcmp(chainmode, "ecb") && !ivmode) { 100272d94861SAlasdair G Kergon ti->error = "This chaining mode requires an IV mechanism"; 1003636d5786SMilan Broz goto bad_cipher; 10041da177e4SLinus Torvalds } 10051da177e4SLinus Torvalds 1006d469f841SMilan Broz if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)", 1007d469f841SMilan Broz chainmode, cipher) >= CRYPTO_MAX_ALG_NAME) { 1008d1806f6aSHerbert Xu ti->error = "Chain mode + cipher name is too long"; 1009636d5786SMilan Broz goto bad_cipher; 1010d1806f6aSHerbert Xu } 1011d1806f6aSHerbert Xu 10123a7f6c99SMilan Broz tfm = crypto_alloc_ablkcipher(cc->cipher, 0, 0); 1013d1806f6aSHerbert Xu if (IS_ERR(tfm)) { 101472d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm"; 1015636d5786SMilan Broz goto bad_cipher; 10161da177e4SLinus Torvalds } 10171da177e4SLinus Torvalds 1018d1806f6aSHerbert Xu strcpy(cc->cipher, cipher); 1019d1806f6aSHerbert Xu strcpy(cc->chainmode, chainmode); 10201da177e4SLinus Torvalds cc->tfm = tfm; 10211da177e4SLinus Torvalds 10220b430958SMilan Broz if (crypt_set_key(cc, argv[1]) < 0) { 10230b430958SMilan Broz ti->error = "Error decoding and setting key"; 10240b430958SMilan Broz goto bad_ivmode; 10250b430958SMilan Broz } 10260b430958SMilan Broz 10271da177e4SLinus Torvalds /* 102848527fa7SRik Snel * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi". 10291da177e4SLinus Torvalds * See comments at iv code 10301da177e4SLinus Torvalds */ 10311da177e4SLinus Torvalds 10321da177e4SLinus Torvalds if (ivmode == NULL) 10331da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 10341da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 10351da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 10361da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 10371da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 103848527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 103948527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 104046b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 104146b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 10421da177e4SLinus Torvalds else { 104372d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 1044636d5786SMilan Broz goto bad_ivmode; 10451da177e4SLinus Torvalds } 10461da177e4SLinus Torvalds 10471da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->ctr && 10481da177e4SLinus Torvalds cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0) 1049636d5786SMilan Broz goto bad_ivmode; 10501da177e4SLinus Torvalds 10513a7f6c99SMilan Broz cc->iv_size = crypto_ablkcipher_ivsize(tfm); 1052d1806f6aSHerbert Xu if (cc->iv_size) 10531da177e4SLinus Torvalds /* at least a 64 bit sector number should fit in our buffer */ 1054d1806f6aSHerbert Xu cc->iv_size = max(cc->iv_size, 10551da177e4SLinus Torvalds (unsigned int)(sizeof(u64) / sizeof(u8))); 10561da177e4SLinus Torvalds else { 10571da177e4SLinus Torvalds if (cc->iv_gen_ops) { 105872d94861SAlasdair G Kergon DMWARN("Selected cipher does not support IVs"); 10591da177e4SLinus Torvalds if (cc->iv_gen_ops->dtr) 10601da177e4SLinus Torvalds cc->iv_gen_ops->dtr(cc); 10611da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 10621da177e4SLinus Torvalds } 10631da177e4SLinus Torvalds } 10641da177e4SLinus Torvalds 106593d2341cSMatthew Dobson cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 10661da177e4SLinus Torvalds if (!cc->io_pool) { 106772d94861SAlasdair G Kergon ti->error = "Cannot allocate crypt io mempool"; 1068636d5786SMilan Broz goto bad_slab_pool; 10691da177e4SLinus Torvalds } 10701da177e4SLinus Torvalds 1071ddd42edfSMilan Broz cc->dmreq_start = sizeof(struct ablkcipher_request); 10723a7f6c99SMilan Broz cc->dmreq_start += crypto_ablkcipher_reqsize(tfm); 1073ddd42edfSMilan Broz cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment()); 10743a7f6c99SMilan Broz cc->dmreq_start += crypto_ablkcipher_alignmask(tfm) & 10753a7f6c99SMilan Broz ~(crypto_tfm_ctx_alignment() - 1); 1076ddd42edfSMilan Broz 1077ddd42edfSMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1078ddd42edfSMilan Broz sizeof(struct dm_crypt_request) + cc->iv_size); 1079ddd42edfSMilan Broz if (!cc->req_pool) { 1080ddd42edfSMilan Broz ti->error = "Cannot allocate crypt request mempool"; 1081ddd42edfSMilan Broz goto bad_req_pool; 1082ddd42edfSMilan Broz } 1083ddd42edfSMilan Broz cc->req = NULL; 1084ddd42edfSMilan Broz 1085a19b27ceSMatthew Dobson cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 10861da177e4SLinus Torvalds if (!cc->page_pool) { 108772d94861SAlasdair G Kergon ti->error = "Cannot allocate page mempool"; 1088636d5786SMilan Broz goto bad_page_pool; 10891da177e4SLinus Torvalds } 10901da177e4SLinus Torvalds 1091bb799ca0SJens Axboe cc->bs = bioset_create(MIN_IOS, 0); 10926a24c718SMilan Broz if (!cc->bs) { 10936a24c718SMilan Broz ti->error = "Cannot allocate crypt bioset"; 10946a24c718SMilan Broz goto bad_bs; 10956a24c718SMilan Broz } 10966a24c718SMilan Broz 10974ee218cdSAndrew Morton if (sscanf(argv[2], "%llu", &tmpll) != 1) { 109872d94861SAlasdair G Kergon ti->error = "Invalid iv_offset sector"; 1099636d5786SMilan Broz goto bad_device; 11001da177e4SLinus Torvalds } 11014ee218cdSAndrew Morton cc->iv_offset = tmpll; 11021da177e4SLinus Torvalds 11034ee218cdSAndrew Morton if (sscanf(argv[4], "%llu", &tmpll) != 1) { 110472d94861SAlasdair G Kergon ti->error = "Invalid device sector"; 1105636d5786SMilan Broz goto bad_device; 11061da177e4SLinus Torvalds } 11074ee218cdSAndrew Morton cc->start = tmpll; 11081da177e4SLinus Torvalds 11091da177e4SLinus Torvalds if (dm_get_device(ti, argv[3], cc->start, ti->len, 11101da177e4SLinus Torvalds dm_table_get_mode(ti->table), &cc->dev)) { 111172d94861SAlasdair G Kergon ti->error = "Device lookup failed"; 1112636d5786SMilan Broz goto bad_device; 11131da177e4SLinus Torvalds } 11141da177e4SLinus Torvalds 11151da177e4SLinus Torvalds if (ivmode && cc->iv_gen_ops) { 11161da177e4SLinus Torvalds if (ivopts) 11171da177e4SLinus Torvalds *(ivopts - 1) = ':'; 11181da177e4SLinus Torvalds cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL); 11191da177e4SLinus Torvalds if (!cc->iv_mode) { 112072d94861SAlasdair G Kergon ti->error = "Error kmallocing iv_mode string"; 1121636d5786SMilan Broz goto bad_ivmode_string; 11221da177e4SLinus Torvalds } 11231da177e4SLinus Torvalds strcpy(cc->iv_mode, ivmode); 11241da177e4SLinus Torvalds } else 11251da177e4SLinus Torvalds cc->iv_mode = NULL; 11261da177e4SLinus Torvalds 1127cabf08e4SMilan Broz cc->io_queue = create_singlethread_workqueue("kcryptd_io"); 1128cabf08e4SMilan Broz if (!cc->io_queue) { 1129cabf08e4SMilan Broz ti->error = "Couldn't create kcryptd io queue"; 1130cabf08e4SMilan Broz goto bad_io_queue; 1131cabf08e4SMilan Broz } 1132cabf08e4SMilan Broz 1133cabf08e4SMilan Broz cc->crypt_queue = create_singlethread_workqueue("kcryptd"); 1134cabf08e4SMilan Broz if (!cc->crypt_queue) { 11359934a8beSMilan Broz ti->error = "Couldn't create kcryptd queue"; 1136cabf08e4SMilan Broz goto bad_crypt_queue; 11379934a8beSMilan Broz } 11389934a8beSMilan Broz 1139647c7db1SMikulas Patocka ti->num_flush_requests = 1; 11401da177e4SLinus Torvalds ti->private = cc; 11411da177e4SLinus Torvalds return 0; 11421da177e4SLinus Torvalds 1143cabf08e4SMilan Broz bad_crypt_queue: 1144cabf08e4SMilan Broz destroy_workqueue(cc->io_queue); 1145cabf08e4SMilan Broz bad_io_queue: 11469934a8beSMilan Broz kfree(cc->iv_mode); 1147636d5786SMilan Broz bad_ivmode_string: 114855b42c5aSDmitry Monakhov dm_put_device(ti, cc->dev); 1149636d5786SMilan Broz bad_device: 11506a24c718SMilan Broz bioset_free(cc->bs); 11516a24c718SMilan Broz bad_bs: 11521da177e4SLinus Torvalds mempool_destroy(cc->page_pool); 1153636d5786SMilan Broz bad_page_pool: 1154ddd42edfSMilan Broz mempool_destroy(cc->req_pool); 1155ddd42edfSMilan Broz bad_req_pool: 11561da177e4SLinus Torvalds mempool_destroy(cc->io_pool); 1157636d5786SMilan Broz bad_slab_pool: 11581da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 11591da177e4SLinus Torvalds cc->iv_gen_ops->dtr(cc); 1160636d5786SMilan Broz bad_ivmode: 11613a7f6c99SMilan Broz crypto_free_ablkcipher(tfm); 1162636d5786SMilan Broz bad_cipher: 11639d3520a3SStefan Rompf /* Must zero key material before freeing */ 1164b81d6cf7SJohannes Weiner kzfree(cc); 11651da177e4SLinus Torvalds return -EINVAL; 11661da177e4SLinus Torvalds } 11671da177e4SLinus Torvalds 11681da177e4SLinus Torvalds static void crypt_dtr(struct dm_target *ti) 11691da177e4SLinus Torvalds { 11701da177e4SLinus Torvalds struct crypt_config *cc = (struct crypt_config *) ti->private; 11711da177e4SLinus Torvalds 1172cabf08e4SMilan Broz destroy_workqueue(cc->io_queue); 1173cabf08e4SMilan Broz destroy_workqueue(cc->crypt_queue); 117480b16c19SMilan Broz 1175ddd42edfSMilan Broz if (cc->req) 1176ddd42edfSMilan Broz mempool_free(cc->req, cc->req_pool); 1177ddd42edfSMilan Broz 11786a24c718SMilan Broz bioset_free(cc->bs); 11791da177e4SLinus Torvalds mempool_destroy(cc->page_pool); 1180ddd42edfSMilan Broz mempool_destroy(cc->req_pool); 11811da177e4SLinus Torvalds mempool_destroy(cc->io_pool); 11821da177e4SLinus Torvalds 11831da177e4SLinus Torvalds kfree(cc->iv_mode); 11841da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 11851da177e4SLinus Torvalds cc->iv_gen_ops->dtr(cc); 11863a7f6c99SMilan Broz crypto_free_ablkcipher(cc->tfm); 11871da177e4SLinus Torvalds dm_put_device(ti, cc->dev); 11889d3520a3SStefan Rompf 11899d3520a3SStefan Rompf /* Must zero key material before freeing */ 1190b81d6cf7SJohannes Weiner kzfree(cc); 11911da177e4SLinus Torvalds } 11921da177e4SLinus Torvalds 11931da177e4SLinus Torvalds static int crypt_map(struct dm_target *ti, struct bio *bio, 11941da177e4SLinus Torvalds union map_info *map_context) 11951da177e4SLinus Torvalds { 1196028867acSAlasdair G Kergon struct dm_crypt_io *io; 1197647c7db1SMikulas Patocka struct crypt_config *cc; 1198647c7db1SMikulas Patocka 1199647c7db1SMikulas Patocka if (unlikely(bio_empty_barrier(bio))) { 1200647c7db1SMikulas Patocka cc = ti->private; 1201647c7db1SMikulas Patocka bio->bi_bdev = cc->dev->bdev; 1202647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 1203647c7db1SMikulas Patocka } 12041da177e4SLinus Torvalds 1205dc440d1eSMilan Broz io = crypt_io_alloc(ti, bio, bio->bi_sector - ti->begin); 1206cabf08e4SMilan Broz 1207cabf08e4SMilan Broz if (bio_data_dir(io->base_bio) == READ) 120823541d2dSMilan Broz kcryptd_queue_io(io); 1209cabf08e4SMilan Broz else 1210cabf08e4SMilan Broz kcryptd_queue_crypt(io); 12111da177e4SLinus Torvalds 1212d2a7ad29SKiyoshi Ueda return DM_MAPIO_SUBMITTED; 12131da177e4SLinus Torvalds } 12141da177e4SLinus Torvalds 12151da177e4SLinus Torvalds static int crypt_status(struct dm_target *ti, status_type_t type, 12161da177e4SLinus Torvalds char *result, unsigned int maxlen) 12171da177e4SLinus Torvalds { 12181da177e4SLinus Torvalds struct crypt_config *cc = (struct crypt_config *) ti->private; 12191da177e4SLinus Torvalds unsigned int sz = 0; 12201da177e4SLinus Torvalds 12211da177e4SLinus Torvalds switch (type) { 12221da177e4SLinus Torvalds case STATUSTYPE_INFO: 12231da177e4SLinus Torvalds result[0] = '\0'; 12241da177e4SLinus Torvalds break; 12251da177e4SLinus Torvalds 12261da177e4SLinus Torvalds case STATUSTYPE_TABLE: 12271da177e4SLinus Torvalds if (cc->iv_mode) 122837af6560SChristophe Saout DMEMIT("%s-%s-%s ", cc->cipher, cc->chainmode, 122937af6560SChristophe Saout cc->iv_mode); 12301da177e4SLinus Torvalds else 123137af6560SChristophe Saout DMEMIT("%s-%s ", cc->cipher, cc->chainmode); 12321da177e4SLinus Torvalds 12331da177e4SLinus Torvalds if (cc->key_size > 0) { 12341da177e4SLinus Torvalds if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 12351da177e4SLinus Torvalds return -ENOMEM; 12361da177e4SLinus Torvalds 12371da177e4SLinus Torvalds crypt_encode_key(result + sz, cc->key, cc->key_size); 12381da177e4SLinus Torvalds sz += cc->key_size << 1; 12391da177e4SLinus Torvalds } else { 12401da177e4SLinus Torvalds if (sz >= maxlen) 12411da177e4SLinus Torvalds return -ENOMEM; 12421da177e4SLinus Torvalds result[sz++] = '-'; 12431da177e4SLinus Torvalds } 12441da177e4SLinus Torvalds 12454ee218cdSAndrew Morton DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 12464ee218cdSAndrew Morton cc->dev->name, (unsigned long long)cc->start); 12471da177e4SLinus Torvalds break; 12481da177e4SLinus Torvalds } 12491da177e4SLinus Torvalds return 0; 12501da177e4SLinus Torvalds } 12511da177e4SLinus Torvalds 1252e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 1253e48d4bbfSMilan Broz { 1254e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1255e48d4bbfSMilan Broz 1256e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1257e48d4bbfSMilan Broz } 1258e48d4bbfSMilan Broz 1259e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 1260e48d4bbfSMilan Broz { 1261e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1262e48d4bbfSMilan Broz 1263e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1264e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 1265e48d4bbfSMilan Broz return -EAGAIN; 1266e48d4bbfSMilan Broz } 1267e48d4bbfSMilan Broz 1268e48d4bbfSMilan Broz return 0; 1269e48d4bbfSMilan Broz } 1270e48d4bbfSMilan Broz 1271e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 1272e48d4bbfSMilan Broz { 1273e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1274e48d4bbfSMilan Broz 1275e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1276e48d4bbfSMilan Broz } 1277e48d4bbfSMilan Broz 1278e48d4bbfSMilan Broz /* Message interface 1279e48d4bbfSMilan Broz * key set <key> 1280e48d4bbfSMilan Broz * key wipe 1281e48d4bbfSMilan Broz */ 1282e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1283e48d4bbfSMilan Broz { 1284e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1285e48d4bbfSMilan Broz 1286e48d4bbfSMilan Broz if (argc < 2) 1287e48d4bbfSMilan Broz goto error; 1288e48d4bbfSMilan Broz 1289e48d4bbfSMilan Broz if (!strnicmp(argv[0], MESG_STR("key"))) { 1290e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1291e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 1292e48d4bbfSMilan Broz return -EINVAL; 1293e48d4bbfSMilan Broz } 1294e48d4bbfSMilan Broz if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) 1295e48d4bbfSMilan Broz return crypt_set_key(cc, argv[2]); 1296e48d4bbfSMilan Broz if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) 1297e48d4bbfSMilan Broz return crypt_wipe_key(cc); 1298e48d4bbfSMilan Broz } 1299e48d4bbfSMilan Broz 1300e48d4bbfSMilan Broz error: 1301e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 1302e48d4bbfSMilan Broz return -EINVAL; 1303e48d4bbfSMilan Broz } 1304e48d4bbfSMilan Broz 1305d41e26b9SMilan Broz static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 1306d41e26b9SMilan Broz struct bio_vec *biovec, int max_size) 1307d41e26b9SMilan Broz { 1308d41e26b9SMilan Broz struct crypt_config *cc = ti->private; 1309d41e26b9SMilan Broz struct request_queue *q = bdev_get_queue(cc->dev->bdev); 1310d41e26b9SMilan Broz 1311d41e26b9SMilan Broz if (!q->merge_bvec_fn) 1312d41e26b9SMilan Broz return max_size; 1313d41e26b9SMilan Broz 1314d41e26b9SMilan Broz bvm->bi_bdev = cc->dev->bdev; 1315d41e26b9SMilan Broz bvm->bi_sector = cc->start + bvm->bi_sector - ti->begin; 1316d41e26b9SMilan Broz 1317d41e26b9SMilan Broz return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 1318d41e26b9SMilan Broz } 1319d41e26b9SMilan Broz 1320af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 1321af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 1322af4874e0SMike Snitzer { 1323af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 1324af4874e0SMike Snitzer 13255dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 1326af4874e0SMike Snitzer } 1327af4874e0SMike Snitzer 13281da177e4SLinus Torvalds static struct target_type crypt_target = { 13291da177e4SLinus Torvalds .name = "crypt", 1330af4874e0SMike Snitzer .version = {1, 7, 0}, 13311da177e4SLinus Torvalds .module = THIS_MODULE, 13321da177e4SLinus Torvalds .ctr = crypt_ctr, 13331da177e4SLinus Torvalds .dtr = crypt_dtr, 13341da177e4SLinus Torvalds .map = crypt_map, 13351da177e4SLinus Torvalds .status = crypt_status, 1336e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 1337e48d4bbfSMilan Broz .preresume = crypt_preresume, 1338e48d4bbfSMilan Broz .resume = crypt_resume, 1339e48d4bbfSMilan Broz .message = crypt_message, 1340d41e26b9SMilan Broz .merge = crypt_merge, 1341af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 13421da177e4SLinus Torvalds }; 13431da177e4SLinus Torvalds 13441da177e4SLinus Torvalds static int __init dm_crypt_init(void) 13451da177e4SLinus Torvalds { 13461da177e4SLinus Torvalds int r; 13471da177e4SLinus Torvalds 1348028867acSAlasdair G Kergon _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 13491da177e4SLinus Torvalds if (!_crypt_io_pool) 13501da177e4SLinus Torvalds return -ENOMEM; 13511da177e4SLinus Torvalds 13521da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 13531da177e4SLinus Torvalds if (r < 0) { 135472d94861SAlasdair G Kergon DMERR("register failed %d", r); 13559934a8beSMilan Broz kmem_cache_destroy(_crypt_io_pool); 13561da177e4SLinus Torvalds } 13571da177e4SLinus Torvalds 13581da177e4SLinus Torvalds return r; 13591da177e4SLinus Torvalds } 13601da177e4SLinus Torvalds 13611da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 13621da177e4SLinus Torvalds { 136310d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 13641da177e4SLinus Torvalds kmem_cache_destroy(_crypt_io_pool); 13651da177e4SLinus Torvalds } 13661da177e4SLinus Torvalds 13671da177e4SLinus Torvalds module_init(dm_crypt_init); 13681da177e4SLinus Torvalds module_exit(dm_crypt_exit); 13691da177e4SLinus Torvalds 13701da177e4SLinus Torvalds MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 13711da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 13721da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1373