11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4542da317SMilan Broz * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 943d69034SMilan Broz #include <linux/completion.h> 10d1806f6aSHerbert Xu #include <linux/err.h> 111da177e4SLinus Torvalds #include <linux/module.h> 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/kernel.h> 141da177e4SLinus Torvalds #include <linux/bio.h> 151da177e4SLinus Torvalds #include <linux/blkdev.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/crypto.h> 191da177e4SLinus Torvalds #include <linux/workqueue.h> 203fcfab16SAndrew Morton #include <linux/backing-dev.h> 211da177e4SLinus Torvalds #include <asm/atomic.h> 22378f058cSDavid Hardeman #include <linux/scatterlist.h> 231da177e4SLinus Torvalds #include <asm/page.h> 2448527fa7SRik Snel #include <asm/unaligned.h> 251da177e4SLinus Torvalds 26586e80e6SMikulas Patocka #include <linux/device-mapper.h> 271da177e4SLinus Torvalds 2872d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 29e48d4bbfSMilan Broz #define MESG_STR(x) x, sizeof(x) 301da177e4SLinus Torvalds 311da177e4SLinus Torvalds /* 321da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 331da177e4SLinus Torvalds */ 341da177e4SLinus Torvalds struct convert_context { 3543d69034SMilan Broz struct completion restart; 361da177e4SLinus Torvalds struct bio *bio_in; 371da177e4SLinus Torvalds struct bio *bio_out; 381da177e4SLinus Torvalds unsigned int offset_in; 391da177e4SLinus Torvalds unsigned int offset_out; 401da177e4SLinus Torvalds unsigned int idx_in; 411da177e4SLinus Torvalds unsigned int idx_out; 421da177e4SLinus Torvalds sector_t sector; 4343d69034SMilan Broz atomic_t pending; 441da177e4SLinus Torvalds }; 451da177e4SLinus Torvalds 4653017030SMilan Broz /* 4753017030SMilan Broz * per bio private data 4853017030SMilan Broz */ 4953017030SMilan Broz struct dm_crypt_io { 5053017030SMilan Broz struct dm_target *target; 5153017030SMilan Broz struct bio *base_bio; 5253017030SMilan Broz struct work_struct work; 5353017030SMilan Broz 5453017030SMilan Broz struct convert_context ctx; 5553017030SMilan Broz 5653017030SMilan Broz atomic_t pending; 5753017030SMilan Broz int error; 580c395b0fSMilan Broz sector_t sector; 59393b47efSMilan Broz struct dm_crypt_io *base_io; 6053017030SMilan Broz }; 6153017030SMilan Broz 6201482b76SMilan Broz struct dm_crypt_request { 63b2174eebSHuang Ying struct convert_context *ctx; 6401482b76SMilan Broz struct scatterlist sg_in; 6501482b76SMilan Broz struct scatterlist sg_out; 6601482b76SMilan Broz }; 6701482b76SMilan Broz 681da177e4SLinus Torvalds struct crypt_config; 691da177e4SLinus Torvalds 701da177e4SLinus Torvalds struct crypt_iv_operations { 711da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 721da177e4SLinus Torvalds const char *opts); 731da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 74b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 75542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 761da177e4SLinus Torvalds int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector); 771da177e4SLinus Torvalds }; 781da177e4SLinus Torvalds 7960473592SMilan Broz struct iv_essiv_private { 8060473592SMilan Broz struct crypto_cipher *tfm; 81b95bf2d3SMilan Broz struct crypto_hash *hash_tfm; 82b95bf2d3SMilan Broz u8 *salt; 8360473592SMilan Broz }; 8460473592SMilan Broz 8560473592SMilan Broz struct iv_benbi_private { 8660473592SMilan Broz int shift; 8760473592SMilan Broz }; 8860473592SMilan Broz 891da177e4SLinus Torvalds /* 901da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 911da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 921da177e4SLinus Torvalds */ 93e48d4bbfSMilan Broz enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 941da177e4SLinus Torvalds struct crypt_config { 951da177e4SLinus Torvalds struct dm_dev *dev; 961da177e4SLinus Torvalds sector_t start; 971da177e4SLinus Torvalds 981da177e4SLinus Torvalds /* 99ddd42edfSMilan Broz * pool for per bio private data, crypto requests and 100ddd42edfSMilan Broz * encryption requeusts/buffer pages 1011da177e4SLinus Torvalds */ 1021da177e4SLinus Torvalds mempool_t *io_pool; 103ddd42edfSMilan Broz mempool_t *req_pool; 1041da177e4SLinus Torvalds mempool_t *page_pool; 1056a24c718SMilan Broz struct bio_set *bs; 1061da177e4SLinus Torvalds 107cabf08e4SMilan Broz struct workqueue_struct *io_queue; 108cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1093f1e9070SMilan Broz 1105ebaee6dSMilan Broz char *cipher; 1115ebaee6dSMilan Broz char *cipher_mode; 1125ebaee6dSMilan Broz 1131da177e4SLinus Torvalds struct crypt_iv_operations *iv_gen_ops; 11479066ad3SHerbert Xu union { 11560473592SMilan Broz struct iv_essiv_private essiv; 11660473592SMilan Broz struct iv_benbi_private benbi; 11779066ad3SHerbert Xu } iv_gen_private; 1181da177e4SLinus Torvalds sector_t iv_offset; 1191da177e4SLinus Torvalds unsigned int iv_size; 1201da177e4SLinus Torvalds 121ddd42edfSMilan Broz /* 122ddd42edfSMilan Broz * Layout of each crypto request: 123ddd42edfSMilan Broz * 124ddd42edfSMilan Broz * struct ablkcipher_request 125ddd42edfSMilan Broz * context 126ddd42edfSMilan Broz * padding 127ddd42edfSMilan Broz * struct dm_crypt_request 128ddd42edfSMilan Broz * padding 129ddd42edfSMilan Broz * IV 130ddd42edfSMilan Broz * 131ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 132ddd42edfSMilan Broz * correctly aligned. 133ddd42edfSMilan Broz */ 134ddd42edfSMilan Broz unsigned int dmreq_start; 135ddd42edfSMilan Broz struct ablkcipher_request *req; 136ddd42edfSMilan Broz 1373a7f6c99SMilan Broz struct crypto_ablkcipher *tfm; 138e48d4bbfSMilan Broz unsigned long flags; 1391da177e4SLinus Torvalds unsigned int key_size; 1401da177e4SLinus Torvalds u8 key[0]; 1411da177e4SLinus Torvalds }; 1421da177e4SLinus Torvalds 1436a24c718SMilan Broz #define MIN_IOS 16 1441da177e4SLinus Torvalds #define MIN_POOL_PAGES 32 1451da177e4SLinus Torvalds #define MIN_BIO_PAGES 8 1461da177e4SLinus Torvalds 147e18b890bSChristoph Lameter static struct kmem_cache *_crypt_io_pool; 1481da177e4SLinus Torvalds 149028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 150395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 151027581f3SOlaf Kirch 1521da177e4SLinus Torvalds /* 1531da177e4SLinus Torvalds * Different IV generation algorithms: 1541da177e4SLinus Torvalds * 1553c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 1563a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 1571da177e4SLinus Torvalds * 15861afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 15961afef61SMilan Broz * number, padded with zeros if necessary. 16061afef61SMilan Broz * 1613c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 1621da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 1631da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 1641da177e4SLinus Torvalds * 16548527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 16648527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 16748527fa7SRik Snel * 16846b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 16946b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 17046b47730SLudwig Nussel * 1711da177e4SLinus Torvalds * plumb: unimplemented, see: 1721da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 1731da177e4SLinus Torvalds */ 1741da177e4SLinus Torvalds 1751da177e4SLinus Torvalds static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 1761da177e4SLinus Torvalds { 1771da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 1781da177e4SLinus Torvalds *(u32 *)iv = cpu_to_le32(sector & 0xffffffff); 1791da177e4SLinus Torvalds 1801da177e4SLinus Torvalds return 0; 1811da177e4SLinus Torvalds } 1821da177e4SLinus Torvalds 18361afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 18461afef61SMilan Broz sector_t sector) 18561afef61SMilan Broz { 18661afef61SMilan Broz memset(iv, 0, cc->iv_size); 18761afef61SMilan Broz *(u64 *)iv = cpu_to_le64(sector); 18861afef61SMilan Broz 18961afef61SMilan Broz return 0; 19061afef61SMilan Broz } 19161afef61SMilan Broz 192b95bf2d3SMilan Broz /* Initialise ESSIV - compute salt but no local memory allocations */ 193b95bf2d3SMilan Broz static int crypt_iv_essiv_init(struct crypt_config *cc) 194b95bf2d3SMilan Broz { 195b95bf2d3SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 196b95bf2d3SMilan Broz struct hash_desc desc; 197b95bf2d3SMilan Broz struct scatterlist sg; 198b95bf2d3SMilan Broz int err; 199b95bf2d3SMilan Broz 200b95bf2d3SMilan Broz sg_init_one(&sg, cc->key, cc->key_size); 201b95bf2d3SMilan Broz desc.tfm = essiv->hash_tfm; 202b95bf2d3SMilan Broz desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 203b95bf2d3SMilan Broz 204b95bf2d3SMilan Broz err = crypto_hash_digest(&desc, &sg, cc->key_size, essiv->salt); 205b95bf2d3SMilan Broz if (err) 206b95bf2d3SMilan Broz return err; 207b95bf2d3SMilan Broz 208b95bf2d3SMilan Broz return crypto_cipher_setkey(essiv->tfm, essiv->salt, 209b95bf2d3SMilan Broz crypto_hash_digestsize(essiv->hash_tfm)); 210b95bf2d3SMilan Broz } 211b95bf2d3SMilan Broz 212542da317SMilan Broz /* Wipe salt and reset key derived from volume key */ 213542da317SMilan Broz static int crypt_iv_essiv_wipe(struct crypt_config *cc) 214542da317SMilan Broz { 215542da317SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 216542da317SMilan Broz unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm); 217542da317SMilan Broz 218542da317SMilan Broz memset(essiv->salt, 0, salt_size); 219542da317SMilan Broz 220542da317SMilan Broz return crypto_cipher_setkey(essiv->tfm, essiv->salt, salt_size); 221542da317SMilan Broz } 222542da317SMilan Broz 22360473592SMilan Broz static void crypt_iv_essiv_dtr(struct crypt_config *cc) 22460473592SMilan Broz { 22560473592SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 22660473592SMilan Broz 22760473592SMilan Broz crypto_free_cipher(essiv->tfm); 22860473592SMilan Broz essiv->tfm = NULL; 229b95bf2d3SMilan Broz 230b95bf2d3SMilan Broz crypto_free_hash(essiv->hash_tfm); 231b95bf2d3SMilan Broz essiv->hash_tfm = NULL; 232b95bf2d3SMilan Broz 233b95bf2d3SMilan Broz kzfree(essiv->salt); 234b95bf2d3SMilan Broz essiv->salt = NULL; 23560473592SMilan Broz } 23660473592SMilan Broz 2371da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 2381da177e4SLinus Torvalds const char *opts) 2391da177e4SLinus Torvalds { 2405861f1beSMilan Broz struct crypto_cipher *essiv_tfm = NULL; 2415861f1beSMilan Broz struct crypto_hash *hash_tfm = NULL; 2425861f1beSMilan Broz u8 *salt = NULL; 243d1806f6aSHerbert Xu int err; 2441da177e4SLinus Torvalds 2455861f1beSMilan Broz if (!opts) { 24672d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 2471da177e4SLinus Torvalds return -EINVAL; 2481da177e4SLinus Torvalds } 2491da177e4SLinus Torvalds 250b95bf2d3SMilan Broz /* Allocate hash algorithm */ 25135058687SHerbert Xu hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 25235058687SHerbert Xu if (IS_ERR(hash_tfm)) { 25372d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 2545861f1beSMilan Broz err = PTR_ERR(hash_tfm); 2555861f1beSMilan Broz goto bad; 2561da177e4SLinus Torvalds } 2571da177e4SLinus Torvalds 258b95bf2d3SMilan Broz salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL); 2595861f1beSMilan Broz if (!salt) { 26072d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 2615861f1beSMilan Broz err = -ENOMEM; 2625861f1beSMilan Broz goto bad; 2631da177e4SLinus Torvalds } 2641da177e4SLinus Torvalds 265b95bf2d3SMilan Broz /* Allocate essiv_tfm */ 266d1806f6aSHerbert Xu essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 267d1806f6aSHerbert Xu if (IS_ERR(essiv_tfm)) { 26872d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm for ESSIV"; 2695861f1beSMilan Broz err = PTR_ERR(essiv_tfm); 2705861f1beSMilan Broz goto bad; 2711da177e4SLinus Torvalds } 272d1806f6aSHerbert Xu if (crypto_cipher_blocksize(essiv_tfm) != 2733a7f6c99SMilan Broz crypto_ablkcipher_ivsize(cc->tfm)) { 27472d94861SAlasdair G Kergon ti->error = "Block size of ESSIV cipher does " 2751da177e4SLinus Torvalds "not match IV size of block cipher"; 2765861f1beSMilan Broz err = -EINVAL; 2775861f1beSMilan Broz goto bad; 2781da177e4SLinus Torvalds } 2791da177e4SLinus Torvalds 280b95bf2d3SMilan Broz cc->iv_gen_private.essiv.salt = salt; 28160473592SMilan Broz cc->iv_gen_private.essiv.tfm = essiv_tfm; 282b95bf2d3SMilan Broz cc->iv_gen_private.essiv.hash_tfm = hash_tfm; 283b95bf2d3SMilan Broz 2841da177e4SLinus Torvalds return 0; 2855861f1beSMilan Broz 2865861f1beSMilan Broz bad: 2875861f1beSMilan Broz if (essiv_tfm && !IS_ERR(essiv_tfm)) 2885861f1beSMilan Broz crypto_free_cipher(essiv_tfm); 2895861f1beSMilan Broz if (hash_tfm && !IS_ERR(hash_tfm)) 2905861f1beSMilan Broz crypto_free_hash(hash_tfm); 291b95bf2d3SMilan Broz kfree(salt); 2925861f1beSMilan Broz return err; 2931da177e4SLinus Torvalds } 2941da177e4SLinus Torvalds 2951da177e4SLinus Torvalds static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 2961da177e4SLinus Torvalds { 2971da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 2981da177e4SLinus Torvalds *(u64 *)iv = cpu_to_le64(sector); 29960473592SMilan Broz crypto_cipher_encrypt_one(cc->iv_gen_private.essiv.tfm, iv, iv); 3001da177e4SLinus Torvalds return 0; 3011da177e4SLinus Torvalds } 3021da177e4SLinus Torvalds 30348527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 30448527fa7SRik Snel const char *opts) 30548527fa7SRik Snel { 3063a7f6c99SMilan Broz unsigned bs = crypto_ablkcipher_blocksize(cc->tfm); 307f0d1b0b3SDavid Howells int log = ilog2(bs); 30848527fa7SRik Snel 30948527fa7SRik Snel /* we need to calculate how far we must shift the sector count 31048527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 31148527fa7SRik Snel 31248527fa7SRik Snel if (1 << log != bs) { 31348527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 31448527fa7SRik Snel return -EINVAL; 31548527fa7SRik Snel } 31648527fa7SRik Snel 31748527fa7SRik Snel if (log > 9) { 31848527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 31948527fa7SRik Snel return -EINVAL; 32048527fa7SRik Snel } 32148527fa7SRik Snel 32260473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 32348527fa7SRik Snel 32448527fa7SRik Snel return 0; 32548527fa7SRik Snel } 32648527fa7SRik Snel 32748527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 32848527fa7SRik Snel { 32948527fa7SRik Snel } 33048527fa7SRik Snel 33148527fa7SRik Snel static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 33248527fa7SRik Snel { 33379066ad3SHerbert Xu __be64 val; 33479066ad3SHerbert Xu 33548527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 33679066ad3SHerbert Xu 33760473592SMilan Broz val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi.shift) + 1); 33879066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 33948527fa7SRik Snel 3401da177e4SLinus Torvalds return 0; 3411da177e4SLinus Torvalds } 3421da177e4SLinus Torvalds 34346b47730SLudwig Nussel static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 34446b47730SLudwig Nussel { 34546b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 34646b47730SLudwig Nussel 34746b47730SLudwig Nussel return 0; 34846b47730SLudwig Nussel } 34946b47730SLudwig Nussel 3501da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_plain_ops = { 3511da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 3521da177e4SLinus Torvalds }; 3531da177e4SLinus Torvalds 35461afef61SMilan Broz static struct crypt_iv_operations crypt_iv_plain64_ops = { 35561afef61SMilan Broz .generator = crypt_iv_plain64_gen 35661afef61SMilan Broz }; 35761afef61SMilan Broz 3581da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_essiv_ops = { 3591da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 3601da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 361b95bf2d3SMilan Broz .init = crypt_iv_essiv_init, 362542da317SMilan Broz .wipe = crypt_iv_essiv_wipe, 3631da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 3641da177e4SLinus Torvalds }; 3651da177e4SLinus Torvalds 36648527fa7SRik Snel static struct crypt_iv_operations crypt_iv_benbi_ops = { 36748527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 36848527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 36948527fa7SRik Snel .generator = crypt_iv_benbi_gen 37048527fa7SRik Snel }; 3711da177e4SLinus Torvalds 37246b47730SLudwig Nussel static struct crypt_iv_operations crypt_iv_null_ops = { 37346b47730SLudwig Nussel .generator = crypt_iv_null_gen 37446b47730SLudwig Nussel }; 37546b47730SLudwig Nussel 376d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 377d469f841SMilan Broz struct convert_context *ctx, 3781da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 379fcd369daSMilan Broz sector_t sector) 3801da177e4SLinus Torvalds { 3811da177e4SLinus Torvalds ctx->bio_in = bio_in; 3821da177e4SLinus Torvalds ctx->bio_out = bio_out; 3831da177e4SLinus Torvalds ctx->offset_in = 0; 3841da177e4SLinus Torvalds ctx->offset_out = 0; 3851da177e4SLinus Torvalds ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 3861da177e4SLinus Torvalds ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 3871da177e4SLinus Torvalds ctx->sector = sector + cc->iv_offset; 38843d69034SMilan Broz init_completion(&ctx->restart); 3891da177e4SLinus Torvalds } 3901da177e4SLinus Torvalds 391b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 392b2174eebSHuang Ying struct ablkcipher_request *req) 393b2174eebSHuang Ying { 394b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 395b2174eebSHuang Ying } 396b2174eebSHuang Ying 397b2174eebSHuang Ying static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc, 398b2174eebSHuang Ying struct dm_crypt_request *dmreq) 399b2174eebSHuang Ying { 400b2174eebSHuang Ying return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start); 401b2174eebSHuang Ying } 402b2174eebSHuang Ying 40301482b76SMilan Broz static int crypt_convert_block(struct crypt_config *cc, 4043a7f6c99SMilan Broz struct convert_context *ctx, 4053a7f6c99SMilan Broz struct ablkcipher_request *req) 40601482b76SMilan Broz { 40701482b76SMilan Broz struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 40801482b76SMilan Broz struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 4093a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 4103a7f6c99SMilan Broz u8 *iv; 4113a7f6c99SMilan Broz int r = 0; 41201482b76SMilan Broz 413b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 4143a7f6c99SMilan Broz iv = (u8 *)ALIGN((unsigned long)(dmreq + 1), 4153a7f6c99SMilan Broz crypto_ablkcipher_alignmask(cc->tfm) + 1); 4163a7f6c99SMilan Broz 417b2174eebSHuang Ying dmreq->ctx = ctx; 4183a7f6c99SMilan Broz sg_init_table(&dmreq->sg_in, 1); 4193a7f6c99SMilan Broz sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT, 42001482b76SMilan Broz bv_in->bv_offset + ctx->offset_in); 42101482b76SMilan Broz 4223a7f6c99SMilan Broz sg_init_table(&dmreq->sg_out, 1); 4233a7f6c99SMilan Broz sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT, 42401482b76SMilan Broz bv_out->bv_offset + ctx->offset_out); 42501482b76SMilan Broz 42601482b76SMilan Broz ctx->offset_in += 1 << SECTOR_SHIFT; 42701482b76SMilan Broz if (ctx->offset_in >= bv_in->bv_len) { 42801482b76SMilan Broz ctx->offset_in = 0; 42901482b76SMilan Broz ctx->idx_in++; 43001482b76SMilan Broz } 43101482b76SMilan Broz 43201482b76SMilan Broz ctx->offset_out += 1 << SECTOR_SHIFT; 43301482b76SMilan Broz if (ctx->offset_out >= bv_out->bv_len) { 43401482b76SMilan Broz ctx->offset_out = 0; 43501482b76SMilan Broz ctx->idx_out++; 43601482b76SMilan Broz } 43701482b76SMilan Broz 4383a7f6c99SMilan Broz if (cc->iv_gen_ops) { 4393a7f6c99SMilan Broz r = cc->iv_gen_ops->generator(cc, iv, ctx->sector); 4403a7f6c99SMilan Broz if (r < 0) 4413a7f6c99SMilan Broz return r; 4423a7f6c99SMilan Broz } 4433a7f6c99SMilan Broz 4443a7f6c99SMilan Broz ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out, 4453a7f6c99SMilan Broz 1 << SECTOR_SHIFT, iv); 4463a7f6c99SMilan Broz 4473a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 4483a7f6c99SMilan Broz r = crypto_ablkcipher_encrypt(req); 4493a7f6c99SMilan Broz else 4503a7f6c99SMilan Broz r = crypto_ablkcipher_decrypt(req); 4513a7f6c99SMilan Broz 4523a7f6c99SMilan Broz return r; 45301482b76SMilan Broz } 45401482b76SMilan Broz 45595497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 45695497a96SMilan Broz int error); 457ddd42edfSMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 458ddd42edfSMilan Broz struct convert_context *ctx) 459ddd42edfSMilan Broz { 460ddd42edfSMilan Broz if (!cc->req) 461ddd42edfSMilan Broz cc->req = mempool_alloc(cc->req_pool, GFP_NOIO); 46295497a96SMilan Broz ablkcipher_request_set_tfm(cc->req, cc->tfm); 46395497a96SMilan Broz ablkcipher_request_set_callback(cc->req, CRYPTO_TFM_REQ_MAY_BACKLOG | 46495497a96SMilan Broz CRYPTO_TFM_REQ_MAY_SLEEP, 465b2174eebSHuang Ying kcryptd_async_done, 466b2174eebSHuang Ying dmreq_of_req(cc, cc->req)); 467ddd42edfSMilan Broz } 468ddd42edfSMilan Broz 4691da177e4SLinus Torvalds /* 4701da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 4711da177e4SLinus Torvalds */ 4721da177e4SLinus Torvalds static int crypt_convert(struct crypt_config *cc, 4731da177e4SLinus Torvalds struct convert_context *ctx) 4741da177e4SLinus Torvalds { 4753f1e9070SMilan Broz int r; 4761da177e4SLinus Torvalds 477c8081618SMilan Broz atomic_set(&ctx->pending, 1); 478c8081618SMilan Broz 4791da177e4SLinus Torvalds while(ctx->idx_in < ctx->bio_in->bi_vcnt && 4801da177e4SLinus Torvalds ctx->idx_out < ctx->bio_out->bi_vcnt) { 4811da177e4SLinus Torvalds 4823a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 4833a7f6c99SMilan Broz 4843f1e9070SMilan Broz atomic_inc(&ctx->pending); 4853f1e9070SMilan Broz 4863a7f6c99SMilan Broz r = crypt_convert_block(cc, ctx, cc->req); 4873a7f6c99SMilan Broz 4883a7f6c99SMilan Broz switch (r) { 4893f1e9070SMilan Broz /* async */ 4903a7f6c99SMilan Broz case -EBUSY: 4913a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 4923a7f6c99SMilan Broz INIT_COMPLETION(ctx->restart); 4933a7f6c99SMilan Broz /* fall through*/ 4943a7f6c99SMilan Broz case -EINPROGRESS: 4953a7f6c99SMilan Broz cc->req = NULL; 4961da177e4SLinus Torvalds ctx->sector++; 4973a7f6c99SMilan Broz continue; 4983a7f6c99SMilan Broz 4993f1e9070SMilan Broz /* sync */ 5003f1e9070SMilan Broz case 0: 5013f1e9070SMilan Broz atomic_dec(&ctx->pending); 5023f1e9070SMilan Broz ctx->sector++; 503c7f1b204SMilan Broz cond_resched(); 5043f1e9070SMilan Broz continue; 5051da177e4SLinus Torvalds 5063f1e9070SMilan Broz /* error */ 5073f1e9070SMilan Broz default: 5083f1e9070SMilan Broz atomic_dec(&ctx->pending); 5091da177e4SLinus Torvalds return r; 5101da177e4SLinus Torvalds } 5113f1e9070SMilan Broz } 5123f1e9070SMilan Broz 5133f1e9070SMilan Broz return 0; 5143f1e9070SMilan Broz } 5151da177e4SLinus Torvalds 5166a24c718SMilan Broz static void dm_crypt_bio_destructor(struct bio *bio) 5176a24c718SMilan Broz { 518028867acSAlasdair G Kergon struct dm_crypt_io *io = bio->bi_private; 5196a24c718SMilan Broz struct crypt_config *cc = io->target->private; 5206a24c718SMilan Broz 5216a24c718SMilan Broz bio_free(bio, cc->bs); 5226a24c718SMilan Broz } 5236a24c718SMilan Broz 5241da177e4SLinus Torvalds /* 5251da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 5261da177e4SLinus Torvalds * This should never violate the device limitations 527933f01d4SMilan Broz * May return a smaller bio when running out of pages, indicated by 528933f01d4SMilan Broz * *out_of_pages set to 1. 5291da177e4SLinus Torvalds */ 530933f01d4SMilan Broz static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, 531933f01d4SMilan Broz unsigned *out_of_pages) 5321da177e4SLinus Torvalds { 533027581f3SOlaf Kirch struct crypt_config *cc = io->target->private; 5348b004457SMilan Broz struct bio *clone; 5351da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 536b4e3ca1aSAl Viro gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 53791e10625SMilan Broz unsigned i, len; 53891e10625SMilan Broz struct page *page; 5391da177e4SLinus Torvalds 5406a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 5418b004457SMilan Broz if (!clone) 5421da177e4SLinus Torvalds return NULL; 5431da177e4SLinus Torvalds 544027581f3SOlaf Kirch clone_init(io, clone); 545933f01d4SMilan Broz *out_of_pages = 0; 5466a24c718SMilan Broz 547f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 54891e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 549933f01d4SMilan Broz if (!page) { 550933f01d4SMilan Broz *out_of_pages = 1; 5511da177e4SLinus Torvalds break; 552933f01d4SMilan Broz } 5531da177e4SLinus Torvalds 5541da177e4SLinus Torvalds /* 5551da177e4SLinus Torvalds * if additional pages cannot be allocated without waiting, 5561da177e4SLinus Torvalds * return a partially allocated bio, the caller will then try 5571da177e4SLinus Torvalds * to allocate additional bios while submitting this partial bio 5581da177e4SLinus Torvalds */ 559f97380bcSOlaf Kirch if (i == (MIN_BIO_PAGES - 1)) 5601da177e4SLinus Torvalds gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 5611da177e4SLinus Torvalds 56291e10625SMilan Broz len = (size > PAGE_SIZE) ? PAGE_SIZE : size; 5631da177e4SLinus Torvalds 56491e10625SMilan Broz if (!bio_add_page(clone, page, len, 0)) { 56591e10625SMilan Broz mempool_free(page, cc->page_pool); 56691e10625SMilan Broz break; 56791e10625SMilan Broz } 56891e10625SMilan Broz 56991e10625SMilan Broz size -= len; 5701da177e4SLinus Torvalds } 5711da177e4SLinus Torvalds 5728b004457SMilan Broz if (!clone->bi_size) { 5738b004457SMilan Broz bio_put(clone); 5741da177e4SLinus Torvalds return NULL; 5751da177e4SLinus Torvalds } 5761da177e4SLinus Torvalds 5778b004457SMilan Broz return clone; 5781da177e4SLinus Torvalds } 5791da177e4SLinus Torvalds 580644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 5811da177e4SLinus Torvalds { 582644bd2f0SNeil Brown unsigned int i; 5831da177e4SLinus Torvalds struct bio_vec *bv; 5841da177e4SLinus Torvalds 585644bd2f0SNeil Brown for (i = 0; i < clone->bi_vcnt; i++) { 5868b004457SMilan Broz bv = bio_iovec_idx(clone, i); 5871da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 5881da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 5891da177e4SLinus Torvalds bv->bv_page = NULL; 5901da177e4SLinus Torvalds } 5911da177e4SLinus Torvalds } 5921da177e4SLinus Torvalds 593dc440d1eSMilan Broz static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti, 594dc440d1eSMilan Broz struct bio *bio, sector_t sector) 595dc440d1eSMilan Broz { 596dc440d1eSMilan Broz struct crypt_config *cc = ti->private; 597dc440d1eSMilan Broz struct dm_crypt_io *io; 598dc440d1eSMilan Broz 599dc440d1eSMilan Broz io = mempool_alloc(cc->io_pool, GFP_NOIO); 600dc440d1eSMilan Broz io->target = ti; 601dc440d1eSMilan Broz io->base_bio = bio; 602dc440d1eSMilan Broz io->sector = sector; 603dc440d1eSMilan Broz io->error = 0; 604393b47efSMilan Broz io->base_io = NULL; 605dc440d1eSMilan Broz atomic_set(&io->pending, 0); 606dc440d1eSMilan Broz 607dc440d1eSMilan Broz return io; 608dc440d1eSMilan Broz } 609dc440d1eSMilan Broz 6103e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 6113e1a8bddSMilan Broz { 6123e1a8bddSMilan Broz atomic_inc(&io->pending); 6133e1a8bddSMilan Broz } 6143e1a8bddSMilan Broz 6151da177e4SLinus Torvalds /* 6161da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 6171da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 618393b47efSMilan Broz * If base_io is set, wait for the last fragment to complete. 6191da177e4SLinus Torvalds */ 6205742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 6211da177e4SLinus Torvalds { 6225742fd77SMilan Broz struct crypt_config *cc = io->target->private; 623b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 624b35f8caaSMilan Broz struct dm_crypt_io *base_io = io->base_io; 625b35f8caaSMilan Broz int error = io->error; 6261da177e4SLinus Torvalds 6271da177e4SLinus Torvalds if (!atomic_dec_and_test(&io->pending)) 6281da177e4SLinus Torvalds return; 6291da177e4SLinus Torvalds 6301da177e4SLinus Torvalds mempool_free(io, cc->io_pool); 631b35f8caaSMilan Broz 632b35f8caaSMilan Broz if (likely(!base_io)) 633b35f8caaSMilan Broz bio_endio(base_bio, error); 634b35f8caaSMilan Broz else { 635b35f8caaSMilan Broz if (error && !base_io->error) 636b35f8caaSMilan Broz base_io->error = error; 637b35f8caaSMilan Broz crypt_dec_pending(base_io); 638b35f8caaSMilan Broz } 6391da177e4SLinus Torvalds } 6401da177e4SLinus Torvalds 6411da177e4SLinus Torvalds /* 642cabf08e4SMilan Broz * kcryptd/kcryptd_io: 6431da177e4SLinus Torvalds * 6441da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 64523541d2dSMilan Broz * interrupt context. 646cabf08e4SMilan Broz * 647cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 648cabf08e4SMilan Broz * 649cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 650cabf08e4SMilan Broz * 651cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 652cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 653cabf08e4SMilan Broz * to memory allocation. 6541da177e4SLinus Torvalds */ 6556712ecf8SNeilBrown static void crypt_endio(struct bio *clone, int error) 6568b004457SMilan Broz { 657028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 6588b004457SMilan Broz struct crypt_config *cc = io->target->private; 659ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 6608b004457SMilan Broz 661adfe4770SMilan Broz if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error)) 662adfe4770SMilan Broz error = -EIO; 663adfe4770SMilan Broz 6648b004457SMilan Broz /* 6656712ecf8SNeilBrown * free the processed pages 6668b004457SMilan Broz */ 667ee7a491eSMilan Broz if (rw == WRITE) 668644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 6698b004457SMilan Broz 6708b004457SMilan Broz bio_put(clone); 671ee7a491eSMilan Broz 672ee7a491eSMilan Broz if (rw == READ && !error) { 673cabf08e4SMilan Broz kcryptd_queue_crypt(io); 6746712ecf8SNeilBrown return; 675ee7a491eSMilan Broz } 6765742fd77SMilan Broz 6775742fd77SMilan Broz if (unlikely(error)) 6785742fd77SMilan Broz io->error = error; 6795742fd77SMilan Broz 6805742fd77SMilan Broz crypt_dec_pending(io); 6818b004457SMilan Broz } 6828b004457SMilan Broz 683028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 6848b004457SMilan Broz { 6858b004457SMilan Broz struct crypt_config *cc = io->target->private; 6868b004457SMilan Broz 6878b004457SMilan Broz clone->bi_private = io; 6888b004457SMilan Broz clone->bi_end_io = crypt_endio; 6898b004457SMilan Broz clone->bi_bdev = cc->dev->bdev; 6908b004457SMilan Broz clone->bi_rw = io->base_bio->bi_rw; 691027581f3SOlaf Kirch clone->bi_destructor = dm_crypt_bio_destructor; 6928b004457SMilan Broz } 6938b004457SMilan Broz 6944e4eef64SMilan Broz static void kcryptd_io_read(struct dm_crypt_io *io) 6958b004457SMilan Broz { 6968b004457SMilan Broz struct crypt_config *cc = io->target->private; 6978b004457SMilan Broz struct bio *base_bio = io->base_bio; 6988b004457SMilan Broz struct bio *clone; 69993e605c2SMilan Broz 7003e1a8bddSMilan Broz crypt_inc_pending(io); 7018b004457SMilan Broz 7028b004457SMilan Broz /* 7038b004457SMilan Broz * The block layer might modify the bvec array, so always 7048b004457SMilan Broz * copy the required bvecs because we need the original 7058b004457SMilan Broz * one in order to decrypt the whole bio data *afterwards*. 7068b004457SMilan Broz */ 7076a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs); 70893e605c2SMilan Broz if (unlikely(!clone)) { 7095742fd77SMilan Broz io->error = -ENOMEM; 7105742fd77SMilan Broz crypt_dec_pending(io); 71123541d2dSMilan Broz return; 71293e605c2SMilan Broz } 7138b004457SMilan Broz 7148b004457SMilan Broz clone_init(io, clone); 7158b004457SMilan Broz clone->bi_idx = 0; 7168b004457SMilan Broz clone->bi_vcnt = bio_segments(base_bio); 7178b004457SMilan Broz clone->bi_size = base_bio->bi_size; 7180c395b0fSMilan Broz clone->bi_sector = cc->start + io->sector; 7198b004457SMilan Broz memcpy(clone->bi_io_vec, bio_iovec(base_bio), 7208b004457SMilan Broz sizeof(struct bio_vec) * clone->bi_vcnt); 7218b004457SMilan Broz 72293e605c2SMilan Broz generic_make_request(clone); 7238b004457SMilan Broz } 7248b004457SMilan Broz 7254e4eef64SMilan Broz static void kcryptd_io_write(struct dm_crypt_io *io) 7264e4eef64SMilan Broz { 72795497a96SMilan Broz struct bio *clone = io->ctx.bio_out; 72895497a96SMilan Broz generic_make_request(clone); 7294e4eef64SMilan Broz } 7304e4eef64SMilan Broz 731395b167cSAlasdair G Kergon static void kcryptd_io(struct work_struct *work) 732395b167cSAlasdair G Kergon { 733395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 734395b167cSAlasdair G Kergon 735395b167cSAlasdair G Kergon if (bio_data_dir(io->base_bio) == READ) 736395b167cSAlasdair G Kergon kcryptd_io_read(io); 737395b167cSAlasdair G Kergon else 738395b167cSAlasdair G Kergon kcryptd_io_write(io); 739395b167cSAlasdair G Kergon } 740395b167cSAlasdair G Kergon 741395b167cSAlasdair G Kergon static void kcryptd_queue_io(struct dm_crypt_io *io) 742395b167cSAlasdair G Kergon { 743395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 744395b167cSAlasdair G Kergon 745395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_io); 746395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 747395b167cSAlasdair G Kergon } 748395b167cSAlasdair G Kergon 74995497a96SMilan Broz static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, 75095497a96SMilan Broz int error, int async) 7514e4eef64SMilan Broz { 752dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 753dec1cedfSMilan Broz struct crypt_config *cc = io->target->private; 754dec1cedfSMilan Broz 755dec1cedfSMilan Broz if (unlikely(error < 0)) { 756dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 757dec1cedfSMilan Broz bio_put(clone); 758dec1cedfSMilan Broz io->error = -EIO; 7596c031f41SMilan Broz crypt_dec_pending(io); 760dec1cedfSMilan Broz return; 761dec1cedfSMilan Broz } 762dec1cedfSMilan Broz 763dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 764dec1cedfSMilan Broz BUG_ON(io->ctx.idx_out < clone->bi_vcnt); 765dec1cedfSMilan Broz 766dec1cedfSMilan Broz clone->bi_sector = cc->start + io->sector; 767899c95d3SMilan Broz 76895497a96SMilan Broz if (async) 76995497a96SMilan Broz kcryptd_queue_io(io); 7701e37bb8eSAlasdair G Kergon else 771899c95d3SMilan Broz generic_make_request(clone); 7724e4eef64SMilan Broz } 7734e4eef64SMilan Broz 774fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 7758b004457SMilan Broz { 7768b004457SMilan Broz struct crypt_config *cc = io->target->private; 7778b004457SMilan Broz struct bio *clone; 778393b47efSMilan Broz struct dm_crypt_io *new_io; 779c8081618SMilan Broz int crypt_finished; 780933f01d4SMilan Broz unsigned out_of_pages = 0; 781dec1cedfSMilan Broz unsigned remaining = io->base_bio->bi_size; 782b635b00eSMilan Broz sector_t sector = io->sector; 783dec1cedfSMilan Broz int r; 7848b004457SMilan Broz 78593e605c2SMilan Broz /* 786fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 787fc5a5e9aSMilan Broz */ 788fc5a5e9aSMilan Broz crypt_inc_pending(io); 789b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 790fc5a5e9aSMilan Broz 791fc5a5e9aSMilan Broz /* 79293e605c2SMilan Broz * The allocated buffers can be smaller than the whole bio, 79393e605c2SMilan Broz * so repeat the whole process until all the data can be handled. 79493e605c2SMilan Broz */ 79593e605c2SMilan Broz while (remaining) { 796933f01d4SMilan Broz clone = crypt_alloc_buffer(io, remaining, &out_of_pages); 79723541d2dSMilan Broz if (unlikely(!clone)) { 7985742fd77SMilan Broz io->error = -ENOMEM; 799fc5a5e9aSMilan Broz break; 80023541d2dSMilan Broz } 8018b004457SMilan Broz 80253017030SMilan Broz io->ctx.bio_out = clone; 80353017030SMilan Broz io->ctx.idx_out = 0; 8048b004457SMilan Broz 80593e605c2SMilan Broz remaining -= clone->bi_size; 806b635b00eSMilan Broz sector += bio_sectors(clone); 807dec1cedfSMilan Broz 8084e594098SMilan Broz crypt_inc_pending(io); 809dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 810c8081618SMilan Broz crypt_finished = atomic_dec_and_test(&io->ctx.pending); 811dec1cedfSMilan Broz 812c8081618SMilan Broz /* Encryption was already finished, submit io now */ 813c8081618SMilan Broz if (crypt_finished) { 81495497a96SMilan Broz kcryptd_crypt_write_io_submit(io, r, 0); 815c8081618SMilan Broz 816c8081618SMilan Broz /* 817c8081618SMilan Broz * If there was an error, do not try next fragments. 818c8081618SMilan Broz * For async, error is processed in async handler. 819c8081618SMilan Broz */ 8206c031f41SMilan Broz if (unlikely(r < 0)) 821fc5a5e9aSMilan Broz break; 822b635b00eSMilan Broz 823b635b00eSMilan Broz io->sector = sector; 8244e594098SMilan Broz } 82593e605c2SMilan Broz 826933f01d4SMilan Broz /* 827933f01d4SMilan Broz * Out of memory -> run queues 828933f01d4SMilan Broz * But don't wait if split was due to the io size restriction 829933f01d4SMilan Broz */ 830933f01d4SMilan Broz if (unlikely(out_of_pages)) 8318aa7e847SJens Axboe congestion_wait(BLK_RW_ASYNC, HZ/100); 832933f01d4SMilan Broz 833393b47efSMilan Broz /* 834393b47efSMilan Broz * With async crypto it is unsafe to share the crypto context 835393b47efSMilan Broz * between fragments, so switch to a new dm_crypt_io structure. 836393b47efSMilan Broz */ 837393b47efSMilan Broz if (unlikely(!crypt_finished && remaining)) { 838393b47efSMilan Broz new_io = crypt_io_alloc(io->target, io->base_bio, 839393b47efSMilan Broz sector); 840393b47efSMilan Broz crypt_inc_pending(new_io); 841393b47efSMilan Broz crypt_convert_init(cc, &new_io->ctx, NULL, 842393b47efSMilan Broz io->base_bio, sector); 843393b47efSMilan Broz new_io->ctx.idx_in = io->ctx.idx_in; 844393b47efSMilan Broz new_io->ctx.offset_in = io->ctx.offset_in; 845393b47efSMilan Broz 846393b47efSMilan Broz /* 847393b47efSMilan Broz * Fragments after the first use the base_io 848393b47efSMilan Broz * pending count. 849393b47efSMilan Broz */ 850393b47efSMilan Broz if (!io->base_io) 851393b47efSMilan Broz new_io->base_io = io; 852393b47efSMilan Broz else { 853393b47efSMilan Broz new_io->base_io = io->base_io; 854393b47efSMilan Broz crypt_inc_pending(io->base_io); 855393b47efSMilan Broz crypt_dec_pending(io); 856393b47efSMilan Broz } 857393b47efSMilan Broz 858393b47efSMilan Broz io = new_io; 859393b47efSMilan Broz } 8608b004457SMilan Broz } 861899c95d3SMilan Broz 862899c95d3SMilan Broz crypt_dec_pending(io); 86384131db6SMilan Broz } 86484131db6SMilan Broz 8654e4eef64SMilan Broz static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error) 8665742fd77SMilan Broz { 8675742fd77SMilan Broz if (unlikely(error < 0)) 8685742fd77SMilan Broz io->error = -EIO; 8695742fd77SMilan Broz 8705742fd77SMilan Broz crypt_dec_pending(io); 8715742fd77SMilan Broz } 8725742fd77SMilan Broz 8734e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 8748b004457SMilan Broz { 8758b004457SMilan Broz struct crypt_config *cc = io->target->private; 8765742fd77SMilan Broz int r = 0; 8778b004457SMilan Broz 8783e1a8bddSMilan Broz crypt_inc_pending(io); 8793a7f6c99SMilan Broz 88053017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 8810c395b0fSMilan Broz io->sector); 8828b004457SMilan Broz 8835742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 8845742fd77SMilan Broz 8853f1e9070SMilan Broz if (atomic_dec_and_test(&io->ctx.pending)) 8864e4eef64SMilan Broz kcryptd_crypt_read_done(io, r); 8873a7f6c99SMilan Broz 8883a7f6c99SMilan Broz crypt_dec_pending(io); 8898b004457SMilan Broz } 8908b004457SMilan Broz 89195497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 89295497a96SMilan Broz int error) 89395497a96SMilan Broz { 894b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 895b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 89695497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 89795497a96SMilan Broz struct crypt_config *cc = io->target->private; 89895497a96SMilan Broz 89995497a96SMilan Broz if (error == -EINPROGRESS) { 90095497a96SMilan Broz complete(&ctx->restart); 90195497a96SMilan Broz return; 90295497a96SMilan Broz } 90395497a96SMilan Broz 904b2174eebSHuang Ying mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool); 90595497a96SMilan Broz 90695497a96SMilan Broz if (!atomic_dec_and_test(&ctx->pending)) 90795497a96SMilan Broz return; 90895497a96SMilan Broz 90995497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 91095497a96SMilan Broz kcryptd_crypt_read_done(io, error); 91195497a96SMilan Broz else 91295497a96SMilan Broz kcryptd_crypt_write_io_submit(io, error, 1); 91395497a96SMilan Broz } 91495497a96SMilan Broz 9154e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 9164e4eef64SMilan Broz { 9174e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 9184e4eef64SMilan Broz 9194e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 9204e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 9214e4eef64SMilan Broz else 9224e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 9238b004457SMilan Broz } 9248b004457SMilan Broz 925395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 926395b167cSAlasdair G Kergon { 927395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 928395b167cSAlasdair G Kergon 929395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 930395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 931395b167cSAlasdair G Kergon } 932395b167cSAlasdair G Kergon 9331da177e4SLinus Torvalds /* 9341da177e4SLinus Torvalds * Decode key from its hex representation 9351da177e4SLinus Torvalds */ 9361da177e4SLinus Torvalds static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 9371da177e4SLinus Torvalds { 9381da177e4SLinus Torvalds char buffer[3]; 9391da177e4SLinus Torvalds char *endp; 9401da177e4SLinus Torvalds unsigned int i; 9411da177e4SLinus Torvalds 9421da177e4SLinus Torvalds buffer[2] = '\0'; 9431da177e4SLinus Torvalds 9441da177e4SLinus Torvalds for (i = 0; i < size; i++) { 9451da177e4SLinus Torvalds buffer[0] = *hex++; 9461da177e4SLinus Torvalds buffer[1] = *hex++; 9471da177e4SLinus Torvalds 9481da177e4SLinus Torvalds key[i] = (u8)simple_strtoul(buffer, &endp, 16); 9491da177e4SLinus Torvalds 9501da177e4SLinus Torvalds if (endp != &buffer[2]) 9511da177e4SLinus Torvalds return -EINVAL; 9521da177e4SLinus Torvalds } 9531da177e4SLinus Torvalds 9541da177e4SLinus Torvalds if (*hex != '\0') 9551da177e4SLinus Torvalds return -EINVAL; 9561da177e4SLinus Torvalds 9571da177e4SLinus Torvalds return 0; 9581da177e4SLinus Torvalds } 9591da177e4SLinus Torvalds 9601da177e4SLinus Torvalds /* 9611da177e4SLinus Torvalds * Encode key into its hex representation 9621da177e4SLinus Torvalds */ 9631da177e4SLinus Torvalds static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 9641da177e4SLinus Torvalds { 9651da177e4SLinus Torvalds unsigned int i; 9661da177e4SLinus Torvalds 9671da177e4SLinus Torvalds for (i = 0; i < size; i++) { 9681da177e4SLinus Torvalds sprintf(hex, "%02x", *key); 9691da177e4SLinus Torvalds hex += 2; 9701da177e4SLinus Torvalds key++; 9711da177e4SLinus Torvalds } 9721da177e4SLinus Torvalds } 9731da177e4SLinus Torvalds 974e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 975e48d4bbfSMilan Broz { 976e48d4bbfSMilan Broz unsigned key_size = strlen(key) >> 1; 977e48d4bbfSMilan Broz 978e48d4bbfSMilan Broz if (cc->key_size && cc->key_size != key_size) 979e48d4bbfSMilan Broz return -EINVAL; 980e48d4bbfSMilan Broz 981e48d4bbfSMilan Broz cc->key_size = key_size; /* initial settings */ 982e48d4bbfSMilan Broz 983e48d4bbfSMilan Broz if ((!key_size && strcmp(key, "-")) || 984e48d4bbfSMilan Broz (key_size && crypt_decode_key(cc->key, key, key_size) < 0)) 985e48d4bbfSMilan Broz return -EINVAL; 986e48d4bbfSMilan Broz 987e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 988e48d4bbfSMilan Broz 9890b430958SMilan Broz return crypto_ablkcipher_setkey(cc->tfm, cc->key, cc->key_size); 990e48d4bbfSMilan Broz } 991e48d4bbfSMilan Broz 992e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 993e48d4bbfSMilan Broz { 994e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 995e48d4bbfSMilan Broz memset(&cc->key, 0, cc->key_size * sizeof(u8)); 9960b430958SMilan Broz return crypto_ablkcipher_setkey(cc->tfm, cc->key, cc->key_size); 997e48d4bbfSMilan Broz } 998e48d4bbfSMilan Broz 99928513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 100028513fccSMilan Broz { 100128513fccSMilan Broz struct crypt_config *cc = ti->private; 100228513fccSMilan Broz 100328513fccSMilan Broz ti->private = NULL; 100428513fccSMilan Broz 100528513fccSMilan Broz if (!cc) 100628513fccSMilan Broz return; 100728513fccSMilan Broz 100828513fccSMilan Broz if (cc->io_queue) 100928513fccSMilan Broz destroy_workqueue(cc->io_queue); 101028513fccSMilan Broz if (cc->crypt_queue) 101128513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 101228513fccSMilan Broz 101328513fccSMilan Broz if (cc->bs) 101428513fccSMilan Broz bioset_free(cc->bs); 101528513fccSMilan Broz 101628513fccSMilan Broz if (cc->page_pool) 101728513fccSMilan Broz mempool_destroy(cc->page_pool); 101828513fccSMilan Broz if (cc->req_pool) 101928513fccSMilan Broz mempool_destroy(cc->req_pool); 102028513fccSMilan Broz if (cc->io_pool) 102128513fccSMilan Broz mempool_destroy(cc->io_pool); 102228513fccSMilan Broz 102328513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 102428513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 102528513fccSMilan Broz 102628513fccSMilan Broz if (cc->tfm && !IS_ERR(cc->tfm)) 102728513fccSMilan Broz crypto_free_ablkcipher(cc->tfm); 102828513fccSMilan Broz 102928513fccSMilan Broz if (cc->dev) 103028513fccSMilan Broz dm_put_device(ti, cc->dev); 103128513fccSMilan Broz 10325ebaee6dSMilan Broz kzfree(cc->cipher); 10335ebaee6dSMilan Broz kzfree(cc->cipher_mode); 103428513fccSMilan Broz 103528513fccSMilan Broz /* Must zero key material before freeing */ 103628513fccSMilan Broz kzfree(cc); 103728513fccSMilan Broz } 103828513fccSMilan Broz 10395ebaee6dSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, 10405ebaee6dSMilan Broz char *cipher_in, char *key) 10411da177e4SLinus Torvalds { 10425ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 10435ebaee6dSMilan Broz char *tmp, *cipher, *chainmode, *ivmode, *ivopts; 10445ebaee6dSMilan Broz char *cipher_api = NULL; 104528513fccSMilan Broz int ret = -EINVAL; 10461da177e4SLinus Torvalds 10475ebaee6dSMilan Broz /* Convert to crypto api definition? */ 10485ebaee6dSMilan Broz if (strchr(cipher_in, '(')) { 10495ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 10501da177e4SLinus Torvalds return -EINVAL; 10511da177e4SLinus Torvalds } 10521da177e4SLinus Torvalds 10535ebaee6dSMilan Broz /* 10545ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 10555ebaee6dSMilan Broz * cipher-mode-iv:ivopts 10565ebaee6dSMilan Broz */ 10575ebaee6dSMilan Broz tmp = cipher_in; 10581da177e4SLinus Torvalds cipher = strsep(&tmp, "-"); 10595ebaee6dSMilan Broz 10605ebaee6dSMilan Broz cc->cipher = kstrdup(cipher, GFP_KERNEL); 10615ebaee6dSMilan Broz if (!cc->cipher) 10625ebaee6dSMilan Broz goto bad_mem; 10635ebaee6dSMilan Broz 10645ebaee6dSMilan Broz if (tmp) { 10655ebaee6dSMilan Broz cc->cipher_mode = kstrdup(tmp, GFP_KERNEL); 10665ebaee6dSMilan Broz if (!cc->cipher_mode) 10675ebaee6dSMilan Broz goto bad_mem; 10685ebaee6dSMilan Broz } 10695ebaee6dSMilan Broz 10701da177e4SLinus Torvalds chainmode = strsep(&tmp, "-"); 10711da177e4SLinus Torvalds ivopts = strsep(&tmp, "-"); 10721da177e4SLinus Torvalds ivmode = strsep(&ivopts, ":"); 10731da177e4SLinus Torvalds 10741da177e4SLinus Torvalds if (tmp) 10755ebaee6dSMilan Broz DMWARN("Ignoring unexpected additional cipher options"); 10761da177e4SLinus Torvalds 10775ebaee6dSMilan Broz /* Compatibility mode for old dm-crypt mappings */ 10785ebaee6dSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) { 10795ebaee6dSMilan Broz kfree(cc->cipher_mode); 10805ebaee6dSMilan Broz cc->cipher_mode = kstrdup("cbc-plain", GFP_KERNEL); 10811da177e4SLinus Torvalds chainmode = "cbc"; 10821da177e4SLinus Torvalds ivmode = "plain"; 10831da177e4SLinus Torvalds } 10841da177e4SLinus Torvalds 1085d1806f6aSHerbert Xu if (strcmp(chainmode, "ecb") && !ivmode) { 10865ebaee6dSMilan Broz ti->error = "IV mechanism required"; 10875ebaee6dSMilan Broz return -EINVAL; 10881da177e4SLinus Torvalds } 10891da177e4SLinus Torvalds 10905ebaee6dSMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 10915ebaee6dSMilan Broz if (!cipher_api) 10925ebaee6dSMilan Broz goto bad_mem; 10935ebaee6dSMilan Broz 10945ebaee6dSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 10955ebaee6dSMilan Broz "%s(%s)", chainmode, cipher); 10965ebaee6dSMilan Broz if (ret < 0) { 10975ebaee6dSMilan Broz kfree(cipher_api); 10985ebaee6dSMilan Broz goto bad_mem; 1099d1806f6aSHerbert Xu } 1100d1806f6aSHerbert Xu 11015ebaee6dSMilan Broz /* Allocate cipher */ 11025ebaee6dSMilan Broz cc->tfm = crypto_alloc_ablkcipher(cipher_api, 0, 0); 110328513fccSMilan Broz if (IS_ERR(cc->tfm)) { 11045ebaee6dSMilan Broz ret = PTR_ERR(cc->tfm); 110572d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm"; 110628513fccSMilan Broz goto bad; 11071da177e4SLinus Torvalds } 11081da177e4SLinus Torvalds 11095ebaee6dSMilan Broz /* Initialize and set key */ 11105ebaee6dSMilan Broz ret = crypt_set_key(cc, key); 111128513fccSMilan Broz if (ret < 0) { 11120b430958SMilan Broz ti->error = "Error decoding and setting key"; 111328513fccSMilan Broz goto bad; 11140b430958SMilan Broz } 11150b430958SMilan Broz 11165ebaee6dSMilan Broz /* Initialize IV */ 11175ebaee6dSMilan Broz cc->iv_size = crypto_ablkcipher_ivsize(cc->tfm); 11185ebaee6dSMilan Broz if (cc->iv_size) 11195ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 11205ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 11215ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 11225ebaee6dSMilan Broz else if (ivmode) { 11235ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 11245ebaee6dSMilan Broz ivmode = NULL; 11255ebaee6dSMilan Broz } 11265ebaee6dSMilan Broz 11275ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 11281da177e4SLinus Torvalds if (ivmode == NULL) 11291da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 11301da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 11311da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 113261afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 113361afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 11341da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 11351da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 113648527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 113748527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 113846b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 113946b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 11401da177e4SLinus Torvalds else { 11415ebaee6dSMilan Broz ret = -EINVAL; 114272d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 114328513fccSMilan Broz goto bad; 11441da177e4SLinus Torvalds } 11451da177e4SLinus Torvalds 114628513fccSMilan Broz /* Allocate IV */ 114728513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 114828513fccSMilan Broz ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 114928513fccSMilan Broz if (ret < 0) { 115028513fccSMilan Broz ti->error = "Error creating IV"; 115128513fccSMilan Broz goto bad; 115228513fccSMilan Broz } 115328513fccSMilan Broz } 11541da177e4SLinus Torvalds 115528513fccSMilan Broz /* Initialize IV (set keys for ESSIV etc) */ 115628513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 115728513fccSMilan Broz ret = cc->iv_gen_ops->init(cc); 115828513fccSMilan Broz if (ret < 0) { 1159b95bf2d3SMilan Broz ti->error = "Error initialising IV"; 116028513fccSMilan Broz goto bad; 116128513fccSMilan Broz } 1162b95bf2d3SMilan Broz } 1163b95bf2d3SMilan Broz 11645ebaee6dSMilan Broz ret = 0; 11655ebaee6dSMilan Broz bad: 11665ebaee6dSMilan Broz kfree(cipher_api); 11675ebaee6dSMilan Broz return ret; 11685ebaee6dSMilan Broz 11695ebaee6dSMilan Broz bad_mem: 11705ebaee6dSMilan Broz ti->error = "Cannot allocate cipher strings"; 11715ebaee6dSMilan Broz return -ENOMEM; 11721da177e4SLinus Torvalds } 11735ebaee6dSMilan Broz 11745ebaee6dSMilan Broz /* 11755ebaee6dSMilan Broz * Construct an encryption mapping: 11765ebaee6dSMilan Broz * <cipher> <key> <iv_offset> <dev_path> <start> 11775ebaee6dSMilan Broz */ 11785ebaee6dSMilan Broz static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 11795ebaee6dSMilan Broz { 11805ebaee6dSMilan Broz struct crypt_config *cc; 11815ebaee6dSMilan Broz unsigned int key_size; 11825ebaee6dSMilan Broz unsigned long long tmpll; 11835ebaee6dSMilan Broz int ret; 11845ebaee6dSMilan Broz 11855ebaee6dSMilan Broz if (argc != 5) { 11865ebaee6dSMilan Broz ti->error = "Not enough arguments"; 11875ebaee6dSMilan Broz return -EINVAL; 11881da177e4SLinus Torvalds } 11891da177e4SLinus Torvalds 11905ebaee6dSMilan Broz key_size = strlen(argv[1]) >> 1; 11915ebaee6dSMilan Broz 11925ebaee6dSMilan Broz cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 11935ebaee6dSMilan Broz if (!cc) { 11945ebaee6dSMilan Broz ti->error = "Cannot allocate encryption context"; 11955ebaee6dSMilan Broz return -ENOMEM; 11965ebaee6dSMilan Broz } 11975ebaee6dSMilan Broz 11985ebaee6dSMilan Broz ti->private = cc; 11995ebaee6dSMilan Broz ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 12005ebaee6dSMilan Broz if (ret < 0) 12015ebaee6dSMilan Broz goto bad; 12025ebaee6dSMilan Broz 120328513fccSMilan Broz ret = -ENOMEM; 120493d2341cSMatthew Dobson cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 12051da177e4SLinus Torvalds if (!cc->io_pool) { 120672d94861SAlasdair G Kergon ti->error = "Cannot allocate crypt io mempool"; 120728513fccSMilan Broz goto bad; 12081da177e4SLinus Torvalds } 12091da177e4SLinus Torvalds 1210ddd42edfSMilan Broz cc->dmreq_start = sizeof(struct ablkcipher_request); 121128513fccSMilan Broz cc->dmreq_start += crypto_ablkcipher_reqsize(cc->tfm); 1212ddd42edfSMilan Broz cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment()); 121328513fccSMilan Broz cc->dmreq_start += crypto_ablkcipher_alignmask(cc->tfm) & 12143a7f6c99SMilan Broz ~(crypto_tfm_ctx_alignment() - 1); 1215ddd42edfSMilan Broz 1216ddd42edfSMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1217ddd42edfSMilan Broz sizeof(struct dm_crypt_request) + cc->iv_size); 1218ddd42edfSMilan Broz if (!cc->req_pool) { 1219ddd42edfSMilan Broz ti->error = "Cannot allocate crypt request mempool"; 122028513fccSMilan Broz goto bad; 1221ddd42edfSMilan Broz } 1222ddd42edfSMilan Broz cc->req = NULL; 1223ddd42edfSMilan Broz 1224a19b27ceSMatthew Dobson cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 12251da177e4SLinus Torvalds if (!cc->page_pool) { 122672d94861SAlasdair G Kergon ti->error = "Cannot allocate page mempool"; 122728513fccSMilan Broz goto bad; 12281da177e4SLinus Torvalds } 12291da177e4SLinus Torvalds 1230bb799ca0SJens Axboe cc->bs = bioset_create(MIN_IOS, 0); 12316a24c718SMilan Broz if (!cc->bs) { 12326a24c718SMilan Broz ti->error = "Cannot allocate crypt bioset"; 123328513fccSMilan Broz goto bad; 12346a24c718SMilan Broz } 12356a24c718SMilan Broz 123628513fccSMilan Broz ret = -EINVAL; 12374ee218cdSAndrew Morton if (sscanf(argv[2], "%llu", &tmpll) != 1) { 123872d94861SAlasdair G Kergon ti->error = "Invalid iv_offset sector"; 123928513fccSMilan Broz goto bad; 12401da177e4SLinus Torvalds } 12414ee218cdSAndrew Morton cc->iv_offset = tmpll; 12421da177e4SLinus Torvalds 124328513fccSMilan Broz if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev)) { 124428513fccSMilan Broz ti->error = "Device lookup failed"; 124528513fccSMilan Broz goto bad; 124628513fccSMilan Broz } 124728513fccSMilan Broz 12484ee218cdSAndrew Morton if (sscanf(argv[4], "%llu", &tmpll) != 1) { 124972d94861SAlasdair G Kergon ti->error = "Invalid device sector"; 125028513fccSMilan Broz goto bad; 12511da177e4SLinus Torvalds } 12524ee218cdSAndrew Morton cc->start = tmpll; 12531da177e4SLinus Torvalds 125428513fccSMilan Broz ret = -ENOMEM; 1255cabf08e4SMilan Broz cc->io_queue = create_singlethread_workqueue("kcryptd_io"); 1256cabf08e4SMilan Broz if (!cc->io_queue) { 1257cabf08e4SMilan Broz ti->error = "Couldn't create kcryptd io queue"; 125828513fccSMilan Broz goto bad; 1259cabf08e4SMilan Broz } 1260cabf08e4SMilan Broz 1261cabf08e4SMilan Broz cc->crypt_queue = create_singlethread_workqueue("kcryptd"); 1262cabf08e4SMilan Broz if (!cc->crypt_queue) { 12639934a8beSMilan Broz ti->error = "Couldn't create kcryptd queue"; 126428513fccSMilan Broz goto bad; 12659934a8beSMilan Broz } 12669934a8beSMilan Broz 1267647c7db1SMikulas Patocka ti->num_flush_requests = 1; 12681da177e4SLinus Torvalds return 0; 12691da177e4SLinus Torvalds 127028513fccSMilan Broz bad: 127128513fccSMilan Broz crypt_dtr(ti); 127228513fccSMilan Broz return ret; 12731da177e4SLinus Torvalds } 12741da177e4SLinus Torvalds 12751da177e4SLinus Torvalds static int crypt_map(struct dm_target *ti, struct bio *bio, 12761da177e4SLinus Torvalds union map_info *map_context) 12771da177e4SLinus Torvalds { 1278028867acSAlasdair G Kergon struct dm_crypt_io *io; 1279647c7db1SMikulas Patocka struct crypt_config *cc; 1280647c7db1SMikulas Patocka 1281647c7db1SMikulas Patocka if (unlikely(bio_empty_barrier(bio))) { 1282647c7db1SMikulas Patocka cc = ti->private; 1283647c7db1SMikulas Patocka bio->bi_bdev = cc->dev->bdev; 1284647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 1285647c7db1SMikulas Patocka } 12861da177e4SLinus Torvalds 1287b441a262SAlasdair G Kergon io = crypt_io_alloc(ti, bio, dm_target_offset(ti, bio->bi_sector)); 1288cabf08e4SMilan Broz 1289cabf08e4SMilan Broz if (bio_data_dir(io->base_bio) == READ) 129023541d2dSMilan Broz kcryptd_queue_io(io); 1291cabf08e4SMilan Broz else 1292cabf08e4SMilan Broz kcryptd_queue_crypt(io); 12931da177e4SLinus Torvalds 1294d2a7ad29SKiyoshi Ueda return DM_MAPIO_SUBMITTED; 12951da177e4SLinus Torvalds } 12961da177e4SLinus Torvalds 12971da177e4SLinus Torvalds static int crypt_status(struct dm_target *ti, status_type_t type, 12981da177e4SLinus Torvalds char *result, unsigned int maxlen) 12991da177e4SLinus Torvalds { 13005ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 13011da177e4SLinus Torvalds unsigned int sz = 0; 13021da177e4SLinus Torvalds 13031da177e4SLinus Torvalds switch (type) { 13041da177e4SLinus Torvalds case STATUSTYPE_INFO: 13051da177e4SLinus Torvalds result[0] = '\0'; 13061da177e4SLinus Torvalds break; 13071da177e4SLinus Torvalds 13081da177e4SLinus Torvalds case STATUSTYPE_TABLE: 13095ebaee6dSMilan Broz if (cc->cipher_mode) 13105ebaee6dSMilan Broz DMEMIT("%s-%s ", cc->cipher, cc->cipher_mode); 13111da177e4SLinus Torvalds else 13125ebaee6dSMilan Broz DMEMIT("%s ", cc->cipher); 13131da177e4SLinus Torvalds 13141da177e4SLinus Torvalds if (cc->key_size > 0) { 13151da177e4SLinus Torvalds if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 13161da177e4SLinus Torvalds return -ENOMEM; 13171da177e4SLinus Torvalds 13181da177e4SLinus Torvalds crypt_encode_key(result + sz, cc->key, cc->key_size); 13191da177e4SLinus Torvalds sz += cc->key_size << 1; 13201da177e4SLinus Torvalds } else { 13211da177e4SLinus Torvalds if (sz >= maxlen) 13221da177e4SLinus Torvalds return -ENOMEM; 13231da177e4SLinus Torvalds result[sz++] = '-'; 13241da177e4SLinus Torvalds } 13251da177e4SLinus Torvalds 13264ee218cdSAndrew Morton DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 13274ee218cdSAndrew Morton cc->dev->name, (unsigned long long)cc->start); 13281da177e4SLinus Torvalds break; 13291da177e4SLinus Torvalds } 13301da177e4SLinus Torvalds return 0; 13311da177e4SLinus Torvalds } 13321da177e4SLinus Torvalds 1333e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 1334e48d4bbfSMilan Broz { 1335e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1336e48d4bbfSMilan Broz 1337e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1338e48d4bbfSMilan Broz } 1339e48d4bbfSMilan Broz 1340e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 1341e48d4bbfSMilan Broz { 1342e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1343e48d4bbfSMilan Broz 1344e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1345e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 1346e48d4bbfSMilan Broz return -EAGAIN; 1347e48d4bbfSMilan Broz } 1348e48d4bbfSMilan Broz 1349e48d4bbfSMilan Broz return 0; 1350e48d4bbfSMilan Broz } 1351e48d4bbfSMilan Broz 1352e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 1353e48d4bbfSMilan Broz { 1354e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1355e48d4bbfSMilan Broz 1356e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1357e48d4bbfSMilan Broz } 1358e48d4bbfSMilan Broz 1359e48d4bbfSMilan Broz /* Message interface 1360e48d4bbfSMilan Broz * key set <key> 1361e48d4bbfSMilan Broz * key wipe 1362e48d4bbfSMilan Broz */ 1363e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1364e48d4bbfSMilan Broz { 1365e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1366542da317SMilan Broz int ret = -EINVAL; 1367e48d4bbfSMilan Broz 1368e48d4bbfSMilan Broz if (argc < 2) 1369e48d4bbfSMilan Broz goto error; 1370e48d4bbfSMilan Broz 1371e48d4bbfSMilan Broz if (!strnicmp(argv[0], MESG_STR("key"))) { 1372e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1373e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 1374e48d4bbfSMilan Broz return -EINVAL; 1375e48d4bbfSMilan Broz } 1376542da317SMilan Broz if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) { 1377542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 1378542da317SMilan Broz if (ret) 1379542da317SMilan Broz return ret; 1380542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 1381542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 1382542da317SMilan Broz return ret; 1383542da317SMilan Broz } 1384542da317SMilan Broz if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) { 1385542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 1386542da317SMilan Broz ret = cc->iv_gen_ops->wipe(cc); 1387542da317SMilan Broz if (ret) 1388542da317SMilan Broz return ret; 1389542da317SMilan Broz } 1390e48d4bbfSMilan Broz return crypt_wipe_key(cc); 1391e48d4bbfSMilan Broz } 1392542da317SMilan Broz } 1393e48d4bbfSMilan Broz 1394e48d4bbfSMilan Broz error: 1395e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 1396e48d4bbfSMilan Broz return -EINVAL; 1397e48d4bbfSMilan Broz } 1398e48d4bbfSMilan Broz 1399d41e26b9SMilan Broz static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 1400d41e26b9SMilan Broz struct bio_vec *biovec, int max_size) 1401d41e26b9SMilan Broz { 1402d41e26b9SMilan Broz struct crypt_config *cc = ti->private; 1403d41e26b9SMilan Broz struct request_queue *q = bdev_get_queue(cc->dev->bdev); 1404d41e26b9SMilan Broz 1405d41e26b9SMilan Broz if (!q->merge_bvec_fn) 1406d41e26b9SMilan Broz return max_size; 1407d41e26b9SMilan Broz 1408d41e26b9SMilan Broz bvm->bi_bdev = cc->dev->bdev; 1409b441a262SAlasdair G Kergon bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector); 1410d41e26b9SMilan Broz 1411d41e26b9SMilan Broz return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 1412d41e26b9SMilan Broz } 1413d41e26b9SMilan Broz 1414af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 1415af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 1416af4874e0SMike Snitzer { 1417af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 1418af4874e0SMike Snitzer 14195dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 1420af4874e0SMike Snitzer } 1421af4874e0SMike Snitzer 14221da177e4SLinus Torvalds static struct target_type crypt_target = { 14231da177e4SLinus Torvalds .name = "crypt", 1424af4874e0SMike Snitzer .version = {1, 7, 0}, 14251da177e4SLinus Torvalds .module = THIS_MODULE, 14261da177e4SLinus Torvalds .ctr = crypt_ctr, 14271da177e4SLinus Torvalds .dtr = crypt_dtr, 14281da177e4SLinus Torvalds .map = crypt_map, 14291da177e4SLinus Torvalds .status = crypt_status, 1430e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 1431e48d4bbfSMilan Broz .preresume = crypt_preresume, 1432e48d4bbfSMilan Broz .resume = crypt_resume, 1433e48d4bbfSMilan Broz .message = crypt_message, 1434d41e26b9SMilan Broz .merge = crypt_merge, 1435af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 14361da177e4SLinus Torvalds }; 14371da177e4SLinus Torvalds 14381da177e4SLinus Torvalds static int __init dm_crypt_init(void) 14391da177e4SLinus Torvalds { 14401da177e4SLinus Torvalds int r; 14411da177e4SLinus Torvalds 1442028867acSAlasdair G Kergon _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 14431da177e4SLinus Torvalds if (!_crypt_io_pool) 14441da177e4SLinus Torvalds return -ENOMEM; 14451da177e4SLinus Torvalds 14461da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 14471da177e4SLinus Torvalds if (r < 0) { 144872d94861SAlasdair G Kergon DMERR("register failed %d", r); 14499934a8beSMilan Broz kmem_cache_destroy(_crypt_io_pool); 14501da177e4SLinus Torvalds } 14511da177e4SLinus Torvalds 14521da177e4SLinus Torvalds return r; 14531da177e4SLinus Torvalds } 14541da177e4SLinus Torvalds 14551da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 14561da177e4SLinus Torvalds { 145710d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 14581da177e4SLinus Torvalds kmem_cache_destroy(_crypt_io_pool); 14591da177e4SLinus Torvalds } 14601da177e4SLinus Torvalds 14611da177e4SLinus Torvalds module_init(dm_crypt_init); 14621da177e4SLinus Torvalds module_exit(dm_crypt_exit); 14631da177e4SLinus Torvalds 14641da177e4SLinus Torvalds MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 14651da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 14661da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1467