11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4542da317SMilan Broz * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 943d69034SMilan Broz #include <linux/completion.h> 10d1806f6aSHerbert Xu #include <linux/err.h> 111da177e4SLinus Torvalds #include <linux/module.h> 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/kernel.h> 141da177e4SLinus Torvalds #include <linux/bio.h> 151da177e4SLinus Torvalds #include <linux/blkdev.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/crypto.h> 191da177e4SLinus Torvalds #include <linux/workqueue.h> 203fcfab16SAndrew Morton #include <linux/backing-dev.h> 21c0297721SAndi Kleen #include <linux/percpu.h> 2260063497SArun Sharma #include <linux/atomic.h> 23378f058cSDavid Hardeman #include <linux/scatterlist.h> 241da177e4SLinus Torvalds #include <asm/page.h> 2548527fa7SRik Snel #include <asm/unaligned.h> 2634745785SMilan Broz #include <crypto/hash.h> 2734745785SMilan Broz #include <crypto/md5.h> 2834745785SMilan Broz #include <crypto/algapi.h> 291da177e4SLinus Torvalds 30586e80e6SMikulas Patocka #include <linux/device-mapper.h> 311da177e4SLinus Torvalds 3272d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 331da177e4SLinus Torvalds 341da177e4SLinus Torvalds /* 351da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 361da177e4SLinus Torvalds */ 371da177e4SLinus Torvalds struct convert_context { 3843d69034SMilan Broz struct completion restart; 391da177e4SLinus Torvalds struct bio *bio_in; 401da177e4SLinus Torvalds struct bio *bio_out; 411da177e4SLinus Torvalds unsigned int offset_in; 421da177e4SLinus Torvalds unsigned int offset_out; 431da177e4SLinus Torvalds unsigned int idx_in; 441da177e4SLinus Torvalds unsigned int idx_out; 451da177e4SLinus Torvalds sector_t sector; 4640b6229bSMikulas Patocka atomic_t cc_pending; 471da177e4SLinus Torvalds }; 481da177e4SLinus Torvalds 4953017030SMilan Broz /* 5053017030SMilan Broz * per bio private data 5153017030SMilan Broz */ 5253017030SMilan Broz struct dm_crypt_io { 5353017030SMilan Broz struct dm_target *target; 5453017030SMilan Broz struct bio *base_bio; 5553017030SMilan Broz struct work_struct work; 5653017030SMilan Broz 5753017030SMilan Broz struct convert_context ctx; 5853017030SMilan Broz 5940b6229bSMikulas Patocka atomic_t io_pending; 6053017030SMilan Broz int error; 610c395b0fSMilan Broz sector_t sector; 62393b47efSMilan Broz struct dm_crypt_io *base_io; 6353017030SMilan Broz }; 6453017030SMilan Broz 6501482b76SMilan Broz struct dm_crypt_request { 66b2174eebSHuang Ying struct convert_context *ctx; 6701482b76SMilan Broz struct scatterlist sg_in; 6801482b76SMilan Broz struct scatterlist sg_out; 692dc5327dSMilan Broz sector_t iv_sector; 7001482b76SMilan Broz }; 7101482b76SMilan Broz 721da177e4SLinus Torvalds struct crypt_config; 731da177e4SLinus Torvalds 741da177e4SLinus Torvalds struct crypt_iv_operations { 751da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 761da177e4SLinus Torvalds const char *opts); 771da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 78b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 79542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 802dc5327dSMilan Broz int (*generator)(struct crypt_config *cc, u8 *iv, 812dc5327dSMilan Broz struct dm_crypt_request *dmreq); 822dc5327dSMilan Broz int (*post)(struct crypt_config *cc, u8 *iv, 832dc5327dSMilan Broz struct dm_crypt_request *dmreq); 841da177e4SLinus Torvalds }; 851da177e4SLinus Torvalds 8660473592SMilan Broz struct iv_essiv_private { 87b95bf2d3SMilan Broz struct crypto_hash *hash_tfm; 88b95bf2d3SMilan Broz u8 *salt; 8960473592SMilan Broz }; 9060473592SMilan Broz 9160473592SMilan Broz struct iv_benbi_private { 9260473592SMilan Broz int shift; 9360473592SMilan Broz }; 9460473592SMilan Broz 9534745785SMilan Broz #define LMK_SEED_SIZE 64 /* hash + 0 */ 9634745785SMilan Broz struct iv_lmk_private { 9734745785SMilan Broz struct crypto_shash *hash_tfm; 9834745785SMilan Broz u8 *seed; 9934745785SMilan Broz }; 10034745785SMilan Broz 1011da177e4SLinus Torvalds /* 1021da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 1031da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 1041da177e4SLinus Torvalds */ 105e48d4bbfSMilan Broz enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 106c0297721SAndi Kleen 107c0297721SAndi Kleen /* 108c0297721SAndi Kleen * Duplicated per-CPU state for cipher. 109c0297721SAndi Kleen */ 110c0297721SAndi Kleen struct crypt_cpu { 111c0297721SAndi Kleen struct ablkcipher_request *req; 112c0297721SAndi Kleen }; 113c0297721SAndi Kleen 114c0297721SAndi Kleen /* 115c0297721SAndi Kleen * The fields in here must be read only after initialization, 116c0297721SAndi Kleen * changing state should be in crypt_cpu. 117c0297721SAndi Kleen */ 1181da177e4SLinus Torvalds struct crypt_config { 1191da177e4SLinus Torvalds struct dm_dev *dev; 1201da177e4SLinus Torvalds sector_t start; 1211da177e4SLinus Torvalds 1221da177e4SLinus Torvalds /* 123ddd42edfSMilan Broz * pool for per bio private data, crypto requests and 124ddd42edfSMilan Broz * encryption requeusts/buffer pages 1251da177e4SLinus Torvalds */ 1261da177e4SLinus Torvalds mempool_t *io_pool; 127ddd42edfSMilan Broz mempool_t *req_pool; 1281da177e4SLinus Torvalds mempool_t *page_pool; 1296a24c718SMilan Broz struct bio_set *bs; 1301da177e4SLinus Torvalds 131cabf08e4SMilan Broz struct workqueue_struct *io_queue; 132cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1333f1e9070SMilan Broz 1345ebaee6dSMilan Broz char *cipher; 1357dbcd137SMilan Broz char *cipher_string; 1365ebaee6dSMilan Broz 1371da177e4SLinus Torvalds struct crypt_iv_operations *iv_gen_ops; 13879066ad3SHerbert Xu union { 13960473592SMilan Broz struct iv_essiv_private essiv; 14060473592SMilan Broz struct iv_benbi_private benbi; 14134745785SMilan Broz struct iv_lmk_private lmk; 14279066ad3SHerbert Xu } iv_gen_private; 1431da177e4SLinus Torvalds sector_t iv_offset; 1441da177e4SLinus Torvalds unsigned int iv_size; 1451da177e4SLinus Torvalds 146ddd42edfSMilan Broz /* 147c0297721SAndi Kleen * Duplicated per cpu state. Access through 148c0297721SAndi Kleen * per_cpu_ptr() only. 149c0297721SAndi Kleen */ 150c0297721SAndi Kleen struct crypt_cpu __percpu *cpu; 151fd2d231fSMikulas Patocka 152fd2d231fSMikulas Patocka /* ESSIV: struct crypto_cipher *essiv_tfm */ 153fd2d231fSMikulas Patocka void *iv_private; 154fd2d231fSMikulas Patocka struct crypto_ablkcipher **tfms; 155d1f96423SMilan Broz unsigned tfms_count; 156c0297721SAndi Kleen 157c0297721SAndi Kleen /* 158ddd42edfSMilan Broz * Layout of each crypto request: 159ddd42edfSMilan Broz * 160ddd42edfSMilan Broz * struct ablkcipher_request 161ddd42edfSMilan Broz * context 162ddd42edfSMilan Broz * padding 163ddd42edfSMilan Broz * struct dm_crypt_request 164ddd42edfSMilan Broz * padding 165ddd42edfSMilan Broz * IV 166ddd42edfSMilan Broz * 167ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 168ddd42edfSMilan Broz * correctly aligned. 169ddd42edfSMilan Broz */ 170ddd42edfSMilan Broz unsigned int dmreq_start; 171ddd42edfSMilan Broz 172e48d4bbfSMilan Broz unsigned long flags; 1731da177e4SLinus Torvalds unsigned int key_size; 174d1f96423SMilan Broz unsigned int key_parts; 1751da177e4SLinus Torvalds u8 key[0]; 1761da177e4SLinus Torvalds }; 1771da177e4SLinus Torvalds 1786a24c718SMilan Broz #define MIN_IOS 16 1791da177e4SLinus Torvalds #define MIN_POOL_PAGES 32 1801da177e4SLinus Torvalds 181e18b890bSChristoph Lameter static struct kmem_cache *_crypt_io_pool; 1821da177e4SLinus Torvalds 183028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 184395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 1852dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq); 186027581f3SOlaf Kirch 187c0297721SAndi Kleen static struct crypt_cpu *this_crypt_config(struct crypt_config *cc) 188c0297721SAndi Kleen { 189c0297721SAndi Kleen return this_cpu_ptr(cc->cpu); 190c0297721SAndi Kleen } 191c0297721SAndi Kleen 192c0297721SAndi Kleen /* 193c0297721SAndi Kleen * Use this to access cipher attributes that are the same for each CPU. 194c0297721SAndi Kleen */ 195c0297721SAndi Kleen static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc) 196c0297721SAndi Kleen { 197fd2d231fSMikulas Patocka return cc->tfms[0]; 198c0297721SAndi Kleen } 199c0297721SAndi Kleen 2001da177e4SLinus Torvalds /* 2011da177e4SLinus Torvalds * Different IV generation algorithms: 2021da177e4SLinus Torvalds * 2033c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 2043a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 2051da177e4SLinus Torvalds * 20661afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 20761afef61SMilan Broz * number, padded with zeros if necessary. 20861afef61SMilan Broz * 2093c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 2101da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 2111da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 2121da177e4SLinus Torvalds * 21348527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 21448527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 21548527fa7SRik Snel * 21646b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 21746b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 21846b47730SLudwig Nussel * 21934745785SMilan Broz * lmk: Compatible implementation of the block chaining mode used 22034745785SMilan Broz * by the Loop-AES block device encryption system 22134745785SMilan Broz * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/ 22234745785SMilan Broz * It operates on full 512 byte sectors and uses CBC 22334745785SMilan Broz * with an IV derived from the sector number, the data and 22434745785SMilan Broz * optionally extra IV seed. 22534745785SMilan Broz * This means that after decryption the first block 22634745785SMilan Broz * of sector must be tweaked according to decrypted data. 22734745785SMilan Broz * Loop-AES can use three encryption schemes: 22834745785SMilan Broz * version 1: is plain aes-cbc mode 22934745785SMilan Broz * version 2: uses 64 multikey scheme with lmk IV generator 23034745785SMilan Broz * version 3: the same as version 2 with additional IV seed 23134745785SMilan Broz * (it uses 65 keys, last key is used as IV seed) 23234745785SMilan Broz * 2331da177e4SLinus Torvalds * plumb: unimplemented, see: 2341da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 2351da177e4SLinus Torvalds */ 2361da177e4SLinus Torvalds 2372dc5327dSMilan Broz static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, 2382dc5327dSMilan Broz struct dm_crypt_request *dmreq) 2391da177e4SLinus Torvalds { 2401da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 241283a8328SAlasdair G Kergon *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff); 2421da177e4SLinus Torvalds 2431da177e4SLinus Torvalds return 0; 2441da177e4SLinus Torvalds } 2451da177e4SLinus Torvalds 24661afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 2472dc5327dSMilan Broz struct dm_crypt_request *dmreq) 24861afef61SMilan Broz { 24961afef61SMilan Broz memset(iv, 0, cc->iv_size); 250283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 25161afef61SMilan Broz 25261afef61SMilan Broz return 0; 25361afef61SMilan Broz } 25461afef61SMilan Broz 255b95bf2d3SMilan Broz /* Initialise ESSIV - compute salt but no local memory allocations */ 256b95bf2d3SMilan Broz static int crypt_iv_essiv_init(struct crypt_config *cc) 257b95bf2d3SMilan Broz { 258b95bf2d3SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 259b95bf2d3SMilan Broz struct hash_desc desc; 260b95bf2d3SMilan Broz struct scatterlist sg; 261c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 262fd2d231fSMikulas Patocka int err; 263b95bf2d3SMilan Broz 264b95bf2d3SMilan Broz sg_init_one(&sg, cc->key, cc->key_size); 265b95bf2d3SMilan Broz desc.tfm = essiv->hash_tfm; 266b95bf2d3SMilan Broz desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 267b95bf2d3SMilan Broz 268b95bf2d3SMilan Broz err = crypto_hash_digest(&desc, &sg, cc->key_size, essiv->salt); 269b95bf2d3SMilan Broz if (err) 270b95bf2d3SMilan Broz return err; 271b95bf2d3SMilan Broz 272fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 273c0297721SAndi Kleen 274c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, essiv->salt, 275b95bf2d3SMilan Broz crypto_hash_digestsize(essiv->hash_tfm)); 276c0297721SAndi Kleen if (err) 277c0297721SAndi Kleen return err; 278c0297721SAndi Kleen 279c0297721SAndi Kleen return 0; 280b95bf2d3SMilan Broz } 281b95bf2d3SMilan Broz 282542da317SMilan Broz /* Wipe salt and reset key derived from volume key */ 283542da317SMilan Broz static int crypt_iv_essiv_wipe(struct crypt_config *cc) 284542da317SMilan Broz { 285542da317SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 286542da317SMilan Broz unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm); 287c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 288fd2d231fSMikulas Patocka int r, err = 0; 289542da317SMilan Broz 290542da317SMilan Broz memset(essiv->salt, 0, salt_size); 291542da317SMilan Broz 292fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 293c0297721SAndi Kleen r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size); 294c0297721SAndi Kleen if (r) 295c0297721SAndi Kleen err = r; 296c0297721SAndi Kleen 297c0297721SAndi Kleen return err; 298c0297721SAndi Kleen } 299c0297721SAndi Kleen 300c0297721SAndi Kleen /* Set up per cpu cipher state */ 301c0297721SAndi Kleen static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc, 302c0297721SAndi Kleen struct dm_target *ti, 303c0297721SAndi Kleen u8 *salt, unsigned saltsize) 304c0297721SAndi Kleen { 305c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 306c0297721SAndi Kleen int err; 307c0297721SAndi Kleen 308c0297721SAndi Kleen /* Setup the essiv_tfm with the given salt */ 309c0297721SAndi Kleen essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 310c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 311c0297721SAndi Kleen ti->error = "Error allocating crypto tfm for ESSIV"; 312c0297721SAndi Kleen return essiv_tfm; 313c0297721SAndi Kleen } 314c0297721SAndi Kleen 315c0297721SAndi Kleen if (crypto_cipher_blocksize(essiv_tfm) != 316c0297721SAndi Kleen crypto_ablkcipher_ivsize(any_tfm(cc))) { 317c0297721SAndi Kleen ti->error = "Block size of ESSIV cipher does " 318c0297721SAndi Kleen "not match IV size of block cipher"; 319c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 320c0297721SAndi Kleen return ERR_PTR(-EINVAL); 321c0297721SAndi Kleen } 322c0297721SAndi Kleen 323c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 324c0297721SAndi Kleen if (err) { 325c0297721SAndi Kleen ti->error = "Failed to set key for ESSIV cipher"; 326c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 327c0297721SAndi Kleen return ERR_PTR(err); 328c0297721SAndi Kleen } 329c0297721SAndi Kleen 330c0297721SAndi Kleen return essiv_tfm; 331542da317SMilan Broz } 332542da317SMilan Broz 33360473592SMilan Broz static void crypt_iv_essiv_dtr(struct crypt_config *cc) 33460473592SMilan Broz { 335c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 33660473592SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 33760473592SMilan Broz 338b95bf2d3SMilan Broz crypto_free_hash(essiv->hash_tfm); 339b95bf2d3SMilan Broz essiv->hash_tfm = NULL; 340b95bf2d3SMilan Broz 341b95bf2d3SMilan Broz kzfree(essiv->salt); 342b95bf2d3SMilan Broz essiv->salt = NULL; 343c0297721SAndi Kleen 344fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 345c0297721SAndi Kleen 346c0297721SAndi Kleen if (essiv_tfm) 347c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 348c0297721SAndi Kleen 349fd2d231fSMikulas Patocka cc->iv_private = NULL; 35060473592SMilan Broz } 35160473592SMilan Broz 3521da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 3531da177e4SLinus Torvalds const char *opts) 3541da177e4SLinus Torvalds { 3555861f1beSMilan Broz struct crypto_cipher *essiv_tfm = NULL; 3565861f1beSMilan Broz struct crypto_hash *hash_tfm = NULL; 3575861f1beSMilan Broz u8 *salt = NULL; 358fd2d231fSMikulas Patocka int err; 3591da177e4SLinus Torvalds 3605861f1beSMilan Broz if (!opts) { 36172d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 3621da177e4SLinus Torvalds return -EINVAL; 3631da177e4SLinus Torvalds } 3641da177e4SLinus Torvalds 365b95bf2d3SMilan Broz /* Allocate hash algorithm */ 36635058687SHerbert Xu hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 36735058687SHerbert Xu if (IS_ERR(hash_tfm)) { 36872d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 3695861f1beSMilan Broz err = PTR_ERR(hash_tfm); 3705861f1beSMilan Broz goto bad; 3711da177e4SLinus Torvalds } 3721da177e4SLinus Torvalds 373b95bf2d3SMilan Broz salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL); 3745861f1beSMilan Broz if (!salt) { 37572d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 3765861f1beSMilan Broz err = -ENOMEM; 3775861f1beSMilan Broz goto bad; 3781da177e4SLinus Torvalds } 3791da177e4SLinus Torvalds 380b95bf2d3SMilan Broz cc->iv_gen_private.essiv.salt = salt; 381b95bf2d3SMilan Broz cc->iv_gen_private.essiv.hash_tfm = hash_tfm; 382b95bf2d3SMilan Broz 383c0297721SAndi Kleen essiv_tfm = setup_essiv_cpu(cc, ti, salt, 384c0297721SAndi Kleen crypto_hash_digestsize(hash_tfm)); 385c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 386c0297721SAndi Kleen crypt_iv_essiv_dtr(cc); 387c0297721SAndi Kleen return PTR_ERR(essiv_tfm); 388c0297721SAndi Kleen } 389fd2d231fSMikulas Patocka cc->iv_private = essiv_tfm; 390c0297721SAndi Kleen 3911da177e4SLinus Torvalds return 0; 3925861f1beSMilan Broz 3935861f1beSMilan Broz bad: 3945861f1beSMilan Broz if (hash_tfm && !IS_ERR(hash_tfm)) 3955861f1beSMilan Broz crypto_free_hash(hash_tfm); 396b95bf2d3SMilan Broz kfree(salt); 3975861f1beSMilan Broz return err; 3981da177e4SLinus Torvalds } 3991da177e4SLinus Torvalds 4002dc5327dSMilan Broz static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, 4012dc5327dSMilan Broz struct dm_crypt_request *dmreq) 4021da177e4SLinus Torvalds { 403fd2d231fSMikulas Patocka struct crypto_cipher *essiv_tfm = cc->iv_private; 404c0297721SAndi Kleen 4051da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 406283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 407c0297721SAndi Kleen crypto_cipher_encrypt_one(essiv_tfm, iv, iv); 408c0297721SAndi Kleen 4091da177e4SLinus Torvalds return 0; 4101da177e4SLinus Torvalds } 4111da177e4SLinus Torvalds 41248527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 41348527fa7SRik Snel const char *opts) 41448527fa7SRik Snel { 415c0297721SAndi Kleen unsigned bs = crypto_ablkcipher_blocksize(any_tfm(cc)); 416f0d1b0b3SDavid Howells int log = ilog2(bs); 41748527fa7SRik Snel 41848527fa7SRik Snel /* we need to calculate how far we must shift the sector count 41948527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 42048527fa7SRik Snel 42148527fa7SRik Snel if (1 << log != bs) { 42248527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 42348527fa7SRik Snel return -EINVAL; 42448527fa7SRik Snel } 42548527fa7SRik Snel 42648527fa7SRik Snel if (log > 9) { 42748527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 42848527fa7SRik Snel return -EINVAL; 42948527fa7SRik Snel } 43048527fa7SRik Snel 43160473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 43248527fa7SRik Snel 43348527fa7SRik Snel return 0; 43448527fa7SRik Snel } 43548527fa7SRik Snel 43648527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 43748527fa7SRik Snel { 43848527fa7SRik Snel } 43948527fa7SRik Snel 4402dc5327dSMilan Broz static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, 4412dc5327dSMilan Broz struct dm_crypt_request *dmreq) 44248527fa7SRik Snel { 44379066ad3SHerbert Xu __be64 val; 44479066ad3SHerbert Xu 44548527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 44679066ad3SHerbert Xu 4472dc5327dSMilan Broz val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1); 44879066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 44948527fa7SRik Snel 4501da177e4SLinus Torvalds return 0; 4511da177e4SLinus Torvalds } 4521da177e4SLinus Torvalds 4532dc5327dSMilan Broz static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, 4542dc5327dSMilan Broz struct dm_crypt_request *dmreq) 45546b47730SLudwig Nussel { 45646b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 45746b47730SLudwig Nussel 45846b47730SLudwig Nussel return 0; 45946b47730SLudwig Nussel } 46046b47730SLudwig Nussel 46134745785SMilan Broz static void crypt_iv_lmk_dtr(struct crypt_config *cc) 46234745785SMilan Broz { 46334745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 46434745785SMilan Broz 46534745785SMilan Broz if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm)) 46634745785SMilan Broz crypto_free_shash(lmk->hash_tfm); 46734745785SMilan Broz lmk->hash_tfm = NULL; 46834745785SMilan Broz 46934745785SMilan Broz kzfree(lmk->seed); 47034745785SMilan Broz lmk->seed = NULL; 47134745785SMilan Broz } 47234745785SMilan Broz 47334745785SMilan Broz static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti, 47434745785SMilan Broz const char *opts) 47534745785SMilan Broz { 47634745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 47734745785SMilan Broz 47834745785SMilan Broz lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0); 47934745785SMilan Broz if (IS_ERR(lmk->hash_tfm)) { 48034745785SMilan Broz ti->error = "Error initializing LMK hash"; 48134745785SMilan Broz return PTR_ERR(lmk->hash_tfm); 48234745785SMilan Broz } 48334745785SMilan Broz 48434745785SMilan Broz /* No seed in LMK version 2 */ 48534745785SMilan Broz if (cc->key_parts == cc->tfms_count) { 48634745785SMilan Broz lmk->seed = NULL; 48734745785SMilan Broz return 0; 48834745785SMilan Broz } 48934745785SMilan Broz 49034745785SMilan Broz lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL); 49134745785SMilan Broz if (!lmk->seed) { 49234745785SMilan Broz crypt_iv_lmk_dtr(cc); 49334745785SMilan Broz ti->error = "Error kmallocing seed storage in LMK"; 49434745785SMilan Broz return -ENOMEM; 49534745785SMilan Broz } 49634745785SMilan Broz 49734745785SMilan Broz return 0; 49834745785SMilan Broz } 49934745785SMilan Broz 50034745785SMilan Broz static int crypt_iv_lmk_init(struct crypt_config *cc) 50134745785SMilan Broz { 50234745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 50334745785SMilan Broz int subkey_size = cc->key_size / cc->key_parts; 50434745785SMilan Broz 50534745785SMilan Broz /* LMK seed is on the position of LMK_KEYS + 1 key */ 50634745785SMilan Broz if (lmk->seed) 50734745785SMilan Broz memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size), 50834745785SMilan Broz crypto_shash_digestsize(lmk->hash_tfm)); 50934745785SMilan Broz 51034745785SMilan Broz return 0; 51134745785SMilan Broz } 51234745785SMilan Broz 51334745785SMilan Broz static int crypt_iv_lmk_wipe(struct crypt_config *cc) 51434745785SMilan Broz { 51534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 51634745785SMilan Broz 51734745785SMilan Broz if (lmk->seed) 51834745785SMilan Broz memset(lmk->seed, 0, LMK_SEED_SIZE); 51934745785SMilan Broz 52034745785SMilan Broz return 0; 52134745785SMilan Broz } 52234745785SMilan Broz 52334745785SMilan Broz static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv, 52434745785SMilan Broz struct dm_crypt_request *dmreq, 52534745785SMilan Broz u8 *data) 52634745785SMilan Broz { 52734745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 52834745785SMilan Broz struct { 52934745785SMilan Broz struct shash_desc desc; 53034745785SMilan Broz char ctx[crypto_shash_descsize(lmk->hash_tfm)]; 53134745785SMilan Broz } sdesc; 53234745785SMilan Broz struct md5_state md5state; 53334745785SMilan Broz u32 buf[4]; 53434745785SMilan Broz int i, r; 53534745785SMilan Broz 53634745785SMilan Broz sdesc.desc.tfm = lmk->hash_tfm; 53734745785SMilan Broz sdesc.desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 53834745785SMilan Broz 53934745785SMilan Broz r = crypto_shash_init(&sdesc.desc); 54034745785SMilan Broz if (r) 54134745785SMilan Broz return r; 54234745785SMilan Broz 54334745785SMilan Broz if (lmk->seed) { 54434745785SMilan Broz r = crypto_shash_update(&sdesc.desc, lmk->seed, LMK_SEED_SIZE); 54534745785SMilan Broz if (r) 54634745785SMilan Broz return r; 54734745785SMilan Broz } 54834745785SMilan Broz 54934745785SMilan Broz /* Sector is always 512B, block size 16, add data of blocks 1-31 */ 55034745785SMilan Broz r = crypto_shash_update(&sdesc.desc, data + 16, 16 * 31); 55134745785SMilan Broz if (r) 55234745785SMilan Broz return r; 55334745785SMilan Broz 55434745785SMilan Broz /* Sector is cropped to 56 bits here */ 55534745785SMilan Broz buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF); 55634745785SMilan Broz buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000); 55734745785SMilan Broz buf[2] = cpu_to_le32(4024); 55834745785SMilan Broz buf[3] = 0; 55934745785SMilan Broz r = crypto_shash_update(&sdesc.desc, (u8 *)buf, sizeof(buf)); 56034745785SMilan Broz if (r) 56134745785SMilan Broz return r; 56234745785SMilan Broz 56334745785SMilan Broz /* No MD5 padding here */ 56434745785SMilan Broz r = crypto_shash_export(&sdesc.desc, &md5state); 56534745785SMilan Broz if (r) 56634745785SMilan Broz return r; 56734745785SMilan Broz 56834745785SMilan Broz for (i = 0; i < MD5_HASH_WORDS; i++) 56934745785SMilan Broz __cpu_to_le32s(&md5state.hash[i]); 57034745785SMilan Broz memcpy(iv, &md5state.hash, cc->iv_size); 57134745785SMilan Broz 57234745785SMilan Broz return 0; 57334745785SMilan Broz } 57434745785SMilan Broz 57534745785SMilan Broz static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv, 57634745785SMilan Broz struct dm_crypt_request *dmreq) 57734745785SMilan Broz { 57834745785SMilan Broz u8 *src; 57934745785SMilan Broz int r = 0; 58034745785SMilan Broz 58134745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 582c2e022cbSCong Wang src = kmap_atomic(sg_page(&dmreq->sg_in)); 58334745785SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, src + dmreq->sg_in.offset); 584c2e022cbSCong Wang kunmap_atomic(src); 58534745785SMilan Broz } else 58634745785SMilan Broz memset(iv, 0, cc->iv_size); 58734745785SMilan Broz 58834745785SMilan Broz return r; 58934745785SMilan Broz } 59034745785SMilan Broz 59134745785SMilan Broz static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv, 59234745785SMilan Broz struct dm_crypt_request *dmreq) 59334745785SMilan Broz { 59434745785SMilan Broz u8 *dst; 59534745785SMilan Broz int r; 59634745785SMilan Broz 59734745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) 59834745785SMilan Broz return 0; 59934745785SMilan Broz 600c2e022cbSCong Wang dst = kmap_atomic(sg_page(&dmreq->sg_out)); 60134745785SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, dst + dmreq->sg_out.offset); 60234745785SMilan Broz 60334745785SMilan Broz /* Tweak the first block of plaintext sector */ 60434745785SMilan Broz if (!r) 60534745785SMilan Broz crypto_xor(dst + dmreq->sg_out.offset, iv, cc->iv_size); 60634745785SMilan Broz 607c2e022cbSCong Wang kunmap_atomic(dst); 60834745785SMilan Broz return r; 60934745785SMilan Broz } 61034745785SMilan Broz 6111da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_plain_ops = { 6121da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 6131da177e4SLinus Torvalds }; 6141da177e4SLinus Torvalds 61561afef61SMilan Broz static struct crypt_iv_operations crypt_iv_plain64_ops = { 61661afef61SMilan Broz .generator = crypt_iv_plain64_gen 61761afef61SMilan Broz }; 61861afef61SMilan Broz 6191da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_essiv_ops = { 6201da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 6211da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 622b95bf2d3SMilan Broz .init = crypt_iv_essiv_init, 623542da317SMilan Broz .wipe = crypt_iv_essiv_wipe, 6241da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 6251da177e4SLinus Torvalds }; 6261da177e4SLinus Torvalds 62748527fa7SRik Snel static struct crypt_iv_operations crypt_iv_benbi_ops = { 62848527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 62948527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 63048527fa7SRik Snel .generator = crypt_iv_benbi_gen 63148527fa7SRik Snel }; 6321da177e4SLinus Torvalds 63346b47730SLudwig Nussel static struct crypt_iv_operations crypt_iv_null_ops = { 63446b47730SLudwig Nussel .generator = crypt_iv_null_gen 63546b47730SLudwig Nussel }; 63646b47730SLudwig Nussel 63734745785SMilan Broz static struct crypt_iv_operations crypt_iv_lmk_ops = { 63834745785SMilan Broz .ctr = crypt_iv_lmk_ctr, 63934745785SMilan Broz .dtr = crypt_iv_lmk_dtr, 64034745785SMilan Broz .init = crypt_iv_lmk_init, 64134745785SMilan Broz .wipe = crypt_iv_lmk_wipe, 64234745785SMilan Broz .generator = crypt_iv_lmk_gen, 64334745785SMilan Broz .post = crypt_iv_lmk_post 64434745785SMilan Broz }; 64534745785SMilan Broz 646d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 647d469f841SMilan Broz struct convert_context *ctx, 6481da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 649fcd369daSMilan Broz sector_t sector) 6501da177e4SLinus Torvalds { 6511da177e4SLinus Torvalds ctx->bio_in = bio_in; 6521da177e4SLinus Torvalds ctx->bio_out = bio_out; 6531da177e4SLinus Torvalds ctx->offset_in = 0; 6541da177e4SLinus Torvalds ctx->offset_out = 0; 6551da177e4SLinus Torvalds ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 6561da177e4SLinus Torvalds ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 6571da177e4SLinus Torvalds ctx->sector = sector + cc->iv_offset; 65843d69034SMilan Broz init_completion(&ctx->restart); 6591da177e4SLinus Torvalds } 6601da177e4SLinus Torvalds 661b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 662b2174eebSHuang Ying struct ablkcipher_request *req) 663b2174eebSHuang Ying { 664b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 665b2174eebSHuang Ying } 666b2174eebSHuang Ying 667b2174eebSHuang Ying static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc, 668b2174eebSHuang Ying struct dm_crypt_request *dmreq) 669b2174eebSHuang Ying { 670b2174eebSHuang Ying return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start); 671b2174eebSHuang Ying } 672b2174eebSHuang Ying 6732dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, 6742dc5327dSMilan Broz struct dm_crypt_request *dmreq) 6752dc5327dSMilan Broz { 6762dc5327dSMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 6772dc5327dSMilan Broz crypto_ablkcipher_alignmask(any_tfm(cc)) + 1); 6782dc5327dSMilan Broz } 6792dc5327dSMilan Broz 68001482b76SMilan Broz static int crypt_convert_block(struct crypt_config *cc, 6813a7f6c99SMilan Broz struct convert_context *ctx, 6823a7f6c99SMilan Broz struct ablkcipher_request *req) 68301482b76SMilan Broz { 68401482b76SMilan Broz struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 68501482b76SMilan Broz struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 6863a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 6873a7f6c99SMilan Broz u8 *iv; 68840b6229bSMikulas Patocka int r; 68901482b76SMilan Broz 690b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 6912dc5327dSMilan Broz iv = iv_of_dmreq(cc, dmreq); 6923a7f6c99SMilan Broz 6932dc5327dSMilan Broz dmreq->iv_sector = ctx->sector; 694b2174eebSHuang Ying dmreq->ctx = ctx; 6953a7f6c99SMilan Broz sg_init_table(&dmreq->sg_in, 1); 6963a7f6c99SMilan Broz sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT, 69701482b76SMilan Broz bv_in->bv_offset + ctx->offset_in); 69801482b76SMilan Broz 6993a7f6c99SMilan Broz sg_init_table(&dmreq->sg_out, 1); 7003a7f6c99SMilan Broz sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT, 70101482b76SMilan Broz bv_out->bv_offset + ctx->offset_out); 70201482b76SMilan Broz 70301482b76SMilan Broz ctx->offset_in += 1 << SECTOR_SHIFT; 70401482b76SMilan Broz if (ctx->offset_in >= bv_in->bv_len) { 70501482b76SMilan Broz ctx->offset_in = 0; 70601482b76SMilan Broz ctx->idx_in++; 70701482b76SMilan Broz } 70801482b76SMilan Broz 70901482b76SMilan Broz ctx->offset_out += 1 << SECTOR_SHIFT; 71001482b76SMilan Broz if (ctx->offset_out >= bv_out->bv_len) { 71101482b76SMilan Broz ctx->offset_out = 0; 71201482b76SMilan Broz ctx->idx_out++; 71301482b76SMilan Broz } 71401482b76SMilan Broz 7153a7f6c99SMilan Broz if (cc->iv_gen_ops) { 7162dc5327dSMilan Broz r = cc->iv_gen_ops->generator(cc, iv, dmreq); 7173a7f6c99SMilan Broz if (r < 0) 7183a7f6c99SMilan Broz return r; 7193a7f6c99SMilan Broz } 7203a7f6c99SMilan Broz 7213a7f6c99SMilan Broz ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out, 7223a7f6c99SMilan Broz 1 << SECTOR_SHIFT, iv); 7233a7f6c99SMilan Broz 7243a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 7253a7f6c99SMilan Broz r = crypto_ablkcipher_encrypt(req); 7263a7f6c99SMilan Broz else 7273a7f6c99SMilan Broz r = crypto_ablkcipher_decrypt(req); 7283a7f6c99SMilan Broz 7292dc5327dSMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 7302dc5327dSMilan Broz r = cc->iv_gen_ops->post(cc, iv, dmreq); 7312dc5327dSMilan Broz 7323a7f6c99SMilan Broz return r; 73301482b76SMilan Broz } 73401482b76SMilan Broz 73595497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 73695497a96SMilan Broz int error); 737c0297721SAndi Kleen 738ddd42edfSMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 739ddd42edfSMilan Broz struct convert_context *ctx) 740ddd42edfSMilan Broz { 741c0297721SAndi Kleen struct crypt_cpu *this_cc = this_crypt_config(cc); 742d1f96423SMilan Broz unsigned key_index = ctx->sector & (cc->tfms_count - 1); 743c0297721SAndi Kleen 744c0297721SAndi Kleen if (!this_cc->req) 745c0297721SAndi Kleen this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO); 746c0297721SAndi Kleen 747fd2d231fSMikulas Patocka ablkcipher_request_set_tfm(this_cc->req, cc->tfms[key_index]); 748c0297721SAndi Kleen ablkcipher_request_set_callback(this_cc->req, 749c0297721SAndi Kleen CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, 750c0297721SAndi Kleen kcryptd_async_done, dmreq_of_req(cc, this_cc->req)); 751ddd42edfSMilan Broz } 752ddd42edfSMilan Broz 7531da177e4SLinus Torvalds /* 7541da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 7551da177e4SLinus Torvalds */ 7561da177e4SLinus Torvalds static int crypt_convert(struct crypt_config *cc, 7571da177e4SLinus Torvalds struct convert_context *ctx) 7581da177e4SLinus Torvalds { 759c0297721SAndi Kleen struct crypt_cpu *this_cc = this_crypt_config(cc); 7603f1e9070SMilan Broz int r; 7611da177e4SLinus Torvalds 76240b6229bSMikulas Patocka atomic_set(&ctx->cc_pending, 1); 763c8081618SMilan Broz 7641da177e4SLinus Torvalds while(ctx->idx_in < ctx->bio_in->bi_vcnt && 7651da177e4SLinus Torvalds ctx->idx_out < ctx->bio_out->bi_vcnt) { 7661da177e4SLinus Torvalds 7673a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 7683a7f6c99SMilan Broz 76940b6229bSMikulas Patocka atomic_inc(&ctx->cc_pending); 7703f1e9070SMilan Broz 771c0297721SAndi Kleen r = crypt_convert_block(cc, ctx, this_cc->req); 7723a7f6c99SMilan Broz 7733a7f6c99SMilan Broz switch (r) { 7743f1e9070SMilan Broz /* async */ 7753a7f6c99SMilan Broz case -EBUSY: 7763a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 7773a7f6c99SMilan Broz INIT_COMPLETION(ctx->restart); 7783a7f6c99SMilan Broz /* fall through*/ 7793a7f6c99SMilan Broz case -EINPROGRESS: 780c0297721SAndi Kleen this_cc->req = NULL; 7811da177e4SLinus Torvalds ctx->sector++; 7823a7f6c99SMilan Broz continue; 7833a7f6c99SMilan Broz 7843f1e9070SMilan Broz /* sync */ 7853f1e9070SMilan Broz case 0: 78640b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 7873f1e9070SMilan Broz ctx->sector++; 788c7f1b204SMilan Broz cond_resched(); 7893f1e9070SMilan Broz continue; 7901da177e4SLinus Torvalds 7913f1e9070SMilan Broz /* error */ 7923f1e9070SMilan Broz default: 79340b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 7941da177e4SLinus Torvalds return r; 7951da177e4SLinus Torvalds } 7963f1e9070SMilan Broz } 7973f1e9070SMilan Broz 7983f1e9070SMilan Broz return 0; 7993f1e9070SMilan Broz } 8001da177e4SLinus Torvalds 8016a24c718SMilan Broz static void dm_crypt_bio_destructor(struct bio *bio) 8026a24c718SMilan Broz { 803028867acSAlasdair G Kergon struct dm_crypt_io *io = bio->bi_private; 8046a24c718SMilan Broz struct crypt_config *cc = io->target->private; 8056a24c718SMilan Broz 8066a24c718SMilan Broz bio_free(bio, cc->bs); 8076a24c718SMilan Broz } 8086a24c718SMilan Broz 8091da177e4SLinus Torvalds /* 8101da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 8111da177e4SLinus Torvalds * This should never violate the device limitations 812933f01d4SMilan Broz * May return a smaller bio when running out of pages, indicated by 813933f01d4SMilan Broz * *out_of_pages set to 1. 8141da177e4SLinus Torvalds */ 815933f01d4SMilan Broz static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, 816933f01d4SMilan Broz unsigned *out_of_pages) 8171da177e4SLinus Torvalds { 818027581f3SOlaf Kirch struct crypt_config *cc = io->target->private; 8198b004457SMilan Broz struct bio *clone; 8201da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 821b4e3ca1aSAl Viro gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 82291e10625SMilan Broz unsigned i, len; 82391e10625SMilan Broz struct page *page; 8241da177e4SLinus Torvalds 8256a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 8268b004457SMilan Broz if (!clone) 8271da177e4SLinus Torvalds return NULL; 8281da177e4SLinus Torvalds 829027581f3SOlaf Kirch clone_init(io, clone); 830933f01d4SMilan Broz *out_of_pages = 0; 8316a24c718SMilan Broz 832f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 83391e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 834933f01d4SMilan Broz if (!page) { 835933f01d4SMilan Broz *out_of_pages = 1; 8361da177e4SLinus Torvalds break; 837933f01d4SMilan Broz } 8381da177e4SLinus Torvalds 8391da177e4SLinus Torvalds /* 840aeb2deaeSMikulas Patocka * If additional pages cannot be allocated without waiting, 841aeb2deaeSMikulas Patocka * return a partially-allocated bio. The caller will then try 842aeb2deaeSMikulas Patocka * to allocate more bios while submitting this partial bio. 8431da177e4SLinus Torvalds */ 8441da177e4SLinus Torvalds gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 8451da177e4SLinus Torvalds 84691e10625SMilan Broz len = (size > PAGE_SIZE) ? PAGE_SIZE : size; 8471da177e4SLinus Torvalds 84891e10625SMilan Broz if (!bio_add_page(clone, page, len, 0)) { 84991e10625SMilan Broz mempool_free(page, cc->page_pool); 85091e10625SMilan Broz break; 85191e10625SMilan Broz } 85291e10625SMilan Broz 85391e10625SMilan Broz size -= len; 8541da177e4SLinus Torvalds } 8551da177e4SLinus Torvalds 8568b004457SMilan Broz if (!clone->bi_size) { 8578b004457SMilan Broz bio_put(clone); 8581da177e4SLinus Torvalds return NULL; 8591da177e4SLinus Torvalds } 8601da177e4SLinus Torvalds 8618b004457SMilan Broz return clone; 8621da177e4SLinus Torvalds } 8631da177e4SLinus Torvalds 864644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 8651da177e4SLinus Torvalds { 866644bd2f0SNeil Brown unsigned int i; 8671da177e4SLinus Torvalds struct bio_vec *bv; 8681da177e4SLinus Torvalds 869644bd2f0SNeil Brown for (i = 0; i < clone->bi_vcnt; i++) { 8708b004457SMilan Broz bv = bio_iovec_idx(clone, i); 8711da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 8721da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 8731da177e4SLinus Torvalds bv->bv_page = NULL; 8741da177e4SLinus Torvalds } 8751da177e4SLinus Torvalds } 8761da177e4SLinus Torvalds 877dc440d1eSMilan Broz static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti, 878dc440d1eSMilan Broz struct bio *bio, sector_t sector) 879dc440d1eSMilan Broz { 880dc440d1eSMilan Broz struct crypt_config *cc = ti->private; 881dc440d1eSMilan Broz struct dm_crypt_io *io; 882dc440d1eSMilan Broz 883dc440d1eSMilan Broz io = mempool_alloc(cc->io_pool, GFP_NOIO); 884dc440d1eSMilan Broz io->target = ti; 885dc440d1eSMilan Broz io->base_bio = bio; 886dc440d1eSMilan Broz io->sector = sector; 887dc440d1eSMilan Broz io->error = 0; 888393b47efSMilan Broz io->base_io = NULL; 88940b6229bSMikulas Patocka atomic_set(&io->io_pending, 0); 890dc440d1eSMilan Broz 891dc440d1eSMilan Broz return io; 892dc440d1eSMilan Broz } 893dc440d1eSMilan Broz 8943e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 8953e1a8bddSMilan Broz { 89640b6229bSMikulas Patocka atomic_inc(&io->io_pending); 8973e1a8bddSMilan Broz } 8983e1a8bddSMilan Broz 8991da177e4SLinus Torvalds /* 9001da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 9011da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 902393b47efSMilan Broz * If base_io is set, wait for the last fragment to complete. 9031da177e4SLinus Torvalds */ 9045742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 9051da177e4SLinus Torvalds { 9065742fd77SMilan Broz struct crypt_config *cc = io->target->private; 907b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 908b35f8caaSMilan Broz struct dm_crypt_io *base_io = io->base_io; 909b35f8caaSMilan Broz int error = io->error; 9101da177e4SLinus Torvalds 91140b6229bSMikulas Patocka if (!atomic_dec_and_test(&io->io_pending)) 9121da177e4SLinus Torvalds return; 9131da177e4SLinus Torvalds 9141da177e4SLinus Torvalds mempool_free(io, cc->io_pool); 915b35f8caaSMilan Broz 916b35f8caaSMilan Broz if (likely(!base_io)) 917b35f8caaSMilan Broz bio_endio(base_bio, error); 918b35f8caaSMilan Broz else { 919b35f8caaSMilan Broz if (error && !base_io->error) 920b35f8caaSMilan Broz base_io->error = error; 921b35f8caaSMilan Broz crypt_dec_pending(base_io); 922b35f8caaSMilan Broz } 9231da177e4SLinus Torvalds } 9241da177e4SLinus Torvalds 9251da177e4SLinus Torvalds /* 926cabf08e4SMilan Broz * kcryptd/kcryptd_io: 9271da177e4SLinus Torvalds * 9281da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 92923541d2dSMilan Broz * interrupt context. 930cabf08e4SMilan Broz * 931cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 932cabf08e4SMilan Broz * 933cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 934cabf08e4SMilan Broz * 935cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 936cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 937cabf08e4SMilan Broz * to memory allocation. 938c0297721SAndi Kleen * 939c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances. 940c0297721SAndi Kleen * They should not depend on each other and do not block. 9411da177e4SLinus Torvalds */ 9426712ecf8SNeilBrown static void crypt_endio(struct bio *clone, int error) 9438b004457SMilan Broz { 944028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 9458b004457SMilan Broz struct crypt_config *cc = io->target->private; 946ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 9478b004457SMilan Broz 948adfe4770SMilan Broz if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error)) 949adfe4770SMilan Broz error = -EIO; 950adfe4770SMilan Broz 9518b004457SMilan Broz /* 9526712ecf8SNeilBrown * free the processed pages 9538b004457SMilan Broz */ 954ee7a491eSMilan Broz if (rw == WRITE) 955644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 9568b004457SMilan Broz 9578b004457SMilan Broz bio_put(clone); 958ee7a491eSMilan Broz 959ee7a491eSMilan Broz if (rw == READ && !error) { 960cabf08e4SMilan Broz kcryptd_queue_crypt(io); 9616712ecf8SNeilBrown return; 962ee7a491eSMilan Broz } 9635742fd77SMilan Broz 9645742fd77SMilan Broz if (unlikely(error)) 9655742fd77SMilan Broz io->error = error; 9665742fd77SMilan Broz 9675742fd77SMilan Broz crypt_dec_pending(io); 9688b004457SMilan Broz } 9698b004457SMilan Broz 970028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 9718b004457SMilan Broz { 9728b004457SMilan Broz struct crypt_config *cc = io->target->private; 9738b004457SMilan Broz 9748b004457SMilan Broz clone->bi_private = io; 9758b004457SMilan Broz clone->bi_end_io = crypt_endio; 9768b004457SMilan Broz clone->bi_bdev = cc->dev->bdev; 9778b004457SMilan Broz clone->bi_rw = io->base_bio->bi_rw; 978027581f3SOlaf Kirch clone->bi_destructor = dm_crypt_bio_destructor; 9798b004457SMilan Broz } 9808b004457SMilan Broz 98120c82538SMilan Broz static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) 9828b004457SMilan Broz { 9838b004457SMilan Broz struct crypt_config *cc = io->target->private; 9848b004457SMilan Broz struct bio *base_bio = io->base_bio; 9858b004457SMilan Broz struct bio *clone; 98693e605c2SMilan Broz 9878b004457SMilan Broz /* 9888b004457SMilan Broz * The block layer might modify the bvec array, so always 9898b004457SMilan Broz * copy the required bvecs because we need the original 9908b004457SMilan Broz * one in order to decrypt the whole bio data *afterwards*. 9918b004457SMilan Broz */ 99220c82538SMilan Broz clone = bio_alloc_bioset(gfp, bio_segments(base_bio), cc->bs); 9937eaceaccSJens Axboe if (!clone) 99420c82538SMilan Broz return 1; 9958b004457SMilan Broz 99620c82538SMilan Broz crypt_inc_pending(io); 99720c82538SMilan Broz 9988b004457SMilan Broz clone_init(io, clone); 9998b004457SMilan Broz clone->bi_idx = 0; 10008b004457SMilan Broz clone->bi_vcnt = bio_segments(base_bio); 10018b004457SMilan Broz clone->bi_size = base_bio->bi_size; 10020c395b0fSMilan Broz clone->bi_sector = cc->start + io->sector; 10038b004457SMilan Broz memcpy(clone->bi_io_vec, bio_iovec(base_bio), 10048b004457SMilan Broz sizeof(struct bio_vec) * clone->bi_vcnt); 10058b004457SMilan Broz 100693e605c2SMilan Broz generic_make_request(clone); 100720c82538SMilan Broz return 0; 10088b004457SMilan Broz } 10098b004457SMilan Broz 10104e4eef64SMilan Broz static void kcryptd_io_write(struct dm_crypt_io *io) 10114e4eef64SMilan Broz { 101295497a96SMilan Broz struct bio *clone = io->ctx.bio_out; 101395497a96SMilan Broz generic_make_request(clone); 10144e4eef64SMilan Broz } 10154e4eef64SMilan Broz 1016395b167cSAlasdair G Kergon static void kcryptd_io(struct work_struct *work) 1017395b167cSAlasdair G Kergon { 1018395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 1019395b167cSAlasdair G Kergon 102020c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 102120c82538SMilan Broz crypt_inc_pending(io); 102220c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOIO)) 102320c82538SMilan Broz io->error = -ENOMEM; 102420c82538SMilan Broz crypt_dec_pending(io); 102520c82538SMilan Broz } else 1026395b167cSAlasdair G Kergon kcryptd_io_write(io); 1027395b167cSAlasdair G Kergon } 1028395b167cSAlasdair G Kergon 1029395b167cSAlasdair G Kergon static void kcryptd_queue_io(struct dm_crypt_io *io) 1030395b167cSAlasdair G Kergon { 1031395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 1032395b167cSAlasdair G Kergon 1033395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_io); 1034395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 1035395b167cSAlasdair G Kergon } 1036395b167cSAlasdair G Kergon 103772c6e7afSMikulas Patocka static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) 10384e4eef64SMilan Broz { 1039dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 1040dec1cedfSMilan Broz struct crypt_config *cc = io->target->private; 1041dec1cedfSMilan Broz 104272c6e7afSMikulas Patocka if (unlikely(io->error < 0)) { 1043dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 1044dec1cedfSMilan Broz bio_put(clone); 10456c031f41SMilan Broz crypt_dec_pending(io); 1046dec1cedfSMilan Broz return; 1047dec1cedfSMilan Broz } 1048dec1cedfSMilan Broz 1049dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 1050dec1cedfSMilan Broz BUG_ON(io->ctx.idx_out < clone->bi_vcnt); 1051dec1cedfSMilan Broz 1052dec1cedfSMilan Broz clone->bi_sector = cc->start + io->sector; 1053899c95d3SMilan Broz 105495497a96SMilan Broz if (async) 105595497a96SMilan Broz kcryptd_queue_io(io); 10561e37bb8eSAlasdair G Kergon else 1057899c95d3SMilan Broz generic_make_request(clone); 10584e4eef64SMilan Broz } 10594e4eef64SMilan Broz 1060fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 10618b004457SMilan Broz { 10628b004457SMilan Broz struct crypt_config *cc = io->target->private; 10638b004457SMilan Broz struct bio *clone; 1064393b47efSMilan Broz struct dm_crypt_io *new_io; 1065c8081618SMilan Broz int crypt_finished; 1066933f01d4SMilan Broz unsigned out_of_pages = 0; 1067dec1cedfSMilan Broz unsigned remaining = io->base_bio->bi_size; 1068b635b00eSMilan Broz sector_t sector = io->sector; 1069dec1cedfSMilan Broz int r; 10708b004457SMilan Broz 107193e605c2SMilan Broz /* 1072fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 1073fc5a5e9aSMilan Broz */ 1074fc5a5e9aSMilan Broz crypt_inc_pending(io); 1075b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 1076fc5a5e9aSMilan Broz 1077fc5a5e9aSMilan Broz /* 107893e605c2SMilan Broz * The allocated buffers can be smaller than the whole bio, 107993e605c2SMilan Broz * so repeat the whole process until all the data can be handled. 108093e605c2SMilan Broz */ 108193e605c2SMilan Broz while (remaining) { 1082933f01d4SMilan Broz clone = crypt_alloc_buffer(io, remaining, &out_of_pages); 108323541d2dSMilan Broz if (unlikely(!clone)) { 10845742fd77SMilan Broz io->error = -ENOMEM; 1085fc5a5e9aSMilan Broz break; 108623541d2dSMilan Broz } 10878b004457SMilan Broz 108853017030SMilan Broz io->ctx.bio_out = clone; 108953017030SMilan Broz io->ctx.idx_out = 0; 10908b004457SMilan Broz 109193e605c2SMilan Broz remaining -= clone->bi_size; 1092b635b00eSMilan Broz sector += bio_sectors(clone); 1093dec1cedfSMilan Broz 10944e594098SMilan Broz crypt_inc_pending(io); 109572c6e7afSMikulas Patocka 1096dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 109772c6e7afSMikulas Patocka if (r < 0) 109872c6e7afSMikulas Patocka io->error = -EIO; 109972c6e7afSMikulas Patocka 110040b6229bSMikulas Patocka crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); 1101dec1cedfSMilan Broz 1102c8081618SMilan Broz /* Encryption was already finished, submit io now */ 1103c8081618SMilan Broz if (crypt_finished) { 110472c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 0); 1105c8081618SMilan Broz 1106c8081618SMilan Broz /* 1107c8081618SMilan Broz * If there was an error, do not try next fragments. 1108c8081618SMilan Broz * For async, error is processed in async handler. 1109c8081618SMilan Broz */ 11106c031f41SMilan Broz if (unlikely(r < 0)) 1111fc5a5e9aSMilan Broz break; 1112b635b00eSMilan Broz 1113b635b00eSMilan Broz io->sector = sector; 11144e594098SMilan Broz } 111593e605c2SMilan Broz 1116933f01d4SMilan Broz /* 1117933f01d4SMilan Broz * Out of memory -> run queues 1118933f01d4SMilan Broz * But don't wait if split was due to the io size restriction 1119933f01d4SMilan Broz */ 1120933f01d4SMilan Broz if (unlikely(out_of_pages)) 11218aa7e847SJens Axboe congestion_wait(BLK_RW_ASYNC, HZ/100); 1122933f01d4SMilan Broz 1123393b47efSMilan Broz /* 1124393b47efSMilan Broz * With async crypto it is unsafe to share the crypto context 1125393b47efSMilan Broz * between fragments, so switch to a new dm_crypt_io structure. 1126393b47efSMilan Broz */ 1127393b47efSMilan Broz if (unlikely(!crypt_finished && remaining)) { 1128393b47efSMilan Broz new_io = crypt_io_alloc(io->target, io->base_bio, 1129393b47efSMilan Broz sector); 1130393b47efSMilan Broz crypt_inc_pending(new_io); 1131393b47efSMilan Broz crypt_convert_init(cc, &new_io->ctx, NULL, 1132393b47efSMilan Broz io->base_bio, sector); 1133393b47efSMilan Broz new_io->ctx.idx_in = io->ctx.idx_in; 1134393b47efSMilan Broz new_io->ctx.offset_in = io->ctx.offset_in; 1135393b47efSMilan Broz 1136393b47efSMilan Broz /* 1137393b47efSMilan Broz * Fragments after the first use the base_io 1138393b47efSMilan Broz * pending count. 1139393b47efSMilan Broz */ 1140393b47efSMilan Broz if (!io->base_io) 1141393b47efSMilan Broz new_io->base_io = io; 1142393b47efSMilan Broz else { 1143393b47efSMilan Broz new_io->base_io = io->base_io; 1144393b47efSMilan Broz crypt_inc_pending(io->base_io); 1145393b47efSMilan Broz crypt_dec_pending(io); 1146393b47efSMilan Broz } 1147393b47efSMilan Broz 1148393b47efSMilan Broz io = new_io; 1149393b47efSMilan Broz } 11508b004457SMilan Broz } 1151899c95d3SMilan Broz 1152899c95d3SMilan Broz crypt_dec_pending(io); 115384131db6SMilan Broz } 115484131db6SMilan Broz 115572c6e7afSMikulas Patocka static void kcryptd_crypt_read_done(struct dm_crypt_io *io) 11565742fd77SMilan Broz { 11575742fd77SMilan Broz crypt_dec_pending(io); 11585742fd77SMilan Broz } 11595742fd77SMilan Broz 11604e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 11618b004457SMilan Broz { 11628b004457SMilan Broz struct crypt_config *cc = io->target->private; 11635742fd77SMilan Broz int r = 0; 11648b004457SMilan Broz 11653e1a8bddSMilan Broz crypt_inc_pending(io); 11663a7f6c99SMilan Broz 116753017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 11680c395b0fSMilan Broz io->sector); 11698b004457SMilan Broz 11705742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 117172c6e7afSMikulas Patocka if (r < 0) 117272c6e7afSMikulas Patocka io->error = -EIO; 11735742fd77SMilan Broz 117440b6229bSMikulas Patocka if (atomic_dec_and_test(&io->ctx.cc_pending)) 117572c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 11763a7f6c99SMilan Broz 11773a7f6c99SMilan Broz crypt_dec_pending(io); 11788b004457SMilan Broz } 11798b004457SMilan Broz 118095497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 118195497a96SMilan Broz int error) 118295497a96SMilan Broz { 1183b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 1184b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 118595497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 118695497a96SMilan Broz struct crypt_config *cc = io->target->private; 118795497a96SMilan Broz 118895497a96SMilan Broz if (error == -EINPROGRESS) { 118995497a96SMilan Broz complete(&ctx->restart); 119095497a96SMilan Broz return; 119195497a96SMilan Broz } 119295497a96SMilan Broz 11932dc5327dSMilan Broz if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post) 11942dc5327dSMilan Broz error = cc->iv_gen_ops->post(cc, iv_of_dmreq(cc, dmreq), dmreq); 11952dc5327dSMilan Broz 119672c6e7afSMikulas Patocka if (error < 0) 119772c6e7afSMikulas Patocka io->error = -EIO; 119872c6e7afSMikulas Patocka 1199b2174eebSHuang Ying mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool); 120095497a96SMilan Broz 120140b6229bSMikulas Patocka if (!atomic_dec_and_test(&ctx->cc_pending)) 120295497a96SMilan Broz return; 120395497a96SMilan Broz 120495497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 120572c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 120695497a96SMilan Broz else 120772c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 1); 120895497a96SMilan Broz } 120995497a96SMilan Broz 12104e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 12114e4eef64SMilan Broz { 12124e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 12134e4eef64SMilan Broz 12144e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 12154e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 12164e4eef64SMilan Broz else 12174e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 12188b004457SMilan Broz } 12198b004457SMilan Broz 1220395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 1221395b167cSAlasdair G Kergon { 1222395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 1223395b167cSAlasdair G Kergon 1224395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 1225395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 1226395b167cSAlasdair G Kergon } 1227395b167cSAlasdair G Kergon 12281da177e4SLinus Torvalds /* 12291da177e4SLinus Torvalds * Decode key from its hex representation 12301da177e4SLinus Torvalds */ 12311da177e4SLinus Torvalds static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 12321da177e4SLinus Torvalds { 12331da177e4SLinus Torvalds char buffer[3]; 12341da177e4SLinus Torvalds unsigned int i; 12351da177e4SLinus Torvalds 12361da177e4SLinus Torvalds buffer[2] = '\0'; 12371da177e4SLinus Torvalds 12381da177e4SLinus Torvalds for (i = 0; i < size; i++) { 12391da177e4SLinus Torvalds buffer[0] = *hex++; 12401da177e4SLinus Torvalds buffer[1] = *hex++; 12411da177e4SLinus Torvalds 12421a66a08aSmajianpeng if (kstrtou8(buffer, 16, &key[i])) 12431da177e4SLinus Torvalds return -EINVAL; 12441da177e4SLinus Torvalds } 12451da177e4SLinus Torvalds 12461da177e4SLinus Torvalds if (*hex != '\0') 12471da177e4SLinus Torvalds return -EINVAL; 12481da177e4SLinus Torvalds 12491da177e4SLinus Torvalds return 0; 12501da177e4SLinus Torvalds } 12511da177e4SLinus Torvalds 12521da177e4SLinus Torvalds /* 12531da177e4SLinus Torvalds * Encode key into its hex representation 12541da177e4SLinus Torvalds */ 12551da177e4SLinus Torvalds static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 12561da177e4SLinus Torvalds { 12571da177e4SLinus Torvalds unsigned int i; 12581da177e4SLinus Torvalds 12591da177e4SLinus Torvalds for (i = 0; i < size; i++) { 12601da177e4SLinus Torvalds sprintf(hex, "%02x", *key); 12611da177e4SLinus Torvalds hex += 2; 12621da177e4SLinus Torvalds key++; 12631da177e4SLinus Torvalds } 12641da177e4SLinus Torvalds } 12651da177e4SLinus Torvalds 1266fd2d231fSMikulas Patocka static void crypt_free_tfms(struct crypt_config *cc) 1267d1f96423SMilan Broz { 1268d1f96423SMilan Broz unsigned i; 1269d1f96423SMilan Broz 1270fd2d231fSMikulas Patocka if (!cc->tfms) 1271fd2d231fSMikulas Patocka return; 1272fd2d231fSMikulas Patocka 1273d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) 1274fd2d231fSMikulas Patocka if (cc->tfms[i] && !IS_ERR(cc->tfms[i])) { 1275fd2d231fSMikulas Patocka crypto_free_ablkcipher(cc->tfms[i]); 1276fd2d231fSMikulas Patocka cc->tfms[i] = NULL; 1277d1f96423SMilan Broz } 1278d1f96423SMilan Broz 1279fd2d231fSMikulas Patocka kfree(cc->tfms); 1280fd2d231fSMikulas Patocka cc->tfms = NULL; 1281fd2d231fSMikulas Patocka } 1282fd2d231fSMikulas Patocka 1283fd2d231fSMikulas Patocka static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode) 1284d1f96423SMilan Broz { 1285d1f96423SMilan Broz unsigned i; 1286d1f96423SMilan Broz int err; 1287d1f96423SMilan Broz 1288fd2d231fSMikulas Patocka cc->tfms = kmalloc(cc->tfms_count * sizeof(struct crypto_ablkcipher *), 1289fd2d231fSMikulas Patocka GFP_KERNEL); 1290fd2d231fSMikulas Patocka if (!cc->tfms) 1291fd2d231fSMikulas Patocka return -ENOMEM; 1292fd2d231fSMikulas Patocka 1293d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 1294fd2d231fSMikulas Patocka cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0); 1295fd2d231fSMikulas Patocka if (IS_ERR(cc->tfms[i])) { 1296fd2d231fSMikulas Patocka err = PTR_ERR(cc->tfms[i]); 1297fd2d231fSMikulas Patocka crypt_free_tfms(cc); 1298d1f96423SMilan Broz return err; 1299d1f96423SMilan Broz } 1300d1f96423SMilan Broz } 1301d1f96423SMilan Broz 1302d1f96423SMilan Broz return 0; 1303d1f96423SMilan Broz } 1304d1f96423SMilan Broz 1305c0297721SAndi Kleen static int crypt_setkey_allcpus(struct crypt_config *cc) 1306c0297721SAndi Kleen { 1307d1f96423SMilan Broz unsigned subkey_size = cc->key_size >> ilog2(cc->tfms_count); 1308fd2d231fSMikulas Patocka int err = 0, i, r; 1309c0297721SAndi Kleen 1310d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 1311fd2d231fSMikulas Patocka r = crypto_ablkcipher_setkey(cc->tfms[i], 1312fd2d231fSMikulas Patocka cc->key + (i * subkey_size), 1313fd2d231fSMikulas Patocka subkey_size); 1314c0297721SAndi Kleen if (r) 1315c0297721SAndi Kleen err = r; 1316c0297721SAndi Kleen } 1317c0297721SAndi Kleen 1318c0297721SAndi Kleen return err; 1319c0297721SAndi Kleen } 1320c0297721SAndi Kleen 1321e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 1322e48d4bbfSMilan Broz { 1323de8be5acSMilan Broz int r = -EINVAL; 1324de8be5acSMilan Broz int key_string_len = strlen(key); 1325de8be5acSMilan Broz 132669a8cfcdSMilan Broz /* The key size may not be changed. */ 1327de8be5acSMilan Broz if (cc->key_size != (key_string_len >> 1)) 1328de8be5acSMilan Broz goto out; 1329e48d4bbfSMilan Broz 133069a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */ 133169a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-")) 1332de8be5acSMilan Broz goto out; 1333e48d4bbfSMilan Broz 133469a8cfcdSMilan Broz if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0) 1335de8be5acSMilan Broz goto out; 1336e48d4bbfSMilan Broz 1337e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1338e48d4bbfSMilan Broz 1339de8be5acSMilan Broz r = crypt_setkey_allcpus(cc); 1340de8be5acSMilan Broz 1341de8be5acSMilan Broz out: 1342de8be5acSMilan Broz /* Hex key string not needed after here, so wipe it. */ 1343de8be5acSMilan Broz memset(key, '0', key_string_len); 1344de8be5acSMilan Broz 1345de8be5acSMilan Broz return r; 1346e48d4bbfSMilan Broz } 1347e48d4bbfSMilan Broz 1348e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 1349e48d4bbfSMilan Broz { 1350e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1351e48d4bbfSMilan Broz memset(&cc->key, 0, cc->key_size * sizeof(u8)); 1352c0297721SAndi Kleen 1353c0297721SAndi Kleen return crypt_setkey_allcpus(cc); 1354e48d4bbfSMilan Broz } 1355e48d4bbfSMilan Broz 135628513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 135728513fccSMilan Broz { 135828513fccSMilan Broz struct crypt_config *cc = ti->private; 1359c0297721SAndi Kleen struct crypt_cpu *cpu_cc; 1360c0297721SAndi Kleen int cpu; 136128513fccSMilan Broz 136228513fccSMilan Broz ti->private = NULL; 136328513fccSMilan Broz 136428513fccSMilan Broz if (!cc) 136528513fccSMilan Broz return; 136628513fccSMilan Broz 136728513fccSMilan Broz if (cc->io_queue) 136828513fccSMilan Broz destroy_workqueue(cc->io_queue); 136928513fccSMilan Broz if (cc->crypt_queue) 137028513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 137128513fccSMilan Broz 1372c0297721SAndi Kleen if (cc->cpu) 1373c0297721SAndi Kleen for_each_possible_cpu(cpu) { 1374c0297721SAndi Kleen cpu_cc = per_cpu_ptr(cc->cpu, cpu); 1375c0297721SAndi Kleen if (cpu_cc->req) 1376c0297721SAndi Kleen mempool_free(cpu_cc->req, cc->req_pool); 1377c0297721SAndi Kleen } 1378c0297721SAndi Kleen 1379fd2d231fSMikulas Patocka crypt_free_tfms(cc); 1380fd2d231fSMikulas Patocka 138128513fccSMilan Broz if (cc->bs) 138228513fccSMilan Broz bioset_free(cc->bs); 138328513fccSMilan Broz 138428513fccSMilan Broz if (cc->page_pool) 138528513fccSMilan Broz mempool_destroy(cc->page_pool); 138628513fccSMilan Broz if (cc->req_pool) 138728513fccSMilan Broz mempool_destroy(cc->req_pool); 138828513fccSMilan Broz if (cc->io_pool) 138928513fccSMilan Broz mempool_destroy(cc->io_pool); 139028513fccSMilan Broz 139128513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 139228513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 139328513fccSMilan Broz 139428513fccSMilan Broz if (cc->dev) 139528513fccSMilan Broz dm_put_device(ti, cc->dev); 139628513fccSMilan Broz 1397c0297721SAndi Kleen if (cc->cpu) 1398c0297721SAndi Kleen free_percpu(cc->cpu); 1399c0297721SAndi Kleen 14005ebaee6dSMilan Broz kzfree(cc->cipher); 14017dbcd137SMilan Broz kzfree(cc->cipher_string); 140228513fccSMilan Broz 140328513fccSMilan Broz /* Must zero key material before freeing */ 140428513fccSMilan Broz kzfree(cc); 140528513fccSMilan Broz } 140628513fccSMilan Broz 14075ebaee6dSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, 14085ebaee6dSMilan Broz char *cipher_in, char *key) 14091da177e4SLinus Torvalds { 14105ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 1411d1f96423SMilan Broz char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount; 14125ebaee6dSMilan Broz char *cipher_api = NULL; 1413fd2d231fSMikulas Patocka int ret = -EINVAL; 141431998ef1SMikulas Patocka char dummy; 14151da177e4SLinus Torvalds 14165ebaee6dSMilan Broz /* Convert to crypto api definition? */ 14175ebaee6dSMilan Broz if (strchr(cipher_in, '(')) { 14185ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 14191da177e4SLinus Torvalds return -EINVAL; 14201da177e4SLinus Torvalds } 14211da177e4SLinus Torvalds 14227dbcd137SMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL); 14237dbcd137SMilan Broz if (!cc->cipher_string) 14247dbcd137SMilan Broz goto bad_mem; 14257dbcd137SMilan Broz 14265ebaee6dSMilan Broz /* 14275ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 1428d1f96423SMilan Broz * cipher[:keycount]-mode-iv:ivopts 14295ebaee6dSMilan Broz */ 14305ebaee6dSMilan Broz tmp = cipher_in; 1431d1f96423SMilan Broz keycount = strsep(&tmp, "-"); 1432d1f96423SMilan Broz cipher = strsep(&keycount, ":"); 1433d1f96423SMilan Broz 1434d1f96423SMilan Broz if (!keycount) 1435d1f96423SMilan Broz cc->tfms_count = 1; 143631998ef1SMikulas Patocka else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 || 1437d1f96423SMilan Broz !is_power_of_2(cc->tfms_count)) { 1438d1f96423SMilan Broz ti->error = "Bad cipher key count specification"; 1439d1f96423SMilan Broz return -EINVAL; 1440d1f96423SMilan Broz } 1441d1f96423SMilan Broz cc->key_parts = cc->tfms_count; 14425ebaee6dSMilan Broz 14435ebaee6dSMilan Broz cc->cipher = kstrdup(cipher, GFP_KERNEL); 14445ebaee6dSMilan Broz if (!cc->cipher) 14455ebaee6dSMilan Broz goto bad_mem; 14465ebaee6dSMilan Broz 14471da177e4SLinus Torvalds chainmode = strsep(&tmp, "-"); 14481da177e4SLinus Torvalds ivopts = strsep(&tmp, "-"); 14491da177e4SLinus Torvalds ivmode = strsep(&ivopts, ":"); 14501da177e4SLinus Torvalds 14511da177e4SLinus Torvalds if (tmp) 14525ebaee6dSMilan Broz DMWARN("Ignoring unexpected additional cipher options"); 14531da177e4SLinus Torvalds 1454fd2d231fSMikulas Patocka cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)), 1455d1f96423SMilan Broz __alignof__(struct crypt_cpu)); 1456c0297721SAndi Kleen if (!cc->cpu) { 1457c0297721SAndi Kleen ti->error = "Cannot allocate per cpu state"; 1458c0297721SAndi Kleen goto bad_mem; 1459c0297721SAndi Kleen } 1460c0297721SAndi Kleen 14617dbcd137SMilan Broz /* 14627dbcd137SMilan Broz * For compatibility with the original dm-crypt mapping format, if 14637dbcd137SMilan Broz * only the cipher name is supplied, use cbc-plain. 14647dbcd137SMilan Broz */ 14655ebaee6dSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) { 14661da177e4SLinus Torvalds chainmode = "cbc"; 14671da177e4SLinus Torvalds ivmode = "plain"; 14681da177e4SLinus Torvalds } 14691da177e4SLinus Torvalds 1470d1806f6aSHerbert Xu if (strcmp(chainmode, "ecb") && !ivmode) { 14715ebaee6dSMilan Broz ti->error = "IV mechanism required"; 14725ebaee6dSMilan Broz return -EINVAL; 14731da177e4SLinus Torvalds } 14741da177e4SLinus Torvalds 14755ebaee6dSMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 14765ebaee6dSMilan Broz if (!cipher_api) 14775ebaee6dSMilan Broz goto bad_mem; 14785ebaee6dSMilan Broz 14795ebaee6dSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 14805ebaee6dSMilan Broz "%s(%s)", chainmode, cipher); 14815ebaee6dSMilan Broz if (ret < 0) { 14825ebaee6dSMilan Broz kfree(cipher_api); 14835ebaee6dSMilan Broz goto bad_mem; 1484d1806f6aSHerbert Xu } 1485d1806f6aSHerbert Xu 14865ebaee6dSMilan Broz /* Allocate cipher */ 1487fd2d231fSMikulas Patocka ret = crypt_alloc_tfms(cc, cipher_api); 1488d1f96423SMilan Broz if (ret < 0) { 148972d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm"; 149028513fccSMilan Broz goto bad; 14911da177e4SLinus Torvalds } 14921da177e4SLinus Torvalds 14935ebaee6dSMilan Broz /* Initialize and set key */ 14945ebaee6dSMilan Broz ret = crypt_set_key(cc, key); 149528513fccSMilan Broz if (ret < 0) { 14960b430958SMilan Broz ti->error = "Error decoding and setting key"; 149728513fccSMilan Broz goto bad; 14980b430958SMilan Broz } 14990b430958SMilan Broz 15005ebaee6dSMilan Broz /* Initialize IV */ 1501c0297721SAndi Kleen cc->iv_size = crypto_ablkcipher_ivsize(any_tfm(cc)); 15025ebaee6dSMilan Broz if (cc->iv_size) 15035ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 15045ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 15055ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 15065ebaee6dSMilan Broz else if (ivmode) { 15075ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 15085ebaee6dSMilan Broz ivmode = NULL; 15095ebaee6dSMilan Broz } 15105ebaee6dSMilan Broz 15115ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 15121da177e4SLinus Torvalds if (ivmode == NULL) 15131da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 15141da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 15151da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 151661afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 151761afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 15181da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 15191da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 152048527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 152148527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 152246b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 152346b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 152434745785SMilan Broz else if (strcmp(ivmode, "lmk") == 0) { 152534745785SMilan Broz cc->iv_gen_ops = &crypt_iv_lmk_ops; 152634745785SMilan Broz /* Version 2 and 3 is recognised according 152734745785SMilan Broz * to length of provided multi-key string. 152834745785SMilan Broz * If present (version 3), last key is used as IV seed. 152934745785SMilan Broz */ 153034745785SMilan Broz if (cc->key_size % cc->key_parts) 153134745785SMilan Broz cc->key_parts++; 153234745785SMilan Broz } else { 15335ebaee6dSMilan Broz ret = -EINVAL; 153472d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 153528513fccSMilan Broz goto bad; 15361da177e4SLinus Torvalds } 15371da177e4SLinus Torvalds 153828513fccSMilan Broz /* Allocate IV */ 153928513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 154028513fccSMilan Broz ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 154128513fccSMilan Broz if (ret < 0) { 154228513fccSMilan Broz ti->error = "Error creating IV"; 154328513fccSMilan Broz goto bad; 154428513fccSMilan Broz } 154528513fccSMilan Broz } 15461da177e4SLinus Torvalds 154728513fccSMilan Broz /* Initialize IV (set keys for ESSIV etc) */ 154828513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 154928513fccSMilan Broz ret = cc->iv_gen_ops->init(cc); 155028513fccSMilan Broz if (ret < 0) { 1551b95bf2d3SMilan Broz ti->error = "Error initialising IV"; 155228513fccSMilan Broz goto bad; 155328513fccSMilan Broz } 1554b95bf2d3SMilan Broz } 1555b95bf2d3SMilan Broz 15565ebaee6dSMilan Broz ret = 0; 15575ebaee6dSMilan Broz bad: 15585ebaee6dSMilan Broz kfree(cipher_api); 15595ebaee6dSMilan Broz return ret; 15605ebaee6dSMilan Broz 15615ebaee6dSMilan Broz bad_mem: 15625ebaee6dSMilan Broz ti->error = "Cannot allocate cipher strings"; 15635ebaee6dSMilan Broz return -ENOMEM; 15641da177e4SLinus Torvalds } 15655ebaee6dSMilan Broz 15665ebaee6dSMilan Broz /* 15675ebaee6dSMilan Broz * Construct an encryption mapping: 15685ebaee6dSMilan Broz * <cipher> <key> <iv_offset> <dev_path> <start> 15695ebaee6dSMilan Broz */ 15705ebaee6dSMilan Broz static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 15715ebaee6dSMilan Broz { 15725ebaee6dSMilan Broz struct crypt_config *cc; 1573772ae5f5SMilan Broz unsigned int key_size, opt_params; 15745ebaee6dSMilan Broz unsigned long long tmpll; 15755ebaee6dSMilan Broz int ret; 1576772ae5f5SMilan Broz struct dm_arg_set as; 1577772ae5f5SMilan Broz const char *opt_string; 157831998ef1SMikulas Patocka char dummy; 15795ebaee6dSMilan Broz 1580772ae5f5SMilan Broz static struct dm_arg _args[] = { 1581772ae5f5SMilan Broz {0, 1, "Invalid number of feature args"}, 1582772ae5f5SMilan Broz }; 1583772ae5f5SMilan Broz 1584772ae5f5SMilan Broz if (argc < 5) { 15855ebaee6dSMilan Broz ti->error = "Not enough arguments"; 15865ebaee6dSMilan Broz return -EINVAL; 15871da177e4SLinus Torvalds } 15881da177e4SLinus Torvalds 15895ebaee6dSMilan Broz key_size = strlen(argv[1]) >> 1; 15905ebaee6dSMilan Broz 15915ebaee6dSMilan Broz cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 15925ebaee6dSMilan Broz if (!cc) { 15935ebaee6dSMilan Broz ti->error = "Cannot allocate encryption context"; 15945ebaee6dSMilan Broz return -ENOMEM; 15955ebaee6dSMilan Broz } 159669a8cfcdSMilan Broz cc->key_size = key_size; 15975ebaee6dSMilan Broz 15985ebaee6dSMilan Broz ti->private = cc; 15995ebaee6dSMilan Broz ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 16005ebaee6dSMilan Broz if (ret < 0) 16015ebaee6dSMilan Broz goto bad; 16025ebaee6dSMilan Broz 160328513fccSMilan Broz ret = -ENOMEM; 160493d2341cSMatthew Dobson cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 16051da177e4SLinus Torvalds if (!cc->io_pool) { 160672d94861SAlasdair G Kergon ti->error = "Cannot allocate crypt io mempool"; 160728513fccSMilan Broz goto bad; 16081da177e4SLinus Torvalds } 16091da177e4SLinus Torvalds 1610ddd42edfSMilan Broz cc->dmreq_start = sizeof(struct ablkcipher_request); 1611c0297721SAndi Kleen cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc)); 1612ddd42edfSMilan Broz cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment()); 1613c0297721SAndi Kleen cc->dmreq_start += crypto_ablkcipher_alignmask(any_tfm(cc)) & 16143a7f6c99SMilan Broz ~(crypto_tfm_ctx_alignment() - 1); 1615ddd42edfSMilan Broz 1616ddd42edfSMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1617ddd42edfSMilan Broz sizeof(struct dm_crypt_request) + cc->iv_size); 1618ddd42edfSMilan Broz if (!cc->req_pool) { 1619ddd42edfSMilan Broz ti->error = "Cannot allocate crypt request mempool"; 162028513fccSMilan Broz goto bad; 1621ddd42edfSMilan Broz } 1622ddd42edfSMilan Broz 1623a19b27ceSMatthew Dobson cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 16241da177e4SLinus Torvalds if (!cc->page_pool) { 162572d94861SAlasdair G Kergon ti->error = "Cannot allocate page mempool"; 162628513fccSMilan Broz goto bad; 16271da177e4SLinus Torvalds } 16281da177e4SLinus Torvalds 1629bb799ca0SJens Axboe cc->bs = bioset_create(MIN_IOS, 0); 16306a24c718SMilan Broz if (!cc->bs) { 16316a24c718SMilan Broz ti->error = "Cannot allocate crypt bioset"; 163228513fccSMilan Broz goto bad; 16336a24c718SMilan Broz } 16346a24c718SMilan Broz 163528513fccSMilan Broz ret = -EINVAL; 163631998ef1SMikulas Patocka if (sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) { 163772d94861SAlasdair G Kergon ti->error = "Invalid iv_offset sector"; 163828513fccSMilan Broz goto bad; 16391da177e4SLinus Torvalds } 16404ee218cdSAndrew Morton cc->iv_offset = tmpll; 16411da177e4SLinus Torvalds 164228513fccSMilan Broz if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev)) { 164328513fccSMilan Broz ti->error = "Device lookup failed"; 164428513fccSMilan Broz goto bad; 164528513fccSMilan Broz } 164628513fccSMilan Broz 164731998ef1SMikulas Patocka if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1) { 164872d94861SAlasdair G Kergon ti->error = "Invalid device sector"; 164928513fccSMilan Broz goto bad; 16501da177e4SLinus Torvalds } 16514ee218cdSAndrew Morton cc->start = tmpll; 16521da177e4SLinus Torvalds 1653772ae5f5SMilan Broz argv += 5; 1654772ae5f5SMilan Broz argc -= 5; 1655772ae5f5SMilan Broz 1656772ae5f5SMilan Broz /* Optional parameters */ 1657772ae5f5SMilan Broz if (argc) { 1658772ae5f5SMilan Broz as.argc = argc; 1659772ae5f5SMilan Broz as.argv = argv; 1660772ae5f5SMilan Broz 1661772ae5f5SMilan Broz ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 1662772ae5f5SMilan Broz if (ret) 1663772ae5f5SMilan Broz goto bad; 1664772ae5f5SMilan Broz 1665772ae5f5SMilan Broz opt_string = dm_shift_arg(&as); 1666772ae5f5SMilan Broz 1667772ae5f5SMilan Broz if (opt_params == 1 && opt_string && 1668772ae5f5SMilan Broz !strcasecmp(opt_string, "allow_discards")) 1669772ae5f5SMilan Broz ti->num_discard_requests = 1; 1670772ae5f5SMilan Broz else if (opt_params) { 1671772ae5f5SMilan Broz ret = -EINVAL; 1672772ae5f5SMilan Broz ti->error = "Invalid feature arguments"; 1673772ae5f5SMilan Broz goto bad; 1674772ae5f5SMilan Broz } 1675772ae5f5SMilan Broz } 1676772ae5f5SMilan Broz 167728513fccSMilan Broz ret = -ENOMEM; 1678c0297721SAndi Kleen cc->io_queue = alloc_workqueue("kcryptd_io", 1679c0297721SAndi Kleen WQ_NON_REENTRANT| 1680c0297721SAndi Kleen WQ_MEM_RECLAIM, 1681c0297721SAndi Kleen 1); 1682cabf08e4SMilan Broz if (!cc->io_queue) { 1683cabf08e4SMilan Broz ti->error = "Couldn't create kcryptd io queue"; 168428513fccSMilan Broz goto bad; 1685cabf08e4SMilan Broz } 1686cabf08e4SMilan Broz 1687c0297721SAndi Kleen cc->crypt_queue = alloc_workqueue("kcryptd", 1688c0297721SAndi Kleen WQ_NON_REENTRANT| 1689c0297721SAndi Kleen WQ_CPU_INTENSIVE| 1690c0297721SAndi Kleen WQ_MEM_RECLAIM, 1691c0297721SAndi Kleen 1); 1692cabf08e4SMilan Broz if (!cc->crypt_queue) { 16939934a8beSMilan Broz ti->error = "Couldn't create kcryptd queue"; 169428513fccSMilan Broz goto bad; 16959934a8beSMilan Broz } 16969934a8beSMilan Broz 1697647c7db1SMikulas Patocka ti->num_flush_requests = 1; 1698983c7db3SMilan Broz ti->discard_zeroes_data_unsupported = 1; 1699983c7db3SMilan Broz 17001da177e4SLinus Torvalds return 0; 17011da177e4SLinus Torvalds 170228513fccSMilan Broz bad: 170328513fccSMilan Broz crypt_dtr(ti); 170428513fccSMilan Broz return ret; 17051da177e4SLinus Torvalds } 17061da177e4SLinus Torvalds 17071da177e4SLinus Torvalds static int crypt_map(struct dm_target *ti, struct bio *bio, 17081da177e4SLinus Torvalds union map_info *map_context) 17091da177e4SLinus Torvalds { 1710028867acSAlasdair G Kergon struct dm_crypt_io *io; 1711647c7db1SMikulas Patocka struct crypt_config *cc; 1712647c7db1SMikulas Patocka 1713772ae5f5SMilan Broz /* 1714772ae5f5SMilan Broz * If bio is REQ_FLUSH or REQ_DISCARD, just bypass crypt queues. 1715772ae5f5SMilan Broz * - for REQ_FLUSH device-mapper core ensures that no IO is in-flight 1716772ae5f5SMilan Broz * - for REQ_DISCARD caller must use flush if IO ordering matters 1717772ae5f5SMilan Broz */ 1718772ae5f5SMilan Broz if (unlikely(bio->bi_rw & (REQ_FLUSH | REQ_DISCARD))) { 1719647c7db1SMikulas Patocka cc = ti->private; 1720647c7db1SMikulas Patocka bio->bi_bdev = cc->dev->bdev; 1721772ae5f5SMilan Broz if (bio_sectors(bio)) 1722772ae5f5SMilan Broz bio->bi_sector = cc->start + dm_target_offset(ti, bio->bi_sector); 1723647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 1724647c7db1SMikulas Patocka } 17251da177e4SLinus Torvalds 1726b441a262SAlasdair G Kergon io = crypt_io_alloc(ti, bio, dm_target_offset(ti, bio->bi_sector)); 1727cabf08e4SMilan Broz 172820c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 172920c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOWAIT)) 173023541d2dSMilan Broz kcryptd_queue_io(io); 173120c82538SMilan Broz } else 1732cabf08e4SMilan Broz kcryptd_queue_crypt(io); 17331da177e4SLinus Torvalds 1734d2a7ad29SKiyoshi Ueda return DM_MAPIO_SUBMITTED; 17351da177e4SLinus Torvalds } 17361da177e4SLinus Torvalds 17371da177e4SLinus Torvalds static int crypt_status(struct dm_target *ti, status_type_t type, 17381da177e4SLinus Torvalds char *result, unsigned int maxlen) 17391da177e4SLinus Torvalds { 17405ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 17411da177e4SLinus Torvalds unsigned int sz = 0; 17421da177e4SLinus Torvalds 17431da177e4SLinus Torvalds switch (type) { 17441da177e4SLinus Torvalds case STATUSTYPE_INFO: 17451da177e4SLinus Torvalds result[0] = '\0'; 17461da177e4SLinus Torvalds break; 17471da177e4SLinus Torvalds 17481da177e4SLinus Torvalds case STATUSTYPE_TABLE: 17497dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string); 17501da177e4SLinus Torvalds 17511da177e4SLinus Torvalds if (cc->key_size > 0) { 17521da177e4SLinus Torvalds if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 17531da177e4SLinus Torvalds return -ENOMEM; 17541da177e4SLinus Torvalds 17551da177e4SLinus Torvalds crypt_encode_key(result + sz, cc->key, cc->key_size); 17561da177e4SLinus Torvalds sz += cc->key_size << 1; 17571da177e4SLinus Torvalds } else { 17581da177e4SLinus Torvalds if (sz >= maxlen) 17591da177e4SLinus Torvalds return -ENOMEM; 17601da177e4SLinus Torvalds result[sz++] = '-'; 17611da177e4SLinus Torvalds } 17621da177e4SLinus Torvalds 17634ee218cdSAndrew Morton DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 17644ee218cdSAndrew Morton cc->dev->name, (unsigned long long)cc->start); 1765772ae5f5SMilan Broz 1766772ae5f5SMilan Broz if (ti->num_discard_requests) 1767772ae5f5SMilan Broz DMEMIT(" 1 allow_discards"); 1768772ae5f5SMilan Broz 17691da177e4SLinus Torvalds break; 17701da177e4SLinus Torvalds } 17711da177e4SLinus Torvalds return 0; 17721da177e4SLinus Torvalds } 17731da177e4SLinus Torvalds 1774e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 1775e48d4bbfSMilan Broz { 1776e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1777e48d4bbfSMilan Broz 1778e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1779e48d4bbfSMilan Broz } 1780e48d4bbfSMilan Broz 1781e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 1782e48d4bbfSMilan Broz { 1783e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1784e48d4bbfSMilan Broz 1785e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1786e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 1787e48d4bbfSMilan Broz return -EAGAIN; 1788e48d4bbfSMilan Broz } 1789e48d4bbfSMilan Broz 1790e48d4bbfSMilan Broz return 0; 1791e48d4bbfSMilan Broz } 1792e48d4bbfSMilan Broz 1793e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 1794e48d4bbfSMilan Broz { 1795e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1796e48d4bbfSMilan Broz 1797e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1798e48d4bbfSMilan Broz } 1799e48d4bbfSMilan Broz 1800e48d4bbfSMilan Broz /* Message interface 1801e48d4bbfSMilan Broz * key set <key> 1802e48d4bbfSMilan Broz * key wipe 1803e48d4bbfSMilan Broz */ 1804e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1805e48d4bbfSMilan Broz { 1806e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1807542da317SMilan Broz int ret = -EINVAL; 1808e48d4bbfSMilan Broz 1809e48d4bbfSMilan Broz if (argc < 2) 1810e48d4bbfSMilan Broz goto error; 1811e48d4bbfSMilan Broz 1812498f0103SMike Snitzer if (!strcasecmp(argv[0], "key")) { 1813e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1814e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 1815e48d4bbfSMilan Broz return -EINVAL; 1816e48d4bbfSMilan Broz } 1817498f0103SMike Snitzer if (argc == 3 && !strcasecmp(argv[1], "set")) { 1818542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 1819542da317SMilan Broz if (ret) 1820542da317SMilan Broz return ret; 1821542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 1822542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 1823542da317SMilan Broz return ret; 1824542da317SMilan Broz } 1825498f0103SMike Snitzer if (argc == 2 && !strcasecmp(argv[1], "wipe")) { 1826542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 1827542da317SMilan Broz ret = cc->iv_gen_ops->wipe(cc); 1828542da317SMilan Broz if (ret) 1829542da317SMilan Broz return ret; 1830542da317SMilan Broz } 1831e48d4bbfSMilan Broz return crypt_wipe_key(cc); 1832e48d4bbfSMilan Broz } 1833542da317SMilan Broz } 1834e48d4bbfSMilan Broz 1835e48d4bbfSMilan Broz error: 1836e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 1837e48d4bbfSMilan Broz return -EINVAL; 1838e48d4bbfSMilan Broz } 1839e48d4bbfSMilan Broz 1840d41e26b9SMilan Broz static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 1841d41e26b9SMilan Broz struct bio_vec *biovec, int max_size) 1842d41e26b9SMilan Broz { 1843d41e26b9SMilan Broz struct crypt_config *cc = ti->private; 1844d41e26b9SMilan Broz struct request_queue *q = bdev_get_queue(cc->dev->bdev); 1845d41e26b9SMilan Broz 1846d41e26b9SMilan Broz if (!q->merge_bvec_fn) 1847d41e26b9SMilan Broz return max_size; 1848d41e26b9SMilan Broz 1849d41e26b9SMilan Broz bvm->bi_bdev = cc->dev->bdev; 1850b441a262SAlasdair G Kergon bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector); 1851d41e26b9SMilan Broz 1852d41e26b9SMilan Broz return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 1853d41e26b9SMilan Broz } 1854d41e26b9SMilan Broz 1855af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 1856af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 1857af4874e0SMike Snitzer { 1858af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 1859af4874e0SMike Snitzer 18605dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 1861af4874e0SMike Snitzer } 1862af4874e0SMike Snitzer 18631da177e4SLinus Torvalds static struct target_type crypt_target = { 18641da177e4SLinus Torvalds .name = "crypt", 1865772ae5f5SMilan Broz .version = {1, 11, 0}, 18661da177e4SLinus Torvalds .module = THIS_MODULE, 18671da177e4SLinus Torvalds .ctr = crypt_ctr, 18681da177e4SLinus Torvalds .dtr = crypt_dtr, 18691da177e4SLinus Torvalds .map = crypt_map, 18701da177e4SLinus Torvalds .status = crypt_status, 1871e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 1872e48d4bbfSMilan Broz .preresume = crypt_preresume, 1873e48d4bbfSMilan Broz .resume = crypt_resume, 1874e48d4bbfSMilan Broz .message = crypt_message, 1875d41e26b9SMilan Broz .merge = crypt_merge, 1876af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 18771da177e4SLinus Torvalds }; 18781da177e4SLinus Torvalds 18791da177e4SLinus Torvalds static int __init dm_crypt_init(void) 18801da177e4SLinus Torvalds { 18811da177e4SLinus Torvalds int r; 18821da177e4SLinus Torvalds 1883028867acSAlasdair G Kergon _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 18841da177e4SLinus Torvalds if (!_crypt_io_pool) 18851da177e4SLinus Torvalds return -ENOMEM; 18861da177e4SLinus Torvalds 18871da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 18881da177e4SLinus Torvalds if (r < 0) { 188972d94861SAlasdair G Kergon DMERR("register failed %d", r); 18909934a8beSMilan Broz kmem_cache_destroy(_crypt_io_pool); 18911da177e4SLinus Torvalds } 18921da177e4SLinus Torvalds 18931da177e4SLinus Torvalds return r; 18941da177e4SLinus Torvalds } 18951da177e4SLinus Torvalds 18961da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 18971da177e4SLinus Torvalds { 189810d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 18991da177e4SLinus Torvalds kmem_cache_destroy(_crypt_io_pool); 19001da177e4SLinus Torvalds } 19011da177e4SLinus Torvalds 19021da177e4SLinus Torvalds module_init(dm_crypt_init); 19031da177e4SLinus Torvalds module_exit(dm_crypt_exit); 19041da177e4SLinus Torvalds 19051da177e4SLinus Torvalds MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 19061da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 19071da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1908