11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4542da317SMilan Broz * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 943d69034SMilan Broz #include <linux/completion.h> 10d1806f6aSHerbert Xu #include <linux/err.h> 111da177e4SLinus Torvalds #include <linux/module.h> 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/kernel.h> 141da177e4SLinus Torvalds #include <linux/bio.h> 151da177e4SLinus Torvalds #include <linux/blkdev.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/crypto.h> 191da177e4SLinus Torvalds #include <linux/workqueue.h> 203fcfab16SAndrew Morton #include <linux/backing-dev.h> 21c0297721SAndi Kleen #include <linux/percpu.h> 221da177e4SLinus Torvalds #include <asm/atomic.h> 23378f058cSDavid Hardeman #include <linux/scatterlist.h> 241da177e4SLinus Torvalds #include <asm/page.h> 2548527fa7SRik Snel #include <asm/unaligned.h> 2634745785SMilan Broz #include <crypto/hash.h> 2734745785SMilan Broz #include <crypto/md5.h> 2834745785SMilan Broz #include <crypto/algapi.h> 291da177e4SLinus Torvalds 30586e80e6SMikulas Patocka #include <linux/device-mapper.h> 311da177e4SLinus Torvalds 3272d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 33e48d4bbfSMilan Broz #define MESG_STR(x) x, sizeof(x) 341da177e4SLinus Torvalds 351da177e4SLinus Torvalds /* 361da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 371da177e4SLinus Torvalds */ 381da177e4SLinus Torvalds struct convert_context { 3943d69034SMilan Broz struct completion restart; 401da177e4SLinus Torvalds struct bio *bio_in; 411da177e4SLinus Torvalds struct bio *bio_out; 421da177e4SLinus Torvalds unsigned int offset_in; 431da177e4SLinus Torvalds unsigned int offset_out; 441da177e4SLinus Torvalds unsigned int idx_in; 451da177e4SLinus Torvalds unsigned int idx_out; 461da177e4SLinus Torvalds sector_t sector; 4743d69034SMilan Broz atomic_t pending; 481da177e4SLinus Torvalds }; 491da177e4SLinus Torvalds 5053017030SMilan Broz /* 5153017030SMilan Broz * per bio private data 5253017030SMilan Broz */ 5353017030SMilan Broz struct dm_crypt_io { 5453017030SMilan Broz struct dm_target *target; 5553017030SMilan Broz struct bio *base_bio; 5653017030SMilan Broz struct work_struct work; 5753017030SMilan Broz 5853017030SMilan Broz struct convert_context ctx; 5953017030SMilan Broz 6053017030SMilan Broz atomic_t pending; 6153017030SMilan Broz int error; 620c395b0fSMilan Broz sector_t sector; 63393b47efSMilan Broz struct dm_crypt_io *base_io; 6453017030SMilan Broz }; 6553017030SMilan Broz 6601482b76SMilan Broz struct dm_crypt_request { 67b2174eebSHuang Ying struct convert_context *ctx; 6801482b76SMilan Broz struct scatterlist sg_in; 6901482b76SMilan Broz struct scatterlist sg_out; 702dc5327dSMilan Broz sector_t iv_sector; 7101482b76SMilan Broz }; 7201482b76SMilan Broz 731da177e4SLinus Torvalds struct crypt_config; 741da177e4SLinus Torvalds 751da177e4SLinus Torvalds struct crypt_iv_operations { 761da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 771da177e4SLinus Torvalds const char *opts); 781da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 79b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 80542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 812dc5327dSMilan Broz int (*generator)(struct crypt_config *cc, u8 *iv, 822dc5327dSMilan Broz struct dm_crypt_request *dmreq); 832dc5327dSMilan Broz int (*post)(struct crypt_config *cc, u8 *iv, 842dc5327dSMilan Broz struct dm_crypt_request *dmreq); 851da177e4SLinus Torvalds }; 861da177e4SLinus Torvalds 8760473592SMilan Broz struct iv_essiv_private { 88b95bf2d3SMilan Broz struct crypto_hash *hash_tfm; 89b95bf2d3SMilan Broz u8 *salt; 9060473592SMilan Broz }; 9160473592SMilan Broz 9260473592SMilan Broz struct iv_benbi_private { 9360473592SMilan Broz int shift; 9460473592SMilan Broz }; 9560473592SMilan Broz 9634745785SMilan Broz #define LMK_SEED_SIZE 64 /* hash + 0 */ 9734745785SMilan Broz struct iv_lmk_private { 9834745785SMilan Broz struct crypto_shash *hash_tfm; 9934745785SMilan Broz u8 *seed; 10034745785SMilan Broz }; 10134745785SMilan Broz 1021da177e4SLinus Torvalds /* 1031da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 1041da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 1051da177e4SLinus Torvalds */ 106e48d4bbfSMilan Broz enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 107c0297721SAndi Kleen 108c0297721SAndi Kleen /* 109c0297721SAndi Kleen * Duplicated per-CPU state for cipher. 110c0297721SAndi Kleen */ 111c0297721SAndi Kleen struct crypt_cpu { 112c0297721SAndi Kleen struct ablkcipher_request *req; 113c0297721SAndi Kleen /* ESSIV: struct crypto_cipher *essiv_tfm */ 114c0297721SAndi Kleen void *iv_private; 115d1f96423SMilan Broz struct crypto_ablkcipher *tfms[0]; 116c0297721SAndi Kleen }; 117c0297721SAndi Kleen 118c0297721SAndi Kleen /* 119c0297721SAndi Kleen * The fields in here must be read only after initialization, 120c0297721SAndi Kleen * changing state should be in crypt_cpu. 121c0297721SAndi Kleen */ 1221da177e4SLinus Torvalds struct crypt_config { 1231da177e4SLinus Torvalds struct dm_dev *dev; 1241da177e4SLinus Torvalds sector_t start; 1251da177e4SLinus Torvalds 1261da177e4SLinus Torvalds /* 127ddd42edfSMilan Broz * pool for per bio private data, crypto requests and 128ddd42edfSMilan Broz * encryption requeusts/buffer pages 1291da177e4SLinus Torvalds */ 1301da177e4SLinus Torvalds mempool_t *io_pool; 131ddd42edfSMilan Broz mempool_t *req_pool; 1321da177e4SLinus Torvalds mempool_t *page_pool; 1336a24c718SMilan Broz struct bio_set *bs; 1341da177e4SLinus Torvalds 135cabf08e4SMilan Broz struct workqueue_struct *io_queue; 136cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1373f1e9070SMilan Broz 1385ebaee6dSMilan Broz char *cipher; 1397dbcd137SMilan Broz char *cipher_string; 1405ebaee6dSMilan Broz 1411da177e4SLinus Torvalds struct crypt_iv_operations *iv_gen_ops; 14279066ad3SHerbert Xu union { 14360473592SMilan Broz struct iv_essiv_private essiv; 14460473592SMilan Broz struct iv_benbi_private benbi; 14534745785SMilan Broz struct iv_lmk_private lmk; 14679066ad3SHerbert Xu } iv_gen_private; 1471da177e4SLinus Torvalds sector_t iv_offset; 1481da177e4SLinus Torvalds unsigned int iv_size; 1491da177e4SLinus Torvalds 150ddd42edfSMilan Broz /* 151c0297721SAndi Kleen * Duplicated per cpu state. Access through 152c0297721SAndi Kleen * per_cpu_ptr() only. 153c0297721SAndi Kleen */ 154c0297721SAndi Kleen struct crypt_cpu __percpu *cpu; 155d1f96423SMilan Broz unsigned tfms_count; 156c0297721SAndi Kleen 157c0297721SAndi Kleen /* 158ddd42edfSMilan Broz * Layout of each crypto request: 159ddd42edfSMilan Broz * 160ddd42edfSMilan Broz * struct ablkcipher_request 161ddd42edfSMilan Broz * context 162ddd42edfSMilan Broz * padding 163ddd42edfSMilan Broz * struct dm_crypt_request 164ddd42edfSMilan Broz * padding 165ddd42edfSMilan Broz * IV 166ddd42edfSMilan Broz * 167ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 168ddd42edfSMilan Broz * correctly aligned. 169ddd42edfSMilan Broz */ 170ddd42edfSMilan Broz unsigned int dmreq_start; 171ddd42edfSMilan Broz 172e48d4bbfSMilan Broz unsigned long flags; 1731da177e4SLinus Torvalds unsigned int key_size; 174d1f96423SMilan Broz unsigned int key_parts; 1751da177e4SLinus Torvalds u8 key[0]; 1761da177e4SLinus Torvalds }; 1771da177e4SLinus Torvalds 1786a24c718SMilan Broz #define MIN_IOS 16 1791da177e4SLinus Torvalds #define MIN_POOL_PAGES 32 1801da177e4SLinus Torvalds #define MIN_BIO_PAGES 8 1811da177e4SLinus Torvalds 182e18b890bSChristoph Lameter static struct kmem_cache *_crypt_io_pool; 1831da177e4SLinus Torvalds 184028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 185395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 1862dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq); 187027581f3SOlaf Kirch 188c0297721SAndi Kleen static struct crypt_cpu *this_crypt_config(struct crypt_config *cc) 189c0297721SAndi Kleen { 190c0297721SAndi Kleen return this_cpu_ptr(cc->cpu); 191c0297721SAndi Kleen } 192c0297721SAndi Kleen 193c0297721SAndi Kleen /* 194c0297721SAndi Kleen * Use this to access cipher attributes that are the same for each CPU. 195c0297721SAndi Kleen */ 196c0297721SAndi Kleen static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc) 197c0297721SAndi Kleen { 198d1f96423SMilan Broz return __this_cpu_ptr(cc->cpu)->tfms[0]; 199c0297721SAndi Kleen } 200c0297721SAndi Kleen 2011da177e4SLinus Torvalds /* 2021da177e4SLinus Torvalds * Different IV generation algorithms: 2031da177e4SLinus Torvalds * 2043c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 2053a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 2061da177e4SLinus Torvalds * 20761afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 20861afef61SMilan Broz * number, padded with zeros if necessary. 20961afef61SMilan Broz * 2103c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 2111da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 2121da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 2131da177e4SLinus Torvalds * 21448527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 21548527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 21648527fa7SRik Snel * 21746b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 21846b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 21946b47730SLudwig Nussel * 22034745785SMilan Broz * lmk: Compatible implementation of the block chaining mode used 22134745785SMilan Broz * by the Loop-AES block device encryption system 22234745785SMilan Broz * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/ 22334745785SMilan Broz * It operates on full 512 byte sectors and uses CBC 22434745785SMilan Broz * with an IV derived from the sector number, the data and 22534745785SMilan Broz * optionally extra IV seed. 22634745785SMilan Broz * This means that after decryption the first block 22734745785SMilan Broz * of sector must be tweaked according to decrypted data. 22834745785SMilan Broz * Loop-AES can use three encryption schemes: 22934745785SMilan Broz * version 1: is plain aes-cbc mode 23034745785SMilan Broz * version 2: uses 64 multikey scheme with lmk IV generator 23134745785SMilan Broz * version 3: the same as version 2 with additional IV seed 23234745785SMilan Broz * (it uses 65 keys, last key is used as IV seed) 23334745785SMilan Broz * 2341da177e4SLinus Torvalds * plumb: unimplemented, see: 2351da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 2361da177e4SLinus Torvalds */ 2371da177e4SLinus Torvalds 2382dc5327dSMilan Broz static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, 2392dc5327dSMilan Broz struct dm_crypt_request *dmreq) 2401da177e4SLinus Torvalds { 2411da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 2422dc5327dSMilan Broz *(u32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff); 2431da177e4SLinus Torvalds 2441da177e4SLinus Torvalds return 0; 2451da177e4SLinus Torvalds } 2461da177e4SLinus Torvalds 24761afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 2482dc5327dSMilan Broz struct dm_crypt_request *dmreq) 24961afef61SMilan Broz { 25061afef61SMilan Broz memset(iv, 0, cc->iv_size); 2512dc5327dSMilan Broz *(u64 *)iv = cpu_to_le64(dmreq->iv_sector); 25261afef61SMilan Broz 25361afef61SMilan Broz return 0; 25461afef61SMilan Broz } 25561afef61SMilan Broz 256b95bf2d3SMilan Broz /* Initialise ESSIV - compute salt but no local memory allocations */ 257b95bf2d3SMilan Broz static int crypt_iv_essiv_init(struct crypt_config *cc) 258b95bf2d3SMilan Broz { 259b95bf2d3SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 260b95bf2d3SMilan Broz struct hash_desc desc; 261b95bf2d3SMilan Broz struct scatterlist sg; 262c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 263c0297721SAndi Kleen int err, cpu; 264b95bf2d3SMilan Broz 265b95bf2d3SMilan Broz sg_init_one(&sg, cc->key, cc->key_size); 266b95bf2d3SMilan Broz desc.tfm = essiv->hash_tfm; 267b95bf2d3SMilan Broz desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 268b95bf2d3SMilan Broz 269b95bf2d3SMilan Broz err = crypto_hash_digest(&desc, &sg, cc->key_size, essiv->salt); 270b95bf2d3SMilan Broz if (err) 271b95bf2d3SMilan Broz return err; 272b95bf2d3SMilan Broz 273c0297721SAndi Kleen for_each_possible_cpu(cpu) { 274c0297721SAndi Kleen essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private, 275c0297721SAndi Kleen 276c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, essiv->salt, 277b95bf2d3SMilan Broz crypto_hash_digestsize(essiv->hash_tfm)); 278c0297721SAndi Kleen if (err) 279c0297721SAndi Kleen return err; 280c0297721SAndi Kleen } 281c0297721SAndi Kleen 282c0297721SAndi Kleen return 0; 283b95bf2d3SMilan Broz } 284b95bf2d3SMilan Broz 285542da317SMilan Broz /* Wipe salt and reset key derived from volume key */ 286542da317SMilan Broz static int crypt_iv_essiv_wipe(struct crypt_config *cc) 287542da317SMilan Broz { 288542da317SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 289542da317SMilan Broz unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm); 290c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 291c0297721SAndi Kleen int cpu, r, err = 0; 292542da317SMilan Broz 293542da317SMilan Broz memset(essiv->salt, 0, salt_size); 294542da317SMilan Broz 295c0297721SAndi Kleen for_each_possible_cpu(cpu) { 296c0297721SAndi Kleen essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private; 297c0297721SAndi Kleen r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size); 298c0297721SAndi Kleen if (r) 299c0297721SAndi Kleen err = r; 300c0297721SAndi Kleen } 301c0297721SAndi Kleen 302c0297721SAndi Kleen return err; 303c0297721SAndi Kleen } 304c0297721SAndi Kleen 305c0297721SAndi Kleen /* Set up per cpu cipher state */ 306c0297721SAndi Kleen static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc, 307c0297721SAndi Kleen struct dm_target *ti, 308c0297721SAndi Kleen u8 *salt, unsigned saltsize) 309c0297721SAndi Kleen { 310c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 311c0297721SAndi Kleen int err; 312c0297721SAndi Kleen 313c0297721SAndi Kleen /* Setup the essiv_tfm with the given salt */ 314c0297721SAndi Kleen essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 315c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 316c0297721SAndi Kleen ti->error = "Error allocating crypto tfm for ESSIV"; 317c0297721SAndi Kleen return essiv_tfm; 318c0297721SAndi Kleen } 319c0297721SAndi Kleen 320c0297721SAndi Kleen if (crypto_cipher_blocksize(essiv_tfm) != 321c0297721SAndi Kleen crypto_ablkcipher_ivsize(any_tfm(cc))) { 322c0297721SAndi Kleen ti->error = "Block size of ESSIV cipher does " 323c0297721SAndi Kleen "not match IV size of block cipher"; 324c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 325c0297721SAndi Kleen return ERR_PTR(-EINVAL); 326c0297721SAndi Kleen } 327c0297721SAndi Kleen 328c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 329c0297721SAndi Kleen if (err) { 330c0297721SAndi Kleen ti->error = "Failed to set key for ESSIV cipher"; 331c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 332c0297721SAndi Kleen return ERR_PTR(err); 333c0297721SAndi Kleen } 334c0297721SAndi Kleen 335c0297721SAndi Kleen return essiv_tfm; 336542da317SMilan Broz } 337542da317SMilan Broz 33860473592SMilan Broz static void crypt_iv_essiv_dtr(struct crypt_config *cc) 33960473592SMilan Broz { 340c0297721SAndi Kleen int cpu; 341c0297721SAndi Kleen struct crypt_cpu *cpu_cc; 342c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 34360473592SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 34460473592SMilan Broz 345b95bf2d3SMilan Broz crypto_free_hash(essiv->hash_tfm); 346b95bf2d3SMilan Broz essiv->hash_tfm = NULL; 347b95bf2d3SMilan Broz 348b95bf2d3SMilan Broz kzfree(essiv->salt); 349b95bf2d3SMilan Broz essiv->salt = NULL; 350c0297721SAndi Kleen 351c0297721SAndi Kleen for_each_possible_cpu(cpu) { 352c0297721SAndi Kleen cpu_cc = per_cpu_ptr(cc->cpu, cpu); 353c0297721SAndi Kleen essiv_tfm = cpu_cc->iv_private; 354c0297721SAndi Kleen 355c0297721SAndi Kleen if (essiv_tfm) 356c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 357c0297721SAndi Kleen 358c0297721SAndi Kleen cpu_cc->iv_private = NULL; 359c0297721SAndi Kleen } 36060473592SMilan Broz } 36160473592SMilan Broz 3621da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 3631da177e4SLinus Torvalds const char *opts) 3641da177e4SLinus Torvalds { 3655861f1beSMilan Broz struct crypto_cipher *essiv_tfm = NULL; 3665861f1beSMilan Broz struct crypto_hash *hash_tfm = NULL; 3675861f1beSMilan Broz u8 *salt = NULL; 368c0297721SAndi Kleen int err, cpu; 3691da177e4SLinus Torvalds 3705861f1beSMilan Broz if (!opts) { 37172d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 3721da177e4SLinus Torvalds return -EINVAL; 3731da177e4SLinus Torvalds } 3741da177e4SLinus Torvalds 375b95bf2d3SMilan Broz /* Allocate hash algorithm */ 37635058687SHerbert Xu hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 37735058687SHerbert Xu if (IS_ERR(hash_tfm)) { 37872d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 3795861f1beSMilan Broz err = PTR_ERR(hash_tfm); 3805861f1beSMilan Broz goto bad; 3811da177e4SLinus Torvalds } 3821da177e4SLinus Torvalds 383b95bf2d3SMilan Broz salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL); 3845861f1beSMilan Broz if (!salt) { 38572d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 3865861f1beSMilan Broz err = -ENOMEM; 3875861f1beSMilan Broz goto bad; 3881da177e4SLinus Torvalds } 3891da177e4SLinus Torvalds 390b95bf2d3SMilan Broz cc->iv_gen_private.essiv.salt = salt; 391b95bf2d3SMilan Broz cc->iv_gen_private.essiv.hash_tfm = hash_tfm; 392b95bf2d3SMilan Broz 393c0297721SAndi Kleen for_each_possible_cpu(cpu) { 394c0297721SAndi Kleen essiv_tfm = setup_essiv_cpu(cc, ti, salt, 395c0297721SAndi Kleen crypto_hash_digestsize(hash_tfm)); 396c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 397c0297721SAndi Kleen crypt_iv_essiv_dtr(cc); 398c0297721SAndi Kleen return PTR_ERR(essiv_tfm); 399c0297721SAndi Kleen } 400c0297721SAndi Kleen per_cpu_ptr(cc->cpu, cpu)->iv_private = essiv_tfm; 401c0297721SAndi Kleen } 402c0297721SAndi Kleen 4031da177e4SLinus Torvalds return 0; 4045861f1beSMilan Broz 4055861f1beSMilan Broz bad: 4065861f1beSMilan Broz if (hash_tfm && !IS_ERR(hash_tfm)) 4075861f1beSMilan Broz crypto_free_hash(hash_tfm); 408b95bf2d3SMilan Broz kfree(salt); 4095861f1beSMilan Broz return err; 4101da177e4SLinus Torvalds } 4111da177e4SLinus Torvalds 4122dc5327dSMilan Broz static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, 4132dc5327dSMilan Broz struct dm_crypt_request *dmreq) 4141da177e4SLinus Torvalds { 415c0297721SAndi Kleen struct crypto_cipher *essiv_tfm = this_crypt_config(cc)->iv_private; 416c0297721SAndi Kleen 4171da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 4182dc5327dSMilan Broz *(u64 *)iv = cpu_to_le64(dmreq->iv_sector); 419c0297721SAndi Kleen crypto_cipher_encrypt_one(essiv_tfm, iv, iv); 420c0297721SAndi Kleen 4211da177e4SLinus Torvalds return 0; 4221da177e4SLinus Torvalds } 4231da177e4SLinus Torvalds 42448527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 42548527fa7SRik Snel const char *opts) 42648527fa7SRik Snel { 427c0297721SAndi Kleen unsigned bs = crypto_ablkcipher_blocksize(any_tfm(cc)); 428f0d1b0b3SDavid Howells int log = ilog2(bs); 42948527fa7SRik Snel 43048527fa7SRik Snel /* we need to calculate how far we must shift the sector count 43148527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 43248527fa7SRik Snel 43348527fa7SRik Snel if (1 << log != bs) { 43448527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 43548527fa7SRik Snel return -EINVAL; 43648527fa7SRik Snel } 43748527fa7SRik Snel 43848527fa7SRik Snel if (log > 9) { 43948527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 44048527fa7SRik Snel return -EINVAL; 44148527fa7SRik Snel } 44248527fa7SRik Snel 44360473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 44448527fa7SRik Snel 44548527fa7SRik Snel return 0; 44648527fa7SRik Snel } 44748527fa7SRik Snel 44848527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 44948527fa7SRik Snel { 45048527fa7SRik Snel } 45148527fa7SRik Snel 4522dc5327dSMilan Broz static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, 4532dc5327dSMilan Broz struct dm_crypt_request *dmreq) 45448527fa7SRik Snel { 45579066ad3SHerbert Xu __be64 val; 45679066ad3SHerbert Xu 45748527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 45879066ad3SHerbert Xu 4592dc5327dSMilan Broz val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1); 46079066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 46148527fa7SRik Snel 4621da177e4SLinus Torvalds return 0; 4631da177e4SLinus Torvalds } 4641da177e4SLinus Torvalds 4652dc5327dSMilan Broz static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, 4662dc5327dSMilan Broz struct dm_crypt_request *dmreq) 46746b47730SLudwig Nussel { 46846b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 46946b47730SLudwig Nussel 47046b47730SLudwig Nussel return 0; 47146b47730SLudwig Nussel } 47246b47730SLudwig Nussel 47334745785SMilan Broz static void crypt_iv_lmk_dtr(struct crypt_config *cc) 47434745785SMilan Broz { 47534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 47634745785SMilan Broz 47734745785SMilan Broz if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm)) 47834745785SMilan Broz crypto_free_shash(lmk->hash_tfm); 47934745785SMilan Broz lmk->hash_tfm = NULL; 48034745785SMilan Broz 48134745785SMilan Broz kzfree(lmk->seed); 48234745785SMilan Broz lmk->seed = NULL; 48334745785SMilan Broz } 48434745785SMilan Broz 48534745785SMilan Broz static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti, 48634745785SMilan Broz const char *opts) 48734745785SMilan Broz { 48834745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 48934745785SMilan Broz 49034745785SMilan Broz lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0); 49134745785SMilan Broz if (IS_ERR(lmk->hash_tfm)) { 49234745785SMilan Broz ti->error = "Error initializing LMK hash"; 49334745785SMilan Broz return PTR_ERR(lmk->hash_tfm); 49434745785SMilan Broz } 49534745785SMilan Broz 49634745785SMilan Broz /* No seed in LMK version 2 */ 49734745785SMilan Broz if (cc->key_parts == cc->tfms_count) { 49834745785SMilan Broz lmk->seed = NULL; 49934745785SMilan Broz return 0; 50034745785SMilan Broz } 50134745785SMilan Broz 50234745785SMilan Broz lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL); 50334745785SMilan Broz if (!lmk->seed) { 50434745785SMilan Broz crypt_iv_lmk_dtr(cc); 50534745785SMilan Broz ti->error = "Error kmallocing seed storage in LMK"; 50634745785SMilan Broz return -ENOMEM; 50734745785SMilan Broz } 50834745785SMilan Broz 50934745785SMilan Broz return 0; 51034745785SMilan Broz } 51134745785SMilan Broz 51234745785SMilan Broz static int crypt_iv_lmk_init(struct crypt_config *cc) 51334745785SMilan Broz { 51434745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 51534745785SMilan Broz int subkey_size = cc->key_size / cc->key_parts; 51634745785SMilan Broz 51734745785SMilan Broz /* LMK seed is on the position of LMK_KEYS + 1 key */ 51834745785SMilan Broz if (lmk->seed) 51934745785SMilan Broz memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size), 52034745785SMilan Broz crypto_shash_digestsize(lmk->hash_tfm)); 52134745785SMilan Broz 52234745785SMilan Broz return 0; 52334745785SMilan Broz } 52434745785SMilan Broz 52534745785SMilan Broz static int crypt_iv_lmk_wipe(struct crypt_config *cc) 52634745785SMilan Broz { 52734745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 52834745785SMilan Broz 52934745785SMilan Broz if (lmk->seed) 53034745785SMilan Broz memset(lmk->seed, 0, LMK_SEED_SIZE); 53134745785SMilan Broz 53234745785SMilan Broz return 0; 53334745785SMilan Broz } 53434745785SMilan Broz 53534745785SMilan Broz static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv, 53634745785SMilan Broz struct dm_crypt_request *dmreq, 53734745785SMilan Broz u8 *data) 53834745785SMilan Broz { 53934745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 54034745785SMilan Broz struct { 54134745785SMilan Broz struct shash_desc desc; 54234745785SMilan Broz char ctx[crypto_shash_descsize(lmk->hash_tfm)]; 54334745785SMilan Broz } sdesc; 54434745785SMilan Broz struct md5_state md5state; 54534745785SMilan Broz u32 buf[4]; 54634745785SMilan Broz int i, r; 54734745785SMilan Broz 54834745785SMilan Broz sdesc.desc.tfm = lmk->hash_tfm; 54934745785SMilan Broz sdesc.desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 55034745785SMilan Broz 55134745785SMilan Broz r = crypto_shash_init(&sdesc.desc); 55234745785SMilan Broz if (r) 55334745785SMilan Broz return r; 55434745785SMilan Broz 55534745785SMilan Broz if (lmk->seed) { 55634745785SMilan Broz r = crypto_shash_update(&sdesc.desc, lmk->seed, LMK_SEED_SIZE); 55734745785SMilan Broz if (r) 55834745785SMilan Broz return r; 55934745785SMilan Broz } 56034745785SMilan Broz 56134745785SMilan Broz /* Sector is always 512B, block size 16, add data of blocks 1-31 */ 56234745785SMilan Broz r = crypto_shash_update(&sdesc.desc, data + 16, 16 * 31); 56334745785SMilan Broz if (r) 56434745785SMilan Broz return r; 56534745785SMilan Broz 56634745785SMilan Broz /* Sector is cropped to 56 bits here */ 56734745785SMilan Broz buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF); 56834745785SMilan Broz buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000); 56934745785SMilan Broz buf[2] = cpu_to_le32(4024); 57034745785SMilan Broz buf[3] = 0; 57134745785SMilan Broz r = crypto_shash_update(&sdesc.desc, (u8 *)buf, sizeof(buf)); 57234745785SMilan Broz if (r) 57334745785SMilan Broz return r; 57434745785SMilan Broz 57534745785SMilan Broz /* No MD5 padding here */ 57634745785SMilan Broz r = crypto_shash_export(&sdesc.desc, &md5state); 57734745785SMilan Broz if (r) 57834745785SMilan Broz return r; 57934745785SMilan Broz 58034745785SMilan Broz for (i = 0; i < MD5_HASH_WORDS; i++) 58134745785SMilan Broz __cpu_to_le32s(&md5state.hash[i]); 58234745785SMilan Broz memcpy(iv, &md5state.hash, cc->iv_size); 58334745785SMilan Broz 58434745785SMilan Broz return 0; 58534745785SMilan Broz } 58634745785SMilan Broz 58734745785SMilan Broz static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv, 58834745785SMilan Broz struct dm_crypt_request *dmreq) 58934745785SMilan Broz { 59034745785SMilan Broz u8 *src; 59134745785SMilan Broz int r = 0; 59234745785SMilan Broz 59334745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 59434745785SMilan Broz src = kmap_atomic(sg_page(&dmreq->sg_in), KM_USER0); 59534745785SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, src + dmreq->sg_in.offset); 59634745785SMilan Broz kunmap_atomic(src, KM_USER0); 59734745785SMilan Broz } else 59834745785SMilan Broz memset(iv, 0, cc->iv_size); 59934745785SMilan Broz 60034745785SMilan Broz return r; 60134745785SMilan Broz } 60234745785SMilan Broz 60334745785SMilan Broz static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv, 60434745785SMilan Broz struct dm_crypt_request *dmreq) 60534745785SMilan Broz { 60634745785SMilan Broz u8 *dst; 60734745785SMilan Broz int r; 60834745785SMilan Broz 60934745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) 61034745785SMilan Broz return 0; 61134745785SMilan Broz 61234745785SMilan Broz dst = kmap_atomic(sg_page(&dmreq->sg_out), KM_USER0); 61334745785SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, dst + dmreq->sg_out.offset); 61434745785SMilan Broz 61534745785SMilan Broz /* Tweak the first block of plaintext sector */ 61634745785SMilan Broz if (!r) 61734745785SMilan Broz crypto_xor(dst + dmreq->sg_out.offset, iv, cc->iv_size); 61834745785SMilan Broz 61934745785SMilan Broz kunmap_atomic(dst, KM_USER0); 62034745785SMilan Broz return r; 62134745785SMilan Broz } 62234745785SMilan Broz 6231da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_plain_ops = { 6241da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 6251da177e4SLinus Torvalds }; 6261da177e4SLinus Torvalds 62761afef61SMilan Broz static struct crypt_iv_operations crypt_iv_plain64_ops = { 62861afef61SMilan Broz .generator = crypt_iv_plain64_gen 62961afef61SMilan Broz }; 63061afef61SMilan Broz 6311da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_essiv_ops = { 6321da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 6331da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 634b95bf2d3SMilan Broz .init = crypt_iv_essiv_init, 635542da317SMilan Broz .wipe = crypt_iv_essiv_wipe, 6361da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 6371da177e4SLinus Torvalds }; 6381da177e4SLinus Torvalds 63948527fa7SRik Snel static struct crypt_iv_operations crypt_iv_benbi_ops = { 64048527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 64148527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 64248527fa7SRik Snel .generator = crypt_iv_benbi_gen 64348527fa7SRik Snel }; 6441da177e4SLinus Torvalds 64546b47730SLudwig Nussel static struct crypt_iv_operations crypt_iv_null_ops = { 64646b47730SLudwig Nussel .generator = crypt_iv_null_gen 64746b47730SLudwig Nussel }; 64846b47730SLudwig Nussel 64934745785SMilan Broz static struct crypt_iv_operations crypt_iv_lmk_ops = { 65034745785SMilan Broz .ctr = crypt_iv_lmk_ctr, 65134745785SMilan Broz .dtr = crypt_iv_lmk_dtr, 65234745785SMilan Broz .init = crypt_iv_lmk_init, 65334745785SMilan Broz .wipe = crypt_iv_lmk_wipe, 65434745785SMilan Broz .generator = crypt_iv_lmk_gen, 65534745785SMilan Broz .post = crypt_iv_lmk_post 65634745785SMilan Broz }; 65734745785SMilan Broz 658d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 659d469f841SMilan Broz struct convert_context *ctx, 6601da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 661fcd369daSMilan Broz sector_t sector) 6621da177e4SLinus Torvalds { 6631da177e4SLinus Torvalds ctx->bio_in = bio_in; 6641da177e4SLinus Torvalds ctx->bio_out = bio_out; 6651da177e4SLinus Torvalds ctx->offset_in = 0; 6661da177e4SLinus Torvalds ctx->offset_out = 0; 6671da177e4SLinus Torvalds ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 6681da177e4SLinus Torvalds ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 6691da177e4SLinus Torvalds ctx->sector = sector + cc->iv_offset; 67043d69034SMilan Broz init_completion(&ctx->restart); 6711da177e4SLinus Torvalds } 6721da177e4SLinus Torvalds 673b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 674b2174eebSHuang Ying struct ablkcipher_request *req) 675b2174eebSHuang Ying { 676b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 677b2174eebSHuang Ying } 678b2174eebSHuang Ying 679b2174eebSHuang Ying static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc, 680b2174eebSHuang Ying struct dm_crypt_request *dmreq) 681b2174eebSHuang Ying { 682b2174eebSHuang Ying return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start); 683b2174eebSHuang Ying } 684b2174eebSHuang Ying 6852dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, 6862dc5327dSMilan Broz struct dm_crypt_request *dmreq) 6872dc5327dSMilan Broz { 6882dc5327dSMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 6892dc5327dSMilan Broz crypto_ablkcipher_alignmask(any_tfm(cc)) + 1); 6902dc5327dSMilan Broz } 6912dc5327dSMilan Broz 69201482b76SMilan Broz static int crypt_convert_block(struct crypt_config *cc, 6933a7f6c99SMilan Broz struct convert_context *ctx, 6943a7f6c99SMilan Broz struct ablkcipher_request *req) 69501482b76SMilan Broz { 69601482b76SMilan Broz struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 69701482b76SMilan Broz struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 6983a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 6993a7f6c99SMilan Broz u8 *iv; 7003a7f6c99SMilan Broz int r = 0; 70101482b76SMilan Broz 702b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 7032dc5327dSMilan Broz iv = iv_of_dmreq(cc, dmreq); 7043a7f6c99SMilan Broz 7052dc5327dSMilan Broz dmreq->iv_sector = ctx->sector; 706b2174eebSHuang Ying dmreq->ctx = ctx; 7073a7f6c99SMilan Broz sg_init_table(&dmreq->sg_in, 1); 7083a7f6c99SMilan Broz sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT, 70901482b76SMilan Broz bv_in->bv_offset + ctx->offset_in); 71001482b76SMilan Broz 7113a7f6c99SMilan Broz sg_init_table(&dmreq->sg_out, 1); 7123a7f6c99SMilan Broz sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT, 71301482b76SMilan Broz bv_out->bv_offset + ctx->offset_out); 71401482b76SMilan Broz 71501482b76SMilan Broz ctx->offset_in += 1 << SECTOR_SHIFT; 71601482b76SMilan Broz if (ctx->offset_in >= bv_in->bv_len) { 71701482b76SMilan Broz ctx->offset_in = 0; 71801482b76SMilan Broz ctx->idx_in++; 71901482b76SMilan Broz } 72001482b76SMilan Broz 72101482b76SMilan Broz ctx->offset_out += 1 << SECTOR_SHIFT; 72201482b76SMilan Broz if (ctx->offset_out >= bv_out->bv_len) { 72301482b76SMilan Broz ctx->offset_out = 0; 72401482b76SMilan Broz ctx->idx_out++; 72501482b76SMilan Broz } 72601482b76SMilan Broz 7273a7f6c99SMilan Broz if (cc->iv_gen_ops) { 7282dc5327dSMilan Broz r = cc->iv_gen_ops->generator(cc, iv, dmreq); 7293a7f6c99SMilan Broz if (r < 0) 7303a7f6c99SMilan Broz return r; 7313a7f6c99SMilan Broz } 7323a7f6c99SMilan Broz 7333a7f6c99SMilan Broz ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out, 7343a7f6c99SMilan Broz 1 << SECTOR_SHIFT, iv); 7353a7f6c99SMilan Broz 7363a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 7373a7f6c99SMilan Broz r = crypto_ablkcipher_encrypt(req); 7383a7f6c99SMilan Broz else 7393a7f6c99SMilan Broz r = crypto_ablkcipher_decrypt(req); 7403a7f6c99SMilan Broz 7412dc5327dSMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 7422dc5327dSMilan Broz r = cc->iv_gen_ops->post(cc, iv, dmreq); 7432dc5327dSMilan Broz 7443a7f6c99SMilan Broz return r; 74501482b76SMilan Broz } 74601482b76SMilan Broz 74795497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 74895497a96SMilan Broz int error); 749c0297721SAndi Kleen 750ddd42edfSMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 751ddd42edfSMilan Broz struct convert_context *ctx) 752ddd42edfSMilan Broz { 753c0297721SAndi Kleen struct crypt_cpu *this_cc = this_crypt_config(cc); 754d1f96423SMilan Broz unsigned key_index = ctx->sector & (cc->tfms_count - 1); 755c0297721SAndi Kleen 756c0297721SAndi Kleen if (!this_cc->req) 757c0297721SAndi Kleen this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO); 758c0297721SAndi Kleen 759d1f96423SMilan Broz ablkcipher_request_set_tfm(this_cc->req, this_cc->tfms[key_index]); 760c0297721SAndi Kleen ablkcipher_request_set_callback(this_cc->req, 761c0297721SAndi Kleen CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, 762c0297721SAndi Kleen kcryptd_async_done, dmreq_of_req(cc, this_cc->req)); 763ddd42edfSMilan Broz } 764ddd42edfSMilan Broz 7651da177e4SLinus Torvalds /* 7661da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 7671da177e4SLinus Torvalds */ 7681da177e4SLinus Torvalds static int crypt_convert(struct crypt_config *cc, 7691da177e4SLinus Torvalds struct convert_context *ctx) 7701da177e4SLinus Torvalds { 771c0297721SAndi Kleen struct crypt_cpu *this_cc = this_crypt_config(cc); 7723f1e9070SMilan Broz int r; 7731da177e4SLinus Torvalds 774c8081618SMilan Broz atomic_set(&ctx->pending, 1); 775c8081618SMilan Broz 7761da177e4SLinus Torvalds while(ctx->idx_in < ctx->bio_in->bi_vcnt && 7771da177e4SLinus Torvalds ctx->idx_out < ctx->bio_out->bi_vcnt) { 7781da177e4SLinus Torvalds 7793a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 7803a7f6c99SMilan Broz 7813f1e9070SMilan Broz atomic_inc(&ctx->pending); 7823f1e9070SMilan Broz 783c0297721SAndi Kleen r = crypt_convert_block(cc, ctx, this_cc->req); 7843a7f6c99SMilan Broz 7853a7f6c99SMilan Broz switch (r) { 7863f1e9070SMilan Broz /* async */ 7873a7f6c99SMilan Broz case -EBUSY: 7883a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 7893a7f6c99SMilan Broz INIT_COMPLETION(ctx->restart); 7903a7f6c99SMilan Broz /* fall through*/ 7913a7f6c99SMilan Broz case -EINPROGRESS: 792c0297721SAndi Kleen this_cc->req = NULL; 7931da177e4SLinus Torvalds ctx->sector++; 7943a7f6c99SMilan Broz continue; 7953a7f6c99SMilan Broz 7963f1e9070SMilan Broz /* sync */ 7973f1e9070SMilan Broz case 0: 7983f1e9070SMilan Broz atomic_dec(&ctx->pending); 7993f1e9070SMilan Broz ctx->sector++; 800c7f1b204SMilan Broz cond_resched(); 8013f1e9070SMilan Broz continue; 8021da177e4SLinus Torvalds 8033f1e9070SMilan Broz /* error */ 8043f1e9070SMilan Broz default: 8053f1e9070SMilan Broz atomic_dec(&ctx->pending); 8061da177e4SLinus Torvalds return r; 8071da177e4SLinus Torvalds } 8083f1e9070SMilan Broz } 8093f1e9070SMilan Broz 8103f1e9070SMilan Broz return 0; 8113f1e9070SMilan Broz } 8121da177e4SLinus Torvalds 8136a24c718SMilan Broz static void dm_crypt_bio_destructor(struct bio *bio) 8146a24c718SMilan Broz { 815028867acSAlasdair G Kergon struct dm_crypt_io *io = bio->bi_private; 8166a24c718SMilan Broz struct crypt_config *cc = io->target->private; 8176a24c718SMilan Broz 8186a24c718SMilan Broz bio_free(bio, cc->bs); 8196a24c718SMilan Broz } 8206a24c718SMilan Broz 8211da177e4SLinus Torvalds /* 8221da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 8231da177e4SLinus Torvalds * This should never violate the device limitations 824933f01d4SMilan Broz * May return a smaller bio when running out of pages, indicated by 825933f01d4SMilan Broz * *out_of_pages set to 1. 8261da177e4SLinus Torvalds */ 827933f01d4SMilan Broz static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, 828933f01d4SMilan Broz unsigned *out_of_pages) 8291da177e4SLinus Torvalds { 830027581f3SOlaf Kirch struct crypt_config *cc = io->target->private; 8318b004457SMilan Broz struct bio *clone; 8321da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 833b4e3ca1aSAl Viro gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 83491e10625SMilan Broz unsigned i, len; 83591e10625SMilan Broz struct page *page; 8361da177e4SLinus Torvalds 8376a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 8388b004457SMilan Broz if (!clone) 8391da177e4SLinus Torvalds return NULL; 8401da177e4SLinus Torvalds 841027581f3SOlaf Kirch clone_init(io, clone); 842933f01d4SMilan Broz *out_of_pages = 0; 8436a24c718SMilan Broz 844f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 84591e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 846933f01d4SMilan Broz if (!page) { 847933f01d4SMilan Broz *out_of_pages = 1; 8481da177e4SLinus Torvalds break; 849933f01d4SMilan Broz } 8501da177e4SLinus Torvalds 8511da177e4SLinus Torvalds /* 8521da177e4SLinus Torvalds * if additional pages cannot be allocated without waiting, 8531da177e4SLinus Torvalds * return a partially allocated bio, the caller will then try 8541da177e4SLinus Torvalds * to allocate additional bios while submitting this partial bio 8551da177e4SLinus Torvalds */ 856f97380bcSOlaf Kirch if (i == (MIN_BIO_PAGES - 1)) 8571da177e4SLinus Torvalds gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 8581da177e4SLinus Torvalds 85991e10625SMilan Broz len = (size > PAGE_SIZE) ? PAGE_SIZE : size; 8601da177e4SLinus Torvalds 86191e10625SMilan Broz if (!bio_add_page(clone, page, len, 0)) { 86291e10625SMilan Broz mempool_free(page, cc->page_pool); 86391e10625SMilan Broz break; 86491e10625SMilan Broz } 86591e10625SMilan Broz 86691e10625SMilan Broz size -= len; 8671da177e4SLinus Torvalds } 8681da177e4SLinus Torvalds 8698b004457SMilan Broz if (!clone->bi_size) { 8708b004457SMilan Broz bio_put(clone); 8711da177e4SLinus Torvalds return NULL; 8721da177e4SLinus Torvalds } 8731da177e4SLinus Torvalds 8748b004457SMilan Broz return clone; 8751da177e4SLinus Torvalds } 8761da177e4SLinus Torvalds 877644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 8781da177e4SLinus Torvalds { 879644bd2f0SNeil Brown unsigned int i; 8801da177e4SLinus Torvalds struct bio_vec *bv; 8811da177e4SLinus Torvalds 882644bd2f0SNeil Brown for (i = 0; i < clone->bi_vcnt; i++) { 8838b004457SMilan Broz bv = bio_iovec_idx(clone, i); 8841da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 8851da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 8861da177e4SLinus Torvalds bv->bv_page = NULL; 8871da177e4SLinus Torvalds } 8881da177e4SLinus Torvalds } 8891da177e4SLinus Torvalds 890dc440d1eSMilan Broz static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti, 891dc440d1eSMilan Broz struct bio *bio, sector_t sector) 892dc440d1eSMilan Broz { 893dc440d1eSMilan Broz struct crypt_config *cc = ti->private; 894dc440d1eSMilan Broz struct dm_crypt_io *io; 895dc440d1eSMilan Broz 896dc440d1eSMilan Broz io = mempool_alloc(cc->io_pool, GFP_NOIO); 897dc440d1eSMilan Broz io->target = ti; 898dc440d1eSMilan Broz io->base_bio = bio; 899dc440d1eSMilan Broz io->sector = sector; 900dc440d1eSMilan Broz io->error = 0; 901393b47efSMilan Broz io->base_io = NULL; 902dc440d1eSMilan Broz atomic_set(&io->pending, 0); 903dc440d1eSMilan Broz 904dc440d1eSMilan Broz return io; 905dc440d1eSMilan Broz } 906dc440d1eSMilan Broz 9073e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 9083e1a8bddSMilan Broz { 9093e1a8bddSMilan Broz atomic_inc(&io->pending); 9103e1a8bddSMilan Broz } 9113e1a8bddSMilan Broz 9121da177e4SLinus Torvalds /* 9131da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 9141da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 915393b47efSMilan Broz * If base_io is set, wait for the last fragment to complete. 9161da177e4SLinus Torvalds */ 9175742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 9181da177e4SLinus Torvalds { 9195742fd77SMilan Broz struct crypt_config *cc = io->target->private; 920b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 921b35f8caaSMilan Broz struct dm_crypt_io *base_io = io->base_io; 922b35f8caaSMilan Broz int error = io->error; 9231da177e4SLinus Torvalds 9241da177e4SLinus Torvalds if (!atomic_dec_and_test(&io->pending)) 9251da177e4SLinus Torvalds return; 9261da177e4SLinus Torvalds 9271da177e4SLinus Torvalds mempool_free(io, cc->io_pool); 928b35f8caaSMilan Broz 929b35f8caaSMilan Broz if (likely(!base_io)) 930b35f8caaSMilan Broz bio_endio(base_bio, error); 931b35f8caaSMilan Broz else { 932b35f8caaSMilan Broz if (error && !base_io->error) 933b35f8caaSMilan Broz base_io->error = error; 934b35f8caaSMilan Broz crypt_dec_pending(base_io); 935b35f8caaSMilan Broz } 9361da177e4SLinus Torvalds } 9371da177e4SLinus Torvalds 9381da177e4SLinus Torvalds /* 939cabf08e4SMilan Broz * kcryptd/kcryptd_io: 9401da177e4SLinus Torvalds * 9411da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 94223541d2dSMilan Broz * interrupt context. 943cabf08e4SMilan Broz * 944cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 945cabf08e4SMilan Broz * 946cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 947cabf08e4SMilan Broz * 948cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 949cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 950cabf08e4SMilan Broz * to memory allocation. 951c0297721SAndi Kleen * 952c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances. 953c0297721SAndi Kleen * They should not depend on each other and do not block. 9541da177e4SLinus Torvalds */ 9556712ecf8SNeilBrown static void crypt_endio(struct bio *clone, int error) 9568b004457SMilan Broz { 957028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 9588b004457SMilan Broz struct crypt_config *cc = io->target->private; 959ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 9608b004457SMilan Broz 961adfe4770SMilan Broz if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error)) 962adfe4770SMilan Broz error = -EIO; 963adfe4770SMilan Broz 9648b004457SMilan Broz /* 9656712ecf8SNeilBrown * free the processed pages 9668b004457SMilan Broz */ 967ee7a491eSMilan Broz if (rw == WRITE) 968644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 9698b004457SMilan Broz 9708b004457SMilan Broz bio_put(clone); 971ee7a491eSMilan Broz 972ee7a491eSMilan Broz if (rw == READ && !error) { 973cabf08e4SMilan Broz kcryptd_queue_crypt(io); 9746712ecf8SNeilBrown return; 975ee7a491eSMilan Broz } 9765742fd77SMilan Broz 9775742fd77SMilan Broz if (unlikely(error)) 9785742fd77SMilan Broz io->error = error; 9795742fd77SMilan Broz 9805742fd77SMilan Broz crypt_dec_pending(io); 9818b004457SMilan Broz } 9828b004457SMilan Broz 983028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 9848b004457SMilan Broz { 9858b004457SMilan Broz struct crypt_config *cc = io->target->private; 9868b004457SMilan Broz 9878b004457SMilan Broz clone->bi_private = io; 9888b004457SMilan Broz clone->bi_end_io = crypt_endio; 9898b004457SMilan Broz clone->bi_bdev = cc->dev->bdev; 9908b004457SMilan Broz clone->bi_rw = io->base_bio->bi_rw; 991027581f3SOlaf Kirch clone->bi_destructor = dm_crypt_bio_destructor; 9928b004457SMilan Broz } 9938b004457SMilan Broz 99420c82538SMilan Broz static void kcryptd_unplug(struct crypt_config *cc) 99520c82538SMilan Broz { 99620c82538SMilan Broz blk_unplug(bdev_get_queue(cc->dev->bdev)); 99720c82538SMilan Broz } 99820c82538SMilan Broz 99920c82538SMilan Broz static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) 10008b004457SMilan Broz { 10018b004457SMilan Broz struct crypt_config *cc = io->target->private; 10028b004457SMilan Broz struct bio *base_bio = io->base_bio; 10038b004457SMilan Broz struct bio *clone; 100493e605c2SMilan Broz 10058b004457SMilan Broz /* 10068b004457SMilan Broz * The block layer might modify the bvec array, so always 10078b004457SMilan Broz * copy the required bvecs because we need the original 10088b004457SMilan Broz * one in order to decrypt the whole bio data *afterwards*. 10098b004457SMilan Broz */ 101020c82538SMilan Broz clone = bio_alloc_bioset(gfp, bio_segments(base_bio), cc->bs); 101120c82538SMilan Broz if (!clone) { 101220c82538SMilan Broz kcryptd_unplug(cc); 101320c82538SMilan Broz return 1; 101493e605c2SMilan Broz } 10158b004457SMilan Broz 101620c82538SMilan Broz crypt_inc_pending(io); 101720c82538SMilan Broz 10188b004457SMilan Broz clone_init(io, clone); 10198b004457SMilan Broz clone->bi_idx = 0; 10208b004457SMilan Broz clone->bi_vcnt = bio_segments(base_bio); 10218b004457SMilan Broz clone->bi_size = base_bio->bi_size; 10220c395b0fSMilan Broz clone->bi_sector = cc->start + io->sector; 10238b004457SMilan Broz memcpy(clone->bi_io_vec, bio_iovec(base_bio), 10248b004457SMilan Broz sizeof(struct bio_vec) * clone->bi_vcnt); 10258b004457SMilan Broz 102693e605c2SMilan Broz generic_make_request(clone); 102720c82538SMilan Broz return 0; 10288b004457SMilan Broz } 10298b004457SMilan Broz 10304e4eef64SMilan Broz static void kcryptd_io_write(struct dm_crypt_io *io) 10314e4eef64SMilan Broz { 103295497a96SMilan Broz struct bio *clone = io->ctx.bio_out; 103395497a96SMilan Broz generic_make_request(clone); 10344e4eef64SMilan Broz } 10354e4eef64SMilan Broz 1036395b167cSAlasdair G Kergon static void kcryptd_io(struct work_struct *work) 1037395b167cSAlasdair G Kergon { 1038395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 1039395b167cSAlasdair G Kergon 104020c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 104120c82538SMilan Broz crypt_inc_pending(io); 104220c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOIO)) 104320c82538SMilan Broz io->error = -ENOMEM; 104420c82538SMilan Broz crypt_dec_pending(io); 104520c82538SMilan Broz } else 1046395b167cSAlasdair G Kergon kcryptd_io_write(io); 1047395b167cSAlasdair G Kergon } 1048395b167cSAlasdair G Kergon 1049395b167cSAlasdair G Kergon static void kcryptd_queue_io(struct dm_crypt_io *io) 1050395b167cSAlasdair G Kergon { 1051395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 1052395b167cSAlasdair G Kergon 1053395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_io); 1054395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 1055395b167cSAlasdair G Kergon } 1056395b167cSAlasdair G Kergon 105795497a96SMilan Broz static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, 105895497a96SMilan Broz int error, int async) 10594e4eef64SMilan Broz { 1060dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 1061dec1cedfSMilan Broz struct crypt_config *cc = io->target->private; 1062dec1cedfSMilan Broz 1063dec1cedfSMilan Broz if (unlikely(error < 0)) { 1064dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 1065dec1cedfSMilan Broz bio_put(clone); 1066dec1cedfSMilan Broz io->error = -EIO; 10676c031f41SMilan Broz crypt_dec_pending(io); 1068dec1cedfSMilan Broz return; 1069dec1cedfSMilan Broz } 1070dec1cedfSMilan Broz 1071dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 1072dec1cedfSMilan Broz BUG_ON(io->ctx.idx_out < clone->bi_vcnt); 1073dec1cedfSMilan Broz 1074dec1cedfSMilan Broz clone->bi_sector = cc->start + io->sector; 1075899c95d3SMilan Broz 107695497a96SMilan Broz if (async) 107795497a96SMilan Broz kcryptd_queue_io(io); 10781e37bb8eSAlasdair G Kergon else 1079899c95d3SMilan Broz generic_make_request(clone); 10804e4eef64SMilan Broz } 10814e4eef64SMilan Broz 1082fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 10838b004457SMilan Broz { 10848b004457SMilan Broz struct crypt_config *cc = io->target->private; 10858b004457SMilan Broz struct bio *clone; 1086393b47efSMilan Broz struct dm_crypt_io *new_io; 1087c8081618SMilan Broz int crypt_finished; 1088933f01d4SMilan Broz unsigned out_of_pages = 0; 1089dec1cedfSMilan Broz unsigned remaining = io->base_bio->bi_size; 1090b635b00eSMilan Broz sector_t sector = io->sector; 1091dec1cedfSMilan Broz int r; 10928b004457SMilan Broz 109393e605c2SMilan Broz /* 1094fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 1095fc5a5e9aSMilan Broz */ 1096fc5a5e9aSMilan Broz crypt_inc_pending(io); 1097b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 1098fc5a5e9aSMilan Broz 1099fc5a5e9aSMilan Broz /* 110093e605c2SMilan Broz * The allocated buffers can be smaller than the whole bio, 110193e605c2SMilan Broz * so repeat the whole process until all the data can be handled. 110293e605c2SMilan Broz */ 110393e605c2SMilan Broz while (remaining) { 1104933f01d4SMilan Broz clone = crypt_alloc_buffer(io, remaining, &out_of_pages); 110523541d2dSMilan Broz if (unlikely(!clone)) { 11065742fd77SMilan Broz io->error = -ENOMEM; 1107fc5a5e9aSMilan Broz break; 110823541d2dSMilan Broz } 11098b004457SMilan Broz 111053017030SMilan Broz io->ctx.bio_out = clone; 111153017030SMilan Broz io->ctx.idx_out = 0; 11128b004457SMilan Broz 111393e605c2SMilan Broz remaining -= clone->bi_size; 1114b635b00eSMilan Broz sector += bio_sectors(clone); 1115dec1cedfSMilan Broz 11164e594098SMilan Broz crypt_inc_pending(io); 1117dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 1118c8081618SMilan Broz crypt_finished = atomic_dec_and_test(&io->ctx.pending); 1119dec1cedfSMilan Broz 1120c8081618SMilan Broz /* Encryption was already finished, submit io now */ 1121c8081618SMilan Broz if (crypt_finished) { 112295497a96SMilan Broz kcryptd_crypt_write_io_submit(io, r, 0); 1123c8081618SMilan Broz 1124c8081618SMilan Broz /* 1125c8081618SMilan Broz * If there was an error, do not try next fragments. 1126c8081618SMilan Broz * For async, error is processed in async handler. 1127c8081618SMilan Broz */ 11286c031f41SMilan Broz if (unlikely(r < 0)) 1129fc5a5e9aSMilan Broz break; 1130b635b00eSMilan Broz 1131b635b00eSMilan Broz io->sector = sector; 11324e594098SMilan Broz } 113393e605c2SMilan Broz 1134933f01d4SMilan Broz /* 1135933f01d4SMilan Broz * Out of memory -> run queues 1136933f01d4SMilan Broz * But don't wait if split was due to the io size restriction 1137933f01d4SMilan Broz */ 1138933f01d4SMilan Broz if (unlikely(out_of_pages)) 11398aa7e847SJens Axboe congestion_wait(BLK_RW_ASYNC, HZ/100); 1140933f01d4SMilan Broz 1141393b47efSMilan Broz /* 1142393b47efSMilan Broz * With async crypto it is unsafe to share the crypto context 1143393b47efSMilan Broz * between fragments, so switch to a new dm_crypt_io structure. 1144393b47efSMilan Broz */ 1145393b47efSMilan Broz if (unlikely(!crypt_finished && remaining)) { 1146393b47efSMilan Broz new_io = crypt_io_alloc(io->target, io->base_bio, 1147393b47efSMilan Broz sector); 1148393b47efSMilan Broz crypt_inc_pending(new_io); 1149393b47efSMilan Broz crypt_convert_init(cc, &new_io->ctx, NULL, 1150393b47efSMilan Broz io->base_bio, sector); 1151393b47efSMilan Broz new_io->ctx.idx_in = io->ctx.idx_in; 1152393b47efSMilan Broz new_io->ctx.offset_in = io->ctx.offset_in; 1153393b47efSMilan Broz 1154393b47efSMilan Broz /* 1155393b47efSMilan Broz * Fragments after the first use the base_io 1156393b47efSMilan Broz * pending count. 1157393b47efSMilan Broz */ 1158393b47efSMilan Broz if (!io->base_io) 1159393b47efSMilan Broz new_io->base_io = io; 1160393b47efSMilan Broz else { 1161393b47efSMilan Broz new_io->base_io = io->base_io; 1162393b47efSMilan Broz crypt_inc_pending(io->base_io); 1163393b47efSMilan Broz crypt_dec_pending(io); 1164393b47efSMilan Broz } 1165393b47efSMilan Broz 1166393b47efSMilan Broz io = new_io; 1167393b47efSMilan Broz } 11688b004457SMilan Broz } 1169899c95d3SMilan Broz 1170899c95d3SMilan Broz crypt_dec_pending(io); 117184131db6SMilan Broz } 117284131db6SMilan Broz 11734e4eef64SMilan Broz static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error) 11745742fd77SMilan Broz { 11755742fd77SMilan Broz if (unlikely(error < 0)) 11765742fd77SMilan Broz io->error = -EIO; 11775742fd77SMilan Broz 11785742fd77SMilan Broz crypt_dec_pending(io); 11795742fd77SMilan Broz } 11805742fd77SMilan Broz 11814e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 11828b004457SMilan Broz { 11838b004457SMilan Broz struct crypt_config *cc = io->target->private; 11845742fd77SMilan Broz int r = 0; 11858b004457SMilan Broz 11863e1a8bddSMilan Broz crypt_inc_pending(io); 11873a7f6c99SMilan Broz 118853017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 11890c395b0fSMilan Broz io->sector); 11908b004457SMilan Broz 11915742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 11925742fd77SMilan Broz 11933f1e9070SMilan Broz if (atomic_dec_and_test(&io->ctx.pending)) 11944e4eef64SMilan Broz kcryptd_crypt_read_done(io, r); 11953a7f6c99SMilan Broz 11963a7f6c99SMilan Broz crypt_dec_pending(io); 11978b004457SMilan Broz } 11988b004457SMilan Broz 119995497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 120095497a96SMilan Broz int error) 120195497a96SMilan Broz { 1202b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 1203b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 120495497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 120595497a96SMilan Broz struct crypt_config *cc = io->target->private; 120695497a96SMilan Broz 120795497a96SMilan Broz if (error == -EINPROGRESS) { 120895497a96SMilan Broz complete(&ctx->restart); 120995497a96SMilan Broz return; 121095497a96SMilan Broz } 121195497a96SMilan Broz 12122dc5327dSMilan Broz if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post) 12132dc5327dSMilan Broz error = cc->iv_gen_ops->post(cc, iv_of_dmreq(cc, dmreq), dmreq); 12142dc5327dSMilan Broz 1215b2174eebSHuang Ying mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool); 121695497a96SMilan Broz 121795497a96SMilan Broz if (!atomic_dec_and_test(&ctx->pending)) 121895497a96SMilan Broz return; 121995497a96SMilan Broz 122095497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 122195497a96SMilan Broz kcryptd_crypt_read_done(io, error); 122295497a96SMilan Broz else 122395497a96SMilan Broz kcryptd_crypt_write_io_submit(io, error, 1); 122495497a96SMilan Broz } 122595497a96SMilan Broz 12264e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 12274e4eef64SMilan Broz { 12284e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 12294e4eef64SMilan Broz 12304e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 12314e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 12324e4eef64SMilan Broz else 12334e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 12348b004457SMilan Broz } 12358b004457SMilan Broz 1236395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 1237395b167cSAlasdair G Kergon { 1238395b167cSAlasdair G Kergon struct crypt_config *cc = io->target->private; 1239395b167cSAlasdair G Kergon 1240395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 1241395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 1242395b167cSAlasdair G Kergon } 1243395b167cSAlasdair G Kergon 12441da177e4SLinus Torvalds /* 12451da177e4SLinus Torvalds * Decode key from its hex representation 12461da177e4SLinus Torvalds */ 12471da177e4SLinus Torvalds static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 12481da177e4SLinus Torvalds { 12491da177e4SLinus Torvalds char buffer[3]; 12501da177e4SLinus Torvalds char *endp; 12511da177e4SLinus Torvalds unsigned int i; 12521da177e4SLinus Torvalds 12531da177e4SLinus Torvalds buffer[2] = '\0'; 12541da177e4SLinus Torvalds 12551da177e4SLinus Torvalds for (i = 0; i < size; i++) { 12561da177e4SLinus Torvalds buffer[0] = *hex++; 12571da177e4SLinus Torvalds buffer[1] = *hex++; 12581da177e4SLinus Torvalds 12591da177e4SLinus Torvalds key[i] = (u8)simple_strtoul(buffer, &endp, 16); 12601da177e4SLinus Torvalds 12611da177e4SLinus Torvalds if (endp != &buffer[2]) 12621da177e4SLinus Torvalds return -EINVAL; 12631da177e4SLinus Torvalds } 12641da177e4SLinus Torvalds 12651da177e4SLinus Torvalds if (*hex != '\0') 12661da177e4SLinus Torvalds return -EINVAL; 12671da177e4SLinus Torvalds 12681da177e4SLinus Torvalds return 0; 12691da177e4SLinus Torvalds } 12701da177e4SLinus Torvalds 12711da177e4SLinus Torvalds /* 12721da177e4SLinus Torvalds * Encode key into its hex representation 12731da177e4SLinus Torvalds */ 12741da177e4SLinus Torvalds static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 12751da177e4SLinus Torvalds { 12761da177e4SLinus Torvalds unsigned int i; 12771da177e4SLinus Torvalds 12781da177e4SLinus Torvalds for (i = 0; i < size; i++) { 12791da177e4SLinus Torvalds sprintf(hex, "%02x", *key); 12801da177e4SLinus Torvalds hex += 2; 12811da177e4SLinus Torvalds key++; 12821da177e4SLinus Torvalds } 12831da177e4SLinus Torvalds } 12841da177e4SLinus Torvalds 1285d1f96423SMilan Broz static void crypt_free_tfms(struct crypt_config *cc, int cpu) 1286d1f96423SMilan Broz { 1287d1f96423SMilan Broz struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu); 1288d1f96423SMilan Broz unsigned i; 1289d1f96423SMilan Broz 1290d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) 1291d1f96423SMilan Broz if (cpu_cc->tfms[i] && !IS_ERR(cpu_cc->tfms[i])) { 1292d1f96423SMilan Broz crypto_free_ablkcipher(cpu_cc->tfms[i]); 1293d1f96423SMilan Broz cpu_cc->tfms[i] = NULL; 1294d1f96423SMilan Broz } 1295d1f96423SMilan Broz } 1296d1f96423SMilan Broz 1297d1f96423SMilan Broz static int crypt_alloc_tfms(struct crypt_config *cc, int cpu, char *ciphermode) 1298d1f96423SMilan Broz { 1299d1f96423SMilan Broz struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu); 1300d1f96423SMilan Broz unsigned i; 1301d1f96423SMilan Broz int err; 1302d1f96423SMilan Broz 1303d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 1304d1f96423SMilan Broz cpu_cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0); 1305d1f96423SMilan Broz if (IS_ERR(cpu_cc->tfms[i])) { 1306d1f96423SMilan Broz err = PTR_ERR(cpu_cc->tfms[i]); 1307d1f96423SMilan Broz crypt_free_tfms(cc, cpu); 1308d1f96423SMilan Broz return err; 1309d1f96423SMilan Broz } 1310d1f96423SMilan Broz } 1311d1f96423SMilan Broz 1312d1f96423SMilan Broz return 0; 1313d1f96423SMilan Broz } 1314d1f96423SMilan Broz 1315c0297721SAndi Kleen static int crypt_setkey_allcpus(struct crypt_config *cc) 1316c0297721SAndi Kleen { 1317d1f96423SMilan Broz unsigned subkey_size = cc->key_size >> ilog2(cc->tfms_count); 1318d1f96423SMilan Broz int cpu, err = 0, i, r; 1319c0297721SAndi Kleen 1320c0297721SAndi Kleen for_each_possible_cpu(cpu) { 1321d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 1322d1f96423SMilan Broz r = crypto_ablkcipher_setkey(per_cpu_ptr(cc->cpu, cpu)->tfms[i], 1323d1f96423SMilan Broz cc->key + (i * subkey_size), subkey_size); 1324c0297721SAndi Kleen if (r) 1325c0297721SAndi Kleen err = r; 1326c0297721SAndi Kleen } 1327d1f96423SMilan Broz } 1328c0297721SAndi Kleen 1329c0297721SAndi Kleen return err; 1330c0297721SAndi Kleen } 1331c0297721SAndi Kleen 1332e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 1333e48d4bbfSMilan Broz { 133469a8cfcdSMilan Broz /* The key size may not be changed. */ 133569a8cfcdSMilan Broz if (cc->key_size != (strlen(key) >> 1)) 1336e48d4bbfSMilan Broz return -EINVAL; 1337e48d4bbfSMilan Broz 133869a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */ 133969a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-")) 134069a8cfcdSMilan Broz return -EINVAL; 1341e48d4bbfSMilan Broz 134269a8cfcdSMilan Broz if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0) 1343e48d4bbfSMilan Broz return -EINVAL; 1344e48d4bbfSMilan Broz 1345e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1346e48d4bbfSMilan Broz 1347c0297721SAndi Kleen return crypt_setkey_allcpus(cc); 1348e48d4bbfSMilan Broz } 1349e48d4bbfSMilan Broz 1350e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 1351e48d4bbfSMilan Broz { 1352e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1353e48d4bbfSMilan Broz memset(&cc->key, 0, cc->key_size * sizeof(u8)); 1354c0297721SAndi Kleen 1355c0297721SAndi Kleen return crypt_setkey_allcpus(cc); 1356e48d4bbfSMilan Broz } 1357e48d4bbfSMilan Broz 135828513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 135928513fccSMilan Broz { 136028513fccSMilan Broz struct crypt_config *cc = ti->private; 1361c0297721SAndi Kleen struct crypt_cpu *cpu_cc; 1362c0297721SAndi Kleen int cpu; 136328513fccSMilan Broz 136428513fccSMilan Broz ti->private = NULL; 136528513fccSMilan Broz 136628513fccSMilan Broz if (!cc) 136728513fccSMilan Broz return; 136828513fccSMilan Broz 136928513fccSMilan Broz if (cc->io_queue) 137028513fccSMilan Broz destroy_workqueue(cc->io_queue); 137128513fccSMilan Broz if (cc->crypt_queue) 137228513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 137328513fccSMilan Broz 1374c0297721SAndi Kleen if (cc->cpu) 1375c0297721SAndi Kleen for_each_possible_cpu(cpu) { 1376c0297721SAndi Kleen cpu_cc = per_cpu_ptr(cc->cpu, cpu); 1377c0297721SAndi Kleen if (cpu_cc->req) 1378c0297721SAndi Kleen mempool_free(cpu_cc->req, cc->req_pool); 1379d1f96423SMilan Broz crypt_free_tfms(cc, cpu); 1380c0297721SAndi Kleen } 1381c0297721SAndi Kleen 138228513fccSMilan Broz if (cc->bs) 138328513fccSMilan Broz bioset_free(cc->bs); 138428513fccSMilan Broz 138528513fccSMilan Broz if (cc->page_pool) 138628513fccSMilan Broz mempool_destroy(cc->page_pool); 138728513fccSMilan Broz if (cc->req_pool) 138828513fccSMilan Broz mempool_destroy(cc->req_pool); 138928513fccSMilan Broz if (cc->io_pool) 139028513fccSMilan Broz mempool_destroy(cc->io_pool); 139128513fccSMilan Broz 139228513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 139328513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 139428513fccSMilan Broz 139528513fccSMilan Broz if (cc->dev) 139628513fccSMilan Broz dm_put_device(ti, cc->dev); 139728513fccSMilan Broz 1398c0297721SAndi Kleen if (cc->cpu) 1399c0297721SAndi Kleen free_percpu(cc->cpu); 1400c0297721SAndi Kleen 14015ebaee6dSMilan Broz kzfree(cc->cipher); 14027dbcd137SMilan Broz kzfree(cc->cipher_string); 140328513fccSMilan Broz 140428513fccSMilan Broz /* Must zero key material before freeing */ 140528513fccSMilan Broz kzfree(cc); 140628513fccSMilan Broz } 140728513fccSMilan Broz 14085ebaee6dSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, 14095ebaee6dSMilan Broz char *cipher_in, char *key) 14101da177e4SLinus Torvalds { 14115ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 1412d1f96423SMilan Broz char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount; 14135ebaee6dSMilan Broz char *cipher_api = NULL; 1414c0297721SAndi Kleen int cpu, ret = -EINVAL; 14151da177e4SLinus Torvalds 14165ebaee6dSMilan Broz /* Convert to crypto api definition? */ 14175ebaee6dSMilan Broz if (strchr(cipher_in, '(')) { 14185ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 14191da177e4SLinus Torvalds return -EINVAL; 14201da177e4SLinus Torvalds } 14211da177e4SLinus Torvalds 14227dbcd137SMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL); 14237dbcd137SMilan Broz if (!cc->cipher_string) 14247dbcd137SMilan Broz goto bad_mem; 14257dbcd137SMilan Broz 14265ebaee6dSMilan Broz /* 14275ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 1428d1f96423SMilan Broz * cipher[:keycount]-mode-iv:ivopts 14295ebaee6dSMilan Broz */ 14305ebaee6dSMilan Broz tmp = cipher_in; 1431d1f96423SMilan Broz keycount = strsep(&tmp, "-"); 1432d1f96423SMilan Broz cipher = strsep(&keycount, ":"); 1433d1f96423SMilan Broz 1434d1f96423SMilan Broz if (!keycount) 1435d1f96423SMilan Broz cc->tfms_count = 1; 1436d1f96423SMilan Broz else if (sscanf(keycount, "%u", &cc->tfms_count) != 1 || 1437d1f96423SMilan Broz !is_power_of_2(cc->tfms_count)) { 1438d1f96423SMilan Broz ti->error = "Bad cipher key count specification"; 1439d1f96423SMilan Broz return -EINVAL; 1440d1f96423SMilan Broz } 1441d1f96423SMilan Broz cc->key_parts = cc->tfms_count; 14425ebaee6dSMilan Broz 14435ebaee6dSMilan Broz cc->cipher = kstrdup(cipher, GFP_KERNEL); 14445ebaee6dSMilan Broz if (!cc->cipher) 14455ebaee6dSMilan Broz goto bad_mem; 14465ebaee6dSMilan Broz 14471da177e4SLinus Torvalds chainmode = strsep(&tmp, "-"); 14481da177e4SLinus Torvalds ivopts = strsep(&tmp, "-"); 14491da177e4SLinus Torvalds ivmode = strsep(&ivopts, ":"); 14501da177e4SLinus Torvalds 14511da177e4SLinus Torvalds if (tmp) 14525ebaee6dSMilan Broz DMWARN("Ignoring unexpected additional cipher options"); 14531da177e4SLinus Torvalds 1454d1f96423SMilan Broz cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)) + 1455d1f96423SMilan Broz cc->tfms_count * sizeof(*(cc->cpu->tfms)), 1456d1f96423SMilan Broz __alignof__(struct crypt_cpu)); 1457c0297721SAndi Kleen if (!cc->cpu) { 1458c0297721SAndi Kleen ti->error = "Cannot allocate per cpu state"; 1459c0297721SAndi Kleen goto bad_mem; 1460c0297721SAndi Kleen } 1461c0297721SAndi Kleen 14627dbcd137SMilan Broz /* 14637dbcd137SMilan Broz * For compatibility with the original dm-crypt mapping format, if 14647dbcd137SMilan Broz * only the cipher name is supplied, use cbc-plain. 14657dbcd137SMilan Broz */ 14665ebaee6dSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) { 14671da177e4SLinus Torvalds chainmode = "cbc"; 14681da177e4SLinus Torvalds ivmode = "plain"; 14691da177e4SLinus Torvalds } 14701da177e4SLinus Torvalds 1471d1806f6aSHerbert Xu if (strcmp(chainmode, "ecb") && !ivmode) { 14725ebaee6dSMilan Broz ti->error = "IV mechanism required"; 14735ebaee6dSMilan Broz return -EINVAL; 14741da177e4SLinus Torvalds } 14751da177e4SLinus Torvalds 14765ebaee6dSMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 14775ebaee6dSMilan Broz if (!cipher_api) 14785ebaee6dSMilan Broz goto bad_mem; 14795ebaee6dSMilan Broz 14805ebaee6dSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 14815ebaee6dSMilan Broz "%s(%s)", chainmode, cipher); 14825ebaee6dSMilan Broz if (ret < 0) { 14835ebaee6dSMilan Broz kfree(cipher_api); 14845ebaee6dSMilan Broz goto bad_mem; 1485d1806f6aSHerbert Xu } 1486d1806f6aSHerbert Xu 14875ebaee6dSMilan Broz /* Allocate cipher */ 1488c0297721SAndi Kleen for_each_possible_cpu(cpu) { 1489d1f96423SMilan Broz ret = crypt_alloc_tfms(cc, cpu, cipher_api); 1490d1f96423SMilan Broz if (ret < 0) { 149172d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm"; 149228513fccSMilan Broz goto bad; 14931da177e4SLinus Torvalds } 1494c0297721SAndi Kleen } 14951da177e4SLinus Torvalds 14965ebaee6dSMilan Broz /* Initialize and set key */ 14975ebaee6dSMilan Broz ret = crypt_set_key(cc, key); 149828513fccSMilan Broz if (ret < 0) { 14990b430958SMilan Broz ti->error = "Error decoding and setting key"; 150028513fccSMilan Broz goto bad; 15010b430958SMilan Broz } 15020b430958SMilan Broz 15035ebaee6dSMilan Broz /* Initialize IV */ 1504c0297721SAndi Kleen cc->iv_size = crypto_ablkcipher_ivsize(any_tfm(cc)); 15055ebaee6dSMilan Broz if (cc->iv_size) 15065ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 15075ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 15085ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 15095ebaee6dSMilan Broz else if (ivmode) { 15105ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 15115ebaee6dSMilan Broz ivmode = NULL; 15125ebaee6dSMilan Broz } 15135ebaee6dSMilan Broz 15145ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 15151da177e4SLinus Torvalds if (ivmode == NULL) 15161da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 15171da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 15181da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 151961afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 152061afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 15211da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 15221da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 152348527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 152448527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 152546b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 152646b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 152734745785SMilan Broz else if (strcmp(ivmode, "lmk") == 0) { 152834745785SMilan Broz cc->iv_gen_ops = &crypt_iv_lmk_ops; 152934745785SMilan Broz /* Version 2 and 3 is recognised according 153034745785SMilan Broz * to length of provided multi-key string. 153134745785SMilan Broz * If present (version 3), last key is used as IV seed. 153234745785SMilan Broz */ 153334745785SMilan Broz if (cc->key_size % cc->key_parts) 153434745785SMilan Broz cc->key_parts++; 153534745785SMilan Broz } else { 15365ebaee6dSMilan Broz ret = -EINVAL; 153772d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 153828513fccSMilan Broz goto bad; 15391da177e4SLinus Torvalds } 15401da177e4SLinus Torvalds 154128513fccSMilan Broz /* Allocate IV */ 154228513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 154328513fccSMilan Broz ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 154428513fccSMilan Broz if (ret < 0) { 154528513fccSMilan Broz ti->error = "Error creating IV"; 154628513fccSMilan Broz goto bad; 154728513fccSMilan Broz } 154828513fccSMilan Broz } 15491da177e4SLinus Torvalds 155028513fccSMilan Broz /* Initialize IV (set keys for ESSIV etc) */ 155128513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 155228513fccSMilan Broz ret = cc->iv_gen_ops->init(cc); 155328513fccSMilan Broz if (ret < 0) { 1554b95bf2d3SMilan Broz ti->error = "Error initialising IV"; 155528513fccSMilan Broz goto bad; 155628513fccSMilan Broz } 1557b95bf2d3SMilan Broz } 1558b95bf2d3SMilan Broz 15595ebaee6dSMilan Broz ret = 0; 15605ebaee6dSMilan Broz bad: 15615ebaee6dSMilan Broz kfree(cipher_api); 15625ebaee6dSMilan Broz return ret; 15635ebaee6dSMilan Broz 15645ebaee6dSMilan Broz bad_mem: 15655ebaee6dSMilan Broz ti->error = "Cannot allocate cipher strings"; 15665ebaee6dSMilan Broz return -ENOMEM; 15671da177e4SLinus Torvalds } 15685ebaee6dSMilan Broz 15695ebaee6dSMilan Broz /* 15705ebaee6dSMilan Broz * Construct an encryption mapping: 15715ebaee6dSMilan Broz * <cipher> <key> <iv_offset> <dev_path> <start> 15725ebaee6dSMilan Broz */ 15735ebaee6dSMilan Broz static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 15745ebaee6dSMilan Broz { 15755ebaee6dSMilan Broz struct crypt_config *cc; 15765ebaee6dSMilan Broz unsigned int key_size; 15775ebaee6dSMilan Broz unsigned long long tmpll; 15785ebaee6dSMilan Broz int ret; 15795ebaee6dSMilan Broz 15805ebaee6dSMilan Broz if (argc != 5) { 15815ebaee6dSMilan Broz ti->error = "Not enough arguments"; 15825ebaee6dSMilan Broz return -EINVAL; 15831da177e4SLinus Torvalds } 15841da177e4SLinus Torvalds 15855ebaee6dSMilan Broz key_size = strlen(argv[1]) >> 1; 15865ebaee6dSMilan Broz 15875ebaee6dSMilan Broz cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 15885ebaee6dSMilan Broz if (!cc) { 15895ebaee6dSMilan Broz ti->error = "Cannot allocate encryption context"; 15905ebaee6dSMilan Broz return -ENOMEM; 15915ebaee6dSMilan Broz } 159269a8cfcdSMilan Broz cc->key_size = key_size; 15935ebaee6dSMilan Broz 15945ebaee6dSMilan Broz ti->private = cc; 15955ebaee6dSMilan Broz ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 15965ebaee6dSMilan Broz if (ret < 0) 15975ebaee6dSMilan Broz goto bad; 15985ebaee6dSMilan Broz 159928513fccSMilan Broz ret = -ENOMEM; 160093d2341cSMatthew Dobson cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 16011da177e4SLinus Torvalds if (!cc->io_pool) { 160272d94861SAlasdair G Kergon ti->error = "Cannot allocate crypt io mempool"; 160328513fccSMilan Broz goto bad; 16041da177e4SLinus Torvalds } 16051da177e4SLinus Torvalds 1606ddd42edfSMilan Broz cc->dmreq_start = sizeof(struct ablkcipher_request); 1607c0297721SAndi Kleen cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc)); 1608ddd42edfSMilan Broz cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment()); 1609c0297721SAndi Kleen cc->dmreq_start += crypto_ablkcipher_alignmask(any_tfm(cc)) & 16103a7f6c99SMilan Broz ~(crypto_tfm_ctx_alignment() - 1); 1611ddd42edfSMilan Broz 1612ddd42edfSMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1613ddd42edfSMilan Broz sizeof(struct dm_crypt_request) + cc->iv_size); 1614ddd42edfSMilan Broz if (!cc->req_pool) { 1615ddd42edfSMilan Broz ti->error = "Cannot allocate crypt request mempool"; 161628513fccSMilan Broz goto bad; 1617ddd42edfSMilan Broz } 1618ddd42edfSMilan Broz 1619a19b27ceSMatthew Dobson cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 16201da177e4SLinus Torvalds if (!cc->page_pool) { 162172d94861SAlasdair G Kergon ti->error = "Cannot allocate page mempool"; 162228513fccSMilan Broz goto bad; 16231da177e4SLinus Torvalds } 16241da177e4SLinus Torvalds 1625bb799ca0SJens Axboe cc->bs = bioset_create(MIN_IOS, 0); 16266a24c718SMilan Broz if (!cc->bs) { 16276a24c718SMilan Broz ti->error = "Cannot allocate crypt bioset"; 162828513fccSMilan Broz goto bad; 16296a24c718SMilan Broz } 16306a24c718SMilan Broz 163128513fccSMilan Broz ret = -EINVAL; 16324ee218cdSAndrew Morton if (sscanf(argv[2], "%llu", &tmpll) != 1) { 163372d94861SAlasdair G Kergon ti->error = "Invalid iv_offset sector"; 163428513fccSMilan Broz goto bad; 16351da177e4SLinus Torvalds } 16364ee218cdSAndrew Morton cc->iv_offset = tmpll; 16371da177e4SLinus Torvalds 163828513fccSMilan Broz if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev)) { 163928513fccSMilan Broz ti->error = "Device lookup failed"; 164028513fccSMilan Broz goto bad; 164128513fccSMilan Broz } 164228513fccSMilan Broz 16434ee218cdSAndrew Morton if (sscanf(argv[4], "%llu", &tmpll) != 1) { 164472d94861SAlasdair G Kergon ti->error = "Invalid device sector"; 164528513fccSMilan Broz goto bad; 16461da177e4SLinus Torvalds } 16474ee218cdSAndrew Morton cc->start = tmpll; 16481da177e4SLinus Torvalds 164928513fccSMilan Broz ret = -ENOMEM; 1650c0297721SAndi Kleen cc->io_queue = alloc_workqueue("kcryptd_io", 1651c0297721SAndi Kleen WQ_NON_REENTRANT| 1652c0297721SAndi Kleen WQ_MEM_RECLAIM, 1653c0297721SAndi Kleen 1); 1654cabf08e4SMilan Broz if (!cc->io_queue) { 1655cabf08e4SMilan Broz ti->error = "Couldn't create kcryptd io queue"; 165628513fccSMilan Broz goto bad; 1657cabf08e4SMilan Broz } 1658cabf08e4SMilan Broz 1659c0297721SAndi Kleen cc->crypt_queue = alloc_workqueue("kcryptd", 1660c0297721SAndi Kleen WQ_NON_REENTRANT| 1661c0297721SAndi Kleen WQ_CPU_INTENSIVE| 1662c0297721SAndi Kleen WQ_MEM_RECLAIM, 1663c0297721SAndi Kleen 1); 1664cabf08e4SMilan Broz if (!cc->crypt_queue) { 16659934a8beSMilan Broz ti->error = "Couldn't create kcryptd queue"; 166628513fccSMilan Broz goto bad; 16679934a8beSMilan Broz } 16689934a8beSMilan Broz 1669647c7db1SMikulas Patocka ti->num_flush_requests = 1; 16701da177e4SLinus Torvalds return 0; 16711da177e4SLinus Torvalds 167228513fccSMilan Broz bad: 167328513fccSMilan Broz crypt_dtr(ti); 167428513fccSMilan Broz return ret; 16751da177e4SLinus Torvalds } 16761da177e4SLinus Torvalds 16771da177e4SLinus Torvalds static int crypt_map(struct dm_target *ti, struct bio *bio, 16781da177e4SLinus Torvalds union map_info *map_context) 16791da177e4SLinus Torvalds { 1680028867acSAlasdair G Kergon struct dm_crypt_io *io; 1681647c7db1SMikulas Patocka struct crypt_config *cc; 1682647c7db1SMikulas Patocka 1683d87f4c14STejun Heo if (bio->bi_rw & REQ_FLUSH) { 1684647c7db1SMikulas Patocka cc = ti->private; 1685647c7db1SMikulas Patocka bio->bi_bdev = cc->dev->bdev; 1686647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 1687647c7db1SMikulas Patocka } 16881da177e4SLinus Torvalds 1689b441a262SAlasdair G Kergon io = crypt_io_alloc(ti, bio, dm_target_offset(ti, bio->bi_sector)); 1690cabf08e4SMilan Broz 169120c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 169220c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOWAIT)) 169323541d2dSMilan Broz kcryptd_queue_io(io); 169420c82538SMilan Broz } else 1695cabf08e4SMilan Broz kcryptd_queue_crypt(io); 16961da177e4SLinus Torvalds 1697d2a7ad29SKiyoshi Ueda return DM_MAPIO_SUBMITTED; 16981da177e4SLinus Torvalds } 16991da177e4SLinus Torvalds 17001da177e4SLinus Torvalds static int crypt_status(struct dm_target *ti, status_type_t type, 17011da177e4SLinus Torvalds char *result, unsigned int maxlen) 17021da177e4SLinus Torvalds { 17035ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 17041da177e4SLinus Torvalds unsigned int sz = 0; 17051da177e4SLinus Torvalds 17061da177e4SLinus Torvalds switch (type) { 17071da177e4SLinus Torvalds case STATUSTYPE_INFO: 17081da177e4SLinus Torvalds result[0] = '\0'; 17091da177e4SLinus Torvalds break; 17101da177e4SLinus Torvalds 17111da177e4SLinus Torvalds case STATUSTYPE_TABLE: 17127dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string); 17131da177e4SLinus Torvalds 17141da177e4SLinus Torvalds if (cc->key_size > 0) { 17151da177e4SLinus Torvalds if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 17161da177e4SLinus Torvalds return -ENOMEM; 17171da177e4SLinus Torvalds 17181da177e4SLinus Torvalds crypt_encode_key(result + sz, cc->key, cc->key_size); 17191da177e4SLinus Torvalds sz += cc->key_size << 1; 17201da177e4SLinus Torvalds } else { 17211da177e4SLinus Torvalds if (sz >= maxlen) 17221da177e4SLinus Torvalds return -ENOMEM; 17231da177e4SLinus Torvalds result[sz++] = '-'; 17241da177e4SLinus Torvalds } 17251da177e4SLinus Torvalds 17264ee218cdSAndrew Morton DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 17274ee218cdSAndrew Morton cc->dev->name, (unsigned long long)cc->start); 17281da177e4SLinus Torvalds break; 17291da177e4SLinus Torvalds } 17301da177e4SLinus Torvalds return 0; 17311da177e4SLinus Torvalds } 17321da177e4SLinus Torvalds 1733e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 1734e48d4bbfSMilan Broz { 1735e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1736e48d4bbfSMilan Broz 1737e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1738e48d4bbfSMilan Broz } 1739e48d4bbfSMilan Broz 1740e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 1741e48d4bbfSMilan Broz { 1742e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1743e48d4bbfSMilan Broz 1744e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1745e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 1746e48d4bbfSMilan Broz return -EAGAIN; 1747e48d4bbfSMilan Broz } 1748e48d4bbfSMilan Broz 1749e48d4bbfSMilan Broz return 0; 1750e48d4bbfSMilan Broz } 1751e48d4bbfSMilan Broz 1752e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 1753e48d4bbfSMilan Broz { 1754e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1755e48d4bbfSMilan Broz 1756e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1757e48d4bbfSMilan Broz } 1758e48d4bbfSMilan Broz 1759e48d4bbfSMilan Broz /* Message interface 1760e48d4bbfSMilan Broz * key set <key> 1761e48d4bbfSMilan Broz * key wipe 1762e48d4bbfSMilan Broz */ 1763e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1764e48d4bbfSMilan Broz { 1765e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1766542da317SMilan Broz int ret = -EINVAL; 1767e48d4bbfSMilan Broz 1768e48d4bbfSMilan Broz if (argc < 2) 1769e48d4bbfSMilan Broz goto error; 1770e48d4bbfSMilan Broz 1771e48d4bbfSMilan Broz if (!strnicmp(argv[0], MESG_STR("key"))) { 1772e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1773e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 1774e48d4bbfSMilan Broz return -EINVAL; 1775e48d4bbfSMilan Broz } 1776542da317SMilan Broz if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) { 1777542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 1778542da317SMilan Broz if (ret) 1779542da317SMilan Broz return ret; 1780542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 1781542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 1782542da317SMilan Broz return ret; 1783542da317SMilan Broz } 1784542da317SMilan Broz if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) { 1785542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 1786542da317SMilan Broz ret = cc->iv_gen_ops->wipe(cc); 1787542da317SMilan Broz if (ret) 1788542da317SMilan Broz return ret; 1789542da317SMilan Broz } 1790e48d4bbfSMilan Broz return crypt_wipe_key(cc); 1791e48d4bbfSMilan Broz } 1792542da317SMilan Broz } 1793e48d4bbfSMilan Broz 1794e48d4bbfSMilan Broz error: 1795e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 1796e48d4bbfSMilan Broz return -EINVAL; 1797e48d4bbfSMilan Broz } 1798e48d4bbfSMilan Broz 1799d41e26b9SMilan Broz static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 1800d41e26b9SMilan Broz struct bio_vec *biovec, int max_size) 1801d41e26b9SMilan Broz { 1802d41e26b9SMilan Broz struct crypt_config *cc = ti->private; 1803d41e26b9SMilan Broz struct request_queue *q = bdev_get_queue(cc->dev->bdev); 1804d41e26b9SMilan Broz 1805d41e26b9SMilan Broz if (!q->merge_bvec_fn) 1806d41e26b9SMilan Broz return max_size; 1807d41e26b9SMilan Broz 1808d41e26b9SMilan Broz bvm->bi_bdev = cc->dev->bdev; 1809b441a262SAlasdair G Kergon bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector); 1810d41e26b9SMilan Broz 1811d41e26b9SMilan Broz return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 1812d41e26b9SMilan Broz } 1813d41e26b9SMilan Broz 1814af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 1815af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 1816af4874e0SMike Snitzer { 1817af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 1818af4874e0SMike Snitzer 18195dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 1820af4874e0SMike Snitzer } 1821af4874e0SMike Snitzer 18221da177e4SLinus Torvalds static struct target_type crypt_target = { 18231da177e4SLinus Torvalds .name = "crypt", 1824d1f96423SMilan Broz .version = {1, 10, 0}, 18251da177e4SLinus Torvalds .module = THIS_MODULE, 18261da177e4SLinus Torvalds .ctr = crypt_ctr, 18271da177e4SLinus Torvalds .dtr = crypt_dtr, 18281da177e4SLinus Torvalds .map = crypt_map, 18291da177e4SLinus Torvalds .status = crypt_status, 1830e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 1831e48d4bbfSMilan Broz .preresume = crypt_preresume, 1832e48d4bbfSMilan Broz .resume = crypt_resume, 1833e48d4bbfSMilan Broz .message = crypt_message, 1834d41e26b9SMilan Broz .merge = crypt_merge, 1835af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 18361da177e4SLinus Torvalds }; 18371da177e4SLinus Torvalds 18381da177e4SLinus Torvalds static int __init dm_crypt_init(void) 18391da177e4SLinus Torvalds { 18401da177e4SLinus Torvalds int r; 18411da177e4SLinus Torvalds 1842028867acSAlasdair G Kergon _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 18431da177e4SLinus Torvalds if (!_crypt_io_pool) 18441da177e4SLinus Torvalds return -ENOMEM; 18451da177e4SLinus Torvalds 18461da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 18471da177e4SLinus Torvalds if (r < 0) { 184872d94861SAlasdair G Kergon DMERR("register failed %d", r); 18499934a8beSMilan Broz kmem_cache_destroy(_crypt_io_pool); 18501da177e4SLinus Torvalds } 18511da177e4SLinus Torvalds 18521da177e4SLinus Torvalds return r; 18531da177e4SLinus Torvalds } 18541da177e4SLinus Torvalds 18551da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 18561da177e4SLinus Torvalds { 185710d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 18581da177e4SLinus Torvalds kmem_cache_destroy(_crypt_io_pool); 18591da177e4SLinus Torvalds } 18601da177e4SLinus Torvalds 18611da177e4SLinus Torvalds module_init(dm_crypt_init); 18621da177e4SLinus Torvalds module_exit(dm_crypt_exit); 18631da177e4SLinus Torvalds 18641da177e4SLinus Torvalds MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 18651da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 18661da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1867