11da177e4SLinus Torvalds /* 2bf14299fSJana Saout * Copyright (C) 2003 Jana Saout <jana@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 454cea3f6SMilan Broz * Copyright (C) 2006-2015 Red Hat, Inc. All rights reserved. 5ed04d981SMilan Broz * Copyright (C) 2013 Milan Broz <gmazyland@gmail.com> 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * This file is released under the GPL. 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds 1043d69034SMilan Broz #include <linux/completion.h> 11d1806f6aSHerbert Xu #include <linux/err.h> 121da177e4SLinus Torvalds #include <linux/module.h> 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/kernel.h> 151da177e4SLinus Torvalds #include <linux/bio.h> 161da177e4SLinus Torvalds #include <linux/blkdev.h> 171da177e4SLinus Torvalds #include <linux/mempool.h> 181da177e4SLinus Torvalds #include <linux/slab.h> 191da177e4SLinus Torvalds #include <linux/crypto.h> 201da177e4SLinus Torvalds #include <linux/workqueue.h> 21dc267621SMikulas Patocka #include <linux/kthread.h> 223fcfab16SAndrew Morton #include <linux/backing-dev.h> 2360063497SArun Sharma #include <linux/atomic.h> 24378f058cSDavid Hardeman #include <linux/scatterlist.h> 25b3c5fd30SMikulas Patocka #include <linux/rbtree.h> 261da177e4SLinus Torvalds #include <asm/page.h> 2748527fa7SRik Snel #include <asm/unaligned.h> 2834745785SMilan Broz #include <crypto/hash.h> 2934745785SMilan Broz #include <crypto/md5.h> 3034745785SMilan Broz #include <crypto/algapi.h> 31bbdb23b5SHerbert Xu #include <crypto/skcipher.h> 321da177e4SLinus Torvalds 33586e80e6SMikulas Patocka #include <linux/device-mapper.h> 341da177e4SLinus Torvalds 3572d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 361da177e4SLinus Torvalds 371da177e4SLinus Torvalds /* 381da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 391da177e4SLinus Torvalds */ 401da177e4SLinus Torvalds struct convert_context { 4143d69034SMilan Broz struct completion restart; 421da177e4SLinus Torvalds struct bio *bio_in; 431da177e4SLinus Torvalds struct bio *bio_out; 44003b5c57SKent Overstreet struct bvec_iter iter_in; 45003b5c57SKent Overstreet struct bvec_iter iter_out; 46c66029f4SMikulas Patocka sector_t cc_sector; 4740b6229bSMikulas Patocka atomic_t cc_pending; 48bbdb23b5SHerbert Xu struct skcipher_request *req; 491da177e4SLinus Torvalds }; 501da177e4SLinus Torvalds 5153017030SMilan Broz /* 5253017030SMilan Broz * per bio private data 5353017030SMilan Broz */ 5453017030SMilan Broz struct dm_crypt_io { 5549a8a920SAlasdair G Kergon struct crypt_config *cc; 5653017030SMilan Broz struct bio *base_bio; 5753017030SMilan Broz struct work_struct work; 5853017030SMilan Broz 5953017030SMilan Broz struct convert_context ctx; 6053017030SMilan Broz 6140b6229bSMikulas Patocka atomic_t io_pending; 6253017030SMilan Broz int error; 630c395b0fSMilan Broz sector_t sector; 64dc267621SMikulas Patocka 65b3c5fd30SMikulas Patocka struct rb_node rb_node; 66298a9fa0SMikulas Patocka } CRYPTO_MINALIGN_ATTR; 6753017030SMilan Broz 6801482b76SMilan Broz struct dm_crypt_request { 69b2174eebSHuang Ying struct convert_context *ctx; 7001482b76SMilan Broz struct scatterlist sg_in; 7101482b76SMilan Broz struct scatterlist sg_out; 722dc5327dSMilan Broz sector_t iv_sector; 7301482b76SMilan Broz }; 7401482b76SMilan Broz 751da177e4SLinus Torvalds struct crypt_config; 761da177e4SLinus Torvalds 771da177e4SLinus Torvalds struct crypt_iv_operations { 781da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 791da177e4SLinus Torvalds const char *opts); 801da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 81b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 82542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 832dc5327dSMilan Broz int (*generator)(struct crypt_config *cc, u8 *iv, 842dc5327dSMilan Broz struct dm_crypt_request *dmreq); 852dc5327dSMilan Broz int (*post)(struct crypt_config *cc, u8 *iv, 862dc5327dSMilan Broz struct dm_crypt_request *dmreq); 871da177e4SLinus Torvalds }; 881da177e4SLinus Torvalds 8960473592SMilan Broz struct iv_essiv_private { 90bbdb23b5SHerbert Xu struct crypto_ahash *hash_tfm; 91b95bf2d3SMilan Broz u8 *salt; 9260473592SMilan Broz }; 9360473592SMilan Broz 9460473592SMilan Broz struct iv_benbi_private { 9560473592SMilan Broz int shift; 9660473592SMilan Broz }; 9760473592SMilan Broz 9834745785SMilan Broz #define LMK_SEED_SIZE 64 /* hash + 0 */ 9934745785SMilan Broz struct iv_lmk_private { 10034745785SMilan Broz struct crypto_shash *hash_tfm; 10134745785SMilan Broz u8 *seed; 10234745785SMilan Broz }; 10334745785SMilan Broz 104ed04d981SMilan Broz #define TCW_WHITENING_SIZE 16 105ed04d981SMilan Broz struct iv_tcw_private { 106ed04d981SMilan Broz struct crypto_shash *crc32_tfm; 107ed04d981SMilan Broz u8 *iv_seed; 108ed04d981SMilan Broz u8 *whitening; 109ed04d981SMilan Broz }; 110ed04d981SMilan Broz 1111da177e4SLinus Torvalds /* 1121da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 1131da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 1141da177e4SLinus Torvalds */ 1150f5d8e6eSMikulas Patocka enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID, 116bcbd94ffSMikulas Patocka DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD, 117bcbd94ffSMikulas Patocka DM_CRYPT_EXIT_THREAD}; 118c0297721SAndi Kleen 119c0297721SAndi Kleen /* 120610f2de3SMikulas Patocka * The fields in here must be read only after initialization. 121c0297721SAndi Kleen */ 1221da177e4SLinus Torvalds struct crypt_config { 1231da177e4SLinus Torvalds struct dm_dev *dev; 1241da177e4SLinus Torvalds sector_t start; 1251da177e4SLinus Torvalds 1261da177e4SLinus Torvalds /* 127ddd42edfSMilan Broz * pool for per bio private data, crypto requests and 128ddd42edfSMilan Broz * encryption requeusts/buffer pages 1291da177e4SLinus Torvalds */ 130ddd42edfSMilan Broz mempool_t *req_pool; 1311da177e4SLinus Torvalds mempool_t *page_pool; 1326a24c718SMilan Broz struct bio_set *bs; 1337145c241SMikulas Patocka struct mutex bio_alloc_lock; 1341da177e4SLinus Torvalds 135cabf08e4SMilan Broz struct workqueue_struct *io_queue; 136cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1373f1e9070SMilan Broz 138dc267621SMikulas Patocka struct task_struct *write_thread; 139dc267621SMikulas Patocka wait_queue_head_t write_thread_wait; 140b3c5fd30SMikulas Patocka struct rb_root write_tree; 141dc267621SMikulas Patocka 1425ebaee6dSMilan Broz char *cipher; 1437dbcd137SMilan Broz char *cipher_string; 1445ebaee6dSMilan Broz 1451da177e4SLinus Torvalds struct crypt_iv_operations *iv_gen_ops; 14679066ad3SHerbert Xu union { 14760473592SMilan Broz struct iv_essiv_private essiv; 14860473592SMilan Broz struct iv_benbi_private benbi; 14934745785SMilan Broz struct iv_lmk_private lmk; 150ed04d981SMilan Broz struct iv_tcw_private tcw; 15179066ad3SHerbert Xu } iv_gen_private; 1521da177e4SLinus Torvalds sector_t iv_offset; 1531da177e4SLinus Torvalds unsigned int iv_size; 1541da177e4SLinus Torvalds 155fd2d231fSMikulas Patocka /* ESSIV: struct crypto_cipher *essiv_tfm */ 156fd2d231fSMikulas Patocka void *iv_private; 157bbdb23b5SHerbert Xu struct crypto_skcipher **tfms; 158d1f96423SMilan Broz unsigned tfms_count; 159c0297721SAndi Kleen 160c0297721SAndi Kleen /* 161ddd42edfSMilan Broz * Layout of each crypto request: 162ddd42edfSMilan Broz * 163bbdb23b5SHerbert Xu * struct skcipher_request 164ddd42edfSMilan Broz * context 165ddd42edfSMilan Broz * padding 166ddd42edfSMilan Broz * struct dm_crypt_request 167ddd42edfSMilan Broz * padding 168ddd42edfSMilan Broz * IV 169ddd42edfSMilan Broz * 170ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 171ddd42edfSMilan Broz * correctly aligned. 172ddd42edfSMilan Broz */ 173ddd42edfSMilan Broz unsigned int dmreq_start; 174ddd42edfSMilan Broz 175298a9fa0SMikulas Patocka unsigned int per_bio_data_size; 176298a9fa0SMikulas Patocka 177e48d4bbfSMilan Broz unsigned long flags; 1781da177e4SLinus Torvalds unsigned int key_size; 179da31a078SMilan Broz unsigned int key_parts; /* independent parts in key buffer */ 180da31a078SMilan Broz unsigned int key_extra_size; /* additional keys length */ 1811da177e4SLinus Torvalds u8 key[0]; 1821da177e4SLinus Torvalds }; 1831da177e4SLinus Torvalds 1846a24c718SMilan Broz #define MIN_IOS 16 1851da177e4SLinus Torvalds 186028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 187395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 1882dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq); 189027581f3SOlaf Kirch 190c0297721SAndi Kleen /* 191c0297721SAndi Kleen * Use this to access cipher attributes that are the same for each CPU. 192c0297721SAndi Kleen */ 193bbdb23b5SHerbert Xu static struct crypto_skcipher *any_tfm(struct crypt_config *cc) 194c0297721SAndi Kleen { 195fd2d231fSMikulas Patocka return cc->tfms[0]; 196c0297721SAndi Kleen } 197c0297721SAndi Kleen 1981da177e4SLinus Torvalds /* 1991da177e4SLinus Torvalds * Different IV generation algorithms: 2001da177e4SLinus Torvalds * 2013c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 2023a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 2031da177e4SLinus Torvalds * 20461afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 20561afef61SMilan Broz * number, padded with zeros if necessary. 20661afef61SMilan Broz * 2073c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 2081da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 2091da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 2101da177e4SLinus Torvalds * 21148527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 21248527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 21348527fa7SRik Snel * 21446b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 21546b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 21646b47730SLudwig Nussel * 21734745785SMilan Broz * lmk: Compatible implementation of the block chaining mode used 21834745785SMilan Broz * by the Loop-AES block device encryption system 21934745785SMilan Broz * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/ 22034745785SMilan Broz * It operates on full 512 byte sectors and uses CBC 22134745785SMilan Broz * with an IV derived from the sector number, the data and 22234745785SMilan Broz * optionally extra IV seed. 22334745785SMilan Broz * This means that after decryption the first block 22434745785SMilan Broz * of sector must be tweaked according to decrypted data. 22534745785SMilan Broz * Loop-AES can use three encryption schemes: 22634745785SMilan Broz * version 1: is plain aes-cbc mode 22734745785SMilan Broz * version 2: uses 64 multikey scheme with lmk IV generator 22834745785SMilan Broz * version 3: the same as version 2 with additional IV seed 22934745785SMilan Broz * (it uses 65 keys, last key is used as IV seed) 23034745785SMilan Broz * 231ed04d981SMilan Broz * tcw: Compatible implementation of the block chaining mode used 232ed04d981SMilan Broz * by the TrueCrypt device encryption system (prior to version 4.1). 233e44f23b3SMilan Broz * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat 234ed04d981SMilan Broz * It operates on full 512 byte sectors and uses CBC 235ed04d981SMilan Broz * with an IV derived from initial key and the sector number. 236ed04d981SMilan Broz * In addition, whitening value is applied on every sector, whitening 237ed04d981SMilan Broz * is calculated from initial key, sector number and mixed using CRC32. 238ed04d981SMilan Broz * Note that this encryption scheme is vulnerable to watermarking attacks 239ed04d981SMilan Broz * and should be used for old compatible containers access only. 240ed04d981SMilan Broz * 2411da177e4SLinus Torvalds * plumb: unimplemented, see: 2421da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 2431da177e4SLinus Torvalds */ 2441da177e4SLinus Torvalds 2452dc5327dSMilan Broz static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, 2462dc5327dSMilan Broz struct dm_crypt_request *dmreq) 2471da177e4SLinus Torvalds { 2481da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 249283a8328SAlasdair G Kergon *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff); 2501da177e4SLinus Torvalds 2511da177e4SLinus Torvalds return 0; 2521da177e4SLinus Torvalds } 2531da177e4SLinus Torvalds 25461afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 2552dc5327dSMilan Broz struct dm_crypt_request *dmreq) 25661afef61SMilan Broz { 25761afef61SMilan Broz memset(iv, 0, cc->iv_size); 258283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 25961afef61SMilan Broz 26061afef61SMilan Broz return 0; 26161afef61SMilan Broz } 26261afef61SMilan Broz 263b95bf2d3SMilan Broz /* Initialise ESSIV - compute salt but no local memory allocations */ 264b95bf2d3SMilan Broz static int crypt_iv_essiv_init(struct crypt_config *cc) 265b95bf2d3SMilan Broz { 266b95bf2d3SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 267bbdb23b5SHerbert Xu AHASH_REQUEST_ON_STACK(req, essiv->hash_tfm); 268b95bf2d3SMilan Broz struct scatterlist sg; 269c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 270fd2d231fSMikulas Patocka int err; 271b95bf2d3SMilan Broz 272b95bf2d3SMilan Broz sg_init_one(&sg, cc->key, cc->key_size); 273bbdb23b5SHerbert Xu ahash_request_set_tfm(req, essiv->hash_tfm); 274bbdb23b5SHerbert Xu ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 275bbdb23b5SHerbert Xu ahash_request_set_crypt(req, &sg, essiv->salt, cc->key_size); 276b95bf2d3SMilan Broz 277bbdb23b5SHerbert Xu err = crypto_ahash_digest(req); 278bbdb23b5SHerbert Xu ahash_request_zero(req); 279b95bf2d3SMilan Broz if (err) 280b95bf2d3SMilan Broz return err; 281b95bf2d3SMilan Broz 282fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 283c0297721SAndi Kleen 284c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, essiv->salt, 285bbdb23b5SHerbert Xu crypto_ahash_digestsize(essiv->hash_tfm)); 286c0297721SAndi Kleen if (err) 287c0297721SAndi Kleen return err; 288c0297721SAndi Kleen 289c0297721SAndi Kleen return 0; 290b95bf2d3SMilan Broz } 291b95bf2d3SMilan Broz 292542da317SMilan Broz /* Wipe salt and reset key derived from volume key */ 293542da317SMilan Broz static int crypt_iv_essiv_wipe(struct crypt_config *cc) 294542da317SMilan Broz { 295542da317SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 296bbdb23b5SHerbert Xu unsigned salt_size = crypto_ahash_digestsize(essiv->hash_tfm); 297c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 298fd2d231fSMikulas Patocka int r, err = 0; 299542da317SMilan Broz 300542da317SMilan Broz memset(essiv->salt, 0, salt_size); 301542da317SMilan Broz 302fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 303c0297721SAndi Kleen r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size); 304c0297721SAndi Kleen if (r) 305c0297721SAndi Kleen err = r; 306c0297721SAndi Kleen 307c0297721SAndi Kleen return err; 308c0297721SAndi Kleen } 309c0297721SAndi Kleen 310c0297721SAndi Kleen /* Set up per cpu cipher state */ 311c0297721SAndi Kleen static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc, 312c0297721SAndi Kleen struct dm_target *ti, 313c0297721SAndi Kleen u8 *salt, unsigned saltsize) 314c0297721SAndi Kleen { 315c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 316c0297721SAndi Kleen int err; 317c0297721SAndi Kleen 318c0297721SAndi Kleen /* Setup the essiv_tfm with the given salt */ 319c0297721SAndi Kleen essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 320c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 321c0297721SAndi Kleen ti->error = "Error allocating crypto tfm for ESSIV"; 322c0297721SAndi Kleen return essiv_tfm; 323c0297721SAndi Kleen } 324c0297721SAndi Kleen 325c0297721SAndi Kleen if (crypto_cipher_blocksize(essiv_tfm) != 326bbdb23b5SHerbert Xu crypto_skcipher_ivsize(any_tfm(cc))) { 327c0297721SAndi Kleen ti->error = "Block size of ESSIV cipher does " 328c0297721SAndi Kleen "not match IV size of block cipher"; 329c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 330c0297721SAndi Kleen return ERR_PTR(-EINVAL); 331c0297721SAndi Kleen } 332c0297721SAndi Kleen 333c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 334c0297721SAndi Kleen if (err) { 335c0297721SAndi Kleen ti->error = "Failed to set key for ESSIV cipher"; 336c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 337c0297721SAndi Kleen return ERR_PTR(err); 338c0297721SAndi Kleen } 339c0297721SAndi Kleen 340c0297721SAndi Kleen return essiv_tfm; 341542da317SMilan Broz } 342542da317SMilan Broz 34360473592SMilan Broz static void crypt_iv_essiv_dtr(struct crypt_config *cc) 34460473592SMilan Broz { 345c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 34660473592SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 34760473592SMilan Broz 348bbdb23b5SHerbert Xu crypto_free_ahash(essiv->hash_tfm); 349b95bf2d3SMilan Broz essiv->hash_tfm = NULL; 350b95bf2d3SMilan Broz 351b95bf2d3SMilan Broz kzfree(essiv->salt); 352b95bf2d3SMilan Broz essiv->salt = NULL; 353c0297721SAndi Kleen 354fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 355c0297721SAndi Kleen 356c0297721SAndi Kleen if (essiv_tfm) 357c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 358c0297721SAndi Kleen 359fd2d231fSMikulas Patocka cc->iv_private = NULL; 36060473592SMilan Broz } 36160473592SMilan Broz 3621da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 3631da177e4SLinus Torvalds const char *opts) 3641da177e4SLinus Torvalds { 3655861f1beSMilan Broz struct crypto_cipher *essiv_tfm = NULL; 366bbdb23b5SHerbert Xu struct crypto_ahash *hash_tfm = NULL; 3675861f1beSMilan Broz u8 *salt = NULL; 368fd2d231fSMikulas Patocka int err; 3691da177e4SLinus Torvalds 3705861f1beSMilan Broz if (!opts) { 37172d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 3721da177e4SLinus Torvalds return -EINVAL; 3731da177e4SLinus Torvalds } 3741da177e4SLinus Torvalds 375b95bf2d3SMilan Broz /* Allocate hash algorithm */ 376bbdb23b5SHerbert Xu hash_tfm = crypto_alloc_ahash(opts, 0, CRYPTO_ALG_ASYNC); 37735058687SHerbert Xu if (IS_ERR(hash_tfm)) { 37872d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 3795861f1beSMilan Broz err = PTR_ERR(hash_tfm); 3805861f1beSMilan Broz goto bad; 3811da177e4SLinus Torvalds } 3821da177e4SLinus Torvalds 383bbdb23b5SHerbert Xu salt = kzalloc(crypto_ahash_digestsize(hash_tfm), GFP_KERNEL); 3845861f1beSMilan Broz if (!salt) { 38572d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 3865861f1beSMilan Broz err = -ENOMEM; 3875861f1beSMilan Broz goto bad; 3881da177e4SLinus Torvalds } 3891da177e4SLinus Torvalds 390b95bf2d3SMilan Broz cc->iv_gen_private.essiv.salt = salt; 391b95bf2d3SMilan Broz cc->iv_gen_private.essiv.hash_tfm = hash_tfm; 392b95bf2d3SMilan Broz 393c0297721SAndi Kleen essiv_tfm = setup_essiv_cpu(cc, ti, salt, 394bbdb23b5SHerbert Xu crypto_ahash_digestsize(hash_tfm)); 395c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 396c0297721SAndi Kleen crypt_iv_essiv_dtr(cc); 397c0297721SAndi Kleen return PTR_ERR(essiv_tfm); 398c0297721SAndi Kleen } 399fd2d231fSMikulas Patocka cc->iv_private = essiv_tfm; 400c0297721SAndi Kleen 4011da177e4SLinus Torvalds return 0; 4025861f1beSMilan Broz 4035861f1beSMilan Broz bad: 4045861f1beSMilan Broz if (hash_tfm && !IS_ERR(hash_tfm)) 405bbdb23b5SHerbert Xu crypto_free_ahash(hash_tfm); 406b95bf2d3SMilan Broz kfree(salt); 4075861f1beSMilan Broz return err; 4081da177e4SLinus Torvalds } 4091da177e4SLinus Torvalds 4102dc5327dSMilan Broz static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, 4112dc5327dSMilan Broz struct dm_crypt_request *dmreq) 4121da177e4SLinus Torvalds { 413fd2d231fSMikulas Patocka struct crypto_cipher *essiv_tfm = cc->iv_private; 414c0297721SAndi Kleen 4151da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 416283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 417c0297721SAndi Kleen crypto_cipher_encrypt_one(essiv_tfm, iv, iv); 418c0297721SAndi Kleen 4191da177e4SLinus Torvalds return 0; 4201da177e4SLinus Torvalds } 4211da177e4SLinus Torvalds 42248527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 42348527fa7SRik Snel const char *opts) 42448527fa7SRik Snel { 425bbdb23b5SHerbert Xu unsigned bs = crypto_skcipher_blocksize(any_tfm(cc)); 426f0d1b0b3SDavid Howells int log = ilog2(bs); 42748527fa7SRik Snel 42848527fa7SRik Snel /* we need to calculate how far we must shift the sector count 42948527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 43048527fa7SRik Snel 43148527fa7SRik Snel if (1 << log != bs) { 43248527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 43348527fa7SRik Snel return -EINVAL; 43448527fa7SRik Snel } 43548527fa7SRik Snel 43648527fa7SRik Snel if (log > 9) { 43748527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 43848527fa7SRik Snel return -EINVAL; 43948527fa7SRik Snel } 44048527fa7SRik Snel 44160473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 44248527fa7SRik Snel 44348527fa7SRik Snel return 0; 44448527fa7SRik Snel } 44548527fa7SRik Snel 44648527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 44748527fa7SRik Snel { 44848527fa7SRik Snel } 44948527fa7SRik Snel 4502dc5327dSMilan Broz static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, 4512dc5327dSMilan Broz struct dm_crypt_request *dmreq) 45248527fa7SRik Snel { 45379066ad3SHerbert Xu __be64 val; 45479066ad3SHerbert Xu 45548527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 45679066ad3SHerbert Xu 4572dc5327dSMilan Broz val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1); 45879066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 45948527fa7SRik Snel 4601da177e4SLinus Torvalds return 0; 4611da177e4SLinus Torvalds } 4621da177e4SLinus Torvalds 4632dc5327dSMilan Broz static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, 4642dc5327dSMilan Broz struct dm_crypt_request *dmreq) 46546b47730SLudwig Nussel { 46646b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 46746b47730SLudwig Nussel 46846b47730SLudwig Nussel return 0; 46946b47730SLudwig Nussel } 47046b47730SLudwig Nussel 47134745785SMilan Broz static void crypt_iv_lmk_dtr(struct crypt_config *cc) 47234745785SMilan Broz { 47334745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 47434745785SMilan Broz 47534745785SMilan Broz if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm)) 47634745785SMilan Broz crypto_free_shash(lmk->hash_tfm); 47734745785SMilan Broz lmk->hash_tfm = NULL; 47834745785SMilan Broz 47934745785SMilan Broz kzfree(lmk->seed); 48034745785SMilan Broz lmk->seed = NULL; 48134745785SMilan Broz } 48234745785SMilan Broz 48334745785SMilan Broz static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti, 48434745785SMilan Broz const char *opts) 48534745785SMilan Broz { 48634745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 48734745785SMilan Broz 48834745785SMilan Broz lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0); 48934745785SMilan Broz if (IS_ERR(lmk->hash_tfm)) { 49034745785SMilan Broz ti->error = "Error initializing LMK hash"; 49134745785SMilan Broz return PTR_ERR(lmk->hash_tfm); 49234745785SMilan Broz } 49334745785SMilan Broz 49434745785SMilan Broz /* No seed in LMK version 2 */ 49534745785SMilan Broz if (cc->key_parts == cc->tfms_count) { 49634745785SMilan Broz lmk->seed = NULL; 49734745785SMilan Broz return 0; 49834745785SMilan Broz } 49934745785SMilan Broz 50034745785SMilan Broz lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL); 50134745785SMilan Broz if (!lmk->seed) { 50234745785SMilan Broz crypt_iv_lmk_dtr(cc); 50334745785SMilan Broz ti->error = "Error kmallocing seed storage in LMK"; 50434745785SMilan Broz return -ENOMEM; 50534745785SMilan Broz } 50634745785SMilan Broz 50734745785SMilan Broz return 0; 50834745785SMilan Broz } 50934745785SMilan Broz 51034745785SMilan Broz static int crypt_iv_lmk_init(struct crypt_config *cc) 51134745785SMilan Broz { 51234745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 51334745785SMilan Broz int subkey_size = cc->key_size / cc->key_parts; 51434745785SMilan Broz 51534745785SMilan Broz /* LMK seed is on the position of LMK_KEYS + 1 key */ 51634745785SMilan Broz if (lmk->seed) 51734745785SMilan Broz memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size), 51834745785SMilan Broz crypto_shash_digestsize(lmk->hash_tfm)); 51934745785SMilan Broz 52034745785SMilan Broz return 0; 52134745785SMilan Broz } 52234745785SMilan Broz 52334745785SMilan Broz static int crypt_iv_lmk_wipe(struct crypt_config *cc) 52434745785SMilan Broz { 52534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 52634745785SMilan Broz 52734745785SMilan Broz if (lmk->seed) 52834745785SMilan Broz memset(lmk->seed, 0, LMK_SEED_SIZE); 52934745785SMilan Broz 53034745785SMilan Broz return 0; 53134745785SMilan Broz } 53234745785SMilan Broz 53334745785SMilan Broz static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv, 53434745785SMilan Broz struct dm_crypt_request *dmreq, 53534745785SMilan Broz u8 *data) 53634745785SMilan Broz { 53734745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 538b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, lmk->hash_tfm); 53934745785SMilan Broz struct md5_state md5state; 540da31a078SMilan Broz __le32 buf[4]; 54134745785SMilan Broz int i, r; 54234745785SMilan Broz 543b6106265SJan-Simon Möller desc->tfm = lmk->hash_tfm; 544b6106265SJan-Simon Möller desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 54534745785SMilan Broz 546b6106265SJan-Simon Möller r = crypto_shash_init(desc); 54734745785SMilan Broz if (r) 54834745785SMilan Broz return r; 54934745785SMilan Broz 55034745785SMilan Broz if (lmk->seed) { 551b6106265SJan-Simon Möller r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE); 55234745785SMilan Broz if (r) 55334745785SMilan Broz return r; 55434745785SMilan Broz } 55534745785SMilan Broz 55634745785SMilan Broz /* Sector is always 512B, block size 16, add data of blocks 1-31 */ 557b6106265SJan-Simon Möller r = crypto_shash_update(desc, data + 16, 16 * 31); 55834745785SMilan Broz if (r) 55934745785SMilan Broz return r; 56034745785SMilan Broz 56134745785SMilan Broz /* Sector is cropped to 56 bits here */ 56234745785SMilan Broz buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF); 56334745785SMilan Broz buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000); 56434745785SMilan Broz buf[2] = cpu_to_le32(4024); 56534745785SMilan Broz buf[3] = 0; 566b6106265SJan-Simon Möller r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf)); 56734745785SMilan Broz if (r) 56834745785SMilan Broz return r; 56934745785SMilan Broz 57034745785SMilan Broz /* No MD5 padding here */ 571b6106265SJan-Simon Möller r = crypto_shash_export(desc, &md5state); 57234745785SMilan Broz if (r) 57334745785SMilan Broz return r; 57434745785SMilan Broz 57534745785SMilan Broz for (i = 0; i < MD5_HASH_WORDS; i++) 57634745785SMilan Broz __cpu_to_le32s(&md5state.hash[i]); 57734745785SMilan Broz memcpy(iv, &md5state.hash, cc->iv_size); 57834745785SMilan Broz 57934745785SMilan Broz return 0; 58034745785SMilan Broz } 58134745785SMilan Broz 58234745785SMilan Broz static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv, 58334745785SMilan Broz struct dm_crypt_request *dmreq) 58434745785SMilan Broz { 58534745785SMilan Broz u8 *src; 58634745785SMilan Broz int r = 0; 58734745785SMilan Broz 58834745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 589c2e022cbSCong Wang src = kmap_atomic(sg_page(&dmreq->sg_in)); 59034745785SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, src + dmreq->sg_in.offset); 591c2e022cbSCong Wang kunmap_atomic(src); 59234745785SMilan Broz } else 59334745785SMilan Broz memset(iv, 0, cc->iv_size); 59434745785SMilan Broz 59534745785SMilan Broz return r; 59634745785SMilan Broz } 59734745785SMilan Broz 59834745785SMilan Broz static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv, 59934745785SMilan Broz struct dm_crypt_request *dmreq) 60034745785SMilan Broz { 60134745785SMilan Broz u8 *dst; 60234745785SMilan Broz int r; 60334745785SMilan Broz 60434745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) 60534745785SMilan Broz return 0; 60634745785SMilan Broz 607c2e022cbSCong Wang dst = kmap_atomic(sg_page(&dmreq->sg_out)); 60834745785SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, dst + dmreq->sg_out.offset); 60934745785SMilan Broz 61034745785SMilan Broz /* Tweak the first block of plaintext sector */ 61134745785SMilan Broz if (!r) 61234745785SMilan Broz crypto_xor(dst + dmreq->sg_out.offset, iv, cc->iv_size); 61334745785SMilan Broz 614c2e022cbSCong Wang kunmap_atomic(dst); 61534745785SMilan Broz return r; 61634745785SMilan Broz } 61734745785SMilan Broz 618ed04d981SMilan Broz static void crypt_iv_tcw_dtr(struct crypt_config *cc) 619ed04d981SMilan Broz { 620ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 621ed04d981SMilan Broz 622ed04d981SMilan Broz kzfree(tcw->iv_seed); 623ed04d981SMilan Broz tcw->iv_seed = NULL; 624ed04d981SMilan Broz kzfree(tcw->whitening); 625ed04d981SMilan Broz tcw->whitening = NULL; 626ed04d981SMilan Broz 627ed04d981SMilan Broz if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm)) 628ed04d981SMilan Broz crypto_free_shash(tcw->crc32_tfm); 629ed04d981SMilan Broz tcw->crc32_tfm = NULL; 630ed04d981SMilan Broz } 631ed04d981SMilan Broz 632ed04d981SMilan Broz static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti, 633ed04d981SMilan Broz const char *opts) 634ed04d981SMilan Broz { 635ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 636ed04d981SMilan Broz 637ed04d981SMilan Broz if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) { 638ed04d981SMilan Broz ti->error = "Wrong key size for TCW"; 639ed04d981SMilan Broz return -EINVAL; 640ed04d981SMilan Broz } 641ed04d981SMilan Broz 642ed04d981SMilan Broz tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 0); 643ed04d981SMilan Broz if (IS_ERR(tcw->crc32_tfm)) { 644ed04d981SMilan Broz ti->error = "Error initializing CRC32 in TCW"; 645ed04d981SMilan Broz return PTR_ERR(tcw->crc32_tfm); 646ed04d981SMilan Broz } 647ed04d981SMilan Broz 648ed04d981SMilan Broz tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL); 649ed04d981SMilan Broz tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL); 650ed04d981SMilan Broz if (!tcw->iv_seed || !tcw->whitening) { 651ed04d981SMilan Broz crypt_iv_tcw_dtr(cc); 652ed04d981SMilan Broz ti->error = "Error allocating seed storage in TCW"; 653ed04d981SMilan Broz return -ENOMEM; 654ed04d981SMilan Broz } 655ed04d981SMilan Broz 656ed04d981SMilan Broz return 0; 657ed04d981SMilan Broz } 658ed04d981SMilan Broz 659ed04d981SMilan Broz static int crypt_iv_tcw_init(struct crypt_config *cc) 660ed04d981SMilan Broz { 661ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 662ed04d981SMilan Broz int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE; 663ed04d981SMilan Broz 664ed04d981SMilan Broz memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size); 665ed04d981SMilan Broz memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size], 666ed04d981SMilan Broz TCW_WHITENING_SIZE); 667ed04d981SMilan Broz 668ed04d981SMilan Broz return 0; 669ed04d981SMilan Broz } 670ed04d981SMilan Broz 671ed04d981SMilan Broz static int crypt_iv_tcw_wipe(struct crypt_config *cc) 672ed04d981SMilan Broz { 673ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 674ed04d981SMilan Broz 675ed04d981SMilan Broz memset(tcw->iv_seed, 0, cc->iv_size); 676ed04d981SMilan Broz memset(tcw->whitening, 0, TCW_WHITENING_SIZE); 677ed04d981SMilan Broz 678ed04d981SMilan Broz return 0; 679ed04d981SMilan Broz } 680ed04d981SMilan Broz 681ed04d981SMilan Broz static int crypt_iv_tcw_whitening(struct crypt_config *cc, 682ed04d981SMilan Broz struct dm_crypt_request *dmreq, 683ed04d981SMilan Broz u8 *data) 684ed04d981SMilan Broz { 685ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 686ed04d981SMilan Broz u64 sector = cpu_to_le64((u64)dmreq->iv_sector); 687ed04d981SMilan Broz u8 buf[TCW_WHITENING_SIZE]; 688b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm); 689ed04d981SMilan Broz int i, r; 690ed04d981SMilan Broz 691ed04d981SMilan Broz /* xor whitening with sector number */ 692ed04d981SMilan Broz memcpy(buf, tcw->whitening, TCW_WHITENING_SIZE); 693ed04d981SMilan Broz crypto_xor(buf, (u8 *)§or, 8); 694ed04d981SMilan Broz crypto_xor(&buf[8], (u8 *)§or, 8); 695ed04d981SMilan Broz 696ed04d981SMilan Broz /* calculate crc32 for every 32bit part and xor it */ 697b6106265SJan-Simon Möller desc->tfm = tcw->crc32_tfm; 698b6106265SJan-Simon Möller desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 699ed04d981SMilan Broz for (i = 0; i < 4; i++) { 700b6106265SJan-Simon Möller r = crypto_shash_init(desc); 701ed04d981SMilan Broz if (r) 702ed04d981SMilan Broz goto out; 703b6106265SJan-Simon Möller r = crypto_shash_update(desc, &buf[i * 4], 4); 704ed04d981SMilan Broz if (r) 705ed04d981SMilan Broz goto out; 706b6106265SJan-Simon Möller r = crypto_shash_final(desc, &buf[i * 4]); 707ed04d981SMilan Broz if (r) 708ed04d981SMilan Broz goto out; 709ed04d981SMilan Broz } 710ed04d981SMilan Broz crypto_xor(&buf[0], &buf[12], 4); 711ed04d981SMilan Broz crypto_xor(&buf[4], &buf[8], 4); 712ed04d981SMilan Broz 713ed04d981SMilan Broz /* apply whitening (8 bytes) to whole sector */ 714ed04d981SMilan Broz for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++) 715ed04d981SMilan Broz crypto_xor(data + i * 8, buf, 8); 716ed04d981SMilan Broz out: 7171a71d6ffSMilan Broz memzero_explicit(buf, sizeof(buf)); 718ed04d981SMilan Broz return r; 719ed04d981SMilan Broz } 720ed04d981SMilan Broz 721ed04d981SMilan Broz static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv, 722ed04d981SMilan Broz struct dm_crypt_request *dmreq) 723ed04d981SMilan Broz { 724ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 725ed04d981SMilan Broz u64 sector = cpu_to_le64((u64)dmreq->iv_sector); 726ed04d981SMilan Broz u8 *src; 727ed04d981SMilan Broz int r = 0; 728ed04d981SMilan Broz 729ed04d981SMilan Broz /* Remove whitening from ciphertext */ 730ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) { 731ed04d981SMilan Broz src = kmap_atomic(sg_page(&dmreq->sg_in)); 732ed04d981SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, src + dmreq->sg_in.offset); 733ed04d981SMilan Broz kunmap_atomic(src); 734ed04d981SMilan Broz } 735ed04d981SMilan Broz 736ed04d981SMilan Broz /* Calculate IV */ 737ed04d981SMilan Broz memcpy(iv, tcw->iv_seed, cc->iv_size); 738ed04d981SMilan Broz crypto_xor(iv, (u8 *)§or, 8); 739ed04d981SMilan Broz if (cc->iv_size > 8) 740ed04d981SMilan Broz crypto_xor(&iv[8], (u8 *)§or, cc->iv_size - 8); 741ed04d981SMilan Broz 742ed04d981SMilan Broz return r; 743ed04d981SMilan Broz } 744ed04d981SMilan Broz 745ed04d981SMilan Broz static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv, 746ed04d981SMilan Broz struct dm_crypt_request *dmreq) 747ed04d981SMilan Broz { 748ed04d981SMilan Broz u8 *dst; 749ed04d981SMilan Broz int r; 750ed04d981SMilan Broz 751ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) 752ed04d981SMilan Broz return 0; 753ed04d981SMilan Broz 754ed04d981SMilan Broz /* Apply whitening on ciphertext */ 755ed04d981SMilan Broz dst = kmap_atomic(sg_page(&dmreq->sg_out)); 756ed04d981SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, dst + dmreq->sg_out.offset); 757ed04d981SMilan Broz kunmap_atomic(dst); 758ed04d981SMilan Broz 759ed04d981SMilan Broz return r; 760ed04d981SMilan Broz } 761ed04d981SMilan Broz 7621da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_plain_ops = { 7631da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 7641da177e4SLinus Torvalds }; 7651da177e4SLinus Torvalds 76661afef61SMilan Broz static struct crypt_iv_operations crypt_iv_plain64_ops = { 76761afef61SMilan Broz .generator = crypt_iv_plain64_gen 76861afef61SMilan Broz }; 76961afef61SMilan Broz 7701da177e4SLinus Torvalds static struct crypt_iv_operations crypt_iv_essiv_ops = { 7711da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 7721da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 773b95bf2d3SMilan Broz .init = crypt_iv_essiv_init, 774542da317SMilan Broz .wipe = crypt_iv_essiv_wipe, 7751da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 7761da177e4SLinus Torvalds }; 7771da177e4SLinus Torvalds 77848527fa7SRik Snel static struct crypt_iv_operations crypt_iv_benbi_ops = { 77948527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 78048527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 78148527fa7SRik Snel .generator = crypt_iv_benbi_gen 78248527fa7SRik Snel }; 7831da177e4SLinus Torvalds 78446b47730SLudwig Nussel static struct crypt_iv_operations crypt_iv_null_ops = { 78546b47730SLudwig Nussel .generator = crypt_iv_null_gen 78646b47730SLudwig Nussel }; 78746b47730SLudwig Nussel 78834745785SMilan Broz static struct crypt_iv_operations crypt_iv_lmk_ops = { 78934745785SMilan Broz .ctr = crypt_iv_lmk_ctr, 79034745785SMilan Broz .dtr = crypt_iv_lmk_dtr, 79134745785SMilan Broz .init = crypt_iv_lmk_init, 79234745785SMilan Broz .wipe = crypt_iv_lmk_wipe, 79334745785SMilan Broz .generator = crypt_iv_lmk_gen, 79434745785SMilan Broz .post = crypt_iv_lmk_post 79534745785SMilan Broz }; 79634745785SMilan Broz 797ed04d981SMilan Broz static struct crypt_iv_operations crypt_iv_tcw_ops = { 798ed04d981SMilan Broz .ctr = crypt_iv_tcw_ctr, 799ed04d981SMilan Broz .dtr = crypt_iv_tcw_dtr, 800ed04d981SMilan Broz .init = crypt_iv_tcw_init, 801ed04d981SMilan Broz .wipe = crypt_iv_tcw_wipe, 802ed04d981SMilan Broz .generator = crypt_iv_tcw_gen, 803ed04d981SMilan Broz .post = crypt_iv_tcw_post 804ed04d981SMilan Broz }; 805ed04d981SMilan Broz 806d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 807d469f841SMilan Broz struct convert_context *ctx, 8081da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 809fcd369daSMilan Broz sector_t sector) 8101da177e4SLinus Torvalds { 8111da177e4SLinus Torvalds ctx->bio_in = bio_in; 8121da177e4SLinus Torvalds ctx->bio_out = bio_out; 813003b5c57SKent Overstreet if (bio_in) 814003b5c57SKent Overstreet ctx->iter_in = bio_in->bi_iter; 815003b5c57SKent Overstreet if (bio_out) 816003b5c57SKent Overstreet ctx->iter_out = bio_out->bi_iter; 817c66029f4SMikulas Patocka ctx->cc_sector = sector + cc->iv_offset; 81843d69034SMilan Broz init_completion(&ctx->restart); 8191da177e4SLinus Torvalds } 8201da177e4SLinus Torvalds 821b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 822bbdb23b5SHerbert Xu struct skcipher_request *req) 823b2174eebSHuang Ying { 824b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 825b2174eebSHuang Ying } 826b2174eebSHuang Ying 827bbdb23b5SHerbert Xu static struct skcipher_request *req_of_dmreq(struct crypt_config *cc, 828b2174eebSHuang Ying struct dm_crypt_request *dmreq) 829b2174eebSHuang Ying { 830bbdb23b5SHerbert Xu return (struct skcipher_request *)((char *)dmreq - cc->dmreq_start); 831b2174eebSHuang Ying } 832b2174eebSHuang Ying 8332dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, 8342dc5327dSMilan Broz struct dm_crypt_request *dmreq) 8352dc5327dSMilan Broz { 8362dc5327dSMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 837bbdb23b5SHerbert Xu crypto_skcipher_alignmask(any_tfm(cc)) + 1); 8382dc5327dSMilan Broz } 8392dc5327dSMilan Broz 84001482b76SMilan Broz static int crypt_convert_block(struct crypt_config *cc, 8413a7f6c99SMilan Broz struct convert_context *ctx, 842bbdb23b5SHerbert Xu struct skcipher_request *req) 84301482b76SMilan Broz { 844003b5c57SKent Overstreet struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in); 845003b5c57SKent Overstreet struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out); 8463a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 8473a7f6c99SMilan Broz u8 *iv; 84840b6229bSMikulas Patocka int r; 84901482b76SMilan Broz 850b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 8512dc5327dSMilan Broz iv = iv_of_dmreq(cc, dmreq); 8523a7f6c99SMilan Broz 853c66029f4SMikulas Patocka dmreq->iv_sector = ctx->cc_sector; 854b2174eebSHuang Ying dmreq->ctx = ctx; 8553a7f6c99SMilan Broz sg_init_table(&dmreq->sg_in, 1); 856003b5c57SKent Overstreet sg_set_page(&dmreq->sg_in, bv_in.bv_page, 1 << SECTOR_SHIFT, 857003b5c57SKent Overstreet bv_in.bv_offset); 85801482b76SMilan Broz 8593a7f6c99SMilan Broz sg_init_table(&dmreq->sg_out, 1); 860003b5c57SKent Overstreet sg_set_page(&dmreq->sg_out, bv_out.bv_page, 1 << SECTOR_SHIFT, 861003b5c57SKent Overstreet bv_out.bv_offset); 86201482b76SMilan Broz 863003b5c57SKent Overstreet bio_advance_iter(ctx->bio_in, &ctx->iter_in, 1 << SECTOR_SHIFT); 864003b5c57SKent Overstreet bio_advance_iter(ctx->bio_out, &ctx->iter_out, 1 << SECTOR_SHIFT); 86501482b76SMilan Broz 8663a7f6c99SMilan Broz if (cc->iv_gen_ops) { 8672dc5327dSMilan Broz r = cc->iv_gen_ops->generator(cc, iv, dmreq); 8683a7f6c99SMilan Broz if (r < 0) 8693a7f6c99SMilan Broz return r; 8703a7f6c99SMilan Broz } 8713a7f6c99SMilan Broz 872bbdb23b5SHerbert Xu skcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out, 8733a7f6c99SMilan Broz 1 << SECTOR_SHIFT, iv); 8743a7f6c99SMilan Broz 8753a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 876bbdb23b5SHerbert Xu r = crypto_skcipher_encrypt(req); 8773a7f6c99SMilan Broz else 878bbdb23b5SHerbert Xu r = crypto_skcipher_decrypt(req); 8793a7f6c99SMilan Broz 8802dc5327dSMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 8812dc5327dSMilan Broz r = cc->iv_gen_ops->post(cc, iv, dmreq); 8822dc5327dSMilan Broz 8833a7f6c99SMilan Broz return r; 88401482b76SMilan Broz } 88501482b76SMilan Broz 88695497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 88795497a96SMilan Broz int error); 888c0297721SAndi Kleen 889ddd42edfSMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 890ddd42edfSMilan Broz struct convert_context *ctx) 891ddd42edfSMilan Broz { 892c66029f4SMikulas Patocka unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1); 893c0297721SAndi Kleen 894610f2de3SMikulas Patocka if (!ctx->req) 895610f2de3SMikulas Patocka ctx->req = mempool_alloc(cc->req_pool, GFP_NOIO); 896c0297721SAndi Kleen 897bbdb23b5SHerbert Xu skcipher_request_set_tfm(ctx->req, cc->tfms[key_index]); 89854cea3f6SMilan Broz 89954cea3f6SMilan Broz /* 90054cea3f6SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs 90154cea3f6SMilan Broz * requests if driver request queue is full. 90254cea3f6SMilan Broz */ 903bbdb23b5SHerbert Xu skcipher_request_set_callback(ctx->req, 904c0297721SAndi Kleen CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, 905610f2de3SMikulas Patocka kcryptd_async_done, dmreq_of_req(cc, ctx->req)); 906ddd42edfSMilan Broz } 907ddd42edfSMilan Broz 908298a9fa0SMikulas Patocka static void crypt_free_req(struct crypt_config *cc, 909bbdb23b5SHerbert Xu struct skcipher_request *req, struct bio *base_bio) 910298a9fa0SMikulas Patocka { 911298a9fa0SMikulas Patocka struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); 912298a9fa0SMikulas Patocka 913bbdb23b5SHerbert Xu if ((struct skcipher_request *)(io + 1) != req) 914298a9fa0SMikulas Patocka mempool_free(req, cc->req_pool); 915298a9fa0SMikulas Patocka } 916298a9fa0SMikulas Patocka 9171da177e4SLinus Torvalds /* 9181da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 9191da177e4SLinus Torvalds */ 9201da177e4SLinus Torvalds static int crypt_convert(struct crypt_config *cc, 9211da177e4SLinus Torvalds struct convert_context *ctx) 9221da177e4SLinus Torvalds { 9233f1e9070SMilan Broz int r; 9241da177e4SLinus Torvalds 92540b6229bSMikulas Patocka atomic_set(&ctx->cc_pending, 1); 926c8081618SMilan Broz 927003b5c57SKent Overstreet while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) { 9281da177e4SLinus Torvalds 9293a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 9303a7f6c99SMilan Broz 93140b6229bSMikulas Patocka atomic_inc(&ctx->cc_pending); 9323f1e9070SMilan Broz 933610f2de3SMikulas Patocka r = crypt_convert_block(cc, ctx, ctx->req); 9343a7f6c99SMilan Broz 9353a7f6c99SMilan Broz switch (r) { 93654cea3f6SMilan Broz /* 93754cea3f6SMilan Broz * The request was queued by a crypto driver 93854cea3f6SMilan Broz * but the driver request queue is full, let's wait. 93954cea3f6SMilan Broz */ 9403a7f6c99SMilan Broz case -EBUSY: 9413a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 94216735d02SWolfram Sang reinit_completion(&ctx->restart); 943c0403ec0SRabin Vincent /* fall through */ 94454cea3f6SMilan Broz /* 94554cea3f6SMilan Broz * The request is queued and processed asynchronously, 94654cea3f6SMilan Broz * completion function kcryptd_async_done() will be called. 94754cea3f6SMilan Broz */ 948c0403ec0SRabin Vincent case -EINPROGRESS: 949610f2de3SMikulas Patocka ctx->req = NULL; 950c66029f4SMikulas Patocka ctx->cc_sector++; 9513a7f6c99SMilan Broz continue; 95254cea3f6SMilan Broz /* 95354cea3f6SMilan Broz * The request was already processed (synchronously). 95454cea3f6SMilan Broz */ 9553f1e9070SMilan Broz case 0: 95640b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 957c66029f4SMikulas Patocka ctx->cc_sector++; 958c7f1b204SMilan Broz cond_resched(); 9593f1e9070SMilan Broz continue; 9601da177e4SLinus Torvalds 96154cea3f6SMilan Broz /* There was an error while processing the request. */ 9623f1e9070SMilan Broz default: 96340b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 9641da177e4SLinus Torvalds return r; 9651da177e4SLinus Torvalds } 9663f1e9070SMilan Broz } 9673f1e9070SMilan Broz 9683f1e9070SMilan Broz return 0; 9693f1e9070SMilan Broz } 9701da177e4SLinus Torvalds 971cf2f1abfSMikulas Patocka static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone); 972cf2f1abfSMikulas Patocka 9731da177e4SLinus Torvalds /* 9741da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 975586b286bSMike Snitzer * This should never violate the device limitations (but only because 976586b286bSMike Snitzer * max_segment_size is being constrained to PAGE_SIZE). 9777145c241SMikulas Patocka * 9787145c241SMikulas Patocka * This function may be called concurrently. If we allocate from the mempool 9797145c241SMikulas Patocka * concurrently, there is a possibility of deadlock. For example, if we have 9807145c241SMikulas Patocka * mempool of 256 pages, two processes, each wanting 256, pages allocate from 9817145c241SMikulas Patocka * the mempool concurrently, it may deadlock in a situation where both processes 9827145c241SMikulas Patocka * have allocated 128 pages and the mempool is exhausted. 9837145c241SMikulas Patocka * 9847145c241SMikulas Patocka * In order to avoid this scenario we allocate the pages under a mutex. 9857145c241SMikulas Patocka * 9867145c241SMikulas Patocka * In order to not degrade performance with excessive locking, we try 9877145c241SMikulas Patocka * non-blocking allocations without a mutex first but on failure we fallback 9887145c241SMikulas Patocka * to blocking allocations with a mutex. 9891da177e4SLinus Torvalds */ 990cf2f1abfSMikulas Patocka static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) 9911da177e4SLinus Torvalds { 99249a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 9938b004457SMilan Broz struct bio *clone; 9941da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 9957145c241SMikulas Patocka gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; 9967145c241SMikulas Patocka unsigned i, len, remaining_size; 99791e10625SMilan Broz struct page *page; 998cf2f1abfSMikulas Patocka struct bio_vec *bvec; 9991da177e4SLinus Torvalds 10007145c241SMikulas Patocka retry: 1001d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 10027145c241SMikulas Patocka mutex_lock(&cc->bio_alloc_lock); 10037145c241SMikulas Patocka 10046a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 10058b004457SMilan Broz if (!clone) 10067145c241SMikulas Patocka goto return_clone; 10071da177e4SLinus Torvalds 1008027581f3SOlaf Kirch clone_init(io, clone); 10096a24c718SMilan Broz 10107145c241SMikulas Patocka remaining_size = size; 10117145c241SMikulas Patocka 1012f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 101391e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 10147145c241SMikulas Patocka if (!page) { 10157145c241SMikulas Patocka crypt_free_buffer_pages(cc, clone); 10167145c241SMikulas Patocka bio_put(clone); 1017d0164adcSMel Gorman gfp_mask |= __GFP_DIRECT_RECLAIM; 10187145c241SMikulas Patocka goto retry; 10197145c241SMikulas Patocka } 10201da177e4SLinus Torvalds 10217145c241SMikulas Patocka len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size; 10221da177e4SLinus Torvalds 1023cf2f1abfSMikulas Patocka bvec = &clone->bi_io_vec[clone->bi_vcnt++]; 1024cf2f1abfSMikulas Patocka bvec->bv_page = page; 1025cf2f1abfSMikulas Patocka bvec->bv_len = len; 1026cf2f1abfSMikulas Patocka bvec->bv_offset = 0; 1027cf2f1abfSMikulas Patocka 1028cf2f1abfSMikulas Patocka clone->bi_iter.bi_size += len; 102991e10625SMilan Broz 10307145c241SMikulas Patocka remaining_size -= len; 10311da177e4SLinus Torvalds } 10321da177e4SLinus Torvalds 10337145c241SMikulas Patocka return_clone: 1034d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 10357145c241SMikulas Patocka mutex_unlock(&cc->bio_alloc_lock); 10367145c241SMikulas Patocka 10378b004457SMilan Broz return clone; 10381da177e4SLinus Torvalds } 10391da177e4SLinus Torvalds 1040644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 10411da177e4SLinus Torvalds { 1042644bd2f0SNeil Brown unsigned int i; 10431da177e4SLinus Torvalds struct bio_vec *bv; 10441da177e4SLinus Torvalds 1045cb34e057SKent Overstreet bio_for_each_segment_all(bv, clone, i) { 10461da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 10471da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 10481da177e4SLinus Torvalds bv->bv_page = NULL; 10491da177e4SLinus Torvalds } 10501da177e4SLinus Torvalds } 10511da177e4SLinus Torvalds 1052298a9fa0SMikulas Patocka static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc, 1053dc440d1eSMilan Broz struct bio *bio, sector_t sector) 1054dc440d1eSMilan Broz { 105549a8a920SAlasdair G Kergon io->cc = cc; 1056dc440d1eSMilan Broz io->base_bio = bio; 1057dc440d1eSMilan Broz io->sector = sector; 1058dc440d1eSMilan Broz io->error = 0; 1059610f2de3SMikulas Patocka io->ctx.req = NULL; 106040b6229bSMikulas Patocka atomic_set(&io->io_pending, 0); 1061dc440d1eSMilan Broz } 1062dc440d1eSMilan Broz 10633e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 10643e1a8bddSMilan Broz { 106540b6229bSMikulas Patocka atomic_inc(&io->io_pending); 10663e1a8bddSMilan Broz } 10673e1a8bddSMilan Broz 10681da177e4SLinus Torvalds /* 10691da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 10701da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 10711da177e4SLinus Torvalds */ 10725742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 10731da177e4SLinus Torvalds { 107449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1075b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 1076b35f8caaSMilan Broz int error = io->error; 10771da177e4SLinus Torvalds 107840b6229bSMikulas Patocka if (!atomic_dec_and_test(&io->io_pending)) 10791da177e4SLinus Torvalds return; 10801da177e4SLinus Torvalds 1081610f2de3SMikulas Patocka if (io->ctx.req) 1082298a9fa0SMikulas Patocka crypt_free_req(cc, io->ctx.req, base_bio); 1083b35f8caaSMilan Broz 10844246a0b6SChristoph Hellwig base_bio->bi_error = error; 10854246a0b6SChristoph Hellwig bio_endio(base_bio); 10861da177e4SLinus Torvalds } 10871da177e4SLinus Torvalds 10881da177e4SLinus Torvalds /* 1089cabf08e4SMilan Broz * kcryptd/kcryptd_io: 10901da177e4SLinus Torvalds * 10911da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 109223541d2dSMilan Broz * interrupt context. 1093cabf08e4SMilan Broz * 1094cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 1095cabf08e4SMilan Broz * 1096cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 1097cabf08e4SMilan Broz * 1098cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 1099cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 1100cabf08e4SMilan Broz * to memory allocation. 1101c0297721SAndi Kleen * 1102c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances. 1103c0297721SAndi Kleen * They should not depend on each other and do not block. 11041da177e4SLinus Torvalds */ 11054246a0b6SChristoph Hellwig static void crypt_endio(struct bio *clone) 11068b004457SMilan Broz { 1107028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 110849a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1109ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 11109b81c842SSasha Levin int error; 11118b004457SMilan Broz 11128b004457SMilan Broz /* 11136712ecf8SNeilBrown * free the processed pages 11148b004457SMilan Broz */ 1115ee7a491eSMilan Broz if (rw == WRITE) 1116644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 11178b004457SMilan Broz 11189b81c842SSasha Levin error = clone->bi_error; 11198b004457SMilan Broz bio_put(clone); 1120ee7a491eSMilan Broz 11219b81c842SSasha Levin if (rw == READ && !error) { 1122cabf08e4SMilan Broz kcryptd_queue_crypt(io); 11236712ecf8SNeilBrown return; 1124ee7a491eSMilan Broz } 11255742fd77SMilan Broz 11269b81c842SSasha Levin if (unlikely(error)) 11279b81c842SSasha Levin io->error = error; 11285742fd77SMilan Broz 11295742fd77SMilan Broz crypt_dec_pending(io); 11308b004457SMilan Broz } 11318b004457SMilan Broz 1132028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 11338b004457SMilan Broz { 113449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 11358b004457SMilan Broz 11368b004457SMilan Broz clone->bi_private = io; 11378b004457SMilan Broz clone->bi_end_io = crypt_endio; 11388b004457SMilan Broz clone->bi_bdev = cc->dev->bdev; 1139e6047149SMike Christie bio_set_op_attrs(clone, bio_op(io->base_bio), io->base_bio->bi_rw); 11408b004457SMilan Broz } 11418b004457SMilan Broz 114220c82538SMilan Broz static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) 11438b004457SMilan Broz { 114449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 11458b004457SMilan Broz struct bio *clone; 114693e605c2SMilan Broz 11478b004457SMilan Broz /* 114859779079SMike Snitzer * We need the original biovec array in order to decrypt 114959779079SMike Snitzer * the whole bio data *afterwards* -- thanks to immutable 115059779079SMike Snitzer * biovecs we don't need to worry about the block layer 115159779079SMike Snitzer * modifying the biovec array; so leverage bio_clone_fast(). 11528b004457SMilan Broz */ 115359779079SMike Snitzer clone = bio_clone_fast(io->base_bio, gfp, cc->bs); 11547eaceaccSJens Axboe if (!clone) 115520c82538SMilan Broz return 1; 11568b004457SMilan Broz 115720c82538SMilan Broz crypt_inc_pending(io); 115820c82538SMilan Broz 11598b004457SMilan Broz clone_init(io, clone); 11604f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 11618b004457SMilan Broz 116293e605c2SMilan Broz generic_make_request(clone); 116320c82538SMilan Broz return 0; 11648b004457SMilan Broz } 11658b004457SMilan Broz 1166dc267621SMikulas Patocka static void kcryptd_io_read_work(struct work_struct *work) 1167395b167cSAlasdair G Kergon { 1168395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 1169395b167cSAlasdair G Kergon 117020c82538SMilan Broz crypt_inc_pending(io); 117120c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOIO)) 117220c82538SMilan Broz io->error = -ENOMEM; 117320c82538SMilan Broz crypt_dec_pending(io); 1174395b167cSAlasdair G Kergon } 1175395b167cSAlasdair G Kergon 1176dc267621SMikulas Patocka static void kcryptd_queue_read(struct dm_crypt_io *io) 1177395b167cSAlasdair G Kergon { 117849a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1179395b167cSAlasdair G Kergon 1180dc267621SMikulas Patocka INIT_WORK(&io->work, kcryptd_io_read_work); 1181395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 1182395b167cSAlasdair G Kergon } 1183395b167cSAlasdair G Kergon 1184dc267621SMikulas Patocka static void kcryptd_io_write(struct dm_crypt_io *io) 1185dc267621SMikulas Patocka { 1186dc267621SMikulas Patocka struct bio *clone = io->ctx.bio_out; 1187dc267621SMikulas Patocka 1188dc267621SMikulas Patocka generic_make_request(clone); 1189dc267621SMikulas Patocka } 1190dc267621SMikulas Patocka 1191b3c5fd30SMikulas Patocka #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node) 1192b3c5fd30SMikulas Patocka 1193dc267621SMikulas Patocka static int dmcrypt_write(void *data) 1194dc267621SMikulas Patocka { 1195dc267621SMikulas Patocka struct crypt_config *cc = data; 1196b3c5fd30SMikulas Patocka struct dm_crypt_io *io; 1197b3c5fd30SMikulas Patocka 1198dc267621SMikulas Patocka while (1) { 1199b3c5fd30SMikulas Patocka struct rb_root write_tree; 1200dc267621SMikulas Patocka struct blk_plug plug; 1201dc267621SMikulas Patocka 1202dc267621SMikulas Patocka DECLARE_WAITQUEUE(wait, current); 1203dc267621SMikulas Patocka 1204dc267621SMikulas Patocka spin_lock_irq(&cc->write_thread_wait.lock); 1205dc267621SMikulas Patocka continue_locked: 1206dc267621SMikulas Patocka 1207b3c5fd30SMikulas Patocka if (!RB_EMPTY_ROOT(&cc->write_tree)) 1208dc267621SMikulas Patocka goto pop_from_list; 1209dc267621SMikulas Patocka 1210bcbd94ffSMikulas Patocka if (unlikely(test_bit(DM_CRYPT_EXIT_THREAD, &cc->flags))) { 1211bcbd94ffSMikulas Patocka spin_unlock_irq(&cc->write_thread_wait.lock); 1212bcbd94ffSMikulas Patocka break; 1213bcbd94ffSMikulas Patocka } 1214bcbd94ffSMikulas Patocka 1215dc267621SMikulas Patocka __set_current_state(TASK_INTERRUPTIBLE); 1216dc267621SMikulas Patocka __add_wait_queue(&cc->write_thread_wait, &wait); 1217dc267621SMikulas Patocka 1218dc267621SMikulas Patocka spin_unlock_irq(&cc->write_thread_wait.lock); 1219dc267621SMikulas Patocka 1220dc267621SMikulas Patocka schedule(); 1221dc267621SMikulas Patocka 1222dc267621SMikulas Patocka spin_lock_irq(&cc->write_thread_wait.lock); 1223dc267621SMikulas Patocka __remove_wait_queue(&cc->write_thread_wait, &wait); 1224dc267621SMikulas Patocka goto continue_locked; 1225dc267621SMikulas Patocka 1226dc267621SMikulas Patocka pop_from_list: 1227b3c5fd30SMikulas Patocka write_tree = cc->write_tree; 1228b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 1229dc267621SMikulas Patocka spin_unlock_irq(&cc->write_thread_wait.lock); 1230dc267621SMikulas Patocka 1231b3c5fd30SMikulas Patocka BUG_ON(rb_parent(write_tree.rb_node)); 1232b3c5fd30SMikulas Patocka 1233b3c5fd30SMikulas Patocka /* 1234b3c5fd30SMikulas Patocka * Note: we cannot walk the tree here with rb_next because 1235b3c5fd30SMikulas Patocka * the structures may be freed when kcryptd_io_write is called. 1236b3c5fd30SMikulas Patocka */ 1237dc267621SMikulas Patocka blk_start_plug(&plug); 1238dc267621SMikulas Patocka do { 1239b3c5fd30SMikulas Patocka io = crypt_io_from_node(rb_first(&write_tree)); 1240b3c5fd30SMikulas Patocka rb_erase(&io->rb_node, &write_tree); 1241dc267621SMikulas Patocka kcryptd_io_write(io); 1242b3c5fd30SMikulas Patocka } while (!RB_EMPTY_ROOT(&write_tree)); 1243dc267621SMikulas Patocka blk_finish_plug(&plug); 1244dc267621SMikulas Patocka } 1245dc267621SMikulas Patocka return 0; 1246dc267621SMikulas Patocka } 1247dc267621SMikulas Patocka 124872c6e7afSMikulas Patocka static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) 12494e4eef64SMilan Broz { 1250dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 125149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1252dc267621SMikulas Patocka unsigned long flags; 1253b3c5fd30SMikulas Patocka sector_t sector; 1254b3c5fd30SMikulas Patocka struct rb_node **rbp, *parent; 1255dec1cedfSMilan Broz 125672c6e7afSMikulas Patocka if (unlikely(io->error < 0)) { 1257dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 1258dec1cedfSMilan Broz bio_put(clone); 12596c031f41SMilan Broz crypt_dec_pending(io); 1260dec1cedfSMilan Broz return; 1261dec1cedfSMilan Broz } 1262dec1cedfSMilan Broz 1263dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 1264003b5c57SKent Overstreet BUG_ON(io->ctx.iter_out.bi_size); 1265dec1cedfSMilan Broz 12664f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 1267899c95d3SMilan Broz 12680f5d8e6eSMikulas Patocka if (likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) { 12690f5d8e6eSMikulas Patocka generic_make_request(clone); 12700f5d8e6eSMikulas Patocka return; 12710f5d8e6eSMikulas Patocka } 12720f5d8e6eSMikulas Patocka 1273dc267621SMikulas Patocka spin_lock_irqsave(&cc->write_thread_wait.lock, flags); 1274b3c5fd30SMikulas Patocka rbp = &cc->write_tree.rb_node; 1275b3c5fd30SMikulas Patocka parent = NULL; 1276b3c5fd30SMikulas Patocka sector = io->sector; 1277b3c5fd30SMikulas Patocka while (*rbp) { 1278b3c5fd30SMikulas Patocka parent = *rbp; 1279b3c5fd30SMikulas Patocka if (sector < crypt_io_from_node(parent)->sector) 1280b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_left; 1281b3c5fd30SMikulas Patocka else 1282b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_right; 1283b3c5fd30SMikulas Patocka } 1284b3c5fd30SMikulas Patocka rb_link_node(&io->rb_node, parent, rbp); 1285b3c5fd30SMikulas Patocka rb_insert_color(&io->rb_node, &cc->write_tree); 1286b3c5fd30SMikulas Patocka 1287dc267621SMikulas Patocka wake_up_locked(&cc->write_thread_wait); 1288dc267621SMikulas Patocka spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags); 12894e4eef64SMilan Broz } 12904e4eef64SMilan Broz 1291fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 12928b004457SMilan Broz { 129349a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 12948b004457SMilan Broz struct bio *clone; 1295c8081618SMilan Broz int crypt_finished; 1296b635b00eSMilan Broz sector_t sector = io->sector; 1297dec1cedfSMilan Broz int r; 12988b004457SMilan Broz 129993e605c2SMilan Broz /* 1300fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 1301fc5a5e9aSMilan Broz */ 1302fc5a5e9aSMilan Broz crypt_inc_pending(io); 1303b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 1304fc5a5e9aSMilan Broz 1305cf2f1abfSMikulas Patocka clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size); 130623541d2dSMilan Broz if (unlikely(!clone)) { 1307cf2f1abfSMikulas Patocka io->error = -EIO; 1308cf2f1abfSMikulas Patocka goto dec; 130923541d2dSMilan Broz } 13108b004457SMilan Broz 131153017030SMilan Broz io->ctx.bio_out = clone; 1312003b5c57SKent Overstreet io->ctx.iter_out = clone->bi_iter; 13138b004457SMilan Broz 1314b635b00eSMilan Broz sector += bio_sectors(clone); 1315dec1cedfSMilan Broz 13164e594098SMilan Broz crypt_inc_pending(io); 1317dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 1318cf2f1abfSMikulas Patocka if (r) 131972c6e7afSMikulas Patocka io->error = -EIO; 132040b6229bSMikulas Patocka crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); 1321dec1cedfSMilan Broz 1322c8081618SMilan Broz /* Encryption was already finished, submit io now */ 1323c8081618SMilan Broz if (crypt_finished) { 132472c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 0); 1325b635b00eSMilan Broz io->sector = sector; 13264e594098SMilan Broz } 132793e605c2SMilan Broz 1328cf2f1abfSMikulas Patocka dec: 1329899c95d3SMilan Broz crypt_dec_pending(io); 133084131db6SMilan Broz } 133184131db6SMilan Broz 133272c6e7afSMikulas Patocka static void kcryptd_crypt_read_done(struct dm_crypt_io *io) 13335742fd77SMilan Broz { 13345742fd77SMilan Broz crypt_dec_pending(io); 13355742fd77SMilan Broz } 13365742fd77SMilan Broz 13374e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 13388b004457SMilan Broz { 133949a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 13405742fd77SMilan Broz int r = 0; 13418b004457SMilan Broz 13423e1a8bddSMilan Broz crypt_inc_pending(io); 13433a7f6c99SMilan Broz 134453017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 13450c395b0fSMilan Broz io->sector); 13468b004457SMilan Broz 13475742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 134872c6e7afSMikulas Patocka if (r < 0) 134972c6e7afSMikulas Patocka io->error = -EIO; 13505742fd77SMilan Broz 135140b6229bSMikulas Patocka if (atomic_dec_and_test(&io->ctx.cc_pending)) 135272c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 13533a7f6c99SMilan Broz 13543a7f6c99SMilan Broz crypt_dec_pending(io); 13558b004457SMilan Broz } 13568b004457SMilan Broz 135795497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 135895497a96SMilan Broz int error) 135995497a96SMilan Broz { 1360b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 1361b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 136295497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 136349a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 136495497a96SMilan Broz 136554cea3f6SMilan Broz /* 136654cea3f6SMilan Broz * A request from crypto driver backlog is going to be processed now, 136754cea3f6SMilan Broz * finish the completion and continue in crypt_convert(). 136854cea3f6SMilan Broz * (Callback will be called for the second time for this request.) 136954cea3f6SMilan Broz */ 1370c0403ec0SRabin Vincent if (error == -EINPROGRESS) { 1371c0403ec0SRabin Vincent complete(&ctx->restart); 137295497a96SMilan Broz return; 1373c0403ec0SRabin Vincent } 137495497a96SMilan Broz 13752dc5327dSMilan Broz if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post) 13762dc5327dSMilan Broz error = cc->iv_gen_ops->post(cc, iv_of_dmreq(cc, dmreq), dmreq); 13772dc5327dSMilan Broz 137872c6e7afSMikulas Patocka if (error < 0) 137972c6e7afSMikulas Patocka io->error = -EIO; 138072c6e7afSMikulas Patocka 1381298a9fa0SMikulas Patocka crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio); 138295497a96SMilan Broz 138340b6229bSMikulas Patocka if (!atomic_dec_and_test(&ctx->cc_pending)) 1384c0403ec0SRabin Vincent return; 138595497a96SMilan Broz 138695497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 138772c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 138895497a96SMilan Broz else 138972c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 1); 139095497a96SMilan Broz } 139195497a96SMilan Broz 13924e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 13934e4eef64SMilan Broz { 13944e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 13954e4eef64SMilan Broz 13964e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 13974e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 13984e4eef64SMilan Broz else 13994e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 14008b004457SMilan Broz } 14018b004457SMilan Broz 1402395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 1403395b167cSAlasdair G Kergon { 140449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1405395b167cSAlasdair G Kergon 1406395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 1407395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 1408395b167cSAlasdair G Kergon } 1409395b167cSAlasdair G Kergon 14101da177e4SLinus Torvalds /* 14111da177e4SLinus Torvalds * Decode key from its hex representation 14121da177e4SLinus Torvalds */ 14131da177e4SLinus Torvalds static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 14141da177e4SLinus Torvalds { 14151da177e4SLinus Torvalds char buffer[3]; 14161da177e4SLinus Torvalds unsigned int i; 14171da177e4SLinus Torvalds 14181da177e4SLinus Torvalds buffer[2] = '\0'; 14191da177e4SLinus Torvalds 14201da177e4SLinus Torvalds for (i = 0; i < size; i++) { 14211da177e4SLinus Torvalds buffer[0] = *hex++; 14221da177e4SLinus Torvalds buffer[1] = *hex++; 14231da177e4SLinus Torvalds 14241a66a08aSmajianpeng if (kstrtou8(buffer, 16, &key[i])) 14251da177e4SLinus Torvalds return -EINVAL; 14261da177e4SLinus Torvalds } 14271da177e4SLinus Torvalds 14281da177e4SLinus Torvalds if (*hex != '\0') 14291da177e4SLinus Torvalds return -EINVAL; 14301da177e4SLinus Torvalds 14311da177e4SLinus Torvalds return 0; 14321da177e4SLinus Torvalds } 14331da177e4SLinus Torvalds 1434fd2d231fSMikulas Patocka static void crypt_free_tfms(struct crypt_config *cc) 1435d1f96423SMilan Broz { 1436d1f96423SMilan Broz unsigned i; 1437d1f96423SMilan Broz 1438fd2d231fSMikulas Patocka if (!cc->tfms) 1439fd2d231fSMikulas Patocka return; 1440fd2d231fSMikulas Patocka 1441d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) 1442fd2d231fSMikulas Patocka if (cc->tfms[i] && !IS_ERR(cc->tfms[i])) { 1443bbdb23b5SHerbert Xu crypto_free_skcipher(cc->tfms[i]); 1444fd2d231fSMikulas Patocka cc->tfms[i] = NULL; 1445d1f96423SMilan Broz } 1446d1f96423SMilan Broz 1447fd2d231fSMikulas Patocka kfree(cc->tfms); 1448fd2d231fSMikulas Patocka cc->tfms = NULL; 1449fd2d231fSMikulas Patocka } 1450fd2d231fSMikulas Patocka 1451fd2d231fSMikulas Patocka static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode) 1452d1f96423SMilan Broz { 1453d1f96423SMilan Broz unsigned i; 1454d1f96423SMilan Broz int err; 1455d1f96423SMilan Broz 1456bbdb23b5SHerbert Xu cc->tfms = kmalloc(cc->tfms_count * sizeof(struct crypto_skcipher *), 1457fd2d231fSMikulas Patocka GFP_KERNEL); 1458fd2d231fSMikulas Patocka if (!cc->tfms) 1459fd2d231fSMikulas Patocka return -ENOMEM; 1460fd2d231fSMikulas Patocka 1461d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 1462bbdb23b5SHerbert Xu cc->tfms[i] = crypto_alloc_skcipher(ciphermode, 0, 0); 1463fd2d231fSMikulas Patocka if (IS_ERR(cc->tfms[i])) { 1464fd2d231fSMikulas Patocka err = PTR_ERR(cc->tfms[i]); 1465fd2d231fSMikulas Patocka crypt_free_tfms(cc); 1466d1f96423SMilan Broz return err; 1467d1f96423SMilan Broz } 1468d1f96423SMilan Broz } 1469d1f96423SMilan Broz 1470d1f96423SMilan Broz return 0; 1471d1f96423SMilan Broz } 1472d1f96423SMilan Broz 1473c0297721SAndi Kleen static int crypt_setkey_allcpus(struct crypt_config *cc) 1474c0297721SAndi Kleen { 1475da31a078SMilan Broz unsigned subkey_size; 1476fd2d231fSMikulas Patocka int err = 0, i, r; 1477c0297721SAndi Kleen 1478da31a078SMilan Broz /* Ignore extra keys (which are used for IV etc) */ 1479da31a078SMilan Broz subkey_size = (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count); 1480da31a078SMilan Broz 1481d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 1482bbdb23b5SHerbert Xu r = crypto_skcipher_setkey(cc->tfms[i], 1483fd2d231fSMikulas Patocka cc->key + (i * subkey_size), 1484fd2d231fSMikulas Patocka subkey_size); 1485c0297721SAndi Kleen if (r) 1486c0297721SAndi Kleen err = r; 1487c0297721SAndi Kleen } 1488c0297721SAndi Kleen 1489c0297721SAndi Kleen return err; 1490c0297721SAndi Kleen } 1491c0297721SAndi Kleen 1492e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 1493e48d4bbfSMilan Broz { 1494de8be5acSMilan Broz int r = -EINVAL; 1495de8be5acSMilan Broz int key_string_len = strlen(key); 1496de8be5acSMilan Broz 149769a8cfcdSMilan Broz /* The key size may not be changed. */ 1498de8be5acSMilan Broz if (cc->key_size != (key_string_len >> 1)) 1499de8be5acSMilan Broz goto out; 1500e48d4bbfSMilan Broz 150169a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */ 150269a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-")) 1503de8be5acSMilan Broz goto out; 1504e48d4bbfSMilan Broz 150569a8cfcdSMilan Broz if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0) 1506de8be5acSMilan Broz goto out; 1507e48d4bbfSMilan Broz 1508e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1509e48d4bbfSMilan Broz 1510de8be5acSMilan Broz r = crypt_setkey_allcpus(cc); 1511de8be5acSMilan Broz 1512de8be5acSMilan Broz out: 1513de8be5acSMilan Broz /* Hex key string not needed after here, so wipe it. */ 1514de8be5acSMilan Broz memset(key, '0', key_string_len); 1515de8be5acSMilan Broz 1516de8be5acSMilan Broz return r; 1517e48d4bbfSMilan Broz } 1518e48d4bbfSMilan Broz 1519e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 1520e48d4bbfSMilan Broz { 1521e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 1522e48d4bbfSMilan Broz memset(&cc->key, 0, cc->key_size * sizeof(u8)); 1523c0297721SAndi Kleen 1524c0297721SAndi Kleen return crypt_setkey_allcpus(cc); 1525e48d4bbfSMilan Broz } 1526e48d4bbfSMilan Broz 152728513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 152828513fccSMilan Broz { 152928513fccSMilan Broz struct crypt_config *cc = ti->private; 153028513fccSMilan Broz 153128513fccSMilan Broz ti->private = NULL; 153228513fccSMilan Broz 153328513fccSMilan Broz if (!cc) 153428513fccSMilan Broz return; 153528513fccSMilan Broz 1536bcbd94ffSMikulas Patocka if (cc->write_thread) { 1537bcbd94ffSMikulas Patocka spin_lock_irq(&cc->write_thread_wait.lock); 1538bcbd94ffSMikulas Patocka set_bit(DM_CRYPT_EXIT_THREAD, &cc->flags); 1539bcbd94ffSMikulas Patocka wake_up_locked(&cc->write_thread_wait); 1540bcbd94ffSMikulas Patocka spin_unlock_irq(&cc->write_thread_wait.lock); 1541dc267621SMikulas Patocka kthread_stop(cc->write_thread); 1542bcbd94ffSMikulas Patocka } 1543dc267621SMikulas Patocka 154428513fccSMilan Broz if (cc->io_queue) 154528513fccSMilan Broz destroy_workqueue(cc->io_queue); 154628513fccSMilan Broz if (cc->crypt_queue) 154728513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 154828513fccSMilan Broz 1549fd2d231fSMikulas Patocka crypt_free_tfms(cc); 1550fd2d231fSMikulas Patocka 155128513fccSMilan Broz if (cc->bs) 155228513fccSMilan Broz bioset_free(cc->bs); 155328513fccSMilan Broz 155428513fccSMilan Broz mempool_destroy(cc->page_pool); 155528513fccSMilan Broz mempool_destroy(cc->req_pool); 155628513fccSMilan Broz 155728513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 155828513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 155928513fccSMilan Broz 156028513fccSMilan Broz if (cc->dev) 156128513fccSMilan Broz dm_put_device(ti, cc->dev); 156228513fccSMilan Broz 15635ebaee6dSMilan Broz kzfree(cc->cipher); 15647dbcd137SMilan Broz kzfree(cc->cipher_string); 156528513fccSMilan Broz 156628513fccSMilan Broz /* Must zero key material before freeing */ 156728513fccSMilan Broz kzfree(cc); 156828513fccSMilan Broz } 156928513fccSMilan Broz 15705ebaee6dSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, 15715ebaee6dSMilan Broz char *cipher_in, char *key) 15721da177e4SLinus Torvalds { 15735ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 1574d1f96423SMilan Broz char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount; 15755ebaee6dSMilan Broz char *cipher_api = NULL; 1576fd2d231fSMikulas Patocka int ret = -EINVAL; 157731998ef1SMikulas Patocka char dummy; 15781da177e4SLinus Torvalds 15795ebaee6dSMilan Broz /* Convert to crypto api definition? */ 15805ebaee6dSMilan Broz if (strchr(cipher_in, '(')) { 15815ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 15821da177e4SLinus Torvalds return -EINVAL; 15831da177e4SLinus Torvalds } 15841da177e4SLinus Torvalds 15857dbcd137SMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL); 15867dbcd137SMilan Broz if (!cc->cipher_string) 15877dbcd137SMilan Broz goto bad_mem; 15887dbcd137SMilan Broz 15895ebaee6dSMilan Broz /* 15905ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 1591d1f96423SMilan Broz * cipher[:keycount]-mode-iv:ivopts 15925ebaee6dSMilan Broz */ 15935ebaee6dSMilan Broz tmp = cipher_in; 1594d1f96423SMilan Broz keycount = strsep(&tmp, "-"); 1595d1f96423SMilan Broz cipher = strsep(&keycount, ":"); 1596d1f96423SMilan Broz 1597d1f96423SMilan Broz if (!keycount) 1598d1f96423SMilan Broz cc->tfms_count = 1; 159931998ef1SMikulas Patocka else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 || 1600d1f96423SMilan Broz !is_power_of_2(cc->tfms_count)) { 1601d1f96423SMilan Broz ti->error = "Bad cipher key count specification"; 1602d1f96423SMilan Broz return -EINVAL; 1603d1f96423SMilan Broz } 1604d1f96423SMilan Broz cc->key_parts = cc->tfms_count; 1605da31a078SMilan Broz cc->key_extra_size = 0; 16065ebaee6dSMilan Broz 16075ebaee6dSMilan Broz cc->cipher = kstrdup(cipher, GFP_KERNEL); 16085ebaee6dSMilan Broz if (!cc->cipher) 16095ebaee6dSMilan Broz goto bad_mem; 16105ebaee6dSMilan Broz 16111da177e4SLinus Torvalds chainmode = strsep(&tmp, "-"); 16121da177e4SLinus Torvalds ivopts = strsep(&tmp, "-"); 16131da177e4SLinus Torvalds ivmode = strsep(&ivopts, ":"); 16141da177e4SLinus Torvalds 16151da177e4SLinus Torvalds if (tmp) 16165ebaee6dSMilan Broz DMWARN("Ignoring unexpected additional cipher options"); 16171da177e4SLinus Torvalds 16187dbcd137SMilan Broz /* 16197dbcd137SMilan Broz * For compatibility with the original dm-crypt mapping format, if 16207dbcd137SMilan Broz * only the cipher name is supplied, use cbc-plain. 16217dbcd137SMilan Broz */ 16225ebaee6dSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) { 16231da177e4SLinus Torvalds chainmode = "cbc"; 16241da177e4SLinus Torvalds ivmode = "plain"; 16251da177e4SLinus Torvalds } 16261da177e4SLinus Torvalds 1627d1806f6aSHerbert Xu if (strcmp(chainmode, "ecb") && !ivmode) { 16285ebaee6dSMilan Broz ti->error = "IV mechanism required"; 16295ebaee6dSMilan Broz return -EINVAL; 16301da177e4SLinus Torvalds } 16311da177e4SLinus Torvalds 16325ebaee6dSMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 16335ebaee6dSMilan Broz if (!cipher_api) 16345ebaee6dSMilan Broz goto bad_mem; 16355ebaee6dSMilan Broz 16365ebaee6dSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 16375ebaee6dSMilan Broz "%s(%s)", chainmode, cipher); 16385ebaee6dSMilan Broz if (ret < 0) { 16395ebaee6dSMilan Broz kfree(cipher_api); 16405ebaee6dSMilan Broz goto bad_mem; 1641d1806f6aSHerbert Xu } 1642d1806f6aSHerbert Xu 16435ebaee6dSMilan Broz /* Allocate cipher */ 1644fd2d231fSMikulas Patocka ret = crypt_alloc_tfms(cc, cipher_api); 1645d1f96423SMilan Broz if (ret < 0) { 164672d94861SAlasdair G Kergon ti->error = "Error allocating crypto tfm"; 164728513fccSMilan Broz goto bad; 16481da177e4SLinus Torvalds } 16491da177e4SLinus Torvalds 16505ebaee6dSMilan Broz /* Initialize IV */ 1651bbdb23b5SHerbert Xu cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc)); 16525ebaee6dSMilan Broz if (cc->iv_size) 16535ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 16545ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 16555ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 16565ebaee6dSMilan Broz else if (ivmode) { 16575ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 16585ebaee6dSMilan Broz ivmode = NULL; 16595ebaee6dSMilan Broz } 16605ebaee6dSMilan Broz 16615ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 16621da177e4SLinus Torvalds if (ivmode == NULL) 16631da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 16641da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 16651da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 166661afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 166761afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 16681da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 16691da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 167048527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 167148527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 167246b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 167346b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 167434745785SMilan Broz else if (strcmp(ivmode, "lmk") == 0) { 167534745785SMilan Broz cc->iv_gen_ops = &crypt_iv_lmk_ops; 1676ed04d981SMilan Broz /* 1677ed04d981SMilan Broz * Version 2 and 3 is recognised according 167834745785SMilan Broz * to length of provided multi-key string. 167934745785SMilan Broz * If present (version 3), last key is used as IV seed. 1680ed04d981SMilan Broz * All keys (including IV seed) are always the same size. 168134745785SMilan Broz */ 1682da31a078SMilan Broz if (cc->key_size % cc->key_parts) { 168334745785SMilan Broz cc->key_parts++; 1684da31a078SMilan Broz cc->key_extra_size = cc->key_size / cc->key_parts; 1685da31a078SMilan Broz } 1686ed04d981SMilan Broz } else if (strcmp(ivmode, "tcw") == 0) { 1687ed04d981SMilan Broz cc->iv_gen_ops = &crypt_iv_tcw_ops; 1688ed04d981SMilan Broz cc->key_parts += 2; /* IV + whitening */ 1689ed04d981SMilan Broz cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE; 169034745785SMilan Broz } else { 16915ebaee6dSMilan Broz ret = -EINVAL; 169272d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 169328513fccSMilan Broz goto bad; 16941da177e4SLinus Torvalds } 16951da177e4SLinus Torvalds 1696da31a078SMilan Broz /* Initialize and set key */ 1697da31a078SMilan Broz ret = crypt_set_key(cc, key); 1698da31a078SMilan Broz if (ret < 0) { 1699da31a078SMilan Broz ti->error = "Error decoding and setting key"; 1700da31a078SMilan Broz goto bad; 1701da31a078SMilan Broz } 1702da31a078SMilan Broz 170328513fccSMilan Broz /* Allocate IV */ 170428513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 170528513fccSMilan Broz ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 170628513fccSMilan Broz if (ret < 0) { 170728513fccSMilan Broz ti->error = "Error creating IV"; 170828513fccSMilan Broz goto bad; 170928513fccSMilan Broz } 171028513fccSMilan Broz } 17111da177e4SLinus Torvalds 171228513fccSMilan Broz /* Initialize IV (set keys for ESSIV etc) */ 171328513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 171428513fccSMilan Broz ret = cc->iv_gen_ops->init(cc); 171528513fccSMilan Broz if (ret < 0) { 1716b95bf2d3SMilan Broz ti->error = "Error initialising IV"; 171728513fccSMilan Broz goto bad; 171828513fccSMilan Broz } 1719b95bf2d3SMilan Broz } 1720b95bf2d3SMilan Broz 17215ebaee6dSMilan Broz ret = 0; 17225ebaee6dSMilan Broz bad: 17235ebaee6dSMilan Broz kfree(cipher_api); 17245ebaee6dSMilan Broz return ret; 17255ebaee6dSMilan Broz 17265ebaee6dSMilan Broz bad_mem: 17275ebaee6dSMilan Broz ti->error = "Cannot allocate cipher strings"; 17285ebaee6dSMilan Broz return -ENOMEM; 17291da177e4SLinus Torvalds } 17305ebaee6dSMilan Broz 17315ebaee6dSMilan Broz /* 17325ebaee6dSMilan Broz * Construct an encryption mapping: 17335ebaee6dSMilan Broz * <cipher> <key> <iv_offset> <dev_path> <start> 17345ebaee6dSMilan Broz */ 17355ebaee6dSMilan Broz static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 17365ebaee6dSMilan Broz { 17375ebaee6dSMilan Broz struct crypt_config *cc; 1738772ae5f5SMilan Broz unsigned int key_size, opt_params; 17395ebaee6dSMilan Broz unsigned long long tmpll; 17405ebaee6dSMilan Broz int ret; 1741d49ec52fSMikulas Patocka size_t iv_size_padding; 1742772ae5f5SMilan Broz struct dm_arg_set as; 1743772ae5f5SMilan Broz const char *opt_string; 174431998ef1SMikulas Patocka char dummy; 17455ebaee6dSMilan Broz 1746772ae5f5SMilan Broz static struct dm_arg _args[] = { 17470f5d8e6eSMikulas Patocka {0, 3, "Invalid number of feature args"}, 1748772ae5f5SMilan Broz }; 1749772ae5f5SMilan Broz 1750772ae5f5SMilan Broz if (argc < 5) { 17515ebaee6dSMilan Broz ti->error = "Not enough arguments"; 17525ebaee6dSMilan Broz return -EINVAL; 17531da177e4SLinus Torvalds } 17541da177e4SLinus Torvalds 17555ebaee6dSMilan Broz key_size = strlen(argv[1]) >> 1; 17565ebaee6dSMilan Broz 17575ebaee6dSMilan Broz cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 17585ebaee6dSMilan Broz if (!cc) { 17595ebaee6dSMilan Broz ti->error = "Cannot allocate encryption context"; 17605ebaee6dSMilan Broz return -ENOMEM; 17615ebaee6dSMilan Broz } 176269a8cfcdSMilan Broz cc->key_size = key_size; 17635ebaee6dSMilan Broz 17645ebaee6dSMilan Broz ti->private = cc; 17655ebaee6dSMilan Broz ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 17665ebaee6dSMilan Broz if (ret < 0) 17675ebaee6dSMilan Broz goto bad; 17685ebaee6dSMilan Broz 1769bbdb23b5SHerbert Xu cc->dmreq_start = sizeof(struct skcipher_request); 1770bbdb23b5SHerbert Xu cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc)); 1771d49ec52fSMikulas Patocka cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request)); 1772d49ec52fSMikulas Patocka 1773bbdb23b5SHerbert Xu if (crypto_skcipher_alignmask(any_tfm(cc)) < CRYPTO_MINALIGN) { 1774d49ec52fSMikulas Patocka /* Allocate the padding exactly */ 1775d49ec52fSMikulas Patocka iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request)) 1776bbdb23b5SHerbert Xu & crypto_skcipher_alignmask(any_tfm(cc)); 1777d49ec52fSMikulas Patocka } else { 1778d49ec52fSMikulas Patocka /* 1779d49ec52fSMikulas Patocka * If the cipher requires greater alignment than kmalloc 1780d49ec52fSMikulas Patocka * alignment, we don't know the exact position of the 1781d49ec52fSMikulas Patocka * initialization vector. We must assume worst case. 1782d49ec52fSMikulas Patocka */ 1783bbdb23b5SHerbert Xu iv_size_padding = crypto_skcipher_alignmask(any_tfm(cc)); 1784d49ec52fSMikulas Patocka } 1785ddd42edfSMilan Broz 178694f5e024SMikulas Patocka ret = -ENOMEM; 1787ddd42edfSMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1788d49ec52fSMikulas Patocka sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size); 1789ddd42edfSMilan Broz if (!cc->req_pool) { 1790ddd42edfSMilan Broz ti->error = "Cannot allocate crypt request mempool"; 179128513fccSMilan Broz goto bad; 1792ddd42edfSMilan Broz } 1793ddd42edfSMilan Broz 179430187e1dSMike Snitzer cc->per_bio_data_size = ti->per_io_data_size = 1795d49ec52fSMikulas Patocka ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + 1796d49ec52fSMikulas Patocka sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size, 1797d49ec52fSMikulas Patocka ARCH_KMALLOC_MINALIGN); 1798298a9fa0SMikulas Patocka 1799cf2f1abfSMikulas Patocka cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0); 18001da177e4SLinus Torvalds if (!cc->page_pool) { 180172d94861SAlasdair G Kergon ti->error = "Cannot allocate page mempool"; 180228513fccSMilan Broz goto bad; 18031da177e4SLinus Torvalds } 18041da177e4SLinus Torvalds 1805bb799ca0SJens Axboe cc->bs = bioset_create(MIN_IOS, 0); 18066a24c718SMilan Broz if (!cc->bs) { 18076a24c718SMilan Broz ti->error = "Cannot allocate crypt bioset"; 180828513fccSMilan Broz goto bad; 18096a24c718SMilan Broz } 18106a24c718SMilan Broz 18117145c241SMikulas Patocka mutex_init(&cc->bio_alloc_lock); 18127145c241SMikulas Patocka 181328513fccSMilan Broz ret = -EINVAL; 181431998ef1SMikulas Patocka if (sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) { 181572d94861SAlasdair G Kergon ti->error = "Invalid iv_offset sector"; 181628513fccSMilan Broz goto bad; 18171da177e4SLinus Torvalds } 18184ee218cdSAndrew Morton cc->iv_offset = tmpll; 18191da177e4SLinus Torvalds 1820e80d1c80SVivek Goyal ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev); 1821e80d1c80SVivek Goyal if (ret) { 182228513fccSMilan Broz ti->error = "Device lookup failed"; 182328513fccSMilan Broz goto bad; 182428513fccSMilan Broz } 182528513fccSMilan Broz 1826e80d1c80SVivek Goyal ret = -EINVAL; 182731998ef1SMikulas Patocka if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1) { 182872d94861SAlasdair G Kergon ti->error = "Invalid device sector"; 182928513fccSMilan Broz goto bad; 18301da177e4SLinus Torvalds } 18314ee218cdSAndrew Morton cc->start = tmpll; 18321da177e4SLinus Torvalds 1833772ae5f5SMilan Broz argv += 5; 1834772ae5f5SMilan Broz argc -= 5; 1835772ae5f5SMilan Broz 1836772ae5f5SMilan Broz /* Optional parameters */ 1837772ae5f5SMilan Broz if (argc) { 1838772ae5f5SMilan Broz as.argc = argc; 1839772ae5f5SMilan Broz as.argv = argv; 1840772ae5f5SMilan Broz 1841772ae5f5SMilan Broz ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 1842772ae5f5SMilan Broz if (ret) 1843772ae5f5SMilan Broz goto bad; 1844772ae5f5SMilan Broz 184544c144f9SWei Yongjun ret = -EINVAL; 1846f3396c58SMikulas Patocka while (opt_params--) { 1847772ae5f5SMilan Broz opt_string = dm_shift_arg(&as); 1848f3396c58SMikulas Patocka if (!opt_string) { 1849f3396c58SMikulas Patocka ti->error = "Not enough feature arguments"; 1850f3396c58SMikulas Patocka goto bad; 1851f3396c58SMikulas Patocka } 1852772ae5f5SMilan Broz 1853f3396c58SMikulas Patocka if (!strcasecmp(opt_string, "allow_discards")) 185455a62eefSAlasdair G Kergon ti->num_discard_bios = 1; 1855f3396c58SMikulas Patocka 1856f3396c58SMikulas Patocka else if (!strcasecmp(opt_string, "same_cpu_crypt")) 1857f3396c58SMikulas Patocka set_bit(DM_CRYPT_SAME_CPU, &cc->flags); 1858f3396c58SMikulas Patocka 18590f5d8e6eSMikulas Patocka else if (!strcasecmp(opt_string, "submit_from_crypt_cpus")) 18600f5d8e6eSMikulas Patocka set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 18610f5d8e6eSMikulas Patocka 1862f3396c58SMikulas Patocka else { 1863772ae5f5SMilan Broz ti->error = "Invalid feature arguments"; 1864772ae5f5SMilan Broz goto bad; 1865772ae5f5SMilan Broz } 1866772ae5f5SMilan Broz } 1867f3396c58SMikulas Patocka } 1868772ae5f5SMilan Broz 186928513fccSMilan Broz ret = -ENOMEM; 1870670368a8STejun Heo cc->io_queue = alloc_workqueue("kcryptd_io", WQ_MEM_RECLAIM, 1); 1871cabf08e4SMilan Broz if (!cc->io_queue) { 1872cabf08e4SMilan Broz ti->error = "Couldn't create kcryptd io queue"; 187328513fccSMilan Broz goto bad; 1874cabf08e4SMilan Broz } 1875cabf08e4SMilan Broz 1876f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 1877f3396c58SMikulas Patocka cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); 1878f3396c58SMikulas Patocka else 1879f3396c58SMikulas Patocka cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, 1880f3396c58SMikulas Patocka num_online_cpus()); 1881cabf08e4SMilan Broz if (!cc->crypt_queue) { 18829934a8beSMilan Broz ti->error = "Couldn't create kcryptd queue"; 188328513fccSMilan Broz goto bad; 18849934a8beSMilan Broz } 18859934a8beSMilan Broz 1886dc267621SMikulas Patocka init_waitqueue_head(&cc->write_thread_wait); 1887b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 1888dc267621SMikulas Patocka 1889dc267621SMikulas Patocka cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write"); 1890dc267621SMikulas Patocka if (IS_ERR(cc->write_thread)) { 1891dc267621SMikulas Patocka ret = PTR_ERR(cc->write_thread); 1892dc267621SMikulas Patocka cc->write_thread = NULL; 1893dc267621SMikulas Patocka ti->error = "Couldn't spawn write thread"; 1894dc267621SMikulas Patocka goto bad; 1895dc267621SMikulas Patocka } 1896dc267621SMikulas Patocka wake_up_process(cc->write_thread); 1897dc267621SMikulas Patocka 189855a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 18990ac55489SAlasdair G Kergon ti->discard_zeroes_data_unsupported = true; 1900983c7db3SMilan Broz 19011da177e4SLinus Torvalds return 0; 19021da177e4SLinus Torvalds 190328513fccSMilan Broz bad: 190428513fccSMilan Broz crypt_dtr(ti); 190528513fccSMilan Broz return ret; 19061da177e4SLinus Torvalds } 19071da177e4SLinus Torvalds 19087de3ee57SMikulas Patocka static int crypt_map(struct dm_target *ti, struct bio *bio) 19091da177e4SLinus Torvalds { 1910028867acSAlasdair G Kergon struct dm_crypt_io *io; 191149a8a920SAlasdair G Kergon struct crypt_config *cc = ti->private; 1912647c7db1SMikulas Patocka 1913772ae5f5SMilan Broz /* 191428a8f0d3SMike Christie * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues. 191528a8f0d3SMike Christie * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight 1916e6047149SMike Christie * - for REQ_OP_DISCARD caller must use flush if IO ordering matters 1917772ae5f5SMilan Broz */ 191828a8f0d3SMike Christie if (unlikely(bio->bi_rw & REQ_PREFLUSH || 191928a8f0d3SMike Christie bio_op(bio) == REQ_OP_DISCARD)) { 1920647c7db1SMikulas Patocka bio->bi_bdev = cc->dev->bdev; 1921772ae5f5SMilan Broz if (bio_sectors(bio)) 19224f024f37SKent Overstreet bio->bi_iter.bi_sector = cc->start + 19234f024f37SKent Overstreet dm_target_offset(ti, bio->bi_iter.bi_sector); 1924647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 1925647c7db1SMikulas Patocka } 19261da177e4SLinus Torvalds 1927298a9fa0SMikulas Patocka io = dm_per_bio_data(bio, cc->per_bio_data_size); 1928298a9fa0SMikulas Patocka crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector)); 1929bbdb23b5SHerbert Xu io->ctx.req = (struct skcipher_request *)(io + 1); 1930cabf08e4SMilan Broz 193120c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 193220c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOWAIT)) 1933dc267621SMikulas Patocka kcryptd_queue_read(io); 193420c82538SMilan Broz } else 1935cabf08e4SMilan Broz kcryptd_queue_crypt(io); 19361da177e4SLinus Torvalds 1937d2a7ad29SKiyoshi Ueda return DM_MAPIO_SUBMITTED; 19381da177e4SLinus Torvalds } 19391da177e4SLinus Torvalds 1940fd7c092eSMikulas Patocka static void crypt_status(struct dm_target *ti, status_type_t type, 19411f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 19421da177e4SLinus Torvalds { 19435ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 1944fd7c092eSMikulas Patocka unsigned i, sz = 0; 1945f3396c58SMikulas Patocka int num_feature_args = 0; 19461da177e4SLinus Torvalds 19471da177e4SLinus Torvalds switch (type) { 19481da177e4SLinus Torvalds case STATUSTYPE_INFO: 19491da177e4SLinus Torvalds result[0] = '\0'; 19501da177e4SLinus Torvalds break; 19511da177e4SLinus Torvalds 19521da177e4SLinus Torvalds case STATUSTYPE_TABLE: 19537dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string); 19541da177e4SLinus Torvalds 1955fd7c092eSMikulas Patocka if (cc->key_size > 0) 1956fd7c092eSMikulas Patocka for (i = 0; i < cc->key_size; i++) 1957fd7c092eSMikulas Patocka DMEMIT("%02x", cc->key[i]); 1958fd7c092eSMikulas Patocka else 1959fd7c092eSMikulas Patocka DMEMIT("-"); 19601da177e4SLinus Torvalds 19614ee218cdSAndrew Morton DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 19624ee218cdSAndrew Morton cc->dev->name, (unsigned long long)cc->start); 1963772ae5f5SMilan Broz 1964f3396c58SMikulas Patocka num_feature_args += !!ti->num_discard_bios; 1965f3396c58SMikulas Patocka num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags); 19660f5d8e6eSMikulas Patocka num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 1967f3396c58SMikulas Patocka if (num_feature_args) { 1968f3396c58SMikulas Patocka DMEMIT(" %d", num_feature_args); 196955a62eefSAlasdair G Kergon if (ti->num_discard_bios) 1970f3396c58SMikulas Patocka DMEMIT(" allow_discards"); 1971f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 1972f3396c58SMikulas Patocka DMEMIT(" same_cpu_crypt"); 19730f5d8e6eSMikulas Patocka if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) 19740f5d8e6eSMikulas Patocka DMEMIT(" submit_from_crypt_cpus"); 1975f3396c58SMikulas Patocka } 1976772ae5f5SMilan Broz 19771da177e4SLinus Torvalds break; 19781da177e4SLinus Torvalds } 19791da177e4SLinus Torvalds } 19801da177e4SLinus Torvalds 1981e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 1982e48d4bbfSMilan Broz { 1983e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1984e48d4bbfSMilan Broz 1985e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1986e48d4bbfSMilan Broz } 1987e48d4bbfSMilan Broz 1988e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 1989e48d4bbfSMilan Broz { 1990e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 1991e48d4bbfSMilan Broz 1992e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 1993e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 1994e48d4bbfSMilan Broz return -EAGAIN; 1995e48d4bbfSMilan Broz } 1996e48d4bbfSMilan Broz 1997e48d4bbfSMilan Broz return 0; 1998e48d4bbfSMilan Broz } 1999e48d4bbfSMilan Broz 2000e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 2001e48d4bbfSMilan Broz { 2002e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 2003e48d4bbfSMilan Broz 2004e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 2005e48d4bbfSMilan Broz } 2006e48d4bbfSMilan Broz 2007e48d4bbfSMilan Broz /* Message interface 2008e48d4bbfSMilan Broz * key set <key> 2009e48d4bbfSMilan Broz * key wipe 2010e48d4bbfSMilan Broz */ 2011e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 2012e48d4bbfSMilan Broz { 2013e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 2014542da317SMilan Broz int ret = -EINVAL; 2015e48d4bbfSMilan Broz 2016e48d4bbfSMilan Broz if (argc < 2) 2017e48d4bbfSMilan Broz goto error; 2018e48d4bbfSMilan Broz 2019498f0103SMike Snitzer if (!strcasecmp(argv[0], "key")) { 2020e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 2021e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 2022e48d4bbfSMilan Broz return -EINVAL; 2023e48d4bbfSMilan Broz } 2024498f0103SMike Snitzer if (argc == 3 && !strcasecmp(argv[1], "set")) { 2025542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 2026542da317SMilan Broz if (ret) 2027542da317SMilan Broz return ret; 2028542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 2029542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 2030542da317SMilan Broz return ret; 2031542da317SMilan Broz } 2032498f0103SMike Snitzer if (argc == 2 && !strcasecmp(argv[1], "wipe")) { 2033542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 2034542da317SMilan Broz ret = cc->iv_gen_ops->wipe(cc); 2035542da317SMilan Broz if (ret) 2036542da317SMilan Broz return ret; 2037542da317SMilan Broz } 2038e48d4bbfSMilan Broz return crypt_wipe_key(cc); 2039e48d4bbfSMilan Broz } 2040542da317SMilan Broz } 2041e48d4bbfSMilan Broz 2042e48d4bbfSMilan Broz error: 2043e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 2044e48d4bbfSMilan Broz return -EINVAL; 2045e48d4bbfSMilan Broz } 2046e48d4bbfSMilan Broz 2047af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 2048af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 2049af4874e0SMike Snitzer { 2050af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 2051af4874e0SMike Snitzer 20525dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 2053af4874e0SMike Snitzer } 2054af4874e0SMike Snitzer 2055586b286bSMike Snitzer static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits) 2056586b286bSMike Snitzer { 2057586b286bSMike Snitzer /* 2058586b286bSMike Snitzer * Unfortunate constraint that is required to avoid the potential 2059586b286bSMike Snitzer * for exceeding underlying device's max_segments limits -- due to 2060586b286bSMike Snitzer * crypt_alloc_buffer() possibly allocating pages for the encryption 2061586b286bSMike Snitzer * bio that are not as physically contiguous as the original bio. 2062586b286bSMike Snitzer */ 2063586b286bSMike Snitzer limits->max_segment_size = PAGE_SIZE; 2064586b286bSMike Snitzer } 2065586b286bSMike Snitzer 20661da177e4SLinus Torvalds static struct target_type crypt_target = { 20671da177e4SLinus Torvalds .name = "crypt", 2068586b286bSMike Snitzer .version = {1, 14, 1}, 20691da177e4SLinus Torvalds .module = THIS_MODULE, 20701da177e4SLinus Torvalds .ctr = crypt_ctr, 20711da177e4SLinus Torvalds .dtr = crypt_dtr, 20721da177e4SLinus Torvalds .map = crypt_map, 20731da177e4SLinus Torvalds .status = crypt_status, 2074e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 2075e48d4bbfSMilan Broz .preresume = crypt_preresume, 2076e48d4bbfSMilan Broz .resume = crypt_resume, 2077e48d4bbfSMilan Broz .message = crypt_message, 2078af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 2079586b286bSMike Snitzer .io_hints = crypt_io_hints, 20801da177e4SLinus Torvalds }; 20811da177e4SLinus Torvalds 20821da177e4SLinus Torvalds static int __init dm_crypt_init(void) 20831da177e4SLinus Torvalds { 20841da177e4SLinus Torvalds int r; 20851da177e4SLinus Torvalds 20861da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 208794f5e024SMikulas Patocka if (r < 0) 208872d94861SAlasdair G Kergon DMERR("register failed %d", r); 20891da177e4SLinus Torvalds 20901da177e4SLinus Torvalds return r; 20911da177e4SLinus Torvalds } 20921da177e4SLinus Torvalds 20931da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 20941da177e4SLinus Torvalds { 209510d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 20961da177e4SLinus Torvalds } 20971da177e4SLinus Torvalds 20981da177e4SLinus Torvalds module_init(dm_crypt_init); 20991da177e4SLinus Torvalds module_exit(dm_crypt_exit); 21001da177e4SLinus Torvalds 2101bf14299fSJana Saout MODULE_AUTHOR("Jana Saout <jana@saout.de>"); 21021da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 21031da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 2104