11da177e4SLinus Torvalds /* 2bf14299fSJana Saout * Copyright (C) 2003 Jana Saout <jana@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4ef43aa38SMilan Broz * Copyright (C) 2006-2017 Red Hat, Inc. All rights reserved. 5ef43aa38SMilan Broz * Copyright (C) 2013-2017 Milan Broz <gmazyland@gmail.com> 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * This file is released under the GPL. 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds 1043d69034SMilan Broz #include <linux/completion.h> 11d1806f6aSHerbert Xu #include <linux/err.h> 121da177e4SLinus Torvalds #include <linux/module.h> 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/kernel.h> 15c538f6ecSOndrej Kozina #include <linux/key.h> 161da177e4SLinus Torvalds #include <linux/bio.h> 171da177e4SLinus Torvalds #include <linux/blkdev.h> 181da177e4SLinus Torvalds #include <linux/mempool.h> 191da177e4SLinus Torvalds #include <linux/slab.h> 201da177e4SLinus Torvalds #include <linux/crypto.h> 211da177e4SLinus Torvalds #include <linux/workqueue.h> 22dc267621SMikulas Patocka #include <linux/kthread.h> 233fcfab16SAndrew Morton #include <linux/backing-dev.h> 2460063497SArun Sharma #include <linux/atomic.h> 25378f058cSDavid Hardeman #include <linux/scatterlist.h> 26b3c5fd30SMikulas Patocka #include <linux/rbtree.h> 27027c431cSOndrej Kozina #include <linux/ctype.h> 281da177e4SLinus Torvalds #include <asm/page.h> 2948527fa7SRik Snel #include <asm/unaligned.h> 3034745785SMilan Broz #include <crypto/hash.h> 3134745785SMilan Broz #include <crypto/md5.h> 3234745785SMilan Broz #include <crypto/algapi.h> 33bbdb23b5SHerbert Xu #include <crypto/skcipher.h> 34ef43aa38SMilan Broz #include <crypto/aead.h> 35ef43aa38SMilan Broz #include <crypto/authenc.h> 36ef43aa38SMilan Broz #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */ 37c538f6ecSOndrej Kozina #include <keys/user-type.h> 381da177e4SLinus Torvalds 39586e80e6SMikulas Patocka #include <linux/device-mapper.h> 401da177e4SLinus Torvalds 4172d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 421da177e4SLinus Torvalds 431da177e4SLinus Torvalds /* 441da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 451da177e4SLinus Torvalds */ 461da177e4SLinus Torvalds struct convert_context { 4743d69034SMilan Broz struct completion restart; 481da177e4SLinus Torvalds struct bio *bio_in; 491da177e4SLinus Torvalds struct bio *bio_out; 50003b5c57SKent Overstreet struct bvec_iter iter_in; 51003b5c57SKent Overstreet struct bvec_iter iter_out; 52c66029f4SMikulas Patocka sector_t cc_sector; 5340b6229bSMikulas Patocka atomic_t cc_pending; 54ef43aa38SMilan Broz union { 55bbdb23b5SHerbert Xu struct skcipher_request *req; 56ef43aa38SMilan Broz struct aead_request *req_aead; 57ef43aa38SMilan Broz } r; 58ef43aa38SMilan Broz 591da177e4SLinus Torvalds }; 601da177e4SLinus Torvalds 6153017030SMilan Broz /* 6253017030SMilan Broz * per bio private data 6353017030SMilan Broz */ 6453017030SMilan Broz struct dm_crypt_io { 6549a8a920SAlasdair G Kergon struct crypt_config *cc; 6653017030SMilan Broz struct bio *base_bio; 67ef43aa38SMilan Broz u8 *integrity_metadata; 68ef43aa38SMilan Broz bool integrity_metadata_from_pool; 6953017030SMilan Broz struct work_struct work; 7053017030SMilan Broz 7153017030SMilan Broz struct convert_context ctx; 7253017030SMilan Broz 7340b6229bSMikulas Patocka atomic_t io_pending; 744e4cbee9SChristoph Hellwig blk_status_t error; 750c395b0fSMilan Broz sector_t sector; 76dc267621SMikulas Patocka 77b3c5fd30SMikulas Patocka struct rb_node rb_node; 78298a9fa0SMikulas Patocka } CRYPTO_MINALIGN_ATTR; 7953017030SMilan Broz 8001482b76SMilan Broz struct dm_crypt_request { 81b2174eebSHuang Ying struct convert_context *ctx; 82ef43aa38SMilan Broz struct scatterlist sg_in[4]; 83ef43aa38SMilan Broz struct scatterlist sg_out[4]; 842dc5327dSMilan Broz sector_t iv_sector; 8501482b76SMilan Broz }; 8601482b76SMilan Broz 871da177e4SLinus Torvalds struct crypt_config; 881da177e4SLinus Torvalds 891da177e4SLinus Torvalds struct crypt_iv_operations { 901da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 911da177e4SLinus Torvalds const char *opts); 921da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 93b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 94542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 952dc5327dSMilan Broz int (*generator)(struct crypt_config *cc, u8 *iv, 962dc5327dSMilan Broz struct dm_crypt_request *dmreq); 972dc5327dSMilan Broz int (*post)(struct crypt_config *cc, u8 *iv, 982dc5327dSMilan Broz struct dm_crypt_request *dmreq); 991da177e4SLinus Torvalds }; 1001da177e4SLinus Torvalds 10160473592SMilan Broz struct iv_essiv_private { 102bbdb23b5SHerbert Xu struct crypto_ahash *hash_tfm; 103b95bf2d3SMilan Broz u8 *salt; 10460473592SMilan Broz }; 10560473592SMilan Broz 10660473592SMilan Broz struct iv_benbi_private { 10760473592SMilan Broz int shift; 10860473592SMilan Broz }; 10960473592SMilan Broz 11034745785SMilan Broz #define LMK_SEED_SIZE 64 /* hash + 0 */ 11134745785SMilan Broz struct iv_lmk_private { 11234745785SMilan Broz struct crypto_shash *hash_tfm; 11334745785SMilan Broz u8 *seed; 11434745785SMilan Broz }; 11534745785SMilan Broz 116ed04d981SMilan Broz #define TCW_WHITENING_SIZE 16 117ed04d981SMilan Broz struct iv_tcw_private { 118ed04d981SMilan Broz struct crypto_shash *crc32_tfm; 119ed04d981SMilan Broz u8 *iv_seed; 120ed04d981SMilan Broz u8 *whitening; 121ed04d981SMilan Broz }; 122ed04d981SMilan Broz 1231da177e4SLinus Torvalds /* 1241da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 1251da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 1261da177e4SLinus Torvalds */ 1270f5d8e6eSMikulas Patocka enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID, 128f659b100SRabin Vincent DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD }; 129c0297721SAndi Kleen 130ef43aa38SMilan Broz enum cipher_flags { 131ef43aa38SMilan Broz CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cihper */ 1328f0009a2SMilan Broz CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */ 133ef43aa38SMilan Broz }; 134ef43aa38SMilan Broz 135c0297721SAndi Kleen /* 136610f2de3SMikulas Patocka * The fields in here must be read only after initialization. 137c0297721SAndi Kleen */ 1381da177e4SLinus Torvalds struct crypt_config { 1391da177e4SLinus Torvalds struct dm_dev *dev; 1401da177e4SLinus Torvalds sector_t start; 1411da177e4SLinus Torvalds 1421da177e4SLinus Torvalds /* 143ef43aa38SMilan Broz * pool for per bio private data, crypto requests, 144ef43aa38SMilan Broz * encryption requeusts/buffer pages and integrity tags 1451da177e4SLinus Torvalds */ 146ddd42edfSMilan Broz mempool_t *req_pool; 1471da177e4SLinus Torvalds mempool_t *page_pool; 148ef43aa38SMilan Broz mempool_t *tag_pool; 149ef43aa38SMilan Broz unsigned tag_pool_max_sectors; 150ef43aa38SMilan Broz 1516a24c718SMilan Broz struct bio_set *bs; 1527145c241SMikulas Patocka struct mutex bio_alloc_lock; 1531da177e4SLinus Torvalds 154cabf08e4SMilan Broz struct workqueue_struct *io_queue; 155cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1563f1e9070SMilan Broz 157dc267621SMikulas Patocka struct task_struct *write_thread; 158dc267621SMikulas Patocka wait_queue_head_t write_thread_wait; 159b3c5fd30SMikulas Patocka struct rb_root write_tree; 160dc267621SMikulas Patocka 1615ebaee6dSMilan Broz char *cipher; 1627dbcd137SMilan Broz char *cipher_string; 163ef43aa38SMilan Broz char *cipher_auth; 164c538f6ecSOndrej Kozina char *key_string; 1655ebaee6dSMilan Broz 1661b1b58f5SJulia Lawall const struct crypt_iv_operations *iv_gen_ops; 16779066ad3SHerbert Xu union { 16860473592SMilan Broz struct iv_essiv_private essiv; 16960473592SMilan Broz struct iv_benbi_private benbi; 17034745785SMilan Broz struct iv_lmk_private lmk; 171ed04d981SMilan Broz struct iv_tcw_private tcw; 17279066ad3SHerbert Xu } iv_gen_private; 1731da177e4SLinus Torvalds sector_t iv_offset; 1741da177e4SLinus Torvalds unsigned int iv_size; 175ff3af92bSMikulas Patocka unsigned short int sector_size; 176ff3af92bSMikulas Patocka unsigned char sector_shift; 1771da177e4SLinus Torvalds 178fd2d231fSMikulas Patocka /* ESSIV: struct crypto_cipher *essiv_tfm */ 179fd2d231fSMikulas Patocka void *iv_private; 180ef43aa38SMilan Broz union { 181bbdb23b5SHerbert Xu struct crypto_skcipher **tfms; 182ef43aa38SMilan Broz struct crypto_aead **tfms_aead; 183ef43aa38SMilan Broz } cipher_tfm; 184d1f96423SMilan Broz unsigned tfms_count; 185ef43aa38SMilan Broz unsigned long cipher_flags; 186c0297721SAndi Kleen 187c0297721SAndi Kleen /* 188ddd42edfSMilan Broz * Layout of each crypto request: 189ddd42edfSMilan Broz * 190bbdb23b5SHerbert Xu * struct skcipher_request 191ddd42edfSMilan Broz * context 192ddd42edfSMilan Broz * padding 193ddd42edfSMilan Broz * struct dm_crypt_request 194ddd42edfSMilan Broz * padding 195ddd42edfSMilan Broz * IV 196ddd42edfSMilan Broz * 197ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 198ddd42edfSMilan Broz * correctly aligned. 199ddd42edfSMilan Broz */ 200ddd42edfSMilan Broz unsigned int dmreq_start; 201ddd42edfSMilan Broz 202298a9fa0SMikulas Patocka unsigned int per_bio_data_size; 203298a9fa0SMikulas Patocka 204e48d4bbfSMilan Broz unsigned long flags; 2051da177e4SLinus Torvalds unsigned int key_size; 206da31a078SMilan Broz unsigned int key_parts; /* independent parts in key buffer */ 207da31a078SMilan Broz unsigned int key_extra_size; /* additional keys length */ 208ef43aa38SMilan Broz unsigned int key_mac_size; /* MAC key size for authenc(...) */ 209ef43aa38SMilan Broz 210ef43aa38SMilan Broz unsigned int integrity_tag_size; 211ef43aa38SMilan Broz unsigned int integrity_iv_size; 212ef43aa38SMilan Broz unsigned int on_disk_tag_size; 213ef43aa38SMilan Broz 214ef43aa38SMilan Broz u8 *authenc_key; /* space for keys in authenc() format (if used) */ 2151da177e4SLinus Torvalds u8 key[0]; 2161da177e4SLinus Torvalds }; 2171da177e4SLinus Torvalds 2180a83df6cSMikulas Patocka #define MIN_IOS 64 219ef43aa38SMilan Broz #define MAX_TAG_SIZE 480 220ef43aa38SMilan Broz #define POOL_ENTRY_SIZE 512 2211da177e4SLinus Torvalds 222028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 223395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 224ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc, 225ef43aa38SMilan Broz struct scatterlist *sg); 226027581f3SOlaf Kirch 227c0297721SAndi Kleen /* 22886f917adSEric Biggers * Use this to access cipher attributes that are independent of the key. 229c0297721SAndi Kleen */ 230bbdb23b5SHerbert Xu static struct crypto_skcipher *any_tfm(struct crypt_config *cc) 231c0297721SAndi Kleen { 232ef43aa38SMilan Broz return cc->cipher_tfm.tfms[0]; 233ef43aa38SMilan Broz } 234ef43aa38SMilan Broz 235ef43aa38SMilan Broz static struct crypto_aead *any_tfm_aead(struct crypt_config *cc) 236ef43aa38SMilan Broz { 237ef43aa38SMilan Broz return cc->cipher_tfm.tfms_aead[0]; 238c0297721SAndi Kleen } 239c0297721SAndi Kleen 2401da177e4SLinus Torvalds /* 2411da177e4SLinus Torvalds * Different IV generation algorithms: 2421da177e4SLinus Torvalds * 2433c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 2443a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 2451da177e4SLinus Torvalds * 24661afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 24761afef61SMilan Broz * number, padded with zeros if necessary. 24861afef61SMilan Broz * 2497e3fd855SMilan Broz * plain64be: the initial vector is the 64-bit big-endian version of the sector 2507e3fd855SMilan Broz * number, padded with zeros if necessary. 2517e3fd855SMilan Broz * 2523c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 2531da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 2541da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 2551da177e4SLinus Torvalds * 25648527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 25748527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 25848527fa7SRik Snel * 25946b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 26046b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 26146b47730SLudwig Nussel * 26234745785SMilan Broz * lmk: Compatible implementation of the block chaining mode used 26334745785SMilan Broz * by the Loop-AES block device encryption system 26434745785SMilan Broz * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/ 26534745785SMilan Broz * It operates on full 512 byte sectors and uses CBC 26634745785SMilan Broz * with an IV derived from the sector number, the data and 26734745785SMilan Broz * optionally extra IV seed. 26834745785SMilan Broz * This means that after decryption the first block 26934745785SMilan Broz * of sector must be tweaked according to decrypted data. 27034745785SMilan Broz * Loop-AES can use three encryption schemes: 27134745785SMilan Broz * version 1: is plain aes-cbc mode 27234745785SMilan Broz * version 2: uses 64 multikey scheme with lmk IV generator 27334745785SMilan Broz * version 3: the same as version 2 with additional IV seed 27434745785SMilan Broz * (it uses 65 keys, last key is used as IV seed) 27534745785SMilan Broz * 276ed04d981SMilan Broz * tcw: Compatible implementation of the block chaining mode used 277ed04d981SMilan Broz * by the TrueCrypt device encryption system (prior to version 4.1). 278e44f23b3SMilan Broz * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat 279ed04d981SMilan Broz * It operates on full 512 byte sectors and uses CBC 280ed04d981SMilan Broz * with an IV derived from initial key and the sector number. 281ed04d981SMilan Broz * In addition, whitening value is applied on every sector, whitening 282ed04d981SMilan Broz * is calculated from initial key, sector number and mixed using CRC32. 283ed04d981SMilan Broz * Note that this encryption scheme is vulnerable to watermarking attacks 284ed04d981SMilan Broz * and should be used for old compatible containers access only. 285ed04d981SMilan Broz * 2861da177e4SLinus Torvalds * plumb: unimplemented, see: 2871da177e4SLinus Torvalds * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 2881da177e4SLinus Torvalds */ 2891da177e4SLinus Torvalds 2902dc5327dSMilan Broz static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, 2912dc5327dSMilan Broz struct dm_crypt_request *dmreq) 2921da177e4SLinus Torvalds { 2931da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 294283a8328SAlasdair G Kergon *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff); 2951da177e4SLinus Torvalds 2961da177e4SLinus Torvalds return 0; 2971da177e4SLinus Torvalds } 2981da177e4SLinus Torvalds 29961afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 3002dc5327dSMilan Broz struct dm_crypt_request *dmreq) 30161afef61SMilan Broz { 30261afef61SMilan Broz memset(iv, 0, cc->iv_size); 303283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 30461afef61SMilan Broz 30561afef61SMilan Broz return 0; 30661afef61SMilan Broz } 30761afef61SMilan Broz 3087e3fd855SMilan Broz static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv, 3097e3fd855SMilan Broz struct dm_crypt_request *dmreq) 3107e3fd855SMilan Broz { 3117e3fd855SMilan Broz memset(iv, 0, cc->iv_size); 3127e3fd855SMilan Broz /* iv_size is at least of size u64; usually it is 16 bytes */ 3137e3fd855SMilan Broz *(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector); 3147e3fd855SMilan Broz 3157e3fd855SMilan Broz return 0; 3167e3fd855SMilan Broz } 3177e3fd855SMilan Broz 318b95bf2d3SMilan Broz /* Initialise ESSIV - compute salt but no local memory allocations */ 319b95bf2d3SMilan Broz static int crypt_iv_essiv_init(struct crypt_config *cc) 320b95bf2d3SMilan Broz { 321b95bf2d3SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 322bbdb23b5SHerbert Xu AHASH_REQUEST_ON_STACK(req, essiv->hash_tfm); 323b95bf2d3SMilan Broz struct scatterlist sg; 324c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 325fd2d231fSMikulas Patocka int err; 326b95bf2d3SMilan Broz 327b95bf2d3SMilan Broz sg_init_one(&sg, cc->key, cc->key_size); 328bbdb23b5SHerbert Xu ahash_request_set_tfm(req, essiv->hash_tfm); 329bbdb23b5SHerbert Xu ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); 330bbdb23b5SHerbert Xu ahash_request_set_crypt(req, &sg, essiv->salt, cc->key_size); 331b95bf2d3SMilan Broz 332bbdb23b5SHerbert Xu err = crypto_ahash_digest(req); 333bbdb23b5SHerbert Xu ahash_request_zero(req); 334b95bf2d3SMilan Broz if (err) 335b95bf2d3SMilan Broz return err; 336b95bf2d3SMilan Broz 337fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 338c0297721SAndi Kleen 339c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, essiv->salt, 340bbdb23b5SHerbert Xu crypto_ahash_digestsize(essiv->hash_tfm)); 341c0297721SAndi Kleen if (err) 342c0297721SAndi Kleen return err; 343c0297721SAndi Kleen 344c0297721SAndi Kleen return 0; 345b95bf2d3SMilan Broz } 346b95bf2d3SMilan Broz 347542da317SMilan Broz /* Wipe salt and reset key derived from volume key */ 348542da317SMilan Broz static int crypt_iv_essiv_wipe(struct crypt_config *cc) 349542da317SMilan Broz { 350542da317SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 351bbdb23b5SHerbert Xu unsigned salt_size = crypto_ahash_digestsize(essiv->hash_tfm); 352c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 353fd2d231fSMikulas Patocka int r, err = 0; 354542da317SMilan Broz 355542da317SMilan Broz memset(essiv->salt, 0, salt_size); 356542da317SMilan Broz 357fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 358c0297721SAndi Kleen r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size); 359c0297721SAndi Kleen if (r) 360c0297721SAndi Kleen err = r; 361c0297721SAndi Kleen 362c0297721SAndi Kleen return err; 363c0297721SAndi Kleen } 364c0297721SAndi Kleen 36586f917adSEric Biggers /* Allocate the cipher for ESSIV */ 36686f917adSEric Biggers static struct crypto_cipher *alloc_essiv_cipher(struct crypt_config *cc, 367c0297721SAndi Kleen struct dm_target *ti, 36886f917adSEric Biggers const u8 *salt, 36986f917adSEric Biggers unsigned int saltsize) 370c0297721SAndi Kleen { 371c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 372c0297721SAndi Kleen int err; 373c0297721SAndi Kleen 374c0297721SAndi Kleen /* Setup the essiv_tfm with the given salt */ 375c0297721SAndi Kleen essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 376c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 377c0297721SAndi Kleen ti->error = "Error allocating crypto tfm for ESSIV"; 378c0297721SAndi Kleen return essiv_tfm; 379c0297721SAndi Kleen } 380c0297721SAndi Kleen 381ef43aa38SMilan Broz if (crypto_cipher_blocksize(essiv_tfm) != cc->iv_size) { 382c0297721SAndi Kleen ti->error = "Block size of ESSIV cipher does " 383c0297721SAndi Kleen "not match IV size of block cipher"; 384c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 385c0297721SAndi Kleen return ERR_PTR(-EINVAL); 386c0297721SAndi Kleen } 387c0297721SAndi Kleen 388c0297721SAndi Kleen err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 389c0297721SAndi Kleen if (err) { 390c0297721SAndi Kleen ti->error = "Failed to set key for ESSIV cipher"; 391c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 392c0297721SAndi Kleen return ERR_PTR(err); 393c0297721SAndi Kleen } 394c0297721SAndi Kleen 395c0297721SAndi Kleen return essiv_tfm; 396542da317SMilan Broz } 397542da317SMilan Broz 39860473592SMilan Broz static void crypt_iv_essiv_dtr(struct crypt_config *cc) 39960473592SMilan Broz { 400c0297721SAndi Kleen struct crypto_cipher *essiv_tfm; 40160473592SMilan Broz struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; 40260473592SMilan Broz 403bbdb23b5SHerbert Xu crypto_free_ahash(essiv->hash_tfm); 404b95bf2d3SMilan Broz essiv->hash_tfm = NULL; 405b95bf2d3SMilan Broz 406b95bf2d3SMilan Broz kzfree(essiv->salt); 407b95bf2d3SMilan Broz essiv->salt = NULL; 408c0297721SAndi Kleen 409fd2d231fSMikulas Patocka essiv_tfm = cc->iv_private; 410c0297721SAndi Kleen 411c0297721SAndi Kleen if (essiv_tfm) 412c0297721SAndi Kleen crypto_free_cipher(essiv_tfm); 413c0297721SAndi Kleen 414fd2d231fSMikulas Patocka cc->iv_private = NULL; 41560473592SMilan Broz } 41660473592SMilan Broz 4171da177e4SLinus Torvalds static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 4181da177e4SLinus Torvalds const char *opts) 4191da177e4SLinus Torvalds { 4205861f1beSMilan Broz struct crypto_cipher *essiv_tfm = NULL; 421bbdb23b5SHerbert Xu struct crypto_ahash *hash_tfm = NULL; 4225861f1beSMilan Broz u8 *salt = NULL; 423fd2d231fSMikulas Patocka int err; 4241da177e4SLinus Torvalds 4255861f1beSMilan Broz if (!opts) { 42672d94861SAlasdair G Kergon ti->error = "Digest algorithm missing for ESSIV mode"; 4271da177e4SLinus Torvalds return -EINVAL; 4281da177e4SLinus Torvalds } 4291da177e4SLinus Torvalds 430b95bf2d3SMilan Broz /* Allocate hash algorithm */ 431bbdb23b5SHerbert Xu hash_tfm = crypto_alloc_ahash(opts, 0, CRYPTO_ALG_ASYNC); 43235058687SHerbert Xu if (IS_ERR(hash_tfm)) { 43372d94861SAlasdair G Kergon ti->error = "Error initializing ESSIV hash"; 4345861f1beSMilan Broz err = PTR_ERR(hash_tfm); 4355861f1beSMilan Broz goto bad; 4361da177e4SLinus Torvalds } 4371da177e4SLinus Torvalds 438bbdb23b5SHerbert Xu salt = kzalloc(crypto_ahash_digestsize(hash_tfm), GFP_KERNEL); 4395861f1beSMilan Broz if (!salt) { 44072d94861SAlasdair G Kergon ti->error = "Error kmallocing salt storage in ESSIV"; 4415861f1beSMilan Broz err = -ENOMEM; 4425861f1beSMilan Broz goto bad; 4431da177e4SLinus Torvalds } 4441da177e4SLinus Torvalds 445b95bf2d3SMilan Broz cc->iv_gen_private.essiv.salt = salt; 446b95bf2d3SMilan Broz cc->iv_gen_private.essiv.hash_tfm = hash_tfm; 447b95bf2d3SMilan Broz 44886f917adSEric Biggers essiv_tfm = alloc_essiv_cipher(cc, ti, salt, 449bbdb23b5SHerbert Xu crypto_ahash_digestsize(hash_tfm)); 450c0297721SAndi Kleen if (IS_ERR(essiv_tfm)) { 451c0297721SAndi Kleen crypt_iv_essiv_dtr(cc); 452c0297721SAndi Kleen return PTR_ERR(essiv_tfm); 453c0297721SAndi Kleen } 454fd2d231fSMikulas Patocka cc->iv_private = essiv_tfm; 455c0297721SAndi Kleen 4561da177e4SLinus Torvalds return 0; 4575861f1beSMilan Broz 4585861f1beSMilan Broz bad: 4595861f1beSMilan Broz if (hash_tfm && !IS_ERR(hash_tfm)) 460bbdb23b5SHerbert Xu crypto_free_ahash(hash_tfm); 461b95bf2d3SMilan Broz kfree(salt); 4625861f1beSMilan Broz return err; 4631da177e4SLinus Torvalds } 4641da177e4SLinus Torvalds 4652dc5327dSMilan Broz static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, 4662dc5327dSMilan Broz struct dm_crypt_request *dmreq) 4671da177e4SLinus Torvalds { 468fd2d231fSMikulas Patocka struct crypto_cipher *essiv_tfm = cc->iv_private; 469c0297721SAndi Kleen 4701da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 471283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 472c0297721SAndi Kleen crypto_cipher_encrypt_one(essiv_tfm, iv, iv); 473c0297721SAndi Kleen 4741da177e4SLinus Torvalds return 0; 4751da177e4SLinus Torvalds } 4761da177e4SLinus Torvalds 47748527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 47848527fa7SRik Snel const char *opts) 47948527fa7SRik Snel { 480bbdb23b5SHerbert Xu unsigned bs = crypto_skcipher_blocksize(any_tfm(cc)); 481f0d1b0b3SDavid Howells int log = ilog2(bs); 48248527fa7SRik Snel 48348527fa7SRik Snel /* we need to calculate how far we must shift the sector count 48448527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 48548527fa7SRik Snel 48648527fa7SRik Snel if (1 << log != bs) { 48748527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 48848527fa7SRik Snel return -EINVAL; 48948527fa7SRik Snel } 49048527fa7SRik Snel 49148527fa7SRik Snel if (log > 9) { 49248527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 49348527fa7SRik Snel return -EINVAL; 49448527fa7SRik Snel } 49548527fa7SRik Snel 49660473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 49748527fa7SRik Snel 49848527fa7SRik Snel return 0; 49948527fa7SRik Snel } 50048527fa7SRik Snel 50148527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 50248527fa7SRik Snel { 50348527fa7SRik Snel } 50448527fa7SRik Snel 5052dc5327dSMilan Broz static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, 5062dc5327dSMilan Broz struct dm_crypt_request *dmreq) 50748527fa7SRik Snel { 50879066ad3SHerbert Xu __be64 val; 50979066ad3SHerbert Xu 51048527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 51179066ad3SHerbert Xu 5122dc5327dSMilan Broz val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1); 51379066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 51448527fa7SRik Snel 5151da177e4SLinus Torvalds return 0; 5161da177e4SLinus Torvalds } 5171da177e4SLinus Torvalds 5182dc5327dSMilan Broz static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, 5192dc5327dSMilan Broz struct dm_crypt_request *dmreq) 52046b47730SLudwig Nussel { 52146b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 52246b47730SLudwig Nussel 52346b47730SLudwig Nussel return 0; 52446b47730SLudwig Nussel } 52546b47730SLudwig Nussel 52634745785SMilan Broz static void crypt_iv_lmk_dtr(struct crypt_config *cc) 52734745785SMilan Broz { 52834745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 52934745785SMilan Broz 53034745785SMilan Broz if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm)) 53134745785SMilan Broz crypto_free_shash(lmk->hash_tfm); 53234745785SMilan Broz lmk->hash_tfm = NULL; 53334745785SMilan Broz 53434745785SMilan Broz kzfree(lmk->seed); 53534745785SMilan Broz lmk->seed = NULL; 53634745785SMilan Broz } 53734745785SMilan Broz 53834745785SMilan Broz static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti, 53934745785SMilan Broz const char *opts) 54034745785SMilan Broz { 54134745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 54234745785SMilan Broz 5438f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) { 5448f0009a2SMilan Broz ti->error = "Unsupported sector size for LMK"; 5458f0009a2SMilan Broz return -EINVAL; 5468f0009a2SMilan Broz } 5478f0009a2SMilan Broz 54834745785SMilan Broz lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0); 54934745785SMilan Broz if (IS_ERR(lmk->hash_tfm)) { 55034745785SMilan Broz ti->error = "Error initializing LMK hash"; 55134745785SMilan Broz return PTR_ERR(lmk->hash_tfm); 55234745785SMilan Broz } 55334745785SMilan Broz 55434745785SMilan Broz /* No seed in LMK version 2 */ 55534745785SMilan Broz if (cc->key_parts == cc->tfms_count) { 55634745785SMilan Broz lmk->seed = NULL; 55734745785SMilan Broz return 0; 55834745785SMilan Broz } 55934745785SMilan Broz 56034745785SMilan Broz lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL); 56134745785SMilan Broz if (!lmk->seed) { 56234745785SMilan Broz crypt_iv_lmk_dtr(cc); 56334745785SMilan Broz ti->error = "Error kmallocing seed storage in LMK"; 56434745785SMilan Broz return -ENOMEM; 56534745785SMilan Broz } 56634745785SMilan Broz 56734745785SMilan Broz return 0; 56834745785SMilan Broz } 56934745785SMilan Broz 57034745785SMilan Broz static int crypt_iv_lmk_init(struct crypt_config *cc) 57134745785SMilan Broz { 57234745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 57334745785SMilan Broz int subkey_size = cc->key_size / cc->key_parts; 57434745785SMilan Broz 57534745785SMilan Broz /* LMK seed is on the position of LMK_KEYS + 1 key */ 57634745785SMilan Broz if (lmk->seed) 57734745785SMilan Broz memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size), 57834745785SMilan Broz crypto_shash_digestsize(lmk->hash_tfm)); 57934745785SMilan Broz 58034745785SMilan Broz return 0; 58134745785SMilan Broz } 58234745785SMilan Broz 58334745785SMilan Broz static int crypt_iv_lmk_wipe(struct crypt_config *cc) 58434745785SMilan Broz { 58534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 58634745785SMilan Broz 58734745785SMilan Broz if (lmk->seed) 58834745785SMilan Broz memset(lmk->seed, 0, LMK_SEED_SIZE); 58934745785SMilan Broz 59034745785SMilan Broz return 0; 59134745785SMilan Broz } 59234745785SMilan Broz 59334745785SMilan Broz static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv, 59434745785SMilan Broz struct dm_crypt_request *dmreq, 59534745785SMilan Broz u8 *data) 59634745785SMilan Broz { 59734745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 598b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, lmk->hash_tfm); 59934745785SMilan Broz struct md5_state md5state; 600da31a078SMilan Broz __le32 buf[4]; 60134745785SMilan Broz int i, r; 60234745785SMilan Broz 603b6106265SJan-Simon Möller desc->tfm = lmk->hash_tfm; 604b6106265SJan-Simon Möller desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 60534745785SMilan Broz 606b6106265SJan-Simon Möller r = crypto_shash_init(desc); 60734745785SMilan Broz if (r) 60834745785SMilan Broz return r; 60934745785SMilan Broz 61034745785SMilan Broz if (lmk->seed) { 611b6106265SJan-Simon Möller r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE); 61234745785SMilan Broz if (r) 61334745785SMilan Broz return r; 61434745785SMilan Broz } 61534745785SMilan Broz 61634745785SMilan Broz /* Sector is always 512B, block size 16, add data of blocks 1-31 */ 617b6106265SJan-Simon Möller r = crypto_shash_update(desc, data + 16, 16 * 31); 61834745785SMilan Broz if (r) 61934745785SMilan Broz return r; 62034745785SMilan Broz 62134745785SMilan Broz /* Sector is cropped to 56 bits here */ 62234745785SMilan Broz buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF); 62334745785SMilan Broz buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000); 62434745785SMilan Broz buf[2] = cpu_to_le32(4024); 62534745785SMilan Broz buf[3] = 0; 626b6106265SJan-Simon Möller r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf)); 62734745785SMilan Broz if (r) 62834745785SMilan Broz return r; 62934745785SMilan Broz 63034745785SMilan Broz /* No MD5 padding here */ 631b6106265SJan-Simon Möller r = crypto_shash_export(desc, &md5state); 63234745785SMilan Broz if (r) 63334745785SMilan Broz return r; 63434745785SMilan Broz 63534745785SMilan Broz for (i = 0; i < MD5_HASH_WORDS; i++) 63634745785SMilan Broz __cpu_to_le32s(&md5state.hash[i]); 63734745785SMilan Broz memcpy(iv, &md5state.hash, cc->iv_size); 63834745785SMilan Broz 63934745785SMilan Broz return 0; 64034745785SMilan Broz } 64134745785SMilan Broz 64234745785SMilan Broz static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv, 64334745785SMilan Broz struct dm_crypt_request *dmreq) 64434745785SMilan Broz { 645ef43aa38SMilan Broz struct scatterlist *sg; 64634745785SMilan Broz u8 *src; 64734745785SMilan Broz int r = 0; 64834745785SMilan Broz 64934745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 650ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in); 651ef43aa38SMilan Broz src = kmap_atomic(sg_page(sg)); 652ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset); 653c2e022cbSCong Wang kunmap_atomic(src); 65434745785SMilan Broz } else 65534745785SMilan Broz memset(iv, 0, cc->iv_size); 65634745785SMilan Broz 65734745785SMilan Broz return r; 65834745785SMilan Broz } 65934745785SMilan Broz 66034745785SMilan Broz static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv, 66134745785SMilan Broz struct dm_crypt_request *dmreq) 66234745785SMilan Broz { 663ef43aa38SMilan Broz struct scatterlist *sg; 66434745785SMilan Broz u8 *dst; 66534745785SMilan Broz int r; 66634745785SMilan Broz 66734745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) 66834745785SMilan Broz return 0; 66934745785SMilan Broz 670ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 671ef43aa38SMilan Broz dst = kmap_atomic(sg_page(sg)); 672ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset); 67334745785SMilan Broz 67434745785SMilan Broz /* Tweak the first block of plaintext sector */ 67534745785SMilan Broz if (!r) 676ef43aa38SMilan Broz crypto_xor(dst + sg->offset, iv, cc->iv_size); 67734745785SMilan Broz 678c2e022cbSCong Wang kunmap_atomic(dst); 67934745785SMilan Broz return r; 68034745785SMilan Broz } 68134745785SMilan Broz 682ed04d981SMilan Broz static void crypt_iv_tcw_dtr(struct crypt_config *cc) 683ed04d981SMilan Broz { 684ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 685ed04d981SMilan Broz 686ed04d981SMilan Broz kzfree(tcw->iv_seed); 687ed04d981SMilan Broz tcw->iv_seed = NULL; 688ed04d981SMilan Broz kzfree(tcw->whitening); 689ed04d981SMilan Broz tcw->whitening = NULL; 690ed04d981SMilan Broz 691ed04d981SMilan Broz if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm)) 692ed04d981SMilan Broz crypto_free_shash(tcw->crc32_tfm); 693ed04d981SMilan Broz tcw->crc32_tfm = NULL; 694ed04d981SMilan Broz } 695ed04d981SMilan Broz 696ed04d981SMilan Broz static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti, 697ed04d981SMilan Broz const char *opts) 698ed04d981SMilan Broz { 699ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 700ed04d981SMilan Broz 7018f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) { 7028f0009a2SMilan Broz ti->error = "Unsupported sector size for TCW"; 7038f0009a2SMilan Broz return -EINVAL; 7048f0009a2SMilan Broz } 7058f0009a2SMilan Broz 706ed04d981SMilan Broz if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) { 707ed04d981SMilan Broz ti->error = "Wrong key size for TCW"; 708ed04d981SMilan Broz return -EINVAL; 709ed04d981SMilan Broz } 710ed04d981SMilan Broz 711ed04d981SMilan Broz tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 0); 712ed04d981SMilan Broz if (IS_ERR(tcw->crc32_tfm)) { 713ed04d981SMilan Broz ti->error = "Error initializing CRC32 in TCW"; 714ed04d981SMilan Broz return PTR_ERR(tcw->crc32_tfm); 715ed04d981SMilan Broz } 716ed04d981SMilan Broz 717ed04d981SMilan Broz tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL); 718ed04d981SMilan Broz tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL); 719ed04d981SMilan Broz if (!tcw->iv_seed || !tcw->whitening) { 720ed04d981SMilan Broz crypt_iv_tcw_dtr(cc); 721ed04d981SMilan Broz ti->error = "Error allocating seed storage in TCW"; 722ed04d981SMilan Broz return -ENOMEM; 723ed04d981SMilan Broz } 724ed04d981SMilan Broz 725ed04d981SMilan Broz return 0; 726ed04d981SMilan Broz } 727ed04d981SMilan Broz 728ed04d981SMilan Broz static int crypt_iv_tcw_init(struct crypt_config *cc) 729ed04d981SMilan Broz { 730ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 731ed04d981SMilan Broz int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE; 732ed04d981SMilan Broz 733ed04d981SMilan Broz memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size); 734ed04d981SMilan Broz memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size], 735ed04d981SMilan Broz TCW_WHITENING_SIZE); 736ed04d981SMilan Broz 737ed04d981SMilan Broz return 0; 738ed04d981SMilan Broz } 739ed04d981SMilan Broz 740ed04d981SMilan Broz static int crypt_iv_tcw_wipe(struct crypt_config *cc) 741ed04d981SMilan Broz { 742ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 743ed04d981SMilan Broz 744ed04d981SMilan Broz memset(tcw->iv_seed, 0, cc->iv_size); 745ed04d981SMilan Broz memset(tcw->whitening, 0, TCW_WHITENING_SIZE); 746ed04d981SMilan Broz 747ed04d981SMilan Broz return 0; 748ed04d981SMilan Broz } 749ed04d981SMilan Broz 750ed04d981SMilan Broz static int crypt_iv_tcw_whitening(struct crypt_config *cc, 751ed04d981SMilan Broz struct dm_crypt_request *dmreq, 752ed04d981SMilan Broz u8 *data) 753ed04d981SMilan Broz { 754ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 755350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector); 756ed04d981SMilan Broz u8 buf[TCW_WHITENING_SIZE]; 757b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm); 758ed04d981SMilan Broz int i, r; 759ed04d981SMilan Broz 760ed04d981SMilan Broz /* xor whitening with sector number */ 76145fe93dfSArd Biesheuvel crypto_xor_cpy(buf, tcw->whitening, (u8 *)§or, 8); 76245fe93dfSArd Biesheuvel crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)§or, 8); 763ed04d981SMilan Broz 764ed04d981SMilan Broz /* calculate crc32 for every 32bit part and xor it */ 765b6106265SJan-Simon Möller desc->tfm = tcw->crc32_tfm; 766b6106265SJan-Simon Möller desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 767ed04d981SMilan Broz for (i = 0; i < 4; i++) { 768b6106265SJan-Simon Möller r = crypto_shash_init(desc); 769ed04d981SMilan Broz if (r) 770ed04d981SMilan Broz goto out; 771b6106265SJan-Simon Möller r = crypto_shash_update(desc, &buf[i * 4], 4); 772ed04d981SMilan Broz if (r) 773ed04d981SMilan Broz goto out; 774b6106265SJan-Simon Möller r = crypto_shash_final(desc, &buf[i * 4]); 775ed04d981SMilan Broz if (r) 776ed04d981SMilan Broz goto out; 777ed04d981SMilan Broz } 778ed04d981SMilan Broz crypto_xor(&buf[0], &buf[12], 4); 779ed04d981SMilan Broz crypto_xor(&buf[4], &buf[8], 4); 780ed04d981SMilan Broz 781ed04d981SMilan Broz /* apply whitening (8 bytes) to whole sector */ 782ed04d981SMilan Broz for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++) 783ed04d981SMilan Broz crypto_xor(data + i * 8, buf, 8); 784ed04d981SMilan Broz out: 7851a71d6ffSMilan Broz memzero_explicit(buf, sizeof(buf)); 786ed04d981SMilan Broz return r; 787ed04d981SMilan Broz } 788ed04d981SMilan Broz 789ed04d981SMilan Broz static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv, 790ed04d981SMilan Broz struct dm_crypt_request *dmreq) 791ed04d981SMilan Broz { 792ef43aa38SMilan Broz struct scatterlist *sg; 793ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 794350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector); 795ed04d981SMilan Broz u8 *src; 796ed04d981SMilan Broz int r = 0; 797ed04d981SMilan Broz 798ed04d981SMilan Broz /* Remove whitening from ciphertext */ 799ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) { 800ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in); 801ef43aa38SMilan Broz src = kmap_atomic(sg_page(sg)); 802ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset); 803ed04d981SMilan Broz kunmap_atomic(src); 804ed04d981SMilan Broz } 805ed04d981SMilan Broz 806ed04d981SMilan Broz /* Calculate IV */ 80745fe93dfSArd Biesheuvel crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)§or, 8); 808ed04d981SMilan Broz if (cc->iv_size > 8) 80945fe93dfSArd Biesheuvel crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)§or, 81045fe93dfSArd Biesheuvel cc->iv_size - 8); 811ed04d981SMilan Broz 812ed04d981SMilan Broz return r; 813ed04d981SMilan Broz } 814ed04d981SMilan Broz 815ed04d981SMilan Broz static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv, 816ed04d981SMilan Broz struct dm_crypt_request *dmreq) 817ed04d981SMilan Broz { 818ef43aa38SMilan Broz struct scatterlist *sg; 819ed04d981SMilan Broz u8 *dst; 820ed04d981SMilan Broz int r; 821ed04d981SMilan Broz 822ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) 823ed04d981SMilan Broz return 0; 824ed04d981SMilan Broz 825ed04d981SMilan Broz /* Apply whitening on ciphertext */ 826ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 827ef43aa38SMilan Broz dst = kmap_atomic(sg_page(sg)); 828ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset); 829ed04d981SMilan Broz kunmap_atomic(dst); 830ed04d981SMilan Broz 831ed04d981SMilan Broz return r; 832ed04d981SMilan Broz } 833ed04d981SMilan Broz 834ef43aa38SMilan Broz static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv, 835ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 836ef43aa38SMilan Broz { 837ef43aa38SMilan Broz /* Used only for writes, there must be an additional space to store IV */ 838ef43aa38SMilan Broz get_random_bytes(iv, cc->iv_size); 839ef43aa38SMilan Broz return 0; 840ef43aa38SMilan Broz } 841ef43aa38SMilan Broz 8421b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain_ops = { 8431da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 8441da177e4SLinus Torvalds }; 8451da177e4SLinus Torvalds 8461b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain64_ops = { 84761afef61SMilan Broz .generator = crypt_iv_plain64_gen 84861afef61SMilan Broz }; 84961afef61SMilan Broz 8507e3fd855SMilan Broz static const struct crypt_iv_operations crypt_iv_plain64be_ops = { 8517e3fd855SMilan Broz .generator = crypt_iv_plain64be_gen 8527e3fd855SMilan Broz }; 8537e3fd855SMilan Broz 8541b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_essiv_ops = { 8551da177e4SLinus Torvalds .ctr = crypt_iv_essiv_ctr, 8561da177e4SLinus Torvalds .dtr = crypt_iv_essiv_dtr, 857b95bf2d3SMilan Broz .init = crypt_iv_essiv_init, 858542da317SMilan Broz .wipe = crypt_iv_essiv_wipe, 8591da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 8601da177e4SLinus Torvalds }; 8611da177e4SLinus Torvalds 8621b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_benbi_ops = { 86348527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 86448527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 86548527fa7SRik Snel .generator = crypt_iv_benbi_gen 86648527fa7SRik Snel }; 8671da177e4SLinus Torvalds 8681b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_null_ops = { 86946b47730SLudwig Nussel .generator = crypt_iv_null_gen 87046b47730SLudwig Nussel }; 87146b47730SLudwig Nussel 8721b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_lmk_ops = { 87334745785SMilan Broz .ctr = crypt_iv_lmk_ctr, 87434745785SMilan Broz .dtr = crypt_iv_lmk_dtr, 87534745785SMilan Broz .init = crypt_iv_lmk_init, 87634745785SMilan Broz .wipe = crypt_iv_lmk_wipe, 87734745785SMilan Broz .generator = crypt_iv_lmk_gen, 87834745785SMilan Broz .post = crypt_iv_lmk_post 87934745785SMilan Broz }; 88034745785SMilan Broz 8811b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_tcw_ops = { 882ed04d981SMilan Broz .ctr = crypt_iv_tcw_ctr, 883ed04d981SMilan Broz .dtr = crypt_iv_tcw_dtr, 884ed04d981SMilan Broz .init = crypt_iv_tcw_init, 885ed04d981SMilan Broz .wipe = crypt_iv_tcw_wipe, 886ed04d981SMilan Broz .generator = crypt_iv_tcw_gen, 887ed04d981SMilan Broz .post = crypt_iv_tcw_post 888ed04d981SMilan Broz }; 889ed04d981SMilan Broz 890ef43aa38SMilan Broz static struct crypt_iv_operations crypt_iv_random_ops = { 891ef43aa38SMilan Broz .generator = crypt_iv_random_gen 892ef43aa38SMilan Broz }; 893ef43aa38SMilan Broz 894ef43aa38SMilan Broz /* 895ef43aa38SMilan Broz * Integrity extensions 896ef43aa38SMilan Broz */ 897ef43aa38SMilan Broz static bool crypt_integrity_aead(struct crypt_config *cc) 898ef43aa38SMilan Broz { 899ef43aa38SMilan Broz return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags); 900ef43aa38SMilan Broz } 901ef43aa38SMilan Broz 902ef43aa38SMilan Broz static bool crypt_integrity_hmac(struct crypt_config *cc) 903ef43aa38SMilan Broz { 90433d2f09fSMilan Broz return crypt_integrity_aead(cc) && cc->key_mac_size; 905ef43aa38SMilan Broz } 906ef43aa38SMilan Broz 907ef43aa38SMilan Broz /* Get sg containing data */ 908ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc, 909ef43aa38SMilan Broz struct scatterlist *sg) 910ef43aa38SMilan Broz { 91133d2f09fSMilan Broz if (unlikely(crypt_integrity_aead(cc))) 912ef43aa38SMilan Broz return &sg[2]; 913ef43aa38SMilan Broz 914ef43aa38SMilan Broz return sg; 915ef43aa38SMilan Broz } 916ef43aa38SMilan Broz 917ef43aa38SMilan Broz static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio) 918ef43aa38SMilan Broz { 919ef43aa38SMilan Broz struct bio_integrity_payload *bip; 920ef43aa38SMilan Broz unsigned int tag_len; 921ef43aa38SMilan Broz int ret; 922ef43aa38SMilan Broz 923ef43aa38SMilan Broz if (!bio_sectors(bio) || !io->cc->on_disk_tag_size) 924ef43aa38SMilan Broz return 0; 925ef43aa38SMilan Broz 926ef43aa38SMilan Broz bip = bio_integrity_alloc(bio, GFP_NOIO, 1); 927ef43aa38SMilan Broz if (IS_ERR(bip)) 928ef43aa38SMilan Broz return PTR_ERR(bip); 929ef43aa38SMilan Broz 930ef43aa38SMilan Broz tag_len = io->cc->on_disk_tag_size * bio_sectors(bio); 931ef43aa38SMilan Broz 932ef43aa38SMilan Broz bip->bip_iter.bi_size = tag_len; 933ef43aa38SMilan Broz bip->bip_iter.bi_sector = io->cc->start + io->sector; 934ef43aa38SMilan Broz 935ef43aa38SMilan Broz ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata), 936ef43aa38SMilan Broz tag_len, offset_in_page(io->integrity_metadata)); 937ef43aa38SMilan Broz if (unlikely(ret != tag_len)) 938ef43aa38SMilan Broz return -ENOMEM; 939ef43aa38SMilan Broz 940ef43aa38SMilan Broz return 0; 941ef43aa38SMilan Broz } 942ef43aa38SMilan Broz 943ef43aa38SMilan Broz static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti) 944ef43aa38SMilan Broz { 945ef43aa38SMilan Broz #ifdef CONFIG_BLK_DEV_INTEGRITY 946ef43aa38SMilan Broz struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk); 947ef43aa38SMilan Broz 948ef43aa38SMilan Broz /* From now we require underlying device with our integrity profile */ 949ef43aa38SMilan Broz if (!bi || strcasecmp(bi->profile->name, "DM-DIF-EXT-TAG")) { 950ef43aa38SMilan Broz ti->error = "Integrity profile not supported."; 951ef43aa38SMilan Broz return -EINVAL; 952ef43aa38SMilan Broz } 953ef43aa38SMilan Broz 954583fe747SMikulas Patocka if (bi->tag_size != cc->on_disk_tag_size || 955583fe747SMikulas Patocka bi->tuple_size != cc->on_disk_tag_size) { 956ef43aa38SMilan Broz ti->error = "Integrity profile tag size mismatch."; 957ef43aa38SMilan Broz return -EINVAL; 958ef43aa38SMilan Broz } 959583fe747SMikulas Patocka if (1 << bi->interval_exp != cc->sector_size) { 960583fe747SMikulas Patocka ti->error = "Integrity profile sector size mismatch."; 961583fe747SMikulas Patocka return -EINVAL; 962583fe747SMikulas Patocka } 963ef43aa38SMilan Broz 96433d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 965ef43aa38SMilan Broz cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size; 966ef43aa38SMilan Broz DMINFO("Integrity AEAD, tag size %u, IV size %u.", 967ef43aa38SMilan Broz cc->integrity_tag_size, cc->integrity_iv_size); 968ef43aa38SMilan Broz 969ef43aa38SMilan Broz if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) { 970ef43aa38SMilan Broz ti->error = "Integrity AEAD auth tag size is not supported."; 971ef43aa38SMilan Broz return -EINVAL; 972ef43aa38SMilan Broz } 973ef43aa38SMilan Broz } else if (cc->integrity_iv_size) 974ef43aa38SMilan Broz DMINFO("Additional per-sector space %u bytes for IV.", 975ef43aa38SMilan Broz cc->integrity_iv_size); 976ef43aa38SMilan Broz 977ef43aa38SMilan Broz if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) { 978ef43aa38SMilan Broz ti->error = "Not enough space for integrity tag in the profile."; 979ef43aa38SMilan Broz return -EINVAL; 980ef43aa38SMilan Broz } 981ef43aa38SMilan Broz 982ef43aa38SMilan Broz return 0; 983ef43aa38SMilan Broz #else 984ef43aa38SMilan Broz ti->error = "Integrity profile not supported."; 985ef43aa38SMilan Broz return -EINVAL; 986ef43aa38SMilan Broz #endif 987ef43aa38SMilan Broz } 988ef43aa38SMilan Broz 989d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 990d469f841SMilan Broz struct convert_context *ctx, 9911da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 992fcd369daSMilan Broz sector_t sector) 9931da177e4SLinus Torvalds { 9941da177e4SLinus Torvalds ctx->bio_in = bio_in; 9951da177e4SLinus Torvalds ctx->bio_out = bio_out; 996003b5c57SKent Overstreet if (bio_in) 997003b5c57SKent Overstreet ctx->iter_in = bio_in->bi_iter; 998003b5c57SKent Overstreet if (bio_out) 999003b5c57SKent Overstreet ctx->iter_out = bio_out->bi_iter; 1000c66029f4SMikulas Patocka ctx->cc_sector = sector + cc->iv_offset; 100143d69034SMilan Broz init_completion(&ctx->restart); 10021da177e4SLinus Torvalds } 10031da177e4SLinus Torvalds 1004b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 1005ef43aa38SMilan Broz void *req) 1006b2174eebSHuang Ying { 1007b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 1008b2174eebSHuang Ying } 1009b2174eebSHuang Ying 1010ef43aa38SMilan Broz static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq) 1011b2174eebSHuang Ying { 1012ef43aa38SMilan Broz return (void *)((char *)dmreq - cc->dmreq_start); 1013b2174eebSHuang Ying } 1014b2174eebSHuang Ying 10152dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, 10162dc5327dSMilan Broz struct dm_crypt_request *dmreq) 10172dc5327dSMilan Broz { 101833d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1019ef43aa38SMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 1020ef43aa38SMilan Broz crypto_aead_alignmask(any_tfm_aead(cc)) + 1); 1021ef43aa38SMilan Broz else 10222dc5327dSMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 1023bbdb23b5SHerbert Xu crypto_skcipher_alignmask(any_tfm(cc)) + 1); 10242dc5327dSMilan Broz } 10252dc5327dSMilan Broz 1026ef43aa38SMilan Broz static u8 *org_iv_of_dmreq(struct crypt_config *cc, 1027ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1028ef43aa38SMilan Broz { 1029ef43aa38SMilan Broz return iv_of_dmreq(cc, dmreq) + cc->iv_size; 1030ef43aa38SMilan Broz } 1031ef43aa38SMilan Broz 1032ef43aa38SMilan Broz static uint64_t *org_sector_of_dmreq(struct crypt_config *cc, 1033ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1034ef43aa38SMilan Broz { 1035ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size; 1036ef43aa38SMilan Broz return (uint64_t*) ptr; 1037ef43aa38SMilan Broz } 1038ef43aa38SMilan Broz 1039ef43aa38SMilan Broz static unsigned int *org_tag_of_dmreq(struct crypt_config *cc, 1040ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1041ef43aa38SMilan Broz { 1042ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + 1043ef43aa38SMilan Broz cc->iv_size + sizeof(uint64_t); 1044ef43aa38SMilan Broz return (unsigned int*)ptr; 1045ef43aa38SMilan Broz } 1046ef43aa38SMilan Broz 1047ef43aa38SMilan Broz static void *tag_from_dmreq(struct crypt_config *cc, 1048ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1049ef43aa38SMilan Broz { 1050ef43aa38SMilan Broz struct convert_context *ctx = dmreq->ctx; 1051ef43aa38SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 1052ef43aa38SMilan Broz 1053ef43aa38SMilan Broz return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) * 1054ef43aa38SMilan Broz cc->on_disk_tag_size]; 1055ef43aa38SMilan Broz } 1056ef43aa38SMilan Broz 1057ef43aa38SMilan Broz static void *iv_tag_from_dmreq(struct crypt_config *cc, 1058ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1059ef43aa38SMilan Broz { 1060ef43aa38SMilan Broz return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size; 1061ef43aa38SMilan Broz } 1062ef43aa38SMilan Broz 1063ef43aa38SMilan Broz static int crypt_convert_block_aead(struct crypt_config *cc, 10643a7f6c99SMilan Broz struct convert_context *ctx, 1065ef43aa38SMilan Broz struct aead_request *req, 1066ef43aa38SMilan Broz unsigned int tag_offset) 106701482b76SMilan Broz { 1068003b5c57SKent Overstreet struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in); 1069003b5c57SKent Overstreet struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out); 10703a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 1071ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv, *tag; 1072ef43aa38SMilan Broz uint64_t *sector; 1073ef43aa38SMilan Broz int r = 0; 1074ef43aa38SMilan Broz 1075ef43aa38SMilan Broz BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size); 107601482b76SMilan Broz 10778f0009a2SMilan Broz /* Reject unexpected unaligned bio. */ 10780440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1))) 10798f0009a2SMilan Broz return -EIO; 108001482b76SMilan Broz 1081b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 1082c66029f4SMikulas Patocka dmreq->iv_sector = ctx->cc_sector; 10838f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 1084ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift; 1085b2174eebSHuang Ying dmreq->ctx = ctx; 108601482b76SMilan Broz 1087ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset; 108801482b76SMilan Broz 1089ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq); 1090ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset); 1091ef43aa38SMilan Broz 1092ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq); 1093ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq); 1094ef43aa38SMilan Broz tag = tag_from_dmreq(cc, dmreq); 1095ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq); 1096ef43aa38SMilan Broz 1097ef43aa38SMilan Broz /* AEAD request: 1098ef43aa38SMilan Broz * |----- AAD -------|------ DATA -------|-- AUTH TAG --| 1099ef43aa38SMilan Broz * | (authenticated) | (auth+encryption) | | 1100ef43aa38SMilan Broz * | sector_LE | IV | sector in/out | tag in/out | 1101ef43aa38SMilan Broz */ 1102ef43aa38SMilan Broz sg_init_table(dmreq->sg_in, 4); 1103ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t)); 1104ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size); 11058f0009a2SMilan Broz sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset); 1106ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size); 1107ef43aa38SMilan Broz 1108ef43aa38SMilan Broz sg_init_table(dmreq->sg_out, 4); 1109ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t)); 1110ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size); 11118f0009a2SMilan Broz sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset); 1112ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size); 111301482b76SMilan Broz 11143a7f6c99SMilan Broz if (cc->iv_gen_ops) { 1115ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */ 1116ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) { 1117ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->iv_size); 1118ef43aa38SMilan Broz } else { 1119ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq); 11203a7f6c99SMilan Broz if (r < 0) 11213a7f6c99SMilan Broz return r; 1122ef43aa38SMilan Broz /* Store generated IV in integrity metadata */ 1123ef43aa38SMilan Broz if (cc->integrity_iv_size) 1124ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->iv_size); 1125ef43aa38SMilan Broz } 1126ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */ 1127ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size); 1128ef43aa38SMilan Broz } 1129ef43aa38SMilan Broz 1130ef43aa38SMilan Broz aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size); 1131ef43aa38SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) { 1132ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out, 11338f0009a2SMilan Broz cc->sector_size, iv); 1134ef43aa38SMilan Broz r = crypto_aead_encrypt(req); 1135ef43aa38SMilan Broz if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size) 1136ef43aa38SMilan Broz memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0, 1137ef43aa38SMilan Broz cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size)); 1138ef43aa38SMilan Broz } else { 1139ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out, 11408f0009a2SMilan Broz cc->sector_size + cc->integrity_tag_size, iv); 1141ef43aa38SMilan Broz r = crypto_aead_decrypt(req); 1142ef43aa38SMilan Broz } 1143ef43aa38SMilan Broz 1144ef43aa38SMilan Broz if (r == -EBADMSG) 1145ef43aa38SMilan Broz DMERR_LIMIT("INTEGRITY AEAD ERROR, sector %llu", 1146ef43aa38SMilan Broz (unsigned long long)le64_to_cpu(*sector)); 1147ef43aa38SMilan Broz 1148ef43aa38SMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 1149ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq); 1150ef43aa38SMilan Broz 11518f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size); 11528f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size); 1153ef43aa38SMilan Broz 1154ef43aa38SMilan Broz return r; 11553a7f6c99SMilan Broz } 11563a7f6c99SMilan Broz 1157ef43aa38SMilan Broz static int crypt_convert_block_skcipher(struct crypt_config *cc, 1158ef43aa38SMilan Broz struct convert_context *ctx, 1159ef43aa38SMilan Broz struct skcipher_request *req, 1160ef43aa38SMilan Broz unsigned int tag_offset) 1161ef43aa38SMilan Broz { 1162ef43aa38SMilan Broz struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in); 1163ef43aa38SMilan Broz struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out); 1164ef43aa38SMilan Broz struct scatterlist *sg_in, *sg_out; 1165ef43aa38SMilan Broz struct dm_crypt_request *dmreq; 1166ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv; 1167ef43aa38SMilan Broz uint64_t *sector; 1168ef43aa38SMilan Broz int r = 0; 1169ef43aa38SMilan Broz 11708f0009a2SMilan Broz /* Reject unexpected unaligned bio. */ 11710440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1))) 11728f0009a2SMilan Broz return -EIO; 11738f0009a2SMilan Broz 1174ef43aa38SMilan Broz dmreq = dmreq_of_req(cc, req); 1175ef43aa38SMilan Broz dmreq->iv_sector = ctx->cc_sector; 11768f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 1177ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift; 1178ef43aa38SMilan Broz dmreq->ctx = ctx; 1179ef43aa38SMilan Broz 1180ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset; 1181ef43aa38SMilan Broz 1182ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq); 1183ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq); 1184ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq); 1185ef43aa38SMilan Broz 1186ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq); 1187ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset); 1188ef43aa38SMilan Broz 1189ef43aa38SMilan Broz /* For skcipher we use only the first sg item */ 1190ef43aa38SMilan Broz sg_in = &dmreq->sg_in[0]; 1191ef43aa38SMilan Broz sg_out = &dmreq->sg_out[0]; 1192ef43aa38SMilan Broz 1193ef43aa38SMilan Broz sg_init_table(sg_in, 1); 11948f0009a2SMilan Broz sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset); 1195ef43aa38SMilan Broz 1196ef43aa38SMilan Broz sg_init_table(sg_out, 1); 11978f0009a2SMilan Broz sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset); 1198ef43aa38SMilan Broz 1199ef43aa38SMilan Broz if (cc->iv_gen_ops) { 1200ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */ 1201ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) { 1202ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->integrity_iv_size); 1203ef43aa38SMilan Broz } else { 1204ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq); 1205ef43aa38SMilan Broz if (r < 0) 1206ef43aa38SMilan Broz return r; 1207ef43aa38SMilan Broz /* Store generated IV in integrity metadata */ 1208ef43aa38SMilan Broz if (cc->integrity_iv_size) 1209ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->integrity_iv_size); 1210ef43aa38SMilan Broz } 1211ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */ 1212ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size); 1213ef43aa38SMilan Broz } 1214ef43aa38SMilan Broz 12158f0009a2SMilan Broz skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv); 12163a7f6c99SMilan Broz 12173a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 1218bbdb23b5SHerbert Xu r = crypto_skcipher_encrypt(req); 12193a7f6c99SMilan Broz else 1220bbdb23b5SHerbert Xu r = crypto_skcipher_decrypt(req); 12213a7f6c99SMilan Broz 12222dc5327dSMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 1223ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq); 1224ef43aa38SMilan Broz 12258f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size); 12268f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size); 12272dc5327dSMilan Broz 12283a7f6c99SMilan Broz return r; 122901482b76SMilan Broz } 123001482b76SMilan Broz 123195497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 123295497a96SMilan Broz int error); 1233c0297721SAndi Kleen 1234ef43aa38SMilan Broz static void crypt_alloc_req_skcipher(struct crypt_config *cc, 1235ddd42edfSMilan Broz struct convert_context *ctx) 1236ddd42edfSMilan Broz { 1237c66029f4SMikulas Patocka unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1); 1238c0297721SAndi Kleen 1239ef43aa38SMilan Broz if (!ctx->r.req) 1240ef43aa38SMilan Broz ctx->r.req = mempool_alloc(cc->req_pool, GFP_NOIO); 1241c0297721SAndi Kleen 1242ef43aa38SMilan Broz skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]); 124354cea3f6SMilan Broz 124454cea3f6SMilan Broz /* 124554cea3f6SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs 124654cea3f6SMilan Broz * requests if driver request queue is full. 124754cea3f6SMilan Broz */ 1248ef43aa38SMilan Broz skcipher_request_set_callback(ctx->r.req, 1249c0297721SAndi Kleen CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, 1250ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req)); 1251ddd42edfSMilan Broz } 1252ddd42edfSMilan Broz 1253ef43aa38SMilan Broz static void crypt_alloc_req_aead(struct crypt_config *cc, 1254ef43aa38SMilan Broz struct convert_context *ctx) 1255ef43aa38SMilan Broz { 1256ef43aa38SMilan Broz if (!ctx->r.req_aead) 1257ef43aa38SMilan Broz ctx->r.req_aead = mempool_alloc(cc->req_pool, GFP_NOIO); 1258ef43aa38SMilan Broz 1259ef43aa38SMilan Broz aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]); 1260ef43aa38SMilan Broz 1261ef43aa38SMilan Broz /* 1262ef43aa38SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs 1263ef43aa38SMilan Broz * requests if driver request queue is full. 1264ef43aa38SMilan Broz */ 1265ef43aa38SMilan Broz aead_request_set_callback(ctx->r.req_aead, 1266ef43aa38SMilan Broz CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, 1267ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead)); 1268ef43aa38SMilan Broz } 1269ef43aa38SMilan Broz 1270ef43aa38SMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 1271ef43aa38SMilan Broz struct convert_context *ctx) 1272ef43aa38SMilan Broz { 127333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1274ef43aa38SMilan Broz crypt_alloc_req_aead(cc, ctx); 1275ef43aa38SMilan Broz else 1276ef43aa38SMilan Broz crypt_alloc_req_skcipher(cc, ctx); 1277ef43aa38SMilan Broz } 1278ef43aa38SMilan Broz 1279ef43aa38SMilan Broz static void crypt_free_req_skcipher(struct crypt_config *cc, 1280bbdb23b5SHerbert Xu struct skcipher_request *req, struct bio *base_bio) 1281298a9fa0SMikulas Patocka { 1282298a9fa0SMikulas Patocka struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); 1283298a9fa0SMikulas Patocka 1284bbdb23b5SHerbert Xu if ((struct skcipher_request *)(io + 1) != req) 1285298a9fa0SMikulas Patocka mempool_free(req, cc->req_pool); 1286298a9fa0SMikulas Patocka } 1287298a9fa0SMikulas Patocka 1288ef43aa38SMilan Broz static void crypt_free_req_aead(struct crypt_config *cc, 1289ef43aa38SMilan Broz struct aead_request *req, struct bio *base_bio) 1290ef43aa38SMilan Broz { 1291ef43aa38SMilan Broz struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); 1292ef43aa38SMilan Broz 1293ef43aa38SMilan Broz if ((struct aead_request *)(io + 1) != req) 1294ef43aa38SMilan Broz mempool_free(req, cc->req_pool); 1295ef43aa38SMilan Broz } 1296ef43aa38SMilan Broz 1297ef43aa38SMilan Broz static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio) 1298ef43aa38SMilan Broz { 129933d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1300ef43aa38SMilan Broz crypt_free_req_aead(cc, req, base_bio); 1301ef43aa38SMilan Broz else 1302ef43aa38SMilan Broz crypt_free_req_skcipher(cc, req, base_bio); 1303ef43aa38SMilan Broz } 1304ef43aa38SMilan Broz 13051da177e4SLinus Torvalds /* 13061da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 13071da177e4SLinus Torvalds */ 13084e4cbee9SChristoph Hellwig static blk_status_t crypt_convert(struct crypt_config *cc, 13091da177e4SLinus Torvalds struct convert_context *ctx) 13101da177e4SLinus Torvalds { 1311ef43aa38SMilan Broz unsigned int tag_offset = 0; 1312ff3af92bSMikulas Patocka unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT; 13133f1e9070SMilan Broz int r; 13141da177e4SLinus Torvalds 131540b6229bSMikulas Patocka atomic_set(&ctx->cc_pending, 1); 1316c8081618SMilan Broz 1317003b5c57SKent Overstreet while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) { 13181da177e4SLinus Torvalds 13193a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 132040b6229bSMikulas Patocka atomic_inc(&ctx->cc_pending); 13213f1e9070SMilan Broz 132233d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1323ef43aa38SMilan Broz r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset); 1324ef43aa38SMilan Broz else 1325ef43aa38SMilan Broz r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset); 13263a7f6c99SMilan Broz 13273a7f6c99SMilan Broz switch (r) { 132854cea3f6SMilan Broz /* 132954cea3f6SMilan Broz * The request was queued by a crypto driver 133054cea3f6SMilan Broz * but the driver request queue is full, let's wait. 133154cea3f6SMilan Broz */ 13323a7f6c99SMilan Broz case -EBUSY: 13333a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 133416735d02SWolfram Sang reinit_completion(&ctx->restart); 1335c0403ec0SRabin Vincent /* fall through */ 133654cea3f6SMilan Broz /* 133754cea3f6SMilan Broz * The request is queued and processed asynchronously, 133854cea3f6SMilan Broz * completion function kcryptd_async_done() will be called. 133954cea3f6SMilan Broz */ 1340c0403ec0SRabin Vincent case -EINPROGRESS: 1341ef43aa38SMilan Broz ctx->r.req = NULL; 13428f0009a2SMilan Broz ctx->cc_sector += sector_step; 1343583fe747SMikulas Patocka tag_offset++; 13443a7f6c99SMilan Broz continue; 134554cea3f6SMilan Broz /* 134654cea3f6SMilan Broz * The request was already processed (synchronously). 134754cea3f6SMilan Broz */ 13483f1e9070SMilan Broz case 0: 134940b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 13508f0009a2SMilan Broz ctx->cc_sector += sector_step; 1351583fe747SMikulas Patocka tag_offset++; 1352c7f1b204SMilan Broz cond_resched(); 13533f1e9070SMilan Broz continue; 1354ef43aa38SMilan Broz /* 1355ef43aa38SMilan Broz * There was a data integrity error. 1356ef43aa38SMilan Broz */ 1357ef43aa38SMilan Broz case -EBADMSG: 1358ef43aa38SMilan Broz atomic_dec(&ctx->cc_pending); 13594e4cbee9SChristoph Hellwig return BLK_STS_PROTECTION; 1360ef43aa38SMilan Broz /* 1361ef43aa38SMilan Broz * There was an error while processing the request. 1362ef43aa38SMilan Broz */ 13633f1e9070SMilan Broz default: 136440b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 13654e4cbee9SChristoph Hellwig return BLK_STS_IOERR; 13661da177e4SLinus Torvalds } 13673f1e9070SMilan Broz } 13683f1e9070SMilan Broz 13693f1e9070SMilan Broz return 0; 13703f1e9070SMilan Broz } 13711da177e4SLinus Torvalds 1372cf2f1abfSMikulas Patocka static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone); 1373cf2f1abfSMikulas Patocka 13741da177e4SLinus Torvalds /* 13751da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 1376586b286bSMike Snitzer * This should never violate the device limitations (but only because 1377586b286bSMike Snitzer * max_segment_size is being constrained to PAGE_SIZE). 13787145c241SMikulas Patocka * 13797145c241SMikulas Patocka * This function may be called concurrently. If we allocate from the mempool 13807145c241SMikulas Patocka * concurrently, there is a possibility of deadlock. For example, if we have 13817145c241SMikulas Patocka * mempool of 256 pages, two processes, each wanting 256, pages allocate from 13827145c241SMikulas Patocka * the mempool concurrently, it may deadlock in a situation where both processes 13837145c241SMikulas Patocka * have allocated 128 pages and the mempool is exhausted. 13847145c241SMikulas Patocka * 13857145c241SMikulas Patocka * In order to avoid this scenario we allocate the pages under a mutex. 13867145c241SMikulas Patocka * 13877145c241SMikulas Patocka * In order to not degrade performance with excessive locking, we try 13887145c241SMikulas Patocka * non-blocking allocations without a mutex first but on failure we fallback 13897145c241SMikulas Patocka * to blocking allocations with a mutex. 13901da177e4SLinus Torvalds */ 1391cf2f1abfSMikulas Patocka static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) 13921da177e4SLinus Torvalds { 139349a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 13948b004457SMilan Broz struct bio *clone; 13951da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 13967145c241SMikulas Patocka gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; 13977145c241SMikulas Patocka unsigned i, len, remaining_size; 139891e10625SMilan Broz struct page *page; 13991da177e4SLinus Torvalds 14007145c241SMikulas Patocka retry: 1401d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 14027145c241SMikulas Patocka mutex_lock(&cc->bio_alloc_lock); 14037145c241SMikulas Patocka 14046a24c718SMilan Broz clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 14058b004457SMilan Broz if (!clone) 1406ef43aa38SMilan Broz goto out; 14071da177e4SLinus Torvalds 1408027581f3SOlaf Kirch clone_init(io, clone); 14096a24c718SMilan Broz 14107145c241SMikulas Patocka remaining_size = size; 14117145c241SMikulas Patocka 1412f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 141391e10625SMilan Broz page = mempool_alloc(cc->page_pool, gfp_mask); 14147145c241SMikulas Patocka if (!page) { 14157145c241SMikulas Patocka crypt_free_buffer_pages(cc, clone); 14167145c241SMikulas Patocka bio_put(clone); 1417d0164adcSMel Gorman gfp_mask |= __GFP_DIRECT_RECLAIM; 14187145c241SMikulas Patocka goto retry; 14197145c241SMikulas Patocka } 14201da177e4SLinus Torvalds 14217145c241SMikulas Patocka len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size; 14221da177e4SLinus Torvalds 14230dae7fe5SMing Lei bio_add_page(clone, page, len, 0); 142491e10625SMilan Broz 14257145c241SMikulas Patocka remaining_size -= len; 14261da177e4SLinus Torvalds } 14271da177e4SLinus Torvalds 1428ef43aa38SMilan Broz /* Allocate space for integrity tags */ 1429ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) { 1430ef43aa38SMilan Broz crypt_free_buffer_pages(cc, clone); 1431ef43aa38SMilan Broz bio_put(clone); 1432ef43aa38SMilan Broz clone = NULL; 1433ef43aa38SMilan Broz } 1434ef43aa38SMilan Broz out: 1435d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 14367145c241SMikulas Patocka mutex_unlock(&cc->bio_alloc_lock); 14377145c241SMikulas Patocka 14388b004457SMilan Broz return clone; 14391da177e4SLinus Torvalds } 14401da177e4SLinus Torvalds 1441644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 14421da177e4SLinus Torvalds { 1443644bd2f0SNeil Brown unsigned int i; 14441da177e4SLinus Torvalds struct bio_vec *bv; 14451da177e4SLinus Torvalds 1446cb34e057SKent Overstreet bio_for_each_segment_all(bv, clone, i) { 14471da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 14481da177e4SLinus Torvalds mempool_free(bv->bv_page, cc->page_pool); 14491da177e4SLinus Torvalds bv->bv_page = NULL; 14501da177e4SLinus Torvalds } 14511da177e4SLinus Torvalds } 14521da177e4SLinus Torvalds 1453298a9fa0SMikulas Patocka static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc, 1454dc440d1eSMilan Broz struct bio *bio, sector_t sector) 1455dc440d1eSMilan Broz { 145649a8a920SAlasdair G Kergon io->cc = cc; 1457dc440d1eSMilan Broz io->base_bio = bio; 1458dc440d1eSMilan Broz io->sector = sector; 1459dc440d1eSMilan Broz io->error = 0; 1460ef43aa38SMilan Broz io->ctx.r.req = NULL; 1461ef43aa38SMilan Broz io->integrity_metadata = NULL; 1462ef43aa38SMilan Broz io->integrity_metadata_from_pool = false; 146340b6229bSMikulas Patocka atomic_set(&io->io_pending, 0); 1464dc440d1eSMilan Broz } 1465dc440d1eSMilan Broz 14663e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 14673e1a8bddSMilan Broz { 146840b6229bSMikulas Patocka atomic_inc(&io->io_pending); 14693e1a8bddSMilan Broz } 14703e1a8bddSMilan Broz 14711da177e4SLinus Torvalds /* 14721da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 14731da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 14741da177e4SLinus Torvalds */ 14755742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 14761da177e4SLinus Torvalds { 147749a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1478b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 14794e4cbee9SChristoph Hellwig blk_status_t error = io->error; 14801da177e4SLinus Torvalds 148140b6229bSMikulas Patocka if (!atomic_dec_and_test(&io->io_pending)) 14821da177e4SLinus Torvalds return; 14831da177e4SLinus Torvalds 1484ef43aa38SMilan Broz if (io->ctx.r.req) 1485ef43aa38SMilan Broz crypt_free_req(cc, io->ctx.r.req, base_bio); 1486ef43aa38SMilan Broz 1487ef43aa38SMilan Broz if (unlikely(io->integrity_metadata_from_pool)) 1488ef43aa38SMilan Broz mempool_free(io->integrity_metadata, io->cc->tag_pool); 1489ef43aa38SMilan Broz else 1490ef43aa38SMilan Broz kfree(io->integrity_metadata); 1491b35f8caaSMilan Broz 14924e4cbee9SChristoph Hellwig base_bio->bi_status = error; 14934246a0b6SChristoph Hellwig bio_endio(base_bio); 14941da177e4SLinus Torvalds } 14951da177e4SLinus Torvalds 14961da177e4SLinus Torvalds /* 1497cabf08e4SMilan Broz * kcryptd/kcryptd_io: 14981da177e4SLinus Torvalds * 14991da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 150023541d2dSMilan Broz * interrupt context. 1501cabf08e4SMilan Broz * 1502cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 1503cabf08e4SMilan Broz * 1504cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 1505cabf08e4SMilan Broz * 1506cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 1507cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 1508cabf08e4SMilan Broz * to memory allocation. 1509c0297721SAndi Kleen * 1510c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances. 1511c0297721SAndi Kleen * They should not depend on each other and do not block. 15121da177e4SLinus Torvalds */ 15134246a0b6SChristoph Hellwig static void crypt_endio(struct bio *clone) 15148b004457SMilan Broz { 1515028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 151649a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1517ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 15184e4cbee9SChristoph Hellwig blk_status_t error; 15198b004457SMilan Broz 15208b004457SMilan Broz /* 15216712ecf8SNeilBrown * free the processed pages 15228b004457SMilan Broz */ 1523ee7a491eSMilan Broz if (rw == WRITE) 1524644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 15258b004457SMilan Broz 15264e4cbee9SChristoph Hellwig error = clone->bi_status; 15278b004457SMilan Broz bio_put(clone); 1528ee7a491eSMilan Broz 15299b81c842SSasha Levin if (rw == READ && !error) { 1530cabf08e4SMilan Broz kcryptd_queue_crypt(io); 15316712ecf8SNeilBrown return; 1532ee7a491eSMilan Broz } 15335742fd77SMilan Broz 15349b81c842SSasha Levin if (unlikely(error)) 15359b81c842SSasha Levin io->error = error; 15365742fd77SMilan Broz 15375742fd77SMilan Broz crypt_dec_pending(io); 15388b004457SMilan Broz } 15398b004457SMilan Broz 1540028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 15418b004457SMilan Broz { 154249a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 15438b004457SMilan Broz 15448b004457SMilan Broz clone->bi_private = io; 15458b004457SMilan Broz clone->bi_end_io = crypt_endio; 154674d46992SChristoph Hellwig bio_set_dev(clone, cc->dev->bdev); 1547ef295ecfSChristoph Hellwig clone->bi_opf = io->base_bio->bi_opf; 15488b004457SMilan Broz } 15498b004457SMilan Broz 155020c82538SMilan Broz static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) 15518b004457SMilan Broz { 155249a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 15538b004457SMilan Broz struct bio *clone; 155493e605c2SMilan Broz 15558b004457SMilan Broz /* 155659779079SMike Snitzer * We need the original biovec array in order to decrypt 155759779079SMike Snitzer * the whole bio data *afterwards* -- thanks to immutable 155859779079SMike Snitzer * biovecs we don't need to worry about the block layer 155959779079SMike Snitzer * modifying the biovec array; so leverage bio_clone_fast(). 15608b004457SMilan Broz */ 156159779079SMike Snitzer clone = bio_clone_fast(io->base_bio, gfp, cc->bs); 15627eaceaccSJens Axboe if (!clone) 156320c82538SMilan Broz return 1; 15648b004457SMilan Broz 156520c82538SMilan Broz crypt_inc_pending(io); 156620c82538SMilan Broz 15678b004457SMilan Broz clone_init(io, clone); 15684f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 15698b004457SMilan Broz 1570ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) { 1571ef43aa38SMilan Broz crypt_dec_pending(io); 1572ef43aa38SMilan Broz bio_put(clone); 1573ef43aa38SMilan Broz return 1; 1574ef43aa38SMilan Broz } 1575ef43aa38SMilan Broz 157693e605c2SMilan Broz generic_make_request(clone); 157720c82538SMilan Broz return 0; 15788b004457SMilan Broz } 15798b004457SMilan Broz 1580dc267621SMikulas Patocka static void kcryptd_io_read_work(struct work_struct *work) 1581395b167cSAlasdair G Kergon { 1582395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 1583395b167cSAlasdair G Kergon 158420c82538SMilan Broz crypt_inc_pending(io); 158520c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOIO)) 15864e4cbee9SChristoph Hellwig io->error = BLK_STS_RESOURCE; 158720c82538SMilan Broz crypt_dec_pending(io); 1588395b167cSAlasdair G Kergon } 1589395b167cSAlasdair G Kergon 1590dc267621SMikulas Patocka static void kcryptd_queue_read(struct dm_crypt_io *io) 1591395b167cSAlasdair G Kergon { 159249a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1593395b167cSAlasdair G Kergon 1594dc267621SMikulas Patocka INIT_WORK(&io->work, kcryptd_io_read_work); 1595395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 1596395b167cSAlasdair G Kergon } 1597395b167cSAlasdair G Kergon 1598dc267621SMikulas Patocka static void kcryptd_io_write(struct dm_crypt_io *io) 1599dc267621SMikulas Patocka { 1600dc267621SMikulas Patocka struct bio *clone = io->ctx.bio_out; 1601dc267621SMikulas Patocka 1602dc267621SMikulas Patocka generic_make_request(clone); 1603dc267621SMikulas Patocka } 1604dc267621SMikulas Patocka 1605b3c5fd30SMikulas Patocka #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node) 1606b3c5fd30SMikulas Patocka 1607dc267621SMikulas Patocka static int dmcrypt_write(void *data) 1608dc267621SMikulas Patocka { 1609dc267621SMikulas Patocka struct crypt_config *cc = data; 1610b3c5fd30SMikulas Patocka struct dm_crypt_io *io; 1611b3c5fd30SMikulas Patocka 1612dc267621SMikulas Patocka while (1) { 1613b3c5fd30SMikulas Patocka struct rb_root write_tree; 1614dc267621SMikulas Patocka struct blk_plug plug; 1615dc267621SMikulas Patocka 1616dc267621SMikulas Patocka DECLARE_WAITQUEUE(wait, current); 1617dc267621SMikulas Patocka 1618dc267621SMikulas Patocka spin_lock_irq(&cc->write_thread_wait.lock); 1619dc267621SMikulas Patocka continue_locked: 1620dc267621SMikulas Patocka 1621b3c5fd30SMikulas Patocka if (!RB_EMPTY_ROOT(&cc->write_tree)) 1622dc267621SMikulas Patocka goto pop_from_list; 1623dc267621SMikulas Patocka 1624f659b100SRabin Vincent set_current_state(TASK_INTERRUPTIBLE); 1625dc267621SMikulas Patocka __add_wait_queue(&cc->write_thread_wait, &wait); 1626dc267621SMikulas Patocka 1627dc267621SMikulas Patocka spin_unlock_irq(&cc->write_thread_wait.lock); 1628dc267621SMikulas Patocka 1629f659b100SRabin Vincent if (unlikely(kthread_should_stop())) { 1630642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING); 1631f659b100SRabin Vincent remove_wait_queue(&cc->write_thread_wait, &wait); 1632f659b100SRabin Vincent break; 1633f659b100SRabin Vincent } 1634f659b100SRabin Vincent 1635dc267621SMikulas Patocka schedule(); 1636dc267621SMikulas Patocka 1637642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING); 1638dc267621SMikulas Patocka spin_lock_irq(&cc->write_thread_wait.lock); 1639dc267621SMikulas Patocka __remove_wait_queue(&cc->write_thread_wait, &wait); 1640dc267621SMikulas Patocka goto continue_locked; 1641dc267621SMikulas Patocka 1642dc267621SMikulas Patocka pop_from_list: 1643b3c5fd30SMikulas Patocka write_tree = cc->write_tree; 1644b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 1645dc267621SMikulas Patocka spin_unlock_irq(&cc->write_thread_wait.lock); 1646dc267621SMikulas Patocka 1647b3c5fd30SMikulas Patocka BUG_ON(rb_parent(write_tree.rb_node)); 1648b3c5fd30SMikulas Patocka 1649b3c5fd30SMikulas Patocka /* 1650b3c5fd30SMikulas Patocka * Note: we cannot walk the tree here with rb_next because 1651b3c5fd30SMikulas Patocka * the structures may be freed when kcryptd_io_write is called. 1652b3c5fd30SMikulas Patocka */ 1653dc267621SMikulas Patocka blk_start_plug(&plug); 1654dc267621SMikulas Patocka do { 1655b3c5fd30SMikulas Patocka io = crypt_io_from_node(rb_first(&write_tree)); 1656b3c5fd30SMikulas Patocka rb_erase(&io->rb_node, &write_tree); 1657dc267621SMikulas Patocka kcryptd_io_write(io); 1658b3c5fd30SMikulas Patocka } while (!RB_EMPTY_ROOT(&write_tree)); 1659dc267621SMikulas Patocka blk_finish_plug(&plug); 1660dc267621SMikulas Patocka } 1661dc267621SMikulas Patocka return 0; 1662dc267621SMikulas Patocka } 1663dc267621SMikulas Patocka 166472c6e7afSMikulas Patocka static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) 16654e4eef64SMilan Broz { 1666dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 166749a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1668dc267621SMikulas Patocka unsigned long flags; 1669b3c5fd30SMikulas Patocka sector_t sector; 1670b3c5fd30SMikulas Patocka struct rb_node **rbp, *parent; 1671dec1cedfSMilan Broz 16724e4cbee9SChristoph Hellwig if (unlikely(io->error)) { 1673dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 1674dec1cedfSMilan Broz bio_put(clone); 16756c031f41SMilan Broz crypt_dec_pending(io); 1676dec1cedfSMilan Broz return; 1677dec1cedfSMilan Broz } 1678dec1cedfSMilan Broz 1679dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 1680003b5c57SKent Overstreet BUG_ON(io->ctx.iter_out.bi_size); 1681dec1cedfSMilan Broz 16824f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 1683899c95d3SMilan Broz 16840f5d8e6eSMikulas Patocka if (likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) { 16850f5d8e6eSMikulas Patocka generic_make_request(clone); 16860f5d8e6eSMikulas Patocka return; 16870f5d8e6eSMikulas Patocka } 16880f5d8e6eSMikulas Patocka 1689dc267621SMikulas Patocka spin_lock_irqsave(&cc->write_thread_wait.lock, flags); 1690b3c5fd30SMikulas Patocka rbp = &cc->write_tree.rb_node; 1691b3c5fd30SMikulas Patocka parent = NULL; 1692b3c5fd30SMikulas Patocka sector = io->sector; 1693b3c5fd30SMikulas Patocka while (*rbp) { 1694b3c5fd30SMikulas Patocka parent = *rbp; 1695b3c5fd30SMikulas Patocka if (sector < crypt_io_from_node(parent)->sector) 1696b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_left; 1697b3c5fd30SMikulas Patocka else 1698b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_right; 1699b3c5fd30SMikulas Patocka } 1700b3c5fd30SMikulas Patocka rb_link_node(&io->rb_node, parent, rbp); 1701b3c5fd30SMikulas Patocka rb_insert_color(&io->rb_node, &cc->write_tree); 1702b3c5fd30SMikulas Patocka 1703dc267621SMikulas Patocka wake_up_locked(&cc->write_thread_wait); 1704dc267621SMikulas Patocka spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags); 17054e4eef64SMilan Broz } 17064e4eef64SMilan Broz 1707fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 17088b004457SMilan Broz { 170949a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 17108b004457SMilan Broz struct bio *clone; 1711c8081618SMilan Broz int crypt_finished; 1712b635b00eSMilan Broz sector_t sector = io->sector; 17134e4cbee9SChristoph Hellwig blk_status_t r; 17148b004457SMilan Broz 171593e605c2SMilan Broz /* 1716fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 1717fc5a5e9aSMilan Broz */ 1718fc5a5e9aSMilan Broz crypt_inc_pending(io); 1719b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 1720fc5a5e9aSMilan Broz 1721cf2f1abfSMikulas Patocka clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size); 172223541d2dSMilan Broz if (unlikely(!clone)) { 17234e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR; 1724cf2f1abfSMikulas Patocka goto dec; 172523541d2dSMilan Broz } 17268b004457SMilan Broz 172753017030SMilan Broz io->ctx.bio_out = clone; 1728003b5c57SKent Overstreet io->ctx.iter_out = clone->bi_iter; 17298b004457SMilan Broz 1730b635b00eSMilan Broz sector += bio_sectors(clone); 1731dec1cedfSMilan Broz 17324e594098SMilan Broz crypt_inc_pending(io); 1733dec1cedfSMilan Broz r = crypt_convert(cc, &io->ctx); 17344e4cbee9SChristoph Hellwig if (r) 1735ef43aa38SMilan Broz io->error = r; 173640b6229bSMikulas Patocka crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); 1737dec1cedfSMilan Broz 1738c8081618SMilan Broz /* Encryption was already finished, submit io now */ 1739c8081618SMilan Broz if (crypt_finished) { 174072c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 0); 1741b635b00eSMilan Broz io->sector = sector; 17424e594098SMilan Broz } 174393e605c2SMilan Broz 1744cf2f1abfSMikulas Patocka dec: 1745899c95d3SMilan Broz crypt_dec_pending(io); 174684131db6SMilan Broz } 174784131db6SMilan Broz 174872c6e7afSMikulas Patocka static void kcryptd_crypt_read_done(struct dm_crypt_io *io) 17495742fd77SMilan Broz { 17505742fd77SMilan Broz crypt_dec_pending(io); 17515742fd77SMilan Broz } 17525742fd77SMilan Broz 17534e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 17548b004457SMilan Broz { 175549a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 17564e4cbee9SChristoph Hellwig blk_status_t r; 17578b004457SMilan Broz 17583e1a8bddSMilan Broz crypt_inc_pending(io); 17593a7f6c99SMilan Broz 176053017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 17610c395b0fSMilan Broz io->sector); 17628b004457SMilan Broz 17635742fd77SMilan Broz r = crypt_convert(cc, &io->ctx); 17644e4cbee9SChristoph Hellwig if (r) 1765ef43aa38SMilan Broz io->error = r; 17665742fd77SMilan Broz 176740b6229bSMikulas Patocka if (atomic_dec_and_test(&io->ctx.cc_pending)) 176872c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 17693a7f6c99SMilan Broz 17703a7f6c99SMilan Broz crypt_dec_pending(io); 17718b004457SMilan Broz } 17728b004457SMilan Broz 177395497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 177495497a96SMilan Broz int error) 177595497a96SMilan Broz { 1776b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 1777b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 177895497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 177949a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 178095497a96SMilan Broz 178154cea3f6SMilan Broz /* 178254cea3f6SMilan Broz * A request from crypto driver backlog is going to be processed now, 178354cea3f6SMilan Broz * finish the completion and continue in crypt_convert(). 178454cea3f6SMilan Broz * (Callback will be called for the second time for this request.) 178554cea3f6SMilan Broz */ 1786c0403ec0SRabin Vincent if (error == -EINPROGRESS) { 1787c0403ec0SRabin Vincent complete(&ctx->restart); 178895497a96SMilan Broz return; 1789c0403ec0SRabin Vincent } 179095497a96SMilan Broz 17912dc5327dSMilan Broz if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post) 1792ef43aa38SMilan Broz error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq); 17932dc5327dSMilan Broz 1794ef43aa38SMilan Broz if (error == -EBADMSG) { 1795ef43aa38SMilan Broz DMERR_LIMIT("INTEGRITY AEAD ERROR, sector %llu", 1796ef43aa38SMilan Broz (unsigned long long)le64_to_cpu(*org_sector_of_dmreq(cc, dmreq))); 17974e4cbee9SChristoph Hellwig io->error = BLK_STS_PROTECTION; 1798ef43aa38SMilan Broz } else if (error < 0) 17994e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR; 180072c6e7afSMikulas Patocka 1801298a9fa0SMikulas Patocka crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio); 180295497a96SMilan Broz 180340b6229bSMikulas Patocka if (!atomic_dec_and_test(&ctx->cc_pending)) 1804c0403ec0SRabin Vincent return; 180595497a96SMilan Broz 180695497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 180772c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 180895497a96SMilan Broz else 180972c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 1); 181095497a96SMilan Broz } 181195497a96SMilan Broz 18124e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 18134e4eef64SMilan Broz { 18144e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 18154e4eef64SMilan Broz 18164e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 18174e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 18184e4eef64SMilan Broz else 18194e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 18208b004457SMilan Broz } 18218b004457SMilan Broz 1822395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 1823395b167cSAlasdair G Kergon { 182449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1825395b167cSAlasdair G Kergon 1826395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 1827395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 1828395b167cSAlasdair G Kergon } 1829395b167cSAlasdair G Kergon 1830ef43aa38SMilan Broz static void crypt_free_tfms_aead(struct crypt_config *cc) 18311da177e4SLinus Torvalds { 1832ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms_aead) 1833ef43aa38SMilan Broz return; 18341da177e4SLinus Torvalds 1835ef43aa38SMilan Broz if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) { 1836ef43aa38SMilan Broz crypto_free_aead(cc->cipher_tfm.tfms_aead[0]); 1837ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead[0] = NULL; 18381da177e4SLinus Torvalds } 18391da177e4SLinus Torvalds 1840ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms_aead); 1841ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead = NULL; 1842ef43aa38SMilan Broz } 18431da177e4SLinus Torvalds 1844ef43aa38SMilan Broz static void crypt_free_tfms_skcipher(struct crypt_config *cc) 1845d1f96423SMilan Broz { 1846d1f96423SMilan Broz unsigned i; 1847d1f96423SMilan Broz 1848ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 1849fd2d231fSMikulas Patocka return; 1850fd2d231fSMikulas Patocka 1851d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) 1852ef43aa38SMilan Broz if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) { 1853ef43aa38SMilan Broz crypto_free_skcipher(cc->cipher_tfm.tfms[i]); 1854ef43aa38SMilan Broz cc->cipher_tfm.tfms[i] = NULL; 1855d1f96423SMilan Broz } 1856d1f96423SMilan Broz 1857ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms); 1858ef43aa38SMilan Broz cc->cipher_tfm.tfms = NULL; 18591da177e4SLinus Torvalds } 18601da177e4SLinus Torvalds 18611da177e4SLinus Torvalds static void crypt_free_tfms(struct crypt_config *cc) 1862d1f96423SMilan Broz { 186333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1864ef43aa38SMilan Broz crypt_free_tfms_aead(cc); 1865ef43aa38SMilan Broz else 1866ef43aa38SMilan Broz crypt_free_tfms_skcipher(cc); 1867d1f96423SMilan Broz } 1868d1f96423SMilan Broz 1869ef43aa38SMilan Broz static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode) 1870d1f96423SMilan Broz { 1871d1f96423SMilan Broz unsigned i; 1872d1f96423SMilan Broz int err; 1873d1f96423SMilan Broz 1874ef43aa38SMilan Broz cc->cipher_tfm.tfms = kzalloc(cc->tfms_count * 1875ef43aa38SMilan Broz sizeof(struct crypto_skcipher *), GFP_KERNEL); 1876ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 1877fd2d231fSMikulas Patocka return -ENOMEM; 1878fd2d231fSMikulas Patocka 1879d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 1880ef43aa38SMilan Broz cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0, 0); 1881ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms[i])) { 1882ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms[i]); 1883fd2d231fSMikulas Patocka crypt_free_tfms(cc); 1884d1f96423SMilan Broz return err; 1885d1f96423SMilan Broz } 1886d1f96423SMilan Broz } 1887d1f96423SMilan Broz 1888d1f96423SMilan Broz return 0; 1889d1f96423SMilan Broz } 1890d1f96423SMilan Broz 1891ef43aa38SMilan Broz static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode) 1892ef43aa38SMilan Broz { 1893ef43aa38SMilan Broz int err; 1894ef43aa38SMilan Broz 1895ef43aa38SMilan Broz cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL); 1896ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 1897ef43aa38SMilan Broz return -ENOMEM; 1898ef43aa38SMilan Broz 1899ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0, 0); 1900ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) { 1901ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]); 1902ef43aa38SMilan Broz crypt_free_tfms(cc); 1903ef43aa38SMilan Broz return err; 1904ef43aa38SMilan Broz } 1905ef43aa38SMilan Broz 1906ef43aa38SMilan Broz return 0; 1907ef43aa38SMilan Broz } 1908ef43aa38SMilan Broz 1909ef43aa38SMilan Broz static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode) 1910ef43aa38SMilan Broz { 191133d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1912ef43aa38SMilan Broz return crypt_alloc_tfms_aead(cc, ciphermode); 1913ef43aa38SMilan Broz else 1914ef43aa38SMilan Broz return crypt_alloc_tfms_skcipher(cc, ciphermode); 1915ef43aa38SMilan Broz } 1916ef43aa38SMilan Broz 1917ef43aa38SMilan Broz static unsigned crypt_subkey_size(struct crypt_config *cc) 1918ef43aa38SMilan Broz { 1919ef43aa38SMilan Broz return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count); 1920ef43aa38SMilan Broz } 1921ef43aa38SMilan Broz 1922ef43aa38SMilan Broz static unsigned crypt_authenckey_size(struct crypt_config *cc) 1923ef43aa38SMilan Broz { 1924ef43aa38SMilan Broz return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param)); 1925ef43aa38SMilan Broz } 1926ef43aa38SMilan Broz 1927ef43aa38SMilan Broz /* 1928ef43aa38SMilan Broz * If AEAD is composed like authenc(hmac(sha256),xts(aes)), 1929ef43aa38SMilan Broz * the key must be for some reason in special format. 1930ef43aa38SMilan Broz * This funcion converts cc->key to this special format. 1931ef43aa38SMilan Broz */ 1932ef43aa38SMilan Broz static void crypt_copy_authenckey(char *p, const void *key, 1933ef43aa38SMilan Broz unsigned enckeylen, unsigned authkeylen) 1934ef43aa38SMilan Broz { 1935ef43aa38SMilan Broz struct crypto_authenc_key_param *param; 1936ef43aa38SMilan Broz struct rtattr *rta; 1937ef43aa38SMilan Broz 1938ef43aa38SMilan Broz rta = (struct rtattr *)p; 1939ef43aa38SMilan Broz param = RTA_DATA(rta); 1940ef43aa38SMilan Broz param->enckeylen = cpu_to_be32(enckeylen); 1941ef43aa38SMilan Broz rta->rta_len = RTA_LENGTH(sizeof(*param)); 1942ef43aa38SMilan Broz rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; 1943ef43aa38SMilan Broz p += RTA_SPACE(sizeof(*param)); 1944ef43aa38SMilan Broz memcpy(p, key + enckeylen, authkeylen); 1945ef43aa38SMilan Broz p += authkeylen; 1946ef43aa38SMilan Broz memcpy(p, key, enckeylen); 1947ef43aa38SMilan Broz } 1948ef43aa38SMilan Broz 1949671ea6b4SMikulas Patocka static int crypt_setkey(struct crypt_config *cc) 1950c0297721SAndi Kleen { 1951da31a078SMilan Broz unsigned subkey_size; 1952fd2d231fSMikulas Patocka int err = 0, i, r; 1953c0297721SAndi Kleen 1954da31a078SMilan Broz /* Ignore extra keys (which are used for IV etc) */ 1955ef43aa38SMilan Broz subkey_size = crypt_subkey_size(cc); 1956da31a078SMilan Broz 1957ef43aa38SMilan Broz if (crypt_integrity_hmac(cc)) 1958ef43aa38SMilan Broz crypt_copy_authenckey(cc->authenc_key, cc->key, 1959ef43aa38SMilan Broz subkey_size - cc->key_mac_size, 1960ef43aa38SMilan Broz cc->key_mac_size); 1961d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 196233d2f09fSMilan Broz if (crypt_integrity_hmac(cc)) 196333d2f09fSMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i], 196433d2f09fSMilan Broz cc->authenc_key, crypt_authenckey_size(cc)); 196533d2f09fSMilan Broz else if (crypt_integrity_aead(cc)) 1966ef43aa38SMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i], 1967ef43aa38SMilan Broz cc->key + (i * subkey_size), 1968ef43aa38SMilan Broz subkey_size); 1969ef43aa38SMilan Broz else 1970ef43aa38SMilan Broz r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i], 1971fd2d231fSMikulas Patocka cc->key + (i * subkey_size), 1972fd2d231fSMikulas Patocka subkey_size); 1973c0297721SAndi Kleen if (r) 1974c0297721SAndi Kleen err = r; 1975c0297721SAndi Kleen } 1976c0297721SAndi Kleen 1977ef43aa38SMilan Broz if (crypt_integrity_hmac(cc)) 1978ef43aa38SMilan Broz memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc)); 1979ef43aa38SMilan Broz 1980c0297721SAndi Kleen return err; 1981c0297721SAndi Kleen } 1982c0297721SAndi Kleen 1983c538f6ecSOndrej Kozina #ifdef CONFIG_KEYS 1984c538f6ecSOndrej Kozina 1985027c431cSOndrej Kozina static bool contains_whitespace(const char *str) 1986027c431cSOndrej Kozina { 1987027c431cSOndrej Kozina while (*str) 1988027c431cSOndrej Kozina if (isspace(*str++)) 1989027c431cSOndrej Kozina return true; 1990027c431cSOndrej Kozina return false; 1991027c431cSOndrej Kozina } 1992027c431cSOndrej Kozina 1993c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string) 1994c538f6ecSOndrej Kozina { 1995c538f6ecSOndrej Kozina char *new_key_string, *key_desc; 1996c538f6ecSOndrej Kozina int ret; 1997c538f6ecSOndrej Kozina struct key *key; 1998c538f6ecSOndrej Kozina const struct user_key_payload *ukp; 1999c538f6ecSOndrej Kozina 2000027c431cSOndrej Kozina /* 2001027c431cSOndrej Kozina * Reject key_string with whitespace. dm core currently lacks code for 2002027c431cSOndrej Kozina * proper whitespace escaping in arguments on DM_TABLE_STATUS path. 2003027c431cSOndrej Kozina */ 2004027c431cSOndrej Kozina if (contains_whitespace(key_string)) { 2005027c431cSOndrej Kozina DMERR("whitespace chars not allowed in key string"); 2006027c431cSOndrej Kozina return -EINVAL; 2007027c431cSOndrej Kozina } 2008027c431cSOndrej Kozina 2009c538f6ecSOndrej Kozina /* look for next ':' separating key_type from key_description */ 2010c538f6ecSOndrej Kozina key_desc = strpbrk(key_string, ":"); 2011c538f6ecSOndrej Kozina if (!key_desc || key_desc == key_string || !strlen(key_desc + 1)) 2012c538f6ecSOndrej Kozina return -EINVAL; 2013c538f6ecSOndrej Kozina 2014c538f6ecSOndrej Kozina if (strncmp(key_string, "logon:", key_desc - key_string + 1) && 2015c538f6ecSOndrej Kozina strncmp(key_string, "user:", key_desc - key_string + 1)) 2016c538f6ecSOndrej Kozina return -EINVAL; 2017c538f6ecSOndrej Kozina 2018c538f6ecSOndrej Kozina new_key_string = kstrdup(key_string, GFP_KERNEL); 2019c538f6ecSOndrej Kozina if (!new_key_string) 2020c538f6ecSOndrej Kozina return -ENOMEM; 2021c538f6ecSOndrej Kozina 2022c538f6ecSOndrej Kozina key = request_key(key_string[0] == 'l' ? &key_type_logon : &key_type_user, 2023c538f6ecSOndrej Kozina key_desc + 1, NULL); 2024c538f6ecSOndrej Kozina if (IS_ERR(key)) { 2025c538f6ecSOndrej Kozina kzfree(new_key_string); 2026c538f6ecSOndrej Kozina return PTR_ERR(key); 2027c538f6ecSOndrej Kozina } 2028c538f6ecSOndrej Kozina 2029f5b0cba8SOndrej Kozina down_read(&key->sem); 2030c538f6ecSOndrej Kozina 20310837e49aSDavid Howells ukp = user_key_payload_locked(key); 2032c538f6ecSOndrej Kozina if (!ukp) { 2033f5b0cba8SOndrej Kozina up_read(&key->sem); 2034c538f6ecSOndrej Kozina key_put(key); 2035c538f6ecSOndrej Kozina kzfree(new_key_string); 2036c538f6ecSOndrej Kozina return -EKEYREVOKED; 2037c538f6ecSOndrej Kozina } 2038c538f6ecSOndrej Kozina 2039c538f6ecSOndrej Kozina if (cc->key_size != ukp->datalen) { 2040f5b0cba8SOndrej Kozina up_read(&key->sem); 2041c538f6ecSOndrej Kozina key_put(key); 2042c538f6ecSOndrej Kozina kzfree(new_key_string); 2043c538f6ecSOndrej Kozina return -EINVAL; 2044c538f6ecSOndrej Kozina } 2045c538f6ecSOndrej Kozina 2046c538f6ecSOndrej Kozina memcpy(cc->key, ukp->data, cc->key_size); 2047c538f6ecSOndrej Kozina 2048f5b0cba8SOndrej Kozina up_read(&key->sem); 2049c538f6ecSOndrej Kozina key_put(key); 2050c538f6ecSOndrej Kozina 2051c538f6ecSOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */ 2052c538f6ecSOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2053c538f6ecSOndrej Kozina 2054c538f6ecSOndrej Kozina ret = crypt_setkey(cc); 2055c538f6ecSOndrej Kozina 2056c538f6ecSOndrej Kozina /* wipe the kernel key payload copy in each case */ 2057c538f6ecSOndrej Kozina memset(cc->key, 0, cc->key_size * sizeof(u8)); 2058c538f6ecSOndrej Kozina 2059c538f6ecSOndrej Kozina if (!ret) { 2060c538f6ecSOndrej Kozina set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2061c538f6ecSOndrej Kozina kzfree(cc->key_string); 2062c538f6ecSOndrej Kozina cc->key_string = new_key_string; 2063c538f6ecSOndrej Kozina } else 2064c538f6ecSOndrej Kozina kzfree(new_key_string); 2065c538f6ecSOndrej Kozina 2066c538f6ecSOndrej Kozina return ret; 2067c538f6ecSOndrej Kozina } 2068c538f6ecSOndrej Kozina 2069c538f6ecSOndrej Kozina static int get_key_size(char **key_string) 2070c538f6ecSOndrej Kozina { 2071c538f6ecSOndrej Kozina char *colon, dummy; 2072c538f6ecSOndrej Kozina int ret; 2073c538f6ecSOndrej Kozina 2074c538f6ecSOndrej Kozina if (*key_string[0] != ':') 2075c538f6ecSOndrej Kozina return strlen(*key_string) >> 1; 2076c538f6ecSOndrej Kozina 2077c538f6ecSOndrej Kozina /* look for next ':' in key string */ 2078c538f6ecSOndrej Kozina colon = strpbrk(*key_string + 1, ":"); 2079c538f6ecSOndrej Kozina if (!colon) 2080c538f6ecSOndrej Kozina return -EINVAL; 2081c538f6ecSOndrej Kozina 2082c538f6ecSOndrej Kozina if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':') 2083c538f6ecSOndrej Kozina return -EINVAL; 2084c538f6ecSOndrej Kozina 2085c538f6ecSOndrej Kozina *key_string = colon; 2086c538f6ecSOndrej Kozina 2087c538f6ecSOndrej Kozina /* remaining key string should be :<logon|user>:<key_desc> */ 2088c538f6ecSOndrej Kozina 2089c538f6ecSOndrej Kozina return ret; 2090c538f6ecSOndrej Kozina } 2091c538f6ecSOndrej Kozina 2092c538f6ecSOndrej Kozina #else 2093c538f6ecSOndrej Kozina 2094c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string) 2095c538f6ecSOndrej Kozina { 2096c538f6ecSOndrej Kozina return -EINVAL; 2097c538f6ecSOndrej Kozina } 2098c538f6ecSOndrej Kozina 2099c538f6ecSOndrej Kozina static int get_key_size(char **key_string) 2100c538f6ecSOndrej Kozina { 2101c538f6ecSOndrej Kozina return (*key_string[0] == ':') ? -EINVAL : strlen(*key_string) >> 1; 2102c538f6ecSOndrej Kozina } 2103c538f6ecSOndrej Kozina 2104c538f6ecSOndrej Kozina #endif 2105c538f6ecSOndrej Kozina 2106e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 2107e48d4bbfSMilan Broz { 2108de8be5acSMilan Broz int r = -EINVAL; 2109de8be5acSMilan Broz int key_string_len = strlen(key); 2110de8be5acSMilan Broz 211169a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */ 211269a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-")) 2113de8be5acSMilan Broz goto out; 2114e48d4bbfSMilan Broz 2115c538f6ecSOndrej Kozina /* ':' means the key is in kernel keyring, short-circuit normal key processing */ 2116c538f6ecSOndrej Kozina if (key[0] == ':') { 2117c538f6ecSOndrej Kozina r = crypt_set_keyring_key(cc, key + 1); 2118c538f6ecSOndrej Kozina goto out; 2119c538f6ecSOndrej Kozina } 2120c538f6ecSOndrej Kozina 2121265e9098SOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */ 2122265e9098SOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2123265e9098SOndrej Kozina 2124c538f6ecSOndrej Kozina /* wipe references to any kernel keyring key */ 2125c538f6ecSOndrej Kozina kzfree(cc->key_string); 2126c538f6ecSOndrej Kozina cc->key_string = NULL; 2127c538f6ecSOndrej Kozina 2128e944e03eSAndy Shevchenko /* Decode key from its hex representation. */ 2129e944e03eSAndy Shevchenko if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0) 2130de8be5acSMilan Broz goto out; 2131e48d4bbfSMilan Broz 2132671ea6b4SMikulas Patocka r = crypt_setkey(cc); 2133265e9098SOndrej Kozina if (!r) 2134e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2135e48d4bbfSMilan Broz 2136de8be5acSMilan Broz out: 2137de8be5acSMilan Broz /* Hex key string not needed after here, so wipe it. */ 2138de8be5acSMilan Broz memset(key, '0', key_string_len); 2139de8be5acSMilan Broz 2140de8be5acSMilan Broz return r; 2141e48d4bbfSMilan Broz } 2142e48d4bbfSMilan Broz 2143e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 2144e48d4bbfSMilan Broz { 2145c82feeecSOndrej Kozina int r; 2146c82feeecSOndrej Kozina 2147e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2148c82feeecSOndrej Kozina get_random_bytes(&cc->key, cc->key_size); 2149c538f6ecSOndrej Kozina kzfree(cc->key_string); 2150c538f6ecSOndrej Kozina cc->key_string = NULL; 2151c82feeecSOndrej Kozina r = crypt_setkey(cc); 2152c82feeecSOndrej Kozina memset(&cc->key, 0, cc->key_size * sizeof(u8)); 2153c0297721SAndi Kleen 2154c82feeecSOndrej Kozina return r; 2155e48d4bbfSMilan Broz } 2156e48d4bbfSMilan Broz 215728513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 215828513fccSMilan Broz { 215928513fccSMilan Broz struct crypt_config *cc = ti->private; 216028513fccSMilan Broz 216128513fccSMilan Broz ti->private = NULL; 216228513fccSMilan Broz 216328513fccSMilan Broz if (!cc) 216428513fccSMilan Broz return; 216528513fccSMilan Broz 2166f659b100SRabin Vincent if (cc->write_thread) 2167dc267621SMikulas Patocka kthread_stop(cc->write_thread); 2168dc267621SMikulas Patocka 216928513fccSMilan Broz if (cc->io_queue) 217028513fccSMilan Broz destroy_workqueue(cc->io_queue); 217128513fccSMilan Broz if (cc->crypt_queue) 217228513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 217328513fccSMilan Broz 2174fd2d231fSMikulas Patocka crypt_free_tfms(cc); 2175fd2d231fSMikulas Patocka 217628513fccSMilan Broz if (cc->bs) 217728513fccSMilan Broz bioset_free(cc->bs); 217828513fccSMilan Broz 217928513fccSMilan Broz mempool_destroy(cc->page_pool); 218028513fccSMilan Broz mempool_destroy(cc->req_pool); 2181ef43aa38SMilan Broz mempool_destroy(cc->tag_pool); 218228513fccSMilan Broz 218328513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 218428513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 218528513fccSMilan Broz 218628513fccSMilan Broz if (cc->dev) 218728513fccSMilan Broz dm_put_device(ti, cc->dev); 218828513fccSMilan Broz 21895ebaee6dSMilan Broz kzfree(cc->cipher); 21907dbcd137SMilan Broz kzfree(cc->cipher_string); 2191c538f6ecSOndrej Kozina kzfree(cc->key_string); 2192ef43aa38SMilan Broz kzfree(cc->cipher_auth); 2193ef43aa38SMilan Broz kzfree(cc->authenc_key); 219428513fccSMilan Broz 219528513fccSMilan Broz /* Must zero key material before freeing */ 219628513fccSMilan Broz kzfree(cc); 219728513fccSMilan Broz } 219828513fccSMilan Broz 2199e889f97aSMilan Broz static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode) 22001da177e4SLinus Torvalds { 22015ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 22021da177e4SLinus Torvalds 220333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2204e889f97aSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc)); 2205e889f97aSMilan Broz else 2206bbdb23b5SHerbert Xu cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc)); 2207e889f97aSMilan Broz 22085ebaee6dSMilan Broz if (cc->iv_size) 22095ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 22105ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 22115ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 22125ebaee6dSMilan Broz else if (ivmode) { 22135ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 22145ebaee6dSMilan Broz ivmode = NULL; 22155ebaee6dSMilan Broz } 22165ebaee6dSMilan Broz 22175ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 22181da177e4SLinus Torvalds if (ivmode == NULL) 22191da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 22201da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 22211da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 222261afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 222361afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 22247e3fd855SMilan Broz else if (strcmp(ivmode, "plain64be") == 0) 22257e3fd855SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64be_ops; 22261da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 22271da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 222848527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 222948527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 223046b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 223146b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 223234745785SMilan Broz else if (strcmp(ivmode, "lmk") == 0) { 223334745785SMilan Broz cc->iv_gen_ops = &crypt_iv_lmk_ops; 2234ed04d981SMilan Broz /* 2235ed04d981SMilan Broz * Version 2 and 3 is recognised according 223634745785SMilan Broz * to length of provided multi-key string. 223734745785SMilan Broz * If present (version 3), last key is used as IV seed. 2238ed04d981SMilan Broz * All keys (including IV seed) are always the same size. 223934745785SMilan Broz */ 2240da31a078SMilan Broz if (cc->key_size % cc->key_parts) { 224134745785SMilan Broz cc->key_parts++; 2242da31a078SMilan Broz cc->key_extra_size = cc->key_size / cc->key_parts; 2243da31a078SMilan Broz } 2244ed04d981SMilan Broz } else if (strcmp(ivmode, "tcw") == 0) { 2245ed04d981SMilan Broz cc->iv_gen_ops = &crypt_iv_tcw_ops; 2246ed04d981SMilan Broz cc->key_parts += 2; /* IV + whitening */ 2247ed04d981SMilan Broz cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE; 2248e889f97aSMilan Broz } else if (strcmp(ivmode, "random") == 0) { 2249e889f97aSMilan Broz cc->iv_gen_ops = &crypt_iv_random_ops; 2250e889f97aSMilan Broz /* Need storage space in integrity fields. */ 2251e889f97aSMilan Broz cc->integrity_iv_size = cc->iv_size; 225234745785SMilan Broz } else { 225372d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 2254e889f97aSMilan Broz return -EINVAL; 22551da177e4SLinus Torvalds } 22561da177e4SLinus Torvalds 2257e889f97aSMilan Broz return 0; 2258e889f97aSMilan Broz } 2259e889f97aSMilan Broz 226033d2f09fSMilan Broz /* 226133d2f09fSMilan Broz * Workaround to parse cipher algorithm from crypto API spec. 226233d2f09fSMilan Broz * The cc->cipher is currently used only in ESSIV. 226333d2f09fSMilan Broz * This should be probably done by crypto-api calls (once available...) 226433d2f09fSMilan Broz */ 226533d2f09fSMilan Broz static int crypt_ctr_blkdev_cipher(struct crypt_config *cc) 226633d2f09fSMilan Broz { 226733d2f09fSMilan Broz const char *alg_name = NULL; 226833d2f09fSMilan Broz char *start, *end; 226933d2f09fSMilan Broz 227033d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 227133d2f09fSMilan Broz alg_name = crypto_tfm_alg_name(crypto_aead_tfm(any_tfm_aead(cc))); 227233d2f09fSMilan Broz if (!alg_name) 227333d2f09fSMilan Broz return -EINVAL; 227433d2f09fSMilan Broz if (crypt_integrity_hmac(cc)) { 227533d2f09fSMilan Broz alg_name = strchr(alg_name, ','); 227633d2f09fSMilan Broz if (!alg_name) 227733d2f09fSMilan Broz return -EINVAL; 227833d2f09fSMilan Broz } 227933d2f09fSMilan Broz alg_name++; 228033d2f09fSMilan Broz } else { 228133d2f09fSMilan Broz alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(any_tfm(cc))); 228233d2f09fSMilan Broz if (!alg_name) 228333d2f09fSMilan Broz return -EINVAL; 228433d2f09fSMilan Broz } 228533d2f09fSMilan Broz 228633d2f09fSMilan Broz start = strchr(alg_name, '('); 228733d2f09fSMilan Broz end = strchr(alg_name, ')'); 228833d2f09fSMilan Broz 228933d2f09fSMilan Broz if (!start && !end) { 229033d2f09fSMilan Broz cc->cipher = kstrdup(alg_name, GFP_KERNEL); 229133d2f09fSMilan Broz return cc->cipher ? 0 : -ENOMEM; 229233d2f09fSMilan Broz } 229333d2f09fSMilan Broz 229433d2f09fSMilan Broz if (!start || !end || ++start >= end) 229533d2f09fSMilan Broz return -EINVAL; 229633d2f09fSMilan Broz 229733d2f09fSMilan Broz cc->cipher = kzalloc(end - start + 1, GFP_KERNEL); 229833d2f09fSMilan Broz if (!cc->cipher) 229933d2f09fSMilan Broz return -ENOMEM; 230033d2f09fSMilan Broz 230133d2f09fSMilan Broz strncpy(cc->cipher, start, end - start); 230233d2f09fSMilan Broz 230333d2f09fSMilan Broz return 0; 230433d2f09fSMilan Broz } 230533d2f09fSMilan Broz 230633d2f09fSMilan Broz /* 230733d2f09fSMilan Broz * Workaround to parse HMAC algorithm from AEAD crypto API spec. 230833d2f09fSMilan Broz * The HMAC is needed to calculate tag size (HMAC digest size). 230933d2f09fSMilan Broz * This should be probably done by crypto-api calls (once available...) 231033d2f09fSMilan Broz */ 231133d2f09fSMilan Broz static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api) 231233d2f09fSMilan Broz { 231333d2f09fSMilan Broz char *start, *end, *mac_alg = NULL; 231433d2f09fSMilan Broz struct crypto_ahash *mac; 231533d2f09fSMilan Broz 231633d2f09fSMilan Broz if (!strstarts(cipher_api, "authenc(")) 231733d2f09fSMilan Broz return 0; 231833d2f09fSMilan Broz 231933d2f09fSMilan Broz start = strchr(cipher_api, '('); 232033d2f09fSMilan Broz end = strchr(cipher_api, ','); 232133d2f09fSMilan Broz if (!start || !end || ++start > end) 232233d2f09fSMilan Broz return -EINVAL; 232333d2f09fSMilan Broz 232433d2f09fSMilan Broz mac_alg = kzalloc(end - start + 1, GFP_KERNEL); 232533d2f09fSMilan Broz if (!mac_alg) 232633d2f09fSMilan Broz return -ENOMEM; 232733d2f09fSMilan Broz strncpy(mac_alg, start, end - start); 232833d2f09fSMilan Broz 232933d2f09fSMilan Broz mac = crypto_alloc_ahash(mac_alg, 0, 0); 233033d2f09fSMilan Broz kfree(mac_alg); 233133d2f09fSMilan Broz 233233d2f09fSMilan Broz if (IS_ERR(mac)) 233333d2f09fSMilan Broz return PTR_ERR(mac); 233433d2f09fSMilan Broz 233533d2f09fSMilan Broz cc->key_mac_size = crypto_ahash_digestsize(mac); 233633d2f09fSMilan Broz crypto_free_ahash(mac); 233733d2f09fSMilan Broz 233833d2f09fSMilan Broz cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL); 233933d2f09fSMilan Broz if (!cc->authenc_key) 234033d2f09fSMilan Broz return -ENOMEM; 234133d2f09fSMilan Broz 234233d2f09fSMilan Broz return 0; 234333d2f09fSMilan Broz } 234433d2f09fSMilan Broz 234533d2f09fSMilan Broz static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key, 234633d2f09fSMilan Broz char **ivmode, char **ivopts) 23471da177e4SLinus Torvalds { 23485ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 234933d2f09fSMilan Broz char *tmp, *cipher_api; 235033d2f09fSMilan Broz int ret = -EINVAL; 235133d2f09fSMilan Broz 235233d2f09fSMilan Broz cc->tfms_count = 1; 235333d2f09fSMilan Broz 235433d2f09fSMilan Broz /* 235533d2f09fSMilan Broz * New format (capi: prefix) 235633d2f09fSMilan Broz * capi:cipher_api_spec-iv:ivopts 235733d2f09fSMilan Broz */ 235833d2f09fSMilan Broz tmp = &cipher_in[strlen("capi:")]; 235933d2f09fSMilan Broz cipher_api = strsep(&tmp, "-"); 236033d2f09fSMilan Broz *ivmode = strsep(&tmp, ":"); 236133d2f09fSMilan Broz *ivopts = tmp; 236233d2f09fSMilan Broz 236333d2f09fSMilan Broz if (*ivmode && !strcmp(*ivmode, "lmk")) 236433d2f09fSMilan Broz cc->tfms_count = 64; 236533d2f09fSMilan Broz 236633d2f09fSMilan Broz cc->key_parts = cc->tfms_count; 236733d2f09fSMilan Broz 236833d2f09fSMilan Broz /* Allocate cipher */ 236933d2f09fSMilan Broz ret = crypt_alloc_tfms(cc, cipher_api); 237033d2f09fSMilan Broz if (ret < 0) { 237133d2f09fSMilan Broz ti->error = "Error allocating crypto tfm"; 237233d2f09fSMilan Broz return ret; 237333d2f09fSMilan Broz } 237433d2f09fSMilan Broz 237533d2f09fSMilan Broz /* Alloc AEAD, can be used only in new format. */ 237633d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 237733d2f09fSMilan Broz ret = crypt_ctr_auth_cipher(cc, cipher_api); 237833d2f09fSMilan Broz if (ret < 0) { 237933d2f09fSMilan Broz ti->error = "Invalid AEAD cipher spec"; 238033d2f09fSMilan Broz return -ENOMEM; 238133d2f09fSMilan Broz } 238233d2f09fSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc)); 238333d2f09fSMilan Broz } else 238433d2f09fSMilan Broz cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc)); 238533d2f09fSMilan Broz 238633d2f09fSMilan Broz ret = crypt_ctr_blkdev_cipher(cc); 238733d2f09fSMilan Broz if (ret < 0) { 238833d2f09fSMilan Broz ti->error = "Cannot allocate cipher string"; 238933d2f09fSMilan Broz return -ENOMEM; 239033d2f09fSMilan Broz } 239133d2f09fSMilan Broz 239233d2f09fSMilan Broz return 0; 239333d2f09fSMilan Broz } 239433d2f09fSMilan Broz 239533d2f09fSMilan Broz static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key, 239633d2f09fSMilan Broz char **ivmode, char **ivopts) 239733d2f09fSMilan Broz { 239833d2f09fSMilan Broz struct crypt_config *cc = ti->private; 239933d2f09fSMilan Broz char *tmp, *cipher, *chainmode, *keycount; 24005ebaee6dSMilan Broz char *cipher_api = NULL; 24015ebaee6dSMilan Broz int ret = -EINVAL; 24025ebaee6dSMilan Broz char dummy; 24035ebaee6dSMilan Broz 240433d2f09fSMilan Broz if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) { 24055ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 24065ebaee6dSMilan Broz return -EINVAL; 24075ebaee6dSMilan Broz } 24085ebaee6dSMilan Broz 24091da177e4SLinus Torvalds /* 24105ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 24115ebaee6dSMilan Broz * cipher[:keycount]-mode-iv:ivopts 24125ebaee6dSMilan Broz */ 24135ebaee6dSMilan Broz tmp = cipher_in; 24145ebaee6dSMilan Broz keycount = strsep(&tmp, "-"); 24155ebaee6dSMilan Broz cipher = strsep(&keycount, ":"); 24165ebaee6dSMilan Broz 241769a8cfcdSMilan Broz if (!keycount) 24185ebaee6dSMilan Broz cc->tfms_count = 1; 24195ebaee6dSMilan Broz else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 || 24205ebaee6dSMilan Broz !is_power_of_2(cc->tfms_count)) { 24215ebaee6dSMilan Broz ti->error = "Bad cipher key count specification"; 24225ebaee6dSMilan Broz return -EINVAL; 24235ebaee6dSMilan Broz } 242428513fccSMilan Broz cc->key_parts = cc->tfms_count; 24251da177e4SLinus Torvalds 242672d94861SAlasdair G Kergon cc->cipher = kstrdup(cipher, GFP_KERNEL); 242728513fccSMilan Broz if (!cc->cipher) 24281da177e4SLinus Torvalds goto bad_mem; 24291da177e4SLinus Torvalds 2430ddd42edfSMilan Broz chainmode = strsep(&tmp, "-"); 243133d2f09fSMilan Broz *ivopts = strsep(&tmp, "-"); 243233d2f09fSMilan Broz *ivmode = strsep(&*ivopts, ":"); 2433c0297721SAndi Kleen 24343a7f6c99SMilan Broz if (tmp) 2435ddd42edfSMilan Broz DMWARN("Ignoring unexpected additional cipher options"); 2436ddd42edfSMilan Broz 2437ddd42edfSMilan Broz /* 2438ddd42edfSMilan Broz * For compatibility with the original dm-crypt mapping format, if 2439ddd42edfSMilan Broz * only the cipher name is supplied, use cbc-plain. 244028513fccSMilan Broz */ 244133d2f09fSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) { 2442cabf08e4SMilan Broz chainmode = "cbc"; 244333d2f09fSMilan Broz *ivmode = "plain"; 2444cabf08e4SMilan Broz } 2445cabf08e4SMilan Broz 244633d2f09fSMilan Broz if (strcmp(chainmode, "ecb") && !*ivmode) { 2447c0297721SAndi Kleen ti->error = "IV mechanism required"; 2448c0297721SAndi Kleen return -EINVAL; 2449c0297721SAndi Kleen } 2450c0297721SAndi Kleen 2451cabf08e4SMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 24529934a8beSMilan Broz if (!cipher_api) 245328513fccSMilan Broz goto bad_mem; 24549934a8beSMilan Broz 24559934a8beSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 2456647c7db1SMikulas Patocka "%s(%s)", chainmode, cipher); 24571da177e4SLinus Torvalds if (ret < 0) { 24581da177e4SLinus Torvalds kfree(cipher_api); 245928513fccSMilan Broz goto bad_mem; 246028513fccSMilan Broz } 246128513fccSMilan Broz 24621da177e4SLinus Torvalds /* Allocate cipher */ 24631da177e4SLinus Torvalds ret = crypt_alloc_tfms(cc, cipher_api); 24641da177e4SLinus Torvalds if (ret < 0) { 24651da177e4SLinus Torvalds ti->error = "Error allocating crypto tfm"; 246633d2f09fSMilan Broz kfree(cipher_api); 246733d2f09fSMilan Broz return ret; 2468028867acSAlasdair G Kergon } 2469bd86e320SJeffy Chen kfree(cipher_api); 2470647c7db1SMikulas Patocka 247133d2f09fSMilan Broz return 0; 247233d2f09fSMilan Broz bad_mem: 247333d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings"; 247433d2f09fSMilan Broz return -ENOMEM; 247533d2f09fSMilan Broz } 247633d2f09fSMilan Broz 247733d2f09fSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key) 247833d2f09fSMilan Broz { 247933d2f09fSMilan Broz struct crypt_config *cc = ti->private; 248033d2f09fSMilan Broz char *ivmode = NULL, *ivopts = NULL; 248133d2f09fSMilan Broz int ret; 248233d2f09fSMilan Broz 248333d2f09fSMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL); 248433d2f09fSMilan Broz if (!cc->cipher_string) { 248533d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings"; 248633d2f09fSMilan Broz return -ENOMEM; 248733d2f09fSMilan Broz } 248833d2f09fSMilan Broz 248933d2f09fSMilan Broz if (strstarts(cipher_in, "capi:")) 249033d2f09fSMilan Broz ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts); 249133d2f09fSMilan Broz else 249233d2f09fSMilan Broz ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts); 249333d2f09fSMilan Broz if (ret) 249433d2f09fSMilan Broz return ret; 249533d2f09fSMilan Broz 2496647c7db1SMikulas Patocka /* Initialize IV */ 2497e889f97aSMilan Broz ret = crypt_ctr_ivmode(ti, ivmode); 2498e889f97aSMilan Broz if (ret < 0) 249933d2f09fSMilan Broz return ret; 25001da177e4SLinus Torvalds 2501da31a078SMilan Broz /* Initialize and set key */ 2502da31a078SMilan Broz ret = crypt_set_key(cc, key); 2503da31a078SMilan Broz if (ret < 0) { 2504da31a078SMilan Broz ti->error = "Error decoding and setting key"; 250533d2f09fSMilan Broz return ret; 2506da31a078SMilan Broz } 2507da31a078SMilan Broz 25081da177e4SLinus Torvalds /* Allocate IV */ 25091da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 25101da177e4SLinus Torvalds ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 25111da177e4SLinus Torvalds if (ret < 0) { 25121da177e4SLinus Torvalds ti->error = "Error creating IV"; 251333d2f09fSMilan Broz return ret; 25141da177e4SLinus Torvalds } 25151da177e4SLinus Torvalds } 25161da177e4SLinus Torvalds 25171da177e4SLinus Torvalds /* Initialize IV (set keys for ESSIV etc) */ 25181da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 25191da177e4SLinus Torvalds ret = cc->iv_gen_ops->init(cc); 25201da177e4SLinus Torvalds if (ret < 0) { 25211da177e4SLinus Torvalds ti->error = "Error initialising IV"; 25221da177e4SLinus Torvalds return ret; 25231da177e4SLinus Torvalds } 25241da177e4SLinus Torvalds } 25251da177e4SLinus Torvalds 252633d2f09fSMilan Broz return ret; 25271da177e4SLinus Torvalds } 25281da177e4SLinus Torvalds 2529ef43aa38SMilan Broz static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv) 2530ef43aa38SMilan Broz { 2531ef43aa38SMilan Broz struct crypt_config *cc = ti->private; 2532ef43aa38SMilan Broz struct dm_arg_set as; 25335916a22bSEric Biggers static const struct dm_arg _args[] = { 25348f0009a2SMilan Broz {0, 6, "Invalid number of feature args"}, 2535ef43aa38SMilan Broz }; 2536ef43aa38SMilan Broz unsigned int opt_params, val; 2537ef43aa38SMilan Broz const char *opt_string, *sval; 25388f0009a2SMilan Broz char dummy; 2539ef43aa38SMilan Broz int ret; 2540ef43aa38SMilan Broz 2541ef43aa38SMilan Broz /* Optional parameters */ 2542ef43aa38SMilan Broz as.argc = argc; 2543ef43aa38SMilan Broz as.argv = argv; 2544ef43aa38SMilan Broz 2545ef43aa38SMilan Broz ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 2546ef43aa38SMilan Broz if (ret) 25471da177e4SLinus Torvalds return ret; 25481da177e4SLinus Torvalds 2549ef43aa38SMilan Broz while (opt_params--) { 2550ef43aa38SMilan Broz opt_string = dm_shift_arg(&as); 2551ef43aa38SMilan Broz if (!opt_string) { 2552ef43aa38SMilan Broz ti->error = "Not enough feature arguments"; 2553ef43aa38SMilan Broz return -EINVAL; 2554ef43aa38SMilan Broz } 2555ef43aa38SMilan Broz 2556ef43aa38SMilan Broz if (!strcasecmp(opt_string, "allow_discards")) 2557ef43aa38SMilan Broz ti->num_discard_bios = 1; 2558ef43aa38SMilan Broz 2559ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "same_cpu_crypt")) 2560ef43aa38SMilan Broz set_bit(DM_CRYPT_SAME_CPU, &cc->flags); 2561ef43aa38SMilan Broz 2562ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "submit_from_crypt_cpus")) 2563ef43aa38SMilan Broz set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 2564ef43aa38SMilan Broz else if (sscanf(opt_string, "integrity:%u:", &val) == 1) { 2565ef43aa38SMilan Broz if (val == 0 || val > MAX_TAG_SIZE) { 2566ef43aa38SMilan Broz ti->error = "Invalid integrity arguments"; 2567ef43aa38SMilan Broz return -EINVAL; 2568ef43aa38SMilan Broz } 2569ef43aa38SMilan Broz cc->on_disk_tag_size = val; 2570ef43aa38SMilan Broz sval = strchr(opt_string + strlen("integrity:"), ':') + 1; 2571ef43aa38SMilan Broz if (!strcasecmp(sval, "aead")) { 2572ef43aa38SMilan Broz set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags); 2573ef43aa38SMilan Broz } else if (strcasecmp(sval, "none")) { 2574ef43aa38SMilan Broz ti->error = "Unknown integrity profile"; 2575ef43aa38SMilan Broz return -EINVAL; 2576ef43aa38SMilan Broz } 2577ef43aa38SMilan Broz 2578ef43aa38SMilan Broz cc->cipher_auth = kstrdup(sval, GFP_KERNEL); 2579ef43aa38SMilan Broz if (!cc->cipher_auth) 25801da177e4SLinus Torvalds return -ENOMEM; 2581ff3af92bSMikulas Patocka } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) { 25828f0009a2SMilan Broz if (cc->sector_size < (1 << SECTOR_SHIFT) || 25838f0009a2SMilan Broz cc->sector_size > 4096 || 2584ff3af92bSMikulas Patocka (cc->sector_size & (cc->sector_size - 1))) { 25858f0009a2SMilan Broz ti->error = "Invalid feature value for sector_size"; 25868f0009a2SMilan Broz return -EINVAL; 25878f0009a2SMilan Broz } 2588783874b0SMilan Broz if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) { 2589783874b0SMilan Broz ti->error = "Device size is not multiple of sector_size feature"; 2590783874b0SMilan Broz return -EINVAL; 2591783874b0SMilan Broz } 2592ff3af92bSMikulas Patocka cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT; 25938f0009a2SMilan Broz } else if (!strcasecmp(opt_string, "iv_large_sectors")) 25948f0009a2SMilan Broz set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags); 25958f0009a2SMilan Broz else { 2596ef43aa38SMilan Broz ti->error = "Invalid feature arguments"; 2597ef43aa38SMilan Broz return -EINVAL; 2598ef43aa38SMilan Broz } 2599ef43aa38SMilan Broz } 2600ef43aa38SMilan Broz 2601ef43aa38SMilan Broz return 0; 26021da177e4SLinus Torvalds } 26031da177e4SLinus Torvalds 26041da177e4SLinus Torvalds /* 26051da177e4SLinus Torvalds * Construct an encryption mapping: 2606c538f6ecSOndrej Kozina * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start> 26071da177e4SLinus Torvalds */ 26081da177e4SLinus Torvalds static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 26091da177e4SLinus Torvalds { 26101da177e4SLinus Torvalds struct crypt_config *cc; 2611c538f6ecSOndrej Kozina int key_size; 2612ef43aa38SMilan Broz unsigned int align_mask; 26131da177e4SLinus Torvalds unsigned long long tmpll; 26141da177e4SLinus Torvalds int ret; 2615ef43aa38SMilan Broz size_t iv_size_padding, additional_req_size; 261631998ef1SMikulas Patocka char dummy; 26171da177e4SLinus Torvalds 2618772ae5f5SMilan Broz if (argc < 5) { 26191da177e4SLinus Torvalds ti->error = "Not enough arguments"; 26201da177e4SLinus Torvalds return -EINVAL; 26211da177e4SLinus Torvalds } 26221da177e4SLinus Torvalds 2623c538f6ecSOndrej Kozina key_size = get_key_size(&argv[1]); 2624c538f6ecSOndrej Kozina if (key_size < 0) { 2625c538f6ecSOndrej Kozina ti->error = "Cannot parse key size"; 2626c538f6ecSOndrej Kozina return -EINVAL; 2627c538f6ecSOndrej Kozina } 26281da177e4SLinus Torvalds 26291da177e4SLinus Torvalds cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 26301da177e4SLinus Torvalds if (!cc) { 26311da177e4SLinus Torvalds ti->error = "Cannot allocate encryption context"; 26321da177e4SLinus Torvalds return -ENOMEM; 26331da177e4SLinus Torvalds } 26341da177e4SLinus Torvalds cc->key_size = key_size; 26358f0009a2SMilan Broz cc->sector_size = (1 << SECTOR_SHIFT); 2636ff3af92bSMikulas Patocka cc->sector_shift = 0; 26371da177e4SLinus Torvalds 26381da177e4SLinus Torvalds ti->private = cc; 2639ef43aa38SMilan Broz 2640ef43aa38SMilan Broz /* Optional parameters need to be read before cipher constructor */ 2641ef43aa38SMilan Broz if (argc > 5) { 2642ef43aa38SMilan Broz ret = crypt_ctr_optional(ti, argc - 5, &argv[5]); 2643ef43aa38SMilan Broz if (ret) 2644ef43aa38SMilan Broz goto bad; 2645ef43aa38SMilan Broz } 2646ef43aa38SMilan Broz 26471da177e4SLinus Torvalds ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 26481da177e4SLinus Torvalds if (ret < 0) 26491da177e4SLinus Torvalds goto bad; 26501da177e4SLinus Torvalds 265133d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 2652ef43aa38SMilan Broz cc->dmreq_start = sizeof(struct aead_request); 2653ef43aa38SMilan Broz cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc)); 2654ef43aa38SMilan Broz align_mask = crypto_aead_alignmask(any_tfm_aead(cc)); 2655ef43aa38SMilan Broz } else { 2656bbdb23b5SHerbert Xu cc->dmreq_start = sizeof(struct skcipher_request); 2657bbdb23b5SHerbert Xu cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc)); 2658ef43aa38SMilan Broz align_mask = crypto_skcipher_alignmask(any_tfm(cc)); 2659ef43aa38SMilan Broz } 2660d49ec52fSMikulas Patocka cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request)); 2661d49ec52fSMikulas Patocka 2662ef43aa38SMilan Broz if (align_mask < CRYPTO_MINALIGN) { 2663d49ec52fSMikulas Patocka /* Allocate the padding exactly */ 2664d49ec52fSMikulas Patocka iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request)) 2665ef43aa38SMilan Broz & align_mask; 2666d49ec52fSMikulas Patocka } else { 2667d49ec52fSMikulas Patocka /* 2668d49ec52fSMikulas Patocka * If the cipher requires greater alignment than kmalloc 2669d49ec52fSMikulas Patocka * alignment, we don't know the exact position of the 2670d49ec52fSMikulas Patocka * initialization vector. We must assume worst case. 2671d49ec52fSMikulas Patocka */ 2672ef43aa38SMilan Broz iv_size_padding = align_mask; 2673d49ec52fSMikulas Patocka } 26741da177e4SLinus Torvalds 267594f5e024SMikulas Patocka ret = -ENOMEM; 2676ef43aa38SMilan Broz 2677ef43aa38SMilan Broz /* ...| IV + padding | original IV | original sec. number | bio tag offset | */ 2678ef43aa38SMilan Broz additional_req_size = sizeof(struct dm_crypt_request) + 2679ef43aa38SMilan Broz iv_size_padding + cc->iv_size + 2680ef43aa38SMilan Broz cc->iv_size + 2681ef43aa38SMilan Broz sizeof(uint64_t) + 2682ef43aa38SMilan Broz sizeof(unsigned int); 2683ef43aa38SMilan Broz 2684ef43aa38SMilan Broz cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + additional_req_size); 26851da177e4SLinus Torvalds if (!cc->req_pool) { 26861da177e4SLinus Torvalds ti->error = "Cannot allocate crypt request mempool"; 26871da177e4SLinus Torvalds goto bad; 26881da177e4SLinus Torvalds } 26891da177e4SLinus Torvalds 269030187e1dSMike Snitzer cc->per_bio_data_size = ti->per_io_data_size = 2691ef43aa38SMilan Broz ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size, 2692d49ec52fSMikulas Patocka ARCH_KMALLOC_MINALIGN); 2693298a9fa0SMikulas Patocka 2694cf2f1abfSMikulas Patocka cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0); 26951da177e4SLinus Torvalds if (!cc->page_pool) { 26968b004457SMilan Broz ti->error = "Cannot allocate page mempool"; 2697e48d4bbfSMilan Broz goto bad; 26981da177e4SLinus Torvalds } 2699e48d4bbfSMilan Broz 270080cd1757SNeilBrown cc->bs = bioset_create(MIN_IOS, 0, BIOSET_NEED_BVECS); 27018b004457SMilan Broz if (!cc->bs) { 27020c395b0fSMilan Broz ti->error = "Cannot allocate crypt bioset"; 2703cabf08e4SMilan Broz goto bad; 270493e605c2SMilan Broz } 2705cabf08e4SMilan Broz 27067145c241SMikulas Patocka mutex_init(&cc->bio_alloc_lock); 27077145c241SMikulas Patocka 2708cabf08e4SMilan Broz ret = -EINVAL; 27098f0009a2SMilan Broz if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) || 27108f0009a2SMilan Broz (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) { 2711cabf08e4SMilan Broz ti->error = "Invalid iv_offset sector"; 2712cabf08e4SMilan Broz goto bad; 27131da177e4SLinus Torvalds } 2714d2a7ad29SKiyoshi Ueda cc->iv_offset = tmpll; 27151da177e4SLinus Torvalds 2716e80d1c80SVivek Goyal ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev); 2717e80d1c80SVivek Goyal if (ret) { 27181da177e4SLinus Torvalds ti->error = "Device lookup failed"; 27191da177e4SLinus Torvalds goto bad; 27201da177e4SLinus Torvalds } 27211da177e4SLinus Torvalds 2722e80d1c80SVivek Goyal ret = -EINVAL; 272331998ef1SMikulas Patocka if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1) { 27241da177e4SLinus Torvalds ti->error = "Invalid device sector"; 27251da177e4SLinus Torvalds goto bad; 27261da177e4SLinus Torvalds } 27271da177e4SLinus Torvalds cc->start = tmpll; 27281da177e4SLinus Torvalds 272933d2f09fSMilan Broz if (crypt_integrity_aead(cc) || cc->integrity_iv_size) { 2730ef43aa38SMilan Broz ret = crypt_integrity_ctr(cc, ti); 2731772ae5f5SMilan Broz if (ret) 2732772ae5f5SMilan Broz goto bad; 2733772ae5f5SMilan Broz 2734ef43aa38SMilan Broz cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size; 2735ef43aa38SMilan Broz if (!cc->tag_pool_max_sectors) 2736ef43aa38SMilan Broz cc->tag_pool_max_sectors = 1; 2737ef43aa38SMilan Broz 2738ef43aa38SMilan Broz cc->tag_pool = mempool_create_kmalloc_pool(MIN_IOS, 2739ef43aa38SMilan Broz cc->tag_pool_max_sectors * cc->on_disk_tag_size); 2740ef43aa38SMilan Broz if (!cc->tag_pool) { 2741ef43aa38SMilan Broz ti->error = "Cannot allocate integrity tags mempool"; 2742f3396c58SMikulas Patocka goto bad; 2743f3396c58SMikulas Patocka } 2744772ae5f5SMilan Broz 2745583fe747SMikulas Patocka cc->tag_pool_max_sectors <<= cc->sector_shift; 2746f3396c58SMikulas Patocka } 2747772ae5f5SMilan Broz 27481da177e4SLinus Torvalds ret = -ENOMEM; 2749a1b89132STim Murray cc->io_queue = alloc_workqueue("kcryptd_io", WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); 27501da177e4SLinus Torvalds if (!cc->io_queue) { 27511da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd io queue"; 27521da177e4SLinus Torvalds goto bad; 27531da177e4SLinus Torvalds } 275437af6560SChristophe Saout 2755f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 2756a1b89132STim Murray cc->crypt_queue = alloc_workqueue("kcryptd", WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); 2757f3396c58SMikulas Patocka else 2758a1b89132STim Murray cc->crypt_queue = alloc_workqueue("kcryptd", 2759a1b89132STim Murray WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, 2760f3396c58SMikulas Patocka num_online_cpus()); 27611da177e4SLinus Torvalds if (!cc->crypt_queue) { 27621da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd queue"; 27631da177e4SLinus Torvalds goto bad; 27641da177e4SLinus Torvalds } 27651da177e4SLinus Torvalds 2766dc267621SMikulas Patocka init_waitqueue_head(&cc->write_thread_wait); 2767b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 2768dc267621SMikulas Patocka 2769dc267621SMikulas Patocka cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write"); 2770dc267621SMikulas Patocka if (IS_ERR(cc->write_thread)) { 2771dc267621SMikulas Patocka ret = PTR_ERR(cc->write_thread); 2772dc267621SMikulas Patocka cc->write_thread = NULL; 2773dc267621SMikulas Patocka ti->error = "Couldn't spawn write thread"; 2774dc267621SMikulas Patocka goto bad; 2775dc267621SMikulas Patocka } 2776dc267621SMikulas Patocka wake_up_process(cc->write_thread); 2777dc267621SMikulas Patocka 277855a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 2779983c7db3SMilan Broz 27801da177e4SLinus Torvalds return 0; 27811da177e4SLinus Torvalds 27821da177e4SLinus Torvalds bad: 27831da177e4SLinus Torvalds crypt_dtr(ti); 27841da177e4SLinus Torvalds return ret; 2785647c7db1SMikulas Patocka } 2786647c7db1SMikulas Patocka 27877de3ee57SMikulas Patocka static int crypt_map(struct dm_target *ti, struct bio *bio) 27881da177e4SLinus Torvalds { 27891da177e4SLinus Torvalds struct dm_crypt_io *io; 279049a8a920SAlasdair G Kergon struct crypt_config *cc = ti->private; 2791647c7db1SMikulas Patocka 2792772ae5f5SMilan Broz /* 279328a8f0d3SMike Christie * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues. 279428a8f0d3SMike Christie * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight 2795e6047149SMike Christie * - for REQ_OP_DISCARD caller must use flush if IO ordering matters 2796772ae5f5SMilan Broz */ 27971eff9d32SJens Axboe if (unlikely(bio->bi_opf & REQ_PREFLUSH || 279828a8f0d3SMike Christie bio_op(bio) == REQ_OP_DISCARD)) { 279974d46992SChristoph Hellwig bio_set_dev(bio, cc->dev->bdev); 2800772ae5f5SMilan Broz if (bio_sectors(bio)) 28014f024f37SKent Overstreet bio->bi_iter.bi_sector = cc->start + 28024f024f37SKent Overstreet dm_target_offset(ti, bio->bi_iter.bi_sector); 2803647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 2804647c7db1SMikulas Patocka } 28051da177e4SLinus Torvalds 28064e870e94SMikulas Patocka /* 28074e870e94SMikulas Patocka * Check if bio is too large, split as needed. 28084e870e94SMikulas Patocka */ 28094e870e94SMikulas Patocka if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_PAGES << PAGE_SHIFT)) && 2810ef43aa38SMilan Broz (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size)) 28114e870e94SMikulas Patocka dm_accept_partial_bio(bio, ((BIO_MAX_PAGES << PAGE_SHIFT) >> SECTOR_SHIFT)); 28124e870e94SMikulas Patocka 28138f0009a2SMilan Broz /* 28148f0009a2SMilan Broz * Ensure that bio is a multiple of internal sector encryption size 28158f0009a2SMilan Broz * and is aligned to this size as defined in IO hints. 28168f0009a2SMilan Broz */ 28178f0009a2SMilan Broz if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0)) 2818846785e6SChristoph Hellwig return DM_MAPIO_KILL; 28198f0009a2SMilan Broz 28208f0009a2SMilan Broz if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1))) 2821846785e6SChristoph Hellwig return DM_MAPIO_KILL; 28228f0009a2SMilan Broz 2823298a9fa0SMikulas Patocka io = dm_per_bio_data(bio, cc->per_bio_data_size); 2824298a9fa0SMikulas Patocka crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector)); 2825ef43aa38SMilan Broz 2826ef43aa38SMilan Broz if (cc->on_disk_tag_size) { 2827583fe747SMikulas Patocka unsigned tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift); 2828ef43aa38SMilan Broz 2829ef43aa38SMilan Broz if (unlikely(tag_len > KMALLOC_MAX_SIZE) || 2830583fe747SMikulas Patocka unlikely(!(io->integrity_metadata = kmalloc(tag_len, 2831ef43aa38SMilan Broz GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN)))) { 2832ef43aa38SMilan Broz if (bio_sectors(bio) > cc->tag_pool_max_sectors) 2833ef43aa38SMilan Broz dm_accept_partial_bio(bio, cc->tag_pool_max_sectors); 2834ef43aa38SMilan Broz io->integrity_metadata = mempool_alloc(cc->tag_pool, GFP_NOIO); 2835ef43aa38SMilan Broz io->integrity_metadata_from_pool = true; 2836ef43aa38SMilan Broz } 2837ef43aa38SMilan Broz } 2838ef43aa38SMilan Broz 283933d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2840ef43aa38SMilan Broz io->ctx.r.req_aead = (struct aead_request *)(io + 1); 2841ef43aa38SMilan Broz else 2842ef43aa38SMilan Broz io->ctx.r.req = (struct skcipher_request *)(io + 1); 28431da177e4SLinus Torvalds 284420c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 284520c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOWAIT)) 2846dc267621SMikulas Patocka kcryptd_queue_read(io); 284720c82538SMilan Broz } else 28484ee218cdSAndrew Morton kcryptd_queue_crypt(io); 28494ee218cdSAndrew Morton 28501da177e4SLinus Torvalds return DM_MAPIO_SUBMITTED; 28511da177e4SLinus Torvalds } 28521da177e4SLinus Torvalds 2853fd7c092eSMikulas Patocka static void crypt_status(struct dm_target *ti, status_type_t type, 28541f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 28551da177e4SLinus Torvalds { 28565ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 2857fd7c092eSMikulas Patocka unsigned i, sz = 0; 2858f3396c58SMikulas Patocka int num_feature_args = 0; 28591da177e4SLinus Torvalds 28601da177e4SLinus Torvalds switch (type) { 28611da177e4SLinus Torvalds case STATUSTYPE_INFO: 28621da177e4SLinus Torvalds result[0] = '\0'; 28631da177e4SLinus Torvalds break; 28641da177e4SLinus Torvalds 28651da177e4SLinus Torvalds case STATUSTYPE_TABLE: 28667dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string); 28671da177e4SLinus Torvalds 2868c538f6ecSOndrej Kozina if (cc->key_size > 0) { 2869c538f6ecSOndrej Kozina if (cc->key_string) 2870c538f6ecSOndrej Kozina DMEMIT(":%u:%s", cc->key_size, cc->key_string); 2871c538f6ecSOndrej Kozina else 2872fd7c092eSMikulas Patocka for (i = 0; i < cc->key_size; i++) 2873fd7c092eSMikulas Patocka DMEMIT("%02x", cc->key[i]); 2874c538f6ecSOndrej Kozina } else 2875fd7c092eSMikulas Patocka DMEMIT("-"); 28761da177e4SLinus Torvalds 28771da177e4SLinus Torvalds DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 28781da177e4SLinus Torvalds cc->dev->name, (unsigned long long)cc->start); 2879772ae5f5SMilan Broz 2880f3396c58SMikulas Patocka num_feature_args += !!ti->num_discard_bios; 2881f3396c58SMikulas Patocka num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags); 28820f5d8e6eSMikulas Patocka num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 2883ff3af92bSMikulas Patocka num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT); 28848f0009a2SMilan Broz num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags); 2885ef43aa38SMilan Broz if (cc->on_disk_tag_size) 2886ef43aa38SMilan Broz num_feature_args++; 2887f3396c58SMikulas Patocka if (num_feature_args) { 2888f3396c58SMikulas Patocka DMEMIT(" %d", num_feature_args); 288955a62eefSAlasdair G Kergon if (ti->num_discard_bios) 2890f3396c58SMikulas Patocka DMEMIT(" allow_discards"); 2891f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 2892f3396c58SMikulas Patocka DMEMIT(" same_cpu_crypt"); 28930f5d8e6eSMikulas Patocka if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) 28940f5d8e6eSMikulas Patocka DMEMIT(" submit_from_crypt_cpus"); 2895ef43aa38SMilan Broz if (cc->on_disk_tag_size) 2896ef43aa38SMilan Broz DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth); 28978f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) 28988f0009a2SMilan Broz DMEMIT(" sector_size:%d", cc->sector_size); 28998f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 29008f0009a2SMilan Broz DMEMIT(" iv_large_sectors"); 2901f3396c58SMikulas Patocka } 2902772ae5f5SMilan Broz 29031da177e4SLinus Torvalds break; 29041da177e4SLinus Torvalds } 29051da177e4SLinus Torvalds } 29061da177e4SLinus Torvalds 2907e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 2908e48d4bbfSMilan Broz { 2909e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 2910e48d4bbfSMilan Broz 2911e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 2912e48d4bbfSMilan Broz } 2913e48d4bbfSMilan Broz 2914e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 2915e48d4bbfSMilan Broz { 2916e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 2917e48d4bbfSMilan Broz 2918e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 2919e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 2920e48d4bbfSMilan Broz return -EAGAIN; 2921e48d4bbfSMilan Broz } 2922e48d4bbfSMilan Broz 2923e48d4bbfSMilan Broz return 0; 2924e48d4bbfSMilan Broz } 2925e48d4bbfSMilan Broz 2926e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 2927e48d4bbfSMilan Broz { 2928e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 2929e48d4bbfSMilan Broz 2930e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 2931e48d4bbfSMilan Broz } 2932e48d4bbfSMilan Broz 2933e48d4bbfSMilan Broz /* Message interface 2934e48d4bbfSMilan Broz * key set <key> 2935e48d4bbfSMilan Broz * key wipe 2936e48d4bbfSMilan Broz */ 2937e48d4bbfSMilan Broz static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 2938e48d4bbfSMilan Broz { 2939e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 2940c538f6ecSOndrej Kozina int key_size, ret = -EINVAL; 2941e48d4bbfSMilan Broz 2942e48d4bbfSMilan Broz if (argc < 2) 2943e48d4bbfSMilan Broz goto error; 2944e48d4bbfSMilan Broz 2945498f0103SMike Snitzer if (!strcasecmp(argv[0], "key")) { 2946e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 2947e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 2948e48d4bbfSMilan Broz return -EINVAL; 2949e48d4bbfSMilan Broz } 2950498f0103SMike Snitzer if (argc == 3 && !strcasecmp(argv[1], "set")) { 2951c538f6ecSOndrej Kozina /* The key size may not be changed. */ 2952c538f6ecSOndrej Kozina key_size = get_key_size(&argv[2]); 2953c538f6ecSOndrej Kozina if (key_size < 0 || cc->key_size != key_size) { 2954c538f6ecSOndrej Kozina memset(argv[2], '0', strlen(argv[2])); 2955c538f6ecSOndrej Kozina return -EINVAL; 2956c538f6ecSOndrej Kozina } 2957c538f6ecSOndrej Kozina 2958542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 2959542da317SMilan Broz if (ret) 2960542da317SMilan Broz return ret; 2961542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 2962542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 2963542da317SMilan Broz return ret; 2964542da317SMilan Broz } 2965498f0103SMike Snitzer if (argc == 2 && !strcasecmp(argv[1], "wipe")) { 2966542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 2967542da317SMilan Broz ret = cc->iv_gen_ops->wipe(cc); 2968542da317SMilan Broz if (ret) 2969542da317SMilan Broz return ret; 2970542da317SMilan Broz } 2971e48d4bbfSMilan Broz return crypt_wipe_key(cc); 2972e48d4bbfSMilan Broz } 2973542da317SMilan Broz } 2974e48d4bbfSMilan Broz 2975e48d4bbfSMilan Broz error: 2976e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 2977e48d4bbfSMilan Broz return -EINVAL; 2978e48d4bbfSMilan Broz } 2979e48d4bbfSMilan Broz 2980af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 2981af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 2982af4874e0SMike Snitzer { 2983af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 2984af4874e0SMike Snitzer 29855dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 2986af4874e0SMike Snitzer } 2987af4874e0SMike Snitzer 2988586b286bSMike Snitzer static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits) 2989586b286bSMike Snitzer { 29908f0009a2SMilan Broz struct crypt_config *cc = ti->private; 29918f0009a2SMilan Broz 2992586b286bSMike Snitzer /* 2993586b286bSMike Snitzer * Unfortunate constraint that is required to avoid the potential 2994586b286bSMike Snitzer * for exceeding underlying device's max_segments limits -- due to 2995586b286bSMike Snitzer * crypt_alloc_buffer() possibly allocating pages for the encryption 2996586b286bSMike Snitzer * bio that are not as physically contiguous as the original bio. 2997586b286bSMike Snitzer */ 2998586b286bSMike Snitzer limits->max_segment_size = PAGE_SIZE; 29998f0009a2SMilan Broz 30008f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) { 30018f0009a2SMilan Broz limits->logical_block_size = cc->sector_size; 30028f0009a2SMilan Broz limits->physical_block_size = cc->sector_size; 30038f0009a2SMilan Broz blk_limits_io_min(limits, cc->sector_size); 30048f0009a2SMilan Broz } 3005586b286bSMike Snitzer } 3006586b286bSMike Snitzer 30071da177e4SLinus Torvalds static struct target_type crypt_target = { 30081da177e4SLinus Torvalds .name = "crypt", 30097e3fd855SMilan Broz .version = {1, 18, 0}, 30101da177e4SLinus Torvalds .module = THIS_MODULE, 30111da177e4SLinus Torvalds .ctr = crypt_ctr, 30121da177e4SLinus Torvalds .dtr = crypt_dtr, 30131da177e4SLinus Torvalds .map = crypt_map, 30141da177e4SLinus Torvalds .status = crypt_status, 3015e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 3016e48d4bbfSMilan Broz .preresume = crypt_preresume, 3017e48d4bbfSMilan Broz .resume = crypt_resume, 3018e48d4bbfSMilan Broz .message = crypt_message, 3019af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 3020586b286bSMike Snitzer .io_hints = crypt_io_hints, 30211da177e4SLinus Torvalds }; 30221da177e4SLinus Torvalds 30231da177e4SLinus Torvalds static int __init dm_crypt_init(void) 30241da177e4SLinus Torvalds { 30251da177e4SLinus Torvalds int r; 30261da177e4SLinus Torvalds 30271da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 302894f5e024SMikulas Patocka if (r < 0) 302972d94861SAlasdair G Kergon DMERR("register failed %d", r); 30301da177e4SLinus Torvalds 30311da177e4SLinus Torvalds return r; 30321da177e4SLinus Torvalds } 30331da177e4SLinus Torvalds 30341da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 30351da177e4SLinus Torvalds { 303610d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 30371da177e4SLinus Torvalds } 30381da177e4SLinus Torvalds 30391da177e4SLinus Torvalds module_init(dm_crypt_init); 30401da177e4SLinus Torvalds module_exit(dm_crypt_exit); 30411da177e4SLinus Torvalds 3042bf14299fSJana Saout MODULE_AUTHOR("Jana Saout <jana@saout.de>"); 30431da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 30441da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 3045