11da177e4SLinus Torvalds /* 2bf14299fSJana Saout * Copyright (C) 2003 Jana Saout <jana@saout.de> 31da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4bbb16584SMilan Broz * Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved. 5bbb16584SMilan Broz * Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com> 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * This file is released under the GPL. 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds 1043d69034SMilan Broz #include <linux/completion.h> 11d1806f6aSHerbert Xu #include <linux/err.h> 121da177e4SLinus Torvalds #include <linux/module.h> 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/kernel.h> 15c538f6ecSOndrej Kozina #include <linux/key.h> 161da177e4SLinus Torvalds #include <linux/bio.h> 171da177e4SLinus Torvalds #include <linux/blkdev.h> 181da177e4SLinus Torvalds #include <linux/mempool.h> 191da177e4SLinus Torvalds #include <linux/slab.h> 201da177e4SLinus Torvalds #include <linux/crypto.h> 211da177e4SLinus Torvalds #include <linux/workqueue.h> 22dc267621SMikulas Patocka #include <linux/kthread.h> 233fcfab16SAndrew Morton #include <linux/backing-dev.h> 2460063497SArun Sharma #include <linux/atomic.h> 25378f058cSDavid Hardeman #include <linux/scatterlist.h> 26b3c5fd30SMikulas Patocka #include <linux/rbtree.h> 27027c431cSOndrej Kozina #include <linux/ctype.h> 281da177e4SLinus Torvalds #include <asm/page.h> 2948527fa7SRik Snel #include <asm/unaligned.h> 3034745785SMilan Broz #include <crypto/hash.h> 3134745785SMilan Broz #include <crypto/md5.h> 3234745785SMilan Broz #include <crypto/algapi.h> 33bbdb23b5SHerbert Xu #include <crypto/skcipher.h> 34ef43aa38SMilan Broz #include <crypto/aead.h> 35ef43aa38SMilan Broz #include <crypto/authenc.h> 36ef43aa38SMilan Broz #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */ 3727f5411aSDmitry Baryshkov #include <linux/key-type.h> 38c538f6ecSOndrej Kozina #include <keys/user-type.h> 3927f5411aSDmitry Baryshkov #include <keys/encrypted-type.h> 401da177e4SLinus Torvalds 41586e80e6SMikulas Patocka #include <linux/device-mapper.h> 421da177e4SLinus Torvalds 4372d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds /* 461da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 471da177e4SLinus Torvalds */ 481da177e4SLinus Torvalds struct convert_context { 4943d69034SMilan Broz struct completion restart; 501da177e4SLinus Torvalds struct bio *bio_in; 511da177e4SLinus Torvalds struct bio *bio_out; 52003b5c57SKent Overstreet struct bvec_iter iter_in; 53003b5c57SKent Overstreet struct bvec_iter iter_out; 548d683dcdSAliOS system security u64 cc_sector; 5540b6229bSMikulas Patocka atomic_t cc_pending; 56ef43aa38SMilan Broz union { 57bbdb23b5SHerbert Xu struct skcipher_request *req; 58ef43aa38SMilan Broz struct aead_request *req_aead; 59ef43aa38SMilan Broz } r; 60ef43aa38SMilan Broz 611da177e4SLinus Torvalds }; 621da177e4SLinus Torvalds 6353017030SMilan Broz /* 6453017030SMilan Broz * per bio private data 6553017030SMilan Broz */ 6653017030SMilan Broz struct dm_crypt_io { 6749a8a920SAlasdair G Kergon struct crypt_config *cc; 6853017030SMilan Broz struct bio *base_bio; 69ef43aa38SMilan Broz u8 *integrity_metadata; 70ef43aa38SMilan Broz bool integrity_metadata_from_pool; 7153017030SMilan Broz struct work_struct work; 7239d42fa9SIgnat Korchagin struct tasklet_struct tasklet; 7353017030SMilan Broz 7453017030SMilan Broz struct convert_context ctx; 7553017030SMilan Broz 7640b6229bSMikulas Patocka atomic_t io_pending; 774e4cbee9SChristoph Hellwig blk_status_t error; 780c395b0fSMilan Broz sector_t sector; 79dc267621SMikulas Patocka 80b3c5fd30SMikulas Patocka struct rb_node rb_node; 81298a9fa0SMikulas Patocka } CRYPTO_MINALIGN_ATTR; 8253017030SMilan Broz 8301482b76SMilan Broz struct dm_crypt_request { 84b2174eebSHuang Ying struct convert_context *ctx; 85ef43aa38SMilan Broz struct scatterlist sg_in[4]; 86ef43aa38SMilan Broz struct scatterlist sg_out[4]; 878d683dcdSAliOS system security u64 iv_sector; 8801482b76SMilan Broz }; 8901482b76SMilan Broz 901da177e4SLinus Torvalds struct crypt_config; 911da177e4SLinus Torvalds 921da177e4SLinus Torvalds struct crypt_iv_operations { 931da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 941da177e4SLinus Torvalds const char *opts); 951da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 96b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 97542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 982dc5327dSMilan Broz int (*generator)(struct crypt_config *cc, u8 *iv, 992dc5327dSMilan Broz struct dm_crypt_request *dmreq); 1002dc5327dSMilan Broz int (*post)(struct crypt_config *cc, u8 *iv, 1012dc5327dSMilan Broz struct dm_crypt_request *dmreq); 1021da177e4SLinus Torvalds }; 1031da177e4SLinus Torvalds 10460473592SMilan Broz struct iv_benbi_private { 10560473592SMilan Broz int shift; 10660473592SMilan Broz }; 10760473592SMilan Broz 10834745785SMilan Broz #define LMK_SEED_SIZE 64 /* hash + 0 */ 10934745785SMilan Broz struct iv_lmk_private { 11034745785SMilan Broz struct crypto_shash *hash_tfm; 11134745785SMilan Broz u8 *seed; 11234745785SMilan Broz }; 11334745785SMilan Broz 114ed04d981SMilan Broz #define TCW_WHITENING_SIZE 16 115ed04d981SMilan Broz struct iv_tcw_private { 116ed04d981SMilan Broz struct crypto_shash *crc32_tfm; 117ed04d981SMilan Broz u8 *iv_seed; 118ed04d981SMilan Broz u8 *whitening; 119ed04d981SMilan Broz }; 120ed04d981SMilan Broz 121bbb16584SMilan Broz #define ELEPHANT_MAX_KEY_SIZE 32 122bbb16584SMilan Broz struct iv_elephant_private { 123bbb16584SMilan Broz struct crypto_skcipher *tfm; 124bbb16584SMilan Broz }; 125bbb16584SMilan Broz 1261da177e4SLinus Torvalds /* 1271da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 1281da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 1291da177e4SLinus Torvalds */ 1300f5d8e6eSMikulas Patocka enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID, 13139d42fa9SIgnat Korchagin DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD, 13239d42fa9SIgnat Korchagin DM_CRYPT_NO_READ_WORKQUEUE, DM_CRYPT_NO_WRITE_WORKQUEUE }; 133c0297721SAndi Kleen 134ef43aa38SMilan Broz enum cipher_flags { 135ef43aa38SMilan Broz CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cihper */ 1368f0009a2SMilan Broz CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */ 137bbb16584SMilan Broz CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */ 138ef43aa38SMilan Broz }; 139ef43aa38SMilan Broz 140c0297721SAndi Kleen /* 141610f2de3SMikulas Patocka * The fields in here must be read only after initialization. 142c0297721SAndi Kleen */ 1431da177e4SLinus Torvalds struct crypt_config { 1441da177e4SLinus Torvalds struct dm_dev *dev; 1451da177e4SLinus Torvalds sector_t start; 1461da177e4SLinus Torvalds 1475059353dSMikulas Patocka struct percpu_counter n_allocated_pages; 1485059353dSMikulas Patocka 149cabf08e4SMilan Broz struct workqueue_struct *io_queue; 150cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1513f1e9070SMilan Broz 152c7329effSMikulas Patocka spinlock_t write_thread_lock; 15372d711c8SMike Snitzer struct task_struct *write_thread; 154b3c5fd30SMikulas Patocka struct rb_root write_tree; 155dc267621SMikulas Patocka 1567dbcd137SMilan Broz char *cipher_string; 157ef43aa38SMilan Broz char *cipher_auth; 158c538f6ecSOndrej Kozina char *key_string; 1595ebaee6dSMilan Broz 1601b1b58f5SJulia Lawall const struct crypt_iv_operations *iv_gen_ops; 16179066ad3SHerbert Xu union { 16260473592SMilan Broz struct iv_benbi_private benbi; 16334745785SMilan Broz struct iv_lmk_private lmk; 164ed04d981SMilan Broz struct iv_tcw_private tcw; 165bbb16584SMilan Broz struct iv_elephant_private elephant; 16679066ad3SHerbert Xu } iv_gen_private; 1678d683dcdSAliOS system security u64 iv_offset; 1681da177e4SLinus Torvalds unsigned int iv_size; 169ff3af92bSMikulas Patocka unsigned short int sector_size; 170ff3af92bSMikulas Patocka unsigned char sector_shift; 1711da177e4SLinus Torvalds 172ef43aa38SMilan Broz union { 173bbdb23b5SHerbert Xu struct crypto_skcipher **tfms; 174ef43aa38SMilan Broz struct crypto_aead **tfms_aead; 175ef43aa38SMilan Broz } cipher_tfm; 176d1f96423SMilan Broz unsigned tfms_count; 177ef43aa38SMilan Broz unsigned long cipher_flags; 178c0297721SAndi Kleen 179c0297721SAndi Kleen /* 180ddd42edfSMilan Broz * Layout of each crypto request: 181ddd42edfSMilan Broz * 182bbdb23b5SHerbert Xu * struct skcipher_request 183ddd42edfSMilan Broz * context 184ddd42edfSMilan Broz * padding 185ddd42edfSMilan Broz * struct dm_crypt_request 186ddd42edfSMilan Broz * padding 187ddd42edfSMilan Broz * IV 188ddd42edfSMilan Broz * 189ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 190ddd42edfSMilan Broz * correctly aligned. 191ddd42edfSMilan Broz */ 192ddd42edfSMilan Broz unsigned int dmreq_start; 193ddd42edfSMilan Broz 194298a9fa0SMikulas Patocka unsigned int per_bio_data_size; 195298a9fa0SMikulas Patocka 196e48d4bbfSMilan Broz unsigned long flags; 1971da177e4SLinus Torvalds unsigned int key_size; 198da31a078SMilan Broz unsigned int key_parts; /* independent parts in key buffer */ 199da31a078SMilan Broz unsigned int key_extra_size; /* additional keys length */ 200ef43aa38SMilan Broz unsigned int key_mac_size; /* MAC key size for authenc(...) */ 201ef43aa38SMilan Broz 202ef43aa38SMilan Broz unsigned int integrity_tag_size; 203ef43aa38SMilan Broz unsigned int integrity_iv_size; 204ef43aa38SMilan Broz unsigned int on_disk_tag_size; 205ef43aa38SMilan Broz 20672d711c8SMike Snitzer /* 20772d711c8SMike Snitzer * pool for per bio private data, crypto requests, 20872d711c8SMike Snitzer * encryption requeusts/buffer pages and integrity tags 20972d711c8SMike Snitzer */ 21072d711c8SMike Snitzer unsigned tag_pool_max_sectors; 21172d711c8SMike Snitzer mempool_t tag_pool; 21272d711c8SMike Snitzer mempool_t req_pool; 21372d711c8SMike Snitzer mempool_t page_pool; 21472d711c8SMike Snitzer 21572d711c8SMike Snitzer struct bio_set bs; 21672d711c8SMike Snitzer struct mutex bio_alloc_lock; 21772d711c8SMike Snitzer 218ef43aa38SMilan Broz u8 *authenc_key; /* space for keys in authenc() format (if used) */ 219b18ae8ddSGustavo A. R. Silva u8 key[]; 2201da177e4SLinus Torvalds }; 2211da177e4SLinus Torvalds 2220a83df6cSMikulas Patocka #define MIN_IOS 64 223ef43aa38SMilan Broz #define MAX_TAG_SIZE 480 224ef43aa38SMilan Broz #define POOL_ENTRY_SIZE 512 2251da177e4SLinus Torvalds 2265059353dSMikulas Patocka static DEFINE_SPINLOCK(dm_crypt_clients_lock); 2275059353dSMikulas Patocka static unsigned dm_crypt_clients_n = 0; 2285059353dSMikulas Patocka static volatile unsigned long dm_crypt_pages_per_client; 2295059353dSMikulas Patocka #define DM_CRYPT_MEMORY_PERCENT 2 2305059353dSMikulas Patocka #define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_PAGES * 16) 2315059353dSMikulas Patocka 232028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *, struct bio *); 233395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 234ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc, 235ef43aa38SMilan Broz struct scatterlist *sg); 236027581f3SOlaf Kirch 2373fd53533SYang Yingliang static bool crypt_integrity_aead(struct crypt_config *cc); 2383fd53533SYang Yingliang 239c0297721SAndi Kleen /* 24086f917adSEric Biggers * Use this to access cipher attributes that are independent of the key. 241c0297721SAndi Kleen */ 242bbdb23b5SHerbert Xu static struct crypto_skcipher *any_tfm(struct crypt_config *cc) 243c0297721SAndi Kleen { 244ef43aa38SMilan Broz return cc->cipher_tfm.tfms[0]; 245ef43aa38SMilan Broz } 246ef43aa38SMilan Broz 247ef43aa38SMilan Broz static struct crypto_aead *any_tfm_aead(struct crypt_config *cc) 248ef43aa38SMilan Broz { 249ef43aa38SMilan Broz return cc->cipher_tfm.tfms_aead[0]; 250c0297721SAndi Kleen } 251c0297721SAndi Kleen 2521da177e4SLinus Torvalds /* 2531da177e4SLinus Torvalds * Different IV generation algorithms: 2541da177e4SLinus Torvalds * 2553c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 2563a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 2571da177e4SLinus Torvalds * 25861afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 25961afef61SMilan Broz * number, padded with zeros if necessary. 26061afef61SMilan Broz * 2617e3fd855SMilan Broz * plain64be: the initial vector is the 64-bit big-endian version of the sector 2627e3fd855SMilan Broz * number, padded with zeros if necessary. 2637e3fd855SMilan Broz * 2643c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 2651da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 2661da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 2671da177e4SLinus Torvalds * 26848527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 26948527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 27048527fa7SRik Snel * 27146b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 27246b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 27346b47730SLudwig Nussel * 27434745785SMilan Broz * lmk: Compatible implementation of the block chaining mode used 27534745785SMilan Broz * by the Loop-AES block device encryption system 27634745785SMilan Broz * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/ 27734745785SMilan Broz * It operates on full 512 byte sectors and uses CBC 27834745785SMilan Broz * with an IV derived from the sector number, the data and 27934745785SMilan Broz * optionally extra IV seed. 28034745785SMilan Broz * This means that after decryption the first block 28134745785SMilan Broz * of sector must be tweaked according to decrypted data. 28234745785SMilan Broz * Loop-AES can use three encryption schemes: 28334745785SMilan Broz * version 1: is plain aes-cbc mode 28434745785SMilan Broz * version 2: uses 64 multikey scheme with lmk IV generator 28534745785SMilan Broz * version 3: the same as version 2 with additional IV seed 28634745785SMilan Broz * (it uses 65 keys, last key is used as IV seed) 28734745785SMilan Broz * 288ed04d981SMilan Broz * tcw: Compatible implementation of the block chaining mode used 289ed04d981SMilan Broz * by the TrueCrypt device encryption system (prior to version 4.1). 290e44f23b3SMilan Broz * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat 291ed04d981SMilan Broz * It operates on full 512 byte sectors and uses CBC 292ed04d981SMilan Broz * with an IV derived from initial key and the sector number. 293ed04d981SMilan Broz * In addition, whitening value is applied on every sector, whitening 294ed04d981SMilan Broz * is calculated from initial key, sector number and mixed using CRC32. 295ed04d981SMilan Broz * Note that this encryption scheme is vulnerable to watermarking attacks 296ed04d981SMilan Broz * and should be used for old compatible containers access only. 297b9411d73SMilan Broz * 298b9411d73SMilan Broz * eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode) 299b9411d73SMilan Broz * The IV is encrypted little-endian byte-offset (with the same key 300b9411d73SMilan Broz * and cipher as the volume). 301bbb16584SMilan Broz * 302bbb16584SMilan Broz * elephant: The extended version of eboiv with additional Elephant diffuser 303bbb16584SMilan Broz * used with Bitlocker CBC mode. 304bbb16584SMilan Broz * This mode was used in older Windows systems 305bbb16584SMilan Broz * http://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf 3061da177e4SLinus Torvalds */ 3071da177e4SLinus Torvalds 3082dc5327dSMilan Broz static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, 3092dc5327dSMilan Broz struct dm_crypt_request *dmreq) 3101da177e4SLinus Torvalds { 3111da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 312283a8328SAlasdair G Kergon *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff); 3131da177e4SLinus Torvalds 3141da177e4SLinus Torvalds return 0; 3151da177e4SLinus Torvalds } 3161da177e4SLinus Torvalds 31761afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 3182dc5327dSMilan Broz struct dm_crypt_request *dmreq) 31961afef61SMilan Broz { 32061afef61SMilan Broz memset(iv, 0, cc->iv_size); 321283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 32261afef61SMilan Broz 32361afef61SMilan Broz return 0; 32461afef61SMilan Broz } 32561afef61SMilan Broz 3267e3fd855SMilan Broz static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv, 3277e3fd855SMilan Broz struct dm_crypt_request *dmreq) 3287e3fd855SMilan Broz { 3297e3fd855SMilan Broz memset(iv, 0, cc->iv_size); 3307e3fd855SMilan Broz /* iv_size is at least of size u64; usually it is 16 bytes */ 3317e3fd855SMilan Broz *(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector); 3327e3fd855SMilan Broz 3337e3fd855SMilan Broz return 0; 3347e3fd855SMilan Broz } 3357e3fd855SMilan Broz 3362dc5327dSMilan Broz static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, 3372dc5327dSMilan Broz struct dm_crypt_request *dmreq) 3381da177e4SLinus Torvalds { 339a1a262b6SArd Biesheuvel /* 340a1a262b6SArd Biesheuvel * ESSIV encryption of the IV is now handled by the crypto API, 341a1a262b6SArd Biesheuvel * so just pass the plain sector number here. 342a1a262b6SArd Biesheuvel */ 3431da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 344283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 345c0297721SAndi Kleen 3461da177e4SLinus Torvalds return 0; 3471da177e4SLinus Torvalds } 3481da177e4SLinus Torvalds 34948527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 35048527fa7SRik Snel const char *opts) 35148527fa7SRik Snel { 3524ea9471fSMilan Broz unsigned bs; 3534ea9471fSMilan Broz int log; 3544ea9471fSMilan Broz 3553fd53533SYang Yingliang if (crypt_integrity_aead(cc)) 3564ea9471fSMilan Broz bs = crypto_aead_blocksize(any_tfm_aead(cc)); 3574ea9471fSMilan Broz else 3584ea9471fSMilan Broz bs = crypto_skcipher_blocksize(any_tfm(cc)); 3594ea9471fSMilan Broz log = ilog2(bs); 36048527fa7SRik Snel 36148527fa7SRik Snel /* we need to calculate how far we must shift the sector count 36248527fa7SRik Snel * to get the cipher block count, we use this shift in _gen */ 36348527fa7SRik Snel 36448527fa7SRik Snel if (1 << log != bs) { 36548527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 36648527fa7SRik Snel return -EINVAL; 36748527fa7SRik Snel } 36848527fa7SRik Snel 36948527fa7SRik Snel if (log > 9) { 37048527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 37148527fa7SRik Snel return -EINVAL; 37248527fa7SRik Snel } 37348527fa7SRik Snel 37460473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 37548527fa7SRik Snel 37648527fa7SRik Snel return 0; 37748527fa7SRik Snel } 37848527fa7SRik Snel 37948527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 38048527fa7SRik Snel { 38148527fa7SRik Snel } 38248527fa7SRik Snel 3832dc5327dSMilan Broz static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, 3842dc5327dSMilan Broz struct dm_crypt_request *dmreq) 38548527fa7SRik Snel { 38679066ad3SHerbert Xu __be64 val; 38779066ad3SHerbert Xu 38848527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 38979066ad3SHerbert Xu 3902dc5327dSMilan Broz val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1); 39179066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 39248527fa7SRik Snel 3931da177e4SLinus Torvalds return 0; 3941da177e4SLinus Torvalds } 3951da177e4SLinus Torvalds 3962dc5327dSMilan Broz static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, 3972dc5327dSMilan Broz struct dm_crypt_request *dmreq) 39846b47730SLudwig Nussel { 39946b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 40046b47730SLudwig Nussel 40146b47730SLudwig Nussel return 0; 40246b47730SLudwig Nussel } 40346b47730SLudwig Nussel 40434745785SMilan Broz static void crypt_iv_lmk_dtr(struct crypt_config *cc) 40534745785SMilan Broz { 40634745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 40734745785SMilan Broz 40834745785SMilan Broz if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm)) 40934745785SMilan Broz crypto_free_shash(lmk->hash_tfm); 41034745785SMilan Broz lmk->hash_tfm = NULL; 41134745785SMilan Broz 41234745785SMilan Broz kzfree(lmk->seed); 41334745785SMilan Broz lmk->seed = NULL; 41434745785SMilan Broz } 41534745785SMilan Broz 41634745785SMilan Broz static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti, 41734745785SMilan Broz const char *opts) 41834745785SMilan Broz { 41934745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 42034745785SMilan Broz 4218f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) { 4228f0009a2SMilan Broz ti->error = "Unsupported sector size for LMK"; 4238f0009a2SMilan Broz return -EINVAL; 4248f0009a2SMilan Broz } 4258f0009a2SMilan Broz 42634745785SMilan Broz lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0); 42734745785SMilan Broz if (IS_ERR(lmk->hash_tfm)) { 42834745785SMilan Broz ti->error = "Error initializing LMK hash"; 42934745785SMilan Broz return PTR_ERR(lmk->hash_tfm); 43034745785SMilan Broz } 43134745785SMilan Broz 43234745785SMilan Broz /* No seed in LMK version 2 */ 43334745785SMilan Broz if (cc->key_parts == cc->tfms_count) { 43434745785SMilan Broz lmk->seed = NULL; 43534745785SMilan Broz return 0; 43634745785SMilan Broz } 43734745785SMilan Broz 43834745785SMilan Broz lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL); 43934745785SMilan Broz if (!lmk->seed) { 44034745785SMilan Broz crypt_iv_lmk_dtr(cc); 44134745785SMilan Broz ti->error = "Error kmallocing seed storage in LMK"; 44234745785SMilan Broz return -ENOMEM; 44334745785SMilan Broz } 44434745785SMilan Broz 44534745785SMilan Broz return 0; 44634745785SMilan Broz } 44734745785SMilan Broz 44834745785SMilan Broz static int crypt_iv_lmk_init(struct crypt_config *cc) 44934745785SMilan Broz { 45034745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 45134745785SMilan Broz int subkey_size = cc->key_size / cc->key_parts; 45234745785SMilan Broz 45334745785SMilan Broz /* LMK seed is on the position of LMK_KEYS + 1 key */ 45434745785SMilan Broz if (lmk->seed) 45534745785SMilan Broz memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size), 45634745785SMilan Broz crypto_shash_digestsize(lmk->hash_tfm)); 45734745785SMilan Broz 45834745785SMilan Broz return 0; 45934745785SMilan Broz } 46034745785SMilan Broz 46134745785SMilan Broz static int crypt_iv_lmk_wipe(struct crypt_config *cc) 46234745785SMilan Broz { 46334745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 46434745785SMilan Broz 46534745785SMilan Broz if (lmk->seed) 46634745785SMilan Broz memset(lmk->seed, 0, LMK_SEED_SIZE); 46734745785SMilan Broz 46834745785SMilan Broz return 0; 46934745785SMilan Broz } 47034745785SMilan Broz 47134745785SMilan Broz static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv, 47234745785SMilan Broz struct dm_crypt_request *dmreq, 47334745785SMilan Broz u8 *data) 47434745785SMilan Broz { 47534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 476b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, lmk->hash_tfm); 47734745785SMilan Broz struct md5_state md5state; 478da31a078SMilan Broz __le32 buf[4]; 47934745785SMilan Broz int i, r; 48034745785SMilan Broz 481b6106265SJan-Simon Möller desc->tfm = lmk->hash_tfm; 48234745785SMilan Broz 483b6106265SJan-Simon Möller r = crypto_shash_init(desc); 48434745785SMilan Broz if (r) 48534745785SMilan Broz return r; 48634745785SMilan Broz 48734745785SMilan Broz if (lmk->seed) { 488b6106265SJan-Simon Möller r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE); 48934745785SMilan Broz if (r) 49034745785SMilan Broz return r; 49134745785SMilan Broz } 49234745785SMilan Broz 49334745785SMilan Broz /* Sector is always 512B, block size 16, add data of blocks 1-31 */ 494b6106265SJan-Simon Möller r = crypto_shash_update(desc, data + 16, 16 * 31); 49534745785SMilan Broz if (r) 49634745785SMilan Broz return r; 49734745785SMilan Broz 49834745785SMilan Broz /* Sector is cropped to 56 bits here */ 49934745785SMilan Broz buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF); 50034745785SMilan Broz buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000); 50134745785SMilan Broz buf[2] = cpu_to_le32(4024); 50234745785SMilan Broz buf[3] = 0; 503b6106265SJan-Simon Möller r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf)); 50434745785SMilan Broz if (r) 50534745785SMilan Broz return r; 50634745785SMilan Broz 50734745785SMilan Broz /* No MD5 padding here */ 508b6106265SJan-Simon Möller r = crypto_shash_export(desc, &md5state); 50934745785SMilan Broz if (r) 51034745785SMilan Broz return r; 51134745785SMilan Broz 51234745785SMilan Broz for (i = 0; i < MD5_HASH_WORDS; i++) 51334745785SMilan Broz __cpu_to_le32s(&md5state.hash[i]); 51434745785SMilan Broz memcpy(iv, &md5state.hash, cc->iv_size); 51534745785SMilan Broz 51634745785SMilan Broz return 0; 51734745785SMilan Broz } 51834745785SMilan Broz 51934745785SMilan Broz static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv, 52034745785SMilan Broz struct dm_crypt_request *dmreq) 52134745785SMilan Broz { 522ef43aa38SMilan Broz struct scatterlist *sg; 52334745785SMilan Broz u8 *src; 52434745785SMilan Broz int r = 0; 52534745785SMilan Broz 52634745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 527ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in); 528ef43aa38SMilan Broz src = kmap_atomic(sg_page(sg)); 529ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset); 530c2e022cbSCong Wang kunmap_atomic(src); 53134745785SMilan Broz } else 53234745785SMilan Broz memset(iv, 0, cc->iv_size); 53334745785SMilan Broz 53434745785SMilan Broz return r; 53534745785SMilan Broz } 53634745785SMilan Broz 53734745785SMilan Broz static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv, 53834745785SMilan Broz struct dm_crypt_request *dmreq) 53934745785SMilan Broz { 540ef43aa38SMilan Broz struct scatterlist *sg; 54134745785SMilan Broz u8 *dst; 54234745785SMilan Broz int r; 54334745785SMilan Broz 54434745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) 54534745785SMilan Broz return 0; 54634745785SMilan Broz 547ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 548ef43aa38SMilan Broz dst = kmap_atomic(sg_page(sg)); 549ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset); 55034745785SMilan Broz 55134745785SMilan Broz /* Tweak the first block of plaintext sector */ 55234745785SMilan Broz if (!r) 553ef43aa38SMilan Broz crypto_xor(dst + sg->offset, iv, cc->iv_size); 55434745785SMilan Broz 555c2e022cbSCong Wang kunmap_atomic(dst); 55634745785SMilan Broz return r; 55734745785SMilan Broz } 55834745785SMilan Broz 559ed04d981SMilan Broz static void crypt_iv_tcw_dtr(struct crypt_config *cc) 560ed04d981SMilan Broz { 561ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 562ed04d981SMilan Broz 563ed04d981SMilan Broz kzfree(tcw->iv_seed); 564ed04d981SMilan Broz tcw->iv_seed = NULL; 565ed04d981SMilan Broz kzfree(tcw->whitening); 566ed04d981SMilan Broz tcw->whitening = NULL; 567ed04d981SMilan Broz 568ed04d981SMilan Broz if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm)) 569ed04d981SMilan Broz crypto_free_shash(tcw->crc32_tfm); 570ed04d981SMilan Broz tcw->crc32_tfm = NULL; 571ed04d981SMilan Broz } 572ed04d981SMilan Broz 573ed04d981SMilan Broz static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti, 574ed04d981SMilan Broz const char *opts) 575ed04d981SMilan Broz { 576ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 577ed04d981SMilan Broz 5788f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) { 5798f0009a2SMilan Broz ti->error = "Unsupported sector size for TCW"; 5808f0009a2SMilan Broz return -EINVAL; 5818f0009a2SMilan Broz } 5828f0009a2SMilan Broz 583ed04d981SMilan Broz if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) { 584ed04d981SMilan Broz ti->error = "Wrong key size for TCW"; 585ed04d981SMilan Broz return -EINVAL; 586ed04d981SMilan Broz } 587ed04d981SMilan Broz 588ed04d981SMilan Broz tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 0); 589ed04d981SMilan Broz if (IS_ERR(tcw->crc32_tfm)) { 590ed04d981SMilan Broz ti->error = "Error initializing CRC32 in TCW"; 591ed04d981SMilan Broz return PTR_ERR(tcw->crc32_tfm); 592ed04d981SMilan Broz } 593ed04d981SMilan Broz 594ed04d981SMilan Broz tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL); 595ed04d981SMilan Broz tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL); 596ed04d981SMilan Broz if (!tcw->iv_seed || !tcw->whitening) { 597ed04d981SMilan Broz crypt_iv_tcw_dtr(cc); 598ed04d981SMilan Broz ti->error = "Error allocating seed storage in TCW"; 599ed04d981SMilan Broz return -ENOMEM; 600ed04d981SMilan Broz } 601ed04d981SMilan Broz 602ed04d981SMilan Broz return 0; 603ed04d981SMilan Broz } 604ed04d981SMilan Broz 605ed04d981SMilan Broz static int crypt_iv_tcw_init(struct crypt_config *cc) 606ed04d981SMilan Broz { 607ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 608ed04d981SMilan Broz int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE; 609ed04d981SMilan Broz 610ed04d981SMilan Broz memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size); 611ed04d981SMilan Broz memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size], 612ed04d981SMilan Broz TCW_WHITENING_SIZE); 613ed04d981SMilan Broz 614ed04d981SMilan Broz return 0; 615ed04d981SMilan Broz } 616ed04d981SMilan Broz 617ed04d981SMilan Broz static int crypt_iv_tcw_wipe(struct crypt_config *cc) 618ed04d981SMilan Broz { 619ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 620ed04d981SMilan Broz 621ed04d981SMilan Broz memset(tcw->iv_seed, 0, cc->iv_size); 622ed04d981SMilan Broz memset(tcw->whitening, 0, TCW_WHITENING_SIZE); 623ed04d981SMilan Broz 624ed04d981SMilan Broz return 0; 625ed04d981SMilan Broz } 626ed04d981SMilan Broz 627ed04d981SMilan Broz static int crypt_iv_tcw_whitening(struct crypt_config *cc, 628ed04d981SMilan Broz struct dm_crypt_request *dmreq, 629ed04d981SMilan Broz u8 *data) 630ed04d981SMilan Broz { 631ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 632350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector); 633ed04d981SMilan Broz u8 buf[TCW_WHITENING_SIZE]; 634b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm); 635ed04d981SMilan Broz int i, r; 636ed04d981SMilan Broz 637ed04d981SMilan Broz /* xor whitening with sector number */ 63845fe93dfSArd Biesheuvel crypto_xor_cpy(buf, tcw->whitening, (u8 *)§or, 8); 63945fe93dfSArd Biesheuvel crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)§or, 8); 640ed04d981SMilan Broz 641ed04d981SMilan Broz /* calculate crc32 for every 32bit part and xor it */ 642b6106265SJan-Simon Möller desc->tfm = tcw->crc32_tfm; 643ed04d981SMilan Broz for (i = 0; i < 4; i++) { 644b6106265SJan-Simon Möller r = crypto_shash_init(desc); 645ed04d981SMilan Broz if (r) 646ed04d981SMilan Broz goto out; 647b6106265SJan-Simon Möller r = crypto_shash_update(desc, &buf[i * 4], 4); 648ed04d981SMilan Broz if (r) 649ed04d981SMilan Broz goto out; 650b6106265SJan-Simon Möller r = crypto_shash_final(desc, &buf[i * 4]); 651ed04d981SMilan Broz if (r) 652ed04d981SMilan Broz goto out; 653ed04d981SMilan Broz } 654ed04d981SMilan Broz crypto_xor(&buf[0], &buf[12], 4); 655ed04d981SMilan Broz crypto_xor(&buf[4], &buf[8], 4); 656ed04d981SMilan Broz 657ed04d981SMilan Broz /* apply whitening (8 bytes) to whole sector */ 658ed04d981SMilan Broz for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++) 659ed04d981SMilan Broz crypto_xor(data + i * 8, buf, 8); 660ed04d981SMilan Broz out: 6611a71d6ffSMilan Broz memzero_explicit(buf, sizeof(buf)); 662ed04d981SMilan Broz return r; 663ed04d981SMilan Broz } 664ed04d981SMilan Broz 665ed04d981SMilan Broz static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv, 666ed04d981SMilan Broz struct dm_crypt_request *dmreq) 667ed04d981SMilan Broz { 668ef43aa38SMilan Broz struct scatterlist *sg; 669ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 670350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector); 671ed04d981SMilan Broz u8 *src; 672ed04d981SMilan Broz int r = 0; 673ed04d981SMilan Broz 674ed04d981SMilan Broz /* Remove whitening from ciphertext */ 675ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) { 676ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in); 677ef43aa38SMilan Broz src = kmap_atomic(sg_page(sg)); 678ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset); 679ed04d981SMilan Broz kunmap_atomic(src); 680ed04d981SMilan Broz } 681ed04d981SMilan Broz 682ed04d981SMilan Broz /* Calculate IV */ 68345fe93dfSArd Biesheuvel crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)§or, 8); 684ed04d981SMilan Broz if (cc->iv_size > 8) 68545fe93dfSArd Biesheuvel crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)§or, 68645fe93dfSArd Biesheuvel cc->iv_size - 8); 687ed04d981SMilan Broz 688ed04d981SMilan Broz return r; 689ed04d981SMilan Broz } 690ed04d981SMilan Broz 691ed04d981SMilan Broz static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv, 692ed04d981SMilan Broz struct dm_crypt_request *dmreq) 693ed04d981SMilan Broz { 694ef43aa38SMilan Broz struct scatterlist *sg; 695ed04d981SMilan Broz u8 *dst; 696ed04d981SMilan Broz int r; 697ed04d981SMilan Broz 698ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) 699ed04d981SMilan Broz return 0; 700ed04d981SMilan Broz 701ed04d981SMilan Broz /* Apply whitening on ciphertext */ 702ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 703ef43aa38SMilan Broz dst = kmap_atomic(sg_page(sg)); 704ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset); 705ed04d981SMilan Broz kunmap_atomic(dst); 706ed04d981SMilan Broz 707ed04d981SMilan Broz return r; 708ed04d981SMilan Broz } 709ed04d981SMilan Broz 710ef43aa38SMilan Broz static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv, 711ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 712ef43aa38SMilan Broz { 713ef43aa38SMilan Broz /* Used only for writes, there must be an additional space to store IV */ 714ef43aa38SMilan Broz get_random_bytes(iv, cc->iv_size); 715ef43aa38SMilan Broz return 0; 716ef43aa38SMilan Broz } 717ef43aa38SMilan Broz 718b9411d73SMilan Broz static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti, 719b9411d73SMilan Broz const char *opts) 720b9411d73SMilan Broz { 7213fd53533SYang Yingliang if (crypt_integrity_aead(cc)) { 72239d13a1aSArd Biesheuvel ti->error = "AEAD transforms not supported for EBOIV"; 723b9411d73SMilan Broz return -EINVAL; 724b9411d73SMilan Broz } 725b9411d73SMilan Broz 72639d13a1aSArd Biesheuvel if (crypto_skcipher_blocksize(any_tfm(cc)) != cc->iv_size) { 72739d13a1aSArd Biesheuvel ti->error = "Block size of EBOIV cipher does " 72839d13a1aSArd Biesheuvel "not match IV size of block cipher"; 72939d13a1aSArd Biesheuvel return -EINVAL; 730b9411d73SMilan Broz } 731b9411d73SMilan Broz 732b9411d73SMilan Broz return 0; 733b9411d73SMilan Broz } 734b9411d73SMilan Broz 735b9411d73SMilan Broz static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv, 736b9411d73SMilan Broz struct dm_crypt_request *dmreq) 737b9411d73SMilan Broz { 73839d13a1aSArd Biesheuvel u8 buf[MAX_CIPHER_BLOCKSIZE] __aligned(__alignof__(__le64)); 73939d13a1aSArd Biesheuvel struct skcipher_request *req; 74039d13a1aSArd Biesheuvel struct scatterlist src, dst; 74139d13a1aSArd Biesheuvel struct crypto_wait wait; 74239d13a1aSArd Biesheuvel int err; 743b9411d73SMilan Broz 7449402e959SMikulas Patocka req = skcipher_request_alloc(any_tfm(cc), GFP_NOIO); 74539d13a1aSArd Biesheuvel if (!req) 74639d13a1aSArd Biesheuvel return -ENOMEM; 747b9411d73SMilan Broz 74839d13a1aSArd Biesheuvel memset(buf, 0, cc->iv_size); 74939d13a1aSArd Biesheuvel *(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size); 75039d13a1aSArd Biesheuvel 75139d13a1aSArd Biesheuvel sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size); 75239d13a1aSArd Biesheuvel sg_init_one(&dst, iv, cc->iv_size); 75339d13a1aSArd Biesheuvel skcipher_request_set_crypt(req, &src, &dst, cc->iv_size, buf); 75439d13a1aSArd Biesheuvel skcipher_request_set_callback(req, 0, crypto_req_done, &wait); 75539d13a1aSArd Biesheuvel err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); 75639d13a1aSArd Biesheuvel skcipher_request_free(req); 75739d13a1aSArd Biesheuvel 75839d13a1aSArd Biesheuvel return err; 759b9411d73SMilan Broz } 760b9411d73SMilan Broz 761bbb16584SMilan Broz static void crypt_iv_elephant_dtr(struct crypt_config *cc) 762bbb16584SMilan Broz { 763bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 764bbb16584SMilan Broz 765bbb16584SMilan Broz crypto_free_skcipher(elephant->tfm); 766bbb16584SMilan Broz elephant->tfm = NULL; 767bbb16584SMilan Broz } 768bbb16584SMilan Broz 769bbb16584SMilan Broz static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti, 770bbb16584SMilan Broz const char *opts) 771bbb16584SMilan Broz { 772bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 773bbb16584SMilan Broz int r; 774bbb16584SMilan Broz 775bbb16584SMilan Broz elephant->tfm = crypto_alloc_skcipher("ecb(aes)", 0, 0); 776bbb16584SMilan Broz if (IS_ERR(elephant->tfm)) { 777bbb16584SMilan Broz r = PTR_ERR(elephant->tfm); 778bbb16584SMilan Broz elephant->tfm = NULL; 779bbb16584SMilan Broz return r; 780bbb16584SMilan Broz } 781bbb16584SMilan Broz 782bbb16584SMilan Broz r = crypt_iv_eboiv_ctr(cc, ti, NULL); 783bbb16584SMilan Broz if (r) 784bbb16584SMilan Broz crypt_iv_elephant_dtr(cc); 785bbb16584SMilan Broz return r; 786bbb16584SMilan Broz } 787bbb16584SMilan Broz 788bbb16584SMilan Broz static void diffuser_disk_to_cpu(u32 *d, size_t n) 789bbb16584SMilan Broz { 790bbb16584SMilan Broz #ifndef __LITTLE_ENDIAN 791bbb16584SMilan Broz int i; 792bbb16584SMilan Broz 793bbb16584SMilan Broz for (i = 0; i < n; i++) 794bbb16584SMilan Broz d[i] = le32_to_cpu((__le32)d[i]); 795bbb16584SMilan Broz #endif 796bbb16584SMilan Broz } 797bbb16584SMilan Broz 798bbb16584SMilan Broz static void diffuser_cpu_to_disk(__le32 *d, size_t n) 799bbb16584SMilan Broz { 800bbb16584SMilan Broz #ifndef __LITTLE_ENDIAN 801bbb16584SMilan Broz int i; 802bbb16584SMilan Broz 803bbb16584SMilan Broz for (i = 0; i < n; i++) 804bbb16584SMilan Broz d[i] = cpu_to_le32((u32)d[i]); 805bbb16584SMilan Broz #endif 806bbb16584SMilan Broz } 807bbb16584SMilan Broz 808bbb16584SMilan Broz static void diffuser_a_decrypt(u32 *d, size_t n) 809bbb16584SMilan Broz { 810bbb16584SMilan Broz int i, i1, i2, i3; 811bbb16584SMilan Broz 812bbb16584SMilan Broz for (i = 0; i < 5; i++) { 813bbb16584SMilan Broz i1 = 0; 814bbb16584SMilan Broz i2 = n - 2; 815bbb16584SMilan Broz i3 = n - 5; 816bbb16584SMilan Broz 817bbb16584SMilan Broz while (i1 < (n - 1)) { 818bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23); 819bbb16584SMilan Broz i1++; i2++; i3++; 820bbb16584SMilan Broz 821bbb16584SMilan Broz if (i3 >= n) 822bbb16584SMilan Broz i3 -= n; 823bbb16584SMilan Broz 824bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 825bbb16584SMilan Broz i1++; i2++; i3++; 826bbb16584SMilan Broz 827bbb16584SMilan Broz if (i2 >= n) 828bbb16584SMilan Broz i2 -= n; 829bbb16584SMilan Broz 830bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19); 831bbb16584SMilan Broz i1++; i2++; i3++; 832bbb16584SMilan Broz 833bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 834bbb16584SMilan Broz i1++; i2++; i3++; 835bbb16584SMilan Broz } 836bbb16584SMilan Broz } 837bbb16584SMilan Broz } 838bbb16584SMilan Broz 839bbb16584SMilan Broz static void diffuser_a_encrypt(u32 *d, size_t n) 840bbb16584SMilan Broz { 841bbb16584SMilan Broz int i, i1, i2, i3; 842bbb16584SMilan Broz 843bbb16584SMilan Broz for (i = 0; i < 5; i++) { 844bbb16584SMilan Broz i1 = n - 1; 845bbb16584SMilan Broz i2 = n - 2 - 1; 846bbb16584SMilan Broz i3 = n - 5 - 1; 847bbb16584SMilan Broz 848bbb16584SMilan Broz while (i1 > 0) { 849bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 850bbb16584SMilan Broz i1--; i2--; i3--; 851bbb16584SMilan Broz 852bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19); 853bbb16584SMilan Broz i1--; i2--; i3--; 854bbb16584SMilan Broz 855bbb16584SMilan Broz if (i2 < 0) 856bbb16584SMilan Broz i2 += n; 857bbb16584SMilan Broz 858bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 859bbb16584SMilan Broz i1--; i2--; i3--; 860bbb16584SMilan Broz 861bbb16584SMilan Broz if (i3 < 0) 862bbb16584SMilan Broz i3 += n; 863bbb16584SMilan Broz 864bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23); 865bbb16584SMilan Broz i1--; i2--; i3--; 866bbb16584SMilan Broz } 867bbb16584SMilan Broz } 868bbb16584SMilan Broz } 869bbb16584SMilan Broz 870bbb16584SMilan Broz static void diffuser_b_decrypt(u32 *d, size_t n) 871bbb16584SMilan Broz { 872bbb16584SMilan Broz int i, i1, i2, i3; 873bbb16584SMilan Broz 874bbb16584SMilan Broz for (i = 0; i < 3; i++) { 875bbb16584SMilan Broz i1 = 0; 876bbb16584SMilan Broz i2 = 2; 877bbb16584SMilan Broz i3 = 5; 878bbb16584SMilan Broz 879bbb16584SMilan Broz while (i1 < (n - 1)) { 880bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 881bbb16584SMilan Broz i1++; i2++; i3++; 882bbb16584SMilan Broz 883bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22); 884bbb16584SMilan Broz i1++; i2++; i3++; 885bbb16584SMilan Broz 886bbb16584SMilan Broz if (i2 >= n) 887bbb16584SMilan Broz i2 -= n; 888bbb16584SMilan Broz 889bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 890bbb16584SMilan Broz i1++; i2++; i3++; 891bbb16584SMilan Broz 892bbb16584SMilan Broz if (i3 >= n) 893bbb16584SMilan Broz i3 -= n; 894bbb16584SMilan Broz 895bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7); 896bbb16584SMilan Broz i1++; i2++; i3++; 897bbb16584SMilan Broz } 898bbb16584SMilan Broz } 899bbb16584SMilan Broz } 900bbb16584SMilan Broz 901bbb16584SMilan Broz static void diffuser_b_encrypt(u32 *d, size_t n) 902bbb16584SMilan Broz { 903bbb16584SMilan Broz int i, i1, i2, i3; 904bbb16584SMilan Broz 905bbb16584SMilan Broz for (i = 0; i < 3; i++) { 906bbb16584SMilan Broz i1 = n - 1; 907bbb16584SMilan Broz i2 = 2 - 1; 908bbb16584SMilan Broz i3 = 5 - 1; 909bbb16584SMilan Broz 910bbb16584SMilan Broz while (i1 > 0) { 911bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7); 912bbb16584SMilan Broz i1--; i2--; i3--; 913bbb16584SMilan Broz 914bbb16584SMilan Broz if (i3 < 0) 915bbb16584SMilan Broz i3 += n; 916bbb16584SMilan Broz 917bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 918bbb16584SMilan Broz i1--; i2--; i3--; 919bbb16584SMilan Broz 920bbb16584SMilan Broz if (i2 < 0) 921bbb16584SMilan Broz i2 += n; 922bbb16584SMilan Broz 923bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22); 924bbb16584SMilan Broz i1--; i2--; i3--; 925bbb16584SMilan Broz 926bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 927bbb16584SMilan Broz i1--; i2--; i3--; 928bbb16584SMilan Broz } 929bbb16584SMilan Broz } 930bbb16584SMilan Broz } 931bbb16584SMilan Broz 932bbb16584SMilan Broz static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq) 933bbb16584SMilan Broz { 934bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 935bbb16584SMilan Broz u8 *es, *ks, *data, *data2, *data_offset; 936bbb16584SMilan Broz struct skcipher_request *req; 937bbb16584SMilan Broz struct scatterlist *sg, *sg2, src, dst; 938bbb16584SMilan Broz struct crypto_wait wait; 939bbb16584SMilan Broz int i, r; 940bbb16584SMilan Broz 941bbb16584SMilan Broz req = skcipher_request_alloc(elephant->tfm, GFP_NOIO); 942bbb16584SMilan Broz es = kzalloc(16, GFP_NOIO); /* Key for AES */ 943bbb16584SMilan Broz ks = kzalloc(32, GFP_NOIO); /* Elephant sector key */ 944bbb16584SMilan Broz 945bbb16584SMilan Broz if (!req || !es || !ks) { 946bbb16584SMilan Broz r = -ENOMEM; 947bbb16584SMilan Broz goto out; 948bbb16584SMilan Broz } 949bbb16584SMilan Broz 950bbb16584SMilan Broz *(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size); 951bbb16584SMilan Broz 952bbb16584SMilan Broz /* E(Ks, e(s)) */ 953bbb16584SMilan Broz sg_init_one(&src, es, 16); 954bbb16584SMilan Broz sg_init_one(&dst, ks, 16); 955bbb16584SMilan Broz skcipher_request_set_crypt(req, &src, &dst, 16, NULL); 956bbb16584SMilan Broz skcipher_request_set_callback(req, 0, crypto_req_done, &wait); 957bbb16584SMilan Broz r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); 958bbb16584SMilan Broz if (r) 959bbb16584SMilan Broz goto out; 960bbb16584SMilan Broz 961bbb16584SMilan Broz /* E(Ks, e'(s)) */ 962bbb16584SMilan Broz es[15] = 0x80; 963bbb16584SMilan Broz sg_init_one(&dst, &ks[16], 16); 964bbb16584SMilan Broz r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); 965bbb16584SMilan Broz if (r) 966bbb16584SMilan Broz goto out; 967bbb16584SMilan Broz 968bbb16584SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 969bbb16584SMilan Broz data = kmap_atomic(sg_page(sg)); 970bbb16584SMilan Broz data_offset = data + sg->offset; 971bbb16584SMilan Broz 972bbb16584SMilan Broz /* Cannot modify original bio, copy to sg_out and apply Elephant to it */ 973bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 974bbb16584SMilan Broz sg2 = crypt_get_sg_data(cc, dmreq->sg_in); 975bbb16584SMilan Broz data2 = kmap_atomic(sg_page(sg2)); 976bbb16584SMilan Broz memcpy(data_offset, data2 + sg2->offset, cc->sector_size); 977bbb16584SMilan Broz kunmap_atomic(data2); 978bbb16584SMilan Broz } 979bbb16584SMilan Broz 980bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) { 981bbb16584SMilan Broz diffuser_disk_to_cpu((u32*)data_offset, cc->sector_size / sizeof(u32)); 982bbb16584SMilan Broz diffuser_b_decrypt((u32*)data_offset, cc->sector_size / sizeof(u32)); 983bbb16584SMilan Broz diffuser_a_decrypt((u32*)data_offset, cc->sector_size / sizeof(u32)); 984bbb16584SMilan Broz diffuser_cpu_to_disk((__le32*)data_offset, cc->sector_size / sizeof(u32)); 985bbb16584SMilan Broz } 986bbb16584SMilan Broz 987bbb16584SMilan Broz for (i = 0; i < (cc->sector_size / 32); i++) 988bbb16584SMilan Broz crypto_xor(data_offset + i * 32, ks, 32); 989bbb16584SMilan Broz 990bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 991bbb16584SMilan Broz diffuser_disk_to_cpu((u32*)data_offset, cc->sector_size / sizeof(u32)); 992bbb16584SMilan Broz diffuser_a_encrypt((u32*)data_offset, cc->sector_size / sizeof(u32)); 993bbb16584SMilan Broz diffuser_b_encrypt((u32*)data_offset, cc->sector_size / sizeof(u32)); 994bbb16584SMilan Broz diffuser_cpu_to_disk((__le32*)data_offset, cc->sector_size / sizeof(u32)); 995bbb16584SMilan Broz } 996bbb16584SMilan Broz 997bbb16584SMilan Broz kunmap_atomic(data); 998bbb16584SMilan Broz out: 999bbb16584SMilan Broz kzfree(ks); 1000bbb16584SMilan Broz kzfree(es); 1001bbb16584SMilan Broz skcipher_request_free(req); 1002bbb16584SMilan Broz return r; 1003bbb16584SMilan Broz } 1004bbb16584SMilan Broz 1005bbb16584SMilan Broz static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv, 1006bbb16584SMilan Broz struct dm_crypt_request *dmreq) 1007bbb16584SMilan Broz { 1008bbb16584SMilan Broz int r; 1009bbb16584SMilan Broz 1010bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 1011bbb16584SMilan Broz r = crypt_iv_elephant(cc, dmreq); 1012bbb16584SMilan Broz if (r) 1013bbb16584SMilan Broz return r; 1014bbb16584SMilan Broz } 1015bbb16584SMilan Broz 1016bbb16584SMilan Broz return crypt_iv_eboiv_gen(cc, iv, dmreq); 1017bbb16584SMilan Broz } 1018bbb16584SMilan Broz 1019bbb16584SMilan Broz static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv, 1020bbb16584SMilan Broz struct dm_crypt_request *dmreq) 1021bbb16584SMilan Broz { 1022bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) 1023bbb16584SMilan Broz return crypt_iv_elephant(cc, dmreq); 1024bbb16584SMilan Broz 1025bbb16584SMilan Broz return 0; 1026bbb16584SMilan Broz } 1027bbb16584SMilan Broz 1028bbb16584SMilan Broz static int crypt_iv_elephant_init(struct crypt_config *cc) 1029bbb16584SMilan Broz { 1030bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 1031bbb16584SMilan Broz int key_offset = cc->key_size - cc->key_extra_size; 1032bbb16584SMilan Broz 1033bbb16584SMilan Broz return crypto_skcipher_setkey(elephant->tfm, &cc->key[key_offset], cc->key_extra_size); 1034bbb16584SMilan Broz } 1035bbb16584SMilan Broz 1036bbb16584SMilan Broz static int crypt_iv_elephant_wipe(struct crypt_config *cc) 1037bbb16584SMilan Broz { 1038bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 1039bbb16584SMilan Broz u8 key[ELEPHANT_MAX_KEY_SIZE]; 1040bbb16584SMilan Broz 1041bbb16584SMilan Broz memset(key, 0, cc->key_extra_size); 1042bbb16584SMilan Broz return crypto_skcipher_setkey(elephant->tfm, key, cc->key_extra_size); 1043bbb16584SMilan Broz } 1044bbb16584SMilan Broz 10451b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain_ops = { 10461da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 10471da177e4SLinus Torvalds }; 10481da177e4SLinus Torvalds 10491b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain64_ops = { 105061afef61SMilan Broz .generator = crypt_iv_plain64_gen 105161afef61SMilan Broz }; 105261afef61SMilan Broz 10537e3fd855SMilan Broz static const struct crypt_iv_operations crypt_iv_plain64be_ops = { 10547e3fd855SMilan Broz .generator = crypt_iv_plain64be_gen 10557e3fd855SMilan Broz }; 10567e3fd855SMilan Broz 10571b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_essiv_ops = { 10581da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 10591da177e4SLinus Torvalds }; 10601da177e4SLinus Torvalds 10611b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_benbi_ops = { 106248527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 106348527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 106448527fa7SRik Snel .generator = crypt_iv_benbi_gen 106548527fa7SRik Snel }; 10661da177e4SLinus Torvalds 10671b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_null_ops = { 106846b47730SLudwig Nussel .generator = crypt_iv_null_gen 106946b47730SLudwig Nussel }; 107046b47730SLudwig Nussel 10711b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_lmk_ops = { 107234745785SMilan Broz .ctr = crypt_iv_lmk_ctr, 107334745785SMilan Broz .dtr = crypt_iv_lmk_dtr, 107434745785SMilan Broz .init = crypt_iv_lmk_init, 107534745785SMilan Broz .wipe = crypt_iv_lmk_wipe, 107634745785SMilan Broz .generator = crypt_iv_lmk_gen, 107734745785SMilan Broz .post = crypt_iv_lmk_post 107834745785SMilan Broz }; 107934745785SMilan Broz 10801b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_tcw_ops = { 1081ed04d981SMilan Broz .ctr = crypt_iv_tcw_ctr, 1082ed04d981SMilan Broz .dtr = crypt_iv_tcw_dtr, 1083ed04d981SMilan Broz .init = crypt_iv_tcw_init, 1084ed04d981SMilan Broz .wipe = crypt_iv_tcw_wipe, 1085ed04d981SMilan Broz .generator = crypt_iv_tcw_gen, 1086ed04d981SMilan Broz .post = crypt_iv_tcw_post 1087ed04d981SMilan Broz }; 1088ed04d981SMilan Broz 1089ef43aa38SMilan Broz static struct crypt_iv_operations crypt_iv_random_ops = { 1090ef43aa38SMilan Broz .generator = crypt_iv_random_gen 1091ef43aa38SMilan Broz }; 1092ef43aa38SMilan Broz 1093b9411d73SMilan Broz static struct crypt_iv_operations crypt_iv_eboiv_ops = { 1094b9411d73SMilan Broz .ctr = crypt_iv_eboiv_ctr, 1095b9411d73SMilan Broz .generator = crypt_iv_eboiv_gen 1096b9411d73SMilan Broz }; 1097b9411d73SMilan Broz 1098bbb16584SMilan Broz static struct crypt_iv_operations crypt_iv_elephant_ops = { 1099bbb16584SMilan Broz .ctr = crypt_iv_elephant_ctr, 1100bbb16584SMilan Broz .dtr = crypt_iv_elephant_dtr, 1101bbb16584SMilan Broz .init = crypt_iv_elephant_init, 1102bbb16584SMilan Broz .wipe = crypt_iv_elephant_wipe, 1103bbb16584SMilan Broz .generator = crypt_iv_elephant_gen, 1104bbb16584SMilan Broz .post = crypt_iv_elephant_post 1105bbb16584SMilan Broz }; 1106bbb16584SMilan Broz 1107ef43aa38SMilan Broz /* 1108ef43aa38SMilan Broz * Integrity extensions 1109ef43aa38SMilan Broz */ 1110ef43aa38SMilan Broz static bool crypt_integrity_aead(struct crypt_config *cc) 1111ef43aa38SMilan Broz { 1112ef43aa38SMilan Broz return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags); 1113ef43aa38SMilan Broz } 1114ef43aa38SMilan Broz 1115ef43aa38SMilan Broz static bool crypt_integrity_hmac(struct crypt_config *cc) 1116ef43aa38SMilan Broz { 111733d2f09fSMilan Broz return crypt_integrity_aead(cc) && cc->key_mac_size; 1118ef43aa38SMilan Broz } 1119ef43aa38SMilan Broz 1120ef43aa38SMilan Broz /* Get sg containing data */ 1121ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc, 1122ef43aa38SMilan Broz struct scatterlist *sg) 1123ef43aa38SMilan Broz { 112433d2f09fSMilan Broz if (unlikely(crypt_integrity_aead(cc))) 1125ef43aa38SMilan Broz return &sg[2]; 1126ef43aa38SMilan Broz 1127ef43aa38SMilan Broz return sg; 1128ef43aa38SMilan Broz } 1129ef43aa38SMilan Broz 1130ef43aa38SMilan Broz static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio) 1131ef43aa38SMilan Broz { 1132ef43aa38SMilan Broz struct bio_integrity_payload *bip; 1133ef43aa38SMilan Broz unsigned int tag_len; 1134ef43aa38SMilan Broz int ret; 1135ef43aa38SMilan Broz 1136ef43aa38SMilan Broz if (!bio_sectors(bio) || !io->cc->on_disk_tag_size) 1137ef43aa38SMilan Broz return 0; 1138ef43aa38SMilan Broz 1139ef43aa38SMilan Broz bip = bio_integrity_alloc(bio, GFP_NOIO, 1); 1140ef43aa38SMilan Broz if (IS_ERR(bip)) 1141ef43aa38SMilan Broz return PTR_ERR(bip); 1142ef43aa38SMilan Broz 1143ff0c129dSMikulas Patocka tag_len = io->cc->on_disk_tag_size * (bio_sectors(bio) >> io->cc->sector_shift); 1144ef43aa38SMilan Broz 1145ef43aa38SMilan Broz bip->bip_iter.bi_size = tag_len; 1146ef43aa38SMilan Broz bip->bip_iter.bi_sector = io->cc->start + io->sector; 1147ef43aa38SMilan Broz 1148ef43aa38SMilan Broz ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata), 1149ef43aa38SMilan Broz tag_len, offset_in_page(io->integrity_metadata)); 1150ef43aa38SMilan Broz if (unlikely(ret != tag_len)) 1151ef43aa38SMilan Broz return -ENOMEM; 1152ef43aa38SMilan Broz 1153ef43aa38SMilan Broz return 0; 1154ef43aa38SMilan Broz } 1155ef43aa38SMilan Broz 1156ef43aa38SMilan Broz static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti) 1157ef43aa38SMilan Broz { 1158ef43aa38SMilan Broz #ifdef CONFIG_BLK_DEV_INTEGRITY 1159ef43aa38SMilan Broz struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk); 11607a1cd723SMilan Broz struct mapped_device *md = dm_table_get_md(ti->table); 1161ef43aa38SMilan Broz 1162ef43aa38SMilan Broz /* From now we require underlying device with our integrity profile */ 1163ef43aa38SMilan Broz if (!bi || strcasecmp(bi->profile->name, "DM-DIF-EXT-TAG")) { 1164ef43aa38SMilan Broz ti->error = "Integrity profile not supported."; 1165ef43aa38SMilan Broz return -EINVAL; 1166ef43aa38SMilan Broz } 1167ef43aa38SMilan Broz 1168583fe747SMikulas Patocka if (bi->tag_size != cc->on_disk_tag_size || 1169583fe747SMikulas Patocka bi->tuple_size != cc->on_disk_tag_size) { 1170ef43aa38SMilan Broz ti->error = "Integrity profile tag size mismatch."; 1171ef43aa38SMilan Broz return -EINVAL; 1172ef43aa38SMilan Broz } 1173583fe747SMikulas Patocka if (1 << bi->interval_exp != cc->sector_size) { 1174583fe747SMikulas Patocka ti->error = "Integrity profile sector size mismatch."; 1175583fe747SMikulas Patocka return -EINVAL; 1176583fe747SMikulas Patocka } 1177ef43aa38SMilan Broz 117833d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 1179ef43aa38SMilan Broz cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size; 11807a1cd723SMilan Broz DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md), 1181ef43aa38SMilan Broz cc->integrity_tag_size, cc->integrity_iv_size); 1182ef43aa38SMilan Broz 1183ef43aa38SMilan Broz if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) { 1184ef43aa38SMilan Broz ti->error = "Integrity AEAD auth tag size is not supported."; 1185ef43aa38SMilan Broz return -EINVAL; 1186ef43aa38SMilan Broz } 1187ef43aa38SMilan Broz } else if (cc->integrity_iv_size) 11887a1cd723SMilan Broz DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md), 1189ef43aa38SMilan Broz cc->integrity_iv_size); 1190ef43aa38SMilan Broz 1191ef43aa38SMilan Broz if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) { 1192ef43aa38SMilan Broz ti->error = "Not enough space for integrity tag in the profile."; 1193ef43aa38SMilan Broz return -EINVAL; 1194ef43aa38SMilan Broz } 1195ef43aa38SMilan Broz 1196ef43aa38SMilan Broz return 0; 1197ef43aa38SMilan Broz #else 1198ef43aa38SMilan Broz ti->error = "Integrity profile not supported."; 1199ef43aa38SMilan Broz return -EINVAL; 1200ef43aa38SMilan Broz #endif 1201ef43aa38SMilan Broz } 1202ef43aa38SMilan Broz 1203d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 1204d469f841SMilan Broz struct convert_context *ctx, 12051da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 1206fcd369daSMilan Broz sector_t sector) 12071da177e4SLinus Torvalds { 12081da177e4SLinus Torvalds ctx->bio_in = bio_in; 12091da177e4SLinus Torvalds ctx->bio_out = bio_out; 1210003b5c57SKent Overstreet if (bio_in) 1211003b5c57SKent Overstreet ctx->iter_in = bio_in->bi_iter; 1212003b5c57SKent Overstreet if (bio_out) 1213003b5c57SKent Overstreet ctx->iter_out = bio_out->bi_iter; 1214c66029f4SMikulas Patocka ctx->cc_sector = sector + cc->iv_offset; 121543d69034SMilan Broz init_completion(&ctx->restart); 12161da177e4SLinus Torvalds } 12171da177e4SLinus Torvalds 1218b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 1219ef43aa38SMilan Broz void *req) 1220b2174eebSHuang Ying { 1221b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 1222b2174eebSHuang Ying } 1223b2174eebSHuang Ying 1224ef43aa38SMilan Broz static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq) 1225b2174eebSHuang Ying { 1226ef43aa38SMilan Broz return (void *)((char *)dmreq - cc->dmreq_start); 1227b2174eebSHuang Ying } 1228b2174eebSHuang Ying 12292dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, 12302dc5327dSMilan Broz struct dm_crypt_request *dmreq) 12312dc5327dSMilan Broz { 123233d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1233ef43aa38SMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 1234ef43aa38SMilan Broz crypto_aead_alignmask(any_tfm_aead(cc)) + 1); 1235ef43aa38SMilan Broz else 12362dc5327dSMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 1237bbdb23b5SHerbert Xu crypto_skcipher_alignmask(any_tfm(cc)) + 1); 12382dc5327dSMilan Broz } 12392dc5327dSMilan Broz 1240ef43aa38SMilan Broz static u8 *org_iv_of_dmreq(struct crypt_config *cc, 1241ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1242ef43aa38SMilan Broz { 1243ef43aa38SMilan Broz return iv_of_dmreq(cc, dmreq) + cc->iv_size; 1244ef43aa38SMilan Broz } 1245ef43aa38SMilan Broz 1246c13b5487SChristoph Hellwig static __le64 *org_sector_of_dmreq(struct crypt_config *cc, 1247ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1248ef43aa38SMilan Broz { 1249ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size; 1250c13b5487SChristoph Hellwig return (__le64 *) ptr; 1251ef43aa38SMilan Broz } 1252ef43aa38SMilan Broz 1253ef43aa38SMilan Broz static unsigned int *org_tag_of_dmreq(struct crypt_config *cc, 1254ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1255ef43aa38SMilan Broz { 1256ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + 1257ef43aa38SMilan Broz cc->iv_size + sizeof(uint64_t); 1258ef43aa38SMilan Broz return (unsigned int*)ptr; 1259ef43aa38SMilan Broz } 1260ef43aa38SMilan Broz 1261ef43aa38SMilan Broz static void *tag_from_dmreq(struct crypt_config *cc, 1262ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1263ef43aa38SMilan Broz { 1264ef43aa38SMilan Broz struct convert_context *ctx = dmreq->ctx; 1265ef43aa38SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 1266ef43aa38SMilan Broz 1267ef43aa38SMilan Broz return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) * 1268ef43aa38SMilan Broz cc->on_disk_tag_size]; 1269ef43aa38SMilan Broz } 1270ef43aa38SMilan Broz 1271ef43aa38SMilan Broz static void *iv_tag_from_dmreq(struct crypt_config *cc, 1272ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1273ef43aa38SMilan Broz { 1274ef43aa38SMilan Broz return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size; 1275ef43aa38SMilan Broz } 1276ef43aa38SMilan Broz 1277ef43aa38SMilan Broz static int crypt_convert_block_aead(struct crypt_config *cc, 12783a7f6c99SMilan Broz struct convert_context *ctx, 1279ef43aa38SMilan Broz struct aead_request *req, 1280ef43aa38SMilan Broz unsigned int tag_offset) 128101482b76SMilan Broz { 1282003b5c57SKent Overstreet struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in); 1283003b5c57SKent Overstreet struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out); 12843a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 1285ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv, *tag; 1286c13b5487SChristoph Hellwig __le64 *sector; 1287ef43aa38SMilan Broz int r = 0; 1288ef43aa38SMilan Broz 1289ef43aa38SMilan Broz BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size); 129001482b76SMilan Broz 12918f0009a2SMilan Broz /* Reject unexpected unaligned bio. */ 12920440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1))) 12938f0009a2SMilan Broz return -EIO; 129401482b76SMilan Broz 1295b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 1296c66029f4SMikulas Patocka dmreq->iv_sector = ctx->cc_sector; 12978f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 1298ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift; 1299b2174eebSHuang Ying dmreq->ctx = ctx; 130001482b76SMilan Broz 1301ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset; 130201482b76SMilan Broz 1303ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq); 1304ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset); 1305ef43aa38SMilan Broz 1306ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq); 1307ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq); 1308ef43aa38SMilan Broz tag = tag_from_dmreq(cc, dmreq); 1309ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq); 1310ef43aa38SMilan Broz 1311ef43aa38SMilan Broz /* AEAD request: 1312ef43aa38SMilan Broz * |----- AAD -------|------ DATA -------|-- AUTH TAG --| 1313ef43aa38SMilan Broz * | (authenticated) | (auth+encryption) | | 1314ef43aa38SMilan Broz * | sector_LE | IV | sector in/out | tag in/out | 1315ef43aa38SMilan Broz */ 1316ef43aa38SMilan Broz sg_init_table(dmreq->sg_in, 4); 1317ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t)); 1318ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size); 13198f0009a2SMilan Broz sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset); 1320ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size); 1321ef43aa38SMilan Broz 1322ef43aa38SMilan Broz sg_init_table(dmreq->sg_out, 4); 1323ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t)); 1324ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size); 13258f0009a2SMilan Broz sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset); 1326ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size); 132701482b76SMilan Broz 13283a7f6c99SMilan Broz if (cc->iv_gen_ops) { 1329ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */ 1330ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) { 1331ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->iv_size); 1332ef43aa38SMilan Broz } else { 1333ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq); 13343a7f6c99SMilan Broz if (r < 0) 13353a7f6c99SMilan Broz return r; 1336ef43aa38SMilan Broz /* Store generated IV in integrity metadata */ 1337ef43aa38SMilan Broz if (cc->integrity_iv_size) 1338ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->iv_size); 1339ef43aa38SMilan Broz } 1340ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */ 1341ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size); 1342ef43aa38SMilan Broz } 1343ef43aa38SMilan Broz 1344ef43aa38SMilan Broz aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size); 1345ef43aa38SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) { 1346ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out, 13478f0009a2SMilan Broz cc->sector_size, iv); 1348ef43aa38SMilan Broz r = crypto_aead_encrypt(req); 1349ef43aa38SMilan Broz if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size) 1350ef43aa38SMilan Broz memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0, 1351ef43aa38SMilan Broz cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size)); 1352ef43aa38SMilan Broz } else { 1353ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out, 13548f0009a2SMilan Broz cc->sector_size + cc->integrity_tag_size, iv); 1355ef43aa38SMilan Broz r = crypto_aead_decrypt(req); 1356ef43aa38SMilan Broz } 1357ef43aa38SMilan Broz 1358f710126cSMilan Broz if (r == -EBADMSG) { 1359f710126cSMilan Broz char b[BDEVNAME_SIZE]; 1360f710126cSMilan Broz DMERR_LIMIT("%s: INTEGRITY AEAD ERROR, sector %llu", bio_devname(ctx->bio_in, b), 1361ef43aa38SMilan Broz (unsigned long long)le64_to_cpu(*sector)); 1362f710126cSMilan Broz } 1363ef43aa38SMilan Broz 1364ef43aa38SMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 1365ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq); 1366ef43aa38SMilan Broz 13678f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size); 13688f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size); 1369ef43aa38SMilan Broz 1370ef43aa38SMilan Broz return r; 13713a7f6c99SMilan Broz } 13723a7f6c99SMilan Broz 1373ef43aa38SMilan Broz static int crypt_convert_block_skcipher(struct crypt_config *cc, 1374ef43aa38SMilan Broz struct convert_context *ctx, 1375ef43aa38SMilan Broz struct skcipher_request *req, 1376ef43aa38SMilan Broz unsigned int tag_offset) 1377ef43aa38SMilan Broz { 1378ef43aa38SMilan Broz struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in); 1379ef43aa38SMilan Broz struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out); 1380ef43aa38SMilan Broz struct scatterlist *sg_in, *sg_out; 1381ef43aa38SMilan Broz struct dm_crypt_request *dmreq; 1382ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv; 1383c13b5487SChristoph Hellwig __le64 *sector; 1384ef43aa38SMilan Broz int r = 0; 1385ef43aa38SMilan Broz 13868f0009a2SMilan Broz /* Reject unexpected unaligned bio. */ 13870440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1))) 13888f0009a2SMilan Broz return -EIO; 13898f0009a2SMilan Broz 1390ef43aa38SMilan Broz dmreq = dmreq_of_req(cc, req); 1391ef43aa38SMilan Broz dmreq->iv_sector = ctx->cc_sector; 13928f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 1393ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift; 1394ef43aa38SMilan Broz dmreq->ctx = ctx; 1395ef43aa38SMilan Broz 1396ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset; 1397ef43aa38SMilan Broz 1398ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq); 1399ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq); 1400ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq); 1401ef43aa38SMilan Broz 1402ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq); 1403ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset); 1404ef43aa38SMilan Broz 1405ef43aa38SMilan Broz /* For skcipher we use only the first sg item */ 1406ef43aa38SMilan Broz sg_in = &dmreq->sg_in[0]; 1407ef43aa38SMilan Broz sg_out = &dmreq->sg_out[0]; 1408ef43aa38SMilan Broz 1409ef43aa38SMilan Broz sg_init_table(sg_in, 1); 14108f0009a2SMilan Broz sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset); 1411ef43aa38SMilan Broz 1412ef43aa38SMilan Broz sg_init_table(sg_out, 1); 14138f0009a2SMilan Broz sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset); 1414ef43aa38SMilan Broz 1415ef43aa38SMilan Broz if (cc->iv_gen_ops) { 1416ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */ 1417ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) { 1418ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->integrity_iv_size); 1419ef43aa38SMilan Broz } else { 1420ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq); 1421ef43aa38SMilan Broz if (r < 0) 1422ef43aa38SMilan Broz return r; 1423bbb16584SMilan Broz /* Data can be already preprocessed in generator */ 1424bbb16584SMilan Broz if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags)) 1425bbb16584SMilan Broz sg_in = sg_out; 1426ef43aa38SMilan Broz /* Store generated IV in integrity metadata */ 1427ef43aa38SMilan Broz if (cc->integrity_iv_size) 1428ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->integrity_iv_size); 1429ef43aa38SMilan Broz } 1430ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */ 1431ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size); 1432ef43aa38SMilan Broz } 1433ef43aa38SMilan Broz 14348f0009a2SMilan Broz skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv); 14353a7f6c99SMilan Broz 14363a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 1437bbdb23b5SHerbert Xu r = crypto_skcipher_encrypt(req); 14383a7f6c99SMilan Broz else 1439bbdb23b5SHerbert Xu r = crypto_skcipher_decrypt(req); 14403a7f6c99SMilan Broz 14412dc5327dSMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 1442ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq); 1443ef43aa38SMilan Broz 14448f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size); 14458f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size); 14462dc5327dSMilan Broz 14473a7f6c99SMilan Broz return r; 144801482b76SMilan Broz } 144901482b76SMilan Broz 145095497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 145195497a96SMilan Broz int error); 1452c0297721SAndi Kleen 1453ef43aa38SMilan Broz static void crypt_alloc_req_skcipher(struct crypt_config *cc, 1454ddd42edfSMilan Broz struct convert_context *ctx) 1455ddd42edfSMilan Broz { 1456c66029f4SMikulas Patocka unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1); 1457c0297721SAndi Kleen 1458ef43aa38SMilan Broz if (!ctx->r.req) 14596f1c819cSKent Overstreet ctx->r.req = mempool_alloc(&cc->req_pool, GFP_NOIO); 1460c0297721SAndi Kleen 1461ef43aa38SMilan Broz skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]); 146254cea3f6SMilan Broz 146354cea3f6SMilan Broz /* 146454cea3f6SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs 146554cea3f6SMilan Broz * requests if driver request queue is full. 146654cea3f6SMilan Broz */ 1467ef43aa38SMilan Broz skcipher_request_set_callback(ctx->r.req, 1468432061b3SMikulas Patocka CRYPTO_TFM_REQ_MAY_BACKLOG, 1469ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req)); 1470ddd42edfSMilan Broz } 1471ddd42edfSMilan Broz 1472ef43aa38SMilan Broz static void crypt_alloc_req_aead(struct crypt_config *cc, 1473ef43aa38SMilan Broz struct convert_context *ctx) 1474ef43aa38SMilan Broz { 1475ef43aa38SMilan Broz if (!ctx->r.req_aead) 14766f1c819cSKent Overstreet ctx->r.req_aead = mempool_alloc(&cc->req_pool, GFP_NOIO); 1477ef43aa38SMilan Broz 1478ef43aa38SMilan Broz aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]); 1479ef43aa38SMilan Broz 1480ef43aa38SMilan Broz /* 1481ef43aa38SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs 1482ef43aa38SMilan Broz * requests if driver request queue is full. 1483ef43aa38SMilan Broz */ 1484ef43aa38SMilan Broz aead_request_set_callback(ctx->r.req_aead, 1485432061b3SMikulas Patocka CRYPTO_TFM_REQ_MAY_BACKLOG, 1486ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead)); 1487ef43aa38SMilan Broz } 1488ef43aa38SMilan Broz 1489ef43aa38SMilan Broz static void crypt_alloc_req(struct crypt_config *cc, 1490ef43aa38SMilan Broz struct convert_context *ctx) 1491ef43aa38SMilan Broz { 149233d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1493ef43aa38SMilan Broz crypt_alloc_req_aead(cc, ctx); 1494ef43aa38SMilan Broz else 1495ef43aa38SMilan Broz crypt_alloc_req_skcipher(cc, ctx); 1496ef43aa38SMilan Broz } 1497ef43aa38SMilan Broz 1498ef43aa38SMilan Broz static void crypt_free_req_skcipher(struct crypt_config *cc, 1499bbdb23b5SHerbert Xu struct skcipher_request *req, struct bio *base_bio) 1500298a9fa0SMikulas Patocka { 1501298a9fa0SMikulas Patocka struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); 1502298a9fa0SMikulas Patocka 1503bbdb23b5SHerbert Xu if ((struct skcipher_request *)(io + 1) != req) 15046f1c819cSKent Overstreet mempool_free(req, &cc->req_pool); 1505298a9fa0SMikulas Patocka } 1506298a9fa0SMikulas Patocka 1507ef43aa38SMilan Broz static void crypt_free_req_aead(struct crypt_config *cc, 1508ef43aa38SMilan Broz struct aead_request *req, struct bio *base_bio) 1509ef43aa38SMilan Broz { 1510ef43aa38SMilan Broz struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); 1511ef43aa38SMilan Broz 1512ef43aa38SMilan Broz if ((struct aead_request *)(io + 1) != req) 15136f1c819cSKent Overstreet mempool_free(req, &cc->req_pool); 1514ef43aa38SMilan Broz } 1515ef43aa38SMilan Broz 1516ef43aa38SMilan Broz static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio) 1517ef43aa38SMilan Broz { 151833d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1519ef43aa38SMilan Broz crypt_free_req_aead(cc, req, base_bio); 1520ef43aa38SMilan Broz else 1521ef43aa38SMilan Broz crypt_free_req_skcipher(cc, req, base_bio); 1522ef43aa38SMilan Broz } 1523ef43aa38SMilan Broz 15241da177e4SLinus Torvalds /* 15251da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 15261da177e4SLinus Torvalds */ 15274e4cbee9SChristoph Hellwig static blk_status_t crypt_convert(struct crypt_config *cc, 152839d42fa9SIgnat Korchagin struct convert_context *ctx, bool atomic) 15291da177e4SLinus Torvalds { 1530ef43aa38SMilan Broz unsigned int tag_offset = 0; 1531ff3af92bSMikulas Patocka unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT; 15323f1e9070SMilan Broz int r; 15331da177e4SLinus Torvalds 153440b6229bSMikulas Patocka atomic_set(&ctx->cc_pending, 1); 1535c8081618SMilan Broz 1536003b5c57SKent Overstreet while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) { 15371da177e4SLinus Torvalds 15383a7f6c99SMilan Broz crypt_alloc_req(cc, ctx); 153940b6229bSMikulas Patocka atomic_inc(&ctx->cc_pending); 15403f1e9070SMilan Broz 154133d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1542ef43aa38SMilan Broz r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset); 1543ef43aa38SMilan Broz else 1544ef43aa38SMilan Broz r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset); 15453a7f6c99SMilan Broz 15463a7f6c99SMilan Broz switch (r) { 154754cea3f6SMilan Broz /* 154854cea3f6SMilan Broz * The request was queued by a crypto driver 154954cea3f6SMilan Broz * but the driver request queue is full, let's wait. 155054cea3f6SMilan Broz */ 15513a7f6c99SMilan Broz case -EBUSY: 15523a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 155316735d02SWolfram Sang reinit_completion(&ctx->restart); 1554c0403ec0SRabin Vincent /* fall through */ 155554cea3f6SMilan Broz /* 155654cea3f6SMilan Broz * The request is queued and processed asynchronously, 155754cea3f6SMilan Broz * completion function kcryptd_async_done() will be called. 155854cea3f6SMilan Broz */ 1559c0403ec0SRabin Vincent case -EINPROGRESS: 1560ef43aa38SMilan Broz ctx->r.req = NULL; 15618f0009a2SMilan Broz ctx->cc_sector += sector_step; 1562583fe747SMikulas Patocka tag_offset++; 15633a7f6c99SMilan Broz continue; 156454cea3f6SMilan Broz /* 156554cea3f6SMilan Broz * The request was already processed (synchronously). 156654cea3f6SMilan Broz */ 15673f1e9070SMilan Broz case 0: 156840b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 15698f0009a2SMilan Broz ctx->cc_sector += sector_step; 1570583fe747SMikulas Patocka tag_offset++; 157139d42fa9SIgnat Korchagin if (!atomic) 1572c7f1b204SMilan Broz cond_resched(); 15733f1e9070SMilan Broz continue; 1574ef43aa38SMilan Broz /* 1575ef43aa38SMilan Broz * There was a data integrity error. 1576ef43aa38SMilan Broz */ 1577ef43aa38SMilan Broz case -EBADMSG: 1578ef43aa38SMilan Broz atomic_dec(&ctx->cc_pending); 15794e4cbee9SChristoph Hellwig return BLK_STS_PROTECTION; 1580ef43aa38SMilan Broz /* 1581ef43aa38SMilan Broz * There was an error while processing the request. 1582ef43aa38SMilan Broz */ 15833f1e9070SMilan Broz default: 158440b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 15854e4cbee9SChristoph Hellwig return BLK_STS_IOERR; 15861da177e4SLinus Torvalds } 15873f1e9070SMilan Broz } 15883f1e9070SMilan Broz 15893f1e9070SMilan Broz return 0; 15903f1e9070SMilan Broz } 15911da177e4SLinus Torvalds 1592cf2f1abfSMikulas Patocka static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone); 1593cf2f1abfSMikulas Patocka 15941da177e4SLinus Torvalds /* 15951da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 1596586b286bSMike Snitzer * This should never violate the device limitations (but only because 1597586b286bSMike Snitzer * max_segment_size is being constrained to PAGE_SIZE). 15987145c241SMikulas Patocka * 15997145c241SMikulas Patocka * This function may be called concurrently. If we allocate from the mempool 16007145c241SMikulas Patocka * concurrently, there is a possibility of deadlock. For example, if we have 16017145c241SMikulas Patocka * mempool of 256 pages, two processes, each wanting 256, pages allocate from 16027145c241SMikulas Patocka * the mempool concurrently, it may deadlock in a situation where both processes 16037145c241SMikulas Patocka * have allocated 128 pages and the mempool is exhausted. 16047145c241SMikulas Patocka * 16057145c241SMikulas Patocka * In order to avoid this scenario we allocate the pages under a mutex. 16067145c241SMikulas Patocka * 16077145c241SMikulas Patocka * In order to not degrade performance with excessive locking, we try 16087145c241SMikulas Patocka * non-blocking allocations without a mutex first but on failure we fallback 16097145c241SMikulas Patocka * to blocking allocations with a mutex. 16101da177e4SLinus Torvalds */ 1611cf2f1abfSMikulas Patocka static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) 16121da177e4SLinus Torvalds { 161349a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 16148b004457SMilan Broz struct bio *clone; 16151da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 16167145c241SMikulas Patocka gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; 16177145c241SMikulas Patocka unsigned i, len, remaining_size; 161891e10625SMilan Broz struct page *page; 16191da177e4SLinus Torvalds 16207145c241SMikulas Patocka retry: 1621d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 16227145c241SMikulas Patocka mutex_lock(&cc->bio_alloc_lock); 16237145c241SMikulas Patocka 16246f1c819cSKent Overstreet clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, &cc->bs); 16258b004457SMilan Broz if (!clone) 1626ef43aa38SMilan Broz goto out; 16271da177e4SLinus Torvalds 1628027581f3SOlaf Kirch clone_init(io, clone); 16296a24c718SMilan Broz 16307145c241SMikulas Patocka remaining_size = size; 16317145c241SMikulas Patocka 1632f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 16336f1c819cSKent Overstreet page = mempool_alloc(&cc->page_pool, gfp_mask); 16347145c241SMikulas Patocka if (!page) { 16357145c241SMikulas Patocka crypt_free_buffer_pages(cc, clone); 16367145c241SMikulas Patocka bio_put(clone); 1637d0164adcSMel Gorman gfp_mask |= __GFP_DIRECT_RECLAIM; 16387145c241SMikulas Patocka goto retry; 16397145c241SMikulas Patocka } 16401da177e4SLinus Torvalds 16417145c241SMikulas Patocka len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size; 16421da177e4SLinus Torvalds 16430dae7fe5SMing Lei bio_add_page(clone, page, len, 0); 164491e10625SMilan Broz 16457145c241SMikulas Patocka remaining_size -= len; 16461da177e4SLinus Torvalds } 16471da177e4SLinus Torvalds 1648ef43aa38SMilan Broz /* Allocate space for integrity tags */ 1649ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) { 1650ef43aa38SMilan Broz crypt_free_buffer_pages(cc, clone); 1651ef43aa38SMilan Broz bio_put(clone); 1652ef43aa38SMilan Broz clone = NULL; 1653ef43aa38SMilan Broz } 1654ef43aa38SMilan Broz out: 1655d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 16567145c241SMikulas Patocka mutex_unlock(&cc->bio_alloc_lock); 16577145c241SMikulas Patocka 16588b004457SMilan Broz return clone; 16591da177e4SLinus Torvalds } 16601da177e4SLinus Torvalds 1661644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 16621da177e4SLinus Torvalds { 16631da177e4SLinus Torvalds struct bio_vec *bv; 16646dc4f100SMing Lei struct bvec_iter_all iter_all; 16651da177e4SLinus Torvalds 16662b070cfeSChristoph Hellwig bio_for_each_segment_all(bv, clone, iter_all) { 16671da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 16686f1c819cSKent Overstreet mempool_free(bv->bv_page, &cc->page_pool); 16691da177e4SLinus Torvalds } 16701da177e4SLinus Torvalds } 16711da177e4SLinus Torvalds 1672298a9fa0SMikulas Patocka static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc, 1673dc440d1eSMilan Broz struct bio *bio, sector_t sector) 1674dc440d1eSMilan Broz { 167549a8a920SAlasdair G Kergon io->cc = cc; 1676dc440d1eSMilan Broz io->base_bio = bio; 1677dc440d1eSMilan Broz io->sector = sector; 1678dc440d1eSMilan Broz io->error = 0; 1679ef43aa38SMilan Broz io->ctx.r.req = NULL; 1680ef43aa38SMilan Broz io->integrity_metadata = NULL; 1681ef43aa38SMilan Broz io->integrity_metadata_from_pool = false; 168240b6229bSMikulas Patocka atomic_set(&io->io_pending, 0); 1683dc440d1eSMilan Broz } 1684dc440d1eSMilan Broz 16853e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 16863e1a8bddSMilan Broz { 168740b6229bSMikulas Patocka atomic_inc(&io->io_pending); 16883e1a8bddSMilan Broz } 16893e1a8bddSMilan Broz 16901da177e4SLinus Torvalds /* 16911da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 16921da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 16931da177e4SLinus Torvalds */ 16945742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 16951da177e4SLinus Torvalds { 169649a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1697b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 16984e4cbee9SChristoph Hellwig blk_status_t error = io->error; 16991da177e4SLinus Torvalds 170040b6229bSMikulas Patocka if (!atomic_dec_and_test(&io->io_pending)) 17011da177e4SLinus Torvalds return; 17021da177e4SLinus Torvalds 1703ef43aa38SMilan Broz if (io->ctx.r.req) 1704ef43aa38SMilan Broz crypt_free_req(cc, io->ctx.r.req, base_bio); 1705ef43aa38SMilan Broz 1706ef43aa38SMilan Broz if (unlikely(io->integrity_metadata_from_pool)) 17076f1c819cSKent Overstreet mempool_free(io->integrity_metadata, &io->cc->tag_pool); 1708ef43aa38SMilan Broz else 1709ef43aa38SMilan Broz kfree(io->integrity_metadata); 1710b35f8caaSMilan Broz 17114e4cbee9SChristoph Hellwig base_bio->bi_status = error; 17124246a0b6SChristoph Hellwig bio_endio(base_bio); 17131da177e4SLinus Torvalds } 17141da177e4SLinus Torvalds 17151da177e4SLinus Torvalds /* 1716cabf08e4SMilan Broz * kcryptd/kcryptd_io: 17171da177e4SLinus Torvalds * 17181da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 171923541d2dSMilan Broz * interrupt context. 1720cabf08e4SMilan Broz * 1721cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 1722cabf08e4SMilan Broz * 1723cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 1724cabf08e4SMilan Broz * 1725cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 1726cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 1727cabf08e4SMilan Broz * to memory allocation. 1728c0297721SAndi Kleen * 1729c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances. 1730c0297721SAndi Kleen * They should not depend on each other and do not block. 17311da177e4SLinus Torvalds */ 17324246a0b6SChristoph Hellwig static void crypt_endio(struct bio *clone) 17338b004457SMilan Broz { 1734028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 173549a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1736ee7a491eSMilan Broz unsigned rw = bio_data_dir(clone); 17374e4cbee9SChristoph Hellwig blk_status_t error; 17388b004457SMilan Broz 17398b004457SMilan Broz /* 17406712ecf8SNeilBrown * free the processed pages 17418b004457SMilan Broz */ 1742ee7a491eSMilan Broz if (rw == WRITE) 1743644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 17448b004457SMilan Broz 17454e4cbee9SChristoph Hellwig error = clone->bi_status; 17468b004457SMilan Broz bio_put(clone); 1747ee7a491eSMilan Broz 17489b81c842SSasha Levin if (rw == READ && !error) { 1749cabf08e4SMilan Broz kcryptd_queue_crypt(io); 17506712ecf8SNeilBrown return; 1751ee7a491eSMilan Broz } 17525742fd77SMilan Broz 17539b81c842SSasha Levin if (unlikely(error)) 17549b81c842SSasha Levin io->error = error; 17555742fd77SMilan Broz 17565742fd77SMilan Broz crypt_dec_pending(io); 17578b004457SMilan Broz } 17588b004457SMilan Broz 1759028867acSAlasdair G Kergon static void clone_init(struct dm_crypt_io *io, struct bio *clone) 17608b004457SMilan Broz { 176149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 17628b004457SMilan Broz 17638b004457SMilan Broz clone->bi_private = io; 17648b004457SMilan Broz clone->bi_end_io = crypt_endio; 176574d46992SChristoph Hellwig bio_set_dev(clone, cc->dev->bdev); 1766ef295ecfSChristoph Hellwig clone->bi_opf = io->base_bio->bi_opf; 17678b004457SMilan Broz } 17688b004457SMilan Broz 176920c82538SMilan Broz static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) 17708b004457SMilan Broz { 177149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 17728b004457SMilan Broz struct bio *clone; 177393e605c2SMilan Broz 17748b004457SMilan Broz /* 177559779079SMike Snitzer * We need the original biovec array in order to decrypt 177659779079SMike Snitzer * the whole bio data *afterwards* -- thanks to immutable 177759779079SMike Snitzer * biovecs we don't need to worry about the block layer 177859779079SMike Snitzer * modifying the biovec array; so leverage bio_clone_fast(). 17798b004457SMilan Broz */ 17806f1c819cSKent Overstreet clone = bio_clone_fast(io->base_bio, gfp, &cc->bs); 17817eaceaccSJens Axboe if (!clone) 178220c82538SMilan Broz return 1; 17838b004457SMilan Broz 178420c82538SMilan Broz crypt_inc_pending(io); 178520c82538SMilan Broz 17868b004457SMilan Broz clone_init(io, clone); 17874f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 17888b004457SMilan Broz 1789ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) { 1790ef43aa38SMilan Broz crypt_dec_pending(io); 1791ef43aa38SMilan Broz bio_put(clone); 1792ef43aa38SMilan Broz return 1; 1793ef43aa38SMilan Broz } 1794ef43aa38SMilan Broz 179593e605c2SMilan Broz generic_make_request(clone); 179620c82538SMilan Broz return 0; 17978b004457SMilan Broz } 17988b004457SMilan Broz 1799dc267621SMikulas Patocka static void kcryptd_io_read_work(struct work_struct *work) 1800395b167cSAlasdair G Kergon { 1801395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 1802395b167cSAlasdair G Kergon 180320c82538SMilan Broz crypt_inc_pending(io); 180420c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOIO)) 18054e4cbee9SChristoph Hellwig io->error = BLK_STS_RESOURCE; 180620c82538SMilan Broz crypt_dec_pending(io); 1807395b167cSAlasdair G Kergon } 1808395b167cSAlasdair G Kergon 1809dc267621SMikulas Patocka static void kcryptd_queue_read(struct dm_crypt_io *io) 1810395b167cSAlasdair G Kergon { 181149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1812395b167cSAlasdair G Kergon 1813dc267621SMikulas Patocka INIT_WORK(&io->work, kcryptd_io_read_work); 1814395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 1815395b167cSAlasdair G Kergon } 1816395b167cSAlasdair G Kergon 1817dc267621SMikulas Patocka static void kcryptd_io_write(struct dm_crypt_io *io) 1818dc267621SMikulas Patocka { 1819dc267621SMikulas Patocka struct bio *clone = io->ctx.bio_out; 1820dc267621SMikulas Patocka 1821dc267621SMikulas Patocka generic_make_request(clone); 1822dc267621SMikulas Patocka } 1823dc267621SMikulas Patocka 1824b3c5fd30SMikulas Patocka #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node) 1825b3c5fd30SMikulas Patocka 1826dc267621SMikulas Patocka static int dmcrypt_write(void *data) 1827dc267621SMikulas Patocka { 1828dc267621SMikulas Patocka struct crypt_config *cc = data; 1829b3c5fd30SMikulas Patocka struct dm_crypt_io *io; 1830b3c5fd30SMikulas Patocka 1831dc267621SMikulas Patocka while (1) { 1832b3c5fd30SMikulas Patocka struct rb_root write_tree; 1833dc267621SMikulas Patocka struct blk_plug plug; 1834dc267621SMikulas Patocka 1835c7329effSMikulas Patocka spin_lock_irq(&cc->write_thread_lock); 1836dc267621SMikulas Patocka continue_locked: 1837dc267621SMikulas Patocka 1838b3c5fd30SMikulas Patocka if (!RB_EMPTY_ROOT(&cc->write_tree)) 1839dc267621SMikulas Patocka goto pop_from_list; 1840dc267621SMikulas Patocka 1841f659b100SRabin Vincent set_current_state(TASK_INTERRUPTIBLE); 1842dc267621SMikulas Patocka 1843c7329effSMikulas Patocka spin_unlock_irq(&cc->write_thread_lock); 1844dc267621SMikulas Patocka 1845f659b100SRabin Vincent if (unlikely(kthread_should_stop())) { 1846642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING); 1847f659b100SRabin Vincent break; 1848f659b100SRabin Vincent } 1849f659b100SRabin Vincent 1850dc267621SMikulas Patocka schedule(); 1851dc267621SMikulas Patocka 1852642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING); 1853c7329effSMikulas Patocka spin_lock_irq(&cc->write_thread_lock); 1854dc267621SMikulas Patocka goto continue_locked; 1855dc267621SMikulas Patocka 1856dc267621SMikulas Patocka pop_from_list: 1857b3c5fd30SMikulas Patocka write_tree = cc->write_tree; 1858b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 1859c7329effSMikulas Patocka spin_unlock_irq(&cc->write_thread_lock); 1860dc267621SMikulas Patocka 1861b3c5fd30SMikulas Patocka BUG_ON(rb_parent(write_tree.rb_node)); 1862b3c5fd30SMikulas Patocka 1863b3c5fd30SMikulas Patocka /* 1864b3c5fd30SMikulas Patocka * Note: we cannot walk the tree here with rb_next because 1865b3c5fd30SMikulas Patocka * the structures may be freed when kcryptd_io_write is called. 1866b3c5fd30SMikulas Patocka */ 1867dc267621SMikulas Patocka blk_start_plug(&plug); 1868dc267621SMikulas Patocka do { 1869b3c5fd30SMikulas Patocka io = crypt_io_from_node(rb_first(&write_tree)); 1870b3c5fd30SMikulas Patocka rb_erase(&io->rb_node, &write_tree); 1871dc267621SMikulas Patocka kcryptd_io_write(io); 1872b3c5fd30SMikulas Patocka } while (!RB_EMPTY_ROOT(&write_tree)); 1873dc267621SMikulas Patocka blk_finish_plug(&plug); 1874dc267621SMikulas Patocka } 1875dc267621SMikulas Patocka return 0; 1876dc267621SMikulas Patocka } 1877dc267621SMikulas Patocka 187872c6e7afSMikulas Patocka static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) 18794e4eef64SMilan Broz { 1880dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 188149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1882dc267621SMikulas Patocka unsigned long flags; 1883b3c5fd30SMikulas Patocka sector_t sector; 1884b3c5fd30SMikulas Patocka struct rb_node **rbp, *parent; 1885dec1cedfSMilan Broz 18864e4cbee9SChristoph Hellwig if (unlikely(io->error)) { 1887dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 1888dec1cedfSMilan Broz bio_put(clone); 18896c031f41SMilan Broz crypt_dec_pending(io); 1890dec1cedfSMilan Broz return; 1891dec1cedfSMilan Broz } 1892dec1cedfSMilan Broz 1893dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 1894003b5c57SKent Overstreet BUG_ON(io->ctx.iter_out.bi_size); 1895dec1cedfSMilan Broz 18964f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 1897899c95d3SMilan Broz 189839d42fa9SIgnat Korchagin if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) || 189939d42fa9SIgnat Korchagin test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) { 19000f5d8e6eSMikulas Patocka generic_make_request(clone); 19010f5d8e6eSMikulas Patocka return; 19020f5d8e6eSMikulas Patocka } 19030f5d8e6eSMikulas Patocka 1904c7329effSMikulas Patocka spin_lock_irqsave(&cc->write_thread_lock, flags); 1905c7329effSMikulas Patocka if (RB_EMPTY_ROOT(&cc->write_tree)) 1906c7329effSMikulas Patocka wake_up_process(cc->write_thread); 1907b3c5fd30SMikulas Patocka rbp = &cc->write_tree.rb_node; 1908b3c5fd30SMikulas Patocka parent = NULL; 1909b3c5fd30SMikulas Patocka sector = io->sector; 1910b3c5fd30SMikulas Patocka while (*rbp) { 1911b3c5fd30SMikulas Patocka parent = *rbp; 1912b3c5fd30SMikulas Patocka if (sector < crypt_io_from_node(parent)->sector) 1913b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_left; 1914b3c5fd30SMikulas Patocka else 1915b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_right; 1916b3c5fd30SMikulas Patocka } 1917b3c5fd30SMikulas Patocka rb_link_node(&io->rb_node, parent, rbp); 1918b3c5fd30SMikulas Patocka rb_insert_color(&io->rb_node, &cc->write_tree); 1919c7329effSMikulas Patocka spin_unlock_irqrestore(&cc->write_thread_lock, flags); 19204e4eef64SMilan Broz } 19214e4eef64SMilan Broz 1922fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 19238b004457SMilan Broz { 192449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 19258b004457SMilan Broz struct bio *clone; 1926c8081618SMilan Broz int crypt_finished; 1927b635b00eSMilan Broz sector_t sector = io->sector; 19284e4cbee9SChristoph Hellwig blk_status_t r; 19298b004457SMilan Broz 193093e605c2SMilan Broz /* 1931fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 1932fc5a5e9aSMilan Broz */ 1933fc5a5e9aSMilan Broz crypt_inc_pending(io); 1934b635b00eSMilan Broz crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 1935fc5a5e9aSMilan Broz 1936cf2f1abfSMikulas Patocka clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size); 193723541d2dSMilan Broz if (unlikely(!clone)) { 19384e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR; 1939cf2f1abfSMikulas Patocka goto dec; 194023541d2dSMilan Broz } 19418b004457SMilan Broz 194253017030SMilan Broz io->ctx.bio_out = clone; 1943003b5c57SKent Overstreet io->ctx.iter_out = clone->bi_iter; 19448b004457SMilan Broz 1945b635b00eSMilan Broz sector += bio_sectors(clone); 1946dec1cedfSMilan Broz 19474e594098SMilan Broz crypt_inc_pending(io); 194839d42fa9SIgnat Korchagin r = crypt_convert(cc, &io->ctx, 194939d42fa9SIgnat Korchagin test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)); 19504e4cbee9SChristoph Hellwig if (r) 1951ef43aa38SMilan Broz io->error = r; 195240b6229bSMikulas Patocka crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); 1953dec1cedfSMilan Broz 1954c8081618SMilan Broz /* Encryption was already finished, submit io now */ 1955c8081618SMilan Broz if (crypt_finished) { 195672c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 0); 1957b635b00eSMilan Broz io->sector = sector; 19584e594098SMilan Broz } 195993e605c2SMilan Broz 1960cf2f1abfSMikulas Patocka dec: 1961899c95d3SMilan Broz crypt_dec_pending(io); 196284131db6SMilan Broz } 196384131db6SMilan Broz 196472c6e7afSMikulas Patocka static void kcryptd_crypt_read_done(struct dm_crypt_io *io) 19655742fd77SMilan Broz { 19665742fd77SMilan Broz crypt_dec_pending(io); 19675742fd77SMilan Broz } 19685742fd77SMilan Broz 19694e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 19708b004457SMilan Broz { 197149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 19724e4cbee9SChristoph Hellwig blk_status_t r; 19738b004457SMilan Broz 19743e1a8bddSMilan Broz crypt_inc_pending(io); 19753a7f6c99SMilan Broz 197653017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 19770c395b0fSMilan Broz io->sector); 19788b004457SMilan Broz 197939d42fa9SIgnat Korchagin r = crypt_convert(cc, &io->ctx, 198039d42fa9SIgnat Korchagin test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)); 19814e4cbee9SChristoph Hellwig if (r) 1982ef43aa38SMilan Broz io->error = r; 19835742fd77SMilan Broz 198440b6229bSMikulas Patocka if (atomic_dec_and_test(&io->ctx.cc_pending)) 198572c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 19863a7f6c99SMilan Broz 19873a7f6c99SMilan Broz crypt_dec_pending(io); 19888b004457SMilan Broz } 19898b004457SMilan Broz 199095497a96SMilan Broz static void kcryptd_async_done(struct crypto_async_request *async_req, 199195497a96SMilan Broz int error) 199295497a96SMilan Broz { 1993b2174eebSHuang Ying struct dm_crypt_request *dmreq = async_req->data; 1994b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 199595497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 199649a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 199795497a96SMilan Broz 199854cea3f6SMilan Broz /* 199954cea3f6SMilan Broz * A request from crypto driver backlog is going to be processed now, 200054cea3f6SMilan Broz * finish the completion and continue in crypt_convert(). 200154cea3f6SMilan Broz * (Callback will be called for the second time for this request.) 200254cea3f6SMilan Broz */ 2003c0403ec0SRabin Vincent if (error == -EINPROGRESS) { 2004c0403ec0SRabin Vincent complete(&ctx->restart); 200595497a96SMilan Broz return; 2006c0403ec0SRabin Vincent } 200795497a96SMilan Broz 20082dc5327dSMilan Broz if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post) 2009ef43aa38SMilan Broz error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq); 20102dc5327dSMilan Broz 2011ef43aa38SMilan Broz if (error == -EBADMSG) { 2012f710126cSMilan Broz char b[BDEVNAME_SIZE]; 2013f710126cSMilan Broz DMERR_LIMIT("%s: INTEGRITY AEAD ERROR, sector %llu", bio_devname(ctx->bio_in, b), 2014ef43aa38SMilan Broz (unsigned long long)le64_to_cpu(*org_sector_of_dmreq(cc, dmreq))); 20154e4cbee9SChristoph Hellwig io->error = BLK_STS_PROTECTION; 2016ef43aa38SMilan Broz } else if (error < 0) 20174e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR; 201872c6e7afSMikulas Patocka 2019298a9fa0SMikulas Patocka crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio); 202095497a96SMilan Broz 202140b6229bSMikulas Patocka if (!atomic_dec_and_test(&ctx->cc_pending)) 2022c0403ec0SRabin Vincent return; 202395497a96SMilan Broz 202495497a96SMilan Broz if (bio_data_dir(io->base_bio) == READ) 202572c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 202695497a96SMilan Broz else 202772c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 1); 202895497a96SMilan Broz } 202995497a96SMilan Broz 20304e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 20314e4eef64SMilan Broz { 20324e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 20334e4eef64SMilan Broz 20344e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 20354e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 20364e4eef64SMilan Broz else 20374e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 20388b004457SMilan Broz } 20398b004457SMilan Broz 204039d42fa9SIgnat Korchagin static void kcryptd_crypt_tasklet(unsigned long work) 204139d42fa9SIgnat Korchagin { 204239d42fa9SIgnat Korchagin kcryptd_crypt((struct work_struct *)work); 204339d42fa9SIgnat Korchagin } 204439d42fa9SIgnat Korchagin 2045395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 2046395b167cSAlasdair G Kergon { 204749a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 2048395b167cSAlasdair G Kergon 204939d42fa9SIgnat Korchagin if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) || 205039d42fa9SIgnat Korchagin (bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) { 205139d42fa9SIgnat Korchagin if (in_irq()) { 205239d42fa9SIgnat Korchagin /* Crypto API's "skcipher_walk_first() refuses to work in hard IRQ context */ 205339d42fa9SIgnat Korchagin tasklet_init(&io->tasklet, kcryptd_crypt_tasklet, (unsigned long)&io->work); 205439d42fa9SIgnat Korchagin tasklet_schedule(&io->tasklet); 205539d42fa9SIgnat Korchagin return; 205639d42fa9SIgnat Korchagin } 205739d42fa9SIgnat Korchagin 205839d42fa9SIgnat Korchagin kcryptd_crypt(&io->work); 205939d42fa9SIgnat Korchagin return; 206039d42fa9SIgnat Korchagin } 206139d42fa9SIgnat Korchagin 2062395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 2063395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 2064395b167cSAlasdair G Kergon } 2065395b167cSAlasdair G Kergon 2066ef43aa38SMilan Broz static void crypt_free_tfms_aead(struct crypt_config *cc) 20671da177e4SLinus Torvalds { 2068ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms_aead) 2069ef43aa38SMilan Broz return; 20701da177e4SLinus Torvalds 2071ef43aa38SMilan Broz if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) { 2072ef43aa38SMilan Broz crypto_free_aead(cc->cipher_tfm.tfms_aead[0]); 2073ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead[0] = NULL; 20741da177e4SLinus Torvalds } 20751da177e4SLinus Torvalds 2076ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms_aead); 2077ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead = NULL; 2078ef43aa38SMilan Broz } 20791da177e4SLinus Torvalds 2080ef43aa38SMilan Broz static void crypt_free_tfms_skcipher(struct crypt_config *cc) 2081d1f96423SMilan Broz { 2082d1f96423SMilan Broz unsigned i; 2083d1f96423SMilan Broz 2084ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 2085fd2d231fSMikulas Patocka return; 2086fd2d231fSMikulas Patocka 2087d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) 2088ef43aa38SMilan Broz if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) { 2089ef43aa38SMilan Broz crypto_free_skcipher(cc->cipher_tfm.tfms[i]); 2090ef43aa38SMilan Broz cc->cipher_tfm.tfms[i] = NULL; 2091d1f96423SMilan Broz } 2092d1f96423SMilan Broz 2093ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms); 2094ef43aa38SMilan Broz cc->cipher_tfm.tfms = NULL; 20951da177e4SLinus Torvalds } 20961da177e4SLinus Torvalds 20971da177e4SLinus Torvalds static void crypt_free_tfms(struct crypt_config *cc) 2098d1f96423SMilan Broz { 209933d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2100ef43aa38SMilan Broz crypt_free_tfms_aead(cc); 2101ef43aa38SMilan Broz else 2102ef43aa38SMilan Broz crypt_free_tfms_skcipher(cc); 2103d1f96423SMilan Broz } 2104d1f96423SMilan Broz 2105ef43aa38SMilan Broz static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode) 2106d1f96423SMilan Broz { 2107d1f96423SMilan Broz unsigned i; 2108d1f96423SMilan Broz int err; 2109d1f96423SMilan Broz 21106396bb22SKees Cook cc->cipher_tfm.tfms = kcalloc(cc->tfms_count, 21116396bb22SKees Cook sizeof(struct crypto_skcipher *), 21126396bb22SKees Cook GFP_KERNEL); 2113ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 2114fd2d231fSMikulas Patocka return -ENOMEM; 2115fd2d231fSMikulas Patocka 2116d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 2117ef43aa38SMilan Broz cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0, 0); 2118ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms[i])) { 2119ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms[i]); 2120fd2d231fSMikulas Patocka crypt_free_tfms(cc); 2121d1f96423SMilan Broz return err; 2122d1f96423SMilan Broz } 2123d1f96423SMilan Broz } 2124d1f96423SMilan Broz 2125af331ebaSEric Biggers /* 2126af331ebaSEric Biggers * dm-crypt performance can vary greatly depending on which crypto 2127af331ebaSEric Biggers * algorithm implementation is used. Help people debug performance 2128af331ebaSEric Biggers * problems by logging the ->cra_driver_name. 2129af331ebaSEric Biggers */ 21307a1cd723SMilan Broz DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode, 2131af331ebaSEric Biggers crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name); 2132d1f96423SMilan Broz return 0; 2133d1f96423SMilan Broz } 2134d1f96423SMilan Broz 2135ef43aa38SMilan Broz static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode) 2136ef43aa38SMilan Broz { 2137ef43aa38SMilan Broz int err; 2138ef43aa38SMilan Broz 2139ef43aa38SMilan Broz cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL); 2140ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 2141ef43aa38SMilan Broz return -ENOMEM; 2142ef43aa38SMilan Broz 2143ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0, 0); 2144ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) { 2145ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]); 2146ef43aa38SMilan Broz crypt_free_tfms(cc); 2147ef43aa38SMilan Broz return err; 2148ef43aa38SMilan Broz } 2149ef43aa38SMilan Broz 21507a1cd723SMilan Broz DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode, 2151af331ebaSEric Biggers crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name); 2152ef43aa38SMilan Broz return 0; 2153ef43aa38SMilan Broz } 2154ef43aa38SMilan Broz 2155ef43aa38SMilan Broz static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode) 2156ef43aa38SMilan Broz { 215733d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2158ef43aa38SMilan Broz return crypt_alloc_tfms_aead(cc, ciphermode); 2159ef43aa38SMilan Broz else 2160ef43aa38SMilan Broz return crypt_alloc_tfms_skcipher(cc, ciphermode); 2161ef43aa38SMilan Broz } 2162ef43aa38SMilan Broz 2163ef43aa38SMilan Broz static unsigned crypt_subkey_size(struct crypt_config *cc) 2164ef43aa38SMilan Broz { 2165ef43aa38SMilan Broz return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count); 2166ef43aa38SMilan Broz } 2167ef43aa38SMilan Broz 2168ef43aa38SMilan Broz static unsigned crypt_authenckey_size(struct crypt_config *cc) 2169ef43aa38SMilan Broz { 2170ef43aa38SMilan Broz return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param)); 2171ef43aa38SMilan Broz } 2172ef43aa38SMilan Broz 2173ef43aa38SMilan Broz /* 2174ef43aa38SMilan Broz * If AEAD is composed like authenc(hmac(sha256),xts(aes)), 2175ef43aa38SMilan Broz * the key must be for some reason in special format. 2176ef43aa38SMilan Broz * This funcion converts cc->key to this special format. 2177ef43aa38SMilan Broz */ 2178ef43aa38SMilan Broz static void crypt_copy_authenckey(char *p, const void *key, 2179ef43aa38SMilan Broz unsigned enckeylen, unsigned authkeylen) 2180ef43aa38SMilan Broz { 2181ef43aa38SMilan Broz struct crypto_authenc_key_param *param; 2182ef43aa38SMilan Broz struct rtattr *rta; 2183ef43aa38SMilan Broz 2184ef43aa38SMilan Broz rta = (struct rtattr *)p; 2185ef43aa38SMilan Broz param = RTA_DATA(rta); 2186ef43aa38SMilan Broz param->enckeylen = cpu_to_be32(enckeylen); 2187ef43aa38SMilan Broz rta->rta_len = RTA_LENGTH(sizeof(*param)); 2188ef43aa38SMilan Broz rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; 2189ef43aa38SMilan Broz p += RTA_SPACE(sizeof(*param)); 2190ef43aa38SMilan Broz memcpy(p, key + enckeylen, authkeylen); 2191ef43aa38SMilan Broz p += authkeylen; 2192ef43aa38SMilan Broz memcpy(p, key, enckeylen); 2193ef43aa38SMilan Broz } 2194ef43aa38SMilan Broz 2195671ea6b4SMikulas Patocka static int crypt_setkey(struct crypt_config *cc) 2196c0297721SAndi Kleen { 2197da31a078SMilan Broz unsigned subkey_size; 2198fd2d231fSMikulas Patocka int err = 0, i, r; 2199c0297721SAndi Kleen 2200da31a078SMilan Broz /* Ignore extra keys (which are used for IV etc) */ 2201ef43aa38SMilan Broz subkey_size = crypt_subkey_size(cc); 2202da31a078SMilan Broz 220327c70036SMilan Broz if (crypt_integrity_hmac(cc)) { 220427c70036SMilan Broz if (subkey_size < cc->key_mac_size) 220527c70036SMilan Broz return -EINVAL; 220627c70036SMilan Broz 2207ef43aa38SMilan Broz crypt_copy_authenckey(cc->authenc_key, cc->key, 2208ef43aa38SMilan Broz subkey_size - cc->key_mac_size, 2209ef43aa38SMilan Broz cc->key_mac_size); 221027c70036SMilan Broz } 221127c70036SMilan Broz 2212d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 221333d2f09fSMilan Broz if (crypt_integrity_hmac(cc)) 221433d2f09fSMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i], 221533d2f09fSMilan Broz cc->authenc_key, crypt_authenckey_size(cc)); 221633d2f09fSMilan Broz else if (crypt_integrity_aead(cc)) 2217ef43aa38SMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i], 2218ef43aa38SMilan Broz cc->key + (i * subkey_size), 2219ef43aa38SMilan Broz subkey_size); 2220ef43aa38SMilan Broz else 2221ef43aa38SMilan Broz r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i], 2222fd2d231fSMikulas Patocka cc->key + (i * subkey_size), 2223fd2d231fSMikulas Patocka subkey_size); 2224c0297721SAndi Kleen if (r) 2225c0297721SAndi Kleen err = r; 2226c0297721SAndi Kleen } 2227c0297721SAndi Kleen 2228ef43aa38SMilan Broz if (crypt_integrity_hmac(cc)) 2229ef43aa38SMilan Broz memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc)); 2230ef43aa38SMilan Broz 2231c0297721SAndi Kleen return err; 2232c0297721SAndi Kleen } 2233c0297721SAndi Kleen 2234c538f6ecSOndrej Kozina #ifdef CONFIG_KEYS 2235c538f6ecSOndrej Kozina 2236027c431cSOndrej Kozina static bool contains_whitespace(const char *str) 2237027c431cSOndrej Kozina { 2238027c431cSOndrej Kozina while (*str) 2239027c431cSOndrej Kozina if (isspace(*str++)) 2240027c431cSOndrej Kozina return true; 2241027c431cSOndrej Kozina return false; 2242027c431cSOndrej Kozina } 2243027c431cSOndrej Kozina 224427f5411aSDmitry Baryshkov static int set_key_user(struct crypt_config *cc, struct key *key) 224527f5411aSDmitry Baryshkov { 224627f5411aSDmitry Baryshkov const struct user_key_payload *ukp; 224727f5411aSDmitry Baryshkov 224827f5411aSDmitry Baryshkov ukp = user_key_payload_locked(key); 224927f5411aSDmitry Baryshkov if (!ukp) 225027f5411aSDmitry Baryshkov return -EKEYREVOKED; 225127f5411aSDmitry Baryshkov 225227f5411aSDmitry Baryshkov if (cc->key_size != ukp->datalen) 225327f5411aSDmitry Baryshkov return -EINVAL; 225427f5411aSDmitry Baryshkov 225527f5411aSDmitry Baryshkov memcpy(cc->key, ukp->data, cc->key_size); 225627f5411aSDmitry Baryshkov 225727f5411aSDmitry Baryshkov return 0; 225827f5411aSDmitry Baryshkov } 225927f5411aSDmitry Baryshkov 226027f5411aSDmitry Baryshkov #if defined(CONFIG_ENCRYPTED_KEYS) || defined(CONFIG_ENCRYPTED_KEYS_MODULE) 226127f5411aSDmitry Baryshkov static int set_key_encrypted(struct crypt_config *cc, struct key *key) 226227f5411aSDmitry Baryshkov { 226327f5411aSDmitry Baryshkov const struct encrypted_key_payload *ekp; 226427f5411aSDmitry Baryshkov 226527f5411aSDmitry Baryshkov ekp = key->payload.data[0]; 226627f5411aSDmitry Baryshkov if (!ekp) 226727f5411aSDmitry Baryshkov return -EKEYREVOKED; 226827f5411aSDmitry Baryshkov 226927f5411aSDmitry Baryshkov if (cc->key_size != ekp->decrypted_datalen) 227027f5411aSDmitry Baryshkov return -EINVAL; 227127f5411aSDmitry Baryshkov 227227f5411aSDmitry Baryshkov memcpy(cc->key, ekp->decrypted_data, cc->key_size); 227327f5411aSDmitry Baryshkov 227427f5411aSDmitry Baryshkov return 0; 227527f5411aSDmitry Baryshkov } 227627f5411aSDmitry Baryshkov #endif /* CONFIG_ENCRYPTED_KEYS */ 227727f5411aSDmitry Baryshkov 2278c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string) 2279c538f6ecSOndrej Kozina { 2280c538f6ecSOndrej Kozina char *new_key_string, *key_desc; 2281c538f6ecSOndrej Kozina int ret; 228227f5411aSDmitry Baryshkov struct key_type *type; 2283c538f6ecSOndrej Kozina struct key *key; 228427f5411aSDmitry Baryshkov int (*set_key)(struct crypt_config *cc, struct key *key); 2285c538f6ecSOndrej Kozina 2286027c431cSOndrej Kozina /* 2287027c431cSOndrej Kozina * Reject key_string with whitespace. dm core currently lacks code for 2288027c431cSOndrej Kozina * proper whitespace escaping in arguments on DM_TABLE_STATUS path. 2289027c431cSOndrej Kozina */ 2290027c431cSOndrej Kozina if (contains_whitespace(key_string)) { 2291027c431cSOndrej Kozina DMERR("whitespace chars not allowed in key string"); 2292027c431cSOndrej Kozina return -EINVAL; 2293027c431cSOndrej Kozina } 2294027c431cSOndrej Kozina 2295c538f6ecSOndrej Kozina /* look for next ':' separating key_type from key_description */ 2296c538f6ecSOndrej Kozina key_desc = strpbrk(key_string, ":"); 2297c538f6ecSOndrej Kozina if (!key_desc || key_desc == key_string || !strlen(key_desc + 1)) 2298c538f6ecSOndrej Kozina return -EINVAL; 2299c538f6ecSOndrej Kozina 230027f5411aSDmitry Baryshkov if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) { 230127f5411aSDmitry Baryshkov type = &key_type_logon; 230227f5411aSDmitry Baryshkov set_key = set_key_user; 230327f5411aSDmitry Baryshkov } else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) { 230427f5411aSDmitry Baryshkov type = &key_type_user; 230527f5411aSDmitry Baryshkov set_key = set_key_user; 230627f5411aSDmitry Baryshkov #if defined(CONFIG_ENCRYPTED_KEYS) || defined(CONFIG_ENCRYPTED_KEYS_MODULE) 230727f5411aSDmitry Baryshkov } else if (!strncmp(key_string, "encrypted:", key_desc - key_string + 1)) { 230827f5411aSDmitry Baryshkov type = &key_type_encrypted; 230927f5411aSDmitry Baryshkov set_key = set_key_encrypted; 231027f5411aSDmitry Baryshkov #endif 231127f5411aSDmitry Baryshkov } else { 2312c538f6ecSOndrej Kozina return -EINVAL; 231327f5411aSDmitry Baryshkov } 2314c538f6ecSOndrej Kozina 2315c538f6ecSOndrej Kozina new_key_string = kstrdup(key_string, GFP_KERNEL); 2316c538f6ecSOndrej Kozina if (!new_key_string) 2317c538f6ecSOndrej Kozina return -ENOMEM; 2318c538f6ecSOndrej Kozina 231927f5411aSDmitry Baryshkov key = request_key(type, key_desc + 1, NULL); 2320c538f6ecSOndrej Kozina if (IS_ERR(key)) { 2321c538f6ecSOndrej Kozina kzfree(new_key_string); 2322c538f6ecSOndrej Kozina return PTR_ERR(key); 2323c538f6ecSOndrej Kozina } 2324c538f6ecSOndrej Kozina 2325f5b0cba8SOndrej Kozina down_read(&key->sem); 2326c538f6ecSOndrej Kozina 232727f5411aSDmitry Baryshkov ret = set_key(cc, key); 232827f5411aSDmitry Baryshkov if (ret < 0) { 2329f5b0cba8SOndrej Kozina up_read(&key->sem); 2330c538f6ecSOndrej Kozina key_put(key); 2331c538f6ecSOndrej Kozina kzfree(new_key_string); 233227f5411aSDmitry Baryshkov return ret; 2333c538f6ecSOndrej Kozina } 2334c538f6ecSOndrej Kozina 2335f5b0cba8SOndrej Kozina up_read(&key->sem); 2336c538f6ecSOndrej Kozina key_put(key); 2337c538f6ecSOndrej Kozina 2338c538f6ecSOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */ 2339c538f6ecSOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2340c538f6ecSOndrej Kozina 2341c538f6ecSOndrej Kozina ret = crypt_setkey(cc); 2342c538f6ecSOndrej Kozina 2343c538f6ecSOndrej Kozina if (!ret) { 2344c538f6ecSOndrej Kozina set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2345c538f6ecSOndrej Kozina kzfree(cc->key_string); 2346c538f6ecSOndrej Kozina cc->key_string = new_key_string; 2347c538f6ecSOndrej Kozina } else 2348c538f6ecSOndrej Kozina kzfree(new_key_string); 2349c538f6ecSOndrej Kozina 2350c538f6ecSOndrej Kozina return ret; 2351c538f6ecSOndrej Kozina } 2352c538f6ecSOndrej Kozina 2353c538f6ecSOndrej Kozina static int get_key_size(char **key_string) 2354c538f6ecSOndrej Kozina { 2355c538f6ecSOndrej Kozina char *colon, dummy; 2356c538f6ecSOndrej Kozina int ret; 2357c538f6ecSOndrej Kozina 2358c538f6ecSOndrej Kozina if (*key_string[0] != ':') 2359c538f6ecSOndrej Kozina return strlen(*key_string) >> 1; 2360c538f6ecSOndrej Kozina 2361c538f6ecSOndrej Kozina /* look for next ':' in key string */ 2362c538f6ecSOndrej Kozina colon = strpbrk(*key_string + 1, ":"); 2363c538f6ecSOndrej Kozina if (!colon) 2364c538f6ecSOndrej Kozina return -EINVAL; 2365c538f6ecSOndrej Kozina 2366c538f6ecSOndrej Kozina if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':') 2367c538f6ecSOndrej Kozina return -EINVAL; 2368c538f6ecSOndrej Kozina 2369c538f6ecSOndrej Kozina *key_string = colon; 2370c538f6ecSOndrej Kozina 2371c538f6ecSOndrej Kozina /* remaining key string should be :<logon|user>:<key_desc> */ 2372c538f6ecSOndrej Kozina 2373c538f6ecSOndrej Kozina return ret; 2374c538f6ecSOndrej Kozina } 2375c538f6ecSOndrej Kozina 2376c538f6ecSOndrej Kozina #else 2377c538f6ecSOndrej Kozina 2378c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string) 2379c538f6ecSOndrej Kozina { 2380c538f6ecSOndrej Kozina return -EINVAL; 2381c538f6ecSOndrej Kozina } 2382c538f6ecSOndrej Kozina 2383c538f6ecSOndrej Kozina static int get_key_size(char **key_string) 2384c538f6ecSOndrej Kozina { 2385c538f6ecSOndrej Kozina return (*key_string[0] == ':') ? -EINVAL : strlen(*key_string) >> 1; 2386c538f6ecSOndrej Kozina } 2387c538f6ecSOndrej Kozina 238827f5411aSDmitry Baryshkov #endif /* CONFIG_KEYS */ 2389c538f6ecSOndrej Kozina 2390e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 2391e48d4bbfSMilan Broz { 2392de8be5acSMilan Broz int r = -EINVAL; 2393de8be5acSMilan Broz int key_string_len = strlen(key); 2394de8be5acSMilan Broz 239569a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */ 239669a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-")) 2397de8be5acSMilan Broz goto out; 2398e48d4bbfSMilan Broz 2399c538f6ecSOndrej Kozina /* ':' means the key is in kernel keyring, short-circuit normal key processing */ 2400c538f6ecSOndrej Kozina if (key[0] == ':') { 2401c538f6ecSOndrej Kozina r = crypt_set_keyring_key(cc, key + 1); 2402c538f6ecSOndrej Kozina goto out; 2403c538f6ecSOndrej Kozina } 2404c538f6ecSOndrej Kozina 2405265e9098SOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */ 2406265e9098SOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2407265e9098SOndrej Kozina 2408c538f6ecSOndrej Kozina /* wipe references to any kernel keyring key */ 2409c538f6ecSOndrej Kozina kzfree(cc->key_string); 2410c538f6ecSOndrej Kozina cc->key_string = NULL; 2411c538f6ecSOndrej Kozina 2412e944e03eSAndy Shevchenko /* Decode key from its hex representation. */ 2413e944e03eSAndy Shevchenko if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0) 2414de8be5acSMilan Broz goto out; 2415e48d4bbfSMilan Broz 2416671ea6b4SMikulas Patocka r = crypt_setkey(cc); 2417265e9098SOndrej Kozina if (!r) 2418e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2419e48d4bbfSMilan Broz 2420de8be5acSMilan Broz out: 2421de8be5acSMilan Broz /* Hex key string not needed after here, so wipe it. */ 2422de8be5acSMilan Broz memset(key, '0', key_string_len); 2423de8be5acSMilan Broz 2424de8be5acSMilan Broz return r; 2425e48d4bbfSMilan Broz } 2426e48d4bbfSMilan Broz 2427e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 2428e48d4bbfSMilan Broz { 2429c82feeecSOndrej Kozina int r; 2430c82feeecSOndrej Kozina 2431e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2432c82feeecSOndrej Kozina get_random_bytes(&cc->key, cc->key_size); 24334a52ffc7SMilan Broz 24344a52ffc7SMilan Broz /* Wipe IV private keys */ 24354a52ffc7SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 24364a52ffc7SMilan Broz r = cc->iv_gen_ops->wipe(cc); 24374a52ffc7SMilan Broz if (r) 24384a52ffc7SMilan Broz return r; 24394a52ffc7SMilan Broz } 24404a52ffc7SMilan Broz 2441c538f6ecSOndrej Kozina kzfree(cc->key_string); 2442c538f6ecSOndrej Kozina cc->key_string = NULL; 2443c82feeecSOndrej Kozina r = crypt_setkey(cc); 2444c82feeecSOndrej Kozina memset(&cc->key, 0, cc->key_size * sizeof(u8)); 2445c0297721SAndi Kleen 2446c82feeecSOndrej Kozina return r; 2447e48d4bbfSMilan Broz } 2448e48d4bbfSMilan Broz 24495059353dSMikulas Patocka static void crypt_calculate_pages_per_client(void) 24505059353dSMikulas Patocka { 2451ca79b0c2SArun KS unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100; 24525059353dSMikulas Patocka 24535059353dSMikulas Patocka if (!dm_crypt_clients_n) 24545059353dSMikulas Patocka return; 24555059353dSMikulas Patocka 24565059353dSMikulas Patocka pages /= dm_crypt_clients_n; 24575059353dSMikulas Patocka if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT) 24585059353dSMikulas Patocka pages = DM_CRYPT_MIN_PAGES_PER_CLIENT; 24595059353dSMikulas Patocka dm_crypt_pages_per_client = pages; 24605059353dSMikulas Patocka } 24615059353dSMikulas Patocka 24625059353dSMikulas Patocka static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data) 24635059353dSMikulas Patocka { 24645059353dSMikulas Patocka struct crypt_config *cc = pool_data; 24655059353dSMikulas Patocka struct page *page; 24665059353dSMikulas Patocka 24675059353dSMikulas Patocka if (unlikely(percpu_counter_compare(&cc->n_allocated_pages, dm_crypt_pages_per_client) >= 0) && 24685059353dSMikulas Patocka likely(gfp_mask & __GFP_NORETRY)) 24695059353dSMikulas Patocka return NULL; 24705059353dSMikulas Patocka 24715059353dSMikulas Patocka page = alloc_page(gfp_mask); 24725059353dSMikulas Patocka if (likely(page != NULL)) 24735059353dSMikulas Patocka percpu_counter_add(&cc->n_allocated_pages, 1); 24745059353dSMikulas Patocka 24755059353dSMikulas Patocka return page; 24765059353dSMikulas Patocka } 24775059353dSMikulas Patocka 24785059353dSMikulas Patocka static void crypt_page_free(void *page, void *pool_data) 24795059353dSMikulas Patocka { 24805059353dSMikulas Patocka struct crypt_config *cc = pool_data; 24815059353dSMikulas Patocka 24825059353dSMikulas Patocka __free_page(page); 24835059353dSMikulas Patocka percpu_counter_sub(&cc->n_allocated_pages, 1); 24845059353dSMikulas Patocka } 24855059353dSMikulas Patocka 248628513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 248728513fccSMilan Broz { 248828513fccSMilan Broz struct crypt_config *cc = ti->private; 248928513fccSMilan Broz 249028513fccSMilan Broz ti->private = NULL; 249128513fccSMilan Broz 249228513fccSMilan Broz if (!cc) 249328513fccSMilan Broz return; 249428513fccSMilan Broz 2495f659b100SRabin Vincent if (cc->write_thread) 2496dc267621SMikulas Patocka kthread_stop(cc->write_thread); 2497dc267621SMikulas Patocka 249828513fccSMilan Broz if (cc->io_queue) 249928513fccSMilan Broz destroy_workqueue(cc->io_queue); 250028513fccSMilan Broz if (cc->crypt_queue) 250128513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 250228513fccSMilan Broz 2503fd2d231fSMikulas Patocka crypt_free_tfms(cc); 2504fd2d231fSMikulas Patocka 25056f1c819cSKent Overstreet bioset_exit(&cc->bs); 250628513fccSMilan Broz 25076f1c819cSKent Overstreet mempool_exit(&cc->page_pool); 25086f1c819cSKent Overstreet mempool_exit(&cc->req_pool); 25096f1c819cSKent Overstreet mempool_exit(&cc->tag_pool); 25106f1c819cSKent Overstreet 2511d00a11dfSKent Overstreet WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0); 2512d00a11dfSKent Overstreet percpu_counter_destroy(&cc->n_allocated_pages); 2513d00a11dfSKent Overstreet 251428513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 251528513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 251628513fccSMilan Broz 251728513fccSMilan Broz if (cc->dev) 251828513fccSMilan Broz dm_put_device(ti, cc->dev); 251928513fccSMilan Broz 25207dbcd137SMilan Broz kzfree(cc->cipher_string); 2521c538f6ecSOndrej Kozina kzfree(cc->key_string); 2522ef43aa38SMilan Broz kzfree(cc->cipher_auth); 2523ef43aa38SMilan Broz kzfree(cc->authenc_key); 252428513fccSMilan Broz 2525d5ffebddSMike Snitzer mutex_destroy(&cc->bio_alloc_lock); 2526d5ffebddSMike Snitzer 252728513fccSMilan Broz /* Must zero key material before freeing */ 252828513fccSMilan Broz kzfree(cc); 25295059353dSMikulas Patocka 25305059353dSMikulas Patocka spin_lock(&dm_crypt_clients_lock); 25315059353dSMikulas Patocka WARN_ON(!dm_crypt_clients_n); 25325059353dSMikulas Patocka dm_crypt_clients_n--; 25335059353dSMikulas Patocka crypt_calculate_pages_per_client(); 25345059353dSMikulas Patocka spin_unlock(&dm_crypt_clients_lock); 253528513fccSMilan Broz } 253628513fccSMilan Broz 2537e889f97aSMilan Broz static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode) 25381da177e4SLinus Torvalds { 25395ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 25401da177e4SLinus Torvalds 254133d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2542e889f97aSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc)); 2543e889f97aSMilan Broz else 2544bbdb23b5SHerbert Xu cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc)); 2545e889f97aSMilan Broz 25465ebaee6dSMilan Broz if (cc->iv_size) 25475ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 25485ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 25495ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 25505ebaee6dSMilan Broz else if (ivmode) { 25515ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 25525ebaee6dSMilan Broz ivmode = NULL; 25535ebaee6dSMilan Broz } 25545ebaee6dSMilan Broz 25555ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 25561da177e4SLinus Torvalds if (ivmode == NULL) 25571da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 25581da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 25591da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 256061afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 256161afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 25627e3fd855SMilan Broz else if (strcmp(ivmode, "plain64be") == 0) 25637e3fd855SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64be_ops; 25641da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 25651da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 256648527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 256748527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 256846b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 256946b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 2570b9411d73SMilan Broz else if (strcmp(ivmode, "eboiv") == 0) 2571b9411d73SMilan Broz cc->iv_gen_ops = &crypt_iv_eboiv_ops; 2572bbb16584SMilan Broz else if (strcmp(ivmode, "elephant") == 0) { 2573bbb16584SMilan Broz cc->iv_gen_ops = &crypt_iv_elephant_ops; 2574bbb16584SMilan Broz cc->key_parts = 2; 2575bbb16584SMilan Broz cc->key_extra_size = cc->key_size / 2; 2576bbb16584SMilan Broz if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE) 2577bbb16584SMilan Broz return -EINVAL; 2578bbb16584SMilan Broz set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags); 2579bbb16584SMilan Broz } else if (strcmp(ivmode, "lmk") == 0) { 258034745785SMilan Broz cc->iv_gen_ops = &crypt_iv_lmk_ops; 2581ed04d981SMilan Broz /* 2582ed04d981SMilan Broz * Version 2 and 3 is recognised according 258334745785SMilan Broz * to length of provided multi-key string. 258434745785SMilan Broz * If present (version 3), last key is used as IV seed. 2585ed04d981SMilan Broz * All keys (including IV seed) are always the same size. 258634745785SMilan Broz */ 2587da31a078SMilan Broz if (cc->key_size % cc->key_parts) { 258834745785SMilan Broz cc->key_parts++; 2589da31a078SMilan Broz cc->key_extra_size = cc->key_size / cc->key_parts; 2590da31a078SMilan Broz } 2591ed04d981SMilan Broz } else if (strcmp(ivmode, "tcw") == 0) { 2592ed04d981SMilan Broz cc->iv_gen_ops = &crypt_iv_tcw_ops; 2593ed04d981SMilan Broz cc->key_parts += 2; /* IV + whitening */ 2594ed04d981SMilan Broz cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE; 2595e889f97aSMilan Broz } else if (strcmp(ivmode, "random") == 0) { 2596e889f97aSMilan Broz cc->iv_gen_ops = &crypt_iv_random_ops; 2597e889f97aSMilan Broz /* Need storage space in integrity fields. */ 2598e889f97aSMilan Broz cc->integrity_iv_size = cc->iv_size; 259934745785SMilan Broz } else { 260072d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 2601e889f97aSMilan Broz return -EINVAL; 26021da177e4SLinus Torvalds } 26031da177e4SLinus Torvalds 2604e889f97aSMilan Broz return 0; 2605e889f97aSMilan Broz } 2606e889f97aSMilan Broz 260733d2f09fSMilan Broz /* 260833d2f09fSMilan Broz * Workaround to parse HMAC algorithm from AEAD crypto API spec. 260933d2f09fSMilan Broz * The HMAC is needed to calculate tag size (HMAC digest size). 261033d2f09fSMilan Broz * This should be probably done by crypto-api calls (once available...) 261133d2f09fSMilan Broz */ 261233d2f09fSMilan Broz static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api) 261333d2f09fSMilan Broz { 261433d2f09fSMilan Broz char *start, *end, *mac_alg = NULL; 261533d2f09fSMilan Broz struct crypto_ahash *mac; 261633d2f09fSMilan Broz 261733d2f09fSMilan Broz if (!strstarts(cipher_api, "authenc(")) 261833d2f09fSMilan Broz return 0; 261933d2f09fSMilan Broz 262033d2f09fSMilan Broz start = strchr(cipher_api, '('); 262133d2f09fSMilan Broz end = strchr(cipher_api, ','); 262233d2f09fSMilan Broz if (!start || !end || ++start > end) 262333d2f09fSMilan Broz return -EINVAL; 262433d2f09fSMilan Broz 262533d2f09fSMilan Broz mac_alg = kzalloc(end - start + 1, GFP_KERNEL); 262633d2f09fSMilan Broz if (!mac_alg) 262733d2f09fSMilan Broz return -ENOMEM; 262833d2f09fSMilan Broz strncpy(mac_alg, start, end - start); 262933d2f09fSMilan Broz 263033d2f09fSMilan Broz mac = crypto_alloc_ahash(mac_alg, 0, 0); 263133d2f09fSMilan Broz kfree(mac_alg); 263233d2f09fSMilan Broz 263333d2f09fSMilan Broz if (IS_ERR(mac)) 263433d2f09fSMilan Broz return PTR_ERR(mac); 263533d2f09fSMilan Broz 263633d2f09fSMilan Broz cc->key_mac_size = crypto_ahash_digestsize(mac); 263733d2f09fSMilan Broz crypto_free_ahash(mac); 263833d2f09fSMilan Broz 263933d2f09fSMilan Broz cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL); 264033d2f09fSMilan Broz if (!cc->authenc_key) 264133d2f09fSMilan Broz return -ENOMEM; 264233d2f09fSMilan Broz 264333d2f09fSMilan Broz return 0; 264433d2f09fSMilan Broz } 264533d2f09fSMilan Broz 264633d2f09fSMilan Broz static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key, 264733d2f09fSMilan Broz char **ivmode, char **ivopts) 26481da177e4SLinus Torvalds { 26495ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 2650a1a262b6SArd Biesheuvel char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME]; 265133d2f09fSMilan Broz int ret = -EINVAL; 265233d2f09fSMilan Broz 265333d2f09fSMilan Broz cc->tfms_count = 1; 265433d2f09fSMilan Broz 265533d2f09fSMilan Broz /* 265633d2f09fSMilan Broz * New format (capi: prefix) 265733d2f09fSMilan Broz * capi:cipher_api_spec-iv:ivopts 265833d2f09fSMilan Broz */ 265933d2f09fSMilan Broz tmp = &cipher_in[strlen("capi:")]; 26601856b9f7SMilan Broz 26611856b9f7SMilan Broz /* Separate IV options if present, it can contain another '-' in hash name */ 26621856b9f7SMilan Broz *ivopts = strrchr(tmp, ':'); 26631856b9f7SMilan Broz if (*ivopts) { 26641856b9f7SMilan Broz **ivopts = '\0'; 26651856b9f7SMilan Broz (*ivopts)++; 26661856b9f7SMilan Broz } 26671856b9f7SMilan Broz /* Parse IV mode */ 26681856b9f7SMilan Broz *ivmode = strrchr(tmp, '-'); 26691856b9f7SMilan Broz if (*ivmode) { 26701856b9f7SMilan Broz **ivmode = '\0'; 26711856b9f7SMilan Broz (*ivmode)++; 26721856b9f7SMilan Broz } 26731856b9f7SMilan Broz /* The rest is crypto API spec */ 26741856b9f7SMilan Broz cipher_api = tmp; 267533d2f09fSMilan Broz 2676a1a262b6SArd Biesheuvel /* Alloc AEAD, can be used only in new format. */ 2677a1a262b6SArd Biesheuvel if (crypt_integrity_aead(cc)) { 2678a1a262b6SArd Biesheuvel ret = crypt_ctr_auth_cipher(cc, cipher_api); 2679a1a262b6SArd Biesheuvel if (ret < 0) { 2680a1a262b6SArd Biesheuvel ti->error = "Invalid AEAD cipher spec"; 2681a1a262b6SArd Biesheuvel return -ENOMEM; 2682a1a262b6SArd Biesheuvel } 2683a1a262b6SArd Biesheuvel } 2684a1a262b6SArd Biesheuvel 268533d2f09fSMilan Broz if (*ivmode && !strcmp(*ivmode, "lmk")) 268633d2f09fSMilan Broz cc->tfms_count = 64; 268733d2f09fSMilan Broz 2688a1a262b6SArd Biesheuvel if (*ivmode && !strcmp(*ivmode, "essiv")) { 2689a1a262b6SArd Biesheuvel if (!*ivopts) { 2690a1a262b6SArd Biesheuvel ti->error = "Digest algorithm missing for ESSIV mode"; 2691a1a262b6SArd Biesheuvel return -EINVAL; 2692a1a262b6SArd Biesheuvel } 2693a1a262b6SArd Biesheuvel ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, "essiv(%s,%s)", 2694a1a262b6SArd Biesheuvel cipher_api, *ivopts); 2695a1a262b6SArd Biesheuvel if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) { 2696a1a262b6SArd Biesheuvel ti->error = "Cannot allocate cipher string"; 2697a1a262b6SArd Biesheuvel return -ENOMEM; 2698a1a262b6SArd Biesheuvel } 2699a1a262b6SArd Biesheuvel cipher_api = buf; 2700a1a262b6SArd Biesheuvel } 2701a1a262b6SArd Biesheuvel 270233d2f09fSMilan Broz cc->key_parts = cc->tfms_count; 270333d2f09fSMilan Broz 270433d2f09fSMilan Broz /* Allocate cipher */ 270533d2f09fSMilan Broz ret = crypt_alloc_tfms(cc, cipher_api); 270633d2f09fSMilan Broz if (ret < 0) { 270733d2f09fSMilan Broz ti->error = "Error allocating crypto tfm"; 270833d2f09fSMilan Broz return ret; 270933d2f09fSMilan Broz } 271033d2f09fSMilan Broz 2711a1a262b6SArd Biesheuvel if (crypt_integrity_aead(cc)) 271233d2f09fSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc)); 2713a1a262b6SArd Biesheuvel else 271433d2f09fSMilan Broz cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc)); 271533d2f09fSMilan Broz 271633d2f09fSMilan Broz return 0; 271733d2f09fSMilan Broz } 271833d2f09fSMilan Broz 271933d2f09fSMilan Broz static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key, 272033d2f09fSMilan Broz char **ivmode, char **ivopts) 272133d2f09fSMilan Broz { 272233d2f09fSMilan Broz struct crypt_config *cc = ti->private; 272333d2f09fSMilan Broz char *tmp, *cipher, *chainmode, *keycount; 27245ebaee6dSMilan Broz char *cipher_api = NULL; 27255ebaee6dSMilan Broz int ret = -EINVAL; 27265ebaee6dSMilan Broz char dummy; 27275ebaee6dSMilan Broz 272833d2f09fSMilan Broz if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) { 27295ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 27305ebaee6dSMilan Broz return -EINVAL; 27315ebaee6dSMilan Broz } 27325ebaee6dSMilan Broz 27331da177e4SLinus Torvalds /* 27345ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 27355ebaee6dSMilan Broz * cipher[:keycount]-mode-iv:ivopts 27365ebaee6dSMilan Broz */ 27375ebaee6dSMilan Broz tmp = cipher_in; 27385ebaee6dSMilan Broz keycount = strsep(&tmp, "-"); 27395ebaee6dSMilan Broz cipher = strsep(&keycount, ":"); 27405ebaee6dSMilan Broz 274169a8cfcdSMilan Broz if (!keycount) 27425ebaee6dSMilan Broz cc->tfms_count = 1; 27435ebaee6dSMilan Broz else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 || 27445ebaee6dSMilan Broz !is_power_of_2(cc->tfms_count)) { 27455ebaee6dSMilan Broz ti->error = "Bad cipher key count specification"; 27465ebaee6dSMilan Broz return -EINVAL; 27475ebaee6dSMilan Broz } 274828513fccSMilan Broz cc->key_parts = cc->tfms_count; 27491da177e4SLinus Torvalds 2750ddd42edfSMilan Broz chainmode = strsep(&tmp, "-"); 27511856b9f7SMilan Broz *ivmode = strsep(&tmp, ":"); 27521856b9f7SMilan Broz *ivopts = tmp; 2753ddd42edfSMilan Broz 2754ddd42edfSMilan Broz /* 2755ddd42edfSMilan Broz * For compatibility with the original dm-crypt mapping format, if 2756ddd42edfSMilan Broz * only the cipher name is supplied, use cbc-plain. 275728513fccSMilan Broz */ 275833d2f09fSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) { 2759cabf08e4SMilan Broz chainmode = "cbc"; 276033d2f09fSMilan Broz *ivmode = "plain"; 2761cabf08e4SMilan Broz } 2762cabf08e4SMilan Broz 276333d2f09fSMilan Broz if (strcmp(chainmode, "ecb") && !*ivmode) { 2764c0297721SAndi Kleen ti->error = "IV mechanism required"; 2765c0297721SAndi Kleen return -EINVAL; 2766c0297721SAndi Kleen } 2767c0297721SAndi Kleen 2768cabf08e4SMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 27699934a8beSMilan Broz if (!cipher_api) 277028513fccSMilan Broz goto bad_mem; 27719934a8beSMilan Broz 2772a1a262b6SArd Biesheuvel if (*ivmode && !strcmp(*ivmode, "essiv")) { 2773a1a262b6SArd Biesheuvel if (!*ivopts) { 2774a1a262b6SArd Biesheuvel ti->error = "Digest algorithm missing for ESSIV mode"; 2775a1a262b6SArd Biesheuvel kfree(cipher_api); 2776a1a262b6SArd Biesheuvel return -EINVAL; 2777a1a262b6SArd Biesheuvel } 2778a1a262b6SArd Biesheuvel ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 2779a1a262b6SArd Biesheuvel "essiv(%s(%s),%s)", chainmode, cipher, *ivopts); 2780a1a262b6SArd Biesheuvel } else { 27819934a8beSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 2782647c7db1SMikulas Patocka "%s(%s)", chainmode, cipher); 2783a1a262b6SArd Biesheuvel } 2784a1a262b6SArd Biesheuvel if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) { 27851da177e4SLinus Torvalds kfree(cipher_api); 278628513fccSMilan Broz goto bad_mem; 278728513fccSMilan Broz } 278828513fccSMilan Broz 27891da177e4SLinus Torvalds /* Allocate cipher */ 27901da177e4SLinus Torvalds ret = crypt_alloc_tfms(cc, cipher_api); 27911da177e4SLinus Torvalds if (ret < 0) { 27921da177e4SLinus Torvalds ti->error = "Error allocating crypto tfm"; 279333d2f09fSMilan Broz kfree(cipher_api); 279433d2f09fSMilan Broz return ret; 2795028867acSAlasdair G Kergon } 2796bd86e320SJeffy Chen kfree(cipher_api); 2797647c7db1SMikulas Patocka 279833d2f09fSMilan Broz return 0; 279933d2f09fSMilan Broz bad_mem: 280033d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings"; 280133d2f09fSMilan Broz return -ENOMEM; 280233d2f09fSMilan Broz } 280333d2f09fSMilan Broz 280433d2f09fSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key) 280533d2f09fSMilan Broz { 280633d2f09fSMilan Broz struct crypt_config *cc = ti->private; 280733d2f09fSMilan Broz char *ivmode = NULL, *ivopts = NULL; 280833d2f09fSMilan Broz int ret; 280933d2f09fSMilan Broz 281033d2f09fSMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL); 281133d2f09fSMilan Broz if (!cc->cipher_string) { 281233d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings"; 281333d2f09fSMilan Broz return -ENOMEM; 281433d2f09fSMilan Broz } 281533d2f09fSMilan Broz 281633d2f09fSMilan Broz if (strstarts(cipher_in, "capi:")) 281733d2f09fSMilan Broz ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts); 281833d2f09fSMilan Broz else 281933d2f09fSMilan Broz ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts); 282033d2f09fSMilan Broz if (ret) 282133d2f09fSMilan Broz return ret; 282233d2f09fSMilan Broz 2823647c7db1SMikulas Patocka /* Initialize IV */ 2824e889f97aSMilan Broz ret = crypt_ctr_ivmode(ti, ivmode); 2825e889f97aSMilan Broz if (ret < 0) 282633d2f09fSMilan Broz return ret; 28271da177e4SLinus Torvalds 2828da31a078SMilan Broz /* Initialize and set key */ 2829da31a078SMilan Broz ret = crypt_set_key(cc, key); 2830da31a078SMilan Broz if (ret < 0) { 2831da31a078SMilan Broz ti->error = "Error decoding and setting key"; 283233d2f09fSMilan Broz return ret; 2833da31a078SMilan Broz } 2834da31a078SMilan Broz 28351da177e4SLinus Torvalds /* Allocate IV */ 28361da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 28371da177e4SLinus Torvalds ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 28381da177e4SLinus Torvalds if (ret < 0) { 28391da177e4SLinus Torvalds ti->error = "Error creating IV"; 284033d2f09fSMilan Broz return ret; 28411da177e4SLinus Torvalds } 28421da177e4SLinus Torvalds } 28431da177e4SLinus Torvalds 28441da177e4SLinus Torvalds /* Initialize IV (set keys for ESSIV etc) */ 28451da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 28461da177e4SLinus Torvalds ret = cc->iv_gen_ops->init(cc); 28471da177e4SLinus Torvalds if (ret < 0) { 28481da177e4SLinus Torvalds ti->error = "Error initialising IV"; 28491da177e4SLinus Torvalds return ret; 28501da177e4SLinus Torvalds } 28511da177e4SLinus Torvalds } 28521da177e4SLinus Torvalds 2853dc94902bSOndrej Kozina /* wipe the kernel key payload copy */ 2854dc94902bSOndrej Kozina if (cc->key_string) 2855dc94902bSOndrej Kozina memset(cc->key, 0, cc->key_size * sizeof(u8)); 2856dc94902bSOndrej Kozina 285733d2f09fSMilan Broz return ret; 28581da177e4SLinus Torvalds } 28591da177e4SLinus Torvalds 2860ef43aa38SMilan Broz static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv) 2861ef43aa38SMilan Broz { 2862ef43aa38SMilan Broz struct crypt_config *cc = ti->private; 2863ef43aa38SMilan Broz struct dm_arg_set as; 28645916a22bSEric Biggers static const struct dm_arg _args[] = { 286539d42fa9SIgnat Korchagin {0, 8, "Invalid number of feature args"}, 2866ef43aa38SMilan Broz }; 2867ef43aa38SMilan Broz unsigned int opt_params, val; 2868ef43aa38SMilan Broz const char *opt_string, *sval; 28698f0009a2SMilan Broz char dummy; 2870ef43aa38SMilan Broz int ret; 2871ef43aa38SMilan Broz 2872ef43aa38SMilan Broz /* Optional parameters */ 2873ef43aa38SMilan Broz as.argc = argc; 2874ef43aa38SMilan Broz as.argv = argv; 2875ef43aa38SMilan Broz 2876ef43aa38SMilan Broz ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 2877ef43aa38SMilan Broz if (ret) 28781da177e4SLinus Torvalds return ret; 28791da177e4SLinus Torvalds 2880ef43aa38SMilan Broz while (opt_params--) { 2881ef43aa38SMilan Broz opt_string = dm_shift_arg(&as); 2882ef43aa38SMilan Broz if (!opt_string) { 2883ef43aa38SMilan Broz ti->error = "Not enough feature arguments"; 2884ef43aa38SMilan Broz return -EINVAL; 2885ef43aa38SMilan Broz } 2886ef43aa38SMilan Broz 2887ef43aa38SMilan Broz if (!strcasecmp(opt_string, "allow_discards")) 2888ef43aa38SMilan Broz ti->num_discard_bios = 1; 2889ef43aa38SMilan Broz 2890ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "same_cpu_crypt")) 2891ef43aa38SMilan Broz set_bit(DM_CRYPT_SAME_CPU, &cc->flags); 2892ef43aa38SMilan Broz 2893ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "submit_from_crypt_cpus")) 2894ef43aa38SMilan Broz set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 289539d42fa9SIgnat Korchagin else if (!strcasecmp(opt_string, "no_read_workqueue")) 289639d42fa9SIgnat Korchagin set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); 289739d42fa9SIgnat Korchagin else if (!strcasecmp(opt_string, "no_write_workqueue")) 289839d42fa9SIgnat Korchagin set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); 2899ef43aa38SMilan Broz else if (sscanf(opt_string, "integrity:%u:", &val) == 1) { 2900ef43aa38SMilan Broz if (val == 0 || val > MAX_TAG_SIZE) { 2901ef43aa38SMilan Broz ti->error = "Invalid integrity arguments"; 2902ef43aa38SMilan Broz return -EINVAL; 2903ef43aa38SMilan Broz } 2904ef43aa38SMilan Broz cc->on_disk_tag_size = val; 2905ef43aa38SMilan Broz sval = strchr(opt_string + strlen("integrity:"), ':') + 1; 2906ef43aa38SMilan Broz if (!strcasecmp(sval, "aead")) { 2907ef43aa38SMilan Broz set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags); 2908ef43aa38SMilan Broz } else if (strcasecmp(sval, "none")) { 2909ef43aa38SMilan Broz ti->error = "Unknown integrity profile"; 2910ef43aa38SMilan Broz return -EINVAL; 2911ef43aa38SMilan Broz } 2912ef43aa38SMilan Broz 2913ef43aa38SMilan Broz cc->cipher_auth = kstrdup(sval, GFP_KERNEL); 2914ef43aa38SMilan Broz if (!cc->cipher_auth) 29151da177e4SLinus Torvalds return -ENOMEM; 2916ff3af92bSMikulas Patocka } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) { 29178f0009a2SMilan Broz if (cc->sector_size < (1 << SECTOR_SHIFT) || 29188f0009a2SMilan Broz cc->sector_size > 4096 || 2919ff3af92bSMikulas Patocka (cc->sector_size & (cc->sector_size - 1))) { 29208f0009a2SMilan Broz ti->error = "Invalid feature value for sector_size"; 29218f0009a2SMilan Broz return -EINVAL; 29228f0009a2SMilan Broz } 2923783874b0SMilan Broz if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) { 2924783874b0SMilan Broz ti->error = "Device size is not multiple of sector_size feature"; 2925783874b0SMilan Broz return -EINVAL; 2926783874b0SMilan Broz } 2927ff3af92bSMikulas Patocka cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT; 29288f0009a2SMilan Broz } else if (!strcasecmp(opt_string, "iv_large_sectors")) 29298f0009a2SMilan Broz set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags); 29308f0009a2SMilan Broz else { 2931ef43aa38SMilan Broz ti->error = "Invalid feature arguments"; 2932ef43aa38SMilan Broz return -EINVAL; 2933ef43aa38SMilan Broz } 2934ef43aa38SMilan Broz } 2935ef43aa38SMilan Broz 2936ef43aa38SMilan Broz return 0; 29371da177e4SLinus Torvalds } 29381da177e4SLinus Torvalds 29391da177e4SLinus Torvalds /* 29401da177e4SLinus Torvalds * Construct an encryption mapping: 2941c538f6ecSOndrej Kozina * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start> 29421da177e4SLinus Torvalds */ 29431da177e4SLinus Torvalds static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 29441da177e4SLinus Torvalds { 29451da177e4SLinus Torvalds struct crypt_config *cc; 2946ed0302e8SMichał Mirosław const char *devname = dm_table_device_name(ti->table); 2947c538f6ecSOndrej Kozina int key_size; 2948ef43aa38SMilan Broz unsigned int align_mask; 29491da177e4SLinus Torvalds unsigned long long tmpll; 29501da177e4SLinus Torvalds int ret; 2951ef43aa38SMilan Broz size_t iv_size_padding, additional_req_size; 295231998ef1SMikulas Patocka char dummy; 29531da177e4SLinus Torvalds 2954772ae5f5SMilan Broz if (argc < 5) { 29551da177e4SLinus Torvalds ti->error = "Not enough arguments"; 29561da177e4SLinus Torvalds return -EINVAL; 29571da177e4SLinus Torvalds } 29581da177e4SLinus Torvalds 2959c538f6ecSOndrej Kozina key_size = get_key_size(&argv[1]); 2960c538f6ecSOndrej Kozina if (key_size < 0) { 2961c538f6ecSOndrej Kozina ti->error = "Cannot parse key size"; 2962c538f6ecSOndrej Kozina return -EINVAL; 2963c538f6ecSOndrej Kozina } 29641da177e4SLinus Torvalds 29659c81c99bSZhengyuan Liu cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL); 29661da177e4SLinus Torvalds if (!cc) { 29671da177e4SLinus Torvalds ti->error = "Cannot allocate encryption context"; 29681da177e4SLinus Torvalds return -ENOMEM; 29691da177e4SLinus Torvalds } 29701da177e4SLinus Torvalds cc->key_size = key_size; 29718f0009a2SMilan Broz cc->sector_size = (1 << SECTOR_SHIFT); 2972ff3af92bSMikulas Patocka cc->sector_shift = 0; 29731da177e4SLinus Torvalds 29741da177e4SLinus Torvalds ti->private = cc; 2975ef43aa38SMilan Broz 29765059353dSMikulas Patocka spin_lock(&dm_crypt_clients_lock); 29775059353dSMikulas Patocka dm_crypt_clients_n++; 29785059353dSMikulas Patocka crypt_calculate_pages_per_client(); 29795059353dSMikulas Patocka spin_unlock(&dm_crypt_clients_lock); 29805059353dSMikulas Patocka 29815059353dSMikulas Patocka ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL); 29825059353dSMikulas Patocka if (ret < 0) 29835059353dSMikulas Patocka goto bad; 29845059353dSMikulas Patocka 2985ef43aa38SMilan Broz /* Optional parameters need to be read before cipher constructor */ 2986ef43aa38SMilan Broz if (argc > 5) { 2987ef43aa38SMilan Broz ret = crypt_ctr_optional(ti, argc - 5, &argv[5]); 2988ef43aa38SMilan Broz if (ret) 2989ef43aa38SMilan Broz goto bad; 2990ef43aa38SMilan Broz } 2991ef43aa38SMilan Broz 29921da177e4SLinus Torvalds ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 29931da177e4SLinus Torvalds if (ret < 0) 29941da177e4SLinus Torvalds goto bad; 29951da177e4SLinus Torvalds 299633d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 2997ef43aa38SMilan Broz cc->dmreq_start = sizeof(struct aead_request); 2998ef43aa38SMilan Broz cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc)); 2999ef43aa38SMilan Broz align_mask = crypto_aead_alignmask(any_tfm_aead(cc)); 3000ef43aa38SMilan Broz } else { 3001bbdb23b5SHerbert Xu cc->dmreq_start = sizeof(struct skcipher_request); 3002bbdb23b5SHerbert Xu cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc)); 3003ef43aa38SMilan Broz align_mask = crypto_skcipher_alignmask(any_tfm(cc)); 3004ef43aa38SMilan Broz } 3005d49ec52fSMikulas Patocka cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request)); 3006d49ec52fSMikulas Patocka 3007ef43aa38SMilan Broz if (align_mask < CRYPTO_MINALIGN) { 3008d49ec52fSMikulas Patocka /* Allocate the padding exactly */ 3009d49ec52fSMikulas Patocka iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request)) 3010ef43aa38SMilan Broz & align_mask; 3011d49ec52fSMikulas Patocka } else { 3012d49ec52fSMikulas Patocka /* 3013d49ec52fSMikulas Patocka * If the cipher requires greater alignment than kmalloc 3014d49ec52fSMikulas Patocka * alignment, we don't know the exact position of the 3015d49ec52fSMikulas Patocka * initialization vector. We must assume worst case. 3016d49ec52fSMikulas Patocka */ 3017ef43aa38SMilan Broz iv_size_padding = align_mask; 3018d49ec52fSMikulas Patocka } 30191da177e4SLinus Torvalds 3020ef43aa38SMilan Broz /* ...| IV + padding | original IV | original sec. number | bio tag offset | */ 3021ef43aa38SMilan Broz additional_req_size = sizeof(struct dm_crypt_request) + 3022ef43aa38SMilan Broz iv_size_padding + cc->iv_size + 3023ef43aa38SMilan Broz cc->iv_size + 3024ef43aa38SMilan Broz sizeof(uint64_t) + 3025ef43aa38SMilan Broz sizeof(unsigned int); 3026ef43aa38SMilan Broz 30276f1c819cSKent Overstreet ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size); 30286f1c819cSKent Overstreet if (ret) { 30291da177e4SLinus Torvalds ti->error = "Cannot allocate crypt request mempool"; 30301da177e4SLinus Torvalds goto bad; 30311da177e4SLinus Torvalds } 30321da177e4SLinus Torvalds 303330187e1dSMike Snitzer cc->per_bio_data_size = ti->per_io_data_size = 3034ef43aa38SMilan Broz ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size, 3035d49ec52fSMikulas Patocka ARCH_KMALLOC_MINALIGN); 3036298a9fa0SMikulas Patocka 30376f1c819cSKent Overstreet ret = mempool_init(&cc->page_pool, BIO_MAX_PAGES, crypt_page_alloc, crypt_page_free, cc); 30386f1c819cSKent Overstreet if (ret) { 30398b004457SMilan Broz ti->error = "Cannot allocate page mempool"; 3040e48d4bbfSMilan Broz goto bad; 30411da177e4SLinus Torvalds } 3042e48d4bbfSMilan Broz 30436f1c819cSKent Overstreet ret = bioset_init(&cc->bs, MIN_IOS, 0, BIOSET_NEED_BVECS); 30446f1c819cSKent Overstreet if (ret) { 30450c395b0fSMilan Broz ti->error = "Cannot allocate crypt bioset"; 3046cabf08e4SMilan Broz goto bad; 304793e605c2SMilan Broz } 3048cabf08e4SMilan Broz 30497145c241SMikulas Patocka mutex_init(&cc->bio_alloc_lock); 30507145c241SMikulas Patocka 3051cabf08e4SMilan Broz ret = -EINVAL; 30528f0009a2SMilan Broz if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) || 30538f0009a2SMilan Broz (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) { 3054cabf08e4SMilan Broz ti->error = "Invalid iv_offset sector"; 3055cabf08e4SMilan Broz goto bad; 30561da177e4SLinus Torvalds } 3057d2a7ad29SKiyoshi Ueda cc->iv_offset = tmpll; 30581da177e4SLinus Torvalds 3059e80d1c80SVivek Goyal ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev); 3060e80d1c80SVivek Goyal if (ret) { 30611da177e4SLinus Torvalds ti->error = "Device lookup failed"; 30621da177e4SLinus Torvalds goto bad; 30631da177e4SLinus Torvalds } 30641da177e4SLinus Torvalds 3065e80d1c80SVivek Goyal ret = -EINVAL; 3066ef87bfc2SMilan Broz if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) { 30671da177e4SLinus Torvalds ti->error = "Invalid device sector"; 30681da177e4SLinus Torvalds goto bad; 30691da177e4SLinus Torvalds } 30701da177e4SLinus Torvalds cc->start = tmpll; 30711da177e4SLinus Torvalds 307233d2f09fSMilan Broz if (crypt_integrity_aead(cc) || cc->integrity_iv_size) { 3073ef43aa38SMilan Broz ret = crypt_integrity_ctr(cc, ti); 3074772ae5f5SMilan Broz if (ret) 3075772ae5f5SMilan Broz goto bad; 3076772ae5f5SMilan Broz 3077ef43aa38SMilan Broz cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size; 3078ef43aa38SMilan Broz if (!cc->tag_pool_max_sectors) 3079ef43aa38SMilan Broz cc->tag_pool_max_sectors = 1; 3080ef43aa38SMilan Broz 30816f1c819cSKent Overstreet ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS, 3082ef43aa38SMilan Broz cc->tag_pool_max_sectors * cc->on_disk_tag_size); 30836f1c819cSKent Overstreet if (ret) { 3084ef43aa38SMilan Broz ti->error = "Cannot allocate integrity tags mempool"; 3085f3396c58SMikulas Patocka goto bad; 3086f3396c58SMikulas Patocka } 3087772ae5f5SMilan Broz 3088583fe747SMikulas Patocka cc->tag_pool_max_sectors <<= cc->sector_shift; 3089f3396c58SMikulas Patocka } 3090772ae5f5SMilan Broz 30911da177e4SLinus Torvalds ret = -ENOMEM; 3092f612b213SMike Snitzer cc->io_queue = alloc_workqueue("kcryptd_io/%s", WQ_MEM_RECLAIM, 1, devname); 30931da177e4SLinus Torvalds if (!cc->io_queue) { 30941da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd io queue"; 30951da177e4SLinus Torvalds goto bad; 30961da177e4SLinus Torvalds } 309737af6560SChristophe Saout 3098f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 3099f612b213SMike Snitzer cc->crypt_queue = alloc_workqueue("kcryptd/%s", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 3100ed0302e8SMichał Mirosław 1, devname); 3101f3396c58SMikulas Patocka else 3102ed0302e8SMichał Mirosław cc->crypt_queue = alloc_workqueue("kcryptd/%s", 3103f612b213SMike Snitzer WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, 3104ed0302e8SMichał Mirosław num_online_cpus(), devname); 31051da177e4SLinus Torvalds if (!cc->crypt_queue) { 31061da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd queue"; 31071da177e4SLinus Torvalds goto bad; 31081da177e4SLinus Torvalds } 31091da177e4SLinus Torvalds 3110c7329effSMikulas Patocka spin_lock_init(&cc->write_thread_lock); 3111b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 3112dc267621SMikulas Patocka 3113ed0302e8SMichał Mirosław cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write/%s", devname); 3114dc267621SMikulas Patocka if (IS_ERR(cc->write_thread)) { 3115dc267621SMikulas Patocka ret = PTR_ERR(cc->write_thread); 3116dc267621SMikulas Patocka cc->write_thread = NULL; 3117dc267621SMikulas Patocka ti->error = "Couldn't spawn write thread"; 3118dc267621SMikulas Patocka goto bad; 3119dc267621SMikulas Patocka } 3120dc267621SMikulas Patocka wake_up_process(cc->write_thread); 3121dc267621SMikulas Patocka 312255a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 3123983c7db3SMilan Broz 31241da177e4SLinus Torvalds return 0; 31251da177e4SLinus Torvalds 31261da177e4SLinus Torvalds bad: 31271da177e4SLinus Torvalds crypt_dtr(ti); 31281da177e4SLinus Torvalds return ret; 3129647c7db1SMikulas Patocka } 3130647c7db1SMikulas Patocka 31317de3ee57SMikulas Patocka static int crypt_map(struct dm_target *ti, struct bio *bio) 31321da177e4SLinus Torvalds { 31331da177e4SLinus Torvalds struct dm_crypt_io *io; 313449a8a920SAlasdair G Kergon struct crypt_config *cc = ti->private; 3135647c7db1SMikulas Patocka 3136772ae5f5SMilan Broz /* 313728a8f0d3SMike Christie * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues. 313828a8f0d3SMike Christie * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight 3139e6047149SMike Christie * - for REQ_OP_DISCARD caller must use flush if IO ordering matters 3140772ae5f5SMilan Broz */ 31411eff9d32SJens Axboe if (unlikely(bio->bi_opf & REQ_PREFLUSH || 314228a8f0d3SMike Christie bio_op(bio) == REQ_OP_DISCARD)) { 314374d46992SChristoph Hellwig bio_set_dev(bio, cc->dev->bdev); 3144772ae5f5SMilan Broz if (bio_sectors(bio)) 31454f024f37SKent Overstreet bio->bi_iter.bi_sector = cc->start + 31464f024f37SKent Overstreet dm_target_offset(ti, bio->bi_iter.bi_sector); 3147647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 3148647c7db1SMikulas Patocka } 31491da177e4SLinus Torvalds 31504e870e94SMikulas Patocka /* 31514e870e94SMikulas Patocka * Check if bio is too large, split as needed. 31524e870e94SMikulas Patocka */ 31534e870e94SMikulas Patocka if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_PAGES << PAGE_SHIFT)) && 3154ef43aa38SMilan Broz (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size)) 31554e870e94SMikulas Patocka dm_accept_partial_bio(bio, ((BIO_MAX_PAGES << PAGE_SHIFT) >> SECTOR_SHIFT)); 31564e870e94SMikulas Patocka 31578f0009a2SMilan Broz /* 31588f0009a2SMilan Broz * Ensure that bio is a multiple of internal sector encryption size 31598f0009a2SMilan Broz * and is aligned to this size as defined in IO hints. 31608f0009a2SMilan Broz */ 31618f0009a2SMilan Broz if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0)) 3162846785e6SChristoph Hellwig return DM_MAPIO_KILL; 31638f0009a2SMilan Broz 31648f0009a2SMilan Broz if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1))) 3165846785e6SChristoph Hellwig return DM_MAPIO_KILL; 31668f0009a2SMilan Broz 3167298a9fa0SMikulas Patocka io = dm_per_bio_data(bio, cc->per_bio_data_size); 3168298a9fa0SMikulas Patocka crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector)); 3169ef43aa38SMilan Broz 3170ef43aa38SMilan Broz if (cc->on_disk_tag_size) { 3171583fe747SMikulas Patocka unsigned tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift); 3172ef43aa38SMilan Broz 3173ef43aa38SMilan Broz if (unlikely(tag_len > KMALLOC_MAX_SIZE) || 3174583fe747SMikulas Patocka unlikely(!(io->integrity_metadata = kmalloc(tag_len, 3175ef43aa38SMilan Broz GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN)))) { 3176ef43aa38SMilan Broz if (bio_sectors(bio) > cc->tag_pool_max_sectors) 3177ef43aa38SMilan Broz dm_accept_partial_bio(bio, cc->tag_pool_max_sectors); 31786f1c819cSKent Overstreet io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO); 3179ef43aa38SMilan Broz io->integrity_metadata_from_pool = true; 3180ef43aa38SMilan Broz } 3181ef43aa38SMilan Broz } 3182ef43aa38SMilan Broz 318333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 3184ef43aa38SMilan Broz io->ctx.r.req_aead = (struct aead_request *)(io + 1); 3185ef43aa38SMilan Broz else 3186ef43aa38SMilan Broz io->ctx.r.req = (struct skcipher_request *)(io + 1); 31871da177e4SLinus Torvalds 318820c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 318920c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOWAIT)) 3190dc267621SMikulas Patocka kcryptd_queue_read(io); 319120c82538SMilan Broz } else 31924ee218cdSAndrew Morton kcryptd_queue_crypt(io); 31934ee218cdSAndrew Morton 31941da177e4SLinus Torvalds return DM_MAPIO_SUBMITTED; 31951da177e4SLinus Torvalds } 31961da177e4SLinus Torvalds 3197fd7c092eSMikulas Patocka static void crypt_status(struct dm_target *ti, status_type_t type, 31981f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 31991da177e4SLinus Torvalds { 32005ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 3201fd7c092eSMikulas Patocka unsigned i, sz = 0; 3202f3396c58SMikulas Patocka int num_feature_args = 0; 32031da177e4SLinus Torvalds 32041da177e4SLinus Torvalds switch (type) { 32051da177e4SLinus Torvalds case STATUSTYPE_INFO: 32061da177e4SLinus Torvalds result[0] = '\0'; 32071da177e4SLinus Torvalds break; 32081da177e4SLinus Torvalds 32091da177e4SLinus Torvalds case STATUSTYPE_TABLE: 32107dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string); 32111da177e4SLinus Torvalds 3212c538f6ecSOndrej Kozina if (cc->key_size > 0) { 3213c538f6ecSOndrej Kozina if (cc->key_string) 3214c538f6ecSOndrej Kozina DMEMIT(":%u:%s", cc->key_size, cc->key_string); 3215c538f6ecSOndrej Kozina else 3216fd7c092eSMikulas Patocka for (i = 0; i < cc->key_size; i++) 3217fd7c092eSMikulas Patocka DMEMIT("%02x", cc->key[i]); 3218c538f6ecSOndrej Kozina } else 3219fd7c092eSMikulas Patocka DMEMIT("-"); 32201da177e4SLinus Torvalds 32211da177e4SLinus Torvalds DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 32221da177e4SLinus Torvalds cc->dev->name, (unsigned long long)cc->start); 3223772ae5f5SMilan Broz 3224f3396c58SMikulas Patocka num_feature_args += !!ti->num_discard_bios; 3225f3396c58SMikulas Patocka num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags); 32260f5d8e6eSMikulas Patocka num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 322739d42fa9SIgnat Korchagin num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); 322839d42fa9SIgnat Korchagin num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); 3229ff3af92bSMikulas Patocka num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT); 32308f0009a2SMilan Broz num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags); 3231ef43aa38SMilan Broz if (cc->on_disk_tag_size) 3232ef43aa38SMilan Broz num_feature_args++; 3233f3396c58SMikulas Patocka if (num_feature_args) { 3234f3396c58SMikulas Patocka DMEMIT(" %d", num_feature_args); 323555a62eefSAlasdair G Kergon if (ti->num_discard_bios) 3236f3396c58SMikulas Patocka DMEMIT(" allow_discards"); 3237f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 3238f3396c58SMikulas Patocka DMEMIT(" same_cpu_crypt"); 32390f5d8e6eSMikulas Patocka if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) 32400f5d8e6eSMikulas Patocka DMEMIT(" submit_from_crypt_cpus"); 324139d42fa9SIgnat Korchagin if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) 324239d42fa9SIgnat Korchagin DMEMIT(" no_read_workqueue"); 324339d42fa9SIgnat Korchagin if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) 324439d42fa9SIgnat Korchagin DMEMIT(" no_write_workqueue"); 3245ef43aa38SMilan Broz if (cc->on_disk_tag_size) 3246ef43aa38SMilan Broz DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth); 32478f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) 32488f0009a2SMilan Broz DMEMIT(" sector_size:%d", cc->sector_size); 32498f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 32508f0009a2SMilan Broz DMEMIT(" iv_large_sectors"); 3251f3396c58SMikulas Patocka } 3252772ae5f5SMilan Broz 32531da177e4SLinus Torvalds break; 32541da177e4SLinus Torvalds } 32551da177e4SLinus Torvalds } 32561da177e4SLinus Torvalds 3257e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 3258e48d4bbfSMilan Broz { 3259e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3260e48d4bbfSMilan Broz 3261e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 3262e48d4bbfSMilan Broz } 3263e48d4bbfSMilan Broz 3264e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 3265e48d4bbfSMilan Broz { 3266e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3267e48d4bbfSMilan Broz 3268e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 3269e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 3270e48d4bbfSMilan Broz return -EAGAIN; 3271e48d4bbfSMilan Broz } 3272e48d4bbfSMilan Broz 3273e48d4bbfSMilan Broz return 0; 3274e48d4bbfSMilan Broz } 3275e48d4bbfSMilan Broz 3276e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 3277e48d4bbfSMilan Broz { 3278e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3279e48d4bbfSMilan Broz 3280e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 3281e48d4bbfSMilan Broz } 3282e48d4bbfSMilan Broz 3283e48d4bbfSMilan Broz /* Message interface 3284e48d4bbfSMilan Broz * key set <key> 3285e48d4bbfSMilan Broz * key wipe 3286e48d4bbfSMilan Broz */ 32871eb5fa84SMike Snitzer static int crypt_message(struct dm_target *ti, unsigned argc, char **argv, 32881eb5fa84SMike Snitzer char *result, unsigned maxlen) 3289e48d4bbfSMilan Broz { 3290e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3291c538f6ecSOndrej Kozina int key_size, ret = -EINVAL; 3292e48d4bbfSMilan Broz 3293e48d4bbfSMilan Broz if (argc < 2) 3294e48d4bbfSMilan Broz goto error; 3295e48d4bbfSMilan Broz 3296498f0103SMike Snitzer if (!strcasecmp(argv[0], "key")) { 3297e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 3298e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 3299e48d4bbfSMilan Broz return -EINVAL; 3300e48d4bbfSMilan Broz } 3301498f0103SMike Snitzer if (argc == 3 && !strcasecmp(argv[1], "set")) { 3302c538f6ecSOndrej Kozina /* The key size may not be changed. */ 3303c538f6ecSOndrej Kozina key_size = get_key_size(&argv[2]); 3304c538f6ecSOndrej Kozina if (key_size < 0 || cc->key_size != key_size) { 3305c538f6ecSOndrej Kozina memset(argv[2], '0', strlen(argv[2])); 3306c538f6ecSOndrej Kozina return -EINVAL; 3307c538f6ecSOndrej Kozina } 3308c538f6ecSOndrej Kozina 3309542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 3310542da317SMilan Broz if (ret) 3311542da317SMilan Broz return ret; 3312542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 3313542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 3314dc94902bSOndrej Kozina /* wipe the kernel key payload copy */ 3315dc94902bSOndrej Kozina if (cc->key_string) 3316dc94902bSOndrej Kozina memset(cc->key, 0, cc->key_size * sizeof(u8)); 3317542da317SMilan Broz return ret; 3318542da317SMilan Broz } 33194a52ffc7SMilan Broz if (argc == 2 && !strcasecmp(argv[1], "wipe")) 3320e48d4bbfSMilan Broz return crypt_wipe_key(cc); 3321e48d4bbfSMilan Broz } 3322e48d4bbfSMilan Broz 3323e48d4bbfSMilan Broz error: 3324e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 3325e48d4bbfSMilan Broz return -EINVAL; 3326e48d4bbfSMilan Broz } 3327e48d4bbfSMilan Broz 3328af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 3329af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 3330af4874e0SMike Snitzer { 3331af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 3332af4874e0SMike Snitzer 33335dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 3334af4874e0SMike Snitzer } 3335af4874e0SMike Snitzer 3336586b286bSMike Snitzer static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits) 3337586b286bSMike Snitzer { 33388f0009a2SMilan Broz struct crypt_config *cc = ti->private; 33398f0009a2SMilan Broz 3340586b286bSMike Snitzer /* 3341586b286bSMike Snitzer * Unfortunate constraint that is required to avoid the potential 3342586b286bSMike Snitzer * for exceeding underlying device's max_segments limits -- due to 3343586b286bSMike Snitzer * crypt_alloc_buffer() possibly allocating pages for the encryption 3344586b286bSMike Snitzer * bio that are not as physically contiguous as the original bio. 3345586b286bSMike Snitzer */ 3346586b286bSMike Snitzer limits->max_segment_size = PAGE_SIZE; 33478f0009a2SMilan Broz 3348bc9e9cf0SMikulas Patocka limits->logical_block_size = 334964611a15SEric Biggers max_t(unsigned, limits->logical_block_size, cc->sector_size); 3350bc9e9cf0SMikulas Patocka limits->physical_block_size = 3351bc9e9cf0SMikulas Patocka max_t(unsigned, limits->physical_block_size, cc->sector_size); 3352bc9e9cf0SMikulas Patocka limits->io_min = max_t(unsigned, limits->io_min, cc->sector_size); 3353586b286bSMike Snitzer } 3354586b286bSMike Snitzer 33551da177e4SLinus Torvalds static struct target_type crypt_target = { 33561da177e4SLinus Torvalds .name = "crypt", 335739d42fa9SIgnat Korchagin .version = {1, 22, 0}, 33581da177e4SLinus Torvalds .module = THIS_MODULE, 33591da177e4SLinus Torvalds .ctr = crypt_ctr, 33601da177e4SLinus Torvalds .dtr = crypt_dtr, 33611da177e4SLinus Torvalds .map = crypt_map, 33621da177e4SLinus Torvalds .status = crypt_status, 3363e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 3364e48d4bbfSMilan Broz .preresume = crypt_preresume, 3365e48d4bbfSMilan Broz .resume = crypt_resume, 3366e48d4bbfSMilan Broz .message = crypt_message, 3367af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 3368586b286bSMike Snitzer .io_hints = crypt_io_hints, 33691da177e4SLinus Torvalds }; 33701da177e4SLinus Torvalds 33711da177e4SLinus Torvalds static int __init dm_crypt_init(void) 33721da177e4SLinus Torvalds { 33731da177e4SLinus Torvalds int r; 33741da177e4SLinus Torvalds 33751da177e4SLinus Torvalds r = dm_register_target(&crypt_target); 337694f5e024SMikulas Patocka if (r < 0) 337772d94861SAlasdair G Kergon DMERR("register failed %d", r); 33781da177e4SLinus Torvalds 33791da177e4SLinus Torvalds return r; 33801da177e4SLinus Torvalds } 33811da177e4SLinus Torvalds 33821da177e4SLinus Torvalds static void __exit dm_crypt_exit(void) 33831da177e4SLinus Torvalds { 338410d3bd09SMikulas Patocka dm_unregister_target(&crypt_target); 33851da177e4SLinus Torvalds } 33861da177e4SLinus Torvalds 33871da177e4SLinus Torvalds module_init(dm_crypt_init); 33881da177e4SLinus Torvalds module_exit(dm_crypt_exit); 33891da177e4SLinus Torvalds 3390bf14299fSJana Saout MODULE_AUTHOR("Jana Saout <jana@saout.de>"); 33911da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 33921da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 3393