13bd94003SHeinz Mauelshagen // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 3bf14299fSJana Saout * Copyright (C) 2003 Jana Saout <jana@saout.de> 41da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 5bbb16584SMilan Broz * Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved. 6bbb16584SMilan Broz * Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com> 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * This file is released under the GPL. 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 1143d69034SMilan Broz #include <linux/completion.h> 12d1806f6aSHerbert Xu #include <linux/err.h> 131da177e4SLinus Torvalds #include <linux/module.h> 141da177e4SLinus Torvalds #include <linux/init.h> 151da177e4SLinus Torvalds #include <linux/kernel.h> 16c538f6ecSOndrej Kozina #include <linux/key.h> 171da177e4SLinus Torvalds #include <linux/bio.h> 181da177e4SLinus Torvalds #include <linux/blkdev.h> 19fe45e630SChristoph Hellwig #include <linux/blk-integrity.h> 201da177e4SLinus Torvalds #include <linux/mempool.h> 211da177e4SLinus Torvalds #include <linux/slab.h> 221da177e4SLinus Torvalds #include <linux/crypto.h> 231da177e4SLinus Torvalds #include <linux/workqueue.h> 24dc267621SMikulas Patocka #include <linux/kthread.h> 253fcfab16SAndrew Morton #include <linux/backing-dev.h> 2660063497SArun Sharma #include <linux/atomic.h> 27378f058cSDavid Hardeman #include <linux/scatterlist.h> 28b3c5fd30SMikulas Patocka #include <linux/rbtree.h> 29027c431cSOndrej Kozina #include <linux/ctype.h> 301da177e4SLinus Torvalds #include <asm/page.h> 3148527fa7SRik Snel #include <asm/unaligned.h> 3234745785SMilan Broz #include <crypto/hash.h> 3334745785SMilan Broz #include <crypto/md5.h> 3434745785SMilan Broz #include <crypto/algapi.h> 35bbdb23b5SHerbert Xu #include <crypto/skcipher.h> 36ef43aa38SMilan Broz #include <crypto/aead.h> 37ef43aa38SMilan Broz #include <crypto/authenc.h> 38ef43aa38SMilan Broz #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */ 3927f5411aSDmitry Baryshkov #include <linux/key-type.h> 40c538f6ecSOndrej Kozina #include <keys/user-type.h> 4127f5411aSDmitry Baryshkov #include <keys/encrypted-type.h> 42363880c4SAhmad Fatoum #include <keys/trusted-type.h> 431da177e4SLinus Torvalds 44586e80e6SMikulas Patocka #include <linux/device-mapper.h> 451da177e4SLinus Torvalds 4658d0f180SMichael Weiß #include "dm-audit.h" 4758d0f180SMichael Weiß 4872d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt" 491da177e4SLinus Torvalds 501da177e4SLinus Torvalds /* 511da177e4SLinus Torvalds * context holding the current state of a multi-part conversion 521da177e4SLinus Torvalds */ 531da177e4SLinus Torvalds struct convert_context { 5443d69034SMilan Broz struct completion restart; 551da177e4SLinus Torvalds struct bio *bio_in; 561da177e4SLinus Torvalds struct bio *bio_out; 57003b5c57SKent Overstreet struct bvec_iter iter_in; 58003b5c57SKent Overstreet struct bvec_iter iter_out; 598d683dcdSAliOS system security u64 cc_sector; 6040b6229bSMikulas Patocka atomic_t cc_pending; 61ef43aa38SMilan Broz union { 62bbdb23b5SHerbert Xu struct skcipher_request *req; 63ef43aa38SMilan Broz struct aead_request *req_aead; 64ef43aa38SMilan Broz } r; 65ef43aa38SMilan Broz 661da177e4SLinus Torvalds }; 671da177e4SLinus Torvalds 6853017030SMilan Broz /* 6953017030SMilan Broz * per bio private data 7053017030SMilan Broz */ 7153017030SMilan Broz struct dm_crypt_io { 7249a8a920SAlasdair G Kergon struct crypt_config *cc; 7353017030SMilan Broz struct bio *base_bio; 74ef43aa38SMilan Broz u8 *integrity_metadata; 75d9a02e01SMike Snitzer bool integrity_metadata_from_pool:1; 76d9a02e01SMike Snitzer bool in_tasklet:1; 77d9a02e01SMike Snitzer 7853017030SMilan Broz struct work_struct work; 7939d42fa9SIgnat Korchagin struct tasklet_struct tasklet; 8053017030SMilan Broz 8153017030SMilan Broz struct convert_context ctx; 8253017030SMilan Broz 8340b6229bSMikulas Patocka atomic_t io_pending; 844e4cbee9SChristoph Hellwig blk_status_t error; 850c395b0fSMilan Broz sector_t sector; 86dc267621SMikulas Patocka 87b3c5fd30SMikulas Patocka struct rb_node rb_node; 88298a9fa0SMikulas Patocka } CRYPTO_MINALIGN_ATTR; 8953017030SMilan Broz 9001482b76SMilan Broz struct dm_crypt_request { 91b2174eebSHuang Ying struct convert_context *ctx; 92ef43aa38SMilan Broz struct scatterlist sg_in[4]; 93ef43aa38SMilan Broz struct scatterlist sg_out[4]; 948d683dcdSAliOS system security u64 iv_sector; 9501482b76SMilan Broz }; 9601482b76SMilan Broz 971da177e4SLinus Torvalds struct crypt_config; 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds struct crypt_iv_operations { 1001da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 1011da177e4SLinus Torvalds const char *opts); 1021da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc); 103b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc); 104542da317SMilan Broz int (*wipe)(struct crypt_config *cc); 1052dc5327dSMilan Broz int (*generator)(struct crypt_config *cc, u8 *iv, 1062dc5327dSMilan Broz struct dm_crypt_request *dmreq); 1072dc5327dSMilan Broz int (*post)(struct crypt_config *cc, u8 *iv, 1082dc5327dSMilan Broz struct dm_crypt_request *dmreq); 1091da177e4SLinus Torvalds }; 1101da177e4SLinus Torvalds 11160473592SMilan Broz struct iv_benbi_private { 11260473592SMilan Broz int shift; 11360473592SMilan Broz }; 11460473592SMilan Broz 11534745785SMilan Broz #define LMK_SEED_SIZE 64 /* hash + 0 */ 11634745785SMilan Broz struct iv_lmk_private { 11734745785SMilan Broz struct crypto_shash *hash_tfm; 11834745785SMilan Broz u8 *seed; 11934745785SMilan Broz }; 12034745785SMilan Broz 121ed04d981SMilan Broz #define TCW_WHITENING_SIZE 16 122ed04d981SMilan Broz struct iv_tcw_private { 123ed04d981SMilan Broz struct crypto_shash *crc32_tfm; 124ed04d981SMilan Broz u8 *iv_seed; 125ed04d981SMilan Broz u8 *whitening; 126ed04d981SMilan Broz }; 127ed04d981SMilan Broz 128bbb16584SMilan Broz #define ELEPHANT_MAX_KEY_SIZE 32 129bbb16584SMilan Broz struct iv_elephant_private { 130bbb16584SMilan Broz struct crypto_skcipher *tfm; 131bbb16584SMilan Broz }; 132bbb16584SMilan Broz 1331da177e4SLinus Torvalds /* 1341da177e4SLinus Torvalds * Crypt: maps a linear range of a block device 1351da177e4SLinus Torvalds * and encrypts / decrypts at the same time. 1361da177e4SLinus Torvalds */ 1370f5d8e6eSMikulas Patocka enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID, 13839d42fa9SIgnat Korchagin DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD, 1398e225f04SDamien Le Moal DM_CRYPT_NO_READ_WORKQUEUE, DM_CRYPT_NO_WRITE_WORKQUEUE, 1408e225f04SDamien Le Moal DM_CRYPT_WRITE_INLINE }; 141c0297721SAndi Kleen 142ef43aa38SMilan Broz enum cipher_flags { 14374d1da39SGeert Uytterhoeven CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cipher */ 1448f0009a2SMilan Broz CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */ 145bbb16584SMilan Broz CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */ 146ef43aa38SMilan Broz }; 147ef43aa38SMilan Broz 148c0297721SAndi Kleen /* 149610f2de3SMikulas Patocka * The fields in here must be read only after initialization. 150c0297721SAndi Kleen */ 1511da177e4SLinus Torvalds struct crypt_config { 1521da177e4SLinus Torvalds struct dm_dev *dev; 1531da177e4SLinus Torvalds sector_t start; 1541da177e4SLinus Torvalds 1555059353dSMikulas Patocka struct percpu_counter n_allocated_pages; 1565059353dSMikulas Patocka 157cabf08e4SMilan Broz struct workqueue_struct *io_queue; 158cabf08e4SMilan Broz struct workqueue_struct *crypt_queue; 1593f1e9070SMilan Broz 160c7329effSMikulas Patocka spinlock_t write_thread_lock; 16172d711c8SMike Snitzer struct task_struct *write_thread; 162b3c5fd30SMikulas Patocka struct rb_root write_tree; 163dc267621SMikulas Patocka 1647dbcd137SMilan Broz char *cipher_string; 165ef43aa38SMilan Broz char *cipher_auth; 166c538f6ecSOndrej Kozina char *key_string; 1675ebaee6dSMilan Broz 1681b1b58f5SJulia Lawall const struct crypt_iv_operations *iv_gen_ops; 16979066ad3SHerbert Xu union { 17060473592SMilan Broz struct iv_benbi_private benbi; 17134745785SMilan Broz struct iv_lmk_private lmk; 172ed04d981SMilan Broz struct iv_tcw_private tcw; 173bbb16584SMilan Broz struct iv_elephant_private elephant; 17479066ad3SHerbert Xu } iv_gen_private; 1758d683dcdSAliOS system security u64 iv_offset; 1761da177e4SLinus Torvalds unsigned int iv_size; 17786a3238cSHeinz Mauelshagen unsigned short sector_size; 178ff3af92bSMikulas Patocka unsigned char sector_shift; 1791da177e4SLinus Torvalds 180ef43aa38SMilan Broz union { 181bbdb23b5SHerbert Xu struct crypto_skcipher **tfms; 182ef43aa38SMilan Broz struct crypto_aead **tfms_aead; 183ef43aa38SMilan Broz } cipher_tfm; 18486a3238cSHeinz Mauelshagen unsigned int tfms_count; 185ef43aa38SMilan Broz unsigned long cipher_flags; 186c0297721SAndi Kleen 187c0297721SAndi Kleen /* 188ddd42edfSMilan Broz * Layout of each crypto request: 189ddd42edfSMilan Broz * 190bbdb23b5SHerbert Xu * struct skcipher_request 191ddd42edfSMilan Broz * context 192ddd42edfSMilan Broz * padding 193ddd42edfSMilan Broz * struct dm_crypt_request 194ddd42edfSMilan Broz * padding 195ddd42edfSMilan Broz * IV 196ddd42edfSMilan Broz * 197ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are 198ddd42edfSMilan Broz * correctly aligned. 199ddd42edfSMilan Broz */ 200ddd42edfSMilan Broz unsigned int dmreq_start; 201ddd42edfSMilan Broz 202298a9fa0SMikulas Patocka unsigned int per_bio_data_size; 203298a9fa0SMikulas Patocka 204e48d4bbfSMilan Broz unsigned long flags; 2051da177e4SLinus Torvalds unsigned int key_size; 206da31a078SMilan Broz unsigned int key_parts; /* independent parts in key buffer */ 207da31a078SMilan Broz unsigned int key_extra_size; /* additional keys length */ 208ef43aa38SMilan Broz unsigned int key_mac_size; /* MAC key size for authenc(...) */ 209ef43aa38SMilan Broz 210ef43aa38SMilan Broz unsigned int integrity_tag_size; 211ef43aa38SMilan Broz unsigned int integrity_iv_size; 212ef43aa38SMilan Broz unsigned int on_disk_tag_size; 213ef43aa38SMilan Broz 21472d711c8SMike Snitzer /* 21572d711c8SMike Snitzer * pool for per bio private data, crypto requests, 21672d711c8SMike Snitzer * encryption requeusts/buffer pages and integrity tags 21772d711c8SMike Snitzer */ 21886a3238cSHeinz Mauelshagen unsigned int tag_pool_max_sectors; 21972d711c8SMike Snitzer mempool_t tag_pool; 22072d711c8SMike Snitzer mempool_t req_pool; 22172d711c8SMike Snitzer mempool_t page_pool; 22272d711c8SMike Snitzer 22372d711c8SMike Snitzer struct bio_set bs; 22472d711c8SMike Snitzer struct mutex bio_alloc_lock; 22572d711c8SMike Snitzer 226ef43aa38SMilan Broz u8 *authenc_key; /* space for keys in authenc() format (if used) */ 227b18ae8ddSGustavo A. R. Silva u8 key[]; 2281da177e4SLinus Torvalds }; 2291da177e4SLinus Torvalds 2300a83df6cSMikulas Patocka #define MIN_IOS 64 231ef43aa38SMilan Broz #define MAX_TAG_SIZE 480 232ef43aa38SMilan Broz #define POOL_ENTRY_SIZE 512 2331da177e4SLinus Torvalds 2345059353dSMikulas Patocka static DEFINE_SPINLOCK(dm_crypt_clients_lock); 2352f06cd12SHeinz Mauelshagen static unsigned int dm_crypt_clients_n; 2365059353dSMikulas Patocka static volatile unsigned long dm_crypt_pages_per_client; 2375059353dSMikulas Patocka #define DM_CRYPT_MEMORY_PERCENT 2 238a8affc03SChristoph Hellwig #define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_VECS * 16) 2395059353dSMikulas Patocka 2403f868c09SChristoph Hellwig static void crypt_endio(struct bio *clone); 241395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io); 242ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc, 243ef43aa38SMilan Broz struct scatterlist *sg); 244027581f3SOlaf Kirch 2453fd53533SYang Yingliang static bool crypt_integrity_aead(struct crypt_config *cc); 2463fd53533SYang Yingliang 247c0297721SAndi Kleen /* 24886f917adSEric Biggers * Use this to access cipher attributes that are independent of the key. 249c0297721SAndi Kleen */ 250bbdb23b5SHerbert Xu static struct crypto_skcipher *any_tfm(struct crypt_config *cc) 251c0297721SAndi Kleen { 252ef43aa38SMilan Broz return cc->cipher_tfm.tfms[0]; 253ef43aa38SMilan Broz } 254ef43aa38SMilan Broz 255ef43aa38SMilan Broz static struct crypto_aead *any_tfm_aead(struct crypt_config *cc) 256ef43aa38SMilan Broz { 257ef43aa38SMilan Broz return cc->cipher_tfm.tfms_aead[0]; 258c0297721SAndi Kleen } 259c0297721SAndi Kleen 2601da177e4SLinus Torvalds /* 2611da177e4SLinus Torvalds * Different IV generation algorithms: 2621da177e4SLinus Torvalds * 2633c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector 2643a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary. 2651da177e4SLinus Torvalds * 26661afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector 26761afef61SMilan Broz * number, padded with zeros if necessary. 26861afef61SMilan Broz * 2697e3fd855SMilan Broz * plain64be: the initial vector is the 64-bit big-endian version of the sector 2707e3fd855SMilan Broz * number, padded with zeros if necessary. 2717e3fd855SMilan Broz * 2723c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is 2731da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt 2741da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing. 2751da177e4SLinus Torvalds * 27648527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 27748527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes) 27848527fa7SRik Snel * 27946b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with 28046b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices. 28146b47730SLudwig Nussel * 28234745785SMilan Broz * lmk: Compatible implementation of the block chaining mode used 28334745785SMilan Broz * by the Loop-AES block device encryption system 28434745785SMilan Broz * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/ 28534745785SMilan Broz * It operates on full 512 byte sectors and uses CBC 28634745785SMilan Broz * with an IV derived from the sector number, the data and 28734745785SMilan Broz * optionally extra IV seed. 28834745785SMilan Broz * This means that after decryption the first block 28934745785SMilan Broz * of sector must be tweaked according to decrypted data. 29034745785SMilan Broz * Loop-AES can use three encryption schemes: 29134745785SMilan Broz * version 1: is plain aes-cbc mode 29234745785SMilan Broz * version 2: uses 64 multikey scheme with lmk IV generator 29334745785SMilan Broz * version 3: the same as version 2 with additional IV seed 29434745785SMilan Broz * (it uses 65 keys, last key is used as IV seed) 29534745785SMilan Broz * 296ed04d981SMilan Broz * tcw: Compatible implementation of the block chaining mode used 297ed04d981SMilan Broz * by the TrueCrypt device encryption system (prior to version 4.1). 298e44f23b3SMilan Broz * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat 299ed04d981SMilan Broz * It operates on full 512 byte sectors and uses CBC 300ed04d981SMilan Broz * with an IV derived from initial key and the sector number. 301ed04d981SMilan Broz * In addition, whitening value is applied on every sector, whitening 302ed04d981SMilan Broz * is calculated from initial key, sector number and mixed using CRC32. 303ed04d981SMilan Broz * Note that this encryption scheme is vulnerable to watermarking attacks 304ed04d981SMilan Broz * and should be used for old compatible containers access only. 305b9411d73SMilan Broz * 306b9411d73SMilan Broz * eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode) 307b9411d73SMilan Broz * The IV is encrypted little-endian byte-offset (with the same key 308b9411d73SMilan Broz * and cipher as the volume). 309bbb16584SMilan Broz * 310bbb16584SMilan Broz * elephant: The extended version of eboiv with additional Elephant diffuser 311bbb16584SMilan Broz * used with Bitlocker CBC mode. 312bbb16584SMilan Broz * This mode was used in older Windows systems 3136f3bc22bSAlexander A. Klimov * https://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf 3141da177e4SLinus Torvalds */ 3151da177e4SLinus Torvalds 3162dc5327dSMilan Broz static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, 3172dc5327dSMilan Broz struct dm_crypt_request *dmreq) 3181da177e4SLinus Torvalds { 3191da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 320283a8328SAlasdair G Kergon *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff); 3211da177e4SLinus Torvalds 3221da177e4SLinus Torvalds return 0; 3231da177e4SLinus Torvalds } 3241da177e4SLinus Torvalds 32561afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv, 3262dc5327dSMilan Broz struct dm_crypt_request *dmreq) 32761afef61SMilan Broz { 32861afef61SMilan Broz memset(iv, 0, cc->iv_size); 329283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 33061afef61SMilan Broz 33161afef61SMilan Broz return 0; 33261afef61SMilan Broz } 33361afef61SMilan Broz 3347e3fd855SMilan Broz static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv, 3357e3fd855SMilan Broz struct dm_crypt_request *dmreq) 3367e3fd855SMilan Broz { 3377e3fd855SMilan Broz memset(iv, 0, cc->iv_size); 3387e3fd855SMilan Broz /* iv_size is at least of size u64; usually it is 16 bytes */ 3397e3fd855SMilan Broz *(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector); 3407e3fd855SMilan Broz 3417e3fd855SMilan Broz return 0; 3427e3fd855SMilan Broz } 3437e3fd855SMilan Broz 3442dc5327dSMilan Broz static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, 3452dc5327dSMilan Broz struct dm_crypt_request *dmreq) 3461da177e4SLinus Torvalds { 347a1a262b6SArd Biesheuvel /* 348a1a262b6SArd Biesheuvel * ESSIV encryption of the IV is now handled by the crypto API, 349a1a262b6SArd Biesheuvel * so just pass the plain sector number here. 350a1a262b6SArd Biesheuvel */ 3511da177e4SLinus Torvalds memset(iv, 0, cc->iv_size); 352283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector); 353c0297721SAndi Kleen 3541da177e4SLinus Torvalds return 0; 3551da177e4SLinus Torvalds } 3561da177e4SLinus Torvalds 35748527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 35848527fa7SRik Snel const char *opts) 35948527fa7SRik Snel { 36086a3238cSHeinz Mauelshagen unsigned int bs; 3614ea9471fSMilan Broz int log; 3624ea9471fSMilan Broz 3633fd53533SYang Yingliang if (crypt_integrity_aead(cc)) 3644ea9471fSMilan Broz bs = crypto_aead_blocksize(any_tfm_aead(cc)); 3654ea9471fSMilan Broz else 3664ea9471fSMilan Broz bs = crypto_skcipher_blocksize(any_tfm(cc)); 3674ea9471fSMilan Broz log = ilog2(bs); 36848527fa7SRik Snel 369a4a82ce3SHeinz Mauelshagen /* 370a4a82ce3SHeinz Mauelshagen * We need to calculate how far we must shift the sector count 371a4a82ce3SHeinz Mauelshagen * to get the cipher block count, we use this shift in _gen. 372a4a82ce3SHeinz Mauelshagen */ 37348527fa7SRik Snel if (1 << log != bs) { 37448527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2"; 37548527fa7SRik Snel return -EINVAL; 37648527fa7SRik Snel } 37748527fa7SRik Snel 37848527fa7SRik Snel if (log > 9) { 37948527fa7SRik Snel ti->error = "cypher blocksize is > 512"; 38048527fa7SRik Snel return -EINVAL; 38148527fa7SRik Snel } 38248527fa7SRik Snel 38360473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log; 38448527fa7SRik Snel 38548527fa7SRik Snel return 0; 38648527fa7SRik Snel } 38748527fa7SRik Snel 38848527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc) 38948527fa7SRik Snel { 39048527fa7SRik Snel } 39148527fa7SRik Snel 3922dc5327dSMilan Broz static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, 3932dc5327dSMilan Broz struct dm_crypt_request *dmreq) 39448527fa7SRik Snel { 39579066ad3SHerbert Xu __be64 val; 39679066ad3SHerbert Xu 39748527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 39879066ad3SHerbert Xu 3992dc5327dSMilan Broz val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1); 40079066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 40148527fa7SRik Snel 4021da177e4SLinus Torvalds return 0; 4031da177e4SLinus Torvalds } 4041da177e4SLinus Torvalds 4052dc5327dSMilan Broz static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, 4062dc5327dSMilan Broz struct dm_crypt_request *dmreq) 40746b47730SLudwig Nussel { 40846b47730SLudwig Nussel memset(iv, 0, cc->iv_size); 40946b47730SLudwig Nussel 41046b47730SLudwig Nussel return 0; 41146b47730SLudwig Nussel } 41246b47730SLudwig Nussel 41334745785SMilan Broz static void crypt_iv_lmk_dtr(struct crypt_config *cc) 41434745785SMilan Broz { 41534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 41634745785SMilan Broz 41734745785SMilan Broz if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm)) 41834745785SMilan Broz crypto_free_shash(lmk->hash_tfm); 41934745785SMilan Broz lmk->hash_tfm = NULL; 42034745785SMilan Broz 421453431a5SWaiman Long kfree_sensitive(lmk->seed); 42234745785SMilan Broz lmk->seed = NULL; 42334745785SMilan Broz } 42434745785SMilan Broz 42534745785SMilan Broz static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti, 42634745785SMilan Broz const char *opts) 42734745785SMilan Broz { 42834745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 42934745785SMilan Broz 4308f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) { 4318f0009a2SMilan Broz ti->error = "Unsupported sector size for LMK"; 4328f0009a2SMilan Broz return -EINVAL; 4338f0009a2SMilan Broz } 4348f0009a2SMilan Broz 435cd746938SMikulas Patocka lmk->hash_tfm = crypto_alloc_shash("md5", 0, 436cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY); 43734745785SMilan Broz if (IS_ERR(lmk->hash_tfm)) { 43834745785SMilan Broz ti->error = "Error initializing LMK hash"; 43934745785SMilan Broz return PTR_ERR(lmk->hash_tfm); 44034745785SMilan Broz } 44134745785SMilan Broz 44234745785SMilan Broz /* No seed in LMK version 2 */ 44334745785SMilan Broz if (cc->key_parts == cc->tfms_count) { 44434745785SMilan Broz lmk->seed = NULL; 44534745785SMilan Broz return 0; 44634745785SMilan Broz } 44734745785SMilan Broz 44834745785SMilan Broz lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL); 44934745785SMilan Broz if (!lmk->seed) { 45034745785SMilan Broz crypt_iv_lmk_dtr(cc); 45134745785SMilan Broz ti->error = "Error kmallocing seed storage in LMK"; 45234745785SMilan Broz return -ENOMEM; 45334745785SMilan Broz } 45434745785SMilan Broz 45534745785SMilan Broz return 0; 45634745785SMilan Broz } 45734745785SMilan Broz 45834745785SMilan Broz static int crypt_iv_lmk_init(struct crypt_config *cc) 45934745785SMilan Broz { 46034745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 46134745785SMilan Broz int subkey_size = cc->key_size / cc->key_parts; 46234745785SMilan Broz 46334745785SMilan Broz /* LMK seed is on the position of LMK_KEYS + 1 key */ 46434745785SMilan Broz if (lmk->seed) 46534745785SMilan Broz memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size), 46634745785SMilan Broz crypto_shash_digestsize(lmk->hash_tfm)); 46734745785SMilan Broz 46834745785SMilan Broz return 0; 46934745785SMilan Broz } 47034745785SMilan Broz 47134745785SMilan Broz static int crypt_iv_lmk_wipe(struct crypt_config *cc) 47234745785SMilan Broz { 47334745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 47434745785SMilan Broz 47534745785SMilan Broz if (lmk->seed) 47634745785SMilan Broz memset(lmk->seed, 0, LMK_SEED_SIZE); 47734745785SMilan Broz 47834745785SMilan Broz return 0; 47934745785SMilan Broz } 48034745785SMilan Broz 48134745785SMilan Broz static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv, 48234745785SMilan Broz struct dm_crypt_request *dmreq, 48334745785SMilan Broz u8 *data) 48434745785SMilan Broz { 48534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk; 486b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, lmk->hash_tfm); 48734745785SMilan Broz struct md5_state md5state; 488da31a078SMilan Broz __le32 buf[4]; 48934745785SMilan Broz int i, r; 49034745785SMilan Broz 491b6106265SJan-Simon Möller desc->tfm = lmk->hash_tfm; 49234745785SMilan Broz 493b6106265SJan-Simon Möller r = crypto_shash_init(desc); 49434745785SMilan Broz if (r) 49534745785SMilan Broz return r; 49634745785SMilan Broz 49734745785SMilan Broz if (lmk->seed) { 498b6106265SJan-Simon Möller r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE); 49934745785SMilan Broz if (r) 50034745785SMilan Broz return r; 50134745785SMilan Broz } 50234745785SMilan Broz 50334745785SMilan Broz /* Sector is always 512B, block size 16, add data of blocks 1-31 */ 504b6106265SJan-Simon Möller r = crypto_shash_update(desc, data + 16, 16 * 31); 50534745785SMilan Broz if (r) 50634745785SMilan Broz return r; 50734745785SMilan Broz 50834745785SMilan Broz /* Sector is cropped to 56 bits here */ 50934745785SMilan Broz buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF); 51034745785SMilan Broz buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000); 51134745785SMilan Broz buf[2] = cpu_to_le32(4024); 51234745785SMilan Broz buf[3] = 0; 513b6106265SJan-Simon Möller r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf)); 51434745785SMilan Broz if (r) 51534745785SMilan Broz return r; 51634745785SMilan Broz 51734745785SMilan Broz /* No MD5 padding here */ 518b6106265SJan-Simon Möller r = crypto_shash_export(desc, &md5state); 51934745785SMilan Broz if (r) 52034745785SMilan Broz return r; 52134745785SMilan Broz 52234745785SMilan Broz for (i = 0; i < MD5_HASH_WORDS; i++) 52334745785SMilan Broz __cpu_to_le32s(&md5state.hash[i]); 52434745785SMilan Broz memcpy(iv, &md5state.hash, cc->iv_size); 52534745785SMilan Broz 52634745785SMilan Broz return 0; 52734745785SMilan Broz } 52834745785SMilan Broz 52934745785SMilan Broz static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv, 53034745785SMilan Broz struct dm_crypt_request *dmreq) 53134745785SMilan Broz { 532ef43aa38SMilan Broz struct scatterlist *sg; 53334745785SMilan Broz u8 *src; 53434745785SMilan Broz int r = 0; 53534745785SMilan Broz 53634745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 537ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in); 5380d78954aSHeinz Mauelshagen src = kmap_local_page(sg_page(sg)); 539ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset); 5400d78954aSHeinz Mauelshagen kunmap_local(src); 54134745785SMilan Broz } else 54234745785SMilan Broz memset(iv, 0, cc->iv_size); 54334745785SMilan Broz 54434745785SMilan Broz return r; 54534745785SMilan Broz } 54634745785SMilan Broz 54734745785SMilan Broz static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv, 54834745785SMilan Broz struct dm_crypt_request *dmreq) 54934745785SMilan Broz { 550ef43aa38SMilan Broz struct scatterlist *sg; 55134745785SMilan Broz u8 *dst; 55234745785SMilan Broz int r; 55334745785SMilan Broz 55434745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) 55534745785SMilan Broz return 0; 55634745785SMilan Broz 557ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 5580d78954aSHeinz Mauelshagen dst = kmap_local_page(sg_page(sg)); 559ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset); 56034745785SMilan Broz 56134745785SMilan Broz /* Tweak the first block of plaintext sector */ 56234745785SMilan Broz if (!r) 563ef43aa38SMilan Broz crypto_xor(dst + sg->offset, iv, cc->iv_size); 56434745785SMilan Broz 5650d78954aSHeinz Mauelshagen kunmap_local(dst); 56634745785SMilan Broz return r; 56734745785SMilan Broz } 56834745785SMilan Broz 569ed04d981SMilan Broz static void crypt_iv_tcw_dtr(struct crypt_config *cc) 570ed04d981SMilan Broz { 571ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 572ed04d981SMilan Broz 573453431a5SWaiman Long kfree_sensitive(tcw->iv_seed); 574ed04d981SMilan Broz tcw->iv_seed = NULL; 575453431a5SWaiman Long kfree_sensitive(tcw->whitening); 576ed04d981SMilan Broz tcw->whitening = NULL; 577ed04d981SMilan Broz 578ed04d981SMilan Broz if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm)) 579ed04d981SMilan Broz crypto_free_shash(tcw->crc32_tfm); 580ed04d981SMilan Broz tcw->crc32_tfm = NULL; 581ed04d981SMilan Broz } 582ed04d981SMilan Broz 583ed04d981SMilan Broz static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti, 584ed04d981SMilan Broz const char *opts) 585ed04d981SMilan Broz { 586ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 587ed04d981SMilan Broz 5888f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) { 5898f0009a2SMilan Broz ti->error = "Unsupported sector size for TCW"; 5908f0009a2SMilan Broz return -EINVAL; 5918f0009a2SMilan Broz } 5928f0009a2SMilan Broz 593ed04d981SMilan Broz if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) { 594ed04d981SMilan Broz ti->error = "Wrong key size for TCW"; 595ed04d981SMilan Broz return -EINVAL; 596ed04d981SMilan Broz } 597ed04d981SMilan Broz 598cd746938SMikulas Patocka tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 599cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY); 600ed04d981SMilan Broz if (IS_ERR(tcw->crc32_tfm)) { 601ed04d981SMilan Broz ti->error = "Error initializing CRC32 in TCW"; 602ed04d981SMilan Broz return PTR_ERR(tcw->crc32_tfm); 603ed04d981SMilan Broz } 604ed04d981SMilan Broz 605ed04d981SMilan Broz tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL); 606ed04d981SMilan Broz tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL); 607ed04d981SMilan Broz if (!tcw->iv_seed || !tcw->whitening) { 608ed04d981SMilan Broz crypt_iv_tcw_dtr(cc); 609ed04d981SMilan Broz ti->error = "Error allocating seed storage in TCW"; 610ed04d981SMilan Broz return -ENOMEM; 611ed04d981SMilan Broz } 612ed04d981SMilan Broz 613ed04d981SMilan Broz return 0; 614ed04d981SMilan Broz } 615ed04d981SMilan Broz 616ed04d981SMilan Broz static int crypt_iv_tcw_init(struct crypt_config *cc) 617ed04d981SMilan Broz { 618ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 619ed04d981SMilan Broz int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE; 620ed04d981SMilan Broz 621ed04d981SMilan Broz memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size); 622ed04d981SMilan Broz memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size], 623ed04d981SMilan Broz TCW_WHITENING_SIZE); 624ed04d981SMilan Broz 625ed04d981SMilan Broz return 0; 626ed04d981SMilan Broz } 627ed04d981SMilan Broz 628ed04d981SMilan Broz static int crypt_iv_tcw_wipe(struct crypt_config *cc) 629ed04d981SMilan Broz { 630ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 631ed04d981SMilan Broz 632ed04d981SMilan Broz memset(tcw->iv_seed, 0, cc->iv_size); 633ed04d981SMilan Broz memset(tcw->whitening, 0, TCW_WHITENING_SIZE); 634ed04d981SMilan Broz 635ed04d981SMilan Broz return 0; 636ed04d981SMilan Broz } 637ed04d981SMilan Broz 638ed04d981SMilan Broz static int crypt_iv_tcw_whitening(struct crypt_config *cc, 639ed04d981SMilan Broz struct dm_crypt_request *dmreq, 640ed04d981SMilan Broz u8 *data) 641ed04d981SMilan Broz { 642ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 643350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector); 644ed04d981SMilan Broz u8 buf[TCW_WHITENING_SIZE]; 645b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm); 646ed04d981SMilan Broz int i, r; 647ed04d981SMilan Broz 648ed04d981SMilan Broz /* xor whitening with sector number */ 64945fe93dfSArd Biesheuvel crypto_xor_cpy(buf, tcw->whitening, (u8 *)§or, 8); 65045fe93dfSArd Biesheuvel crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)§or, 8); 651ed04d981SMilan Broz 652ed04d981SMilan Broz /* calculate crc32 for every 32bit part and xor it */ 653b6106265SJan-Simon Möller desc->tfm = tcw->crc32_tfm; 654ed04d981SMilan Broz for (i = 0; i < 4; i++) { 655b6106265SJan-Simon Möller r = crypto_shash_init(desc); 656ed04d981SMilan Broz if (r) 657ed04d981SMilan Broz goto out; 658b6106265SJan-Simon Möller r = crypto_shash_update(desc, &buf[i * 4], 4); 659ed04d981SMilan Broz if (r) 660ed04d981SMilan Broz goto out; 661b6106265SJan-Simon Möller r = crypto_shash_final(desc, &buf[i * 4]); 662ed04d981SMilan Broz if (r) 663ed04d981SMilan Broz goto out; 664ed04d981SMilan Broz } 665ed04d981SMilan Broz crypto_xor(&buf[0], &buf[12], 4); 666ed04d981SMilan Broz crypto_xor(&buf[4], &buf[8], 4); 667ed04d981SMilan Broz 668ed04d981SMilan Broz /* apply whitening (8 bytes) to whole sector */ 669ed04d981SMilan Broz for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++) 670ed04d981SMilan Broz crypto_xor(data + i * 8, buf, 8); 671ed04d981SMilan Broz out: 6721a71d6ffSMilan Broz memzero_explicit(buf, sizeof(buf)); 673ed04d981SMilan Broz return r; 674ed04d981SMilan Broz } 675ed04d981SMilan Broz 676ed04d981SMilan Broz static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv, 677ed04d981SMilan Broz struct dm_crypt_request *dmreq) 678ed04d981SMilan Broz { 679ef43aa38SMilan Broz struct scatterlist *sg; 680ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw; 681350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector); 682ed04d981SMilan Broz u8 *src; 683ed04d981SMilan Broz int r = 0; 684ed04d981SMilan Broz 685ed04d981SMilan Broz /* Remove whitening from ciphertext */ 686ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) { 687ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in); 6880d78954aSHeinz Mauelshagen src = kmap_local_page(sg_page(sg)); 689ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset); 6900d78954aSHeinz Mauelshagen kunmap_local(src); 691ed04d981SMilan Broz } 692ed04d981SMilan Broz 693ed04d981SMilan Broz /* Calculate IV */ 69445fe93dfSArd Biesheuvel crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)§or, 8); 695ed04d981SMilan Broz if (cc->iv_size > 8) 69645fe93dfSArd Biesheuvel crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)§or, 69745fe93dfSArd Biesheuvel cc->iv_size - 8); 698ed04d981SMilan Broz 699ed04d981SMilan Broz return r; 700ed04d981SMilan Broz } 701ed04d981SMilan Broz 702ed04d981SMilan Broz static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv, 703ed04d981SMilan Broz struct dm_crypt_request *dmreq) 704ed04d981SMilan Broz { 705ef43aa38SMilan Broz struct scatterlist *sg; 706ed04d981SMilan Broz u8 *dst; 707ed04d981SMilan Broz int r; 708ed04d981SMilan Broz 709ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) 710ed04d981SMilan Broz return 0; 711ed04d981SMilan Broz 712ed04d981SMilan Broz /* Apply whitening on ciphertext */ 713ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 7140d78954aSHeinz Mauelshagen dst = kmap_local_page(sg_page(sg)); 715ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset); 7160d78954aSHeinz Mauelshagen kunmap_local(dst); 717ed04d981SMilan Broz 718ed04d981SMilan Broz return r; 719ed04d981SMilan Broz } 720ed04d981SMilan Broz 721ef43aa38SMilan Broz static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv, 722ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 723ef43aa38SMilan Broz { 724ef43aa38SMilan Broz /* Used only for writes, there must be an additional space to store IV */ 725ef43aa38SMilan Broz get_random_bytes(iv, cc->iv_size); 726ef43aa38SMilan Broz return 0; 727ef43aa38SMilan Broz } 728ef43aa38SMilan Broz 729b9411d73SMilan Broz static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti, 730b9411d73SMilan Broz const char *opts) 731b9411d73SMilan Broz { 7323fd53533SYang Yingliang if (crypt_integrity_aead(cc)) { 73339d13a1aSArd Biesheuvel ti->error = "AEAD transforms not supported for EBOIV"; 734b9411d73SMilan Broz return -EINVAL; 735b9411d73SMilan Broz } 736b9411d73SMilan Broz 73739d13a1aSArd Biesheuvel if (crypto_skcipher_blocksize(any_tfm(cc)) != cc->iv_size) { 7382e84fecfSHeinz Mauelshagen ti->error = "Block size of EBOIV cipher does not match IV size of block cipher"; 73939d13a1aSArd Biesheuvel return -EINVAL; 740b9411d73SMilan Broz } 741b9411d73SMilan Broz 742b9411d73SMilan Broz return 0; 743b9411d73SMilan Broz } 744b9411d73SMilan Broz 745b9411d73SMilan Broz static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv, 746b9411d73SMilan Broz struct dm_crypt_request *dmreq) 747b9411d73SMilan Broz { 74839d13a1aSArd Biesheuvel u8 buf[MAX_CIPHER_BLOCKSIZE] __aligned(__alignof__(__le64)); 74939d13a1aSArd Biesheuvel struct skcipher_request *req; 75039d13a1aSArd Biesheuvel struct scatterlist src, dst; 7517785a9e4SDamien Le Moal DECLARE_CRYPTO_WAIT(wait); 75239d13a1aSArd Biesheuvel int err; 753b9411d73SMilan Broz 7549402e959SMikulas Patocka req = skcipher_request_alloc(any_tfm(cc), GFP_NOIO); 75539d13a1aSArd Biesheuvel if (!req) 75639d13a1aSArd Biesheuvel return -ENOMEM; 757b9411d73SMilan Broz 75839d13a1aSArd Biesheuvel memset(buf, 0, cc->iv_size); 75939d13a1aSArd Biesheuvel *(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size); 76039d13a1aSArd Biesheuvel 76139d13a1aSArd Biesheuvel sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size); 76239d13a1aSArd Biesheuvel sg_init_one(&dst, iv, cc->iv_size); 76339d13a1aSArd Biesheuvel skcipher_request_set_crypt(req, &src, &dst, cc->iv_size, buf); 76439d13a1aSArd Biesheuvel skcipher_request_set_callback(req, 0, crypto_req_done, &wait); 76539d13a1aSArd Biesheuvel err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); 76639d13a1aSArd Biesheuvel skcipher_request_free(req); 76739d13a1aSArd Biesheuvel 76839d13a1aSArd Biesheuvel return err; 769b9411d73SMilan Broz } 770b9411d73SMilan Broz 771bbb16584SMilan Broz static void crypt_iv_elephant_dtr(struct crypt_config *cc) 772bbb16584SMilan Broz { 773bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 774bbb16584SMilan Broz 775bbb16584SMilan Broz crypto_free_skcipher(elephant->tfm); 776bbb16584SMilan Broz elephant->tfm = NULL; 777bbb16584SMilan Broz } 778bbb16584SMilan Broz 779bbb16584SMilan Broz static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti, 780bbb16584SMilan Broz const char *opts) 781bbb16584SMilan Broz { 782bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 783bbb16584SMilan Broz int r; 784bbb16584SMilan Broz 785cd746938SMikulas Patocka elephant->tfm = crypto_alloc_skcipher("ecb(aes)", 0, 786cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY); 787bbb16584SMilan Broz if (IS_ERR(elephant->tfm)) { 788bbb16584SMilan Broz r = PTR_ERR(elephant->tfm); 789bbb16584SMilan Broz elephant->tfm = NULL; 790bbb16584SMilan Broz return r; 791bbb16584SMilan Broz } 792bbb16584SMilan Broz 793bbb16584SMilan Broz r = crypt_iv_eboiv_ctr(cc, ti, NULL); 794bbb16584SMilan Broz if (r) 795bbb16584SMilan Broz crypt_iv_elephant_dtr(cc); 796bbb16584SMilan Broz return r; 797bbb16584SMilan Broz } 798bbb16584SMilan Broz 799bbb16584SMilan Broz static void diffuser_disk_to_cpu(u32 *d, size_t n) 800bbb16584SMilan Broz { 801bbb16584SMilan Broz #ifndef __LITTLE_ENDIAN 802bbb16584SMilan Broz int i; 803bbb16584SMilan Broz 804bbb16584SMilan Broz for (i = 0; i < n; i++) 805bbb16584SMilan Broz d[i] = le32_to_cpu((__le32)d[i]); 806bbb16584SMilan Broz #endif 807bbb16584SMilan Broz } 808bbb16584SMilan Broz 809bbb16584SMilan Broz static void diffuser_cpu_to_disk(__le32 *d, size_t n) 810bbb16584SMilan Broz { 811bbb16584SMilan Broz #ifndef __LITTLE_ENDIAN 812bbb16584SMilan Broz int i; 813bbb16584SMilan Broz 814bbb16584SMilan Broz for (i = 0; i < n; i++) 815bbb16584SMilan Broz d[i] = cpu_to_le32((u32)d[i]); 816bbb16584SMilan Broz #endif 817bbb16584SMilan Broz } 818bbb16584SMilan Broz 819bbb16584SMilan Broz static void diffuser_a_decrypt(u32 *d, size_t n) 820bbb16584SMilan Broz { 821bbb16584SMilan Broz int i, i1, i2, i3; 822bbb16584SMilan Broz 823bbb16584SMilan Broz for (i = 0; i < 5; i++) { 824bbb16584SMilan Broz i1 = 0; 825bbb16584SMilan Broz i2 = n - 2; 826bbb16584SMilan Broz i3 = n - 5; 827bbb16584SMilan Broz 828bbb16584SMilan Broz while (i1 < (n - 1)) { 829bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23); 830bbb16584SMilan Broz i1++; i2++; i3++; 831bbb16584SMilan Broz 832bbb16584SMilan Broz if (i3 >= n) 833bbb16584SMilan Broz i3 -= n; 834bbb16584SMilan Broz 835bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 836bbb16584SMilan Broz i1++; i2++; i3++; 837bbb16584SMilan Broz 838bbb16584SMilan Broz if (i2 >= n) 839bbb16584SMilan Broz i2 -= n; 840bbb16584SMilan Broz 841bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19); 842bbb16584SMilan Broz i1++; i2++; i3++; 843bbb16584SMilan Broz 844bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 845bbb16584SMilan Broz i1++; i2++; i3++; 846bbb16584SMilan Broz } 847bbb16584SMilan Broz } 848bbb16584SMilan Broz } 849bbb16584SMilan Broz 850bbb16584SMilan Broz static void diffuser_a_encrypt(u32 *d, size_t n) 851bbb16584SMilan Broz { 852bbb16584SMilan Broz int i, i1, i2, i3; 853bbb16584SMilan Broz 854bbb16584SMilan Broz for (i = 0; i < 5; i++) { 855bbb16584SMilan Broz i1 = n - 1; 856bbb16584SMilan Broz i2 = n - 2 - 1; 857bbb16584SMilan Broz i3 = n - 5 - 1; 858bbb16584SMilan Broz 859bbb16584SMilan Broz while (i1 > 0) { 860bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 861bbb16584SMilan Broz i1--; i2--; i3--; 862bbb16584SMilan Broz 863bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19); 864bbb16584SMilan Broz i1--; i2--; i3--; 865bbb16584SMilan Broz 866bbb16584SMilan Broz if (i2 < 0) 867bbb16584SMilan Broz i2 += n; 868bbb16584SMilan Broz 869bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 870bbb16584SMilan Broz i1--; i2--; i3--; 871bbb16584SMilan Broz 872bbb16584SMilan Broz if (i3 < 0) 873bbb16584SMilan Broz i3 += n; 874bbb16584SMilan Broz 875bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23); 876bbb16584SMilan Broz i1--; i2--; i3--; 877bbb16584SMilan Broz } 878bbb16584SMilan Broz } 879bbb16584SMilan Broz } 880bbb16584SMilan Broz 881bbb16584SMilan Broz static void diffuser_b_decrypt(u32 *d, size_t n) 882bbb16584SMilan Broz { 883bbb16584SMilan Broz int i, i1, i2, i3; 884bbb16584SMilan Broz 885bbb16584SMilan Broz for (i = 0; i < 3; i++) { 886bbb16584SMilan Broz i1 = 0; 887bbb16584SMilan Broz i2 = 2; 888bbb16584SMilan Broz i3 = 5; 889bbb16584SMilan Broz 890bbb16584SMilan Broz while (i1 < (n - 1)) { 891bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 892bbb16584SMilan Broz i1++; i2++; i3++; 893bbb16584SMilan Broz 894bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22); 895bbb16584SMilan Broz i1++; i2++; i3++; 896bbb16584SMilan Broz 897bbb16584SMilan Broz if (i2 >= n) 898bbb16584SMilan Broz i2 -= n; 899bbb16584SMilan Broz 900bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3]; 901bbb16584SMilan Broz i1++; i2++; i3++; 902bbb16584SMilan Broz 903bbb16584SMilan Broz if (i3 >= n) 904bbb16584SMilan Broz i3 -= n; 905bbb16584SMilan Broz 906bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7); 907bbb16584SMilan Broz i1++; i2++; i3++; 908bbb16584SMilan Broz } 909bbb16584SMilan Broz } 910bbb16584SMilan Broz } 911bbb16584SMilan Broz 912bbb16584SMilan Broz static void diffuser_b_encrypt(u32 *d, size_t n) 913bbb16584SMilan Broz { 914bbb16584SMilan Broz int i, i1, i2, i3; 915bbb16584SMilan Broz 916bbb16584SMilan Broz for (i = 0; i < 3; i++) { 917bbb16584SMilan Broz i1 = n - 1; 918bbb16584SMilan Broz i2 = 2 - 1; 919bbb16584SMilan Broz i3 = 5 - 1; 920bbb16584SMilan Broz 921bbb16584SMilan Broz while (i1 > 0) { 922bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7); 923bbb16584SMilan Broz i1--; i2--; i3--; 924bbb16584SMilan Broz 925bbb16584SMilan Broz if (i3 < 0) 926bbb16584SMilan Broz i3 += n; 927bbb16584SMilan Broz 928bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 929bbb16584SMilan Broz i1--; i2--; i3--; 930bbb16584SMilan Broz 931bbb16584SMilan Broz if (i2 < 0) 932bbb16584SMilan Broz i2 += n; 933bbb16584SMilan Broz 934bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22); 935bbb16584SMilan Broz i1--; i2--; i3--; 936bbb16584SMilan Broz 937bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3]; 938bbb16584SMilan Broz i1--; i2--; i3--; 939bbb16584SMilan Broz } 940bbb16584SMilan Broz } 941bbb16584SMilan Broz } 942bbb16584SMilan Broz 943bbb16584SMilan Broz static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq) 944bbb16584SMilan Broz { 945bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 946bbb16584SMilan Broz u8 *es, *ks, *data, *data2, *data_offset; 947bbb16584SMilan Broz struct skcipher_request *req; 948bbb16584SMilan Broz struct scatterlist *sg, *sg2, src, dst; 9497785a9e4SDamien Le Moal DECLARE_CRYPTO_WAIT(wait); 950bbb16584SMilan Broz int i, r; 951bbb16584SMilan Broz 952bbb16584SMilan Broz req = skcipher_request_alloc(elephant->tfm, GFP_NOIO); 953bbb16584SMilan Broz es = kzalloc(16, GFP_NOIO); /* Key for AES */ 954bbb16584SMilan Broz ks = kzalloc(32, GFP_NOIO); /* Elephant sector key */ 955bbb16584SMilan Broz 956bbb16584SMilan Broz if (!req || !es || !ks) { 957bbb16584SMilan Broz r = -ENOMEM; 958bbb16584SMilan Broz goto out; 959bbb16584SMilan Broz } 960bbb16584SMilan Broz 961bbb16584SMilan Broz *(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size); 962bbb16584SMilan Broz 963bbb16584SMilan Broz /* E(Ks, e(s)) */ 964bbb16584SMilan Broz sg_init_one(&src, es, 16); 965bbb16584SMilan Broz sg_init_one(&dst, ks, 16); 966bbb16584SMilan Broz skcipher_request_set_crypt(req, &src, &dst, 16, NULL); 967bbb16584SMilan Broz skcipher_request_set_callback(req, 0, crypto_req_done, &wait); 968bbb16584SMilan Broz r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); 969bbb16584SMilan Broz if (r) 970bbb16584SMilan Broz goto out; 971bbb16584SMilan Broz 972bbb16584SMilan Broz /* E(Ks, e'(s)) */ 973bbb16584SMilan Broz es[15] = 0x80; 974bbb16584SMilan Broz sg_init_one(&dst, &ks[16], 16); 975bbb16584SMilan Broz r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); 976bbb16584SMilan Broz if (r) 977bbb16584SMilan Broz goto out; 978bbb16584SMilan Broz 979bbb16584SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out); 9800d78954aSHeinz Mauelshagen data = kmap_local_page(sg_page(sg)); 981bbb16584SMilan Broz data_offset = data + sg->offset; 982bbb16584SMilan Broz 983bbb16584SMilan Broz /* Cannot modify original bio, copy to sg_out and apply Elephant to it */ 984bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 985bbb16584SMilan Broz sg2 = crypt_get_sg_data(cc, dmreq->sg_in); 9860d78954aSHeinz Mauelshagen data2 = kmap_local_page(sg_page(sg2)); 987bbb16584SMilan Broz memcpy(data_offset, data2 + sg2->offset, cc->sector_size); 9880d78954aSHeinz Mauelshagen kunmap_local(data2); 989bbb16584SMilan Broz } 990bbb16584SMilan Broz 991bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) { 992bbb16584SMilan Broz diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32)); 993bbb16584SMilan Broz diffuser_b_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32)); 994bbb16584SMilan Broz diffuser_a_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32)); 995bbb16584SMilan Broz diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32)); 996bbb16584SMilan Broz } 997bbb16584SMilan Broz 998bbb16584SMilan Broz for (i = 0; i < (cc->sector_size / 32); i++) 999bbb16584SMilan Broz crypto_xor(data_offset + i * 32, ks, 32); 1000bbb16584SMilan Broz 1001bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 1002bbb16584SMilan Broz diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32)); 1003bbb16584SMilan Broz diffuser_a_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32)); 1004bbb16584SMilan Broz diffuser_b_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32)); 1005bbb16584SMilan Broz diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32)); 1006bbb16584SMilan Broz } 1007bbb16584SMilan Broz 10080d78954aSHeinz Mauelshagen kunmap_local(data); 1009bbb16584SMilan Broz out: 1010453431a5SWaiman Long kfree_sensitive(ks); 1011453431a5SWaiman Long kfree_sensitive(es); 1012bbb16584SMilan Broz skcipher_request_free(req); 1013bbb16584SMilan Broz return r; 1014bbb16584SMilan Broz } 1015bbb16584SMilan Broz 1016bbb16584SMilan Broz static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv, 1017bbb16584SMilan Broz struct dm_crypt_request *dmreq) 1018bbb16584SMilan Broz { 1019bbb16584SMilan Broz int r; 1020bbb16584SMilan Broz 1021bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) { 1022bbb16584SMilan Broz r = crypt_iv_elephant(cc, dmreq); 1023bbb16584SMilan Broz if (r) 1024bbb16584SMilan Broz return r; 1025bbb16584SMilan Broz } 1026bbb16584SMilan Broz 1027bbb16584SMilan Broz return crypt_iv_eboiv_gen(cc, iv, dmreq); 1028bbb16584SMilan Broz } 1029bbb16584SMilan Broz 1030bbb16584SMilan Broz static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv, 1031bbb16584SMilan Broz struct dm_crypt_request *dmreq) 1032bbb16584SMilan Broz { 1033bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) 1034bbb16584SMilan Broz return crypt_iv_elephant(cc, dmreq); 1035bbb16584SMilan Broz 1036bbb16584SMilan Broz return 0; 1037bbb16584SMilan Broz } 1038bbb16584SMilan Broz 1039bbb16584SMilan Broz static int crypt_iv_elephant_init(struct crypt_config *cc) 1040bbb16584SMilan Broz { 1041bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 1042bbb16584SMilan Broz int key_offset = cc->key_size - cc->key_extra_size; 1043bbb16584SMilan Broz 1044bbb16584SMilan Broz return crypto_skcipher_setkey(elephant->tfm, &cc->key[key_offset], cc->key_extra_size); 1045bbb16584SMilan Broz } 1046bbb16584SMilan Broz 1047bbb16584SMilan Broz static int crypt_iv_elephant_wipe(struct crypt_config *cc) 1048bbb16584SMilan Broz { 1049bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant; 1050bbb16584SMilan Broz u8 key[ELEPHANT_MAX_KEY_SIZE]; 1051bbb16584SMilan Broz 1052bbb16584SMilan Broz memset(key, 0, cc->key_extra_size); 1053bbb16584SMilan Broz return crypto_skcipher_setkey(elephant->tfm, key, cc->key_extra_size); 1054bbb16584SMilan Broz } 1055bbb16584SMilan Broz 10561b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain_ops = { 10571da177e4SLinus Torvalds .generator = crypt_iv_plain_gen 10581da177e4SLinus Torvalds }; 10591da177e4SLinus Torvalds 10601b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain64_ops = { 106161afef61SMilan Broz .generator = crypt_iv_plain64_gen 106261afef61SMilan Broz }; 106361afef61SMilan Broz 10647e3fd855SMilan Broz static const struct crypt_iv_operations crypt_iv_plain64be_ops = { 10657e3fd855SMilan Broz .generator = crypt_iv_plain64be_gen 10667e3fd855SMilan Broz }; 10677e3fd855SMilan Broz 10681b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_essiv_ops = { 10691da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen 10701da177e4SLinus Torvalds }; 10711da177e4SLinus Torvalds 10721b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_benbi_ops = { 107348527fa7SRik Snel .ctr = crypt_iv_benbi_ctr, 107448527fa7SRik Snel .dtr = crypt_iv_benbi_dtr, 107548527fa7SRik Snel .generator = crypt_iv_benbi_gen 107648527fa7SRik Snel }; 10771da177e4SLinus Torvalds 10781b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_null_ops = { 107946b47730SLudwig Nussel .generator = crypt_iv_null_gen 108046b47730SLudwig Nussel }; 108146b47730SLudwig Nussel 10821b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_lmk_ops = { 108334745785SMilan Broz .ctr = crypt_iv_lmk_ctr, 108434745785SMilan Broz .dtr = crypt_iv_lmk_dtr, 108534745785SMilan Broz .init = crypt_iv_lmk_init, 108634745785SMilan Broz .wipe = crypt_iv_lmk_wipe, 108734745785SMilan Broz .generator = crypt_iv_lmk_gen, 108834745785SMilan Broz .post = crypt_iv_lmk_post 108934745785SMilan Broz }; 109034745785SMilan Broz 10911b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_tcw_ops = { 1092ed04d981SMilan Broz .ctr = crypt_iv_tcw_ctr, 1093ed04d981SMilan Broz .dtr = crypt_iv_tcw_dtr, 1094ed04d981SMilan Broz .init = crypt_iv_tcw_init, 1095ed04d981SMilan Broz .wipe = crypt_iv_tcw_wipe, 1096ed04d981SMilan Broz .generator = crypt_iv_tcw_gen, 1097ed04d981SMilan Broz .post = crypt_iv_tcw_post 1098ed04d981SMilan Broz }; 1099ed04d981SMilan Broz 1100e8dc79d1SRikard Falkeborn static const struct crypt_iv_operations crypt_iv_random_ops = { 1101ef43aa38SMilan Broz .generator = crypt_iv_random_gen 1102ef43aa38SMilan Broz }; 1103ef43aa38SMilan Broz 1104e8dc79d1SRikard Falkeborn static const struct crypt_iv_operations crypt_iv_eboiv_ops = { 1105b9411d73SMilan Broz .ctr = crypt_iv_eboiv_ctr, 1106b9411d73SMilan Broz .generator = crypt_iv_eboiv_gen 1107b9411d73SMilan Broz }; 1108b9411d73SMilan Broz 1109e8dc79d1SRikard Falkeborn static const struct crypt_iv_operations crypt_iv_elephant_ops = { 1110bbb16584SMilan Broz .ctr = crypt_iv_elephant_ctr, 1111bbb16584SMilan Broz .dtr = crypt_iv_elephant_dtr, 1112bbb16584SMilan Broz .init = crypt_iv_elephant_init, 1113bbb16584SMilan Broz .wipe = crypt_iv_elephant_wipe, 1114bbb16584SMilan Broz .generator = crypt_iv_elephant_gen, 1115bbb16584SMilan Broz .post = crypt_iv_elephant_post 1116bbb16584SMilan Broz }; 1117bbb16584SMilan Broz 1118ef43aa38SMilan Broz /* 1119ef43aa38SMilan Broz * Integrity extensions 1120ef43aa38SMilan Broz */ 1121ef43aa38SMilan Broz static bool crypt_integrity_aead(struct crypt_config *cc) 1122ef43aa38SMilan Broz { 1123ef43aa38SMilan Broz return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags); 1124ef43aa38SMilan Broz } 1125ef43aa38SMilan Broz 1126ef43aa38SMilan Broz static bool crypt_integrity_hmac(struct crypt_config *cc) 1127ef43aa38SMilan Broz { 112833d2f09fSMilan Broz return crypt_integrity_aead(cc) && cc->key_mac_size; 1129ef43aa38SMilan Broz } 1130ef43aa38SMilan Broz 1131ef43aa38SMilan Broz /* Get sg containing data */ 1132ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc, 1133ef43aa38SMilan Broz struct scatterlist *sg) 1134ef43aa38SMilan Broz { 113533d2f09fSMilan Broz if (unlikely(crypt_integrity_aead(cc))) 1136ef43aa38SMilan Broz return &sg[2]; 1137ef43aa38SMilan Broz 1138ef43aa38SMilan Broz return sg; 1139ef43aa38SMilan Broz } 1140ef43aa38SMilan Broz 1141ef43aa38SMilan Broz static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio) 1142ef43aa38SMilan Broz { 1143ef43aa38SMilan Broz struct bio_integrity_payload *bip; 1144ef43aa38SMilan Broz unsigned int tag_len; 1145ef43aa38SMilan Broz int ret; 1146ef43aa38SMilan Broz 1147ef43aa38SMilan Broz if (!bio_sectors(bio) || !io->cc->on_disk_tag_size) 1148ef43aa38SMilan Broz return 0; 1149ef43aa38SMilan Broz 1150ef43aa38SMilan Broz bip = bio_integrity_alloc(bio, GFP_NOIO, 1); 1151ef43aa38SMilan Broz if (IS_ERR(bip)) 1152ef43aa38SMilan Broz return PTR_ERR(bip); 1153ef43aa38SMilan Broz 1154ff0c129dSMikulas Patocka tag_len = io->cc->on_disk_tag_size * (bio_sectors(bio) >> io->cc->sector_shift); 1155ef43aa38SMilan Broz 1156ef43aa38SMilan Broz bip->bip_iter.bi_size = tag_len; 1157ef43aa38SMilan Broz bip->bip_iter.bi_sector = io->cc->start + io->sector; 1158ef43aa38SMilan Broz 1159ef43aa38SMilan Broz ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata), 1160ef43aa38SMilan Broz tag_len, offset_in_page(io->integrity_metadata)); 1161ef43aa38SMilan Broz if (unlikely(ret != tag_len)) 1162ef43aa38SMilan Broz return -ENOMEM; 1163ef43aa38SMilan Broz 1164ef43aa38SMilan Broz return 0; 1165ef43aa38SMilan Broz } 1166ef43aa38SMilan Broz 1167ef43aa38SMilan Broz static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti) 1168ef43aa38SMilan Broz { 1169ef43aa38SMilan Broz #ifdef CONFIG_BLK_DEV_INTEGRITY 1170ef43aa38SMilan Broz struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk); 11717a1cd723SMilan Broz struct mapped_device *md = dm_table_get_md(ti->table); 1172ef43aa38SMilan Broz 1173ef43aa38SMilan Broz /* From now we require underlying device with our integrity profile */ 1174ef43aa38SMilan Broz if (!bi || strcasecmp(bi->profile->name, "DM-DIF-EXT-TAG")) { 1175ef43aa38SMilan Broz ti->error = "Integrity profile not supported."; 1176ef43aa38SMilan Broz return -EINVAL; 1177ef43aa38SMilan Broz } 1178ef43aa38SMilan Broz 1179583fe747SMikulas Patocka if (bi->tag_size != cc->on_disk_tag_size || 1180583fe747SMikulas Patocka bi->tuple_size != cc->on_disk_tag_size) { 1181ef43aa38SMilan Broz ti->error = "Integrity profile tag size mismatch."; 1182ef43aa38SMilan Broz return -EINVAL; 1183ef43aa38SMilan Broz } 1184583fe747SMikulas Patocka if (1 << bi->interval_exp != cc->sector_size) { 1185583fe747SMikulas Patocka ti->error = "Integrity profile sector size mismatch."; 1186583fe747SMikulas Patocka return -EINVAL; 1187583fe747SMikulas Patocka } 1188ef43aa38SMilan Broz 118933d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 1190ef43aa38SMilan Broz cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size; 11917a1cd723SMilan Broz DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md), 1192ef43aa38SMilan Broz cc->integrity_tag_size, cc->integrity_iv_size); 1193ef43aa38SMilan Broz 1194ef43aa38SMilan Broz if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) { 1195ef43aa38SMilan Broz ti->error = "Integrity AEAD auth tag size is not supported."; 1196ef43aa38SMilan Broz return -EINVAL; 1197ef43aa38SMilan Broz } 1198ef43aa38SMilan Broz } else if (cc->integrity_iv_size) 11997a1cd723SMilan Broz DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md), 1200ef43aa38SMilan Broz cc->integrity_iv_size); 1201ef43aa38SMilan Broz 1202ef43aa38SMilan Broz if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) { 1203ef43aa38SMilan Broz ti->error = "Not enough space for integrity tag in the profile."; 1204ef43aa38SMilan Broz return -EINVAL; 1205ef43aa38SMilan Broz } 1206ef43aa38SMilan Broz 1207ef43aa38SMilan Broz return 0; 1208ef43aa38SMilan Broz #else 1209ef43aa38SMilan Broz ti->error = "Integrity profile not supported."; 1210ef43aa38SMilan Broz return -EINVAL; 1211ef43aa38SMilan Broz #endif 1212ef43aa38SMilan Broz } 1213ef43aa38SMilan Broz 1214d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc, 1215d469f841SMilan Broz struct convert_context *ctx, 12161da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in, 1217fcd369daSMilan Broz sector_t sector) 12181da177e4SLinus Torvalds { 12191da177e4SLinus Torvalds ctx->bio_in = bio_in; 12201da177e4SLinus Torvalds ctx->bio_out = bio_out; 1221003b5c57SKent Overstreet if (bio_in) 1222003b5c57SKent Overstreet ctx->iter_in = bio_in->bi_iter; 1223003b5c57SKent Overstreet if (bio_out) 1224003b5c57SKent Overstreet ctx->iter_out = bio_out->bi_iter; 1225c66029f4SMikulas Patocka ctx->cc_sector = sector + cc->iv_offset; 122643d69034SMilan Broz init_completion(&ctx->restart); 12271da177e4SLinus Torvalds } 12281da177e4SLinus Torvalds 1229b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc, 1230ef43aa38SMilan Broz void *req) 1231b2174eebSHuang Ying { 1232b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start); 1233b2174eebSHuang Ying } 1234b2174eebSHuang Ying 1235ef43aa38SMilan Broz static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq) 1236b2174eebSHuang Ying { 1237ef43aa38SMilan Broz return (void *)((char *)dmreq - cc->dmreq_start); 1238b2174eebSHuang Ying } 1239b2174eebSHuang Ying 12402dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc, 12412dc5327dSMilan Broz struct dm_crypt_request *dmreq) 12422dc5327dSMilan Broz { 124333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1244ef43aa38SMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 1245ef43aa38SMilan Broz crypto_aead_alignmask(any_tfm_aead(cc)) + 1); 1246ef43aa38SMilan Broz else 12472dc5327dSMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1), 1248bbdb23b5SHerbert Xu crypto_skcipher_alignmask(any_tfm(cc)) + 1); 12492dc5327dSMilan Broz } 12502dc5327dSMilan Broz 1251ef43aa38SMilan Broz static u8 *org_iv_of_dmreq(struct crypt_config *cc, 1252ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1253ef43aa38SMilan Broz { 1254ef43aa38SMilan Broz return iv_of_dmreq(cc, dmreq) + cc->iv_size; 1255ef43aa38SMilan Broz } 1256ef43aa38SMilan Broz 1257c13b5487SChristoph Hellwig static __le64 *org_sector_of_dmreq(struct crypt_config *cc, 1258ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1259ef43aa38SMilan Broz { 1260ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size; 12610ef0b471SHeinz Mauelshagen 1262c13b5487SChristoph Hellwig return (__le64 *) ptr; 1263ef43aa38SMilan Broz } 1264ef43aa38SMilan Broz 1265ef43aa38SMilan Broz static unsigned int *org_tag_of_dmreq(struct crypt_config *cc, 1266ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1267ef43aa38SMilan Broz { 1268ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + 1269ef43aa38SMilan Broz cc->iv_size + sizeof(uint64_t); 12700ef0b471SHeinz Mauelshagen 1271ef43aa38SMilan Broz return (unsigned int *)ptr; 1272ef43aa38SMilan Broz } 1273ef43aa38SMilan Broz 1274ef43aa38SMilan Broz static void *tag_from_dmreq(struct crypt_config *cc, 1275ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1276ef43aa38SMilan Broz { 1277ef43aa38SMilan Broz struct convert_context *ctx = dmreq->ctx; 1278ef43aa38SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 1279ef43aa38SMilan Broz 1280ef43aa38SMilan Broz return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) * 1281ef43aa38SMilan Broz cc->on_disk_tag_size]; 1282ef43aa38SMilan Broz } 1283ef43aa38SMilan Broz 1284ef43aa38SMilan Broz static void *iv_tag_from_dmreq(struct crypt_config *cc, 1285ef43aa38SMilan Broz struct dm_crypt_request *dmreq) 1286ef43aa38SMilan Broz { 1287ef43aa38SMilan Broz return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size; 1288ef43aa38SMilan Broz } 1289ef43aa38SMilan Broz 1290ef43aa38SMilan Broz static int crypt_convert_block_aead(struct crypt_config *cc, 12913a7f6c99SMilan Broz struct convert_context *ctx, 1292ef43aa38SMilan Broz struct aead_request *req, 1293ef43aa38SMilan Broz unsigned int tag_offset) 129401482b76SMilan Broz { 1295003b5c57SKent Overstreet struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in); 1296003b5c57SKent Overstreet struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out); 12973a7f6c99SMilan Broz struct dm_crypt_request *dmreq; 1298ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv, *tag; 1299c13b5487SChristoph Hellwig __le64 *sector; 1300ef43aa38SMilan Broz int r = 0; 1301ef43aa38SMilan Broz 1302ef43aa38SMilan Broz BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size); 130301482b76SMilan Broz 13048f0009a2SMilan Broz /* Reject unexpected unaligned bio. */ 13050440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1))) 13068f0009a2SMilan Broz return -EIO; 130701482b76SMilan Broz 1308b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req); 1309c66029f4SMikulas Patocka dmreq->iv_sector = ctx->cc_sector; 13108f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 1311ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift; 1312b2174eebSHuang Ying dmreq->ctx = ctx; 131301482b76SMilan Broz 1314ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset; 131501482b76SMilan Broz 1316ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq); 1317ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset); 1318ef43aa38SMilan Broz 1319ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq); 1320ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq); 1321ef43aa38SMilan Broz tag = tag_from_dmreq(cc, dmreq); 1322ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq); 1323ef43aa38SMilan Broz 1324ef43aa38SMilan Broz /* AEAD request: 1325ef43aa38SMilan Broz * |----- AAD -------|------ DATA -------|-- AUTH TAG --| 1326ef43aa38SMilan Broz * | (authenticated) | (auth+encryption) | | 1327ef43aa38SMilan Broz * | sector_LE | IV | sector in/out | tag in/out | 1328ef43aa38SMilan Broz */ 1329ef43aa38SMilan Broz sg_init_table(dmreq->sg_in, 4); 1330ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t)); 1331ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size); 13328f0009a2SMilan Broz sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset); 1333ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size); 1334ef43aa38SMilan Broz 1335ef43aa38SMilan Broz sg_init_table(dmreq->sg_out, 4); 1336ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t)); 1337ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size); 13388f0009a2SMilan Broz sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset); 1339ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size); 134001482b76SMilan Broz 13413a7f6c99SMilan Broz if (cc->iv_gen_ops) { 1342ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */ 1343ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) { 1344ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->iv_size); 1345ef43aa38SMilan Broz } else { 1346ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq); 13473a7f6c99SMilan Broz if (r < 0) 13483a7f6c99SMilan Broz return r; 1349ef43aa38SMilan Broz /* Store generated IV in integrity metadata */ 1350ef43aa38SMilan Broz if (cc->integrity_iv_size) 1351ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->iv_size); 1352ef43aa38SMilan Broz } 1353ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */ 1354ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size); 1355ef43aa38SMilan Broz } 1356ef43aa38SMilan Broz 1357ef43aa38SMilan Broz aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size); 1358ef43aa38SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) { 1359ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out, 13608f0009a2SMilan Broz cc->sector_size, iv); 1361ef43aa38SMilan Broz r = crypto_aead_encrypt(req); 1362ef43aa38SMilan Broz if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size) 1363ef43aa38SMilan Broz memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0, 1364ef43aa38SMilan Broz cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size)); 1365ef43aa38SMilan Broz } else { 1366ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out, 13678f0009a2SMilan Broz cc->sector_size + cc->integrity_tag_size, iv); 1368ef43aa38SMilan Broz r = crypto_aead_decrypt(req); 1369ef43aa38SMilan Broz } 1370ef43aa38SMilan Broz 1371f710126cSMilan Broz if (r == -EBADMSG) { 137258d0f180SMichael Weiß sector_t s = le64_to_cpu(*sector); 137358d0f180SMichael Weiß 137466671719SChristoph Hellwig DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu", 137566671719SChristoph Hellwig ctx->bio_in->bi_bdev, s); 137658d0f180SMichael Weiß dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead", 137758d0f180SMichael Weiß ctx->bio_in, s, 0); 1378f710126cSMilan Broz } 1379ef43aa38SMilan Broz 1380ef43aa38SMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 1381ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq); 1382ef43aa38SMilan Broz 13838f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size); 13848f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size); 1385ef43aa38SMilan Broz 1386ef43aa38SMilan Broz return r; 13873a7f6c99SMilan Broz } 13883a7f6c99SMilan Broz 1389ef43aa38SMilan Broz static int crypt_convert_block_skcipher(struct crypt_config *cc, 1390ef43aa38SMilan Broz struct convert_context *ctx, 1391ef43aa38SMilan Broz struct skcipher_request *req, 1392ef43aa38SMilan Broz unsigned int tag_offset) 1393ef43aa38SMilan Broz { 1394ef43aa38SMilan Broz struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in); 1395ef43aa38SMilan Broz struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out); 1396ef43aa38SMilan Broz struct scatterlist *sg_in, *sg_out; 1397ef43aa38SMilan Broz struct dm_crypt_request *dmreq; 1398ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv; 1399c13b5487SChristoph Hellwig __le64 *sector; 1400ef43aa38SMilan Broz int r = 0; 1401ef43aa38SMilan Broz 14028f0009a2SMilan Broz /* Reject unexpected unaligned bio. */ 14030440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1))) 14048f0009a2SMilan Broz return -EIO; 14058f0009a2SMilan Broz 1406ef43aa38SMilan Broz dmreq = dmreq_of_req(cc, req); 1407ef43aa38SMilan Broz dmreq->iv_sector = ctx->cc_sector; 14088f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 1409ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift; 1410ef43aa38SMilan Broz dmreq->ctx = ctx; 1411ef43aa38SMilan Broz 1412ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset; 1413ef43aa38SMilan Broz 1414ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq); 1415ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq); 1416ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq); 1417ef43aa38SMilan Broz 1418ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq); 1419ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset); 1420ef43aa38SMilan Broz 1421ef43aa38SMilan Broz /* For skcipher we use only the first sg item */ 1422ef43aa38SMilan Broz sg_in = &dmreq->sg_in[0]; 1423ef43aa38SMilan Broz sg_out = &dmreq->sg_out[0]; 1424ef43aa38SMilan Broz 1425ef43aa38SMilan Broz sg_init_table(sg_in, 1); 14268f0009a2SMilan Broz sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset); 1427ef43aa38SMilan Broz 1428ef43aa38SMilan Broz sg_init_table(sg_out, 1); 14298f0009a2SMilan Broz sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset); 1430ef43aa38SMilan Broz 1431ef43aa38SMilan Broz if (cc->iv_gen_ops) { 1432ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */ 1433ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) { 1434ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->integrity_iv_size); 1435ef43aa38SMilan Broz } else { 1436ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq); 1437ef43aa38SMilan Broz if (r < 0) 1438ef43aa38SMilan Broz return r; 1439bbb16584SMilan Broz /* Data can be already preprocessed in generator */ 1440bbb16584SMilan Broz if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags)) 1441bbb16584SMilan Broz sg_in = sg_out; 1442ef43aa38SMilan Broz /* Store generated IV in integrity metadata */ 1443ef43aa38SMilan Broz if (cc->integrity_iv_size) 1444ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->integrity_iv_size); 1445ef43aa38SMilan Broz } 1446ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */ 1447ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size); 1448ef43aa38SMilan Broz } 1449ef43aa38SMilan Broz 14508f0009a2SMilan Broz skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv); 14513a7f6c99SMilan Broz 14523a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) 1453bbdb23b5SHerbert Xu r = crypto_skcipher_encrypt(req); 14543a7f6c99SMilan Broz else 1455bbdb23b5SHerbert Xu r = crypto_skcipher_decrypt(req); 14563a7f6c99SMilan Broz 14572dc5327dSMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post) 1458ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq); 1459ef43aa38SMilan Broz 14608f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size); 14618f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size); 14622dc5327dSMilan Broz 14633a7f6c99SMilan Broz return r; 146401482b76SMilan Broz } 146501482b76SMilan Broz 1466dcfe653dSHerbert Xu static void kcryptd_async_done(void *async_req, int error); 1467c0297721SAndi Kleen 1468d68b2958SIgnat Korchagin static int crypt_alloc_req_skcipher(struct crypt_config *cc, 1469ddd42edfSMilan Broz struct convert_context *ctx) 1470ddd42edfSMilan Broz { 147186a3238cSHeinz Mauelshagen unsigned int key_index = ctx->cc_sector & (cc->tfms_count - 1); 1472c0297721SAndi Kleen 1473d68b2958SIgnat Korchagin if (!ctx->r.req) { 1474d68b2958SIgnat Korchagin ctx->r.req = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO); 1475ef43aa38SMilan Broz if (!ctx->r.req) 1476d68b2958SIgnat Korchagin return -ENOMEM; 1477d68b2958SIgnat Korchagin } 1478c0297721SAndi Kleen 1479ef43aa38SMilan Broz skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]); 148054cea3f6SMilan Broz 148154cea3f6SMilan Broz /* 148254cea3f6SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs 148354cea3f6SMilan Broz * requests if driver request queue is full. 148454cea3f6SMilan Broz */ 1485ef43aa38SMilan Broz skcipher_request_set_callback(ctx->r.req, 1486432061b3SMikulas Patocka CRYPTO_TFM_REQ_MAY_BACKLOG, 1487ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req)); 1488d68b2958SIgnat Korchagin 1489d68b2958SIgnat Korchagin return 0; 1490ddd42edfSMilan Broz } 1491ddd42edfSMilan Broz 1492d68b2958SIgnat Korchagin static int crypt_alloc_req_aead(struct crypt_config *cc, 1493ef43aa38SMilan Broz struct convert_context *ctx) 1494ef43aa38SMilan Broz { 1495004b8ae9SIgnat Korchagin if (!ctx->r.req_aead) { 1496004b8ae9SIgnat Korchagin ctx->r.req_aead = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO); 1497004b8ae9SIgnat Korchagin if (!ctx->r.req_aead) 1498d68b2958SIgnat Korchagin return -ENOMEM; 1499d68b2958SIgnat Korchagin } 1500ef43aa38SMilan Broz 1501ef43aa38SMilan Broz aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]); 1502ef43aa38SMilan Broz 1503ef43aa38SMilan Broz /* 1504ef43aa38SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs 1505ef43aa38SMilan Broz * requests if driver request queue is full. 1506ef43aa38SMilan Broz */ 1507ef43aa38SMilan Broz aead_request_set_callback(ctx->r.req_aead, 1508432061b3SMikulas Patocka CRYPTO_TFM_REQ_MAY_BACKLOG, 1509ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead)); 1510d68b2958SIgnat Korchagin 1511d68b2958SIgnat Korchagin return 0; 1512ef43aa38SMilan Broz } 1513ef43aa38SMilan Broz 1514d68b2958SIgnat Korchagin static int crypt_alloc_req(struct crypt_config *cc, 1515ef43aa38SMilan Broz struct convert_context *ctx) 1516ef43aa38SMilan Broz { 151733d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1518d68b2958SIgnat Korchagin return crypt_alloc_req_aead(cc, ctx); 1519ef43aa38SMilan Broz else 1520d68b2958SIgnat Korchagin return crypt_alloc_req_skcipher(cc, ctx); 1521ef43aa38SMilan Broz } 1522ef43aa38SMilan Broz 1523ef43aa38SMilan Broz static void crypt_free_req_skcipher(struct crypt_config *cc, 1524bbdb23b5SHerbert Xu struct skcipher_request *req, struct bio *base_bio) 1525298a9fa0SMikulas Patocka { 1526298a9fa0SMikulas Patocka struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); 1527298a9fa0SMikulas Patocka 1528bbdb23b5SHerbert Xu if ((struct skcipher_request *)(io + 1) != req) 15296f1c819cSKent Overstreet mempool_free(req, &cc->req_pool); 1530298a9fa0SMikulas Patocka } 1531298a9fa0SMikulas Patocka 1532ef43aa38SMilan Broz static void crypt_free_req_aead(struct crypt_config *cc, 1533ef43aa38SMilan Broz struct aead_request *req, struct bio *base_bio) 1534ef43aa38SMilan Broz { 1535ef43aa38SMilan Broz struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size); 1536ef43aa38SMilan Broz 1537ef43aa38SMilan Broz if ((struct aead_request *)(io + 1) != req) 15386f1c819cSKent Overstreet mempool_free(req, &cc->req_pool); 1539ef43aa38SMilan Broz } 1540ef43aa38SMilan Broz 1541ef43aa38SMilan Broz static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio) 1542ef43aa38SMilan Broz { 154333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1544ef43aa38SMilan Broz crypt_free_req_aead(cc, req, base_bio); 1545ef43aa38SMilan Broz else 1546ef43aa38SMilan Broz crypt_free_req_skcipher(cc, req, base_bio); 1547ef43aa38SMilan Broz } 1548ef43aa38SMilan Broz 15491da177e4SLinus Torvalds /* 15501da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one) 15511da177e4SLinus Torvalds */ 15524e4cbee9SChristoph Hellwig static blk_status_t crypt_convert(struct crypt_config *cc, 15538abec36dSIgnat Korchagin struct convert_context *ctx, bool atomic, bool reset_pending) 15541da177e4SLinus Torvalds { 1555ef43aa38SMilan Broz unsigned int tag_offset = 0; 1556ff3af92bSMikulas Patocka unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT; 15573f1e9070SMilan Broz int r; 15581da177e4SLinus Torvalds 15598abec36dSIgnat Korchagin /* 15608abec36dSIgnat Korchagin * if reset_pending is set we are dealing with the bio for the first time, 15618abec36dSIgnat Korchagin * else we're continuing to work on the previous bio, so don't mess with 15628abec36dSIgnat Korchagin * the cc_pending counter 15638abec36dSIgnat Korchagin */ 15648abec36dSIgnat Korchagin if (reset_pending) 156540b6229bSMikulas Patocka atomic_set(&ctx->cc_pending, 1); 1566c8081618SMilan Broz 1567003b5c57SKent Overstreet while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) { 15681da177e4SLinus Torvalds 1569d68b2958SIgnat Korchagin r = crypt_alloc_req(cc, ctx); 1570d68b2958SIgnat Korchagin if (r) { 1571d68b2958SIgnat Korchagin complete(&ctx->restart); 1572d68b2958SIgnat Korchagin return BLK_STS_DEV_RESOURCE; 1573d68b2958SIgnat Korchagin } 1574d68b2958SIgnat Korchagin 157540b6229bSMikulas Patocka atomic_inc(&ctx->cc_pending); 15763f1e9070SMilan Broz 157733d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 1578ef43aa38SMilan Broz r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset); 1579ef43aa38SMilan Broz else 1580ef43aa38SMilan Broz r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset); 15813a7f6c99SMilan Broz 15823a7f6c99SMilan Broz switch (r) { 158354cea3f6SMilan Broz /* 158454cea3f6SMilan Broz * The request was queued by a crypto driver 158554cea3f6SMilan Broz * but the driver request queue is full, let's wait. 158654cea3f6SMilan Broz */ 15873a7f6c99SMilan Broz case -EBUSY: 15888abec36dSIgnat Korchagin if (in_interrupt()) { 15898abec36dSIgnat Korchagin if (try_wait_for_completion(&ctx->restart)) { 15908abec36dSIgnat Korchagin /* 15918abec36dSIgnat Korchagin * we don't have to block to wait for completion, 15928abec36dSIgnat Korchagin * so proceed 15938abec36dSIgnat Korchagin */ 15948abec36dSIgnat Korchagin } else { 15958abec36dSIgnat Korchagin /* 15968abec36dSIgnat Korchagin * we can't wait for completion without blocking 15978abec36dSIgnat Korchagin * exit and continue processing in a workqueue 15988abec36dSIgnat Korchagin */ 15998abec36dSIgnat Korchagin ctx->r.req = NULL; 16008abec36dSIgnat Korchagin ctx->cc_sector += sector_step; 16018abec36dSIgnat Korchagin tag_offset++; 16028abec36dSIgnat Korchagin return BLK_STS_DEV_RESOURCE; 16038abec36dSIgnat Korchagin } 16048abec36dSIgnat Korchagin } else { 16053a7f6c99SMilan Broz wait_for_completion(&ctx->restart); 16068abec36dSIgnat Korchagin } 160716735d02SWolfram Sang reinit_completion(&ctx->restart); 1608df561f66SGustavo A. R. Silva fallthrough; 160954cea3f6SMilan Broz /* 161054cea3f6SMilan Broz * The request is queued and processed asynchronously, 161154cea3f6SMilan Broz * completion function kcryptd_async_done() will be called. 161254cea3f6SMilan Broz */ 1613c0403ec0SRabin Vincent case -EINPROGRESS: 1614ef43aa38SMilan Broz ctx->r.req = NULL; 16158f0009a2SMilan Broz ctx->cc_sector += sector_step; 1616583fe747SMikulas Patocka tag_offset++; 16173a7f6c99SMilan Broz continue; 161854cea3f6SMilan Broz /* 161954cea3f6SMilan Broz * The request was already processed (synchronously). 162054cea3f6SMilan Broz */ 16213f1e9070SMilan Broz case 0: 162240b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 16238f0009a2SMilan Broz ctx->cc_sector += sector_step; 1624583fe747SMikulas Patocka tag_offset++; 162539d42fa9SIgnat Korchagin if (!atomic) 1626c7f1b204SMilan Broz cond_resched(); 16273f1e9070SMilan Broz continue; 1628ef43aa38SMilan Broz /* 1629ef43aa38SMilan Broz * There was a data integrity error. 1630ef43aa38SMilan Broz */ 1631ef43aa38SMilan Broz case -EBADMSG: 1632ef43aa38SMilan Broz atomic_dec(&ctx->cc_pending); 16334e4cbee9SChristoph Hellwig return BLK_STS_PROTECTION; 1634ef43aa38SMilan Broz /* 1635ef43aa38SMilan Broz * There was an error while processing the request. 1636ef43aa38SMilan Broz */ 16373f1e9070SMilan Broz default: 163840b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending); 16394e4cbee9SChristoph Hellwig return BLK_STS_IOERR; 16401da177e4SLinus Torvalds } 16413f1e9070SMilan Broz } 16423f1e9070SMilan Broz 16433f1e9070SMilan Broz return 0; 16443f1e9070SMilan Broz } 16451da177e4SLinus Torvalds 1646cf2f1abfSMikulas Patocka static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone); 1647cf2f1abfSMikulas Patocka 16481da177e4SLinus Torvalds /* 16491da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size 1650586b286bSMike Snitzer * This should never violate the device limitations (but only because 1651586b286bSMike Snitzer * max_segment_size is being constrained to PAGE_SIZE). 16527145c241SMikulas Patocka * 16537145c241SMikulas Patocka * This function may be called concurrently. If we allocate from the mempool 16547145c241SMikulas Patocka * concurrently, there is a possibility of deadlock. For example, if we have 16557145c241SMikulas Patocka * mempool of 256 pages, two processes, each wanting 256, pages allocate from 16567145c241SMikulas Patocka * the mempool concurrently, it may deadlock in a situation where both processes 16577145c241SMikulas Patocka * have allocated 128 pages and the mempool is exhausted. 16587145c241SMikulas Patocka * 16597145c241SMikulas Patocka * In order to avoid this scenario we allocate the pages under a mutex. 16607145c241SMikulas Patocka * 16617145c241SMikulas Patocka * In order to not degrade performance with excessive locking, we try 16627145c241SMikulas Patocka * non-blocking allocations without a mutex first but on failure we fallback 16637145c241SMikulas Patocka * to blocking allocations with a mutex. 16641da177e4SLinus Torvalds */ 166586a3238cSHeinz Mauelshagen static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size) 16661da177e4SLinus Torvalds { 166749a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 16688b004457SMilan Broz struct bio *clone; 16691da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 16707145c241SMikulas Patocka gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; 167186a3238cSHeinz Mauelshagen unsigned int i, len, remaining_size; 167291e10625SMilan Broz struct page *page; 16731da177e4SLinus Torvalds 16747145c241SMikulas Patocka retry: 1675d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 16767145c241SMikulas Patocka mutex_lock(&cc->bio_alloc_lock); 16777145c241SMikulas Patocka 1678609be106SChristoph Hellwig clone = bio_alloc_bioset(cc->dev->bdev, nr_iovecs, io->base_bio->bi_opf, 1679609be106SChristoph Hellwig GFP_NOIO, &cc->bs); 16803f868c09SChristoph Hellwig clone->bi_private = io; 16813f868c09SChristoph Hellwig clone->bi_end_io = crypt_endio; 16826a24c718SMilan Broz 16837145c241SMikulas Patocka remaining_size = size; 16847145c241SMikulas Patocka 1685f97380bcSOlaf Kirch for (i = 0; i < nr_iovecs; i++) { 16866f1c819cSKent Overstreet page = mempool_alloc(&cc->page_pool, gfp_mask); 16877145c241SMikulas Patocka if (!page) { 16887145c241SMikulas Patocka crypt_free_buffer_pages(cc, clone); 16897145c241SMikulas Patocka bio_put(clone); 1690d0164adcSMel Gorman gfp_mask |= __GFP_DIRECT_RECLAIM; 16917145c241SMikulas Patocka goto retry; 16927145c241SMikulas Patocka } 16931da177e4SLinus Torvalds 16947145c241SMikulas Patocka len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size; 16951da177e4SLinus Torvalds 16960dae7fe5SMing Lei bio_add_page(clone, page, len, 0); 169791e10625SMilan Broz 16987145c241SMikulas Patocka remaining_size -= len; 16991da177e4SLinus Torvalds } 17001da177e4SLinus Torvalds 1701ef43aa38SMilan Broz /* Allocate space for integrity tags */ 1702ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) { 1703ef43aa38SMilan Broz crypt_free_buffer_pages(cc, clone); 1704ef43aa38SMilan Broz bio_put(clone); 1705ef43aa38SMilan Broz clone = NULL; 1706ef43aa38SMilan Broz } 170753db984eSChristoph Hellwig 1708d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM)) 17097145c241SMikulas Patocka mutex_unlock(&cc->bio_alloc_lock); 17107145c241SMikulas Patocka 17118b004457SMilan Broz return clone; 17121da177e4SLinus Torvalds } 17131da177e4SLinus Torvalds 1714644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) 17151da177e4SLinus Torvalds { 17161da177e4SLinus Torvalds struct bio_vec *bv; 17176dc4f100SMing Lei struct bvec_iter_all iter_all; 17181da177e4SLinus Torvalds 17192b070cfeSChristoph Hellwig bio_for_each_segment_all(bv, clone, iter_all) { 17201da177e4SLinus Torvalds BUG_ON(!bv->bv_page); 17216f1c819cSKent Overstreet mempool_free(bv->bv_page, &cc->page_pool); 17221da177e4SLinus Torvalds } 17231da177e4SLinus Torvalds } 17241da177e4SLinus Torvalds 1725298a9fa0SMikulas Patocka static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc, 1726dc440d1eSMilan Broz struct bio *bio, sector_t sector) 1727dc440d1eSMilan Broz { 172849a8a920SAlasdair G Kergon io->cc = cc; 1729dc440d1eSMilan Broz io->base_bio = bio; 1730dc440d1eSMilan Broz io->sector = sector; 1731dc440d1eSMilan Broz io->error = 0; 1732ef43aa38SMilan Broz io->ctx.r.req = NULL; 1733ef43aa38SMilan Broz io->integrity_metadata = NULL; 1734ef43aa38SMilan Broz io->integrity_metadata_from_pool = false; 1735d9a02e01SMike Snitzer io->in_tasklet = false; 173640b6229bSMikulas Patocka atomic_set(&io->io_pending, 0); 1737dc440d1eSMilan Broz } 1738dc440d1eSMilan Broz 17393e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io) 17403e1a8bddSMilan Broz { 174140b6229bSMikulas Patocka atomic_inc(&io->io_pending); 17423e1a8bddSMilan Broz } 17433e1a8bddSMilan Broz 17448e14f610SIgnat Korchagin static void kcryptd_io_bio_endio(struct work_struct *work) 17458e14f610SIgnat Korchagin { 17468e14f610SIgnat Korchagin struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 17470ef0b471SHeinz Mauelshagen 17488e14f610SIgnat Korchagin bio_endio(io->base_bio); 17498e14f610SIgnat Korchagin } 17508e14f610SIgnat Korchagin 17511da177e4SLinus Torvalds /* 17521da177e4SLinus Torvalds * One of the bios was finished. Check for completion of 17531da177e4SLinus Torvalds * the whole request and correctly clean up the buffer. 17541da177e4SLinus Torvalds */ 17555742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io) 17561da177e4SLinus Torvalds { 175749a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1758b35f8caaSMilan Broz struct bio *base_bio = io->base_bio; 17594e4cbee9SChristoph Hellwig blk_status_t error = io->error; 17601da177e4SLinus Torvalds 176140b6229bSMikulas Patocka if (!atomic_dec_and_test(&io->io_pending)) 17621da177e4SLinus Torvalds return; 17631da177e4SLinus Torvalds 1764ef43aa38SMilan Broz if (io->ctx.r.req) 1765ef43aa38SMilan Broz crypt_free_req(cc, io->ctx.r.req, base_bio); 1766ef43aa38SMilan Broz 1767ef43aa38SMilan Broz if (unlikely(io->integrity_metadata_from_pool)) 17686f1c819cSKent Overstreet mempool_free(io->integrity_metadata, &io->cc->tag_pool); 1769ef43aa38SMilan Broz else 1770ef43aa38SMilan Broz kfree(io->integrity_metadata); 1771b35f8caaSMilan Broz 17724e4cbee9SChristoph Hellwig base_bio->bi_status = error; 17738e14f610SIgnat Korchagin 17748e14f610SIgnat Korchagin /* 17758e14f610SIgnat Korchagin * If we are running this function from our tasklet, 17768e14f610SIgnat Korchagin * we can't call bio_endio() here, because it will call 17778e14f610SIgnat Korchagin * clone_endio() from dm.c, which in turn will 17788e14f610SIgnat Korchagin * free the current struct dm_crypt_io structure with 17798e14f610SIgnat Korchagin * our tasklet. In this case we need to delay bio_endio() 17808e14f610SIgnat Korchagin * execution to after the tasklet is done and dequeued. 17818e14f610SIgnat Korchagin */ 1782d9a02e01SMike Snitzer if (io->in_tasklet) { 1783d9a02e01SMike Snitzer INIT_WORK(&io->work, kcryptd_io_bio_endio); 1784d9a02e01SMike Snitzer queue_work(cc->io_queue, &io->work); 17858e14f610SIgnat Korchagin return; 17868e14f610SIgnat Korchagin } 17878e14f610SIgnat Korchagin 1788d9a02e01SMike Snitzer bio_endio(base_bio); 17891da177e4SLinus Torvalds } 17901da177e4SLinus Torvalds 17911da177e4SLinus Torvalds /* 1792cabf08e4SMilan Broz * kcryptd/kcryptd_io: 17931da177e4SLinus Torvalds * 17941da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an 179523541d2dSMilan Broz * interrupt context. 1796cabf08e4SMilan Broz * 1797cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption. 1798cabf08e4SMilan Broz * 1799cabf08e4SMilan Broz * kcryptd_io performs the IO submission. 1800cabf08e4SMilan Broz * 1801cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be 1802cabf08e4SMilan Broz * starved by new requests which can block in the first stages due 1803cabf08e4SMilan Broz * to memory allocation. 1804c0297721SAndi Kleen * 1805c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances. 1806c0297721SAndi Kleen * They should not depend on each other and do not block. 18071da177e4SLinus Torvalds */ 18084246a0b6SChristoph Hellwig static void crypt_endio(struct bio *clone) 18098b004457SMilan Broz { 1810028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private; 181149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 181286a3238cSHeinz Mauelshagen unsigned int rw = bio_data_dir(clone); 18134e4cbee9SChristoph Hellwig blk_status_t error; 18148b004457SMilan Broz 18158b004457SMilan Broz /* 18166712ecf8SNeilBrown * free the processed pages 18178b004457SMilan Broz */ 1818ee7a491eSMilan Broz if (rw == WRITE) 1819644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone); 18208b004457SMilan Broz 18214e4cbee9SChristoph Hellwig error = clone->bi_status; 18228b004457SMilan Broz bio_put(clone); 1823ee7a491eSMilan Broz 18249b81c842SSasha Levin if (rw == READ && !error) { 1825cabf08e4SMilan Broz kcryptd_queue_crypt(io); 18266712ecf8SNeilBrown return; 1827ee7a491eSMilan Broz } 18285742fd77SMilan Broz 18299b81c842SSasha Levin if (unlikely(error)) 18309b81c842SSasha Levin io->error = error; 18315742fd77SMilan Broz 18325742fd77SMilan Broz crypt_dec_pending(io); 18338b004457SMilan Broz } 18348b004457SMilan Broz 1835e5524e12SMike Snitzer #define CRYPT_MAP_READ_GFP GFP_NOWAIT 1836e5524e12SMike Snitzer 183720c82538SMilan Broz static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) 18388b004457SMilan Broz { 183949a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 18408b004457SMilan Broz struct bio *clone; 184193e605c2SMilan Broz 18428b004457SMilan Broz /* 1843abfc426dSChristoph Hellwig * We need the original biovec array in order to decrypt the whole bio 1844abfc426dSChristoph Hellwig * data *afterwards* -- thanks to immutable biovecs we don't need to 1845abfc426dSChristoph Hellwig * worry about the block layer modifying the biovec array; so leverage 1846abfc426dSChristoph Hellwig * bio_alloc_clone(). 18478b004457SMilan Broz */ 1848abfc426dSChristoph Hellwig clone = bio_alloc_clone(cc->dev->bdev, io->base_bio, gfp, &cc->bs); 18497eaceaccSJens Axboe if (!clone) 185020c82538SMilan Broz return 1; 18513f868c09SChristoph Hellwig clone->bi_private = io; 18523f868c09SChristoph Hellwig clone->bi_end_io = crypt_endio; 18538b004457SMilan Broz 185420c82538SMilan Broz crypt_inc_pending(io); 185520c82538SMilan Broz 18564f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 18578b004457SMilan Broz 1858ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) { 1859ef43aa38SMilan Broz crypt_dec_pending(io); 1860ef43aa38SMilan Broz bio_put(clone); 1861ef43aa38SMilan Broz return 1; 1862ef43aa38SMilan Broz } 1863ef43aa38SMilan Broz 1864b7f8dff0SMike Snitzer dm_submit_bio_remap(io->base_bio, clone); 186520c82538SMilan Broz return 0; 18668b004457SMilan Broz } 18678b004457SMilan Broz 1868dc267621SMikulas Patocka static void kcryptd_io_read_work(struct work_struct *work) 1869395b167cSAlasdair G Kergon { 1870395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 1871395b167cSAlasdair G Kergon 187220c82538SMilan Broz crypt_inc_pending(io); 187320c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOIO)) 18744e4cbee9SChristoph Hellwig io->error = BLK_STS_RESOURCE; 187520c82538SMilan Broz crypt_dec_pending(io); 1876395b167cSAlasdair G Kergon } 1877395b167cSAlasdair G Kergon 1878dc267621SMikulas Patocka static void kcryptd_queue_read(struct dm_crypt_io *io) 1879395b167cSAlasdair G Kergon { 188049a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1881395b167cSAlasdair G Kergon 1882dc267621SMikulas Patocka INIT_WORK(&io->work, kcryptd_io_read_work); 1883395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work); 1884395b167cSAlasdair G Kergon } 1885395b167cSAlasdair G Kergon 1886dc267621SMikulas Patocka static void kcryptd_io_write(struct dm_crypt_io *io) 1887dc267621SMikulas Patocka { 1888dc267621SMikulas Patocka struct bio *clone = io->ctx.bio_out; 1889dc267621SMikulas Patocka 1890b7f8dff0SMike Snitzer dm_submit_bio_remap(io->base_bio, clone); 1891dc267621SMikulas Patocka } 1892dc267621SMikulas Patocka 1893b3c5fd30SMikulas Patocka #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node) 1894b3c5fd30SMikulas Patocka 1895dc267621SMikulas Patocka static int dmcrypt_write(void *data) 1896dc267621SMikulas Patocka { 1897dc267621SMikulas Patocka struct crypt_config *cc = data; 1898b3c5fd30SMikulas Patocka struct dm_crypt_io *io; 1899b3c5fd30SMikulas Patocka 1900dc267621SMikulas Patocka while (1) { 1901b3c5fd30SMikulas Patocka struct rb_root write_tree; 1902dc267621SMikulas Patocka struct blk_plug plug; 1903dc267621SMikulas Patocka 1904c7329effSMikulas Patocka spin_lock_irq(&cc->write_thread_lock); 1905dc267621SMikulas Patocka continue_locked: 1906dc267621SMikulas Patocka 1907b3c5fd30SMikulas Patocka if (!RB_EMPTY_ROOT(&cc->write_tree)) 1908dc267621SMikulas Patocka goto pop_from_list; 1909dc267621SMikulas Patocka 1910f659b100SRabin Vincent set_current_state(TASK_INTERRUPTIBLE); 1911dc267621SMikulas Patocka 1912c7329effSMikulas Patocka spin_unlock_irq(&cc->write_thread_lock); 1913dc267621SMikulas Patocka 1914f659b100SRabin Vincent if (unlikely(kthread_should_stop())) { 1915642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING); 1916f659b100SRabin Vincent break; 1917f659b100SRabin Vincent } 1918f659b100SRabin Vincent 1919dc267621SMikulas Patocka schedule(); 1920dc267621SMikulas Patocka 1921642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING); 1922c7329effSMikulas Patocka spin_lock_irq(&cc->write_thread_lock); 1923dc267621SMikulas Patocka goto continue_locked; 1924dc267621SMikulas Patocka 1925dc267621SMikulas Patocka pop_from_list: 1926b3c5fd30SMikulas Patocka write_tree = cc->write_tree; 1927b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 1928c7329effSMikulas Patocka spin_unlock_irq(&cc->write_thread_lock); 1929dc267621SMikulas Patocka 1930b3c5fd30SMikulas Patocka BUG_ON(rb_parent(write_tree.rb_node)); 1931b3c5fd30SMikulas Patocka 1932b3c5fd30SMikulas Patocka /* 1933b3c5fd30SMikulas Patocka * Note: we cannot walk the tree here with rb_next because 1934b3c5fd30SMikulas Patocka * the structures may be freed when kcryptd_io_write is called. 1935b3c5fd30SMikulas Patocka */ 1936dc267621SMikulas Patocka blk_start_plug(&plug); 1937dc267621SMikulas Patocka do { 1938b3c5fd30SMikulas Patocka io = crypt_io_from_node(rb_first(&write_tree)); 1939b3c5fd30SMikulas Patocka rb_erase(&io->rb_node, &write_tree); 1940dc267621SMikulas Patocka kcryptd_io_write(io); 1941fb294b1cSMikulas Patocka cond_resched(); 1942b3c5fd30SMikulas Patocka } while (!RB_EMPTY_ROOT(&write_tree)); 1943dc267621SMikulas Patocka blk_finish_plug(&plug); 1944dc267621SMikulas Patocka } 1945dc267621SMikulas Patocka return 0; 1946dc267621SMikulas Patocka } 1947dc267621SMikulas Patocka 194872c6e7afSMikulas Patocka static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) 19494e4eef64SMilan Broz { 1950dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out; 195149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 1952dc267621SMikulas Patocka unsigned long flags; 1953b3c5fd30SMikulas Patocka sector_t sector; 1954b3c5fd30SMikulas Patocka struct rb_node **rbp, *parent; 1955dec1cedfSMilan Broz 19564e4cbee9SChristoph Hellwig if (unlikely(io->error)) { 1957dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone); 1958dec1cedfSMilan Broz bio_put(clone); 19596c031f41SMilan Broz crypt_dec_pending(io); 1960dec1cedfSMilan Broz return; 1961dec1cedfSMilan Broz } 1962dec1cedfSMilan Broz 1963dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */ 1964003b5c57SKent Overstreet BUG_ON(io->ctx.iter_out.bi_size); 1965dec1cedfSMilan Broz 19664f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector; 1967899c95d3SMilan Broz 196839d42fa9SIgnat Korchagin if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) || 196939d42fa9SIgnat Korchagin test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) { 1970b7f8dff0SMike Snitzer dm_submit_bio_remap(io->base_bio, clone); 19710f5d8e6eSMikulas Patocka return; 19720f5d8e6eSMikulas Patocka } 19730f5d8e6eSMikulas Patocka 1974c7329effSMikulas Patocka spin_lock_irqsave(&cc->write_thread_lock, flags); 1975c7329effSMikulas Patocka if (RB_EMPTY_ROOT(&cc->write_tree)) 1976c7329effSMikulas Patocka wake_up_process(cc->write_thread); 1977b3c5fd30SMikulas Patocka rbp = &cc->write_tree.rb_node; 1978b3c5fd30SMikulas Patocka parent = NULL; 1979b3c5fd30SMikulas Patocka sector = io->sector; 1980b3c5fd30SMikulas Patocka while (*rbp) { 1981b3c5fd30SMikulas Patocka parent = *rbp; 1982b3c5fd30SMikulas Patocka if (sector < crypt_io_from_node(parent)->sector) 1983b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_left; 1984b3c5fd30SMikulas Patocka else 1985b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_right; 1986b3c5fd30SMikulas Patocka } 1987b3c5fd30SMikulas Patocka rb_link_node(&io->rb_node, parent, rbp); 1988b3c5fd30SMikulas Patocka rb_insert_color(&io->rb_node, &cc->write_tree); 1989c7329effSMikulas Patocka spin_unlock_irqrestore(&cc->write_thread_lock, flags); 19904e4eef64SMilan Broz } 19914e4eef64SMilan Broz 19928e225f04SDamien Le Moal static bool kcryptd_crypt_write_inline(struct crypt_config *cc, 19938e225f04SDamien Le Moal struct convert_context *ctx) 19948e225f04SDamien Le Moal 19958e225f04SDamien Le Moal { 19968e225f04SDamien Le Moal if (!test_bit(DM_CRYPT_WRITE_INLINE, &cc->flags)) 19978e225f04SDamien Le Moal return false; 19988e225f04SDamien Le Moal 19998e225f04SDamien Le Moal /* 20008e225f04SDamien Le Moal * Note: zone append writes (REQ_OP_ZONE_APPEND) do not have ordering 20018e225f04SDamien Le Moal * constraints so they do not need to be issued inline by 20028e225f04SDamien Le Moal * kcryptd_crypt_write_convert(). 20038e225f04SDamien Le Moal */ 20048e225f04SDamien Le Moal switch (bio_op(ctx->bio_in)) { 20058e225f04SDamien Le Moal case REQ_OP_WRITE: 20068e225f04SDamien Le Moal case REQ_OP_WRITE_ZEROES: 20078e225f04SDamien Le Moal return true; 20088e225f04SDamien Le Moal default: 20098e225f04SDamien Le Moal return false; 20108e225f04SDamien Le Moal } 20118e225f04SDamien Le Moal } 20128e225f04SDamien Le Moal 20138abec36dSIgnat Korchagin static void kcryptd_crypt_write_continue(struct work_struct *work) 20148abec36dSIgnat Korchagin { 20158abec36dSIgnat Korchagin struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 20168abec36dSIgnat Korchagin struct crypt_config *cc = io->cc; 20178abec36dSIgnat Korchagin struct convert_context *ctx = &io->ctx; 20188abec36dSIgnat Korchagin int crypt_finished; 20198abec36dSIgnat Korchagin sector_t sector = io->sector; 20208abec36dSIgnat Korchagin blk_status_t r; 20218abec36dSIgnat Korchagin 20228abec36dSIgnat Korchagin wait_for_completion(&ctx->restart); 20238abec36dSIgnat Korchagin reinit_completion(&ctx->restart); 20248abec36dSIgnat Korchagin 20258abec36dSIgnat Korchagin r = crypt_convert(cc, &io->ctx, true, false); 20268abec36dSIgnat Korchagin if (r) 20278abec36dSIgnat Korchagin io->error = r; 20288abec36dSIgnat Korchagin crypt_finished = atomic_dec_and_test(&ctx->cc_pending); 20298abec36dSIgnat Korchagin if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) { 20308abec36dSIgnat Korchagin /* Wait for completion signaled by kcryptd_async_done() */ 20318abec36dSIgnat Korchagin wait_for_completion(&ctx->restart); 20328abec36dSIgnat Korchagin crypt_finished = 1; 20338abec36dSIgnat Korchagin } 20348abec36dSIgnat Korchagin 20358abec36dSIgnat Korchagin /* Encryption was already finished, submit io now */ 20368abec36dSIgnat Korchagin if (crypt_finished) { 20378abec36dSIgnat Korchagin kcryptd_crypt_write_io_submit(io, 0); 20388abec36dSIgnat Korchagin io->sector = sector; 20398abec36dSIgnat Korchagin } 20408abec36dSIgnat Korchagin 20418abec36dSIgnat Korchagin crypt_dec_pending(io); 20428abec36dSIgnat Korchagin } 20438abec36dSIgnat Korchagin 2044fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 20458b004457SMilan Broz { 204649a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 20478e225f04SDamien Le Moal struct convert_context *ctx = &io->ctx; 20488b004457SMilan Broz struct bio *clone; 2049c8081618SMilan Broz int crypt_finished; 2050b635b00eSMilan Broz sector_t sector = io->sector; 20514e4cbee9SChristoph Hellwig blk_status_t r; 20528b004457SMilan Broz 205393e605c2SMilan Broz /* 2054fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes. 2055fc5a5e9aSMilan Broz */ 2056fc5a5e9aSMilan Broz crypt_inc_pending(io); 20578e225f04SDamien Le Moal crypt_convert_init(cc, ctx, NULL, io->base_bio, sector); 2058fc5a5e9aSMilan Broz 2059cf2f1abfSMikulas Patocka clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size); 206023541d2dSMilan Broz if (unlikely(!clone)) { 20614e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR; 2062cf2f1abfSMikulas Patocka goto dec; 206323541d2dSMilan Broz } 20648b004457SMilan Broz 206553017030SMilan Broz io->ctx.bio_out = clone; 2066003b5c57SKent Overstreet io->ctx.iter_out = clone->bi_iter; 20678b004457SMilan Broz 2068b635b00eSMilan Broz sector += bio_sectors(clone); 2069dec1cedfSMilan Broz 20704e594098SMilan Broz crypt_inc_pending(io); 20718e225f04SDamien Le Moal r = crypt_convert(cc, ctx, 20728abec36dSIgnat Korchagin test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), true); 20738abec36dSIgnat Korchagin /* 20748abec36dSIgnat Korchagin * Crypto API backlogged the request, because its queue was full 20758abec36dSIgnat Korchagin * and we're in softirq context, so continue from a workqueue 20768abec36dSIgnat Korchagin * (TODO: is it actually possible to be in softirq in the write path?) 20778abec36dSIgnat Korchagin */ 20788abec36dSIgnat Korchagin if (r == BLK_STS_DEV_RESOURCE) { 20798abec36dSIgnat Korchagin INIT_WORK(&io->work, kcryptd_crypt_write_continue); 20808abec36dSIgnat Korchagin queue_work(cc->crypt_queue, &io->work); 20818abec36dSIgnat Korchagin return; 20828abec36dSIgnat Korchagin } 20834e4cbee9SChristoph Hellwig if (r) 2084ef43aa38SMilan Broz io->error = r; 20858e225f04SDamien Le Moal crypt_finished = atomic_dec_and_test(&ctx->cc_pending); 20868e225f04SDamien Le Moal if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) { 20878e225f04SDamien Le Moal /* Wait for completion signaled by kcryptd_async_done() */ 20888e225f04SDamien Le Moal wait_for_completion(&ctx->restart); 20898e225f04SDamien Le Moal crypt_finished = 1; 20908e225f04SDamien Le Moal } 2091dec1cedfSMilan Broz 2092c8081618SMilan Broz /* Encryption was already finished, submit io now */ 2093c8081618SMilan Broz if (crypt_finished) { 209472c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 0); 2095b635b00eSMilan Broz io->sector = sector; 20964e594098SMilan Broz } 209793e605c2SMilan Broz 2098cf2f1abfSMikulas Patocka dec: 2099899c95d3SMilan Broz crypt_dec_pending(io); 210084131db6SMilan Broz } 210184131db6SMilan Broz 210272c6e7afSMikulas Patocka static void kcryptd_crypt_read_done(struct dm_crypt_io *io) 21035742fd77SMilan Broz { 21045742fd77SMilan Broz crypt_dec_pending(io); 21055742fd77SMilan Broz } 21065742fd77SMilan Broz 21078abec36dSIgnat Korchagin static void kcryptd_crypt_read_continue(struct work_struct *work) 21088abec36dSIgnat Korchagin { 21098abec36dSIgnat Korchagin struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 21108abec36dSIgnat Korchagin struct crypt_config *cc = io->cc; 21118abec36dSIgnat Korchagin blk_status_t r; 21128abec36dSIgnat Korchagin 21138abec36dSIgnat Korchagin wait_for_completion(&io->ctx.restart); 21148abec36dSIgnat Korchagin reinit_completion(&io->ctx.restart); 21158abec36dSIgnat Korchagin 21168abec36dSIgnat Korchagin r = crypt_convert(cc, &io->ctx, true, false); 21178abec36dSIgnat Korchagin if (r) 21188abec36dSIgnat Korchagin io->error = r; 21198abec36dSIgnat Korchagin 21208abec36dSIgnat Korchagin if (atomic_dec_and_test(&io->ctx.cc_pending)) 21218abec36dSIgnat Korchagin kcryptd_crypt_read_done(io); 21228abec36dSIgnat Korchagin 21238abec36dSIgnat Korchagin crypt_dec_pending(io); 21248abec36dSIgnat Korchagin } 21258abec36dSIgnat Korchagin 21264e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) 21278b004457SMilan Broz { 212849a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 21294e4cbee9SChristoph Hellwig blk_status_t r; 21308b004457SMilan Broz 21313e1a8bddSMilan Broz crypt_inc_pending(io); 21323a7f6c99SMilan Broz 213353017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, 21340c395b0fSMilan Broz io->sector); 21358b004457SMilan Broz 213639d42fa9SIgnat Korchagin r = crypt_convert(cc, &io->ctx, 21378abec36dSIgnat Korchagin test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true); 21388abec36dSIgnat Korchagin /* 21398abec36dSIgnat Korchagin * Crypto API backlogged the request, because its queue was full 21408abec36dSIgnat Korchagin * and we're in softirq context, so continue from a workqueue 21418abec36dSIgnat Korchagin */ 21428abec36dSIgnat Korchagin if (r == BLK_STS_DEV_RESOURCE) { 21438abec36dSIgnat Korchagin INIT_WORK(&io->work, kcryptd_crypt_read_continue); 21448abec36dSIgnat Korchagin queue_work(cc->crypt_queue, &io->work); 21458abec36dSIgnat Korchagin return; 21468abec36dSIgnat Korchagin } 21474e4cbee9SChristoph Hellwig if (r) 2148ef43aa38SMilan Broz io->error = r; 21495742fd77SMilan Broz 215040b6229bSMikulas Patocka if (atomic_dec_and_test(&io->ctx.cc_pending)) 215172c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 21523a7f6c99SMilan Broz 21533a7f6c99SMilan Broz crypt_dec_pending(io); 21548b004457SMilan Broz } 21558b004457SMilan Broz 2156dcfe653dSHerbert Xu static void kcryptd_async_done(void *data, int error) 215795497a96SMilan Broz { 2158dcfe653dSHerbert Xu struct dm_crypt_request *dmreq = data; 2159b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx; 216095497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx); 216149a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 216295497a96SMilan Broz 216354cea3f6SMilan Broz /* 216454cea3f6SMilan Broz * A request from crypto driver backlog is going to be processed now, 216554cea3f6SMilan Broz * finish the completion and continue in crypt_convert(). 216654cea3f6SMilan Broz * (Callback will be called for the second time for this request.) 216754cea3f6SMilan Broz */ 2168c0403ec0SRabin Vincent if (error == -EINPROGRESS) { 2169c0403ec0SRabin Vincent complete(&ctx->restart); 217095497a96SMilan Broz return; 2171c0403ec0SRabin Vincent } 217295497a96SMilan Broz 21732dc5327dSMilan Broz if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post) 2174ef43aa38SMilan Broz error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq); 21752dc5327dSMilan Broz 2176ef43aa38SMilan Broz if (error == -EBADMSG) { 217758d0f180SMichael Weiß sector_t s = le64_to_cpu(*org_sector_of_dmreq(cc, dmreq)); 217858d0f180SMichael Weiß 217966671719SChristoph Hellwig DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu", 218066671719SChristoph Hellwig ctx->bio_in->bi_bdev, s); 218158d0f180SMichael Weiß dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead", 218258d0f180SMichael Weiß ctx->bio_in, s, 0); 21834e4cbee9SChristoph Hellwig io->error = BLK_STS_PROTECTION; 2184ef43aa38SMilan Broz } else if (error < 0) 21854e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR; 218672c6e7afSMikulas Patocka 2187298a9fa0SMikulas Patocka crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio); 218895497a96SMilan Broz 218940b6229bSMikulas Patocka if (!atomic_dec_and_test(&ctx->cc_pending)) 2190c0403ec0SRabin Vincent return; 219195497a96SMilan Broz 21928e225f04SDamien Le Moal /* 21938e225f04SDamien Le Moal * The request is fully completed: for inline writes, let 21948e225f04SDamien Le Moal * kcryptd_crypt_write_convert() do the IO submission. 21958e225f04SDamien Le Moal */ 21968e225f04SDamien Le Moal if (bio_data_dir(io->base_bio) == READ) { 219772c6e7afSMikulas Patocka kcryptd_crypt_read_done(io); 21988e225f04SDamien Le Moal return; 21998e225f04SDamien Le Moal } 22008e225f04SDamien Le Moal 22018e225f04SDamien Le Moal if (kcryptd_crypt_write_inline(cc, ctx)) { 22028e225f04SDamien Le Moal complete(&ctx->restart); 22038e225f04SDamien Le Moal return; 22048e225f04SDamien Le Moal } 22058e225f04SDamien Le Moal 220672c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 1); 220795497a96SMilan Broz } 220895497a96SMilan Broz 22094e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work) 22104e4eef64SMilan Broz { 22114e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 22124e4eef64SMilan Broz 22134e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ) 22144e4eef64SMilan Broz kcryptd_crypt_read_convert(io); 22154e4eef64SMilan Broz else 22164e4eef64SMilan Broz kcryptd_crypt_write_convert(io); 22178b004457SMilan Broz } 22188b004457SMilan Broz 221939d42fa9SIgnat Korchagin static void kcryptd_crypt_tasklet(unsigned long work) 222039d42fa9SIgnat Korchagin { 222139d42fa9SIgnat Korchagin kcryptd_crypt((struct work_struct *)work); 222239d42fa9SIgnat Korchagin } 222339d42fa9SIgnat Korchagin 2224395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io) 2225395b167cSAlasdair G Kergon { 222649a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc; 2227395b167cSAlasdair G Kergon 222839d42fa9SIgnat Korchagin if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) || 222939d42fa9SIgnat Korchagin (bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) { 2230c87a95dcSIgnat Korchagin /* 2231d3703ef3SChangbin Du * in_hardirq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context. 2232c87a95dcSIgnat Korchagin * irqs_disabled(): the kernel may run some IO completion from the idle thread, but 2233c87a95dcSIgnat Korchagin * it is being executed with irqs disabled. 2234c87a95dcSIgnat Korchagin */ 2235d3703ef3SChangbin Du if (in_hardirq() || irqs_disabled()) { 2236d9a02e01SMike Snitzer io->in_tasklet = true; 223739d42fa9SIgnat Korchagin tasklet_init(&io->tasklet, kcryptd_crypt_tasklet, (unsigned long)&io->work); 223839d42fa9SIgnat Korchagin tasklet_schedule(&io->tasklet); 223939d42fa9SIgnat Korchagin return; 224039d42fa9SIgnat Korchagin } 224139d42fa9SIgnat Korchagin 224239d42fa9SIgnat Korchagin kcryptd_crypt(&io->work); 224339d42fa9SIgnat Korchagin return; 224439d42fa9SIgnat Korchagin } 224539d42fa9SIgnat Korchagin 2246395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt); 2247395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work); 2248395b167cSAlasdair G Kergon } 2249395b167cSAlasdair G Kergon 2250ef43aa38SMilan Broz static void crypt_free_tfms_aead(struct crypt_config *cc) 22511da177e4SLinus Torvalds { 2252ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms_aead) 2253ef43aa38SMilan Broz return; 22541da177e4SLinus Torvalds 2255ef43aa38SMilan Broz if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) { 2256ef43aa38SMilan Broz crypto_free_aead(cc->cipher_tfm.tfms_aead[0]); 2257ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead[0] = NULL; 22581da177e4SLinus Torvalds } 22591da177e4SLinus Torvalds 2260ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms_aead); 2261ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead = NULL; 2262ef43aa38SMilan Broz } 22631da177e4SLinus Torvalds 2264ef43aa38SMilan Broz static void crypt_free_tfms_skcipher(struct crypt_config *cc) 2265d1f96423SMilan Broz { 226686a3238cSHeinz Mauelshagen unsigned int i; 2267d1f96423SMilan Broz 2268ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 2269fd2d231fSMikulas Patocka return; 2270fd2d231fSMikulas Patocka 2271d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) 2272ef43aa38SMilan Broz if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) { 2273ef43aa38SMilan Broz crypto_free_skcipher(cc->cipher_tfm.tfms[i]); 2274ef43aa38SMilan Broz cc->cipher_tfm.tfms[i] = NULL; 2275d1f96423SMilan Broz } 2276d1f96423SMilan Broz 2277ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms); 2278ef43aa38SMilan Broz cc->cipher_tfm.tfms = NULL; 22791da177e4SLinus Torvalds } 22801da177e4SLinus Torvalds 22811da177e4SLinus Torvalds static void crypt_free_tfms(struct crypt_config *cc) 2282d1f96423SMilan Broz { 228333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2284ef43aa38SMilan Broz crypt_free_tfms_aead(cc); 2285ef43aa38SMilan Broz else 2286ef43aa38SMilan Broz crypt_free_tfms_skcipher(cc); 2287d1f96423SMilan Broz } 2288d1f96423SMilan Broz 2289ef43aa38SMilan Broz static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode) 2290d1f96423SMilan Broz { 229186a3238cSHeinz Mauelshagen unsigned int i; 2292d1f96423SMilan Broz int err; 2293d1f96423SMilan Broz 22946396bb22SKees Cook cc->cipher_tfm.tfms = kcalloc(cc->tfms_count, 22956396bb22SKees Cook sizeof(struct crypto_skcipher *), 22966396bb22SKees Cook GFP_KERNEL); 2297ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 2298fd2d231fSMikulas Patocka return -ENOMEM; 2299fd2d231fSMikulas Patocka 2300d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 2301cd746938SMikulas Patocka cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0, 2302cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY); 2303ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms[i])) { 2304ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms[i]); 2305fd2d231fSMikulas Patocka crypt_free_tfms(cc); 2306d1f96423SMilan Broz return err; 2307d1f96423SMilan Broz } 2308d1f96423SMilan Broz } 2309d1f96423SMilan Broz 2310af331ebaSEric Biggers /* 2311af331ebaSEric Biggers * dm-crypt performance can vary greatly depending on which crypto 2312af331ebaSEric Biggers * algorithm implementation is used. Help people debug performance 2313af331ebaSEric Biggers * problems by logging the ->cra_driver_name. 2314af331ebaSEric Biggers */ 23157a1cd723SMilan Broz DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode, 2316af331ebaSEric Biggers crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name); 2317d1f96423SMilan Broz return 0; 2318d1f96423SMilan Broz } 2319d1f96423SMilan Broz 2320ef43aa38SMilan Broz static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode) 2321ef43aa38SMilan Broz { 2322ef43aa38SMilan Broz int err; 2323ef43aa38SMilan Broz 2324ef43aa38SMilan Broz cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL); 2325ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms) 2326ef43aa38SMilan Broz return -ENOMEM; 2327ef43aa38SMilan Broz 2328cd746938SMikulas Patocka cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0, 2329cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY); 2330ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) { 2331ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]); 2332ef43aa38SMilan Broz crypt_free_tfms(cc); 2333ef43aa38SMilan Broz return err; 2334ef43aa38SMilan Broz } 2335ef43aa38SMilan Broz 23367a1cd723SMilan Broz DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode, 2337af331ebaSEric Biggers crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name); 2338ef43aa38SMilan Broz return 0; 2339ef43aa38SMilan Broz } 2340ef43aa38SMilan Broz 2341ef43aa38SMilan Broz static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode) 2342ef43aa38SMilan Broz { 234333d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2344ef43aa38SMilan Broz return crypt_alloc_tfms_aead(cc, ciphermode); 2345ef43aa38SMilan Broz else 2346ef43aa38SMilan Broz return crypt_alloc_tfms_skcipher(cc, ciphermode); 2347ef43aa38SMilan Broz } 2348ef43aa38SMilan Broz 234986a3238cSHeinz Mauelshagen static unsigned int crypt_subkey_size(struct crypt_config *cc) 2350ef43aa38SMilan Broz { 2351ef43aa38SMilan Broz return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count); 2352ef43aa38SMilan Broz } 2353ef43aa38SMilan Broz 235486a3238cSHeinz Mauelshagen static unsigned int crypt_authenckey_size(struct crypt_config *cc) 2355ef43aa38SMilan Broz { 2356ef43aa38SMilan Broz return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param)); 2357ef43aa38SMilan Broz } 2358ef43aa38SMilan Broz 2359ef43aa38SMilan Broz /* 2360ef43aa38SMilan Broz * If AEAD is composed like authenc(hmac(sha256),xts(aes)), 2361ef43aa38SMilan Broz * the key must be for some reason in special format. 2362ef43aa38SMilan Broz * This funcion converts cc->key to this special format. 2363ef43aa38SMilan Broz */ 2364ef43aa38SMilan Broz static void crypt_copy_authenckey(char *p, const void *key, 236586a3238cSHeinz Mauelshagen unsigned int enckeylen, unsigned int authkeylen) 2366ef43aa38SMilan Broz { 2367ef43aa38SMilan Broz struct crypto_authenc_key_param *param; 2368ef43aa38SMilan Broz struct rtattr *rta; 2369ef43aa38SMilan Broz 2370ef43aa38SMilan Broz rta = (struct rtattr *)p; 2371ef43aa38SMilan Broz param = RTA_DATA(rta); 2372ef43aa38SMilan Broz param->enckeylen = cpu_to_be32(enckeylen); 2373ef43aa38SMilan Broz rta->rta_len = RTA_LENGTH(sizeof(*param)); 2374ef43aa38SMilan Broz rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; 2375ef43aa38SMilan Broz p += RTA_SPACE(sizeof(*param)); 2376ef43aa38SMilan Broz memcpy(p, key + enckeylen, authkeylen); 2377ef43aa38SMilan Broz p += authkeylen; 2378ef43aa38SMilan Broz memcpy(p, key, enckeylen); 2379ef43aa38SMilan Broz } 2380ef43aa38SMilan Broz 2381671ea6b4SMikulas Patocka static int crypt_setkey(struct crypt_config *cc) 2382c0297721SAndi Kleen { 238386a3238cSHeinz Mauelshagen unsigned int subkey_size; 2384fd2d231fSMikulas Patocka int err = 0, i, r; 2385c0297721SAndi Kleen 2386da31a078SMilan Broz /* Ignore extra keys (which are used for IV etc) */ 2387ef43aa38SMilan Broz subkey_size = crypt_subkey_size(cc); 2388da31a078SMilan Broz 238927c70036SMilan Broz if (crypt_integrity_hmac(cc)) { 239027c70036SMilan Broz if (subkey_size < cc->key_mac_size) 239127c70036SMilan Broz return -EINVAL; 239227c70036SMilan Broz 2393ef43aa38SMilan Broz crypt_copy_authenckey(cc->authenc_key, cc->key, 2394ef43aa38SMilan Broz subkey_size - cc->key_mac_size, 2395ef43aa38SMilan Broz cc->key_mac_size); 239627c70036SMilan Broz } 239727c70036SMilan Broz 2398d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) { 239933d2f09fSMilan Broz if (crypt_integrity_hmac(cc)) 240033d2f09fSMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i], 240133d2f09fSMilan Broz cc->authenc_key, crypt_authenckey_size(cc)); 240233d2f09fSMilan Broz else if (crypt_integrity_aead(cc)) 2403ef43aa38SMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i], 2404ef43aa38SMilan Broz cc->key + (i * subkey_size), 2405ef43aa38SMilan Broz subkey_size); 2406ef43aa38SMilan Broz else 2407ef43aa38SMilan Broz r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i], 2408fd2d231fSMikulas Patocka cc->key + (i * subkey_size), 2409fd2d231fSMikulas Patocka subkey_size); 2410c0297721SAndi Kleen if (r) 2411c0297721SAndi Kleen err = r; 2412c0297721SAndi Kleen } 2413c0297721SAndi Kleen 2414ef43aa38SMilan Broz if (crypt_integrity_hmac(cc)) 2415ef43aa38SMilan Broz memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc)); 2416ef43aa38SMilan Broz 2417c0297721SAndi Kleen return err; 2418c0297721SAndi Kleen } 2419c0297721SAndi Kleen 2420c538f6ecSOndrej Kozina #ifdef CONFIG_KEYS 2421c538f6ecSOndrej Kozina 2422027c431cSOndrej Kozina static bool contains_whitespace(const char *str) 2423027c431cSOndrej Kozina { 2424027c431cSOndrej Kozina while (*str) 2425027c431cSOndrej Kozina if (isspace(*str++)) 2426027c431cSOndrej Kozina return true; 2427027c431cSOndrej Kozina return false; 2428027c431cSOndrej Kozina } 2429027c431cSOndrej Kozina 243027f5411aSDmitry Baryshkov static int set_key_user(struct crypt_config *cc, struct key *key) 243127f5411aSDmitry Baryshkov { 243227f5411aSDmitry Baryshkov const struct user_key_payload *ukp; 243327f5411aSDmitry Baryshkov 243427f5411aSDmitry Baryshkov ukp = user_key_payload_locked(key); 243527f5411aSDmitry Baryshkov if (!ukp) 243627f5411aSDmitry Baryshkov return -EKEYREVOKED; 243727f5411aSDmitry Baryshkov 243827f5411aSDmitry Baryshkov if (cc->key_size != ukp->datalen) 243927f5411aSDmitry Baryshkov return -EINVAL; 244027f5411aSDmitry Baryshkov 244127f5411aSDmitry Baryshkov memcpy(cc->key, ukp->data, cc->key_size); 244227f5411aSDmitry Baryshkov 244327f5411aSDmitry Baryshkov return 0; 244427f5411aSDmitry Baryshkov } 244527f5411aSDmitry Baryshkov 244627f5411aSDmitry Baryshkov static int set_key_encrypted(struct crypt_config *cc, struct key *key) 244727f5411aSDmitry Baryshkov { 244827f5411aSDmitry Baryshkov const struct encrypted_key_payload *ekp; 244927f5411aSDmitry Baryshkov 245027f5411aSDmitry Baryshkov ekp = key->payload.data[0]; 245127f5411aSDmitry Baryshkov if (!ekp) 245227f5411aSDmitry Baryshkov return -EKEYREVOKED; 245327f5411aSDmitry Baryshkov 245427f5411aSDmitry Baryshkov if (cc->key_size != ekp->decrypted_datalen) 245527f5411aSDmitry Baryshkov return -EINVAL; 245627f5411aSDmitry Baryshkov 245727f5411aSDmitry Baryshkov memcpy(cc->key, ekp->decrypted_data, cc->key_size); 245827f5411aSDmitry Baryshkov 245927f5411aSDmitry Baryshkov return 0; 246027f5411aSDmitry Baryshkov } 246127f5411aSDmitry Baryshkov 2462363880c4SAhmad Fatoum static int set_key_trusted(struct crypt_config *cc, struct key *key) 2463363880c4SAhmad Fatoum { 2464363880c4SAhmad Fatoum const struct trusted_key_payload *tkp; 2465363880c4SAhmad Fatoum 2466363880c4SAhmad Fatoum tkp = key->payload.data[0]; 2467363880c4SAhmad Fatoum if (!tkp) 2468363880c4SAhmad Fatoum return -EKEYREVOKED; 2469363880c4SAhmad Fatoum 2470363880c4SAhmad Fatoum if (cc->key_size != tkp->key_len) 2471363880c4SAhmad Fatoum return -EINVAL; 2472363880c4SAhmad Fatoum 2473363880c4SAhmad Fatoum memcpy(cc->key, tkp->key, cc->key_size); 2474363880c4SAhmad Fatoum 2475363880c4SAhmad Fatoum return 0; 2476363880c4SAhmad Fatoum } 2477363880c4SAhmad Fatoum 2478c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string) 2479c538f6ecSOndrej Kozina { 2480c538f6ecSOndrej Kozina char *new_key_string, *key_desc; 2481c538f6ecSOndrej Kozina int ret; 248227f5411aSDmitry Baryshkov struct key_type *type; 2483c538f6ecSOndrej Kozina struct key *key; 248427f5411aSDmitry Baryshkov int (*set_key)(struct crypt_config *cc, struct key *key); 2485c538f6ecSOndrej Kozina 2486027c431cSOndrej Kozina /* 2487027c431cSOndrej Kozina * Reject key_string with whitespace. dm core currently lacks code for 2488027c431cSOndrej Kozina * proper whitespace escaping in arguments on DM_TABLE_STATUS path. 2489027c431cSOndrej Kozina */ 2490027c431cSOndrej Kozina if (contains_whitespace(key_string)) { 2491027c431cSOndrej Kozina DMERR("whitespace chars not allowed in key string"); 2492027c431cSOndrej Kozina return -EINVAL; 2493027c431cSOndrej Kozina } 2494027c431cSOndrej Kozina 2495c538f6ecSOndrej Kozina /* look for next ':' separating key_type from key_description */ 2496fc772580SChristophe JAILLET key_desc = strchr(key_string, ':'); 2497c538f6ecSOndrej Kozina if (!key_desc || key_desc == key_string || !strlen(key_desc + 1)) 2498c538f6ecSOndrej Kozina return -EINVAL; 2499c538f6ecSOndrej Kozina 250027f5411aSDmitry Baryshkov if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) { 250127f5411aSDmitry Baryshkov type = &key_type_logon; 250227f5411aSDmitry Baryshkov set_key = set_key_user; 250327f5411aSDmitry Baryshkov } else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) { 250427f5411aSDmitry Baryshkov type = &key_type_user; 250527f5411aSDmitry Baryshkov set_key = set_key_user; 2506831475ccSAhmad Fatoum } else if (IS_ENABLED(CONFIG_ENCRYPTED_KEYS) && 2507831475ccSAhmad Fatoum !strncmp(key_string, "encrypted:", key_desc - key_string + 1)) { 250827f5411aSDmitry Baryshkov type = &key_type_encrypted; 250927f5411aSDmitry Baryshkov set_key = set_key_encrypted; 2510363880c4SAhmad Fatoum } else if (IS_ENABLED(CONFIG_TRUSTED_KEYS) && 2511363880c4SAhmad Fatoum !strncmp(key_string, "trusted:", key_desc - key_string + 1)) { 2512363880c4SAhmad Fatoum type = &key_type_trusted; 2513363880c4SAhmad Fatoum set_key = set_key_trusted; 251427f5411aSDmitry Baryshkov } else { 2515c538f6ecSOndrej Kozina return -EINVAL; 251627f5411aSDmitry Baryshkov } 2517c538f6ecSOndrej Kozina 2518c538f6ecSOndrej Kozina new_key_string = kstrdup(key_string, GFP_KERNEL); 2519c538f6ecSOndrej Kozina if (!new_key_string) 2520c538f6ecSOndrej Kozina return -ENOMEM; 2521c538f6ecSOndrej Kozina 252227f5411aSDmitry Baryshkov key = request_key(type, key_desc + 1, NULL); 2523c538f6ecSOndrej Kozina if (IS_ERR(key)) { 2524453431a5SWaiman Long kfree_sensitive(new_key_string); 2525c538f6ecSOndrej Kozina return PTR_ERR(key); 2526c538f6ecSOndrej Kozina } 2527c538f6ecSOndrej Kozina 2528f5b0cba8SOndrej Kozina down_read(&key->sem); 2529c538f6ecSOndrej Kozina 253027f5411aSDmitry Baryshkov ret = set_key(cc, key); 253127f5411aSDmitry Baryshkov if (ret < 0) { 2532f5b0cba8SOndrej Kozina up_read(&key->sem); 2533c538f6ecSOndrej Kozina key_put(key); 2534453431a5SWaiman Long kfree_sensitive(new_key_string); 253527f5411aSDmitry Baryshkov return ret; 2536c538f6ecSOndrej Kozina } 2537c538f6ecSOndrej Kozina 2538f5b0cba8SOndrej Kozina up_read(&key->sem); 2539c538f6ecSOndrej Kozina key_put(key); 2540c538f6ecSOndrej Kozina 2541c538f6ecSOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */ 2542c538f6ecSOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2543c538f6ecSOndrej Kozina 2544c538f6ecSOndrej Kozina ret = crypt_setkey(cc); 2545c538f6ecSOndrej Kozina 2546c538f6ecSOndrej Kozina if (!ret) { 2547c538f6ecSOndrej Kozina set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2548453431a5SWaiman Long kfree_sensitive(cc->key_string); 2549c538f6ecSOndrej Kozina cc->key_string = new_key_string; 2550c538f6ecSOndrej Kozina } else 2551453431a5SWaiman Long kfree_sensitive(new_key_string); 2552c538f6ecSOndrej Kozina 2553c538f6ecSOndrej Kozina return ret; 2554c538f6ecSOndrej Kozina } 2555c538f6ecSOndrej Kozina 2556c538f6ecSOndrej Kozina static int get_key_size(char **key_string) 2557c538f6ecSOndrej Kozina { 2558c538f6ecSOndrej Kozina char *colon, dummy; 2559c538f6ecSOndrej Kozina int ret; 2560c538f6ecSOndrej Kozina 2561c538f6ecSOndrej Kozina if (*key_string[0] != ':') 2562c538f6ecSOndrej Kozina return strlen(*key_string) >> 1; 2563c538f6ecSOndrej Kozina 2564c538f6ecSOndrej Kozina /* look for next ':' in key string */ 2565c538f6ecSOndrej Kozina colon = strpbrk(*key_string + 1, ":"); 2566c538f6ecSOndrej Kozina if (!colon) 2567c538f6ecSOndrej Kozina return -EINVAL; 2568c538f6ecSOndrej Kozina 2569c538f6ecSOndrej Kozina if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':') 2570c538f6ecSOndrej Kozina return -EINVAL; 2571c538f6ecSOndrej Kozina 2572c538f6ecSOndrej Kozina *key_string = colon; 2573c538f6ecSOndrej Kozina 2574c538f6ecSOndrej Kozina /* remaining key string should be :<logon|user>:<key_desc> */ 2575c538f6ecSOndrej Kozina 2576c538f6ecSOndrej Kozina return ret; 2577c538f6ecSOndrej Kozina } 2578c538f6ecSOndrej Kozina 2579c538f6ecSOndrej Kozina #else 2580c538f6ecSOndrej Kozina 2581c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string) 2582c538f6ecSOndrej Kozina { 2583c538f6ecSOndrej Kozina return -EINVAL; 2584c538f6ecSOndrej Kozina } 2585c538f6ecSOndrej Kozina 2586c538f6ecSOndrej Kozina static int get_key_size(char **key_string) 2587c538f6ecSOndrej Kozina { 25886fc51504SAashish Sharma return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1); 2589c538f6ecSOndrej Kozina } 2590c538f6ecSOndrej Kozina 259127f5411aSDmitry Baryshkov #endif /* CONFIG_KEYS */ 2592c538f6ecSOndrej Kozina 2593e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key) 2594e48d4bbfSMilan Broz { 2595de8be5acSMilan Broz int r = -EINVAL; 2596de8be5acSMilan Broz int key_string_len = strlen(key); 2597de8be5acSMilan Broz 259869a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */ 259969a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-")) 2600de8be5acSMilan Broz goto out; 2601e48d4bbfSMilan Broz 2602c538f6ecSOndrej Kozina /* ':' means the key is in kernel keyring, short-circuit normal key processing */ 2603c538f6ecSOndrej Kozina if (key[0] == ':') { 2604c538f6ecSOndrej Kozina r = crypt_set_keyring_key(cc, key + 1); 2605c538f6ecSOndrej Kozina goto out; 2606c538f6ecSOndrej Kozina } 2607c538f6ecSOndrej Kozina 2608265e9098SOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */ 2609265e9098SOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2610265e9098SOndrej Kozina 2611c538f6ecSOndrej Kozina /* wipe references to any kernel keyring key */ 2612453431a5SWaiman Long kfree_sensitive(cc->key_string); 2613c538f6ecSOndrej Kozina cc->key_string = NULL; 2614c538f6ecSOndrej Kozina 2615e944e03eSAndy Shevchenko /* Decode key from its hex representation. */ 2616e944e03eSAndy Shevchenko if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0) 2617de8be5acSMilan Broz goto out; 2618e48d4bbfSMilan Broz 2619671ea6b4SMikulas Patocka r = crypt_setkey(cc); 2620265e9098SOndrej Kozina if (!r) 2621e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2622e48d4bbfSMilan Broz 2623de8be5acSMilan Broz out: 2624de8be5acSMilan Broz /* Hex key string not needed after here, so wipe it. */ 2625de8be5acSMilan Broz memset(key, '0', key_string_len); 2626de8be5acSMilan Broz 2627de8be5acSMilan Broz return r; 2628e48d4bbfSMilan Broz } 2629e48d4bbfSMilan Broz 2630e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc) 2631e48d4bbfSMilan Broz { 2632c82feeecSOndrej Kozina int r; 2633c82feeecSOndrej Kozina 2634e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 2635c82feeecSOndrej Kozina get_random_bytes(&cc->key, cc->key_size); 26364a52ffc7SMilan Broz 26374a52ffc7SMilan Broz /* Wipe IV private keys */ 26384a52ffc7SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) { 26394a52ffc7SMilan Broz r = cc->iv_gen_ops->wipe(cc); 26404a52ffc7SMilan Broz if (r) 26414a52ffc7SMilan Broz return r; 26424a52ffc7SMilan Broz } 26434a52ffc7SMilan Broz 2644453431a5SWaiman Long kfree_sensitive(cc->key_string); 2645c538f6ecSOndrej Kozina cc->key_string = NULL; 2646c82feeecSOndrej Kozina r = crypt_setkey(cc); 2647c82feeecSOndrej Kozina memset(&cc->key, 0, cc->key_size * sizeof(u8)); 2648c0297721SAndi Kleen 2649c82feeecSOndrej Kozina return r; 2650e48d4bbfSMilan Broz } 2651e48d4bbfSMilan Broz 26525059353dSMikulas Patocka static void crypt_calculate_pages_per_client(void) 26535059353dSMikulas Patocka { 2654ca79b0c2SArun KS unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100; 26555059353dSMikulas Patocka 26565059353dSMikulas Patocka if (!dm_crypt_clients_n) 26575059353dSMikulas Patocka return; 26585059353dSMikulas Patocka 26595059353dSMikulas Patocka pages /= dm_crypt_clients_n; 26605059353dSMikulas Patocka if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT) 26615059353dSMikulas Patocka pages = DM_CRYPT_MIN_PAGES_PER_CLIENT; 26625059353dSMikulas Patocka dm_crypt_pages_per_client = pages; 26635059353dSMikulas Patocka } 26645059353dSMikulas Patocka 26655059353dSMikulas Patocka static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data) 26665059353dSMikulas Patocka { 26675059353dSMikulas Patocka struct crypt_config *cc = pool_data; 26685059353dSMikulas Patocka struct page *page; 26695059353dSMikulas Patocka 2670528b16bfSArne Welzel /* 2671528b16bfSArne Welzel * Note, percpu_counter_read_positive() may over (and under) estimate 2672528b16bfSArne Welzel * the current usage by at most (batch - 1) * num_online_cpus() pages, 2673528b16bfSArne Welzel * but avoids potential spinlock contention of an exact result. 2674528b16bfSArne Welzel */ 2675528b16bfSArne Welzel if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) && 26765059353dSMikulas Patocka likely(gfp_mask & __GFP_NORETRY)) 26775059353dSMikulas Patocka return NULL; 26785059353dSMikulas Patocka 26795059353dSMikulas Patocka page = alloc_page(gfp_mask); 26805059353dSMikulas Patocka if (likely(page != NULL)) 26815059353dSMikulas Patocka percpu_counter_add(&cc->n_allocated_pages, 1); 26825059353dSMikulas Patocka 26835059353dSMikulas Patocka return page; 26845059353dSMikulas Patocka } 26855059353dSMikulas Patocka 26865059353dSMikulas Patocka static void crypt_page_free(void *page, void *pool_data) 26875059353dSMikulas Patocka { 26885059353dSMikulas Patocka struct crypt_config *cc = pool_data; 26895059353dSMikulas Patocka 26905059353dSMikulas Patocka __free_page(page); 26915059353dSMikulas Patocka percpu_counter_sub(&cc->n_allocated_pages, 1); 26925059353dSMikulas Patocka } 26935059353dSMikulas Patocka 269428513fccSMilan Broz static void crypt_dtr(struct dm_target *ti) 269528513fccSMilan Broz { 269628513fccSMilan Broz struct crypt_config *cc = ti->private; 269728513fccSMilan Broz 269828513fccSMilan Broz ti->private = NULL; 269928513fccSMilan Broz 270028513fccSMilan Broz if (!cc) 270128513fccSMilan Broz return; 270228513fccSMilan Broz 2703f659b100SRabin Vincent if (cc->write_thread) 2704dc267621SMikulas Patocka kthread_stop(cc->write_thread); 2705dc267621SMikulas Patocka 270628513fccSMilan Broz if (cc->io_queue) 270728513fccSMilan Broz destroy_workqueue(cc->io_queue); 270828513fccSMilan Broz if (cc->crypt_queue) 270928513fccSMilan Broz destroy_workqueue(cc->crypt_queue); 271028513fccSMilan Broz 2711fd2d231fSMikulas Patocka crypt_free_tfms(cc); 2712fd2d231fSMikulas Patocka 27136f1c819cSKent Overstreet bioset_exit(&cc->bs); 271428513fccSMilan Broz 27156f1c819cSKent Overstreet mempool_exit(&cc->page_pool); 27166f1c819cSKent Overstreet mempool_exit(&cc->req_pool); 27176f1c819cSKent Overstreet mempool_exit(&cc->tag_pool); 27186f1c819cSKent Overstreet 2719d00a11dfSKent Overstreet WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0); 2720d00a11dfSKent Overstreet percpu_counter_destroy(&cc->n_allocated_pages); 2721d00a11dfSKent Overstreet 272228513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 272328513fccSMilan Broz cc->iv_gen_ops->dtr(cc); 272428513fccSMilan Broz 272528513fccSMilan Broz if (cc->dev) 272628513fccSMilan Broz dm_put_device(ti, cc->dev); 272728513fccSMilan Broz 2728453431a5SWaiman Long kfree_sensitive(cc->cipher_string); 2729453431a5SWaiman Long kfree_sensitive(cc->key_string); 2730453431a5SWaiman Long kfree_sensitive(cc->cipher_auth); 2731453431a5SWaiman Long kfree_sensitive(cc->authenc_key); 273228513fccSMilan Broz 2733d5ffebddSMike Snitzer mutex_destroy(&cc->bio_alloc_lock); 2734d5ffebddSMike Snitzer 273528513fccSMilan Broz /* Must zero key material before freeing */ 2736453431a5SWaiman Long kfree_sensitive(cc); 27375059353dSMikulas Patocka 27385059353dSMikulas Patocka spin_lock(&dm_crypt_clients_lock); 27395059353dSMikulas Patocka WARN_ON(!dm_crypt_clients_n); 27405059353dSMikulas Patocka dm_crypt_clients_n--; 27415059353dSMikulas Patocka crypt_calculate_pages_per_client(); 27425059353dSMikulas Patocka spin_unlock(&dm_crypt_clients_lock); 274358d0f180SMichael Weiß 274458d0f180SMichael Weiß dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); 274528513fccSMilan Broz } 274628513fccSMilan Broz 2747e889f97aSMilan Broz static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode) 27481da177e4SLinus Torvalds { 27495ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 27501da177e4SLinus Torvalds 275133d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 2752e889f97aSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc)); 2753e889f97aSMilan Broz else 2754bbdb23b5SHerbert Xu cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc)); 2755e889f97aSMilan Broz 27565ebaee6dSMilan Broz if (cc->iv_size) 27575ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */ 27585ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size, 27595ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8))); 27605ebaee6dSMilan Broz else if (ivmode) { 27615ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs"); 27625ebaee6dSMilan Broz ivmode = NULL; 27635ebaee6dSMilan Broz } 27645ebaee6dSMilan Broz 27655ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */ 27661da177e4SLinus Torvalds if (ivmode == NULL) 27671da177e4SLinus Torvalds cc->iv_gen_ops = NULL; 27681da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0) 27691da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops; 277061afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0) 277161afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops; 27727e3fd855SMilan Broz else if (strcmp(ivmode, "plain64be") == 0) 27737e3fd855SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64be_ops; 27741da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0) 27751da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops; 277648527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0) 277748527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops; 277846b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0) 277946b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops; 2780b9411d73SMilan Broz else if (strcmp(ivmode, "eboiv") == 0) 2781b9411d73SMilan Broz cc->iv_gen_ops = &crypt_iv_eboiv_ops; 2782bbb16584SMilan Broz else if (strcmp(ivmode, "elephant") == 0) { 2783bbb16584SMilan Broz cc->iv_gen_ops = &crypt_iv_elephant_ops; 2784bbb16584SMilan Broz cc->key_parts = 2; 2785bbb16584SMilan Broz cc->key_extra_size = cc->key_size / 2; 2786bbb16584SMilan Broz if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE) 2787bbb16584SMilan Broz return -EINVAL; 2788bbb16584SMilan Broz set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags); 2789bbb16584SMilan Broz } else if (strcmp(ivmode, "lmk") == 0) { 279034745785SMilan Broz cc->iv_gen_ops = &crypt_iv_lmk_ops; 2791ed04d981SMilan Broz /* 2792ed04d981SMilan Broz * Version 2 and 3 is recognised according 279334745785SMilan Broz * to length of provided multi-key string. 279434745785SMilan Broz * If present (version 3), last key is used as IV seed. 2795ed04d981SMilan Broz * All keys (including IV seed) are always the same size. 279634745785SMilan Broz */ 2797da31a078SMilan Broz if (cc->key_size % cc->key_parts) { 279834745785SMilan Broz cc->key_parts++; 2799da31a078SMilan Broz cc->key_extra_size = cc->key_size / cc->key_parts; 2800da31a078SMilan Broz } 2801ed04d981SMilan Broz } else if (strcmp(ivmode, "tcw") == 0) { 2802ed04d981SMilan Broz cc->iv_gen_ops = &crypt_iv_tcw_ops; 2803ed04d981SMilan Broz cc->key_parts += 2; /* IV + whitening */ 2804ed04d981SMilan Broz cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE; 2805e889f97aSMilan Broz } else if (strcmp(ivmode, "random") == 0) { 2806e889f97aSMilan Broz cc->iv_gen_ops = &crypt_iv_random_ops; 2807e889f97aSMilan Broz /* Need storage space in integrity fields. */ 2808e889f97aSMilan Broz cc->integrity_iv_size = cc->iv_size; 280934745785SMilan Broz } else { 281072d94861SAlasdair G Kergon ti->error = "Invalid IV mode"; 2811e889f97aSMilan Broz return -EINVAL; 28121da177e4SLinus Torvalds } 28131da177e4SLinus Torvalds 2814e889f97aSMilan Broz return 0; 2815e889f97aSMilan Broz } 2816e889f97aSMilan Broz 281733d2f09fSMilan Broz /* 281833d2f09fSMilan Broz * Workaround to parse HMAC algorithm from AEAD crypto API spec. 281933d2f09fSMilan Broz * The HMAC is needed to calculate tag size (HMAC digest size). 282033d2f09fSMilan Broz * This should be probably done by crypto-api calls (once available...) 282133d2f09fSMilan Broz */ 282233d2f09fSMilan Broz static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api) 282333d2f09fSMilan Broz { 282433d2f09fSMilan Broz char *start, *end, *mac_alg = NULL; 282533d2f09fSMilan Broz struct crypto_ahash *mac; 282633d2f09fSMilan Broz 282733d2f09fSMilan Broz if (!strstarts(cipher_api, "authenc(")) 282833d2f09fSMilan Broz return 0; 282933d2f09fSMilan Broz 283033d2f09fSMilan Broz start = strchr(cipher_api, '('); 283133d2f09fSMilan Broz end = strchr(cipher_api, ','); 283233d2f09fSMilan Broz if (!start || !end || ++start > end) 283333d2f09fSMilan Broz return -EINVAL; 283433d2f09fSMilan Broz 283533d2f09fSMilan Broz mac_alg = kzalloc(end - start + 1, GFP_KERNEL); 283633d2f09fSMilan Broz if (!mac_alg) 283733d2f09fSMilan Broz return -ENOMEM; 283833d2f09fSMilan Broz strncpy(mac_alg, start, end - start); 283933d2f09fSMilan Broz 2840cd746938SMikulas Patocka mac = crypto_alloc_ahash(mac_alg, 0, CRYPTO_ALG_ALLOCATES_MEMORY); 284133d2f09fSMilan Broz kfree(mac_alg); 284233d2f09fSMilan Broz 284333d2f09fSMilan Broz if (IS_ERR(mac)) 284433d2f09fSMilan Broz return PTR_ERR(mac); 284533d2f09fSMilan Broz 284633d2f09fSMilan Broz cc->key_mac_size = crypto_ahash_digestsize(mac); 284733d2f09fSMilan Broz crypto_free_ahash(mac); 284833d2f09fSMilan Broz 284933d2f09fSMilan Broz cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL); 285033d2f09fSMilan Broz if (!cc->authenc_key) 285133d2f09fSMilan Broz return -ENOMEM; 285233d2f09fSMilan Broz 285333d2f09fSMilan Broz return 0; 285433d2f09fSMilan Broz } 285533d2f09fSMilan Broz 285633d2f09fSMilan Broz static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key, 285733d2f09fSMilan Broz char **ivmode, char **ivopts) 28581da177e4SLinus Torvalds { 28595ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 2860a1a262b6SArd Biesheuvel char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME]; 286133d2f09fSMilan Broz int ret = -EINVAL; 286233d2f09fSMilan Broz 286333d2f09fSMilan Broz cc->tfms_count = 1; 286433d2f09fSMilan Broz 286533d2f09fSMilan Broz /* 286633d2f09fSMilan Broz * New format (capi: prefix) 286733d2f09fSMilan Broz * capi:cipher_api_spec-iv:ivopts 286833d2f09fSMilan Broz */ 286933d2f09fSMilan Broz tmp = &cipher_in[strlen("capi:")]; 28701856b9f7SMilan Broz 28711856b9f7SMilan Broz /* Separate IV options if present, it can contain another '-' in hash name */ 28721856b9f7SMilan Broz *ivopts = strrchr(tmp, ':'); 28731856b9f7SMilan Broz if (*ivopts) { 28741856b9f7SMilan Broz **ivopts = '\0'; 28751856b9f7SMilan Broz (*ivopts)++; 28761856b9f7SMilan Broz } 28771856b9f7SMilan Broz /* Parse IV mode */ 28781856b9f7SMilan Broz *ivmode = strrchr(tmp, '-'); 28791856b9f7SMilan Broz if (*ivmode) { 28801856b9f7SMilan Broz **ivmode = '\0'; 28811856b9f7SMilan Broz (*ivmode)++; 28821856b9f7SMilan Broz } 28831856b9f7SMilan Broz /* The rest is crypto API spec */ 28841856b9f7SMilan Broz cipher_api = tmp; 288533d2f09fSMilan Broz 2886a1a262b6SArd Biesheuvel /* Alloc AEAD, can be used only in new format. */ 2887a1a262b6SArd Biesheuvel if (crypt_integrity_aead(cc)) { 2888a1a262b6SArd Biesheuvel ret = crypt_ctr_auth_cipher(cc, cipher_api); 2889a1a262b6SArd Biesheuvel if (ret < 0) { 2890a1a262b6SArd Biesheuvel ti->error = "Invalid AEAD cipher spec"; 2891a1a262b6SArd Biesheuvel return -ENOMEM; 2892a1a262b6SArd Biesheuvel } 2893a1a262b6SArd Biesheuvel } 2894a1a262b6SArd Biesheuvel 289533d2f09fSMilan Broz if (*ivmode && !strcmp(*ivmode, "lmk")) 289633d2f09fSMilan Broz cc->tfms_count = 64; 289733d2f09fSMilan Broz 2898a1a262b6SArd Biesheuvel if (*ivmode && !strcmp(*ivmode, "essiv")) { 2899a1a262b6SArd Biesheuvel if (!*ivopts) { 2900a1a262b6SArd Biesheuvel ti->error = "Digest algorithm missing for ESSIV mode"; 2901a1a262b6SArd Biesheuvel return -EINVAL; 2902a1a262b6SArd Biesheuvel } 2903a1a262b6SArd Biesheuvel ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, "essiv(%s,%s)", 2904a1a262b6SArd Biesheuvel cipher_api, *ivopts); 2905a1a262b6SArd Biesheuvel if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) { 2906a1a262b6SArd Biesheuvel ti->error = "Cannot allocate cipher string"; 2907a1a262b6SArd Biesheuvel return -ENOMEM; 2908a1a262b6SArd Biesheuvel } 2909a1a262b6SArd Biesheuvel cipher_api = buf; 2910a1a262b6SArd Biesheuvel } 2911a1a262b6SArd Biesheuvel 291233d2f09fSMilan Broz cc->key_parts = cc->tfms_count; 291333d2f09fSMilan Broz 291433d2f09fSMilan Broz /* Allocate cipher */ 291533d2f09fSMilan Broz ret = crypt_alloc_tfms(cc, cipher_api); 291633d2f09fSMilan Broz if (ret < 0) { 291733d2f09fSMilan Broz ti->error = "Error allocating crypto tfm"; 291833d2f09fSMilan Broz return ret; 291933d2f09fSMilan Broz } 292033d2f09fSMilan Broz 2921a1a262b6SArd Biesheuvel if (crypt_integrity_aead(cc)) 292233d2f09fSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc)); 2923a1a262b6SArd Biesheuvel else 292433d2f09fSMilan Broz cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc)); 292533d2f09fSMilan Broz 292633d2f09fSMilan Broz return 0; 292733d2f09fSMilan Broz } 292833d2f09fSMilan Broz 292933d2f09fSMilan Broz static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key, 293033d2f09fSMilan Broz char **ivmode, char **ivopts) 293133d2f09fSMilan Broz { 293233d2f09fSMilan Broz struct crypt_config *cc = ti->private; 293333d2f09fSMilan Broz char *tmp, *cipher, *chainmode, *keycount; 29345ebaee6dSMilan Broz char *cipher_api = NULL; 29355ebaee6dSMilan Broz int ret = -EINVAL; 29365ebaee6dSMilan Broz char dummy; 29375ebaee6dSMilan Broz 293833d2f09fSMilan Broz if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) { 29395ebaee6dSMilan Broz ti->error = "Bad cipher specification"; 29405ebaee6dSMilan Broz return -EINVAL; 29415ebaee6dSMilan Broz } 29425ebaee6dSMilan Broz 29431da177e4SLinus Torvalds /* 29445ebaee6dSMilan Broz * Legacy dm-crypt cipher specification 29455ebaee6dSMilan Broz * cipher[:keycount]-mode-iv:ivopts 29465ebaee6dSMilan Broz */ 29475ebaee6dSMilan Broz tmp = cipher_in; 29485ebaee6dSMilan Broz keycount = strsep(&tmp, "-"); 29495ebaee6dSMilan Broz cipher = strsep(&keycount, ":"); 29505ebaee6dSMilan Broz 295169a8cfcdSMilan Broz if (!keycount) 29525ebaee6dSMilan Broz cc->tfms_count = 1; 29535ebaee6dSMilan Broz else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 || 29545ebaee6dSMilan Broz !is_power_of_2(cc->tfms_count)) { 29555ebaee6dSMilan Broz ti->error = "Bad cipher key count specification"; 29565ebaee6dSMilan Broz return -EINVAL; 29575ebaee6dSMilan Broz } 295828513fccSMilan Broz cc->key_parts = cc->tfms_count; 29591da177e4SLinus Torvalds 2960ddd42edfSMilan Broz chainmode = strsep(&tmp, "-"); 29611856b9f7SMilan Broz *ivmode = strsep(&tmp, ":"); 29621856b9f7SMilan Broz *ivopts = tmp; 2963ddd42edfSMilan Broz 2964ddd42edfSMilan Broz /* 2965ddd42edfSMilan Broz * For compatibility with the original dm-crypt mapping format, if 2966ddd42edfSMilan Broz * only the cipher name is supplied, use cbc-plain. 296728513fccSMilan Broz */ 296833d2f09fSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) { 2969cabf08e4SMilan Broz chainmode = "cbc"; 297033d2f09fSMilan Broz *ivmode = "plain"; 2971cabf08e4SMilan Broz } 2972cabf08e4SMilan Broz 297333d2f09fSMilan Broz if (strcmp(chainmode, "ecb") && !*ivmode) { 2974c0297721SAndi Kleen ti->error = "IV mechanism required"; 2975c0297721SAndi Kleen return -EINVAL; 2976c0297721SAndi Kleen } 2977c0297721SAndi Kleen 2978cabf08e4SMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL); 29799934a8beSMilan Broz if (!cipher_api) 298028513fccSMilan Broz goto bad_mem; 29819934a8beSMilan Broz 2982a1a262b6SArd Biesheuvel if (*ivmode && !strcmp(*ivmode, "essiv")) { 2983a1a262b6SArd Biesheuvel if (!*ivopts) { 2984a1a262b6SArd Biesheuvel ti->error = "Digest algorithm missing for ESSIV mode"; 2985a1a262b6SArd Biesheuvel kfree(cipher_api); 2986a1a262b6SArd Biesheuvel return -EINVAL; 2987a1a262b6SArd Biesheuvel } 2988a1a262b6SArd Biesheuvel ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 2989a1a262b6SArd Biesheuvel "essiv(%s(%s),%s)", chainmode, cipher, *ivopts); 2990a1a262b6SArd Biesheuvel } else { 29919934a8beSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, 2992647c7db1SMikulas Patocka "%s(%s)", chainmode, cipher); 2993a1a262b6SArd Biesheuvel } 2994a1a262b6SArd Biesheuvel if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) { 29951da177e4SLinus Torvalds kfree(cipher_api); 299628513fccSMilan Broz goto bad_mem; 299728513fccSMilan Broz } 299828513fccSMilan Broz 29991da177e4SLinus Torvalds /* Allocate cipher */ 30001da177e4SLinus Torvalds ret = crypt_alloc_tfms(cc, cipher_api); 30011da177e4SLinus Torvalds if (ret < 0) { 30021da177e4SLinus Torvalds ti->error = "Error allocating crypto tfm"; 300333d2f09fSMilan Broz kfree(cipher_api); 300433d2f09fSMilan Broz return ret; 3005028867acSAlasdair G Kergon } 3006bd86e320SJeffy Chen kfree(cipher_api); 3007647c7db1SMikulas Patocka 300833d2f09fSMilan Broz return 0; 300933d2f09fSMilan Broz bad_mem: 301033d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings"; 301133d2f09fSMilan Broz return -ENOMEM; 301233d2f09fSMilan Broz } 301333d2f09fSMilan Broz 301433d2f09fSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key) 301533d2f09fSMilan Broz { 301633d2f09fSMilan Broz struct crypt_config *cc = ti->private; 301733d2f09fSMilan Broz char *ivmode = NULL, *ivopts = NULL; 301833d2f09fSMilan Broz int ret; 301933d2f09fSMilan Broz 302033d2f09fSMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL); 302133d2f09fSMilan Broz if (!cc->cipher_string) { 302233d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings"; 302333d2f09fSMilan Broz return -ENOMEM; 302433d2f09fSMilan Broz } 302533d2f09fSMilan Broz 302633d2f09fSMilan Broz if (strstarts(cipher_in, "capi:")) 302733d2f09fSMilan Broz ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts); 302833d2f09fSMilan Broz else 302933d2f09fSMilan Broz ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts); 303033d2f09fSMilan Broz if (ret) 303133d2f09fSMilan Broz return ret; 303233d2f09fSMilan Broz 3033647c7db1SMikulas Patocka /* Initialize IV */ 3034e889f97aSMilan Broz ret = crypt_ctr_ivmode(ti, ivmode); 3035e889f97aSMilan Broz if (ret < 0) 303633d2f09fSMilan Broz return ret; 30371da177e4SLinus Torvalds 3038da31a078SMilan Broz /* Initialize and set key */ 3039da31a078SMilan Broz ret = crypt_set_key(cc, key); 3040da31a078SMilan Broz if (ret < 0) { 3041da31a078SMilan Broz ti->error = "Error decoding and setting key"; 304233d2f09fSMilan Broz return ret; 3043da31a078SMilan Broz } 3044da31a078SMilan Broz 30451da177e4SLinus Torvalds /* Allocate IV */ 30461da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) { 30471da177e4SLinus Torvalds ret = cc->iv_gen_ops->ctr(cc, ti, ivopts); 30481da177e4SLinus Torvalds if (ret < 0) { 30491da177e4SLinus Torvalds ti->error = "Error creating IV"; 305033d2f09fSMilan Broz return ret; 30511da177e4SLinus Torvalds } 30521da177e4SLinus Torvalds } 30531da177e4SLinus Torvalds 30541da177e4SLinus Torvalds /* Initialize IV (set keys for ESSIV etc) */ 30551da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->init) { 30561da177e4SLinus Torvalds ret = cc->iv_gen_ops->init(cc); 30571da177e4SLinus Torvalds if (ret < 0) { 30581da177e4SLinus Torvalds ti->error = "Error initialising IV"; 30591da177e4SLinus Torvalds return ret; 30601da177e4SLinus Torvalds } 30611da177e4SLinus Torvalds } 30621da177e4SLinus Torvalds 3063dc94902bSOndrej Kozina /* wipe the kernel key payload copy */ 3064dc94902bSOndrej Kozina if (cc->key_string) 3065dc94902bSOndrej Kozina memset(cc->key, 0, cc->key_size * sizeof(u8)); 3066dc94902bSOndrej Kozina 306733d2f09fSMilan Broz return ret; 30681da177e4SLinus Torvalds } 30691da177e4SLinus Torvalds 3070ef43aa38SMilan Broz static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv) 3071ef43aa38SMilan Broz { 3072ef43aa38SMilan Broz struct crypt_config *cc = ti->private; 3073ef43aa38SMilan Broz struct dm_arg_set as; 30745916a22bSEric Biggers static const struct dm_arg _args[] = { 307539d42fa9SIgnat Korchagin {0, 8, "Invalid number of feature args"}, 3076ef43aa38SMilan Broz }; 3077ef43aa38SMilan Broz unsigned int opt_params, val; 3078ef43aa38SMilan Broz const char *opt_string, *sval; 30798f0009a2SMilan Broz char dummy; 3080ef43aa38SMilan Broz int ret; 3081ef43aa38SMilan Broz 3082ef43aa38SMilan Broz /* Optional parameters */ 3083ef43aa38SMilan Broz as.argc = argc; 3084ef43aa38SMilan Broz as.argv = argv; 3085ef43aa38SMilan Broz 3086ef43aa38SMilan Broz ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 3087ef43aa38SMilan Broz if (ret) 30881da177e4SLinus Torvalds return ret; 30891da177e4SLinus Torvalds 3090ef43aa38SMilan Broz while (opt_params--) { 3091ef43aa38SMilan Broz opt_string = dm_shift_arg(&as); 3092ef43aa38SMilan Broz if (!opt_string) { 3093ef43aa38SMilan Broz ti->error = "Not enough feature arguments"; 3094ef43aa38SMilan Broz return -EINVAL; 3095ef43aa38SMilan Broz } 3096ef43aa38SMilan Broz 3097ef43aa38SMilan Broz if (!strcasecmp(opt_string, "allow_discards")) 3098ef43aa38SMilan Broz ti->num_discard_bios = 1; 3099ef43aa38SMilan Broz 3100ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "same_cpu_crypt")) 3101ef43aa38SMilan Broz set_bit(DM_CRYPT_SAME_CPU, &cc->flags); 3102ef43aa38SMilan Broz 3103ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "submit_from_crypt_cpus")) 3104ef43aa38SMilan Broz set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 310539d42fa9SIgnat Korchagin else if (!strcasecmp(opt_string, "no_read_workqueue")) 310639d42fa9SIgnat Korchagin set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); 310739d42fa9SIgnat Korchagin else if (!strcasecmp(opt_string, "no_write_workqueue")) 310839d42fa9SIgnat Korchagin set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); 3109ef43aa38SMilan Broz else if (sscanf(opt_string, "integrity:%u:", &val) == 1) { 3110ef43aa38SMilan Broz if (val == 0 || val > MAX_TAG_SIZE) { 3111ef43aa38SMilan Broz ti->error = "Invalid integrity arguments"; 3112ef43aa38SMilan Broz return -EINVAL; 3113ef43aa38SMilan Broz } 3114ef43aa38SMilan Broz cc->on_disk_tag_size = val; 3115ef43aa38SMilan Broz sval = strchr(opt_string + strlen("integrity:"), ':') + 1; 3116ef43aa38SMilan Broz if (!strcasecmp(sval, "aead")) { 3117ef43aa38SMilan Broz set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags); 3118ef43aa38SMilan Broz } else if (strcasecmp(sval, "none")) { 3119ef43aa38SMilan Broz ti->error = "Unknown integrity profile"; 3120ef43aa38SMilan Broz return -EINVAL; 3121ef43aa38SMilan Broz } 3122ef43aa38SMilan Broz 3123ef43aa38SMilan Broz cc->cipher_auth = kstrdup(sval, GFP_KERNEL); 3124ef43aa38SMilan Broz if (!cc->cipher_auth) 31251da177e4SLinus Torvalds return -ENOMEM; 3126ff3af92bSMikulas Patocka } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) { 31278f0009a2SMilan Broz if (cc->sector_size < (1 << SECTOR_SHIFT) || 31288f0009a2SMilan Broz cc->sector_size > 4096 || 3129ff3af92bSMikulas Patocka (cc->sector_size & (cc->sector_size - 1))) { 31308f0009a2SMilan Broz ti->error = "Invalid feature value for sector_size"; 31318f0009a2SMilan Broz return -EINVAL; 31328f0009a2SMilan Broz } 3133783874b0SMilan Broz if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) { 3134783874b0SMilan Broz ti->error = "Device size is not multiple of sector_size feature"; 3135783874b0SMilan Broz return -EINVAL; 3136783874b0SMilan Broz } 3137ff3af92bSMikulas Patocka cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT; 31388f0009a2SMilan Broz } else if (!strcasecmp(opt_string, "iv_large_sectors")) 31398f0009a2SMilan Broz set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags); 31408f0009a2SMilan Broz else { 3141ef43aa38SMilan Broz ti->error = "Invalid feature arguments"; 3142ef43aa38SMilan Broz return -EINVAL; 3143ef43aa38SMilan Broz } 3144ef43aa38SMilan Broz } 3145ef43aa38SMilan Broz 3146ef43aa38SMilan Broz return 0; 31471da177e4SLinus Torvalds } 31481da177e4SLinus Torvalds 31498e225f04SDamien Le Moal #ifdef CONFIG_BLK_DEV_ZONED 31508e225f04SDamien Le Moal static int crypt_report_zones(struct dm_target *ti, 31518e225f04SDamien Le Moal struct dm_report_zones_args *args, unsigned int nr_zones) 31528e225f04SDamien Le Moal { 31538e225f04SDamien Le Moal struct crypt_config *cc = ti->private; 31548e225f04SDamien Le Moal 3155912e8875SDamien Le Moal return dm_report_zones(cc->dev->bdev, cc->start, 3156912e8875SDamien Le Moal cc->start + dm_target_offset(ti, args->next_sector), 3157912e8875SDamien Le Moal args, nr_zones); 31588e225f04SDamien Le Moal } 3159e3290b94SMike Snitzer #else 3160e3290b94SMike Snitzer #define crypt_report_zones NULL 31618e225f04SDamien Le Moal #endif 31628e225f04SDamien Le Moal 31631da177e4SLinus Torvalds /* 31641da177e4SLinus Torvalds * Construct an encryption mapping: 3165c538f6ecSOndrej Kozina * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start> 31661da177e4SLinus Torvalds */ 31671da177e4SLinus Torvalds static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 31681da177e4SLinus Torvalds { 31691da177e4SLinus Torvalds struct crypt_config *cc; 3170ed0302e8SMichał Mirosław const char *devname = dm_table_device_name(ti->table); 3171c538f6ecSOndrej Kozina int key_size; 3172ef43aa38SMilan Broz unsigned int align_mask; 31731da177e4SLinus Torvalds unsigned long long tmpll; 31741da177e4SLinus Torvalds int ret; 3175ef43aa38SMilan Broz size_t iv_size_padding, additional_req_size; 317631998ef1SMikulas Patocka char dummy; 31771da177e4SLinus Torvalds 3178772ae5f5SMilan Broz if (argc < 5) { 31791da177e4SLinus Torvalds ti->error = "Not enough arguments"; 31801da177e4SLinus Torvalds return -EINVAL; 31811da177e4SLinus Torvalds } 31821da177e4SLinus Torvalds 3183c538f6ecSOndrej Kozina key_size = get_key_size(&argv[1]); 3184c538f6ecSOndrej Kozina if (key_size < 0) { 3185c538f6ecSOndrej Kozina ti->error = "Cannot parse key size"; 3186c538f6ecSOndrej Kozina return -EINVAL; 3187c538f6ecSOndrej Kozina } 31881da177e4SLinus Torvalds 31899c81c99bSZhengyuan Liu cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL); 31901da177e4SLinus Torvalds if (!cc) { 31911da177e4SLinus Torvalds ti->error = "Cannot allocate encryption context"; 31921da177e4SLinus Torvalds return -ENOMEM; 31931da177e4SLinus Torvalds } 31941da177e4SLinus Torvalds cc->key_size = key_size; 31958f0009a2SMilan Broz cc->sector_size = (1 << SECTOR_SHIFT); 3196ff3af92bSMikulas Patocka cc->sector_shift = 0; 31971da177e4SLinus Torvalds 31981da177e4SLinus Torvalds ti->private = cc; 3199ef43aa38SMilan Broz 32005059353dSMikulas Patocka spin_lock(&dm_crypt_clients_lock); 32015059353dSMikulas Patocka dm_crypt_clients_n++; 32025059353dSMikulas Patocka crypt_calculate_pages_per_client(); 32035059353dSMikulas Patocka spin_unlock(&dm_crypt_clients_lock); 32045059353dSMikulas Patocka 32055059353dSMikulas Patocka ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL); 32065059353dSMikulas Patocka if (ret < 0) 32075059353dSMikulas Patocka goto bad; 32085059353dSMikulas Patocka 3209ef43aa38SMilan Broz /* Optional parameters need to be read before cipher constructor */ 3210ef43aa38SMilan Broz if (argc > 5) { 3211ef43aa38SMilan Broz ret = crypt_ctr_optional(ti, argc - 5, &argv[5]); 3212ef43aa38SMilan Broz if (ret) 3213ef43aa38SMilan Broz goto bad; 3214ef43aa38SMilan Broz } 3215ef43aa38SMilan Broz 32161da177e4SLinus Torvalds ret = crypt_ctr_cipher(ti, argv[0], argv[1]); 32171da177e4SLinus Torvalds if (ret < 0) 32181da177e4SLinus Torvalds goto bad; 32191da177e4SLinus Torvalds 322033d2f09fSMilan Broz if (crypt_integrity_aead(cc)) { 3221ef43aa38SMilan Broz cc->dmreq_start = sizeof(struct aead_request); 3222ef43aa38SMilan Broz cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc)); 3223ef43aa38SMilan Broz align_mask = crypto_aead_alignmask(any_tfm_aead(cc)); 3224ef43aa38SMilan Broz } else { 3225bbdb23b5SHerbert Xu cc->dmreq_start = sizeof(struct skcipher_request); 3226bbdb23b5SHerbert Xu cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc)); 3227ef43aa38SMilan Broz align_mask = crypto_skcipher_alignmask(any_tfm(cc)); 3228ef43aa38SMilan Broz } 3229d49ec52fSMikulas Patocka cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request)); 3230d49ec52fSMikulas Patocka 3231ef43aa38SMilan Broz if (align_mask < CRYPTO_MINALIGN) { 3232d49ec52fSMikulas Patocka /* Allocate the padding exactly */ 3233d49ec52fSMikulas Patocka iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request)) 3234ef43aa38SMilan Broz & align_mask; 3235d49ec52fSMikulas Patocka } else { 3236d49ec52fSMikulas Patocka /* 3237d49ec52fSMikulas Patocka * If the cipher requires greater alignment than kmalloc 3238d49ec52fSMikulas Patocka * alignment, we don't know the exact position of the 3239d49ec52fSMikulas Patocka * initialization vector. We must assume worst case. 3240d49ec52fSMikulas Patocka */ 3241ef43aa38SMilan Broz iv_size_padding = align_mask; 3242d49ec52fSMikulas Patocka } 32431da177e4SLinus Torvalds 3244ef43aa38SMilan Broz /* ...| IV + padding | original IV | original sec. number | bio tag offset | */ 3245ef43aa38SMilan Broz additional_req_size = sizeof(struct dm_crypt_request) + 3246ef43aa38SMilan Broz iv_size_padding + cc->iv_size + 3247ef43aa38SMilan Broz cc->iv_size + 3248ef43aa38SMilan Broz sizeof(uint64_t) + 3249ef43aa38SMilan Broz sizeof(unsigned int); 3250ef43aa38SMilan Broz 32516f1c819cSKent Overstreet ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size); 32526f1c819cSKent Overstreet if (ret) { 32531da177e4SLinus Torvalds ti->error = "Cannot allocate crypt request mempool"; 32541da177e4SLinus Torvalds goto bad; 32551da177e4SLinus Torvalds } 32561da177e4SLinus Torvalds 325730187e1dSMike Snitzer cc->per_bio_data_size = ti->per_io_data_size = 3258ef43aa38SMilan Broz ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size, 3259d49ec52fSMikulas Patocka ARCH_KMALLOC_MINALIGN); 3260298a9fa0SMikulas Patocka 3261a8affc03SChristoph Hellwig ret = mempool_init(&cc->page_pool, BIO_MAX_VECS, crypt_page_alloc, crypt_page_free, cc); 32626f1c819cSKent Overstreet if (ret) { 32638b004457SMilan Broz ti->error = "Cannot allocate page mempool"; 3264e48d4bbfSMilan Broz goto bad; 32651da177e4SLinus Torvalds } 3266e48d4bbfSMilan Broz 32676f1c819cSKent Overstreet ret = bioset_init(&cc->bs, MIN_IOS, 0, BIOSET_NEED_BVECS); 32686f1c819cSKent Overstreet if (ret) { 32690c395b0fSMilan Broz ti->error = "Cannot allocate crypt bioset"; 3270cabf08e4SMilan Broz goto bad; 327193e605c2SMilan Broz } 3272cabf08e4SMilan Broz 32737145c241SMikulas Patocka mutex_init(&cc->bio_alloc_lock); 32747145c241SMikulas Patocka 3275cabf08e4SMilan Broz ret = -EINVAL; 32768f0009a2SMilan Broz if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) || 32778f0009a2SMilan Broz (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) { 3278cabf08e4SMilan Broz ti->error = "Invalid iv_offset sector"; 3279cabf08e4SMilan Broz goto bad; 32801da177e4SLinus Torvalds } 3281d2a7ad29SKiyoshi Ueda cc->iv_offset = tmpll; 32821da177e4SLinus Torvalds 3283e80d1c80SVivek Goyal ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev); 3284e80d1c80SVivek Goyal if (ret) { 32851da177e4SLinus Torvalds ti->error = "Device lookup failed"; 32861da177e4SLinus Torvalds goto bad; 32871da177e4SLinus Torvalds } 32881da177e4SLinus Torvalds 3289e80d1c80SVivek Goyal ret = -EINVAL; 3290ef87bfc2SMilan Broz if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) { 32911da177e4SLinus Torvalds ti->error = "Invalid device sector"; 32921da177e4SLinus Torvalds goto bad; 32931da177e4SLinus Torvalds } 32941da177e4SLinus Torvalds cc->start = tmpll; 32951da177e4SLinus Torvalds 3296f34ee1dcSDamien Le Moal if (bdev_is_zoned(cc->dev->bdev)) { 32978e225f04SDamien Le Moal /* 32988e225f04SDamien Le Moal * For zoned block devices, we need to preserve the issuer write 32998e225f04SDamien Le Moal * ordering. To do so, disable write workqueues and force inline 33008e225f04SDamien Le Moal * encryption completion. 33018e225f04SDamien Le Moal */ 33028e225f04SDamien Le Moal set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); 33038e225f04SDamien Le Moal set_bit(DM_CRYPT_WRITE_INLINE, &cc->flags); 3304f34ee1dcSDamien Le Moal 3305f34ee1dcSDamien Le Moal /* 3306f34ee1dcSDamien Le Moal * All zone append writes to a zone of a zoned block device will 3307f34ee1dcSDamien Le Moal * have the same BIO sector, the start of the zone. When the 3308f34ee1dcSDamien Le Moal * cypher IV mode uses sector values, all data targeting a 3309f34ee1dcSDamien Le Moal * zone will be encrypted using the first sector numbers of the 3310f34ee1dcSDamien Le Moal * zone. This will not result in write errors but will 3311f34ee1dcSDamien Le Moal * cause most reads to fail as reads will use the sector values 3312f34ee1dcSDamien Le Moal * for the actual data locations, resulting in IV mismatch. 3313f34ee1dcSDamien Le Moal * To avoid this problem, ask DM core to emulate zone append 3314f34ee1dcSDamien Le Moal * operations with regular writes. 3315f34ee1dcSDamien Le Moal */ 3316f34ee1dcSDamien Le Moal DMDEBUG("Zone append operations will be emulated"); 3317f34ee1dcSDamien Le Moal ti->emulate_zone_append = true; 33188e225f04SDamien Le Moal } 33198e225f04SDamien Le Moal 332033d2f09fSMilan Broz if (crypt_integrity_aead(cc) || cc->integrity_iv_size) { 3321ef43aa38SMilan Broz ret = crypt_integrity_ctr(cc, ti); 3322772ae5f5SMilan Broz if (ret) 3323772ae5f5SMilan Broz goto bad; 3324772ae5f5SMilan Broz 3325ef43aa38SMilan Broz cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size; 3326ef43aa38SMilan Broz if (!cc->tag_pool_max_sectors) 3327ef43aa38SMilan Broz cc->tag_pool_max_sectors = 1; 3328ef43aa38SMilan Broz 33296f1c819cSKent Overstreet ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS, 3330ef43aa38SMilan Broz cc->tag_pool_max_sectors * cc->on_disk_tag_size); 33316f1c819cSKent Overstreet if (ret) { 3332ef43aa38SMilan Broz ti->error = "Cannot allocate integrity tags mempool"; 3333f3396c58SMikulas Patocka goto bad; 3334f3396c58SMikulas Patocka } 3335772ae5f5SMilan Broz 3336583fe747SMikulas Patocka cc->tag_pool_max_sectors <<= cc->sector_shift; 3337f3396c58SMikulas Patocka } 3338772ae5f5SMilan Broz 33391da177e4SLinus Torvalds ret = -ENOMEM; 3340f612b213SMike Snitzer cc->io_queue = alloc_workqueue("kcryptd_io/%s", WQ_MEM_RECLAIM, 1, devname); 33411da177e4SLinus Torvalds if (!cc->io_queue) { 33421da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd io queue"; 33431da177e4SLinus Torvalds goto bad; 33441da177e4SLinus Torvalds } 334537af6560SChristophe Saout 3346f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 334748b0777cSMike Snitzer cc->crypt_queue = alloc_workqueue("kcryptd/%s", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 3348ed0302e8SMichał Mirosław 1, devname); 3349f3396c58SMikulas Patocka else 335048b0777cSMike Snitzer cc->crypt_queue = alloc_workqueue("kcryptd/%s", 335148b0777cSMike Snitzer WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, 3352ed0302e8SMichał Mirosław num_online_cpus(), devname); 33531da177e4SLinus Torvalds if (!cc->crypt_queue) { 33541da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd queue"; 33551da177e4SLinus Torvalds goto bad; 33561da177e4SLinus Torvalds } 33571da177e4SLinus Torvalds 3358c7329effSMikulas Patocka spin_lock_init(&cc->write_thread_lock); 3359b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT; 3360dc267621SMikulas Patocka 3361a5217c11SCai Huoqing cc->write_thread = kthread_run(dmcrypt_write, cc, "dmcrypt_write/%s", devname); 3362dc267621SMikulas Patocka if (IS_ERR(cc->write_thread)) { 3363dc267621SMikulas Patocka ret = PTR_ERR(cc->write_thread); 3364dc267621SMikulas Patocka cc->write_thread = NULL; 3365dc267621SMikulas Patocka ti->error = "Couldn't spawn write thread"; 3366dc267621SMikulas Patocka goto bad; 3367dc267621SMikulas Patocka } 3368dc267621SMikulas Patocka 336955a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 3370a666e5c0SMikulas Patocka ti->limit_swap_bios = true; 3371e5524e12SMike Snitzer ti->accounts_remapped_io = true; 3372983c7db3SMilan Broz 337358d0f180SMichael Weiß dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); 33741da177e4SLinus Torvalds return 0; 33751da177e4SLinus Torvalds 33761da177e4SLinus Torvalds bad: 337758d0f180SMichael Weiß dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); 33781da177e4SLinus Torvalds crypt_dtr(ti); 33791da177e4SLinus Torvalds return ret; 3380647c7db1SMikulas Patocka } 3381647c7db1SMikulas Patocka 33827de3ee57SMikulas Patocka static int crypt_map(struct dm_target *ti, struct bio *bio) 33831da177e4SLinus Torvalds { 33841da177e4SLinus Torvalds struct dm_crypt_io *io; 338549a8a920SAlasdair G Kergon struct crypt_config *cc = ti->private; 3386647c7db1SMikulas Patocka 3387772ae5f5SMilan Broz /* 338828a8f0d3SMike Christie * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues. 338928a8f0d3SMike Christie * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight 3390e6047149SMike Christie * - for REQ_OP_DISCARD caller must use flush if IO ordering matters 3391772ae5f5SMilan Broz */ 33921eff9d32SJens Axboe if (unlikely(bio->bi_opf & REQ_PREFLUSH || 339328a8f0d3SMike Christie bio_op(bio) == REQ_OP_DISCARD)) { 339474d46992SChristoph Hellwig bio_set_dev(bio, cc->dev->bdev); 3395772ae5f5SMilan Broz if (bio_sectors(bio)) 33964f024f37SKent Overstreet bio->bi_iter.bi_sector = cc->start + 33974f024f37SKent Overstreet dm_target_offset(ti, bio->bi_iter.bi_sector); 3398647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED; 3399647c7db1SMikulas Patocka } 34001da177e4SLinus Torvalds 34014e870e94SMikulas Patocka /* 34024e870e94SMikulas Patocka * Check if bio is too large, split as needed. 34034e870e94SMikulas Patocka */ 3404a8affc03SChristoph Hellwig if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_VECS << PAGE_SHIFT)) && 3405ef43aa38SMilan Broz (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size)) 3406a8affc03SChristoph Hellwig dm_accept_partial_bio(bio, ((BIO_MAX_VECS << PAGE_SHIFT) >> SECTOR_SHIFT)); 34074e870e94SMikulas Patocka 34088f0009a2SMilan Broz /* 34098f0009a2SMilan Broz * Ensure that bio is a multiple of internal sector encryption size 34108f0009a2SMilan Broz * and is aligned to this size as defined in IO hints. 34118f0009a2SMilan Broz */ 34128f0009a2SMilan Broz if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0)) 3413846785e6SChristoph Hellwig return DM_MAPIO_KILL; 34148f0009a2SMilan Broz 34158f0009a2SMilan Broz if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1))) 3416846785e6SChristoph Hellwig return DM_MAPIO_KILL; 34178f0009a2SMilan Broz 3418298a9fa0SMikulas Patocka io = dm_per_bio_data(bio, cc->per_bio_data_size); 3419298a9fa0SMikulas Patocka crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector)); 3420ef43aa38SMilan Broz 3421ef43aa38SMilan Broz if (cc->on_disk_tag_size) { 342286a3238cSHeinz Mauelshagen unsigned int tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift); 3423ef43aa38SMilan Broz 3424d715fa23SHeinz Mauelshagen if (unlikely(tag_len > KMALLOC_MAX_SIZE)) 3425d715fa23SHeinz Mauelshagen io->integrity_metadata = NULL; 3426d715fa23SHeinz Mauelshagen else 3427d715fa23SHeinz Mauelshagen io->integrity_metadata = kmalloc(tag_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 3428d715fa23SHeinz Mauelshagen 3429d715fa23SHeinz Mauelshagen if (unlikely(!io->integrity_metadata)) { 3430ef43aa38SMilan Broz if (bio_sectors(bio) > cc->tag_pool_max_sectors) 3431ef43aa38SMilan Broz dm_accept_partial_bio(bio, cc->tag_pool_max_sectors); 34326f1c819cSKent Overstreet io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO); 3433ef43aa38SMilan Broz io->integrity_metadata_from_pool = true; 3434ef43aa38SMilan Broz } 3435ef43aa38SMilan Broz } 3436ef43aa38SMilan Broz 343733d2f09fSMilan Broz if (crypt_integrity_aead(cc)) 3438ef43aa38SMilan Broz io->ctx.r.req_aead = (struct aead_request *)(io + 1); 3439ef43aa38SMilan Broz else 3440ef43aa38SMilan Broz io->ctx.r.req = (struct skcipher_request *)(io + 1); 34411da177e4SLinus Torvalds 344220c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) { 3443e5524e12SMike Snitzer if (kcryptd_io_read(io, CRYPT_MAP_READ_GFP)) 3444dc267621SMikulas Patocka kcryptd_queue_read(io); 344520c82538SMilan Broz } else 34464ee218cdSAndrew Morton kcryptd_queue_crypt(io); 34474ee218cdSAndrew Morton 34481da177e4SLinus Torvalds return DM_MAPIO_SUBMITTED; 34491da177e4SLinus Torvalds } 34501da177e4SLinus Torvalds 3451567dd8f3SMikulas Patocka static char hex2asc(unsigned char c) 3452567dd8f3SMikulas Patocka { 345386a3238cSHeinz Mauelshagen return c + '0' + ((unsigned int)(9 - c) >> 4 & 0x27); 3454567dd8f3SMikulas Patocka } 3455567dd8f3SMikulas Patocka 3456fd7c092eSMikulas Patocka static void crypt_status(struct dm_target *ti, status_type_t type, 345786a3238cSHeinz Mauelshagen unsigned int status_flags, char *result, unsigned int maxlen) 34581da177e4SLinus Torvalds { 34595ebaee6dSMilan Broz struct crypt_config *cc = ti->private; 346086a3238cSHeinz Mauelshagen unsigned int i, sz = 0; 3461f3396c58SMikulas Patocka int num_feature_args = 0; 34621da177e4SLinus Torvalds 34631da177e4SLinus Torvalds switch (type) { 34641da177e4SLinus Torvalds case STATUSTYPE_INFO: 34651da177e4SLinus Torvalds result[0] = '\0'; 34661da177e4SLinus Torvalds break; 34671da177e4SLinus Torvalds 34681da177e4SLinus Torvalds case STATUSTYPE_TABLE: 34697dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string); 34701da177e4SLinus Torvalds 3471c538f6ecSOndrej Kozina if (cc->key_size > 0) { 3472c538f6ecSOndrej Kozina if (cc->key_string) 3473c538f6ecSOndrej Kozina DMEMIT(":%u:%s", cc->key_size, cc->key_string); 3474567dd8f3SMikulas Patocka else { 3475567dd8f3SMikulas Patocka for (i = 0; i < cc->key_size; i++) { 3476567dd8f3SMikulas Patocka DMEMIT("%c%c", hex2asc(cc->key[i] >> 4), 3477567dd8f3SMikulas Patocka hex2asc(cc->key[i] & 0xf)); 3478567dd8f3SMikulas Patocka } 3479567dd8f3SMikulas Patocka } 3480c538f6ecSOndrej Kozina } else 3481fd7c092eSMikulas Patocka DMEMIT("-"); 34821da177e4SLinus Torvalds 34831da177e4SLinus Torvalds DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 34841da177e4SLinus Torvalds cc->dev->name, (unsigned long long)cc->start); 3485772ae5f5SMilan Broz 3486f3396c58SMikulas Patocka num_feature_args += !!ti->num_discard_bios; 3487f3396c58SMikulas Patocka num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags); 34880f5d8e6eSMikulas Patocka num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 348939d42fa9SIgnat Korchagin num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); 349039d42fa9SIgnat Korchagin num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); 3491ff3af92bSMikulas Patocka num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT); 34928f0009a2SMilan Broz num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags); 3493ef43aa38SMilan Broz if (cc->on_disk_tag_size) 3494ef43aa38SMilan Broz num_feature_args++; 3495f3396c58SMikulas Patocka if (num_feature_args) { 3496f3396c58SMikulas Patocka DMEMIT(" %d", num_feature_args); 349755a62eefSAlasdair G Kergon if (ti->num_discard_bios) 3498f3396c58SMikulas Patocka DMEMIT(" allow_discards"); 3499f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 3500f3396c58SMikulas Patocka DMEMIT(" same_cpu_crypt"); 35010f5d8e6eSMikulas Patocka if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) 35020f5d8e6eSMikulas Patocka DMEMIT(" submit_from_crypt_cpus"); 350339d42fa9SIgnat Korchagin if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) 350439d42fa9SIgnat Korchagin DMEMIT(" no_read_workqueue"); 350539d42fa9SIgnat Korchagin if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) 350639d42fa9SIgnat Korchagin DMEMIT(" no_write_workqueue"); 3507ef43aa38SMilan Broz if (cc->on_disk_tag_size) 3508ef43aa38SMilan Broz DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth); 35098f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) 35108f0009a2SMilan Broz DMEMIT(" sector_size:%d", cc->sector_size); 35118f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) 35128f0009a2SMilan Broz DMEMIT(" iv_large_sectors"); 3513f3396c58SMikulas Patocka } 35148ec45662STushar Sugandhi break; 3515772ae5f5SMilan Broz 35168ec45662STushar Sugandhi case STATUSTYPE_IMA: 35178ec45662STushar Sugandhi DMEMIT_TARGET_NAME_VERSION(ti->type); 35188ec45662STushar Sugandhi DMEMIT(",allow_discards=%c", ti->num_discard_bios ? 'y' : 'n'); 35198ec45662STushar Sugandhi DMEMIT(",same_cpu_crypt=%c", test_bit(DM_CRYPT_SAME_CPU, &cc->flags) ? 'y' : 'n'); 35208ec45662STushar Sugandhi DMEMIT(",submit_from_crypt_cpus=%c", test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) ? 35218ec45662STushar Sugandhi 'y' : 'n'); 35228ec45662STushar Sugandhi DMEMIT(",no_read_workqueue=%c", test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags) ? 35238ec45662STushar Sugandhi 'y' : 'n'); 35248ec45662STushar Sugandhi DMEMIT(",no_write_workqueue=%c", test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags) ? 35258ec45662STushar Sugandhi 'y' : 'n'); 35268ec45662STushar Sugandhi DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ? 35278ec45662STushar Sugandhi 'y' : 'n'); 35288ec45662STushar Sugandhi 35298ec45662STushar Sugandhi if (cc->on_disk_tag_size) 35308ec45662STushar Sugandhi DMEMIT(",integrity_tag_size=%u,cipher_auth=%s", 35318ec45662STushar Sugandhi cc->on_disk_tag_size, cc->cipher_auth); 35328ec45662STushar Sugandhi if (cc->sector_size != (1 << SECTOR_SHIFT)) 35338ec45662STushar Sugandhi DMEMIT(",sector_size=%d", cc->sector_size); 35348ec45662STushar Sugandhi if (cc->cipher_string) 35358ec45662STushar Sugandhi DMEMIT(",cipher_string=%s", cc->cipher_string); 35368ec45662STushar Sugandhi 35378ec45662STushar Sugandhi DMEMIT(",key_size=%u", cc->key_size); 35388ec45662STushar Sugandhi DMEMIT(",key_parts=%u", cc->key_parts); 35398ec45662STushar Sugandhi DMEMIT(",key_extra_size=%u", cc->key_extra_size); 35408ec45662STushar Sugandhi DMEMIT(",key_mac_size=%u", cc->key_mac_size); 35418ec45662STushar Sugandhi DMEMIT(";"); 35421da177e4SLinus Torvalds break; 35431da177e4SLinus Torvalds } 35441da177e4SLinus Torvalds } 35451da177e4SLinus Torvalds 3546e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti) 3547e48d4bbfSMilan Broz { 3548e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3549e48d4bbfSMilan Broz 3550e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 3551e48d4bbfSMilan Broz } 3552e48d4bbfSMilan Broz 3553e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti) 3554e48d4bbfSMilan Broz { 3555e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3556e48d4bbfSMilan Broz 3557e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 3558e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set."); 3559e48d4bbfSMilan Broz return -EAGAIN; 3560e48d4bbfSMilan Broz } 3561e48d4bbfSMilan Broz 3562e48d4bbfSMilan Broz return 0; 3563e48d4bbfSMilan Broz } 3564e48d4bbfSMilan Broz 3565e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti) 3566e48d4bbfSMilan Broz { 3567e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3568e48d4bbfSMilan Broz 3569e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 3570e48d4bbfSMilan Broz } 3571e48d4bbfSMilan Broz 3572e48d4bbfSMilan Broz /* Message interface 3573e48d4bbfSMilan Broz * key set <key> 3574e48d4bbfSMilan Broz * key wipe 3575e48d4bbfSMilan Broz */ 357686a3238cSHeinz Mauelshagen static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv, 357786a3238cSHeinz Mauelshagen char *result, unsigned int maxlen) 3578e48d4bbfSMilan Broz { 3579e48d4bbfSMilan Broz struct crypt_config *cc = ti->private; 3580c538f6ecSOndrej Kozina int key_size, ret = -EINVAL; 3581e48d4bbfSMilan Broz 3582e48d4bbfSMilan Broz if (argc < 2) 3583e48d4bbfSMilan Broz goto error; 3584e48d4bbfSMilan Broz 3585498f0103SMike Snitzer if (!strcasecmp(argv[0], "key")) { 3586e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 3587e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation."); 3588e48d4bbfSMilan Broz return -EINVAL; 3589e48d4bbfSMilan Broz } 3590498f0103SMike Snitzer if (argc == 3 && !strcasecmp(argv[1], "set")) { 3591c538f6ecSOndrej Kozina /* The key size may not be changed. */ 3592c538f6ecSOndrej Kozina key_size = get_key_size(&argv[2]); 3593c538f6ecSOndrej Kozina if (key_size < 0 || cc->key_size != key_size) { 3594c538f6ecSOndrej Kozina memset(argv[2], '0', strlen(argv[2])); 3595c538f6ecSOndrej Kozina return -EINVAL; 3596c538f6ecSOndrej Kozina } 3597c538f6ecSOndrej Kozina 3598542da317SMilan Broz ret = crypt_set_key(cc, argv[2]); 3599542da317SMilan Broz if (ret) 3600542da317SMilan Broz return ret; 3601542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init) 3602542da317SMilan Broz ret = cc->iv_gen_ops->init(cc); 3603dc94902bSOndrej Kozina /* wipe the kernel key payload copy */ 3604dc94902bSOndrej Kozina if (cc->key_string) 3605dc94902bSOndrej Kozina memset(cc->key, 0, cc->key_size * sizeof(u8)); 3606542da317SMilan Broz return ret; 3607542da317SMilan Broz } 36084a52ffc7SMilan Broz if (argc == 2 && !strcasecmp(argv[1], "wipe")) 3609e48d4bbfSMilan Broz return crypt_wipe_key(cc); 3610e48d4bbfSMilan Broz } 3611e48d4bbfSMilan Broz 3612e48d4bbfSMilan Broz error: 3613e48d4bbfSMilan Broz DMWARN("unrecognised message received."); 3614e48d4bbfSMilan Broz return -EINVAL; 3615e48d4bbfSMilan Broz } 3616e48d4bbfSMilan Broz 3617af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti, 3618af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data) 3619af4874e0SMike Snitzer { 3620af4874e0SMike Snitzer struct crypt_config *cc = ti->private; 3621af4874e0SMike Snitzer 36225dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data); 3623af4874e0SMike Snitzer } 3624af4874e0SMike Snitzer 3625586b286bSMike Snitzer static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits) 3626586b286bSMike Snitzer { 36278f0009a2SMilan Broz struct crypt_config *cc = ti->private; 36288f0009a2SMilan Broz 3629586b286bSMike Snitzer /* 3630586b286bSMike Snitzer * Unfortunate constraint that is required to avoid the potential 3631586b286bSMike Snitzer * for exceeding underlying device's max_segments limits -- due to 3632586b286bSMike Snitzer * crypt_alloc_buffer() possibly allocating pages for the encryption 3633586b286bSMike Snitzer * bio that are not as physically contiguous as the original bio. 3634586b286bSMike Snitzer */ 3635586b286bSMike Snitzer limits->max_segment_size = PAGE_SIZE; 36368f0009a2SMilan Broz 3637bc9e9cf0SMikulas Patocka limits->logical_block_size = 363886a3238cSHeinz Mauelshagen max_t(unsigned int, limits->logical_block_size, cc->sector_size); 3639bc9e9cf0SMikulas Patocka limits->physical_block_size = 364086a3238cSHeinz Mauelshagen max_t(unsigned int, limits->physical_block_size, cc->sector_size); 364186a3238cSHeinz Mauelshagen limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size); 364286e4d3e8SKeith Busch limits->dma_alignment = limits->logical_block_size - 1; 3643586b286bSMike Snitzer } 3644586b286bSMike Snitzer 36451da177e4SLinus Torvalds static struct target_type crypt_target = { 36461da177e4SLinus Torvalds .name = "crypt", 3647e5524e12SMike Snitzer .version = {1, 24, 0}, 36481da177e4SLinus Torvalds .module = THIS_MODULE, 36491da177e4SLinus Torvalds .ctr = crypt_ctr, 36501da177e4SLinus Torvalds .dtr = crypt_dtr, 36518e225f04SDamien Le Moal .features = DM_TARGET_ZONED_HM, 36528e225f04SDamien Le Moal .report_zones = crypt_report_zones, 36531da177e4SLinus Torvalds .map = crypt_map, 36541da177e4SLinus Torvalds .status = crypt_status, 3655e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend, 3656e48d4bbfSMilan Broz .preresume = crypt_preresume, 3657e48d4bbfSMilan Broz .resume = crypt_resume, 3658e48d4bbfSMilan Broz .message = crypt_message, 3659af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices, 3660586b286bSMike Snitzer .io_hints = crypt_io_hints, 36611da177e4SLinus Torvalds }; 3662*3664ff82SYangtao Li module_dm(crypt); 36631da177e4SLinus Torvalds 3664bf14299fSJana Saout MODULE_AUTHOR("Jana Saout <jana@saout.de>"); 36651da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 36661da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 3667