13bd94003SHeinz Mauelshagen // SPDX-License-Identifier: GPL-2.0-only
21da177e4SLinus Torvalds /*
3bf14299fSJana Saout * Copyright (C) 2003 Jana Saout <jana@saout.de>
41da177e4SLinus Torvalds * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
5bbb16584SMilan Broz * Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved.
6bbb16584SMilan Broz * Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com>
71da177e4SLinus Torvalds *
81da177e4SLinus Torvalds * This file is released under the GPL.
91da177e4SLinus Torvalds */
101da177e4SLinus Torvalds
1143d69034SMilan Broz #include <linux/completion.h>
12d1806f6aSHerbert Xu #include <linux/err.h>
131da177e4SLinus Torvalds #include <linux/module.h>
141da177e4SLinus Torvalds #include <linux/init.h>
151da177e4SLinus Torvalds #include <linux/kernel.h>
16c538f6ecSOndrej Kozina #include <linux/key.h>
171da177e4SLinus Torvalds #include <linux/bio.h>
181da177e4SLinus Torvalds #include <linux/blkdev.h>
19fe45e630SChristoph Hellwig #include <linux/blk-integrity.h>
201da177e4SLinus Torvalds #include <linux/mempool.h>
211da177e4SLinus Torvalds #include <linux/slab.h>
221da177e4SLinus Torvalds #include <linux/crypto.h>
231da177e4SLinus Torvalds #include <linux/workqueue.h>
24dc267621SMikulas Patocka #include <linux/kthread.h>
253fcfab16SAndrew Morton #include <linux/backing-dev.h>
2660063497SArun Sharma #include <linux/atomic.h>
27378f058cSDavid Hardeman #include <linux/scatterlist.h>
28b3c5fd30SMikulas Patocka #include <linux/rbtree.h>
29027c431cSOndrej Kozina #include <linux/ctype.h>
301da177e4SLinus Torvalds #include <asm/page.h>
3148527fa7SRik Snel #include <asm/unaligned.h>
3234745785SMilan Broz #include <crypto/hash.h>
3334745785SMilan Broz #include <crypto/md5.h>
34bbdb23b5SHerbert Xu #include <crypto/skcipher.h>
35ef43aa38SMilan Broz #include <crypto/aead.h>
36ef43aa38SMilan Broz #include <crypto/authenc.h>
37e3023094SHerbert Xu #include <crypto/utils.h>
38ef43aa38SMilan Broz #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
3927f5411aSDmitry Baryshkov #include <linux/key-type.h>
40c538f6ecSOndrej Kozina #include <keys/user-type.h>
4127f5411aSDmitry Baryshkov #include <keys/encrypted-type.h>
42363880c4SAhmad Fatoum #include <keys/trusted-type.h>
431da177e4SLinus Torvalds
44586e80e6SMikulas Patocka #include <linux/device-mapper.h>
451da177e4SLinus Torvalds
4658d0f180SMichael Weiß #include "dm-audit.h"
4758d0f180SMichael Weiß
4872d94861SAlasdair G Kergon #define DM_MSG_PREFIX "crypt"
491da177e4SLinus Torvalds
501da177e4SLinus Torvalds /*
511da177e4SLinus Torvalds * context holding the current state of a multi-part conversion
521da177e4SLinus Torvalds */
531da177e4SLinus Torvalds struct convert_context {
5443d69034SMilan Broz struct completion restart;
551da177e4SLinus Torvalds struct bio *bio_in;
56003b5c57SKent Overstreet struct bvec_iter iter_in;
57*978698aaSMikulas Patocka struct bio *bio_out;
58003b5c57SKent Overstreet struct bvec_iter iter_out;
5940b6229bSMikulas Patocka atomic_t cc_pending;
60*978698aaSMikulas Patocka u64 cc_sector;
61ef43aa38SMilan Broz union {
62bbdb23b5SHerbert Xu struct skcipher_request *req;
63ef43aa38SMilan Broz struct aead_request *req_aead;
64ef43aa38SMilan Broz } r;
650f6cf136SMikulas Patocka bool aead_recheck;
660f6cf136SMikulas Patocka bool aead_failed;
67ef43aa38SMilan Broz
681da177e4SLinus Torvalds };
691da177e4SLinus Torvalds
7053017030SMilan Broz /*
7153017030SMilan Broz * per bio private data
7253017030SMilan Broz */
7353017030SMilan Broz struct dm_crypt_io {
7449a8a920SAlasdair G Kergon struct crypt_config *cc;
7553017030SMilan Broz struct bio *base_bio;
76ef43aa38SMilan Broz u8 *integrity_metadata;
77d9a02e01SMike Snitzer bool integrity_metadata_from_pool:1;
78d9a02e01SMike Snitzer
7953017030SMilan Broz struct work_struct work;
8053017030SMilan Broz
8153017030SMilan Broz struct convert_context ctx;
8253017030SMilan Broz
8340b6229bSMikulas Patocka atomic_t io_pending;
844e4cbee9SChristoph Hellwig blk_status_t error;
850c395b0fSMilan Broz sector_t sector;
86dc267621SMikulas Patocka
870f6cf136SMikulas Patocka struct bvec_iter saved_bi_iter;
880f6cf136SMikulas Patocka
89b3c5fd30SMikulas Patocka struct rb_node rb_node;
90298a9fa0SMikulas Patocka } CRYPTO_MINALIGN_ATTR;
9153017030SMilan Broz
9201482b76SMilan Broz struct dm_crypt_request {
93b2174eebSHuang Ying struct convert_context *ctx;
94ef43aa38SMilan Broz struct scatterlist sg_in[4];
95ef43aa38SMilan Broz struct scatterlist sg_out[4];
968d683dcdSAliOS system security u64 iv_sector;
9701482b76SMilan Broz };
9801482b76SMilan Broz
991da177e4SLinus Torvalds struct crypt_config;
1001da177e4SLinus Torvalds
1011da177e4SLinus Torvalds struct crypt_iv_operations {
1021da177e4SLinus Torvalds int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
1031da177e4SLinus Torvalds const char *opts);
1041da177e4SLinus Torvalds void (*dtr)(struct crypt_config *cc);
105b95bf2d3SMilan Broz int (*init)(struct crypt_config *cc);
106542da317SMilan Broz int (*wipe)(struct crypt_config *cc);
1072dc5327dSMilan Broz int (*generator)(struct crypt_config *cc, u8 *iv,
1082dc5327dSMilan Broz struct dm_crypt_request *dmreq);
1092dc5327dSMilan Broz int (*post)(struct crypt_config *cc, u8 *iv,
1102dc5327dSMilan Broz struct dm_crypt_request *dmreq);
1111da177e4SLinus Torvalds };
1121da177e4SLinus Torvalds
11360473592SMilan Broz struct iv_benbi_private {
11460473592SMilan Broz int shift;
11560473592SMilan Broz };
11660473592SMilan Broz
11734745785SMilan Broz #define LMK_SEED_SIZE 64 /* hash + 0 */
11834745785SMilan Broz struct iv_lmk_private {
11934745785SMilan Broz struct crypto_shash *hash_tfm;
12034745785SMilan Broz u8 *seed;
12134745785SMilan Broz };
12234745785SMilan Broz
123ed04d981SMilan Broz #define TCW_WHITENING_SIZE 16
124ed04d981SMilan Broz struct iv_tcw_private {
125ed04d981SMilan Broz struct crypto_shash *crc32_tfm;
126ed04d981SMilan Broz u8 *iv_seed;
127ed04d981SMilan Broz u8 *whitening;
128ed04d981SMilan Broz };
129ed04d981SMilan Broz
130bbb16584SMilan Broz #define ELEPHANT_MAX_KEY_SIZE 32
131bbb16584SMilan Broz struct iv_elephant_private {
132bbb16584SMilan Broz struct crypto_skcipher *tfm;
133bbb16584SMilan Broz };
134bbb16584SMilan Broz
1351da177e4SLinus Torvalds /*
1361da177e4SLinus Torvalds * Crypt: maps a linear range of a block device
1371da177e4SLinus Torvalds * and encrypts / decrypts at the same time.
1381da177e4SLinus Torvalds */
1390f5d8e6eSMikulas Patocka enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
14039d42fa9SIgnat Korchagin DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD,
1418e225f04SDamien Le Moal DM_CRYPT_NO_READ_WORKQUEUE, DM_CRYPT_NO_WRITE_WORKQUEUE,
1428e225f04SDamien Le Moal DM_CRYPT_WRITE_INLINE };
143c0297721SAndi Kleen
144ef43aa38SMilan Broz enum cipher_flags {
14574d1da39SGeert Uytterhoeven CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cipher */
1468f0009a2SMilan Broz CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */
147bbb16584SMilan Broz CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */
148ef43aa38SMilan Broz };
149ef43aa38SMilan Broz
150c0297721SAndi Kleen /*
151610f2de3SMikulas Patocka * The fields in here must be read only after initialization.
152c0297721SAndi Kleen */
1531da177e4SLinus Torvalds struct crypt_config {
1541da177e4SLinus Torvalds struct dm_dev *dev;
1551da177e4SLinus Torvalds sector_t start;
1561da177e4SLinus Torvalds
1575059353dSMikulas Patocka struct percpu_counter n_allocated_pages;
1585059353dSMikulas Patocka
159cabf08e4SMilan Broz struct workqueue_struct *io_queue;
160cabf08e4SMilan Broz struct workqueue_struct *crypt_queue;
1613f1e9070SMilan Broz
162c7329effSMikulas Patocka spinlock_t write_thread_lock;
16372d711c8SMike Snitzer struct task_struct *write_thread;
164b3c5fd30SMikulas Patocka struct rb_root write_tree;
165dc267621SMikulas Patocka
1667dbcd137SMilan Broz char *cipher_string;
167ef43aa38SMilan Broz char *cipher_auth;
168c538f6ecSOndrej Kozina char *key_string;
1695ebaee6dSMilan Broz
1701b1b58f5SJulia Lawall const struct crypt_iv_operations *iv_gen_ops;
17179066ad3SHerbert Xu union {
17260473592SMilan Broz struct iv_benbi_private benbi;
17334745785SMilan Broz struct iv_lmk_private lmk;
174ed04d981SMilan Broz struct iv_tcw_private tcw;
175bbb16584SMilan Broz struct iv_elephant_private elephant;
17679066ad3SHerbert Xu } iv_gen_private;
1778d683dcdSAliOS system security u64 iv_offset;
1781da177e4SLinus Torvalds unsigned int iv_size;
17986a3238cSHeinz Mauelshagen unsigned short sector_size;
180ff3af92bSMikulas Patocka unsigned char sector_shift;
1811da177e4SLinus Torvalds
182ef43aa38SMilan Broz union {
183bbdb23b5SHerbert Xu struct crypto_skcipher **tfms;
184ef43aa38SMilan Broz struct crypto_aead **tfms_aead;
185ef43aa38SMilan Broz } cipher_tfm;
18686a3238cSHeinz Mauelshagen unsigned int tfms_count;
187ef43aa38SMilan Broz unsigned long cipher_flags;
188c0297721SAndi Kleen
189c0297721SAndi Kleen /*
190ddd42edfSMilan Broz * Layout of each crypto request:
191ddd42edfSMilan Broz *
192bbdb23b5SHerbert Xu * struct skcipher_request
193ddd42edfSMilan Broz * context
194ddd42edfSMilan Broz * padding
195ddd42edfSMilan Broz * struct dm_crypt_request
196ddd42edfSMilan Broz * padding
197ddd42edfSMilan Broz * IV
198ddd42edfSMilan Broz *
199ddd42edfSMilan Broz * The padding is added so that dm_crypt_request and the IV are
200ddd42edfSMilan Broz * correctly aligned.
201ddd42edfSMilan Broz */
202ddd42edfSMilan Broz unsigned int dmreq_start;
203ddd42edfSMilan Broz
204298a9fa0SMikulas Patocka unsigned int per_bio_data_size;
205298a9fa0SMikulas Patocka
206e48d4bbfSMilan Broz unsigned long flags;
2071da177e4SLinus Torvalds unsigned int key_size;
208da31a078SMilan Broz unsigned int key_parts; /* independent parts in key buffer */
209da31a078SMilan Broz unsigned int key_extra_size; /* additional keys length */
210ef43aa38SMilan Broz unsigned int key_mac_size; /* MAC key size for authenc(...) */
211ef43aa38SMilan Broz
212ef43aa38SMilan Broz unsigned int integrity_tag_size;
213ef43aa38SMilan Broz unsigned int integrity_iv_size;
214ef43aa38SMilan Broz unsigned int on_disk_tag_size;
215ef43aa38SMilan Broz
21672d711c8SMike Snitzer /*
21772d711c8SMike Snitzer * pool for per bio private data, crypto requests,
21872d711c8SMike Snitzer * encryption requeusts/buffer pages and integrity tags
21972d711c8SMike Snitzer */
22086a3238cSHeinz Mauelshagen unsigned int tag_pool_max_sectors;
22172d711c8SMike Snitzer mempool_t tag_pool;
22272d711c8SMike Snitzer mempool_t req_pool;
22372d711c8SMike Snitzer mempool_t page_pool;
22472d711c8SMike Snitzer
22572d711c8SMike Snitzer struct bio_set bs;
22672d711c8SMike Snitzer struct mutex bio_alloc_lock;
22772d711c8SMike Snitzer
228ef43aa38SMilan Broz u8 *authenc_key; /* space for keys in authenc() format (if used) */
229b18ae8ddSGustavo A. R. Silva u8 key[];
2301da177e4SLinus Torvalds };
2311da177e4SLinus Torvalds
2320a83df6cSMikulas Patocka #define MIN_IOS 64
233ef43aa38SMilan Broz #define MAX_TAG_SIZE 480
234ef43aa38SMilan Broz #define POOL_ENTRY_SIZE 512
2351da177e4SLinus Torvalds
2365059353dSMikulas Patocka static DEFINE_SPINLOCK(dm_crypt_clients_lock);
2372f06cd12SHeinz Mauelshagen static unsigned int dm_crypt_clients_n;
2385059353dSMikulas Patocka static volatile unsigned long dm_crypt_pages_per_client;
2395059353dSMikulas Patocka #define DM_CRYPT_MEMORY_PERCENT 2
240a8affc03SChristoph Hellwig #define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_VECS * 16)
2415059353dSMikulas Patocka
2423f868c09SChristoph Hellwig static void crypt_endio(struct bio *clone);
243395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io);
244ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
245ef43aa38SMilan Broz struct scatterlist *sg);
246027581f3SOlaf Kirch
2473fd53533SYang Yingliang static bool crypt_integrity_aead(struct crypt_config *cc);
2483fd53533SYang Yingliang
249c0297721SAndi Kleen /*
25086f917adSEric Biggers * Use this to access cipher attributes that are independent of the key.
251c0297721SAndi Kleen */
any_tfm(struct crypt_config * cc)252bbdb23b5SHerbert Xu static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
253c0297721SAndi Kleen {
254ef43aa38SMilan Broz return cc->cipher_tfm.tfms[0];
255ef43aa38SMilan Broz }
256ef43aa38SMilan Broz
any_tfm_aead(struct crypt_config * cc)257ef43aa38SMilan Broz static struct crypto_aead *any_tfm_aead(struct crypt_config *cc)
258ef43aa38SMilan Broz {
259ef43aa38SMilan Broz return cc->cipher_tfm.tfms_aead[0];
260c0297721SAndi Kleen }
261c0297721SAndi Kleen
2621da177e4SLinus Torvalds /*
2631da177e4SLinus Torvalds * Different IV generation algorithms:
2641da177e4SLinus Torvalds *
2653c164bd8SRik Snel * plain: the initial vector is the 32-bit little-endian version of the sector
2663a4fa0a2SRobert P. J. Day * number, padded with zeros if necessary.
2671da177e4SLinus Torvalds *
26861afef61SMilan Broz * plain64: the initial vector is the 64-bit little-endian version of the sector
26961afef61SMilan Broz * number, padded with zeros if necessary.
27061afef61SMilan Broz *
2717e3fd855SMilan Broz * plain64be: the initial vector is the 64-bit big-endian version of the sector
2727e3fd855SMilan Broz * number, padded with zeros if necessary.
2737e3fd855SMilan Broz *
2743c164bd8SRik Snel * essiv: "encrypted sector|salt initial vector", the sector number is
2751da177e4SLinus Torvalds * encrypted with the bulk cipher using a salt as key. The salt
2761da177e4SLinus Torvalds * should be derived from the bulk cipher's key via hashing.
2771da177e4SLinus Torvalds *
27848527fa7SRik Snel * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
27948527fa7SRik Snel * (needed for LRW-32-AES and possible other narrow block modes)
28048527fa7SRik Snel *
28146b47730SLudwig Nussel * null: the initial vector is always zero. Provides compatibility with
28246b47730SLudwig Nussel * obsolete loop_fish2 devices. Do not use for new devices.
28346b47730SLudwig Nussel *
28434745785SMilan Broz * lmk: Compatible implementation of the block chaining mode used
28534745785SMilan Broz * by the Loop-AES block device encryption system
28634745785SMilan Broz * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
28734745785SMilan Broz * It operates on full 512 byte sectors and uses CBC
28834745785SMilan Broz * with an IV derived from the sector number, the data and
28934745785SMilan Broz * optionally extra IV seed.
29034745785SMilan Broz * This means that after decryption the first block
29134745785SMilan Broz * of sector must be tweaked according to decrypted data.
29234745785SMilan Broz * Loop-AES can use three encryption schemes:
29334745785SMilan Broz * version 1: is plain aes-cbc mode
29434745785SMilan Broz * version 2: uses 64 multikey scheme with lmk IV generator
29534745785SMilan Broz * version 3: the same as version 2 with additional IV seed
29634745785SMilan Broz * (it uses 65 keys, last key is used as IV seed)
29734745785SMilan Broz *
298ed04d981SMilan Broz * tcw: Compatible implementation of the block chaining mode used
299ed04d981SMilan Broz * by the TrueCrypt device encryption system (prior to version 4.1).
300e44f23b3SMilan Broz * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
301ed04d981SMilan Broz * It operates on full 512 byte sectors and uses CBC
302ed04d981SMilan Broz * with an IV derived from initial key and the sector number.
303ed04d981SMilan Broz * In addition, whitening value is applied on every sector, whitening
304ed04d981SMilan Broz * is calculated from initial key, sector number and mixed using CRC32.
305ed04d981SMilan Broz * Note that this encryption scheme is vulnerable to watermarking attacks
306ed04d981SMilan Broz * and should be used for old compatible containers access only.
307b9411d73SMilan Broz *
308b9411d73SMilan Broz * eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode)
309b9411d73SMilan Broz * The IV is encrypted little-endian byte-offset (with the same key
310b9411d73SMilan Broz * and cipher as the volume).
311bbb16584SMilan Broz *
312bbb16584SMilan Broz * elephant: The extended version of eboiv with additional Elephant diffuser
313bbb16584SMilan Broz * used with Bitlocker CBC mode.
314bbb16584SMilan Broz * This mode was used in older Windows systems
3156f3bc22bSAlexander A. Klimov * https://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf
3161da177e4SLinus Torvalds */
3171da177e4SLinus Torvalds
crypt_iv_plain_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)3182dc5327dSMilan Broz static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
3192dc5327dSMilan Broz struct dm_crypt_request *dmreq)
3201da177e4SLinus Torvalds {
3211da177e4SLinus Torvalds memset(iv, 0, cc->iv_size);
322283a8328SAlasdair G Kergon *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
3231da177e4SLinus Torvalds
3241da177e4SLinus Torvalds return 0;
3251da177e4SLinus Torvalds }
3261da177e4SLinus Torvalds
crypt_iv_plain64_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)32761afef61SMilan Broz static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
3282dc5327dSMilan Broz struct dm_crypt_request *dmreq)
32961afef61SMilan Broz {
33061afef61SMilan Broz memset(iv, 0, cc->iv_size);
331283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
33261afef61SMilan Broz
33361afef61SMilan Broz return 0;
33461afef61SMilan Broz }
33561afef61SMilan Broz
crypt_iv_plain64be_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)3367e3fd855SMilan Broz static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv,
3377e3fd855SMilan Broz struct dm_crypt_request *dmreq)
3387e3fd855SMilan Broz {
3397e3fd855SMilan Broz memset(iv, 0, cc->iv_size);
3407e3fd855SMilan Broz /* iv_size is at least of size u64; usually it is 16 bytes */
3417e3fd855SMilan Broz *(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);
3427e3fd855SMilan Broz
3437e3fd855SMilan Broz return 0;
3447e3fd855SMilan Broz }
3457e3fd855SMilan Broz
crypt_iv_essiv_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)3462dc5327dSMilan Broz static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
3472dc5327dSMilan Broz struct dm_crypt_request *dmreq)
3481da177e4SLinus Torvalds {
349a1a262b6SArd Biesheuvel /*
350a1a262b6SArd Biesheuvel * ESSIV encryption of the IV is now handled by the crypto API,
351a1a262b6SArd Biesheuvel * so just pass the plain sector number here.
352a1a262b6SArd Biesheuvel */
3531da177e4SLinus Torvalds memset(iv, 0, cc->iv_size);
354283a8328SAlasdair G Kergon *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
355c0297721SAndi Kleen
3561da177e4SLinus Torvalds return 0;
3571da177e4SLinus Torvalds }
3581da177e4SLinus Torvalds
crypt_iv_benbi_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)35948527fa7SRik Snel static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
36048527fa7SRik Snel const char *opts)
36148527fa7SRik Snel {
36286a3238cSHeinz Mauelshagen unsigned int bs;
3634ea9471fSMilan Broz int log;
3644ea9471fSMilan Broz
3653fd53533SYang Yingliang if (crypt_integrity_aead(cc))
3664ea9471fSMilan Broz bs = crypto_aead_blocksize(any_tfm_aead(cc));
3674ea9471fSMilan Broz else
3684ea9471fSMilan Broz bs = crypto_skcipher_blocksize(any_tfm(cc));
3694ea9471fSMilan Broz log = ilog2(bs);
37048527fa7SRik Snel
371a4a82ce3SHeinz Mauelshagen /*
372a4a82ce3SHeinz Mauelshagen * We need to calculate how far we must shift the sector count
373a4a82ce3SHeinz Mauelshagen * to get the cipher block count, we use this shift in _gen.
374a4a82ce3SHeinz Mauelshagen */
37548527fa7SRik Snel if (1 << log != bs) {
37648527fa7SRik Snel ti->error = "cypher blocksize is not a power of 2";
37748527fa7SRik Snel return -EINVAL;
37848527fa7SRik Snel }
37948527fa7SRik Snel
38048527fa7SRik Snel if (log > 9) {
38148527fa7SRik Snel ti->error = "cypher blocksize is > 512";
38248527fa7SRik Snel return -EINVAL;
38348527fa7SRik Snel }
38448527fa7SRik Snel
38560473592SMilan Broz cc->iv_gen_private.benbi.shift = 9 - log;
38648527fa7SRik Snel
38748527fa7SRik Snel return 0;
38848527fa7SRik Snel }
38948527fa7SRik Snel
crypt_iv_benbi_dtr(struct crypt_config * cc)39048527fa7SRik Snel static void crypt_iv_benbi_dtr(struct crypt_config *cc)
39148527fa7SRik Snel {
39248527fa7SRik Snel }
39348527fa7SRik Snel
crypt_iv_benbi_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)3942dc5327dSMilan Broz static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
3952dc5327dSMilan Broz struct dm_crypt_request *dmreq)
39648527fa7SRik Snel {
39779066ad3SHerbert Xu __be64 val;
39879066ad3SHerbert Xu
39948527fa7SRik Snel memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
40079066ad3SHerbert Xu
4012dc5327dSMilan Broz val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
40279066ad3SHerbert Xu put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
40348527fa7SRik Snel
4041da177e4SLinus Torvalds return 0;
4051da177e4SLinus Torvalds }
4061da177e4SLinus Torvalds
crypt_iv_null_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)4072dc5327dSMilan Broz static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
4082dc5327dSMilan Broz struct dm_crypt_request *dmreq)
40946b47730SLudwig Nussel {
41046b47730SLudwig Nussel memset(iv, 0, cc->iv_size);
41146b47730SLudwig Nussel
41246b47730SLudwig Nussel return 0;
41346b47730SLudwig Nussel }
41446b47730SLudwig Nussel
crypt_iv_lmk_dtr(struct crypt_config * cc)41534745785SMilan Broz static void crypt_iv_lmk_dtr(struct crypt_config *cc)
41634745785SMilan Broz {
41734745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
41834745785SMilan Broz
41934745785SMilan Broz if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
42034745785SMilan Broz crypto_free_shash(lmk->hash_tfm);
42134745785SMilan Broz lmk->hash_tfm = NULL;
42234745785SMilan Broz
423453431a5SWaiman Long kfree_sensitive(lmk->seed);
42434745785SMilan Broz lmk->seed = NULL;
42534745785SMilan Broz }
42634745785SMilan Broz
crypt_iv_lmk_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)42734745785SMilan Broz static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
42834745785SMilan Broz const char *opts)
42934745785SMilan Broz {
43034745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
43134745785SMilan Broz
4328f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) {
4338f0009a2SMilan Broz ti->error = "Unsupported sector size for LMK";
4348f0009a2SMilan Broz return -EINVAL;
4358f0009a2SMilan Broz }
4368f0009a2SMilan Broz
437cd746938SMikulas Patocka lmk->hash_tfm = crypto_alloc_shash("md5", 0,
438cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY);
43934745785SMilan Broz if (IS_ERR(lmk->hash_tfm)) {
44034745785SMilan Broz ti->error = "Error initializing LMK hash";
44134745785SMilan Broz return PTR_ERR(lmk->hash_tfm);
44234745785SMilan Broz }
44334745785SMilan Broz
44434745785SMilan Broz /* No seed in LMK version 2 */
44534745785SMilan Broz if (cc->key_parts == cc->tfms_count) {
44634745785SMilan Broz lmk->seed = NULL;
44734745785SMilan Broz return 0;
44834745785SMilan Broz }
44934745785SMilan Broz
45034745785SMilan Broz lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
45134745785SMilan Broz if (!lmk->seed) {
45234745785SMilan Broz crypt_iv_lmk_dtr(cc);
45334745785SMilan Broz ti->error = "Error kmallocing seed storage in LMK";
45434745785SMilan Broz return -ENOMEM;
45534745785SMilan Broz }
45634745785SMilan Broz
45734745785SMilan Broz return 0;
45834745785SMilan Broz }
45934745785SMilan Broz
crypt_iv_lmk_init(struct crypt_config * cc)46034745785SMilan Broz static int crypt_iv_lmk_init(struct crypt_config *cc)
46134745785SMilan Broz {
46234745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
46334745785SMilan Broz int subkey_size = cc->key_size / cc->key_parts;
46434745785SMilan Broz
46534745785SMilan Broz /* LMK seed is on the position of LMK_KEYS + 1 key */
46634745785SMilan Broz if (lmk->seed)
46734745785SMilan Broz memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
46834745785SMilan Broz crypto_shash_digestsize(lmk->hash_tfm));
46934745785SMilan Broz
47034745785SMilan Broz return 0;
47134745785SMilan Broz }
47234745785SMilan Broz
crypt_iv_lmk_wipe(struct crypt_config * cc)47334745785SMilan Broz static int crypt_iv_lmk_wipe(struct crypt_config *cc)
47434745785SMilan Broz {
47534745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
47634745785SMilan Broz
47734745785SMilan Broz if (lmk->seed)
47834745785SMilan Broz memset(lmk->seed, 0, LMK_SEED_SIZE);
47934745785SMilan Broz
48034745785SMilan Broz return 0;
48134745785SMilan Broz }
48234745785SMilan Broz
crypt_iv_lmk_one(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq,u8 * data)48334745785SMilan Broz static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
48434745785SMilan Broz struct dm_crypt_request *dmreq,
48534745785SMilan Broz u8 *data)
48634745785SMilan Broz {
48734745785SMilan Broz struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
488b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
48934745785SMilan Broz struct md5_state md5state;
490da31a078SMilan Broz __le32 buf[4];
49134745785SMilan Broz int i, r;
49234745785SMilan Broz
493b6106265SJan-Simon Möller desc->tfm = lmk->hash_tfm;
49434745785SMilan Broz
495b6106265SJan-Simon Möller r = crypto_shash_init(desc);
49634745785SMilan Broz if (r)
49734745785SMilan Broz return r;
49834745785SMilan Broz
49934745785SMilan Broz if (lmk->seed) {
500b6106265SJan-Simon Möller r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE);
50134745785SMilan Broz if (r)
50234745785SMilan Broz return r;
50334745785SMilan Broz }
50434745785SMilan Broz
50534745785SMilan Broz /* Sector is always 512B, block size 16, add data of blocks 1-31 */
506b6106265SJan-Simon Möller r = crypto_shash_update(desc, data + 16, 16 * 31);
50734745785SMilan Broz if (r)
50834745785SMilan Broz return r;
50934745785SMilan Broz
51034745785SMilan Broz /* Sector is cropped to 56 bits here */
51134745785SMilan Broz buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
51234745785SMilan Broz buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
51334745785SMilan Broz buf[2] = cpu_to_le32(4024);
51434745785SMilan Broz buf[3] = 0;
515b6106265SJan-Simon Möller r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf));
51634745785SMilan Broz if (r)
51734745785SMilan Broz return r;
51834745785SMilan Broz
51934745785SMilan Broz /* No MD5 padding here */
520b6106265SJan-Simon Möller r = crypto_shash_export(desc, &md5state);
52134745785SMilan Broz if (r)
52234745785SMilan Broz return r;
52334745785SMilan Broz
52434745785SMilan Broz for (i = 0; i < MD5_HASH_WORDS; i++)
52534745785SMilan Broz __cpu_to_le32s(&md5state.hash[i]);
52634745785SMilan Broz memcpy(iv, &md5state.hash, cc->iv_size);
52734745785SMilan Broz
52834745785SMilan Broz return 0;
52934745785SMilan Broz }
53034745785SMilan Broz
crypt_iv_lmk_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)53134745785SMilan Broz static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
53234745785SMilan Broz struct dm_crypt_request *dmreq)
53334745785SMilan Broz {
534ef43aa38SMilan Broz struct scatterlist *sg;
53534745785SMilan Broz u8 *src;
53634745785SMilan Broz int r = 0;
53734745785SMilan Broz
53834745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
539ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in);
5400d78954aSHeinz Mauelshagen src = kmap_local_page(sg_page(sg));
541ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset);
5420d78954aSHeinz Mauelshagen kunmap_local(src);
54334745785SMilan Broz } else
54434745785SMilan Broz memset(iv, 0, cc->iv_size);
54534745785SMilan Broz
54634745785SMilan Broz return r;
54734745785SMilan Broz }
54834745785SMilan Broz
crypt_iv_lmk_post(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)54934745785SMilan Broz static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
55034745785SMilan Broz struct dm_crypt_request *dmreq)
55134745785SMilan Broz {
552ef43aa38SMilan Broz struct scatterlist *sg;
55334745785SMilan Broz u8 *dst;
55434745785SMilan Broz int r;
55534745785SMilan Broz
55634745785SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
55734745785SMilan Broz return 0;
55834745785SMilan Broz
559ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out);
5600d78954aSHeinz Mauelshagen dst = kmap_local_page(sg_page(sg));
561ef43aa38SMilan Broz r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset);
56234745785SMilan Broz
56334745785SMilan Broz /* Tweak the first block of plaintext sector */
56434745785SMilan Broz if (!r)
565ef43aa38SMilan Broz crypto_xor(dst + sg->offset, iv, cc->iv_size);
56634745785SMilan Broz
5670d78954aSHeinz Mauelshagen kunmap_local(dst);
56834745785SMilan Broz return r;
56934745785SMilan Broz }
57034745785SMilan Broz
crypt_iv_tcw_dtr(struct crypt_config * cc)571ed04d981SMilan Broz static void crypt_iv_tcw_dtr(struct crypt_config *cc)
572ed04d981SMilan Broz {
573ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
574ed04d981SMilan Broz
575453431a5SWaiman Long kfree_sensitive(tcw->iv_seed);
576ed04d981SMilan Broz tcw->iv_seed = NULL;
577453431a5SWaiman Long kfree_sensitive(tcw->whitening);
578ed04d981SMilan Broz tcw->whitening = NULL;
579ed04d981SMilan Broz
580ed04d981SMilan Broz if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm))
581ed04d981SMilan Broz crypto_free_shash(tcw->crc32_tfm);
582ed04d981SMilan Broz tcw->crc32_tfm = NULL;
583ed04d981SMilan Broz }
584ed04d981SMilan Broz
crypt_iv_tcw_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)585ed04d981SMilan Broz static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
586ed04d981SMilan Broz const char *opts)
587ed04d981SMilan Broz {
588ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
589ed04d981SMilan Broz
5908f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT)) {
5918f0009a2SMilan Broz ti->error = "Unsupported sector size for TCW";
5928f0009a2SMilan Broz return -EINVAL;
5938f0009a2SMilan Broz }
5948f0009a2SMilan Broz
595ed04d981SMilan Broz if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
596ed04d981SMilan Broz ti->error = "Wrong key size for TCW";
597ed04d981SMilan Broz return -EINVAL;
598ed04d981SMilan Broz }
599ed04d981SMilan Broz
600cd746938SMikulas Patocka tcw->crc32_tfm = crypto_alloc_shash("crc32", 0,
601cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY);
602ed04d981SMilan Broz if (IS_ERR(tcw->crc32_tfm)) {
603ed04d981SMilan Broz ti->error = "Error initializing CRC32 in TCW";
604ed04d981SMilan Broz return PTR_ERR(tcw->crc32_tfm);
605ed04d981SMilan Broz }
606ed04d981SMilan Broz
607ed04d981SMilan Broz tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
608ed04d981SMilan Broz tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
609ed04d981SMilan Broz if (!tcw->iv_seed || !tcw->whitening) {
610ed04d981SMilan Broz crypt_iv_tcw_dtr(cc);
611ed04d981SMilan Broz ti->error = "Error allocating seed storage in TCW";
612ed04d981SMilan Broz return -ENOMEM;
613ed04d981SMilan Broz }
614ed04d981SMilan Broz
615ed04d981SMilan Broz return 0;
616ed04d981SMilan Broz }
617ed04d981SMilan Broz
crypt_iv_tcw_init(struct crypt_config * cc)618ed04d981SMilan Broz static int crypt_iv_tcw_init(struct crypt_config *cc)
619ed04d981SMilan Broz {
620ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
621ed04d981SMilan Broz int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
622ed04d981SMilan Broz
623ed04d981SMilan Broz memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
624ed04d981SMilan Broz memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
625ed04d981SMilan Broz TCW_WHITENING_SIZE);
626ed04d981SMilan Broz
627ed04d981SMilan Broz return 0;
628ed04d981SMilan Broz }
629ed04d981SMilan Broz
crypt_iv_tcw_wipe(struct crypt_config * cc)630ed04d981SMilan Broz static int crypt_iv_tcw_wipe(struct crypt_config *cc)
631ed04d981SMilan Broz {
632ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
633ed04d981SMilan Broz
634ed04d981SMilan Broz memset(tcw->iv_seed, 0, cc->iv_size);
635ed04d981SMilan Broz memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
636ed04d981SMilan Broz
637ed04d981SMilan Broz return 0;
638ed04d981SMilan Broz }
639ed04d981SMilan Broz
crypt_iv_tcw_whitening(struct crypt_config * cc,struct dm_crypt_request * dmreq,u8 * data)640ed04d981SMilan Broz static int crypt_iv_tcw_whitening(struct crypt_config *cc,
641ed04d981SMilan Broz struct dm_crypt_request *dmreq,
642ed04d981SMilan Broz u8 *data)
643ed04d981SMilan Broz {
644ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
645350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector);
646ed04d981SMilan Broz u8 buf[TCW_WHITENING_SIZE];
647b6106265SJan-Simon Möller SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
648ed04d981SMilan Broz int i, r;
649ed04d981SMilan Broz
650ed04d981SMilan Broz /* xor whitening with sector number */
65145fe93dfSArd Biesheuvel crypto_xor_cpy(buf, tcw->whitening, (u8 *)§or, 8);
65245fe93dfSArd Biesheuvel crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)§or, 8);
653ed04d981SMilan Broz
654ed04d981SMilan Broz /* calculate crc32 for every 32bit part and xor it */
655b6106265SJan-Simon Möller desc->tfm = tcw->crc32_tfm;
656ed04d981SMilan Broz for (i = 0; i < 4; i++) {
657b6106265SJan-Simon Möller r = crypto_shash_init(desc);
658ed04d981SMilan Broz if (r)
659ed04d981SMilan Broz goto out;
660b6106265SJan-Simon Möller r = crypto_shash_update(desc, &buf[i * 4], 4);
661ed04d981SMilan Broz if (r)
662ed04d981SMilan Broz goto out;
663b6106265SJan-Simon Möller r = crypto_shash_final(desc, &buf[i * 4]);
664ed04d981SMilan Broz if (r)
665ed04d981SMilan Broz goto out;
666ed04d981SMilan Broz }
667ed04d981SMilan Broz crypto_xor(&buf[0], &buf[12], 4);
668ed04d981SMilan Broz crypto_xor(&buf[4], &buf[8], 4);
669ed04d981SMilan Broz
670ed04d981SMilan Broz /* apply whitening (8 bytes) to whole sector */
671ed04d981SMilan Broz for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
672ed04d981SMilan Broz crypto_xor(data + i * 8, buf, 8);
673ed04d981SMilan Broz out:
6741a71d6ffSMilan Broz memzero_explicit(buf, sizeof(buf));
675ed04d981SMilan Broz return r;
676ed04d981SMilan Broz }
677ed04d981SMilan Broz
crypt_iv_tcw_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)678ed04d981SMilan Broz static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
679ed04d981SMilan Broz struct dm_crypt_request *dmreq)
680ed04d981SMilan Broz {
681ef43aa38SMilan Broz struct scatterlist *sg;
682ed04d981SMilan Broz struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
683350b5393SBart Van Assche __le64 sector = cpu_to_le64(dmreq->iv_sector);
684ed04d981SMilan Broz u8 *src;
685ed04d981SMilan Broz int r = 0;
686ed04d981SMilan Broz
687ed04d981SMilan Broz /* Remove whitening from ciphertext */
688ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
689ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_in);
6900d78954aSHeinz Mauelshagen src = kmap_local_page(sg_page(sg));
691ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset);
6920d78954aSHeinz Mauelshagen kunmap_local(src);
693ed04d981SMilan Broz }
694ed04d981SMilan Broz
695ed04d981SMilan Broz /* Calculate IV */
69645fe93dfSArd Biesheuvel crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)§or, 8);
697ed04d981SMilan Broz if (cc->iv_size > 8)
69845fe93dfSArd Biesheuvel crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)§or,
69945fe93dfSArd Biesheuvel cc->iv_size - 8);
700ed04d981SMilan Broz
701ed04d981SMilan Broz return r;
702ed04d981SMilan Broz }
703ed04d981SMilan Broz
crypt_iv_tcw_post(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)704ed04d981SMilan Broz static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
705ed04d981SMilan Broz struct dm_crypt_request *dmreq)
706ed04d981SMilan Broz {
707ef43aa38SMilan Broz struct scatterlist *sg;
708ed04d981SMilan Broz u8 *dst;
709ed04d981SMilan Broz int r;
710ed04d981SMilan Broz
711ed04d981SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
712ed04d981SMilan Broz return 0;
713ed04d981SMilan Broz
714ed04d981SMilan Broz /* Apply whitening on ciphertext */
715ef43aa38SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out);
7160d78954aSHeinz Mauelshagen dst = kmap_local_page(sg_page(sg));
717ef43aa38SMilan Broz r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset);
7180d78954aSHeinz Mauelshagen kunmap_local(dst);
719ed04d981SMilan Broz
720ed04d981SMilan Broz return r;
721ed04d981SMilan Broz }
722ed04d981SMilan Broz
crypt_iv_random_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)723ef43aa38SMilan Broz static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,
724ef43aa38SMilan Broz struct dm_crypt_request *dmreq)
725ef43aa38SMilan Broz {
726ef43aa38SMilan Broz /* Used only for writes, there must be an additional space to store IV */
727ef43aa38SMilan Broz get_random_bytes(iv, cc->iv_size);
728ef43aa38SMilan Broz return 0;
729ef43aa38SMilan Broz }
730ef43aa38SMilan Broz
crypt_iv_eboiv_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)731b9411d73SMilan Broz static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti,
732b9411d73SMilan Broz const char *opts)
733b9411d73SMilan Broz {
7343fd53533SYang Yingliang if (crypt_integrity_aead(cc)) {
73539d13a1aSArd Biesheuvel ti->error = "AEAD transforms not supported for EBOIV";
736b9411d73SMilan Broz return -EINVAL;
737b9411d73SMilan Broz }
738b9411d73SMilan Broz
73939d13a1aSArd Biesheuvel if (crypto_skcipher_blocksize(any_tfm(cc)) != cc->iv_size) {
7402e84fecfSHeinz Mauelshagen ti->error = "Block size of EBOIV cipher does not match IV size of block cipher";
74139d13a1aSArd Biesheuvel return -EINVAL;
742b9411d73SMilan Broz }
743b9411d73SMilan Broz
744b9411d73SMilan Broz return 0;
745b9411d73SMilan Broz }
746b9411d73SMilan Broz
crypt_iv_eboiv_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)747b9411d73SMilan Broz static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv,
748b9411d73SMilan Broz struct dm_crypt_request *dmreq)
749b9411d73SMilan Broz {
750e3023094SHerbert Xu struct crypto_skcipher *tfm = any_tfm(cc);
75139d13a1aSArd Biesheuvel struct skcipher_request *req;
75239d13a1aSArd Biesheuvel struct scatterlist src, dst;
7537785a9e4SDamien Le Moal DECLARE_CRYPTO_WAIT(wait);
754e3023094SHerbert Xu unsigned int reqsize;
75539d13a1aSArd Biesheuvel int err;
756e3023094SHerbert Xu u8 *buf;
757b9411d73SMilan Broz
758152d0bcdSHerbert Xu reqsize = sizeof(*req) + crypto_skcipher_reqsize(tfm);
759152d0bcdSHerbert Xu reqsize = ALIGN(reqsize, __alignof__(__le64));
760e3023094SHerbert Xu
761e3023094SHerbert Xu req = kmalloc(reqsize + cc->iv_size, GFP_NOIO);
76239d13a1aSArd Biesheuvel if (!req)
76339d13a1aSArd Biesheuvel return -ENOMEM;
764b9411d73SMilan Broz
765e3023094SHerbert Xu skcipher_request_set_tfm(req, tfm);
766e3023094SHerbert Xu
767e3023094SHerbert Xu buf = (u8 *)req + reqsize;
76839d13a1aSArd Biesheuvel memset(buf, 0, cc->iv_size);
76939d13a1aSArd Biesheuvel *(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
77039d13a1aSArd Biesheuvel
77139d13a1aSArd Biesheuvel sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size);
77239d13a1aSArd Biesheuvel sg_init_one(&dst, iv, cc->iv_size);
77339d13a1aSArd Biesheuvel skcipher_request_set_crypt(req, &src, &dst, cc->iv_size, buf);
77439d13a1aSArd Biesheuvel skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
77539d13a1aSArd Biesheuvel err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
776e3023094SHerbert Xu kfree_sensitive(req);
77739d13a1aSArd Biesheuvel
77839d13a1aSArd Biesheuvel return err;
779b9411d73SMilan Broz }
780b9411d73SMilan Broz
crypt_iv_elephant_dtr(struct crypt_config * cc)781bbb16584SMilan Broz static void crypt_iv_elephant_dtr(struct crypt_config *cc)
782bbb16584SMilan Broz {
783bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
784bbb16584SMilan Broz
785bbb16584SMilan Broz crypto_free_skcipher(elephant->tfm);
786bbb16584SMilan Broz elephant->tfm = NULL;
787bbb16584SMilan Broz }
788bbb16584SMilan Broz
crypt_iv_elephant_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)789bbb16584SMilan Broz static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti,
790bbb16584SMilan Broz const char *opts)
791bbb16584SMilan Broz {
792bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
793bbb16584SMilan Broz int r;
794bbb16584SMilan Broz
795cd746938SMikulas Patocka elephant->tfm = crypto_alloc_skcipher("ecb(aes)", 0,
796cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY);
797bbb16584SMilan Broz if (IS_ERR(elephant->tfm)) {
798bbb16584SMilan Broz r = PTR_ERR(elephant->tfm);
799bbb16584SMilan Broz elephant->tfm = NULL;
800bbb16584SMilan Broz return r;
801bbb16584SMilan Broz }
802bbb16584SMilan Broz
803bbb16584SMilan Broz r = crypt_iv_eboiv_ctr(cc, ti, NULL);
804bbb16584SMilan Broz if (r)
805bbb16584SMilan Broz crypt_iv_elephant_dtr(cc);
806bbb16584SMilan Broz return r;
807bbb16584SMilan Broz }
808bbb16584SMilan Broz
diffuser_disk_to_cpu(u32 * d,size_t n)809bbb16584SMilan Broz static void diffuser_disk_to_cpu(u32 *d, size_t n)
810bbb16584SMilan Broz {
811bbb16584SMilan Broz #ifndef __LITTLE_ENDIAN
812bbb16584SMilan Broz int i;
813bbb16584SMilan Broz
814bbb16584SMilan Broz for (i = 0; i < n; i++)
815bbb16584SMilan Broz d[i] = le32_to_cpu((__le32)d[i]);
816bbb16584SMilan Broz #endif
817bbb16584SMilan Broz }
818bbb16584SMilan Broz
diffuser_cpu_to_disk(__le32 * d,size_t n)819bbb16584SMilan Broz static void diffuser_cpu_to_disk(__le32 *d, size_t n)
820bbb16584SMilan Broz {
821bbb16584SMilan Broz #ifndef __LITTLE_ENDIAN
822bbb16584SMilan Broz int i;
823bbb16584SMilan Broz
824bbb16584SMilan Broz for (i = 0; i < n; i++)
825bbb16584SMilan Broz d[i] = cpu_to_le32((u32)d[i]);
826bbb16584SMilan Broz #endif
827bbb16584SMilan Broz }
828bbb16584SMilan Broz
diffuser_a_decrypt(u32 * d,size_t n)829bbb16584SMilan Broz static void diffuser_a_decrypt(u32 *d, size_t n)
830bbb16584SMilan Broz {
831bbb16584SMilan Broz int i, i1, i2, i3;
832bbb16584SMilan Broz
833bbb16584SMilan Broz for (i = 0; i < 5; i++) {
834bbb16584SMilan Broz i1 = 0;
835bbb16584SMilan Broz i2 = n - 2;
836bbb16584SMilan Broz i3 = n - 5;
837bbb16584SMilan Broz
838bbb16584SMilan Broz while (i1 < (n - 1)) {
839bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
840bbb16584SMilan Broz i1++; i2++; i3++;
841bbb16584SMilan Broz
842bbb16584SMilan Broz if (i3 >= n)
843bbb16584SMilan Broz i3 -= n;
844bbb16584SMilan Broz
845bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3];
846bbb16584SMilan Broz i1++; i2++; i3++;
847bbb16584SMilan Broz
848bbb16584SMilan Broz if (i2 >= n)
849bbb16584SMilan Broz i2 -= n;
850bbb16584SMilan Broz
851bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
852bbb16584SMilan Broz i1++; i2++; i3++;
853bbb16584SMilan Broz
854bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3];
855bbb16584SMilan Broz i1++; i2++; i3++;
856bbb16584SMilan Broz }
857bbb16584SMilan Broz }
858bbb16584SMilan Broz }
859bbb16584SMilan Broz
diffuser_a_encrypt(u32 * d,size_t n)860bbb16584SMilan Broz static void diffuser_a_encrypt(u32 *d, size_t n)
861bbb16584SMilan Broz {
862bbb16584SMilan Broz int i, i1, i2, i3;
863bbb16584SMilan Broz
864bbb16584SMilan Broz for (i = 0; i < 5; i++) {
865bbb16584SMilan Broz i1 = n - 1;
866bbb16584SMilan Broz i2 = n - 2 - 1;
867bbb16584SMilan Broz i3 = n - 5 - 1;
868bbb16584SMilan Broz
869bbb16584SMilan Broz while (i1 > 0) {
870bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3];
871bbb16584SMilan Broz i1--; i2--; i3--;
872bbb16584SMilan Broz
873bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
874bbb16584SMilan Broz i1--; i2--; i3--;
875bbb16584SMilan Broz
876bbb16584SMilan Broz if (i2 < 0)
877bbb16584SMilan Broz i2 += n;
878bbb16584SMilan Broz
879bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3];
880bbb16584SMilan Broz i1--; i2--; i3--;
881bbb16584SMilan Broz
882bbb16584SMilan Broz if (i3 < 0)
883bbb16584SMilan Broz i3 += n;
884bbb16584SMilan Broz
885bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
886bbb16584SMilan Broz i1--; i2--; i3--;
887bbb16584SMilan Broz }
888bbb16584SMilan Broz }
889bbb16584SMilan Broz }
890bbb16584SMilan Broz
diffuser_b_decrypt(u32 * d,size_t n)891bbb16584SMilan Broz static void diffuser_b_decrypt(u32 *d, size_t n)
892bbb16584SMilan Broz {
893bbb16584SMilan Broz int i, i1, i2, i3;
894bbb16584SMilan Broz
895bbb16584SMilan Broz for (i = 0; i < 3; i++) {
896bbb16584SMilan Broz i1 = 0;
897bbb16584SMilan Broz i2 = 2;
898bbb16584SMilan Broz i3 = 5;
899bbb16584SMilan Broz
900bbb16584SMilan Broz while (i1 < (n - 1)) {
901bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3];
902bbb16584SMilan Broz i1++; i2++; i3++;
903bbb16584SMilan Broz
904bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
905bbb16584SMilan Broz i1++; i2++; i3++;
906bbb16584SMilan Broz
907bbb16584SMilan Broz if (i2 >= n)
908bbb16584SMilan Broz i2 -= n;
909bbb16584SMilan Broz
910bbb16584SMilan Broz d[i1] += d[i2] ^ d[i3];
911bbb16584SMilan Broz i1++; i2++; i3++;
912bbb16584SMilan Broz
913bbb16584SMilan Broz if (i3 >= n)
914bbb16584SMilan Broz i3 -= n;
915bbb16584SMilan Broz
916bbb16584SMilan Broz d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
917bbb16584SMilan Broz i1++; i2++; i3++;
918bbb16584SMilan Broz }
919bbb16584SMilan Broz }
920bbb16584SMilan Broz }
921bbb16584SMilan Broz
diffuser_b_encrypt(u32 * d,size_t n)922bbb16584SMilan Broz static void diffuser_b_encrypt(u32 *d, size_t n)
923bbb16584SMilan Broz {
924bbb16584SMilan Broz int i, i1, i2, i3;
925bbb16584SMilan Broz
926bbb16584SMilan Broz for (i = 0; i < 3; i++) {
927bbb16584SMilan Broz i1 = n - 1;
928bbb16584SMilan Broz i2 = 2 - 1;
929bbb16584SMilan Broz i3 = 5 - 1;
930bbb16584SMilan Broz
931bbb16584SMilan Broz while (i1 > 0) {
932bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
933bbb16584SMilan Broz i1--; i2--; i3--;
934bbb16584SMilan Broz
935bbb16584SMilan Broz if (i3 < 0)
936bbb16584SMilan Broz i3 += n;
937bbb16584SMilan Broz
938bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3];
939bbb16584SMilan Broz i1--; i2--; i3--;
940bbb16584SMilan Broz
941bbb16584SMilan Broz if (i2 < 0)
942bbb16584SMilan Broz i2 += n;
943bbb16584SMilan Broz
944bbb16584SMilan Broz d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
945bbb16584SMilan Broz i1--; i2--; i3--;
946bbb16584SMilan Broz
947bbb16584SMilan Broz d[i1] -= d[i2] ^ d[i3];
948bbb16584SMilan Broz i1--; i2--; i3--;
949bbb16584SMilan Broz }
950bbb16584SMilan Broz }
951bbb16584SMilan Broz }
952bbb16584SMilan Broz
crypt_iv_elephant(struct crypt_config * cc,struct dm_crypt_request * dmreq)953bbb16584SMilan Broz static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq)
954bbb16584SMilan Broz {
955bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
956bbb16584SMilan Broz u8 *es, *ks, *data, *data2, *data_offset;
957bbb16584SMilan Broz struct skcipher_request *req;
958bbb16584SMilan Broz struct scatterlist *sg, *sg2, src, dst;
9597785a9e4SDamien Le Moal DECLARE_CRYPTO_WAIT(wait);
960bbb16584SMilan Broz int i, r;
961bbb16584SMilan Broz
962bbb16584SMilan Broz req = skcipher_request_alloc(elephant->tfm, GFP_NOIO);
963bbb16584SMilan Broz es = kzalloc(16, GFP_NOIO); /* Key for AES */
964bbb16584SMilan Broz ks = kzalloc(32, GFP_NOIO); /* Elephant sector key */
965bbb16584SMilan Broz
966bbb16584SMilan Broz if (!req || !es || !ks) {
967bbb16584SMilan Broz r = -ENOMEM;
968bbb16584SMilan Broz goto out;
969bbb16584SMilan Broz }
970bbb16584SMilan Broz
971bbb16584SMilan Broz *(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
972bbb16584SMilan Broz
973bbb16584SMilan Broz /* E(Ks, e(s)) */
974bbb16584SMilan Broz sg_init_one(&src, es, 16);
975bbb16584SMilan Broz sg_init_one(&dst, ks, 16);
976bbb16584SMilan Broz skcipher_request_set_crypt(req, &src, &dst, 16, NULL);
977bbb16584SMilan Broz skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
978bbb16584SMilan Broz r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
979bbb16584SMilan Broz if (r)
980bbb16584SMilan Broz goto out;
981bbb16584SMilan Broz
982bbb16584SMilan Broz /* E(Ks, e'(s)) */
983bbb16584SMilan Broz es[15] = 0x80;
984bbb16584SMilan Broz sg_init_one(&dst, &ks[16], 16);
985bbb16584SMilan Broz r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
986bbb16584SMilan Broz if (r)
987bbb16584SMilan Broz goto out;
988bbb16584SMilan Broz
989bbb16584SMilan Broz sg = crypt_get_sg_data(cc, dmreq->sg_out);
9900d78954aSHeinz Mauelshagen data = kmap_local_page(sg_page(sg));
991bbb16584SMilan Broz data_offset = data + sg->offset;
992bbb16584SMilan Broz
993bbb16584SMilan Broz /* Cannot modify original bio, copy to sg_out and apply Elephant to it */
994bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
995bbb16584SMilan Broz sg2 = crypt_get_sg_data(cc, dmreq->sg_in);
9960d78954aSHeinz Mauelshagen data2 = kmap_local_page(sg_page(sg2));
997bbb16584SMilan Broz memcpy(data_offset, data2 + sg2->offset, cc->sector_size);
9980d78954aSHeinz Mauelshagen kunmap_local(data2);
999bbb16584SMilan Broz }
1000bbb16584SMilan Broz
1001bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
1002bbb16584SMilan Broz diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32));
1003bbb16584SMilan Broz diffuser_b_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1004bbb16584SMilan Broz diffuser_a_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1005bbb16584SMilan Broz diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32));
1006bbb16584SMilan Broz }
1007bbb16584SMilan Broz
1008bbb16584SMilan Broz for (i = 0; i < (cc->sector_size / 32); i++)
1009bbb16584SMilan Broz crypto_xor(data_offset + i * 32, ks, 32);
1010bbb16584SMilan Broz
1011bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1012bbb16584SMilan Broz diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32));
1013bbb16584SMilan Broz diffuser_a_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1014bbb16584SMilan Broz diffuser_b_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1015bbb16584SMilan Broz diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32));
1016bbb16584SMilan Broz }
1017bbb16584SMilan Broz
10180d78954aSHeinz Mauelshagen kunmap_local(data);
1019bbb16584SMilan Broz out:
1020453431a5SWaiman Long kfree_sensitive(ks);
1021453431a5SWaiman Long kfree_sensitive(es);
1022bbb16584SMilan Broz skcipher_request_free(req);
1023bbb16584SMilan Broz return r;
1024bbb16584SMilan Broz }
1025bbb16584SMilan Broz
crypt_iv_elephant_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)1026bbb16584SMilan Broz static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv,
1027bbb16584SMilan Broz struct dm_crypt_request *dmreq)
1028bbb16584SMilan Broz {
1029bbb16584SMilan Broz int r;
1030bbb16584SMilan Broz
1031bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1032bbb16584SMilan Broz r = crypt_iv_elephant(cc, dmreq);
1033bbb16584SMilan Broz if (r)
1034bbb16584SMilan Broz return r;
1035bbb16584SMilan Broz }
1036bbb16584SMilan Broz
1037bbb16584SMilan Broz return crypt_iv_eboiv_gen(cc, iv, dmreq);
1038bbb16584SMilan Broz }
1039bbb16584SMilan Broz
crypt_iv_elephant_post(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)1040bbb16584SMilan Broz static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv,
1041bbb16584SMilan Broz struct dm_crypt_request *dmreq)
1042bbb16584SMilan Broz {
1043bbb16584SMilan Broz if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
1044bbb16584SMilan Broz return crypt_iv_elephant(cc, dmreq);
1045bbb16584SMilan Broz
1046bbb16584SMilan Broz return 0;
1047bbb16584SMilan Broz }
1048bbb16584SMilan Broz
crypt_iv_elephant_init(struct crypt_config * cc)1049bbb16584SMilan Broz static int crypt_iv_elephant_init(struct crypt_config *cc)
1050bbb16584SMilan Broz {
1051bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1052bbb16584SMilan Broz int key_offset = cc->key_size - cc->key_extra_size;
1053bbb16584SMilan Broz
1054bbb16584SMilan Broz return crypto_skcipher_setkey(elephant->tfm, &cc->key[key_offset], cc->key_extra_size);
1055bbb16584SMilan Broz }
1056bbb16584SMilan Broz
crypt_iv_elephant_wipe(struct crypt_config * cc)1057bbb16584SMilan Broz static int crypt_iv_elephant_wipe(struct crypt_config *cc)
1058bbb16584SMilan Broz {
1059bbb16584SMilan Broz struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1060bbb16584SMilan Broz u8 key[ELEPHANT_MAX_KEY_SIZE];
1061bbb16584SMilan Broz
1062bbb16584SMilan Broz memset(key, 0, cc->key_extra_size);
1063bbb16584SMilan Broz return crypto_skcipher_setkey(elephant->tfm, key, cc->key_extra_size);
1064bbb16584SMilan Broz }
1065bbb16584SMilan Broz
10661b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain_ops = {
10671da177e4SLinus Torvalds .generator = crypt_iv_plain_gen
10681da177e4SLinus Torvalds };
10691da177e4SLinus Torvalds
10701b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_plain64_ops = {
107161afef61SMilan Broz .generator = crypt_iv_plain64_gen
107261afef61SMilan Broz };
107361afef61SMilan Broz
10747e3fd855SMilan Broz static const struct crypt_iv_operations crypt_iv_plain64be_ops = {
10757e3fd855SMilan Broz .generator = crypt_iv_plain64be_gen
10767e3fd855SMilan Broz };
10777e3fd855SMilan Broz
10781b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_essiv_ops = {
10791da177e4SLinus Torvalds .generator = crypt_iv_essiv_gen
10801da177e4SLinus Torvalds };
10811da177e4SLinus Torvalds
10821b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_benbi_ops = {
108348527fa7SRik Snel .ctr = crypt_iv_benbi_ctr,
108448527fa7SRik Snel .dtr = crypt_iv_benbi_dtr,
108548527fa7SRik Snel .generator = crypt_iv_benbi_gen
108648527fa7SRik Snel };
10871da177e4SLinus Torvalds
10881b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_null_ops = {
108946b47730SLudwig Nussel .generator = crypt_iv_null_gen
109046b47730SLudwig Nussel };
109146b47730SLudwig Nussel
10921b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_lmk_ops = {
109334745785SMilan Broz .ctr = crypt_iv_lmk_ctr,
109434745785SMilan Broz .dtr = crypt_iv_lmk_dtr,
109534745785SMilan Broz .init = crypt_iv_lmk_init,
109634745785SMilan Broz .wipe = crypt_iv_lmk_wipe,
109734745785SMilan Broz .generator = crypt_iv_lmk_gen,
109834745785SMilan Broz .post = crypt_iv_lmk_post
109934745785SMilan Broz };
110034745785SMilan Broz
11011b1b58f5SJulia Lawall static const struct crypt_iv_operations crypt_iv_tcw_ops = {
1102ed04d981SMilan Broz .ctr = crypt_iv_tcw_ctr,
1103ed04d981SMilan Broz .dtr = crypt_iv_tcw_dtr,
1104ed04d981SMilan Broz .init = crypt_iv_tcw_init,
1105ed04d981SMilan Broz .wipe = crypt_iv_tcw_wipe,
1106ed04d981SMilan Broz .generator = crypt_iv_tcw_gen,
1107ed04d981SMilan Broz .post = crypt_iv_tcw_post
1108ed04d981SMilan Broz };
1109ed04d981SMilan Broz
1110e8dc79d1SRikard Falkeborn static const struct crypt_iv_operations crypt_iv_random_ops = {
1111ef43aa38SMilan Broz .generator = crypt_iv_random_gen
1112ef43aa38SMilan Broz };
1113ef43aa38SMilan Broz
1114e8dc79d1SRikard Falkeborn static const struct crypt_iv_operations crypt_iv_eboiv_ops = {
1115b9411d73SMilan Broz .ctr = crypt_iv_eboiv_ctr,
1116b9411d73SMilan Broz .generator = crypt_iv_eboiv_gen
1117b9411d73SMilan Broz };
1118b9411d73SMilan Broz
1119e8dc79d1SRikard Falkeborn static const struct crypt_iv_operations crypt_iv_elephant_ops = {
1120bbb16584SMilan Broz .ctr = crypt_iv_elephant_ctr,
1121bbb16584SMilan Broz .dtr = crypt_iv_elephant_dtr,
1122bbb16584SMilan Broz .init = crypt_iv_elephant_init,
1123bbb16584SMilan Broz .wipe = crypt_iv_elephant_wipe,
1124bbb16584SMilan Broz .generator = crypt_iv_elephant_gen,
1125bbb16584SMilan Broz .post = crypt_iv_elephant_post
1126bbb16584SMilan Broz };
1127bbb16584SMilan Broz
1128ef43aa38SMilan Broz /*
1129ef43aa38SMilan Broz * Integrity extensions
1130ef43aa38SMilan Broz */
crypt_integrity_aead(struct crypt_config * cc)1131ef43aa38SMilan Broz static bool crypt_integrity_aead(struct crypt_config *cc)
1132ef43aa38SMilan Broz {
1133ef43aa38SMilan Broz return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
1134ef43aa38SMilan Broz }
1135ef43aa38SMilan Broz
crypt_integrity_hmac(struct crypt_config * cc)1136ef43aa38SMilan Broz static bool crypt_integrity_hmac(struct crypt_config *cc)
1137ef43aa38SMilan Broz {
113833d2f09fSMilan Broz return crypt_integrity_aead(cc) && cc->key_mac_size;
1139ef43aa38SMilan Broz }
1140ef43aa38SMilan Broz
1141ef43aa38SMilan Broz /* Get sg containing data */
crypt_get_sg_data(struct crypt_config * cc,struct scatterlist * sg)1142ef43aa38SMilan Broz static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
1143ef43aa38SMilan Broz struct scatterlist *sg)
1144ef43aa38SMilan Broz {
114533d2f09fSMilan Broz if (unlikely(crypt_integrity_aead(cc)))
1146ef43aa38SMilan Broz return &sg[2];
1147ef43aa38SMilan Broz
1148ef43aa38SMilan Broz return sg;
1149ef43aa38SMilan Broz }
1150ef43aa38SMilan Broz
dm_crypt_integrity_io_alloc(struct dm_crypt_io * io,struct bio * bio)1151ef43aa38SMilan Broz static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
1152ef43aa38SMilan Broz {
1153ef43aa38SMilan Broz struct bio_integrity_payload *bip;
1154ef43aa38SMilan Broz unsigned int tag_len;
1155ef43aa38SMilan Broz int ret;
1156ef43aa38SMilan Broz
1157ef43aa38SMilan Broz if (!bio_sectors(bio) || !io->cc->on_disk_tag_size)
1158ef43aa38SMilan Broz return 0;
1159ef43aa38SMilan Broz
1160ef43aa38SMilan Broz bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
1161ef43aa38SMilan Broz if (IS_ERR(bip))
1162ef43aa38SMilan Broz return PTR_ERR(bip);
1163ef43aa38SMilan Broz
1164ff0c129dSMikulas Patocka tag_len = io->cc->on_disk_tag_size * (bio_sectors(bio) >> io->cc->sector_shift);
1165ef43aa38SMilan Broz
1166ef43aa38SMilan Broz bip->bip_iter.bi_sector = io->cc->start + io->sector;
1167ef43aa38SMilan Broz
1168ef43aa38SMilan Broz ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),
1169ef43aa38SMilan Broz tag_len, offset_in_page(io->integrity_metadata));
1170ef43aa38SMilan Broz if (unlikely(ret != tag_len))
1171ef43aa38SMilan Broz return -ENOMEM;
1172ef43aa38SMilan Broz
1173ef43aa38SMilan Broz return 0;
1174ef43aa38SMilan Broz }
1175ef43aa38SMilan Broz
crypt_integrity_ctr(struct crypt_config * cc,struct dm_target * ti)1176ef43aa38SMilan Broz static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
1177ef43aa38SMilan Broz {
1178ef43aa38SMilan Broz #ifdef CONFIG_BLK_DEV_INTEGRITY
1179ef43aa38SMilan Broz struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk);
11807a1cd723SMilan Broz struct mapped_device *md = dm_table_get_md(ti->table);
1181ef43aa38SMilan Broz
1182ef43aa38SMilan Broz /* From now we require underlying device with our integrity profile */
1183ef43aa38SMilan Broz if (!bi || strcasecmp(bi->profile->name, "DM-DIF-EXT-TAG")) {
1184ef43aa38SMilan Broz ti->error = "Integrity profile not supported.";
1185ef43aa38SMilan Broz return -EINVAL;
1186ef43aa38SMilan Broz }
1187ef43aa38SMilan Broz
1188583fe747SMikulas Patocka if (bi->tag_size != cc->on_disk_tag_size ||
1189583fe747SMikulas Patocka bi->tuple_size != cc->on_disk_tag_size) {
1190ef43aa38SMilan Broz ti->error = "Integrity profile tag size mismatch.";
1191ef43aa38SMilan Broz return -EINVAL;
1192ef43aa38SMilan Broz }
1193583fe747SMikulas Patocka if (1 << bi->interval_exp != cc->sector_size) {
1194583fe747SMikulas Patocka ti->error = "Integrity profile sector size mismatch.";
1195583fe747SMikulas Patocka return -EINVAL;
1196583fe747SMikulas Patocka }
1197ef43aa38SMilan Broz
119833d2f09fSMilan Broz if (crypt_integrity_aead(cc)) {
1199ef43aa38SMilan Broz cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size;
12007a1cd723SMilan Broz DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md),
1201ef43aa38SMilan Broz cc->integrity_tag_size, cc->integrity_iv_size);
1202ef43aa38SMilan Broz
1203ef43aa38SMilan Broz if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) {
1204ef43aa38SMilan Broz ti->error = "Integrity AEAD auth tag size is not supported.";
1205ef43aa38SMilan Broz return -EINVAL;
1206ef43aa38SMilan Broz }
1207ef43aa38SMilan Broz } else if (cc->integrity_iv_size)
12087a1cd723SMilan Broz DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md),
1209ef43aa38SMilan Broz cc->integrity_iv_size);
1210ef43aa38SMilan Broz
1211ef43aa38SMilan Broz if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) {
1212ef43aa38SMilan Broz ti->error = "Not enough space for integrity tag in the profile.";
1213ef43aa38SMilan Broz return -EINVAL;
1214ef43aa38SMilan Broz }
1215ef43aa38SMilan Broz
1216ef43aa38SMilan Broz return 0;
1217ef43aa38SMilan Broz #else
1218ef43aa38SMilan Broz ti->error = "Integrity profile not supported.";
1219ef43aa38SMilan Broz return -EINVAL;
1220ef43aa38SMilan Broz #endif
1221ef43aa38SMilan Broz }
1222ef43aa38SMilan Broz
crypt_convert_init(struct crypt_config * cc,struct convert_context * ctx,struct bio * bio_out,struct bio * bio_in,sector_t sector)1223d469f841SMilan Broz static void crypt_convert_init(struct crypt_config *cc,
1224d469f841SMilan Broz struct convert_context *ctx,
12251da177e4SLinus Torvalds struct bio *bio_out, struct bio *bio_in,
1226fcd369daSMilan Broz sector_t sector)
12271da177e4SLinus Torvalds {
12281da177e4SLinus Torvalds ctx->bio_in = bio_in;
12291da177e4SLinus Torvalds ctx->bio_out = bio_out;
1230003b5c57SKent Overstreet if (bio_in)
1231003b5c57SKent Overstreet ctx->iter_in = bio_in->bi_iter;
1232003b5c57SKent Overstreet if (bio_out)
1233003b5c57SKent Overstreet ctx->iter_out = bio_out->bi_iter;
1234c66029f4SMikulas Patocka ctx->cc_sector = sector + cc->iv_offset;
123543d69034SMilan Broz init_completion(&ctx->restart);
12361da177e4SLinus Torvalds }
12371da177e4SLinus Torvalds
dmreq_of_req(struct crypt_config * cc,void * req)1238b2174eebSHuang Ying static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
1239ef43aa38SMilan Broz void *req)
1240b2174eebSHuang Ying {
1241b2174eebSHuang Ying return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
1242b2174eebSHuang Ying }
1243b2174eebSHuang Ying
req_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1244ef43aa38SMilan Broz static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)
1245b2174eebSHuang Ying {
1246ef43aa38SMilan Broz return (void *)((char *)dmreq - cc->dmreq_start);
1247b2174eebSHuang Ying }
1248b2174eebSHuang Ying
iv_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)12492dc5327dSMilan Broz static u8 *iv_of_dmreq(struct crypt_config *cc,
12502dc5327dSMilan Broz struct dm_crypt_request *dmreq)
12512dc5327dSMilan Broz {
125233d2f09fSMilan Broz if (crypt_integrity_aead(cc))
1253ef43aa38SMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1254ef43aa38SMilan Broz crypto_aead_alignmask(any_tfm_aead(cc)) + 1);
1255ef43aa38SMilan Broz else
12562dc5327dSMilan Broz return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1257bbdb23b5SHerbert Xu crypto_skcipher_alignmask(any_tfm(cc)) + 1);
12582dc5327dSMilan Broz }
12592dc5327dSMilan Broz
org_iv_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1260ef43aa38SMilan Broz static u8 *org_iv_of_dmreq(struct crypt_config *cc,
1261ef43aa38SMilan Broz struct dm_crypt_request *dmreq)
1262ef43aa38SMilan Broz {
1263ef43aa38SMilan Broz return iv_of_dmreq(cc, dmreq) + cc->iv_size;
1264ef43aa38SMilan Broz }
1265ef43aa38SMilan Broz
org_sector_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1266c13b5487SChristoph Hellwig static __le64 *org_sector_of_dmreq(struct crypt_config *cc,
1267ef43aa38SMilan Broz struct dm_crypt_request *dmreq)
1268ef43aa38SMilan Broz {
1269ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;
12700ef0b471SHeinz Mauelshagen
1271c13b5487SChristoph Hellwig return (__le64 *) ptr;
1272ef43aa38SMilan Broz }
1273ef43aa38SMilan Broz
org_tag_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1274ef43aa38SMilan Broz static unsigned int *org_tag_of_dmreq(struct crypt_config *cc,
1275ef43aa38SMilan Broz struct dm_crypt_request *dmreq)
1276ef43aa38SMilan Broz {
1277ef43aa38SMilan Broz u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +
1278ef43aa38SMilan Broz cc->iv_size + sizeof(uint64_t);
12790ef0b471SHeinz Mauelshagen
1280ef43aa38SMilan Broz return (unsigned int *)ptr;
1281ef43aa38SMilan Broz }
1282ef43aa38SMilan Broz
tag_from_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1283ef43aa38SMilan Broz static void *tag_from_dmreq(struct crypt_config *cc,
1284ef43aa38SMilan Broz struct dm_crypt_request *dmreq)
1285ef43aa38SMilan Broz {
1286ef43aa38SMilan Broz struct convert_context *ctx = dmreq->ctx;
1287ef43aa38SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1288ef43aa38SMilan Broz
1289ef43aa38SMilan Broz return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *
1290ef43aa38SMilan Broz cc->on_disk_tag_size];
1291ef43aa38SMilan Broz }
1292ef43aa38SMilan Broz
iv_tag_from_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1293ef43aa38SMilan Broz static void *iv_tag_from_dmreq(struct crypt_config *cc,
1294ef43aa38SMilan Broz struct dm_crypt_request *dmreq)
1295ef43aa38SMilan Broz {
1296ef43aa38SMilan Broz return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;
1297ef43aa38SMilan Broz }
1298ef43aa38SMilan Broz
crypt_convert_block_aead(struct crypt_config * cc,struct convert_context * ctx,struct aead_request * req,unsigned int tag_offset)1299ef43aa38SMilan Broz static int crypt_convert_block_aead(struct crypt_config *cc,
13003a7f6c99SMilan Broz struct convert_context *ctx,
1301ef43aa38SMilan Broz struct aead_request *req,
1302ef43aa38SMilan Broz unsigned int tag_offset)
130301482b76SMilan Broz {
1304003b5c57SKent Overstreet struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1305003b5c57SKent Overstreet struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
13063a7f6c99SMilan Broz struct dm_crypt_request *dmreq;
1307ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv, *tag;
1308c13b5487SChristoph Hellwig __le64 *sector;
1309ef43aa38SMilan Broz int r = 0;
1310ef43aa38SMilan Broz
1311ef43aa38SMilan Broz BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size);
131201482b76SMilan Broz
13138f0009a2SMilan Broz /* Reject unexpected unaligned bio. */
13140440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
13158f0009a2SMilan Broz return -EIO;
131601482b76SMilan Broz
1317b2174eebSHuang Ying dmreq = dmreq_of_req(cc, req);
1318c66029f4SMikulas Patocka dmreq->iv_sector = ctx->cc_sector;
13198f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1320ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift;
1321b2174eebSHuang Ying dmreq->ctx = ctx;
132201482b76SMilan Broz
1323ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset;
132401482b76SMilan Broz
1325ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq);
1326ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1327ef43aa38SMilan Broz
1328ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq);
1329ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq);
1330ef43aa38SMilan Broz tag = tag_from_dmreq(cc, dmreq);
1331ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq);
1332ef43aa38SMilan Broz
1333ef43aa38SMilan Broz /* AEAD request:
1334ef43aa38SMilan Broz * |----- AAD -------|------ DATA -------|-- AUTH TAG --|
1335ef43aa38SMilan Broz * | (authenticated) | (auth+encryption) | |
1336ef43aa38SMilan Broz * | sector_LE | IV | sector in/out | tag in/out |
1337ef43aa38SMilan Broz */
1338ef43aa38SMilan Broz sg_init_table(dmreq->sg_in, 4);
1339ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t));
1340ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size);
13418f0009a2SMilan Broz sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
1342ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size);
1343ef43aa38SMilan Broz
1344ef43aa38SMilan Broz sg_init_table(dmreq->sg_out, 4);
1345ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t));
1346ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size);
13478f0009a2SMilan Broz sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
1348ef43aa38SMilan Broz sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size);
134901482b76SMilan Broz
13503a7f6c99SMilan Broz if (cc->iv_gen_ops) {
1351ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */
1352ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1353ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->iv_size);
1354ef43aa38SMilan Broz } else {
1355ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
13563a7f6c99SMilan Broz if (r < 0)
13573a7f6c99SMilan Broz return r;
1358ef43aa38SMilan Broz /* Store generated IV in integrity metadata */
1359ef43aa38SMilan Broz if (cc->integrity_iv_size)
1360ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->iv_size);
1361ef43aa38SMilan Broz }
1362ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */
1363ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size);
1364ef43aa38SMilan Broz }
1365ef43aa38SMilan Broz
1366ef43aa38SMilan Broz aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size);
1367ef43aa38SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE) {
1368ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
13698f0009a2SMilan Broz cc->sector_size, iv);
1370ef43aa38SMilan Broz r = crypto_aead_encrypt(req);
1371ef43aa38SMilan Broz if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size)
1372ef43aa38SMilan Broz memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,
1373ef43aa38SMilan Broz cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size));
1374ef43aa38SMilan Broz } else {
1375ef43aa38SMilan Broz aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
13768f0009a2SMilan Broz cc->sector_size + cc->integrity_tag_size, iv);
1377ef43aa38SMilan Broz r = crypto_aead_decrypt(req);
1378ef43aa38SMilan Broz }
1379ef43aa38SMilan Broz
1380f710126cSMilan Broz if (r == -EBADMSG) {
138158d0f180SMichael Weiß sector_t s = le64_to_cpu(*sector);
138258d0f180SMichael Weiß
13830f6cf136SMikulas Patocka ctx->aead_failed = true;
13840f6cf136SMikulas Patocka if (ctx->aead_recheck) {
138566671719SChristoph Hellwig DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
138666671719SChristoph Hellwig ctx->bio_in->bi_bdev, s);
138758d0f180SMichael Weiß dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
138858d0f180SMichael Weiß ctx->bio_in, s, 0);
1389f710126cSMilan Broz }
13900f6cf136SMikulas Patocka }
1391ef43aa38SMilan Broz
1392ef43aa38SMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1393ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1394ef43aa38SMilan Broz
13958f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
13968f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
1397ef43aa38SMilan Broz
1398ef43aa38SMilan Broz return r;
13993a7f6c99SMilan Broz }
14003a7f6c99SMilan Broz
crypt_convert_block_skcipher(struct crypt_config * cc,struct convert_context * ctx,struct skcipher_request * req,unsigned int tag_offset)1401ef43aa38SMilan Broz static int crypt_convert_block_skcipher(struct crypt_config *cc,
1402ef43aa38SMilan Broz struct convert_context *ctx,
1403ef43aa38SMilan Broz struct skcipher_request *req,
1404ef43aa38SMilan Broz unsigned int tag_offset)
1405ef43aa38SMilan Broz {
1406ef43aa38SMilan Broz struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1407ef43aa38SMilan Broz struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1408ef43aa38SMilan Broz struct scatterlist *sg_in, *sg_out;
1409ef43aa38SMilan Broz struct dm_crypt_request *dmreq;
1410ef43aa38SMilan Broz u8 *iv, *org_iv, *tag_iv;
1411c13b5487SChristoph Hellwig __le64 *sector;
1412ef43aa38SMilan Broz int r = 0;
1413ef43aa38SMilan Broz
14148f0009a2SMilan Broz /* Reject unexpected unaligned bio. */
14150440d5c0SMikulas Patocka if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
14168f0009a2SMilan Broz return -EIO;
14178f0009a2SMilan Broz
1418ef43aa38SMilan Broz dmreq = dmreq_of_req(cc, req);
1419ef43aa38SMilan Broz dmreq->iv_sector = ctx->cc_sector;
14208f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1421ff3af92bSMikulas Patocka dmreq->iv_sector >>= cc->sector_shift;
1422ef43aa38SMilan Broz dmreq->ctx = ctx;
1423ef43aa38SMilan Broz
1424ef43aa38SMilan Broz *org_tag_of_dmreq(cc, dmreq) = tag_offset;
1425ef43aa38SMilan Broz
1426ef43aa38SMilan Broz iv = iv_of_dmreq(cc, dmreq);
1427ef43aa38SMilan Broz org_iv = org_iv_of_dmreq(cc, dmreq);
1428ef43aa38SMilan Broz tag_iv = iv_tag_from_dmreq(cc, dmreq);
1429ef43aa38SMilan Broz
1430ef43aa38SMilan Broz sector = org_sector_of_dmreq(cc, dmreq);
1431ef43aa38SMilan Broz *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1432ef43aa38SMilan Broz
1433ef43aa38SMilan Broz /* For skcipher we use only the first sg item */
1434ef43aa38SMilan Broz sg_in = &dmreq->sg_in[0];
1435ef43aa38SMilan Broz sg_out = &dmreq->sg_out[0];
1436ef43aa38SMilan Broz
1437ef43aa38SMilan Broz sg_init_table(sg_in, 1);
14388f0009a2SMilan Broz sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
1439ef43aa38SMilan Broz
1440ef43aa38SMilan Broz sg_init_table(sg_out, 1);
14418f0009a2SMilan Broz sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
1442ef43aa38SMilan Broz
1443ef43aa38SMilan Broz if (cc->iv_gen_ops) {
1444ef43aa38SMilan Broz /* For READs use IV stored in integrity metadata */
1445ef43aa38SMilan Broz if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1446ef43aa38SMilan Broz memcpy(org_iv, tag_iv, cc->integrity_iv_size);
1447ef43aa38SMilan Broz } else {
1448ef43aa38SMilan Broz r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1449ef43aa38SMilan Broz if (r < 0)
1450ef43aa38SMilan Broz return r;
1451bbb16584SMilan Broz /* Data can be already preprocessed in generator */
1452bbb16584SMilan Broz if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))
1453bbb16584SMilan Broz sg_in = sg_out;
1454ef43aa38SMilan Broz /* Store generated IV in integrity metadata */
1455ef43aa38SMilan Broz if (cc->integrity_iv_size)
1456ef43aa38SMilan Broz memcpy(tag_iv, org_iv, cc->integrity_iv_size);
1457ef43aa38SMilan Broz }
1458ef43aa38SMilan Broz /* Working copy of IV, to be modified in crypto API */
1459ef43aa38SMilan Broz memcpy(iv, org_iv, cc->iv_size);
1460ef43aa38SMilan Broz }
1461ef43aa38SMilan Broz
14628f0009a2SMilan Broz skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv);
14633a7f6c99SMilan Broz
14643a7f6c99SMilan Broz if (bio_data_dir(ctx->bio_in) == WRITE)
1465bbdb23b5SHerbert Xu r = crypto_skcipher_encrypt(req);
14663a7f6c99SMilan Broz else
1467bbdb23b5SHerbert Xu r = crypto_skcipher_decrypt(req);
14683a7f6c99SMilan Broz
14692dc5327dSMilan Broz if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1470ef43aa38SMilan Broz r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1471ef43aa38SMilan Broz
14728f0009a2SMilan Broz bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
14738f0009a2SMilan Broz bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
14742dc5327dSMilan Broz
14753a7f6c99SMilan Broz return r;
147601482b76SMilan Broz }
147701482b76SMilan Broz
1478dcfe653dSHerbert Xu static void kcryptd_async_done(void *async_req, int error);
1479c0297721SAndi Kleen
crypt_alloc_req_skcipher(struct crypt_config * cc,struct convert_context * ctx)1480d68b2958SIgnat Korchagin static int crypt_alloc_req_skcipher(struct crypt_config *cc,
1481ddd42edfSMilan Broz struct convert_context *ctx)
1482ddd42edfSMilan Broz {
148386a3238cSHeinz Mauelshagen unsigned int key_index = ctx->cc_sector & (cc->tfms_count - 1);
1484c0297721SAndi Kleen
1485d68b2958SIgnat Korchagin if (!ctx->r.req) {
1486d68b2958SIgnat Korchagin ctx->r.req = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1487ef43aa38SMilan Broz if (!ctx->r.req)
1488d68b2958SIgnat Korchagin return -ENOMEM;
1489d68b2958SIgnat Korchagin }
1490c0297721SAndi Kleen
1491ef43aa38SMilan Broz skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]);
149254cea3f6SMilan Broz
149354cea3f6SMilan Broz /*
149454cea3f6SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
149554cea3f6SMilan Broz * requests if driver request queue is full.
149654cea3f6SMilan Broz */
1497ef43aa38SMilan Broz skcipher_request_set_callback(ctx->r.req,
1498432061b3SMikulas Patocka CRYPTO_TFM_REQ_MAY_BACKLOG,
1499ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req));
1500d68b2958SIgnat Korchagin
1501d68b2958SIgnat Korchagin return 0;
1502ddd42edfSMilan Broz }
1503ddd42edfSMilan Broz
crypt_alloc_req_aead(struct crypt_config * cc,struct convert_context * ctx)1504d68b2958SIgnat Korchagin static int crypt_alloc_req_aead(struct crypt_config *cc,
1505ef43aa38SMilan Broz struct convert_context *ctx)
1506ef43aa38SMilan Broz {
1507004b8ae9SIgnat Korchagin if (!ctx->r.req_aead) {
1508004b8ae9SIgnat Korchagin ctx->r.req_aead = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1509004b8ae9SIgnat Korchagin if (!ctx->r.req_aead)
1510d68b2958SIgnat Korchagin return -ENOMEM;
1511d68b2958SIgnat Korchagin }
1512ef43aa38SMilan Broz
1513ef43aa38SMilan Broz aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]);
1514ef43aa38SMilan Broz
1515ef43aa38SMilan Broz /*
1516ef43aa38SMilan Broz * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1517ef43aa38SMilan Broz * requests if driver request queue is full.
1518ef43aa38SMilan Broz */
1519ef43aa38SMilan Broz aead_request_set_callback(ctx->r.req_aead,
1520432061b3SMikulas Patocka CRYPTO_TFM_REQ_MAY_BACKLOG,
1521ef43aa38SMilan Broz kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead));
1522d68b2958SIgnat Korchagin
1523d68b2958SIgnat Korchagin return 0;
1524ef43aa38SMilan Broz }
1525ef43aa38SMilan Broz
crypt_alloc_req(struct crypt_config * cc,struct convert_context * ctx)1526d68b2958SIgnat Korchagin static int crypt_alloc_req(struct crypt_config *cc,
1527ef43aa38SMilan Broz struct convert_context *ctx)
1528ef43aa38SMilan Broz {
152933d2f09fSMilan Broz if (crypt_integrity_aead(cc))
1530d68b2958SIgnat Korchagin return crypt_alloc_req_aead(cc, ctx);
1531ef43aa38SMilan Broz else
1532d68b2958SIgnat Korchagin return crypt_alloc_req_skcipher(cc, ctx);
1533ef43aa38SMilan Broz }
1534ef43aa38SMilan Broz
crypt_free_req_skcipher(struct crypt_config * cc,struct skcipher_request * req,struct bio * base_bio)1535ef43aa38SMilan Broz static void crypt_free_req_skcipher(struct crypt_config *cc,
1536bbdb23b5SHerbert Xu struct skcipher_request *req, struct bio *base_bio)
1537298a9fa0SMikulas Patocka {
1538298a9fa0SMikulas Patocka struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
1539298a9fa0SMikulas Patocka
1540bbdb23b5SHerbert Xu if ((struct skcipher_request *)(io + 1) != req)
15416f1c819cSKent Overstreet mempool_free(req, &cc->req_pool);
1542298a9fa0SMikulas Patocka }
1543298a9fa0SMikulas Patocka
crypt_free_req_aead(struct crypt_config * cc,struct aead_request * req,struct bio * base_bio)1544ef43aa38SMilan Broz static void crypt_free_req_aead(struct crypt_config *cc,
1545ef43aa38SMilan Broz struct aead_request *req, struct bio *base_bio)
1546ef43aa38SMilan Broz {
1547ef43aa38SMilan Broz struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
1548ef43aa38SMilan Broz
1549ef43aa38SMilan Broz if ((struct aead_request *)(io + 1) != req)
15506f1c819cSKent Overstreet mempool_free(req, &cc->req_pool);
1551ef43aa38SMilan Broz }
1552ef43aa38SMilan Broz
crypt_free_req(struct crypt_config * cc,void * req,struct bio * base_bio)1553ef43aa38SMilan Broz static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)
1554ef43aa38SMilan Broz {
155533d2f09fSMilan Broz if (crypt_integrity_aead(cc))
1556ef43aa38SMilan Broz crypt_free_req_aead(cc, req, base_bio);
1557ef43aa38SMilan Broz else
1558ef43aa38SMilan Broz crypt_free_req_skcipher(cc, req, base_bio);
1559ef43aa38SMilan Broz }
1560ef43aa38SMilan Broz
15611da177e4SLinus Torvalds /*
15621da177e4SLinus Torvalds * Encrypt / decrypt data from one bio to another one (can be the same one)
15631da177e4SLinus Torvalds */
crypt_convert(struct crypt_config * cc,struct convert_context * ctx,bool atomic,bool reset_pending)15644e4cbee9SChristoph Hellwig static blk_status_t crypt_convert(struct crypt_config *cc,
15658abec36dSIgnat Korchagin struct convert_context *ctx, bool atomic, bool reset_pending)
15661da177e4SLinus Torvalds {
1567ef43aa38SMilan Broz unsigned int tag_offset = 0;
1568ff3af92bSMikulas Patocka unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
15693f1e9070SMilan Broz int r;
15701da177e4SLinus Torvalds
15718abec36dSIgnat Korchagin /*
15728abec36dSIgnat Korchagin * if reset_pending is set we are dealing with the bio for the first time,
15738abec36dSIgnat Korchagin * else we're continuing to work on the previous bio, so don't mess with
15748abec36dSIgnat Korchagin * the cc_pending counter
15758abec36dSIgnat Korchagin */
15768abec36dSIgnat Korchagin if (reset_pending)
157740b6229bSMikulas Patocka atomic_set(&ctx->cc_pending, 1);
1578c8081618SMilan Broz
1579003b5c57SKent Overstreet while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
15801da177e4SLinus Torvalds
1581d68b2958SIgnat Korchagin r = crypt_alloc_req(cc, ctx);
1582d68b2958SIgnat Korchagin if (r) {
1583d68b2958SIgnat Korchagin complete(&ctx->restart);
1584d68b2958SIgnat Korchagin return BLK_STS_DEV_RESOURCE;
1585d68b2958SIgnat Korchagin }
1586d68b2958SIgnat Korchagin
158740b6229bSMikulas Patocka atomic_inc(&ctx->cc_pending);
15883f1e9070SMilan Broz
158933d2f09fSMilan Broz if (crypt_integrity_aead(cc))
1590ef43aa38SMilan Broz r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset);
1591ef43aa38SMilan Broz else
1592ef43aa38SMilan Broz r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset);
15933a7f6c99SMilan Broz
15943a7f6c99SMilan Broz switch (r) {
159554cea3f6SMilan Broz /*
159654cea3f6SMilan Broz * The request was queued by a crypto driver
159754cea3f6SMilan Broz * but the driver request queue is full, let's wait.
159854cea3f6SMilan Broz */
15993a7f6c99SMilan Broz case -EBUSY:
16008abec36dSIgnat Korchagin if (in_interrupt()) {
16018abec36dSIgnat Korchagin if (try_wait_for_completion(&ctx->restart)) {
16028abec36dSIgnat Korchagin /*
16038abec36dSIgnat Korchagin * we don't have to block to wait for completion,
16048abec36dSIgnat Korchagin * so proceed
16058abec36dSIgnat Korchagin */
16068abec36dSIgnat Korchagin } else {
16078abec36dSIgnat Korchagin /*
16088abec36dSIgnat Korchagin * we can't wait for completion without blocking
16098abec36dSIgnat Korchagin * exit and continue processing in a workqueue
16108abec36dSIgnat Korchagin */
16118abec36dSIgnat Korchagin ctx->r.req = NULL;
16128abec36dSIgnat Korchagin ctx->cc_sector += sector_step;
16138abec36dSIgnat Korchagin tag_offset++;
16148abec36dSIgnat Korchagin return BLK_STS_DEV_RESOURCE;
16158abec36dSIgnat Korchagin }
16168abec36dSIgnat Korchagin } else {
16173a7f6c99SMilan Broz wait_for_completion(&ctx->restart);
16188abec36dSIgnat Korchagin }
161916735d02SWolfram Sang reinit_completion(&ctx->restart);
1620df561f66SGustavo A. R. Silva fallthrough;
162154cea3f6SMilan Broz /*
162254cea3f6SMilan Broz * The request is queued and processed asynchronously,
162354cea3f6SMilan Broz * completion function kcryptd_async_done() will be called.
162454cea3f6SMilan Broz */
1625c0403ec0SRabin Vincent case -EINPROGRESS:
1626ef43aa38SMilan Broz ctx->r.req = NULL;
16278f0009a2SMilan Broz ctx->cc_sector += sector_step;
1628583fe747SMikulas Patocka tag_offset++;
16293a7f6c99SMilan Broz continue;
163054cea3f6SMilan Broz /*
163154cea3f6SMilan Broz * The request was already processed (synchronously).
163254cea3f6SMilan Broz */
16333f1e9070SMilan Broz case 0:
163440b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending);
16358f0009a2SMilan Broz ctx->cc_sector += sector_step;
1636583fe747SMikulas Patocka tag_offset++;
163739d42fa9SIgnat Korchagin if (!atomic)
1638c7f1b204SMilan Broz cond_resched();
16393f1e9070SMilan Broz continue;
1640ef43aa38SMilan Broz /*
1641ef43aa38SMilan Broz * There was a data integrity error.
1642ef43aa38SMilan Broz */
1643ef43aa38SMilan Broz case -EBADMSG:
1644ef43aa38SMilan Broz atomic_dec(&ctx->cc_pending);
16454e4cbee9SChristoph Hellwig return BLK_STS_PROTECTION;
1646ef43aa38SMilan Broz /*
1647ef43aa38SMilan Broz * There was an error while processing the request.
1648ef43aa38SMilan Broz */
16493f1e9070SMilan Broz default:
165040b6229bSMikulas Patocka atomic_dec(&ctx->cc_pending);
16514e4cbee9SChristoph Hellwig return BLK_STS_IOERR;
16521da177e4SLinus Torvalds }
16533f1e9070SMilan Broz }
16543f1e9070SMilan Broz
16553f1e9070SMilan Broz return 0;
16563f1e9070SMilan Broz }
16571da177e4SLinus Torvalds
1658cf2f1abfSMikulas Patocka static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
1659cf2f1abfSMikulas Patocka
16601da177e4SLinus Torvalds /*
16611da177e4SLinus Torvalds * Generate a new unfragmented bio with the given size
1662586b286bSMike Snitzer * This should never violate the device limitations (but only because
1663586b286bSMike Snitzer * max_segment_size is being constrained to PAGE_SIZE).
16647145c241SMikulas Patocka *
16657145c241SMikulas Patocka * This function may be called concurrently. If we allocate from the mempool
16667145c241SMikulas Patocka * concurrently, there is a possibility of deadlock. For example, if we have
16677145c241SMikulas Patocka * mempool of 256 pages, two processes, each wanting 256, pages allocate from
16687145c241SMikulas Patocka * the mempool concurrently, it may deadlock in a situation where both processes
16697145c241SMikulas Patocka * have allocated 128 pages and the mempool is exhausted.
16707145c241SMikulas Patocka *
16717145c241SMikulas Patocka * In order to avoid this scenario we allocate the pages under a mutex.
16727145c241SMikulas Patocka *
16737145c241SMikulas Patocka * In order to not degrade performance with excessive locking, we try
16747145c241SMikulas Patocka * non-blocking allocations without a mutex first but on failure we fallback
16757145c241SMikulas Patocka * to blocking allocations with a mutex.
16765054e778SMikulas Patocka *
16775054e778SMikulas Patocka * In order to reduce allocation overhead, we try to allocate compound pages in
16785054e778SMikulas Patocka * the first pass. If they are not available, we fall back to the mempool.
16791da177e4SLinus Torvalds */
crypt_alloc_buffer(struct dm_crypt_io * io,unsigned int size)168086a3238cSHeinz Mauelshagen static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size)
16811da177e4SLinus Torvalds {
168249a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
16838b004457SMilan Broz struct bio *clone;
16841da177e4SLinus Torvalds unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
16857145c241SMikulas Patocka gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
16865054e778SMikulas Patocka unsigned int remaining_size;
1687dfa1898cSMikulas Patocka unsigned int order = MAX_ORDER;
16881da177e4SLinus Torvalds
16897145c241SMikulas Patocka retry:
1690d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
16917145c241SMikulas Patocka mutex_lock(&cc->bio_alloc_lock);
16927145c241SMikulas Patocka
1693609be106SChristoph Hellwig clone = bio_alloc_bioset(cc->dev->bdev, nr_iovecs, io->base_bio->bi_opf,
1694609be106SChristoph Hellwig GFP_NOIO, &cc->bs);
16953f868c09SChristoph Hellwig clone->bi_private = io;
16963f868c09SChristoph Hellwig clone->bi_end_io = crypt_endio;
16976a24c718SMilan Broz
16987145c241SMikulas Patocka remaining_size = size;
16997145c241SMikulas Patocka
17005054e778SMikulas Patocka while (remaining_size) {
17015054e778SMikulas Patocka struct page *pages;
17025054e778SMikulas Patocka unsigned size_to_add;
17035054e778SMikulas Patocka unsigned remaining_order = __fls((remaining_size + PAGE_SIZE - 1) >> PAGE_SHIFT);
17045054e778SMikulas Patocka order = min(order, remaining_order);
17055054e778SMikulas Patocka
17065054e778SMikulas Patocka while (order > 0) {
17070b1f2f89SMikulas Patocka if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) +
17080b1f2f89SMikulas Patocka (1 << order) > dm_crypt_pages_per_client))
17090b1f2f89SMikulas Patocka goto decrease_order;
17105054e778SMikulas Patocka pages = alloc_pages(gfp_mask
17115054e778SMikulas Patocka | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | __GFP_COMP,
17125054e778SMikulas Patocka order);
17130b1f2f89SMikulas Patocka if (likely(pages != NULL)) {
17140b1f2f89SMikulas Patocka percpu_counter_add(&cc->n_allocated_pages, 1 << order);
17155054e778SMikulas Patocka goto have_pages;
17160b1f2f89SMikulas Patocka }
17170b1f2f89SMikulas Patocka decrease_order:
17185054e778SMikulas Patocka order--;
17195054e778SMikulas Patocka }
17205054e778SMikulas Patocka
17215054e778SMikulas Patocka pages = mempool_alloc(&cc->page_pool, gfp_mask);
17225054e778SMikulas Patocka if (!pages) {
17237145c241SMikulas Patocka crypt_free_buffer_pages(cc, clone);
17247145c241SMikulas Patocka bio_put(clone);
1725d0164adcSMel Gorman gfp_mask |= __GFP_DIRECT_RECLAIM;
17265054e778SMikulas Patocka order = 0;
17277145c241SMikulas Patocka goto retry;
17287145c241SMikulas Patocka }
17291da177e4SLinus Torvalds
17305054e778SMikulas Patocka have_pages:
17315054e778SMikulas Patocka size_to_add = min((unsigned)PAGE_SIZE << order, remaining_size);
17325054e778SMikulas Patocka __bio_add_page(clone, pages, size_to_add, 0);
17335054e778SMikulas Patocka remaining_size -= size_to_add;
17341da177e4SLinus Torvalds }
17351da177e4SLinus Torvalds
1736ef43aa38SMilan Broz /* Allocate space for integrity tags */
1737ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) {
1738ef43aa38SMilan Broz crypt_free_buffer_pages(cc, clone);
1739ef43aa38SMilan Broz bio_put(clone);
1740ef43aa38SMilan Broz clone = NULL;
1741ef43aa38SMilan Broz }
174253db984eSChristoph Hellwig
1743d0164adcSMel Gorman if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
17447145c241SMikulas Patocka mutex_unlock(&cc->bio_alloc_lock);
17457145c241SMikulas Patocka
17468b004457SMilan Broz return clone;
17471da177e4SLinus Torvalds }
17481da177e4SLinus Torvalds
crypt_free_buffer_pages(struct crypt_config * cc,struct bio * clone)1749644bd2f0SNeil Brown static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
17501da177e4SLinus Torvalds {
17515054e778SMikulas Patocka struct folio_iter fi;
17521da177e4SLinus Torvalds
17535054e778SMikulas Patocka if (clone->bi_vcnt > 0) { /* bio_for_each_folio_all crashes with an empty bio */
17545054e778SMikulas Patocka bio_for_each_folio_all(fi, clone) {
17550b1f2f89SMikulas Patocka if (folio_test_large(fi.folio)) {
17560b1f2f89SMikulas Patocka percpu_counter_sub(&cc->n_allocated_pages,
17570b1f2f89SMikulas Patocka 1 << folio_order(fi.folio));
17585054e778SMikulas Patocka folio_put(fi.folio);
17590b1f2f89SMikulas Patocka } else {
17605054e778SMikulas Patocka mempool_free(&fi.folio->page, &cc->page_pool);
17615054e778SMikulas Patocka }
17621da177e4SLinus Torvalds }
17631da177e4SLinus Torvalds }
17640b1f2f89SMikulas Patocka }
17651da177e4SLinus Torvalds
crypt_io_init(struct dm_crypt_io * io,struct crypt_config * cc,struct bio * bio,sector_t sector)1766298a9fa0SMikulas Patocka static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
1767dc440d1eSMilan Broz struct bio *bio, sector_t sector)
1768dc440d1eSMilan Broz {
176949a8a920SAlasdair G Kergon io->cc = cc;
1770dc440d1eSMilan Broz io->base_bio = bio;
1771dc440d1eSMilan Broz io->sector = sector;
1772dc440d1eSMilan Broz io->error = 0;
17730f6cf136SMikulas Patocka io->ctx.aead_recheck = false;
17740f6cf136SMikulas Patocka io->ctx.aead_failed = false;
1775ef43aa38SMilan Broz io->ctx.r.req = NULL;
1776ef43aa38SMilan Broz io->integrity_metadata = NULL;
1777ef43aa38SMilan Broz io->integrity_metadata_from_pool = false;
177840b6229bSMikulas Patocka atomic_set(&io->io_pending, 0);
1779dc440d1eSMilan Broz }
1780dc440d1eSMilan Broz
crypt_inc_pending(struct dm_crypt_io * io)17813e1a8bddSMilan Broz static void crypt_inc_pending(struct dm_crypt_io *io)
17823e1a8bddSMilan Broz {
178340b6229bSMikulas Patocka atomic_inc(&io->io_pending);
17843e1a8bddSMilan Broz }
17853e1a8bddSMilan Broz
17860f6cf136SMikulas Patocka static void kcryptd_queue_read(struct dm_crypt_io *io);
17870f6cf136SMikulas Patocka
17881da177e4SLinus Torvalds /*
17891da177e4SLinus Torvalds * One of the bios was finished. Check for completion of
17901da177e4SLinus Torvalds * the whole request and correctly clean up the buffer.
17911da177e4SLinus Torvalds */
crypt_dec_pending(struct dm_crypt_io * io)17925742fd77SMilan Broz static void crypt_dec_pending(struct dm_crypt_io *io)
17931da177e4SLinus Torvalds {
179449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
1795b35f8caaSMilan Broz struct bio *base_bio = io->base_bio;
17964e4cbee9SChristoph Hellwig blk_status_t error = io->error;
17971da177e4SLinus Torvalds
179840b6229bSMikulas Patocka if (!atomic_dec_and_test(&io->io_pending))
17991da177e4SLinus Torvalds return;
18001da177e4SLinus Torvalds
18010f6cf136SMikulas Patocka if (likely(!io->ctx.aead_recheck) && unlikely(io->ctx.aead_failed) &&
18020f6cf136SMikulas Patocka cc->on_disk_tag_size && bio_data_dir(base_bio) == READ) {
18030f6cf136SMikulas Patocka io->ctx.aead_recheck = true;
18040f6cf136SMikulas Patocka io->ctx.aead_failed = false;
18050f6cf136SMikulas Patocka io->error = 0;
18060f6cf136SMikulas Patocka kcryptd_queue_read(io);
18070f6cf136SMikulas Patocka return;
18080f6cf136SMikulas Patocka }
18090f6cf136SMikulas Patocka
1810ef43aa38SMilan Broz if (io->ctx.r.req)
1811ef43aa38SMilan Broz crypt_free_req(cc, io->ctx.r.req, base_bio);
1812ef43aa38SMilan Broz
1813ef43aa38SMilan Broz if (unlikely(io->integrity_metadata_from_pool))
18146f1c819cSKent Overstreet mempool_free(io->integrity_metadata, &io->cc->tag_pool);
1815ef43aa38SMilan Broz else
1816ef43aa38SMilan Broz kfree(io->integrity_metadata);
1817b35f8caaSMilan Broz
18184e4cbee9SChristoph Hellwig base_bio->bi_status = error;
18198e14f610SIgnat Korchagin
1820d9a02e01SMike Snitzer bio_endio(base_bio);
18211da177e4SLinus Torvalds }
18221da177e4SLinus Torvalds
18231da177e4SLinus Torvalds /*
1824cabf08e4SMilan Broz * kcryptd/kcryptd_io:
18251da177e4SLinus Torvalds *
18261da177e4SLinus Torvalds * Needed because it would be very unwise to do decryption in an
182723541d2dSMilan Broz * interrupt context.
1828cabf08e4SMilan Broz *
1829cabf08e4SMilan Broz * kcryptd performs the actual encryption or decryption.
1830cabf08e4SMilan Broz *
1831cabf08e4SMilan Broz * kcryptd_io performs the IO submission.
1832cabf08e4SMilan Broz *
1833cabf08e4SMilan Broz * They must be separated as otherwise the final stages could be
1834cabf08e4SMilan Broz * starved by new requests which can block in the first stages due
1835cabf08e4SMilan Broz * to memory allocation.
1836c0297721SAndi Kleen *
1837c0297721SAndi Kleen * The work is done per CPU global for all dm-crypt instances.
1838c0297721SAndi Kleen * They should not depend on each other and do not block.
18391da177e4SLinus Torvalds */
crypt_endio(struct bio * clone)18404246a0b6SChristoph Hellwig static void crypt_endio(struct bio *clone)
18418b004457SMilan Broz {
1842028867acSAlasdair G Kergon struct dm_crypt_io *io = clone->bi_private;
184349a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
184486a3238cSHeinz Mauelshagen unsigned int rw = bio_data_dir(clone);
18450f6cf136SMikulas Patocka blk_status_t error = clone->bi_status;
18460f6cf136SMikulas Patocka
18470f6cf136SMikulas Patocka if (io->ctx.aead_recheck && !error) {
18480f6cf136SMikulas Patocka kcryptd_queue_crypt(io);
18490f6cf136SMikulas Patocka return;
18500f6cf136SMikulas Patocka }
18518b004457SMilan Broz
18528b004457SMilan Broz /*
18536712ecf8SNeilBrown * free the processed pages
18548b004457SMilan Broz */
18550f6cf136SMikulas Patocka if (rw == WRITE || io->ctx.aead_recheck)
1856644bd2f0SNeil Brown crypt_free_buffer_pages(cc, clone);
18578b004457SMilan Broz
18588b004457SMilan Broz bio_put(clone);
1859ee7a491eSMilan Broz
18609b81c842SSasha Levin if (rw == READ && !error) {
1861cabf08e4SMilan Broz kcryptd_queue_crypt(io);
18626712ecf8SNeilBrown return;
1863ee7a491eSMilan Broz }
18645742fd77SMilan Broz
18659b81c842SSasha Levin if (unlikely(error))
18669b81c842SSasha Levin io->error = error;
18675742fd77SMilan Broz
18685742fd77SMilan Broz crypt_dec_pending(io);
18698b004457SMilan Broz }
18708b004457SMilan Broz
1871e5524e12SMike Snitzer #define CRYPT_MAP_READ_GFP GFP_NOWAIT
1872e5524e12SMike Snitzer
kcryptd_io_read(struct dm_crypt_io * io,gfp_t gfp)187320c82538SMilan Broz static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
18748b004457SMilan Broz {
187549a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
18768b004457SMilan Broz struct bio *clone;
187793e605c2SMilan Broz
18780f6cf136SMikulas Patocka if (io->ctx.aead_recheck) {
18790f6cf136SMikulas Patocka if (!(gfp & __GFP_DIRECT_RECLAIM))
18800f6cf136SMikulas Patocka return 1;
18810f6cf136SMikulas Patocka crypt_inc_pending(io);
18820f6cf136SMikulas Patocka clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
18830f6cf136SMikulas Patocka if (unlikely(!clone)) {
18840f6cf136SMikulas Patocka crypt_dec_pending(io);
18850f6cf136SMikulas Patocka return 1;
18860f6cf136SMikulas Patocka }
18870f6cf136SMikulas Patocka clone->bi_iter.bi_sector = cc->start + io->sector;
18880f6cf136SMikulas Patocka crypt_convert_init(cc, &io->ctx, clone, clone, io->sector);
18890f6cf136SMikulas Patocka io->saved_bi_iter = clone->bi_iter;
18900f6cf136SMikulas Patocka dm_submit_bio_remap(io->base_bio, clone);
18910f6cf136SMikulas Patocka return 0;
18920f6cf136SMikulas Patocka }
18930f6cf136SMikulas Patocka
18948b004457SMilan Broz /*
1895abfc426dSChristoph Hellwig * We need the original biovec array in order to decrypt the whole bio
1896abfc426dSChristoph Hellwig * data *afterwards* -- thanks to immutable biovecs we don't need to
1897abfc426dSChristoph Hellwig * worry about the block layer modifying the biovec array; so leverage
1898abfc426dSChristoph Hellwig * bio_alloc_clone().
18998b004457SMilan Broz */
1900abfc426dSChristoph Hellwig clone = bio_alloc_clone(cc->dev->bdev, io->base_bio, gfp, &cc->bs);
19017eaceaccSJens Axboe if (!clone)
190220c82538SMilan Broz return 1;
19033f868c09SChristoph Hellwig clone->bi_private = io;
19043f868c09SChristoph Hellwig clone->bi_end_io = crypt_endio;
19058b004457SMilan Broz
190620c82538SMilan Broz crypt_inc_pending(io);
190720c82538SMilan Broz
19084f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector;
19098b004457SMilan Broz
1910ef43aa38SMilan Broz if (dm_crypt_integrity_io_alloc(io, clone)) {
1911ef43aa38SMilan Broz crypt_dec_pending(io);
1912ef43aa38SMilan Broz bio_put(clone);
1913ef43aa38SMilan Broz return 1;
1914ef43aa38SMilan Broz }
1915ef43aa38SMilan Broz
1916b7f8dff0SMike Snitzer dm_submit_bio_remap(io->base_bio, clone);
191720c82538SMilan Broz return 0;
19188b004457SMilan Broz }
19198b004457SMilan Broz
kcryptd_io_read_work(struct work_struct * work)1920dc267621SMikulas Patocka static void kcryptd_io_read_work(struct work_struct *work)
1921395b167cSAlasdair G Kergon {
1922395b167cSAlasdair G Kergon struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1923395b167cSAlasdair G Kergon
192420c82538SMilan Broz crypt_inc_pending(io);
192520c82538SMilan Broz if (kcryptd_io_read(io, GFP_NOIO))
19264e4cbee9SChristoph Hellwig io->error = BLK_STS_RESOURCE;
192720c82538SMilan Broz crypt_dec_pending(io);
1928395b167cSAlasdair G Kergon }
1929395b167cSAlasdair G Kergon
kcryptd_queue_read(struct dm_crypt_io * io)1930dc267621SMikulas Patocka static void kcryptd_queue_read(struct dm_crypt_io *io)
1931395b167cSAlasdair G Kergon {
193249a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
1933395b167cSAlasdair G Kergon
1934dc267621SMikulas Patocka INIT_WORK(&io->work, kcryptd_io_read_work);
1935395b167cSAlasdair G Kergon queue_work(cc->io_queue, &io->work);
1936395b167cSAlasdair G Kergon }
1937395b167cSAlasdair G Kergon
kcryptd_io_write(struct dm_crypt_io * io)1938dc267621SMikulas Patocka static void kcryptd_io_write(struct dm_crypt_io *io)
1939dc267621SMikulas Patocka {
1940dc267621SMikulas Patocka struct bio *clone = io->ctx.bio_out;
1941dc267621SMikulas Patocka
1942b7f8dff0SMike Snitzer dm_submit_bio_remap(io->base_bio, clone);
1943dc267621SMikulas Patocka }
1944dc267621SMikulas Patocka
1945b3c5fd30SMikulas Patocka #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1946b3c5fd30SMikulas Patocka
dmcrypt_write(void * data)1947dc267621SMikulas Patocka static int dmcrypt_write(void *data)
1948dc267621SMikulas Patocka {
1949dc267621SMikulas Patocka struct crypt_config *cc = data;
1950b3c5fd30SMikulas Patocka struct dm_crypt_io *io;
1951b3c5fd30SMikulas Patocka
1952dc267621SMikulas Patocka while (1) {
1953b3c5fd30SMikulas Patocka struct rb_root write_tree;
1954dc267621SMikulas Patocka struct blk_plug plug;
1955dc267621SMikulas Patocka
1956c7329effSMikulas Patocka spin_lock_irq(&cc->write_thread_lock);
1957dc267621SMikulas Patocka continue_locked:
1958dc267621SMikulas Patocka
1959b3c5fd30SMikulas Patocka if (!RB_EMPTY_ROOT(&cc->write_tree))
1960dc267621SMikulas Patocka goto pop_from_list;
1961dc267621SMikulas Patocka
1962f659b100SRabin Vincent set_current_state(TASK_INTERRUPTIBLE);
1963dc267621SMikulas Patocka
1964c7329effSMikulas Patocka spin_unlock_irq(&cc->write_thread_lock);
1965dc267621SMikulas Patocka
1966f659b100SRabin Vincent if (unlikely(kthread_should_stop())) {
1967642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING);
1968f659b100SRabin Vincent break;
1969f659b100SRabin Vincent }
1970f659b100SRabin Vincent
1971dc267621SMikulas Patocka schedule();
1972dc267621SMikulas Patocka
1973642fa448SDavidlohr Bueso set_current_state(TASK_RUNNING);
1974c7329effSMikulas Patocka spin_lock_irq(&cc->write_thread_lock);
1975dc267621SMikulas Patocka goto continue_locked;
1976dc267621SMikulas Patocka
1977dc267621SMikulas Patocka pop_from_list:
1978b3c5fd30SMikulas Patocka write_tree = cc->write_tree;
1979b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT;
1980c7329effSMikulas Patocka spin_unlock_irq(&cc->write_thread_lock);
1981dc267621SMikulas Patocka
1982b3c5fd30SMikulas Patocka BUG_ON(rb_parent(write_tree.rb_node));
1983b3c5fd30SMikulas Patocka
1984b3c5fd30SMikulas Patocka /*
1985b3c5fd30SMikulas Patocka * Note: we cannot walk the tree here with rb_next because
1986b3c5fd30SMikulas Patocka * the structures may be freed when kcryptd_io_write is called.
1987b3c5fd30SMikulas Patocka */
1988dc267621SMikulas Patocka blk_start_plug(&plug);
1989dc267621SMikulas Patocka do {
1990b3c5fd30SMikulas Patocka io = crypt_io_from_node(rb_first(&write_tree));
1991b3c5fd30SMikulas Patocka rb_erase(&io->rb_node, &write_tree);
1992dc267621SMikulas Patocka kcryptd_io_write(io);
1993fb294b1cSMikulas Patocka cond_resched();
1994b3c5fd30SMikulas Patocka } while (!RB_EMPTY_ROOT(&write_tree));
1995dc267621SMikulas Patocka blk_finish_plug(&plug);
1996dc267621SMikulas Patocka }
1997dc267621SMikulas Patocka return 0;
1998dc267621SMikulas Patocka }
1999dc267621SMikulas Patocka
kcryptd_crypt_write_io_submit(struct dm_crypt_io * io,int async)200072c6e7afSMikulas Patocka static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
20014e4eef64SMilan Broz {
2002dec1cedfSMilan Broz struct bio *clone = io->ctx.bio_out;
200349a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
2004dc267621SMikulas Patocka unsigned long flags;
2005b3c5fd30SMikulas Patocka sector_t sector;
2006b3c5fd30SMikulas Patocka struct rb_node **rbp, *parent;
2007dec1cedfSMilan Broz
20084e4cbee9SChristoph Hellwig if (unlikely(io->error)) {
2009dec1cedfSMilan Broz crypt_free_buffer_pages(cc, clone);
2010dec1cedfSMilan Broz bio_put(clone);
20116c031f41SMilan Broz crypt_dec_pending(io);
2012dec1cedfSMilan Broz return;
2013dec1cedfSMilan Broz }
2014dec1cedfSMilan Broz
2015dec1cedfSMilan Broz /* crypt_convert should have filled the clone bio */
2016003b5c57SKent Overstreet BUG_ON(io->ctx.iter_out.bi_size);
2017dec1cedfSMilan Broz
20184f024f37SKent Overstreet clone->bi_iter.bi_sector = cc->start + io->sector;
2019899c95d3SMilan Broz
202039d42fa9SIgnat Korchagin if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) ||
202139d42fa9SIgnat Korchagin test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) {
2022b7f8dff0SMike Snitzer dm_submit_bio_remap(io->base_bio, clone);
20230f5d8e6eSMikulas Patocka return;
20240f5d8e6eSMikulas Patocka }
20250f5d8e6eSMikulas Patocka
2026c7329effSMikulas Patocka spin_lock_irqsave(&cc->write_thread_lock, flags);
2027c7329effSMikulas Patocka if (RB_EMPTY_ROOT(&cc->write_tree))
2028c7329effSMikulas Patocka wake_up_process(cc->write_thread);
2029b3c5fd30SMikulas Patocka rbp = &cc->write_tree.rb_node;
2030b3c5fd30SMikulas Patocka parent = NULL;
2031b3c5fd30SMikulas Patocka sector = io->sector;
2032b3c5fd30SMikulas Patocka while (*rbp) {
2033b3c5fd30SMikulas Patocka parent = *rbp;
2034b3c5fd30SMikulas Patocka if (sector < crypt_io_from_node(parent)->sector)
2035b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_left;
2036b3c5fd30SMikulas Patocka else
2037b3c5fd30SMikulas Patocka rbp = &(*rbp)->rb_right;
2038b3c5fd30SMikulas Patocka }
2039b3c5fd30SMikulas Patocka rb_link_node(&io->rb_node, parent, rbp);
2040b3c5fd30SMikulas Patocka rb_insert_color(&io->rb_node, &cc->write_tree);
2041c7329effSMikulas Patocka spin_unlock_irqrestore(&cc->write_thread_lock, flags);
20424e4eef64SMilan Broz }
20434e4eef64SMilan Broz
kcryptd_crypt_write_inline(struct crypt_config * cc,struct convert_context * ctx)20448e225f04SDamien Le Moal static bool kcryptd_crypt_write_inline(struct crypt_config *cc,
20458e225f04SDamien Le Moal struct convert_context *ctx)
20468e225f04SDamien Le Moal
20478e225f04SDamien Le Moal {
20488e225f04SDamien Le Moal if (!test_bit(DM_CRYPT_WRITE_INLINE, &cc->flags))
20498e225f04SDamien Le Moal return false;
20508e225f04SDamien Le Moal
20518e225f04SDamien Le Moal /*
20528e225f04SDamien Le Moal * Note: zone append writes (REQ_OP_ZONE_APPEND) do not have ordering
20538e225f04SDamien Le Moal * constraints so they do not need to be issued inline by
20548e225f04SDamien Le Moal * kcryptd_crypt_write_convert().
20558e225f04SDamien Le Moal */
20568e225f04SDamien Le Moal switch (bio_op(ctx->bio_in)) {
20578e225f04SDamien Le Moal case REQ_OP_WRITE:
20588e225f04SDamien Le Moal case REQ_OP_WRITE_ZEROES:
20598e225f04SDamien Le Moal return true;
20608e225f04SDamien Le Moal default:
20618e225f04SDamien Le Moal return false;
20628e225f04SDamien Le Moal }
20638e225f04SDamien Le Moal }
20648e225f04SDamien Le Moal
kcryptd_crypt_write_continue(struct work_struct * work)20658abec36dSIgnat Korchagin static void kcryptd_crypt_write_continue(struct work_struct *work)
20668abec36dSIgnat Korchagin {
20678abec36dSIgnat Korchagin struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
20688abec36dSIgnat Korchagin struct crypt_config *cc = io->cc;
20698abec36dSIgnat Korchagin struct convert_context *ctx = &io->ctx;
20708abec36dSIgnat Korchagin int crypt_finished;
20718abec36dSIgnat Korchagin sector_t sector = io->sector;
20728abec36dSIgnat Korchagin blk_status_t r;
20738abec36dSIgnat Korchagin
20748abec36dSIgnat Korchagin wait_for_completion(&ctx->restart);
20758abec36dSIgnat Korchagin reinit_completion(&ctx->restart);
20768abec36dSIgnat Korchagin
20778abec36dSIgnat Korchagin r = crypt_convert(cc, &io->ctx, true, false);
20788abec36dSIgnat Korchagin if (r)
20798abec36dSIgnat Korchagin io->error = r;
20808abec36dSIgnat Korchagin crypt_finished = atomic_dec_and_test(&ctx->cc_pending);
20818abec36dSIgnat Korchagin if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
20828abec36dSIgnat Korchagin /* Wait for completion signaled by kcryptd_async_done() */
20838abec36dSIgnat Korchagin wait_for_completion(&ctx->restart);
20848abec36dSIgnat Korchagin crypt_finished = 1;
20858abec36dSIgnat Korchagin }
20868abec36dSIgnat Korchagin
20878abec36dSIgnat Korchagin /* Encryption was already finished, submit io now */
20888abec36dSIgnat Korchagin if (crypt_finished) {
20898abec36dSIgnat Korchagin kcryptd_crypt_write_io_submit(io, 0);
20908abec36dSIgnat Korchagin io->sector = sector;
20918abec36dSIgnat Korchagin }
20928abec36dSIgnat Korchagin
20938abec36dSIgnat Korchagin crypt_dec_pending(io);
20948abec36dSIgnat Korchagin }
20958abec36dSIgnat Korchagin
kcryptd_crypt_write_convert(struct dm_crypt_io * io)2096fc5a5e9aSMilan Broz static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
20978b004457SMilan Broz {
209849a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
20998e225f04SDamien Le Moal struct convert_context *ctx = &io->ctx;
21008b004457SMilan Broz struct bio *clone;
2101c8081618SMilan Broz int crypt_finished;
2102b635b00eSMilan Broz sector_t sector = io->sector;
21034e4cbee9SChristoph Hellwig blk_status_t r;
21048b004457SMilan Broz
210593e605c2SMilan Broz /*
2106fc5a5e9aSMilan Broz * Prevent io from disappearing until this function completes.
2107fc5a5e9aSMilan Broz */
2108fc5a5e9aSMilan Broz crypt_inc_pending(io);
21098e225f04SDamien Le Moal crypt_convert_init(cc, ctx, NULL, io->base_bio, sector);
2110fc5a5e9aSMilan Broz
2111cf2f1abfSMikulas Patocka clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
211223541d2dSMilan Broz if (unlikely(!clone)) {
21134e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR;
2114cf2f1abfSMikulas Patocka goto dec;
211523541d2dSMilan Broz }
21168b004457SMilan Broz
211753017030SMilan Broz io->ctx.bio_out = clone;
2118003b5c57SKent Overstreet io->ctx.iter_out = clone->bi_iter;
21198b004457SMilan Broz
212064ba01a3SMikulas Patocka if (crypt_integrity_aead(cc)) {
212164ba01a3SMikulas Patocka bio_copy_data(clone, io->base_bio);
212264ba01a3SMikulas Patocka io->ctx.bio_in = clone;
212364ba01a3SMikulas Patocka io->ctx.iter_in = clone->bi_iter;
212464ba01a3SMikulas Patocka }
212564ba01a3SMikulas Patocka
2126b635b00eSMilan Broz sector += bio_sectors(clone);
2127dec1cedfSMilan Broz
21284e594098SMilan Broz crypt_inc_pending(io);
21298e225f04SDamien Le Moal r = crypt_convert(cc, ctx,
21308abec36dSIgnat Korchagin test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), true);
21318abec36dSIgnat Korchagin /*
21328abec36dSIgnat Korchagin * Crypto API backlogged the request, because its queue was full
21338abec36dSIgnat Korchagin * and we're in softirq context, so continue from a workqueue
21348abec36dSIgnat Korchagin * (TODO: is it actually possible to be in softirq in the write path?)
21358abec36dSIgnat Korchagin */
21368abec36dSIgnat Korchagin if (r == BLK_STS_DEV_RESOURCE) {
21378abec36dSIgnat Korchagin INIT_WORK(&io->work, kcryptd_crypt_write_continue);
21388abec36dSIgnat Korchagin queue_work(cc->crypt_queue, &io->work);
21398abec36dSIgnat Korchagin return;
21408abec36dSIgnat Korchagin }
21414e4cbee9SChristoph Hellwig if (r)
2142ef43aa38SMilan Broz io->error = r;
21438e225f04SDamien Le Moal crypt_finished = atomic_dec_and_test(&ctx->cc_pending);
21448e225f04SDamien Le Moal if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
21458e225f04SDamien Le Moal /* Wait for completion signaled by kcryptd_async_done() */
21468e225f04SDamien Le Moal wait_for_completion(&ctx->restart);
21478e225f04SDamien Le Moal crypt_finished = 1;
21488e225f04SDamien Le Moal }
2149dec1cedfSMilan Broz
2150c8081618SMilan Broz /* Encryption was already finished, submit io now */
2151c8081618SMilan Broz if (crypt_finished) {
215272c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 0);
2153b635b00eSMilan Broz io->sector = sector;
21544e594098SMilan Broz }
215593e605c2SMilan Broz
2156cf2f1abfSMikulas Patocka dec:
2157899c95d3SMilan Broz crypt_dec_pending(io);
215884131db6SMilan Broz }
215984131db6SMilan Broz
kcryptd_crypt_read_done(struct dm_crypt_io * io)216072c6e7afSMikulas Patocka static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
21615742fd77SMilan Broz {
21620f6cf136SMikulas Patocka if (io->ctx.aead_recheck) {
21630f6cf136SMikulas Patocka if (!io->error) {
21640f6cf136SMikulas Patocka io->ctx.bio_in->bi_iter = io->saved_bi_iter;
21650f6cf136SMikulas Patocka bio_copy_data(io->base_bio, io->ctx.bio_in);
21660f6cf136SMikulas Patocka }
21670f6cf136SMikulas Patocka crypt_free_buffer_pages(io->cc, io->ctx.bio_in);
21680f6cf136SMikulas Patocka bio_put(io->ctx.bio_in);
21690f6cf136SMikulas Patocka }
21705742fd77SMilan Broz crypt_dec_pending(io);
21715742fd77SMilan Broz }
21725742fd77SMilan Broz
kcryptd_crypt_read_continue(struct work_struct * work)21738abec36dSIgnat Korchagin static void kcryptd_crypt_read_continue(struct work_struct *work)
21748abec36dSIgnat Korchagin {
21758abec36dSIgnat Korchagin struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
21768abec36dSIgnat Korchagin struct crypt_config *cc = io->cc;
21778abec36dSIgnat Korchagin blk_status_t r;
21788abec36dSIgnat Korchagin
21798abec36dSIgnat Korchagin wait_for_completion(&io->ctx.restart);
21808abec36dSIgnat Korchagin reinit_completion(&io->ctx.restart);
21818abec36dSIgnat Korchagin
21828abec36dSIgnat Korchagin r = crypt_convert(cc, &io->ctx, true, false);
21838abec36dSIgnat Korchagin if (r)
21848abec36dSIgnat Korchagin io->error = r;
21858abec36dSIgnat Korchagin
21868abec36dSIgnat Korchagin if (atomic_dec_and_test(&io->ctx.cc_pending))
21878abec36dSIgnat Korchagin kcryptd_crypt_read_done(io);
21888abec36dSIgnat Korchagin
21898abec36dSIgnat Korchagin crypt_dec_pending(io);
21908abec36dSIgnat Korchagin }
21918abec36dSIgnat Korchagin
kcryptd_crypt_read_convert(struct dm_crypt_io * io)21924e4eef64SMilan Broz static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
21938b004457SMilan Broz {
219449a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
21954e4cbee9SChristoph Hellwig blk_status_t r;
21968b004457SMilan Broz
21973e1a8bddSMilan Broz crypt_inc_pending(io);
21983a7f6c99SMilan Broz
21990f6cf136SMikulas Patocka if (io->ctx.aead_recheck) {
22000f6cf136SMikulas Patocka io->ctx.cc_sector = io->sector + cc->iv_offset;
22010f6cf136SMikulas Patocka r = crypt_convert(cc, &io->ctx,
22020f6cf136SMikulas Patocka test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
22030f6cf136SMikulas Patocka } else {
220453017030SMilan Broz crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
22050c395b0fSMilan Broz io->sector);
22068b004457SMilan Broz
220739d42fa9SIgnat Korchagin r = crypt_convert(cc, &io->ctx,
22088abec36dSIgnat Korchagin test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
22090f6cf136SMikulas Patocka }
22108abec36dSIgnat Korchagin /*
22118abec36dSIgnat Korchagin * Crypto API backlogged the request, because its queue was full
22128abec36dSIgnat Korchagin * and we're in softirq context, so continue from a workqueue
22138abec36dSIgnat Korchagin */
22148abec36dSIgnat Korchagin if (r == BLK_STS_DEV_RESOURCE) {
22158abec36dSIgnat Korchagin INIT_WORK(&io->work, kcryptd_crypt_read_continue);
22168abec36dSIgnat Korchagin queue_work(cc->crypt_queue, &io->work);
22178abec36dSIgnat Korchagin return;
22188abec36dSIgnat Korchagin }
22194e4cbee9SChristoph Hellwig if (r)
2220ef43aa38SMilan Broz io->error = r;
22215742fd77SMilan Broz
222240b6229bSMikulas Patocka if (atomic_dec_and_test(&io->ctx.cc_pending))
222372c6e7afSMikulas Patocka kcryptd_crypt_read_done(io);
22243a7f6c99SMilan Broz
22253a7f6c99SMilan Broz crypt_dec_pending(io);
22268b004457SMilan Broz }
22278b004457SMilan Broz
kcryptd_async_done(void * data,int error)2228dcfe653dSHerbert Xu static void kcryptd_async_done(void *data, int error)
222995497a96SMilan Broz {
2230dcfe653dSHerbert Xu struct dm_crypt_request *dmreq = data;
2231b2174eebSHuang Ying struct convert_context *ctx = dmreq->ctx;
223295497a96SMilan Broz struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
223349a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
223495497a96SMilan Broz
223554cea3f6SMilan Broz /*
223654cea3f6SMilan Broz * A request from crypto driver backlog is going to be processed now,
223754cea3f6SMilan Broz * finish the completion and continue in crypt_convert().
223854cea3f6SMilan Broz * (Callback will be called for the second time for this request.)
223954cea3f6SMilan Broz */
2240c0403ec0SRabin Vincent if (error == -EINPROGRESS) {
2241c0403ec0SRabin Vincent complete(&ctx->restart);
224295497a96SMilan Broz return;
2243c0403ec0SRabin Vincent }
224495497a96SMilan Broz
22452dc5327dSMilan Broz if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
2246ef43aa38SMilan Broz error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);
22472dc5327dSMilan Broz
2248ef43aa38SMilan Broz if (error == -EBADMSG) {
224958d0f180SMichael Weiß sector_t s = le64_to_cpu(*org_sector_of_dmreq(cc, dmreq));
225058d0f180SMichael Weiß
22510f6cf136SMikulas Patocka ctx->aead_failed = true;
22520f6cf136SMikulas Patocka if (ctx->aead_recheck) {
225366671719SChristoph Hellwig DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
225466671719SChristoph Hellwig ctx->bio_in->bi_bdev, s);
225558d0f180SMichael Weiß dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
225658d0f180SMichael Weiß ctx->bio_in, s, 0);
22570f6cf136SMikulas Patocka }
22584e4cbee9SChristoph Hellwig io->error = BLK_STS_PROTECTION;
2259ef43aa38SMilan Broz } else if (error < 0)
22604e4cbee9SChristoph Hellwig io->error = BLK_STS_IOERR;
226172c6e7afSMikulas Patocka
2262298a9fa0SMikulas Patocka crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);
226395497a96SMilan Broz
226440b6229bSMikulas Patocka if (!atomic_dec_and_test(&ctx->cc_pending))
2265c0403ec0SRabin Vincent return;
226695497a96SMilan Broz
22678e225f04SDamien Le Moal /*
22688e225f04SDamien Le Moal * The request is fully completed: for inline writes, let
22698e225f04SDamien Le Moal * kcryptd_crypt_write_convert() do the IO submission.
22708e225f04SDamien Le Moal */
22718e225f04SDamien Le Moal if (bio_data_dir(io->base_bio) == READ) {
227272c6e7afSMikulas Patocka kcryptd_crypt_read_done(io);
22738e225f04SDamien Le Moal return;
22748e225f04SDamien Le Moal }
22758e225f04SDamien Le Moal
22768e225f04SDamien Le Moal if (kcryptd_crypt_write_inline(cc, ctx)) {
22778e225f04SDamien Le Moal complete(&ctx->restart);
22788e225f04SDamien Le Moal return;
22798e225f04SDamien Le Moal }
22808e225f04SDamien Le Moal
228172c6e7afSMikulas Patocka kcryptd_crypt_write_io_submit(io, 1);
228295497a96SMilan Broz }
228395497a96SMilan Broz
kcryptd_crypt(struct work_struct * work)22844e4eef64SMilan Broz static void kcryptd_crypt(struct work_struct *work)
22854e4eef64SMilan Broz {
22864e4eef64SMilan Broz struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
22874e4eef64SMilan Broz
22884e4eef64SMilan Broz if (bio_data_dir(io->base_bio) == READ)
22894e4eef64SMilan Broz kcryptd_crypt_read_convert(io);
22904e4eef64SMilan Broz else
22914e4eef64SMilan Broz kcryptd_crypt_write_convert(io);
22928b004457SMilan Broz }
22938b004457SMilan Broz
kcryptd_queue_crypt(struct dm_crypt_io * io)2294395b167cSAlasdair G Kergon static void kcryptd_queue_crypt(struct dm_crypt_io *io)
2295395b167cSAlasdair G Kergon {
229649a8a920SAlasdair G Kergon struct crypt_config *cc = io->cc;
2297395b167cSAlasdair G Kergon
229839d42fa9SIgnat Korchagin if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) ||
229939d42fa9SIgnat Korchagin (bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) {
2300c87a95dcSIgnat Korchagin /*
2301d3703ef3SChangbin Du * in_hardirq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.
2302c87a95dcSIgnat Korchagin * irqs_disabled(): the kernel may run some IO completion from the idle thread, but
2303c87a95dcSIgnat Korchagin * it is being executed with irqs disabled.
2304c87a95dcSIgnat Korchagin */
23055735a267SMikulas Patocka if (!(in_hardirq() || irqs_disabled())) {
230639d42fa9SIgnat Korchagin kcryptd_crypt(&io->work);
230739d42fa9SIgnat Korchagin return;
230839d42fa9SIgnat Korchagin }
23095735a267SMikulas Patocka }
231039d42fa9SIgnat Korchagin
2311395b167cSAlasdair G Kergon INIT_WORK(&io->work, kcryptd_crypt);
2312395b167cSAlasdair G Kergon queue_work(cc->crypt_queue, &io->work);
2313395b167cSAlasdair G Kergon }
2314395b167cSAlasdair G Kergon
crypt_free_tfms_aead(struct crypt_config * cc)2315ef43aa38SMilan Broz static void crypt_free_tfms_aead(struct crypt_config *cc)
23161da177e4SLinus Torvalds {
2317ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms_aead)
2318ef43aa38SMilan Broz return;
23191da177e4SLinus Torvalds
2320ef43aa38SMilan Broz if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
2321ef43aa38SMilan Broz crypto_free_aead(cc->cipher_tfm.tfms_aead[0]);
2322ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead[0] = NULL;
23231da177e4SLinus Torvalds }
23241da177e4SLinus Torvalds
2325ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms_aead);
2326ef43aa38SMilan Broz cc->cipher_tfm.tfms_aead = NULL;
2327ef43aa38SMilan Broz }
23281da177e4SLinus Torvalds
crypt_free_tfms_skcipher(struct crypt_config * cc)2329ef43aa38SMilan Broz static void crypt_free_tfms_skcipher(struct crypt_config *cc)
2330d1f96423SMilan Broz {
233186a3238cSHeinz Mauelshagen unsigned int i;
2332d1f96423SMilan Broz
2333ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms)
2334fd2d231fSMikulas Patocka return;
2335fd2d231fSMikulas Patocka
2336d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++)
2337ef43aa38SMilan Broz if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) {
2338ef43aa38SMilan Broz crypto_free_skcipher(cc->cipher_tfm.tfms[i]);
2339ef43aa38SMilan Broz cc->cipher_tfm.tfms[i] = NULL;
2340d1f96423SMilan Broz }
2341d1f96423SMilan Broz
2342ef43aa38SMilan Broz kfree(cc->cipher_tfm.tfms);
2343ef43aa38SMilan Broz cc->cipher_tfm.tfms = NULL;
23441da177e4SLinus Torvalds }
23451da177e4SLinus Torvalds
crypt_free_tfms(struct crypt_config * cc)23461da177e4SLinus Torvalds static void crypt_free_tfms(struct crypt_config *cc)
2347d1f96423SMilan Broz {
234833d2f09fSMilan Broz if (crypt_integrity_aead(cc))
2349ef43aa38SMilan Broz crypt_free_tfms_aead(cc);
2350ef43aa38SMilan Broz else
2351ef43aa38SMilan Broz crypt_free_tfms_skcipher(cc);
2352d1f96423SMilan Broz }
2353d1f96423SMilan Broz
crypt_alloc_tfms_skcipher(struct crypt_config * cc,char * ciphermode)2354ef43aa38SMilan Broz static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode)
2355d1f96423SMilan Broz {
235686a3238cSHeinz Mauelshagen unsigned int i;
2357d1f96423SMilan Broz int err;
2358d1f96423SMilan Broz
23596396bb22SKees Cook cc->cipher_tfm.tfms = kcalloc(cc->tfms_count,
23606396bb22SKees Cook sizeof(struct crypto_skcipher *),
23616396bb22SKees Cook GFP_KERNEL);
2362ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms)
2363fd2d231fSMikulas Patocka return -ENOMEM;
2364fd2d231fSMikulas Patocka
2365d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) {
2366cd746938SMikulas Patocka cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0,
2367cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY);
2368ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms[i])) {
2369ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms[i]);
2370fd2d231fSMikulas Patocka crypt_free_tfms(cc);
2371d1f96423SMilan Broz return err;
2372d1f96423SMilan Broz }
2373d1f96423SMilan Broz }
2374d1f96423SMilan Broz
2375af331ebaSEric Biggers /*
2376af331ebaSEric Biggers * dm-crypt performance can vary greatly depending on which crypto
2377af331ebaSEric Biggers * algorithm implementation is used. Help people debug performance
2378af331ebaSEric Biggers * problems by logging the ->cra_driver_name.
2379af331ebaSEric Biggers */
23807a1cd723SMilan Broz DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2381af331ebaSEric Biggers crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name);
2382d1f96423SMilan Broz return 0;
2383d1f96423SMilan Broz }
2384d1f96423SMilan Broz
crypt_alloc_tfms_aead(struct crypt_config * cc,char * ciphermode)2385ef43aa38SMilan Broz static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)
2386ef43aa38SMilan Broz {
2387ef43aa38SMilan Broz int err;
2388ef43aa38SMilan Broz
2389ef43aa38SMilan Broz cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL);
2390ef43aa38SMilan Broz if (!cc->cipher_tfm.tfms)
2391ef43aa38SMilan Broz return -ENOMEM;
2392ef43aa38SMilan Broz
2393cd746938SMikulas Patocka cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0,
2394cd746938SMikulas Patocka CRYPTO_ALG_ALLOCATES_MEMORY);
2395ef43aa38SMilan Broz if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
2396ef43aa38SMilan Broz err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]);
2397ef43aa38SMilan Broz crypt_free_tfms(cc);
2398ef43aa38SMilan Broz return err;
2399ef43aa38SMilan Broz }
2400ef43aa38SMilan Broz
24017a1cd723SMilan Broz DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2402af331ebaSEric Biggers crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name);
2403ef43aa38SMilan Broz return 0;
2404ef43aa38SMilan Broz }
2405ef43aa38SMilan Broz
crypt_alloc_tfms(struct crypt_config * cc,char * ciphermode)2406ef43aa38SMilan Broz static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
2407ef43aa38SMilan Broz {
240833d2f09fSMilan Broz if (crypt_integrity_aead(cc))
2409ef43aa38SMilan Broz return crypt_alloc_tfms_aead(cc, ciphermode);
2410ef43aa38SMilan Broz else
2411ef43aa38SMilan Broz return crypt_alloc_tfms_skcipher(cc, ciphermode);
2412ef43aa38SMilan Broz }
2413ef43aa38SMilan Broz
crypt_subkey_size(struct crypt_config * cc)241486a3238cSHeinz Mauelshagen static unsigned int crypt_subkey_size(struct crypt_config *cc)
2415ef43aa38SMilan Broz {
2416ef43aa38SMilan Broz return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
2417ef43aa38SMilan Broz }
2418ef43aa38SMilan Broz
crypt_authenckey_size(struct crypt_config * cc)241986a3238cSHeinz Mauelshagen static unsigned int crypt_authenckey_size(struct crypt_config *cc)
2420ef43aa38SMilan Broz {
2421ef43aa38SMilan Broz return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param));
2422ef43aa38SMilan Broz }
2423ef43aa38SMilan Broz
2424ef43aa38SMilan Broz /*
2425ef43aa38SMilan Broz * If AEAD is composed like authenc(hmac(sha256),xts(aes)),
2426ef43aa38SMilan Broz * the key must be for some reason in special format.
2427ef43aa38SMilan Broz * This funcion converts cc->key to this special format.
2428ef43aa38SMilan Broz */
crypt_copy_authenckey(char * p,const void * key,unsigned int enckeylen,unsigned int authkeylen)2429ef43aa38SMilan Broz static void crypt_copy_authenckey(char *p, const void *key,
243086a3238cSHeinz Mauelshagen unsigned int enckeylen, unsigned int authkeylen)
2431ef43aa38SMilan Broz {
2432ef43aa38SMilan Broz struct crypto_authenc_key_param *param;
2433ef43aa38SMilan Broz struct rtattr *rta;
2434ef43aa38SMilan Broz
2435ef43aa38SMilan Broz rta = (struct rtattr *)p;
2436ef43aa38SMilan Broz param = RTA_DATA(rta);
2437ef43aa38SMilan Broz param->enckeylen = cpu_to_be32(enckeylen);
2438ef43aa38SMilan Broz rta->rta_len = RTA_LENGTH(sizeof(*param));
2439ef43aa38SMilan Broz rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
2440ef43aa38SMilan Broz p += RTA_SPACE(sizeof(*param));
2441ef43aa38SMilan Broz memcpy(p, key + enckeylen, authkeylen);
2442ef43aa38SMilan Broz p += authkeylen;
2443ef43aa38SMilan Broz memcpy(p, key, enckeylen);
2444ef43aa38SMilan Broz }
2445ef43aa38SMilan Broz
crypt_setkey(struct crypt_config * cc)2446671ea6b4SMikulas Patocka static int crypt_setkey(struct crypt_config *cc)
2447c0297721SAndi Kleen {
244886a3238cSHeinz Mauelshagen unsigned int subkey_size;
2449fd2d231fSMikulas Patocka int err = 0, i, r;
2450c0297721SAndi Kleen
2451da31a078SMilan Broz /* Ignore extra keys (which are used for IV etc) */
2452ef43aa38SMilan Broz subkey_size = crypt_subkey_size(cc);
2453da31a078SMilan Broz
245427c70036SMilan Broz if (crypt_integrity_hmac(cc)) {
245527c70036SMilan Broz if (subkey_size < cc->key_mac_size)
245627c70036SMilan Broz return -EINVAL;
245727c70036SMilan Broz
2458ef43aa38SMilan Broz crypt_copy_authenckey(cc->authenc_key, cc->key,
2459ef43aa38SMilan Broz subkey_size - cc->key_mac_size,
2460ef43aa38SMilan Broz cc->key_mac_size);
246127c70036SMilan Broz }
246227c70036SMilan Broz
2463d1f96423SMilan Broz for (i = 0; i < cc->tfms_count; i++) {
246433d2f09fSMilan Broz if (crypt_integrity_hmac(cc))
246533d2f09fSMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
246633d2f09fSMilan Broz cc->authenc_key, crypt_authenckey_size(cc));
246733d2f09fSMilan Broz else if (crypt_integrity_aead(cc))
2468ef43aa38SMilan Broz r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
2469ef43aa38SMilan Broz cc->key + (i * subkey_size),
2470ef43aa38SMilan Broz subkey_size);
2471ef43aa38SMilan Broz else
2472ef43aa38SMilan Broz r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i],
2473fd2d231fSMikulas Patocka cc->key + (i * subkey_size),
2474fd2d231fSMikulas Patocka subkey_size);
2475c0297721SAndi Kleen if (r)
2476c0297721SAndi Kleen err = r;
2477c0297721SAndi Kleen }
2478c0297721SAndi Kleen
2479ef43aa38SMilan Broz if (crypt_integrity_hmac(cc))
2480ef43aa38SMilan Broz memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc));
2481ef43aa38SMilan Broz
2482c0297721SAndi Kleen return err;
2483c0297721SAndi Kleen }
2484c0297721SAndi Kleen
2485c538f6ecSOndrej Kozina #ifdef CONFIG_KEYS
2486c538f6ecSOndrej Kozina
contains_whitespace(const char * str)2487027c431cSOndrej Kozina static bool contains_whitespace(const char *str)
2488027c431cSOndrej Kozina {
2489027c431cSOndrej Kozina while (*str)
2490027c431cSOndrej Kozina if (isspace(*str++))
2491027c431cSOndrej Kozina return true;
2492027c431cSOndrej Kozina return false;
2493027c431cSOndrej Kozina }
2494027c431cSOndrej Kozina
set_key_user(struct crypt_config * cc,struct key * key)249527f5411aSDmitry Baryshkov static int set_key_user(struct crypt_config *cc, struct key *key)
249627f5411aSDmitry Baryshkov {
249727f5411aSDmitry Baryshkov const struct user_key_payload *ukp;
249827f5411aSDmitry Baryshkov
249927f5411aSDmitry Baryshkov ukp = user_key_payload_locked(key);
250027f5411aSDmitry Baryshkov if (!ukp)
250127f5411aSDmitry Baryshkov return -EKEYREVOKED;
250227f5411aSDmitry Baryshkov
250327f5411aSDmitry Baryshkov if (cc->key_size != ukp->datalen)
250427f5411aSDmitry Baryshkov return -EINVAL;
250527f5411aSDmitry Baryshkov
250627f5411aSDmitry Baryshkov memcpy(cc->key, ukp->data, cc->key_size);
250727f5411aSDmitry Baryshkov
250827f5411aSDmitry Baryshkov return 0;
250927f5411aSDmitry Baryshkov }
251027f5411aSDmitry Baryshkov
set_key_encrypted(struct crypt_config * cc,struct key * key)251127f5411aSDmitry Baryshkov static int set_key_encrypted(struct crypt_config *cc, struct key *key)
251227f5411aSDmitry Baryshkov {
251327f5411aSDmitry Baryshkov const struct encrypted_key_payload *ekp;
251427f5411aSDmitry Baryshkov
251527f5411aSDmitry Baryshkov ekp = key->payload.data[0];
251627f5411aSDmitry Baryshkov if (!ekp)
251727f5411aSDmitry Baryshkov return -EKEYREVOKED;
251827f5411aSDmitry Baryshkov
251927f5411aSDmitry Baryshkov if (cc->key_size != ekp->decrypted_datalen)
252027f5411aSDmitry Baryshkov return -EINVAL;
252127f5411aSDmitry Baryshkov
252227f5411aSDmitry Baryshkov memcpy(cc->key, ekp->decrypted_data, cc->key_size);
252327f5411aSDmitry Baryshkov
252427f5411aSDmitry Baryshkov return 0;
252527f5411aSDmitry Baryshkov }
252627f5411aSDmitry Baryshkov
set_key_trusted(struct crypt_config * cc,struct key * key)2527363880c4SAhmad Fatoum static int set_key_trusted(struct crypt_config *cc, struct key *key)
2528363880c4SAhmad Fatoum {
2529363880c4SAhmad Fatoum const struct trusted_key_payload *tkp;
2530363880c4SAhmad Fatoum
2531363880c4SAhmad Fatoum tkp = key->payload.data[0];
2532363880c4SAhmad Fatoum if (!tkp)
2533363880c4SAhmad Fatoum return -EKEYREVOKED;
2534363880c4SAhmad Fatoum
2535363880c4SAhmad Fatoum if (cc->key_size != tkp->key_len)
2536363880c4SAhmad Fatoum return -EINVAL;
2537363880c4SAhmad Fatoum
2538363880c4SAhmad Fatoum memcpy(cc->key, tkp->key, cc->key_size);
2539363880c4SAhmad Fatoum
2540363880c4SAhmad Fatoum return 0;
2541363880c4SAhmad Fatoum }
2542363880c4SAhmad Fatoum
crypt_set_keyring_key(struct crypt_config * cc,const char * key_string)2543c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2544c538f6ecSOndrej Kozina {
2545c538f6ecSOndrej Kozina char *new_key_string, *key_desc;
2546c538f6ecSOndrej Kozina int ret;
254727f5411aSDmitry Baryshkov struct key_type *type;
2548c538f6ecSOndrej Kozina struct key *key;
254927f5411aSDmitry Baryshkov int (*set_key)(struct crypt_config *cc, struct key *key);
2550c538f6ecSOndrej Kozina
2551027c431cSOndrej Kozina /*
2552027c431cSOndrej Kozina * Reject key_string with whitespace. dm core currently lacks code for
2553027c431cSOndrej Kozina * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
2554027c431cSOndrej Kozina */
2555027c431cSOndrej Kozina if (contains_whitespace(key_string)) {
2556027c431cSOndrej Kozina DMERR("whitespace chars not allowed in key string");
2557027c431cSOndrej Kozina return -EINVAL;
2558027c431cSOndrej Kozina }
2559027c431cSOndrej Kozina
2560c538f6ecSOndrej Kozina /* look for next ':' separating key_type from key_description */
2561fc772580SChristophe JAILLET key_desc = strchr(key_string, ':');
2562c538f6ecSOndrej Kozina if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
2563c538f6ecSOndrej Kozina return -EINVAL;
2564c538f6ecSOndrej Kozina
256527f5411aSDmitry Baryshkov if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
256627f5411aSDmitry Baryshkov type = &key_type_logon;
256727f5411aSDmitry Baryshkov set_key = set_key_user;
256827f5411aSDmitry Baryshkov } else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) {
256927f5411aSDmitry Baryshkov type = &key_type_user;
257027f5411aSDmitry Baryshkov set_key = set_key_user;
2571831475ccSAhmad Fatoum } else if (IS_ENABLED(CONFIG_ENCRYPTED_KEYS) &&
2572831475ccSAhmad Fatoum !strncmp(key_string, "encrypted:", key_desc - key_string + 1)) {
257327f5411aSDmitry Baryshkov type = &key_type_encrypted;
257427f5411aSDmitry Baryshkov set_key = set_key_encrypted;
2575363880c4SAhmad Fatoum } else if (IS_ENABLED(CONFIG_TRUSTED_KEYS) &&
2576363880c4SAhmad Fatoum !strncmp(key_string, "trusted:", key_desc - key_string + 1)) {
2577363880c4SAhmad Fatoum type = &key_type_trusted;
2578363880c4SAhmad Fatoum set_key = set_key_trusted;
257927f5411aSDmitry Baryshkov } else {
2580c538f6ecSOndrej Kozina return -EINVAL;
258127f5411aSDmitry Baryshkov }
2582c538f6ecSOndrej Kozina
2583c538f6ecSOndrej Kozina new_key_string = kstrdup(key_string, GFP_KERNEL);
2584c538f6ecSOndrej Kozina if (!new_key_string)
2585c538f6ecSOndrej Kozina return -ENOMEM;
2586c538f6ecSOndrej Kozina
258727f5411aSDmitry Baryshkov key = request_key(type, key_desc + 1, NULL);
2588c538f6ecSOndrej Kozina if (IS_ERR(key)) {
2589453431a5SWaiman Long kfree_sensitive(new_key_string);
2590c538f6ecSOndrej Kozina return PTR_ERR(key);
2591c538f6ecSOndrej Kozina }
2592c538f6ecSOndrej Kozina
2593f5b0cba8SOndrej Kozina down_read(&key->sem);
2594c538f6ecSOndrej Kozina
259527f5411aSDmitry Baryshkov ret = set_key(cc, key);
259627f5411aSDmitry Baryshkov if (ret < 0) {
2597f5b0cba8SOndrej Kozina up_read(&key->sem);
2598c538f6ecSOndrej Kozina key_put(key);
2599453431a5SWaiman Long kfree_sensitive(new_key_string);
260027f5411aSDmitry Baryshkov return ret;
2601c538f6ecSOndrej Kozina }
2602c538f6ecSOndrej Kozina
2603f5b0cba8SOndrej Kozina up_read(&key->sem);
2604c538f6ecSOndrej Kozina key_put(key);
2605c538f6ecSOndrej Kozina
2606c538f6ecSOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */
2607c538f6ecSOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2608c538f6ecSOndrej Kozina
2609c538f6ecSOndrej Kozina ret = crypt_setkey(cc);
2610c538f6ecSOndrej Kozina
2611c538f6ecSOndrej Kozina if (!ret) {
2612c538f6ecSOndrej Kozina set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2613453431a5SWaiman Long kfree_sensitive(cc->key_string);
2614c538f6ecSOndrej Kozina cc->key_string = new_key_string;
2615c538f6ecSOndrej Kozina } else
2616453431a5SWaiman Long kfree_sensitive(new_key_string);
2617c538f6ecSOndrej Kozina
2618c538f6ecSOndrej Kozina return ret;
2619c538f6ecSOndrej Kozina }
2620c538f6ecSOndrej Kozina
get_key_size(char ** key_string)2621c538f6ecSOndrej Kozina static int get_key_size(char **key_string)
2622c538f6ecSOndrej Kozina {
2623c538f6ecSOndrej Kozina char *colon, dummy;
2624c538f6ecSOndrej Kozina int ret;
2625c538f6ecSOndrej Kozina
2626c538f6ecSOndrej Kozina if (*key_string[0] != ':')
2627c538f6ecSOndrej Kozina return strlen(*key_string) >> 1;
2628c538f6ecSOndrej Kozina
2629c538f6ecSOndrej Kozina /* look for next ':' in key string */
2630c538f6ecSOndrej Kozina colon = strpbrk(*key_string + 1, ":");
2631c538f6ecSOndrej Kozina if (!colon)
2632c538f6ecSOndrej Kozina return -EINVAL;
2633c538f6ecSOndrej Kozina
2634c538f6ecSOndrej Kozina if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
2635c538f6ecSOndrej Kozina return -EINVAL;
2636c538f6ecSOndrej Kozina
2637c538f6ecSOndrej Kozina *key_string = colon;
2638c538f6ecSOndrej Kozina
2639c538f6ecSOndrej Kozina /* remaining key string should be :<logon|user>:<key_desc> */
2640c538f6ecSOndrej Kozina
2641c538f6ecSOndrej Kozina return ret;
2642c538f6ecSOndrej Kozina }
2643c538f6ecSOndrej Kozina
2644c538f6ecSOndrej Kozina #else
2645c538f6ecSOndrej Kozina
crypt_set_keyring_key(struct crypt_config * cc,const char * key_string)2646c538f6ecSOndrej Kozina static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2647c538f6ecSOndrej Kozina {
2648c538f6ecSOndrej Kozina return -EINVAL;
2649c538f6ecSOndrej Kozina }
2650c538f6ecSOndrej Kozina
get_key_size(char ** key_string)2651c538f6ecSOndrej Kozina static int get_key_size(char **key_string)
2652c538f6ecSOndrej Kozina {
26536fc51504SAashish Sharma return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1);
2654c538f6ecSOndrej Kozina }
2655c538f6ecSOndrej Kozina
265627f5411aSDmitry Baryshkov #endif /* CONFIG_KEYS */
2657c538f6ecSOndrej Kozina
crypt_set_key(struct crypt_config * cc,char * key)2658e48d4bbfSMilan Broz static int crypt_set_key(struct crypt_config *cc, char *key)
2659e48d4bbfSMilan Broz {
2660de8be5acSMilan Broz int r = -EINVAL;
2661de8be5acSMilan Broz int key_string_len = strlen(key);
2662de8be5acSMilan Broz
266369a8cfcdSMilan Broz /* Hyphen (which gives a key_size of zero) means there is no key. */
266469a8cfcdSMilan Broz if (!cc->key_size && strcmp(key, "-"))
2665de8be5acSMilan Broz goto out;
2666e48d4bbfSMilan Broz
2667c538f6ecSOndrej Kozina /* ':' means the key is in kernel keyring, short-circuit normal key processing */
2668c538f6ecSOndrej Kozina if (key[0] == ':') {
2669c538f6ecSOndrej Kozina r = crypt_set_keyring_key(cc, key + 1);
2670c538f6ecSOndrej Kozina goto out;
2671c538f6ecSOndrej Kozina }
2672c538f6ecSOndrej Kozina
2673265e9098SOndrej Kozina /* clear the flag since following operations may invalidate previously valid key */
2674265e9098SOndrej Kozina clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2675265e9098SOndrej Kozina
2676c538f6ecSOndrej Kozina /* wipe references to any kernel keyring key */
2677453431a5SWaiman Long kfree_sensitive(cc->key_string);
2678c538f6ecSOndrej Kozina cc->key_string = NULL;
2679c538f6ecSOndrej Kozina
2680e944e03eSAndy Shevchenko /* Decode key from its hex representation. */
2681e944e03eSAndy Shevchenko if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0)
2682de8be5acSMilan Broz goto out;
2683e48d4bbfSMilan Broz
2684671ea6b4SMikulas Patocka r = crypt_setkey(cc);
2685265e9098SOndrej Kozina if (!r)
2686e48d4bbfSMilan Broz set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2687e48d4bbfSMilan Broz
2688de8be5acSMilan Broz out:
2689de8be5acSMilan Broz /* Hex key string not needed after here, so wipe it. */
2690de8be5acSMilan Broz memset(key, '0', key_string_len);
2691de8be5acSMilan Broz
2692de8be5acSMilan Broz return r;
2693e48d4bbfSMilan Broz }
2694e48d4bbfSMilan Broz
crypt_wipe_key(struct crypt_config * cc)2695e48d4bbfSMilan Broz static int crypt_wipe_key(struct crypt_config *cc)
2696e48d4bbfSMilan Broz {
2697c82feeecSOndrej Kozina int r;
2698c82feeecSOndrej Kozina
2699e48d4bbfSMilan Broz clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2700c82feeecSOndrej Kozina get_random_bytes(&cc->key, cc->key_size);
27014a52ffc7SMilan Broz
27024a52ffc7SMilan Broz /* Wipe IV private keys */
27034a52ffc7SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
27044a52ffc7SMilan Broz r = cc->iv_gen_ops->wipe(cc);
27054a52ffc7SMilan Broz if (r)
27064a52ffc7SMilan Broz return r;
27074a52ffc7SMilan Broz }
27084a52ffc7SMilan Broz
2709453431a5SWaiman Long kfree_sensitive(cc->key_string);
2710c538f6ecSOndrej Kozina cc->key_string = NULL;
2711c82feeecSOndrej Kozina r = crypt_setkey(cc);
2712c82feeecSOndrej Kozina memset(&cc->key, 0, cc->key_size * sizeof(u8));
2713c0297721SAndi Kleen
2714c82feeecSOndrej Kozina return r;
2715e48d4bbfSMilan Broz }
2716e48d4bbfSMilan Broz
crypt_calculate_pages_per_client(void)27175059353dSMikulas Patocka static void crypt_calculate_pages_per_client(void)
27185059353dSMikulas Patocka {
2719ca79b0c2SArun KS unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100;
27205059353dSMikulas Patocka
27215059353dSMikulas Patocka if (!dm_crypt_clients_n)
27225059353dSMikulas Patocka return;
27235059353dSMikulas Patocka
27245059353dSMikulas Patocka pages /= dm_crypt_clients_n;
27255059353dSMikulas Patocka if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT)
27265059353dSMikulas Patocka pages = DM_CRYPT_MIN_PAGES_PER_CLIENT;
27275059353dSMikulas Patocka dm_crypt_pages_per_client = pages;
27285059353dSMikulas Patocka }
27295059353dSMikulas Patocka
crypt_page_alloc(gfp_t gfp_mask,void * pool_data)27305059353dSMikulas Patocka static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)
27315059353dSMikulas Patocka {
27325059353dSMikulas Patocka struct crypt_config *cc = pool_data;
27335059353dSMikulas Patocka struct page *page;
27345059353dSMikulas Patocka
2735528b16bfSArne Welzel /*
2736528b16bfSArne Welzel * Note, percpu_counter_read_positive() may over (and under) estimate
2737528b16bfSArne Welzel * the current usage by at most (batch - 1) * num_online_cpus() pages,
2738528b16bfSArne Welzel * but avoids potential spinlock contention of an exact result.
2739528b16bfSArne Welzel */
2740528b16bfSArne Welzel if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) &&
27415059353dSMikulas Patocka likely(gfp_mask & __GFP_NORETRY))
27425059353dSMikulas Patocka return NULL;
27435059353dSMikulas Patocka
27445059353dSMikulas Patocka page = alloc_page(gfp_mask);
27455059353dSMikulas Patocka if (likely(page != NULL))
27465059353dSMikulas Patocka percpu_counter_add(&cc->n_allocated_pages, 1);
27475059353dSMikulas Patocka
27485059353dSMikulas Patocka return page;
27495059353dSMikulas Patocka }
27505059353dSMikulas Patocka
crypt_page_free(void * page,void * pool_data)27515059353dSMikulas Patocka static void crypt_page_free(void *page, void *pool_data)
27525059353dSMikulas Patocka {
27535059353dSMikulas Patocka struct crypt_config *cc = pool_data;
27545059353dSMikulas Patocka
27555059353dSMikulas Patocka __free_page(page);
27565059353dSMikulas Patocka percpu_counter_sub(&cc->n_allocated_pages, 1);
27575059353dSMikulas Patocka }
27585059353dSMikulas Patocka
crypt_dtr(struct dm_target * ti)275928513fccSMilan Broz static void crypt_dtr(struct dm_target *ti)
276028513fccSMilan Broz {
276128513fccSMilan Broz struct crypt_config *cc = ti->private;
276228513fccSMilan Broz
276328513fccSMilan Broz ti->private = NULL;
276428513fccSMilan Broz
276528513fccSMilan Broz if (!cc)
276628513fccSMilan Broz return;
276728513fccSMilan Broz
2768f659b100SRabin Vincent if (cc->write_thread)
2769dc267621SMikulas Patocka kthread_stop(cc->write_thread);
2770dc267621SMikulas Patocka
277128513fccSMilan Broz if (cc->io_queue)
277228513fccSMilan Broz destroy_workqueue(cc->io_queue);
277328513fccSMilan Broz if (cc->crypt_queue)
277428513fccSMilan Broz destroy_workqueue(cc->crypt_queue);
277528513fccSMilan Broz
2776fd2d231fSMikulas Patocka crypt_free_tfms(cc);
2777fd2d231fSMikulas Patocka
27786f1c819cSKent Overstreet bioset_exit(&cc->bs);
277928513fccSMilan Broz
27806f1c819cSKent Overstreet mempool_exit(&cc->page_pool);
27816f1c819cSKent Overstreet mempool_exit(&cc->req_pool);
27826f1c819cSKent Overstreet mempool_exit(&cc->tag_pool);
27836f1c819cSKent Overstreet
2784d00a11dfSKent Overstreet WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
2785d00a11dfSKent Overstreet percpu_counter_destroy(&cc->n_allocated_pages);
2786d00a11dfSKent Overstreet
278728513fccSMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
278828513fccSMilan Broz cc->iv_gen_ops->dtr(cc);
278928513fccSMilan Broz
279028513fccSMilan Broz if (cc->dev)
279128513fccSMilan Broz dm_put_device(ti, cc->dev);
279228513fccSMilan Broz
2793453431a5SWaiman Long kfree_sensitive(cc->cipher_string);
2794453431a5SWaiman Long kfree_sensitive(cc->key_string);
2795453431a5SWaiman Long kfree_sensitive(cc->cipher_auth);
2796453431a5SWaiman Long kfree_sensitive(cc->authenc_key);
279728513fccSMilan Broz
2798d5ffebddSMike Snitzer mutex_destroy(&cc->bio_alloc_lock);
2799d5ffebddSMike Snitzer
280028513fccSMilan Broz /* Must zero key material before freeing */
2801453431a5SWaiman Long kfree_sensitive(cc);
28025059353dSMikulas Patocka
28035059353dSMikulas Patocka spin_lock(&dm_crypt_clients_lock);
28045059353dSMikulas Patocka WARN_ON(!dm_crypt_clients_n);
28055059353dSMikulas Patocka dm_crypt_clients_n--;
28065059353dSMikulas Patocka crypt_calculate_pages_per_client();
28075059353dSMikulas Patocka spin_unlock(&dm_crypt_clients_lock);
280858d0f180SMichael Weiß
280958d0f180SMichael Weiß dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
281028513fccSMilan Broz }
281128513fccSMilan Broz
crypt_ctr_ivmode(struct dm_target * ti,const char * ivmode)2812e889f97aSMilan Broz static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
28131da177e4SLinus Torvalds {
28145ebaee6dSMilan Broz struct crypt_config *cc = ti->private;
28151da177e4SLinus Torvalds
281633d2f09fSMilan Broz if (crypt_integrity_aead(cc))
2817e889f97aSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
2818e889f97aSMilan Broz else
2819bbdb23b5SHerbert Xu cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
2820e889f97aSMilan Broz
28215ebaee6dSMilan Broz if (cc->iv_size)
28225ebaee6dSMilan Broz /* at least a 64 bit sector number should fit in our buffer */
28235ebaee6dSMilan Broz cc->iv_size = max(cc->iv_size,
28245ebaee6dSMilan Broz (unsigned int)(sizeof(u64) / sizeof(u8)));
28255ebaee6dSMilan Broz else if (ivmode) {
28265ebaee6dSMilan Broz DMWARN("Selected cipher does not support IVs");
28275ebaee6dSMilan Broz ivmode = NULL;
28285ebaee6dSMilan Broz }
28295ebaee6dSMilan Broz
28305ebaee6dSMilan Broz /* Choose ivmode, see comments at iv code. */
28311da177e4SLinus Torvalds if (ivmode == NULL)
28321da177e4SLinus Torvalds cc->iv_gen_ops = NULL;
28331da177e4SLinus Torvalds else if (strcmp(ivmode, "plain") == 0)
28341da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_plain_ops;
283561afef61SMilan Broz else if (strcmp(ivmode, "plain64") == 0)
283661afef61SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64_ops;
28377e3fd855SMilan Broz else if (strcmp(ivmode, "plain64be") == 0)
28387e3fd855SMilan Broz cc->iv_gen_ops = &crypt_iv_plain64be_ops;
28391da177e4SLinus Torvalds else if (strcmp(ivmode, "essiv") == 0)
28401da177e4SLinus Torvalds cc->iv_gen_ops = &crypt_iv_essiv_ops;
284148527fa7SRik Snel else if (strcmp(ivmode, "benbi") == 0)
284248527fa7SRik Snel cc->iv_gen_ops = &crypt_iv_benbi_ops;
284346b47730SLudwig Nussel else if (strcmp(ivmode, "null") == 0)
284446b47730SLudwig Nussel cc->iv_gen_ops = &crypt_iv_null_ops;
2845b9411d73SMilan Broz else if (strcmp(ivmode, "eboiv") == 0)
2846b9411d73SMilan Broz cc->iv_gen_ops = &crypt_iv_eboiv_ops;
2847bbb16584SMilan Broz else if (strcmp(ivmode, "elephant") == 0) {
2848bbb16584SMilan Broz cc->iv_gen_ops = &crypt_iv_elephant_ops;
2849bbb16584SMilan Broz cc->key_parts = 2;
2850bbb16584SMilan Broz cc->key_extra_size = cc->key_size / 2;
2851bbb16584SMilan Broz if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)
2852bbb16584SMilan Broz return -EINVAL;
2853bbb16584SMilan Broz set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags);
2854bbb16584SMilan Broz } else if (strcmp(ivmode, "lmk") == 0) {
285534745785SMilan Broz cc->iv_gen_ops = &crypt_iv_lmk_ops;
2856ed04d981SMilan Broz /*
2857ed04d981SMilan Broz * Version 2 and 3 is recognised according
285834745785SMilan Broz * to length of provided multi-key string.
285934745785SMilan Broz * If present (version 3), last key is used as IV seed.
2860ed04d981SMilan Broz * All keys (including IV seed) are always the same size.
286134745785SMilan Broz */
2862da31a078SMilan Broz if (cc->key_size % cc->key_parts) {
286334745785SMilan Broz cc->key_parts++;
2864da31a078SMilan Broz cc->key_extra_size = cc->key_size / cc->key_parts;
2865da31a078SMilan Broz }
2866ed04d981SMilan Broz } else if (strcmp(ivmode, "tcw") == 0) {
2867ed04d981SMilan Broz cc->iv_gen_ops = &crypt_iv_tcw_ops;
2868ed04d981SMilan Broz cc->key_parts += 2; /* IV + whitening */
2869ed04d981SMilan Broz cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
2870e889f97aSMilan Broz } else if (strcmp(ivmode, "random") == 0) {
2871e889f97aSMilan Broz cc->iv_gen_ops = &crypt_iv_random_ops;
2872e889f97aSMilan Broz /* Need storage space in integrity fields. */
2873e889f97aSMilan Broz cc->integrity_iv_size = cc->iv_size;
287434745785SMilan Broz } else {
287572d94861SAlasdair G Kergon ti->error = "Invalid IV mode";
2876e889f97aSMilan Broz return -EINVAL;
28771da177e4SLinus Torvalds }
28781da177e4SLinus Torvalds
2879e889f97aSMilan Broz return 0;
2880e889f97aSMilan Broz }
2881e889f97aSMilan Broz
288233d2f09fSMilan Broz /*
288333d2f09fSMilan Broz * Workaround to parse HMAC algorithm from AEAD crypto API spec.
288433d2f09fSMilan Broz * The HMAC is needed to calculate tag size (HMAC digest size).
288533d2f09fSMilan Broz * This should be probably done by crypto-api calls (once available...)
288633d2f09fSMilan Broz */
crypt_ctr_auth_cipher(struct crypt_config * cc,char * cipher_api)288733d2f09fSMilan Broz static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api)
288833d2f09fSMilan Broz {
288933d2f09fSMilan Broz char *start, *end, *mac_alg = NULL;
289033d2f09fSMilan Broz struct crypto_ahash *mac;
289133d2f09fSMilan Broz
289233d2f09fSMilan Broz if (!strstarts(cipher_api, "authenc("))
289333d2f09fSMilan Broz return 0;
289433d2f09fSMilan Broz
289533d2f09fSMilan Broz start = strchr(cipher_api, '(');
289633d2f09fSMilan Broz end = strchr(cipher_api, ',');
289733d2f09fSMilan Broz if (!start || !end || ++start > end)
289833d2f09fSMilan Broz return -EINVAL;
289933d2f09fSMilan Broz
290033d2f09fSMilan Broz mac_alg = kzalloc(end - start + 1, GFP_KERNEL);
290133d2f09fSMilan Broz if (!mac_alg)
290233d2f09fSMilan Broz return -ENOMEM;
290333d2f09fSMilan Broz strncpy(mac_alg, start, end - start);
290433d2f09fSMilan Broz
2905cd746938SMikulas Patocka mac = crypto_alloc_ahash(mac_alg, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
290633d2f09fSMilan Broz kfree(mac_alg);
290733d2f09fSMilan Broz
290833d2f09fSMilan Broz if (IS_ERR(mac))
290933d2f09fSMilan Broz return PTR_ERR(mac);
291033d2f09fSMilan Broz
291133d2f09fSMilan Broz cc->key_mac_size = crypto_ahash_digestsize(mac);
291233d2f09fSMilan Broz crypto_free_ahash(mac);
291333d2f09fSMilan Broz
291433d2f09fSMilan Broz cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL);
291533d2f09fSMilan Broz if (!cc->authenc_key)
291633d2f09fSMilan Broz return -ENOMEM;
291733d2f09fSMilan Broz
291833d2f09fSMilan Broz return 0;
291933d2f09fSMilan Broz }
292033d2f09fSMilan Broz
crypt_ctr_cipher_new(struct dm_target * ti,char * cipher_in,char * key,char ** ivmode,char ** ivopts)292133d2f09fSMilan Broz static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key,
292233d2f09fSMilan Broz char **ivmode, char **ivopts)
29231da177e4SLinus Torvalds {
29245ebaee6dSMilan Broz struct crypt_config *cc = ti->private;
2925a1a262b6SArd Biesheuvel char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME];
292633d2f09fSMilan Broz int ret = -EINVAL;
292733d2f09fSMilan Broz
292833d2f09fSMilan Broz cc->tfms_count = 1;
292933d2f09fSMilan Broz
293033d2f09fSMilan Broz /*
293133d2f09fSMilan Broz * New format (capi: prefix)
293233d2f09fSMilan Broz * capi:cipher_api_spec-iv:ivopts
293333d2f09fSMilan Broz */
293433d2f09fSMilan Broz tmp = &cipher_in[strlen("capi:")];
29351856b9f7SMilan Broz
29361856b9f7SMilan Broz /* Separate IV options if present, it can contain another '-' in hash name */
29371856b9f7SMilan Broz *ivopts = strrchr(tmp, ':');
29381856b9f7SMilan Broz if (*ivopts) {
29391856b9f7SMilan Broz **ivopts = '\0';
29401856b9f7SMilan Broz (*ivopts)++;
29411856b9f7SMilan Broz }
29421856b9f7SMilan Broz /* Parse IV mode */
29431856b9f7SMilan Broz *ivmode = strrchr(tmp, '-');
29441856b9f7SMilan Broz if (*ivmode) {
29451856b9f7SMilan Broz **ivmode = '\0';
29461856b9f7SMilan Broz (*ivmode)++;
29471856b9f7SMilan Broz }
29481856b9f7SMilan Broz /* The rest is crypto API spec */
29491856b9f7SMilan Broz cipher_api = tmp;
295033d2f09fSMilan Broz
2951a1a262b6SArd Biesheuvel /* Alloc AEAD, can be used only in new format. */
2952a1a262b6SArd Biesheuvel if (crypt_integrity_aead(cc)) {
2953a1a262b6SArd Biesheuvel ret = crypt_ctr_auth_cipher(cc, cipher_api);
2954a1a262b6SArd Biesheuvel if (ret < 0) {
2955a1a262b6SArd Biesheuvel ti->error = "Invalid AEAD cipher spec";
29562a32897cSMikulas Patocka return ret;
2957a1a262b6SArd Biesheuvel }
2958a1a262b6SArd Biesheuvel }
2959a1a262b6SArd Biesheuvel
296033d2f09fSMilan Broz if (*ivmode && !strcmp(*ivmode, "lmk"))
296133d2f09fSMilan Broz cc->tfms_count = 64;
296233d2f09fSMilan Broz
2963a1a262b6SArd Biesheuvel if (*ivmode && !strcmp(*ivmode, "essiv")) {
2964a1a262b6SArd Biesheuvel if (!*ivopts) {
2965a1a262b6SArd Biesheuvel ti->error = "Digest algorithm missing for ESSIV mode";
2966a1a262b6SArd Biesheuvel return -EINVAL;
2967a1a262b6SArd Biesheuvel }
2968a1a262b6SArd Biesheuvel ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, "essiv(%s,%s)",
2969a1a262b6SArd Biesheuvel cipher_api, *ivopts);
2970a1a262b6SArd Biesheuvel if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
2971a1a262b6SArd Biesheuvel ti->error = "Cannot allocate cipher string";
2972a1a262b6SArd Biesheuvel return -ENOMEM;
2973a1a262b6SArd Biesheuvel }
2974a1a262b6SArd Biesheuvel cipher_api = buf;
2975a1a262b6SArd Biesheuvel }
2976a1a262b6SArd Biesheuvel
297733d2f09fSMilan Broz cc->key_parts = cc->tfms_count;
297833d2f09fSMilan Broz
297933d2f09fSMilan Broz /* Allocate cipher */
298033d2f09fSMilan Broz ret = crypt_alloc_tfms(cc, cipher_api);
298133d2f09fSMilan Broz if (ret < 0) {
298233d2f09fSMilan Broz ti->error = "Error allocating crypto tfm";
298333d2f09fSMilan Broz return ret;
298433d2f09fSMilan Broz }
298533d2f09fSMilan Broz
2986a1a262b6SArd Biesheuvel if (crypt_integrity_aead(cc))
298733d2f09fSMilan Broz cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
2988a1a262b6SArd Biesheuvel else
298933d2f09fSMilan Broz cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
299033d2f09fSMilan Broz
299133d2f09fSMilan Broz return 0;
299233d2f09fSMilan Broz }
299333d2f09fSMilan Broz
crypt_ctr_cipher_old(struct dm_target * ti,char * cipher_in,char * key,char ** ivmode,char ** ivopts)299433d2f09fSMilan Broz static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key,
299533d2f09fSMilan Broz char **ivmode, char **ivopts)
299633d2f09fSMilan Broz {
299733d2f09fSMilan Broz struct crypt_config *cc = ti->private;
299833d2f09fSMilan Broz char *tmp, *cipher, *chainmode, *keycount;
29995ebaee6dSMilan Broz char *cipher_api = NULL;
30005ebaee6dSMilan Broz int ret = -EINVAL;
30015ebaee6dSMilan Broz char dummy;
30025ebaee6dSMilan Broz
300333d2f09fSMilan Broz if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) {
30045ebaee6dSMilan Broz ti->error = "Bad cipher specification";
30055ebaee6dSMilan Broz return -EINVAL;
30065ebaee6dSMilan Broz }
30075ebaee6dSMilan Broz
30081da177e4SLinus Torvalds /*
30095ebaee6dSMilan Broz * Legacy dm-crypt cipher specification
30105ebaee6dSMilan Broz * cipher[:keycount]-mode-iv:ivopts
30115ebaee6dSMilan Broz */
30125ebaee6dSMilan Broz tmp = cipher_in;
30135ebaee6dSMilan Broz keycount = strsep(&tmp, "-");
30145ebaee6dSMilan Broz cipher = strsep(&keycount, ":");
30155ebaee6dSMilan Broz
301669a8cfcdSMilan Broz if (!keycount)
30175ebaee6dSMilan Broz cc->tfms_count = 1;
30185ebaee6dSMilan Broz else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
30195ebaee6dSMilan Broz !is_power_of_2(cc->tfms_count)) {
30205ebaee6dSMilan Broz ti->error = "Bad cipher key count specification";
30215ebaee6dSMilan Broz return -EINVAL;
30225ebaee6dSMilan Broz }
302328513fccSMilan Broz cc->key_parts = cc->tfms_count;
30241da177e4SLinus Torvalds
3025ddd42edfSMilan Broz chainmode = strsep(&tmp, "-");
30261856b9f7SMilan Broz *ivmode = strsep(&tmp, ":");
30271856b9f7SMilan Broz *ivopts = tmp;
3028ddd42edfSMilan Broz
3029ddd42edfSMilan Broz /*
3030ddd42edfSMilan Broz * For compatibility with the original dm-crypt mapping format, if
3031ddd42edfSMilan Broz * only the cipher name is supplied, use cbc-plain.
303228513fccSMilan Broz */
303333d2f09fSMilan Broz if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) {
3034cabf08e4SMilan Broz chainmode = "cbc";
303533d2f09fSMilan Broz *ivmode = "plain";
3036cabf08e4SMilan Broz }
3037cabf08e4SMilan Broz
303833d2f09fSMilan Broz if (strcmp(chainmode, "ecb") && !*ivmode) {
3039c0297721SAndi Kleen ti->error = "IV mechanism required";
3040c0297721SAndi Kleen return -EINVAL;
3041c0297721SAndi Kleen }
3042c0297721SAndi Kleen
3043cabf08e4SMilan Broz cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
30449934a8beSMilan Broz if (!cipher_api)
304528513fccSMilan Broz goto bad_mem;
30469934a8beSMilan Broz
3047a1a262b6SArd Biesheuvel if (*ivmode && !strcmp(*ivmode, "essiv")) {
3048a1a262b6SArd Biesheuvel if (!*ivopts) {
3049a1a262b6SArd Biesheuvel ti->error = "Digest algorithm missing for ESSIV mode";
3050a1a262b6SArd Biesheuvel kfree(cipher_api);
3051a1a262b6SArd Biesheuvel return -EINVAL;
3052a1a262b6SArd Biesheuvel }
3053a1a262b6SArd Biesheuvel ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
3054a1a262b6SArd Biesheuvel "essiv(%s(%s),%s)", chainmode, cipher, *ivopts);
3055a1a262b6SArd Biesheuvel } else {
30569934a8beSMilan Broz ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
3057647c7db1SMikulas Patocka "%s(%s)", chainmode, cipher);
3058a1a262b6SArd Biesheuvel }
3059a1a262b6SArd Biesheuvel if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
30601da177e4SLinus Torvalds kfree(cipher_api);
306128513fccSMilan Broz goto bad_mem;
306228513fccSMilan Broz }
306328513fccSMilan Broz
30641da177e4SLinus Torvalds /* Allocate cipher */
30651da177e4SLinus Torvalds ret = crypt_alloc_tfms(cc, cipher_api);
30661da177e4SLinus Torvalds if (ret < 0) {
30671da177e4SLinus Torvalds ti->error = "Error allocating crypto tfm";
306833d2f09fSMilan Broz kfree(cipher_api);
306933d2f09fSMilan Broz return ret;
3070028867acSAlasdair G Kergon }
3071bd86e320SJeffy Chen kfree(cipher_api);
3072647c7db1SMikulas Patocka
307333d2f09fSMilan Broz return 0;
307433d2f09fSMilan Broz bad_mem:
307533d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings";
307633d2f09fSMilan Broz return -ENOMEM;
307733d2f09fSMilan Broz }
307833d2f09fSMilan Broz
crypt_ctr_cipher(struct dm_target * ti,char * cipher_in,char * key)307933d2f09fSMilan Broz static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
308033d2f09fSMilan Broz {
308133d2f09fSMilan Broz struct crypt_config *cc = ti->private;
308233d2f09fSMilan Broz char *ivmode = NULL, *ivopts = NULL;
308333d2f09fSMilan Broz int ret;
308433d2f09fSMilan Broz
308533d2f09fSMilan Broz cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);
308633d2f09fSMilan Broz if (!cc->cipher_string) {
308733d2f09fSMilan Broz ti->error = "Cannot allocate cipher strings";
308833d2f09fSMilan Broz return -ENOMEM;
308933d2f09fSMilan Broz }
309033d2f09fSMilan Broz
309133d2f09fSMilan Broz if (strstarts(cipher_in, "capi:"))
309233d2f09fSMilan Broz ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts);
309333d2f09fSMilan Broz else
309433d2f09fSMilan Broz ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts);
309533d2f09fSMilan Broz if (ret)
309633d2f09fSMilan Broz return ret;
309733d2f09fSMilan Broz
3098647c7db1SMikulas Patocka /* Initialize IV */
3099e889f97aSMilan Broz ret = crypt_ctr_ivmode(ti, ivmode);
3100e889f97aSMilan Broz if (ret < 0)
310133d2f09fSMilan Broz return ret;
31021da177e4SLinus Torvalds
3103da31a078SMilan Broz /* Initialize and set key */
3104da31a078SMilan Broz ret = crypt_set_key(cc, key);
3105da31a078SMilan Broz if (ret < 0) {
3106da31a078SMilan Broz ti->error = "Error decoding and setting key";
310733d2f09fSMilan Broz return ret;
3108da31a078SMilan Broz }
3109da31a078SMilan Broz
31101da177e4SLinus Torvalds /* Allocate IV */
31111da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
31121da177e4SLinus Torvalds ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
31131da177e4SLinus Torvalds if (ret < 0) {
31141da177e4SLinus Torvalds ti->error = "Error creating IV";
311533d2f09fSMilan Broz return ret;
31161da177e4SLinus Torvalds }
31171da177e4SLinus Torvalds }
31181da177e4SLinus Torvalds
31191da177e4SLinus Torvalds /* Initialize IV (set keys for ESSIV etc) */
31201da177e4SLinus Torvalds if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
31211da177e4SLinus Torvalds ret = cc->iv_gen_ops->init(cc);
31221da177e4SLinus Torvalds if (ret < 0) {
31231da177e4SLinus Torvalds ti->error = "Error initialising IV";
31241da177e4SLinus Torvalds return ret;
31251da177e4SLinus Torvalds }
31261da177e4SLinus Torvalds }
31271da177e4SLinus Torvalds
3128dc94902bSOndrej Kozina /* wipe the kernel key payload copy */
3129dc94902bSOndrej Kozina if (cc->key_string)
3130dc94902bSOndrej Kozina memset(cc->key, 0, cc->key_size * sizeof(u8));
3131dc94902bSOndrej Kozina
313233d2f09fSMilan Broz return ret;
31331da177e4SLinus Torvalds }
31341da177e4SLinus Torvalds
crypt_ctr_optional(struct dm_target * ti,unsigned int argc,char ** argv)3135ef43aa38SMilan Broz static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv)
3136ef43aa38SMilan Broz {
3137ef43aa38SMilan Broz struct crypt_config *cc = ti->private;
3138ef43aa38SMilan Broz struct dm_arg_set as;
31395916a22bSEric Biggers static const struct dm_arg _args[] = {
314039d42fa9SIgnat Korchagin {0, 8, "Invalid number of feature args"},
3141ef43aa38SMilan Broz };
3142ef43aa38SMilan Broz unsigned int opt_params, val;
3143ef43aa38SMilan Broz const char *opt_string, *sval;
31448f0009a2SMilan Broz char dummy;
3145ef43aa38SMilan Broz int ret;
3146ef43aa38SMilan Broz
3147ef43aa38SMilan Broz /* Optional parameters */
3148ef43aa38SMilan Broz as.argc = argc;
3149ef43aa38SMilan Broz as.argv = argv;
3150ef43aa38SMilan Broz
3151ef43aa38SMilan Broz ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
3152ef43aa38SMilan Broz if (ret)
31531da177e4SLinus Torvalds return ret;
31541da177e4SLinus Torvalds
3155ef43aa38SMilan Broz while (opt_params--) {
3156ef43aa38SMilan Broz opt_string = dm_shift_arg(&as);
3157ef43aa38SMilan Broz if (!opt_string) {
3158ef43aa38SMilan Broz ti->error = "Not enough feature arguments";
3159ef43aa38SMilan Broz return -EINVAL;
3160ef43aa38SMilan Broz }
3161ef43aa38SMilan Broz
3162ef43aa38SMilan Broz if (!strcasecmp(opt_string, "allow_discards"))
3163ef43aa38SMilan Broz ti->num_discard_bios = 1;
3164ef43aa38SMilan Broz
3165ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "same_cpu_crypt"))
3166ef43aa38SMilan Broz set_bit(DM_CRYPT_SAME_CPU, &cc->flags);
3167ef43aa38SMilan Broz
3168ef43aa38SMilan Broz else if (!strcasecmp(opt_string, "submit_from_crypt_cpus"))
3169ef43aa38SMilan Broz set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
317039d42fa9SIgnat Korchagin else if (!strcasecmp(opt_string, "no_read_workqueue"))
317139d42fa9SIgnat Korchagin set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
317239d42fa9SIgnat Korchagin else if (!strcasecmp(opt_string, "no_write_workqueue"))
317339d42fa9SIgnat Korchagin set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
3174ef43aa38SMilan Broz else if (sscanf(opt_string, "integrity:%u:", &val) == 1) {
3175ef43aa38SMilan Broz if (val == 0 || val > MAX_TAG_SIZE) {
3176ef43aa38SMilan Broz ti->error = "Invalid integrity arguments";
3177ef43aa38SMilan Broz return -EINVAL;
3178ef43aa38SMilan Broz }
3179ef43aa38SMilan Broz cc->on_disk_tag_size = val;
3180ef43aa38SMilan Broz sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
3181ef43aa38SMilan Broz if (!strcasecmp(sval, "aead")) {
3182ef43aa38SMilan Broz set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
3183ef43aa38SMilan Broz } else if (strcasecmp(sval, "none")) {
3184ef43aa38SMilan Broz ti->error = "Unknown integrity profile";
3185ef43aa38SMilan Broz return -EINVAL;
3186ef43aa38SMilan Broz }
3187ef43aa38SMilan Broz
3188ef43aa38SMilan Broz cc->cipher_auth = kstrdup(sval, GFP_KERNEL);
3189ef43aa38SMilan Broz if (!cc->cipher_auth)
31901da177e4SLinus Torvalds return -ENOMEM;
3191ff3af92bSMikulas Patocka } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) {
31928f0009a2SMilan Broz if (cc->sector_size < (1 << SECTOR_SHIFT) ||
31938f0009a2SMilan Broz cc->sector_size > 4096 ||
3194ff3af92bSMikulas Patocka (cc->sector_size & (cc->sector_size - 1))) {
31958f0009a2SMilan Broz ti->error = "Invalid feature value for sector_size";
31968f0009a2SMilan Broz return -EINVAL;
31978f0009a2SMilan Broz }
3198783874b0SMilan Broz if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) {
3199783874b0SMilan Broz ti->error = "Device size is not multiple of sector_size feature";
3200783874b0SMilan Broz return -EINVAL;
3201783874b0SMilan Broz }
3202ff3af92bSMikulas Patocka cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT;
32038f0009a2SMilan Broz } else if (!strcasecmp(opt_string, "iv_large_sectors"))
32048f0009a2SMilan Broz set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
32058f0009a2SMilan Broz else {
3206ef43aa38SMilan Broz ti->error = "Invalid feature arguments";
3207ef43aa38SMilan Broz return -EINVAL;
3208ef43aa38SMilan Broz }
3209ef43aa38SMilan Broz }
3210ef43aa38SMilan Broz
3211ef43aa38SMilan Broz return 0;
32121da177e4SLinus Torvalds }
32131da177e4SLinus Torvalds
32148e225f04SDamien Le Moal #ifdef CONFIG_BLK_DEV_ZONED
crypt_report_zones(struct dm_target * ti,struct dm_report_zones_args * args,unsigned int nr_zones)32158e225f04SDamien Le Moal static int crypt_report_zones(struct dm_target *ti,
32168e225f04SDamien Le Moal struct dm_report_zones_args *args, unsigned int nr_zones)
32178e225f04SDamien Le Moal {
32188e225f04SDamien Le Moal struct crypt_config *cc = ti->private;
32198e225f04SDamien Le Moal
3220912e8875SDamien Le Moal return dm_report_zones(cc->dev->bdev, cc->start,
3221912e8875SDamien Le Moal cc->start + dm_target_offset(ti, args->next_sector),
3222912e8875SDamien Le Moal args, nr_zones);
32238e225f04SDamien Le Moal }
3224e3290b94SMike Snitzer #else
3225e3290b94SMike Snitzer #define crypt_report_zones NULL
32268e225f04SDamien Le Moal #endif
32278e225f04SDamien Le Moal
32281da177e4SLinus Torvalds /*
32291da177e4SLinus Torvalds * Construct an encryption mapping:
3230c538f6ecSOndrej Kozina * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
32311da177e4SLinus Torvalds */
crypt_ctr(struct dm_target * ti,unsigned int argc,char ** argv)32321da177e4SLinus Torvalds static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
32331da177e4SLinus Torvalds {
32341da177e4SLinus Torvalds struct crypt_config *cc;
3235ed0302e8SMichał Mirosław const char *devname = dm_table_device_name(ti->table);
3236c538f6ecSOndrej Kozina int key_size;
3237ef43aa38SMilan Broz unsigned int align_mask;
32381da177e4SLinus Torvalds unsigned long long tmpll;
32391da177e4SLinus Torvalds int ret;
3240ef43aa38SMilan Broz size_t iv_size_padding, additional_req_size;
324131998ef1SMikulas Patocka char dummy;
32421da177e4SLinus Torvalds
3243772ae5f5SMilan Broz if (argc < 5) {
32441da177e4SLinus Torvalds ti->error = "Not enough arguments";
32451da177e4SLinus Torvalds return -EINVAL;
32461da177e4SLinus Torvalds }
32471da177e4SLinus Torvalds
3248c538f6ecSOndrej Kozina key_size = get_key_size(&argv[1]);
3249c538f6ecSOndrej Kozina if (key_size < 0) {
3250c538f6ecSOndrej Kozina ti->error = "Cannot parse key size";
3251c538f6ecSOndrej Kozina return -EINVAL;
3252c538f6ecSOndrej Kozina }
32531da177e4SLinus Torvalds
32549c81c99bSZhengyuan Liu cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL);
32551da177e4SLinus Torvalds if (!cc) {
32561da177e4SLinus Torvalds ti->error = "Cannot allocate encryption context";
32571da177e4SLinus Torvalds return -ENOMEM;
32581da177e4SLinus Torvalds }
32591da177e4SLinus Torvalds cc->key_size = key_size;
32608f0009a2SMilan Broz cc->sector_size = (1 << SECTOR_SHIFT);
3261ff3af92bSMikulas Patocka cc->sector_shift = 0;
32621da177e4SLinus Torvalds
32631da177e4SLinus Torvalds ti->private = cc;
3264ef43aa38SMilan Broz
32655059353dSMikulas Patocka spin_lock(&dm_crypt_clients_lock);
32665059353dSMikulas Patocka dm_crypt_clients_n++;
32675059353dSMikulas Patocka crypt_calculate_pages_per_client();
32685059353dSMikulas Patocka spin_unlock(&dm_crypt_clients_lock);
32695059353dSMikulas Patocka
32705059353dSMikulas Patocka ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL);
32715059353dSMikulas Patocka if (ret < 0)
32725059353dSMikulas Patocka goto bad;
32735059353dSMikulas Patocka
3274ef43aa38SMilan Broz /* Optional parameters need to be read before cipher constructor */
3275ef43aa38SMilan Broz if (argc > 5) {
3276ef43aa38SMilan Broz ret = crypt_ctr_optional(ti, argc - 5, &argv[5]);
3277ef43aa38SMilan Broz if (ret)
3278ef43aa38SMilan Broz goto bad;
3279ef43aa38SMilan Broz }
3280ef43aa38SMilan Broz
32811da177e4SLinus Torvalds ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
32821da177e4SLinus Torvalds if (ret < 0)
32831da177e4SLinus Torvalds goto bad;
32841da177e4SLinus Torvalds
328533d2f09fSMilan Broz if (crypt_integrity_aead(cc)) {
3286ef43aa38SMilan Broz cc->dmreq_start = sizeof(struct aead_request);
3287ef43aa38SMilan Broz cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc));
3288ef43aa38SMilan Broz align_mask = crypto_aead_alignmask(any_tfm_aead(cc));
3289ef43aa38SMilan Broz } else {
3290bbdb23b5SHerbert Xu cc->dmreq_start = sizeof(struct skcipher_request);
3291bbdb23b5SHerbert Xu cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc));
3292ef43aa38SMilan Broz align_mask = crypto_skcipher_alignmask(any_tfm(cc));
3293ef43aa38SMilan Broz }
3294d49ec52fSMikulas Patocka cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
3295d49ec52fSMikulas Patocka
3296ef43aa38SMilan Broz if (align_mask < CRYPTO_MINALIGN) {
3297d49ec52fSMikulas Patocka /* Allocate the padding exactly */
3298d49ec52fSMikulas Patocka iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
3299ef43aa38SMilan Broz & align_mask;
3300d49ec52fSMikulas Patocka } else {
3301d49ec52fSMikulas Patocka /*
3302d49ec52fSMikulas Patocka * If the cipher requires greater alignment than kmalloc
3303d49ec52fSMikulas Patocka * alignment, we don't know the exact position of the
3304d49ec52fSMikulas Patocka * initialization vector. We must assume worst case.
3305d49ec52fSMikulas Patocka */
3306ef43aa38SMilan Broz iv_size_padding = align_mask;
3307d49ec52fSMikulas Patocka }
33081da177e4SLinus Torvalds
3309ef43aa38SMilan Broz /* ...| IV + padding | original IV | original sec. number | bio tag offset | */
3310ef43aa38SMilan Broz additional_req_size = sizeof(struct dm_crypt_request) +
3311ef43aa38SMilan Broz iv_size_padding + cc->iv_size +
3312ef43aa38SMilan Broz cc->iv_size +
3313ef43aa38SMilan Broz sizeof(uint64_t) +
3314ef43aa38SMilan Broz sizeof(unsigned int);
3315ef43aa38SMilan Broz
33166f1c819cSKent Overstreet ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size);
33176f1c819cSKent Overstreet if (ret) {
33181da177e4SLinus Torvalds ti->error = "Cannot allocate crypt request mempool";
33191da177e4SLinus Torvalds goto bad;
33201da177e4SLinus Torvalds }
33211da177e4SLinus Torvalds
332230187e1dSMike Snitzer cc->per_bio_data_size = ti->per_io_data_size =
3323ef43aa38SMilan Broz ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
33247bc75714SCatalin Marinas ARCH_DMA_MINALIGN);
3325298a9fa0SMikulas Patocka
3326a8affc03SChristoph Hellwig ret = mempool_init(&cc->page_pool, BIO_MAX_VECS, crypt_page_alloc, crypt_page_free, cc);
33276f1c819cSKent Overstreet if (ret) {
33288b004457SMilan Broz ti->error = "Cannot allocate page mempool";
3329e48d4bbfSMilan Broz goto bad;
33301da177e4SLinus Torvalds }
3331e48d4bbfSMilan Broz
33326f1c819cSKent Overstreet ret = bioset_init(&cc->bs, MIN_IOS, 0, BIOSET_NEED_BVECS);
33336f1c819cSKent Overstreet if (ret) {
33340c395b0fSMilan Broz ti->error = "Cannot allocate crypt bioset";
3335cabf08e4SMilan Broz goto bad;
333693e605c2SMilan Broz }
3337cabf08e4SMilan Broz
33387145c241SMikulas Patocka mutex_init(&cc->bio_alloc_lock);
33397145c241SMikulas Patocka
3340cabf08e4SMilan Broz ret = -EINVAL;
33418f0009a2SMilan Broz if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) ||
33428f0009a2SMilan Broz (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) {
3343cabf08e4SMilan Broz ti->error = "Invalid iv_offset sector";
3344cabf08e4SMilan Broz goto bad;
33451da177e4SLinus Torvalds }
3346d2a7ad29SKiyoshi Ueda cc->iv_offset = tmpll;
33471da177e4SLinus Torvalds
3348e80d1c80SVivek Goyal ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev);
3349e80d1c80SVivek Goyal if (ret) {
33501da177e4SLinus Torvalds ti->error = "Device lookup failed";
33511da177e4SLinus Torvalds goto bad;
33521da177e4SLinus Torvalds }
33531da177e4SLinus Torvalds
3354e80d1c80SVivek Goyal ret = -EINVAL;
3355ef87bfc2SMilan Broz if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) {
33561da177e4SLinus Torvalds ti->error = "Invalid device sector";
33571da177e4SLinus Torvalds goto bad;
33581da177e4SLinus Torvalds }
33591da177e4SLinus Torvalds cc->start = tmpll;
33601da177e4SLinus Torvalds
3361f34ee1dcSDamien Le Moal if (bdev_is_zoned(cc->dev->bdev)) {
33628e225f04SDamien Le Moal /*
33638e225f04SDamien Le Moal * For zoned block devices, we need to preserve the issuer write
33648e225f04SDamien Le Moal * ordering. To do so, disable write workqueues and force inline
33658e225f04SDamien Le Moal * encryption completion.
33668e225f04SDamien Le Moal */
33678e225f04SDamien Le Moal set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
33688e225f04SDamien Le Moal set_bit(DM_CRYPT_WRITE_INLINE, &cc->flags);
3369f34ee1dcSDamien Le Moal
3370f34ee1dcSDamien Le Moal /*
3371f34ee1dcSDamien Le Moal * All zone append writes to a zone of a zoned block device will
3372f34ee1dcSDamien Le Moal * have the same BIO sector, the start of the zone. When the
3373f34ee1dcSDamien Le Moal * cypher IV mode uses sector values, all data targeting a
3374f34ee1dcSDamien Le Moal * zone will be encrypted using the first sector numbers of the
3375f34ee1dcSDamien Le Moal * zone. This will not result in write errors but will
3376f34ee1dcSDamien Le Moal * cause most reads to fail as reads will use the sector values
3377f34ee1dcSDamien Le Moal * for the actual data locations, resulting in IV mismatch.
3378f34ee1dcSDamien Le Moal * To avoid this problem, ask DM core to emulate zone append
3379f34ee1dcSDamien Le Moal * operations with regular writes.
3380f34ee1dcSDamien Le Moal */
3381f34ee1dcSDamien Le Moal DMDEBUG("Zone append operations will be emulated");
3382f34ee1dcSDamien Le Moal ti->emulate_zone_append = true;
33838e225f04SDamien Le Moal }
33848e225f04SDamien Le Moal
338533d2f09fSMilan Broz if (crypt_integrity_aead(cc) || cc->integrity_iv_size) {
3386ef43aa38SMilan Broz ret = crypt_integrity_ctr(cc, ti);
3387772ae5f5SMilan Broz if (ret)
3388772ae5f5SMilan Broz goto bad;
3389772ae5f5SMilan Broz
3390ef43aa38SMilan Broz cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size;
3391ef43aa38SMilan Broz if (!cc->tag_pool_max_sectors)
3392ef43aa38SMilan Broz cc->tag_pool_max_sectors = 1;
3393ef43aa38SMilan Broz
33946f1c819cSKent Overstreet ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS,
3395ef43aa38SMilan Broz cc->tag_pool_max_sectors * cc->on_disk_tag_size);
33966f1c819cSKent Overstreet if (ret) {
3397ef43aa38SMilan Broz ti->error = "Cannot allocate integrity tags mempool";
3398f3396c58SMikulas Patocka goto bad;
3399f3396c58SMikulas Patocka }
3400772ae5f5SMilan Broz
3401583fe747SMikulas Patocka cc->tag_pool_max_sectors <<= cc->sector_shift;
3402f3396c58SMikulas Patocka }
3403772ae5f5SMilan Broz
34041da177e4SLinus Torvalds ret = -ENOMEM;
3405f612b213SMike Snitzer cc->io_queue = alloc_workqueue("kcryptd_io/%s", WQ_MEM_RECLAIM, 1, devname);
34061da177e4SLinus Torvalds if (!cc->io_queue) {
34071da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd io queue";
34081da177e4SLinus Torvalds goto bad;
34091da177e4SLinus Torvalds }
341037af6560SChristophe Saout
3411f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
341248b0777cSMike Snitzer cc->crypt_queue = alloc_workqueue("kcryptd/%s", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM,
3413ed0302e8SMichał Mirosław 1, devname);
3414f3396c58SMikulas Patocka else
341548b0777cSMike Snitzer cc->crypt_queue = alloc_workqueue("kcryptd/%s",
341648b0777cSMike Snitzer WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND,
3417ed0302e8SMichał Mirosław num_online_cpus(), devname);
34181da177e4SLinus Torvalds if (!cc->crypt_queue) {
34191da177e4SLinus Torvalds ti->error = "Couldn't create kcryptd queue";
34201da177e4SLinus Torvalds goto bad;
34211da177e4SLinus Torvalds }
34221da177e4SLinus Torvalds
3423c7329effSMikulas Patocka spin_lock_init(&cc->write_thread_lock);
3424b3c5fd30SMikulas Patocka cc->write_tree = RB_ROOT;
3425dc267621SMikulas Patocka
3426a5217c11SCai Huoqing cc->write_thread = kthread_run(dmcrypt_write, cc, "dmcrypt_write/%s", devname);
3427dc267621SMikulas Patocka if (IS_ERR(cc->write_thread)) {
3428dc267621SMikulas Patocka ret = PTR_ERR(cc->write_thread);
3429dc267621SMikulas Patocka cc->write_thread = NULL;
3430dc267621SMikulas Patocka ti->error = "Couldn't spawn write thread";
3431dc267621SMikulas Patocka goto bad;
3432dc267621SMikulas Patocka }
3433dc267621SMikulas Patocka
343455a62eefSAlasdair G Kergon ti->num_flush_bios = 1;
3435a666e5c0SMikulas Patocka ti->limit_swap_bios = true;
3436e5524e12SMike Snitzer ti->accounts_remapped_io = true;
3437983c7db3SMilan Broz
343858d0f180SMichael Weiß dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);
34391da177e4SLinus Torvalds return 0;
34401da177e4SLinus Torvalds
34411da177e4SLinus Torvalds bad:
344258d0f180SMichael Weiß dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);
34431da177e4SLinus Torvalds crypt_dtr(ti);
34441da177e4SLinus Torvalds return ret;
3445647c7db1SMikulas Patocka }
3446647c7db1SMikulas Patocka
crypt_map(struct dm_target * ti,struct bio * bio)34477de3ee57SMikulas Patocka static int crypt_map(struct dm_target *ti, struct bio *bio)
34481da177e4SLinus Torvalds {
34491da177e4SLinus Torvalds struct dm_crypt_io *io;
345049a8a920SAlasdair G Kergon struct crypt_config *cc = ti->private;
3451647c7db1SMikulas Patocka
3452772ae5f5SMilan Broz /*
345328a8f0d3SMike Christie * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
345428a8f0d3SMike Christie * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
3455e6047149SMike Christie * - for REQ_OP_DISCARD caller must use flush if IO ordering matters
3456772ae5f5SMilan Broz */
34571eff9d32SJens Axboe if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
345828a8f0d3SMike Christie bio_op(bio) == REQ_OP_DISCARD)) {
345974d46992SChristoph Hellwig bio_set_dev(bio, cc->dev->bdev);
3460772ae5f5SMilan Broz if (bio_sectors(bio))
34614f024f37SKent Overstreet bio->bi_iter.bi_sector = cc->start +
34624f024f37SKent Overstreet dm_target_offset(ti, bio->bi_iter.bi_sector);
3463647c7db1SMikulas Patocka return DM_MAPIO_REMAPPED;
3464647c7db1SMikulas Patocka }
34651da177e4SLinus Torvalds
34664e870e94SMikulas Patocka /*
34674e870e94SMikulas Patocka * Check if bio is too large, split as needed.
34684e870e94SMikulas Patocka */
3469a8affc03SChristoph Hellwig if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_VECS << PAGE_SHIFT)) &&
3470ef43aa38SMilan Broz (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size))
3471a8affc03SChristoph Hellwig dm_accept_partial_bio(bio, ((BIO_MAX_VECS << PAGE_SHIFT) >> SECTOR_SHIFT));
34724e870e94SMikulas Patocka
34738f0009a2SMilan Broz /*
34748f0009a2SMilan Broz * Ensure that bio is a multiple of internal sector encryption size
34758f0009a2SMilan Broz * and is aligned to this size as defined in IO hints.
34768f0009a2SMilan Broz */
34778f0009a2SMilan Broz if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0))
3478846785e6SChristoph Hellwig return DM_MAPIO_KILL;
34798f0009a2SMilan Broz
34808f0009a2SMilan Broz if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1)))
3481846785e6SChristoph Hellwig return DM_MAPIO_KILL;
34828f0009a2SMilan Broz
3483298a9fa0SMikulas Patocka io = dm_per_bio_data(bio, cc->per_bio_data_size);
3484298a9fa0SMikulas Patocka crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
3485ef43aa38SMilan Broz
3486ef43aa38SMilan Broz if (cc->on_disk_tag_size) {
348786a3238cSHeinz Mauelshagen unsigned int tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift);
3488ef43aa38SMilan Broz
3489d715fa23SHeinz Mauelshagen if (unlikely(tag_len > KMALLOC_MAX_SIZE))
3490d715fa23SHeinz Mauelshagen io->integrity_metadata = NULL;
3491d715fa23SHeinz Mauelshagen else
3492d715fa23SHeinz Mauelshagen io->integrity_metadata = kmalloc(tag_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
3493d715fa23SHeinz Mauelshagen
3494d715fa23SHeinz Mauelshagen if (unlikely(!io->integrity_metadata)) {
3495ef43aa38SMilan Broz if (bio_sectors(bio) > cc->tag_pool_max_sectors)
3496ef43aa38SMilan Broz dm_accept_partial_bio(bio, cc->tag_pool_max_sectors);
34976f1c819cSKent Overstreet io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO);
3498ef43aa38SMilan Broz io->integrity_metadata_from_pool = true;
3499ef43aa38SMilan Broz }
3500ef43aa38SMilan Broz }
3501ef43aa38SMilan Broz
350233d2f09fSMilan Broz if (crypt_integrity_aead(cc))
3503ef43aa38SMilan Broz io->ctx.r.req_aead = (struct aead_request *)(io + 1);
3504ef43aa38SMilan Broz else
3505ef43aa38SMilan Broz io->ctx.r.req = (struct skcipher_request *)(io + 1);
35061da177e4SLinus Torvalds
350720c82538SMilan Broz if (bio_data_dir(io->base_bio) == READ) {
3508e5524e12SMike Snitzer if (kcryptd_io_read(io, CRYPT_MAP_READ_GFP))
3509dc267621SMikulas Patocka kcryptd_queue_read(io);
351020c82538SMilan Broz } else
35114ee218cdSAndrew Morton kcryptd_queue_crypt(io);
35124ee218cdSAndrew Morton
35131da177e4SLinus Torvalds return DM_MAPIO_SUBMITTED;
35141da177e4SLinus Torvalds }
35151da177e4SLinus Torvalds
hex2asc(unsigned char c)3516567dd8f3SMikulas Patocka static char hex2asc(unsigned char c)
3517567dd8f3SMikulas Patocka {
351886a3238cSHeinz Mauelshagen return c + '0' + ((unsigned int)(9 - c) >> 4 & 0x27);
3519567dd8f3SMikulas Patocka }
3520567dd8f3SMikulas Patocka
crypt_status(struct dm_target * ti,status_type_t type,unsigned int status_flags,char * result,unsigned int maxlen)3521fd7c092eSMikulas Patocka static void crypt_status(struct dm_target *ti, status_type_t type,
352286a3238cSHeinz Mauelshagen unsigned int status_flags, char *result, unsigned int maxlen)
35231da177e4SLinus Torvalds {
35245ebaee6dSMilan Broz struct crypt_config *cc = ti->private;
352586a3238cSHeinz Mauelshagen unsigned int i, sz = 0;
3526f3396c58SMikulas Patocka int num_feature_args = 0;
35271da177e4SLinus Torvalds
35281da177e4SLinus Torvalds switch (type) {
35291da177e4SLinus Torvalds case STATUSTYPE_INFO:
35301da177e4SLinus Torvalds result[0] = '\0';
35311da177e4SLinus Torvalds break;
35321da177e4SLinus Torvalds
35331da177e4SLinus Torvalds case STATUSTYPE_TABLE:
35347dbcd137SMilan Broz DMEMIT("%s ", cc->cipher_string);
35351da177e4SLinus Torvalds
3536c538f6ecSOndrej Kozina if (cc->key_size > 0) {
3537c538f6ecSOndrej Kozina if (cc->key_string)
3538c538f6ecSOndrej Kozina DMEMIT(":%u:%s", cc->key_size, cc->key_string);
3539567dd8f3SMikulas Patocka else {
3540567dd8f3SMikulas Patocka for (i = 0; i < cc->key_size; i++) {
3541567dd8f3SMikulas Patocka DMEMIT("%c%c", hex2asc(cc->key[i] >> 4),
3542567dd8f3SMikulas Patocka hex2asc(cc->key[i] & 0xf));
3543567dd8f3SMikulas Patocka }
3544567dd8f3SMikulas Patocka }
3545c538f6ecSOndrej Kozina } else
3546fd7c092eSMikulas Patocka DMEMIT("-");
35471da177e4SLinus Torvalds
35481da177e4SLinus Torvalds DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
35491da177e4SLinus Torvalds cc->dev->name, (unsigned long long)cc->start);
3550772ae5f5SMilan Broz
3551f3396c58SMikulas Patocka num_feature_args += !!ti->num_discard_bios;
3552f3396c58SMikulas Patocka num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);
35530f5d8e6eSMikulas Patocka num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
355439d42fa9SIgnat Korchagin num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
355539d42fa9SIgnat Korchagin num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
3556ff3af92bSMikulas Patocka num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);
35578f0009a2SMilan Broz num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
3558ef43aa38SMilan Broz if (cc->on_disk_tag_size)
3559ef43aa38SMilan Broz num_feature_args++;
3560f3396c58SMikulas Patocka if (num_feature_args) {
3561f3396c58SMikulas Patocka DMEMIT(" %d", num_feature_args);
356255a62eefSAlasdair G Kergon if (ti->num_discard_bios)
3563f3396c58SMikulas Patocka DMEMIT(" allow_discards");
3564f3396c58SMikulas Patocka if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
3565f3396c58SMikulas Patocka DMEMIT(" same_cpu_crypt");
35660f5d8e6eSMikulas Patocka if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))
35670f5d8e6eSMikulas Patocka DMEMIT(" submit_from_crypt_cpus");
356839d42fa9SIgnat Korchagin if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags))
356939d42fa9SIgnat Korchagin DMEMIT(" no_read_workqueue");
357039d42fa9SIgnat Korchagin if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))
357139d42fa9SIgnat Korchagin DMEMIT(" no_write_workqueue");
3572ef43aa38SMilan Broz if (cc->on_disk_tag_size)
3573ef43aa38SMilan Broz DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth);
35748f0009a2SMilan Broz if (cc->sector_size != (1 << SECTOR_SHIFT))
35758f0009a2SMilan Broz DMEMIT(" sector_size:%d", cc->sector_size);
35768f0009a2SMilan Broz if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
35778f0009a2SMilan Broz DMEMIT(" iv_large_sectors");
3578f3396c58SMikulas Patocka }
35798ec45662STushar Sugandhi break;
3580772ae5f5SMilan Broz
35818ec45662STushar Sugandhi case STATUSTYPE_IMA:
35828ec45662STushar Sugandhi DMEMIT_TARGET_NAME_VERSION(ti->type);
35838ec45662STushar Sugandhi DMEMIT(",allow_discards=%c", ti->num_discard_bios ? 'y' : 'n');
35848ec45662STushar Sugandhi DMEMIT(",same_cpu_crypt=%c", test_bit(DM_CRYPT_SAME_CPU, &cc->flags) ? 'y' : 'n');
35858ec45662STushar Sugandhi DMEMIT(",submit_from_crypt_cpus=%c", test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) ?
35868ec45662STushar Sugandhi 'y' : 'n');
35878ec45662STushar Sugandhi DMEMIT(",no_read_workqueue=%c", test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags) ?
35888ec45662STushar Sugandhi 'y' : 'n');
35898ec45662STushar Sugandhi DMEMIT(",no_write_workqueue=%c", test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags) ?
35908ec45662STushar Sugandhi 'y' : 'n');
35918ec45662STushar Sugandhi DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ?
35928ec45662STushar Sugandhi 'y' : 'n');
35938ec45662STushar Sugandhi
35948ec45662STushar Sugandhi if (cc->on_disk_tag_size)
35958ec45662STushar Sugandhi DMEMIT(",integrity_tag_size=%u,cipher_auth=%s",
35968ec45662STushar Sugandhi cc->on_disk_tag_size, cc->cipher_auth);
35978ec45662STushar Sugandhi if (cc->sector_size != (1 << SECTOR_SHIFT))
35988ec45662STushar Sugandhi DMEMIT(",sector_size=%d", cc->sector_size);
35998ec45662STushar Sugandhi if (cc->cipher_string)
36008ec45662STushar Sugandhi DMEMIT(",cipher_string=%s", cc->cipher_string);
36018ec45662STushar Sugandhi
36028ec45662STushar Sugandhi DMEMIT(",key_size=%u", cc->key_size);
36038ec45662STushar Sugandhi DMEMIT(",key_parts=%u", cc->key_parts);
36048ec45662STushar Sugandhi DMEMIT(",key_extra_size=%u", cc->key_extra_size);
36058ec45662STushar Sugandhi DMEMIT(",key_mac_size=%u", cc->key_mac_size);
36068ec45662STushar Sugandhi DMEMIT(";");
36071da177e4SLinus Torvalds break;
36081da177e4SLinus Torvalds }
36091da177e4SLinus Torvalds }
36101da177e4SLinus Torvalds
crypt_postsuspend(struct dm_target * ti)3611e48d4bbfSMilan Broz static void crypt_postsuspend(struct dm_target *ti)
3612e48d4bbfSMilan Broz {
3613e48d4bbfSMilan Broz struct crypt_config *cc = ti->private;
3614e48d4bbfSMilan Broz
3615e48d4bbfSMilan Broz set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
3616e48d4bbfSMilan Broz }
3617e48d4bbfSMilan Broz
crypt_preresume(struct dm_target * ti)3618e48d4bbfSMilan Broz static int crypt_preresume(struct dm_target *ti)
3619e48d4bbfSMilan Broz {
3620e48d4bbfSMilan Broz struct crypt_config *cc = ti->private;
3621e48d4bbfSMilan Broz
3622e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
3623e48d4bbfSMilan Broz DMERR("aborting resume - crypt key is not set.");
3624e48d4bbfSMilan Broz return -EAGAIN;
3625e48d4bbfSMilan Broz }
3626e48d4bbfSMilan Broz
3627e48d4bbfSMilan Broz return 0;
3628e48d4bbfSMilan Broz }
3629e48d4bbfSMilan Broz
crypt_resume(struct dm_target * ti)3630e48d4bbfSMilan Broz static void crypt_resume(struct dm_target *ti)
3631e48d4bbfSMilan Broz {
3632e48d4bbfSMilan Broz struct crypt_config *cc = ti->private;
3633e48d4bbfSMilan Broz
3634e48d4bbfSMilan Broz clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
3635e48d4bbfSMilan Broz }
3636e48d4bbfSMilan Broz
3637e48d4bbfSMilan Broz /* Message interface
3638e48d4bbfSMilan Broz * key set <key>
3639e48d4bbfSMilan Broz * key wipe
3640e48d4bbfSMilan Broz */
crypt_message(struct dm_target * ti,unsigned int argc,char ** argv,char * result,unsigned int maxlen)364186a3238cSHeinz Mauelshagen static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv,
364286a3238cSHeinz Mauelshagen char *result, unsigned int maxlen)
3643e48d4bbfSMilan Broz {
3644e48d4bbfSMilan Broz struct crypt_config *cc = ti->private;
3645c538f6ecSOndrej Kozina int key_size, ret = -EINVAL;
3646e48d4bbfSMilan Broz
3647e48d4bbfSMilan Broz if (argc < 2)
3648e48d4bbfSMilan Broz goto error;
3649e48d4bbfSMilan Broz
3650498f0103SMike Snitzer if (!strcasecmp(argv[0], "key")) {
3651e48d4bbfSMilan Broz if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
3652e48d4bbfSMilan Broz DMWARN("not suspended during key manipulation.");
3653e48d4bbfSMilan Broz return -EINVAL;
3654e48d4bbfSMilan Broz }
3655498f0103SMike Snitzer if (argc == 3 && !strcasecmp(argv[1], "set")) {
3656c538f6ecSOndrej Kozina /* The key size may not be changed. */
3657c538f6ecSOndrej Kozina key_size = get_key_size(&argv[2]);
3658c538f6ecSOndrej Kozina if (key_size < 0 || cc->key_size != key_size) {
3659c538f6ecSOndrej Kozina memset(argv[2], '0', strlen(argv[2]));
3660c538f6ecSOndrej Kozina return -EINVAL;
3661c538f6ecSOndrej Kozina }
3662c538f6ecSOndrej Kozina
3663542da317SMilan Broz ret = crypt_set_key(cc, argv[2]);
3664542da317SMilan Broz if (ret)
3665542da317SMilan Broz return ret;
3666542da317SMilan Broz if (cc->iv_gen_ops && cc->iv_gen_ops->init)
3667542da317SMilan Broz ret = cc->iv_gen_ops->init(cc);
3668dc94902bSOndrej Kozina /* wipe the kernel key payload copy */
3669dc94902bSOndrej Kozina if (cc->key_string)
3670dc94902bSOndrej Kozina memset(cc->key, 0, cc->key_size * sizeof(u8));
3671542da317SMilan Broz return ret;
3672542da317SMilan Broz }
36734a52ffc7SMilan Broz if (argc == 2 && !strcasecmp(argv[1], "wipe"))
3674e48d4bbfSMilan Broz return crypt_wipe_key(cc);
3675e48d4bbfSMilan Broz }
3676e48d4bbfSMilan Broz
3677e48d4bbfSMilan Broz error:
3678e48d4bbfSMilan Broz DMWARN("unrecognised message received.");
3679e48d4bbfSMilan Broz return -EINVAL;
3680e48d4bbfSMilan Broz }
3681e48d4bbfSMilan Broz
crypt_iterate_devices(struct dm_target * ti,iterate_devices_callout_fn fn,void * data)3682af4874e0SMike Snitzer static int crypt_iterate_devices(struct dm_target *ti,
3683af4874e0SMike Snitzer iterate_devices_callout_fn fn, void *data)
3684af4874e0SMike Snitzer {
3685af4874e0SMike Snitzer struct crypt_config *cc = ti->private;
3686af4874e0SMike Snitzer
36875dea271bSMike Snitzer return fn(ti, cc->dev, cc->start, ti->len, data);
3688af4874e0SMike Snitzer }
3689af4874e0SMike Snitzer
crypt_io_hints(struct dm_target * ti,struct queue_limits * limits)3690586b286bSMike Snitzer static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
3691586b286bSMike Snitzer {
36928f0009a2SMilan Broz struct crypt_config *cc = ti->private;
36938f0009a2SMilan Broz
3694586b286bSMike Snitzer /*
3695586b286bSMike Snitzer * Unfortunate constraint that is required to avoid the potential
3696586b286bSMike Snitzer * for exceeding underlying device's max_segments limits -- due to
3697586b286bSMike Snitzer * crypt_alloc_buffer() possibly allocating pages for the encryption
3698586b286bSMike Snitzer * bio that are not as physically contiguous as the original bio.
3699586b286bSMike Snitzer */
3700586b286bSMike Snitzer limits->max_segment_size = PAGE_SIZE;
37018f0009a2SMilan Broz
3702bc9e9cf0SMikulas Patocka limits->logical_block_size =
370386a3238cSHeinz Mauelshagen max_t(unsigned int, limits->logical_block_size, cc->sector_size);
3704bc9e9cf0SMikulas Patocka limits->physical_block_size =
370586a3238cSHeinz Mauelshagen max_t(unsigned int, limits->physical_block_size, cc->sector_size);
370686a3238cSHeinz Mauelshagen limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size);
370786e4d3e8SKeith Busch limits->dma_alignment = limits->logical_block_size - 1;
3708586b286bSMike Snitzer }
3709586b286bSMike Snitzer
37101da177e4SLinus Torvalds static struct target_type crypt_target = {
37111da177e4SLinus Torvalds .name = "crypt",
3712e5524e12SMike Snitzer .version = {1, 24, 0},
37131da177e4SLinus Torvalds .module = THIS_MODULE,
37141da177e4SLinus Torvalds .ctr = crypt_ctr,
37151da177e4SLinus Torvalds .dtr = crypt_dtr,
37168e225f04SDamien Le Moal .features = DM_TARGET_ZONED_HM,
37178e225f04SDamien Le Moal .report_zones = crypt_report_zones,
37181da177e4SLinus Torvalds .map = crypt_map,
37191da177e4SLinus Torvalds .status = crypt_status,
3720e48d4bbfSMilan Broz .postsuspend = crypt_postsuspend,
3721e48d4bbfSMilan Broz .preresume = crypt_preresume,
3722e48d4bbfSMilan Broz .resume = crypt_resume,
3723e48d4bbfSMilan Broz .message = crypt_message,
3724af4874e0SMike Snitzer .iterate_devices = crypt_iterate_devices,
3725586b286bSMike Snitzer .io_hints = crypt_io_hints,
37261da177e4SLinus Torvalds };
37273664ff82SYangtao Li module_dm(crypt);
37281da177e4SLinus Torvalds
3729bf14299fSJana Saout MODULE_AUTHOR("Jana Saout <jana@saout.de>");
37301da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
37311da177e4SLinus Torvalds MODULE_LICENSE("GPL");
3732