1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. 4 * 5 * Author: Mikulas Patocka <mpatocka@redhat.com> 6 * 7 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors 8 * 9 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set 10 * default prefetch value. Data are read in "prefetch_cluster" chunks from the 11 * hash device. Setting this greatly improves performance when data and hash 12 * are on the same disk on different partitions on devices with poor random 13 * access behavior. 14 */ 15 16 #include "dm-verity.h" 17 #include "dm-verity-fec.h" 18 #include "dm-verity-verify-sig.h" 19 #include "dm-audit.h" 20 #include <linux/module.h> 21 #include <linux/reboot.h> 22 #include <linux/scatterlist.h> 23 #include <linux/string.h> 24 #include <linux/jump_label.h> 25 26 #define DM_MSG_PREFIX "verity" 27 28 #define DM_VERITY_ENV_LENGTH 42 29 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" 30 31 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 32 33 #define DM_VERITY_MAX_CORRUPTED_ERRS 100 34 35 #define DM_VERITY_OPT_LOGGING "ignore_corruption" 36 #define DM_VERITY_OPT_RESTART "restart_on_corruption" 37 #define DM_VERITY_OPT_PANIC "panic_on_corruption" 38 #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks" 39 #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once" 40 #define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet" 41 42 #define DM_VERITY_OPTS_MAX (4 + DM_VERITY_OPTS_FEC + \ 43 DM_VERITY_ROOT_HASH_VERIFICATION_OPTS) 44 45 static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; 46 47 module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); 48 49 static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled); 50 51 struct dm_verity_prefetch_work { 52 struct work_struct work; 53 struct dm_verity *v; 54 sector_t block; 55 unsigned int n_blocks; 56 }; 57 58 /* 59 * Auxiliary structure appended to each dm-bufio buffer. If the value 60 * hash_verified is nonzero, hash of the block has been verified. 61 * 62 * The variable hash_verified is set to 0 when allocating the buffer, then 63 * it can be changed to 1 and it is never reset to 0 again. 64 * 65 * There is no lock around this value, a race condition can at worst cause 66 * that multiple processes verify the hash of the same buffer simultaneously 67 * and write 1 to hash_verified simultaneously. 68 * This condition is harmless, so we don't need locking. 69 */ 70 struct buffer_aux { 71 int hash_verified; 72 }; 73 74 /* 75 * Initialize struct buffer_aux for a freshly created buffer. 76 */ 77 static void dm_bufio_alloc_callback(struct dm_buffer *buf) 78 { 79 struct buffer_aux *aux = dm_bufio_get_aux_data(buf); 80 81 aux->hash_verified = 0; 82 } 83 84 /* 85 * Translate input sector number to the sector number on the target device. 86 */ 87 static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) 88 { 89 return v->data_start + dm_target_offset(v->ti, bi_sector); 90 } 91 92 /* 93 * Return hash position of a specified block at a specified tree level 94 * (0 is the lowest level). 95 * The lowest "hash_per_block_bits"-bits of the result denote hash position 96 * inside a hash block. The remaining bits denote location of the hash block. 97 */ 98 static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, 99 int level) 100 { 101 return block >> (level * v->hash_per_block_bits); 102 } 103 104 static int verity_hash_update(struct dm_verity *v, struct ahash_request *req, 105 const u8 *data, size_t len, 106 struct crypto_wait *wait) 107 { 108 struct scatterlist sg; 109 110 if (likely(!is_vmalloc_addr(data))) { 111 sg_init_one(&sg, data, len); 112 ahash_request_set_crypt(req, &sg, NULL, len); 113 return crypto_wait_req(crypto_ahash_update(req), wait); 114 } 115 116 do { 117 int r; 118 size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data)); 119 120 flush_kernel_vmap_range((void *)data, this_step); 121 sg_init_table(&sg, 1); 122 sg_set_page(&sg, vmalloc_to_page(data), this_step, offset_in_page(data)); 123 ahash_request_set_crypt(req, &sg, NULL, this_step); 124 r = crypto_wait_req(crypto_ahash_update(req), wait); 125 if (unlikely(r)) 126 return r; 127 data += this_step; 128 len -= this_step; 129 } while (len); 130 131 return 0; 132 } 133 134 /* 135 * Wrapper for crypto_ahash_init, which handles verity salting. 136 */ 137 static int verity_hash_init(struct dm_verity *v, struct ahash_request *req, 138 struct crypto_wait *wait) 139 { 140 int r; 141 142 ahash_request_set_tfm(req, v->tfm); 143 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 144 CRYPTO_TFM_REQ_MAY_BACKLOG, 145 crypto_req_done, (void *)wait); 146 crypto_init_wait(wait); 147 148 r = crypto_wait_req(crypto_ahash_init(req), wait); 149 150 if (unlikely(r < 0)) { 151 DMERR("crypto_ahash_init failed: %d", r); 152 return r; 153 } 154 155 if (likely(v->salt_size && (v->version >= 1))) 156 r = verity_hash_update(v, req, v->salt, v->salt_size, wait); 157 158 return r; 159 } 160 161 static int verity_hash_final(struct dm_verity *v, struct ahash_request *req, 162 u8 *digest, struct crypto_wait *wait) 163 { 164 int r; 165 166 if (unlikely(v->salt_size && (!v->version))) { 167 r = verity_hash_update(v, req, v->salt, v->salt_size, wait); 168 169 if (r < 0) { 170 DMERR("%s failed updating salt: %d", __func__, r); 171 goto out; 172 } 173 } 174 175 ahash_request_set_crypt(req, NULL, digest, 0); 176 r = crypto_wait_req(crypto_ahash_final(req), wait); 177 out: 178 return r; 179 } 180 181 int verity_hash(struct dm_verity *v, struct ahash_request *req, 182 const u8 *data, size_t len, u8 *digest) 183 { 184 int r; 185 struct crypto_wait wait; 186 187 r = verity_hash_init(v, req, &wait); 188 if (unlikely(r < 0)) 189 goto out; 190 191 r = verity_hash_update(v, req, data, len, &wait); 192 if (unlikely(r < 0)) 193 goto out; 194 195 r = verity_hash_final(v, req, digest, &wait); 196 197 out: 198 return r; 199 } 200 201 static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, 202 sector_t *hash_block, unsigned int *offset) 203 { 204 sector_t position = verity_position_at_level(v, block, level); 205 unsigned int idx; 206 207 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); 208 209 if (!offset) 210 return; 211 212 idx = position & ((1 << v->hash_per_block_bits) - 1); 213 if (!v->version) 214 *offset = idx * v->digest_size; 215 else 216 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); 217 } 218 219 /* 220 * Handle verification errors. 221 */ 222 static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, 223 unsigned long long block) 224 { 225 char verity_env[DM_VERITY_ENV_LENGTH]; 226 char *envp[] = { verity_env, NULL }; 227 const char *type_str = ""; 228 struct mapped_device *md = dm_table_get_md(v->ti->table); 229 230 /* Corruption should be visible in device status in all modes */ 231 v->hash_failed = true; 232 233 if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) 234 goto out; 235 236 v->corrupted_errs++; 237 238 switch (type) { 239 case DM_VERITY_BLOCK_TYPE_DATA: 240 type_str = "data"; 241 break; 242 case DM_VERITY_BLOCK_TYPE_METADATA: 243 type_str = "metadata"; 244 break; 245 default: 246 BUG(); 247 } 248 249 DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, 250 type_str, block); 251 252 if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) { 253 DMERR("%s: reached maximum errors", v->data_dev->name); 254 dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0); 255 } 256 257 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", 258 DM_VERITY_ENV_VAR_NAME, type, block); 259 260 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); 261 262 out: 263 if (v->mode == DM_VERITY_MODE_LOGGING) 264 return 0; 265 266 if (v->mode == DM_VERITY_MODE_RESTART) 267 kernel_restart("dm-verity device corrupted"); 268 269 if (v->mode == DM_VERITY_MODE_PANIC) 270 panic("dm-verity device corrupted"); 271 272 return 1; 273 } 274 275 /* 276 * Verify hash of a metadata block pertaining to the specified data block 277 * ("block" argument) at a specified level ("level" argument). 278 * 279 * On successful return, verity_io_want_digest(v, io) contains the hash value 280 * for a lower tree level or for the data block (if we're at the lowest level). 281 * 282 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. 283 * If "skip_unverified" is false, unverified buffer is hashed and verified 284 * against current value of verity_io_want_digest(v, io). 285 */ 286 static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, 287 sector_t block, int level, bool skip_unverified, 288 u8 *want_digest) 289 { 290 struct dm_buffer *buf; 291 struct buffer_aux *aux; 292 u8 *data; 293 int r; 294 sector_t hash_block; 295 unsigned int offset; 296 297 verity_hash_at_level(v, block, level, &hash_block, &offset); 298 299 if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { 300 data = dm_bufio_get(v->bufio, hash_block, &buf); 301 if (data == NULL) { 302 /* 303 * In tasklet and the hash was not in the bufio cache. 304 * Return early and resume execution from a work-queue 305 * to read the hash from disk. 306 */ 307 return -EAGAIN; 308 } 309 } else 310 data = dm_bufio_read(v->bufio, hash_block, &buf); 311 312 if (IS_ERR(data)) 313 return PTR_ERR(data); 314 315 aux = dm_bufio_get_aux_data(buf); 316 317 if (!aux->hash_verified) { 318 if (skip_unverified) { 319 r = 1; 320 goto release_ret_r; 321 } 322 323 r = verity_hash(v, verity_io_hash_req(v, io), 324 data, 1 << v->hash_dev_block_bits, 325 verity_io_real_digest(v, io)); 326 if (unlikely(r < 0)) 327 goto release_ret_r; 328 329 if (likely(memcmp(verity_io_real_digest(v, io), want_digest, 330 v->digest_size) == 0)) 331 aux->hash_verified = 1; 332 else if (static_branch_unlikely(&use_tasklet_enabled) && 333 io->in_tasklet) { 334 /* 335 * Error handling code (FEC included) cannot be run in a 336 * tasklet since it may sleep, so fallback to work-queue. 337 */ 338 r = -EAGAIN; 339 goto release_ret_r; 340 } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA, 341 hash_block, data, NULL) == 0) 342 aux->hash_verified = 1; 343 else if (verity_handle_err(v, 344 DM_VERITY_BLOCK_TYPE_METADATA, 345 hash_block)) { 346 struct bio *bio = 347 dm_bio_from_per_bio_data(io, 348 v->ti->per_io_data_size); 349 dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio, 350 block, 0); 351 r = -EIO; 352 goto release_ret_r; 353 } 354 } 355 356 data += offset; 357 memcpy(want_digest, data, v->digest_size); 358 r = 0; 359 360 release_ret_r: 361 dm_bufio_release(buf); 362 return r; 363 } 364 365 /* 366 * Find a hash for a given block, write it to digest and verify the integrity 367 * of the hash tree if necessary. 368 */ 369 int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, 370 sector_t block, u8 *digest, bool *is_zero) 371 { 372 int r = 0, i; 373 374 if (likely(v->levels)) { 375 /* 376 * First, we try to get the requested hash for 377 * the current block. If the hash block itself is 378 * verified, zero is returned. If it isn't, this 379 * function returns 1 and we fall back to whole 380 * chain verification. 381 */ 382 r = verity_verify_level(v, io, block, 0, true, digest); 383 if (likely(r <= 0)) 384 goto out; 385 } 386 387 memcpy(digest, v->root_digest, v->digest_size); 388 389 for (i = v->levels - 1; i >= 0; i--) { 390 r = verity_verify_level(v, io, block, i, false, digest); 391 if (unlikely(r)) 392 goto out; 393 } 394 out: 395 if (!r && v->zero_digest) 396 *is_zero = !memcmp(v->zero_digest, digest, v->digest_size); 397 else 398 *is_zero = false; 399 400 return r; 401 } 402 403 /* 404 * Calculates the digest for the given bio 405 */ 406 static int verity_for_io_block(struct dm_verity *v, struct dm_verity_io *io, 407 struct bvec_iter *iter, struct crypto_wait *wait) 408 { 409 unsigned int todo = 1 << v->data_dev_block_bits; 410 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 411 struct scatterlist sg; 412 struct ahash_request *req = verity_io_hash_req(v, io); 413 414 do { 415 int r; 416 unsigned int len; 417 struct bio_vec bv = bio_iter_iovec(bio, *iter); 418 419 sg_init_table(&sg, 1); 420 421 len = bv.bv_len; 422 423 if (likely(len >= todo)) 424 len = todo; 425 /* 426 * Operating on a single page at a time looks suboptimal 427 * until you consider the typical block size is 4,096B. 428 * Going through this loops twice should be very rare. 429 */ 430 sg_set_page(&sg, bv.bv_page, len, bv.bv_offset); 431 ahash_request_set_crypt(req, &sg, NULL, len); 432 r = crypto_wait_req(crypto_ahash_update(req), wait); 433 434 if (unlikely(r < 0)) { 435 DMERR("%s crypto op failed: %d", __func__, r); 436 return r; 437 } 438 439 bio_advance_iter(bio, iter, len); 440 todo -= len; 441 } while (todo); 442 443 return 0; 444 } 445 446 /* 447 * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec 448 * starting from iter. 449 */ 450 int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, 451 struct bvec_iter *iter, 452 int (*process)(struct dm_verity *v, 453 struct dm_verity_io *io, u8 *data, 454 size_t len)) 455 { 456 unsigned int todo = 1 << v->data_dev_block_bits; 457 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 458 459 do { 460 int r; 461 u8 *page; 462 unsigned int len; 463 struct bio_vec bv = bio_iter_iovec(bio, *iter); 464 465 page = bvec_kmap_local(&bv); 466 len = bv.bv_len; 467 468 if (likely(len >= todo)) 469 len = todo; 470 471 r = process(v, io, page, len); 472 kunmap_local(page); 473 474 if (r < 0) 475 return r; 476 477 bio_advance_iter(bio, iter, len); 478 todo -= len; 479 } while (todo); 480 481 return 0; 482 } 483 484 static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io, 485 u8 *data, size_t len) 486 { 487 memset(data, 0, len); 488 return 0; 489 } 490 491 /* 492 * Moves the bio iter one data block forward. 493 */ 494 static inline void verity_bv_skip_block(struct dm_verity *v, 495 struct dm_verity_io *io, 496 struct bvec_iter *iter) 497 { 498 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 499 500 bio_advance_iter(bio, iter, 1 << v->data_dev_block_bits); 501 } 502 503 /* 504 * Verify one "dm_verity_io" structure. 505 */ 506 static int verity_verify_io(struct dm_verity_io *io) 507 { 508 bool is_zero; 509 struct dm_verity *v = io->v; 510 #if defined(CONFIG_DM_VERITY_FEC) 511 struct bvec_iter start; 512 #endif 513 struct bvec_iter iter_copy; 514 struct bvec_iter *iter; 515 struct crypto_wait wait; 516 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 517 unsigned int b; 518 519 if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { 520 /* 521 * Copy the iterator in case we need to restart 522 * verification in a work-queue. 523 */ 524 iter_copy = io->iter; 525 iter = &iter_copy; 526 } else 527 iter = &io->iter; 528 529 for (b = 0; b < io->n_blocks; b++) { 530 int r; 531 sector_t cur_block = io->block + b; 532 struct ahash_request *req = verity_io_hash_req(v, io); 533 534 if (v->validated_blocks && bio->bi_status == BLK_STS_OK && 535 likely(test_bit(cur_block, v->validated_blocks))) { 536 verity_bv_skip_block(v, io, iter); 537 continue; 538 } 539 540 r = verity_hash_for_block(v, io, cur_block, 541 verity_io_want_digest(v, io), 542 &is_zero); 543 if (unlikely(r < 0)) 544 return r; 545 546 if (is_zero) { 547 /* 548 * If we expect a zero block, don't validate, just 549 * return zeros. 550 */ 551 r = verity_for_bv_block(v, io, iter, 552 verity_bv_zero); 553 if (unlikely(r < 0)) 554 return r; 555 556 continue; 557 } 558 559 r = verity_hash_init(v, req, &wait); 560 if (unlikely(r < 0)) 561 return r; 562 563 #if defined(CONFIG_DM_VERITY_FEC) 564 if (verity_fec_is_enabled(v)) 565 start = *iter; 566 #endif 567 r = verity_for_io_block(v, io, iter, &wait); 568 if (unlikely(r < 0)) 569 return r; 570 571 r = verity_hash_final(v, req, verity_io_real_digest(v, io), 572 &wait); 573 if (unlikely(r < 0)) 574 return r; 575 576 if (likely(memcmp(verity_io_real_digest(v, io), 577 verity_io_want_digest(v, io), v->digest_size) == 0)) { 578 if (v->validated_blocks) 579 set_bit(cur_block, v->validated_blocks); 580 continue; 581 } else if (static_branch_unlikely(&use_tasklet_enabled) && 582 io->in_tasklet) { 583 /* 584 * Error handling code (FEC included) cannot be run in a 585 * tasklet since it may sleep, so fallback to work-queue. 586 */ 587 return -EAGAIN; 588 #if defined(CONFIG_DM_VERITY_FEC) 589 } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, 590 cur_block, NULL, &start) == 0) { 591 continue; 592 #endif 593 } else { 594 if (bio->bi_status) { 595 /* 596 * Error correction failed; Just return error 597 */ 598 return -EIO; 599 } 600 if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, 601 cur_block)) { 602 dm_audit_log_bio(DM_MSG_PREFIX, "verify-data", 603 bio, cur_block, 0); 604 return -EIO; 605 } 606 } 607 } 608 609 return 0; 610 } 611 612 /* 613 * Skip verity work in response to I/O error when system is shutting down. 614 */ 615 static inline bool verity_is_system_shutting_down(void) 616 { 617 return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF 618 || system_state == SYSTEM_RESTART; 619 } 620 621 /* 622 * End one "io" structure with a given error. 623 */ 624 static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) 625 { 626 struct dm_verity *v = io->v; 627 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 628 629 bio->bi_end_io = io->orig_bi_end_io; 630 bio->bi_status = status; 631 632 if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet) 633 verity_fec_finish_io(io); 634 635 bio_endio(bio); 636 } 637 638 static void verity_work(struct work_struct *w) 639 { 640 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); 641 642 io->in_tasklet = false; 643 644 verity_fec_init_io(io); 645 verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); 646 } 647 648 static void verity_tasklet(unsigned long data) 649 { 650 struct dm_verity_io *io = (struct dm_verity_io *)data; 651 int err; 652 653 io->in_tasklet = true; 654 err = verity_verify_io(io); 655 if (err == -EAGAIN) { 656 /* fallback to retrying with work-queue */ 657 INIT_WORK(&io->work, verity_work); 658 queue_work(io->v->verify_wq, &io->work); 659 return; 660 } 661 662 verity_finish_io(io, errno_to_blk_status(err)); 663 } 664 665 static void verity_end_io(struct bio *bio) 666 { 667 struct dm_verity_io *io = bio->bi_private; 668 669 if (bio->bi_status && 670 (!verity_fec_is_enabled(io->v) || verity_is_system_shutting_down())) { 671 verity_finish_io(io, bio->bi_status); 672 return; 673 } 674 675 if (static_branch_unlikely(&use_tasklet_enabled) && io->v->use_tasklet) { 676 tasklet_init(&io->tasklet, verity_tasklet, (unsigned long)io); 677 tasklet_schedule(&io->tasklet); 678 } else { 679 INIT_WORK(&io->work, verity_work); 680 queue_work(io->v->verify_wq, &io->work); 681 } 682 } 683 684 /* 685 * Prefetch buffers for the specified io. 686 * The root buffer is not prefetched, it is assumed that it will be cached 687 * all the time. 688 */ 689 static void verity_prefetch_io(struct work_struct *work) 690 { 691 struct dm_verity_prefetch_work *pw = 692 container_of(work, struct dm_verity_prefetch_work, work); 693 struct dm_verity *v = pw->v; 694 int i; 695 696 for (i = v->levels - 2; i >= 0; i--) { 697 sector_t hash_block_start; 698 sector_t hash_block_end; 699 700 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); 701 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); 702 703 if (!i) { 704 unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster); 705 706 cluster >>= v->data_dev_block_bits; 707 if (unlikely(!cluster)) 708 goto no_prefetch_cluster; 709 710 if (unlikely(cluster & (cluster - 1))) 711 cluster = 1 << __fls(cluster); 712 713 hash_block_start &= ~(sector_t)(cluster - 1); 714 hash_block_end |= cluster - 1; 715 if (unlikely(hash_block_end >= v->hash_blocks)) 716 hash_block_end = v->hash_blocks - 1; 717 } 718 no_prefetch_cluster: 719 dm_bufio_prefetch(v->bufio, hash_block_start, 720 hash_block_end - hash_block_start + 1); 721 } 722 723 kfree(pw); 724 } 725 726 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) 727 { 728 sector_t block = io->block; 729 unsigned int n_blocks = io->n_blocks; 730 struct dm_verity_prefetch_work *pw; 731 732 if (v->validated_blocks) { 733 while (n_blocks && test_bit(block, v->validated_blocks)) { 734 block++; 735 n_blocks--; 736 } 737 while (n_blocks && test_bit(block + n_blocks - 1, 738 v->validated_blocks)) 739 n_blocks--; 740 if (!n_blocks) 741 return; 742 } 743 744 pw = kmalloc(sizeof(struct dm_verity_prefetch_work), 745 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 746 747 if (!pw) 748 return; 749 750 INIT_WORK(&pw->work, verity_prefetch_io); 751 pw->v = v; 752 pw->block = block; 753 pw->n_blocks = n_blocks; 754 queue_work(v->verify_wq, &pw->work); 755 } 756 757 /* 758 * Bio map function. It allocates dm_verity_io structure and bio vector and 759 * fills them. Then it issues prefetches and the I/O. 760 */ 761 static int verity_map(struct dm_target *ti, struct bio *bio) 762 { 763 struct dm_verity *v = ti->private; 764 struct dm_verity_io *io; 765 766 bio_set_dev(bio, v->data_dev->bdev); 767 bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); 768 769 if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) & 770 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { 771 DMERR_LIMIT("unaligned io"); 772 return DM_MAPIO_KILL; 773 } 774 775 if (bio_end_sector(bio) >> 776 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { 777 DMERR_LIMIT("io out of range"); 778 return DM_MAPIO_KILL; 779 } 780 781 if (bio_data_dir(bio) == WRITE) 782 return DM_MAPIO_KILL; 783 784 io = dm_per_bio_data(bio, ti->per_io_data_size); 785 io->v = v; 786 io->orig_bi_end_io = bio->bi_end_io; 787 io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); 788 io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; 789 790 bio->bi_end_io = verity_end_io; 791 bio->bi_private = io; 792 io->iter = bio->bi_iter; 793 794 verity_submit_prefetch(v, io); 795 796 submit_bio_noacct(bio); 797 798 return DM_MAPIO_SUBMITTED; 799 } 800 801 /* 802 * Status: V (valid) or C (corruption found) 803 */ 804 static void verity_status(struct dm_target *ti, status_type_t type, 805 unsigned int status_flags, char *result, unsigned int maxlen) 806 { 807 struct dm_verity *v = ti->private; 808 unsigned int args = 0; 809 unsigned int sz = 0; 810 unsigned int x; 811 812 switch (type) { 813 case STATUSTYPE_INFO: 814 DMEMIT("%c", v->hash_failed ? 'C' : 'V'); 815 break; 816 case STATUSTYPE_TABLE: 817 DMEMIT("%u %s %s %u %u %llu %llu %s ", 818 v->version, 819 v->data_dev->name, 820 v->hash_dev->name, 821 1 << v->data_dev_block_bits, 822 1 << v->hash_dev_block_bits, 823 (unsigned long long)v->data_blocks, 824 (unsigned long long)v->hash_start, 825 v->alg_name 826 ); 827 for (x = 0; x < v->digest_size; x++) 828 DMEMIT("%02x", v->root_digest[x]); 829 DMEMIT(" "); 830 if (!v->salt_size) 831 DMEMIT("-"); 832 else 833 for (x = 0; x < v->salt_size; x++) 834 DMEMIT("%02x", v->salt[x]); 835 if (v->mode != DM_VERITY_MODE_EIO) 836 args++; 837 if (verity_fec_is_enabled(v)) 838 args += DM_VERITY_OPTS_FEC; 839 if (v->zero_digest) 840 args++; 841 if (v->validated_blocks) 842 args++; 843 if (v->use_tasklet) 844 args++; 845 if (v->signature_key_desc) 846 args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; 847 if (!args) 848 return; 849 DMEMIT(" %u", args); 850 if (v->mode != DM_VERITY_MODE_EIO) { 851 DMEMIT(" "); 852 switch (v->mode) { 853 case DM_VERITY_MODE_LOGGING: 854 DMEMIT(DM_VERITY_OPT_LOGGING); 855 break; 856 case DM_VERITY_MODE_RESTART: 857 DMEMIT(DM_VERITY_OPT_RESTART); 858 break; 859 case DM_VERITY_MODE_PANIC: 860 DMEMIT(DM_VERITY_OPT_PANIC); 861 break; 862 default: 863 BUG(); 864 } 865 } 866 if (v->zero_digest) 867 DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); 868 if (v->validated_blocks) 869 DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); 870 if (v->use_tasklet) 871 DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); 872 sz = verity_fec_status_table(v, sz, result, maxlen); 873 if (v->signature_key_desc) 874 DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY 875 " %s", v->signature_key_desc); 876 break; 877 878 case STATUSTYPE_IMA: 879 DMEMIT_TARGET_NAME_VERSION(ti->type); 880 DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V'); 881 DMEMIT(",verity_version=%u", v->version); 882 DMEMIT(",data_device_name=%s", v->data_dev->name); 883 DMEMIT(",hash_device_name=%s", v->hash_dev->name); 884 DMEMIT(",verity_algorithm=%s", v->alg_name); 885 886 DMEMIT(",root_digest="); 887 for (x = 0; x < v->digest_size; x++) 888 DMEMIT("%02x", v->root_digest[x]); 889 890 DMEMIT(",salt="); 891 if (!v->salt_size) 892 DMEMIT("-"); 893 else 894 for (x = 0; x < v->salt_size; x++) 895 DMEMIT("%02x", v->salt[x]); 896 897 DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n'); 898 DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n'); 899 if (v->signature_key_desc) 900 DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc); 901 902 if (v->mode != DM_VERITY_MODE_EIO) { 903 DMEMIT(",verity_mode="); 904 switch (v->mode) { 905 case DM_VERITY_MODE_LOGGING: 906 DMEMIT(DM_VERITY_OPT_LOGGING); 907 break; 908 case DM_VERITY_MODE_RESTART: 909 DMEMIT(DM_VERITY_OPT_RESTART); 910 break; 911 case DM_VERITY_MODE_PANIC: 912 DMEMIT(DM_VERITY_OPT_PANIC); 913 break; 914 default: 915 DMEMIT("invalid"); 916 } 917 } 918 DMEMIT(";"); 919 break; 920 } 921 } 922 923 static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) 924 { 925 struct dm_verity *v = ti->private; 926 927 *bdev = v->data_dev->bdev; 928 929 if (v->data_start || ti->len != bdev_nr_sectors(v->data_dev->bdev)) 930 return 1; 931 return 0; 932 } 933 934 static int verity_iterate_devices(struct dm_target *ti, 935 iterate_devices_callout_fn fn, void *data) 936 { 937 struct dm_verity *v = ti->private; 938 939 return fn(ti, v->data_dev, v->data_start, ti->len, data); 940 } 941 942 static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) 943 { 944 struct dm_verity *v = ti->private; 945 946 if (limits->logical_block_size < 1 << v->data_dev_block_bits) 947 limits->logical_block_size = 1 << v->data_dev_block_bits; 948 949 if (limits->physical_block_size < 1 << v->data_dev_block_bits) 950 limits->physical_block_size = 1 << v->data_dev_block_bits; 951 952 blk_limits_io_min(limits, limits->logical_block_size); 953 } 954 955 static void verity_dtr(struct dm_target *ti) 956 { 957 struct dm_verity *v = ti->private; 958 959 if (v->verify_wq) 960 destroy_workqueue(v->verify_wq); 961 962 if (v->bufio) 963 dm_bufio_client_destroy(v->bufio); 964 965 kvfree(v->validated_blocks); 966 kfree(v->salt); 967 kfree(v->root_digest); 968 kfree(v->zero_digest); 969 970 if (v->tfm) 971 crypto_free_ahash(v->tfm); 972 973 kfree(v->alg_name); 974 975 if (v->hash_dev) 976 dm_put_device(ti, v->hash_dev); 977 978 if (v->data_dev) 979 dm_put_device(ti, v->data_dev); 980 981 verity_fec_dtr(v); 982 983 kfree(v->signature_key_desc); 984 985 if (v->use_tasklet) 986 static_branch_dec(&use_tasklet_enabled); 987 988 kfree(v); 989 990 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); 991 } 992 993 static int verity_alloc_most_once(struct dm_verity *v) 994 { 995 struct dm_target *ti = v->ti; 996 997 /* the bitset can only handle INT_MAX blocks */ 998 if (v->data_blocks > INT_MAX) { 999 ti->error = "device too large to use check_at_most_once"; 1000 return -E2BIG; 1001 } 1002 1003 v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks), 1004 sizeof(unsigned long), 1005 GFP_KERNEL); 1006 if (!v->validated_blocks) { 1007 ti->error = "failed to allocate bitset for check_at_most_once"; 1008 return -ENOMEM; 1009 } 1010 1011 return 0; 1012 } 1013 1014 static int verity_alloc_zero_digest(struct dm_verity *v) 1015 { 1016 int r = -ENOMEM; 1017 struct ahash_request *req; 1018 u8 *zero_data; 1019 1020 v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); 1021 1022 if (!v->zero_digest) 1023 return r; 1024 1025 req = kmalloc(v->ahash_reqsize, GFP_KERNEL); 1026 1027 if (!req) 1028 return r; /* verity_dtr will free zero_digest */ 1029 1030 zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); 1031 1032 if (!zero_data) 1033 goto out; 1034 1035 r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits, 1036 v->zero_digest); 1037 1038 out: 1039 kfree(req); 1040 kfree(zero_data); 1041 1042 return r; 1043 } 1044 1045 static inline bool verity_is_verity_mode(const char *arg_name) 1046 { 1047 return (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING) || 1048 !strcasecmp(arg_name, DM_VERITY_OPT_RESTART) || 1049 !strcasecmp(arg_name, DM_VERITY_OPT_PANIC)); 1050 } 1051 1052 static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name) 1053 { 1054 if (v->mode) 1055 return -EINVAL; 1056 1057 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) 1058 v->mode = DM_VERITY_MODE_LOGGING; 1059 else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) 1060 v->mode = DM_VERITY_MODE_RESTART; 1061 else if (!strcasecmp(arg_name, DM_VERITY_OPT_PANIC)) 1062 v->mode = DM_VERITY_MODE_PANIC; 1063 1064 return 0; 1065 } 1066 1067 static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, 1068 struct dm_verity_sig_opts *verify_args, 1069 bool only_modifier_opts) 1070 { 1071 int r = 0; 1072 unsigned int argc; 1073 struct dm_target *ti = v->ti; 1074 const char *arg_name; 1075 1076 static const struct dm_arg _args[] = { 1077 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, 1078 }; 1079 1080 r = dm_read_arg_group(_args, as, &argc, &ti->error); 1081 if (r) 1082 return -EINVAL; 1083 1084 if (!argc) 1085 return 0; 1086 1087 do { 1088 arg_name = dm_shift_arg(as); 1089 argc--; 1090 1091 if (verity_is_verity_mode(arg_name)) { 1092 if (only_modifier_opts) 1093 continue; 1094 r = verity_parse_verity_mode(v, arg_name); 1095 if (r) { 1096 ti->error = "Conflicting error handling parameters"; 1097 return r; 1098 } 1099 continue; 1100 1101 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) { 1102 if (only_modifier_opts) 1103 continue; 1104 r = verity_alloc_zero_digest(v); 1105 if (r) { 1106 ti->error = "Cannot allocate zero digest"; 1107 return r; 1108 } 1109 continue; 1110 1111 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) { 1112 if (only_modifier_opts) 1113 continue; 1114 r = verity_alloc_most_once(v); 1115 if (r) 1116 return r; 1117 continue; 1118 1119 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { 1120 v->use_tasklet = true; 1121 static_branch_inc(&use_tasklet_enabled); 1122 continue; 1123 1124 } else if (verity_is_fec_opt_arg(arg_name)) { 1125 if (only_modifier_opts) 1126 continue; 1127 r = verity_fec_parse_opt_args(as, v, &argc, arg_name); 1128 if (r) 1129 return r; 1130 continue; 1131 1132 } else if (verity_verify_is_sig_opt_arg(arg_name)) { 1133 if (only_modifier_opts) 1134 continue; 1135 r = verity_verify_sig_parse_opt_args(as, v, 1136 verify_args, 1137 &argc, arg_name); 1138 if (r) 1139 return r; 1140 continue; 1141 1142 } else if (only_modifier_opts) { 1143 /* 1144 * Ignore unrecognized opt, could easily be an extra 1145 * argument to an option whose parsing was skipped. 1146 * Normal parsing (@only_modifier_opts=false) will 1147 * properly parse all options (and their extra args). 1148 */ 1149 continue; 1150 } 1151 1152 DMERR("Unrecognized verity feature request: %s", arg_name); 1153 ti->error = "Unrecognized verity feature request"; 1154 return -EINVAL; 1155 } while (argc && !r); 1156 1157 return r; 1158 } 1159 1160 /* 1161 * Target parameters: 1162 * <version> The current format is version 1. 1163 * Vsn 0 is compatible with original Chromium OS releases. 1164 * <data device> 1165 * <hash device> 1166 * <data block size> 1167 * <hash block size> 1168 * <the number of data blocks> 1169 * <hash start block> 1170 * <algorithm> 1171 * <digest> 1172 * <salt> Hex string or "-" if no salt. 1173 */ 1174 static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) 1175 { 1176 struct dm_verity *v; 1177 struct dm_verity_sig_opts verify_args = {0}; 1178 struct dm_arg_set as; 1179 unsigned int num; 1180 unsigned long long num_ll; 1181 int r; 1182 int i; 1183 sector_t hash_position; 1184 char dummy; 1185 char *root_hash_digest_to_validate; 1186 1187 v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); 1188 if (!v) { 1189 ti->error = "Cannot allocate verity structure"; 1190 return -ENOMEM; 1191 } 1192 ti->private = v; 1193 v->ti = ti; 1194 1195 r = verity_fec_ctr_alloc(v); 1196 if (r) 1197 goto bad; 1198 1199 if ((dm_table_get_mode(ti->table) & ~BLK_OPEN_READ)) { 1200 ti->error = "Device must be readonly"; 1201 r = -EINVAL; 1202 goto bad; 1203 } 1204 1205 if (argc < 10) { 1206 ti->error = "Not enough arguments"; 1207 r = -EINVAL; 1208 goto bad; 1209 } 1210 1211 /* Parse optional parameters that modify primary args */ 1212 if (argc > 10) { 1213 as.argc = argc - 10; 1214 as.argv = argv + 10; 1215 r = verity_parse_opt_args(&as, v, &verify_args, true); 1216 if (r < 0) 1217 goto bad; 1218 } 1219 1220 if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || 1221 num > 1) { 1222 ti->error = "Invalid version"; 1223 r = -EINVAL; 1224 goto bad; 1225 } 1226 v->version = num; 1227 1228 r = dm_get_device(ti, argv[1], BLK_OPEN_READ, &v->data_dev); 1229 if (r) { 1230 ti->error = "Data device lookup failed"; 1231 goto bad; 1232 } 1233 1234 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &v->hash_dev); 1235 if (r) { 1236 ti->error = "Hash device lookup failed"; 1237 goto bad; 1238 } 1239 1240 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || 1241 !num || (num & (num - 1)) || 1242 num < bdev_logical_block_size(v->data_dev->bdev) || 1243 num > PAGE_SIZE) { 1244 ti->error = "Invalid data device block size"; 1245 r = -EINVAL; 1246 goto bad; 1247 } 1248 v->data_dev_block_bits = __ffs(num); 1249 1250 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || 1251 !num || (num & (num - 1)) || 1252 num < bdev_logical_block_size(v->hash_dev->bdev) || 1253 num > INT_MAX) { 1254 ti->error = "Invalid hash device block size"; 1255 r = -EINVAL; 1256 goto bad; 1257 } 1258 v->hash_dev_block_bits = __ffs(num); 1259 1260 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || 1261 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) 1262 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { 1263 ti->error = "Invalid data blocks"; 1264 r = -EINVAL; 1265 goto bad; 1266 } 1267 v->data_blocks = num_ll; 1268 1269 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { 1270 ti->error = "Data device is too small"; 1271 r = -EINVAL; 1272 goto bad; 1273 } 1274 1275 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || 1276 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) 1277 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { 1278 ti->error = "Invalid hash start"; 1279 r = -EINVAL; 1280 goto bad; 1281 } 1282 v->hash_start = num_ll; 1283 1284 v->alg_name = kstrdup(argv[7], GFP_KERNEL); 1285 if (!v->alg_name) { 1286 ti->error = "Cannot allocate algorithm name"; 1287 r = -ENOMEM; 1288 goto bad; 1289 } 1290 1291 v->tfm = crypto_alloc_ahash(v->alg_name, 0, 1292 v->use_tasklet ? CRYPTO_ALG_ASYNC : 0); 1293 if (IS_ERR(v->tfm)) { 1294 ti->error = "Cannot initialize hash function"; 1295 r = PTR_ERR(v->tfm); 1296 v->tfm = NULL; 1297 goto bad; 1298 } 1299 1300 /* 1301 * dm-verity performance can vary greatly depending on which hash 1302 * algorithm implementation is used. Help people debug performance 1303 * problems by logging the ->cra_driver_name. 1304 */ 1305 DMINFO("%s using implementation \"%s\"", v->alg_name, 1306 crypto_hash_alg_common(v->tfm)->base.cra_driver_name); 1307 1308 v->digest_size = crypto_ahash_digestsize(v->tfm); 1309 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { 1310 ti->error = "Digest size too big"; 1311 r = -EINVAL; 1312 goto bad; 1313 } 1314 v->ahash_reqsize = sizeof(struct ahash_request) + 1315 crypto_ahash_reqsize(v->tfm); 1316 1317 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); 1318 if (!v->root_digest) { 1319 ti->error = "Cannot allocate root digest"; 1320 r = -ENOMEM; 1321 goto bad; 1322 } 1323 if (strlen(argv[8]) != v->digest_size * 2 || 1324 hex2bin(v->root_digest, argv[8], v->digest_size)) { 1325 ti->error = "Invalid root digest"; 1326 r = -EINVAL; 1327 goto bad; 1328 } 1329 root_hash_digest_to_validate = argv[8]; 1330 1331 if (strcmp(argv[9], "-")) { 1332 v->salt_size = strlen(argv[9]) / 2; 1333 v->salt = kmalloc(v->salt_size, GFP_KERNEL); 1334 if (!v->salt) { 1335 ti->error = "Cannot allocate salt"; 1336 r = -ENOMEM; 1337 goto bad; 1338 } 1339 if (strlen(argv[9]) != v->salt_size * 2 || 1340 hex2bin(v->salt, argv[9], v->salt_size)) { 1341 ti->error = "Invalid salt"; 1342 r = -EINVAL; 1343 goto bad; 1344 } 1345 } 1346 1347 argv += 10; 1348 argc -= 10; 1349 1350 /* Optional parameters */ 1351 if (argc) { 1352 as.argc = argc; 1353 as.argv = argv; 1354 r = verity_parse_opt_args(&as, v, &verify_args, false); 1355 if (r < 0) 1356 goto bad; 1357 } 1358 1359 /* Root hash signature is a optional parameter*/ 1360 r = verity_verify_root_hash(root_hash_digest_to_validate, 1361 strlen(root_hash_digest_to_validate), 1362 verify_args.sig, 1363 verify_args.sig_size); 1364 if (r < 0) { 1365 ti->error = "Root hash verification failed"; 1366 goto bad; 1367 } 1368 v->hash_per_block_bits = 1369 __fls((1 << v->hash_dev_block_bits) / v->digest_size); 1370 1371 v->levels = 0; 1372 if (v->data_blocks) 1373 while (v->hash_per_block_bits * v->levels < 64 && 1374 (unsigned long long)(v->data_blocks - 1) >> 1375 (v->hash_per_block_bits * v->levels)) 1376 v->levels++; 1377 1378 if (v->levels > DM_VERITY_MAX_LEVELS) { 1379 ti->error = "Too many tree levels"; 1380 r = -E2BIG; 1381 goto bad; 1382 } 1383 1384 hash_position = v->hash_start; 1385 for (i = v->levels - 1; i >= 0; i--) { 1386 sector_t s; 1387 1388 v->hash_level_block[i] = hash_position; 1389 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) 1390 >> ((i + 1) * v->hash_per_block_bits); 1391 if (hash_position + s < hash_position) { 1392 ti->error = "Hash device offset overflow"; 1393 r = -E2BIG; 1394 goto bad; 1395 } 1396 hash_position += s; 1397 } 1398 v->hash_blocks = hash_position; 1399 1400 v->bufio = dm_bufio_client_create(v->hash_dev->bdev, 1401 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), 1402 dm_bufio_alloc_callback, NULL, 1403 v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0); 1404 if (IS_ERR(v->bufio)) { 1405 ti->error = "Cannot initialize dm-bufio"; 1406 r = PTR_ERR(v->bufio); 1407 v->bufio = NULL; 1408 goto bad; 1409 } 1410 1411 if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) { 1412 ti->error = "Hash device is too small"; 1413 r = -E2BIG; 1414 goto bad; 1415 } 1416 1417 /* 1418 * Using WQ_HIGHPRI improves throughput and completion latency by 1419 * reducing wait times when reading from a dm-verity device. 1420 * 1421 * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI 1422 * allows verify_wq to preempt softirq since verification in tasklet 1423 * will fall-back to using it for error handling (or if the bufio cache 1424 * doesn't have required hashes). 1425 */ 1426 v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1427 if (!v->verify_wq) { 1428 ti->error = "Cannot allocate workqueue"; 1429 r = -ENOMEM; 1430 goto bad; 1431 } 1432 1433 ti->per_io_data_size = sizeof(struct dm_verity_io) + 1434 v->ahash_reqsize + v->digest_size * 2; 1435 1436 r = verity_fec_ctr(v); 1437 if (r) 1438 goto bad; 1439 1440 ti->per_io_data_size = roundup(ti->per_io_data_size, 1441 __alignof__(struct dm_verity_io)); 1442 1443 verity_verify_sig_opts_cleanup(&verify_args); 1444 1445 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); 1446 1447 return 0; 1448 1449 bad: 1450 1451 verity_verify_sig_opts_cleanup(&verify_args); 1452 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); 1453 verity_dtr(ti); 1454 1455 return r; 1456 } 1457 1458 /* 1459 * Check whether a DM target is a verity target. 1460 */ 1461 bool dm_is_verity_target(struct dm_target *ti) 1462 { 1463 return ti->type->module == THIS_MODULE; 1464 } 1465 1466 /* 1467 * Get the verity mode (error behavior) of a verity target. 1468 * 1469 * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity 1470 * target. 1471 */ 1472 int dm_verity_get_mode(struct dm_target *ti) 1473 { 1474 struct dm_verity *v = ti->private; 1475 1476 if (!dm_is_verity_target(ti)) 1477 return -EINVAL; 1478 1479 return v->mode; 1480 } 1481 1482 /* 1483 * Get the root digest of a verity target. 1484 * 1485 * Returns a copy of the root digest, the caller is responsible for 1486 * freeing the memory of the digest. 1487 */ 1488 int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size) 1489 { 1490 struct dm_verity *v = ti->private; 1491 1492 if (!dm_is_verity_target(ti)) 1493 return -EINVAL; 1494 1495 *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL); 1496 if (*root_digest == NULL) 1497 return -ENOMEM; 1498 1499 *digest_size = v->digest_size; 1500 1501 return 0; 1502 } 1503 1504 static struct target_type verity_target = { 1505 .name = "verity", 1506 .features = DM_TARGET_IMMUTABLE, 1507 .version = {1, 9, 0}, 1508 .module = THIS_MODULE, 1509 .ctr = verity_ctr, 1510 .dtr = verity_dtr, 1511 .map = verity_map, 1512 .status = verity_status, 1513 .prepare_ioctl = verity_prepare_ioctl, 1514 .iterate_devices = verity_iterate_devices, 1515 .io_hints = verity_io_hints, 1516 }; 1517 module_dm(verity); 1518 1519 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); 1520 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>"); 1521 MODULE_AUTHOR("Will Drewry <wad@chromium.org>"); 1522 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); 1523 MODULE_LICENSE("GPL"); 1524