1/* 2 * Copyright (C) 2012 Red Hat, Inc. 3 * 4 * Author: Mikulas Patocka <mpatocka@redhat.com> 5 * 6 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors 7 * 8 * This file is released under the GPLv2. 9 * 10 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set 11 * default prefetch value. Data are read in "prefetch_cluster" chunks from the 12 * hash device. Setting this greatly improves performance when data and hash 13 * are on the same disk on different partitions on devices with poor random 14 * access behavior. 15 */ 16 17#include "dm-bufio.h" 18 19#include <linux/module.h> 20#include <linux/device-mapper.h> 21#include <linux/reboot.h> 22#include <crypto/hash.h> 23 24#define DM_MSG_PREFIX "verity" 25 26#define DM_VERITY_ENV_LENGTH 42 27#define DM_VERITY_ENV_VAR_NAME "VERITY_ERR_BLOCK_NR" 28 29#define DM_VERITY_IO_VEC_INLINE 16 30#define DM_VERITY_MEMPOOL_SIZE 4 31#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 32 33#define DM_VERITY_MAX_LEVELS 63 34#define DM_VERITY_MAX_CORRUPTED_ERRS 100 35 36static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; 37 38module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR); 39 40enum verity_mode { 41 DM_VERITY_MODE_EIO = 0, 42 DM_VERITY_MODE_LOGGING = 1, 43 DM_VERITY_MODE_RESTART = 2 44}; 45 46enum verity_block_type { 47 DM_VERITY_BLOCK_TYPE_DATA, 48 DM_VERITY_BLOCK_TYPE_METADATA 49}; 50 51struct dm_verity { 52 struct dm_dev *data_dev; 53 struct dm_dev *hash_dev; 54 struct dm_target *ti; 55 struct dm_bufio_client *bufio; 56 char *alg_name; 57 struct crypto_shash *tfm; 58 u8 *root_digest; /* digest of the root block */ 59 u8 *salt; /* salt: its size is salt_size */ 60 unsigned salt_size; 61 sector_t data_start; /* data offset in 512-byte sectors */ 62 sector_t hash_start; /* hash start in blocks */ 63 sector_t data_blocks; /* the number of data blocks */ 64 sector_t hash_blocks; /* the number of hash blocks */ 65 unsigned char data_dev_block_bits; /* log2(data blocksize) */ 66 unsigned char hash_dev_block_bits; /* log2(hash blocksize) */ 67 unsigned char hash_per_block_bits; /* log2(hashes in hash block) */ 68 unsigned char levels; /* the number of tree levels */ 69 unsigned char version; 70 unsigned digest_size; /* digest size for the current hash algorithm */ 71 unsigned shash_descsize;/* the size of temporary space for crypto */ 72 int hash_failed; /* set to 1 if hash of any block failed */ 73 enum verity_mode mode; /* mode for handling verification errors */ 74 unsigned corrupted_errs;/* Number of errors for corrupted blocks */ 75 76 mempool_t *vec_mempool; /* mempool of bio vector */ 77 78 struct workqueue_struct *verify_wq; 79 80 /* starting blocks for each tree level. 0 is the lowest level. */ 81 sector_t hash_level_block[DM_VERITY_MAX_LEVELS]; 82}; 83 84struct dm_verity_io { 85 struct dm_verity *v; 86 87 /* original values of bio->bi_end_io and bio->bi_private */ 88 bio_end_io_t *orig_bi_end_io; 89 void *orig_bi_private; 90 91 sector_t block; 92 unsigned n_blocks; 93 94 struct bvec_iter iter; 95 96 struct work_struct work; 97 98 /* 99 * Three variably-size fields follow this struct: 100 * 101 * u8 hash_desc[v->shash_descsize]; 102 * u8 real_digest[v->digest_size]; 103 * u8 want_digest[v->digest_size]; 104 * 105 * To access them use: io_hash_desc(), io_real_digest() and io_want_digest(). 106 */ 107}; 108 109struct dm_verity_prefetch_work { 110 struct work_struct work; 111 struct dm_verity *v; 112 sector_t block; 113 unsigned n_blocks; 114}; 115 116static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io) 117{ 118 return (struct shash_desc *)(io + 1); 119} 120 121static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io) 122{ 123 return (u8 *)(io + 1) + v->shash_descsize; 124} 125 126static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io) 127{ 128 return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; 129} 130 131/* 132 * Auxiliary structure appended to each dm-bufio buffer. If the value 133 * hash_verified is nonzero, hash of the block has been verified. 134 * 135 * The variable hash_verified is set to 0 when allocating the buffer, then 136 * it can be changed to 1 and it is never reset to 0 again. 137 * 138 * There is no lock around this value, a race condition can at worst cause 139 * that multiple processes verify the hash of the same buffer simultaneously 140 * and write 1 to hash_verified simultaneously. 141 * This condition is harmless, so we don't need locking. 142 */ 143struct buffer_aux { 144 int hash_verified; 145}; 146 147/* 148 * Initialize struct buffer_aux for a freshly created buffer. 149 */ 150static void dm_bufio_alloc_callback(struct dm_buffer *buf) 151{ 152 struct buffer_aux *aux = dm_bufio_get_aux_data(buf); 153 154 aux->hash_verified = 0; 155} 156 157/* 158 * Translate input sector number to the sector number on the target device. 159 */ 160static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) 161{ 162 return v->data_start + dm_target_offset(v->ti, bi_sector); 163} 164 165/* 166 * Return hash position of a specified block at a specified tree level 167 * (0 is the lowest level). 168 * The lowest "hash_per_block_bits"-bits of the result denote hash position 169 * inside a hash block. The remaining bits denote location of the hash block. 170 */ 171static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, 172 int level) 173{ 174 return block >> (level * v->hash_per_block_bits); 175} 176 177static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, 178 sector_t *hash_block, unsigned *offset) 179{ 180 sector_t position = verity_position_at_level(v, block, level); 181 unsigned idx; 182 183 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); 184 185 if (!offset) 186 return; 187 188 idx = position & ((1 << v->hash_per_block_bits) - 1); 189 if (!v->version) 190 *offset = idx * v->digest_size; 191 else 192 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); 193} 194 195/* 196 * Handle verification errors. 197 */ 198static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, 199 unsigned long long block) 200{ 201 char verity_env[DM_VERITY_ENV_LENGTH]; 202 char *envp[] = { verity_env, NULL }; 203 const char *type_str = ""; 204 struct mapped_device *md = dm_table_get_md(v->ti->table); 205 206 if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) 207 goto out; 208 209 ++v->corrupted_errs; 210 211 switch (type) { 212 case DM_VERITY_BLOCK_TYPE_DATA: 213 type_str = "data"; 214 break; 215 case DM_VERITY_BLOCK_TYPE_METADATA: 216 type_str = "metadata"; 217 break; 218 default: 219 BUG(); 220 } 221 222 DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, 223 type_str, block); 224 225 if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) 226 DMERR("%s: reached maximum errors", v->data_dev->name); 227 228 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", 229 DM_VERITY_ENV_VAR_NAME, type, block); 230 231 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); 232 233out: 234 if (v->mode == DM_VERITY_MODE_LOGGING) 235 return 0; 236 237 if (v->mode == DM_VERITY_MODE_RESTART) 238 kernel_restart("dm-verity device corrupted"); 239 240 return 1; 241} 242 243/* 244 * Verify hash of a metadata block pertaining to the specified data block 245 * ("block" argument) at a specified level ("level" argument). 246 * 247 * On successful return, io_want_digest(v, io) contains the hash value for 248 * a lower tree level or for the data block (if we're at the lowest leve). 249 * 250 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. 251 * If "skip_unverified" is false, unverified buffer is hashed and verified 252 * against current value of io_want_digest(v, io). 253 */ 254static int verity_verify_level(struct dm_verity_io *io, sector_t block, 255 int level, bool skip_unverified) 256{ 257 struct dm_verity *v = io->v; 258 struct dm_buffer *buf; 259 struct buffer_aux *aux; 260 u8 *data; 261 int r; 262 sector_t hash_block; 263 unsigned offset; 264 265 verity_hash_at_level(v, block, level, &hash_block, &offset); 266 267 data = dm_bufio_read(v->bufio, hash_block, &buf); 268 if (unlikely(IS_ERR(data))) 269 return PTR_ERR(data); 270 271 aux = dm_bufio_get_aux_data(buf); 272 273 if (!aux->hash_verified) { 274 struct shash_desc *desc; 275 u8 *result; 276 277 if (skip_unverified) { 278 r = 1; 279 goto release_ret_r; 280 } 281 282 desc = io_hash_desc(v, io); 283 desc->tfm = v->tfm; 284 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 285 r = crypto_shash_init(desc); 286 if (r < 0) { 287 DMERR("crypto_shash_init failed: %d", r); 288 goto release_ret_r; 289 } 290 291 if (likely(v->version >= 1)) { 292 r = crypto_shash_update(desc, v->salt, v->salt_size); 293 if (r < 0) { 294 DMERR("crypto_shash_update failed: %d", r); 295 goto release_ret_r; 296 } 297 } 298 299 r = crypto_shash_update(desc, data, 1 << v->hash_dev_block_bits); 300 if (r < 0) { 301 DMERR("crypto_shash_update failed: %d", r); 302 goto release_ret_r; 303 } 304 305 if (!v->version) { 306 r = crypto_shash_update(desc, v->salt, v->salt_size); 307 if (r < 0) { 308 DMERR("crypto_shash_update failed: %d", r); 309 goto release_ret_r; 310 } 311 } 312 313 result = io_real_digest(v, io); 314 r = crypto_shash_final(desc, result); 315 if (r < 0) { 316 DMERR("crypto_shash_final failed: %d", r); 317 goto release_ret_r; 318 } 319 if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) { 320 v->hash_failed = 1; 321 322 if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_METADATA, 323 hash_block)) { 324 r = -EIO; 325 goto release_ret_r; 326 } 327 } else 328 aux->hash_verified = 1; 329 } 330 331 data += offset; 332 333 memcpy(io_want_digest(v, io), data, v->digest_size); 334 335 dm_bufio_release(buf); 336 return 0; 337 338release_ret_r: 339 dm_bufio_release(buf); 340 341 return r; 342} 343 344/* 345 * Verify one "dm_verity_io" structure. 346 */ 347static int verity_verify_io(struct dm_verity_io *io) 348{ 349 struct dm_verity *v = io->v; 350 struct bio *bio = dm_bio_from_per_bio_data(io, 351 v->ti->per_bio_data_size); 352 unsigned b; 353 int i; 354 355 for (b = 0; b < io->n_blocks; b++) { 356 struct shash_desc *desc; 357 u8 *result; 358 int r; 359 unsigned todo; 360 361 if (likely(v->levels)) { 362 /* 363 * First, we try to get the requested hash for 364 * the current block. If the hash block itself is 365 * verified, zero is returned. If it isn't, this 366 * function returns 0 and we fall back to whole 367 * chain verification. 368 */ 369 int r = verity_verify_level(io, io->block + b, 0, true); 370 if (likely(!r)) 371 goto test_block_hash; 372 if (r < 0) 373 return r; 374 } 375 376 memcpy(io_want_digest(v, io), v->root_digest, v->digest_size); 377 378 for (i = v->levels - 1; i >= 0; i--) { 379 int r = verity_verify_level(io, io->block + b, i, false); 380 if (unlikely(r)) 381 return r; 382 } 383 384test_block_hash: 385 desc = io_hash_desc(v, io); 386 desc->tfm = v->tfm; 387 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 388 r = crypto_shash_init(desc); 389 if (r < 0) { 390 DMERR("crypto_shash_init failed: %d", r); 391 return r; 392 } 393 394 if (likely(v->version >= 1)) { 395 r = crypto_shash_update(desc, v->salt, v->salt_size); 396 if (r < 0) { 397 DMERR("crypto_shash_update failed: %d", r); 398 return r; 399 } 400 } 401 todo = 1 << v->data_dev_block_bits; 402 do { 403 u8 *page; 404 unsigned len; 405 struct bio_vec bv = bio_iter_iovec(bio, io->iter); 406 407 page = kmap_atomic(bv.bv_page); 408 len = bv.bv_len; 409 if (likely(len >= todo)) 410 len = todo; 411 r = crypto_shash_update(desc, page + bv.bv_offset, len); 412 kunmap_atomic(page); 413 414 if (r < 0) { 415 DMERR("crypto_shash_update failed: %d", r); 416 return r; 417 } 418 419 bio_advance_iter(bio, &io->iter, len); 420 todo -= len; 421 } while (todo); 422 423 if (!v->version) { 424 r = crypto_shash_update(desc, v->salt, v->salt_size); 425 if (r < 0) { 426 DMERR("crypto_shash_update failed: %d", r); 427 return r; 428 } 429 } 430 431 result = io_real_digest(v, io); 432 r = crypto_shash_final(desc, result); 433 if (r < 0) { 434 DMERR("crypto_shash_final failed: %d", r); 435 return r; 436 } 437 if (unlikely(memcmp(result, io_want_digest(v, io), v->digest_size))) { 438 v->hash_failed = 1; 439 440 if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, 441 io->block + b)) 442 return -EIO; 443 } 444 } 445 446 return 0; 447} 448 449/* 450 * End one "io" structure with a given error. 451 */ 452static void verity_finish_io(struct dm_verity_io *io, int error) 453{ 454 struct dm_verity *v = io->v; 455 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size); 456 457 bio->bi_end_io = io->orig_bi_end_io; 458 bio->bi_private = io->orig_bi_private; 459 460 bio_endio_nodec(bio, error); 461} 462 463static void verity_work(struct work_struct *w) 464{ 465 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); 466 467 verity_finish_io(io, verity_verify_io(io)); 468} 469 470static void verity_end_io(struct bio *bio, int error) 471{ 472 struct dm_verity_io *io = bio->bi_private; 473 474 if (error) { 475 verity_finish_io(io, error); 476 return; 477 } 478 479 INIT_WORK(&io->work, verity_work); 480 queue_work(io->v->verify_wq, &io->work); 481} 482 483/* 484 * Prefetch buffers for the specified io. 485 * The root buffer is not prefetched, it is assumed that it will be cached 486 * all the time. 487 */ 488static void verity_prefetch_io(struct work_struct *work) 489{ 490 struct dm_verity_prefetch_work *pw = 491 container_of(work, struct dm_verity_prefetch_work, work); 492 struct dm_verity *v = pw->v; 493 int i; 494 495 for (i = v->levels - 2; i >= 0; i--) { 496 sector_t hash_block_start; 497 sector_t hash_block_end; 498 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); 499 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); 500 if (!i) { 501 unsigned cluster = ACCESS_ONCE(dm_verity_prefetch_cluster); 502 503 cluster >>= v->data_dev_block_bits; 504 if (unlikely(!cluster)) 505 goto no_prefetch_cluster; 506 507 if (unlikely(cluster & (cluster - 1))) 508 cluster = 1 << __fls(cluster); 509 510 hash_block_start &= ~(sector_t)(cluster - 1); 511 hash_block_end |= cluster - 1; 512 if (unlikely(hash_block_end >= v->hash_blocks)) 513 hash_block_end = v->hash_blocks - 1; 514 } 515no_prefetch_cluster: 516 dm_bufio_prefetch(v->bufio, hash_block_start, 517 hash_block_end - hash_block_start + 1); 518 } 519 520 kfree(pw); 521} 522 523static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) 524{ 525 struct dm_verity_prefetch_work *pw; 526 527 pw = kmalloc(sizeof(struct dm_verity_prefetch_work), 528 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 529 530 if (!pw) 531 return; 532 533 INIT_WORK(&pw->work, verity_prefetch_io); 534 pw->v = v; 535 pw->block = io->block; 536 pw->n_blocks = io->n_blocks; 537 queue_work(v->verify_wq, &pw->work); 538} 539 540/* 541 * Bio map function. It allocates dm_verity_io structure and bio vector and 542 * fills them. Then it issues prefetches and the I/O. 543 */ 544static int verity_map(struct dm_target *ti, struct bio *bio) 545{ 546 struct dm_verity *v = ti->private; 547 struct dm_verity_io *io; 548 549 bio->bi_bdev = v->data_dev->bdev; 550 bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); 551 552 if (((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) & 553 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { 554 DMERR_LIMIT("unaligned io"); 555 return -EIO; 556 } 557 558 if (bio_end_sector(bio) >> 559 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { 560 DMERR_LIMIT("io out of range"); 561 return -EIO; 562 } 563 564 if (bio_data_dir(bio) == WRITE) 565 return -EIO; 566 567 io = dm_per_bio_data(bio, ti->per_bio_data_size); 568 io->v = v; 569 io->orig_bi_end_io = bio->bi_end_io; 570 io->orig_bi_private = bio->bi_private; 571 io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); 572 io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; 573 574 bio->bi_end_io = verity_end_io; 575 bio->bi_private = io; 576 io->iter = bio->bi_iter; 577 578 verity_submit_prefetch(v, io); 579 580 generic_make_request(bio); 581 582 return DM_MAPIO_SUBMITTED; 583} 584 585/* 586 * Status: V (valid) or C (corruption found) 587 */ 588static void verity_status(struct dm_target *ti, status_type_t type, 589 unsigned status_flags, char *result, unsigned maxlen) 590{ 591 struct dm_verity *v = ti->private; 592 unsigned sz = 0; 593 unsigned x; 594 595 switch (type) { 596 case STATUSTYPE_INFO: 597 DMEMIT("%c", v->hash_failed ? 'C' : 'V'); 598 break; 599 case STATUSTYPE_TABLE: 600 DMEMIT("%u %s %s %u %u %llu %llu %s ", 601 v->version, 602 v->data_dev->name, 603 v->hash_dev->name, 604 1 << v->data_dev_block_bits, 605 1 << v->hash_dev_block_bits, 606 (unsigned long long)v->data_blocks, 607 (unsigned long long)v->hash_start, 608 v->alg_name 609 ); 610 for (x = 0; x < v->digest_size; x++) 611 DMEMIT("%02x", v->root_digest[x]); 612 DMEMIT(" "); 613 if (!v->salt_size) 614 DMEMIT("-"); 615 else 616 for (x = 0; x < v->salt_size; x++) 617 DMEMIT("%02x", v->salt[x]); 618 break; 619 } 620} 621 622static int verity_ioctl(struct dm_target *ti, unsigned cmd, 623 unsigned long arg) 624{ 625 struct dm_verity *v = ti->private; 626 int r = 0; 627 628 if (v->data_start || 629 ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) 630 r = scsi_verify_blk_ioctl(NULL, cmd); 631 632 return r ? : __blkdev_driver_ioctl(v->data_dev->bdev, v->data_dev->mode, 633 cmd, arg); 634} 635 636static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 637 struct bio_vec *biovec, int max_size) 638{ 639 struct dm_verity *v = ti->private; 640 struct request_queue *q = bdev_get_queue(v->data_dev->bdev); 641 642 if (!q->merge_bvec_fn) 643 return max_size; 644 645 bvm->bi_bdev = v->data_dev->bdev; 646 bvm->bi_sector = verity_map_sector(v, bvm->bi_sector); 647 648 return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 649} 650 651static int verity_iterate_devices(struct dm_target *ti, 652 iterate_devices_callout_fn fn, void *data) 653{ 654 struct dm_verity *v = ti->private; 655 656 return fn(ti, v->data_dev, v->data_start, ti->len, data); 657} 658 659static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) 660{ 661 struct dm_verity *v = ti->private; 662 663 if (limits->logical_block_size < 1 << v->data_dev_block_bits) 664 limits->logical_block_size = 1 << v->data_dev_block_bits; 665 666 if (limits->physical_block_size < 1 << v->data_dev_block_bits) 667 limits->physical_block_size = 1 << v->data_dev_block_bits; 668 669 blk_limits_io_min(limits, limits->logical_block_size); 670} 671 672static void verity_dtr(struct dm_target *ti) 673{ 674 struct dm_verity *v = ti->private; 675 676 if (v->verify_wq) 677 destroy_workqueue(v->verify_wq); 678 679 if (v->vec_mempool) 680 mempool_destroy(v->vec_mempool); 681 682 if (v->bufio) 683 dm_bufio_client_destroy(v->bufio); 684 685 kfree(v->salt); 686 kfree(v->root_digest); 687 688 if (v->tfm) 689 crypto_free_shash(v->tfm); 690 691 kfree(v->alg_name); 692 693 if (v->hash_dev) 694 dm_put_device(ti, v->hash_dev); 695 696 if (v->data_dev) 697 dm_put_device(ti, v->data_dev); 698 699 kfree(v); 700} 701 702/* 703 * Target parameters: 704 * <version> The current format is version 1. 705 * Vsn 0 is compatible with original Chromium OS releases. 706 * <data device> 707 * <hash device> 708 * <data block size> 709 * <hash block size> 710 * <the number of data blocks> 711 * <hash start block> 712 * <algorithm> 713 * <digest> 714 * <salt> Hex string or "-" if no salt. 715 */ 716static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) 717{ 718 struct dm_verity *v; 719 unsigned num; 720 unsigned long long num_ll; 721 int r; 722 int i; 723 sector_t hash_position; 724 char dummy; 725 726 v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); 727 if (!v) { 728 ti->error = "Cannot allocate verity structure"; 729 return -ENOMEM; 730 } 731 ti->private = v; 732 v->ti = ti; 733 734 if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) { 735 ti->error = "Device must be readonly"; 736 r = -EINVAL; 737 goto bad; 738 } 739 740 if (argc < 10 || argc > 11) { 741 ti->error = "Invalid argument count: 10-11 arguments required"; 742 r = -EINVAL; 743 goto bad; 744 } 745 746 if (sscanf(argv[0], "%d%c", &num, &dummy) != 1 || 747 num < 0 || num > 1) { 748 ti->error = "Invalid version"; 749 r = -EINVAL; 750 goto bad; 751 } 752 v->version = num; 753 754 r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev); 755 if (r) { 756 ti->error = "Data device lookup failed"; 757 goto bad; 758 } 759 760 r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev); 761 if (r) { 762 ti->error = "Data device lookup failed"; 763 goto bad; 764 } 765 766 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || 767 !num || (num & (num - 1)) || 768 num < bdev_logical_block_size(v->data_dev->bdev) || 769 num > PAGE_SIZE) { 770 ti->error = "Invalid data device block size"; 771 r = -EINVAL; 772 goto bad; 773 } 774 v->data_dev_block_bits = __ffs(num); 775 776 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || 777 !num || (num & (num - 1)) || 778 num < bdev_logical_block_size(v->hash_dev->bdev) || 779 num > INT_MAX) { 780 ti->error = "Invalid hash device block size"; 781 r = -EINVAL; 782 goto bad; 783 } 784 v->hash_dev_block_bits = __ffs(num); 785 786 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || 787 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) 788 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { 789 ti->error = "Invalid data blocks"; 790 r = -EINVAL; 791 goto bad; 792 } 793 v->data_blocks = num_ll; 794 795 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { 796 ti->error = "Data device is too small"; 797 r = -EINVAL; 798 goto bad; 799 } 800 801 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || 802 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) 803 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { 804 ti->error = "Invalid hash start"; 805 r = -EINVAL; 806 goto bad; 807 } 808 v->hash_start = num_ll; 809 810 v->alg_name = kstrdup(argv[7], GFP_KERNEL); 811 if (!v->alg_name) { 812 ti->error = "Cannot allocate algorithm name"; 813 r = -ENOMEM; 814 goto bad; 815 } 816 817 v->tfm = crypto_alloc_shash(v->alg_name, 0, 0); 818 if (IS_ERR(v->tfm)) { 819 ti->error = "Cannot initialize hash function"; 820 r = PTR_ERR(v->tfm); 821 v->tfm = NULL; 822 goto bad; 823 } 824 v->digest_size = crypto_shash_digestsize(v->tfm); 825 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { 826 ti->error = "Digest size too big"; 827 r = -EINVAL; 828 goto bad; 829 } 830 v->shash_descsize = 831 sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm); 832 833 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); 834 if (!v->root_digest) { 835 ti->error = "Cannot allocate root digest"; 836 r = -ENOMEM; 837 goto bad; 838 } 839 if (strlen(argv[8]) != v->digest_size * 2 || 840 hex2bin(v->root_digest, argv[8], v->digest_size)) { 841 ti->error = "Invalid root digest"; 842 r = -EINVAL; 843 goto bad; 844 } 845 846 if (strcmp(argv[9], "-")) { 847 v->salt_size = strlen(argv[9]) / 2; 848 v->salt = kmalloc(v->salt_size, GFP_KERNEL); 849 if (!v->salt) { 850 ti->error = "Cannot allocate salt"; 851 r = -ENOMEM; 852 goto bad; 853 } 854 if (strlen(argv[9]) != v->salt_size * 2 || 855 hex2bin(v->salt, argv[9], v->salt_size)) { 856 ti->error = "Invalid salt"; 857 r = -EINVAL; 858 goto bad; 859 } 860 } 861 862 if (argc > 10) { 863 if (sscanf(argv[10], "%d%c", &num, &dummy) != 1 || 864 num < DM_VERITY_MODE_EIO || 865 num > DM_VERITY_MODE_RESTART) { 866 ti->error = "Invalid mode"; 867 r = -EINVAL; 868 goto bad; 869 } 870 v->mode = num; 871 } 872 873 v->hash_per_block_bits = 874 __fls((1 << v->hash_dev_block_bits) / v->digest_size); 875 876 v->levels = 0; 877 if (v->data_blocks) 878 while (v->hash_per_block_bits * v->levels < 64 && 879 (unsigned long long)(v->data_blocks - 1) >> 880 (v->hash_per_block_bits * v->levels)) 881 v->levels++; 882 883 if (v->levels > DM_VERITY_MAX_LEVELS) { 884 ti->error = "Too many tree levels"; 885 r = -E2BIG; 886 goto bad; 887 } 888 889 hash_position = v->hash_start; 890 for (i = v->levels - 1; i >= 0; i--) { 891 sector_t s; 892 v->hash_level_block[i] = hash_position; 893 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) 894 >> ((i + 1) * v->hash_per_block_bits); 895 if (hash_position + s < hash_position) { 896 ti->error = "Hash device offset overflow"; 897 r = -E2BIG; 898 goto bad; 899 } 900 hash_position += s; 901 } 902 v->hash_blocks = hash_position; 903 904 v->bufio = dm_bufio_client_create(v->hash_dev->bdev, 905 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), 906 dm_bufio_alloc_callback, NULL); 907 if (IS_ERR(v->bufio)) { 908 ti->error = "Cannot initialize dm-bufio"; 909 r = PTR_ERR(v->bufio); 910 v->bufio = NULL; 911 goto bad; 912 } 913 914 if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) { 915 ti->error = "Hash device is too small"; 916 r = -E2BIG; 917 goto bad; 918 } 919 920 ti->per_bio_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io)); 921 922 v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE, 923 BIO_MAX_PAGES * sizeof(struct bio_vec)); 924 if (!v->vec_mempool) { 925 ti->error = "Cannot allocate vector mempool"; 926 r = -ENOMEM; 927 goto bad; 928 } 929 930 /* WQ_UNBOUND greatly improves performance when running on ramdisk */ 931 v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus()); 932 if (!v->verify_wq) { 933 ti->error = "Cannot allocate workqueue"; 934 r = -ENOMEM; 935 goto bad; 936 } 937 938 return 0; 939 940bad: 941 verity_dtr(ti); 942 943 return r; 944} 945 946static struct target_type verity_target = { 947 .name = "verity", 948 .version = {1, 2, 0}, 949 .module = THIS_MODULE, 950 .ctr = verity_ctr, 951 .dtr = verity_dtr, 952 .map = verity_map, 953 .status = verity_status, 954 .ioctl = verity_ioctl, 955 .merge = verity_merge, 956 .iterate_devices = verity_iterate_devices, 957 .io_hints = verity_io_hints, 958}; 959 960static int __init dm_verity_init(void) 961{ 962 int r; 963 964 r = dm_register_target(&verity_target); 965 if (r < 0) 966 DMERR("register failed %d", r); 967 968 return r; 969} 970 971static void __exit dm_verity_exit(void) 972{ 973 dm_unregister_target(&verity_target); 974} 975 976module_init(dm_verity_init); 977module_exit(dm_verity_exit); 978 979MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); 980MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>"); 981MODULE_AUTHOR("Will Drewry <wad@chromium.org>"); 982MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); 983MODULE_LICENSE("GPL"); 984