1/* 2 * Copyright (C) 2001 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8#include "dm.h" 9 10#include <linux/module.h> 11#include <linux/vmalloc.h> 12#include <linux/blkdev.h> 13#include <linux/namei.h> 14#include <linux/ctype.h> 15#include <linux/string.h> 16#include <linux/slab.h> 17#include <linux/interrupt.h> 18#include <linux/mutex.h> 19#include <linux/delay.h> 20#include <linux/atomic.h> 21 22#define DM_MSG_PREFIX "table" 23 24#define MAX_DEPTH 16 25#define NODE_SIZE L1_CACHE_BYTES 26#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t)) 27#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1) 28 29struct dm_table { 30 struct mapped_device *md; 31 unsigned type; 32 33 /* btree table */ 34 unsigned int depth; 35 unsigned int counts[MAX_DEPTH]; /* in nodes */ 36 sector_t *index[MAX_DEPTH]; 37 38 unsigned int num_targets; 39 unsigned int num_allocated; 40 sector_t *highs; 41 struct dm_target *targets; 42 43 struct target_type *immutable_target_type; 44 unsigned integrity_supported:1; 45 unsigned singleton:1; 46 47 /* 48 * Indicates the rw permissions for the new logical 49 * device. This should be a combination of FMODE_READ 50 * and FMODE_WRITE. 51 */ 52 fmode_t mode; 53 54 /* a list of devices used by this table */ 55 struct list_head devices; 56 57 /* events get handed up using this callback */ 58 void (*event_fn)(void *); 59 void *event_context; 60 61 struct dm_md_mempools *mempools; 62 63 struct list_head target_callbacks; 64}; 65 66/* 67 * Similar to ceiling(log_size(n)) 68 */ 69static unsigned int int_log(unsigned int n, unsigned int base) 70{ 71 int result = 0; 72 73 while (n > 1) { 74 n = dm_div_up(n, base); 75 result++; 76 } 77 78 return result; 79} 80 81/* 82 * Calculate the index of the child node of the n'th node k'th key. 83 */ 84static inline unsigned int get_child(unsigned int n, unsigned int k) 85{ 86 return (n * CHILDREN_PER_NODE) + k; 87} 88 89/* 90 * Return the n'th node of level l from table t. 91 */ 92static inline sector_t *get_node(struct dm_table *t, 93 unsigned int l, unsigned int n) 94{ 95 return t->index[l] + (n * KEYS_PER_NODE); 96} 97 98/* 99 * Return the highest key that you could lookup from the n'th 100 * node on level l of the btree. 101 */ 102static sector_t high(struct dm_table *t, unsigned int l, unsigned int n) 103{ 104 for (; l < t->depth - 1; l++) 105 n = get_child(n, CHILDREN_PER_NODE - 1); 106 107 if (n >= t->counts[l]) 108 return (sector_t) - 1; 109 110 return get_node(t, l, n)[KEYS_PER_NODE - 1]; 111} 112 113/* 114 * Fills in a level of the btree based on the highs of the level 115 * below it. 116 */ 117static int setup_btree_index(unsigned int l, struct dm_table *t) 118{ 119 unsigned int n, k; 120 sector_t *node; 121 122 for (n = 0U; n < t->counts[l]; n++) { 123 node = get_node(t, l, n); 124 125 for (k = 0U; k < KEYS_PER_NODE; k++) 126 node[k] = high(t, l + 1, get_child(n, k)); 127 } 128 129 return 0; 130} 131 132void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size) 133{ 134 unsigned long size; 135 void *addr; 136 137 /* 138 * Check that we're not going to overflow. 139 */ 140 if (nmemb > (ULONG_MAX / elem_size)) 141 return NULL; 142 143 size = nmemb * elem_size; 144 addr = vzalloc(size); 145 146 return addr; 147} 148EXPORT_SYMBOL(dm_vcalloc); 149 150/* 151 * highs, and targets are managed as dynamic arrays during a 152 * table load. 153 */ 154static int alloc_targets(struct dm_table *t, unsigned int num) 155{ 156 sector_t *n_highs; 157 struct dm_target *n_targets; 158 159 /* 160 * Allocate both the target array and offset array at once. 161 * Append an empty entry to catch sectors beyond the end of 162 * the device. 163 */ 164 n_highs = (sector_t *) dm_vcalloc(num + 1, sizeof(struct dm_target) + 165 sizeof(sector_t)); 166 if (!n_highs) 167 return -ENOMEM; 168 169 n_targets = (struct dm_target *) (n_highs + num); 170 171 memset(n_highs, -1, sizeof(*n_highs) * num); 172 vfree(t->highs); 173 174 t->num_allocated = num; 175 t->highs = n_highs; 176 t->targets = n_targets; 177 178 return 0; 179} 180 181int dm_table_create(struct dm_table **result, fmode_t mode, 182 unsigned num_targets, struct mapped_device *md) 183{ 184 struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL); 185 186 if (!t) 187 return -ENOMEM; 188 189 INIT_LIST_HEAD(&t->devices); 190 INIT_LIST_HEAD(&t->target_callbacks); 191 192 if (!num_targets) 193 num_targets = KEYS_PER_NODE; 194 195 num_targets = dm_round_up(num_targets, KEYS_PER_NODE); 196 197 if (!num_targets) { 198 kfree(t); 199 return -ENOMEM; 200 } 201 202 if (alloc_targets(t, num_targets)) { 203 kfree(t); 204 return -ENOMEM; 205 } 206 207 t->mode = mode; 208 t->md = md; 209 *result = t; 210 return 0; 211} 212 213static void free_devices(struct list_head *devices, struct mapped_device *md) 214{ 215 struct list_head *tmp, *next; 216 217 list_for_each_safe(tmp, next, devices) { 218 struct dm_dev_internal *dd = 219 list_entry(tmp, struct dm_dev_internal, list); 220 DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s", 221 dm_device_name(md), dd->dm_dev->name); 222 dm_put_table_device(md, dd->dm_dev); 223 kfree(dd); 224 } 225} 226 227void dm_table_destroy(struct dm_table *t) 228{ 229 unsigned int i; 230 231 if (!t) 232 return; 233 234 /* free the indexes */ 235 if (t->depth >= 2) 236 vfree(t->index[t->depth - 2]); 237 238 /* free the targets */ 239 for (i = 0; i < t->num_targets; i++) { 240 struct dm_target *tgt = t->targets + i; 241 242 if (tgt->type->dtr) 243 tgt->type->dtr(tgt); 244 245 dm_put_target_type(tgt->type); 246 } 247 248 vfree(t->highs); 249 250 /* free the device list */ 251 free_devices(&t->devices, t->md); 252 253 dm_free_md_mempools(t->mempools); 254 255 kfree(t); 256} 257 258/* 259 * See if we've already got a device in the list. 260 */ 261static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev) 262{ 263 struct dm_dev_internal *dd; 264 265 list_for_each_entry (dd, l, list) 266 if (dd->dm_dev->bdev->bd_dev == dev) 267 return dd; 268 269 return NULL; 270} 271 272/* 273 * If possible, this checks an area of a destination device is invalid. 274 */ 275static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev, 276 sector_t start, sector_t len, void *data) 277{ 278 struct request_queue *q; 279 struct queue_limits *limits = data; 280 struct block_device *bdev = dev->bdev; 281 sector_t dev_size = 282 i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; 283 unsigned short logical_block_size_sectors = 284 limits->logical_block_size >> SECTOR_SHIFT; 285 char b[BDEVNAME_SIZE]; 286 287 /* 288 * Some devices exist without request functions, 289 * such as loop devices not yet bound to backing files. 290 * Forbid the use of such devices. 291 */ 292 q = bdev_get_queue(bdev); 293 if (!q || !q->make_request_fn) { 294 DMWARN("%s: %s is not yet initialised: " 295 "start=%llu, len=%llu, dev_size=%llu", 296 dm_device_name(ti->table->md), bdevname(bdev, b), 297 (unsigned long long)start, 298 (unsigned long long)len, 299 (unsigned long long)dev_size); 300 return 1; 301 } 302 303 if (!dev_size) 304 return 0; 305 306 if ((start >= dev_size) || (start + len > dev_size)) { 307 DMWARN("%s: %s too small for target: " 308 "start=%llu, len=%llu, dev_size=%llu", 309 dm_device_name(ti->table->md), bdevname(bdev, b), 310 (unsigned long long)start, 311 (unsigned long long)len, 312 (unsigned long long)dev_size); 313 return 1; 314 } 315 316 if (logical_block_size_sectors <= 1) 317 return 0; 318 319 if (start & (logical_block_size_sectors - 1)) { 320 DMWARN("%s: start=%llu not aligned to h/w " 321 "logical block size %u of %s", 322 dm_device_name(ti->table->md), 323 (unsigned long long)start, 324 limits->logical_block_size, bdevname(bdev, b)); 325 return 1; 326 } 327 328 if (len & (logical_block_size_sectors - 1)) { 329 DMWARN("%s: len=%llu not aligned to h/w " 330 "logical block size %u of %s", 331 dm_device_name(ti->table->md), 332 (unsigned long long)len, 333 limits->logical_block_size, bdevname(bdev, b)); 334 return 1; 335 } 336 337 return 0; 338} 339 340/* 341 * This upgrades the mode on an already open dm_dev, being 342 * careful to leave things as they were if we fail to reopen the 343 * device and not to touch the existing bdev field in case 344 * it is accessed concurrently inside dm_table_any_congested(). 345 */ 346static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode, 347 struct mapped_device *md) 348{ 349 int r; 350 struct dm_dev *old_dev, *new_dev; 351 352 old_dev = dd->dm_dev; 353 354 r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev, 355 dd->dm_dev->mode | new_mode, &new_dev); 356 if (r) 357 return r; 358 359 dd->dm_dev = new_dev; 360 dm_put_table_device(md, old_dev); 361 362 return 0; 363} 364 365/* 366 * Add a device to the list, or just increment the usage count if 367 * it's already present. 368 */ 369int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode, 370 struct dm_dev **result) 371{ 372 int r; 373 dev_t uninitialized_var(dev); 374 struct dm_dev_internal *dd; 375 unsigned int major, minor; 376 struct dm_table *t = ti->table; 377 char dummy; 378 379 BUG_ON(!t); 380 381 if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) { 382 /* Extract the major/minor numbers */ 383 dev = MKDEV(major, minor); 384 if (MAJOR(dev) != major || MINOR(dev) != minor) 385 return -EOVERFLOW; 386 } else { 387 /* convert the path to a device */ 388 struct block_device *bdev = lookup_bdev(path); 389 390 if (IS_ERR(bdev)) 391 return PTR_ERR(bdev); 392 dev = bdev->bd_dev; 393 bdput(bdev); 394 } 395 396 dd = find_device(&t->devices, dev); 397 if (!dd) { 398 dd = kmalloc(sizeof(*dd), GFP_KERNEL); 399 if (!dd) 400 return -ENOMEM; 401 402 if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) { 403 kfree(dd); 404 return r; 405 } 406 407 atomic_set(&dd->count, 0); 408 list_add(&dd->list, &t->devices); 409 410 } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) { 411 r = upgrade_mode(dd, mode, t->md); 412 if (r) 413 return r; 414 } 415 atomic_inc(&dd->count); 416 417 *result = dd->dm_dev; 418 return 0; 419} 420EXPORT_SYMBOL(dm_get_device); 421 422static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev, 423 sector_t start, sector_t len, void *data) 424{ 425 struct queue_limits *limits = data; 426 struct block_device *bdev = dev->bdev; 427 struct request_queue *q = bdev_get_queue(bdev); 428 char b[BDEVNAME_SIZE]; 429 430 if (unlikely(!q)) { 431 DMWARN("%s: Cannot set limits for nonexistent device %s", 432 dm_device_name(ti->table->md), bdevname(bdev, b)); 433 return 0; 434 } 435 436 if (bdev_stack_limits(limits, bdev, start) < 0) 437 DMWARN("%s: adding target device %s caused an alignment inconsistency: " 438 "physical_block_size=%u, logical_block_size=%u, " 439 "alignment_offset=%u, start=%llu", 440 dm_device_name(ti->table->md), bdevname(bdev, b), 441 q->limits.physical_block_size, 442 q->limits.logical_block_size, 443 q->limits.alignment_offset, 444 (unsigned long long) start << SECTOR_SHIFT); 445 446 /* 447 * Check if merge fn is supported. 448 * If not we'll force DM to use PAGE_SIZE or 449 * smaller I/O, just to be safe. 450 */ 451 if (dm_queue_merge_is_compulsory(q) && !ti->type->merge) 452 blk_limits_max_hw_sectors(limits, 453 (unsigned int) (PAGE_SIZE >> 9)); 454 return 0; 455} 456 457/* 458 * Decrement a device's use count and remove it if necessary. 459 */ 460void dm_put_device(struct dm_target *ti, struct dm_dev *d) 461{ 462 int found = 0; 463 struct list_head *devices = &ti->table->devices; 464 struct dm_dev_internal *dd; 465 466 list_for_each_entry(dd, devices, list) { 467 if (dd->dm_dev == d) { 468 found = 1; 469 break; 470 } 471 } 472 if (!found) { 473 DMWARN("%s: device %s not in table devices list", 474 dm_device_name(ti->table->md), d->name); 475 return; 476 } 477 if (atomic_dec_and_test(&dd->count)) { 478 dm_put_table_device(ti->table->md, d); 479 list_del(&dd->list); 480 kfree(dd); 481 } 482} 483EXPORT_SYMBOL(dm_put_device); 484 485/* 486 * Checks to see if the target joins onto the end of the table. 487 */ 488static int adjoin(struct dm_table *table, struct dm_target *ti) 489{ 490 struct dm_target *prev; 491 492 if (!table->num_targets) 493 return !ti->begin; 494 495 prev = &table->targets[table->num_targets - 1]; 496 return (ti->begin == (prev->begin + prev->len)); 497} 498 499/* 500 * Used to dynamically allocate the arg array. 501 * 502 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must 503 * process messages even if some device is suspended. These messages have a 504 * small fixed number of arguments. 505 * 506 * On the other hand, dm-switch needs to process bulk data using messages and 507 * excessive use of GFP_NOIO could cause trouble. 508 */ 509static char **realloc_argv(unsigned *array_size, char **old_argv) 510{ 511 char **argv; 512 unsigned new_size; 513 gfp_t gfp; 514 515 if (*array_size) { 516 new_size = *array_size * 2; 517 gfp = GFP_KERNEL; 518 } else { 519 new_size = 8; 520 gfp = GFP_NOIO; 521 } 522 argv = kmalloc(new_size * sizeof(*argv), gfp); 523 if (argv) { 524 memcpy(argv, old_argv, *array_size * sizeof(*argv)); 525 *array_size = new_size; 526 } 527 528 kfree(old_argv); 529 return argv; 530} 531 532/* 533 * Destructively splits up the argument list to pass to ctr. 534 */ 535int dm_split_args(int *argc, char ***argvp, char *input) 536{ 537 char *start, *end = input, *out, **argv = NULL; 538 unsigned array_size = 0; 539 540 *argc = 0; 541 542 if (!input) { 543 *argvp = NULL; 544 return 0; 545 } 546 547 argv = realloc_argv(&array_size, argv); 548 if (!argv) 549 return -ENOMEM; 550 551 while (1) { 552 /* Skip whitespace */ 553 start = skip_spaces(end); 554 555 if (!*start) 556 break; /* success, we hit the end */ 557 558 /* 'out' is used to remove any back-quotes */ 559 end = out = start; 560 while (*end) { 561 /* Everything apart from '\0' can be quoted */ 562 if (*end == '\\' && *(end + 1)) { 563 *out++ = *(end + 1); 564 end += 2; 565 continue; 566 } 567 568 if (isspace(*end)) 569 break; /* end of token */ 570 571 *out++ = *end++; 572 } 573 574 /* have we already filled the array ? */ 575 if ((*argc + 1) > array_size) { 576 argv = realloc_argv(&array_size, argv); 577 if (!argv) 578 return -ENOMEM; 579 } 580 581 /* we know this is whitespace */ 582 if (*end) 583 end++; 584 585 /* terminate the string and put it in the array */ 586 *out = '\0'; 587 argv[*argc] = start; 588 (*argc)++; 589 } 590 591 *argvp = argv; 592 return 0; 593} 594 595/* 596 * Impose necessary and sufficient conditions on a devices's table such 597 * that any incoming bio which respects its logical_block_size can be 598 * processed successfully. If it falls across the boundary between 599 * two or more targets, the size of each piece it gets split into must 600 * be compatible with the logical_block_size of the target processing it. 601 */ 602static int validate_hardware_logical_block_alignment(struct dm_table *table, 603 struct queue_limits *limits) 604{ 605 /* 606 * This function uses arithmetic modulo the logical_block_size 607 * (in units of 512-byte sectors). 608 */ 609 unsigned short device_logical_block_size_sects = 610 limits->logical_block_size >> SECTOR_SHIFT; 611 612 /* 613 * Offset of the start of the next table entry, mod logical_block_size. 614 */ 615 unsigned short next_target_start = 0; 616 617 /* 618 * Given an aligned bio that extends beyond the end of a 619 * target, how many sectors must the next target handle? 620 */ 621 unsigned short remaining = 0; 622 623 struct dm_target *uninitialized_var(ti); 624 struct queue_limits ti_limits; 625 unsigned i = 0; 626 627 /* 628 * Check each entry in the table in turn. 629 */ 630 while (i < dm_table_get_num_targets(table)) { 631 ti = dm_table_get_target(table, i++); 632 633 blk_set_stacking_limits(&ti_limits); 634 635 /* combine all target devices' limits */ 636 if (ti->type->iterate_devices) 637 ti->type->iterate_devices(ti, dm_set_device_limits, 638 &ti_limits); 639 640 /* 641 * If the remaining sectors fall entirely within this 642 * table entry are they compatible with its logical_block_size? 643 */ 644 if (remaining < ti->len && 645 remaining & ((ti_limits.logical_block_size >> 646 SECTOR_SHIFT) - 1)) 647 break; /* Error */ 648 649 next_target_start = 650 (unsigned short) ((next_target_start + ti->len) & 651 (device_logical_block_size_sects - 1)); 652 remaining = next_target_start ? 653 device_logical_block_size_sects - next_target_start : 0; 654 } 655 656 if (remaining) { 657 DMWARN("%s: table line %u (start sect %llu len %llu) " 658 "not aligned to h/w logical block size %u", 659 dm_device_name(table->md), i, 660 (unsigned long long) ti->begin, 661 (unsigned long long) ti->len, 662 limits->logical_block_size); 663 return -EINVAL; 664 } 665 666 return 0; 667} 668 669int dm_table_add_target(struct dm_table *t, const char *type, 670 sector_t start, sector_t len, char *params) 671{ 672 int r = -EINVAL, argc; 673 char **argv; 674 struct dm_target *tgt; 675 676 if (t->singleton) { 677 DMERR("%s: target type %s must appear alone in table", 678 dm_device_name(t->md), t->targets->type->name); 679 return -EINVAL; 680 } 681 682 BUG_ON(t->num_targets >= t->num_allocated); 683 684 tgt = t->targets + t->num_targets; 685 memset(tgt, 0, sizeof(*tgt)); 686 687 if (!len) { 688 DMERR("%s: zero-length target", dm_device_name(t->md)); 689 return -EINVAL; 690 } 691 692 tgt->type = dm_get_target_type(type); 693 if (!tgt->type) { 694 DMERR("%s: %s: unknown target type", dm_device_name(t->md), 695 type); 696 return -EINVAL; 697 } 698 699 if (dm_target_needs_singleton(tgt->type)) { 700 if (t->num_targets) { 701 DMERR("%s: target type %s must appear alone in table", 702 dm_device_name(t->md), type); 703 return -EINVAL; 704 } 705 t->singleton = 1; 706 } 707 708 if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) { 709 DMERR("%s: target type %s may not be included in read-only tables", 710 dm_device_name(t->md), type); 711 return -EINVAL; 712 } 713 714 if (t->immutable_target_type) { 715 if (t->immutable_target_type != tgt->type) { 716 DMERR("%s: immutable target type %s cannot be mixed with other target types", 717 dm_device_name(t->md), t->immutable_target_type->name); 718 return -EINVAL; 719 } 720 } else if (dm_target_is_immutable(tgt->type)) { 721 if (t->num_targets) { 722 DMERR("%s: immutable target type %s cannot be mixed with other target types", 723 dm_device_name(t->md), tgt->type->name); 724 return -EINVAL; 725 } 726 t->immutable_target_type = tgt->type; 727 } 728 729 tgt->table = t; 730 tgt->begin = start; 731 tgt->len = len; 732 tgt->error = "Unknown error"; 733 734 /* 735 * Does this target adjoin the previous one ? 736 */ 737 if (!adjoin(t, tgt)) { 738 tgt->error = "Gap in table"; 739 r = -EINVAL; 740 goto bad; 741 } 742 743 r = dm_split_args(&argc, &argv, params); 744 if (r) { 745 tgt->error = "couldn't split parameters (insufficient memory)"; 746 goto bad; 747 } 748 749 r = tgt->type->ctr(tgt, argc, argv); 750 kfree(argv); 751 if (r) 752 goto bad; 753 754 t->highs[t->num_targets++] = tgt->begin + tgt->len - 1; 755 756 if (!tgt->num_discard_bios && tgt->discards_supported) 757 DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.", 758 dm_device_name(t->md), type); 759 760 return 0; 761 762 bad: 763 DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error); 764 dm_put_target_type(tgt->type); 765 return r; 766} 767 768/* 769 * Target argument parsing helpers. 770 */ 771static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, 772 unsigned *value, char **error, unsigned grouped) 773{ 774 const char *arg_str = dm_shift_arg(arg_set); 775 char dummy; 776 777 if (!arg_str || 778 (sscanf(arg_str, "%u%c", value, &dummy) != 1) || 779 (*value < arg->min) || 780 (*value > arg->max) || 781 (grouped && arg_set->argc < *value)) { 782 *error = arg->error; 783 return -EINVAL; 784 } 785 786 return 0; 787} 788 789int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, 790 unsigned *value, char **error) 791{ 792 return validate_next_arg(arg, arg_set, value, error, 0); 793} 794EXPORT_SYMBOL(dm_read_arg); 795 796int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set, 797 unsigned *value, char **error) 798{ 799 return validate_next_arg(arg, arg_set, value, error, 1); 800} 801EXPORT_SYMBOL(dm_read_arg_group); 802 803const char *dm_shift_arg(struct dm_arg_set *as) 804{ 805 char *r; 806 807 if (as->argc) { 808 as->argc--; 809 r = *as->argv; 810 as->argv++; 811 return r; 812 } 813 814 return NULL; 815} 816EXPORT_SYMBOL(dm_shift_arg); 817 818void dm_consume_args(struct dm_arg_set *as, unsigned num_args) 819{ 820 BUG_ON(as->argc < num_args); 821 as->argc -= num_args; 822 as->argv += num_args; 823} 824EXPORT_SYMBOL(dm_consume_args); 825 826static int dm_table_set_type(struct dm_table *t) 827{ 828 unsigned i; 829 unsigned bio_based = 0, request_based = 0, hybrid = 0; 830 struct dm_target *tgt; 831 struct dm_dev_internal *dd; 832 struct list_head *devices; 833 unsigned live_md_type; 834 835 for (i = 0; i < t->num_targets; i++) { 836 tgt = t->targets + i; 837 if (dm_target_hybrid(tgt)) 838 hybrid = 1; 839 else if (dm_target_request_based(tgt)) 840 request_based = 1; 841 else 842 bio_based = 1; 843 844 if (bio_based && request_based) { 845 DMWARN("Inconsistent table: different target types" 846 " can't be mixed up"); 847 return -EINVAL; 848 } 849 } 850 851 if (hybrid && !bio_based && !request_based) { 852 /* 853 * The targets can work either way. 854 * Determine the type from the live device. 855 * Default to bio-based if device is new. 856 */ 857 live_md_type = dm_get_md_type(t->md); 858 if (live_md_type == DM_TYPE_REQUEST_BASED) 859 request_based = 1; 860 else 861 bio_based = 1; 862 } 863 864 if (bio_based) { 865 /* We must use this table as bio-based */ 866 t->type = DM_TYPE_BIO_BASED; 867 return 0; 868 } 869 870 BUG_ON(!request_based); /* No targets in this table */ 871 872 /* Non-request-stackable devices can't be used for request-based dm */ 873 devices = dm_table_get_devices(t); 874 list_for_each_entry(dd, devices, list) { 875 if (!blk_queue_stackable(bdev_get_queue(dd->dm_dev->bdev))) { 876 DMWARN("table load rejected: including" 877 " non-request-stackable devices"); 878 return -EINVAL; 879 } 880 } 881 882 /* 883 * Request-based dm supports only tables that have a single target now. 884 * To support multiple targets, request splitting support is needed, 885 * and that needs lots of changes in the block-layer. 886 * (e.g. request completion process for partial completion.) 887 */ 888 if (t->num_targets > 1) { 889 DMWARN("Request-based dm doesn't support multiple targets yet"); 890 return -EINVAL; 891 } 892 893 t->type = DM_TYPE_REQUEST_BASED; 894 895 return 0; 896} 897 898unsigned dm_table_get_type(struct dm_table *t) 899{ 900 return t->type; 901} 902 903struct target_type *dm_table_get_immutable_target_type(struct dm_table *t) 904{ 905 return t->immutable_target_type; 906} 907 908bool dm_table_request_based(struct dm_table *t) 909{ 910 return dm_table_get_type(t) == DM_TYPE_REQUEST_BASED; 911} 912 913static int dm_table_alloc_md_mempools(struct dm_table *t) 914{ 915 unsigned type = dm_table_get_type(t); 916 unsigned per_bio_data_size = 0; 917 struct dm_target *tgt; 918 unsigned i; 919 920 if (unlikely(type == DM_TYPE_NONE)) { 921 DMWARN("no table type is set, can't allocate mempools"); 922 return -EINVAL; 923 } 924 925 if (type == DM_TYPE_BIO_BASED) 926 for (i = 0; i < t->num_targets; i++) { 927 tgt = t->targets + i; 928 per_bio_data_size = max(per_bio_data_size, tgt->per_bio_data_size); 929 } 930 931 t->mempools = dm_alloc_md_mempools(type, t->integrity_supported, per_bio_data_size); 932 if (!t->mempools) 933 return -ENOMEM; 934 935 return 0; 936} 937 938void dm_table_free_md_mempools(struct dm_table *t) 939{ 940 dm_free_md_mempools(t->mempools); 941 t->mempools = NULL; 942} 943 944struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t) 945{ 946 return t->mempools; 947} 948 949static int setup_indexes(struct dm_table *t) 950{ 951 int i; 952 unsigned int total = 0; 953 sector_t *indexes; 954 955 /* allocate the space for *all* the indexes */ 956 for (i = t->depth - 2; i >= 0; i--) { 957 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE); 958 total += t->counts[i]; 959 } 960 961 indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE); 962 if (!indexes) 963 return -ENOMEM; 964 965 /* set up internal nodes, bottom-up */ 966 for (i = t->depth - 2; i >= 0; i--) { 967 t->index[i] = indexes; 968 indexes += (KEYS_PER_NODE * t->counts[i]); 969 setup_btree_index(i, t); 970 } 971 972 return 0; 973} 974 975/* 976 * Builds the btree to index the map. 977 */ 978static int dm_table_build_index(struct dm_table *t) 979{ 980 int r = 0; 981 unsigned int leaf_nodes; 982 983 /* how many indexes will the btree have ? */ 984 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); 985 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); 986 987 /* leaf layer has already been set up */ 988 t->counts[t->depth - 1] = leaf_nodes; 989 t->index[t->depth - 1] = t->highs; 990 991 if (t->depth >= 2) 992 r = setup_indexes(t); 993 994 return r; 995} 996 997/* 998 * Get a disk whose integrity profile reflects the table's profile. 999 * If %match_all is true, all devices' profiles must match. 1000 * If %match_all is false, all devices must at least have an 1001 * allocated integrity profile; but uninitialized is ok. 1002 * Returns NULL if integrity support was inconsistent or unavailable. 1003 */ 1004static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t, 1005 bool match_all) 1006{ 1007 struct list_head *devices = dm_table_get_devices(t); 1008 struct dm_dev_internal *dd = NULL; 1009 struct gendisk *prev_disk = NULL, *template_disk = NULL; 1010 1011 list_for_each_entry(dd, devices, list) { 1012 template_disk = dd->dm_dev->bdev->bd_disk; 1013 if (!blk_get_integrity(template_disk)) 1014 goto no_integrity; 1015 if (!match_all && !blk_integrity_is_initialized(template_disk)) 1016 continue; /* skip uninitialized profiles */ 1017 else if (prev_disk && 1018 blk_integrity_compare(prev_disk, template_disk) < 0) 1019 goto no_integrity; 1020 prev_disk = template_disk; 1021 } 1022 1023 return template_disk; 1024 1025no_integrity: 1026 if (prev_disk) 1027 DMWARN("%s: integrity not set: %s and %s profile mismatch", 1028 dm_device_name(t->md), 1029 prev_disk->disk_name, 1030 template_disk->disk_name); 1031 return NULL; 1032} 1033 1034/* 1035 * Register the mapped device for blk_integrity support if 1036 * the underlying devices have an integrity profile. But all devices 1037 * may not have matching profiles (checking all devices isn't reliable 1038 * during table load because this table may use other DM device(s) which 1039 * must be resumed before they will have an initialized integity profile). 1040 * Stacked DM devices force a 2 stage integrity profile validation: 1041 * 1 - during load, validate all initialized integrity profiles match 1042 * 2 - during resume, validate all integrity profiles match 1043 */ 1044static int dm_table_prealloc_integrity(struct dm_table *t, struct mapped_device *md) 1045{ 1046 struct gendisk *template_disk = NULL; 1047 1048 template_disk = dm_table_get_integrity_disk(t, false); 1049 if (!template_disk) 1050 return 0; 1051 1052 if (!blk_integrity_is_initialized(dm_disk(md))) { 1053 t->integrity_supported = 1; 1054 return blk_integrity_register(dm_disk(md), NULL); 1055 } 1056 1057 /* 1058 * If DM device already has an initalized integrity 1059 * profile the new profile should not conflict. 1060 */ 1061 if (blk_integrity_is_initialized(template_disk) && 1062 blk_integrity_compare(dm_disk(md), template_disk) < 0) { 1063 DMWARN("%s: conflict with existing integrity profile: " 1064 "%s profile mismatch", 1065 dm_device_name(t->md), 1066 template_disk->disk_name); 1067 return 1; 1068 } 1069 1070 /* Preserve existing initialized integrity profile */ 1071 t->integrity_supported = 1; 1072 return 0; 1073} 1074 1075/* 1076 * Prepares the table for use by building the indices, 1077 * setting the type, and allocating mempools. 1078 */ 1079int dm_table_complete(struct dm_table *t) 1080{ 1081 int r; 1082 1083 r = dm_table_set_type(t); 1084 if (r) { 1085 DMERR("unable to set table type"); 1086 return r; 1087 } 1088 1089 r = dm_table_build_index(t); 1090 if (r) { 1091 DMERR("unable to build btrees"); 1092 return r; 1093 } 1094 1095 r = dm_table_prealloc_integrity(t, t->md); 1096 if (r) { 1097 DMERR("could not register integrity profile."); 1098 return r; 1099 } 1100 1101 r = dm_table_alloc_md_mempools(t); 1102 if (r) 1103 DMERR("unable to allocate mempools"); 1104 1105 return r; 1106} 1107 1108static DEFINE_MUTEX(_event_lock); 1109void dm_table_event_callback(struct dm_table *t, 1110 void (*fn)(void *), void *context) 1111{ 1112 mutex_lock(&_event_lock); 1113 t->event_fn = fn; 1114 t->event_context = context; 1115 mutex_unlock(&_event_lock); 1116} 1117 1118void dm_table_event(struct dm_table *t) 1119{ 1120 /* 1121 * You can no longer call dm_table_event() from interrupt 1122 * context, use a bottom half instead. 1123 */ 1124 BUG_ON(in_interrupt()); 1125 1126 mutex_lock(&_event_lock); 1127 if (t->event_fn) 1128 t->event_fn(t->event_context); 1129 mutex_unlock(&_event_lock); 1130} 1131EXPORT_SYMBOL(dm_table_event); 1132 1133sector_t dm_table_get_size(struct dm_table *t) 1134{ 1135 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0; 1136} 1137EXPORT_SYMBOL(dm_table_get_size); 1138 1139struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index) 1140{ 1141 if (index >= t->num_targets) 1142 return NULL; 1143 1144 return t->targets + index; 1145} 1146 1147/* 1148 * Search the btree for the correct target. 1149 * 1150 * Caller should check returned pointer with dm_target_is_valid() 1151 * to trap I/O beyond end of device. 1152 */ 1153struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector) 1154{ 1155 unsigned int l, n = 0, k = 0; 1156 sector_t *node; 1157 1158 for (l = 0; l < t->depth; l++) { 1159 n = get_child(n, k); 1160 node = get_node(t, l, n); 1161 1162 for (k = 0; k < KEYS_PER_NODE; k++) 1163 if (node[k] >= sector) 1164 break; 1165 } 1166 1167 return &t->targets[(KEYS_PER_NODE * n) + k]; 1168} 1169 1170static int count_device(struct dm_target *ti, struct dm_dev *dev, 1171 sector_t start, sector_t len, void *data) 1172{ 1173 unsigned *num_devices = data; 1174 1175 (*num_devices)++; 1176 1177 return 0; 1178} 1179 1180/* 1181 * Check whether a table has no data devices attached using each 1182 * target's iterate_devices method. 1183 * Returns false if the result is unknown because a target doesn't 1184 * support iterate_devices. 1185 */ 1186bool dm_table_has_no_data_devices(struct dm_table *table) 1187{ 1188 struct dm_target *uninitialized_var(ti); 1189 unsigned i = 0, num_devices = 0; 1190 1191 while (i < dm_table_get_num_targets(table)) { 1192 ti = dm_table_get_target(table, i++); 1193 1194 if (!ti->type->iterate_devices) 1195 return false; 1196 1197 ti->type->iterate_devices(ti, count_device, &num_devices); 1198 if (num_devices) 1199 return false; 1200 } 1201 1202 return true; 1203} 1204 1205/* 1206 * Establish the new table's queue_limits and validate them. 1207 */ 1208int dm_calculate_queue_limits(struct dm_table *table, 1209 struct queue_limits *limits) 1210{ 1211 struct dm_target *uninitialized_var(ti); 1212 struct queue_limits ti_limits; 1213 unsigned i = 0; 1214 1215 blk_set_stacking_limits(limits); 1216 1217 while (i < dm_table_get_num_targets(table)) { 1218 blk_set_stacking_limits(&ti_limits); 1219 1220 ti = dm_table_get_target(table, i++); 1221 1222 if (!ti->type->iterate_devices) 1223 goto combine_limits; 1224 1225 /* 1226 * Combine queue limits of all the devices this target uses. 1227 */ 1228 ti->type->iterate_devices(ti, dm_set_device_limits, 1229 &ti_limits); 1230 1231 /* Set I/O hints portion of queue limits */ 1232 if (ti->type->io_hints) 1233 ti->type->io_hints(ti, &ti_limits); 1234 1235 /* 1236 * Check each device area is consistent with the target's 1237 * overall queue limits. 1238 */ 1239 if (ti->type->iterate_devices(ti, device_area_is_invalid, 1240 &ti_limits)) 1241 return -EINVAL; 1242 1243combine_limits: 1244 /* 1245 * Merge this target's queue limits into the overall limits 1246 * for the table. 1247 */ 1248 if (blk_stack_limits(limits, &ti_limits, 0) < 0) 1249 DMWARN("%s: adding target device " 1250 "(start sect %llu len %llu) " 1251 "caused an alignment inconsistency", 1252 dm_device_name(table->md), 1253 (unsigned long long) ti->begin, 1254 (unsigned long long) ti->len); 1255 } 1256 1257 return validate_hardware_logical_block_alignment(table, limits); 1258} 1259 1260/* 1261 * Set the integrity profile for this device if all devices used have 1262 * matching profiles. We're quite deep in the resume path but still 1263 * don't know if all devices (particularly DM devices this device 1264 * may be stacked on) have matching profiles. Even if the profiles 1265 * don't match we have no way to fail (to resume) at this point. 1266 */ 1267static void dm_table_set_integrity(struct dm_table *t) 1268{ 1269 struct gendisk *template_disk = NULL; 1270 1271 if (!blk_get_integrity(dm_disk(t->md))) 1272 return; 1273 1274 template_disk = dm_table_get_integrity_disk(t, true); 1275 if (template_disk) 1276 blk_integrity_register(dm_disk(t->md), 1277 blk_get_integrity(template_disk)); 1278 else if (blk_integrity_is_initialized(dm_disk(t->md))) 1279 DMWARN("%s: device no longer has a valid integrity profile", 1280 dm_device_name(t->md)); 1281 else 1282 DMWARN("%s: unable to establish an integrity profile", 1283 dm_device_name(t->md)); 1284} 1285 1286static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev, 1287 sector_t start, sector_t len, void *data) 1288{ 1289 unsigned flush = (*(unsigned *)data); 1290 struct request_queue *q = bdev_get_queue(dev->bdev); 1291 1292 return q && (q->flush_flags & flush); 1293} 1294 1295static bool dm_table_supports_flush(struct dm_table *t, unsigned flush) 1296{ 1297 struct dm_target *ti; 1298 unsigned i = 0; 1299 1300 /* 1301 * Require at least one underlying device to support flushes. 1302 * t->devices includes internal dm devices such as mirror logs 1303 * so we need to use iterate_devices here, which targets 1304 * supporting flushes must provide. 1305 */ 1306 while (i < dm_table_get_num_targets(t)) { 1307 ti = dm_table_get_target(t, i++); 1308 1309 if (!ti->num_flush_bios) 1310 continue; 1311 1312 if (ti->flush_supported) 1313 return 1; 1314 1315 if (ti->type->iterate_devices && 1316 ti->type->iterate_devices(ti, device_flush_capable, &flush)) 1317 return 1; 1318 } 1319 1320 return 0; 1321} 1322 1323static bool dm_table_discard_zeroes_data(struct dm_table *t) 1324{ 1325 struct dm_target *ti; 1326 unsigned i = 0; 1327 1328 /* Ensure that all targets supports discard_zeroes_data. */ 1329 while (i < dm_table_get_num_targets(t)) { 1330 ti = dm_table_get_target(t, i++); 1331 1332 if (ti->discard_zeroes_data_unsupported) 1333 return 0; 1334 } 1335 1336 return 1; 1337} 1338 1339static int device_is_nonrot(struct dm_target *ti, struct dm_dev *dev, 1340 sector_t start, sector_t len, void *data) 1341{ 1342 struct request_queue *q = bdev_get_queue(dev->bdev); 1343 1344 return q && blk_queue_nonrot(q); 1345} 1346 1347static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev, 1348 sector_t start, sector_t len, void *data) 1349{ 1350 struct request_queue *q = bdev_get_queue(dev->bdev); 1351 1352 return q && !blk_queue_add_random(q); 1353} 1354 1355static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev, 1356 sector_t start, sector_t len, void *data) 1357{ 1358 struct request_queue *q = bdev_get_queue(dev->bdev); 1359 1360 return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags); 1361} 1362 1363static bool dm_table_all_devices_attribute(struct dm_table *t, 1364 iterate_devices_callout_fn func) 1365{ 1366 struct dm_target *ti; 1367 unsigned i = 0; 1368 1369 while (i < dm_table_get_num_targets(t)) { 1370 ti = dm_table_get_target(t, i++); 1371 1372 if (!ti->type->iterate_devices || 1373 !ti->type->iterate_devices(ti, func, NULL)) 1374 return 0; 1375 } 1376 1377 return 1; 1378} 1379 1380static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev, 1381 sector_t start, sector_t len, void *data) 1382{ 1383 struct request_queue *q = bdev_get_queue(dev->bdev); 1384 1385 return q && !q->limits.max_write_same_sectors; 1386} 1387 1388static bool dm_table_supports_write_same(struct dm_table *t) 1389{ 1390 struct dm_target *ti; 1391 unsigned i = 0; 1392 1393 while (i < dm_table_get_num_targets(t)) { 1394 ti = dm_table_get_target(t, i++); 1395 1396 if (!ti->num_write_same_bios) 1397 return false; 1398 1399 if (!ti->type->iterate_devices || 1400 ti->type->iterate_devices(ti, device_not_write_same_capable, NULL)) 1401 return false; 1402 } 1403 1404 return true; 1405} 1406 1407static int device_discard_capable(struct dm_target *ti, struct dm_dev *dev, 1408 sector_t start, sector_t len, void *data) 1409{ 1410 struct request_queue *q = bdev_get_queue(dev->bdev); 1411 1412 return q && blk_queue_discard(q); 1413} 1414 1415static bool dm_table_supports_discards(struct dm_table *t) 1416{ 1417 struct dm_target *ti; 1418 unsigned i = 0; 1419 1420 /* 1421 * Unless any target used by the table set discards_supported, 1422 * require at least one underlying device to support discards. 1423 * t->devices includes internal dm devices such as mirror logs 1424 * so we need to use iterate_devices here, which targets 1425 * supporting discard selectively must provide. 1426 */ 1427 while (i < dm_table_get_num_targets(t)) { 1428 ti = dm_table_get_target(t, i++); 1429 1430 if (!ti->num_discard_bios) 1431 continue; 1432 1433 if (ti->discards_supported) 1434 return 1; 1435 1436 if (ti->type->iterate_devices && 1437 ti->type->iterate_devices(ti, device_discard_capable, NULL)) 1438 return 1; 1439 } 1440 1441 return 0; 1442} 1443 1444void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, 1445 struct queue_limits *limits) 1446{ 1447 unsigned flush = 0; 1448 1449 /* 1450 * Copy table's limits to the DM device's request_queue 1451 */ 1452 q->limits = *limits; 1453 1454 if (!dm_table_supports_discards(t)) 1455 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); 1456 else 1457 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 1458 1459 if (dm_table_supports_flush(t, REQ_FLUSH)) { 1460 flush |= REQ_FLUSH; 1461 if (dm_table_supports_flush(t, REQ_FUA)) 1462 flush |= REQ_FUA; 1463 } 1464 blk_queue_flush(q, flush); 1465 1466 if (!dm_table_discard_zeroes_data(t)) 1467 q->limits.discard_zeroes_data = 0; 1468 1469 /* Ensure that all underlying devices are non-rotational. */ 1470 if (dm_table_all_devices_attribute(t, device_is_nonrot)) 1471 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); 1472 else 1473 queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, q); 1474 1475 if (!dm_table_supports_write_same(t)) 1476 q->limits.max_write_same_sectors = 0; 1477 1478 if (dm_table_all_devices_attribute(t, queue_supports_sg_merge)) 1479 queue_flag_clear_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); 1480 else 1481 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); 1482 1483 dm_table_set_integrity(t); 1484 1485 /* 1486 * Determine whether or not this queue's I/O timings contribute 1487 * to the entropy pool, Only request-based targets use this. 1488 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not 1489 * have it set. 1490 */ 1491 if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random)) 1492 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q); 1493 1494 /* 1495 * QUEUE_FLAG_STACKABLE must be set after all queue settings are 1496 * visible to other CPUs because, once the flag is set, incoming bios 1497 * are processed by request-based dm, which refers to the queue 1498 * settings. 1499 * Until the flag set, bios are passed to bio-based dm and queued to 1500 * md->deferred where queue settings are not needed yet. 1501 * Those bios are passed to request-based dm at the resume time. 1502 */ 1503 smp_mb(); 1504 if (dm_table_request_based(t)) 1505 queue_flag_set_unlocked(QUEUE_FLAG_STACKABLE, q); 1506} 1507 1508unsigned int dm_table_get_num_targets(struct dm_table *t) 1509{ 1510 return t->num_targets; 1511} 1512 1513struct list_head *dm_table_get_devices(struct dm_table *t) 1514{ 1515 return &t->devices; 1516} 1517 1518fmode_t dm_table_get_mode(struct dm_table *t) 1519{ 1520 return t->mode; 1521} 1522EXPORT_SYMBOL(dm_table_get_mode); 1523 1524static void suspend_targets(struct dm_table *t, unsigned postsuspend) 1525{ 1526 int i = t->num_targets; 1527 struct dm_target *ti = t->targets; 1528 1529 while (i--) { 1530 if (postsuspend) { 1531 if (ti->type->postsuspend) 1532 ti->type->postsuspend(ti); 1533 } else if (ti->type->presuspend) 1534 ti->type->presuspend(ti); 1535 1536 ti++; 1537 } 1538} 1539 1540void dm_table_presuspend_targets(struct dm_table *t) 1541{ 1542 if (!t) 1543 return; 1544 1545 suspend_targets(t, 0); 1546} 1547 1548void dm_table_postsuspend_targets(struct dm_table *t) 1549{ 1550 if (!t) 1551 return; 1552 1553 suspend_targets(t, 1); 1554} 1555 1556int dm_table_resume_targets(struct dm_table *t) 1557{ 1558 int i, r = 0; 1559 1560 for (i = 0; i < t->num_targets; i++) { 1561 struct dm_target *ti = t->targets + i; 1562 1563 if (!ti->type->preresume) 1564 continue; 1565 1566 r = ti->type->preresume(ti); 1567 if (r) { 1568 DMERR("%s: %s: preresume failed, error = %d", 1569 dm_device_name(t->md), ti->type->name, r); 1570 return r; 1571 } 1572 } 1573 1574 for (i = 0; i < t->num_targets; i++) { 1575 struct dm_target *ti = t->targets + i; 1576 1577 if (ti->type->resume) 1578 ti->type->resume(ti); 1579 } 1580 1581 return 0; 1582} 1583 1584void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb) 1585{ 1586 list_add(&cb->list, &t->target_callbacks); 1587} 1588EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks); 1589 1590int dm_table_any_congested(struct dm_table *t, int bdi_bits) 1591{ 1592 struct dm_dev_internal *dd; 1593 struct list_head *devices = dm_table_get_devices(t); 1594 struct dm_target_callbacks *cb; 1595 int r = 0; 1596 1597 list_for_each_entry(dd, devices, list) { 1598 struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev); 1599 char b[BDEVNAME_SIZE]; 1600 1601 if (likely(q)) 1602 r |= bdi_congested(&q->backing_dev_info, bdi_bits); 1603 else 1604 DMWARN_LIMIT("%s: any_congested: nonexistent device %s", 1605 dm_device_name(t->md), 1606 bdevname(dd->dm_dev->bdev, b)); 1607 } 1608 1609 list_for_each_entry(cb, &t->target_callbacks, list) 1610 if (cb->congested_fn) 1611 r |= cb->congested_fn(cb, bdi_bits); 1612 1613 return r; 1614} 1615 1616int dm_table_any_busy_target(struct dm_table *t) 1617{ 1618 unsigned i; 1619 struct dm_target *ti; 1620 1621 for (i = 0; i < t->num_targets; i++) { 1622 ti = t->targets + i; 1623 if (ti->type->busy && ti->type->busy(ti)) 1624 return 1; 1625 } 1626 1627 return 0; 1628} 1629 1630struct mapped_device *dm_table_get_md(struct dm_table *t) 1631{ 1632 return t->md; 1633} 1634EXPORT_SYMBOL(dm_table_get_md); 1635 1636void dm_table_run_md_queue_async(struct dm_table *t) 1637{ 1638 struct mapped_device *md; 1639 struct request_queue *queue; 1640 unsigned long flags; 1641 1642 if (!dm_table_request_based(t)) 1643 return; 1644 1645 md = dm_table_get_md(t); 1646 queue = dm_get_md_queue(md); 1647 if (queue) { 1648 spin_lock_irqsave(queue->queue_lock, flags); 1649 blk_run_queue_async(queue); 1650 spin_unlock_irqrestore(queue->queue_lock, flags); 1651 } 1652} 1653EXPORT_SYMBOL(dm_table_run_md_queue_async); 1654 1655