1/* 2 * Copyright (C) 1991, 1992 Linus Torvalds 3 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics 4 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE 5 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de> 6 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> 7 * - July2000 8 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001 9 */ 10 11/* 12 * This handles all read/write requests to block devices 13 */ 14#include <linux/kernel.h> 15#include <linux/module.h> 16#include <linux/backing-dev.h> 17#include <linux/bio.h> 18#include <linux/blkdev.h> 19#include <linux/blk-mq.h> 20#include <linux/highmem.h> 21#include <linux/mm.h> 22#include <linux/kernel_stat.h> 23#include <linux/string.h> 24#include <linux/init.h> 25#include <linux/completion.h> 26#include <linux/slab.h> 27#include <linux/swap.h> 28#include <linux/writeback.h> 29#include <linux/task_io_accounting_ops.h> 30#include <linux/fault-inject.h> 31#include <linux/list_sort.h> 32#include <linux/delay.h> 33#include <linux/ratelimit.h> 34#include <linux/pm_runtime.h> 35 36#define CREATE_TRACE_POINTS 37#include <trace/events/block.h> 38 39#include "blk.h" 40#include "blk-cgroup.h" 41#include "blk-mq.h" 42 43EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); 44EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); 45EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); 46EXPORT_TRACEPOINT_SYMBOL_GPL(block_split); 47EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug); 48 49DEFINE_IDA(blk_queue_ida); 50 51/* 52 * For the allocated request tables 53 */ 54struct kmem_cache *request_cachep = NULL; 55 56/* 57 * For queue allocation 58 */ 59struct kmem_cache *blk_requestq_cachep; 60 61/* 62 * Controlling structure to kblockd 63 */ 64static struct workqueue_struct *kblockd_workqueue; 65 66void blk_queue_congestion_threshold(struct request_queue *q) 67{ 68 int nr; 69 70 nr = q->nr_requests - (q->nr_requests / 8) + 1; 71 if (nr > q->nr_requests) 72 nr = q->nr_requests; 73 q->nr_congestion_on = nr; 74 75 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1; 76 if (nr < 1) 77 nr = 1; 78 q->nr_congestion_off = nr; 79} 80 81/** 82 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info 83 * @bdev: device 84 * 85 * Locates the passed device's request queue and returns the address of its 86 * backing_dev_info. This function can only be called if @bdev is opened 87 * and the return value is never NULL. 88 */ 89struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev) 90{ 91 struct request_queue *q = bdev_get_queue(bdev); 92 93 return &q->backing_dev_info; 94} 95EXPORT_SYMBOL(blk_get_backing_dev_info); 96 97void blk_rq_init(struct request_queue *q, struct request *rq) 98{ 99 memset(rq, 0, sizeof(*rq)); 100 101 INIT_LIST_HEAD(&rq->queuelist); 102 INIT_LIST_HEAD(&rq->timeout_list); 103 rq->cpu = -1; 104 rq->q = q; 105 rq->__sector = (sector_t) -1; 106 INIT_HLIST_NODE(&rq->hash); 107 RB_CLEAR_NODE(&rq->rb_node); 108 rq->cmd = rq->__cmd; 109 rq->cmd_len = BLK_MAX_CDB; 110 rq->tag = -1; 111 rq->start_time = jiffies; 112 set_start_time_ns(rq); 113 rq->part = NULL; 114} 115EXPORT_SYMBOL(blk_rq_init); 116 117static void req_bio_endio(struct request *rq, struct bio *bio, 118 unsigned int nbytes, int error) 119{ 120 if (error) 121 clear_bit(BIO_UPTODATE, &bio->bi_flags); 122 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) 123 error = -EIO; 124 125 if (unlikely(rq->cmd_flags & REQ_QUIET)) 126 set_bit(BIO_QUIET, &bio->bi_flags); 127 128 bio_advance(bio, nbytes); 129 130 /* don't actually finish bio if it's part of flush sequence */ 131 if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) 132 bio_endio(bio, error); 133} 134 135void blk_dump_rq_flags(struct request *rq, char *msg) 136{ 137 int bit; 138 139 printk(KERN_INFO "%s: dev %s: type=%x, flags=%llx\n", msg, 140 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type, 141 (unsigned long long) rq->cmd_flags); 142 143 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", 144 (unsigned long long)blk_rq_pos(rq), 145 blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); 146 printk(KERN_INFO " bio %p, biotail %p, len %u\n", 147 rq->bio, rq->biotail, blk_rq_bytes(rq)); 148 149 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { 150 printk(KERN_INFO " cdb: "); 151 for (bit = 0; bit < BLK_MAX_CDB; bit++) 152 printk("%02x ", rq->cmd[bit]); 153 printk("\n"); 154 } 155} 156EXPORT_SYMBOL(blk_dump_rq_flags); 157 158static void blk_delay_work(struct work_struct *work) 159{ 160 struct request_queue *q; 161 162 q = container_of(work, struct request_queue, delay_work.work); 163 spin_lock_irq(q->queue_lock); 164 __blk_run_queue(q); 165 spin_unlock_irq(q->queue_lock); 166} 167 168/** 169 * blk_delay_queue - restart queueing after defined interval 170 * @q: The &struct request_queue in question 171 * @msecs: Delay in msecs 172 * 173 * Description: 174 * Sometimes queueing needs to be postponed for a little while, to allow 175 * resources to come back. This function will make sure that queueing is 176 * restarted around the specified time. Queue lock must be held. 177 */ 178void blk_delay_queue(struct request_queue *q, unsigned long msecs) 179{ 180 if (likely(!blk_queue_dead(q))) 181 queue_delayed_work(kblockd_workqueue, &q->delay_work, 182 msecs_to_jiffies(msecs)); 183} 184EXPORT_SYMBOL(blk_delay_queue); 185 186/** 187 * blk_start_queue - restart a previously stopped queue 188 * @q: The &struct request_queue in question 189 * 190 * Description: 191 * blk_start_queue() will clear the stop flag on the queue, and call 192 * the request_fn for the queue if it was in a stopped state when 193 * entered. Also see blk_stop_queue(). Queue lock must be held. 194 **/ 195void blk_start_queue(struct request_queue *q) 196{ 197 WARN_ON(!irqs_disabled()); 198 199 queue_flag_clear(QUEUE_FLAG_STOPPED, q); 200 __blk_run_queue(q); 201} 202EXPORT_SYMBOL(blk_start_queue); 203 204/** 205 * blk_stop_queue - stop a queue 206 * @q: The &struct request_queue in question 207 * 208 * Description: 209 * The Linux block layer assumes that a block driver will consume all 210 * entries on the request queue when the request_fn strategy is called. 211 * Often this will not happen, because of hardware limitations (queue 212 * depth settings). If a device driver gets a 'queue full' response, 213 * or if it simply chooses not to queue more I/O at one point, it can 214 * call this function to prevent the request_fn from being called until 215 * the driver has signalled it's ready to go again. This happens by calling 216 * blk_start_queue() to restart queue operations. Queue lock must be held. 217 **/ 218void blk_stop_queue(struct request_queue *q) 219{ 220 cancel_delayed_work(&q->delay_work); 221 queue_flag_set(QUEUE_FLAG_STOPPED, q); 222} 223EXPORT_SYMBOL(blk_stop_queue); 224 225/** 226 * blk_sync_queue - cancel any pending callbacks on a queue 227 * @q: the queue 228 * 229 * Description: 230 * The block layer may perform asynchronous callback activity 231 * on a queue, such as calling the unplug function after a timeout. 232 * A block device may call blk_sync_queue to ensure that any 233 * such activity is cancelled, thus allowing it to release resources 234 * that the callbacks might use. The caller must already have made sure 235 * that its ->make_request_fn will not re-add plugging prior to calling 236 * this function. 237 * 238 * This function does not cancel any asynchronous activity arising 239 * out of elevator or throttling code. That would require elevator_exit() 240 * and blkcg_exit_queue() to be called with queue lock initialized. 241 * 242 */ 243void blk_sync_queue(struct request_queue *q) 244{ 245 del_timer_sync(&q->timeout); 246 247 if (q->mq_ops) { 248 struct blk_mq_hw_ctx *hctx; 249 int i; 250 251 queue_for_each_hw_ctx(q, hctx, i) { 252 cancel_delayed_work_sync(&hctx->run_work); 253 cancel_delayed_work_sync(&hctx->delay_work); 254 } 255 } else { 256 cancel_delayed_work_sync(&q->delay_work); 257 } 258} 259EXPORT_SYMBOL(blk_sync_queue); 260 261/** 262 * __blk_run_queue_uncond - run a queue whether or not it has been stopped 263 * @q: The queue to run 264 * 265 * Description: 266 * Invoke request handling on a queue if there are any pending requests. 267 * May be used to restart request handling after a request has completed. 268 * This variant runs the queue whether or not the queue has been 269 * stopped. Must be called with the queue lock held and interrupts 270 * disabled. See also @blk_run_queue. 271 */ 272inline void __blk_run_queue_uncond(struct request_queue *q) 273{ 274 if (unlikely(blk_queue_dead(q))) 275 return; 276 277 /* 278 * Some request_fn implementations, e.g. scsi_request_fn(), unlock 279 * the queue lock internally. As a result multiple threads may be 280 * running such a request function concurrently. Keep track of the 281 * number of active request_fn invocations such that blk_drain_queue() 282 * can wait until all these request_fn calls have finished. 283 */ 284 q->request_fn_active++; 285 q->request_fn(q); 286 q->request_fn_active--; 287} 288 289/** 290 * __blk_run_queue - run a single device queue 291 * @q: The queue to run 292 * 293 * Description: 294 * See @blk_run_queue. This variant must be called with the queue lock 295 * held and interrupts disabled. 296 */ 297void __blk_run_queue(struct request_queue *q) 298{ 299 if (unlikely(blk_queue_stopped(q))) 300 return; 301 302 __blk_run_queue_uncond(q); 303} 304EXPORT_SYMBOL(__blk_run_queue); 305 306/** 307 * blk_run_queue_async - run a single device queue in workqueue context 308 * @q: The queue to run 309 * 310 * Description: 311 * Tells kblockd to perform the equivalent of @blk_run_queue on behalf 312 * of us. The caller must hold the queue lock. 313 */ 314void blk_run_queue_async(struct request_queue *q) 315{ 316 if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q))) 317 mod_delayed_work(kblockd_workqueue, &q->delay_work, 0); 318} 319EXPORT_SYMBOL(blk_run_queue_async); 320 321/** 322 * blk_run_queue - run a single device queue 323 * @q: The queue to run 324 * 325 * Description: 326 * Invoke request handling on this queue, if it has pending work to do. 327 * May be used to restart queueing when a request has completed. 328 */ 329void blk_run_queue(struct request_queue *q) 330{ 331 unsigned long flags; 332 333 spin_lock_irqsave(q->queue_lock, flags); 334 __blk_run_queue(q); 335 spin_unlock_irqrestore(q->queue_lock, flags); 336} 337EXPORT_SYMBOL(blk_run_queue); 338 339void blk_put_queue(struct request_queue *q) 340{ 341 kobject_put(&q->kobj); 342} 343EXPORT_SYMBOL(blk_put_queue); 344 345/** 346 * __blk_drain_queue - drain requests from request_queue 347 * @q: queue to drain 348 * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV 349 * 350 * Drain requests from @q. If @drain_all is set, all requests are drained. 351 * If not, only ELVPRIV requests are drained. The caller is responsible 352 * for ensuring that no new requests which need to be drained are queued. 353 */ 354static void __blk_drain_queue(struct request_queue *q, bool drain_all) 355 __releases(q->queue_lock) 356 __acquires(q->queue_lock) 357{ 358 int i; 359 360 lockdep_assert_held(q->queue_lock); 361 362 while (true) { 363 bool drain = false; 364 365 /* 366 * The caller might be trying to drain @q before its 367 * elevator is initialized. 368 */ 369 if (q->elevator) 370 elv_drain_elevator(q); 371 372 blkcg_drain_queue(q); 373 374 /* 375 * This function might be called on a queue which failed 376 * driver init after queue creation or is not yet fully 377 * active yet. Some drivers (e.g. fd and loop) get unhappy 378 * in such cases. Kick queue iff dispatch queue has 379 * something on it and @q has request_fn set. 380 */ 381 if (!list_empty(&q->queue_head) && q->request_fn) 382 __blk_run_queue(q); 383 384 drain |= q->nr_rqs_elvpriv; 385 drain |= q->request_fn_active; 386 387 /* 388 * Unfortunately, requests are queued at and tracked from 389 * multiple places and there's no single counter which can 390 * be drained. Check all the queues and counters. 391 */ 392 if (drain_all) { 393 struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); 394 drain |= !list_empty(&q->queue_head); 395 for (i = 0; i < 2; i++) { 396 drain |= q->nr_rqs[i]; 397 drain |= q->in_flight[i]; 398 if (fq) 399 drain |= !list_empty(&fq->flush_queue[i]); 400 } 401 } 402 403 if (!drain) 404 break; 405 406 spin_unlock_irq(q->queue_lock); 407 408 msleep(10); 409 410 spin_lock_irq(q->queue_lock); 411 } 412 413 /* 414 * With queue marked dead, any woken up waiter will fail the 415 * allocation path, so the wakeup chaining is lost and we're 416 * left with hung waiters. We need to wake up those waiters. 417 */ 418 if (q->request_fn) { 419 struct request_list *rl; 420 421 blk_queue_for_each_rl(rl, q) 422 for (i = 0; i < ARRAY_SIZE(rl->wait); i++) 423 wake_up_all(&rl->wait[i]); 424 } 425} 426 427/** 428 * blk_queue_bypass_start - enter queue bypass mode 429 * @q: queue of interest 430 * 431 * In bypass mode, only the dispatch FIFO queue of @q is used. This 432 * function makes @q enter bypass mode and drains all requests which were 433 * throttled or issued before. On return, it's guaranteed that no request 434 * is being throttled or has ELVPRIV set and blk_queue_bypass() %true 435 * inside queue or RCU read lock. 436 */ 437void blk_queue_bypass_start(struct request_queue *q) 438{ 439 spin_lock_irq(q->queue_lock); 440 q->bypass_depth++; 441 queue_flag_set(QUEUE_FLAG_BYPASS, q); 442 spin_unlock_irq(q->queue_lock); 443 444 /* 445 * Queues start drained. Skip actual draining till init is 446 * complete. This avoids lenghty delays during queue init which 447 * can happen many times during boot. 448 */ 449 if (blk_queue_init_done(q)) { 450 spin_lock_irq(q->queue_lock); 451 __blk_drain_queue(q, false); 452 spin_unlock_irq(q->queue_lock); 453 454 /* ensure blk_queue_bypass() is %true inside RCU read lock */ 455 synchronize_rcu(); 456 } 457} 458EXPORT_SYMBOL_GPL(blk_queue_bypass_start); 459 460/** 461 * blk_queue_bypass_end - leave queue bypass mode 462 * @q: queue of interest 463 * 464 * Leave bypass mode and restore the normal queueing behavior. 465 */ 466void blk_queue_bypass_end(struct request_queue *q) 467{ 468 spin_lock_irq(q->queue_lock); 469 if (!--q->bypass_depth) 470 queue_flag_clear(QUEUE_FLAG_BYPASS, q); 471 WARN_ON_ONCE(q->bypass_depth < 0); 472 spin_unlock_irq(q->queue_lock); 473} 474EXPORT_SYMBOL_GPL(blk_queue_bypass_end); 475 476/** 477 * blk_cleanup_queue - shutdown a request queue 478 * @q: request queue to shutdown 479 * 480 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and 481 * put it. All future requests will be failed immediately with -ENODEV. 482 */ 483void blk_cleanup_queue(struct request_queue *q) 484{ 485 spinlock_t *lock = q->queue_lock; 486 487 /* mark @q DYING, no new request or merges will be allowed afterwards */ 488 mutex_lock(&q->sysfs_lock); 489 queue_flag_set_unlocked(QUEUE_FLAG_DYING, q); 490 spin_lock_irq(lock); 491 492 /* 493 * A dying queue is permanently in bypass mode till released. Note 494 * that, unlike blk_queue_bypass_start(), we aren't performing 495 * synchronize_rcu() after entering bypass mode to avoid the delay 496 * as some drivers create and destroy a lot of queues while 497 * probing. This is still safe because blk_release_queue() will be 498 * called only after the queue refcnt drops to zero and nothing, 499 * RCU or not, would be traversing the queue by then. 500 */ 501 q->bypass_depth++; 502 queue_flag_set(QUEUE_FLAG_BYPASS, q); 503 504 queue_flag_set(QUEUE_FLAG_NOMERGES, q); 505 queue_flag_set(QUEUE_FLAG_NOXMERGES, q); 506 queue_flag_set(QUEUE_FLAG_DYING, q); 507 spin_unlock_irq(lock); 508 mutex_unlock(&q->sysfs_lock); 509 510 /* 511 * Drain all requests queued before DYING marking. Set DEAD flag to 512 * prevent that q->request_fn() gets invoked after draining finished. 513 */ 514 if (q->mq_ops) { 515 blk_mq_freeze_queue(q); 516 spin_lock_irq(lock); 517 } else { 518 spin_lock_irq(lock); 519 __blk_drain_queue(q, true); 520 } 521 queue_flag_set(QUEUE_FLAG_DEAD, q); 522 spin_unlock_irq(lock); 523 524 /* @q won't process any more request, flush async actions */ 525 del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer); 526 blk_sync_queue(q); 527 528 spin_lock_irq(lock); 529 if (q->queue_lock != &q->__queue_lock) 530 q->queue_lock = &q->__queue_lock; 531 spin_unlock_irq(lock); 532 533 /* @q is and will stay empty, shutdown and put */ 534 blk_put_queue(q); 535} 536EXPORT_SYMBOL(blk_cleanup_queue); 537 538int blk_init_rl(struct request_list *rl, struct request_queue *q, 539 gfp_t gfp_mask) 540{ 541 if (unlikely(rl->rq_pool)) 542 return 0; 543 544 rl->q = q; 545 rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0; 546 rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0; 547 init_waitqueue_head(&rl->wait[BLK_RW_SYNC]); 548 init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]); 549 550 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab, 551 mempool_free_slab, request_cachep, 552 gfp_mask, q->node); 553 if (!rl->rq_pool) 554 return -ENOMEM; 555 556 return 0; 557} 558 559void blk_exit_rl(struct request_list *rl) 560{ 561 if (rl->rq_pool) 562 mempool_destroy(rl->rq_pool); 563} 564 565struct request_queue *blk_alloc_queue(gfp_t gfp_mask) 566{ 567 return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE); 568} 569EXPORT_SYMBOL(blk_alloc_queue); 570 571struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) 572{ 573 struct request_queue *q; 574 int err; 575 576 q = kmem_cache_alloc_node(blk_requestq_cachep, 577 gfp_mask | __GFP_ZERO, node_id); 578 if (!q) 579 return NULL; 580 581 q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask); 582 if (q->id < 0) 583 goto fail_q; 584 585 q->backing_dev_info.ra_pages = 586 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; 587 q->backing_dev_info.state = 0; 588 q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY; 589 q->backing_dev_info.name = "block"; 590 q->node = node_id; 591 592 err = bdi_init(&q->backing_dev_info); 593 if (err) 594 goto fail_id; 595 596 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, 597 laptop_mode_timer_fn, (unsigned long) q); 598 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); 599 INIT_LIST_HEAD(&q->queue_head); 600 INIT_LIST_HEAD(&q->timeout_list); 601 INIT_LIST_HEAD(&q->icq_list); 602#ifdef CONFIG_BLK_CGROUP 603 INIT_LIST_HEAD(&q->blkg_list); 604#endif 605 INIT_DELAYED_WORK(&q->delay_work, blk_delay_work); 606 607 kobject_init(&q->kobj, &blk_queue_ktype); 608 609 mutex_init(&q->sysfs_lock); 610 spin_lock_init(&q->__queue_lock); 611 612 /* 613 * By default initialize queue_lock to internal lock and driver can 614 * override it later if need be. 615 */ 616 q->queue_lock = &q->__queue_lock; 617 618 /* 619 * A queue starts its life with bypass turned on to avoid 620 * unnecessary bypass on/off overhead and nasty surprises during 621 * init. The initial bypass will be finished when the queue is 622 * registered by blk_register_queue(). 623 */ 624 q->bypass_depth = 1; 625 __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags); 626 627 init_waitqueue_head(&q->mq_freeze_wq); 628 629 if (blkcg_init_queue(q)) 630 goto fail_bdi; 631 632 return q; 633 634fail_bdi: 635 bdi_destroy(&q->backing_dev_info); 636fail_id: 637 ida_simple_remove(&blk_queue_ida, q->id); 638fail_q: 639 kmem_cache_free(blk_requestq_cachep, q); 640 return NULL; 641} 642EXPORT_SYMBOL(blk_alloc_queue_node); 643 644/** 645 * blk_init_queue - prepare a request queue for use with a block device 646 * @rfn: The function to be called to process requests that have been 647 * placed on the queue. 648 * @lock: Request queue spin lock 649 * 650 * Description: 651 * If a block device wishes to use the standard request handling procedures, 652 * which sorts requests and coalesces adjacent requests, then it must 653 * call blk_init_queue(). The function @rfn will be called when there 654 * are requests on the queue that need to be processed. If the device 655 * supports plugging, then @rfn may not be called immediately when requests 656 * are available on the queue, but may be called at some time later instead. 657 * Plugged queues are generally unplugged when a buffer belonging to one 658 * of the requests on the queue is needed, or due to memory pressure. 659 * 660 * @rfn is not required, or even expected, to remove all requests off the 661 * queue, but only as many as it can handle at a time. If it does leave 662 * requests on the queue, it is responsible for arranging that the requests 663 * get dealt with eventually. 664 * 665 * The queue spin lock must be held while manipulating the requests on the 666 * request queue; this lock will be taken also from interrupt context, so irq 667 * disabling is needed for it. 668 * 669 * Function returns a pointer to the initialized request queue, or %NULL if 670 * it didn't succeed. 671 * 672 * Note: 673 * blk_init_queue() must be paired with a blk_cleanup_queue() call 674 * when the block device is deactivated (such as at module unload). 675 **/ 676 677struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) 678{ 679 return blk_init_queue_node(rfn, lock, NUMA_NO_NODE); 680} 681EXPORT_SYMBOL(blk_init_queue); 682 683struct request_queue * 684blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id) 685{ 686 struct request_queue *uninit_q, *q; 687 688 uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id); 689 if (!uninit_q) 690 return NULL; 691 692 q = blk_init_allocated_queue(uninit_q, rfn, lock); 693 if (!q) 694 blk_cleanup_queue(uninit_q); 695 696 return q; 697} 698EXPORT_SYMBOL(blk_init_queue_node); 699 700struct request_queue * 701blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, 702 spinlock_t *lock) 703{ 704 if (!q) 705 return NULL; 706 707 q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, 0); 708 if (!q->fq) 709 return NULL; 710 711 if (blk_init_rl(&q->root_rl, q, GFP_KERNEL)) 712 goto fail; 713 714 q->request_fn = rfn; 715 q->prep_rq_fn = NULL; 716 q->unprep_rq_fn = NULL; 717 q->queue_flags |= QUEUE_FLAG_DEFAULT; 718 719 /* Override internal queue lock with supplied lock pointer */ 720 if (lock) 721 q->queue_lock = lock; 722 723 /* 724 * This also sets hw/phys segments, boundary and size 725 */ 726 blk_queue_make_request(q, blk_queue_bio); 727 728 q->sg_reserved_size = INT_MAX; 729 730 /* Protect q->elevator from elevator_change */ 731 mutex_lock(&q->sysfs_lock); 732 733 /* init elevator */ 734 if (elevator_init(q, NULL)) { 735 mutex_unlock(&q->sysfs_lock); 736 goto fail; 737 } 738 739 mutex_unlock(&q->sysfs_lock); 740 741 return q; 742 743fail: 744 blk_free_flush_queue(q->fq); 745 return NULL; 746} 747EXPORT_SYMBOL(blk_init_allocated_queue); 748 749bool blk_get_queue(struct request_queue *q) 750{ 751 if (likely(!blk_queue_dying(q))) { 752 __blk_get_queue(q); 753 return true; 754 } 755 756 return false; 757} 758EXPORT_SYMBOL(blk_get_queue); 759 760static inline void blk_free_request(struct request_list *rl, struct request *rq) 761{ 762 if (rq->cmd_flags & REQ_ELVPRIV) { 763 elv_put_request(rl->q, rq); 764 if (rq->elv.icq) 765 put_io_context(rq->elv.icq->ioc); 766 } 767 768 mempool_free(rq, rl->rq_pool); 769} 770 771/* 772 * ioc_batching returns true if the ioc is a valid batching request and 773 * should be given priority access to a request. 774 */ 775static inline int ioc_batching(struct request_queue *q, struct io_context *ioc) 776{ 777 if (!ioc) 778 return 0; 779 780 /* 781 * Make sure the process is able to allocate at least 1 request 782 * even if the batch times out, otherwise we could theoretically 783 * lose wakeups. 784 */ 785 return ioc->nr_batch_requests == q->nr_batching || 786 (ioc->nr_batch_requests > 0 787 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME)); 788} 789 790/* 791 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This 792 * will cause the process to be a "batcher" on all queues in the system. This 793 * is the behaviour we want though - once it gets a wakeup it should be given 794 * a nice run. 795 */ 796static void ioc_set_batching(struct request_queue *q, struct io_context *ioc) 797{ 798 if (!ioc || ioc_batching(q, ioc)) 799 return; 800 801 ioc->nr_batch_requests = q->nr_batching; 802 ioc->last_waited = jiffies; 803} 804 805static void __freed_request(struct request_list *rl, int sync) 806{ 807 struct request_queue *q = rl->q; 808 809 /* 810 * bdi isn't aware of blkcg yet. As all async IOs end up root 811 * blkcg anyway, just use root blkcg state. 812 */ 813 if (rl == &q->root_rl && 814 rl->count[sync] < queue_congestion_off_threshold(q)) 815 blk_clear_queue_congested(q, sync); 816 817 if (rl->count[sync] + 1 <= q->nr_requests) { 818 if (waitqueue_active(&rl->wait[sync])) 819 wake_up(&rl->wait[sync]); 820 821 blk_clear_rl_full(rl, sync); 822 } 823} 824 825/* 826 * A request has just been released. Account for it, update the full and 827 * congestion status, wake up any waiters. Called under q->queue_lock. 828 */ 829static void freed_request(struct request_list *rl, unsigned int flags) 830{ 831 struct request_queue *q = rl->q; 832 int sync = rw_is_sync(flags); 833 834 q->nr_rqs[sync]--; 835 rl->count[sync]--; 836 if (flags & REQ_ELVPRIV) 837 q->nr_rqs_elvpriv--; 838 839 __freed_request(rl, sync); 840 841 if (unlikely(rl->starved[sync ^ 1])) 842 __freed_request(rl, sync ^ 1); 843} 844 845int blk_update_nr_requests(struct request_queue *q, unsigned int nr) 846{ 847 struct request_list *rl; 848 849 spin_lock_irq(q->queue_lock); 850 q->nr_requests = nr; 851 blk_queue_congestion_threshold(q); 852 853 /* congestion isn't cgroup aware and follows root blkcg for now */ 854 rl = &q->root_rl; 855 856 if (rl->count[BLK_RW_SYNC] >= queue_congestion_on_threshold(q)) 857 blk_set_queue_congested(q, BLK_RW_SYNC); 858 else if (rl->count[BLK_RW_SYNC] < queue_congestion_off_threshold(q)) 859 blk_clear_queue_congested(q, BLK_RW_SYNC); 860 861 if (rl->count[BLK_RW_ASYNC] >= queue_congestion_on_threshold(q)) 862 blk_set_queue_congested(q, BLK_RW_ASYNC); 863 else if (rl->count[BLK_RW_ASYNC] < queue_congestion_off_threshold(q)) 864 blk_clear_queue_congested(q, BLK_RW_ASYNC); 865 866 blk_queue_for_each_rl(rl, q) { 867 if (rl->count[BLK_RW_SYNC] >= q->nr_requests) { 868 blk_set_rl_full(rl, BLK_RW_SYNC); 869 } else { 870 blk_clear_rl_full(rl, BLK_RW_SYNC); 871 wake_up(&rl->wait[BLK_RW_SYNC]); 872 } 873 874 if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) { 875 blk_set_rl_full(rl, BLK_RW_ASYNC); 876 } else { 877 blk_clear_rl_full(rl, BLK_RW_ASYNC); 878 wake_up(&rl->wait[BLK_RW_ASYNC]); 879 } 880 } 881 882 spin_unlock_irq(q->queue_lock); 883 return 0; 884} 885 886/* 887 * Determine if elevator data should be initialized when allocating the 888 * request associated with @bio. 889 */ 890static bool blk_rq_should_init_elevator(struct bio *bio) 891{ 892 if (!bio) 893 return true; 894 895 /* 896 * Flush requests do not use the elevator so skip initialization. 897 * This allows a request to share the flush and elevator data. 898 */ 899 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) 900 return false; 901 902 return true; 903} 904 905/** 906 * rq_ioc - determine io_context for request allocation 907 * @bio: request being allocated is for this bio (can be %NULL) 908 * 909 * Determine io_context to use for request allocation for @bio. May return 910 * %NULL if %current->io_context doesn't exist. 911 */ 912static struct io_context *rq_ioc(struct bio *bio) 913{ 914#ifdef CONFIG_BLK_CGROUP 915 if (bio && bio->bi_ioc) 916 return bio->bi_ioc; 917#endif 918 return current->io_context; 919} 920 921/** 922 * __get_request - get a free request 923 * @rl: request list to allocate from 924 * @rw_flags: RW and SYNC flags 925 * @bio: bio to allocate request for (can be %NULL) 926 * @gfp_mask: allocation mask 927 * 928 * Get a free request from @q. This function may fail under memory 929 * pressure or if @q is dead. 930 * 931 * Must be called with @q->queue_lock held and, 932 * Returns ERR_PTR on failure, with @q->queue_lock held. 933 * Returns request pointer on success, with @q->queue_lock *not held*. 934 */ 935static struct request *__get_request(struct request_list *rl, int rw_flags, 936 struct bio *bio, gfp_t gfp_mask) 937{ 938 struct request_queue *q = rl->q; 939 struct request *rq; 940 struct elevator_type *et = q->elevator->type; 941 struct io_context *ioc = rq_ioc(bio); 942 struct io_cq *icq = NULL; 943 const bool is_sync = rw_is_sync(rw_flags) != 0; 944 int may_queue; 945 946 if (unlikely(blk_queue_dying(q))) 947 return ERR_PTR(-ENODEV); 948 949 may_queue = elv_may_queue(q, rw_flags); 950 if (may_queue == ELV_MQUEUE_NO) 951 goto rq_starved; 952 953 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) { 954 if (rl->count[is_sync]+1 >= q->nr_requests) { 955 /* 956 * The queue will fill after this allocation, so set 957 * it as full, and mark this process as "batching". 958 * This process will be allowed to complete a batch of 959 * requests, others will be blocked. 960 */ 961 if (!blk_rl_full(rl, is_sync)) { 962 ioc_set_batching(q, ioc); 963 blk_set_rl_full(rl, is_sync); 964 } else { 965 if (may_queue != ELV_MQUEUE_MUST 966 && !ioc_batching(q, ioc)) { 967 /* 968 * The queue is full and the allocating 969 * process is not a "batcher", and not 970 * exempted by the IO scheduler 971 */ 972 return ERR_PTR(-ENOMEM); 973 } 974 } 975 } 976 /* 977 * bdi isn't aware of blkcg yet. As all async IOs end up 978 * root blkcg anyway, just use root blkcg state. 979 */ 980 if (rl == &q->root_rl) 981 blk_set_queue_congested(q, is_sync); 982 } 983 984 /* 985 * Only allow batching queuers to allocate up to 50% over the defined 986 * limit of requests, otherwise we could have thousands of requests 987 * allocated with any setting of ->nr_requests 988 */ 989 if (rl->count[is_sync] >= (3 * q->nr_requests / 2)) 990 return ERR_PTR(-ENOMEM); 991 992 q->nr_rqs[is_sync]++; 993 rl->count[is_sync]++; 994 rl->starved[is_sync] = 0; 995 996 /* 997 * Decide whether the new request will be managed by elevator. If 998 * so, mark @rw_flags and increment elvpriv. Non-zero elvpriv will 999 * prevent the current elevator from being destroyed until the new 1000 * request is freed. This guarantees icq's won't be destroyed and 1001 * makes creating new ones safe. 1002 * 1003 * Also, lookup icq while holding queue_lock. If it doesn't exist, 1004 * it will be created after releasing queue_lock. 1005 */ 1006 if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) { 1007 rw_flags |= REQ_ELVPRIV; 1008 q->nr_rqs_elvpriv++; 1009 if (et->icq_cache && ioc) 1010 icq = ioc_lookup_icq(ioc, q); 1011 } 1012 1013 if (blk_queue_io_stat(q)) 1014 rw_flags |= REQ_IO_STAT; 1015 spin_unlock_irq(q->queue_lock); 1016 1017 /* allocate and init request */ 1018 rq = mempool_alloc(rl->rq_pool, gfp_mask); 1019 if (!rq) 1020 goto fail_alloc; 1021 1022 blk_rq_init(q, rq); 1023 blk_rq_set_rl(rq, rl); 1024 rq->cmd_flags = rw_flags | REQ_ALLOCED; 1025 1026 /* init elvpriv */ 1027 if (rw_flags & REQ_ELVPRIV) { 1028 if (unlikely(et->icq_cache && !icq)) { 1029 if (ioc) 1030 icq = ioc_create_icq(ioc, q, gfp_mask); 1031 if (!icq) 1032 goto fail_elvpriv; 1033 } 1034 1035 rq->elv.icq = icq; 1036 if (unlikely(elv_set_request(q, rq, bio, gfp_mask))) 1037 goto fail_elvpriv; 1038 1039 /* @rq->elv.icq holds io_context until @rq is freed */ 1040 if (icq) 1041 get_io_context(icq->ioc); 1042 } 1043out: 1044 /* 1045 * ioc may be NULL here, and ioc_batching will be false. That's 1046 * OK, if the queue is under the request limit then requests need 1047 * not count toward the nr_batch_requests limit. There will always 1048 * be some limit enforced by BLK_BATCH_TIME. 1049 */ 1050 if (ioc_batching(q, ioc)) 1051 ioc->nr_batch_requests--; 1052 1053 trace_block_getrq(q, bio, rw_flags & 1); 1054 return rq; 1055 1056fail_elvpriv: 1057 /* 1058 * elvpriv init failed. ioc, icq and elvpriv aren't mempool backed 1059 * and may fail indefinitely under memory pressure and thus 1060 * shouldn't stall IO. Treat this request as !elvpriv. This will 1061 * disturb iosched and blkcg but weird is bettern than dead. 1062 */ 1063 printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n", 1064 __func__, dev_name(q->backing_dev_info.dev)); 1065 1066 rq->cmd_flags &= ~REQ_ELVPRIV; 1067 rq->elv.icq = NULL; 1068 1069 spin_lock_irq(q->queue_lock); 1070 q->nr_rqs_elvpriv--; 1071 spin_unlock_irq(q->queue_lock); 1072 goto out; 1073 1074fail_alloc: 1075 /* 1076 * Allocation failed presumably due to memory. Undo anything we 1077 * might have messed up. 1078 * 1079 * Allocating task should really be put onto the front of the wait 1080 * queue, but this is pretty rare. 1081 */ 1082 spin_lock_irq(q->queue_lock); 1083 freed_request(rl, rw_flags); 1084 1085 /* 1086 * in the very unlikely event that allocation failed and no 1087 * requests for this direction was pending, mark us starved so that 1088 * freeing of a request in the other direction will notice 1089 * us. another possible fix would be to split the rq mempool into 1090 * READ and WRITE 1091 */ 1092rq_starved: 1093 if (unlikely(rl->count[is_sync] == 0)) 1094 rl->starved[is_sync] = 1; 1095 return ERR_PTR(-ENOMEM); 1096} 1097 1098/** 1099 * get_request - get a free request 1100 * @q: request_queue to allocate request from 1101 * @rw_flags: RW and SYNC flags 1102 * @bio: bio to allocate request for (can be %NULL) 1103 * @gfp_mask: allocation mask 1104 * 1105 * Get a free request from @q. If %__GFP_WAIT is set in @gfp_mask, this 1106 * function keeps retrying under memory pressure and fails iff @q is dead. 1107 * 1108 * Must be called with @q->queue_lock held and, 1109 * Returns ERR_PTR on failure, with @q->queue_lock held. 1110 * Returns request pointer on success, with @q->queue_lock *not held*. 1111 */ 1112static struct request *get_request(struct request_queue *q, int rw_flags, 1113 struct bio *bio, gfp_t gfp_mask) 1114{ 1115 const bool is_sync = rw_is_sync(rw_flags) != 0; 1116 DEFINE_WAIT(wait); 1117 struct request_list *rl; 1118 struct request *rq; 1119 1120 rl = blk_get_rl(q, bio); /* transferred to @rq on success */ 1121retry: 1122 rq = __get_request(rl, rw_flags, bio, gfp_mask); 1123 if (!IS_ERR(rq)) 1124 return rq; 1125 1126 if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) { 1127 blk_put_rl(rl); 1128 return rq; 1129 } 1130 1131 /* wait on @rl and retry */ 1132 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait, 1133 TASK_UNINTERRUPTIBLE); 1134 1135 trace_block_sleeprq(q, bio, rw_flags & 1); 1136 1137 spin_unlock_irq(q->queue_lock); 1138 io_schedule(); 1139 1140 /* 1141 * After sleeping, we become a "batching" process and will be able 1142 * to allocate at least one request, and up to a big batch of them 1143 * for a small period time. See ioc_batching, ioc_set_batching 1144 */ 1145 ioc_set_batching(q, current->io_context); 1146 1147 spin_lock_irq(q->queue_lock); 1148 finish_wait(&rl->wait[is_sync], &wait); 1149 1150 goto retry; 1151} 1152 1153static struct request *blk_old_get_request(struct request_queue *q, int rw, 1154 gfp_t gfp_mask) 1155{ 1156 struct request *rq; 1157 1158 BUG_ON(rw != READ && rw != WRITE); 1159 1160 /* create ioc upfront */ 1161 create_io_context(gfp_mask, q->node); 1162 1163 spin_lock_irq(q->queue_lock); 1164 rq = get_request(q, rw, NULL, gfp_mask); 1165 if (IS_ERR(rq)) 1166 spin_unlock_irq(q->queue_lock); 1167 /* q->queue_lock is unlocked at this point */ 1168 1169 return rq; 1170} 1171 1172struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask) 1173{ 1174 if (q->mq_ops) 1175 return blk_mq_alloc_request(q, rw, gfp_mask, false); 1176 else 1177 return blk_old_get_request(q, rw, gfp_mask); 1178} 1179EXPORT_SYMBOL(blk_get_request); 1180 1181/** 1182 * blk_make_request - given a bio, allocate a corresponding struct request. 1183 * @q: target request queue 1184 * @bio: The bio describing the memory mappings that will be submitted for IO. 1185 * It may be a chained-bio properly constructed by block/bio layer. 1186 * @gfp_mask: gfp flags to be used for memory allocation 1187 * 1188 * blk_make_request is the parallel of generic_make_request for BLOCK_PC 1189 * type commands. Where the struct request needs to be farther initialized by 1190 * the caller. It is passed a &struct bio, which describes the memory info of 1191 * the I/O transfer. 1192 * 1193 * The caller of blk_make_request must make sure that bi_io_vec 1194 * are set to describe the memory buffers. That bio_data_dir() will return 1195 * the needed direction of the request. (And all bio's in the passed bio-chain 1196 * are properly set accordingly) 1197 * 1198 * If called under none-sleepable conditions, mapped bio buffers must not 1199 * need bouncing, by calling the appropriate masked or flagged allocator, 1200 * suitable for the target device. Otherwise the call to blk_queue_bounce will 1201 * BUG. 1202 * 1203 * WARNING: When allocating/cloning a bio-chain, careful consideration should be 1204 * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for 1205 * anything but the first bio in the chain. Otherwise you risk waiting for IO 1206 * completion of a bio that hasn't been submitted yet, thus resulting in a 1207 * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead 1208 * of bio_alloc(), as that avoids the mempool deadlock. 1209 * If possible a big IO should be split into smaller parts when allocation 1210 * fails. Partial allocation should not be an error, or you risk a live-lock. 1211 */ 1212struct request *blk_make_request(struct request_queue *q, struct bio *bio, 1213 gfp_t gfp_mask) 1214{ 1215 struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask); 1216 1217 if (IS_ERR(rq)) 1218 return rq; 1219 1220 blk_rq_set_block_pc(rq); 1221 1222 for_each_bio(bio) { 1223 struct bio *bounce_bio = bio; 1224 int ret; 1225 1226 blk_queue_bounce(q, &bounce_bio); 1227 ret = blk_rq_append_bio(q, rq, bounce_bio); 1228 if (unlikely(ret)) { 1229 blk_put_request(rq); 1230 return ERR_PTR(ret); 1231 } 1232 } 1233 1234 return rq; 1235} 1236EXPORT_SYMBOL(blk_make_request); 1237 1238/** 1239 * blk_rq_set_block_pc - initialize a request to type BLOCK_PC 1240 * @rq: request to be initialized 1241 * 1242 */ 1243void blk_rq_set_block_pc(struct request *rq) 1244{ 1245 rq->cmd_type = REQ_TYPE_BLOCK_PC; 1246 rq->__data_len = 0; 1247 rq->__sector = (sector_t) -1; 1248 rq->bio = rq->biotail = NULL; 1249 memset(rq->__cmd, 0, sizeof(rq->__cmd)); 1250} 1251EXPORT_SYMBOL(blk_rq_set_block_pc); 1252 1253/** 1254 * blk_requeue_request - put a request back on queue 1255 * @q: request queue where request should be inserted 1256 * @rq: request to be inserted 1257 * 1258 * Description: 1259 * Drivers often keep queueing requests until the hardware cannot accept 1260 * more, when that condition happens we need to put the request back 1261 * on the queue. Must be called with queue lock held. 1262 */ 1263void blk_requeue_request(struct request_queue *q, struct request *rq) 1264{ 1265 blk_delete_timer(rq); 1266 blk_clear_rq_complete(rq); 1267 trace_block_rq_requeue(q, rq); 1268 1269 if (blk_rq_tagged(rq)) 1270 blk_queue_end_tag(q, rq); 1271 1272 BUG_ON(blk_queued_rq(rq)); 1273 1274 elv_requeue_request(q, rq); 1275} 1276EXPORT_SYMBOL(blk_requeue_request); 1277 1278static void add_acct_request(struct request_queue *q, struct request *rq, 1279 int where) 1280{ 1281 blk_account_io_start(rq, true); 1282 __elv_add_request(q, rq, where); 1283} 1284 1285static void part_round_stats_single(int cpu, struct hd_struct *part, 1286 unsigned long now) 1287{ 1288 int inflight; 1289 1290 if (now == part->stamp) 1291 return; 1292 1293 inflight = part_in_flight(part); 1294 if (inflight) { 1295 __part_stat_add(cpu, part, time_in_queue, 1296 inflight * (now - part->stamp)); 1297 __part_stat_add(cpu, part, io_ticks, (now - part->stamp)); 1298 } 1299 part->stamp = now; 1300} 1301 1302/** 1303 * part_round_stats() - Round off the performance stats on a struct disk_stats. 1304 * @cpu: cpu number for stats access 1305 * @part: target partition 1306 * 1307 * The average IO queue length and utilisation statistics are maintained 1308 * by observing the current state of the queue length and the amount of 1309 * time it has been in this state for. 1310 * 1311 * Normally, that accounting is done on IO completion, but that can result 1312 * in more than a second's worth of IO being accounted for within any one 1313 * second, leading to >100% utilisation. To deal with that, we call this 1314 * function to do a round-off before returning the results when reading 1315 * /proc/diskstats. This accounts immediately for all queue usage up to 1316 * the current jiffies and restarts the counters again. 1317 */ 1318void part_round_stats(int cpu, struct hd_struct *part) 1319{ 1320 unsigned long now = jiffies; 1321 1322 if (part->partno) 1323 part_round_stats_single(cpu, &part_to_disk(part)->part0, now); 1324 part_round_stats_single(cpu, part, now); 1325} 1326EXPORT_SYMBOL_GPL(part_round_stats); 1327 1328#ifdef CONFIG_PM_RUNTIME 1329static void blk_pm_put_request(struct request *rq) 1330{ 1331 if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && !--rq->q->nr_pending) 1332 pm_runtime_mark_last_busy(rq->q->dev); 1333} 1334#else 1335static inline void blk_pm_put_request(struct request *rq) {} 1336#endif 1337 1338/* 1339 * queue lock must be held 1340 */ 1341void __blk_put_request(struct request_queue *q, struct request *req) 1342{ 1343 if (unlikely(!q)) 1344 return; 1345 1346 if (q->mq_ops) { 1347 blk_mq_free_request(req); 1348 return; 1349 } 1350 1351 blk_pm_put_request(req); 1352 1353 elv_completed_request(q, req); 1354 1355 /* this is a bio leak */ 1356 WARN_ON(req->bio != NULL); 1357 1358 /* 1359 * Request may not have originated from ll_rw_blk. if not, 1360 * it didn't come out of our reserved rq pools 1361 */ 1362 if (req->cmd_flags & REQ_ALLOCED) { 1363 unsigned int flags = req->cmd_flags; 1364 struct request_list *rl = blk_rq_rl(req); 1365 1366 BUG_ON(!list_empty(&req->queuelist)); 1367 BUG_ON(ELV_ON_HASH(req)); 1368 1369 blk_free_request(rl, req); 1370 freed_request(rl, flags); 1371 blk_put_rl(rl); 1372 } 1373} 1374EXPORT_SYMBOL_GPL(__blk_put_request); 1375 1376void blk_put_request(struct request *req) 1377{ 1378 struct request_queue *q = req->q; 1379 1380 if (q->mq_ops) 1381 blk_mq_free_request(req); 1382 else { 1383 unsigned long flags; 1384 1385 spin_lock_irqsave(q->queue_lock, flags); 1386 __blk_put_request(q, req); 1387 spin_unlock_irqrestore(q->queue_lock, flags); 1388 } 1389} 1390EXPORT_SYMBOL(blk_put_request); 1391 1392/** 1393 * blk_add_request_payload - add a payload to a request 1394 * @rq: request to update 1395 * @page: page backing the payload 1396 * @len: length of the payload. 1397 * 1398 * This allows to later add a payload to an already submitted request by 1399 * a block driver. The driver needs to take care of freeing the payload 1400 * itself. 1401 * 1402 * Note that this is a quite horrible hack and nothing but handling of 1403 * discard requests should ever use it. 1404 */ 1405void blk_add_request_payload(struct request *rq, struct page *page, 1406 unsigned int len) 1407{ 1408 struct bio *bio = rq->bio; 1409 1410 bio->bi_io_vec->bv_page = page; 1411 bio->bi_io_vec->bv_offset = 0; 1412 bio->bi_io_vec->bv_len = len; 1413 1414 bio->bi_iter.bi_size = len; 1415 bio->bi_vcnt = 1; 1416 bio->bi_phys_segments = 1; 1417 1418 rq->__data_len = rq->resid_len = len; 1419 rq->nr_phys_segments = 1; 1420} 1421EXPORT_SYMBOL_GPL(blk_add_request_payload); 1422 1423bool bio_attempt_back_merge(struct request_queue *q, struct request *req, 1424 struct bio *bio) 1425{ 1426 const int ff = bio->bi_rw & REQ_FAILFAST_MASK; 1427 1428 if (!ll_back_merge_fn(q, req, bio)) 1429 return false; 1430 1431 trace_block_bio_backmerge(q, req, bio); 1432 1433 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) 1434 blk_rq_set_mixed_merge(req); 1435 1436 req->biotail->bi_next = bio; 1437 req->biotail = bio; 1438 req->__data_len += bio->bi_iter.bi_size; 1439 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); 1440 1441 blk_account_io_start(req, false); 1442 return true; 1443} 1444 1445bool bio_attempt_front_merge(struct request_queue *q, struct request *req, 1446 struct bio *bio) 1447{ 1448 const int ff = bio->bi_rw & REQ_FAILFAST_MASK; 1449 1450 if (!ll_front_merge_fn(q, req, bio)) 1451 return false; 1452 1453 trace_block_bio_frontmerge(q, req, bio); 1454 1455 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) 1456 blk_rq_set_mixed_merge(req); 1457 1458 bio->bi_next = req->bio; 1459 req->bio = bio; 1460 1461 req->__sector = bio->bi_iter.bi_sector; 1462 req->__data_len += bio->bi_iter.bi_size; 1463 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); 1464 1465 blk_account_io_start(req, false); 1466 return true; 1467} 1468 1469/** 1470 * blk_attempt_plug_merge - try to merge with %current's plugged list 1471 * @q: request_queue new bio is being queued at 1472 * @bio: new bio being queued 1473 * @request_count: out parameter for number of traversed plugged requests 1474 * 1475 * Determine whether @bio being queued on @q can be merged with a request 1476 * on %current's plugged list. Returns %true if merge was successful, 1477 * otherwise %false. 1478 * 1479 * Plugging coalesces IOs from the same issuer for the same purpose without 1480 * going through @q->queue_lock. As such it's more of an issuing mechanism 1481 * than scheduling, and the request, while may have elvpriv data, is not 1482 * added on the elevator at this point. In addition, we don't have 1483 * reliable access to the elevator outside queue lock. Only check basic 1484 * merging parameters without querying the elevator. 1485 * 1486 * Caller must ensure !blk_queue_nomerges(q) beforehand. 1487 */ 1488bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 1489 unsigned int *request_count) 1490{ 1491 struct blk_plug *plug; 1492 struct request *rq; 1493 bool ret = false; 1494 struct list_head *plug_list; 1495 1496 plug = current->plug; 1497 if (!plug) 1498 goto out; 1499 *request_count = 0; 1500 1501 if (q->mq_ops) 1502 plug_list = &plug->mq_list; 1503 else 1504 plug_list = &plug->list; 1505 1506 list_for_each_entry_reverse(rq, plug_list, queuelist) { 1507 int el_ret; 1508 1509 if (rq->q == q) 1510 (*request_count)++; 1511 1512 if (rq->q != q || !blk_rq_merge_ok(rq, bio)) 1513 continue; 1514 1515 el_ret = blk_try_merge(rq, bio); 1516 if (el_ret == ELEVATOR_BACK_MERGE) { 1517 ret = bio_attempt_back_merge(q, rq, bio); 1518 if (ret) 1519 break; 1520 } else if (el_ret == ELEVATOR_FRONT_MERGE) { 1521 ret = bio_attempt_front_merge(q, rq, bio); 1522 if (ret) 1523 break; 1524 } 1525 } 1526out: 1527 return ret; 1528} 1529 1530void init_request_from_bio(struct request *req, struct bio *bio) 1531{ 1532 req->cmd_type = REQ_TYPE_FS; 1533 1534 req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK; 1535 if (bio->bi_rw & REQ_RAHEAD) 1536 req->cmd_flags |= REQ_FAILFAST_MASK; 1537 1538 req->errors = 0; 1539 req->__sector = bio->bi_iter.bi_sector; 1540 req->ioprio = bio_prio(bio); 1541 blk_rq_bio_prep(req->q, req, bio); 1542} 1543 1544void blk_queue_bio(struct request_queue *q, struct bio *bio) 1545{ 1546 const bool sync = !!(bio->bi_rw & REQ_SYNC); 1547 struct blk_plug *plug; 1548 int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT; 1549 struct request *req; 1550 unsigned int request_count = 0; 1551 1552 /* 1553 * low level driver can indicate that it wants pages above a 1554 * certain limit bounced to low memory (ie for highmem, or even 1555 * ISA dma in theory) 1556 */ 1557 blk_queue_bounce(q, &bio); 1558 1559 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1560 bio_endio(bio, -EIO); 1561 return; 1562 } 1563 1564 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { 1565 spin_lock_irq(q->queue_lock); 1566 where = ELEVATOR_INSERT_FLUSH; 1567 goto get_rq; 1568 } 1569 1570 /* 1571 * Check if we can merge with the plugged list before grabbing 1572 * any locks. 1573 */ 1574 if (!blk_queue_nomerges(q) && 1575 blk_attempt_plug_merge(q, bio, &request_count)) 1576 return; 1577 1578 spin_lock_irq(q->queue_lock); 1579 1580 el_ret = elv_merge(q, &req, bio); 1581 if (el_ret == ELEVATOR_BACK_MERGE) { 1582 if (bio_attempt_back_merge(q, req, bio)) { 1583 elv_bio_merged(q, req, bio); 1584 if (!attempt_back_merge(q, req)) 1585 elv_merged_request(q, req, el_ret); 1586 goto out_unlock; 1587 } 1588 } else if (el_ret == ELEVATOR_FRONT_MERGE) { 1589 if (bio_attempt_front_merge(q, req, bio)) { 1590 elv_bio_merged(q, req, bio); 1591 if (!attempt_front_merge(q, req)) 1592 elv_merged_request(q, req, el_ret); 1593 goto out_unlock; 1594 } 1595 } 1596 1597get_rq: 1598 /* 1599 * This sync check and mask will be re-done in init_request_from_bio(), 1600 * but we need to set it earlier to expose the sync flag to the 1601 * rq allocator and io schedulers. 1602 */ 1603 rw_flags = bio_data_dir(bio); 1604 if (sync) 1605 rw_flags |= REQ_SYNC; 1606 1607 /* 1608 * Grab a free request. This is might sleep but can not fail. 1609 * Returns with the queue unlocked. 1610 */ 1611 req = get_request(q, rw_flags, bio, GFP_NOIO); 1612 if (IS_ERR(req)) { 1613 bio_endio(bio, PTR_ERR(req)); /* @q is dead */ 1614 goto out_unlock; 1615 } 1616 1617 /* 1618 * After dropping the lock and possibly sleeping here, our request 1619 * may now be mergeable after it had proven unmergeable (above). 1620 * We don't worry about that case for efficiency. It won't happen 1621 * often, and the elevators are able to handle it. 1622 */ 1623 init_request_from_bio(req, bio); 1624 1625 if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) 1626 req->cpu = raw_smp_processor_id(); 1627 1628 plug = current->plug; 1629 if (plug) { 1630 /* 1631 * If this is the first request added after a plug, fire 1632 * of a plug trace. 1633 */ 1634 if (!request_count) 1635 trace_block_plug(q); 1636 else { 1637 if (request_count >= BLK_MAX_REQUEST_COUNT) { 1638 blk_flush_plug_list(plug, false); 1639 trace_block_plug(q); 1640 } 1641 } 1642 list_add_tail(&req->queuelist, &plug->list); 1643 blk_account_io_start(req, true); 1644 } else { 1645 spin_lock_irq(q->queue_lock); 1646 add_acct_request(q, req, where); 1647 __blk_run_queue(q); 1648out_unlock: 1649 spin_unlock_irq(q->queue_lock); 1650 } 1651} 1652EXPORT_SYMBOL_GPL(blk_queue_bio); /* for device mapper only */ 1653 1654/* 1655 * If bio->bi_dev is a partition, remap the location 1656 */ 1657static inline void blk_partition_remap(struct bio *bio) 1658{ 1659 struct block_device *bdev = bio->bi_bdev; 1660 1661 if (bio_sectors(bio) && bdev != bdev->bd_contains) { 1662 struct hd_struct *p = bdev->bd_part; 1663 1664 bio->bi_iter.bi_sector += p->start_sect; 1665 bio->bi_bdev = bdev->bd_contains; 1666 1667 trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio, 1668 bdev->bd_dev, 1669 bio->bi_iter.bi_sector - p->start_sect); 1670 } 1671} 1672 1673static void handle_bad_sector(struct bio *bio) 1674{ 1675 char b[BDEVNAME_SIZE]; 1676 1677 printk(KERN_INFO "attempt to access beyond end of device\n"); 1678 printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n", 1679 bdevname(bio->bi_bdev, b), 1680 bio->bi_rw, 1681 (unsigned long long)bio_end_sector(bio), 1682 (long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9)); 1683 1684 set_bit(BIO_EOF, &bio->bi_flags); 1685} 1686 1687#ifdef CONFIG_FAIL_MAKE_REQUEST 1688 1689static DECLARE_FAULT_ATTR(fail_make_request); 1690 1691static int __init setup_fail_make_request(char *str) 1692{ 1693 return setup_fault_attr(&fail_make_request, str); 1694} 1695__setup("fail_make_request=", setup_fail_make_request); 1696 1697static bool should_fail_request(struct hd_struct *part, unsigned int bytes) 1698{ 1699 return part->make_it_fail && should_fail(&fail_make_request, bytes); 1700} 1701 1702static int __init fail_make_request_debugfs(void) 1703{ 1704 struct dentry *dir = fault_create_debugfs_attr("fail_make_request", 1705 NULL, &fail_make_request); 1706 1707 return PTR_ERR_OR_ZERO(dir); 1708} 1709 1710late_initcall(fail_make_request_debugfs); 1711 1712#else /* CONFIG_FAIL_MAKE_REQUEST */ 1713 1714static inline bool should_fail_request(struct hd_struct *part, 1715 unsigned int bytes) 1716{ 1717 return false; 1718} 1719 1720#endif /* CONFIG_FAIL_MAKE_REQUEST */ 1721 1722/* 1723 * Check whether this bio extends beyond the end of the device. 1724 */ 1725static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors) 1726{ 1727 sector_t maxsector; 1728 1729 if (!nr_sectors) 1730 return 0; 1731 1732 /* Test device or partition size, when known. */ 1733 maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9; 1734 if (maxsector) { 1735 sector_t sector = bio->bi_iter.bi_sector; 1736 1737 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) { 1738 /* 1739 * This may well happen - the kernel calls bread() 1740 * without checking the size of the device, e.g., when 1741 * mounting a device. 1742 */ 1743 handle_bad_sector(bio); 1744 return 1; 1745 } 1746 } 1747 1748 return 0; 1749} 1750 1751static noinline_for_stack bool 1752generic_make_request_checks(struct bio *bio) 1753{ 1754 struct request_queue *q; 1755 int nr_sectors = bio_sectors(bio); 1756 int err = -EIO; 1757 char b[BDEVNAME_SIZE]; 1758 struct hd_struct *part; 1759 1760 might_sleep(); 1761 1762 if (bio_check_eod(bio, nr_sectors)) 1763 goto end_io; 1764 1765 q = bdev_get_queue(bio->bi_bdev); 1766 if (unlikely(!q)) { 1767 printk(KERN_ERR 1768 "generic_make_request: Trying to access " 1769 "nonexistent block-device %s (%Lu)\n", 1770 bdevname(bio->bi_bdev, b), 1771 (long long) bio->bi_iter.bi_sector); 1772 goto end_io; 1773 } 1774 1775 if (likely(bio_is_rw(bio) && 1776 nr_sectors > queue_max_hw_sectors(q))) { 1777 printk(KERN_ERR "bio too big device %s (%u > %u)\n", 1778 bdevname(bio->bi_bdev, b), 1779 bio_sectors(bio), 1780 queue_max_hw_sectors(q)); 1781 goto end_io; 1782 } 1783 1784 part = bio->bi_bdev->bd_part; 1785 if (should_fail_request(part, bio->bi_iter.bi_size) || 1786 should_fail_request(&part_to_disk(part)->part0, 1787 bio->bi_iter.bi_size)) 1788 goto end_io; 1789 1790 /* 1791 * If this device has partitions, remap block n 1792 * of partition p to block n+start(p) of the disk. 1793 */ 1794 blk_partition_remap(bio); 1795 1796 if (bio_check_eod(bio, nr_sectors)) 1797 goto end_io; 1798 1799 /* 1800 * Filter flush bio's early so that make_request based 1801 * drivers without flush support don't have to worry 1802 * about them. 1803 */ 1804 if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) { 1805 bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA); 1806 if (!nr_sectors) { 1807 err = 0; 1808 goto end_io; 1809 } 1810 } 1811 1812 if ((bio->bi_rw & REQ_DISCARD) && 1813 (!blk_queue_discard(q) || 1814 ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) { 1815 err = -EOPNOTSUPP; 1816 goto end_io; 1817 } 1818 1819 if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) { 1820 err = -EOPNOTSUPP; 1821 goto end_io; 1822 } 1823 1824 /* 1825 * Various block parts want %current->io_context and lazy ioc 1826 * allocation ends up trading a lot of pain for a small amount of 1827 * memory. Just allocate it upfront. This may fail and block 1828 * layer knows how to live with it. 1829 */ 1830 create_io_context(GFP_ATOMIC, q->node); 1831 1832 if (blk_throtl_bio(q, bio)) 1833 return false; /* throttled, will be resubmitted later */ 1834 1835 trace_block_bio_queue(q, bio); 1836 return true; 1837 1838end_io: 1839 bio_endio(bio, err); 1840 return false; 1841} 1842 1843/** 1844 * generic_make_request - hand a buffer to its device driver for I/O 1845 * @bio: The bio describing the location in memory and on the device. 1846 * 1847 * generic_make_request() is used to make I/O requests of block 1848 * devices. It is passed a &struct bio, which describes the I/O that needs 1849 * to be done. 1850 * 1851 * generic_make_request() does not return any status. The 1852 * success/failure status of the request, along with notification of 1853 * completion, is delivered asynchronously through the bio->bi_end_io 1854 * function described (one day) else where. 1855 * 1856 * The caller of generic_make_request must make sure that bi_io_vec 1857 * are set to describe the memory buffer, and that bi_dev and bi_sector are 1858 * set to describe the device address, and the 1859 * bi_end_io and optionally bi_private are set to describe how 1860 * completion notification should be signaled. 1861 * 1862 * generic_make_request and the drivers it calls may use bi_next if this 1863 * bio happens to be merged with someone else, and may resubmit the bio to 1864 * a lower device by calling into generic_make_request recursively, which 1865 * means the bio should NOT be touched after the call to ->make_request_fn. 1866 */ 1867void generic_make_request(struct bio *bio) 1868{ 1869 struct bio_list bio_list_on_stack; 1870 1871 if (!generic_make_request_checks(bio)) 1872 return; 1873 1874 /* 1875 * We only want one ->make_request_fn to be active at a time, else 1876 * stack usage with stacked devices could be a problem. So use 1877 * current->bio_list to keep a list of requests submited by a 1878 * make_request_fn function. current->bio_list is also used as a 1879 * flag to say if generic_make_request is currently active in this 1880 * task or not. If it is NULL, then no make_request is active. If 1881 * it is non-NULL, then a make_request is active, and new requests 1882 * should be added at the tail 1883 */ 1884 if (current->bio_list) { 1885 bio_list_add(current->bio_list, bio); 1886 return; 1887 } 1888 1889 /* following loop may be a bit non-obvious, and so deserves some 1890 * explanation. 1891 * Before entering the loop, bio->bi_next is NULL (as all callers 1892 * ensure that) so we have a list with a single bio. 1893 * We pretend that we have just taken it off a longer list, so 1894 * we assign bio_list to a pointer to the bio_list_on_stack, 1895 * thus initialising the bio_list of new bios to be 1896 * added. ->make_request() may indeed add some more bios 1897 * through a recursive call to generic_make_request. If it 1898 * did, we find a non-NULL value in bio_list and re-enter the loop 1899 * from the top. In this case we really did just take the bio 1900 * of the top of the list (no pretending) and so remove it from 1901 * bio_list, and call into ->make_request() again. 1902 */ 1903 BUG_ON(bio->bi_next); 1904 bio_list_init(&bio_list_on_stack); 1905 current->bio_list = &bio_list_on_stack; 1906 do { 1907 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 1908 1909 q->make_request_fn(q, bio); 1910 1911 bio = bio_list_pop(current->bio_list); 1912 } while (bio); 1913 current->bio_list = NULL; /* deactivate */ 1914} 1915EXPORT_SYMBOL(generic_make_request); 1916 1917/** 1918 * submit_bio - submit a bio to the block device layer for I/O 1919 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) 1920 * @bio: The &struct bio which describes the I/O 1921 * 1922 * submit_bio() is very similar in purpose to generic_make_request(), and 1923 * uses that function to do most of the work. Both are fairly rough 1924 * interfaces; @bio must be presetup and ready for I/O. 1925 * 1926 */ 1927void submit_bio(int rw, struct bio *bio) 1928{ 1929 bio->bi_rw |= rw; 1930 1931 /* 1932 * If it's a regular read/write or a barrier with data attached, 1933 * go through the normal accounting stuff before submission. 1934 */ 1935 if (bio_has_data(bio)) { 1936 unsigned int count; 1937 1938 if (unlikely(rw & REQ_WRITE_SAME)) 1939 count = bdev_logical_block_size(bio->bi_bdev) >> 9; 1940 else 1941 count = bio_sectors(bio); 1942 1943 if (rw & WRITE) { 1944 count_vm_events(PGPGOUT, count); 1945 } else { 1946 task_io_account_read(bio->bi_iter.bi_size); 1947 count_vm_events(PGPGIN, count); 1948 } 1949 1950 if (unlikely(block_dump)) { 1951 char b[BDEVNAME_SIZE]; 1952 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", 1953 current->comm, task_pid_nr(current), 1954 (rw & WRITE) ? "WRITE" : "READ", 1955 (unsigned long long)bio->bi_iter.bi_sector, 1956 bdevname(bio->bi_bdev, b), 1957 count); 1958 } 1959 } 1960 1961 generic_make_request(bio); 1962} 1963EXPORT_SYMBOL(submit_bio); 1964 1965/** 1966 * blk_rq_check_limits - Helper function to check a request for the queue limit 1967 * @q: the queue 1968 * @rq: the request being checked 1969 * 1970 * Description: 1971 * @rq may have been made based on weaker limitations of upper-level queues 1972 * in request stacking drivers, and it may violate the limitation of @q. 1973 * Since the block layer and the underlying device driver trust @rq 1974 * after it is inserted to @q, it should be checked against @q before 1975 * the insertion using this generic function. 1976 * 1977 * This function should also be useful for request stacking drivers 1978 * in some cases below, so export this function. 1979 * Request stacking drivers like request-based dm may change the queue 1980 * limits while requests are in the queue (e.g. dm's table swapping). 1981 * Such request stacking drivers should check those requests against 1982 * the new queue limits again when they dispatch those requests, 1983 * although such checkings are also done against the old queue limits 1984 * when submitting requests. 1985 */ 1986int blk_rq_check_limits(struct request_queue *q, struct request *rq) 1987{ 1988 if (!rq_mergeable(rq)) 1989 return 0; 1990 1991 if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) { 1992 printk(KERN_ERR "%s: over max size limit.\n", __func__); 1993 return -EIO; 1994 } 1995 1996 /* 1997 * queue's settings related to segment counting like q->bounce_pfn 1998 * may differ from that of other stacking queues. 1999 * Recalculate it to check the request correctly on this queue's 2000 * limitation. 2001 */ 2002 blk_recalc_rq_segments(rq); 2003 if (rq->nr_phys_segments > queue_max_segments(q)) { 2004 printk(KERN_ERR "%s: over max segments limit.\n", __func__); 2005 return -EIO; 2006 } 2007 2008 return 0; 2009} 2010EXPORT_SYMBOL_GPL(blk_rq_check_limits); 2011 2012/** 2013 * blk_insert_cloned_request - Helper for stacking drivers to submit a request 2014 * @q: the queue to submit the request 2015 * @rq: the request being queued 2016 */ 2017int blk_insert_cloned_request(struct request_queue *q, struct request *rq) 2018{ 2019 unsigned long flags; 2020 int where = ELEVATOR_INSERT_BACK; 2021 2022 if (blk_rq_check_limits(q, rq)) 2023 return -EIO; 2024 2025 if (rq->rq_disk && 2026 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq))) 2027 return -EIO; 2028 2029 spin_lock_irqsave(q->queue_lock, flags); 2030 if (unlikely(blk_queue_dying(q))) { 2031 spin_unlock_irqrestore(q->queue_lock, flags); 2032 return -ENODEV; 2033 } 2034 2035 /* 2036 * Submitting request must be dequeued before calling this function 2037 * because it will be linked to another request_queue 2038 */ 2039 BUG_ON(blk_queued_rq(rq)); 2040 2041 if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA)) 2042 where = ELEVATOR_INSERT_FLUSH; 2043 2044 add_acct_request(q, rq, where); 2045 if (where == ELEVATOR_INSERT_FLUSH) 2046 __blk_run_queue(q); 2047 spin_unlock_irqrestore(q->queue_lock, flags); 2048 2049 return 0; 2050} 2051EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 2052 2053/** 2054 * blk_rq_err_bytes - determine number of bytes till the next failure boundary 2055 * @rq: request to examine 2056 * 2057 * Description: 2058 * A request could be merge of IOs which require different failure 2059 * handling. This function determines the number of bytes which 2060 * can be failed from the beginning of the request without 2061 * crossing into area which need to be retried further. 2062 * 2063 * Return: 2064 * The number of bytes to fail. 2065 * 2066 * Context: 2067 * queue_lock must be held. 2068 */ 2069unsigned int blk_rq_err_bytes(const struct request *rq) 2070{ 2071 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; 2072 unsigned int bytes = 0; 2073 struct bio *bio; 2074 2075 if (!(rq->cmd_flags & REQ_MIXED_MERGE)) 2076 return blk_rq_bytes(rq); 2077 2078 /* 2079 * Currently the only 'mixing' which can happen is between 2080 * different fastfail types. We can safely fail portions 2081 * which have all the failfast bits that the first one has - 2082 * the ones which are at least as eager to fail as the first 2083 * one. 2084 */ 2085 for (bio = rq->bio; bio; bio = bio->bi_next) { 2086 if ((bio->bi_rw & ff) != ff) 2087 break; 2088 bytes += bio->bi_iter.bi_size; 2089 } 2090 2091 /* this could lead to infinite loop */ 2092 BUG_ON(blk_rq_bytes(rq) && !bytes); 2093 return bytes; 2094} 2095EXPORT_SYMBOL_GPL(blk_rq_err_bytes); 2096 2097void blk_account_io_completion(struct request *req, unsigned int bytes) 2098{ 2099 if (blk_do_io_stat(req)) { 2100 const int rw = rq_data_dir(req); 2101 struct hd_struct *part; 2102 int cpu; 2103 2104 cpu = part_stat_lock(); 2105 part = req->part; 2106 part_stat_add(cpu, part, sectors[rw], bytes >> 9); 2107 part_stat_unlock(); 2108 } 2109} 2110 2111void blk_account_io_done(struct request *req) 2112{ 2113 /* 2114 * Account IO completion. flush_rq isn't accounted as a 2115 * normal IO on queueing nor completion. Accounting the 2116 * containing request is enough. 2117 */ 2118 if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) { 2119 unsigned long duration = jiffies - req->start_time; 2120 const int rw = rq_data_dir(req); 2121 struct hd_struct *part; 2122 int cpu; 2123 2124 cpu = part_stat_lock(); 2125 part = req->part; 2126 2127 part_stat_inc(cpu, part, ios[rw]); 2128 part_stat_add(cpu, part, ticks[rw], duration); 2129 part_round_stats(cpu, part); 2130 part_dec_in_flight(part, rw); 2131 2132 hd_struct_put(part); 2133 part_stat_unlock(); 2134 } 2135} 2136 2137#ifdef CONFIG_PM_RUNTIME 2138/* 2139 * Don't process normal requests when queue is suspended 2140 * or in the process of suspending/resuming 2141 */ 2142static struct request *blk_pm_peek_request(struct request_queue *q, 2143 struct request *rq) 2144{ 2145 if (q->dev && (q->rpm_status == RPM_SUSPENDED || 2146 (q->rpm_status != RPM_ACTIVE && !(rq->cmd_flags & REQ_PM)))) 2147 return NULL; 2148 else 2149 return rq; 2150} 2151#else 2152static inline struct request *blk_pm_peek_request(struct request_queue *q, 2153 struct request *rq) 2154{ 2155 return rq; 2156} 2157#endif 2158 2159void blk_account_io_start(struct request *rq, bool new_io) 2160{ 2161 struct hd_struct *part; 2162 int rw = rq_data_dir(rq); 2163 int cpu; 2164 2165 if (!blk_do_io_stat(rq)) 2166 return; 2167 2168 cpu = part_stat_lock(); 2169 2170 if (!new_io) { 2171 part = rq->part; 2172 part_stat_inc(cpu, part, merges[rw]); 2173 } else { 2174 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); 2175 if (!hd_struct_try_get(part)) { 2176 /* 2177 * The partition is already being removed, 2178 * the request will be accounted on the disk only 2179 * 2180 * We take a reference on disk->part0 although that 2181 * partition will never be deleted, so we can treat 2182 * it as any other partition. 2183 */ 2184 part = &rq->rq_disk->part0; 2185 hd_struct_get(part); 2186 } 2187 part_round_stats(cpu, part); 2188 part_inc_in_flight(part, rw); 2189 rq->part = part; 2190 } 2191 2192 part_stat_unlock(); 2193} 2194 2195/** 2196 * blk_peek_request - peek at the top of a request queue 2197 * @q: request queue to peek at 2198 * 2199 * Description: 2200 * Return the request at the top of @q. The returned request 2201 * should be started using blk_start_request() before LLD starts 2202 * processing it. 2203 * 2204 * Return: 2205 * Pointer to the request at the top of @q if available. Null 2206 * otherwise. 2207 * 2208 * Context: 2209 * queue_lock must be held. 2210 */ 2211struct request *blk_peek_request(struct request_queue *q) 2212{ 2213 struct request *rq; 2214 int ret; 2215 2216 while ((rq = __elv_next_request(q)) != NULL) { 2217 2218 rq = blk_pm_peek_request(q, rq); 2219 if (!rq) 2220 break; 2221 2222 if (!(rq->cmd_flags & REQ_STARTED)) { 2223 /* 2224 * This is the first time the device driver 2225 * sees this request (possibly after 2226 * requeueing). Notify IO scheduler. 2227 */ 2228 if (rq->cmd_flags & REQ_SORTED) 2229 elv_activate_rq(q, rq); 2230 2231 /* 2232 * just mark as started even if we don't start 2233 * it, a request that has been delayed should 2234 * not be passed by new incoming requests 2235 */ 2236 rq->cmd_flags |= REQ_STARTED; 2237 trace_block_rq_issue(q, rq); 2238 } 2239 2240 if (!q->boundary_rq || q->boundary_rq == rq) { 2241 q->end_sector = rq_end_sector(rq); 2242 q->boundary_rq = NULL; 2243 } 2244 2245 if (rq->cmd_flags & REQ_DONTPREP) 2246 break; 2247 2248 if (q->dma_drain_size && blk_rq_bytes(rq)) { 2249 /* 2250 * make sure space for the drain appears we 2251 * know we can do this because max_hw_segments 2252 * has been adjusted to be one fewer than the 2253 * device can handle 2254 */ 2255 rq->nr_phys_segments++; 2256 } 2257 2258 if (!q->prep_rq_fn) 2259 break; 2260 2261 ret = q->prep_rq_fn(q, rq); 2262 if (ret == BLKPREP_OK) { 2263 break; 2264 } else if (ret == BLKPREP_DEFER) { 2265 /* 2266 * the request may have been (partially) prepped. 2267 * we need to keep this request in the front to 2268 * avoid resource deadlock. REQ_STARTED will 2269 * prevent other fs requests from passing this one. 2270 */ 2271 if (q->dma_drain_size && blk_rq_bytes(rq) && 2272 !(rq->cmd_flags & REQ_DONTPREP)) { 2273 /* 2274 * remove the space for the drain we added 2275 * so that we don't add it again 2276 */ 2277 --rq->nr_phys_segments; 2278 } 2279 2280 rq = NULL; 2281 break; 2282 } else if (ret == BLKPREP_KILL) { 2283 rq->cmd_flags |= REQ_QUIET; 2284 /* 2285 * Mark this request as started so we don't trigger 2286 * any debug logic in the end I/O path. 2287 */ 2288 blk_start_request(rq); 2289 __blk_end_request_all(rq, -EIO); 2290 } else { 2291 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret); 2292 break; 2293 } 2294 } 2295 2296 return rq; 2297} 2298EXPORT_SYMBOL(blk_peek_request); 2299 2300void blk_dequeue_request(struct request *rq) 2301{ 2302 struct request_queue *q = rq->q; 2303 2304 BUG_ON(list_empty(&rq->queuelist)); 2305 BUG_ON(ELV_ON_HASH(rq)); 2306 2307 list_del_init(&rq->queuelist); 2308 2309 /* 2310 * the time frame between a request being removed from the lists 2311 * and to it is freed is accounted as io that is in progress at 2312 * the driver side. 2313 */ 2314 if (blk_account_rq(rq)) { 2315 q->in_flight[rq_is_sync(rq)]++; 2316 set_io_start_time_ns(rq); 2317 } 2318} 2319 2320/** 2321 * blk_start_request - start request processing on the driver 2322 * @req: request to dequeue 2323 * 2324 * Description: 2325 * Dequeue @req and start timeout timer on it. This hands off the 2326 * request to the driver. 2327 * 2328 * Block internal functions which don't want to start timer should 2329 * call blk_dequeue_request(). 2330 * 2331 * Context: 2332 * queue_lock must be held. 2333 */ 2334void blk_start_request(struct request *req) 2335{ 2336 blk_dequeue_request(req); 2337 2338 /* 2339 * We are now handing the request to the hardware, initialize 2340 * resid_len to full count and add the timeout handler. 2341 */ 2342 req->resid_len = blk_rq_bytes(req); 2343 if (unlikely(blk_bidi_rq(req))) 2344 req->next_rq->resid_len = blk_rq_bytes(req->next_rq); 2345 2346 BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags)); 2347 blk_add_timer(req); 2348} 2349EXPORT_SYMBOL(blk_start_request); 2350 2351/** 2352 * blk_fetch_request - fetch a request from a request queue 2353 * @q: request queue to fetch a request from 2354 * 2355 * Description: 2356 * Return the request at the top of @q. The request is started on 2357 * return and LLD can start processing it immediately. 2358 * 2359 * Return: 2360 * Pointer to the request at the top of @q if available. Null 2361 * otherwise. 2362 * 2363 * Context: 2364 * queue_lock must be held. 2365 */ 2366struct request *blk_fetch_request(struct request_queue *q) 2367{ 2368 struct request *rq; 2369 2370 rq = blk_peek_request(q); 2371 if (rq) 2372 blk_start_request(rq); 2373 return rq; 2374} 2375EXPORT_SYMBOL(blk_fetch_request); 2376 2377/** 2378 * blk_update_request - Special helper function for request stacking drivers 2379 * @req: the request being processed 2380 * @error: %0 for success, < %0 for error 2381 * @nr_bytes: number of bytes to complete @req 2382 * 2383 * Description: 2384 * Ends I/O on a number of bytes attached to @req, but doesn't complete 2385 * the request structure even if @req doesn't have leftover. 2386 * If @req has leftover, sets it up for the next range of segments. 2387 * 2388 * This special helper function is only for request stacking drivers 2389 * (e.g. request-based dm) so that they can handle partial completion. 2390 * Actual device drivers should use blk_end_request instead. 2391 * 2392 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees 2393 * %false return from this function. 2394 * 2395 * Return: 2396 * %false - this request doesn't have any more data 2397 * %true - this request has more data 2398 **/ 2399bool blk_update_request(struct request *req, int error, unsigned int nr_bytes) 2400{ 2401 int total_bytes; 2402 2403 trace_block_rq_complete(req->q, req, nr_bytes); 2404 2405 if (!req->bio) 2406 return false; 2407 2408 /* 2409 * For fs requests, rq is just carrier of independent bio's 2410 * and each partial completion should be handled separately. 2411 * Reset per-request error on each partial completion. 2412 * 2413 * TODO: tj: This is too subtle. It would be better to let 2414 * low level drivers do what they see fit. 2415 */ 2416 if (req->cmd_type == REQ_TYPE_FS) 2417 req->errors = 0; 2418 2419 if (error && req->cmd_type == REQ_TYPE_FS && 2420 !(req->cmd_flags & REQ_QUIET)) { 2421 char *error_type; 2422 2423 switch (error) { 2424 case -ENOLINK: 2425 error_type = "recoverable transport"; 2426 break; 2427 case -EREMOTEIO: 2428 error_type = "critical target"; 2429 break; 2430 case -EBADE: 2431 error_type = "critical nexus"; 2432 break; 2433 case -ETIMEDOUT: 2434 error_type = "timeout"; 2435 break; 2436 case -ENOSPC: 2437 error_type = "critical space allocation"; 2438 break; 2439 case -ENODATA: 2440 error_type = "critical medium"; 2441 break; 2442 case -EIO: 2443 default: 2444 error_type = "I/O"; 2445 break; 2446 } 2447 printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n", 2448 __func__, error_type, req->rq_disk ? 2449 req->rq_disk->disk_name : "?", 2450 (unsigned long long)blk_rq_pos(req)); 2451 2452 } 2453 2454 blk_account_io_completion(req, nr_bytes); 2455 2456 total_bytes = 0; 2457 while (req->bio) { 2458 struct bio *bio = req->bio; 2459 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); 2460 2461 if (bio_bytes == bio->bi_iter.bi_size) 2462 req->bio = bio->bi_next; 2463 2464 req_bio_endio(req, bio, bio_bytes, error); 2465 2466 total_bytes += bio_bytes; 2467 nr_bytes -= bio_bytes; 2468 2469 if (!nr_bytes) 2470 break; 2471 } 2472 2473 /* 2474 * completely done 2475 */ 2476 if (!req->bio) { 2477 /* 2478 * Reset counters so that the request stacking driver 2479 * can find how many bytes remain in the request 2480 * later. 2481 */ 2482 req->__data_len = 0; 2483 return false; 2484 } 2485 2486 req->__data_len -= total_bytes; 2487 2488 /* update sector only for requests with clear definition of sector */ 2489 if (req->cmd_type == REQ_TYPE_FS) 2490 req->__sector += total_bytes >> 9; 2491 2492 /* mixed attributes always follow the first bio */ 2493 if (req->cmd_flags & REQ_MIXED_MERGE) { 2494 req->cmd_flags &= ~REQ_FAILFAST_MASK; 2495 req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK; 2496 } 2497 2498 /* 2499 * If total number of sectors is less than the first segment 2500 * size, something has gone terribly wrong. 2501 */ 2502 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { 2503 blk_dump_rq_flags(req, "request botched"); 2504 req->__data_len = blk_rq_cur_bytes(req); 2505 } 2506 2507 /* recalculate the number of segments */ 2508 blk_recalc_rq_segments(req); 2509 2510 return true; 2511} 2512EXPORT_SYMBOL_GPL(blk_update_request); 2513 2514static bool blk_update_bidi_request(struct request *rq, int error, 2515 unsigned int nr_bytes, 2516 unsigned int bidi_bytes) 2517{ 2518 if (blk_update_request(rq, error, nr_bytes)) 2519 return true; 2520 2521 /* Bidi request must be completed as a whole */ 2522 if (unlikely(blk_bidi_rq(rq)) && 2523 blk_update_request(rq->next_rq, error, bidi_bytes)) 2524 return true; 2525 2526 if (blk_queue_add_random(rq->q)) 2527 add_disk_randomness(rq->rq_disk); 2528 2529 return false; 2530} 2531 2532/** 2533 * blk_unprep_request - unprepare a request 2534 * @req: the request 2535 * 2536 * This function makes a request ready for complete resubmission (or 2537 * completion). It happens only after all error handling is complete, 2538 * so represents the appropriate moment to deallocate any resources 2539 * that were allocated to the request in the prep_rq_fn. The queue 2540 * lock is held when calling this. 2541 */ 2542void blk_unprep_request(struct request *req) 2543{ 2544 struct request_queue *q = req->q; 2545 2546 req->cmd_flags &= ~REQ_DONTPREP; 2547 if (q->unprep_rq_fn) 2548 q->unprep_rq_fn(q, req); 2549} 2550EXPORT_SYMBOL_GPL(blk_unprep_request); 2551 2552/* 2553 * queue lock must be held 2554 */ 2555void blk_finish_request(struct request *req, int error) 2556{ 2557 if (blk_rq_tagged(req)) 2558 blk_queue_end_tag(req->q, req); 2559 2560 BUG_ON(blk_queued_rq(req)); 2561 2562 if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS) 2563 laptop_io_completion(&req->q->backing_dev_info); 2564 2565 blk_delete_timer(req); 2566 2567 if (req->cmd_flags & REQ_DONTPREP) 2568 blk_unprep_request(req); 2569 2570 blk_account_io_done(req); 2571 2572 if (req->end_io) 2573 req->end_io(req, error); 2574 else { 2575 if (blk_bidi_rq(req)) 2576 __blk_put_request(req->next_rq->q, req->next_rq); 2577 2578 __blk_put_request(req->q, req); 2579 } 2580} 2581EXPORT_SYMBOL(blk_finish_request); 2582 2583/** 2584 * blk_end_bidi_request - Complete a bidi request 2585 * @rq: the request to complete 2586 * @error: %0 for success, < %0 for error 2587 * @nr_bytes: number of bytes to complete @rq 2588 * @bidi_bytes: number of bytes to complete @rq->next_rq 2589 * 2590 * Description: 2591 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. 2592 * Drivers that supports bidi can safely call this member for any 2593 * type of request, bidi or uni. In the later case @bidi_bytes is 2594 * just ignored. 2595 * 2596 * Return: 2597 * %false - we are done with this request 2598 * %true - still buffers pending for this request 2599 **/ 2600static bool blk_end_bidi_request(struct request *rq, int error, 2601 unsigned int nr_bytes, unsigned int bidi_bytes) 2602{ 2603 struct request_queue *q = rq->q; 2604 unsigned long flags; 2605 2606 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) 2607 return true; 2608 2609 spin_lock_irqsave(q->queue_lock, flags); 2610 blk_finish_request(rq, error); 2611 spin_unlock_irqrestore(q->queue_lock, flags); 2612 2613 return false; 2614} 2615 2616/** 2617 * __blk_end_bidi_request - Complete a bidi request with queue lock held 2618 * @rq: the request to complete 2619 * @error: %0 for success, < %0 for error 2620 * @nr_bytes: number of bytes to complete @rq 2621 * @bidi_bytes: number of bytes to complete @rq->next_rq 2622 * 2623 * Description: 2624 * Identical to blk_end_bidi_request() except that queue lock is 2625 * assumed to be locked on entry and remains so on return. 2626 * 2627 * Return: 2628 * %false - we are done with this request 2629 * %true - still buffers pending for this request 2630 **/ 2631bool __blk_end_bidi_request(struct request *rq, int error, 2632 unsigned int nr_bytes, unsigned int bidi_bytes) 2633{ 2634 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) 2635 return true; 2636 2637 blk_finish_request(rq, error); 2638 2639 return false; 2640} 2641 2642/** 2643 * blk_end_request - Helper function for drivers to complete the request. 2644 * @rq: the request being processed 2645 * @error: %0 for success, < %0 for error 2646 * @nr_bytes: number of bytes to complete 2647 * 2648 * Description: 2649 * Ends I/O on a number of bytes attached to @rq. 2650 * If @rq has leftover, sets it up for the next range of segments. 2651 * 2652 * Return: 2653 * %false - we are done with this request 2654 * %true - still buffers pending for this request 2655 **/ 2656bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes) 2657{ 2658 return blk_end_bidi_request(rq, error, nr_bytes, 0); 2659} 2660EXPORT_SYMBOL(blk_end_request); 2661 2662/** 2663 * blk_end_request_all - Helper function for drives to finish the request. 2664 * @rq: the request to finish 2665 * @error: %0 for success, < %0 for error 2666 * 2667 * Description: 2668 * Completely finish @rq. 2669 */ 2670void blk_end_request_all(struct request *rq, int error) 2671{ 2672 bool pending; 2673 unsigned int bidi_bytes = 0; 2674 2675 if (unlikely(blk_bidi_rq(rq))) 2676 bidi_bytes = blk_rq_bytes(rq->next_rq); 2677 2678 pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); 2679 BUG_ON(pending); 2680} 2681EXPORT_SYMBOL(blk_end_request_all); 2682 2683/** 2684 * blk_end_request_cur - Helper function to finish the current request chunk. 2685 * @rq: the request to finish the current chunk for 2686 * @error: %0 for success, < %0 for error 2687 * 2688 * Description: 2689 * Complete the current consecutively mapped chunk from @rq. 2690 * 2691 * Return: 2692 * %false - we are done with this request 2693 * %true - still buffers pending for this request 2694 */ 2695bool blk_end_request_cur(struct request *rq, int error) 2696{ 2697 return blk_end_request(rq, error, blk_rq_cur_bytes(rq)); 2698} 2699EXPORT_SYMBOL(blk_end_request_cur); 2700 2701/** 2702 * blk_end_request_err - Finish a request till the next failure boundary. 2703 * @rq: the request to finish till the next failure boundary for 2704 * @error: must be negative errno 2705 * 2706 * Description: 2707 * Complete @rq till the next failure boundary. 2708 * 2709 * Return: 2710 * %false - we are done with this request 2711 * %true - still buffers pending for this request 2712 */ 2713bool blk_end_request_err(struct request *rq, int error) 2714{ 2715 WARN_ON(error >= 0); 2716 return blk_end_request(rq, error, blk_rq_err_bytes(rq)); 2717} 2718EXPORT_SYMBOL_GPL(blk_end_request_err); 2719 2720/** 2721 * __blk_end_request - Helper function for drivers to complete the request. 2722 * @rq: the request being processed 2723 * @error: %0 for success, < %0 for error 2724 * @nr_bytes: number of bytes to complete 2725 * 2726 * Description: 2727 * Must be called with queue lock held unlike blk_end_request(). 2728 * 2729 * Return: 2730 * %false - we are done with this request 2731 * %true - still buffers pending for this request 2732 **/ 2733bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes) 2734{ 2735 return __blk_end_bidi_request(rq, error, nr_bytes, 0); 2736} 2737EXPORT_SYMBOL(__blk_end_request); 2738 2739/** 2740 * __blk_end_request_all - Helper function for drives to finish the request. 2741 * @rq: the request to finish 2742 * @error: %0 for success, < %0 for error 2743 * 2744 * Description: 2745 * Completely finish @rq. Must be called with queue lock held. 2746 */ 2747void __blk_end_request_all(struct request *rq, int error) 2748{ 2749 bool pending; 2750 unsigned int bidi_bytes = 0; 2751 2752 if (unlikely(blk_bidi_rq(rq))) 2753 bidi_bytes = blk_rq_bytes(rq->next_rq); 2754 2755 pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); 2756 BUG_ON(pending); 2757} 2758EXPORT_SYMBOL(__blk_end_request_all); 2759 2760/** 2761 * __blk_end_request_cur - Helper function to finish the current request chunk. 2762 * @rq: the request to finish the current chunk for 2763 * @error: %0 for success, < %0 for error 2764 * 2765 * Description: 2766 * Complete the current consecutively mapped chunk from @rq. Must 2767 * be called with queue lock held. 2768 * 2769 * Return: 2770 * %false - we are done with this request 2771 * %true - still buffers pending for this request 2772 */ 2773bool __blk_end_request_cur(struct request *rq, int error) 2774{ 2775 return __blk_end_request(rq, error, blk_rq_cur_bytes(rq)); 2776} 2777EXPORT_SYMBOL(__blk_end_request_cur); 2778 2779/** 2780 * __blk_end_request_err - Finish a request till the next failure boundary. 2781 * @rq: the request to finish till the next failure boundary for 2782 * @error: must be negative errno 2783 * 2784 * Description: 2785 * Complete @rq till the next failure boundary. Must be called 2786 * with queue lock held. 2787 * 2788 * Return: 2789 * %false - we are done with this request 2790 * %true - still buffers pending for this request 2791 */ 2792bool __blk_end_request_err(struct request *rq, int error) 2793{ 2794 WARN_ON(error >= 0); 2795 return __blk_end_request(rq, error, blk_rq_err_bytes(rq)); 2796} 2797EXPORT_SYMBOL_GPL(__blk_end_request_err); 2798 2799void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 2800 struct bio *bio) 2801{ 2802 /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */ 2803 rq->cmd_flags |= bio->bi_rw & REQ_WRITE; 2804 2805 if (bio_has_data(bio)) 2806 rq->nr_phys_segments = bio_phys_segments(q, bio); 2807 2808 rq->__data_len = bio->bi_iter.bi_size; 2809 rq->bio = rq->biotail = bio; 2810 2811 if (bio->bi_bdev) 2812 rq->rq_disk = bio->bi_bdev->bd_disk; 2813} 2814 2815#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 2816/** 2817 * rq_flush_dcache_pages - Helper function to flush all pages in a request 2818 * @rq: the request to be flushed 2819 * 2820 * Description: 2821 * Flush all pages in @rq. 2822 */ 2823void rq_flush_dcache_pages(struct request *rq) 2824{ 2825 struct req_iterator iter; 2826 struct bio_vec bvec; 2827 2828 rq_for_each_segment(bvec, rq, iter) 2829 flush_dcache_page(bvec.bv_page); 2830} 2831EXPORT_SYMBOL_GPL(rq_flush_dcache_pages); 2832#endif 2833 2834/** 2835 * blk_lld_busy - Check if underlying low-level drivers of a device are busy 2836 * @q : the queue of the device being checked 2837 * 2838 * Description: 2839 * Check if underlying low-level drivers of a device are busy. 2840 * If the drivers want to export their busy state, they must set own 2841 * exporting function using blk_queue_lld_busy() first. 2842 * 2843 * Basically, this function is used only by request stacking drivers 2844 * to stop dispatching requests to underlying devices when underlying 2845 * devices are busy. This behavior helps more I/O merging on the queue 2846 * of the request stacking driver and prevents I/O throughput regression 2847 * on burst I/O load. 2848 * 2849 * Return: 2850 * 0 - Not busy (The request stacking driver should dispatch request) 2851 * 1 - Busy (The request stacking driver should stop dispatching request) 2852 */ 2853int blk_lld_busy(struct request_queue *q) 2854{ 2855 if (q->lld_busy_fn) 2856 return q->lld_busy_fn(q); 2857 2858 return 0; 2859} 2860EXPORT_SYMBOL_GPL(blk_lld_busy); 2861 2862/** 2863 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request 2864 * @rq: the clone request to be cleaned up 2865 * 2866 * Description: 2867 * Free all bios in @rq for a cloned request. 2868 */ 2869void blk_rq_unprep_clone(struct request *rq) 2870{ 2871 struct bio *bio; 2872 2873 while ((bio = rq->bio) != NULL) { 2874 rq->bio = bio->bi_next; 2875 2876 bio_put(bio); 2877 } 2878} 2879EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); 2880 2881/* 2882 * Copy attributes of the original request to the clone request. 2883 * The actual data parts (e.g. ->cmd, ->sense) are not copied. 2884 */ 2885static void __blk_rq_prep_clone(struct request *dst, struct request *src) 2886{ 2887 dst->cpu = src->cpu; 2888 dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE; 2889 dst->cmd_type = src->cmd_type; 2890 dst->__sector = blk_rq_pos(src); 2891 dst->__data_len = blk_rq_bytes(src); 2892 dst->nr_phys_segments = src->nr_phys_segments; 2893 dst->ioprio = src->ioprio; 2894 dst->extra_len = src->extra_len; 2895} 2896 2897/** 2898 * blk_rq_prep_clone - Helper function to setup clone request 2899 * @rq: the request to be setup 2900 * @rq_src: original request to be cloned 2901 * @bs: bio_set that bios for clone are allocated from 2902 * @gfp_mask: memory allocation mask for bio 2903 * @bio_ctr: setup function to be called for each clone bio. 2904 * Returns %0 for success, non %0 for failure. 2905 * @data: private data to be passed to @bio_ctr 2906 * 2907 * Description: 2908 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. 2909 * The actual data parts of @rq_src (e.g. ->cmd, ->sense) 2910 * are not copied, and copying such parts is the caller's responsibility. 2911 * Also, pages which the original bios are pointing to are not copied 2912 * and the cloned bios just point same pages. 2913 * So cloned bios must be completed before original bios, which means 2914 * the caller must complete @rq before @rq_src. 2915 */ 2916int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 2917 struct bio_set *bs, gfp_t gfp_mask, 2918 int (*bio_ctr)(struct bio *, struct bio *, void *), 2919 void *data) 2920{ 2921 struct bio *bio, *bio_src; 2922 2923 if (!bs) 2924 bs = fs_bio_set; 2925 2926 blk_rq_init(NULL, rq); 2927 2928 __rq_for_each_bio(bio_src, rq_src) { 2929 bio = bio_clone_fast(bio_src, gfp_mask, bs); 2930 if (!bio) 2931 goto free_and_out; 2932 2933 if (bio_ctr && bio_ctr(bio, bio_src, data)) 2934 goto free_and_out; 2935 2936 if (rq->bio) { 2937 rq->biotail->bi_next = bio; 2938 rq->biotail = bio; 2939 } else 2940 rq->bio = rq->biotail = bio; 2941 } 2942 2943 __blk_rq_prep_clone(rq, rq_src); 2944 2945 return 0; 2946 2947free_and_out: 2948 if (bio) 2949 bio_put(bio); 2950 blk_rq_unprep_clone(rq); 2951 2952 return -ENOMEM; 2953} 2954EXPORT_SYMBOL_GPL(blk_rq_prep_clone); 2955 2956int kblockd_schedule_work(struct work_struct *work) 2957{ 2958 return queue_work(kblockd_workqueue, work); 2959} 2960EXPORT_SYMBOL(kblockd_schedule_work); 2961 2962int kblockd_schedule_delayed_work(struct delayed_work *dwork, 2963 unsigned long delay) 2964{ 2965 return queue_delayed_work(kblockd_workqueue, dwork, delay); 2966} 2967EXPORT_SYMBOL(kblockd_schedule_delayed_work); 2968 2969int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork, 2970 unsigned long delay) 2971{ 2972 return queue_delayed_work_on(cpu, kblockd_workqueue, dwork, delay); 2973} 2974EXPORT_SYMBOL(kblockd_schedule_delayed_work_on); 2975 2976/** 2977 * blk_start_plug - initialize blk_plug and track it inside the task_struct 2978 * @plug: The &struct blk_plug that needs to be initialized 2979 * 2980 * Description: 2981 * Tracking blk_plug inside the task_struct will help with auto-flushing the 2982 * pending I/O should the task end up blocking between blk_start_plug() and 2983 * blk_finish_plug(). This is important from a performance perspective, but 2984 * also ensures that we don't deadlock. For instance, if the task is blocking 2985 * for a memory allocation, memory reclaim could end up wanting to free a 2986 * page belonging to that request that is currently residing in our private 2987 * plug. By flushing the pending I/O when the process goes to sleep, we avoid 2988 * this kind of deadlock. 2989 */ 2990void blk_start_plug(struct blk_plug *plug) 2991{ 2992 struct task_struct *tsk = current; 2993 2994 INIT_LIST_HEAD(&plug->list); 2995 INIT_LIST_HEAD(&plug->mq_list); 2996 INIT_LIST_HEAD(&plug->cb_list); 2997 2998 /* 2999 * If this is a nested plug, don't actually assign it. It will be 3000 * flushed on its own. 3001 */ 3002 if (!tsk->plug) { 3003 /* 3004 * Store ordering should not be needed here, since a potential 3005 * preempt will imply a full memory barrier 3006 */ 3007 tsk->plug = plug; 3008 } 3009} 3010EXPORT_SYMBOL(blk_start_plug); 3011 3012static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) 3013{ 3014 struct request *rqa = container_of(a, struct request, queuelist); 3015 struct request *rqb = container_of(b, struct request, queuelist); 3016 3017 return !(rqa->q < rqb->q || 3018 (rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb))); 3019} 3020 3021/* 3022 * If 'from_schedule' is true, then postpone the dispatch of requests 3023 * until a safe kblockd context. We due this to avoid accidental big 3024 * additional stack usage in driver dispatch, in places where the originally 3025 * plugger did not intend it. 3026 */ 3027static void queue_unplugged(struct request_queue *q, unsigned int depth, 3028 bool from_schedule) 3029 __releases(q->queue_lock) 3030{ 3031 trace_block_unplug(q, depth, !from_schedule); 3032 3033 if (from_schedule) 3034 blk_run_queue_async(q); 3035 else 3036 __blk_run_queue(q); 3037 spin_unlock(q->queue_lock); 3038} 3039 3040static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule) 3041{ 3042 LIST_HEAD(callbacks); 3043 3044 while (!list_empty(&plug->cb_list)) { 3045 list_splice_init(&plug->cb_list, &callbacks); 3046 3047 while (!list_empty(&callbacks)) { 3048 struct blk_plug_cb *cb = list_first_entry(&callbacks, 3049 struct blk_plug_cb, 3050 list); 3051 list_del(&cb->list); 3052 cb->callback(cb, from_schedule); 3053 } 3054 } 3055} 3056 3057struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data, 3058 int size) 3059{ 3060 struct blk_plug *plug = current->plug; 3061 struct blk_plug_cb *cb; 3062 3063 if (!plug) 3064 return NULL; 3065 3066 list_for_each_entry(cb, &plug->cb_list, list) 3067 if (cb->callback == unplug && cb->data == data) 3068 return cb; 3069 3070 /* Not currently on the callback list */ 3071 BUG_ON(size < sizeof(*cb)); 3072 cb = kzalloc(size, GFP_ATOMIC); 3073 if (cb) { 3074 cb->data = data; 3075 cb->callback = unplug; 3076 list_add(&cb->list, &plug->cb_list); 3077 } 3078 return cb; 3079} 3080EXPORT_SYMBOL(blk_check_plugged); 3081 3082void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) 3083{ 3084 struct request_queue *q; 3085 unsigned long flags; 3086 struct request *rq; 3087 LIST_HEAD(list); 3088 unsigned int depth; 3089 3090 flush_plug_callbacks(plug, from_schedule); 3091 3092 if (!list_empty(&plug->mq_list)) 3093 blk_mq_flush_plug_list(plug, from_schedule); 3094 3095 if (list_empty(&plug->list)) 3096 return; 3097 3098 list_splice_init(&plug->list, &list); 3099 3100 list_sort(NULL, &list, plug_rq_cmp); 3101 3102 q = NULL; 3103 depth = 0; 3104 3105 /* 3106 * Save and disable interrupts here, to avoid doing it for every 3107 * queue lock we have to take. 3108 */ 3109 local_irq_save(flags); 3110 while (!list_empty(&list)) { 3111 rq = list_entry_rq(list.next); 3112 list_del_init(&rq->queuelist); 3113 BUG_ON(!rq->q); 3114 if (rq->q != q) { 3115 /* 3116 * This drops the queue lock 3117 */ 3118 if (q) 3119 queue_unplugged(q, depth, from_schedule); 3120 q = rq->q; 3121 depth = 0; 3122 spin_lock(q->queue_lock); 3123 } 3124 3125 /* 3126 * Short-circuit if @q is dead 3127 */ 3128 if (unlikely(blk_queue_dying(q))) { 3129 __blk_end_request_all(rq, -ENODEV); 3130 continue; 3131 } 3132 3133 /* 3134 * rq is already accounted, so use raw insert 3135 */ 3136 if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) 3137 __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH); 3138 else 3139 __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE); 3140 3141 depth++; 3142 } 3143 3144 /* 3145 * This drops the queue lock 3146 */ 3147 if (q) 3148 queue_unplugged(q, depth, from_schedule); 3149 3150 local_irq_restore(flags); 3151} 3152 3153void blk_finish_plug(struct blk_plug *plug) 3154{ 3155 blk_flush_plug_list(plug, false); 3156 3157 if (plug == current->plug) 3158 current->plug = NULL; 3159} 3160EXPORT_SYMBOL(blk_finish_plug); 3161 3162#ifdef CONFIG_PM_RUNTIME 3163/** 3164 * blk_pm_runtime_init - Block layer runtime PM initialization routine 3165 * @q: the queue of the device 3166 * @dev: the device the queue belongs to 3167 * 3168 * Description: 3169 * Initialize runtime-PM-related fields for @q and start auto suspend for 3170 * @dev. Drivers that want to take advantage of request-based runtime PM 3171 * should call this function after @dev has been initialized, and its 3172 * request queue @q has been allocated, and runtime PM for it can not happen 3173 * yet(either due to disabled/forbidden or its usage_count > 0). In most 3174 * cases, driver should call this function before any I/O has taken place. 3175 * 3176 * This function takes care of setting up using auto suspend for the device, 3177 * the autosuspend delay is set to -1 to make runtime suspend impossible 3178 * until an updated value is either set by user or by driver. Drivers do 3179 * not need to touch other autosuspend settings. 3180 * 3181 * The block layer runtime PM is request based, so only works for drivers 3182 * that use request as their IO unit instead of those directly use bio's. 3183 */ 3184void blk_pm_runtime_init(struct request_queue *q, struct device *dev) 3185{ 3186 q->dev = dev; 3187 q->rpm_status = RPM_ACTIVE; 3188 pm_runtime_set_autosuspend_delay(q->dev, -1); 3189 pm_runtime_use_autosuspend(q->dev); 3190} 3191EXPORT_SYMBOL(blk_pm_runtime_init); 3192 3193/** 3194 * blk_pre_runtime_suspend - Pre runtime suspend check 3195 * @q: the queue of the device 3196 * 3197 * Description: 3198 * This function will check if runtime suspend is allowed for the device 3199 * by examining if there are any requests pending in the queue. If there 3200 * are requests pending, the device can not be runtime suspended; otherwise, 3201 * the queue's status will be updated to SUSPENDING and the driver can 3202 * proceed to suspend the device. 3203 * 3204 * For the not allowed case, we mark last busy for the device so that 3205 * runtime PM core will try to autosuspend it some time later. 3206 * 3207 * This function should be called near the start of the device's 3208 * runtime_suspend callback. 3209 * 3210 * Return: 3211 * 0 - OK to runtime suspend the device 3212 * -EBUSY - Device should not be runtime suspended 3213 */ 3214int blk_pre_runtime_suspend(struct request_queue *q) 3215{ 3216 int ret = 0; 3217 3218 spin_lock_irq(q->queue_lock); 3219 if (q->nr_pending) { 3220 ret = -EBUSY; 3221 pm_runtime_mark_last_busy(q->dev); 3222 } else { 3223 q->rpm_status = RPM_SUSPENDING; 3224 } 3225 spin_unlock_irq(q->queue_lock); 3226 return ret; 3227} 3228EXPORT_SYMBOL(blk_pre_runtime_suspend); 3229 3230/** 3231 * blk_post_runtime_suspend - Post runtime suspend processing 3232 * @q: the queue of the device 3233 * @err: return value of the device's runtime_suspend function 3234 * 3235 * Description: 3236 * Update the queue's runtime status according to the return value of the 3237 * device's runtime suspend function and mark last busy for the device so 3238 * that PM core will try to auto suspend the device at a later time. 3239 * 3240 * This function should be called near the end of the device's 3241 * runtime_suspend callback. 3242 */ 3243void blk_post_runtime_suspend(struct request_queue *q, int err) 3244{ 3245 spin_lock_irq(q->queue_lock); 3246 if (!err) { 3247 q->rpm_status = RPM_SUSPENDED; 3248 } else { 3249 q->rpm_status = RPM_ACTIVE; 3250 pm_runtime_mark_last_busy(q->dev); 3251 } 3252 spin_unlock_irq(q->queue_lock); 3253} 3254EXPORT_SYMBOL(blk_post_runtime_suspend); 3255 3256/** 3257 * blk_pre_runtime_resume - Pre runtime resume processing 3258 * @q: the queue of the device 3259 * 3260 * Description: 3261 * Update the queue's runtime status to RESUMING in preparation for the 3262 * runtime resume of the device. 3263 * 3264 * This function should be called near the start of the device's 3265 * runtime_resume callback. 3266 */ 3267void blk_pre_runtime_resume(struct request_queue *q) 3268{ 3269 spin_lock_irq(q->queue_lock); 3270 q->rpm_status = RPM_RESUMING; 3271 spin_unlock_irq(q->queue_lock); 3272} 3273EXPORT_SYMBOL(blk_pre_runtime_resume); 3274 3275/** 3276 * blk_post_runtime_resume - Post runtime resume processing 3277 * @q: the queue of the device 3278 * @err: return value of the device's runtime_resume function 3279 * 3280 * Description: 3281 * Update the queue's runtime status according to the return value of the 3282 * device's runtime_resume function. If it is successfully resumed, process 3283 * the requests that are queued into the device's queue when it is resuming 3284 * and then mark last busy and initiate autosuspend for it. 3285 * 3286 * This function should be called near the end of the device's 3287 * runtime_resume callback. 3288 */ 3289void blk_post_runtime_resume(struct request_queue *q, int err) 3290{ 3291 spin_lock_irq(q->queue_lock); 3292 if (!err) { 3293 q->rpm_status = RPM_ACTIVE; 3294 __blk_run_queue(q); 3295 pm_runtime_mark_last_busy(q->dev); 3296 pm_request_autosuspend(q->dev); 3297 } else { 3298 q->rpm_status = RPM_SUSPENDED; 3299 } 3300 spin_unlock_irq(q->queue_lock); 3301} 3302EXPORT_SYMBOL(blk_post_runtime_resume); 3303#endif 3304 3305int __init blk_dev_init(void) 3306{ 3307 BUILD_BUG_ON(__REQ_NR_BITS > 8 * 3308 sizeof(((struct request *)0)->cmd_flags)); 3309 3310 /* used for unplugging and affects IO latency/throughput - HIGHPRI */ 3311 kblockd_workqueue = alloc_workqueue("kblockd", 3312 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 3313 if (!kblockd_workqueue) 3314 panic("Failed to create kblockd\n"); 3315 3316 request_cachep = kmem_cache_create("blkdev_requests", 3317 sizeof(struct request), 0, SLAB_PANIC, NULL); 3318 3319 blk_requestq_cachep = kmem_cache_create("blkdev_queue", 3320 sizeof(struct request_queue), 0, SLAB_PANIC, NULL); 3321 3322 return 0; 3323} 3324