scsi_lib.c revision 7c72ce81870ded9365f4bc5caa98ef1591dd18dd
15a23b0762c9095e137ce9a559cc7c37b2f8fd083Igor M. Liplianin/* 25a23b0762c9095e137ce9a559cc7c37b2f8fd083Igor M. Liplianin * scsi_lib.c Copyright (C) 1999 Eric Youngdale 3e5514f104d875b3d28cbcd5d4f2b96ab2fca1e29Andy Walls * 4e5514f104d875b3d28cbcd5d4f2b96ab2fca1e29Andy Walls * SCSI queueing library. 5e5514f104d875b3d28cbcd5d4f2b96ab2fca1e29Andy Walls * Initial versions: Eric Youngdale (eric@andante.org). 6d19770e5178a4bc49641711246360c25781d20a4Steven Toth * Based upon conversations with large numbers 7d19770e5178a4bc49641711246360c25781d20a4Steven Toth * of people at Linux Expo. 8d19770e5178a4bc49641711246360c25781d20a4Steven Toth */ 9d19770e5178a4bc49641711246360c25781d20a4Steven Toth 107c91f0624a9a2b8b9b122cf94fef34bc7f7347a6Mauro Carvalho Chehab#include <linux/bio.h> 11d19770e5178a4bc49641711246360c25781d20a4Steven Toth#include <linux/blkdev.h> 12d19770e5178a4bc49641711246360c25781d20a4Steven Toth#include <linux/completion.h> 13d19770e5178a4bc49641711246360c25781d20a4Steven Toth#include <linux/kernel.h> 14d19770e5178a4bc49641711246360c25781d20a4Steven Toth#include <linux/mempool.h> 15#include <linux/slab.h> 16#include <linux/init.h> 17#include <linux/pci.h> 18#include <linux/delay.h> 19 20#include <scsi/scsi.h> 21#include <scsi/scsi_dbg.h> 22#include <scsi/scsi_device.h> 23#include <scsi/scsi_driver.h> 24#include <scsi/scsi_eh.h> 25#include <scsi/scsi_host.h> 26#include <scsi/scsi_request.h> 27 28#include "scsi_priv.h" 29#include "scsi_logging.h" 30 31 32#define SG_MEMPOOL_NR (sizeof(scsi_sg_pools)/sizeof(struct scsi_host_sg_pool)) 33#define SG_MEMPOOL_SIZE 32 34 35struct scsi_host_sg_pool { 36 size_t size; 37 char *name; 38 kmem_cache_t *slab; 39 mempool_t *pool; 40}; 41 42#if (SCSI_MAX_PHYS_SEGMENTS < 32) 43#error SCSI_MAX_PHYS_SEGMENTS is too small 44#endif 45 46#define SP(x) { x, "sgpool-" #x } 47static struct scsi_host_sg_pool scsi_sg_pools[] = { 48 SP(8), 49 SP(16), 50 SP(32), 51#if (SCSI_MAX_PHYS_SEGMENTS > 32) 52 SP(64), 53#if (SCSI_MAX_PHYS_SEGMENTS > 64) 54 SP(128), 55#if (SCSI_MAX_PHYS_SEGMENTS > 128) 56 SP(256), 57#if (SCSI_MAX_PHYS_SEGMENTS > 256) 58#error SCSI_MAX_PHYS_SEGMENTS is too large 59#endif 60#endif 61#endif 62#endif 63}; 64#undef SP 65 66 67/* 68 * Function: scsi_insert_special_req() 69 * 70 * Purpose: Insert pre-formed request into request queue. 71 * 72 * Arguments: sreq - request that is ready to be queued. 73 * at_head - boolean. True if we should insert at head 74 * of queue, false if we should insert at tail. 75 * 76 * Lock status: Assumed that lock is not held upon entry. 77 * 78 * Returns: Nothing 79 * 80 * Notes: This function is called from character device and from 81 * ioctl types of functions where the caller knows exactly 82 * what SCSI command needs to be issued. The idea is that 83 * we merely inject the command into the queue (at the head 84 * for now), and then call the queue request function to actually 85 * process it. 86 */ 87int scsi_insert_special_req(struct scsi_request *sreq, int at_head) 88{ 89 /* 90 * Because users of this function are apt to reuse requests with no 91 * modification, we have to sanitise the request flags here 92 */ 93 sreq->sr_request->flags &= ~REQ_DONTPREP; 94 blk_insert_request(sreq->sr_device->request_queue, sreq->sr_request, 95 at_head, sreq); 96 return 0; 97} 98 99static void scsi_run_queue(struct request_queue *q); 100 101/* 102 * Function: scsi_unprep_request() 103 * 104 * Purpose: Remove all preparation done for a request, including its 105 * associated scsi_cmnd, so that it can be requeued. 106 * 107 * Arguments: req - request to unprepare 108 * 109 * Lock status: Assumed that no locks are held upon entry. 110 * 111 * Returns: Nothing. 112 */ 113static void scsi_unprep_request(struct request *req) 114{ 115 struct scsi_cmnd *cmd = req->special; 116 117 req->flags &= ~REQ_DONTPREP; 118 req->special = (req->flags & REQ_SPECIAL) ? cmd->sc_request : NULL; 119 120 scsi_put_command(cmd); 121} 122 123/* 124 * Function: scsi_queue_insert() 125 * 126 * Purpose: Insert a command in the midlevel queue. 127 * 128 * Arguments: cmd - command that we are adding to queue. 129 * reason - why we are inserting command to queue. 130 * 131 * Lock status: Assumed that lock is not held upon entry. 132 * 133 * Returns: Nothing. 134 * 135 * Notes: We do this for one of two cases. Either the host is busy 136 * and it cannot accept any more commands for the time being, 137 * or the device returned QUEUE_FULL and can accept no more 138 * commands. 139 * Notes: This could be called either from an interrupt context or a 140 * normal process context. 141 */ 142int scsi_queue_insert(struct scsi_cmnd *cmd, int reason) 143{ 144 struct Scsi_Host *host = cmd->device->host; 145 struct scsi_device *device = cmd->device; 146 struct request_queue *q = device->request_queue; 147 unsigned long flags; 148 149 SCSI_LOG_MLQUEUE(1, 150 printk("Inserting command %p into mlqueue\n", cmd)); 151 152 /* 153 * Set the appropriate busy bit for the device/host. 154 * 155 * If the host/device isn't busy, assume that something actually 156 * completed, and that we should be able to queue a command now. 157 * 158 * Note that the prior mid-layer assumption that any host could 159 * always queue at least one command is now broken. The mid-layer 160 * will implement a user specifiable stall (see 161 * scsi_host.max_host_blocked and scsi_device.max_device_blocked) 162 * if a command is requeued with no other commands outstanding 163 * either for the device or for the host. 164 */ 165 if (reason == SCSI_MLQUEUE_HOST_BUSY) 166 host->host_blocked = host->max_host_blocked; 167 else if (reason == SCSI_MLQUEUE_DEVICE_BUSY) 168 device->device_blocked = device->max_device_blocked; 169 170 /* 171 * Decrement the counters, since these commands are no longer 172 * active on the host/device. 173 */ 174 scsi_device_unbusy(device); 175 176 /* 177 * Requeue this command. It will go before all other commands 178 * that are already in the queue. 179 * 180 * NOTE: there is magic here about the way the queue is plugged if 181 * we have no outstanding commands. 182 * 183 * Although we *don't* plug the queue, we call the request 184 * function. The SCSI request function detects the blocked condition 185 * and plugs the queue appropriately. 186 */ 187 spin_lock_irqsave(q->queue_lock, flags); 188 blk_requeue_request(q, cmd->request); 189 spin_unlock_irqrestore(q->queue_lock, flags); 190 191 scsi_run_queue(q); 192 193 return 0; 194} 195 196/* 197 * Function: scsi_do_req 198 * 199 * Purpose: Queue a SCSI request 200 * 201 * Arguments: sreq - command descriptor. 202 * cmnd - actual SCSI command to be performed. 203 * buffer - data buffer. 204 * bufflen - size of data buffer. 205 * done - completion function to be run. 206 * timeout - how long to let it run before timeout. 207 * retries - number of retries we allow. 208 * 209 * Lock status: No locks held upon entry. 210 * 211 * Returns: Nothing. 212 * 213 * Notes: This function is only used for queueing requests for things 214 * like ioctls and character device requests - this is because 215 * we essentially just inject a request into the queue for the 216 * device. 217 * 218 * In order to support the scsi_device_quiesce function, we 219 * now inject requests on the *head* of the device queue 220 * rather than the tail. 221 */ 222void scsi_do_req(struct scsi_request *sreq, const void *cmnd, 223 void *buffer, unsigned bufflen, 224 void (*done)(struct scsi_cmnd *), 225 int timeout, int retries) 226{ 227 /* 228 * If the upper level driver is reusing these things, then 229 * we should release the low-level block now. Another one will 230 * be allocated later when this request is getting queued. 231 */ 232 __scsi_release_request(sreq); 233 234 /* 235 * Our own function scsi_done (which marks the host as not busy, 236 * disables the timeout counter, etc) will be called by us or by the 237 * scsi_hosts[host].queuecommand() function needs to also call 238 * the completion function for the high level driver. 239 */ 240 memcpy(sreq->sr_cmnd, cmnd, sizeof(sreq->sr_cmnd)); 241 sreq->sr_bufflen = bufflen; 242 sreq->sr_buffer = buffer; 243 sreq->sr_allowed = retries; 244 sreq->sr_done = done; 245 sreq->sr_timeout_per_command = timeout; 246 247 if (sreq->sr_cmd_len == 0) 248 sreq->sr_cmd_len = COMMAND_SIZE(sreq->sr_cmnd[0]); 249 250 /* 251 * head injection *required* here otherwise quiesce won't work 252 */ 253 scsi_insert_special_req(sreq, 1); 254} 255EXPORT_SYMBOL(scsi_do_req); 256 257/* This is the end routine we get to if a command was never attached 258 * to the request. Simply complete the request without changing 259 * rq_status; this will cause a DRIVER_ERROR. */ 260static void scsi_wait_req_end_io(struct request *req) 261{ 262 BUG_ON(!req->waiting); 263 264 complete(req->waiting); 265} 266 267void scsi_wait_req(struct scsi_request *sreq, const void *cmnd, void *buffer, 268 unsigned bufflen, int timeout, int retries) 269{ 270 DECLARE_COMPLETION(wait); 271 int write = (sreq->sr_data_direction == DMA_TO_DEVICE); 272 struct request *req; 273 274 req = blk_get_request(sreq->sr_device->request_queue, write, 275 __GFP_WAIT); 276 if (bufflen && blk_rq_map_kern(sreq->sr_device->request_queue, req, 277 buffer, bufflen, __GFP_WAIT)) { 278 sreq->sr_result = DRIVER_ERROR << 24; 279 blk_put_request(req); 280 return; 281 } 282 283 req->flags |= REQ_NOMERGE; 284 req->waiting = &wait; 285 req->end_io = scsi_wait_req_end_io; 286 req->cmd_len = COMMAND_SIZE(((u8 *)cmnd)[0]); 287 req->sense = sreq->sr_sense_buffer; 288 req->sense_len = 0; 289 memcpy(req->cmd, cmnd, req->cmd_len); 290 req->timeout = timeout; 291 req->flags |= REQ_BLOCK_PC; 292 req->rq_disk = NULL; 293 blk_insert_request(sreq->sr_device->request_queue, req, 294 sreq->sr_data_direction == DMA_TO_DEVICE, NULL); 295 wait_for_completion(&wait); 296 sreq->sr_request->waiting = NULL; 297 sreq->sr_result = req->errors; 298 if (req->errors) 299 sreq->sr_result |= (DRIVER_ERROR << 24); 300 301 blk_put_request(req); 302} 303 304EXPORT_SYMBOL(scsi_wait_req); 305 306/** 307 * scsi_execute - insert request and wait for the result 308 * @sdev: scsi device 309 * @cmd: scsi command 310 * @data_direction: data direction 311 * @buffer: data buffer 312 * @bufflen: len of buffer 313 * @sense: optional sense buffer 314 * @timeout: request timeout in seconds 315 * @retries: number of times to retry request 316 * @flags: or into request flags; 317 * 318 * returns the req->errors value which is the the scsi_cmnd result 319 * field. 320 **/ 321int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, 322 int data_direction, void *buffer, unsigned bufflen, 323 unsigned char *sense, int timeout, int retries, int flags) 324{ 325 struct request *req; 326 int write = (data_direction == DMA_TO_DEVICE); 327 int ret = DRIVER_ERROR << 24; 328 329 req = blk_get_request(sdev->request_queue, write, __GFP_WAIT); 330 331 if (bufflen && blk_rq_map_kern(sdev->request_queue, req, 332 buffer, bufflen, __GFP_WAIT)) 333 goto out; 334 335 req->cmd_len = COMMAND_SIZE(cmd[0]); 336 memcpy(req->cmd, cmd, req->cmd_len); 337 req->sense = sense; 338 req->sense_len = 0; 339 req->timeout = timeout; 340 req->flags |= flags | REQ_BLOCK_PC | REQ_SPECIAL | REQ_QUIET; 341 342 /* 343 * head injection *required* here otherwise quiesce won't work 344 */ 345 blk_execute_rq(req->q, NULL, req, 1); 346 347 ret = req->errors; 348 out: 349 blk_put_request(req); 350 351 return ret; 352} 353EXPORT_SYMBOL(scsi_execute); 354 355 356int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd, 357 int data_direction, void *buffer, unsigned bufflen, 358 struct scsi_sense_hdr *sshdr, int timeout, int retries) 359{ 360 char *sense = NULL; 361 int result; 362 363 if (sshdr) { 364 sense = kmalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO); 365 if (!sense) 366 return DRIVER_ERROR << 24; 367 memset(sense, 0, SCSI_SENSE_BUFFERSIZE); 368 } 369 result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen, 370 sense, timeout, retries, 0); 371 if (sshdr) 372 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr); 373 374 kfree(sense); 375 return result; 376} 377EXPORT_SYMBOL(scsi_execute_req); 378 379/* 380 * Function: scsi_init_cmd_errh() 381 * 382 * Purpose: Initialize cmd fields related to error handling. 383 * 384 * Arguments: cmd - command that is ready to be queued. 385 * 386 * Returns: Nothing 387 * 388 * Notes: This function has the job of initializing a number of 389 * fields related to error handling. Typically this will 390 * be called once for each command, as required. 391 */ 392static int scsi_init_cmd_errh(struct scsi_cmnd *cmd) 393{ 394 cmd->serial_number = 0; 395 396 memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer); 397 398 if (cmd->cmd_len == 0) 399 cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]); 400 401 /* 402 * We need saved copies of a number of fields - this is because 403 * error handling may need to overwrite these with different values 404 * to run different commands, and once error handling is complete, 405 * we will need to restore these values prior to running the actual 406 * command. 407 */ 408 cmd->old_use_sg = cmd->use_sg; 409 cmd->old_cmd_len = cmd->cmd_len; 410 cmd->sc_old_data_direction = cmd->sc_data_direction; 411 cmd->old_underflow = cmd->underflow; 412 memcpy(cmd->data_cmnd, cmd->cmnd, sizeof(cmd->cmnd)); 413 cmd->buffer = cmd->request_buffer; 414 cmd->bufflen = cmd->request_bufflen; 415 416 return 1; 417} 418 419/* 420 * Function: scsi_setup_cmd_retry() 421 * 422 * Purpose: Restore the command state for a retry 423 * 424 * Arguments: cmd - command to be restored 425 * 426 * Returns: Nothing 427 * 428 * Notes: Immediately prior to retrying a command, we need 429 * to restore certain fields that we saved above. 430 */ 431void scsi_setup_cmd_retry(struct scsi_cmnd *cmd) 432{ 433 memcpy(cmd->cmnd, cmd->data_cmnd, sizeof(cmd->data_cmnd)); 434 cmd->request_buffer = cmd->buffer; 435 cmd->request_bufflen = cmd->bufflen; 436 cmd->use_sg = cmd->old_use_sg; 437 cmd->cmd_len = cmd->old_cmd_len; 438 cmd->sc_data_direction = cmd->sc_old_data_direction; 439 cmd->underflow = cmd->old_underflow; 440} 441 442void scsi_device_unbusy(struct scsi_device *sdev) 443{ 444 struct Scsi_Host *shost = sdev->host; 445 unsigned long flags; 446 447 spin_lock_irqsave(shost->host_lock, flags); 448 shost->host_busy--; 449 if (unlikely(scsi_host_in_recovery(shost) && 450 shost->host_failed)) 451 scsi_eh_wakeup(shost); 452 spin_unlock(shost->host_lock); 453 spin_lock(sdev->request_queue->queue_lock); 454 sdev->device_busy--; 455 spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags); 456} 457 458/* 459 * Called for single_lun devices on IO completion. Clear starget_sdev_user, 460 * and call blk_run_queue for all the scsi_devices on the target - 461 * including current_sdev first. 462 * 463 * Called with *no* scsi locks held. 464 */ 465static void scsi_single_lun_run(struct scsi_device *current_sdev) 466{ 467 struct Scsi_Host *shost = current_sdev->host; 468 struct scsi_device *sdev, *tmp; 469 struct scsi_target *starget = scsi_target(current_sdev); 470 unsigned long flags; 471 472 spin_lock_irqsave(shost->host_lock, flags); 473 starget->starget_sdev_user = NULL; 474 spin_unlock_irqrestore(shost->host_lock, flags); 475 476 /* 477 * Call blk_run_queue for all LUNs on the target, starting with 478 * current_sdev. We race with others (to set starget_sdev_user), 479 * but in most cases, we will be first. Ideally, each LU on the 480 * target would get some limited time or requests on the target. 481 */ 482 blk_run_queue(current_sdev->request_queue); 483 484 spin_lock_irqsave(shost->host_lock, flags); 485 if (starget->starget_sdev_user) 486 goto out; 487 list_for_each_entry_safe(sdev, tmp, &starget->devices, 488 same_target_siblings) { 489 if (sdev == current_sdev) 490 continue; 491 if (scsi_device_get(sdev)) 492 continue; 493 494 spin_unlock_irqrestore(shost->host_lock, flags); 495 blk_run_queue(sdev->request_queue); 496 spin_lock_irqsave(shost->host_lock, flags); 497 498 scsi_device_put(sdev); 499 } 500 out: 501 spin_unlock_irqrestore(shost->host_lock, flags); 502} 503 504/* 505 * Function: scsi_run_queue() 506 * 507 * Purpose: Select a proper request queue to serve next 508 * 509 * Arguments: q - last request's queue 510 * 511 * Returns: Nothing 512 * 513 * Notes: The previous command was completely finished, start 514 * a new one if possible. 515 */ 516static void scsi_run_queue(struct request_queue *q) 517{ 518 struct scsi_device *sdev = q->queuedata; 519 struct Scsi_Host *shost = sdev->host; 520 unsigned long flags; 521 522 if (sdev->single_lun) 523 scsi_single_lun_run(sdev); 524 525 spin_lock_irqsave(shost->host_lock, flags); 526 while (!list_empty(&shost->starved_list) && 527 !shost->host_blocked && !shost->host_self_blocked && 528 !((shost->can_queue > 0) && 529 (shost->host_busy >= shost->can_queue))) { 530 /* 531 * As long as shost is accepting commands and we have 532 * starved queues, call blk_run_queue. scsi_request_fn 533 * drops the queue_lock and can add us back to the 534 * starved_list. 535 * 536 * host_lock protects the starved_list and starved_entry. 537 * scsi_request_fn must get the host_lock before checking 538 * or modifying starved_list or starved_entry. 539 */ 540 sdev = list_entry(shost->starved_list.next, 541 struct scsi_device, starved_entry); 542 list_del_init(&sdev->starved_entry); 543 spin_unlock_irqrestore(shost->host_lock, flags); 544 545 blk_run_queue(sdev->request_queue); 546 547 spin_lock_irqsave(shost->host_lock, flags); 548 if (unlikely(!list_empty(&sdev->starved_entry))) 549 /* 550 * sdev lost a race, and was put back on the 551 * starved list. This is unlikely but without this 552 * in theory we could loop forever. 553 */ 554 break; 555 } 556 spin_unlock_irqrestore(shost->host_lock, flags); 557 558 blk_run_queue(q); 559} 560 561/* 562 * Function: scsi_requeue_command() 563 * 564 * Purpose: Handle post-processing of completed commands. 565 * 566 * Arguments: q - queue to operate on 567 * cmd - command that may need to be requeued. 568 * 569 * Returns: Nothing 570 * 571 * Notes: After command completion, there may be blocks left 572 * over which weren't finished by the previous command 573 * this can be for a number of reasons - the main one is 574 * I/O errors in the middle of the request, in which case 575 * we need to request the blocks that come after the bad 576 * sector. 577 * Notes: Upon return, cmd is a stale pointer. 578 */ 579static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd) 580{ 581 struct request *req = cmd->request; 582 unsigned long flags; 583 584 scsi_unprep_request(req); 585 spin_lock_irqsave(q->queue_lock, flags); 586 blk_requeue_request(q, req); 587 spin_unlock_irqrestore(q->queue_lock, flags); 588 589 scsi_run_queue(q); 590} 591 592void scsi_next_command(struct scsi_cmnd *cmd) 593{ 594 struct request_queue *q = cmd->device->request_queue; 595 596 scsi_put_command(cmd); 597 scsi_run_queue(q); 598} 599 600void scsi_run_host_queues(struct Scsi_Host *shost) 601{ 602 struct scsi_device *sdev; 603 604 shost_for_each_device(sdev, shost) 605 scsi_run_queue(sdev->request_queue); 606} 607 608/* 609 * Function: scsi_end_request() 610 * 611 * Purpose: Post-processing of completed commands (usually invoked at end 612 * of upper level post-processing and scsi_io_completion). 613 * 614 * Arguments: cmd - command that is complete. 615 * uptodate - 1 if I/O indicates success, <= 0 for I/O error. 616 * bytes - number of bytes of completed I/O 617 * requeue - indicates whether we should requeue leftovers. 618 * 619 * Lock status: Assumed that lock is not held upon entry. 620 * 621 * Returns: cmd if requeue required, NULL otherwise. 622 * 623 * Notes: This is called for block device requests in order to 624 * mark some number of sectors as complete. 625 * 626 * We are guaranteeing that the request queue will be goosed 627 * at some point during this call. 628 * Notes: If cmd was requeued, upon return it will be a stale pointer. 629 */ 630static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate, 631 int bytes, int requeue) 632{ 633 request_queue_t *q = cmd->device->request_queue; 634 struct request *req = cmd->request; 635 unsigned long flags; 636 637 /* 638 * If there are blocks left over at the end, set up the command 639 * to queue the remainder of them. 640 */ 641 if (end_that_request_chunk(req, uptodate, bytes)) { 642 int leftover = (req->hard_nr_sectors << 9); 643 644 if (blk_pc_request(req)) 645 leftover = req->data_len; 646 647 /* kill remainder if no retrys */ 648 if (!uptodate && blk_noretry_request(req)) 649 end_that_request_chunk(req, 0, leftover); 650 else { 651 if (requeue) { 652 /* 653 * Bleah. Leftovers again. Stick the 654 * leftovers in the front of the 655 * queue, and goose the queue again. 656 */ 657 scsi_requeue_command(q, cmd); 658 cmd = NULL; 659 } 660 return cmd; 661 } 662 } 663 664 add_disk_randomness(req->rq_disk); 665 666 spin_lock_irqsave(q->queue_lock, flags); 667 if (blk_rq_tagged(req)) 668 blk_queue_end_tag(q, req); 669 end_that_request_last(req); 670 spin_unlock_irqrestore(q->queue_lock, flags); 671 672 /* 673 * This will goose the queue request function at the end, so we don't 674 * need to worry about launching another command. 675 */ 676 scsi_next_command(cmd); 677 return NULL; 678} 679 680static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, int gfp_mask) 681{ 682 struct scsi_host_sg_pool *sgp; 683 struct scatterlist *sgl; 684 685 BUG_ON(!cmd->use_sg); 686 687 switch (cmd->use_sg) { 688 case 1 ... 8: 689 cmd->sglist_len = 0; 690 break; 691 case 9 ... 16: 692 cmd->sglist_len = 1; 693 break; 694 case 17 ... 32: 695 cmd->sglist_len = 2; 696 break; 697#if (SCSI_MAX_PHYS_SEGMENTS > 32) 698 case 33 ... 64: 699 cmd->sglist_len = 3; 700 break; 701#if (SCSI_MAX_PHYS_SEGMENTS > 64) 702 case 65 ... 128: 703 cmd->sglist_len = 4; 704 break; 705#if (SCSI_MAX_PHYS_SEGMENTS > 128) 706 case 129 ... 256: 707 cmd->sglist_len = 5; 708 break; 709#endif 710#endif 711#endif 712 default: 713 return NULL; 714 } 715 716 sgp = scsi_sg_pools + cmd->sglist_len; 717 sgl = mempool_alloc(sgp->pool, gfp_mask); 718 return sgl; 719} 720 721static void scsi_free_sgtable(struct scatterlist *sgl, int index) 722{ 723 struct scsi_host_sg_pool *sgp; 724 725 BUG_ON(index >= SG_MEMPOOL_NR); 726 727 sgp = scsi_sg_pools + index; 728 mempool_free(sgl, sgp->pool); 729} 730 731/* 732 * Function: scsi_release_buffers() 733 * 734 * Purpose: Completion processing for block device I/O requests. 735 * 736 * Arguments: cmd - command that we are bailing. 737 * 738 * Lock status: Assumed that no lock is held upon entry. 739 * 740 * Returns: Nothing 741 * 742 * Notes: In the event that an upper level driver rejects a 743 * command, we must release resources allocated during 744 * the __init_io() function. Primarily this would involve 745 * the scatter-gather table, and potentially any bounce 746 * buffers. 747 */ 748static void scsi_release_buffers(struct scsi_cmnd *cmd) 749{ 750 struct request *req = cmd->request; 751 752 /* 753 * Free up any indirection buffers we allocated for DMA purposes. 754 */ 755 if (cmd->use_sg) 756 scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len); 757 else if (cmd->request_buffer != req->buffer) 758 kfree(cmd->request_buffer); 759 760 /* 761 * Zero these out. They now point to freed memory, and it is 762 * dangerous to hang onto the pointers. 763 */ 764 cmd->buffer = NULL; 765 cmd->bufflen = 0; 766 cmd->request_buffer = NULL; 767 cmd->request_bufflen = 0; 768} 769 770/* 771 * Function: scsi_io_completion() 772 * 773 * Purpose: Completion processing for block device I/O requests. 774 * 775 * Arguments: cmd - command that is finished. 776 * 777 * Lock status: Assumed that no lock is held upon entry. 778 * 779 * Returns: Nothing 780 * 781 * Notes: This function is matched in terms of capabilities to 782 * the function that created the scatter-gather list. 783 * In other words, if there are no bounce buffers 784 * (the normal case for most drivers), we don't need 785 * the logic to deal with cleaning up afterwards. 786 * 787 * We must do one of several things here: 788 * 789 * a) Call scsi_end_request. This will finish off the 790 * specified number of sectors. If we are done, the 791 * command block will be released, and the queue 792 * function will be goosed. If we are not done, then 793 * scsi_end_request will directly goose the queue. 794 * 795 * b) We can just use scsi_requeue_command() here. This would 796 * be used if we just wanted to retry, for example. 797 */ 798void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes, 799 unsigned int block_bytes) 800{ 801 int result = cmd->result; 802 int this_count = cmd->bufflen; 803 request_queue_t *q = cmd->device->request_queue; 804 struct request *req = cmd->request; 805 int clear_errors = 1; 806 struct scsi_sense_hdr sshdr; 807 int sense_valid = 0; 808 int sense_deferred = 0; 809 810 if (blk_complete_barrier_rq(q, req, good_bytes >> 9)) 811 return; 812 813 /* 814 * Free up any indirection buffers we allocated for DMA purposes. 815 * For the case of a READ, we need to copy the data out of the 816 * bounce buffer and into the real buffer. 817 */ 818 if (cmd->use_sg) 819 scsi_free_sgtable(cmd->buffer, cmd->sglist_len); 820 else if (cmd->buffer != req->buffer) { 821 if (rq_data_dir(req) == READ) { 822 unsigned long flags; 823 char *to = bio_kmap_irq(req->bio, &flags); 824 memcpy(to, cmd->buffer, cmd->bufflen); 825 bio_kunmap_irq(to, &flags); 826 } 827 kfree(cmd->buffer); 828 } 829 830 if (result) { 831 sense_valid = scsi_command_normalize_sense(cmd, &sshdr); 832 if (sense_valid) 833 sense_deferred = scsi_sense_is_deferred(&sshdr); 834 } 835 if (blk_pc_request(req)) { /* SG_IO ioctl from block level */ 836 req->errors = result; 837 if (result) { 838 clear_errors = 0; 839 if (sense_valid && req->sense) { 840 /* 841 * SG_IO wants current and deferred errors 842 */ 843 int len = 8 + cmd->sense_buffer[7]; 844 845 if (len > SCSI_SENSE_BUFFERSIZE) 846 len = SCSI_SENSE_BUFFERSIZE; 847 memcpy(req->sense, cmd->sense_buffer, len); 848 req->sense_len = len; 849 } 850 } else 851 req->data_len = cmd->resid; 852 } 853 854 /* 855 * Zero these out. They now point to freed memory, and it is 856 * dangerous to hang onto the pointers. 857 */ 858 cmd->buffer = NULL; 859 cmd->bufflen = 0; 860 cmd->request_buffer = NULL; 861 cmd->request_bufflen = 0; 862 863 /* 864 * Next deal with any sectors which we were able to correctly 865 * handle. 866 */ 867 if (good_bytes >= 0) { 868 SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, %d bytes done.\n", 869 req->nr_sectors, good_bytes)); 870 SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg)); 871 872 if (clear_errors) 873 req->errors = 0; 874 /* 875 * If multiple sectors are requested in one buffer, then 876 * they will have been finished off by the first command. 877 * If not, then we have a multi-buffer command. 878 * 879 * If block_bytes != 0, it means we had a medium error 880 * of some sort, and that we want to mark some number of 881 * sectors as not uptodate. Thus we want to inhibit 882 * requeueing right here - we will requeue down below 883 * when we handle the bad sectors. 884 */ 885 886 /* 887 * If the command completed without error, then either 888 * finish off the rest of the command, or start a new one. 889 */ 890 if (scsi_end_request(cmd, 1, good_bytes, result == 0) == NULL) 891 return; 892 } 893 /* 894 * Now, if we were good little boys and girls, Santa left us a request 895 * sense buffer. We can extract information from this, so we 896 * can choose a block to remap, etc. 897 */ 898 if (sense_valid && !sense_deferred) { 899 switch (sshdr.sense_key) { 900 case UNIT_ATTENTION: 901 if (cmd->device->removable) { 902 /* detected disc change. set a bit 903 * and quietly refuse further access. 904 */ 905 cmd->device->changed = 1; 906 scsi_end_request(cmd, 0, 907 this_count, 1); 908 return; 909 } else { 910 /* 911 * Must have been a power glitch, or a 912 * bus reset. Could not have been a 913 * media change, so we just retry the 914 * request and see what happens. 915 */ 916 scsi_requeue_command(q, cmd); 917 return; 918 } 919 break; 920 case ILLEGAL_REQUEST: 921 /* 922 * If we had an ILLEGAL REQUEST returned, then we may 923 * have performed an unsupported command. The only 924 * thing this should be would be a ten byte read where 925 * only a six byte read was supported. Also, on a 926 * system where READ CAPACITY failed, we may have read 927 * past the end of the disk. 928 */ 929 if (cmd->device->use_10_for_rw && 930 (cmd->cmnd[0] == READ_10 || 931 cmd->cmnd[0] == WRITE_10)) { 932 cmd->device->use_10_for_rw = 0; 933 /* 934 * This will cause a retry with a 6-byte 935 * command. 936 */ 937 scsi_requeue_command(q, cmd); 938 result = 0; 939 } else { 940 scsi_end_request(cmd, 0, this_count, 1); 941 return; 942 } 943 break; 944 case NOT_READY: 945 /* 946 * If the device is in the process of becoming ready, 947 * retry. 948 */ 949 if (sshdr.asc == 0x04 && sshdr.ascq == 0x01) { 950 scsi_requeue_command(q, cmd); 951 return; 952 } 953 if (!(req->flags & REQ_QUIET)) 954 dev_printk(KERN_INFO, 955 &cmd->device->sdev_gendev, 956 "Device not ready.\n"); 957 scsi_end_request(cmd, 0, this_count, 1); 958 return; 959 case VOLUME_OVERFLOW: 960 if (!(req->flags & REQ_QUIET)) { 961 dev_printk(KERN_INFO, 962 &cmd->device->sdev_gendev, 963 "Volume overflow, CDB: "); 964 __scsi_print_command(cmd->data_cmnd); 965 scsi_print_sense("", cmd); 966 } 967 scsi_end_request(cmd, 0, block_bytes, 1); 968 return; 969 default: 970 break; 971 } 972 } /* driver byte != 0 */ 973 if (host_byte(result) == DID_RESET) { 974 /* 975 * Third party bus reset or reset for error 976 * recovery reasons. Just retry the request 977 * and see what happens. 978 */ 979 scsi_requeue_command(q, cmd); 980 return; 981 } 982 if (result) { 983 if (!(req->flags & REQ_QUIET)) { 984 dev_printk(KERN_INFO, &cmd->device->sdev_gendev, 985 "SCSI error: return code = 0x%x\n", result); 986 987 if (driver_byte(result) & DRIVER_SENSE) 988 scsi_print_sense("", cmd); 989 } 990 /* 991 * Mark a single buffer as not uptodate. Queue the remainder. 992 * We sometimes get this cruft in the event that a medium error 993 * isn't properly reported. 994 */ 995 block_bytes = req->hard_cur_sectors << 9; 996 if (!block_bytes) 997 block_bytes = req->data_len; 998 scsi_end_request(cmd, 0, block_bytes, 1); 999 } 1000} 1001EXPORT_SYMBOL(scsi_io_completion); 1002 1003/* 1004 * Function: scsi_init_io() 1005 * 1006 * Purpose: SCSI I/O initialize function. 1007 * 1008 * Arguments: cmd - Command descriptor we wish to initialize 1009 * 1010 * Returns: 0 on success 1011 * BLKPREP_DEFER if the failure is retryable 1012 * BLKPREP_KILL if the failure is fatal 1013 */ 1014static int scsi_init_io(struct scsi_cmnd *cmd) 1015{ 1016 struct request *req = cmd->request; 1017 struct scatterlist *sgpnt; 1018 int count; 1019 1020 /* 1021 * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer 1022 */ 1023 if ((req->flags & REQ_BLOCK_PC) && !req->bio) { 1024 cmd->request_bufflen = req->data_len; 1025 cmd->request_buffer = req->data; 1026 req->buffer = req->data; 1027 cmd->use_sg = 0; 1028 return 0; 1029 } 1030 1031 /* 1032 * we used to not use scatter-gather for single segment request, 1033 * but now we do (it makes highmem I/O easier to support without 1034 * kmapping pages) 1035 */ 1036 cmd->use_sg = req->nr_phys_segments; 1037 1038 /* 1039 * if sg table allocation fails, requeue request later. 1040 */ 1041 sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC); 1042 if (unlikely(!sgpnt)) { 1043 scsi_unprep_request(req); 1044 return BLKPREP_DEFER; 1045 } 1046 1047 cmd->request_buffer = (char *) sgpnt; 1048 cmd->request_bufflen = req->nr_sectors << 9; 1049 if (blk_pc_request(req)) 1050 cmd->request_bufflen = req->data_len; 1051 req->buffer = NULL; 1052 1053 /* 1054 * Next, walk the list, and fill in the addresses and sizes of 1055 * each segment. 1056 */ 1057 count = blk_rq_map_sg(req->q, req, cmd->request_buffer); 1058 1059 /* 1060 * mapped well, send it off 1061 */ 1062 if (likely(count <= cmd->use_sg)) { 1063 cmd->use_sg = count; 1064 return 0; 1065 } 1066 1067 printk(KERN_ERR "Incorrect number of segments after building list\n"); 1068 printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg); 1069 printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors, 1070 req->current_nr_sectors); 1071 1072 /* release the command and kill it */ 1073 scsi_release_buffers(cmd); 1074 scsi_put_command(cmd); 1075 return BLKPREP_KILL; 1076} 1077 1078static int scsi_prepare_flush_fn(request_queue_t *q, struct request *rq) 1079{ 1080 struct scsi_device *sdev = q->queuedata; 1081 struct scsi_driver *drv; 1082 1083 if (sdev->sdev_state == SDEV_RUNNING) { 1084 drv = *(struct scsi_driver **) rq->rq_disk->private_data; 1085 1086 if (drv->prepare_flush) 1087 return drv->prepare_flush(q, rq); 1088 } 1089 1090 return 0; 1091} 1092 1093static void scsi_end_flush_fn(request_queue_t *q, struct request *rq) 1094{ 1095 struct scsi_device *sdev = q->queuedata; 1096 struct request *flush_rq = rq->end_io_data; 1097 struct scsi_driver *drv; 1098 1099 if (flush_rq->errors) { 1100 printk("scsi: barrier error, disabling flush support\n"); 1101 blk_queue_ordered(q, QUEUE_ORDERED_NONE); 1102 } 1103 1104 if (sdev->sdev_state == SDEV_RUNNING) { 1105 drv = *(struct scsi_driver **) rq->rq_disk->private_data; 1106 drv->end_flush(q, rq); 1107 } 1108} 1109 1110static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk, 1111 sector_t *error_sector) 1112{ 1113 struct scsi_device *sdev = q->queuedata; 1114 struct scsi_driver *drv; 1115 1116 if (sdev->sdev_state != SDEV_RUNNING) 1117 return -ENXIO; 1118 1119 drv = *(struct scsi_driver **) disk->private_data; 1120 if (drv->issue_flush) 1121 return drv->issue_flush(&sdev->sdev_gendev, error_sector); 1122 1123 return -EOPNOTSUPP; 1124} 1125 1126static void scsi_generic_done(struct scsi_cmnd *cmd) 1127{ 1128 BUG_ON(!blk_pc_request(cmd->request)); 1129 scsi_io_completion(cmd, cmd->result == 0 ? cmd->bufflen : 0, 0); 1130} 1131 1132static int scsi_prep_fn(struct request_queue *q, struct request *req) 1133{ 1134 struct scsi_device *sdev = q->queuedata; 1135 struct scsi_cmnd *cmd; 1136 int specials_only = 0; 1137 1138 /* 1139 * Just check to see if the device is online. If it isn't, we 1140 * refuse to process any commands. The device must be brought 1141 * online before trying any recovery commands 1142 */ 1143 if (unlikely(!scsi_device_online(sdev))) { 1144 printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n", 1145 sdev->host->host_no, sdev->id, sdev->lun); 1146 goto kill; 1147 } 1148 if (unlikely(sdev->sdev_state != SDEV_RUNNING)) { 1149 /* OK, we're not in a running state don't prep 1150 * user commands */ 1151 if (sdev->sdev_state == SDEV_DEL) { 1152 /* Device is fully deleted, no commands 1153 * at all allowed down */ 1154 printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to dead device\n", 1155 sdev->host->host_no, sdev->id, sdev->lun); 1156 goto kill; 1157 } 1158 /* OK, we only allow special commands (i.e. not 1159 * user initiated ones */ 1160 specials_only = sdev->sdev_state; 1161 } 1162 1163 /* 1164 * Find the actual device driver associated with this command. 1165 * The SPECIAL requests are things like character device or 1166 * ioctls, which did not originate from ll_rw_blk. Note that 1167 * the special field is also used to indicate the cmd for 1168 * the remainder of a partially fulfilled request that can 1169 * come up when there is a medium error. We have to treat 1170 * these two cases differently. We differentiate by looking 1171 * at request->cmd, as this tells us the real story. 1172 */ 1173 if (req->flags & REQ_SPECIAL && req->special) { 1174 struct scsi_request *sreq = req->special; 1175 1176 if (sreq->sr_magic == SCSI_REQ_MAGIC) { 1177 cmd = scsi_get_command(sreq->sr_device, GFP_ATOMIC); 1178 if (unlikely(!cmd)) 1179 goto defer; 1180 scsi_init_cmd_from_req(cmd, sreq); 1181 } else 1182 cmd = req->special; 1183 } else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) { 1184 1185 if(unlikely(specials_only) && !(req->flags & REQ_SPECIAL)) { 1186 if(specials_only == SDEV_QUIESCE || 1187 specials_only == SDEV_BLOCK) 1188 goto defer; 1189 1190 printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to device being removed\n", 1191 sdev->host->host_no, sdev->id, sdev->lun); 1192 goto kill; 1193 } 1194 1195 1196 /* 1197 * Now try and find a command block that we can use. 1198 */ 1199 if (!req->special) { 1200 cmd = scsi_get_command(sdev, GFP_ATOMIC); 1201 if (unlikely(!cmd)) 1202 goto defer; 1203 } else 1204 cmd = req->special; 1205 1206 /* pull a tag out of the request if we have one */ 1207 cmd->tag = req->tag; 1208 } else { 1209 blk_dump_rq_flags(req, "SCSI bad req"); 1210 goto kill; 1211 } 1212 1213 /* note the overloading of req->special. When the tag 1214 * is active it always means cmd. If the tag goes 1215 * back for re-queueing, it may be reset */ 1216 req->special = cmd; 1217 cmd->request = req; 1218 1219 /* 1220 * FIXME: drop the lock here because the functions below 1221 * expect to be called without the queue lock held. Also, 1222 * previously, we dequeued the request before dropping the 1223 * lock. We hope REQ_STARTED prevents anything untoward from 1224 * happening now. 1225 */ 1226 if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) { 1227 struct scsi_driver *drv; 1228 int ret; 1229 1230 /* 1231 * This will do a couple of things: 1232 * 1) Fill in the actual SCSI command. 1233 * 2) Fill in any other upper-level specific fields 1234 * (timeout). 1235 * 1236 * If this returns 0, it means that the request failed 1237 * (reading past end of disk, reading offline device, 1238 * etc). This won't actually talk to the device, but 1239 * some kinds of consistency checking may cause the 1240 * request to be rejected immediately. 1241 */ 1242 1243 /* 1244 * This sets up the scatter-gather table (allocating if 1245 * required). 1246 */ 1247 ret = scsi_init_io(cmd); 1248 switch(ret) { 1249 /* For BLKPREP_KILL/DEFER the cmd was released */ 1250 case BLKPREP_KILL: 1251 goto kill; 1252 case BLKPREP_DEFER: 1253 goto defer; 1254 } 1255 1256 /* 1257 * Initialize the actual SCSI command for this request. 1258 */ 1259 if (req->rq_disk) { 1260 drv = *(struct scsi_driver **)req->rq_disk->private_data; 1261 if (unlikely(!drv->init_command(cmd))) { 1262 scsi_release_buffers(cmd); 1263 scsi_put_command(cmd); 1264 goto kill; 1265 } 1266 } else { 1267 memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd)); 1268 cmd->cmd_len = req->cmd_len; 1269 if (rq_data_dir(req) == WRITE) 1270 cmd->sc_data_direction = DMA_TO_DEVICE; 1271 else if (req->data_len) 1272 cmd->sc_data_direction = DMA_FROM_DEVICE; 1273 else 1274 cmd->sc_data_direction = DMA_NONE; 1275 1276 cmd->transfersize = req->data_len; 1277 cmd->allowed = 3; 1278 cmd->timeout_per_command = req->timeout; 1279 cmd->done = scsi_generic_done; 1280 } 1281 } 1282 1283 /* 1284 * The request is now prepped, no need to come back here 1285 */ 1286 req->flags |= REQ_DONTPREP; 1287 return BLKPREP_OK; 1288 1289 defer: 1290 /* If we defer, the elv_next_request() returns NULL, but the 1291 * queue must be restarted, so we plug here if no returning 1292 * command will automatically do that. */ 1293 if (sdev->device_busy == 0) 1294 blk_plug_device(q); 1295 return BLKPREP_DEFER; 1296 kill: 1297 req->errors = DID_NO_CONNECT << 16; 1298 return BLKPREP_KILL; 1299} 1300 1301/* 1302 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else 1303 * return 0. 1304 * 1305 * Called with the queue_lock held. 1306 */ 1307static inline int scsi_dev_queue_ready(struct request_queue *q, 1308 struct scsi_device *sdev) 1309{ 1310 if (sdev->device_busy >= sdev->queue_depth) 1311 return 0; 1312 if (sdev->device_busy == 0 && sdev->device_blocked) { 1313 /* 1314 * unblock after device_blocked iterates to zero 1315 */ 1316 if (--sdev->device_blocked == 0) { 1317 SCSI_LOG_MLQUEUE(3, 1318 printk("scsi%d (%d:%d) unblocking device at" 1319 " zero depth\n", sdev->host->host_no, 1320 sdev->id, sdev->lun)); 1321 } else { 1322 blk_plug_device(q); 1323 return 0; 1324 } 1325 } 1326 if (sdev->device_blocked) 1327 return 0; 1328 1329 return 1; 1330} 1331 1332/* 1333 * scsi_host_queue_ready: if we can send requests to shost, return 1 else 1334 * return 0. We must end up running the queue again whenever 0 is 1335 * returned, else IO can hang. 1336 * 1337 * Called with host_lock held. 1338 */ 1339static inline int scsi_host_queue_ready(struct request_queue *q, 1340 struct Scsi_Host *shost, 1341 struct scsi_device *sdev) 1342{ 1343 if (scsi_host_in_recovery(shost)) 1344 return 0; 1345 if (shost->host_busy == 0 && shost->host_blocked) { 1346 /* 1347 * unblock after host_blocked iterates to zero 1348 */ 1349 if (--shost->host_blocked == 0) { 1350 SCSI_LOG_MLQUEUE(3, 1351 printk("scsi%d unblocking host at zero depth\n", 1352 shost->host_no)); 1353 } else { 1354 blk_plug_device(q); 1355 return 0; 1356 } 1357 } 1358 if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) || 1359 shost->host_blocked || shost->host_self_blocked) { 1360 if (list_empty(&sdev->starved_entry)) 1361 list_add_tail(&sdev->starved_entry, &shost->starved_list); 1362 return 0; 1363 } 1364 1365 /* We're OK to process the command, so we can't be starved */ 1366 if (!list_empty(&sdev->starved_entry)) 1367 list_del_init(&sdev->starved_entry); 1368 1369 return 1; 1370} 1371 1372/* 1373 * Kill a request for a dead device 1374 */ 1375static void scsi_kill_request(struct request *req, request_queue_t *q) 1376{ 1377 struct scsi_cmnd *cmd = req->special; 1378 1379 blkdev_dequeue_request(req); 1380 1381 if (unlikely(cmd == NULL)) { 1382 printk(KERN_CRIT "impossible request in %s.\n", 1383 __FUNCTION__); 1384 BUG(); 1385 } 1386 1387 scsi_init_cmd_errh(cmd); 1388 cmd->result = DID_NO_CONNECT << 16; 1389 atomic_inc(&cmd->device->iorequest_cnt); 1390 __scsi_done(cmd); 1391} 1392 1393/* 1394 * Function: scsi_request_fn() 1395 * 1396 * Purpose: Main strategy routine for SCSI. 1397 * 1398 * Arguments: q - Pointer to actual queue. 1399 * 1400 * Returns: Nothing 1401 * 1402 * Lock status: IO request lock assumed to be held when called. 1403 */ 1404static void scsi_request_fn(struct request_queue *q) 1405{ 1406 struct scsi_device *sdev = q->queuedata; 1407 struct Scsi_Host *shost; 1408 struct scsi_cmnd *cmd; 1409 struct request *req; 1410 1411 if (!sdev) { 1412 printk("scsi: killing requests for dead queue\n"); 1413 while ((req = elv_next_request(q)) != NULL) 1414 scsi_kill_request(req, q); 1415 return; 1416 } 1417 1418 if(!get_device(&sdev->sdev_gendev)) 1419 /* We must be tearing the block queue down already */ 1420 return; 1421 1422 /* 1423 * To start with, we keep looping until the queue is empty, or until 1424 * the host is no longer able to accept any more requests. 1425 */ 1426 shost = sdev->host; 1427 while (!blk_queue_plugged(q)) { 1428 int rtn; 1429 /* 1430 * get next queueable request. We do this early to make sure 1431 * that the request is fully prepared even if we cannot 1432 * accept it. 1433 */ 1434 req = elv_next_request(q); 1435 if (!req || !scsi_dev_queue_ready(q, sdev)) 1436 break; 1437 1438 if (unlikely(!scsi_device_online(sdev))) { 1439 printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n", 1440 sdev->host->host_no, sdev->id, sdev->lun); 1441 scsi_kill_request(req, q); 1442 continue; 1443 } 1444 1445 1446 /* 1447 * Remove the request from the request list. 1448 */ 1449 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req))) 1450 blkdev_dequeue_request(req); 1451 sdev->device_busy++; 1452 1453 spin_unlock(q->queue_lock); 1454 cmd = req->special; 1455 if (unlikely(cmd == NULL)) { 1456 printk(KERN_CRIT "impossible request in %s.\n" 1457 "please mail a stack trace to " 1458 "linux-scsi@vger.kernel.org", 1459 __FUNCTION__); 1460 BUG(); 1461 } 1462 spin_lock(shost->host_lock); 1463 1464 if (!scsi_host_queue_ready(q, shost, sdev)) 1465 goto not_ready; 1466 if (sdev->single_lun) { 1467 if (scsi_target(sdev)->starget_sdev_user && 1468 scsi_target(sdev)->starget_sdev_user != sdev) 1469 goto not_ready; 1470 scsi_target(sdev)->starget_sdev_user = sdev; 1471 } 1472 shost->host_busy++; 1473 1474 /* 1475 * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will 1476 * take the lock again. 1477 */ 1478 spin_unlock_irq(shost->host_lock); 1479 1480 /* 1481 * Finally, initialize any error handling parameters, and set up 1482 * the timers for timeouts. 1483 */ 1484 scsi_init_cmd_errh(cmd); 1485 1486 /* 1487 * Dispatch the command to the low-level driver. 1488 */ 1489 rtn = scsi_dispatch_cmd(cmd); 1490 spin_lock_irq(q->queue_lock); 1491 if(rtn) { 1492 /* we're refusing the command; because of 1493 * the way locks get dropped, we need to 1494 * check here if plugging is required */ 1495 if(sdev->device_busy == 0) 1496 blk_plug_device(q); 1497 1498 break; 1499 } 1500 } 1501 1502 goto out; 1503 1504 not_ready: 1505 spin_unlock_irq(shost->host_lock); 1506 1507 /* 1508 * lock q, handle tag, requeue req, and decrement device_busy. We 1509 * must return with queue_lock held. 1510 * 1511 * Decrementing device_busy without checking it is OK, as all such 1512 * cases (host limits or settings) should run the queue at some 1513 * later time. 1514 */ 1515 spin_lock_irq(q->queue_lock); 1516 blk_requeue_request(q, req); 1517 sdev->device_busy--; 1518 if(sdev->device_busy == 0) 1519 blk_plug_device(q); 1520 out: 1521 /* must be careful here...if we trigger the ->remove() function 1522 * we cannot be holding the q lock */ 1523 spin_unlock_irq(q->queue_lock); 1524 put_device(&sdev->sdev_gendev); 1525 spin_lock_irq(q->queue_lock); 1526} 1527 1528u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost) 1529{ 1530 struct device *host_dev; 1531 u64 bounce_limit = 0xffffffff; 1532 1533 if (shost->unchecked_isa_dma) 1534 return BLK_BOUNCE_ISA; 1535 /* 1536 * Platforms with virtual-DMA translation 1537 * hardware have no practical limit. 1538 */ 1539 if (!PCI_DMA_BUS_IS_PHYS) 1540 return BLK_BOUNCE_ANY; 1541 1542 host_dev = scsi_get_device(shost); 1543 if (host_dev && host_dev->dma_mask) 1544 bounce_limit = *host_dev->dma_mask; 1545 1546 return bounce_limit; 1547} 1548EXPORT_SYMBOL(scsi_calculate_bounce_limit); 1549 1550struct request_queue *scsi_alloc_queue(struct scsi_device *sdev) 1551{ 1552 struct Scsi_Host *shost = sdev->host; 1553 struct request_queue *q; 1554 1555 q = blk_init_queue(scsi_request_fn, NULL); 1556 if (!q) 1557 return NULL; 1558 1559 blk_queue_prep_rq(q, scsi_prep_fn); 1560 1561 blk_queue_max_hw_segments(q, shost->sg_tablesize); 1562 blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS); 1563 blk_queue_max_sectors(q, shost->max_sectors); 1564 blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost)); 1565 blk_queue_segment_boundary(q, shost->dma_boundary); 1566 blk_queue_issue_flush_fn(q, scsi_issue_flush_fn); 1567 1568 /* 1569 * ordered tags are superior to flush ordering 1570 */ 1571 if (shost->ordered_tag) 1572 blk_queue_ordered(q, QUEUE_ORDERED_TAG); 1573 else if (shost->ordered_flush) { 1574 blk_queue_ordered(q, QUEUE_ORDERED_FLUSH); 1575 q->prepare_flush_fn = scsi_prepare_flush_fn; 1576 q->end_flush_fn = scsi_end_flush_fn; 1577 } 1578 1579 if (!shost->use_clustering) 1580 clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags); 1581 return q; 1582} 1583 1584void scsi_free_queue(struct request_queue *q) 1585{ 1586 blk_cleanup_queue(q); 1587} 1588 1589/* 1590 * Function: scsi_block_requests() 1591 * 1592 * Purpose: Utility function used by low-level drivers to prevent further 1593 * commands from being queued to the device. 1594 * 1595 * Arguments: shost - Host in question 1596 * 1597 * Returns: Nothing 1598 * 1599 * Lock status: No locks are assumed held. 1600 * 1601 * Notes: There is no timer nor any other means by which the requests 1602 * get unblocked other than the low-level driver calling 1603 * scsi_unblock_requests(). 1604 */ 1605void scsi_block_requests(struct Scsi_Host *shost) 1606{ 1607 shost->host_self_blocked = 1; 1608} 1609EXPORT_SYMBOL(scsi_block_requests); 1610 1611/* 1612 * Function: scsi_unblock_requests() 1613 * 1614 * Purpose: Utility function used by low-level drivers to allow further 1615 * commands from being queued to the device. 1616 * 1617 * Arguments: shost - Host in question 1618 * 1619 * Returns: Nothing 1620 * 1621 * Lock status: No locks are assumed held. 1622 * 1623 * Notes: There is no timer nor any other means by which the requests 1624 * get unblocked other than the low-level driver calling 1625 * scsi_unblock_requests(). 1626 * 1627 * This is done as an API function so that changes to the 1628 * internals of the scsi mid-layer won't require wholesale 1629 * changes to drivers that use this feature. 1630 */ 1631void scsi_unblock_requests(struct Scsi_Host *shost) 1632{ 1633 shost->host_self_blocked = 0; 1634 scsi_run_host_queues(shost); 1635} 1636EXPORT_SYMBOL(scsi_unblock_requests); 1637 1638int __init scsi_init_queue(void) 1639{ 1640 int i; 1641 1642 for (i = 0; i < SG_MEMPOOL_NR; i++) { 1643 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 1644 int size = sgp->size * sizeof(struct scatterlist); 1645 1646 sgp->slab = kmem_cache_create(sgp->name, size, 0, 1647 SLAB_HWCACHE_ALIGN, NULL, NULL); 1648 if (!sgp->slab) { 1649 printk(KERN_ERR "SCSI: can't init sg slab %s\n", 1650 sgp->name); 1651 } 1652 1653 sgp->pool = mempool_create(SG_MEMPOOL_SIZE, 1654 mempool_alloc_slab, mempool_free_slab, 1655 sgp->slab); 1656 if (!sgp->pool) { 1657 printk(KERN_ERR "SCSI: can't init sg mempool %s\n", 1658 sgp->name); 1659 } 1660 } 1661 1662 return 0; 1663} 1664 1665void scsi_exit_queue(void) 1666{ 1667 int i; 1668 1669 for (i = 0; i < SG_MEMPOOL_NR; i++) { 1670 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i; 1671 mempool_destroy(sgp->pool); 1672 kmem_cache_destroy(sgp->slab); 1673 } 1674} 1675/** 1676 * scsi_mode_sense - issue a mode sense, falling back from 10 to 1677 * six bytes if necessary. 1678 * @sdev: SCSI device to be queried 1679 * @dbd: set if mode sense will allow block descriptors to be returned 1680 * @modepage: mode page being requested 1681 * @buffer: request buffer (may not be smaller than eight bytes) 1682 * @len: length of request buffer. 1683 * @timeout: command timeout 1684 * @retries: number of retries before failing 1685 * @data: returns a structure abstracting the mode header data 1686 * @sense: place to put sense data (or NULL if no sense to be collected). 1687 * must be SCSI_SENSE_BUFFERSIZE big. 1688 * 1689 * Returns zero if unsuccessful, or the header offset (either 4 1690 * or 8 depending on whether a six or ten byte command was 1691 * issued) if successful. 1692 **/ 1693int 1694scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, 1695 unsigned char *buffer, int len, int timeout, int retries, 1696 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) { 1697 unsigned char cmd[12]; 1698 int use_10_for_ms; 1699 int header_length; 1700 int result; 1701 struct scsi_sense_hdr my_sshdr; 1702 1703 memset(data, 0, sizeof(*data)); 1704 memset(&cmd[0], 0, 12); 1705 cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */ 1706 cmd[2] = modepage; 1707 1708 /* caller might not be interested in sense, but we need it */ 1709 if (!sshdr) 1710 sshdr = &my_sshdr; 1711 1712 retry: 1713 use_10_for_ms = sdev->use_10_for_ms; 1714 1715 if (use_10_for_ms) { 1716 if (len < 8) 1717 len = 8; 1718 1719 cmd[0] = MODE_SENSE_10; 1720 cmd[8] = len; 1721 header_length = 8; 1722 } else { 1723 if (len < 4) 1724 len = 4; 1725 1726 cmd[0] = MODE_SENSE; 1727 cmd[4] = len; 1728 header_length = 4; 1729 } 1730 1731 memset(buffer, 0, len); 1732 1733 result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len, 1734 sshdr, timeout, retries); 1735 1736 /* This code looks awful: what it's doing is making sure an 1737 * ILLEGAL REQUEST sense return identifies the actual command 1738 * byte as the problem. MODE_SENSE commands can return 1739 * ILLEGAL REQUEST if the code page isn't supported */ 1740 1741 if (use_10_for_ms && !scsi_status_is_good(result) && 1742 (driver_byte(result) & DRIVER_SENSE)) { 1743 if (scsi_sense_valid(sshdr)) { 1744 if ((sshdr->sense_key == ILLEGAL_REQUEST) && 1745 (sshdr->asc == 0x20) && (sshdr->ascq == 0)) { 1746 /* 1747 * Invalid command operation code 1748 */ 1749 sdev->use_10_for_ms = 0; 1750 goto retry; 1751 } 1752 } 1753 } 1754 1755 if(scsi_status_is_good(result)) { 1756 data->header_length = header_length; 1757 if(use_10_for_ms) { 1758 data->length = buffer[0]*256 + buffer[1] + 2; 1759 data->medium_type = buffer[2]; 1760 data->device_specific = buffer[3]; 1761 data->longlba = buffer[4] & 0x01; 1762 data->block_descriptor_length = buffer[6]*256 1763 + buffer[7]; 1764 } else { 1765 data->length = buffer[0] + 1; 1766 data->medium_type = buffer[1]; 1767 data->device_specific = buffer[2]; 1768 data->block_descriptor_length = buffer[3]; 1769 } 1770 } 1771 1772 return result; 1773} 1774EXPORT_SYMBOL(scsi_mode_sense); 1775 1776int 1777scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries) 1778{ 1779 char cmd[] = { 1780 TEST_UNIT_READY, 0, 0, 0, 0, 0, 1781 }; 1782 struct scsi_sense_hdr sshdr; 1783 int result; 1784 1785 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, &sshdr, 1786 timeout, retries); 1787 1788 if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) { 1789 1790 if ((scsi_sense_valid(&sshdr)) && 1791 ((sshdr.sense_key == UNIT_ATTENTION) || 1792 (sshdr.sense_key == NOT_READY))) { 1793 sdev->changed = 1; 1794 result = 0; 1795 } 1796 } 1797 return result; 1798} 1799EXPORT_SYMBOL(scsi_test_unit_ready); 1800 1801/** 1802 * scsi_device_set_state - Take the given device through the device 1803 * state model. 1804 * @sdev: scsi device to change the state of. 1805 * @state: state to change to. 1806 * 1807 * Returns zero if unsuccessful or an error if the requested 1808 * transition is illegal. 1809 **/ 1810int 1811scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state) 1812{ 1813 enum scsi_device_state oldstate = sdev->sdev_state; 1814 1815 if (state == oldstate) 1816 return 0; 1817 1818 switch (state) { 1819 case SDEV_CREATED: 1820 /* There are no legal states that come back to 1821 * created. This is the manually initialised start 1822 * state */ 1823 goto illegal; 1824 1825 case SDEV_RUNNING: 1826 switch (oldstate) { 1827 case SDEV_CREATED: 1828 case SDEV_OFFLINE: 1829 case SDEV_QUIESCE: 1830 case SDEV_BLOCK: 1831 break; 1832 default: 1833 goto illegal; 1834 } 1835 break; 1836 1837 case SDEV_QUIESCE: 1838 switch (oldstate) { 1839 case SDEV_RUNNING: 1840 case SDEV_OFFLINE: 1841 break; 1842 default: 1843 goto illegal; 1844 } 1845 break; 1846 1847 case SDEV_OFFLINE: 1848 switch (oldstate) { 1849 case SDEV_CREATED: 1850 case SDEV_RUNNING: 1851 case SDEV_QUIESCE: 1852 case SDEV_BLOCK: 1853 break; 1854 default: 1855 goto illegal; 1856 } 1857 break; 1858 1859 case SDEV_BLOCK: 1860 switch (oldstate) { 1861 case SDEV_CREATED: 1862 case SDEV_RUNNING: 1863 break; 1864 default: 1865 goto illegal; 1866 } 1867 break; 1868 1869 case SDEV_CANCEL: 1870 switch (oldstate) { 1871 case SDEV_CREATED: 1872 case SDEV_RUNNING: 1873 case SDEV_OFFLINE: 1874 case SDEV_BLOCK: 1875 break; 1876 default: 1877 goto illegal; 1878 } 1879 break; 1880 1881 case SDEV_DEL: 1882 switch (oldstate) { 1883 case SDEV_CANCEL: 1884 break; 1885 default: 1886 goto illegal; 1887 } 1888 break; 1889 1890 } 1891 sdev->sdev_state = state; 1892 return 0; 1893 1894 illegal: 1895 SCSI_LOG_ERROR_RECOVERY(1, 1896 dev_printk(KERN_ERR, &sdev->sdev_gendev, 1897 "Illegal state transition %s->%s\n", 1898 scsi_device_state_name(oldstate), 1899 scsi_device_state_name(state)) 1900 ); 1901 return -EINVAL; 1902} 1903EXPORT_SYMBOL(scsi_device_set_state); 1904 1905/** 1906 * scsi_device_quiesce - Block user issued commands. 1907 * @sdev: scsi device to quiesce. 1908 * 1909 * This works by trying to transition to the SDEV_QUIESCE state 1910 * (which must be a legal transition). When the device is in this 1911 * state, only special requests will be accepted, all others will 1912 * be deferred. Since special requests may also be requeued requests, 1913 * a successful return doesn't guarantee the device will be 1914 * totally quiescent. 1915 * 1916 * Must be called with user context, may sleep. 1917 * 1918 * Returns zero if unsuccessful or an error if not. 1919 **/ 1920int 1921scsi_device_quiesce(struct scsi_device *sdev) 1922{ 1923 int err = scsi_device_set_state(sdev, SDEV_QUIESCE); 1924 if (err) 1925 return err; 1926 1927 scsi_run_queue(sdev->request_queue); 1928 while (sdev->device_busy) { 1929 msleep_interruptible(200); 1930 scsi_run_queue(sdev->request_queue); 1931 } 1932 return 0; 1933} 1934EXPORT_SYMBOL(scsi_device_quiesce); 1935 1936/** 1937 * scsi_device_resume - Restart user issued commands to a quiesced device. 1938 * @sdev: scsi device to resume. 1939 * 1940 * Moves the device from quiesced back to running and restarts the 1941 * queues. 1942 * 1943 * Must be called with user context, may sleep. 1944 **/ 1945void 1946scsi_device_resume(struct scsi_device *sdev) 1947{ 1948 if(scsi_device_set_state(sdev, SDEV_RUNNING)) 1949 return; 1950 scsi_run_queue(sdev->request_queue); 1951} 1952EXPORT_SYMBOL(scsi_device_resume); 1953 1954static void 1955device_quiesce_fn(struct scsi_device *sdev, void *data) 1956{ 1957 scsi_device_quiesce(sdev); 1958} 1959 1960void 1961scsi_target_quiesce(struct scsi_target *starget) 1962{ 1963 starget_for_each_device(starget, NULL, device_quiesce_fn); 1964} 1965EXPORT_SYMBOL(scsi_target_quiesce); 1966 1967static void 1968device_resume_fn(struct scsi_device *sdev, void *data) 1969{ 1970 scsi_device_resume(sdev); 1971} 1972 1973void 1974scsi_target_resume(struct scsi_target *starget) 1975{ 1976 starget_for_each_device(starget, NULL, device_resume_fn); 1977} 1978EXPORT_SYMBOL(scsi_target_resume); 1979 1980/** 1981 * scsi_internal_device_block - internal function to put a device 1982 * temporarily into the SDEV_BLOCK state 1983 * @sdev: device to block 1984 * 1985 * Block request made by scsi lld's to temporarily stop all 1986 * scsi commands on the specified device. Called from interrupt 1987 * or normal process context. 1988 * 1989 * Returns zero if successful or error if not 1990 * 1991 * Notes: 1992 * This routine transitions the device to the SDEV_BLOCK state 1993 * (which must be a legal transition). When the device is in this 1994 * state, all commands are deferred until the scsi lld reenables 1995 * the device with scsi_device_unblock or device_block_tmo fires. 1996 * This routine assumes the host_lock is held on entry. 1997 **/ 1998int 1999scsi_internal_device_block(struct scsi_device *sdev) 2000{ 2001 request_queue_t *q = sdev->request_queue; 2002 unsigned long flags; 2003 int err = 0; 2004 2005 err = scsi_device_set_state(sdev, SDEV_BLOCK); 2006 if (err) 2007 return err; 2008 2009 /* 2010 * The device has transitioned to SDEV_BLOCK. Stop the 2011 * block layer from calling the midlayer with this device's 2012 * request queue. 2013 */ 2014 spin_lock_irqsave(q->queue_lock, flags); 2015 blk_stop_queue(q); 2016 spin_unlock_irqrestore(q->queue_lock, flags); 2017 2018 return 0; 2019} 2020EXPORT_SYMBOL_GPL(scsi_internal_device_block); 2021 2022/** 2023 * scsi_internal_device_unblock - resume a device after a block request 2024 * @sdev: device to resume 2025 * 2026 * Called by scsi lld's or the midlayer to restart the device queue 2027 * for the previously suspended scsi device. Called from interrupt or 2028 * normal process context. 2029 * 2030 * Returns zero if successful or error if not. 2031 * 2032 * Notes: 2033 * This routine transitions the device to the SDEV_RUNNING state 2034 * (which must be a legal transition) allowing the midlayer to 2035 * goose the queue for this device. This routine assumes the 2036 * host_lock is held upon entry. 2037 **/ 2038int 2039scsi_internal_device_unblock(struct scsi_device *sdev) 2040{ 2041 request_queue_t *q = sdev->request_queue; 2042 int err; 2043 unsigned long flags; 2044 2045 /* 2046 * Try to transition the scsi device to SDEV_RUNNING 2047 * and goose the device queue if successful. 2048 */ 2049 err = scsi_device_set_state(sdev, SDEV_RUNNING); 2050 if (err) 2051 return err; 2052 2053 spin_lock_irqsave(q->queue_lock, flags); 2054 blk_start_queue(q); 2055 spin_unlock_irqrestore(q->queue_lock, flags); 2056 2057 return 0; 2058} 2059EXPORT_SYMBOL_GPL(scsi_internal_device_unblock); 2060 2061static void 2062device_block(struct scsi_device *sdev, void *data) 2063{ 2064 scsi_internal_device_block(sdev); 2065} 2066 2067static int 2068target_block(struct device *dev, void *data) 2069{ 2070 if (scsi_is_target_device(dev)) 2071 starget_for_each_device(to_scsi_target(dev), NULL, 2072 device_block); 2073 return 0; 2074} 2075 2076void 2077scsi_target_block(struct device *dev) 2078{ 2079 if (scsi_is_target_device(dev)) 2080 starget_for_each_device(to_scsi_target(dev), NULL, 2081 device_block); 2082 else 2083 device_for_each_child(dev, NULL, target_block); 2084} 2085EXPORT_SYMBOL_GPL(scsi_target_block); 2086 2087static void 2088device_unblock(struct scsi_device *sdev, void *data) 2089{ 2090 scsi_internal_device_unblock(sdev); 2091} 2092 2093static int 2094target_unblock(struct device *dev, void *data) 2095{ 2096 if (scsi_is_target_device(dev)) 2097 starget_for_each_device(to_scsi_target(dev), NULL, 2098 device_unblock); 2099 return 0; 2100} 2101 2102void 2103scsi_target_unblock(struct device *dev) 2104{ 2105 if (scsi_is_target_device(dev)) 2106 starget_for_each_device(to_scsi_target(dev), NULL, 2107 device_unblock); 2108 else 2109 device_for_each_child(dev, NULL, target_unblock); 2110} 2111EXPORT_SYMBOL_GPL(scsi_target_unblock); 2112