scsi_lib.c revision 0d95716d6a1308c465d8c17ed1a217628936bb0c
1/*
2 *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
3 *
4 *  SCSI queueing library.
5 *      Initial versions: Eric Youngdale (eric@andante.org).
6 *                        Based upon conversations with large numbers
7 *                        of people at Linux Expo.
8 */
9
10#include <linux/bio.h>
11#include <linux/blkdev.h>
12#include <linux/completion.h>
13#include <linux/kernel.h>
14#include <linux/mempool.h>
15#include <linux/slab.h>
16#include <linux/init.h>
17#include <linux/pci.h>
18#include <linux/delay.h>
19
20#include <scsi/scsi.h>
21#include <scsi/scsi_dbg.h>
22#include <scsi/scsi_device.h>
23#include <scsi/scsi_driver.h>
24#include <scsi/scsi_eh.h>
25#include <scsi/scsi_host.h>
26#include <scsi/scsi_request.h>
27
28#include "scsi_priv.h"
29#include "scsi_logging.h"
30
31
32#define SG_MEMPOOL_NR		(sizeof(scsi_sg_pools)/sizeof(struct scsi_host_sg_pool))
33#define SG_MEMPOOL_SIZE		32
34
35struct scsi_host_sg_pool {
36	size_t		size;
37	char		*name;
38	kmem_cache_t	*slab;
39	mempool_t	*pool;
40};
41
42#if (SCSI_MAX_PHYS_SEGMENTS < 32)
43#error SCSI_MAX_PHYS_SEGMENTS is too small
44#endif
45
46#define SP(x) { x, "sgpool-" #x }
47static struct scsi_host_sg_pool scsi_sg_pools[] = {
48	SP(8),
49	SP(16),
50	SP(32),
51#if (SCSI_MAX_PHYS_SEGMENTS > 32)
52	SP(64),
53#if (SCSI_MAX_PHYS_SEGMENTS > 64)
54	SP(128),
55#if (SCSI_MAX_PHYS_SEGMENTS > 128)
56	SP(256),
57#if (SCSI_MAX_PHYS_SEGMENTS > 256)
58#error SCSI_MAX_PHYS_SEGMENTS is too large
59#endif
60#endif
61#endif
62#endif
63};
64#undef SP
65
66static void scsi_run_queue(struct request_queue *q);
67
68/*
69 * Function:	scsi_unprep_request()
70 *
71 * Purpose:	Remove all preparation done for a request, including its
72 *		associated scsi_cmnd, so that it can be requeued.
73 *
74 * Arguments:	req	- request to unprepare
75 *
76 * Lock status:	Assumed that no locks are held upon entry.
77 *
78 * Returns:	Nothing.
79 */
80static void scsi_unprep_request(struct request *req)
81{
82	struct scsi_cmnd *cmd = req->special;
83
84	req->flags &= ~REQ_DONTPREP;
85	req->special = (req->flags & REQ_SPECIAL) ? cmd->sc_request : NULL;
86
87	scsi_put_command(cmd);
88}
89
90/*
91 * Function:    scsi_queue_insert()
92 *
93 * Purpose:     Insert a command in the midlevel queue.
94 *
95 * Arguments:   cmd    - command that we are adding to queue.
96 *              reason - why we are inserting command to queue.
97 *
98 * Lock status: Assumed that lock is not held upon entry.
99 *
100 * Returns:     Nothing.
101 *
102 * Notes:       We do this for one of two cases.  Either the host is busy
103 *              and it cannot accept any more commands for the time being,
104 *              or the device returned QUEUE_FULL and can accept no more
105 *              commands.
106 * Notes:       This could be called either from an interrupt context or a
107 *              normal process context.
108 */
109int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
110{
111	struct Scsi_Host *host = cmd->device->host;
112	struct scsi_device *device = cmd->device;
113	struct request_queue *q = device->request_queue;
114	unsigned long flags;
115
116	SCSI_LOG_MLQUEUE(1,
117		 printk("Inserting command %p into mlqueue\n", cmd));
118
119	/*
120	 * Set the appropriate busy bit for the device/host.
121	 *
122	 * If the host/device isn't busy, assume that something actually
123	 * completed, and that we should be able to queue a command now.
124	 *
125	 * Note that the prior mid-layer assumption that any host could
126	 * always queue at least one command is now broken.  The mid-layer
127	 * will implement a user specifiable stall (see
128	 * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
129	 * if a command is requeued with no other commands outstanding
130	 * either for the device or for the host.
131	 */
132	if (reason == SCSI_MLQUEUE_HOST_BUSY)
133		host->host_blocked = host->max_host_blocked;
134	else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
135		device->device_blocked = device->max_device_blocked;
136
137	/*
138	 * Decrement the counters, since these commands are no longer
139	 * active on the host/device.
140	 */
141	scsi_device_unbusy(device);
142
143	/*
144	 * Requeue this command.  It will go before all other commands
145	 * that are already in the queue.
146	 *
147	 * NOTE: there is magic here about the way the queue is plugged if
148	 * we have no outstanding commands.
149	 *
150	 * Although we *don't* plug the queue, we call the request
151	 * function.  The SCSI request function detects the blocked condition
152	 * and plugs the queue appropriately.
153         */
154	spin_lock_irqsave(q->queue_lock, flags);
155	blk_requeue_request(q, cmd->request);
156	spin_unlock_irqrestore(q->queue_lock, flags);
157
158	scsi_run_queue(q);
159
160	return 0;
161}
162
163/*
164 * Function:    scsi_do_req
165 *
166 * Purpose:     Queue a SCSI request
167 *
168 * Arguments:   sreq	  - command descriptor.
169 *              cmnd      - actual SCSI command to be performed.
170 *              buffer    - data buffer.
171 *              bufflen   - size of data buffer.
172 *              done      - completion function to be run.
173 *              timeout   - how long to let it run before timeout.
174 *              retries   - number of retries we allow.
175 *
176 * Lock status: No locks held upon entry.
177 *
178 * Returns:     Nothing.
179 *
180 * Notes:	This function is only used for queueing requests for things
181 *		like ioctls and character device requests - this is because
182 *		we essentially just inject a request into the queue for the
183 *		device.
184 *
185 *		In order to support the scsi_device_quiesce function, we
186 *		now inject requests on the *head* of the device queue
187 *		rather than the tail.
188 */
189void scsi_do_req(struct scsi_request *sreq, const void *cmnd,
190		 void *buffer, unsigned bufflen,
191		 void (*done)(struct scsi_cmnd *),
192		 int timeout, int retries)
193{
194	/*
195	 * If the upper level driver is reusing these things, then
196	 * we should release the low-level block now.  Another one will
197	 * be allocated later when this request is getting queued.
198	 */
199	__scsi_release_request(sreq);
200
201	/*
202	 * Our own function scsi_done (which marks the host as not busy,
203	 * disables the timeout counter, etc) will be called by us or by the
204	 * scsi_hosts[host].queuecommand() function needs to also call
205	 * the completion function for the high level driver.
206	 */
207	memcpy(sreq->sr_cmnd, cmnd, sizeof(sreq->sr_cmnd));
208	sreq->sr_bufflen = bufflen;
209	sreq->sr_buffer = buffer;
210	sreq->sr_allowed = retries;
211	sreq->sr_done = done;
212	sreq->sr_timeout_per_command = timeout;
213
214	if (sreq->sr_cmd_len == 0)
215		sreq->sr_cmd_len = COMMAND_SIZE(sreq->sr_cmnd[0]);
216
217	/*
218	 * head injection *required* here otherwise quiesce won't work
219	 *
220	 * Because users of this function are apt to reuse requests with no
221	 * modification, we have to sanitise the request flags here
222	 */
223	sreq->sr_request->flags &= ~REQ_DONTPREP;
224	blk_insert_request(sreq->sr_device->request_queue, sreq->sr_request,
225		       	   1, sreq);
226}
227EXPORT_SYMBOL(scsi_do_req);
228
229/**
230 * scsi_execute - insert request and wait for the result
231 * @sdev:	scsi device
232 * @cmd:	scsi command
233 * @data_direction: data direction
234 * @buffer:	data buffer
235 * @bufflen:	len of buffer
236 * @sense:	optional sense buffer
237 * @timeout:	request timeout in seconds
238 * @retries:	number of times to retry request
239 * @flags:	or into request flags;
240 *
241 * returns the req->errors value which is the the scsi_cmnd result
242 * field.
243 **/
244int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
245		 int data_direction, void *buffer, unsigned bufflen,
246		 unsigned char *sense, int timeout, int retries, int flags)
247{
248	struct request *req;
249	int write = (data_direction == DMA_TO_DEVICE);
250	int ret = DRIVER_ERROR << 24;
251
252	req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
253
254	if (bufflen &&	blk_rq_map_kern(sdev->request_queue, req,
255					buffer, bufflen, __GFP_WAIT))
256		goto out;
257
258	req->cmd_len = COMMAND_SIZE(cmd[0]);
259	memcpy(req->cmd, cmd, req->cmd_len);
260	req->sense = sense;
261	req->sense_len = 0;
262	req->retries = retries;
263	req->timeout = timeout;
264	req->flags |= flags | REQ_BLOCK_PC | REQ_SPECIAL | REQ_QUIET;
265
266	/*
267	 * head injection *required* here otherwise quiesce won't work
268	 */
269	blk_execute_rq(req->q, NULL, req, 1);
270
271	ret = req->errors;
272 out:
273	blk_put_request(req);
274
275	return ret;
276}
277EXPORT_SYMBOL(scsi_execute);
278
279
280int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
281		     int data_direction, void *buffer, unsigned bufflen,
282		     struct scsi_sense_hdr *sshdr, int timeout, int retries)
283{
284	char *sense = NULL;
285	int result;
286
287	if (sshdr) {
288		sense = kmalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
289		if (!sense)
290			return DRIVER_ERROR << 24;
291		memset(sense, 0, SCSI_SENSE_BUFFERSIZE);
292	}
293	result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
294				  sense, timeout, retries, 0);
295	if (sshdr)
296		scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
297
298	kfree(sense);
299	return result;
300}
301EXPORT_SYMBOL(scsi_execute_req);
302
303struct scsi_io_context {
304	void *data;
305	void (*done)(void *data, char *sense, int result, int resid);
306	char sense[SCSI_SENSE_BUFFERSIZE];
307};
308
309static void scsi_end_async(struct request *req)
310{
311	struct scsi_io_context *sioc = req->end_io_data;
312
313	if (sioc->done)
314		sioc->done(sioc->data, sioc->sense, req->errors, req->data_len);
315
316	kfree(sioc);
317	__blk_put_request(req->q, req);
318}
319
320static int scsi_merge_bio(struct request *rq, struct bio *bio)
321{
322	struct request_queue *q = rq->q;
323
324	bio->bi_flags &= ~(1 << BIO_SEG_VALID);
325	if (rq_data_dir(rq) == WRITE)
326		bio->bi_rw |= (1 << BIO_RW);
327	blk_queue_bounce(q, &bio);
328
329	if (!rq->bio)
330		blk_rq_bio_prep(q, rq, bio);
331	else if (!q->back_merge_fn(q, rq, bio))
332		return -EINVAL;
333	else {
334		rq->biotail->bi_next = bio;
335		rq->biotail = bio;
336		rq->hard_nr_sectors += bio_sectors(bio);
337		rq->nr_sectors = rq->hard_nr_sectors;
338	}
339
340	return 0;
341}
342
343static int scsi_bi_endio(struct bio *bio, unsigned int bytes_done, int error)
344{
345	if (bio->bi_size)
346		return 1;
347
348	bio_put(bio);
349	return 0;
350}
351
352/**
353 * scsi_req_map_sg - map a scatterlist into a request
354 * @rq:		request to fill
355 * @sg:		scatterlist
356 * @nsegs:	number of elements
357 * @bufflen:	len of buffer
358 * @gfp:	memory allocation flags
359 *
360 * scsi_req_map_sg maps a scatterlist into a request so that the
361 * request can be sent to the block layer. We do not trust the scatterlist
362 * sent to use, as some ULDs use that struct to only organize the pages.
363 */
364static int scsi_req_map_sg(struct request *rq, struct scatterlist *sgl,
365			   int nsegs, unsigned bufflen, gfp_t gfp)
366{
367	struct request_queue *q = rq->q;
368	int nr_pages = (bufflen + PAGE_SIZE - 1) >> PAGE_SHIFT;
369	unsigned int data_len = 0, len, bytes, off;
370	struct page *page;
371	struct bio *bio = NULL;
372	int i, err, nr_vecs = 0;
373
374	for (i = 0; i < nsegs; i++) {
375		page = sgl[i].page;
376		off = sgl[i].offset;
377		len = sgl[i].length;
378		data_len += len;
379
380		while (len > 0) {
381			bytes = min_t(unsigned int, len, PAGE_SIZE - off);
382
383			if (!bio) {
384				nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages);
385				nr_pages -= nr_vecs;
386
387				bio = bio_alloc(gfp, nr_vecs);
388				if (!bio) {
389					err = -ENOMEM;
390					goto free_bios;
391				}
392				bio->bi_end_io = scsi_bi_endio;
393			}
394
395			if (bio_add_pc_page(q, bio, page, bytes, off) !=
396			    bytes) {
397				bio_put(bio);
398				err = -EINVAL;
399				goto free_bios;
400			}
401
402			if (bio->bi_vcnt >= nr_vecs) {
403				err = scsi_merge_bio(rq, bio);
404				if (err) {
405					bio_endio(bio, bio->bi_size, 0);
406					goto free_bios;
407				}
408				bio = NULL;
409			}
410
411			page++;
412			len -= bytes;
413			off = 0;
414		}
415	}
416
417	rq->buffer = rq->data = NULL;
418	rq->data_len = data_len;
419	return 0;
420
421free_bios:
422	while ((bio = rq->bio) != NULL) {
423		rq->bio = bio->bi_next;
424		/*
425		 * call endio instead of bio_put incase it was bounced
426		 */
427		bio_endio(bio, bio->bi_size, 0);
428	}
429
430	return err;
431}
432
433/**
434 * scsi_execute_async - insert request
435 * @sdev:	scsi device
436 * @cmd:	scsi command
437 * @data_direction: data direction
438 * @buffer:	data buffer (this can be a kernel buffer or scatterlist)
439 * @bufflen:	len of buffer
440 * @use_sg:	if buffer is a scatterlist this is the number of elements
441 * @timeout:	request timeout in seconds
442 * @retries:	number of times to retry request
443 * @flags:	or into request flags
444 **/
445int scsi_execute_async(struct scsi_device *sdev, const unsigned char *cmd,
446		       int data_direction, void *buffer, unsigned bufflen,
447		       int use_sg, int timeout, int retries, void *privdata,
448		       void (*done)(void *, char *, int, int), gfp_t gfp)
449{
450	struct request *req;
451	struct scsi_io_context *sioc;
452	int err = 0;
453	int write = (data_direction == DMA_TO_DEVICE);
454
455	sioc = kzalloc(sizeof(*sioc), gfp);
456	if (!sioc)
457		return DRIVER_ERROR << 24;
458
459	req = blk_get_request(sdev->request_queue, write, gfp);
460	if (!req)
461		goto free_sense;
462
463	if (use_sg)
464		err = scsi_req_map_sg(req, buffer, use_sg, bufflen, gfp);
465	else if (bufflen)
466		err = blk_rq_map_kern(req->q, req, buffer, bufflen, gfp);
467
468	if (err)
469		goto free_req;
470
471	req->cmd_len = COMMAND_SIZE(cmd[0]);
472	memcpy(req->cmd, cmd, req->cmd_len);
473	req->sense = sioc->sense;
474	req->sense_len = 0;
475	req->timeout = timeout;
476	req->retries = retries;
477	req->flags |= REQ_BLOCK_PC | REQ_QUIET;
478	req->end_io_data = sioc;
479
480	sioc->data = privdata;
481	sioc->done = done;
482
483	blk_execute_rq_nowait(req->q, NULL, req, 1, scsi_end_async);
484	return 0;
485
486free_req:
487	blk_put_request(req);
488free_sense:
489	kfree(sioc);
490	return DRIVER_ERROR << 24;
491}
492EXPORT_SYMBOL_GPL(scsi_execute_async);
493
494/*
495 * Function:    scsi_init_cmd_errh()
496 *
497 * Purpose:     Initialize cmd fields related to error handling.
498 *
499 * Arguments:   cmd	- command that is ready to be queued.
500 *
501 * Returns:     Nothing
502 *
503 * Notes:       This function has the job of initializing a number of
504 *              fields related to error handling.   Typically this will
505 *              be called once for each command, as required.
506 */
507static int scsi_init_cmd_errh(struct scsi_cmnd *cmd)
508{
509	cmd->serial_number = 0;
510
511	memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
512
513	if (cmd->cmd_len == 0)
514		cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
515
516	/*
517	 * We need saved copies of a number of fields - this is because
518	 * error handling may need to overwrite these with different values
519	 * to run different commands, and once error handling is complete,
520	 * we will need to restore these values prior to running the actual
521	 * command.
522	 */
523	cmd->old_use_sg = cmd->use_sg;
524	cmd->old_cmd_len = cmd->cmd_len;
525	cmd->sc_old_data_direction = cmd->sc_data_direction;
526	cmd->old_underflow = cmd->underflow;
527	memcpy(cmd->data_cmnd, cmd->cmnd, sizeof(cmd->cmnd));
528	cmd->buffer = cmd->request_buffer;
529	cmd->bufflen = cmd->request_bufflen;
530
531	return 1;
532}
533
534/*
535 * Function:   scsi_setup_cmd_retry()
536 *
537 * Purpose:    Restore the command state for a retry
538 *
539 * Arguments:  cmd	- command to be restored
540 *
541 * Returns:    Nothing
542 *
543 * Notes:      Immediately prior to retrying a command, we need
544 *             to restore certain fields that we saved above.
545 */
546void scsi_setup_cmd_retry(struct scsi_cmnd *cmd)
547{
548	memcpy(cmd->cmnd, cmd->data_cmnd, sizeof(cmd->data_cmnd));
549	cmd->request_buffer = cmd->buffer;
550	cmd->request_bufflen = cmd->bufflen;
551	cmd->use_sg = cmd->old_use_sg;
552	cmd->cmd_len = cmd->old_cmd_len;
553	cmd->sc_data_direction = cmd->sc_old_data_direction;
554	cmd->underflow = cmd->old_underflow;
555}
556
557void scsi_device_unbusy(struct scsi_device *sdev)
558{
559	struct Scsi_Host *shost = sdev->host;
560	unsigned long flags;
561
562	spin_lock_irqsave(shost->host_lock, flags);
563	shost->host_busy--;
564	if (unlikely(scsi_host_in_recovery(shost) &&
565		     shost->host_failed))
566		scsi_eh_wakeup(shost);
567	spin_unlock(shost->host_lock);
568	spin_lock(sdev->request_queue->queue_lock);
569	sdev->device_busy--;
570	spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
571}
572
573/*
574 * Called for single_lun devices on IO completion. Clear starget_sdev_user,
575 * and call blk_run_queue for all the scsi_devices on the target -
576 * including current_sdev first.
577 *
578 * Called with *no* scsi locks held.
579 */
580static void scsi_single_lun_run(struct scsi_device *current_sdev)
581{
582	struct Scsi_Host *shost = current_sdev->host;
583	struct scsi_device *sdev, *tmp;
584	struct scsi_target *starget = scsi_target(current_sdev);
585	unsigned long flags;
586
587	spin_lock_irqsave(shost->host_lock, flags);
588	starget->starget_sdev_user = NULL;
589	spin_unlock_irqrestore(shost->host_lock, flags);
590
591	/*
592	 * Call blk_run_queue for all LUNs on the target, starting with
593	 * current_sdev. We race with others (to set starget_sdev_user),
594	 * but in most cases, we will be first. Ideally, each LU on the
595	 * target would get some limited time or requests on the target.
596	 */
597	blk_run_queue(current_sdev->request_queue);
598
599	spin_lock_irqsave(shost->host_lock, flags);
600	if (starget->starget_sdev_user)
601		goto out;
602	list_for_each_entry_safe(sdev, tmp, &starget->devices,
603			same_target_siblings) {
604		if (sdev == current_sdev)
605			continue;
606		if (scsi_device_get(sdev))
607			continue;
608
609		spin_unlock_irqrestore(shost->host_lock, flags);
610		blk_run_queue(sdev->request_queue);
611		spin_lock_irqsave(shost->host_lock, flags);
612
613		scsi_device_put(sdev);
614	}
615 out:
616	spin_unlock_irqrestore(shost->host_lock, flags);
617}
618
619/*
620 * Function:	scsi_run_queue()
621 *
622 * Purpose:	Select a proper request queue to serve next
623 *
624 * Arguments:	q	- last request's queue
625 *
626 * Returns:     Nothing
627 *
628 * Notes:	The previous command was completely finished, start
629 *		a new one if possible.
630 */
631static void scsi_run_queue(struct request_queue *q)
632{
633	struct scsi_device *sdev = q->queuedata;
634	struct Scsi_Host *shost = sdev->host;
635	unsigned long flags;
636
637	if (sdev->single_lun)
638		scsi_single_lun_run(sdev);
639
640	spin_lock_irqsave(shost->host_lock, flags);
641	while (!list_empty(&shost->starved_list) &&
642	       !shost->host_blocked && !shost->host_self_blocked &&
643		!((shost->can_queue > 0) &&
644		  (shost->host_busy >= shost->can_queue))) {
645		/*
646		 * As long as shost is accepting commands and we have
647		 * starved queues, call blk_run_queue. scsi_request_fn
648		 * drops the queue_lock and can add us back to the
649		 * starved_list.
650		 *
651		 * host_lock protects the starved_list and starved_entry.
652		 * scsi_request_fn must get the host_lock before checking
653		 * or modifying starved_list or starved_entry.
654		 */
655		sdev = list_entry(shost->starved_list.next,
656					  struct scsi_device, starved_entry);
657		list_del_init(&sdev->starved_entry);
658		spin_unlock_irqrestore(shost->host_lock, flags);
659
660		blk_run_queue(sdev->request_queue);
661
662		spin_lock_irqsave(shost->host_lock, flags);
663		if (unlikely(!list_empty(&sdev->starved_entry)))
664			/*
665			 * sdev lost a race, and was put back on the
666			 * starved list. This is unlikely but without this
667			 * in theory we could loop forever.
668			 */
669			break;
670	}
671	spin_unlock_irqrestore(shost->host_lock, flags);
672
673	blk_run_queue(q);
674}
675
676/*
677 * Function:	scsi_requeue_command()
678 *
679 * Purpose:	Handle post-processing of completed commands.
680 *
681 * Arguments:	q	- queue to operate on
682 *		cmd	- command that may need to be requeued.
683 *
684 * Returns:	Nothing
685 *
686 * Notes:	After command completion, there may be blocks left
687 *		over which weren't finished by the previous command
688 *		this can be for a number of reasons - the main one is
689 *		I/O errors in the middle of the request, in which case
690 *		we need to request the blocks that come after the bad
691 *		sector.
692 * Notes:	Upon return, cmd is a stale pointer.
693 */
694static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
695{
696	struct request *req = cmd->request;
697	unsigned long flags;
698
699	scsi_unprep_request(req);
700	spin_lock_irqsave(q->queue_lock, flags);
701	blk_requeue_request(q, req);
702	spin_unlock_irqrestore(q->queue_lock, flags);
703
704	scsi_run_queue(q);
705}
706
707void scsi_next_command(struct scsi_cmnd *cmd)
708{
709	struct scsi_device *sdev = cmd->device;
710	struct request_queue *q = sdev->request_queue;
711
712	/* need to hold a reference on the device before we let go of the cmd */
713	get_device(&sdev->sdev_gendev);
714
715	scsi_put_command(cmd);
716	scsi_run_queue(q);
717
718	/* ok to remove device now */
719	put_device(&sdev->sdev_gendev);
720}
721
722void scsi_run_host_queues(struct Scsi_Host *shost)
723{
724	struct scsi_device *sdev;
725
726	shost_for_each_device(sdev, shost)
727		scsi_run_queue(sdev->request_queue);
728}
729
730/*
731 * Function:    scsi_end_request()
732 *
733 * Purpose:     Post-processing of completed commands (usually invoked at end
734 *		of upper level post-processing and scsi_io_completion).
735 *
736 * Arguments:   cmd	 - command that is complete.
737 *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
738 *              bytes    - number of bytes of completed I/O
739 *		requeue  - indicates whether we should requeue leftovers.
740 *
741 * Lock status: Assumed that lock is not held upon entry.
742 *
743 * Returns:     cmd if requeue required, NULL otherwise.
744 *
745 * Notes:       This is called for block device requests in order to
746 *              mark some number of sectors as complete.
747 *
748 *		We are guaranteeing that the request queue will be goosed
749 *		at some point during this call.
750 * Notes:	If cmd was requeued, upon return it will be a stale pointer.
751 */
752static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
753					  int bytes, int requeue)
754{
755	request_queue_t *q = cmd->device->request_queue;
756	struct request *req = cmd->request;
757	unsigned long flags;
758
759	/*
760	 * If there are blocks left over at the end, set up the command
761	 * to queue the remainder of them.
762	 */
763	if (end_that_request_chunk(req, uptodate, bytes)) {
764		int leftover = (req->hard_nr_sectors << 9);
765
766		if (blk_pc_request(req))
767			leftover = req->data_len;
768
769		/* kill remainder if no retrys */
770		if (!uptodate && blk_noretry_request(req))
771			end_that_request_chunk(req, 0, leftover);
772		else {
773			if (requeue) {
774				/*
775				 * Bleah.  Leftovers again.  Stick the
776				 * leftovers in the front of the
777				 * queue, and goose the queue again.
778				 */
779				scsi_requeue_command(q, cmd);
780				cmd = NULL;
781			}
782			return cmd;
783		}
784	}
785
786	add_disk_randomness(req->rq_disk);
787
788	spin_lock_irqsave(q->queue_lock, flags);
789	if (blk_rq_tagged(req))
790		blk_queue_end_tag(q, req);
791	end_that_request_last(req);
792	spin_unlock_irqrestore(q->queue_lock, flags);
793
794	/*
795	 * This will goose the queue request function at the end, so we don't
796	 * need to worry about launching another command.
797	 */
798	scsi_next_command(cmd);
799	return NULL;
800}
801
802static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, gfp_t gfp_mask)
803{
804	struct scsi_host_sg_pool *sgp;
805	struct scatterlist *sgl;
806
807	BUG_ON(!cmd->use_sg);
808
809	switch (cmd->use_sg) {
810	case 1 ... 8:
811		cmd->sglist_len = 0;
812		break;
813	case 9 ... 16:
814		cmd->sglist_len = 1;
815		break;
816	case 17 ... 32:
817		cmd->sglist_len = 2;
818		break;
819#if (SCSI_MAX_PHYS_SEGMENTS > 32)
820	case 33 ... 64:
821		cmd->sglist_len = 3;
822		break;
823#if (SCSI_MAX_PHYS_SEGMENTS > 64)
824	case 65 ... 128:
825		cmd->sglist_len = 4;
826		break;
827#if (SCSI_MAX_PHYS_SEGMENTS  > 128)
828	case 129 ... 256:
829		cmd->sglist_len = 5;
830		break;
831#endif
832#endif
833#endif
834	default:
835		return NULL;
836	}
837
838	sgp = scsi_sg_pools + cmd->sglist_len;
839	sgl = mempool_alloc(sgp->pool, gfp_mask);
840	return sgl;
841}
842
843static void scsi_free_sgtable(struct scatterlist *sgl, int index)
844{
845	struct scsi_host_sg_pool *sgp;
846
847	BUG_ON(index >= SG_MEMPOOL_NR);
848
849	sgp = scsi_sg_pools + index;
850	mempool_free(sgl, sgp->pool);
851}
852
853/*
854 * Function:    scsi_release_buffers()
855 *
856 * Purpose:     Completion processing for block device I/O requests.
857 *
858 * Arguments:   cmd	- command that we are bailing.
859 *
860 * Lock status: Assumed that no lock is held upon entry.
861 *
862 * Returns:     Nothing
863 *
864 * Notes:       In the event that an upper level driver rejects a
865 *		command, we must release resources allocated during
866 *		the __init_io() function.  Primarily this would involve
867 *		the scatter-gather table, and potentially any bounce
868 *		buffers.
869 */
870static void scsi_release_buffers(struct scsi_cmnd *cmd)
871{
872	struct request *req = cmd->request;
873
874	/*
875	 * Free up any indirection buffers we allocated for DMA purposes.
876	 */
877	if (cmd->use_sg)
878		scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len);
879	else if (cmd->request_buffer != req->buffer)
880		kfree(cmd->request_buffer);
881
882	/*
883	 * Zero these out.  They now point to freed memory, and it is
884	 * dangerous to hang onto the pointers.
885	 */
886	cmd->buffer  = NULL;
887	cmd->bufflen = 0;
888	cmd->request_buffer = NULL;
889	cmd->request_bufflen = 0;
890}
891
892/*
893 * Function:    scsi_io_completion()
894 *
895 * Purpose:     Completion processing for block device I/O requests.
896 *
897 * Arguments:   cmd   - command that is finished.
898 *
899 * Lock status: Assumed that no lock is held upon entry.
900 *
901 * Returns:     Nothing
902 *
903 * Notes:       This function is matched in terms of capabilities to
904 *              the function that created the scatter-gather list.
905 *              In other words, if there are no bounce buffers
906 *              (the normal case for most drivers), we don't need
907 *              the logic to deal with cleaning up afterwards.
908 *
909 *		We must do one of several things here:
910 *
911 *		a) Call scsi_end_request.  This will finish off the
912 *		   specified number of sectors.  If we are done, the
913 *		   command block will be released, and the queue
914 *		   function will be goosed.  If we are not done, then
915 *		   scsi_end_request will directly goose the queue.
916 *
917 *		b) We can just use scsi_requeue_command() here.  This would
918 *		   be used if we just wanted to retry, for example.
919 */
920void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes,
921			unsigned int block_bytes)
922{
923	int result = cmd->result;
924	int this_count = cmd->bufflen;
925	request_queue_t *q = cmd->device->request_queue;
926	struct request *req = cmd->request;
927	int clear_errors = 1;
928	struct scsi_sense_hdr sshdr;
929	int sense_valid = 0;
930	int sense_deferred = 0;
931
932	if (blk_complete_barrier_rq(q, req, good_bytes >> 9))
933		return;
934
935	/*
936	 * Free up any indirection buffers we allocated for DMA purposes.
937	 * For the case of a READ, we need to copy the data out of the
938	 * bounce buffer and into the real buffer.
939	 */
940	if (cmd->use_sg)
941		scsi_free_sgtable(cmd->buffer, cmd->sglist_len);
942	else if (cmd->buffer != req->buffer) {
943		if (rq_data_dir(req) == READ) {
944			unsigned long flags;
945			char *to = bio_kmap_irq(req->bio, &flags);
946			memcpy(to, cmd->buffer, cmd->bufflen);
947			bio_kunmap_irq(to, &flags);
948		}
949		kfree(cmd->buffer);
950	}
951
952	if (result) {
953		sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
954		if (sense_valid)
955			sense_deferred = scsi_sense_is_deferred(&sshdr);
956	}
957	if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
958		req->errors = result;
959		if (result) {
960			clear_errors = 0;
961			if (sense_valid && req->sense) {
962				/*
963				 * SG_IO wants current and deferred errors
964				 */
965				int len = 8 + cmd->sense_buffer[7];
966
967				if (len > SCSI_SENSE_BUFFERSIZE)
968					len = SCSI_SENSE_BUFFERSIZE;
969				memcpy(req->sense, cmd->sense_buffer,  len);
970				req->sense_len = len;
971			}
972		} else
973			req->data_len = cmd->resid;
974	}
975
976	/*
977	 * Zero these out.  They now point to freed memory, and it is
978	 * dangerous to hang onto the pointers.
979	 */
980	cmd->buffer  = NULL;
981	cmd->bufflen = 0;
982	cmd->request_buffer = NULL;
983	cmd->request_bufflen = 0;
984
985	/*
986	 * Next deal with any sectors which we were able to correctly
987	 * handle.
988	 */
989	if (good_bytes >= 0) {
990		SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, %d bytes done.\n",
991					      req->nr_sectors, good_bytes));
992		SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
993
994		if (clear_errors)
995			req->errors = 0;
996		/*
997		 * If multiple sectors are requested in one buffer, then
998		 * they will have been finished off by the first command.
999		 * If not, then we have a multi-buffer command.
1000		 *
1001		 * If block_bytes != 0, it means we had a medium error
1002		 * of some sort, and that we want to mark some number of
1003		 * sectors as not uptodate.  Thus we want to inhibit
1004		 * requeueing right here - we will requeue down below
1005		 * when we handle the bad sectors.
1006		 */
1007
1008		/*
1009		 * If the command completed without error, then either
1010		 * finish off the rest of the command, or start a new one.
1011		 */
1012		if (scsi_end_request(cmd, 1, good_bytes, result == 0) == NULL)
1013			return;
1014	}
1015	/*
1016	 * Now, if we were good little boys and girls, Santa left us a request
1017	 * sense buffer.  We can extract information from this, so we
1018	 * can choose a block to remap, etc.
1019	 */
1020	if (sense_valid && !sense_deferred) {
1021		switch (sshdr.sense_key) {
1022		case UNIT_ATTENTION:
1023			if (cmd->device->removable) {
1024				/* detected disc change.  set a bit
1025				 * and quietly refuse further access.
1026				 */
1027				cmd->device->changed = 1;
1028				scsi_end_request(cmd, 0,
1029						this_count, 1);
1030				return;
1031			} else {
1032				/*
1033				* Must have been a power glitch, or a
1034				* bus reset.  Could not have been a
1035				* media change, so we just retry the
1036				* request and see what happens.
1037				*/
1038				scsi_requeue_command(q, cmd);
1039				return;
1040			}
1041			break;
1042		case ILLEGAL_REQUEST:
1043			/*
1044		 	* If we had an ILLEGAL REQUEST returned, then we may
1045		 	* have performed an unsupported command.  The only
1046		 	* thing this should be would be a ten byte read where
1047			* only a six byte read was supported.  Also, on a
1048			* system where READ CAPACITY failed, we may have read
1049			* past the end of the disk.
1050		 	*/
1051			if ((cmd->device->use_10_for_rw &&
1052			    sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
1053			    (cmd->cmnd[0] == READ_10 ||
1054			     cmd->cmnd[0] == WRITE_10)) {
1055				cmd->device->use_10_for_rw = 0;
1056				/*
1057				 * This will cause a retry with a 6-byte
1058				 * command.
1059				 */
1060				scsi_requeue_command(q, cmd);
1061				result = 0;
1062			} else {
1063				scsi_end_request(cmd, 0, this_count, 1);
1064				return;
1065			}
1066			break;
1067		case NOT_READY:
1068			/*
1069			 * If the device is in the process of becoming ready,
1070			 * retry.
1071			 */
1072			if (sshdr.asc == 0x04 && sshdr.ascq == 0x01) {
1073				scsi_requeue_command(q, cmd);
1074				return;
1075			}
1076			if (!(req->flags & REQ_QUIET))
1077				scmd_printk(KERN_INFO, cmd,
1078					   "Device not ready.\n");
1079			scsi_end_request(cmd, 0, this_count, 1);
1080			return;
1081		case VOLUME_OVERFLOW:
1082			if (!(req->flags & REQ_QUIET)) {
1083				scmd_printk(KERN_INFO, cmd,
1084					   "Volume overflow, CDB: ");
1085				__scsi_print_command(cmd->data_cmnd);
1086				scsi_print_sense("", cmd);
1087			}
1088			scsi_end_request(cmd, 0, block_bytes, 1);
1089			return;
1090		default:
1091			break;
1092		}
1093	}			/* driver byte != 0 */
1094	if (host_byte(result) == DID_RESET) {
1095		/*
1096		 * Third party bus reset or reset for error
1097		 * recovery reasons.  Just retry the request
1098		 * and see what happens.
1099		 */
1100		scsi_requeue_command(q, cmd);
1101		return;
1102	}
1103	if (result) {
1104		if (!(req->flags & REQ_QUIET)) {
1105			scmd_printk(KERN_INFO, cmd,
1106				   "SCSI error: return code = 0x%x\n", result);
1107
1108			if (driver_byte(result) & DRIVER_SENSE)
1109				scsi_print_sense("", cmd);
1110		}
1111		/*
1112		 * Mark a single buffer as not uptodate.  Queue the remainder.
1113		 * We sometimes get this cruft in the event that a medium error
1114		 * isn't properly reported.
1115		 */
1116		block_bytes = req->hard_cur_sectors << 9;
1117		if (!block_bytes)
1118			block_bytes = req->data_len;
1119		scsi_end_request(cmd, 0, block_bytes, 1);
1120	}
1121}
1122EXPORT_SYMBOL(scsi_io_completion);
1123
1124/*
1125 * Function:    scsi_init_io()
1126 *
1127 * Purpose:     SCSI I/O initialize function.
1128 *
1129 * Arguments:   cmd   - Command descriptor we wish to initialize
1130 *
1131 * Returns:     0 on success
1132 *		BLKPREP_DEFER if the failure is retryable
1133 *		BLKPREP_KILL if the failure is fatal
1134 */
1135static int scsi_init_io(struct scsi_cmnd *cmd)
1136{
1137	struct request     *req = cmd->request;
1138	struct scatterlist *sgpnt;
1139	int		   count;
1140
1141	/*
1142	 * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer
1143	 */
1144	if ((req->flags & REQ_BLOCK_PC) && !req->bio) {
1145		cmd->request_bufflen = req->data_len;
1146		cmd->request_buffer = req->data;
1147		req->buffer = req->data;
1148		cmd->use_sg = 0;
1149		return 0;
1150	}
1151
1152	/*
1153	 * we used to not use scatter-gather for single segment request,
1154	 * but now we do (it makes highmem I/O easier to support without
1155	 * kmapping pages)
1156	 */
1157	cmd->use_sg = req->nr_phys_segments;
1158
1159	/*
1160	 * if sg table allocation fails, requeue request later.
1161	 */
1162	sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
1163	if (unlikely(!sgpnt)) {
1164		scsi_unprep_request(req);
1165		return BLKPREP_DEFER;
1166	}
1167
1168	cmd->request_buffer = (char *) sgpnt;
1169	cmd->request_bufflen = req->nr_sectors << 9;
1170	if (blk_pc_request(req))
1171		cmd->request_bufflen = req->data_len;
1172	req->buffer = NULL;
1173
1174	/*
1175	 * Next, walk the list, and fill in the addresses and sizes of
1176	 * each segment.
1177	 */
1178	count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
1179
1180	/*
1181	 * mapped well, send it off
1182	 */
1183	if (likely(count <= cmd->use_sg)) {
1184		cmd->use_sg = count;
1185		return 0;
1186	}
1187
1188	printk(KERN_ERR "Incorrect number of segments after building list\n");
1189	printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg);
1190	printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors,
1191			req->current_nr_sectors);
1192
1193	/* release the command and kill it */
1194	scsi_release_buffers(cmd);
1195	scsi_put_command(cmd);
1196	return BLKPREP_KILL;
1197}
1198
1199static int scsi_prepare_flush_fn(request_queue_t *q, struct request *rq)
1200{
1201	struct scsi_device *sdev = q->queuedata;
1202	struct scsi_driver *drv;
1203
1204	if (sdev->sdev_state == SDEV_RUNNING) {
1205		drv = *(struct scsi_driver **) rq->rq_disk->private_data;
1206
1207		if (drv->prepare_flush)
1208			return drv->prepare_flush(q, rq);
1209	}
1210
1211	return 0;
1212}
1213
1214static void scsi_end_flush_fn(request_queue_t *q, struct request *rq)
1215{
1216	struct scsi_device *sdev = q->queuedata;
1217	struct request *flush_rq = rq->end_io_data;
1218	struct scsi_driver *drv;
1219
1220	if (flush_rq->errors) {
1221		printk("scsi: barrier error, disabling flush support\n");
1222		blk_queue_ordered(q, QUEUE_ORDERED_NONE);
1223	}
1224
1225	if (sdev->sdev_state == SDEV_RUNNING) {
1226		drv = *(struct scsi_driver **) rq->rq_disk->private_data;
1227		drv->end_flush(q, rq);
1228	}
1229}
1230
1231static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk,
1232			       sector_t *error_sector)
1233{
1234	struct scsi_device *sdev = q->queuedata;
1235	struct scsi_driver *drv;
1236
1237	if (sdev->sdev_state != SDEV_RUNNING)
1238		return -ENXIO;
1239
1240	drv = *(struct scsi_driver **) disk->private_data;
1241	if (drv->issue_flush)
1242		return drv->issue_flush(&sdev->sdev_gendev, error_sector);
1243
1244	return -EOPNOTSUPP;
1245}
1246
1247static void scsi_generic_done(struct scsi_cmnd *cmd)
1248{
1249	BUG_ON(!blk_pc_request(cmd->request));
1250	/*
1251	 * This will complete the whole command with uptodate=1 so
1252	 * as far as the block layer is concerned the command completed
1253	 * successfully. Since this is a REQ_BLOCK_PC command the
1254	 * caller should check the request's errors value
1255	 */
1256	scsi_io_completion(cmd, cmd->bufflen, 0);
1257}
1258
1259static int scsi_prep_fn(struct request_queue *q, struct request *req)
1260{
1261	struct scsi_device *sdev = q->queuedata;
1262	struct scsi_cmnd *cmd;
1263	int specials_only = 0;
1264
1265	/*
1266	 * Just check to see if the device is online.  If it isn't, we
1267	 * refuse to process any commands.  The device must be brought
1268	 * online before trying any recovery commands
1269	 */
1270	if (unlikely(!scsi_device_online(sdev))) {
1271		sdev_printk(KERN_ERR, sdev,
1272			    "rejecting I/O to offline device\n");
1273		goto kill;
1274	}
1275	if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1276		/* OK, we're not in a running state don't prep
1277		 * user commands */
1278		if (sdev->sdev_state == SDEV_DEL) {
1279			/* Device is fully deleted, no commands
1280			 * at all allowed down */
1281			sdev_printk(KERN_ERR, sdev,
1282				    "rejecting I/O to dead device\n");
1283			goto kill;
1284		}
1285		/* OK, we only allow special commands (i.e. not
1286		 * user initiated ones */
1287		specials_only = sdev->sdev_state;
1288	}
1289
1290	/*
1291	 * Find the actual device driver associated with this command.
1292	 * The SPECIAL requests are things like character device or
1293	 * ioctls, which did not originate from ll_rw_blk.  Note that
1294	 * the special field is also used to indicate the cmd for
1295	 * the remainder of a partially fulfilled request that can
1296	 * come up when there is a medium error.  We have to treat
1297	 * these two cases differently.  We differentiate by looking
1298	 * at request->cmd, as this tells us the real story.
1299	 */
1300	if (req->flags & REQ_SPECIAL && req->special) {
1301		struct scsi_request *sreq = req->special;
1302
1303		if (sreq->sr_magic == SCSI_REQ_MAGIC) {
1304			cmd = scsi_get_command(sreq->sr_device, GFP_ATOMIC);
1305			if (unlikely(!cmd))
1306				goto defer;
1307			scsi_init_cmd_from_req(cmd, sreq);
1308		} else
1309			cmd = req->special;
1310	} else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1311
1312		if(unlikely(specials_only) && !(req->flags & REQ_SPECIAL)) {
1313			if(specials_only == SDEV_QUIESCE ||
1314					specials_only == SDEV_BLOCK)
1315				goto defer;
1316
1317			sdev_printk(KERN_ERR, sdev,
1318				    "rejecting I/O to device being removed\n");
1319			goto kill;
1320		}
1321
1322
1323		/*
1324		 * Now try and find a command block that we can use.
1325		 */
1326		if (!req->special) {
1327			cmd = scsi_get_command(sdev, GFP_ATOMIC);
1328			if (unlikely(!cmd))
1329				goto defer;
1330		} else
1331			cmd = req->special;
1332
1333		/* pull a tag out of the request if we have one */
1334		cmd->tag = req->tag;
1335	} else {
1336		blk_dump_rq_flags(req, "SCSI bad req");
1337		goto kill;
1338	}
1339
1340	/* note the overloading of req->special.  When the tag
1341	 * is active it always means cmd.  If the tag goes
1342	 * back for re-queueing, it may be reset */
1343	req->special = cmd;
1344	cmd->request = req;
1345
1346	/*
1347	 * FIXME: drop the lock here because the functions below
1348	 * expect to be called without the queue lock held.  Also,
1349	 * previously, we dequeued the request before dropping the
1350	 * lock.  We hope REQ_STARTED prevents anything untoward from
1351	 * happening now.
1352	 */
1353	if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1354		struct scsi_driver *drv;
1355		int ret;
1356
1357		/*
1358		 * This will do a couple of things:
1359		 *  1) Fill in the actual SCSI command.
1360		 *  2) Fill in any other upper-level specific fields
1361		 * (timeout).
1362		 *
1363		 * If this returns 0, it means that the request failed
1364		 * (reading past end of disk, reading offline device,
1365		 * etc).   This won't actually talk to the device, but
1366		 * some kinds of consistency checking may cause the
1367		 * request to be rejected immediately.
1368		 */
1369
1370		/*
1371		 * This sets up the scatter-gather table (allocating if
1372		 * required).
1373		 */
1374		ret = scsi_init_io(cmd);
1375		switch(ret) {
1376			/* For BLKPREP_KILL/DEFER the cmd was released */
1377		case BLKPREP_KILL:
1378			goto kill;
1379		case BLKPREP_DEFER:
1380			goto defer;
1381		}
1382
1383		/*
1384		 * Initialize the actual SCSI command for this request.
1385		 */
1386		if (req->rq_disk) {
1387			drv = *(struct scsi_driver **)req->rq_disk->private_data;
1388			if (unlikely(!drv->init_command(cmd))) {
1389				scsi_release_buffers(cmd);
1390				scsi_put_command(cmd);
1391				goto kill;
1392			}
1393		} else {
1394			memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd));
1395			cmd->cmd_len = req->cmd_len;
1396			if (rq_data_dir(req) == WRITE)
1397				cmd->sc_data_direction = DMA_TO_DEVICE;
1398			else if (req->data_len)
1399				cmd->sc_data_direction = DMA_FROM_DEVICE;
1400			else
1401				cmd->sc_data_direction = DMA_NONE;
1402
1403			cmd->transfersize = req->data_len;
1404			cmd->allowed = req->retries;
1405			cmd->timeout_per_command = req->timeout;
1406			cmd->done = scsi_generic_done;
1407		}
1408	}
1409
1410	/*
1411	 * The request is now prepped, no need to come back here
1412	 */
1413	req->flags |= REQ_DONTPREP;
1414	return BLKPREP_OK;
1415
1416 defer:
1417	/* If we defer, the elv_next_request() returns NULL, but the
1418	 * queue must be restarted, so we plug here if no returning
1419	 * command will automatically do that. */
1420	if (sdev->device_busy == 0)
1421		blk_plug_device(q);
1422	return BLKPREP_DEFER;
1423 kill:
1424	req->errors = DID_NO_CONNECT << 16;
1425	return BLKPREP_KILL;
1426}
1427
1428/*
1429 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1430 * return 0.
1431 *
1432 * Called with the queue_lock held.
1433 */
1434static inline int scsi_dev_queue_ready(struct request_queue *q,
1435				  struct scsi_device *sdev)
1436{
1437	if (sdev->device_busy >= sdev->queue_depth)
1438		return 0;
1439	if (sdev->device_busy == 0 && sdev->device_blocked) {
1440		/*
1441		 * unblock after device_blocked iterates to zero
1442		 */
1443		if (--sdev->device_blocked == 0) {
1444			SCSI_LOG_MLQUEUE(3,
1445				   sdev_printk(KERN_INFO, sdev,
1446				   "unblocking device at zero depth\n"));
1447		} else {
1448			blk_plug_device(q);
1449			return 0;
1450		}
1451	}
1452	if (sdev->device_blocked)
1453		return 0;
1454
1455	return 1;
1456}
1457
1458/*
1459 * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1460 * return 0. We must end up running the queue again whenever 0 is
1461 * returned, else IO can hang.
1462 *
1463 * Called with host_lock held.
1464 */
1465static inline int scsi_host_queue_ready(struct request_queue *q,
1466				   struct Scsi_Host *shost,
1467				   struct scsi_device *sdev)
1468{
1469	if (scsi_host_in_recovery(shost))
1470		return 0;
1471	if (shost->host_busy == 0 && shost->host_blocked) {
1472		/*
1473		 * unblock after host_blocked iterates to zero
1474		 */
1475		if (--shost->host_blocked == 0) {
1476			SCSI_LOG_MLQUEUE(3,
1477				printk("scsi%d unblocking host at zero depth\n",
1478					shost->host_no));
1479		} else {
1480			blk_plug_device(q);
1481			return 0;
1482		}
1483	}
1484	if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
1485	    shost->host_blocked || shost->host_self_blocked) {
1486		if (list_empty(&sdev->starved_entry))
1487			list_add_tail(&sdev->starved_entry, &shost->starved_list);
1488		return 0;
1489	}
1490
1491	/* We're OK to process the command, so we can't be starved */
1492	if (!list_empty(&sdev->starved_entry))
1493		list_del_init(&sdev->starved_entry);
1494
1495	return 1;
1496}
1497
1498/*
1499 * Kill a request for a dead device
1500 */
1501static void scsi_kill_request(struct request *req, request_queue_t *q)
1502{
1503	struct scsi_cmnd *cmd = req->special;
1504
1505	blkdev_dequeue_request(req);
1506
1507	if (unlikely(cmd == NULL)) {
1508		printk(KERN_CRIT "impossible request in %s.\n",
1509				 __FUNCTION__);
1510		BUG();
1511	}
1512
1513	scsi_init_cmd_errh(cmd);
1514	cmd->result = DID_NO_CONNECT << 16;
1515	atomic_inc(&cmd->device->iorequest_cnt);
1516	__scsi_done(cmd);
1517}
1518
1519/*
1520 * Function:    scsi_request_fn()
1521 *
1522 * Purpose:     Main strategy routine for SCSI.
1523 *
1524 * Arguments:   q       - Pointer to actual queue.
1525 *
1526 * Returns:     Nothing
1527 *
1528 * Lock status: IO request lock assumed to be held when called.
1529 */
1530static void scsi_request_fn(struct request_queue *q)
1531{
1532	struct scsi_device *sdev = q->queuedata;
1533	struct Scsi_Host *shost;
1534	struct scsi_cmnd *cmd;
1535	struct request *req;
1536
1537	if (!sdev) {
1538		printk("scsi: killing requests for dead queue\n");
1539		while ((req = elv_next_request(q)) != NULL)
1540			scsi_kill_request(req, q);
1541		return;
1542	}
1543
1544	if(!get_device(&sdev->sdev_gendev))
1545		/* We must be tearing the block queue down already */
1546		return;
1547
1548	/*
1549	 * To start with, we keep looping until the queue is empty, or until
1550	 * the host is no longer able to accept any more requests.
1551	 */
1552	shost = sdev->host;
1553	while (!blk_queue_plugged(q)) {
1554		int rtn;
1555		/*
1556		 * get next queueable request.  We do this early to make sure
1557		 * that the request is fully prepared even if we cannot
1558		 * accept it.
1559		 */
1560		req = elv_next_request(q);
1561		if (!req || !scsi_dev_queue_ready(q, sdev))
1562			break;
1563
1564		if (unlikely(!scsi_device_online(sdev))) {
1565			sdev_printk(KERN_ERR, sdev,
1566				    "rejecting I/O to offline device\n");
1567			scsi_kill_request(req, q);
1568			continue;
1569		}
1570
1571
1572		/*
1573		 * Remove the request from the request list.
1574		 */
1575		if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1576			blkdev_dequeue_request(req);
1577		sdev->device_busy++;
1578
1579		spin_unlock(q->queue_lock);
1580		cmd = req->special;
1581		if (unlikely(cmd == NULL)) {
1582			printk(KERN_CRIT "impossible request in %s.\n"
1583					 "please mail a stack trace to "
1584					 "linux-scsi@vger.kernel.org",
1585					 __FUNCTION__);
1586			BUG();
1587		}
1588		spin_lock(shost->host_lock);
1589
1590		if (!scsi_host_queue_ready(q, shost, sdev))
1591			goto not_ready;
1592		if (sdev->single_lun) {
1593			if (scsi_target(sdev)->starget_sdev_user &&
1594			    scsi_target(sdev)->starget_sdev_user != sdev)
1595				goto not_ready;
1596			scsi_target(sdev)->starget_sdev_user = sdev;
1597		}
1598		shost->host_busy++;
1599
1600		/*
1601		 * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1602		 *		take the lock again.
1603		 */
1604		spin_unlock_irq(shost->host_lock);
1605
1606		/*
1607		 * Finally, initialize any error handling parameters, and set up
1608		 * the timers for timeouts.
1609		 */
1610		scsi_init_cmd_errh(cmd);
1611
1612		/*
1613		 * Dispatch the command to the low-level driver.
1614		 */
1615		rtn = scsi_dispatch_cmd(cmd);
1616		spin_lock_irq(q->queue_lock);
1617		if(rtn) {
1618			/* we're refusing the command; because of
1619			 * the way locks get dropped, we need to
1620			 * check here if plugging is required */
1621			if(sdev->device_busy == 0)
1622				blk_plug_device(q);
1623
1624			break;
1625		}
1626	}
1627
1628	goto out;
1629
1630 not_ready:
1631	spin_unlock_irq(shost->host_lock);
1632
1633	/*
1634	 * lock q, handle tag, requeue req, and decrement device_busy. We
1635	 * must return with queue_lock held.
1636	 *
1637	 * Decrementing device_busy without checking it is OK, as all such
1638	 * cases (host limits or settings) should run the queue at some
1639	 * later time.
1640	 */
1641	spin_lock_irq(q->queue_lock);
1642	blk_requeue_request(q, req);
1643	sdev->device_busy--;
1644	if(sdev->device_busy == 0)
1645		blk_plug_device(q);
1646 out:
1647	/* must be careful here...if we trigger the ->remove() function
1648	 * we cannot be holding the q lock */
1649	spin_unlock_irq(q->queue_lock);
1650	put_device(&sdev->sdev_gendev);
1651	spin_lock_irq(q->queue_lock);
1652}
1653
1654u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1655{
1656	struct device *host_dev;
1657	u64 bounce_limit = 0xffffffff;
1658
1659	if (shost->unchecked_isa_dma)
1660		return BLK_BOUNCE_ISA;
1661	/*
1662	 * Platforms with virtual-DMA translation
1663	 * hardware have no practical limit.
1664	 */
1665	if (!PCI_DMA_BUS_IS_PHYS)
1666		return BLK_BOUNCE_ANY;
1667
1668	host_dev = scsi_get_device(shost);
1669	if (host_dev && host_dev->dma_mask)
1670		bounce_limit = *host_dev->dma_mask;
1671
1672	return bounce_limit;
1673}
1674EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1675
1676struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1677{
1678	struct Scsi_Host *shost = sdev->host;
1679	struct request_queue *q;
1680
1681	q = blk_init_queue(scsi_request_fn, NULL);
1682	if (!q)
1683		return NULL;
1684
1685	blk_queue_prep_rq(q, scsi_prep_fn);
1686
1687	blk_queue_max_hw_segments(q, shost->sg_tablesize);
1688	blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS);
1689	blk_queue_max_sectors(q, shost->max_sectors);
1690	blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1691	blk_queue_segment_boundary(q, shost->dma_boundary);
1692	blk_queue_issue_flush_fn(q, scsi_issue_flush_fn);
1693
1694	/*
1695	 * ordered tags are superior to flush ordering
1696	 */
1697	if (shost->ordered_tag)
1698		blk_queue_ordered(q, QUEUE_ORDERED_TAG);
1699	else if (shost->ordered_flush) {
1700		blk_queue_ordered(q, QUEUE_ORDERED_FLUSH);
1701		q->prepare_flush_fn = scsi_prepare_flush_fn;
1702		q->end_flush_fn = scsi_end_flush_fn;
1703	}
1704
1705	if (!shost->use_clustering)
1706		clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
1707	return q;
1708}
1709
1710void scsi_free_queue(struct request_queue *q)
1711{
1712	blk_cleanup_queue(q);
1713}
1714
1715/*
1716 * Function:    scsi_block_requests()
1717 *
1718 * Purpose:     Utility function used by low-level drivers to prevent further
1719 *		commands from being queued to the device.
1720 *
1721 * Arguments:   shost       - Host in question
1722 *
1723 * Returns:     Nothing
1724 *
1725 * Lock status: No locks are assumed held.
1726 *
1727 * Notes:       There is no timer nor any other means by which the requests
1728 *		get unblocked other than the low-level driver calling
1729 *		scsi_unblock_requests().
1730 */
1731void scsi_block_requests(struct Scsi_Host *shost)
1732{
1733	shost->host_self_blocked = 1;
1734}
1735EXPORT_SYMBOL(scsi_block_requests);
1736
1737/*
1738 * Function:    scsi_unblock_requests()
1739 *
1740 * Purpose:     Utility function used by low-level drivers to allow further
1741 *		commands from being queued to the device.
1742 *
1743 * Arguments:   shost       - Host in question
1744 *
1745 * Returns:     Nothing
1746 *
1747 * Lock status: No locks are assumed held.
1748 *
1749 * Notes:       There is no timer nor any other means by which the requests
1750 *		get unblocked other than the low-level driver calling
1751 *		scsi_unblock_requests().
1752 *
1753 *		This is done as an API function so that changes to the
1754 *		internals of the scsi mid-layer won't require wholesale
1755 *		changes to drivers that use this feature.
1756 */
1757void scsi_unblock_requests(struct Scsi_Host *shost)
1758{
1759	shost->host_self_blocked = 0;
1760	scsi_run_host_queues(shost);
1761}
1762EXPORT_SYMBOL(scsi_unblock_requests);
1763
1764int __init scsi_init_queue(void)
1765{
1766	int i;
1767
1768	for (i = 0; i < SG_MEMPOOL_NR; i++) {
1769		struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1770		int size = sgp->size * sizeof(struct scatterlist);
1771
1772		sgp->slab = kmem_cache_create(sgp->name, size, 0,
1773				SLAB_HWCACHE_ALIGN, NULL, NULL);
1774		if (!sgp->slab) {
1775			printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1776					sgp->name);
1777		}
1778
1779		sgp->pool = mempool_create(SG_MEMPOOL_SIZE,
1780				mempool_alloc_slab, mempool_free_slab,
1781				sgp->slab);
1782		if (!sgp->pool) {
1783			printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1784					sgp->name);
1785		}
1786	}
1787
1788	return 0;
1789}
1790
1791void scsi_exit_queue(void)
1792{
1793	int i;
1794
1795	for (i = 0; i < SG_MEMPOOL_NR; i++) {
1796		struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1797		mempool_destroy(sgp->pool);
1798		kmem_cache_destroy(sgp->slab);
1799	}
1800}
1801/**
1802 *	scsi_mode_sense - issue a mode sense, falling back from 10 to
1803 *		six bytes if necessary.
1804 *	@sdev:	SCSI device to be queried
1805 *	@dbd:	set if mode sense will allow block descriptors to be returned
1806 *	@modepage: mode page being requested
1807 *	@buffer: request buffer (may not be smaller than eight bytes)
1808 *	@len:	length of request buffer.
1809 *	@timeout: command timeout
1810 *	@retries: number of retries before failing
1811 *	@data: returns a structure abstracting the mode header data
1812 *	@sense: place to put sense data (or NULL if no sense to be collected).
1813 *		must be SCSI_SENSE_BUFFERSIZE big.
1814 *
1815 *	Returns zero if unsuccessful, or the header offset (either 4
1816 *	or 8 depending on whether a six or ten byte command was
1817 *	issued) if successful.
1818 **/
1819int
1820scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1821		  unsigned char *buffer, int len, int timeout, int retries,
1822		  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) {
1823	unsigned char cmd[12];
1824	int use_10_for_ms;
1825	int header_length;
1826	int result;
1827	struct scsi_sense_hdr my_sshdr;
1828
1829	memset(data, 0, sizeof(*data));
1830	memset(&cmd[0], 0, 12);
1831	cmd[1] = dbd & 0x18;	/* allows DBD and LLBA bits */
1832	cmd[2] = modepage;
1833
1834	/* caller might not be interested in sense, but we need it */
1835	if (!sshdr)
1836		sshdr = &my_sshdr;
1837
1838 retry:
1839	use_10_for_ms = sdev->use_10_for_ms;
1840
1841	if (use_10_for_ms) {
1842		if (len < 8)
1843			len = 8;
1844
1845		cmd[0] = MODE_SENSE_10;
1846		cmd[8] = len;
1847		header_length = 8;
1848	} else {
1849		if (len < 4)
1850			len = 4;
1851
1852		cmd[0] = MODE_SENSE;
1853		cmd[4] = len;
1854		header_length = 4;
1855	}
1856
1857	memset(buffer, 0, len);
1858
1859	result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
1860				  sshdr, timeout, retries);
1861
1862	/* This code looks awful: what it's doing is making sure an
1863	 * ILLEGAL REQUEST sense return identifies the actual command
1864	 * byte as the problem.  MODE_SENSE commands can return
1865	 * ILLEGAL REQUEST if the code page isn't supported */
1866
1867	if (use_10_for_ms && !scsi_status_is_good(result) &&
1868	    (driver_byte(result) & DRIVER_SENSE)) {
1869		if (scsi_sense_valid(sshdr)) {
1870			if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
1871			    (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
1872				/*
1873				 * Invalid command operation code
1874				 */
1875				sdev->use_10_for_ms = 0;
1876				goto retry;
1877			}
1878		}
1879	}
1880
1881	if(scsi_status_is_good(result)) {
1882		data->header_length = header_length;
1883		if(use_10_for_ms) {
1884			data->length = buffer[0]*256 + buffer[1] + 2;
1885			data->medium_type = buffer[2];
1886			data->device_specific = buffer[3];
1887			data->longlba = buffer[4] & 0x01;
1888			data->block_descriptor_length = buffer[6]*256
1889				+ buffer[7];
1890		} else {
1891			data->length = buffer[0] + 1;
1892			data->medium_type = buffer[1];
1893			data->device_specific = buffer[2];
1894			data->block_descriptor_length = buffer[3];
1895		}
1896	}
1897
1898	return result;
1899}
1900EXPORT_SYMBOL(scsi_mode_sense);
1901
1902int
1903scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
1904{
1905	char cmd[] = {
1906		TEST_UNIT_READY, 0, 0, 0, 0, 0,
1907	};
1908	struct scsi_sense_hdr sshdr;
1909	int result;
1910
1911	result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
1912				  timeout, retries);
1913
1914	if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) {
1915
1916		if ((scsi_sense_valid(&sshdr)) &&
1917		    ((sshdr.sense_key == UNIT_ATTENTION) ||
1918		     (sshdr.sense_key == NOT_READY))) {
1919			sdev->changed = 1;
1920			result = 0;
1921		}
1922	}
1923	return result;
1924}
1925EXPORT_SYMBOL(scsi_test_unit_ready);
1926
1927/**
1928 *	scsi_device_set_state - Take the given device through the device
1929 *		state model.
1930 *	@sdev:	scsi device to change the state of.
1931 *	@state:	state to change to.
1932 *
1933 *	Returns zero if unsuccessful or an error if the requested
1934 *	transition is illegal.
1935 **/
1936int
1937scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
1938{
1939	enum scsi_device_state oldstate = sdev->sdev_state;
1940
1941	if (state == oldstate)
1942		return 0;
1943
1944	switch (state) {
1945	case SDEV_CREATED:
1946		/* There are no legal states that come back to
1947		 * created.  This is the manually initialised start
1948		 * state */
1949		goto illegal;
1950
1951	case SDEV_RUNNING:
1952		switch (oldstate) {
1953		case SDEV_CREATED:
1954		case SDEV_OFFLINE:
1955		case SDEV_QUIESCE:
1956		case SDEV_BLOCK:
1957			break;
1958		default:
1959			goto illegal;
1960		}
1961		break;
1962
1963	case SDEV_QUIESCE:
1964		switch (oldstate) {
1965		case SDEV_RUNNING:
1966		case SDEV_OFFLINE:
1967			break;
1968		default:
1969			goto illegal;
1970		}
1971		break;
1972
1973	case SDEV_OFFLINE:
1974		switch (oldstate) {
1975		case SDEV_CREATED:
1976		case SDEV_RUNNING:
1977		case SDEV_QUIESCE:
1978		case SDEV_BLOCK:
1979			break;
1980		default:
1981			goto illegal;
1982		}
1983		break;
1984
1985	case SDEV_BLOCK:
1986		switch (oldstate) {
1987		case SDEV_CREATED:
1988		case SDEV_RUNNING:
1989			break;
1990		default:
1991			goto illegal;
1992		}
1993		break;
1994
1995	case SDEV_CANCEL:
1996		switch (oldstate) {
1997		case SDEV_CREATED:
1998		case SDEV_RUNNING:
1999		case SDEV_OFFLINE:
2000		case SDEV_BLOCK:
2001			break;
2002		default:
2003			goto illegal;
2004		}
2005		break;
2006
2007	case SDEV_DEL:
2008		switch (oldstate) {
2009		case SDEV_CANCEL:
2010			break;
2011		default:
2012			goto illegal;
2013		}
2014		break;
2015
2016	}
2017	sdev->sdev_state = state;
2018	return 0;
2019
2020 illegal:
2021	SCSI_LOG_ERROR_RECOVERY(1,
2022				sdev_printk(KERN_ERR, sdev,
2023					    "Illegal state transition %s->%s\n",
2024					    scsi_device_state_name(oldstate),
2025					    scsi_device_state_name(state))
2026				);
2027	return -EINVAL;
2028}
2029EXPORT_SYMBOL(scsi_device_set_state);
2030
2031/**
2032 *	scsi_device_quiesce - Block user issued commands.
2033 *	@sdev:	scsi device to quiesce.
2034 *
2035 *	This works by trying to transition to the SDEV_QUIESCE state
2036 *	(which must be a legal transition).  When the device is in this
2037 *	state, only special requests will be accepted, all others will
2038 *	be deferred.  Since special requests may also be requeued requests,
2039 *	a successful return doesn't guarantee the device will be
2040 *	totally quiescent.
2041 *
2042 *	Must be called with user context, may sleep.
2043 *
2044 *	Returns zero if unsuccessful or an error if not.
2045 **/
2046int
2047scsi_device_quiesce(struct scsi_device *sdev)
2048{
2049	int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
2050	if (err)
2051		return err;
2052
2053	scsi_run_queue(sdev->request_queue);
2054	while (sdev->device_busy) {
2055		msleep_interruptible(200);
2056		scsi_run_queue(sdev->request_queue);
2057	}
2058	return 0;
2059}
2060EXPORT_SYMBOL(scsi_device_quiesce);
2061
2062/**
2063 *	scsi_device_resume - Restart user issued commands to a quiesced device.
2064 *	@sdev:	scsi device to resume.
2065 *
2066 *	Moves the device from quiesced back to running and restarts the
2067 *	queues.
2068 *
2069 *	Must be called with user context, may sleep.
2070 **/
2071void
2072scsi_device_resume(struct scsi_device *sdev)
2073{
2074	if(scsi_device_set_state(sdev, SDEV_RUNNING))
2075		return;
2076	scsi_run_queue(sdev->request_queue);
2077}
2078EXPORT_SYMBOL(scsi_device_resume);
2079
2080static void
2081device_quiesce_fn(struct scsi_device *sdev, void *data)
2082{
2083	scsi_device_quiesce(sdev);
2084}
2085
2086void
2087scsi_target_quiesce(struct scsi_target *starget)
2088{
2089	starget_for_each_device(starget, NULL, device_quiesce_fn);
2090}
2091EXPORT_SYMBOL(scsi_target_quiesce);
2092
2093static void
2094device_resume_fn(struct scsi_device *sdev, void *data)
2095{
2096	scsi_device_resume(sdev);
2097}
2098
2099void
2100scsi_target_resume(struct scsi_target *starget)
2101{
2102	starget_for_each_device(starget, NULL, device_resume_fn);
2103}
2104EXPORT_SYMBOL(scsi_target_resume);
2105
2106/**
2107 * scsi_internal_device_block - internal function to put a device
2108 *				temporarily into the SDEV_BLOCK state
2109 * @sdev:	device to block
2110 *
2111 * Block request made by scsi lld's to temporarily stop all
2112 * scsi commands on the specified device.  Called from interrupt
2113 * or normal process context.
2114 *
2115 * Returns zero if successful or error if not
2116 *
2117 * Notes:
2118 *	This routine transitions the device to the SDEV_BLOCK state
2119 *	(which must be a legal transition).  When the device is in this
2120 *	state, all commands are deferred until the scsi lld reenables
2121 *	the device with scsi_device_unblock or device_block_tmo fires.
2122 *	This routine assumes the host_lock is held on entry.
2123 **/
2124int
2125scsi_internal_device_block(struct scsi_device *sdev)
2126{
2127	request_queue_t *q = sdev->request_queue;
2128	unsigned long flags;
2129	int err = 0;
2130
2131	err = scsi_device_set_state(sdev, SDEV_BLOCK);
2132	if (err)
2133		return err;
2134
2135	/*
2136	 * The device has transitioned to SDEV_BLOCK.  Stop the
2137	 * block layer from calling the midlayer with this device's
2138	 * request queue.
2139	 */
2140	spin_lock_irqsave(q->queue_lock, flags);
2141	blk_stop_queue(q);
2142	spin_unlock_irqrestore(q->queue_lock, flags);
2143
2144	return 0;
2145}
2146EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2147
2148/**
2149 * scsi_internal_device_unblock - resume a device after a block request
2150 * @sdev:	device to resume
2151 *
2152 * Called by scsi lld's or the midlayer to restart the device queue
2153 * for the previously suspended scsi device.  Called from interrupt or
2154 * normal process context.
2155 *
2156 * Returns zero if successful or error if not.
2157 *
2158 * Notes:
2159 *	This routine transitions the device to the SDEV_RUNNING state
2160 *	(which must be a legal transition) allowing the midlayer to
2161 *	goose the queue for this device.  This routine assumes the
2162 *	host_lock is held upon entry.
2163 **/
2164int
2165scsi_internal_device_unblock(struct scsi_device *sdev)
2166{
2167	request_queue_t *q = sdev->request_queue;
2168	int err;
2169	unsigned long flags;
2170
2171	/*
2172	 * Try to transition the scsi device to SDEV_RUNNING
2173	 * and goose the device queue if successful.
2174	 */
2175	err = scsi_device_set_state(sdev, SDEV_RUNNING);
2176	if (err)
2177		return err;
2178
2179	spin_lock_irqsave(q->queue_lock, flags);
2180	blk_start_queue(q);
2181	spin_unlock_irqrestore(q->queue_lock, flags);
2182
2183	return 0;
2184}
2185EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
2186
2187static void
2188device_block(struct scsi_device *sdev, void *data)
2189{
2190	scsi_internal_device_block(sdev);
2191}
2192
2193static int
2194target_block(struct device *dev, void *data)
2195{
2196	if (scsi_is_target_device(dev))
2197		starget_for_each_device(to_scsi_target(dev), NULL,
2198					device_block);
2199	return 0;
2200}
2201
2202void
2203scsi_target_block(struct device *dev)
2204{
2205	if (scsi_is_target_device(dev))
2206		starget_for_each_device(to_scsi_target(dev), NULL,
2207					device_block);
2208	else
2209		device_for_each_child(dev, NULL, target_block);
2210}
2211EXPORT_SYMBOL_GPL(scsi_target_block);
2212
2213static void
2214device_unblock(struct scsi_device *sdev, void *data)
2215{
2216	scsi_internal_device_unblock(sdev);
2217}
2218
2219static int
2220target_unblock(struct device *dev, void *data)
2221{
2222	if (scsi_is_target_device(dev))
2223		starget_for_each_device(to_scsi_target(dev), NULL,
2224					device_unblock);
2225	return 0;
2226}
2227
2228void
2229scsi_target_unblock(struct device *dev)
2230{
2231	if (scsi_is_target_device(dev))
2232		starget_for_each_device(to_scsi_target(dev), NULL,
2233					device_unblock);
2234	else
2235		device_for_each_child(dev, NULL, target_unblock);
2236}
2237EXPORT_SYMBOL_GPL(scsi_target_unblock);
2238