[PATCH] libata: add ap->pflags and move core dynamic flags to it
[linux-2.6] / drivers / scsi / scsi_lib.c
1 /*
2  *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
3  *
4  *  SCSI queueing library.
5  *      Initial versions: Eric Youngdale (eric@andante.org).
6  *                        Based upon conversations with large numbers
7  *                        of people at Linux Expo.
8  */
9
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/completion.h>
13 #include <linux/kernel.h>
14 #include <linux/mempool.h>
15 #include <linux/slab.h>
16 #include <linux/init.h>
17 #include <linux/pci.h>
18 #include <linux/delay.h>
19 #include <linux/hardirq.h>
20
21 #include <scsi/scsi.h>
22 #include <scsi/scsi_cmnd.h>
23 #include <scsi/scsi_dbg.h>
24 #include <scsi/scsi_device.h>
25 #include <scsi/scsi_driver.h>
26 #include <scsi/scsi_eh.h>
27 #include <scsi/scsi_host.h>
28
29 #include "scsi_priv.h"
30 #include "scsi_logging.h"
31
32
33 #define SG_MEMPOOL_NR           ARRAY_SIZE(scsi_sg_pools)
34 #define SG_MEMPOOL_SIZE         32
35
36 struct scsi_host_sg_pool {
37         size_t          size;
38         char            *name; 
39         kmem_cache_t    *slab;
40         mempool_t       *pool;
41 };
42
43 #if (SCSI_MAX_PHYS_SEGMENTS < 32)
44 #error SCSI_MAX_PHYS_SEGMENTS is too small
45 #endif
46
47 #define SP(x) { x, "sgpool-" #x } 
48 static struct scsi_host_sg_pool scsi_sg_pools[] = {
49         SP(8),
50         SP(16),
51         SP(32),
52 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
53         SP(64),
54 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
55         SP(128),
56 #if (SCSI_MAX_PHYS_SEGMENTS > 128)
57         SP(256),
58 #if (SCSI_MAX_PHYS_SEGMENTS > 256)
59 #error SCSI_MAX_PHYS_SEGMENTS is too large
60 #endif
61 #endif
62 #endif
63 #endif
64 };      
65 #undef SP
66
67 static void scsi_run_queue(struct request_queue *q);
68
69 /*
70  * Function:    scsi_unprep_request()
71  *
72  * Purpose:     Remove all preparation done for a request, including its
73  *              associated scsi_cmnd, so that it can be requeued.
74  *
75  * Arguments:   req     - request to unprepare
76  *
77  * Lock status: Assumed that no locks are held upon entry.
78  *
79  * Returns:     Nothing.
80  */
81 static void scsi_unprep_request(struct request *req)
82 {
83         struct scsi_cmnd *cmd = req->special;
84
85         req->flags &= ~REQ_DONTPREP;
86         req->special = NULL;
87
88         scsi_put_command(cmd);
89 }
90
91 /*
92  * Function:    scsi_queue_insert()
93  *
94  * Purpose:     Insert a command in the midlevel queue.
95  *
96  * Arguments:   cmd    - command that we are adding to queue.
97  *              reason - why we are inserting command to queue.
98  *
99  * Lock status: Assumed that lock is not held upon entry.
100  *
101  * Returns:     Nothing.
102  *
103  * Notes:       We do this for one of two cases.  Either the host is busy
104  *              and it cannot accept any more commands for the time being,
105  *              or the device returned QUEUE_FULL and can accept no more
106  *              commands.
107  * Notes:       This could be called either from an interrupt context or a
108  *              normal process context.
109  */
110 int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
111 {
112         struct Scsi_Host *host = cmd->device->host;
113         struct scsi_device *device = cmd->device;
114         struct request_queue *q = device->request_queue;
115         unsigned long flags;
116
117         SCSI_LOG_MLQUEUE(1,
118                  printk("Inserting command %p into mlqueue\n", cmd));
119
120         /*
121          * Set the appropriate busy bit for the device/host.
122          *
123          * If the host/device isn't busy, assume that something actually
124          * completed, and that we should be able to queue a command now.
125          *
126          * Note that the prior mid-layer assumption that any host could
127          * always queue at least one command is now broken.  The mid-layer
128          * will implement a user specifiable stall (see
129          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
130          * if a command is requeued with no other commands outstanding
131          * either for the device or for the host.
132          */
133         if (reason == SCSI_MLQUEUE_HOST_BUSY)
134                 host->host_blocked = host->max_host_blocked;
135         else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
136                 device->device_blocked = device->max_device_blocked;
137
138         /*
139          * Decrement the counters, since these commands are no longer
140          * active on the host/device.
141          */
142         scsi_device_unbusy(device);
143
144         /*
145          * Requeue this command.  It will go before all other commands
146          * that are already in the queue.
147          *
148          * NOTE: there is magic here about the way the queue is plugged if
149          * we have no outstanding commands.
150          * 
151          * Although we *don't* plug the queue, we call the request
152          * function.  The SCSI request function detects the blocked condition
153          * and plugs the queue appropriately.
154          */
155         spin_lock_irqsave(q->queue_lock, flags);
156         blk_requeue_request(q, cmd->request);
157         spin_unlock_irqrestore(q->queue_lock, flags);
158
159         scsi_run_queue(q);
160
161         return 0;
162 }
163
164 /**
165  * scsi_execute - insert request and wait for the result
166  * @sdev:       scsi device
167  * @cmd:        scsi command
168  * @data_direction: data direction
169  * @buffer:     data buffer
170  * @bufflen:    len of buffer
171  * @sense:      optional sense buffer
172  * @timeout:    request timeout in seconds
173  * @retries:    number of times to retry request
174  * @flags:      or into request flags;
175  *
176  * returns the req->errors value which is the the scsi_cmnd result
177  * field.
178  **/
179 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
180                  int data_direction, void *buffer, unsigned bufflen,
181                  unsigned char *sense, int timeout, int retries, int flags)
182 {
183         struct request *req;
184         int write = (data_direction == DMA_TO_DEVICE);
185         int ret = DRIVER_ERROR << 24;
186
187         req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
188
189         if (bufflen &&  blk_rq_map_kern(sdev->request_queue, req,
190                                         buffer, bufflen, __GFP_WAIT))
191                 goto out;
192
193         req->cmd_len = COMMAND_SIZE(cmd[0]);
194         memcpy(req->cmd, cmd, req->cmd_len);
195         req->sense = sense;
196         req->sense_len = 0;
197         req->retries = retries;
198         req->timeout = timeout;
199         req->flags |= flags | REQ_BLOCK_PC | REQ_SPECIAL | REQ_QUIET;
200
201         /*
202          * head injection *required* here otherwise quiesce won't work
203          */
204         blk_execute_rq(req->q, NULL, req, 1);
205
206         ret = req->errors;
207  out:
208         blk_put_request(req);
209
210         return ret;
211 }
212 EXPORT_SYMBOL(scsi_execute);
213
214
215 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
216                      int data_direction, void *buffer, unsigned bufflen,
217                      struct scsi_sense_hdr *sshdr, int timeout, int retries)
218 {
219         char *sense = NULL;
220         int result;
221         
222         if (sshdr) {
223                 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
224                 if (!sense)
225                         return DRIVER_ERROR << 24;
226         }
227         result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
228                               sense, timeout, retries, 0);
229         if (sshdr)
230                 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
231
232         kfree(sense);
233         return result;
234 }
235 EXPORT_SYMBOL(scsi_execute_req);
236
237 struct scsi_io_context {
238         void *data;
239         void (*done)(void *data, char *sense, int result, int resid);
240         char sense[SCSI_SENSE_BUFFERSIZE];
241 };
242
243 static kmem_cache_t *scsi_io_context_cache;
244
245 static void scsi_end_async(struct request *req, int uptodate)
246 {
247         struct scsi_io_context *sioc = req->end_io_data;
248
249         if (sioc->done)
250                 sioc->done(sioc->data, sioc->sense, req->errors, req->data_len);
251
252         kmem_cache_free(scsi_io_context_cache, sioc);
253         __blk_put_request(req->q, req);
254 }
255
256 static int scsi_merge_bio(struct request *rq, struct bio *bio)
257 {
258         struct request_queue *q = rq->q;
259
260         bio->bi_flags &= ~(1 << BIO_SEG_VALID);
261         if (rq_data_dir(rq) == WRITE)
262                 bio->bi_rw |= (1 << BIO_RW);
263         blk_queue_bounce(q, &bio);
264
265         if (!rq->bio)
266                 blk_rq_bio_prep(q, rq, bio);
267         else if (!q->back_merge_fn(q, rq, bio))
268                 return -EINVAL;
269         else {
270                 rq->biotail->bi_next = bio;
271                 rq->biotail = bio;
272                 rq->hard_nr_sectors += bio_sectors(bio);
273                 rq->nr_sectors = rq->hard_nr_sectors;
274         }
275
276         return 0;
277 }
278
279 static int scsi_bi_endio(struct bio *bio, unsigned int bytes_done, int error)
280 {
281         if (bio->bi_size)
282                 return 1;
283
284         bio_put(bio);
285         return 0;
286 }
287
288 /**
289  * scsi_req_map_sg - map a scatterlist into a request
290  * @rq:         request to fill
291  * @sg:         scatterlist
292  * @nsegs:      number of elements
293  * @bufflen:    len of buffer
294  * @gfp:        memory allocation flags
295  *
296  * scsi_req_map_sg maps a scatterlist into a request so that the
297  * request can be sent to the block layer. We do not trust the scatterlist
298  * sent to use, as some ULDs use that struct to only organize the pages.
299  */
300 static int scsi_req_map_sg(struct request *rq, struct scatterlist *sgl,
301                            int nsegs, unsigned bufflen, gfp_t gfp)
302 {
303         struct request_queue *q = rq->q;
304         int nr_pages = (bufflen + sgl[0].offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
305         unsigned int data_len = 0, len, bytes, off;
306         struct page *page;
307         struct bio *bio = NULL;
308         int i, err, nr_vecs = 0;
309
310         for (i = 0; i < nsegs; i++) {
311                 page = sgl[i].page;
312                 off = sgl[i].offset;
313                 len = sgl[i].length;
314                 data_len += len;
315
316                 while (len > 0) {
317                         bytes = min_t(unsigned int, len, PAGE_SIZE - off);
318
319                         if (!bio) {
320                                 nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages);
321                                 nr_pages -= nr_vecs;
322
323                                 bio = bio_alloc(gfp, nr_vecs);
324                                 if (!bio) {
325                                         err = -ENOMEM;
326                                         goto free_bios;
327                                 }
328                                 bio->bi_end_io = scsi_bi_endio;
329                         }
330
331                         if (bio_add_pc_page(q, bio, page, bytes, off) !=
332                             bytes) {
333                                 bio_put(bio);
334                                 err = -EINVAL;
335                                 goto free_bios;
336                         }
337
338                         if (bio->bi_vcnt >= nr_vecs) {
339                                 err = scsi_merge_bio(rq, bio);
340                                 if (err) {
341                                         bio_endio(bio, bio->bi_size, 0);
342                                         goto free_bios;
343                                 }
344                                 bio = NULL;
345                         }
346
347                         page++;
348                         len -= bytes;
349                         off = 0;
350                 }
351         }
352
353         rq->buffer = rq->data = NULL;
354         rq->data_len = data_len;
355         return 0;
356
357 free_bios:
358         while ((bio = rq->bio) != NULL) {
359                 rq->bio = bio->bi_next;
360                 /*
361                  * call endio instead of bio_put incase it was bounced
362                  */
363                 bio_endio(bio, bio->bi_size, 0);
364         }
365
366         return err;
367 }
368
369 /**
370  * scsi_execute_async - insert request
371  * @sdev:       scsi device
372  * @cmd:        scsi command
373  * @cmd_len:    length of scsi cdb
374  * @data_direction: data direction
375  * @buffer:     data buffer (this can be a kernel buffer or scatterlist)
376  * @bufflen:    len of buffer
377  * @use_sg:     if buffer is a scatterlist this is the number of elements
378  * @timeout:    request timeout in seconds
379  * @retries:    number of times to retry request
380  * @flags:      or into request flags
381  **/
382 int scsi_execute_async(struct scsi_device *sdev, const unsigned char *cmd,
383                        int cmd_len, int data_direction, void *buffer, unsigned bufflen,
384                        int use_sg, int timeout, int retries, void *privdata,
385                        void (*done)(void *, char *, int, int), gfp_t gfp)
386 {
387         struct request *req;
388         struct scsi_io_context *sioc;
389         int err = 0;
390         int write = (data_direction == DMA_TO_DEVICE);
391
392         sioc = kmem_cache_alloc(scsi_io_context_cache, gfp);
393         if (!sioc)
394                 return DRIVER_ERROR << 24;
395         memset(sioc, 0, sizeof(*sioc));
396
397         req = blk_get_request(sdev->request_queue, write, gfp);
398         if (!req)
399                 goto free_sense;
400         req->flags |= REQ_BLOCK_PC | REQ_QUIET;
401
402         if (use_sg)
403                 err = scsi_req_map_sg(req, buffer, use_sg, bufflen, gfp);
404         else if (bufflen)
405                 err = blk_rq_map_kern(req->q, req, buffer, bufflen, gfp);
406
407         if (err)
408                 goto free_req;
409
410         req->cmd_len = cmd_len;
411         memcpy(req->cmd, cmd, req->cmd_len);
412         req->sense = sioc->sense;
413         req->sense_len = 0;
414         req->timeout = timeout;
415         req->retries = retries;
416         req->end_io_data = sioc;
417
418         sioc->data = privdata;
419         sioc->done = done;
420
421         blk_execute_rq_nowait(req->q, NULL, req, 1, scsi_end_async);
422         return 0;
423
424 free_req:
425         blk_put_request(req);
426 free_sense:
427         kfree(sioc);
428         return DRIVER_ERROR << 24;
429 }
430 EXPORT_SYMBOL_GPL(scsi_execute_async);
431
432 /*
433  * Function:    scsi_init_cmd_errh()
434  *
435  * Purpose:     Initialize cmd fields related to error handling.
436  *
437  * Arguments:   cmd     - command that is ready to be queued.
438  *
439  * Returns:     Nothing
440  *
441  * Notes:       This function has the job of initializing a number of
442  *              fields related to error handling.   Typically this will
443  *              be called once for each command, as required.
444  */
445 static int scsi_init_cmd_errh(struct scsi_cmnd *cmd)
446 {
447         cmd->serial_number = 0;
448
449         memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
450
451         if (cmd->cmd_len == 0)
452                 cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
453
454         /*
455          * We need saved copies of a number of fields - this is because
456          * error handling may need to overwrite these with different values
457          * to run different commands, and once error handling is complete,
458          * we will need to restore these values prior to running the actual
459          * command.
460          */
461         cmd->old_use_sg = cmd->use_sg;
462         cmd->old_cmd_len = cmd->cmd_len;
463         cmd->sc_old_data_direction = cmd->sc_data_direction;
464         cmd->old_underflow = cmd->underflow;
465         memcpy(cmd->data_cmnd, cmd->cmnd, sizeof(cmd->cmnd));
466         cmd->buffer = cmd->request_buffer;
467         cmd->bufflen = cmd->request_bufflen;
468
469         return 1;
470 }
471
472 /*
473  * Function:   scsi_setup_cmd_retry()
474  *
475  * Purpose:    Restore the command state for a retry
476  *
477  * Arguments:  cmd      - command to be restored
478  *
479  * Returns:    Nothing
480  *
481  * Notes:      Immediately prior to retrying a command, we need
482  *             to restore certain fields that we saved above.
483  */
484 void scsi_setup_cmd_retry(struct scsi_cmnd *cmd)
485 {
486         memcpy(cmd->cmnd, cmd->data_cmnd, sizeof(cmd->data_cmnd));
487         cmd->request_buffer = cmd->buffer;
488         cmd->request_bufflen = cmd->bufflen;
489         cmd->use_sg = cmd->old_use_sg;
490         cmd->cmd_len = cmd->old_cmd_len;
491         cmd->sc_data_direction = cmd->sc_old_data_direction;
492         cmd->underflow = cmd->old_underflow;
493 }
494
495 void scsi_device_unbusy(struct scsi_device *sdev)
496 {
497         struct Scsi_Host *shost = sdev->host;
498         unsigned long flags;
499
500         spin_lock_irqsave(shost->host_lock, flags);
501         shost->host_busy--;
502         if (unlikely(scsi_host_in_recovery(shost) &&
503                      (shost->host_failed || shost->host_eh_scheduled)))
504                 scsi_eh_wakeup(shost);
505         spin_unlock(shost->host_lock);
506         spin_lock(sdev->request_queue->queue_lock);
507         sdev->device_busy--;
508         spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
509 }
510
511 /*
512  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
513  * and call blk_run_queue for all the scsi_devices on the target -
514  * including current_sdev first.
515  *
516  * Called with *no* scsi locks held.
517  */
518 static void scsi_single_lun_run(struct scsi_device *current_sdev)
519 {
520         struct Scsi_Host *shost = current_sdev->host;
521         struct scsi_device *sdev, *tmp;
522         struct scsi_target *starget = scsi_target(current_sdev);
523         unsigned long flags;
524
525         spin_lock_irqsave(shost->host_lock, flags);
526         starget->starget_sdev_user = NULL;
527         spin_unlock_irqrestore(shost->host_lock, flags);
528
529         /*
530          * Call blk_run_queue for all LUNs on the target, starting with
531          * current_sdev. We race with others (to set starget_sdev_user),
532          * but in most cases, we will be first. Ideally, each LU on the
533          * target would get some limited time or requests on the target.
534          */
535         blk_run_queue(current_sdev->request_queue);
536
537         spin_lock_irqsave(shost->host_lock, flags);
538         if (starget->starget_sdev_user)
539                 goto out;
540         list_for_each_entry_safe(sdev, tmp, &starget->devices,
541                         same_target_siblings) {
542                 if (sdev == current_sdev)
543                         continue;
544                 if (scsi_device_get(sdev))
545                         continue;
546
547                 spin_unlock_irqrestore(shost->host_lock, flags);
548                 blk_run_queue(sdev->request_queue);
549                 spin_lock_irqsave(shost->host_lock, flags);
550         
551                 scsi_device_put(sdev);
552         }
553  out:
554         spin_unlock_irqrestore(shost->host_lock, flags);
555 }
556
557 /*
558  * Function:    scsi_run_queue()
559  *
560  * Purpose:     Select a proper request queue to serve next
561  *
562  * Arguments:   q       - last request's queue
563  *
564  * Returns:     Nothing
565  *
566  * Notes:       The previous command was completely finished, start
567  *              a new one if possible.
568  */
569 static void scsi_run_queue(struct request_queue *q)
570 {
571         struct scsi_device *sdev = q->queuedata;
572         struct Scsi_Host *shost = sdev->host;
573         unsigned long flags;
574
575         if (sdev->single_lun)
576                 scsi_single_lun_run(sdev);
577
578         spin_lock_irqsave(shost->host_lock, flags);
579         while (!list_empty(&shost->starved_list) &&
580                !shost->host_blocked && !shost->host_self_blocked &&
581                 !((shost->can_queue > 0) &&
582                   (shost->host_busy >= shost->can_queue))) {
583                 /*
584                  * As long as shost is accepting commands and we have
585                  * starved queues, call blk_run_queue. scsi_request_fn
586                  * drops the queue_lock and can add us back to the
587                  * starved_list.
588                  *
589                  * host_lock protects the starved_list and starved_entry.
590                  * scsi_request_fn must get the host_lock before checking
591                  * or modifying starved_list or starved_entry.
592                  */
593                 sdev = list_entry(shost->starved_list.next,
594                                           struct scsi_device, starved_entry);
595                 list_del_init(&sdev->starved_entry);
596                 spin_unlock_irqrestore(shost->host_lock, flags);
597
598                 blk_run_queue(sdev->request_queue);
599
600                 spin_lock_irqsave(shost->host_lock, flags);
601                 if (unlikely(!list_empty(&sdev->starved_entry)))
602                         /*
603                          * sdev lost a race, and was put back on the
604                          * starved list. This is unlikely but without this
605                          * in theory we could loop forever.
606                          */
607                         break;
608         }
609         spin_unlock_irqrestore(shost->host_lock, flags);
610
611         blk_run_queue(q);
612 }
613
614 /*
615  * Function:    scsi_requeue_command()
616  *
617  * Purpose:     Handle post-processing of completed commands.
618  *
619  * Arguments:   q       - queue to operate on
620  *              cmd     - command that may need to be requeued.
621  *
622  * Returns:     Nothing
623  *
624  * Notes:       After command completion, there may be blocks left
625  *              over which weren't finished by the previous command
626  *              this can be for a number of reasons - the main one is
627  *              I/O errors in the middle of the request, in which case
628  *              we need to request the blocks that come after the bad
629  *              sector.
630  * Notes:       Upon return, cmd is a stale pointer.
631  */
632 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
633 {
634         struct request *req = cmd->request;
635         unsigned long flags;
636
637         scsi_unprep_request(req);
638         spin_lock_irqsave(q->queue_lock, flags);
639         blk_requeue_request(q, req);
640         spin_unlock_irqrestore(q->queue_lock, flags);
641
642         scsi_run_queue(q);
643 }
644
645 void scsi_next_command(struct scsi_cmnd *cmd)
646 {
647         struct scsi_device *sdev = cmd->device;
648         struct request_queue *q = sdev->request_queue;
649
650         /* need to hold a reference on the device before we let go of the cmd */
651         get_device(&sdev->sdev_gendev);
652
653         scsi_put_command(cmd);
654         scsi_run_queue(q);
655
656         /* ok to remove device now */
657         put_device(&sdev->sdev_gendev);
658 }
659
660 void scsi_run_host_queues(struct Scsi_Host *shost)
661 {
662         struct scsi_device *sdev;
663
664         shost_for_each_device(sdev, shost)
665                 scsi_run_queue(sdev->request_queue);
666 }
667
668 /*
669  * Function:    scsi_end_request()
670  *
671  * Purpose:     Post-processing of completed commands (usually invoked at end
672  *              of upper level post-processing and scsi_io_completion).
673  *
674  * Arguments:   cmd      - command that is complete.
675  *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
676  *              bytes    - number of bytes of completed I/O
677  *              requeue  - indicates whether we should requeue leftovers.
678  *
679  * Lock status: Assumed that lock is not held upon entry.
680  *
681  * Returns:     cmd if requeue required, NULL otherwise.
682  *
683  * Notes:       This is called for block device requests in order to
684  *              mark some number of sectors as complete.
685  * 
686  *              We are guaranteeing that the request queue will be goosed
687  *              at some point during this call.
688  * Notes:       If cmd was requeued, upon return it will be a stale pointer.
689  */
690 static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
691                                           int bytes, int requeue)
692 {
693         request_queue_t *q = cmd->device->request_queue;
694         struct request *req = cmd->request;
695         unsigned long flags;
696
697         /*
698          * If there are blocks left over at the end, set up the command
699          * to queue the remainder of them.
700          */
701         if (end_that_request_chunk(req, uptodate, bytes)) {
702                 int leftover = (req->hard_nr_sectors << 9);
703
704                 if (blk_pc_request(req))
705                         leftover = req->data_len;
706
707                 /* kill remainder if no retrys */
708                 if (!uptodate && blk_noretry_request(req))
709                         end_that_request_chunk(req, 0, leftover);
710                 else {
711                         if (requeue) {
712                                 /*
713                                  * Bleah.  Leftovers again.  Stick the
714                                  * leftovers in the front of the
715                                  * queue, and goose the queue again.
716                                  */
717                                 scsi_requeue_command(q, cmd);
718                                 cmd = NULL;
719                         }
720                         return cmd;
721                 }
722         }
723
724         add_disk_randomness(req->rq_disk);
725
726         spin_lock_irqsave(q->queue_lock, flags);
727         if (blk_rq_tagged(req))
728                 blk_queue_end_tag(q, req);
729         end_that_request_last(req, uptodate);
730         spin_unlock_irqrestore(q->queue_lock, flags);
731
732         /*
733          * This will goose the queue request function at the end, so we don't
734          * need to worry about launching another command.
735          */
736         scsi_next_command(cmd);
737         return NULL;
738 }
739
740 static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, gfp_t gfp_mask)
741 {
742         struct scsi_host_sg_pool *sgp;
743         struct scatterlist *sgl;
744
745         BUG_ON(!cmd->use_sg);
746
747         switch (cmd->use_sg) {
748         case 1 ... 8:
749                 cmd->sglist_len = 0;
750                 break;
751         case 9 ... 16:
752                 cmd->sglist_len = 1;
753                 break;
754         case 17 ... 32:
755                 cmd->sglist_len = 2;
756                 break;
757 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
758         case 33 ... 64:
759                 cmd->sglist_len = 3;
760                 break;
761 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
762         case 65 ... 128:
763                 cmd->sglist_len = 4;
764                 break;
765 #if (SCSI_MAX_PHYS_SEGMENTS  > 128)
766         case 129 ... 256:
767                 cmd->sglist_len = 5;
768                 break;
769 #endif
770 #endif
771 #endif
772         default:
773                 return NULL;
774         }
775
776         sgp = scsi_sg_pools + cmd->sglist_len;
777         sgl = mempool_alloc(sgp->pool, gfp_mask);
778         return sgl;
779 }
780
781 static void scsi_free_sgtable(struct scatterlist *sgl, int index)
782 {
783         struct scsi_host_sg_pool *sgp;
784
785         BUG_ON(index >= SG_MEMPOOL_NR);
786
787         sgp = scsi_sg_pools + index;
788         mempool_free(sgl, sgp->pool);
789 }
790
791 /*
792  * Function:    scsi_release_buffers()
793  *
794  * Purpose:     Completion processing for block device I/O requests.
795  *
796  * Arguments:   cmd     - command that we are bailing.
797  *
798  * Lock status: Assumed that no lock is held upon entry.
799  *
800  * Returns:     Nothing
801  *
802  * Notes:       In the event that an upper level driver rejects a
803  *              command, we must release resources allocated during
804  *              the __init_io() function.  Primarily this would involve
805  *              the scatter-gather table, and potentially any bounce
806  *              buffers.
807  */
808 static void scsi_release_buffers(struct scsi_cmnd *cmd)
809 {
810         struct request *req = cmd->request;
811
812         /*
813          * Free up any indirection buffers we allocated for DMA purposes. 
814          */
815         if (cmd->use_sg)
816                 scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len);
817         else if (cmd->request_buffer != req->buffer)
818                 kfree(cmd->request_buffer);
819
820         /*
821          * Zero these out.  They now point to freed memory, and it is
822          * dangerous to hang onto the pointers.
823          */
824         cmd->buffer  = NULL;
825         cmd->bufflen = 0;
826         cmd->request_buffer = NULL;
827         cmd->request_bufflen = 0;
828 }
829
830 /*
831  * Function:    scsi_io_completion()
832  *
833  * Purpose:     Completion processing for block device I/O requests.
834  *
835  * Arguments:   cmd   - command that is finished.
836  *
837  * Lock status: Assumed that no lock is held upon entry.
838  *
839  * Returns:     Nothing
840  *
841  * Notes:       This function is matched in terms of capabilities to
842  *              the function that created the scatter-gather list.
843  *              In other words, if there are no bounce buffers
844  *              (the normal case for most drivers), we don't need
845  *              the logic to deal with cleaning up afterwards.
846  *
847  *              We must do one of several things here:
848  *
849  *              a) Call scsi_end_request.  This will finish off the
850  *                 specified number of sectors.  If we are done, the
851  *                 command block will be released, and the queue
852  *                 function will be goosed.  If we are not done, then
853  *                 scsi_end_request will directly goose the queue.
854  *
855  *              b) We can just use scsi_requeue_command() here.  This would
856  *                 be used if we just wanted to retry, for example.
857  */
858 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
859 {
860         int result = cmd->result;
861         int this_count = cmd->bufflen;
862         request_queue_t *q = cmd->device->request_queue;
863         struct request *req = cmd->request;
864         int clear_errors = 1;
865         struct scsi_sense_hdr sshdr;
866         int sense_valid = 0;
867         int sense_deferred = 0;
868
869         /*
870          * Free up any indirection buffers we allocated for DMA purposes. 
871          * For the case of a READ, we need to copy the data out of the
872          * bounce buffer and into the real buffer.
873          */
874         if (cmd->use_sg)
875                 scsi_free_sgtable(cmd->buffer, cmd->sglist_len);
876         else if (cmd->buffer != req->buffer) {
877                 if (rq_data_dir(req) == READ) {
878                         unsigned long flags;
879                         char *to = bio_kmap_irq(req->bio, &flags);
880                         memcpy(to, cmd->buffer, cmd->bufflen);
881                         bio_kunmap_irq(to, &flags);
882                 }
883                 kfree(cmd->buffer);
884         }
885
886         if (result) {
887                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
888                 if (sense_valid)
889                         sense_deferred = scsi_sense_is_deferred(&sshdr);
890         }
891         if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
892                 req->errors = result;
893                 if (result) {
894                         clear_errors = 0;
895                         if (sense_valid && req->sense) {
896                                 /*
897                                  * SG_IO wants current and deferred errors
898                                  */
899                                 int len = 8 + cmd->sense_buffer[7];
900
901                                 if (len > SCSI_SENSE_BUFFERSIZE)
902                                         len = SCSI_SENSE_BUFFERSIZE;
903                                 memcpy(req->sense, cmd->sense_buffer,  len);
904                                 req->sense_len = len;
905                         }
906                 } else
907                         req->data_len = cmd->resid;
908         }
909
910         /*
911          * Zero these out.  They now point to freed memory, and it is
912          * dangerous to hang onto the pointers.
913          */
914         cmd->buffer  = NULL;
915         cmd->bufflen = 0;
916         cmd->request_buffer = NULL;
917         cmd->request_bufflen = 0;
918
919         /*
920          * Next deal with any sectors which we were able to correctly
921          * handle.
922          */
923         SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, "
924                                       "%d bytes done.\n",
925                                       req->nr_sectors, good_bytes));
926         SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
927
928         if (clear_errors)
929                 req->errors = 0;
930
931         /* A number of bytes were successfully read.  If there
932          * are leftovers and there is some kind of error
933          * (result != 0), retry the rest.
934          */
935         if (scsi_end_request(cmd, 1, good_bytes, result == 0) == NULL)
936                 return;
937
938         /* good_bytes = 0, or (inclusive) there were leftovers and
939          * result = 0, so scsi_end_request couldn't retry.
940          */
941         if (sense_valid && !sense_deferred) {
942                 switch (sshdr.sense_key) {
943                 case UNIT_ATTENTION:
944                         if (cmd->device->removable) {
945                                 /* Detected disc change.  Set a bit
946                                  * and quietly refuse further access.
947                                  */
948                                 cmd->device->changed = 1;
949                                 scsi_end_request(cmd, 0, this_count, 1);
950                                 return;
951                         } else {
952                                 /* Must have been a power glitch, or a
953                                  * bus reset.  Could not have been a
954                                  * media change, so we just retry the
955                                  * request and see what happens.
956                                  */
957                                 scsi_requeue_command(q, cmd);
958                                 return;
959                         }
960                         break;
961                 case ILLEGAL_REQUEST:
962                         /* If we had an ILLEGAL REQUEST returned, then
963                          * we may have performed an unsupported
964                          * command.  The only thing this should be
965                          * would be a ten byte read where only a six
966                          * byte read was supported.  Also, on a system
967                          * where READ CAPACITY failed, we may have
968                          * read past the end of the disk.
969                          */
970                         if ((cmd->device->use_10_for_rw &&
971                             sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
972                             (cmd->cmnd[0] == READ_10 ||
973                              cmd->cmnd[0] == WRITE_10)) {
974                                 cmd->device->use_10_for_rw = 0;
975                                 /* This will cause a retry with a
976                                  * 6-byte command.
977                                  */
978                                 scsi_requeue_command(q, cmd);
979                                 return;
980                         } else {
981                                 scsi_end_request(cmd, 0, this_count, 1);
982                                 return;
983                         }
984                         break;
985                 case NOT_READY:
986                         /* If the device is in the process of becoming
987                          * ready, or has a temporary blockage, retry.
988                          */
989                         if (sshdr.asc == 0x04) {
990                                 switch (sshdr.ascq) {
991                                 case 0x01: /* becoming ready */
992                                 case 0x04: /* format in progress */
993                                 case 0x05: /* rebuild in progress */
994                                 case 0x06: /* recalculation in progress */
995                                 case 0x07: /* operation in progress */
996                                 case 0x08: /* Long write in progress */
997                                 case 0x09: /* self test in progress */
998                                         scsi_requeue_command(q, cmd);
999                                         return;
1000                                 default:
1001                                         break;
1002                                 }
1003                         }
1004                         if (!(req->flags & REQ_QUIET)) {
1005                                 scmd_printk(KERN_INFO, cmd,
1006                                             "Device not ready: ");
1007                                 scsi_print_sense_hdr("", &sshdr);
1008                         }
1009                         scsi_end_request(cmd, 0, this_count, 1);
1010                         return;
1011                 case VOLUME_OVERFLOW:
1012                         if (!(req->flags & REQ_QUIET)) {
1013                                 scmd_printk(KERN_INFO, cmd,
1014                                             "Volume overflow, CDB: ");
1015                                 __scsi_print_command(cmd->data_cmnd);
1016                                 scsi_print_sense("", cmd);
1017                         }
1018                         /* See SSC3rXX or current. */
1019                         scsi_end_request(cmd, 0, this_count, 1);
1020                         return;
1021                 default:
1022                         break;
1023                 }
1024         }
1025         if (host_byte(result) == DID_RESET) {
1026                 /* Third party bus reset or reset for error recovery
1027                  * reasons.  Just retry the request and see what
1028                  * happens.
1029                  */
1030                 scsi_requeue_command(q, cmd);
1031                 return;
1032         }
1033         if (result) {
1034                 if (!(req->flags & REQ_QUIET)) {
1035                         scmd_printk(KERN_INFO, cmd,
1036                                     "SCSI error: return code = 0x%08x\n",
1037                                     result);
1038                         if (driver_byte(result) & DRIVER_SENSE)
1039                                 scsi_print_sense("", cmd);
1040                 }
1041         }
1042         scsi_end_request(cmd, 0, this_count, !result);
1043 }
1044 EXPORT_SYMBOL(scsi_io_completion);
1045
1046 /*
1047  * Function:    scsi_init_io()
1048  *
1049  * Purpose:     SCSI I/O initialize function.
1050  *
1051  * Arguments:   cmd   - Command descriptor we wish to initialize
1052  *
1053  * Returns:     0 on success
1054  *              BLKPREP_DEFER if the failure is retryable
1055  *              BLKPREP_KILL if the failure is fatal
1056  */
1057 static int scsi_init_io(struct scsi_cmnd *cmd)
1058 {
1059         struct request     *req = cmd->request;
1060         struct scatterlist *sgpnt;
1061         int                count;
1062
1063         /*
1064          * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer
1065          */
1066         if ((req->flags & REQ_BLOCK_PC) && !req->bio) {
1067                 cmd->request_bufflen = req->data_len;
1068                 cmd->request_buffer = req->data;
1069                 req->buffer = req->data;
1070                 cmd->use_sg = 0;
1071                 return 0;
1072         }
1073
1074         /*
1075          * we used to not use scatter-gather for single segment request,
1076          * but now we do (it makes highmem I/O easier to support without
1077          * kmapping pages)
1078          */
1079         cmd->use_sg = req->nr_phys_segments;
1080
1081         /*
1082          * if sg table allocation fails, requeue request later.
1083          */
1084         sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
1085         if (unlikely(!sgpnt)) {
1086                 scsi_unprep_request(req);
1087                 return BLKPREP_DEFER;
1088         }
1089
1090         cmd->request_buffer = (char *) sgpnt;
1091         cmd->request_bufflen = req->nr_sectors << 9;
1092         if (blk_pc_request(req))
1093                 cmd->request_bufflen = req->data_len;
1094         req->buffer = NULL;
1095
1096         /* 
1097          * Next, walk the list, and fill in the addresses and sizes of
1098          * each segment.
1099          */
1100         count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
1101
1102         /*
1103          * mapped well, send it off
1104          */
1105         if (likely(count <= cmd->use_sg)) {
1106                 cmd->use_sg = count;
1107                 return 0;
1108         }
1109
1110         printk(KERN_ERR "Incorrect number of segments after building list\n");
1111         printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg);
1112         printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors,
1113                         req->current_nr_sectors);
1114
1115         /* release the command and kill it */
1116         scsi_release_buffers(cmd);
1117         scsi_put_command(cmd);
1118         return BLKPREP_KILL;
1119 }
1120
1121 static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk,
1122                                sector_t *error_sector)
1123 {
1124         struct scsi_device *sdev = q->queuedata;
1125         struct scsi_driver *drv;
1126
1127         if (sdev->sdev_state != SDEV_RUNNING)
1128                 return -ENXIO;
1129
1130         drv = *(struct scsi_driver **) disk->private_data;
1131         if (drv->issue_flush)
1132                 return drv->issue_flush(&sdev->sdev_gendev, error_sector);
1133
1134         return -EOPNOTSUPP;
1135 }
1136
1137 static void scsi_blk_pc_done(struct scsi_cmnd *cmd)
1138 {
1139         BUG_ON(!blk_pc_request(cmd->request));
1140         /*
1141          * This will complete the whole command with uptodate=1 so
1142          * as far as the block layer is concerned the command completed
1143          * successfully. Since this is a REQ_BLOCK_PC command the
1144          * caller should check the request's errors value
1145          */
1146         scsi_io_completion(cmd, cmd->bufflen);
1147 }
1148
1149 static void scsi_setup_blk_pc_cmnd(struct scsi_cmnd *cmd)
1150 {
1151         struct request *req = cmd->request;
1152
1153         BUG_ON(sizeof(req->cmd) > sizeof(cmd->cmnd));
1154         memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd));
1155         cmd->cmd_len = req->cmd_len;
1156         if (!req->data_len)
1157                 cmd->sc_data_direction = DMA_NONE;
1158         else if (rq_data_dir(req) == WRITE)
1159                 cmd->sc_data_direction = DMA_TO_DEVICE;
1160         else
1161                 cmd->sc_data_direction = DMA_FROM_DEVICE;
1162         
1163         cmd->transfersize = req->data_len;
1164         cmd->allowed = req->retries;
1165         cmd->timeout_per_command = req->timeout;
1166         cmd->done = scsi_blk_pc_done;
1167 }
1168
1169 static int scsi_prep_fn(struct request_queue *q, struct request *req)
1170 {
1171         struct scsi_device *sdev = q->queuedata;
1172         struct scsi_cmnd *cmd;
1173         int specials_only = 0;
1174
1175         /*
1176          * Just check to see if the device is online.  If it isn't, we
1177          * refuse to process any commands.  The device must be brought
1178          * online before trying any recovery commands
1179          */
1180         if (unlikely(!scsi_device_online(sdev))) {
1181                 sdev_printk(KERN_ERR, sdev,
1182                             "rejecting I/O to offline device\n");
1183                 goto kill;
1184         }
1185         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1186                 /* OK, we're not in a running state don't prep
1187                  * user commands */
1188                 if (sdev->sdev_state == SDEV_DEL) {
1189                         /* Device is fully deleted, no commands
1190                          * at all allowed down */
1191                         sdev_printk(KERN_ERR, sdev,
1192                                     "rejecting I/O to dead device\n");
1193                         goto kill;
1194                 }
1195                 /* OK, we only allow special commands (i.e. not
1196                  * user initiated ones */
1197                 specials_only = sdev->sdev_state;
1198         }
1199
1200         /*
1201          * Find the actual device driver associated with this command.
1202          * The SPECIAL requests are things like character device or
1203          * ioctls, which did not originate from ll_rw_blk.  Note that
1204          * the special field is also used to indicate the cmd for
1205          * the remainder of a partially fulfilled request that can 
1206          * come up when there is a medium error.  We have to treat
1207          * these two cases differently.  We differentiate by looking
1208          * at request->cmd, as this tells us the real story.
1209          */
1210         if (req->flags & REQ_SPECIAL && req->special) {
1211                 cmd = req->special;
1212         } else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1213
1214                 if(unlikely(specials_only) && !(req->flags & REQ_SPECIAL)) {
1215                         if(specials_only == SDEV_QUIESCE ||
1216                                         specials_only == SDEV_BLOCK)
1217                                 goto defer;
1218                         
1219                         sdev_printk(KERN_ERR, sdev,
1220                                     "rejecting I/O to device being removed\n");
1221                         goto kill;
1222                 }
1223                         
1224                         
1225                 /*
1226                  * Now try and find a command block that we can use.
1227                  */
1228                 if (!req->special) {
1229                         cmd = scsi_get_command(sdev, GFP_ATOMIC);
1230                         if (unlikely(!cmd))
1231                                 goto defer;
1232                 } else
1233                         cmd = req->special;
1234                 
1235                 /* pull a tag out of the request if we have one */
1236                 cmd->tag = req->tag;
1237         } else {
1238                 blk_dump_rq_flags(req, "SCSI bad req");
1239                 goto kill;
1240         }
1241         
1242         /* note the overloading of req->special.  When the tag
1243          * is active it always means cmd.  If the tag goes
1244          * back for re-queueing, it may be reset */
1245         req->special = cmd;
1246         cmd->request = req;
1247         
1248         /*
1249          * FIXME: drop the lock here because the functions below
1250          * expect to be called without the queue lock held.  Also,
1251          * previously, we dequeued the request before dropping the
1252          * lock.  We hope REQ_STARTED prevents anything untoward from
1253          * happening now.
1254          */
1255         if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1256                 int ret;
1257
1258                 /*
1259                  * This will do a couple of things:
1260                  *  1) Fill in the actual SCSI command.
1261                  *  2) Fill in any other upper-level specific fields
1262                  * (timeout).
1263                  *
1264                  * If this returns 0, it means that the request failed
1265                  * (reading past end of disk, reading offline device,
1266                  * etc).   This won't actually talk to the device, but
1267                  * some kinds of consistency checking may cause the     
1268                  * request to be rejected immediately.
1269                  */
1270
1271                 /* 
1272                  * This sets up the scatter-gather table (allocating if
1273                  * required).
1274                  */
1275                 ret = scsi_init_io(cmd);
1276                 switch(ret) {
1277                         /* For BLKPREP_KILL/DEFER the cmd was released */
1278                 case BLKPREP_KILL:
1279                         goto kill;
1280                 case BLKPREP_DEFER:
1281                         goto defer;
1282                 }
1283                 
1284                 /*
1285                  * Initialize the actual SCSI command for this request.
1286                  */
1287                 if (req->flags & REQ_BLOCK_PC) {
1288                         scsi_setup_blk_pc_cmnd(cmd);
1289                 } else if (req->rq_disk) {
1290                         struct scsi_driver *drv;
1291
1292                         drv = *(struct scsi_driver **)req->rq_disk->private_data;
1293                         if (unlikely(!drv->init_command(cmd))) {
1294                                 scsi_release_buffers(cmd);
1295                                 scsi_put_command(cmd);
1296                                 goto kill;
1297                         }
1298                 }
1299         }
1300
1301         /*
1302          * The request is now prepped, no need to come back here
1303          */
1304         req->flags |= REQ_DONTPREP;
1305         return BLKPREP_OK;
1306
1307  defer:
1308         /* If we defer, the elv_next_request() returns NULL, but the
1309          * queue must be restarted, so we plug here if no returning
1310          * command will automatically do that. */
1311         if (sdev->device_busy == 0)
1312                 blk_plug_device(q);
1313         return BLKPREP_DEFER;
1314  kill:
1315         req->errors = DID_NO_CONNECT << 16;
1316         return BLKPREP_KILL;
1317 }
1318
1319 /*
1320  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1321  * return 0.
1322  *
1323  * Called with the queue_lock held.
1324  */
1325 static inline int scsi_dev_queue_ready(struct request_queue *q,
1326                                   struct scsi_device *sdev)
1327 {
1328         if (sdev->device_busy >= sdev->queue_depth)
1329                 return 0;
1330         if (sdev->device_busy == 0 && sdev->device_blocked) {
1331                 /*
1332                  * unblock after device_blocked iterates to zero
1333                  */
1334                 if (--sdev->device_blocked == 0) {
1335                         SCSI_LOG_MLQUEUE(3,
1336                                    sdev_printk(KERN_INFO, sdev,
1337                                    "unblocking device at zero depth\n"));
1338                 } else {
1339                         blk_plug_device(q);
1340                         return 0;
1341                 }
1342         }
1343         if (sdev->device_blocked)
1344                 return 0;
1345
1346         return 1;
1347 }
1348
1349 /*
1350  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1351  * return 0. We must end up running the queue again whenever 0 is
1352  * returned, else IO can hang.
1353  *
1354  * Called with host_lock held.
1355  */
1356 static inline int scsi_host_queue_ready(struct request_queue *q,
1357                                    struct Scsi_Host *shost,
1358                                    struct scsi_device *sdev)
1359 {
1360         if (scsi_host_in_recovery(shost))
1361                 return 0;
1362         if (shost->host_busy == 0 && shost->host_blocked) {
1363                 /*
1364                  * unblock after host_blocked iterates to zero
1365                  */
1366                 if (--shost->host_blocked == 0) {
1367                         SCSI_LOG_MLQUEUE(3,
1368                                 printk("scsi%d unblocking host at zero depth\n",
1369                                         shost->host_no));
1370                 } else {
1371                         blk_plug_device(q);
1372                         return 0;
1373                 }
1374         }
1375         if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
1376             shost->host_blocked || shost->host_self_blocked) {
1377                 if (list_empty(&sdev->starved_entry))
1378                         list_add_tail(&sdev->starved_entry, &shost->starved_list);
1379                 return 0;
1380         }
1381
1382         /* We're OK to process the command, so we can't be starved */
1383         if (!list_empty(&sdev->starved_entry))
1384                 list_del_init(&sdev->starved_entry);
1385
1386         return 1;
1387 }
1388
1389 /*
1390  * Kill a request for a dead device
1391  */
1392 static void scsi_kill_request(struct request *req, request_queue_t *q)
1393 {
1394         struct scsi_cmnd *cmd = req->special;
1395         struct scsi_device *sdev = cmd->device;
1396         struct Scsi_Host *shost = sdev->host;
1397
1398         blkdev_dequeue_request(req);
1399
1400         if (unlikely(cmd == NULL)) {
1401                 printk(KERN_CRIT "impossible request in %s.\n",
1402                                  __FUNCTION__);
1403                 BUG();
1404         }
1405
1406         scsi_init_cmd_errh(cmd);
1407         cmd->result = DID_NO_CONNECT << 16;
1408         atomic_inc(&cmd->device->iorequest_cnt);
1409
1410         /*
1411          * SCSI request completion path will do scsi_device_unbusy(),
1412          * bump busy counts.  To bump the counters, we need to dance
1413          * with the locks as normal issue path does.
1414          */
1415         sdev->device_busy++;
1416         spin_unlock(sdev->request_queue->queue_lock);
1417         spin_lock(shost->host_lock);
1418         shost->host_busy++;
1419         spin_unlock(shost->host_lock);
1420         spin_lock(sdev->request_queue->queue_lock);
1421
1422         __scsi_done(cmd);
1423 }
1424
1425 static void scsi_softirq_done(struct request *rq)
1426 {
1427         struct scsi_cmnd *cmd = rq->completion_data;
1428         unsigned long wait_for = (cmd->allowed + 1) * cmd->timeout_per_command;
1429         int disposition;
1430
1431         INIT_LIST_HEAD(&cmd->eh_entry);
1432
1433         disposition = scsi_decide_disposition(cmd);
1434         if (disposition != SUCCESS &&
1435             time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
1436                 sdev_printk(KERN_ERR, cmd->device,
1437                             "timing out command, waited %lus\n",
1438                             wait_for/HZ);
1439                 disposition = SUCCESS;
1440         }
1441                         
1442         scsi_log_completion(cmd, disposition);
1443
1444         switch (disposition) {
1445                 case SUCCESS:
1446                         scsi_finish_command(cmd);
1447                         break;
1448                 case NEEDS_RETRY:
1449                         scsi_retry_command(cmd);
1450                         break;
1451                 case ADD_TO_MLQUEUE:
1452                         scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
1453                         break;
1454                 default:
1455                         if (!scsi_eh_scmd_add(cmd, 0))
1456                                 scsi_finish_command(cmd);
1457         }
1458 }
1459
1460 /*
1461  * Function:    scsi_request_fn()
1462  *
1463  * Purpose:     Main strategy routine for SCSI.
1464  *
1465  * Arguments:   q       - Pointer to actual queue.
1466  *
1467  * Returns:     Nothing
1468  *
1469  * Lock status: IO request lock assumed to be held when called.
1470  */
1471 static void scsi_request_fn(struct request_queue *q)
1472 {
1473         struct scsi_device *sdev = q->queuedata;
1474         struct Scsi_Host *shost;
1475         struct scsi_cmnd *cmd;
1476         struct request *req;
1477
1478         if (!sdev) {
1479                 printk("scsi: killing requests for dead queue\n");
1480                 while ((req = elv_next_request(q)) != NULL)
1481                         scsi_kill_request(req, q);
1482                 return;
1483         }
1484
1485         if(!get_device(&sdev->sdev_gendev))
1486                 /* We must be tearing the block queue down already */
1487                 return;
1488
1489         /*
1490          * To start with, we keep looping until the queue is empty, or until
1491          * the host is no longer able to accept any more requests.
1492          */
1493         shost = sdev->host;
1494         while (!blk_queue_plugged(q)) {
1495                 int rtn;
1496                 /*
1497                  * get next queueable request.  We do this early to make sure
1498                  * that the request is fully prepared even if we cannot 
1499                  * accept it.
1500                  */
1501                 req = elv_next_request(q);
1502                 if (!req || !scsi_dev_queue_ready(q, sdev))
1503                         break;
1504
1505                 if (unlikely(!scsi_device_online(sdev))) {
1506                         sdev_printk(KERN_ERR, sdev,
1507                                     "rejecting I/O to offline device\n");
1508                         scsi_kill_request(req, q);
1509                         continue;
1510                 }
1511
1512
1513                 /*
1514                  * Remove the request from the request list.
1515                  */
1516                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1517                         blkdev_dequeue_request(req);
1518                 sdev->device_busy++;
1519
1520                 spin_unlock(q->queue_lock);
1521                 cmd = req->special;
1522                 if (unlikely(cmd == NULL)) {
1523                         printk(KERN_CRIT "impossible request in %s.\n"
1524                                          "please mail a stack trace to "
1525                                          "linux-scsi@vger.kernel.org",
1526                                          __FUNCTION__);
1527                         BUG();
1528                 }
1529                 spin_lock(shost->host_lock);
1530
1531                 if (!scsi_host_queue_ready(q, shost, sdev))
1532                         goto not_ready;
1533                 if (sdev->single_lun) {
1534                         if (scsi_target(sdev)->starget_sdev_user &&
1535                             scsi_target(sdev)->starget_sdev_user != sdev)
1536                                 goto not_ready;
1537                         scsi_target(sdev)->starget_sdev_user = sdev;
1538                 }
1539                 shost->host_busy++;
1540
1541                 /*
1542                  * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1543                  *              take the lock again.
1544                  */
1545                 spin_unlock_irq(shost->host_lock);
1546
1547                 /*
1548                  * Finally, initialize any error handling parameters, and set up
1549                  * the timers for timeouts.
1550                  */
1551                 scsi_init_cmd_errh(cmd);
1552
1553                 /*
1554                  * Dispatch the command to the low-level driver.
1555                  */
1556                 rtn = scsi_dispatch_cmd(cmd);
1557                 spin_lock_irq(q->queue_lock);
1558                 if(rtn) {
1559                         /* we're refusing the command; because of
1560                          * the way locks get dropped, we need to 
1561                          * check here if plugging is required */
1562                         if(sdev->device_busy == 0)
1563                                 blk_plug_device(q);
1564
1565                         break;
1566                 }
1567         }
1568
1569         goto out;
1570
1571  not_ready:
1572         spin_unlock_irq(shost->host_lock);
1573
1574         /*
1575          * lock q, handle tag, requeue req, and decrement device_busy. We
1576          * must return with queue_lock held.
1577          *
1578          * Decrementing device_busy without checking it is OK, as all such
1579          * cases (host limits or settings) should run the queue at some
1580          * later time.
1581          */
1582         spin_lock_irq(q->queue_lock);
1583         blk_requeue_request(q, req);
1584         sdev->device_busy--;
1585         if(sdev->device_busy == 0)
1586                 blk_plug_device(q);
1587  out:
1588         /* must be careful here...if we trigger the ->remove() function
1589          * we cannot be holding the q lock */
1590         spin_unlock_irq(q->queue_lock);
1591         put_device(&sdev->sdev_gendev);
1592         spin_lock_irq(q->queue_lock);
1593 }
1594
1595 u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1596 {
1597         struct device *host_dev;
1598         u64 bounce_limit = 0xffffffff;
1599
1600         if (shost->unchecked_isa_dma)
1601                 return BLK_BOUNCE_ISA;
1602         /*
1603          * Platforms with virtual-DMA translation
1604          * hardware have no practical limit.
1605          */
1606         if (!PCI_DMA_BUS_IS_PHYS)
1607                 return BLK_BOUNCE_ANY;
1608
1609         host_dev = scsi_get_device(shost);
1610         if (host_dev && host_dev->dma_mask)
1611                 bounce_limit = *host_dev->dma_mask;
1612
1613         return bounce_limit;
1614 }
1615 EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1616
1617 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1618 {
1619         struct Scsi_Host *shost = sdev->host;
1620         struct request_queue *q;
1621
1622         q = blk_init_queue(scsi_request_fn, NULL);
1623         if (!q)
1624                 return NULL;
1625
1626         blk_queue_prep_rq(q, scsi_prep_fn);
1627
1628         blk_queue_max_hw_segments(q, shost->sg_tablesize);
1629         blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS);
1630         blk_queue_max_sectors(q, shost->max_sectors);
1631         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1632         blk_queue_segment_boundary(q, shost->dma_boundary);
1633         blk_queue_issue_flush_fn(q, scsi_issue_flush_fn);
1634         blk_queue_softirq_done(q, scsi_softirq_done);
1635
1636         if (!shost->use_clustering)
1637                 clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
1638         return q;
1639 }
1640
1641 void scsi_free_queue(struct request_queue *q)
1642 {
1643         blk_cleanup_queue(q);
1644 }
1645
1646 /*
1647  * Function:    scsi_block_requests()
1648  *
1649  * Purpose:     Utility function used by low-level drivers to prevent further
1650  *              commands from being queued to the device.
1651  *
1652  * Arguments:   shost       - Host in question
1653  *
1654  * Returns:     Nothing
1655  *
1656  * Lock status: No locks are assumed held.
1657  *
1658  * Notes:       There is no timer nor any other means by which the requests
1659  *              get unblocked other than the low-level driver calling
1660  *              scsi_unblock_requests().
1661  */
1662 void scsi_block_requests(struct Scsi_Host *shost)
1663 {
1664         shost->host_self_blocked = 1;
1665 }
1666 EXPORT_SYMBOL(scsi_block_requests);
1667
1668 /*
1669  * Function:    scsi_unblock_requests()
1670  *
1671  * Purpose:     Utility function used by low-level drivers to allow further
1672  *              commands from being queued to the device.
1673  *
1674  * Arguments:   shost       - Host in question
1675  *
1676  * Returns:     Nothing
1677  *
1678  * Lock status: No locks are assumed held.
1679  *
1680  * Notes:       There is no timer nor any other means by which the requests
1681  *              get unblocked other than the low-level driver calling
1682  *              scsi_unblock_requests().
1683  *
1684  *              This is done as an API function so that changes to the
1685  *              internals of the scsi mid-layer won't require wholesale
1686  *              changes to drivers that use this feature.
1687  */
1688 void scsi_unblock_requests(struct Scsi_Host *shost)
1689 {
1690         shost->host_self_blocked = 0;
1691         scsi_run_host_queues(shost);
1692 }
1693 EXPORT_SYMBOL(scsi_unblock_requests);
1694
1695 int __init scsi_init_queue(void)
1696 {
1697         int i;
1698
1699         scsi_io_context_cache = kmem_cache_create("scsi_io_context",
1700                                         sizeof(struct scsi_io_context),
1701                                         0, 0, NULL, NULL);
1702         if (!scsi_io_context_cache) {
1703                 printk(KERN_ERR "SCSI: can't init scsi io context cache\n");
1704                 return -ENOMEM;
1705         }
1706
1707         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1708                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1709                 int size = sgp->size * sizeof(struct scatterlist);
1710
1711                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
1712                                 SLAB_HWCACHE_ALIGN, NULL, NULL);
1713                 if (!sgp->slab) {
1714                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1715                                         sgp->name);
1716                 }
1717
1718                 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
1719                                                      sgp->slab);
1720                 if (!sgp->pool) {
1721                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1722                                         sgp->name);
1723                 }
1724         }
1725
1726         return 0;
1727 }
1728
1729 void scsi_exit_queue(void)
1730 {
1731         int i;
1732
1733         kmem_cache_destroy(scsi_io_context_cache);
1734
1735         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1736                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1737                 mempool_destroy(sgp->pool);
1738                 kmem_cache_destroy(sgp->slab);
1739         }
1740 }
1741
1742 /**
1743  *      scsi_mode_select - issue a mode select
1744  *      @sdev:  SCSI device to be queried
1745  *      @pf:    Page format bit (1 == standard, 0 == vendor specific)
1746  *      @sp:    Save page bit (0 == don't save, 1 == save)
1747  *      @modepage: mode page being requested
1748  *      @buffer: request buffer (may not be smaller than eight bytes)
1749  *      @len:   length of request buffer.
1750  *      @timeout: command timeout
1751  *      @retries: number of retries before failing
1752  *      @data: returns a structure abstracting the mode header data
1753  *      @sense: place to put sense data (or NULL if no sense to be collected).
1754  *              must be SCSI_SENSE_BUFFERSIZE big.
1755  *
1756  *      Returns zero if successful; negative error number or scsi
1757  *      status on error
1758  *
1759  */
1760 int
1761 scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
1762                  unsigned char *buffer, int len, int timeout, int retries,
1763                  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1764 {
1765         unsigned char cmd[10];
1766         unsigned char *real_buffer;
1767         int ret;
1768
1769         memset(cmd, 0, sizeof(cmd));
1770         cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
1771
1772         if (sdev->use_10_for_ms) {
1773                 if (len > 65535)
1774                         return -EINVAL;
1775                 real_buffer = kmalloc(8 + len, GFP_KERNEL);
1776                 if (!real_buffer)
1777                         return -ENOMEM;
1778                 memcpy(real_buffer + 8, buffer, len);
1779                 len += 8;
1780                 real_buffer[0] = 0;
1781                 real_buffer[1] = 0;
1782                 real_buffer[2] = data->medium_type;
1783                 real_buffer[3] = data->device_specific;
1784                 real_buffer[4] = data->longlba ? 0x01 : 0;
1785                 real_buffer[5] = 0;
1786                 real_buffer[6] = data->block_descriptor_length >> 8;
1787                 real_buffer[7] = data->block_descriptor_length;
1788
1789                 cmd[0] = MODE_SELECT_10;
1790                 cmd[7] = len >> 8;
1791                 cmd[8] = len;
1792         } else {
1793                 if (len > 255 || data->block_descriptor_length > 255 ||
1794                     data->longlba)
1795                         return -EINVAL;
1796
1797                 real_buffer = kmalloc(4 + len, GFP_KERNEL);
1798                 if (!real_buffer)
1799                         return -ENOMEM;
1800                 memcpy(real_buffer + 4, buffer, len);
1801                 len += 4;
1802                 real_buffer[0] = 0;
1803                 real_buffer[1] = data->medium_type;
1804                 real_buffer[2] = data->device_specific;
1805                 real_buffer[3] = data->block_descriptor_length;
1806                 
1807
1808                 cmd[0] = MODE_SELECT;
1809                 cmd[4] = len;
1810         }
1811
1812         ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
1813                                sshdr, timeout, retries);
1814         kfree(real_buffer);
1815         return ret;
1816 }
1817 EXPORT_SYMBOL_GPL(scsi_mode_select);
1818
1819 /**
1820  *      scsi_mode_sense - issue a mode sense, falling back from 10 to 
1821  *              six bytes if necessary.
1822  *      @sdev:  SCSI device to be queried
1823  *      @dbd:   set if mode sense will allow block descriptors to be returned
1824  *      @modepage: mode page being requested
1825  *      @buffer: request buffer (may not be smaller than eight bytes)
1826  *      @len:   length of request buffer.
1827  *      @timeout: command timeout
1828  *      @retries: number of retries before failing
1829  *      @data: returns a structure abstracting the mode header data
1830  *      @sense: place to put sense data (or NULL if no sense to be collected).
1831  *              must be SCSI_SENSE_BUFFERSIZE big.
1832  *
1833  *      Returns zero if unsuccessful, or the header offset (either 4
1834  *      or 8 depending on whether a six or ten byte command was
1835  *      issued) if successful.
1836  **/
1837 int
1838 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1839                   unsigned char *buffer, int len, int timeout, int retries,
1840                   struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
1841 {
1842         unsigned char cmd[12];
1843         int use_10_for_ms;
1844         int header_length;
1845         int result;
1846         struct scsi_sense_hdr my_sshdr;
1847
1848         memset(data, 0, sizeof(*data));
1849         memset(&cmd[0], 0, 12);
1850         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
1851         cmd[2] = modepage;
1852
1853         /* caller might not be interested in sense, but we need it */
1854         if (!sshdr)
1855                 sshdr = &my_sshdr;
1856
1857  retry:
1858         use_10_for_ms = sdev->use_10_for_ms;
1859
1860         if (use_10_for_ms) {
1861                 if (len < 8)
1862                         len = 8;
1863
1864                 cmd[0] = MODE_SENSE_10;
1865                 cmd[8] = len;
1866                 header_length = 8;
1867         } else {
1868                 if (len < 4)
1869                         len = 4;
1870
1871                 cmd[0] = MODE_SENSE;
1872                 cmd[4] = len;
1873                 header_length = 4;
1874         }
1875
1876         memset(buffer, 0, len);
1877
1878         result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
1879                                   sshdr, timeout, retries);
1880
1881         /* This code looks awful: what it's doing is making sure an
1882          * ILLEGAL REQUEST sense return identifies the actual command
1883          * byte as the problem.  MODE_SENSE commands can return
1884          * ILLEGAL REQUEST if the code page isn't supported */
1885
1886         if (use_10_for_ms && !scsi_status_is_good(result) &&
1887             (driver_byte(result) & DRIVER_SENSE)) {
1888                 if (scsi_sense_valid(sshdr)) {
1889                         if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
1890                             (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
1891                                 /* 
1892                                  * Invalid command operation code
1893                                  */
1894                                 sdev->use_10_for_ms = 0;
1895                                 goto retry;
1896                         }
1897                 }
1898         }
1899
1900         if(scsi_status_is_good(result)) {
1901                 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
1902                              (modepage == 6 || modepage == 8))) {
1903                         /* Initio breakage? */
1904                         header_length = 0;
1905                         data->length = 13;
1906                         data->medium_type = 0;
1907                         data->device_specific = 0;
1908                         data->longlba = 0;
1909                         data->block_descriptor_length = 0;
1910                 } else if(use_10_for_ms) {
1911                         data->length = buffer[0]*256 + buffer[1] + 2;
1912                         data->medium_type = buffer[2];
1913                         data->device_specific = buffer[3];
1914                         data->longlba = buffer[4] & 0x01;
1915                         data->block_descriptor_length = buffer[6]*256
1916                                 + buffer[7];
1917                 } else {
1918                         data->length = buffer[0] + 1;
1919                         data->medium_type = buffer[1];
1920                         data->device_specific = buffer[2];
1921                         data->block_descriptor_length = buffer[3];
1922                 }
1923                 data->header_length = header_length;
1924         }
1925
1926         return result;
1927 }
1928 EXPORT_SYMBOL(scsi_mode_sense);
1929
1930 int
1931 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
1932 {
1933         char cmd[] = {
1934                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
1935         };
1936         struct scsi_sense_hdr sshdr;
1937         int result;
1938         
1939         result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
1940                                   timeout, retries);
1941
1942         if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) {
1943
1944                 if ((scsi_sense_valid(&sshdr)) &&
1945                     ((sshdr.sense_key == UNIT_ATTENTION) ||
1946                      (sshdr.sense_key == NOT_READY))) {
1947                         sdev->changed = 1;
1948                         result = 0;
1949                 }
1950         }
1951         return result;
1952 }
1953 EXPORT_SYMBOL(scsi_test_unit_ready);
1954
1955 /**
1956  *      scsi_device_set_state - Take the given device through the device
1957  *              state model.
1958  *      @sdev:  scsi device to change the state of.
1959  *      @state: state to change to.
1960  *
1961  *      Returns zero if unsuccessful or an error if the requested 
1962  *      transition is illegal.
1963  **/
1964 int
1965 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
1966 {
1967         enum scsi_device_state oldstate = sdev->sdev_state;
1968
1969         if (state == oldstate)
1970                 return 0;
1971
1972         switch (state) {
1973         case SDEV_CREATED:
1974                 /* There are no legal states that come back to
1975                  * created.  This is the manually initialised start
1976                  * state */
1977                 goto illegal;
1978                         
1979         case SDEV_RUNNING:
1980                 switch (oldstate) {
1981                 case SDEV_CREATED:
1982                 case SDEV_OFFLINE:
1983                 case SDEV_QUIESCE:
1984                 case SDEV_BLOCK:
1985                         break;
1986                 default:
1987                         goto illegal;
1988                 }
1989                 break;
1990
1991         case SDEV_QUIESCE:
1992                 switch (oldstate) {
1993                 case SDEV_RUNNING:
1994                 case SDEV_OFFLINE:
1995                         break;
1996                 default:
1997                         goto illegal;
1998                 }
1999                 break;
2000
2001         case SDEV_OFFLINE:
2002                 switch (oldstate) {
2003                 case SDEV_CREATED:
2004                 case SDEV_RUNNING:
2005                 case SDEV_QUIESCE:
2006                 case SDEV_BLOCK:
2007                         break;
2008                 default:
2009                         goto illegal;
2010                 }
2011                 break;
2012
2013         case SDEV_BLOCK:
2014                 switch (oldstate) {
2015                 case SDEV_CREATED:
2016                 case SDEV_RUNNING:
2017                         break;
2018                 default:
2019                         goto illegal;
2020                 }
2021                 break;
2022
2023         case SDEV_CANCEL:
2024                 switch (oldstate) {
2025                 case SDEV_CREATED:
2026                 case SDEV_RUNNING:
2027                 case SDEV_QUIESCE:
2028                 case SDEV_OFFLINE:
2029                 case SDEV_BLOCK:
2030                         break;
2031                 default:
2032                         goto illegal;
2033                 }
2034                 break;
2035
2036         case SDEV_DEL:
2037                 switch (oldstate) {
2038                 case SDEV_CREATED:
2039                 case SDEV_RUNNING:
2040                 case SDEV_OFFLINE:
2041                 case SDEV_CANCEL:
2042                         break;
2043                 default:
2044                         goto illegal;
2045                 }
2046                 break;
2047
2048         }
2049         sdev->sdev_state = state;
2050         return 0;
2051
2052  illegal:
2053         SCSI_LOG_ERROR_RECOVERY(1, 
2054                                 sdev_printk(KERN_ERR, sdev,
2055                                             "Illegal state transition %s->%s\n",
2056                                             scsi_device_state_name(oldstate),
2057                                             scsi_device_state_name(state))
2058                                 );
2059         return -EINVAL;
2060 }
2061 EXPORT_SYMBOL(scsi_device_set_state);
2062
2063 /**
2064  *      scsi_device_quiesce - Block user issued commands.
2065  *      @sdev:  scsi device to quiesce.
2066  *
2067  *      This works by trying to transition to the SDEV_QUIESCE state
2068  *      (which must be a legal transition).  When the device is in this
2069  *      state, only special requests will be accepted, all others will
2070  *      be deferred.  Since special requests may also be requeued requests,
2071  *      a successful return doesn't guarantee the device will be 
2072  *      totally quiescent.
2073  *
2074  *      Must be called with user context, may sleep.
2075  *
2076  *      Returns zero if unsuccessful or an error if not.
2077  **/
2078 int
2079 scsi_device_quiesce(struct scsi_device *sdev)
2080 {
2081         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
2082         if (err)
2083                 return err;
2084
2085         scsi_run_queue(sdev->request_queue);
2086         while (sdev->device_busy) {
2087                 msleep_interruptible(200);
2088                 scsi_run_queue(sdev->request_queue);
2089         }
2090         return 0;
2091 }
2092 EXPORT_SYMBOL(scsi_device_quiesce);
2093
2094 /**
2095  *      scsi_device_resume - Restart user issued commands to a quiesced device.
2096  *      @sdev:  scsi device to resume.
2097  *
2098  *      Moves the device from quiesced back to running and restarts the
2099  *      queues.
2100  *
2101  *      Must be called with user context, may sleep.
2102  **/
2103 void
2104 scsi_device_resume(struct scsi_device *sdev)
2105 {
2106         if(scsi_device_set_state(sdev, SDEV_RUNNING))
2107                 return;
2108         scsi_run_queue(sdev->request_queue);
2109 }
2110 EXPORT_SYMBOL(scsi_device_resume);
2111
2112 static void
2113 device_quiesce_fn(struct scsi_device *sdev, void *data)
2114 {
2115         scsi_device_quiesce(sdev);
2116 }
2117
2118 void
2119 scsi_target_quiesce(struct scsi_target *starget)
2120 {
2121         starget_for_each_device(starget, NULL, device_quiesce_fn);
2122 }
2123 EXPORT_SYMBOL(scsi_target_quiesce);
2124
2125 static void
2126 device_resume_fn(struct scsi_device *sdev, void *data)
2127 {
2128         scsi_device_resume(sdev);
2129 }
2130
2131 void
2132 scsi_target_resume(struct scsi_target *starget)
2133 {
2134         starget_for_each_device(starget, NULL, device_resume_fn);
2135 }
2136 EXPORT_SYMBOL(scsi_target_resume);
2137
2138 /**
2139  * scsi_internal_device_block - internal function to put a device
2140  *                              temporarily into the SDEV_BLOCK state
2141  * @sdev:       device to block
2142  *
2143  * Block request made by scsi lld's to temporarily stop all
2144  * scsi commands on the specified device.  Called from interrupt
2145  * or normal process context.
2146  *
2147  * Returns zero if successful or error if not
2148  *
2149  * Notes:       
2150  *      This routine transitions the device to the SDEV_BLOCK state
2151  *      (which must be a legal transition).  When the device is in this
2152  *      state, all commands are deferred until the scsi lld reenables
2153  *      the device with scsi_device_unblock or device_block_tmo fires.
2154  *      This routine assumes the host_lock is held on entry.
2155  **/
2156 int
2157 scsi_internal_device_block(struct scsi_device *sdev)
2158 {
2159         request_queue_t *q = sdev->request_queue;
2160         unsigned long flags;
2161         int err = 0;
2162
2163         err = scsi_device_set_state(sdev, SDEV_BLOCK);
2164         if (err)
2165                 return err;
2166
2167         /* 
2168          * The device has transitioned to SDEV_BLOCK.  Stop the
2169          * block layer from calling the midlayer with this device's
2170          * request queue. 
2171          */
2172         spin_lock_irqsave(q->queue_lock, flags);
2173         blk_stop_queue(q);
2174         spin_unlock_irqrestore(q->queue_lock, flags);
2175
2176         return 0;
2177 }
2178 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2179  
2180 /**
2181  * scsi_internal_device_unblock - resume a device after a block request
2182  * @sdev:       device to resume
2183  *
2184  * Called by scsi lld's or the midlayer to restart the device queue
2185  * for the previously suspended scsi device.  Called from interrupt or
2186  * normal process context.
2187  *
2188  * Returns zero if successful or error if not.
2189  *
2190  * Notes:       
2191  *      This routine transitions the device to the SDEV_RUNNING state
2192  *      (which must be a legal transition) allowing the midlayer to
2193  *      goose the queue for this device.  This routine assumes the 
2194  *      host_lock is held upon entry.
2195  **/
2196 int
2197 scsi_internal_device_unblock(struct scsi_device *sdev)
2198 {
2199         request_queue_t *q = sdev->request_queue; 
2200         int err;
2201         unsigned long flags;
2202         
2203         /* 
2204          * Try to transition the scsi device to SDEV_RUNNING
2205          * and goose the device queue if successful.  
2206          */
2207         err = scsi_device_set_state(sdev, SDEV_RUNNING);
2208         if (err)
2209                 return err;
2210
2211         spin_lock_irqsave(q->queue_lock, flags);
2212         blk_start_queue(q);
2213         spin_unlock_irqrestore(q->queue_lock, flags);
2214
2215         return 0;
2216 }
2217 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
2218
2219 static void
2220 device_block(struct scsi_device *sdev, void *data)
2221 {
2222         scsi_internal_device_block(sdev);
2223 }
2224
2225 static int
2226 target_block(struct device *dev, void *data)
2227 {
2228         if (scsi_is_target_device(dev))
2229                 starget_for_each_device(to_scsi_target(dev), NULL,
2230                                         device_block);
2231         return 0;
2232 }
2233
2234 void
2235 scsi_target_block(struct device *dev)
2236 {
2237         if (scsi_is_target_device(dev))
2238                 starget_for_each_device(to_scsi_target(dev), NULL,
2239                                         device_block);
2240         else
2241                 device_for_each_child(dev, NULL, target_block);
2242 }
2243 EXPORT_SYMBOL_GPL(scsi_target_block);
2244
2245 static void
2246 device_unblock(struct scsi_device *sdev, void *data)
2247 {
2248         scsi_internal_device_unblock(sdev);
2249 }
2250
2251 static int
2252 target_unblock(struct device *dev, void *data)
2253 {
2254         if (scsi_is_target_device(dev))
2255                 starget_for_each_device(to_scsi_target(dev), NULL,
2256                                         device_unblock);
2257         return 0;
2258 }
2259
2260 void
2261 scsi_target_unblock(struct device *dev)
2262 {
2263         if (scsi_is_target_device(dev))
2264                 starget_for_each_device(to_scsi_target(dev), NULL,
2265                                         device_unblock);
2266         else
2267                 device_for_each_child(dev, NULL, target_unblock);
2268 }
2269 EXPORT_SYMBOL_GPL(scsi_target_unblock);
2270
2271 /**
2272  * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
2273  * @sg:         scatter-gather list
2274  * @sg_count:   number of segments in sg
2275  * @offset:     offset in bytes into sg, on return offset into the mapped area
2276  * @len:        bytes to map, on return number of bytes mapped
2277  *
2278  * Returns virtual address of the start of the mapped page
2279  */
2280 void *scsi_kmap_atomic_sg(struct scatterlist *sg, int sg_count,
2281                           size_t *offset, size_t *len)
2282 {
2283         int i;
2284         size_t sg_len = 0, len_complete = 0;
2285         struct page *page;
2286
2287         for (i = 0; i < sg_count; i++) {
2288                 len_complete = sg_len; /* Complete sg-entries */
2289                 sg_len += sg[i].length;
2290                 if (sg_len > *offset)
2291                         break;
2292         }
2293
2294         if (unlikely(i == sg_count)) {
2295                 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
2296                         "elements %d\n",
2297                        __FUNCTION__, sg_len, *offset, sg_count);
2298                 WARN_ON(1);
2299                 return NULL;
2300         }
2301
2302         /* Offset starting from the beginning of first page in this sg-entry */
2303         *offset = *offset - len_complete + sg[i].offset;
2304
2305         /* Assumption: contiguous pages can be accessed as "page + i" */
2306         page = nth_page(sg[i].page, (*offset >> PAGE_SHIFT));
2307         *offset &= ~PAGE_MASK;
2308
2309         /* Bytes in this sg-entry from *offset to the end of the page */
2310         sg_len = PAGE_SIZE - *offset;
2311         if (*len > sg_len)
2312                 *len = sg_len;
2313
2314         return kmap_atomic(page, KM_BIO_SRC_IRQ);
2315 }
2316 EXPORT_SYMBOL(scsi_kmap_atomic_sg);
2317
2318 /**
2319  * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously
2320  *                         mapped with scsi_kmap_atomic_sg
2321  * @virt:       virtual address to be unmapped
2322  */
2323 void scsi_kunmap_atomic_sg(void *virt)
2324 {
2325         kunmap_atomic(virt, KM_BIO_SRC_IRQ);
2326 }
2327 EXPORT_SYMBOL(scsi_kunmap_atomic_sg);