[SCSI] make scsi_requeue_request() use blk_requeue_request()
[linux-2.6] / drivers / scsi / scsi_lib.c
1 /*
2  *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
3  *
4  *  SCSI queueing library.
5  *      Initial versions: Eric Youngdale (eric@andante.org).
6  *                        Based upon conversations with large numbers
7  *                        of people at Linux Expo.
8  */
9
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/completion.h>
13 #include <linux/kernel.h>
14 #include <linux/mempool.h>
15 #include <linux/slab.h>
16 #include <linux/init.h>
17 #include <linux/pci.h>
18 #include <linux/delay.h>
19
20 #include <scsi/scsi.h>
21 #include <scsi/scsi_dbg.h>
22 #include <scsi/scsi_device.h>
23 #include <scsi/scsi_driver.h>
24 #include <scsi/scsi_eh.h>
25 #include <scsi/scsi_host.h>
26 #include <scsi/scsi_request.h>
27
28 #include "scsi_priv.h"
29 #include "scsi_logging.h"
30
31
32 #define SG_MEMPOOL_NR           (sizeof(scsi_sg_pools)/sizeof(struct scsi_host_sg_pool))
33 #define SG_MEMPOOL_SIZE         32
34
35 struct scsi_host_sg_pool {
36         size_t          size;
37         char            *name; 
38         kmem_cache_t    *slab;
39         mempool_t       *pool;
40 };
41
42 #if (SCSI_MAX_PHYS_SEGMENTS < 32)
43 #error SCSI_MAX_PHYS_SEGMENTS is too small
44 #endif
45
46 #define SP(x) { x, "sgpool-" #x } 
47 struct scsi_host_sg_pool scsi_sg_pools[] = { 
48         SP(8),
49         SP(16),
50         SP(32),
51 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
52         SP(64),
53 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
54         SP(128),
55 #if (SCSI_MAX_PHYS_SEGMENTS > 128)
56         SP(256),
57 #if (SCSI_MAX_PHYS_SEGMENTS > 256)
58 #error SCSI_MAX_PHYS_SEGMENTS is too large
59 #endif
60 #endif
61 #endif
62 #endif
63 };      
64 #undef SP
65
66
67 /*
68  * Function:    scsi_insert_special_req()
69  *
70  * Purpose:     Insert pre-formed request into request queue.
71  *
72  * Arguments:   sreq    - request that is ready to be queued.
73  *              at_head - boolean.  True if we should insert at head
74  *                        of queue, false if we should insert at tail.
75  *
76  * Lock status: Assumed that lock is not held upon entry.
77  *
78  * Returns:     Nothing
79  *
80  * Notes:       This function is called from character device and from
81  *              ioctl types of functions where the caller knows exactly
82  *              what SCSI command needs to be issued.   The idea is that
83  *              we merely inject the command into the queue (at the head
84  *              for now), and then call the queue request function to actually
85  *              process it.
86  */
87 int scsi_insert_special_req(struct scsi_request *sreq, int at_head)
88 {
89         /*
90          * Because users of this function are apt to reuse requests with no
91          * modification, we have to sanitise the request flags here
92          */
93         sreq->sr_request->flags &= ~REQ_DONTPREP;
94         blk_insert_request(sreq->sr_device->request_queue, sreq->sr_request,
95                            at_head, sreq);
96         return 0;
97 }
98
99 /*
100  * Function:    scsi_queue_insert()
101  *
102  * Purpose:     Insert a command in the midlevel queue.
103  *
104  * Arguments:   cmd    - command that we are adding to queue.
105  *              reason - why we are inserting command to queue.
106  *
107  * Lock status: Assumed that lock is not held upon entry.
108  *
109  * Returns:     Nothing.
110  *
111  * Notes:       We do this for one of two cases.  Either the host is busy
112  *              and it cannot accept any more commands for the time being,
113  *              or the device returned QUEUE_FULL and can accept no more
114  *              commands.
115  * Notes:       This could be called either from an interrupt context or a
116  *              normal process context.
117  */
118 int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
119 {
120         struct Scsi_Host *host = cmd->device->host;
121         struct scsi_device *device = cmd->device;
122
123         SCSI_LOG_MLQUEUE(1,
124                  printk("Inserting command %p into mlqueue\n", cmd));
125
126         /*
127          * We are inserting the command into the ml queue.  First, we
128          * cancel the timer, so it doesn't time out.
129          */
130         scsi_delete_timer(cmd);
131
132         /*
133          * Next, set the appropriate busy bit for the device/host.
134          *
135          * If the host/device isn't busy, assume that something actually
136          * completed, and that we should be able to queue a command now.
137          *
138          * Note that the prior mid-layer assumption that any host could
139          * always queue at least one command is now broken.  The mid-layer
140          * will implement a user specifiable stall (see
141          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
142          * if a command is requeued with no other commands outstanding
143          * either for the device or for the host.
144          */
145         if (reason == SCSI_MLQUEUE_HOST_BUSY)
146                 host->host_blocked = host->max_host_blocked;
147         else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
148                 device->device_blocked = device->max_device_blocked;
149
150         /*
151          * Register the fact that we own the thing for now.
152          */
153         cmd->state = SCSI_STATE_MLQUEUE;
154         cmd->owner = SCSI_OWNER_MIDLEVEL;
155
156         /*
157          * Decrement the counters, since these commands are no longer
158          * active on the host/device.
159          */
160         scsi_device_unbusy(device);
161
162         /*
163          * Insert this command at the head of the queue for it's device.
164          * It will go before all other commands that are already in the queue.
165          *
166          * NOTE: there is magic here about the way the queue is plugged if
167          * we have no outstanding commands.
168          * 
169          * Although this *doesn't* plug the queue, it does call the request
170          * function.  The SCSI request function detects the blocked condition
171          * and plugs the queue appropriately.
172          */
173         blk_insert_request(device->request_queue, cmd->request, 1, cmd, 1);
174         return 0;
175 }
176
177 /*
178  * Function:    scsi_do_req
179  *
180  * Purpose:     Queue a SCSI request
181  *
182  * Arguments:   sreq      - command descriptor.
183  *              cmnd      - actual SCSI command to be performed.
184  *              buffer    - data buffer.
185  *              bufflen   - size of data buffer.
186  *              done      - completion function to be run.
187  *              timeout   - how long to let it run before timeout.
188  *              retries   - number of retries we allow.
189  *
190  * Lock status: No locks held upon entry.
191  *
192  * Returns:     Nothing.
193  *
194  * Notes:       This function is only used for queueing requests for things
195  *              like ioctls and character device requests - this is because
196  *              we essentially just inject a request into the queue for the
197  *              device.
198  *
199  *              In order to support the scsi_device_quiesce function, we
200  *              now inject requests on the *head* of the device queue
201  *              rather than the tail.
202  */
203 void scsi_do_req(struct scsi_request *sreq, const void *cmnd,
204                  void *buffer, unsigned bufflen,
205                  void (*done)(struct scsi_cmnd *),
206                  int timeout, int retries)
207 {
208         /*
209          * If the upper level driver is reusing these things, then
210          * we should release the low-level block now.  Another one will
211          * be allocated later when this request is getting queued.
212          */
213         __scsi_release_request(sreq);
214
215         /*
216          * Our own function scsi_done (which marks the host as not busy,
217          * disables the timeout counter, etc) will be called by us or by the
218          * scsi_hosts[host].queuecommand() function needs to also call
219          * the completion function for the high level driver.
220          */
221         memcpy(sreq->sr_cmnd, cmnd, sizeof(sreq->sr_cmnd));
222         sreq->sr_bufflen = bufflen;
223         sreq->sr_buffer = buffer;
224         sreq->sr_allowed = retries;
225         sreq->sr_done = done;
226         sreq->sr_timeout_per_command = timeout;
227
228         if (sreq->sr_cmd_len == 0)
229                 sreq->sr_cmd_len = COMMAND_SIZE(sreq->sr_cmnd[0]);
230
231         /*
232          * head injection *required* here otherwise quiesce won't work
233          */
234         scsi_insert_special_req(sreq, 1);
235 }
236 EXPORT_SYMBOL(scsi_do_req);
237
238 static void scsi_wait_done(struct scsi_cmnd *cmd)
239 {
240         struct request *req = cmd->request;
241         struct request_queue *q = cmd->device->request_queue;
242         unsigned long flags;
243
244         req->rq_status = RQ_SCSI_DONE;  /* Busy, but indicate request done */
245
246         spin_lock_irqsave(q->queue_lock, flags);
247         if (blk_rq_tagged(req))
248                 blk_queue_end_tag(q, req);
249         spin_unlock_irqrestore(q->queue_lock, flags);
250
251         if (req->waiting)
252                 complete(req->waiting);
253 }
254
255 /* This is the end routine we get to if a command was never attached
256  * to the request.  Simply complete the request without changing
257  * rq_status; this will cause a DRIVER_ERROR. */
258 static void scsi_wait_req_end_io(struct request *req)
259 {
260         BUG_ON(!req->waiting);
261
262         complete(req->waiting);
263 }
264
265 void scsi_wait_req(struct scsi_request *sreq, const void *cmnd, void *buffer,
266                    unsigned bufflen, int timeout, int retries)
267 {
268         DECLARE_COMPLETION(wait);
269         
270         sreq->sr_request->waiting = &wait;
271         sreq->sr_request->rq_status = RQ_SCSI_BUSY;
272         sreq->sr_request->end_io = scsi_wait_req_end_io;
273         scsi_do_req(sreq, cmnd, buffer, bufflen, scsi_wait_done,
274                         timeout, retries);
275         wait_for_completion(&wait);
276         sreq->sr_request->waiting = NULL;
277         if (sreq->sr_request->rq_status != RQ_SCSI_DONE)
278                 sreq->sr_result |= (DRIVER_ERROR << 24);
279
280         __scsi_release_request(sreq);
281 }
282 EXPORT_SYMBOL(scsi_wait_req);
283
284 /*
285  * Function:    scsi_init_cmd_errh()
286  *
287  * Purpose:     Initialize cmd fields related to error handling.
288  *
289  * Arguments:   cmd     - command that is ready to be queued.
290  *
291  * Returns:     Nothing
292  *
293  * Notes:       This function has the job of initializing a number of
294  *              fields related to error handling.   Typically this will
295  *              be called once for each command, as required.
296  */
297 static int scsi_init_cmd_errh(struct scsi_cmnd *cmd)
298 {
299         cmd->owner = SCSI_OWNER_MIDLEVEL;
300         cmd->serial_number = 0;
301         cmd->abort_reason = 0;
302
303         memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
304
305         if (cmd->cmd_len == 0)
306                 cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
307
308         /*
309          * We need saved copies of a number of fields - this is because
310          * error handling may need to overwrite these with different values
311          * to run different commands, and once error handling is complete,
312          * we will need to restore these values prior to running the actual
313          * command.
314          */
315         cmd->old_use_sg = cmd->use_sg;
316         cmd->old_cmd_len = cmd->cmd_len;
317         cmd->sc_old_data_direction = cmd->sc_data_direction;
318         cmd->old_underflow = cmd->underflow;
319         memcpy(cmd->data_cmnd, cmd->cmnd, sizeof(cmd->cmnd));
320         cmd->buffer = cmd->request_buffer;
321         cmd->bufflen = cmd->request_bufflen;
322         cmd->abort_reason = 0;
323
324         return 1;
325 }
326
327 /*
328  * Function:   scsi_setup_cmd_retry()
329  *
330  * Purpose:    Restore the command state for a retry
331  *
332  * Arguments:  cmd      - command to be restored
333  *
334  * Returns:    Nothing
335  *
336  * Notes:      Immediately prior to retrying a command, we need
337  *             to restore certain fields that we saved above.
338  */
339 void scsi_setup_cmd_retry(struct scsi_cmnd *cmd)
340 {
341         memcpy(cmd->cmnd, cmd->data_cmnd, sizeof(cmd->data_cmnd));
342         cmd->request_buffer = cmd->buffer;
343         cmd->request_bufflen = cmd->bufflen;
344         cmd->use_sg = cmd->old_use_sg;
345         cmd->cmd_len = cmd->old_cmd_len;
346         cmd->sc_data_direction = cmd->sc_old_data_direction;
347         cmd->underflow = cmd->old_underflow;
348 }
349
350 void scsi_device_unbusy(struct scsi_device *sdev)
351 {
352         struct Scsi_Host *shost = sdev->host;
353         unsigned long flags;
354
355         spin_lock_irqsave(shost->host_lock, flags);
356         shost->host_busy--;
357         if (unlikely(test_bit(SHOST_RECOVERY, &shost->shost_state) &&
358                      shost->host_failed))
359                 scsi_eh_wakeup(shost);
360         spin_unlock(shost->host_lock);
361         spin_lock(sdev->request_queue->queue_lock);
362         sdev->device_busy--;
363         spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
364 }
365
366 /*
367  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
368  * and call blk_run_queue for all the scsi_devices on the target -
369  * including current_sdev first.
370  *
371  * Called with *no* scsi locks held.
372  */
373 static void scsi_single_lun_run(struct scsi_device *current_sdev)
374 {
375         struct Scsi_Host *shost = current_sdev->host;
376         struct scsi_device *sdev, *tmp;
377         struct scsi_target *starget = scsi_target(current_sdev);
378         unsigned long flags;
379
380         spin_lock_irqsave(shost->host_lock, flags);
381         starget->starget_sdev_user = NULL;
382         spin_unlock_irqrestore(shost->host_lock, flags);
383
384         /*
385          * Call blk_run_queue for all LUNs on the target, starting with
386          * current_sdev. We race with others (to set starget_sdev_user),
387          * but in most cases, we will be first. Ideally, each LU on the
388          * target would get some limited time or requests on the target.
389          */
390         blk_run_queue(current_sdev->request_queue);
391
392         spin_lock_irqsave(shost->host_lock, flags);
393         if (starget->starget_sdev_user)
394                 goto out;
395         list_for_each_entry_safe(sdev, tmp, &starget->devices,
396                         same_target_siblings) {
397                 if (sdev == current_sdev)
398                         continue;
399                 if (scsi_device_get(sdev))
400                         continue;
401
402                 spin_unlock_irqrestore(shost->host_lock, flags);
403                 blk_run_queue(sdev->request_queue);
404                 spin_lock_irqsave(shost->host_lock, flags);
405         
406                 scsi_device_put(sdev);
407         }
408  out:
409         spin_unlock_irqrestore(shost->host_lock, flags);
410 }
411
412 /*
413  * Function:    scsi_run_queue()
414  *
415  * Purpose:     Select a proper request queue to serve next
416  *
417  * Arguments:   q       - last request's queue
418  *
419  * Returns:     Nothing
420  *
421  * Notes:       The previous command was completely finished, start
422  *              a new one if possible.
423  */
424 static void scsi_run_queue(struct request_queue *q)
425 {
426         struct scsi_device *sdev = q->queuedata;
427         struct Scsi_Host *shost = sdev->host;
428         unsigned long flags;
429
430         if (sdev->single_lun)
431                 scsi_single_lun_run(sdev);
432
433         spin_lock_irqsave(shost->host_lock, flags);
434         while (!list_empty(&shost->starved_list) &&
435                !shost->host_blocked && !shost->host_self_blocked &&
436                 !((shost->can_queue > 0) &&
437                   (shost->host_busy >= shost->can_queue))) {
438                 /*
439                  * As long as shost is accepting commands and we have
440                  * starved queues, call blk_run_queue. scsi_request_fn
441                  * drops the queue_lock and can add us back to the
442                  * starved_list.
443                  *
444                  * host_lock protects the starved_list and starved_entry.
445                  * scsi_request_fn must get the host_lock before checking
446                  * or modifying starved_list or starved_entry.
447                  */
448                 sdev = list_entry(shost->starved_list.next,
449                                           struct scsi_device, starved_entry);
450                 list_del_init(&sdev->starved_entry);
451                 spin_unlock_irqrestore(shost->host_lock, flags);
452
453                 blk_run_queue(sdev->request_queue);
454
455                 spin_lock_irqsave(shost->host_lock, flags);
456                 if (unlikely(!list_empty(&sdev->starved_entry)))
457                         /*
458                          * sdev lost a race, and was put back on the
459                          * starved list. This is unlikely but without this
460                          * in theory we could loop forever.
461                          */
462                         break;
463         }
464         spin_unlock_irqrestore(shost->host_lock, flags);
465
466         blk_run_queue(q);
467 }
468
469 /*
470  * Function:    scsi_requeue_command()
471  *
472  * Purpose:     Handle post-processing of completed commands.
473  *
474  * Arguments:   q       - queue to operate on
475  *              cmd     - command that may need to be requeued.
476  *
477  * Returns:     Nothing
478  *
479  * Notes:       After command completion, there may be blocks left
480  *              over which weren't finished by the previous command
481  *              this can be for a number of reasons - the main one is
482  *              I/O errors in the middle of the request, in which case
483  *              we need to request the blocks that come after the bad
484  *              sector.
485  */
486 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
487 {
488         unsigned long flags;
489
490         cmd->request->flags &= ~REQ_DONTPREP;
491
492         spin_lock_irqsave(q->queue_lock, flags);
493         blk_requeue_request(q, cmd->request);
494         spin_unlock_irqrestore(q->queue_lock, flags);
495
496         scsi_run_queue(q);
497 }
498
499 void scsi_next_command(struct scsi_cmnd *cmd)
500 {
501         struct request_queue *q = cmd->device->request_queue;
502
503         scsi_put_command(cmd);
504         scsi_run_queue(q);
505 }
506
507 void scsi_run_host_queues(struct Scsi_Host *shost)
508 {
509         struct scsi_device *sdev;
510
511         shost_for_each_device(sdev, shost)
512                 scsi_run_queue(sdev->request_queue);
513 }
514
515 /*
516  * Function:    scsi_end_request()
517  *
518  * Purpose:     Post-processing of completed commands (usually invoked at end
519  *              of upper level post-processing and scsi_io_completion).
520  *
521  * Arguments:   cmd      - command that is complete.
522  *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
523  *              bytes    - number of bytes of completed I/O
524  *              requeue  - indicates whether we should requeue leftovers.
525  *
526  * Lock status: Assumed that lock is not held upon entry.
527  *
528  * Returns:     cmd if requeue done or required, NULL otherwise
529  *
530  * Notes:       This is called for block device requests in order to
531  *              mark some number of sectors as complete.
532  * 
533  *              We are guaranteeing that the request queue will be goosed
534  *              at some point during this call.
535  */
536 static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
537                                           int bytes, int requeue)
538 {
539         request_queue_t *q = cmd->device->request_queue;
540         struct request *req = cmd->request;
541         unsigned long flags;
542
543         /*
544          * If there are blocks left over at the end, set up the command
545          * to queue the remainder of them.
546          */
547         if (end_that_request_chunk(req, uptodate, bytes)) {
548                 int leftover = (req->hard_nr_sectors << 9);
549
550                 if (blk_pc_request(req))
551                         leftover = req->data_len;
552
553                 /* kill remainder if no retrys */
554                 if (!uptodate && blk_noretry_request(req))
555                         end_that_request_chunk(req, 0, leftover);
556                 else {
557                         if (requeue)
558                                 /*
559                                  * Bleah.  Leftovers again.  Stick the
560                                  * leftovers in the front of the
561                                  * queue, and goose the queue again.
562                                  */
563                                 scsi_requeue_command(q, cmd);
564
565                         return cmd;
566                 }
567         }
568
569         add_disk_randomness(req->rq_disk);
570
571         spin_lock_irqsave(q->queue_lock, flags);
572         if (blk_rq_tagged(req))
573                 blk_queue_end_tag(q, req);
574         end_that_request_last(req);
575         spin_unlock_irqrestore(q->queue_lock, flags);
576
577         /*
578          * This will goose the queue request function at the end, so we don't
579          * need to worry about launching another command.
580          */
581         scsi_next_command(cmd);
582         return NULL;
583 }
584
585 static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, int gfp_mask)
586 {
587         struct scsi_host_sg_pool *sgp;
588         struct scatterlist *sgl;
589
590         BUG_ON(!cmd->use_sg);
591
592         switch (cmd->use_sg) {
593         case 1 ... 8:
594                 cmd->sglist_len = 0;
595                 break;
596         case 9 ... 16:
597                 cmd->sglist_len = 1;
598                 break;
599         case 17 ... 32:
600                 cmd->sglist_len = 2;
601                 break;
602 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
603         case 33 ... 64:
604                 cmd->sglist_len = 3;
605                 break;
606 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
607         case 65 ... 128:
608                 cmd->sglist_len = 4;
609                 break;
610 #if (SCSI_MAX_PHYS_SEGMENTS  > 128)
611         case 129 ... 256:
612                 cmd->sglist_len = 5;
613                 break;
614 #endif
615 #endif
616 #endif
617         default:
618                 return NULL;
619         }
620
621         sgp = scsi_sg_pools + cmd->sglist_len;
622         sgl = mempool_alloc(sgp->pool, gfp_mask);
623         if (sgl)
624                 memset(sgl, 0, sgp->size);
625         return sgl;
626 }
627
628 static void scsi_free_sgtable(struct scatterlist *sgl, int index)
629 {
630         struct scsi_host_sg_pool *sgp;
631
632         BUG_ON(index > SG_MEMPOOL_NR);
633
634         sgp = scsi_sg_pools + index;
635         mempool_free(sgl, sgp->pool);
636 }
637
638 /*
639  * Function:    scsi_release_buffers()
640  *
641  * Purpose:     Completion processing for block device I/O requests.
642  *
643  * Arguments:   cmd     - command that we are bailing.
644  *
645  * Lock status: Assumed that no lock is held upon entry.
646  *
647  * Returns:     Nothing
648  *
649  * Notes:       In the event that an upper level driver rejects a
650  *              command, we must release resources allocated during
651  *              the __init_io() function.  Primarily this would involve
652  *              the scatter-gather table, and potentially any bounce
653  *              buffers.
654  */
655 static void scsi_release_buffers(struct scsi_cmnd *cmd)
656 {
657         struct request *req = cmd->request;
658
659         /*
660          * Free up any indirection buffers we allocated for DMA purposes. 
661          */
662         if (cmd->use_sg)
663                 scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len);
664         else if (cmd->request_buffer != req->buffer)
665                 kfree(cmd->request_buffer);
666
667         /*
668          * Zero these out.  They now point to freed memory, and it is
669          * dangerous to hang onto the pointers.
670          */
671         cmd->buffer  = NULL;
672         cmd->bufflen = 0;
673         cmd->request_buffer = NULL;
674         cmd->request_bufflen = 0;
675 }
676
677 /*
678  * Function:    scsi_io_completion()
679  *
680  * Purpose:     Completion processing for block device I/O requests.
681  *
682  * Arguments:   cmd   - command that is finished.
683  *
684  * Lock status: Assumed that no lock is held upon entry.
685  *
686  * Returns:     Nothing
687  *
688  * Notes:       This function is matched in terms of capabilities to
689  *              the function that created the scatter-gather list.
690  *              In other words, if there are no bounce buffers
691  *              (the normal case for most drivers), we don't need
692  *              the logic to deal with cleaning up afterwards.
693  *
694  *              We must do one of several things here:
695  *
696  *              a) Call scsi_end_request.  This will finish off the
697  *                 specified number of sectors.  If we are done, the
698  *                 command block will be released, and the queue
699  *                 function will be goosed.  If we are not done, then
700  *                 scsi_end_request will directly goose the queue.
701  *
702  *              b) We can just use scsi_requeue_command() here.  This would
703  *                 be used if we just wanted to retry, for example.
704  */
705 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes,
706                         unsigned int block_bytes)
707 {
708         int result = cmd->result;
709         int this_count = cmd->bufflen;
710         request_queue_t *q = cmd->device->request_queue;
711         struct request *req = cmd->request;
712         int clear_errors = 1;
713         struct scsi_sense_hdr sshdr;
714         int sense_valid = 0;
715         int sense_deferred = 0;
716
717         if (blk_complete_barrier_rq(q, req, good_bytes >> 9))
718                 return;
719
720         /*
721          * Free up any indirection buffers we allocated for DMA purposes. 
722          * For the case of a READ, we need to copy the data out of the
723          * bounce buffer and into the real buffer.
724          */
725         if (cmd->use_sg)
726                 scsi_free_sgtable(cmd->buffer, cmd->sglist_len);
727         else if (cmd->buffer != req->buffer) {
728                 if (rq_data_dir(req) == READ) {
729                         unsigned long flags;
730                         char *to = bio_kmap_irq(req->bio, &flags);
731                         memcpy(to, cmd->buffer, cmd->bufflen);
732                         bio_kunmap_irq(to, &flags);
733                 }
734                 kfree(cmd->buffer);
735         }
736
737         if (result) {
738                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
739                 if (sense_valid)
740                         sense_deferred = scsi_sense_is_deferred(&sshdr);
741         }
742         if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
743                 req->errors = result;
744                 if (result) {
745                         clear_errors = 0;
746                         if (sense_valid && req->sense) {
747                                 /*
748                                  * SG_IO wants current and deferred errors
749                                  */
750                                 int len = 8 + cmd->sense_buffer[7];
751
752                                 if (len > SCSI_SENSE_BUFFERSIZE)
753                                         len = SCSI_SENSE_BUFFERSIZE;
754                                 memcpy(req->sense, cmd->sense_buffer,  len);
755                                 req->sense_len = len;
756                         }
757                 } else
758                         req->data_len = cmd->resid;
759         }
760
761         /*
762          * Zero these out.  They now point to freed memory, and it is
763          * dangerous to hang onto the pointers.
764          */
765         cmd->buffer  = NULL;
766         cmd->bufflen = 0;
767         cmd->request_buffer = NULL;
768         cmd->request_bufflen = 0;
769
770         /*
771          * Next deal with any sectors which we were able to correctly
772          * handle.
773          */
774         if (good_bytes >= 0) {
775                 SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, %d bytes done.\n",
776                                               req->nr_sectors, good_bytes));
777                 SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
778
779                 if (clear_errors)
780                         req->errors = 0;
781                 /*
782                  * If multiple sectors are requested in one buffer, then
783                  * they will have been finished off by the first command.
784                  * If not, then we have a multi-buffer command.
785                  *
786                  * If block_bytes != 0, it means we had a medium error
787                  * of some sort, and that we want to mark some number of
788                  * sectors as not uptodate.  Thus we want to inhibit
789                  * requeueing right here - we will requeue down below
790                  * when we handle the bad sectors.
791                  */
792                 cmd = scsi_end_request(cmd, 1, good_bytes, result == 0);
793
794                 /*
795                  * If the command completed without error, then either finish off the
796                  * rest of the command, or start a new one.
797                  */
798                 if (result == 0 || cmd == NULL ) {
799                         return;
800                 }
801         }
802         /*
803          * Now, if we were good little boys and girls, Santa left us a request
804          * sense buffer.  We can extract information from this, so we
805          * can choose a block to remap, etc.
806          */
807         if (sense_valid && !sense_deferred) {
808                 switch (sshdr.sense_key) {
809                 case UNIT_ATTENTION:
810                         if (cmd->device->removable) {
811                                 /* detected disc change.  set a bit 
812                                  * and quietly refuse further access.
813                                  */
814                                 cmd->device->changed = 1;
815                                 cmd = scsi_end_request(cmd, 0,
816                                                 this_count, 1);
817                                 return;
818                         } else {
819                                 /*
820                                 * Must have been a power glitch, or a
821                                 * bus reset.  Could not have been a
822                                 * media change, so we just retry the
823                                 * request and see what happens.  
824                                 */
825                                 scsi_requeue_command(q, cmd);
826                                 return;
827                         }
828                         break;
829                 case ILLEGAL_REQUEST:
830                         /*
831                         * If we had an ILLEGAL REQUEST returned, then we may
832                         * have performed an unsupported command.  The only
833                         * thing this should be would be a ten byte read where
834                         * only a six byte read was supported.  Also, on a
835                         * system where READ CAPACITY failed, we may have read
836                         * past the end of the disk.
837                         */
838                         if (cmd->device->use_10_for_rw &&
839                             (cmd->cmnd[0] == READ_10 ||
840                              cmd->cmnd[0] == WRITE_10)) {
841                                 cmd->device->use_10_for_rw = 0;
842                                 /*
843                                  * This will cause a retry with a 6-byte
844                                  * command.
845                                  */
846                                 scsi_requeue_command(q, cmd);
847                                 result = 0;
848                         } else {
849                                 cmd = scsi_end_request(cmd, 0, this_count, 1);
850                                 return;
851                         }
852                         break;
853                 case NOT_READY:
854                         /*
855                          * If the device is in the process of becoming ready,
856                          * retry.
857                          */
858                         if (sshdr.asc == 0x04 && sshdr.ascq == 0x01) {
859                                 scsi_requeue_command(q, cmd);
860                                 return;
861                         }
862                         printk(KERN_INFO "Device %s not ready.\n",
863                                req->rq_disk ? req->rq_disk->disk_name : "");
864                         cmd = scsi_end_request(cmd, 0, this_count, 1);
865                         return;
866                 case VOLUME_OVERFLOW:
867                         printk(KERN_INFO "Volume overflow <%d %d %d %d> CDB: ",
868                                cmd->device->host->host_no,
869                                (int)cmd->device->channel,
870                                (int)cmd->device->id, (int)cmd->device->lun);
871                         __scsi_print_command(cmd->data_cmnd);
872                         scsi_print_sense("", cmd);
873                         cmd = scsi_end_request(cmd, 0, block_bytes, 1);
874                         return;
875                 default:
876                         break;
877                 }
878         }                       /* driver byte != 0 */
879         if (host_byte(result) == DID_RESET) {
880                 /*
881                  * Third party bus reset or reset for error
882                  * recovery reasons.  Just retry the request
883                  * and see what happens.  
884                  */
885                 scsi_requeue_command(q, cmd);
886                 return;
887         }
888         if (result) {
889                 printk(KERN_INFO "SCSI error : <%d %d %d %d> return code "
890                        "= 0x%x\n", cmd->device->host->host_no,
891                        cmd->device->channel,
892                        cmd->device->id,
893                        cmd->device->lun, result);
894
895                 if (driver_byte(result) & DRIVER_SENSE)
896                         scsi_print_sense("", cmd);
897                 /*
898                  * Mark a single buffer as not uptodate.  Queue the remainder.
899                  * We sometimes get this cruft in the event that a medium error
900                  * isn't properly reported.
901                  */
902                 block_bytes = req->hard_cur_sectors << 9;
903                 if (!block_bytes)
904                         block_bytes = req->data_len;
905                 cmd = scsi_end_request(cmd, 0, block_bytes, 1);
906         }
907 }
908 EXPORT_SYMBOL(scsi_io_completion);
909
910 /*
911  * Function:    scsi_init_io()
912  *
913  * Purpose:     SCSI I/O initialize function.
914  *
915  * Arguments:   cmd   - Command descriptor we wish to initialize
916  *
917  * Returns:     0 on success
918  *              BLKPREP_DEFER if the failure is retryable
919  *              BLKPREP_KILL if the failure is fatal
920  */
921 static int scsi_init_io(struct scsi_cmnd *cmd)
922 {
923         struct request     *req = cmd->request;
924         struct scatterlist *sgpnt;
925         int                count;
926
927         /*
928          * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer
929          */
930         if ((req->flags & REQ_BLOCK_PC) && !req->bio) {
931                 cmd->request_bufflen = req->data_len;
932                 cmd->request_buffer = req->data;
933                 req->buffer = req->data;
934                 cmd->use_sg = 0;
935                 return 0;
936         }
937
938         /*
939          * we used to not use scatter-gather for single segment request,
940          * but now we do (it makes highmem I/O easier to support without
941          * kmapping pages)
942          */
943         cmd->use_sg = req->nr_phys_segments;
944
945         /*
946          * if sg table allocation fails, requeue request later.
947          */
948         sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
949         if (unlikely(!sgpnt))
950                 return BLKPREP_DEFER;
951
952         cmd->request_buffer = (char *) sgpnt;
953         cmd->request_bufflen = req->nr_sectors << 9;
954         if (blk_pc_request(req))
955                 cmd->request_bufflen = req->data_len;
956         req->buffer = NULL;
957
958         /* 
959          * Next, walk the list, and fill in the addresses and sizes of
960          * each segment.
961          */
962         count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
963
964         /*
965          * mapped well, send it off
966          */
967         if (likely(count <= cmd->use_sg)) {
968                 cmd->use_sg = count;
969                 return 0;
970         }
971
972         printk(KERN_ERR "Incorrect number of segments after building list\n");
973         printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg);
974         printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors,
975                         req->current_nr_sectors);
976
977         /* release the command and kill it */
978         scsi_release_buffers(cmd);
979         scsi_put_command(cmd);
980         return BLKPREP_KILL;
981 }
982
983 static int scsi_prepare_flush_fn(request_queue_t *q, struct request *rq)
984 {
985         struct scsi_device *sdev = q->queuedata;
986         struct scsi_driver *drv;
987
988         if (sdev->sdev_state == SDEV_RUNNING) {
989                 drv = *(struct scsi_driver **) rq->rq_disk->private_data;
990
991                 if (drv->prepare_flush)
992                         return drv->prepare_flush(q, rq);
993         }
994
995         return 0;
996 }
997
998 static void scsi_end_flush_fn(request_queue_t *q, struct request *rq)
999 {
1000         struct scsi_device *sdev = q->queuedata;
1001         struct request *flush_rq = rq->end_io_data;
1002         struct scsi_driver *drv;
1003
1004         if (flush_rq->errors) {
1005                 printk("scsi: barrier error, disabling flush support\n");
1006                 blk_queue_ordered(q, QUEUE_ORDERED_NONE);
1007         }
1008
1009         if (sdev->sdev_state == SDEV_RUNNING) {
1010                 drv = *(struct scsi_driver **) rq->rq_disk->private_data;
1011                 drv->end_flush(q, rq);
1012         }
1013 }
1014
1015 static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk,
1016                                sector_t *error_sector)
1017 {
1018         struct scsi_device *sdev = q->queuedata;
1019         struct scsi_driver *drv;
1020
1021         if (sdev->sdev_state != SDEV_RUNNING)
1022                 return -ENXIO;
1023
1024         drv = *(struct scsi_driver **) disk->private_data;
1025         if (drv->issue_flush)
1026                 return drv->issue_flush(&sdev->sdev_gendev, error_sector);
1027
1028         return -EOPNOTSUPP;
1029 }
1030
1031 static int scsi_prep_fn(struct request_queue *q, struct request *req)
1032 {
1033         struct scsi_device *sdev = q->queuedata;
1034         struct scsi_cmnd *cmd;
1035         int specials_only = 0;
1036
1037         /*
1038          * Just check to see if the device is online.  If it isn't, we
1039          * refuse to process any commands.  The device must be brought
1040          * online before trying any recovery commands
1041          */
1042         if (unlikely(!scsi_device_online(sdev))) {
1043                 printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n",
1044                        sdev->host->host_no, sdev->id, sdev->lun);
1045                 return BLKPREP_KILL;
1046         }
1047         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1048                 /* OK, we're not in a running state don't prep
1049                  * user commands */
1050                 if (sdev->sdev_state == SDEV_DEL) {
1051                         /* Device is fully deleted, no commands
1052                          * at all allowed down */
1053                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to dead device\n",
1054                                sdev->host->host_no, sdev->id, sdev->lun);
1055                         return BLKPREP_KILL;
1056                 }
1057                 /* OK, we only allow special commands (i.e. not
1058                  * user initiated ones */
1059                 specials_only = sdev->sdev_state;
1060         }
1061
1062         /*
1063          * Find the actual device driver associated with this command.
1064          * The SPECIAL requests are things like character device or
1065          * ioctls, which did not originate from ll_rw_blk.  Note that
1066          * the special field is also used to indicate the cmd for
1067          * the remainder of a partially fulfilled request that can 
1068          * come up when there is a medium error.  We have to treat
1069          * these two cases differently.  We differentiate by looking
1070          * at request->cmd, as this tells us the real story.
1071          */
1072         if (req->flags & REQ_SPECIAL) {
1073                 struct scsi_request *sreq = req->special;
1074
1075                 if (sreq->sr_magic == SCSI_REQ_MAGIC) {
1076                         cmd = scsi_get_command(sreq->sr_device, GFP_ATOMIC);
1077                         if (unlikely(!cmd))
1078                                 goto defer;
1079                         scsi_init_cmd_from_req(cmd, sreq);
1080                 } else
1081                         cmd = req->special;
1082         } else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1083
1084                 if(unlikely(specials_only)) {
1085                         if(specials_only == SDEV_QUIESCE ||
1086                                         specials_only == SDEV_BLOCK)
1087                                 return BLKPREP_DEFER;
1088                         
1089                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to device being removed\n",
1090                                sdev->host->host_no, sdev->id, sdev->lun);
1091                         return BLKPREP_KILL;
1092                 }
1093                         
1094                         
1095                 /*
1096                  * Now try and find a command block that we can use.
1097                  */
1098                 if (!req->special) {
1099                         cmd = scsi_get_command(sdev, GFP_ATOMIC);
1100                         if (unlikely(!cmd))
1101                                 goto defer;
1102                 } else
1103                         cmd = req->special;
1104                 
1105                 /* pull a tag out of the request if we have one */
1106                 cmd->tag = req->tag;
1107         } else {
1108                 blk_dump_rq_flags(req, "SCSI bad req");
1109                 return BLKPREP_KILL;
1110         }
1111         
1112         /* note the overloading of req->special.  When the tag
1113          * is active it always means cmd.  If the tag goes
1114          * back for re-queueing, it may be reset */
1115         req->special = cmd;
1116         cmd->request = req;
1117         
1118         /*
1119          * FIXME: drop the lock here because the functions below
1120          * expect to be called without the queue lock held.  Also,
1121          * previously, we dequeued the request before dropping the
1122          * lock.  We hope REQ_STARTED prevents anything untoward from
1123          * happening now.
1124          */
1125         if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1126                 struct scsi_driver *drv;
1127                 int ret;
1128
1129                 /*
1130                  * This will do a couple of things:
1131                  *  1) Fill in the actual SCSI command.
1132                  *  2) Fill in any other upper-level specific fields
1133                  * (timeout).
1134                  *
1135                  * If this returns 0, it means that the request failed
1136                  * (reading past end of disk, reading offline device,
1137                  * etc).   This won't actually talk to the device, but
1138                  * some kinds of consistency checking may cause the     
1139                  * request to be rejected immediately.
1140                  */
1141
1142                 /* 
1143                  * This sets up the scatter-gather table (allocating if
1144                  * required).
1145                  */
1146                 ret = scsi_init_io(cmd);
1147                 if (ret)        /* BLKPREP_KILL return also releases the command */
1148                         return ret;
1149                 
1150                 /*
1151                  * Initialize the actual SCSI command for this request.
1152                  */
1153                 drv = *(struct scsi_driver **)req->rq_disk->private_data;
1154                 if (unlikely(!drv->init_command(cmd))) {
1155                         scsi_release_buffers(cmd);
1156                         scsi_put_command(cmd);
1157                         return BLKPREP_KILL;
1158                 }
1159         }
1160
1161         /*
1162          * The request is now prepped, no need to come back here
1163          */
1164         req->flags |= REQ_DONTPREP;
1165         return BLKPREP_OK;
1166
1167  defer:
1168         /* If we defer, the elv_next_request() returns NULL, but the
1169          * queue must be restarted, so we plug here if no returning
1170          * command will automatically do that. */
1171         if (sdev->device_busy == 0)
1172                 blk_plug_device(q);
1173         return BLKPREP_DEFER;
1174 }
1175
1176 /*
1177  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1178  * return 0.
1179  *
1180  * Called with the queue_lock held.
1181  */
1182 static inline int scsi_dev_queue_ready(struct request_queue *q,
1183                                   struct scsi_device *sdev)
1184 {
1185         if (sdev->device_busy >= sdev->queue_depth)
1186                 return 0;
1187         if (sdev->device_busy == 0 && sdev->device_blocked) {
1188                 /*
1189                  * unblock after device_blocked iterates to zero
1190                  */
1191                 if (--sdev->device_blocked == 0) {
1192                         SCSI_LOG_MLQUEUE(3,
1193                                 printk("scsi%d (%d:%d) unblocking device at"
1194                                        " zero depth\n", sdev->host->host_no,
1195                                        sdev->id, sdev->lun));
1196                 } else {
1197                         blk_plug_device(q);
1198                         return 0;
1199                 }
1200         }
1201         if (sdev->device_blocked)
1202                 return 0;
1203
1204         return 1;
1205 }
1206
1207 /*
1208  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1209  * return 0. We must end up running the queue again whenever 0 is
1210  * returned, else IO can hang.
1211  *
1212  * Called with host_lock held.
1213  */
1214 static inline int scsi_host_queue_ready(struct request_queue *q,
1215                                    struct Scsi_Host *shost,
1216                                    struct scsi_device *sdev)
1217 {
1218         if (test_bit(SHOST_RECOVERY, &shost->shost_state))
1219                 return 0;
1220         if (shost->host_busy == 0 && shost->host_blocked) {
1221                 /*
1222                  * unblock after host_blocked iterates to zero
1223                  */
1224                 if (--shost->host_blocked == 0) {
1225                         SCSI_LOG_MLQUEUE(3,
1226                                 printk("scsi%d unblocking host at zero depth\n",
1227                                         shost->host_no));
1228                 } else {
1229                         blk_plug_device(q);
1230                         return 0;
1231                 }
1232         }
1233         if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
1234             shost->host_blocked || shost->host_self_blocked) {
1235                 if (list_empty(&sdev->starved_entry))
1236                         list_add_tail(&sdev->starved_entry, &shost->starved_list);
1237                 return 0;
1238         }
1239
1240         /* We're OK to process the command, so we can't be starved */
1241         if (!list_empty(&sdev->starved_entry))
1242                 list_del_init(&sdev->starved_entry);
1243
1244         return 1;
1245 }
1246
1247 /*
1248  * Kill requests for a dead device
1249  */
1250 static void scsi_kill_requests(request_queue_t *q)
1251 {
1252         struct request *req;
1253
1254         while ((req = elv_next_request(q)) != NULL) {
1255                 blkdev_dequeue_request(req);
1256                 req->flags |= REQ_QUIET;
1257                 while (end_that_request_first(req, 0, req->nr_sectors))
1258                         ;
1259                 end_that_request_last(req);
1260         }
1261 }
1262
1263 /*
1264  * Function:    scsi_request_fn()
1265  *
1266  * Purpose:     Main strategy routine for SCSI.
1267  *
1268  * Arguments:   q       - Pointer to actual queue.
1269  *
1270  * Returns:     Nothing
1271  *
1272  * Lock status: IO request lock assumed to be held when called.
1273  */
1274 static void scsi_request_fn(struct request_queue *q)
1275 {
1276         struct scsi_device *sdev = q->queuedata;
1277         struct Scsi_Host *shost;
1278         struct scsi_cmnd *cmd;
1279         struct request *req;
1280
1281         if (!sdev) {
1282                 printk("scsi: killing requests for dead queue\n");
1283                 scsi_kill_requests(q);
1284                 return;
1285         }
1286
1287         if(!get_device(&sdev->sdev_gendev))
1288                 /* We must be tearing the block queue down already */
1289                 return;
1290
1291         /*
1292          * To start with, we keep looping until the queue is empty, or until
1293          * the host is no longer able to accept any more requests.
1294          */
1295         shost = sdev->host;
1296         while (!blk_queue_plugged(q)) {
1297                 int rtn;
1298                 /*
1299                  * get next queueable request.  We do this early to make sure
1300                  * that the request is fully prepared even if we cannot 
1301                  * accept it.
1302                  */
1303                 req = elv_next_request(q);
1304                 if (!req || !scsi_dev_queue_ready(q, sdev))
1305                         break;
1306
1307                 if (unlikely(!scsi_device_online(sdev))) {
1308                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n",
1309                                sdev->host->host_no, sdev->id, sdev->lun);
1310                         blkdev_dequeue_request(req);
1311                         req->flags |= REQ_QUIET;
1312                         while (end_that_request_first(req, 0, req->nr_sectors))
1313                                 ;
1314                         end_that_request_last(req);
1315                         continue;
1316                 }
1317
1318
1319                 /*
1320                  * Remove the request from the request list.
1321                  */
1322                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1323                         blkdev_dequeue_request(req);
1324                 sdev->device_busy++;
1325
1326                 spin_unlock(q->queue_lock);
1327                 spin_lock(shost->host_lock);
1328
1329                 if (!scsi_host_queue_ready(q, shost, sdev))
1330                         goto not_ready;
1331                 if (sdev->single_lun) {
1332                         if (scsi_target(sdev)->starget_sdev_user &&
1333                             scsi_target(sdev)->starget_sdev_user != sdev)
1334                                 goto not_ready;
1335                         scsi_target(sdev)->starget_sdev_user = sdev;
1336                 }
1337                 shost->host_busy++;
1338
1339                 /*
1340                  * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1341                  *              take the lock again.
1342                  */
1343                 spin_unlock_irq(shost->host_lock);
1344
1345                 cmd = req->special;
1346                 if (unlikely(cmd == NULL)) {
1347                         printk(KERN_CRIT "impossible request in %s.\n"
1348                                          "please mail a stack trace to "
1349                                          "linux-scsi@vger.kernel.org",
1350                                          __FUNCTION__);
1351                         BUG();
1352                 }
1353
1354                 /*
1355                  * Finally, initialize any error handling parameters, and set up
1356                  * the timers for timeouts.
1357                  */
1358                 scsi_init_cmd_errh(cmd);
1359
1360                 /*
1361                  * Dispatch the command to the low-level driver.
1362                  */
1363                 rtn = scsi_dispatch_cmd(cmd);
1364                 spin_lock_irq(q->queue_lock);
1365                 if(rtn) {
1366                         /* we're refusing the command; because of
1367                          * the way locks get dropped, we need to 
1368                          * check here if plugging is required */
1369                         if(sdev->device_busy == 0)
1370                                 blk_plug_device(q);
1371
1372                         break;
1373                 }
1374         }
1375
1376         goto out;
1377
1378  not_ready:
1379         spin_unlock_irq(shost->host_lock);
1380
1381         /*
1382          * lock q, handle tag, requeue req, and decrement device_busy. We
1383          * must return with queue_lock held.
1384          *
1385          * Decrementing device_busy without checking it is OK, as all such
1386          * cases (host limits or settings) should run the queue at some
1387          * later time.
1388          */
1389         spin_lock_irq(q->queue_lock);
1390         blk_requeue_request(q, req);
1391         sdev->device_busy--;
1392         if(sdev->device_busy == 0)
1393                 blk_plug_device(q);
1394  out:
1395         /* must be careful here...if we trigger the ->remove() function
1396          * we cannot be holding the q lock */
1397         spin_unlock_irq(q->queue_lock);
1398         put_device(&sdev->sdev_gendev);
1399         spin_lock_irq(q->queue_lock);
1400 }
1401
1402 u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1403 {
1404         struct device *host_dev;
1405         u64 bounce_limit = 0xffffffff;
1406
1407         if (shost->unchecked_isa_dma)
1408                 return BLK_BOUNCE_ISA;
1409         /*
1410          * Platforms with virtual-DMA translation
1411          * hardware have no practical limit.
1412          */
1413         if (!PCI_DMA_BUS_IS_PHYS)
1414                 return BLK_BOUNCE_ANY;
1415
1416         host_dev = scsi_get_device(shost);
1417         if (host_dev && host_dev->dma_mask)
1418                 bounce_limit = *host_dev->dma_mask;
1419
1420         return bounce_limit;
1421 }
1422 EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1423
1424 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1425 {
1426         struct Scsi_Host *shost = sdev->host;
1427         struct request_queue *q;
1428
1429         q = blk_init_queue(scsi_request_fn, NULL);
1430         if (!q)
1431                 return NULL;
1432
1433         blk_queue_prep_rq(q, scsi_prep_fn);
1434
1435         blk_queue_max_hw_segments(q, shost->sg_tablesize);
1436         blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS);
1437         blk_queue_max_sectors(q, shost->max_sectors);
1438         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1439         blk_queue_segment_boundary(q, shost->dma_boundary);
1440         blk_queue_issue_flush_fn(q, scsi_issue_flush_fn);
1441
1442         /*
1443          * ordered tags are superior to flush ordering
1444          */
1445         if (shost->ordered_tag)
1446                 blk_queue_ordered(q, QUEUE_ORDERED_TAG);
1447         else if (shost->ordered_flush) {
1448                 blk_queue_ordered(q, QUEUE_ORDERED_FLUSH);
1449                 q->prepare_flush_fn = scsi_prepare_flush_fn;
1450                 q->end_flush_fn = scsi_end_flush_fn;
1451         }
1452
1453         if (!shost->use_clustering)
1454                 clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
1455         return q;
1456 }
1457
1458 void scsi_free_queue(struct request_queue *q)
1459 {
1460         blk_cleanup_queue(q);
1461 }
1462
1463 /*
1464  * Function:    scsi_block_requests()
1465  *
1466  * Purpose:     Utility function used by low-level drivers to prevent further
1467  *              commands from being queued to the device.
1468  *
1469  * Arguments:   shost       - Host in question
1470  *
1471  * Returns:     Nothing
1472  *
1473  * Lock status: No locks are assumed held.
1474  *
1475  * Notes:       There is no timer nor any other means by which the requests
1476  *              get unblocked other than the low-level driver calling
1477  *              scsi_unblock_requests().
1478  */
1479 void scsi_block_requests(struct Scsi_Host *shost)
1480 {
1481         shost->host_self_blocked = 1;
1482 }
1483 EXPORT_SYMBOL(scsi_block_requests);
1484
1485 /*
1486  * Function:    scsi_unblock_requests()
1487  *
1488  * Purpose:     Utility function used by low-level drivers to allow further
1489  *              commands from being queued to the device.
1490  *
1491  * Arguments:   shost       - Host in question
1492  *
1493  * Returns:     Nothing
1494  *
1495  * Lock status: No locks are assumed held.
1496  *
1497  * Notes:       There is no timer nor any other means by which the requests
1498  *              get unblocked other than the low-level driver calling
1499  *              scsi_unblock_requests().
1500  *
1501  *              This is done as an API function so that changes to the
1502  *              internals of the scsi mid-layer won't require wholesale
1503  *              changes to drivers that use this feature.
1504  */
1505 void scsi_unblock_requests(struct Scsi_Host *shost)
1506 {
1507         shost->host_self_blocked = 0;
1508         scsi_run_host_queues(shost);
1509 }
1510 EXPORT_SYMBOL(scsi_unblock_requests);
1511
1512 int __init scsi_init_queue(void)
1513 {
1514         int i;
1515
1516         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1517                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1518                 int size = sgp->size * sizeof(struct scatterlist);
1519
1520                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
1521                                 SLAB_HWCACHE_ALIGN, NULL, NULL);
1522                 if (!sgp->slab) {
1523                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1524                                         sgp->name);
1525                 }
1526
1527                 sgp->pool = mempool_create(SG_MEMPOOL_SIZE,
1528                                 mempool_alloc_slab, mempool_free_slab,
1529                                 sgp->slab);
1530                 if (!sgp->pool) {
1531                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1532                                         sgp->name);
1533                 }
1534         }
1535
1536         return 0;
1537 }
1538
1539 void scsi_exit_queue(void)
1540 {
1541         int i;
1542
1543         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1544                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1545                 mempool_destroy(sgp->pool);
1546                 kmem_cache_destroy(sgp->slab);
1547         }
1548 }
1549 /**
1550  *      __scsi_mode_sense - issue a mode sense, falling back from 10 to 
1551  *              six bytes if necessary.
1552  *      @sreq:  SCSI request to fill in with the MODE_SENSE
1553  *      @dbd:   set if mode sense will allow block descriptors to be returned
1554  *      @modepage: mode page being requested
1555  *      @buffer: request buffer (may not be smaller than eight bytes)
1556  *      @len:   length of request buffer.
1557  *      @timeout: command timeout
1558  *      @retries: number of retries before failing
1559  *      @data: returns a structure abstracting the mode header data
1560  *
1561  *      Returns zero if unsuccessful, or the header offset (either 4
1562  *      or 8 depending on whether a six or ten byte command was
1563  *      issued) if successful.
1564  **/
1565 int
1566 __scsi_mode_sense(struct scsi_request *sreq, int dbd, int modepage,
1567                   unsigned char *buffer, int len, int timeout, int retries,
1568                   struct scsi_mode_data *data) {
1569         unsigned char cmd[12];
1570         int use_10_for_ms;
1571         int header_length;
1572
1573         memset(data, 0, sizeof(*data));
1574         memset(&cmd[0], 0, 12);
1575         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
1576         cmd[2] = modepage;
1577
1578  retry:
1579         use_10_for_ms = sreq->sr_device->use_10_for_ms;
1580
1581         if (use_10_for_ms) {
1582                 if (len < 8)
1583                         len = 8;
1584
1585                 cmd[0] = MODE_SENSE_10;
1586                 cmd[8] = len;
1587                 header_length = 8;
1588         } else {
1589                 if (len < 4)
1590                         len = 4;
1591
1592                 cmd[0] = MODE_SENSE;
1593                 cmd[4] = len;
1594                 header_length = 4;
1595         }
1596
1597         sreq->sr_cmd_len = 0;
1598         memset(sreq->sr_sense_buffer, 0, sizeof(sreq->sr_sense_buffer));
1599         sreq->sr_data_direction = DMA_FROM_DEVICE;
1600
1601         memset(buffer, 0, len);
1602
1603         scsi_wait_req(sreq, cmd, buffer, len, timeout, retries);
1604
1605         /* This code looks awful: what it's doing is making sure an
1606          * ILLEGAL REQUEST sense return identifies the actual command
1607          * byte as the problem.  MODE_SENSE commands can return
1608          * ILLEGAL REQUEST if the code page isn't supported */
1609
1610         if (use_10_for_ms && !scsi_status_is_good(sreq->sr_result) &&
1611             (driver_byte(sreq->sr_result) & DRIVER_SENSE)) {
1612                 struct scsi_sense_hdr sshdr;
1613
1614                 if (scsi_request_normalize_sense(sreq, &sshdr)) {
1615                         if ((sshdr.sense_key == ILLEGAL_REQUEST) &&
1616                             (sshdr.asc == 0x20) && (sshdr.ascq == 0)) {
1617                                 /* 
1618                                  * Invalid command operation code
1619                                  */
1620                                 sreq->sr_device->use_10_for_ms = 0;
1621                                 goto retry;
1622                         }
1623                 }
1624         }
1625
1626         if(scsi_status_is_good(sreq->sr_result)) {
1627                 data->header_length = header_length;
1628                 if(use_10_for_ms) {
1629                         data->length = buffer[0]*256 + buffer[1] + 2;
1630                         data->medium_type = buffer[2];
1631                         data->device_specific = buffer[3];
1632                         data->longlba = buffer[4] & 0x01;
1633                         data->block_descriptor_length = buffer[6]*256
1634                                 + buffer[7];
1635                 } else {
1636                         data->length = buffer[0] + 1;
1637                         data->medium_type = buffer[1];
1638                         data->device_specific = buffer[2];
1639                         data->block_descriptor_length = buffer[3];
1640                 }
1641         }
1642
1643         return sreq->sr_result;
1644 }
1645 EXPORT_SYMBOL(__scsi_mode_sense);
1646
1647 /**
1648  *      scsi_mode_sense - issue a mode sense, falling back from 10 to 
1649  *              six bytes if necessary.
1650  *      @sdev:  scsi device to send command to.
1651  *      @dbd:   set if mode sense will disable block descriptors in the return
1652  *      @modepage: mode page being requested
1653  *      @buffer: request buffer (may not be smaller than eight bytes)
1654  *      @len:   length of request buffer.
1655  *      @timeout: command timeout
1656  *      @retries: number of retries before failing
1657  *
1658  *      Returns zero if unsuccessful, or the header offset (either 4
1659  *      or 8 depending on whether a six or ten byte command was
1660  *      issued) if successful.
1661  **/
1662 int
1663 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1664                 unsigned char *buffer, int len, int timeout, int retries,
1665                 struct scsi_mode_data *data)
1666 {
1667         struct scsi_request *sreq = scsi_allocate_request(sdev, GFP_KERNEL);
1668         int ret;
1669
1670         if (!sreq)
1671                 return -1;
1672
1673         ret = __scsi_mode_sense(sreq, dbd, modepage, buffer, len,
1674                                 timeout, retries, data);
1675
1676         scsi_release_request(sreq);
1677
1678         return ret;
1679 }
1680 EXPORT_SYMBOL(scsi_mode_sense);
1681
1682 int
1683 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
1684 {
1685         struct scsi_request *sreq;
1686         char cmd[] = {
1687                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
1688         };
1689         int result;
1690         
1691         sreq = scsi_allocate_request(sdev, GFP_KERNEL);
1692         if (!sreq)
1693                 return -ENOMEM;
1694
1695         sreq->sr_data_direction = DMA_NONE;
1696         scsi_wait_req(sreq, cmd, NULL, 0, timeout, retries);
1697
1698         if ((driver_byte(sreq->sr_result) & DRIVER_SENSE) && sdev->removable) {
1699                 struct scsi_sense_hdr sshdr;
1700
1701                 if ((scsi_request_normalize_sense(sreq, &sshdr)) &&
1702                     ((sshdr.sense_key == UNIT_ATTENTION) ||
1703                      (sshdr.sense_key == NOT_READY))) {
1704                         sdev->changed = 1;
1705                         sreq->sr_result = 0;
1706                 }
1707         }
1708         result = sreq->sr_result;
1709         scsi_release_request(sreq);
1710         return result;
1711 }
1712 EXPORT_SYMBOL(scsi_test_unit_ready);
1713
1714 /**
1715  *      scsi_device_set_state - Take the given device through the device
1716  *              state model.
1717  *      @sdev:  scsi device to change the state of.
1718  *      @state: state to change to.
1719  *
1720  *      Returns zero if unsuccessful or an error if the requested 
1721  *      transition is illegal.
1722  **/
1723 int
1724 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
1725 {
1726         enum scsi_device_state oldstate = sdev->sdev_state;
1727
1728         if (state == oldstate)
1729                 return 0;
1730
1731         switch (state) {
1732         case SDEV_CREATED:
1733                 /* There are no legal states that come back to
1734                  * created.  This is the manually initialised start
1735                  * state */
1736                 goto illegal;
1737                         
1738         case SDEV_RUNNING:
1739                 switch (oldstate) {
1740                 case SDEV_CREATED:
1741                 case SDEV_OFFLINE:
1742                 case SDEV_QUIESCE:
1743                 case SDEV_BLOCK:
1744                         break;
1745                 default:
1746                         goto illegal;
1747                 }
1748                 break;
1749
1750         case SDEV_QUIESCE:
1751                 switch (oldstate) {
1752                 case SDEV_RUNNING:
1753                 case SDEV_OFFLINE:
1754                         break;
1755                 default:
1756                         goto illegal;
1757                 }
1758                 break;
1759
1760         case SDEV_OFFLINE:
1761                 switch (oldstate) {
1762                 case SDEV_CREATED:
1763                 case SDEV_RUNNING:
1764                 case SDEV_QUIESCE:
1765                 case SDEV_BLOCK:
1766                         break;
1767                 default:
1768                         goto illegal;
1769                 }
1770                 break;
1771
1772         case SDEV_BLOCK:
1773                 switch (oldstate) {
1774                 case SDEV_CREATED:
1775                 case SDEV_RUNNING:
1776                         break;
1777                 default:
1778                         goto illegal;
1779                 }
1780                 break;
1781
1782         case SDEV_CANCEL:
1783                 switch (oldstate) {
1784                 case SDEV_CREATED:
1785                 case SDEV_RUNNING:
1786                 case SDEV_OFFLINE:
1787                 case SDEV_BLOCK:
1788                         break;
1789                 default:
1790                         goto illegal;
1791                 }
1792                 break;
1793
1794         case SDEV_DEL:
1795                 switch (oldstate) {
1796                 case SDEV_CANCEL:
1797                         break;
1798                 default:
1799                         goto illegal;
1800                 }
1801                 break;
1802
1803         }
1804         sdev->sdev_state = state;
1805         return 0;
1806
1807  illegal:
1808         SCSI_LOG_ERROR_RECOVERY(1, 
1809                                 dev_printk(KERN_ERR, &sdev->sdev_gendev,
1810                                            "Illegal state transition %s->%s\n",
1811                                            scsi_device_state_name(oldstate),
1812                                            scsi_device_state_name(state))
1813                                 );
1814         return -EINVAL;
1815 }
1816 EXPORT_SYMBOL(scsi_device_set_state);
1817
1818 /**
1819  *      scsi_device_quiesce - Block user issued commands.
1820  *      @sdev:  scsi device to quiesce.
1821  *
1822  *      This works by trying to transition to the SDEV_QUIESCE state
1823  *      (which must be a legal transition).  When the device is in this
1824  *      state, only special requests will be accepted, all others will
1825  *      be deferred.  Since special requests may also be requeued requests,
1826  *      a successful return doesn't guarantee the device will be 
1827  *      totally quiescent.
1828  *
1829  *      Must be called with user context, may sleep.
1830  *
1831  *      Returns zero if unsuccessful or an error if not.
1832  **/
1833 int
1834 scsi_device_quiesce(struct scsi_device *sdev)
1835 {
1836         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
1837         if (err)
1838                 return err;
1839
1840         scsi_run_queue(sdev->request_queue);
1841         while (sdev->device_busy) {
1842                 msleep_interruptible(200);
1843                 scsi_run_queue(sdev->request_queue);
1844         }
1845         return 0;
1846 }
1847 EXPORT_SYMBOL(scsi_device_quiesce);
1848
1849 /**
1850  *      scsi_device_resume - Restart user issued commands to a quiesced device.
1851  *      @sdev:  scsi device to resume.
1852  *
1853  *      Moves the device from quiesced back to running and restarts the
1854  *      queues.
1855  *
1856  *      Must be called with user context, may sleep.
1857  **/
1858 void
1859 scsi_device_resume(struct scsi_device *sdev)
1860 {
1861         if(scsi_device_set_state(sdev, SDEV_RUNNING))
1862                 return;
1863         scsi_run_queue(sdev->request_queue);
1864 }
1865 EXPORT_SYMBOL(scsi_device_resume);
1866
1867 static void
1868 device_quiesce_fn(struct scsi_device *sdev, void *data)
1869 {
1870         scsi_device_quiesce(sdev);
1871 }
1872
1873 void
1874 scsi_target_quiesce(struct scsi_target *starget)
1875 {
1876         starget_for_each_device(starget, NULL, device_quiesce_fn);
1877 }
1878 EXPORT_SYMBOL(scsi_target_quiesce);
1879
1880 static void
1881 device_resume_fn(struct scsi_device *sdev, void *data)
1882 {
1883         scsi_device_resume(sdev);
1884 }
1885
1886 void
1887 scsi_target_resume(struct scsi_target *starget)
1888 {
1889         starget_for_each_device(starget, NULL, device_resume_fn);
1890 }
1891 EXPORT_SYMBOL(scsi_target_resume);
1892
1893 /**
1894  * scsi_internal_device_block - internal function to put a device
1895  *                              temporarily into the SDEV_BLOCK state
1896  * @sdev:       device to block
1897  *
1898  * Block request made by scsi lld's to temporarily stop all
1899  * scsi commands on the specified device.  Called from interrupt
1900  * or normal process context.
1901  *
1902  * Returns zero if successful or error if not
1903  *
1904  * Notes:       
1905  *      This routine transitions the device to the SDEV_BLOCK state
1906  *      (which must be a legal transition).  When the device is in this
1907  *      state, all commands are deferred until the scsi lld reenables
1908  *      the device with scsi_device_unblock or device_block_tmo fires.
1909  *      This routine assumes the host_lock is held on entry.
1910  **/
1911 int
1912 scsi_internal_device_block(struct scsi_device *sdev)
1913 {
1914         request_queue_t *q = sdev->request_queue;
1915         unsigned long flags;
1916         int err = 0;
1917
1918         err = scsi_device_set_state(sdev, SDEV_BLOCK);
1919         if (err)
1920                 return err;
1921
1922         /* 
1923          * The device has transitioned to SDEV_BLOCK.  Stop the
1924          * block layer from calling the midlayer with this device's
1925          * request queue. 
1926          */
1927         spin_lock_irqsave(q->queue_lock, flags);
1928         blk_stop_queue(q);
1929         spin_unlock_irqrestore(q->queue_lock, flags);
1930
1931         return 0;
1932 }
1933 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
1934  
1935 /**
1936  * scsi_internal_device_unblock - resume a device after a block request
1937  * @sdev:       device to resume
1938  *
1939  * Called by scsi lld's or the midlayer to restart the device queue
1940  * for the previously suspended scsi device.  Called from interrupt or
1941  * normal process context.
1942  *
1943  * Returns zero if successful or error if not.
1944  *
1945  * Notes:       
1946  *      This routine transitions the device to the SDEV_RUNNING state
1947  *      (which must be a legal transition) allowing the midlayer to
1948  *      goose the queue for this device.  This routine assumes the 
1949  *      host_lock is held upon entry.
1950  **/
1951 int
1952 scsi_internal_device_unblock(struct scsi_device *sdev)
1953 {
1954         request_queue_t *q = sdev->request_queue; 
1955         int err;
1956         unsigned long flags;
1957         
1958         /* 
1959          * Try to transition the scsi device to SDEV_RUNNING
1960          * and goose the device queue if successful.  
1961          */
1962         err = scsi_device_set_state(sdev, SDEV_RUNNING);
1963         if (err)
1964                 return err;
1965
1966         spin_lock_irqsave(q->queue_lock, flags);
1967         blk_start_queue(q);
1968         spin_unlock_irqrestore(q->queue_lock, flags);
1969
1970         return 0;
1971 }
1972 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
1973
1974 static void
1975 device_block(struct scsi_device *sdev, void *data)
1976 {
1977         scsi_internal_device_block(sdev);
1978 }
1979
1980 static int
1981 target_block(struct device *dev, void *data)
1982 {
1983         if (scsi_is_target_device(dev))
1984                 starget_for_each_device(to_scsi_target(dev), NULL,
1985                                         device_block);
1986         return 0;
1987 }
1988
1989 void
1990 scsi_target_block(struct device *dev)
1991 {
1992         if (scsi_is_target_device(dev))
1993                 starget_for_each_device(to_scsi_target(dev), NULL,
1994                                         device_block);
1995         else
1996                 device_for_each_child(dev, NULL, target_block);
1997 }
1998 EXPORT_SYMBOL_GPL(scsi_target_block);
1999
2000 static void
2001 device_unblock(struct scsi_device *sdev, void *data)
2002 {
2003         scsi_internal_device_unblock(sdev);
2004 }
2005
2006 static int
2007 target_unblock(struct device *dev, void *data)
2008 {
2009         if (scsi_is_target_device(dev))
2010                 starget_for_each_device(to_scsi_target(dev), NULL,
2011                                         device_unblock);
2012         return 0;
2013 }
2014
2015 void
2016 scsi_target_unblock(struct device *dev)
2017 {
2018         if (scsi_is_target_device(dev))
2019                 starget_for_each_device(to_scsi_target(dev), NULL,
2020                                         device_unblock);
2021         else
2022                 device_for_each_child(dev, NULL, target_unblock);
2023 }
2024 EXPORT_SYMBOL_GPL(scsi_target_unblock);