Merge master.kernel.org:/pub/scm/linux/kernel/git/acme/net-2.6
[linux-2.6] / drivers / scsi / scsi_lib.c
1 /*
2  *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
3  *
4  *  SCSI queueing library.
5  *      Initial versions: Eric Youngdale (eric@andante.org).
6  *                        Based upon conversations with large numbers
7  *                        of people at Linux Expo.
8  */
9
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/completion.h>
13 #include <linux/kernel.h>
14 #include <linux/mempool.h>
15 #include <linux/slab.h>
16 #include <linux/init.h>
17 #include <linux/pci.h>
18 #include <linux/delay.h>
19
20 #include <scsi/scsi.h>
21 #include <scsi/scsi_dbg.h>
22 #include <scsi/scsi_device.h>
23 #include <scsi/scsi_driver.h>
24 #include <scsi/scsi_eh.h>
25 #include <scsi/scsi_host.h>
26 #include <scsi/scsi_request.h>
27
28 #include "scsi_priv.h"
29 #include "scsi_logging.h"
30
31
32 #define SG_MEMPOOL_NR           (sizeof(scsi_sg_pools)/sizeof(struct scsi_host_sg_pool))
33 #define SG_MEMPOOL_SIZE         32
34
35 struct scsi_host_sg_pool {
36         size_t          size;
37         char            *name; 
38         kmem_cache_t    *slab;
39         mempool_t       *pool;
40 };
41
42 #if (SCSI_MAX_PHYS_SEGMENTS < 32)
43 #error SCSI_MAX_PHYS_SEGMENTS is too small
44 #endif
45
46 #define SP(x) { x, "sgpool-" #x } 
47 static struct scsi_host_sg_pool scsi_sg_pools[] = {
48         SP(8),
49         SP(16),
50         SP(32),
51 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
52         SP(64),
53 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
54         SP(128),
55 #if (SCSI_MAX_PHYS_SEGMENTS > 128)
56         SP(256),
57 #if (SCSI_MAX_PHYS_SEGMENTS > 256)
58 #error SCSI_MAX_PHYS_SEGMENTS is too large
59 #endif
60 #endif
61 #endif
62 #endif
63 };      
64 #undef SP
65
66
67 /*
68  * Function:    scsi_insert_special_req()
69  *
70  * Purpose:     Insert pre-formed request into request queue.
71  *
72  * Arguments:   sreq    - request that is ready to be queued.
73  *              at_head - boolean.  True if we should insert at head
74  *                        of queue, false if we should insert at tail.
75  *
76  * Lock status: Assumed that lock is not held upon entry.
77  *
78  * Returns:     Nothing
79  *
80  * Notes:       This function is called from character device and from
81  *              ioctl types of functions where the caller knows exactly
82  *              what SCSI command needs to be issued.   The idea is that
83  *              we merely inject the command into the queue (at the head
84  *              for now), and then call the queue request function to actually
85  *              process it.
86  */
87 int scsi_insert_special_req(struct scsi_request *sreq, int at_head)
88 {
89         /*
90          * Because users of this function are apt to reuse requests with no
91          * modification, we have to sanitise the request flags here
92          */
93         sreq->sr_request->flags &= ~REQ_DONTPREP;
94         blk_insert_request(sreq->sr_device->request_queue, sreq->sr_request,
95                            at_head, sreq);
96         return 0;
97 }
98
99 static void scsi_run_queue(struct request_queue *q);
100 static void scsi_release_buffers(struct scsi_cmnd *cmd);
101
102 /*
103  * Function:    scsi_unprep_request()
104  *
105  * Purpose:     Remove all preparation done for a request, including its
106  *              associated scsi_cmnd, so that it can be requeued.
107  *
108  * Arguments:   req     - request to unprepare
109  *
110  * Lock status: Assumed that no locks are held upon entry.
111  *
112  * Returns:     Nothing.
113  */
114 static void scsi_unprep_request(struct request *req)
115 {
116         struct scsi_cmnd *cmd = req->special;
117
118         req->flags &= ~REQ_DONTPREP;
119         req->special = (req->flags & REQ_SPECIAL) ? cmd->sc_request : NULL;
120
121         scsi_release_buffers(cmd);
122         scsi_put_command(cmd);
123 }
124
125 /*
126  * Function:    scsi_queue_insert()
127  *
128  * Purpose:     Insert a command in the midlevel queue.
129  *
130  * Arguments:   cmd    - command that we are adding to queue.
131  *              reason - why we are inserting command to queue.
132  *
133  * Lock status: Assumed that lock is not held upon entry.
134  *
135  * Returns:     Nothing.
136  *
137  * Notes:       We do this for one of two cases.  Either the host is busy
138  *              and it cannot accept any more commands for the time being,
139  *              or the device returned QUEUE_FULL and can accept no more
140  *              commands.
141  * Notes:       This could be called either from an interrupt context or a
142  *              normal process context.
143  * Notes:       Upon return, cmd is a stale pointer.
144  */
145 int scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
146 {
147         struct Scsi_Host *host = cmd->device->host;
148         struct scsi_device *device = cmd->device;
149         struct request_queue *q = device->request_queue;
150         struct request *req = cmd->request;
151         unsigned long flags;
152
153         SCSI_LOG_MLQUEUE(1,
154                  printk("Inserting command %p into mlqueue\n", cmd));
155
156         /*
157          * Set the appropriate busy bit for the device/host.
158          *
159          * If the host/device isn't busy, assume that something actually
160          * completed, and that we should be able to queue a command now.
161          *
162          * Note that the prior mid-layer assumption that any host could
163          * always queue at least one command is now broken.  The mid-layer
164          * will implement a user specifiable stall (see
165          * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
166          * if a command is requeued with no other commands outstanding
167          * either for the device or for the host.
168          */
169         if (reason == SCSI_MLQUEUE_HOST_BUSY)
170                 host->host_blocked = host->max_host_blocked;
171         else if (reason == SCSI_MLQUEUE_DEVICE_BUSY)
172                 device->device_blocked = device->max_device_blocked;
173
174         /*
175          * Decrement the counters, since these commands are no longer
176          * active on the host/device.
177          */
178         scsi_device_unbusy(device);
179
180         /*
181          * Requeue this command.  It will go before all other commands
182          * that are already in the queue.
183          *
184          * NOTE: there is magic here about the way the queue is plugged if
185          * we have no outstanding commands.
186          * 
187          * Although we *don't* plug the queue, we call the request
188          * function.  The SCSI request function detects the blocked condition
189          * and plugs the queue appropriately.
190          */
191         scsi_unprep_request(req);
192         spin_lock_irqsave(q->queue_lock, flags);
193         blk_requeue_request(q, req);
194         spin_unlock_irqrestore(q->queue_lock, flags);
195
196         scsi_run_queue(q);
197
198         return 0;
199 }
200
201 /*
202  * Function:    scsi_do_req
203  *
204  * Purpose:     Queue a SCSI request
205  *
206  * Arguments:   sreq      - command descriptor.
207  *              cmnd      - actual SCSI command to be performed.
208  *              buffer    - data buffer.
209  *              bufflen   - size of data buffer.
210  *              done      - completion function to be run.
211  *              timeout   - how long to let it run before timeout.
212  *              retries   - number of retries we allow.
213  *
214  * Lock status: No locks held upon entry.
215  *
216  * Returns:     Nothing.
217  *
218  * Notes:       This function is only used for queueing requests for things
219  *              like ioctls and character device requests - this is because
220  *              we essentially just inject a request into the queue for the
221  *              device.
222  *
223  *              In order to support the scsi_device_quiesce function, we
224  *              now inject requests on the *head* of the device queue
225  *              rather than the tail.
226  */
227 void scsi_do_req(struct scsi_request *sreq, const void *cmnd,
228                  void *buffer, unsigned bufflen,
229                  void (*done)(struct scsi_cmnd *),
230                  int timeout, int retries)
231 {
232         /*
233          * If the upper level driver is reusing these things, then
234          * we should release the low-level block now.  Another one will
235          * be allocated later when this request is getting queued.
236          */
237         __scsi_release_request(sreq);
238
239         /*
240          * Our own function scsi_done (which marks the host as not busy,
241          * disables the timeout counter, etc) will be called by us or by the
242          * scsi_hosts[host].queuecommand() function needs to also call
243          * the completion function for the high level driver.
244          */
245         memcpy(sreq->sr_cmnd, cmnd, sizeof(sreq->sr_cmnd));
246         sreq->sr_bufflen = bufflen;
247         sreq->sr_buffer = buffer;
248         sreq->sr_allowed = retries;
249         sreq->sr_done = done;
250         sreq->sr_timeout_per_command = timeout;
251
252         if (sreq->sr_cmd_len == 0)
253                 sreq->sr_cmd_len = COMMAND_SIZE(sreq->sr_cmnd[0]);
254
255         /*
256          * head injection *required* here otherwise quiesce won't work
257          */
258         scsi_insert_special_req(sreq, 1);
259 }
260 EXPORT_SYMBOL(scsi_do_req);
261
262 /* This is the end routine we get to if a command was never attached
263  * to the request.  Simply complete the request without changing
264  * rq_status; this will cause a DRIVER_ERROR. */
265 static void scsi_wait_req_end_io(struct request *req)
266 {
267         BUG_ON(!req->waiting);
268
269         complete(req->waiting);
270 }
271
272 void scsi_wait_req(struct scsi_request *sreq, const void *cmnd, void *buffer,
273                    unsigned bufflen, int timeout, int retries)
274 {
275         DECLARE_COMPLETION(wait);
276         int write = (sreq->sr_data_direction == DMA_TO_DEVICE);
277         struct request *req;
278
279         req = blk_get_request(sreq->sr_device->request_queue, write,
280                               __GFP_WAIT);
281         if (bufflen && blk_rq_map_kern(sreq->sr_device->request_queue, req,
282                                        buffer, bufflen, __GFP_WAIT)) {
283                 sreq->sr_result = DRIVER_ERROR << 24;
284                 blk_put_request(req);
285                 return;
286         }
287
288         req->flags |= REQ_NOMERGE;
289         req->waiting = &wait;
290         req->end_io = scsi_wait_req_end_io;
291         req->cmd_len = COMMAND_SIZE(((u8 *)cmnd)[0]);
292         req->sense = sreq->sr_sense_buffer;
293         req->sense_len = 0;
294         memcpy(req->cmd, cmnd, req->cmd_len);
295         req->timeout = timeout;
296         req->flags |= REQ_BLOCK_PC;
297         req->rq_disk = NULL;
298         blk_insert_request(sreq->sr_device->request_queue, req,
299                            sreq->sr_data_direction == DMA_TO_DEVICE, NULL);
300         wait_for_completion(&wait);
301         sreq->sr_request->waiting = NULL;
302         sreq->sr_result = req->errors;
303         if (req->errors)
304                 sreq->sr_result |= (DRIVER_ERROR << 24);
305
306         blk_put_request(req);
307 }
308
309 EXPORT_SYMBOL(scsi_wait_req);
310
311 /**
312  * scsi_execute - insert request and wait for the result
313  * @sdev:       scsi device
314  * @cmd:        scsi command
315  * @data_direction: data direction
316  * @buffer:     data buffer
317  * @bufflen:    len of buffer
318  * @sense:      optional sense buffer
319  * @timeout:    request timeout in seconds
320  * @retries:    number of times to retry request
321  * @flags:      or into request flags;
322  *
323  * returns the req->errors value which is the the scsi_cmnd result
324  * field.
325  **/
326 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
327                  int data_direction, void *buffer, unsigned bufflen,
328                  unsigned char *sense, int timeout, int retries, int flags)
329 {
330         struct request *req;
331         int write = (data_direction == DMA_TO_DEVICE);
332         int ret = DRIVER_ERROR << 24;
333
334         req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
335
336         if (bufflen &&  blk_rq_map_kern(sdev->request_queue, req,
337                                         buffer, bufflen, __GFP_WAIT))
338                 goto out;
339
340         req->cmd_len = COMMAND_SIZE(cmd[0]);
341         memcpy(req->cmd, cmd, req->cmd_len);
342         req->sense = sense;
343         req->sense_len = 0;
344         req->timeout = timeout;
345         req->flags |= flags | REQ_BLOCK_PC | REQ_SPECIAL | REQ_QUIET;
346
347         /*
348          * head injection *required* here otherwise quiesce won't work
349          */
350         blk_execute_rq(req->q, NULL, req, 1);
351
352         ret = req->errors;
353  out:
354         blk_put_request(req);
355
356         return ret;
357 }
358 EXPORT_SYMBOL(scsi_execute);
359
360
361 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
362                      int data_direction, void *buffer, unsigned bufflen,
363                      struct scsi_sense_hdr *sshdr, int timeout, int retries)
364 {
365         char *sense = NULL;
366         int result;
367         
368         if (sshdr) {
369                 sense = kmalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
370                 if (!sense)
371                         return DRIVER_ERROR << 24;
372                 memset(sense, 0, SCSI_SENSE_BUFFERSIZE);
373         }
374         result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
375                                   sense, timeout, retries, 0);
376         if (sshdr)
377                 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
378
379         kfree(sense);
380         return result;
381 }
382 EXPORT_SYMBOL(scsi_execute_req);
383
384 /*
385  * Function:    scsi_init_cmd_errh()
386  *
387  * Purpose:     Initialize cmd fields related to error handling.
388  *
389  * Arguments:   cmd     - command that is ready to be queued.
390  *
391  * Returns:     Nothing
392  *
393  * Notes:       This function has the job of initializing a number of
394  *              fields related to error handling.   Typically this will
395  *              be called once for each command, as required.
396  */
397 static int scsi_init_cmd_errh(struct scsi_cmnd *cmd)
398 {
399         cmd->serial_number = 0;
400
401         memset(cmd->sense_buffer, 0, sizeof cmd->sense_buffer);
402
403         if (cmd->cmd_len == 0)
404                 cmd->cmd_len = COMMAND_SIZE(cmd->cmnd[0]);
405
406         /*
407          * We need saved copies of a number of fields - this is because
408          * error handling may need to overwrite these with different values
409          * to run different commands, and once error handling is complete,
410          * we will need to restore these values prior to running the actual
411          * command.
412          */
413         cmd->old_use_sg = cmd->use_sg;
414         cmd->old_cmd_len = cmd->cmd_len;
415         cmd->sc_old_data_direction = cmd->sc_data_direction;
416         cmd->old_underflow = cmd->underflow;
417         memcpy(cmd->data_cmnd, cmd->cmnd, sizeof(cmd->cmnd));
418         cmd->buffer = cmd->request_buffer;
419         cmd->bufflen = cmd->request_bufflen;
420
421         return 1;
422 }
423
424 /*
425  * Function:   scsi_setup_cmd_retry()
426  *
427  * Purpose:    Restore the command state for a retry
428  *
429  * Arguments:  cmd      - command to be restored
430  *
431  * Returns:    Nothing
432  *
433  * Notes:      Immediately prior to retrying a command, we need
434  *             to restore certain fields that we saved above.
435  */
436 void scsi_setup_cmd_retry(struct scsi_cmnd *cmd)
437 {
438         memcpy(cmd->cmnd, cmd->data_cmnd, sizeof(cmd->data_cmnd));
439         cmd->request_buffer = cmd->buffer;
440         cmd->request_bufflen = cmd->bufflen;
441         cmd->use_sg = cmd->old_use_sg;
442         cmd->cmd_len = cmd->old_cmd_len;
443         cmd->sc_data_direction = cmd->sc_old_data_direction;
444         cmd->underflow = cmd->old_underflow;
445 }
446
447 void scsi_device_unbusy(struct scsi_device *sdev)
448 {
449         struct Scsi_Host *shost = sdev->host;
450         unsigned long flags;
451
452         spin_lock_irqsave(shost->host_lock, flags);
453         shost->host_busy--;
454         if (unlikely((shost->shost_state == SHOST_RECOVERY) &&
455                      shost->host_failed))
456                 scsi_eh_wakeup(shost);
457         spin_unlock(shost->host_lock);
458         spin_lock(sdev->request_queue->queue_lock);
459         sdev->device_busy--;
460         spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
461 }
462
463 /*
464  * Called for single_lun devices on IO completion. Clear starget_sdev_user,
465  * and call blk_run_queue for all the scsi_devices on the target -
466  * including current_sdev first.
467  *
468  * Called with *no* scsi locks held.
469  */
470 static void scsi_single_lun_run(struct scsi_device *current_sdev)
471 {
472         struct Scsi_Host *shost = current_sdev->host;
473         struct scsi_device *sdev, *tmp;
474         struct scsi_target *starget = scsi_target(current_sdev);
475         unsigned long flags;
476
477         spin_lock_irqsave(shost->host_lock, flags);
478         starget->starget_sdev_user = NULL;
479         spin_unlock_irqrestore(shost->host_lock, flags);
480
481         /*
482          * Call blk_run_queue for all LUNs on the target, starting with
483          * current_sdev. We race with others (to set starget_sdev_user),
484          * but in most cases, we will be first. Ideally, each LU on the
485          * target would get some limited time or requests on the target.
486          */
487         blk_run_queue(current_sdev->request_queue);
488
489         spin_lock_irqsave(shost->host_lock, flags);
490         if (starget->starget_sdev_user)
491                 goto out;
492         list_for_each_entry_safe(sdev, tmp, &starget->devices,
493                         same_target_siblings) {
494                 if (sdev == current_sdev)
495                         continue;
496                 if (scsi_device_get(sdev))
497                         continue;
498
499                 spin_unlock_irqrestore(shost->host_lock, flags);
500                 blk_run_queue(sdev->request_queue);
501                 spin_lock_irqsave(shost->host_lock, flags);
502         
503                 scsi_device_put(sdev);
504         }
505  out:
506         spin_unlock_irqrestore(shost->host_lock, flags);
507 }
508
509 /*
510  * Function:    scsi_run_queue()
511  *
512  * Purpose:     Select a proper request queue to serve next
513  *
514  * Arguments:   q       - last request's queue
515  *
516  * Returns:     Nothing
517  *
518  * Notes:       The previous command was completely finished, start
519  *              a new one if possible.
520  */
521 static void scsi_run_queue(struct request_queue *q)
522 {
523         struct scsi_device *sdev = q->queuedata;
524         struct Scsi_Host *shost = sdev->host;
525         unsigned long flags;
526
527         if (sdev->single_lun)
528                 scsi_single_lun_run(sdev);
529
530         spin_lock_irqsave(shost->host_lock, flags);
531         while (!list_empty(&shost->starved_list) &&
532                !shost->host_blocked && !shost->host_self_blocked &&
533                 !((shost->can_queue > 0) &&
534                   (shost->host_busy >= shost->can_queue))) {
535                 /*
536                  * As long as shost is accepting commands and we have
537                  * starved queues, call blk_run_queue. scsi_request_fn
538                  * drops the queue_lock and can add us back to the
539                  * starved_list.
540                  *
541                  * host_lock protects the starved_list and starved_entry.
542                  * scsi_request_fn must get the host_lock before checking
543                  * or modifying starved_list or starved_entry.
544                  */
545                 sdev = list_entry(shost->starved_list.next,
546                                           struct scsi_device, starved_entry);
547                 list_del_init(&sdev->starved_entry);
548                 spin_unlock_irqrestore(shost->host_lock, flags);
549
550                 blk_run_queue(sdev->request_queue);
551
552                 spin_lock_irqsave(shost->host_lock, flags);
553                 if (unlikely(!list_empty(&sdev->starved_entry)))
554                         /*
555                          * sdev lost a race, and was put back on the
556                          * starved list. This is unlikely but without this
557                          * in theory we could loop forever.
558                          */
559                         break;
560         }
561         spin_unlock_irqrestore(shost->host_lock, flags);
562
563         blk_run_queue(q);
564 }
565
566 /*
567  * Function:    scsi_requeue_command()
568  *
569  * Purpose:     Handle post-processing of completed commands.
570  *
571  * Arguments:   q       - queue to operate on
572  *              cmd     - command that may need to be requeued.
573  *
574  * Returns:     Nothing
575  *
576  * Notes:       After command completion, there may be blocks left
577  *              over which weren't finished by the previous command
578  *              this can be for a number of reasons - the main one is
579  *              I/O errors in the middle of the request, in which case
580  *              we need to request the blocks that come after the bad
581  *              sector.
582  * Notes:       Upon return, cmd is a stale pointer.
583  */
584 static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
585 {
586         struct request *req = cmd->request;
587         unsigned long flags;
588
589         scsi_unprep_request(req);
590         spin_lock_irqsave(q->queue_lock, flags);
591         blk_requeue_request(q, req);
592         spin_unlock_irqrestore(q->queue_lock, flags);
593
594         scsi_run_queue(q);
595 }
596
597 void scsi_next_command(struct scsi_cmnd *cmd)
598 {
599         struct request_queue *q = cmd->device->request_queue;
600
601         scsi_put_command(cmd);
602         scsi_run_queue(q);
603 }
604
605 void scsi_run_host_queues(struct Scsi_Host *shost)
606 {
607         struct scsi_device *sdev;
608
609         shost_for_each_device(sdev, shost)
610                 scsi_run_queue(sdev->request_queue);
611 }
612
613 /*
614  * Function:    scsi_end_request()
615  *
616  * Purpose:     Post-processing of completed commands (usually invoked at end
617  *              of upper level post-processing and scsi_io_completion).
618  *
619  * Arguments:   cmd      - command that is complete.
620  *              uptodate - 1 if I/O indicates success, <= 0 for I/O error.
621  *              bytes    - number of bytes of completed I/O
622  *              requeue  - indicates whether we should requeue leftovers.
623  *
624  * Lock status: Assumed that lock is not held upon entry.
625  *
626  * Returns:     cmd if requeue required, NULL otherwise.
627  *
628  * Notes:       This is called for block device requests in order to
629  *              mark some number of sectors as complete.
630  * 
631  *              We are guaranteeing that the request queue will be goosed
632  *              at some point during this call.
633  * Notes:       If cmd was requeued, upon return it will be a stale pointer.
634  */
635 static struct scsi_cmnd *scsi_end_request(struct scsi_cmnd *cmd, int uptodate,
636                                           int bytes, int requeue)
637 {
638         request_queue_t *q = cmd->device->request_queue;
639         struct request *req = cmd->request;
640         unsigned long flags;
641
642         /*
643          * If there are blocks left over at the end, set up the command
644          * to queue the remainder of them.
645          */
646         if (end_that_request_chunk(req, uptodate, bytes)) {
647                 int leftover = (req->hard_nr_sectors << 9);
648
649                 if (blk_pc_request(req))
650                         leftover = req->data_len;
651
652                 /* kill remainder if no retrys */
653                 if (!uptodate && blk_noretry_request(req))
654                         end_that_request_chunk(req, 0, leftover);
655                 else {
656                         if (requeue) {
657                                 /*
658                                  * Bleah.  Leftovers again.  Stick the
659                                  * leftovers in the front of the
660                                  * queue, and goose the queue again.
661                                  */
662                                 scsi_requeue_command(q, cmd);
663                                 cmd = NULL;
664                         }
665                         return cmd;
666                 }
667         }
668
669         add_disk_randomness(req->rq_disk);
670
671         spin_lock_irqsave(q->queue_lock, flags);
672         if (blk_rq_tagged(req))
673                 blk_queue_end_tag(q, req);
674         end_that_request_last(req);
675         spin_unlock_irqrestore(q->queue_lock, flags);
676
677         /*
678          * This will goose the queue request function at the end, so we don't
679          * need to worry about launching another command.
680          */
681         scsi_next_command(cmd);
682         return NULL;
683 }
684
685 static struct scatterlist *scsi_alloc_sgtable(struct scsi_cmnd *cmd, int gfp_mask)
686 {
687         struct scsi_host_sg_pool *sgp;
688         struct scatterlist *sgl;
689
690         BUG_ON(!cmd->use_sg);
691
692         switch (cmd->use_sg) {
693         case 1 ... 8:
694                 cmd->sglist_len = 0;
695                 break;
696         case 9 ... 16:
697                 cmd->sglist_len = 1;
698                 break;
699         case 17 ... 32:
700                 cmd->sglist_len = 2;
701                 break;
702 #if (SCSI_MAX_PHYS_SEGMENTS > 32)
703         case 33 ... 64:
704                 cmd->sglist_len = 3;
705                 break;
706 #if (SCSI_MAX_PHYS_SEGMENTS > 64)
707         case 65 ... 128:
708                 cmd->sglist_len = 4;
709                 break;
710 #if (SCSI_MAX_PHYS_SEGMENTS  > 128)
711         case 129 ... 256:
712                 cmd->sglist_len = 5;
713                 break;
714 #endif
715 #endif
716 #endif
717         default:
718                 return NULL;
719         }
720
721         sgp = scsi_sg_pools + cmd->sglist_len;
722         sgl = mempool_alloc(sgp->pool, gfp_mask);
723         return sgl;
724 }
725
726 static void scsi_free_sgtable(struct scatterlist *sgl, int index)
727 {
728         struct scsi_host_sg_pool *sgp;
729
730         BUG_ON(index >= SG_MEMPOOL_NR);
731
732         sgp = scsi_sg_pools + index;
733         mempool_free(sgl, sgp->pool);
734 }
735
736 /*
737  * Function:    scsi_release_buffers()
738  *
739  * Purpose:     Completion processing for block device I/O requests.
740  *
741  * Arguments:   cmd     - command that we are bailing.
742  *
743  * Lock status: Assumed that no lock is held upon entry.
744  *
745  * Returns:     Nothing
746  *
747  * Notes:       In the event that an upper level driver rejects a
748  *              command, we must release resources allocated during
749  *              the __init_io() function.  Primarily this would involve
750  *              the scatter-gather table, and potentially any bounce
751  *              buffers.
752  */
753 static void scsi_release_buffers(struct scsi_cmnd *cmd)
754 {
755         struct request *req = cmd->request;
756
757         /*
758          * Free up any indirection buffers we allocated for DMA purposes. 
759          */
760         if (cmd->use_sg)
761                 scsi_free_sgtable(cmd->request_buffer, cmd->sglist_len);
762         else if (cmd->request_buffer != req->buffer)
763                 kfree(cmd->request_buffer);
764
765         /*
766          * Zero these out.  They now point to freed memory, and it is
767          * dangerous to hang onto the pointers.
768          */
769         cmd->buffer  = NULL;
770         cmd->bufflen = 0;
771         cmd->request_buffer = NULL;
772         cmd->request_bufflen = 0;
773 }
774
775 /*
776  * Function:    scsi_io_completion()
777  *
778  * Purpose:     Completion processing for block device I/O requests.
779  *
780  * Arguments:   cmd   - command that is finished.
781  *
782  * Lock status: Assumed that no lock is held upon entry.
783  *
784  * Returns:     Nothing
785  *
786  * Notes:       This function is matched in terms of capabilities to
787  *              the function that created the scatter-gather list.
788  *              In other words, if there are no bounce buffers
789  *              (the normal case for most drivers), we don't need
790  *              the logic to deal with cleaning up afterwards.
791  *
792  *              We must do one of several things here:
793  *
794  *              a) Call scsi_end_request.  This will finish off the
795  *                 specified number of sectors.  If we are done, the
796  *                 command block will be released, and the queue
797  *                 function will be goosed.  If we are not done, then
798  *                 scsi_end_request will directly goose the queue.
799  *
800  *              b) We can just use scsi_requeue_command() here.  This would
801  *                 be used if we just wanted to retry, for example.
802  */
803 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes,
804                         unsigned int block_bytes)
805 {
806         int result = cmd->result;
807         int this_count = cmd->bufflen;
808         request_queue_t *q = cmd->device->request_queue;
809         struct request *req = cmd->request;
810         int clear_errors = 1;
811         struct scsi_sense_hdr sshdr;
812         int sense_valid = 0;
813         int sense_deferred = 0;
814
815         if (blk_complete_barrier_rq(q, req, good_bytes >> 9))
816                 return;
817
818         /*
819          * Free up any indirection buffers we allocated for DMA purposes. 
820          * For the case of a READ, we need to copy the data out of the
821          * bounce buffer and into the real buffer.
822          */
823         if (cmd->use_sg)
824                 scsi_free_sgtable(cmd->buffer, cmd->sglist_len);
825         else if (cmd->buffer != req->buffer) {
826                 if (rq_data_dir(req) == READ) {
827                         unsigned long flags;
828                         char *to = bio_kmap_irq(req->bio, &flags);
829                         memcpy(to, cmd->buffer, cmd->bufflen);
830                         bio_kunmap_irq(to, &flags);
831                 }
832                 kfree(cmd->buffer);
833         }
834
835         if (result) {
836                 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
837                 if (sense_valid)
838                         sense_deferred = scsi_sense_is_deferred(&sshdr);
839         }
840         if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
841                 req->errors = result;
842                 if (result) {
843                         clear_errors = 0;
844                         if (sense_valid && req->sense) {
845                                 /*
846                                  * SG_IO wants current and deferred errors
847                                  */
848                                 int len = 8 + cmd->sense_buffer[7];
849
850                                 if (len > SCSI_SENSE_BUFFERSIZE)
851                                         len = SCSI_SENSE_BUFFERSIZE;
852                                 memcpy(req->sense, cmd->sense_buffer,  len);
853                                 req->sense_len = len;
854                         }
855                 } else
856                         req->data_len = cmd->resid;
857         }
858
859         /*
860          * Zero these out.  They now point to freed memory, and it is
861          * dangerous to hang onto the pointers.
862          */
863         cmd->buffer  = NULL;
864         cmd->bufflen = 0;
865         cmd->request_buffer = NULL;
866         cmd->request_bufflen = 0;
867
868         /*
869          * Next deal with any sectors which we were able to correctly
870          * handle.
871          */
872         if (good_bytes >= 0) {
873                 SCSI_LOG_HLCOMPLETE(1, printk("%ld sectors total, %d bytes done.\n",
874                                               req->nr_sectors, good_bytes));
875                 SCSI_LOG_HLCOMPLETE(1, printk("use_sg is %d\n", cmd->use_sg));
876
877                 if (clear_errors)
878                         req->errors = 0;
879                 /*
880                  * If multiple sectors are requested in one buffer, then
881                  * they will have been finished off by the first command.
882                  * If not, then we have a multi-buffer command.
883                  *
884                  * If block_bytes != 0, it means we had a medium error
885                  * of some sort, and that we want to mark some number of
886                  * sectors as not uptodate.  Thus we want to inhibit
887                  * requeueing right here - we will requeue down below
888                  * when we handle the bad sectors.
889                  */
890
891                 /*
892                  * If the command completed without error, then either
893                  * finish off the rest of the command, or start a new one.
894                  */
895                 if (scsi_end_request(cmd, 1, good_bytes, result == 0) == NULL)
896                         return;
897         }
898         /*
899          * Now, if we were good little boys and girls, Santa left us a request
900          * sense buffer.  We can extract information from this, so we
901          * can choose a block to remap, etc.
902          */
903         if (sense_valid && !sense_deferred) {
904                 switch (sshdr.sense_key) {
905                 case UNIT_ATTENTION:
906                         if (cmd->device->removable) {
907                                 /* detected disc change.  set a bit 
908                                  * and quietly refuse further access.
909                                  */
910                                 cmd->device->changed = 1;
911                                 scsi_end_request(cmd, 0,
912                                                 this_count, 1);
913                                 return;
914                         } else {
915                                 /*
916                                 * Must have been a power glitch, or a
917                                 * bus reset.  Could not have been a
918                                 * media change, so we just retry the
919                                 * request and see what happens.  
920                                 */
921                                 scsi_requeue_command(q, cmd);
922                                 return;
923                         }
924                         break;
925                 case ILLEGAL_REQUEST:
926                         /*
927                         * If we had an ILLEGAL REQUEST returned, then we may
928                         * have performed an unsupported command.  The only
929                         * thing this should be would be a ten byte read where
930                         * only a six byte read was supported.  Also, on a
931                         * system where READ CAPACITY failed, we may have read
932                         * past the end of the disk.
933                         */
934                         if (cmd->device->use_10_for_rw &&
935                             (cmd->cmnd[0] == READ_10 ||
936                              cmd->cmnd[0] == WRITE_10)) {
937                                 cmd->device->use_10_for_rw = 0;
938                                 /*
939                                  * This will cause a retry with a 6-byte
940                                  * command.
941                                  */
942                                 scsi_requeue_command(q, cmd);
943                                 result = 0;
944                         } else {
945                                 scsi_end_request(cmd, 0, this_count, 1);
946                                 return;
947                         }
948                         break;
949                 case NOT_READY:
950                         /*
951                          * If the device is in the process of becoming ready,
952                          * retry.
953                          */
954                         if (sshdr.asc == 0x04 && sshdr.ascq == 0x01) {
955                                 scsi_requeue_command(q, cmd);
956                                 return;
957                         }
958                         if (!(req->flags & REQ_QUIET))
959                                 dev_printk(KERN_INFO,
960                                            &cmd->device->sdev_gendev,
961                                            "Device not ready.\n");
962                         scsi_end_request(cmd, 0, this_count, 1);
963                         return;
964                 case VOLUME_OVERFLOW:
965                         if (!(req->flags & REQ_QUIET)) {
966                                 dev_printk(KERN_INFO,
967                                            &cmd->device->sdev_gendev,
968                                            "Volume overflow, CDB: ");
969                                 __scsi_print_command(cmd->data_cmnd);
970                                 scsi_print_sense("", cmd);
971                         }
972                         scsi_end_request(cmd, 0, block_bytes, 1);
973                         return;
974                 default:
975                         break;
976                 }
977         }                       /* driver byte != 0 */
978         if (host_byte(result) == DID_RESET) {
979                 /*
980                  * Third party bus reset or reset for error
981                  * recovery reasons.  Just retry the request
982                  * and see what happens.  
983                  */
984                 scsi_requeue_command(q, cmd);
985                 return;
986         }
987         if (result) {
988                 if (!(req->flags & REQ_QUIET)) {
989                         dev_printk(KERN_INFO, &cmd->device->sdev_gendev,
990                                    "SCSI error: return code = 0x%x\n", result);
991
992                         if (driver_byte(result) & DRIVER_SENSE)
993                                 scsi_print_sense("", cmd);
994                 }
995                 /*
996                  * Mark a single buffer as not uptodate.  Queue the remainder.
997                  * We sometimes get this cruft in the event that a medium error
998                  * isn't properly reported.
999                  */
1000                 block_bytes = req->hard_cur_sectors << 9;
1001                 if (!block_bytes)
1002                         block_bytes = req->data_len;
1003                 scsi_end_request(cmd, 0, block_bytes, 1);
1004         }
1005 }
1006 EXPORT_SYMBOL(scsi_io_completion);
1007
1008 /*
1009  * Function:    scsi_init_io()
1010  *
1011  * Purpose:     SCSI I/O initialize function.
1012  *
1013  * Arguments:   cmd   - Command descriptor we wish to initialize
1014  *
1015  * Returns:     0 on success
1016  *              BLKPREP_DEFER if the failure is retryable
1017  *              BLKPREP_KILL if the failure is fatal
1018  */
1019 static int scsi_init_io(struct scsi_cmnd *cmd)
1020 {
1021         struct request     *req = cmd->request;
1022         struct scatterlist *sgpnt;
1023         int                count;
1024
1025         /*
1026          * if this is a rq->data based REQ_BLOCK_PC, setup for a non-sg xfer
1027          */
1028         if ((req->flags & REQ_BLOCK_PC) && !req->bio) {
1029                 cmd->request_bufflen = req->data_len;
1030                 cmd->request_buffer = req->data;
1031                 req->buffer = req->data;
1032                 cmd->use_sg = 0;
1033                 return 0;
1034         }
1035
1036         /*
1037          * we used to not use scatter-gather for single segment request,
1038          * but now we do (it makes highmem I/O easier to support without
1039          * kmapping pages)
1040          */
1041         cmd->use_sg = req->nr_phys_segments;
1042
1043         /*
1044          * if sg table allocation fails, requeue request later.
1045          */
1046         sgpnt = scsi_alloc_sgtable(cmd, GFP_ATOMIC);
1047         if (unlikely(!sgpnt))
1048                 return BLKPREP_DEFER;
1049
1050         cmd->request_buffer = (char *) sgpnt;
1051         cmd->request_bufflen = req->nr_sectors << 9;
1052         if (blk_pc_request(req))
1053                 cmd->request_bufflen = req->data_len;
1054         req->buffer = NULL;
1055
1056         /* 
1057          * Next, walk the list, and fill in the addresses and sizes of
1058          * each segment.
1059          */
1060         count = blk_rq_map_sg(req->q, req, cmd->request_buffer);
1061
1062         /*
1063          * mapped well, send it off
1064          */
1065         if (likely(count <= cmd->use_sg)) {
1066                 cmd->use_sg = count;
1067                 return 0;
1068         }
1069
1070         printk(KERN_ERR "Incorrect number of segments after building list\n");
1071         printk(KERN_ERR "counted %d, received %d\n", count, cmd->use_sg);
1072         printk(KERN_ERR "req nr_sec %lu, cur_nr_sec %u\n", req->nr_sectors,
1073                         req->current_nr_sectors);
1074
1075         /* release the command and kill it */
1076         scsi_release_buffers(cmd);
1077         scsi_put_command(cmd);
1078         return BLKPREP_KILL;
1079 }
1080
1081 static int scsi_prepare_flush_fn(request_queue_t *q, struct request *rq)
1082 {
1083         struct scsi_device *sdev = q->queuedata;
1084         struct scsi_driver *drv;
1085
1086         if (sdev->sdev_state == SDEV_RUNNING) {
1087                 drv = *(struct scsi_driver **) rq->rq_disk->private_data;
1088
1089                 if (drv->prepare_flush)
1090                         return drv->prepare_flush(q, rq);
1091         }
1092
1093         return 0;
1094 }
1095
1096 static void scsi_end_flush_fn(request_queue_t *q, struct request *rq)
1097 {
1098         struct scsi_device *sdev = q->queuedata;
1099         struct request *flush_rq = rq->end_io_data;
1100         struct scsi_driver *drv;
1101
1102         if (flush_rq->errors) {
1103                 printk("scsi: barrier error, disabling flush support\n");
1104                 blk_queue_ordered(q, QUEUE_ORDERED_NONE);
1105         }
1106
1107         if (sdev->sdev_state == SDEV_RUNNING) {
1108                 drv = *(struct scsi_driver **) rq->rq_disk->private_data;
1109                 drv->end_flush(q, rq);
1110         }
1111 }
1112
1113 static int scsi_issue_flush_fn(request_queue_t *q, struct gendisk *disk,
1114                                sector_t *error_sector)
1115 {
1116         struct scsi_device *sdev = q->queuedata;
1117         struct scsi_driver *drv;
1118
1119         if (sdev->sdev_state != SDEV_RUNNING)
1120                 return -ENXIO;
1121
1122         drv = *(struct scsi_driver **) disk->private_data;
1123         if (drv->issue_flush)
1124                 return drv->issue_flush(&sdev->sdev_gendev, error_sector);
1125
1126         return -EOPNOTSUPP;
1127 }
1128
1129 static void scsi_generic_done(struct scsi_cmnd *cmd)
1130 {
1131         BUG_ON(!blk_pc_request(cmd->request));
1132         scsi_io_completion(cmd, cmd->result == 0 ? cmd->bufflen : 0, 0);
1133 }
1134
1135 static int scsi_prep_fn(struct request_queue *q, struct request *req)
1136 {
1137         struct scsi_device *sdev = q->queuedata;
1138         struct scsi_cmnd *cmd;
1139         int specials_only = 0;
1140
1141         /*
1142          * Just check to see if the device is online.  If it isn't, we
1143          * refuse to process any commands.  The device must be brought
1144          * online before trying any recovery commands
1145          */
1146         if (unlikely(!scsi_device_online(sdev))) {
1147                 printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n",
1148                        sdev->host->host_no, sdev->id, sdev->lun);
1149                 goto kill;
1150         }
1151         if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1152                 /* OK, we're not in a running state don't prep
1153                  * user commands */
1154                 if (sdev->sdev_state == SDEV_DEL) {
1155                         /* Device is fully deleted, no commands
1156                          * at all allowed down */
1157                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to dead device\n",
1158                                sdev->host->host_no, sdev->id, sdev->lun);
1159                         goto kill;
1160                 }
1161                 /* OK, we only allow special commands (i.e. not
1162                  * user initiated ones */
1163                 specials_only = sdev->sdev_state;
1164         }
1165
1166         /*
1167          * Find the actual device driver associated with this command.
1168          * The SPECIAL requests are things like character device or
1169          * ioctls, which did not originate from ll_rw_blk.  Note that
1170          * the special field is also used to indicate the cmd for
1171          * the remainder of a partially fulfilled request that can 
1172          * come up when there is a medium error.  We have to treat
1173          * these two cases differently.  We differentiate by looking
1174          * at request->cmd, as this tells us the real story.
1175          */
1176         if (req->flags & REQ_SPECIAL && req->special) {
1177                 struct scsi_request *sreq = req->special;
1178
1179                 if (sreq->sr_magic == SCSI_REQ_MAGIC) {
1180                         cmd = scsi_get_command(sreq->sr_device, GFP_ATOMIC);
1181                         if (unlikely(!cmd))
1182                                 goto defer;
1183                         scsi_init_cmd_from_req(cmd, sreq);
1184                 } else
1185                         cmd = req->special;
1186         } else if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1187
1188                 if(unlikely(specials_only) && !(req->flags & REQ_SPECIAL)) {
1189                         if(specials_only == SDEV_QUIESCE ||
1190                                         specials_only == SDEV_BLOCK)
1191                                 goto defer;
1192                         
1193                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to device being removed\n",
1194                                sdev->host->host_no, sdev->id, sdev->lun);
1195                         goto kill;
1196                 }
1197                         
1198                         
1199                 /*
1200                  * Now try and find a command block that we can use.
1201                  */
1202                 if (!req->special) {
1203                         cmd = scsi_get_command(sdev, GFP_ATOMIC);
1204                         if (unlikely(!cmd))
1205                                 goto defer;
1206                 } else
1207                         cmd = req->special;
1208                 
1209                 /* pull a tag out of the request if we have one */
1210                 cmd->tag = req->tag;
1211         } else {
1212                 blk_dump_rq_flags(req, "SCSI bad req");
1213                 goto kill;
1214         }
1215         
1216         /* note the overloading of req->special.  When the tag
1217          * is active it always means cmd.  If the tag goes
1218          * back for re-queueing, it may be reset */
1219         req->special = cmd;
1220         cmd->request = req;
1221         
1222         /*
1223          * FIXME: drop the lock here because the functions below
1224          * expect to be called without the queue lock held.  Also,
1225          * previously, we dequeued the request before dropping the
1226          * lock.  We hope REQ_STARTED prevents anything untoward from
1227          * happening now.
1228          */
1229         if (req->flags & (REQ_CMD | REQ_BLOCK_PC)) {
1230                 struct scsi_driver *drv;
1231                 int ret;
1232
1233                 /*
1234                  * This will do a couple of things:
1235                  *  1) Fill in the actual SCSI command.
1236                  *  2) Fill in any other upper-level specific fields
1237                  * (timeout).
1238                  *
1239                  * If this returns 0, it means that the request failed
1240                  * (reading past end of disk, reading offline device,
1241                  * etc).   This won't actually talk to the device, but
1242                  * some kinds of consistency checking may cause the     
1243                  * request to be rejected immediately.
1244                  */
1245
1246                 /* 
1247                  * This sets up the scatter-gather table (allocating if
1248                  * required).
1249                  */
1250                 ret = scsi_init_io(cmd);
1251                 switch(ret) {
1252                 case BLKPREP_KILL:
1253                         /* BLKPREP_KILL return also releases the command */
1254                         goto kill;
1255                 case BLKPREP_DEFER:
1256                         goto defer;
1257                 }
1258                 
1259                 /*
1260                  * Initialize the actual SCSI command for this request.
1261                  */
1262                 if (req->rq_disk) {
1263                         drv = *(struct scsi_driver **)req->rq_disk->private_data;
1264                         if (unlikely(!drv->init_command(cmd))) {
1265                                 scsi_release_buffers(cmd);
1266                                 scsi_put_command(cmd);
1267                                 goto kill;
1268                         }
1269                 } else {
1270                         memcpy(cmd->cmnd, req->cmd, sizeof(cmd->cmnd));
1271                         if (rq_data_dir(req) == WRITE)
1272                                 cmd->sc_data_direction = DMA_TO_DEVICE;
1273                         else if (req->data_len)
1274                                 cmd->sc_data_direction = DMA_FROM_DEVICE;
1275                         else
1276                                 cmd->sc_data_direction = DMA_NONE;
1277                         
1278                         cmd->transfersize = req->data_len;
1279                         cmd->allowed = 3;
1280                         cmd->timeout_per_command = req->timeout;
1281                         cmd->done = scsi_generic_done;
1282                 }
1283         }
1284
1285         /*
1286          * The request is now prepped, no need to come back here
1287          */
1288         req->flags |= REQ_DONTPREP;
1289         return BLKPREP_OK;
1290
1291  defer:
1292         /* If we defer, the elv_next_request() returns NULL, but the
1293          * queue must be restarted, so we plug here if no returning
1294          * command will automatically do that. */
1295         if (sdev->device_busy == 0)
1296                 blk_plug_device(q);
1297         return BLKPREP_DEFER;
1298  kill:
1299         req->errors = DID_NO_CONNECT << 16;
1300         return BLKPREP_KILL;
1301 }
1302
1303 /*
1304  * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1305  * return 0.
1306  *
1307  * Called with the queue_lock held.
1308  */
1309 static inline int scsi_dev_queue_ready(struct request_queue *q,
1310                                   struct scsi_device *sdev)
1311 {
1312         if (sdev->device_busy >= sdev->queue_depth)
1313                 return 0;
1314         if (sdev->device_busy == 0 && sdev->device_blocked) {
1315                 /*
1316                  * unblock after device_blocked iterates to zero
1317                  */
1318                 if (--sdev->device_blocked == 0) {
1319                         SCSI_LOG_MLQUEUE(3,
1320                                 printk("scsi%d (%d:%d) unblocking device at"
1321                                        " zero depth\n", sdev->host->host_no,
1322                                        sdev->id, sdev->lun));
1323                 } else {
1324                         blk_plug_device(q);
1325                         return 0;
1326                 }
1327         }
1328         if (sdev->device_blocked)
1329                 return 0;
1330
1331         return 1;
1332 }
1333
1334 /*
1335  * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1336  * return 0. We must end up running the queue again whenever 0 is
1337  * returned, else IO can hang.
1338  *
1339  * Called with host_lock held.
1340  */
1341 static inline int scsi_host_queue_ready(struct request_queue *q,
1342                                    struct Scsi_Host *shost,
1343                                    struct scsi_device *sdev)
1344 {
1345         if (shost->shost_state == SHOST_RECOVERY)
1346                 return 0;
1347         if (shost->host_busy == 0 && shost->host_blocked) {
1348                 /*
1349                  * unblock after host_blocked iterates to zero
1350                  */
1351                 if (--shost->host_blocked == 0) {
1352                         SCSI_LOG_MLQUEUE(3,
1353                                 printk("scsi%d unblocking host at zero depth\n",
1354                                         shost->host_no));
1355                 } else {
1356                         blk_plug_device(q);
1357                         return 0;
1358                 }
1359         }
1360         if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
1361             shost->host_blocked || shost->host_self_blocked) {
1362                 if (list_empty(&sdev->starved_entry))
1363                         list_add_tail(&sdev->starved_entry, &shost->starved_list);
1364                 return 0;
1365         }
1366
1367         /* We're OK to process the command, so we can't be starved */
1368         if (!list_empty(&sdev->starved_entry))
1369                 list_del_init(&sdev->starved_entry);
1370
1371         return 1;
1372 }
1373
1374 /*
1375  * Kill a request for a dead device
1376  */
1377 static void scsi_kill_request(struct request *req, request_queue_t *q)
1378 {
1379         struct scsi_cmnd *cmd = req->special;
1380
1381         blkdev_dequeue_request(req);
1382
1383         if (unlikely(cmd == NULL)) {
1384                 printk(KERN_CRIT "impossible request in %s.\n",
1385                                  __FUNCTION__);
1386                 BUG();
1387         }
1388
1389         scsi_init_cmd_errh(cmd);
1390         cmd->result = DID_NO_CONNECT << 16;
1391         atomic_inc(&cmd->device->iorequest_cnt);
1392         __scsi_done(cmd);
1393 }
1394
1395 /*
1396  * Function:    scsi_request_fn()
1397  *
1398  * Purpose:     Main strategy routine for SCSI.
1399  *
1400  * Arguments:   q       - Pointer to actual queue.
1401  *
1402  * Returns:     Nothing
1403  *
1404  * Lock status: IO request lock assumed to be held when called.
1405  */
1406 static void scsi_request_fn(struct request_queue *q)
1407 {
1408         struct scsi_device *sdev = q->queuedata;
1409         struct Scsi_Host *shost;
1410         struct scsi_cmnd *cmd;
1411         struct request *req;
1412
1413         if (!sdev) {
1414                 printk("scsi: killing requests for dead queue\n");
1415                 while ((req = elv_next_request(q)) != NULL)
1416                         scsi_kill_request(req, q);
1417                 return;
1418         }
1419
1420         if(!get_device(&sdev->sdev_gendev))
1421                 /* We must be tearing the block queue down already */
1422                 return;
1423
1424         /*
1425          * To start with, we keep looping until the queue is empty, or until
1426          * the host is no longer able to accept any more requests.
1427          */
1428         shost = sdev->host;
1429         while (!blk_queue_plugged(q)) {
1430                 int rtn;
1431                 /*
1432                  * get next queueable request.  We do this early to make sure
1433                  * that the request is fully prepared even if we cannot 
1434                  * accept it.
1435                  */
1436                 req = elv_next_request(q);
1437                 if (!req || !scsi_dev_queue_ready(q, sdev))
1438                         break;
1439
1440                 if (unlikely(!scsi_device_online(sdev))) {
1441                         printk(KERN_ERR "scsi%d (%d:%d): rejecting I/O to offline device\n",
1442                                sdev->host->host_no, sdev->id, sdev->lun);
1443                         scsi_kill_request(req, q);
1444                         continue;
1445                 }
1446
1447
1448                 /*
1449                  * Remove the request from the request list.
1450                  */
1451                 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1452                         blkdev_dequeue_request(req);
1453                 sdev->device_busy++;
1454
1455                 spin_unlock(q->queue_lock);
1456                 cmd = req->special;
1457                 if (unlikely(cmd == NULL)) {
1458                         printk(KERN_CRIT "impossible request in %s.\n"
1459                                          "please mail a stack trace to "
1460                                          "linux-scsi@vger.kernel.org",
1461                                          __FUNCTION__);
1462                         BUG();
1463                 }
1464                 spin_lock(shost->host_lock);
1465
1466                 if (!scsi_host_queue_ready(q, shost, sdev))
1467                         goto not_ready;
1468                 if (sdev->single_lun) {
1469                         if (scsi_target(sdev)->starget_sdev_user &&
1470                             scsi_target(sdev)->starget_sdev_user != sdev)
1471                                 goto not_ready;
1472                         scsi_target(sdev)->starget_sdev_user = sdev;
1473                 }
1474                 shost->host_busy++;
1475
1476                 /*
1477                  * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
1478                  *              take the lock again.
1479                  */
1480                 spin_unlock_irq(shost->host_lock);
1481
1482                 /*
1483                  * Finally, initialize any error handling parameters, and set up
1484                  * the timers for timeouts.
1485                  */
1486                 scsi_init_cmd_errh(cmd);
1487
1488                 /*
1489                  * Dispatch the command to the low-level driver.
1490                  */
1491                 rtn = scsi_dispatch_cmd(cmd);
1492                 spin_lock_irq(q->queue_lock);
1493                 if(rtn) {
1494                         /* we're refusing the command; because of
1495                          * the way locks get dropped, we need to 
1496                          * check here if plugging is required */
1497                         if(sdev->device_busy == 0)
1498                                 blk_plug_device(q);
1499
1500                         break;
1501                 }
1502         }
1503
1504         goto out;
1505
1506  not_ready:
1507         spin_unlock_irq(shost->host_lock);
1508
1509         /*
1510          * lock q, handle tag, requeue req, and decrement device_busy. We
1511          * must return with queue_lock held.
1512          *
1513          * Decrementing device_busy without checking it is OK, as all such
1514          * cases (host limits or settings) should run the queue at some
1515          * later time.
1516          */
1517         scsi_unprep_request(req);
1518         spin_lock_irq(q->queue_lock);
1519         blk_requeue_request(q, req);
1520         sdev->device_busy--;
1521         if(sdev->device_busy == 0)
1522                 blk_plug_device(q);
1523  out:
1524         /* must be careful here...if we trigger the ->remove() function
1525          * we cannot be holding the q lock */
1526         spin_unlock_irq(q->queue_lock);
1527         put_device(&sdev->sdev_gendev);
1528         spin_lock_irq(q->queue_lock);
1529 }
1530
1531 u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
1532 {
1533         struct device *host_dev;
1534         u64 bounce_limit = 0xffffffff;
1535
1536         if (shost->unchecked_isa_dma)
1537                 return BLK_BOUNCE_ISA;
1538         /*
1539          * Platforms with virtual-DMA translation
1540          * hardware have no practical limit.
1541          */
1542         if (!PCI_DMA_BUS_IS_PHYS)
1543                 return BLK_BOUNCE_ANY;
1544
1545         host_dev = scsi_get_device(shost);
1546         if (host_dev && host_dev->dma_mask)
1547                 bounce_limit = *host_dev->dma_mask;
1548
1549         return bounce_limit;
1550 }
1551 EXPORT_SYMBOL(scsi_calculate_bounce_limit);
1552
1553 struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
1554 {
1555         struct Scsi_Host *shost = sdev->host;
1556         struct request_queue *q;
1557
1558         q = blk_init_queue(scsi_request_fn, NULL);
1559         if (!q)
1560                 return NULL;
1561
1562         blk_queue_prep_rq(q, scsi_prep_fn);
1563
1564         blk_queue_max_hw_segments(q, shost->sg_tablesize);
1565         blk_queue_max_phys_segments(q, SCSI_MAX_PHYS_SEGMENTS);
1566         blk_queue_max_sectors(q, shost->max_sectors);
1567         blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
1568         blk_queue_segment_boundary(q, shost->dma_boundary);
1569         blk_queue_issue_flush_fn(q, scsi_issue_flush_fn);
1570
1571         /*
1572          * ordered tags are superior to flush ordering
1573          */
1574         if (shost->ordered_tag)
1575                 blk_queue_ordered(q, QUEUE_ORDERED_TAG);
1576         else if (shost->ordered_flush) {
1577                 blk_queue_ordered(q, QUEUE_ORDERED_FLUSH);
1578                 q->prepare_flush_fn = scsi_prepare_flush_fn;
1579                 q->end_flush_fn = scsi_end_flush_fn;
1580         }
1581
1582         if (!shost->use_clustering)
1583                 clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
1584         return q;
1585 }
1586
1587 void scsi_free_queue(struct request_queue *q)
1588 {
1589         blk_cleanup_queue(q);
1590 }
1591
1592 /*
1593  * Function:    scsi_block_requests()
1594  *
1595  * Purpose:     Utility function used by low-level drivers to prevent further
1596  *              commands from being queued to the device.
1597  *
1598  * Arguments:   shost       - Host in question
1599  *
1600  * Returns:     Nothing
1601  *
1602  * Lock status: No locks are assumed held.
1603  *
1604  * Notes:       There is no timer nor any other means by which the requests
1605  *              get unblocked other than the low-level driver calling
1606  *              scsi_unblock_requests().
1607  */
1608 void scsi_block_requests(struct Scsi_Host *shost)
1609 {
1610         shost->host_self_blocked = 1;
1611 }
1612 EXPORT_SYMBOL(scsi_block_requests);
1613
1614 /*
1615  * Function:    scsi_unblock_requests()
1616  *
1617  * Purpose:     Utility function used by low-level drivers to allow further
1618  *              commands from being queued to the device.
1619  *
1620  * Arguments:   shost       - Host in question
1621  *
1622  * Returns:     Nothing
1623  *
1624  * Lock status: No locks are assumed held.
1625  *
1626  * Notes:       There is no timer nor any other means by which the requests
1627  *              get unblocked other than the low-level driver calling
1628  *              scsi_unblock_requests().
1629  *
1630  *              This is done as an API function so that changes to the
1631  *              internals of the scsi mid-layer won't require wholesale
1632  *              changes to drivers that use this feature.
1633  */
1634 void scsi_unblock_requests(struct Scsi_Host *shost)
1635 {
1636         shost->host_self_blocked = 0;
1637         scsi_run_host_queues(shost);
1638 }
1639 EXPORT_SYMBOL(scsi_unblock_requests);
1640
1641 int __init scsi_init_queue(void)
1642 {
1643         int i;
1644
1645         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1646                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1647                 int size = sgp->size * sizeof(struct scatterlist);
1648
1649                 sgp->slab = kmem_cache_create(sgp->name, size, 0,
1650                                 SLAB_HWCACHE_ALIGN, NULL, NULL);
1651                 if (!sgp->slab) {
1652                         printk(KERN_ERR "SCSI: can't init sg slab %s\n",
1653                                         sgp->name);
1654                 }
1655
1656                 sgp->pool = mempool_create(SG_MEMPOOL_SIZE,
1657                                 mempool_alloc_slab, mempool_free_slab,
1658                                 sgp->slab);
1659                 if (!sgp->pool) {
1660                         printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
1661                                         sgp->name);
1662                 }
1663         }
1664
1665         return 0;
1666 }
1667
1668 void scsi_exit_queue(void)
1669 {
1670         int i;
1671
1672         for (i = 0; i < SG_MEMPOOL_NR; i++) {
1673                 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
1674                 mempool_destroy(sgp->pool);
1675                 kmem_cache_destroy(sgp->slab);
1676         }
1677 }
1678 /**
1679  *      scsi_mode_sense - issue a mode sense, falling back from 10 to 
1680  *              six bytes if necessary.
1681  *      @sdev:  SCSI device to be queried
1682  *      @dbd:   set if mode sense will allow block descriptors to be returned
1683  *      @modepage: mode page being requested
1684  *      @buffer: request buffer (may not be smaller than eight bytes)
1685  *      @len:   length of request buffer.
1686  *      @timeout: command timeout
1687  *      @retries: number of retries before failing
1688  *      @data: returns a structure abstracting the mode header data
1689  *      @sense: place to put sense data (or NULL if no sense to be collected).
1690  *              must be SCSI_SENSE_BUFFERSIZE big.
1691  *
1692  *      Returns zero if unsuccessful, or the header offset (either 4
1693  *      or 8 depending on whether a six or ten byte command was
1694  *      issued) if successful.
1695  **/
1696 int
1697 scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
1698                   unsigned char *buffer, int len, int timeout, int retries,
1699                   struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr) {
1700         unsigned char cmd[12];
1701         int use_10_for_ms;
1702         int header_length;
1703         int result;
1704         struct scsi_sense_hdr my_sshdr;
1705
1706         memset(data, 0, sizeof(*data));
1707         memset(&cmd[0], 0, 12);
1708         cmd[1] = dbd & 0x18;    /* allows DBD and LLBA bits */
1709         cmd[2] = modepage;
1710
1711         /* caller might not be interested in sense, but we need it */
1712         if (!sshdr)
1713                 sshdr = &my_sshdr;
1714
1715  retry:
1716         use_10_for_ms = sdev->use_10_for_ms;
1717
1718         if (use_10_for_ms) {
1719                 if (len < 8)
1720                         len = 8;
1721
1722                 cmd[0] = MODE_SENSE_10;
1723                 cmd[8] = len;
1724                 header_length = 8;
1725         } else {
1726                 if (len < 4)
1727                         len = 4;
1728
1729                 cmd[0] = MODE_SENSE;
1730                 cmd[4] = len;
1731                 header_length = 4;
1732         }
1733
1734         memset(buffer, 0, len);
1735
1736         result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
1737                                   sshdr, timeout, retries);
1738
1739         /* This code looks awful: what it's doing is making sure an
1740          * ILLEGAL REQUEST sense return identifies the actual command
1741          * byte as the problem.  MODE_SENSE commands can return
1742          * ILLEGAL REQUEST if the code page isn't supported */
1743
1744         if (use_10_for_ms && !scsi_status_is_good(result) &&
1745             (driver_byte(result) & DRIVER_SENSE)) {
1746                 if (scsi_sense_valid(sshdr)) {
1747                         if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
1748                             (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
1749                                 /* 
1750                                  * Invalid command operation code
1751                                  */
1752                                 sdev->use_10_for_ms = 0;
1753                                 goto retry;
1754                         }
1755                 }
1756         }
1757
1758         if(scsi_status_is_good(result)) {
1759                 data->header_length = header_length;
1760                 if(use_10_for_ms) {
1761                         data->length = buffer[0]*256 + buffer[1] + 2;
1762                         data->medium_type = buffer[2];
1763                         data->device_specific = buffer[3];
1764                         data->longlba = buffer[4] & 0x01;
1765                         data->block_descriptor_length = buffer[6]*256
1766                                 + buffer[7];
1767                 } else {
1768                         data->length = buffer[0] + 1;
1769                         data->medium_type = buffer[1];
1770                         data->device_specific = buffer[2];
1771                         data->block_descriptor_length = buffer[3];
1772                 }
1773         }
1774
1775         return result;
1776 }
1777 EXPORT_SYMBOL(scsi_mode_sense);
1778
1779 int
1780 scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries)
1781 {
1782         char cmd[] = {
1783                 TEST_UNIT_READY, 0, 0, 0, 0, 0,
1784         };
1785         struct scsi_sense_hdr sshdr;
1786         int result;
1787         
1788         result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, &sshdr,
1789                                   timeout, retries);
1790
1791         if ((driver_byte(result) & DRIVER_SENSE) && sdev->removable) {
1792
1793                 if ((scsi_sense_valid(&sshdr)) &&
1794                     ((sshdr.sense_key == UNIT_ATTENTION) ||
1795                      (sshdr.sense_key == NOT_READY))) {
1796                         sdev->changed = 1;
1797                         result = 0;
1798                 }
1799         }
1800         return result;
1801 }
1802 EXPORT_SYMBOL(scsi_test_unit_ready);
1803
1804 /**
1805  *      scsi_device_set_state - Take the given device through the device
1806  *              state model.
1807  *      @sdev:  scsi device to change the state of.
1808  *      @state: state to change to.
1809  *
1810  *      Returns zero if unsuccessful or an error if the requested 
1811  *      transition is illegal.
1812  **/
1813 int
1814 scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
1815 {
1816         enum scsi_device_state oldstate = sdev->sdev_state;
1817
1818         if (state == oldstate)
1819                 return 0;
1820
1821         switch (state) {
1822         case SDEV_CREATED:
1823                 /* There are no legal states that come back to
1824                  * created.  This is the manually initialised start
1825                  * state */
1826                 goto illegal;
1827                         
1828         case SDEV_RUNNING:
1829                 switch (oldstate) {
1830                 case SDEV_CREATED:
1831                 case SDEV_OFFLINE:
1832                 case SDEV_QUIESCE:
1833                 case SDEV_BLOCK:
1834                         break;
1835                 default:
1836                         goto illegal;
1837                 }
1838                 break;
1839
1840         case SDEV_QUIESCE:
1841                 switch (oldstate) {
1842                 case SDEV_RUNNING:
1843                 case SDEV_OFFLINE:
1844                         break;
1845                 default:
1846                         goto illegal;
1847                 }
1848                 break;
1849
1850         case SDEV_OFFLINE:
1851                 switch (oldstate) {
1852                 case SDEV_CREATED:
1853                 case SDEV_RUNNING:
1854                 case SDEV_QUIESCE:
1855                 case SDEV_BLOCK:
1856                         break;
1857                 default:
1858                         goto illegal;
1859                 }
1860                 break;
1861
1862         case SDEV_BLOCK:
1863                 switch (oldstate) {
1864                 case SDEV_CREATED:
1865                 case SDEV_RUNNING:
1866                         break;
1867                 default:
1868                         goto illegal;
1869                 }
1870                 break;
1871
1872         case SDEV_CANCEL:
1873                 switch (oldstate) {
1874                 case SDEV_CREATED:
1875                 case SDEV_RUNNING:
1876                 case SDEV_OFFLINE:
1877                 case SDEV_BLOCK:
1878                         break;
1879                 default:
1880                         goto illegal;
1881                 }
1882                 break;
1883
1884         case SDEV_DEL:
1885                 switch (oldstate) {
1886                 case SDEV_CANCEL:
1887                         break;
1888                 default:
1889                         goto illegal;
1890                 }
1891                 break;
1892
1893         }
1894         sdev->sdev_state = state;
1895         return 0;
1896
1897  illegal:
1898         SCSI_LOG_ERROR_RECOVERY(1, 
1899                                 dev_printk(KERN_ERR, &sdev->sdev_gendev,
1900                                            "Illegal state transition %s->%s\n",
1901                                            scsi_device_state_name(oldstate),
1902                                            scsi_device_state_name(state))
1903                                 );
1904         return -EINVAL;
1905 }
1906 EXPORT_SYMBOL(scsi_device_set_state);
1907
1908 /**
1909  *      scsi_device_quiesce - Block user issued commands.
1910  *      @sdev:  scsi device to quiesce.
1911  *
1912  *      This works by trying to transition to the SDEV_QUIESCE state
1913  *      (which must be a legal transition).  When the device is in this
1914  *      state, only special requests will be accepted, all others will
1915  *      be deferred.  Since special requests may also be requeued requests,
1916  *      a successful return doesn't guarantee the device will be 
1917  *      totally quiescent.
1918  *
1919  *      Must be called with user context, may sleep.
1920  *
1921  *      Returns zero if unsuccessful or an error if not.
1922  **/
1923 int
1924 scsi_device_quiesce(struct scsi_device *sdev)
1925 {
1926         int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
1927         if (err)
1928                 return err;
1929
1930         scsi_run_queue(sdev->request_queue);
1931         while (sdev->device_busy) {
1932                 msleep_interruptible(200);
1933                 scsi_run_queue(sdev->request_queue);
1934         }
1935         return 0;
1936 }
1937 EXPORT_SYMBOL(scsi_device_quiesce);
1938
1939 /**
1940  *      scsi_device_resume - Restart user issued commands to a quiesced device.
1941  *      @sdev:  scsi device to resume.
1942  *
1943  *      Moves the device from quiesced back to running and restarts the
1944  *      queues.
1945  *
1946  *      Must be called with user context, may sleep.
1947  **/
1948 void
1949 scsi_device_resume(struct scsi_device *sdev)
1950 {
1951         if(scsi_device_set_state(sdev, SDEV_RUNNING))
1952                 return;
1953         scsi_run_queue(sdev->request_queue);
1954 }
1955 EXPORT_SYMBOL(scsi_device_resume);
1956
1957 static void
1958 device_quiesce_fn(struct scsi_device *sdev, void *data)
1959 {
1960         scsi_device_quiesce(sdev);
1961 }
1962
1963 void
1964 scsi_target_quiesce(struct scsi_target *starget)
1965 {
1966         starget_for_each_device(starget, NULL, device_quiesce_fn);
1967 }
1968 EXPORT_SYMBOL(scsi_target_quiesce);
1969
1970 static void
1971 device_resume_fn(struct scsi_device *sdev, void *data)
1972 {
1973         scsi_device_resume(sdev);
1974 }
1975
1976 void
1977 scsi_target_resume(struct scsi_target *starget)
1978 {
1979         starget_for_each_device(starget, NULL, device_resume_fn);
1980 }
1981 EXPORT_SYMBOL(scsi_target_resume);
1982
1983 /**
1984  * scsi_internal_device_block - internal function to put a device
1985  *                              temporarily into the SDEV_BLOCK state
1986  * @sdev:       device to block
1987  *
1988  * Block request made by scsi lld's to temporarily stop all
1989  * scsi commands on the specified device.  Called from interrupt
1990  * or normal process context.
1991  *
1992  * Returns zero if successful or error if not
1993  *
1994  * Notes:       
1995  *      This routine transitions the device to the SDEV_BLOCK state
1996  *      (which must be a legal transition).  When the device is in this
1997  *      state, all commands are deferred until the scsi lld reenables
1998  *      the device with scsi_device_unblock or device_block_tmo fires.
1999  *      This routine assumes the host_lock is held on entry.
2000  **/
2001 int
2002 scsi_internal_device_block(struct scsi_device *sdev)
2003 {
2004         request_queue_t *q = sdev->request_queue;
2005         unsigned long flags;
2006         int err = 0;
2007
2008         err = scsi_device_set_state(sdev, SDEV_BLOCK);
2009         if (err)
2010                 return err;
2011
2012         /* 
2013          * The device has transitioned to SDEV_BLOCK.  Stop the
2014          * block layer from calling the midlayer with this device's
2015          * request queue. 
2016          */
2017         spin_lock_irqsave(q->queue_lock, flags);
2018         blk_stop_queue(q);
2019         spin_unlock_irqrestore(q->queue_lock, flags);
2020
2021         return 0;
2022 }
2023 EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2024  
2025 /**
2026  * scsi_internal_device_unblock - resume a device after a block request
2027  * @sdev:       device to resume
2028  *
2029  * Called by scsi lld's or the midlayer to restart the device queue
2030  * for the previously suspended scsi device.  Called from interrupt or
2031  * normal process context.
2032  *
2033  * Returns zero if successful or error if not.
2034  *
2035  * Notes:       
2036  *      This routine transitions the device to the SDEV_RUNNING state
2037  *      (which must be a legal transition) allowing the midlayer to
2038  *      goose the queue for this device.  This routine assumes the 
2039  *      host_lock is held upon entry.
2040  **/
2041 int
2042 scsi_internal_device_unblock(struct scsi_device *sdev)
2043 {
2044         request_queue_t *q = sdev->request_queue; 
2045         int err;
2046         unsigned long flags;
2047         
2048         /* 
2049          * Try to transition the scsi device to SDEV_RUNNING
2050          * and goose the device queue if successful.  
2051          */
2052         err = scsi_device_set_state(sdev, SDEV_RUNNING);
2053         if (err)
2054                 return err;
2055
2056         spin_lock_irqsave(q->queue_lock, flags);
2057         blk_start_queue(q);
2058         spin_unlock_irqrestore(q->queue_lock, flags);
2059
2060         return 0;
2061 }
2062 EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
2063
2064 static void
2065 device_block(struct scsi_device *sdev, void *data)
2066 {
2067         scsi_internal_device_block(sdev);
2068 }
2069
2070 static int
2071 target_block(struct device *dev, void *data)
2072 {
2073         if (scsi_is_target_device(dev))
2074                 starget_for_each_device(to_scsi_target(dev), NULL,
2075                                         device_block);
2076         return 0;
2077 }
2078
2079 void
2080 scsi_target_block(struct device *dev)
2081 {
2082         if (scsi_is_target_device(dev))
2083                 starget_for_each_device(to_scsi_target(dev), NULL,
2084                                         device_block);
2085         else
2086                 device_for_each_child(dev, NULL, target_block);
2087 }
2088 EXPORT_SYMBOL_GPL(scsi_target_block);
2089
2090 static void
2091 device_unblock(struct scsi_device *sdev, void *data)
2092 {
2093         scsi_internal_device_unblock(sdev);
2094 }
2095
2096 static int
2097 target_unblock(struct device *dev, void *data)
2098 {
2099         if (scsi_is_target_device(dev))
2100                 starget_for_each_device(to_scsi_target(dev), NULL,
2101                                         device_unblock);
2102         return 0;
2103 }
2104
2105 void
2106 scsi_target_unblock(struct device *dev)
2107 {
2108         if (scsi_is_target_device(dev))
2109                 starget_for_each_device(to_scsi_target(dev), NULL,
2110                                         device_unblock);
2111         else
2112                 device_for_each_child(dev, NULL, target_unblock);
2113 }
2114 EXPORT_SYMBOL_GPL(scsi_target_unblock);