Pull netlink into release branch
[linux-2.6] / drivers / ide / ide-io.c
1 /*
2  *      IDE I/O functions
3  *
4  *      Basic PIO and command management functionality.
5  *
6  * This code was split off from ide.c. See ide.c for history and original
7  * copyrights.
8  *
9  * This program is free software; you can redistribute it and/or modify it
10  * under the terms of the GNU General Public License as published by the
11  * Free Software Foundation; either version 2, or (at your option) any
12  * later version.
13  *
14  * This program is distributed in the hope that it will be useful, but
15  * WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
17  * General Public License for more details.
18  *
19  * For the avoidance of doubt the "preferred form" of this code is one which
20  * is in an open non patent encumbered format. Where cryptographic key signing
21  * forms part of the process of creating an executable the information
22  * including keys needed to generate an equivalently functional executable
23  * are deemed to be part of the source code.
24  */
25  
26  
27 #include <linux/module.h>
28 #include <linux/types.h>
29 #include <linux/string.h>
30 #include <linux/kernel.h>
31 #include <linux/timer.h>
32 #include <linux/mm.h>
33 #include <linux/interrupt.h>
34 #include <linux/major.h>
35 #include <linux/errno.h>
36 #include <linux/genhd.h>
37 #include <linux/blkpg.h>
38 #include <linux/slab.h>
39 #include <linux/init.h>
40 #include <linux/pci.h>
41 #include <linux/delay.h>
42 #include <linux/ide.h>
43 #include <linux/completion.h>
44 #include <linux/reboot.h>
45 #include <linux/cdrom.h>
46 #include <linux/seq_file.h>
47 #include <linux/device.h>
48 #include <linux/kmod.h>
49 #include <linux/scatterlist.h>
50
51 #include <asm/byteorder.h>
52 #include <asm/irq.h>
53 #include <asm/uaccess.h>
54 #include <asm/io.h>
55 #include <asm/bitops.h>
56
57 static int __ide_end_request(ide_drive_t *drive, struct request *rq,
58                              int uptodate, unsigned int nr_bytes)
59 {
60         int ret = 1;
61
62         /*
63          * if failfast is set on a request, override number of sectors and
64          * complete the whole request right now
65          */
66         if (blk_noretry_request(rq) && end_io_error(uptodate))
67                 nr_bytes = rq->hard_nr_sectors << 9;
68
69         if (!blk_fs_request(rq) && end_io_error(uptodate) && !rq->errors)
70                 rq->errors = -EIO;
71
72         /*
73          * decide whether to reenable DMA -- 3 is a random magic for now,
74          * if we DMA timeout more than 3 times, just stay in PIO
75          */
76         if (drive->state == DMA_PIO_RETRY && drive->retry_pio <= 3) {
77                 drive->state = 0;
78                 HWGROUP(drive)->hwif->ide_dma_on(drive);
79         }
80
81         if (!end_that_request_chunk(rq, uptodate, nr_bytes)) {
82                 add_disk_randomness(rq->rq_disk);
83                 if (!list_empty(&rq->queuelist))
84                         blkdev_dequeue_request(rq);
85                 HWGROUP(drive)->rq = NULL;
86                 end_that_request_last(rq, uptodate);
87                 ret = 0;
88         }
89
90         return ret;
91 }
92
93 /**
94  *      ide_end_request         -       complete an IDE I/O
95  *      @drive: IDE device for the I/O
96  *      @uptodate:
97  *      @nr_sectors: number of sectors completed
98  *
99  *      This is our end_request wrapper function. We complete the I/O
100  *      update random number input and dequeue the request, which if
101  *      it was tagged may be out of order.
102  */
103
104 int ide_end_request (ide_drive_t *drive, int uptodate, int nr_sectors)
105 {
106         unsigned int nr_bytes = nr_sectors << 9;
107         struct request *rq;
108         unsigned long flags;
109         int ret = 1;
110
111         /*
112          * room for locking improvements here, the calls below don't
113          * need the queue lock held at all
114          */
115         spin_lock_irqsave(&ide_lock, flags);
116         rq = HWGROUP(drive)->rq;
117
118         if (!nr_bytes) {
119                 if (blk_pc_request(rq))
120                         nr_bytes = rq->data_len;
121                 else
122                         nr_bytes = rq->hard_cur_sectors << 9;
123         }
124
125         ret = __ide_end_request(drive, rq, uptodate, nr_bytes);
126
127         spin_unlock_irqrestore(&ide_lock, flags);
128         return ret;
129 }
130 EXPORT_SYMBOL(ide_end_request);
131
132 /*
133  * Power Management state machine. This one is rather trivial for now,
134  * we should probably add more, like switching back to PIO on suspend
135  * to help some BIOSes, re-do the door locking on resume, etc...
136  */
137
138 enum {
139         ide_pm_flush_cache      = ide_pm_state_start_suspend,
140         idedisk_pm_standby,
141
142         idedisk_pm_restore_pio  = ide_pm_state_start_resume,
143         idedisk_pm_idle,
144         ide_pm_restore_dma,
145 };
146
147 static void ide_complete_power_step(ide_drive_t *drive, struct request *rq, u8 stat, u8 error)
148 {
149         struct request_pm_state *pm = rq->data;
150
151         if (drive->media != ide_disk)
152                 return;
153
154         switch (pm->pm_step) {
155         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) complete */
156                 if (pm->pm_state == PM_EVENT_FREEZE)
157                         pm->pm_step = ide_pm_state_completed;
158                 else
159                         pm->pm_step = idedisk_pm_standby;
160                 break;
161         case idedisk_pm_standby:        /* Suspend step 2 (standby) complete */
162                 pm->pm_step = ide_pm_state_completed;
163                 break;
164         case idedisk_pm_restore_pio:    /* Resume step 1 complete */
165                 pm->pm_step = idedisk_pm_idle;
166                 break;
167         case idedisk_pm_idle:           /* Resume step 2 (idle) complete */
168                 pm->pm_step = ide_pm_restore_dma;
169                 break;
170         }
171 }
172
173 static ide_startstop_t ide_start_power_step(ide_drive_t *drive, struct request *rq)
174 {
175         struct request_pm_state *pm = rq->data;
176         ide_task_t *args = rq->special;
177
178         memset(args, 0, sizeof(*args));
179
180         switch (pm->pm_step) {
181         case ide_pm_flush_cache:        /* Suspend step 1 (flush cache) */
182                 if (drive->media != ide_disk)
183                         break;
184                 /* Not supported? Switch to next step now. */
185                 if (!drive->wcache || !ide_id_has_flush_cache(drive->id)) {
186                         ide_complete_power_step(drive, rq, 0, 0);
187                         return ide_stopped;
188                 }
189                 if (ide_id_has_flush_cache_ext(drive->id))
190                         args->tfRegister[IDE_COMMAND_OFFSET] = WIN_FLUSH_CACHE_EXT;
191                 else
192                         args->tfRegister[IDE_COMMAND_OFFSET] = WIN_FLUSH_CACHE;
193                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
194                 args->handler      = &task_no_data_intr;
195                 return do_rw_taskfile(drive, args);
196
197         case idedisk_pm_standby:        /* Suspend step 2 (standby) */
198                 args->tfRegister[IDE_COMMAND_OFFSET] = WIN_STANDBYNOW1;
199                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
200                 args->handler      = &task_no_data_intr;
201                 return do_rw_taskfile(drive, args);
202
203         case idedisk_pm_restore_pio:    /* Resume step 1 (restore PIO) */
204                 if (drive->hwif->tuneproc != NULL)
205                         drive->hwif->tuneproc(drive, 255);
206                 /*
207                  * skip idedisk_pm_idle for ATAPI devices
208                  */
209                 if (drive->media != ide_disk)
210                         pm->pm_step = ide_pm_restore_dma;
211                 else
212                         ide_complete_power_step(drive, rq, 0, 0);
213                 return ide_stopped;
214
215         case idedisk_pm_idle:           /* Resume step 2 (idle) */
216                 args->tfRegister[IDE_COMMAND_OFFSET] = WIN_IDLEIMMEDIATE;
217                 args->command_type = IDE_DRIVE_TASK_NO_DATA;
218                 args->handler = task_no_data_intr;
219                 return do_rw_taskfile(drive, args);
220
221         case ide_pm_restore_dma:        /* Resume step 3 (restore DMA) */
222                 /*
223                  * Right now, all we do is call hwif->ide_dma_check(drive),
224                  * we could be smarter and check for current xfer_speed
225                  * in struct drive etc...
226                  */
227                 if (drive->hwif->ide_dma_check == NULL)
228                         break;
229                 drive->hwif->dma_off_quietly(drive);
230                 /*
231                  * TODO: respect ->using_dma setting
232                  */
233                 ide_set_dma(drive);
234                 break;
235         }
236         pm->pm_step = ide_pm_state_completed;
237         return ide_stopped;
238 }
239
240 /**
241  *      ide_end_dequeued_request        -       complete an IDE I/O
242  *      @drive: IDE device for the I/O
243  *      @uptodate:
244  *      @nr_sectors: number of sectors completed
245  *
246  *      Complete an I/O that is no longer on the request queue. This
247  *      typically occurs when we pull the request and issue a REQUEST_SENSE.
248  *      We must still finish the old request but we must not tamper with the
249  *      queue in the meantime.
250  *
251  *      NOTE: This path does not handle barrier, but barrier is not supported
252  *      on ide-cd anyway.
253  */
254
255 int ide_end_dequeued_request(ide_drive_t *drive, struct request *rq,
256                              int uptodate, int nr_sectors)
257 {
258         unsigned long flags;
259         int ret = 1;
260
261         spin_lock_irqsave(&ide_lock, flags);
262
263         BUG_ON(!blk_rq_started(rq));
264
265         /*
266          * if failfast is set on a request, override number of sectors and
267          * complete the whole request right now
268          */
269         if (blk_noretry_request(rq) && end_io_error(uptodate))
270                 nr_sectors = rq->hard_nr_sectors;
271
272         if (!blk_fs_request(rq) && end_io_error(uptodate) && !rq->errors)
273                 rq->errors = -EIO;
274
275         /*
276          * decide whether to reenable DMA -- 3 is a random magic for now,
277          * if we DMA timeout more than 3 times, just stay in PIO
278          */
279         if (drive->state == DMA_PIO_RETRY && drive->retry_pio <= 3) {
280                 drive->state = 0;
281                 HWGROUP(drive)->hwif->ide_dma_on(drive);
282         }
283
284         if (!end_that_request_first(rq, uptodate, nr_sectors)) {
285                 add_disk_randomness(rq->rq_disk);
286                 if (blk_rq_tagged(rq))
287                         blk_queue_end_tag(drive->queue, rq);
288                 end_that_request_last(rq, uptodate);
289                 ret = 0;
290         }
291         spin_unlock_irqrestore(&ide_lock, flags);
292         return ret;
293 }
294 EXPORT_SYMBOL_GPL(ide_end_dequeued_request);
295
296
297 /**
298  *      ide_complete_pm_request - end the current Power Management request
299  *      @drive: target drive
300  *      @rq: request
301  *
302  *      This function cleans up the current PM request and stops the queue
303  *      if necessary.
304  */
305 static void ide_complete_pm_request (ide_drive_t *drive, struct request *rq)
306 {
307         unsigned long flags;
308
309 #ifdef DEBUG_PM
310         printk("%s: completing PM request, %s\n", drive->name,
311                blk_pm_suspend_request(rq) ? "suspend" : "resume");
312 #endif
313         spin_lock_irqsave(&ide_lock, flags);
314         if (blk_pm_suspend_request(rq)) {
315                 blk_stop_queue(drive->queue);
316         } else {
317                 drive->blocked = 0;
318                 blk_start_queue(drive->queue);
319         }
320         blkdev_dequeue_request(rq);
321         HWGROUP(drive)->rq = NULL;
322         end_that_request_last(rq, 1);
323         spin_unlock_irqrestore(&ide_lock, flags);
324 }
325
326 /*
327  * FIXME: probably move this somewhere else, name is bad too :)
328  */
329 u64 ide_get_error_location(ide_drive_t *drive, char *args)
330 {
331         u32 high, low;
332         u8 hcyl, lcyl, sect;
333         u64 sector;
334
335         high = 0;
336         hcyl = args[5];
337         lcyl = args[4];
338         sect = args[3];
339
340         if (ide_id_has_flush_cache_ext(drive->id)) {
341                 low = (hcyl << 16) | (lcyl << 8) | sect;
342                 HWIF(drive)->OUTB(drive->ctl|0x80, IDE_CONTROL_REG);
343                 high = ide_read_24(drive);
344         } else {
345                 u8 cur = HWIF(drive)->INB(IDE_SELECT_REG);
346                 if (cur & 0x40) {
347                         high = cur & 0xf;
348                         low = (hcyl << 16) | (lcyl << 8) | sect;
349                 } else {
350                         low = hcyl * drive->head * drive->sect;
351                         low += lcyl * drive->sect;
352                         low += sect - 1;
353                 }
354         }
355
356         sector = ((u64) high << 24) | low;
357         return sector;
358 }
359 EXPORT_SYMBOL(ide_get_error_location);
360
361 /**
362  *      ide_end_drive_cmd       -       end an explicit drive command
363  *      @drive: command 
364  *      @stat: status bits
365  *      @err: error bits
366  *
367  *      Clean up after success/failure of an explicit drive command.
368  *      These get thrown onto the queue so they are synchronized with
369  *      real I/O operations on the drive.
370  *
371  *      In LBA48 mode we have to read the register set twice to get
372  *      all the extra information out.
373  */
374  
375 void ide_end_drive_cmd (ide_drive_t *drive, u8 stat, u8 err)
376 {
377         ide_hwif_t *hwif = HWIF(drive);
378         unsigned long flags;
379         struct request *rq;
380
381         spin_lock_irqsave(&ide_lock, flags);
382         rq = HWGROUP(drive)->rq;
383         spin_unlock_irqrestore(&ide_lock, flags);
384
385         if (rq->cmd_type == REQ_TYPE_ATA_CMD) {
386                 u8 *args = (u8 *) rq->buffer;
387                 if (rq->errors == 0)
388                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
389
390                 if (args) {
391                         args[0] = stat;
392                         args[1] = err;
393                         args[2] = hwif->INB(IDE_NSECTOR_REG);
394                 }
395         } else if (rq->cmd_type == REQ_TYPE_ATA_TASK) {
396                 u8 *args = (u8 *) rq->buffer;
397                 if (rq->errors == 0)
398                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
399
400                 if (args) {
401                         args[0] = stat;
402                         args[1] = err;
403                         args[2] = hwif->INB(IDE_NSECTOR_REG);
404                         args[3] = hwif->INB(IDE_SECTOR_REG);
405                         args[4] = hwif->INB(IDE_LCYL_REG);
406                         args[5] = hwif->INB(IDE_HCYL_REG);
407                         args[6] = hwif->INB(IDE_SELECT_REG);
408                 }
409         } else if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE) {
410                 ide_task_t *args = (ide_task_t *) rq->special;
411                 if (rq->errors == 0)
412                         rq->errors = !OK_STAT(stat,READY_STAT,BAD_STAT);
413                         
414                 if (args) {
415                         if (args->tf_in_flags.b.data) {
416                                 u16 data                                = hwif->INW(IDE_DATA_REG);
417                                 args->tfRegister[IDE_DATA_OFFSET]       = (data) & 0xFF;
418                                 args->hobRegister[IDE_DATA_OFFSET]      = (data >> 8) & 0xFF;
419                         }
420                         args->tfRegister[IDE_ERROR_OFFSET]   = err;
421                         /* be sure we're looking at the low order bits */
422                         hwif->OUTB(drive->ctl & ~0x80, IDE_CONTROL_REG);
423                         args->tfRegister[IDE_NSECTOR_OFFSET] = hwif->INB(IDE_NSECTOR_REG);
424                         args->tfRegister[IDE_SECTOR_OFFSET]  = hwif->INB(IDE_SECTOR_REG);
425                         args->tfRegister[IDE_LCYL_OFFSET]    = hwif->INB(IDE_LCYL_REG);
426                         args->tfRegister[IDE_HCYL_OFFSET]    = hwif->INB(IDE_HCYL_REG);
427                         args->tfRegister[IDE_SELECT_OFFSET]  = hwif->INB(IDE_SELECT_REG);
428                         args->tfRegister[IDE_STATUS_OFFSET]  = stat;
429
430                         if (drive->addressing == 1) {
431                                 hwif->OUTB(drive->ctl|0x80, IDE_CONTROL_REG);
432                                 args->hobRegister[IDE_FEATURE_OFFSET]   = hwif->INB(IDE_FEATURE_REG);
433                                 args->hobRegister[IDE_NSECTOR_OFFSET]   = hwif->INB(IDE_NSECTOR_REG);
434                                 args->hobRegister[IDE_SECTOR_OFFSET]    = hwif->INB(IDE_SECTOR_REG);
435                                 args->hobRegister[IDE_LCYL_OFFSET]      = hwif->INB(IDE_LCYL_REG);
436                                 args->hobRegister[IDE_HCYL_OFFSET]      = hwif->INB(IDE_HCYL_REG);
437                         }
438                 }
439         } else if (blk_pm_request(rq)) {
440                 struct request_pm_state *pm = rq->data;
441 #ifdef DEBUG_PM
442                 printk("%s: complete_power_step(step: %d, stat: %x, err: %x)\n",
443                         drive->name, rq->pm->pm_step, stat, err);
444 #endif
445                 ide_complete_power_step(drive, rq, stat, err);
446                 if (pm->pm_step == ide_pm_state_completed)
447                         ide_complete_pm_request(drive, rq);
448                 return;
449         }
450
451         spin_lock_irqsave(&ide_lock, flags);
452         blkdev_dequeue_request(rq);
453         HWGROUP(drive)->rq = NULL;
454         rq->errors = err;
455         end_that_request_last(rq, !rq->errors);
456         spin_unlock_irqrestore(&ide_lock, flags);
457 }
458
459 EXPORT_SYMBOL(ide_end_drive_cmd);
460
461 /**
462  *      try_to_flush_leftover_data      -       flush junk
463  *      @drive: drive to flush
464  *
465  *      try_to_flush_leftover_data() is invoked in response to a drive
466  *      unexpectedly having its DRQ_STAT bit set.  As an alternative to
467  *      resetting the drive, this routine tries to clear the condition
468  *      by read a sector's worth of data from the drive.  Of course,
469  *      this may not help if the drive is *waiting* for data from *us*.
470  */
471 static void try_to_flush_leftover_data (ide_drive_t *drive)
472 {
473         int i = (drive->mult_count ? drive->mult_count : 1) * SECTOR_WORDS;
474
475         if (drive->media != ide_disk)
476                 return;
477         while (i > 0) {
478                 u32 buffer[16];
479                 u32 wcount = (i > 16) ? 16 : i;
480
481                 i -= wcount;
482                 HWIF(drive)->ata_input_data(drive, buffer, wcount);
483         }
484 }
485
486 static void ide_kill_rq(ide_drive_t *drive, struct request *rq)
487 {
488         if (rq->rq_disk) {
489                 ide_driver_t *drv;
490
491                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
492                 drv->end_request(drive, 0, 0);
493         } else
494                 ide_end_request(drive, 0, 0);
495 }
496
497 static ide_startstop_t ide_ata_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
498 {
499         ide_hwif_t *hwif = drive->hwif;
500
501         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
502                 /* other bits are useless when BUSY */
503                 rq->errors |= ERROR_RESET;
504         } else if (stat & ERR_STAT) {
505                 /* err has different meaning on cdrom and tape */
506                 if (err == ABRT_ERR) {
507                         if (drive->select.b.lba &&
508                             /* some newer drives don't support WIN_SPECIFY */
509                             hwif->INB(IDE_COMMAND_REG) == WIN_SPECIFY)
510                                 return ide_stopped;
511                 } else if ((err & BAD_CRC) == BAD_CRC) {
512                         /* UDMA crc error, just retry the operation */
513                         drive->crc_count++;
514                 } else if (err & (BBD_ERR | ECC_ERR)) {
515                         /* retries won't help these */
516                         rq->errors = ERROR_MAX;
517                 } else if (err & TRK0_ERR) {
518                         /* help it find track zero */
519                         rq->errors |= ERROR_RECAL;
520                 }
521         }
522
523         if ((stat & DRQ_STAT) && rq_data_dir(rq) == READ && hwif->err_stops_fifo == 0)
524                 try_to_flush_leftover_data(drive);
525
526         if (rq->errors >= ERROR_MAX || blk_noretry_request(rq)) {
527                 ide_kill_rq(drive, rq);
528                 return ide_stopped;
529         }
530
531         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
532                 rq->errors |= ERROR_RESET;
533
534         if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
535                 ++rq->errors;
536                 return ide_do_reset(drive);
537         }
538
539         if ((rq->errors & ERROR_RECAL) == ERROR_RECAL)
540                 drive->special.b.recalibrate = 1;
541
542         ++rq->errors;
543
544         return ide_stopped;
545 }
546
547 static ide_startstop_t ide_atapi_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
548 {
549         ide_hwif_t *hwif = drive->hwif;
550
551         if (stat & BUSY_STAT || ((stat & WRERR_STAT) && !drive->nowerr)) {
552                 /* other bits are useless when BUSY */
553                 rq->errors |= ERROR_RESET;
554         } else {
555                 /* add decoding error stuff */
556         }
557
558         if (hwif->INB(IDE_STATUS_REG) & (BUSY_STAT|DRQ_STAT))
559                 /* force an abort */
560                 hwif->OUTB(WIN_IDLEIMMEDIATE, IDE_COMMAND_REG);
561
562         if (rq->errors >= ERROR_MAX) {
563                 ide_kill_rq(drive, rq);
564         } else {
565                 if ((rq->errors & ERROR_RESET) == ERROR_RESET) {
566                         ++rq->errors;
567                         return ide_do_reset(drive);
568                 }
569                 ++rq->errors;
570         }
571
572         return ide_stopped;
573 }
574
575 ide_startstop_t
576 __ide_error(ide_drive_t *drive, struct request *rq, u8 stat, u8 err)
577 {
578         if (drive->media == ide_disk)
579                 return ide_ata_error(drive, rq, stat, err);
580         return ide_atapi_error(drive, rq, stat, err);
581 }
582
583 EXPORT_SYMBOL_GPL(__ide_error);
584
585 /**
586  *      ide_error       -       handle an error on the IDE
587  *      @drive: drive the error occurred on
588  *      @msg: message to report
589  *      @stat: status bits
590  *
591  *      ide_error() takes action based on the error returned by the drive.
592  *      For normal I/O that may well include retries. We deal with
593  *      both new-style (taskfile) and old style command handling here.
594  *      In the case of taskfile command handling there is work left to
595  *      do
596  */
597  
598 ide_startstop_t ide_error (ide_drive_t *drive, const char *msg, u8 stat)
599 {
600         struct request *rq;
601         u8 err;
602
603         err = ide_dump_status(drive, msg, stat);
604
605         if ((rq = HWGROUP(drive)->rq) == NULL)
606                 return ide_stopped;
607
608         /* retry only "normal" I/O: */
609         if (!blk_fs_request(rq)) {
610                 rq->errors = 1;
611                 ide_end_drive_cmd(drive, stat, err);
612                 return ide_stopped;
613         }
614
615         if (rq->rq_disk) {
616                 ide_driver_t *drv;
617
618                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
619                 return drv->error(drive, rq, stat, err);
620         } else
621                 return __ide_error(drive, rq, stat, err);
622 }
623
624 EXPORT_SYMBOL_GPL(ide_error);
625
626 ide_startstop_t __ide_abort(ide_drive_t *drive, struct request *rq)
627 {
628         if (drive->media != ide_disk)
629                 rq->errors |= ERROR_RESET;
630
631         ide_kill_rq(drive, rq);
632
633         return ide_stopped;
634 }
635
636 EXPORT_SYMBOL_GPL(__ide_abort);
637
638 /**
639  *      ide_abort       -       abort pending IDE operations
640  *      @drive: drive the error occurred on
641  *      @msg: message to report
642  *
643  *      ide_abort kills and cleans up when we are about to do a 
644  *      host initiated reset on active commands. Longer term we
645  *      want handlers to have sensible abort handling themselves
646  *
647  *      This differs fundamentally from ide_error because in 
648  *      this case the command is doing just fine when we
649  *      blow it away.
650  */
651  
652 ide_startstop_t ide_abort(ide_drive_t *drive, const char *msg)
653 {
654         struct request *rq;
655
656         if (drive == NULL || (rq = HWGROUP(drive)->rq) == NULL)
657                 return ide_stopped;
658
659         /* retry only "normal" I/O: */
660         if (!blk_fs_request(rq)) {
661                 rq->errors = 1;
662                 ide_end_drive_cmd(drive, BUSY_STAT, 0);
663                 return ide_stopped;
664         }
665
666         if (rq->rq_disk) {
667                 ide_driver_t *drv;
668
669                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
670                 return drv->abort(drive, rq);
671         } else
672                 return __ide_abort(drive, rq);
673 }
674
675 /**
676  *      ide_cmd         -       issue a simple drive command
677  *      @drive: drive the command is for
678  *      @cmd: command byte
679  *      @nsect: sector byte
680  *      @handler: handler for the command completion
681  *
682  *      Issue a simple drive command with interrupts.
683  *      The drive must be selected beforehand.
684  */
685
686 static void ide_cmd (ide_drive_t *drive, u8 cmd, u8 nsect,
687                 ide_handler_t *handler)
688 {
689         ide_hwif_t *hwif = HWIF(drive);
690         if (IDE_CONTROL_REG)
691                 hwif->OUTB(drive->ctl,IDE_CONTROL_REG); /* clear nIEN */
692         SELECT_MASK(drive,0);
693         hwif->OUTB(nsect,IDE_NSECTOR_REG);
694         ide_execute_command(drive, cmd, handler, WAIT_CMD, NULL);
695 }
696
697 /**
698  *      drive_cmd_intr          -       drive command completion interrupt
699  *      @drive: drive the completion interrupt occurred on
700  *
701  *      drive_cmd_intr() is invoked on completion of a special DRIVE_CMD.
702  *      We do any necessary data reading and then wait for the drive to
703  *      go non busy. At that point we may read the error data and complete
704  *      the request
705  */
706  
707 static ide_startstop_t drive_cmd_intr (ide_drive_t *drive)
708 {
709         struct request *rq = HWGROUP(drive)->rq;
710         ide_hwif_t *hwif = HWIF(drive);
711         u8 *args = (u8 *) rq->buffer;
712         u8 stat = hwif->INB(IDE_STATUS_REG);
713         int retries = 10;
714
715         local_irq_enable_in_hardirq();
716         if ((stat & DRQ_STAT) && args && args[3]) {
717                 u8 io_32bit = drive->io_32bit;
718                 drive->io_32bit = 0;
719                 hwif->ata_input_data(drive, &args[4], args[3] * SECTOR_WORDS);
720                 drive->io_32bit = io_32bit;
721                 while (((stat = hwif->INB(IDE_STATUS_REG)) & BUSY_STAT) && retries--)
722                         udelay(100);
723         }
724
725         if (!OK_STAT(stat, READY_STAT, BAD_STAT))
726                 return ide_error(drive, "drive_cmd", stat);
727                 /* calls ide_end_drive_cmd */
728         ide_end_drive_cmd(drive, stat, hwif->INB(IDE_ERROR_REG));
729         return ide_stopped;
730 }
731
732 static void ide_init_specify_cmd(ide_drive_t *drive, ide_task_t *task)
733 {
734         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->sect;
735         task->tfRegister[IDE_SECTOR_OFFSET]  = drive->sect;
736         task->tfRegister[IDE_LCYL_OFFSET]    = drive->cyl;
737         task->tfRegister[IDE_HCYL_OFFSET]    = drive->cyl>>8;
738         task->tfRegister[IDE_SELECT_OFFSET]  = ((drive->head-1)|drive->select.all)&0xBF;
739         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_SPECIFY;
740
741         task->handler = &set_geometry_intr;
742 }
743
744 static void ide_init_restore_cmd(ide_drive_t *drive, ide_task_t *task)
745 {
746         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->sect;
747         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_RESTORE;
748
749         task->handler = &recal_intr;
750 }
751
752 static void ide_init_setmult_cmd(ide_drive_t *drive, ide_task_t *task)
753 {
754         task->tfRegister[IDE_NSECTOR_OFFSET] = drive->mult_req;
755         task->tfRegister[IDE_COMMAND_OFFSET] = WIN_SETMULT;
756
757         task->handler = &set_multmode_intr;
758 }
759
760 static ide_startstop_t ide_disk_special(ide_drive_t *drive)
761 {
762         special_t *s = &drive->special;
763         ide_task_t args;
764
765         memset(&args, 0, sizeof(ide_task_t));
766         args.command_type = IDE_DRIVE_TASK_NO_DATA;
767
768         if (s->b.set_geometry) {
769                 s->b.set_geometry = 0;
770                 ide_init_specify_cmd(drive, &args);
771         } else if (s->b.recalibrate) {
772                 s->b.recalibrate = 0;
773                 ide_init_restore_cmd(drive, &args);
774         } else if (s->b.set_multmode) {
775                 s->b.set_multmode = 0;
776                 if (drive->mult_req > drive->id->max_multsect)
777                         drive->mult_req = drive->id->max_multsect;
778                 ide_init_setmult_cmd(drive, &args);
779         } else if (s->all) {
780                 int special = s->all;
781                 s->all = 0;
782                 printk(KERN_ERR "%s: bad special flag: 0x%02x\n", drive->name, special);
783                 return ide_stopped;
784         }
785
786         do_rw_taskfile(drive, &args);
787
788         return ide_started;
789 }
790
791 /**
792  *      do_special              -       issue some special commands
793  *      @drive: drive the command is for
794  *
795  *      do_special() is used to issue WIN_SPECIFY, WIN_RESTORE, and WIN_SETMULT
796  *      commands to a drive.  It used to do much more, but has been scaled
797  *      back.
798  */
799
800 static ide_startstop_t do_special (ide_drive_t *drive)
801 {
802         special_t *s = &drive->special;
803
804 #ifdef DEBUG
805         printk("%s: do_special: 0x%02x\n", drive->name, s->all);
806 #endif
807         if (s->b.set_tune) {
808                 s->b.set_tune = 0;
809                 if (HWIF(drive)->tuneproc != NULL)
810                         HWIF(drive)->tuneproc(drive, drive->tune_req);
811                 return ide_stopped;
812         } else {
813                 if (drive->media == ide_disk)
814                         return ide_disk_special(drive);
815
816                 s->all = 0;
817                 drive->mult_req = 0;
818                 return ide_stopped;
819         }
820 }
821
822 void ide_map_sg(ide_drive_t *drive, struct request *rq)
823 {
824         ide_hwif_t *hwif = drive->hwif;
825         struct scatterlist *sg = hwif->sg_table;
826
827         if (hwif->sg_mapped)    /* needed by ide-scsi */
828                 return;
829
830         if (rq->cmd_type != REQ_TYPE_ATA_TASKFILE) {
831                 hwif->sg_nents = blk_rq_map_sg(drive->queue, rq, sg);
832         } else {
833                 sg_init_one(sg, rq->buffer, rq->nr_sectors * SECTOR_SIZE);
834                 hwif->sg_nents = 1;
835         }
836 }
837
838 EXPORT_SYMBOL_GPL(ide_map_sg);
839
840 void ide_init_sg_cmd(ide_drive_t *drive, struct request *rq)
841 {
842         ide_hwif_t *hwif = drive->hwif;
843
844         hwif->nsect = hwif->nleft = rq->nr_sectors;
845         hwif->cursg = hwif->cursg_ofs = 0;
846 }
847
848 EXPORT_SYMBOL_GPL(ide_init_sg_cmd);
849
850 /**
851  *      execute_drive_command   -       issue special drive command
852  *      @drive: the drive to issue the command on
853  *      @rq: the request structure holding the command
854  *
855  *      execute_drive_cmd() issues a special drive command,  usually 
856  *      initiated by ioctl() from the external hdparm program. The
857  *      command can be a drive command, drive task or taskfile 
858  *      operation. Weirdly you can call it with NULL to wait for
859  *      all commands to finish. Don't do this as that is due to change
860  */
861
862 static ide_startstop_t execute_drive_cmd (ide_drive_t *drive,
863                 struct request *rq)
864 {
865         ide_hwif_t *hwif = HWIF(drive);
866         if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE) {
867                 ide_task_t *args = rq->special;
868  
869                 if (!args)
870                         goto done;
871
872                 hwif->data_phase = args->data_phase;
873
874                 switch (hwif->data_phase) {
875                 case TASKFILE_MULTI_OUT:
876                 case TASKFILE_OUT:
877                 case TASKFILE_MULTI_IN:
878                 case TASKFILE_IN:
879                         ide_init_sg_cmd(drive, rq);
880                         ide_map_sg(drive, rq);
881                 default:
882                         break;
883                 }
884
885                 if (args->tf_out_flags.all != 0) 
886                         return flagged_taskfile(drive, args);
887                 return do_rw_taskfile(drive, args);
888         } else if (rq->cmd_type == REQ_TYPE_ATA_TASK) {
889                 u8 *args = rq->buffer;
890                 u8 sel;
891  
892                 if (!args)
893                         goto done;
894 #ifdef DEBUG
895                 printk("%s: DRIVE_TASK_CMD ", drive->name);
896                 printk("cmd=0x%02x ", args[0]);
897                 printk("fr=0x%02x ", args[1]);
898                 printk("ns=0x%02x ", args[2]);
899                 printk("sc=0x%02x ", args[3]);
900                 printk("lcyl=0x%02x ", args[4]);
901                 printk("hcyl=0x%02x ", args[5]);
902                 printk("sel=0x%02x\n", args[6]);
903 #endif
904                 hwif->OUTB(args[1], IDE_FEATURE_REG);
905                 hwif->OUTB(args[3], IDE_SECTOR_REG);
906                 hwif->OUTB(args[4], IDE_LCYL_REG);
907                 hwif->OUTB(args[5], IDE_HCYL_REG);
908                 sel = (args[6] & ~0x10);
909                 if (drive->select.b.unit)
910                         sel |= 0x10;
911                 hwif->OUTB(sel, IDE_SELECT_REG);
912                 ide_cmd(drive, args[0], args[2], &drive_cmd_intr);
913                 return ide_started;
914         } else if (rq->cmd_type == REQ_TYPE_ATA_CMD) {
915                 u8 *args = rq->buffer;
916
917                 if (!args)
918                         goto done;
919 #ifdef DEBUG
920                 printk("%s: DRIVE_CMD ", drive->name);
921                 printk("cmd=0x%02x ", args[0]);
922                 printk("sc=0x%02x ", args[1]);
923                 printk("fr=0x%02x ", args[2]);
924                 printk("xx=0x%02x\n", args[3]);
925 #endif
926                 if (args[0] == WIN_SMART) {
927                         hwif->OUTB(0x4f, IDE_LCYL_REG);
928                         hwif->OUTB(0xc2, IDE_HCYL_REG);
929                         hwif->OUTB(args[2],IDE_FEATURE_REG);
930                         hwif->OUTB(args[1],IDE_SECTOR_REG);
931                         ide_cmd(drive, args[0], args[3], &drive_cmd_intr);
932                         return ide_started;
933                 }
934                 hwif->OUTB(args[2],IDE_FEATURE_REG);
935                 ide_cmd(drive, args[0], args[1], &drive_cmd_intr);
936                 return ide_started;
937         }
938
939 done:
940         /*
941          * NULL is actually a valid way of waiting for
942          * all current requests to be flushed from the queue.
943          */
944 #ifdef DEBUG
945         printk("%s: DRIVE_CMD (null)\n", drive->name);
946 #endif
947         ide_end_drive_cmd(drive,
948                         hwif->INB(IDE_STATUS_REG),
949                         hwif->INB(IDE_ERROR_REG));
950         return ide_stopped;
951 }
952
953 static void ide_check_pm_state(ide_drive_t *drive, struct request *rq)
954 {
955         struct request_pm_state *pm = rq->data;
956
957         if (blk_pm_suspend_request(rq) &&
958             pm->pm_step == ide_pm_state_start_suspend)
959                 /* Mark drive blocked when starting the suspend sequence. */
960                 drive->blocked = 1;
961         else if (blk_pm_resume_request(rq) &&
962                  pm->pm_step == ide_pm_state_start_resume) {
963                 /* 
964                  * The first thing we do on wakeup is to wait for BSY bit to
965                  * go away (with a looong timeout) as a drive on this hwif may
966                  * just be POSTing itself.
967                  * We do that before even selecting as the "other" device on
968                  * the bus may be broken enough to walk on our toes at this
969                  * point.
970                  */
971                 int rc;
972 #ifdef DEBUG_PM
973                 printk("%s: Wakeup request inited, waiting for !BSY...\n", drive->name);
974 #endif
975                 rc = ide_wait_not_busy(HWIF(drive), 35000);
976                 if (rc)
977                         printk(KERN_WARNING "%s: bus not ready on wakeup\n", drive->name);
978                 SELECT_DRIVE(drive);
979                 HWIF(drive)->OUTB(8, HWIF(drive)->io_ports[IDE_CONTROL_OFFSET]);
980                 rc = ide_wait_not_busy(HWIF(drive), 100000);
981                 if (rc)
982                         printk(KERN_WARNING "%s: drive not ready on wakeup\n", drive->name);
983         }
984 }
985
986 /**
987  *      start_request   -       start of I/O and command issuing for IDE
988  *
989  *      start_request() initiates handling of a new I/O request. It
990  *      accepts commands and I/O (read/write) requests. It also does
991  *      the final remapping for weird stuff like EZDrive. Once 
992  *      device mapper can work sector level the EZDrive stuff can go away
993  *
994  *      FIXME: this function needs a rename
995  */
996  
997 static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
998 {
999         ide_startstop_t startstop;
1000         sector_t block;
1001
1002         BUG_ON(!blk_rq_started(rq));
1003
1004 #ifdef DEBUG
1005         printk("%s: start_request: current=0x%08lx\n",
1006                 HWIF(drive)->name, (unsigned long) rq);
1007 #endif
1008
1009         /* bail early if we've exceeded max_failures */
1010         if (drive->max_failures && (drive->failures > drive->max_failures)) {
1011                 goto kill_rq;
1012         }
1013
1014         block    = rq->sector;
1015         if (blk_fs_request(rq) &&
1016             (drive->media == ide_disk || drive->media == ide_floppy)) {
1017                 block += drive->sect0;
1018         }
1019         /* Yecch - this will shift the entire interval,
1020            possibly killing some innocent following sector */
1021         if (block == 0 && drive->remap_0_to_1 == 1)
1022                 block = 1;  /* redirect MBR access to EZ-Drive partn table */
1023
1024         if (blk_pm_request(rq))
1025                 ide_check_pm_state(drive, rq);
1026
1027         SELECT_DRIVE(drive);
1028         if (ide_wait_stat(&startstop, drive, drive->ready_stat, BUSY_STAT|DRQ_STAT, WAIT_READY)) {
1029                 printk(KERN_ERR "%s: drive not ready for command\n", drive->name);
1030                 return startstop;
1031         }
1032         if (!drive->special.all) {
1033                 ide_driver_t *drv;
1034
1035                 /*
1036                  * We reset the drive so we need to issue a SETFEATURES.
1037                  * Do it _after_ do_special() restored device parameters.
1038                  */
1039                 if (drive->current_speed == 0xff)
1040                         ide_config_drive_speed(drive, drive->desired_speed);
1041
1042                 if (rq->cmd_type == REQ_TYPE_ATA_CMD ||
1043                     rq->cmd_type == REQ_TYPE_ATA_TASK ||
1044                     rq->cmd_type == REQ_TYPE_ATA_TASKFILE)
1045                         return execute_drive_cmd(drive, rq);
1046                 else if (blk_pm_request(rq)) {
1047                         struct request_pm_state *pm = rq->data;
1048 #ifdef DEBUG_PM
1049                         printk("%s: start_power_step(step: %d)\n",
1050                                 drive->name, rq->pm->pm_step);
1051 #endif
1052                         startstop = ide_start_power_step(drive, rq);
1053                         if (startstop == ide_stopped &&
1054                             pm->pm_step == ide_pm_state_completed)
1055                                 ide_complete_pm_request(drive, rq);
1056                         return startstop;
1057                 }
1058
1059                 drv = *(ide_driver_t **)rq->rq_disk->private_data;
1060                 return drv->do_request(drive, rq, block);
1061         }
1062         return do_special(drive);
1063 kill_rq:
1064         ide_kill_rq(drive, rq);
1065         return ide_stopped;
1066 }
1067
1068 /**
1069  *      ide_stall_queue         -       pause an IDE device
1070  *      @drive: drive to stall
1071  *      @timeout: time to stall for (jiffies)
1072  *
1073  *      ide_stall_queue() can be used by a drive to give excess bandwidth back
1074  *      to the hwgroup by sleeping for timeout jiffies.
1075  */
1076  
1077 void ide_stall_queue (ide_drive_t *drive, unsigned long timeout)
1078 {
1079         if (timeout > WAIT_WORSTCASE)
1080                 timeout = WAIT_WORSTCASE;
1081         drive->sleep = timeout + jiffies;
1082         drive->sleeping = 1;
1083 }
1084
1085 EXPORT_SYMBOL(ide_stall_queue);
1086
1087 #define WAKEUP(drive)   ((drive)->service_start + 2 * (drive)->service_time)
1088
1089 /**
1090  *      choose_drive            -       select a drive to service
1091  *      @hwgroup: hardware group to select on
1092  *
1093  *      choose_drive() selects the next drive which will be serviced.
1094  *      This is necessary because the IDE layer can't issue commands
1095  *      to both drives on the same cable, unlike SCSI.
1096  */
1097  
1098 static inline ide_drive_t *choose_drive (ide_hwgroup_t *hwgroup)
1099 {
1100         ide_drive_t *drive, *best;
1101
1102 repeat: 
1103         best = NULL;
1104         drive = hwgroup->drive;
1105
1106         /*
1107          * drive is doing pre-flush, ordered write, post-flush sequence. even
1108          * though that is 3 requests, it must be seen as a single transaction.
1109          * we must not preempt this drive until that is complete
1110          */
1111         if (blk_queue_flushing(drive->queue)) {
1112                 /*
1113                  * small race where queue could get replugged during
1114                  * the 3-request flush cycle, just yank the plug since
1115                  * we want it to finish asap
1116                  */
1117                 blk_remove_plug(drive->queue);
1118                 return drive;
1119         }
1120
1121         do {
1122                 if ((!drive->sleeping || time_after_eq(jiffies, drive->sleep))
1123                     && !elv_queue_empty(drive->queue)) {
1124                         if (!best
1125                          || (drive->sleeping && (!best->sleeping || time_before(drive->sleep, best->sleep)))
1126                          || (!best->sleeping && time_before(WAKEUP(drive), WAKEUP(best))))
1127                         {
1128                                 if (!blk_queue_plugged(drive->queue))
1129                                         best = drive;
1130                         }
1131                 }
1132         } while ((drive = drive->next) != hwgroup->drive);
1133         if (best && best->nice1 && !best->sleeping && best != hwgroup->drive && best->service_time > WAIT_MIN_SLEEP) {
1134                 long t = (signed long)(WAKEUP(best) - jiffies);
1135                 if (t >= WAIT_MIN_SLEEP) {
1136                 /*
1137                  * We *may* have some time to spare, but first let's see if
1138                  * someone can potentially benefit from our nice mood today..
1139                  */
1140                         drive = best->next;
1141                         do {
1142                                 if (!drive->sleeping
1143                                  && time_before(jiffies - best->service_time, WAKEUP(drive))
1144                                  && time_before(WAKEUP(drive), jiffies + t))
1145                                 {
1146                                         ide_stall_queue(best, min_t(long, t, 10 * WAIT_MIN_SLEEP));
1147                                         goto repeat;
1148                                 }
1149                         } while ((drive = drive->next) != best);
1150                 }
1151         }
1152         return best;
1153 }
1154
1155 /*
1156  * Issue a new request to a drive from hwgroup
1157  * Caller must have already done spin_lock_irqsave(&ide_lock, ..);
1158  *
1159  * A hwgroup is a serialized group of IDE interfaces.  Usually there is
1160  * exactly one hwif (interface) per hwgroup, but buggy controllers (eg. CMD640)
1161  * may have both interfaces in a single hwgroup to "serialize" access.
1162  * Or possibly multiple ISA interfaces can share a common IRQ by being grouped
1163  * together into one hwgroup for serialized access.
1164  *
1165  * Note also that several hwgroups can end up sharing a single IRQ,
1166  * possibly along with many other devices.  This is especially common in
1167  * PCI-based systems with off-board IDE controller cards.
1168  *
1169  * The IDE driver uses the single global ide_lock spinlock to protect
1170  * access to the request queues, and to protect the hwgroup->busy flag.
1171  *
1172  * The first thread into the driver for a particular hwgroup sets the
1173  * hwgroup->busy flag to indicate that this hwgroup is now active,
1174  * and then initiates processing of the top request from the request queue.
1175  *
1176  * Other threads attempting entry notice the busy setting, and will simply
1177  * queue their new requests and exit immediately.  Note that hwgroup->busy
1178  * remains set even when the driver is merely awaiting the next interrupt.
1179  * Thus, the meaning is "this hwgroup is busy processing a request".
1180  *
1181  * When processing of a request completes, the completing thread or IRQ-handler
1182  * will start the next request from the queue.  If no more work remains,
1183  * the driver will clear the hwgroup->busy flag and exit.
1184  *
1185  * The ide_lock (spinlock) is used to protect all access to the
1186  * hwgroup->busy flag, but is otherwise not needed for most processing in
1187  * the driver.  This makes the driver much more friendlier to shared IRQs
1188  * than previous designs, while remaining 100% (?) SMP safe and capable.
1189  */
1190 static void ide_do_request (ide_hwgroup_t *hwgroup, int masked_irq)
1191 {
1192         ide_drive_t     *drive;
1193         ide_hwif_t      *hwif;
1194         struct request  *rq;
1195         ide_startstop_t startstop;
1196         int             loops = 0;
1197
1198         /* for atari only: POSSIBLY BROKEN HERE(?) */
1199         ide_get_lock(ide_intr, hwgroup);
1200
1201         /* caller must own ide_lock */
1202         BUG_ON(!irqs_disabled());
1203
1204         while (!hwgroup->busy) {
1205                 hwgroup->busy = 1;
1206                 drive = choose_drive(hwgroup);
1207                 if (drive == NULL) {
1208                         int sleeping = 0;
1209                         unsigned long sleep = 0; /* shut up, gcc */
1210                         hwgroup->rq = NULL;
1211                         drive = hwgroup->drive;
1212                         do {
1213                                 if (drive->sleeping && (!sleeping || time_before(drive->sleep, sleep))) {
1214                                         sleeping = 1;
1215                                         sleep = drive->sleep;
1216                                 }
1217                         } while ((drive = drive->next) != hwgroup->drive);
1218                         if (sleeping) {
1219                 /*
1220                  * Take a short snooze, and then wake up this hwgroup again.
1221                  * This gives other hwgroups on the same a chance to
1222                  * play fairly with us, just in case there are big differences
1223                  * in relative throughputs.. don't want to hog the cpu too much.
1224                  */
1225                                 if (time_before(sleep, jiffies + WAIT_MIN_SLEEP))
1226                                         sleep = jiffies + WAIT_MIN_SLEEP;
1227 #if 1
1228                                 if (timer_pending(&hwgroup->timer))
1229                                         printk(KERN_CRIT "ide_set_handler: timer already active\n");
1230 #endif
1231                                 /* so that ide_timer_expiry knows what to do */
1232                                 hwgroup->sleeping = 1;
1233                                 hwgroup->req_gen_timer = hwgroup->req_gen;
1234                                 mod_timer(&hwgroup->timer, sleep);
1235                                 /* we purposely leave hwgroup->busy==1
1236                                  * while sleeping */
1237                         } else {
1238                                 /* Ugly, but how can we sleep for the lock
1239                                  * otherwise? perhaps from tq_disk?
1240                                  */
1241
1242                                 /* for atari only */
1243                                 ide_release_lock();
1244                                 hwgroup->busy = 0;
1245                         }
1246
1247                         /* no more work for this hwgroup (for now) */
1248                         return;
1249                 }
1250         again:
1251                 hwif = HWIF(drive);
1252                 if (hwgroup->hwif->sharing_irq &&
1253                     hwif != hwgroup->hwif &&
1254                     hwif->io_ports[IDE_CONTROL_OFFSET]) {
1255                         /* set nIEN for previous hwif */
1256                         SELECT_INTERRUPT(drive);
1257                 }
1258                 hwgroup->hwif = hwif;
1259                 hwgroup->drive = drive;
1260                 drive->sleeping = 0;
1261                 drive->service_start = jiffies;
1262
1263                 if (blk_queue_plugged(drive->queue)) {
1264                         printk(KERN_ERR "ide: huh? queue was plugged!\n");
1265                         break;
1266                 }
1267
1268                 /*
1269                  * we know that the queue isn't empty, but this can happen
1270                  * if the q->prep_rq_fn() decides to kill a request
1271                  */
1272                 rq = elv_next_request(drive->queue);
1273                 if (!rq) {
1274                         hwgroup->busy = 0;
1275                         break;
1276                 }
1277
1278                 /*
1279                  * Sanity: don't accept a request that isn't a PM request
1280                  * if we are currently power managed. This is very important as
1281                  * blk_stop_queue() doesn't prevent the elv_next_request()
1282                  * above to return us whatever is in the queue. Since we call
1283                  * ide_do_request() ourselves, we end up taking requests while
1284                  * the queue is blocked...
1285                  * 
1286                  * We let requests forced at head of queue with ide-preempt
1287                  * though. I hope that doesn't happen too much, hopefully not
1288                  * unless the subdriver triggers such a thing in its own PM
1289                  * state machine.
1290                  *
1291                  * We count how many times we loop here to make sure we service
1292                  * all drives in the hwgroup without looping for ever
1293                  */
1294                 if (drive->blocked && !blk_pm_request(rq) && !(rq->cmd_flags & REQ_PREEMPT)) {
1295                         drive = drive->next ? drive->next : hwgroup->drive;
1296                         if (loops++ < 4 && !blk_queue_plugged(drive->queue))
1297                                 goto again;
1298                         /* We clear busy, there should be no pending ATA command at this point. */
1299                         hwgroup->busy = 0;
1300                         break;
1301                 }
1302
1303                 hwgroup->rq = rq;
1304
1305                 /*
1306                  * Some systems have trouble with IDE IRQs arriving while
1307                  * the driver is still setting things up.  So, here we disable
1308                  * the IRQ used by this interface while the request is being started.
1309                  * This may look bad at first, but pretty much the same thing
1310                  * happens anyway when any interrupt comes in, IDE or otherwise
1311                  *  -- the kernel masks the IRQ while it is being handled.
1312                  */
1313                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1314                         disable_irq_nosync(hwif->irq);
1315                 spin_unlock(&ide_lock);
1316                 local_irq_enable_in_hardirq();
1317                         /* allow other IRQs while we start this request */
1318                 startstop = start_request(drive, rq);
1319                 spin_lock_irq(&ide_lock);
1320                 if (masked_irq != IDE_NO_IRQ && hwif->irq != masked_irq)
1321                         enable_irq(hwif->irq);
1322                 if (startstop == ide_stopped)
1323                         hwgroup->busy = 0;
1324         }
1325 }
1326
1327 /*
1328  * Passes the stuff to ide_do_request
1329  */
1330 void do_ide_request(request_queue_t *q)
1331 {
1332         ide_drive_t *drive = q->queuedata;
1333
1334         ide_do_request(HWGROUP(drive), IDE_NO_IRQ);
1335 }
1336
1337 /*
1338  * un-busy the hwgroup etc, and clear any pending DMA status. we want to
1339  * retry the current request in pio mode instead of risking tossing it
1340  * all away
1341  */
1342 static ide_startstop_t ide_dma_timeout_retry(ide_drive_t *drive, int error)
1343 {
1344         ide_hwif_t *hwif = HWIF(drive);
1345         struct request *rq;
1346         ide_startstop_t ret = ide_stopped;
1347
1348         /*
1349          * end current dma transaction
1350          */
1351
1352         if (error < 0) {
1353                 printk(KERN_WARNING "%s: DMA timeout error\n", drive->name);
1354                 (void)HWIF(drive)->ide_dma_end(drive);
1355                 ret = ide_error(drive, "dma timeout error",
1356                                                 hwif->INB(IDE_STATUS_REG));
1357         } else {
1358                 printk(KERN_WARNING "%s: DMA timeout retry\n", drive->name);
1359                 hwif->dma_timeout(drive);
1360         }
1361
1362         /*
1363          * disable dma for now, but remember that we did so because of
1364          * a timeout -- we'll reenable after we finish this next request
1365          * (or rather the first chunk of it) in pio.
1366          */
1367         drive->retry_pio++;
1368         drive->state = DMA_PIO_RETRY;
1369         hwif->dma_off_quietly(drive);
1370
1371         /*
1372          * un-busy drive etc (hwgroup->busy is cleared on return) and
1373          * make sure request is sane
1374          */
1375         rq = HWGROUP(drive)->rq;
1376
1377         if (!rq)
1378                 goto out;
1379
1380         HWGROUP(drive)->rq = NULL;
1381
1382         rq->errors = 0;
1383
1384         if (!rq->bio)
1385                 goto out;
1386
1387         rq->sector = rq->bio->bi_sector;
1388         rq->current_nr_sectors = bio_iovec(rq->bio)->bv_len >> 9;
1389         rq->hard_cur_sectors = rq->current_nr_sectors;
1390         rq->buffer = bio_data(rq->bio);
1391 out:
1392         return ret;
1393 }
1394
1395 /**
1396  *      ide_timer_expiry        -       handle lack of an IDE interrupt
1397  *      @data: timer callback magic (hwgroup)
1398  *
1399  *      An IDE command has timed out before the expected drive return
1400  *      occurred. At this point we attempt to clean up the current
1401  *      mess. If the current handler includes an expiry handler then
1402  *      we invoke the expiry handler, and providing it is happy the
1403  *      work is done. If that fails we apply generic recovery rules
1404  *      invoking the handler and checking the drive DMA status. We
1405  *      have an excessively incestuous relationship with the DMA
1406  *      logic that wants cleaning up.
1407  */
1408  
1409 void ide_timer_expiry (unsigned long data)
1410 {
1411         ide_hwgroup_t   *hwgroup = (ide_hwgroup_t *) data;
1412         ide_handler_t   *handler;
1413         ide_expiry_t    *expiry;
1414         unsigned long   flags;
1415         unsigned long   wait = -1;
1416
1417         spin_lock_irqsave(&ide_lock, flags);
1418
1419         if (((handler = hwgroup->handler) == NULL) ||
1420             (hwgroup->req_gen != hwgroup->req_gen_timer)) {
1421                 /*
1422                  * Either a marginal timeout occurred
1423                  * (got the interrupt just as timer expired),
1424                  * or we were "sleeping" to give other devices a chance.
1425                  * Either way, we don't really want to complain about anything.
1426                  */
1427                 if (hwgroup->sleeping) {
1428                         hwgroup->sleeping = 0;
1429                         hwgroup->busy = 0;
1430                 }
1431         } else {
1432                 ide_drive_t *drive = hwgroup->drive;
1433                 if (!drive) {
1434                         printk(KERN_ERR "ide_timer_expiry: hwgroup->drive was NULL\n");
1435                         hwgroup->handler = NULL;
1436                 } else {
1437                         ide_hwif_t *hwif;
1438                         ide_startstop_t startstop = ide_stopped;
1439                         if (!hwgroup->busy) {
1440                                 hwgroup->busy = 1;      /* paranoia */
1441                                 printk(KERN_ERR "%s: ide_timer_expiry: hwgroup->busy was 0 ??\n", drive->name);
1442                         }
1443                         if ((expiry = hwgroup->expiry) != NULL) {
1444                                 /* continue */
1445                                 if ((wait = expiry(drive)) > 0) {
1446                                         /* reset timer */
1447                                         hwgroup->timer.expires  = jiffies + wait;
1448                                         hwgroup->req_gen_timer = hwgroup->req_gen;
1449                                         add_timer(&hwgroup->timer);
1450                                         spin_unlock_irqrestore(&ide_lock, flags);
1451                                         return;
1452                                 }
1453                         }
1454                         hwgroup->handler = NULL;
1455                         /*
1456                          * We need to simulate a real interrupt when invoking
1457                          * the handler() function, which means we need to
1458                          * globally mask the specific IRQ:
1459                          */
1460                         spin_unlock(&ide_lock);
1461                         hwif  = HWIF(drive);
1462 #if DISABLE_IRQ_NOSYNC
1463                         disable_irq_nosync(hwif->irq);
1464 #else
1465                         /* disable_irq_nosync ?? */
1466                         disable_irq(hwif->irq);
1467 #endif /* DISABLE_IRQ_NOSYNC */
1468                         /* local CPU only,
1469                          * as if we were handling an interrupt */
1470                         local_irq_disable();
1471                         if (hwgroup->polling) {
1472                                 startstop = handler(drive);
1473                         } else if (drive_is_ready(drive)) {
1474                                 if (drive->waiting_for_dma)
1475                                         hwgroup->hwif->dma_lost_irq(drive);
1476                                 (void)ide_ack_intr(hwif);
1477                                 printk(KERN_WARNING "%s: lost interrupt\n", drive->name);
1478                                 startstop = handler(drive);
1479                         } else {
1480                                 if (drive->waiting_for_dma) {
1481                                         startstop = ide_dma_timeout_retry(drive, wait);
1482                                 } else
1483                                         startstop =
1484                                         ide_error(drive, "irq timeout", hwif->INB(IDE_STATUS_REG));
1485                         }
1486                         drive->service_time = jiffies - drive->service_start;
1487                         spin_lock_irq(&ide_lock);
1488                         enable_irq(hwif->irq);
1489                         if (startstop == ide_stopped)
1490                                 hwgroup->busy = 0;
1491                 }
1492         }
1493         ide_do_request(hwgroup, IDE_NO_IRQ);
1494         spin_unlock_irqrestore(&ide_lock, flags);
1495 }
1496
1497 /**
1498  *      unexpected_intr         -       handle an unexpected IDE interrupt
1499  *      @irq: interrupt line
1500  *      @hwgroup: hwgroup being processed
1501  *
1502  *      There's nothing really useful we can do with an unexpected interrupt,
1503  *      other than reading the status register (to clear it), and logging it.
1504  *      There should be no way that an irq can happen before we're ready for it,
1505  *      so we needn't worry much about losing an "important" interrupt here.
1506  *
1507  *      On laptops (and "green" PCs), an unexpected interrupt occurs whenever
1508  *      the drive enters "idle", "standby", or "sleep" mode, so if the status
1509  *      looks "good", we just ignore the interrupt completely.
1510  *
1511  *      This routine assumes __cli() is in effect when called.
1512  *
1513  *      If an unexpected interrupt happens on irq15 while we are handling irq14
1514  *      and if the two interfaces are "serialized" (CMD640), then it looks like
1515  *      we could screw up by interfering with a new request being set up for 
1516  *      irq15.
1517  *
1518  *      In reality, this is a non-issue.  The new command is not sent unless 
1519  *      the drive is ready to accept one, in which case we know the drive is
1520  *      not trying to interrupt us.  And ide_set_handler() is always invoked
1521  *      before completing the issuance of any new drive command, so we will not
1522  *      be accidentally invoked as a result of any valid command completion
1523  *      interrupt.
1524  *
1525  *      Note that we must walk the entire hwgroup here. We know which hwif
1526  *      is doing the current command, but we don't know which hwif burped
1527  *      mysteriously.
1528  */
1529  
1530 static void unexpected_intr (int irq, ide_hwgroup_t *hwgroup)
1531 {
1532         u8 stat;
1533         ide_hwif_t *hwif = hwgroup->hwif;
1534
1535         /*
1536          * handle the unexpected interrupt
1537          */
1538         do {
1539                 if (hwif->irq == irq) {
1540                         stat = hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1541                         if (!OK_STAT(stat, READY_STAT, BAD_STAT)) {
1542                                 /* Try to not flood the console with msgs */
1543                                 static unsigned long last_msgtime, count;
1544                                 ++count;
1545                                 if (time_after(jiffies, last_msgtime + HZ)) {
1546                                         last_msgtime = jiffies;
1547                                         printk(KERN_ERR "%s%s: unexpected interrupt, "
1548                                                 "status=0x%02x, count=%ld\n",
1549                                                 hwif->name,
1550                                                 (hwif->next==hwgroup->hwif) ? "" : "(?)", stat, count);
1551                                 }
1552                         }
1553                 }
1554         } while ((hwif = hwif->next) != hwgroup->hwif);
1555 }
1556
1557 /**
1558  *      ide_intr        -       default IDE interrupt handler
1559  *      @irq: interrupt number
1560  *      @dev_id: hwif group
1561  *      @regs: unused weirdness from the kernel irq layer
1562  *
1563  *      This is the default IRQ handler for the IDE layer. You should
1564  *      not need to override it. If you do be aware it is subtle in
1565  *      places
1566  *
1567  *      hwgroup->hwif is the interface in the group currently performing
1568  *      a command. hwgroup->drive is the drive and hwgroup->handler is
1569  *      the IRQ handler to call. As we issue a command the handlers
1570  *      step through multiple states, reassigning the handler to the
1571  *      next step in the process. Unlike a smart SCSI controller IDE
1572  *      expects the main processor to sequence the various transfer
1573  *      stages. We also manage a poll timer to catch up with most
1574  *      timeout situations. There are still a few where the handlers
1575  *      don't ever decide to give up.
1576  *
1577  *      The handler eventually returns ide_stopped to indicate the
1578  *      request completed. At this point we issue the next request
1579  *      on the hwgroup and the process begins again.
1580  */
1581  
1582 irqreturn_t ide_intr (int irq, void *dev_id)
1583 {
1584         unsigned long flags;
1585         ide_hwgroup_t *hwgroup = (ide_hwgroup_t *)dev_id;
1586         ide_hwif_t *hwif;
1587         ide_drive_t *drive;
1588         ide_handler_t *handler;
1589         ide_startstop_t startstop;
1590
1591         spin_lock_irqsave(&ide_lock, flags);
1592         hwif = hwgroup->hwif;
1593
1594         if (!ide_ack_intr(hwif)) {
1595                 spin_unlock_irqrestore(&ide_lock, flags);
1596                 return IRQ_NONE;
1597         }
1598
1599         if ((handler = hwgroup->handler) == NULL || hwgroup->polling) {
1600                 /*
1601                  * Not expecting an interrupt from this drive.
1602                  * That means this could be:
1603                  *      (1) an interrupt from another PCI device
1604                  *      sharing the same PCI INT# as us.
1605                  * or   (2) a drive just entered sleep or standby mode,
1606                  *      and is interrupting to let us know.
1607                  * or   (3) a spurious interrupt of unknown origin.
1608                  *
1609                  * For PCI, we cannot tell the difference,
1610                  * so in that case we just ignore it and hope it goes away.
1611                  *
1612                  * FIXME: unexpected_intr should be hwif-> then we can
1613                  * remove all the ifdef PCI crap
1614                  */
1615 #ifdef CONFIG_BLK_DEV_IDEPCI
1616                 if (hwif->pci_dev && !hwif->pci_dev->vendor)
1617 #endif  /* CONFIG_BLK_DEV_IDEPCI */
1618                 {
1619                         /*
1620                          * Probably not a shared PCI interrupt,
1621                          * so we can safely try to do something about it:
1622                          */
1623                         unexpected_intr(irq, hwgroup);
1624 #ifdef CONFIG_BLK_DEV_IDEPCI
1625                 } else {
1626                         /*
1627                          * Whack the status register, just in case
1628                          * we have a leftover pending IRQ.
1629                          */
1630                         (void) hwif->INB(hwif->io_ports[IDE_STATUS_OFFSET]);
1631 #endif /* CONFIG_BLK_DEV_IDEPCI */
1632                 }
1633                 spin_unlock_irqrestore(&ide_lock, flags);
1634                 return IRQ_NONE;
1635         }
1636         drive = hwgroup->drive;
1637         if (!drive) {
1638                 /*
1639                  * This should NEVER happen, and there isn't much
1640                  * we could do about it here.
1641                  *
1642                  * [Note - this can occur if the drive is hot unplugged]
1643                  */
1644                 spin_unlock_irqrestore(&ide_lock, flags);
1645                 return IRQ_HANDLED;
1646         }
1647         if (!drive_is_ready(drive)) {
1648                 /*
1649                  * This happens regularly when we share a PCI IRQ with
1650                  * another device.  Unfortunately, it can also happen
1651                  * with some buggy drives that trigger the IRQ before
1652                  * their status register is up to date.  Hopefully we have
1653                  * enough advance overhead that the latter isn't a problem.
1654                  */
1655                 spin_unlock_irqrestore(&ide_lock, flags);
1656                 return IRQ_NONE;
1657         }
1658         if (!hwgroup->busy) {
1659                 hwgroup->busy = 1;      /* paranoia */
1660                 printk(KERN_ERR "%s: ide_intr: hwgroup->busy was 0 ??\n", drive->name);
1661         }
1662         hwgroup->handler = NULL;
1663         hwgroup->req_gen++;
1664         del_timer(&hwgroup->timer);
1665         spin_unlock(&ide_lock);
1666
1667         /* Some controllers might set DMA INTR no matter DMA or PIO;
1668          * bmdma status might need to be cleared even for
1669          * PIO interrupts to prevent spurious/lost irq.
1670          */
1671         if (hwif->ide_dma_clear_irq && !(drive->waiting_for_dma))
1672                 /* ide_dma_end() needs bmdma status for error checking.
1673                  * So, skip clearing bmdma status here and leave it
1674                  * to ide_dma_end() if this is dma interrupt.
1675                  */
1676                 hwif->ide_dma_clear_irq(drive);
1677
1678         if (drive->unmask)
1679                 local_irq_enable_in_hardirq();
1680         /* service this interrupt, may set handler for next interrupt */
1681         startstop = handler(drive);
1682         spin_lock_irq(&ide_lock);
1683
1684         /*
1685          * Note that handler() may have set things up for another
1686          * interrupt to occur soon, but it cannot happen until
1687          * we exit from this routine, because it will be the
1688          * same irq as is currently being serviced here, and Linux
1689          * won't allow another of the same (on any CPU) until we return.
1690          */
1691         drive->service_time = jiffies - drive->service_start;
1692         if (startstop == ide_stopped) {
1693                 if (hwgroup->handler == NULL) { /* paranoia */
1694                         hwgroup->busy = 0;
1695                         ide_do_request(hwgroup, hwif->irq);
1696                 } else {
1697                         printk(KERN_ERR "%s: ide_intr: huh? expected NULL handler "
1698                                 "on exit\n", drive->name);
1699                 }
1700         }
1701         spin_unlock_irqrestore(&ide_lock, flags);
1702         return IRQ_HANDLED;
1703 }
1704
1705 /**
1706  *      ide_init_drive_cmd      -       initialize a drive command request
1707  *      @rq: request object
1708  *
1709  *      Initialize a request before we fill it in and send it down to
1710  *      ide_do_drive_cmd. Commands must be set up by this function. Right
1711  *      now it doesn't do a lot, but if that changes abusers will have a
1712  *      nasty surprise.
1713  */
1714
1715 void ide_init_drive_cmd (struct request *rq)
1716 {
1717         memset(rq, 0, sizeof(*rq));
1718         rq->cmd_type = REQ_TYPE_ATA_CMD;
1719         rq->ref_count = 1;
1720 }
1721
1722 EXPORT_SYMBOL(ide_init_drive_cmd);
1723
1724 /**
1725  *      ide_do_drive_cmd        -       issue IDE special command
1726  *      @drive: device to issue command
1727  *      @rq: request to issue
1728  *      @action: action for processing
1729  *
1730  *      This function issues a special IDE device request
1731  *      onto the request queue.
1732  *
1733  *      If action is ide_wait, then the rq is queued at the end of the
1734  *      request queue, and the function sleeps until it has been processed.
1735  *      This is for use when invoked from an ioctl handler.
1736  *
1737  *      If action is ide_preempt, then the rq is queued at the head of
1738  *      the request queue, displacing the currently-being-processed
1739  *      request and this function returns immediately without waiting
1740  *      for the new rq to be completed.  This is VERY DANGEROUS, and is
1741  *      intended for careful use by the ATAPI tape/cdrom driver code.
1742  *
1743  *      If action is ide_end, then the rq is queued at the end of the
1744  *      request queue, and the function returns immediately without waiting
1745  *      for the new rq to be completed. This is again intended for careful
1746  *      use by the ATAPI tape/cdrom driver code.
1747  */
1748  
1749 int ide_do_drive_cmd (ide_drive_t *drive, struct request *rq, ide_action_t action)
1750 {
1751         unsigned long flags;
1752         ide_hwgroup_t *hwgroup = HWGROUP(drive);
1753         DECLARE_COMPLETION_ONSTACK(wait);
1754         int where = ELEVATOR_INSERT_BACK, err;
1755         int must_wait = (action == ide_wait || action == ide_head_wait);
1756
1757         rq->errors = 0;
1758
1759         /*
1760          * we need to hold an extra reference to request for safe inspection
1761          * after completion
1762          */
1763         if (must_wait) {
1764                 rq->ref_count++;
1765                 rq->end_io_data = &wait;
1766                 rq->end_io = blk_end_sync_rq;
1767         }
1768
1769         spin_lock_irqsave(&ide_lock, flags);
1770         if (action == ide_preempt)
1771                 hwgroup->rq = NULL;
1772         if (action == ide_preempt || action == ide_head_wait) {
1773                 where = ELEVATOR_INSERT_FRONT;
1774                 rq->cmd_flags |= REQ_PREEMPT;
1775         }
1776         __elv_add_request(drive->queue, rq, where, 0);
1777         ide_do_request(hwgroup, IDE_NO_IRQ);
1778         spin_unlock_irqrestore(&ide_lock, flags);
1779
1780         err = 0;
1781         if (must_wait) {
1782                 wait_for_completion(&wait);
1783                 if (rq->errors)
1784                         err = -EIO;
1785
1786                 blk_put_request(rq);
1787         }
1788
1789         return err;
1790 }
1791
1792 EXPORT_SYMBOL(ide_do_drive_cmd);