Merge branch 'ucc_geth' of master.kernel.org:/pub/scm/linux/kernel/git/leo/fsl-soc...
[linux-2.6] / drivers / scsi / aic7xxx / aic7xxx_core.c
1 /*
2  * Core routines and tables shareable across OS platforms.
3  *
4  * Copyright (c) 1994-2002 Justin T. Gibbs.
5  * Copyright (c) 2000-2002 Adaptec Inc.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions, and the following disclaimer,
13  *    without modification.
14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
15  *    substantially similar to the "NO WARRANTY" disclaimer below
16  *    ("Disclaimer") and any redistribution must be conditioned upon
17  *    including a substantially similar Disclaimer requirement for further
18  *    binary redistribution.
19  * 3. Neither the names of the above-listed copyright holders nor the names
20  *    of any contributors may be used to endorse or promote products derived
21  *    from this software without specific prior written permission.
22  *
23  * Alternatively, this software may be distributed under the terms of the
24  * GNU General Public License ("GPL") version 2 as published by the Free
25  * Software Foundation.
26  *
27  * NO WARRANTY
28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38  * POSSIBILITY OF SUCH DAMAGES.
39  *
40  * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#155 $
41  */
42
43 #ifdef __linux__
44 #include "aic7xxx_osm.h"
45 #include "aic7xxx_inline.h"
46 #include "aicasm/aicasm_insformat.h"
47 #else
48 #include <dev/aic7xxx/aic7xxx_osm.h>
49 #include <dev/aic7xxx/aic7xxx_inline.h>
50 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
51 #endif
52
53 /***************************** Lookup Tables **********************************/
54 char *ahc_chip_names[] =
55 {
56         "NONE",
57         "aic7770",
58         "aic7850",
59         "aic7855",
60         "aic7859",
61         "aic7860",
62         "aic7870",
63         "aic7880",
64         "aic7895",
65         "aic7895C",
66         "aic7890/91",
67         "aic7896/97",
68         "aic7892",
69         "aic7899"
70 };
71 static const u_int num_chip_names = ARRAY_SIZE(ahc_chip_names);
72
73 /*
74  * Hardware error codes.
75  */
76 struct ahc_hard_error_entry {
77         uint8_t errno;
78         char *errmesg;
79 };
80
81 static struct ahc_hard_error_entry ahc_hard_errors[] = {
82         { ILLHADDR,     "Illegal Host Access" },
83         { ILLSADDR,     "Illegal Sequencer Address referrenced" },
84         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
85         { SQPARERR,     "Sequencer Parity Error" },
86         { DPARERR,      "Data-path Parity Error" },
87         { MPARERR,      "Scratch or SCB Memory Parity Error" },
88         { PCIERRSTAT,   "PCI Error detected" },
89         { CIOPARERR,    "CIOBUS Parity Error" },
90 };
91 static const u_int num_errors = ARRAY_SIZE(ahc_hard_errors);
92
93 static struct ahc_phase_table_entry ahc_phase_table[] =
94 {
95         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
96         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
97         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
98         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
99         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
100         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
101         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
102         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
103         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
104         { 0,            MSG_NOOP,               "in unknown phase"      }
105 };
106
107 /*
108  * In most cases we only wish to itterate over real phases, so
109  * exclude the last element from the count.
110  */
111 static const u_int num_phases = ARRAY_SIZE(ahc_phase_table) - 1;
112
113 /*
114  * Valid SCSIRATE values.  (p. 3-17)
115  * Provides a mapping of tranfer periods in ns to the proper value to
116  * stick in the scsixfer reg.
117  */
118 static struct ahc_syncrate ahc_syncrates[] =
119 {
120       /* ultra2    fast/ultra  period     rate */
121         { 0x42,      0x000,      9,      "80.0" },
122         { 0x03,      0x000,     10,      "40.0" },
123         { 0x04,      0x000,     11,      "33.0" },
124         { 0x05,      0x100,     12,      "20.0" },
125         { 0x06,      0x110,     15,      "16.0" },
126         { 0x07,      0x120,     18,      "13.4" },
127         { 0x08,      0x000,     25,      "10.0" },
128         { 0x19,      0x010,     31,      "8.0"  },
129         { 0x1a,      0x020,     37,      "6.67" },
130         { 0x1b,      0x030,     43,      "5.7"  },
131         { 0x1c,      0x040,     50,      "5.0"  },
132         { 0x00,      0x050,     56,      "4.4"  },
133         { 0x00,      0x060,     62,      "4.0"  },
134         { 0x00,      0x070,     68,      "3.6"  },
135         { 0x00,      0x000,      0,      NULL   }
136 };
137
138 /* Our Sequencer Program */
139 #include "aic7xxx_seq.h"
140
141 /**************************** Function Declarations ***************************/
142 static void             ahc_force_renegotiation(struct ahc_softc *ahc,
143                                                 struct ahc_devinfo *devinfo);
144 static struct ahc_tmode_tstate*
145                         ahc_alloc_tstate(struct ahc_softc *ahc,
146                                          u_int scsi_id, char channel);
147 #ifdef AHC_TARGET_MODE
148 static void             ahc_free_tstate(struct ahc_softc *ahc,
149                                         u_int scsi_id, char channel, int force);
150 #endif
151 static struct ahc_syncrate*
152                         ahc_devlimited_syncrate(struct ahc_softc *ahc,
153                                                 struct ahc_initiator_tinfo *,
154                                                 u_int *period,
155                                                 u_int *ppr_options,
156                                                 role_t role);
157 static void             ahc_update_pending_scbs(struct ahc_softc *ahc);
158 static void             ahc_fetch_devinfo(struct ahc_softc *ahc,
159                                           struct ahc_devinfo *devinfo);
160 static void             ahc_scb_devinfo(struct ahc_softc *ahc,
161                                         struct ahc_devinfo *devinfo,
162                                         struct scb *scb);
163 static void             ahc_assert_atn(struct ahc_softc *ahc);
164 static void             ahc_setup_initiator_msgout(struct ahc_softc *ahc,
165                                                    struct ahc_devinfo *devinfo,
166                                                    struct scb *scb);
167 static void             ahc_build_transfer_msg(struct ahc_softc *ahc,
168                                                struct ahc_devinfo *devinfo);
169 static void             ahc_construct_sdtr(struct ahc_softc *ahc,
170                                            struct ahc_devinfo *devinfo,
171                                            u_int period, u_int offset);
172 static void             ahc_construct_wdtr(struct ahc_softc *ahc,
173                                            struct ahc_devinfo *devinfo,
174                                            u_int bus_width);
175 static void             ahc_construct_ppr(struct ahc_softc *ahc,
176                                           struct ahc_devinfo *devinfo,
177                                           u_int period, u_int offset,
178                                           u_int bus_width, u_int ppr_options);
179 static void             ahc_clear_msg_state(struct ahc_softc *ahc);
180 static void             ahc_handle_proto_violation(struct ahc_softc *ahc);
181 static void             ahc_handle_message_phase(struct ahc_softc *ahc);
182 typedef enum {
183         AHCMSG_1B,
184         AHCMSG_2B,
185         AHCMSG_EXT
186 } ahc_msgtype;
187 static int              ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
188                                      u_int msgval, int full);
189 static int              ahc_parse_msg(struct ahc_softc *ahc,
190                                       struct ahc_devinfo *devinfo);
191 static int              ahc_handle_msg_reject(struct ahc_softc *ahc,
192                                               struct ahc_devinfo *devinfo);
193 static void             ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
194                                                 struct ahc_devinfo *devinfo);
195 static void             ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
196 static void             ahc_handle_devreset(struct ahc_softc *ahc,
197                                             struct ahc_devinfo *devinfo,
198                                             cam_status status, char *message,
199                                             int verbose_level);
200 #ifdef AHC_TARGET_MODE
201 static void             ahc_setup_target_msgin(struct ahc_softc *ahc,
202                                                struct ahc_devinfo *devinfo,
203                                                struct scb *scb);
204 #endif
205
206 static bus_dmamap_callback_t    ahc_dmamap_cb; 
207 static void                     ahc_build_free_scb_list(struct ahc_softc *ahc);
208 static int                      ahc_init_scbdata(struct ahc_softc *ahc);
209 static void                     ahc_fini_scbdata(struct ahc_softc *ahc);
210 static void             ahc_qinfifo_requeue(struct ahc_softc *ahc,
211                                             struct scb *prev_scb,
212                                             struct scb *scb);
213 static int              ahc_qinfifo_count(struct ahc_softc *ahc);
214 static u_int            ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
215                                                    u_int prev, u_int scbptr);
216 static void             ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
217 static u_int            ahc_rem_wscb(struct ahc_softc *ahc,
218                                      u_int scbpos, u_int prev);
219 static void             ahc_reset_current_bus(struct ahc_softc *ahc);
220 #ifdef AHC_DUMP_SEQ
221 static void             ahc_dumpseq(struct ahc_softc *ahc);
222 #endif
223 static int              ahc_loadseq(struct ahc_softc *ahc);
224 static int              ahc_check_patch(struct ahc_softc *ahc,
225                                         struct patch **start_patch,
226                                         u_int start_instr, u_int *skip_addr);
227 static void             ahc_download_instr(struct ahc_softc *ahc,
228                                            u_int instrptr, uint8_t *dconsts);
229 #ifdef AHC_TARGET_MODE
230 static void             ahc_queue_lstate_event(struct ahc_softc *ahc,
231                                                struct ahc_tmode_lstate *lstate,
232                                                u_int initiator_id,
233                                                u_int event_type,
234                                                u_int event_arg);
235 static void             ahc_update_scsiid(struct ahc_softc *ahc,
236                                           u_int targid_mask);
237 static int              ahc_handle_target_cmd(struct ahc_softc *ahc,
238                                               struct target_cmd *cmd);
239 #endif
240 /************************* Sequencer Execution Control ************************/
241 /*
242  * Restart the sequencer program from address zero
243  */
244 void
245 ahc_restart(struct ahc_softc *ahc)
246 {
247
248         ahc_pause(ahc);
249
250         /* No more pending messages. */
251         ahc_clear_msg_state(ahc);
252
253         ahc_outb(ahc, SCSISIGO, 0);             /* De-assert BSY */
254         ahc_outb(ahc, MSG_OUT, MSG_NOOP);       /* No message to send */
255         ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
256         ahc_outb(ahc, LASTPHASE, P_BUSFREE);
257         ahc_outb(ahc, SAVED_SCSIID, 0xFF);
258         ahc_outb(ahc, SAVED_LUN, 0xFF);
259
260         /*
261          * Ensure that the sequencer's idea of TQINPOS
262          * matches our own.  The sequencer increments TQINPOS
263          * only after it sees a DMA complete and a reset could
264          * occur before the increment leaving the kernel to believe
265          * the command arrived but the sequencer to not.
266          */
267         ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
268
269         /* Always allow reselection */
270         ahc_outb(ahc, SCSISEQ,
271                  ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
272         if ((ahc->features & AHC_CMD_CHAN) != 0) {
273                 /* Ensure that no DMA operations are in progress */
274                 ahc_outb(ahc, CCSCBCNT, 0);
275                 ahc_outb(ahc, CCSGCTL, 0);
276                 ahc_outb(ahc, CCSCBCTL, 0);
277         }
278         /*
279          * If we were in the process of DMA'ing SCB data into
280          * an SCB, replace that SCB on the free list.  This prevents
281          * an SCB leak.
282          */
283         if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
284                 ahc_add_curscb_to_free_list(ahc);
285                 ahc_outb(ahc, SEQ_FLAGS2,
286                          ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
287         }
288
289         /*
290          * Clear any pending sequencer interrupt.  It is no
291          * longer relevant since we're resetting the Program
292          * Counter.
293          */
294         ahc_outb(ahc, CLRINT, CLRSEQINT);
295
296         ahc_outb(ahc, MWI_RESIDUAL, 0);
297         ahc_outb(ahc, SEQCTL, ahc->seqctl);
298         ahc_outb(ahc, SEQADDR0, 0);
299         ahc_outb(ahc, SEQADDR1, 0);
300
301         ahc_unpause(ahc);
302 }
303
304 /************************* Input/Output Queues ********************************/
305 void
306 ahc_run_qoutfifo(struct ahc_softc *ahc)
307 {
308         struct scb *scb;
309         u_int  scb_index;
310
311         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
312         while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
313
314                 scb_index = ahc->qoutfifo[ahc->qoutfifonext];
315                 if ((ahc->qoutfifonext & 0x03) == 0x03) {
316                         u_int modnext;
317
318                         /*
319                          * Clear 32bits of QOUTFIFO at a time
320                          * so that we don't clobber an incoming
321                          * byte DMA to the array on architectures
322                          * that only support 32bit load and store
323                          * operations.
324                          */
325                         modnext = ahc->qoutfifonext & ~0x3;
326                         *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
327                         ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
328                                         ahc->shared_data_dmamap,
329                                         /*offset*/modnext, /*len*/4,
330                                         BUS_DMASYNC_PREREAD);
331                 }
332                 ahc->qoutfifonext++;
333
334                 scb = ahc_lookup_scb(ahc, scb_index);
335                 if (scb == NULL) {
336                         printf("%s: WARNING no command for scb %d "
337                                "(cmdcmplt)\nQOUTPOS = %d\n",
338                                ahc_name(ahc), scb_index,
339                                (ahc->qoutfifonext - 1) & 0xFF);
340                         continue;
341                 }
342
343                 /*
344                  * Save off the residual
345                  * if there is one.
346                  */
347                 ahc_update_residual(ahc, scb);
348                 ahc_done(ahc, scb);
349         }
350 }
351
352 void
353 ahc_run_untagged_queues(struct ahc_softc *ahc)
354 {
355         int i;
356
357         for (i = 0; i < 16; i++)
358                 ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
359 }
360
361 void
362 ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
363 {
364         struct scb *scb;
365
366         if (ahc->untagged_queue_lock != 0)
367                 return;
368
369         if ((scb = TAILQ_FIRST(queue)) != NULL
370          && (scb->flags & SCB_ACTIVE) == 0) {
371                 scb->flags |= SCB_ACTIVE;
372                 ahc_queue_scb(ahc, scb);
373         }
374 }
375
376 /************************* Interrupt Handling *********************************/
377 void
378 ahc_handle_brkadrint(struct ahc_softc *ahc)
379 {
380         /*
381          * We upset the sequencer :-(
382          * Lookup the error message
383          */
384         int i;
385         int error;
386
387         error = ahc_inb(ahc, ERROR);
388         for (i = 0; error != 1 && i < num_errors; i++)
389                 error >>= 1;
390         printf("%s: brkadrint, %s at seqaddr = 0x%x\n",
391                ahc_name(ahc), ahc_hard_errors[i].errmesg,
392                ahc_inb(ahc, SEQADDR0) |
393                (ahc_inb(ahc, SEQADDR1) << 8));
394
395         ahc_dump_card_state(ahc);
396
397         /* Tell everyone that this HBA is no longer available */
398         ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
399                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
400                        CAM_NO_HBA);
401
402         /* Disable all interrupt sources by resetting the controller */
403         ahc_shutdown(ahc);
404 }
405
406 void
407 ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
408 {
409         struct scb *scb;
410         struct ahc_devinfo devinfo;
411         
412         ahc_fetch_devinfo(ahc, &devinfo);
413
414         /*
415          * Clear the upper byte that holds SEQINT status
416          * codes and clear the SEQINT bit. We will unpause
417          * the sequencer, if appropriate, after servicing
418          * the request.
419          */
420         ahc_outb(ahc, CLRINT, CLRSEQINT);
421         switch (intstat & SEQINT_MASK) {
422         case BAD_STATUS:
423         {
424                 u_int  scb_index;
425                 struct hardware_scb *hscb;
426
427                 /*
428                  * Set the default return value to 0 (don't
429                  * send sense).  The sense code will change
430                  * this if needed.
431                  */
432                 ahc_outb(ahc, RETURN_1, 0);
433
434                 /*
435                  * The sequencer will notify us when a command
436                  * has an error that would be of interest to
437                  * the kernel.  This allows us to leave the sequencer
438                  * running in the common case of command completes
439                  * without error.  The sequencer will already have
440                  * dma'd the SCB back up to us, so we can reference
441                  * the in kernel copy directly.
442                  */
443                 scb_index = ahc_inb(ahc, SCB_TAG);
444                 scb = ahc_lookup_scb(ahc, scb_index);
445                 if (scb == NULL) {
446                         ahc_print_devinfo(ahc, &devinfo);
447                         printf("ahc_intr - referenced scb "
448                                "not valid during seqint 0x%x scb(%d)\n",
449                                intstat, scb_index);
450                         ahc_dump_card_state(ahc);
451                         panic("for safety");
452                         goto unpause;
453                 }
454
455                 hscb = scb->hscb; 
456
457                 /* Don't want to clobber the original sense code */
458                 if ((scb->flags & SCB_SENSE) != 0) {
459                         /*
460                          * Clear the SCB_SENSE Flag and have
461                          * the sequencer do a normal command
462                          * complete.
463                          */
464                         scb->flags &= ~SCB_SENSE;
465                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
466                         break;
467                 }
468                 ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
469                 /* Freeze the queue until the client sees the error. */
470                 ahc_freeze_devq(ahc, scb);
471                 ahc_freeze_scb(scb);
472                 ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
473                 switch (hscb->shared_data.status.scsi_status) {
474                 case SCSI_STATUS_OK:
475                         printf("%s: Interrupted for staus of 0???\n",
476                                ahc_name(ahc));
477                         break;
478                 case SCSI_STATUS_CMD_TERMINATED:
479                 case SCSI_STATUS_CHECK_COND:
480                 {
481                         struct ahc_dma_seg *sg;
482                         struct scsi_sense *sc;
483                         struct ahc_initiator_tinfo *targ_info;
484                         struct ahc_tmode_tstate *tstate;
485                         struct ahc_transinfo *tinfo;
486 #ifdef AHC_DEBUG
487                         if (ahc_debug & AHC_SHOW_SENSE) {
488                                 ahc_print_path(ahc, scb);
489                                 printf("SCB %d: requests Check Status\n",
490                                        scb->hscb->tag);
491                         }
492 #endif
493
494                         if (ahc_perform_autosense(scb) == 0)
495                                 break;
496
497                         targ_info = ahc_fetch_transinfo(ahc,
498                                                         devinfo.channel,
499                                                         devinfo.our_scsiid,
500                                                         devinfo.target,
501                                                         &tstate);
502                         tinfo = &targ_info->curr;
503                         sg = scb->sg_list;
504                         sc = (struct scsi_sense *)(&hscb->shared_data.cdb); 
505                         /*
506                          * Save off the residual if there is one.
507                          */
508                         ahc_update_residual(ahc, scb);
509 #ifdef AHC_DEBUG
510                         if (ahc_debug & AHC_SHOW_SENSE) {
511                                 ahc_print_path(ahc, scb);
512                                 printf("Sending Sense\n");
513                         }
514 #endif
515                         sg->addr = ahc_get_sense_bufaddr(ahc, scb);
516                         sg->len = ahc_get_sense_bufsize(ahc, scb);
517                         sg->len |= AHC_DMA_LAST_SEG;
518
519                         /* Fixup byte order */
520                         sg->addr = ahc_htole32(sg->addr);
521                         sg->len = ahc_htole32(sg->len);
522
523                         sc->opcode = REQUEST_SENSE;
524                         sc->byte2 = 0;
525                         if (tinfo->protocol_version <= SCSI_REV_2
526                          && SCB_GET_LUN(scb) < 8)
527                                 sc->byte2 = SCB_GET_LUN(scb) << 5;
528                         sc->unused[0] = 0;
529                         sc->unused[1] = 0;
530                         sc->length = sg->len;
531                         sc->control = 0;
532
533                         /*
534                          * We can't allow the target to disconnect.
535                          * This will be an untagged transaction and
536                          * having the target disconnect will make this
537                          * transaction indestinguishable from outstanding
538                          * tagged transactions.
539                          */
540                         hscb->control = 0;
541
542                         /*
543                          * This request sense could be because the
544                          * the device lost power or in some other
545                          * way has lost our transfer negotiations.
546                          * Renegotiate if appropriate.  Unit attention
547                          * errors will be reported before any data
548                          * phases occur.
549                          */
550                         if (ahc_get_residual(scb) 
551                          == ahc_get_transfer_length(scb)) {
552                                 ahc_update_neg_request(ahc, &devinfo,
553                                                        tstate, targ_info,
554                                                        AHC_NEG_IF_NON_ASYNC);
555                         }
556                         if (tstate->auto_negotiate & devinfo.target_mask) {
557                                 hscb->control |= MK_MESSAGE;
558                                 scb->flags &= ~SCB_NEGOTIATE;
559                                 scb->flags |= SCB_AUTO_NEGOTIATE;
560                         }
561                         hscb->cdb_len = sizeof(*sc);
562                         hscb->dataptr = sg->addr; 
563                         hscb->datacnt = sg->len;
564                         hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
565                         hscb->sgptr = ahc_htole32(hscb->sgptr);
566                         scb->sg_count = 1;
567                         scb->flags |= SCB_SENSE;
568                         ahc_qinfifo_requeue_tail(ahc, scb);
569                         ahc_outb(ahc, RETURN_1, SEND_SENSE);
570                         /*
571                          * Ensure we have enough time to actually
572                          * retrieve the sense.
573                          */
574                         ahc_scb_timer_reset(scb, 5 * 1000000);
575                         break;
576                 }
577                 default:
578                         break;
579                 }
580                 break;
581         }
582         case NO_MATCH:
583         {
584                 /* Ensure we don't leave the selection hardware on */
585                 ahc_outb(ahc, SCSISEQ,
586                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
587
588                 printf("%s:%c:%d: no active SCB for reconnecting "
589                        "target - issuing BUS DEVICE RESET\n",
590                        ahc_name(ahc), devinfo.channel, devinfo.target);
591                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
592                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
593                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
594                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
595                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
596                        "SINDEX == 0x%x\n",
597                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
598                        ahc_index_busy_tcl(ahc,
599                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
600                                       ahc_inb(ahc, SAVED_LUN))),
601                        ahc_inb(ahc, SINDEX));
602                 printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
603                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
604                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
605                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
606                        ahc_inb(ahc, SCB_CONTROL));
607                 printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
608                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
609                 printf("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
610                 printf("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
611                 ahc_dump_card_state(ahc);
612                 ahc->msgout_buf[0] = MSG_BUS_DEV_RESET;
613                 ahc->msgout_len = 1;
614                 ahc->msgout_index = 0;
615                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
616                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
617                 ahc_assert_atn(ahc);
618                 break;
619         }
620         case SEND_REJECT: 
621         {
622                 u_int rejbyte = ahc_inb(ahc, ACCUM);
623                 printf("%s:%c:%d: Warning - unknown message received from "
624                        "target (0x%x).  Rejecting\n", 
625                        ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
626                 break; 
627         }
628         case PROTO_VIOLATION:
629         {
630                 ahc_handle_proto_violation(ahc);
631                 break;
632         }
633         case IGN_WIDE_RES:
634                 ahc_handle_ign_wide_residue(ahc, &devinfo);
635                 break;
636         case PDATA_REINIT:
637                 ahc_reinitialize_dataptrs(ahc);
638                 break;
639         case BAD_PHASE:
640         {
641                 u_int lastphase;
642
643                 lastphase = ahc_inb(ahc, LASTPHASE);
644                 printf("%s:%c:%d: unknown scsi bus phase %x, "
645                        "lastphase = 0x%x.  Attempting to continue\n",
646                        ahc_name(ahc), devinfo.channel, devinfo.target,
647                        lastphase, ahc_inb(ahc, SCSISIGI));
648                 break;
649         }
650         case MISSED_BUSFREE:
651         {
652                 u_int lastphase;
653
654                 lastphase = ahc_inb(ahc, LASTPHASE);
655                 printf("%s:%c:%d: Missed busfree. "
656                        "Lastphase = 0x%x, Curphase = 0x%x\n",
657                        ahc_name(ahc), devinfo.channel, devinfo.target,
658                        lastphase, ahc_inb(ahc, SCSISIGI));
659                 ahc_restart(ahc);
660                 return;
661         }
662         case HOST_MSG_LOOP:
663         {
664                 /*
665                  * The sequencer has encountered a message phase
666                  * that requires host assistance for completion.
667                  * While handling the message phase(s), we will be
668                  * notified by the sequencer after each byte is
669                  * transfered so we can track bus phase changes.
670                  *
671                  * If this is the first time we've seen a HOST_MSG_LOOP
672                  * interrupt, initialize the state of the host message
673                  * loop.
674                  */
675                 if (ahc->msg_type == MSG_TYPE_NONE) {
676                         struct scb *scb;
677                         u_int scb_index;
678                         u_int bus_phase;
679
680                         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
681                         if (bus_phase != P_MESGIN
682                          && bus_phase != P_MESGOUT) {
683                                 printf("ahc_intr: HOST_MSG_LOOP bad "
684                                        "phase 0x%x\n",
685                                       bus_phase);
686                                 /*
687                                  * Probably transitioned to bus free before
688                                  * we got here.  Just punt the message.
689                                  */
690                                 ahc_clear_intstat(ahc);
691                                 ahc_restart(ahc);
692                                 return;
693                         }
694
695                         scb_index = ahc_inb(ahc, SCB_TAG);
696                         scb = ahc_lookup_scb(ahc, scb_index);
697                         if (devinfo.role == ROLE_INITIATOR) {
698                                 if (scb == NULL)
699                                         panic("HOST_MSG_LOOP with "
700                                               "invalid SCB %x\n", scb_index);
701
702                                 if (bus_phase == P_MESGOUT)
703                                         ahc_setup_initiator_msgout(ahc,
704                                                                    &devinfo,
705                                                                    scb);
706                                 else {
707                                         ahc->msg_type =
708                                             MSG_TYPE_INITIATOR_MSGIN;
709                                         ahc->msgin_index = 0;
710                                 }
711                         }
712 #ifdef AHC_TARGET_MODE
713                         else {
714                                 if (bus_phase == P_MESGOUT) {
715                                         ahc->msg_type =
716                                             MSG_TYPE_TARGET_MSGOUT;
717                                         ahc->msgin_index = 0;
718                                 }
719                                 else 
720                                         ahc_setup_target_msgin(ahc,
721                                                                &devinfo,
722                                                                scb);
723                         }
724 #endif
725                 }
726
727                 ahc_handle_message_phase(ahc);
728                 break;
729         }
730         case PERR_DETECTED:
731         {
732                 /*
733                  * If we've cleared the parity error interrupt
734                  * but the sequencer still believes that SCSIPERR
735                  * is true, it must be that the parity error is
736                  * for the currently presented byte on the bus,
737                  * and we are not in a phase (data-in) where we will
738                  * eventually ack this byte.  Ack the byte and
739                  * throw it away in the hope that the target will
740                  * take us to message out to deliver the appropriate
741                  * error message.
742                  */
743                 if ((intstat & SCSIINT) == 0
744                  && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
745
746                         if ((ahc->features & AHC_DT) == 0) {
747                                 u_int curphase;
748
749                                 /*
750                                  * The hardware will only let you ack bytes
751                                  * if the expected phase in SCSISIGO matches
752                                  * the current phase.  Make sure this is
753                                  * currently the case.
754                                  */
755                                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
756                                 ahc_outb(ahc, LASTPHASE, curphase);
757                                 ahc_outb(ahc, SCSISIGO, curphase);
758                         }
759                         if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
760                                 int wait;
761
762                                 /*
763                                  * In a data phase.  Faster to bitbucket
764                                  * the data than to individually ack each
765                                  * byte.  This is also the only strategy
766                                  * that will work with AUTOACK enabled.
767                                  */
768                                 ahc_outb(ahc, SXFRCTL1,
769                                          ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
770                                 wait = 5000;
771                                 while (--wait != 0) {
772                                         if ((ahc_inb(ahc, SCSISIGI)
773                                           & (CDI|MSGI)) != 0)
774                                                 break;
775                                         ahc_delay(100);
776                                 }
777                                 ahc_outb(ahc, SXFRCTL1,
778                                          ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
779                                 if (wait == 0) {
780                                         struct  scb *scb;
781                                         u_int   scb_index;
782
783                                         ahc_print_devinfo(ahc, &devinfo);
784                                         printf("Unable to clear parity error.  "
785                                                "Resetting bus.\n");
786                                         scb_index = ahc_inb(ahc, SCB_TAG);
787                                         scb = ahc_lookup_scb(ahc, scb_index);
788                                         if (scb != NULL)
789                                                 ahc_set_transaction_status(scb,
790                                                     CAM_UNCOR_PARITY);
791                                         ahc_reset_channel(ahc, devinfo.channel, 
792                                                           /*init reset*/TRUE);
793                                 }
794                         } else {
795                                 ahc_inb(ahc, SCSIDATL);
796                         }
797                 }
798                 break;
799         }
800         case DATA_OVERRUN:
801         {
802                 /*
803                  * When the sequencer detects an overrun, it
804                  * places the controller in "BITBUCKET" mode
805                  * and allows the target to complete its transfer.
806                  * Unfortunately, none of the counters get updated
807                  * when the controller is in this mode, so we have
808                  * no way of knowing how large the overrun was.
809                  */
810                 u_int scbindex = ahc_inb(ahc, SCB_TAG);
811                 u_int lastphase = ahc_inb(ahc, LASTPHASE);
812                 u_int i;
813
814                 scb = ahc_lookup_scb(ahc, scbindex);
815                 for (i = 0; i < num_phases; i++) {
816                         if (lastphase == ahc_phase_table[i].phase)
817                                 break;
818                 }
819                 ahc_print_path(ahc, scb);
820                 printf("data overrun detected %s."
821                        "  Tag == 0x%x.\n",
822                        ahc_phase_table[i].phasemsg,
823                        scb->hscb->tag);
824                 ahc_print_path(ahc, scb);
825                 printf("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
826                        ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
827                        ahc_get_transfer_length(scb), scb->sg_count);
828                 if (scb->sg_count > 0) {
829                         for (i = 0; i < scb->sg_count; i++) {
830
831                                 printf("sg[%d] - Addr 0x%x%x : Length %d\n",
832                                        i,
833                                        (ahc_le32toh(scb->sg_list[i].len) >> 24
834                                         & SG_HIGH_ADDR_BITS),
835                                        ahc_le32toh(scb->sg_list[i].addr),
836                                        ahc_le32toh(scb->sg_list[i].len)
837                                        & AHC_SG_LEN_MASK);
838                         }
839                 }
840                 /*
841                  * Set this and it will take effect when the
842                  * target does a command complete.
843                  */
844                 ahc_freeze_devq(ahc, scb);
845                 if ((scb->flags & SCB_SENSE) == 0) {
846                         ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
847                 } else {
848                         scb->flags &= ~SCB_SENSE;
849                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
850                 }
851                 ahc_freeze_scb(scb);
852
853                 if ((ahc->features & AHC_ULTRA2) != 0) {
854                         /*
855                          * Clear the channel in case we return
856                          * to data phase later.
857                          */
858                         ahc_outb(ahc, SXFRCTL0,
859                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
860                         ahc_outb(ahc, SXFRCTL0,
861                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
862                 }
863                 if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
864                         u_int dscommand1;
865
866                         /* Ensure HHADDR is 0 for future DMA operations. */
867                         dscommand1 = ahc_inb(ahc, DSCOMMAND1);
868                         ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
869                         ahc_outb(ahc, HADDR, 0);
870                         ahc_outb(ahc, DSCOMMAND1, dscommand1);
871                 }
872                 break;
873         }
874         case MKMSG_FAILED:
875         {
876                 u_int scbindex;
877
878                 printf("%s:%c:%d:%d: Attempt to issue message failed\n",
879                        ahc_name(ahc), devinfo.channel, devinfo.target,
880                        devinfo.lun);
881                 scbindex = ahc_inb(ahc, SCB_TAG);
882                 scb = ahc_lookup_scb(ahc, scbindex);
883                 if (scb != NULL
884                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
885                         /*
886                          * Ensure that we didn't put a second instance of this
887                          * SCB into the QINFIFO.
888                          */
889                         ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
890                                            SCB_GET_CHANNEL(ahc, scb),
891                                            SCB_GET_LUN(scb), scb->hscb->tag,
892                                            ROLE_INITIATOR, /*status*/0,
893                                            SEARCH_REMOVE);
894                 break;
895         }
896         case NO_FREE_SCB:
897         {
898                 printf("%s: No free or disconnected SCBs\n", ahc_name(ahc));
899                 ahc_dump_card_state(ahc);
900                 panic("for safety");
901                 break;
902         }
903         case SCB_MISMATCH:
904         {
905                 u_int scbptr;
906
907                 scbptr = ahc_inb(ahc, SCBPTR);
908                 printf("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
909                        scbptr, ahc_inb(ahc, ARG_1),
910                        ahc->scb_data->hscbs[scbptr].tag);
911                 ahc_dump_card_state(ahc);
912                 panic("for saftey");
913                 break;
914         }
915         case OUT_OF_RANGE:
916         {
917                 printf("%s: BTT calculation out of range\n", ahc_name(ahc));
918                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
919                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
920                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
921                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
922                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
923                        "SINDEX == 0x%x\n, A == 0x%x\n",
924                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
925                        ahc_index_busy_tcl(ahc,
926                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
927                                       ahc_inb(ahc, SAVED_LUN))),
928                        ahc_inb(ahc, SINDEX),
929                        ahc_inb(ahc, ACCUM));
930                 printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
931                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
932                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
933                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
934                        ahc_inb(ahc, SCB_CONTROL));
935                 printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
936                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
937                 ahc_dump_card_state(ahc);
938                 panic("for safety");
939                 break;
940         }
941         default:
942                 printf("ahc_intr: seqint, "
943                        "intstat == 0x%x, scsisigi = 0x%x\n",
944                        intstat, ahc_inb(ahc, SCSISIGI));
945                 break;
946         }
947 unpause:
948         /*
949          *  The sequencer is paused immediately on
950          *  a SEQINT, so we should restart it when
951          *  we're done.
952          */
953         ahc_unpause(ahc);
954 }
955
956 void
957 ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
958 {
959         u_int   scb_index;
960         u_int   status0;
961         u_int   status;
962         struct  scb *scb;
963         char    cur_channel;
964         char    intr_channel;
965
966         if ((ahc->features & AHC_TWIN) != 0
967          && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
968                 cur_channel = 'B';
969         else
970                 cur_channel = 'A';
971         intr_channel = cur_channel;
972
973         if ((ahc->features & AHC_ULTRA2) != 0)
974                 status0 = ahc_inb(ahc, SSTAT0) & IOERR;
975         else
976                 status0 = 0;
977         status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
978         if (status == 0 && status0 == 0) {
979                 if ((ahc->features & AHC_TWIN) != 0) {
980                         /* Try the other channel */
981                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
982                         status = ahc_inb(ahc, SSTAT1)
983                                & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
984                         intr_channel = (cur_channel == 'A') ? 'B' : 'A';
985                 }
986                 if (status == 0) {
987                         printf("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
988                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
989                         ahc_unpause(ahc);
990                         return;
991                 }
992         }
993
994         /* Make sure the sequencer is in a safe location. */
995         ahc_clear_critical_section(ahc);
996
997         scb_index = ahc_inb(ahc, SCB_TAG);
998         scb = ahc_lookup_scb(ahc, scb_index);
999         if (scb != NULL
1000          && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1001                 scb = NULL;
1002
1003         if ((ahc->features & AHC_ULTRA2) != 0
1004          && (status0 & IOERR) != 0) {
1005                 int now_lvd;
1006
1007                 now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
1008                 printf("%s: Transceiver State Has Changed to %s mode\n",
1009                        ahc_name(ahc), now_lvd ? "LVD" : "SE");
1010                 ahc_outb(ahc, CLRSINT0, CLRIOERR);
1011                 /*
1012                  * When transitioning to SE mode, the reset line
1013                  * glitches, triggering an arbitration bug in some
1014                  * Ultra2 controllers.  This bug is cleared when we
1015                  * assert the reset line.  Since a reset glitch has
1016                  * already occurred with this transition and a
1017                  * transceiver state change is handled just like
1018                  * a bus reset anyway, asserting the reset line
1019                  * ourselves is safe.
1020                  */
1021                 ahc_reset_channel(ahc, intr_channel,
1022                                  /*Initiate Reset*/now_lvd == 0);
1023         } else if ((status & SCSIRSTI) != 0) {
1024                 printf("%s: Someone reset channel %c\n",
1025                         ahc_name(ahc), intr_channel);
1026                 if (intr_channel != cur_channel)
1027                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1028                 ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1029         } else if ((status & SCSIPERR) != 0) {
1030                 /*
1031                  * Determine the bus phase and queue an appropriate message.
1032                  * SCSIPERR is latched true as soon as a parity error
1033                  * occurs.  If the sequencer acked the transfer that
1034                  * caused the parity error and the currently presented
1035                  * transfer on the bus has correct parity, SCSIPERR will
1036                  * be cleared by CLRSCSIPERR.  Use this to determine if
1037                  * we should look at the last phase the sequencer recorded,
1038                  * or the current phase presented on the bus.
1039                  */
1040                 struct  ahc_devinfo devinfo;
1041                 u_int   mesg_out;
1042                 u_int   curphase;
1043                 u_int   errorphase;
1044                 u_int   lastphase;
1045                 u_int   scsirate;
1046                 u_int   i;
1047                 u_int   sstat2;
1048                 int     silent;
1049
1050                 lastphase = ahc_inb(ahc, LASTPHASE);
1051                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1052                 sstat2 = ahc_inb(ahc, SSTAT2);
1053                 ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1054                 /*
1055                  * For all phases save DATA, the sequencer won't
1056                  * automatically ack a byte that has a parity error
1057                  * in it.  So the only way that the current phase
1058                  * could be 'data-in' is if the parity error is for
1059                  * an already acked byte in the data phase.  During
1060                  * synchronous data-in transfers, we may actually
1061                  * ack bytes before latching the current phase in
1062                  * LASTPHASE, leading to the discrepancy between
1063                  * curphase and lastphase.
1064                  */
1065                 if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1066                  || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1067                         errorphase = curphase;
1068                 else
1069                         errorphase = lastphase;
1070
1071                 for (i = 0; i < num_phases; i++) {
1072                         if (errorphase == ahc_phase_table[i].phase)
1073                                 break;
1074                 }
1075                 mesg_out = ahc_phase_table[i].mesg_out;
1076                 silent = FALSE;
1077                 if (scb != NULL) {
1078                         if (SCB_IS_SILENT(scb))
1079                                 silent = TRUE;
1080                         else
1081                                 ahc_print_path(ahc, scb);
1082                         scb->flags |= SCB_TRANSMISSION_ERROR;
1083                 } else
1084                         printf("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1085                                SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1086                 scsirate = ahc_inb(ahc, SCSIRATE);
1087                 if (silent == FALSE) {
1088                         printf("parity error detected %s. "
1089                                "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1090                                ahc_phase_table[i].phasemsg,
1091                                ahc_inw(ahc, SEQADDR0),
1092                                scsirate);
1093                         if ((ahc->features & AHC_DT) != 0) {
1094                                 if ((sstat2 & CRCVALERR) != 0)
1095                                         printf("\tCRC Value Mismatch\n");
1096                                 if ((sstat2 & CRCENDERR) != 0)
1097                                         printf("\tNo terminal CRC packet "
1098                                                "recevied\n");
1099                                 if ((sstat2 & CRCREQERR) != 0)
1100                                         printf("\tIllegal CRC packet "
1101                                                "request\n");
1102                                 if ((sstat2 & DUAL_EDGE_ERR) != 0)
1103                                         printf("\tUnexpected %sDT Data Phase\n",
1104                                                (scsirate & SINGLE_EDGE)
1105                                              ? "" : "non-");
1106                         }
1107                 }
1108
1109                 if ((ahc->features & AHC_DT) != 0
1110                  && (sstat2 & DUAL_EDGE_ERR) != 0) {
1111                         /*
1112                          * This error applies regardless of
1113                          * data direction, so ignore the value
1114                          * in the phase table.
1115                          */
1116                         mesg_out = MSG_INITIATOR_DET_ERR;
1117                 }
1118
1119                 /*
1120                  * We've set the hardware to assert ATN if we   
1121                  * get a parity error on "in" phases, so all we  
1122                  * need to do is stuff the message buffer with
1123                  * the appropriate message.  "In" phases have set
1124                  * mesg_out to something other than MSG_NOP.
1125                  */
1126                 if (mesg_out != MSG_NOOP) {
1127                         if (ahc->msg_type != MSG_TYPE_NONE)
1128                                 ahc->send_msg_perror = TRUE;
1129                         else
1130                                 ahc_outb(ahc, MSG_OUT, mesg_out);
1131                 }
1132                 /*
1133                  * Force a renegotiation with this target just in
1134                  * case we are out of sync for some external reason
1135                  * unknown (or unreported) by the target.
1136                  */
1137                 ahc_fetch_devinfo(ahc, &devinfo);
1138                 ahc_force_renegotiation(ahc, &devinfo);
1139
1140                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1141                 ahc_unpause(ahc);
1142         } else if ((status & SELTO) != 0) {
1143                 u_int   scbptr;
1144
1145                 /* Stop the selection */
1146                 ahc_outb(ahc, SCSISEQ, 0);
1147
1148                 /* No more pending messages */
1149                 ahc_clear_msg_state(ahc);
1150
1151                 /* Clear interrupt state */
1152                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1153                 ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1154
1155                 /*
1156                  * Although the driver does not care about the
1157                  * 'Selection in Progress' status bit, the busy
1158                  * LED does.  SELINGO is only cleared by a sucessfull
1159                  * selection, so we must manually clear it to insure
1160                  * the LED turns off just incase no future successful
1161                  * selections occur (e.g. no devices on the bus).
1162                  */
1163                 ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1164
1165                 scbptr = ahc_inb(ahc, WAITING_SCBH);
1166                 ahc_outb(ahc, SCBPTR, scbptr);
1167                 scb_index = ahc_inb(ahc, SCB_TAG);
1168
1169                 scb = ahc_lookup_scb(ahc, scb_index);
1170                 if (scb == NULL) {
1171                         printf("%s: ahc_intr - referenced scb not "
1172                                "valid during SELTO scb(%d, %d)\n",
1173                                ahc_name(ahc), scbptr, scb_index);
1174                         ahc_dump_card_state(ahc);
1175                 } else {
1176                         struct ahc_devinfo devinfo;
1177 #ifdef AHC_DEBUG
1178                         if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1179                                 ahc_print_path(ahc, scb);
1180                                 printf("Saw Selection Timeout for SCB 0x%x\n",
1181                                        scb_index);
1182                         }
1183 #endif
1184                         ahc_scb_devinfo(ahc, &devinfo, scb);
1185                         ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1186                         ahc_freeze_devq(ahc, scb);
1187
1188                         /*
1189                          * Cancel any pending transactions on the device
1190                          * now that it seems to be missing.  This will
1191                          * also revert us to async/narrow transfers until
1192                          * we can renegotiate with the device.
1193                          */
1194                         ahc_handle_devreset(ahc, &devinfo,
1195                                             CAM_SEL_TIMEOUT,
1196                                             "Selection Timeout",
1197                                             /*verbose_level*/1);
1198                 }
1199                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1200                 ahc_restart(ahc);
1201         } else if ((status & BUSFREE) != 0
1202                 && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1203                 struct  ahc_devinfo devinfo;
1204                 u_int   lastphase;
1205                 u_int   saved_scsiid;
1206                 u_int   saved_lun;
1207                 u_int   target;
1208                 u_int   initiator_role_id;
1209                 char    channel;
1210                 int     printerror;
1211
1212                 /*
1213                  * Clear our selection hardware as soon as possible.
1214                  * We may have an entry in the waiting Q for this target,
1215                  * that is affected by this busfree and we don't want to
1216                  * go about selecting the target while we handle the event.
1217                  */
1218                 ahc_outb(ahc, SCSISEQ,
1219                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1220
1221                 /*
1222                  * Disable busfree interrupts and clear the busfree
1223                  * interrupt status.  We do this here so that several
1224                  * bus transactions occur prior to clearing the SCSIINT
1225                  * latch.  It can take a bit for the clearing to take effect.
1226                  */
1227                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1228                 ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1229
1230                 /*
1231                  * Look at what phase we were last in.
1232                  * If its message out, chances are pretty good
1233                  * that the busfree was in response to one of
1234                  * our abort requests.
1235                  */
1236                 lastphase = ahc_inb(ahc, LASTPHASE);
1237                 saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1238                 saved_lun = ahc_inb(ahc, SAVED_LUN);
1239                 target = SCSIID_TARGET(ahc, saved_scsiid);
1240                 initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1241                 channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1242                 ahc_compile_devinfo(&devinfo, initiator_role_id,
1243                                     target, saved_lun, channel, ROLE_INITIATOR);
1244                 printerror = 1;
1245
1246                 if (lastphase == P_MESGOUT) {
1247                         u_int tag;
1248
1249                         tag = SCB_LIST_NULL;
1250                         if (ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT_TAG, TRUE)
1251                          || ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT, TRUE)) {
1252                                 if (ahc->msgout_buf[ahc->msgout_index - 1]
1253                                  == MSG_ABORT_TAG)
1254                                         tag = scb->hscb->tag;
1255                                 ahc_print_path(ahc, scb);
1256                                 printf("SCB %d - Abort%s Completed.\n",
1257                                        scb->hscb->tag, tag == SCB_LIST_NULL ?
1258                                        "" : " Tag");
1259                                 ahc_abort_scbs(ahc, target, channel,
1260                                                saved_lun, tag,
1261                                                ROLE_INITIATOR,
1262                                                CAM_REQ_ABORTED);
1263                                 printerror = 0;
1264                         } else if (ahc_sent_msg(ahc, AHCMSG_1B,
1265                                                 MSG_BUS_DEV_RESET, TRUE)) {
1266 #ifdef __FreeBSD__
1267                                 /*
1268                                  * Don't mark the user's request for this BDR
1269                                  * as completing with CAM_BDR_SENT.  CAM3
1270                                  * specifies CAM_REQ_CMP.
1271                                  */
1272                                 if (scb != NULL
1273                                  && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
1274                                  && ahc_match_scb(ahc, scb, target, channel,
1275                                                   CAM_LUN_WILDCARD,
1276                                                   SCB_LIST_NULL,
1277                                                   ROLE_INITIATOR)) {
1278                                         ahc_set_transaction_status(scb, CAM_REQ_CMP);
1279                                 }
1280 #endif
1281                                 ahc_compile_devinfo(&devinfo,
1282                                                     initiator_role_id,
1283                                                     target,
1284                                                     CAM_LUN_WILDCARD,
1285                                                     channel,
1286                                                     ROLE_INITIATOR);
1287                                 ahc_handle_devreset(ahc, &devinfo,
1288                                                     CAM_BDR_SENT,
1289                                                     "Bus Device Reset",
1290                                                     /*verbose_level*/0);
1291                                 printerror = 0;
1292                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1293                                                 MSG_EXT_PPR, FALSE)) {
1294                                 struct ahc_initiator_tinfo *tinfo;
1295                                 struct ahc_tmode_tstate *tstate;
1296
1297                                 /*
1298                                  * PPR Rejected.  Try non-ppr negotiation
1299                                  * and retry command.
1300                                  */
1301                                 tinfo = ahc_fetch_transinfo(ahc,
1302                                                             devinfo.channel,
1303                                                             devinfo.our_scsiid,
1304                                                             devinfo.target,
1305                                                             &tstate);
1306                                 tinfo->curr.transport_version = 2;
1307                                 tinfo->goal.transport_version = 2;
1308                                 tinfo->goal.ppr_options = 0;
1309                                 ahc_qinfifo_requeue_tail(ahc, scb);
1310                                 printerror = 0;
1311                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1312                                                 MSG_EXT_WDTR, FALSE)) {
1313                                 /*
1314                                  * Negotiation Rejected.  Go-narrow and
1315                                  * retry command.
1316                                  */
1317                                 ahc_set_width(ahc, &devinfo,
1318                                               MSG_EXT_WDTR_BUS_8_BIT,
1319                                               AHC_TRANS_CUR|AHC_TRANS_GOAL,
1320                                               /*paused*/TRUE);
1321                                 ahc_qinfifo_requeue_tail(ahc, scb);
1322                                 printerror = 0;
1323                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1324                                                 MSG_EXT_SDTR, FALSE)) {
1325                                 /*
1326                                  * Negotiation Rejected.  Go-async and
1327                                  * retry command.
1328                                  */
1329                                 ahc_set_syncrate(ahc, &devinfo,
1330                                                 /*syncrate*/NULL,
1331                                                 /*period*/0, /*offset*/0,
1332                                                 /*ppr_options*/0,
1333                                                 AHC_TRANS_CUR|AHC_TRANS_GOAL,
1334                                                 /*paused*/TRUE);
1335                                 ahc_qinfifo_requeue_tail(ahc, scb);
1336                                 printerror = 0;
1337                         }
1338                 }
1339                 if (printerror != 0) {
1340                         u_int i;
1341
1342                         if (scb != NULL) {
1343                                 u_int tag;
1344
1345                                 if ((scb->hscb->control & TAG_ENB) != 0)
1346                                         tag = scb->hscb->tag;
1347                                 else
1348                                         tag = SCB_LIST_NULL;
1349                                 ahc_print_path(ahc, scb);
1350                                 ahc_abort_scbs(ahc, target, channel,
1351                                                SCB_GET_LUN(scb), tag,
1352                                                ROLE_INITIATOR,
1353                                                CAM_UNEXP_BUSFREE);
1354                         } else {
1355                                 /*
1356                                  * We had not fully identified this connection,
1357                                  * so we cannot abort anything.
1358                                  */
1359                                 printf("%s: ", ahc_name(ahc));
1360                         }
1361                         for (i = 0; i < num_phases; i++) {
1362                                 if (lastphase == ahc_phase_table[i].phase)
1363                                         break;
1364                         }
1365                         if (lastphase != P_BUSFREE) {
1366                                 /*
1367                                  * Renegotiate with this device at the
1368                                  * next oportunity just in case this busfree
1369                                  * is due to a negotiation mismatch with the
1370                                  * device.
1371                                  */
1372                                 ahc_force_renegotiation(ahc, &devinfo);
1373                         }
1374                         printf("Unexpected busfree %s\n"
1375                                "SEQADDR == 0x%x\n",
1376                                ahc_phase_table[i].phasemsg,
1377                                ahc_inb(ahc, SEQADDR0)
1378                                 | (ahc_inb(ahc, SEQADDR1) << 8));
1379                 }
1380                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1381                 ahc_restart(ahc);
1382         } else {
1383                 printf("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1384                        ahc_name(ahc), status);
1385                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1386         }
1387 }
1388
1389 /*
1390  * Force renegotiation to occur the next time we initiate
1391  * a command to the current device.
1392  */
1393 static void
1394 ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1395 {
1396         struct  ahc_initiator_tinfo *targ_info;
1397         struct  ahc_tmode_tstate *tstate;
1398
1399         targ_info = ahc_fetch_transinfo(ahc,
1400                                         devinfo->channel,
1401                                         devinfo->our_scsiid,
1402                                         devinfo->target,
1403                                         &tstate);
1404         ahc_update_neg_request(ahc, devinfo, tstate,
1405                                targ_info, AHC_NEG_IF_NON_ASYNC);
1406 }
1407
1408 #define AHC_MAX_STEPS 2000
1409 void
1410 ahc_clear_critical_section(struct ahc_softc *ahc)
1411 {
1412         int     stepping;
1413         int     steps;
1414         u_int   simode0;
1415         u_int   simode1;
1416
1417         if (ahc->num_critical_sections == 0)
1418                 return;
1419
1420         stepping = FALSE;
1421         steps = 0;
1422         simode0 = 0;
1423         simode1 = 0;
1424         for (;;) {
1425                 struct  cs *cs;
1426                 u_int   seqaddr;
1427                 u_int   i;
1428
1429                 seqaddr = ahc_inb(ahc, SEQADDR0)
1430                         | (ahc_inb(ahc, SEQADDR1) << 8);
1431
1432                 /*
1433                  * Seqaddr represents the next instruction to execute, 
1434                  * so we are really executing the instruction just
1435                  * before it.
1436                  */
1437                 if (seqaddr != 0)
1438                         seqaddr -= 1;
1439                 cs = ahc->critical_sections;
1440                 for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
1441                         
1442                         if (cs->begin < seqaddr && cs->end >= seqaddr)
1443                                 break;
1444                 }
1445
1446                 if (i == ahc->num_critical_sections)
1447                         break;
1448
1449                 if (steps > AHC_MAX_STEPS) {
1450                         printf("%s: Infinite loop in critical section\n",
1451                                ahc_name(ahc));
1452                         ahc_dump_card_state(ahc);
1453                         panic("critical section loop");
1454                 }
1455
1456                 steps++;
1457                 if (stepping == FALSE) {
1458
1459                         /*
1460                          * Disable all interrupt sources so that the
1461                          * sequencer will not be stuck by a pausing
1462                          * interrupt condition while we attempt to
1463                          * leave a critical section.
1464                          */
1465                         simode0 = ahc_inb(ahc, SIMODE0);
1466                         ahc_outb(ahc, SIMODE0, 0);
1467                         simode1 = ahc_inb(ahc, SIMODE1);
1468                         if ((ahc->features & AHC_DT) != 0)
1469                                 /*
1470                                  * On DT class controllers, we
1471                                  * use the enhanced busfree logic.
1472                                  * Unfortunately we cannot re-enable
1473                                  * busfree detection within the
1474                                  * current connection, so we must
1475                                  * leave it on while single stepping.
1476                                  */
1477                                 ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
1478                         else
1479                                 ahc_outb(ahc, SIMODE1, 0);
1480                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1481                         ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
1482                         stepping = TRUE;
1483                 }
1484                 if ((ahc->features & AHC_DT) != 0) {
1485                         ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
1486                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1487                 }
1488                 ahc_outb(ahc, HCNTRL, ahc->unpause);
1489                 while (!ahc_is_paused(ahc))
1490                         ahc_delay(200);
1491         }
1492         if (stepping) {
1493                 ahc_outb(ahc, SIMODE0, simode0);
1494                 ahc_outb(ahc, SIMODE1, simode1);
1495                 ahc_outb(ahc, SEQCTL, ahc->seqctl);
1496         }
1497 }
1498
1499 /*
1500  * Clear any pending interrupt status.
1501  */
1502 void
1503 ahc_clear_intstat(struct ahc_softc *ahc)
1504 {
1505         /* Clear any interrupt conditions this may have caused */
1506         ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
1507                                 |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
1508                                 CLRREQINIT);
1509         ahc_flush_device_writes(ahc);
1510         ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
1511         ahc_flush_device_writes(ahc);
1512         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1513         ahc_flush_device_writes(ahc);
1514 }
1515
1516 /**************************** Debugging Routines ******************************/
1517 #ifdef AHC_DEBUG
1518 uint32_t ahc_debug = AHC_DEBUG_OPTS;
1519 #endif
1520
1521 void
1522 ahc_print_scb(struct scb *scb)
1523 {
1524         int i;
1525
1526         struct hardware_scb *hscb = scb->hscb;
1527
1528         printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
1529                (void *)scb,
1530                hscb->control,
1531                hscb->scsiid,
1532                hscb->lun,
1533                hscb->cdb_len);
1534         printf("Shared Data: ");
1535         for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
1536                 printf("%#02x", hscb->shared_data.cdb[i]);
1537         printf("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
1538                 ahc_le32toh(hscb->dataptr),
1539                 ahc_le32toh(hscb->datacnt),
1540                 ahc_le32toh(hscb->sgptr),
1541                 hscb->tag);
1542         if (scb->sg_count > 0) {
1543                 for (i = 0; i < scb->sg_count; i++) {
1544                         printf("sg[%d] - Addr 0x%x%x : Length %d\n",
1545                                i,
1546                                (ahc_le32toh(scb->sg_list[i].len) >> 24
1547                                 & SG_HIGH_ADDR_BITS),
1548                                ahc_le32toh(scb->sg_list[i].addr),
1549                                ahc_le32toh(scb->sg_list[i].len));
1550                 }
1551         }
1552 }
1553
1554 /************************* Transfer Negotiation *******************************/
1555 /*
1556  * Allocate per target mode instance (ID we respond to as a target)
1557  * transfer negotiation data structures.
1558  */
1559 static struct ahc_tmode_tstate *
1560 ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
1561 {
1562         struct ahc_tmode_tstate *master_tstate;
1563         struct ahc_tmode_tstate *tstate;
1564         int i;
1565
1566         master_tstate = ahc->enabled_targets[ahc->our_id];
1567         if (channel == 'B') {
1568                 scsi_id += 8;
1569                 master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
1570         }
1571         if (ahc->enabled_targets[scsi_id] != NULL
1572          && ahc->enabled_targets[scsi_id] != master_tstate)
1573                 panic("%s: ahc_alloc_tstate - Target already allocated",
1574                       ahc_name(ahc));
1575         tstate = (struct ahc_tmode_tstate*)malloc(sizeof(*tstate),
1576                                                    M_DEVBUF, M_NOWAIT);
1577         if (tstate == NULL)
1578                 return (NULL);
1579
1580         /*
1581          * If we have allocated a master tstate, copy user settings from
1582          * the master tstate (taken from SRAM or the EEPROM) for this
1583          * channel, but reset our current and goal settings to async/narrow
1584          * until an initiator talks to us.
1585          */
1586         if (master_tstate != NULL) {
1587                 memcpy(tstate, master_tstate, sizeof(*tstate));
1588                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
1589                 tstate->ultraenb = 0;
1590                 for (i = 0; i < AHC_NUM_TARGETS; i++) {
1591                         memset(&tstate->transinfo[i].curr, 0,
1592                               sizeof(tstate->transinfo[i].curr));
1593                         memset(&tstate->transinfo[i].goal, 0,
1594                               sizeof(tstate->transinfo[i].goal));
1595                 }
1596         } else
1597                 memset(tstate, 0, sizeof(*tstate));
1598         ahc->enabled_targets[scsi_id] = tstate;
1599         return (tstate);
1600 }
1601
1602 #ifdef AHC_TARGET_MODE
1603 /*
1604  * Free per target mode instance (ID we respond to as a target)
1605  * transfer negotiation data structures.
1606  */
1607 static void
1608 ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
1609 {
1610         struct ahc_tmode_tstate *tstate;
1611
1612         /*
1613          * Don't clean up our "master" tstate.
1614          * It has our default user settings.
1615          */
1616         if (((channel == 'B' && scsi_id == ahc->our_id_b)
1617           || (channel == 'A' && scsi_id == ahc->our_id))
1618          && force == FALSE)
1619                 return;
1620
1621         if (channel == 'B')
1622                 scsi_id += 8;
1623         tstate = ahc->enabled_targets[scsi_id];
1624         if (tstate != NULL)
1625                 free(tstate, M_DEVBUF);
1626         ahc->enabled_targets[scsi_id] = NULL;
1627 }
1628 #endif
1629
1630 /*
1631  * Called when we have an active connection to a target on the bus,
1632  * this function finds the nearest syncrate to the input period limited
1633  * by the capabilities of the bus connectivity of and sync settings for
1634  * the target.
1635  */
1636 struct ahc_syncrate *
1637 ahc_devlimited_syncrate(struct ahc_softc *ahc,
1638                         struct ahc_initiator_tinfo *tinfo,
1639                         u_int *period, u_int *ppr_options, role_t role)
1640 {
1641         struct  ahc_transinfo *transinfo;
1642         u_int   maxsync;
1643
1644         if ((ahc->features & AHC_ULTRA2) != 0) {
1645                 if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
1646                  && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
1647                         maxsync = AHC_SYNCRATE_DT;
1648                 } else {
1649                         maxsync = AHC_SYNCRATE_ULTRA;
1650                         /* Can't do DT on an SE bus */
1651                         *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1652                 }
1653         } else if ((ahc->features & AHC_ULTRA) != 0) {
1654                 maxsync = AHC_SYNCRATE_ULTRA;
1655         } else {
1656                 maxsync = AHC_SYNCRATE_FAST;
1657         }
1658         /*
1659          * Never allow a value higher than our current goal
1660          * period otherwise we may allow a target initiated
1661          * negotiation to go above the limit as set by the
1662          * user.  In the case of an initiator initiated
1663          * sync negotiation, we limit based on the user
1664          * setting.  This allows the system to still accept
1665          * incoming negotiations even if target initiated
1666          * negotiation is not performed.
1667          */
1668         if (role == ROLE_TARGET)
1669                 transinfo = &tinfo->user;
1670         else 
1671                 transinfo = &tinfo->goal;
1672         *ppr_options &= transinfo->ppr_options;
1673         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
1674                 maxsync = max(maxsync, (u_int)AHC_SYNCRATE_ULTRA2);
1675                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1676         }
1677         if (transinfo->period == 0) {
1678                 *period = 0;
1679                 *ppr_options = 0;
1680                 return (NULL);
1681         }
1682         *period = max(*period, (u_int)transinfo->period);
1683         return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
1684 }
1685
1686 /*
1687  * Look up the valid period to SCSIRATE conversion in our table.
1688  * Return the period and offset that should be sent to the target
1689  * if this was the beginning of an SDTR.
1690  */
1691 struct ahc_syncrate *
1692 ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
1693                   u_int *ppr_options, u_int maxsync)
1694 {
1695         struct ahc_syncrate *syncrate;
1696
1697         if ((ahc->features & AHC_DT) == 0)
1698                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1699
1700         /* Skip all DT only entries if DT is not available */
1701         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
1702          && maxsync < AHC_SYNCRATE_ULTRA2)
1703                 maxsync = AHC_SYNCRATE_ULTRA2;
1704
1705         /* Now set the maxsync based on the card capabilities
1706          * DT is already done above */
1707         if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
1708             && maxsync < AHC_SYNCRATE_ULTRA)
1709                 maxsync = AHC_SYNCRATE_ULTRA;
1710         if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
1711             && maxsync < AHC_SYNCRATE_FAST)
1712                 maxsync = AHC_SYNCRATE_FAST;
1713
1714         for (syncrate = &ahc_syncrates[maxsync];
1715              syncrate->rate != NULL;
1716              syncrate++) {
1717
1718                 /*
1719                  * The Ultra2 table doesn't go as low
1720                  * as for the Fast/Ultra cards.
1721                  */
1722                 if ((ahc->features & AHC_ULTRA2) != 0
1723                  && (syncrate->sxfr_u2 == 0))
1724                         break;
1725
1726                 if (*period <= syncrate->period) {
1727                         /*
1728                          * When responding to a target that requests
1729                          * sync, the requested rate may fall between
1730                          * two rates that we can output, but still be
1731                          * a rate that we can receive.  Because of this,
1732                          * we want to respond to the target with
1733                          * the same rate that it sent to us even
1734                          * if the period we use to send data to it
1735                          * is lower.  Only lower the response period
1736                          * if we must.
1737                          */
1738                         if (syncrate == &ahc_syncrates[maxsync])
1739                                 *period = syncrate->period;
1740
1741                         /*
1742                          * At some speeds, we only support
1743                          * ST transfers.
1744                          */
1745                         if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
1746                                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1747                         break;
1748                 }
1749         }
1750
1751         if ((*period == 0)
1752          || (syncrate->rate == NULL)
1753          || ((ahc->features & AHC_ULTRA2) != 0
1754           && (syncrate->sxfr_u2 == 0))) {
1755                 /* Use asynchronous transfers. */
1756                 *period = 0;
1757                 syncrate = NULL;
1758                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1759         }
1760         return (syncrate);
1761 }
1762
1763 /*
1764  * Convert from an entry in our syncrate table to the SCSI equivalent
1765  * sync "period" factor.
1766  */
1767 u_int
1768 ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
1769 {
1770         struct ahc_syncrate *syncrate;
1771
1772         if ((ahc->features & AHC_ULTRA2) != 0)
1773                 scsirate &= SXFR_ULTRA2;
1774         else
1775                 scsirate &= SXFR;
1776
1777         /* now set maxsync based on card capabilities */
1778         if ((ahc->features & AHC_DT) == 0 && maxsync < AHC_SYNCRATE_ULTRA2)
1779                 maxsync = AHC_SYNCRATE_ULTRA2;
1780         if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
1781             && maxsync < AHC_SYNCRATE_ULTRA)
1782                 maxsync = AHC_SYNCRATE_ULTRA;
1783         if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
1784             && maxsync < AHC_SYNCRATE_FAST)
1785                 maxsync = AHC_SYNCRATE_FAST;
1786
1787
1788         syncrate = &ahc_syncrates[maxsync];
1789         while (syncrate->rate != NULL) {
1790
1791                 if ((ahc->features & AHC_ULTRA2) != 0) {
1792                         if (syncrate->sxfr_u2 == 0)
1793                                 break;
1794                         else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
1795                                 return (syncrate->period);
1796                 } else if (scsirate == (syncrate->sxfr & SXFR)) {
1797                                 return (syncrate->period);
1798                 }
1799                 syncrate++;
1800         }
1801         return (0); /* async */
1802 }
1803
1804 /*
1805  * Truncate the given synchronous offset to a value the
1806  * current adapter type and syncrate are capable of.
1807  */
1808 void
1809 ahc_validate_offset(struct ahc_softc *ahc,
1810                     struct ahc_initiator_tinfo *tinfo,
1811                     struct ahc_syncrate *syncrate,
1812                     u_int *offset, int wide, role_t role)
1813 {
1814         u_int maxoffset;
1815
1816         /* Limit offset to what we can do */
1817         if (syncrate == NULL) {
1818                 maxoffset = 0;
1819         } else if ((ahc->features & AHC_ULTRA2) != 0) {
1820                 maxoffset = MAX_OFFSET_ULTRA2;
1821         } else {
1822                 if (wide)
1823                         maxoffset = MAX_OFFSET_16BIT;
1824                 else
1825                         maxoffset = MAX_OFFSET_8BIT;
1826         }
1827         *offset = min(*offset, maxoffset);
1828         if (tinfo != NULL) {
1829                 if (role == ROLE_TARGET)
1830                         *offset = min(*offset, (u_int)tinfo->user.offset);
1831                 else
1832                         *offset = min(*offset, (u_int)tinfo->goal.offset);
1833         }
1834 }
1835
1836 /*
1837  * Truncate the given transfer width parameter to a value the
1838  * current adapter type is capable of.
1839  */
1840 void
1841 ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
1842                    u_int *bus_width, role_t role)
1843 {
1844         switch (*bus_width) {
1845         default:
1846                 if (ahc->features & AHC_WIDE) {
1847                         /* Respond Wide */
1848                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
1849                         break;
1850                 }
1851                 /* FALLTHROUGH */
1852         case MSG_EXT_WDTR_BUS_8_BIT:
1853                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
1854                 break;
1855         }
1856         if (tinfo != NULL) {
1857                 if (role == ROLE_TARGET)
1858                         *bus_width = min((u_int)tinfo->user.width, *bus_width);
1859                 else
1860                         *bus_width = min((u_int)tinfo->goal.width, *bus_width);
1861         }
1862 }
1863
1864 /*
1865  * Update the bitmask of targets for which the controller should
1866  * negotiate with at the next convenient oportunity.  This currently
1867  * means the next time we send the initial identify messages for
1868  * a new transaction.
1869  */
1870 int
1871 ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
1872                        struct ahc_tmode_tstate *tstate,
1873                        struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
1874 {
1875         u_int auto_negotiate_orig;
1876
1877         auto_negotiate_orig = tstate->auto_negotiate;
1878         if (neg_type == AHC_NEG_ALWAYS) {
1879                 /*
1880                  * Force our "current" settings to be
1881                  * unknown so that unless a bus reset
1882                  * occurs the need to renegotiate is
1883                  * recorded persistently.
1884                  */
1885                 if ((ahc->features & AHC_WIDE) != 0)
1886                         tinfo->curr.width = AHC_WIDTH_UNKNOWN;
1887                 tinfo->curr.period = AHC_PERIOD_UNKNOWN;
1888                 tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
1889         }
1890         if (tinfo->curr.period != tinfo->goal.period
1891          || tinfo->curr.width != tinfo->goal.width
1892          || tinfo->curr.offset != tinfo->goal.offset
1893          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
1894          || (neg_type == AHC_NEG_IF_NON_ASYNC
1895           && (tinfo->goal.offset != 0
1896            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
1897            || tinfo->goal.ppr_options != 0)))
1898                 tstate->auto_negotiate |= devinfo->target_mask;
1899         else
1900                 tstate->auto_negotiate &= ~devinfo->target_mask;
1901
1902         return (auto_negotiate_orig != tstate->auto_negotiate);
1903 }
1904
1905 /*
1906  * Update the user/goal/curr tables of synchronous negotiation
1907  * parameters as well as, in the case of a current or active update,
1908  * any data structures on the host controller.  In the case of an
1909  * active update, the specified target is currently talking to us on
1910  * the bus, so the transfer parameter update must take effect
1911  * immediately.
1912  */
1913 void
1914 ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
1915                  struct ahc_syncrate *syncrate, u_int period,
1916                  u_int offset, u_int ppr_options, u_int type, int paused)
1917 {
1918         struct  ahc_initiator_tinfo *tinfo;
1919         struct  ahc_tmode_tstate *tstate;
1920         u_int   old_period;
1921         u_int   old_offset;
1922         u_int   old_ppr;
1923         int     active;
1924         int     update_needed;
1925
1926         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
1927         update_needed = 0;
1928
1929         if (syncrate == NULL) {
1930                 period = 0;
1931                 offset = 0;
1932         }
1933
1934         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
1935                                     devinfo->target, &tstate);
1936
1937         if ((type & AHC_TRANS_USER) != 0) {
1938                 tinfo->user.period = period;
1939                 tinfo->user.offset = offset;
1940                 tinfo->user.ppr_options = ppr_options;
1941         }
1942
1943         if ((type & AHC_TRANS_GOAL) != 0) {
1944                 tinfo->goal.period = period;
1945                 tinfo->goal.offset = offset;
1946                 tinfo->goal.ppr_options = ppr_options;
1947         }
1948
1949         old_period = tinfo->curr.period;
1950         old_offset = tinfo->curr.offset;
1951         old_ppr    = tinfo->curr.ppr_options;
1952
1953         if ((type & AHC_TRANS_CUR) != 0
1954          && (old_period != period
1955           || old_offset != offset
1956           || old_ppr != ppr_options)) {
1957                 u_int   scsirate;
1958
1959                 update_needed++;
1960                 scsirate = tinfo->scsirate;
1961                 if ((ahc->features & AHC_ULTRA2) != 0) {
1962
1963                         scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
1964                         if (syncrate != NULL) {
1965                                 scsirate |= syncrate->sxfr_u2;
1966                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
1967                                         scsirate |= ENABLE_CRC;
1968                                 else
1969                                         scsirate |= SINGLE_EDGE;
1970                         }
1971                 } else {
1972
1973                         scsirate &= ~(SXFR|SOFS);
1974                         /*
1975                          * Ensure Ultra mode is set properly for
1976                          * this target.
1977                          */
1978                         tstate->ultraenb &= ~devinfo->target_mask;
1979                         if (syncrate != NULL) {
1980                                 if (syncrate->sxfr & ULTRA_SXFR) {
1981                                         tstate->ultraenb |=
1982                                                 devinfo->target_mask;
1983                                 }
1984                                 scsirate |= syncrate->sxfr & SXFR;
1985                                 scsirate |= offset & SOFS;
1986                         }
1987                         if (active) {
1988                                 u_int sxfrctl0;
1989
1990                                 sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
1991                                 sxfrctl0 &= ~FAST20;
1992                                 if (tstate->ultraenb & devinfo->target_mask)
1993                                         sxfrctl0 |= FAST20;
1994                                 ahc_outb(ahc, SXFRCTL0, sxfrctl0);
1995                         }
1996                 }
1997                 if (active) {
1998                         ahc_outb(ahc, SCSIRATE, scsirate);
1999                         if ((ahc->features & AHC_ULTRA2) != 0)
2000                                 ahc_outb(ahc, SCSIOFFSET, offset);
2001                 }
2002
2003                 tinfo->scsirate = scsirate;
2004                 tinfo->curr.period = period;
2005                 tinfo->curr.offset = offset;
2006                 tinfo->curr.ppr_options = ppr_options;
2007
2008                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2009                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2010                 if (bootverbose) {
2011                         if (offset != 0) {
2012                                 printf("%s: target %d synchronous at %sMHz%s, "
2013                                        "offset = 0x%x\n", ahc_name(ahc),
2014                                        devinfo->target, syncrate->rate,
2015                                        (ppr_options & MSG_EXT_PPR_DT_REQ)
2016                                        ? " DT" : "", offset);
2017                         } else {
2018                                 printf("%s: target %d using "
2019                                        "asynchronous transfers\n",
2020                                        ahc_name(ahc), devinfo->target);
2021                         }
2022                 }
2023         }
2024
2025         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2026                                                 tinfo, AHC_NEG_TO_GOAL);
2027
2028         if (update_needed)
2029                 ahc_update_pending_scbs(ahc);
2030 }
2031
2032 /*
2033  * Update the user/goal/curr tables of wide negotiation
2034  * parameters as well as, in the case of a current or active update,
2035  * any data structures on the host controller.  In the case of an
2036  * active update, the specified target is currently talking to us on
2037  * the bus, so the transfer parameter update must take effect
2038  * immediately.
2039  */
2040 void
2041 ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2042               u_int width, u_int type, int paused)
2043 {
2044         struct  ahc_initiator_tinfo *tinfo;
2045         struct  ahc_tmode_tstate *tstate;
2046         u_int   oldwidth;
2047         int     active;
2048         int     update_needed;
2049
2050         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2051         update_needed = 0;
2052         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2053                                     devinfo->target, &tstate);
2054
2055         if ((type & AHC_TRANS_USER) != 0)
2056                 tinfo->user.width = width;
2057
2058         if ((type & AHC_TRANS_GOAL) != 0)
2059                 tinfo->goal.width = width;
2060
2061         oldwidth = tinfo->curr.width;
2062         if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2063                 u_int   scsirate;
2064
2065                 update_needed++;
2066                 scsirate =  tinfo->scsirate;
2067                 scsirate &= ~WIDEXFER;
2068                 if (width == MSG_EXT_WDTR_BUS_16_BIT)
2069                         scsirate |= WIDEXFER;
2070
2071                 tinfo->scsirate = scsirate;
2072
2073                 if (active)
2074                         ahc_outb(ahc, SCSIRATE, scsirate);
2075
2076                 tinfo->curr.width = width;
2077
2078                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2079                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2080                 if (bootverbose) {
2081                         printf("%s: target %d using %dbit transfers\n",
2082                                ahc_name(ahc), devinfo->target,
2083                                8 * (0x01 << width));
2084                 }
2085         }
2086
2087         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2088                                                 tinfo, AHC_NEG_TO_GOAL);
2089         if (update_needed)
2090                 ahc_update_pending_scbs(ahc);
2091 }
2092
2093 /*
2094  * Update the current state of tagged queuing for a given target.
2095  */
2096 static void
2097 ahc_set_tags(struct ahc_softc *ahc, struct scsi_cmnd *cmd,
2098              struct ahc_devinfo *devinfo, ahc_queue_alg alg)
2099 {
2100         struct scsi_device *sdev = cmd->device;
2101
2102         ahc_platform_set_tags(ahc, sdev, devinfo, alg);
2103         ahc_send_async(ahc, devinfo->channel, devinfo->target,
2104                        devinfo->lun, AC_TRANSFER_NEG);
2105 }
2106
2107 /*
2108  * When the transfer settings for a connection change, update any
2109  * in-transit SCBs to contain the new data so the hardware will
2110  * be set correctly during future (re)selections.
2111  */
2112 static void
2113 ahc_update_pending_scbs(struct ahc_softc *ahc)
2114 {
2115         struct  scb *pending_scb;
2116         int     pending_scb_count;
2117         int     i;
2118         int     paused;
2119         u_int   saved_scbptr;
2120
2121         /*
2122          * Traverse the pending SCB list and ensure that all of the
2123          * SCBs there have the proper settings.
2124          */
2125         pending_scb_count = 0;
2126         LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2127                 struct ahc_devinfo devinfo;
2128                 struct hardware_scb *pending_hscb;
2129                 struct ahc_initiator_tinfo *tinfo;
2130                 struct ahc_tmode_tstate *tstate;
2131
2132                 ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2133                 tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2134                                             devinfo.our_scsiid,
2135                                             devinfo.target, &tstate);
2136                 pending_hscb = pending_scb->hscb;
2137                 pending_hscb->control &= ~ULTRAENB;
2138                 if ((tstate->ultraenb & devinfo.target_mask) != 0)
2139                         pending_hscb->control |= ULTRAENB;
2140                 pending_hscb->scsirate = tinfo->scsirate;
2141                 pending_hscb->scsioffset = tinfo->curr.offset;
2142                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2143                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2144                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2145                         pending_hscb->control &= ~MK_MESSAGE;
2146                 }
2147                 ahc_sync_scb(ahc, pending_scb,
2148                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2149                 pending_scb_count++;
2150         }
2151
2152         if (pending_scb_count == 0)
2153                 return;
2154
2155         if (ahc_is_paused(ahc)) {
2156                 paused = 1;
2157         } else {
2158                 paused = 0;
2159                 ahc_pause(ahc);
2160         }
2161
2162         saved_scbptr = ahc_inb(ahc, SCBPTR);
2163         /* Ensure that the hscbs down on the card match the new information */
2164         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2165                 struct  hardware_scb *pending_hscb;
2166                 u_int   control;
2167                 u_int   scb_tag;
2168
2169                 ahc_outb(ahc, SCBPTR, i);
2170                 scb_tag = ahc_inb(ahc, SCB_TAG);
2171                 pending_scb = ahc_lookup_scb(ahc, scb_tag);
2172                 if (pending_scb == NULL)
2173                         continue;
2174
2175                 pending_hscb = pending_scb->hscb;
2176                 control = ahc_inb(ahc, SCB_CONTROL);
2177                 control &= ~(ULTRAENB|MK_MESSAGE);
2178                 control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2179                 ahc_outb(ahc, SCB_CONTROL, control);
2180                 ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2181                 ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2182         }
2183         ahc_outb(ahc, SCBPTR, saved_scbptr);
2184
2185         if (paused == 0)
2186                 ahc_unpause(ahc);
2187 }
2188
2189 /**************************** Pathing Information *****************************/
2190 static void
2191 ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2192 {
2193         u_int   saved_scsiid;
2194         role_t  role;
2195         int     our_id;
2196
2197         if (ahc_inb(ahc, SSTAT0) & TARGET)
2198                 role = ROLE_TARGET;
2199         else
2200                 role = ROLE_INITIATOR;
2201
2202         if (role == ROLE_TARGET
2203          && (ahc->features & AHC_MULTI_TID) != 0
2204          && (ahc_inb(ahc, SEQ_FLAGS)
2205            & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2206                 /* We were selected, so pull our id from TARGIDIN */
2207                 our_id = ahc_inb(ahc, TARGIDIN) & OID;
2208         } else if ((ahc->features & AHC_ULTRA2) != 0)
2209                 our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2210         else
2211                 our_id = ahc_inb(ahc, SCSIID) & OID;
2212
2213         saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2214         ahc_compile_devinfo(devinfo,
2215                             our_id,
2216                             SCSIID_TARGET(ahc, saved_scsiid),
2217                             ahc_inb(ahc, SAVED_LUN),
2218                             SCSIID_CHANNEL(ahc, saved_scsiid),
2219                             role);
2220 }
2221
2222 struct ahc_phase_table_entry*
2223 ahc_lookup_phase_entry(int phase)
2224 {
2225         struct ahc_phase_table_entry *entry;
2226         struct ahc_phase_table_entry *last_entry;
2227
2228         /*
2229          * num_phases doesn't include the default entry which
2230          * will be returned if the phase doesn't match.
2231          */
2232         last_entry = &ahc_phase_table[num_phases];
2233         for (entry = ahc_phase_table; entry < last_entry; entry++) {
2234                 if (phase == entry->phase)
2235                         break;
2236         }
2237         return (entry);
2238 }
2239
2240 void
2241 ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2242                     u_int lun, char channel, role_t role)
2243 {
2244         devinfo->our_scsiid = our_id;
2245         devinfo->target = target;
2246         devinfo->lun = lun;
2247         devinfo->target_offset = target;
2248         devinfo->channel = channel;
2249         devinfo->role = role;
2250         if (channel == 'B')
2251                 devinfo->target_offset += 8;
2252         devinfo->target_mask = (0x01 << devinfo->target_offset);
2253 }
2254
2255 void
2256 ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2257 {
2258         printf("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2259                devinfo->target, devinfo->lun);
2260 }
2261
2262 static void
2263 ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2264                 struct scb *scb)
2265 {
2266         role_t  role;
2267         int     our_id;
2268
2269         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2270         role = ROLE_INITIATOR;
2271         if ((scb->flags & SCB_TARGET_SCB) != 0)
2272                 role = ROLE_TARGET;
2273         ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2274                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2275 }
2276
2277
2278 /************************ Message Phase Processing ****************************/
2279 static void
2280 ahc_assert_atn(struct ahc_softc *ahc)
2281 {
2282         u_int scsisigo;
2283
2284         scsisigo = ATNO;
2285         if ((ahc->features & AHC_DT) == 0)
2286                 scsisigo |= ahc_inb(ahc, SCSISIGI);
2287         ahc_outb(ahc, SCSISIGO, scsisigo);
2288 }
2289
2290 /*
2291  * When an initiator transaction with the MK_MESSAGE flag either reconnects
2292  * or enters the initial message out phase, we are interrupted.  Fill our
2293  * outgoing message buffer with the appropriate message and beging handing
2294  * the message phase(s) manually.
2295  */
2296 static void
2297 ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2298                            struct scb *scb)
2299 {
2300         /*
2301          * To facilitate adding multiple messages together,
2302          * each routine should increment the index and len
2303          * variables instead of setting them explicitly.
2304          */
2305         ahc->msgout_index = 0;
2306         ahc->msgout_len = 0;
2307
2308         if ((scb->flags & SCB_DEVICE_RESET) == 0
2309          && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2310                 u_int identify_msg;
2311
2312                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2313                 if ((scb->hscb->control & DISCENB) != 0)
2314                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
2315                 ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2316                 ahc->msgout_len++;
2317
2318                 if ((scb->hscb->control & TAG_ENB) != 0) {
2319                         ahc->msgout_buf[ahc->msgout_index++] =
2320                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2321                         ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2322                         ahc->msgout_len += 2;
2323                 }
2324         }
2325
2326         if (scb->flags & SCB_DEVICE_RESET) {
2327                 ahc->msgout_buf[ahc->msgout_index++] = MSG_BUS_DEV_RESET;
2328                 ahc->msgout_len++;
2329                 ahc_print_path(ahc, scb);
2330                 printf("Bus Device Reset Message Sent\n");
2331                 /*
2332                  * Clear our selection hardware in advance of
2333                  * the busfree.  We may have an entry in the waiting
2334                  * Q for this target, and we don't want to go about
2335                  * selecting while we handle the busfree and blow it
2336                  * away.
2337                  */
2338                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2339         } else if ((scb->flags & SCB_ABORT) != 0) {
2340                 if ((scb->hscb->control & TAG_ENB) != 0)
2341                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT_TAG;
2342                 else
2343                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT;
2344                 ahc->msgout_len++;
2345                 ahc_print_path(ahc, scb);
2346                 printf("Abort%s Message Sent\n",
2347                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2348                 /*
2349                  * Clear our selection hardware in advance of
2350                  * the busfree.  We may have an entry in the waiting
2351                  * Q for this target, and we don't want to go about
2352                  * selecting while we handle the busfree and blow it
2353                  * away.
2354                  */
2355                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2356         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2357                 ahc_build_transfer_msg(ahc, devinfo);
2358         } else {
2359                 printf("ahc_intr: AWAITING_MSG for an SCB that "
2360                        "does not have a waiting message\n");
2361                 printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2362                        devinfo->target_mask);
2363                 panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2364                       "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2365                       ahc_inb(ahc, MSG_OUT), scb->flags);
2366         }
2367
2368         /*
2369          * Clear the MK_MESSAGE flag from the SCB so we aren't
2370          * asked to send this message again.
2371          */
2372         ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2373         scb->hscb->control &= ~MK_MESSAGE;
2374         ahc->msgout_index = 0;
2375         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2376 }
2377
2378 /*
2379  * Build an appropriate transfer negotiation message for the
2380  * currently active target.
2381  */
2382 static void
2383 ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2384 {
2385         /*
2386          * We need to initiate transfer negotiations.
2387          * If our current and goal settings are identical,
2388          * we want to renegotiate due to a check condition.
2389          */
2390         struct  ahc_initiator_tinfo *tinfo;
2391         struct  ahc_tmode_tstate *tstate;
2392         struct  ahc_syncrate *rate;
2393         int     dowide;
2394         int     dosync;
2395         int     doppr;
2396         u_int   period;
2397         u_int   ppr_options;
2398         u_int   offset;
2399
2400         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2401                                     devinfo->target, &tstate);
2402         /*
2403          * Filter our period based on the current connection.
2404          * If we can't perform DT transfers on this segment (not in LVD
2405          * mode for instance), then our decision to issue a PPR message
2406          * may change.
2407          */
2408         period = tinfo->goal.period;
2409         offset = tinfo->goal.offset;
2410         ppr_options = tinfo->goal.ppr_options;
2411         /* Target initiated PPR is not allowed in the SCSI spec */
2412         if (devinfo->role == ROLE_TARGET)
2413                 ppr_options = 0;
2414         rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2415                                        &ppr_options, devinfo->role);
2416         dowide = tinfo->curr.width != tinfo->goal.width;
2417         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2418         /*
2419          * Only use PPR if we have options that need it, even if the device
2420          * claims to support it.  There might be an expander in the way
2421          * that doesn't.
2422          */
2423         doppr = ppr_options != 0;
2424
2425         if (!dowide && !dosync && !doppr) {
2426                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
2427                 dosync = tinfo->goal.offset != 0;
2428         }
2429
2430         if (!dowide && !dosync && !doppr) {
2431                 /*
2432                  * Force async with a WDTR message if we have a wide bus,
2433                  * or just issue an SDTR with a 0 offset.
2434                  */
2435                 if ((ahc->features & AHC_WIDE) != 0)
2436                         dowide = 1;
2437                 else
2438                         dosync = 1;
2439
2440                 if (bootverbose) {
2441                         ahc_print_devinfo(ahc, devinfo);
2442                         printf("Ensuring async\n");
2443                 }
2444         }
2445
2446         /* Target initiated PPR is not allowed in the SCSI spec */
2447         if (devinfo->role == ROLE_TARGET)
2448                 doppr = 0;
2449
2450         /*
2451          * Both the PPR message and SDTR message require the
2452          * goal syncrate to be limited to what the target device
2453          * is capable of handling (based on whether an LVD->SE
2454          * expander is on the bus), so combine these two cases.
2455          * Regardless, guarantee that if we are using WDTR and SDTR
2456          * messages that WDTR comes first.
2457          */
2458         if (doppr || (dosync && !dowide)) {
2459
2460                 offset = tinfo->goal.offset;
2461                 ahc_validate_offset(ahc, tinfo, rate, &offset,
2462                                     doppr ? tinfo->goal.width
2463                                           : tinfo->curr.width,
2464                                     devinfo->role);
2465                 if (doppr) {
2466                         ahc_construct_ppr(ahc, devinfo, period, offset,
2467                                           tinfo->goal.width, ppr_options);
2468                 } else {
2469                         ahc_construct_sdtr(ahc, devinfo, period, offset);
2470                 }
2471         } else {
2472                 ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
2473         }
2474 }
2475
2476 /*
2477  * Build a synchronous negotiation message in our message
2478  * buffer based on the input parameters.
2479  */
2480 static void
2481 ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2482                    u_int period, u_int offset)
2483 {
2484         if (offset == 0)
2485                 period = AHC_ASYNC_XFER_PERIOD;
2486         ahc->msgout_index += spi_populate_sync_msg(
2487                         ahc->msgout_buf + ahc->msgout_index, period, offset);
2488         ahc->msgout_len += 5;
2489         if (bootverbose) {
2490                 printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
2491                        ahc_name(ahc), devinfo->channel, devinfo->target,
2492                        devinfo->lun, period, offset);
2493         }
2494 }
2495
2496 /*
2497  * Build a wide negotiation message in our message
2498  * buffer based on the input parameters.
2499  */
2500 static void
2501 ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2502                    u_int bus_width)
2503 {
2504         ahc->msgout_index += spi_populate_width_msg(
2505                         ahc->msgout_buf + ahc->msgout_index, bus_width);
2506         ahc->msgout_len += 4;
2507         if (bootverbose) {
2508                 printf("(%s:%c:%d:%d): Sending WDTR %x\n",
2509                        ahc_name(ahc), devinfo->channel, devinfo->target,
2510                        devinfo->lun, bus_width);
2511         }
2512 }
2513
2514 /*
2515  * Build a parallel protocol request message in our message
2516  * buffer based on the input parameters.
2517  */
2518 static void
2519 ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2520                   u_int period, u_int offset, u_int bus_width,
2521                   u_int ppr_options)
2522 {
2523         if (offset == 0)
2524                 period = AHC_ASYNC_XFER_PERIOD;
2525         ahc->msgout_index += spi_populate_ppr_msg(
2526                         ahc->msgout_buf + ahc->msgout_index, period, offset,
2527                         bus_width, ppr_options);
2528         ahc->msgout_len += 8;
2529         if (bootverbose) {
2530                 printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
2531                        "offset %x, ppr_options %x\n", ahc_name(ahc),
2532                        devinfo->channel, devinfo->target, devinfo->lun,
2533                        bus_width, period, offset, ppr_options);
2534         }
2535 }
2536
2537 /*
2538  * Clear any active message state.
2539  */
2540 static void
2541 ahc_clear_msg_state(struct ahc_softc *ahc)
2542 {
2543         ahc->msgout_len = 0;
2544         ahc->msgin_index = 0;
2545         ahc->msg_type = MSG_TYPE_NONE;
2546         if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
2547                 /*
2548                  * The target didn't care to respond to our
2549                  * message request, so clear ATN.
2550                  */
2551                 ahc_outb(ahc, CLRSINT1, CLRATNO);
2552         }
2553         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
2554         ahc_outb(ahc, SEQ_FLAGS2,
2555                  ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
2556 }
2557
2558 static void
2559 ahc_handle_proto_violation(struct ahc_softc *ahc)
2560 {
2561         struct  ahc_devinfo devinfo;
2562         struct  scb *scb;
2563         u_int   scbid;
2564         u_int   seq_flags;
2565         u_int   curphase;
2566         u_int   lastphase;
2567         int     found;
2568
2569         ahc_fetch_devinfo(ahc, &devinfo);
2570         scbid = ahc_inb(ahc, SCB_TAG);
2571         scb = ahc_lookup_scb(ahc, scbid);
2572         seq_flags = ahc_inb(ahc, SEQ_FLAGS);
2573         curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
2574         lastphase = ahc_inb(ahc, LASTPHASE);
2575         if ((seq_flags & NOT_IDENTIFIED) != 0) {
2576
2577                 /*
2578                  * The reconnecting target either did not send an
2579                  * identify message, or did, but we didn't find an SCB
2580                  * to match.
2581                  */
2582                 ahc_print_devinfo(ahc, &devinfo);
2583                 printf("Target did not send an IDENTIFY message. "
2584                        "LASTPHASE = 0x%x.\n", lastphase);
2585                 scb = NULL;
2586         } else if (scb == NULL) {
2587                 /*
2588                  * We don't seem to have an SCB active for this
2589                  * transaction.  Print an error and reset the bus.
2590                  */
2591                 ahc_print_devinfo(ahc, &devinfo);
2592                 printf("No SCB found during protocol violation\n");
2593                 goto proto_violation_reset;
2594         } else {
2595                 ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
2596                 if ((seq_flags & NO_CDB_SENT) != 0) {
2597                         ahc_print_path(ahc, scb);
2598                         printf("No or incomplete CDB sent to device.\n");
2599                 } else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
2600                         /*
2601                          * The target never bothered to provide status to
2602                          * us prior to completing the command.  Since we don't
2603                          * know the disposition of this command, we must attempt
2604                          * to abort it.  Assert ATN and prepare to send an abort
2605                          * message.
2606                          */
2607                         ahc_print_path(ahc, scb);
2608                         printf("Completed command without status.\n");
2609                 } else {
2610                         ahc_print_path(ahc, scb);
2611                         printf("Unknown protocol violation.\n");
2612                         ahc_dump_card_state(ahc);
2613                 }
2614         }
2615         if ((lastphase & ~P_DATAIN_DT) == 0
2616          || lastphase == P_COMMAND) {
2617 proto_violation_reset:
2618                 /*
2619                  * Target either went directly to data/command
2620                  * phase or didn't respond to our ATN.
2621                  * The only safe thing to do is to blow
2622                  * it away with a bus reset.
2623                  */
2624                 found = ahc_reset_channel(ahc, 'A', TRUE);
2625                 printf("%s: Issued Channel %c Bus Reset. "
2626                        "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
2627         } else {
2628                 /*
2629                  * Leave the selection hardware off in case
2630                  * this abort attempt will affect yet to
2631                  * be sent commands.
2632                  */
2633                 ahc_outb(ahc, SCSISEQ,
2634                          ahc_inb(ahc, SCSISEQ) & ~ENSELO);
2635                 ahc_assert_atn(ahc);
2636                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
2637                 if (scb == NULL) {
2638                         ahc_print_devinfo(ahc, &devinfo);
2639                         ahc->msgout_buf[0] = MSG_ABORT_TASK;
2640                         ahc->msgout_len = 1;
2641                         ahc->msgout_index = 0;
2642                         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2643                 } else {
2644                         ahc_print_path(ahc, scb);
2645                         scb->flags |= SCB_ABORT;
2646                 }
2647                 printf("Protocol violation %s.  Attempting to abort.\n",
2648                        ahc_lookup_phase_entry(curphase)->phasemsg);
2649         }
2650 }
2651
2652 /*
2653  * Manual message loop handler.
2654  */
2655 static void
2656 ahc_handle_message_phase(struct ahc_softc *ahc)
2657
2658         struct  ahc_devinfo devinfo;
2659         u_int   bus_phase;
2660         int     end_session;
2661
2662         ahc_fetch_devinfo(ahc, &devinfo);
2663         end_session = FALSE;
2664         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
2665
2666 reswitch:
2667         switch (ahc->msg_type) {
2668         case MSG_TYPE_INITIATOR_MSGOUT:
2669         {
2670                 int lastbyte;
2671                 int phasemis;
2672                 int msgdone;
2673
2674                 if (ahc->msgout_len == 0)
2675                         panic("HOST_MSG_LOOP interrupt with no active message");
2676
2677 #ifdef AHC_DEBUG
2678                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2679                         ahc_print_devinfo(ahc, &devinfo);
2680                         printf("INITIATOR_MSG_OUT");
2681                 }
2682 #endif
2683                 phasemis = bus_phase != P_MESGOUT;
2684                 if (phasemis) {
2685 #ifdef AHC_DEBUG
2686                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2687                                 printf(" PHASEMIS %s\n",
2688                                        ahc_lookup_phase_entry(bus_phase)
2689                                                              ->phasemsg);
2690                         }
2691 #endif
2692                         if (bus_phase == P_MESGIN) {
2693                                 /*
2694                                  * Change gears and see if
2695                                  * this messages is of interest to
2696                                  * us or should be passed back to
2697                                  * the sequencer.
2698                                  */
2699                                 ahc_outb(ahc, CLRSINT1, CLRATNO);
2700                                 ahc->send_msg_perror = FALSE;
2701                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
2702                                 ahc->msgin_index = 0;
2703                                 goto reswitch;
2704                         }
2705                         end_session = TRUE;
2706                         break;
2707                 }
2708
2709                 if (ahc->send_msg_perror) {
2710                         ahc_outb(ahc, CLRSINT1, CLRATNO);
2711                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2712 #ifdef AHC_DEBUG
2713                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2714                                 printf(" byte 0x%x\n", ahc->send_msg_perror);
2715 #endif
2716                         ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
2717                         break;
2718                 }
2719
2720                 msgdone = ahc->msgout_index == ahc->msgout_len;
2721                 if (msgdone) {
2722                         /*
2723                          * The target has requested a retry.
2724                          * Re-assert ATN, reset our message index to
2725                          * 0, and try again.
2726                          */
2727                         ahc->msgout_index = 0;
2728                         ahc_assert_atn(ahc);
2729                 }
2730
2731                 lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
2732                 if (lastbyte) {
2733                         /* Last byte is signified by dropping ATN */
2734                         ahc_outb(ahc, CLRSINT1, CLRATNO);
2735                 }
2736
2737                 /*
2738                  * Clear our interrupt status and present
2739                  * the next byte on the bus.
2740                  */
2741                 ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2742 #ifdef AHC_DEBUG
2743                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2744                         printf(" byte 0x%x\n",
2745                                ahc->msgout_buf[ahc->msgout_index]);
2746 #endif
2747                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
2748                 break;
2749         }
2750         case MSG_TYPE_INITIATOR_MSGIN:
2751         {
2752                 int phasemis;
2753                 int message_done;
2754
2755 #ifdef AHC_DEBUG
2756                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2757                         ahc_print_devinfo(ahc, &devinfo);
2758                         printf("INITIATOR_MSG_IN");
2759                 }
2760 #endif
2761                 phasemis = bus_phase != P_MESGIN;
2762                 if (phasemis) {
2763 #ifdef AHC_DEBUG
2764                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2765                                 printf(" PHASEMIS %s\n",
2766                                        ahc_lookup_phase_entry(bus_phase)
2767                                                              ->phasemsg);
2768                         }
2769 #endif
2770                         ahc->msgin_index = 0;
2771                         if (bus_phase == P_MESGOUT
2772                          && (ahc->send_msg_perror == TRUE
2773                           || (ahc->msgout_len != 0
2774                            && ahc->msgout_index == 0))) {
2775                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2776                                 goto reswitch;
2777                         }
2778                         end_session = TRUE;
2779                         break;
2780                 }
2781
2782                 /* Pull the byte in without acking it */
2783                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
2784 #ifdef AHC_DEBUG
2785                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2786                         printf(" byte 0x%x\n",
2787                                ahc->msgin_buf[ahc->msgin_index]);
2788 #endif
2789
2790                 message_done = ahc_parse_msg(ahc, &devinfo);
2791
2792                 if (message_done) {
2793                         /*
2794                          * Clear our incoming message buffer in case there
2795                          * is another message following this one.
2796                          */
2797                         ahc->msgin_index = 0;
2798
2799                         /*
2800                          * If this message illicited a response,
2801                          * assert ATN so the target takes us to the
2802                          * message out phase.
2803                          */
2804                         if (ahc->msgout_len != 0) {
2805 #ifdef AHC_DEBUG
2806                                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2807                                         ahc_print_devinfo(ahc, &devinfo);
2808                                         printf("Asserting ATN for response\n");
2809                                 }
2810 #endif
2811                                 ahc_assert_atn(ahc);
2812                         }
2813                 } else 
2814                         ahc->msgin_index++;
2815
2816                 if (message_done == MSGLOOP_TERMINATED) {
2817                         end_session = TRUE;
2818                 } else {
2819                         /* Ack the byte */
2820                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2821                         ahc_inb(ahc, SCSIDATL);
2822                 }
2823                 break;
2824         }
2825         case MSG_TYPE_TARGET_MSGIN:
2826         {
2827                 int msgdone;
2828                 int msgout_request;
2829
2830                 if (ahc->msgout_len == 0)
2831                         panic("Target MSGIN with no active message");
2832
2833                 /*
2834                  * If we interrupted a mesgout session, the initiator
2835                  * will not know this until our first REQ.  So, we
2836                  * only honor mesgout requests after we've sent our
2837                  * first byte.
2838                  */
2839                 if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
2840                  && ahc->msgout_index > 0)
2841                         msgout_request = TRUE;
2842                 else
2843                         msgout_request = FALSE;
2844
2845                 if (msgout_request) {
2846
2847                         /*
2848                          * Change gears and see if
2849                          * this messages is of interest to
2850                          * us or should be passed back to
2851                          * the sequencer.
2852                          */
2853                         ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
2854                         ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
2855                         ahc->msgin_index = 0;
2856                         /* Dummy read to REQ for first byte */
2857                         ahc_inb(ahc, SCSIDATL);
2858                         ahc_outb(ahc, SXFRCTL0,
2859                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2860                         break;
2861                 }
2862
2863                 msgdone = ahc->msgout_index == ahc->msgout_len;
2864                 if (msgdone) {
2865                         ahc_outb(ahc, SXFRCTL0,
2866                                  ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
2867                         end_session = TRUE;
2868                         break;
2869                 }
2870
2871                 /*
2872                  * Present the next byte on the bus.
2873                  */
2874                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2875                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
2876                 break;
2877         }
2878         case MSG_TYPE_TARGET_MSGOUT:
2879         {
2880                 int lastbyte;
2881                 int msgdone;
2882
2883                 /*
2884                  * The initiator signals that this is
2885                  * the last byte by dropping ATN.
2886                  */
2887                 lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
2888
2889                 /*
2890                  * Read the latched byte, but turn off SPIOEN first
2891                  * so that we don't inadvertently cause a REQ for the
2892                  * next byte.
2893                  */
2894                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
2895                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
2896                 msgdone = ahc_parse_msg(ahc, &devinfo);
2897                 if (msgdone == MSGLOOP_TERMINATED) {
2898                         /*
2899                          * The message is *really* done in that it caused
2900                          * us to go to bus free.  The sequencer has already
2901                          * been reset at this point, so pull the ejection
2902                          * handle.
2903                          */
2904                         return;
2905                 }
2906                 
2907                 ahc->msgin_index++;
2908
2909                 /*
2910                  * XXX Read spec about initiator dropping ATN too soon
2911                  *     and use msgdone to detect it.
2912                  */
2913                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
2914                         ahc->msgin_index = 0;
2915
2916                         /*
2917                          * If this message illicited a response, transition
2918                          * to the Message in phase and send it.
2919                          */
2920                         if (ahc->msgout_len != 0) {
2921                                 ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
2922                                 ahc_outb(ahc, SXFRCTL0,
2923                                          ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2924                                 ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
2925                                 ahc->msgin_index = 0;
2926                                 break;
2927                         }
2928                 }
2929
2930                 if (lastbyte)
2931                         end_session = TRUE;
2932                 else {
2933                         /* Ask for the next byte. */
2934                         ahc_outb(ahc, SXFRCTL0,
2935                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2936                 }
2937
2938                 break;
2939         }
2940         default:
2941                 panic("Unknown REQINIT message type");
2942         }
2943
2944         if (end_session) {
2945                 ahc_clear_msg_state(ahc);
2946                 ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
2947         } else
2948                 ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
2949 }
2950
2951 /*
2952  * See if we sent a particular extended message to the target.
2953  * If "full" is true, return true only if the target saw the full
2954  * message.  If "full" is false, return true if the target saw at
2955  * least the first byte of the message.
2956  */
2957 static int
2958 ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
2959 {
2960         int found;
2961         u_int index;
2962
2963         found = FALSE;
2964         index = 0;
2965
2966         while (index < ahc->msgout_len) {
2967                 if (ahc->msgout_buf[index] == MSG_EXTENDED) {
2968                         u_int end_index;
2969
2970                         end_index = index + 1 + ahc->msgout_buf[index + 1];
2971                         if (ahc->msgout_buf[index+2] == msgval
2972                          && type == AHCMSG_EXT) {
2973
2974                                 if (full) {
2975                                         if (ahc->msgout_index > end_index)
2976                                                 found = TRUE;
2977                                 } else if (ahc->msgout_index > index)
2978                                         found = TRUE;
2979                         }
2980                         index = end_index;
2981                 } else if (ahc->msgout_buf[index] >= MSG_SIMPLE_TASK
2982                         && ahc->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
2983
2984                         /* Skip tag type and tag id or residue param*/
2985                         index += 2;
2986                 } else {
2987                         /* Single byte message */
2988                         if (type == AHCMSG_1B
2989                          && ahc->msgout_buf[index] == msgval
2990                          && ahc->msgout_index > index)
2991                                 found = TRUE;
2992                         index++;
2993                 }
2994
2995                 if (found)
2996                         break;
2997         }
2998         return (found);
2999 }
3000
3001 /*
3002  * Wait for a complete incoming message, parse it, and respond accordingly.
3003  */
3004 static int
3005 ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3006 {
3007         struct  ahc_initiator_tinfo *tinfo;
3008         struct  ahc_tmode_tstate *tstate;
3009         int     reject;
3010         int     done;
3011         int     response;
3012         u_int   targ_scsirate;
3013
3014         done = MSGLOOP_IN_PROG;
3015         response = FALSE;
3016         reject = FALSE;
3017         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
3018                                     devinfo->target, &tstate);
3019         targ_scsirate = tinfo->scsirate;
3020
3021         /*
3022          * Parse as much of the message as is available,
3023          * rejecting it if we don't support it.  When
3024          * the entire message is available and has been
3025          * handled, return MSGLOOP_MSGCOMPLETE, indicating
3026          * that we have parsed an entire message.
3027          *
3028          * In the case of extended messages, we accept the length
3029          * byte outright and perform more checking once we know the
3030          * extended message type.
3031          */
3032         switch (ahc->msgin_buf[0]) {
3033         case MSG_DISCONNECT:
3034         case MSG_SAVEDATAPOINTER:
3035         case MSG_CMDCOMPLETE:
3036         case MSG_RESTOREPOINTERS:
3037         case MSG_IGN_WIDE_RESIDUE:
3038                 /*
3039                  * End our message loop as these are messages
3040                  * the sequencer handles on its own.
3041                  */
3042                 done = MSGLOOP_TERMINATED;
3043                 break;
3044         case MSG_MESSAGE_REJECT:
3045                 response = ahc_handle_msg_reject(ahc, devinfo);
3046                 /* FALLTHROUGH */
3047         case MSG_NOOP:
3048                 done = MSGLOOP_MSGCOMPLETE;
3049                 break;
3050         case MSG_EXTENDED:
3051         {
3052                 /* Wait for enough of the message to begin validation */
3053                 if (ahc->msgin_index < 2)
3054                         break;
3055                 switch (ahc->msgin_buf[2]) {
3056                 case MSG_EXT_SDTR:
3057                 {
3058                         struct   ahc_syncrate *syncrate;
3059                         u_int    period;
3060                         u_int    ppr_options;
3061                         u_int    offset;
3062                         u_int    saved_offset;
3063                         
3064                         if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3065                                 reject = TRUE;
3066                                 break;
3067                         }
3068
3069                         /*
3070                          * Wait until we have both args before validating
3071                          * and acting on this message.
3072                          *
3073                          * Add one to MSG_EXT_SDTR_LEN to account for
3074                          * the extended message preamble.
3075                          */
3076                         if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3077                                 break;
3078
3079                         period = ahc->msgin_buf[3];
3080                         ppr_options = 0;
3081                         saved_offset = offset = ahc->msgin_buf[4];
3082                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3083                                                            &ppr_options,
3084                                                            devinfo->role);
3085                         ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3086                                             targ_scsirate & WIDEXFER,
3087                                             devinfo->role);
3088                         if (bootverbose) {
3089                                 printf("(%s:%c:%d:%d): Received "
3090                                        "SDTR period %x, offset %x\n\t"
3091                                        "Filtered to period %x, offset %x\n",
3092                                        ahc_name(ahc), devinfo->channel,
3093                                        devinfo->target, devinfo->lun,
3094                                        ahc->msgin_buf[3], saved_offset,
3095                                        period, offset);
3096                         }
3097                         ahc_set_syncrate(ahc, devinfo, 
3098                                          syncrate, period,
3099                                          offset, ppr_options,
3100                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3101                                          /*paused*/TRUE);
3102
3103                         /*
3104                          * See if we initiated Sync Negotiation
3105                          * and didn't have to fall down to async
3106                          * transfers.
3107                          */
3108                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3109                                 /* We started it */
3110                                 if (saved_offset != offset) {
3111                                         /* Went too low - force async */
3112                                         reject = TRUE;
3113                                 }
3114                         } else {
3115                                 /*
3116                                  * Send our own SDTR in reply
3117                                  */
3118                                 if (bootverbose
3119                                  && devinfo->role == ROLE_INITIATOR) {
3120                                         printf("(%s:%c:%d:%d): Target "
3121                                                "Initiated SDTR\n",
3122                                                ahc_name(ahc), devinfo->channel,
3123                                                devinfo->target, devinfo->lun);
3124                                 }
3125                                 ahc->msgout_index = 0;
3126                                 ahc->msgout_len = 0;
3127                                 ahc_construct_sdtr(ahc, devinfo,
3128                                                    period, offset);
3129                                 ahc->msgout_index = 0;
3130                                 response = TRUE;
3131                         }
3132                         done = MSGLOOP_MSGCOMPLETE;
3133                         break;
3134                 }
3135                 case MSG_EXT_WDTR:
3136                 {
3137                         u_int bus_width;
3138                         u_int saved_width;
3139                         u_int sending_reply;
3140
3141                         sending_reply = FALSE;
3142                         if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3143                                 reject = TRUE;
3144                                 break;
3145                         }
3146
3147                         /*
3148                          * Wait until we have our arg before validating
3149                          * and acting on this message.
3150                          *
3151                          * Add one to MSG_EXT_WDTR_LEN to account for
3152                          * the extended message preamble.
3153                          */
3154                         if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3155                                 break;
3156
3157                         bus_width = ahc->msgin_buf[3];
3158                         saved_width = bus_width;
3159                         ahc_validate_width(ahc, tinfo, &bus_width,
3160                                            devinfo->role);
3161                         if (bootverbose) {
3162                                 printf("(%s:%c:%d:%d): Received WDTR "
3163                                        "%x filtered to %x\n",
3164                                        ahc_name(ahc), devinfo->channel,
3165                                        devinfo->target, devinfo->lun,
3166                                        saved_width, bus_width);
3167                         }
3168
3169                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3170                                 /*
3171                                  * Don't send a WDTR back to the
3172                                  * target, since we asked first.
3173                                  * If the width went higher than our
3174                                  * request, reject it.
3175                                  */
3176                                 if (saved_width > bus_width) {
3177                                         reject = TRUE;
3178                                         printf("(%s:%c:%d:%d): requested %dBit "
3179                                                "transfers.  Rejecting...\n",
3180                                                ahc_name(ahc), devinfo->channel,
3181                                                devinfo->target, devinfo->lun,
3182                                                8 * (0x01 << bus_width));
3183                                         bus_width = 0;
3184                                 }
3185                         } else {
3186                                 /*
3187                                  * Send our own WDTR in reply
3188                                  */
3189                                 if (bootverbose
3190                                  && devinfo->role == ROLE_INITIATOR) {
3191                                         printf("(%s:%c:%d:%d): Target "
3192                                                "Initiated WDTR\n",
3193                                                ahc_name(ahc), devinfo->channel,
3194                                                devinfo->target, devinfo->lun);
3195                                 }
3196                                 ahc->msgout_index = 0;
3197                                 ahc->msgout_len = 0;
3198                                 ahc_construct_wdtr(ahc, devinfo, bus_width);
3199                                 ahc->msgout_index = 0;
3200                                 response = TRUE;
3201                                 sending_reply = TRUE;
3202                         }
3203                         /*
3204                          * After a wide message, we are async, but
3205                          * some devices don't seem to honor this portion
3206                          * of the spec.  Force a renegotiation of the
3207                          * sync component of our transfer agreement even
3208                          * if our goal is async.  By updating our width
3209                          * after forcing the negotiation, we avoid
3210                          * renegotiating for width.
3211                          */
3212                         ahc_update_neg_request(ahc, devinfo, tstate,
3213                                                tinfo, AHC_NEG_ALWAYS);
3214                         ahc_set_width(ahc, devinfo, bus_width,
3215                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3216                                       /*paused*/TRUE);
3217                         if (sending_reply == FALSE && reject == FALSE) {
3218
3219                                 /*
3220                                  * We will always have an SDTR to send.
3221                                  */
3222                                 ahc->msgout_index = 0;
3223                                 ahc->msgout_len = 0;
3224                                 ahc_build_transfer_msg(ahc, devinfo);
3225                                 ahc->msgout_index = 0;
3226                                 response = TRUE;
3227                         }
3228                         done = MSGLOOP_MSGCOMPLETE;
3229                         break;
3230                 }
3231                 case MSG_EXT_PPR:
3232                 {
3233                         struct  ahc_syncrate *syncrate;
3234                         u_int   period;
3235                         u_int   offset;
3236                         u_int   bus_width;
3237                         u_int   ppr_options;
3238                         u_int   saved_width;
3239                         u_int   saved_offset;
3240                         u_int   saved_ppr_options;
3241
3242                         if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3243                                 reject = TRUE;
3244                                 break;
3245                         }
3246
3247                         /*
3248                          * Wait until we have all args before validating
3249                          * and acting on this message.
3250                          *
3251                          * Add one to MSG_EXT_PPR_LEN to account for
3252                          * the extended message preamble.
3253                          */
3254                         if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3255                                 break;
3256
3257                         period = ahc->msgin_buf[3];
3258                         offset = ahc->msgin_buf[5];
3259                         bus_width = ahc->msgin_buf[6];
3260                         saved_width = bus_width;
3261                         ppr_options = ahc->msgin_buf[7];
3262                         /*
3263                          * According to the spec, a DT only
3264                          * period factor with no DT option
3265                          * set implies async.
3266                          */
3267                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3268                          && period == 9)
3269                                 offset = 0;
3270                         saved_ppr_options = ppr_options;
3271                         saved_offset = offset;
3272
3273                         /*
3274                          * Mask out any options we don't support
3275                          * on any controller.  Transfer options are
3276                          * only available if we are negotiating wide.
3277                          */
3278                         ppr_options &= MSG_EXT_PPR_DT_REQ;
3279                         if (bus_width == 0)
3280                                 ppr_options = 0;
3281
3282                         ahc_validate_width(ahc, tinfo, &bus_width,
3283                                            devinfo->role);
3284                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3285                                                            &ppr_options,
3286                                                            devinfo->role);
3287                         ahc_validate_offset(ahc, tinfo, syncrate,
3288                                             &offset, bus_width,
3289                                             devinfo->role);
3290
3291                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, TRUE)) {
3292                                 /*
3293                                  * If we are unable to do any of the
3294                                  * requested options (we went too low),
3295                                  * then we'll have to reject the message.
3296                                  */
3297                                 if (saved_width > bus_width
3298                                  || saved_offset != offset
3299                                  || saved_ppr_options != ppr_options) {
3300                                         reject = TRUE;
3301                                         period = 0;
3302                                         offset = 0;
3303                                         bus_width = 0;
3304                                         ppr_options = 0;
3305                                         syncrate = NULL;
3306                                 }
3307                         } else {
3308                                 if (devinfo->role != ROLE_TARGET)
3309                                         printf("(%s:%c:%d:%d): Target "
3310                                                "Initiated PPR\n",
3311                                                ahc_name(ahc), devinfo->channel,
3312                                                devinfo->target, devinfo->lun);
3313                                 else
3314                                         printf("(%s:%c:%d:%d): Initiator "
3315                                                "Initiated PPR\n",
3316                                                ahc_name(ahc), devinfo->channel,
3317                                                devinfo->target, devinfo->lun);
3318                                 ahc->msgout_index = 0;
3319                                 ahc->msgout_len = 0;
3320                                 ahc_construct_ppr(ahc, devinfo, period, offset,
3321                                                   bus_width, ppr_options);
3322                                 ahc->msgout_index = 0;
3323                                 response = TRUE;
3324                         }
3325                         if (bootverbose) {
3326                                 printf("(%s:%c:%d:%d): Received PPR width %x, "
3327                                        "period %x, offset %x,options %x\n"
3328                                        "\tFiltered to width %x, period %x, "
3329                                        "offset %x, options %x\n",
3330                                        ahc_name(ahc), devinfo->channel,
3331                                        devinfo->target, devinfo->lun,
3332                                        saved_width, ahc->msgin_buf[3],
3333                                        saved_offset, saved_ppr_options,
3334                                        bus_width, period, offset, ppr_options);
3335                         }
3336                         ahc_set_width(ahc, devinfo, bus_width,
3337                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3338                                       /*paused*/TRUE);
3339                         ahc_set_syncrate(ahc, devinfo,
3340                                          syncrate, period,
3341                                          offset, ppr_options,
3342                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3343                                          /*paused*/TRUE);
3344                         done = MSGLOOP_MSGCOMPLETE;
3345                         break;
3346                 }
3347                 default:
3348                         /* Unknown extended message.  Reject it. */
3349                         reject = TRUE;
3350                         break;
3351                 }
3352                 break;
3353         }
3354 #ifdef AHC_TARGET_MODE
3355         case MSG_BUS_DEV_RESET:
3356                 ahc_handle_devreset(ahc, devinfo,
3357                                     CAM_BDR_SENT,
3358                                     "Bus Device Reset Received",
3359                                     /*verbose_level*/0);
3360                 ahc_restart(ahc);
3361                 done = MSGLOOP_TERMINATED;
3362                 break;
3363         case MSG_ABORT_TAG:
3364         case MSG_ABORT:
3365         case MSG_CLEAR_QUEUE:
3366         {
3367                 int tag;
3368
3369                 /* Target mode messages */
3370                 if (devinfo->role != ROLE_TARGET) {
3371                         reject = TRUE;
3372                         break;
3373                 }
3374                 tag = SCB_LIST_NULL;
3375                 if (ahc->msgin_buf[0] == MSG_ABORT_TAG)
3376                         tag = ahc_inb(ahc, INITIATOR_TAG);
3377                 ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3378                                devinfo->lun, tag, ROLE_TARGET,
3379                                CAM_REQ_ABORTED);
3380
3381                 tstate = ahc->enabled_targets[devinfo->our_scsiid];
3382                 if (tstate != NULL) {
3383                         struct ahc_tmode_lstate* lstate;
3384
3385                         lstate = tstate->enabled_luns[devinfo->lun];
3386                         if (lstate != NULL) {
3387                                 ahc_queue_lstate_event(ahc, lstate,
3388                                                        devinfo->our_scsiid,
3389                                                        ahc->msgin_buf[0],
3390                                                        /*arg*/tag);
3391                                 ahc_send_lstate_events(ahc, lstate);
3392                         }
3393                 }
3394                 ahc_restart(ahc);
3395                 done = MSGLOOP_TERMINATED;
3396                 break;
3397         }
3398 #endif
3399         case MSG_TERM_IO_PROC:
3400         default:
3401                 reject = TRUE;
3402                 break;
3403         }
3404
3405         if (reject) {
3406                 /*
3407                  * Setup to reject the message.
3408                  */
3409                 ahc->msgout_index = 0;
3410                 ahc->msgout_len = 1;
3411                 ahc->msgout_buf[0] = MSG_MESSAGE_REJECT;
3412                 done = MSGLOOP_MSGCOMPLETE;
3413                 response = TRUE;
3414         }
3415
3416         if (done != MSGLOOP_IN_PROG && !response)
3417                 /* Clear the outgoing message buffer */
3418                 ahc->msgout_len = 0;
3419
3420         return (done);
3421 }
3422
3423 /*
3424  * Process a message reject message.
3425  */
3426 static int
3427 ahc_handle_msg_reject(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3428 {
3429         /*
3430          * What we care about here is if we had an
3431          * outstanding SDTR or WDTR message for this
3432          * target.  If we did, this is a signal that
3433          * the target is refusing negotiation.
3434          */
3435         struct scb *scb;
3436         struct ahc_initiator_tinfo *tinfo;
3437         struct ahc_tmode_tstate *tstate;
3438         u_int scb_index;
3439         u_int last_msg;
3440         int   response = 0;
3441
3442         scb_index = ahc_inb(ahc, SCB_TAG);
3443         scb = ahc_lookup_scb(ahc, scb_index);
3444         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel,
3445                                     devinfo->our_scsiid,
3446                                     devinfo->target, &tstate);
3447         /* Might be necessary */
3448         last_msg = ahc_inb(ahc, LAST_MSG);
3449
3450         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
3451                 /*
3452                  * Target does not support the PPR message.
3453                  * Attempt to negotiate SPI-2 style.
3454                  */
3455                 if (bootverbose) {
3456                         printf("(%s:%c:%d:%d): PPR Rejected. "
3457                                "Trying WDTR/SDTR\n",
3458                                ahc_name(ahc), devinfo->channel,
3459                                devinfo->target, devinfo->lun);
3460                 }
3461                 tinfo->goal.ppr_options = 0;
3462                 tinfo->curr.transport_version = 2;
3463                 tinfo->goal.transport_version = 2;
3464                 ahc->msgout_index = 0;
3465                 ahc->msgout_len = 0;
3466                 ahc_build_transfer_msg(ahc, devinfo);
3467                 ahc->msgout_index = 0;
3468                 response = 1;
3469         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
3470
3471                 /* note 8bit xfers */
3472                 printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
3473                        "8bit transfers\n", ahc_name(ahc),
3474                        devinfo->channel, devinfo->target, devinfo->lun);
3475                 ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3476                               AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3477                               /*paused*/TRUE);
3478                 /*
3479                  * No need to clear the sync rate.  If the target
3480                  * did not accept the command, our syncrate is
3481                  * unaffected.  If the target started the negotiation,
3482                  * but rejected our response, we already cleared the
3483                  * sync rate before sending our WDTR.
3484                  */
3485                 if (tinfo->goal.offset != tinfo->curr.offset) {
3486
3487                         /* Start the sync negotiation */
3488                         ahc->msgout_index = 0;
3489                         ahc->msgout_len = 0;
3490                         ahc_build_transfer_msg(ahc, devinfo);
3491                         ahc->msgout_index = 0;
3492                         response = 1;
3493                 }
3494         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
3495                 /* note asynch xfers and clear flag */
3496                 ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL, /*period*/0,
3497                                  /*offset*/0, /*ppr_options*/0,
3498                                  AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3499                                  /*paused*/TRUE);
3500                 printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
3501                        "Using asynchronous transfers\n",
3502                        ahc_name(ahc), devinfo->channel,
3503                        devinfo->target, devinfo->lun);
3504         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
3505                 int tag_type;
3506                 int mask;
3507
3508                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
3509
3510                 if (tag_type == MSG_SIMPLE_TASK) {
3511                         printf("(%s:%c:%d:%d): refuses tagged commands.  "
3512                                "Performing non-tagged I/O\n", ahc_name(ahc),
3513                                devinfo->channel, devinfo->target, devinfo->lun);
3514                         ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_NONE);
3515                         mask = ~0x23;
3516                 } else {
3517                         printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
3518                                "Performing simple queue tagged I/O only\n",
3519                                ahc_name(ahc), devinfo->channel, devinfo->target,
3520                                devinfo->lun, tag_type == MSG_ORDERED_TASK
3521                                ? "ordered" : "head of queue");
3522                         ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_BASIC);
3523                         mask = ~0x03;
3524                 }
3525
3526                 /*
3527                  * Resend the identify for this CCB as the target
3528                  * may believe that the selection is invalid otherwise.
3529                  */
3530                 ahc_outb(ahc, SCB_CONTROL,
3531                          ahc_inb(ahc, SCB_CONTROL) & mask);
3532                 scb->hscb->control &= mask;
3533                 ahc_set_transaction_tag(scb, /*enabled*/FALSE,
3534                                         /*type*/MSG_SIMPLE_TASK);
3535                 ahc_outb(ahc, MSG_OUT, MSG_IDENTIFYFLAG);
3536                 ahc_assert_atn(ahc);
3537
3538                 /*
3539                  * This transaction is now at the head of
3540                  * the untagged queue for this target.
3541                  */
3542                 if ((ahc->flags & AHC_SCB_BTT) == 0) {
3543                         struct scb_tailq *untagged_q;
3544
3545                         untagged_q =
3546                             &(ahc->untagged_queues[devinfo->target_offset]);
3547                         TAILQ_INSERT_HEAD(untagged_q, scb, links.tqe);
3548                         scb->flags |= SCB_UNTAGGEDQ;
3549                 }
3550                 ahc_busy_tcl(ahc, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
3551                              scb->hscb->tag);
3552
3553                 /*
3554                  * Requeue all tagged commands for this target
3555                  * currently in our posession so they can be
3556                  * converted to untagged commands.
3557                  */
3558                 ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
3559                                    SCB_GET_CHANNEL(ahc, scb),
3560                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
3561                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
3562                                    SEARCH_COMPLETE);
3563         } else {
3564                 /*
3565                  * Otherwise, we ignore it.
3566                  */
3567                 printf("%s:%c:%d: Message reject for %x -- ignored\n",
3568                        ahc_name(ahc), devinfo->channel, devinfo->target,
3569                        last_msg);
3570         }
3571         return (response);
3572 }
3573
3574 /*
3575  * Process an ingnore wide residue message.
3576  */
3577 static void
3578 ahc_handle_ign_wide_residue(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3579 {
3580         u_int scb_index;
3581         struct scb *scb;
3582
3583         scb_index = ahc_inb(ahc, SCB_TAG);
3584         scb = ahc_lookup_scb(ahc, scb_index);
3585         /*
3586          * XXX Actually check data direction in the sequencer?
3587          * Perhaps add datadir to some spare bits in the hscb?
3588          */
3589         if ((ahc_inb(ahc, SEQ_FLAGS) & DPHASE) == 0
3590          || ahc_get_transfer_dir(scb) != CAM_DIR_IN) {
3591                 /*
3592                  * Ignore the message if we haven't
3593                  * seen an appropriate data phase yet.
3594                  */
3595         } else {
3596                 /*
3597                  * If the residual occurred on the last
3598                  * transfer and the transfer request was
3599                  * expected to end on an odd count, do
3600                  * nothing.  Otherwise, subtract a byte
3601                  * and update the residual count accordingly.
3602                  */
3603                 uint32_t sgptr;
3604
3605                 sgptr = ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
3606                 if ((sgptr & SG_LIST_NULL) != 0
3607                  && (ahc_inb(ahc, SCB_LUN) & SCB_XFERLEN_ODD) != 0) {
3608                         /*
3609                          * If the residual occurred on the last
3610                          * transfer and the transfer request was
3611                          * expected to end on an odd count, do
3612                          * nothing.
3613                          */
3614                 } else {
3615                         struct ahc_dma_seg *sg;
3616                         uint32_t data_cnt;
3617                         uint32_t data_addr;
3618                         uint32_t sglen;
3619
3620                         /* Pull in all of the sgptr */
3621                         sgptr = ahc_inl(ahc, SCB_RESIDUAL_SGPTR);
3622                         data_cnt = ahc_inl(ahc, SCB_RESIDUAL_DATACNT);
3623
3624                         if ((sgptr & SG_LIST_NULL) != 0) {
3625                                 /*
3626                                  * The residual data count is not updated
3627                                  * for the command run to completion case.
3628                                  * Explicitly zero the count.
3629                                  */
3630                                 data_cnt &= ~AHC_SG_LEN_MASK;
3631                         }
3632
3633                         data_addr = ahc_inl(ahc, SHADDR);
3634
3635                         data_cnt += 1;
3636                         data_addr -= 1;
3637                         sgptr &= SG_PTR_MASK;
3638
3639                         sg = ahc_sg_bus_to_virt(scb, sgptr);
3640
3641                         /*
3642                          * The residual sg ptr points to the next S/G
3643                          * to load so we must go back one.
3644                          */
3645                         sg--;
3646                         sglen = ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
3647                         if (sg != scb->sg_list
3648                          && sglen < (data_cnt & AHC_SG_LEN_MASK)) {
3649
3650                                 sg--;
3651                                 sglen = ahc_le32toh(sg->len);
3652                                 /*
3653                                  * Preserve High Address and SG_LIST bits
3654                                  * while setting the count to 1.
3655                                  */
3656                                 data_cnt = 1 | (sglen & (~AHC_SG_LEN_MASK));
3657                                 data_addr = ahc_le32toh(sg->addr)
3658                                           + (sglen & AHC_SG_LEN_MASK) - 1;
3659
3660                                 /*
3661                                  * Increment sg so it points to the
3662                                  * "next" sg.
3663                                  */
3664                                 sg++;
3665                                 sgptr = ahc_sg_virt_to_bus(scb, sg);
3666                         }
3667                         ahc_outl(ahc, SCB_RESIDUAL_SGPTR, sgptr);
3668                         ahc_outl(ahc, SCB_RESIDUAL_DATACNT, data_cnt);
3669                         /*
3670                          * Toggle the "oddness" of the transfer length
3671                          * to handle this mid-transfer ignore wide
3672                          * residue.  This ensures that the oddness is
3673                          * correct for subsequent data transfers.
3674                          */
3675                         ahc_outb(ahc, SCB_LUN,
3676                                  ahc_inb(ahc, SCB_LUN) ^ SCB_XFERLEN_ODD);
3677                 }
3678         }
3679 }
3680
3681
3682 /*
3683  * Reinitialize the data pointers for the active transfer
3684  * based on its current residual.
3685  */
3686 static void
3687 ahc_reinitialize_dataptrs(struct ahc_softc *ahc)
3688 {
3689         struct   scb *scb;
3690         struct   ahc_dma_seg *sg;
3691         u_int    scb_index;
3692         uint32_t sgptr;
3693         uint32_t resid;
3694         uint32_t dataptr;
3695
3696         scb_index = ahc_inb(ahc, SCB_TAG);
3697         scb = ahc_lookup_scb(ahc, scb_index);
3698         sgptr = (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 3) << 24)
3699               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 2) << 16)
3700               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 1) << 8)
3701               | ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
3702
3703         sgptr &= SG_PTR_MASK;
3704         sg = ahc_sg_bus_to_virt(scb, sgptr);
3705
3706         /* The residual sg_ptr always points to the next sg */
3707         sg--;
3708
3709         resid = (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 2) << 16)
3710               | (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 1) << 8)
3711               | ahc_inb(ahc, SCB_RESIDUAL_DATACNT);
3712
3713         dataptr = ahc_le32toh(sg->addr)
3714                 + (ahc_le32toh(sg->len) & AHC_SG_LEN_MASK)
3715                 - resid;
3716         if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
3717                 u_int dscommand1;
3718
3719                 dscommand1 = ahc_inb(ahc, DSCOMMAND1);
3720                 ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
3721                 ahc_outb(ahc, HADDR,
3722                          (ahc_le32toh(sg->len) >> 24) & SG_HIGH_ADDR_BITS);
3723                 ahc_outb(ahc, DSCOMMAND1, dscommand1);
3724         }
3725         ahc_outb(ahc, HADDR + 3, dataptr >> 24);
3726         ahc_outb(ahc, HADDR + 2, dataptr >> 16);
3727         ahc_outb(ahc, HADDR + 1, dataptr >> 8);
3728         ahc_outb(ahc, HADDR, dataptr);
3729         ahc_outb(ahc, HCNT + 2, resid >> 16);
3730         ahc_outb(ahc, HCNT + 1, resid >> 8);
3731         ahc_outb(ahc, HCNT, resid);
3732         if ((ahc->features & AHC_ULTRA2) == 0) {
3733                 ahc_outb(ahc, STCNT + 2, resid >> 16);
3734                 ahc_outb(ahc, STCNT + 1, resid >> 8);
3735                 ahc_outb(ahc, STCNT, resid);
3736         }
3737 }
3738
3739 /*
3740  * Handle the effects of issuing a bus device reset message.
3741  */
3742 static void
3743 ahc_handle_devreset(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3744                     cam_status status, char *message, int verbose_level)
3745 {
3746 #ifdef AHC_TARGET_MODE
3747         struct ahc_tmode_tstate* tstate;
3748         u_int lun;
3749 #endif
3750         int found;
3751
3752         found = ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3753                                CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
3754                                status);
3755
3756 #ifdef AHC_TARGET_MODE
3757         /*
3758          * Send an immediate notify ccb to all target mord peripheral
3759          * drivers affected by this action.
3760          */
3761         tstate = ahc->enabled_targets[devinfo->our_scsiid];
3762         if (tstate != NULL) {
3763                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
3764                         struct ahc_tmode_lstate* lstate;
3765
3766                         lstate = tstate->enabled_luns[lun];
3767                         if (lstate == NULL)
3768                                 continue;
3769
3770                         ahc_queue_lstate_event(ahc, lstate, devinfo->our_scsiid,
3771                                                MSG_BUS_DEV_RESET, /*arg*/0);
3772                         ahc_send_lstate_events(ahc, lstate);
3773                 }
3774         }
3775 #endif
3776
3777         /*
3778          * Go back to async/narrow transfers and renegotiate.
3779          */
3780         ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3781                       AHC_TRANS_CUR, /*paused*/TRUE);
3782         ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL,
3783                          /*period*/0, /*offset*/0, /*ppr_options*/0,
3784                          AHC_TRANS_CUR, /*paused*/TRUE);
3785         
3786         if (status != CAM_SEL_TIMEOUT)
3787                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
3788                                CAM_LUN_WILDCARD, AC_SENT_BDR);
3789
3790         if (message != NULL
3791          && (verbose_level <= bootverbose))
3792                 printf("%s: %s on %c:%d. %d SCBs aborted\n", ahc_name(ahc),
3793                        message, devinfo->channel, devinfo->target, found);
3794 }
3795
3796 #ifdef AHC_TARGET_MODE
3797 static void
3798 ahc_setup_target_msgin(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3799                        struct scb *scb)
3800 {
3801
3802         /*              
3803          * To facilitate adding multiple messages together,
3804          * each routine should increment the index and len
3805          * variables instead of setting them explicitly.
3806          */             
3807         ahc->msgout_index = 0;
3808         ahc->msgout_len = 0;
3809
3810         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
3811                 ahc_build_transfer_msg(ahc, devinfo);
3812         else
3813                 panic("ahc_intr: AWAITING target message with no message");
3814
3815         ahc->msgout_index = 0;
3816         ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3817 }
3818 #endif
3819 /**************************** Initialization **********************************/
3820 /*
3821  * Allocate a controller structure for a new device
3822  * and perform initial initializion.
3823  */
3824 struct ahc_softc *
3825 ahc_alloc(void *platform_arg, char *name)
3826 {
3827         struct  ahc_softc *ahc;
3828         int     i;
3829
3830 #ifndef __FreeBSD__
3831         ahc = malloc(sizeof(*ahc), M_DEVBUF, M_NOWAIT);
3832         if (!ahc) {
3833                 printf("aic7xxx: cannot malloc softc!\n");
3834                 free(name, M_DEVBUF);
3835                 return NULL;
3836         }
3837 #else
3838         ahc = device_get_softc((device_t)platform_arg);
3839 #endif
3840         memset(ahc, 0, sizeof(*ahc));
3841         ahc->seep_config = malloc(sizeof(*ahc->seep_config),
3842                                   M_DEVBUF, M_NOWAIT);
3843         if (ahc->seep_config == NULL) {
3844 #ifndef __FreeBSD__
3845                 free(ahc, M_DEVBUF);
3846 #endif
3847                 free(name, M_DEVBUF);
3848                 return (NULL);
3849         }
3850         LIST_INIT(&ahc->pending_scbs);
3851         /* We don't know our unit number until the OSM sets it */
3852         ahc->name = name;
3853         ahc->unit = -1;
3854         ahc->description = NULL;
3855         ahc->channel = 'A';
3856         ahc->channel_b = 'B';
3857         ahc->chip = AHC_NONE;
3858         ahc->features = AHC_FENONE;
3859         ahc->bugs = AHC_BUGNONE;
3860         ahc->flags = AHC_FNONE;
3861         /*
3862          * Default to all error reporting enabled with the
3863          * sequencer operating at its fastest speed.
3864          * The bus attach code may modify this.
3865          */
3866         ahc->seqctl = FASTMODE;
3867
3868         for (i = 0; i < AHC_NUM_TARGETS; i++)
3869                 TAILQ_INIT(&ahc->untagged_queues[i]);
3870         if (ahc_platform_alloc(ahc, platform_arg) != 0) {
3871                 ahc_free(ahc);
3872                 ahc = NULL;
3873         }
3874         return (ahc);
3875 }
3876
3877 int
3878 ahc_softc_init(struct ahc_softc *ahc)
3879 {
3880
3881         /* The IRQMS bit is only valid on VL and EISA chips */
3882         if ((ahc->chip & AHC_PCI) == 0)
3883                 ahc->unpause = ahc_inb(ahc, HCNTRL) & IRQMS;
3884         else
3885                 ahc->unpause = 0;
3886         ahc->pause = ahc->unpause | PAUSE; 
3887         /* XXX The shared scb data stuff should be deprecated */
3888         if (ahc->scb_data == NULL) {
3889                 ahc->scb_data = malloc(sizeof(*ahc->scb_data),
3890                                        M_DEVBUF, M_NOWAIT);
3891                 if (ahc->scb_data == NULL)
3892                         return (ENOMEM);
3893                 memset(ahc->scb_data, 0, sizeof(*ahc->scb_data));
3894         }
3895
3896         return (0);
3897 }
3898
3899 void
3900 ahc_set_unit(struct ahc_softc *ahc, int unit)
3901 {
3902         ahc->unit = unit;
3903 }
3904
3905 void
3906 ahc_set_name(struct ahc_softc *ahc, char *name)
3907 {
3908         if (ahc->name != NULL)
3909                 free(ahc->name, M_DEVBUF);
3910         ahc->name = name;
3911 }
3912
3913 void
3914 ahc_free(struct ahc_softc *ahc)
3915 {
3916         int i;
3917
3918         switch (ahc->init_level) {
3919         default:
3920         case 5:
3921                 ahc_shutdown(ahc);
3922                 /* FALLTHROUGH */
3923         case 4:
3924                 ahc_dmamap_unload(ahc, ahc->shared_data_dmat,
3925                                   ahc->shared_data_dmamap);
3926                 /* FALLTHROUGH */
3927         case 3:
3928                 ahc_dmamem_free(ahc, ahc->shared_data_dmat, ahc->qoutfifo,
3929                                 ahc->shared_data_dmamap);
3930                 ahc_dmamap_destroy(ahc, ahc->shared_data_dmat,
3931                                    ahc->shared_data_dmamap);
3932                 /* FALLTHROUGH */
3933         case 2:
3934                 ahc_dma_tag_destroy(ahc, ahc->shared_data_dmat);
3935         case 1:
3936 #ifndef __linux__
3937                 ahc_dma_tag_destroy(ahc, ahc->buffer_dmat);
3938 #endif
3939                 break;
3940         case 0:
3941                 break;
3942         }
3943
3944 #ifndef __linux__
3945         ahc_dma_tag_destroy(ahc, ahc->parent_dmat);
3946 #endif
3947         ahc_platform_free(ahc);
3948         ahc_fini_scbdata(ahc);
3949         for (i = 0; i < AHC_NUM_TARGETS; i++) {
3950                 struct ahc_tmode_tstate *tstate;
3951
3952                 tstate = ahc->enabled_targets[i];
3953                 if (tstate != NULL) {
3954 #ifdef AHC_TARGET_MODE
3955                         int j;
3956
3957                         for (j = 0; j < AHC_NUM_LUNS; j++) {
3958                                 struct ahc_tmode_lstate *lstate;
3959
3960                                 lstate = tstate->enabled_luns[j];
3961                                 if (lstate != NULL) {
3962                                         xpt_free_path(lstate->path);
3963                                         free(lstate, M_DEVBUF);
3964                                 }
3965                         }
3966 #endif
3967                         free(tstate, M_DEVBUF);
3968                 }
3969         }
3970 #ifdef AHC_TARGET_MODE
3971         if (ahc->black_hole != NULL) {
3972                 xpt_free_path(ahc->black_hole->path);
3973                 free(ahc->black_hole, M_DEVBUF);
3974         }
3975 #endif
3976         if (ahc->name != NULL)
3977                 free(ahc->name, M_DEVBUF);
3978         if (ahc->seep_config != NULL)
3979                 free(ahc->seep_config, M_DEVBUF);
3980 #ifndef __FreeBSD__
3981         free(ahc, M_DEVBUF);
3982 #endif
3983         return;
3984 }
3985
3986 void
3987 ahc_shutdown(void *arg)
3988 {
3989         struct  ahc_softc *ahc;
3990         int     i;
3991
3992         ahc = (struct ahc_softc *)arg;
3993
3994         /* This will reset most registers to 0, but not all */
3995         ahc_reset(ahc, /*reinit*/FALSE);
3996         ahc_outb(ahc, SCSISEQ, 0);
3997         ahc_outb(ahc, SXFRCTL0, 0);
3998         ahc_outb(ahc, DSPCISTATUS, 0);
3999
4000         for (i = TARG_SCSIRATE; i < SCSICONF; i++)
4001                 ahc_outb(ahc, i, 0);
4002 }
4003
4004 /*
4005  * Reset the controller and record some information about it
4006  * that is only available just after a reset.  If "reinit" is
4007  * non-zero, this reset occured after initial configuration
4008  * and the caller requests that the chip be fully reinitialized
4009  * to a runable state.  Chip interrupts are *not* enabled after
4010  * a reinitialization.  The caller must enable interrupts via
4011  * ahc_intr_enable().
4012  */
4013 int
4014 ahc_reset(struct ahc_softc *ahc, int reinit)
4015 {
4016         u_int   sblkctl;
4017         u_int   sxfrctl1_a, sxfrctl1_b;
4018         int     error;
4019         int     wait;
4020         
4021         /*
4022          * Preserve the value of the SXFRCTL1 register for all channels.
4023          * It contains settings that affect termination and we don't want
4024          * to disturb the integrity of the bus.
4025          */
4026         ahc_pause(ahc);
4027         sxfrctl1_b = 0;
4028         if ((ahc->chip & AHC_CHIPID_MASK) == AHC_AIC7770) {
4029                 u_int sblkctl;
4030
4031                 /*
4032                  * Save channel B's settings in case this chip
4033                  * is setup for TWIN channel operation.
4034                  */
4035                 sblkctl = ahc_inb(ahc, SBLKCTL);
4036                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4037                 sxfrctl1_b = ahc_inb(ahc, SXFRCTL1);
4038                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4039         }
4040         sxfrctl1_a = ahc_inb(ahc, SXFRCTL1);
4041
4042         ahc_outb(ahc, HCNTRL, CHIPRST | ahc->pause);
4043
4044         /*
4045          * Ensure that the reset has finished.  We delay 1000us
4046          * prior to reading the register to make sure the chip
4047          * has sufficiently completed its reset to handle register
4048          * accesses.
4049          */
4050         wait = 1000;
4051         do {
4052                 ahc_delay(1000);
4053         } while (--wait && !(ahc_inb(ahc, HCNTRL) & CHIPRSTACK));
4054
4055         if (wait == 0) {
4056                 printf("%s: WARNING - Failed chip reset!  "
4057                        "Trying to initialize anyway.\n", ahc_name(ahc));
4058         }
4059         ahc_outb(ahc, HCNTRL, ahc->pause);
4060
4061         /* Determine channel configuration */
4062         sblkctl = ahc_inb(ahc, SBLKCTL) & (SELBUSB|SELWIDE);
4063         /* No Twin Channel PCI cards */
4064         if ((ahc->chip & AHC_PCI) != 0)
4065                 sblkctl &= ~SELBUSB;
4066         switch (sblkctl) {
4067         case 0:
4068                 /* Single Narrow Channel */
4069                 break;
4070         case 2:
4071                 /* Wide Channel */
4072                 ahc->features |= AHC_WIDE;
4073                 break;
4074         case 8:
4075                 /* Twin Channel */
4076                 ahc->features |= AHC_TWIN;
4077                 break;
4078         default:
4079                 printf(" Unsupported adapter type.  Ignoring\n");
4080                 return(-1);
4081         }
4082
4083         /*
4084          * Reload sxfrctl1.
4085          *
4086          * We must always initialize STPWEN to 1 before we
4087          * restore the saved values.  STPWEN is initialized
4088          * to a tri-state condition which can only be cleared
4089          * by turning it on.
4090          */
4091         if ((ahc->features & AHC_TWIN) != 0) {
4092                 u_int sblkctl;
4093
4094                 sblkctl = ahc_inb(ahc, SBLKCTL);
4095                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4096                 ahc_outb(ahc, SXFRCTL1, sxfrctl1_b);
4097                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4098         }
4099         ahc_outb(ahc, SXFRCTL1, sxfrctl1_a);
4100
4101         error = 0;
4102         if (reinit != 0)
4103                 /*
4104                  * If a recovery action has forced a chip reset,
4105                  * re-initialize the chip to our liking.
4106                  */
4107                 error = ahc->bus_chip_init(ahc);
4108 #ifdef AHC_DUMP_SEQ
4109         else 
4110                 ahc_dumpseq(ahc);
4111 #endif
4112
4113         return (error);
4114 }
4115
4116 /*
4117  * Determine the number of SCBs available on the controller
4118  */
4119 int
4120 ahc_probe_scbs(struct ahc_softc *ahc) {
4121         int i;
4122
4123         for (i = 0; i < AHC_SCB_MAX; i++) {
4124
4125                 ahc_outb(ahc, SCBPTR, i);
4126                 ahc_outb(ahc, SCB_BASE, i);
4127                 if (ahc_inb(ahc, SCB_BASE) != i)
4128                         break;
4129                 ahc_outb(ahc, SCBPTR, 0);
4130                 if (ahc_inb(ahc, SCB_BASE) != 0)
4131                         break;
4132         }
4133         return (i);
4134 }
4135
4136 static void
4137 ahc_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 
4138 {
4139         dma_addr_t *baddr;
4140
4141         baddr = (dma_addr_t *)arg;
4142         *baddr = segs->ds_addr;
4143 }
4144
4145 static void
4146 ahc_build_free_scb_list(struct ahc_softc *ahc)
4147 {
4148         int scbsize;
4149         int i;
4150
4151         scbsize = 32;
4152         if ((ahc->flags & AHC_LSCBS_ENABLED) != 0)
4153                 scbsize = 64;
4154
4155         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
4156                 int j;
4157
4158                 ahc_outb(ahc, SCBPTR, i);
4159
4160                 /*
4161                  * Touch all SCB bytes to avoid parity errors
4162                  * should one of our debugging routines read
4163                  * an otherwise uninitiatlized byte.
4164                  */
4165                 for (j = 0; j < scbsize; j++)
4166                         ahc_outb(ahc, SCB_BASE+j, 0xFF);
4167
4168                 /* Clear the control byte. */
4169                 ahc_outb(ahc, SCB_CONTROL, 0);
4170
4171                 /* Set the next pointer */
4172                 if ((ahc->flags & AHC_PAGESCBS) != 0)
4173                         ahc_outb(ahc, SCB_NEXT, i+1);
4174                 else 
4175                         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4176
4177                 /* Make the tag number, SCSIID, and lun invalid */
4178                 ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
4179                 ahc_outb(ahc, SCB_SCSIID, 0xFF);
4180                 ahc_outb(ahc, SCB_LUN, 0xFF);
4181         }
4182
4183         if ((ahc->flags & AHC_PAGESCBS) != 0) {
4184                 /* SCB 0 heads the free list. */
4185                 ahc_outb(ahc, FREE_SCBH, 0);
4186         } else {
4187                 /* No free list. */
4188                 ahc_outb(ahc, FREE_SCBH, SCB_LIST_NULL);
4189         }
4190
4191         /* Make sure that the last SCB terminates the free list */
4192         ahc_outb(ahc, SCBPTR, i-1);
4193         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4194 }
4195
4196 static int
4197 ahc_init_scbdata(struct ahc_softc *ahc)
4198 {
4199         struct scb_data *scb_data;
4200
4201         scb_data = ahc->scb_data;
4202         SLIST_INIT(&scb_data->free_scbs);
4203         SLIST_INIT(&scb_data->sg_maps);
4204
4205         /* Allocate SCB resources */
4206         scb_data->scbarray =
4207             (struct scb *)malloc(sizeof(struct scb) * AHC_SCB_MAX_ALLOC,
4208                                  M_DEVBUF, M_NOWAIT);
4209         if (scb_data->scbarray == NULL)
4210                 return (ENOMEM);
4211         memset(scb_data->scbarray, 0, sizeof(struct scb) * AHC_SCB_MAX_ALLOC);
4212
4213         /* Determine the number of hardware SCBs and initialize them */
4214
4215         scb_data->maxhscbs = ahc_probe_scbs(ahc);
4216         if (ahc->scb_data->maxhscbs == 0) {
4217                 printf("%s: No SCB space found\n", ahc_name(ahc));
4218                 return (ENXIO);
4219         }
4220
4221         /*
4222          * Create our DMA tags.  These tags define the kinds of device
4223          * accessible memory allocations and memory mappings we will
4224          * need to perform during normal operation.
4225          *
4226          * Unless we need to further restrict the allocation, we rely
4227          * on the restrictions of the parent dmat, hence the common
4228          * use of MAXADDR and MAXSIZE.
4229          */
4230
4231         /* DMA tag for our hardware scb structures */
4232         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4233                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4234                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4235                                /*highaddr*/BUS_SPACE_MAXADDR,
4236                                /*filter*/NULL, /*filterarg*/NULL,
4237                                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4238                                /*nsegments*/1,
4239                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4240                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
4241                 goto error_exit;
4242         }
4243
4244         scb_data->init_level++;
4245
4246         /* Allocation for our hscbs */
4247         if (ahc_dmamem_alloc(ahc, scb_data->hscb_dmat,
4248                              (void **)&scb_data->hscbs,
4249                              BUS_DMA_NOWAIT, &scb_data->hscb_dmamap) != 0) {
4250                 goto error_exit;
4251         }
4252
4253         scb_data->init_level++;
4254
4255         /* And permanently map them */
4256         ahc_dmamap_load(ahc, scb_data->hscb_dmat, scb_data->hscb_dmamap,
4257                         scb_data->hscbs,
4258                         AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4259                         ahc_dmamap_cb, &scb_data->hscb_busaddr, /*flags*/0);
4260
4261         scb_data->init_level++;
4262
4263         /* DMA tag for our sense buffers */
4264         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4265                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4266                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4267                                /*highaddr*/BUS_SPACE_MAXADDR,
4268                                /*filter*/NULL, /*filterarg*/NULL,
4269                                AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4270                                /*nsegments*/1,
4271                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4272                                /*flags*/0, &scb_data->sense_dmat) != 0) {
4273                 goto error_exit;
4274         }
4275
4276         scb_data->init_level++;
4277
4278         /* Allocate them */
4279         if (ahc_dmamem_alloc(ahc, scb_data->sense_dmat,
4280                              (void **)&scb_data->sense,
4281                              BUS_DMA_NOWAIT, &scb_data->sense_dmamap) != 0) {
4282                 goto error_exit;
4283         }
4284
4285         scb_data->init_level++;
4286
4287         /* And permanently map them */
4288         ahc_dmamap_load(ahc, scb_data->sense_dmat, scb_data->sense_dmamap,
4289                         scb_data->sense,
4290                         AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4291                         ahc_dmamap_cb, &scb_data->sense_busaddr, /*flags*/0);
4292
4293         scb_data->init_level++;
4294
4295         /* DMA tag for our S/G structures.  We allocate in page sized chunks */
4296         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/8,
4297                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4298                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4299                                /*highaddr*/BUS_SPACE_MAXADDR,
4300                                /*filter*/NULL, /*filterarg*/NULL,
4301                                PAGE_SIZE, /*nsegments*/1,
4302                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4303                                /*flags*/0, &scb_data->sg_dmat) != 0) {
4304                 goto error_exit;
4305         }
4306
4307         scb_data->init_level++;
4308
4309         /* Perform initial CCB allocation */
4310         memset(scb_data->hscbs, 0,
4311                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb));
4312         ahc_alloc_scbs(ahc);
4313
4314         if (scb_data->numscbs == 0) {
4315                 printf("%s: ahc_init_scbdata - "
4316                        "Unable to allocate initial scbs\n",
4317                        ahc_name(ahc));
4318                 goto error_exit;
4319         }
4320
4321         /*
4322          * Reserve the next queued SCB.
4323          */
4324         ahc->next_queued_scb = ahc_get_scb(ahc);
4325
4326         /*
4327          * Note that we were successfull
4328          */
4329         return (0); 
4330
4331 error_exit:
4332
4333         return (ENOMEM);
4334 }
4335
4336 static void
4337 ahc_fini_scbdata(struct ahc_softc *ahc)
4338 {
4339         struct scb_data *scb_data;
4340
4341         scb_data = ahc->scb_data;
4342         if (scb_data == NULL)
4343                 return;
4344
4345         switch (scb_data->init_level) {
4346         default:
4347         case 7:
4348         {
4349                 struct sg_map_node *sg_map;
4350
4351                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps))!= NULL) {
4352                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4353                         ahc_dmamap_unload(ahc, scb_data->sg_dmat,
4354                                           sg_map->sg_dmamap);
4355                         ahc_dmamem_free(ahc, scb_data->sg_dmat,
4356                                         sg_map->sg_vaddr,
4357                                         sg_map->sg_dmamap);
4358                         free(sg_map, M_DEVBUF);
4359                 }
4360                 ahc_dma_tag_destroy(ahc, scb_data->sg_dmat);
4361         }
4362         case 6:
4363                 ahc_dmamap_unload(ahc, scb_data->sense_dmat,
4364                                   scb_data->sense_dmamap);
4365         case 5:
4366                 ahc_dmamem_free(ahc, scb_data->sense_dmat, scb_data->sense,
4367                                 scb_data->sense_dmamap);
4368                 ahc_dmamap_destroy(ahc, scb_data->sense_dmat,
4369                                    scb_data->sense_dmamap);
4370         case 4:
4371                 ahc_dma_tag_destroy(ahc, scb_data->sense_dmat);
4372         case 3:
4373                 ahc_dmamap_unload(ahc, scb_data->hscb_dmat,
4374                                   scb_data->hscb_dmamap);
4375         case 2:
4376                 ahc_dmamem_free(ahc, scb_data->hscb_dmat, scb_data->hscbs,
4377                                 scb_data->hscb_dmamap);
4378                 ahc_dmamap_destroy(ahc, scb_data->hscb_dmat,
4379                                    scb_data->hscb_dmamap);
4380         case 1:
4381                 ahc_dma_tag_destroy(ahc, scb_data->hscb_dmat);
4382                 break;
4383         case 0:
4384                 break;
4385         }
4386         if (scb_data->scbarray != NULL)
4387                 free(scb_data->scbarray, M_DEVBUF);
4388 }
4389
4390 void
4391 ahc_alloc_scbs(struct ahc_softc *ahc)
4392 {
4393         struct scb_data *scb_data;
4394         struct scb *next_scb;
4395         struct sg_map_node *sg_map;
4396         dma_addr_t physaddr;
4397         struct ahc_dma_seg *segs;
4398         int newcount;
4399         int i;
4400
4401         scb_data = ahc->scb_data;
4402         if (scb_data->numscbs >= AHC_SCB_MAX_ALLOC)
4403                 /* Can't allocate any more */
4404                 return;
4405
4406         next_scb = &scb_data->scbarray[scb_data->numscbs];
4407
4408         sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
4409
4410         if (sg_map == NULL)
4411                 return;
4412
4413         /* Allocate S/G space for the next batch of SCBS */
4414         if (ahc_dmamem_alloc(ahc, scb_data->sg_dmat,
4415                              (void **)&sg_map->sg_vaddr,
4416                              BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
4417                 free(sg_map, M_DEVBUF);
4418                 return;
4419         }
4420
4421         SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
4422
4423         ahc_dmamap_load(ahc, scb_data->sg_dmat, sg_map->sg_dmamap,
4424                         sg_map->sg_vaddr, PAGE_SIZE, ahc_dmamap_cb,
4425                         &sg_map->sg_physaddr, /*flags*/0);
4426
4427         segs = sg_map->sg_vaddr;
4428         physaddr = sg_map->sg_physaddr;
4429
4430         newcount = (PAGE_SIZE / (AHC_NSEG * sizeof(struct ahc_dma_seg)));
4431         newcount = min(newcount, (AHC_SCB_MAX_ALLOC - scb_data->numscbs));
4432         for (i = 0; i < newcount; i++) {
4433                 struct scb_platform_data *pdata;
4434 #ifndef __linux__
4435                 int error;
4436 #endif
4437                 pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
4438                                                            M_DEVBUF, M_NOWAIT);
4439                 if (pdata == NULL)
4440                         break;
4441                 next_scb->platform_data = pdata;
4442                 next_scb->sg_map = sg_map;
4443                 next_scb->sg_list = segs;
4444                 /*
4445                  * The sequencer always starts with the second entry.
4446                  * The first entry is embedded in the scb.
4447                  */
4448                 next_scb->sg_list_phys = physaddr + sizeof(struct ahc_dma_seg);
4449                 next_scb->ahc_softc = ahc;
4450                 next_scb->flags = SCB_FREE;
4451 #ifndef __linux__
4452                 error = ahc_dmamap_create(ahc, ahc->buffer_dmat, /*flags*/0,
4453                                           &next_scb->dmamap);
4454                 if (error != 0)
4455                         break;
4456 #endif
4457                 next_scb->hscb = &scb_data->hscbs[scb_data->numscbs];
4458                 next_scb->hscb->tag = ahc->scb_data->numscbs;
4459                 SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs,
4460                                   next_scb, links.sle);
4461                 segs += AHC_NSEG;
4462                 physaddr += (AHC_NSEG * sizeof(struct ahc_dma_seg));
4463                 next_scb++;
4464                 ahc->scb_data->numscbs++;
4465         }
4466 }
4467
4468 void
4469 ahc_controller_info(struct ahc_softc *ahc, char *buf)
4470 {
4471         int len;
4472
4473         len = sprintf(buf, "%s: ", ahc_chip_names[ahc->chip & AHC_CHIPID_MASK]);
4474         buf += len;
4475         if ((ahc->features & AHC_TWIN) != 0)
4476                 len = sprintf(buf, "Twin Channel, A SCSI Id=%d, "
4477                               "B SCSI Id=%d, primary %c, ",
4478                               ahc->our_id, ahc->our_id_b,
4479                               (ahc->flags & AHC_PRIMARY_CHANNEL) + 'A');
4480         else {
4481                 const char *speed;
4482                 const char *type;
4483
4484                 speed = "";
4485                 if ((ahc->features & AHC_ULTRA) != 0) {
4486                         speed = "Ultra ";
4487                 } else if ((ahc->features & AHC_DT) != 0) {
4488                         speed = "Ultra160 ";
4489                 } else if ((ahc->features & AHC_ULTRA2) != 0) {
4490                         speed = "Ultra2 ";
4491                 }
4492                 if ((ahc->features & AHC_WIDE) != 0) {
4493                         type = "Wide";
4494                 } else {
4495                         type = "Single";
4496                 }
4497                 len = sprintf(buf, "%s%s Channel %c, SCSI Id=%d, ",
4498                               speed, type, ahc->channel, ahc->our_id);
4499         }
4500         buf += len;
4501
4502         if ((ahc->flags & AHC_PAGESCBS) != 0)
4503                 sprintf(buf, "%d/%d SCBs",
4504                         ahc->scb_data->maxhscbs, AHC_MAX_QUEUE);
4505         else
4506                 sprintf(buf, "%d SCBs", ahc->scb_data->maxhscbs);
4507 }
4508
4509 int
4510 ahc_chip_init(struct ahc_softc *ahc)
4511 {
4512         int      term;
4513         int      error;
4514         u_int    i;
4515         u_int    scsi_conf;
4516         u_int    scsiseq_template;
4517         uint32_t physaddr;
4518
4519         ahc_outb(ahc, SEQ_FLAGS, 0);
4520         ahc_outb(ahc, SEQ_FLAGS2, 0);
4521
4522         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1, for both channels*/
4523         if (ahc->features & AHC_TWIN) {
4524
4525                 /*
4526                  * Setup Channel B first.
4527                  */
4528                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) | SELBUSB);
4529                 term = (ahc->flags & AHC_TERM_ENB_B) != 0 ? STPWEN : 0;
4530                 ahc_outb(ahc, SCSIID, ahc->our_id_b);
4531                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
4532                 ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
4533                                         |term|ahc->seltime_b|ENSTIMER|ACTNEGEN);
4534                 if ((ahc->features & AHC_ULTRA2) != 0)
4535                         ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
4536                 ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
4537                 ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
4538
4539                 /* Select Channel A */
4540                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) & ~SELBUSB);
4541         }
4542         term = (ahc->flags & AHC_TERM_ENB_A) != 0 ? STPWEN : 0;
4543         if ((ahc->features & AHC_ULTRA2) != 0)
4544                 ahc_outb(ahc, SCSIID_ULTRA2, ahc->our_id);
4545         else
4546                 ahc_outb(ahc, SCSIID, ahc->our_id);
4547         scsi_conf = ahc_inb(ahc, SCSICONF);
4548         ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
4549                                 |term|ahc->seltime
4550                                 |ENSTIMER|ACTNEGEN);
4551         if ((ahc->features & AHC_ULTRA2) != 0)
4552                 ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
4553         ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
4554         ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
4555
4556         /* There are no untagged SCBs active yet. */
4557         for (i = 0; i < 16; i++) {
4558                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, 0));
4559                 if ((ahc->flags & AHC_SCB_BTT) != 0) {
4560                         int lun;
4561
4562                         /*
4563                          * The SCB based BTT allows an entry per
4564                          * target and lun pair.
4565                          */
4566                         for (lun = 1; lun < AHC_NUM_LUNS; lun++)
4567                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, lun));
4568                 }
4569         }
4570
4571         /* All of our queues are empty */
4572         for (i = 0; i < 256; i++)
4573                 ahc->qoutfifo[i] = SCB_LIST_NULL;
4574         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_PREREAD);
4575
4576         for (i = 0; i < 256; i++)
4577                 ahc->qinfifo[i] = SCB_LIST_NULL;
4578
4579         if ((ahc->features & AHC_MULTI_TID) != 0) {
4580                 ahc_outb(ahc, TARGID, 0);
4581                 ahc_outb(ahc, TARGID + 1, 0);
4582         }
4583
4584         /*
4585          * Tell the sequencer where it can find our arrays in memory.
4586          */
4587         physaddr = ahc->scb_data->hscb_busaddr;
4588         ahc_outb(ahc, HSCB_ADDR, physaddr & 0xFF);
4589         ahc_outb(ahc, HSCB_ADDR + 1, (physaddr >> 8) & 0xFF);
4590         ahc_outb(ahc, HSCB_ADDR + 2, (physaddr >> 16) & 0xFF);
4591         ahc_outb(ahc, HSCB_ADDR + 3, (physaddr >> 24) & 0xFF);
4592
4593         physaddr = ahc->shared_data_busaddr;
4594         ahc_outb(ahc, SHARED_DATA_ADDR, physaddr & 0xFF);
4595         ahc_outb(ahc, SHARED_DATA_ADDR + 1, (physaddr >> 8) & 0xFF);
4596         ahc_outb(ahc, SHARED_DATA_ADDR + 2, (physaddr >> 16) & 0xFF);
4597         ahc_outb(ahc, SHARED_DATA_ADDR + 3, (physaddr >> 24) & 0xFF);
4598
4599         /*
4600          * Initialize the group code to command length table.
4601          * This overrides the values in TARG_SCSIRATE, so only
4602          * setup the table after we have processed that information.
4603          */
4604         ahc_outb(ahc, CMDSIZE_TABLE, 5);
4605         ahc_outb(ahc, CMDSIZE_TABLE + 1, 9);
4606         ahc_outb(ahc, CMDSIZE_TABLE + 2, 9);
4607         ahc_outb(ahc, CMDSIZE_TABLE + 3, 0);
4608         ahc_outb(ahc, CMDSIZE_TABLE + 4, 15);
4609         ahc_outb(ahc, CMDSIZE_TABLE + 5, 11);
4610         ahc_outb(ahc, CMDSIZE_TABLE + 6, 0);
4611         ahc_outb(ahc, CMDSIZE_TABLE + 7, 0);
4612                 
4613         if ((ahc->features & AHC_HS_MAILBOX) != 0)
4614                 ahc_outb(ahc, HS_MAILBOX, 0);
4615
4616         /* Tell the sequencer of our initial queue positions */
4617         if ((ahc->features & AHC_TARGETMODE) != 0) {
4618                 ahc->tqinfifonext = 1;
4619                 ahc_outb(ahc, KERNEL_TQINPOS, ahc->tqinfifonext - 1);
4620                 ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
4621         }
4622         ahc->qinfifonext = 0;
4623         ahc->qoutfifonext = 0;
4624         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
4625                 ahc_outb(ahc, QOFF_CTLSTA, SCB_QSIZE_256);
4626                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
4627                 ahc_outb(ahc, SNSCB_QOFF, ahc->qinfifonext);
4628                 ahc_outb(ahc, SDSCB_QOFF, 0);
4629         } else {
4630                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
4631                 ahc_outb(ahc, QINPOS, ahc->qinfifonext);
4632                 ahc_outb(ahc, QOUTPOS, ahc->qoutfifonext);
4633         }
4634
4635         /* We don't have any waiting selections */
4636         ahc_outb(ahc, WAITING_SCBH, SCB_LIST_NULL);
4637
4638         /* Our disconnection list is empty too */
4639         ahc_outb(ahc, DISCONNECTED_SCBH, SCB_LIST_NULL);
4640
4641         /* Message out buffer starts empty */
4642         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
4643
4644         /*
4645          * Setup the allowed SCSI Sequences based on operational mode.
4646          * If we are a target, we'll enalbe select in operations once
4647          * we've had a lun enabled.
4648          */
4649         scsiseq_template = ENSELO|ENAUTOATNO|ENAUTOATNP;
4650         if ((ahc->flags & AHC_INITIATORROLE) != 0)
4651                 scsiseq_template |= ENRSELI;
4652         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq_template);
4653
4654         /* Initialize our list of free SCBs. */
4655         ahc_build_free_scb_list(ahc);
4656
4657         /*
4658          * Tell the sequencer which SCB will be the next one it receives.
4659          */
4660         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
4661
4662         /*
4663          * Load the Sequencer program and Enable the adapter
4664          * in "fast" mode.
4665          */
4666         if (bootverbose)
4667                 printf("%s: Downloading Sequencer Program...",
4668                        ahc_name(ahc));
4669
4670         error = ahc_loadseq(ahc);
4671         if (error != 0)
4672                 return (error);
4673
4674         if ((ahc->features & AHC_ULTRA2) != 0) {
4675                 int wait;
4676
4677                 /*
4678                  * Wait for up to 500ms for our transceivers
4679                  * to settle.  If the adapter does not have
4680                  * a cable attached, the transceivers may
4681                  * never settle, so don't complain if we
4682                  * fail here.
4683                  */
4684                 for (wait = 5000;
4685                      (ahc_inb(ahc, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
4686                      wait--)
4687                         ahc_delay(100);
4688         }
4689         ahc_restart(ahc);
4690         return (0);
4691 }
4692
4693 /*
4694  * Start the board, ready for normal operation
4695  */
4696 int
4697 ahc_init(struct ahc_softc *ahc)
4698 {
4699         int      max_targ;
4700         u_int    i;
4701         u_int    scsi_conf;
4702         u_int    ultraenb;
4703         u_int    discenable;
4704         u_int    tagenable;
4705         size_t   driver_data_size;
4706
4707 #ifdef AHC_DEBUG
4708         if ((ahc_debug & AHC_DEBUG_SEQUENCER) != 0)
4709                 ahc->flags |= AHC_SEQUENCER_DEBUG;
4710 #endif
4711
4712 #ifdef AHC_PRINT_SRAM
4713         printf("Scratch Ram:");
4714         for (i = 0x20; i < 0x5f; i++) {
4715                 if (((i % 8) == 0) && (i != 0)) {
4716                         printf ("\n              ");
4717                 }
4718                 printf (" 0x%x", ahc_inb(ahc, i));
4719         }
4720         if ((ahc->features & AHC_MORE_SRAM) != 0) {
4721                 for (i = 0x70; i < 0x7f; i++) {
4722                         if (((i % 8) == 0) && (i != 0)) {
4723                                 printf ("\n              ");
4724                         }
4725                         printf (" 0x%x", ahc_inb(ahc, i));
4726                 }
4727         }
4728         printf ("\n");
4729         /*
4730          * Reading uninitialized scratch ram may
4731          * generate parity errors.
4732          */
4733         ahc_outb(ahc, CLRINT, CLRPARERR);
4734         ahc_outb(ahc, CLRINT, CLRBRKADRINT);
4735 #endif
4736         max_targ = 15;
4737
4738         /*
4739          * Assume we have a board at this stage and it has been reset.
4740          */
4741         if ((ahc->flags & AHC_USEDEFAULTS) != 0)
4742                 ahc->our_id = ahc->our_id_b = 7;
4743         
4744         /*
4745          * Default to allowing initiator operations.
4746          */
4747         ahc->flags |= AHC_INITIATORROLE;
4748
4749         /*
4750          * Only allow target mode features if this unit has them enabled.
4751          */
4752         if ((AHC_TMODE_ENABLE & (0x1 << ahc->unit)) == 0)
4753                 ahc->features &= ~AHC_TARGETMODE;
4754
4755 #ifndef __linux__
4756         /* DMA tag for mapping buffers into device visible space. */
4757         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4758                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4759                                /*lowaddr*/ahc->flags & AHC_39BIT_ADDRESSING
4760                                         ? (dma_addr_t)0x7FFFFFFFFFULL
4761                                         : BUS_SPACE_MAXADDR_32BIT,
4762                                /*highaddr*/BUS_SPACE_MAXADDR,
4763                                /*filter*/NULL, /*filterarg*/NULL,
4764                                /*maxsize*/(AHC_NSEG - 1) * PAGE_SIZE,
4765                                /*nsegments*/AHC_NSEG,
4766                                /*maxsegsz*/AHC_MAXTRANSFER_SIZE,
4767                                /*flags*/BUS_DMA_ALLOCNOW,
4768                                &ahc->buffer_dmat) != 0) {
4769                 return (ENOMEM);
4770         }
4771 #endif
4772
4773         ahc->init_level++;
4774
4775         /*
4776          * DMA tag for our command fifos and other data in system memory
4777          * the card's sequencer must be able to access.  For initiator
4778          * roles, we need to allocate space for the qinfifo and qoutfifo.
4779          * The qinfifo and qoutfifo are composed of 256 1 byte elements. 
4780          * When providing for the target mode role, we must additionally
4781          * provide space for the incoming target command fifo and an extra
4782          * byte to deal with a dma bug in some chip versions.
4783          */
4784         driver_data_size = 2 * 256 * sizeof(uint8_t);
4785         if ((ahc->features & AHC_TARGETMODE) != 0)
4786                 driver_data_size += AHC_TMODE_CMDS * sizeof(struct target_cmd)
4787                                  + /*DMA WideOdd Bug Buffer*/1;
4788         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4789                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4790                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4791                                /*highaddr*/BUS_SPACE_MAXADDR,
4792                                /*filter*/NULL, /*filterarg*/NULL,
4793                                driver_data_size,
4794                                /*nsegments*/1,
4795                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4796                                /*flags*/0, &ahc->shared_data_dmat) != 0) {
4797                 return (ENOMEM);
4798         }
4799
4800         ahc->init_level++;
4801
4802         /* Allocation of driver data */
4803         if (ahc_dmamem_alloc(ahc, ahc->shared_data_dmat,
4804                              (void **)&ahc->qoutfifo,
4805                              BUS_DMA_NOWAIT, &ahc->shared_data_dmamap) != 0) {
4806                 return (ENOMEM);
4807         }
4808
4809         ahc->init_level++;
4810
4811         /* And permanently map it in */
4812         ahc_dmamap_load(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
4813                         ahc->qoutfifo, driver_data_size, ahc_dmamap_cb,
4814                         &ahc->shared_data_busaddr, /*flags*/0);
4815
4816         if ((ahc->features & AHC_TARGETMODE) != 0) {
4817                 ahc->targetcmds = (struct target_cmd *)ahc->qoutfifo;
4818                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[AHC_TMODE_CMDS];
4819                 ahc->dma_bug_buf = ahc->shared_data_busaddr
4820                                  + driver_data_size - 1;
4821                 /* All target command blocks start out invalid. */
4822                 for (i = 0; i < AHC_TMODE_CMDS; i++)
4823                         ahc->targetcmds[i].cmd_valid = 0;
4824                 ahc_sync_tqinfifo(ahc, BUS_DMASYNC_PREREAD);
4825                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[256];
4826         }
4827         ahc->qinfifo = &ahc->qoutfifo[256];
4828
4829         ahc->init_level++;
4830
4831         /* Allocate SCB data now that buffer_dmat is initialized */
4832         if (ahc->scb_data->maxhscbs == 0)
4833                 if (ahc_init_scbdata(ahc) != 0)
4834                         return (ENOMEM);
4835
4836         /*
4837          * Allocate a tstate to house information for our
4838          * initiator presence on the bus as well as the user
4839          * data for any target mode initiator.
4840          */
4841         if (ahc_alloc_tstate(ahc, ahc->our_id, 'A') == NULL) {
4842                 printf("%s: unable to allocate ahc_tmode_tstate.  "
4843                        "Failing attach\n", ahc_name(ahc));
4844                 return (ENOMEM);
4845         }
4846
4847         if ((ahc->features & AHC_TWIN) != 0) {
4848                 if (ahc_alloc_tstate(ahc, ahc->our_id_b, 'B') == NULL) {
4849                         printf("%s: unable to allocate ahc_tmode_tstate.  "
4850                                "Failing attach\n", ahc_name(ahc));
4851                         return (ENOMEM);
4852                 }
4853         }
4854
4855         if (ahc->scb_data->maxhscbs < AHC_SCB_MAX_ALLOC) {
4856                 ahc->flags |= AHC_PAGESCBS;
4857         } else {
4858                 ahc->flags &= ~AHC_PAGESCBS;
4859         }
4860
4861 #ifdef AHC_DEBUG
4862         if (ahc_debug & AHC_SHOW_MISC) {
4863                 printf("%s: hardware scb %u bytes; kernel scb %u bytes; "
4864                        "ahc_dma %u bytes\n",
4865                         ahc_name(ahc),
4866                         (u_int)sizeof(struct hardware_scb),
4867                         (u_int)sizeof(struct scb),
4868                         (u_int)sizeof(struct ahc_dma_seg));
4869         }
4870 #endif /* AHC_DEBUG */
4871
4872         /*
4873          * Look at the information that board initialization or
4874          * the board bios has left us.
4875          */
4876         if (ahc->features & AHC_TWIN) {
4877                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
4878                 if ((scsi_conf & RESET_SCSI) != 0
4879                  && (ahc->flags & AHC_INITIATORROLE) != 0)
4880                         ahc->flags |= AHC_RESET_BUS_B;
4881         }
4882
4883         scsi_conf = ahc_inb(ahc, SCSICONF);
4884         if ((scsi_conf & RESET_SCSI) != 0
4885          && (ahc->flags & AHC_INITIATORROLE) != 0)
4886                 ahc->flags |= AHC_RESET_BUS_A;
4887
4888         ultraenb = 0;   
4889         tagenable = ALL_TARGETS_MASK;
4890
4891         /* Grab the disconnection disable table and invert it for our needs */
4892         if ((ahc->flags & AHC_USEDEFAULTS) != 0) {
4893                 printf("%s: Host Adapter Bios disabled.  Using default SCSI "
4894                         "device parameters\n", ahc_name(ahc));
4895                 ahc->flags |= AHC_EXTENDED_TRANS_A|AHC_EXTENDED_TRANS_B|
4896                               AHC_TERM_ENB_A|AHC_TERM_ENB_B;
4897                 discenable = ALL_TARGETS_MASK;
4898                 if ((ahc->features & AHC_ULTRA) != 0)
4899                         ultraenb = ALL_TARGETS_MASK;
4900         } else {
4901                 discenable = ~((ahc_inb(ahc, DISC_DSB + 1) << 8)
4902                            | ahc_inb(ahc, DISC_DSB));
4903                 if ((ahc->features & (AHC_ULTRA|AHC_ULTRA2)) != 0)
4904                         ultraenb = (ahc_inb(ahc, ULTRA_ENB + 1) << 8)
4905                                       | ahc_inb(ahc, ULTRA_ENB);
4906         }
4907
4908         if ((ahc->features & (AHC_WIDE|AHC_TWIN)) == 0)
4909                 max_targ = 7;
4910
4911         for (i = 0; i <= max_targ; i++) {
4912                 struct ahc_initiator_tinfo *tinfo;
4913                 struct ahc_tmode_tstate *tstate;
4914                 u_int our_id;
4915                 u_int target_id;
4916                 char channel;
4917
4918                 channel = 'A';
4919                 our_id = ahc->our_id;
4920                 target_id = i;
4921                 if (i > 7 && (ahc->features & AHC_TWIN) != 0) {
4922                         channel = 'B';
4923                         our_id = ahc->our_id_b;
4924                         target_id = i % 8;
4925                 }
4926                 tinfo = ahc_fetch_transinfo(ahc, channel, our_id,
4927                                             target_id, &tstate);
4928                 /* Default to async narrow across the board */
4929                 memset(tinfo, 0, sizeof(*tinfo));
4930                 if (ahc->flags & AHC_USEDEFAULTS) {
4931                         if ((ahc->features & AHC_WIDE) != 0)
4932                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
4933
4934                         /*
4935                          * These will be truncated when we determine the
4936                          * connection type we have with the target.
4937                          */
4938                         tinfo->user.period = ahc_syncrates->period;
4939                         tinfo->user.offset = MAX_OFFSET;
4940                 } else {
4941                         u_int scsirate;
4942                         uint16_t mask;
4943
4944                         /* Take the settings leftover in scratch RAM. */
4945                         scsirate = ahc_inb(ahc, TARG_SCSIRATE + i);
4946                         mask = (0x01 << i);
4947                         if ((ahc->features & AHC_ULTRA2) != 0) {
4948                                 u_int offset;
4949                                 u_int maxsync;
4950
4951                                 if ((scsirate & SOFS) == 0x0F) {
4952                                         /*
4953                                          * Haven't negotiated yet,
4954                                          * so the format is different.
4955                                          */
4956                                         scsirate = (scsirate & SXFR) >> 4
4957                                                  | (ultraenb & mask)
4958                                                   ? 0x08 : 0x0
4959                                                  | (scsirate & WIDEXFER);
4960                                         offset = MAX_OFFSET_ULTRA2;
4961                                 } else
4962                                         offset = ahc_inb(ahc, TARG_OFFSET + i);
4963                                 if ((scsirate & ~WIDEXFER) == 0 && offset != 0)
4964                                         /* Set to the lowest sync rate, 5MHz */
4965                                         scsirate |= 0x1c;
4966                                 maxsync = AHC_SYNCRATE_ULTRA2;
4967                                 if ((ahc->features & AHC_DT) != 0)
4968                                         maxsync = AHC_SYNCRATE_DT;
4969                                 tinfo->user.period =
4970                                     ahc_find_period(ahc, scsirate, maxsync);
4971                                 if (offset == 0)
4972                                         tinfo->user.period = 0;
4973                                 else
4974                                         tinfo->user.offset = MAX_OFFSET;
4975                                 if ((scsirate & SXFR_ULTRA2) <= 8/*10MHz*/
4976                                  && (ahc->features & AHC_DT) != 0)
4977                                         tinfo->user.ppr_options =
4978                                             MSG_EXT_PPR_DT_REQ;
4979                         } else if ((scsirate & SOFS) != 0) {
4980                                 if ((scsirate & SXFR) == 0x40
4981                                  && (ultraenb & mask) != 0) {
4982                                         /* Treat 10MHz as a non-ultra speed */
4983                                         scsirate &= ~SXFR;
4984                                         ultraenb &= ~mask;
4985                                 }
4986                                 tinfo->user.period = 
4987                                     ahc_find_period(ahc, scsirate,
4988                                                     (ultraenb & mask)
4989                                                    ? AHC_SYNCRATE_ULTRA
4990                                                    : AHC_SYNCRATE_FAST);
4991                                 if (tinfo->user.period != 0)
4992                                         tinfo->user.offset = MAX_OFFSET;
4993                         }
4994                         if (tinfo->user.period == 0)
4995                                 tinfo->user.offset = 0;
4996                         if ((scsirate & WIDEXFER) != 0
4997                          && (ahc->features & AHC_WIDE) != 0)
4998                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
4999                         tinfo->user.protocol_version = 4;
5000                         if ((ahc->features & AHC_DT) != 0)
5001                                 tinfo->user.transport_version = 3;
5002                         else
5003                                 tinfo->user.transport_version = 2;
5004                         tinfo->goal.protocol_version = 2;
5005                         tinfo->goal.transport_version = 2;
5006                         tinfo->curr.protocol_version = 2;
5007                         tinfo->curr.transport_version = 2;
5008                 }
5009                 tstate->ultraenb = 0;
5010         }
5011         ahc->user_discenable = discenable;
5012         ahc->user_tagenable = tagenable;
5013
5014         return (ahc->bus_chip_init(ahc));
5015 }
5016
5017 void
5018 ahc_intr_enable(struct ahc_softc *ahc, int enable)
5019 {
5020         u_int hcntrl;
5021
5022         hcntrl = ahc_inb(ahc, HCNTRL);
5023         hcntrl &= ~INTEN;
5024         ahc->pause &= ~INTEN;
5025         ahc->unpause &= ~INTEN;
5026         if (enable) {
5027                 hcntrl |= INTEN;
5028                 ahc->pause |= INTEN;
5029                 ahc->unpause |= INTEN;
5030         }
5031         ahc_outb(ahc, HCNTRL, hcntrl);
5032 }
5033
5034 /*
5035  * Ensure that the card is paused in a location
5036  * outside of all critical sections and that all
5037  * pending work is completed prior to returning.
5038  * This routine should only be called from outside
5039  * an interrupt context.
5040  */
5041 void
5042 ahc_pause_and_flushwork(struct ahc_softc *ahc)
5043 {
5044         int intstat;
5045         int maxloops;
5046         int paused;
5047
5048         maxloops = 1000;
5049         ahc->flags |= AHC_ALL_INTERRUPTS;
5050         paused = FALSE;
5051         do {
5052                 if (paused) {
5053                         ahc_unpause(ahc);
5054                         /*
5055                          * Give the sequencer some time to service
5056                          * any active selections.
5057                          */
5058                         ahc_delay(500);
5059                 }
5060                 ahc_intr(ahc);
5061                 ahc_pause(ahc);
5062                 paused = TRUE;
5063                 ahc_outb(ahc, SCSISEQ, ahc_inb(ahc, SCSISEQ) & ~ENSELO);
5064                 intstat = ahc_inb(ahc, INTSTAT);
5065                 if ((intstat & INT_PEND) == 0) {
5066                         ahc_clear_critical_section(ahc);
5067                         intstat = ahc_inb(ahc, INTSTAT);
5068                 }
5069         } while (--maxloops
5070               && (intstat != 0xFF || (ahc->features & AHC_REMOVABLE) == 0)
5071               && ((intstat & INT_PEND) != 0
5072                || (ahc_inb(ahc, SSTAT0) & (SELDO|SELINGO)) != 0));
5073         if (maxloops == 0) {
5074                 printf("Infinite interrupt loop, INTSTAT = %x",
5075                        ahc_inb(ahc, INTSTAT));
5076         }
5077         ahc_platform_flushwork(ahc);
5078         ahc->flags &= ~AHC_ALL_INTERRUPTS;
5079 }
5080
5081 int
5082 ahc_suspend(struct ahc_softc *ahc)
5083 {
5084
5085         ahc_pause_and_flushwork(ahc);
5086
5087         if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
5088                 ahc_unpause(ahc);
5089                 return (EBUSY);
5090         }
5091
5092 #ifdef AHC_TARGET_MODE
5093         /*
5094          * XXX What about ATIOs that have not yet been serviced?
5095          * Perhaps we should just refuse to be suspended if we
5096          * are acting in a target role.
5097          */
5098         if (ahc->pending_device != NULL) {
5099                 ahc_unpause(ahc);
5100                 return (EBUSY);
5101         }
5102 #endif
5103         ahc_shutdown(ahc);
5104         return (0);
5105 }
5106
5107 int
5108 ahc_resume(struct ahc_softc *ahc)
5109 {
5110
5111         ahc_reset(ahc, /*reinit*/TRUE);
5112         ahc_intr_enable(ahc, TRUE); 
5113         ahc_restart(ahc);
5114         return (0);
5115 }
5116
5117 /************************** Busy Target Table *********************************/
5118 /*
5119  * Return the untagged transaction id for a given target/channel lun.
5120  * Optionally, clear the entry.
5121  */
5122 u_int
5123 ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl)
5124 {
5125         u_int scbid;
5126         u_int target_offset;
5127
5128         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5129                 u_int saved_scbptr;
5130                 
5131                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5132                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5133                 scbid = ahc_inb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl));
5134                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5135         } else {
5136                 target_offset = TCL_TARGET_OFFSET(tcl);
5137                 scbid = ahc_inb(ahc, BUSY_TARGETS + target_offset);
5138         }
5139
5140         return (scbid);
5141 }
5142
5143 void
5144 ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl)
5145 {
5146         u_int target_offset;
5147
5148         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5149                 u_int saved_scbptr;
5150                 
5151                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5152                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5153                 ahc_outb(ahc, SCB_64_BTT+TCL_TARGET_OFFSET(tcl), SCB_LIST_NULL);
5154                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5155         } else {
5156                 target_offset = TCL_TARGET_OFFSET(tcl);
5157                 ahc_outb(ahc, BUSY_TARGETS + target_offset, SCB_LIST_NULL);
5158         }
5159 }
5160
5161 void
5162 ahc_busy_tcl(struct ahc_softc *ahc, u_int tcl, u_int scbid)
5163 {
5164         u_int target_offset;
5165
5166         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5167                 u_int saved_scbptr;
5168                 
5169                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5170                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5171                 ahc_outb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl), scbid);
5172                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5173         } else {
5174                 target_offset = TCL_TARGET_OFFSET(tcl);
5175                 ahc_outb(ahc, BUSY_TARGETS + target_offset, scbid);
5176         }
5177 }
5178
5179 /************************** SCB and SCB queue management **********************/
5180 int
5181 ahc_match_scb(struct ahc_softc *ahc, struct scb *scb, int target,
5182               char channel, int lun, u_int tag, role_t role)
5183 {
5184         int targ = SCB_GET_TARGET(ahc, scb);
5185         char chan = SCB_GET_CHANNEL(ahc, scb);
5186         int slun = SCB_GET_LUN(scb);
5187         int match;
5188
5189         match = ((chan == channel) || (channel == ALL_CHANNELS));
5190         if (match != 0)
5191                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
5192         if (match != 0)
5193                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
5194         if (match != 0) {
5195 #ifdef AHC_TARGET_MODE
5196                 int group;
5197
5198                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
5199                 if (role == ROLE_INITIATOR) {
5200                         match = (group != XPT_FC_GROUP_TMODE)
5201                               && ((tag == scb->hscb->tag)
5202                                || (tag == SCB_LIST_NULL));
5203                 } else if (role == ROLE_TARGET) {
5204                         match = (group == XPT_FC_GROUP_TMODE)
5205                               && ((tag == scb->io_ctx->csio.tag_id)
5206                                || (tag == SCB_LIST_NULL));
5207                 }
5208 #else /* !AHC_TARGET_MODE */
5209                 match = ((tag == scb->hscb->tag) || (tag == SCB_LIST_NULL));
5210 #endif /* AHC_TARGET_MODE */
5211         }
5212
5213         return match;
5214 }
5215
5216 void
5217 ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb)
5218 {
5219         int     target;
5220         char    channel;
5221         int     lun;
5222
5223         target = SCB_GET_TARGET(ahc, scb);
5224         lun = SCB_GET_LUN(scb);
5225         channel = SCB_GET_CHANNEL(ahc, scb);
5226         
5227         ahc_search_qinfifo(ahc, target, channel, lun,
5228                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
5229                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5230
5231         ahc_platform_freeze_devq(ahc, scb);
5232 }
5233
5234 void
5235 ahc_qinfifo_requeue_tail(struct ahc_softc *ahc, struct scb *scb)
5236 {
5237         struct scb *prev_scb;
5238
5239         prev_scb = NULL;
5240         if (ahc_qinfifo_count(ahc) != 0) {
5241                 u_int prev_tag;
5242                 uint8_t prev_pos;
5243
5244                 prev_pos = ahc->qinfifonext - 1;
5245                 prev_tag = ahc->qinfifo[prev_pos];
5246                 prev_scb = ahc_lookup_scb(ahc, prev_tag);
5247         }
5248         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5249         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5250                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5251         } else {
5252                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5253         }
5254 }
5255
5256 static void
5257 ahc_qinfifo_requeue(struct ahc_softc *ahc, struct scb *prev_scb,
5258                     struct scb *scb)
5259 {
5260         if (prev_scb == NULL) {
5261                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5262         } else {
5263                 prev_scb->hscb->next = scb->hscb->tag;
5264                 ahc_sync_scb(ahc, prev_scb, 
5265                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5266         }
5267         ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
5268         scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5269         ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5270 }
5271
5272 static int
5273 ahc_qinfifo_count(struct ahc_softc *ahc)
5274 {
5275         uint8_t qinpos;
5276         uint8_t diff;
5277
5278         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5279                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
5280                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
5281         } else
5282                 qinpos = ahc_inb(ahc, QINPOS);
5283         diff = ahc->qinfifonext - qinpos;
5284         return (diff);
5285 }
5286
5287 int
5288 ahc_search_qinfifo(struct ahc_softc *ahc, int target, char channel,
5289                    int lun, u_int tag, role_t role, uint32_t status,
5290                    ahc_search_action action)
5291 {
5292         struct  scb *scb;
5293         struct  scb *prev_scb;
5294         uint8_t qinstart;
5295         uint8_t qinpos;
5296         uint8_t qintail;
5297         uint8_t next;
5298         uint8_t prev;
5299         uint8_t curscbptr;
5300         int     found;
5301         int     have_qregs;
5302
5303         qintail = ahc->qinfifonext;
5304         have_qregs = (ahc->features & AHC_QUEUE_REGS) != 0;
5305         if (have_qregs) {
5306                 qinstart = ahc_inb(ahc, SNSCB_QOFF);
5307                 ahc_outb(ahc, SNSCB_QOFF, qinstart);
5308         } else
5309                 qinstart = ahc_inb(ahc, QINPOS);
5310         qinpos = qinstart;
5311         found = 0;
5312         prev_scb = NULL;
5313
5314         if (action == SEARCH_COMPLETE) {
5315                 /*
5316                  * Don't attempt to run any queued untagged transactions
5317                  * until we are done with the abort process.
5318                  */
5319                 ahc_freeze_untagged_queues(ahc);
5320         }
5321
5322         /*
5323          * Start with an empty queue.  Entries that are not chosen
5324          * for removal will be re-added to the queue as we go.
5325          */
5326         ahc->qinfifonext = qinpos;
5327         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5328
5329         while (qinpos != qintail) {
5330                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinpos]);
5331                 if (scb == NULL) {
5332                         printf("qinpos = %d, SCB index = %d\n",
5333                                 qinpos, ahc->qinfifo[qinpos]);
5334                         panic("Loop 1\n");
5335                 }
5336
5337                 if (ahc_match_scb(ahc, scb, target, channel, lun, tag, role)) {
5338                         /*
5339                          * We found an scb that needs to be acted on.
5340                          */
5341                         found++;
5342                         switch (action) {
5343                         case SEARCH_COMPLETE:
5344                         {
5345                                 cam_status ostat;
5346                                 cam_status cstat;
5347
5348                                 ostat = ahc_get_transaction_status(scb);
5349                                 if (ostat == CAM_REQ_INPROG)
5350                                         ahc_set_transaction_status(scb, status);
5351                                 cstat = ahc_get_transaction_status(scb);
5352                                 if (cstat != CAM_REQ_CMP)
5353                                         ahc_freeze_scb(scb);
5354                                 if ((scb->flags & SCB_ACTIVE) == 0)
5355                                         printf("Inactive SCB in qinfifo\n");
5356                                 ahc_done(ahc, scb);
5357
5358                                 /* FALLTHROUGH */
5359                         }
5360                         case SEARCH_REMOVE:
5361                                 break;
5362                         case SEARCH_COUNT:
5363                                 ahc_qinfifo_requeue(ahc, prev_scb, scb);
5364                                 prev_scb = scb;
5365                                 break;
5366                         }
5367                 } else {
5368                         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5369                         prev_scb = scb;
5370                 }
5371                 qinpos++;
5372         }
5373
5374         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5375                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5376         } else {
5377                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5378         }
5379
5380         if (action != SEARCH_COUNT
5381          && (found != 0)
5382          && (qinstart != ahc->qinfifonext)) {
5383                 /*
5384                  * The sequencer may be in the process of dmaing
5385                  * down the SCB at the beginning of the queue.
5386                  * This could be problematic if either the first,
5387                  * or the second SCB is removed from the queue
5388                  * (the first SCB includes a pointer to the "next"
5389                  * SCB to dma). If we have removed any entries, swap
5390                  * the first element in the queue with the next HSCB
5391                  * so the sequencer will notice that NEXT_QUEUED_SCB
5392                  * has changed during its dma attempt and will retry
5393                  * the DMA.
5394                  */
5395                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinstart]);
5396
5397                 if (scb == NULL) {
5398                         printf("found = %d, qinstart = %d, qinfifionext = %d\n",
5399                                 found, qinstart, ahc->qinfifonext);
5400                         panic("First/Second Qinfifo fixup\n");
5401                 }
5402                 /*
5403                  * ahc_swap_with_next_hscb forces our next pointer to
5404                  * point to the reserved SCB for future commands.  Save
5405                  * and restore our original next pointer to maintain
5406                  * queue integrity.
5407                  */
5408                 next = scb->hscb->next;
5409                 ahc->scb_data->scbindex[scb->hscb->tag] = NULL;
5410                 ahc_swap_with_next_hscb(ahc, scb);
5411                 scb->hscb->next = next;
5412                 ahc->qinfifo[qinstart] = scb->hscb->tag;
5413
5414                 /* Tell the card about the new head of the qinfifo. */
5415                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5416
5417                 /* Fixup the tail "next" pointer. */
5418                 qintail = ahc->qinfifonext - 1;
5419                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qintail]);
5420                 scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5421         }
5422
5423         /*
5424          * Search waiting for selection list.
5425          */
5426         curscbptr = ahc_inb(ahc, SCBPTR);
5427         next = ahc_inb(ahc, WAITING_SCBH);  /* Start at head of list. */
5428         prev = SCB_LIST_NULL;
5429
5430         while (next != SCB_LIST_NULL) {
5431                 uint8_t scb_index;
5432
5433                 ahc_outb(ahc, SCBPTR, next);
5434                 scb_index = ahc_inb(ahc, SCB_TAG);
5435                 if (scb_index >= ahc->scb_data->numscbs) {
5436                         printf("Waiting List inconsistency. "
5437                                "SCB index == %d, yet numscbs == %d.",
5438                                scb_index, ahc->scb_data->numscbs);
5439                         ahc_dump_card_state(ahc);
5440                         panic("for safety");
5441                 }
5442                 scb = ahc_lookup_scb(ahc, scb_index);
5443                 if (scb == NULL) {
5444                         printf("scb_index = %d, next = %d\n",
5445                                 scb_index, next);
5446                         panic("Waiting List traversal\n");
5447                 }
5448                 if (ahc_match_scb(ahc, scb, target, channel,
5449                                   lun, SCB_LIST_NULL, role)) {
5450                         /*
5451                          * We found an scb that needs to be acted on.
5452                          */
5453                         found++;
5454                         switch (action) {
5455                         case SEARCH_COMPLETE:
5456                         {
5457                                 cam_status ostat;
5458                                 cam_status cstat;
5459
5460                                 ostat = ahc_get_transaction_status(scb);
5461                                 if (ostat == CAM_REQ_INPROG)
5462                                         ahc_set_transaction_status(scb,
5463                                                                    status);
5464                                 cstat = ahc_get_transaction_status(scb);
5465                                 if (cstat != CAM_REQ_CMP)
5466                                         ahc_freeze_scb(scb);
5467                                 if ((scb->flags & SCB_ACTIVE) == 0)
5468                                         printf("Inactive SCB in Waiting List\n");
5469                                 ahc_done(ahc, scb);
5470                                 /* FALLTHROUGH */
5471                         }
5472                         case SEARCH_REMOVE:
5473                                 next = ahc_rem_wscb(ahc, next, prev);
5474                                 break;
5475                         case SEARCH_COUNT:
5476                                 prev = next;
5477                                 next = ahc_inb(ahc, SCB_NEXT);
5478                                 break;
5479                         }
5480                 } else {
5481                         
5482                         prev = next;
5483                         next = ahc_inb(ahc, SCB_NEXT);
5484                 }
5485         }
5486         ahc_outb(ahc, SCBPTR, curscbptr);
5487
5488         found += ahc_search_untagged_queues(ahc, /*ahc_io_ctx_t*/NULL, target,
5489                                             channel, lun, status, action);
5490
5491         if (action == SEARCH_COMPLETE)
5492                 ahc_release_untagged_queues(ahc);
5493         return (found);
5494 }
5495
5496 int
5497 ahc_search_untagged_queues(struct ahc_softc *ahc, ahc_io_ctx_t ctx,
5498                            int target, char channel, int lun, uint32_t status,
5499                            ahc_search_action action)
5500 {
5501         struct  scb *scb;
5502         int     maxtarget;
5503         int     found;
5504         int     i;
5505
5506         if (action == SEARCH_COMPLETE) {
5507                 /*
5508                  * Don't attempt to run any queued untagged transactions
5509                  * until we are done with the abort process.
5510                  */
5511                 ahc_freeze_untagged_queues(ahc);
5512         }
5513
5514         found = 0;
5515         i = 0;
5516         if ((ahc->flags & AHC_SCB_BTT) == 0) {
5517
5518                 maxtarget = 16;
5519                 if (target != CAM_TARGET_WILDCARD) {
5520
5521                         i = target;
5522                         if (channel == 'B')
5523                                 i += 8;
5524                         maxtarget = i + 1;
5525                 }
5526         } else {
5527                 maxtarget = 0;
5528         }
5529
5530         for (; i < maxtarget; i++) {
5531                 struct scb_tailq *untagged_q;
5532                 struct scb *next_scb;
5533
5534                 untagged_q = &(ahc->untagged_queues[i]);
5535                 next_scb = TAILQ_FIRST(untagged_q);
5536                 while (next_scb != NULL) {
5537
5538                         scb = next_scb;
5539                         next_scb = TAILQ_NEXT(scb, links.tqe);
5540
5541                         /*
5542                          * The head of the list may be the currently
5543                          * active untagged command for a device.
5544                          * We're only searching for commands that
5545                          * have not been started.  A transaction
5546                          * marked active but still in the qinfifo
5547                          * is removed by the qinfifo scanning code
5548                          * above.
5549                          */
5550                         if ((scb->flags & SCB_ACTIVE) != 0)
5551                                 continue;
5552
5553                         if (ahc_match_scb(ahc, scb, target, channel, lun,
5554                                           SCB_LIST_NULL, ROLE_INITIATOR) == 0
5555                          || (ctx != NULL && ctx != scb->io_ctx))
5556                                 continue;
5557
5558                         /*
5559                          * We found an scb that needs to be acted on.
5560                          */
5561                         found++;
5562                         switch (action) {
5563                         case SEARCH_COMPLETE:
5564                         {
5565                                 cam_status ostat;
5566                                 cam_status cstat;
5567
5568                                 ostat = ahc_get_transaction_status(scb);
5569                                 if (ostat == CAM_REQ_INPROG)
5570                                         ahc_set_transaction_status(scb, status);
5571                                 cstat = ahc_get_transaction_status(scb);
5572                                 if (cstat != CAM_REQ_CMP)
5573                                         ahc_freeze_scb(scb);
5574                                 if ((scb->flags & SCB_ACTIVE) == 0)
5575                                         printf("Inactive SCB in untaggedQ\n");
5576                                 ahc_done(ahc, scb);
5577                                 break;
5578                         }
5579                         case SEARCH_REMOVE:
5580                                 scb->flags &= ~SCB_UNTAGGEDQ;
5581                                 TAILQ_REMOVE(untagged_q, scb, links.tqe);
5582                                 break;
5583                         case SEARCH_COUNT:
5584                                 break;
5585                         }
5586                 }
5587         }
5588
5589         if (action == SEARCH_COMPLETE)
5590                 ahc_release_untagged_queues(ahc);
5591         return (found);
5592 }
5593
5594 int
5595 ahc_search_disc_list(struct ahc_softc *ahc, int target, char channel,
5596                      int lun, u_int tag, int stop_on_first, int remove,
5597                      int save_state)
5598 {
5599         struct  scb *scbp;
5600         u_int   next;
5601         u_int   prev;
5602         u_int   count;
5603         u_int   active_scb;
5604
5605         count = 0;
5606         next = ahc_inb(ahc, DISCONNECTED_SCBH);
5607         prev = SCB_LIST_NULL;
5608
5609         if (save_state) {
5610                 /* restore this when we're done */
5611                 active_scb = ahc_inb(ahc, SCBPTR);
5612         } else
5613                 /* Silence compiler */
5614                 active_scb = SCB_LIST_NULL;
5615
5616         while (next != SCB_LIST_NULL) {
5617                 u_int scb_index;
5618
5619                 ahc_outb(ahc, SCBPTR, next);
5620                 scb_index = ahc_inb(ahc, SCB_TAG);
5621                 if (scb_index >= ahc->scb_data->numscbs) {
5622                         printf("Disconnected List inconsistency. "
5623                                "SCB index == %d, yet numscbs == %d.",
5624                                scb_index, ahc->scb_data->numscbs);
5625                         ahc_dump_card_state(ahc);
5626                         panic("for safety");
5627                 }
5628
5629                 if (next == prev) {
5630                         panic("Disconnected List Loop. "
5631                               "cur SCBPTR == %x, prev SCBPTR == %x.",
5632                               next, prev);
5633                 }
5634                 scbp = ahc_lookup_scb(ahc, scb_index);
5635                 if (ahc_match_scb(ahc, scbp, target, channel, lun,
5636                                   tag, ROLE_INITIATOR)) {
5637                         count++;
5638                         if (remove) {
5639                                 next =
5640                                     ahc_rem_scb_from_disc_list(ahc, prev, next);
5641                         } else {
5642                                 prev = next;
5643                                 next = ahc_inb(ahc, SCB_NEXT);
5644                         }
5645                         if (stop_on_first)
5646                                 break;
5647                 } else {
5648                         prev = next;
5649                         next = ahc_inb(ahc, SCB_NEXT);
5650                 }
5651         }
5652         if (save_state)
5653                 ahc_outb(ahc, SCBPTR, active_scb);
5654         return (count);
5655 }
5656
5657 /*
5658  * Remove an SCB from the on chip list of disconnected transactions.
5659  * This is empty/unused if we are not performing SCB paging.
5660  */
5661 static u_int
5662 ahc_rem_scb_from_disc_list(struct ahc_softc *ahc, u_int prev, u_int scbptr)
5663 {
5664         u_int next;
5665
5666         ahc_outb(ahc, SCBPTR, scbptr);
5667         next = ahc_inb(ahc, SCB_NEXT);
5668
5669         ahc_outb(ahc, SCB_CONTROL, 0);
5670
5671         ahc_add_curscb_to_free_list(ahc);
5672
5673         if (prev != SCB_LIST_NULL) {
5674                 ahc_outb(ahc, SCBPTR, prev);
5675                 ahc_outb(ahc, SCB_NEXT, next);
5676         } else
5677                 ahc_outb(ahc, DISCONNECTED_SCBH, next);
5678
5679         return (next);
5680 }
5681
5682 /*
5683  * Add the SCB as selected by SCBPTR onto the on chip list of
5684  * free hardware SCBs.  This list is empty/unused if we are not
5685  * performing SCB paging.
5686  */
5687 static void
5688 ahc_add_curscb_to_free_list(struct ahc_softc *ahc)
5689 {
5690         /*
5691          * Invalidate the tag so that our abort
5692          * routines don't think it's active.
5693          */
5694         ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
5695
5696         if ((ahc->flags & AHC_PAGESCBS) != 0) {
5697                 ahc_outb(ahc, SCB_NEXT, ahc_inb(ahc, FREE_SCBH));
5698                 ahc_outb(ahc, FREE_SCBH, ahc_inb(ahc, SCBPTR));
5699         }
5700 }
5701
5702 /*
5703  * Manipulate the waiting for selection list and return the
5704  * scb that follows the one that we remove.
5705  */
5706 static u_int
5707 ahc_rem_wscb(struct ahc_softc *ahc, u_int scbpos, u_int prev)
5708 {       
5709         u_int curscb, next;
5710
5711         /*
5712          * Select the SCB we want to abort and
5713          * pull the next pointer out of it.
5714          */
5715         curscb = ahc_inb(ahc, SCBPTR);
5716         ahc_outb(ahc, SCBPTR, scbpos);
5717         next = ahc_inb(ahc, SCB_NEXT);
5718
5719         /* Clear the necessary fields */
5720         ahc_outb(ahc, SCB_CONTROL, 0);
5721
5722         ahc_add_curscb_to_free_list(ahc);
5723
5724         /* update the waiting list */
5725         if (prev == SCB_LIST_NULL) {
5726                 /* First in the list */
5727                 ahc_outb(ahc, WAITING_SCBH, next); 
5728
5729                 /*
5730                  * Ensure we aren't attempting to perform
5731                  * selection for this entry.
5732                  */
5733                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
5734         } else {
5735                 /*
5736                  * Select the scb that pointed to us 
5737                  * and update its next pointer.
5738                  */
5739                 ahc_outb(ahc, SCBPTR, prev);
5740                 ahc_outb(ahc, SCB_NEXT, next);
5741         }
5742
5743         /*
5744          * Point us back at the original scb position.
5745          */
5746         ahc_outb(ahc, SCBPTR, curscb);
5747         return next;
5748 }
5749
5750 /******************************** Error Handling ******************************/
5751 /*
5752  * Abort all SCBs that match the given description (target/channel/lun/tag),
5753  * setting their status to the passed in status if the status has not already
5754  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
5755  * is paused before it is called.
5756  */
5757 int
5758 ahc_abort_scbs(struct ahc_softc *ahc, int target, char channel,
5759                int lun, u_int tag, role_t role, uint32_t status)
5760 {
5761         struct  scb *scbp;
5762         struct  scb *scbp_next;
5763         u_int   active_scb;
5764         int     i, j;
5765         int     maxtarget;
5766         int     minlun;
5767         int     maxlun;
5768
5769         int     found;
5770
5771         /*
5772          * Don't attempt to run any queued untagged transactions
5773          * until we are done with the abort process.
5774          */
5775         ahc_freeze_untagged_queues(ahc);
5776
5777         /* restore this when we're done */
5778         active_scb = ahc_inb(ahc, SCBPTR);
5779
5780         found = ahc_search_qinfifo(ahc, target, channel, lun, SCB_LIST_NULL,
5781                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5782
5783         /*
5784          * Clean out the busy target table for any untagged commands.
5785          */
5786         i = 0;
5787         maxtarget = 16;
5788         if (target != CAM_TARGET_WILDCARD) {
5789                 i = target;
5790                 if (channel == 'B')
5791                         i += 8;
5792                 maxtarget = i + 1;
5793         }
5794
5795         if (lun == CAM_LUN_WILDCARD) {
5796
5797                 /*
5798                  * Unless we are using an SCB based
5799                  * busy targets table, there is only
5800                  * one table entry for all luns of
5801                  * a target.
5802                  */
5803                 minlun = 0;
5804                 maxlun = 1;
5805                 if ((ahc->flags & AHC_SCB_BTT) != 0)
5806                         maxlun = AHC_NUM_LUNS;
5807         } else {
5808                 minlun = lun;
5809                 maxlun = lun + 1;
5810         }
5811
5812         if (role != ROLE_TARGET) {
5813                 for (;i < maxtarget; i++) {
5814                         for (j = minlun;j < maxlun; j++) {
5815                                 u_int scbid;
5816                                 u_int tcl;
5817
5818                                 tcl = BUILD_TCL(i << 4, j);
5819                                 scbid = ahc_index_busy_tcl(ahc, tcl);
5820                                 scbp = ahc_lookup_scb(ahc, scbid);
5821                                 if (scbp == NULL
5822                                  || ahc_match_scb(ahc, scbp, target, channel,
5823                                                   lun, tag, role) == 0)
5824                                         continue;
5825                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, j));
5826                         }
5827                 }
5828
5829                 /*
5830                  * Go through the disconnected list and remove any entries we
5831                  * have queued for completion, 0'ing their control byte too.
5832                  * We save the active SCB and restore it ourselves, so there
5833                  * is no reason for this search to restore it too.
5834                  */
5835                 ahc_search_disc_list(ahc, target, channel, lun, tag,
5836                                      /*stop_on_first*/FALSE, /*remove*/TRUE,
5837                                      /*save_state*/FALSE);
5838         }
5839
5840         /*
5841          * Go through the hardware SCB array looking for commands that
5842          * were active but not on any list.  In some cases, these remnants
5843          * might not still have mappings in the scbindex array (e.g. unexpected
5844          * bus free with the same scb queued for an abort).  Don't hold this
5845          * against them.
5846          */
5847         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
5848                 u_int scbid;
5849
5850                 ahc_outb(ahc, SCBPTR, i);
5851                 scbid = ahc_inb(ahc, SCB_TAG);
5852                 scbp = ahc_lookup_scb(ahc, scbid);
5853                 if ((scbp == NULL && scbid != SCB_LIST_NULL)
5854                  || (scbp != NULL
5855                   && ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)))
5856                         ahc_add_curscb_to_free_list(ahc);
5857         }
5858
5859         /*
5860          * Go through the pending CCB list and look for
5861          * commands for this target that are still active.
5862          * These are other tagged commands that were
5863          * disconnected when the reset occurred.
5864          */
5865         scbp_next = LIST_FIRST(&ahc->pending_scbs);
5866         while (scbp_next != NULL) {
5867                 scbp = scbp_next;
5868                 scbp_next = LIST_NEXT(scbp, pending_links);
5869                 if (ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)) {
5870                         cam_status ostat;
5871
5872                         ostat = ahc_get_transaction_status(scbp);
5873                         if (ostat == CAM_REQ_INPROG)
5874                                 ahc_set_transaction_status(scbp, status);
5875                         if (ahc_get_transaction_status(scbp) != CAM_REQ_CMP)
5876                                 ahc_freeze_scb(scbp);
5877                         if ((scbp->flags & SCB_ACTIVE) == 0)
5878                                 printf("Inactive SCB on pending list\n");
5879                         ahc_done(ahc, scbp);
5880                         found++;
5881                 }
5882         }
5883         ahc_outb(ahc, SCBPTR, active_scb);
5884         ahc_platform_abort_scbs(ahc, target, channel, lun, tag, role, status);
5885         ahc_release_untagged_queues(ahc);
5886         return found;
5887 }
5888
5889 static void
5890 ahc_reset_current_bus(struct ahc_softc *ahc)
5891 {
5892         uint8_t scsiseq;
5893
5894         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENSCSIRST);
5895         scsiseq = ahc_inb(ahc, SCSISEQ);
5896         ahc_outb(ahc, SCSISEQ, scsiseq | SCSIRSTO);
5897         ahc_flush_device_writes(ahc);
5898         ahc_delay(AHC_BUSRESET_DELAY);
5899         /* Turn off the bus reset */
5900         ahc_outb(ahc, SCSISEQ, scsiseq & ~SCSIRSTO);
5901
5902         ahc_clear_intstat(ahc);
5903
5904         /* Re-enable reset interrupts */
5905         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) | ENSCSIRST);
5906 }
5907
5908 int
5909 ahc_reset_channel(struct ahc_softc *ahc, char channel, int initiate_reset)
5910 {
5911         struct  ahc_devinfo devinfo;
5912         u_int   initiator, target, max_scsiid;
5913         u_int   sblkctl;
5914         u_int   scsiseq;
5915         u_int   simode1;
5916         int     found;
5917         int     restart_needed;
5918         char    cur_channel;
5919
5920         ahc->pending_device = NULL;
5921
5922         ahc_compile_devinfo(&devinfo,
5923                             CAM_TARGET_WILDCARD,
5924                             CAM_TARGET_WILDCARD,
5925                             CAM_LUN_WILDCARD,
5926                             channel, ROLE_UNKNOWN);
5927         ahc_pause(ahc);
5928
5929         /* Make sure the sequencer is in a safe location. */
5930         ahc_clear_critical_section(ahc);
5931
5932         /*
5933          * Run our command complete fifos to ensure that we perform
5934          * completion processing on any commands that 'completed'
5935          * before the reset occurred.
5936          */
5937         ahc_run_qoutfifo(ahc);
5938 #ifdef AHC_TARGET_MODE
5939         /*
5940          * XXX - In Twin mode, the tqinfifo may have commands
5941          *       for an unaffected channel in it.  However, if
5942          *       we have run out of ATIO resources to drain that
5943          *       queue, we may not get them all out here.  Further,
5944          *       the blocked transactions for the reset channel
5945          *       should just be killed off, irrespecitve of whether
5946          *       we are blocked on ATIO resources.  Write a routine
5947          *       to compact the tqinfifo appropriately.
5948          */
5949         if ((ahc->flags & AHC_TARGETROLE) != 0) {
5950                 ahc_run_tqinfifo(ahc, /*paused*/TRUE);
5951         }
5952 #endif
5953
5954         /*
5955          * Reset the bus if we are initiating this reset
5956          */
5957         sblkctl = ahc_inb(ahc, SBLKCTL);
5958         cur_channel = 'A';
5959         if ((ahc->features & AHC_TWIN) != 0
5960          && ((sblkctl & SELBUSB) != 0))
5961             cur_channel = 'B';
5962         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
5963         if (cur_channel != channel) {
5964                 /* Case 1: Command for another bus is active
5965                  * Stealthily reset the other bus without
5966                  * upsetting the current bus.
5967                  */
5968                 ahc_outb(ahc, SBLKCTL, sblkctl ^ SELBUSB);
5969                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
5970 #ifdef AHC_TARGET_MODE
5971                 /*
5972                  * Bus resets clear ENSELI, so we cannot
5973                  * defer re-enabling bus reset interrupts
5974                  * if we are in target mode.
5975                  */
5976                 if ((ahc->flags & AHC_TARGETROLE) != 0)
5977                         simode1 |= ENSCSIRST;
5978 #endif
5979                 ahc_outb(ahc, SIMODE1, simode1);
5980                 if (initiate_reset)
5981                         ahc_reset_current_bus(ahc);
5982                 ahc_clear_intstat(ahc);
5983                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
5984                 ahc_outb(ahc, SBLKCTL, sblkctl);
5985                 restart_needed = FALSE;
5986         } else {
5987                 /* Case 2: A command from this bus is active or we're idle */
5988                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
5989 #ifdef AHC_TARGET_MODE
5990                 /*
5991                  * Bus resets clear ENSELI, so we cannot
5992                  * defer re-enabling bus reset interrupts
5993                  * if we are in target mode.
5994                  */
5995                 if ((ahc->flags & AHC_TARGETROLE) != 0)
5996                         simode1 |= ENSCSIRST;
5997 #endif
5998                 ahc_outb(ahc, SIMODE1, simode1);
5999                 if (initiate_reset)
6000                         ahc_reset_current_bus(ahc);
6001                 ahc_clear_intstat(ahc);
6002                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6003                 restart_needed = TRUE;
6004         }
6005
6006         /*
6007          * Clean up all the state information for the
6008          * pending transactions on this bus.
6009          */
6010         found = ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, channel,
6011                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
6012                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6013
6014         max_scsiid = (ahc->features & AHC_WIDE) ? 15 : 7;
6015
6016 #ifdef AHC_TARGET_MODE
6017         /*
6018          * Send an immediate notify ccb to all target more peripheral
6019          * drivers affected by this action.
6020          */
6021         for (target = 0; target <= max_scsiid; target++) {
6022                 struct ahc_tmode_tstate* tstate;
6023                 u_int lun;
6024
6025                 tstate = ahc->enabled_targets[target];
6026                 if (tstate == NULL)
6027                         continue;
6028                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
6029                         struct ahc_tmode_lstate* lstate;
6030
6031                         lstate = tstate->enabled_luns[lun];
6032                         if (lstate == NULL)
6033                                 continue;
6034
6035                         ahc_queue_lstate_event(ahc, lstate, CAM_TARGET_WILDCARD,
6036                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
6037                         ahc_send_lstate_events(ahc, lstate);
6038                 }
6039         }
6040 #endif
6041         /* Notify the XPT that a bus reset occurred */
6042         ahc_send_async(ahc, devinfo.channel, CAM_TARGET_WILDCARD,
6043                        CAM_LUN_WILDCARD, AC_BUS_RESET);
6044
6045         /*
6046          * Revert to async/narrow transfers until we renegotiate.
6047          */
6048         for (target = 0; target <= max_scsiid; target++) {
6049
6050                 if (ahc->enabled_targets[target] == NULL)
6051                         continue;
6052                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
6053                         struct ahc_devinfo devinfo;
6054
6055                         ahc_compile_devinfo(&devinfo, target, initiator,
6056                                             CAM_LUN_WILDCARD,
6057                                             channel, ROLE_UNKNOWN);
6058                         ahc_set_width(ahc, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6059                                       AHC_TRANS_CUR, /*paused*/TRUE);
6060                         ahc_set_syncrate(ahc, &devinfo, /*syncrate*/NULL,
6061                                          /*period*/0, /*offset*/0,
6062                                          /*ppr_options*/0, AHC_TRANS_CUR,
6063                                          /*paused*/TRUE);
6064                 }
6065         }
6066
6067         if (restart_needed)
6068                 ahc_restart(ahc);
6069         else
6070                 ahc_unpause(ahc);
6071         return found;
6072 }
6073
6074
6075 /***************************** Residual Processing ****************************/
6076 /*
6077  * Calculate the residual for a just completed SCB.
6078  */
6079 void
6080 ahc_calc_residual(struct ahc_softc *ahc, struct scb *scb)
6081 {
6082         struct hardware_scb *hscb;
6083         struct status_pkt *spkt;
6084         uint32_t sgptr;
6085         uint32_t resid_sgptr;
6086         uint32_t resid;
6087
6088         /*
6089          * 5 cases.
6090          * 1) No residual.
6091          *    SG_RESID_VALID clear in sgptr.
6092          * 2) Transferless command
6093          * 3) Never performed any transfers.
6094          *    sgptr has SG_FULL_RESID set.
6095          * 4) No residual but target did not
6096          *    save data pointers after the
6097          *    last transfer, so sgptr was
6098          *    never updated.
6099          * 5) We have a partial residual.
6100          *    Use residual_sgptr to determine
6101          *    where we are.
6102          */
6103
6104         hscb = scb->hscb;
6105         sgptr = ahc_le32toh(hscb->sgptr);
6106         if ((sgptr & SG_RESID_VALID) == 0)
6107                 /* Case 1 */
6108                 return;
6109         sgptr &= ~SG_RESID_VALID;
6110
6111         if ((sgptr & SG_LIST_NULL) != 0)
6112                 /* Case 2 */
6113                 return;
6114
6115         spkt = &hscb->shared_data.status;
6116         resid_sgptr = ahc_le32toh(spkt->residual_sg_ptr);
6117         if ((sgptr & SG_FULL_RESID) != 0) {
6118                 /* Case 3 */
6119                 resid = ahc_get_transfer_length(scb);
6120         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
6121                 /* Case 4 */
6122                 return;
6123         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
6124                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
6125         } else {
6126                 struct ahc_dma_seg *sg;
6127
6128                 /*
6129                  * Remainder of the SG where the transfer
6130                  * stopped.  
6131                  */
6132                 resid = ahc_le32toh(spkt->residual_datacnt) & AHC_SG_LEN_MASK;
6133                 sg = ahc_sg_bus_to_virt(scb, resid_sgptr & SG_PTR_MASK);
6134
6135                 /* The residual sg_ptr always points to the next sg */
6136                 sg--;
6137
6138                 /*
6139                  * Add up the contents of all residual
6140                  * SG segments that are after the SG where
6141                  * the transfer stopped.
6142                  */
6143                 while ((ahc_le32toh(sg->len) & AHC_DMA_LAST_SEG) == 0) {
6144                         sg++;
6145                         resid += ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
6146                 }
6147         }
6148         if ((scb->flags & SCB_SENSE) == 0)
6149                 ahc_set_residual(scb, resid);
6150         else
6151                 ahc_set_sense_residual(scb, resid);
6152
6153 #ifdef AHC_DEBUG
6154         if ((ahc_debug & AHC_SHOW_MISC) != 0) {
6155                 ahc_print_path(ahc, scb);
6156                 printf("Handled %sResidual of %d bytes\n",
6157                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
6158         }
6159 #endif
6160 }
6161
6162 /******************************* Target Mode **********************************/
6163 #ifdef AHC_TARGET_MODE
6164 /*
6165  * Add a target mode event to this lun's queue
6166  */
6167 static void
6168 ahc_queue_lstate_event(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate,
6169                        u_int initiator_id, u_int event_type, u_int event_arg)
6170 {
6171         struct ahc_tmode_event *event;
6172         int pending;
6173
6174         xpt_freeze_devq(lstate->path, /*count*/1);
6175         if (lstate->event_w_idx >= lstate->event_r_idx)
6176                 pending = lstate->event_w_idx - lstate->event_r_idx;
6177         else
6178                 pending = AHC_TMODE_EVENT_BUFFER_SIZE + 1
6179                         - (lstate->event_r_idx - lstate->event_w_idx);
6180
6181         if (event_type == EVENT_TYPE_BUS_RESET
6182          || event_type == MSG_BUS_DEV_RESET) {
6183                 /*
6184                  * Any earlier events are irrelevant, so reset our buffer.
6185                  * This has the effect of allowing us to deal with reset
6186                  * floods (an external device holding down the reset line)
6187                  * without losing the event that is really interesting.
6188                  */
6189                 lstate->event_r_idx = 0;
6190                 lstate->event_w_idx = 0;
6191                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
6192         }
6193
6194         if (pending == AHC_TMODE_EVENT_BUFFER_SIZE) {
6195                 xpt_print_path(lstate->path);
6196                 printf("immediate event %x:%x lost\n",
6197                        lstate->event_buffer[lstate->event_r_idx].event_type,
6198                        lstate->event_buffer[lstate->event_r_idx].event_arg);
6199                 lstate->event_r_idx++;
6200                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6201                         lstate->event_r_idx = 0;
6202                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
6203         }
6204
6205         event = &lstate->event_buffer[lstate->event_w_idx];
6206         event->initiator_id = initiator_id;
6207         event->event_type = event_type;
6208         event->event_arg = event_arg;
6209         lstate->event_w_idx++;
6210         if (lstate->event_w_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6211                 lstate->event_w_idx = 0;
6212 }
6213
6214 /*
6215  * Send any target mode events queued up waiting
6216  * for immediate notify resources.
6217  */
6218 void
6219 ahc_send_lstate_events(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate)
6220 {
6221         struct ccb_hdr *ccbh;
6222         struct ccb_immed_notify *inot;
6223
6224         while (lstate->event_r_idx != lstate->event_w_idx
6225             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
6226                 struct ahc_tmode_event *event;
6227
6228                 event = &lstate->event_buffer[lstate->event_r_idx];
6229                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
6230                 inot = (struct ccb_immed_notify *)ccbh;
6231                 switch (event->event_type) {
6232                 case EVENT_TYPE_BUS_RESET:
6233                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
6234                         break;
6235                 default:
6236                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
6237                         inot->message_args[0] = event->event_type;
6238                         inot->message_args[1] = event->event_arg;
6239                         break;
6240                 }
6241                 inot->initiator_id = event->initiator_id;
6242                 inot->sense_len = 0;
6243                 xpt_done((union ccb *)inot);
6244                 lstate->event_r_idx++;
6245                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6246                         lstate->event_r_idx = 0;
6247         }
6248 }
6249 #endif
6250
6251 /******************** Sequencer Program Patching/Download *********************/
6252
6253 #ifdef AHC_DUMP_SEQ
6254 void
6255 ahc_dumpseq(struct ahc_softc* ahc)
6256 {
6257         int i;
6258
6259         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6260         ahc_outb(ahc, SEQADDR0, 0);
6261         ahc_outb(ahc, SEQADDR1, 0);
6262         for (i = 0; i < ahc->instruction_ram_size; i++) {
6263                 uint8_t ins_bytes[4];
6264
6265                 ahc_insb(ahc, SEQRAM, ins_bytes, 4);
6266                 printf("0x%08x\n", ins_bytes[0] << 24
6267                                  | ins_bytes[1] << 16
6268                                  | ins_bytes[2] << 8
6269                                  | ins_bytes[3]);
6270         }
6271 }
6272 #endif
6273
6274 static int
6275 ahc_loadseq(struct ahc_softc *ahc)
6276 {
6277         struct  cs cs_table[num_critical_sections];
6278         u_int   begin_set[num_critical_sections];
6279         u_int   end_set[num_critical_sections];
6280         struct  patch *cur_patch;
6281         u_int   cs_count;
6282         u_int   cur_cs;
6283         u_int   i;
6284         u_int   skip_addr;
6285         u_int   sg_prefetch_cnt;
6286         int     downloaded;
6287         uint8_t download_consts[7];
6288
6289         /*
6290          * Start out with 0 critical sections
6291          * that apply to this firmware load.
6292          */
6293         cs_count = 0;
6294         cur_cs = 0;
6295         memset(begin_set, 0, sizeof(begin_set));
6296         memset(end_set, 0, sizeof(end_set));
6297
6298         /* Setup downloadable constant table */
6299         download_consts[QOUTFIFO_OFFSET] = 0;
6300         if (ahc->targetcmds != NULL)
6301                 download_consts[QOUTFIFO_OFFSET] += 32;
6302         download_consts[QINFIFO_OFFSET] = download_consts[QOUTFIFO_OFFSET] + 1;
6303         download_consts[CACHESIZE_MASK] = ahc->pci_cachesize - 1;
6304         download_consts[INVERTED_CACHESIZE_MASK] = ~(ahc->pci_cachesize - 1);
6305         sg_prefetch_cnt = ahc->pci_cachesize;
6306         if (sg_prefetch_cnt < (2 * sizeof(struct ahc_dma_seg)))
6307                 sg_prefetch_cnt = 2 * sizeof(struct ahc_dma_seg);
6308         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
6309         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_cnt - 1);
6310         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_cnt - 1);
6311
6312         cur_patch = patches;
6313         downloaded = 0;
6314         skip_addr = 0;
6315         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6316         ahc_outb(ahc, SEQADDR0, 0);
6317         ahc_outb(ahc, SEQADDR1, 0);
6318
6319         for (i = 0; i < sizeof(seqprog)/4; i++) {
6320                 if (ahc_check_patch(ahc, &cur_patch, i, &skip_addr) == 0) {
6321                         /*
6322                          * Don't download this instruction as it
6323                          * is in a patch that was removed.
6324                          */
6325                         continue;
6326                 }
6327
6328                 if (downloaded == ahc->instruction_ram_size) {
6329                         /*
6330                          * We're about to exceed the instruction
6331                          * storage capacity for this chip.  Fail
6332                          * the load.
6333                          */
6334                         printf("\n%s: Program too large for instruction memory "
6335                                "size of %d!\n", ahc_name(ahc),
6336                                ahc->instruction_ram_size);
6337                         return (ENOMEM);
6338                 }
6339
6340                 /*
6341                  * Move through the CS table until we find a CS
6342                  * that might apply to this instruction.
6343                  */
6344                 for (; cur_cs < num_critical_sections; cur_cs++) {
6345                         if (critical_sections[cur_cs].end <= i) {
6346                                 if (begin_set[cs_count] == TRUE
6347                                  && end_set[cs_count] == FALSE) {
6348                                         cs_table[cs_count].end = downloaded;
6349                                         end_set[cs_count] = TRUE;
6350                                         cs_count++;
6351                                 }
6352                                 continue;
6353                         }
6354                         if (critical_sections[cur_cs].begin <= i
6355                          && begin_set[cs_count] == FALSE) {
6356                                 cs_table[cs_count].begin = downloaded;
6357                                 begin_set[cs_count] = TRUE;
6358                         }
6359                         break;
6360                 }
6361                 ahc_download_instr(ahc, i, download_consts);
6362                 downloaded++;
6363         }
6364
6365         ahc->num_critical_sections = cs_count;
6366         if (cs_count != 0) {
6367
6368                 cs_count *= sizeof(struct cs);
6369                 ahc->critical_sections = malloc(cs_count, M_DEVBUF, M_NOWAIT);
6370                 if (ahc->critical_sections == NULL)
6371                         panic("ahc_loadseq: Could not malloc");
6372                 memcpy(ahc->critical_sections, cs_table, cs_count);
6373         }
6374         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE);
6375
6376         if (bootverbose) {
6377                 printf(" %d instructions downloaded\n", downloaded);
6378                 printf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
6379                        ahc_name(ahc), ahc->features, ahc->bugs, ahc->flags);
6380         }
6381         return (0);
6382 }
6383
6384 static int
6385 ahc_check_patch(struct ahc_softc *ahc, struct patch **start_patch,
6386                 u_int start_instr, u_int *skip_addr)
6387 {
6388         struct  patch *cur_patch;
6389         struct  patch *last_patch;
6390         u_int   num_patches;
6391
6392         num_patches = ARRAY_SIZE(patches);
6393         last_patch = &patches[num_patches];
6394         cur_patch = *start_patch;
6395
6396         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
6397
6398                 if (cur_patch->patch_func(ahc) == 0) {
6399
6400                         /* Start rejecting code */
6401                         *skip_addr = start_instr + cur_patch->skip_instr;
6402                         cur_patch += cur_patch->skip_patch;
6403                 } else {
6404                         /* Accepted this patch.  Advance to the next
6405                          * one and wait for our intruction pointer to
6406                          * hit this point.
6407                          */
6408                         cur_patch++;
6409                 }
6410         }
6411
6412         *start_patch = cur_patch;
6413         if (start_instr < *skip_addr)
6414                 /* Still skipping */
6415                 return (0);
6416
6417         return (1);
6418 }
6419
6420 static void
6421 ahc_download_instr(struct ahc_softc *ahc, u_int instrptr, uint8_t *dconsts)
6422 {
6423         union   ins_formats instr;
6424         struct  ins_format1 *fmt1_ins;
6425         struct  ins_format3 *fmt3_ins;
6426         u_int   opcode;
6427
6428         /*
6429          * The firmware is always compiled into a little endian format.
6430          */
6431         instr.integer = ahc_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
6432
6433         fmt1_ins = &instr.format1;
6434         fmt3_ins = NULL;
6435
6436         /* Pull the opcode */
6437         opcode = instr.format1.opcode;
6438         switch (opcode) {
6439         case AIC_OP_JMP:
6440         case AIC_OP_JC:
6441         case AIC_OP_JNC:
6442         case AIC_OP_CALL:
6443         case AIC_OP_JNE:
6444         case AIC_OP_JNZ:
6445         case AIC_OP_JE:
6446         case AIC_OP_JZ:
6447         {
6448                 struct patch *cur_patch;
6449                 int address_offset;
6450                 u_int address;
6451                 u_int skip_addr;
6452                 u_int i;
6453
6454                 fmt3_ins = &instr.format3;
6455                 address_offset = 0;
6456                 address = fmt3_ins->address;
6457                 cur_patch = patches;
6458                 skip_addr = 0;
6459
6460                 for (i = 0; i < address;) {
6461
6462                         ahc_check_patch(ahc, &cur_patch, i, &skip_addr);
6463
6464                         if (skip_addr > i) {
6465                                 int end_addr;
6466
6467                                 end_addr = min(address, skip_addr);
6468                                 address_offset += end_addr - i;
6469                                 i = skip_addr;
6470                         } else {
6471                                 i++;
6472                         }
6473                 }
6474                 address -= address_offset;
6475                 fmt3_ins->address = address;
6476                 /* FALLTHROUGH */
6477         }
6478         case AIC_OP_OR:
6479         case AIC_OP_AND:
6480         case AIC_OP_XOR:
6481         case AIC_OP_ADD:
6482         case AIC_OP_ADC:
6483         case AIC_OP_BMOV:
6484                 if (fmt1_ins->parity != 0) {
6485                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
6486                 }
6487                 fmt1_ins->parity = 0;
6488                 if ((ahc->features & AHC_CMD_CHAN) == 0
6489                  && opcode == AIC_OP_BMOV) {
6490                         /*
6491                          * Block move was added at the same time
6492                          * as the command channel.  Verify that
6493                          * this is only a move of a single element
6494                          * and convert the BMOV to a MOV
6495                          * (AND with an immediate of FF).
6496                          */
6497                         if (fmt1_ins->immediate != 1)
6498                                 panic("%s: BMOV not supported\n",
6499                                       ahc_name(ahc));
6500                         fmt1_ins->opcode = AIC_OP_AND;
6501                         fmt1_ins->immediate = 0xff;
6502                 }
6503                 /* FALLTHROUGH */
6504         case AIC_OP_ROL:
6505                 if ((ahc->features & AHC_ULTRA2) != 0) {
6506                         int i, count;
6507
6508                         /* Calculate odd parity for the instruction */
6509                         for (i = 0, count = 0; i < 31; i++) {
6510                                 uint32_t mask;
6511
6512                                 mask = 0x01 << i;
6513                                 if ((instr.integer & mask) != 0)
6514                                         count++;
6515                         }
6516                         if ((count & 0x01) == 0)
6517                                 instr.format1.parity = 1;
6518                 } else {
6519                         /* Compress the instruction for older sequencers */
6520                         if (fmt3_ins != NULL) {
6521                                 instr.integer =
6522                                         fmt3_ins->immediate
6523                                       | (fmt3_ins->source << 8)
6524                                       | (fmt3_ins->address << 16)
6525                                       | (fmt3_ins->opcode << 25);
6526                         } else {
6527                                 instr.integer =
6528                                         fmt1_ins->immediate
6529                                       | (fmt1_ins->source << 8)
6530                                       | (fmt1_ins->destination << 16)
6531                                       | (fmt1_ins->ret << 24)
6532                                       | (fmt1_ins->opcode << 25);
6533                         }
6534                 }
6535                 /* The sequencer is a little endian cpu */
6536                 instr.integer = ahc_htole32(instr.integer);
6537                 ahc_outsb(ahc, SEQRAM, instr.bytes, 4);
6538                 break;
6539         default:
6540                 panic("Unknown opcode encountered in seq program");
6541                 break;
6542         }
6543 }
6544
6545 int
6546 ahc_print_register(ahc_reg_parse_entry_t *table, u_int num_entries,
6547                    const char *name, u_int address, u_int value,
6548                    u_int *cur_column, u_int wrap_point)
6549 {
6550         int     printed;
6551         u_int   printed_mask;
6552
6553         if (cur_column != NULL && *cur_column >= wrap_point) {
6554                 printf("\n");
6555                 *cur_column = 0;
6556         }
6557         printed = printf("%s[0x%x]", name, value);
6558         if (table == NULL) {
6559                 printed += printf(" ");
6560                 *cur_column += printed;
6561                 return (printed);
6562         }
6563         printed_mask = 0;
6564         while (printed_mask != 0xFF) {
6565                 int entry;
6566
6567                 for (entry = 0; entry < num_entries; entry++) {
6568                         if (((value & table[entry].mask)
6569                           != table[entry].value)
6570                          || ((printed_mask & table[entry].mask)
6571                           == table[entry].mask))
6572                                 continue;
6573
6574                         printed += printf("%s%s",
6575                                           printed_mask == 0 ? ":(" : "|",
6576                                           table[entry].name);
6577                         printed_mask |= table[entry].mask;
6578                         
6579                         break;
6580                 }
6581                 if (entry >= num_entries)
6582                         break;
6583         }
6584         if (printed_mask != 0)
6585                 printed += printf(") ");
6586         else
6587                 printed += printf(" ");
6588         if (cur_column != NULL)
6589                 *cur_column += printed;
6590         return (printed);
6591 }
6592
6593 void
6594 ahc_dump_card_state(struct ahc_softc *ahc)
6595 {
6596         struct  scb *scb;
6597         struct  scb_tailq *untagged_q;
6598         u_int   cur_col;
6599         int     paused;
6600         int     target;
6601         int     maxtarget;
6602         int     i;
6603         uint8_t last_phase;
6604         uint8_t qinpos;
6605         uint8_t qintail;
6606         uint8_t qoutpos;
6607         uint8_t scb_index;
6608         uint8_t saved_scbptr;
6609
6610         if (ahc_is_paused(ahc)) {
6611                 paused = 1;
6612         } else {
6613                 paused = 0;
6614                 ahc_pause(ahc);
6615         }
6616
6617         saved_scbptr = ahc_inb(ahc, SCBPTR);
6618         last_phase = ahc_inb(ahc, LASTPHASE);
6619         printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
6620                "%s: Dumping Card State %s, at SEQADDR 0x%x\n",
6621                ahc_name(ahc), ahc_lookup_phase_entry(last_phase)->phasemsg,
6622                ahc_inb(ahc, SEQADDR0) | (ahc_inb(ahc, SEQADDR1) << 8));
6623         if (paused)
6624                 printf("Card was paused\n");
6625         printf("ACCUM = 0x%x, SINDEX = 0x%x, DINDEX = 0x%x, ARG_2 = 0x%x\n",
6626                ahc_inb(ahc, ACCUM), ahc_inb(ahc, SINDEX), ahc_inb(ahc, DINDEX),
6627                ahc_inb(ahc, ARG_2));
6628         printf("HCNT = 0x%x SCBPTR = 0x%x\n", ahc_inb(ahc, HCNT),
6629                ahc_inb(ahc, SCBPTR));
6630         cur_col = 0;
6631         if ((ahc->features & AHC_DT) != 0)
6632                 ahc_scsiphase_print(ahc_inb(ahc, SCSIPHASE), &cur_col, 50);
6633         ahc_scsisigi_print(ahc_inb(ahc, SCSISIGI), &cur_col, 50);
6634         ahc_error_print(ahc_inb(ahc, ERROR), &cur_col, 50);
6635         ahc_scsibusl_print(ahc_inb(ahc, SCSIBUSL), &cur_col, 50);
6636         ahc_lastphase_print(ahc_inb(ahc, LASTPHASE), &cur_col, 50);
6637         ahc_scsiseq_print(ahc_inb(ahc, SCSISEQ), &cur_col, 50);
6638         ahc_sblkctl_print(ahc_inb(ahc, SBLKCTL), &cur_col, 50);
6639         ahc_scsirate_print(ahc_inb(ahc, SCSIRATE), &cur_col, 50);
6640         ahc_seqctl_print(ahc_inb(ahc, SEQCTL), &cur_col, 50);
6641         ahc_seq_flags_print(ahc_inb(ahc, SEQ_FLAGS), &cur_col, 50);
6642         ahc_sstat0_print(ahc_inb(ahc, SSTAT0), &cur_col, 50);
6643         ahc_sstat1_print(ahc_inb(ahc, SSTAT1), &cur_col, 50);
6644         ahc_sstat2_print(ahc_inb(ahc, SSTAT2), &cur_col, 50);
6645         ahc_sstat3_print(ahc_inb(ahc, SSTAT3), &cur_col, 50);
6646         ahc_simode0_print(ahc_inb(ahc, SIMODE0), &cur_col, 50);
6647         ahc_simode1_print(ahc_inb(ahc, SIMODE1), &cur_col, 50);
6648         ahc_sxfrctl0_print(ahc_inb(ahc, SXFRCTL0), &cur_col, 50);
6649         ahc_dfcntrl_print(ahc_inb(ahc, DFCNTRL), &cur_col, 50);
6650         ahc_dfstatus_print(ahc_inb(ahc, DFSTATUS), &cur_col, 50);
6651         if (cur_col != 0)
6652                 printf("\n");
6653         printf("STACK:");
6654         for (i = 0; i < STACK_SIZE; i++)
6655                printf(" 0x%x", ahc_inb(ahc, STACK)|(ahc_inb(ahc, STACK) << 8));
6656         printf("\nSCB count = %d\n", ahc->scb_data->numscbs);
6657         printf("Kernel NEXTQSCB = %d\n", ahc->next_queued_scb->hscb->tag);
6658         printf("Card NEXTQSCB = %d\n", ahc_inb(ahc, NEXT_QUEUED_SCB));
6659         /* QINFIFO */
6660         printf("QINFIFO entries: ");
6661         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
6662                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
6663                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
6664         } else
6665                 qinpos = ahc_inb(ahc, QINPOS);
6666         qintail = ahc->qinfifonext;
6667         while (qinpos != qintail) {
6668                 printf("%d ", ahc->qinfifo[qinpos]);
6669                 qinpos++;
6670         }
6671         printf("\n");
6672
6673         printf("Waiting Queue entries: ");
6674         scb_index = ahc_inb(ahc, WAITING_SCBH);
6675         i = 0;
6676         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6677                 ahc_outb(ahc, SCBPTR, scb_index);
6678                 printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
6679                 scb_index = ahc_inb(ahc, SCB_NEXT);
6680         }
6681         printf("\n");
6682
6683         printf("Disconnected Queue entries: ");
6684         scb_index = ahc_inb(ahc, DISCONNECTED_SCBH);
6685         i = 0;
6686         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6687                 ahc_outb(ahc, SCBPTR, scb_index);
6688                 printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
6689                 scb_index = ahc_inb(ahc, SCB_NEXT);
6690         }
6691         printf("\n");
6692                 
6693         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
6694         printf("QOUTFIFO entries: ");
6695         qoutpos = ahc->qoutfifonext;
6696         i = 0;
6697         while (ahc->qoutfifo[qoutpos] != SCB_LIST_NULL && i++ < 256) {
6698                 printf("%d ", ahc->qoutfifo[qoutpos]);
6699                 qoutpos++;
6700         }
6701         printf("\n");
6702
6703         printf("Sequencer Free SCB List: ");
6704         scb_index = ahc_inb(ahc, FREE_SCBH);
6705         i = 0;
6706         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6707                 ahc_outb(ahc, SCBPTR, scb_index);
6708                 printf("%d ", scb_index);
6709                 scb_index = ahc_inb(ahc, SCB_NEXT);
6710         }
6711         printf("\n");
6712
6713         printf("Sequencer SCB Info: ");
6714         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
6715                 ahc_outb(ahc, SCBPTR, i);
6716                 cur_col = printf("\n%3d ", i);
6717
6718                 ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL), &cur_col, 60);
6719                 ahc_scb_scsiid_print(ahc_inb(ahc, SCB_SCSIID), &cur_col, 60);
6720                 ahc_scb_lun_print(ahc_inb(ahc, SCB_LUN), &cur_col, 60);
6721                 ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
6722         }
6723         printf("\n");
6724
6725         printf("Pending list: ");
6726         i = 0;
6727         LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
6728                 if (i++ > 256)
6729                         break;
6730                 cur_col = printf("\n%3d ", scb->hscb->tag);
6731                 ahc_scb_control_print(scb->hscb->control, &cur_col, 60);
6732                 ahc_scb_scsiid_print(scb->hscb->scsiid, &cur_col, 60);
6733                 ahc_scb_lun_print(scb->hscb->lun, &cur_col, 60);
6734                 if ((ahc->flags & AHC_PAGESCBS) == 0) {
6735                         ahc_outb(ahc, SCBPTR, scb->hscb->tag);
6736                         printf("(");
6737                         ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL),
6738                                               &cur_col, 60);
6739                         ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
6740                         printf(")");
6741                 }
6742         }
6743         printf("\n");
6744
6745         printf("Kernel Free SCB list: ");
6746         i = 0;
6747         SLIST_FOREACH(scb, &ahc->scb_data->free_scbs, links.sle) {
6748                 if (i++ > 256)
6749                         break;
6750                 printf("%d ", scb->hscb->tag);
6751         }
6752         printf("\n");
6753
6754         maxtarget = (ahc->features & (AHC_WIDE|AHC_TWIN)) ? 15 : 7;
6755         for (target = 0; target <= maxtarget; target++) {
6756                 untagged_q = &ahc->untagged_queues[target];
6757                 if (TAILQ_FIRST(untagged_q) == NULL)
6758                         continue;
6759                 printf("Untagged Q(%d): ", target);
6760                 i = 0;
6761                 TAILQ_FOREACH(scb, untagged_q, links.tqe) {
6762                         if (i++ > 256)
6763                                 break;
6764                         printf("%d ", scb->hscb->tag);
6765                 }
6766                 printf("\n");
6767         }
6768
6769         ahc_platform_dump_card_state(ahc);
6770         printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
6771         ahc_outb(ahc, SCBPTR, saved_scbptr);
6772         if (paused == 0)
6773                 ahc_unpause(ahc);
6774 }
6775
6776 /************************* Target Mode ****************************************/
6777 #ifdef AHC_TARGET_MODE
6778 cam_status
6779 ahc_find_tmode_devs(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb,
6780                     struct ahc_tmode_tstate **tstate,
6781                     struct ahc_tmode_lstate **lstate,
6782                     int notfound_failure)
6783 {
6784
6785         if ((ahc->features & AHC_TARGETMODE) == 0)
6786                 return (CAM_REQ_INVALID);
6787
6788         /*
6789          * Handle the 'black hole' device that sucks up
6790          * requests to unattached luns on enabled targets.
6791          */
6792         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
6793          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
6794                 *tstate = NULL;
6795                 *lstate = ahc->black_hole;
6796         } else {
6797                 u_int max_id;
6798
6799                 max_id = (ahc->features & AHC_WIDE) ? 16 : 8;
6800                 if (ccb->ccb_h.target_id >= max_id)
6801                         return (CAM_TID_INVALID);
6802
6803                 if (ccb->ccb_h.target_lun >= AHC_NUM_LUNS)
6804                         return (CAM_LUN_INVALID);
6805
6806                 *tstate = ahc->enabled_targets[ccb->ccb_h.target_id];
6807                 *lstate = NULL;
6808                 if (*tstate != NULL)
6809                         *lstate =
6810                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
6811         }
6812
6813         if (notfound_failure != 0 && *lstate == NULL)
6814                 return (CAM_PATH_INVALID);
6815
6816         return (CAM_REQ_CMP);
6817 }
6818
6819 void
6820 ahc_handle_en_lun(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb)
6821 {
6822         struct     ahc_tmode_tstate *tstate;
6823         struct     ahc_tmode_lstate *lstate;
6824         struct     ccb_en_lun *cel;
6825         cam_status status;
6826         u_long     s;
6827         u_int      target;
6828         u_int      lun;
6829         u_int      target_mask;
6830         u_int      our_id;
6831         int        error;
6832         char       channel;
6833
6834         status = ahc_find_tmode_devs(ahc, sim, ccb, &tstate, &lstate,
6835                                      /*notfound_failure*/FALSE);
6836
6837         if (status != CAM_REQ_CMP) {
6838                 ccb->ccb_h.status = status;
6839                 return;
6840         }
6841
6842         if (cam_sim_bus(sim) == 0)
6843                 our_id = ahc->our_id;
6844         else
6845                 our_id = ahc->our_id_b;
6846
6847         if (ccb->ccb_h.target_id != our_id) {
6848                 /*
6849                  * our_id represents our initiator ID, or
6850                  * the ID of the first target to have an
6851                  * enabled lun in target mode.  There are
6852                  * two cases that may preclude enabling a
6853                  * target id other than our_id.
6854                  *
6855                  *   o our_id is for an active initiator role.
6856                  *     Since the hardware does not support
6857                  *     reselections to the initiator role at
6858                  *     anything other than our_id, and our_id
6859                  *     is used by the hardware to indicate the
6860                  *     ID to use for both select-out and
6861                  *     reselect-out operations, the only target
6862                  *     ID we can support in this mode is our_id.
6863                  *
6864                  *   o The MULTARGID feature is not available and
6865                  *     a previous target mode ID has been enabled.
6866                  */
6867                 if ((ahc->features & AHC_MULTIROLE) != 0) {
6868
6869                         if ((ahc->features & AHC_MULTI_TID) != 0
6870                          && (ahc->flags & AHC_INITIATORROLE) != 0) {
6871                                 /*
6872                                  * Only allow additional targets if
6873                                  * the initiator role is disabled.
6874                                  * The hardware cannot handle a re-select-in
6875                                  * on the initiator id during a re-select-out
6876                                  * on a different target id.
6877                                  */
6878                                 status = CAM_TID_INVALID;
6879                         } else if ((ahc->flags & AHC_INITIATORROLE) != 0
6880                                 || ahc->enabled_luns > 0) {
6881                                 /*
6882                                  * Only allow our target id to change
6883                                  * if the initiator role is not configured
6884                                  * and there are no enabled luns which
6885                                  * are attached to the currently registered
6886                                  * scsi id.
6887                                  */
6888                                 status = CAM_TID_INVALID;
6889                         }
6890                 } else if ((ahc->features & AHC_MULTI_TID) == 0
6891                         && ahc->enabled_luns > 0) {
6892
6893                         status = CAM_TID_INVALID;
6894                 }
6895         }
6896
6897         if (status != CAM_REQ_CMP) {
6898                 ccb->ccb_h.status = status;
6899                 return;
6900         }
6901
6902         /*
6903          * We now have an id that is valid.
6904          * If we aren't in target mode, switch modes.
6905          */
6906         if ((ahc->flags & AHC_TARGETROLE) == 0
6907          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
6908                 u_long   s;
6909                 ahc_flag saved_flags;
6910
6911                 printf("Configuring Target Mode\n");
6912                 ahc_lock(ahc, &s);
6913                 if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
6914                         ccb->ccb_h.status = CAM_BUSY;
6915                         ahc_unlock(ahc, &s);
6916                         return;
6917                 }
6918                 saved_flags = ahc->flags;
6919                 ahc->flags |= AHC_TARGETROLE;
6920                 if ((ahc->features & AHC_MULTIROLE) == 0)
6921                         ahc->flags &= ~AHC_INITIATORROLE;
6922                 ahc_pause(ahc);
6923                 error = ahc_loadseq(ahc);
6924                 if (error != 0) {
6925                         /*
6926                          * Restore original configuration and notify
6927                          * the caller that we cannot support target mode.
6928                          * Since the adapter started out in this
6929                          * configuration, the firmware load will succeed,
6930                          * so there is no point in checking ahc_loadseq's
6931                          * return value.
6932                          */
6933                         ahc->flags = saved_flags;
6934                         (void)ahc_loadseq(ahc);
6935                         ahc_restart(ahc);
6936                         ahc_unlock(ahc, &s);
6937                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
6938                         return;
6939                 }
6940                 ahc_restart(ahc);
6941                 ahc_unlock(ahc, &s);
6942         }
6943         cel = &ccb->cel;
6944         target = ccb->ccb_h.target_id;
6945         lun = ccb->ccb_h.target_lun;
6946         channel = SIM_CHANNEL(ahc, sim);
6947         target_mask = 0x01 << target;
6948         if (channel == 'B')
6949                 target_mask <<= 8;
6950
6951         if (cel->enable != 0) {
6952                 u_int scsiseq;
6953
6954                 /* Are we already enabled?? */
6955                 if (lstate != NULL) {
6956                         xpt_print_path(ccb->ccb_h.path);
6957                         printf("Lun already enabled\n");
6958                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
6959                         return;
6960                 }
6961
6962                 if (cel->grp6_len != 0
6963                  || cel->grp7_len != 0) {
6964                         /*
6965                          * Don't (yet?) support vendor
6966                          * specific commands.
6967                          */
6968                         ccb->ccb_h.status = CAM_REQ_INVALID;
6969                         printf("Non-zero Group Codes\n");
6970                         return;
6971                 }
6972
6973                 /*
6974                  * Seems to be okay.
6975                  * Setup our data structures.
6976                  */
6977                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
6978                         tstate = ahc_alloc_tstate(ahc, target, channel);
6979                         if (tstate == NULL) {
6980                                 xpt_print_path(ccb->ccb_h.path);
6981                                 printf("Couldn't allocate tstate\n");
6982                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
6983                                 return;
6984                         }
6985                 }
6986                 lstate = malloc(sizeof(*lstate), M_DEVBUF, M_NOWAIT);
6987                 if (lstate == NULL) {
6988                         xpt_print_path(ccb->ccb_h.path);
6989                         printf("Couldn't allocate lstate\n");
6990                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
6991                         return;
6992                 }
6993                 memset(lstate, 0, sizeof(*lstate));
6994                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
6995                                          xpt_path_path_id(ccb->ccb_h.path),
6996                                          xpt_path_target_id(ccb->ccb_h.path),
6997                                          xpt_path_lun_id(ccb->ccb_h.path));
6998                 if (status != CAM_REQ_CMP) {
6999                         free(lstate, M_DEVBUF);
7000                         xpt_print_path(ccb->ccb_h.path);
7001                         printf("Couldn't allocate path\n");
7002                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7003                         return;
7004                 }
7005                 SLIST_INIT(&lstate->accept_tios);
7006                 SLIST_INIT(&lstate->immed_notifies);
7007                 ahc_lock(ahc, &s);
7008                 ahc_pause(ahc);
7009                 if (target != CAM_TARGET_WILDCARD) {
7010                         tstate->enabled_luns[lun] = lstate;
7011                         ahc->enabled_luns++;
7012
7013                         if ((ahc->features & AHC_MULTI_TID) != 0) {
7014                                 u_int targid_mask;
7015
7016                                 targid_mask = ahc_inb(ahc, TARGID)
7017                                             | (ahc_inb(ahc, TARGID + 1) << 8);
7018
7019                                 targid_mask |= target_mask;
7020                                 ahc_outb(ahc, TARGID, targid_mask);
7021                                 ahc_outb(ahc, TARGID+1, (targid_mask >> 8));
7022                                 
7023                                 ahc_update_scsiid(ahc, targid_mask);
7024                         } else {
7025                                 u_int our_id;
7026                                 char  channel;
7027
7028                                 channel = SIM_CHANNEL(ahc, sim);
7029                                 our_id = SIM_SCSI_ID(ahc, sim);
7030
7031                                 /*
7032                                  * This can only happen if selections
7033                                  * are not enabled
7034                                  */
7035                                 if (target != our_id) {
7036                                         u_int sblkctl;
7037                                         char  cur_channel;
7038                                         int   swap;
7039
7040                                         sblkctl = ahc_inb(ahc, SBLKCTL);
7041                                         cur_channel = (sblkctl & SELBUSB)
7042                                                     ? 'B' : 'A';
7043                                         if ((ahc->features & AHC_TWIN) == 0)
7044                                                 cur_channel = 'A';
7045                                         swap = cur_channel != channel;
7046                                         if (channel == 'A')
7047                                                 ahc->our_id = target;
7048                                         else
7049                                                 ahc->our_id_b = target;
7050
7051                                         if (swap)
7052                                                 ahc_outb(ahc, SBLKCTL,
7053                                                          sblkctl ^ SELBUSB);
7054
7055                                         ahc_outb(ahc, SCSIID, target);
7056
7057                                         if (swap)
7058                                                 ahc_outb(ahc, SBLKCTL, sblkctl);
7059                                 }
7060                         }
7061                 } else
7062                         ahc->black_hole = lstate;
7063                 /* Allow select-in operations */
7064                 if (ahc->black_hole != NULL && ahc->enabled_luns > 0) {
7065                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7066                         scsiseq |= ENSELI;
7067                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7068                         scsiseq = ahc_inb(ahc, SCSISEQ);
7069                         scsiseq |= ENSELI;
7070                         ahc_outb(ahc, SCSISEQ, scsiseq);
7071                 }
7072                 ahc_unpause(ahc);
7073                 ahc_unlock(ahc, &s);
7074                 ccb->ccb_h.status = CAM_REQ_CMP;
7075                 xpt_print_path(ccb->ccb_h.path);
7076                 printf("Lun now enabled for target mode\n");
7077         } else {
7078                 struct scb *scb;
7079                 int i, empty;
7080
7081                 if (lstate == NULL) {
7082                         ccb->ccb_h.status = CAM_LUN_INVALID;
7083                         return;
7084                 }
7085
7086                 ahc_lock(ahc, &s);
7087                 
7088                 ccb->ccb_h.status = CAM_REQ_CMP;
7089                 LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7090                         struct ccb_hdr *ccbh;
7091
7092                         ccbh = &scb->io_ctx->ccb_h;
7093                         if (ccbh->func_code == XPT_CONT_TARGET_IO
7094                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
7095                                 printf("CTIO pending\n");
7096                                 ccb->ccb_h.status = CAM_REQ_INVALID;
7097                                 ahc_unlock(ahc, &s);
7098                                 return;
7099                         }
7100                 }
7101
7102                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
7103                         printf("ATIOs pending\n");
7104                         ccb->ccb_h.status = CAM_REQ_INVALID;
7105                 }
7106
7107                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
7108                         printf("INOTs pending\n");
7109                         ccb->ccb_h.status = CAM_REQ_INVALID;
7110                 }
7111
7112                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
7113                         ahc_unlock(ahc, &s);
7114                         return;
7115                 }
7116
7117                 xpt_print_path(ccb->ccb_h.path);
7118                 printf("Target mode disabled\n");
7119                 xpt_free_path(lstate->path);
7120                 free(lstate, M_DEVBUF);
7121
7122                 ahc_pause(ahc);
7123                 /* Can we clean up the target too? */
7124                 if (target != CAM_TARGET_WILDCARD) {
7125                         tstate->enabled_luns[lun] = NULL;
7126                         ahc->enabled_luns--;
7127                         for (empty = 1, i = 0; i < 8; i++)
7128                                 if (tstate->enabled_luns[i] != NULL) {
7129                                         empty = 0;
7130                                         break;
7131                                 }
7132
7133                         if (empty) {
7134                                 ahc_free_tstate(ahc, target, channel,
7135                                                 /*force*/FALSE);
7136                                 if (ahc->features & AHC_MULTI_TID) {
7137                                         u_int targid_mask;
7138
7139                                         targid_mask = ahc_inb(ahc, TARGID)
7140                                                     | (ahc_inb(ahc, TARGID + 1)
7141                                                        << 8);
7142
7143                                         targid_mask &= ~target_mask;
7144                                         ahc_outb(ahc, TARGID, targid_mask);
7145                                         ahc_outb(ahc, TARGID+1,
7146                                                  (targid_mask >> 8));
7147                                         ahc_update_scsiid(ahc, targid_mask);
7148                                 }
7149                         }
7150                 } else {
7151
7152                         ahc->black_hole = NULL;
7153
7154                         /*
7155                          * We can't allow selections without
7156                          * our black hole device.
7157                          */
7158                         empty = TRUE;
7159                 }
7160                 if (ahc->enabled_luns == 0) {
7161                         /* Disallow select-in */
7162                         u_int scsiseq;
7163
7164                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7165                         scsiseq &= ~ENSELI;
7166                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7167                         scsiseq = ahc_inb(ahc, SCSISEQ);
7168                         scsiseq &= ~ENSELI;
7169                         ahc_outb(ahc, SCSISEQ, scsiseq);
7170
7171                         if ((ahc->features & AHC_MULTIROLE) == 0) {
7172                                 printf("Configuring Initiator Mode\n");
7173                                 ahc->flags &= ~AHC_TARGETROLE;
7174                                 ahc->flags |= AHC_INITIATORROLE;
7175                                 /*
7176                                  * Returning to a configuration that
7177                                  * fit previously will always succeed.
7178                                  */
7179                                 (void)ahc_loadseq(ahc);
7180                                 ahc_restart(ahc);
7181                                 /*
7182                                  * Unpaused.  The extra unpause
7183                                  * that follows is harmless.
7184                                  */
7185                         }
7186                 }
7187                 ahc_unpause(ahc);
7188                 ahc_unlock(ahc, &s);
7189         }
7190 }
7191
7192 static void
7193 ahc_update_scsiid(struct ahc_softc *ahc, u_int targid_mask)
7194 {
7195         u_int scsiid_mask;
7196         u_int scsiid;
7197
7198         if ((ahc->features & AHC_MULTI_TID) == 0)
7199                 panic("ahc_update_scsiid called on non-multitid unit\n");
7200
7201         /*
7202          * Since we will rely on the TARGID mask
7203          * for selection enables, ensure that OID
7204          * in SCSIID is not set to some other ID
7205          * that we don't want to allow selections on.
7206          */
7207         if ((ahc->features & AHC_ULTRA2) != 0)
7208                 scsiid = ahc_inb(ahc, SCSIID_ULTRA2);
7209         else
7210                 scsiid = ahc_inb(ahc, SCSIID);
7211         scsiid_mask = 0x1 << (scsiid & OID);
7212         if ((targid_mask & scsiid_mask) == 0) {
7213                 u_int our_id;
7214
7215                 /* ffs counts from 1 */
7216                 our_id = ffs(targid_mask);
7217                 if (our_id == 0)
7218                         our_id = ahc->our_id;
7219                 else
7220                         our_id--;
7221                 scsiid &= TID;
7222                 scsiid |= our_id;
7223         }
7224         if ((ahc->features & AHC_ULTRA2) != 0)
7225                 ahc_outb(ahc, SCSIID_ULTRA2, scsiid);
7226         else
7227                 ahc_outb(ahc, SCSIID, scsiid);
7228 }
7229
7230 void
7231 ahc_run_tqinfifo(struct ahc_softc *ahc, int paused)
7232 {
7233         struct target_cmd *cmd;
7234
7235         /*
7236          * If the card supports auto-access pause,
7237          * we can access the card directly regardless
7238          * of whether it is paused or not.
7239          */
7240         if ((ahc->features & AHC_AUTOPAUSE) != 0)
7241                 paused = TRUE;
7242
7243         ahc_sync_tqinfifo(ahc, BUS_DMASYNC_POSTREAD);
7244         while ((cmd = &ahc->targetcmds[ahc->tqinfifonext])->cmd_valid != 0) {
7245
7246                 /*
7247                  * Only advance through the queue if we
7248                  * have the resources to process the command.
7249                  */
7250                 if (ahc_handle_target_cmd(ahc, cmd) != 0)
7251                         break;
7252
7253                 cmd->cmd_valid = 0;
7254                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
7255                                 ahc->shared_data_dmamap,
7256                                 ahc_targetcmd_offset(ahc, ahc->tqinfifonext),
7257                                 sizeof(struct target_cmd),
7258                                 BUS_DMASYNC_PREREAD);
7259                 ahc->tqinfifonext++;
7260
7261                 /*
7262                  * Lazily update our position in the target mode incoming
7263                  * command queue as seen by the sequencer.
7264                  */
7265                 if ((ahc->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
7266                         if ((ahc->features & AHC_HS_MAILBOX) != 0) {
7267                                 u_int hs_mailbox;
7268
7269                                 hs_mailbox = ahc_inb(ahc, HS_MAILBOX);
7270                                 hs_mailbox &= ~HOST_TQINPOS;
7271                                 hs_mailbox |= ahc->tqinfifonext & HOST_TQINPOS;
7272                                 ahc_outb(ahc, HS_MAILBOX, hs_mailbox);
7273                         } else {
7274                                 if (!paused)
7275                                         ahc_pause(ahc); 
7276                                 ahc_outb(ahc, KERNEL_TQINPOS,
7277                                          ahc->tqinfifonext & HOST_TQINPOS);
7278                                 if (!paused)
7279                                         ahc_unpause(ahc);
7280                         }
7281                 }
7282         }
7283 }
7284
7285 static int
7286 ahc_handle_target_cmd(struct ahc_softc *ahc, struct target_cmd *cmd)
7287 {
7288         struct    ahc_tmode_tstate *tstate;
7289         struct    ahc_tmode_lstate *lstate;
7290         struct    ccb_accept_tio *atio;
7291         uint8_t *byte;
7292         int       initiator;
7293         int       target;
7294         int       lun;
7295
7296         initiator = SCSIID_TARGET(ahc, cmd->scsiid);
7297         target = SCSIID_OUR_ID(cmd->scsiid);
7298         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
7299
7300         byte = cmd->bytes;
7301         tstate = ahc->enabled_targets[target];
7302         lstate = NULL;
7303         if (tstate != NULL)
7304                 lstate = tstate->enabled_luns[lun];
7305
7306         /*
7307          * Commands for disabled luns go to the black hole driver.
7308          */
7309         if (lstate == NULL)
7310                 lstate = ahc->black_hole;
7311
7312         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
7313         if (atio == NULL) {
7314                 ahc->flags |= AHC_TQINFIFO_BLOCKED;
7315                 /*
7316                  * Wait for more ATIOs from the peripheral driver for this lun.
7317                  */
7318                 if (bootverbose)
7319                         printf("%s: ATIOs exhausted\n", ahc_name(ahc));
7320                 return (1);
7321         } else
7322                 ahc->flags &= ~AHC_TQINFIFO_BLOCKED;
7323 #if 0
7324         printf("Incoming command from %d for %d:%d%s\n",
7325                initiator, target, lun,
7326                lstate == ahc->black_hole ? "(Black Holed)" : "");
7327 #endif
7328         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
7329
7330         if (lstate == ahc->black_hole) {
7331                 /* Fill in the wildcards */
7332                 atio->ccb_h.target_id = target;
7333                 atio->ccb_h.target_lun = lun;
7334         }
7335
7336         /*
7337          * Package it up and send it off to
7338          * whomever has this lun enabled.
7339          */
7340         atio->sense_len = 0;
7341         atio->init_id = initiator;
7342         if (byte[0] != 0xFF) {
7343                 /* Tag was included */
7344                 atio->tag_action = *byte++;
7345                 atio->tag_id = *byte++;
7346                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
7347         } else {
7348                 atio->ccb_h.flags = 0;
7349         }
7350         byte++;
7351
7352         /* Okay.  Now determine the cdb size based on the command code */
7353         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
7354         case 0:
7355                 atio->cdb_len = 6;
7356                 break;
7357         case 1:
7358         case 2:
7359                 atio->cdb_len = 10;
7360                 break;
7361         case 4:
7362                 atio->cdb_len = 16;
7363                 break;
7364         case 5:
7365                 atio->cdb_len = 12;
7366                 break;
7367         case 3:
7368         default:
7369                 /* Only copy the opcode. */
7370                 atio->cdb_len = 1;
7371                 printf("Reserved or VU command code type encountered\n");
7372                 break;
7373         }
7374         
7375         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
7376
7377         atio->ccb_h.status |= CAM_CDB_RECVD;
7378
7379         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
7380                 /*
7381                  * We weren't allowed to disconnect.
7382                  * We're hanging on the bus until a
7383                  * continue target I/O comes in response
7384                  * to this accept tio.
7385                  */
7386 #if 0
7387                 printf("Received Immediate Command %d:%d:%d - %p\n",
7388                        initiator, target, lun, ahc->pending_device);
7389 #endif
7390                 ahc->pending_device = lstate;
7391                 ahc_freeze_ccb((union ccb *)atio);
7392                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
7393         }
7394         xpt_done((union ccb*)atio);
7395         return (0);
7396 }
7397
7398 #endif