git.oblomov.eu Git - linux-2.6/blob - drivers/scsi/aic7xxx/aic79xx_core.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2003 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#250 $
  41  */
  42
  43 #ifdef __linux__
  44 #include "aic79xx_osm.h"
  45 #include "aic79xx_inline.h"
  46 #include "aicasm/aicasm_insformat.h"
  47 #else
  48 #include <dev/aic7xxx/aic79xx_osm.h>
  49 #include <dev/aic7xxx/aic79xx_inline.h>
  50 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
  51 #endif
  52
  53
  54 /***************************** Lookup Tables **********************************/
  55 char *ahd_chip_names[] =
  56 {
  57         "NONE",
  58         "aic7901",
  59         "aic7902",
  60         "aic7901A"
  61 };
  62 static const u_int num_chip_names = NUM_ELEMENTS(ahd_chip_names);
  63
  64 /*
  65  * Hardware error codes.
  66  */
  67 struct ahd_hard_error_entry {
  68         uint8_t errno;
  69         char *errmesg;
  70 };
  71
  72 static struct ahd_hard_error_entry ahd_hard_errors[] = {
  73         { DSCTMOUT,     "Discard Timer has timed out" },
  74         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  75         { SQPARERR,     "Sequencer Parity Error" },
  76         { DPARERR,      "Data-path Parity Error" },
  77         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  78         { CIOPARERR,    "CIOBUS Parity Error" },
  79 };
  80 static const u_int num_errors = NUM_ELEMENTS(ahd_hard_errors);
  81
  82 static struct ahd_phase_table_entry ahd_phase_table[] =
  83 {
  84         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  85         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  86         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  87         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  88         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
  89         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
  90         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
  91         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
  92         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
  93         { 0,            MSG_NOOP,               "in unknown phase"      }
  94 };
  95
  96 /*
  97  * In most cases we only wish to itterate over real phases, so
  98  * exclude the last element from the count.
  99  */
 100 static const u_int num_phases = NUM_ELEMENTS(ahd_phase_table) - 1;
 101
 102 /* Our Sequencer Program */
 103 #include "aic79xx_seq.h"
 104
 105 /**************************** Function Declarations ***************************/
 106 static void             ahd_handle_transmission_error(struct ahd_softc *ahd);
 107 static void             ahd_handle_lqiphase_error(struct ahd_softc *ahd,
 108                                                   u_int lqistat1);
 109 static int              ahd_handle_pkt_busfree(struct ahd_softc *ahd,
 110                                                u_int busfreetime);
 111 static int              ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
 112 static void             ahd_handle_proto_violation(struct ahd_softc *ahd);
 113 static void             ahd_force_renegotiation(struct ahd_softc *ahd,
 114                                                 struct ahd_devinfo *devinfo);
 115
 116 static struct ahd_tmode_tstate*
 117                         ahd_alloc_tstate(struct ahd_softc *ahd,
 118                                          u_int scsi_id, char channel);
 119 #ifdef AHD_TARGET_MODE
 120 static void             ahd_free_tstate(struct ahd_softc *ahd,
 121                                         u_int scsi_id, char channel, int force);
 122 #endif
 123 static void             ahd_devlimited_syncrate(struct ahd_softc *ahd,
 124                                                 struct ahd_initiator_tinfo *,
 125                                                 u_int *period,
 126                                                 u_int *ppr_options,
 127                                                 role_t role);
 128 static void             ahd_update_neg_table(struct ahd_softc *ahd,
 129                                              struct ahd_devinfo *devinfo,
 130                                              struct ahd_transinfo *tinfo);
 131 static void             ahd_update_pending_scbs(struct ahd_softc *ahd);
 132 static void             ahd_fetch_devinfo(struct ahd_softc *ahd,
 133                                           struct ahd_devinfo *devinfo);
 134 static void             ahd_scb_devinfo(struct ahd_softc *ahd,
 135                                         struct ahd_devinfo *devinfo,
 136                                         struct scb *scb);
 137 static void             ahd_setup_initiator_msgout(struct ahd_softc *ahd,
 138                                                    struct ahd_devinfo *devinfo,
 139                                                    struct scb *scb);
 140 static void             ahd_build_transfer_msg(struct ahd_softc *ahd,
 141                                                struct ahd_devinfo *devinfo);
 142 static void             ahd_construct_sdtr(struct ahd_softc *ahd,
 143                                            struct ahd_devinfo *devinfo,
 144                                            u_int period, u_int offset);
 145 static void             ahd_construct_wdtr(struct ahd_softc *ahd,
 146                                            struct ahd_devinfo *devinfo,
 147                                            u_int bus_width);
 148 static void             ahd_construct_ppr(struct ahd_softc *ahd,
 149                                           struct ahd_devinfo *devinfo,
 150                                           u_int period, u_int offset,
 151                                           u_int bus_width, u_int ppr_options);
 152 static void             ahd_clear_msg_state(struct ahd_softc *ahd);
 153 static void             ahd_handle_message_phase(struct ahd_softc *ahd);
 154 typedef enum {
 155         AHDMSG_1B,
 156         AHDMSG_2B,
 157         AHDMSG_EXT
 158 } ahd_msgtype;
 159 static int              ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
 160                                      u_int msgval, int full);
 161 static int              ahd_parse_msg(struct ahd_softc *ahd,
 162                                       struct ahd_devinfo *devinfo);
 163 static int              ahd_handle_msg_reject(struct ahd_softc *ahd,
 164                                               struct ahd_devinfo *devinfo);
 165 static void             ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
 166                                                 struct ahd_devinfo *devinfo);
 167 static void             ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
 168 static void             ahd_handle_devreset(struct ahd_softc *ahd,
 169                                             struct ahd_devinfo *devinfo,
 170                                             u_int lun, cam_status status,
 171                                             char *message, int verbose_level);
 172 #ifdef AHD_TARGET_MODE
 173 static void             ahd_setup_target_msgin(struct ahd_softc *ahd,
 174                                                struct ahd_devinfo *devinfo,
 175                                                struct scb *scb);
 176 #endif
 177
 178 static u_int            ahd_sglist_size(struct ahd_softc *ahd);
 179 static u_int            ahd_sglist_allocsize(struct ahd_softc *ahd);
 180 static bus_dmamap_callback_t
 181                         ahd_dmamap_cb;
 182 static void             ahd_initialize_hscbs(struct ahd_softc *ahd);
 183 static int              ahd_init_scbdata(struct ahd_softc *ahd);
 184 static void             ahd_fini_scbdata(struct ahd_softc *ahd);
 185 static void             ahd_setup_iocell_workaround(struct ahd_softc *ahd);
 186 static void             ahd_iocell_first_selection(struct ahd_softc *ahd);
 187 static void             ahd_add_col_list(struct ahd_softc *ahd,
 188                                          struct scb *scb, u_int col_idx);
 189 static void             ahd_rem_col_list(struct ahd_softc *ahd,
 190                                          struct scb *scb);
 191 static void             ahd_chip_init(struct ahd_softc *ahd);
 192 static void             ahd_qinfifo_requeue(struct ahd_softc *ahd,
 193                                             struct scb *prev_scb,
 194                                             struct scb *scb);
 195 static int              ahd_qinfifo_count(struct ahd_softc *ahd);
 196 static int              ahd_search_scb_list(struct ahd_softc *ahd, int target,
 197                                             char channel, int lun, u_int tag,
 198                                             role_t role, uint32_t status,
 199                                             ahd_search_action action,
 200                                             u_int *list_head, u_int *list_tail,
 201                                             u_int tid);
 202 static void             ahd_stitch_tid_list(struct ahd_softc *ahd,
 203                                             u_int tid_prev, u_int tid_cur,
 204                                             u_int tid_next);
 205 static void             ahd_add_scb_to_free_list(struct ahd_softc *ahd,
 206                                                  u_int scbid);
 207 static u_int            ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
 208                                      u_int prev, u_int next, u_int tid);
 209 static void             ahd_reset_current_bus(struct ahd_softc *ahd);
 210 static ahd_callback_t   ahd_reset_poll;
 211 static ahd_callback_t   ahd_stat_timer;
 212 #ifdef AHD_DUMP_SEQ
 213 static void             ahd_dumpseq(struct ahd_softc *ahd);
 214 #endif
 215 static void             ahd_loadseq(struct ahd_softc *ahd);
 216 static int              ahd_check_patch(struct ahd_softc *ahd,
 217                                         struct patch **start_patch,
 218                                         u_int start_instr, u_int *skip_addr);
 219 static u_int            ahd_resolve_seqaddr(struct ahd_softc *ahd,
 220                                             u_int address);
 221 static void             ahd_download_instr(struct ahd_softc *ahd,
 222                                            u_int instrptr, uint8_t *dconsts);
 223 static int              ahd_probe_stack_size(struct ahd_softc *ahd);
 224 static int              ahd_scb_active_in_fifo(struct ahd_softc *ahd,
 225                                                struct scb *scb);
 226 static void             ahd_run_data_fifo(struct ahd_softc *ahd,
 227                                           struct scb *scb);
 228
 229 #ifdef AHD_TARGET_MODE
 230 static void             ahd_queue_lstate_event(struct ahd_softc *ahd,
 231                                                struct ahd_tmode_lstate *lstate,
 232                                                u_int initiator_id,
 233                                                u_int event_type,
 234                                                u_int event_arg);
 235 static void             ahd_update_scsiid(struct ahd_softc *ahd,
 236                                           u_int targid_mask);
 237 static int              ahd_handle_target_cmd(struct ahd_softc *ahd,
 238                                               struct target_cmd *cmd);
 239 #endif
 240
 241 /******************************** Private Inlines *****************************/
 242 static __inline void    ahd_assert_atn(struct ahd_softc *ahd);
 243 static __inline int     ahd_currently_packetized(struct ahd_softc *ahd);
 244 static __inline int     ahd_set_active_fifo(struct ahd_softc *ahd);
 245
 246 static __inline void
 247 ahd_assert_atn(struct ahd_softc *ahd)
 248 {
 249         ahd_outb(ahd, SCSISIGO, ATNO);
 250 }
 251
 252 /*
 253  * Determine if the current connection has a packetized
 254  * agreement.  This does not necessarily mean that we
 255  * are currently in a packetized transfer.  We could
 256  * just as easily be sending or receiving a message.
 257  */
 258 static __inline int
 259 ahd_currently_packetized(struct ahd_softc *ahd)
 260 {
 261         ahd_mode_state   saved_modes;
 262         int              packetized;
 263
 264         saved_modes = ahd_save_modes(ahd);
 265         if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
 266                 /*
 267                  * The packetized bit refers to the last
 268                  * connection, not the current one.  Check
 269                  * for non-zero LQISTATE instead.
 270                  */
 271                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 272                 packetized = ahd_inb(ahd, LQISTATE) != 0;
 273         } else {
 274                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 275                 packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
 276         }
 277         ahd_restore_modes(ahd, saved_modes);
 278         return (packetized);
 279 }
 280
 281 static __inline int
 282 ahd_set_active_fifo(struct ahd_softc *ahd)
 283 {
 284         u_int active_fifo;
 285
 286         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 287         active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
 288         switch (active_fifo) {
 289         case 0:
 290         case 1:
 291                 ahd_set_modes(ahd, active_fifo, active_fifo);
 292                 return (1);
 293         default:
 294                 return (0);
 295         }
 296 }
 297
 298 /************************* Sequencer Execution Control ************************/
 299 /*
 300  * Restart the sequencer program from address zero
 301  */
 302 void
 303 ahd_restart(struct ahd_softc *ahd)
 304 {
 305
 306         ahd_pause(ahd);
 307
 308         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 309
 310         /* No more pending messages */
 311         ahd_clear_msg_state(ahd);
 312         ahd_outb(ahd, SCSISIGO, 0);             /* De-assert BSY */
 313         ahd_outb(ahd, MSG_OUT, MSG_NOOP);       /* No message to send */
 314         ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
 315         ahd_outb(ahd, SEQINTCTL, 0);
 316         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
 317         ahd_outb(ahd, SEQ_FLAGS, 0);
 318         ahd_outb(ahd, SAVED_SCSIID, 0xFF);
 319         ahd_outb(ahd, SAVED_LUN, 0xFF);
 320
 321         /*
 322          * Ensure that the sequencer's idea of TQINPOS
 323          * matches our own.  The sequencer increments TQINPOS
 324          * only after it sees a DMA complete and a reset could
 325          * occur before the increment leaving the kernel to believe
 326          * the command arrived but the sequencer to not.
 327          */
 328         ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
 329
 330         /* Always allow reselection */
 331         ahd_outb(ahd, SCSISEQ1,
 332                  ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 333         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 334
 335         /*
 336          * Clear any pending sequencer interrupt.  It is no
 337          * longer relevant since we're resetting the Program
 338          * Counter.
 339          */
 340         ahd_outb(ahd, CLRINT, CLRSEQINT);
 341
 342         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
 343         ahd_unpause(ahd);
 344 }
 345
 346 void
 347 ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
 348 {
 349         ahd_mode_state   saved_modes;
 350
 351 #ifdef AHD_DEBUG
 352         if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
 353                 printf("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
 354 #endif
 355         saved_modes = ahd_save_modes(ahd);
 356         ahd_set_modes(ahd, fifo, fifo);
 357         ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
 358         if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 359                 ahd_outb(ahd, CCSGCTL, CCSGRESET);
 360         ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 361         ahd_outb(ahd, SG_STATE, 0);
 362         ahd_restore_modes(ahd, saved_modes);
 363 }
 364
 365 /************************* Input/Output Queues ********************************/
 366 /*
 367  * Flush and completed commands that are sitting in the command
 368  * complete queues down on the chip but have yet to be dma'ed back up.
 369  */
 370 void
 371 ahd_flush_qoutfifo(struct ahd_softc *ahd)
 372 {
 373         struct          scb *scb;
 374         ahd_mode_state  saved_modes;
 375         u_int           saved_scbptr;
 376         u_int           ccscbctl;
 377         u_int           scbid;
 378         u_int           next_scbid;
 379
 380         saved_modes = ahd_save_modes(ahd);
 381
 382         /*
 383          * Flush the good status FIFO for completed packetized commands.
 384          */
 385         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 386         saved_scbptr = ahd_get_scbptr(ahd);
 387         while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
 388                 u_int fifo_mode;
 389                 u_int i;
 390
 391                 scbid = ahd_inw(ahd, GSFIFO);
 392                 scb = ahd_lookup_scb(ahd, scbid);
 393                 if (scb == NULL) {
 394                         printf("%s: Warning - GSFIFO SCB %d invalid\n",
 395                                ahd_name(ahd), scbid);
 396                         continue;
 397                 }
 398                 /*
 399                  * Determine if this transaction is still active in
 400                  * any FIFO.  If it is, we must flush that FIFO to
 401                  * the host before completing the  command.
 402                  */
 403                 fifo_mode = 0;
 404 rescan_fifos:
 405                 for (i = 0; i < 2; i++) {
 406                         /* Toggle to the other mode. */
 407                         fifo_mode ^= 1;
 408                         ahd_set_modes(ahd, fifo_mode, fifo_mode);
 409
 410                         if (ahd_scb_active_in_fifo(ahd, scb) == 0)
 411                                 continue;
 412
 413                         ahd_run_data_fifo(ahd, scb);
 414
 415                         /*
 416                          * Running this FIFO may cause a CFG4DATA for
 417                          * this same transaction to assert in the other
 418                          * FIFO or a new snapshot SAVEPTRS interrupt
 419                          * in this FIFO.  Even running a FIFO may not
 420                          * clear the transaction if we are still waiting
 421                          * for data to drain to the host. We must loop
 422                          * until the transaction is not active in either
 423                          * FIFO just to be sure.  Reset our loop counter
 424                          * so we will visit both FIFOs again before
 425                          * declaring this transaction finished.  We
 426                          * also delay a bit so that status has a chance
 427                          * to change before we look at this FIFO again.
 428                          */
 429                         ahd_delay(200);
 430                         goto rescan_fifos;
 431                 }
 432                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 433                 ahd_set_scbptr(ahd, scbid);
 434                 if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
 435                  && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
 436                   || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
 437                       & SG_LIST_NULL) != 0)) {
 438                         u_int comp_head;
 439
 440                         /*
 441                          * The transfer completed with a residual.
 442                          * Place this SCB on the complete DMA list
 443                          * so that we update our in-core copy of the
 444                          * SCB before completing the command.
 445                          */
 446                         ahd_outb(ahd, SCB_SCSI_STATUS, 0);
 447                         ahd_outb(ahd, SCB_SGPTR,
 448                                  ahd_inb_scbram(ahd, SCB_SGPTR)
 449                                  | SG_STATUS_VALID);
 450                         ahd_outw(ahd, SCB_TAG, scbid);
 451                         ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
 452                         comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 453                         if (SCBID_IS_NULL(comp_head)) {
 454                                 ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
 455                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 456                         } else {
 457                                 u_int tail;
 458
 459                                 tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
 460                                 ahd_set_scbptr(ahd, tail);
 461                                 ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
 462                                 ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 463                                 ahd_set_scbptr(ahd, scbid);
 464                         }
 465                 } else
 466                         ahd_complete_scb(ahd, scb);
 467         }
 468         ahd_set_scbptr(ahd, saved_scbptr);
 469
 470         /*
 471          * Setup for command channel portion of flush.
 472          */
 473         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 474
 475         /*
 476          * Wait for any inprogress DMA to complete and clear DMA state
 477          * if this if for an SCB in the qinfifo.
 478          */
 479         while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
 480
 481                 if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
 482                         if ((ccscbctl & ARRDONE) != 0)
 483                                 break;
 484                 } else if ((ccscbctl & CCSCBDONE) != 0)
 485                         break;
 486                 ahd_delay(200);
 487         }
 488         /*
 489          * We leave the sequencer to cleanup in the case of DMA's to
 490          * update the qoutfifo.  In all other cases (DMA's to the
 491          * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
 492          * we disable the DMA engine so that the sequencer will not
 493          * attempt to handle the DMA completion.
 494          */
 495         if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
 496                 ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
 497
 498         /*
 499          * Complete any SCBs that just finished
 500          * being DMA'ed into the qoutfifo.
 501          */
 502         ahd_run_qoutfifo(ahd);
 503
 504         saved_scbptr = ahd_get_scbptr(ahd);
 505         /*
 506          * Manually update/complete any completed SCBs that are waiting to be
 507          * DMA'ed back up to the host.
 508          */
 509         scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 510         while (!SCBID_IS_NULL(scbid)) {
 511                 uint8_t *hscb_ptr;
 512                 u_int    i;
 513
 514                 ahd_set_scbptr(ahd, scbid);
 515                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 516                 scb = ahd_lookup_scb(ahd, scbid);
 517                 if (scb == NULL) {
 518                         printf("%s: Warning - DMA-up and complete "
 519                                "SCB %d invalid\n", ahd_name(ahd), scbid);
 520                         continue;
 521                 }
 522                 hscb_ptr = (uint8_t *)scb->hscb;
 523                 for (i = 0; i < sizeof(struct hardware_scb); i++)
 524                         *hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
 525
 526                 ahd_complete_scb(ahd, scb);
 527                 scbid = next_scbid;
 528         }
 529         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
 530         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
 531
 532         scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
 533         while (!SCBID_IS_NULL(scbid)) {
 534
 535                 ahd_set_scbptr(ahd, scbid);
 536                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 537                 scb = ahd_lookup_scb(ahd, scbid);
 538                 if (scb == NULL) {
 539                         printf("%s: Warning - Complete Qfrz SCB %d invalid\n",
 540                                ahd_name(ahd), scbid);
 541                         continue;
 542                 }
 543
 544                 ahd_complete_scb(ahd, scb);
 545                 scbid = next_scbid;
 546         }
 547         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
 548
 549         scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
 550         while (!SCBID_IS_NULL(scbid)) {
 551
 552                 ahd_set_scbptr(ahd, scbid);
 553                 next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 554                 scb = ahd_lookup_scb(ahd, scbid);
 555                 if (scb == NULL) {
 556                         printf("%s: Warning - Complete SCB %d invalid\n",
 557                                ahd_name(ahd), scbid);
 558                         continue;
 559                 }
 560
 561                 ahd_complete_scb(ahd, scb);
 562                 scbid = next_scbid;
 563         }
 564         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
 565
 566         /*
 567          * Restore state.
 568          */
 569         ahd_set_scbptr(ahd, saved_scbptr);
 570         ahd_restore_modes(ahd, saved_modes);
 571         ahd->flags |= AHD_UPDATE_PEND_CMDS;
 572 }
 573
 574 /*
 575  * Determine if an SCB for a packetized transaction
 576  * is active in a FIFO.
 577  */
 578 static int
 579 ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
 580 {
 581
 582         /*
 583          * The FIFO is only active for our transaction if
 584          * the SCBPTR matches the SCB's ID and the firmware
 585          * has installed a handler for the FIFO or we have
 586          * a pending SAVEPTRS or CFG4DATA interrupt.
 587          */
 588         if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
 589          || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
 590           && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
 591                 return (0);
 592
 593         return (1);
 594 }
 595
 596 /*
 597  * Run a data fifo to completion for a transaction we know
 598  * has completed across the SCSI bus (good status has been
 599  * received).  We are already set to the correct FIFO mode
 600  * on entry to this routine.
 601  *
 602  * This function attempts to operate exactly as the firmware
 603  * would when running this FIFO.  Care must be taken to update
 604  * this routine any time the firmware's FIFO algorithm is
 605  * changed.
 606  */
 607 static void
 608 ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
 609 {
 610         u_int seqintsrc;
 611
 612         seqintsrc = ahd_inb(ahd, SEQINTSRC);
 613         if ((seqintsrc & CFG4DATA) != 0) {
 614                 uint32_t datacnt;
 615                 uint32_t sgptr;
 616
 617                 /*
 618                  * Clear full residual flag.
 619                  */
 620                 sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
 621                 ahd_outb(ahd, SCB_SGPTR, sgptr);
 622
 623                 /*
 624                  * Load datacnt and address.
 625                  */
 626                 datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
 627                 if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
 628                         sgptr |= LAST_SEG;
 629                         ahd_outb(ahd, SG_STATE, 0);
 630                 } else
 631                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 632                 ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
 633                 ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
 634                 ahd_outb(ahd, SG_CACHE_PRE, sgptr);
 635                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
 636
 637                 /*
 638                  * Initialize Residual Fields.
 639                  */
 640                 ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
 641                 ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
 642
 643                 /*
 644                  * Mark the SCB as having a FIFO in use.
 645                  */
 646                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 647                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
 648
 649                 /*
 650                  * Install a "fake" handler for this FIFO.
 651                  */
 652                 ahd_outw(ahd, LONGJMP_ADDR, 0);
 653
 654                 /*
 655                  * Notify the hardware that we have satisfied
 656                  * this sequencer interrupt.
 657                  */
 658                 ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
 659         } else if ((seqintsrc & SAVEPTRS) != 0) {
 660                 uint32_t sgptr;
 661                 uint32_t resid;
 662
 663                 if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
 664                         /*
 665                          * Snapshot Save Pointers.  All that
 666                          * is necessary to clear the snapshot
 667                          * is a CLRCHN.
 668                          */
 669                         goto clrchn;
 670                 }
 671
 672                 /*
 673                  * Disable S/G fetch so the DMA engine
 674                  * is available to future users.
 675                  */
 676                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 677                         ahd_outb(ahd, CCSGCTL, 0);
 678                 ahd_outb(ahd, SG_STATE, 0);
 679
 680                 /*
 681                  * Flush the data FIFO.  Strickly only
 682                  * necessary for Rev A parts.
 683                  */
 684                 ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
 685
 686                 /*
 687                  * Calculate residual.
 688                  */
 689                 sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 690                 resid = ahd_inl(ahd, SHCNT);
 691                 resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
 692                 ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
 693                 if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
 694                         /*
 695                          * Must back up to the correct S/G element.
 696                          * Typically this just means resetting our
 697                          * low byte to the offset in the SG_CACHE,
 698                          * but if we wrapped, we have to correct
 699                          * the other bytes of the sgptr too.
 700                          */
 701                         if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
 702                          && (sgptr & 0x80) == 0)
 703                                 sgptr -= 0x100;
 704                         sgptr &= ~0xFF;
 705                         sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
 706                                & SG_ADDR_MASK;
 707                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 708                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
 709                 } else if ((resid & AHD_SG_LEN_MASK) == 0) {
 710                         ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
 711                                  sgptr | SG_LIST_NULL);
 712                 }
 713                 /*
 714                  * Save Pointers.
 715                  */
 716                 ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
 717                 ahd_outl(ahd, SCB_DATACNT, resid);
 718                 ahd_outl(ahd, SCB_SGPTR, sgptr);
 719                 ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
 720                 ahd_outb(ahd, SEQIMODE,
 721                          ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
 722                 /*
 723                  * If the data is to the SCSI bus, we are
 724                  * done, otherwise wait for FIFOEMP.
 725                  */
 726                 if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
 727                         goto clrchn;
 728         } else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
 729                 uint32_t sgptr;
 730                 uint64_t data_addr;
 731                 uint32_t data_len;
 732                 u_int    dfcntrl;
 733
 734                 /*
 735                  * Disable S/G fetch so the DMA engine
 736                  * is available to future users.  We won't
 737                  * be using the DMA engine to load segments.
 738                  */
 739                 if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
 740                         ahd_outb(ahd, CCSGCTL, 0);
 741                         ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 742                 }
 743
 744                 /*
 745                  * Wait for the DMA engine to notice that the
 746                  * host transfer is enabled and that there is
 747                  * space in the S/G FIFO for new segments before
 748                  * loading more segments.
 749                  */
 750                 if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
 751                  && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
 752
 753                         /*
 754                          * Determine the offset of the next S/G
 755                          * element to load.
 756                          */
 757                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 758                         sgptr &= SG_PTR_MASK;
 759                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 760                                 struct ahd_dma64_seg *sg;
 761
 762                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 763                                 data_addr = sg->addr;
 764                                 data_len = sg->len;
 765                                 sgptr += sizeof(*sg);
 766                         } else {
 767                                 struct  ahd_dma_seg *sg;
 768
 769                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 770                                 data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
 771                                 data_addr <<= 8;
 772                                 data_addr |= sg->addr;
 773                                 data_len = sg->len;
 774                                 sgptr += sizeof(*sg);
 775                         }
 776
 777                         /*
 778                          * Update residual information.
 779                          */
 780                         ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
 781                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 782
 783                         /*
 784                          * Load the S/G.
 785                          */
 786                         if (data_len & AHD_DMA_LAST_SEG) {
 787                                 sgptr |= LAST_SEG;
 788                                 ahd_outb(ahd, SG_STATE, 0);
 789                         }
 790                         ahd_outq(ahd, HADDR, data_addr);
 791                         ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
 792                         ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
 793
 794                         /*
 795                          * Advertise the segment to the hardware.
 796                          */
 797                         dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
 798                         if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
 799                                 /*
 800                                  * Use SCSIENWRDIS so that SCSIEN
 801                                  * is never modified by this
 802                                  * operation.
 803                                  */
 804                                 dfcntrl |= SCSIENWRDIS;
 805                         }
 806                         ahd_outb(ahd, DFCNTRL, dfcntrl);
 807                 }
 808         } else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
 809
 810                 /*
 811                  * Transfer completed to the end of SG list
 812                  * and has flushed to the host.
 813                  */
 814                 ahd_outb(ahd, SCB_SGPTR,
 815                          ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
 816                 goto clrchn;
 817         } else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
 818 clrchn:
 819                 /*
 820                  * Clear any handler for this FIFO, decrement
 821                  * the FIFO use count for the SCB, and release
 822                  * the FIFO.
 823                  */
 824                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 825                 ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 826                          ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
 827                 ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
 828         }
 829 }
 830
 831 /*
 832  * Look for entries in the QoutFIFO that have completed.
 833  * The valid_tag completion field indicates the validity
 834  * of the entry - the valid value toggles each time through
 835  * the queue. We use the sg_status field in the completion
 836  * entry to avoid referencing the hscb if the completion
 837  * occurred with no errors and no residual.  sg_status is
 838  * a copy of the first byte (little endian) of the sgptr
 839  * hscb field.
 840  */
 841 void
 842 ahd_run_qoutfifo(struct ahd_softc *ahd)
 843 {
 844         struct ahd_completion *completion;
 845         struct scb *scb;
 846         u_int  scb_index;
 847
 848         if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
 849                 panic("ahd_run_qoutfifo recursion");
 850         ahd->flags |= AHD_RUNNING_QOUTFIFO;
 851         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
 852         for (;;) {
 853                 completion = &ahd->qoutfifo[ahd->qoutfifonext];
 854
 855                 if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
 856                         break;
 857
 858                 scb_index = ahd_le16toh(completion->tag);
 859                 scb = ahd_lookup_scb(ahd, scb_index);
 860                 if (scb == NULL) {
 861                         printf("%s: WARNING no command for scb %d "
 862                                "(cmdcmplt)\nQOUTPOS = %d\n",
 863                                ahd_name(ahd), scb_index,
 864                                ahd->qoutfifonext);
 865                         ahd_dump_card_state(ahd);
 866                 } else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
 867                         ahd_handle_scb_status(ahd, scb);
 868                 } else {
 869                         ahd_done(ahd, scb);
 870                 }
 871
 872                 ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
 873                 if (ahd->qoutfifonext == 0)
 874                         ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
 875         }
 876         ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
 877 }
 878
 879 /************************* Interrupt Handling *********************************/
 880 void
 881 ahd_handle_hwerrint(struct ahd_softc *ahd)
 882 {
 883         /*
 884          * Some catastrophic hardware error has occurred.
 885          * Print it for the user and disable the controller.
 886          */
 887         int i;
 888         int error;
 889
 890         error = ahd_inb(ahd, ERROR);
 891         for (i = 0; i < num_errors; i++) {
 892                 if ((error & ahd_hard_errors[i].errno) != 0)
 893                         printf("%s: hwerrint, %s\n",
 894                                ahd_name(ahd), ahd_hard_errors[i].errmesg);
 895         }
 896
 897         ahd_dump_card_state(ahd);
 898         panic("BRKADRINT");
 899
 900         /* Tell everyone that this HBA is no longer available */
 901         ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 902                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 903                        CAM_NO_HBA);
 904
 905         /* Tell the system that this controller has gone away. */
 906         ahd_free(ahd);
 907 }
 908
 909 void
 910 ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
 911 {
 912         u_int seqintcode;
 913
 914         /*
 915          * Save the sequencer interrupt code and clear the SEQINT
 916          * bit. We will unpause the sequencer, if appropriate,
 917          * after servicing the request.
 918          */
 919         seqintcode = ahd_inb(ahd, SEQINTCODE);
 920         ahd_outb(ahd, CLRINT, CLRSEQINT);
 921         if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
 922                 /*
 923                  * Unpause the sequencer and let it clear
 924                  * SEQINT by writing NO_SEQINT to it.  This
 925                  * will cause the sequencer to be paused again,
 926                  * which is the expected state of this routine.
 927                  */
 928                 ahd_unpause(ahd);
 929                 while (!ahd_is_paused(ahd))
 930                         ;
 931                 ahd_outb(ahd, CLRINT, CLRSEQINT);
 932         }
 933         ahd_update_modes(ahd);
 934 #ifdef AHD_DEBUG
 935         if ((ahd_debug & AHD_SHOW_MISC) != 0)
 936                 printf("%s: Handle Seqint Called for code %d\n",
 937                        ahd_name(ahd), seqintcode);
 938 #endif
 939         switch (seqintcode) {
 940         case ENTERING_NONPACK:
 941         {
 942                 struct  scb *scb;
 943                 u_int   scbid;
 944
 945                 AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 946                                  ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 947                 scbid = ahd_get_scbptr(ahd);
 948                 scb = ahd_lookup_scb(ahd, scbid);
 949                 if (scb == NULL) {
 950                         /*
 951                          * Somehow need to know if this
 952                          * is from a selection or reselection.
 953                          * From that, we can determine target
 954                          * ID so we at least have an I_T nexus.
 955                          */
 956                 } else {
 957                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
 958                         ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
 959                         ahd_outb(ahd, SEQ_FLAGS, 0x0);
 960                 }
 961                 if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
 962                  && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
 963                         /*
 964                          * Phase change after read stream with
 965                          * CRC error with P0 asserted on last
 966                          * packet.
 967                          */
 968 #ifdef AHD_DEBUG
 969                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 970                                 printf("%s: Assuming LQIPHASE_NLQ with "
 971                                        "P0 assertion\n", ahd_name(ahd));
 972 #endif
 973                 }
 974 #ifdef AHD_DEBUG
 975                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 976                         printf("%s: Entering NONPACK\n", ahd_name(ahd));
 977 #endif
 978                 break;
 979         }
 980         case INVALID_SEQINT:
 981                 printf("%s: Invalid Sequencer interrupt occurred.\n",
 982                        ahd_name(ahd));
 983                 ahd_dump_card_state(ahd);
 984                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 985                 break;
 986         case STATUS_OVERRUN:
 987         {
 988                 struct  scb *scb;
 989                 u_int   scbid;
 990
 991                 scbid = ahd_get_scbptr(ahd);
 992                 scb = ahd_lookup_scb(ahd, scbid);
 993                 if (scb != NULL)
 994                         ahd_print_path(ahd, scb);
 995                 else
 996                         printf("%s: ", ahd_name(ahd));
 997                 printf("SCB %d Packetized Status Overrun", scbid);
 998                 ahd_dump_card_state(ahd);
 999                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1000                 break;
1001         }
1002         case CFG4ISTAT_INTR:
1003         {
1004                 struct  scb *scb;
1005                 u_int   scbid;
1006
1007                 scbid = ahd_get_scbptr(ahd);
1008                 scb = ahd_lookup_scb(ahd, scbid);
1009                 if (scb == NULL) {
1010                         ahd_dump_card_state(ahd);
1011                         printf("CFG4ISTAT: Free SCB %d referenced", scbid);
1012                         panic("For safety");
1013                 }
1014                 ahd_outq(ahd, HADDR, scb->sense_busaddr);
1015                 ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
1016                 ahd_outb(ahd, HCNT + 2, 0);
1017                 ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
1018                 ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1019                 break;
1020         }
1021         case ILLEGAL_PHASE:
1022         {
1023                 u_int bus_phase;
1024
1025                 bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1026                 printf("%s: ILLEGAL_PHASE 0x%x\n",
1027                        ahd_name(ahd), bus_phase);
1028
1029                 switch (bus_phase) {
1030                 case P_DATAOUT:
1031                 case P_DATAIN:
1032                 case P_DATAOUT_DT:
1033                 case P_DATAIN_DT:
1034                 case P_MESGOUT:
1035                 case P_STATUS:
1036                 case P_MESGIN:
1037                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1038                         printf("%s: Issued Bus Reset.\n", ahd_name(ahd));
1039                         break;
1040                 case P_COMMAND:
1041                 {
1042                         struct  ahd_devinfo devinfo;
1043                         struct  scb *scb;
1044                         struct  ahd_initiator_tinfo *targ_info;
1045                         struct  ahd_tmode_tstate *tstate;
1046                         struct  ahd_transinfo *tinfo;
1047                         u_int   scbid;
1048
1049                         /*
1050                          * If a target takes us into the command phase
1051                          * assume that it has been externally reset and
1052                          * has thus lost our previous packetized negotiation
1053                          * agreement.  Since we have not sent an identify
1054                          * message and may not have fully qualified the
1055                          * connection, we change our command to TUR, assert
1056                          * ATN and ABORT the task when we go to message in
1057                          * phase.  The OSM will see the REQUEUE_REQUEST
1058                          * status and retry the command.
1059                          */
1060                         scbid = ahd_get_scbptr(ahd);
1061                         scb = ahd_lookup_scb(ahd, scbid);
1062                         if (scb == NULL) {
1063                                 printf("Invalid phase with no valid SCB.  "
1064                                        "Resetting bus.\n");
1065                                 ahd_reset_channel(ahd, 'A',
1066                                                   /*Initiate Reset*/TRUE);
1067                                 break;
1068                         }
1069                         ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
1070                                             SCB_GET_TARGET(ahd, scb),
1071                                             SCB_GET_LUN(scb),
1072                                             SCB_GET_CHANNEL(ahd, scb),
1073                                             ROLE_INITIATOR);
1074                         targ_info = ahd_fetch_transinfo(ahd,
1075                                                         devinfo.channel,
1076                                                         devinfo.our_scsiid,
1077                                                         devinfo.target,
1078                                                         &tstate);
1079                         tinfo = &targ_info->curr;
1080                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
1081                                       AHD_TRANS_ACTIVE, /*paused*/TRUE);
1082                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
1083                                          /*offset*/0, /*ppr_options*/0,
1084                                          AHD_TRANS_ACTIVE, /*paused*/TRUE);
1085                         ahd_outb(ahd, SCB_CDB_STORE, 0);
1086                         ahd_outb(ahd, SCB_CDB_STORE+1, 0);
1087                         ahd_outb(ahd, SCB_CDB_STORE+2, 0);
1088                         ahd_outb(ahd, SCB_CDB_STORE+3, 0);
1089                         ahd_outb(ahd, SCB_CDB_STORE+4, 0);
1090                         ahd_outb(ahd, SCB_CDB_STORE+5, 0);
1091                         ahd_outb(ahd, SCB_CDB_LEN, 6);
1092                         scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
1093                         scb->hscb->control |= MK_MESSAGE;
1094                         ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
1095                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1096                         ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1097                         /*
1098                          * The lun is 0, regardless of the SCB's lun
1099                          * as we have not sent an identify message.
1100                          */
1101                         ahd_outb(ahd, SAVED_LUN, 0);
1102                         ahd_outb(ahd, SEQ_FLAGS, 0);
1103                         ahd_assert_atn(ahd);
1104                         scb->flags &= ~SCB_PACKETIZED;
1105                         scb->flags |= SCB_ABORT|SCB_CMDPHASE_ABORT;
1106                         ahd_freeze_devq(ahd, scb);
1107                         ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
1108                         ahd_freeze_scb(scb);
1109
1110                         /*
1111                          * Allow the sequencer to continue with
1112                          * non-pack processing.
1113                          */
1114                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1115                         ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
1116                         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1117                                 ahd_outb(ahd, CLRLQOINT1, 0);
1118                         }
1119 #ifdef AHD_DEBUG
1120                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1121                                 ahd_print_path(ahd, scb);
1122                                 printf("Unexpected command phase from "
1123                                        "packetized target\n");
1124                         }
1125 #endif
1126                         break;
1127                 }
1128                 }
1129                 break;
1130         }
1131         case CFG4OVERRUN:
1132         {
1133                 struct  scb *scb;
1134                 u_int   scb_index;
1135
1136 #ifdef AHD_DEBUG
1137                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1138                         printf("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
1139                                ahd_inb(ahd, MODE_PTR));
1140                 }
1141 #endif
1142                 scb_index = ahd_get_scbptr(ahd);
1143                 scb = ahd_lookup_scb(ahd, scb_index);
1144                 if (scb == NULL) {
1145                         /*
1146                          * Attempt to transfer to an SCB that is
1147                          * not outstanding.
1148                          */
1149                         ahd_assert_atn(ahd);
1150                         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1151                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
1152                         ahd->msgout_len = 1;
1153                         ahd->msgout_index = 0;
1154                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1155                         /*
1156                          * Clear status received flag to prevent any
1157                          * attempt to complete this bogus SCB.
1158                          */
1159                         ahd_outb(ahd, SCB_CONTROL,
1160                                  ahd_inb_scbram(ahd, SCB_CONTROL)
1161                                  & ~STATUS_RCVD);
1162                 }
1163                 break;
1164         }
1165         case DUMP_CARD_STATE:
1166         {
1167                 ahd_dump_card_state(ahd);
1168                 break;
1169         }
1170         case PDATA_REINIT:
1171         {
1172 #ifdef AHD_DEBUG
1173                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1174                         printf("%s: PDATA_REINIT - DFCNTRL = 0x%x "
1175                                "SG_CACHE_SHADOW = 0x%x\n",
1176                                ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
1177                                ahd_inb(ahd, SG_CACHE_SHADOW));
1178                 }
1179 #endif
1180                 ahd_reinitialize_dataptrs(ahd);
1181                 break;
1182         }
1183         case HOST_MSG_LOOP:
1184         {
1185                 struct ahd_devinfo devinfo;
1186
1187                 /*
1188                  * The sequencer has encountered a message phase
1189                  * that requires host assistance for completion.
1190                  * While handling the message phase(s), we will be
1191                  * notified by the sequencer after each byte is
1192                  * transfered so we can track bus phase changes.
1193                  *
1194                  * If this is the first time we've seen a HOST_MSG_LOOP
1195                  * interrupt, initialize the state of the host message
1196                  * loop.
1197                  */
1198                 ahd_fetch_devinfo(ahd, &devinfo);
1199                 if (ahd->msg_type == MSG_TYPE_NONE) {
1200                         struct scb *scb;
1201                         u_int scb_index;
1202                         u_int bus_phase;
1203
1204                         bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1205                         if (bus_phase != P_MESGIN
1206                          && bus_phase != P_MESGOUT) {
1207                                 printf("ahd_intr: HOST_MSG_LOOP bad "
1208                                        "phase 0x%x\n", bus_phase);
1209                                 /*
1210                                  * Probably transitioned to bus free before
1211                                  * we got here.  Just punt the message.
1212                                  */
1213                                 ahd_dump_card_state(ahd);
1214                                 ahd_clear_intstat(ahd);
1215                                 ahd_restart(ahd);
1216                                 return;
1217                         }
1218
1219                         scb_index = ahd_get_scbptr(ahd);
1220                         scb = ahd_lookup_scb(ahd, scb_index);
1221                         if (devinfo.role == ROLE_INITIATOR) {
1222                                 if (bus_phase == P_MESGOUT)
1223                                         ahd_setup_initiator_msgout(ahd,
1224                                                                    &devinfo,
1225                                                                    scb);
1226                                 else {
1227                                         ahd->msg_type =
1228                                             MSG_TYPE_INITIATOR_MSGIN;
1229                                         ahd->msgin_index = 0;
1230                                 }
1231                         }
1232 #ifdef AHD_TARGET_MODE
1233                         else {
1234                                 if (bus_phase == P_MESGOUT) {
1235                                         ahd->msg_type =
1236                                             MSG_TYPE_TARGET_MSGOUT;
1237                                         ahd->msgin_index = 0;
1238                                 }
1239                                 else
1240                                         ahd_setup_target_msgin(ahd,
1241                                                                &devinfo,
1242                                                                scb);
1243                         }
1244 #endif
1245                 }
1246
1247                 ahd_handle_message_phase(ahd);
1248                 break;
1249         }
1250         case NO_MATCH:
1251         {
1252                 /* Ensure we don't leave the selection hardware on */
1253                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
1254                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
1255
1256                 printf("%s:%c:%d: no active SCB for reconnecting "
1257                        "target - issuing BUS DEVICE RESET\n",
1258                        ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
1259                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1260                        "REG0 == 0x%x ACCUM = 0x%x\n",
1261                        ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
1262                        ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
1263                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1264                        "SINDEX == 0x%x\n",
1265                        ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
1266                        ahd_find_busy_tcl(ahd,
1267                                          BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
1268                                                    ahd_inb(ahd, SAVED_LUN))),
1269                        ahd_inw(ahd, SINDEX));
1270                 printf("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1271                        "SCB_CONTROL == 0x%x\n",
1272                        ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
1273                        ahd_inb_scbram(ahd, SCB_LUN),
1274                        ahd_inb_scbram(ahd, SCB_CONTROL));
1275                 printf("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
1276                        ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
1277                 printf("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
1278                 printf("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
1279                 ahd_dump_card_state(ahd);
1280                 ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
1281                 ahd->msgout_len = 1;
1282                 ahd->msgout_index = 0;
1283                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1284                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
1285                 ahd_assert_atn(ahd);
1286                 break;
1287         }
1288         case PROTO_VIOLATION:
1289         {
1290                 ahd_handle_proto_violation(ahd);
1291                 break;
1292         }
1293         case IGN_WIDE_RES:
1294         {
1295                 struct ahd_devinfo devinfo;
1296
1297                 ahd_fetch_devinfo(ahd, &devinfo);
1298                 ahd_handle_ign_wide_residue(ahd, &devinfo);
1299                 break;
1300         }
1301         case BAD_PHASE:
1302         {
1303                 u_int lastphase;
1304
1305                 lastphase = ahd_inb(ahd, LASTPHASE);
1306                 printf("%s:%c:%d: unknown scsi bus phase %x, "
1307                        "lastphase = 0x%x.  Attempting to continue\n",
1308                        ahd_name(ahd), 'A',
1309                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
1310                        lastphase, ahd_inb(ahd, SCSISIGI));
1311                 break;
1312         }
1313         case MISSED_BUSFREE:
1314         {
1315                 u_int lastphase;
1316
1317                 lastphase = ahd_inb(ahd, LASTPHASE);
1318                 printf("%s:%c:%d: Missed busfree. "
1319                        "Lastphase = 0x%x, Curphase = 0x%x\n",
1320                        ahd_name(ahd), 'A',
1321                        SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
1322                        lastphase, ahd_inb(ahd, SCSISIGI));
1323                 ahd_restart(ahd);
1324                 return;
1325         }
1326         case DATA_OVERRUN:
1327         {
1328                 /*
1329                  * When the sequencer detects an overrun, it
1330                  * places the controller in "BITBUCKET" mode
1331                  * and allows the target to complete its transfer.
1332                  * Unfortunately, none of the counters get updated
1333                  * when the controller is in this mode, so we have
1334                  * no way of knowing how large the overrun was.
1335                  */
1336                 struct  scb *scb;
1337                 u_int   scbindex;
1338 #ifdef AHD_DEBUG
1339                 u_int   lastphase;
1340 #endif
1341
1342                 scbindex = ahd_get_scbptr(ahd);
1343                 scb = ahd_lookup_scb(ahd, scbindex);
1344 #ifdef AHD_DEBUG
1345                 lastphase = ahd_inb(ahd, LASTPHASE);
1346                 if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1347                         ahd_print_path(ahd, scb);
1348                         printf("data overrun detected %s.  Tag == 0x%x.\n",
1349                                ahd_lookup_phase_entry(lastphase)->phasemsg,
1350                                SCB_GET_TAG(scb));
1351                         ahd_print_path(ahd, scb);
1352                         printf("%s seen Data Phase.  Length = %ld.  "
1353                                "NumSGs = %d.\n",
1354                                ahd_inb(ahd, SEQ_FLAGS) & DPHASE
1355                                ? "Have" : "Haven't",
1356                                ahd_get_transfer_length(scb), scb->sg_count);
1357                         ahd_dump_sglist(scb);
1358                 }
1359 #endif
1360
1361                 /*
1362                  * Set this and it will take effect when the
1363                  * target does a command complete.
1364                  */
1365                 ahd_freeze_devq(ahd, scb);
1366                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1367                 ahd_freeze_scb(scb);
1368                 break;
1369         }
1370         case MKMSG_FAILED:
1371         {
1372                 struct ahd_devinfo devinfo;
1373                 struct scb *scb;
1374                 u_int scbid;
1375
1376                 ahd_fetch_devinfo(ahd, &devinfo);
1377                 printf("%s:%c:%d:%d: Attempt to issue message failed\n",
1378                        ahd_name(ahd), devinfo.channel, devinfo.target,
1379                        devinfo.lun);
1380                 scbid = ahd_get_scbptr(ahd);
1381                 scb = ahd_lookup_scb(ahd, scbid);
1382                 if (scb != NULL
1383                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
1384                         /*
1385                          * Ensure that we didn't put a second instance of this
1386                          * SCB into the QINFIFO.
1387                          */
1388                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
1389                                            SCB_GET_CHANNEL(ahd, scb),
1390                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
1391                                            ROLE_INITIATOR, /*status*/0,
1392                                            SEARCH_REMOVE);
1393                 ahd_outb(ahd, SCB_CONTROL,
1394                          ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
1395                 break;
1396         }
1397         case TASKMGMT_FUNC_COMPLETE:
1398         {
1399                 u_int   scbid;
1400                 struct  scb *scb;
1401
1402                 scbid = ahd_get_scbptr(ahd);
1403                 scb = ahd_lookup_scb(ahd, scbid);
1404                 if (scb != NULL) {
1405                         u_int      lun;
1406                         u_int      tag;
1407                         cam_status error;
1408
1409                         ahd_print_path(ahd, scb);
1410                         printf("Task Management Func 0x%x Complete\n",
1411                                scb->hscb->task_management);
1412                         lun = CAM_LUN_WILDCARD;
1413                         tag = SCB_LIST_NULL;
1414
1415                         switch (scb->hscb->task_management) {
1416                         case SIU_TASKMGMT_ABORT_TASK:
1417                                 tag = SCB_GET_TAG(scb);
1418                         case SIU_TASKMGMT_ABORT_TASK_SET:
1419                         case SIU_TASKMGMT_CLEAR_TASK_SET:
1420                                 lun = scb->hscb->lun;
1421                                 error = CAM_REQ_ABORTED;
1422                                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
1423                                                'A', lun, tag, ROLE_INITIATOR,
1424                                                error);
1425                                 break;
1426                         case SIU_TASKMGMT_LUN_RESET:
1427                                 lun = scb->hscb->lun;
1428                         case SIU_TASKMGMT_TARGET_RESET:
1429                         {
1430                                 struct ahd_devinfo devinfo;
1431
1432                                 ahd_scb_devinfo(ahd, &devinfo, scb);
1433                                 error = CAM_BDR_SENT;
1434                                 ahd_handle_devreset(ahd, &devinfo, lun,
1435                                                     CAM_BDR_SENT,
1436                                                     lun != CAM_LUN_WILDCARD
1437                                                     ? "Lun Reset"
1438                                                     : "Target Reset",
1439                                                     /*verbose_level*/0);
1440                                 break;
1441                         }
1442                         default:
1443                                 panic("Unexpected TaskMgmt Func\n");
1444                                 break;
1445                         }
1446                 }
1447                 break;
1448         }
1449         case TASKMGMT_CMD_CMPLT_OKAY:
1450         {
1451                 u_int   scbid;
1452                 struct  scb *scb;
1453
1454                 /*
1455                  * An ABORT TASK TMF failed to be delivered before
1456                  * the targeted command completed normally.
1457                  */
1458                 scbid = ahd_get_scbptr(ahd);
1459                 scb = ahd_lookup_scb(ahd, scbid);
1460                 if (scb != NULL) {
1461                         /*
1462                          * Remove the second instance of this SCB from
1463                          * the QINFIFO if it is still there.
1464                          */
1465                         ahd_print_path(ahd, scb);
1466                         printf("SCB completes before TMF\n");
1467                         /*
1468                          * Handle losing the race.  Wait until any
1469                          * current selection completes.  We will then
1470                          * set the TMF back to zero in this SCB so that
1471                          * the sequencer doesn't bother to issue another
1472                          * sequencer interrupt for its completion.
1473                          */
1474                         while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
1475                             && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
1476                             && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
1477                                 ;
1478                         ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
1479                         ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
1480                                            SCB_GET_CHANNEL(ahd, scb),
1481                                            SCB_GET_LUN(scb), SCB_GET_TAG(scb),
1482                                            ROLE_INITIATOR, /*status*/0,
1483                                            SEARCH_REMOVE);
1484                 }
1485                 break;
1486         }
1487         case TRACEPOINT0:
1488         case TRACEPOINT1:
1489         case TRACEPOINT2:
1490         case TRACEPOINT3:
1491                 printf("%s: Tracepoint %d\n", ahd_name(ahd),
1492                        seqintcode - TRACEPOINT0);
1493                 break;
1494         case NO_SEQINT:
1495                 break;
1496         case SAW_HWERR:
1497                 ahd_handle_hwerrint(ahd);
1498                 break;
1499         default:
1500                 printf("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
1501                        seqintcode);
1502                 break;
1503         }
1504         /*
1505          *  The sequencer is paused immediately on
1506          *  a SEQINT, so we should restart it when
1507          *  we're done.
1508          */
1509         ahd_unpause(ahd);
1510 }
1511
1512 void
1513 ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
1514 {
1515         struct scb      *scb;
1516         u_int            status0;
1517         u_int            status3;
1518         u_int            status;
1519         u_int            lqistat1;
1520         u_int            lqostat0;
1521         u_int            scbid;
1522         u_int            busfreetime;
1523
1524         ahd_update_modes(ahd);
1525         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1526
1527         status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
1528         status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
1529         status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1530         lqistat1 = ahd_inb(ahd, LQISTAT1);
1531         lqostat0 = ahd_inb(ahd, LQOSTAT0);
1532         busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
1533         if ((status0 & (SELDI|SELDO)) != 0) {
1534                 u_int simode0;
1535
1536                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1537                 simode0 = ahd_inb(ahd, SIMODE0);
1538                 status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
1539                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1540         }
1541         scbid = ahd_get_scbptr(ahd);
1542         scb = ahd_lookup_scb(ahd, scbid);
1543         if (scb != NULL
1544          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1545                 scb = NULL;
1546
1547         if ((status0 & IOERR) != 0) {
1548                 u_int now_lvd;
1549
1550                 now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
1551                 printf("%s: Transceiver State Has Changed to %s mode\n",
1552                        ahd_name(ahd), now_lvd ? "LVD" : "SE");
1553                 ahd_outb(ahd, CLRSINT0, CLRIOERR);
1554                 /*
1555                  * A change in I/O mode is equivalent to a bus reset.
1556                  */
1557                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1558                 ahd_pause(ahd);
1559                 ahd_setup_iocell_workaround(ahd);
1560                 ahd_unpause(ahd);
1561         } else if ((status0 & OVERRUN) != 0) {
1562
1563                 printf("%s: SCSI offset overrun detected.  Resetting bus.\n",
1564                        ahd_name(ahd));
1565                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1566         } else if ((status & SCSIRSTI) != 0) {
1567
1568                 printf("%s: Someone reset channel A\n", ahd_name(ahd));
1569                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
1570         } else if ((status & SCSIPERR) != 0) {
1571
1572                 /* Make sure the sequencer is in a safe location. */
1573                 ahd_clear_critical_section(ahd);
1574
1575                 ahd_handle_transmission_error(ahd);
1576         } else if (lqostat0 != 0) {
1577
1578                 printf("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
1579                 ahd_outb(ahd, CLRLQOINT0, lqostat0);
1580                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
1581                         ahd_outb(ahd, CLRLQOINT1, 0);
1582         } else if ((status & SELTO) != 0) {
1583                 u_int  scbid;
1584
1585                 /* Stop the selection */
1586                 ahd_outb(ahd, SCSISEQ0, 0);
1587
1588                 /* Make sure the sequencer is in a safe location. */
1589                 ahd_clear_critical_section(ahd);
1590
1591                 /* No more pending messages */
1592                 ahd_clear_msg_state(ahd);
1593
1594                 /* Clear interrupt state */
1595                 ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1596
1597                 /*
1598                  * Although the driver does not care about the
1599                  * 'Selection in Progress' status bit, the busy
1600                  * LED does.  SELINGO is only cleared by a sucessfull
1601                  * selection, so we must manually clear it to insure
1602                  * the LED turns off just incase no future successful
1603                  * selections occur (e.g. no devices on the bus).
1604                  */
1605                 ahd_outb(ahd, CLRSINT0, CLRSELINGO);
1606
1607                 scbid = ahd_inw(ahd, WAITING_TID_HEAD);
1608                 scb = ahd_lookup_scb(ahd, scbid);
1609                 if (scb == NULL) {
1610                         printf("%s: ahd_intr - referenced scb not "
1611                                "valid during SELTO scb(0x%x)\n",
1612                                ahd_name(ahd), scbid);
1613                         ahd_dump_card_state(ahd);
1614                 } else {
1615                         struct ahd_devinfo devinfo;
1616 #ifdef AHD_DEBUG
1617                         if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
1618                                 ahd_print_path(ahd, scb);
1619                                 printf("Saw Selection Timeout for SCB 0x%x\n",
1620                                        scbid);
1621                         }
1622 #endif
1623                         ahd_scb_devinfo(ahd, &devinfo, scb);
1624                         ahd_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1625                         ahd_freeze_devq(ahd, scb);
1626
1627                         /*
1628                          * Cancel any pending transactions on the device
1629                          * now that it seems to be missing.  This will
1630                          * also revert us to async/narrow transfers until
1631                          * we can renegotiate with the device.
1632                          */
1633                         ahd_handle_devreset(ahd, &devinfo,
1634                                             CAM_LUN_WILDCARD,
1635                                             CAM_SEL_TIMEOUT,
1636                                             "Selection Timeout",
1637                                             /*verbose_level*/1);
1638                 }
1639                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
1640                 ahd_iocell_first_selection(ahd);
1641                 ahd_unpause(ahd);
1642         } else if ((status0 & (SELDI|SELDO)) != 0) {
1643
1644                 ahd_iocell_first_selection(ahd);
1645                 ahd_unpause(ahd);
1646         } else if (status3 != 0) {
1647                 printf("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
1648                        ahd_name(ahd), status3);
1649                 ahd_outb(ahd, CLRSINT3, status3);
1650         } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
1651
1652                 /* Make sure the sequencer is in a safe location. */
1653                 ahd_clear_critical_section(ahd);
1654
1655                 ahd_handle_lqiphase_error(ahd, lqistat1);
1656         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
1657                 /*
1658                  * This status can be delayed during some
1659                  * streaming operations.  The SCSIPHASE
1660                  * handler has already dealt with this case
1661                  * so just clear the error.
1662                  */
1663                 ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
1664         } else if ((status & BUSFREE) != 0
1665                 || (lqistat1 & LQOBUSFREE) != 0) {
1666                 u_int lqostat1;
1667                 int   restart;
1668                 int   clear_fifo;
1669                 int   packetized;
1670                 u_int mode;
1671
1672                 /*
1673                  * Clear our selection hardware as soon as possible.
1674                  * We may have an entry in the waiting Q for this target,
1675                  * that is affected by this busfree and we don't want to
1676                  * go about selecting the target while we handle the event.
1677                  */
1678                 ahd_outb(ahd, SCSISEQ0, 0);
1679
1680                 /* Make sure the sequencer is in a safe location. */
1681                 ahd_clear_critical_section(ahd);
1682
1683                 /*
1684                  * Determine what we were up to at the time of
1685                  * the busfree.
1686                  */
1687                 mode = AHD_MODE_SCSI;
1688                 busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
1689                 lqostat1 = ahd_inb(ahd, LQOSTAT1);
1690                 switch (busfreetime) {
1691                 case BUSFREE_DFF0:
1692                 case BUSFREE_DFF1:
1693                 {
1694                         u_int   scbid;
1695                         struct  scb *scb;
1696
1697                         mode = busfreetime == BUSFREE_DFF0
1698                              ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
1699                         ahd_set_modes(ahd, mode, mode);
1700                         scbid = ahd_get_scbptr(ahd);
1701                         scb = ahd_lookup_scb(ahd, scbid);
1702                         if (scb == NULL) {
1703                                 printf("%s: Invalid SCB %d in DFF%d "
1704                                        "during unexpected busfree\n",
1705                                        ahd_name(ahd), scbid, mode);
1706                                 packetized = 0;
1707                         } else
1708                                 packetized = (scb->flags & SCB_PACKETIZED) != 0;
1709                         clear_fifo = 1;
1710                         break;
1711                 }
1712                 case BUSFREE_LQO:
1713                         clear_fifo = 0;
1714                         packetized = 1;
1715                         break;
1716                 default:
1717                         clear_fifo = 0;
1718                         packetized =  (lqostat1 & LQOBUSFREE) != 0;
1719                         if (!packetized
1720                          && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
1721                          && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
1722                          && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
1723                           || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
1724                                 /*
1725                                  * Assume packetized if we are not
1726                                  * on the bus in a non-packetized
1727                                  * capacity and any pending selection
1728                                  * was a packetized selection.
1729                                  */
1730                                 packetized = 1;
1731                         break;
1732                 }
1733
1734 #ifdef AHD_DEBUG
1735                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
1736                         printf("Saw Busfree.  Busfreetime = 0x%x.\n",
1737                                busfreetime);
1738 #endif
1739                 /*
1740                  * Busfrees that occur in non-packetized phases are
1741                  * handled by the nonpkt_busfree handler.
1742                  */
1743                 if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
1744                         restart = ahd_handle_pkt_busfree(ahd, busfreetime);
1745                 } else {
1746                         packetized = 0;
1747                         restart = ahd_handle_nonpkt_busfree(ahd);
1748                 }
1749                 /*
1750                  * Clear the busfree interrupt status.  The setting of
1751                  * the interrupt is a pulse, so in a perfect world, we
1752                  * would not need to muck with the ENBUSFREE logic.  This
1753                  * would ensure that if the bus moves on to another
1754                  * connection, busfree protection is still in force.  If
1755                  * BUSFREEREV is broken, however, we must manually clear
1756                  * the ENBUSFREE if the busfree occurred during a non-pack
1757                  * connection so that we don't get false positives during
1758                  * future, packetized, connections.
1759                  */
1760                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
1761                 if (packetized == 0
1762                  && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
1763                         ahd_outb(ahd, SIMODE1,
1764                                  ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
1765
1766                 if (clear_fifo)
1767                         ahd_clear_fifo(ahd, mode);
1768
1769                 ahd_clear_msg_state(ahd);
1770                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
1771                 if (restart) {
1772                         ahd_restart(ahd);
1773                 } else {
1774                         ahd_unpause(ahd);
1775                 }
1776         } else {
1777                 printf("%s: Missing case in ahd_handle_scsiint. status = %x\n",
1778                        ahd_name(ahd), status);
1779                 ahd_dump_card_state(ahd);
1780                 ahd_clear_intstat(ahd);
1781                 ahd_unpause(ahd);
1782         }
1783 }
1784
1785 static void
1786 ahd_handle_transmission_error(struct ahd_softc *ahd)
1787 {
1788         struct  scb *scb;
1789         u_int   scbid;
1790         u_int   lqistat1;
1791         u_int   lqistat2;
1792         u_int   msg_out;
1793         u_int   curphase;
1794         u_int   lastphase;
1795         u_int   perrdiag;
1796         u_int   cur_col;
1797         int     silent;
1798
1799         scb = NULL;
1800         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1801         lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
1802         lqistat2 = ahd_inb(ahd, LQISTAT2);
1803         if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
1804          && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
1805                 u_int lqistate;
1806
1807                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1808                 lqistate = ahd_inb(ahd, LQISTATE);
1809                 if ((lqistate >= 0x1E && lqistate <= 0x24)
1810                  || (lqistate == 0x29)) {
1811 #ifdef AHD_DEBUG
1812                         if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1813                                 printf("%s: NLQCRC found via LQISTATE\n",
1814                                        ahd_name(ahd));
1815                         }
1816 #endif
1817                         lqistat1 |= LQICRCI_NLQ;
1818                 }
1819                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1820         }
1821
1822         ahd_outb(ahd, CLRLQIINT1, lqistat1);
1823         lastphase = ahd_inb(ahd, LASTPHASE);
1824         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1825         perrdiag = ahd_inb(ahd, PERRDIAG);
1826         msg_out = MSG_INITIATOR_DET_ERR;
1827         ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
1828
1829         /*
1830          * Try to find the SCB associated with this error.
1831          */
1832         silent = FALSE;
1833         if (lqistat1 == 0
1834          || (lqistat1 & LQICRCI_NLQ) != 0) {
1835                 if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
1836                         ahd_set_active_fifo(ahd);
1837                 scbid = ahd_get_scbptr(ahd);
1838                 scb = ahd_lookup_scb(ahd, scbid);
1839                 if (scb != NULL && SCB_IS_SILENT(scb))
1840                         silent = TRUE;
1841         }
1842
1843         cur_col = 0;
1844         if (silent == FALSE) {
1845                 printf("%s: Transmission error detected\n", ahd_name(ahd));
1846                 ahd_lqistat1_print(lqistat1, &cur_col, 50);
1847                 ahd_lastphase_print(lastphase, &cur_col, 50);
1848                 ahd_scsisigi_print(curphase, &cur_col, 50);
1849                 ahd_perrdiag_print(perrdiag, &cur_col, 50);
1850                 printf("\n");
1851                 ahd_dump_card_state(ahd);
1852         }
1853
1854         if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
1855                 if (silent == FALSE) {
1856                         printf("%s: Gross protocol error during incoming "
1857                                "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
1858                                ahd_name(ahd), lqistat1);
1859                 }
1860                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1861                 return;
1862         } else if ((lqistat1 & LQICRCI_LQ) != 0) {
1863                 /*
1864                  * A CRC error has been detected on an incoming LQ.
1865                  * The bus is currently hung on the last ACK.
1866                  * Hit LQIRETRY to release the last ack, and
1867                  * wait for the sequencer to determine that ATNO
1868                  * is asserted while in message out to take us
1869                  * to our host message loop.  No NONPACKREQ or
1870                  * LQIPHASE type errors will occur in this
1871                  * scenario.  After this first LQIRETRY, the LQI
1872                  * manager will be in ISELO where it will
1873                  * happily sit until another packet phase begins.
1874                  * Unexpected bus free detection is enabled
1875                  * through any phases that occur after we release
1876                  * this last ack until the LQI manager sees a
1877                  * packet phase.  This implies we may have to
1878                  * ignore a perfectly valid "unexected busfree"
1879                  * after our "initiator detected error" message is
1880                  * sent.  A busfree is the expected response after
1881                  * we tell the target that it's L_Q was corrupted.
1882                  * (SPI4R09 10.7.3.3.3)
1883                  */
1884                 ahd_outb(ahd, LQCTL2, LQIRETRY);
1885                 printf("LQIRetry for LQICRCI_LQ to release ACK\n");
1886         } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
1887                 /*
1888                  * We detected a CRC error in a NON-LQ packet.
1889                  * The hardware has varying behavior in this situation
1890                  * depending on whether this packet was part of a
1891                  * stream or not.
1892                  *
1893                  * PKT by PKT mode:
1894                  * The hardware has already acked the complete packet.
1895                  * If the target honors our outstanding ATN condition,
1896                  * we should be (or soon will be) in MSGOUT phase.
1897                  * This will trigger the LQIPHASE_LQ status bit as the
1898                  * hardware was expecting another LQ.  Unexpected
1899                  * busfree detection is enabled.  Once LQIPHASE_LQ is
1900                  * true (first entry into host message loop is much
1901                  * the same), we must clear LQIPHASE_LQ and hit
1902                  * LQIRETRY so the hardware is ready to handle
1903                  * a future LQ.  NONPACKREQ will not be asserted again
1904                  * once we hit LQIRETRY until another packet is
1905                  * processed.  The target may either go busfree
1906                  * or start another packet in response to our message.
1907                  *
1908                  * Read Streaming P0 asserted:
1909                  * If we raise ATN and the target completes the entire
1910                  * stream (P0 asserted during the last packet), the
1911                  * hardware will ack all data and return to the ISTART
1912                  * state.  When the target reponds to our ATN condition,
1913                  * LQIPHASE_LQ will be asserted.  We should respond to
1914                  * this with an LQIRETRY to prepare for any future
1915                  * packets.  NONPACKREQ will not be asserted again
1916                  * once we hit LQIRETRY until another packet is
1917                  * processed.  The target may either go busfree or
1918                  * start another packet in response to our message.
1919                  * Busfree detection is enabled.
1920                  *
1921                  * Read Streaming P0 not asserted:
1922                  * If we raise ATN and the target transitions to
1923                  * MSGOUT in or after a packet where P0 is not
1924                  * asserted, the hardware will assert LQIPHASE_NLQ.
1925                  * We should respond to the LQIPHASE_NLQ with an
1926                  * LQIRETRY.  Should the target stay in a non-pkt
1927                  * phase after we send our message, the hardware
1928                  * will assert LQIPHASE_LQ.  Recovery is then just as
1929                  * listed above for the read streaming with P0 asserted.
1930                  * Busfree detection is enabled.
1931                  */
1932                 if (silent == FALSE)
1933                         printf("LQICRC_NLQ\n");
1934                 if (scb == NULL) {
1935                         printf("%s: No SCB valid for LQICRC_NLQ.  "
1936                                "Resetting bus\n", ahd_name(ahd));
1937                         ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1938                         return;
1939                 }
1940         } else if ((lqistat1 & LQIBADLQI) != 0) {
1941                 printf("Need to handle BADLQI!\n");
1942                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1943                 return;
1944         } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
1945                 if ((curphase & ~P_DATAIN_DT) != 0) {
1946                         /* Ack the byte.  So we can continue. */
1947                         if (silent == FALSE)
1948                                 printf("Acking %s to clear perror\n",
1949                                     ahd_lookup_phase_entry(curphase)->phasemsg);
1950                         ahd_inb(ahd, SCSIDAT);
1951                 }
1952
1953                 if (curphase == P_MESGIN)
1954                         msg_out = MSG_PARITY_ERROR;
1955         }
1956
1957         /*
1958          * We've set the hardware to assert ATN if we
1959          * get a parity error on "in" phases, so all we
1960          * need to do is stuff the message buffer with
1961          * the appropriate message.  "In" phases have set
1962          * mesg_out to something other than MSG_NOP.
1963          */
1964         ahd->send_msg_perror = msg_out;
1965         if (scb != NULL && msg_out == MSG_INITIATOR_DET_ERR)
1966                 scb->flags |= SCB_TRANSMISSION_ERROR;
1967         ahd_outb(ahd, MSG_OUT, HOST_MSG);
1968         ahd_outb(ahd, CLRINT, CLRSCSIINT);
1969         ahd_unpause(ahd);
1970 }
1971
1972 static void
1973 ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
1974 {
1975         /*
1976          * Clear the sources of the interrupts.
1977          */
1978         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1979         ahd_outb(ahd, CLRLQIINT1, lqistat1);
1980
1981         /*
1982          * If the "illegal" phase changes were in response
1983          * to our ATN to flag a CRC error, AND we ended up
1984          * on packet boundaries, clear the error, restart the
1985          * LQI manager as appropriate, and go on our merry
1986          * way toward sending the message.  Otherwise, reset
1987          * the bus to clear the error.
1988          */
1989         ahd_set_active_fifo(ahd);
1990         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
1991          && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
1992                 if ((lqistat1 & LQIPHASE_LQ) != 0) {
1993                         printf("LQIRETRY for LQIPHASE_LQ\n");
1994                         ahd_outb(ahd, LQCTL2, LQIRETRY);
1995                 } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
1996                         printf("LQIRETRY for LQIPHASE_NLQ\n");
1997                         ahd_outb(ahd, LQCTL2, LQIRETRY);
1998                 } else
1999                         panic("ahd_handle_lqiphase_error: No phase errors\n");
2000                 ahd_dump_card_state(ahd);
2001                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2002                 ahd_unpause(ahd);
2003         } else {
2004                 printf("Reseting Channel for LQI Phase error\n");
2005                 ahd_dump_card_state(ahd);
2006                 ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2007         }
2008 }
2009
2010 /*
2011  * Packetized unexpected or expected busfree.
2012  * Entered in mode based on busfreetime.
2013  */
2014 static int
2015 ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
2016 {
2017         u_int lqostat1;
2018
2019         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2020                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2021         lqostat1 = ahd_inb(ahd, LQOSTAT1);
2022         if ((lqostat1 & LQOBUSFREE) != 0) {
2023                 struct scb *scb;
2024                 u_int scbid;
2025                 u_int saved_scbptr;
2026                 u_int waiting_h;
2027                 u_int waiting_t;
2028                 u_int next;
2029
2030                 /*
2031                  * The LQO manager detected an unexpected busfree
2032                  * either:
2033                  *
2034                  * 1) During an outgoing LQ.
2035                  * 2) After an outgoing LQ but before the first
2036                  *    REQ of the command packet.
2037                  * 3) During an outgoing command packet.
2038                  *
2039                  * In all cases, CURRSCB is pointing to the
2040                  * SCB that encountered the failure.  Clean
2041                  * up the queue, clear SELDO and LQOBUSFREE,
2042                  * and allow the sequencer to restart the select
2043                  * out at its lesure.
2044                  */
2045                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2046                 scbid = ahd_inw(ahd, CURRSCB);
2047                 scb = ahd_lookup_scb(ahd, scbid);
2048                 if (scb == NULL)
2049                        panic("SCB not valid during LQOBUSFREE");
2050                 /*
2051                  * Clear the status.
2052                  */
2053                 ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
2054                 if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2055                         ahd_outb(ahd, CLRLQOINT1, 0);
2056                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2057                 ahd_flush_device_writes(ahd);
2058                 ahd_outb(ahd, CLRSINT0, CLRSELDO);
2059
2060                 /*
2061                  * Return the LQO manager to its idle loop.  It will
2062                  * not do this automatically if the busfree occurs
2063                  * after the first REQ of either the LQ or command
2064                  * packet or between the LQ and command packet.
2065                  */
2066                 ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
2067
2068                 /*
2069                  * Update the waiting for selection queue so
2070                  * we restart on the correct SCB.
2071                  */
2072                 waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
2073                 saved_scbptr = ahd_get_scbptr(ahd);
2074                 if (waiting_h != scbid) {
2075
2076                         ahd_outw(ahd, WAITING_TID_HEAD, scbid);
2077                         waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
2078                         if (waiting_t == waiting_h) {
2079                                 ahd_outw(ahd, WAITING_TID_TAIL, scbid);
2080                                 next = SCB_LIST_NULL;
2081                         } else {
2082                                 ahd_set_scbptr(ahd, waiting_h);
2083                                 next = ahd_inw_scbram(ahd, SCB_NEXT2);
2084                         }
2085                         ahd_set_scbptr(ahd, scbid);
2086                         ahd_outw(ahd, SCB_NEXT2, next);
2087                 }
2088                 ahd_set_scbptr(ahd, saved_scbptr);
2089                 if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
2090                         if (SCB_IS_SILENT(scb) == FALSE) {
2091                                 ahd_print_path(ahd, scb);
2092                                 printf("Probable outgoing LQ CRC error.  "
2093                                        "Retrying command\n");
2094                         }
2095                         scb->crc_retry_count++;
2096                 } else {
2097                         ahd_set_transaction_status(scb, CAM_UNCOR_PARITY);
2098                         ahd_freeze_scb(scb);
2099                         ahd_freeze_devq(ahd, scb);
2100                 }
2101                 /* Return unpausing the sequencer. */
2102                 return (0);
2103         } else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
2104                 /*
2105                  * Ignore what are really parity errors that
2106                  * occur on the last REQ of a free running
2107                  * clock prior to going busfree.  Some drives
2108                  * do not properly active negate just before
2109                  * going busfree resulting in a parity glitch.
2110                  */
2111                 ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
2112 #ifdef AHD_DEBUG
2113                 if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
2114                         printf("%s: Parity on last REQ detected "
2115                                "during busfree phase.\n",
2116                                ahd_name(ahd));
2117 #endif
2118                 /* Return unpausing the sequencer. */
2119                 return (0);
2120         }
2121         if (ahd->src_mode != AHD_MODE_SCSI) {
2122                 u_int   scbid;
2123                 struct  scb *scb;
2124
2125                 scbid = ahd_get_scbptr(ahd);
2126                 scb = ahd_lookup_scb(ahd, scbid);
2127                 ahd_print_path(ahd, scb);
2128                 printf("Unexpected PKT busfree condition\n");
2129                 ahd_dump_card_state(ahd);
2130                 ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
2131                                SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2132                                ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
2133
2134                 /* Return restarting the sequencer. */
2135                 return (1);
2136         }
2137         printf("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
2138         ahd_dump_card_state(ahd);
2139         /* Restart the sequencer. */
2140         return (1);
2141 }
2142
2143 /*
2144  * Non-packetized unexpected or expected busfree.
2145  */
2146 static int
2147 ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
2148 {
2149         struct  ahd_devinfo devinfo;
2150         struct  scb *scb;
2151         u_int   lastphase;
2152         u_int   saved_scsiid;
2153         u_int   saved_lun;
2154         u_int   target;
2155         u_int   initiator_role_id;
2156         u_int   scbid;
2157         u_int   ppr_busfree;
2158         int     printerror;
2159
2160         /*
2161          * Look at what phase we were last in.  If its message out,
2162          * chances are pretty good that the busfree was in response
2163          * to one of our abort requests.
2164          */
2165         lastphase = ahd_inb(ahd, LASTPHASE);
2166         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
2167         saved_lun = ahd_inb(ahd, SAVED_LUN);
2168         target = SCSIID_TARGET(ahd, saved_scsiid);
2169         initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
2170         ahd_compile_devinfo(&devinfo, initiator_role_id,
2171                             target, saved_lun, 'A', ROLE_INITIATOR);
2172         printerror = 1;
2173
2174         scbid = ahd_get_scbptr(ahd);
2175         scb = ahd_lookup_scb(ahd, scbid);
2176         if (scb != NULL
2177          && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
2178                 scb = NULL;
2179
2180         ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
2181         if (lastphase == P_MESGOUT) {
2182                 u_int tag;
2183
2184                 tag = SCB_LIST_NULL;
2185                 if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
2186                  || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
2187                         int found;
2188                         int sent_msg;
2189
2190                         if (scb == NULL) {
2191                                 ahd_print_devinfo(ahd, &devinfo);
2192                                 printf("Abort for unidentified "
2193                                        "connection completed.\n");
2194                                 /* restart the sequencer. */
2195                                 return (1);
2196                         }
2197                         sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
2198                         ahd_print_path(ahd, scb);
2199                         printf("SCB %d - Abort%s Completed.\n",
2200                                SCB_GET_TAG(scb),
2201                                sent_msg == MSG_ABORT_TAG ? "" : " Tag");
2202
2203                         if (sent_msg == MSG_ABORT_TAG)
2204                                 tag = SCB_GET_TAG(scb);
2205
2206                         if ((scb->flags & SCB_CMDPHASE_ABORT) != 0) {
2207                                 /*
2208                                  * This abort is in response to an
2209                                  * unexpected switch to command phase
2210                                  * for a packetized connection.  Since
2211                                  * the identify message was never sent,
2212                                  * "saved lun" is 0.  We really want to
2213                                  * abort only the SCB that encountered
2214                                  * this error, which could have a different
2215                                  * lun.  The SCB will be retried so the OS
2216                                  * will see the UA after renegotiating to
2217                                  * packetized.
2218                                  */
2219                                 tag = SCB_GET_TAG(scb);
2220                                 saved_lun = scb->hscb->lun;
2221                         }
2222                         found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
2223                                                tag, ROLE_INITIATOR,
2224                                                CAM_REQ_ABORTED);
2225                         printf("found == 0x%x\n", found);
2226                         printerror = 0;
2227                 } else if (ahd_sent_msg(ahd, AHDMSG_1B,
2228                                         MSG_BUS_DEV_RESET, TRUE)) {
2229 #ifdef __FreeBSD__
2230                         /*
2231                          * Don't mark the user's request for this BDR
2232                          * as completing with CAM_BDR_SENT.  CAM3
2233                          * specifies CAM_REQ_CMP.
2234                          */
2235                         if (scb != NULL
2236                          && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
2237                          && ahd_match_scb(ahd, scb, target, 'A',
2238                                           CAM_LUN_WILDCARD, SCB_LIST_NULL,
2239                                           ROLE_INITIATOR))
2240                                 ahd_set_transaction_status(scb, CAM_REQ_CMP);
2241 #endif
2242                         ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
2243                                             CAM_BDR_SENT, "Bus Device Reset",
2244                                             /*verbose_level*/0);
2245                         printerror = 0;
2246                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)
2247                         && ppr_busfree == 0) {
2248                         struct ahd_initiator_tinfo *tinfo;
2249                         struct ahd_tmode_tstate *tstate;
2250
2251                         /*
2252                          * PPR Rejected.
2253                          *
2254                          * If the previous negotiation was packetized,
2255                          * this could be because the device has been
2256                          * reset without our knowledge.  Force our
2257                          * current negotiation to async and retry the
2258                          * negotiation.  Otherwise retry the command
2259                          * with non-ppr negotiation.
2260                          */
2261 #ifdef AHD_DEBUG
2262                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2263                                 printf("PPR negotiation rejected busfree.\n");
2264 #endif
2265                         tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
2266                                                     devinfo.our_scsiid,
2267                                                     devinfo.target, &tstate);
2268                         if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
2269                                 ahd_set_width(ahd, &devinfo,
2270                                               MSG_EXT_WDTR_BUS_8_BIT,
2271                                               AHD_TRANS_CUR,
2272                                               /*paused*/TRUE);
2273                                 ahd_set_syncrate(ahd, &devinfo,
2274                                                 /*period*/0, /*offset*/0,
2275                                                 /*ppr_options*/0,
2276                                                 AHD_TRANS_CUR,
2277                                                 /*paused*/TRUE);
2278                                 /*
2279                                  * The expect PPR busfree handler below
2280                                  * will effect the retry and necessary
2281                                  * abort.
2282                                  */
2283                         } else {
2284                                 tinfo->curr.transport_version = 2;
2285                                 tinfo->goal.transport_version = 2;
2286                                 tinfo->goal.ppr_options = 0;
2287                                 /*
2288                                  * Remove any SCBs in the waiting for selection
2289                                  * queue that may also be for this target so
2290                                  * that command ordering is preserved.
2291                                  */
2292                                 ahd_freeze_devq(ahd, scb);
2293                                 ahd_qinfifo_requeue_tail(ahd, scb);
2294                                 printerror = 0;
2295                         }
2296                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
2297                         && ppr_busfree == 0) {
2298                         /*
2299                          * Negotiation Rejected.  Go-narrow and
2300                          * retry command.
2301                          */
2302 #ifdef AHD_DEBUG
2303                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2304                                 printf("WDTR negotiation rejected busfree.\n");
2305 #endif
2306                         ahd_set_width(ahd, &devinfo,
2307                                       MSG_EXT_WDTR_BUS_8_BIT,
2308                                       AHD_TRANS_CUR|AHD_TRANS_GOAL,
2309                                       /*paused*/TRUE);
2310                         /*
2311                          * Remove any SCBs in the waiting for selection
2312                          * queue that may also be for this target so that
2313                          * command ordering is preserved.
2314                          */
2315                         ahd_freeze_devq(ahd, scb);
2316                         ahd_qinfifo_requeue_tail(ahd, scb);
2317                         printerror = 0;
2318                 } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)
2319                         && ppr_busfree == 0) {
2320                         /*
2321                          * Negotiation Rejected.  Go-async and
2322                          * retry command.
2323                          */
2324 #ifdef AHD_DEBUG
2325                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2326                                 printf("SDTR negotiation rejected busfree.\n");
2327 #endif
2328                         ahd_set_syncrate(ahd, &devinfo,
2329                                         /*period*/0, /*offset*/0,
2330                                         /*ppr_options*/0,
2331                                         AHD_TRANS_CUR|AHD_TRANS_GOAL,
2332                                         /*paused*/TRUE);
2333                         /*
2334                          * Remove any SCBs in the waiting for selection
2335                          * queue that may also be for this target so that
2336                          * command ordering is preserved.
2337                          */
2338                         ahd_freeze_devq(ahd, scb);
2339                         ahd_qinfifo_requeue_tail(ahd, scb);
2340                         printerror = 0;
2341                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
2342                         && ahd_sent_msg(ahd, AHDMSG_1B,
2343                                          MSG_INITIATOR_DET_ERR, TRUE)) {
2344
2345 #ifdef AHD_DEBUG
2346                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2347                                 printf("Expected IDE Busfree\n");
2348 #endif
2349                         printerror = 0;
2350                 } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
2351                         && ahd_sent_msg(ahd, AHDMSG_1B,
2352                                         MSG_MESSAGE_REJECT, TRUE)) {
2353
2354 #ifdef AHD_DEBUG
2355                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2356                                 printf("Expected QAS Reject Busfree\n");
2357 #endif
2358                         printerror = 0;
2359                 }
2360         }
2361
2362         /*
2363          * The busfree required flag is honored at the end of
2364          * the message phases.  We check it last in case we
2365          * had to send some other message that caused a busfree.
2366          */
2367         if (printerror != 0
2368          && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
2369          && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
2370
2371                 ahd_freeze_devq(ahd, scb);
2372                 ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
2373                 ahd_freeze_scb(scb);
2374                 if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
2375                         ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
2376                                        SCB_GET_CHANNEL(ahd, scb),
2377                                        SCB_GET_LUN(scb), SCB_LIST_NULL,
2378                                        ROLE_INITIATOR, CAM_REQ_ABORTED);
2379                 } else {
2380 #ifdef AHD_DEBUG
2381                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
2382                                 printf("PPR Negotiation Busfree.\n");
2383 #endif
2384                         ahd_done(ahd, scb);
2385                 }
2386                 printerror = 0;
2387         }
2388         if (printerror != 0) {
2389                 int aborted;
2390
2391                 aborted = 0;
2392                 if (scb != NULL) {
2393                         u_int tag;
2394
2395                         if ((scb->hscb->control & TAG_ENB) != 0)
2396                                 tag = SCB_GET_TAG(scb);
2397                         else
2398                                 tag = SCB_LIST_NULL;
2399                         ahd_print_path(ahd, scb);
2400                         aborted = ahd_abort_scbs(ahd, target, 'A',
2401                                        SCB_GET_LUN(scb), tag,
2402                                        ROLE_INITIATOR,
2403                                        CAM_UNEXP_BUSFREE);
2404                 } else {
2405                         /*
2406                          * We had not fully identified this connection,
2407                          * so we cannot abort anything.
2408                          */
2409                         printf("%s: ", ahd_name(ahd));
2410                 }
2411                 printf("Unexpected busfree %s, %d SCBs aborted, "
2412                        "PRGMCNT == 0x%x\n",
2413                        ahd_lookup_phase_entry(lastphase)->phasemsg,
2414                        aborted,
2415                        ahd_inw(ahd, PRGMCNT));
2416                 ahd_dump_card_state(ahd);
2417                 if (lastphase != P_BUSFREE)
2418                         ahd_force_renegotiation(ahd, &devinfo);
2419         }
2420         /* Always restart the sequencer. */
2421         return (1);
2422 }
2423
2424 static void
2425 ahd_handle_proto_violation(struct ahd_softc *ahd)
2426 {
2427         struct  ahd_devinfo devinfo;
2428         struct  scb *scb;
2429         u_int   scbid;
2430         u_int   seq_flags;
2431         u_int   curphase;
2432         u_int   lastphase;
2433         int     found;
2434
2435         ahd_fetch_devinfo(ahd, &devinfo);
2436         scbid = ahd_get_scbptr(ahd);
2437         scb = ahd_lookup_scb(ahd, scbid);
2438         seq_flags = ahd_inb(ahd, SEQ_FLAGS);
2439         curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2440         lastphase = ahd_inb(ahd, LASTPHASE);
2441         if ((seq_flags & NOT_IDENTIFIED) != 0) {
2442
2443                 /*
2444                  * The reconnecting target either did not send an
2445                  * identify message, or did, but we didn't find an SCB
2446                  * to match.
2447                  */
2448                 ahd_print_devinfo(ahd, &devinfo);
2449                 printf("Target did not send an IDENTIFY message. "
2450                        "LASTPHASE = 0x%x.\n", lastphase);
2451                 scb = NULL;
2452         } else if (scb == NULL) {
2453                 /*
2454                  * We don't seem to have an SCB active for this
2455                  * transaction.  Print an error and reset the bus.
2456                  */
2457                 ahd_print_devinfo(ahd, &devinfo);
2458                 printf("No SCB found during protocol violation\n");
2459                 goto proto_violation_reset;
2460         } else {
2461                 ahd_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
2462                 if ((seq_flags & NO_CDB_SENT) != 0) {
2463                         ahd_print_path(ahd, scb);
2464                         printf("No or incomplete CDB sent to device.\n");
2465                 } else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
2466                           & STATUS_RCVD) == 0) {
2467                         /*
2468                          * The target never bothered to provide status to
2469                          * us prior to completing the command.  Since we don't
2470                          * know the disposition of this command, we must attempt
2471                          * to abort it.  Assert ATN and prepare to send an abort
2472                          * message.
2473                          */
2474                         ahd_print_path(ahd, scb);
2475                         printf("Completed command without status.\n");
2476                 } else {
2477                         ahd_print_path(ahd, scb);
2478                         printf("Unknown protocol violation.\n");
2479                         ahd_dump_card_state(ahd);
2480                 }
2481         }
2482         if ((lastphase & ~P_DATAIN_DT) == 0
2483          || lastphase == P_COMMAND) {
2484 proto_violation_reset:
2485                 /*
2486                  * Target either went directly to data
2487                  * phase or didn't respond to our ATN.
2488                  * The only safe thing to do is to blow
2489                  * it away with a bus reset.
2490                  */
2491                 found = ahd_reset_channel(ahd, 'A', TRUE);
2492                 printf("%s: Issued Channel %c Bus Reset. "
2493                        "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
2494         } else {
2495                 /*
2496                  * Leave the selection hardware off in case
2497                  * this abort attempt will affect yet to
2498                  * be sent commands.
2499                  */
2500                 ahd_outb(ahd, SCSISEQ0,
2501                          ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2502                 ahd_assert_atn(ahd);
2503                 ahd_outb(ahd, MSG_OUT, HOST_MSG);
2504                 if (scb == NULL) {
2505                         ahd_print_devinfo(ahd, &devinfo);
2506                         ahd->msgout_buf[0] = MSG_ABORT_TASK;
2507                         ahd->msgout_len = 1;
2508                         ahd->msgout_index = 0;
2509                         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2510                 } else {
2511                         ahd_print_path(ahd, scb);
2512                         scb->flags |= SCB_ABORT;
2513                 }
2514                 printf("Protocol violation %s.  Attempting to abort.\n",
2515                        ahd_lookup_phase_entry(curphase)->phasemsg);
2516         }
2517 }
2518
2519 /*
2520  * Force renegotiation to occur the next time we initiate
2521  * a command to the current device.
2522  */
2523 static void
2524 ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
2525 {
2526         struct  ahd_initiator_tinfo *targ_info;
2527         struct  ahd_tmode_tstate *tstate;
2528
2529 #ifdef AHD_DEBUG
2530         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
2531                 ahd_print_devinfo(ahd, devinfo);
2532                 printf("Forcing renegotiation\n");
2533         }
2534 #endif
2535         targ_info = ahd_fetch_transinfo(ahd,
2536                                         devinfo->channel,
2537                                         devinfo->our_scsiid,
2538                                         devinfo->target,
2539                                         &tstate);
2540         ahd_update_neg_request(ahd, devinfo, tstate,
2541                                targ_info, AHD_NEG_IF_NON_ASYNC);
2542 }
2543
2544 #define AHD_MAX_STEPS 2000
2545 void
2546 ahd_clear_critical_section(struct ahd_softc *ahd)
2547 {
2548         ahd_mode_state  saved_modes;
2549         int             stepping;
2550         int             steps;
2551         int             first_instr;
2552         u_int           simode0;
2553         u_int           simode1;
2554         u_int           simode3;
2555         u_int           lqimode0;
2556         u_int           lqimode1;
2557         u_int           lqomode0;
2558         u_int           lqomode1;
2559
2560         if (ahd->num_critical_sections == 0)
2561                 return;
2562
2563         stepping = FALSE;
2564         steps = 0;
2565         first_instr = 0;
2566         simode0 = 0;
2567         simode1 = 0;
2568         simode3 = 0;
2569         lqimode0 = 0;
2570         lqimode1 = 0;
2571         lqomode0 = 0;
2572         lqomode1 = 0;
2573         saved_modes = ahd_save_modes(ahd);
2574         for (;;) {
2575                 struct  cs *cs;
2576                 u_int   seqaddr;
2577                 u_int   i;
2578
2579                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2580                 seqaddr = ahd_inw(ahd, CURADDR);
2581
2582                 cs = ahd->critical_sections;
2583                 for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
2584
2585                         if (cs->begin < seqaddr && cs->end >= seqaddr)
2586                                 break;
2587                 }
2588
2589                 if (i == ahd->num_critical_sections)
2590                         break;
2591
2592                 if (steps > AHD_MAX_STEPS) {
2593                         printf("%s: Infinite loop in critical section\n"
2594                                "%s: First Instruction 0x%x now 0x%x\n",
2595                                ahd_name(ahd), ahd_name(ahd), first_instr,
2596                                seqaddr);
2597                         ahd_dump_card_state(ahd);
2598                         panic("critical section loop");
2599                 }
2600
2601                 steps++;
2602 #ifdef AHD_DEBUG
2603                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
2604                         printf("%s: Single stepping at 0x%x\n", ahd_name(ahd),
2605                                seqaddr);
2606 #endif
2607                 if (stepping == FALSE) {
2608
2609                         first_instr = seqaddr;
2610                         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2611                         simode0 = ahd_inb(ahd, SIMODE0);
2612                         simode3 = ahd_inb(ahd, SIMODE3);
2613                         lqimode0 = ahd_inb(ahd, LQIMODE0);
2614                         lqimode1 = ahd_inb(ahd, LQIMODE1);
2615                         lqomode0 = ahd_inb(ahd, LQOMODE0);
2616                         lqomode1 = ahd_inb(ahd, LQOMODE1);
2617                         ahd_outb(ahd, SIMODE0, 0);
2618                         ahd_outb(ahd, SIMODE3, 0);
2619                         ahd_outb(ahd, LQIMODE0, 0);
2620                         ahd_outb(ahd, LQIMODE1, 0);
2621                         ahd_outb(ahd, LQOMODE0, 0);
2622                         ahd_outb(ahd, LQOMODE1, 0);
2623                         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2624                         simode1 = ahd_inb(ahd, SIMODE1);
2625                         /*
2626                          * We don't clear ENBUSFREE.  Unfortunately
2627                          * we cannot re-enable busfree detection within
2628                          * the current connection, so we must leave it
2629                          * on while single stepping.
2630                          */
2631                         ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
2632                         ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
2633                         stepping = TRUE;
2634                 }
2635                 ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
2636                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2637                 ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
2638                 ahd_outb(ahd, HCNTRL, ahd->unpause);
2639                 while (!ahd_is_paused(ahd))
2640                         ahd_delay(200);
2641                 ahd_update_modes(ahd);
2642         }
2643         if (stepping) {
2644                 ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2645                 ahd_outb(ahd, SIMODE0, simode0);
2646                 ahd_outb(ahd, SIMODE3, simode3);
2647                 ahd_outb(ahd, LQIMODE0, lqimode0);
2648                 ahd_outb(ahd, LQIMODE1, lqimode1);
2649                 ahd_outb(ahd, LQOMODE0, lqomode0);
2650                 ahd_outb(ahd, LQOMODE1, lqomode1);
2651                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2652                 ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
2653                 ahd_outb(ahd, SIMODE1, simode1);
2654                 /*
2655                  * SCSIINT seems to glitch occassionally when
2656                  * the interrupt masks are restored.  Clear SCSIINT
2657                  * one more time so that only persistent errors
2658                  * are seen as a real interrupt.
2659                  */
2660                 ahd_outb(ahd, CLRINT, CLRSCSIINT);
2661         }
2662         ahd_restore_modes(ahd, saved_modes);
2663 }
2664
2665 /*
2666  * Clear any pending interrupt status.
2667  */
2668 void
2669 ahd_clear_intstat(struct ahd_softc *ahd)
2670 {
2671         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2672                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2673         /* Clear any interrupt conditions this may have caused */
2674         ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
2675                                  |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
2676         ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
2677                                  |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
2678                                  |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
2679         ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
2680                                  |CLRLQOATNPKT|CLRLQOTCRC);
2681         ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
2682                                  |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
2683         if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
2684                 ahd_outb(ahd, CLRLQOINT0, 0);
2685                 ahd_outb(ahd, CLRLQOINT1, 0);
2686         }
2687         ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
2688         ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2689                                 |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
2690         ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
2691                                 |CLRIOERR|CLROVERRUN);
2692         ahd_outb(ahd, CLRINT, CLRSCSIINT);
2693 }
2694
2695 /**************************** Debugging Routines ******************************/
2696 #ifdef AHD_DEBUG
2697 uint32_t ahd_debug = AHD_DEBUG_OPTS;
2698 #endif
2699 void
2700 ahd_print_scb(struct scb *scb)
2701 {
2702         struct hardware_scb *hscb;
2703         int i;
2704
2705         hscb = scb->hscb;
2706         printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2707                (void *)scb,
2708                hscb->control,
2709                hscb->scsiid,
2710                hscb->lun,
2711                hscb->cdb_len);
2712         printf("Shared Data: ");
2713         for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
2714                 printf("%#02x", hscb->shared_data.idata.cdb[i]);
2715         printf("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
2716                (uint32_t)((ahd_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
2717                (uint32_t)(ahd_le64toh(hscb->dataptr) & 0xFFFFFFFF),
2718                ahd_le32toh(hscb->datacnt),
2719                ahd_le32toh(hscb->sgptr),
2720                SCB_GET_TAG(scb));
2721         ahd_dump_sglist(scb);
2722 }
2723
2724 void
2725 ahd_dump_sglist(struct scb *scb)
2726 {
2727         int i;
2728
2729         if (scb->sg_count > 0) {
2730                 if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
2731                         struct ahd_dma64_seg *sg_list;
2732
2733                         sg_list = (struct ahd_dma64_seg*)scb->sg_list;
2734                         for (i = 0; i < scb->sg_count; i++) {
2735                                 uint64_t addr;
2736                                 uint32_t len;
2737
2738                                 addr = ahd_le64toh(sg_list[i].addr);
2739                                 len = ahd_le32toh(sg_list[i].len);
2740                                 printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
2741                                        i,
2742                                        (uint32_t)((addr >> 32) & 0xFFFFFFFF),
2743                                        (uint32_t)(addr & 0xFFFFFFFF),
2744                                        sg_list[i].len & AHD_SG_LEN_MASK,
2745                                        (sg_list[i].len & AHD_DMA_LAST_SEG)
2746                                      ? " Last" : "");
2747                         }
2748                 } else {
2749                         struct ahd_dma_seg *sg_list;
2750
2751                         sg_list = (struct ahd_dma_seg*)scb->sg_list;
2752                         for (i = 0; i < scb->sg_count; i++) {
2753                                 uint32_t len;
2754
2755                                 len = ahd_le32toh(sg_list[i].len);
2756                                 printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
2757                                        i,
2758                                        (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
2759                                        ahd_le32toh(sg_list[i].addr),
2760                                        len & AHD_SG_LEN_MASK,
2761                                        len & AHD_DMA_LAST_SEG ? " Last" : "");
2762                         }
2763                 }
2764         }
2765 }
2766
2767 /************************* Transfer Negotiation *******************************/
2768 /*
2769  * Allocate per target mode instance (ID we respond to as a target)
2770  * transfer negotiation data structures.
2771  */
2772 static struct ahd_tmode_tstate *
2773 ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
2774 {
2775         struct ahd_tmode_tstate *master_tstate;
2776         struct ahd_tmode_tstate *tstate;
2777         int i;
2778
2779         master_tstate = ahd->enabled_targets[ahd->our_id];
2780         if (ahd->enabled_targets[scsi_id] != NULL
2781          && ahd->enabled_targets[scsi_id] != master_tstate)
2782                 panic("%s: ahd_alloc_tstate - Target already allocated",
2783                       ahd_name(ahd));
2784         tstate = malloc(sizeof(*tstate), M_DEVBUF, M_NOWAIT);
2785         if (tstate == NULL)
2786                 return (NULL);
2787
2788         /*
2789          * If we have allocated a master tstate, copy user settings from
2790          * the master tstate (taken from SRAM or the EEPROM) for this
2791          * channel, but reset our current and goal settings to async/narrow
2792          * until an initiator talks to us.
2793          */
2794         if (master_tstate != NULL) {
2795                 memcpy(tstate, master_tstate, sizeof(*tstate));
2796                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2797                 for (i = 0; i < 16; i++) {
2798                         memset(&tstate->transinfo[i].curr, 0,
2799                               sizeof(tstate->transinfo[i].curr));
2800                         memset(&tstate->transinfo[i].goal, 0,
2801                               sizeof(tstate->transinfo[i].goal));
2802                 }
2803         } else
2804                 memset(tstate, 0, sizeof(*tstate));
2805         ahd->enabled_targets[scsi_id] = tstate;
2806         return (tstate);
2807 }
2808
2809 #ifdef AHD_TARGET_MODE
2810 /*
2811  * Free per target mode instance (ID we respond to as a target)
2812  * transfer negotiation data structures.
2813  */
2814 static void
2815 ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
2816 {
2817         struct ahd_tmode_tstate *tstate;
2818
2819         /*
2820          * Don't clean up our "master" tstate.
2821          * It has our default user settings.
2822          */
2823         if (scsi_id == ahd->our_id
2824          && force == FALSE)
2825                 return;
2826
2827         tstate = ahd->enabled_targets[scsi_id];
2828         if (tstate != NULL)
2829                 free(tstate, M_DEVBUF);
2830         ahd->enabled_targets[scsi_id] = NULL;
2831 }
2832 #endif
2833
2834 /*
2835  * Called when we have an active connection to a target on the bus,
2836  * this function finds the nearest period to the input period limited
2837  * by the capabilities of the bus connectivity of and sync settings for
2838  * the target.
2839  */
2840 void
2841 ahd_devlimited_syncrate(struct ahd_softc *ahd,
2842                         struct ahd_initiator_tinfo *tinfo,
2843                         u_int *period, u_int *ppr_options, role_t role)
2844 {
2845         struct  ahd_transinfo *transinfo;
2846         u_int   maxsync;
2847
2848         if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
2849          && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
2850                 maxsync = AHD_SYNCRATE_PACED;
2851         } else {
2852                 maxsync = AHD_SYNCRATE_ULTRA;
2853                 /* Can't do DT related options on an SE bus */
2854                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
2855         }
2856         /*
2857          * Never allow a value higher than our current goal
2858          * period otherwise we may allow a target initiated
2859          * negotiation to go above the limit as set by the
2860          * user.  In the case of an initiator initiated
2861          * sync negotiation, we limit based on the user
2862          * setting.  This allows the system to still accept
2863          * incoming negotiations even if target initiated
2864          * negotiation is not performed.
2865          */
2866         if (role == ROLE_TARGET)
2867                 transinfo = &tinfo->user;
2868         else
2869                 transinfo = &tinfo->goal;
2870         *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
2871         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2872                 maxsync = MAX(maxsync, AHD_SYNCRATE_ULTRA2);
2873                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2874         }
2875         if (transinfo->period == 0) {
2876                 *period = 0;
2877                 *ppr_options = 0;
2878         } else {
2879                 *period = MAX(*period, transinfo->period);
2880                 ahd_find_syncrate(ahd, period, ppr_options, maxsync);
2881         }
2882 }
2883
2884 /*
2885  * Look up the valid period to SCSIRATE conversion in our table.
2886  * Return the period and offset that should be sent to the target
2887  * if this was the beginning of an SDTR.
2888  */
2889 void
2890 ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
2891                   u_int *ppr_options, u_int maxsync)
2892 {
2893         if (*period < maxsync)
2894                 *period = maxsync;
2895
2896         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
2897          && *period > AHD_SYNCRATE_MIN_DT)
2898                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2899
2900         if (*period > AHD_SYNCRATE_MIN)
2901                 *period = 0;
2902
2903         /* Honor PPR option conformance rules. */
2904         if (*period > AHD_SYNCRATE_PACED)
2905                 *ppr_options &= ~MSG_EXT_PPR_RTI;
2906
2907         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
2908                 *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
2909
2910         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
2911                 *ppr_options &= MSG_EXT_PPR_QAS_REQ;
2912
2913         /* Skip all PACED only entries if IU is not available */
2914         if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
2915          && *period < AHD_SYNCRATE_DT)
2916                 *period = AHD_SYNCRATE_DT;
2917
2918         /* Skip all DT only entries if DT is not available */
2919         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2920          && *period < AHD_SYNCRATE_ULTRA2)
2921                 *period = AHD_SYNCRATE_ULTRA2;
2922 }
2923
2924 /*
2925  * Truncate the given synchronous offset to a value the
2926  * current adapter type and syncrate are capable of.
2927  */
2928 void
2929 ahd_validate_offset(struct ahd_softc *ahd,
2930                     struct ahd_initiator_tinfo *tinfo,
2931                     u_int period, u_int *offset, int wide,
2932                     role_t role)
2933 {
2934         u_int maxoffset;
2935
2936         /* Limit offset to what we can do */
2937         if (period == 0)
2938                 maxoffset = 0;
2939         else if (period <= AHD_SYNCRATE_PACED) {
2940                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
2941                         maxoffset = MAX_OFFSET_PACED_BUG;
2942                 else
2943                         maxoffset = MAX_OFFSET_PACED;
2944         } else
2945                 maxoffset = MAX_OFFSET_NON_PACED;
2946         *offset = MIN(*offset, maxoffset);
2947         if (tinfo != NULL) {
2948                 if (role == ROLE_TARGET)
2949                         *offset = MIN(*offset, tinfo->user.offset);
2950                 else
2951                         *offset = MIN(*offset, tinfo->goal.offset);
2952         }
2953 }
2954
2955 /*
2956  * Truncate the given transfer width parameter to a value the
2957  * current adapter type is capable of.
2958  */
2959 void
2960 ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
2961                    u_int *bus_width, role_t role)
2962 {
2963         switch (*bus_width) {
2964         default:
2965                 if (ahd->features & AHD_WIDE) {
2966                         /* Respond Wide */
2967                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
2968                         break;
2969                 }
2970                 /* FALLTHROUGH */
2971         case MSG_EXT_WDTR_BUS_8_BIT:
2972                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
2973                 break;
2974         }
2975         if (tinfo != NULL) {
2976                 if (role == ROLE_TARGET)
2977                         *bus_width = MIN(tinfo->user.width, *bus_width);
2978                 else
2979                         *bus_width = MIN(tinfo->goal.width, *bus_width);
2980         }
2981 }
2982
2983 /*
2984  * Update the bitmask of targets for which the controller should
2985  * negotiate with at the next convenient oportunity.  This currently
2986  * means the next time we send the initial identify messages for
2987  * a new transaction.
2988  */
2989 int
2990 ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
2991                        struct ahd_tmode_tstate *tstate,
2992                        struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
2993 {
2994         u_int auto_negotiate_orig;
2995
2996         auto_negotiate_orig = tstate->auto_negotiate;
2997         if (neg_type == AHD_NEG_ALWAYS) {
2998                 /*
2999                  * Force our "current" settings to be
3000                  * unknown so that unless a bus reset
3001                  * occurs the need to renegotiate is
3002                  * recorded persistently.
3003                  */
3004                 if ((ahd->features & AHD_WIDE) != 0)
3005                         tinfo->curr.width = AHD_WIDTH_UNKNOWN;
3006                 tinfo->curr.period = AHD_PERIOD_UNKNOWN;
3007                 tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
3008         }
3009         if (tinfo->curr.period != tinfo->goal.period
3010          || tinfo->curr.width != tinfo->goal.width
3011          || tinfo->curr.offset != tinfo->goal.offset
3012          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
3013          || (neg_type == AHD_NEG_IF_NON_ASYNC
3014           && (tinfo->goal.offset != 0
3015            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
3016            || tinfo->goal.ppr_options != 0)))
3017                 tstate->auto_negotiate |= devinfo->target_mask;
3018         else
3019                 tstate->auto_negotiate &= ~devinfo->target_mask;
3020
3021         return (auto_negotiate_orig != tstate->auto_negotiate);
3022 }
3023
3024 /*
3025  * Update the user/goal/curr tables of synchronous negotiation
3026  * parameters as well as, in the case of a current or active update,
3027  * any data structures on the host controller.  In the case of an
3028  * active update, the specified target is currently talking to us on
3029  * the bus, so the transfer parameter update must take effect
3030  * immediately.
3031  */
3032 void
3033 ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3034                  u_int period, u_int offset, u_int ppr_options,
3035                  u_int type, int paused)
3036 {
3037         struct  ahd_initiator_tinfo *tinfo;
3038         struct  ahd_tmode_tstate *tstate;
3039         u_int   old_period;
3040         u_int   old_offset;
3041         u_int   old_ppr;
3042         int     active;
3043         int     update_needed;
3044
3045         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3046         update_needed = 0;
3047
3048         if (period == 0 || offset == 0) {
3049                 period = 0;
3050                 offset = 0;
3051         }
3052
3053         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3054                                     devinfo->target, &tstate);
3055
3056         if ((type & AHD_TRANS_USER) != 0) {
3057                 tinfo->user.period = period;
3058                 tinfo->user.offset = offset;
3059                 tinfo->user.ppr_options = ppr_options;
3060         }
3061
3062         if ((type & AHD_TRANS_GOAL) != 0) {
3063                 tinfo->goal.period = period;
3064                 tinfo->goal.offset = offset;
3065                 tinfo->goal.ppr_options = ppr_options;
3066         }
3067
3068         old_period = tinfo->curr.period;
3069         old_offset = tinfo->curr.offset;
3070         old_ppr    = tinfo->curr.ppr_options;
3071
3072         if ((type & AHD_TRANS_CUR) != 0
3073          && (old_period != period
3074           || old_offset != offset
3075           || old_ppr != ppr_options)) {
3076
3077                 update_needed++;
3078
3079                 tinfo->curr.period = period;
3080                 tinfo->curr.offset = offset;
3081                 tinfo->curr.ppr_options = ppr_options;
3082
3083                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3084                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
3085                 if (bootverbose) {
3086                         if (offset != 0) {
3087                                 int options;
3088
3089                                 printf("%s: target %d synchronous with "
3090                                        "period = 0x%x, offset = 0x%x",
3091                                        ahd_name(ahd), devinfo->target,
3092                                        period, offset);
3093                                 options = 0;
3094                                 if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
3095                                         printf("(RDSTRM");
3096                                         options++;
3097                                 }
3098                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
3099                                         printf("%s", options ? "|DT" : "(DT");
3100                                         options++;
3101                                 }
3102                                 if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
3103                                         printf("%s", options ? "|IU" : "(IU");
3104                                         options++;
3105                                 }
3106                                 if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
3107                                         printf("%s", options ? "|RTI" : "(RTI");
3108                                         options++;
3109                                 }
3110                                 if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
3111                                         printf("%s", options ? "|QAS" : "(QAS");
3112                                         options++;
3113                                 }
3114                                 if (options != 0)
3115                                         printf(")\n");
3116                                 else
3117                                         printf("\n");
3118                         } else {
3119                                 printf("%s: target %d using "
3120                                        "asynchronous transfers%s\n",
3121                                        ahd_name(ahd), devinfo->target,
3122                                        (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
3123                                      ?  "(QAS)" : "");
3124                         }
3125                 }
3126         }
3127         /*
3128          * Always refresh the neg-table to handle the case of the
3129          * sequencer setting the ENATNO bit for a MK_MESSAGE request.
3130          * We will always renegotiate in that case if this is a
3131          * packetized request.  Also manage the busfree expected flag
3132          * from this common routine so that we catch changes due to
3133          * WDTR or SDTR messages.
3134          */
3135         if ((type & AHD_TRANS_CUR) != 0) {
3136                 if (!paused)
3137                         ahd_pause(ahd);
3138                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3139                 if (!paused)
3140                         ahd_unpause(ahd);
3141                 if (ahd->msg_type != MSG_TYPE_NONE) {
3142                         if ((old_ppr & MSG_EXT_PPR_IU_REQ)
3143                          != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
3144 #ifdef AHD_DEBUG
3145                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3146                                         ahd_print_devinfo(ahd, devinfo);
3147                                         printf("Expecting IU Change busfree\n");
3148                                 }
3149 #endif
3150                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
3151                                                |  MSG_FLAG_IU_REQ_CHANGED;
3152                         }
3153                         if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
3154 #ifdef AHD_DEBUG
3155                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3156                                         printf("PPR with IU_REQ outstanding\n");
3157 #endif
3158                                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
3159                         }
3160                 }
3161         }
3162
3163         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3164                                                 tinfo, AHD_NEG_TO_GOAL);
3165
3166         if (update_needed && active)
3167                 ahd_update_pending_scbs(ahd);
3168 }
3169
3170 /*
3171  * Update the user/goal/curr tables of wide negotiation
3172  * parameters as well as, in the case of a current or active update,
3173  * any data structures on the host controller.  In the case of an
3174  * active update, the specified target is currently talking to us on
3175  * the bus, so the transfer parameter update must take effect
3176  * immediately.
3177  */
3178 void
3179 ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3180               u_int width, u_int type, int paused)
3181 {
3182         struct  ahd_initiator_tinfo *tinfo;
3183         struct  ahd_tmode_tstate *tstate;
3184         u_int   oldwidth;
3185         int     active;
3186         int     update_needed;
3187
3188         active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3189         update_needed = 0;
3190         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3191                                     devinfo->target, &tstate);
3192
3193         if ((type & AHD_TRANS_USER) != 0)
3194                 tinfo->user.width = width;
3195
3196         if ((type & AHD_TRANS_GOAL) != 0)
3197                 tinfo->goal.width = width;
3198
3199         oldwidth = tinfo->curr.width;
3200         if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
3201
3202                 update_needed++;
3203
3204                 tinfo->curr.width = width;
3205                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
3206                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
3207                 if (bootverbose) {
3208                         printf("%s: target %d using %dbit transfers\n",
3209                                ahd_name(ahd), devinfo->target,
3210                                8 * (0x01 << width));
3211                 }
3212         }
3213
3214         if ((type & AHD_TRANS_CUR) != 0) {
3215                 if (!paused)
3216                         ahd_pause(ahd);
3217                 ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3218                 if (!paused)
3219                         ahd_unpause(ahd);
3220         }
3221
3222         update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
3223                                                 tinfo, AHD_NEG_TO_GOAL);
3224         if (update_needed && active)
3225                 ahd_update_pending_scbs(ahd);
3226
3227 }
3228
3229 /*
3230  * Update the current state of tagged queuing for a given target.
3231  */
3232 void
3233 ahd_set_tags(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3234              ahd_queue_alg alg)
3235 {
3236         ahd_platform_set_tags(ahd, devinfo, alg);
3237         ahd_send_async(ahd, devinfo->channel, devinfo->target,
3238                        devinfo->lun, AC_TRANSFER_NEG, &alg);
3239 }
3240
3241 static void
3242 ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3243                      struct ahd_transinfo *tinfo)
3244 {
3245         ahd_mode_state  saved_modes;
3246         u_int           period;
3247         u_int           ppr_opts;
3248         u_int           con_opts;
3249         u_int           offset;
3250         u_int           saved_negoaddr;
3251         uint8_t         iocell_opts[sizeof(ahd->iocell_opts)];
3252
3253         saved_modes = ahd_save_modes(ahd);
3254         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3255
3256         saved_negoaddr = ahd_inb(ahd, NEGOADDR);
3257         ahd_outb(ahd, NEGOADDR, devinfo->target);
3258         period = tinfo->period;
3259         offset = tinfo->offset;
3260         memcpy(iocell_opts, ahd->iocell_opts, sizeof(ahd->iocell_opts));
3261         ppr_opts = tinfo->ppr_options & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ
3262                                         |MSG_EXT_PPR_IU_REQ|MSG_EXT_PPR_RTI);
3263         con_opts = 0;
3264         if (period == 0)
3265                 period = AHD_SYNCRATE_ASYNC;
3266         if (period == AHD_SYNCRATE_160) {
3267
3268                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
3269                         /*
3270                          * When the SPI4 spec was finalized, PACE transfers
3271                          * was not made a configurable option in the PPR
3272                          * message.  Instead it is assumed to be enabled for
3273                          * any syncrate faster than 80MHz.  Nevertheless,
3274                          * Harpoon2A4 allows this to be configurable.
3275                          *
3276                          * Harpoon2A4 also assumes at most 2 data bytes per
3277                          * negotiated REQ/ACK offset.  Paced transfers take
3278                          * 4, so we must adjust our offset.
3279                          */
3280                         ppr_opts |= PPROPT_PACE;
3281                         offset *= 2;
3282
3283                         /*
3284                          * Harpoon2A assumed that there would be a
3285                          * fallback rate between 160MHz and 80Mhz,
3286                          * so 7 is used as the period factor rather
3287                          * than 8 for 160MHz.
3288                          */
3289                         period = AHD_SYNCRATE_REVA_160;
3290                 }
3291                 if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) == 0)
3292                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
3293                             ~AHD_PRECOMP_MASK;
3294         } else {
3295                 /*
3296                  * Precomp should be disabled for non-paced transfers.
3297                  */
3298                 iocell_opts[AHD_PRECOMP_SLEW_INDEX] &= ~AHD_PRECOMP_MASK;
3299
3300                 if ((ahd->features & AHD_NEW_IOCELL_OPTS) != 0
3301                  && (ppr_opts & MSG_EXT_PPR_DT_REQ) != 0
3302                  && (ppr_opts & MSG_EXT_PPR_IU_REQ) == 0) {
3303                         /*
3304                          * Slow down our CRC interval to be
3305                          * compatible with non-packetized
3306                          * U160 devices that can't handle a
3307                          * CRC at full speed.
3308                          */
3309                         con_opts |= ENSLOWCRC;
3310                 }
3311
3312                 if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
3313                         /*
3314                          * On H2A4, revert to a slower slewrate
3315                          * on non-paced transfers.
3316                          */
3317                         iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
3318                             ~AHD_SLEWRATE_MASK;
3319                 }
3320         }
3321
3322         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP_SLEW);
3323         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_PRECOMP_SLEW_INDEX]);
3324         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_AMPLITUDE);
3325         ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_AMPLITUDE_INDEX]);
3326
3327         ahd_outb(ahd, NEGPERIOD, period);
3328         ahd_outb(ahd, NEGPPROPTS, ppr_opts);
3329         ahd_outb(ahd, NEGOFFSET, offset);
3330
3331         if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
3332                 con_opts |= WIDEXFER;
3333
3334         /*
3335          * Slow down our CRC interval to be
3336          * compatible with packetized U320 devices
3337          * that can't handle a CRC at full speed
3338          */
3339         if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
3340                 con_opts |= ENSLOWCRC;
3341         }
3342
3343         /*
3344          * During packetized transfers, the target will
3345          * give us the oportunity to send command packets
3346          * without us asserting attention.
3347          */
3348         if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
3349                 con_opts |= ENAUTOATNO;
3350         ahd_outb(ahd, NEGCONOPTS, con_opts);
3351         ahd_outb(ahd, NEGOADDR, saved_negoaddr);
3352         ahd_restore_modes(ahd, saved_modes);
3353 }
3354
3355 /*
3356  * When the transfer settings for a connection change, setup for
3357  * negotiation in pending SCBs to effect the change as quickly as
3358  * possible.  We also cancel any negotiations that are scheduled
3359  * for inflight SCBs that have not been started yet.
3360  */
3361 static void
3362 ahd_update_pending_scbs(struct ahd_softc *ahd)
3363 {
3364         struct          scb *pending_scb;
3365         int             pending_scb_count;
3366         int             paused;
3367         u_int           saved_scbptr;
3368         ahd_mode_state  saved_modes;
3369
3370         /*
3371          * Traverse the pending SCB list and ensure that all of the
3372          * SCBs there have the proper settings.  We can only safely
3373          * clear the negotiation required flag (setting requires the
3374          * execution queue to be modified) and this is only possible
3375          * if we are not already attempting to select out for this
3376          * SCB.  For this reason, all callers only call this routine
3377          * if we are changing the negotiation settings for the currently
3378          * active transaction on the bus.
3379          */
3380         pending_scb_count = 0;
3381         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
3382                 struct ahd_devinfo devinfo;
3383                 struct ahd_initiator_tinfo *tinfo;
3384                 struct ahd_tmode_tstate *tstate;
3385
3386                 ahd_scb_devinfo(ahd, &devinfo, pending_scb);
3387                 tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
3388                                             devinfo.our_scsiid,
3389                                             devinfo.target, &tstate);
3390                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
3391                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
3392                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
3393                         pending_scb->hscb->control &= ~MK_MESSAGE;
3394                 }
3395                 ahd_sync_scb(ahd, pending_scb,
3396                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
3397                 pending_scb_count++;
3398         }
3399
3400         if (pending_scb_count == 0)
3401                 return;
3402
3403         if (ahd_is_paused(ahd)) {
3404                 paused = 1;
3405         } else {
3406                 paused = 0;
3407                 ahd_pause(ahd);
3408         }
3409
3410         /*
3411          * Force the sequencer to reinitialize the selection for
3412          * the command at the head of the execution queue if it
3413          * has already been setup.  The negotiation changes may
3414          * effect whether we select-out with ATN.  It is only
3415          * safe to clear ENSELO when the bus is not free and no
3416          * selection is in progres or completed.
3417          */
3418         saved_modes = ahd_save_modes(ahd);
3419         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3420         if ((ahd_inb(ahd, SCSISIGI) & BSYI) != 0
3421          && (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) == 0)
3422                 ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
3423         saved_scbptr = ahd_get_scbptr(ahd);
3424         /* Ensure that the hscbs down on the card match the new information */
3425         LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
3426                 u_int   scb_tag;
3427                 u_int   control;
3428
3429                 scb_tag = SCB_GET_TAG(pending_scb);
3430                 ahd_set_scbptr(ahd, scb_tag);
3431                 control = ahd_inb_scbram(ahd, SCB_CONTROL);
3432                 control &= ~MK_MESSAGE;
3433                 control |= pending_scb->hscb->control & MK_MESSAGE;
3434                 ahd_outb(ahd, SCB_CONTROL, control);
3435         }
3436         ahd_set_scbptr(ahd, saved_scbptr);
3437         ahd_restore_modes(ahd, saved_modes);
3438
3439         if (paused == 0)
3440                 ahd_unpause(ahd);
3441 }
3442
3443 /**************************** Pathing Information *****************************/
3444 static void
3445 ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3446 {
3447         ahd_mode_state  saved_modes;
3448         u_int           saved_scsiid;
3449         role_t          role;
3450         int             our_id;
3451
3452         saved_modes = ahd_save_modes(ahd);
3453         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3454
3455         if (ahd_inb(ahd, SSTAT0) & TARGET)
3456                 role = ROLE_TARGET;
3457         else
3458                 role = ROLE_INITIATOR;
3459
3460         if (role == ROLE_TARGET
3461          && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
3462                 /* We were selected, so pull our id from TARGIDIN */
3463                 our_id = ahd_inb(ahd, TARGIDIN) & OID;
3464         } else if (role == ROLE_TARGET)
3465                 our_id = ahd_inb(ahd, TOWNID);
3466         else
3467                 our_id = ahd_inb(ahd, IOWNID);
3468
3469         saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
3470         ahd_compile_devinfo(devinfo,
3471                             our_id,
3472                             SCSIID_TARGET(ahd, saved_scsiid),
3473                             ahd_inb(ahd, SAVED_LUN),
3474                             SCSIID_CHANNEL(ahd, saved_scsiid),
3475                             role);
3476         ahd_restore_modes(ahd, saved_modes);
3477 }
3478
3479 void
3480 ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3481 {
3482         printf("%s:%c:%d:%d: ", ahd_name(ahd), 'A',
3483                devinfo->target, devinfo->lun);
3484 }
3485
3486 struct ahd_phase_table_entry*
3487 ahd_lookup_phase_entry(int phase)
3488 {
3489         struct ahd_phase_table_entry *entry;
3490         struct ahd_phase_table_entry *last_entry;
3491
3492         /*
3493          * num_phases doesn't include the default entry which
3494          * will be returned if the phase doesn't match.
3495          */
3496         last_entry = &ahd_phase_table[num_phases];
3497         for (entry = ahd_phase_table; entry < last_entry; entry++) {
3498                 if (phase == entry->phase)
3499                         break;
3500         }
3501         return (entry);
3502 }
3503
3504 void
3505 ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
3506                     u_int lun, char channel, role_t role)
3507 {
3508         devinfo->our_scsiid = our_id;
3509         devinfo->target = target;
3510         devinfo->lun = lun;
3511         devinfo->target_offset = target;
3512         devinfo->channel = channel;
3513         devinfo->role = role;
3514         if (channel == 'B')
3515                 devinfo->target_offset += 8;
3516         devinfo->target_mask = (0x01 << devinfo->target_offset);
3517 }
3518
3519 static void
3520 ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3521                 struct scb *scb)
3522 {
3523         role_t  role;
3524         int     our_id;
3525
3526         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
3527         role = ROLE_INITIATOR;
3528         if ((scb->hscb->control & TARGET_SCB) != 0)
3529                 role = ROLE_TARGET;
3530         ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
3531                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
3532 }
3533
3534
3535 /************************ Message Phase Processing ****************************/
3536 /*
3537  * When an initiator transaction with the MK_MESSAGE flag either reconnects
3538  * or enters the initial message out phase, we are interrupted.  Fill our
3539  * outgoing message buffer with the appropriate message and beging handing
3540  * the message phase(s) manually.
3541  */
3542 static void
3543 ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3544                            struct scb *scb)
3545 {
3546         /*
3547          * To facilitate adding multiple messages together,
3548          * each routine should increment the index and len
3549          * variables instead of setting them explicitly.
3550          */
3551         ahd->msgout_index = 0;
3552         ahd->msgout_len = 0;
3553
3554         if (ahd_currently_packetized(ahd))
3555                 ahd->msg_flags |= MSG_FLAG_PACKETIZED;
3556
3557         if (ahd->send_msg_perror
3558          && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
3559                 ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
3560                 ahd->msgout_len++;
3561                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3562 #ifdef AHD_DEBUG
3563                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3564                         printf("Setting up for Parity Error delivery\n");
3565 #endif
3566                 return;
3567         } else if (scb == NULL) {
3568                 printf("%s: WARNING. No pending message for "
3569                        "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
3570                 ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
3571                 ahd->msgout_len++;
3572                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3573                 return;
3574         }
3575
3576         if ((scb->flags & SCB_DEVICE_RESET) == 0
3577          && (scb->flags & SCB_PACKETIZED) == 0
3578          && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
3579                 u_int identify_msg;
3580
3581                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
3582                 if ((scb->hscb->control & DISCENB) != 0)
3583                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
3584                 ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
3585                 ahd->msgout_len++;
3586
3587                 if ((scb->hscb->control & TAG_ENB) != 0) {
3588                         ahd->msgout_buf[ahd->msgout_index++] =
3589                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
3590                         ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
3591                         ahd->msgout_len += 2;
3592                 }
3593         }
3594
3595         if (scb->flags & SCB_DEVICE_RESET) {
3596                 ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
3597                 ahd->msgout_len++;
3598                 ahd_print_path(ahd, scb);
3599                 printf("Bus Device Reset Message Sent\n");
3600                 /*
3601                  * Clear our selection hardware in advance of
3602                  * the busfree.  We may have an entry in the waiting
3603                  * Q for this target, and we don't want to go about
3604                  * selecting while we handle the busfree and blow it
3605                  * away.
3606                  */
3607                 ahd_outb(ahd, SCSISEQ0, 0);
3608         } else if ((scb->flags & SCB_ABORT) != 0) {
3609
3610                 if ((scb->hscb->control & TAG_ENB) != 0) {
3611                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
3612                 } else {
3613                         ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
3614                 }
3615                 ahd->msgout_len++;
3616                 ahd_print_path(ahd, scb);
3617                 printf("Abort%s Message Sent\n",
3618                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
3619                 /*
3620                  * Clear our selection hardware in advance of
3621                  * the busfree.  We may have an entry in the waiting
3622                  * Q for this target, and we don't want to go about
3623                  * selecting while we handle the busfree and blow it
3624                  * away.
3625                  */
3626                 ahd_outb(ahd, SCSISEQ0, 0);
3627         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
3628                 ahd_build_transfer_msg(ahd, devinfo);
3629                 /*
3630                  * Clear our selection hardware in advance of potential
3631                  * PPR IU status change busfree.  We may have an entry in
3632                  * the waiting Q for this target, and we don't want to go
3633                  * about selecting while we handle the busfree and blow
3634                  * it away.
3635                  */
3636                 ahd_outb(ahd, SCSISEQ0, 0);
3637         } else {
3638                 printf("ahd_intr: AWAITING_MSG for an SCB that "
3639                        "does not have a waiting message\n");
3640                 printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
3641                        devinfo->target_mask);
3642                 panic("SCB = %d, SCB Control = %x:%x, MSG_OUT = %x "
3643                       "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
3644                       ahd_inb_scbram(ahd, SCB_CONTROL), ahd_inb(ahd, MSG_OUT),
3645                       scb->flags);
3646         }
3647
3648         /*
3649          * Clear the MK_MESSAGE flag from the SCB so we aren't
3650          * asked to send this message again.
3651          */
3652         ahd_outb(ahd, SCB_CONTROL,
3653                  ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
3654         scb->hscb->control &= ~MK_MESSAGE;
3655         ahd->msgout_index = 0;
3656         ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3657 }
3658
3659 /*
3660  * Build an appropriate transfer negotiation message for the
3661  * currently active target.
3662  */
3663 static void
3664 ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3665 {
3666         /*
3667          * We need to initiate transfer negotiations.
3668          * If our current and goal settings are identical,
3669          * we want to renegotiate due to a check condition.
3670          */
3671         struct  ahd_initiator_tinfo *tinfo;
3672         struct  ahd_tmode_tstate *tstate;
3673         int     dowide;
3674         int     dosync;
3675         int     doppr;
3676         u_int   period;
3677         u_int   ppr_options;
3678         u_int   offset;
3679
3680         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3681                                     devinfo->target, &tstate);
3682         /*
3683          * Filter our period based on the current connection.
3684          * If we can't perform DT transfers on this segment (not in LVD
3685          * mode for instance), then our decision to issue a PPR message
3686          * may change.
3687          */
3688         period = tinfo->goal.period;
3689         offset = tinfo->goal.offset;
3690         ppr_options = tinfo->goal.ppr_options;
3691         /* Target initiated PPR is not allowed in the SCSI spec */
3692         if (devinfo->role == ROLE_TARGET)
3693                 ppr_options = 0;
3694         ahd_devlimited_syncrate(ahd, tinfo, &period,
3695                                 &ppr_options, devinfo->role);
3696         dowide = tinfo->curr.width != tinfo->goal.width;
3697         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
3698         /*
3699          * Only use PPR if we have options that need it, even if the device
3700          * claims to support it.  There might be an expander in the way
3701          * that doesn't.
3702          */
3703         doppr = ppr_options != 0;
3704
3705         if (!dowide && !dosync && !doppr) {
3706                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
3707                 dosync = tinfo->goal.offset != 0;
3708         }
3709
3710         if (!dowide && !dosync && !doppr) {
3711                 /*
3712                  * Force async with a WDTR message if we have a wide bus,
3713                  * or just issue an SDTR with a 0 offset.
3714                  */
3715                 if ((ahd->features & AHD_WIDE) != 0)
3716                         dowide = 1;
3717                 else
3718                         dosync = 1;
3719
3720                 if (bootverbose) {
3721                         ahd_print_devinfo(ahd, devinfo);
3722                         printf("Ensuring async\n");
3723                 }
3724         }
3725         /* Target initiated PPR is not allowed in the SCSI spec */
3726         if (devinfo->role == ROLE_TARGET)
3727                 doppr = 0;
3728
3729         /*
3730          * Both the PPR message and SDTR message require the
3731          * goal syncrate to be limited to what the target device
3732          * is capable of handling (based on whether an LVD->SE
3733          * expander is on the bus), so combine these two cases.
3734          * Regardless, guarantee that if we are using WDTR and SDTR
3735          * messages that WDTR comes first.
3736          */
3737         if (doppr || (dosync && !dowide)) {
3738
3739                 offset = tinfo->goal.offset;
3740                 ahd_validate_offset(ahd, tinfo, period, &offset,
3741                                     doppr ? tinfo->goal.width
3742                                           : tinfo->curr.width,
3743                                     devinfo->role);
3744                 if (doppr) {
3745                         ahd_construct_ppr(ahd, devinfo, period, offset,
3746                                           tinfo->goal.width, ppr_options);
3747                 } else {
3748                         ahd_construct_sdtr(ahd, devinfo, period, offset);
3749                 }
3750         } else {
3751                 ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
3752         }
3753 }
3754
3755 /*
3756  * Build a synchronous negotiation message in our message
3757  * buffer based on the input parameters.
3758  */
3759 static void
3760 ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3761                    u_int period, u_int offset)
3762 {
3763         if (offset == 0)
3764                 period = AHD_ASYNC_XFER_PERIOD;
3765         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3766         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR_LEN;
3767         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_SDTR;
3768         ahd->msgout_buf[ahd->msgout_index++] = period;
3769         ahd->msgout_buf[ahd->msgout_index++] = offset;
3770         ahd->msgout_len += 5;
3771         if (bootverbose) {
3772                 printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
3773                        ahd_name(ahd), devinfo->channel, devinfo->target,
3774                        devinfo->lun, period, offset);
3775         }
3776 }
3777
3778 /*
3779  * Build a wide negotiateion message in our message
3780  * buffer based on the input parameters.
3781  */
3782 static void
3783 ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3784                    u_int bus_width)
3785 {
3786         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3787         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR_LEN;
3788         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_WDTR;
3789         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
3790         ahd->msgout_len += 4;
3791         if (bootverbose) {
3792                 printf("(%s:%c:%d:%d): Sending WDTR %x\n",
3793                        ahd_name(ahd), devinfo->channel, devinfo->target,
3794                        devinfo->lun, bus_width);
3795         }
3796 }
3797
3798 /*
3799  * Build a parallel protocol request message in our message
3800  * buffer based on the input parameters.
3801  */
3802 static void
3803 ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3804                   u_int period, u_int offset, u_int bus_width,
3805                   u_int ppr_options)
3806 {
3807         /*
3808          * Always request precompensation from
3809          * the other target if we are running
3810          * at paced syncrates.
3811          */
3812         if (period <= AHD_SYNCRATE_PACED)
3813                 ppr_options |= MSG_EXT_PPR_PCOMP_EN;
3814         if (offset == 0)
3815                 period = AHD_ASYNC_XFER_PERIOD;
3816         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXTENDED;
3817         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR_LEN;
3818         ahd->msgout_buf[ahd->msgout_index++] = MSG_EXT_PPR;
3819         ahd->msgout_buf[ahd->msgout_index++] = period;
3820         ahd->msgout_buf[ahd->msgout_index++] = 0;
3821         ahd->msgout_buf[ahd->msgout_index++] = offset;
3822         ahd->msgout_buf[ahd->msgout_index++] = bus_width;
3823         ahd->msgout_buf[ahd->msgout_index++] = ppr_options;
3824         ahd->msgout_len += 8;
3825         if (bootverbose) {
3826                 printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3827                        "offset %x, ppr_options %x\n", ahd_name(ahd),
3828                        devinfo->channel, devinfo->target, devinfo->lun,
3829                        bus_width, period, offset, ppr_options);
3830         }
3831 }
3832
3833 /*
3834  * Clear any active message state.
3835  */
3836 static void
3837 ahd_clear_msg_state(struct ahd_softc *ahd)
3838 {
3839         ahd_mode_state saved_modes;
3840
3841         saved_modes = ahd_save_modes(ahd);
3842         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3843         ahd->send_msg_perror = 0;
3844         ahd->msg_flags = MSG_FLAG_NONE;
3845         ahd->msgout_len = 0;
3846         ahd->msgin_index = 0;
3847         ahd->msg_type = MSG_TYPE_NONE;
3848         if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
3849                 /*
3850                  * The target didn't care to respond to our
3851                  * message request, so clear ATN.
3852                  */
3853                 ahd_outb(ahd, CLRSINT1, CLRATNO);
3854         }
3855         ahd_outb(ahd, MSG_OUT, MSG_NOOP);
3856         ahd_outb(ahd, SEQ_FLAGS2,
3857                  ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3858         ahd_restore_modes(ahd, saved_modes);
3859 }
3860
3861 /*
3862  * Manual message loop handler.
3863  */
3864 static void
3865 ahd_handle_message_phase(struct ahd_softc *ahd)
3866 {
3867         struct  ahd_devinfo devinfo;
3868         u_int   bus_phase;
3869         int     end_session;
3870
3871         ahd_fetch_devinfo(ahd, &devinfo);
3872         end_session = FALSE;
3873         bus_phase = ahd_inb(ahd, LASTPHASE);
3874
3875         if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
3876                 printf("LQIRETRY for LQIPHASE_OUTPKT\n");
3877                 ahd_outb(ahd, LQCTL2, LQIRETRY);
3878         }
3879 reswitch:
3880         switch (ahd->msg_type) {
3881         case MSG_TYPE_INITIATOR_MSGOUT:
3882         {
3883                 int lastbyte;
3884                 int phasemis;
3885                 int msgdone;
3886
3887                 if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
3888                         panic("HOST_MSG_LOOP interrupt with no active message");
3889
3890 #ifdef AHD_DEBUG
3891                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3892                         ahd_print_devinfo(ahd, &devinfo);
3893                         printf("INITIATOR_MSG_OUT");
3894                 }
3895 #endif
3896                 phasemis = bus_phase != P_MESGOUT;
3897                 if (phasemis) {
3898 #ifdef AHD_DEBUG
3899                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3900                                 printf(" PHASEMIS %s\n",
3901                                        ahd_lookup_phase_entry(bus_phase)
3902                                                              ->phasemsg);
3903                         }
3904 #endif
3905                         if (bus_phase == P_MESGIN) {
3906                                 /*
3907                                  * Change gears and see if
3908                                  * this messages is of interest to
3909                                  * us or should be passed back to
3910                                  * the sequencer.
3911                                  */
3912                                 ahd_outb(ahd, CLRSINT1, CLRATNO);
3913                                 ahd->send_msg_perror = 0;
3914                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3915                                 ahd->msgin_index = 0;
3916                                 goto reswitch;
3917                         }
3918                         end_session = TRUE;
3919                         break;
3920                 }
3921
3922                 if (ahd->send_msg_perror) {
3923                         ahd_outb(ahd, CLRSINT1, CLRATNO);
3924                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
3925 #ifdef AHD_DEBUG
3926                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3927                                 printf(" byte 0x%x\n", ahd->send_msg_perror);
3928 #endif
3929                         /*
3930                          * If we are notifying the target of a CRC error
3931                          * during packetized operations, the target is
3932                          * within its rights to acknowledge our message
3933                          * with a busfree.
3934                          */
3935                         if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
3936                          && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
3937                                 ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
3938
3939                         ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
3940                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
3941                         break;
3942                 }
3943
3944                 msgdone = ahd->msgout_index == ahd->msgout_len;
3945                 if (msgdone) {
3946                         /*
3947                          * The target has requested a retry.
3948                          * Re-assert ATN, reset our message index to
3949                          * 0, and try again.
3950                          */
3951                         ahd->msgout_index = 0;
3952                         ahd_assert_atn(ahd);
3953                 }
3954
3955                 lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
3956                 if (lastbyte) {
3957                         /* Last byte is signified by dropping ATN */
3958                         ahd_outb(ahd, CLRSINT1, CLRATNO);
3959                 }
3960
3961                 /*
3962                  * Clear our interrupt status and present
3963                  * the next byte on the bus.
3964                  */
3965                 ahd_outb(ahd, CLRSINT1, CLRREQINIT);
3966 #ifdef AHD_DEBUG
3967                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3968                         printf(" byte 0x%x\n",
3969                                ahd->msgout_buf[ahd->msgout_index]);
3970 #endif
3971                 ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
3972                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
3973                 break;
3974         }
3975         case MSG_TYPE_INITIATOR_MSGIN:
3976         {
3977                 int phasemis;
3978                 int message_done;
3979
3980 #ifdef AHD_DEBUG
3981                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3982                         ahd_print_devinfo(ahd, &devinfo);
3983                         printf("INITIATOR_MSG_IN");
3984                 }
3985 #endif
3986                 phasemis = bus_phase != P_MESGIN;
3987                 if (phasemis) {
3988 #ifdef AHD_DEBUG
3989                         if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3990                                 printf(" PHASEMIS %s\n",
3991                                        ahd_lookup_phase_entry(bus_phase)
3992                                                              ->phasemsg);
3993                         }
3994 #endif
3995                         ahd->msgin_index = 0;
3996                         if (bus_phase == P_MESGOUT
3997                          && (ahd->send_msg_perror != 0
3998                           || (ahd->msgout_len != 0
3999                            && ahd->msgout_index == 0))) {
4000                                 ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4001                                 goto reswitch;
4002                         }
4003                         end_session = TRUE;
4004                         break;
4005                 }
4006
4007                 /* Pull the byte in without acking it */
4008                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
4009 #ifdef AHD_DEBUG
4010                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4011                         printf(" byte 0x%x\n",
4012                                ahd->msgin_buf[ahd->msgin_index]);
4013 #endif
4014
4015                 message_done = ahd_parse_msg(ahd, &devinfo);
4016
4017                 if (message_done) {
4018                         /*
4019                          * Clear our incoming message buffer in case there
4020                          * is another message following this one.
4021                          */
4022                         ahd->msgin_index = 0;
4023
4024                         /*
4025                          * If this message illicited a response,
4026                          * assert ATN so the target takes us to the
4027                          * message out phase.
4028                          */
4029                         if (ahd->msgout_len != 0) {
4030 #ifdef AHD_DEBUG
4031                                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4032                                         ahd_print_devinfo(ahd, &devinfo);
4033                                         printf("Asserting ATN for response\n");
4034                                 }
4035 #endif
4036                                 ahd_assert_atn(ahd);
4037                         }
4038                 } else
4039                         ahd->msgin_index++;
4040
4041                 if (message_done == MSGLOOP_TERMINATED) {
4042                         end_session = TRUE;
4043                 } else {
4044                         /* Ack the byte */
4045                         ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4046                         ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
4047                 }
4048                 break;
4049         }
4050         case MSG_TYPE_TARGET_MSGIN:
4051         {
4052                 int msgdone;
4053                 int msgout_request;
4054
4055                 /*
4056                  * By default, the message loop will continue.
4057                  */
4058                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4059
4060                 if (ahd->msgout_len == 0)
4061                         panic("Target MSGIN with no active message");
4062
4063                 /*
4064                  * If we interrupted a mesgout session, the initiator
4065                  * will not know this until our first REQ.  So, we
4066                  * only honor mesgout requests after we've sent our
4067                  * first byte.
4068                  */
4069                 if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
4070                  && ahd->msgout_index > 0)
4071                         msgout_request = TRUE;
4072                 else
4073                         msgout_request = FALSE;
4074
4075                 if (msgout_request) {
4076
4077                         /*
4078                          * Change gears and see if
4079                          * this messages is of interest to
4080                          * us or should be passed back to
4081                          * the sequencer.
4082                          */
4083                         ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
4084                         ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
4085                         ahd->msgin_index = 0;
4086                         /* Dummy read to REQ for first byte */
4087                         ahd_inb(ahd, SCSIDAT);
4088                         ahd_outb(ahd, SXFRCTL0,
4089                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4090                         break;
4091                 }
4092
4093                 msgdone = ahd->msgout_index == ahd->msgout_len;
4094                 if (msgdone) {
4095                         ahd_outb(ahd, SXFRCTL0,
4096                                  ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4097                         end_session = TRUE;
4098                         break;
4099                 }
4100
4101                 /*
4102                  * Present the next byte on the bus.
4103                  */
4104                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4105                 ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
4106                 break;
4107         }
4108         case MSG_TYPE_TARGET_MSGOUT:
4109         {
4110                 int lastbyte;
4111                 int msgdone;
4112
4113                 /*
4114                  * By default, the message loop will continue.
4115                  */
4116                 ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4117
4118                 /*
4119                  * The initiator signals that this is
4120                  * the last byte by dropping ATN.
4121                  */
4122                 lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
4123
4124                 /*
4125                  * Read the latched byte, but turn off SPIOEN first
4126                  * so that we don't inadvertently cause a REQ for the
4127                  * next byte.
4128                  */
4129                 ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4130                 ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
4131                 msgdone = ahd_parse_msg(ahd, &devinfo);
4132                 if (msgdone == MSGLOOP_TERMINATED) {
4133                         /*
4134                          * The message is *really* done in that it caused
4135                          * us to go to bus free.  The sequencer has already
4136                          * been reset at this point, so pull the ejection
4137                          * handle.
4138                          */
4139                         return;
4140                 }
4141
4142                 ahd->msgin_index++;
4143
4144                 /*
4145                  * XXX Read spec about initiator dropping ATN too soon
4146                  *     and use msgdone to detect it.
4147                  */
4148                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
4149                         ahd->msgin_index = 0;
4150
4151                         /*
4152                          * If this message illicited a response, transition
4153                          * to the Message in phase and send it.
4154                          */
4155                         if (ahd->msgout_len != 0) {
4156                                 ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
4157                                 ahd_outb(ahd, SXFRCTL0,
4158                                          ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4159                                 ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
4160                                 ahd->msgin_index = 0;
4161                                 break;
4162                         }
4163                 }
4164
4165                 if (lastbyte)
4166                         end_session = TRUE;
4167                 else {
4168                         /* Ask for the next byte. */
4169                         ahd_outb(ahd, SXFRCTL0,
4170                                  ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4171                 }
4172
4173                 break;
4174         }
4175         default:
4176                 panic("Unknown REQINIT message type");
4177         }
4178
4179         if (end_session) {
4180                 if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
4181                         printf("%s: Returning to Idle Loop\n",
4182                                ahd_name(ahd));
4183                         ahd_clear_msg_state(ahd);
4184
4185                         /*
4186                          * Perform the equivalent of a clear_target_state.
4187                          */
4188                         ahd_outb(ahd, LASTPHASE, P_BUSFREE);
4189                         ahd_outb(ahd, SEQ_FLAGS, NOT_IDENTIFIED|NO_CDB_SENT);
4190                         ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
4191                 } else {
4192                         ahd_clear_msg_state(ahd);
4193                         ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
4194                 }
4195         }
4196 }
4197
4198 /*
4199  * See if we sent a particular extended message to the target.
4200  * If "full" is true, return true only if the target saw the full
4201  * message.  If "full" is false, return true if the target saw at
4202  * least the first byte of the message.
4203  */
4204 static int
4205 ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
4206 {
4207         int found;
4208         u_int index;
4209
4210         found = FALSE;
4211         index = 0;
4212
4213         while (index < ahd->msgout_len) {
4214                 if (ahd->msgout_buf[index] == MSG_EXTENDED) {
4215                         u_int end_index;
4216
4217                         end_index = index + 1 + ahd->msgout_buf[index + 1];
4218                         if (ahd->msgout_buf[index+2] == msgval
4219                          && type == AHDMSG_EXT) {
4220
4221                                 if (full) {
4222                                         if (ahd->msgout_index > end_index)
4223                                                 found = TRUE;
4224                                 } else if (ahd->msgout_index > index)
4225                                         found = TRUE;
4226                         }
4227                         index = end_index;
4228                 } else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
4229                         && ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
4230
4231                         /* Skip tag type and tag id or residue param*/
4232                         index += 2;
4233                 } else {
4234                         /* Single byte message */
4235                         if (type == AHDMSG_1B
4236                          && ahd->msgout_index > index
4237                          && (ahd->msgout_buf[index] == msgval
4238                           || ((ahd->msgout_buf[index] & MSG_IDENTIFYFLAG) != 0
4239                            && msgval == MSG_IDENTIFYFLAG)))
4240                                 found = TRUE;
4241                         index++;
4242                 }
4243
4244                 if (found)
4245                         break;
4246         }
4247         return (found);
4248 }
4249
4250 /*
4251  * Wait for a complete incoming message, parse it, and respond accordingly.
4252  */
4253 static int
4254 ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4255 {
4256         struct  ahd_initiator_tinfo *tinfo;
4257         struct  ahd_tmode_tstate *tstate;
4258         int     reject;
4259         int     done;
4260         int     response;
4261
4262         done = MSGLOOP_IN_PROG;
4263         response = FALSE;
4264         reject = FALSE;
4265         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4266                                     devinfo->target, &tstate);
4267
4268         /*
4269          * Parse as much of the message as is available,
4270          * rejecting it if we don't support it.  When
4271          * the entire message is available and has been
4272          * handled, return MSGLOOP_MSGCOMPLETE, indicating
4273          * that we have parsed an entire message.
4274          *
4275          * In the case of extended messages, we accept the length
4276          * byte outright and perform more checking once we know the
4277          * extended message type.
4278          */
4279         switch (ahd->msgin_buf[0]) {
4280         case MSG_DISCONNECT:
4281         case MSG_SAVEDATAPOINTER:
4282         case MSG_CMDCOMPLETE:
4283         case MSG_RESTOREPOINTERS:
4284         case MSG_IGN_WIDE_RESIDUE:
4285                 /*
4286                  * End our message loop as these are messages
4287                  * the sequencer handles on its own.
4288                  */
4289                 done = MSGLOOP_TERMINATED;
4290                 break;
4291         case MSG_MESSAGE_REJECT:
4292                 response = ahd_handle_msg_reject(ahd, devinfo);
4293                 /* FALLTHROUGH */
4294         case MSG_NOOP:
4295                 done = MSGLOOP_MSGCOMPLETE;
4296                 break;
4297         case MSG_EXTENDED:
4298         {
4299                 /* Wait for enough of the message to begin validation */
4300                 if (ahd->msgin_index < 2)
4301                         break;
4302                 switch (ahd->msgin_buf[2]) {
4303                 case MSG_EXT_SDTR:
4304                 {
4305                         u_int    period;
4306                         u_int    ppr_options;
4307                         u_int    offset;
4308                         u_int    saved_offset;
4309
4310                         if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
4311                                 reject = TRUE;
4312                                 break;
4313                         }
4314
4315                         /*
4316                          * Wait until we have both args before validating
4317                          * and acting on this message.
4318                          *
4319                          * Add one to MSG_EXT_SDTR_LEN to account for
4320                          * the extended message preamble.
4321                          */
4322                         if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
4323                                 break;
4324
4325                         period = ahd->msgin_buf[3];
4326                         ppr_options = 0;
4327                         saved_offset = offset = ahd->msgin_buf[4];
4328                         ahd_devlimited_syncrate(ahd, tinfo, &period,
4329                                                 &ppr_options, devinfo->role);
4330                         ahd_validate_offset(ahd, tinfo, period, &offset,
4331                                             tinfo->curr.width, devinfo->role);
4332                         if (bootverbose) {
4333                                 printf("(%s:%c:%d:%d): Received "
4334                                        "SDTR period %x, offset %x\n\t"
4335                                        "Filtered to period %x, offset %x\n",
4336                                        ahd_name(ahd), devinfo->channel,
4337                                        devinfo->target, devinfo->lun,
4338                                        ahd->msgin_buf[3], saved_offset,
4339                                        period, offset);
4340                         }
4341                         ahd_set_syncrate(ahd, devinfo, period,
4342                                          offset, ppr_options,
4343                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4344                                          /*paused*/TRUE);
4345
4346                         /*
4347                          * See if we initiated Sync Negotiation
4348                          * and didn't have to fall down to async
4349                          * transfers.
4350                          */
4351                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
4352                                 /* We started it */
4353                                 if (saved_offset != offset) {
4354                                         /* Went too low - force async */
4355                                         reject = TRUE;
4356                                 }
4357                         } else {
4358                                 /*
4359                                  * Send our own SDTR in reply
4360                                  */
4361                                 if (bootverbose
4362                                  && devinfo->role == ROLE_INITIATOR) {
4363                                         printf("(%s:%c:%d:%d): Target "
4364                                                "Initiated SDTR\n",
4365                                                ahd_name(ahd), devinfo->channel,
4366                                                devinfo->target, devinfo->lun);
4367                                 }
4368                                 ahd->msgout_index = 0;
4369                                 ahd->msgout_len = 0;
4370                                 ahd_construct_sdtr(ahd, devinfo,
4371                                                    period, offset);
4372                                 ahd->msgout_index = 0;
4373                                 response = TRUE;
4374                         }
4375                         done = MSGLOOP_MSGCOMPLETE;
4376                         break;
4377                 }
4378                 case MSG_EXT_WDTR:
4379                 {
4380                         u_int bus_width;
4381                         u_int saved_width;
4382                         u_int sending_reply;
4383
4384                         sending_reply = FALSE;
4385                         if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
4386                                 reject = TRUE;
4387                                 break;
4388                         }
4389
4390                         /*
4391                          * Wait until we have our arg before validating
4392                          * and acting on this message.
4393                          *
4394                          * Add one to MSG_EXT_WDTR_LEN to account for
4395                          * the extended message preamble.
4396                          */
4397                         if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
4398                                 break;
4399
4400                         bus_width = ahd->msgin_buf[3];
4401                         saved_width = bus_width;
4402                         ahd_validate_width(ahd, tinfo, &bus_width,
4403                                            devinfo->role);
4404                         if (bootverbose) {
4405                                 printf("(%s:%c:%d:%d): Received WDTR "
4406                                        "%x filtered to %x\n",
4407                                        ahd_name(ahd), devinfo->channel,
4408                                        devinfo->target, devinfo->lun,
4409                                        saved_width, bus_width);
4410                         }
4411
4412                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
4413                                 /*
4414                                  * Don't send a WDTR back to the
4415                                  * target, since we asked first.
4416                                  * If the width went higher than our
4417                                  * request, reject it.
4418                                  */
4419                                 if (saved_width > bus_width) {
4420                                         reject = TRUE;
4421                                         printf("(%s:%c:%d:%d): requested %dBit "
4422                                                "transfers.  Rejecting...\n",
4423                                                ahd_name(ahd), devinfo->channel,
4424                                                devinfo->target, devinfo->lun,
4425                                                8 * (0x01 << bus_width));
4426                                         bus_width = 0;
4427                                 }
4428                         } else {
4429                                 /*
4430                                  * Send our own WDTR in reply
4431                                  */
4432                                 if (bootverbose
4433                                  && devinfo->role == ROLE_INITIATOR) {
4434                                         printf("(%s:%c:%d:%d): Target "
4435                                                "Initiated WDTR\n",
4436                                                ahd_name(ahd), devinfo->channel,
4437                                                devinfo->target, devinfo->lun);
4438                                 }
4439                                 ahd->msgout_index = 0;
4440                                 ahd->msgout_len = 0;
4441                                 ahd_construct_wdtr(ahd, devinfo, bus_width);
4442                                 ahd->msgout_index = 0;
4443                                 response = TRUE;
4444                                 sending_reply = TRUE;
4445                         }
4446                         /*
4447                          * After a wide message, we are async, but
4448                          * some devices don't seem to honor this portion
4449                          * of the spec.  Force a renegotiation of the
4450                          * sync component of our transfer agreement even
4451                          * if our goal is async.  By updating our width
4452                          * after forcing the negotiation, we avoid
4453                          * renegotiating for width.
4454                          */
4455                         ahd_update_neg_request(ahd, devinfo, tstate,
4456                                                tinfo, AHD_NEG_ALWAYS);
4457                         ahd_set_width(ahd, devinfo, bus_width,
4458                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4459                                       /*paused*/TRUE);
4460                         if (sending_reply == FALSE && reject == FALSE) {
4461
4462                                 /*
4463                                  * We will always have an SDTR to send.
4464                                  */
4465                                 ahd->msgout_index = 0;
4466                                 ahd->msgout_len = 0;
4467                                 ahd_build_transfer_msg(ahd, devinfo);
4468                                 ahd->msgout_index = 0;
4469                                 response = TRUE;
4470                         }
4471                         done = MSGLOOP_MSGCOMPLETE;
4472                         break;
4473                 }
4474                 case MSG_EXT_PPR:
4475                 {
4476                         u_int   period;
4477                         u_int   offset;
4478                         u_int   bus_width;
4479                         u_int   ppr_options;
4480                         u_int   saved_width;
4481                         u_int   saved_offset;
4482                         u_int   saved_ppr_options;
4483
4484                         if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
4485                                 reject = TRUE;
4486                                 break;
4487                         }
4488
4489                         /*
4490                          * Wait until we have all args before validating
4491                          * and acting on this message.
4492                          *
4493                          * Add one to MSG_EXT_PPR_LEN to account for
4494                          * the extended message preamble.
4495                          */
4496                         if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
4497                                 break;
4498
4499                         period = ahd->msgin_buf[3];
4500                         offset = ahd->msgin_buf[5];
4501                         bus_width = ahd->msgin_buf[6];
4502                         saved_width = bus_width;
4503                         ppr_options = ahd->msgin_buf[7];
4504                         /*
4505                          * According to the spec, a DT only
4506                          * period factor with no DT option
4507                          * set implies async.
4508                          */
4509                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
4510                          && period <= 9)
4511                                 offset = 0;
4512                         saved_ppr_options = ppr_options;
4513                         saved_offset = offset;
4514
4515                         /*
4516                          * Transfer options are only available if we
4517                          * are negotiating wide.
4518                          */
4519                         if (bus_width == 0)
4520                                 ppr_options &= MSG_EXT_PPR_QAS_REQ;
4521
4522                         ahd_validate_width(ahd, tinfo, &bus_width,
4523                                            devinfo->role);
4524                         ahd_devlimited_syncrate(ahd, tinfo, &period,
4525                                                 &ppr_options, devinfo->role);
4526                         ahd_validate_offset(ahd, tinfo, period, &offset,
4527                                             bus_width, devinfo->role);
4528
4529                         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
4530                                 /*
4531                                  * If we are unable to do any of the
4532                                  * requested options (we went too low),
4533                                  * then we'll have to reject the message.
4534                                  */
4535                                 if (saved_width > bus_width
4536                                  || saved_offset != offset
4537                                  || saved_ppr_options != ppr_options) {
4538                                         reject = TRUE;
4539                                         period = 0;
4540                                         offset = 0;
4541                                         bus_width = 0;
4542                                         ppr_options = 0;
4543                                 }
4544                         } else {
4545                                 if (devinfo->role != ROLE_TARGET)
4546                                         printf("(%s:%c:%d:%d): Target "
4547                                                "Initiated PPR\n",
4548                                                ahd_name(ahd), devinfo->channel,
4549                                                devinfo->target, devinfo->lun);
4550                                 else
4551                                         printf("(%s:%c:%d:%d): Initiator "
4552                                                "Initiated PPR\n",
4553                                                ahd_name(ahd), devinfo->channel,
4554                                                devinfo->target, devinfo->lun);
4555                                 ahd->msgout_index = 0;
4556                                 ahd->msgout_len = 0;
4557                                 ahd_construct_ppr(ahd, devinfo, period, offset,
4558                                                   bus_width, ppr_options);
4559                                 ahd->msgout_index = 0;
4560                                 response = TRUE;
4561                         }
4562                         if (bootverbose) {
4563                                 printf("(%s:%c:%d:%d): Received PPR width %x, "
4564                                        "period %x, offset %x,options %x\n"
4565                                        "\tFiltered to width %x, period %x, "
4566                                        "offset %x, options %x\n",
4567                                        ahd_name(ahd), devinfo->channel,
4568                                        devinfo->target, devinfo->lun,
4569                                        saved_width, ahd->msgin_buf[3],
4570                                        saved_offset, saved_ppr_options,
4571                                        bus_width, period, offset, ppr_options);
4572                         }
4573                         ahd_set_width(ahd, devinfo, bus_width,
4574                                       AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4575                                       /*paused*/TRUE);
4576                         ahd_set_syncrate(ahd, devinfo, period,
4577                                          offset, ppr_options,
4578                                          AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4579                                          /*paused*/TRUE);
4580
4581                         done = MSGLOOP_MSGCOMPLETE;
4582                         break;
4583                 }
4584                 default:
4585                         /* Unknown extended message.  Reject it. */
4586                         reject = TRUE;
4587                         break;
4588                 }
4589                 break;
4590         }
4591 #ifdef AHD_TARGET_MODE
4592         case MSG_BUS_DEV_RESET:
4593                 ahd_handle_devreset(ahd, devinfo, CAM_LUN_WILDCARD,
4594                                     CAM_BDR_SENT,
4595                                     "Bus Device Reset Received",
4596                                     /*verbose_level*/0);
4597                 ahd_restart(ahd);
4598                 done = MSGLOOP_TERMINATED;
4599                 break;
4600         case MSG_ABORT_TAG:
4601         case MSG_ABORT:
4602         case MSG_CLEAR_QUEUE:
4603         {
4604                 int tag;
4605
4606                 /* Target mode messages */
4607                 if (devinfo->role != ROLE_TARGET) {
4608                         reject = TRUE;
4609                         break;
4610                 }
4611                 tag = SCB_LIST_NULL;
4612                 if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
4613                         tag = ahd_inb(ahd, INITIATOR_TAG);
4614                 ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
4615                                devinfo->lun, tag, ROLE_TARGET,
4616                                CAM_REQ_ABORTED);
4617
4618                 tstate = ahd->enabled_targets[devinfo->our_scsiid];
4619                 if (tstate != NULL) {
4620                         struct ahd_tmode_lstate* lstate;
4621
4622                         lstate = tstate->enabled_luns[devinfo->lun];
4623                         if (lstate != NULL) {
4624                                 ahd_queue_lstate_event(ahd, lstate,
4625                                                        devinfo->our_scsiid,
4626                                                        ahd->msgin_buf[0],
4627                                                        /*arg*/tag);
4628                                 ahd_send_lstate_events(ahd, lstate);
4629                         }
4630                 }
4631                 ahd_restart(ahd);
4632                 done = MSGLOOP_TERMINATED;
4633                 break;
4634         }
4635 #endif
4636         case MSG_QAS_REQUEST:
4637 #ifdef AHD_DEBUG
4638                 if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4639                         printf("%s: QAS request.  SCSISIGI == 0x%x\n",
4640                                ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
4641 #endif
4642                 ahd->msg_flags |= MSG_FLAG_EXPECT_QASREJ_BUSFREE;
4643                 /* FALLTHROUGH */
4644         case MSG_TERM_IO_PROC:
4645         default:
4646                 reject = TRUE;
4647                 break;
4648         }
4649
4650         if (reject) {
4651                 /*
4652                  * Setup to reject the message.
4653                  */
4654                 ahd->msgout_index = 0;
4655                 ahd->msgout_len = 1;
4656                 ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
4657                 done = MSGLOOP_MSGCOMPLETE;
4658                 response = TRUE;
4659         }
4660
4661         if (done != MSGLOOP_IN_PROG && !response)
4662                 /* Clear the outgoing message buffer */
4663                 ahd->msgout_len = 0;
4664
4665         return (done);
4666 }
4667
4668 /*
4669  * Process a message reject message.
4670  */
4671 static int
4672 ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4673 {
4674         /*
4675          * What we care about here is if we had an
4676          * outstanding SDTR or WDTR message for this
4677          * target.  If we did, this is a signal that
4678          * the target is refusing negotiation.
4679          */
4680         struct scb *scb;
4681         struct ahd_initiator_tinfo *tinfo;
4682         struct ahd_tmode_tstate *tstate;
4683         u_int scb_index;
4684         u_int last_msg;
4685         int   response = 0;
4686
4687         scb_index = ahd_get_scbptr(ahd);
4688         scb = ahd_lookup_scb(ahd, scb_index);
4689         tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
4690                                     devinfo->our_scsiid,
4691                                     devinfo->target, &tstate);
4692         /* Might be necessary */
4693         last_msg = ahd_inb(ahd, LAST_MSG);
4694
4695         if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
4696                 if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/TRUE)
4697                  && tinfo->goal.period <= AHD_SYNCRATE_PACED) {
4698                         /*
4699                          * Target may not like our SPI-4 PPR Options.
4700                          * Attempt to negotiate 80MHz which will turn
4701                          * off these options.
4702                          */
4703                         if (bootverbose) {
4704                                 printf("(%s:%c:%d:%d): PPR Rejected. "
4705                                        "Trying simple U160 PPR\n",
4706                                        ahd_name(ahd), devinfo->channel,
4707                                        devinfo->target, devinfo->lun);
4708                         }
4709                         tinfo->goal.period = AHD_SYNCRATE_DT;
4710                         tinfo->goal.ppr_options &= MSG_EXT_PPR_IU_REQ
4711                                                 |  MSG_EXT_PPR_QAS_REQ
4712                                                 |  MSG_EXT_PPR_DT_REQ;
4713                 } else {
4714                         /*
4715                          * Target does not support the PPR message.
4716                          * Attempt to negotiate SPI-2 style.
4717                          */
4718                         if (bootverbose) {
4719                                 printf("(%s:%c:%d:%d): PPR Rejected. "
4720                                        "Trying WDTR/SDTR\n",
4721                                        ahd_name(ahd), devinfo->channel,
4722                                        devinfo->target, devinfo->lun);
4723                         }
4724                         tinfo->goal.ppr_options = 0;
4725                         tinfo->curr.transport_version = 2;
4726                         tinfo->goal.transport_version = 2;
4727                 }
4728                 ahd->msgout_index = 0;
4729                 ahd->msgout_len = 0;
4730                 ahd_build_transfer_msg(ahd, devinfo);
4731                 ahd->msgout_index = 0;
4732                 response = 1;
4733         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
4734
4735                 /* note 8bit xfers */
4736                 printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
4737                        "8bit transfers\n", ahd_name(ahd),
4738                        devinfo->channel, devinfo->target, devinfo->lun);
4739                 ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4740                               AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4741                               /*paused*/TRUE);
4742                 /*
4743                  * No need to clear the sync rate.  If the target
4744                  * did not accept the command, our syncrate is
4745                  * unaffected.  If the target started the negotiation,
4746                  * but rejected our response, we already cleared the
4747                  * sync rate before sending our WDTR.
4748                  */
4749                 if (tinfo->goal.offset != tinfo->curr.offset) {
4750
4751                         /* Start the sync negotiation */
4752                         ahd->msgout_index = 0;
4753                         ahd->msgout_len = 0;
4754                         ahd_build_transfer_msg(ahd, devinfo);
4755                         ahd->msgout_index = 0;
4756                         response = 1;
4757                 }
4758         } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
4759                 /* note asynch xfers and clear flag */
4760                 ahd_set_syncrate(ahd, devinfo, /*period*/0,
4761                                  /*offset*/0, /*ppr_options*/0,
4762                                  AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
4763                                  /*paused*/TRUE);
4764                 printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
4765                        "Using asynchronous transfers\n",
4766                        ahd_name(ahd), devinfo->channel,
4767                        devinfo->target, devinfo->lun);
4768         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
4769                 int tag_type;
4770                 int mask;
4771
4772                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
4773
4774                 if (tag_type == MSG_SIMPLE_TASK) {
4775                         printf("(%s:%c:%d:%d): refuses tagged commands.  "
4776                                "Performing non-tagged I/O\n", ahd_name(ahd),
4777                                devinfo->channel, devinfo->target, devinfo->lun);
4778                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_NONE);
4779                         mask = ~0x23;
4780                 } else {
4781                         printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
4782                                "Performing simple queue tagged I/O only\n",
4783                                ahd_name(ahd), devinfo->channel, devinfo->target,
4784                                devinfo->lun, tag_type == MSG_ORDERED_TASK
4785                                ? "ordered" : "head of queue");
4786                         ahd_set_tags(ahd, devinfo, AHD_QUEUE_BASIC);
4787                         mask = ~0x03;
4788                 }
4789
4790                 /*
4791                  * Resend the identify for this CCB as the target
4792                  * may believe that the selection is invalid otherwise.
4793                  */
4794                 ahd_outb(ahd, SCB_CONTROL,
4795                          ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
4796                 scb->hscb->control &= mask;
4797                 ahd_set_transaction_tag(scb, /*enabled*/FALSE,
4798                                         /*type*/MSG_SIMPLE_TASK);
4799                 ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
4800                 ahd_assert_atn(ahd);
4801                 ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
4802                              SCB_GET_TAG(scb));
4803
4804                 /*
4805                  * Requeue all tagged commands for this target
4806                  * currently in our posession so they can be
4807                  * converted to untagged commands.
4808                  */
4809                 ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
4810                                    SCB_GET_CHANNEL(ahd, scb),
4811                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
4812                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
4813                                    SEARCH_COMPLETE);
4814         } else if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_IDENTIFYFLAG, TRUE)) {
4815                 /*
4816                  * Most likely the device believes that we had
4817                  * previously negotiated packetized.
4818                  */
4819                 ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
4820                                |  MSG_FLAG_IU_REQ_CHANGED;
4821
4822                 ahd_force_renegotiation(ahd, devinfo);
4823                 ahd->msgout_index = 0;
4824                 ahd->msgout_len = 0;
4825                 ahd_build_transfer_msg(ahd, devinfo);
4826                 ahd->msgout_index = 0;
4827                 response = 1;
4828         } else {
4829                 /*
4830                  * Otherwise, we ignore it.
4831                  */
4832                 printf("%s:%c:%d: Message reject for %x -- ignored\n",
4833                        ahd_name(ahd), devinfo->channel, devinfo->target,
4834                        last_msg);
4835         }
4836         return (response);
4837 }
4838
4839 /*
4840  * Process an ingnore wide residue message.
4841  */
4842 static void
4843 ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4844 {
4845         u_int scb_index;
4846         struct scb *scb;
4847
4848         scb_index = ahd_get_scbptr(ahd);
4849         scb = ahd_lookup_scb(ahd, scb_index);
4850         /*
4851          * XXX Actually check data direction in the sequencer?
4852          * Perhaps add datadir to some spare bits in the hscb?
4853          */
4854         if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
4855          || ahd_get_transfer_dir(scb) != CAM_DIR_IN) {
4856                 /*
4857                  * Ignore the message if we haven't
4858                  * seen an appropriate data phase yet.
4859                  */
4860         } else {
4861                 /*
4862                  * If the residual occurred on the last
4863                  * transfer and the transfer request was
4864                  * expected to end on an odd count, do
4865                  * nothing.  Otherwise, subtract a byte
4866                  * and update the residual count accordingly.
4867                  */
4868                 uint32_t sgptr;
4869
4870                 sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
4871                 if ((sgptr & SG_LIST_NULL) != 0
4872                  && (ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
4873                      & SCB_XFERLEN_ODD) != 0) {
4874                         /*
4875                          * If the residual occurred on the last
4876                          * transfer and the transfer request was
4877                          * expected to end on an odd count, do
4878                          * nothing.
4879                          */
4880                 } else {
4881                         uint32_t data_cnt;
4882                         uint64_t data_addr;
4883                         uint32_t sglen;
4884
4885                         /* Pull in the rest of the sgptr */
4886                         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
4887                         data_cnt = ahd_inl_scbram(ahd, SCB_RESIDUAL_DATACNT);
4888                         if ((sgptr & SG_LIST_NULL) != 0) {
4889                                 /*
4890                                  * The residual data count is not updated
4891                                  * for the command run to completion case.
4892                                  * Explicitly zero the count.
4893                                  */
4894                                 data_cnt &= ~AHD_SG_LEN_MASK;
4895                         }
4896                         data_addr = ahd_inq(ahd, SHADDR);
4897                         data_cnt += 1;
4898                         data_addr -= 1;
4899                         sgptr &= SG_PTR_MASK;
4900                         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
4901                                 struct ahd_dma64_seg *sg;
4902
4903                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
4904
4905                                 /*
4906                                  * The residual sg ptr points to the next S/G
4907                                  * to load so we must go back one.
4908                                  */
4909                                 sg--;
4910                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
4911                                 if (sg != scb->sg_list
4912                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
4913
4914                                         sg--;
4915                                         sglen = ahd_le32toh(sg->len);
4916                                         /*
4917                                          * Preserve High Address and SG_LIST
4918                                          * bits while setting the count to 1.
4919                                          */
4920                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
4921                                         data_addr = ahd_le64toh(sg->addr)
4922                                                   + (sglen & AHD_SG_LEN_MASK)
4923                                                   - 1;
4924
4925                                         /*
4926                                          * Increment sg so it points to the
4927                                          * "next" sg.
4928                                          */
4929                                         sg++;
4930                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
4931                                                                    sg);
4932                                 }
4933                         } else {
4934                                 struct ahd_dma_seg *sg;
4935
4936                                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
4937
4938                                 /*
4939                                  * The residual sg ptr points to the next S/G
4940                                  * to load so we must go back one.
4941                                  */
4942                                 sg--;
4943                                 sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
4944                                 if (sg != scb->sg_list
4945                                  && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
4946
4947                                         sg--;
4948                                         sglen = ahd_le32toh(sg->len);
4949                                         /*
4950                                          * Preserve High Address and SG_LIST
4951                                          * bits while setting the count to 1.
4952                                          */
4953                                         data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
4954                                         data_addr = ahd_le32toh(sg->addr)
4955                                                   + (sglen & AHD_SG_LEN_MASK)
4956                                                   - 1;
4957
4958                                         /*
4959                                          * Increment sg so it points to the
4960                                          * "next" sg.
4961                                          */
4962                                         sg++;
4963                                         sgptr = ahd_sg_virt_to_bus(ahd, scb,
4964                                                                   sg);
4965                                 }
4966                         }
4967                         /*
4968                          * Toggle the "oddness" of the transfer length
4969                          * to handle this mid-transfer ignore wide
4970                          * residue.  This ensures that the oddness is
4971                          * correct for subsequent data transfers.
4972                          */
4973                         ahd_outb(ahd, SCB_TASK_ATTRIBUTE,
4974                             ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
4975                             ^ SCB_XFERLEN_ODD);
4976
4977                         ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
4978                         ahd_outl(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
4979                         /*
4980                          * The FIFO's pointers will be updated if/when the
4981                          * sequencer re-enters a data phase.
4982                          */
4983                 }
4984         }
4985 }
4986
4987
4988 /*
4989  * Reinitialize the data pointers for the active transfer
4990  * based on its current residual.
4991  */
4992 static void
4993 ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
4994 {
4995         struct           scb *scb;
4996         ahd_mode_state   saved_modes;
4997         u_int            scb_index;
4998         u_int            wait;
4999         uint32_t         sgptr;
5000         uint32_t         resid;
5001         uint64_t         dataptr;
5002
5003         AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
5004                          AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
5005
5006         scb_index = ahd_get_scbptr(ahd);
5007         scb = ahd_lookup_scb(ahd, scb_index);
5008
5009         /*
5010          * Release and reacquire the FIFO so we
5011          * have a clean slate.
5012          */
5013         ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
5014         wait = 1000;
5015         while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE))
5016                 ahd_delay(100);
5017         if (wait == 0) {
5018                 ahd_print_path(ahd, scb);
5019                 printf("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
5020                 ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
5021         }
5022         saved_modes = ahd_save_modes(ahd);
5023         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5024         ahd_outb(ahd, DFFSTAT,
5025                  ahd_inb(ahd, DFFSTAT)
5026                 | (saved_modes == 0x11 ? CURRFIFO_1 : CURRFIFO_0));
5027
5028         /*
5029          * Determine initial values for data_addr and data_cnt
5030          * for resuming the data phase.
5031          */
5032         sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5033         sgptr &= SG_PTR_MASK;
5034
5035         resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
5036               | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
5037               | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
5038
5039         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5040                 struct ahd_dma64_seg *sg;
5041
5042                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5043
5044                 /* The residual sg_ptr always points to the next sg */
5045                 sg--;
5046
5047                 dataptr = ahd_le64toh(sg->addr)
5048                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5049                         - resid;
5050                 ahd_outl(ahd, HADDR + 4, dataptr >> 32);
5051         } else {
5052                 struct   ahd_dma_seg *sg;
5053
5054                 sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5055
5056                 /* The residual sg_ptr always points to the next sg */
5057                 sg--;
5058
5059                 dataptr = ahd_le32toh(sg->addr)
5060                         + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5061                         - resid;
5062                 ahd_outb(ahd, HADDR + 4,
5063                          (ahd_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
5064         }
5065         ahd_outl(ahd, HADDR, dataptr);
5066         ahd_outb(ahd, HCNT + 2, resid >> 16);
5067         ahd_outb(ahd, HCNT + 1, resid >> 8);
5068         ahd_outb(ahd, HCNT, resid);
5069 }
5070
5071 /*
5072  * Handle the effects of issuing a bus device reset message.
5073  */
5074 static void
5075 ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5076                     u_int lun, cam_status status, char *message,
5077                     int verbose_level)
5078 {
5079 #ifdef AHD_TARGET_MODE
5080         struct ahd_tmode_tstate* tstate;
5081 #endif
5082         int found;
5083
5084         found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5085                                lun, SCB_LIST_NULL, devinfo->role,
5086                                status);
5087
5088 #ifdef AHD_TARGET_MODE
5089         /*
5090          * Send an immediate notify ccb to all target mord peripheral
5091          * drivers affected by this action.
5092          */
5093         tstate = ahd->enabled_targets[devinfo->our_scsiid];
5094         if (tstate != NULL) {
5095                 u_int cur_lun;
5096                 u_int max_lun;
5097
5098                 if (lun != CAM_LUN_WILDCARD) {
5099                         cur_lun = 0;
5100                         max_lun = AHD_NUM_LUNS - 1;
5101                 } else {
5102                         cur_lun = lun;
5103                         max_lun = lun;
5104                 }
5105                 for (cur_lun <= max_lun; cur_lun++) {
5106                         struct ahd_tmode_lstate* lstate;
5107
5108                         lstate = tstate->enabled_luns[cur_lun];
5109                         if (lstate == NULL)
5110                                 continue;
5111
5112                         ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
5113                                                MSG_BUS_DEV_RESET, /*arg*/0);
5114                         ahd_send_lstate_events(ahd, lstate);
5115                 }
5116         }
5117 #endif
5118
5119         /*
5120          * Go back to async/narrow transfers and renegotiate.
5121          */
5122         ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5123                       AHD_TRANS_CUR, /*paused*/TRUE);
5124         ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
5125                          /*ppr_options*/0, AHD_TRANS_CUR,
5126                          /*paused*/TRUE);
5127
5128         if (status != CAM_SEL_TIMEOUT)
5129                 ahd_send_async(ahd, devinfo->channel, devinfo->target,
5130                                CAM_LUN_WILDCARD, AC_SENT_BDR, NULL);
5131
5132         if (message != NULL && bootverbose)
5133                 printf("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
5134                        message, devinfo->channel, devinfo->target, found);
5135 }
5136
5137 #ifdef AHD_TARGET_MODE
5138 static void
5139 ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5140                        struct scb *scb)
5141 {
5142
5143         /*
5144          * To facilitate adding multiple messages together,
5145          * each routine should increment the index and len
5146          * variables instead of setting them explicitly.
5147          */
5148         ahd->msgout_index = 0;
5149         ahd->msgout_len = 0;
5150
5151         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
5152                 ahd_build_transfer_msg(ahd, devinfo);
5153         else
5154                 panic("ahd_intr: AWAITING target message with no message");
5155
5156         ahd->msgout_index = 0;
5157         ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
5158 }
5159 #endif
5160 /**************************** Initialization **********************************/
5161 static u_int
5162 ahd_sglist_size(struct ahd_softc *ahd)
5163 {
5164         bus_size_t list_size;
5165
5166         list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
5167         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
5168                 list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
5169         return (list_size);
5170 }
5171
5172 /*
5173  * Calculate the optimum S/G List allocation size.  S/G elements used
5174  * for a given transaction must be physically contiguous.  Assume the
5175  * OS will allocate full pages to us, so it doesn't make sense to request
5176  * less than a page.
5177  */
5178 static u_int
5179 ahd_sglist_allocsize(struct ahd_softc *ahd)
5180 {
5181         bus_size_t sg_list_increment;
5182         bus_size_t sg_list_size;
5183         bus_size_t max_list_size;
5184         bus_size_t best_list_size;
5185
5186         /* Start out with the minimum required for AHD_NSEG. */
5187         sg_list_increment = ahd_sglist_size(ahd);
5188         sg_list_size = sg_list_increment;
5189
5190         /* Get us as close as possible to a page in size. */
5191         while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
5192                 sg_list_size += sg_list_increment;
5193
5194         /*
5195          * Try to reduce the amount of wastage by allocating
5196          * multiple pages.
5197          */
5198         best_list_size = sg_list_size;
5199         max_list_size = roundup(sg_list_increment, PAGE_SIZE);
5200         if (max_list_size < 4 * PAGE_SIZE)
5201                 max_list_size = 4 * PAGE_SIZE;
5202         if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
5203                 max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
5204         while ((sg_list_size + sg_list_increment) <= max_list_size
5205            &&  (sg_list_size % PAGE_SIZE) != 0) {
5206                 bus_size_t new_mod;
5207                 bus_size_t best_mod;
5208
5209                 sg_list_size += sg_list_increment;
5210                 new_mod = sg_list_size % PAGE_SIZE;
5211                 best_mod = best_list_size % PAGE_SIZE;
5212                 if (new_mod > best_mod || new_mod == 0) {
5213                         best_list_size = sg_list_size;
5214                 }
5215         }
5216         return (best_list_size);
5217 }
5218
5219 /*
5220  * Allocate a controller structure for a new device
5221  * and perform initial initializion.
5222  */
5223 struct ahd_softc *
5224 ahd_alloc(void *platform_arg, char *name)
5225 {
5226         struct  ahd_softc *ahd;
5227
5228 #ifndef __FreeBSD__
5229         ahd = malloc(sizeof(*ahd), M_DEVBUF, M_NOWAIT);
5230         if (!ahd) {
5231                 printf("aic7xxx: cannot malloc softc!\n");
5232                 free(name, M_DEVBUF);
5233                 return NULL;
5234         }
5235 #else
5236         ahd = device_get_softc((device_t)platform_arg);
5237 #endif
5238         memset(ahd, 0, sizeof(*ahd));
5239         ahd->seep_config = malloc(sizeof(*ahd->seep_config),
5240                                   M_DEVBUF, M_NOWAIT);
5241         if (ahd->seep_config == NULL) {
5242 #ifndef __FreeBSD__
5243                 free(ahd, M_DEVBUF);
5244 #endif
5245                 free(name, M_DEVBUF);
5246                 return (NULL);
5247         }
5248         LIST_INIT(&ahd->pending_scbs);
5249         /* We don't know our unit number until the OSM sets it */
5250         ahd->name = name;
5251         ahd->unit = -1;
5252         ahd->description = NULL;
5253         ahd->bus_description = NULL;
5254         ahd->channel = 'A';
5255         ahd->chip = AHD_NONE;
5256         ahd->features = AHD_FENONE;
5257         ahd->bugs = AHD_BUGNONE;
5258         ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
5259                    | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
5260         ahd_timer_init(&ahd->reset_timer);
5261         ahd_timer_init(&ahd->stat_timer);
5262         ahd->int_coalescing_timer = AHD_INT_COALESCING_TIMER_DEFAULT;
5263         ahd->int_coalescing_maxcmds = AHD_INT_COALESCING_MAXCMDS_DEFAULT;
5264         ahd->int_coalescing_mincmds = AHD_INT_COALESCING_MINCMDS_DEFAULT;
5265         ahd->int_coalescing_threshold = AHD_INT_COALESCING_THRESHOLD_DEFAULT;
5266         ahd->int_coalescing_stop_threshold =
5267             AHD_INT_COALESCING_STOP_THRESHOLD_DEFAULT;
5268
5269         if (ahd_platform_alloc(ahd, platform_arg) != 0) {
5270                 ahd_free(ahd);
5271                 ahd = NULL;
5272         }
5273 #ifdef AHD_DEBUG
5274         if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
5275                 printf("%s: scb size = 0x%x, hscb size = 0x%x\n",
5276                        ahd_name(ahd), (u_int)sizeof(struct scb),
5277                        (u_int)sizeof(struct hardware_scb));
5278         }
5279 #endif
5280         return (ahd);
5281 }
5282
5283 int
5284 ahd_softc_init(struct ahd_softc *ahd)
5285 {
5286
5287         ahd->unpause = 0;
5288         ahd->pause = PAUSE;
5289         return (0);
5290 }
5291
5292 void
5293 ahd_set_unit(struct ahd_softc *ahd, int unit)
5294 {
5295         ahd->unit = unit;
5296 }
5297
5298 void
5299 ahd_set_name(struct ahd_softc *ahd, char *name)
5300 {
5301         if (ahd->name != NULL)
5302                 free(ahd->name, M_DEVBUF);
5303         ahd->name = name;
5304 }
5305
5306 void
5307 ahd_free(struct ahd_softc *ahd)
5308 {
5309         int i;
5310
5311         switch (ahd->init_level) {
5312         default:
5313         case 5:
5314                 ahd_shutdown(ahd);
5315                 /* FALLTHROUGH */
5316         case 4:
5317                 ahd_dmamap_unload(ahd, ahd->shared_data_dmat,
5318                                   ahd->shared_data_map.dmamap);
5319                 /* FALLTHROUGH */
5320         case 3:
5321                 ahd_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
5322                                 ahd->shared_data_map.dmamap);
5323                 ahd_dmamap_destroy(ahd, ahd->shared_data_dmat,
5324                                    ahd->shared_data_map.dmamap);
5325                 /* FALLTHROUGH */
5326         case 2:
5327                 ahd_dma_tag_destroy(ahd, ahd->shared_data_dmat);
5328         case 1:
5329 #ifndef __linux__
5330                 ahd_dma_tag_destroy(ahd, ahd->buffer_dmat);
5331 #endif
5332                 break;
5333         case 0:
5334                 break;
5335         }
5336
5337 #ifndef __linux__
5338         ahd_dma_tag_destroy(ahd, ahd->parent_dmat);
5339 #endif
5340         ahd_platform_free(ahd);
5341         ahd_fini_scbdata(ahd);
5342         for (i = 0; i < AHD_NUM_TARGETS; i++) {
5343                 struct ahd_tmode_tstate *tstate;
5344
5345                 tstate = ahd->enabled_targets[i];
5346                 if (tstate != NULL) {
5347 #ifdef AHD_TARGET_MODE
5348                         int j;
5349
5350                         for (j = 0; j < AHD_NUM_LUNS; j++) {
5351                                 struct ahd_tmode_lstate *lstate;
5352
5353                                 lstate = tstate->enabled_luns[j];
5354                                 if (lstate != NULL) {
5355                                         xpt_free_path(lstate->path);
5356                                         free(lstate, M_DEVBUF);
5357                                 }
5358                         }
5359 #endif
5360                         free(tstate, M_DEVBUF);
5361                 }
5362         }
5363 #ifdef AHD_TARGET_MODE
5364         if (ahd->black_hole != NULL) {
5365                 xpt_free_path(ahd->black_hole->path);
5366                 free(ahd->black_hole, M_DEVBUF);
5367         }
5368 #endif
5369         if (ahd->name != NULL)
5370                 free(ahd->name, M_DEVBUF);
5371         if (ahd->seep_config != NULL)
5372                 free(ahd->seep_config, M_DEVBUF);
5373         if (ahd->saved_stack != NULL)
5374                 free(ahd->saved_stack, M_DEVBUF);
5375 #ifndef __FreeBSD__
5376         free(ahd, M_DEVBUF);
5377 #endif
5378         return;
5379 }
5380
5381 void
5382 ahd_shutdown(void *arg)
5383 {
5384         struct  ahd_softc *ahd;
5385
5386         ahd = (struct ahd_softc *)arg;
5387
5388         /*
5389          * Stop periodic timer callbacks.
5390          */
5391         ahd_timer_stop(&ahd->reset_timer);
5392         ahd_timer_stop(&ahd->stat_timer);
5393
5394         /* This will reset most registers to 0, but not all */
5395         ahd_reset(ahd, /*reinit*/FALSE);
5396 }
5397
5398 /*
5399  * Reset the controller and record some information about it
5400  * that is only available just after a reset.  If "reinit" is
5401  * non-zero, this reset occured after initial configuration
5402  * and the caller requests that the chip be fully reinitialized
5403  * to a runable state.  Chip interrupts are *not* enabled after
5404  * a reinitialization.  The caller must enable interrupts via
5405  * ahd_intr_enable().
5406  */
5407 int
5408 ahd_reset(struct ahd_softc *ahd, int reinit)
5409 {
5410         u_int    sxfrctl1;
5411         int      wait;
5412         uint32_t cmd;
5413
5414         /*
5415          * Preserve the value of the SXFRCTL1 register for all channels.
5416          * It contains settings that affect termination and we don't want
5417          * to disturb the integrity of the bus.
5418          */
5419         ahd_pause(ahd);
5420         ahd_update_modes(ahd);
5421         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5422         sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
5423
5424         cmd = ahd_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
5425         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
5426                 uint32_t mod_cmd;
5427
5428                 /*
5429                  * A4 Razor #632
5430                  * During the assertion of CHIPRST, the chip
5431                  * does not disable its parity logic prior to
5432                  * the start of the reset.  This may cause a
5433                  * parity error to be detected and thus a
5434                  * spurious SERR or PERR assertion.  Disble
5435                  * PERR and SERR responses during the CHIPRST.
5436                  */
5437                 mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
5438                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
5439                                      mod_cmd, /*bytes*/2);
5440         }
5441         ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
5442
5443         /*
5444          * Ensure that the reset has finished.  We delay 1000us
5445          * prior to reading the register to make sure the chip
5446          * has sufficiently completed its reset to handle register
5447          * accesses.
5448          */
5449         wait = 1000;
5450         do {
5451                 ahd_delay(1000);
5452         } while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
5453
5454         if (wait == 0) {
5455                 printf("%s: WARNING - Failed chip reset!  "
5456                        "Trying to initialize anyway.\n", ahd_name(ahd));
5457         }
5458         ahd_outb(ahd, HCNTRL, ahd->pause);
5459
5460         if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
5461                 /*
5462                  * Clear any latched PCI error status and restore
5463                  * previous SERR and PERR response enables.
5464                  */
5465                 ahd_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
5466                                      0xFF, /*bytes*/1);
5467                 ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
5468                                      cmd, /*bytes*/2);
5469         }
5470
5471         /*
5472          * Mode should be SCSI after a chip reset, but lets
5473          * set it just to be safe.  We touch the MODE_PTR
5474          * register directly so as to bypass the lazy update
5475          * code in ahd_set_modes().
5476          */
5477         ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5478         ahd_outb(ahd, MODE_PTR,
5479                  ahd_build_mode_state(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI));
5480
5481         /*
5482          * Restore SXFRCTL1.
5483          *
5484          * We must always initialize STPWEN to 1 before we
5485          * restore the saved values.  STPWEN is initialized
5486          * to a tri-state condition which can only be cleared
5487          * by turning it on.
5488          */
5489         ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
5490         ahd_outb(ahd, SXFRCTL1, sxfrctl1);
5491
5492         /* Determine chip configuration */
5493         ahd->features &= ~AHD_WIDE;
5494         if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
5495                 ahd->features |= AHD_WIDE;
5496
5497         /*
5498          * If a recovery action has forced a chip reset,
5499          * re-initialize the chip to our liking.
5500          */
5501         if (reinit != 0)
5502                 ahd_chip_init(ahd);
5503
5504         return (0);
5505 }
5506
5507 /*
5508  * Determine the number of SCBs available on the controller
5509  */
5510 int
5511 ahd_probe_scbs(struct ahd_softc *ahd) {
5512         int i;
5513
5514         AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
5515                          ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
5516         for (i = 0; i < AHD_SCB_MAX; i++) {
5517                 int j;
5518
5519                 ahd_set_scbptr(ahd, i);
5520                 ahd_outw(ahd, SCB_BASE, i);
5521                 for (j = 2; j < 64; j++)
5522                         ahd_outb(ahd, SCB_BASE+j, 0);
5523                 /* Start out life as unallocated (needing an abort) */
5524                 ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
5525                 if (ahd_inw_scbram(ahd, SCB_BASE) != i)
5526                         break;
5527                 ahd_set_scbptr(ahd, 0);
5528                 if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
5529                         break;
5530         }
5531         return (i);
5532 }
5533
5534 static void
5535 ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
5536 {
5537         dma_addr_t *baddr;
5538
5539         baddr = (dma_addr_t *)arg;
5540         *baddr = segs->ds_addr;
5541 }
5542
5543 static void
5544 ahd_initialize_hscbs(struct ahd_softc *ahd)
5545 {
5546         int i;
5547
5548         for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
5549                 ahd_set_scbptr(ahd, i);
5550
5551                 /* Clear the control byte. */
5552                 ahd_outb(ahd, SCB_CONTROL, 0);
5553
5554                 /* Set the next pointer */
5555                 ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
5556         }
5557 }
5558
5559 static int
5560 ahd_init_scbdata(struct ahd_softc *ahd)
5561 {
5562         struct  scb_data *scb_data;
5563         int     i;
5564
5565         scb_data = &ahd->scb_data;
5566         TAILQ_INIT(&scb_data->free_scbs);
5567         for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
5568                 LIST_INIT(&scb_data->free_scb_lists[i]);
5569         LIST_INIT(&scb_data->any_dev_free_scb_list);
5570         SLIST_INIT(&scb_data->hscb_maps);
5571         SLIST_INIT(&scb_data->sg_maps);
5572         SLIST_INIT(&scb_data->sense_maps);
5573
5574         /* Determine the number of hardware SCBs and initialize them */
5575         scb_data->maxhscbs = ahd_probe_scbs(ahd);
5576         if (scb_data->maxhscbs == 0) {
5577                 printf("%s: No SCB space found\n", ahd_name(ahd));
5578                 return (ENXIO);
5579         }
5580
5581         ahd_initialize_hscbs(ahd);
5582
5583         /*
5584          * Create our DMA tags.  These tags define the kinds of device
5585          * accessible memory allocations and memory mappings we will
5586          * need to perform during normal operation.
5587          *
5588          * Unless we need to further restrict the allocation, we rely
5589          * on the restrictions of the parent dmat, hence the common
5590          * use of MAXADDR and MAXSIZE.
5591          */
5592
5593         /* DMA tag for our hardware scb structures */
5594         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5595                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5596                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5597                                /*highaddr*/BUS_SPACE_MAXADDR,
5598                                /*filter*/NULL, /*filterarg*/NULL,
5599                                PAGE_SIZE, /*nsegments*/1,
5600                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5601                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
5602                 goto error_exit;
5603         }
5604
5605         scb_data->init_level++;
5606
5607         /* DMA tag for our S/G structures. */
5608         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/8,
5609                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5610                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5611                                /*highaddr*/BUS_SPACE_MAXADDR,
5612                                /*filter*/NULL, /*filterarg*/NULL,
5613                                ahd_sglist_allocsize(ahd), /*nsegments*/1,
5614                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5615                                /*flags*/0, &scb_data->sg_dmat) != 0) {
5616                 goto error_exit;
5617         }
5618 #ifdef AHD_DEBUG
5619         if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
5620                 printf("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
5621                        ahd_sglist_allocsize(ahd));
5622 #endif
5623
5624         scb_data->init_level++;
5625
5626         /* DMA tag for our sense buffers.  We allocate in page sized chunks */
5627         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
5628                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5629                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5630                                /*highaddr*/BUS_SPACE_MAXADDR,
5631                                /*filter*/NULL, /*filterarg*/NULL,
5632                                PAGE_SIZE, /*nsegments*/1,
5633                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5634                                /*flags*/0, &scb_data->sense_dmat) != 0) {
5635                 goto error_exit;
5636         }
5637
5638         scb_data->init_level++;
5639
5640         /* Perform initial CCB allocation */
5641         ahd_alloc_scbs(ahd);
5642
5643         if (scb_data->numscbs == 0) {
5644                 printf("%s: ahd_init_scbdata - "
5645                        "Unable to allocate initial scbs\n",
5646                        ahd_name(ahd));
5647                 goto error_exit;
5648         }
5649
5650         /*
5651          * Note that we were successfull
5652          */
5653         return (0);
5654
5655 error_exit:
5656
5657         return (ENOMEM);
5658 }
5659
5660 static struct scb *
5661 ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
5662 {
5663         struct scb *scb;
5664
5665         /*
5666          * Look on the pending list.
5667          */
5668         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
5669                 if (SCB_GET_TAG(scb) == tag)
5670                         return (scb);
5671         }
5672
5673         /*
5674          * Then on all of the collision free lists.
5675          */
5676         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
5677                 struct scb *list_scb;
5678
5679                 list_scb = scb;
5680                 do {
5681                         if (SCB_GET_TAG(list_scb) == tag)
5682                                 return (list_scb);
5683                         list_scb = LIST_NEXT(list_scb, collision_links);
5684                 } while (list_scb);
5685         }
5686
5687         /*
5688          * And finally on the generic free list.
5689          */
5690         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
5691                 if (SCB_GET_TAG(scb) == tag)
5692                         return (scb);
5693         }
5694
5695         return (NULL);
5696 }
5697
5698 static void
5699 ahd_fini_scbdata(struct ahd_softc *ahd)
5700 {
5701         struct scb_data *scb_data;
5702
5703         scb_data = &ahd->scb_data;
5704         if (scb_data == NULL)
5705                 return;
5706
5707         switch (scb_data->init_level) {
5708         default:
5709         case 7:
5710         {
5711                 struct map_node *sns_map;
5712
5713                 while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
5714                         SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
5715                         ahd_dmamap_unload(ahd, scb_data->sense_dmat,
5716                                           sns_map->dmamap);
5717                         ahd_dmamem_free(ahd, scb_data->sense_dmat,
5718                                         sns_map->vaddr, sns_map->dmamap);
5719                         free(sns_map, M_DEVBUF);
5720                 }
5721                 ahd_dma_tag_destroy(ahd, scb_data->sense_dmat);
5722                 /* FALLTHROUGH */
5723         }
5724         case 6:
5725         {
5726                 struct map_node *sg_map;
5727
5728                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
5729                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
5730                         ahd_dmamap_unload(ahd, scb_data->sg_dmat,
5731                                           sg_map->dmamap);
5732                         ahd_dmamem_free(ahd, scb_data->sg_dmat,
5733                                         sg_map->vaddr, sg_map->dmamap);
5734                         free(sg_map, M_DEVBUF);
5735                 }
5736                 ahd_dma_tag_destroy(ahd, scb_data->sg_dmat);
5737                 /* FALLTHROUGH */
5738         }
5739         case 5:
5740         {
5741                 struct map_node *hscb_map;
5742
5743                 while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
5744                         SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
5745                         ahd_dmamap_unload(ahd, scb_data->hscb_dmat,
5746                                           hscb_map->dmamap);
5747                         ahd_dmamem_free(ahd, scb_data->hscb_dmat,
5748                                         hscb_map->vaddr, hscb_map->dmamap);
5749                         free(hscb_map, M_DEVBUF);
5750                 }
5751                 ahd_dma_tag_destroy(ahd, scb_data->hscb_dmat);
5752                 /* FALLTHROUGH */
5753         }
5754         case 4:
5755         case 3:
5756         case 2:
5757         case 1:
5758         case 0:
5759                 break;
5760         }
5761 }
5762
5763 /*
5764  * DSP filter Bypass must be enabled until the first selection
5765  * after a change in bus mode (Razor #491 and #493).
5766  */
5767 static void
5768 ahd_setup_iocell_workaround(struct ahd_softc *ahd)
5769 {
5770         ahd_mode_state saved_modes;
5771
5772         saved_modes = ahd_save_modes(ahd);
5773         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
5774         ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
5775                | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
5776         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
5777 #ifdef AHD_DEBUG
5778         if ((ahd_debug & AHD_SHOW_MISC) != 0)
5779                 printf("%s: Setting up iocell workaround\n", ahd_name(ahd));
5780 #endif
5781         ahd_restore_modes(ahd, saved_modes);
5782         ahd->flags &= ~AHD_HAD_FIRST_SEL;
5783 }
5784
5785 static void
5786 ahd_iocell_first_selection(struct ahd_softc *ahd)
5787 {
5788         ahd_mode_state  saved_modes;
5789         u_int           sblkctl;
5790
5791         if ((ahd->flags & AHD_HAD_FIRST_SEL) != 0)
5792                 return;
5793         saved_modes = ahd_save_modes(ahd);
5794         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5795         sblkctl = ahd_inb(ahd, SBLKCTL);
5796         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
5797 #ifdef AHD_DEBUG
5798         if ((ahd_debug & AHD_SHOW_MISC) != 0)
5799                 printf("%s: iocell first selection\n", ahd_name(ahd));
5800 #endif
5801         if ((sblkctl & ENAB40) != 0) {
5802                 ahd_outb(ahd, DSPDATACTL,
5803                          ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
5804 #ifdef AHD_DEBUG
5805                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
5806                         printf("%s: BYPASS now disabled\n", ahd_name(ahd));
5807 #endif
5808         }
5809         ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
5810         ahd_outb(ahd, CLRINT, CLRSCSIINT);
5811         ahd_restore_modes(ahd, saved_modes);
5812         ahd->flags |= AHD_HAD_FIRST_SEL;
5813 }
5814
5815 /*************************** SCB Management ***********************************/
5816 static void
5817 ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
5818 {
5819         struct  scb_list *free_list;
5820         struct  scb_tailq *free_tailq;
5821         struct  scb *first_scb;
5822
5823         scb->flags |= SCB_ON_COL_LIST;
5824         AHD_SET_SCB_COL_IDX(scb, col_idx);
5825         free_list = &ahd->scb_data.free_scb_lists[col_idx];
5826         free_tailq = &ahd->scb_data.free_scbs;
5827         first_scb = LIST_FIRST(free_list);
5828         if (first_scb != NULL) {
5829                 LIST_INSERT_AFTER(first_scb, scb, collision_links);
5830         } else {
5831                 LIST_INSERT_HEAD(free_list, scb, collision_links);
5832                 TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
5833         }
5834 }
5835
5836 static void
5837 ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
5838 {
5839         struct  scb_list *free_list;
5840         struct  scb_tailq *free_tailq;
5841         struct  scb *first_scb;
5842         u_int   col_idx;
5843
5844         scb->flags &= ~SCB_ON_COL_LIST;
5845         col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
5846         free_list = &ahd->scb_data.free_scb_lists[col_idx];
5847         free_tailq = &ahd->scb_data.free_scbs;
5848         first_scb = LIST_FIRST(free_list);
5849         if (first_scb == scb) {
5850                 struct scb *next_scb;
5851
5852                 /*
5853                  * Maintain order in the collision free
5854                  * lists for fairness if this device has
5855                  * other colliding tags active.
5856                  */
5857                 next_scb = LIST_NEXT(scb, collision_links);
5858                 if (next_scb != NULL) {
5859                         TAILQ_INSERT_AFTER(free_tailq, scb,
5860                                            next_scb, links.tqe);
5861                 }
5862                 TAILQ_REMOVE(free_tailq, scb, links.tqe);
5863         }
5864         LIST_REMOVE(scb, collision_links);
5865 }
5866
5867 /*
5868  * Get a free scb. If there are none, see if we can allocate a new SCB.
5869  */
5870 struct scb *
5871 ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
5872 {
5873         struct scb *scb;
5874         int tries;
5875
5876         tries = 0;
5877 look_again:
5878         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
5879                 if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
5880                         ahd_rem_col_list(ahd, scb);
5881                         goto found;
5882                 }
5883         }
5884         if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
5885
5886                 if (tries++ != 0)
5887                         return (NULL);
5888                 ahd_alloc_scbs(ahd);
5889                 goto look_again;
5890         }
5891         LIST_REMOVE(scb, links.le);
5892         if (col_idx != AHD_NEVER_COL_IDX
5893          && (scb->col_scb != NULL)
5894          && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
5895                 LIST_REMOVE(scb->col_scb, links.le);
5896                 ahd_add_col_list(ahd, scb->col_scb, col_idx);
5897         }
5898 found:
5899         scb->flags |= SCB_ACTIVE;
5900         return (scb);
5901 }
5902
5903 /*
5904  * Return an SCB resource to the free list.
5905  */
5906 void
5907 ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
5908 {
5909
5910         /* Clean up for the next user */
5911         scb->flags = SCB_FLAG_NONE;
5912         scb->hscb->control = 0;
5913         ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = NULL;
5914
5915         if (scb->col_scb == NULL) {
5916
5917                 /*
5918                  * No collision possible.  Just free normally.
5919                  */
5920                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
5921                                  scb, links.le);
5922         } else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
5923
5924                 /*
5925                  * The SCB we might have collided with is on
5926                  * a free collision list.  Put both SCBs on
5927                  * the generic list.
5928                  */
5929                 ahd_rem_col_list(ahd, scb->col_scb);
5930                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
5931                                  scb, links.le);
5932                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
5933                                  scb->col_scb, links.le);
5934         } else if ((scb->col_scb->flags
5935                   & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
5936                 && (scb->col_scb->hscb->control & TAG_ENB) != 0) {
5937
5938                 /*
5939                  * The SCB we might collide with on the next allocation
5940                  * is still active in a non-packetized, tagged, context.
5941                  * Put us on the SCB collision list.
5942                  */
5943                 ahd_add_col_list(ahd, scb,
5944                                  AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
5945         } else {
5946                 /*
5947                  * The SCB we might collide with on the next allocation
5948                  * is either active in a packetized context, or free.
5949                  * Since we can't collide, put this SCB on the generic
5950                  * free list.
5951                  */
5952                 LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
5953                                  scb, links.le);
5954         }
5955
5956         ahd_platform_scb_free(ahd, scb);
5957 }
5958
5959 void
5960 ahd_alloc_scbs(struct ahd_softc *ahd)
5961 {
5962         struct scb_data *scb_data;
5963         struct scb      *next_scb;
5964         struct hardware_scb *hscb;
5965         struct map_node *hscb_map;
5966         struct map_node *sg_map;
5967         struct map_node *sense_map;
5968         uint8_t         *segs;
5969         uint8_t         *sense_data;
5970         dma_addr_t       hscb_busaddr;
5971         dma_addr_t       sg_busaddr;
5972         dma_addr_t       sense_busaddr;
5973         int              newcount;
5974         int              i;
5975
5976         scb_data = &ahd->scb_data;
5977         if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
5978                 /* Can't allocate any more */
5979                 return;
5980
5981         if (scb_data->scbs_left != 0) {
5982                 int offset;
5983
5984                 offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
5985                 hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
5986                 hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
5987                 hscb_busaddr = hscb_map->physaddr + (offset * sizeof(*hscb));
5988         } else {
5989                 hscb_map = malloc(sizeof(*hscb_map), M_DEVBUF, M_NOWAIT);
5990
5991                 if (hscb_map == NULL)
5992                         return;
5993
5994                 /* Allocate the next batch of hardware SCBs */
5995                 if (ahd_dmamem_alloc(ahd, scb_data->hscb_dmat,
5996                                      (void **)&hscb_map->vaddr,
5997                                      BUS_DMA_NOWAIT, &hscb_map->dmamap) != 0) {
5998                         free(hscb_map, M_DEVBUF);
5999                         return;
6000                 }
6001
6002                 SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
6003
6004                 ahd_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
6005                                 hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6006                                 &hscb_map->physaddr, /*flags*/0);
6007
6008                 hscb = (struct hardware_scb *)hscb_map->vaddr;
6009                 hscb_busaddr = hscb_map->physaddr;
6010                 scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
6011         }
6012
6013         if (scb_data->sgs_left != 0) {
6014                 int offset;
6015
6016                 offset = ((ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd))
6017                        - scb_data->sgs_left) * ahd_sglist_size(ahd);
6018                 sg_map = SLIST_FIRST(&scb_data->sg_maps);
6019                 segs = sg_map->vaddr + offset;
6020                 sg_busaddr = sg_map->physaddr + offset;
6021         } else {
6022                 sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
6023
6024                 if (sg_map == NULL)
6025                         return;
6026
6027                 /* Allocate the next batch of S/G lists */
6028                 if (ahd_dmamem_alloc(ahd, scb_data->sg_dmat,
6029                                      (void **)&sg_map->vaddr,
6030                                      BUS_DMA_NOWAIT, &sg_map->dmamap) != 0) {
6031                         free(sg_map, M_DEVBUF);
6032                         return;
6033                 }
6034
6035                 SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
6036
6037                 ahd_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
6038                                 sg_map->vaddr, ahd_sglist_allocsize(ahd),
6039                                 ahd_dmamap_cb, &sg_map->physaddr, /*flags*/0);
6040
6041                 segs = sg_map->vaddr;
6042                 sg_busaddr = sg_map->physaddr;
6043                 scb_data->sgs_left =
6044                     ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
6045 #ifdef AHD_DEBUG
6046                 if (ahd_debug & AHD_SHOW_MEMORY)
6047                         printf("Mapped SG data\n");
6048 #endif
6049         }
6050
6051         if (scb_data->sense_left != 0) {
6052                 int offset;
6053
6054                 offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
6055                 sense_map = SLIST_FIRST(&scb_data->sense_maps);
6056                 sense_data = sense_map->vaddr + offset;
6057                 sense_busaddr = sense_map->physaddr + offset;
6058         } else {
6059                 sense_map = malloc(sizeof(*sense_map), M_DEVBUF, M_NOWAIT);
6060
6061                 if (sense_map == NULL)
6062                         return;
6063
6064                 /* Allocate the next batch of sense buffers */
6065                 if (ahd_dmamem_alloc(ahd, scb_data->sense_dmat,
6066                                      (void **)&sense_map->vaddr,
6067                                      BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
6068                         free(sense_map, M_DEVBUF);
6069                         return;
6070                 }
6071
6072                 SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
6073
6074                 ahd_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
6075                                 sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6076                                 &sense_map->physaddr, /*flags*/0);
6077
6078                 sense_data = sense_map->vaddr;
6079                 sense_busaddr = sense_map->physaddr;
6080                 scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
6081 #ifdef AHD_DEBUG
6082                 if (ahd_debug & AHD_SHOW_MEMORY)
6083                         printf("Mapped sense data\n");
6084 #endif
6085         }
6086
6087         newcount = MIN(scb_data->sense_left, scb_data->scbs_left);
6088         newcount = MIN(newcount, scb_data->sgs_left);
6089         newcount = MIN(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
6090         for (i = 0; i < newcount; i++) {
6091                 struct scb_platform_data *pdata;
6092                 u_int col_tag;
6093 #ifndef __linux__
6094                 int error;
6095 #endif
6096
6097                 next_scb = (struct scb *)malloc(sizeof(*next_scb),
6098                                                 M_DEVBUF, M_NOWAIT);
6099                 if (next_scb == NULL)
6100                         break;
6101
6102                 pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
6103                                                            M_DEVBUF, M_NOWAIT);
6104                 if (pdata == NULL) {
6105                         free(next_scb, M_DEVBUF);
6106                         break;
6107                 }
6108                 next_scb->platform_data = pdata;
6109                 next_scb->hscb_map = hscb_map;
6110                 next_scb->sg_map = sg_map;
6111                 next_scb->sense_map = sense_map;
6112                 next_scb->sg_list = segs;
6113                 next_scb->sense_data = sense_data;
6114                 next_scb->sense_busaddr = sense_busaddr;
6115                 memset(hscb, 0, sizeof(*hscb));
6116                 next_scb->hscb = hscb;
6117                 hscb->hscb_busaddr = ahd_htole32(hscb_busaddr);
6118
6119                 /*
6120                  * The sequencer always starts with the second entry.
6121                  * The first entry is embedded in the scb.
6122                  */
6123                 next_scb->sg_list_busaddr = sg_busaddr;
6124                 if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
6125                         next_scb->sg_list_busaddr
6126                             += sizeof(struct ahd_dma64_seg);
6127                 else
6128                         next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
6129                 next_scb->ahd_softc = ahd;
6130                 next_scb->flags = SCB_FLAG_NONE;
6131 #ifndef __linux__
6132                 error = ahd_dmamap_create(ahd, ahd->buffer_dmat, /*flags*/0,
6133                                           &next_scb->dmamap);
6134                 if (error != 0) {
6135                         free(next_scb, M_DEVBUF);
6136                         free(pdata, M_DEVBUF);
6137                         break;
6138                 }
6139 #endif
6140                 next_scb->hscb->tag = ahd_htole16(scb_data->numscbs);
6141                 col_tag = scb_data->numscbs ^ 0x100;
6142                 next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
6143                 if (next_scb->col_scb != NULL)
6144                         next_scb->col_scb->col_scb = next_scb;
6145                 ahd_free_scb(ahd, next_scb);
6146                 hscb++;
6147                 hscb_busaddr += sizeof(*hscb);
6148                 segs += ahd_sglist_size(ahd);
6149                 sg_busaddr += ahd_sglist_size(ahd);
6150                 sense_data += AHD_SENSE_BUFSIZE;
6151                 sense_busaddr += AHD_SENSE_BUFSIZE;
6152                 scb_data->numscbs++;
6153                 scb_data->sense_left--;
6154                 scb_data->scbs_left--;
6155                 scb_data->sgs_left--;
6156         }
6157 }
6158
6159 void
6160 ahd_controller_info(struct ahd_softc *ahd, char *buf)
6161 {
6162         const char *speed;
6163         const char *type;
6164         int len;
6165
6166         len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
6167         buf += len;
6168
6169         speed = "Ultra320 ";
6170         if ((ahd->features & AHD_WIDE) != 0) {
6171                 type = "Wide ";
6172         } else {
6173                 type = "Single ";
6174         }
6175         len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
6176                       speed, type, ahd->channel, ahd->our_id);
6177         buf += len;
6178
6179         sprintf(buf, "%s, %d SCBs", ahd->bus_description,
6180                 ahd->scb_data.maxhscbs);
6181 }
6182
6183 static const char *channel_strings[] = {
6184         "Primary Low",
6185         "Primary High",
6186         "Secondary Low",
6187         "Secondary High"
6188 };
6189
6190 static const char *termstat_strings[] = {
6191         "Terminated Correctly",
6192         "Over Terminated",
6193         "Under Terminated",
6194         "Not Configured"
6195 };
6196
6197 /*
6198  * Start the board, ready for normal operation
6199  */
6200 int
6201 ahd_init(struct ahd_softc *ahd)
6202 {
6203         uint8_t         *next_vaddr;
6204         dma_addr_t       next_baddr;
6205         size_t           driver_data_size;
6206         int              i;
6207         int              error;
6208         u_int            warn_user;
6209         uint8_t          current_sensing;
6210         uint8_t          fstat;
6211
6212         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6213
6214         ahd->stack_size = ahd_probe_stack_size(ahd);
6215         ahd->saved_stack = malloc(ahd->stack_size * sizeof(uint16_t),
6216                                   M_DEVBUF, M_NOWAIT);
6217         if (ahd->saved_stack == NULL)
6218                 return (ENOMEM);
6219
6220         /*
6221          * Verify that the compiler hasn't over-agressively
6222          * padded important structures.
6223          */
6224         if (sizeof(struct hardware_scb) != 64)
6225                 panic("Hardware SCB size is incorrect");
6226
6227 #ifdef AHD_DEBUG
6228         if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
6229                 ahd->flags |= AHD_SEQUENCER_DEBUG;
6230 #endif
6231
6232         /*
6233          * Default to allowing initiator operations.
6234          */
6235         ahd->flags |= AHD_INITIATORROLE;
6236
6237         /*
6238          * Only allow target mode features if this unit has them enabled.
6239          */
6240         if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
6241                 ahd->features &= ~AHD_TARGETMODE;
6242
6243 #ifndef __linux__
6244         /* DMA tag for mapping buffers into device visible space. */
6245         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6246                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6247                                /*lowaddr*/ahd->flags & AHD_39BIT_ADDRESSING
6248                                         ? (dma_addr_t)0x7FFFFFFFFFULL
6249                                         : BUS_SPACE_MAXADDR_32BIT,
6250                                /*highaddr*/BUS_SPACE_MAXADDR,
6251                                /*filter*/NULL, /*filterarg*/NULL,
6252                                /*maxsize*/(AHD_NSEG - 1) * PAGE_SIZE,
6253                                /*nsegments*/AHD_NSEG,
6254                                /*maxsegsz*/AHD_MAXTRANSFER_SIZE,
6255                                /*flags*/BUS_DMA_ALLOCNOW,
6256                                &ahd->buffer_dmat) != 0) {
6257                 return (ENOMEM);
6258         }
6259 #endif
6260
6261         ahd->init_level++;
6262
6263         /*
6264          * DMA tag for our command fifos and other data in system memory
6265          * the card's sequencer must be able to access.  For initiator
6266          * roles, we need to allocate space for the qoutfifo.  When providing
6267          * for the target mode role, we must additionally provide space for
6268          * the incoming target command fifo.
6269          */
6270         driver_data_size = AHD_SCB_MAX * sizeof(*ahd->qoutfifo)
6271                          + sizeof(struct hardware_scb);
6272         if ((ahd->features & AHD_TARGETMODE) != 0)
6273                 driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6274         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
6275                 driver_data_size += PKT_OVERRUN_BUFSIZE;
6276         if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6277                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6278                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6279                                /*highaddr*/BUS_SPACE_MAXADDR,
6280                                /*filter*/NULL, /*filterarg*/NULL,
6281                                driver_data_size,
6282                                /*nsegments*/1,
6283                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6284                                /*flags*/0, &ahd->shared_data_dmat) != 0) {
6285                 return (ENOMEM);
6286         }
6287
6288         ahd->init_level++;
6289
6290         /* Allocation of driver data */
6291         if (ahd_dmamem_alloc(ahd, ahd->shared_data_dmat,
6292                              (void **)&ahd->shared_data_map.vaddr,
6293                              BUS_DMA_NOWAIT,
6294                              &ahd->shared_data_map.dmamap) != 0) {
6295                 return (ENOMEM);
6296         }
6297
6298         ahd->init_level++;
6299
6300         /* And permanently map it in */
6301         ahd_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
6302                         ahd->shared_data_map.vaddr, driver_data_size,
6303                         ahd_dmamap_cb, &ahd->shared_data_map.physaddr,
6304                         /*flags*/0);
6305         ahd->qoutfifo = (struct ahd_completion *)ahd->shared_data_map.vaddr;
6306         next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
6307         next_baddr = ahd->shared_data_map.physaddr
6308                    + AHD_QOUT_SIZE*sizeof(struct ahd_completion);
6309         if ((ahd->features & AHD_TARGETMODE) != 0) {
6310                 ahd->targetcmds = (struct target_cmd *)next_vaddr;
6311                 next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6312                 next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
6313         }
6314
6315         if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
6316                 ahd->overrun_buf = next_vaddr;
6317                 next_vaddr += PKT_OVERRUN_BUFSIZE;
6318                 next_baddr += PKT_OVERRUN_BUFSIZE;
6319         }
6320
6321         /*
6322          * We need one SCB to serve as the "next SCB".  Since the
6323          * tag identifier in this SCB will never be used, there is
6324          * no point in using a valid HSCB tag from an SCB pulled from
6325          * the standard free pool.  So, we allocate this "sentinel"
6326          * specially from the DMA safe memory chunk used for the QOUTFIFO.
6327          */
6328         ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
6329         ahd->next_queued_hscb_map = &ahd->shared_data_map;
6330         ahd->next_queued_hscb->hscb_busaddr = ahd_htole32(next_baddr);
6331
6332         ahd->init_level++;
6333
6334         /* Allocate SCB data now that buffer_dmat is initialized */
6335         if (ahd_init_scbdata(ahd) != 0)
6336                 return (ENOMEM);
6337
6338         if ((ahd->flags & AHD_INITIATORROLE) == 0)
6339                 ahd->flags &= ~AHD_RESET_BUS_A;
6340
6341         /*
6342          * Before committing these settings to the chip, give
6343          * the OSM one last chance to modify our configuration.
6344          */
6345         ahd_platform_init(ahd);
6346
6347         /* Bring up the chip. */
6348         ahd_chip_init(ahd);
6349
6350         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
6351
6352         if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
6353                 goto init_done;
6354
6355         /*
6356          * Verify termination based on current draw and
6357          * warn user if the bus is over/under terminated.
6358          */
6359         error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
6360                                    CURSENSE_ENB);
6361         if (error != 0) {
6362                 printf("%s: current sensing timeout 1\n", ahd_name(ahd));
6363                 goto init_done;
6364         }
6365         for (i = 20, fstat = FLX_FSTAT_BUSY;
6366              (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
6367                 error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
6368                 if (error != 0) {
6369                         printf("%s: current sensing timeout 2\n",
6370                                ahd_name(ahd));
6371                         goto init_done;
6372                 }
6373         }
6374         if (i == 0) {
6375                 printf("%s: Timedout during current-sensing test\n",
6376                        ahd_name(ahd));
6377                 goto init_done;
6378         }
6379
6380         /* Latch Current Sensing status. */
6381         error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
6382         if (error != 0) {
6383                 printf("%s: current sensing timeout 3\n", ahd_name(ahd));
6384                 goto init_done;
6385         }
6386
6387         /* Diable current sensing. */
6388         ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
6389
6390 #ifdef AHD_DEBUG
6391         if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
6392                 printf("%s: current_sensing == 0x%x\n",
6393                        ahd_name(ahd), current_sensing);
6394         }
6395 #endif
6396         warn_user = 0;
6397         for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
6398                 u_int term_stat;
6399
6400                 term_stat = (current_sensing & FLX_CSTAT_MASK);
6401                 switch (term_stat) {
6402                 case FLX_CSTAT_OVER:
6403                 case FLX_CSTAT_UNDER:
6404                         warn_user++;
6405                 case FLX_CSTAT_INVALID:
6406                 case FLX_CSTAT_OKAY:
6407                         if (warn_user == 0 && bootverbose == 0)
6408                                 break;
6409                         printf("%s: %s Channel %s\n", ahd_name(ahd),
6410                                channel_strings[i], termstat_strings[term_stat]);
6411                         break;
6412                 }
6413         }
6414         if (warn_user) {
6415                 printf("%s: WARNING. Termination is not configured correctly.\n"
6416                        "%s: WARNING. SCSI bus operations may FAIL.\n",
6417                        ahd_name(ahd), ahd_name(ahd));
6418         }
6419 init_done:
6420         ahd_restart(ahd);
6421         ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US,
6422                         ahd_stat_timer, ahd);
6423         return (0);
6424 }
6425
6426 /*
6427  * (Re)initialize chip state after a chip reset.
6428  */
6429 static void
6430 ahd_chip_init(struct ahd_softc *ahd)
6431 {
6432         uint32_t busaddr;
6433         u_int    sxfrctl1;
6434         u_int    scsiseq_template;
6435         u_int    wait;
6436         u_int    i;
6437         u_int    target;
6438
6439         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6440         /*
6441          * Take the LED out of diagnostic mode
6442          */
6443         ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
6444
6445         /*
6446          * Return HS_MAILBOX to its default value.
6447          */
6448         ahd->hs_mailbox = 0;
6449         ahd_outb(ahd, HS_MAILBOX, 0);
6450
6451         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
6452         ahd_outb(ahd, IOWNID, ahd->our_id);
6453         ahd_outb(ahd, TOWNID, ahd->our_id);
6454         sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
6455         sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
6456         if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
6457          && (ahd->seltime != STIMESEL_MIN)) {
6458                 /*
6459                  * The selection timer duration is twice as long
6460                  * as it should be.  Halve it by adding "1" to
6461                  * the user specified setting.
6462                  */
6463                 sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
6464         } else {
6465                 sxfrctl1 |= ahd->seltime;
6466         }
6467
6468         ahd_outb(ahd, SXFRCTL0, DFON);
6469         ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
6470         ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
6471
6472         /*
6473          * Now that termination is set, wait for up
6474          * to 500ms for our transceivers to settle.  If
6475          * the adapter does not have a cable attached,
6476          * the transceivers may never settle, so don't
6477          * complain if we fail here.
6478          */
6479         for (wait = 10000;
6480              (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
6481              wait--)
6482                 ahd_delay(100);
6483
6484         /* Clear any false bus resets due to the transceivers settling */
6485         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
6486         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6487
6488         /* Initialize mode specific S/G state. */
6489         for (i = 0; i < 2; i++) {
6490                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
6491                 ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
6492                 ahd_outb(ahd, SG_STATE, 0);
6493                 ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
6494                 ahd_outb(ahd, SEQIMODE,
6495                          ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
6496                         |ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
6497         }
6498
6499         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6500         ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
6501         ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
6502         ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
6503         ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
6504         if ((ahd->bugs & AHD_BUSFREEREV_BUG) != 0) {
6505                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|AUTO_MSGOUT_DE);
6506         } else {
6507                 ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
6508         }
6509         ahd_outb(ahd, SCSCHKN, CURRFIFODEF|WIDERESEN|SHVALIDSTDIS);
6510         if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
6511                 /*
6512                  * Do not issue a target abort when a split completion
6513                  * error occurs.  Let our PCIX interrupt handler deal
6514                  * with it instead. H2A4 Razor #625
6515                  */
6516                 ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
6517
6518         if ((ahd->bugs & AHD_LQOOVERRUN_BUG) != 0)
6519                 ahd_outb(ahd, LQOSCSCTL, LQONOCHKOVER);
6520
6521         /*
6522          * Tweak IOCELL settings.
6523          */
6524         if ((ahd->flags & AHD_HP_BOARD) != 0) {
6525                 for (i = 0; i < NUMDSPS; i++) {
6526                         ahd_outb(ahd, DSPSELECT, i);
6527                         ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_HP_DEFAULT);
6528                 }
6529 #ifdef AHD_DEBUG
6530                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
6531                         printf("%s: WRTBIASCTL now 0x%x\n", ahd_name(ahd),
6532                                WRTBIASCTL_HP_DEFAULT);
6533 #endif
6534         }
6535         ahd_setup_iocell_workaround(ahd);
6536
6537         /*
6538          * Enable LQI Manager interrupts.
6539          */
6540         ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
6541                               | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
6542                               | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
6543         ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
6544         /*
6545          * We choose to have the sequencer catch LQOPHCHGINPKT errors
6546          * manually for the command phase at the start of a packetized
6547          * selection case.  ENLQOBUSFREE should be made redundant by
6548          * the BUSFREE interrupt, but it seems that some LQOBUSFREE
6549          * events fail to assert the BUSFREE interrupt so we must
6550          * also enable LQOBUSFREE interrupts.
6551          */
6552         ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE);
6553
6554         /*
6555          * Setup sequencer interrupt handlers.
6556          */
6557         ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
6558         ahd_outw(ahd, INTVEC2_ADDR, ahd_resolve_seqaddr(ahd, LABEL_timer_isr));
6559
6560         /*
6561          * Setup SCB Offset registers.
6562          */
6563         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
6564                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
6565                          pkt_long_lun));
6566         } else {
6567                 ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
6568         }
6569         ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
6570         ahd_outb(ahd, ATTRPTR, offsetof(struct hardware_scb, task_attribute));
6571         ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
6572         ahd_outb(ahd, CMDPTR, offsetof(struct hardware_scb,
6573                                        shared_data.idata.cdb));
6574         ahd_outb(ahd, QNEXTPTR,
6575                  offsetof(struct hardware_scb, next_hscb_busaddr));
6576         ahd_outb(ahd, ABRTBITPTR, MK_MESSAGE_BIT_OFFSET);
6577         ahd_outb(ahd, ABRTBYTEPTR, offsetof(struct hardware_scb, control));
6578         if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
6579                 ahd_outb(ahd, LUNLEN,
6580                          sizeof(ahd->next_queued_hscb->pkt_long_lun) - 1);
6581         } else {
6582                 ahd_outb(ahd, LUNLEN, LUNLEN_SINGLE_LEVEL_LUN);
6583         }
6584         ahd_outb(ahd, CDBLIMIT, SCB_CDB_LEN_PTR - 1);
6585         ahd_outb(ahd, MAXCMD, 0xFF);
6586         ahd_outb(ahd, SCBAUTOPTR,
6587                  AUSCBPTR_EN | offsetof(struct hardware_scb, tag));
6588
6589         /* We haven't been enabled for target mode yet. */
6590         ahd_outb(ahd, MULTARGID, 0);
6591         ahd_outb(ahd, MULTARGID + 1, 0);
6592
6593         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6594         /* Initialize the negotiation table. */
6595         if ((ahd->features & AHD_NEW_IOCELL_OPTS) == 0) {
6596                 /*
6597                  * Clear the spare bytes in the neg table to avoid
6598                  * spurious parity errors.
6599                  */
6600                 for (target = 0; target < AHD_NUM_TARGETS; target++) {
6601                         ahd_outb(ahd, NEGOADDR, target);
6602                         ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PER_DEV0);
6603                         for (i = 0; i < AHD_NUM_PER_DEV_ANNEXCOLS; i++)
6604                                 ahd_outb(ahd, ANNEXDAT, 0);
6605                 }
6606         }
6607         for (target = 0; target < AHD_NUM_TARGETS; target++) {
6608                 struct   ahd_devinfo devinfo;
6609                 struct   ahd_initiator_tinfo *tinfo;
6610                 struct   ahd_tmode_tstate *tstate;
6611
6612                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6613                                             target, &tstate);
6614                 ahd_compile_devinfo(&devinfo, ahd->our_id,
6615                                     target, CAM_LUN_WILDCARD,
6616                                     'A', ROLE_INITIATOR);
6617                 ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
6618         }
6619
6620         ahd_outb(ahd, CLRSINT3, NTRAMPERR|OSRAMPERR);
6621         ahd_outb(ahd, CLRINT, CLRSCSIINT);
6622
6623 #ifdef NEEDS_MORE_TESTING
6624         /*
6625          * Always enable abort on incoming L_Qs if this feature is
6626          * supported.  We use this to catch invalid SCB references.
6627          */
6628         if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
6629                 ahd_outb(ahd, LQCTL1, ABORTPENDING);
6630         else
6631 #endif
6632                 ahd_outb(ahd, LQCTL1, 0);
6633
6634         /* All of our queues are empty */
6635         ahd->qoutfifonext = 0;
6636         ahd->qoutfifonext_valid_tag = QOUTFIFO_ENTRY_VALID;
6637         ahd_outb(ahd, QOUTFIFO_ENTRY_VALID_TAG, QOUTFIFO_ENTRY_VALID);
6638         for (i = 0; i < AHD_QOUT_SIZE; i++)
6639                 ahd->qoutfifo[i].valid_tag = 0;
6640         ahd_sync_qoutfifo(ahd, BUS_DMASYNC_PREREAD);
6641
6642         ahd->qinfifonext = 0;
6643         for (i = 0; i < AHD_QIN_SIZE; i++)
6644                 ahd->qinfifo[i] = SCB_LIST_NULL;
6645
6646         if ((ahd->features & AHD_TARGETMODE) != 0) {
6647                 /* All target command blocks start out invalid. */
6648                 for (i = 0; i < AHD_TMODE_CMDS; i++)
6649                         ahd->targetcmds[i].cmd_valid = 0;
6650                 ahd_sync_tqinfifo(ahd, BUS_DMASYNC_PREREAD);
6651                 ahd->tqinfifonext = 1;
6652                 ahd_outb(ahd, KERNEL_TQINPOS, ahd->tqinfifonext - 1);
6653                 ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
6654         }
6655
6656         /* Initialize Scratch Ram. */
6657         ahd_outb(ahd, SEQ_FLAGS, 0);
6658         ahd_outb(ahd, SEQ_FLAGS2, 0);
6659
6660         /* We don't have any waiting selections */
6661         ahd_outw(ahd, WAITING_TID_HEAD, SCB_LIST_NULL);
6662         ahd_outw(ahd, WAITING_TID_TAIL, SCB_LIST_NULL);
6663         ahd_outw(ahd, MK_MESSAGE_SCB, SCB_LIST_NULL);
6664         ahd_outw(ahd, MK_MESSAGE_SCSIID, 0xFF);
6665         for (i = 0; i < AHD_NUM_TARGETS; i++)
6666                 ahd_outw(ahd, WAITING_SCB_TAILS + (2 * i), SCB_LIST_NULL);
6667
6668         /*
6669          * Nobody is waiting to be DMAed into the QOUTFIFO.
6670          */
6671         ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
6672         ahd_outw(ahd, COMPLETE_SCB_DMAINPROG_HEAD, SCB_LIST_NULL);
6673         ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
6674         ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
6675         ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
6676
6677         /*
6678          * The Freeze Count is 0.
6679          */
6680         ahd->qfreeze_cnt = 0;
6681         ahd_outw(ahd, QFREEZE_COUNT, 0);
6682         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, 0);
6683
6684         /*
6685          * Tell the sequencer where it can find our arrays in memory.
6686          */
6687         busaddr = ahd->shared_data_map.physaddr;
6688         ahd_outl(ahd, SHARED_DATA_ADDR, busaddr);
6689         ahd_outl(ahd, QOUTFIFO_NEXT_ADDR, busaddr);
6690
6691         /*
6692          * Setup the allowed SCSI Sequences based on operational mode.
6693          * If we are a target, we'll enable select in operations once
6694          * we've had a lun enabled.
6695          */
6696         scsiseq_template = ENAUTOATNP;
6697         if ((ahd->flags & AHD_INITIATORROLE) != 0)
6698                 scsiseq_template |= ENRSELI;
6699         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq_template);
6700
6701         /* There are no busy SCBs yet. */
6702         for (target = 0; target < AHD_NUM_TARGETS; target++) {
6703                 int lun;
6704
6705                 for (lun = 0; lun < AHD_NUM_LUNS_NONPKT; lun++)
6706                         ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(target, 'A', lun));
6707         }
6708
6709         /*
6710          * Initialize the group code to command length table.
6711          * Vendor Unique codes are set to 0 so we only capture
6712          * the first byte of the cdb.  These can be overridden
6713          * when target mode is enabled.
6714          */
6715         ahd_outb(ahd, CMDSIZE_TABLE, 5);
6716         ahd_outb(ahd, CMDSIZE_TABLE + 1, 9);
6717         ahd_outb(ahd, CMDSIZE_TABLE + 2, 9);
6718         ahd_outb(ahd, CMDSIZE_TABLE + 3, 0);
6719         ahd_outb(ahd, CMDSIZE_TABLE + 4, 15);
6720         ahd_outb(ahd, CMDSIZE_TABLE + 5, 11);
6721         ahd_outb(ahd, CMDSIZE_TABLE + 6, 0);
6722         ahd_outb(ahd, CMDSIZE_TABLE + 7, 0);
6723
6724         /* Tell the sequencer of our initial queue positions */
6725         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
6726         ahd_outb(ahd, QOFF_CTLSTA, SCB_QSIZE_512);
6727         ahd->qinfifonext = 0;
6728         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
6729         ahd_set_hescb_qoff(ahd, 0);
6730         ahd_set_snscb_qoff(ahd, 0);
6731         ahd_set_sescb_qoff(ahd, 0);
6732         ahd_set_sdscb_qoff(ahd, 0);
6733
6734         /*
6735          * Tell the sequencer which SCB will be the next one it receives.
6736          */
6737         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
6738         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
6739
6740         /*
6741          * Default to coalescing disabled.
6742          */
6743         ahd_outw(ahd, INT_COALESCING_CMDCOUNT, 0);
6744         ahd_outw(ahd, CMDS_PENDING, 0);
6745         ahd_update_coalescing_values(ahd, ahd->int_coalescing_timer,
6746                                      ahd->int_coalescing_maxcmds,
6747                                      ahd->int_coalescing_mincmds);
6748         ahd_enable_coalescing(ahd, FALSE);
6749
6750         ahd_loadseq(ahd);
6751         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6752
6753         if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
6754                 u_int negodat3 = ahd_inb(ahd, NEGCONOPTS);
6755
6756                 negodat3 |= ENSLOWCRC;
6757                 ahd_outb(ahd, NEGCONOPTS, negodat3);
6758                 negodat3 = ahd_inb(ahd, NEGCONOPTS);
6759                 if (!(negodat3 & ENSLOWCRC))
6760                         printf("aic79xx: failed to set the SLOWCRC bit\n");
6761                 else
6762                         printf("aic79xx: SLOWCRC bit set\n");
6763         }
6764 }
6765
6766 /*
6767  * Setup default device and controller settings.
6768  * This should only be called if our probe has
6769  * determined that no configuration data is available.
6770  */
6771 int
6772 ahd_default_config(struct ahd_softc *ahd)
6773 {
6774         int     targ;
6775
6776         ahd->our_id = 7;
6777
6778         /*
6779          * Allocate a tstate to house information for our
6780          * initiator presence on the bus as well as the user
6781          * data for any target mode initiator.
6782          */
6783         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
6784                 printf("%s: unable to allocate ahd_tmode_tstate.  "
6785                        "Failing attach\n", ahd_name(ahd));
6786                 return (ENOMEM);
6787         }
6788
6789         for (targ = 0; targ < AHD_NUM_TARGETS; targ++) {
6790                 struct   ahd_devinfo devinfo;
6791                 struct   ahd_initiator_tinfo *tinfo;
6792                 struct   ahd_tmode_tstate *tstate;
6793                 uint16_t target_mask;
6794
6795                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6796                                             targ, &tstate);
6797                 /*
6798                  * We support SPC2 and SPI4.
6799                  */
6800                 tinfo->user.protocol_version = 4;
6801                 tinfo->user.transport_version = 4;
6802
6803                 target_mask = 0x01 << targ;
6804                 ahd->user_discenable |= target_mask;
6805                 tstate->discenable |= target_mask;
6806                 ahd->user_tagenable |= target_mask;
6807 #ifdef AHD_FORCE_160
6808                 tinfo->user.period = AHD_SYNCRATE_DT;
6809 #else
6810                 tinfo->user.period = AHD_SYNCRATE_160;
6811 #endif
6812                 tinfo->user.offset = MAX_OFFSET;
6813                 tinfo->user.ppr_options = MSG_EXT_PPR_RD_STRM
6814                                         | MSG_EXT_PPR_WR_FLOW
6815                                         | MSG_EXT_PPR_HOLD_MCS
6816                                         | MSG_EXT_PPR_IU_REQ
6817                                         | MSG_EXT_PPR_QAS_REQ
6818                                         | MSG_EXT_PPR_DT_REQ;
6819                 if ((ahd->features & AHD_RTI) != 0)
6820                         tinfo->user.ppr_options |= MSG_EXT_PPR_RTI;
6821
6822                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
6823
6824                 /*
6825                  * Start out Async/Narrow/Untagged and with
6826                  * conservative protocol support.
6827                  */
6828                 tinfo->goal.protocol_version = 2;
6829                 tinfo->goal.transport_version = 2;
6830                 tinfo->curr.protocol_version = 2;
6831                 tinfo->curr.transport_version = 2;
6832                 ahd_compile_devinfo(&devinfo, ahd->our_id,
6833                                     targ, CAM_LUN_WILDCARD,
6834                                     'A', ROLE_INITIATOR);
6835                 tstate->tagenable &= ~target_mask;
6836                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6837                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
6838                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
6839                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
6840                                  /*paused*/TRUE);
6841         }
6842         return (0);
6843 }
6844
6845 /*
6846  * Parse device configuration information.
6847  */
6848 int
6849 ahd_parse_cfgdata(struct ahd_softc *ahd, struct seeprom_config *sc)
6850 {
6851         int targ;
6852         int max_targ;
6853
6854         max_targ = sc->max_targets & CFMAXTARG;
6855         ahd->our_id = sc->brtime_id & CFSCSIID;
6856
6857         /*
6858          * Allocate a tstate to house information for our
6859          * initiator presence on the bus as well as the user
6860          * data for any target mode initiator.
6861          */
6862         if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
6863                 printf("%s: unable to allocate ahd_tmode_tstate.  "
6864                        "Failing attach\n", ahd_name(ahd));
6865                 return (ENOMEM);
6866         }
6867
6868         for (targ = 0; targ < max_targ; targ++) {
6869                 struct   ahd_devinfo devinfo;
6870                 struct   ahd_initiator_tinfo *tinfo;
6871                 struct   ahd_transinfo *user_tinfo;
6872                 struct   ahd_tmode_tstate *tstate;
6873                 uint16_t target_mask;
6874
6875                 tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
6876                                             targ, &tstate);
6877                 user_tinfo = &tinfo->user;
6878
6879                 /*
6880                  * We support SPC2 and SPI4.
6881                  */
6882                 tinfo->user.protocol_version = 4;
6883                 tinfo->user.transport_version = 4;
6884
6885                 target_mask = 0x01 << targ;
6886                 ahd->user_discenable &= ~target_mask;
6887                 tstate->discenable &= ~target_mask;
6888                 ahd->user_tagenable &= ~target_mask;
6889                 if (sc->device_flags[targ] & CFDISC) {
6890                         tstate->discenable |= target_mask;
6891                         ahd->user_discenable |= target_mask;
6892                         ahd->user_tagenable |= target_mask;
6893                 } else {
6894                         /*
6895                          * Cannot be packetized without disconnection.
6896                          */
6897                         sc->device_flags[targ] &= ~CFPACKETIZED;
6898                 }
6899
6900                 user_tinfo->ppr_options = 0;
6901                 user_tinfo->period = (sc->device_flags[targ] & CFXFER);
6902                 if (user_tinfo->period < CFXFER_ASYNC) {
6903                         if (user_tinfo->period <= AHD_PERIOD_10MHz)
6904                                 user_tinfo->ppr_options |= MSG_EXT_PPR_DT_REQ;
6905                         user_tinfo->offset = MAX_OFFSET;
6906                 } else  {
6907                         user_tinfo->offset = 0;
6908                         user_tinfo->period = AHD_ASYNC_XFER_PERIOD;
6909                 }
6910 #ifdef AHD_FORCE_160
6911                 if (user_tinfo->period <= AHD_SYNCRATE_160)
6912                         user_tinfo->period = AHD_SYNCRATE_DT;
6913 #endif
6914
6915                 if ((sc->device_flags[targ] & CFPACKETIZED) != 0) {
6916                         user_tinfo->ppr_options |= MSG_EXT_PPR_RD_STRM
6917                                                 |  MSG_EXT_PPR_WR_FLOW
6918                                                 |  MSG_EXT_PPR_HOLD_MCS
6919                                                 |  MSG_EXT_PPR_IU_REQ;
6920                         if ((ahd->features & AHD_RTI) != 0)
6921                                 user_tinfo->ppr_options |= MSG_EXT_PPR_RTI;
6922                 }
6923
6924                 if ((sc->device_flags[targ] & CFQAS) != 0)
6925                         user_tinfo->ppr_options |= MSG_EXT_PPR_QAS_REQ;
6926
6927                 if ((sc->device_flags[targ] & CFWIDEB) != 0)
6928                         user_tinfo->width = MSG_EXT_WDTR_BUS_16_BIT;
6929                 else
6930                         user_tinfo->width = MSG_EXT_WDTR_BUS_8_BIT;
6931 #ifdef AHD_DEBUG
6932                 if ((ahd_debug & AHD_SHOW_MISC) != 0)
6933                         printf("(%d): %x:%x:%x:%x\n", targ, user_tinfo->width,
6934                                user_tinfo->period, user_tinfo->offset,
6935                                user_tinfo->ppr_options);
6936 #endif
6937                 /*
6938                  * Start out Async/Narrow/Untagged and with
6939                  * conservative protocol support.
6940                  */
6941                 tstate->tagenable &= ~target_mask;
6942                 tinfo->goal.protocol_version = 2;
6943                 tinfo->goal.transport_version = 2;
6944                 tinfo->curr.protocol_version = 2;
6945                 tinfo->curr.transport_version = 2;
6946                 ahd_compile_devinfo(&devinfo, ahd->our_id,
6947                                     targ, CAM_LUN_WILDCARD,
6948                                     'A', ROLE_INITIATOR);
6949                 ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6950                               AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
6951                 ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
6952                                  /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
6953                                  /*paused*/TRUE);
6954         }
6955
6956         ahd->flags &= ~AHD_SPCHK_ENB_A;
6957         if (sc->bios_control & CFSPARITY)
6958                 ahd->flags |= AHD_SPCHK_ENB_A;
6959
6960         ahd->flags &= ~AHD_RESET_BUS_A;
6961         if (sc->bios_control & CFRESETB)
6962                 ahd->flags |= AHD_RESET_BUS_A;
6963
6964         ahd->flags &= ~AHD_EXTENDED_TRANS_A;
6965         if (sc->bios_control & CFEXTEND)
6966                 ahd->flags |= AHD_EXTENDED_TRANS_A;
6967
6968         ahd->flags &= ~AHD_BIOS_ENABLED;
6969         if ((sc->bios_control & CFBIOSSTATE) == CFBS_ENABLED)
6970                 ahd->flags |= AHD_BIOS_ENABLED;
6971
6972         ahd->flags &= ~AHD_STPWLEVEL_A;
6973         if ((sc->adapter_control & CFSTPWLEVEL) != 0)
6974                 ahd->flags |= AHD_STPWLEVEL_A;
6975
6976         return (0);
6977 }
6978
6979 /*
6980  * Parse device configuration information.
6981  */
6982 int
6983 ahd_parse_vpddata(struct ahd_softc *ahd, struct vpd_config *vpd)
6984 {
6985         int error;
6986
6987         error = ahd_verify_vpd_cksum(vpd);
6988         if (error == 0)
6989                 return (EINVAL);
6990         if ((vpd->bios_flags & VPDBOOTHOST) != 0)
6991                 ahd->flags |= AHD_BOOT_CHANNEL;
6992         return (0);
6993 }
6994
6995 void
6996 ahd_intr_enable(struct ahd_softc *ahd, int enable)
6997 {
6998         u_int hcntrl;
6999
7000         hcntrl = ahd_inb(ahd, HCNTRL);
7001         hcntrl &= ~INTEN;
7002         ahd->pause &= ~INTEN;
7003         ahd->unpause &= ~INTEN;
7004         if (enable) {
7005                 hcntrl |= INTEN;
7006                 ahd->pause |= INTEN;
7007                 ahd->unpause |= INTEN;
7008         }
7009         ahd_outb(ahd, HCNTRL, hcntrl);
7010 }
7011
7012 void
7013 ahd_update_coalescing_values(struct ahd_softc *ahd, u_int timer, u_int maxcmds,
7014                              u_int mincmds)
7015 {
7016         if (timer > AHD_TIMER_MAX_US)
7017                 timer = AHD_TIMER_MAX_US;
7018         ahd->int_coalescing_timer = timer;
7019
7020         if (maxcmds > AHD_INT_COALESCING_MAXCMDS_MAX)
7021                 maxcmds = AHD_INT_COALESCING_MAXCMDS_MAX;
7022         if (mincmds > AHD_INT_COALESCING_MINCMDS_MAX)
7023                 mincmds = AHD_INT_COALESCING_MINCMDS_MAX;
7024         ahd->int_coalescing_maxcmds = maxcmds;
7025         ahd_outw(ahd, INT_COALESCING_TIMER, timer / AHD_TIMER_US_PER_TICK);
7026         ahd_outb(ahd, INT_COALESCING_MAXCMDS, -maxcmds);
7027         ahd_outb(ahd, INT_COALESCING_MINCMDS, -mincmds);
7028 }
7029
7030 void
7031 ahd_enable_coalescing(struct ahd_softc *ahd, int enable)
7032 {
7033
7034         ahd->hs_mailbox &= ~ENINT_COALESCE;
7035         if (enable)
7036                 ahd->hs_mailbox |= ENINT_COALESCE;
7037         ahd_outb(ahd, HS_MAILBOX, ahd->hs_mailbox);
7038         ahd_flush_device_writes(ahd);
7039         ahd_run_qoutfifo(ahd);
7040 }
7041
7042 /*
7043  * Ensure that the card is paused in a location
7044  * outside of all critical sections and that all
7045  * pending work is completed prior to returning.
7046  * This routine should only be called from outside
7047  * an interrupt context.
7048  */
7049 void
7050 ahd_pause_and_flushwork(struct ahd_softc *ahd)
7051 {
7052         u_int intstat;
7053         u_int maxloops;
7054
7055         maxloops = 1000;
7056         ahd->flags |= AHD_ALL_INTERRUPTS;
7057         ahd_pause(ahd);
7058         /*
7059          * Freeze the outgoing selections.  We do this only
7060          * until we are safely paused without further selections
7061          * pending.
7062          */
7063         ahd->qfreeze_cnt--;
7064         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7065         ahd_outb(ahd, SEQ_FLAGS2, ahd_inb(ahd, SEQ_FLAGS2) | SELECTOUT_QFROZEN);
7066         do {
7067
7068                 ahd_unpause(ahd);
7069                 /*
7070                  * Give the sequencer some time to service
7071                  * any active selections.
7072                  */
7073                 ahd_delay(500);
7074
7075                 ahd_intr(ahd);
7076                 ahd_pause(ahd);
7077                 intstat = ahd_inb(ahd, INTSTAT);
7078                 if ((intstat & INT_PEND) == 0) {
7079                         ahd_clear_critical_section(ahd);
7080                         intstat = ahd_inb(ahd, INTSTAT);
7081                 }
7082         } while (--maxloops
7083               && (intstat != 0xFF || (ahd->features & AHD_REMOVABLE) == 0)
7084               && ((intstat & INT_PEND) != 0
7085                || (ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
7086                || (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) != 0));
7087
7088         if (maxloops == 0) {
7089                 printf("Infinite interrupt loop, INTSTAT = %x",
7090                       ahd_inb(ahd, INTSTAT));
7091         }
7092         ahd->qfreeze_cnt++;
7093         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7094
7095         ahd_flush_qoutfifo(ahd);
7096
7097         ahd_platform_flushwork(ahd);
7098         ahd->flags &= ~AHD_ALL_INTERRUPTS;
7099 }
7100
7101 int
7102 ahd_suspend(struct ahd_softc *ahd)
7103 {
7104
7105         ahd_pause_and_flushwork(ahd);
7106
7107         if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
7108                 ahd_unpause(ahd);
7109                 return (EBUSY);
7110         }
7111         ahd_shutdown(ahd);
7112         return (0);
7113 }
7114
7115 int
7116 ahd_resume(struct ahd_softc *ahd)
7117 {
7118
7119         ahd_reset(ahd, /*reinit*/TRUE);
7120         ahd_intr_enable(ahd, TRUE);
7121         ahd_restart(ahd);
7122         return (0);
7123 }
7124
7125 /************************** Busy Target Table *********************************/
7126 /*
7127  * Set SCBPTR to the SCB that contains the busy
7128  * table entry for TCL.  Return the offset into
7129  * the SCB that contains the entry for TCL.
7130  * saved_scbid is dereferenced and set to the
7131  * scbid that should be restored once manipualtion
7132  * of the TCL entry is complete.
7133  */
7134 static __inline u_int
7135 ahd_index_busy_tcl(struct ahd_softc *ahd, u_int *saved_scbid, u_int tcl)
7136 {
7137         /*
7138          * Index to the SCB that contains the busy entry.
7139          */
7140         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7141         *saved_scbid = ahd_get_scbptr(ahd);
7142         ahd_set_scbptr(ahd, TCL_LUN(tcl)
7143                      | ((TCL_TARGET_OFFSET(tcl) & 0xC) << 4));
7144
7145         /*
7146          * And now calculate the SCB offset to the entry.
7147          * Each entry is 2 bytes wide, hence the
7148          * multiplication by 2.
7149          */
7150         return (((TCL_TARGET_OFFSET(tcl) & 0x3) << 1) + SCB_DISCONNECTED_LISTS);
7151 }
7152
7153 /*
7154  * Return the untagged transaction id for a given target/channel lun.
7155  */
7156 u_int
7157 ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl)
7158 {
7159         u_int scbid;
7160         u_int scb_offset;
7161         u_int saved_scbptr;
7162
7163         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7164         scbid = ahd_inw_scbram(ahd, scb_offset);
7165         ahd_set_scbptr(ahd, saved_scbptr);
7166         return (scbid);
7167 }
7168
7169 void
7170 ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl, u_int scbid)
7171 {
7172         u_int scb_offset;
7173         u_int saved_scbptr;
7174
7175         scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
7176         ahd_outw(ahd, scb_offset, scbid);
7177         ahd_set_scbptr(ahd, saved_scbptr);
7178 }
7179
7180 /************************** SCB and SCB queue management **********************/
7181 int
7182 ahd_match_scb(struct ahd_softc *ahd, struct scb *scb, int target,
7183               char channel, int lun, u_int tag, role_t role)
7184 {
7185         int targ = SCB_GET_TARGET(ahd, scb);
7186         char chan = SCB_GET_CHANNEL(ahd, scb);
7187         int slun = SCB_GET_LUN(scb);
7188         int match;
7189
7190         match = ((chan == channel) || (channel == ALL_CHANNELS));
7191         if (match != 0)
7192                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
7193         if (match != 0)
7194                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
7195         if (match != 0) {
7196 #ifdef AHD_TARGET_MODE
7197                 int group;
7198
7199                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
7200                 if (role == ROLE_INITIATOR) {
7201                         match = (group != XPT_FC_GROUP_TMODE)
7202                               && ((tag == SCB_GET_TAG(scb))
7203                                || (tag == SCB_LIST_NULL));
7204                 } else if (role == ROLE_TARGET) {
7205                         match = (group == XPT_FC_GROUP_TMODE)
7206                               && ((tag == scb->io_ctx->csio.tag_id)
7207                                || (tag == SCB_LIST_NULL));
7208                 }
7209 #else /* !AHD_TARGET_MODE */
7210                 match = ((tag == SCB_GET_TAG(scb)) || (tag == SCB_LIST_NULL));
7211 #endif /* AHD_TARGET_MODE */
7212         }
7213
7214         return match;
7215 }
7216
7217 void
7218 ahd_freeze_devq(struct ahd_softc *ahd, struct scb *scb)
7219 {
7220         int     target;
7221         char    channel;
7222         int     lun;
7223
7224         target = SCB_GET_TARGET(ahd, scb);
7225         lun = SCB_GET_LUN(scb);
7226         channel = SCB_GET_CHANNEL(ahd, scb);
7227
7228         ahd_search_qinfifo(ahd, target, channel, lun,
7229                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
7230                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
7231
7232         ahd_platform_freeze_devq(ahd, scb);
7233 }
7234
7235 void
7236 ahd_qinfifo_requeue_tail(struct ahd_softc *ahd, struct scb *scb)
7237 {
7238         struct scb      *prev_scb;
7239         ahd_mode_state   saved_modes;
7240
7241         saved_modes = ahd_save_modes(ahd);
7242         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7243         prev_scb = NULL;
7244         if (ahd_qinfifo_count(ahd) != 0) {
7245                 u_int prev_tag;
7246                 u_int prev_pos;
7247
7248                 prev_pos = AHD_QIN_WRAP(ahd->qinfifonext - 1);
7249                 prev_tag = ahd->qinfifo[prev_pos];
7250                 prev_scb = ahd_lookup_scb(ahd, prev_tag);
7251         }
7252         ahd_qinfifo_requeue(ahd, prev_scb, scb);
7253         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7254         ahd_restore_modes(ahd, saved_modes);
7255 }
7256
7257 static void
7258 ahd_qinfifo_requeue(struct ahd_softc *ahd, struct scb *prev_scb,
7259                     struct scb *scb)
7260 {
7261         if (prev_scb == NULL) {
7262                 uint32_t busaddr;
7263
7264                 busaddr = ahd_le32toh(scb->hscb->hscb_busaddr);
7265                 ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7266         } else {
7267                 prev_scb->hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
7268                 ahd_sync_scb(ahd, prev_scb,
7269                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
7270         }
7271         ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
7272         ahd->qinfifonext++;
7273         scb->hscb->next_hscb_busaddr = ahd->next_queued_hscb->hscb_busaddr;
7274         ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
7275 }
7276
7277 static int
7278 ahd_qinfifo_count(struct ahd_softc *ahd)
7279 {
7280         u_int qinpos;
7281         u_int wrap_qinpos;
7282         u_int wrap_qinfifonext;
7283
7284         AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
7285         qinpos = ahd_get_snscb_qoff(ahd);
7286         wrap_qinpos = AHD_QIN_WRAP(qinpos);
7287         wrap_qinfifonext = AHD_QIN_WRAP(ahd->qinfifonext);
7288         if (wrap_qinfifonext >= wrap_qinpos)
7289                 return (wrap_qinfifonext - wrap_qinpos);
7290         else
7291                 return (wrap_qinfifonext
7292                       + NUM_ELEMENTS(ahd->qinfifo) - wrap_qinpos);
7293 }
7294
7295 void
7296 ahd_reset_cmds_pending(struct ahd_softc *ahd)
7297 {
7298         struct          scb *scb;
7299         ahd_mode_state  saved_modes;
7300         u_int           pending_cmds;
7301
7302         saved_modes = ahd_save_modes(ahd);
7303         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7304
7305         /*
7306          * Don't count any commands as outstanding that the
7307          * sequencer has already marked for completion.
7308          */
7309         ahd_flush_qoutfifo(ahd);
7310
7311         pending_cmds = 0;
7312         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
7313                 pending_cmds++;
7314         }
7315         ahd_outw(ahd, CMDS_PENDING, pending_cmds - ahd_qinfifo_count(ahd));
7316         ahd_restore_modes(ahd, saved_modes);
7317         ahd->flags &= ~AHD_UPDATE_PEND_CMDS;
7318 }
7319
7320 void
7321 ahd_done_with_status(struct ahd_softc *ahd, struct scb *scb, uint32_t status)
7322 {
7323         cam_status ostat;
7324         cam_status cstat;
7325
7326         ostat = ahd_get_transaction_status(scb);
7327         if (ostat == CAM_REQ_INPROG)
7328                 ahd_set_transaction_status(scb, status);
7329         cstat = ahd_get_transaction_status(scb);
7330         if (cstat != CAM_REQ_CMP)
7331                 ahd_freeze_scb(scb);
7332         ahd_done(ahd, scb);
7333 }
7334
7335 int
7336 ahd_search_qinfifo(struct ahd_softc *ahd, int target, char channel,
7337                    int lun, u_int tag, role_t role, uint32_t status,
7338                    ahd_search_action action)
7339 {
7340         struct scb      *scb;
7341         struct scb      *mk_msg_scb;
7342         struct scb      *prev_scb;
7343         ahd_mode_state   saved_modes;
7344         u_int            qinstart;
7345         u_int            qinpos;
7346         u_int            qintail;
7347         u_int            tid_next;
7348         u_int            tid_prev;
7349         u_int            scbid;
7350         u_int            seq_flags2;
7351         u_int            savedscbptr;
7352         uint32_t         busaddr;
7353         int              found;
7354         int              targets;
7355
7356         /* Must be in CCHAN mode */
7357         saved_modes = ahd_save_modes(ahd);
7358         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7359
7360         /*
7361          * Halt any pending SCB DMA.  The sequencer will reinitiate
7362          * this dma if the qinfifo is not empty once we unpause.
7363          */
7364         if ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN|CCSCBDIR))
7365          == (CCARREN|CCSCBEN|CCSCBDIR)) {
7366                 ahd_outb(ahd, CCSCBCTL,
7367                          ahd_inb(ahd, CCSCBCTL) & ~(CCARREN|CCSCBEN));
7368                 while ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN)) != 0)
7369                         ;
7370         }
7371         /* Determine sequencer's position in the qinfifo. */
7372         qintail = AHD_QIN_WRAP(ahd->qinfifonext);
7373         qinstart = ahd_get_snscb_qoff(ahd);
7374         qinpos = AHD_QIN_WRAP(qinstart);
7375         found = 0;
7376         prev_scb = NULL;
7377
7378         if (action == SEARCH_PRINT) {
7379                 printf("qinstart = %d qinfifonext = %d\nQINFIFO:",
7380                        qinstart, ahd->qinfifonext);
7381         }
7382
7383         /*
7384          * Start with an empty queue.  Entries that are not chosen
7385          * for removal will be re-added to the queue as we go.
7386          */
7387         ahd->qinfifonext = qinstart;
7388         busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
7389         ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7390
7391         while (qinpos != qintail) {
7392                 scb = ahd_lookup_scb(ahd, ahd->qinfifo[qinpos]);
7393                 if (scb == NULL) {
7394                         printf("qinpos = %d, SCB index = %d\n",
7395                                 qinpos, ahd->qinfifo[qinpos]);
7396                         panic("Loop 1\n");
7397                 }
7398
7399                 if (ahd_match_scb(ahd, scb, target, channel, lun, tag, role)) {
7400                         /*
7401                          * We found an scb that needs to be acted on.
7402                          */
7403                         found++;
7404                         switch (action) {
7405                         case SEARCH_COMPLETE:
7406                                 if ((scb->flags & SCB_ACTIVE) == 0)
7407                                         printf("Inactive SCB in qinfifo\n");
7408                                 ahd_done_with_status(ahd, scb, status);
7409                                 /* FALLTHROUGH */
7410                         case SEARCH_REMOVE:
7411                                 break;
7412                         case SEARCH_PRINT:
7413                                 printf(" 0x%x", ahd->qinfifo[qinpos]);
7414                                 /* FALLTHROUGH */
7415                         case SEARCH_COUNT:
7416                                 ahd_qinfifo_requeue(ahd, prev_scb, scb);
7417                                 prev_scb = scb;
7418                                 break;
7419                         }
7420                 } else {
7421                         ahd_qinfifo_requeue(ahd, prev_scb, scb);
7422                         prev_scb = scb;
7423                 }
7424                 qinpos = AHD_QIN_WRAP(qinpos+1);
7425         }
7426
7427         ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7428
7429         if (action == SEARCH_PRINT)
7430                 printf("\nWAITING_TID_QUEUES:\n");
7431
7432         /*
7433          * Search waiting for selection lists.  We traverse the
7434          * list of "their ids" waiting for selection and, if
7435          * appropriate, traverse the SCBs of each "their id"
7436          * looking for matches.
7437          */
7438         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7439         seq_flags2 = ahd_inb(ahd, SEQ_FLAGS2);
7440         if ((seq_flags2 & PENDING_MK_MESSAGE) != 0) {
7441                 scbid = ahd_inw(ahd, MK_MESSAGE_SCB);
7442                 mk_msg_scb = ahd_lookup_scb(ahd, scbid);
7443         } else
7444                 mk_msg_scb = NULL;
7445         savedscbptr = ahd_get_scbptr(ahd);
7446         tid_next = ahd_inw(ahd, WAITING_TID_HEAD);
7447         tid_prev = SCB_LIST_NULL;
7448         targets = 0;
7449         for (scbid = tid_next; !SCBID_IS_NULL(scbid); scbid = tid_next) {
7450                 u_int tid_head;
7451                 u_int tid_tail;
7452
7453                 targets++;
7454                 if (targets > AHD_NUM_TARGETS)
7455                         panic("TID LIST LOOP");
7456
7457                 if (scbid >= ahd->scb_data.numscbs) {
7458                         printf("%s: Waiting TID List inconsistency. "
7459                                "SCB index == 0x%x, yet numscbs == 0x%x.",
7460                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
7461                         ahd_dump_card_state(ahd);
7462                         panic("for safety");
7463                 }
7464                 scb = ahd_lookup_scb(ahd, scbid);
7465                 if (scb == NULL) {
7466                         printf("%s: SCB = 0x%x Not Active!\n",
7467                                ahd_name(ahd), scbid);
7468                         panic("Waiting TID List traversal\n");
7469                 }
7470                 ahd_set_scbptr(ahd, scbid);
7471                 tid_next = ahd_inw_scbram(ahd, SCB_NEXT2);
7472                 if (ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
7473                                   SCB_LIST_NULL, ROLE_UNKNOWN) == 0) {
7474                         tid_prev = scbid;
7475                         continue;
7476                 }
7477
7478                 /*
7479                  * We found a list of scbs that needs to be searched.
7480                  */
7481                 if (action == SEARCH_PRINT)
7482                         printf("       %d ( ", SCB_GET_TARGET(ahd, scb));
7483                 tid_head = scbid;
7484                 found += ahd_search_scb_list(ahd, target, channel,
7485                                              lun, tag, role, status,
7486                                              action, &tid_head, &tid_tail,
7487                                              SCB_GET_TARGET(ahd, scb));
7488                 /*
7489                  * Check any MK_MESSAGE SCB that is still waiting to
7490                  * enter this target's waiting for selection queue.
7491                  */
7492                 if (mk_msg_scb != NULL
7493                  && ahd_match_scb(ahd, mk_msg_scb, target, channel,
7494                                   lun, tag, role)) {
7495
7496                         /*
7497                          * We found an scb that needs to be acted on.
7498                          */
7499                         found++;
7500                         switch (action) {
7501                         case SEARCH_COMPLETE:
7502                                 if ((mk_msg_scb->flags & SCB_ACTIVE) == 0)
7503                                         printf("Inactive SCB pending MK_MSG\n");
7504                                 ahd_done_with_status(ahd, mk_msg_scb, status);
7505                                 /* FALLTHROUGH */
7506                         case SEARCH_REMOVE:
7507                         {
7508                                 u_int tail_offset;
7509
7510                                 printf("Removing MK_MSG scb\n");
7511
7512                                 /*
7513                                  * Reset our tail to the tail of the
7514                                  * main per-target list.
7515                                  */
7516                                 tail_offset = WAITING_SCB_TAILS
7517                                     + (2 * SCB_GET_TARGET(ahd, mk_msg_scb));
7518                                 ahd_outw(ahd, tail_offset, tid_tail);
7519
7520                                 seq_flags2 &= ~PENDING_MK_MESSAGE;
7521                                 ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
7522                                 ahd_outw(ahd, CMDS_PENDING,
7523                                          ahd_inw(ahd, CMDS_PENDING)-1);
7524                                 mk_msg_scb = NULL;
7525                                 break;
7526                         }
7527                         case SEARCH_PRINT:
7528                                 printf(" 0x%x", SCB_GET_TAG(scb));
7529                                 /* FALLTHROUGH */
7530                         case SEARCH_COUNT:
7531                                 break;
7532                         }
7533                 }
7534
7535                 if (mk_msg_scb != NULL
7536                  && SCBID_IS_NULL(tid_head)
7537                  && ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
7538                                   SCB_LIST_NULL, ROLE_UNKNOWN)) {
7539
7540                         /*
7541                          * When removing the last SCB for a target
7542                          * queue with a pending MK_MESSAGE scb, we
7543                          * must queue the MK_MESSAGE scb.
7544                          */
7545                         printf("Queueing mk_msg_scb\n");
7546                         tid_head = ahd_inw(ahd, MK_MESSAGE_SCB);
7547                         seq_flags2 &= ~PENDING_MK_MESSAGE;
7548                         ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
7549                         mk_msg_scb = NULL;
7550                 }
7551                 if (tid_head != scbid)
7552                         ahd_stitch_tid_list(ahd, tid_prev, tid_head, tid_next);
7553                 if (!SCBID_IS_NULL(tid_head))
7554                         tid_prev = tid_head;
7555                 if (action == SEARCH_PRINT)
7556                         printf(")\n");
7557         }
7558
7559         /* Restore saved state. */
7560         ahd_set_scbptr(ahd, savedscbptr);
7561         ahd_restore_modes(ahd, saved_modes);
7562         return (found);
7563 }
7564
7565 static int
7566 ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel,
7567                     int lun, u_int tag, role_t role, uint32_t status,
7568                     ahd_search_action action, u_int *list_head,
7569                     u_int *list_tail, u_int tid)
7570 {
7571         struct  scb *scb;
7572         u_int   scbid;
7573         u_int   next;
7574         u_int   prev;
7575         int     found;
7576
7577         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7578         found = 0;
7579         prev = SCB_LIST_NULL;
7580         next = *list_head;
7581         *list_tail = SCB_LIST_NULL;
7582         for (scbid = next; !SCBID_IS_NULL(scbid); scbid = next) {
7583                 if (scbid >= ahd->scb_data.numscbs) {
7584                         printf("%s:SCB List inconsistency. "
7585                                "SCB == 0x%x, yet numscbs == 0x%x.",
7586                                ahd_name(ahd), scbid, ahd->scb_data.numscbs);
7587                         ahd_dump_card_state(ahd);
7588                         panic("for safety");
7589                 }
7590                 scb = ahd_lookup_scb(ahd, scbid);
7591                 if (scb == NULL) {
7592                         printf("%s: SCB = %d Not Active!\n",
7593                                ahd_name(ahd), scbid);
7594                         panic("Waiting List traversal\n");
7595                 }
7596                 ahd_set_scbptr(ahd, scbid);
7597                 *list_tail = scbid;
7598                 next = ahd_inw_scbram(ahd, SCB_NEXT);
7599                 if (ahd_match_scb(ahd, scb, target, channel,
7600                                   lun, SCB_LIST_NULL, role) == 0) {
7601                         prev = scbid;
7602                         continue;
7603                 }
7604                 found++;
7605                 switch (action) {
7606                 case SEARCH_COMPLETE:
7607                         if ((scb->flags & SCB_ACTIVE) == 0)
7608                                 printf("Inactive SCB in Waiting List\n");
7609                         ahd_done_with_status(ahd, scb, status);
7610                         /* FALLTHROUGH */
7611                 case SEARCH_REMOVE:
7612                         ahd_rem_wscb(ahd, scbid, prev, next, tid);
7613                         *list_tail = prev;
7614                         if (SCBID_IS_NULL(prev))
7615                                 *list_head = next;
7616                         break;
7617                 case SEARCH_PRINT:
7618                         printf("0x%x ", scbid);
7619                 case SEARCH_COUNT:
7620                         prev = scbid;
7621                         break;
7622                 }
7623                 if (found > AHD_SCB_MAX)
7624                         panic("SCB LIST LOOP");
7625         }
7626         if (action == SEARCH_COMPLETE
7627          || action == SEARCH_REMOVE)
7628                 ahd_outw(ahd, CMDS_PENDING, ahd_inw(ahd, CMDS_PENDING) - found);
7629         return (found);
7630 }
7631
7632 static void
7633 ahd_stitch_tid_list(struct ahd_softc *ahd, u_int tid_prev,
7634                     u_int tid_cur, u_int tid_next)
7635 {
7636         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7637
7638         if (SCBID_IS_NULL(tid_cur)) {
7639
7640                 /* Bypass current TID list */
7641                 if (SCBID_IS_NULL(tid_prev)) {
7642                         ahd_outw(ahd, WAITING_TID_HEAD, tid_next);
7643                 } else {
7644                         ahd_set_scbptr(ahd, tid_prev);
7645                         ahd_outw(ahd, SCB_NEXT2, tid_next);
7646                 }
7647                 if (SCBID_IS_NULL(tid_next))
7648                         ahd_outw(ahd, WAITING_TID_TAIL, tid_prev);
7649         } else {
7650
7651                 /* Stitch through tid_cur */
7652                 if (SCBID_IS_NULL(tid_prev)) {
7653                         ahd_outw(ahd, WAITING_TID_HEAD, tid_cur);
7654                 } else {
7655                         ahd_set_scbptr(ahd, tid_prev);
7656                         ahd_outw(ahd, SCB_NEXT2, tid_cur);
7657                 }
7658                 ahd_set_scbptr(ahd, tid_cur);
7659                 ahd_outw(ahd, SCB_NEXT2, tid_next);
7660
7661                 if (SCBID_IS_NULL(tid_next))
7662                         ahd_outw(ahd, WAITING_TID_TAIL, tid_cur);
7663         }
7664 }
7665
7666 /*
7667  * Manipulate the waiting for selection list and return the
7668  * scb that follows the one that we remove.
7669  */
7670 static u_int
7671 ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
7672              u_int prev, u_int next, u_int tid)
7673 {
7674         u_int tail_offset;
7675
7676         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7677         if (!SCBID_IS_NULL(prev)) {
7678                 ahd_set_scbptr(ahd, prev);
7679                 ahd_outw(ahd, SCB_NEXT, next);
7680         }
7681
7682         /*
7683          * SCBs that have MK_MESSAGE set in them may
7684          * cause the tail pointer to be updated without
7685          * setting the next pointer of the previous tail.
7686          * Only clear the tail if the removed SCB was
7687          * the tail.
7688          */
7689         tail_offset = WAITING_SCB_TAILS + (2 * tid);
7690         if (SCBID_IS_NULL(next)
7691          && ahd_inw(ahd, tail_offset) == scbid)
7692                 ahd_outw(ahd, tail_offset, prev);
7693
7694         ahd_add_scb_to_free_list(ahd, scbid);
7695         return (next);
7696 }
7697
7698 /*
7699  * Add the SCB as selected by SCBPTR onto the on chip list of
7700  * free hardware SCBs.  This list is empty/unused if we are not
7701  * performing SCB paging.
7702  */
7703 static void
7704 ahd_add_scb_to_free_list(struct ahd_softc *ahd, u_int scbid)
7705 {
7706 /* XXX Need some other mechanism to designate "free". */
7707         /*
7708          * Invalidate the tag so that our abort
7709          * routines don't think it's active.
7710         ahd_outb(ahd, SCB_TAG, SCB_LIST_NULL);
7711          */
7712 }
7713
7714 /******************************** Error Handling ******************************/
7715 /*
7716  * Abort all SCBs that match the given description (target/channel/lun/tag),
7717  * setting their status to the passed in status if the status has not already
7718  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
7719  * is paused before it is called.
7720  */
7721 int
7722 ahd_abort_scbs(struct ahd_softc *ahd, int target, char channel,
7723                int lun, u_int tag, role_t role, uint32_t status)
7724 {
7725         struct          scb *scbp;
7726         struct          scb *scbp_next;
7727         u_int           i, j;
7728         u_int           maxtarget;
7729         u_int           minlun;
7730         u_int           maxlun;
7731         int             found;
7732         ahd_mode_state  saved_modes;
7733
7734         /* restore this when we're done */
7735         saved_modes = ahd_save_modes(ahd);
7736         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7737
7738         found = ahd_search_qinfifo(ahd, target, channel, lun, SCB_LIST_NULL,
7739                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
7740
7741         /*
7742          * Clean out the busy target table for any untagged commands.
7743          */
7744         i = 0;
7745         maxtarget = 16;
7746         if (target != CAM_TARGET_WILDCARD) {
7747                 i = target;
7748                 if (channel == 'B')
7749                         i += 8;
7750                 maxtarget = i + 1;
7751         }
7752
7753         if (lun == CAM_LUN_WILDCARD) {
7754                 minlun = 0;
7755                 maxlun = AHD_NUM_LUNS_NONPKT;
7756         } else if (lun >= AHD_NUM_LUNS_NONPKT) {
7757                 minlun = maxlun = 0;
7758         } else {
7759                 minlun = lun;
7760                 maxlun = lun + 1;
7761         }
7762
7763         if (role != ROLE_TARGET) {
7764                 for (;i < maxtarget; i++) {
7765                         for (j = minlun;j < maxlun; j++) {
7766                                 u_int scbid;
7767                                 u_int tcl;
7768
7769                                 tcl = BUILD_TCL_RAW(i, 'A', j);
7770                                 scbid = ahd_find_busy_tcl(ahd, tcl);
7771                                 scbp = ahd_lookup_scb(ahd, scbid);
7772                                 if (scbp == NULL
7773                                  || ahd_match_scb(ahd, scbp, target, channel,
7774                                                   lun, tag, role) == 0)
7775                                         continue;
7776                                 ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(i, 'A', j));
7777                         }
7778                 }
7779         }
7780
7781         /*
7782          * Don't abort commands that have already completed,
7783          * but haven't quite made it up to the host yet.
7784          */
7785         ahd_flush_qoutfifo(ahd);
7786
7787         /*
7788          * Go through the pending CCB list and look for
7789          * commands for this target that are still active.
7790          * These are other tagged commands that were
7791          * disconnected when the reset occurred.
7792          */
7793         scbp_next = LIST_FIRST(&ahd->pending_scbs);
7794         while (scbp_next != NULL) {
7795                 scbp = scbp_next;
7796                 scbp_next = LIST_NEXT(scbp, pending_links);
7797                 if (ahd_match_scb(ahd, scbp, target, channel, lun, tag, role)) {
7798                         cam_status ostat;
7799
7800                         ostat = ahd_get_transaction_status(scbp);
7801                         if (ostat == CAM_REQ_INPROG)
7802                                 ahd_set_transaction_status(scbp, status);
7803                         if (ahd_get_transaction_status(scbp) != CAM_REQ_CMP)
7804                                 ahd_freeze_scb(scbp);
7805                         if ((scbp->flags & SCB_ACTIVE) == 0)
7806                                 printf("Inactive SCB on pending list\n");
7807                         ahd_done(ahd, scbp);
7808                         found++;
7809                 }
7810         }
7811         ahd_restore_modes(ahd, saved_modes);
7812         ahd_platform_abort_scbs(ahd, target, channel, lun, tag, role, status);
7813         ahd->flags |= AHD_UPDATE_PEND_CMDS;
7814         return found;
7815 }
7816
7817 static void
7818 ahd_reset_current_bus(struct ahd_softc *ahd)
7819 {
7820         uint8_t scsiseq;
7821
7822         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7823         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~ENSCSIRST);
7824         scsiseq = ahd_inb(ahd, SCSISEQ0) & ~(ENSELO|ENARBO|SCSIRSTO);
7825         ahd_outb(ahd, SCSISEQ0, scsiseq | SCSIRSTO);
7826         ahd_flush_device_writes(ahd);
7827         ahd_delay(AHD_BUSRESET_DELAY);
7828         /* Turn off the bus reset */
7829         ahd_outb(ahd, SCSISEQ0, scsiseq);
7830         ahd_flush_device_writes(ahd);
7831         ahd_delay(AHD_BUSRESET_DELAY);
7832         if ((ahd->bugs & AHD_SCSIRST_BUG) != 0) {
7833                 /*
7834                  * 2A Razor #474
7835                  * Certain chip state is not cleared for
7836                  * SCSI bus resets that we initiate, so
7837                  * we must reset the chip.
7838                  */
7839                 ahd_reset(ahd, /*reinit*/TRUE);
7840                 ahd_intr_enable(ahd, /*enable*/TRUE);
7841                 AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7842         }
7843
7844         ahd_clear_intstat(ahd);
7845 }
7846
7847 int
7848 ahd_reset_channel(struct ahd_softc *ahd, char channel, int initiate_reset)
7849 {
7850         struct  ahd_devinfo devinfo;
7851         u_int   initiator;
7852         u_int   target;
7853         u_int   max_scsiid;
7854         int     found;
7855         u_int   fifo;
7856         u_int   next_fifo;
7857
7858         ahd->pending_device = NULL;
7859
7860         ahd_compile_devinfo(&devinfo,
7861                             CAM_TARGET_WILDCARD,
7862                             CAM_TARGET_WILDCARD,
7863                             CAM_LUN_WILDCARD,
7864                             channel, ROLE_UNKNOWN);
7865         ahd_pause(ahd);
7866
7867         /* Make sure the sequencer is in a safe location. */
7868         ahd_clear_critical_section(ahd);
7869
7870 #ifdef AHD_TARGET_MODE
7871         if ((ahd->flags & AHD_TARGETROLE) != 0) {
7872                 ahd_run_tqinfifo(ahd, /*paused*/TRUE);
7873         }
7874 #endif
7875         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7876
7877         /*
7878          * Disable selections so no automatic hardware
7879          * functions will modify chip state.
7880          */
7881         ahd_outb(ahd, SCSISEQ0, 0);
7882         ahd_outb(ahd, SCSISEQ1, 0);
7883
7884         /*
7885          * Safely shut down our DMA engines.  Always start with
7886          * the FIFO that is not currently active (if any are
7887          * actively connected).
7888          */
7889         next_fifo = fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
7890         if (next_fifo > CURRFIFO_1)
7891                 /* If disconneced, arbitrarily start with FIFO1. */
7892                 next_fifo = fifo = 0;
7893         do {
7894                 next_fifo ^= CURRFIFO_1;
7895                 ahd_set_modes(ahd, next_fifo, next_fifo);
7896                 ahd_outb(ahd, DFCNTRL,
7897                          ahd_inb(ahd, DFCNTRL) & ~(SCSIEN|HDMAEN));
7898                 while ((ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0)
7899                         ahd_delay(10);
7900                 /*
7901                  * Set CURRFIFO to the now inactive channel.
7902                  */
7903                 ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7904                 ahd_outb(ahd, DFFSTAT, next_fifo);
7905         } while (next_fifo != fifo);
7906
7907         /*
7908          * Reset the bus if we are initiating this reset
7909          */
7910         ahd_clear_msg_state(ahd);
7911         ahd_outb(ahd, SIMODE1,
7912                  ahd_inb(ahd, SIMODE1) & ~(ENBUSFREE|ENSCSIRST));
7913
7914         if (initiate_reset)
7915                 ahd_reset_current_bus(ahd);
7916
7917         ahd_clear_intstat(ahd);
7918
7919         /*
7920          * Clean up all the state information for the
7921          * pending transactions on this bus.
7922          */
7923         found = ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, channel,
7924                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
7925                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
7926
7927         /*
7928          * Cleanup anything left in the FIFOs.
7929          */
7930         ahd_clear_fifo(ahd, 0);
7931         ahd_clear_fifo(ahd, 1);
7932
7933         /*
7934          * Revert to async/narrow transfers until we renegotiate.
7935          */
7936         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
7937         for (target = 0; target <= max_scsiid; target++) {
7938
7939                 if (ahd->enabled_targets[target] == NULL)
7940                         continue;
7941                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
7942                         struct ahd_devinfo devinfo;
7943
7944                         ahd_compile_devinfo(&devinfo, target, initiator,
7945                                             CAM_LUN_WILDCARD,
7946                                             'A', ROLE_UNKNOWN);
7947                         ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7948                                       AHD_TRANS_CUR, /*paused*/TRUE);
7949                         ahd_set_syncrate(ahd, &devinfo, /*period*/0,
7950                                          /*offset*/0, /*ppr_options*/0,
7951                                          AHD_TRANS_CUR, /*paused*/TRUE);
7952                 }
7953         }
7954
7955 #ifdef AHD_TARGET_MODE
7956         max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
7957
7958         /*
7959          * Send an immediate notify ccb to all target more peripheral
7960          * drivers affected by this action.
7961          */
7962         for (target = 0; target <= max_scsiid; target++) {
7963                 struct ahd_tmode_tstate* tstate;
7964                 u_int lun;
7965
7966                 tstate = ahd->enabled_targets[target];
7967                 if (tstate == NULL)
7968                         continue;
7969                 for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
7970                         struct ahd_tmode_lstate* lstate;
7971
7972                         lstate = tstate->enabled_luns[lun];
7973                         if (lstate == NULL)
7974                                 continue;
7975
7976                         ahd_queue_lstate_event(ahd, lstate, CAM_TARGET_WILDCARD,
7977                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
7978                         ahd_send_lstate_events(ahd, lstate);
7979                 }
7980         }
7981 #endif
7982         /* Notify the XPT that a bus reset occurred */
7983         ahd_send_async(ahd, devinfo.channel, CAM_TARGET_WILDCARD,
7984                        CAM_LUN_WILDCARD, AC_BUS_RESET, NULL);
7985         ahd_restart(ahd);
7986         /*
7987          * Freeze the SIMQ until our poller can determine that
7988          * the bus reset has really gone away.  We set the initial
7989          * timer to 0 to have the check performed as soon as possible
7990          * from the timer context.
7991          */
7992         if ((ahd->flags & AHD_RESET_POLL_ACTIVE) == 0) {
7993                 ahd->flags |= AHD_RESET_POLL_ACTIVE;
7994                 ahd_freeze_simq(ahd);
7995                 ahd_timer_reset(&ahd->reset_timer, 0, ahd_reset_poll, ahd);
7996         }
7997         return (found);
7998 }
7999
8000
8001 #define AHD_RESET_POLL_US 1000
8002 static void
8003 ahd_reset_poll(void *arg)
8004 {
8005         struct  ahd_softc *ahd = arg;
8006         u_int   scsiseq1;
8007         u_long  s;
8008
8009         ahd_lock(ahd, &s);
8010         ahd_pause(ahd);
8011         ahd_update_modes(ahd);
8012         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8013         ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
8014         if ((ahd_inb(ahd, SSTAT1) & SCSIRSTI) != 0) {
8015                 ahd_timer_reset(&ahd->reset_timer, AHD_RESET_POLL_US,
8016                                 ahd_reset_poll, ahd);
8017                 ahd_unpause(ahd);
8018                 ahd_unlock(ahd, &s);
8019                 return;
8020         }
8021
8022         /* Reset is now low.  Complete chip reinitialization. */
8023         ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) | ENSCSIRST);
8024         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8025         ahd_outb(ahd, SCSISEQ1, scsiseq1 & (ENSELI|ENRSELI|ENAUTOATNP));
8026         ahd_unpause(ahd);
8027         ahd->flags &= ~AHD_RESET_POLL_ACTIVE;
8028         ahd_unlock(ahd, &s);
8029         ahd_release_simq(ahd);
8030 }
8031
8032 /**************************** Statistics Processing ***************************/
8033 static void
8034 ahd_stat_timer(void *arg)
8035 {
8036         struct  ahd_softc *ahd = arg;
8037         u_long  s;
8038         int     enint_coal;
8039
8040         ahd_lock(ahd, &s);
8041
8042         enint_coal = ahd->hs_mailbox & ENINT_COALESCE;
8043         if (ahd->cmdcmplt_total > ahd->int_coalescing_threshold)
8044                 enint_coal |= ENINT_COALESCE;
8045         else if (ahd->cmdcmplt_total < ahd->int_coalescing_stop_threshold)
8046                 enint_coal &= ~ENINT_COALESCE;
8047
8048         if (enint_coal != (ahd->hs_mailbox & ENINT_COALESCE)) {
8049                 ahd_enable_coalescing(ahd, enint_coal);
8050 #ifdef AHD_DEBUG
8051                 if ((ahd_debug & AHD_SHOW_INT_COALESCING) != 0)
8052                         printf("%s: Interrupt coalescing "
8053                                "now %sabled. Cmds %d\n",
8054                                ahd_name(ahd),
8055                                (enint_coal & ENINT_COALESCE) ? "en" : "dis",
8056                                ahd->cmdcmplt_total);
8057 #endif
8058         }
8059
8060         ahd->cmdcmplt_bucket = (ahd->cmdcmplt_bucket+1) & (AHD_STAT_BUCKETS-1);
8061         ahd->cmdcmplt_total -= ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket];
8062         ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket] = 0;
8063         ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US,
8064                         ahd_stat_timer, ahd);
8065         ahd_unlock(ahd, &s);
8066 }
8067
8068 /****************************** Status Processing *****************************/
8069 void
8070 ahd_handle_scb_status(struct ahd_softc *ahd, struct scb *scb)
8071 {
8072         if (scb->hscb->shared_data.istatus.scsi_status != 0) {
8073                 ahd_handle_scsi_status(ahd, scb);
8074         } else {
8075                 ahd_calc_residual(ahd, scb);
8076                 ahd_done(ahd, scb);
8077         }
8078 }
8079
8080 void
8081 ahd_handle_scsi_status(struct ahd_softc *ahd, struct scb *scb)
8082 {
8083         struct  hardware_scb *hscb;
8084         int     paused;
8085
8086         /*
8087          * The sequencer freezes its select-out queue
8088          * anytime a SCSI status error occurs.  We must
8089          * handle the error and increment our qfreeze count
8090          * to allow the sequencer to continue.  We don't
8091          * bother clearing critical sections here since all
8092          * operations are on data structures that the sequencer
8093          * is not touching once the queue is frozen.
8094          */
8095         hscb = scb->hscb;
8096
8097         if (ahd_is_paused(ahd)) {
8098                 paused = 1;
8099         } else {
8100                 paused = 0;
8101                 ahd_pause(ahd);
8102         }
8103
8104         /* Freeze the queue until the client sees the error. */
8105         ahd_freeze_devq(ahd, scb);
8106         ahd_freeze_scb(scb);
8107         ahd->qfreeze_cnt++;
8108         ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
8109
8110         if (paused == 0)
8111                 ahd_unpause(ahd);
8112
8113         /* Don't want to clobber the original sense code */
8114         if ((scb->flags & SCB_SENSE) != 0) {
8115                 /*
8116                  * Clear the SCB_SENSE Flag and perform
8117                  * a normal command completion.
8118                  */
8119                 scb->flags &= ~SCB_SENSE;
8120                 ahd_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
8121                 ahd_done(ahd, scb);
8122                 return;
8123         }
8124         ahd_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
8125         ahd_set_scsi_status(scb, hscb->shared_data.istatus.scsi_status);
8126         switch (hscb->shared_data.istatus.scsi_status) {
8127         case STATUS_PKT_SENSE:
8128         {
8129                 struct scsi_status_iu_header *siu;
8130
8131                 ahd_sync_sense(ahd, scb, BUS_DMASYNC_POSTREAD);
8132                 siu = (struct scsi_status_iu_header *)scb->sense_data;
8133                 ahd_set_scsi_status(scb, siu->status);
8134 #ifdef AHD_DEBUG
8135                 if ((ahd_debug & AHD_SHOW_SENSE) != 0) {
8136                         ahd_print_path(ahd, scb);
8137                         printf("SCB 0x%x Received PKT Status of 0x%x\n",
8138                                SCB_GET_TAG(scb), siu->status);
8139                         printf("\tflags = 0x%x, sense len = 0x%x, "
8140                                "pktfail = 0x%x\n",
8141                                siu->flags, scsi_4btoul(siu->sense_length),
8142                                scsi_4btoul(siu->pkt_failures_length));
8143                 }
8144 #endif
8145                 if ((siu->flags & SIU_RSPVALID) != 0) {
8146                         ahd_print_path(ahd, scb);
8147                         if (scsi_4btoul(siu->pkt_failures_length) < 4) {
8148                                 printf("Unable to parse pkt_failures\n");
8149                         } else {
8150
8151                                 switch (SIU_PKTFAIL_CODE(siu)) {
8152                                 case SIU_PFC_NONE:
8153                                         printf("No packet failure found\n");
8154                                         break;
8155                                 case SIU_PFC_CIU_FIELDS_INVALID:
8156                                         printf("Invalid Command IU Field\n");
8157                                         break;
8158                                 case SIU_PFC_TMF_NOT_SUPPORTED:
8159                                         printf("TMF not supportd\n");
8160                                         break;
8161                                 case SIU_PFC_TMF_FAILED:
8162                                         printf("TMF failed\n");
8163                                         break;
8164                                 case SIU_PFC_INVALID_TYPE_CODE:
8165                                         printf("Invalid L_Q Type code\n");
8166                                         break;
8167                                 case SIU_PFC_ILLEGAL_REQUEST:
8168                                         printf("Illegal request\n");
8169                                 default:
8170                                         break;
8171                                 }
8172                         }
8173                         if (siu->status == SCSI_STATUS_OK)
8174                                 ahd_set_transaction_status(scb,
8175                                                            CAM_REQ_CMP_ERR);
8176                 }
8177                 if ((siu->flags & SIU_SNSVALID) != 0) {
8178                         scb->flags |= SCB_PKT_SENSE;
8179 #ifdef AHD_DEBUG
8180                         if ((ahd_debug & AHD_SHOW_SENSE) != 0)
8181                                 printf("Sense data available\n");
8182 #endif
8183                 }
8184                 ahd_done(ahd, scb);
8185                 break;
8186         }
8187         case SCSI_STATUS_CMD_TERMINATED:
8188         case SCSI_STATUS_CHECK_COND:
8189         {
8190                 struct ahd_devinfo devinfo;
8191                 struct ahd_dma_seg *sg;
8192                 struct scsi_sense *sc;
8193                 struct ahd_initiator_tinfo *targ_info;
8194                 struct ahd_tmode_tstate *tstate;
8195                 struct ahd_transinfo *tinfo;
8196 #ifdef AHD_DEBUG
8197                 if (ahd_debug & AHD_SHOW_SENSE) {
8198                         ahd_print_path(ahd, scb);
8199                         printf("SCB %d: requests Check Status\n",
8200                                SCB_GET_TAG(scb));
8201                 }
8202 #endif
8203
8204                 if (ahd_perform_autosense(scb) == 0)
8205                         break;
8206
8207                 ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
8208                                     SCB_GET_TARGET(ahd, scb),
8209                                     SCB_GET_LUN(scb),
8210                                     SCB_GET_CHANNEL(ahd, scb),
8211                                     ROLE_INITIATOR);
8212                 targ_info = ahd_fetch_transinfo(ahd,
8213                                                 devinfo.channel,
8214                                                 devinfo.our_scsiid,
8215                                                 devinfo.target,
8216                                                 &tstate);
8217                 tinfo = &targ_info->curr;
8218                 sg = scb->sg_list;
8219                 sc = (struct scsi_sense *)hscb->shared_data.idata.cdb;
8220                 /*
8221                  * Save off the residual if there is one.
8222                  */
8223                 ahd_update_residual(ahd, scb);
8224 #ifdef AHD_DEBUG
8225                 if (ahd_debug & AHD_SHOW_SENSE) {
8226                         ahd_print_path(ahd, scb);
8227                         printf("Sending Sense\n");
8228                 }
8229 #endif
8230                 scb->sg_count = 0;
8231                 sg = ahd_sg_setup(ahd, scb, sg, ahd_get_sense_bufaddr(ahd, scb),
8232                                   ahd_get_sense_bufsize(ahd, scb),
8233                                   /*last*/TRUE);
8234                 sc->opcode = REQUEST_SENSE;
8235                 sc->byte2 = 0;
8236                 if (tinfo->protocol_version <= SCSI_REV_2
8237                  && SCB_GET_LUN(scb) < 8)
8238                         sc->byte2 = SCB_GET_LUN(scb) << 5;
8239                 sc->unused[0] = 0;
8240                 sc->unused[1] = 0;
8241                 sc->length = ahd_get_sense_bufsize(ahd, scb);
8242                 sc->control = 0;
8243
8244                 /*
8245                  * We can't allow the target to disconnect.
8246                  * This will be an untagged transaction and
8247                  * having the target disconnect will make this
8248                  * transaction indestinguishable from outstanding
8249                  * tagged transactions.
8250                  */
8251                 hscb->control = 0;
8252
8253                 /*
8254                  * This request sense could be because the
8255                  * the device lost power or in some other
8256                  * way has lost our transfer negotiations.
8257                  * Renegotiate if appropriate.  Unit attention
8258                  * errors will be reported before any data
8259                  * phases occur.
8260                  */
8261                 if (ahd_get_residual(scb) == ahd_get_transfer_length(scb)) {
8262                         ahd_update_neg_request(ahd, &devinfo,
8263                                                tstate, targ_info,
8264                                                AHD_NEG_IF_NON_ASYNC);
8265                 }
8266                 if (tstate->auto_negotiate & devinfo.target_mask) {
8267                         hscb->control |= MK_MESSAGE;
8268                         scb->flags &=
8269                             ~(SCB_NEGOTIATE|SCB_ABORT|SCB_DEVICE_RESET);
8270                         scb->flags |= SCB_AUTO_NEGOTIATE;
8271                 }
8272                 hscb->cdb_len = sizeof(*sc);
8273                 ahd_setup_data_scb(ahd, scb);
8274                 scb->flags |= SCB_SENSE;
8275                 ahd_queue_scb(ahd, scb);
8276                 break;
8277         }
8278         case SCSI_STATUS_OK:
8279                 printf("%s: Interrupted for staus of 0???\n",
8280                        ahd_name(ahd));
8281                 /* FALLTHROUGH */
8282         default:
8283                 ahd_done(ahd, scb);
8284                 break;
8285         }
8286 }
8287
8288 /*
8289  * Calculate the residual for a just completed SCB.
8290  */
8291 void
8292 ahd_calc_residual(struct ahd_softc *ahd, struct scb *scb)
8293 {
8294         struct hardware_scb *hscb;
8295         struct initiator_status *spkt;
8296         uint32_t sgptr;
8297         uint32_t resid_sgptr;
8298         uint32_t resid;
8299
8300         /*
8301          * 5 cases.
8302          * 1) No residual.
8303          *    SG_STATUS_VALID clear in sgptr.
8304          * 2) Transferless command
8305          * 3) Never performed any transfers.
8306          *    sgptr has SG_FULL_RESID set.
8307          * 4) No residual but target did not
8308          *    save data pointers after the
8309          *    last transfer, so sgptr was
8310          *    never updated.
8311          * 5) We have a partial residual.
8312          *    Use residual_sgptr to determine
8313          *    where we are.
8314          */
8315
8316         hscb = scb->hscb;
8317         sgptr = ahd_le32toh(hscb->sgptr);
8318         if ((sgptr & SG_STATUS_VALID) == 0)
8319                 /* Case 1 */
8320                 return;
8321         sgptr &= ~SG_STATUS_VALID;
8322
8323         if ((sgptr & SG_LIST_NULL) != 0)
8324                 /* Case 2 */
8325                 return;
8326
8327         /*
8328          * Residual fields are the same in both
8329          * target and initiator status packets,
8330          * so we can always use the initiator fields
8331          * regardless of the role for this SCB.
8332          */
8333         spkt = &hscb->shared_data.istatus;
8334         resid_sgptr = ahd_le32toh(spkt->residual_sgptr);
8335         if ((sgptr & SG_FULL_RESID) != 0) {
8336                 /* Case 3 */
8337                 resid = ahd_get_transfer_length(scb);
8338         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
8339                 /* Case 4 */
8340                 return;
8341         } else if ((resid_sgptr & SG_OVERRUN_RESID) != 0) {
8342                 ahd_print_path(ahd, scb);
8343                 printf("data overrun detected Tag == 0x%x.\n",
8344                        SCB_GET_TAG(scb));
8345                 ahd_freeze_devq(ahd, scb);
8346                 ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
8347                 ahd_freeze_scb(scb);
8348                 return;
8349         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
8350                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
8351                 /* NOTREACHED */
8352         } else {
8353                 struct ahd_dma_seg *sg;
8354
8355                 /*
8356                  * Remainder of the SG where the transfer
8357                  * stopped.
8358                  */
8359                 resid = ahd_le32toh(spkt->residual_datacnt) & AHD_SG_LEN_MASK;
8360                 sg = ahd_sg_bus_to_virt(ahd, scb, resid_sgptr & SG_PTR_MASK);
8361
8362                 /* The residual sg_ptr always points to the next sg */
8363                 sg--;
8364
8365                 /*
8366                  * Add up the contents of all residual
8367                  * SG segments that are after the SG where
8368                  * the transfer stopped.
8369                  */
8370                 while ((ahd_le32toh(sg->len) & AHD_DMA_LAST_SEG) == 0) {
8371                         sg++;
8372                         resid += ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
8373                 }
8374         }
8375         if ((scb->flags & SCB_SENSE) == 0)
8376                 ahd_set_residual(scb, resid);
8377         else
8378                 ahd_set_sense_residual(scb, resid);
8379
8380 #ifdef AHD_DEBUG
8381         if ((ahd_debug & AHD_SHOW_MISC) != 0) {
8382                 ahd_print_path(ahd, scb);
8383                 printf("Handled %sResidual of %d bytes\n",
8384                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
8385         }
8386 #endif
8387 }
8388
8389 /******************************* Target Mode **********************************/
8390 #ifdef AHD_TARGET_MODE
8391 /*
8392  * Add a target mode event to this lun's queue
8393  */
8394 static void
8395 ahd_queue_lstate_event(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate,
8396                        u_int initiator_id, u_int event_type, u_int event_arg)
8397 {
8398         struct ahd_tmode_event *event;
8399         int pending;
8400
8401         xpt_freeze_devq(lstate->path, /*count*/1);
8402         if (lstate->event_w_idx >= lstate->event_r_idx)
8403                 pending = lstate->event_w_idx - lstate->event_r_idx;
8404         else
8405                 pending = AHD_TMODE_EVENT_BUFFER_SIZE + 1
8406                         - (lstate->event_r_idx - lstate->event_w_idx);
8407
8408         if (event_type == EVENT_TYPE_BUS_RESET
8409          || event_type == MSG_BUS_DEV_RESET) {
8410                 /*
8411                  * Any earlier events are irrelevant, so reset our buffer.
8412                  * This has the effect of allowing us to deal with reset
8413                  * floods (an external device holding down the reset line)
8414                  * without losing the event that is really interesting.
8415                  */
8416                 lstate->event_r_idx = 0;
8417                 lstate->event_w_idx = 0;
8418                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
8419         }
8420
8421         if (pending == AHD_TMODE_EVENT_BUFFER_SIZE) {
8422                 xpt_print_path(lstate->path);
8423                 printf("immediate event %x:%x lost\n",
8424                        lstate->event_buffer[lstate->event_r_idx].event_type,
8425                        lstate->event_buffer[lstate->event_r_idx].event_arg);
8426                 lstate->event_r_idx++;
8427                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8428                         lstate->event_r_idx = 0;
8429                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
8430         }
8431
8432         event = &lstate->event_buffer[lstate->event_w_idx];
8433         event->initiator_id = initiator_id;
8434         event->event_type = event_type;
8435         event->event_arg = event_arg;
8436         lstate->event_w_idx++;
8437         if (lstate->event_w_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8438                 lstate->event_w_idx = 0;
8439 }
8440
8441 /*
8442  * Send any target mode events queued up waiting
8443  * for immediate notify resources.
8444  */
8445 void
8446 ahd_send_lstate_events(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate)
8447 {
8448         struct ccb_hdr *ccbh;
8449         struct ccb_immed_notify *inot;
8450
8451         while (lstate->event_r_idx != lstate->event_w_idx
8452             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
8453                 struct ahd_tmode_event *event;
8454
8455                 event = &lstate->event_buffer[lstate->event_r_idx];
8456                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
8457                 inot = (struct ccb_immed_notify *)ccbh;
8458                 switch (event->event_type) {
8459                 case EVENT_TYPE_BUS_RESET:
8460                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
8461                         break;
8462                 default:
8463                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
8464                         inot->message_args[0] = event->event_type;
8465                         inot->message_args[1] = event->event_arg;
8466                         break;
8467                 }
8468                 inot->initiator_id = event->initiator_id;
8469                 inot->sense_len = 0;
8470                 xpt_done((union ccb *)inot);
8471                 lstate->event_r_idx++;
8472                 if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
8473                         lstate->event_r_idx = 0;
8474         }
8475 }
8476 #endif
8477
8478 /******************** Sequencer Program Patching/Download *********************/
8479
8480 #ifdef AHD_DUMP_SEQ
8481 void
8482 ahd_dumpseq(struct ahd_softc* ahd)
8483 {
8484         int i;
8485         int max_prog;
8486
8487         max_prog = 2048;
8488
8489         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
8490         ahd_outw(ahd, PRGMCNT, 0);
8491         for (i = 0; i < max_prog; i++) {
8492                 uint8_t ins_bytes[4];
8493
8494                 ahd_insb(ahd, SEQRAM, ins_bytes, 4);
8495                 printf("0x%08x\n", ins_bytes[0] << 24
8496                                  | ins_bytes[1] << 16
8497                                  | ins_bytes[2] << 8
8498                                  | ins_bytes[3]);
8499         }
8500 }
8501 #endif
8502
8503 static void
8504 ahd_loadseq(struct ahd_softc *ahd)
8505 {
8506         struct  cs cs_table[num_critical_sections];
8507         u_int   begin_set[num_critical_sections];
8508         u_int   end_set[num_critical_sections];
8509         struct  patch *cur_patch;
8510         u_int   cs_count;
8511         u_int   cur_cs;
8512         u_int   i;
8513         int     downloaded;
8514         u_int   skip_addr;
8515         u_int   sg_prefetch_cnt;
8516         u_int   sg_prefetch_cnt_limit;
8517         u_int   sg_prefetch_align;
8518         u_int   sg_size;
8519         u_int   cacheline_mask;
8520         uint8_t download_consts[DOWNLOAD_CONST_COUNT];
8521
8522         if (bootverbose)
8523                 printf("%s: Downloading Sequencer Program...",
8524                        ahd_name(ahd));
8525
8526 #if DOWNLOAD_CONST_COUNT != 8
8527 #error "Download Const Mismatch"
8528 #endif
8529         /*
8530          * Start out with 0 critical sections
8531          * that apply to this firmware load.
8532          */
8533         cs_count = 0;
8534         cur_cs = 0;
8535         memset(begin_set, 0, sizeof(begin_set));
8536         memset(end_set, 0, sizeof(end_set));
8537
8538         /*
8539          * Setup downloadable constant table.
8540          *
8541          * The computation for the S/G prefetch variables is
8542          * a bit complicated.  We would like to always fetch
8543          * in terms of cachelined sized increments.  However,
8544          * if the cacheline is not an even multiple of the
8545          * SG element size or is larger than our SG RAM, using
8546          * just the cache size might leave us with only a portion
8547          * of an SG element at the tail of a prefetch.  If the
8548          * cacheline is larger than our S/G prefetch buffer less
8549          * the size of an SG element, we may round down to a cacheline
8550          * that doesn't contain any or all of the S/G of interest
8551          * within the bounds of our S/G ram.  Provide variables to
8552          * the sequencer that will allow it to handle these edge
8553          * cases.
8554          */
8555         /* Start by aligning to the nearest cacheline. */
8556         sg_prefetch_align = ahd->pci_cachesize;
8557         if (sg_prefetch_align == 0)
8558                 sg_prefetch_align = 8;
8559         /* Round down to the nearest power of 2. */
8560         while (powerof2(sg_prefetch_align) == 0)
8561                 sg_prefetch_align--;
8562
8563         cacheline_mask = sg_prefetch_align - 1;
8564
8565         /*
8566          * If the cacheline boundary is greater than half our prefetch RAM
8567          * we risk not being able to fetch even a single complete S/G
8568          * segment if we align to that boundary.
8569          */
8570         if (sg_prefetch_align > CCSGADDR_MAX/2)
8571                 sg_prefetch_align = CCSGADDR_MAX/2;
8572         /* Start by fetching a single cacheline. */
8573         sg_prefetch_cnt = sg_prefetch_align;
8574         /*
8575          * Increment the prefetch count by cachelines until
8576          * at least one S/G element will fit.
8577          */
8578         sg_size = sizeof(struct ahd_dma_seg);
8579         if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
8580                 sg_size = sizeof(struct ahd_dma64_seg);
8581         while (sg_prefetch_cnt < sg_size)
8582                 sg_prefetch_cnt += sg_prefetch_align;
8583         /*
8584          * If the cacheline is not an even multiple of
8585          * the S/G size, we may only get a partial S/G when
8586          * we align. Add a cacheline if this is the case.
8587          */
8588         if ((sg_prefetch_align % sg_size) != 0
8589          && (sg_prefetch_cnt < CCSGADDR_MAX))
8590                 sg_prefetch_cnt += sg_prefetch_align;
8591         /*
8592          * Lastly, compute a value that the sequencer can use
8593          * to determine if the remainder of the CCSGRAM buffer
8594          * has a full S/G element in it.
8595          */
8596         sg_prefetch_cnt_limit = -(sg_prefetch_cnt - sg_size + 1);
8597         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
8598         download_consts[SG_PREFETCH_CNT_LIMIT] = sg_prefetch_cnt_limit;
8599         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_align - 1);
8600         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_align - 1);
8601         download_consts[SG_SIZEOF] = sg_size;
8602         download_consts[PKT_OVERRUN_BUFOFFSET] =
8603                 (ahd->overrun_buf - (uint8_t *)ahd->qoutfifo) / 256;
8604         download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_1BYTE_LUN;
8605         download_consts[CACHELINE_MASK] = cacheline_mask;
8606         cur_patch = patches;
8607         downloaded = 0;
8608         skip_addr = 0;
8609         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
8610         ahd_outw(ahd, PRGMCNT, 0);
8611
8612         for (i = 0; i < sizeof(seqprog)/4; i++) {
8613                 if (ahd_check_patch(ahd, &cur_patch, i, &skip_addr) == 0) {
8614                         /*
8615                          * Don't download this instruction as it
8616                          * is in a patch that was removed.
8617                          */
8618                         continue;
8619                 }
8620                 /*
8621                  * Move through the CS table until we find a CS
8622                  * that might apply to this instruction.
8623                  */
8624                 for (; cur_cs < num_critical_sections; cur_cs++) {
8625                         if (critical_sections[cur_cs].end <= i) {
8626                                 if (begin_set[cs_count] == TRUE
8627                                  && end_set[cs_count] == FALSE) {
8628                                         cs_table[cs_count].end = downloaded;
8629                                         end_set[cs_count] = TRUE;
8630                                         cs_count++;
8631                                 }
8632                                 continue;
8633                         }
8634                         if (critical_sections[cur_cs].begin <= i
8635                          && begin_set[cs_count] == FALSE) {
8636                                 cs_table[cs_count].begin = downloaded;
8637                                 begin_set[cs_count] = TRUE;
8638                         }
8639                         break;
8640                 }
8641                 ahd_download_instr(ahd, i, download_consts);
8642                 downloaded++;
8643         }
8644
8645         ahd->num_critical_sections = cs_count;
8646         if (cs_count != 0) {
8647
8648                 cs_count *= sizeof(struct cs);
8649                 ahd->critical_sections = malloc(cs_count, M_DEVBUF, M_NOWAIT);
8650                 if (ahd->critical_sections == NULL)
8651                         panic("ahd_loadseq: Could not malloc");
8652                 memcpy(ahd->critical_sections, cs_table, cs_count);
8653         }
8654         ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE);
8655
8656         if (bootverbose) {
8657                 printf(" %d instructions downloaded\n", downloaded);
8658                 printf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
8659                        ahd_name(ahd), ahd->features, ahd->bugs, ahd->flags);
8660         }
8661 }
8662
8663 static int
8664 ahd_check_patch(struct ahd_softc *ahd, struct patch **start_patch,
8665                 u_int start_instr, u_int *skip_addr)
8666 {
8667         struct  patch *cur_patch;
8668         struct  patch *last_patch;
8669         u_int   num_patches;
8670
8671         num_patches = sizeof(patches)/sizeof(struct patch);
8672         last_patch = &patches[num_patches];
8673         cur_patch = *start_patch;
8674
8675         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
8676
8677                 if (cur_patch->patch_func(ahd) == 0) {
8678
8679                         /* Start rejecting code */
8680                         *skip_addr = start_instr + cur_patch->skip_instr;
8681                         cur_patch += cur_patch->skip_patch;
8682                 } else {
8683                         /* Accepted this patch.  Advance to the next
8684                          * one and wait for our intruction pointer to
8685                          * hit this point.
8686                          */
8687                         cur_patch++;
8688                 }
8689         }
8690
8691         *start_patch = cur_patch;
8692         if (start_instr < *skip_addr)
8693                 /* Still skipping */
8694                 return (0);
8695
8696         return (1);
8697 }
8698
8699 static u_int
8700 ahd_resolve_seqaddr(struct ahd_softc *ahd, u_int address)
8701 {
8702         struct patch *cur_patch;
8703         int address_offset;
8704         u_int skip_addr;
8705         u_int i;
8706
8707         address_offset = 0;
8708         cur_patch = patches;
8709         skip_addr = 0;
8710
8711         for (i = 0; i < address;) {
8712
8713                 ahd_check_patch(ahd, &cur_patch, i, &skip_addr);
8714
8715                 if (skip_addr > i) {
8716                         int end_addr;
8717
8718                         end_addr = MIN(address, skip_addr);
8719                         address_offset += end_addr - i;
8720                         i = skip_addr;
8721                 } else {
8722                         i++;
8723                 }
8724         }
8725         return (address - address_offset);
8726 }
8727
8728 static void
8729 ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts)
8730 {
8731         union   ins_formats instr;
8732         struct  ins_format1 *fmt1_ins;
8733         struct  ins_format3 *fmt3_ins;
8734         u_int   opcode;
8735
8736         /*
8737          * The firmware is always compiled into a little endian format.
8738          */
8739         instr.integer = ahd_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
8740
8741         fmt1_ins = &instr.format1;
8742         fmt3_ins = NULL;
8743
8744         /* Pull the opcode */
8745         opcode = instr.format1.opcode;
8746         switch (opcode) {
8747         case AIC_OP_JMP:
8748         case AIC_OP_JC:
8749         case AIC_OP_JNC:
8750         case AIC_OP_CALL:
8751         case AIC_OP_JNE:
8752         case AIC_OP_JNZ:
8753         case AIC_OP_JE:
8754         case AIC_OP_JZ:
8755         {
8756                 fmt3_ins = &instr.format3;
8757                 fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address);
8758                 /* FALLTHROUGH */
8759         }
8760         case AIC_OP_OR:
8761         case AIC_OP_AND:
8762         case AIC_OP_XOR:
8763         case AIC_OP_ADD:
8764         case AIC_OP_ADC:
8765         case AIC_OP_BMOV:
8766                 if (fmt1_ins->parity != 0) {
8767                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
8768                 }
8769                 fmt1_ins->parity = 0;
8770                 /* FALLTHROUGH */
8771         case AIC_OP_ROL:
8772         {
8773                 int i, count;
8774
8775                 /* Calculate odd parity for the instruction */
8776                 for (i = 0, count = 0; i < 31; i++) {
8777                         uint32_t mask;
8778
8779                         mask = 0x01 << i;
8780                         if ((instr.integer & mask) != 0)
8781                                 count++;
8782                 }
8783                 if ((count & 0x01) == 0)
8784                         instr.format1.parity = 1;
8785
8786                 /* The sequencer is a little endian cpu */
8787                 instr.integer = ahd_htole32(instr.integer);
8788                 ahd_outsb(ahd, SEQRAM, instr.bytes, 4);
8789                 break;
8790         }
8791         default:
8792                 panic("Unknown opcode encountered in seq program");
8793                 break;
8794         }
8795 }
8796
8797 static int
8798 ahd_probe_stack_size(struct ahd_softc *ahd)
8799 {
8800         int last_probe;
8801
8802         last_probe = 0;
8803         while (1) {
8804                 int i;
8805
8806                 /*
8807                  * We avoid using 0 as a pattern to avoid
8808                  * confusion if the stack implementation
8809                  * "back-fills" with zeros when "poping'
8810                  * entries.
8811                  */
8812                 for (i = 1; i <= last_probe+1; i++) {
8813                        ahd_outb(ahd, STACK, i & 0xFF);
8814                        ahd_outb(ahd, STACK, (i >> 8) & 0xFF);
8815                 }
8816
8817                 /* Verify */
8818                 for (i = last_probe+1; i > 0; i--) {
8819                         u_int stack_entry;
8820
8821                         stack_entry = ahd_inb(ahd, STACK)
8822                                     |(ahd_inb(ahd, STACK) << 8);
8823                         if (stack_entry != i)
8824                                 goto sized;
8825                 }
8826                 last_probe++;
8827         }
8828 sized:
8829         return (last_probe);
8830 }
8831
8832 int
8833 ahd_print_register(ahd_reg_parse_entry_t *table, u_int num_entries,
8834                    const char *name, u_int address, u_int value,
8835                    u_int *cur_column, u_int wrap_point)
8836 {
8837         int     printed;
8838         u_int   printed_mask;
8839
8840         if (cur_column != NULL && *cur_column >= wrap_point) {
8841                 printf("\n");
8842                 *cur_column = 0;
8843         }
8844         printed = printf("%s[0x%x]", name, value);
8845         if (table == NULL) {
8846                 printed += printf(" ");
8847                 *cur_column += printed;
8848                 return (printed);
8849         }
8850         printed_mask = 0;
8851         while (printed_mask != 0xFF) {
8852                 int entry;
8853
8854                 for (entry = 0; entry < num_entries; entry++) {
8855                         if (((value & table[entry].mask)
8856                           != table[entry].value)
8857                          || ((printed_mask & table[entry].mask)
8858                           == table[entry].mask))
8859                                 continue;
8860
8861                         printed += printf("%s%s",
8862                                           printed_mask == 0 ? ":(" : "|",
8863                                           table[entry].name);
8864                         printed_mask |= table[entry].mask;
8865
8866                         break;
8867                 }
8868                 if (entry >= num_entries)
8869                         break;
8870         }
8871         if (printed_mask != 0)
8872                 printed += printf(") ");
8873         else
8874                 printed += printf(" ");
8875         if (cur_column != NULL)
8876                 *cur_column += printed;
8877         return (printed);
8878 }
8879
8880 void
8881 ahd_dump_card_state(struct ahd_softc *ahd)
8882 {
8883         struct scb      *scb;
8884         ahd_mode_state   saved_modes;
8885         u_int            dffstat;
8886         int              paused;
8887         u_int            scb_index;
8888         u_int            saved_scb_index;
8889         u_int            cur_col;
8890         int              i;
8891
8892         if (ahd_is_paused(ahd)) {
8893                 paused = 1;
8894         } else {
8895                 paused = 0;
8896                 ahd_pause(ahd);
8897         }
8898         saved_modes = ahd_save_modes(ahd);
8899         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8900         printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
8901                "%s: Dumping Card State at program address 0x%x Mode 0x%x\n",
8902                ahd_name(ahd),
8903                ahd_inw(ahd, CURADDR),
8904                ahd_build_mode_state(ahd, ahd->saved_src_mode,
8905                                     ahd->saved_dst_mode));
8906         if (paused)
8907                 printf("Card was paused\n");
8908
8909         if (ahd_check_cmdcmpltqueues(ahd))
8910                 printf("Completions are pending\n");
8911
8912         /*
8913          * Mode independent registers.
8914          */
8915         cur_col = 0;
8916         ahd_intstat_print(ahd_inb(ahd, INTSTAT), &cur_col, 50);
8917         ahd_seloid_print(ahd_inb(ahd, SELOID), &cur_col, 50);
8918         ahd_selid_print(ahd_inb(ahd, SELID), &cur_col, 50);
8919         ahd_hs_mailbox_print(ahd_inb(ahd, LOCAL_HS_MAILBOX), &cur_col, 50);
8920         ahd_intctl_print(ahd_inb(ahd, INTCTL), &cur_col, 50);
8921         ahd_seqintstat_print(ahd_inb(ahd, SEQINTSTAT), &cur_col, 50);
8922         ahd_saved_mode_print(ahd_inb(ahd, SAVED_MODE), &cur_col, 50);
8923         ahd_dffstat_print(ahd_inb(ahd, DFFSTAT), &cur_col, 50);
8924         ahd_scsisigi_print(ahd_inb(ahd, SCSISIGI), &cur_col, 50);
8925         ahd_scsiphase_print(ahd_inb(ahd, SCSIPHASE), &cur_col, 50);
8926         ahd_scsibus_print(ahd_inb(ahd, SCSIBUS), &cur_col, 50);
8927         ahd_lastphase_print(ahd_inb(ahd, LASTPHASE), &cur_col, 50);
8928         ahd_scsiseq0_print(ahd_inb(ahd, SCSISEQ0), &cur_col, 50);
8929         ahd_scsiseq1_print(ahd_inb(ahd, SCSISEQ1), &cur_col, 50);
8930         ahd_seqctl0_print(ahd_inb(ahd, SEQCTL0), &cur_col, 50);
8931         ahd_seqintctl_print(ahd_inb(ahd, SEQINTCTL), &cur_col, 50);
8932         ahd_seq_flags_print(ahd_inb(ahd, SEQ_FLAGS), &cur_col, 50);
8933         ahd_seq_flags2_print(ahd_inb(ahd, SEQ_FLAGS2), &cur_col, 50);
8934         ahd_qfreeze_count_print(ahd_inw(ahd, QFREEZE_COUNT), &cur_col, 50);
8935         ahd_kernel_qfreeze_count_print(ahd_inw(ahd, KERNEL_QFREEZE_COUNT),
8936                                        &cur_col, 50);
8937         ahd_mk_message_scb_print(ahd_inw(ahd, MK_MESSAGE_SCB), &cur_col, 50);
8938         ahd_mk_message_scsiid_print(ahd_inb(ahd, MK_MESSAGE_SCSIID),
8939                                     &cur_col, 50);
8940         ahd_sstat0_print(ahd_inb(ahd, SSTAT0), &cur_col, 50);
8941         ahd_sstat1_print(ahd_inb(ahd, SSTAT1), &cur_col, 50);
8942         ahd_sstat2_print(ahd_inb(ahd, SSTAT2), &cur_col, 50);
8943         ahd_sstat3_print(ahd_inb(ahd, SSTAT3), &cur_col, 50);
8944         ahd_perrdiag_print(ahd_inb(ahd, PERRDIAG), &cur_col, 50);
8945         ahd_simode1_print(ahd_inb(ahd, SIMODE1), &cur_col, 50);
8946         ahd_lqistat0_print(ahd_inb(ahd, LQISTAT0), &cur_col, 50);
8947         ahd_lqistat1_print(ahd_inb(ahd, LQISTAT1), &cur_col, 50);
8948         ahd_lqistat2_print(ahd_inb(ahd, LQISTAT2), &cur_col, 50);
8949         ahd_lqostat0_print(ahd_inb(ahd, LQOSTAT0), &cur_col, 50);
8950         ahd_lqostat1_print(ahd_inb(ahd, LQOSTAT1), &cur_col, 50);
8951         ahd_lqostat2_print(ahd_inb(ahd, LQOSTAT2), &cur_col, 50);
8952         printf("\n");
8953         printf("\nSCB Count = %d CMDS_PENDING = %d LASTSCB 0x%x "
8954                "CURRSCB 0x%x NEXTSCB 0x%x\n",
8955                ahd->scb_data.numscbs, ahd_inw(ahd, CMDS_PENDING),
8956                ahd_inw(ahd, LASTSCB), ahd_inw(ahd, CURRSCB),
8957                ahd_inw(ahd, NEXTSCB));
8958         cur_col = 0;
8959         /* QINFIFO */
8960         ahd_search_qinfifo(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
8961                            CAM_LUN_WILDCARD, SCB_LIST_NULL,
8962                            ROLE_UNKNOWN, /*status*/0, SEARCH_PRINT);
8963         saved_scb_index = ahd_get_scbptr(ahd);
8964         printf("Pending list:");
8965         i = 0;
8966         LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
8967                 if (i++ > AHD_SCB_MAX)
8968                         break;
8969                 cur_col = printf("\n%3d FIFO_USE[0x%x] ", SCB_GET_TAG(scb),
8970                                  ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT));
8971                 ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
8972                 ahd_scb_control_print(ahd_inb_scbram(ahd, SCB_CONTROL),
8973                                       &cur_col, 60);
8974                 ahd_scb_scsiid_print(ahd_inb_scbram(ahd, SCB_SCSIID),
8975                                      &cur_col, 60);
8976         }
8977         printf("\nTotal %d\n", i);
8978
8979         printf("Kernel Free SCB list: ");
8980         i = 0;
8981         TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
8982                 struct scb *list_scb;
8983
8984                 list_scb = scb;
8985                 do {
8986                         printf("%d ", SCB_GET_TAG(list_scb));
8987                         list_scb = LIST_NEXT(list_scb, collision_links);
8988                 } while (list_scb && i++ < AHD_SCB_MAX);
8989         }
8990
8991         LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
8992                 if (i++ > AHD_SCB_MAX)
8993                         break;
8994                 printf("%d ", SCB_GET_TAG(scb));
8995         }
8996         printf("\n");
8997
8998         printf("Sequencer Complete DMA-inprog list: ");
8999         scb_index = ahd_inw(ahd, COMPLETE_SCB_DMAINPROG_HEAD);
9000         i = 0;
9001         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9002                 ahd_set_scbptr(ahd, scb_index);
9003                 printf("%d ", scb_index);
9004                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9005         }
9006         printf("\n");
9007
9008         printf("Sequencer Complete list: ");
9009         scb_index = ahd_inw(ahd, COMPLETE_SCB_HEAD);
9010         i = 0;
9011         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9012                 ahd_set_scbptr(ahd, scb_index);
9013                 printf("%d ", scb_index);
9014                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9015         }
9016         printf("\n");
9017
9018
9019         printf("Sequencer DMA-Up and Complete list: ");
9020         scb_index = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
9021         i = 0;
9022         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9023                 ahd_set_scbptr(ahd, scb_index);
9024                 printf("%d ", scb_index);
9025                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9026         }
9027         printf("\n");
9028         printf("Sequencer On QFreeze and Complete list: ");
9029         scb_index = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
9030         i = 0;
9031         while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9032                 ahd_set_scbptr(ahd, scb_index);
9033                 printf("%d ", scb_index);
9034                 scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9035         }
9036         printf("\n");
9037         ahd_set_scbptr(ahd, saved_scb_index);
9038         dffstat = ahd_inb(ahd, DFFSTAT);
9039         for (i = 0; i < 2; i++) {
9040 #ifdef AHD_DEBUG
9041                 struct scb *fifo_scb;
9042 #endif
9043                 u_int       fifo_scbptr;
9044
9045                 ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
9046                 fifo_scbptr = ahd_get_scbptr(ahd);
9047                 printf("\n\n%s: FIFO%d %s, LONGJMP == 0x%x, SCB 0x%x\n",
9048                        ahd_name(ahd), i,
9049                        (dffstat & (FIFO0FREE << i)) ? "Free" : "Active",
9050                        ahd_inw(ahd, LONGJMP_ADDR), fifo_scbptr);
9051                 cur_col = 0;
9052                 ahd_seqimode_print(ahd_inb(ahd, SEQIMODE), &cur_col, 50);
9053                 ahd_seqintsrc_print(ahd_inb(ahd, SEQINTSRC), &cur_col, 50);
9054                 ahd_dfcntrl_print(ahd_inb(ahd, DFCNTRL), &cur_col, 50);
9055                 ahd_dfstatus_print(ahd_inb(ahd, DFSTATUS), &cur_col, 50);
9056                 ahd_sg_cache_shadow_print(ahd_inb(ahd, SG_CACHE_SHADOW),
9057                                           &cur_col, 50);
9058                 ahd_sg_state_print(ahd_inb(ahd, SG_STATE), &cur_col, 50);
9059                 ahd_dffsxfrctl_print(ahd_inb(ahd, DFFSXFRCTL), &cur_col, 50);
9060                 ahd_soffcnt_print(ahd_inb(ahd, SOFFCNT), &cur_col, 50);
9061                 ahd_mdffstat_print(ahd_inb(ahd, MDFFSTAT), &cur_col, 50);
9062                 if (cur_col > 50) {
9063                         printf("\n");
9064                         cur_col = 0;
9065                 }
9066                 cur_col += printf("SHADDR = 0x%x%x, SHCNT = 0x%x ",
9067                                   ahd_inl(ahd, SHADDR+4),
9068                                   ahd_inl(ahd, SHADDR),
9069                                   (ahd_inb(ahd, SHCNT)
9070                                 | (ahd_inb(ahd, SHCNT + 1) << 8)
9071                                 | (ahd_inb(ahd, SHCNT + 2) << 16)));
9072                 if (cur_col > 50) {
9073                         printf("\n");
9074                         cur_col = 0;
9075                 }
9076                 cur_col += printf("HADDR = 0x%x%x, HCNT = 0x%x ",
9077                                   ahd_inl(ahd, HADDR+4),
9078                                   ahd_inl(ahd, HADDR),
9079                                   (ahd_inb(ahd, HCNT)
9080                                 | (ahd_inb(ahd, HCNT + 1) << 8)
9081                                 | (ahd_inb(ahd, HCNT + 2) << 16)));
9082                 ahd_ccsgctl_print(ahd_inb(ahd, CCSGCTL), &cur_col, 50);
9083 #ifdef AHD_DEBUG
9084                 if ((ahd_debug & AHD_SHOW_SG) != 0) {
9085                         fifo_scb = ahd_lookup_scb(ahd, fifo_scbptr);
9086                         if (fifo_scb != NULL)
9087                                 ahd_dump_sglist(fifo_scb);
9088                 }
9089 #endif
9090         }
9091         printf("\nLQIN: ");
9092         for (i = 0; i < 20; i++)
9093                 printf("0x%x ", ahd_inb(ahd, LQIN + i));
9094         printf("\n");
9095         ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
9096         printf("%s: LQISTATE = 0x%x, LQOSTATE = 0x%x, OPTIONMODE = 0x%x\n",
9097                ahd_name(ahd), ahd_inb(ahd, LQISTATE), ahd_inb(ahd, LQOSTATE),
9098                ahd_inb(ahd, OPTIONMODE));
9099         printf("%s: OS_SPACE_CNT = 0x%x MAXCMDCNT = 0x%x\n",
9100                ahd_name(ahd), ahd_inb(ahd, OS_SPACE_CNT),
9101                ahd_inb(ahd, MAXCMDCNT));
9102         printf("%s: SAVED_SCSIID = 0x%x SAVED_LUN = 0x%x\n",
9103                ahd_name(ahd), ahd_inb(ahd, SAVED_SCSIID),
9104                ahd_inb(ahd, SAVED_LUN));
9105         ahd_simode0_print(ahd_inb(ahd, SIMODE0), &cur_col, 50);
9106         printf("\n");
9107         ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
9108         cur_col = 0;
9109         ahd_ccscbctl_print(ahd_inb(ahd, CCSCBCTL), &cur_col, 50);
9110         printf("\n");
9111         ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
9112         printf("%s: REG0 == 0x%x, SINDEX = 0x%x, DINDEX = 0x%x\n",
9113                ahd_name(ahd), ahd_inw(ahd, REG0), ahd_inw(ahd, SINDEX),
9114                ahd_inw(ahd, DINDEX));
9115         printf("%s: SCBPTR == 0x%x, SCB_NEXT == 0x%x, SCB_NEXT2 == 0x%x\n",
9116                ahd_name(ahd), ahd_get_scbptr(ahd),
9117                ahd_inw_scbram(ahd, SCB_NEXT),
9118                ahd_inw_scbram(ahd, SCB_NEXT2));
9119         printf("CDB %x %x %x %x %x %x\n",
9120                ahd_inb_scbram(ahd, SCB_CDB_STORE),
9121                ahd_inb_scbram(ahd, SCB_CDB_STORE+1),
9122                ahd_inb_scbram(ahd, SCB_CDB_STORE+2),
9123                ahd_inb_scbram(ahd, SCB_CDB_STORE+3),
9124                ahd_inb_scbram(ahd, SCB_CDB_STORE+4),
9125                ahd_inb_scbram(ahd, SCB_CDB_STORE+5));
9126         printf("STACK:");
9127         for (i = 0; i < ahd->stack_size; i++) {
9128                 ahd->saved_stack[i] =
9129                     ahd_inb(ahd, STACK)|(ahd_inb(ahd, STACK) << 8);
9130                 printf(" 0x%x", ahd->saved_stack[i]);
9131         }
9132         for (i = ahd->stack_size-1; i >= 0; i--) {
9133                 ahd_outb(ahd, STACK, ahd->saved_stack[i] & 0xFF);
9134                 ahd_outb(ahd, STACK, (ahd->saved_stack[i] >> 8) & 0xFF);
9135         }
9136         printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
9137         ahd_restore_modes(ahd, saved_modes);
9138         if (paused == 0)
9139                 ahd_unpause(ahd);
9140 }
9141
9142 void
9143 ahd_dump_scbs(struct ahd_softc *ahd)
9144 {
9145         ahd_mode_state saved_modes;
9146         u_int          saved_scb_index;
9147         int            i;
9148
9149         saved_modes = ahd_save_modes(ahd);
9150         ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9151         saved_scb_index = ahd_get_scbptr(ahd);
9152         for (i = 0; i < AHD_SCB_MAX; i++) {
9153                 ahd_set_scbptr(ahd, i);
9154                 printf("%3d", i);
9155                 printf("(CTRL 0x%x ID 0x%x N 0x%x N2 0x%x SG 0x%x, RSG 0x%x)\n",
9156                        ahd_inb_scbram(ahd, SCB_CONTROL),
9157                        ahd_inb_scbram(ahd, SCB_SCSIID),
9158                        ahd_inw_scbram(ahd, SCB_NEXT),
9159                        ahd_inw_scbram(ahd, SCB_NEXT2),
9160                        ahd_inl_scbram(ahd, SCB_SGPTR),
9161                        ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR));
9162         }
9163         printf("\n");
9164         ahd_set_scbptr(ahd, saved_scb_index);
9165         ahd_restore_modes(ahd, saved_modes);
9166 }
9167
9168 /**************************** Flexport Logic **********************************/
9169 /*
9170  * Read count 16bit words from 16bit word address start_addr from the
9171  * SEEPROM attached to the controller, into buf, using the controller's
9172  * SEEPROM reading state machine.  Optionally treat the data as a byte
9173  * stream in terms of byte order.
9174  */
9175 int
9176 ahd_read_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9177                  u_int start_addr, u_int count, int bytestream)
9178 {
9179         u_int cur_addr;
9180         u_int end_addr;
9181         int   error;
9182
9183         /*
9184          * If we never make it through the loop even once,
9185          * we were passed invalid arguments.
9186          */
9187         error = EINVAL;
9188         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9189         end_addr = start_addr + count;
9190         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9191
9192                 ahd_outb(ahd, SEEADR, cur_addr);
9193                 ahd_outb(ahd, SEECTL, SEEOP_READ | SEESTART);
9194
9195                 error = ahd_wait_seeprom(ahd);
9196                 if (error)
9197                         break;
9198                 if (bytestream != 0) {
9199                         uint8_t *bytestream_ptr;
9200
9201                         bytestream_ptr = (uint8_t *)buf;
9202                         *bytestream_ptr++ = ahd_inb(ahd, SEEDAT);
9203                         *bytestream_ptr = ahd_inb(ahd, SEEDAT+1);
9204                 } else {
9205                         /*
9206                          * ahd_inw() already handles machine byte order.
9207                          */
9208                         *buf = ahd_inw(ahd, SEEDAT);
9209                 }
9210                 buf++;
9211         }
9212         return (error);
9213 }
9214
9215 /*
9216  * Write count 16bit words from buf, into SEEPROM attache to the
9217  * controller starting at 16bit word address start_addr, using the
9218  * controller's SEEPROM writing state machine.
9219  */
9220 int
9221 ahd_write_seeprom(struct ahd_softc *ahd, uint16_t *buf,
9222                   u_int start_addr, u_int count)
9223 {
9224         u_int cur_addr;
9225         u_int end_addr;
9226         int   error;
9227         int   retval;
9228
9229         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9230         error = ENOENT;
9231
9232         /* Place the chip into write-enable mode */
9233         ahd_outb(ahd, SEEADR, SEEOP_EWEN_ADDR);
9234         ahd_outb(ahd, SEECTL, SEEOP_EWEN | SEESTART);
9235         error = ahd_wait_seeprom(ahd);
9236         if (error)
9237                 return (error);
9238
9239         /*
9240          * Write the data.  If we don't get throught the loop at
9241          * least once, the arguments were invalid.
9242          */
9243         retval = EINVAL;
9244         end_addr = start_addr + count;
9245         for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
9246                 ahd_outw(ahd, SEEDAT, *buf++);
9247                 ahd_outb(ahd, SEEADR, cur_addr);
9248                 ahd_outb(ahd, SEECTL, SEEOP_WRITE | SEESTART);
9249
9250                 retval = ahd_wait_seeprom(ahd);
9251                 if (retval)
9252                         break;
9253         }
9254
9255         /*
9256          * Disable writes.
9257          */
9258         ahd_outb(ahd, SEEADR, SEEOP_EWDS_ADDR);
9259         ahd_outb(ahd, SEECTL, SEEOP_EWDS | SEESTART);
9260         error = ahd_wait_seeprom(ahd);
9261         if (error)
9262                 return (error);
9263         return (retval);
9264 }
9265
9266 /*
9267  * Wait ~100us for the serial eeprom to satisfy our request.
9268  */
9269 int
9270 ahd_wait_seeprom(struct ahd_softc *ahd)
9271 {
9272         int cnt;
9273
9274         cnt = 5000;
9275         while ((ahd_inb(ahd, SEESTAT) & (SEEARBACK|SEEBUSY)) != 0 && --cnt)
9276                 ahd_delay(5);
9277
9278         if (cnt == 0)
9279                 return (ETIMEDOUT);
9280         return (0);
9281 }
9282
9283 /*
9284  * Validate the two checksums in the per_channel
9285  * vital product data struct.
9286  */
9287 int
9288 ahd_verify_vpd_cksum(struct vpd_config *vpd)
9289 {
9290         int i;
9291         int maxaddr;
9292         uint32_t checksum;
9293         uint8_t *vpdarray;
9294
9295         vpdarray = (uint8_t *)vpd;
9296         maxaddr = offsetof(struct vpd_config, vpd_checksum);
9297         checksum = 0;
9298         for (i = offsetof(struct vpd_config, resource_type); i < maxaddr; i++)
9299                 checksum = checksum + vpdarray[i];
9300         if (checksum == 0
9301          || (-checksum & 0xFF) != vpd->vpd_checksum)
9302                 return (0);
9303
9304         checksum = 0;
9305         maxaddr = offsetof(struct vpd_config, checksum);
9306         for (i = offsetof(struct vpd_config, default_target_flags);
9307              i < maxaddr; i++)
9308                 checksum = checksum + vpdarray[i];
9309         if (checksum == 0
9310          || (-checksum & 0xFF) != vpd->checksum)
9311                 return (0);
9312         return (1);
9313 }
9314
9315 int
9316 ahd_verify_cksum(struct seeprom_config *sc)
9317 {
9318         int i;
9319         int maxaddr;
9320         uint32_t checksum;
9321         uint16_t *scarray;
9322
9323         maxaddr = (sizeof(*sc)/2) - 1;
9324         checksum = 0;
9325         scarray = (uint16_t *)sc;
9326
9327         for (i = 0; i < maxaddr; i++)
9328                 checksum = checksum + scarray[i];
9329         if (checksum == 0
9330          || (checksum & 0xFFFF) != sc->checksum) {
9331                 return (0);
9332         } else {
9333                 return (1);
9334         }
9335 }
9336
9337 int
9338 ahd_acquire_seeprom(struct ahd_softc *ahd)
9339 {
9340         /*
9341          * We should be able to determine the SEEPROM type
9342          * from the flexport logic, but unfortunately not
9343          * all implementations have this logic and there is
9344          * no programatic method for determining if the logic
9345          * is present.
9346          */
9347         return (1);
9348 #if 0
9349         uint8_t seetype;
9350         int     error;
9351
9352         error = ahd_read_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, &seetype);
9353         if (error != 0
9354          || ((seetype & FLX_ROMSTAT_SEECFG) == FLX_ROMSTAT_SEE_NONE))
9355                 return (0);
9356         return (1);
9357 #endif
9358 }
9359
9360 void
9361 ahd_release_seeprom(struct ahd_softc *ahd)
9362 {
9363         /* Currently a no-op */
9364 }
9365
9366 int
9367 ahd_write_flexport(struct ahd_softc *ahd, u_int addr, u_int value)
9368 {
9369         int error;
9370
9371         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9372         if (addr > 7)
9373                 panic("ahd_write_flexport: address out of range");
9374         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
9375         error = ahd_wait_flexport(ahd);
9376         if (error != 0)
9377                 return (error);
9378         ahd_outb(ahd, BRDDAT, value);
9379         ahd_flush_device_writes(ahd);
9380         ahd_outb(ahd, BRDCTL, BRDSTB|BRDEN|(addr << 3));
9381         ahd_flush_device_writes(ahd);
9382         ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
9383         ahd_flush_device_writes(ahd);
9384         ahd_outb(ahd, BRDCTL, 0);
9385         ahd_flush_device_writes(ahd);
9386         return (0);
9387 }
9388
9389 int
9390 ahd_read_flexport(struct ahd_softc *ahd, u_int addr, uint8_t *value)
9391 {
9392         int     error;
9393
9394         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9395         if (addr > 7)
9396                 panic("ahd_read_flexport: address out of range");
9397         ahd_outb(ahd, BRDCTL, BRDRW|BRDEN|(addr << 3));
9398         error = ahd_wait_flexport(ahd);
9399         if (error != 0)
9400                 return (error);
9401         *value = ahd_inb(ahd, BRDDAT);
9402         ahd_outb(ahd, BRDCTL, 0);
9403         ahd_flush_device_writes(ahd);
9404         return (0);
9405 }
9406
9407 /*
9408  * Wait at most 2 seconds for flexport arbitration to succeed.
9409  */
9410 int
9411 ahd_wait_flexport(struct ahd_softc *ahd)
9412 {
9413         int cnt;
9414
9415         AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
9416         cnt = 1000000 * 2 / 5;
9417         while ((ahd_inb(ahd, BRDCTL) & FLXARBACK) == 0 && --cnt)
9418                 ahd_delay(5);
9419
9420         if (cnt == 0)
9421                 return (ETIMEDOUT);
9422         return (0);
9423 }
9424
9425 /************************* Target Mode ****************************************/
9426 #ifdef AHD_TARGET_MODE
9427 cam_status
9428 ahd_find_tmode_devs(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb,
9429                     struct ahd_tmode_tstate **tstate,
9430                     struct ahd_tmode_lstate **lstate,
9431                     int notfound_failure)
9432 {
9433
9434         if ((ahd->features & AHD_TARGETMODE) == 0)
9435                 return (CAM_REQ_INVALID);
9436
9437         /*
9438          * Handle the 'black hole' device that sucks up
9439          * requests to unattached luns on enabled targets.
9440          */
9441         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
9442          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
9443                 *tstate = NULL;
9444                 *lstate = ahd->black_hole;
9445         } else {
9446                 u_int max_id;
9447
9448                 max_id = (ahd->features & AHD_WIDE) ? 15 : 7;
9449                 if (ccb->ccb_h.target_id > max_id)
9450                         return (CAM_TID_INVALID);
9451
9452                 if (ccb->ccb_h.target_lun >= AHD_NUM_LUNS)
9453                         return (CAM_LUN_INVALID);
9454
9455                 *tstate = ahd->enabled_targets[ccb->ccb_h.target_id];
9456                 *lstate = NULL;
9457                 if (*tstate != NULL)
9458                         *lstate =
9459                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
9460         }
9461
9462         if (notfound_failure != 0 && *lstate == NULL)
9463                 return (CAM_PATH_INVALID);
9464
9465         return (CAM_REQ_CMP);
9466 }
9467
9468 void
9469 ahd_handle_en_lun(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb)
9470 {
9471 #if NOT_YET
9472         struct     ahd_tmode_tstate *tstate;
9473         struct     ahd_tmode_lstate *lstate;
9474         struct     ccb_en_lun *cel;
9475         cam_status status;
9476         u_int      target;
9477         u_int      lun;
9478         u_int      target_mask;
9479         u_long     s;
9480         char       channel;
9481
9482         status = ahd_find_tmode_devs(ahd, sim, ccb, &tstate, &lstate,
9483                                      /*notfound_failure*/FALSE);
9484
9485         if (status != CAM_REQ_CMP) {
9486                 ccb->ccb_h.status = status;
9487                 return;
9488         }
9489
9490         if ((ahd->features & AHD_MULTIROLE) != 0) {
9491                 u_int      our_id;
9492
9493                 our_id = ahd->our_id;
9494                 if (ccb->ccb_h.target_id != our_id) {
9495                         if ((ahd->features & AHD_MULTI_TID) != 0
9496                          && (ahd->flags & AHD_INITIATORROLE) != 0) {
9497                                 /*
9498                                  * Only allow additional targets if
9499                                  * the initiator role is disabled.
9500                                  * The hardware cannot handle a re-select-in
9501                                  * on the initiator id during a re-select-out
9502                                  * on a different target id.
9503                                  */
9504                                 status = CAM_TID_INVALID;
9505                         } else if ((ahd->flags & AHD_INITIATORROLE) != 0
9506                                 || ahd->enabled_luns > 0) {
9507                                 /*
9508                                  * Only allow our target id to change
9509                                  * if the initiator role is not configured
9510                                  * and there are no enabled luns which
9511                                  * are attached to the currently registered
9512                                  * scsi id.
9513                                  */
9514                                 status = CAM_TID_INVALID;
9515                         }
9516                 }
9517         }
9518
9519         if (status != CAM_REQ_CMP) {
9520                 ccb->ccb_h.status = status;
9521                 return;
9522         }
9523
9524         /*
9525          * We now have an id that is valid.
9526          * If we aren't in target mode, switch modes.
9527          */
9528         if ((ahd->flags & AHD_TARGETROLE) == 0
9529          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
9530                 u_long  s;
9531
9532                 printf("Configuring Target Mode\n");
9533                 ahd_lock(ahd, &s);
9534                 if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
9535                         ccb->ccb_h.status = CAM_BUSY;
9536                         ahd_unlock(ahd, &s);
9537                         return;
9538                 }
9539                 ahd->flags |= AHD_TARGETROLE;
9540                 if ((ahd->features & AHD_MULTIROLE) == 0)
9541                         ahd->flags &= ~AHD_INITIATORROLE;
9542                 ahd_pause(ahd);
9543                 ahd_loadseq(ahd);
9544                 ahd_restart(ahd);
9545                 ahd_unlock(ahd, &s);
9546         }
9547         cel = &ccb->cel;
9548         target = ccb->ccb_h.target_id;
9549         lun = ccb->ccb_h.target_lun;
9550         channel = SIM_CHANNEL(ahd, sim);
9551         target_mask = 0x01 << target;
9552         if (channel == 'B')
9553                 target_mask <<= 8;
9554
9555         if (cel->enable != 0) {
9556                 u_int scsiseq1;
9557
9558                 /* Are we already enabled?? */
9559                 if (lstate != NULL) {
9560                         xpt_print_path(ccb->ccb_h.path);
9561                         printf("Lun already enabled\n");
9562                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
9563                         return;
9564                 }
9565
9566                 if (cel->grp6_len != 0
9567                  || cel->grp7_len != 0) {
9568                         /*
9569                          * Don't (yet?) support vendor
9570                          * specific commands.
9571                          */
9572                         ccb->ccb_h.status = CAM_REQ_INVALID;
9573                         printf("Non-zero Group Codes\n");
9574                         return;
9575                 }
9576
9577                 /*
9578                  * Seems to be okay.
9579                  * Setup our data structures.
9580                  */
9581                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
9582                         tstate = ahd_alloc_tstate(ahd, target, channel);
9583                         if (tstate == NULL) {
9584                                 xpt_print_path(ccb->ccb_h.path);
9585                                 printf("Couldn't allocate tstate\n");
9586                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
9587                                 return;
9588                         }
9589                 }
9590                 lstate = malloc(sizeof(*lstate), M_DEVBUF, M_NOWAIT);
9591                 if (lstate == NULL) {
9592                         xpt_print_path(ccb->ccb_h.path);
9593                         printf("Couldn't allocate lstate\n");
9594                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
9595                         return;
9596                 }
9597                 memset(lstate, 0, sizeof(*lstate));
9598                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
9599                                          xpt_path_path_id(ccb->ccb_h.path),
9600                                          xpt_path_target_id(ccb->ccb_h.path),
9601                                          xpt_path_lun_id(ccb->ccb_h.path));
9602                 if (status != CAM_REQ_CMP) {
9603                         free(lstate, M_DEVBUF);
9604                         xpt_print_path(ccb->ccb_h.path);
9605                         printf("Couldn't allocate path\n");
9606                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
9607                         return;
9608                 }
9609                 SLIST_INIT(&lstate->accept_tios);
9610                 SLIST_INIT(&lstate->immed_notifies);
9611                 ahd_lock(ahd, &s);
9612                 ahd_pause(ahd);
9613                 if (target != CAM_TARGET_WILDCARD) {
9614                         tstate->enabled_luns[lun] = lstate;
9615                         ahd->enabled_luns++;
9616
9617                         if ((ahd->features & AHD_MULTI_TID) != 0) {
9618                                 u_int targid_mask;
9619
9620                                 targid_mask = ahd_inw(ahd, TARGID);
9621                                 targid_mask |= target_mask;
9622                                 ahd_outw(ahd, TARGID, targid_mask);
9623                                 ahd_update_scsiid(ahd, targid_mask);
9624                         } else {
9625                                 u_int our_id;
9626                                 char  channel;
9627
9628                                 channel = SIM_CHANNEL(ahd, sim);
9629                                 our_id = SIM_SCSI_ID(ahd, sim);
9630
9631                                 /*
9632                                  * This can only happen if selections
9633                                  * are not enabled
9634                                  */
9635                                 if (target != our_id) {
9636                                         u_int sblkctl;
9637                                         char  cur_channel;
9638                                         int   swap;
9639
9640                                         sblkctl = ahd_inb(ahd, SBLKCTL);
9641                                         cur_channel = (sblkctl & SELBUSB)
9642                                                     ? 'B' : 'A';
9643                                         if ((ahd->features & AHD_TWIN) == 0)
9644                                                 cur_channel = 'A';
9645                                         swap = cur_channel != channel;
9646                                         ahd->our_id = target;
9647
9648                                         if (swap)
9649                                                 ahd_outb(ahd, SBLKCTL,
9650                                                          sblkctl ^ SELBUSB);
9651
9652                                         ahd_outb(ahd, SCSIID, target);
9653
9654                                         if (swap)
9655                                                 ahd_outb(ahd, SBLKCTL, sblkctl);
9656                                 }
9657                         }
9658                 } else
9659                         ahd->black_hole = lstate;
9660                 /* Allow select-in operations */
9661                 if (ahd->black_hole != NULL && ahd->enabled_luns > 0) {
9662                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
9663                         scsiseq1 |= ENSELI;
9664                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
9665                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
9666                         scsiseq1 |= ENSELI;
9667                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
9668                 }
9669                 ahd_unpause(ahd);
9670                 ahd_unlock(ahd, &s);
9671                 ccb->ccb_h.status = CAM_REQ_CMP;
9672                 xpt_print_path(ccb->ccb_h.path);
9673                 printf("Lun now enabled for target mode\n");
9674         } else {
9675                 struct scb *scb;
9676                 int i, empty;
9677
9678                 if (lstate == NULL) {
9679                         ccb->ccb_h.status = CAM_LUN_INVALID;
9680                         return;
9681                 }
9682
9683                 ahd_lock(ahd, &s);
9684
9685                 ccb->ccb_h.status = CAM_REQ_CMP;
9686                 LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
9687                         struct ccb_hdr *ccbh;
9688
9689                         ccbh = &scb->io_ctx->ccb_h;
9690                         if (ccbh->func_code == XPT_CONT_TARGET_IO
9691                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
9692                                 printf("CTIO pending\n");
9693                                 ccb->ccb_h.status = CAM_REQ_INVALID;
9694                                 ahd_unlock(ahd, &s);
9695                                 return;
9696                         }
9697                 }
9698
9699                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
9700                         printf("ATIOs pending\n");
9701                         ccb->ccb_h.status = CAM_REQ_INVALID;
9702                 }
9703
9704                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
9705                         printf("INOTs pending\n");
9706                         ccb->ccb_h.status = CAM_REQ_INVALID;
9707                 }
9708
9709                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
9710                         ahd_unlock(ahd, &s);
9711                         return;
9712                 }
9713
9714                 xpt_print_path(ccb->ccb_h.path);
9715                 printf("Target mode disabled\n");
9716                 xpt_free_path(lstate->path);
9717                 free(lstate, M_DEVBUF);
9718
9719                 ahd_pause(ahd);
9720                 /* Can we clean up the target too? */
9721                 if (target != CAM_TARGET_WILDCARD) {
9722                         tstate->enabled_luns[lun] = NULL;
9723                         ahd->enabled_luns--;
9724                         for (empty = 1, i = 0; i < 8; i++)
9725                                 if (tstate->enabled_luns[i] != NULL) {
9726                                         empty = 0;
9727                                         break;
9728                                 }
9729
9730                         if (empty) {
9731                                 ahd_free_tstate(ahd, target, channel,
9732                                                 /*force*/FALSE);
9733                                 if (ahd->features & AHD_MULTI_TID) {
9734                                         u_int targid_mask;
9735
9736                                         targid_mask = ahd_inw(ahd, TARGID);
9737                                         targid_mask &= ~target_mask;
9738                                         ahd_outw(ahd, TARGID, targid_mask);
9739                                         ahd_update_scsiid(ahd, targid_mask);
9740                                 }
9741                         }
9742                 } else {
9743
9744                         ahd->black_hole = NULL;
9745
9746                         /*
9747                          * We can't allow selections without
9748                          * our black hole device.
9749                          */
9750                         empty = TRUE;
9751                 }
9752                 if (ahd->enabled_luns == 0) {
9753                         /* Disallow select-in */
9754                         u_int scsiseq1;
9755
9756                         scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
9757                         scsiseq1 &= ~ENSELI;
9758                         ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
9759                         scsiseq1 = ahd_inb(ahd, SCSISEQ1);
9760                         scsiseq1 &= ~ENSELI;
9761                         ahd_outb(ahd, SCSISEQ1, scsiseq1);
9762
9763                         if ((ahd->features & AHD_MULTIROLE) == 0) {
9764                                 printf("Configuring Initiator Mode\n");
9765                                 ahd->flags &= ~AHD_TARGETROLE;
9766                                 ahd->flags |= AHD_INITIATORROLE;
9767                                 ahd_pause(ahd);
9768                                 ahd_loadseq(ahd);
9769                                 ahd_restart(ahd);
9770                                 /*
9771                                  * Unpaused.  The extra unpause
9772                                  * that follows is harmless.
9773                                  */
9774                         }
9775                 }
9776                 ahd_unpause(ahd);
9777                 ahd_unlock(ahd, &s);
9778         }
9779 #endif
9780 }
9781
9782 static void
9783 ahd_update_scsiid(struct ahd_softc *ahd, u_int targid_mask)
9784 {
9785 #if NOT_YET
9786         u_int scsiid_mask;
9787         u_int scsiid;
9788
9789         if ((ahd->features & AHD_MULTI_TID) == 0)
9790                 panic("ahd_update_scsiid called on non-multitid unit\n");
9791
9792         /*
9793          * Since we will rely on the TARGID mask
9794          * for selection enables, ensure that OID
9795          * in SCSIID is not set to some other ID
9796          * that we don't want to allow selections on.
9797          */
9798         if ((ahd->features & AHD_ULTRA2) != 0)
9799                 scsiid = ahd_inb(ahd, SCSIID_ULTRA2);
9800         else
9801                 scsiid = ahd_inb(ahd, SCSIID);
9802         scsiid_mask = 0x1 << (scsiid & OID);
9803         if ((targid_mask & scsiid_mask) == 0) {
9804                 u_int our_id;
9805
9806                 /* ffs counts from 1 */
9807                 our_id = ffs(targid_mask);
9808                 if (our_id == 0)
9809                         our_id = ahd->our_id;
9810                 else
9811                         our_id--;
9812                 scsiid &= TID;
9813                 scsiid |= our_id;
9814         }
9815         if ((ahd->features & AHD_ULTRA2) != 0)
9816                 ahd_outb(ahd, SCSIID_ULTRA2, scsiid);
9817         else
9818                 ahd_outb(ahd, SCSIID, scsiid);
9819 #endif
9820 }
9821
9822 void
9823 ahd_run_tqinfifo(struct ahd_softc *ahd, int paused)
9824 {
9825         struct target_cmd *cmd;
9826
9827         ahd_sync_tqinfifo(ahd, BUS_DMASYNC_POSTREAD);
9828         while ((cmd = &ahd->targetcmds[ahd->tqinfifonext])->cmd_valid != 0) {
9829
9830                 /*
9831                  * Only advance through the queue if we
9832                  * have the resources to process the command.
9833                  */
9834                 if (ahd_handle_target_cmd(ahd, cmd) != 0)
9835                         break;
9836
9837                 cmd->cmd_valid = 0;
9838                 ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
9839                                 ahd->shared_data_map.dmamap,
9840                                 ahd_targetcmd_offset(ahd, ahd->tqinfifonext),
9841                                 sizeof(struct target_cmd),
9842                                 BUS_DMASYNC_PREREAD);
9843                 ahd->tqinfifonext++;
9844
9845                 /*
9846                  * Lazily update our position in the target mode incoming
9847                  * command queue as seen by the sequencer.
9848                  */
9849                 if ((ahd->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
9850                         u_int hs_mailbox;
9851
9852                         hs_mailbox = ahd_inb(ahd, HS_MAILBOX);
9853                         hs_mailbox &= ~HOST_TQINPOS;
9854                         hs_mailbox |= ahd->tqinfifonext & HOST_TQINPOS;
9855                         ahd_outb(ahd, HS_MAILBOX, hs_mailbox);
9856                 }
9857         }
9858 }
9859
9860 static int
9861 ahd_handle_target_cmd(struct ahd_softc *ahd, struct target_cmd *cmd)
9862 {
9863         struct    ahd_tmode_tstate *tstate;
9864         struct    ahd_tmode_lstate *lstate;
9865         struct    ccb_accept_tio *atio;
9866         uint8_t *byte;
9867         int       initiator;
9868         int       target;
9869         int       lun;
9870
9871         initiator = SCSIID_TARGET(ahd, cmd->scsiid);
9872         target = SCSIID_OUR_ID(cmd->scsiid);
9873         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
9874
9875         byte = cmd->bytes;
9876         tstate = ahd->enabled_targets[target];
9877         lstate = NULL;
9878         if (tstate != NULL)
9879                 lstate = tstate->enabled_luns[lun];
9880
9881         /*
9882          * Commands for disabled luns go to the black hole driver.
9883          */
9884         if (lstate == NULL)
9885                 lstate = ahd->black_hole;
9886
9887         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
9888         if (atio == NULL) {
9889                 ahd->flags |= AHD_TQINFIFO_BLOCKED;
9890                 /*
9891                  * Wait for more ATIOs from the peripheral driver for this lun.
9892                  */
9893                 return (1);
9894         } else
9895                 ahd->flags &= ~AHD_TQINFIFO_BLOCKED;
9896 #ifdef AHD_DEBUG
9897         if ((ahd_debug & AHD_SHOW_TQIN) != 0)
9898                 printf("Incoming command from %d for %d:%d%s\n",
9899                        initiator, target, lun,
9900                        lstate == ahd->black_hole ? "(Black Holed)" : "");
9901 #endif
9902         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
9903
9904         if (lstate == ahd->black_hole) {
9905                 /* Fill in the wildcards */
9906                 atio->ccb_h.target_id = target;
9907                 atio->ccb_h.target_lun = lun;
9908         }
9909
9910         /*
9911          * Package it up and send it off to
9912          * whomever has this lun enabled.
9913          */
9914         atio->sense_len = 0;
9915         atio->init_id = initiator;
9916         if (byte[0] != 0xFF) {
9917                 /* Tag was included */
9918                 atio->tag_action = *byte++;
9919                 atio->tag_id = *byte++;
9920                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
9921         } else {
9922                 atio->ccb_h.flags = 0;
9923         }
9924         byte++;
9925
9926         /* Okay.  Now determine the cdb size based on the command code */
9927         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
9928         case 0:
9929                 atio->cdb_len = 6;
9930                 break;
9931         case 1:
9932         case 2:
9933                 atio->cdb_len = 10;
9934                 break;
9935         case 4:
9936                 atio->cdb_len = 16;
9937                 break;
9938         case 5:
9939                 atio->cdb_len = 12;
9940                 break;
9941         case 3:
9942         default:
9943                 /* Only copy the opcode. */
9944                 atio->cdb_len = 1;
9945                 printf("Reserved or VU command code type encountered\n");
9946                 break;
9947         }
9948
9949         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
9950
9951         atio->ccb_h.status |= CAM_CDB_RECVD;
9952
9953         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
9954                 /*
9955                  * We weren't allowed to disconnect.
9956                  * We're hanging on the bus until a
9957                  * continue target I/O comes in response
9958                  * to this accept tio.
9959                  */
9960 #ifdef AHD_DEBUG
9961                 if ((ahd_debug & AHD_SHOW_TQIN) != 0)
9962                         printf("Received Immediate Command %d:%d:%d - %p\n",
9963                                initiator, target, lun, ahd->pending_device);
9964 #endif
9965                 ahd->pending_device = lstate;
9966                 ahd_freeze_ccb((union ccb *)atio);
9967                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
9968         }
9969         xpt_done((union ccb*)atio);
9970         return (0);
9971 }
9972
9973 #endif