1 /*
2  * Core routines and tables shareable across OS platforms.
3  *
4  * Copyright (c) 1994-2002 Justin T. Gibbs.
5  * Copyright (c) 2000-2002 Adaptec Inc.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions, and the following disclaimer,
13  *    without modification.
14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
15  *    substantially similar to the "NO WARRANTY" disclaimer below
16  *    ("Disclaimer") and any redistribution must be conditioned upon
17  *    including a substantially similar Disclaimer requirement for further
18  *    binary redistribution.
19  * 3. Neither the names of the above-listed copyright holders nor the names
20  *    of any contributors may be used to endorse or promote products derived
21  *    from this software without specific prior written permission.
22  *
23  * Alternatively, this software may be distributed under the terms of the
24  * GNU General Public License ("GPL") version 2 as published by the Free
25  * Software Foundation.
26  *
27  * NO WARRANTY
28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38  * POSSIBILITY OF SUCH DAMAGES.
39  *
40  * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#155 $
41  */
42 
43 #ifdef __linux__
44 #include "aic7xxx_osm.h"
45 #include "aic7xxx_inline.h"
46 #include "aicasm/aicasm_insformat.h"
47 #else
48 #include <dev/aic7xxx/aic7xxx_osm.h>
49 #include <dev/aic7xxx/aic7xxx_inline.h>
50 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
51 #endif
52 
53 /***************************** Lookup Tables **********************************/
54 static const char *const ahc_chip_names[] = {
55 	"NONE",
56 	"aic7770",
57 	"aic7850",
58 	"aic7855",
59 	"aic7859",
60 	"aic7860",
61 	"aic7870",
62 	"aic7880",
63 	"aic7895",
64 	"aic7895C",
65 	"aic7890/91",
66 	"aic7896/97",
67 	"aic7892",
68 	"aic7899"
69 };
70 static const u_int num_chip_names = ARRAY_SIZE(ahc_chip_names);
71 
72 /*
73  * Hardware error codes.
74  */
75 struct ahc_hard_error_entry {
76         uint8_t errno;
77 	const char *errmesg;
78 };
79 
80 static const struct ahc_hard_error_entry ahc_hard_errors[] = {
81 	{ ILLHADDR,	"Illegal Host Access" },
82 	{ ILLSADDR,	"Illegal Sequencer Address referenced" },
83 	{ ILLOPCODE,	"Illegal Opcode in sequencer program" },
84 	{ SQPARERR,	"Sequencer Parity Error" },
85 	{ DPARERR,	"Data-path Parity Error" },
86 	{ MPARERR,	"Scratch or SCB Memory Parity Error" },
87 	{ PCIERRSTAT,	"PCI Error detected" },
88 	{ CIOPARERR,	"CIOBUS Parity Error" },
89 };
90 static const u_int num_errors = ARRAY_SIZE(ahc_hard_errors);
91 
92 static const struct ahc_phase_table_entry ahc_phase_table[] =
93 {
94 	{ P_DATAOUT,	MSG_NOOP,		"in Data-out phase"	},
95 	{ P_DATAIN,	MSG_INITIATOR_DET_ERR,	"in Data-in phase"	},
96 	{ P_DATAOUT_DT,	MSG_NOOP,		"in DT Data-out phase"	},
97 	{ P_DATAIN_DT,	MSG_INITIATOR_DET_ERR,	"in DT Data-in phase"	},
98 	{ P_COMMAND,	MSG_NOOP,		"in Command phase"	},
99 	{ P_MESGOUT,	MSG_NOOP,		"in Message-out phase"	},
100 	{ P_STATUS,	MSG_INITIATOR_DET_ERR,	"in Status phase"	},
101 	{ P_MESGIN,	MSG_PARITY_ERROR,	"in Message-in phase"	},
102 	{ P_BUSFREE,	MSG_NOOP,		"while idle"		},
103 	{ 0,		MSG_NOOP,		"in unknown phase"	}
104 };
105 
106 /*
107  * In most cases we only wish to itterate over real phases, so
108  * exclude the last element from the count.
109  */
110 static const u_int num_phases = ARRAY_SIZE(ahc_phase_table) - 1;
111 
112 /*
113  * Valid SCSIRATE values.  (p. 3-17)
114  * Provides a mapping of tranfer periods in ns to the proper value to
115  * stick in the scsixfer reg.
116  */
117 static const struct ahc_syncrate ahc_syncrates[] =
118 {
119       /* ultra2    fast/ultra  period     rate */
120 	{ 0x42,      0x000,      9,      "80.0" },
121 	{ 0x03,      0x000,     10,      "40.0" },
122 	{ 0x04,      0x000,     11,      "33.0" },
123 	{ 0x05,      0x100,     12,      "20.0" },
124 	{ 0x06,      0x110,     15,      "16.0" },
125 	{ 0x07,      0x120,     18,      "13.4" },
126 	{ 0x08,      0x000,     25,      "10.0" },
127 	{ 0x19,      0x010,     31,      "8.0"  },
128 	{ 0x1a,      0x020,     37,      "6.67" },
129 	{ 0x1b,      0x030,     43,      "5.7"  },
130 	{ 0x1c,      0x040,     50,      "5.0"  },
131 	{ 0x00,      0x050,     56,      "4.4"  },
132 	{ 0x00,      0x060,     62,      "4.0"  },
133 	{ 0x00,      0x070,     68,      "3.6"  },
134 	{ 0x00,      0x000,      0,      NULL   }
135 };
136 
137 /* Our Sequencer Program */
138 #include "aic7xxx_seq.h"
139 
140 /**************************** Function Declarations ***************************/
141 static void		ahc_force_renegotiation(struct ahc_softc *ahc,
142 						struct ahc_devinfo *devinfo);
143 static struct ahc_tmode_tstate*
144 			ahc_alloc_tstate(struct ahc_softc *ahc,
145 					 u_int scsi_id, char channel);
146 #ifdef AHC_TARGET_MODE
147 static void		ahc_free_tstate(struct ahc_softc *ahc,
148 					u_int scsi_id, char channel, int force);
149 #endif
150 static const struct ahc_syncrate*
151 			ahc_devlimited_syncrate(struct ahc_softc *ahc,
152 					        struct ahc_initiator_tinfo *,
153 						u_int *period,
154 						u_int *ppr_options,
155 						role_t role);
156 static void		ahc_update_pending_scbs(struct ahc_softc *ahc);
157 static void		ahc_fetch_devinfo(struct ahc_softc *ahc,
158 					  struct ahc_devinfo *devinfo);
159 static void		ahc_scb_devinfo(struct ahc_softc *ahc,
160 					struct ahc_devinfo *devinfo,
161 					struct scb *scb);
162 static void		ahc_assert_atn(struct ahc_softc *ahc);
163 static void		ahc_setup_initiator_msgout(struct ahc_softc *ahc,
164 						   struct ahc_devinfo *devinfo,
165 						   struct scb *scb);
166 static void		ahc_build_transfer_msg(struct ahc_softc *ahc,
167 					       struct ahc_devinfo *devinfo);
168 static void		ahc_construct_sdtr(struct ahc_softc *ahc,
169 					   struct ahc_devinfo *devinfo,
170 					   u_int period, u_int offset);
171 static void		ahc_construct_wdtr(struct ahc_softc *ahc,
172 					   struct ahc_devinfo *devinfo,
173 					   u_int bus_width);
174 static void		ahc_construct_ppr(struct ahc_softc *ahc,
175 					  struct ahc_devinfo *devinfo,
176 					  u_int period, u_int offset,
177 					  u_int bus_width, u_int ppr_options);
178 static void		ahc_clear_msg_state(struct ahc_softc *ahc);
179 static void		ahc_handle_proto_violation(struct ahc_softc *ahc);
180 static void		ahc_handle_message_phase(struct ahc_softc *ahc);
181 typedef enum {
182 	AHCMSG_1B,
183 	AHCMSG_2B,
184 	AHCMSG_EXT
185 } ahc_msgtype;
186 static int		ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
187 				     u_int msgval, int full);
188 static int		ahc_parse_msg(struct ahc_softc *ahc,
189 				      struct ahc_devinfo *devinfo);
190 static int		ahc_handle_msg_reject(struct ahc_softc *ahc,
191 					      struct ahc_devinfo *devinfo);
192 static void		ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
193 						struct ahc_devinfo *devinfo);
194 static void		ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
195 static void		ahc_handle_devreset(struct ahc_softc *ahc,
196 					    struct ahc_devinfo *devinfo,
197 					    cam_status status, char *message,
198 					    int verbose_level);
199 #ifdef AHC_TARGET_MODE
200 static void		ahc_setup_target_msgin(struct ahc_softc *ahc,
201 					       struct ahc_devinfo *devinfo,
202 					       struct scb *scb);
203 #endif
204 
205 static bus_dmamap_callback_t	ahc_dmamap_cb;
206 static void		ahc_build_free_scb_list(struct ahc_softc *ahc);
207 static int		ahc_init_scbdata(struct ahc_softc *ahc);
208 static void		ahc_fini_scbdata(struct ahc_softc *ahc);
209 static void		ahc_qinfifo_requeue(struct ahc_softc *ahc,
210 					    struct scb *prev_scb,
211 					    struct scb *scb);
212 static int		ahc_qinfifo_count(struct ahc_softc *ahc);
213 static u_int		ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
214 						   u_int prev, u_int scbptr);
215 static void		ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
216 static u_int		ahc_rem_wscb(struct ahc_softc *ahc,
217 				     u_int scbpos, u_int prev);
218 static void		ahc_reset_current_bus(struct ahc_softc *ahc);
219 #ifdef AHC_DUMP_SEQ
220 static void		ahc_dumpseq(struct ahc_softc *ahc);
221 #endif
222 static int		ahc_loadseq(struct ahc_softc *ahc);
223 static int		ahc_check_patch(struct ahc_softc *ahc,
224 					const struct patch **start_patch,
225 					u_int start_instr, u_int *skip_addr);
226 static void		ahc_download_instr(struct ahc_softc *ahc,
227 					   u_int instrptr, uint8_t *dconsts);
228 #ifdef AHC_TARGET_MODE
229 static void		ahc_queue_lstate_event(struct ahc_softc *ahc,
230 					       struct ahc_tmode_lstate *lstate,
231 					       u_int initiator_id,
232 					       u_int event_type,
233 					       u_int event_arg);
234 static void		ahc_update_scsiid(struct ahc_softc *ahc,
235 					  u_int targid_mask);
236 static int		ahc_handle_target_cmd(struct ahc_softc *ahc,
237 					      struct target_cmd *cmd);
238 #endif
239 
240 static u_int		ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl);
241 static void		ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl);
242 static void		ahc_busy_tcl(struct ahc_softc *ahc,
243 				     u_int tcl, u_int busyid);
244 
245 /************************** SCB and SCB queue management **********************/
246 static void		ahc_run_untagged_queues(struct ahc_softc *ahc);
247 static void		ahc_run_untagged_queue(struct ahc_softc *ahc,
248 					       struct scb_tailq *queue);
249 
250 /****************************** Initialization ********************************/
251 static void		 ahc_alloc_scbs(struct ahc_softc *ahc);
252 static void		 ahc_shutdown(void *arg);
253 
254 /*************************** Interrupt Services *******************************/
255 static void		ahc_clear_intstat(struct ahc_softc *ahc);
256 static void		ahc_run_qoutfifo(struct ahc_softc *ahc);
257 #ifdef AHC_TARGET_MODE
258 static void		ahc_run_tqinfifo(struct ahc_softc *ahc, int paused);
259 #endif
260 static void		ahc_handle_brkadrint(struct ahc_softc *ahc);
261 static void		ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat);
262 static void		ahc_handle_scsiint(struct ahc_softc *ahc,
263 					   u_int intstat);
264 static void		ahc_clear_critical_section(struct ahc_softc *ahc);
265 
266 /***************************** Error Recovery *********************************/
267 static void		ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb);
268 static int		ahc_abort_scbs(struct ahc_softc *ahc, int target,
269 				       char channel, int lun, u_int tag,
270 				       role_t role, uint32_t status);
271 static void		ahc_calc_residual(struct ahc_softc *ahc,
272 					  struct scb *scb);
273 
274 /*********************** Untagged Transaction Routines ************************/
275 static inline void	ahc_freeze_untagged_queues(struct ahc_softc *ahc);
276 static inline void	ahc_release_untagged_queues(struct ahc_softc *ahc);
277 
278 /*
279  * Block our completion routine from starting the next untagged
280  * transaction for this target or target lun.
281  */
282 static inline void
283 ahc_freeze_untagged_queues(struct ahc_softc *ahc)
284 {
285 	if ((ahc->flags & AHC_SCB_BTT) == 0)
286 		ahc->untagged_queue_lock++;
287 }
288 
289 /*
290  * Allow the next untagged transaction for this target or target lun
291  * to be executed.  We use a counting semaphore to allow the lock
292  * to be acquired recursively.  Once the count drops to zero, the
293  * transaction queues will be run.
294  */
295 static inline void
296 ahc_release_untagged_queues(struct ahc_softc *ahc)
297 {
298 	if ((ahc->flags & AHC_SCB_BTT) == 0) {
299 		ahc->untagged_queue_lock--;
300 		if (ahc->untagged_queue_lock == 0)
301 			ahc_run_untagged_queues(ahc);
302 	}
303 }
304 
305 /************************* Sequencer Execution Control ************************/
306 /*
307  * Work around any chip bugs related to halting sequencer execution.
308  * On Ultra2 controllers, we must clear the CIOBUS stretch signal by
309  * reading a register that will set this signal and deassert it.
310  * Without this workaround, if the chip is paused, by an interrupt or
311  * manual pause while accessing scb ram, accesses to certain registers
312  * will hang the system (infinite pci retries).
313  */
314 static void
315 ahc_pause_bug_fix(struct ahc_softc *ahc)
316 {
317 	if ((ahc->features & AHC_ULTRA2) != 0)
318 		(void)ahc_inb(ahc, CCSCBCTL);
319 }
320 
321 /*
322  * Determine whether the sequencer has halted code execution.
323  * Returns non-zero status if the sequencer is stopped.
324  */
325 int
326 ahc_is_paused(struct ahc_softc *ahc)
327 {
328 	return ((ahc_inb(ahc, HCNTRL) & PAUSE) != 0);
329 }
330 
331 /*
332  * Request that the sequencer stop and wait, indefinitely, for it
333  * to stop.  The sequencer will only acknowledge that it is paused
334  * once it has reached an instruction boundary and PAUSEDIS is
335  * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
336  * for critical sections.
337  */
338 void
339 ahc_pause(struct ahc_softc *ahc)
340 {
341 	ahc_outb(ahc, HCNTRL, ahc->pause);
342 
343 	/*
344 	 * Since the sequencer can disable pausing in a critical section, we
345 	 * must loop until it actually stops.
346 	 */
347 	while (ahc_is_paused(ahc) == 0)
348 		;
349 
350 	ahc_pause_bug_fix(ahc);
351 }
352 
353 /*
354  * Allow the sequencer to continue program execution.
355  * We check here to ensure that no additional interrupt
356  * sources that would cause the sequencer to halt have been
357  * asserted.  If, for example, a SCSI bus reset is detected
358  * while we are fielding a different, pausing, interrupt type,
359  * we don't want to release the sequencer before going back
360  * into our interrupt handler and dealing with this new
361  * condition.
362  */
363 void
364 ahc_unpause(struct ahc_softc *ahc)
365 {
366 	if ((ahc_inb(ahc, INTSTAT) & (SCSIINT | SEQINT | BRKADRINT)) == 0)
367 		ahc_outb(ahc, HCNTRL, ahc->unpause);
368 }
369 
370 /************************** Memory mapping routines ***************************/
371 static struct ahc_dma_seg *
372 ahc_sg_bus_to_virt(struct scb *scb, uint32_t sg_busaddr)
373 {
374 	int sg_index;
375 
376 	sg_index = (sg_busaddr - scb->sg_list_phys)/sizeof(struct ahc_dma_seg);
377 	/* sg_list_phys points to entry 1, not 0 */
378 	sg_index++;
379 
380 	return (&scb->sg_list[sg_index]);
381 }
382 
383 static uint32_t
384 ahc_sg_virt_to_bus(struct scb *scb, struct ahc_dma_seg *sg)
385 {
386 	int sg_index;
387 
388 	/* sg_list_phys points to entry 1, not 0 */
389 	sg_index = sg - &scb->sg_list[1];
390 
391 	return (scb->sg_list_phys + (sg_index * sizeof(*scb->sg_list)));
392 }
393 
394 static uint32_t
395 ahc_hscb_busaddr(struct ahc_softc *ahc, u_int index)
396 {
397 	return (ahc->scb_data->hscb_busaddr
398 		+ (sizeof(struct hardware_scb) * index));
399 }
400 
401 static void
402 ahc_sync_scb(struct ahc_softc *ahc, struct scb *scb, int op)
403 {
404 	ahc_dmamap_sync(ahc, ahc->scb_data->hscb_dmat,
405 			ahc->scb_data->hscb_dmamap,
406 			/*offset*/(scb->hscb - ahc->hscbs) * sizeof(*scb->hscb),
407 			/*len*/sizeof(*scb->hscb), op);
408 }
409 
410 void
411 ahc_sync_sglist(struct ahc_softc *ahc, struct scb *scb, int op)
412 {
413 	if (scb->sg_count == 0)
414 		return;
415 
416 	ahc_dmamap_sync(ahc, ahc->scb_data->sg_dmat, scb->sg_map->sg_dmamap,
417 			/*offset*/(scb->sg_list - scb->sg_map->sg_vaddr)
418 				* sizeof(struct ahc_dma_seg),
419 			/*len*/sizeof(struct ahc_dma_seg) * scb->sg_count, op);
420 }
421 
422 #ifdef AHC_TARGET_MODE
423 static uint32_t
424 ahc_targetcmd_offset(struct ahc_softc *ahc, u_int index)
425 {
426 	return (((uint8_t *)&ahc->targetcmds[index]) - ahc->qoutfifo);
427 }
428 #endif
429 
430 /*********************** Miscellaneous Support Functions ***********************/
431 /*
432  * Determine whether the sequencer reported a residual
433  * for this SCB/transaction.
434  */
435 static void
436 ahc_update_residual(struct ahc_softc *ahc, struct scb *scb)
437 {
438 	uint32_t sgptr;
439 
440 	sgptr = ahc_le32toh(scb->hscb->sgptr);
441 	if ((sgptr & SG_RESID_VALID) != 0)
442 		ahc_calc_residual(ahc, scb);
443 }
444 
445 /*
446  * Return pointers to the transfer negotiation information
447  * for the specified our_id/remote_id pair.
448  */
449 struct ahc_initiator_tinfo *
450 ahc_fetch_transinfo(struct ahc_softc *ahc, char channel, u_int our_id,
451 		    u_int remote_id, struct ahc_tmode_tstate **tstate)
452 {
453 	/*
454 	 * Transfer data structures are stored from the perspective
455 	 * of the target role.  Since the parameters for a connection
456 	 * in the initiator role to a given target are the same as
457 	 * when the roles are reversed, we pretend we are the target.
458 	 */
459 	if (channel == 'B')
460 		our_id += 8;
461 	*tstate = ahc->enabled_targets[our_id];
462 	return (&(*tstate)->transinfo[remote_id]);
463 }
464 
465 uint16_t
466 ahc_inw(struct ahc_softc *ahc, u_int port)
467 {
468 	uint16_t r = ahc_inb(ahc, port+1) << 8;
469 	return r | ahc_inb(ahc, port);
470 }
471 
472 void
473 ahc_outw(struct ahc_softc *ahc, u_int port, u_int value)
474 {
475 	ahc_outb(ahc, port, value & 0xFF);
476 	ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
477 }
478 
479 uint32_t
480 ahc_inl(struct ahc_softc *ahc, u_int port)
481 {
482 	return ((ahc_inb(ahc, port))
483 	      | (ahc_inb(ahc, port+1) << 8)
484 	      | (ahc_inb(ahc, port+2) << 16)
485 	      | (ahc_inb(ahc, port+3) << 24));
486 }
487 
488 void
489 ahc_outl(struct ahc_softc *ahc, u_int port, uint32_t value)
490 {
491 	ahc_outb(ahc, port, (value) & 0xFF);
492 	ahc_outb(ahc, port+1, ((value) >> 8) & 0xFF);
493 	ahc_outb(ahc, port+2, ((value) >> 16) & 0xFF);
494 	ahc_outb(ahc, port+3, ((value) >> 24) & 0xFF);
495 }
496 
497 uint64_t
498 ahc_inq(struct ahc_softc *ahc, u_int port)
499 {
500 	return ((ahc_inb(ahc, port))
501 	      | (ahc_inb(ahc, port+1) << 8)
502 	      | (ahc_inb(ahc, port+2) << 16)
503 	      | (ahc_inb(ahc, port+3) << 24)
504 	      | (((uint64_t)ahc_inb(ahc, port+4)) << 32)
505 	      | (((uint64_t)ahc_inb(ahc, port+5)) << 40)
506 	      | (((uint64_t)ahc_inb(ahc, port+6)) << 48)
507 	      | (((uint64_t)ahc_inb(ahc, port+7)) << 56));
508 }
509 
510 void
511 ahc_outq(struct ahc_softc *ahc, u_int port, uint64_t value)
512 {
513 	ahc_outb(ahc, port, value & 0xFF);
514 	ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
515 	ahc_outb(ahc, port+2, (value >> 16) & 0xFF);
516 	ahc_outb(ahc, port+3, (value >> 24) & 0xFF);
517 	ahc_outb(ahc, port+4, (value >> 32) & 0xFF);
518 	ahc_outb(ahc, port+5, (value >> 40) & 0xFF);
519 	ahc_outb(ahc, port+6, (value >> 48) & 0xFF);
520 	ahc_outb(ahc, port+7, (value >> 56) & 0xFF);
521 }
522 
523 /*
524  * Get a free scb. If there are none, see if we can allocate a new SCB.
525  */
526 struct scb *
527 ahc_get_scb(struct ahc_softc *ahc)
528 {
529 	struct scb *scb;
530 
531 	if ((scb = SLIST_FIRST(&ahc->scb_data->free_scbs)) == NULL) {
532 		ahc_alloc_scbs(ahc);
533 		scb = SLIST_FIRST(&ahc->scb_data->free_scbs);
534 		if (scb == NULL)
535 			return (NULL);
536 	}
537 	SLIST_REMOVE_HEAD(&ahc->scb_data->free_scbs, links.sle);
538 	return (scb);
539 }
540 
541 /*
542  * Return an SCB resource to the free list.
543  */
544 void
545 ahc_free_scb(struct ahc_softc *ahc, struct scb *scb)
546 {
547 	struct hardware_scb *hscb;
548 
549 	hscb = scb->hscb;
550 	/* Clean up for the next user */
551 	ahc->scb_data->scbindex[hscb->tag] = NULL;
552 	scb->flags = SCB_FREE;
553 	hscb->control = 0;
554 
555 	SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs, scb, links.sle);
556 
557 	/* Notify the OSM that a resource is now available. */
558 	ahc_platform_scb_free(ahc, scb);
559 }
560 
561 struct scb *
562 ahc_lookup_scb(struct ahc_softc *ahc, u_int tag)
563 {
564 	struct scb* scb;
565 
566 	scb = ahc->scb_data->scbindex[tag];
567 	if (scb != NULL)
568 		ahc_sync_scb(ahc, scb,
569 			     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
570 	return (scb);
571 }
572 
573 static void
574 ahc_swap_with_next_hscb(struct ahc_softc *ahc, struct scb *scb)
575 {
576 	struct hardware_scb *q_hscb;
577 	u_int  saved_tag;
578 
579 	/*
580 	 * Our queuing method is a bit tricky.  The card
581 	 * knows in advance which HSCB to download, and we
582 	 * can't disappoint it.  To achieve this, the next
583 	 * SCB to download is saved off in ahc->next_queued_scb.
584 	 * When we are called to queue "an arbitrary scb",
585 	 * we copy the contents of the incoming HSCB to the one
586 	 * the sequencer knows about, swap HSCB pointers and
587 	 * finally assign the SCB to the tag indexed location
588 	 * in the scb_array.  This makes sure that we can still
589 	 * locate the correct SCB by SCB_TAG.
590 	 */
591 	q_hscb = ahc->next_queued_scb->hscb;
592 	saved_tag = q_hscb->tag;
593 	memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
594 	if ((scb->flags & SCB_CDB32_PTR) != 0) {
595 		q_hscb->shared_data.cdb_ptr =
596 		    ahc_htole32(ahc_hscb_busaddr(ahc, q_hscb->tag)
597 			      + offsetof(struct hardware_scb, cdb32));
598 	}
599 	q_hscb->tag = saved_tag;
600 	q_hscb->next = scb->hscb->tag;
601 
602 	/* Now swap HSCB pointers. */
603 	ahc->next_queued_scb->hscb = scb->hscb;
604 	scb->hscb = q_hscb;
605 
606 	/* Now define the mapping from tag to SCB in the scbindex */
607 	ahc->scb_data->scbindex[scb->hscb->tag] = scb;
608 }
609 
610 /*
611  * Tell the sequencer about a new transaction to execute.
612  */
613 void
614 ahc_queue_scb(struct ahc_softc *ahc, struct scb *scb)
615 {
616 	ahc_swap_with_next_hscb(ahc, scb);
617 
618 	if (scb->hscb->tag == SCB_LIST_NULL
619 	 || scb->hscb->next == SCB_LIST_NULL)
620 		panic("Attempt to queue invalid SCB tag %x:%x\n",
621 		      scb->hscb->tag, scb->hscb->next);
622 
623 	/*
624 	 * Setup data "oddness".
625 	 */
626 	scb->hscb->lun &= LID;
627 	if (ahc_get_transfer_length(scb) & 0x1)
628 		scb->hscb->lun |= SCB_XFERLEN_ODD;
629 
630 	/*
631 	 * Keep a history of SCBs we've downloaded in the qinfifo.
632 	 */
633 	ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
634 
635 	/*
636 	 * Make sure our data is consistent from the
637 	 * perspective of the adapter.
638 	 */
639 	ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
640 
641 	/* Tell the adapter about the newly queued SCB */
642 	if ((ahc->features & AHC_QUEUE_REGS) != 0) {
643 		ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
644 	} else {
645 		if ((ahc->features & AHC_AUTOPAUSE) == 0)
646 			ahc_pause(ahc);
647 		ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
648 		if ((ahc->features & AHC_AUTOPAUSE) == 0)
649 			ahc_unpause(ahc);
650 	}
651 }
652 
653 struct scsi_sense_data *
654 ahc_get_sense_buf(struct ahc_softc *ahc, struct scb *scb)
655 {
656 	int offset;
657 
658 	offset = scb - ahc->scb_data->scbarray;
659 	return (&ahc->scb_data->sense[offset]);
660 }
661 
662 static uint32_t
663 ahc_get_sense_bufaddr(struct ahc_softc *ahc, struct scb *scb)
664 {
665 	int offset;
666 
667 	offset = scb - ahc->scb_data->scbarray;
668 	return (ahc->scb_data->sense_busaddr
669 	      + (offset * sizeof(struct scsi_sense_data)));
670 }
671 
672 /************************** Interrupt Processing ******************************/
673 static void
674 ahc_sync_qoutfifo(struct ahc_softc *ahc, int op)
675 {
676 	ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
677 			/*offset*/0, /*len*/256, op);
678 }
679 
680 static void
681 ahc_sync_tqinfifo(struct ahc_softc *ahc, int op)
682 {
683 #ifdef AHC_TARGET_MODE
684 	if ((ahc->flags & AHC_TARGETROLE) != 0) {
685 		ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
686 				ahc->shared_data_dmamap,
687 				ahc_targetcmd_offset(ahc, 0),
688 				sizeof(struct target_cmd) * AHC_TMODE_CMDS,
689 				op);
690 	}
691 #endif
692 }
693 
694 /*
695  * See if the firmware has posted any completed commands
696  * into our in-core command complete fifos.
697  */
698 #define AHC_RUN_QOUTFIFO 0x1
699 #define AHC_RUN_TQINFIFO 0x2
700 static u_int
701 ahc_check_cmdcmpltqueues(struct ahc_softc *ahc)
702 {
703 	u_int retval;
704 
705 	retval = 0;
706 	ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
707 			/*offset*/ahc->qoutfifonext, /*len*/1,
708 			BUS_DMASYNC_POSTREAD);
709 	if (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL)
710 		retval |= AHC_RUN_QOUTFIFO;
711 #ifdef AHC_TARGET_MODE
712 	if ((ahc->flags & AHC_TARGETROLE) != 0
713 	 && (ahc->flags & AHC_TQINFIFO_BLOCKED) == 0) {
714 		ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
715 				ahc->shared_data_dmamap,
716 				ahc_targetcmd_offset(ahc, ahc->tqinfifofnext),
717 				/*len*/sizeof(struct target_cmd),
718 				BUS_DMASYNC_POSTREAD);
719 		if (ahc->targetcmds[ahc->tqinfifonext].cmd_valid != 0)
720 			retval |= AHC_RUN_TQINFIFO;
721 	}
722 #endif
723 	return (retval);
724 }
725 
726 /*
727  * Catch an interrupt from the adapter
728  */
729 int
730 ahc_intr(struct ahc_softc *ahc)
731 {
732 	u_int	intstat;
733 
734 	if ((ahc->pause & INTEN) == 0) {
735 		/*
736 		 * Our interrupt is not enabled on the chip
737 		 * and may be disabled for re-entrancy reasons,
738 		 * so just return.  This is likely just a shared
739 		 * interrupt.
740 		 */
741 		return (0);
742 	}
743 	/*
744 	 * Instead of directly reading the interrupt status register,
745 	 * infer the cause of the interrupt by checking our in-core
746 	 * completion queues.  This avoids a costly PCI bus read in
747 	 * most cases.
748 	 */
749 	if ((ahc->flags & (AHC_ALL_INTERRUPTS|AHC_EDGE_INTERRUPT)) == 0
750 	 && (ahc_check_cmdcmpltqueues(ahc) != 0))
751 		intstat = CMDCMPLT;
752 	else {
753 		intstat = ahc_inb(ahc, INTSTAT);
754 	}
755 
756 	if ((intstat & INT_PEND) == 0) {
757 #if AHC_PCI_CONFIG > 0
758 		if (ahc->unsolicited_ints > 500) {
759 			ahc->unsolicited_ints = 0;
760 			if ((ahc->chip & AHC_PCI) != 0
761 			 && (ahc_inb(ahc, ERROR) & PCIERRSTAT) != 0)
762 				ahc->bus_intr(ahc);
763 		}
764 #endif
765 		ahc->unsolicited_ints++;
766 		return (0);
767 	}
768 	ahc->unsolicited_ints = 0;
769 
770 	if (intstat & CMDCMPLT) {
771 		ahc_outb(ahc, CLRINT, CLRCMDINT);
772 
773 		/*
774 		 * Ensure that the chip sees that we've cleared
775 		 * this interrupt before we walk the output fifo.
776 		 * Otherwise, we may, due to posted bus writes,
777 		 * clear the interrupt after we finish the scan,
778 		 * and after the sequencer has added new entries
779 		 * and asserted the interrupt again.
780 		 */
781 		ahc_flush_device_writes(ahc);
782 		ahc_run_qoutfifo(ahc);
783 #ifdef AHC_TARGET_MODE
784 		if ((ahc->flags & AHC_TARGETROLE) != 0)
785 			ahc_run_tqinfifo(ahc, /*paused*/FALSE);
786 #endif
787 	}
788 
789 	/*
790 	 * Handle statuses that may invalidate our cached
791 	 * copy of INTSTAT separately.
792 	 */
793 	if (intstat == 0xFF && (ahc->features & AHC_REMOVABLE) != 0) {
794 		/* Hot eject.  Do nothing */
795 	} else if (intstat & BRKADRINT) {
796 		ahc_handle_brkadrint(ahc);
797 	} else if ((intstat & (SEQINT|SCSIINT)) != 0) {
798 
799 		ahc_pause_bug_fix(ahc);
800 
801 		if ((intstat & SEQINT) != 0)
802 			ahc_handle_seqint(ahc, intstat);
803 
804 		if ((intstat & SCSIINT) != 0)
805 			ahc_handle_scsiint(ahc, intstat);
806 	}
807 	return (1);
808 }
809 
810 /************************* Sequencer Execution Control ************************/
811 /*
812  * Restart the sequencer program from address zero
813  */
814 static void
815 ahc_restart(struct ahc_softc *ahc)
816 {
817 	uint8_t	sblkctl;
818 
819 	ahc_pause(ahc);
820 
821 	/* No more pending messages. */
822 	ahc_clear_msg_state(ahc);
823 
824 	ahc_outb(ahc, SCSISIGO, 0);		/* De-assert BSY */
825 	ahc_outb(ahc, MSG_OUT, MSG_NOOP);	/* No message to send */
826 	ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
827 	ahc_outb(ahc, LASTPHASE, P_BUSFREE);
828 	ahc_outb(ahc, SAVED_SCSIID, 0xFF);
829 	ahc_outb(ahc, SAVED_LUN, 0xFF);
830 
831 	/*
832 	 * Ensure that the sequencer's idea of TQINPOS
833 	 * matches our own.  The sequencer increments TQINPOS
834 	 * only after it sees a DMA complete and a reset could
835 	 * occur before the increment leaving the kernel to believe
836 	 * the command arrived but the sequencer to not.
837 	 */
838 	ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
839 
840 	/* Always allow reselection */
841 	ahc_outb(ahc, SCSISEQ,
842 		 ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
843 	if ((ahc->features & AHC_CMD_CHAN) != 0) {
844 		/* Ensure that no DMA operations are in progress */
845 		ahc_outb(ahc, CCSCBCNT, 0);
846 		ahc_outb(ahc, CCSGCTL, 0);
847 		ahc_outb(ahc, CCSCBCTL, 0);
848 	}
849 	/*
850 	 * If we were in the process of DMA'ing SCB data into
851 	 * an SCB, replace that SCB on the free list.  This prevents
852 	 * an SCB leak.
853 	 */
854 	if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
855 		ahc_add_curscb_to_free_list(ahc);
856 		ahc_outb(ahc, SEQ_FLAGS2,
857 			 ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
858 	}
859 
860 	/*
861 	 * Clear any pending sequencer interrupt.  It is no
862 	 * longer relevant since we're resetting the Program
863 	 * Counter.
864 	 */
865 	ahc_outb(ahc, CLRINT, CLRSEQINT);
866 
867 	ahc_outb(ahc, MWI_RESIDUAL, 0);
868 	ahc_outb(ahc, SEQCTL, ahc->seqctl);
869 	ahc_outb(ahc, SEQADDR0, 0);
870 	ahc_outb(ahc, SEQADDR1, 0);
871 
872 	/*
873 	 * Take the LED out of diagnostic mode on PM resume, too
874 	 */
875 	sblkctl = ahc_inb(ahc, SBLKCTL);
876 	ahc_outb(ahc, SBLKCTL, (sblkctl & ~(DIAGLEDEN|DIAGLEDON)));
877 
878 	ahc_unpause(ahc);
879 }
880 
881 /************************* Input/Output Queues ********************************/
882 static void
883 ahc_run_qoutfifo(struct ahc_softc *ahc)
884 {
885 	struct scb *scb;
886 	u_int  scb_index;
887 
888 	ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
889 	while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
890 
891 		scb_index = ahc->qoutfifo[ahc->qoutfifonext];
892 		if ((ahc->qoutfifonext & 0x03) == 0x03) {
893 			u_int modnext;
894 
895 			/*
896 			 * Clear 32bits of QOUTFIFO at a time
897 			 * so that we don't clobber an incoming
898 			 * byte DMA to the array on architectures
899 			 * that only support 32bit load and store
900 			 * operations.
901 			 */
902 			modnext = ahc->qoutfifonext & ~0x3;
903 			*((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
904 			ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
905 					ahc->shared_data_dmamap,
906 					/*offset*/modnext, /*len*/4,
907 					BUS_DMASYNC_PREREAD);
908 		}
909 		ahc->qoutfifonext++;
910 
911 		scb = ahc_lookup_scb(ahc, scb_index);
912 		if (scb == NULL) {
913 			printk("%s: WARNING no command for scb %d "
914 			       "(cmdcmplt)\nQOUTPOS = %d\n",
915 			       ahc_name(ahc), scb_index,
916 			       (ahc->qoutfifonext - 1) & 0xFF);
917 			continue;
918 		}
919 
920 		/*
921 		 * Save off the residual
922 		 * if there is one.
923 		 */
924 		ahc_update_residual(ahc, scb);
925 		ahc_done(ahc, scb);
926 	}
927 }
928 
929 static void
930 ahc_run_untagged_queues(struct ahc_softc *ahc)
931 {
932 	int i;
933 
934 	for (i = 0; i < 16; i++)
935 		ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
936 }
937 
938 static void
939 ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
940 {
941 	struct scb *scb;
942 
943 	if (ahc->untagged_queue_lock != 0)
944 		return;
945 
946 	if ((scb = TAILQ_FIRST(queue)) != NULL
947 	 && (scb->flags & SCB_ACTIVE) == 0) {
948 		scb->flags |= SCB_ACTIVE;
949 		ahc_queue_scb(ahc, scb);
950 	}
951 }
952 
953 /************************* Interrupt Handling *********************************/
954 static void
955 ahc_handle_brkadrint(struct ahc_softc *ahc)
956 {
957 	/*
958 	 * We upset the sequencer :-(
959 	 * Lookup the error message
960 	 */
961 	int i;
962 	int error;
963 
964 	error = ahc_inb(ahc, ERROR);
965 	for (i = 0; error != 1 && i < num_errors; i++)
966 		error >>= 1;
967 	printk("%s: brkadrint, %s at seqaddr = 0x%x\n",
968 	       ahc_name(ahc), ahc_hard_errors[i].errmesg,
969 	       ahc_inb(ahc, SEQADDR0) |
970 	       (ahc_inb(ahc, SEQADDR1) << 8));
971 
972 	ahc_dump_card_state(ahc);
973 
974 	/* Tell everyone that this HBA is no longer available */
975 	ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
976 		       CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
977 		       CAM_NO_HBA);
978 
979 	/* Disable all interrupt sources by resetting the controller */
980 	ahc_shutdown(ahc);
981 }
982 
983 static void
984 ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
985 {
986 	struct scb *scb;
987 	struct ahc_devinfo devinfo;
988 
989 	ahc_fetch_devinfo(ahc, &devinfo);
990 
991 	/*
992 	 * Clear the upper byte that holds SEQINT status
993 	 * codes and clear the SEQINT bit. We will unpause
994 	 * the sequencer, if appropriate, after servicing
995 	 * the request.
996 	 */
997 	ahc_outb(ahc, CLRINT, CLRSEQINT);
998 	switch (intstat & SEQINT_MASK) {
999 	case BAD_STATUS:
1000 	{
1001 		u_int  scb_index;
1002 		struct hardware_scb *hscb;
1003 
1004 		/*
1005 		 * Set the default return value to 0 (don't
1006 		 * send sense).  The sense code will change
1007 		 * this if needed.
1008 		 */
1009 		ahc_outb(ahc, RETURN_1, 0);
1010 
1011 		/*
1012 		 * The sequencer will notify us when a command
1013 		 * has an error that would be of interest to
1014 		 * the kernel.  This allows us to leave the sequencer
1015 		 * running in the common case of command completes
1016 		 * without error.  The sequencer will already have
1017 		 * dma'd the SCB back up to us, so we can reference
1018 		 * the in kernel copy directly.
1019 		 */
1020 		scb_index = ahc_inb(ahc, SCB_TAG);
1021 		scb = ahc_lookup_scb(ahc, scb_index);
1022 		if (scb == NULL) {
1023 			ahc_print_devinfo(ahc, &devinfo);
1024 			printk("ahc_intr - referenced scb "
1025 			       "not valid during seqint 0x%x scb(%d)\n",
1026 			       intstat, scb_index);
1027 			ahc_dump_card_state(ahc);
1028 			panic("for safety");
1029 			goto unpause;
1030 		}
1031 
1032 		hscb = scb->hscb;
1033 
1034 		/* Don't want to clobber the original sense code */
1035 		if ((scb->flags & SCB_SENSE) != 0) {
1036 			/*
1037 			 * Clear the SCB_SENSE Flag and have
1038 			 * the sequencer do a normal command
1039 			 * complete.
1040 			 */
1041 			scb->flags &= ~SCB_SENSE;
1042 			ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1043 			break;
1044 		}
1045 		ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
1046 		/* Freeze the queue until the client sees the error. */
1047 		ahc_freeze_devq(ahc, scb);
1048 		ahc_freeze_scb(scb);
1049 		ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
1050 		switch (hscb->shared_data.status.scsi_status) {
1051 		case SCSI_STATUS_OK:
1052 			printk("%s: Interrupted for status of 0???\n",
1053 			       ahc_name(ahc));
1054 			break;
1055 		case SCSI_STATUS_CMD_TERMINATED:
1056 		case SCSI_STATUS_CHECK_COND:
1057 		{
1058 			struct ahc_dma_seg *sg;
1059 			struct scsi_sense *sc;
1060 			struct ahc_initiator_tinfo *targ_info;
1061 			struct ahc_tmode_tstate *tstate;
1062 			struct ahc_transinfo *tinfo;
1063 #ifdef AHC_DEBUG
1064 			if (ahc_debug & AHC_SHOW_SENSE) {
1065 				ahc_print_path(ahc, scb);
1066 				printk("SCB %d: requests Check Status\n",
1067 				       scb->hscb->tag);
1068 			}
1069 #endif
1070 
1071 			if (ahc_perform_autosense(scb) == 0)
1072 				break;
1073 
1074 			targ_info = ahc_fetch_transinfo(ahc,
1075 							devinfo.channel,
1076 							devinfo.our_scsiid,
1077 							devinfo.target,
1078 							&tstate);
1079 			tinfo = &targ_info->curr;
1080 			sg = scb->sg_list;
1081 			sc = (struct scsi_sense *)(&hscb->shared_data.cdb);
1082 			/*
1083 			 * Save off the residual if there is one.
1084 			 */
1085 			ahc_update_residual(ahc, scb);
1086 #ifdef AHC_DEBUG
1087 			if (ahc_debug & AHC_SHOW_SENSE) {
1088 				ahc_print_path(ahc, scb);
1089 				printk("Sending Sense\n");
1090 			}
1091 #endif
1092 			sg->addr = ahc_get_sense_bufaddr(ahc, scb);
1093 			sg->len = ahc_get_sense_bufsize(ahc, scb);
1094 			sg->len |= AHC_DMA_LAST_SEG;
1095 
1096 			/* Fixup byte order */
1097 			sg->addr = ahc_htole32(sg->addr);
1098 			sg->len = ahc_htole32(sg->len);
1099 
1100 			sc->opcode = REQUEST_SENSE;
1101 			sc->byte2 = 0;
1102 			if (tinfo->protocol_version <= SCSI_REV_2
1103 			 && SCB_GET_LUN(scb) < 8)
1104 				sc->byte2 = SCB_GET_LUN(scb) << 5;
1105 			sc->unused[0] = 0;
1106 			sc->unused[1] = 0;
1107 			sc->length = sg->len;
1108 			sc->control = 0;
1109 
1110 			/*
1111 			 * We can't allow the target to disconnect.
1112 			 * This will be an untagged transaction and
1113 			 * having the target disconnect will make this
1114 			 * transaction indestinguishable from outstanding
1115 			 * tagged transactions.
1116 			 */
1117 			hscb->control = 0;
1118 
1119 			/*
1120 			 * This request sense could be because the
1121 			 * the device lost power or in some other
1122 			 * way has lost our transfer negotiations.
1123 			 * Renegotiate if appropriate.  Unit attention
1124 			 * errors will be reported before any data
1125 			 * phases occur.
1126 			 */
1127 			if (ahc_get_residual(scb)
1128 			 == ahc_get_transfer_length(scb)) {
1129 				ahc_update_neg_request(ahc, &devinfo,
1130 						       tstate, targ_info,
1131 						       AHC_NEG_IF_NON_ASYNC);
1132 			}
1133 			if (tstate->auto_negotiate & devinfo.target_mask) {
1134 				hscb->control |= MK_MESSAGE;
1135 				scb->flags &= ~SCB_NEGOTIATE;
1136 				scb->flags |= SCB_AUTO_NEGOTIATE;
1137 			}
1138 			hscb->cdb_len = sizeof(*sc);
1139 			hscb->dataptr = sg->addr;
1140 			hscb->datacnt = sg->len;
1141 			hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
1142 			hscb->sgptr = ahc_htole32(hscb->sgptr);
1143 			scb->sg_count = 1;
1144 			scb->flags |= SCB_SENSE;
1145 			ahc_qinfifo_requeue_tail(ahc, scb);
1146 			ahc_outb(ahc, RETURN_1, SEND_SENSE);
1147 			/*
1148 			 * Ensure we have enough time to actually
1149 			 * retrieve the sense.
1150 			 */
1151 			ahc_scb_timer_reset(scb, 5 * 1000000);
1152 			break;
1153 		}
1154 		default:
1155 			break;
1156 		}
1157 		break;
1158 	}
1159 	case NO_MATCH:
1160 	{
1161 		/* Ensure we don't leave the selection hardware on */
1162 		ahc_outb(ahc, SCSISEQ,
1163 			 ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1164 
1165 		printk("%s:%c:%d: no active SCB for reconnecting "
1166 		       "target - issuing BUS DEVICE RESET\n",
1167 		       ahc_name(ahc), devinfo.channel, devinfo.target);
1168 		printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1169 		       "ARG_1 == 0x%x ACCUM = 0x%x\n",
1170 		       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1171 		       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1172 		printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1173 		       "SINDEX == 0x%x\n",
1174 		       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1175 		       ahc_index_busy_tcl(ahc,
1176 			    BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1177 				      ahc_inb(ahc, SAVED_LUN))),
1178 		       ahc_inb(ahc, SINDEX));
1179 		printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1180 		       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1181 		       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1182 		       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1183 		       ahc_inb(ahc, SCB_CONTROL));
1184 		printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1185 		       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1186 		printk("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
1187 		printk("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
1188 		ahc_dump_card_state(ahc);
1189 		ahc->msgout_buf[0] = MSG_BUS_DEV_RESET;
1190 		ahc->msgout_len = 1;
1191 		ahc->msgout_index = 0;
1192 		ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1193 		ahc_outb(ahc, MSG_OUT, HOST_MSG);
1194 		ahc_assert_atn(ahc);
1195 		break;
1196 	}
1197 	case SEND_REJECT:
1198 	{
1199 		u_int rejbyte = ahc_inb(ahc, ACCUM);
1200 		printk("%s:%c:%d: Warning - unknown message received from "
1201 		       "target (0x%x).  Rejecting\n",
1202 		       ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
1203 		break;
1204 	}
1205 	case PROTO_VIOLATION:
1206 	{
1207 		ahc_handle_proto_violation(ahc);
1208 		break;
1209 	}
1210 	case IGN_WIDE_RES:
1211 		ahc_handle_ign_wide_residue(ahc, &devinfo);
1212 		break;
1213 	case PDATA_REINIT:
1214 		ahc_reinitialize_dataptrs(ahc);
1215 		break;
1216 	case BAD_PHASE:
1217 	{
1218 		u_int lastphase;
1219 
1220 		lastphase = ahc_inb(ahc, LASTPHASE);
1221 		printk("%s:%c:%d: unknown scsi bus phase %x, "
1222 		       "lastphase = 0x%x.  Attempting to continue\n",
1223 		       ahc_name(ahc), devinfo.channel, devinfo.target,
1224 		       lastphase, ahc_inb(ahc, SCSISIGI));
1225 		break;
1226 	}
1227 	case MISSED_BUSFREE:
1228 	{
1229 		u_int lastphase;
1230 
1231 		lastphase = ahc_inb(ahc, LASTPHASE);
1232 		printk("%s:%c:%d: Missed busfree. "
1233 		       "Lastphase = 0x%x, Curphase = 0x%x\n",
1234 		       ahc_name(ahc), devinfo.channel, devinfo.target,
1235 		       lastphase, ahc_inb(ahc, SCSISIGI));
1236 		ahc_restart(ahc);
1237 		return;
1238 	}
1239 	case HOST_MSG_LOOP:
1240 	{
1241 		/*
1242 		 * The sequencer has encountered a message phase
1243 		 * that requires host assistance for completion.
1244 		 * While handling the message phase(s), we will be
1245 		 * notified by the sequencer after each byte is
1246 		 * transferred so we can track bus phase changes.
1247 		 *
1248 		 * If this is the first time we've seen a HOST_MSG_LOOP
1249 		 * interrupt, initialize the state of the host message
1250 		 * loop.
1251 		 */
1252 		if (ahc->msg_type == MSG_TYPE_NONE) {
1253 			struct scb *scb;
1254 			u_int scb_index;
1255 			u_int bus_phase;
1256 
1257 			bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1258 			if (bus_phase != P_MESGIN
1259 			 && bus_phase != P_MESGOUT) {
1260 				printk("ahc_intr: HOST_MSG_LOOP bad "
1261 				       "phase 0x%x\n",
1262 				      bus_phase);
1263 				/*
1264 				 * Probably transitioned to bus free before
1265 				 * we got here.  Just punt the message.
1266 				 */
1267 				ahc_clear_intstat(ahc);
1268 				ahc_restart(ahc);
1269 				return;
1270 			}
1271 
1272 			scb_index = ahc_inb(ahc, SCB_TAG);
1273 			scb = ahc_lookup_scb(ahc, scb_index);
1274 			if (devinfo.role == ROLE_INITIATOR) {
1275 				if (bus_phase == P_MESGOUT) {
1276 					if (scb == NULL)
1277 						panic("HOST_MSG_LOOP with "
1278 						      "invalid SCB %x\n",
1279 						      scb_index);
1280 
1281 					ahc_setup_initiator_msgout(ahc,
1282 								   &devinfo,
1283 								   scb);
1284 				} else {
1285 					ahc->msg_type =
1286 					    MSG_TYPE_INITIATOR_MSGIN;
1287 					ahc->msgin_index = 0;
1288 				}
1289 			}
1290 #ifdef AHC_TARGET_MODE
1291 			else {
1292 				if (bus_phase == P_MESGOUT) {
1293 					ahc->msg_type =
1294 					    MSG_TYPE_TARGET_MSGOUT;
1295 					ahc->msgin_index = 0;
1296 				}
1297 				else
1298 					ahc_setup_target_msgin(ahc,
1299 							       &devinfo,
1300 							       scb);
1301 			}
1302 #endif
1303 		}
1304 
1305 		ahc_handle_message_phase(ahc);
1306 		break;
1307 	}
1308 	case PERR_DETECTED:
1309 	{
1310 		/*
1311 		 * If we've cleared the parity error interrupt
1312 		 * but the sequencer still believes that SCSIPERR
1313 		 * is true, it must be that the parity error is
1314 		 * for the currently presented byte on the bus,
1315 		 * and we are not in a phase (data-in) where we will
1316 		 * eventually ack this byte.  Ack the byte and
1317 		 * throw it away in the hope that the target will
1318 		 * take us to message out to deliver the appropriate
1319 		 * error message.
1320 		 */
1321 		if ((intstat & SCSIINT) == 0
1322 		 && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
1323 
1324 			if ((ahc->features & AHC_DT) == 0) {
1325 				u_int curphase;
1326 
1327 				/*
1328 				 * The hardware will only let you ack bytes
1329 				 * if the expected phase in SCSISIGO matches
1330 				 * the current phase.  Make sure this is
1331 				 * currently the case.
1332 				 */
1333 				curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1334 				ahc_outb(ahc, LASTPHASE, curphase);
1335 				ahc_outb(ahc, SCSISIGO, curphase);
1336 			}
1337 			if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
1338 				int wait;
1339 
1340 				/*
1341 				 * In a data phase.  Faster to bitbucket
1342 				 * the data than to individually ack each
1343 				 * byte.  This is also the only strategy
1344 				 * that will work with AUTOACK enabled.
1345 				 */
1346 				ahc_outb(ahc, SXFRCTL1,
1347 					 ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
1348 				wait = 5000;
1349 				while (--wait != 0) {
1350 					if ((ahc_inb(ahc, SCSISIGI)
1351 					  & (CDI|MSGI)) != 0)
1352 						break;
1353 					ahc_delay(100);
1354 				}
1355 				ahc_outb(ahc, SXFRCTL1,
1356 					 ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
1357 				if (wait == 0) {
1358 					struct	scb *scb;
1359 					u_int	scb_index;
1360 
1361 					ahc_print_devinfo(ahc, &devinfo);
1362 					printk("Unable to clear parity error.  "
1363 					       "Resetting bus.\n");
1364 					scb_index = ahc_inb(ahc, SCB_TAG);
1365 					scb = ahc_lookup_scb(ahc, scb_index);
1366 					if (scb != NULL)
1367 						ahc_set_transaction_status(scb,
1368 						    CAM_UNCOR_PARITY);
1369 					ahc_reset_channel(ahc, devinfo.channel,
1370 							  /*init reset*/TRUE);
1371 				}
1372 			} else {
1373 				ahc_inb(ahc, SCSIDATL);
1374 			}
1375 		}
1376 		break;
1377 	}
1378 	case DATA_OVERRUN:
1379 	{
1380 		/*
1381 		 * When the sequencer detects an overrun, it
1382 		 * places the controller in "BITBUCKET" mode
1383 		 * and allows the target to complete its transfer.
1384 		 * Unfortunately, none of the counters get updated
1385 		 * when the controller is in this mode, so we have
1386 		 * no way of knowing how large the overrun was.
1387 		 */
1388 		u_int scbindex = ahc_inb(ahc, SCB_TAG);
1389 		u_int lastphase = ahc_inb(ahc, LASTPHASE);
1390 		u_int i;
1391 
1392 		scb = ahc_lookup_scb(ahc, scbindex);
1393 		for (i = 0; i < num_phases; i++) {
1394 			if (lastphase == ahc_phase_table[i].phase)
1395 				break;
1396 		}
1397 		ahc_print_path(ahc, scb);
1398 		printk("data overrun detected %s."
1399 		       "  Tag == 0x%x.\n",
1400 		       ahc_phase_table[i].phasemsg,
1401   		       scb->hscb->tag);
1402 		ahc_print_path(ahc, scb);
1403 		printk("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
1404 		       ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
1405 		       ahc_get_transfer_length(scb), scb->sg_count);
1406 		if (scb->sg_count > 0) {
1407 			for (i = 0; i < scb->sg_count; i++) {
1408 
1409 				printk("sg[%d] - Addr 0x%x%x : Length %d\n",
1410 				       i,
1411 				       (ahc_le32toh(scb->sg_list[i].len) >> 24
1412 				        & SG_HIGH_ADDR_BITS),
1413 				       ahc_le32toh(scb->sg_list[i].addr),
1414 				       ahc_le32toh(scb->sg_list[i].len)
1415 				       & AHC_SG_LEN_MASK);
1416 			}
1417 		}
1418 		/*
1419 		 * Set this and it will take effect when the
1420 		 * target does a command complete.
1421 		 */
1422 		ahc_freeze_devq(ahc, scb);
1423 		if ((scb->flags & SCB_SENSE) == 0) {
1424 			ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1425 		} else {
1426 			scb->flags &= ~SCB_SENSE;
1427 			ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1428 		}
1429 		ahc_freeze_scb(scb);
1430 
1431 		if ((ahc->features & AHC_ULTRA2) != 0) {
1432 			/*
1433 			 * Clear the channel in case we return
1434 			 * to data phase later.
1435 			 */
1436 			ahc_outb(ahc, SXFRCTL0,
1437 				 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1438 			ahc_outb(ahc, SXFRCTL0,
1439 				 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1440 		}
1441 		if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
1442 			u_int dscommand1;
1443 
1444 			/* Ensure HHADDR is 0 for future DMA operations. */
1445 			dscommand1 = ahc_inb(ahc, DSCOMMAND1);
1446 			ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
1447 			ahc_outb(ahc, HADDR, 0);
1448 			ahc_outb(ahc, DSCOMMAND1, dscommand1);
1449 		}
1450 		break;
1451 	}
1452 	case MKMSG_FAILED:
1453 	{
1454 		u_int scbindex;
1455 
1456 		printk("%s:%c:%d:%d: Attempt to issue message failed\n",
1457 		       ahc_name(ahc), devinfo.channel, devinfo.target,
1458 		       devinfo.lun);
1459 		scbindex = ahc_inb(ahc, SCB_TAG);
1460 		scb = ahc_lookup_scb(ahc, scbindex);
1461 		if (scb != NULL
1462 		 && (scb->flags & SCB_RECOVERY_SCB) != 0)
1463 			/*
1464 			 * Ensure that we didn't put a second instance of this
1465 			 * SCB into the QINFIFO.
1466 			 */
1467 			ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
1468 					   SCB_GET_CHANNEL(ahc, scb),
1469 					   SCB_GET_LUN(scb), scb->hscb->tag,
1470 					   ROLE_INITIATOR, /*status*/0,
1471 					   SEARCH_REMOVE);
1472 		break;
1473 	}
1474 	case NO_FREE_SCB:
1475 	{
1476 		printk("%s: No free or disconnected SCBs\n", ahc_name(ahc));
1477 		ahc_dump_card_state(ahc);
1478 		panic("for safety");
1479 		break;
1480 	}
1481 	case SCB_MISMATCH:
1482 	{
1483 		u_int scbptr;
1484 
1485 		scbptr = ahc_inb(ahc, SCBPTR);
1486 		printk("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
1487 		       scbptr, ahc_inb(ahc, ARG_1),
1488 		       ahc->scb_data->hscbs[scbptr].tag);
1489 		ahc_dump_card_state(ahc);
1490 		panic("for safety");
1491 		break;
1492 	}
1493 	case OUT_OF_RANGE:
1494 	{
1495 		printk("%s: BTT calculation out of range\n", ahc_name(ahc));
1496 		printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1497 		       "ARG_1 == 0x%x ACCUM = 0x%x\n",
1498 		       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1499 		       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1500 		printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1501 		       "SINDEX == 0x%x\n, A == 0x%x\n",
1502 		       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1503 		       ahc_index_busy_tcl(ahc,
1504 			    BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1505 				      ahc_inb(ahc, SAVED_LUN))),
1506 		       ahc_inb(ahc, SINDEX),
1507 		       ahc_inb(ahc, ACCUM));
1508 		printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1509 		       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1510 		       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1511 		       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1512 		       ahc_inb(ahc, SCB_CONTROL));
1513 		printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1514 		       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1515 		ahc_dump_card_state(ahc);
1516 		panic("for safety");
1517 		break;
1518 	}
1519 	default:
1520 		printk("ahc_intr: seqint, "
1521 		       "intstat == 0x%x, scsisigi = 0x%x\n",
1522 		       intstat, ahc_inb(ahc, SCSISIGI));
1523 		break;
1524 	}
1525 unpause:
1526 	/*
1527 	 *  The sequencer is paused immediately on
1528 	 *  a SEQINT, so we should restart it when
1529 	 *  we're done.
1530 	 */
1531 	ahc_unpause(ahc);
1532 }
1533 
1534 static void
1535 ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
1536 {
1537 	u_int	scb_index;
1538 	u_int	status0;
1539 	u_int	status;
1540 	struct	scb *scb;
1541 	char	cur_channel;
1542 	char	intr_channel;
1543 
1544 	if ((ahc->features & AHC_TWIN) != 0
1545 	 && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
1546 		cur_channel = 'B';
1547 	else
1548 		cur_channel = 'A';
1549 	intr_channel = cur_channel;
1550 
1551 	if ((ahc->features & AHC_ULTRA2) != 0)
1552 		status0 = ahc_inb(ahc, SSTAT0) & IOERR;
1553 	else
1554 		status0 = 0;
1555 	status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1556 	if (status == 0 && status0 == 0) {
1557 		if ((ahc->features & AHC_TWIN) != 0) {
1558 			/* Try the other channel */
1559 		 	ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1560 			status = ahc_inb(ahc, SSTAT1)
1561 			       & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1562 			intr_channel = (cur_channel == 'A') ? 'B' : 'A';
1563 		}
1564 		if (status == 0) {
1565 			printk("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
1566 			ahc_outb(ahc, CLRINT, CLRSCSIINT);
1567 			ahc_unpause(ahc);
1568 			return;
1569 		}
1570 	}
1571 
1572 	/* Make sure the sequencer is in a safe location. */
1573 	ahc_clear_critical_section(ahc);
1574 
1575 	scb_index = ahc_inb(ahc, SCB_TAG);
1576 	scb = ahc_lookup_scb(ahc, scb_index);
1577 	if (scb != NULL
1578 	 && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1579 		scb = NULL;
1580 
1581 	if ((ahc->features & AHC_ULTRA2) != 0
1582 	 && (status0 & IOERR) != 0) {
1583 		int now_lvd;
1584 
1585 		now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
1586 		printk("%s: Transceiver State Has Changed to %s mode\n",
1587 		       ahc_name(ahc), now_lvd ? "LVD" : "SE");
1588 		ahc_outb(ahc, CLRSINT0, CLRIOERR);
1589 		/*
1590 		 * When transitioning to SE mode, the reset line
1591 		 * glitches, triggering an arbitration bug in some
1592 		 * Ultra2 controllers.  This bug is cleared when we
1593 		 * assert the reset line.  Since a reset glitch has
1594 		 * already occurred with this transition and a
1595 		 * transceiver state change is handled just like
1596 		 * a bus reset anyway, asserting the reset line
1597 		 * ourselves is safe.
1598 		 */
1599 		ahc_reset_channel(ahc, intr_channel,
1600 				 /*Initiate Reset*/now_lvd == 0);
1601 	} else if ((status & SCSIRSTI) != 0) {
1602 		printk("%s: Someone reset channel %c\n",
1603 			ahc_name(ahc), intr_channel);
1604 		if (intr_channel != cur_channel)
1605 		 	ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1606 		ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1607 	} else if ((status & SCSIPERR) != 0) {
1608 		/*
1609 		 * Determine the bus phase and queue an appropriate message.
1610 		 * SCSIPERR is latched true as soon as a parity error
1611 		 * occurs.  If the sequencer acked the transfer that
1612 		 * caused the parity error and the currently presented
1613 		 * transfer on the bus has correct parity, SCSIPERR will
1614 		 * be cleared by CLRSCSIPERR.  Use this to determine if
1615 		 * we should look at the last phase the sequencer recorded,
1616 		 * or the current phase presented on the bus.
1617 		 */
1618 		struct	ahc_devinfo devinfo;
1619 		u_int	mesg_out;
1620 		u_int	curphase;
1621 		u_int	errorphase;
1622 		u_int	lastphase;
1623 		u_int	scsirate;
1624 		u_int	i;
1625 		u_int	sstat2;
1626 		int	silent;
1627 
1628 		lastphase = ahc_inb(ahc, LASTPHASE);
1629 		curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1630 		sstat2 = ahc_inb(ahc, SSTAT2);
1631 		ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1632 		/*
1633 		 * For all phases save DATA, the sequencer won't
1634 		 * automatically ack a byte that has a parity error
1635 		 * in it.  So the only way that the current phase
1636 		 * could be 'data-in' is if the parity error is for
1637 		 * an already acked byte in the data phase.  During
1638 		 * synchronous data-in transfers, we may actually
1639 		 * ack bytes before latching the current phase in
1640 		 * LASTPHASE, leading to the discrepancy between
1641 		 * curphase and lastphase.
1642 		 */
1643 		if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1644 		 || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1645 			errorphase = curphase;
1646 		else
1647 			errorphase = lastphase;
1648 
1649 		for (i = 0; i < num_phases; i++) {
1650 			if (errorphase == ahc_phase_table[i].phase)
1651 				break;
1652 		}
1653 		mesg_out = ahc_phase_table[i].mesg_out;
1654 		silent = FALSE;
1655 		if (scb != NULL) {
1656 			if (SCB_IS_SILENT(scb))
1657 				silent = TRUE;
1658 			else
1659 				ahc_print_path(ahc, scb);
1660 			scb->flags |= SCB_TRANSMISSION_ERROR;
1661 		} else
1662 			printk("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1663 			       SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1664 		scsirate = ahc_inb(ahc, SCSIRATE);
1665 		if (silent == FALSE) {
1666 			printk("parity error detected %s. "
1667 			       "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1668 			       ahc_phase_table[i].phasemsg,
1669 			       ahc_inw(ahc, SEQADDR0),
1670 			       scsirate);
1671 			if ((ahc->features & AHC_DT) != 0) {
1672 				if ((sstat2 & CRCVALERR) != 0)
1673 					printk("\tCRC Value Mismatch\n");
1674 				if ((sstat2 & CRCENDERR) != 0)
1675 					printk("\tNo terminal CRC packet "
1676 					       "recevied\n");
1677 				if ((sstat2 & CRCREQERR) != 0)
1678 					printk("\tIllegal CRC packet "
1679 					       "request\n");
1680 				if ((sstat2 & DUAL_EDGE_ERR) != 0)
1681 					printk("\tUnexpected %sDT Data Phase\n",
1682 					       (scsirate & SINGLE_EDGE)
1683 					     ? "" : "non-");
1684 			}
1685 		}
1686 
1687 		if ((ahc->features & AHC_DT) != 0
1688 		 && (sstat2 & DUAL_EDGE_ERR) != 0) {
1689 			/*
1690 			 * This error applies regardless of
1691 			 * data direction, so ignore the value
1692 			 * in the phase table.
1693 			 */
1694 			mesg_out = MSG_INITIATOR_DET_ERR;
1695 		}
1696 
1697 		/*
1698 		 * We've set the hardware to assert ATN if we
1699 		 * get a parity error on "in" phases, so all we
1700 		 * need to do is stuff the message buffer with
1701 		 * the appropriate message.  "In" phases have set
1702 		 * mesg_out to something other than MSG_NOP.
1703 		 */
1704 		if (mesg_out != MSG_NOOP) {
1705 			if (ahc->msg_type != MSG_TYPE_NONE)
1706 				ahc->send_msg_perror = TRUE;
1707 			else
1708 				ahc_outb(ahc, MSG_OUT, mesg_out);
1709 		}
1710 		/*
1711 		 * Force a renegotiation with this target just in
1712 		 * case we are out of sync for some external reason
1713 		 * unknown (or unreported) by the target.
1714 		 */
1715 		ahc_fetch_devinfo(ahc, &devinfo);
1716 		ahc_force_renegotiation(ahc, &devinfo);
1717 
1718 		ahc_outb(ahc, CLRINT, CLRSCSIINT);
1719 		ahc_unpause(ahc);
1720 	} else if ((status & SELTO) != 0) {
1721 		u_int	scbptr;
1722 
1723 		/* Stop the selection */
1724 		ahc_outb(ahc, SCSISEQ, 0);
1725 
1726 		/* No more pending messages */
1727 		ahc_clear_msg_state(ahc);
1728 
1729 		/* Clear interrupt state */
1730 		ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1731 		ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1732 
1733 		/*
1734 		 * Although the driver does not care about the
1735 		 * 'Selection in Progress' status bit, the busy
1736 		 * LED does.  SELINGO is only cleared by a successful
1737 		 * selection, so we must manually clear it to insure
1738 		 * the LED turns off just incase no future successful
1739 		 * selections occur (e.g. no devices on the bus).
1740 		 */
1741 		ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1742 
1743 		scbptr = ahc_inb(ahc, WAITING_SCBH);
1744 		ahc_outb(ahc, SCBPTR, scbptr);
1745 		scb_index = ahc_inb(ahc, SCB_TAG);
1746 
1747 		scb = ahc_lookup_scb(ahc, scb_index);
1748 		if (scb == NULL) {
1749 			printk("%s: ahc_intr - referenced scb not "
1750 			       "valid during SELTO scb(%d, %d)\n",
1751 			       ahc_name(ahc), scbptr, scb_index);
1752 			ahc_dump_card_state(ahc);
1753 		} else {
1754 			struct ahc_devinfo devinfo;
1755 #ifdef AHC_DEBUG
1756 			if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1757 				ahc_print_path(ahc, scb);
1758 				printk("Saw Selection Timeout for SCB 0x%x\n",
1759 				       scb_index);
1760 			}
1761 #endif
1762 			ahc_scb_devinfo(ahc, &devinfo, scb);
1763 			ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1764 			ahc_freeze_devq(ahc, scb);
1765 
1766 			/*
1767 			 * Cancel any pending transactions on the device
1768 			 * now that it seems to be missing.  This will
1769 			 * also revert us to async/narrow transfers until
1770 			 * we can renegotiate with the device.
1771 			 */
1772 			ahc_handle_devreset(ahc, &devinfo,
1773 					    CAM_SEL_TIMEOUT,
1774 					    "Selection Timeout",
1775 					    /*verbose_level*/1);
1776 		}
1777 		ahc_outb(ahc, CLRINT, CLRSCSIINT);
1778 		ahc_restart(ahc);
1779 	} else if ((status & BUSFREE) != 0
1780 		&& (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1781 		struct	ahc_devinfo devinfo;
1782 		u_int	lastphase;
1783 		u_int	saved_scsiid;
1784 		u_int	saved_lun;
1785 		u_int	target;
1786 		u_int	initiator_role_id;
1787 		char	channel;
1788 		int	printerror;
1789 
1790 		/*
1791 		 * Clear our selection hardware as soon as possible.
1792 		 * We may have an entry in the waiting Q for this target,
1793 		 * that is affected by this busfree and we don't want to
1794 		 * go about selecting the target while we handle the event.
1795 		 */
1796 		ahc_outb(ahc, SCSISEQ,
1797 			 ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1798 
1799 		/*
1800 		 * Disable busfree interrupts and clear the busfree
1801 		 * interrupt status.  We do this here so that several
1802 		 * bus transactions occur prior to clearing the SCSIINT
1803 		 * latch.  It can take a bit for the clearing to take effect.
1804 		 */
1805 		ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1806 		ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1807 
1808 		/*
1809 		 * Look at what phase we were last in.
1810 		 * If its message out, chances are pretty good
1811 		 * that the busfree was in response to one of
1812 		 * our abort requests.
1813 		 */
1814 		lastphase = ahc_inb(ahc, LASTPHASE);
1815 		saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1816 		saved_lun = ahc_inb(ahc, SAVED_LUN);
1817 		target = SCSIID_TARGET(ahc, saved_scsiid);
1818 		initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1819 		channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1820 		ahc_compile_devinfo(&devinfo, initiator_role_id,
1821 				    target, saved_lun, channel, ROLE_INITIATOR);
1822 		printerror = 1;
1823 
1824 		if (lastphase == P_MESGOUT) {
1825 			u_int tag;
1826 
1827 			tag = SCB_LIST_NULL;
1828 			if (ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT_TAG, TRUE)
1829 			 || ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT, TRUE)) {
1830 				if (ahc->msgout_buf[ahc->msgout_index - 1]
1831 				 == MSG_ABORT_TAG)
1832 					tag = scb->hscb->tag;
1833 				ahc_print_path(ahc, scb);
1834 				printk("SCB %d - Abort%s Completed.\n",
1835 				       scb->hscb->tag, tag == SCB_LIST_NULL ?
1836 				       "" : " Tag");
1837 				ahc_abort_scbs(ahc, target, channel,
1838 					       saved_lun, tag,
1839 					       ROLE_INITIATOR,
1840 					       CAM_REQ_ABORTED);
1841 				printerror = 0;
1842 			} else if (ahc_sent_msg(ahc, AHCMSG_1B,
1843 						MSG_BUS_DEV_RESET, TRUE)) {
1844 #ifdef __FreeBSD__
1845 				/*
1846 				 * Don't mark the user's request for this BDR
1847 				 * as completing with CAM_BDR_SENT.  CAM3
1848 				 * specifies CAM_REQ_CMP.
1849 				 */
1850 				if (scb != NULL
1851 				 && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
1852 				 && ahc_match_scb(ahc, scb, target, channel,
1853 						  CAM_LUN_WILDCARD,
1854 						  SCB_LIST_NULL,
1855 						  ROLE_INITIATOR)) {
1856 					ahc_set_transaction_status(scb, CAM_REQ_CMP);
1857 				}
1858 #endif
1859 				ahc_compile_devinfo(&devinfo,
1860 						    initiator_role_id,
1861 						    target,
1862 						    CAM_LUN_WILDCARD,
1863 						    channel,
1864 						    ROLE_INITIATOR);
1865 				ahc_handle_devreset(ahc, &devinfo,
1866 						    CAM_BDR_SENT,
1867 						    "Bus Device Reset",
1868 						    /*verbose_level*/0);
1869 				printerror = 0;
1870 			} else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1871 						MSG_EXT_PPR, FALSE)) {
1872 				struct ahc_initiator_tinfo *tinfo;
1873 				struct ahc_tmode_tstate *tstate;
1874 
1875 				/*
1876 				 * PPR Rejected.  Try non-ppr negotiation
1877 				 * and retry command.
1878 				 */
1879 				tinfo = ahc_fetch_transinfo(ahc,
1880 							    devinfo.channel,
1881 							    devinfo.our_scsiid,
1882 							    devinfo.target,
1883 							    &tstate);
1884 				tinfo->curr.transport_version = 2;
1885 				tinfo->goal.transport_version = 2;
1886 				tinfo->goal.ppr_options = 0;
1887 				ahc_qinfifo_requeue_tail(ahc, scb);
1888 				printerror = 0;
1889 			} else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1890 						MSG_EXT_WDTR, FALSE)) {
1891 				/*
1892 				 * Negotiation Rejected.  Go-narrow and
1893 				 * retry command.
1894 				 */
1895 				ahc_set_width(ahc, &devinfo,
1896 					      MSG_EXT_WDTR_BUS_8_BIT,
1897 					      AHC_TRANS_CUR|AHC_TRANS_GOAL,
1898 					      /*paused*/TRUE);
1899 				ahc_qinfifo_requeue_tail(ahc, scb);
1900 				printerror = 0;
1901 			} else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1902 						MSG_EXT_SDTR, FALSE)) {
1903 				/*
1904 				 * Negotiation Rejected.  Go-async and
1905 				 * retry command.
1906 				 */
1907 				ahc_set_syncrate(ahc, &devinfo,
1908 						/*syncrate*/NULL,
1909 						/*period*/0, /*offset*/0,
1910 						/*ppr_options*/0,
1911 						AHC_TRANS_CUR|AHC_TRANS_GOAL,
1912 						/*paused*/TRUE);
1913 				ahc_qinfifo_requeue_tail(ahc, scb);
1914 				printerror = 0;
1915 			}
1916 		}
1917 		if (printerror != 0) {
1918 			u_int i;
1919 
1920 			if (scb != NULL) {
1921 				u_int tag;
1922 
1923 				if ((scb->hscb->control & TAG_ENB) != 0)
1924 					tag = scb->hscb->tag;
1925 				else
1926 					tag = SCB_LIST_NULL;
1927 				ahc_print_path(ahc, scb);
1928 				ahc_abort_scbs(ahc, target, channel,
1929 					       SCB_GET_LUN(scb), tag,
1930 					       ROLE_INITIATOR,
1931 					       CAM_UNEXP_BUSFREE);
1932 			} else {
1933 				/*
1934 				 * We had not fully identified this connection,
1935 				 * so we cannot abort anything.
1936 				 */
1937 				printk("%s: ", ahc_name(ahc));
1938 			}
1939 			for (i = 0; i < num_phases; i++) {
1940 				if (lastphase == ahc_phase_table[i].phase)
1941 					break;
1942 			}
1943 			if (lastphase != P_BUSFREE) {
1944 				/*
1945 				 * Renegotiate with this device at the
1946 				 * next opportunity just in case this busfree
1947 				 * is due to a negotiation mismatch with the
1948 				 * device.
1949 				 */
1950 				ahc_force_renegotiation(ahc, &devinfo);
1951 			}
1952 			printk("Unexpected busfree %s\n"
1953 			       "SEQADDR == 0x%x\n",
1954 			       ahc_phase_table[i].phasemsg,
1955 			       ahc_inb(ahc, SEQADDR0)
1956 				| (ahc_inb(ahc, SEQADDR1) << 8));
1957 		}
1958 		ahc_outb(ahc, CLRINT, CLRSCSIINT);
1959 		ahc_restart(ahc);
1960 	} else {
1961 		printk("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1962 		       ahc_name(ahc), status);
1963 		ahc_outb(ahc, CLRINT, CLRSCSIINT);
1964 	}
1965 }
1966 
1967 /*
1968  * Force renegotiation to occur the next time we initiate
1969  * a command to the current device.
1970  */
1971 static void
1972 ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1973 {
1974 	struct	ahc_initiator_tinfo *targ_info;
1975 	struct	ahc_tmode_tstate *tstate;
1976 
1977 	targ_info = ahc_fetch_transinfo(ahc,
1978 					devinfo->channel,
1979 					devinfo->our_scsiid,
1980 					devinfo->target,
1981 					&tstate);
1982 	ahc_update_neg_request(ahc, devinfo, tstate,
1983 			       targ_info, AHC_NEG_IF_NON_ASYNC);
1984 }
1985 
1986 #define AHC_MAX_STEPS 2000
1987 static void
1988 ahc_clear_critical_section(struct ahc_softc *ahc)
1989 {
1990 	int	stepping;
1991 	int	steps;
1992 	u_int	simode0;
1993 	u_int	simode1;
1994 
1995 	if (ahc->num_critical_sections == 0)
1996 		return;
1997 
1998 	stepping = FALSE;
1999 	steps = 0;
2000 	simode0 = 0;
2001 	simode1 = 0;
2002 	for (;;) {
2003 		struct	cs *cs;
2004 		u_int	seqaddr;
2005 		u_int	i;
2006 
2007 		seqaddr = ahc_inb(ahc, SEQADDR0)
2008 			| (ahc_inb(ahc, SEQADDR1) << 8);
2009 
2010 		/*
2011 		 * Seqaddr represents the next instruction to execute,
2012 		 * so we are really executing the instruction just
2013 		 * before it.
2014 		 */
2015 		if (seqaddr != 0)
2016 			seqaddr -= 1;
2017 		cs = ahc->critical_sections;
2018 		for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
2019 
2020 			if (cs->begin < seqaddr && cs->end >= seqaddr)
2021 				break;
2022 		}
2023 
2024 		if (i == ahc->num_critical_sections)
2025 			break;
2026 
2027 		if (steps > AHC_MAX_STEPS) {
2028 			printk("%s: Infinite loop in critical section\n",
2029 			       ahc_name(ahc));
2030 			ahc_dump_card_state(ahc);
2031 			panic("critical section loop");
2032 		}
2033 
2034 		steps++;
2035 		if (stepping == FALSE) {
2036 
2037 			/*
2038 			 * Disable all interrupt sources so that the
2039 			 * sequencer will not be stuck by a pausing
2040 			 * interrupt condition while we attempt to
2041 			 * leave a critical section.
2042 			 */
2043 			simode0 = ahc_inb(ahc, SIMODE0);
2044 			ahc_outb(ahc, SIMODE0, 0);
2045 			simode1 = ahc_inb(ahc, SIMODE1);
2046 			if ((ahc->features & AHC_DT) != 0)
2047 				/*
2048 				 * On DT class controllers, we
2049 				 * use the enhanced busfree logic.
2050 				 * Unfortunately we cannot re-enable
2051 				 * busfree detection within the
2052 				 * current connection, so we must
2053 				 * leave it on while single stepping.
2054 				 */
2055 				ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
2056 			else
2057 				ahc_outb(ahc, SIMODE1, 0);
2058 			ahc_outb(ahc, CLRINT, CLRSCSIINT);
2059 			ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
2060 			stepping = TRUE;
2061 		}
2062 		if ((ahc->features & AHC_DT) != 0) {
2063 			ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
2064 			ahc_outb(ahc, CLRINT, CLRSCSIINT);
2065 		}
2066 		ahc_outb(ahc, HCNTRL, ahc->unpause);
2067 		while (!ahc_is_paused(ahc))
2068 			ahc_delay(200);
2069 	}
2070 	if (stepping) {
2071 		ahc_outb(ahc, SIMODE0, simode0);
2072 		ahc_outb(ahc, SIMODE1, simode1);
2073 		ahc_outb(ahc, SEQCTL, ahc->seqctl);
2074 	}
2075 }
2076 
2077 /*
2078  * Clear any pending interrupt status.
2079  */
2080 static void
2081 ahc_clear_intstat(struct ahc_softc *ahc)
2082 {
2083 	/* Clear any interrupt conditions this may have caused */
2084 	ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2085 				|CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
2086 				CLRREQINIT);
2087 	ahc_flush_device_writes(ahc);
2088 	ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
2089  	ahc_flush_device_writes(ahc);
2090 	ahc_outb(ahc, CLRINT, CLRSCSIINT);
2091 	ahc_flush_device_writes(ahc);
2092 }
2093 
2094 /**************************** Debugging Routines ******************************/
2095 #ifdef AHC_DEBUG
2096 uint32_t ahc_debug = AHC_DEBUG_OPTS;
2097 #endif
2098 
2099 #if 0 /* unused */
2100 static void
2101 ahc_print_scb(struct scb *scb)
2102 {
2103 	int i;
2104 
2105 	struct hardware_scb *hscb = scb->hscb;
2106 
2107 	printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2108 	       (void *)scb,
2109 	       hscb->control,
2110 	       hscb->scsiid,
2111 	       hscb->lun,
2112 	       hscb->cdb_len);
2113 	printk("Shared Data: ");
2114 	for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
2115 		printk("%#02x", hscb->shared_data.cdb[i]);
2116 	printk("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
2117 		ahc_le32toh(hscb->dataptr),
2118 		ahc_le32toh(hscb->datacnt),
2119 		ahc_le32toh(hscb->sgptr),
2120 		hscb->tag);
2121 	if (scb->sg_count > 0) {
2122 		for (i = 0; i < scb->sg_count; i++) {
2123 			printk("sg[%d] - Addr 0x%x%x : Length %d\n",
2124 			       i,
2125 			       (ahc_le32toh(scb->sg_list[i].len) >> 24
2126 			        & SG_HIGH_ADDR_BITS),
2127 			       ahc_le32toh(scb->sg_list[i].addr),
2128 			       ahc_le32toh(scb->sg_list[i].len));
2129 		}
2130 	}
2131 }
2132 #endif
2133 
2134 /************************* Transfer Negotiation *******************************/
2135 /*
2136  * Allocate per target mode instance (ID we respond to as a target)
2137  * transfer negotiation data structures.
2138  */
2139 static struct ahc_tmode_tstate *
2140 ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
2141 {
2142 	struct ahc_tmode_tstate *master_tstate;
2143 	struct ahc_tmode_tstate *tstate;
2144 	int i;
2145 
2146 	master_tstate = ahc->enabled_targets[ahc->our_id];
2147 	if (channel == 'B') {
2148 		scsi_id += 8;
2149 		master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
2150 	}
2151 	if (ahc->enabled_targets[scsi_id] != NULL
2152 	 && ahc->enabled_targets[scsi_id] != master_tstate)
2153 		panic("%s: ahc_alloc_tstate - Target already allocated",
2154 		      ahc_name(ahc));
2155 	tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
2156 	if (tstate == NULL)
2157 		return (NULL);
2158 
2159 	/*
2160 	 * If we have allocated a master tstate, copy user settings from
2161 	 * the master tstate (taken from SRAM or the EEPROM) for this
2162 	 * channel, but reset our current and goal settings to async/narrow
2163 	 * until an initiator talks to us.
2164 	 */
2165 	if (master_tstate != NULL) {
2166 		memcpy(tstate, master_tstate, sizeof(*tstate));
2167 		memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2168 		tstate->ultraenb = 0;
2169 		for (i = 0; i < AHC_NUM_TARGETS; i++) {
2170 			memset(&tstate->transinfo[i].curr, 0,
2171 			      sizeof(tstate->transinfo[i].curr));
2172 			memset(&tstate->transinfo[i].goal, 0,
2173 			      sizeof(tstate->transinfo[i].goal));
2174 		}
2175 	} else
2176 		memset(tstate, 0, sizeof(*tstate));
2177 	ahc->enabled_targets[scsi_id] = tstate;
2178 	return (tstate);
2179 }
2180 
2181 #ifdef AHC_TARGET_MODE
2182 /*
2183  * Free per target mode instance (ID we respond to as a target)
2184  * transfer negotiation data structures.
2185  */
2186 static void
2187 ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
2188 {
2189 	struct ahc_tmode_tstate *tstate;
2190 
2191 	/*
2192 	 * Don't clean up our "master" tstate.
2193 	 * It has our default user settings.
2194 	 */
2195 	if (((channel == 'B' && scsi_id == ahc->our_id_b)
2196 	  || (channel == 'A' && scsi_id == ahc->our_id))
2197 	 && force == FALSE)
2198 		return;
2199 
2200 	if (channel == 'B')
2201 		scsi_id += 8;
2202 	tstate = ahc->enabled_targets[scsi_id];
2203 	if (tstate != NULL)
2204 		kfree(tstate);
2205 	ahc->enabled_targets[scsi_id] = NULL;
2206 }
2207 #endif
2208 
2209 /*
2210  * Called when we have an active connection to a target on the bus,
2211  * this function finds the nearest syncrate to the input period limited
2212  * by the capabilities of the bus connectivity of and sync settings for
2213  * the target.
2214  */
2215 const struct ahc_syncrate *
2216 ahc_devlimited_syncrate(struct ahc_softc *ahc,
2217 			struct ahc_initiator_tinfo *tinfo,
2218 			u_int *period, u_int *ppr_options, role_t role)
2219 {
2220 	struct	ahc_transinfo *transinfo;
2221 	u_int	maxsync;
2222 
2223 	if ((ahc->features & AHC_ULTRA2) != 0) {
2224 		if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
2225 		 && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
2226 			maxsync = AHC_SYNCRATE_DT;
2227 		} else {
2228 			maxsync = AHC_SYNCRATE_ULTRA;
2229 			/* Can't do DT on an SE bus */
2230 			*ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2231 		}
2232 	} else if ((ahc->features & AHC_ULTRA) != 0) {
2233 		maxsync = AHC_SYNCRATE_ULTRA;
2234 	} else {
2235 		maxsync = AHC_SYNCRATE_FAST;
2236 	}
2237 	/*
2238 	 * Never allow a value higher than our current goal
2239 	 * period otherwise we may allow a target initiated
2240 	 * negotiation to go above the limit as set by the
2241 	 * user.  In the case of an initiator initiated
2242 	 * sync negotiation, we limit based on the user
2243 	 * setting.  This allows the system to still accept
2244 	 * incoming negotiations even if target initiated
2245 	 * negotiation is not performed.
2246 	 */
2247 	if (role == ROLE_TARGET)
2248 		transinfo = &tinfo->user;
2249 	else
2250 		transinfo = &tinfo->goal;
2251 	*ppr_options &= transinfo->ppr_options;
2252 	if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2253 		maxsync = max(maxsync, (u_int)AHC_SYNCRATE_ULTRA2);
2254 		*ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2255 	}
2256 	if (transinfo->period == 0) {
2257 		*period = 0;
2258 		*ppr_options = 0;
2259 		return (NULL);
2260 	}
2261 	*period = max(*period, (u_int)transinfo->period);
2262 	return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
2263 }
2264 
2265 /*
2266  * Look up the valid period to SCSIRATE conversion in our table.
2267  * Return the period and offset that should be sent to the target
2268  * if this was the beginning of an SDTR.
2269  */
2270 const struct ahc_syncrate *
2271 ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
2272 		  u_int *ppr_options, u_int maxsync)
2273 {
2274 	const struct ahc_syncrate *syncrate;
2275 
2276 	if ((ahc->features & AHC_DT) == 0)
2277 		*ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2278 
2279 	/* Skip all DT only entries if DT is not available */
2280 	if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2281 	 && maxsync < AHC_SYNCRATE_ULTRA2)
2282 		maxsync = AHC_SYNCRATE_ULTRA2;
2283 
2284 	/* Now set the maxsync based on the card capabilities
2285 	 * DT is already done above */
2286 	if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2287 	    && maxsync < AHC_SYNCRATE_ULTRA)
2288 		maxsync = AHC_SYNCRATE_ULTRA;
2289 	if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2290 	    && maxsync < AHC_SYNCRATE_FAST)
2291 		maxsync = AHC_SYNCRATE_FAST;
2292 
2293 	for (syncrate = &ahc_syncrates[maxsync];
2294 	     syncrate->rate != NULL;
2295 	     syncrate++) {
2296 
2297 		/*
2298 		 * The Ultra2 table doesn't go as low
2299 		 * as for the Fast/Ultra cards.
2300 		 */
2301 		if ((ahc->features & AHC_ULTRA2) != 0
2302 		 && (syncrate->sxfr_u2 == 0))
2303 			break;
2304 
2305 		if (*period <= syncrate->period) {
2306 			/*
2307 			 * When responding to a target that requests
2308 			 * sync, the requested rate may fall between
2309 			 * two rates that we can output, but still be
2310 			 * a rate that we can receive.  Because of this,
2311 			 * we want to respond to the target with
2312 			 * the same rate that it sent to us even
2313 			 * if the period we use to send data to it
2314 			 * is lower.  Only lower the response period
2315 			 * if we must.
2316 			 */
2317 			if (syncrate == &ahc_syncrates[maxsync])
2318 				*period = syncrate->period;
2319 
2320 			/*
2321 			 * At some speeds, we only support
2322 			 * ST transfers.
2323 			 */
2324 		 	if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
2325 				*ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2326 			break;
2327 		}
2328 	}
2329 
2330 	if ((*period == 0)
2331 	 || (syncrate->rate == NULL)
2332 	 || ((ahc->features & AHC_ULTRA2) != 0
2333 	  && (syncrate->sxfr_u2 == 0))) {
2334 		/* Use asynchronous transfers. */
2335 		*period = 0;
2336 		syncrate = NULL;
2337 		*ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2338 	}
2339 	return (syncrate);
2340 }
2341 
2342 /*
2343  * Convert from an entry in our syncrate table to the SCSI equivalent
2344  * sync "period" factor.
2345  */
2346 u_int
2347 ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
2348 {
2349 	const struct ahc_syncrate *syncrate;
2350 
2351 	if ((ahc->features & AHC_ULTRA2) != 0)
2352 		scsirate &= SXFR_ULTRA2;
2353 	else
2354 		scsirate &= SXFR;
2355 
2356 	/* now set maxsync based on card capabilities */
2357 	if ((ahc->features & AHC_DT) == 0 && maxsync < AHC_SYNCRATE_ULTRA2)
2358 		maxsync = AHC_SYNCRATE_ULTRA2;
2359 	if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2360 	    && maxsync < AHC_SYNCRATE_ULTRA)
2361 		maxsync = AHC_SYNCRATE_ULTRA;
2362 	if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2363 	    && maxsync < AHC_SYNCRATE_FAST)
2364 		maxsync = AHC_SYNCRATE_FAST;
2365 
2366 
2367 	syncrate = &ahc_syncrates[maxsync];
2368 	while (syncrate->rate != NULL) {
2369 
2370 		if ((ahc->features & AHC_ULTRA2) != 0) {
2371 			if (syncrate->sxfr_u2 == 0)
2372 				break;
2373 			else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
2374 				return (syncrate->period);
2375 		} else if (scsirate == (syncrate->sxfr & SXFR)) {
2376 				return (syncrate->period);
2377 		}
2378 		syncrate++;
2379 	}
2380 	return (0); /* async */
2381 }
2382 
2383 /*
2384  * Truncate the given synchronous offset to a value the
2385  * current adapter type and syncrate are capable of.
2386  */
2387 static void
2388 ahc_validate_offset(struct ahc_softc *ahc,
2389 		    struct ahc_initiator_tinfo *tinfo,
2390 		    const struct ahc_syncrate *syncrate,
2391 		    u_int *offset, int wide, role_t role)
2392 {
2393 	u_int maxoffset;
2394 
2395 	/* Limit offset to what we can do */
2396 	if (syncrate == NULL) {
2397 		maxoffset = 0;
2398 	} else if ((ahc->features & AHC_ULTRA2) != 0) {
2399 		maxoffset = MAX_OFFSET_ULTRA2;
2400 	} else {
2401 		if (wide)
2402 			maxoffset = MAX_OFFSET_16BIT;
2403 		else
2404 			maxoffset = MAX_OFFSET_8BIT;
2405 	}
2406 	*offset = min(*offset, maxoffset);
2407 	if (tinfo != NULL) {
2408 		if (role == ROLE_TARGET)
2409 			*offset = min(*offset, (u_int)tinfo->user.offset);
2410 		else
2411 			*offset = min(*offset, (u_int)tinfo->goal.offset);
2412 	}
2413 }
2414 
2415 /*
2416  * Truncate the given transfer width parameter to a value the
2417  * current adapter type is capable of.
2418  */
2419 static void
2420 ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
2421 		   u_int *bus_width, role_t role)
2422 {
2423 	switch (*bus_width) {
2424 	default:
2425 		if (ahc->features & AHC_WIDE) {
2426 			/* Respond Wide */
2427 			*bus_width = MSG_EXT_WDTR_BUS_16_BIT;
2428 			break;
2429 		}
2430 		/* FALLTHROUGH */
2431 	case MSG_EXT_WDTR_BUS_8_BIT:
2432 		*bus_width = MSG_EXT_WDTR_BUS_8_BIT;
2433 		break;
2434 	}
2435 	if (tinfo != NULL) {
2436 		if (role == ROLE_TARGET)
2437 			*bus_width = min((u_int)tinfo->user.width, *bus_width);
2438 		else
2439 			*bus_width = min((u_int)tinfo->goal.width, *bus_width);
2440 	}
2441 }
2442 
2443 /*
2444  * Update the bitmask of targets for which the controller should
2445  * negotiate with at the next convenient opportunity.  This currently
2446  * means the next time we send the initial identify messages for
2447  * a new transaction.
2448  */
2449 int
2450 ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2451 		       struct ahc_tmode_tstate *tstate,
2452 		       struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
2453 {
2454 	u_int auto_negotiate_orig;
2455 
2456 	auto_negotiate_orig = tstate->auto_negotiate;
2457 	if (neg_type == AHC_NEG_ALWAYS) {
2458 		/*
2459 		 * Force our "current" settings to be
2460 		 * unknown so that unless a bus reset
2461 		 * occurs the need to renegotiate is
2462 		 * recorded persistently.
2463 		 */
2464 		if ((ahc->features & AHC_WIDE) != 0)
2465 			tinfo->curr.width = AHC_WIDTH_UNKNOWN;
2466 		tinfo->curr.period = AHC_PERIOD_UNKNOWN;
2467 		tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
2468 	}
2469 	if (tinfo->curr.period != tinfo->goal.period
2470 	 || tinfo->curr.width != tinfo->goal.width
2471 	 || tinfo->curr.offset != tinfo->goal.offset
2472 	 || tinfo->curr.ppr_options != tinfo->goal.ppr_options
2473 	 || (neg_type == AHC_NEG_IF_NON_ASYNC
2474 	  && (tinfo->goal.offset != 0
2475 	   || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
2476 	   || tinfo->goal.ppr_options != 0)))
2477 		tstate->auto_negotiate |= devinfo->target_mask;
2478 	else
2479 		tstate->auto_negotiate &= ~devinfo->target_mask;
2480 
2481 	return (auto_negotiate_orig != tstate->auto_negotiate);
2482 }
2483 
2484 /*
2485  * Update the user/goal/curr tables of synchronous negotiation
2486  * parameters as well as, in the case of a current or active update,
2487  * any data structures on the host controller.  In the case of an
2488  * active update, the specified target is currently talking to us on
2489  * the bus, so the transfer parameter update must take effect
2490  * immediately.
2491  */
2492 void
2493 ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2494 		 const struct ahc_syncrate *syncrate, u_int period,
2495 		 u_int offset, u_int ppr_options, u_int type, int paused)
2496 {
2497 	struct	ahc_initiator_tinfo *tinfo;
2498 	struct	ahc_tmode_tstate *tstate;
2499 	u_int	old_period;
2500 	u_int	old_offset;
2501 	u_int	old_ppr;
2502 	int	active;
2503 	int	update_needed;
2504 
2505 	active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2506 	update_needed = 0;
2507 
2508 	if (syncrate == NULL) {
2509 		period = 0;
2510 		offset = 0;
2511 	}
2512 
2513 	tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2514 				    devinfo->target, &tstate);
2515 
2516 	if ((type & AHC_TRANS_USER) != 0) {
2517 		tinfo->user.period = period;
2518 		tinfo->user.offset = offset;
2519 		tinfo->user.ppr_options = ppr_options;
2520 	}
2521 
2522 	if ((type & AHC_TRANS_GOAL) != 0) {
2523 		tinfo->goal.period = period;
2524 		tinfo->goal.offset = offset;
2525 		tinfo->goal.ppr_options = ppr_options;
2526 	}
2527 
2528 	old_period = tinfo->curr.period;
2529 	old_offset = tinfo->curr.offset;
2530 	old_ppr	   = tinfo->curr.ppr_options;
2531 
2532 	if ((type & AHC_TRANS_CUR) != 0
2533 	 && (old_period != period
2534 	  || old_offset != offset
2535 	  || old_ppr != ppr_options)) {
2536 		u_int	scsirate;
2537 
2538 		update_needed++;
2539 		scsirate = tinfo->scsirate;
2540 		if ((ahc->features & AHC_ULTRA2) != 0) {
2541 
2542 			scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
2543 			if (syncrate != NULL) {
2544 				scsirate |= syncrate->sxfr_u2;
2545 				if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
2546 					scsirate |= ENABLE_CRC;
2547 				else
2548 					scsirate |= SINGLE_EDGE;
2549 			}
2550 		} else {
2551 
2552 			scsirate &= ~(SXFR|SOFS);
2553 			/*
2554 			 * Ensure Ultra mode is set properly for
2555 			 * this target.
2556 			 */
2557 			tstate->ultraenb &= ~devinfo->target_mask;
2558 			if (syncrate != NULL) {
2559 				if (syncrate->sxfr & ULTRA_SXFR) {
2560 					tstate->ultraenb |=
2561 						devinfo->target_mask;
2562 				}
2563 				scsirate |= syncrate->sxfr & SXFR;
2564 				scsirate |= offset & SOFS;
2565 			}
2566 			if (active) {
2567 				u_int sxfrctl0;
2568 
2569 				sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
2570 				sxfrctl0 &= ~FAST20;
2571 				if (tstate->ultraenb & devinfo->target_mask)
2572 					sxfrctl0 |= FAST20;
2573 				ahc_outb(ahc, SXFRCTL0, sxfrctl0);
2574 			}
2575 		}
2576 		if (active) {
2577 			ahc_outb(ahc, SCSIRATE, scsirate);
2578 			if ((ahc->features & AHC_ULTRA2) != 0)
2579 				ahc_outb(ahc, SCSIOFFSET, offset);
2580 		}
2581 
2582 		tinfo->scsirate = scsirate;
2583 		tinfo->curr.period = period;
2584 		tinfo->curr.offset = offset;
2585 		tinfo->curr.ppr_options = ppr_options;
2586 
2587 		ahc_send_async(ahc, devinfo->channel, devinfo->target,
2588 			       CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2589 		if (bootverbose) {
2590 			if (offset != 0) {
2591 				printk("%s: target %d synchronous at %sMHz%s, "
2592 				       "offset = 0x%x\n", ahc_name(ahc),
2593 				       devinfo->target, syncrate->rate,
2594 				       (ppr_options & MSG_EXT_PPR_DT_REQ)
2595 				       ? " DT" : "", offset);
2596 			} else {
2597 				printk("%s: target %d using "
2598 				       "asynchronous transfers\n",
2599 				       ahc_name(ahc), devinfo->target);
2600 			}
2601 		}
2602 	}
2603 
2604 	update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2605 						tinfo, AHC_NEG_TO_GOAL);
2606 
2607 	if (update_needed)
2608 		ahc_update_pending_scbs(ahc);
2609 }
2610 
2611 /*
2612  * Update the user/goal/curr tables of wide negotiation
2613  * parameters as well as, in the case of a current or active update,
2614  * any data structures on the host controller.  In the case of an
2615  * active update, the specified target is currently talking to us on
2616  * the bus, so the transfer parameter update must take effect
2617  * immediately.
2618  */
2619 void
2620 ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2621 	      u_int width, u_int type, int paused)
2622 {
2623 	struct	ahc_initiator_tinfo *tinfo;
2624 	struct	ahc_tmode_tstate *tstate;
2625 	u_int	oldwidth;
2626 	int	active;
2627 	int	update_needed;
2628 
2629 	active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2630 	update_needed = 0;
2631 	tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2632 				    devinfo->target, &tstate);
2633 
2634 	if ((type & AHC_TRANS_USER) != 0)
2635 		tinfo->user.width = width;
2636 
2637 	if ((type & AHC_TRANS_GOAL) != 0)
2638 		tinfo->goal.width = width;
2639 
2640 	oldwidth = tinfo->curr.width;
2641 	if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2642 		u_int	scsirate;
2643 
2644 		update_needed++;
2645 		scsirate =  tinfo->scsirate;
2646 		scsirate &= ~WIDEXFER;
2647 		if (width == MSG_EXT_WDTR_BUS_16_BIT)
2648 			scsirate |= WIDEXFER;
2649 
2650 		tinfo->scsirate = scsirate;
2651 
2652 		if (active)
2653 			ahc_outb(ahc, SCSIRATE, scsirate);
2654 
2655 		tinfo->curr.width = width;
2656 
2657 		ahc_send_async(ahc, devinfo->channel, devinfo->target,
2658 			       CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2659 		if (bootverbose) {
2660 			printk("%s: target %d using %dbit transfers\n",
2661 			       ahc_name(ahc), devinfo->target,
2662 			       8 * (0x01 << width));
2663 		}
2664 	}
2665 
2666 	update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2667 						tinfo, AHC_NEG_TO_GOAL);
2668 	if (update_needed)
2669 		ahc_update_pending_scbs(ahc);
2670 }
2671 
2672 /*
2673  * Update the current state of tagged queuing for a given target.
2674  */
2675 static void
2676 ahc_set_tags(struct ahc_softc *ahc, struct scsi_cmnd *cmd,
2677 	     struct ahc_devinfo *devinfo, ahc_queue_alg alg)
2678 {
2679 	struct scsi_device *sdev = cmd->device;
2680 
2681  	ahc_platform_set_tags(ahc, sdev, devinfo, alg);
2682  	ahc_send_async(ahc, devinfo->channel, devinfo->target,
2683  		       devinfo->lun, AC_TRANSFER_NEG);
2684 }
2685 
2686 /*
2687  * When the transfer settings for a connection change, update any
2688  * in-transit SCBs to contain the new data so the hardware will
2689  * be set correctly during future (re)selections.
2690  */
2691 static void
2692 ahc_update_pending_scbs(struct ahc_softc *ahc)
2693 {
2694 	struct	scb *pending_scb;
2695 	int	pending_scb_count;
2696 	int	i;
2697 	int	paused;
2698 	u_int	saved_scbptr;
2699 
2700 	/*
2701 	 * Traverse the pending SCB list and ensure that all of the
2702 	 * SCBs there have the proper settings.
2703 	 */
2704 	pending_scb_count = 0;
2705 	LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2706 		struct ahc_devinfo devinfo;
2707 		struct hardware_scb *pending_hscb;
2708 		struct ahc_initiator_tinfo *tinfo;
2709 		struct ahc_tmode_tstate *tstate;
2710 
2711 		ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2712 		tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2713 					    devinfo.our_scsiid,
2714 					    devinfo.target, &tstate);
2715 		pending_hscb = pending_scb->hscb;
2716 		pending_hscb->control &= ~ULTRAENB;
2717 		if ((tstate->ultraenb & devinfo.target_mask) != 0)
2718 			pending_hscb->control |= ULTRAENB;
2719 		pending_hscb->scsirate = tinfo->scsirate;
2720 		pending_hscb->scsioffset = tinfo->curr.offset;
2721 		if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2722 		 && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2723 			pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2724 			pending_hscb->control &= ~MK_MESSAGE;
2725 		}
2726 		ahc_sync_scb(ahc, pending_scb,
2727 			     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2728 		pending_scb_count++;
2729 	}
2730 
2731 	if (pending_scb_count == 0)
2732 		return;
2733 
2734 	if (ahc_is_paused(ahc)) {
2735 		paused = 1;
2736 	} else {
2737 		paused = 0;
2738 		ahc_pause(ahc);
2739 	}
2740 
2741 	saved_scbptr = ahc_inb(ahc, SCBPTR);
2742 	/* Ensure that the hscbs down on the card match the new information */
2743 	for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2744 		struct	hardware_scb *pending_hscb;
2745 		u_int	control;
2746 		u_int	scb_tag;
2747 
2748 		ahc_outb(ahc, SCBPTR, i);
2749 		scb_tag = ahc_inb(ahc, SCB_TAG);
2750 		pending_scb = ahc_lookup_scb(ahc, scb_tag);
2751 		if (pending_scb == NULL)
2752 			continue;
2753 
2754 		pending_hscb = pending_scb->hscb;
2755 		control = ahc_inb(ahc, SCB_CONTROL);
2756 		control &= ~(ULTRAENB|MK_MESSAGE);
2757 		control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2758 		ahc_outb(ahc, SCB_CONTROL, control);
2759 		ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2760 		ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2761 	}
2762 	ahc_outb(ahc, SCBPTR, saved_scbptr);
2763 
2764 	if (paused == 0)
2765 		ahc_unpause(ahc);
2766 }
2767 
2768 /**************************** Pathing Information *****************************/
2769 static void
2770 ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2771 {
2772 	u_int	saved_scsiid;
2773 	role_t	role;
2774 	int	our_id;
2775 
2776 	if (ahc_inb(ahc, SSTAT0) & TARGET)
2777 		role = ROLE_TARGET;
2778 	else
2779 		role = ROLE_INITIATOR;
2780 
2781 	if (role == ROLE_TARGET
2782 	 && (ahc->features & AHC_MULTI_TID) != 0
2783 	 && (ahc_inb(ahc, SEQ_FLAGS)
2784  	   & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2785 		/* We were selected, so pull our id from TARGIDIN */
2786 		our_id = ahc_inb(ahc, TARGIDIN) & OID;
2787 	} else if ((ahc->features & AHC_ULTRA2) != 0)
2788 		our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2789 	else
2790 		our_id = ahc_inb(ahc, SCSIID) & OID;
2791 
2792 	saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2793 	ahc_compile_devinfo(devinfo,
2794 			    our_id,
2795 			    SCSIID_TARGET(ahc, saved_scsiid),
2796 			    ahc_inb(ahc, SAVED_LUN),
2797 			    SCSIID_CHANNEL(ahc, saved_scsiid),
2798 			    role);
2799 }
2800 
2801 static const struct ahc_phase_table_entry*
2802 ahc_lookup_phase_entry(int phase)
2803 {
2804 	const struct ahc_phase_table_entry *entry;
2805 	const struct ahc_phase_table_entry *last_entry;
2806 
2807 	/*
2808 	 * num_phases doesn't include the default entry which
2809 	 * will be returned if the phase doesn't match.
2810 	 */
2811 	last_entry = &ahc_phase_table[num_phases];
2812 	for (entry = ahc_phase_table; entry < last_entry; entry++) {
2813 		if (phase == entry->phase)
2814 			break;
2815 	}
2816 	return (entry);
2817 }
2818 
2819 void
2820 ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2821 		    u_int lun, char channel, role_t role)
2822 {
2823 	devinfo->our_scsiid = our_id;
2824 	devinfo->target = target;
2825 	devinfo->lun = lun;
2826 	devinfo->target_offset = target;
2827 	devinfo->channel = channel;
2828 	devinfo->role = role;
2829 	if (channel == 'B')
2830 		devinfo->target_offset += 8;
2831 	devinfo->target_mask = (0x01 << devinfo->target_offset);
2832 }
2833 
2834 void
2835 ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2836 {
2837 	printk("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2838 	       devinfo->target, devinfo->lun);
2839 }
2840 
2841 static void
2842 ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2843 		struct scb *scb)
2844 {
2845 	role_t	role;
2846 	int	our_id;
2847 
2848 	our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2849 	role = ROLE_INITIATOR;
2850 	if ((scb->flags & SCB_TARGET_SCB) != 0)
2851 		role = ROLE_TARGET;
2852 	ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2853 			    SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2854 }
2855 
2856 
2857 /************************ Message Phase Processing ****************************/
2858 static void
2859 ahc_assert_atn(struct ahc_softc *ahc)
2860 {
2861 	u_int scsisigo;
2862 
2863 	scsisigo = ATNO;
2864 	if ((ahc->features & AHC_DT) == 0)
2865 		scsisigo |= ahc_inb(ahc, SCSISIGI);
2866 	ahc_outb(ahc, SCSISIGO, scsisigo);
2867 }
2868 
2869 /*
2870  * When an initiator transaction with the MK_MESSAGE flag either reconnects
2871  * or enters the initial message out phase, we are interrupted.  Fill our
2872  * outgoing message buffer with the appropriate message and beging handing
2873  * the message phase(s) manually.
2874  */
2875 static void
2876 ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2877 			   struct scb *scb)
2878 {
2879 	/*
2880 	 * To facilitate adding multiple messages together,
2881 	 * each routine should increment the index and len
2882 	 * variables instead of setting them explicitly.
2883 	 */
2884 	ahc->msgout_index = 0;
2885 	ahc->msgout_len = 0;
2886 
2887 	if ((scb->flags & SCB_DEVICE_RESET) == 0
2888 	 && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2889 		u_int identify_msg;
2890 
2891 		identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2892 		if ((scb->hscb->control & DISCENB) != 0)
2893 			identify_msg |= MSG_IDENTIFY_DISCFLAG;
2894 		ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2895 		ahc->msgout_len++;
2896 
2897 		if ((scb->hscb->control & TAG_ENB) != 0) {
2898 			ahc->msgout_buf[ahc->msgout_index++] =
2899 			    scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2900 			ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2901 			ahc->msgout_len += 2;
2902 		}
2903 	}
2904 
2905 	if (scb->flags & SCB_DEVICE_RESET) {
2906 		ahc->msgout_buf[ahc->msgout_index++] = MSG_BUS_DEV_RESET;
2907 		ahc->msgout_len++;
2908 		ahc_print_path(ahc, scb);
2909 		printk("Bus Device Reset Message Sent\n");
2910 		/*
2911 		 * Clear our selection hardware in advance of
2912 		 * the busfree.  We may have an entry in the waiting
2913 		 * Q for this target, and we don't want to go about
2914 		 * selecting while we handle the busfree and blow it
2915 		 * away.
2916 		 */
2917 		ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2918 	} else if ((scb->flags & SCB_ABORT) != 0) {
2919 		if ((scb->hscb->control & TAG_ENB) != 0)
2920 			ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT_TAG;
2921 		else
2922 			ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT;
2923 		ahc->msgout_len++;
2924 		ahc_print_path(ahc, scb);
2925 		printk("Abort%s Message Sent\n",
2926 		       (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2927 		/*
2928 		 * Clear our selection hardware in advance of
2929 		 * the busfree.  We may have an entry in the waiting
2930 		 * Q for this target, and we don't want to go about
2931 		 * selecting while we handle the busfree and blow it
2932 		 * away.
2933 		 */
2934 		ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2935 	} else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2936 		ahc_build_transfer_msg(ahc, devinfo);
2937 	} else {
2938 		printk("ahc_intr: AWAITING_MSG for an SCB that "
2939 		       "does not have a waiting message\n");
2940 		printk("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2941 		       devinfo->target_mask);
2942 		panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2943 		      "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2944 		      ahc_inb(ahc, MSG_OUT), scb->flags);
2945 	}
2946 
2947 	/*
2948 	 * Clear the MK_MESSAGE flag from the SCB so we aren't
2949 	 * asked to send this message again.
2950 	 */
2951 	ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2952 	scb->hscb->control &= ~MK_MESSAGE;
2953 	ahc->msgout_index = 0;
2954 	ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2955 }
2956 
2957 /*
2958  * Build an appropriate transfer negotiation message for the
2959  * currently active target.
2960  */
2961 static void
2962 ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2963 {
2964 	/*
2965 	 * We need to initiate transfer negotiations.
2966 	 * If our current and goal settings are identical,
2967 	 * we want to renegotiate due to a check condition.
2968 	 */
2969 	struct	ahc_initiator_tinfo *tinfo;
2970 	struct	ahc_tmode_tstate *tstate;
2971 	const struct ahc_syncrate *rate;
2972 	int	dowide;
2973 	int	dosync;
2974 	int	doppr;
2975 	u_int	period;
2976 	u_int	ppr_options;
2977 	u_int	offset;
2978 
2979 	tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2980 				    devinfo->target, &tstate);
2981 	/*
2982 	 * Filter our period based on the current connection.
2983 	 * If we can't perform DT transfers on this segment (not in LVD
2984 	 * mode for instance), then our decision to issue a PPR message
2985 	 * may change.
2986 	 */
2987 	period = tinfo->goal.period;
2988 	offset = tinfo->goal.offset;
2989 	ppr_options = tinfo->goal.ppr_options;
2990 	/* Target initiated PPR is not allowed in the SCSI spec */
2991 	if (devinfo->role == ROLE_TARGET)
2992 		ppr_options = 0;
2993 	rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2994 				       &ppr_options, devinfo->role);
2995 	dowide = tinfo->curr.width != tinfo->goal.width;
2996 	dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2997 	/*
2998 	 * Only use PPR if we have options that need it, even if the device
2999 	 * claims to support it.  There might be an expander in the way
3000 	 * that doesn't.
3001 	 */
3002 	doppr = ppr_options != 0;
3003 
3004 	if (!dowide && !dosync && !doppr) {
3005 		dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
3006 		dosync = tinfo->goal.offset != 0;
3007 	}
3008 
3009 	if (!dowide && !dosync && !doppr) {
3010 		/*
3011 		 * Force async with a WDTR message if we have a wide bus,
3012 		 * or just issue an SDTR with a 0 offset.
3013 		 */
3014 		if ((ahc->features & AHC_WIDE) != 0)
3015 			dowide = 1;
3016 		else
3017 			dosync = 1;
3018 
3019 		if (bootverbose) {
3020 			ahc_print_devinfo(ahc, devinfo);
3021 			printk("Ensuring async\n");
3022 		}
3023 	}
3024 
3025 	/* Target initiated PPR is not allowed in the SCSI spec */
3026 	if (devinfo->role == ROLE_TARGET)
3027 		doppr = 0;
3028 
3029 	/*
3030 	 * Both the PPR message and SDTR message require the
3031 	 * goal syncrate to be limited to what the target device
3032 	 * is capable of handling (based on whether an LVD->SE
3033 	 * expander is on the bus), so combine these two cases.
3034 	 * Regardless, guarantee that if we are using WDTR and SDTR
3035 	 * messages that WDTR comes first.
3036 	 */
3037 	if (doppr || (dosync && !dowide)) {
3038 
3039 		offset = tinfo->goal.offset;
3040 		ahc_validate_offset(ahc, tinfo, rate, &offset,
3041 				    doppr ? tinfo->goal.width
3042 					  : tinfo->curr.width,
3043 				    devinfo->role);
3044 		if (doppr) {
3045 			ahc_construct_ppr(ahc, devinfo, period, offset,
3046 					  tinfo->goal.width, ppr_options);
3047 		} else {
3048 			ahc_construct_sdtr(ahc, devinfo, period, offset);
3049 		}
3050 	} else {
3051 		ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
3052 	}
3053 }
3054 
3055 /*
3056  * Build a synchronous negotiation message in our message
3057  * buffer based on the input parameters.
3058  */
3059 static void
3060 ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3061 		   u_int period, u_int offset)
3062 {
3063 	if (offset == 0)
3064 		period = AHC_ASYNC_XFER_PERIOD;
3065 	ahc->msgout_index += spi_populate_sync_msg(
3066 			ahc->msgout_buf + ahc->msgout_index, period, offset);
3067 	ahc->msgout_len += 5;
3068 	if (bootverbose) {
3069 		printk("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
3070 		       ahc_name(ahc), devinfo->channel, devinfo->target,
3071 		       devinfo->lun, period, offset);
3072 	}
3073 }
3074 
3075 /*
3076  * Build a wide negotiation message in our message
3077  * buffer based on the input parameters.
3078  */
3079 static void
3080 ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3081 		   u_int bus_width)
3082 {
3083 	ahc->msgout_index += spi_populate_width_msg(
3084 			ahc->msgout_buf + ahc->msgout_index, bus_width);
3085 	ahc->msgout_len += 4;
3086 	if (bootverbose) {
3087 		printk("(%s:%c:%d:%d): Sending WDTR %x\n",
3088 		       ahc_name(ahc), devinfo->channel, devinfo->target,
3089 		       devinfo->lun, bus_width);
3090 	}
3091 }
3092 
3093 /*
3094  * Build a parallel protocol request message in our message
3095  * buffer based on the input parameters.
3096  */
3097 static void
3098 ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3099 		  u_int period, u_int offset, u_int bus_width,
3100 		  u_int ppr_options)
3101 {
3102 	if (offset == 0)
3103 		period = AHC_ASYNC_XFER_PERIOD;
3104 	ahc->msgout_index += spi_populate_ppr_msg(
3105 			ahc->msgout_buf + ahc->msgout_index, period, offset,
3106 			bus_width, ppr_options);
3107 	ahc->msgout_len += 8;
3108 	if (bootverbose) {
3109 		printk("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3110 		       "offset %x, ppr_options %x\n", ahc_name(ahc),
3111 		       devinfo->channel, devinfo->target, devinfo->lun,
3112 		       bus_width, period, offset, ppr_options);
3113 	}
3114 }
3115 
3116 /*
3117  * Clear any active message state.
3118  */
3119 static void
3120 ahc_clear_msg_state(struct ahc_softc *ahc)
3121 {
3122 	ahc->msgout_len = 0;
3123 	ahc->msgin_index = 0;
3124 	ahc->msg_type = MSG_TYPE_NONE;
3125 	if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
3126 		/*
3127 		 * The target didn't care to respond to our
3128 		 * message request, so clear ATN.
3129 		 */
3130 		ahc_outb(ahc, CLRSINT1, CLRATNO);
3131 	}
3132 	ahc_outb(ahc, MSG_OUT, MSG_NOOP);
3133 	ahc_outb(ahc, SEQ_FLAGS2,
3134 		 ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3135 }
3136 
3137 static void
3138 ahc_handle_proto_violation(struct ahc_softc *ahc)
3139 {
3140 	struct	ahc_devinfo devinfo;
3141 	struct	scb *scb;
3142 	u_int	scbid;
3143 	u_int	seq_flags;
3144 	u_int	curphase;
3145 	u_int	lastphase;
3146 	int	found;
3147 
3148 	ahc_fetch_devinfo(ahc, &devinfo);
3149 	scbid = ahc_inb(ahc, SCB_TAG);
3150 	scb = ahc_lookup_scb(ahc, scbid);
3151 	seq_flags = ahc_inb(ahc, SEQ_FLAGS);
3152 	curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3153 	lastphase = ahc_inb(ahc, LASTPHASE);
3154 	if ((seq_flags & NOT_IDENTIFIED) != 0) {
3155 
3156 		/*
3157 		 * The reconnecting target either did not send an
3158 		 * identify message, or did, but we didn't find an SCB
3159 		 * to match.
3160 		 */
3161 		ahc_print_devinfo(ahc, &devinfo);
3162 		printk("Target did not send an IDENTIFY message. "
3163 		       "LASTPHASE = 0x%x.\n", lastphase);
3164 		scb = NULL;
3165 	} else if (scb == NULL) {
3166 		/*
3167 		 * We don't seem to have an SCB active for this
3168 		 * transaction.  Print an error and reset the bus.
3169 		 */
3170 		ahc_print_devinfo(ahc, &devinfo);
3171 		printk("No SCB found during protocol violation\n");
3172 		goto proto_violation_reset;
3173 	} else {
3174 		ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3175 		if ((seq_flags & NO_CDB_SENT) != 0) {
3176 			ahc_print_path(ahc, scb);
3177 			printk("No or incomplete CDB sent to device.\n");
3178 		} else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
3179 			/*
3180 			 * The target never bothered to provide status to
3181 			 * us prior to completing the command.  Since we don't
3182 			 * know the disposition of this command, we must attempt
3183 			 * to abort it.  Assert ATN and prepare to send an abort
3184 			 * message.
3185 			 */
3186 			ahc_print_path(ahc, scb);
3187 			printk("Completed command without status.\n");
3188 		} else {
3189 			ahc_print_path(ahc, scb);
3190 			printk("Unknown protocol violation.\n");
3191 			ahc_dump_card_state(ahc);
3192 		}
3193 	}
3194 	if ((lastphase & ~P_DATAIN_DT) == 0
3195 	 || lastphase == P_COMMAND) {
3196 proto_violation_reset:
3197 		/*
3198 		 * Target either went directly to data/command
3199 		 * phase or didn't respond to our ATN.
3200 		 * The only safe thing to do is to blow
3201 		 * it away with a bus reset.
3202 		 */
3203 		found = ahc_reset_channel(ahc, 'A', TRUE);
3204 		printk("%s: Issued Channel %c Bus Reset. "
3205 		       "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
3206 	} else {
3207 		/*
3208 		 * Leave the selection hardware off in case
3209 		 * this abort attempt will affect yet to
3210 		 * be sent commands.
3211 		 */
3212 		ahc_outb(ahc, SCSISEQ,
3213 			 ahc_inb(ahc, SCSISEQ) & ~ENSELO);
3214 		ahc_assert_atn(ahc);
3215 		ahc_outb(ahc, MSG_OUT, HOST_MSG);
3216 		if (scb == NULL) {
3217 			ahc_print_devinfo(ahc, &devinfo);
3218 			ahc->msgout_buf[0] = MSG_ABORT_TASK;
3219 			ahc->msgout_len = 1;
3220 			ahc->msgout_index = 0;
3221 			ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3222 		} else {
3223 			ahc_print_path(ahc, scb);
3224 			scb->flags |= SCB_ABORT;
3225 		}
3226 		printk("Protocol violation %s.  Attempting to abort.\n",
3227 		       ahc_lookup_phase_entry(curphase)->phasemsg);
3228 	}
3229 }
3230 
3231 /*
3232  * Manual message loop handler.
3233  */
3234 static void
3235 ahc_handle_message_phase(struct ahc_softc *ahc)
3236 {
3237 	struct	ahc_devinfo devinfo;
3238 	u_int	bus_phase;
3239 	int	end_session;
3240 
3241 	ahc_fetch_devinfo(ahc, &devinfo);
3242 	end_session = FALSE;
3243 	bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3244 
3245 reswitch:
3246 	switch (ahc->msg_type) {
3247 	case MSG_TYPE_INITIATOR_MSGOUT:
3248 	{
3249 		int lastbyte;
3250 		int phasemis;
3251 		int msgdone;
3252 
3253 		if (ahc->msgout_len == 0)
3254 			panic("HOST_MSG_LOOP interrupt with no active message");
3255 
3256 #ifdef AHC_DEBUG
3257 		if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3258 			ahc_print_devinfo(ahc, &devinfo);
3259 			printk("INITIATOR_MSG_OUT");
3260 		}
3261 #endif
3262 		phasemis = bus_phase != P_MESGOUT;
3263 		if (phasemis) {
3264 #ifdef AHC_DEBUG
3265 			if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3266 				printk(" PHASEMIS %s\n",
3267 				       ahc_lookup_phase_entry(bus_phase)
3268 							     ->phasemsg);
3269 			}
3270 #endif
3271 			if (bus_phase == P_MESGIN) {
3272 				/*
3273 				 * Change gears and see if
3274 				 * this messages is of interest to
3275 				 * us or should be passed back to
3276 				 * the sequencer.
3277 				 */
3278 				ahc_outb(ahc, CLRSINT1, CLRATNO);
3279 				ahc->send_msg_perror = FALSE;
3280 				ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3281 				ahc->msgin_index = 0;
3282 				goto reswitch;
3283 			}
3284 			end_session = TRUE;
3285 			break;
3286 		}
3287 
3288 		if (ahc->send_msg_perror) {
3289 			ahc_outb(ahc, CLRSINT1, CLRATNO);
3290 			ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3291 #ifdef AHC_DEBUG
3292 			if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3293 				printk(" byte 0x%x\n", ahc->send_msg_perror);
3294 #endif
3295 			ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
3296 			break;
3297 		}
3298 
3299 		msgdone	= ahc->msgout_index == ahc->msgout_len;
3300 		if (msgdone) {
3301 			/*
3302 			 * The target has requested a retry.
3303 			 * Re-assert ATN, reset our message index to
3304 			 * 0, and try again.
3305 			 */
3306 			ahc->msgout_index = 0;
3307 			ahc_assert_atn(ahc);
3308 		}
3309 
3310 		lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
3311 		if (lastbyte) {
3312 			/* Last byte is signified by dropping ATN */
3313 			ahc_outb(ahc, CLRSINT1, CLRATNO);
3314 		}
3315 
3316 		/*
3317 		 * Clear our interrupt status and present
3318 		 * the next byte on the bus.
3319 		 */
3320 		ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3321 #ifdef AHC_DEBUG
3322 		if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3323 			printk(" byte 0x%x\n",
3324 			       ahc->msgout_buf[ahc->msgout_index]);
3325 #endif
3326 		ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3327 		break;
3328 	}
3329 	case MSG_TYPE_INITIATOR_MSGIN:
3330 	{
3331 		int phasemis;
3332 		int message_done;
3333 
3334 #ifdef AHC_DEBUG
3335 		if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3336 			ahc_print_devinfo(ahc, &devinfo);
3337 			printk("INITIATOR_MSG_IN");
3338 		}
3339 #endif
3340 		phasemis = bus_phase != P_MESGIN;
3341 		if (phasemis) {
3342 #ifdef AHC_DEBUG
3343 			if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3344 				printk(" PHASEMIS %s\n",
3345 				       ahc_lookup_phase_entry(bus_phase)
3346 							     ->phasemsg);
3347 			}
3348 #endif
3349 			ahc->msgin_index = 0;
3350 			if (bus_phase == P_MESGOUT
3351 			 && (ahc->send_msg_perror == TRUE
3352 			  || (ahc->msgout_len != 0
3353 			   && ahc->msgout_index == 0))) {
3354 				ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3355 				goto reswitch;
3356 			}
3357 			end_session = TRUE;
3358 			break;
3359 		}
3360 
3361 		/* Pull the byte in without acking it */
3362 		ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
3363 #ifdef AHC_DEBUG
3364 		if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3365 			printk(" byte 0x%x\n",
3366 			       ahc->msgin_buf[ahc->msgin_index]);
3367 #endif
3368 
3369 		message_done = ahc_parse_msg(ahc, &devinfo);
3370 
3371 		if (message_done) {
3372 			/*
3373 			 * Clear our incoming message buffer in case there
3374 			 * is another message following this one.
3375 			 */
3376 			ahc->msgin_index = 0;
3377 
3378 			/*
3379 			 * If this message illicited a response,
3380 			 * assert ATN so the target takes us to the
3381 			 * message out phase.
3382 			 */
3383 			if (ahc->msgout_len != 0) {
3384 #ifdef AHC_DEBUG
3385 				if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3386 					ahc_print_devinfo(ahc, &devinfo);
3387 					printk("Asserting ATN for response\n");
3388 				}
3389 #endif
3390 				ahc_assert_atn(ahc);
3391 			}
3392 		} else
3393 			ahc->msgin_index++;
3394 
3395 		if (message_done == MSGLOOP_TERMINATED) {
3396 			end_session = TRUE;
3397 		} else {
3398 			/* Ack the byte */
3399 			ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3400 			ahc_inb(ahc, SCSIDATL);
3401 		}
3402 		break;
3403 	}
3404 	case MSG_TYPE_TARGET_MSGIN:
3405 	{
3406 		int msgdone;
3407 		int msgout_request;
3408 
3409 		if (ahc->msgout_len == 0)
3410 			panic("Target MSGIN with no active message");
3411 
3412 		/*
3413 		 * If we interrupted a mesgout session, the initiator
3414 		 * will not know this until our first REQ.  So, we
3415 		 * only honor mesgout requests after we've sent our
3416 		 * first byte.
3417 		 */
3418 		if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
3419 		 && ahc->msgout_index > 0)
3420 			msgout_request = TRUE;
3421 		else
3422 			msgout_request = FALSE;
3423 
3424 		if (msgout_request) {
3425 
3426 			/*
3427 			 * Change gears and see if
3428 			 * this messages is of interest to
3429 			 * us or should be passed back to
3430 			 * the sequencer.
3431 			 */
3432 			ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
3433 			ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
3434 			ahc->msgin_index = 0;
3435 			/* Dummy read to REQ for first byte */
3436 			ahc_inb(ahc, SCSIDATL);
3437 			ahc_outb(ahc, SXFRCTL0,
3438 				 ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3439 			break;
3440 		}
3441 
3442 		msgdone = ahc->msgout_index == ahc->msgout_len;
3443 		if (msgdone) {
3444 			ahc_outb(ahc, SXFRCTL0,
3445 				 ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3446 			end_session = TRUE;
3447 			break;
3448 		}
3449 
3450 		/*
3451 		 * Present the next byte on the bus.
3452 		 */
3453 		ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3454 		ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3455 		break;
3456 	}
3457 	case MSG_TYPE_TARGET_MSGOUT:
3458 	{
3459 		int lastbyte;
3460 		int msgdone;
3461 
3462 		/*
3463 		 * The initiator signals that this is
3464 		 * the last byte by dropping ATN.
3465 		 */
3466 		lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
3467 
3468 		/*
3469 		 * Read the latched byte, but turn off SPIOEN first
3470 		 * so that we don't inadvertently cause a REQ for the
3471 		 * next byte.
3472 		 */
3473 		ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3474 		ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
3475 		msgdone = ahc_parse_msg(ahc, &devinfo);
3476 		if (msgdone == MSGLOOP_TERMINATED) {
3477 			/*
3478 			 * The message is *really* done in that it caused
3479 			 * us to go to bus free.  The sequencer has already
3480 			 * been reset at this point, so pull the ejection
3481 			 * handle.
3482 			 */
3483 			return;
3484 		}
3485 
3486 		ahc->msgin_index++;
3487 
3488 		/*
3489 		 * XXX Read spec about initiator dropping ATN too soon
3490 		 *     and use msgdone to detect it.
3491 		 */
3492 		if (msgdone == MSGLOOP_MSGCOMPLETE) {
3493 			ahc->msgin_index = 0;
3494 
3495 			/*
3496 			 * If this message illicited a response, transition
3497 			 * to the Message in phase and send it.
3498 			 */
3499 			if (ahc->msgout_len != 0) {
3500 				ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
3501 				ahc_outb(ahc, SXFRCTL0,
3502 					 ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3503 				ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3504 				ahc->msgin_index = 0;
3505 				break;
3506 			}
3507 		}
3508 
3509 		if (lastbyte)
3510 			end_session = TRUE;
3511 		else {
3512 			/* Ask for the next byte. */
3513 			ahc_outb(ahc, SXFRCTL0,
3514 				 ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3515 		}
3516 
3517 		break;
3518 	}
3519 	default:
3520 		panic("Unknown REQINIT message type");
3521 	}
3522 
3523 	if (end_session) {
3524 		ahc_clear_msg_state(ahc);
3525 		ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
3526 	} else
3527 		ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
3528 }
3529 
3530 /*
3531  * See if we sent a particular extended message to the target.
3532  * If "full" is true, return true only if the target saw the full
3533  * message.  If "full" is false, return true if the target saw at
3534  * least the first byte of the message.
3535  */
3536 static int
3537 ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
3538 {
3539 	int found;
3540 	u_int index;
3541 
3542 	found = FALSE;
3543 	index = 0;
3544 
3545 	while (index < ahc->msgout_len) {
3546 		if (ahc->msgout_buf[index] == MSG_EXTENDED) {
3547 			u_int end_index;
3548 
3549 			end_index = index + 1 + ahc->msgout_buf[index + 1];
3550 			if (ahc->msgout_buf[index+2] == msgval
3551 			 && type == AHCMSG_EXT) {
3552 
3553 				if (full) {
3554 					if (ahc->msgout_index > end_index)
3555 						found = TRUE;
3556 				} else if (ahc->msgout_index > index)
3557 					found = TRUE;
3558 			}
3559 			index = end_index;
3560 		} else if (ahc->msgout_buf[index] >= MSG_SIMPLE_TASK
3561 			&& ahc->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
3562 
3563 			/* Skip tag type and tag id or residue param*/
3564 			index += 2;
3565 		} else {
3566 			/* Single byte message */
3567 			if (type == AHCMSG_1B
3568 			 && ahc->msgout_buf[index] == msgval
3569 			 && ahc->msgout_index > index)
3570 				found = TRUE;
3571 			index++;
3572 		}
3573 
3574 		if (found)
3575 			break;
3576 	}
3577 	return (found);
3578 }
3579 
3580 /*
3581  * Wait for a complete incoming message, parse it, and respond accordingly.
3582  */
3583 static int
3584 ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3585 {
3586 	struct	ahc_initiator_tinfo *tinfo;
3587 	struct	ahc_tmode_tstate *tstate;
3588 	int	reject;
3589 	int	done;
3590 	int	response;
3591 	u_int	targ_scsirate;
3592 
3593 	done = MSGLOOP_IN_PROG;
3594 	response = FALSE;
3595 	reject = FALSE;
3596 	tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
3597 				    devinfo->target, &tstate);
3598 	targ_scsirate = tinfo->scsirate;
3599 
3600 	/*
3601 	 * Parse as much of the message as is available,
3602 	 * rejecting it if we don't support it.  When
3603 	 * the entire message is available and has been
3604 	 * handled, return MSGLOOP_MSGCOMPLETE, indicating
3605 	 * that we have parsed an entire message.
3606 	 *
3607 	 * In the case of extended messages, we accept the length
3608 	 * byte outright and perform more checking once we know the
3609 	 * extended message type.
3610 	 */
3611 	switch (ahc->msgin_buf[0]) {
3612 	case MSG_DISCONNECT:
3613 	case MSG_SAVEDATAPOINTER:
3614 	case MSG_CMDCOMPLETE:
3615 	case MSG_RESTOREPOINTERS:
3616 	case MSG_IGN_WIDE_RESIDUE:
3617 		/*
3618 		 * End our message loop as these are messages
3619 		 * the sequencer handles on its own.
3620 		 */
3621 		done = MSGLOOP_TERMINATED;
3622 		break;
3623 	case MSG_MESSAGE_REJECT:
3624 		response = ahc_handle_msg_reject(ahc, devinfo);
3625 		/* FALLTHROUGH */
3626 	case MSG_NOOP:
3627 		done = MSGLOOP_MSGCOMPLETE;
3628 		break;
3629 	case MSG_EXTENDED:
3630 	{
3631 		/* Wait for enough of the message to begin validation */
3632 		if (ahc->msgin_index < 2)
3633 			break;
3634 		switch (ahc->msgin_buf[2]) {
3635 		case MSG_EXT_SDTR:
3636 		{
3637 			const struct ahc_syncrate *syncrate;
3638 			u_int	 period;
3639 			u_int	 ppr_options;
3640 			u_int	 offset;
3641 			u_int	 saved_offset;
3642 
3643 			if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3644 				reject = TRUE;
3645 				break;
3646 			}
3647 
3648 			/*
3649 			 * Wait until we have both args before validating
3650 			 * and acting on this message.
3651 			 *
3652 			 * Add one to MSG_EXT_SDTR_LEN to account for
3653 			 * the extended message preamble.
3654 			 */
3655 			if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3656 				break;
3657 
3658 			period = ahc->msgin_buf[3];
3659 			ppr_options = 0;
3660 			saved_offset = offset = ahc->msgin_buf[4];
3661 			syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3662 							   &ppr_options,
3663 							   devinfo->role);
3664 			ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3665 					    targ_scsirate & WIDEXFER,
3666 					    devinfo->role);
3667 			if (bootverbose) {
3668 				printk("(%s:%c:%d:%d): Received "
3669 				       "SDTR period %x, offset %x\n\t"
3670 				       "Filtered to period %x, offset %x\n",
3671 				       ahc_name(ahc), devinfo->channel,
3672 				       devinfo->target, devinfo->lun,
3673 				       ahc->msgin_buf[3], saved_offset,
3674 				       period, offset);
3675 			}
3676 			ahc_set_syncrate(ahc, devinfo,
3677 					 syncrate, period,
3678 					 offset, ppr_options,
3679 					 AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3680 					 /*paused*/TRUE);
3681 
3682 			/*
3683 			 * See if we initiated Sync Negotiation
3684 			 * and didn't have to fall down to async
3685 			 * transfers.
3686 			 */
3687 			if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3688 				/* We started it */
3689 				if (saved_offset != offset) {
3690 					/* Went too low - force async */
3691 					reject = TRUE;
3692 				}
3693 			} else {
3694 				/*
3695 				 * Send our own SDTR in reply
3696 				 */
3697 				if (bootverbose
3698 				 && devinfo->role == ROLE_INITIATOR) {
3699 					printk("(%s:%c:%d:%d): Target "
3700 					       "Initiated SDTR\n",
3701 					       ahc_name(ahc), devinfo->channel,
3702 					       devinfo->target, devinfo->lun);
3703 				}
3704 				ahc->msgout_index = 0;
3705 				ahc->msgout_len = 0;
3706 				ahc_construct_sdtr(ahc, devinfo,
3707 						   period, offset);
3708 				ahc->msgout_index = 0;
3709 				response = TRUE;
3710 			}
3711 			done = MSGLOOP_MSGCOMPLETE;
3712 			break;
3713 		}
3714 		case MSG_EXT_WDTR:
3715 		{
3716 			u_int bus_width;
3717 			u_int saved_width;
3718 			u_int sending_reply;
3719 
3720 			sending_reply = FALSE;
3721 			if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3722 				reject = TRUE;
3723 				break;
3724 			}
3725 
3726 			/*
3727 			 * Wait until we have our arg before validating
3728 			 * and acting on this message.
3729 			 *
3730 			 * Add one to MSG_EXT_WDTR_LEN to account for
3731 			 * the extended message preamble.
3732 			 */
3733 			if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3734 				break;
3735 
3736 			bus_width = ahc->msgin_buf[3];
3737 			saved_width = bus_width;
3738 			ahc_validate_width(ahc, tinfo, &bus_width,
3739 					   devinfo->role);
3740 			if (bootverbose) {
3741 				printk("(%s:%c:%d:%d): Received WDTR "
3742 				       "%x filtered to %x\n",
3743 				       ahc_name(ahc), devinfo->channel,
3744 				       devinfo->target, devinfo->lun,
3745 				       saved_width, bus_width);
3746 			}
3747 
3748 			if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3749 				/*
3750 				 * Don't send a WDTR back to the
3751 				 * target, since we asked first.
3752 				 * If the width went higher than our
3753 				 * request, reject it.
3754 				 */
3755 				if (saved_width > bus_width) {
3756 					reject = TRUE;
3757 					printk("(%s:%c:%d:%d): requested %dBit "
3758 					       "transfers.  Rejecting...\n",
3759 					       ahc_name(ahc), devinfo->channel,
3760 					       devinfo->target, devinfo->lun,
3761 					       8 * (0x01 << bus_width));
3762 					bus_width = 0;
3763 				}
3764 			} else {
3765 				/*
3766 				 * Send our own WDTR in reply
3767 				 */
3768 				if (bootverbose
3769 				 && devinfo->role == ROLE_INITIATOR) {
3770 					printk("(%s:%c:%d:%d): Target "
3771 					       "Initiated WDTR\n",
3772 					       ahc_name(ahc), devinfo->channel,
3773 					       devinfo->target, devinfo->lun);
3774 				}
3775 				ahc->msgout_index = 0;
3776 				ahc->msgout_len = 0;
3777 				ahc_construct_wdtr(ahc, devinfo, bus_width);
3778 				ahc->msgout_index = 0;
3779 				response = TRUE;
3780 				sending_reply = TRUE;
3781 			}
3782 			/*
3783 			 * After a wide message, we are async, but
3784 			 * some devices don't seem to honor this portion
3785 			 * of the spec.  Force a renegotiation of the
3786 			 * sync component of our transfer agreement even
3787 			 * if our goal is async.  By updating our width
3788 			 * after forcing the negotiation, we avoid
3789 			 * renegotiating for width.
3790 			 */
3791 			ahc_update_neg_request(ahc, devinfo, tstate,
3792 					       tinfo, AHC_NEG_ALWAYS);
3793 			ahc_set_width(ahc, devinfo, bus_width,
3794 				      AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3795 				      /*paused*/TRUE);
3796 			if (sending_reply == FALSE && reject == FALSE) {
3797 
3798 				/*
3799 				 * We will always have an SDTR to send.
3800 				 */
3801 				ahc->msgout_index = 0;
3802 				ahc->msgout_len = 0;
3803 				ahc_build_transfer_msg(ahc, devinfo);
3804 				ahc->msgout_index = 0;
3805 				response = TRUE;
3806 			}
3807 			done = MSGLOOP_MSGCOMPLETE;
3808 			break;
3809 		}
3810 		case MSG_EXT_PPR:
3811 		{
3812 			const struct ahc_syncrate *syncrate;
3813 			u_int	period;
3814 			u_int	offset;
3815 			u_int	bus_width;
3816 			u_int	ppr_options;
3817 			u_int	saved_width;
3818 			u_int	saved_offset;
3819 			u_int	saved_ppr_options;
3820 
3821 			if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3822 				reject = TRUE;
3823 				break;
3824 			}
3825 
3826 			/*
3827 			 * Wait until we have all args before validating
3828 			 * and acting on this message.
3829 			 *
3830 			 * Add one to MSG_EXT_PPR_LEN to account for
3831 			 * the extended message preamble.
3832 			 */
3833 			if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3834 				break;
3835 
3836 			period = ahc->msgin_buf[3];
3837 			offset = ahc->msgin_buf[5];
3838 			bus_width = ahc->msgin_buf[6];
3839 			saved_width = bus_width;
3840 			ppr_options = ahc->msgin_buf[7];
3841 			/*
3842 			 * According to the spec, a DT only
3843 			 * period factor with no DT option
3844 			 * set implies async.
3845 			 */
3846 			if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3847 			 && period == 9)
3848 				offset = 0;
3849 			saved_ppr_options = ppr_options;
3850 			saved_offset = offset;
3851 
3852 			/*
3853 			 * Mask out any options we don't support
3854 			 * on any controller.  Transfer options are
3855 			 * only available if we are negotiating wide.
3856 			 */
3857 			ppr_options &= MSG_EXT_PPR_DT_REQ;
3858 			if (bus_width == 0)
3859 				ppr_options = 0;
3860 
3861 			ahc_validate_width(ahc, tinfo, &bus_width,
3862 					   devinfo->role);
3863 			syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3864 							   &ppr_options,
3865 							   devinfo->role);
3866 			ahc_validate_offset(ahc, tinfo, syncrate,
3867 					    &offset, bus_width,
3868 					    devinfo->role);
3869 
3870 			if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, TRUE)) {
3871 				/*
3872 				 * If we are unable to do any of the
3873 				 * requested options (we went too low),
3874 				 * then we'll have to reject the message.
3875 				 */
3876 				if (saved_width > bus_width
3877 				 || saved_offset != offset
3878 				 || saved_ppr_options != ppr_options) {
3879 					reject = TRUE;
3880 					period = 0;
3881 					offset = 0;
3882 					bus_width = 0;
3883 					ppr_options = 0;
3884 					syncrate = NULL;
3885 				}
3886 			} else {
3887 				if (devinfo->role != ROLE_TARGET)
3888 					printk("(%s:%c:%d:%d): Target "
3889 					       "Initiated PPR\n",
3890 					       ahc_name(ahc), devinfo->channel,
3891 					       devinfo->target, devinfo->lun);
3892 				else
3893 					printk("(%s:%c:%d:%d): Initiator "
3894 					       "Initiated PPR\n",
3895 					       ahc_name(ahc), devinfo->channel,
3896 					       devinfo->target, devinfo->lun);
3897 				ahc->msgout_index = 0;
3898 				ahc->msgout_len = 0;
3899 				ahc_construct_ppr(ahc, devinfo, period, offset,
3900 						  bus_width, ppr_options);
3901 				ahc->msgout_index = 0;
3902 				response = TRUE;
3903 			}
3904 			if (bootverbose) {
3905 				printk("(%s:%c:%d:%d): Received PPR width %x, "
3906 				       "period %x, offset %x,options %x\n"
3907 				       "\tFiltered to width %x, period %x, "
3908 				       "offset %x, options %x\n",
3909 				       ahc_name(ahc), devinfo->channel,
3910 				       devinfo->target, devinfo->lun,
3911 				       saved_width, ahc->msgin_buf[3],
3912 				       saved_offset, saved_ppr_options,
3913 				       bus_width, period, offset, ppr_options);
3914 			}
3915 			ahc_set_width(ahc, devinfo, bus_width,
3916 				      AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3917 				      /*paused*/TRUE);
3918 			ahc_set_syncrate(ahc, devinfo,
3919 					 syncrate, period,
3920 					 offset, ppr_options,
3921 					 AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3922 					 /*paused*/TRUE);
3923 			done = MSGLOOP_MSGCOMPLETE;
3924 			break;
3925 		}
3926 		default:
3927 			/* Unknown extended message.  Reject it. */
3928 			reject = TRUE;
3929 			break;
3930 		}
3931 		break;
3932 	}
3933 #ifdef AHC_TARGET_MODE
3934 	case MSG_BUS_DEV_RESET:
3935 		ahc_handle_devreset(ahc, devinfo,
3936 				    CAM_BDR_SENT,
3937 				    "Bus Device Reset Received",
3938 				    /*verbose_level*/0);
3939 		ahc_restart(ahc);
3940 		done = MSGLOOP_TERMINATED;
3941 		break;
3942 	case MSG_ABORT_TAG:
3943 	case MSG_ABORT:
3944 	case MSG_CLEAR_QUEUE:
3945 	{
3946 		int tag;
3947 
3948 		/* Target mode messages */
3949 		if (devinfo->role != ROLE_TARGET) {
3950 			reject = TRUE;
3951 			break;
3952 		}
3953 		tag = SCB_LIST_NULL;
3954 		if (ahc->msgin_buf[0] == MSG_ABORT_TAG)
3955 			tag = ahc_inb(ahc, INITIATOR_TAG);
3956 		ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3957 			       devinfo->lun, tag, ROLE_TARGET,
3958 			       CAM_REQ_ABORTED);
3959 
3960 		tstate = ahc->enabled_targets[devinfo->our_scsiid];
3961 		if (tstate != NULL) {
3962 			struct ahc_tmode_lstate* lstate;
3963 
3964 			lstate = tstate->enabled_luns[devinfo->lun];
3965 			if (lstate != NULL) {
3966 				ahc_queue_lstate_event(ahc, lstate,
3967 						       devinfo->our_scsiid,
3968 						       ahc->msgin_buf[0],
3969 						       /*arg*/tag);
3970 				ahc_send_lstate_events(ahc, lstate);
3971 			}
3972 		}
3973 		ahc_restart(ahc);
3974 		done = MSGLOOP_TERMINATED;
3975 		break;
3976 	}
3977 #endif
3978 	case MSG_TERM_IO_PROC:
3979 	default:
3980 		reject = TRUE;
3981 		break;
3982 	}
3983 
3984 	if (reject) {
3985 		/*
3986 		 * Setup to reject the message.
3987 		 */
3988 		ahc->msgout_index = 0;
3989 		ahc->msgout_len = 1;
3990 		ahc->msgout_buf[0] = MSG_MESSAGE_REJECT;
3991 		done = MSGLOOP_MSGCOMPLETE;
3992 		response = TRUE;
3993 	}
3994 
3995 	if (done != MSGLOOP_IN_PROG && !response)
3996 		/* Clear the outgoing message buffer */
3997 		ahc->msgout_len = 0;
3998 
3999 	return (done);
4000 }
4001 
4002 /*
4003  * Process a message reject message.
4004  */
4005 static int
4006 ahc_handle_msg_reject(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
4007 {
4008 	/*
4009 	 * What we care about here is if we had an
4010 	 * outstanding SDTR or WDTR message for this
4011 	 * target.  If we did, this is a signal that
4012 	 * the target is refusing negotiation.
4013 	 */
4014 	struct scb *scb;
4015 	struct ahc_initiator_tinfo *tinfo;
4016 	struct ahc_tmode_tstate *tstate;
4017 	u_int scb_index;
4018 	u_int last_msg;
4019 	int   response = 0;
4020 
4021 	scb_index = ahc_inb(ahc, SCB_TAG);
4022 	scb = ahc_lookup_scb(ahc, scb_index);
4023 	tinfo = ahc_fetch_transinfo(ahc, devinfo->channel,
4024 				    devinfo->our_scsiid,
4025 				    devinfo->target, &tstate);
4026 	/* Might be necessary */
4027 	last_msg = ahc_inb(ahc, LAST_MSG);
4028 
4029 	if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
4030 		/*
4031 		 * Target does not support the PPR message.
4032 		 * Attempt to negotiate SPI-2 style.
4033 		 */
4034 		if (bootverbose) {
4035 			printk("(%s:%c:%d:%d): PPR Rejected. "
4036 			       "Trying WDTR/SDTR\n",
4037 			       ahc_name(ahc), devinfo->channel,
4038 			       devinfo->target, devinfo->lun);
4039 		}
4040 		tinfo->goal.ppr_options = 0;
4041 		tinfo->curr.transport_version = 2;
4042 		tinfo->goal.transport_version = 2;
4043 		ahc->msgout_index = 0;
4044 		ahc->msgout_len = 0;
4045 		ahc_build_transfer_msg(ahc, devinfo);
4046 		ahc->msgout_index = 0;
4047 		response = 1;
4048 	} else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
4049 
4050 		/* note 8bit xfers */
4051 		printk("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
4052 		       "8bit transfers\n", ahc_name(ahc),
4053 		       devinfo->channel, devinfo->target, devinfo->lun);
4054 		ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4055 			      AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
4056 			      /*paused*/TRUE);
4057 		/*
4058 		 * No need to clear the sync rate.  If the target
4059 		 * did not accept the command, our syncrate is
4060 		 * unaffected.  If the target started the negotiation,
4061 		 * but rejected our response, we already cleared the
4062 		 * sync rate before sending our WDTR.
4063 		 */
4064 		if (tinfo->goal.offset != tinfo->curr.offset) {
4065 
4066 			/* Start the sync negotiation */
4067 			ahc->msgout_index = 0;
4068 			ahc->msgout_len = 0;
4069 			ahc_build_transfer_msg(ahc, devinfo);
4070 			ahc->msgout_index = 0;
4071 			response = 1;
4072 		}
4073 	} else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
4074 		/* note asynch xfers and clear flag */
4075 		ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL, /*period*/0,
4076 				 /*offset*/0, /*ppr_options*/0,
4077 				 AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
4078 				 /*paused*/TRUE);
4079 		printk("(%s:%c:%d:%d): refuses synchronous negotiation. "
4080 		       "Using asynchronous transfers\n",
4081 		       ahc_name(ahc), devinfo->channel,
4082 		       devinfo->target, devinfo->lun);
4083 	} else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
4084 		int tag_type;
4085 		int mask;
4086 
4087 		tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
4088 
4089 		if (tag_type == MSG_SIMPLE_TASK) {
4090 			printk("(%s:%c:%d:%d): refuses tagged commands.  "
4091 			       "Performing non-tagged I/O\n", ahc_name(ahc),
4092 			       devinfo->channel, devinfo->target, devinfo->lun);
4093 			ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_NONE);
4094 			mask = ~0x23;
4095 		} else {
4096 			printk("(%s:%c:%d:%d): refuses %s tagged commands.  "
4097 			       "Performing simple queue tagged I/O only\n",
4098 			       ahc_name(ahc), devinfo->channel, devinfo->target,
4099 			       devinfo->lun, tag_type == MSG_ORDERED_TASK
4100 			       ? "ordered" : "head of queue");
4101 			ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_BASIC);
4102 			mask = ~0x03;
4103 		}
4104 
4105 		/*
4106 		 * Resend the identify for this CCB as the target
4107 		 * may believe that the selection is invalid otherwise.
4108 		 */
4109 		ahc_outb(ahc, SCB_CONTROL,
4110 			 ahc_inb(ahc, SCB_CONTROL) & mask);
4111 	 	scb->hscb->control &= mask;
4112 		ahc_set_transaction_tag(scb, /*enabled*/FALSE,
4113 					/*type*/MSG_SIMPLE_TASK);
4114 		ahc_outb(ahc, MSG_OUT, MSG_IDENTIFYFLAG);
4115 		ahc_assert_atn(ahc);
4116 
4117 		/*
4118 		 * This transaction is now at the head of
4119 		 * the untagged queue for this target.
4120 		 */
4121 		if ((ahc->flags & AHC_SCB_BTT) == 0) {
4122 			struct scb_tailq *untagged_q;
4123 
4124 			untagged_q =
4125 			    &(ahc->untagged_queues[devinfo->target_offset]);
4126 			TAILQ_INSERT_HEAD(untagged_q, scb, links.tqe);
4127 			scb->flags |= SCB_UNTAGGEDQ;
4128 		}
4129 		ahc_busy_tcl(ahc, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
4130 			     scb->hscb->tag);
4131 
4132 		/*
4133 		 * Requeue all tagged commands for this target
4134 		 * currently in our possession so they can be
4135 		 * converted to untagged commands.
4136 		 */
4137 		ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
4138 				   SCB_GET_CHANNEL(ahc, scb),
4139 				   SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
4140 				   ROLE_INITIATOR, CAM_REQUEUE_REQ,
4141 				   SEARCH_COMPLETE);
4142 	} else {
4143 		/*
4144 		 * Otherwise, we ignore it.
4145 		 */
4146 		printk("%s:%c:%d: Message reject for %x -- ignored\n",
4147 		       ahc_name(ahc), devinfo->channel, devinfo->target,
4148 		       last_msg);
4149 	}
4150 	return (response);
4151 }
4152 
4153 /*
4154  * Process an ingnore wide residue message.
4155  */
4156 static void
4157 ahc_handle_ign_wide_residue(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
4158 {
4159 	u_int scb_index;
4160 	struct scb *scb;
4161 
4162 	scb_index = ahc_inb(ahc, SCB_TAG);
4163 	scb = ahc_lookup_scb(ahc, scb_index);
4164 	/*
4165 	 * XXX Actually check data direction in the sequencer?
4166 	 * Perhaps add datadir to some spare bits in the hscb?
4167 	 */
4168 	if ((ahc_inb(ahc, SEQ_FLAGS) & DPHASE) == 0
4169 	 || ahc_get_transfer_dir(scb) != CAM_DIR_IN) {
4170 		/*
4171 		 * Ignore the message if we haven't
4172 		 * seen an appropriate data phase yet.
4173 		 */
4174 	} else {
4175 		/*
4176 		 * If the residual occurred on the last
4177 		 * transfer and the transfer request was
4178 		 * expected to end on an odd count, do
4179 		 * nothing.  Otherwise, subtract a byte
4180 		 * and update the residual count accordingly.
4181 		 */
4182 		uint32_t sgptr;
4183 
4184 		sgptr = ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4185 		if ((sgptr & SG_LIST_NULL) != 0
4186 		 && (ahc_inb(ahc, SCB_LUN) & SCB_XFERLEN_ODD) != 0) {
4187 			/*
4188 			 * If the residual occurred on the last
4189 			 * transfer and the transfer request was
4190 			 * expected to end on an odd count, do
4191 			 * nothing.
4192 			 */
4193 		} else {
4194 			struct ahc_dma_seg *sg;
4195 			uint32_t data_cnt;
4196 			uint32_t data_addr;
4197 			uint32_t sglen;
4198 
4199 			/* Pull in all of the sgptr */
4200 			sgptr = ahc_inl(ahc, SCB_RESIDUAL_SGPTR);
4201 			data_cnt = ahc_inl(ahc, SCB_RESIDUAL_DATACNT);
4202 
4203 			if ((sgptr & SG_LIST_NULL) != 0) {
4204 				/*
4205 				 * The residual data count is not updated
4206 				 * for the command run to completion case.
4207 				 * Explicitly zero the count.
4208 				 */
4209 				data_cnt &= ~AHC_SG_LEN_MASK;
4210 			}
4211 
4212 			data_addr = ahc_inl(ahc, SHADDR);
4213 
4214 			data_cnt += 1;
4215 			data_addr -= 1;
4216 			sgptr &= SG_PTR_MASK;
4217 
4218 			sg = ahc_sg_bus_to_virt(scb, sgptr);
4219 
4220 			/*
4221 			 * The residual sg ptr points to the next S/G
4222 			 * to load so we must go back one.
4223 			 */
4224 			sg--;
4225 			sglen = ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
4226 			if (sg != scb->sg_list
4227 			 && sglen < (data_cnt & AHC_SG_LEN_MASK)) {
4228 
4229 				sg--;
4230 				sglen = ahc_le32toh(sg->len);
4231 				/*
4232 				 * Preserve High Address and SG_LIST bits
4233 				 * while setting the count to 1.
4234 				 */
4235 				data_cnt = 1 | (sglen & (~AHC_SG_LEN_MASK));
4236 				data_addr = ahc_le32toh(sg->addr)
4237 					  + (sglen & AHC_SG_LEN_MASK) - 1;
4238 
4239 				/*
4240 				 * Increment sg so it points to the
4241 				 * "next" sg.
4242 				 */
4243 				sg++;
4244 				sgptr = ahc_sg_virt_to_bus(scb, sg);
4245 			}
4246 			ahc_outl(ahc, SCB_RESIDUAL_SGPTR, sgptr);
4247 			ahc_outl(ahc, SCB_RESIDUAL_DATACNT, data_cnt);
4248 			/*
4249 			 * Toggle the "oddness" of the transfer length
4250 			 * to handle this mid-transfer ignore wide
4251 			 * residue.  This ensures that the oddness is
4252 			 * correct for subsequent data transfers.
4253 			 */
4254 			ahc_outb(ahc, SCB_LUN,
4255 				 ahc_inb(ahc, SCB_LUN) ^ SCB_XFERLEN_ODD);
4256 		}
4257 	}
4258 }
4259 
4260 
4261 /*
4262  * Reinitialize the data pointers for the active transfer
4263  * based on its current residual.
4264  */
4265 static void
4266 ahc_reinitialize_dataptrs(struct ahc_softc *ahc)
4267 {
4268 	struct	 scb *scb;
4269 	struct	 ahc_dma_seg *sg;
4270 	u_int	 scb_index;
4271 	uint32_t sgptr;
4272 	uint32_t resid;
4273 	uint32_t dataptr;
4274 
4275 	scb_index = ahc_inb(ahc, SCB_TAG);
4276 	scb = ahc_lookup_scb(ahc, scb_index);
4277 	sgptr = (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 3) << 24)
4278 	      | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 2) << 16)
4279 	      | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 1) << 8)
4280 	      |	ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4281 
4282 	sgptr &= SG_PTR_MASK;
4283 	sg = ahc_sg_bus_to_virt(scb, sgptr);
4284 
4285 	/* The residual sg_ptr always points to the next sg */
4286 	sg--;
4287 
4288 	resid = (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 2) << 16)
4289 	      | (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 1) << 8)
4290 	      | ahc_inb(ahc, SCB_RESIDUAL_DATACNT);
4291 
4292 	dataptr = ahc_le32toh(sg->addr)
4293 		+ (ahc_le32toh(sg->len) & AHC_SG_LEN_MASK)
4294 		- resid;
4295 	if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
4296 		u_int dscommand1;
4297 
4298 		dscommand1 = ahc_inb(ahc, DSCOMMAND1);
4299 		ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
4300 		ahc_outb(ahc, HADDR,
4301 			 (ahc_le32toh(sg->len) >> 24) & SG_HIGH_ADDR_BITS);
4302 		ahc_outb(ahc, DSCOMMAND1, dscommand1);
4303 	}
4304 	ahc_outb(ahc, HADDR + 3, dataptr >> 24);
4305 	ahc_outb(ahc, HADDR + 2, dataptr >> 16);
4306 	ahc_outb(ahc, HADDR + 1, dataptr >> 8);
4307 	ahc_outb(ahc, HADDR, dataptr);
4308 	ahc_outb(ahc, HCNT + 2, resid >> 16);
4309 	ahc_outb(ahc, HCNT + 1, resid >> 8);
4310 	ahc_outb(ahc, HCNT, resid);
4311 	if ((ahc->features & AHC_ULTRA2) == 0) {
4312 		ahc_outb(ahc, STCNT + 2, resid >> 16);
4313 		ahc_outb(ahc, STCNT + 1, resid >> 8);
4314 		ahc_outb(ahc, STCNT, resid);
4315 	}
4316 }
4317 
4318 /*
4319  * Handle the effects of issuing a bus device reset message.
4320  */
4321 static void
4322 ahc_handle_devreset(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4323 		    cam_status status, char *message, int verbose_level)
4324 {
4325 #ifdef AHC_TARGET_MODE
4326 	struct ahc_tmode_tstate* tstate;
4327 	u_int lun;
4328 #endif
4329 	int found;
4330 
4331 	found = ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
4332 			       CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
4333 			       status);
4334 
4335 #ifdef AHC_TARGET_MODE
4336 	/*
4337 	 * Send an immediate notify ccb to all target mord peripheral
4338 	 * drivers affected by this action.
4339 	 */
4340 	tstate = ahc->enabled_targets[devinfo->our_scsiid];
4341 	if (tstate != NULL) {
4342 		for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
4343 			struct ahc_tmode_lstate* lstate;
4344 
4345 			lstate = tstate->enabled_luns[lun];
4346 			if (lstate == NULL)
4347 				continue;
4348 
4349 			ahc_queue_lstate_event(ahc, lstate, devinfo->our_scsiid,
4350 					       MSG_BUS_DEV_RESET, /*arg*/0);
4351 			ahc_send_lstate_events(ahc, lstate);
4352 		}
4353 	}
4354 #endif
4355 
4356 	/*
4357 	 * Go back to async/narrow transfers and renegotiate.
4358 	 */
4359 	ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4360 		      AHC_TRANS_CUR, /*paused*/TRUE);
4361 	ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL,
4362 			 /*period*/0, /*offset*/0, /*ppr_options*/0,
4363 			 AHC_TRANS_CUR, /*paused*/TRUE);
4364 
4365 	if (status != CAM_SEL_TIMEOUT)
4366 		ahc_send_async(ahc, devinfo->channel, devinfo->target,
4367 			       CAM_LUN_WILDCARD, AC_SENT_BDR);
4368 
4369 	if (message != NULL
4370 	 && (verbose_level <= bootverbose))
4371 		printk("%s: %s on %c:%d. %d SCBs aborted\n", ahc_name(ahc),
4372 		       message, devinfo->channel, devinfo->target, found);
4373 }
4374 
4375 #ifdef AHC_TARGET_MODE
4376 static void
4377 ahc_setup_target_msgin(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4378 		       struct scb *scb)
4379 {
4380 
4381 	/*
4382 	 * To facilitate adding multiple messages together,
4383 	 * each routine should increment the index and len
4384 	 * variables instead of setting them explicitly.
4385 	 */
4386 	ahc->msgout_index = 0;
4387 	ahc->msgout_len = 0;
4388 
4389 	if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
4390 		ahc_build_transfer_msg(ahc, devinfo);
4391 	else
4392 		panic("ahc_intr: AWAITING target message with no message");
4393 
4394 	ahc->msgout_index = 0;
4395 	ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
4396 }
4397 #endif
4398 /**************************** Initialization **********************************/
4399 /*
4400  * Allocate a controller structure for a new device
4401  * and perform initial initializion.
4402  */
4403 struct ahc_softc *
4404 ahc_alloc(void *platform_arg, char *name)
4405 {
4406 	struct  ahc_softc *ahc;
4407 	int	i;
4408 
4409 #ifndef	__FreeBSD__
4410 	ahc = kmalloc(sizeof(*ahc), GFP_ATOMIC);
4411 	if (!ahc) {
4412 		printk("aic7xxx: cannot malloc softc!\n");
4413 		kfree(name);
4414 		return NULL;
4415 	}
4416 #else
4417 	ahc = device_get_softc((device_t)platform_arg);
4418 #endif
4419 	memset(ahc, 0, sizeof(*ahc));
4420 	ahc->seep_config = kmalloc(sizeof(*ahc->seep_config), GFP_ATOMIC);
4421 	if (ahc->seep_config == NULL) {
4422 #ifndef	__FreeBSD__
4423 		kfree(ahc);
4424 #endif
4425 		kfree(name);
4426 		return (NULL);
4427 	}
4428 	LIST_INIT(&ahc->pending_scbs);
4429 	/* We don't know our unit number until the OSM sets it */
4430 	ahc->name = name;
4431 	ahc->unit = -1;
4432 	ahc->description = NULL;
4433 	ahc->channel = 'A';
4434 	ahc->channel_b = 'B';
4435 	ahc->chip = AHC_NONE;
4436 	ahc->features = AHC_FENONE;
4437 	ahc->bugs = AHC_BUGNONE;
4438 	ahc->flags = AHC_FNONE;
4439 	/*
4440 	 * Default to all error reporting enabled with the
4441 	 * sequencer operating at its fastest speed.
4442 	 * The bus attach code may modify this.
4443 	 */
4444 	ahc->seqctl = FASTMODE;
4445 
4446 	for (i = 0; i < AHC_NUM_TARGETS; i++)
4447 		TAILQ_INIT(&ahc->untagged_queues[i]);
4448 	if (ahc_platform_alloc(ahc, platform_arg) != 0) {
4449 		ahc_free(ahc);
4450 		ahc = NULL;
4451 	}
4452 	return (ahc);
4453 }
4454 
4455 int
4456 ahc_softc_init(struct ahc_softc *ahc)
4457 {
4458 
4459 	/* The IRQMS bit is only valid on VL and EISA chips */
4460 	if ((ahc->chip & AHC_PCI) == 0)
4461 		ahc->unpause = ahc_inb(ahc, HCNTRL) & IRQMS;
4462 	else
4463 		ahc->unpause = 0;
4464 	ahc->pause = ahc->unpause | PAUSE;
4465 	/* XXX The shared scb data stuff should be deprecated */
4466 	if (ahc->scb_data == NULL) {
4467 		ahc->scb_data = kzalloc(sizeof(*ahc->scb_data), GFP_ATOMIC);
4468 		if (ahc->scb_data == NULL)
4469 			return (ENOMEM);
4470 	}
4471 
4472 	return (0);
4473 }
4474 
4475 void
4476 ahc_set_unit(struct ahc_softc *ahc, int unit)
4477 {
4478 	ahc->unit = unit;
4479 }
4480 
4481 void
4482 ahc_set_name(struct ahc_softc *ahc, char *name)
4483 {
4484 	if (ahc->name != NULL)
4485 		kfree(ahc->name);
4486 	ahc->name = name;
4487 }
4488 
4489 void
4490 ahc_free(struct ahc_softc *ahc)
4491 {
4492 	int i;
4493 
4494 	switch (ahc->init_level) {
4495 	default:
4496 	case 5:
4497 		ahc_shutdown(ahc);
4498 		/* FALLTHROUGH */
4499 	case 4:
4500 		ahc_dmamap_unload(ahc, ahc->shared_data_dmat,
4501 				  ahc->shared_data_dmamap);
4502 		/* FALLTHROUGH */
4503 	case 3:
4504 		ahc_dmamem_free(ahc, ahc->shared_data_dmat, ahc->qoutfifo,
4505 				ahc->shared_data_dmamap);
4506 		ahc_dmamap_destroy(ahc, ahc->shared_data_dmat,
4507 				   ahc->shared_data_dmamap);
4508 		/* FALLTHROUGH */
4509 	case 2:
4510 		ahc_dma_tag_destroy(ahc, ahc->shared_data_dmat);
4511 	case 1:
4512 #ifndef __linux__
4513 		ahc_dma_tag_destroy(ahc, ahc->buffer_dmat);
4514 #endif
4515 		break;
4516 	case 0:
4517 		break;
4518 	}
4519 
4520 #ifndef __linux__
4521 	ahc_dma_tag_destroy(ahc, ahc->parent_dmat);
4522 #endif
4523 	ahc_platform_free(ahc);
4524 	ahc_fini_scbdata(ahc);
4525 	for (i = 0; i < AHC_NUM_TARGETS; i++) {
4526 		struct ahc_tmode_tstate *tstate;
4527 
4528 		tstate = ahc->enabled_targets[i];
4529 		if (tstate != NULL) {
4530 #ifdef AHC_TARGET_MODE
4531 			int j;
4532 
4533 			for (j = 0; j < AHC_NUM_LUNS; j++) {
4534 				struct ahc_tmode_lstate *lstate;
4535 
4536 				lstate = tstate->enabled_luns[j];
4537 				if (lstate != NULL) {
4538 					xpt_free_path(lstate->path);
4539 					kfree(lstate);
4540 				}
4541 			}
4542 #endif
4543 			kfree(tstate);
4544 		}
4545 	}
4546 #ifdef AHC_TARGET_MODE
4547 	if (ahc->black_hole != NULL) {
4548 		xpt_free_path(ahc->black_hole->path);
4549 		kfree(ahc->black_hole);
4550 	}
4551 #endif
4552 	if (ahc->name != NULL)
4553 		kfree(ahc->name);
4554 	if (ahc->seep_config != NULL)
4555 		kfree(ahc->seep_config);
4556 #ifndef __FreeBSD__
4557 	kfree(ahc);
4558 #endif
4559 	return;
4560 }
4561 
4562 static void
4563 ahc_shutdown(void *arg)
4564 {
4565 	struct	ahc_softc *ahc;
4566 	int	i;
4567 
4568 	ahc = (struct ahc_softc *)arg;
4569 
4570 	/* This will reset most registers to 0, but not all */
4571 	ahc_reset(ahc, /*reinit*/FALSE);
4572 	ahc_outb(ahc, SCSISEQ, 0);
4573 	ahc_outb(ahc, SXFRCTL0, 0);
4574 	ahc_outb(ahc, DSPCISTATUS, 0);
4575 
4576 	for (i = TARG_SCSIRATE; i < SCSICONF; i++)
4577 		ahc_outb(ahc, i, 0);
4578 }
4579 
4580 /*
4581  * Reset the controller and record some information about it
4582  * that is only available just after a reset.  If "reinit" is
4583  * non-zero, this reset occurred after initial configuration
4584  * and the caller requests that the chip be fully reinitialized
4585  * to a runable state.  Chip interrupts are *not* enabled after
4586  * a reinitialization.  The caller must enable interrupts via
4587  * ahc_intr_enable().
4588  */
4589 int
4590 ahc_reset(struct ahc_softc *ahc, int reinit)
4591 {
4592 	u_int	sblkctl;
4593 	u_int	sxfrctl1_a, sxfrctl1_b;
4594 	int	error;
4595 	int	wait;
4596 
4597 	/*
4598 	 * Preserve the value of the SXFRCTL1 register for all channels.
4599 	 * It contains settings that affect termination and we don't want
4600 	 * to disturb the integrity of the bus.
4601 	 */
4602 	ahc_pause(ahc);
4603 	sxfrctl1_b = 0;
4604 	if ((ahc->chip & AHC_CHIPID_MASK) == AHC_AIC7770) {
4605 		u_int sblkctl;
4606 
4607 		/*
4608 		 * Save channel B's settings in case this chip
4609 		 * is setup for TWIN channel operation.
4610 		 */
4611 		sblkctl = ahc_inb(ahc, SBLKCTL);
4612 		ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4613 		sxfrctl1_b = ahc_inb(ahc, SXFRCTL1);
4614 		ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4615 	}
4616 	sxfrctl1_a = ahc_inb(ahc, SXFRCTL1);
4617 
4618 	ahc_outb(ahc, HCNTRL, CHIPRST | ahc->pause);
4619 
4620 	/*
4621 	 * Ensure that the reset has finished.  We delay 1000us
4622 	 * prior to reading the register to make sure the chip
4623 	 * has sufficiently completed its reset to handle register
4624 	 * accesses.
4625 	 */
4626 	wait = 1000;
4627 	do {
4628 		ahc_delay(1000);
4629 	} while (--wait && !(ahc_inb(ahc, HCNTRL) & CHIPRSTACK));
4630 
4631 	if (wait == 0) {
4632 		printk("%s: WARNING - Failed chip reset!  "
4633 		       "Trying to initialize anyway.\n", ahc_name(ahc));
4634 	}
4635 	ahc_outb(ahc, HCNTRL, ahc->pause);
4636 
4637 	/* Determine channel configuration */
4638 	sblkctl = ahc_inb(ahc, SBLKCTL) & (SELBUSB|SELWIDE);
4639 	/* No Twin Channel PCI cards */
4640 	if ((ahc->chip & AHC_PCI) != 0)
4641 		sblkctl &= ~SELBUSB;
4642 	switch (sblkctl) {
4643 	case 0:
4644 		/* Single Narrow Channel */
4645 		break;
4646 	case 2:
4647 		/* Wide Channel */
4648 		ahc->features |= AHC_WIDE;
4649 		break;
4650 	case 8:
4651 		/* Twin Channel */
4652 		ahc->features |= AHC_TWIN;
4653 		break;
4654 	default:
4655 		printk(" Unsupported adapter type.  Ignoring\n");
4656 		return(-1);
4657 	}
4658 
4659 	/*
4660 	 * Reload sxfrctl1.
4661 	 *
4662 	 * We must always initialize STPWEN to 1 before we
4663 	 * restore the saved values.  STPWEN is initialized
4664 	 * to a tri-state condition which can only be cleared
4665 	 * by turning it on.
4666 	 */
4667 	if ((ahc->features & AHC_TWIN) != 0) {
4668 		u_int sblkctl;
4669 
4670 		sblkctl = ahc_inb(ahc, SBLKCTL);
4671 		ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4672 		ahc_outb(ahc, SXFRCTL1, sxfrctl1_b);
4673 		ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4674 	}
4675 	ahc_outb(ahc, SXFRCTL1, sxfrctl1_a);
4676 
4677 	error = 0;
4678 	if (reinit != 0)
4679 		/*
4680 		 * If a recovery action has forced a chip reset,
4681 		 * re-initialize the chip to our liking.
4682 		 */
4683 		error = ahc->bus_chip_init(ahc);
4684 #ifdef AHC_DUMP_SEQ
4685 	else
4686 		ahc_dumpseq(ahc);
4687 #endif
4688 
4689 	return (error);
4690 }
4691 
4692 /*
4693  * Determine the number of SCBs available on the controller
4694  */
4695 int
4696 ahc_probe_scbs(struct ahc_softc *ahc) {
4697 	int i;
4698 
4699 	for (i = 0; i < AHC_SCB_MAX; i++) {
4700 
4701 		ahc_outb(ahc, SCBPTR, i);
4702 		ahc_outb(ahc, SCB_BASE, i);
4703 		if (ahc_inb(ahc, SCB_BASE) != i)
4704 			break;
4705 		ahc_outb(ahc, SCBPTR, 0);
4706 		if (ahc_inb(ahc, SCB_BASE) != 0)
4707 			break;
4708 	}
4709 	return (i);
4710 }
4711 
4712 static void
4713 ahc_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4714 {
4715 	dma_addr_t *baddr;
4716 
4717 	baddr = (dma_addr_t *)arg;
4718 	*baddr = segs->ds_addr;
4719 }
4720 
4721 static void
4722 ahc_build_free_scb_list(struct ahc_softc *ahc)
4723 {
4724 	int scbsize;
4725 	int i;
4726 
4727 	scbsize = 32;
4728 	if ((ahc->flags & AHC_LSCBS_ENABLED) != 0)
4729 		scbsize = 64;
4730 
4731 	for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
4732 		int j;
4733 
4734 		ahc_outb(ahc, SCBPTR, i);
4735 
4736 		/*
4737 		 * Touch all SCB bytes to avoid parity errors
4738 		 * should one of our debugging routines read
4739 		 * an otherwise uninitiatlized byte.
4740 		 */
4741 		for (j = 0; j < scbsize; j++)
4742 			ahc_outb(ahc, SCB_BASE+j, 0xFF);
4743 
4744 		/* Clear the control byte. */
4745 		ahc_outb(ahc, SCB_CONTROL, 0);
4746 
4747 		/* Set the next pointer */
4748 		if ((ahc->flags & AHC_PAGESCBS) != 0)
4749 			ahc_outb(ahc, SCB_NEXT, i+1);
4750 		else
4751 			ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4752 
4753 		/* Make the tag number, SCSIID, and lun invalid */
4754 		ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
4755 		ahc_outb(ahc, SCB_SCSIID, 0xFF);
4756 		ahc_outb(ahc, SCB_LUN, 0xFF);
4757 	}
4758 
4759 	if ((ahc->flags & AHC_PAGESCBS) != 0) {
4760 		/* SCB 0 heads the free list. */
4761 		ahc_outb(ahc, FREE_SCBH, 0);
4762 	} else {
4763 		/* No free list. */
4764 		ahc_outb(ahc, FREE_SCBH, SCB_LIST_NULL);
4765 	}
4766 
4767 	/* Make sure that the last SCB terminates the free list */
4768 	ahc_outb(ahc, SCBPTR, i-1);
4769 	ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4770 }
4771 
4772 static int
4773 ahc_init_scbdata(struct ahc_softc *ahc)
4774 {
4775 	struct scb_data *scb_data;
4776 
4777 	scb_data = ahc->scb_data;
4778 	SLIST_INIT(&scb_data->free_scbs);
4779 	SLIST_INIT(&scb_data->sg_maps);
4780 
4781 	/* Allocate SCB resources */
4782 	scb_data->scbarray = kzalloc(sizeof(struct scb) * AHC_SCB_MAX_ALLOC,
4783 				GFP_ATOMIC);
4784 	if (scb_data->scbarray == NULL)
4785 		return (ENOMEM);
4786 
4787 	/* Determine the number of hardware SCBs and initialize them */
4788 
4789 	scb_data->maxhscbs = ahc_probe_scbs(ahc);
4790 	if (ahc->scb_data->maxhscbs == 0) {
4791 		printk("%s: No SCB space found\n", ahc_name(ahc));
4792 		return (ENXIO);
4793 	}
4794 
4795 	/*
4796 	 * Create our DMA tags.  These tags define the kinds of device
4797 	 * accessible memory allocations and memory mappings we will
4798 	 * need to perform during normal operation.
4799 	 *
4800 	 * Unless we need to further restrict the allocation, we rely
4801 	 * on the restrictions of the parent dmat, hence the common
4802 	 * use of MAXADDR and MAXSIZE.
4803 	 */
4804 
4805 	/* DMA tag for our hardware scb structures */
4806 	if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4807 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4808 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4809 			       /*highaddr*/BUS_SPACE_MAXADDR,
4810 			       /*filter*/NULL, /*filterarg*/NULL,
4811 			       AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4812 			       /*nsegments*/1,
4813 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4814 			       /*flags*/0, &scb_data->hscb_dmat) != 0) {
4815 		goto error_exit;
4816 	}
4817 
4818 	scb_data->init_level++;
4819 
4820 	/* Allocation for our hscbs */
4821 	if (ahc_dmamem_alloc(ahc, scb_data->hscb_dmat,
4822 			     (void **)&scb_data->hscbs,
4823 			     BUS_DMA_NOWAIT, &scb_data->hscb_dmamap) != 0) {
4824 		goto error_exit;
4825 	}
4826 
4827 	scb_data->init_level++;
4828 
4829 	/* And permanently map them */
4830 	ahc_dmamap_load(ahc, scb_data->hscb_dmat, scb_data->hscb_dmamap,
4831 			scb_data->hscbs,
4832 			AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4833 			ahc_dmamap_cb, &scb_data->hscb_busaddr, /*flags*/0);
4834 
4835 	scb_data->init_level++;
4836 
4837 	/* DMA tag for our sense buffers */
4838 	if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4839 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4840 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4841 			       /*highaddr*/BUS_SPACE_MAXADDR,
4842 			       /*filter*/NULL, /*filterarg*/NULL,
4843 			       AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4844 			       /*nsegments*/1,
4845 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4846 			       /*flags*/0, &scb_data->sense_dmat) != 0) {
4847 		goto error_exit;
4848 	}
4849 
4850 	scb_data->init_level++;
4851 
4852 	/* Allocate them */
4853 	if (ahc_dmamem_alloc(ahc, scb_data->sense_dmat,
4854 			     (void **)&scb_data->sense,
4855 			     BUS_DMA_NOWAIT, &scb_data->sense_dmamap) != 0) {
4856 		goto error_exit;
4857 	}
4858 
4859 	scb_data->init_level++;
4860 
4861 	/* And permanently map them */
4862 	ahc_dmamap_load(ahc, scb_data->sense_dmat, scb_data->sense_dmamap,
4863 			scb_data->sense,
4864 			AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4865 			ahc_dmamap_cb, &scb_data->sense_busaddr, /*flags*/0);
4866 
4867 	scb_data->init_level++;
4868 
4869 	/* DMA tag for our S/G structures.  We allocate in page sized chunks */
4870 	if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/8,
4871 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4872 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4873 			       /*highaddr*/BUS_SPACE_MAXADDR,
4874 			       /*filter*/NULL, /*filterarg*/NULL,
4875 			       PAGE_SIZE, /*nsegments*/1,
4876 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4877 			       /*flags*/0, &scb_data->sg_dmat) != 0) {
4878 		goto error_exit;
4879 	}
4880 
4881 	scb_data->init_level++;
4882 
4883 	/* Perform initial CCB allocation */
4884 	memset(scb_data->hscbs, 0,
4885 	       AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb));
4886 	ahc_alloc_scbs(ahc);
4887 
4888 	if (scb_data->numscbs == 0) {
4889 		printk("%s: ahc_init_scbdata - "
4890 		       "Unable to allocate initial scbs\n",
4891 		       ahc_name(ahc));
4892 		goto error_exit;
4893 	}
4894 
4895 	/*
4896 	 * Reserve the next queued SCB.
4897 	 */
4898 	ahc->next_queued_scb = ahc_get_scb(ahc);
4899 
4900 	/*
4901 	 * Note that we were successful
4902 	 */
4903 	return (0);
4904 
4905 error_exit:
4906 
4907 	return (ENOMEM);
4908 }
4909 
4910 static void
4911 ahc_fini_scbdata(struct ahc_softc *ahc)
4912 {
4913 	struct scb_data *scb_data;
4914 
4915 	scb_data = ahc->scb_data;
4916 	if (scb_data == NULL)
4917 		return;
4918 
4919 	switch (scb_data->init_level) {
4920 	default:
4921 	case 7:
4922 	{
4923 		struct sg_map_node *sg_map;
4924 
4925 		while ((sg_map = SLIST_FIRST(&scb_data->sg_maps))!= NULL) {
4926 			SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4927 			ahc_dmamap_unload(ahc, scb_data->sg_dmat,
4928 					  sg_map->sg_dmamap);
4929 			ahc_dmamem_free(ahc, scb_data->sg_dmat,
4930 					sg_map->sg_vaddr,
4931 					sg_map->sg_dmamap);
4932 			kfree(sg_map);
4933 		}
4934 		ahc_dma_tag_destroy(ahc, scb_data->sg_dmat);
4935 	}
4936 	case 6:
4937 		ahc_dmamap_unload(ahc, scb_data->sense_dmat,
4938 				  scb_data->sense_dmamap);
4939 	case 5:
4940 		ahc_dmamem_free(ahc, scb_data->sense_dmat, scb_data->sense,
4941 				scb_data->sense_dmamap);
4942 		ahc_dmamap_destroy(ahc, scb_data->sense_dmat,
4943 				   scb_data->sense_dmamap);
4944 	case 4:
4945 		ahc_dma_tag_destroy(ahc, scb_data->sense_dmat);
4946 	case 3:
4947 		ahc_dmamap_unload(ahc, scb_data->hscb_dmat,
4948 				  scb_data->hscb_dmamap);
4949 	case 2:
4950 		ahc_dmamem_free(ahc, scb_data->hscb_dmat, scb_data->hscbs,
4951 				scb_data->hscb_dmamap);
4952 		ahc_dmamap_destroy(ahc, scb_data->hscb_dmat,
4953 				   scb_data->hscb_dmamap);
4954 	case 1:
4955 		ahc_dma_tag_destroy(ahc, scb_data->hscb_dmat);
4956 		break;
4957 	case 0:
4958 		break;
4959 	}
4960 	if (scb_data->scbarray != NULL)
4961 		kfree(scb_data->scbarray);
4962 }
4963 
4964 static void
4965 ahc_alloc_scbs(struct ahc_softc *ahc)
4966 {
4967 	struct scb_data *scb_data;
4968 	struct scb *next_scb;
4969 	struct sg_map_node *sg_map;
4970 	dma_addr_t physaddr;
4971 	struct ahc_dma_seg *segs;
4972 	int newcount;
4973 	int i;
4974 
4975 	scb_data = ahc->scb_data;
4976 	if (scb_data->numscbs >= AHC_SCB_MAX_ALLOC)
4977 		/* Can't allocate any more */
4978 		return;
4979 
4980 	next_scb = &scb_data->scbarray[scb_data->numscbs];
4981 
4982 	sg_map = kmalloc(sizeof(*sg_map), GFP_ATOMIC);
4983 
4984 	if (sg_map == NULL)
4985 		return;
4986 
4987 	/* Allocate S/G space for the next batch of SCBS */
4988 	if (ahc_dmamem_alloc(ahc, scb_data->sg_dmat,
4989 			     (void **)&sg_map->sg_vaddr,
4990 			     BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
4991 		kfree(sg_map);
4992 		return;
4993 	}
4994 
4995 	SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
4996 
4997 	ahc_dmamap_load(ahc, scb_data->sg_dmat, sg_map->sg_dmamap,
4998 			sg_map->sg_vaddr, PAGE_SIZE, ahc_dmamap_cb,
4999 			&sg_map->sg_physaddr, /*flags*/0);
5000 
5001 	segs = sg_map->sg_vaddr;
5002 	physaddr = sg_map->sg_physaddr;
5003 
5004 	newcount = (PAGE_SIZE / (AHC_NSEG * sizeof(struct ahc_dma_seg)));
5005 	newcount = min(newcount, (AHC_SCB_MAX_ALLOC - scb_data->numscbs));
5006 	for (i = 0; i < newcount; i++) {
5007 		struct scb_platform_data *pdata;
5008 #ifndef __linux__
5009 		int error;
5010 #endif
5011 		pdata = kmalloc(sizeof(*pdata), GFP_ATOMIC);
5012 		if (pdata == NULL)
5013 			break;
5014 		next_scb->platform_data = pdata;
5015 		next_scb->sg_map = sg_map;
5016 		next_scb->sg_list = segs;
5017 		/*
5018 		 * The sequencer always starts with the second entry.
5019 		 * The first entry is embedded in the scb.
5020 		 */
5021 		next_scb->sg_list_phys = physaddr + sizeof(struct ahc_dma_seg);
5022 		next_scb->ahc_softc = ahc;
5023 		next_scb->flags = SCB_FREE;
5024 #ifndef __linux__
5025 		error = ahc_dmamap_create(ahc, ahc->buffer_dmat, /*flags*/0,
5026 					  &next_scb->dmamap);
5027 		if (error != 0)
5028 			break;
5029 #endif
5030 		next_scb->hscb = &scb_data->hscbs[scb_data->numscbs];
5031 		next_scb->hscb->tag = ahc->scb_data->numscbs;
5032 		SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs,
5033 				  next_scb, links.sle);
5034 		segs += AHC_NSEG;
5035 		physaddr += (AHC_NSEG * sizeof(struct ahc_dma_seg));
5036 		next_scb++;
5037 		ahc->scb_data->numscbs++;
5038 	}
5039 }
5040 
5041 void
5042 ahc_controller_info(struct ahc_softc *ahc, char *buf)
5043 {
5044 	int len;
5045 
5046 	len = sprintf(buf, "%s: ", ahc_chip_names[ahc->chip & AHC_CHIPID_MASK]);
5047 	buf += len;
5048 	if ((ahc->features & AHC_TWIN) != 0)
5049  		len = sprintf(buf, "Twin Channel, A SCSI Id=%d, "
5050 			      "B SCSI Id=%d, primary %c, ",
5051 			      ahc->our_id, ahc->our_id_b,
5052 			      (ahc->flags & AHC_PRIMARY_CHANNEL) + 'A');
5053 	else {
5054 		const char *speed;
5055 		const char *type;
5056 
5057 		speed = "";
5058 		if ((ahc->features & AHC_ULTRA) != 0) {
5059 			speed = "Ultra ";
5060 		} else if ((ahc->features & AHC_DT) != 0) {
5061 			speed = "Ultra160 ";
5062 		} else if ((ahc->features & AHC_ULTRA2) != 0) {
5063 			speed = "Ultra2 ";
5064 		}
5065 		if ((ahc->features & AHC_WIDE) != 0) {
5066 			type = "Wide";
5067 		} else {
5068 			type = "Single";
5069 		}
5070 		len = sprintf(buf, "%s%s Channel %c, SCSI Id=%d, ",
5071 			      speed, type, ahc->channel, ahc->our_id);
5072 	}
5073 	buf += len;
5074 
5075 	if ((ahc->flags & AHC_PAGESCBS) != 0)
5076 		sprintf(buf, "%d/%d SCBs",
5077 			ahc->scb_data->maxhscbs, AHC_MAX_QUEUE);
5078 	else
5079 		sprintf(buf, "%d SCBs", ahc->scb_data->maxhscbs);
5080 }
5081 
5082 int
5083 ahc_chip_init(struct ahc_softc *ahc)
5084 {
5085 	int	 term;
5086 	int	 error;
5087 	u_int	 i;
5088 	u_int	 scsi_conf;
5089 	u_int	 scsiseq_template;
5090 	uint32_t physaddr;
5091 
5092 	ahc_outb(ahc, SEQ_FLAGS, 0);
5093 	ahc_outb(ahc, SEQ_FLAGS2, 0);
5094 
5095 	/* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1, for both channels*/
5096 	if (ahc->features & AHC_TWIN) {
5097 
5098 		/*
5099 		 * Setup Channel B first.
5100 		 */
5101 		ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) | SELBUSB);
5102 		term = (ahc->flags & AHC_TERM_ENB_B) != 0 ? STPWEN : 0;
5103 		ahc_outb(ahc, SCSIID, ahc->our_id_b);
5104 		scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5105 		ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5106 					|term|ahc->seltime_b|ENSTIMER|ACTNEGEN);
5107 		if ((ahc->features & AHC_ULTRA2) != 0)
5108 			ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5109 		ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5110 		ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5111 
5112 		/* Select Channel A */
5113 		ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) & ~SELBUSB);
5114 	}
5115 	term = (ahc->flags & AHC_TERM_ENB_A) != 0 ? STPWEN : 0;
5116 	if ((ahc->features & AHC_ULTRA2) != 0)
5117 		ahc_outb(ahc, SCSIID_ULTRA2, ahc->our_id);
5118 	else
5119 		ahc_outb(ahc, SCSIID, ahc->our_id);
5120 	scsi_conf = ahc_inb(ahc, SCSICONF);
5121 	ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5122 				|term|ahc->seltime
5123 				|ENSTIMER|ACTNEGEN);
5124 	if ((ahc->features & AHC_ULTRA2) != 0)
5125 		ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5126 	ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5127 	ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5128 
5129 	/* There are no untagged SCBs active yet. */
5130 	for (i = 0; i < 16; i++) {
5131 		ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, 0));
5132 		if ((ahc->flags & AHC_SCB_BTT) != 0) {
5133 			int lun;
5134 
5135 			/*
5136 			 * The SCB based BTT allows an entry per
5137 			 * target and lun pair.
5138 			 */
5139 			for (lun = 1; lun < AHC_NUM_LUNS; lun++)
5140 				ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, lun));
5141 		}
5142 	}
5143 
5144 	/* All of our queues are empty */
5145 	for (i = 0; i < 256; i++)
5146 		ahc->qoutfifo[i] = SCB_LIST_NULL;
5147 	ahc_sync_qoutfifo(ahc, BUS_DMASYNC_PREREAD);
5148 
5149 	for (i = 0; i < 256; i++)
5150 		ahc->qinfifo[i] = SCB_LIST_NULL;
5151 
5152 	if ((ahc->features & AHC_MULTI_TID) != 0) {
5153 		ahc_outb(ahc, TARGID, 0);
5154 		ahc_outb(ahc, TARGID + 1, 0);
5155 	}
5156 
5157 	/*
5158 	 * Tell the sequencer where it can find our arrays in memory.
5159 	 */
5160 	physaddr = ahc->scb_data->hscb_busaddr;
5161 	ahc_outb(ahc, HSCB_ADDR, physaddr & 0xFF);
5162 	ahc_outb(ahc, HSCB_ADDR + 1, (physaddr >> 8) & 0xFF);
5163 	ahc_outb(ahc, HSCB_ADDR + 2, (physaddr >> 16) & 0xFF);
5164 	ahc_outb(ahc, HSCB_ADDR + 3, (physaddr >> 24) & 0xFF);
5165 
5166 	physaddr = ahc->shared_data_busaddr;
5167 	ahc_outb(ahc, SHARED_DATA_ADDR, physaddr & 0xFF);
5168 	ahc_outb(ahc, SHARED_DATA_ADDR + 1, (physaddr >> 8) & 0xFF);
5169 	ahc_outb(ahc, SHARED_DATA_ADDR + 2, (physaddr >> 16) & 0xFF);
5170 	ahc_outb(ahc, SHARED_DATA_ADDR + 3, (physaddr >> 24) & 0xFF);
5171 
5172 	/*
5173 	 * Initialize the group code to command length table.
5174 	 * This overrides the values in TARG_SCSIRATE, so only
5175 	 * setup the table after we have processed that information.
5176 	 */
5177 	ahc_outb(ahc, CMDSIZE_TABLE, 5);
5178 	ahc_outb(ahc, CMDSIZE_TABLE + 1, 9);
5179 	ahc_outb(ahc, CMDSIZE_TABLE + 2, 9);
5180 	ahc_outb(ahc, CMDSIZE_TABLE + 3, 0);
5181 	ahc_outb(ahc, CMDSIZE_TABLE + 4, 15);
5182 	ahc_outb(ahc, CMDSIZE_TABLE + 5, 11);
5183 	ahc_outb(ahc, CMDSIZE_TABLE + 6, 0);
5184 	ahc_outb(ahc, CMDSIZE_TABLE + 7, 0);
5185 
5186 	if ((ahc->features & AHC_HS_MAILBOX) != 0)
5187 		ahc_outb(ahc, HS_MAILBOX, 0);
5188 
5189 	/* Tell the sequencer of our initial queue positions */
5190 	if ((ahc->features & AHC_TARGETMODE) != 0) {
5191 		ahc->tqinfifonext = 1;
5192 		ahc_outb(ahc, KERNEL_TQINPOS, ahc->tqinfifonext - 1);
5193 		ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
5194 	}
5195 	ahc->qinfifonext = 0;
5196 	ahc->qoutfifonext = 0;
5197 	if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5198 		ahc_outb(ahc, QOFF_CTLSTA, SCB_QSIZE_256);
5199 		ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5200 		ahc_outb(ahc, SNSCB_QOFF, ahc->qinfifonext);
5201 		ahc_outb(ahc, SDSCB_QOFF, 0);
5202 	} else {
5203 		ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5204 		ahc_outb(ahc, QINPOS, ahc->qinfifonext);
5205 		ahc_outb(ahc, QOUTPOS, ahc->qoutfifonext);
5206 	}
5207 
5208 	/* We don't have any waiting selections */
5209 	ahc_outb(ahc, WAITING_SCBH, SCB_LIST_NULL);
5210 
5211 	/* Our disconnection list is empty too */
5212 	ahc_outb(ahc, DISCONNECTED_SCBH, SCB_LIST_NULL);
5213 
5214 	/* Message out buffer starts empty */
5215 	ahc_outb(ahc, MSG_OUT, MSG_NOOP);
5216 
5217 	/*
5218 	 * Setup the allowed SCSI Sequences based on operational mode.
5219 	 * If we are a target, we'll enable select in operations once
5220 	 * we've had a lun enabled.
5221 	 */
5222 	scsiseq_template = ENSELO|ENAUTOATNO|ENAUTOATNP;
5223 	if ((ahc->flags & AHC_INITIATORROLE) != 0)
5224 		scsiseq_template |= ENRSELI;
5225 	ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq_template);
5226 
5227 	/* Initialize our list of free SCBs. */
5228 	ahc_build_free_scb_list(ahc);
5229 
5230 	/*
5231 	 * Tell the sequencer which SCB will be the next one it receives.
5232 	 */
5233 	ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5234 
5235 	/*
5236 	 * Load the Sequencer program and Enable the adapter
5237 	 * in "fast" mode.
5238 	 */
5239 	if (bootverbose)
5240 		printk("%s: Downloading Sequencer Program...",
5241 		       ahc_name(ahc));
5242 
5243 	error = ahc_loadseq(ahc);
5244 	if (error != 0)
5245 		return (error);
5246 
5247 	if ((ahc->features & AHC_ULTRA2) != 0) {
5248 		int wait;
5249 
5250 		/*
5251 		 * Wait for up to 500ms for our transceivers
5252 		 * to settle.  If the adapter does not have
5253 		 * a cable attached, the transceivers may
5254 		 * never settle, so don't complain if we
5255 		 * fail here.
5256 		 */
5257 		for (wait = 5000;
5258 		     (ahc_inb(ahc, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
5259 		     wait--)
5260 			ahc_delay(100);
5261 	}
5262 	ahc_restart(ahc);
5263 	return (0);
5264 }
5265 
5266 /*
5267  * Start the board, ready for normal operation
5268  */
5269 int
5270 ahc_init(struct ahc_softc *ahc)
5271 {
5272 	int	 max_targ;
5273 	u_int	 i;
5274 	u_int	 scsi_conf;
5275 	u_int	 ultraenb;
5276 	u_int	 discenable;
5277 	u_int	 tagenable;
5278 	size_t	 driver_data_size;
5279 
5280 #ifdef AHC_DEBUG
5281 	if ((ahc_debug & AHC_DEBUG_SEQUENCER) != 0)
5282 		ahc->flags |= AHC_SEQUENCER_DEBUG;
5283 #endif
5284 
5285 #ifdef AHC_PRINT_SRAM
5286 	printk("Scratch Ram:");
5287 	for (i = 0x20; i < 0x5f; i++) {
5288 		if (((i % 8) == 0) && (i != 0)) {
5289 			printk ("\n              ");
5290 		}
5291 		printk (" 0x%x", ahc_inb(ahc, i));
5292 	}
5293 	if ((ahc->features & AHC_MORE_SRAM) != 0) {
5294 		for (i = 0x70; i < 0x7f; i++) {
5295 			if (((i % 8) == 0) && (i != 0)) {
5296 				printk ("\n              ");
5297 			}
5298 			printk (" 0x%x", ahc_inb(ahc, i));
5299 		}
5300 	}
5301 	printk ("\n");
5302 	/*
5303 	 * Reading uninitialized scratch ram may
5304 	 * generate parity errors.
5305 	 */
5306 	ahc_outb(ahc, CLRINT, CLRPARERR);
5307 	ahc_outb(ahc, CLRINT, CLRBRKADRINT);
5308 #endif
5309 	max_targ = 15;
5310 
5311 	/*
5312 	 * Assume we have a board at this stage and it has been reset.
5313 	 */
5314 	if ((ahc->flags & AHC_USEDEFAULTS) != 0)
5315 		ahc->our_id = ahc->our_id_b = 7;
5316 
5317 	/*
5318 	 * Default to allowing initiator operations.
5319 	 */
5320 	ahc->flags |= AHC_INITIATORROLE;
5321 
5322 	/*
5323 	 * Only allow target mode features if this unit has them enabled.
5324 	 */
5325 	if ((AHC_TMODE_ENABLE & (0x1 << ahc->unit)) == 0)
5326 		ahc->features &= ~AHC_TARGETMODE;
5327 
5328 #ifndef __linux__
5329 	/* DMA tag for mapping buffers into device visible space. */
5330 	if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
5331 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5332 			       /*lowaddr*/ahc->flags & AHC_39BIT_ADDRESSING
5333 					? (dma_addr_t)0x7FFFFFFFFFULL
5334 					: BUS_SPACE_MAXADDR_32BIT,
5335 			       /*highaddr*/BUS_SPACE_MAXADDR,
5336 			       /*filter*/NULL, /*filterarg*/NULL,
5337 			       /*maxsize*/(AHC_NSEG - 1) * PAGE_SIZE,
5338 			       /*nsegments*/AHC_NSEG,
5339 			       /*maxsegsz*/AHC_MAXTRANSFER_SIZE,
5340 			       /*flags*/BUS_DMA_ALLOCNOW,
5341 			       &ahc->buffer_dmat) != 0) {
5342 		return (ENOMEM);
5343 	}
5344 #endif
5345 
5346 	ahc->init_level++;
5347 
5348 	/*
5349 	 * DMA tag for our command fifos and other data in system memory
5350 	 * the card's sequencer must be able to access.  For initiator
5351 	 * roles, we need to allocate space for the qinfifo and qoutfifo.
5352 	 * The qinfifo and qoutfifo are composed of 256 1 byte elements.
5353 	 * When providing for the target mode role, we must additionally
5354 	 * provide space for the incoming target command fifo and an extra
5355 	 * byte to deal with a dma bug in some chip versions.
5356 	 */
5357 	driver_data_size = 2 * 256 * sizeof(uint8_t);
5358 	if ((ahc->features & AHC_TARGETMODE) != 0)
5359 		driver_data_size += AHC_TMODE_CMDS * sizeof(struct target_cmd)
5360 				 + /*DMA WideOdd Bug Buffer*/1;
5361 	if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
5362 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5363 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5364 			       /*highaddr*/BUS_SPACE_MAXADDR,
5365 			       /*filter*/NULL, /*filterarg*/NULL,
5366 			       driver_data_size,
5367 			       /*nsegments*/1,
5368 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5369 			       /*flags*/0, &ahc->shared_data_dmat) != 0) {
5370 		return (ENOMEM);
5371 	}
5372 
5373 	ahc->init_level++;
5374 
5375 	/* Allocation of driver data */
5376 	if (ahc_dmamem_alloc(ahc, ahc->shared_data_dmat,
5377 			     (void **)&ahc->qoutfifo,
5378 			     BUS_DMA_NOWAIT, &ahc->shared_data_dmamap) != 0) {
5379 		return (ENOMEM);
5380 	}
5381 
5382 	ahc->init_level++;
5383 
5384 	/* And permanently map it in */
5385 	ahc_dmamap_load(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
5386 			ahc->qoutfifo, driver_data_size, ahc_dmamap_cb,
5387 			&ahc->shared_data_busaddr, /*flags*/0);
5388 
5389 	if ((ahc->features & AHC_TARGETMODE) != 0) {
5390 		ahc->targetcmds = (struct target_cmd *)ahc->qoutfifo;
5391 		ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[AHC_TMODE_CMDS];
5392 		ahc->dma_bug_buf = ahc->shared_data_busaddr
5393 				 + driver_data_size - 1;
5394 		/* All target command blocks start out invalid. */
5395 		for (i = 0; i < AHC_TMODE_CMDS; i++)
5396 			ahc->targetcmds[i].cmd_valid = 0;
5397 		ahc_sync_tqinfifo(ahc, BUS_DMASYNC_PREREAD);
5398 		ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[256];
5399 	}
5400 	ahc->qinfifo = &ahc->qoutfifo[256];
5401 
5402 	ahc->init_level++;
5403 
5404 	/* Allocate SCB data now that buffer_dmat is initialized */
5405 	if (ahc->scb_data->maxhscbs == 0)
5406 		if (ahc_init_scbdata(ahc) != 0)
5407 			return (ENOMEM);
5408 
5409 	/*
5410 	 * Allocate a tstate to house information for our
5411 	 * initiator presence on the bus as well as the user
5412 	 * data for any target mode initiator.
5413 	 */
5414 	if (ahc_alloc_tstate(ahc, ahc->our_id, 'A') == NULL) {
5415 		printk("%s: unable to allocate ahc_tmode_tstate.  "
5416 		       "Failing attach\n", ahc_name(ahc));
5417 		return (ENOMEM);
5418 	}
5419 
5420 	if ((ahc->features & AHC_TWIN) != 0) {
5421 		if (ahc_alloc_tstate(ahc, ahc->our_id_b, 'B') == NULL) {
5422 			printk("%s: unable to allocate ahc_tmode_tstate.  "
5423 			       "Failing attach\n", ahc_name(ahc));
5424 			return (ENOMEM);
5425 		}
5426 	}
5427 
5428 	if (ahc->scb_data->maxhscbs < AHC_SCB_MAX_ALLOC) {
5429 		ahc->flags |= AHC_PAGESCBS;
5430 	} else {
5431 		ahc->flags &= ~AHC_PAGESCBS;
5432 	}
5433 
5434 #ifdef AHC_DEBUG
5435 	if (ahc_debug & AHC_SHOW_MISC) {
5436 		printk("%s: hardware scb %u bytes; kernel scb %u bytes; "
5437 		       "ahc_dma %u bytes\n",
5438 			ahc_name(ahc),
5439 			(u_int)sizeof(struct hardware_scb),
5440 			(u_int)sizeof(struct scb),
5441 			(u_int)sizeof(struct ahc_dma_seg));
5442 	}
5443 #endif /* AHC_DEBUG */
5444 
5445 	/*
5446 	 * Look at the information that board initialization or
5447 	 * the board bios has left us.
5448 	 */
5449 	if (ahc->features & AHC_TWIN) {
5450 		scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5451 		if ((scsi_conf & RESET_SCSI) != 0
5452 		 && (ahc->flags & AHC_INITIATORROLE) != 0)
5453 			ahc->flags |= AHC_RESET_BUS_B;
5454 	}
5455 
5456 	scsi_conf = ahc_inb(ahc, SCSICONF);
5457 	if ((scsi_conf & RESET_SCSI) != 0
5458 	 && (ahc->flags & AHC_INITIATORROLE) != 0)
5459 		ahc->flags |= AHC_RESET_BUS_A;
5460 
5461 	ultraenb = 0;
5462 	tagenable = ALL_TARGETS_MASK;
5463 
5464 	/* Grab the disconnection disable table and invert it for our needs */
5465 	if ((ahc->flags & AHC_USEDEFAULTS) != 0) {
5466 		printk("%s: Host Adapter Bios disabled.  Using default SCSI "
5467 			"device parameters\n", ahc_name(ahc));
5468 		ahc->flags |= AHC_EXTENDED_TRANS_A|AHC_EXTENDED_TRANS_B|
5469 			      AHC_TERM_ENB_A|AHC_TERM_ENB_B;
5470 		discenable = ALL_TARGETS_MASK;
5471 		if ((ahc->features & AHC_ULTRA) != 0)
5472 			ultraenb = ALL_TARGETS_MASK;
5473 	} else {
5474 		discenable = ~((ahc_inb(ahc, DISC_DSB + 1) << 8)
5475 			   | ahc_inb(ahc, DISC_DSB));
5476 		if ((ahc->features & (AHC_ULTRA|AHC_ULTRA2)) != 0)
5477 			ultraenb = (ahc_inb(ahc, ULTRA_ENB + 1) << 8)
5478 				      | ahc_inb(ahc, ULTRA_ENB);
5479 	}
5480 
5481 	if ((ahc->features & (AHC_WIDE|AHC_TWIN)) == 0)
5482 		max_targ = 7;
5483 
5484 	for (i = 0; i <= max_targ; i++) {
5485 		struct ahc_initiator_tinfo *tinfo;
5486 		struct ahc_tmode_tstate *tstate;
5487 		u_int our_id;
5488 		u_int target_id;
5489 		char channel;
5490 
5491 		channel = 'A';
5492 		our_id = ahc->our_id;
5493 		target_id = i;
5494 		if (i > 7 && (ahc->features & AHC_TWIN) != 0) {
5495 			channel = 'B';
5496 			our_id = ahc->our_id_b;
5497 			target_id = i % 8;
5498 		}
5499 		tinfo = ahc_fetch_transinfo(ahc, channel, our_id,
5500 					    target_id, &tstate);
5501 		/* Default to async narrow across the board */
5502 		memset(tinfo, 0, sizeof(*tinfo));
5503 		if (ahc->flags & AHC_USEDEFAULTS) {
5504 			if ((ahc->features & AHC_WIDE) != 0)
5505 				tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5506 
5507 			/*
5508 			 * These will be truncated when we determine the
5509 			 * connection type we have with the target.
5510 			 */
5511 			tinfo->user.period = ahc_syncrates->period;
5512 			tinfo->user.offset = MAX_OFFSET;
5513 		} else {
5514 			u_int scsirate;
5515 			uint16_t mask;
5516 
5517 			/* Take the settings leftover in scratch RAM. */
5518 			scsirate = ahc_inb(ahc, TARG_SCSIRATE + i);
5519 			mask = (0x01 << i);
5520 			if ((ahc->features & AHC_ULTRA2) != 0) {
5521 				u_int offset;
5522 				u_int maxsync;
5523 
5524 				if ((scsirate & SOFS) == 0x0F) {
5525 					/*
5526 					 * Haven't negotiated yet,
5527 					 * so the format is different.
5528 					 */
5529 					scsirate = (scsirate & SXFR) >> 4
5530 						 | (ultraenb & mask)
5531 						  ? 0x08 : 0x0
5532 						 | (scsirate & WIDEXFER);
5533 					offset = MAX_OFFSET_ULTRA2;
5534 				} else
5535 					offset = ahc_inb(ahc, TARG_OFFSET + i);
5536 				if ((scsirate & ~WIDEXFER) == 0 && offset != 0)
5537 					/* Set to the lowest sync rate, 5MHz */
5538 					scsirate |= 0x1c;
5539 				maxsync = AHC_SYNCRATE_ULTRA2;
5540 				if ((ahc->features & AHC_DT) != 0)
5541 					maxsync = AHC_SYNCRATE_DT;
5542 				tinfo->user.period =
5543 				    ahc_find_period(ahc, scsirate, maxsync);
5544 				if (offset == 0)
5545 					tinfo->user.period = 0;
5546 				else
5547 					tinfo->user.offset = MAX_OFFSET;
5548 				if ((scsirate & SXFR_ULTRA2) <= 8/*10MHz*/
5549 				 && (ahc->features & AHC_DT) != 0)
5550 					tinfo->user.ppr_options =
5551 					    MSG_EXT_PPR_DT_REQ;
5552 			} else if ((scsirate & SOFS) != 0) {
5553 				if ((scsirate & SXFR) == 0x40
5554 				 && (ultraenb & mask) != 0) {
5555 					/* Treat 10MHz as a non-ultra speed */
5556 					scsirate &= ~SXFR;
5557 				 	ultraenb &= ~mask;
5558 				}
5559 				tinfo->user.period =
5560 				    ahc_find_period(ahc, scsirate,
5561 						    (ultraenb & mask)
5562 						   ? AHC_SYNCRATE_ULTRA
5563 						   : AHC_SYNCRATE_FAST);
5564 				if (tinfo->user.period != 0)
5565 					tinfo->user.offset = MAX_OFFSET;
5566 			}
5567 			if (tinfo->user.period == 0)
5568 				tinfo->user.offset = 0;
5569 			if ((scsirate & WIDEXFER) != 0
5570 			 && (ahc->features & AHC_WIDE) != 0)
5571 				tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5572 			tinfo->user.protocol_version = 4;
5573 			if ((ahc->features & AHC_DT) != 0)
5574 				tinfo->user.transport_version = 3;
5575 			else
5576 				tinfo->user.transport_version = 2;
5577 			tinfo->goal.protocol_version = 2;
5578 			tinfo->goal.transport_version = 2;
5579 			tinfo->curr.protocol_version = 2;
5580 			tinfo->curr.transport_version = 2;
5581 		}
5582 		tstate->ultraenb = 0;
5583 	}
5584 	ahc->user_discenable = discenable;
5585 	ahc->user_tagenable = tagenable;
5586 
5587 	return (ahc->bus_chip_init(ahc));
5588 }
5589 
5590 void
5591 ahc_intr_enable(struct ahc_softc *ahc, int enable)
5592 {
5593 	u_int hcntrl;
5594 
5595 	hcntrl = ahc_inb(ahc, HCNTRL);
5596 	hcntrl &= ~INTEN;
5597 	ahc->pause &= ~INTEN;
5598 	ahc->unpause &= ~INTEN;
5599 	if (enable) {
5600 		hcntrl |= INTEN;
5601 		ahc->pause |= INTEN;
5602 		ahc->unpause |= INTEN;
5603 	}
5604 	ahc_outb(ahc, HCNTRL, hcntrl);
5605 }
5606 
5607 /*
5608  * Ensure that the card is paused in a location
5609  * outside of all critical sections and that all
5610  * pending work is completed prior to returning.
5611  * This routine should only be called from outside
5612  * an interrupt context.
5613  */
5614 void
5615 ahc_pause_and_flushwork(struct ahc_softc *ahc)
5616 {
5617 	int intstat;
5618 	int maxloops;
5619 	int paused;
5620 
5621 	maxloops = 1000;
5622 	ahc->flags |= AHC_ALL_INTERRUPTS;
5623 	paused = FALSE;
5624 	do {
5625 		if (paused) {
5626 			ahc_unpause(ahc);
5627 			/*
5628 			 * Give the sequencer some time to service
5629 			 * any active selections.
5630 			 */
5631 			ahc_delay(500);
5632 		}
5633 		ahc_intr(ahc);
5634 		ahc_pause(ahc);
5635 		paused = TRUE;
5636 		ahc_outb(ahc, SCSISEQ, ahc_inb(ahc, SCSISEQ) & ~ENSELO);
5637 		intstat = ahc_inb(ahc, INTSTAT);
5638 		if ((intstat & INT_PEND) == 0) {
5639 			ahc_clear_critical_section(ahc);
5640 			intstat = ahc_inb(ahc, INTSTAT);
5641 		}
5642 	} while (--maxloops
5643 	      && (intstat != 0xFF || (ahc->features & AHC_REMOVABLE) == 0)
5644 	      && ((intstat & INT_PEND) != 0
5645 	       || (ahc_inb(ahc, SSTAT0) & (SELDO|SELINGO)) != 0));
5646 	if (maxloops == 0) {
5647 		printk("Infinite interrupt loop, INTSTAT = %x",
5648 		       ahc_inb(ahc, INTSTAT));
5649 	}
5650 	ahc_platform_flushwork(ahc);
5651 	ahc->flags &= ~AHC_ALL_INTERRUPTS;
5652 }
5653 
5654 #ifdef CONFIG_PM
5655 int
5656 ahc_suspend(struct ahc_softc *ahc)
5657 {
5658 
5659 	ahc_pause_and_flushwork(ahc);
5660 
5661 	if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
5662 		ahc_unpause(ahc);
5663 		return (EBUSY);
5664 	}
5665 
5666 #ifdef AHC_TARGET_MODE
5667 	/*
5668 	 * XXX What about ATIOs that have not yet been serviced?
5669 	 * Perhaps we should just refuse to be suspended if we
5670 	 * are acting in a target role.
5671 	 */
5672 	if (ahc->pending_device != NULL) {
5673 		ahc_unpause(ahc);
5674 		return (EBUSY);
5675 	}
5676 #endif
5677 	ahc_shutdown(ahc);
5678 	return (0);
5679 }
5680 
5681 int
5682 ahc_resume(struct ahc_softc *ahc)
5683 {
5684 
5685 	ahc_reset(ahc, /*reinit*/TRUE);
5686 	ahc_intr_enable(ahc, TRUE);
5687 	ahc_restart(ahc);
5688 	return (0);
5689 }
5690 #endif
5691 /************************** Busy Target Table *********************************/
5692 /*
5693  * Return the untagged transaction id for a given target/channel lun.
5694  * Optionally, clear the entry.
5695  */
5696 static u_int
5697 ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl)
5698 {
5699 	u_int scbid;
5700 	u_int target_offset;
5701 
5702 	if ((ahc->flags & AHC_SCB_BTT) != 0) {
5703 		u_int saved_scbptr;
5704 
5705 		saved_scbptr = ahc_inb(ahc, SCBPTR);
5706 		ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5707 		scbid = ahc_inb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl));
5708 		ahc_outb(ahc, SCBPTR, saved_scbptr);
5709 	} else {
5710 		target_offset = TCL_TARGET_OFFSET(tcl);
5711 		scbid = ahc_inb(ahc, BUSY_TARGETS + target_offset);
5712 	}
5713 
5714 	return (scbid);
5715 }
5716 
5717 static void
5718 ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl)
5719 {
5720 	u_int target_offset;
5721 
5722 	if ((ahc->flags & AHC_SCB_BTT) != 0) {
5723 		u_int saved_scbptr;
5724 
5725 		saved_scbptr = ahc_inb(ahc, SCBPTR);
5726 		ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5727 		ahc_outb(ahc, SCB_64_BTT+TCL_TARGET_OFFSET(tcl), SCB_LIST_NULL);
5728 		ahc_outb(ahc, SCBPTR, saved_scbptr);
5729 	} else {
5730 		target_offset = TCL_TARGET_OFFSET(tcl);
5731 		ahc_outb(ahc, BUSY_TARGETS + target_offset, SCB_LIST_NULL);
5732 	}
5733 }
5734 
5735 static void
5736 ahc_busy_tcl(struct ahc_softc *ahc, u_int tcl, u_int scbid)
5737 {
5738 	u_int target_offset;
5739 
5740 	if ((ahc->flags & AHC_SCB_BTT) != 0) {
5741 		u_int saved_scbptr;
5742 
5743 		saved_scbptr = ahc_inb(ahc, SCBPTR);
5744 		ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5745 		ahc_outb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl), scbid);
5746 		ahc_outb(ahc, SCBPTR, saved_scbptr);
5747 	} else {
5748 		target_offset = TCL_TARGET_OFFSET(tcl);
5749 		ahc_outb(ahc, BUSY_TARGETS + target_offset, scbid);
5750 	}
5751 }
5752 
5753 /************************** SCB and SCB queue management **********************/
5754 int
5755 ahc_match_scb(struct ahc_softc *ahc, struct scb *scb, int target,
5756 	      char channel, int lun, u_int tag, role_t role)
5757 {
5758 	int targ = SCB_GET_TARGET(ahc, scb);
5759 	char chan = SCB_GET_CHANNEL(ahc, scb);
5760 	int slun = SCB_GET_LUN(scb);
5761 	int match;
5762 
5763 	match = ((chan == channel) || (channel == ALL_CHANNELS));
5764 	if (match != 0)
5765 		match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
5766 	if (match != 0)
5767 		match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
5768 	if (match != 0) {
5769 #ifdef AHC_TARGET_MODE
5770 		int group;
5771 
5772 		group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
5773 		if (role == ROLE_INITIATOR) {
5774 			match = (group != XPT_FC_GROUP_TMODE)
5775 			      && ((tag == scb->hscb->tag)
5776 			       || (tag == SCB_LIST_NULL));
5777 		} else if (role == ROLE_TARGET) {
5778 			match = (group == XPT_FC_GROUP_TMODE)
5779 			      && ((tag == scb->io_ctx->csio.tag_id)
5780 			       || (tag == SCB_LIST_NULL));
5781 		}
5782 #else /* !AHC_TARGET_MODE */
5783 		match = ((tag == scb->hscb->tag) || (tag == SCB_LIST_NULL));
5784 #endif /* AHC_TARGET_MODE */
5785 	}
5786 
5787 	return match;
5788 }
5789 
5790 static void
5791 ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb)
5792 {
5793 	int	target;
5794 	char	channel;
5795 	int	lun;
5796 
5797 	target = SCB_GET_TARGET(ahc, scb);
5798 	lun = SCB_GET_LUN(scb);
5799 	channel = SCB_GET_CHANNEL(ahc, scb);
5800 
5801 	ahc_search_qinfifo(ahc, target, channel, lun,
5802 			   /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
5803 			   CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5804 
5805 	ahc_platform_freeze_devq(ahc, scb);
5806 }
5807 
5808 void
5809 ahc_qinfifo_requeue_tail(struct ahc_softc *ahc, struct scb *scb)
5810 {
5811 	struct scb *prev_scb;
5812 
5813 	prev_scb = NULL;
5814 	if (ahc_qinfifo_count(ahc) != 0) {
5815 		u_int prev_tag;
5816 		uint8_t prev_pos;
5817 
5818 		prev_pos = ahc->qinfifonext - 1;
5819 		prev_tag = ahc->qinfifo[prev_pos];
5820 		prev_scb = ahc_lookup_scb(ahc, prev_tag);
5821 	}
5822 	ahc_qinfifo_requeue(ahc, prev_scb, scb);
5823 	if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5824 		ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5825 	} else {
5826 		ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5827 	}
5828 }
5829 
5830 static void
5831 ahc_qinfifo_requeue(struct ahc_softc *ahc, struct scb *prev_scb,
5832 		    struct scb *scb)
5833 {
5834 	if (prev_scb == NULL) {
5835 		ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5836 	} else {
5837 		prev_scb->hscb->next = scb->hscb->tag;
5838 		ahc_sync_scb(ahc, prev_scb,
5839 			     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5840 	}
5841 	ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
5842 	scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5843 	ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5844 }
5845 
5846 static int
5847 ahc_qinfifo_count(struct ahc_softc *ahc)
5848 {
5849 	uint8_t qinpos;
5850 	uint8_t diff;
5851 
5852 	if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5853 		qinpos = ahc_inb(ahc, SNSCB_QOFF);
5854 		ahc_outb(ahc, SNSCB_QOFF, qinpos);
5855 	} else
5856 		qinpos = ahc_inb(ahc, QINPOS);
5857 	diff = ahc->qinfifonext - qinpos;
5858 	return (diff);
5859 }
5860 
5861 int
5862 ahc_search_qinfifo(struct ahc_softc *ahc, int target, char channel,
5863 		   int lun, u_int tag, role_t role, uint32_t status,
5864 		   ahc_search_action action)
5865 {
5866 	struct	scb *scb;
5867 	struct	scb *prev_scb;
5868 	uint8_t qinstart;
5869 	uint8_t qinpos;
5870 	uint8_t qintail;
5871 	uint8_t next;
5872 	uint8_t prev;
5873 	uint8_t curscbptr;
5874 	int	found;
5875 	int	have_qregs;
5876 
5877 	qintail = ahc->qinfifonext;
5878 	have_qregs = (ahc->features & AHC_QUEUE_REGS) != 0;
5879 	if (have_qregs) {
5880 		qinstart = ahc_inb(ahc, SNSCB_QOFF);
5881 		ahc_outb(ahc, SNSCB_QOFF, qinstart);
5882 	} else
5883 		qinstart = ahc_inb(ahc, QINPOS);
5884 	qinpos = qinstart;
5885 	found = 0;
5886 	prev_scb = NULL;
5887 
5888 	if (action == SEARCH_COMPLETE) {
5889 		/*
5890 		 * Don't attempt to run any queued untagged transactions
5891 		 * until we are done with the abort process.
5892 		 */
5893 		ahc_freeze_untagged_queues(ahc);
5894 	}
5895 
5896 	/*
5897 	 * Start with an empty queue.  Entries that are not chosen
5898 	 * for removal will be re-added to the queue as we go.
5899 	 */
5900 	ahc->qinfifonext = qinpos;
5901 	ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5902 
5903 	while (qinpos != qintail) {
5904 		scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinpos]);
5905 		if (scb == NULL) {
5906 			printk("qinpos = %d, SCB index = %d\n",
5907 				qinpos, ahc->qinfifo[qinpos]);
5908 			panic("Loop 1\n");
5909 		}
5910 
5911 		if (ahc_match_scb(ahc, scb, target, channel, lun, tag, role)) {
5912 			/*
5913 			 * We found an scb that needs to be acted on.
5914 			 */
5915 			found++;
5916 			switch (action) {
5917 			case SEARCH_COMPLETE:
5918 			{
5919 				cam_status ostat;
5920 				cam_status cstat;
5921 
5922 				ostat = ahc_get_transaction_status(scb);
5923 				if (ostat == CAM_REQ_INPROG)
5924 					ahc_set_transaction_status(scb, status);
5925 				cstat = ahc_get_transaction_status(scb);
5926 				if (cstat != CAM_REQ_CMP)
5927 					ahc_freeze_scb(scb);
5928 				if ((scb->flags & SCB_ACTIVE) == 0)
5929 					printk("Inactive SCB in qinfifo\n");
5930 				ahc_done(ahc, scb);
5931 
5932 				/* FALLTHROUGH */
5933 			}
5934 			case SEARCH_REMOVE:
5935 				break;
5936 			case SEARCH_COUNT:
5937 				ahc_qinfifo_requeue(ahc, prev_scb, scb);
5938 				prev_scb = scb;
5939 				break;
5940 			}
5941 		} else {
5942 			ahc_qinfifo_requeue(ahc, prev_scb, scb);
5943 			prev_scb = scb;
5944 		}
5945 		qinpos++;
5946 	}
5947 
5948 	if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5949 		ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5950 	} else {
5951 		ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5952 	}
5953 
5954 	if (action != SEARCH_COUNT
5955 	 && (found != 0)
5956 	 && (qinstart != ahc->qinfifonext)) {
5957 		/*
5958 		 * The sequencer may be in the process of dmaing
5959 		 * down the SCB at the beginning of the queue.
5960 		 * This could be problematic if either the first,
5961 		 * or the second SCB is removed from the queue
5962 		 * (the first SCB includes a pointer to the "next"
5963 		 * SCB to dma). If we have removed any entries, swap
5964 		 * the first element in the queue with the next HSCB
5965 		 * so the sequencer will notice that NEXT_QUEUED_SCB
5966 		 * has changed during its dma attempt and will retry
5967 		 * the DMA.
5968 		 */
5969 		scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinstart]);
5970 
5971 		if (scb == NULL) {
5972 			printk("found = %d, qinstart = %d, qinfifionext = %d\n",
5973 				found, qinstart, ahc->qinfifonext);
5974 			panic("First/Second Qinfifo fixup\n");
5975 		}
5976 		/*
5977 		 * ahc_swap_with_next_hscb forces our next pointer to
5978 		 * point to the reserved SCB for future commands.  Save
5979 		 * and restore our original next pointer to maintain
5980 		 * queue integrity.
5981 		 */
5982 		next = scb->hscb->next;
5983 		ahc->scb_data->scbindex[scb->hscb->tag] = NULL;
5984 		ahc_swap_with_next_hscb(ahc, scb);
5985 		scb->hscb->next = next;
5986 		ahc->qinfifo[qinstart] = scb->hscb->tag;
5987 
5988 		/* Tell the card about the new head of the qinfifo. */
5989 		ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5990 
5991 		/* Fixup the tail "next" pointer. */
5992 		qintail = ahc->qinfifonext - 1;
5993 		scb = ahc_lookup_scb(ahc, ahc->qinfifo[qintail]);
5994 		scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5995 	}
5996 
5997 	/*
5998 	 * Search waiting for selection list.
5999 	 */
6000 	curscbptr = ahc_inb(ahc, SCBPTR);
6001 	next = ahc_inb(ahc, WAITING_SCBH);  /* Start at head of list. */
6002 	prev = SCB_LIST_NULL;
6003 
6004 	while (next != SCB_LIST_NULL) {
6005 		uint8_t scb_index;
6006 
6007 		ahc_outb(ahc, SCBPTR, next);
6008 		scb_index = ahc_inb(ahc, SCB_TAG);
6009 		if (scb_index >= ahc->scb_data->numscbs) {
6010 			printk("Waiting List inconsistency. "
6011 			       "SCB index == %d, yet numscbs == %d.",
6012 			       scb_index, ahc->scb_data->numscbs);
6013 			ahc_dump_card_state(ahc);
6014 			panic("for safety");
6015 		}
6016 		scb = ahc_lookup_scb(ahc, scb_index);
6017 		if (scb == NULL) {
6018 			printk("scb_index = %d, next = %d\n",
6019 				scb_index, next);
6020 			panic("Waiting List traversal\n");
6021 		}
6022 		if (ahc_match_scb(ahc, scb, target, channel,
6023 				  lun, SCB_LIST_NULL, role)) {
6024 			/*
6025 			 * We found an scb that needs to be acted on.
6026 			 */
6027 			found++;
6028 			switch (action) {
6029 			case SEARCH_COMPLETE:
6030 			{
6031 				cam_status ostat;
6032 				cam_status cstat;
6033 
6034 				ostat = ahc_get_transaction_status(scb);
6035 				if (ostat == CAM_REQ_INPROG)
6036 					ahc_set_transaction_status(scb,
6037 								   status);
6038 				cstat = ahc_get_transaction_status(scb);
6039 				if (cstat != CAM_REQ_CMP)
6040 					ahc_freeze_scb(scb);
6041 				if ((scb->flags & SCB_ACTIVE) == 0)
6042 					printk("Inactive SCB in Waiting List\n");
6043 				ahc_done(ahc, scb);
6044 				/* FALLTHROUGH */
6045 			}
6046 			case SEARCH_REMOVE:
6047 				next = ahc_rem_wscb(ahc, next, prev);
6048 				break;
6049 			case SEARCH_COUNT:
6050 				prev = next;
6051 				next = ahc_inb(ahc, SCB_NEXT);
6052 				break;
6053 			}
6054 		} else {
6055 
6056 			prev = next;
6057 			next = ahc_inb(ahc, SCB_NEXT);
6058 		}
6059 	}
6060 	ahc_outb(ahc, SCBPTR, curscbptr);
6061 
6062 	found += ahc_search_untagged_queues(ahc, /*ahc_io_ctx_t*/NULL, target,
6063 					    channel, lun, status, action);
6064 
6065 	if (action == SEARCH_COMPLETE)
6066 		ahc_release_untagged_queues(ahc);
6067 	return (found);
6068 }
6069 
6070 int
6071 ahc_search_untagged_queues(struct ahc_softc *ahc, ahc_io_ctx_t ctx,
6072 			   int target, char channel, int lun, uint32_t status,
6073 			   ahc_search_action action)
6074 {
6075 	struct	scb *scb;
6076 	int	maxtarget;
6077 	int	found;
6078 	int	i;
6079 
6080 	if (action == SEARCH_COMPLETE) {
6081 		/*
6082 		 * Don't attempt to run any queued untagged transactions
6083 		 * until we are done with the abort process.
6084 		 */
6085 		ahc_freeze_untagged_queues(ahc);
6086 	}
6087 
6088 	found = 0;
6089 	i = 0;
6090 	if ((ahc->flags & AHC_SCB_BTT) == 0) {
6091 
6092 		maxtarget = 16;
6093 		if (target != CAM_TARGET_WILDCARD) {
6094 
6095 			i = target;
6096 			if (channel == 'B')
6097 				i += 8;
6098 			maxtarget = i + 1;
6099 		}
6100 	} else {
6101 		maxtarget = 0;
6102 	}
6103 
6104 	for (; i < maxtarget; i++) {
6105 		struct scb_tailq *untagged_q;
6106 		struct scb *next_scb;
6107 
6108 		untagged_q = &(ahc->untagged_queues[i]);
6109 		next_scb = TAILQ_FIRST(untagged_q);
6110 		while (next_scb != NULL) {
6111 
6112 			scb = next_scb;
6113 			next_scb = TAILQ_NEXT(scb, links.tqe);
6114 
6115 			/*
6116 			 * The head of the list may be the currently
6117 			 * active untagged command for a device.
6118 			 * We're only searching for commands that
6119 			 * have not been started.  A transaction
6120 			 * marked active but still in the qinfifo
6121 			 * is removed by the qinfifo scanning code
6122 			 * above.
6123 			 */
6124 			if ((scb->flags & SCB_ACTIVE) != 0)
6125 				continue;
6126 
6127 			if (ahc_match_scb(ahc, scb, target, channel, lun,
6128 					  SCB_LIST_NULL, ROLE_INITIATOR) == 0
6129 			 || (ctx != NULL && ctx != scb->io_ctx))
6130 				continue;
6131 
6132 			/*
6133 			 * We found an scb that needs to be acted on.
6134 			 */
6135 			found++;
6136 			switch (action) {
6137 			case SEARCH_COMPLETE:
6138 			{
6139 				cam_status ostat;
6140 				cam_status cstat;
6141 
6142 				ostat = ahc_get_transaction_status(scb);
6143 				if (ostat == CAM_REQ_INPROG)
6144 					ahc_set_transaction_status(scb, status);
6145 				cstat = ahc_get_transaction_status(scb);
6146 				if (cstat != CAM_REQ_CMP)
6147 					ahc_freeze_scb(scb);
6148 				if ((scb->flags & SCB_ACTIVE) == 0)
6149 					printk("Inactive SCB in untaggedQ\n");
6150 				ahc_done(ahc, scb);
6151 				break;
6152 			}
6153 			case SEARCH_REMOVE:
6154 				scb->flags &= ~SCB_UNTAGGEDQ;
6155 				TAILQ_REMOVE(untagged_q, scb, links.tqe);
6156 				break;
6157 			case SEARCH_COUNT:
6158 				break;
6159 			}
6160 		}
6161 	}
6162 
6163 	if (action == SEARCH_COMPLETE)
6164 		ahc_release_untagged_queues(ahc);
6165 	return (found);
6166 }
6167 
6168 int
6169 ahc_search_disc_list(struct ahc_softc *ahc, int target, char channel,
6170 		     int lun, u_int tag, int stop_on_first, int remove,
6171 		     int save_state)
6172 {
6173 	struct	scb *scbp;
6174 	u_int	next;
6175 	u_int	prev;
6176 	u_int	count;
6177 	u_int	active_scb;
6178 
6179 	count = 0;
6180 	next = ahc_inb(ahc, DISCONNECTED_SCBH);
6181 	prev = SCB_LIST_NULL;
6182 
6183 	if (save_state) {
6184 		/* restore this when we're done */
6185 		active_scb = ahc_inb(ahc, SCBPTR);
6186 	} else
6187 		/* Silence compiler */
6188 		active_scb = SCB_LIST_NULL;
6189 
6190 	while (next != SCB_LIST_NULL) {
6191 		u_int scb_index;
6192 
6193 		ahc_outb(ahc, SCBPTR, next);
6194 		scb_index = ahc_inb(ahc, SCB_TAG);
6195 		if (scb_index >= ahc->scb_data->numscbs) {
6196 			printk("Disconnected List inconsistency. "
6197 			       "SCB index == %d, yet numscbs == %d.",
6198 			       scb_index, ahc->scb_data->numscbs);
6199 			ahc_dump_card_state(ahc);
6200 			panic("for safety");
6201 		}
6202 
6203 		if (next == prev) {
6204 			panic("Disconnected List Loop. "
6205 			      "cur SCBPTR == %x, prev SCBPTR == %x.",
6206 			      next, prev);
6207 		}
6208 		scbp = ahc_lookup_scb(ahc, scb_index);
6209 		if (ahc_match_scb(ahc, scbp, target, channel, lun,
6210 				  tag, ROLE_INITIATOR)) {
6211 			count++;
6212 			if (remove) {
6213 				next =
6214 				    ahc_rem_scb_from_disc_list(ahc, prev, next);
6215 			} else {
6216 				prev = next;
6217 				next = ahc_inb(ahc, SCB_NEXT);
6218 			}
6219 			if (stop_on_first)
6220 				break;
6221 		} else {
6222 			prev = next;
6223 			next = ahc_inb(ahc, SCB_NEXT);
6224 		}
6225 	}
6226 	if (save_state)
6227 		ahc_outb(ahc, SCBPTR, active_scb);
6228 	return (count);
6229 }
6230 
6231 /*
6232  * Remove an SCB from the on chip list of disconnected transactions.
6233  * This is empty/unused if we are not performing SCB paging.
6234  */
6235 static u_int
6236 ahc_rem_scb_from_disc_list(struct ahc_softc *ahc, u_int prev, u_int scbptr)
6237 {
6238 	u_int next;
6239 
6240 	ahc_outb(ahc, SCBPTR, scbptr);
6241 	next = ahc_inb(ahc, SCB_NEXT);
6242 
6243 	ahc_outb(ahc, SCB_CONTROL, 0);
6244 
6245 	ahc_add_curscb_to_free_list(ahc);
6246 
6247 	if (prev != SCB_LIST_NULL) {
6248 		ahc_outb(ahc, SCBPTR, prev);
6249 		ahc_outb(ahc, SCB_NEXT, next);
6250 	} else
6251 		ahc_outb(ahc, DISCONNECTED_SCBH, next);
6252 
6253 	return (next);
6254 }
6255 
6256 /*
6257  * Add the SCB as selected by SCBPTR onto the on chip list of
6258  * free hardware SCBs.  This list is empty/unused if we are not
6259  * performing SCB paging.
6260  */
6261 static void
6262 ahc_add_curscb_to_free_list(struct ahc_softc *ahc)
6263 {
6264 	/*
6265 	 * Invalidate the tag so that our abort
6266 	 * routines don't think it's active.
6267 	 */
6268 	ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
6269 
6270 	if ((ahc->flags & AHC_PAGESCBS) != 0) {
6271 		ahc_outb(ahc, SCB_NEXT, ahc_inb(ahc, FREE_SCBH));
6272 		ahc_outb(ahc, FREE_SCBH, ahc_inb(ahc, SCBPTR));
6273 	}
6274 }
6275 
6276 /*
6277  * Manipulate the waiting for selection list and return the
6278  * scb that follows the one that we remove.
6279  */
6280 static u_int
6281 ahc_rem_wscb(struct ahc_softc *ahc, u_int scbpos, u_int prev)
6282 {
6283 	u_int curscb, next;
6284 
6285 	/*
6286 	 * Select the SCB we want to abort and
6287 	 * pull the next pointer out of it.
6288 	 */
6289 	curscb = ahc_inb(ahc, SCBPTR);
6290 	ahc_outb(ahc, SCBPTR, scbpos);
6291 	next = ahc_inb(ahc, SCB_NEXT);
6292 
6293 	/* Clear the necessary fields */
6294 	ahc_outb(ahc, SCB_CONTROL, 0);
6295 
6296 	ahc_add_curscb_to_free_list(ahc);
6297 
6298 	/* update the waiting list */
6299 	if (prev == SCB_LIST_NULL) {
6300 		/* First in the list */
6301 		ahc_outb(ahc, WAITING_SCBH, next);
6302 
6303 		/*
6304 		 * Ensure we aren't attempting to perform
6305 		 * selection for this entry.
6306 		 */
6307 		ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
6308 	} else {
6309 		/*
6310 		 * Select the scb that pointed to us
6311 		 * and update its next pointer.
6312 		 */
6313 		ahc_outb(ahc, SCBPTR, prev);
6314 		ahc_outb(ahc, SCB_NEXT, next);
6315 	}
6316 
6317 	/*
6318 	 * Point us back at the original scb position.
6319 	 */
6320 	ahc_outb(ahc, SCBPTR, curscb);
6321 	return next;
6322 }
6323 
6324 /******************************** Error Handling ******************************/
6325 /*
6326  * Abort all SCBs that match the given description (target/channel/lun/tag),
6327  * setting their status to the passed in status if the status has not already
6328  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
6329  * is paused before it is called.
6330  */
6331 static int
6332 ahc_abort_scbs(struct ahc_softc *ahc, int target, char channel,
6333 	       int lun, u_int tag, role_t role, uint32_t status)
6334 {
6335 	struct	scb *scbp;
6336 	struct	scb *scbp_next;
6337 	u_int	active_scb;
6338 	int	i, j;
6339 	int	maxtarget;
6340 	int	minlun;
6341 	int	maxlun;
6342 
6343 	int	found;
6344 
6345 	/*
6346 	 * Don't attempt to run any queued untagged transactions
6347 	 * until we are done with the abort process.
6348 	 */
6349 	ahc_freeze_untagged_queues(ahc);
6350 
6351 	/* restore this when we're done */
6352 	active_scb = ahc_inb(ahc, SCBPTR);
6353 
6354 	found = ahc_search_qinfifo(ahc, target, channel, lun, SCB_LIST_NULL,
6355 				   role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
6356 
6357 	/*
6358 	 * Clean out the busy target table for any untagged commands.
6359 	 */
6360 	i = 0;
6361 	maxtarget = 16;
6362 	if (target != CAM_TARGET_WILDCARD) {
6363 		i = target;
6364 		if (channel == 'B')
6365 			i += 8;
6366 		maxtarget = i + 1;
6367 	}
6368 
6369 	if (lun == CAM_LUN_WILDCARD) {
6370 
6371 		/*
6372 		 * Unless we are using an SCB based
6373 		 * busy targets table, there is only
6374 		 * one table entry for all luns of
6375 		 * a target.
6376 		 */
6377 		minlun = 0;
6378 		maxlun = 1;
6379 		if ((ahc->flags & AHC_SCB_BTT) != 0)
6380 			maxlun = AHC_NUM_LUNS;
6381 	} else {
6382 		minlun = lun;
6383 		maxlun = lun + 1;
6384 	}
6385 
6386 	if (role != ROLE_TARGET) {
6387 		for (;i < maxtarget; i++) {
6388 			for (j = minlun;j < maxlun; j++) {
6389 				u_int scbid;
6390 				u_int tcl;
6391 
6392 				tcl = BUILD_TCL(i << 4, j);
6393 				scbid = ahc_index_busy_tcl(ahc, tcl);
6394 				scbp = ahc_lookup_scb(ahc, scbid);
6395 				if (scbp == NULL
6396 				 || ahc_match_scb(ahc, scbp, target, channel,
6397 						  lun, tag, role) == 0)
6398 					continue;
6399 				ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, j));
6400 			}
6401 		}
6402 
6403 		/*
6404 		 * Go through the disconnected list and remove any entries we
6405 		 * have queued for completion, 0'ing their control byte too.
6406 		 * We save the active SCB and restore it ourselves, so there
6407 		 * is no reason for this search to restore it too.
6408 		 */
6409 		ahc_search_disc_list(ahc, target, channel, lun, tag,
6410 				     /*stop_on_first*/FALSE, /*remove*/TRUE,
6411 				     /*save_state*/FALSE);
6412 	}
6413 
6414 	/*
6415 	 * Go through the hardware SCB array looking for commands that
6416 	 * were active but not on any list.  In some cases, these remnants
6417 	 * might not still have mappings in the scbindex array (e.g. unexpected
6418 	 * bus free with the same scb queued for an abort).  Don't hold this
6419 	 * against them.
6420 	 */
6421 	for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
6422 		u_int scbid;
6423 
6424 		ahc_outb(ahc, SCBPTR, i);
6425 		scbid = ahc_inb(ahc, SCB_TAG);
6426 		scbp = ahc_lookup_scb(ahc, scbid);
6427 		if ((scbp == NULL && scbid != SCB_LIST_NULL)
6428 		 || (scbp != NULL
6429 		  && ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)))
6430 			ahc_add_curscb_to_free_list(ahc);
6431 	}
6432 
6433 	/*
6434 	 * Go through the pending CCB list and look for
6435 	 * commands for this target that are still active.
6436 	 * These are other tagged commands that were
6437 	 * disconnected when the reset occurred.
6438 	 */
6439 	scbp_next = LIST_FIRST(&ahc->pending_scbs);
6440 	while (scbp_next != NULL) {
6441 		scbp = scbp_next;
6442 		scbp_next = LIST_NEXT(scbp, pending_links);
6443 		if (ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)) {
6444 			cam_status ostat;
6445 
6446 			ostat = ahc_get_transaction_status(scbp);
6447 			if (ostat == CAM_REQ_INPROG)
6448 				ahc_set_transaction_status(scbp, status);
6449 			if (ahc_get_transaction_status(scbp) != CAM_REQ_CMP)
6450 				ahc_freeze_scb(scbp);
6451 			if ((scbp->flags & SCB_ACTIVE) == 0)
6452 				printk("Inactive SCB on pending list\n");
6453 			ahc_done(ahc, scbp);
6454 			found++;
6455 		}
6456 	}
6457 	ahc_outb(ahc, SCBPTR, active_scb);
6458 	ahc_platform_abort_scbs(ahc, target, channel, lun, tag, role, status);
6459 	ahc_release_untagged_queues(ahc);
6460 	return found;
6461 }
6462 
6463 static void
6464 ahc_reset_current_bus(struct ahc_softc *ahc)
6465 {
6466 	uint8_t scsiseq;
6467 
6468 	ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENSCSIRST);
6469 	scsiseq = ahc_inb(ahc, SCSISEQ);
6470 	ahc_outb(ahc, SCSISEQ, scsiseq | SCSIRSTO);
6471 	ahc_flush_device_writes(ahc);
6472 	ahc_delay(AHC_BUSRESET_DELAY);
6473 	/* Turn off the bus reset */
6474 	ahc_outb(ahc, SCSISEQ, scsiseq & ~SCSIRSTO);
6475 
6476 	ahc_clear_intstat(ahc);
6477 
6478 	/* Re-enable reset interrupts */
6479 	ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) | ENSCSIRST);
6480 }
6481 
6482 int
6483 ahc_reset_channel(struct ahc_softc *ahc, char channel, int initiate_reset)
6484 {
6485 	struct	ahc_devinfo devinfo;
6486 	u_int	initiator, target, max_scsiid;
6487 	u_int	sblkctl;
6488 	u_int	scsiseq;
6489 	u_int	simode1;
6490 	int	found;
6491 	int	restart_needed;
6492 	char	cur_channel;
6493 
6494 	ahc->pending_device = NULL;
6495 
6496 	ahc_compile_devinfo(&devinfo,
6497 			    CAM_TARGET_WILDCARD,
6498 			    CAM_TARGET_WILDCARD,
6499 			    CAM_LUN_WILDCARD,
6500 			    channel, ROLE_UNKNOWN);
6501 	ahc_pause(ahc);
6502 
6503 	/* Make sure the sequencer is in a safe location. */
6504 	ahc_clear_critical_section(ahc);
6505 
6506 	/*
6507 	 * Run our command complete fifos to ensure that we perform
6508 	 * completion processing on any commands that 'completed'
6509 	 * before the reset occurred.
6510 	 */
6511 	ahc_run_qoutfifo(ahc);
6512 #ifdef AHC_TARGET_MODE
6513 	/*
6514 	 * XXX - In Twin mode, the tqinfifo may have commands
6515 	 *	 for an unaffected channel in it.  However, if
6516 	 *	 we have run out of ATIO resources to drain that
6517 	 *	 queue, we may not get them all out here.  Further,
6518 	 *	 the blocked transactions for the reset channel
6519 	 *	 should just be killed off, irrespecitve of whether
6520 	 *	 we are blocked on ATIO resources.  Write a routine
6521 	 *	 to compact the tqinfifo appropriately.
6522 	 */
6523 	if ((ahc->flags & AHC_TARGETROLE) != 0) {
6524 		ahc_run_tqinfifo(ahc, /*paused*/TRUE);
6525 	}
6526 #endif
6527 
6528 	/*
6529 	 * Reset the bus if we are initiating this reset
6530 	 */
6531 	sblkctl = ahc_inb(ahc, SBLKCTL);
6532 	cur_channel = 'A';
6533 	if ((ahc->features & AHC_TWIN) != 0
6534 	 && ((sblkctl & SELBUSB) != 0))
6535 	    cur_channel = 'B';
6536 	scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
6537 	if (cur_channel != channel) {
6538 		/* Case 1: Command for another bus is active
6539 		 * Stealthily reset the other bus without
6540 		 * upsetting the current bus.
6541 		 */
6542 		ahc_outb(ahc, SBLKCTL, sblkctl ^ SELBUSB);
6543 		simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6544 #ifdef AHC_TARGET_MODE
6545 		/*
6546 		 * Bus resets clear ENSELI, so we cannot
6547 		 * defer re-enabling bus reset interrupts
6548 		 * if we are in target mode.
6549 		 */
6550 		if ((ahc->flags & AHC_TARGETROLE) != 0)
6551 			simode1 |= ENSCSIRST;
6552 #endif
6553 		ahc_outb(ahc, SIMODE1, simode1);
6554 		if (initiate_reset)
6555 			ahc_reset_current_bus(ahc);
6556 		ahc_clear_intstat(ahc);
6557 		ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6558 		ahc_outb(ahc, SBLKCTL, sblkctl);
6559 		restart_needed = FALSE;
6560 	} else {
6561 		/* Case 2: A command from this bus is active or we're idle */
6562 		simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6563 #ifdef AHC_TARGET_MODE
6564 		/*
6565 		 * Bus resets clear ENSELI, so we cannot
6566 		 * defer re-enabling bus reset interrupts
6567 		 * if we are in target mode.
6568 		 */
6569 		if ((ahc->flags & AHC_TARGETROLE) != 0)
6570 			simode1 |= ENSCSIRST;
6571 #endif
6572 		ahc_outb(ahc, SIMODE1, simode1);
6573 		if (initiate_reset)
6574 			ahc_reset_current_bus(ahc);
6575 		ahc_clear_intstat(ahc);
6576 		ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6577 		restart_needed = TRUE;
6578 	}
6579 
6580 	/*
6581 	 * Clean up all the state information for the
6582 	 * pending transactions on this bus.
6583 	 */
6584 	found = ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, channel,
6585 			       CAM_LUN_WILDCARD, SCB_LIST_NULL,
6586 			       ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6587 
6588 	max_scsiid = (ahc->features & AHC_WIDE) ? 15 : 7;
6589 
6590 #ifdef AHC_TARGET_MODE
6591 	/*
6592 	 * Send an immediate notify ccb to all target more peripheral
6593 	 * drivers affected by this action.
6594 	 */
6595 	for (target = 0; target <= max_scsiid; target++) {
6596 		struct ahc_tmode_tstate* tstate;
6597 		u_int lun;
6598 
6599 		tstate = ahc->enabled_targets[target];
6600 		if (tstate == NULL)
6601 			continue;
6602 		for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
6603 			struct ahc_tmode_lstate* lstate;
6604 
6605 			lstate = tstate->enabled_luns[lun];
6606 			if (lstate == NULL)
6607 				continue;
6608 
6609 			ahc_queue_lstate_event(ahc, lstate, CAM_TARGET_WILDCARD,
6610 					       EVENT_TYPE_BUS_RESET, /*arg*/0);
6611 			ahc_send_lstate_events(ahc, lstate);
6612 		}
6613 	}
6614 #endif
6615 	/* Notify the XPT that a bus reset occurred */
6616 	ahc_send_async(ahc, devinfo.channel, CAM_TARGET_WILDCARD,
6617 		       CAM_LUN_WILDCARD, AC_BUS_RESET);
6618 
6619 	/*
6620 	 * Revert to async/narrow transfers until we renegotiate.
6621 	 */
6622 	for (target = 0; target <= max_scsiid; target++) {
6623 
6624 		if (ahc->enabled_targets[target] == NULL)
6625 			continue;
6626 		for (initiator = 0; initiator <= max_scsiid; initiator++) {
6627 			struct ahc_devinfo devinfo;
6628 
6629 			ahc_compile_devinfo(&devinfo, target, initiator,
6630 					    CAM_LUN_WILDCARD,
6631 					    channel, ROLE_UNKNOWN);
6632 			ahc_set_width(ahc, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6633 				      AHC_TRANS_CUR, /*paused*/TRUE);
6634 			ahc_set_syncrate(ahc, &devinfo, /*syncrate*/NULL,
6635 					 /*period*/0, /*offset*/0,
6636 					 /*ppr_options*/0, AHC_TRANS_CUR,
6637 					 /*paused*/TRUE);
6638 		}
6639 	}
6640 
6641 	if (restart_needed)
6642 		ahc_restart(ahc);
6643 	else
6644 		ahc_unpause(ahc);
6645 	return found;
6646 }
6647 
6648 
6649 /***************************** Residual Processing ****************************/
6650 /*
6651  * Calculate the residual for a just completed SCB.
6652  */
6653 static void
6654 ahc_calc_residual(struct ahc_softc *ahc, struct scb *scb)
6655 {
6656 	struct hardware_scb *hscb;
6657 	struct status_pkt *spkt;
6658 	uint32_t sgptr;
6659 	uint32_t resid_sgptr;
6660 	uint32_t resid;
6661 
6662 	/*
6663 	 * 5 cases.
6664 	 * 1) No residual.
6665 	 *    SG_RESID_VALID clear in sgptr.
6666 	 * 2) Transferless command
6667 	 * 3) Never performed any transfers.
6668 	 *    sgptr has SG_FULL_RESID set.
6669 	 * 4) No residual but target did not
6670 	 *    save data pointers after the
6671 	 *    last transfer, so sgptr was
6672 	 *    never updated.
6673 	 * 5) We have a partial residual.
6674 	 *    Use residual_sgptr to determine
6675 	 *    where we are.
6676 	 */
6677 
6678 	hscb = scb->hscb;
6679 	sgptr = ahc_le32toh(hscb->sgptr);
6680 	if ((sgptr & SG_RESID_VALID) == 0)
6681 		/* Case 1 */
6682 		return;
6683 	sgptr &= ~SG_RESID_VALID;
6684 
6685 	if ((sgptr & SG_LIST_NULL) != 0)
6686 		/* Case 2 */
6687 		return;
6688 
6689 	spkt = &hscb->shared_data.status;
6690 	resid_sgptr = ahc_le32toh(spkt->residual_sg_ptr);
6691 	if ((sgptr & SG_FULL_RESID) != 0) {
6692 		/* Case 3 */
6693 		resid = ahc_get_transfer_length(scb);
6694 	} else if ((resid_sgptr & SG_LIST_NULL) != 0) {
6695 		/* Case 4 */
6696 		return;
6697 	} else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
6698 		panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
6699 	} else {
6700 		struct ahc_dma_seg *sg;
6701 
6702 		/*
6703 		 * Remainder of the SG where the transfer
6704 		 * stopped.
6705 		 */
6706 		resid = ahc_le32toh(spkt->residual_datacnt) & AHC_SG_LEN_MASK;
6707 		sg = ahc_sg_bus_to_virt(scb, resid_sgptr & SG_PTR_MASK);
6708 
6709 		/* The residual sg_ptr always points to the next sg */
6710 		sg--;
6711 
6712 		/*
6713 		 * Add up the contents of all residual
6714 		 * SG segments that are after the SG where
6715 		 * the transfer stopped.
6716 		 */
6717 		while ((ahc_le32toh(sg->len) & AHC_DMA_LAST_SEG) == 0) {
6718 			sg++;
6719 			resid += ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
6720 		}
6721 	}
6722 	if ((scb->flags & SCB_SENSE) == 0)
6723 		ahc_set_residual(scb, resid);
6724 	else
6725 		ahc_set_sense_residual(scb, resid);
6726 
6727 #ifdef AHC_DEBUG
6728 	if ((ahc_debug & AHC_SHOW_MISC) != 0) {
6729 		ahc_print_path(ahc, scb);
6730 		printk("Handled %sResidual of %d bytes\n",
6731 		       (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
6732 	}
6733 #endif
6734 }
6735 
6736 /******************************* Target Mode **********************************/
6737 #ifdef AHC_TARGET_MODE
6738 /*
6739  * Add a target mode event to this lun's queue
6740  */
6741 static void
6742 ahc_queue_lstate_event(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate,
6743 		       u_int initiator_id, u_int event_type, u_int event_arg)
6744 {
6745 	struct ahc_tmode_event *event;
6746 	int pending;
6747 
6748 	xpt_freeze_devq(lstate->path, /*count*/1);
6749 	if (lstate->event_w_idx >= lstate->event_r_idx)
6750 		pending = lstate->event_w_idx - lstate->event_r_idx;
6751 	else
6752 		pending = AHC_TMODE_EVENT_BUFFER_SIZE + 1
6753 			- (lstate->event_r_idx - lstate->event_w_idx);
6754 
6755 	if (event_type == EVENT_TYPE_BUS_RESET
6756 	 || event_type == MSG_BUS_DEV_RESET) {
6757 		/*
6758 		 * Any earlier events are irrelevant, so reset our buffer.
6759 		 * This has the effect of allowing us to deal with reset
6760 		 * floods (an external device holding down the reset line)
6761 		 * without losing the event that is really interesting.
6762 		 */
6763 		lstate->event_r_idx = 0;
6764 		lstate->event_w_idx = 0;
6765 		xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
6766 	}
6767 
6768 	if (pending == AHC_TMODE_EVENT_BUFFER_SIZE) {
6769 		xpt_print_path(lstate->path);
6770 		printk("immediate event %x:%x lost\n",
6771 		       lstate->event_buffer[lstate->event_r_idx].event_type,
6772 		       lstate->event_buffer[lstate->event_r_idx].event_arg);
6773 		lstate->event_r_idx++;
6774 		if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6775 			lstate->event_r_idx = 0;
6776 		xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
6777 	}
6778 
6779 	event = &lstate->event_buffer[lstate->event_w_idx];
6780 	event->initiator_id = initiator_id;
6781 	event->event_type = event_type;
6782 	event->event_arg = event_arg;
6783 	lstate->event_w_idx++;
6784 	if (lstate->event_w_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6785 		lstate->event_w_idx = 0;
6786 }
6787 
6788 /*
6789  * Send any target mode events queued up waiting
6790  * for immediate notify resources.
6791  */
6792 void
6793 ahc_send_lstate_events(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate)
6794 {
6795 	struct ccb_hdr *ccbh;
6796 	struct ccb_immed_notify *inot;
6797 
6798 	while (lstate->event_r_idx != lstate->event_w_idx
6799 	    && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
6800 		struct ahc_tmode_event *event;
6801 
6802 		event = &lstate->event_buffer[lstate->event_r_idx];
6803 		SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
6804 		inot = (struct ccb_immed_notify *)ccbh;
6805 		switch (event->event_type) {
6806 		case EVENT_TYPE_BUS_RESET:
6807 			ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
6808 			break;
6809 		default:
6810 			ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
6811 			inot->message_args[0] = event->event_type;
6812 			inot->message_args[1] = event->event_arg;
6813 			break;
6814 		}
6815 		inot->initiator_id = event->initiator_id;
6816 		inot->sense_len = 0;
6817 		xpt_done((union ccb *)inot);
6818 		lstate->event_r_idx++;
6819 		if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6820 			lstate->event_r_idx = 0;
6821 	}
6822 }
6823 #endif
6824 
6825 /******************** Sequencer Program Patching/Download *********************/
6826 
6827 #ifdef AHC_DUMP_SEQ
6828 void
6829 ahc_dumpseq(struct ahc_softc* ahc)
6830 {
6831 	int i;
6832 
6833 	ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6834 	ahc_outb(ahc, SEQADDR0, 0);
6835 	ahc_outb(ahc, SEQADDR1, 0);
6836 	for (i = 0; i < ahc->instruction_ram_size; i++) {
6837 		uint8_t ins_bytes[4];
6838 
6839 		ahc_insb(ahc, SEQRAM, ins_bytes, 4);
6840 		printk("0x%08x\n", ins_bytes[0] << 24
6841 				 | ins_bytes[1] << 16
6842 				 | ins_bytes[2] << 8
6843 				 | ins_bytes[3]);
6844 	}
6845 }
6846 #endif
6847 
6848 static int
6849 ahc_loadseq(struct ahc_softc *ahc)
6850 {
6851 	struct	cs cs_table[num_critical_sections];
6852 	u_int	begin_set[num_critical_sections];
6853 	u_int	end_set[num_critical_sections];
6854 	const struct patch *cur_patch;
6855 	u_int	cs_count;
6856 	u_int	cur_cs;
6857 	u_int	i;
6858 	u_int	skip_addr;
6859 	u_int	sg_prefetch_cnt;
6860 	int	downloaded;
6861 	uint8_t	download_consts[7];
6862 
6863 	/*
6864 	 * Start out with 0 critical sections
6865 	 * that apply to this firmware load.
6866 	 */
6867 	cs_count = 0;
6868 	cur_cs = 0;
6869 	memset(begin_set, 0, sizeof(begin_set));
6870 	memset(end_set, 0, sizeof(end_set));
6871 
6872 	/* Setup downloadable constant table */
6873 	download_consts[QOUTFIFO_OFFSET] = 0;
6874 	if (ahc->targetcmds != NULL)
6875 		download_consts[QOUTFIFO_OFFSET] += 32;
6876 	download_consts[QINFIFO_OFFSET] = download_consts[QOUTFIFO_OFFSET] + 1;
6877 	download_consts[CACHESIZE_MASK] = ahc->pci_cachesize - 1;
6878 	download_consts[INVERTED_CACHESIZE_MASK] = ~(ahc->pci_cachesize - 1);
6879 	sg_prefetch_cnt = ahc->pci_cachesize;
6880 	if (sg_prefetch_cnt < (2 * sizeof(struct ahc_dma_seg)))
6881 		sg_prefetch_cnt = 2 * sizeof(struct ahc_dma_seg);
6882 	download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
6883 	download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_cnt - 1);
6884 	download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_cnt - 1);
6885 
6886 	cur_patch = patches;
6887 	downloaded = 0;
6888 	skip_addr = 0;
6889 	ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6890 	ahc_outb(ahc, SEQADDR0, 0);
6891 	ahc_outb(ahc, SEQADDR1, 0);
6892 
6893 	for (i = 0; i < sizeof(seqprog)/4; i++) {
6894 		if (ahc_check_patch(ahc, &cur_patch, i, &skip_addr) == 0) {
6895 			/*
6896 			 * Don't download this instruction as it
6897 			 * is in a patch that was removed.
6898 			 */
6899 			continue;
6900 		}
6901 
6902 		if (downloaded == ahc->instruction_ram_size) {
6903 			/*
6904 			 * We're about to exceed the instruction
6905 			 * storage capacity for this chip.  Fail
6906 			 * the load.
6907 			 */
6908 			printk("\n%s: Program too large for instruction memory "
6909 			       "size of %d!\n", ahc_name(ahc),
6910 			       ahc->instruction_ram_size);
6911 			return (ENOMEM);
6912 		}
6913 
6914 		/*
6915 		 * Move through the CS table until we find a CS
6916 		 * that might apply to this instruction.
6917 		 */
6918 		for (; cur_cs < num_critical_sections; cur_cs++) {
6919 			if (critical_sections[cur_cs].end <= i) {
6920 				if (begin_set[cs_count] == TRUE
6921 				 && end_set[cs_count] == FALSE) {
6922 					cs_table[cs_count].end = downloaded;
6923 				 	end_set[cs_count] = TRUE;
6924 					cs_count++;
6925 				}
6926 				continue;
6927 			}
6928 			if (critical_sections[cur_cs].begin <= i
6929 			 && begin_set[cs_count] == FALSE) {
6930 				cs_table[cs_count].begin = downloaded;
6931 				begin_set[cs_count] = TRUE;
6932 			}
6933 			break;
6934 		}
6935 		ahc_download_instr(ahc, i, download_consts);
6936 		downloaded++;
6937 	}
6938 
6939 	ahc->num_critical_sections = cs_count;
6940 	if (cs_count != 0) {
6941 
6942 		cs_count *= sizeof(struct cs);
6943 		ahc->critical_sections = kmalloc(cs_count, GFP_ATOMIC);
6944 		if (ahc->critical_sections == NULL)
6945 			panic("ahc_loadseq: Could not malloc");
6946 		memcpy(ahc->critical_sections, cs_table, cs_count);
6947 	}
6948 	ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE);
6949 
6950 	if (bootverbose) {
6951 		printk(" %d instructions downloaded\n", downloaded);
6952 		printk("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
6953 		       ahc_name(ahc), ahc->features, ahc->bugs, ahc->flags);
6954 	}
6955 	return (0);
6956 }
6957 
6958 static int
6959 ahc_check_patch(struct ahc_softc *ahc, const struct patch **start_patch,
6960 		u_int start_instr, u_int *skip_addr)
6961 {
6962 	const struct patch *cur_patch;
6963 	const struct patch *last_patch;
6964 	u_int	num_patches;
6965 
6966 	num_patches = ARRAY_SIZE(patches);
6967 	last_patch = &patches[num_patches];
6968 	cur_patch = *start_patch;
6969 
6970 	while (cur_patch < last_patch && start_instr == cur_patch->begin) {
6971 
6972 		if (cur_patch->patch_func(ahc) == 0) {
6973 
6974 			/* Start rejecting code */
6975 			*skip_addr = start_instr + cur_patch->skip_instr;
6976 			cur_patch += cur_patch->skip_patch;
6977 		} else {
6978 			/* Accepted this patch.  Advance to the next
6979 			 * one and wait for our intruction pointer to
6980 			 * hit this point.
6981 			 */
6982 			cur_patch++;
6983 		}
6984 	}
6985 
6986 	*start_patch = cur_patch;
6987 	if (start_instr < *skip_addr)
6988 		/* Still skipping */
6989 		return (0);
6990 
6991 	return (1);
6992 }
6993 
6994 static void
6995 ahc_download_instr(struct ahc_softc *ahc, u_int instrptr, uint8_t *dconsts)
6996 {
6997 	union	ins_formats instr;
6998 	struct	ins_format1 *fmt1_ins;
6999 	struct	ins_format3 *fmt3_ins;
7000 	u_int	opcode;
7001 
7002 	/*
7003 	 * The firmware is always compiled into a little endian format.
7004 	 */
7005 	instr.integer = ahc_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
7006 
7007 	fmt1_ins = &instr.format1;
7008 	fmt3_ins = NULL;
7009 
7010 	/* Pull the opcode */
7011 	opcode = instr.format1.opcode;
7012 	switch (opcode) {
7013 	case AIC_OP_JMP:
7014 	case AIC_OP_JC:
7015 	case AIC_OP_JNC:
7016 	case AIC_OP_CALL:
7017 	case AIC_OP_JNE:
7018 	case AIC_OP_JNZ:
7019 	case AIC_OP_JE:
7020 	case AIC_OP_JZ:
7021 	{
7022 		const struct patch *cur_patch;
7023 		int address_offset;
7024 		u_int address;
7025 		u_int skip_addr;
7026 		u_int i;
7027 
7028 		fmt3_ins = &instr.format3;
7029 		address_offset = 0;
7030 		address = fmt3_ins->address;
7031 		cur_patch = patches;
7032 		skip_addr = 0;
7033 
7034 		for (i = 0; i < address;) {
7035 
7036 			ahc_check_patch(ahc, &cur_patch, i, &skip_addr);
7037 
7038 			if (skip_addr > i) {
7039 				int end_addr;
7040 
7041 				end_addr = min(address, skip_addr);
7042 				address_offset += end_addr - i;
7043 				i = skip_addr;
7044 			} else {
7045 				i++;
7046 			}
7047 		}
7048 		address -= address_offset;
7049 		fmt3_ins->address = address;
7050 		/* FALLTHROUGH */
7051 	}
7052 	case AIC_OP_OR:
7053 	case AIC_OP_AND:
7054 	case AIC_OP_XOR:
7055 	case AIC_OP_ADD:
7056 	case AIC_OP_ADC:
7057 	case AIC_OP_BMOV:
7058 		if (fmt1_ins->parity != 0) {
7059 			fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
7060 		}
7061 		fmt1_ins->parity = 0;
7062 		if ((ahc->features & AHC_CMD_CHAN) == 0
7063 		 && opcode == AIC_OP_BMOV) {
7064 			/*
7065 			 * Block move was added at the same time
7066 			 * as the command channel.  Verify that
7067 			 * this is only a move of a single element
7068 			 * and convert the BMOV to a MOV
7069 			 * (AND with an immediate of FF).
7070 			 */
7071 			if (fmt1_ins->immediate != 1)
7072 				panic("%s: BMOV not supported\n",
7073 				      ahc_name(ahc));
7074 			fmt1_ins->opcode = AIC_OP_AND;
7075 			fmt1_ins->immediate = 0xff;
7076 		}
7077 		/* FALLTHROUGH */
7078 	case AIC_OP_ROL:
7079 		if ((ahc->features & AHC_ULTRA2) != 0) {
7080 			int i, count;
7081 
7082 			/* Calculate odd parity for the instruction */
7083 			for (i = 0, count = 0; i < 31; i++) {
7084 				uint32_t mask;
7085 
7086 				mask = 0x01 << i;
7087 				if ((instr.integer & mask) != 0)
7088 					count++;
7089 			}
7090 			if ((count & 0x01) == 0)
7091 				instr.format1.parity = 1;
7092 		} else {
7093 			/* Compress the instruction for older sequencers */
7094 			if (fmt3_ins != NULL) {
7095 				instr.integer =
7096 					fmt3_ins->immediate
7097 				      | (fmt3_ins->source << 8)
7098 				      | (fmt3_ins->address << 16)
7099 				      |	(fmt3_ins->opcode << 25);
7100 			} else {
7101 				instr.integer =
7102 					fmt1_ins->immediate
7103 				      | (fmt1_ins->source << 8)
7104 				      | (fmt1_ins->destination << 16)
7105 				      |	(fmt1_ins->ret << 24)
7106 				      |	(fmt1_ins->opcode << 25);
7107 			}
7108 		}
7109 		/* The sequencer is a little endian cpu */
7110 		instr.integer = ahc_htole32(instr.integer);
7111 		ahc_outsb(ahc, SEQRAM, instr.bytes, 4);
7112 		break;
7113 	default:
7114 		panic("Unknown opcode encountered in seq program");
7115 		break;
7116 	}
7117 }
7118 
7119 int
7120 ahc_print_register(const ahc_reg_parse_entry_t *table, u_int num_entries,
7121 		   const char *name, u_int address, u_int value,
7122 		   u_int *cur_column, u_int wrap_point)
7123 {
7124 	int	printed;
7125 	u_int	printed_mask;
7126 
7127 	if (cur_column != NULL && *cur_column >= wrap_point) {
7128 		printk("\n");
7129 		*cur_column = 0;
7130 	}
7131 	printed  = printk("%s[0x%x]", name, value);
7132 	if (table == NULL) {
7133 		printed += printk(" ");
7134 		*cur_column += printed;
7135 		return (printed);
7136 	}
7137 	printed_mask = 0;
7138 	while (printed_mask != 0xFF) {
7139 		int entry;
7140 
7141 		for (entry = 0; entry < num_entries; entry++) {
7142 			if (((value & table[entry].mask)
7143 			  != table[entry].value)
7144 			 || ((printed_mask & table[entry].mask)
7145 			  == table[entry].mask))
7146 				continue;
7147 
7148 			printed += printk("%s%s",
7149 					  printed_mask == 0 ? ":(" : "|",
7150 					  table[entry].name);
7151 			printed_mask |= table[entry].mask;
7152 
7153 			break;
7154 		}
7155 		if (entry >= num_entries)
7156 			break;
7157 	}
7158 	if (printed_mask != 0)
7159 		printed += printk(") ");
7160 	else
7161 		printed += printk(" ");
7162 	if (cur_column != NULL)
7163 		*cur_column += printed;
7164 	return (printed);
7165 }
7166 
7167 void
7168 ahc_dump_card_state(struct ahc_softc *ahc)
7169 {
7170 	struct	scb *scb;
7171 	struct	scb_tailq *untagged_q;
7172 	u_int	cur_col;
7173 	int	paused;
7174 	int	target;
7175 	int	maxtarget;
7176 	int	i;
7177 	uint8_t last_phase;
7178 	uint8_t qinpos;
7179 	uint8_t qintail;
7180 	uint8_t qoutpos;
7181 	uint8_t scb_index;
7182 	uint8_t saved_scbptr;
7183 
7184 	if (ahc_is_paused(ahc)) {
7185 		paused = 1;
7186 	} else {
7187 		paused = 0;
7188 		ahc_pause(ahc);
7189 	}
7190 
7191 	saved_scbptr = ahc_inb(ahc, SCBPTR);
7192 	last_phase = ahc_inb(ahc, LASTPHASE);
7193 	printk(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
7194 	       "%s: Dumping Card State %s, at SEQADDR 0x%x\n",
7195 	       ahc_name(ahc), ahc_lookup_phase_entry(last_phase)->phasemsg,
7196 	       ahc_inb(ahc, SEQADDR0) | (ahc_inb(ahc, SEQADDR1) << 8));
7197 	if (paused)
7198 		printk("Card was paused\n");
7199 	printk("ACCUM = 0x%x, SINDEX = 0x%x, DINDEX = 0x%x, ARG_2 = 0x%x\n",
7200 	       ahc_inb(ahc, ACCUM), ahc_inb(ahc, SINDEX), ahc_inb(ahc, DINDEX),
7201 	       ahc_inb(ahc, ARG_2));
7202 	printk("HCNT = 0x%x SCBPTR = 0x%x\n", ahc_inb(ahc, HCNT),
7203 	       ahc_inb(ahc, SCBPTR));
7204 	cur_col = 0;
7205 	if ((ahc->features & AHC_DT) != 0)
7206 		ahc_scsiphase_print(ahc_inb(ahc, SCSIPHASE), &cur_col, 50);
7207 	ahc_scsisigi_print(ahc_inb(ahc, SCSISIGI), &cur_col, 50);
7208 	ahc_error_print(ahc_inb(ahc, ERROR), &cur_col, 50);
7209 	ahc_scsibusl_print(ahc_inb(ahc, SCSIBUSL), &cur_col, 50);
7210 	ahc_lastphase_print(ahc_inb(ahc, LASTPHASE), &cur_col, 50);
7211 	ahc_scsiseq_print(ahc_inb(ahc, SCSISEQ), &cur_col, 50);
7212 	ahc_sblkctl_print(ahc_inb(ahc, SBLKCTL), &cur_col, 50);
7213 	ahc_scsirate_print(ahc_inb(ahc, SCSIRATE), &cur_col, 50);
7214 	ahc_seqctl_print(ahc_inb(ahc, SEQCTL), &cur_col, 50);
7215 	ahc_seq_flags_print(ahc_inb(ahc, SEQ_FLAGS), &cur_col, 50);
7216 	ahc_sstat0_print(ahc_inb(ahc, SSTAT0), &cur_col, 50);
7217 	ahc_sstat1_print(ahc_inb(ahc, SSTAT1), &cur_col, 50);
7218 	ahc_sstat2_print(ahc_inb(ahc, SSTAT2), &cur_col, 50);
7219 	ahc_sstat3_print(ahc_inb(ahc, SSTAT3), &cur_col, 50);
7220 	ahc_simode0_print(ahc_inb(ahc, SIMODE0), &cur_col, 50);
7221 	ahc_simode1_print(ahc_inb(ahc, SIMODE1), &cur_col, 50);
7222 	ahc_sxfrctl0_print(ahc_inb(ahc, SXFRCTL0), &cur_col, 50);
7223 	ahc_dfcntrl_print(ahc_inb(ahc, DFCNTRL), &cur_col, 50);
7224 	ahc_dfstatus_print(ahc_inb(ahc, DFSTATUS), &cur_col, 50);
7225 	if (cur_col != 0)
7226 		printk("\n");
7227 	printk("STACK:");
7228 	for (i = 0; i < STACK_SIZE; i++)
7229 		printk(" 0x%x", ahc_inb(ahc, STACK)|(ahc_inb(ahc, STACK) << 8));
7230 	printk("\nSCB count = %d\n", ahc->scb_data->numscbs);
7231 	printk("Kernel NEXTQSCB = %d\n", ahc->next_queued_scb->hscb->tag);
7232 	printk("Card NEXTQSCB = %d\n", ahc_inb(ahc, NEXT_QUEUED_SCB));
7233 	/* QINFIFO */
7234 	printk("QINFIFO entries: ");
7235 	if ((ahc->features & AHC_QUEUE_REGS) != 0) {
7236 		qinpos = ahc_inb(ahc, SNSCB_QOFF);
7237 		ahc_outb(ahc, SNSCB_QOFF, qinpos);
7238 	} else
7239 		qinpos = ahc_inb(ahc, QINPOS);
7240 	qintail = ahc->qinfifonext;
7241 	while (qinpos != qintail) {
7242 		printk("%d ", ahc->qinfifo[qinpos]);
7243 		qinpos++;
7244 	}
7245 	printk("\n");
7246 
7247 	printk("Waiting Queue entries: ");
7248 	scb_index = ahc_inb(ahc, WAITING_SCBH);
7249 	i = 0;
7250 	while (scb_index != SCB_LIST_NULL && i++ < 256) {
7251 		ahc_outb(ahc, SCBPTR, scb_index);
7252 		printk("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7253 		scb_index = ahc_inb(ahc, SCB_NEXT);
7254 	}
7255 	printk("\n");
7256 
7257 	printk("Disconnected Queue entries: ");
7258 	scb_index = ahc_inb(ahc, DISCONNECTED_SCBH);
7259 	i = 0;
7260 	while (scb_index != SCB_LIST_NULL && i++ < 256) {
7261 		ahc_outb(ahc, SCBPTR, scb_index);
7262 		printk("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7263 		scb_index = ahc_inb(ahc, SCB_NEXT);
7264 	}
7265 	printk("\n");
7266 
7267 	ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
7268 	printk("QOUTFIFO entries: ");
7269 	qoutpos = ahc->qoutfifonext;
7270 	i = 0;
7271 	while (ahc->qoutfifo[qoutpos] != SCB_LIST_NULL && i++ < 256) {
7272 		printk("%d ", ahc->qoutfifo[qoutpos]);
7273 		qoutpos++;
7274 	}
7275 	printk("\n");
7276 
7277 	printk("Sequencer Free SCB List: ");
7278 	scb_index = ahc_inb(ahc, FREE_SCBH);
7279 	i = 0;
7280 	while (scb_index != SCB_LIST_NULL && i++ < 256) {
7281 		ahc_outb(ahc, SCBPTR, scb_index);
7282 		printk("%d ", scb_index);
7283 		scb_index = ahc_inb(ahc, SCB_NEXT);
7284 	}
7285 	printk("\n");
7286 
7287 	printk("Sequencer SCB Info: ");
7288 	for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
7289 		ahc_outb(ahc, SCBPTR, i);
7290 		cur_col  = printk("\n%3d ", i);
7291 
7292 		ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL), &cur_col, 60);
7293 		ahc_scb_scsiid_print(ahc_inb(ahc, SCB_SCSIID), &cur_col, 60);
7294 		ahc_scb_lun_print(ahc_inb(ahc, SCB_LUN), &cur_col, 60);
7295 		ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7296 	}
7297 	printk("\n");
7298 
7299 	printk("Pending list: ");
7300 	i = 0;
7301 	LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7302 		if (i++ > 256)
7303 			break;
7304 		cur_col  = printk("\n%3d ", scb->hscb->tag);
7305 		ahc_scb_control_print(scb->hscb->control, &cur_col, 60);
7306 		ahc_scb_scsiid_print(scb->hscb->scsiid, &cur_col, 60);
7307 		ahc_scb_lun_print(scb->hscb->lun, &cur_col, 60);
7308 		if ((ahc->flags & AHC_PAGESCBS) == 0) {
7309 			ahc_outb(ahc, SCBPTR, scb->hscb->tag);
7310 			printk("(");
7311 			ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL),
7312 					      &cur_col, 60);
7313 			ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7314 			printk(")");
7315 		}
7316 	}
7317 	printk("\n");
7318 
7319 	printk("Kernel Free SCB list: ");
7320 	i = 0;
7321 	SLIST_FOREACH(scb, &ahc->scb_data->free_scbs, links.sle) {
7322 		if (i++ > 256)
7323 			break;
7324 		printk("%d ", scb->hscb->tag);
7325 	}
7326 	printk("\n");
7327 
7328 	maxtarget = (ahc->features & (AHC_WIDE|AHC_TWIN)) ? 15 : 7;
7329 	for (target = 0; target <= maxtarget; target++) {
7330 		untagged_q = &ahc->untagged_queues[target];
7331 		if (TAILQ_FIRST(untagged_q) == NULL)
7332 			continue;
7333 		printk("Untagged Q(%d): ", target);
7334 		i = 0;
7335 		TAILQ_FOREACH(scb, untagged_q, links.tqe) {
7336 			if (i++ > 256)
7337 				break;
7338 			printk("%d ", scb->hscb->tag);
7339 		}
7340 		printk("\n");
7341 	}
7342 
7343 	ahc_platform_dump_card_state(ahc);
7344 	printk("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
7345 	ahc_outb(ahc, SCBPTR, saved_scbptr);
7346 	if (paused == 0)
7347 		ahc_unpause(ahc);
7348 }
7349 
7350 /************************* Target Mode ****************************************/
7351 #ifdef AHC_TARGET_MODE
7352 cam_status
7353 ahc_find_tmode_devs(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb,
7354 		    struct ahc_tmode_tstate **tstate,
7355 		    struct ahc_tmode_lstate **lstate,
7356 		    int notfound_failure)
7357 {
7358 
7359 	if ((ahc->features & AHC_TARGETMODE) == 0)
7360 		return (CAM_REQ_INVALID);
7361 
7362 	/*
7363 	 * Handle the 'black hole' device that sucks up
7364 	 * requests to unattached luns on enabled targets.
7365 	 */
7366 	if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
7367 	 && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
7368 		*tstate = NULL;
7369 		*lstate = ahc->black_hole;
7370 	} else {
7371 		u_int max_id;
7372 
7373 		max_id = (ahc->features & AHC_WIDE) ? 16 : 8;
7374 		if (ccb->ccb_h.target_id >= max_id)
7375 			return (CAM_TID_INVALID);
7376 
7377 		if (ccb->ccb_h.target_lun >= AHC_NUM_LUNS)
7378 			return (CAM_LUN_INVALID);
7379 
7380 		*tstate = ahc->enabled_targets[ccb->ccb_h.target_id];
7381 		*lstate = NULL;
7382 		if (*tstate != NULL)
7383 			*lstate =
7384 			    (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
7385 	}
7386 
7387 	if (notfound_failure != 0 && *lstate == NULL)
7388 		return (CAM_PATH_INVALID);
7389 
7390 	return (CAM_REQ_CMP);
7391 }
7392 
7393 void
7394 ahc_handle_en_lun(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb)
7395 {
7396 	struct	   ahc_tmode_tstate *tstate;
7397 	struct	   ahc_tmode_lstate *lstate;
7398 	struct	   ccb_en_lun *cel;
7399 	cam_status status;
7400 	u_long	   s;
7401 	u_int	   target;
7402 	u_int	   lun;
7403 	u_int	   target_mask;
7404 	u_int	   our_id;
7405 	int	   error;
7406 	char	   channel;
7407 
7408 	status = ahc_find_tmode_devs(ahc, sim, ccb, &tstate, &lstate,
7409 				     /*notfound_failure*/FALSE);
7410 
7411 	if (status != CAM_REQ_CMP) {
7412 		ccb->ccb_h.status = status;
7413 		return;
7414 	}
7415 
7416 	if (cam_sim_bus(sim) == 0)
7417 		our_id = ahc->our_id;
7418 	else
7419 		our_id = ahc->our_id_b;
7420 
7421 	if (ccb->ccb_h.target_id != our_id) {
7422 		/*
7423 		 * our_id represents our initiator ID, or
7424 		 * the ID of the first target to have an
7425 		 * enabled lun in target mode.  There are
7426 		 * two cases that may preclude enabling a
7427 		 * target id other than our_id.
7428 		 *
7429 		 *   o our_id is for an active initiator role.
7430 		 *     Since the hardware does not support
7431 		 *     reselections to the initiator role at
7432 		 *     anything other than our_id, and our_id
7433 		 *     is used by the hardware to indicate the
7434 		 *     ID to use for both select-out and
7435 		 *     reselect-out operations, the only target
7436 		 *     ID we can support in this mode is our_id.
7437 		 *
7438 		 *   o The MULTARGID feature is not available and
7439 		 *     a previous target mode ID has been enabled.
7440 		 */
7441 		if ((ahc->features & AHC_MULTIROLE) != 0) {
7442 
7443 			if ((ahc->features & AHC_MULTI_TID) != 0
7444 		   	 && (ahc->flags & AHC_INITIATORROLE) != 0) {
7445 				/*
7446 				 * Only allow additional targets if
7447 				 * the initiator role is disabled.
7448 				 * The hardware cannot handle a re-select-in
7449 				 * on the initiator id during a re-select-out
7450 				 * on a different target id.
7451 				 */
7452 				status = CAM_TID_INVALID;
7453 			} else if ((ahc->flags & AHC_INITIATORROLE) != 0
7454 				|| ahc->enabled_luns > 0) {
7455 				/*
7456 				 * Only allow our target id to change
7457 				 * if the initiator role is not configured
7458 				 * and there are no enabled luns which
7459 				 * are attached to the currently registered
7460 				 * scsi id.
7461 				 */
7462 				status = CAM_TID_INVALID;
7463 			}
7464 		} else if ((ahc->features & AHC_MULTI_TID) == 0
7465 			&& ahc->enabled_luns > 0) {
7466 
7467 			status = CAM_TID_INVALID;
7468 		}
7469 	}
7470 
7471 	if (status != CAM_REQ_CMP) {
7472 		ccb->ccb_h.status = status;
7473 		return;
7474 	}
7475 
7476 	/*
7477 	 * We now have an id that is valid.
7478 	 * If we aren't in target mode, switch modes.
7479 	 */
7480 	if ((ahc->flags & AHC_TARGETROLE) == 0
7481 	 && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
7482 		u_long	 s;
7483 		ahc_flag saved_flags;
7484 
7485 		printk("Configuring Target Mode\n");
7486 		ahc_lock(ahc, &s);
7487 		if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
7488 			ccb->ccb_h.status = CAM_BUSY;
7489 			ahc_unlock(ahc, &s);
7490 			return;
7491 		}
7492 		saved_flags = ahc->flags;
7493 		ahc->flags |= AHC_TARGETROLE;
7494 		if ((ahc->features & AHC_MULTIROLE) == 0)
7495 			ahc->flags &= ~AHC_INITIATORROLE;
7496 		ahc_pause(ahc);
7497 		error = ahc_loadseq(ahc);
7498 		if (error != 0) {
7499 			/*
7500 			 * Restore original configuration and notify
7501 			 * the caller that we cannot support target mode.
7502 			 * Since the adapter started out in this
7503 			 * configuration, the firmware load will succeed,
7504 			 * so there is no point in checking ahc_loadseq's
7505 			 * return value.
7506 			 */
7507 			ahc->flags = saved_flags;
7508 			(void)ahc_loadseq(ahc);
7509 			ahc_restart(ahc);
7510 			ahc_unlock(ahc, &s);
7511 			ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
7512 			return;
7513 		}
7514 		ahc_restart(ahc);
7515 		ahc_unlock(ahc, &s);
7516 	}
7517 	cel = &ccb->cel;
7518 	target = ccb->ccb_h.target_id;
7519 	lun = ccb->ccb_h.target_lun;
7520 	channel = SIM_CHANNEL(ahc, sim);
7521 	target_mask = 0x01 << target;
7522 	if (channel == 'B')
7523 		target_mask <<= 8;
7524 
7525 	if (cel->enable != 0) {
7526 		u_int scsiseq;
7527 
7528 		/* Are we already enabled?? */
7529 		if (lstate != NULL) {
7530 			xpt_print_path(ccb->ccb_h.path);
7531 			printk("Lun already enabled\n");
7532 			ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
7533 			return;
7534 		}
7535 
7536 		if (cel->grp6_len != 0
7537 		 || cel->grp7_len != 0) {
7538 			/*
7539 			 * Don't (yet?) support vendor
7540 			 * specific commands.
7541 			 */
7542 			ccb->ccb_h.status = CAM_REQ_INVALID;
7543 			printk("Non-zero Group Codes\n");
7544 			return;
7545 		}
7546 
7547 		/*
7548 		 * Seems to be okay.
7549 		 * Setup our data structures.
7550 		 */
7551 		if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
7552 			tstate = ahc_alloc_tstate(ahc, target, channel);
7553 			if (tstate == NULL) {
7554 				xpt_print_path(ccb->ccb_h.path);
7555 				printk("Couldn't allocate tstate\n");
7556 				ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7557 				return;
7558 			}
7559 		}
7560 		lstate = kzalloc(sizeof(*lstate), GFP_ATOMIC);
7561 		if (lstate == NULL) {
7562 			xpt_print_path(ccb->ccb_h.path);
7563 			printk("Couldn't allocate lstate\n");
7564 			ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7565 			return;
7566 		}
7567 		status = xpt_create_path(&lstate->path, /*periph*/NULL,
7568 					 xpt_path_path_id(ccb->ccb_h.path),
7569 					 xpt_path_target_id(ccb->ccb_h.path),
7570 					 xpt_path_lun_id(ccb->ccb_h.path));
7571 		if (status != CAM_REQ_CMP) {
7572 			kfree(lstate);
7573 			xpt_print_path(ccb->ccb_h.path);
7574 			printk("Couldn't allocate path\n");
7575 			ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7576 			return;
7577 		}
7578 		SLIST_INIT(&lstate->accept_tios);
7579 		SLIST_INIT(&lstate->immed_notifies);
7580 		ahc_lock(ahc, &s);
7581 		ahc_pause(ahc);
7582 		if (target != CAM_TARGET_WILDCARD) {
7583 			tstate->enabled_luns[lun] = lstate;
7584 			ahc->enabled_luns++;
7585 
7586 			if ((ahc->features & AHC_MULTI_TID) != 0) {
7587 				u_int targid_mask;
7588 
7589 				targid_mask = ahc_inb(ahc, TARGID)
7590 					    | (ahc_inb(ahc, TARGID + 1) << 8);
7591 
7592 				targid_mask |= target_mask;
7593 				ahc_outb(ahc, TARGID, targid_mask);
7594 				ahc_outb(ahc, TARGID+1, (targid_mask >> 8));
7595 
7596 				ahc_update_scsiid(ahc, targid_mask);
7597 			} else {
7598 				u_int our_id;
7599 				char  channel;
7600 
7601 				channel = SIM_CHANNEL(ahc, sim);
7602 				our_id = SIM_SCSI_ID(ahc, sim);
7603 
7604 				/*
7605 				 * This can only happen if selections
7606 				 * are not enabled
7607 				 */
7608 				if (target != our_id) {
7609 					u_int sblkctl;
7610 					char  cur_channel;
7611 					int   swap;
7612 
7613 					sblkctl = ahc_inb(ahc, SBLKCTL);
7614 					cur_channel = (sblkctl & SELBUSB)
7615 						    ? 'B' : 'A';
7616 					if ((ahc->features & AHC_TWIN) == 0)
7617 						cur_channel = 'A';
7618 					swap = cur_channel != channel;
7619 					if (channel == 'A')
7620 						ahc->our_id = target;
7621 					else
7622 						ahc->our_id_b = target;
7623 
7624 					if (swap)
7625 						ahc_outb(ahc, SBLKCTL,
7626 							 sblkctl ^ SELBUSB);
7627 
7628 					ahc_outb(ahc, SCSIID, target);
7629 
7630 					if (swap)
7631 						ahc_outb(ahc, SBLKCTL, sblkctl);
7632 				}
7633 			}
7634 		} else
7635 			ahc->black_hole = lstate;
7636 		/* Allow select-in operations */
7637 		if (ahc->black_hole != NULL && ahc->enabled_luns > 0) {
7638 			scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7639 			scsiseq |= ENSELI;
7640 			ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7641 			scsiseq = ahc_inb(ahc, SCSISEQ);
7642 			scsiseq |= ENSELI;
7643 			ahc_outb(ahc, SCSISEQ, scsiseq);
7644 		}
7645 		ahc_unpause(ahc);
7646 		ahc_unlock(ahc, &s);
7647 		ccb->ccb_h.status = CAM_REQ_CMP;
7648 		xpt_print_path(ccb->ccb_h.path);
7649 		printk("Lun now enabled for target mode\n");
7650 	} else {
7651 		struct scb *scb;
7652 		int i, empty;
7653 
7654 		if (lstate == NULL) {
7655 			ccb->ccb_h.status = CAM_LUN_INVALID;
7656 			return;
7657 		}
7658 
7659 		ahc_lock(ahc, &s);
7660 
7661 		ccb->ccb_h.status = CAM_REQ_CMP;
7662 		LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7663 			struct ccb_hdr *ccbh;
7664 
7665 			ccbh = &scb->io_ctx->ccb_h;
7666 			if (ccbh->func_code == XPT_CONT_TARGET_IO
7667 			 && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
7668 				printk("CTIO pending\n");
7669 				ccb->ccb_h.status = CAM_REQ_INVALID;
7670 				ahc_unlock(ahc, &s);
7671 				return;
7672 			}
7673 		}
7674 
7675 		if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
7676 			printk("ATIOs pending\n");
7677 			ccb->ccb_h.status = CAM_REQ_INVALID;
7678 		}
7679 
7680 		if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
7681 			printk("INOTs pending\n");
7682 			ccb->ccb_h.status = CAM_REQ_INVALID;
7683 		}
7684 
7685 		if (ccb->ccb_h.status != CAM_REQ_CMP) {
7686 			ahc_unlock(ahc, &s);
7687 			return;
7688 		}
7689 
7690 		xpt_print_path(ccb->ccb_h.path);
7691 		printk("Target mode disabled\n");
7692 		xpt_free_path(lstate->path);
7693 		kfree(lstate);
7694 
7695 		ahc_pause(ahc);
7696 		/* Can we clean up the target too? */
7697 		if (target != CAM_TARGET_WILDCARD) {
7698 			tstate->enabled_luns[lun] = NULL;
7699 			ahc->enabled_luns--;
7700 			for (empty = 1, i = 0; i < 8; i++)
7701 				if (tstate->enabled_luns[i] != NULL) {
7702 					empty = 0;
7703 					break;
7704 				}
7705 
7706 			if (empty) {
7707 				ahc_free_tstate(ahc, target, channel,
7708 						/*force*/FALSE);
7709 				if (ahc->features & AHC_MULTI_TID) {
7710 					u_int targid_mask;
7711 
7712 					targid_mask = ahc_inb(ahc, TARGID)
7713 						    | (ahc_inb(ahc, TARGID + 1)
7714 						       << 8);
7715 
7716 					targid_mask &= ~target_mask;
7717 					ahc_outb(ahc, TARGID, targid_mask);
7718 					ahc_outb(ahc, TARGID+1,
7719 					 	 (targid_mask >> 8));
7720 					ahc_update_scsiid(ahc, targid_mask);
7721 				}
7722 			}
7723 		} else {
7724 
7725 			ahc->black_hole = NULL;
7726 
7727 			/*
7728 			 * We can't allow selections without
7729 			 * our black hole device.
7730 			 */
7731 			empty = TRUE;
7732 		}
7733 		if (ahc->enabled_luns == 0) {
7734 			/* Disallow select-in */
7735 			u_int scsiseq;
7736 
7737 			scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7738 			scsiseq &= ~ENSELI;
7739 			ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7740 			scsiseq = ahc_inb(ahc, SCSISEQ);
7741 			scsiseq &= ~ENSELI;
7742 			ahc_outb(ahc, SCSISEQ, scsiseq);
7743 
7744 			if ((ahc->features & AHC_MULTIROLE) == 0) {
7745 				printk("Configuring Initiator Mode\n");
7746 				ahc->flags &= ~AHC_TARGETROLE;
7747 				ahc->flags |= AHC_INITIATORROLE;
7748 				/*
7749 				 * Returning to a configuration that
7750 				 * fit previously will always succeed.
7751 				 */
7752 				(void)ahc_loadseq(ahc);
7753 				ahc_restart(ahc);
7754 				/*
7755 				 * Unpaused.  The extra unpause
7756 				 * that follows is harmless.
7757 				 */
7758 			}
7759 		}
7760 		ahc_unpause(ahc);
7761 		ahc_unlock(ahc, &s);
7762 	}
7763 }
7764 
7765 static void
7766 ahc_update_scsiid(struct ahc_softc *ahc, u_int targid_mask)
7767 {
7768 	u_int scsiid_mask;
7769 	u_int scsiid;
7770 
7771 	if ((ahc->features & AHC_MULTI_TID) == 0)
7772 		panic("ahc_update_scsiid called on non-multitid unit\n");
7773 
7774 	/*
7775 	 * Since we will rely on the TARGID mask
7776 	 * for selection enables, ensure that OID
7777 	 * in SCSIID is not set to some other ID
7778 	 * that we don't want to allow selections on.
7779 	 */
7780 	if ((ahc->features & AHC_ULTRA2) != 0)
7781 		scsiid = ahc_inb(ahc, SCSIID_ULTRA2);
7782 	else
7783 		scsiid = ahc_inb(ahc, SCSIID);
7784 	scsiid_mask = 0x1 << (scsiid & OID);
7785 	if ((targid_mask & scsiid_mask) == 0) {
7786 		u_int our_id;
7787 
7788 		/* ffs counts from 1 */
7789 		our_id = ffs(targid_mask);
7790 		if (our_id == 0)
7791 			our_id = ahc->our_id;
7792 		else
7793 			our_id--;
7794 		scsiid &= TID;
7795 		scsiid |= our_id;
7796 	}
7797 	if ((ahc->features & AHC_ULTRA2) != 0)
7798 		ahc_outb(ahc, SCSIID_ULTRA2, scsiid);
7799 	else
7800 		ahc_outb(ahc, SCSIID, scsiid);
7801 }
7802 
7803 static void
7804 ahc_run_tqinfifo(struct ahc_softc *ahc, int paused)
7805 {
7806 	struct target_cmd *cmd;
7807 
7808 	/*
7809 	 * If the card supports auto-access pause,
7810 	 * we can access the card directly regardless
7811 	 * of whether it is paused or not.
7812 	 */
7813 	if ((ahc->features & AHC_AUTOPAUSE) != 0)
7814 		paused = TRUE;
7815 
7816 	ahc_sync_tqinfifo(ahc, BUS_DMASYNC_POSTREAD);
7817 	while ((cmd = &ahc->targetcmds[ahc->tqinfifonext])->cmd_valid != 0) {
7818 
7819 		/*
7820 		 * Only advance through the queue if we
7821 		 * have the resources to process the command.
7822 		 */
7823 		if (ahc_handle_target_cmd(ahc, cmd) != 0)
7824 			break;
7825 
7826 		cmd->cmd_valid = 0;
7827 		ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
7828 				ahc->shared_data_dmamap,
7829 				ahc_targetcmd_offset(ahc, ahc->tqinfifonext),
7830 				sizeof(struct target_cmd),
7831 				BUS_DMASYNC_PREREAD);
7832 		ahc->tqinfifonext++;
7833 
7834 		/*
7835 		 * Lazily update our position in the target mode incoming
7836 		 * command queue as seen by the sequencer.
7837 		 */
7838 		if ((ahc->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
7839 			if ((ahc->features & AHC_HS_MAILBOX) != 0) {
7840 				u_int hs_mailbox;
7841 
7842 				hs_mailbox = ahc_inb(ahc, HS_MAILBOX);
7843 				hs_mailbox &= ~HOST_TQINPOS;
7844 				hs_mailbox |= ahc->tqinfifonext & HOST_TQINPOS;
7845 				ahc_outb(ahc, HS_MAILBOX, hs_mailbox);
7846 			} else {
7847 				if (!paused)
7848 					ahc_pause(ahc);
7849 				ahc_outb(ahc, KERNEL_TQINPOS,
7850 					 ahc->tqinfifonext & HOST_TQINPOS);
7851 				if (!paused)
7852 					ahc_unpause(ahc);
7853 			}
7854 		}
7855 	}
7856 }
7857 
7858 static int
7859 ahc_handle_target_cmd(struct ahc_softc *ahc, struct target_cmd *cmd)
7860 {
7861 	struct	  ahc_tmode_tstate *tstate;
7862 	struct	  ahc_tmode_lstate *lstate;
7863 	struct	  ccb_accept_tio *atio;
7864 	uint8_t *byte;
7865 	int	  initiator;
7866 	int	  target;
7867 	int	  lun;
7868 
7869 	initiator = SCSIID_TARGET(ahc, cmd->scsiid);
7870 	target = SCSIID_OUR_ID(cmd->scsiid);
7871 	lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
7872 
7873 	byte = cmd->bytes;
7874 	tstate = ahc->enabled_targets[target];
7875 	lstate = NULL;
7876 	if (tstate != NULL)
7877 		lstate = tstate->enabled_luns[lun];
7878 
7879 	/*
7880 	 * Commands for disabled luns go to the black hole driver.
7881 	 */
7882 	if (lstate == NULL)
7883 		lstate = ahc->black_hole;
7884 
7885 	atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
7886 	if (atio == NULL) {
7887 		ahc->flags |= AHC_TQINFIFO_BLOCKED;
7888 		/*
7889 		 * Wait for more ATIOs from the peripheral driver for this lun.
7890 		 */
7891 		if (bootverbose)
7892 			printk("%s: ATIOs exhausted\n", ahc_name(ahc));
7893 		return (1);
7894 	} else
7895 		ahc->flags &= ~AHC_TQINFIFO_BLOCKED;
7896 #if 0
7897 	printk("Incoming command from %d for %d:%d%s\n",
7898 	       initiator, target, lun,
7899 	       lstate == ahc->black_hole ? "(Black Holed)" : "");
7900 #endif
7901 	SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
7902 
7903 	if (lstate == ahc->black_hole) {
7904 		/* Fill in the wildcards */
7905 		atio->ccb_h.target_id = target;
7906 		atio->ccb_h.target_lun = lun;
7907 	}
7908 
7909 	/*
7910 	 * Package it up and send it off to
7911 	 * whomever has this lun enabled.
7912 	 */
7913 	atio->sense_len = 0;
7914 	atio->init_id = initiator;
7915 	if (byte[0] != 0xFF) {
7916 		/* Tag was included */
7917 		atio->tag_action = *byte++;
7918 		atio->tag_id = *byte++;
7919 		atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
7920 	} else {
7921 		atio->ccb_h.flags = 0;
7922 	}
7923 	byte++;
7924 
7925 	/* Okay.  Now determine the cdb size based on the command code */
7926 	switch (*byte >> CMD_GROUP_CODE_SHIFT) {
7927 	case 0:
7928 		atio->cdb_len = 6;
7929 		break;
7930 	case 1:
7931 	case 2:
7932 		atio->cdb_len = 10;
7933 		break;
7934 	case 4:
7935 		atio->cdb_len = 16;
7936 		break;
7937 	case 5:
7938 		atio->cdb_len = 12;
7939 		break;
7940 	case 3:
7941 	default:
7942 		/* Only copy the opcode. */
7943 		atio->cdb_len = 1;
7944 		printk("Reserved or VU command code type encountered\n");
7945 		break;
7946 	}
7947 
7948 	memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
7949 
7950 	atio->ccb_h.status |= CAM_CDB_RECVD;
7951 
7952 	if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
7953 		/*
7954 		 * We weren't allowed to disconnect.
7955 		 * We're hanging on the bus until a
7956 		 * continue target I/O comes in response
7957 		 * to this accept tio.
7958 		 */
7959 #if 0
7960 		printk("Received Immediate Command %d:%d:%d - %p\n",
7961 		       initiator, target, lun, ahc->pending_device);
7962 #endif
7963 		ahc->pending_device = lstate;
7964 		ahc_freeze_ccb((union ccb *)atio);
7965 		atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
7966 	}
7967 	xpt_done((union ccb*)atio);
7968 	return (0);
7969 }
7970 
7971 #endif
7972