-static __inline uint16_t
-ahd_inw(struct ahd_softc *ahd, u_int port)
-{
- /*
- * Read high byte first as some registers increment
- * or have other side effects when the low byte is
- * read.
- */
- uint16_t r = ahd_inb(ahd, port+1) << 8;
- return r | ahd_inb(ahd, port);
-}
-
-static __inline void
-ahd_outw(struct ahd_softc *ahd, u_int port, u_int value)
-{
- /*
- * Write low byte first to accomodate registers
- * such as PRGMCNT where the order maters.
- */
- ahd_outb(ahd, port, value & 0xFF);
- ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
-}
-
-static __inline uint32_t
-ahd_inl(struct ahd_softc *ahd, u_int port)
-{
- return ((ahd_inb(ahd, port))
- | (ahd_inb(ahd, port+1) << 8)
- | (ahd_inb(ahd, port+2) << 16)
- | (ahd_inb(ahd, port+3) << 24));
-}
-
-static __inline void
-ahd_outl(struct ahd_softc *ahd, u_int port, uint32_t value)
-{
- ahd_outb(ahd, port, (value) & 0xFF);
- ahd_outb(ahd, port+1, ((value) >> 8) & 0xFF);
- ahd_outb(ahd, port+2, ((value) >> 16) & 0xFF);
- ahd_outb(ahd, port+3, ((value) >> 24) & 0xFF);
-}
-
-static __inline uint64_t
-ahd_inq(struct ahd_softc *ahd, u_int port)
-{
- return ((ahd_inb(ahd, port))
- | (ahd_inb(ahd, port+1) << 8)
- | (ahd_inb(ahd, port+2) << 16)
- | (ahd_inb(ahd, port+3) << 24)
- | (((uint64_t)ahd_inb(ahd, port+4)) << 32)
- | (((uint64_t)ahd_inb(ahd, port+5)) << 40)
- | (((uint64_t)ahd_inb(ahd, port+6)) << 48)
- | (((uint64_t)ahd_inb(ahd, port+7)) << 56));
-}
-
-static __inline void
-ahd_outq(struct ahd_softc *ahd, u_int port, uint64_t value)
-{
- ahd_outb(ahd, port, value & 0xFF);
- ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
- ahd_outb(ahd, port+2, (value >> 16) & 0xFF);
- ahd_outb(ahd, port+3, (value >> 24) & 0xFF);
- ahd_outb(ahd, port+4, (value >> 32) & 0xFF);
- ahd_outb(ahd, port+5, (value >> 40) & 0xFF);
- ahd_outb(ahd, port+6, (value >> 48) & 0xFF);
- ahd_outb(ahd, port+7, (value >> 56) & 0xFF);
-}
-
-static __inline u_int
-ahd_get_scbptr(struct ahd_softc *ahd)
-{
- AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
- ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
- return (ahd_inb(ahd, SCBPTR) | (ahd_inb(ahd, SCBPTR + 1) << 8));
-}
-
-static __inline void
-ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr)
-{
- AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
- ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
- ahd_outb(ahd, SCBPTR, scbptr & 0xFF);
- ahd_outb(ahd, SCBPTR+1, (scbptr >> 8) & 0xFF);
-}
-
-static __inline u_int
-ahd_get_hnscb_qoff(struct ahd_softc *ahd)
-{
- return (ahd_inw_atomic(ahd, HNSCB_QOFF));
-}
-
-static __inline void
-ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value)
-{
- ahd_outw_atomic(ahd, HNSCB_QOFF, value);
-}
-
-static __inline u_int
-ahd_get_hescb_qoff(struct ahd_softc *ahd)
-{
- return (ahd_inb(ahd, HESCB_QOFF));
-}
-
-static __inline void
-ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value)
-{
- ahd_outb(ahd, HESCB_QOFF, value);
-}
-
-static __inline u_int
-ahd_get_snscb_qoff(struct ahd_softc *ahd)
-{
- u_int oldvalue;
-
- AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
- oldvalue = ahd_inw(ahd, SNSCB_QOFF);
- ahd_outw(ahd, SNSCB_QOFF, oldvalue);
- return (oldvalue);
-}
-
-static __inline void
-ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value)
-{
- AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
- ahd_outw(ahd, SNSCB_QOFF, value);
-}
-
-static __inline u_int
-ahd_get_sescb_qoff(struct ahd_softc *ahd)
-{
- AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
- return (ahd_inb(ahd, SESCB_QOFF));
-}
-
-static __inline void
-ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value)
-{
- AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
- ahd_outb(ahd, SESCB_QOFF, value);
-}
-
-static __inline u_int
-ahd_get_sdscb_qoff(struct ahd_softc *ahd)
-{
- AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
- return (ahd_inb(ahd, SDSCB_QOFF) | (ahd_inb(ahd, SDSCB_QOFF + 1) << 8));
-}
-
-static __inline void
-ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value)
-{
- AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
- ahd_outb(ahd, SDSCB_QOFF, value & 0xFF);
- ahd_outb(ahd, SDSCB_QOFF+1, (value >> 8) & 0xFF);
-}
-
-static __inline u_int
-ahd_inb_scbram(struct ahd_softc *ahd, u_int offset)
-{
- u_int value;
-
- /*
- * Workaround PCI-X Rev A. hardware bug.
- * After a host read of SCB memory, the chip
- * may become confused into thinking prefetch
- * was required. This starts the discard timer
- * running and can cause an unexpected discard
- * timer interrupt. The work around is to read
- * a normal register prior to the exhaustion of
- * the discard timer. The mode pointer register
- * has no side effects and so serves well for
- * this purpose.
- *
- * Razor #528
- */
- value = ahd_inb(ahd, offset);
- if ((ahd->bugs & AHD_PCIX_SCBRAM_RD_BUG) != 0)
- ahd_inb(ahd, MODE_PTR);
- return (value);
-}
-
-static __inline u_int
-ahd_inw_scbram(struct ahd_softc *ahd, u_int offset)
-{
- return (ahd_inb_scbram(ahd, offset)
- | (ahd_inb_scbram(ahd, offset+1) << 8));
-}
-
-static __inline uint32_t
-ahd_inl_scbram(struct ahd_softc *ahd, u_int offset)
-{
- return (ahd_inw_scbram(ahd, offset)
- | (ahd_inw_scbram(ahd, offset+2) << 16));
-}
-
-static __inline uint64_t
-ahd_inq_scbram(struct ahd_softc *ahd, u_int offset)
-{
- return (ahd_inl_scbram(ahd, offset)
- | ((uint64_t)ahd_inl_scbram(ahd, offset+4)) << 32);
-}
-
-static __inline struct scb *
-ahd_lookup_scb(struct ahd_softc *ahd, u_int tag)
-{
- struct scb* scb;
-
- if (tag >= AHD_SCB_MAX)
- return (NULL);
- scb = ahd->scb_data.scbindex[tag];
- if (scb != NULL)
- ahd_sync_scb(ahd, scb,
- BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
- return (scb);
-}
-
-static __inline void
-ahd_swap_with_next_hscb(struct ahd_softc *ahd, struct scb *scb)
-{
- struct hardware_scb *q_hscb;
- struct map_node *q_hscb_map;
- uint32_t saved_hscb_busaddr;
-
- /*
- * Our queuing method is a bit tricky. The card
- * knows in advance which HSCB (by address) to download,
- * and we can't disappoint it. To achieve this, the next
- * HSCB to download is saved off in ahd->next_queued_hscb.
- * When we are called to queue "an arbitrary scb",
- * we copy the contents of the incoming HSCB to the one
- * the sequencer knows about, swap HSCB pointers and
- * finally assign the SCB to the tag indexed location
- * in the scb_array. This makes sure that we can still
- * locate the correct SCB by SCB_TAG.
- */
- q_hscb = ahd->next_queued_hscb;
- q_hscb_map = ahd->next_queued_hscb_map;
- saved_hscb_busaddr = q_hscb->hscb_busaddr;
- memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
- q_hscb->hscb_busaddr = saved_hscb_busaddr;
- q_hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
-
- /* Now swap HSCB pointers. */
- ahd->next_queued_hscb = scb->hscb;
- ahd->next_queued_hscb_map = scb->hscb_map;
- scb->hscb = q_hscb;
- scb->hscb_map = q_hscb_map;
-
- /* Now define the mapping from tag to SCB in the scbindex */
- ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = scb;
-}
-
-/*
- * Tell the sequencer about a new transaction to execute.
- */
-static __inline void
-ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb)
-{
- ahd_swap_with_next_hscb(ahd, scb);
-
- if (SCBID_IS_NULL(SCB_GET_TAG(scb)))
- panic("Attempt to queue invalid SCB tag %x\n",
- SCB_GET_TAG(scb));
-
- /*
- * Keep a history of SCBs we've downloaded in the qinfifo.
- */
- ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
- ahd->qinfifonext++;
-
- if (scb->sg_count != 0)
- ahd_setup_data_scb(ahd, scb);
- else
- ahd_setup_noxfer_scb(ahd, scb);
- ahd_setup_scb_common(ahd, scb);
-
- /*
- * Make sure our data is consistent from the
- * perspective of the adapter.
- */
- ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
-
-#ifdef AHD_DEBUG
- if ((ahd_debug & AHD_SHOW_QUEUE) != 0) {
- uint64_t host_dataptr;
-
- host_dataptr = ahd_le64toh(scb->hscb->dataptr);
- printf("%s: Queueing SCB %d:0x%x bus addr 0x%x - 0x%x%x/0x%x\n",
- ahd_name(ahd),
- SCB_GET_TAG(scb), scb->hscb->scsiid,
- ahd_le32toh(scb->hscb->hscb_busaddr),
- (u_int)((host_dataptr >> 32) & 0xFFFFFFFF),
- (u_int)(host_dataptr & 0xFFFFFFFF),
- ahd_le32toh(scb->hscb->datacnt));
- }