atmel_spi: chain DMA transfers

Add support for chained transfers in the atmel_spi driver, letting the DMA
controller switch to the next buffer pair without CPU intervention.  This
reduced I/O latencies by about 2% in one bulk I/O test.  It should also help
work around several interrelated errata affecting chipselect 0 on at91rm9200
chips.

Almost all of the changes are in the reworked atmel_spi_next_xfer() function.
That's now called with the driver in one of three states:

 1. It isn't transferring anything (in which case the first transfer
    of the current message is going to be sent)
 2. It has finished transfering a non-chainable transfer (in which
    case it will go to the next transfer in the message)
 3. It has finished transfering a chained transfer (in which case the
    next transfer is already queued)

After that it will queue the next transfer if it can be chained.

Signed-off-by: Szilveszter Ordog <slipszi@gmail.com>
Acked-by: Haavard Skinnemoen <hskinnemoen@atmel.com>
Signed-off-by: David Brownell <dbrownell@users.sourceforge.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/drivers/spi/atmel_spi.c b/drivers/spi/atmel_spi.c
index b09d33678dd8fc36c9edcc3c9e9773ba4c1f0d41..545519a063b803ab80936397c4f44134b6ac46c3 100644 (file)
--- a/drivers/spi/atmel_spi.c
+++ b/drivers/spi/atmel_spi.c
@@ -51,7 +51,9 @@ struct atmel_spi {
        u8                      stopping;
        struct list_head        queue;
        struct spi_transfer     *current_transfer;
-       unsigned long           remaining_bytes;
+       unsigned long           current_remaining_bytes;
+       struct spi_transfer     *next_transfer;
+       unsigned long           next_remaining_bytes;
 
        void                    *buffer;
        dma_addr_t              buffer_dma;
@@ -121,6 +123,48 @@ static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)
                gpio_set_value(gpio, !active);
 }
 
+static inline int atmel_spi_xfer_is_last(struct spi_message *msg,
+                                       struct spi_transfer *xfer)
+{
+       return msg->transfers.prev == &xfer->transfer_list;
+}
+
+static inline int atmel_spi_xfer_can_be_chained(struct spi_transfer *xfer)
+{
+       return xfer->delay_usecs == 0 && !xfer->cs_change;
+}
+
+static void atmel_spi_next_xfer_data(struct spi_master *master,
+                               struct spi_transfer *xfer,
+                               dma_addr_t *tx_dma,
+                               dma_addr_t *rx_dma,
+                               u32 *plen)
+{
+       struct atmel_spi        *as = spi_master_get_devdata(master);
+       u32                     len = *plen;
+
+       /* use scratch buffer only when rx or tx data is unspecified */
+       if (xfer->rx_buf)
+               *rx_dma = xfer->rx_dma + xfer->len - len;
+       else {
+               *rx_dma = as->buffer_dma;
+               if (len > BUFFER_SIZE)
+                       len = BUFFER_SIZE;
+       }
+       if (xfer->tx_buf)
+               *tx_dma = xfer->tx_dma + xfer->len - len;
+       else {
+               *tx_dma = as->buffer_dma;
+               if (len > BUFFER_SIZE)
+                       len = BUFFER_SIZE;
+               memset(as->buffer, 0, len);
+               dma_sync_single_for_device(&as->pdev->dev,
+                               as->buffer_dma, len, DMA_TO_DEVICE);
+       }
+
+       *plen = len;
+}
+
 /*
  * Submit next transfer for DMA.
  * lock is held, spi irq is blocked
@@ -130,53 +174,68 @@ static void atmel_spi_next_xfer(struct spi_master *master,
 {
        struct atmel_spi        *as = spi_master_get_devdata(master);
        struct spi_transfer     *xfer;
-       u32                     len;
+       u32                     len, remaining, total;
        dma_addr_t              tx_dma, rx_dma;
 
-       xfer = as->current_transfer;
-       if (!xfer || as->remaining_bytes == 0) {
-               if (xfer)
-                       xfer = list_entry(xfer->transfer_list.next,
-                                       struct spi_transfer, transfer_list);
-               else
-                       xfer = list_entry(msg->transfers.next,
-                                       struct spi_transfer, transfer_list);
-               as->remaining_bytes = xfer->len;
-               as->current_transfer = xfer;
-       }
+       if (!as->current_transfer)
+               xfer = list_entry(msg->transfers.next,
+                               struct spi_transfer, transfer_list);
+       else if (!as->next_transfer)
+               xfer = list_entry(as->current_transfer->transfer_list.next,
+                               struct spi_transfer, transfer_list);
+       else
+               xfer = NULL;
 
-       len = as->remaining_bytes;
+       if (xfer) {
+               len = xfer->len;
+               atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);
+               remaining = xfer->len - len;
 
-       tx_dma = xfer->tx_dma + xfer->len - len;
-       rx_dma = xfer->rx_dma + xfer->len - len;
+               spi_writel(as, RPR, rx_dma);
+               spi_writel(as, TPR, tx_dma);
 
-       /* use scratch buffer only when rx or tx data is unspecified */
-       if (!xfer->rx_buf) {
-               rx_dma = as->buffer_dma;
-               if (len > BUFFER_SIZE)
-                       len = BUFFER_SIZE;
-       }
-       if (!xfer->tx_buf) {
-               tx_dma = as->buffer_dma;
-               if (len > BUFFER_SIZE)
-                       len = BUFFER_SIZE;
-               memset(as->buffer, 0, len);
-               dma_sync_single_for_device(&as->pdev->dev,
-                               as->buffer_dma, len, DMA_TO_DEVICE);
+               if (msg->spi->bits_per_word > 8)
+                       len >>= 1;
+               spi_writel(as, RCR, len);
+               spi_writel(as, TCR, len);
+       } else {
+               xfer = as->next_transfer;
+               remaining = as->next_remaining_bytes;
        }
 
-       spi_writel(as, RPR, rx_dma);
-       spi_writel(as, TPR, tx_dma);
+       as->current_transfer = xfer;
+       as->current_remaining_bytes = remaining;
 
-       as->remaining_bytes -= len;
-       if (msg->spi->bits_per_word > 8)
-               len >>= 1;
+       if (remaining > 0)
+               len = remaining;
+       else if (!atmel_spi_xfer_is_last(msg, xfer) &&
+               atmel_spi_xfer_can_be_chained(xfer)) {
+               xfer = list_entry(xfer->transfer_list.next,
+                               struct spi_transfer, transfer_list);
+               len = xfer->len;
+       } else
+               xfer = NULL;
 
-       /* REVISIT: when xfer->delay_usecs == 0, the PDC "next transfer"
-        * mechanism might help avoid the IRQ latency between transfers
-        * (and improve the nCS0 errata handling on at91rm9200 chips)
-        *
-        * We're also waiting for ENDRX before we start the next
+       as->next_transfer = xfer;
+
+       if (xfer) {
+               total = len;
+               atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);
+               as->next_remaining_bytes = total - len;
+
+               spi_writel(as, RNPR, rx_dma);
+               spi_writel(as, TNPR, tx_dma);
+
+               if (msg->spi->bits_per_word > 8)
+                       len >>= 1;
+               spi_writel(as, RNCR, len);
+               spi_writel(as, TNCR, len);
+       } else {
+               spi_writel(as, RNCR, 0);
+               spi_writel(as, TNCR, 0);
+       }
+
+       /* REVISIT: We're waiting for ENDRX before we start the next
         * transfer because we need to handle some difficult timing
         * issues otherwise. If we wait for ENDTX in one transfer and
         * then starts waiting for ENDRX in the next, it's difficult
@@ -186,8 +245,6 @@ static void atmel_spi_next_xfer(struct spi_master *master,
         *
         * It should be doable, though. Just not now...
         */
-       spi_writel(as, TNCR, 0);
-       spi_writel(as, RNCR, 0);
        spi_writel(as, IER, SPI_BIT(ENDRX) | SPI_BIT(OVRES));
 
        dev_dbg(&msg->spi->dev,
@@ -195,8 +252,6 @@ static void atmel_spi_next_xfer(struct spi_master *master,
                xfer, xfer->len, xfer->tx_buf, xfer->tx_dma,
                xfer->rx_buf, xfer->rx_dma, spi_readl(as, IMR));
 
-       spi_writel(as, RCR, len);
-       spi_writel(as, TCR, len);
        spi_writel(as, PTCR, SPI_BIT(TXTEN) | SPI_BIT(RXTEN));
 }
 
@@ -294,6 +349,7 @@ atmel_spi_msg_done(struct spi_master *master, struct atmel_spi *as,
        spin_lock(&as->lock);
 
        as->current_transfer = NULL;
+       as->next_transfer = NULL;
 
        /* continue if needed */
        if (list_empty(&as->queue) || as->stopping)
@@ -377,7 +433,7 @@ atmel_spi_interrupt(int irq, void *dev_id)
 
                spi_writel(as, IDR, pending);
 
-               if (as->remaining_bytes == 0) {
+               if (as->current_remaining_bytes == 0) {
                        msg->actual_length += xfer->len;
 
                        if (!msg->is_dma_mapped)
@@ -387,7 +443,7 @@ atmel_spi_interrupt(int irq, void *dev_id)
                        if (xfer->delay_usecs)
                                udelay(xfer->delay_usecs);
 
-                       if (msg->transfers.prev == &xfer->transfer_list) {
+                       if (atmel_spi_xfer_is_last(msg, xfer)) {
                                /* report completed message */
                                atmel_spi_msg_done(master, as, msg, 0,
                                                xfer->cs_change);