2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2007 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
24 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
36 #include "ioatdma_registers.h"
37 #include "ioatdma_hw.h"
39 #define INITIAL_IOAT_DESC_COUNT 128
41 #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
42 #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
43 #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
44 #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
46 /* internal functions */
47 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
48 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
50 static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
55 struct ioat_dma_chan *ioat_chan;
57 device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
58 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
59 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
61 for (i = 0; i < device->common.chancnt; i++) {
62 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
64 device->common.chancnt = i;
68 ioat_chan->device = device;
69 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
70 ioat_chan->xfercap = xfercap;
71 spin_lock_init(&ioat_chan->cleanup_lock);
72 spin_lock_init(&ioat_chan->desc_lock);
73 INIT_LIST_HEAD(&ioat_chan->free_desc);
74 INIT_LIST_HEAD(&ioat_chan->used_desc);
75 /* This should be made common somewhere in dmaengine.c */
76 ioat_chan->common.device = &device->common;
77 list_add_tail(&ioat_chan->common.device_node,
78 &device->common.channels);
80 return device->common.chancnt;
83 static void ioat_set_src(dma_addr_t addr,
84 struct dma_async_tx_descriptor *tx,
87 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
88 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
90 pci_unmap_addr_set(desc, src, addr);
92 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
93 iter->hw->src_addr = addr;
94 addr += ioat_chan->xfercap;
99 static void ioat_set_dest(dma_addr_t addr,
100 struct dma_async_tx_descriptor *tx,
103 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
104 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
106 pci_unmap_addr_set(desc, dst, addr);
108 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
109 iter->hw->dst_addr = addr;
110 addr += ioat_chan->xfercap;
114 static dma_cookie_t ioat_tx_submit(struct dma_async_tx_descriptor *tx)
116 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
117 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
120 struct ioat_desc_sw *group_start;
122 group_start = list_entry(desc->async_tx.tx_list.next,
123 struct ioat_desc_sw, node);
124 spin_lock_bh(&ioat_chan->desc_lock);
125 /* cookie incr and addition to used_list must be atomic */
126 cookie = ioat_chan->common.cookie;
130 ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
132 /* write address into NextDescriptor field of last desc in chain */
133 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
134 group_start->async_tx.phys;
135 list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
137 ioat_chan->pending += desc->tx_cnt;
138 if (ioat_chan->pending >= 4) {
140 ioat_chan->pending = 0;
142 spin_unlock_bh(&ioat_chan->desc_lock);
145 writeb(IOAT_CHANCMD_APPEND,
146 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
151 static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
152 struct ioat_dma_chan *ioat_chan,
155 struct ioat_dma_descriptor *desc;
156 struct ioat_desc_sw *desc_sw;
157 struct ioatdma_device *ioatdma_device;
160 ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
161 desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
165 desc_sw = kzalloc(sizeof(*desc_sw), flags);
166 if (unlikely(!desc_sw)) {
167 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
171 memset(desc, 0, sizeof(*desc));
172 dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
173 desc_sw->async_tx.tx_set_src = ioat_set_src;
174 desc_sw->async_tx.tx_set_dest = ioat_set_dest;
175 desc_sw->async_tx.tx_submit = ioat_tx_submit;
176 INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
178 desc_sw->async_tx.phys = phys;
183 /* returns the actual number of allocated descriptors */
184 static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
186 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
187 struct ioat_desc_sw *desc = NULL;
193 /* have we already been set up? */
194 if (!list_empty(&ioat_chan->free_desc))
195 return INITIAL_IOAT_DESC_COUNT;
197 /* Setup register to interrupt and write completion status on error */
198 chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
199 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
200 IOAT_CHANCTRL_ERR_COMPLETION_EN;
201 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
203 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
205 dev_err(&ioat_chan->device->pdev->dev,
206 "ioatdma: CHANERR = %x, clearing\n", chanerr);
207 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
210 /* Allocate descriptors */
211 for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
212 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
214 dev_err(&ioat_chan->device->pdev->dev,
215 "ioatdma: Only %d initial descriptors\n", i);
218 list_add_tail(&desc->node, &tmp_list);
220 spin_lock_bh(&ioat_chan->desc_lock);
221 list_splice(&tmp_list, &ioat_chan->free_desc);
222 spin_unlock_bh(&ioat_chan->desc_lock);
224 /* allocate a completion writeback area */
225 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
226 ioat_chan->completion_virt =
227 pci_pool_alloc(ioat_chan->device->completion_pool,
229 &ioat_chan->completion_addr);
230 memset(ioat_chan->completion_virt, 0,
231 sizeof(*ioat_chan->completion_virt));
232 writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
233 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
234 writel(((u64) ioat_chan->completion_addr) >> 32,
235 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
237 ioat_dma_start_null_desc(ioat_chan);
241 static void ioat_dma_free_chan_resources(struct dma_chan *chan)
243 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
244 struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);
245 struct ioat_desc_sw *desc, *_desc;
246 int in_use_descs = 0;
248 ioat_dma_memcpy_cleanup(ioat_chan);
250 writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
252 spin_lock_bh(&ioat_chan->desc_lock);
253 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
255 list_del(&desc->node);
256 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
257 desc->async_tx.phys);
260 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
261 list_del(&desc->node);
262 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
263 desc->async_tx.phys);
266 spin_unlock_bh(&ioat_chan->desc_lock);
268 pci_pool_free(ioatdma_device->completion_pool,
269 ioat_chan->completion_virt,
270 ioat_chan->completion_addr);
272 /* one is ok since we left it on there on purpose */
273 if (in_use_descs > 1)
274 dev_err(&ioat_chan->device->pdev->dev,
275 "ioatdma: Freeing %d in use descriptors!\n",
278 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
281 static struct dma_async_tx_descriptor *ioat_dma_prep_memcpy(
282 struct dma_chan *chan,
286 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
287 struct ioat_desc_sw *first, *prev, *new;
288 LIST_HEAD(new_chain);
301 spin_lock_bh(&ioat_chan->desc_lock);
303 if (!list_empty(&ioat_chan->free_desc)) {
304 new = to_ioat_desc(ioat_chan->free_desc.next);
305 list_del(&new->node);
307 /* try to get another desc */
308 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
309 /* will this ever happen? */
310 /* TODO add upper limit on these */
314 copy = min((u32) len, ioat_chan->xfercap);
316 new->hw->size = copy;
318 new->async_tx.cookie = 0;
319 new->async_tx.ack = 1;
321 /* chain together the physical address list for the HW */
325 prev->hw->next = (u64) new->async_tx.phys;
329 list_add_tail(&new->node, &new_chain);
333 list_splice(&new_chain, &new->async_tx.tx_list);
335 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
337 new->tx_cnt = desc_count;
338 new->async_tx.ack = 0; /* client is in control of this ack */
339 new->async_tx.cookie = -EBUSY;
341 pci_unmap_len_set(new, len, orig_len);
342 spin_unlock_bh(&ioat_chan->desc_lock);
344 return new ? &new->async_tx : NULL;
348 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
350 * @chan: DMA channel handle
352 static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
354 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
356 if (ioat_chan->pending != 0) {
357 ioat_chan->pending = 0;
358 writeb(IOAT_CHANCMD_APPEND,
359 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
363 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)
365 unsigned long phys_complete;
366 struct ioat_desc_sw *desc, *_desc;
367 dma_cookie_t cookie = 0;
369 prefetch(ioat_chan->completion_virt);
371 if (!spin_trylock(&ioat_chan->cleanup_lock))
374 /* The completion writeback can happen at any time,
375 so reads by the driver need to be atomic operations
376 The descriptor physical addresses are limited to 32-bits
377 when the CPU can only do a 32-bit mov */
379 #if (BITS_PER_LONG == 64)
381 ioat_chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
383 phys_complete = ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
386 if ((ioat_chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
387 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
388 dev_err(&ioat_chan->device->pdev->dev,
389 "ioatdma: Channel halted, chanerr = %x\n",
390 readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET));
392 /* TODO do something to salvage the situation */
395 if (phys_complete == ioat_chan->last_completion) {
396 spin_unlock(&ioat_chan->cleanup_lock);
400 spin_lock_bh(&ioat_chan->desc_lock);
401 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
404 * Incoming DMA requests may use multiple descriptors, due to
405 * exceeding xfercap, perhaps. If so, only the last one will
406 * have a cookie, and require unmapping.
408 if (desc->async_tx.cookie) {
409 cookie = desc->async_tx.cookie;
412 * yes we are unmapping both _page and _single alloc'd
413 * regions with unmap_page. Is this *really* that bad?
415 pci_unmap_page(ioat_chan->device->pdev,
416 pci_unmap_addr(desc, dst),
417 pci_unmap_len(desc, len),
419 pci_unmap_page(ioat_chan->device->pdev,
420 pci_unmap_addr(desc, src),
421 pci_unmap_len(desc, len),
425 if (desc->async_tx.phys != phys_complete) {
427 * a completed entry, but not the last, so cleanup
428 * if the client is done with the descriptor
430 if (desc->async_tx.ack) {
431 list_del(&desc->node);
432 list_add_tail(&desc->node,
433 &ioat_chan->free_desc);
435 desc->async_tx.cookie = 0;
438 * last used desc. Do not remove, so we can append from
439 * it, but don't look at it next time, either
441 desc->async_tx.cookie = 0;
443 /* TODO check status bits? */
448 spin_unlock_bh(&ioat_chan->desc_lock);
450 ioat_chan->last_completion = phys_complete;
452 ioat_chan->completed_cookie = cookie;
454 spin_unlock(&ioat_chan->cleanup_lock);
457 static void ioat_dma_dependency_added(struct dma_chan *chan)
459 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
460 spin_lock_bh(&ioat_chan->desc_lock);
461 if (ioat_chan->pending == 0) {
462 spin_unlock_bh(&ioat_chan->desc_lock);
463 ioat_dma_memcpy_cleanup(ioat_chan);
465 spin_unlock_bh(&ioat_chan->desc_lock);
469 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
470 * @chan: IOAT DMA channel handle
471 * @cookie: DMA transaction identifier
472 * @done: if not %NULL, updated with last completed transaction
473 * @used: if not %NULL, updated with last used transaction
475 static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
480 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
481 dma_cookie_t last_used;
482 dma_cookie_t last_complete;
485 last_used = chan->cookie;
486 last_complete = ioat_chan->completed_cookie;
489 *done = last_complete;
493 ret = dma_async_is_complete(cookie, last_complete, last_used);
494 if (ret == DMA_SUCCESS)
497 ioat_dma_memcpy_cleanup(ioat_chan);
499 last_used = chan->cookie;
500 last_complete = ioat_chan->completed_cookie;
503 *done = last_complete;
507 return dma_async_is_complete(cookie, last_complete, last_used);
512 static irqreturn_t ioat_do_interrupt(int irq, void *data)
514 struct ioatdma_device *instance = data;
515 unsigned long attnstatus;
518 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
520 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
523 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
524 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
528 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
530 printk(KERN_ERR "ioatdma: interrupt! status %lx\n", attnstatus);
532 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
536 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)
538 struct ioat_desc_sw *desc;
540 spin_lock_bh(&ioat_chan->desc_lock);
542 if (!list_empty(&ioat_chan->free_desc)) {
543 desc = to_ioat_desc(ioat_chan->free_desc.next);
544 list_del(&desc->node);
546 /* try to get another desc */
547 spin_unlock_bh(&ioat_chan->desc_lock);
548 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
549 spin_lock_bh(&ioat_chan->desc_lock);
550 /* will this ever happen? */
554 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
556 desc->async_tx.ack = 1;
558 list_add_tail(&desc->node, &ioat_chan->used_desc);
559 spin_unlock_bh(&ioat_chan->desc_lock);
561 writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
562 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
563 writel(((u64) desc->async_tx.phys) >> 32,
564 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
566 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
570 * Perform a IOAT transaction to verify the HW works.
572 #define IOAT_TEST_SIZE 2000
574 static int ioat_self_test(struct ioatdma_device *device)
579 struct dma_chan *dma_chan;
580 struct dma_async_tx_descriptor *tx;
585 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
588 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
594 /* Fill in src buffer */
595 for (i = 0; i < IOAT_TEST_SIZE; i++)
598 /* Start copy, using first DMA channel */
599 dma_chan = container_of(device->common.channels.next,
602 if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
603 dev_err(&device->pdev->dev,
604 "selftest cannot allocate chan resource\n");
609 tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
611 addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
613 ioat_set_src(addr, tx, 0);
614 addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
616 ioat_set_dest(addr, tx, 0);
617 cookie = ioat_tx_submit(tx);
618 ioat_dma_memcpy_issue_pending(dma_chan);
621 if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
622 dev_err(&device->pdev->dev,
623 "ioatdma: Self-test copy timed out, disabling\n");
627 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
628 dev_err(&device->pdev->dev,
629 "ioatdma: Self-test copy failed compare, disabling\n");
635 ioat_dma_free_chan_resources(dma_chan);
642 struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
643 void __iomem *iobase)
646 struct ioatdma_device *device;
648 device = kzalloc(sizeof(*device), GFP_KERNEL);
654 device->reg_base = iobase;
655 device->version = readb(device->reg_base + IOAT_VER_OFFSET);
657 /* DMA coherent memory pool for DMA descriptor allocations */
658 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
659 sizeof(struct ioat_dma_descriptor),
661 if (!device->dma_pool) {
666 device->completion_pool = pci_pool_create("completion_pool", pdev,
667 sizeof(u64), SMP_CACHE_BYTES,
669 if (!device->completion_pool) {
671 goto err_completion_pool;
674 INIT_LIST_HEAD(&device->common.channels);
675 ioat_dma_enumerate_channels(device);
677 dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
678 device->common.device_alloc_chan_resources =
679 ioat_dma_alloc_chan_resources;
680 device->common.device_free_chan_resources =
681 ioat_dma_free_chan_resources;
682 device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
683 device->common.device_is_tx_complete = ioat_dma_is_complete;
684 device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
685 device->common.device_dependency_added = ioat_dma_dependency_added;
686 device->common.dev = &pdev->dev;
687 printk(KERN_INFO "ioatdma: Intel(R) I/OAT DMA Engine found,"
688 " %d channels, device version 0x%02x\n",
689 device->common.chancnt, device->version);
691 pci_set_drvdata(pdev, device);
692 err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat",
697 writeb(IOAT_INTRCTRL_MASTER_INT_EN,
698 device->reg_base + IOAT_INTRCTRL_OFFSET);
699 pci_set_master(pdev);
701 err = ioat_self_test(device);
705 dma_async_device_register(&device->common);
710 free_irq(device->pdev->irq, device);
712 pci_pool_destroy(device->completion_pool);
714 pci_pool_destroy(device->dma_pool);
720 "ioatdma: Intel(R) I/OAT DMA Engine initialization failed\n");
724 void ioat_dma_remove(struct ioatdma_device *device)
726 struct dma_chan *chan, *_chan;
727 struct ioat_dma_chan *ioat_chan;
729 dma_async_device_unregister(&device->common);
731 free_irq(device->pdev->irq, device);
733 pci_pool_destroy(device->dma_pool);
734 pci_pool_destroy(device->completion_pool);
736 list_for_each_entry_safe(chan, _chan,
737 &device->common.channels, device_node) {
738 ioat_chan = to_ioat_chan(chan);
739 list_del(&chan->device_node);