2 * Dynamic DMA mapping support.
5 #include <linux/types.h>
7 #include <linux/string.h>
9 #include <linux/module.h>
10 #include <linux/dmar.h>
11 #include <linux/bootmem.h>
12 #include <asm/proto.h>
15 #include <asm/calgary.h>
17 int iommu_merge __read_mostly = 0;
19 dma_addr_t bad_dma_address __read_mostly;
20 EXPORT_SYMBOL(bad_dma_address);
22 /* This tells the BIO block layer to assume merging. Default to off
23 because we cannot guarantee merging later. */
24 int iommu_bio_merge __read_mostly = 0;
25 EXPORT_SYMBOL(iommu_bio_merge);
27 static int iommu_sac_force __read_mostly = 0;
29 int no_iommu __read_mostly;
30 /* Set this to 1 if there is a HW IOMMU in the system */
31 int iommu_detected __read_mostly = 0;
33 /* Dummy device used for NULL arguments (normally ISA). Better would
34 be probably a smaller DMA mask, but this is bug-to-bug compatible
36 struct device fallback_dev = {
37 .bus_id = "fallback device",
38 .coherent_dma_mask = DMA_32BIT_MASK,
39 .dma_mask = &fallback_dev.coherent_dma_mask,
42 /* Allocate DMA memory on node near device */
43 noinline static void *
44 dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
49 node = dev_to_node(dev);
51 page = alloc_pages_node(node, gfp, order);
52 return page ? page_address(page) : NULL;
56 * Allocate memory for a coherent mapping.
59 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
63 unsigned long dma_mask = 0;
68 dma_mask = dev->coherent_dma_mask;
70 dma_mask = DMA_32BIT_MASK;
72 /* Device not DMA able */
73 if (dev->dma_mask == NULL)
76 /* Don't invoke OOM killer */
79 /* Kludge to make it bug-to-bug compatible with i386. i386
80 uses the normal dma_mask for alloc_coherent. */
81 dma_mask &= *dev->dma_mask;
83 /* Why <=? Even when the mask is smaller than 4GB it is often
84 larger than 16MB and in this case we have a chance of
85 finding fitting memory in the next higher zone first. If
86 not retry with true GFP_DMA. -AK */
87 if (dma_mask <= DMA_32BIT_MASK)
91 memory = dma_alloc_pages(dev, gfp, get_order(size));
97 bus = virt_to_bus(memory);
98 high = (bus + size) >= dma_mask;
100 if (force_iommu && !(gfp & GFP_DMA))
103 free_pages((unsigned long)memory,
106 /* Don't use the 16MB ZONE_DMA unless absolutely
107 needed. It's better to use remapping first. */
108 if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
109 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
113 /* Let low level make its own zone decisions */
114 gfp &= ~(GFP_DMA32|GFP_DMA);
116 if (dma_ops->alloc_coherent)
117 return dma_ops->alloc_coherent(dev, size,
122 memset(memory, 0, size);
124 *dma_handle = virt_to_bus(memory);
129 if (dma_ops->alloc_coherent) {
130 free_pages((unsigned long)memory, get_order(size));
131 gfp &= ~(GFP_DMA|GFP_DMA32);
132 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
135 if (dma_ops->map_simple) {
136 *dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
138 PCI_DMA_BIDIRECTIONAL);
139 if (*dma_handle != bad_dma_address)
143 if (panic_on_overflow)
144 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
145 free_pages((unsigned long)memory, get_order(size));
148 EXPORT_SYMBOL(dma_alloc_coherent);
151 * Unmap coherent memory.
152 * The caller must ensure that the device has finished accessing the mapping.
154 void dma_free_coherent(struct device *dev, size_t size,
155 void *vaddr, dma_addr_t bus)
157 WARN_ON(irqs_disabled()); /* for portability */
158 if (dma_ops->unmap_single)
159 dma_ops->unmap_single(dev, bus, size, 0);
160 free_pages((unsigned long)vaddr, get_order(size));
162 EXPORT_SYMBOL(dma_free_coherent);
164 int dma_supported(struct device *dev, u64 mask)
167 if (mask > 0xffffffff && forbid_dac > 0) {
171 printk(KERN_INFO "PCI: Disallowing DAC for device %s\n", dev->bus_id);
176 if (dma_ops->dma_supported)
177 return dma_ops->dma_supported(dev, mask);
179 /* Copied from i386. Doesn't make much sense, because it will
180 only work for pci_alloc_coherent.
181 The caller just has to use GFP_DMA in this case. */
182 if (mask < DMA_24BIT_MASK)
185 /* Tell the device to use SAC when IOMMU force is on. This
186 allows the driver to use cheaper accesses in some cases.
188 Problem with this is that if we overflow the IOMMU area and
189 return DAC as fallback address the device may not handle it
192 As a special case some controllers have a 39bit address
193 mode that is as efficient as 32bit (aic79xx). Don't force
194 SAC for these. Assume all masks <= 40 bits are of this
195 type. Normally this doesn't make any difference, but gives
196 more gentle handling of IOMMU overflow. */
197 if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
198 printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
204 EXPORT_SYMBOL(dma_supported);
207 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
210 static __init int iommu_setup(char *p)
218 if (!strncmp(p, "off", 3))
220 /* gart_parse_options has more force support */
221 if (!strncmp(p, "force", 5))
223 if (!strncmp(p, "noforce", 7)) {
228 if (!strncmp(p, "biomerge", 8)) {
229 iommu_bio_merge = 4096;
233 if (!strncmp(p, "panic", 5))
234 panic_on_overflow = 1;
235 if (!strncmp(p, "nopanic", 7))
236 panic_on_overflow = 0;
237 if (!strncmp(p, "merge", 5)) {
241 if (!strncmp(p, "nomerge", 7))
243 if (!strncmp(p, "forcesac", 8))
245 if (!strncmp(p, "allowdac", 8))
247 if (!strncmp(p, "nodac", 5))
250 #ifdef CONFIG_SWIOTLB
251 if (!strncmp(p, "soft", 4))
255 #ifdef CONFIG_GART_IOMMU
256 gart_parse_options(p);
259 #ifdef CONFIG_CALGARY_IOMMU
260 if (!strncmp(p, "calgary", 7))
262 #endif /* CONFIG_CALGARY_IOMMU */
264 p += strcspn(p, ",");
270 early_param("iommu", iommu_setup);