X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=lib%2Fswiotlb.c;h=10625785eefd90d412668fce760190b122569bb9;hb=45c091bb2d453ce4a8b06cf19872ec7a77fc4799;hp=875b0c16250cdea9e03b7a2ed7f334392af83d5e;hpb=6c654b5fdf093cd05f35f7c9c2a00182fa5636dc;p=linux-2.6 diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 875b0c1625..10625785ee 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -1,7 +1,7 @@ /* * Dynamic DMA mapping support. * - * This implementation is for IA-64 platforms that do not support + * This implementation is for IA-64 and EM64T platforms that do not support * I/O TLBs (aka DMA address translation hardware). * Copyright (C) 2000 Asit Mallick * Copyright (C) 2000 Goutham Rao @@ -11,21 +11,23 @@ * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API. * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid * unnecessary i-cache flushing. - * 04/07/.. ak Better overflow handling. Assorted fixes. + * 04/07/.. ak Better overflow handling. Assorted fixes. + * 05/09/10 linville Add support for syncing ranges, support syncing for + * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup. */ #include +#include #include #include -#include #include #include #include #include #include -#include #include +#include #include #include @@ -58,6 +60,14 @@ */ #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) +/* + * Enumeration for sync targets + */ +enum dma_sync_target { + SYNC_FOR_CPU = 0, + SYNC_FOR_DEVICE = 1, +}; + int swiotlb_force; /* @@ -117,7 +127,7 @@ __setup("swiotlb=", setup_io_tlb_npages); /* * Statically reserve bounce buffer space and initialize bounce buffer data - * structures for the software IO TLB used to implement the PCI DMA API. + * structures for the software IO TLB used to implement the DMA API. */ void swiotlb_init_with_default_size (size_t default_size) @@ -132,8 +142,7 @@ swiotlb_init_with_default_size (size_t default_size) /* * Get IO TLB memory from the low pages */ - io_tlb_start = alloc_bootmem_low_pages(io_tlb_nslabs * - (1 << IO_TLB_SHIFT)); + io_tlb_start = alloc_bootmem_low_pages(io_tlb_nslabs * (1 << IO_TLB_SHIFT)); if (!io_tlb_start) panic("Cannot allocate SWIOTLB buffer"); io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT); @@ -287,8 +296,7 @@ map_single(struct device *hwdev, char *buffer, size_t size, int dir) else stride = 1; - if (!nslots) - BUG(); + BUG_ON(!nslots); /* * Find suitable number of IO TLB entries size that will fit this @@ -397,26 +405,33 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir) } static void -sync_single(struct device *hwdev, char *dma_addr, size_t size, int dir) +sync_single(struct device *hwdev, char *dma_addr, size_t size, + int dir, int target) { int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; char *buffer = io_tlb_orig_addr[index]; - /* - * bounce... copy the data back into/from the original buffer - * XXX How do you handle DMA_BIDIRECTIONAL here ? - */ - if (dir == DMA_FROM_DEVICE) - memcpy(buffer, dma_addr, size); - else if (dir == DMA_TO_DEVICE) - memcpy(dma_addr, buffer, size); - else + switch (target) { + case SYNC_FOR_CPU: + if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) + memcpy(buffer, dma_addr, size); + else + BUG_ON(dir != DMA_TO_DEVICE); + break; + case SYNC_FOR_DEVICE: + if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) + memcpy(dma_addr, buffer, size); + else + BUG_ON(dir != DMA_FROM_DEVICE); + break; + default: BUG(); + } } void * swiotlb_alloc_coherent(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, int flags) + dma_addr_t *dma_handle, gfp_t flags) { unsigned long dev_addr; void *ret; @@ -447,7 +462,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, */ dma_addr_t handle; handle = swiotlb_map_single(NULL, NULL, size, DMA_FROM_DEVICE); - if (dma_mapping_error(handle)) + if (swiotlb_dma_mapping_error(handle)) return NULL; ret = phys_to_virt(handle); @@ -485,24 +500,24 @@ swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) /* * Ran out of IOMMU space for this operation. This is very bad. * Unfortunately the drivers cannot handle this operation properly. - * unless they check for pci_dma_mapping_error (most don't) + * unless they check for dma_mapping_error (most don't) * When the mapping is small enough return a static buffer to limit * the damage, or panic when the transfer is too big. */ - printk(KERN_ERR "PCI-DMA: Out of SW-IOMMU space for %lu bytes at " + printk(KERN_ERR "DMA: Out of SW-IOMMU space for %lu bytes at " "device %s\n", size, dev ? dev->bus_id : "?"); if (size > io_tlb_overflow && do_panic) { - if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL) - panic("PCI-DMA: Memory would be corrupted\n"); - if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL) - panic("PCI-DMA: Random memory would be DMAed\n"); + if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) + panic("DMA: Memory would be corrupted\n"); + if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) + panic("DMA: Random memory would be DMAed\n"); } } /* * Map a single buffer of the indicated size for DMA in streaming mode. The - * PCI address to use is returned. + * physical address to use is returned. * * Once the device is given the dma address, the device owns this memory until * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed. @@ -513,8 +528,7 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir) unsigned long dev_addr = virt_to_phys(ptr); void *map; - if (dir == DMA_NONE) - BUG(); + BUG_ON(dir == DMA_NONE); /* * If the pointer passed in happens to be in the device's DMA window, * we can safely return the device addr and not worry about bounce @@ -576,8 +590,7 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size, { char *dma_addr = phys_to_virt(dev_addr); - if (dir == DMA_NONE) - BUG(); + BUG_ON(dir == DMA_NONE); if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end) unmap_single(hwdev, dma_addr, size, dir); else if (dir == DMA_FROM_DEVICE) @@ -589,39 +602,71 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size, * after a transfer. * * If you perform a swiotlb_map_single() but wish to interrogate the buffer - * using the cpu, yet do not wish to teardown the PCI dma mapping, you must - * call this function before doing so. At the next point you give the PCI dma + * using the cpu, yet do not wish to teardown the dma mapping, you must + * call this function before doing so. At the next point you give the dma * address back to the card, you must first perform a * swiotlb_dma_sync_for_device, and then the device again owns the buffer */ -void -swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, - size_t size, int dir) +static inline void +swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, + size_t size, int dir, int target) { char *dma_addr = phys_to_virt(dev_addr); - if (dir == DMA_NONE) - BUG(); + BUG_ON(dir == DMA_NONE); if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end) - sync_single(hwdev, dma_addr, size, dir); + sync_single(hwdev, dma_addr, size, dir, target); else if (dir == DMA_FROM_DEVICE) mark_clean(dma_addr, size); } +void +swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, + size_t size, int dir) +{ + swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); +} + void swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, size_t size, int dir) { - char *dma_addr = phys_to_virt(dev_addr); + swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); +} - if (dir == DMA_NONE) - BUG(); +/* + * Same as above, but for a sub-range of the mapping. + */ +static inline void +swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr, + unsigned long offset, size_t size, + int dir, int target) +{ + char *dma_addr = phys_to_virt(dev_addr) + offset; + + BUG_ON(dir == DMA_NONE); if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end) - sync_single(hwdev, dma_addr, size, dir); + sync_single(hwdev, dma_addr, size, dir, target); else if (dir == DMA_FROM_DEVICE) mark_clean(dma_addr, size); } +void +swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr, + unsigned long offset, size_t size, int dir) +{ + swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, + SYNC_FOR_CPU); +} + +void +swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, + unsigned long offset, size_t size, int dir) +{ + swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, + SYNC_FOR_DEVICE); +} + /* * Map a set of buffers described by scatterlist in streaming mode for DMA. * This is the scatter-gather version of the above swiotlb_map_single @@ -646,15 +691,15 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems, unsigned long dev_addr; int i; - if (dir == DMA_NONE) - BUG(); + BUG_ON(dir == DMA_NONE); for (i = 0; i < nelems; i++, sg++) { addr = SG_ENT_VIRT_ADDRESS(sg); dev_addr = virt_to_phys(addr); if (swiotlb_force || address_needs_mapping(hwdev, dev_addr)) { - sg->dma_address = (dma_addr_t) virt_to_phys(map_single(hwdev, addr, sg->length, dir)); - if (!sg->dma_address) { + void *map = map_single(hwdev, addr, sg->length, dir); + sg->dma_address = virt_to_bus(map); + if (!map) { /* Don't panic here, we expect map_sg users to do proper error handling. */ swiotlb_full(hwdev, sg->length, dir, 0); @@ -679,8 +724,7 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems, { int i; - if (dir == DMA_NONE) - BUG(); + BUG_ON(dir == DMA_NONE); for (i = 0; i < nelems; i++, sg++) if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) @@ -696,34 +740,32 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems, * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules * and usage. */ -void -swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, - int nelems, int dir) +static inline void +swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sg, + int nelems, int dir, int target) { int i; - if (dir == DMA_NONE) - BUG(); + BUG_ON(dir == DMA_NONE); for (i = 0; i < nelems; i++, sg++) if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) sync_single(hwdev, (void *) sg->dma_address, - sg->dma_length, dir); + sg->dma_length, dir, target); +} + +void +swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, + int nelems, int dir) +{ + swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); } void swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, int nelems, int dir) { - int i; - - if (dir == DMA_NONE) - BUG(); - - for (i = 0; i < nelems; i++, sg++) - if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) - sync_single(hwdev, (void *) sg->dma_address, - sg->dma_length, dir); + swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); } int @@ -733,9 +775,9 @@ swiotlb_dma_mapping_error(dma_addr_t dma_addr) } /* - * Return whether the given PCI device DMA address mask can be supported + * Return whether the given device DMA address mask can be supported * properly. For example, if your device can only drive the low 24-bits - * during PCI bus mastering, then you would pass 0x00ffffff as the mask to + * during bus mastering, then you would pass 0x00ffffff as the mask to * this function. */ int @@ -751,6 +793,8 @@ EXPORT_SYMBOL(swiotlb_map_sg); EXPORT_SYMBOL(swiotlb_unmap_sg); EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); EXPORT_SYMBOL(swiotlb_sync_single_for_device); +EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu); +EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device); EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); EXPORT_SYMBOL(swiotlb_sync_sg_for_device); EXPORT_SYMBOL(swiotlb_dma_mapping_error);