powerpc/cell: Add DMA_ATTR_WEAK_ORDERING dma attribute and use in Cell IOMMU code

Introduce a new dma attriblue DMA_ATTR_WEAK_ORDERING to use weak ordering
on DMA mappings in the Cell processor. Add the code to the Cell's IOMMU
implementation to use this code.

Dynamic mappings can be weakly or strongly ordered on an individual basis
but the fixed mapping has to be either completely strong or completely weak.
This is currently decided by a kernel boot option (pass iommu_fixed=weak
for a weakly ordered fixed linear mapping, strongly ordered is the default).

Signed-off-by: Mark Nelson <markn@au1.ibm.com>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

diff --git a/Documentation/DMA-attributes.txt b/Documentation/DMA-attributes.txt
index 6d772f84b477c9f5a4698e87d50fb63649af83d3..b768cc0e402b8e6f1cfa8ae0b0f23c4e1c50168a 100644 (file)
--- a/Documentation/DMA-attributes.txt
+++ b/Documentation/DMA-attributes.txt
@@ -22,3 +22,12 @@ ready and available in memory.  The DMA of the "completion indication"
 could race with data DMA.  Mapping the memory used for completion
 indications with DMA_ATTR_WRITE_BARRIER would prevent the race.
 
+DMA_ATTR_WEAK_ORDERING
+----------------------
+
+DMA_ATTR_WEAK_ORDERING specifies that reads and writes to the mapping
+may be weakly ordered, that is that reads and writes may pass each other.
+
+Since it is optional for platforms to implement DMA_ATTR_WEAK_ORDERING,
+those that do not will simply ignore the attribute and exhibit default
+behavior.

diff --git a/arch/powerpc/platforms/cell/iommu.c b/arch/powerpc/platforms/cell/iommu.c
index 3b7078453e7fbf9a8e2b4e5e3e3badbcc2655031..208005ca262c62ac58b466c94e21ff3f47932924 100644 (file)
--- a/arch/powerpc/platforms/cell/iommu.c
+++ b/arch/powerpc/platforms/cell/iommu.c
@@ -199,6 +199,8 @@ static void tce_build_cell(struct iommu_table *tbl, long index, long npages,
        base_pte = IOPTE_PP_W | IOPTE_PP_R | IOPTE_M | IOPTE_SO_RW |
                (window->ioid & IOPTE_IOID_Mask);
 #endif
+       if (unlikely(dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs)))
+               base_pte &= ~IOPTE_SO_RW;
 
        io_pte = (unsigned long *)tbl->it_base + (index - tbl->it_offset);
 
@@ -539,7 +541,9 @@ static struct cbe_iommu *cell_iommu_for_node(int nid)
 static unsigned long cell_dma_direct_offset;
 
 static unsigned long dma_iommu_fixed_base;
-struct dma_mapping_ops dma_iommu_fixed_ops;
+
+/* iommu_fixed_is_weak is set if booted with iommu_fixed=weak */
+static int iommu_fixed_is_weak;
 
 static struct iommu_table *cell_get_iommu_table(struct device *dev)
 {
@@ -563,6 +567,98 @@ static struct iommu_table *cell_get_iommu_table(struct device *dev)
        return &window->table;
 }
 
+/* A coherent allocation implies strong ordering */
+
+static void *dma_fixed_alloc_coherent(struct device *dev, size_t size,
+                                     dma_addr_t *dma_handle, gfp_t flag)
+{
+       if (iommu_fixed_is_weak)
+               return iommu_alloc_coherent(dev, cell_get_iommu_table(dev),
+                                           size, dma_handle,
+                                           device_to_mask(dev), flag,
+                                           dev->archdata.numa_node);
+       else
+               return dma_direct_ops.alloc_coherent(dev, size, dma_handle,
+                                                    flag);
+}
+
+static void dma_fixed_free_coherent(struct device *dev, size_t size,
+                                   void *vaddr, dma_addr_t dma_handle)
+{
+       if (iommu_fixed_is_weak)
+               iommu_free_coherent(cell_get_iommu_table(dev), size, vaddr,
+                                   dma_handle);
+       else
+               dma_direct_ops.free_coherent(dev, size, vaddr, dma_handle);
+}
+
+static dma_addr_t dma_fixed_map_single(struct device *dev, void *ptr,
+                                      size_t size,
+                                      enum dma_data_direction direction,
+                                      struct dma_attrs *attrs)
+{
+       if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs))
+               return dma_direct_ops.map_single(dev, ptr, size, direction,
+                                                attrs);
+       else
+               return iommu_map_single(dev, cell_get_iommu_table(dev), ptr,
+                                       size, device_to_mask(dev), direction,
+                                       attrs);
+}
+
+static void dma_fixed_unmap_single(struct device *dev, dma_addr_t dma_addr,
+                                  size_t size,
+                                  enum dma_data_direction direction,
+                                  struct dma_attrs *attrs)
+{
+       if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs))
+               dma_direct_ops.unmap_single(dev, dma_addr, size, direction,
+                                           attrs);
+       else
+               iommu_unmap_single(cell_get_iommu_table(dev), dma_addr, size,
+                                  direction, attrs);
+}
+
+static int dma_fixed_map_sg(struct device *dev, struct scatterlist *sg,
+                          int nents, enum dma_data_direction direction,
+                          struct dma_attrs *attrs)
+{
+       if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs))
+               return dma_direct_ops.map_sg(dev, sg, nents, direction, attrs);
+       else
+               return iommu_map_sg(dev, cell_get_iommu_table(dev), sg, nents,
+                                   device_to_mask(dev), direction, attrs);
+}
+
+static void dma_fixed_unmap_sg(struct device *dev, struct scatterlist *sg,
+                              int nents, enum dma_data_direction direction,
+                              struct dma_attrs *attrs)
+{
+       if (iommu_fixed_is_weak == dma_get_attr(DMA_ATTR_WEAK_ORDERING, attrs))
+               dma_direct_ops.unmap_sg(dev, sg, nents, direction, attrs);
+       else
+               iommu_unmap_sg(cell_get_iommu_table(dev), sg, nents, direction,
+                              attrs);
+}
+
+static int dma_fixed_dma_supported(struct device *dev, u64 mask)
+{
+       return mask == DMA_64BIT_MASK;
+}
+
+static int dma_set_mask_and_switch(struct device *dev, u64 dma_mask);
+
+struct dma_mapping_ops dma_iommu_fixed_ops = {
+       .alloc_coherent = dma_fixed_alloc_coherent,
+       .free_coherent  = dma_fixed_free_coherent,
+       .map_single     = dma_fixed_map_single,
+       .unmap_single   = dma_fixed_unmap_single,
+       .map_sg         = dma_fixed_map_sg,
+       .unmap_sg       = dma_fixed_unmap_sg,
+       .dma_supported  = dma_fixed_dma_supported,
+       .set_dma_mask   = dma_set_mask_and_switch,
+};
+
 static void cell_dma_dev_setup_fixed(struct device *dev);
 
 static void cell_dma_dev_setup(struct device *dev)
@@ -919,9 +1015,16 @@ static void cell_iommu_setup_fixed_ptab(struct cbe_iommu *iommu,
 
        pr_debug("iommu: mapping 0x%lx pages from 0x%lx\n", fsize, fbase);
 
-       base_pte = IOPTE_PP_W | IOPTE_PP_R | IOPTE_M | IOPTE_SO_RW
+       base_pte = IOPTE_PP_W | IOPTE_PP_R | IOPTE_M
                    | (cell_iommu_get_ioid(np) & IOPTE_IOID_Mask);
 
+       if (iommu_fixed_is_weak)
+               pr_info("IOMMU: Using weak ordering for fixed mapping\n");
+       else {
+               pr_info("IOMMU: Using strong ordering for fixed mapping\n");
+               base_pte |= IOPTE_SO_RW;
+       }
+
        for (uaddr = 0; uaddr < fsize; uaddr += (1 << 24)) {
                /* Don't touch the dynamic region */
                ioaddr = uaddr + fbase;
@@ -1037,9 +1140,6 @@ static int __init cell_iommu_fixed_mapping_init(void)
                cell_iommu_setup_window(iommu, np, dbase, dsize, 0);
        }
 
-       dma_iommu_fixed_ops = dma_direct_ops;
-       dma_iommu_fixed_ops.set_dma_mask = dma_set_mask_and_switch;
-
        dma_iommu_ops.set_dma_mask = dma_set_mask_and_switch;
        set_pci_dma_ops(&dma_iommu_ops);
 
@@ -1053,6 +1153,9 @@ static int __init setup_iommu_fixed(char *str)
        if (strcmp(str, "off") == 0)
                iommu_fixed_disabled = 1;
 
+       else if (strcmp(str, "weak") == 0)
+               iommu_fixed_is_weak = 1;
+
        return 1;
 }
 __setup("iommu_fixed=", setup_iommu_fixed);

diff --git a/include/linux/dma-attrs.h b/include/linux/dma-attrs.h
index 1677e2bfa00cea03f216b196b32cf42791aa9521..71ad34eca6e3d124f9175e79d536a39379ee02fa 100644 (file)
--- a/include/linux/dma-attrs.h
+++ b/include/linux/dma-attrs.h
@@ -12,6 +12,7 @@
  */
 enum dma_attr {
        DMA_ATTR_WRITE_BARRIER,
+       DMA_ATTR_WEAK_ORDERING,
        DMA_ATTR_MAX,
 };