From 6a32fd4d0d42258004631dc0ac90665382a2e5dc Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sun, 19 Feb 2006 22:21:32 -0800 Subject: [PATCH] [SPARC64]: Remove PGLIST_NENTS PCI IOMMU mapping limitation on SUN4V. Use a batching queue system for IOMMU mapping setup, with a page sized batch. Signed-off-by: David S. Miller --- arch/sparc64/kernel/pci_sun4v.c | 233 +++++++++++++++++++--------- arch/sparc64/kernel/pci_sun4v.h | 10 +- arch/sparc64/kernel/pci_sun4v_asm.S | 11 +- 3 files changed, 171 insertions(+), 83 deletions(-) diff --git a/arch/sparc64/kernel/pci_sun4v.c b/arch/sparc64/kernel/pci_sun4v.c index 902d07c714..4e9d3c451a 100644 --- a/arch/sparc64/kernel/pci_sun4v.c +++ b/arch/sparc64/kernel/pci_sun4v.c @@ -26,11 +26,86 @@ #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) -struct sun4v_pglist { - u64 *pglist; +struct pci_iommu_batch { + struct pci_dev *pdev; /* Device mapping is for. */ + unsigned long prot; /* IOMMU page protections */ + unsigned long entry; /* Index into IOTSB. */ + u64 *pglist; /* List of physical pages */ + unsigned long npages; /* Number of pages in list. */ }; -static DEFINE_PER_CPU(struct sun4v_pglist, iommu_pglists); +static DEFINE_PER_CPU(struct pci_iommu_batch, pci_iommu_batch); + +/* Interrupts must be disabled. */ +static inline void pci_iommu_batch_start(struct pci_dev *pdev, unsigned long prot, unsigned long entry) +{ + struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch); + + p->pdev = pdev; + p->prot = prot; + p->entry = entry; + p->npages = 0; +} + +/* Interrupts must be disabled. */ +static long pci_iommu_batch_flush(struct pci_iommu_batch *p) +{ + struct pcidev_cookie *pcp = p->pdev->sysdata; + unsigned long devhandle = pcp->pbm->devhandle; + unsigned long prot = p->prot; + unsigned long entry = p->entry; + u64 *pglist = p->pglist; + unsigned long npages = p->npages; + + do { + long num; + + num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), + npages, prot, __pa(pglist)); + if (unlikely(num < 0)) { + if (printk_ratelimit()) + printk("pci_iommu_batch_flush: IOMMU map of " + "[%08lx:%08lx:%lx:%lx:%lx] failed with " + "status %ld\n", + devhandle, HV_PCI_TSBID(0, entry), + npages, prot, __pa(pglist), num); + return -1; + } + + entry += num; + npages -= num; + pglist += num; + } while (npages != 0); + + p->entry = entry; + p->npages = 0; + + return 0; +} + +/* Interrupts must be disabled. */ +static inline long pci_iommu_batch_add(u64 phys_page) +{ + struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch); + + BUG_ON(p->npages >= PGLIST_NENTS); + + p->pglist[p->npages++] = phys_page; + if (p->npages == PGLIST_NENTS) + return pci_iommu_batch_flush(p); + + return 0; +} + +/* Interrupts must be disabled. */ +static inline long pci_iommu_batch_end(void) +{ + struct pci_iommu_batch *p = &__get_cpu_var(pci_iommu_batch); + + BUG_ON(p->npages >= PGLIST_NENTS); + + return pci_iommu_batch_flush(p); +} static long pci_arena_alloc(struct pci_iommu_arena *arena, unsigned long npages) { @@ -86,65 +161,64 @@ static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr unsigned long flags, order, first_page, npages, n; void *ret; long entry; - u64 *pglist; - u32 devhandle; - int cpu; size = IO_PAGE_ALIGN(size); order = get_order(size); - if (order >= MAX_ORDER) + if (unlikely(order >= MAX_ORDER)) return NULL; npages = size >> IO_PAGE_SHIFT; - if (npages > PGLIST_NENTS) - return NULL; first_page = __get_free_pages(GFP_ATOMIC, order); - if (first_page == 0UL) + if (unlikely(first_page == 0UL)) return NULL; memset((char *)first_page, 0, PAGE_SIZE << order); pcp = pdev->sysdata; - devhandle = pcp->pbm->devhandle; iommu = pcp->pbm->iommu; spin_lock_irqsave(&iommu->lock, flags); entry = pci_arena_alloc(&iommu->arena, npages); spin_unlock_irqrestore(&iommu->lock, flags); - if (unlikely(entry < 0L)) { - free_pages(first_page, order); - return NULL; - } + if (unlikely(entry < 0L)) + goto arena_alloc_fail; *dma_addrp = (iommu->page_table_map_base + (entry << IO_PAGE_SHIFT)); ret = (void *) first_page; first_page = __pa(first_page); - cpu = get_cpu(); + local_irq_save(flags); - pglist = __get_cpu_var(iommu_pglists).pglist; - for (n = 0; n < npages; n++) - pglist[n] = first_page + (n * PAGE_SIZE); + pci_iommu_batch_start(pdev, + (HV_PCI_MAP_ATTR_READ | + HV_PCI_MAP_ATTR_WRITE), + entry); - do { - unsigned long num; + for (n = 0; n < npages; n++) { + long err = pci_iommu_batch_add(first_page + (n * PAGE_SIZE)); + if (unlikely(err < 0L)) + goto iommu_map_fail; + } - num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), - npages, - (HV_PCI_MAP_ATTR_READ | - HV_PCI_MAP_ATTR_WRITE), - __pa(pglist)); - entry += num; - npages -= num; - pglist += num; - } while (npages != 0); + if (unlikely(pci_iommu_batch_end() < 0L)) + goto iommu_map_fail; - put_cpu(); + local_irq_restore(flags); return ret; + +iommu_map_fail: + /* Interrupts are disabled. */ + spin_lock(&iommu->lock); + pci_arena_free(&iommu->arena, entry, npages); + spin_unlock_irqrestore(&iommu->lock, flags); + +arena_alloc_fail: + free_pages(first_page, order); + return NULL; } static void pci_4v_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma) @@ -186,15 +260,12 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, struct pci_iommu *iommu; unsigned long flags, npages, oaddr; unsigned long i, base_paddr; - u32 devhandle, bus_addr, ret; + u32 bus_addr, ret; unsigned long prot; long entry; - u64 *pglist; - int cpu; pcp = pdev->sysdata; iommu = pcp->pbm->iommu; - devhandle = pcp->pbm->devhandle; if (unlikely(direction == PCI_DMA_NONE)) goto bad; @@ -202,8 +273,6 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, oaddr = (unsigned long)ptr; npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; - if (unlikely(npages > PGLIST_NENTS)) - goto bad; spin_lock_irqsave(&iommu->lock, flags); entry = pci_arena_alloc(&iommu->arena, npages); @@ -220,24 +289,19 @@ static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, if (direction != PCI_DMA_TODEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; - cpu = get_cpu(); - - pglist = __get_cpu_var(iommu_pglists).pglist; - for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) - pglist[i] = base_paddr; + local_irq_save(flags); - do { - unsigned long num; + pci_iommu_batch_start(pdev, prot, entry); - num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), - npages, prot, - __pa(pglist)); - entry += num; - npages -= num; - pglist += num; - } while (npages != 0); + for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { + long err = pci_iommu_batch_add(base_paddr); + if (unlikely(err < 0L)) + goto iommu_map_fail; + } + if (unlikely(pci_iommu_batch_end() < 0L)) + goto iommu_map_fail; - put_cpu(); + local_irq_restore(flags); return ret; @@ -245,6 +309,14 @@ bad: if (printk_ratelimit()) WARN_ON(1); return PCI_DMA_ERROR_CODE; + +iommu_map_fail: + /* Interrupts are disabled. */ + spin_lock(&iommu->lock); + pci_arena_free(&iommu->arena, entry, npages); + spin_unlock_irqrestore(&iommu->lock, flags); + + return PCI_DMA_ERROR_CODE; } static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) @@ -289,18 +361,19 @@ static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_ #define SG_ENT_PHYS_ADDRESS(SG) \ (__pa(page_address((SG)->page)) + (SG)->offset) -static inline void fill_sg(long entry, u32 devhandle, +static inline long fill_sg(long entry, struct pci_dev *pdev, struct scatterlist *sg, int nused, int nelems, unsigned long prot) { struct scatterlist *dma_sg = sg; struct scatterlist *sg_end = sg + nelems; - int i, cpu, pglist_ent; - u64 *pglist; + unsigned long flags; + int i; + + local_irq_save(flags); + + pci_iommu_batch_start(pdev, prot, entry); - cpu = get_cpu(); - pglist = __get_cpu_var(iommu_pglists).pglist; - pglist_ent = 0; for (i = 0; i < nused; i++) { unsigned long pteval = ~0UL; u32 dma_npages; @@ -338,7 +411,12 @@ static inline void fill_sg(long entry, u32 devhandle, pteval = (pteval & IOPTE_PAGE); while (len > 0) { - pglist[pglist_ent++] = pteval; + long err; + + err = pci_iommu_batch_add(pteval); + if (unlikely(err < 0L)) + goto iommu_map_failed; + pteval += IO_PAGE_SIZE; len -= (IO_PAGE_SIZE - offset); offset = 0; @@ -366,18 +444,15 @@ static inline void fill_sg(long entry, u32 devhandle, dma_sg++; } - BUG_ON(pglist_ent == 0); + if (unlikely(pci_iommu_batch_end() < 0L)) + goto iommu_map_failed; - do { - unsigned long num; - - num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), - pglist_ent); - entry += num; - pglist_ent -= num; - } while (pglist_ent != 0); + local_irq_restore(flags); + return 0; - put_cpu(); +iommu_map_failed: + local_irq_restore(flags); + return -1L; } static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) @@ -385,9 +460,9 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n struct pcidev_cookie *pcp; struct pci_iommu *iommu; unsigned long flags, npages, prot; - u32 devhandle, dma_base; + u32 dma_base; struct scatterlist *sgtmp; - long entry; + long entry, err; int used; /* Fast path single entry scatterlists. */ @@ -404,7 +479,6 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n pcp = pdev->sysdata; iommu = pcp->pbm->iommu; - devhandle = pcp->pbm->devhandle; if (unlikely(direction == PCI_DMA_NONE)) goto bad; @@ -441,7 +515,9 @@ static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int n if (direction != PCI_DMA_TODEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; - fill_sg(entry, devhandle, sglist, used, nelems, prot); + err = fill_sg(entry, pdev, sglist, used, nelems, prot); + if (unlikely(err < 0L)) + goto iommu_map_failed; return used; @@ -449,6 +525,13 @@ bad: if (printk_ratelimit()) WARN_ON(1); return 0; + +iommu_map_failed: + spin_lock_irqsave(&iommu->lock, flags); + pci_arena_free(&iommu->arena, entry, npages); + spin_unlock_irqrestore(&iommu->lock, flags); + + return 0; } static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) @@ -1011,13 +1094,13 @@ void sun4v_pci_init(int node, char *model_name) } } - for (i = 0; i < NR_CPUS; i++) { + for_each_cpu(i) { unsigned long page = get_zeroed_page(GFP_ATOMIC); if (!page) goto fatal_memory_error; - per_cpu(iommu_pglists, i).pglist = (u64 *) page; + per_cpu(pci_iommu_batch, i).pglist = (u64 *) page; } p = kmalloc(sizeof(struct pci_controller_info), GFP_ATOMIC); diff --git a/arch/sparc64/kernel/pci_sun4v.h b/arch/sparc64/kernel/pci_sun4v.h index 88f199e11a..884d25f615 100644 --- a/arch/sparc64/kernel/pci_sun4v.h +++ b/arch/sparc64/kernel/pci_sun4v.h @@ -6,11 +6,11 @@ #ifndef _PCI_SUN4V_H #define _PCI_SUN4V_H -extern unsigned long pci_sun4v_iommu_map(unsigned long devhandle, - unsigned long tsbid, - unsigned long num_ttes, - unsigned long io_attributes, - unsigned long io_page_list_pa); +extern long pci_sun4v_iommu_map(unsigned long devhandle, + unsigned long tsbid, + unsigned long num_ttes, + unsigned long io_attributes, + unsigned long io_page_list_pa); extern unsigned long pci_sun4v_iommu_demap(unsigned long devhandle, unsigned long tsbid, unsigned long num_ttes); diff --git a/arch/sparc64/kernel/pci_sun4v_asm.S b/arch/sparc64/kernel/pci_sun4v_asm.S index 424db65266..6604fdbf74 100644 --- a/arch/sparc64/kernel/pci_sun4v_asm.S +++ b/arch/sparc64/kernel/pci_sun4v_asm.S @@ -11,14 +11,19 @@ * %o3: io_attributes * %o4: io_page_list phys address * - * returns %o0: num ttes mapped + * returns %o0: -status if status was non-zero, else + * %o0: num pages mapped */ .globl pci_sun4v_iommu_map pci_sun4v_iommu_map: + mov %o5, %g1 mov HV_FAST_PCI_IOMMU_MAP, %o5 ta HV_FAST_TRAP - retl - mov %o1, %o0 + brnz,pn %o0, 1f + sub %g0, %o0, %o0 + mov %o1, %o0 +1: retl + nop /* %o0: devhandle * %o1: tsbid -- 2.39.5