From 7f8e33546d17c7d8849be3a6623c3b6b3c9b588b Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 6 Feb 2007 17:29:53 +0000 Subject: [ARM] Don't call consistent_sync() for DMA coherent memory Memory allocated by the coherent memory allocators will be marked uncacheable, which means it's pointless calling consistent_sync() to perform cache maintainence on this memory; it's just a waste of CPU cycles. Moreover, with the (subsequent) merge of outer cache support, it actually breaks things to call consistent_sync() on anything but direct-mapped memory. Signed-off-by: Russell King --- arch/arm/common/dmabounce.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c index 2e635b814c1..272702accd8 100644 --- a/arch/arm/common/dmabounce.c +++ b/arch/arm/common/dmabounce.c @@ -281,10 +281,14 @@ map_single(struct device *dev, void *ptr, size_t size, ptr = buf->safe; dma_addr = buf->safe_dma_addr; + } else { + /* + * We don't need to sync the DMA buffer since + * it was allocated via the coherent allocators. + */ + consistent_sync(ptr, size, dir); } - consistent_sync(ptr, size, dir); - return dma_addr; } @@ -397,7 +401,10 @@ sync_single(struct device *dev, dma_addr_t dma_addr, size_t size, default: BUG(); } - consistent_sync(buf->safe, size, dir); + /* + * No need to sync the safe buffer - it was allocated + * via the coherent allocators. + */ } else { consistent_sync(dma_to_virt(dev, dma_addr), size, dir); } -- cgit v1.2.3-70-g09d2 From 953233dc9958ba2b29753d0f24e37a33a076a5f6 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 5 Feb 2007 14:48:08 +0100 Subject: [ARM] 4134/1: Add generic support for outer caches The outer cache can be L2 as on RealView/EB MPCore platform or even L3 or further on ARMv7 cores. This patch adds the generic support for flushing the outer cache in the DMA operations. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/common/dmabounce.c | 1 + arch/arm/kernel/setup.c | 3 +++ arch/arm/mm/Kconfig | 3 +++ arch/arm/mm/consistent.c | 6 ++++++ include/asm-arm/cacheflush.h | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 50 insertions(+) (limited to 'arch/arm/common') diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c index 272702accd8..b4748e3171c 100644 --- a/arch/arm/common/dmabounce.c +++ b/arch/arm/common/dmabounce.c @@ -338,6 +338,7 @@ unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, */ ptr = (unsigned long)buf->ptr; dmac_clean_range(ptr, ptr + size); + outer_clean_range(__pa(ptr), __pa(ptr) + size); } free_safe_buffer(device_info, buf); } diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index bbab134cd82..243aea45805 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -88,6 +88,9 @@ struct cpu_user_fns cpu_user; #ifdef MULTI_CACHE struct cpu_cache_fns cpu_cache; #endif +#ifdef CONFIG_OUTER_CACHE +struct outer_cache_fns outer_cache; +#endif struct stack { u32 irq[3]; diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index aade2f72c92..a84eed9f854 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -609,3 +609,6 @@ config NEEDS_SYSCALL_FOR_CMPXCHG Forget about fast user space cmpxchg support. It is just not possible. +config OUTER_CACHE + bool + default n diff --git a/arch/arm/mm/consistent.c b/arch/arm/mm/consistent.c index 6a9c362fef5..83bd035c7d5 100644 --- a/arch/arm/mm/consistent.c +++ b/arch/arm/mm/consistent.c @@ -208,6 +208,7 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, unsigned long kaddr = (unsigned long)page_address(page); memset(page_address(page), 0, size); dmac_flush_range(kaddr, kaddr + size); + outer_flush_range(__pa(kaddr), __pa(kaddr) + size); } /* @@ -485,15 +486,20 @@ void consistent_sync(void *vaddr, size_t size, int direction) unsigned long start = (unsigned long)vaddr; unsigned long end = start + size; + BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(end)); + switch (direction) { case DMA_FROM_DEVICE: /* invalidate only */ dmac_inv_range(start, end); + outer_inv_range(__pa(start), __pa(end)); break; case DMA_TO_DEVICE: /* writeback only */ dmac_clean_range(start, end); + outer_clean_range(__pa(start), __pa(end)); break; case DMA_BIDIRECTIONAL: /* writeback and invalidate */ dmac_flush_range(start, end); + outer_flush_range(__pa(start), __pa(end)); break; default: BUG(); diff --git a/include/asm-arm/cacheflush.h b/include/asm-arm/cacheflush.h index 5f531ea0305..ce60b3702ba 100644 --- a/include/asm-arm/cacheflush.h +++ b/include/asm-arm/cacheflush.h @@ -190,6 +190,12 @@ struct cpu_cache_fns { void (*dma_flush_range)(unsigned long, unsigned long); }; +struct outer_cache_fns { + void (*inv_range)(unsigned long, unsigned long); + void (*clean_range)(unsigned long, unsigned long); + void (*flush_range)(unsigned long, unsigned long); +}; + /* * Select the calling method */ @@ -246,6 +252,37 @@ extern void dmac_flush_range(unsigned long, unsigned long); #endif +#ifdef CONFIG_OUTER_CACHE + +extern struct outer_cache_fns outer_cache; + +static inline void outer_inv_range(unsigned long start, unsigned long end) +{ + if (outer_cache.inv_range) + outer_cache.inv_range(start, end); +} +static inline void outer_clean_range(unsigned long start, unsigned long end) +{ + if (outer_cache.clean_range) + outer_cache.clean_range(start, end); +} +static inline void outer_flush_range(unsigned long start, unsigned long end) +{ + if (outer_cache.flush_range) + outer_cache.flush_range(start, end); +} + +#else + +static inline void outer_inv_range(unsigned long start, unsigned long end) +{ } +static inline void outer_clean_range(unsigned long start, unsigned long end) +{ } +static inline void outer_flush_range(unsigned long start, unsigned long end) +{ } + +#endif + /* * flush_cache_vmap() is used when creating mappings (eg, via vmap, * vmalloc, ioremap etc) in kernel space for pages. Since the -- cgit v1.2.3-70-g09d2 From 7ae5a761d2ffc4cf7d3248e09f4d3da234434f30 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 6 Feb 2007 17:39:31 +0000 Subject: [ARM] Convert DMA cache handling to take const void * args The DMA cache handling functions take virtual addresses, but in the form of unsigned long arguments. This leads to a little confusion about what exactly they take. So, convert them to take const void * instead. Signed-off-by: Russell King --- arch/arm/common/dmabounce.c | 7 +++---- arch/arm/mm/consistent.c | 13 ++++++------- include/asm-arm/cacheflush.h | 12 ++++++------ include/asm-arm/dma-mapping.h | 2 +- 4 files changed, 16 insertions(+), 18 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c index b4748e3171c..2362c498f52 100644 --- a/arch/arm/common/dmabounce.c +++ b/arch/arm/common/dmabounce.c @@ -321,12 +321,12 @@ unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, DO_STATS ( device_info->bounce_count++ ); if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { - unsigned long ptr; + void *ptr = buf->ptr; dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n", - __func__, buf->safe, buf->ptr, size); - memcpy(buf->ptr, buf->safe, size); + __func__, buf->safe, ptr, size); + memcpy(ptr, buf->safe, size); /* * DMA buffers must have the same cache properties @@ -336,7 +336,6 @@ unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, * bidirectional case because we know the cache * lines will be coherent with the data written. */ - ptr = (unsigned long)buf->ptr; dmac_clean_range(ptr, ptr + size); outer_clean_range(__pa(ptr), __pa(ptr) + size); } diff --git a/arch/arm/mm/consistent.c b/arch/arm/mm/consistent.c index 83bd035c7d5..166aee13c4b 100644 --- a/arch/arm/mm/consistent.c +++ b/arch/arm/mm/consistent.c @@ -205,10 +205,10 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, * kernel direct-mapped region for device DMA. */ { - unsigned long kaddr = (unsigned long)page_address(page); - memset(page_address(page), 0, size); - dmac_flush_range(kaddr, kaddr + size); - outer_flush_range(__pa(kaddr), __pa(kaddr) + size); + void *ptr = page_address(page); + memset(ptr, 0, size); + dmac_flush_range(ptr, ptr + size); + outer_flush_range(__pa(ptr), __pa(ptr) + size); } /* @@ -481,10 +481,9 @@ core_initcall(consistent_init); * platforms with CONFIG_DMABOUNCE. * Use the driver DMA support - see dma-mapping.h (dma_sync_*) */ -void consistent_sync(void *vaddr, size_t size, int direction) +void consistent_sync(const void *start, size_t size, int direction) { - unsigned long start = (unsigned long)vaddr; - unsigned long end = start + size; + const void *end = start + size; BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(end)); diff --git a/include/asm-arm/cacheflush.h b/include/asm-arm/cacheflush.h index ce60b3702ba..afad32c76e6 100644 --- a/include/asm-arm/cacheflush.h +++ b/include/asm-arm/cacheflush.h @@ -185,9 +185,9 @@ struct cpu_cache_fns { void (*coherent_user_range)(unsigned long, unsigned long); void (*flush_kern_dcache_page)(void *); - void (*dma_inv_range)(unsigned long, unsigned long); - void (*dma_clean_range)(unsigned long, unsigned long); - void (*dma_flush_range)(unsigned long, unsigned long); + void (*dma_inv_range)(const void *, const void *); + void (*dma_clean_range)(const void *, const void *); + void (*dma_flush_range)(const void *, const void *); }; struct outer_cache_fns { @@ -246,9 +246,9 @@ extern void __cpuc_flush_dcache_page(void *); #define dmac_clean_range __glue(_CACHE,_dma_clean_range) #define dmac_flush_range __glue(_CACHE,_dma_flush_range) -extern void dmac_inv_range(unsigned long, unsigned long); -extern void dmac_clean_range(unsigned long, unsigned long); -extern void dmac_flush_range(unsigned long, unsigned long); +extern void dmac_inv_range(const void *, const void *); +extern void dmac_clean_range(const void *, const void *); +extern void dmac_flush_range(const void *, const void *); #endif diff --git a/include/asm-arm/dma-mapping.h b/include/asm-arm/dma-mapping.h index 9bc46b486af..a1d574cdcc1 100644 --- a/include/asm-arm/dma-mapping.h +++ b/include/asm-arm/dma-mapping.h @@ -17,7 +17,7 @@ * platforms with CONFIG_DMABOUNCE. * Use the driver DMA support - see dma-mapping.h (dma_sync_*) */ -extern void consistent_sync(void *kaddr, size_t size, int rw); +extern void consistent_sync(const void *kaddr, size_t size, int rw); /* * Return whether the given device DMA address mask can be supported -- cgit v1.2.3-70-g09d2 From b3a1bde4db9889feb116330bff21214811c940e4 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 14 Feb 2007 19:14:56 +0100 Subject: [ARM] 4108/2: Allow multiple GIC interrupt controllers in a system The current implementation only assumes one GIC to be present in the system. However, there are platforms with more than one cascaded interrupt controllers (RealView/EB MPCore for example). Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/common/gic.c | 109 ++++++++++++++++++++++++++++++----- arch/arm/mach-realview/platsmp.c | 2 +- arch/arm/mach-realview/realview_eb.c | 4 +- include/asm-arm/hardware/gic.h | 5 +- 4 files changed, 101 insertions(+), 19 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/gic.c b/arch/arm/common/gic.c index 09b9d1b6844..4deece5fbdf 100644 --- a/arch/arm/common/gic.c +++ b/arch/arm/common/gic.c @@ -14,7 +14,9 @@ * * o There is one CPU Interface per CPU, which sends interrupts sent * by the Distributor, and interrupts generated locally, to the - * associated CPU. + * associated CPU. The base address of the CPU interface is usually + * aliased so that the same address points to different chips depending + * on the CPU it is accessed from. * * Note that IRQs 0-31 are special - they are local to each CPU. * As such, the enable set/clear, pending set/clear and active bit @@ -31,10 +33,38 @@ #include #include -static void __iomem *gic_dist_base; -static void __iomem *gic_cpu_base; static DEFINE_SPINLOCK(irq_controller_lock); +struct gic_chip_data { + unsigned int irq_offset; + void __iomem *dist_base; + void __iomem *cpu_base; +}; + +#ifndef MAX_GIC_NR +#define MAX_GIC_NR 1 +#endif + +static struct gic_chip_data gic_data[MAX_GIC_NR]; + +static inline void __iomem *gic_dist_base(unsigned int irq) +{ + struct gic_chip_data *gic_data = get_irq_chip_data(irq); + return gic_data->dist_base; +} + +static inline void __iomem *gic_cpu_base(unsigned int irq) +{ + struct gic_chip_data *gic_data = get_irq_chip_data(irq); + return gic_data->cpu_base; +} + +static inline unsigned int gic_irq(unsigned int irq) +{ + struct gic_chip_data *gic_data = get_irq_chip_data(irq); + return irq - gic_data->irq_offset; +} + /* * Routines to acknowledge, disable and enable interrupts * @@ -55,8 +85,8 @@ static void gic_ack_irq(unsigned int irq) u32 mask = 1 << (irq % 32); spin_lock(&irq_controller_lock); - writel(mask, gic_dist_base + GIC_DIST_ENABLE_CLEAR + (irq / 32) * 4); - writel(irq, gic_cpu_base + GIC_CPU_EOI); + writel(mask, gic_dist_base(irq) + GIC_DIST_ENABLE_CLEAR + (gic_irq(irq) / 32) * 4); + writel(gic_irq(irq), gic_cpu_base(irq) + GIC_CPU_EOI); spin_unlock(&irq_controller_lock); } @@ -65,7 +95,7 @@ static void gic_mask_irq(unsigned int irq) u32 mask = 1 << (irq % 32); spin_lock(&irq_controller_lock); - writel(mask, gic_dist_base + GIC_DIST_ENABLE_CLEAR + (irq / 32) * 4); + writel(mask, gic_dist_base(irq) + GIC_DIST_ENABLE_CLEAR + (gic_irq(irq) / 32) * 4); spin_unlock(&irq_controller_lock); } @@ -74,14 +104,14 @@ static void gic_unmask_irq(unsigned int irq) u32 mask = 1 << (irq % 32); spin_lock(&irq_controller_lock); - writel(mask, gic_dist_base + GIC_DIST_ENABLE_SET + (irq / 32) * 4); + writel(mask, gic_dist_base(irq) + GIC_DIST_ENABLE_SET + (gic_irq(irq) / 32) * 4); spin_unlock(&irq_controller_lock); } #ifdef CONFIG_SMP static void gic_set_cpu(unsigned int irq, cpumask_t mask_val) { - void __iomem *reg = gic_dist_base + GIC_DIST_TARGET + (irq & ~3); + void __iomem *reg = gic_dist_base(irq) + GIC_DIST_TARGET + (gic_irq(irq) & ~3); unsigned int shift = (irq % 4) * 8; unsigned int cpu = first_cpu(mask_val); u32 val; @@ -95,6 +125,37 @@ static void gic_set_cpu(unsigned int irq, cpumask_t mask_val) } #endif +static void fastcall gic_handle_cascade_irq(unsigned int irq, + struct irq_desc *desc) +{ + struct gic_chip_data *chip_data = get_irq_data(irq); + struct irq_chip *chip = get_irq_chip(irq); + unsigned int cascade_irq; + unsigned long status; + + /* primary controller ack'ing */ + chip->ack(irq); + + spin_lock(&irq_controller_lock); + status = readl(chip_data->cpu_base + GIC_CPU_INTACK); + spin_unlock(&irq_controller_lock); + + cascade_irq = (status & 0x3ff); + if (cascade_irq > 1020) + goto out; + if (cascade_irq < 32 || cascade_irq >= NR_IRQS) { + do_bad_IRQ(cascade_irq, desc); + goto out; + } + + cascade_irq += chip_data->irq_offset; + generic_handle_irq(cascade_irq); + + out: + /* primary controller unmasking */ + chip->unmask(irq); +} + static struct irq_chip gic_chip = { .name = "GIC", .ack = gic_ack_irq, @@ -105,15 +166,29 @@ static struct irq_chip gic_chip = { #endif }; -void __init gic_dist_init(void __iomem *base) +void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq) +{ + if (gic_nr >= MAX_GIC_NR) + BUG(); + if (set_irq_data(irq, &gic_data[gic_nr]) != 0) + BUG(); + set_irq_chained_handler(irq, gic_handle_cascade_irq); +} + +void __init gic_dist_init(unsigned int gic_nr, void __iomem *base, + unsigned int irq_start) { unsigned int max_irq, i; u32 cpumask = 1 << smp_processor_id(); + if (gic_nr >= MAX_GIC_NR) + BUG(); + cpumask |= cpumask << 8; cpumask |= cpumask << 16; - gic_dist_base = base; + gic_data[gic_nr].dist_base = base; + gic_data[gic_nr].irq_offset = (irq_start - 1) & ~31; writel(0, base + GIC_DIST_CTRL); @@ -158,8 +233,9 @@ void __init gic_dist_init(void __iomem *base) /* * Setup the Linux IRQ subsystem. */ - for (i = 29; i < max_irq; i++) { + for (i = irq_start; i < gic_data[gic_nr].irq_offset + max_irq; i++) { set_irq_chip(i, &gic_chip); + set_irq_chip_data(i, &gic_data[gic_nr]); set_irq_handler(i, handle_level_irq); set_irq_flags(i, IRQF_VALID | IRQF_PROBE); } @@ -167,9 +243,13 @@ void __init gic_dist_init(void __iomem *base) writel(1, base + GIC_DIST_CTRL); } -void __cpuinit gic_cpu_init(void __iomem *base) +void __cpuinit gic_cpu_init(unsigned int gic_nr, void __iomem *base) { - gic_cpu_base = base; + if (gic_nr >= MAX_GIC_NR) + BUG(); + + gic_data[gic_nr].cpu_base = base; + writel(0xf0, base + GIC_CPU_PRIMASK); writel(1, base + GIC_CPU_CTRL); } @@ -179,6 +259,7 @@ void gic_raise_softirq(cpumask_t cpumask, unsigned int irq) { unsigned long map = *cpus_addr(cpumask); - writel(map << 16 | irq, gic_dist_base + GIC_DIST_SOFTINT); + /* this always happens on GIC0 */ + writel(map << 16 | irq, gic_data[0].dist_base + GIC_DIST_SOFTINT); } #endif diff --git a/arch/arm/mach-realview/platsmp.c b/arch/arm/mach-realview/platsmp.c index b8484e15dac..709a9b1ac63 100644 --- a/arch/arm/mach-realview/platsmp.c +++ b/arch/arm/mach-realview/platsmp.c @@ -52,7 +52,7 @@ void __cpuinit platform_secondary_init(unsigned int cpu) * core (e.g. timer irq), then they will not have been enabled * for us: do so */ - gic_cpu_init(__io_address(REALVIEW_GIC_CPU_BASE)); + gic_cpu_init(0, __io_address(REALVIEW_GIC_CPU_BASE)); /* * let the primary processor know we're out of the diff --git a/arch/arm/mach-realview/realview_eb.c b/arch/arm/mach-realview/realview_eb.c index 9741b4d3c9c..b6a6f68cb69 100644 --- a/arch/arm/mach-realview/realview_eb.c +++ b/arch/arm/mach-realview/realview_eb.c @@ -143,8 +143,8 @@ static void __init gic_init_irq(void) writel(pldctrl, __io_address(REALVIEW_SYS_BASE) + 0xd8); writel(0x00000000, __io_address(REALVIEW_SYS_LOCK)); #endif - gic_dist_init(__io_address(REALVIEW_GIC_DIST_BASE)); - gic_cpu_init(__io_address(REALVIEW_GIC_CPU_BASE)); + gic_dist_init(0, __io_address(REALVIEW_GIC_DIST_BASE), 29); + gic_cpu_init(0, __io_address(REALVIEW_GIC_CPU_BASE)); } static void __init realview_eb_init(void) diff --git a/include/asm-arm/hardware/gic.h b/include/asm-arm/hardware/gic.h index 3fa5eb70f64..966e428ad32 100644 --- a/include/asm-arm/hardware/gic.h +++ b/include/asm-arm/hardware/gic.h @@ -33,8 +33,9 @@ #define GIC_DIST_SOFTINT 0xf00 #ifndef __ASSEMBLY__ -void gic_dist_init(void __iomem *base); -void gic_cpu_init(void __iomem *base); +void gic_dist_init(unsigned int gic_nr, void __iomem *base, unsigned int irq_start); +void gic_cpu_init(unsigned int gic_nr, void __iomem *base); +void gic_cascade_irq(unsigned int gic_nr, unsigned int irq); void gic_raise_softirq(cpumask_t cpumask, unsigned int irq); #endif -- cgit v1.2.3-70-g09d2