From 9a6655e49fd98f3748bb80da20705448aad9ee57 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 31 Aug 2010 13:05:22 +0100 Subject: ARM: Improve the L2 cache performance when PL310 is used With this L2 cache controller, the cache maintenance by PA and sync operations are atomic and do not require a "wait" loop. This patch conditionally defines the cache_wait() function. Since L2x0 cache controllers do not work with ARMv7 CPUs, the patch automatically enables CACHE_PL310 when only CPU_V7 is defined. Signed-off-by: Catalin Marinas --- arch/arm/mm/Kconfig | 8 ++++++++ arch/arm/mm/cache-l2x0.c | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index a0a2928ae4d..4414a01e1e8 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -779,6 +779,14 @@ config CACHE_L2X0 help This option enables the L2x0 PrimeCell. +config CACHE_PL310 + bool + depends on CACHE_L2X0 + default y if CPU_V7 && !CPU_V6 + help + This option enables optimisations for the PL310 cache + controller. + config CACHE_TAUROS2 bool "Enable the Tauros2 L2 cache controller" depends on (ARCH_DOVE || ARCH_MMP) diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 9982eb385c0..edb43ff7aee 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -29,13 +29,22 @@ static void __iomem *l2x0_base; static DEFINE_SPINLOCK(l2x0_lock); static uint32_t l2x0_way_mask; /* Bitmask of active ways */ -static inline void cache_wait(void __iomem *reg, unsigned long mask) +static inline void cache_wait_way(void __iomem *reg, unsigned long mask) { - /* wait for the operation to complete */ + /* wait for cache operation by line or way to complete */ while (readl_relaxed(reg) & mask) ; } +#ifdef CONFIG_CACHE_PL310 +static inline void cache_wait(void __iomem *reg, unsigned long mask) +{ + /* cache operations by line are atomic on PL310 */ +} +#else +#define cache_wait cache_wait_way +#endif + static inline void cache_sync(void) { void __iomem *base = l2x0_base; @@ -110,7 +119,7 @@ static inline void l2x0_inv_all(void) /* invalidate all ways */ spin_lock_irqsave(&l2x0_lock, flags); writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY); - cache_wait(l2x0_base + L2X0_INV_WAY, l2x0_way_mask); + cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask); cache_sync(); spin_unlock_irqrestore(&l2x0_lock, flags); } -- cgit v1.2.3-70-g09d2 From 2fd8658931193599c867fd6974fa184ec34af16c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner <[tglx@linutronix.de]> Date: Sat, 31 Jul 2010 21:05:24 +0530 Subject: arm: Implement l2x0 cache disable functions Add flush_all, inv_all and disable functions to the l2x0 code. These functions are called from kexec code to prevent random crashes in the new kernel. Platforms like OMAP which control L2 enable/disable via SMI mode can override the outer_cache.disable() function to implement their own. Signed-off-by: Thomas Gleixner Acked-by: Catalin Marinas Acked-by: Linus Walleij --- arch/arm/mm/cache-l2x0.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index edb43ff7aee..9310d618070 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -112,12 +112,26 @@ static void l2x0_cache_sync(void) spin_unlock_irqrestore(&l2x0_lock, flags); } -static inline void l2x0_inv_all(void) +static void l2x0_flush_all(void) +{ + unsigned long flags; + + /* clean all ways */ + spin_lock_irqsave(&l2x0_lock, flags); + writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_INV_WAY); + cache_wait_way(l2x0_base + L2X0_CLEAN_INV_WAY, l2x0_way_mask); + cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); +} + +static void l2x0_inv_all(void) { unsigned long flags; /* invalidate all ways */ spin_lock_irqsave(&l2x0_lock, flags); + /* Invalidating when L2 is enabled is a nono */ + BUG_ON(readl(l2x0_base + L2X0_CTRL) & 1); writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_INV_WAY); cache_wait_way(l2x0_base + L2X0_INV_WAY, l2x0_way_mask); cache_sync(); @@ -215,6 +229,15 @@ static void l2x0_flush_range(unsigned long start, unsigned long end) spin_unlock_irqrestore(&l2x0_lock, flags); } +static void l2x0_disable(void) +{ + unsigned long flags; + + spin_lock_irqsave(&l2x0_lock, flags); + writel(0, l2x0_base + L2X0_CTRL); + spin_unlock_irqrestore(&l2x0_lock, flags); +} + void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) { __u32 aux; @@ -272,6 +295,9 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) outer_cache.clean_range = l2x0_clean_range; outer_cache.flush_range = l2x0_flush_range; outer_cache.sync = l2x0_cache_sync; + outer_cache.flush_all = l2x0_flush_all; + outer_cache.inv_all = l2x0_inv_all; + outer_cache.disable = l2x0_disable; printk(KERN_INFO "%s cache controller enabled\n", type); printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n", -- cgit v1.2.3-70-g09d2 From 5ba70372289a1fb378b95cee2cf46b0203d65291 Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Sun, 11 Jul 2010 14:35:37 +0530 Subject: ARM: l2x0: Determine the cache size The cache size is needed for to optimise range based maintainance operations Signed-off-by: Santosh Shilimkar Acked-by: Catalin Marinas Acked-by: Linus Walleij --- arch/arm/include/asm/hardware/cache-l2x0.h | 1 + arch/arm/mm/cache-l2x0.c | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/include/asm/hardware/cache-l2x0.h b/arch/arm/include/asm/hardware/cache-l2x0.h index d833355569c..4633d2a8817 100644 --- a/arch/arm/include/asm/hardware/cache-l2x0.h +++ b/arch/arm/include/asm/hardware/cache-l2x0.h @@ -55,6 +55,7 @@ #define L2X0_CACHE_ID_PART_MASK (0xf << 6) #define L2X0_CACHE_ID_PART_L210 (1 << 6) #define L2X0_CACHE_ID_PART_L310 (3 << 6) +#define L2X0_AUX_CTRL_WAY_SIZE_MASK (0x3 << 17) #ifndef __ASSEMBLY__ extern void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask); diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 9310d618070..262c7529bcd 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -28,6 +28,7 @@ static void __iomem *l2x0_base; static DEFINE_SPINLOCK(l2x0_lock); static uint32_t l2x0_way_mask; /* Bitmask of active ways */ +static uint32_t l2x0_size; static inline void cache_wait_way(void __iomem *reg, unsigned long mask) { @@ -242,6 +243,7 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) { __u32 aux; __u32 cache_id; + __u32 way_size = 0; int ways; const char *type; @@ -275,6 +277,13 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) l2x0_way_mask = (1 << ways) - 1; + /* + * L2 cache Size = Way size * Number of ways + */ + way_size = (aux & L2X0_AUX_CTRL_WAY_SIZE_MASK) >> 17; + way_size = 1 << (way_size + 3); + l2x0_size = ways * way_size * SZ_1K; + /* * Check if l2x0 controller is already enabled. * If you are booting from non-secure mode @@ -300,6 +309,6 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) outer_cache.disable = l2x0_disable; printk(KERN_INFO "%s cache controller enabled\n", type); - printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n", - ways, cache_id, aux); + printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d B\n", + ways, cache_id, aux, l2x0_size); } -- cgit v1.2.3-70-g09d2 From 444457c1f59d58bc48acf5b4fc585225106c11ff Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Sun, 11 Jul 2010 14:58:41 +0530 Subject: ARM: l2x0: Optimise the range based operations For the big buffers which are in excess of cache size, the maintaince operations by PA are very slow. For such buffers the maintainace operations can be speeded up by using the WAY based method. Signed-off-by: Santosh Shilimkar Acked-by: Catalin Marinas Acked-by: Linus Walleij --- arch/arm/mm/cache-l2x0.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 262c7529bcd..170c9bb9586 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -125,6 +125,18 @@ static void l2x0_flush_all(void) spin_unlock_irqrestore(&l2x0_lock, flags); } +static void l2x0_clean_all(void) +{ + unsigned long flags; + + /* clean all ways */ + spin_lock_irqsave(&l2x0_lock, flags); + writel_relaxed(l2x0_way_mask, l2x0_base + L2X0_CLEAN_WAY); + cache_wait_way(l2x0_base + L2X0_CLEAN_WAY, l2x0_way_mask); + cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); +} + static void l2x0_inv_all(void) { unsigned long flags; @@ -183,6 +195,11 @@ static void l2x0_clean_range(unsigned long start, unsigned long end) void __iomem *base = l2x0_base; unsigned long flags; + if ((end - start) >= l2x0_size) { + l2x0_clean_all(); + return; + } + spin_lock_irqsave(&l2x0_lock, flags); start &= ~(CACHE_LINE_SIZE - 1); while (start < end) { @@ -208,6 +225,11 @@ static void l2x0_flush_range(unsigned long start, unsigned long end) void __iomem *base = l2x0_base; unsigned long flags; + if ((end - start) >= l2x0_size) { + l2x0_flush_all(); + return; + } + spin_lock_irqsave(&l2x0_lock, flags); start &= ~(CACHE_LINE_SIZE - 1); while (start < end) { -- cgit v1.2.3-70-g09d2 From f444fce3c0aa199facc8c2be7635a4a82b84b71e Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Mon, 18 Oct 2010 09:03:03 +0100 Subject: ARM: 6445/1: fixup TCM memory types After Santosh's fixup of the generic MT_MEMORY and MT_MEMORY_NONCACHED I add this fix to the TCM memory types. The main change is that the ITCM memory is L_PTE_WRITE and DOMAIN_KERNEL which works just fine. The changed to the DTCM is just cosmetic to fit with surrounding code. Cc: Santosh Shilimkar Cc: Rickard Andersson Signed-off-by: Linus Walleij Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index c32f731d56d..d184d25e9a5 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -265,17 +265,17 @@ static struct mem_type mem_types[] = { .domain = DOMAIN_KERNEL, }, [MT_MEMORY_DTCM] = { - .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | - L_PTE_DIRTY | L_PTE_WRITE, - .prot_l1 = PMD_TYPE_TABLE, - .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, - .domain = DOMAIN_KERNEL, + .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | + L_PTE_WRITE, + .prot_l1 = PMD_TYPE_TABLE, + .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, + .domain = DOMAIN_KERNEL, }, [MT_MEMORY_ITCM] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | - L_PTE_USER | L_PTE_EXEC, + L_PTE_WRITE | L_PTE_EXEC, .prot_l1 = PMD_TYPE_TABLE, - .domain = DOMAIN_IO, + .domain = DOMAIN_KERNEL, }, }; -- cgit v1.2.3-70-g09d2 From 4e929d2bcf13eeaa9636448c55690b383a910391 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 18:09:59 +0100 Subject: ARM: fix memblock breakage Will says: | Commit e63075a3 removed the explicit MEMBLOCK_REAL_LIMIT #define | and introduced the requirement that arch code calls | memblock_set_current_limit to ensure that the __va macro can | be used on physical addresses returned from memblock_alloc. Unfortunately, ARM was missed out of this change. Fix this. Reported-by: Will Deacon Signed-off-by: Russell King --- arch/arm/include/asm/memblock.h | 7 ------- arch/arm/mm/mmu.c | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/include/asm/memblock.h b/arch/arm/include/asm/memblock.h index fdbc43b2e6c..b8da2e415e4 100644 --- a/arch/arm/include/asm/memblock.h +++ b/arch/arm/include/asm/memblock.h @@ -1,13 +1,6 @@ #ifndef _ASM_ARM_MEMBLOCK_H #define _ASM_ARM_MEMBLOCK_H -#ifdef CONFIG_MMU -extern phys_addr_t lowmem_end_addr; -#define MEMBLOCK_REAL_LIMIT lowmem_end_addr -#else -#define MEMBLOCK_REAL_LIMIT 0 -#endif - struct meminfo; struct machine_desc; diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index d184d25e9a5..8e21b4e721e 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -745,13 +745,11 @@ static int __init early_vmalloc(char *arg) } early_param("vmalloc", early_vmalloc); -phys_addr_t lowmem_end_addr; - static void __init sanity_check_meminfo(void) { int i, j, highmem = 0; - lowmem_end_addr = __pa(vmalloc_min - 1) + 1; + memblock_set_current_limit(__pa(vmalloc_min - 1) + 1); for (i = 0, j = 0; i < meminfo.nr_banks; i++) { struct membank *bank = &meminfo.bank[j]; -- cgit v1.2.3-70-g09d2 From 4e54d93d3c9846ba1c2644ad06463dafa690d1b7 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Thu, 28 Oct 2010 11:45:22 +0100 Subject: ARM: 6464/2: fix spinlock recursion in adjust_pte() When running following code in a machine which has VIVT caches and USE_SPLIT_PTLOCKS is not defined: fd = open("/etc/passwd", O_RDONLY); addr = mmap(NULL, 4096, PROT_READ, MAP_SHARED, fd, 0); addr2 = mmap(NULL, 4096, PROT_READ, MAP_SHARED, fd, 0); v = *((int *)addr); we will hang in spinlock recursion in the page fault handler: BUG: spinlock recursion on CPU#0, mmap_test/717 lock: c5e295d8, .magic: dead4ead, .owner: mmap_test/717, .owner_cpu: 0 [] (unwind_backtrace+0x0/0xec) [] (do_raw_spin_lock+0x40/0x140) [] (update_mmu_cache+0x208/0x250) [] (__do_fault+0x320/0x3ec) [] (handle_mm_fault+0x2f0/0x6d8) [] (do_page_fault+0xdc/0x1cc) [] (do_DataAbort+0x34/0x94) This comes from the fact that when USE_SPLIT_PTLOCKS is not defined, the only lock protecting the page tables is mm->page_table_lock which is already locked before update_mmu_cache() is called. Signed-off-by: Mika Westerberg Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index c493d7244d3..83e59f87042 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -66,6 +66,30 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, return ret; } +#if USE_SPLIT_PTLOCKS +/* + * If we are using split PTE locks, then we need to take the page + * lock here. Otherwise we are using shared mm->page_table_lock + * which is already locked, thus cannot take it. + */ +static inline void do_pte_lock(spinlock_t *ptl) +{ + /* + * Use nested version here to indicate that we are already + * holding one similar spinlock. + */ + spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); +} + +static inline void do_pte_unlock(spinlock_t *ptl) +{ + spin_unlock(ptl); +} +#else /* !USE_SPLIT_PTLOCKS */ +static inline void do_pte_lock(spinlock_t *ptl) {} +static inline void do_pte_unlock(spinlock_t *ptl) {} +#endif /* USE_SPLIT_PTLOCKS */ + static int adjust_pte(struct vm_area_struct *vma, unsigned long address, unsigned long pfn) { @@ -90,11 +114,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, */ ptl = pte_lockptr(vma->vm_mm, pmd); pte = pte_offset_map(pmd, address); - spin_lock(ptl); + do_pte_lock(ptl); ret = do_adjust_pte(vma, address, pfn, pte); - spin_unlock(ptl); + do_pte_unlock(ptl); pte_unmap(pte); return ret; -- cgit v1.2.3-70-g09d2 From c8c90860cd3592fac83a349f84a20360a6498727 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Thu, 28 Oct 2010 11:27:40 +0100 Subject: ARM: 6466/1: implement flush_icache_all for the rest of the CPUs Commit 81d11955bf0 ("ARM: 6405/1: Handle __flush_icache_all for CONFIG_SMP_ON_UP") added a new function to struct cpu_cache_fns: flush_icache_all(). It also implemented this for v6 and v7 but not for v5 and backwards. Without the function pointer in place, we will be calling wrong cache functions. For example with ep93xx we get following: Unable to handle kernel paging request at virtual address ee070f38 pgd = c0004000 [ee070f38] *pgd=00000000 Internal error: Oops: 80000005 [#1] PREEMPT last sysfs file: Modules linked in: CPU: 0 Not tainted (2.6.36+ #1) PC is at 0xee070f38 LR is at __dma_alloc+0x11c/0x2d0 pc : [] lr : [] psr: 60000013 sp : c581bde0 ip : 00000000 fp : c0472000 r10: c0472000 r9 : 000000d0 r8 : 00020000 r7 : 0001ffff r6 : 00000000 r5 : c0472400 r4 : c5980000 r3 : c03ab7e0 r2 : 00000000 r1 : c59a0000 r0 : c5980000 Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment kernel Control: c000717f Table: c0004000 DAC: 00000017 Process swapper (pid: 1, stack limit = 0xc581a270) [] (__dma_alloc+0x11c/0x2d0) [] (dma_alloc_writecombine+0x1c/0x24) [] (ep93xx_pcm_preallocate_dma_buffer+0x44/0x60) [] (ep93xx_pcm_new+0x5c/0x88) [] (snd_soc_instantiate_cards+0x8a8/0xbc0) [] (soc_probe+0xfc/0x134) [] (platform_drv_probe+0x18/0x1c) [] (driver_probe_device+0xb0/0x16c) [] (bus_for_each_drv+0x48/0x84) [] (device_attach+0x50/0x68) [] (bus_probe_device+0x24/0x44) [] (device_add+0x2fc/0x44c) [] (platform_device_add+0x104/0x15c) [] (simone_init+0x60/0x94) [] (do_one_initcall+0xd0/0x1a4) __dma_alloc() calls (inlined) __dma_alloc_buffer() which ends up calling dmac_flush_range(). Now since the entries in the arm920_cache_fns are shifted by one, we jump into address 0xee070f38 which is actually next instruction after the arm920_cache_fns structure. So implement flush_icache_all() for the rest of the supported CPUs using a generic 'invalidate I cache' instruction. Signed-off-by: Mika Westerberg Signed-off-by: Russell King --- arch/arm/mm/cache-fa.S | 12 ++++++++++++ arch/arm/mm/cache-v3.S | 10 ++++++++++ arch/arm/mm/cache-v4.S | 10 ++++++++++ arch/arm/mm/cache-v4wb.S | 12 ++++++++++++ arch/arm/mm/cache-v4wt.S | 12 ++++++++++++ arch/arm/mm/proc-arm1020.S | 15 +++++++++++++++ arch/arm/mm/proc-arm1020e.S | 15 +++++++++++++++ arch/arm/mm/proc-arm1022.S | 15 +++++++++++++++ arch/arm/mm/proc-arm1026.S | 15 +++++++++++++++ arch/arm/mm/proc-arm920.S | 12 ++++++++++++ arch/arm/mm/proc-arm922.S | 12 ++++++++++++ arch/arm/mm/proc-arm925.S | 12 ++++++++++++ arch/arm/mm/proc-arm926.S | 12 ++++++++++++ arch/arm/mm/proc-arm940.S | 12 ++++++++++++ arch/arm/mm/proc-arm946.S | 12 ++++++++++++ arch/arm/mm/proc-feroceon.S | 13 +++++++++++++ arch/arm/mm/proc-xsc3.S | 12 ++++++++++++ arch/arm/mm/proc-xscale.S | 12 ++++++++++++ 18 files changed, 225 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S index 7148e53e607..1fa6f71470d 100644 --- a/arch/arm/mm/cache-fa.S +++ b/arch/arm/mm/cache-fa.S @@ -37,6 +37,17 @@ /* FIXME: put optimal value here. Current one is just estimation */ #define CACHE_DLIMIT (CACHE_DSIZE * 2) +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(fa_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(fa_flush_icache_all) + /* * flush_user_cache_all() * @@ -233,6 +244,7 @@ ENDPROC(fa_dma_unmap_area) .type fa_cache_fns, #object ENTRY(fa_cache_fns) + .long fa_flush_icache_all .long fa_flush_kern_cache_all .long fa_flush_user_cache_all .long fa_flush_user_cache_range diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S index c2ff3c599fe..2e2bc406a18 100644 --- a/arch/arm/mm/cache-v3.S +++ b/arch/arm/mm/cache-v3.S @@ -12,6 +12,15 @@ #include #include "proc-macros.S" +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(v3_flush_icache_all) + mov pc, lr +ENDPROC(v3_flush_icache_all) + /* * flush_user_cache_all() * @@ -122,6 +131,7 @@ ENDPROC(v3_dma_map_area) .type v3_cache_fns, #object ENTRY(v3_cache_fns) + .long v3_flush_icache_all .long v3_flush_kern_cache_all .long v3_flush_user_cache_all .long v3_flush_user_cache_range diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S index 4810f7e3e81..a8fefb523f1 100644 --- a/arch/arm/mm/cache-v4.S +++ b/arch/arm/mm/cache-v4.S @@ -12,6 +12,15 @@ #include #include "proc-macros.S" +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(v4_flush_icache_all) + mov pc, lr +ENDPROC(v4_flush_icache_all) + /* * flush_user_cache_all() * @@ -134,6 +143,7 @@ ENDPROC(v4_dma_map_area) .type v4_cache_fns, #object ENTRY(v4_cache_fns) + .long v4_flush_icache_all .long v4_flush_kern_cache_all .long v4_flush_user_cache_all .long v4_flush_user_cache_range diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S index df8368afa10..d3644db467b 100644 --- a/arch/arm/mm/cache-v4wb.S +++ b/arch/arm/mm/cache-v4wb.S @@ -50,6 +50,17 @@ flush_base: .long FLUSH_BASE .text +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(v4wb_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(v4wb_flush_icache_all) + /* * flush_user_cache_all() * @@ -244,6 +255,7 @@ ENDPROC(v4wb_dma_unmap_area) .type v4wb_cache_fns, #object ENTRY(v4wb_cache_fns) + .long v4wb_flush_icache_all .long v4wb_flush_kern_cache_all .long v4wb_flush_user_cache_all .long v4wb_flush_user_cache_range diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S index 45c70312f43..49c2b66cf3d 100644 --- a/arch/arm/mm/cache-v4wt.S +++ b/arch/arm/mm/cache-v4wt.S @@ -40,6 +40,17 @@ */ #define CACHE_DLIMIT 16384 +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(v4wt_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(v4wt_flush_icache_all) + /* * flush_user_cache_all() * @@ -188,6 +199,7 @@ ENDPROC(v4wt_dma_map_area) .type v4wt_cache_fns, #object ENTRY(v4wt_cache_fns) + .long v4wt_flush_icache_all .long v4wt_flush_kern_cache_all .long v4wt_flush_user_cache_all .long v4wt_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index a6f5f8475b9..bcf748d9f4e 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -119,6 +119,20 @@ ENTRY(cpu_arm1020_do_idle) /* ================================= CACHE ================================ */ .align 5 + +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm1020_flush_icache_all) +#ifndef CONFIG_CPU_ICACHE_DISABLE + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache +#endif + mov pc, lr +ENDPROC(arm1020_flush_icache_all) + /* * flush_user_cache_all() * @@ -351,6 +365,7 @@ ENTRY(arm1020_dma_unmap_area) ENDPROC(arm1020_dma_unmap_area) ENTRY(arm1020_cache_fns) + .long arm1020_flush_icache_all .long arm1020_flush_kern_cache_all .long arm1020_flush_user_cache_all .long arm1020_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S index afc06b9c313..ab7ec26657e 100644 --- a/arch/arm/mm/proc-arm1020e.S +++ b/arch/arm/mm/proc-arm1020e.S @@ -119,6 +119,20 @@ ENTRY(cpu_arm1020e_do_idle) /* ================================= CACHE ================================ */ .align 5 + +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm1020e_flush_icache_all) +#ifndef CONFIG_CPU_ICACHE_DISABLE + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache +#endif + mov pc, lr +ENDPROC(arm1020e_flush_icache_all) + /* * flush_user_cache_all() * @@ -337,6 +351,7 @@ ENTRY(arm1020e_dma_unmap_area) ENDPROC(arm1020e_dma_unmap_area) ENTRY(arm1020e_cache_fns) + .long arm1020e_flush_icache_all .long arm1020e_flush_kern_cache_all .long arm1020e_flush_user_cache_all .long arm1020e_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S index 8915e0ba3fe..831c5e54e22 100644 --- a/arch/arm/mm/proc-arm1022.S +++ b/arch/arm/mm/proc-arm1022.S @@ -108,6 +108,20 @@ ENTRY(cpu_arm1022_do_idle) /* ================================= CACHE ================================ */ .align 5 + +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm1022_flush_icache_all) +#ifndef CONFIG_CPU_ICACHE_DISABLE + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache +#endif + mov pc, lr +ENDPROC(arm1022_flush_icache_all) + /* * flush_user_cache_all() * @@ -326,6 +340,7 @@ ENTRY(arm1022_dma_unmap_area) ENDPROC(arm1022_dma_unmap_area) ENTRY(arm1022_cache_fns) + .long arm1022_flush_icache_all .long arm1022_flush_kern_cache_all .long arm1022_flush_user_cache_all .long arm1022_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S index ff446c5d476..e3f7e9a166b 100644 --- a/arch/arm/mm/proc-arm1026.S +++ b/arch/arm/mm/proc-arm1026.S @@ -108,6 +108,20 @@ ENTRY(cpu_arm1026_do_idle) /* ================================= CACHE ================================ */ .align 5 + +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm1026_flush_icache_all) +#ifndef CONFIG_CPU_ICACHE_DISABLE + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache +#endif + mov pc, lr +ENDPROC(arm1026_flush_icache_all) + /* * flush_user_cache_all() * @@ -320,6 +334,7 @@ ENTRY(arm1026_dma_unmap_area) ENDPROC(arm1026_dma_unmap_area) ENTRY(arm1026_cache_fns) + .long arm1026_flush_icache_all .long arm1026_flush_kern_cache_all .long arm1026_flush_user_cache_all .long arm1026_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index fecf570939f..6109f278a90 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -109,6 +109,17 @@ ENTRY(cpu_arm920_do_idle) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm920_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(arm920_flush_icache_all) + /* * flush_user_cache_all() * @@ -305,6 +316,7 @@ ENTRY(arm920_dma_unmap_area) ENDPROC(arm920_dma_unmap_area) ENTRY(arm920_cache_fns) + .long arm920_flush_icache_all .long arm920_flush_kern_cache_all .long arm920_flush_user_cache_all .long arm920_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index e3cbf87c948..bb2f0f46a5e 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -111,6 +111,17 @@ ENTRY(cpu_arm922_do_idle) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm922_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(arm922_flush_icache_all) + /* * flush_user_cache_all() * @@ -307,6 +318,7 @@ ENTRY(arm922_dma_unmap_area) ENDPROC(arm922_dma_unmap_area) ENTRY(arm922_cache_fns) + .long arm922_flush_icache_all .long arm922_flush_kern_cache_all .long arm922_flush_user_cache_all .long arm922_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S index 572424c867b..c13e01accfe 100644 --- a/arch/arm/mm/proc-arm925.S +++ b/arch/arm/mm/proc-arm925.S @@ -144,6 +144,17 @@ ENTRY(cpu_arm925_do_idle) mcr p15, 0, r1, c1, c0, 0 @ Restore ICache enable mov pc, lr +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm925_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(arm925_flush_icache_all) + /* * flush_user_cache_all() * @@ -362,6 +373,7 @@ ENTRY(arm925_dma_unmap_area) ENDPROC(arm925_dma_unmap_area) ENTRY(arm925_cache_fns) + .long arm925_flush_icache_all .long arm925_flush_kern_cache_all .long arm925_flush_user_cache_all .long arm925_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 63d168b4ebe..42eb4315740 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -110,6 +110,17 @@ ENTRY(cpu_arm926_do_idle) msr cpsr_c, r3 @ Restore FIQ state mov pc, lr +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm926_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(arm926_flush_icache_all) + /* * flush_user_cache_all() * @@ -325,6 +336,7 @@ ENTRY(arm926_dma_unmap_area) ENDPROC(arm926_dma_unmap_area) ENTRY(arm926_cache_fns) + .long arm926_flush_icache_all .long arm926_flush_kern_cache_all .long arm926_flush_user_cache_all .long arm926_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S index f6a62822418..7b11cdb9935 100644 --- a/arch/arm/mm/proc-arm940.S +++ b/arch/arm/mm/proc-arm940.S @@ -67,6 +67,17 @@ ENTRY(cpu_arm940_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt mov pc, lr +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm940_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(arm940_flush_icache_all) + /* * flush_user_cache_all() */ @@ -254,6 +265,7 @@ ENTRY(arm940_dma_unmap_area) ENDPROC(arm940_dma_unmap_area) ENTRY(arm940_cache_fns) + .long arm940_flush_icache_all .long arm940_flush_kern_cache_all .long arm940_flush_user_cache_all .long arm940_flush_user_cache_range diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S index ea2e7f2eb95..1a5bbf08034 100644 --- a/arch/arm/mm/proc-arm946.S +++ b/arch/arm/mm/proc-arm946.S @@ -74,6 +74,17 @@ ENTRY(cpu_arm946_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt mov pc, lr +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(arm946_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(arm946_flush_icache_all) + /* * flush_user_cache_all() */ @@ -296,6 +307,7 @@ ENTRY(arm946_dma_unmap_area) ENDPROC(arm946_dma_unmap_area) ENTRY(arm946_cache_fns) + .long arm946_flush_icache_all .long arm946_flush_kern_cache_all .long arm946_flush_user_cache_all .long arm946_flush_user_cache_range diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index 578da69200c..b4597edbff9 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -123,6 +123,17 @@ ENTRY(cpu_feroceon_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt mov pc, lr +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(feroceon_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(feroceon_flush_icache_all) + /* * flush_user_cache_all() * @@ -401,6 +412,7 @@ ENTRY(feroceon_dma_unmap_area) ENDPROC(feroceon_dma_unmap_area) ENTRY(feroceon_cache_fns) + .long feroceon_flush_icache_all .long feroceon_flush_kern_cache_all .long feroceon_flush_user_cache_all .long feroceon_flush_user_cache_range @@ -412,6 +424,7 @@ ENTRY(feroceon_cache_fns) .long feroceon_dma_flush_range ENTRY(feroceon_range_cache_fns) + .long feroceon_flush_icache_all .long feroceon_flush_kern_cache_all .long feroceon_flush_user_cache_all .long feroceon_flush_user_cache_range diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index cad07e40304..ec26355cb7c 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -140,6 +140,17 @@ ENTRY(cpu_xsc3_do_idle) /* ================================= CACHE ================================ */ +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(xsc3_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(xsc3_flush_icache_all) + /* * flush_user_cache_all() * @@ -325,6 +336,7 @@ ENTRY(xsc3_dma_unmap_area) ENDPROC(xsc3_dma_unmap_area) ENTRY(xsc3_cache_fns) + .long xsc3_flush_icache_all .long xsc3_flush_kern_cache_all .long xsc3_flush_user_cache_all .long xsc3_flush_user_cache_range diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index cb245edb2c2..523408c0bb3 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -180,6 +180,17 @@ ENTRY(cpu_xscale_do_idle) /* ================================= CACHE ================================ */ +/* + * flush_icache_all() + * + * Unconditionally clean and invalidate the entire icache. + */ +ENTRY(xscale_flush_icache_all) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache + mov pc, lr +ENDPROC(xscale_flush_icache_all) + /* * flush_user_cache_all() * @@ -397,6 +408,7 @@ ENTRY(xscale_dma_unmap_area) ENDPROC(xscale_dma_unmap_area) ENTRY(xscale_cache_fns) + .long xscale_flush_icache_all .long xscale_flush_kern_cache_all .long xscale_flush_user_cache_all .long xscale_flush_user_cache_range -- cgit v1.2.3-70-g09d2 From 7dc50ec7283391dd7a29a80e2a0fb76731a6a7c7 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 18:14:56 +0100 Subject: ARM: ensure membank array is always sorted This was missing from the noMMU code, so there was the possibility of things not working as expected if out of order memory information was passed. Signed-off-by: Russell King --- arch/arm/mm/init.c | 10 ++++++++++ arch/arm/mm/mmu.c | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 7fd9b5eb177..759878355a0 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -256,10 +257,19 @@ static void arm_memory_present(void) } #endif +static int __init meminfo_cmp(const void *_a, const void *_b) +{ + const struct membank *a = _a, *b = _b; + long cmp = bank_pfn_start(a) - bank_pfn_start(b); + return cmp < 0 ? -1 : cmp > 0 ? 1 : 0; +} + void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc) { int i; + sort(&meminfo.bank, meminfo.nr_banks, sizeof(meminfo.bank[0]), meminfo_cmp, NULL); + memblock_init(); for (i = 0; i < mi->nr_banks; i++) memblock_add(mi->bank[i].start, mi->bank[i].size); diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 8e21b4e721e..5222d385096 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -1011,13 +1010,6 @@ static void __init map_lowmem(void) } } -static int __init meminfo_cmp(const void *_a, const void *_b) -{ - const struct membank *a = _a, *b = _b; - long cmp = bank_pfn_start(a) - bank_pfn_start(b); - return cmp < 0 ? -1 : cmp > 0 ? 1 : 0; -} - /* * paging_init() sets up the page tables, initialises the zone memory * maps, and sets up the zero page, bad page and bad page tables. @@ -1026,8 +1018,6 @@ void __init paging_init(struct machine_desc *mdesc) { void *zero_page; - sort(&meminfo.bank, meminfo.nr_banks, sizeof(meminfo.bank[0]), meminfo_cmp, NULL); - build_mem_type_table(); sanity_check_meminfo(); prepare_page_table(); -- cgit v1.2.3-70-g09d2 From a2c54d2af848fafd53f0314947d0741313205dbc Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 19:17:31 +0100 Subject: ARM: memblock: use memblock when initializing memory allocators Signed-off-by: Russell King --- arch/arm/mm/init.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 759878355a0..f68c93bb1fd 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -192,11 +192,11 @@ static void __init arm_bootmem_init(struct meminfo *mi, } } -static void __init arm_bootmem_free(struct meminfo *mi, unsigned long min, - unsigned long max_low, unsigned long max_high) +static void __init arm_bootmem_free(unsigned long min, unsigned long max_low, + unsigned long max_high) { unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; - int i; + struct memblock_region *reg; /* * initialise the zones. @@ -218,13 +218,20 @@ static void __init arm_bootmem_free(struct meminfo *mi, unsigned long min, * holes = node_size - sum(bank_sizes) */ memcpy(zhole_size, zone_size, sizeof(zhole_size)); - for_each_bank(i, mi) { - int idx = 0; + for_each_memblock(memory, reg) { + unsigned long start = memblock_region_memory_base_pfn(reg); + unsigned long end = memblock_region_memory_end_pfn(reg); + + if (start < max_low) { + unsigned long low_end = min(end, max_low); + zhole_size[0] -= low_end - start; + } #ifdef CONFIG_HIGHMEM - if (mi->bank[i].highmem) - idx = ZONE_HIGHMEM; + if (end > max_low) { + unsigned long high_start = max(start, max_low); + zhole_size[ZONE_HIGHMEM] -= end - high_start; + } #endif - zhole_size[idx] -= bank_pfn_size(&mi->bank[i]); } /* @@ -327,7 +334,7 @@ void __init bootmem_init(void) * the sparse mem_map arrays initialized by sparse_init() * for memmap_init_zone(), otherwise all PFNs are invalid. */ - arm_bootmem_free(mi, min, max_low, max_high); + arm_bootmem_free(min, max_low, max_high); high_memory = __va((max_low << PAGE_SHIFT) - 1) + 1; -- cgit v1.2.3-70-g09d2 From a801d2764033063b313d0c55704de8fb36e1efd1 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 19:27:44 +0100 Subject: ARM: memblock: use memblock to free memory into arm_bootmem_init() Switch arm_bootmem_init() to use memblock instead of membank to free memory into bootmem. Signed-off-by: Russell King --- arch/arm/mm/init.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index f68c93bb1fd..8e1edbc6116 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -148,14 +148,13 @@ static void __init find_limits(struct meminfo *mi, } } -static void __init arm_bootmem_init(struct meminfo *mi, - unsigned long start_pfn, unsigned long end_pfn) +static void __init arm_bootmem_init(unsigned long start_pfn, + unsigned long end_pfn) { struct memblock_region *reg; unsigned int boot_pages; phys_addr_t bitmap; pg_data_t *pgdat; - int i; /* * Allocate the bootmem bitmap page. This must be in a region @@ -173,22 +172,31 @@ static void __init arm_bootmem_init(struct meminfo *mi, pgdat = NODE_DATA(0); init_bootmem_node(pgdat, __phys_to_pfn(bitmap), start_pfn, end_pfn); - for_each_bank(i, mi) { - struct membank *bank = &mi->bank[i]; - if (!bank->highmem) - free_bootmem(bank_phys_start(bank), bank_phys_size(bank)); + /* Free the lowmem regions from memblock into bootmem. */ + for_each_memblock(memory, reg) { + unsigned long start = memblock_region_memory_base_pfn(reg); + unsigned long end = memblock_region_memory_end_pfn(reg); + + if (end >= end_pfn) + end = end_pfn; + if (start >= end) + break; + + free_bootmem(__pfn_to_phys(start), (end - start) << PAGE_SHIFT); } - /* - * Reserve the memblock reserved regions in bootmem. - */ + /* Reserve the lowmem memblock reserved regions in bootmem. */ for_each_memblock(reserved, reg) { - phys_addr_t start = memblock_region_reserved_base_pfn(reg); - phys_addr_t end = memblock_region_reserved_end_pfn(reg); - if (start >= start_pfn && end <= end_pfn) - reserve_bootmem_node(pgdat, __pfn_to_phys(start), - (end - start) << PAGE_SHIFT, - BOOTMEM_DEFAULT); + unsigned long start = memblock_region_reserved_base_pfn(reg); + unsigned long end = memblock_region_reserved_end_pfn(reg); + + if (end >= end_pfn) + end = end_pfn; + if (start >= end) + break; + + reserve_bootmem(__pfn_to_phys(start), + (end - start) << PAGE_SHIFT, BOOTMEM_DEFAULT); } } @@ -316,7 +324,7 @@ void __init bootmem_init(void) find_limits(mi, &min, &max_low, &max_high); - arm_bootmem_init(mi, min, max_low); + arm_bootmem_init(min, max_low); /* * Sparsemem tries to allocate bootmem in memory_present(), -- cgit v1.2.3-70-g09d2 From 47ea3c15498154f634c304e08dee284efdd7dceb Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 19:35:29 +0100 Subject: ARM: memblock: convert memory detail printing to use memblock Signed-off-by: Russell King --- arch/arm/mm/init.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 8e1edbc6116..58b90ad4949 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -444,6 +444,7 @@ static void __init free_unused_memmap(struct meminfo *mi) void __init mem_init(void) { unsigned long reserved_pages, free_pages; + struct memblock_region *reg; int i; #ifdef CONFIG_HAVE_TCM /* These pointers are filled in on TCM detection */ @@ -503,9 +504,11 @@ void __init mem_init(void) */ printk(KERN_INFO "Memory:"); num_physpages = 0; - for (i = 0; i < meminfo.nr_banks; i++) { - num_physpages += bank_pfn_size(&meminfo.bank[i]); - printk(" %ldMB", bank_phys_size(&meminfo.bank[i]) >> 20); + for_each_memblock(memory, reg) { + unsigned long pages = memblock_region_memory_end_pfn(reg) - + memblock_region_memory_base_pfn(reg); + num_physpages += pages; + printk(" %ldMB", pages >> (20 - PAGE_SHIFT)); } printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT)); -- cgit v1.2.3-70-g09d2 From d0e775afb94d9b61ba6c63299169ef7a87b68189 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 19:37:06 +0100 Subject: ARM: move freeing of highmem pages out of mem_init() Signed-off-by: Russell King --- arch/arm/mm/init.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 58b90ad4949..1b4e0abf113 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -436,6 +436,22 @@ static void __init free_unused_memmap(struct meminfo *mi) } } +static void __init free_highpages(void) +{ +#ifdef CONFIG_HIGHMEM + int i; + + /* set highmem page free */ + for_each_bank (i, &meminfo) { + unsigned long start = bank_pfn_start(&meminfo.bank[i]); + unsigned long end = bank_pfn_end(&meminfo.bank[i]); + if (start >= max_low_pfn + PHYS_PFN_OFFSET) + totalhigh_pages += free_area(start, end, NULL); + } + totalram_pages += totalhigh_pages; +#endif +} + /* * mem_init() marks the free areas in the mem_map and tells us how much * memory is free. This is done after various parts of the system have @@ -465,16 +481,7 @@ void __init mem_init(void) __phys_to_pfn(__pa(swapper_pg_dir)), NULL); #endif -#ifdef CONFIG_HIGHMEM - /* set highmem page free */ - for_each_bank (i, &meminfo) { - unsigned long start = bank_pfn_start(&meminfo.bank[i]); - unsigned long end = bank_pfn_end(&meminfo.bank[i]); - if (start >= max_low_pfn + PHYS_PFN_OFFSET) - totalhigh_pages += free_area(start, end, NULL); - } - totalram_pages += totalhigh_pages; -#endif + free_highpages(); reserved_pages = free_pages = 0; -- cgit v1.2.3-70-g09d2 From df4f14c7b22e43e67c0e4e3b005ff897a0a72f4d Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 19:45:49 +0100 Subject: ARM: memblock: convert free_highpages() to use memblock Free the high pages using the memblock memory lists - and more importantly, exclude any memblock allocations in highmem from the free'd memory. Signed-off-by: Russell King --- arch/arm/mm/init.c | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 1b4e0abf113..5422821d372 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -439,13 +439,47 @@ static void __init free_unused_memmap(struct meminfo *mi) static void __init free_highpages(void) { #ifdef CONFIG_HIGHMEM - int i; + unsigned long max_low = max_low_pfn + PHYS_PFN_OFFSET; + struct memblock_region *mem, *res; /* set highmem page free */ - for_each_bank (i, &meminfo) { - unsigned long start = bank_pfn_start(&meminfo.bank[i]); - unsigned long end = bank_pfn_end(&meminfo.bank[i]); - if (start >= max_low_pfn + PHYS_PFN_OFFSET) + for_each_memblock(memory, mem) { + unsigned long start = memblock_region_memory_base_pfn(mem); + unsigned long end = memblock_region_memory_end_pfn(mem); + + /* Ignore complete lowmem entries */ + if (end <= max_low) + continue; + + /* Truncate partial highmem entries */ + if (start < max_low) + start = max_low; + + /* Find and exclude any reserved regions */ + for_each_memblock(reserved, res) { + unsigned long res_start, res_end; + + res_start = memblock_region_reserved_base_pfn(res); + res_end = memblock_region_reserved_end_pfn(res); + + if (res_end < start) + continue; + if (res_start < start) + res_start = start; + if (res_start > end) + res_start = end; + if (res_end > end) + res_end = end; + if (res_start != start) + totalhigh_pages += free_area(start, res_start, + NULL); + start = res_end; + if (start == end) + break; + } + + /* And now free anything which remains */ + if (start < end) totalhigh_pages += free_area(start, end, NULL); } totalram_pages += totalhigh_pages; -- cgit v1.2.3-70-g09d2 From f25b4b4c89ff118df72421dd2cb080a6380896ac Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 19:49:33 +0100 Subject: ARM: memblock: move meminfo into find_limits directly bootmem_init() no longer makes several uses of the membank information, so move this into the one remaining called function which does use it. Signed-off-by: Russell King --- arch/arm/mm/init.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 5422821d372..5164069ced4 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -122,9 +122,10 @@ void show_mem(void) printk("%d pages swap cached\n", cached); } -static void __init find_limits(struct meminfo *mi, - unsigned long *min, unsigned long *max_low, unsigned long *max_high) +static void __init find_limits(unsigned long *min, unsigned long *max_low, + unsigned long *max_high) { + struct meminfo *mi = &meminfo; int i; *min = -1UL; @@ -317,12 +318,11 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc) void __init bootmem_init(void) { - struct meminfo *mi = &meminfo; unsigned long min, max_low, max_high; max_low = max_high = 0; - find_limits(mi, &min, &max_low, &max_high); + find_limits(&min, &max_low, &max_high); arm_bootmem_init(min, max_low); -- cgit v1.2.3-70-g09d2 From 8df6516864462cb7a6f87d5a46df68fb0faebbb5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 27 Oct 2010 19:57:38 +0100 Subject: ARM: memblock: setup lowmem mappings using memblock Use memblock information to setup lowmem mappings rather than the membank array. This allows platforms to manipulate the memblock information during initialization to reserve (and remove) memory from the kernel's view of memory - and thus allowing platforms to setup their own private mappings for this memory without causing problems with multiple aliasing mappings: size = min(size, SZ_2M); base = memblock_alloc(size, min(align, SZ_2M)); memblock_free(base, size); memblock_remove(base, size); This is needed because multiple mappings of regions with differing attributes (sharability, type, cache) are not permitted with ARMv6 and above. Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 49 +++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 20 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 5222d385096..72ad3e1f56c 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -744,11 +744,14 @@ static int __init early_vmalloc(char *arg) } early_param("vmalloc", early_vmalloc); +static phys_addr_t lowmem_limit __initdata = 0; + static void __init sanity_check_meminfo(void) { int i, j, highmem = 0; - memblock_set_current_limit(__pa(vmalloc_min - 1) + 1); + lowmem_limit = __pa(vmalloc_min - 1) + 1; + memblock_set_current_limit(lowmem_limit); for (i = 0, j = 0; i < meminfo.nr_banks; i++) { struct membank *bank = &meminfo.bank[j]; @@ -849,6 +852,7 @@ static void __init sanity_check_meminfo(void) static inline void prepare_page_table(void) { unsigned long addr; + phys_addr_t end; /* * Clear out all the mappings below the kernel image. @@ -863,11 +867,18 @@ static inline void prepare_page_table(void) for ( ; addr < PAGE_OFFSET; addr += PGDIR_SIZE) pmd_clear(pmd_off_k(addr)); + /* + * Find the end of the first block of lowmem. + */ + end = memblock.memory.regions[0].base + memblock.memory.regions[0].size; + if (end >= lowmem_limit) + end = lowmem_limit; + /* * Clear out all the kernel space mappings, except for the first * memory bank, up to the end of the vmalloc region. */ - for (addr = __phys_to_virt(bank_phys_end(&meminfo.bank[0])); + for (addr = __phys_to_virt(end); addr < VMALLOC_END; addr += PGDIR_SIZE) pmd_clear(pmd_off_k(addr)); } @@ -984,29 +995,27 @@ static void __init kmap_init(void) #endif } -static inline void map_memory_bank(struct membank *bank) -{ - struct map_desc map; - - map.pfn = bank_pfn_start(bank); - map.virtual = __phys_to_virt(bank_phys_start(bank)); - map.length = bank_phys_size(bank); - map.type = MT_MEMORY; - - create_mapping(&map); -} - static void __init map_lowmem(void) { - struct meminfo *mi = &meminfo; - int i; + struct memblock_region *reg; /* Map all the lowmem memory banks. */ - for (i = 0; i < mi->nr_banks; i++) { - struct membank *bank = &mi->bank[i]; + for_each_memblock(memory, reg) { + phys_addr_t start = reg->base; + phys_addr_t end = start + reg->size; + struct map_desc map; + + if (end > lowmem_limit) + end = lowmem_limit; + if (start >= end) + break; + + map.pfn = __phys_to_pfn(start); + map.virtual = __phys_to_virt(start); + map.length = end - start; + map.type = MT_MEMORY; - if (!bank->highmem) - map_memory_bank(bank); + create_mapping(&map); } } -- cgit v1.2.3-70-g09d2