From 55a8173cfe1c6b489f8f5705282c762aed2e265e Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Sun, 18 Sep 2011 22:40:00 -0400 Subject: ARM: move initialization of the high_memory variable earlier Some upcoming changes must know the VMALLOC_START value, which is based on high_memory, before bootmem_init() is called. The best location to set it is in sanity_check_meminfo() where the needed computation is already done, and in the non MMU case it is trivial to do now that the meminfo array is already sorted at that point. Signed-off-by: Nicolas Pitre --- arch/arm/mm/mmu.c | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/arm/mm/mmu.c') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index dc8c550e6cb..0aa8b7d5b21 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -860,6 +860,7 @@ void __init sanity_check_meminfo(void) } #endif meminfo.nr_banks = j; + high_memory = __va(lowmem_limit - 1) + 1; memblock_set_current_limit(lowmem_limit); } -- cgit v1.2.3-70-g09d2 From 0536bdf33faff4d940ac094c77998cfac368cfff Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 25 Aug 2011 00:35:59 -0400 Subject: ARM: move iotable mappings within the vmalloc region In order to remove the build time variation between different SOCs with regards to VMALLOC_END, the iotable mappings are now allocated inside the vmalloc region. This allows for VMALLOC_END to be identical across all machines. The value for VMALLOC_END is now set to 0xff000000 which is right where the consistent DMA area starts. To accommodate all static mappings on machines with possible highmem usage, the default vmalloc area size is changed to 240 MB so that VMALLOC_START is no higher than 0xf0000000 by default. Signed-off-by: Nicolas Pitre Tested-by: Stephen Warren Tested-by: Kevin Hilman Tested-by: Jamie Iles --- Documentation/arm/memory.txt | 11 +++++----- arch/arm/include/asm/pgtable.h | 8 +------ arch/arm/mm/mmu.c | 49 ++++++++++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 27 deletions(-) (limited to 'arch/arm/mm/mmu.c') diff --git a/Documentation/arm/memory.txt b/Documentation/arm/memory.txt index 771d48d3b33..208a2d465b9 100644 --- a/Documentation/arm/memory.txt +++ b/Documentation/arm/memory.txt @@ -51,15 +51,14 @@ ffc00000 ffefffff DMA memory mapping region. Memory returned ff000000 ffbfffff Reserved for future expansion of DMA mapping region. -VMALLOC_END feffffff Free for platform use, recommended. - VMALLOC_END must be aligned to a 2MB - boundary. - VMALLOC_START VMALLOC_END-1 vmalloc() / ioremap() space. Memory returned by vmalloc/ioremap will be dynamically placed in this region. - VMALLOC_START may be based upon the value - of the high_memory variable. + Machine specific static mappings are also + located here through iotable_init(). + VMALLOC_START is based upon the value + of the high_memory variable, and VMALLOC_END + is equal to 0xff000000. PAGE_OFFSET high_memory-1 Kernel direct-mapped RAM region. This maps the platforms RAM, and typically diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 9451dce3a55..6cdd55cb0b8 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -21,7 +21,6 @@ #else #include -#include #include #include @@ -33,15 +32,10 @@ * any out-of-bounds memory accesses will hopefully be caught. * The vmalloc() routines leaves a hole of 4kB between each vmalloced * area for the same reason. ;) - * - * Note that platforms may override VMALLOC_START, but they must provide - * VMALLOC_END. VMALLOC_END defines the (exclusive) limit of this space, - * which may not overlap IO space. */ -#ifndef VMALLOC_START #define VMALLOC_OFFSET (8*1024*1024) #define VMALLOC_START (((unsigned long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)) -#endif +#define VMALLOC_END 0xff000000UL #define LIBRARY_TEXT_START 0x0c000000 diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 0aa8b7d5b21..c61481577ae 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -529,13 +530,18 @@ EXPORT_SYMBOL(phys_mem_access_prot); #define vectors_base() (vectors_high() ? 0xffff0000 : 0) -static void __init *early_alloc(unsigned long sz) +static void __init *early_alloc_aligned(unsigned long sz, unsigned long align) { - void *ptr = __va(memblock_alloc(sz, sz)); + void *ptr = __va(memblock_alloc(sz, align)); memset(ptr, 0, sz); return ptr; } +static void __init *early_alloc(unsigned long sz) +{ + return early_alloc_aligned(sz, sz); +} + static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot) { if (pmd_none(*pmd)) { @@ -685,9 +691,10 @@ static void __init create_mapping(struct map_desc *md) } if ((md->type == MT_DEVICE || md->type == MT_ROM) && - md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) { + md->virtual >= PAGE_OFFSET && + (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) { printk(KERN_WARNING "BUG: mapping for 0x%08llx" - " at 0x%08lx overlaps vmalloc space\n", + " at 0x%08lx out of vmalloc space\n", (long long)__pfn_to_phys((u64)md->pfn), md->virtual); } @@ -729,18 +736,32 @@ static void __init create_mapping(struct map_desc *md) */ void __init iotable_init(struct map_desc *io_desc, int nr) { - int i; + struct map_desc *md; + struct vm_struct *vm; + + if (!nr) + return; - for (i = 0; i < nr; i++) - create_mapping(io_desc + i); + vm = early_alloc_aligned(sizeof(*vm) * nr, __alignof__(*vm)); + + for (md = io_desc; nr; md++, nr--) { + create_mapping(md); + vm->addr = (void *)(md->virtual & PAGE_MASK); + vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK)); + vm->phys_addr = __pfn_to_phys(md->pfn); + vm->flags = VM_IOREMAP; + vm->caller = iotable_init; + vm_area_add_early(vm++); + } } -static void * __initdata vmalloc_min = (void *)(VMALLOC_END - SZ_128M); +static void * __initdata vmalloc_min = + (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET); /* * vmalloc=size forces the vmalloc area to be exactly 'size' * bytes. This can be used to increase (or decrease) the vmalloc - * area - the default is 128m. + * area - the default is 240m. */ static int __init early_vmalloc(char *arg) { @@ -891,10 +912,10 @@ static inline void prepare_page_table(void) /* * Clear out all the kernel space mappings, except for the first - * memory bank, up to the end of the vmalloc region. + * memory bank, up to the vmalloc region. */ for (addr = __phys_to_virt(end); - addr < VMALLOC_END; addr += PMD_SIZE) + addr < VMALLOC_START; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); } @@ -921,8 +942,8 @@ void __init arm_mm_memblock_reserve(void) } /* - * Set up device the mappings. Since we clear out the page tables for all - * mappings above VMALLOC_END, we will remove any debug device mappings. + * Set up the device mappings. Since we clear out the page tables for all + * mappings above VMALLOC_START, we will remove any debug device mappings. * This means you have to be careful how you debug this function, or any * called function. This means you can't use any function or debugging * method which may touch any device, otherwise the kernel _will_ crash. @@ -937,7 +958,7 @@ static void __init devicemaps_init(struct machine_desc *mdesc) */ vectors_page = early_alloc(PAGE_SIZE); - for (addr = VMALLOC_END; addr; addr += PMD_SIZE) + for (addr = VMALLOC_START; addr; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); /* -- cgit v1.2.3-70-g09d2 From 576d2f2525612ecb5af029a76f21f22a3b82563d Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Fri, 16 Sep 2011 01:14:23 -0400 Subject: ARM: add generic ioremap optimization by reusing static mappings Now that we have all the static mappings from iotable_init() located in the vmalloc area, it is trivial to optimize ioremap by reusing those static mappings when the requested physical area fits in one of them, and so in a generic way for all platforms. Signed-off-by: Nicolas Pitre Tested-by: Stephen Warren Tested-by: Kevin Hilman Tested-by: Jamie Iles --- arch/arm/mm/ioremap.c | 72 ++++++++++++++++++++++++++++++++++----------------- arch/arm/mm/mm.h | 14 ++++++++++ arch/arm/mm/mmu.c | 3 ++- 3 files changed, 64 insertions(+), 25 deletions(-) (limited to 'arch/arm/mm/mmu.c') diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index bc7d9bd766d..12c7ad215ce 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -36,12 +36,6 @@ #include #include "mm.h" -/* - * Used by ioremap() and iounmap() code to mark (super)section-mapped - * I/O regions in vm_struct->flags field. - */ -#define VM_ARM_SECTION_MAPPING 0x80000000 - int ioremap_page(unsigned long virt, unsigned long phys, const struct mem_type *mtype) { @@ -201,12 +195,6 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn, if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK)) return NULL; - /* - * Don't allow RAM to be mapped - this causes problems with ARMv6+ - */ - if (WARN_ON(pfn_valid(pfn))) - return NULL; - type = get_mem_type(mtype); if (!type) return NULL; @@ -216,6 +204,34 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn, */ size = PAGE_ALIGN(offset + size); + /* + * Try to reuse one of the static mapping whenever possible. + */ + read_lock(&vmlist_lock); + for (area = vmlist; area; area = area->next) { + if (!size || (sizeof(phys_addr_t) == 4 && pfn >= 0x100000)) + break; + if (!(area->flags & VM_ARM_STATIC_MAPPING)) + continue; + if ((area->flags & VM_ARM_MTYPE_MASK) != VM_ARM_MTYPE(mtype)) + continue; + if (__phys_to_pfn(area->phys_addr) > pfn || + __pfn_to_phys(pfn) + size-1 > area->phys_addr + area->size-1) + continue; + /* we can drop the lock here as we know *area is static */ + read_unlock(&vmlist_lock); + addr = (unsigned long)area->addr; + addr += __pfn_to_phys(pfn) - area->phys_addr; + return (void __iomem *) (offset + addr); + } + read_unlock(&vmlist_lock); + + /* + * Don't allow RAM to be mapped - this causes problems with ARMv6+ + */ + if (WARN_ON(pfn_valid(pfn))) + return NULL; + area = get_vm_area_caller(size, VM_IOREMAP, caller); if (!area) return NULL; @@ -313,26 +329,34 @@ __arm_ioremap_exec(unsigned long phys_addr, size_t size, bool cached) void __iounmap(volatile void __iomem *io_addr) { void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr); -#ifndef CONFIG_SMP struct vm_struct *vm; - /* - * If this is a section based mapping we need to handle it - * specially as the VM subsystem does not know how to handle - * such a beast. - */ read_lock(&vmlist_lock); for (vm = vmlist; vm; vm = vm->next) { - if ((vm->flags & VM_IOREMAP) && (vm->addr == addr)) { - if (vm->flags & VM_ARM_SECTION_MAPPING) { - unmap_area_sections((unsigned long)vm->addr, - vm->size); - } + if (vm->addr > addr) break; + if (!(vm->flags & VM_IOREMAP)) + continue; + /* If this is a static mapping we must leave it alone */ + if ((vm->flags & VM_ARM_STATIC_MAPPING) && + (vm->addr <= addr) && (vm->addr + vm->size > addr)) { + read_unlock(&vmlist_lock); + return; } +#ifndef CONFIG_SMP + /* + * If this is a section based mapping we need to handle it + * specially as the VM subsystem does not know how to handle + * such a beast. + */ + if ((vm->addr == addr) && + (vm->flags & VM_ARM_SECTION_MAPPING)) { + unmap_area_sections((unsigned long)vm->addr, vm->size); + break; + } +#endif } read_unlock(&vmlist_lock); -#endif vunmap(addr); } diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index ad7cce3bc43..70f6d3ea483 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -21,6 +21,20 @@ const struct mem_type *get_mem_type(unsigned int type); extern void __flush_dcache_page(struct address_space *mapping, struct page *page); +/* + * ARM specific vm_struct->flags bits. + */ + +/* (super)section-mapped I/O regions used by ioremap()/iounmap() */ +#define VM_ARM_SECTION_MAPPING 0x80000000 + +/* permanent static mappings from iotable_init() */ +#define VM_ARM_STATIC_MAPPING 0x40000000 + +/* mapping type (attributes) for permanent static mappings */ +#define VM_ARM_MTYPE(mt) ((mt) << 20) +#define VM_ARM_MTYPE_MASK (0x1f << 20) + #endif #ifdef CONFIG_ZONE_DMA diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index c61481577ae..27e366af67f 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -749,7 +749,8 @@ void __init iotable_init(struct map_desc *io_desc, int nr) vm->addr = (void *)(md->virtual & PAGE_MASK); vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK)); vm->phys_addr = __pfn_to_phys(md->pfn); - vm->flags = VM_IOREMAP; + vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING; + vm->flags |= VM_ARM_MTYPE(md->type); vm->caller = iotable_init; vm_area_add_early(vm++); } -- cgit v1.2.3-70-g09d2 From 1b6ba46b7efa31055eb993a6f2c6bbcb8b35b001 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 22 Nov 2011 17:30:29 +0000 Subject: ARM: LPAE: MMU setup for the 3-level page table format This patch adds the MMU initialisation for the LPAE page table format. The swapper_pg_dir size with LPAE is 5 rather than 4 pages. A new proc-v7-3level.S file contains the TTB initialisation, context switch and PTE setting code with the LPAE. The TTBRx split is based on the PAGE_OFFSET with TTBR1 used for the kernel mappings. The 36-bit mappings (supersections) and a few other memory types in mmu.c are conditionally compiled. Signed-off-by: Catalin Marinas --- arch/arm/kernel/head.S | 45 ++++++++++++- arch/arm/mm/mmu.c | 30 +++++++++ arch/arm/mm/proc-macros.S | 5 +- arch/arm/mm/proc-v7-3level.S | 150 +++++++++++++++++++++++++++++++++++++++++++ arch/arm/mm/proc-v7.S | 25 +++++--- 5 files changed, 243 insertions(+), 12 deletions(-) create mode 100644 arch/arm/mm/proc-v7-3level.S (limited to 'arch/arm/mm/mmu.c') diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 54eb94aff6c..c8e797baaf5 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -39,8 +39,14 @@ #error KERNEL_RAM_VADDR must start at 0xXXXX8000 #endif +#ifdef CONFIG_ARM_LPAE + /* LPAE requires an additional page for the PGD */ +#define PG_DIR_SIZE 0x5000 +#define PMD_ORDER 3 +#else #define PG_DIR_SIZE 0x4000 #define PMD_ORDER 2 +#endif .globl swapper_pg_dir .equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE @@ -164,6 +170,25 @@ __create_page_tables: teq r0, r6 bne 1b +#ifdef CONFIG_ARM_LPAE + /* + * Build the PGD table (first level) to point to the PMD table. A PGD + * entry is 64-bit wide. + */ + mov r0, r4 + add r3, r4, #0x1000 @ first PMD table address + orr r3, r3, #3 @ PGD block type + mov r6, #4 @ PTRS_PER_PGD + mov r7, #1 << (55 - 32) @ L_PGD_SWAPPER +1: str r3, [r0], #4 @ set bottom PGD entry bits + str r7, [r0], #4 @ set top PGD entry bits + add r3, r3, #0x1000 @ next PMD table + subs r6, r6, #1 + bne 1b + + add r4, r4, #0x1000 @ point to the PMD tables +#endif + ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags /* @@ -219,8 +244,8 @@ __create_page_tables: #endif /* - * Then map boot params address in r2 or - * the first 1MB of ram if boot params address is not specified. + * Then map boot params address in r2 or the first 1MB (2MB with LPAE) + * of ram if boot params address is not specified. */ mov r0, r2, lsr #SECTION_SHIFT movs r0, r0, lsl #SECTION_SHIFT @@ -251,7 +276,15 @@ __create_page_tables: mov r3, r7, lsr #SECTION_SHIFT ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags orr r3, r7, r3, lsl #SECTION_SHIFT +#ifdef CONFIG_ARM_LPAE + mov r7, #1 << (54 - 32) @ XN +#else + orr r3, r3, #PMD_SECT_XN +#endif 1: str r3, [r0], #4 +#ifdef CONFIG_ARM_LPAE + str r7, [r0], #4 +#endif add r3, r3, #1 << SECTION_SHIFT cmp r0, r6 blo 1b @@ -282,6 +315,9 @@ __create_page_tables: add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ORDER) str r3, [r0] #endif +#endif +#ifdef CONFIG_ARM_LPAE + sub r4, r4, #0x1000 @ point to the PGD table #endif mov pc, lr ENDPROC(__create_page_tables) @@ -374,12 +410,17 @@ __enable_mmu: #ifdef CONFIG_CPU_ICACHE_DISABLE bic r0, r0, #CR_I #endif +#ifdef CONFIG_ARM_LPAE + mov r5, #0 + mcrr p15, 0, r4, r5, c2 @ load TTBR0 +#else mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ domain_val(DOMAIN_IO, DOMAIN_CLIENT)) mcr p15, 0, r5, c3, c0, 0 @ load domain access register mcr p15, 0, r4, c2, c0, 0 @ load page table pointer +#endif b __turn_mmu_on ENDPROC(__enable_mmu) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index dc8c550e6cb..1935311e17f 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -150,6 +150,7 @@ static int __init early_nowrite(char *__unused) } early_param("nowb", early_nowrite); +#ifndef CONFIG_ARM_LPAE static int __init early_ecc(char *p) { if (memcmp(p, "on", 2) == 0) @@ -159,6 +160,7 @@ static int __init early_ecc(char *p) return 0; } early_param("ecc", early_ecc); +#endif static int __init noalign_setup(char *__unused) { @@ -228,10 +230,12 @@ static struct mem_type mem_types[] = { .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, .domain = DOMAIN_KERNEL, }, +#ifndef CONFIG_ARM_LPAE [MT_MINICLEAN] = { .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE, .domain = DOMAIN_KERNEL, }, +#endif [MT_LOW_VECTORS] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_RDONLY, @@ -429,6 +433,7 @@ static void __init build_mem_type_table(void) * ARMv6 and above have extended page tables. */ if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { +#ifndef CONFIG_ARM_LPAE /* * Mark cache clean areas and XIP ROM read only * from SVC mode and no access from userspace. @@ -436,6 +441,7 @@ static void __init build_mem_type_table(void) mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; +#endif if (is_smp()) { /* @@ -474,6 +480,18 @@ static void __init build_mem_type_table(void) mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE; } +#ifdef CONFIG_ARM_LPAE + /* + * Do not generate access flag faults for the kernel mappings. + */ + for (i = 0; i < ARRAY_SIZE(mem_types); i++) { + mem_types[i].prot_pte |= PTE_EXT_AF; + mem_types[i].prot_sect |= PMD_SECT_AF; + } + kern_pgprot |= PTE_EXT_AF; + vecs_pgprot |= PTE_EXT_AF; +#endif + for (i = 0; i < 16; i++) { unsigned long v = pgprot_val(protection_map[i]); protection_map[i] = __pgprot(v | user_pgprot); @@ -572,8 +590,10 @@ static void __init alloc_init_section(pud_t *pud, unsigned long addr, if (((addr | end | phys) & ~SECTION_MASK) == 0) { pmd_t *p = pmd; +#ifndef CONFIG_ARM_LPAE if (addr & SECTION_SIZE) pmd++; +#endif do { *pmd = __pmd(phys | type->prot_sect); @@ -603,6 +623,7 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, } while (pud++, addr = next, addr != end); } +#ifndef CONFIG_ARM_LPAE static void __init create_36bit_mapping(struct map_desc *md, const struct mem_type *type) { @@ -662,6 +683,7 @@ static void __init create_36bit_mapping(struct map_desc *md, pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT; } while (addr != end); } +#endif /* !CONFIG_ARM_LPAE */ /* * Create the page directory entries and any necessary @@ -693,6 +715,7 @@ static void __init create_mapping(struct map_desc *md) type = &mem_types[md->type]; +#ifndef CONFIG_ARM_LPAE /* * Catch 36-bit addresses */ @@ -700,6 +723,7 @@ static void __init create_mapping(struct map_desc *md) create_36bit_mapping(md, type); return; } +#endif addr = md->virtual & PAGE_MASK; phys = __pfn_to_phys(md->pfn); @@ -897,7 +921,13 @@ static inline void prepare_page_table(void) pmd_clear(pmd_off_k(addr)); } +#ifdef CONFIG_ARM_LPAE +/* the first page is reserved for pgd */ +#define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \ + PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t)) +#else #define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) +#endif /* * Reserve the special regions of memory diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index 307a4def8d3..2d8ff3ad86d 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -91,8 +91,9 @@ #if L_PTE_SHARED != PTE_EXT_SHARED #error PTE shared bit mismatch #endif -#if (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\ - L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED +#if !defined (CONFIG_ARM_LPAE) && \ + (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\ + L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED #error Invalid Linux PTE bit settings #endif #endif /* CONFIG_MMU */ diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S new file mode 100644 index 00000000000..8de0f1dd154 --- /dev/null +++ b/arch/arm/mm/proc-v7-3level.S @@ -0,0 +1,150 @@ +/* + * arch/arm/mm/proc-v7-3level.S + * + * Copyright (C) 2001 Deep Blue Solutions Ltd. + * Copyright (C) 2011 ARM Ltd. + * Author: Catalin Marinas + * based on arch/arm/mm/proc-v7-2level.S + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#define TTB_IRGN_NC (0 << 8) +#define TTB_IRGN_WBWA (1 << 8) +#define TTB_IRGN_WT (2 << 8) +#define TTB_IRGN_WB (3 << 8) +#define TTB_RGN_NC (0 << 10) +#define TTB_RGN_OC_WBWA (1 << 10) +#define TTB_RGN_OC_WT (2 << 10) +#define TTB_RGN_OC_WB (3 << 10) +#define TTB_S (3 << 12) +#define TTB_EAE (1 << 31) + +/* PTWs cacheable, inner WB not shareable, outer WB not shareable */ +#define TTB_FLAGS_UP (TTB_IRGN_WB|TTB_RGN_OC_WB) +#define PMD_FLAGS_UP (PMD_SECT_WB) + +/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */ +#define TTB_FLAGS_SMP (TTB_IRGN_WBWA|TTB_S|TTB_RGN_OC_WBWA) +#define PMD_FLAGS_SMP (PMD_SECT_WBWA|PMD_SECT_S) + +/* + * cpu_v7_switch_mm(pgd_phys, tsk) + * + * Set the translation table base pointer to be pgd_phys (physical address of + * the new TTB). + */ +ENTRY(cpu_v7_switch_mm) +#ifdef CONFIG_MMU + ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id + and r3, r1, #0xff + mov r3, r3, lsl #(48 - 32) @ ASID + mcrr p15, 0, r0, r3, c2 @ set TTB 0 + isb +#endif + mov pc, lr +ENDPROC(cpu_v7_switch_mm) + +/* + * cpu_v7_set_pte_ext(ptep, pte) + * + * Set a level 2 translation table entry. + * - ptep - pointer to level 3 translation table entry + * - pte - PTE value to store (64-bit in r2 and r3) + */ +ENTRY(cpu_v7_set_pte_ext) +#ifdef CONFIG_MMU + tst r2, #L_PTE_PRESENT + beq 1f + tst r3, #1 << (55 - 32) @ L_PTE_DIRTY + orreq r2, #L_PTE_RDONLY +1: strd r2, r3, [r0] + mcr p15, 0, r0, c7, c10, 1 @ flush_pte +#endif + mov pc, lr +ENDPROC(cpu_v7_set_pte_ext) + + /* + * Memory region attributes for LPAE (defined in pgtable-3level.h): + * + * n = AttrIndx[2:0] + * + * n MAIR + * UNCACHED 000 00000000 + * BUFFERABLE 001 01000100 + * DEV_WC 001 01000100 + * WRITETHROUGH 010 10101010 + * WRITEBACK 011 11101110 + * DEV_CACHED 011 11101110 + * DEV_SHARED 100 00000100 + * DEV_NONSHARED 100 00000100 + * unused 101 + * unused 110 + * WRITEALLOC 111 11111111 + */ +.equ PRRR, 0xeeaa4400 @ MAIR0 +.equ NMRR, 0xff000004 @ MAIR1 + + /* + * Macro for setting up the TTBRx and TTBCR registers. + * - \ttbr1 updated. + */ + .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp + ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address + cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? (branch below) + mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register + orr \tmp, \tmp, #TTB_EAE + ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP) + ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP) + ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP << 16) + ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP << 16) + /* + * TTBR0/TTBR1 split (PAGE_OFFSET): + * 0x40000000: T0SZ = 2, T1SZ = 0 (not used) + * 0x80000000: T0SZ = 0, T1SZ = 1 + * 0xc0000000: T0SZ = 0, T1SZ = 2 + * + * Only use this feature if PHYS_OFFSET <= PAGE_OFFSET, otherwise + * booting secondary CPUs would end up using TTBR1 for the identity + * mapping set up in TTBR0. + */ + bhi 9001f @ PHYS_OFFSET > PAGE_OFFSET? + orr \tmp, \tmp, #(((PAGE_OFFSET >> 30) - 1) << 16) @ TTBCR.T1SZ +#if defined CONFIG_VMSPLIT_2G + /* PAGE_OFFSET == 0x80000000, T1SZ == 1 */ + add \ttbr1, \ttbr1, #1 << 4 @ skip two L1 entries +#elif defined CONFIG_VMSPLIT_3G + /* PAGE_OFFSET == 0xc0000000, T1SZ == 2 */ + add \ttbr1, \ttbr1, #4096 * (1 + 3) @ only L2 used, skip pgd+3*pmd +#endif + /* CONFIG_VMSPLIT_1G does not need TTBR1 adjustment */ +9001: mcr p15, 0, \tmp, c2, c0, 2 @ TTB control register + mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1 + .endm + + __CPUINIT + + /* + * AT + * TFR EV X F IHD LR S + * .EEE ..EE PUI. .TAT 4RVI ZWRS BLDP WCAM + * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced + * 11 0 110 1 0011 1100 .111 1101 < we want + */ + .align 2 + .type v7_crval, #object +v7_crval: + crval clear=0x0120c302, mmuset=0x30c23c7d, ucset=0x00c01c7c + + .previous diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index ed1a4d11533..7efa2a721d5 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -19,7 +19,11 @@ #include "proc-macros.S" +#ifdef CONFIG_ARM_LPAE +#include "proc-v7-3level.S" +#else #include "proc-v7-2level.S" +#endif ENTRY(cpu_v7_proc_init) mov pc, lr @@ -87,7 +91,7 @@ ENDPROC(cpu_v7_dcache_clean_area) /* Suspend/resume support: derived from arch/arm/mach-s5pv210/sleep.S */ .globl cpu_v7_suspend_size -.equ cpu_v7_suspend_size, 4 * 7 +.equ cpu_v7_suspend_size, 4 * 8 #ifdef CONFIG_ARM_CPU_SUSPEND ENTRY(cpu_v7_do_suspend) stmfd sp!, {r4 - r10, lr} @@ -96,10 +100,11 @@ ENTRY(cpu_v7_do_suspend) stmia r0!, {r4 - r5} mrc p15, 0, r6, c3, c0, 0 @ Domain ID mrc p15, 0, r7, c2, c0, 1 @ TTB 1 + mrc p15, 0, r11, c2, c0, 2 @ TTB control register mrc p15, 0, r8, c1, c0, 0 @ Control register mrc p15, 0, r9, c1, c0, 1 @ Auxiliary control register mrc p15, 0, r10, c1, c0, 2 @ Co-processor access control - stmia r0, {r6 - r10} + stmia r0, {r6 - r11} ldmfd sp!, {r4 - r10, pc} ENDPROC(cpu_v7_do_suspend) @@ -111,13 +116,15 @@ ENTRY(cpu_v7_do_resume) ldmia r0!, {r4 - r5} mcr p15, 0, r4, c13, c0, 0 @ FCSE/PID mcr p15, 0, r5, c13, c0, 3 @ User r/o thread ID - ldmia r0, {r6 - r10} + ldmia r0, {r6 - r11} mcr p15, 0, r6, c3, c0, 0 @ Domain ID +#ifndef CONFIG_ARM_LPAE ALT_SMP(orr r1, r1, #TTB_FLAGS_SMP) ALT_UP(orr r1, r1, #TTB_FLAGS_UP) +#endif mcr p15, 0, r1, c2, c0, 0 @ TTB 0 mcr p15, 0, r7, c2, c0, 1 @ TTB 1 - mcr p15, 0, ip, c2, c0, 2 @ TTB control register + mcr p15, 0, r11, c2, c0, 2 @ TTB control register mrc p15, 0, r4, c1, c0, 1 @ Read Auxiliary control register teq r4, r9 @ Is it already set? mcrne p15, 0, r9, c1, c0, 1 @ No, so write it @@ -291,11 +298,11 @@ __v7_setup_stack: */ .macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0 ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ - PMD_FLAGS_SMP | \mm_mmuflags) + PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags) ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ - PMD_FLAGS_UP | \mm_mmuflags) - .long PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_AP_WRITE | \ - PMD_SECT_AP_READ | \io_mmuflags + PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags) + .long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags W(b) \initfunc .long cpu_arch_name .long cpu_elf_name @@ -308,6 +315,7 @@ __v7_setup_stack: .long v7_cache_fns .endm +#ifndef CONFIG_ARM_LPAE /* * ARM Ltd. Cortex A5 processor. */ @@ -327,6 +335,7 @@ __v7_ca9mp_proc_info: .long 0xff0ffff0 __v7_proc __v7_ca9mp_setup .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info +#endif /* CONFIG_ARM_LPAE */ /* * ARM Ltd. Cortex A15 processor. -- cgit v1.2.3-70-g09d2 From 77f73a2c8e869b035e71eea5cae07c30fe4bded0 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 22 Nov 2011 17:30:32 +0000 Subject: ARM: LPAE: mark memory banks with start > ULONG_MAX as highmem Memory banks living outside of the 32-bit physical address space do not have a 1:1 pa <-> va mapping and therefore the __va macro may wrap. This patch ensures that such banks are marked as highmem so that the Kernel doesn't try to split them up when it sees that the wrapped virtual address overlaps the vmalloc space. Signed-off-by: Will Deacon Signed-off-by: Catalin Marinas Acked-by: Nicolas Pitre --- arch/arm/mm/mmu.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'arch/arm/mm/mmu.c') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 1935311e17f..b836d6b2258 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -799,6 +799,9 @@ void __init sanity_check_meminfo(void) struct membank *bank = &meminfo.bank[j]; *bank = meminfo.bank[i]; + if (bank->start > ULONG_MAX) + highmem = 1; + #ifdef CONFIG_HIGHMEM if (__va(bank->start) >= vmalloc_min || __va(bank->start) < (void *)PAGE_OFFSET) @@ -810,7 +813,7 @@ void __init sanity_check_meminfo(void) * Split those memory banks which are partially overlapping * the vmalloc area greatly simplifying things later. */ - if (__va(bank->start) < vmalloc_min && + if (!highmem && __va(bank->start) < vmalloc_min && bank->size > vmalloc_min - __va(bank->start)) { if (meminfo.nr_banks >= NR_BANKS) { printk(KERN_CRIT "NR_BANKS too low, " @@ -830,6 +833,17 @@ void __init sanity_check_meminfo(void) #else bank->highmem = highmem; + /* + * Highmem banks not allowed with !CONFIG_HIGHMEM. + */ + if (highmem) { + printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx " + "(!CONFIG_HIGHMEM).\n", + (unsigned long long)bank->start, + (unsigned long long)bank->start + bank->size - 1); + continue; + } + /* * Check whether this memory bank would entirely overlap * the vmalloc area. -- cgit v1.2.3-70-g09d2