diff options
Diffstat (limited to 'arch/arm/mm')
| -rw-r--r-- | arch/arm/mm/Kconfig | 21 | ||||
| -rw-r--r-- | arch/arm/mm/Makefile | 1 | ||||
| -rw-r--r-- | arch/arm/mm/cache-l2x0.c | 5 | ||||
| -rw-r--r-- | arch/arm/mm/cache-v6.S | 18 | ||||
| -rw-r--r-- | arch/arm/mm/discontig.c | 45 | ||||
| -rw-r--r-- | arch/arm/mm/dma-mapping.c | 18 | ||||
| -rw-r--r-- | arch/arm/mm/init.c | 392 | ||||
| -rw-r--r-- | arch/arm/mm/mm.h | 4 | ||||
| -rw-r--r-- | arch/arm/mm/mmu.c | 161 | ||||
| -rw-r--r-- | arch/arm/mm/nommu.c | 22 |
10 files changed, 239 insertions, 448 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 71d5d5efcee..87ec141fcaa 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -724,6 +724,25 @@ config NEEDS_SYSCALL_FOR_CMPXCHG Forget about fast user space cmpxchg support. It is just not possible. +config DMA_CACHE_RWFO + bool "Enable read/write for ownership DMA cache maintenance" + depends on CPU_V6 && SMP + default y + help + The Snoop Control Unit on ARM11MPCore does not detect the + cache maintenance operations and the dma_{map,unmap}_area() + functions may leave stale cache entries on other CPUs. By + enabling this option, Read or Write For Ownership in the ARMv6 + DMA cache maintenance functions is performed. These LDR/STR + instructions change the cache line state to shared or modified + so that the cache operation has the desired effect. + + Note that the workaround is only valid on processors that do + not perform speculative loads into the D-cache. For such + processors, if cache maintenance operations are not broadcast + in hardware, other workarounds are needed (e.g. cache + maintenance broadcasting in software via FIQ). + config OUTER_CACHE bool @@ -783,6 +802,8 @@ config ARM_L1_CACHE_SHIFT config ARM_DMA_MEM_BUFFERABLE bool "Use non-cacheable memory for DMA" if CPU_V6 && !CPU_V7 + depends on !(MACH_REALVIEW_PB1176 || REALVIEW_EB_ARM11MP || \ + MACH_REALVIEW_PB11MP) default y if CPU_V6 || CPU_V7 help Historically, the kernel has used strongly ordered mappings to diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index e8d34a80851..d63b6c41375 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -15,7 +15,6 @@ endif obj-$(CONFIG_MODULES) += proc-syms.o obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o -obj-$(CONFIG_DISCONTIGMEM) += discontig.o obj-$(CONFIG_HIGHMEM) += highmem.o obj-$(CONFIG_CPU_ABRT_NOMMU) += abort-nommu.o diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 9819869d2bc..df4955885b2 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -218,6 +218,9 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) cache_id = readl(l2x0_base + L2X0_CACHE_ID); aux = readl(l2x0_base + L2X0_AUX_CTRL); + aux &= aux_mask; + aux |= aux_val; + /* Determine the number of ways */ switch (cache_id & L2X0_CACHE_ID_PART_MASK) { case L2X0_CACHE_ID_PART_L310: @@ -248,8 +251,6 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) if (!(readl(l2x0_base + L2X0_CTRL) & 1)) { /* l2x0 controller is disabled */ - aux &= aux_mask; - aux |= aux_val; writel(aux, l2x0_base + L2X0_AUX_CTRL); l2x0_inv_all(); diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index e46ecd84713..86aa689ef1a 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -211,8 +211,9 @@ v6_dma_inv_range: mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line #endif 1: -#ifdef CONFIG_SMP - str r0, [r0] @ write for ownership +#ifdef CONFIG_DMA_CACHE_RWFO + ldr r2, [r0] @ read for ownership + str r2, [r0] @ write for ownership #endif #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c6, 1 @ invalidate D line @@ -234,7 +235,7 @@ v6_dma_inv_range: v6_dma_clean_range: bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: -#ifdef CONFIG_SMP +#ifdef CONFIG_DMA_CACHE_RWFO ldr r2, [r0] @ read for ownership #endif #ifdef HARVARD_CACHE @@ -257,7 +258,7 @@ v6_dma_clean_range: ENTRY(v6_dma_flush_range) bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: -#ifdef CONFIG_SMP +#ifdef CONFIG_DMA_CACHE_RWFO ldr r2, [r0] @ read for ownership str r2, [r0] @ write for ownership #endif @@ -283,9 +284,13 @@ ENTRY(v6_dma_map_area) add r1, r1, r0 teq r2, #DMA_FROM_DEVICE beq v6_dma_inv_range +#ifndef CONFIG_DMA_CACHE_RWFO + b v6_dma_clean_range +#else teq r2, #DMA_TO_DEVICE beq v6_dma_clean_range b v6_dma_flush_range +#endif ENDPROC(v6_dma_map_area) /* @@ -295,6 +300,11 @@ ENDPROC(v6_dma_map_area) * - dir - DMA direction */ ENTRY(v6_dma_unmap_area) +#ifndef CONFIG_DMA_CACHE_RWFO + add r1, r1, r0 + teq r2, #DMA_TO_DEVICE + bne v6_dma_inv_range +#endif mov pc, lr ENDPROC(v6_dma_unmap_area) diff --git a/arch/arm/mm/discontig.c b/arch/arm/mm/discontig.c deleted file mode 100644 index c8c0c4b0f0a..00000000000 --- a/arch/arm/mm/discontig.c +++ /dev/null @@ -1,45 +0,0 @@ -/* - * linux/arch/arm/mm/discontig.c - * - * Discontiguous memory support. - * - * Initial code: Copyright (C) 1999-2000 Nicolas Pitre - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#include <linux/module.h> -#include <linux/mmzone.h> -#include <linux/bootmem.h> - -#if MAX_NUMNODES != 4 && MAX_NUMNODES != 16 -# error Fix Me Please -#endif - -/* - * Our node_data structure for discontiguous memory. - */ - -pg_data_t discontig_node_data[MAX_NUMNODES] = { - { .bdata = &bootmem_node_data[0] }, - { .bdata = &bootmem_node_data[1] }, - { .bdata = &bootmem_node_data[2] }, - { .bdata = &bootmem_node_data[3] }, -#if MAX_NUMNODES == 16 - { .bdata = &bootmem_node_data[4] }, - { .bdata = &bootmem_node_data[5] }, - { .bdata = &bootmem_node_data[6] }, - { .bdata = &bootmem_node_data[7] }, - { .bdata = &bootmem_node_data[8] }, - { .bdata = &bootmem_node_data[9] }, - { .bdata = &bootmem_node_data[10] }, - { .bdata = &bootmem_node_data[11] }, - { .bdata = &bootmem_node_data[12] }, - { .bdata = &bootmem_node_data[13] }, - { .bdata = &bootmem_node_data[14] }, - { .bdata = &bootmem_node_data[15] }, -#endif -}; - -EXPORT_SYMBOL(discontig_node_data); diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 13fa536d82e..9e7742f0a10 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -24,15 +24,6 @@ #include <asm/tlbflush.h> #include <asm/sizes.h> -/* Sanity check size */ -#if (CONSISTENT_DMA_SIZE % SZ_2M) -#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB" -#endif - -#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT) -#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT) -#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT) - static u64 get_coherent_dma_mask(struct device *dev) { u64 mask = ISA_DMA_THRESHOLD; @@ -123,6 +114,15 @@ static void __dma_free_buffer(struct page *page, size_t size) } #ifdef CONFIG_MMU +/* Sanity check size */ +#if (CONSISTENT_DMA_SIZE % SZ_2M) +#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB" +#endif + +#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT) +#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT) +#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT) + /* * These are the page tables (2MB each) covering uncached, DMA consistent allocations */ diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index f6a99946532..599d121c81e 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -17,6 +17,7 @@ #include <linux/initrd.h> #include <linux/highmem.h> #include <linux/gfp.h> +#include <linux/memblock.h> #include <asm/mach-types.h> #include <asm/sections.h> @@ -79,38 +80,37 @@ struct meminfo meminfo; void show_mem(void) { int free = 0, total = 0, reserved = 0; - int shared = 0, cached = 0, slab = 0, node, i; + int shared = 0, cached = 0, slab = 0, i; struct meminfo * mi = &meminfo; printk("Mem-info:\n"); show_free_areas(); - for_each_online_node(node) { - for_each_nodebank (i,mi,node) { - struct membank *bank = &mi->bank[i]; - unsigned int pfn1, pfn2; - struct page *page, *end; - - pfn1 = bank_pfn_start(bank); - pfn2 = bank_pfn_end(bank); - - page = pfn_to_page(pfn1); - end = pfn_to_page(pfn2 - 1) + 1; - - do { - total++; - if (PageReserved(page)) - reserved++; - else if (PageSwapCache(page)) - cached++; - else if (PageSlab(page)) - slab++; - else if (!page_count(page)) - free++; - else - shared += page_count(page) - 1; - page++; - } while (page < end); - } + + for_each_bank (i, mi) { + struct membank *bank = &mi->bank[i]; + unsigned int pfn1, pfn2; + struct page *page, *end; + + pfn1 = bank_pfn_start(bank); + pfn2 = bank_pfn_end(bank); + + page = pfn_to_page(pfn1); + end = pfn_to_page(pfn2 - 1) + 1; + + do { + total++; + if (PageReserved(page)) + reserved++; + else if (PageSwapCache(page)) + cached++; + else if (PageSlab(page)) + slab++; + else if (!page_count(page)) + free++; + else + shared += page_count(page) - 1; + page++; + } while (page < end); } printk("%d pages of RAM\n", total); @@ -121,7 +121,7 @@ void show_mem(void) printk("%d pages swap cached\n", cached); } -static void __init find_node_limits(int node, struct meminfo *mi, +static void __init find_limits(struct meminfo *mi, unsigned long *min, unsigned long *max_low, unsigned long *max_high) { int i; @@ -129,7 +129,7 @@ static void __init find_node_limits(int node, struct meminfo *mi, *min = -1UL; *max_low = *max_high = 0; - for_each_nodebank(i, mi, node) { + for_each_bank (i, mi) { struct membank *bank = &mi->bank[i]; unsigned long start, end; @@ -147,155 +147,64 @@ static void __init find_node_limits(int node, struct meminfo *mi, } } -/* - * FIXME: We really want to avoid allocating the bootmap bitmap - * over the top of the initrd. Hopefully, this is located towards - * the start of a bank, so if we allocate the bootmap bitmap at - * the end, we won't clash. - */ -static unsigned int __init -find_bootmap_pfn(int node, struct meminfo *mi, unsigned int bootmap_pages) -{ - unsigned int start_pfn, i, bootmap_pfn; - - start_pfn = PAGE_ALIGN(__pa(_end)) >> PAGE_SHIFT; - bootmap_pfn = 0; - - for_each_nodebank(i, mi, node) { - struct membank *bank = &mi->bank[i]; - unsigned int start, end; - - start = bank_pfn_start(bank); - end = bank_pfn_end(bank); - - if (end < start_pfn) - continue; - - if (start < start_pfn) - start = start_pfn; - - if (end <= start) - continue; - - if (end - start >= bootmap_pages) { - bootmap_pfn = start; - break; - } - } - - if (bootmap_pfn == 0) - BUG(); - - return bootmap_pfn; -} - -static int __init check_initrd(struct meminfo *mi) -{ - int initrd_node = -2; -#ifdef CONFIG_BLK_DEV_INITRD - unsigned long end = phys_initrd_start + phys_initrd_size; - - /* - * Make sure that the initrd is within a valid area of - * memory. - */ - if (phys_initrd_size) { - unsigned int i; - - initrd_node = -1; - - for (i = 0; i < mi->nr_banks; i++) { - struct membank *bank = &mi->bank[i]; - if (bank_phys_start(bank) <= phys_initrd_start && - end <= bank_phys_end(bank)) - initrd_node = bank->node; - } - } - - if (initrd_node == -1) { - printk(KERN_ERR "INITRD: 0x%08lx+0x%08lx extends beyond " - "physical memory - disabling initrd\n", - phys_initrd_start, phys_initrd_size); - phys_initrd_start = phys_initrd_size = 0; - } -#endif - - return initrd_node; -} - -static void __init bootmem_init_node(int node, struct meminfo *mi, +static void __init arm_bootmem_init(struct meminfo *mi, unsigned long start_pfn, unsigned long end_pfn) { - unsigned long boot_pfn; unsigned int boot_pages; + phys_addr_t bitmap; pg_data_t *pgdat; int i; /* - * Allocate the bootmem bitmap page. + * Allocate the bootmem bitmap page. This must be in a region + * of memory which has already been mapped. */ boot_pages = bootmem_bootmap_pages(end_pfn - start_pfn); - boot_pfn = find_bootmap_pfn(node, mi, boot_pages); + bitmap = memblock_alloc_base(boot_pages << PAGE_SHIFT, L1_CACHE_BYTES, + __pfn_to_phys(end_pfn)); /* - * Initialise the bootmem allocator for this node, handing the + * Initialise the bootmem allocator, handing the * memory banks over to bootmem. */ - node_set_online(node); - pgdat = NODE_DATA(node); - init_bootmem_node(pgdat, boot_pfn, start_pfn, end_pfn); + node_set_online(0); + pgdat = NODE_DATA(0); + init_bootmem_node(pgdat, __phys_to_pfn(bitmap), start_pfn, end_pfn); - for_each_nodebank(i, mi, node) { + for_each_bank(i, mi) { struct membank *bank = &mi->bank[i]; if (!bank->highmem) - free_bootmem_node(pgdat, bank_phys_start(bank), bank_phys_size(bank)); + free_bootmem(bank_phys_start(bank), bank_phys_size(bank)); } /* - * Reserve the bootmem bitmap for this node. + * Reserve the memblock reserved regions in bootmem. */ - reserve_bootmem_node(pgdat, boot_pfn << PAGE_SHIFT, - boot_pages << PAGE_SHIFT, BOOTMEM_DEFAULT); -} - -static void __init bootmem_reserve_initrd(int node) -{ -#ifdef CONFIG_BLK_DEV_INITRD - pg_data_t *pgdat = NODE_DATA(node); - int res; - - res = reserve_bootmem_node(pgdat, phys_initrd_start, - phys_initrd_size, BOOTMEM_EXCLUSIVE); - - if (res == 0) { - initrd_start = __phys_to_virt(phys_initrd_start); - initrd_end = initrd_start + phys_initrd_size; - } else { - printk(KERN_ERR - "INITRD: 0x%08lx+0x%08lx overlaps in-use " - "memory region - disabling initrd\n", - phys_initrd_start, phys_initrd_size); + for (i = 0; i < memblock.reserved.cnt; i++) { + phys_addr_t start = memblock_start_pfn(&memblock.reserved, i); + if (start >= start_pfn && + memblock_end_pfn(&memblock.reserved, i) <= end_pfn) + reserve_bootmem_node(pgdat, __pfn_to_phys(start), + memblock_size_bytes(&memblock.reserved, i), + BOOTMEM_DEFAULT); } -#endif } -static void __init bootmem_free_node(int node, struct meminfo *mi) +static void __init arm_bootmem_free(struct meminfo *mi, unsigned long min, + unsigned long max_low, unsigned long max_high) { unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; - unsigned long min, max_low, max_high; int i; - find_node_limits(node, mi, &min, &max_low, &max_high); - /* - * initialise the zones within this node. + * initialise the zones. */ memset(zone_size, 0, sizeof(zone_size)); /* - * The size of this node has already been determined. If we need - * to do anything fancy with the allocation of this memory to the - * zones, now is the time to do it. + * The memory size has already been determined. If we need + * to do anything fancy with the allocation of this memory + * to the zones, now is the time to do it. */ zone_size[0] = max_low - min; #ifdef CONFIG_HIGHMEM @@ -303,11 +212,11 @@ static void __init bootmem_free_node(int node, struct meminfo *mi) #endif /* - * For each bank in this node, calculate the size of the holes. - * holes = node_size - sum(bank_sizes_in_node) + * Calculate the size of the holes. + * holes = node_size - sum(bank_sizes) */ memcpy(zhole_size, zone_size, sizeof(zhole_size)); - for_each_nodebank(i, mi, node) { + for_each_bank(i, mi) { int idx = 0; #ifdef CONFIG_HIGHMEM if (mi->bank[i].highmem) @@ -320,24 +229,23 @@ static void __init bootmem_free_node(int node, struct meminfo *mi) * Adjust the sizes according to any special requirements for * this machine type. */ - arch_adjust_zones(node, zone_size, zhole_size); + arch_adjust_zones(zone_size, zhole_size); - free_area_init_node(node, zone_size, min, zhole_size); + free_area_init_node(0, zone_size, min, zhole_size); } #ifndef CONFIG_SPARSEMEM int pfn_valid(unsigned long pfn) { - struct meminfo *mi = &meminfo; - unsigned int left = 0, right = mi->nr_banks; + struct memblock_region *mem = &memblock.memory; + unsigned int left = 0, right = mem->cnt; do { unsigned int mid = (right + left) / 2; - struct membank *bank = &mi->bank[mid]; - if (pfn < bank_pfn_start(bank)) + if (pfn < memblock_start_pfn(mem, mid)) right = mid; - else if (pfn >= bank_pfn_end(bank)) + else if (pfn >= memblock_end_pfn(mem, mid)) left = mid + 1; else return 1; @@ -346,73 +254,69 @@ int pfn_valid(unsigned long pfn) } EXPORT_SYMBOL(pfn_valid); -static void arm_memory_present(struct meminfo *mi, int node) +static void arm_memory_present(void) { } #else -static void arm_memory_present(struct meminfo *mi, int node) +static void arm_memory_present(void) { int i; - for_each_nodebank(i, mi, node) { - struct membank *bank = &mi->bank[i]; - memory_present(node, bank_pfn_start(bank), bank_pfn_end(bank)); - } + for (i = 0; i < memblock.memory.cnt; i++) + memory_present(0, memblock_start_pfn(&memblock.memory, i), + memblock_end_pfn(&memblock.memory, i)); } #endif -void __init bootmem_init(void) +void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc) { - struct meminfo *mi = &meminfo; - unsigned long min, max_low, max_high; - int node, initrd_node; + int i; - /* - * Locate which node contains the ramdisk image, if any. - */ - initrd_node = check_initrd(mi); + memblock_init(); + for (i = 0; i < mi->nr_banks; i++) + memblock_add(mi->bank[i].start, mi->bank[i].size); - max_low = max_high = 0; + /* Register the kernel text, kernel data and initrd with memblock. */ +#ifdef CONFIG_XIP_KERNEL + memblock_reserve(__pa(_data), _end - _data); +#else + memblock_reserve(__pa(_stext), _end - _stext); +#endif +#ifdef CONFIG_BLK_DEV_INITRD + if (phys_initrd_size) { + memblock_reserve(phys_initrd_start, phys_initrd_size); - /* - * Run through each node initialising the bootmem allocator. - */ - for_each_node(node) { - unsigned long node_low, node_high; + /* Now convert initrd to virtual addresses */ + initrd_start = __phys_to_virt(phys_initrd_start); + initrd_end = initrd_start + phys_initrd_size; + } +#endif - find_node_limits(node, mi, &min, &node_low, &node_high); + arm_mm_memblock_reserve(); - if (node_low > max_low) - max_low = node_low; - if (node_high > max_high) - max_high = node_high; + /* reserve any platform specific memblock areas */ + if (mdesc->reserve) + mdesc->reserve(); - /* - * If there is no memory in this node, ignore it. - * (We can't have nodes which have no lowmem) - */ - if (node_low == 0) - continue; + memblock_analyze(); + memblock_dump_all(); +} - bootmem_init_node(node, mi, min, node_low); +void __init bootmem_init(void) +{ + struct meminfo *mi = &meminfo; + unsigned long min, max_low, max_high; - /* - * Reserve any special node zero regions. - */ - if (node == 0) - reserve_node_zero(NODE_DATA(node)); + max_low = max_high = 0; - /* - * If the initrd is in this node, reserve its memory. - */ - if (node == initrd_node) - bootmem_reserve_initrd(node); + find_limits(mi, &min, &max_low, &max_high); - /* - * Sparsemem tries to allocate bootmem in memory_present(), - * so must be done after the fixed reservations - */ - arm_memory_present(mi, node); - } + arm_bootmem_init(mi, min, max_low); + + /* + * Sparsemem tries to allocate bootmem in memory_present(), + * so must be done after the fixed reservations + */ + arm_memory_present(); /* * sparse_init() needs the bootmem allocator up and running. @@ -420,12 +324,11 @@ void __init bootmem_init(void) sparse_init(); /* - * Now free memory in each node - free_area_init_node needs + * Now free the memory - free_area_init_node needs * the sparse mem_map arrays initialized by sparse_init() * for memmap_init_zone(), otherwise all PFNs are invalid. */ - for_each_node(node) - bootmem_free_node(node, mi); + arm_bootmem_free(mi, min, max_low, max_high); high_memory = __va((max_low << PAGE_SHIFT) - 1) + 1; @@ -460,7 +363,7 @@ static inline int free_area(unsigned long pfn, unsigned long end, char *s) } static inline void -free_memmap(int node, unsigned long start_pfn, unsigned long end_pfn) +free_memmap(unsigned long start_pfn, unsigned long end_pfn) { struct page *start_pg, *end_pg; unsigned long pg, pgend; @@ -483,13 +386,13 @@ free_memmap(int node, unsigned long start_pfn, unsigned long end_pfn) * free the section of the memmap array. */ if (pg < pgend) - free_bootmem_node(NODE_DATA(node), pg, pgend - pg); + free_bootmem(pg, pgend - pg); } /* * The mem_map array can get very big. Free the unused area of the memory map. */ -static void __init free_unused_memmap_node(int node, struct meminfo *mi) +static void __init free_unused_memmap(struct meminfo *mi) { unsigned long bank_start, prev_bank_end = 0; unsigned int i; @@ -499,7 +402,7 @@ static void __init free_unused_memmap_node(int node, struct meminfo *mi) * may not be the case, especially if the user has provided the * information on the command line. */ - for_each_nodebank(i, mi, node) { + for_each_bank(i, mi) { struct membank *bank = &mi->bank[i]; bank_start = bank_pfn_start(bank); @@ -514,7 +417,7 @@ static void __init free_unused_memmap_node(int node, struct meminfo *mi) * between the current bank and the previous, free it. */ if (prev_bank_end && prev_bank_end != bank_start) - free_memmap(node, prev_bank_end, bank_start); + free_memmap(prev_bank_end, bank_start); prev_bank_end = bank_pfn_end(bank); } @@ -528,21 +431,14 @@ static void __init free_unused_memmap_node(int node, struct meminfo *mi) void __init mem_init(void) { unsigned long reserved_pages, free_pages; - int i, node; + int i; -#ifndef CONFIG_DISCONTIGMEM max_mapnr = pfn_to_page(max_pfn + PHYS_PFN_OFFSET) - mem_map; -#endif /* this will put all unused low memory onto the freelists */ - for_each_online_node(node) { - pg_data_t *pgdat = NODE_DATA(node); - - free_unused_memmap_node(node, &meminfo); + free_unused_memmap(&meminfo); - if (pgdat->node_spanned_pages != 0) - totalram_pages += free_all_bootmem_node(pgdat); - } + totalram_pages += free_all_bootmem(); #ifdef CONFIG_SA1111 /* now that our DMA memory is actually so designated, we can free it */ @@ -552,39 +448,35 @@ void __init mem_init(void) #ifdef CONFIG_HIGHMEM /* set highmem page free */ - for_each_online_node(node) { - for_each_nodebank (i, &meminfo, node) { - unsigned long start = bank_pfn_start(&meminfo.bank[i]); - unsigned long end = bank_pfn_end(&meminfo.bank[i]); - if (start >= max_low_pfn + PHYS_PFN_OFFSET) - totalhigh_pages += free_area(start, end, NULL); - } + for_each_bank (i, &meminfo) { + unsigned long start = bank_pfn_start(&meminfo.bank[i]); + unsigned long end = bank_pfn_end(&meminfo.bank[i]); + if (start >= max_low_pfn + PHYS_PFN_OFFSET) + totalhigh_pages += free_area(start, end, NULL); } totalram_pages += totalhigh_pages; #endif reserved_pages = free_pages = 0; - for_each_online_node(node) { - for_each_nodebank(i, &meminfo, node) { - struct membank *bank = &meminfo.bank[i]; - unsigned int pfn1, pfn2; - struct page *page, *end; - - pfn1 = bank_pfn_start(bank); - pfn2 = bank_pfn_end(bank); - - page = pfn_to_page(pfn1); - end = pfn_to_page(pfn2 - 1) + 1; - - do { - if (PageReserved(page)) - reserved_pages++; - else if (!page_count(page)) - free_pages++; - page++; - } while (page < end); - } + for_each_bank(i, &meminfo) { + struct membank *bank = &meminfo.bank[i]; + unsigned int pfn1, pfn2; + struct page *page, *end; + + pfn1 = bank_pfn_start(bank); + pfn2 = bank_pfn_end(bank); + + page = pfn_to_page(pfn1); + end = pfn_to_page(pfn2 - 1) + 1; + + do { + if (PageReserved(page)) + reserved_pages++; + else if (!page_count(page)) + free_pages++; + page++; + } while (page < end); } /* diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 815d08eecbb..6630620380a 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -28,7 +28,5 @@ extern void __flush_dcache_page(struct address_space *mapping, struct page *page #endif -struct pglist_data; - void __init bootmem_init(void); -void reserve_node_zero(struct pglist_data *pgdat); +void arm_mm_memblock_reserve(void); diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 28589417118..d5541adc352 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -11,13 +11,12 @@ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/init.h> -#include <linux/bootmem.h> #include <linux/mman.h> #include <linux/nodemask.h> +#include <linux/memblock.h> #include <linux/sort.h> #include <asm/cputype.h> -#include <asm/mach-types.h> #include <asm/sections.h> #include <asm/cachetype.h> #include <asm/setup.h> @@ -488,18 +487,28 @@ static void __init build_mem_type_table(void) #define vectors_base() (vectors_high() ? 0xffff0000 : 0) -static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, - unsigned long end, unsigned long pfn, - const struct mem_type *type) +static void __init *early_alloc(unsigned long sz) { - pte_t *pte; + void *ptr = __va(memblock_alloc(sz, sz)); + memset(ptr, 0, sz); + return ptr; +} +static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot) +{ if (pmd_none(*pmd)) { - pte = alloc_bootmem_low_pages(2 * PTRS_PER_PTE * sizeof(pte_t)); - __pmd_populate(pmd, __pa(pte) | type->prot_l1); + pte_t *pte = early_alloc(2 * PTRS_PER_PTE * sizeof(pte_t)); + __pmd_populate(pmd, __pa(pte) | prot); } + BUG_ON(pmd_bad(*pmd)); + return pte_offset_kernel(pmd, addr); +} - pte = pte_offset_kernel(pmd, addr); +static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, + unsigned long end, unsigned long pfn, + const struct mem_type *type) +{ + pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1); do { set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0); pfn++; @@ -668,7 +677,7 @@ void __init iotable_init(struct map_desc *io_desc, int nr) create_mapping(io_desc + i); } -static unsigned long __initdata vmalloc_reserve = SZ_128M; +static void * __initdata vmalloc_min = (void *)(VMALLOC_END - SZ_128M); /* * vmalloc=size forces the vmalloc area to be exactly 'size' @@ -677,7 +686,7 @@ static unsigned long __initdata vmalloc_reserve = SZ_128M; */ static int __init early_vmalloc(char *arg) { - vmalloc_reserve = memparse(arg, NULL); + unsigned long vmalloc_reserve = memparse(arg, NULL); if (vmalloc_reserve < SZ_16M) { vmalloc_reserve = SZ_16M; @@ -692,22 +701,26 @@ static int __init early_vmalloc(char *arg) "vmalloc area is too big, limiting to %luMB\n", vmalloc_reserve >> 20); } + + vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve); return 0; } early_param("vmalloc", early_vmalloc); -#define VMALLOC_MIN (void *)(VMALLOC_END - vmalloc_reserve) +phys_addr_t lowmem_end_addr; static void __init sanity_check_meminfo(void) { int i, j, highmem = 0; + lowmem_end_addr = __pa(vmalloc_min - 1) + 1; + for (i = 0, j = 0; i < meminfo.nr_banks; i++) { struct membank *bank = &meminfo.bank[j]; *bank = meminfo.bank[i]; #ifdef CONFIG_HIGHMEM - if (__va(bank->start) > VMALLOC_MIN || + if (__va(bank->start) > vmalloc_min || __va(bank->start) < (void *)PAGE_OFFSET) highmem = 1; @@ -717,8 +730,8 @@ static void __init sanity_check_meminfo(void) * Split those memory banks which are partially overlapping * the vmalloc area greatly simplifying things later. */ - if (__va(bank->start) < VMALLOC_MIN && - bank->size > VMALLOC_MIN - __va(bank->start)) { + if (__va(bank->start) < vmalloc_min && + bank->size > vmalloc_min - __va(bank->start)) { if (meminfo.nr_banks >= NR_BANKS) { printk(KERN_CRIT "NR_BANKS too low, " "ignoring high memory\n"); @@ -727,12 +740,12 @@ static void __init sanity_check_meminfo(void) (meminfo.nr_banks - i) * sizeof(*bank)); meminfo.nr_banks++; i++; - bank[1].size -= VMALLOC_MIN - __va(bank->start); - bank[1].start = __pa(VMALLOC_MIN - 1) + 1; + bank[1].size -= vmalloc_min - __va(bank->start); + bank[1].start = __pa(vmalloc_min - 1) + 1; bank[1].highmem = highmem = 1; j++; } - bank->size = VMALLOC_MIN - __va(bank->start); + bank->size = vmalloc_min - __va(bank->start); } #else bank->highmem = highmem; @@ -741,7 +754,7 @@ static void __init sanity_check_meminfo(void) * Check whether this memory bank would entirely overlap * the vmalloc area. */ - if (__va(bank->start) >= VMALLOC_MIN || + if (__va(bank->start) >= vmalloc_min || __va(bank->start) < (void *)PAGE_OFFSET) { printk(KERN_NOTICE "Ignoring RAM at %.8lx-%.8lx " "(vmalloc region overlap).\n", @@ -753,9 +766,9 @@ static void __init sanity_check_meminfo(void) * Check whether this memory bank would partially overlap * the vmalloc area. */ - if (__va(bank->start + bank->size) > VMALLOC_MIN || + if (__va(bank->start + bank->size) > vmalloc_min || __va(bank->start + bank->size) < __va(bank->start)) { - unsigned long newsize = VMALLOC_MIN - __va(bank->start); + unsigned long newsize = vmalloc_min - __va(bank->start); printk(KERN_NOTICE "Truncating RAM at %.8lx-%.8lx " "to -%.8lx (vmalloc region overlap).\n", bank->start, bank->start + bank->size - 1, @@ -827,101 +840,23 @@ static inline void prepare_page_table(void) } /* - * Reserve the various regions of node 0 + * Reserve the special regions of memory */ -void __init reserve_node_zero(pg_data_t *pgdat) +void __init arm_mm_memblock_reserve(void) { - unsigned long res_size = 0; - - /* - * Register the kernel text and data with bootmem. - * Note that this can only be in node 0. - */ -#ifdef CONFIG_XIP_KERNEL - reserve_bootmem_node(pgdat, __pa(_data), _end - _data, - BOOTMEM_DEFAULT); -#else - reserve_bootmem_node(pgdat, __pa(_stext), _end - _stext, - BOOTMEM_DEFAULT); -#endif - /* * Reserve the page tables. These are already in use, * and can only be in node 0. */ - reserve_bootmem_node(pgdat, __pa(swapper_pg_dir), - PTRS_PER_PGD * sizeof(pgd_t), BOOTMEM_DEFAULT); - - /* - * Hmm... This should go elsewhere, but we really really need to - * stop things allocating the low memory; ideally we need a better - * implementation of GFP_DMA which does not assume that DMA-able - * memory starts at zero. - */ - if (machine_is_integrator() || machine_is_cintegrator()) - res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; - - /* - * These should likewise go elsewhere. They pre-reserve the - * screen memory region at the start of main system memory. - */ - if (machine_is_edb7211()) - res_size = 0x00020000; - if (machine_is_p720t()) - res_size = 0x00014000; - - /* H1940, RX3715 and RX1950 need to reserve this for suspend */ - - if (machine_is_h1940() || machine_is_rx3715() - || machine_is_rx1950()) { - reserve_bootmem_node(pgdat, 0x30003000, 0x1000, - BOOTMEM_DEFAULT); - reserve_bootmem_node(pgdat, 0x30081000, 0x1000, - BOOTMEM_DEFAULT); - } - - if (machine_is_palmld() || machine_is_palmtx()) { - reserve_bootmem_node(pgdat, 0xa0000000, 0x1000, - BOOTMEM_EXCLUSIVE); - reserve_bootmem_node(pgdat, 0xa0200000, 0x1000, - BOOTMEM_EXCLUSIVE); - } - - if (machine_is_treo680() || machine_is_centro()) { - reserve_bootmem_node(pgdat, 0xa0000000, 0x1000, - BOOTMEM_EXCLUSIVE); - reserve_bootmem_node(pgdat, 0xa2000000, 0x1000, - BOOTMEM_EXCLUSIVE); - } - - if (machine_is_palmt5()) - reserve_bootmem_node(pgdat, 0xa0200000, 0x1000, - BOOTMEM_EXCLUSIVE); - - /* - * U300 - This platform family can share physical memory - * between two ARM cpus, one running Linux and the other - * running another OS. - */ - if (machine_is_u300()) { -#ifdef CONFIG_MACH_U300_SINGLE_RAM -#if ((CONFIG_MACH_U300_ACCESS_MEM_SIZE & 1) == 1) && \ - CONFIG_MACH_U300_2MB_ALIGNMENT_FIX - res_size = 0x00100000; -#endif -#endif - } + memblock_reserve(__pa(swapper_pg_dir), PTRS_PER_PGD * sizeof(pgd_t)); #ifdef CONFIG_SA1111 /* * Because of the SA1111 DMA bug, we want to preserve our * precious DMA-able memory... */ - res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; + memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET); #endif - if (res_size) - reserve_bootmem_node(pgdat, PHYS_OFFSET, res_size, - BOOTMEM_DEFAULT); } /* @@ -940,7 +875,7 @@ static void __init devicemaps_init(struct machine_desc *mdesc) /* * Allocate the vector page early. */ - vectors = alloc_bootmem_low_pages(PAGE_SIZE); + vectors = early_alloc(PAGE_SIZE); for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE) pmd_clear(pmd_off_k(addr)); @@ -1011,11 +946,8 @@ static void __init devicemaps_init(struct machine_desc *mdesc) static void __init kmap_init(void) { #ifdef CONFIG_HIGHMEM - pmd_t *pmd = pmd_off_k(PKMAP_BASE); - pte_t *pte = alloc_bootmem_low_pages(2 * PTRS_PER_PTE * sizeof(pte_t)); - BUG_ON(!pmd_none(*pmd) || !pte); - __pmd_populate(pmd, __pa(pte) | _PAGE_KERNEL_TABLE); - pkmap_page_table = pte + PTRS_PER_PTE; + pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE), + PKMAP_BASE, _PAGE_KERNEL_TABLE); #endif } @@ -1066,17 +998,16 @@ void __init paging_init(struct machine_desc *mdesc) sanity_check_meminfo(); prepare_page_table(); map_lowmem(); - bootmem_init(); devicemaps_init(mdesc); kmap_init(); top_pmd = pmd_off_k(0xffff0000); - /* - * allocate the zero page. Note that this always succeeds and - * returns a zeroed result. - */ - zero_page = alloc_bootmem_low_pages(PAGE_SIZE); + /* allocate the zero page. */ + zero_page = early_alloc(PAGE_SIZE); + + bootmem_init(); + empty_zero_page = virt_to_page(zero_page); __flush_dcache_page(NULL, empty_zero_page); } diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 33b327379f0..687d02319a4 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -6,8 +6,8 @@ #include <linux/module.h> #include <linux/mm.h> #include <linux/pagemap.h> -#include <linux/bootmem.h> #include <linux/io.h> +#include <linux/memblock.h> #include <asm/cacheflush.h> #include <asm/sections.h> @@ -17,30 +17,14 @@ #include "mm.h" -/* - * Reserve the various regions of node 0 - */ -void __init reserve_node_zero(pg_data_t *pgdat) +void __init arm_mm_memblock_reserve(void) { /* - * Register the kernel text and data with bootmem. - * Note that this can only be in node 0. - */ -#ifdef CONFIG_XIP_KERNEL - reserve_bootmem_node(pgdat, __pa(_data), _end - _data, - BOOTMEM_DEFAULT); -#else - reserve_bootmem_node(pgdat, __pa(_stext), _end - _stext, - BOOTMEM_DEFAULT); -#endif - - /* * Register the exception vector page. * some architectures which the DRAM is the exception vector to trap, * alloc_page breaks with error, although it is not NULL, but "0." */ - reserve_bootmem_node(pgdat, CONFIG_VECTORS_BASE, PAGE_SIZE, - BOOTMEM_DEFAULT); + memblock_reserve(CONFIG_VECTORS_BASE, PAGE_SIZE); } /* |