From b61bfa3c462671c48a51fb5c31af337c5a996a04 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 23 Jul 2008 21:26:55 -0700 Subject: mm: move bootmem descriptors definition to a single place There are a lot of places that define either a single bootmem descriptor or an array of them. Use only one central array with MAX_NUMNODES items instead. Signed-off-by: Johannes Weiner Acked-by: Ralf Baechle Cc: Ingo Molnar Cc: Richard Henderson Cc: Russell King Cc: Tony Luck Cc: Hirokazu Takata Cc: Geert Uytterhoeven Cc: Kyle McMartin Cc: Paul Mackerras Cc: Paul Mundt Cc: David S. Miller Cc: Yinghai Lu Cc: Christoph Lameter Cc: Mel Gorman Cc: Andy Whitcroft Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/bootmem.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/bootmem.h') diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index a1d9b79078e..2599c741405 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -38,6 +38,8 @@ typedef struct bootmem_data { struct list_head list; } bootmem_data_t; +extern bootmem_data_t bootmem_node_data[]; + extern unsigned long bootmem_bootmap_pages(unsigned long); extern unsigned long init_bootmem(unsigned long addr, unsigned long memend); extern void free_bootmem(unsigned long addr, unsigned long size); -- cgit v1.2.3-70-g09d2 From ffc6421f0720f433b5b35b89ff56e998eabff93b Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 23 Jul 2008 21:26:59 -0700 Subject: mm: unexport __alloc_bootmem_core() This function has no external callers, so unexport it. Also fix its naming inconsistency. Signed-off-by: Johannes Weiner Cc: Ingo Molnar Cc: Yinghai Lu Cc: Christoph Lameter Cc: Mel Gorman Cc: Andy Whitcroft Cc: Mel Gorman Cc: Andy Whitcroft Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/bootmem.h | 5 ----- mm/bootmem.c | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 17 deletions(-) (limited to 'include/linux/bootmem.h') diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index 2599c741405..dd8fee6c46d 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -56,11 +56,6 @@ extern void *__alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal); -extern void *__alloc_bootmem_core(struct bootmem_data *bdata, - unsigned long size, - unsigned long align, - unsigned long goal, - unsigned long limit); /* * flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE, diff --git a/mm/bootmem.c b/mm/bootmem.c index 251c66c5d96..4bc6ae2fbaa 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c @@ -234,9 +234,9 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, * * NOTE: This function is _not_ reentrant. */ -void * __init -__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, - unsigned long align, unsigned long goal, unsigned long limit) +static void * __init +alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, + unsigned long align, unsigned long goal, unsigned long limit) { unsigned long areasize, preferred; unsigned long i, start = 0, incr, eidx, end_pfn; @@ -245,7 +245,7 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, void *node_bootmem_map; if (!size) { - printk("__alloc_bootmem_core(): zero-sized request\n"); + printk("alloc_bootmem_core(): zero-sized request\n"); BUG(); } BUG_ON(align & (align-1)); @@ -512,7 +512,7 @@ void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, void *ptr; list_for_each_entry(bdata, &bdata_list, list) { - ptr = __alloc_bootmem_core(bdata, size, align, goal, 0); + ptr = alloc_bootmem_core(bdata, size, align, goal, 0); if (ptr) return ptr; } @@ -540,7 +540,7 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, { void *ptr; - ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); + ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); if (ptr) return ptr; @@ -559,8 +559,8 @@ void * __init alloc_bootmem_section(unsigned long size, goal = PFN_PHYS(pfn); limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1; pgdat = NODE_DATA(early_pfn_to_nid(pfn)); - ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal, - limit); + ptr = alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal, + limit); if (!ptr) return NULL; @@ -589,8 +589,8 @@ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, void *ptr; list_for_each_entry(bdata, &bdata_list, list) { - ptr = __alloc_bootmem_core(bdata, size, align, goal, - ARCH_LOW_ADDRESS_LIMIT); + ptr = alloc_bootmem_core(bdata, size, align, goal, + ARCH_LOW_ADDRESS_LIMIT); if (ptr) return ptr; } @@ -606,6 +606,6 @@ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { - return __alloc_bootmem_core(pgdat->bdata, size, align, goal, - ARCH_LOW_ADDRESS_LIMIT); + return alloc_bootmem_core(pgdat->bdata, size, align, goal, + ARCH_LOW_ADDRESS_LIMIT); } -- cgit v1.2.3-70-g09d2 From b54bbf7b81170f03597c17dd0b559e3006bc9868 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Wed, 23 Jul 2008 21:27:45 -0700 Subject: mm: introduce non panic alloc_bootmem Straight forward variant of the existing __alloc_bootmem_node, only subsequent patch when allocating giant hugepages at boot -- don't want to panic if we can't allocate as many as the user asked for. Signed-off-by: Andi Kleen Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/bootmem.h | 4 ++++ mm/bootmem.c | 12 ++++++++++++ 2 files changed, 16 insertions(+) (limited to 'include/linux/bootmem.h') diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index dd8fee6c46d..f352c5f125b 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -89,6 +89,10 @@ extern void *__alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal); +extern void *__alloc_bootmem_node_nopanic(pg_data_t *pgdat, + unsigned long size, + unsigned long align, + unsigned long goal); extern unsigned long init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn, unsigned long startpfn, diff --git a/mm/bootmem.c b/mm/bootmem.c index 4bc6ae2fbaa..9ac972535ff 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c @@ -578,6 +578,18 @@ void * __init alloc_bootmem_section(unsigned long size, } #endif +void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, + unsigned long align, unsigned long goal) +{ + void *ptr; + + ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); + if (ptr) + return ptr; + + return __alloc_bootmem_nopanic(size, align, goal); +} + #ifndef ARCH_LOW_ADDRESS_LIMIT #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL #endif -- cgit v1.2.3-70-g09d2 From 223e8dc9249c9e15f6c8b638d73fcad78ccb0a88 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 23 Jul 2008 21:28:00 -0700 Subject: bootmem: reorder code to match new bootmem structure This only reorders functions so that further patches will be easier to read. No code changed. Signed-off-by: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/bootmem.h | 86 ++++++------ mm/bootmem.c | 356 ++++++++++++++++++++++++------------------------ 2 files changed, 222 insertions(+), 220 deletions(-) (limited to 'include/linux/bootmem.h') diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index f352c5f125b..5000fd70b04 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -41,36 +41,62 @@ typedef struct bootmem_data { extern bootmem_data_t bootmem_node_data[]; extern unsigned long bootmem_bootmap_pages(unsigned long); + +extern unsigned long init_bootmem_node(pg_data_t *pgdat, + unsigned long freepfn, + unsigned long startpfn, + unsigned long endpfn); extern unsigned long init_bootmem(unsigned long addr, unsigned long memend); + +extern unsigned long free_all_bootmem_node(pg_data_t *pgdat); +extern unsigned long free_all_bootmem(void); + +extern void free_bootmem_node(pg_data_t *pgdat, + unsigned long addr, + unsigned long size); extern void free_bootmem(unsigned long addr, unsigned long size); -extern void *__alloc_bootmem(unsigned long size, + +/* + * Flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE, + * the architecture-specific code should honor this). + * + * If flags is 0, then the return value is always 0 (success). If + * flags contains BOOTMEM_EXCLUSIVE, then -EBUSY is returned if the + * memory already was reserved. + */ +#define BOOTMEM_DEFAULT 0 +#define BOOTMEM_EXCLUSIVE (1<<0) + +extern int reserve_bootmem_node(pg_data_t *pgdat, + unsigned long physaddr, + unsigned long size, + int flags); +#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE +extern int reserve_bootmem(unsigned long addr, unsigned long size, int flags); +#endif + +extern void *__alloc_bootmem_nopanic(unsigned long size, unsigned long align, unsigned long goal); -extern void *__alloc_bootmem_nopanic(unsigned long size, +extern void *__alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal); extern void *__alloc_bootmem_low(unsigned long size, unsigned long align, unsigned long goal); +extern void *__alloc_bootmem_node(pg_data_t *pgdat, + unsigned long size, + unsigned long align, + unsigned long goal); +extern void *__alloc_bootmem_node_nopanic(pg_data_t *pgdat, + unsigned long size, + unsigned long align, + unsigned long goal); extern void *__alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal); - -/* - * flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE, - * the architecture-specific code should honor this) - */ -#define BOOTMEM_DEFAULT 0 -#define BOOTMEM_EXCLUSIVE (1<<0) - #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE -/* - * If flags is 0, then the return value is always 0 (success). If - * flags contains BOOTMEM_EXCLUSIVE, then -EBUSY is returned if the - * memory already was reserved. - */ -extern int reserve_bootmem(unsigned long addr, unsigned long size, int flags); #define alloc_bootmem(x) \ __alloc_bootmem(x, SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_low(x) \ @@ -83,38 +109,16 @@ extern int reserve_bootmem(unsigned long addr, unsigned long size, int flags); extern int reserve_bootmem_generic(unsigned long addr, unsigned long size, int flags); -extern unsigned long free_all_bootmem(void); -extern unsigned long free_all_bootmem_node(pg_data_t *pgdat); -extern void *__alloc_bootmem_node(pg_data_t *pgdat, - unsigned long size, - unsigned long align, - unsigned long goal); -extern void *__alloc_bootmem_node_nopanic(pg_data_t *pgdat, - unsigned long size, - unsigned long align, - unsigned long goal); -extern unsigned long init_bootmem_node(pg_data_t *pgdat, - unsigned long freepfn, - unsigned long startpfn, - unsigned long endpfn); -extern int reserve_bootmem_node(pg_data_t *pgdat, - unsigned long physaddr, - unsigned long size, - int flags); -extern void free_bootmem_node(pg_data_t *pgdat, - unsigned long addr, - unsigned long size); -extern void *alloc_bootmem_section(unsigned long size, - unsigned long section_nr); -#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE #define alloc_bootmem_node(pgdat, x) \ __alloc_bootmem_node(pgdat, x, SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_pages_node(pgdat, x) \ __alloc_bootmem_node(pgdat, x, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_low_pages_node(pgdat, x) \ __alloc_bootmem_low_node(pgdat, x, PAGE_SIZE, 0) -#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ + +extern void *alloc_bootmem_section(unsigned long size, + unsigned long section_nr); #ifdef CONFIG_HAVE_ARCH_ALLOC_REMAP extern void *alloc_remap(int nid, unsigned long size); diff --git a/mm/bootmem.c b/mm/bootmem.c index 9ac972535ff..24eacf52c50 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c @@ -38,6 +38,19 @@ unsigned long saved_max_pfn; bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata; +/* + * Given an initialised bdata, it returns the size of the boot bitmap + */ +static unsigned long __init get_mapsize(bootmem_data_t *bdata) +{ + unsigned long mapsize; + unsigned long start = PFN_DOWN(bdata->node_boot_start); + unsigned long end = bdata->node_low_pfn; + + mapsize = ((end - start) + 7) / 8; + return ALIGN(mapsize, sizeof(long)); +} + /* return the number of _pages_ that will be allocated for the boot bitmap */ unsigned long __init bootmem_bootmap_pages(unsigned long pages) { @@ -71,19 +84,6 @@ static void __init link_bootmem(bootmem_data_t *bdata) list_add_tail(&bdata->list, &bdata_list); } -/* - * Given an initialised bdata, it returns the size of the boot bitmap - */ -static unsigned long __init get_mapsize(bootmem_data_t *bdata) -{ - unsigned long mapsize; - unsigned long start = PFN_DOWN(bdata->node_boot_start); - unsigned long end = bdata->node_low_pfn; - - mapsize = ((end - start) + 7) / 8; - return ALIGN(mapsize, sizeof(long)); -} - /* * Called once to set up the allocator itself. */ @@ -108,6 +108,146 @@ static unsigned long __init init_bootmem_core(bootmem_data_t *bdata, return mapsize; } +unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn, + unsigned long startpfn, unsigned long endpfn) +{ + return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn); +} + +unsigned long __init init_bootmem(unsigned long start, unsigned long pages) +{ + max_low_pfn = pages; + min_low_pfn = start; + return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); +} + +static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) +{ + struct page *page; + unsigned long pfn; + unsigned long i, count; + unsigned long idx; + unsigned long *map; + int gofast = 0; + + BUG_ON(!bdata->node_bootmem_map); + + count = 0; + /* first extant page of the node */ + pfn = PFN_DOWN(bdata->node_boot_start); + idx = bdata->node_low_pfn - pfn; + map = bdata->node_bootmem_map; + /* + * Check if we are aligned to BITS_PER_LONG pages. If so, we might + * be able to free page orders of that size at once. + */ + if (!(pfn & (BITS_PER_LONG-1))) + gofast = 1; + + for (i = 0; i < idx; ) { + unsigned long v = ~map[i / BITS_PER_LONG]; + + if (gofast && v == ~0UL) { + int order; + + page = pfn_to_page(pfn); + count += BITS_PER_LONG; + order = ffs(BITS_PER_LONG) - 1; + __free_pages_bootmem(page, order); + i += BITS_PER_LONG; + page += BITS_PER_LONG; + } else if (v) { + unsigned long m; + + page = pfn_to_page(pfn); + for (m = 1; m && i < idx; m<<=1, page++, i++) { + if (v & m) { + count++; + __free_pages_bootmem(page, 0); + } + } + } else { + i += BITS_PER_LONG; + } + pfn += BITS_PER_LONG; + } + + /* + * Now free the allocator bitmap itself, it's not + * needed anymore: + */ + page = virt_to_page(bdata->node_bootmem_map); + idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT; + for (i = 0; i < idx; i++, page++) + __free_pages_bootmem(page, 0); + count += i; + bdata->node_bootmem_map = NULL; + + return count; +} + +unsigned long __init free_all_bootmem_node(pg_data_t *pgdat) +{ + register_page_bootmem_info_node(pgdat); + return free_all_bootmem_core(pgdat->bdata); +} + +unsigned long __init free_all_bootmem(void) +{ + return free_all_bootmem_core(NODE_DATA(0)->bdata); +} + +static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, + unsigned long size) +{ + unsigned long sidx, eidx; + unsigned long i; + + BUG_ON(!size); + + /* out range */ + if (addr + size < bdata->node_boot_start || + PFN_DOWN(addr) > bdata->node_low_pfn) + return; + /* + * round down end of usable mem, partially free pages are + * considered reserved. + */ + + if (addr >= bdata->node_boot_start && addr < bdata->last_success) + bdata->last_success = addr; + + /* + * Round up to index to the range. + */ + if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start)) + sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start); + else + sidx = 0; + + eidx = PFN_DOWN(addr + size - bdata->node_boot_start); + if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start)) + eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start); + + for (i = sidx; i < eidx; i++) { + if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map))) + BUG(); + } +} + +void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, + unsigned long size) +{ + free_bootmem_core(pgdat->bdata, physaddr, size); +} + +void __init free_bootmem(unsigned long addr, unsigned long size) +{ + bootmem_data_t *bdata; + list_for_each_entry(bdata, &bdata_list, list) + free_bootmem_core(bdata, addr, size); +} + /* * Marks a particular physical memory range as unallocatable. Usable RAM * might be used for boot-time allocations - or it might get added @@ -183,43 +323,36 @@ static void __init reserve_bootmem_core(bootmem_data_t *bdata, } } -static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, - unsigned long size) +int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, + unsigned long size, int flags) { - unsigned long sidx, eidx; - unsigned long i; - - BUG_ON(!size); - - /* out range */ - if (addr + size < bdata->node_boot_start || - PFN_DOWN(addr) > bdata->node_low_pfn) - return; - /* - * round down end of usable mem, partially free pages are - * considered reserved. - */ - - if (addr >= bdata->node_boot_start && addr < bdata->last_success) - bdata->last_success = addr; + int ret; - /* - * Round up to index to the range. - */ - if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start)) - sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start); - else - sidx = 0; + ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); + if (ret < 0) + return -ENOMEM; + reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); + return 0; +} - eidx = PFN_DOWN(addr + size - bdata->node_boot_start); - if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start)) - eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start); +#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE +int __init reserve_bootmem(unsigned long addr, unsigned long size, + int flags) +{ + bootmem_data_t *bdata; + int ret; - for (i = sidx; i < eidx; i++) { - if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map))) - BUG(); + list_for_each_entry(bdata, &bdata_list, list) { + ret = can_reserve_bootmem_core(bdata, addr, size, flags); + if (ret < 0) + return ret; } + list_for_each_entry(bdata, &bdata_list, list) + reserve_bootmem_core(bdata, addr, size, flags); + + return 0; } +#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ /* * We 'merge' subsequent allocations to save space. We might 'lose' @@ -371,140 +504,6 @@ found: return ret; } -static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) -{ - struct page *page; - unsigned long pfn; - unsigned long i, count; - unsigned long idx; - unsigned long *map; - int gofast = 0; - - BUG_ON(!bdata->node_bootmem_map); - - count = 0; - /* first extant page of the node */ - pfn = PFN_DOWN(bdata->node_boot_start); - idx = bdata->node_low_pfn - pfn; - map = bdata->node_bootmem_map; - /* - * Check if we are aligned to BITS_PER_LONG pages. If so, we might - * be able to free page orders of that size at once. - */ - if (!(pfn & (BITS_PER_LONG-1))) - gofast = 1; - - for (i = 0; i < idx; ) { - unsigned long v = ~map[i / BITS_PER_LONG]; - - if (gofast && v == ~0UL) { - int order; - - page = pfn_to_page(pfn); - count += BITS_PER_LONG; - order = ffs(BITS_PER_LONG) - 1; - __free_pages_bootmem(page, order); - i += BITS_PER_LONG; - page += BITS_PER_LONG; - } else if (v) { - unsigned long m; - - page = pfn_to_page(pfn); - for (m = 1; m && i < idx; m<<=1, page++, i++) { - if (v & m) { - count++; - __free_pages_bootmem(page, 0); - } - } - } else { - i += BITS_PER_LONG; - } - pfn += BITS_PER_LONG; - } - - /* - * Now free the allocator bitmap itself, it's not - * needed anymore: - */ - page = virt_to_page(bdata->node_bootmem_map); - idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT; - for (i = 0; i < idx; i++, page++) - __free_pages_bootmem(page, 0); - count += i; - bdata->node_bootmem_map = NULL; - - return count; -} - -unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn, - unsigned long startpfn, unsigned long endpfn) -{ - return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn); -} - -int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, - unsigned long size, int flags) -{ - int ret; - - ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); - if (ret < 0) - return -ENOMEM; - reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); - - return 0; -} - -void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, - unsigned long size) -{ - free_bootmem_core(pgdat->bdata, physaddr, size); -} - -unsigned long __init free_all_bootmem_node(pg_data_t *pgdat) -{ - register_page_bootmem_info_node(pgdat); - return free_all_bootmem_core(pgdat->bdata); -} - -unsigned long __init init_bootmem(unsigned long start, unsigned long pages) -{ - max_low_pfn = pages; - min_low_pfn = start; - return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); -} - -#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE -int __init reserve_bootmem(unsigned long addr, unsigned long size, - int flags) -{ - bootmem_data_t *bdata; - int ret; - - list_for_each_entry(bdata, &bdata_list, list) { - ret = can_reserve_bootmem_core(bdata, addr, size, flags); - if (ret < 0) - return ret; - } - list_for_each_entry(bdata, &bdata_list, list) - reserve_bootmem_core(bdata, addr, size, flags); - - return 0; -} -#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ - -void __init free_bootmem(unsigned long addr, unsigned long size) -{ - bootmem_data_t *bdata; - list_for_each_entry(bdata, &bdata_list, list) - free_bootmem_core(bdata, addr, size); -} - -unsigned long __init free_all_bootmem(void) -{ - return free_all_bootmem_core(NODE_DATA(0)->bdata); -} - void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, unsigned long goal) { @@ -534,7 +533,6 @@ void * __init __alloc_bootmem(unsigned long size, unsigned long align, return NULL; } - void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { -- cgit v1.2.3-70-g09d2 From 5f2809e69c7128f86316048221cf45146f69a4a0 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 23 Jul 2008 21:28:05 -0700 Subject: bootmem: clean up alloc_bootmem_core alloc_bootmem_core has become quite nasty to read over time. This is a clean rewrite that keeps the semantics. bdata->last_pos has been dropped. bdata->last_success has been renamed to hint_idx and it is now an index relative to the node's range. Since further block searching might start at this index, it is now set to the end of a succeeded allocation rather than its beginning. bdata->last_offset has been renamed to last_end_off to be more clear that it represents the ending address of the last allocation relative to the node. [y-goto@jp.fujitsu.com: fix new alloc_bootmem_core()] Signed-off-by: Johannes Weiner Signed-off-by: Yasunori Goto Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/bootmem.h | 6 +- mm/bootmem.c | 212 +++++++++++++++++------------------------------- 2 files changed, 78 insertions(+), 140 deletions(-) (limited to 'include/linux/bootmem.h') diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index 5000fd70b04..90921d10ffa 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -31,10 +31,8 @@ typedef struct bootmem_data { unsigned long node_boot_start; unsigned long node_low_pfn; void *node_bootmem_map; - unsigned long last_offset; - unsigned long last_pos; - unsigned long last_success; /* Previous allocation point. To speed - * up searching */ + unsigned long last_end_off; + unsigned long hint_idx; struct list_head list; } bootmem_data_t; diff --git a/mm/bootmem.c b/mm/bootmem.c index 300d126ec53..94ea612decc 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c @@ -242,8 +242,9 @@ static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, * considered reserved. */ - if (addr >= bdata->node_boot_start && addr < bdata->last_success) - bdata->last_success = addr; + if (addr >= bdata->node_boot_start && + PFN_DOWN(addr - bdata->node_boot_start) < bdata->hint_idx) + bdata->hint_idx = PFN_DOWN(addr - bdata->node_boot_start); /* * Round up to index to the range. @@ -431,36 +432,16 @@ int __init reserve_bootmem(unsigned long addr, unsigned long size, } #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ -/* - * We 'merge' subsequent allocations to save space. We might 'lose' - * some fraction of a page if allocations cannot be satisfied due to - * size constraints on boxes where there is physical RAM space - * fragmentation - in these cases (mostly large memory boxes) this - * is not a problem. - * - * On low memory boxes we get it right in 100% of the cases. - * - * alignment has to be a power of 2 value. - * - * NOTE: This function is _not_ reentrant. - */ -static void * __init -alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, - unsigned long align, unsigned long goal, unsigned long limit) +static void * __init alloc_bootmem_core(struct bootmem_data *bdata, + unsigned long size, unsigned long align, + unsigned long goal, unsigned long limit) { - unsigned long areasize, preferred; - unsigned long i, start = 0, incr, eidx, end_pfn; - void *ret; - unsigned long node_boot_start; - void *node_bootmem_map; - - if (!size) { - printk("alloc_bootmem_core(): zero-sized request\n"); - BUG(); - } - BUG_ON(align & (align-1)); + unsigned long min, max, start, sidx, midx, step; + + BUG_ON(!size); + BUG_ON(align & (align - 1)); + BUG_ON(limit && goal + size > limit); - /* on nodes without memory - bootmem_map is NULL */ if (!bdata->node_bootmem_map) return NULL; @@ -468,126 +449,85 @@ alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT, align, goal, limit); - /* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */ - node_boot_start = bdata->node_boot_start; - node_bootmem_map = bdata->node_bootmem_map; - if (align) { - node_boot_start = ALIGN(bdata->node_boot_start, align); - if (node_boot_start > bdata->node_boot_start) - node_bootmem_map = (unsigned long *)bdata->node_bootmem_map + - PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG; - } + min = PFN_DOWN(bdata->node_boot_start); + max = bdata->node_low_pfn; - if (limit && node_boot_start >= limit) + goal >>= PAGE_SHIFT; + limit >>= PAGE_SHIFT; + + if (limit && max > limit) + max = limit; + if (max <= min) return NULL; - end_pfn = bdata->node_low_pfn; - limit = PFN_DOWN(limit); - if (limit && end_pfn > limit) - end_pfn = limit; + step = max(align >> PAGE_SHIFT, 1UL); - eidx = end_pfn - PFN_DOWN(node_boot_start); + if (goal && min < goal && goal < max) + start = ALIGN(goal, step); + else + start = ALIGN(min, step); - /* - * We try to allocate bootmem pages above 'goal' - * first, then we try to allocate lower pages. - */ - preferred = 0; - if (goal && PFN_DOWN(goal) < end_pfn) { - if (goal > node_boot_start) - preferred = goal - node_boot_start; - - if (bdata->last_success > node_boot_start && - bdata->last_success - node_boot_start >= preferred) - if (!limit || (limit && limit > bdata->last_success)) - preferred = bdata->last_success - node_boot_start; - } + sidx = start - PFN_DOWN(bdata->node_boot_start); + midx = max - PFN_DOWN(bdata->node_boot_start); - preferred = PFN_DOWN(ALIGN(preferred, align)); - areasize = (size + PAGE_SIZE-1) / PAGE_SIZE; - incr = align >> PAGE_SHIFT ? : 1; + if (bdata->hint_idx > sidx) { + /* Make sure we retry on failure */ + goal = 1; + sidx = ALIGN(bdata->hint_idx, step); + } -restart_scan: - for (i = preferred; i < eidx;) { - unsigned long j; + while (1) { + int merge; + void *region; + unsigned long eidx, i, start_off, end_off; +find_block: + sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx); + sidx = ALIGN(sidx, step); + eidx = sidx + PFN_UP(size); - i = find_next_zero_bit(node_bootmem_map, eidx, i); - i = ALIGN(i, incr); - if (i >= eidx) + if (sidx >= midx || eidx > midx) break; - if (test_bit(i, node_bootmem_map)) { - i += incr; - continue; - } - for (j = i + 1; j < i + areasize; ++j) { - if (j >= eidx) - goto fail_block; - if (test_bit(j, node_bootmem_map)) - goto fail_block; - } - start = i; - goto found; - fail_block: - i = ALIGN(j, incr); - if (i == j) - i += incr; - } - - if (preferred > 0) { - preferred = 0; - goto restart_scan; - } - return NULL; -found: - bdata->last_success = PFN_PHYS(start) + node_boot_start; - BUG_ON(start >= eidx); + for (i = sidx; i < eidx; i++) + if (test_bit(i, bdata->node_bootmem_map)) { + sidx = ALIGN(i, step); + if (sidx == i) + sidx += step; + goto find_block; + } - /* - * Is the next page of the previous allocation-end the start - * of this allocation's buffer? If yes then we can 'merge' - * the previous partial page with this allocation. - */ - if (align < PAGE_SIZE && - bdata->last_offset && bdata->last_pos+1 == start) { - unsigned long offset, remaining_size; - offset = ALIGN(bdata->last_offset, align); - BUG_ON(offset > PAGE_SIZE); - remaining_size = PAGE_SIZE - offset; - if (size < remaining_size) { - areasize = 0; - /* last_pos unchanged */ - bdata->last_offset = offset + size; - ret = phys_to_virt(bdata->last_pos * PAGE_SIZE + - offset + node_boot_start); - } else { - remaining_size = size - remaining_size; - areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE; - ret = phys_to_virt(bdata->last_pos * PAGE_SIZE + - offset + node_boot_start); - bdata->last_pos = start + areasize - 1; - bdata->last_offset = remaining_size; - } - bdata->last_offset &= ~PAGE_MASK; - } else { - bdata->last_pos = start + areasize - 1; - bdata->last_offset = size & ~PAGE_MASK; - ret = phys_to_virt(start * PAGE_SIZE + node_boot_start); + if (bdata->last_end_off && + PFN_DOWN(bdata->last_end_off) + 1 == sidx) + start_off = ALIGN(bdata->last_end_off, align); + else + start_off = PFN_PHYS(sidx); + + merge = PFN_DOWN(start_off) < sidx; + end_off = start_off + size; + + bdata->last_end_off = end_off; + bdata->hint_idx = PFN_UP(end_off); + + /* + * Reserve the area now: + */ + for (i = PFN_DOWN(start_off) + merge; + i < PFN_UP(end_off); i++) + if (test_and_set_bit(i, bdata->node_bootmem_map)) + BUG(); + + region = phys_to_virt(bdata->node_boot_start + start_off); + memset(region, 0, size); + return region; } - bdebug("nid=%td start=%lx end=%lx\n", - bdata - bootmem_node_data, - start + PFN_DOWN(bdata->node_boot_start), - start + areasize + PFN_DOWN(bdata->node_boot_start)); + if (goal) { + goal = 0; + sidx = 0; + goto find_block; + } - /* - * Reserve the area now: - */ - for (i = start; i < start + areasize; i++) - if (unlikely(test_and_set_bit(i, node_bootmem_map))) - BUG(); - memset(ret, 0, size); - return ret; + return NULL; } /** -- cgit v1.2.3-70-g09d2 From 3560e249abda6bee41a07a7bf0383a6e193e2839 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 23 Jul 2008 21:28:09 -0700 Subject: bootmem: replace node_boot_start in struct bootmem_data Almost all users of this field need a PFN instead of a physical address, so replace node_boot_start with node_min_pfn. [Lee.Schermerhorn@hp.com: fix spurious BUG_ON() in mark_bootmem()] Signed-off-by: Johannes Weiner Cc: Signed-off-by: Lee Schermerhorn Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/mm/numa.c | 2 +- arch/arm/plat-omap/fb.c | 4 +--- arch/avr32/mm/init.c | 3 +-- arch/ia64/mm/discontig.c | 19 ++++++++++--------- arch/m32r/mm/discontig.c | 3 +-- arch/m32r/mm/init.c | 4 +--- arch/mn10300/mm/init.c | 6 +++--- arch/sh/mm/init.c | 2 +- include/linux/bootmem.h | 2 +- mm/bootmem.c | 40 +++++++++++++++++++++------------------- 10 files changed, 41 insertions(+), 44 deletions(-) (limited to 'include/linux/bootmem.h') diff --git a/arch/alpha/mm/numa.c b/arch/alpha/mm/numa.c index def0c74a78a..d8c4ceaf00b 100644 --- a/arch/alpha/mm/numa.c +++ b/arch/alpha/mm/numa.c @@ -304,7 +304,7 @@ void __init paging_init(void) for_each_online_node(nid) { bootmem_data_t *bdata = &bootmem_node_data[nid]; - unsigned long start_pfn = bdata->node_boot_start >> PAGE_SHIFT; + unsigned long start_pfn = bdata->node_min_pfn; unsigned long end_pfn = bdata->node_low_pfn; if (dma_local_pfn >= end_pfn - start_pfn) diff --git a/arch/arm/plat-omap/fb.c b/arch/arm/plat-omap/fb.c index 7854f19b77c..96d6f061973 100644 --- a/arch/arm/plat-omap/fb.c +++ b/arch/arm/plat-omap/fb.c @@ -182,7 +182,7 @@ void __init omapfb_reserve_sdram(void) return; bdata = NODE_DATA(0)->bdata; - sdram_start = bdata->node_boot_start; + sdram_start = bdata->node_min_pfn << PAGE_SHIFT; sdram_size = (bdata->node_low_pfn << PAGE_SHIFT) - sdram_start; reserved = 0; for (i = 0; ; i++) { @@ -340,5 +340,3 @@ unsigned long omapfb_reserve_sram(unsigned long sram_pstart, #endif - - diff --git a/arch/avr32/mm/init.c b/arch/avr32/mm/init.c index 786de88a82a..3c85fdaa948 100644 --- a/arch/avr32/mm/init.c +++ b/arch/avr32/mm/init.c @@ -119,8 +119,7 @@ void __init paging_init(void) unsigned long zones_size[MAX_NR_ZONES]; unsigned long low, start_pfn; - start_pfn = pgdat->bdata->node_boot_start; - start_pfn >>= PAGE_SHIFT; + start_pfn = pgdat->bdata->node_min_pfn; low = pgdat->bdata->node_low_pfn; memset(zones_size, 0, sizeof(zones_size)); diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index 2fcf8464331..d83125e1ed2 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -74,17 +74,17 @@ pg_data_t *pgdat_list[MAX_NUMNODES]; static int __init build_node_maps(unsigned long start, unsigned long len, int node) { - unsigned long cstart, epfn, end = start + len; + unsigned long spfn, epfn, end = start + len; struct bootmem_data *bdp = &bootmem_node_data[node]; epfn = GRANULEROUNDUP(end) >> PAGE_SHIFT; - cstart = GRANULEROUNDDOWN(start); + spfn = GRANULEROUNDDOWN(start) >> PAGE_SHIFT; if (!bdp->node_low_pfn) { - bdp->node_boot_start = cstart; + bdp->node_min_pfn = spfn; bdp->node_low_pfn = epfn; } else { - bdp->node_boot_start = min(cstart, bdp->node_boot_start); + bdp->node_min_pfn = min(spfn, bdp->node_min_pfn); bdp->node_low_pfn = max(epfn, bdp->node_low_pfn); } @@ -221,20 +221,21 @@ static void __init fill_pernode(int node, unsigned long pernode, static int __init find_pernode_space(unsigned long start, unsigned long len, int node) { - unsigned long epfn; + unsigned long spfn, epfn; unsigned long pernodesize = 0, pernode, pages, mapsize; struct bootmem_data *bdp = &bootmem_node_data[node]; + spfn = start >> PAGE_SHIFT; epfn = (start + len) >> PAGE_SHIFT; - pages = bdp->node_low_pfn - (bdp->node_boot_start >> PAGE_SHIFT); + pages = bdp->node_low_pfn - bdp->node_min_pfn; mapsize = bootmem_bootmap_pages(pages) << PAGE_SHIFT; /* * Make sure this memory falls within this node's usable memory * since we may have thrown some away in build_maps(). */ - if (start < bdp->node_boot_start || epfn > bdp->node_low_pfn) + if (spfn < bdp->node_min_pfn || epfn > bdp->node_low_pfn) return 0; /* Don't setup this node's local space twice... */ @@ -296,7 +297,7 @@ static void __init reserve_pernode_space(void) bdp = pdp->bdata; /* First the bootmem_map itself */ - pages = bdp->node_low_pfn - (bdp->node_boot_start>>PAGE_SHIFT); + pages = bdp->node_low_pfn - bdp->node_min_pfn; size = bootmem_bootmap_pages(pages) << PAGE_SHIFT; base = __pa(bdp->node_bootmem_map); reserve_bootmem_node(pdp, base, size, BOOTMEM_DEFAULT); @@ -466,7 +467,7 @@ void __init find_memory(void) init_bootmem_node(pgdat_list[node], map>>PAGE_SHIFT, - bdp->node_boot_start>>PAGE_SHIFT, + bdp->node_min_pfn, bdp->node_low_pfn); } diff --git a/arch/m32r/mm/discontig.c b/arch/m32r/mm/discontig.c index cc23934bc41..cbc3c4c5456 100644 --- a/arch/m32r/mm/discontig.c +++ b/arch/m32r/mm/discontig.c @@ -123,8 +123,7 @@ unsigned long __init setup_memory(void) return max_low_pfn; } -#define START_PFN(nid) \ - (NODE_DATA(nid)->bdata->node_boot_start >> PAGE_SHIFT) +#define START_PFN(nid) (NODE_DATA(nid)->bdata->node_min_pfn) #define MAX_LOW_PFN(nid) (NODE_DATA(nid)->bdata->node_low_pfn) unsigned long __init zone_sizes_init(void) diff --git a/arch/m32r/mm/init.c b/arch/m32r/mm/init.c index 28799af15e9..2554eb59cfe 100644 --- a/arch/m32r/mm/init.c +++ b/arch/m32r/mm/init.c @@ -93,8 +93,7 @@ void free_initrd_mem(unsigned long, unsigned long); #endif /* It'd be good if these lines were in the standard header file. */ -#define START_PFN(nid) \ - (NODE_DATA(nid)->bdata->node_boot_start >> PAGE_SHIFT) +#define START_PFN(nid) (NODE_DATA(nid)->bdata->node_min_pfn) #define MAX_LOW_PFN(nid) (NODE_DATA(nid)->bdata->node_low_pfn) #ifndef CONFIG_DISCONTIGMEM @@ -252,4 +251,3 @@ void free_initrd_mem(unsigned long start, unsigned long end) printk (KERN_INFO "Freeing initrd memory: %ldk freed\n", (end - start) >> 10); } #endif - diff --git a/arch/mn10300/mm/init.c b/arch/mn10300/mm/init.c index 8c5d88c7b90..8cee387a24f 100644 --- a/arch/mn10300/mm/init.c +++ b/arch/mn10300/mm/init.c @@ -67,8 +67,8 @@ void __init paging_init(void) /* declare the sizes of the RAM zones (only use the normal zone) */ zones_size[ZONE_NORMAL] = - (contig_page_data.bdata->node_low_pfn) - - (contig_page_data.bdata->node_boot_start >> PAGE_SHIFT); + contig_page_data.bdata->node_low_pfn - + contig_page_data.bdata->node_min_pfn; /* pass the memory from the bootmem allocator to the main allocator */ free_area_init(zones_size); @@ -87,7 +87,7 @@ void __init mem_init(void) if (!mem_map) BUG(); -#define START_PFN (contig_page_data.bdata->node_boot_start >> PAGE_SHIFT) +#define START_PFN (contig_page_data.bdata->node_min_pfn) #define MAX_LOW_PFN (contig_page_data.bdata->node_low_pfn) max_mapnr = num_physpages = MAX_LOW_PFN - START_PFN; diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index d7df26bd1e5..d652d375eb1 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -191,7 +191,7 @@ void __init paging_init(void) pg_data_t *pgdat = NODE_DATA(nid); unsigned long low, start_pfn; - start_pfn = pgdat->bdata->node_boot_start >> PAGE_SHIFT; + start_pfn = pgdat->bdata->node_min_pfn; low = pgdat->bdata->node_low_pfn; if (max_zone_pfns[ZONE_NORMAL] < low) diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index 90921d10ffa..4ddf2922fc8 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -28,7 +28,7 @@ extern unsigned long saved_max_pfn; * memory pages (including holes) on the node. */ typedef struct bootmem_data { - unsigned long node_boot_start; + unsigned long node_min_pfn; unsigned long node_low_pfn; void *node_bootmem_map; unsigned long last_end_off; diff --git a/mm/bootmem.c b/mm/bootmem.c index 282b786c2b1..4af15d0340a 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c @@ -80,7 +80,7 @@ static void __init link_bootmem(bootmem_data_t *bdata) bootmem_data_t *ent; ent = list_entry(iter, bootmem_data_t, list); - if (bdata->node_boot_start < ent->node_boot_start) + if (bdata->node_min_pfn < ent->node_min_pfn) break; } list_add_tail(&bdata->list, iter); @@ -96,7 +96,7 @@ static unsigned long __init init_bootmem_core(bootmem_data_t *bdata, mminit_validate_memmodel_limits(&start, &end); bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart)); - bdata->node_boot_start = PFN_PHYS(start); + bdata->node_min_pfn = start; bdata->node_low_pfn = end; link_bootmem(bdata); @@ -151,7 +151,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) if (!bdata->node_bootmem_map) return 0; - start = PFN_DOWN(bdata->node_boot_start); + start = bdata->node_min_pfn; end = bdata->node_low_pfn; /* @@ -167,7 +167,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) unsigned long *map, idx, vec; map = bdata->node_bootmem_map; - idx = start - PFN_DOWN(bdata->node_boot_start); + idx = start - bdata->node_min_pfn; vec = ~map[idx / BITS_PER_LONG]; if (aligned && vec == ~0UL && start + BITS_PER_LONG < end) { @@ -192,7 +192,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) } page = virt_to_page(bdata->node_bootmem_map); - pages = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start); + pages = bdata->node_low_pfn - bdata->node_min_pfn; pages = bootmem_bootmap_pages(pages); count += pages; while (pages--) @@ -231,8 +231,8 @@ static void __init __free(bootmem_data_t *bdata, unsigned long idx; bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data, - sidx + PFN_DOWN(bdata->node_boot_start), - eidx + PFN_DOWN(bdata->node_boot_start)); + sidx + bdata->node_min_pfn, + eidx + bdata->node_min_pfn); if (bdata->hint_idx > sidx) bdata->hint_idx = sidx; @@ -250,8 +250,8 @@ static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx, bdebug("nid=%td start=%lx end=%lx flags=%x\n", bdata - bootmem_node_data, - sidx + PFN_DOWN(bdata->node_boot_start), - eidx + PFN_DOWN(bdata->node_boot_start), + sidx + bdata->node_min_pfn, + eidx + bdata->node_min_pfn, flags); for (idx = sidx; idx < eidx; idx++) @@ -261,7 +261,7 @@ static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx, return -EBUSY; } bdebug("silent double reserve of PFN %lx\n", - idx + PFN_DOWN(bdata->node_boot_start)); + idx + bdata->node_min_pfn); } return 0; } @@ -275,11 +275,11 @@ static int __init mark_bootmem_node(bootmem_data_t *bdata, bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x\n", bdata - bootmem_node_data, start, end, reserve, flags); - BUG_ON(start < PFN_DOWN(bdata->node_boot_start)); + BUG_ON(start < bdata->node_min_pfn); BUG_ON(end > bdata->node_low_pfn); - sidx = start - PFN_DOWN(bdata->node_boot_start); - eidx = end - PFN_DOWN(bdata->node_boot_start); + sidx = start - bdata->node_min_pfn; + eidx = end - bdata->node_min_pfn; if (reserve) return __reserve(bdata, sidx, eidx, flags); @@ -299,7 +299,8 @@ static int __init mark_bootmem(unsigned long start, unsigned long end, int err; unsigned long max; - if (pos < PFN_DOWN(bdata->node_boot_start)) { + if (pos < bdata->node_min_pfn || + pos >= bdata->node_low_pfn) { BUG_ON(pos != start); continue; } @@ -422,7 +423,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT, align, goal, limit); - min = PFN_DOWN(bdata->node_boot_start); + min = bdata->node_min_pfn; max = bdata->node_low_pfn; goal >>= PAGE_SHIFT; @@ -440,8 +441,8 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, else start = ALIGN(min, step); - sidx = start - PFN_DOWN(bdata->node_boot_start); - midx = max - PFN_DOWN(bdata->node_boot_start); + sidx = start - bdata->node_min_pfn;; + midx = max - bdata->node_min_pfn; if (bdata->hint_idx > sidx) { /* @@ -491,7 +492,8 @@ find_block: PFN_UP(end_off), BOOTMEM_EXCLUSIVE)) BUG(); - region = phys_to_virt(bdata->node_boot_start + start_off); + region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) + + start_off); memset(region, 0, size); return region; } @@ -518,7 +520,7 @@ restart: if (goal && bdata->node_low_pfn <= PFN_DOWN(goal)) continue; - if (limit && bdata->node_boot_start >= limit) + if (limit && bdata->node_min_pfn >= PFN_DOWN(limit)) break; region = alloc_bootmem_core(bdata, size, align, goal, limit); -- cgit v1.2.3-70-g09d2 From c6af5e9f8a57467df2e55e428316a43480174521 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Fri, 25 Jul 2008 15:48:04 +0200 Subject: bootmem: Move node allocation macros back to !HAVE_ARCH_BOOTMEM_NODE These got unintentionally moved, put them back as x86 provides its own versions. Signed-off-by: Johannes Weiner Signed-off-by: Linus Torvalds --- include/linux/bootmem.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux/bootmem.h') diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index 4ddf2922fc8..652470b687c 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -103,17 +103,16 @@ extern void *__alloc_bootmem_low_node(pg_data_t *pgdat, __alloc_bootmem(x, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_low_pages(x) \ __alloc_bootmem_low(x, PAGE_SIZE, 0) -#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ - -extern int reserve_bootmem_generic(unsigned long addr, unsigned long size, - int flags); - #define alloc_bootmem_node(pgdat, x) \ __alloc_bootmem_node(pgdat, x, SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_pages_node(pgdat, x) \ __alloc_bootmem_node(pgdat, x, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_low_pages_node(pgdat, x) \ __alloc_bootmem_low_node(pgdat, x, PAGE_SIZE, 0) +#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ + +extern int reserve_bootmem_generic(unsigned long addr, unsigned long size, + int flags); extern void *alloc_bootmem_section(unsigned long size, unsigned long section_nr); -- cgit v1.2.3-70-g09d2 From 74768ed833344bb0f82b97cee46320a3d7f09ecd Mon Sep 17 00:00:00 2001 From: Jan Beulich Date: Tue, 12 Aug 2008 15:08:39 -0700 Subject: page allocator: use no-panic variant of alloc_bootmem() in alloc_large_system_hash() .. since a failed allocation is being (initially) handled gracefully, and panic()-ed upon failure explicitly in the function if retries with smaller sizes failed. Signed-off-by: Jan Beulich Signed-off-by: David Howells Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/bootmem.h | 4 ++++ mm/page_alloc.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux/bootmem.h') diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index 652470b687c..95837bfb525 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h @@ -97,10 +97,14 @@ extern void *__alloc_bootmem_low_node(pg_data_t *pgdat, #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE #define alloc_bootmem(x) \ __alloc_bootmem(x, SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS)) +#define alloc_bootmem_nopanic(x) \ + __alloc_bootmem_nopanic(x, SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_low(x) \ __alloc_bootmem_low(x, SMP_CACHE_BYTES, 0) #define alloc_bootmem_pages(x) \ __alloc_bootmem(x, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)) +#define alloc_bootmem_pages_nopanic(x) \ + __alloc_bootmem_nopanic(x, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)) #define alloc_bootmem_low_pages(x) \ __alloc_bootmem_low(x, PAGE_SIZE, 0) #define alloc_bootmem_node(pgdat, x) \ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 401d104d2bb..af982f7cdb2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4437,7 +4437,7 @@ void *__init alloc_large_system_hash(const char *tablename, do { size = bucketsize << log2qty; if (flags & HASH_EARLY) - table = alloc_bootmem(size); + table = alloc_bootmem_nopanic(size); else if (hashdist) table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); else { -- cgit v1.2.3-70-g09d2