From b888132b0ff29ca0733589a594c243ed612438eb Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Wed, 11 Oct 2006 01:20:40 -0700 Subject: [PATCH] mm: remove memmap_zone_idx() memmap_zone_idx() is not used anymore. It was required by an earlier version of account-for-memmap-and-optionally-the-kernel-image-as-holes.patch but not any more. Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'mm/page_alloc.c') diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a8c003e7b3d..8246e834b2f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2294,19 +2294,6 @@ unsigned long __init zone_absent_pages_in_node(int nid, return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); } -/* Return the zone index a PFN is in */ -int memmap_zone_idx(struct page *lmem_map) -{ - int i; - unsigned long phys_addr = virt_to_phys(lmem_map); - unsigned long pfn = phys_addr >> PAGE_SHIFT; - - for (i = 0; i < MAX_NR_ZONES; i++) - if (pfn < arch_zone_highest_possible_pfn[i]) - break; - - return i; -} #else static inline unsigned long zone_spanned_pages_in_node(int nid, unsigned long zone_type, @@ -2325,10 +2312,6 @@ static inline unsigned long zone_absent_pages_in_node(int nid, return zholes_size[zone_type]; } -static inline int memmap_zone_idx(struct page *lmem_map) -{ - return MAX_NR_ZONES; -} #endif static void __init calculate_node_totalpages(struct pglist_data *pgdat, -- cgit v1.2.3-70-g09d2 From dafb13673c463bc2aade4a4819704dde0f5fa37f Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 11 Oct 2006 01:21:30 -0700 Subject: [PATCH] mm: arch_free_page fix After the PG_reserved check was added, arch_free_page was being called in the wrong place (it could be called for a page we don't actually want to free). Fix that. Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'mm/page_alloc.c') diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 8246e834b2f..c5caac2c3c5 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -495,7 +495,6 @@ static void __free_pages_ok(struct page *page, unsigned int order) int i; int reserved = 0; - arch_free_page(page, order); if (!PageHighMem(page)) debug_check_no_locks_freed(page_address(page), PAGE_SIZE<mapping = NULL; if (free_pages_check(page)) return; + arch_free_page(page, 0); kernel_map_pages(page, 1, 0); pcp = &zone_pcp(zone, get_cpu())->pcp[cold]; -- cgit v1.2.3-70-g09d2 From 9858db504caedb2424b9a32744c23f9a81ec1731 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 11 Oct 2006 01:21:30 -0700 Subject: [PATCH] mm: locks_freed fix Move the lock debug checks below the page reserved checks. Also, having debug_check_no_locks_freed in kernel_map_pages is wrong. Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 7 +------ mm/page_alloc.c | 8 ++++---- 2 files changed, 5 insertions(+), 10 deletions(-) (limited to 'mm/page_alloc.c') diff --git a/include/linux/mm.h b/include/linux/mm.h index 26146623be2..5a6068ff555 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1103,12 +1103,7 @@ static inline void vm_stat_account(struct mm_struct *mm, #ifndef CONFIG_DEBUG_PAGEALLOC static inline void -kernel_map_pages(struct page *page, int numpages, int enable) -{ - if (!PageHighMem(page) && !enable) - debug_check_no_locks_freed(page_address(page), - numpages * PAGE_SIZE); -} +kernel_map_pages(struct page *page, int numpages, int enable) {} #endif extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c5caac2c3c5..40db96a655d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -495,15 +495,13 @@ static void __free_pages_ok(struct page *page, unsigned int order) int i; int reserved = 0; - if (!PageHighMem(page)) - debug_check_no_locks_freed(page_address(page), - PAGE_SIZE< Date: Thu, 19 Oct 2006 23:28:16 -0700 Subject: [PATCH] separate bdi congestion functions from queue congestion functions Separate out the concept of "queue congestion" from "backing-dev congestion". Congestion is a backing-dev concept, not a queue concept. The blk_* congestion functions are retained, as wrappers around the core backing-dev congestion functions. This proper layering is needed so that NFS can cleanly use the congestion functions, and so that CONFIG_BLOCK=n actually links. Cc: "Thomas Maier" Cc: "Jens Axboe" Cc: Trond Myklebust Cc: David Howells Cc: Peter Osterlund Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/i386/lib/usercopy.c | 3 +- block/ll_rw_blk.c | 71 --------------------------------------------- drivers/md/dm-crypt.c | 3 +- fs/fat/file.c | 3 +- fs/nfs/write.c | 4 ++- fs/reiserfs/journal.c | 3 +- fs/xfs/linux-2.6/kmem.c | 5 ++-- fs/xfs/linux-2.6/xfs_buf.c | 3 +- include/linux/backing-dev.h | 7 +++++ include/linux/blkdev.h | 24 ++++++++++++--- include/linux/writeback.h | 1 - mm/Makefile | 3 +- mm/backing-dev.c | 69 +++++++++++++++++++++++++++++++++++++++++++ mm/page-writeback.c | 17 +++-------- mm/page_alloc.c | 5 ++-- mm/shmem.c | 3 +- mm/vmscan.c | 6 ++-- 17 files changed, 126 insertions(+), 104 deletions(-) create mode 100644 mm/backing-dev.c (limited to 'mm/page_alloc.c') diff --git a/arch/i386/lib/usercopy.c b/arch/i386/lib/usercopy.c index 258df6b4d7d..d22cfc9d656 100644 --- a/arch/i386/lib/usercopy.c +++ b/arch/i386/lib/usercopy.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -741,7 +742,7 @@ survive: if (retval == -ENOMEM && is_init(current)) { up_read(¤t->mm->mmap_sem); - blk_congestion_wait(WRITE, HZ/50); + congestion_wait(WRITE, HZ/50); goto survive; } diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 132a858ce2c..136066583c6 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -56,11 +56,6 @@ static kmem_cache_t *requestq_cachep; */ static kmem_cache_t *iocontext_cachep; -static wait_queue_head_t congestion_wqh[2] = { - __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]), - __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1]) - }; - /* * Controlling structure to kblockd */ @@ -112,37 +107,6 @@ static void blk_queue_congestion_threshold(struct request_queue *q) q->nr_congestion_off = nr; } -/* - * A queue has just exitted congestion. Note this in the global counter of - * congested queues, and wake up anyone who was waiting for requests to be - * put back. - */ -void blk_clear_queue_congested(request_queue_t *q, int rw) -{ - enum bdi_state bit; - wait_queue_head_t *wqh = &congestion_wqh[rw]; - - bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested; - clear_bit(bit, &q->backing_dev_info.state); - smp_mb__after_clear_bit(); - if (waitqueue_active(wqh)) - wake_up(wqh); -} -EXPORT_SYMBOL(blk_clear_queue_congested); - -/* - * A queue has just entered congestion. Flag that in the queue's VM-visible - * state flags and increment the global gounter of congested queues. - */ -void blk_set_queue_congested(request_queue_t *q, int rw) -{ - enum bdi_state bit; - - bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested; - set_bit(bit, &q->backing_dev_info.state); -} -EXPORT_SYMBOL(blk_set_queue_congested); - /** * blk_get_backing_dev_info - get the address of a queue's backing_dev_info * @bdev: device @@ -2755,41 +2719,6 @@ void blk_end_sync_rq(struct request *rq, int error) } EXPORT_SYMBOL(blk_end_sync_rq); -/** - * blk_congestion_wait - wait for a queue to become uncongested - * @rw: READ or WRITE - * @timeout: timeout in jiffies - * - * Waits for up to @timeout jiffies for a queue (any queue) to exit congestion. - * If no queues are congested then just wait for the next request to be - * returned. - */ -long blk_congestion_wait(int rw, long timeout) -{ - long ret; - DEFINE_WAIT(wait); - wait_queue_head_t *wqh = &congestion_wqh[rw]; - - prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); - ret = io_schedule_timeout(timeout); - finish_wait(wqh, &wait); - return ret; -} - -EXPORT_SYMBOL(blk_congestion_wait); - -/** - * blk_congestion_end - wake up sleepers on a congestion queue - * @rw: READ or WRITE - */ -void blk_congestion_end(int rw) -{ - wait_queue_head_t *wqh = &congestion_wqh[rw]; - - if (waitqueue_active(wqh)) - wake_up(wqh); -} - /* * Has to be called with the request spinlock acquired */ diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 655d816760e..a625576fdee 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -602,7 +603,7 @@ static void process_write(struct crypt_io *io) /* out of memory -> run queues */ if (remaining) - blk_congestion_wait(bio_data_dir(clone), HZ/100); + congestion_wait(bio_data_dir(clone), HZ/100); } } diff --git a/fs/fat/file.c b/fs/fat/file.c index f4b8f8b3fbd..8337451e789 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -13,6 +13,7 @@ #include #include #include +#include #include int fat_generic_ioctl(struct inode *inode, struct file *filp, @@ -118,7 +119,7 @@ static int fat_file_release(struct inode *inode, struct file *filp) if ((filp->f_mode & FMODE_WRITE) && MSDOS_SB(inode->i_sb)->options.flush) { fat_flush_inodes(inode->i_sb, inode, NULL); - blk_congestion_wait(WRITE, HZ/10); + congestion_wait(WRITE, HZ/10); } return 0; } diff --git a/fs/nfs/write.c b/fs/nfs/write.c index f6675d2c386..ca92ac36fe9 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -57,6 +57,8 @@ #include #include #include +#include + #include #include @@ -395,7 +397,7 @@ int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) out: clear_bit(BDI_write_congested, &bdi->state); wake_up_all(&nfs_write_congestion); - writeback_congestion_end(); + congestion_end(WRITE); return err; } diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c index ad8cbc49883..85ce2326830 100644 --- a/fs/reiserfs/journal.c +++ b/fs/reiserfs/journal.c @@ -53,6 +53,7 @@ #include #include #include +#include /* gets a struct reiserfs_journal_list * from a list head */ #define JOURNAL_LIST_ENTRY(h) (list_entry((h), struct reiserfs_journal_list, \ @@ -970,7 +971,7 @@ int reiserfs_async_progress_wait(struct super_block *s) DEFINE_WAIT(wait); struct reiserfs_journal *j = SB_JOURNAL(s); if (atomic_read(&j->j_async_throttle)) - blk_congestion_wait(WRITE, HZ / 10); + congestion_wait(WRITE, HZ / 10); return 0; } diff --git a/fs/xfs/linux-2.6/kmem.c b/fs/xfs/linux-2.6/kmem.c index d5973758981..004baf60061 100644 --- a/fs/xfs/linux-2.6/kmem.c +++ b/fs/xfs/linux-2.6/kmem.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "time.h" #include "kmem.h" @@ -53,7 +54,7 @@ kmem_alloc(size_t size, unsigned int __nocast flags) printk(KERN_ERR "XFS: possible memory allocation " "deadlock in %s (mode:0x%x)\n", __FUNCTION__, lflags); - blk_congestion_wait(WRITE, HZ/50); + congestion_wait(WRITE, HZ/50); } while (1); } @@ -131,7 +132,7 @@ kmem_zone_alloc(kmem_zone_t *zone, unsigned int __nocast flags) printk(KERN_ERR "XFS: possible memory allocation " "deadlock in %s (mode:0x%x)\n", __FUNCTION__, lflags); - blk_congestion_wait(WRITE, HZ/50); + congestion_wait(WRITE, HZ/50); } while (1); } diff --git a/fs/xfs/linux-2.6/xfs_buf.c b/fs/xfs/linux-2.6/xfs_buf.c index 9bbadafdcb0..db5f5a3608c 100644 --- a/fs/xfs/linux-2.6/xfs_buf.c +++ b/fs/xfs/linux-2.6/xfs_buf.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "xfs_linux.h" STATIC kmem_zone_t *xfs_buf_zone; @@ -395,7 +396,7 @@ _xfs_buf_lookup_pages( XFS_STATS_INC(xb_page_retries); xfsbufd_wakeup(0, gfp_mask); - blk_congestion_wait(WRITE, HZ/50); + congestion_wait(WRITE, HZ/50); goto retry; } diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index f7a1390d67f..7011d625559 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -10,6 +10,8 @@ #include +struct page; + /* * Bits in backing_dev_info.state */ @@ -88,6 +90,11 @@ static inline int bdi_rw_congested(struct backing_dev_info *bdi) (1 << BDI_write_congested)); } +void clear_bdi_congested(struct backing_dev_info *bdi, int rw); +void set_bdi_congested(struct backing_dev_info *bdi, int rw); +long congestion_wait(int rw, long timeout); +void congestion_end(int rw); + #define bdi_cap_writeback_dirty(bdi) \ (!((bdi)->capabilities & BDI_CAP_NO_WRITEBACK)) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 9575e3a5ff2..7bfcde2d557 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -651,8 +651,26 @@ extern void blk_recount_segments(request_queue_t *, struct bio *); extern int scsi_cmd_ioctl(struct file *, struct gendisk *, unsigned int, void __user *); extern int sg_scsi_ioctl(struct file *, struct request_queue *, struct gendisk *, struct scsi_ioctl_command __user *); -extern void blk_clear_queue_congested(request_queue_t *q, int rw); -extern void blk_set_queue_congested(request_queue_t *q, int rw); + +/* + * A queue has just exitted congestion. Note this in the global counter of + * congested queues, and wake up anyone who was waiting for requests to be + * put back. + */ +static inline void blk_clear_queue_congested(request_queue_t *q, int rw) +{ + clear_bdi_congested(&q->backing_dev_info, rw); +} + +/* + * A queue has just entered congestion. Flag that in the queue's VM-visible + * state flags and increment the global gounter of congested queues. + */ +static inline void blk_set_queue_congested(request_queue_t *q, int rw) +{ + set_bdi_congested(&q->backing_dev_info, rw); +} + extern void blk_start_queue(request_queue_t *q); extern void blk_stop_queue(request_queue_t *q); extern void blk_sync_queue(struct request_queue *q); @@ -767,10 +785,8 @@ extern int blk_queue_init_tags(request_queue_t *, int, struct blk_queue_tag *); extern void blk_queue_free_tags(request_queue_t *); extern int blk_queue_resize_tags(request_queue_t *, int); extern void blk_queue_invalidate_tags(request_queue_t *); -extern long blk_congestion_wait(int rw, long timeout); extern struct blk_queue_tag *blk_init_tags(int); extern void blk_free_tags(struct blk_queue_tag *); -extern void blk_congestion_end(int rw); static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, int tag) diff --git a/include/linux/writeback.h b/include/linux/writeback.h index a341c803286..fc35e6bdfb9 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -85,7 +85,6 @@ int wakeup_pdflush(long nr_pages); void laptop_io_completion(void); void laptop_sync_completion(void); void throttle_vm_writeout(void); -void writeback_congestion_end(void); /* These are exported to sysctl. */ extern int dirty_background_ratio; diff --git a/mm/Makefile b/mm/Makefile index 12b3a4eee88..f3c077eb0b8 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -10,7 +10,8 @@ mmu-$(CONFIG_MMU) := fremap.o highmem.o madvise.o memory.o mincore.o \ obj-y := bootmem.o filemap.o mempool.o oom_kill.o fadvise.o \ page_alloc.o page-writeback.o pdflush.o \ readahead.o swap.o truncate.o vmscan.o \ - prio_tree.o util.o mmzone.o vmstat.o $(mmu-y) + prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \ + $(mmu-y) ifeq ($(CONFIG_MMU)$(CONFIG_BLOCK),yy) obj-y += bounce.o diff --git a/mm/backing-dev.c b/mm/backing-dev.c new file mode 100644 index 00000000000..f50a2811f9d --- /dev/null +++ b/mm/backing-dev.c @@ -0,0 +1,69 @@ + +#include +#include +#include +#include +#include + +static wait_queue_head_t congestion_wqh[2] = { + __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]), + __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1]) + }; + + +void clear_bdi_congested(struct backing_dev_info *bdi, int rw) +{ + enum bdi_state bit; + wait_queue_head_t *wqh = &congestion_wqh[rw]; + + bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested; + clear_bit(bit, &bdi->state); + smp_mb__after_clear_bit(); + if (waitqueue_active(wqh)) + wake_up(wqh); +} +EXPORT_SYMBOL(clear_bdi_congested); + +void set_bdi_congested(struct backing_dev_info *bdi, int rw) +{ + enum bdi_state bit; + + bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested; + set_bit(bit, &bdi->state); +} +EXPORT_SYMBOL(set_bdi_congested); + +/** + * congestion_wait - wait for a backing_dev to become uncongested + * @rw: READ or WRITE + * @timeout: timeout in jiffies + * + * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit + * write congestion. If no backing_devs are congested then just wait for the + * next write to be completed. + */ +long congestion_wait(int rw, long timeout) +{ + long ret; + DEFINE_WAIT(wait); + wait_queue_head_t *wqh = &congestion_wqh[rw]; + + prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); + ret = io_schedule_timeout(timeout); + finish_wait(wqh, &wait); + return ret; +} +EXPORT_SYMBOL(congestion_wait); + +/** + * congestion_end - wake up sleepers on a congested backing_dev_info + * @rw: READ or WRITE + */ +void congestion_end(int rw) +{ + wait_queue_head_t *wqh = &congestion_wqh[rw]; + + if (waitqueue_active(wqh)) + wake_up(wqh); +} +EXPORT_SYMBOL(congestion_end); diff --git a/mm/page-writeback.c b/mm/page-writeback.c index a0f33905744..8d9b19f239c 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -222,7 +222,7 @@ static void balance_dirty_pages(struct address_space *mapping) if (pages_written >= write_chunk) break; /* We've done our duty */ } - blk_congestion_wait(WRITE, HZ/10); + congestion_wait(WRITE, HZ/10); } if (nr_reclaimable + global_page_state(NR_WRITEBACK) @@ -314,7 +314,7 @@ void throttle_vm_writeout(void) if (global_page_state(NR_UNSTABLE_NFS) + global_page_state(NR_WRITEBACK) <= dirty_thresh) break; - blk_congestion_wait(WRITE, HZ/10); + congestion_wait(WRITE, HZ/10); } } @@ -351,7 +351,7 @@ static void background_writeout(unsigned long _min_pages) min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write; if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) { /* Wrote less than expected */ - blk_congestion_wait(WRITE, HZ/10); + congestion_wait(WRITE, HZ/10); if (!wbc.encountered_congestion) break; } @@ -422,7 +422,7 @@ static void wb_kupdate(unsigned long arg) writeback_inodes(&wbc); if (wbc.nr_to_write > 0) { if (wbc.encountered_congestion) - blk_congestion_wait(WRITE, HZ/10); + congestion_wait(WRITE, HZ/10); else break; /* All the old data is written */ } @@ -955,15 +955,6 @@ int test_set_page_writeback(struct page *page) } EXPORT_SYMBOL(test_set_page_writeback); -/* - * Wakes up tasks that are being throttled due to writeback congestion - */ -void writeback_congestion_end(void) -{ - blk_congestion_end(WRITE); -} -EXPORT_SYMBOL(writeback_congestion_end); - /* * Return true if any of the pages in the mapping are marged with the * passed tag. diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 40db96a655d..afee38f04d8 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -1050,7 +1051,7 @@ nofail_alloc: if (page) goto got_pg; if (gfp_mask & __GFP_NOFAIL) { - blk_congestion_wait(WRITE, HZ/50); + congestion_wait(WRITE, HZ/50); goto nofail_alloc; } } @@ -1113,7 +1114,7 @@ rebalance: do_retry = 1; } if (do_retry) { - blk_congestion_wait(WRITE, HZ/50); + congestion_wait(WRITE, HZ/50); goto rebalance; } diff --git a/mm/shmem.c b/mm/shmem.c index b378f66cf2f..4959535fc14 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -1131,7 +1132,7 @@ repeat: page_cache_release(swappage); if (error == -ENOMEM) { /* let kswapd refresh zone for GFP_ATOMICs */ - blk_congestion_wait(WRITE, HZ/50); + congestion_wait(WRITE, HZ/50); } goto repeat; } diff --git a/mm/vmscan.c b/mm/vmscan.c index af73c14f9d8..f05527bf792 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1059,7 +1059,7 @@ unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask) /* Take a nap, wait for some writeback to complete */ if (sc.nr_scanned && priority < DEF_PRIORITY - 2) - blk_congestion_wait(WRITE, HZ/10); + congestion_wait(WRITE, HZ/10); } /* top priority shrink_caches still had more to do? don't OOM, then */ if (!sc.all_unreclaimable) @@ -1214,7 +1214,7 @@ scan: * another pass across the zones. */ if (total_scanned && priority < DEF_PRIORITY - 2) - blk_congestion_wait(WRITE, HZ/10); + congestion_wait(WRITE, HZ/10); /* * We do this so kswapd doesn't build up large priorities for @@ -1458,7 +1458,7 @@ unsigned long shrink_all_memory(unsigned long nr_pages) goto out; if (sc.nr_scanned && prio < DEF_PRIORITY - 2) - blk_congestion_wait(WRITE, HZ / 10); + congestion_wait(WRITE, HZ / 10); } lru_pages = 0; -- cgit v1.2.3-70-g09d2 From 6220ec7844fda2686496013a66b5b9169976b991 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Thu, 19 Oct 2006 23:29:05 -0700 Subject: [PATCH] highest_possible_node_id() linkage fix Qooting Adrian: - net/sunrpc/svc.c uses highest_possible_node_id() - include/linux/nodemask.h says highest_possible_node_id() is out-of-line #if MAX_NUMNODES > 1 - the out-of-line highest_possible_node_id() is in lib/cpumask.c - lib/Makefile: lib-$(CONFIG_SMP) += cpumask.o CONFIG_ARCH_DISCONTIGMEM_ENABLE=y, CONFIG_SMP=n, CONFIG_SUNRPC=y -> highest_possible_node_id() is used in net/sunrpc/svc.c CONFIG_NODES_SHIFT defined and > 0 -> include/linux/numa.h: MAX_NUMNODES > 1 -> compile error The bug is not present on architectures where ARCH_DISCONTIGMEM_ENABLE depends on NUMA (but m32r isn't the only affected architecture). So move the function into page_alloc.c Cc: Adrian Bunk Cc: Paul Jackson Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/cpumask.c | 16 ---------------- mm/page_alloc.c | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) (limited to 'mm/page_alloc.c') diff --git a/lib/cpumask.c b/lib/cpumask.c index 7a2a73f88d5..3a67dc5ada7 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -43,19 +43,3 @@ int __any_online_cpu(const cpumask_t *mask) return cpu; } EXPORT_SYMBOL(__any_online_cpu); - -#if MAX_NUMNODES > 1 -/* - * Find the highest possible node id. - */ -int highest_possible_node_id(void) -{ - unsigned int node; - unsigned int highest = 0; - - for_each_node_mask(node, node_possible_map) - highest = node; - return highest; -} -EXPORT_SYMBOL(highest_possible_node_id); -#endif diff --git a/mm/page_alloc.c b/mm/page_alloc.c index afee38f04d8..ebd425c2e2a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3120,3 +3120,19 @@ unsigned long page_to_pfn(struct page *page) EXPORT_SYMBOL(pfn_to_page); EXPORT_SYMBOL(page_to_pfn); #endif /* CONFIG_OUT_OF_LINE_PFN_TO_PAGE */ + +#if MAX_NUMNODES > 1 +/* + * Find the highest possible node id. + */ +int highest_possible_node_id(void) +{ + unsigned int node; + unsigned int highest = 0; + + for_each_node_mask(node, node_possible_map) + highest = node; + return highest; +} +EXPORT_SYMBOL(highest_possible_node_id); +#endif -- cgit v1.2.3-70-g09d2 From 7516795739bd53175629b90fab0ad488d7a6a9f7 Mon Sep 17 00:00:00 2001 From: Andy Whitcroft Date: Sat, 21 Oct 2006 10:24:14 -0700 Subject: [PATCH] Reintroduce NODES_SPAN_OTHER_NODES for powerpc Reintroduce NODES_SPAN_OTHER_NODES for powerpc Revert "[PATCH] Remove SPAN_OTHER_NODES config definition" This reverts commit f62859bb6871c5e4a8e591c60befc8caaf54db8c. Revert "[PATCH] mm: remove arch independent NODES_SPAN_OTHER_NODES" This reverts commit a94b3ab7eab4edcc9b2cb474b188f774c331adf7. Also update the comments to indicate that this is still required and where its used. Signed-off-by: Andy Whitcroft Cc: Paul Mackerras Cc: Mike Kravetz Cc: Benjamin Herrenschmidt Acked-by: Mel Gorman Acked-by: Will Schmidt Cc: Christoph Lameter Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/powerpc/Kconfig | 9 +++++++++ arch/powerpc/configs/pseries_defconfig | 1 + include/linux/mmzone.h | 6 ++++++ mm/page_alloc.c | 2 ++ 4 files changed, 18 insertions(+) (limited to 'mm/page_alloc.c') diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 8b691046557..2bd9b7fb0f6 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -751,6 +751,15 @@ config ARCH_MEMORY_PROBE def_bool y depends on MEMORY_HOTPLUG +# Some NUMA nodes have memory ranges that span +# other nodes. Even though a pfn is valid and +# between a node's start and end pfns, it may not +# reside on that node. See memmap_init_zone() +# for details. +config NODES_SPAN_OTHER_NODES + def_bool y + depends on NEED_MULTIPLE_NODES + config PPC_64K_PAGES bool "64k page size" depends on PPC64 diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig index 9828663652e..d2833c1a1f3 100644 --- a/arch/powerpc/configs/pseries_defconfig +++ b/arch/powerpc/configs/pseries_defconfig @@ -184,6 +184,7 @@ CONFIG_SPLIT_PTLOCK_CPUS=4 CONFIG_MIGRATION=y CONFIG_RESOURCES_64BIT=y CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID=y +CONFIG_NODES_SPAN_OTHER_NODES=y # CONFIG_PPC_64K_PAGES is not set CONFIG_SCHED_SMT=y CONFIG_PROC_DEVICETREE=y diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 59855b8718a..ed0762b283a 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -674,6 +674,12 @@ void sparse_init(void); #define sparse_index_init(_sec, _nid) do {} while (0) #endif /* CONFIG_SPARSEMEM */ +#ifdef CONFIG_NODES_SPAN_OTHER_NODES +#define early_pfn_in_nid(pfn, nid) (early_pfn_to_nid(pfn) == (nid)) +#else +#define early_pfn_in_nid(pfn, nid) (1) +#endif + #ifndef early_pfn_valid #define early_pfn_valid(pfn) (1) #endif diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ebd425c2e2a..f5fc45472d5 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1689,6 +1689,8 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, for (pfn = start_pfn; pfn < end_pfn; pfn++) { if (!early_pfn_valid(pfn)) continue; + if (!early_pfn_in_nid(pfn, nid)) + continue; page = pfn_to_page(pfn); set_page_links(page, zone, nid, pfn); init_page_count(page); -- cgit v1.2.3-70-g09d2 From 3bb1a852ab6c9cdf211a2f4a2f502340c8c38eca Mon Sep 17 00:00:00 2001 From: Martin Bligh Date: Sat, 28 Oct 2006 10:38:24 -0700 Subject: [PATCH] vmscan: Fix temp_priority race The temp_priority field in zone is racy, as we can walk through a reclaim path, and just before we copy it into prev_priority, it can be overwritten (say with DEF_PRIORITY) by another reclaimer. The same bug is contained in both try_to_free_pages and balance_pgdat, but it is fixed slightly differently. In balance_pgdat, we keep a separate priority record per zone in a local array. In try_to_free_pages there is no need to do this, as the priority level is the same for all zones that we reclaim from. Impact of this bug is that temp_priority is copied into prev_priority, and setting this artificially high causes reclaimers to set distress artificially low. They then fail to reclaim mapped pages, when they are, in fact, under severe memory pressure (their priority may be as low as 0). This causes the OOM killer to fire incorrectly. From: Andrew Morton __zone_reclaim() isn't modifying zone->prev_priority. But zone->prev_priority is used in the decision whether or not to bring mapped pages onto the inactive list. Hence there's a risk here that __zone_reclaim() will fail because zone->prev_priority ir large (ie: low urgency) and lots of mapped pages end up stuck on the active list. Fix that up by decreasing (ie making more urgent) zone->prev_priority as __zone_reclaim() scans the zone's pages. This bug perhaps explains why ZONE_RECLAIM_PRIORITY was created. It should be possible to remove that now, and to just start out at DEF_PRIORITY? Cc: Nick Piggin Cc: Christoph Lameter Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mmzone.h | 6 +----- mm/page_alloc.c | 2 +- mm/vmscan.c | 55 +++++++++++++++++++++++++++++++++++++------------- mm/vmstat.c | 2 -- 4 files changed, 43 insertions(+), 22 deletions(-) (limited to 'mm/page_alloc.c') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ed0762b283a..e06683e2bea 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -218,13 +218,9 @@ struct zone { * under - it drives the swappiness decision: whether to unmap mapped * pages. * - * temp_priority is used to remember the scanning priority at which - * this zone was successfully refilled to free_pages == pages_high. - * - * Access to both these fields is quite racy even on uniprocessor. But + * Access to both this field is quite racy even on uniprocessor. But * it is expected to average out OK. */ - int temp_priority; int prev_priority; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f5fc45472d5..ecf853b5e30 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2407,7 +2407,7 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat, zone->zone_pgdat = pgdat; zone->free_pages = 0; - zone->temp_priority = zone->prev_priority = DEF_PRIORITY; + zone->prev_priority = DEF_PRIORITY; zone_pcp_init(zone); INIT_LIST_HEAD(&zone->active_list); diff --git a/mm/vmscan.c b/mm/vmscan.c index f05527bf792..b32560ead5c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -723,6 +723,20 @@ done: return nr_reclaimed; } +/* + * We are about to scan this zone at a certain priority level. If that priority + * level is smaller (ie: more urgent) than the previous priority, then note + * that priority level within the zone. This is done so that when the next + * process comes in to scan this zone, it will immediately start out at this + * priority level rather than having to build up its own scanning priority. + * Here, this priority affects only the reclaim-mapped threshold. + */ +static inline void note_zone_scanning_priority(struct zone *zone, int priority) +{ + if (priority < zone->prev_priority) + zone->prev_priority = priority; +} + static inline int zone_is_near_oom(struct zone *zone) { return zone->pages_scanned >= (zone->nr_active + zone->nr_inactive)*3; @@ -972,9 +986,7 @@ static unsigned long shrink_zones(int priority, struct zone **zones, if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) continue; - zone->temp_priority = priority; - if (zone->prev_priority > priority) - zone->prev_priority = priority; + note_zone_scanning_priority(zone, priority); if (zone->all_unreclaimable && priority != DEF_PRIORITY) continue; /* Let kswapd poll it */ @@ -1024,7 +1036,6 @@ unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask) if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) continue; - zone->temp_priority = DEF_PRIORITY; lru_pages += zone->nr_active + zone->nr_inactive; } @@ -1065,13 +1076,22 @@ unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask) if (!sc.all_unreclaimable) ret = 1; out: + /* + * Now that we've scanned all the zones at this priority level, note + * that level within the zone so that the next thread which performs + * scanning of this zone will immediately start out at this priority + * level. This affects only the decision whether or not to bring + * mapped pages onto the inactive list. + */ + if (priority < 0) + priority = 0; for (i = 0; zones[i] != 0; i++) { struct zone *zone = zones[i]; if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) continue; - zone->prev_priority = zone->temp_priority; + zone->prev_priority = priority; } return ret; } @@ -1111,6 +1131,11 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order) .swap_cluster_max = SWAP_CLUSTER_MAX, .swappiness = vm_swappiness, }; + /* + * temp_priority is used to remember the scanning priority at which + * this zone was successfully refilled to free_pages == pages_high. + */ + int temp_priority[MAX_NR_ZONES]; loop_again: total_scanned = 0; @@ -1118,11 +1143,8 @@ loop_again: sc.may_writepage = !laptop_mode; count_vm_event(PAGEOUTRUN); - for (i = 0; i < pgdat->nr_zones; i++) { - struct zone *zone = pgdat->node_zones + i; - - zone->temp_priority = DEF_PRIORITY; - } + for (i = 0; i < pgdat->nr_zones; i++) + temp_priority[i] = DEF_PRIORITY; for (priority = DEF_PRIORITY; priority >= 0; priority--) { int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */ @@ -1183,10 +1205,9 @@ scan: if (!zone_watermark_ok(zone, order, zone->pages_high, end_zone, 0)) all_zones_ok = 0; - zone->temp_priority = priority; - if (zone->prev_priority > priority) - zone->prev_priority = priority; + temp_priority[i] = priority; sc.nr_scanned = 0; + note_zone_scanning_priority(zone, priority); nr_reclaimed += shrink_zone(priority, zone, &sc); reclaim_state->reclaimed_slab = 0; nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL, @@ -1226,10 +1247,15 @@ scan: break; } out: + /* + * Note within each zone the priority level at which this zone was + * brought into a happy state. So that the next thread which scans this + * zone will start out at that priority level. + */ for (i = 0; i < pgdat->nr_zones; i++) { struct zone *zone = pgdat->node_zones + i; - zone->prev_priority = zone->temp_priority; + zone->prev_priority = temp_priority[i]; } if (!all_zones_ok) { cond_resched(); @@ -1614,6 +1640,7 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) */ priority = ZONE_RECLAIM_PRIORITY; do { + note_zone_scanning_priority(zone, priority); nr_reclaimed += shrink_zone(priority, zone, &sc); priority--; } while (priority >= 0 && nr_reclaimed < nr_pages); diff --git a/mm/vmstat.c b/mm/vmstat.c index 45b124e012f..8614e8f6743 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -587,11 +587,9 @@ static int zoneinfo_show(struct seq_file *m, void *arg) seq_printf(m, "\n all_unreclaimable: %u" "\n prev_priority: %i" - "\n temp_priority: %i" "\n start_pfn: %lu", zone->all_unreclaimable, zone->prev_priority, - zone->temp_priority, zone->zone_start_pfn); spin_unlock_irqrestore(&zone->lock, flags); seq_putc(m, '\n'); -- cgit v1.2.3-70-g09d2 From 0c6cb974636dd29681b03f8eb0ae227decab01fb Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Sat, 28 Oct 2006 10:38:59 -0700 Subject: [PATCH] Calculation fix for memory holes beyong the end of physical memory absent_pages_in_range() made the assumption that users of the arch-independent zone-sizing API would not care about holes beyound the end of physical memory. This was not the case and was "fixed" in a patch called "Account for holes that are outside the range of physical memory". However, when given a range that started before a hole in "real" memory and ended beyond the end of memory, it would get the result wrong. The bug is in mainline but a patch is below. It has been tested successfully on a number of machines and architectures. Additional credit to Keith Mannthey for discovering the problem, helping identify the correct fix and confirming it Worked For Him. Signed-off-by: Mel Gorman Cc: keith mannthey Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mm/page_alloc.c') diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ecf853b5e30..b55bb358b83 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2261,7 +2261,7 @@ unsigned long __init __absent_pages_in_range(int nid, /* Account for ranges past physical memory on this node */ if (range_end_pfn > prev_end_pfn) - hole_pages = range_end_pfn - + hole_pages += range_end_pfn - max(range_start_pfn, prev_end_pfn); return hole_pages; -- cgit v1.2.3-70-g09d2 From 941c7105dc4f4961727acc518e18e00b9a03cbf3 Mon Sep 17 00:00:00 2001 From: nkalmala Date: Thu, 2 Nov 2006 22:07:04 -0800 Subject: [PATCH] mm: un-needed add-store operation wastes a few bytes Un-needed add-store operation wastes a few bytes. 8 bytes wasted with -O2, on a ppc. Signed-off-by: nkalmala Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mm/page_alloc.c') diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b55bb358b83..bf2f6cff1d6 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -853,7 +853,7 @@ again: pcp = &zone_pcp(zone, cpu)->pcp[cold]; local_irq_save(flags); if (!pcp->count) { - pcp->count += rmqueue_bulk(zone, 0, + pcp->count = rmqueue_bulk(zone, 0, pcp->batch, &pcp->list); if (unlikely(!pcp->count)) goto failed; -- cgit v1.2.3-70-g09d2