From 46453a6e194a8c55fe6cf3dc8e1c4f24e2abc013 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 22 Mar 2006 00:07:58 -0800 Subject: [PATCH] mm: never ClearPageLRU released pages If vmscan finds a zero refcount page on the lru list, never ClearPageLRU it. This means the release code need not hold ->lru_lock to stabilise PageLRU, so that lock may be skipped entirely when releasing !PageLRU pages (because we know PageLRU won't have been temporarily cleared by vmscan, which was previously guaranteed by holding the lock to synchronise against vmscan). Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/swap.c | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) (limited to 'mm/swap.c') diff --git a/mm/swap.c b/mm/swap.c index b524ea90bdd..3045a0f4c45 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -209,19 +209,18 @@ int lru_add_drain_all(void) */ void fastcall __page_cache_release(struct page *page) { - unsigned long flags; - struct zone *zone = page_zone(page); + if (PageLRU(page)) { + unsigned long flags; + struct zone *zone = page_zone(page); - spin_lock_irqsave(&zone->lru_lock, flags); - if (TestClearPageLRU(page)) + spin_lock_irqsave(&zone->lru_lock, flags); + if (!TestClearPageLRU(page)) + BUG(); del_page_from_lru(zone, page); - if (page_count(page) != 0) - page = NULL; - spin_unlock_irqrestore(&zone->lru_lock, flags); - if (page) - free_hot_page(page); + spin_unlock_irqrestore(&zone->lru_lock, flags); + } + free_hot_page(page); } - EXPORT_SYMBOL(__page_cache_release); /* @@ -245,7 +244,6 @@ void release_pages(struct page **pages, int nr, int cold) pagevec_init(&pages_to_free, cold); for (i = 0; i < nr; i++) { struct page *page = pages[i]; - struct zone *pagezone; if (unlikely(PageCompound(page))) { if (zone) { @@ -259,23 +257,27 @@ void release_pages(struct page **pages, int nr, int cold) if (!put_page_testzero(page)) continue; - pagezone = page_zone(page); - if (pagezone != zone) { - if (zone) - spin_unlock_irq(&zone->lru_lock); - zone = pagezone; - spin_lock_irq(&zone->lru_lock); - } - if (TestClearPageLRU(page)) + if (PageLRU(page)) { + struct zone *pagezone = page_zone(page); + if (pagezone != zone) { + if (zone) + spin_unlock_irq(&zone->lru_lock); + zone = pagezone; + spin_lock_irq(&zone->lru_lock); + } + if (!TestClearPageLRU(page)) + BUG(); del_page_from_lru(zone, page); - if (page_count(page) == 0) { - if (!pagevec_add(&pages_to_free, page)) { + } + + if (!pagevec_add(&pages_to_free, page)) { + if (zone) { spin_unlock_irq(&zone->lru_lock); - __pagevec_free(&pages_to_free); - pagevec_reinit(&pages_to_free); - zone = NULL; /* No lock is held */ + zone = NULL; } - } + __pagevec_free(&pages_to_free); + pagevec_reinit(&pages_to_free); + } } if (zone) spin_unlock_irq(&zone->lru_lock); -- cgit v1.2.3-70-g09d2 From 8d438f96d2b8eade6cbcd8adfc22dae6f5cbd6c0 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 22 Mar 2006 00:07:59 -0800 Subject: [PATCH] mm: PageLRU no testset PG_lru is protected by zone->lru_lock. It does not need TestSet/TestClear operations. Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/page-flags.h | 5 ++--- mm/swap.c | 16 ++++++++-------- mm/vmscan.c | 20 +++++++++++--------- 3 files changed, 21 insertions(+), 20 deletions(-) (limited to 'mm/swap.c') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index d52999c4333..58856c823f8 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -239,10 +239,9 @@ extern void __mod_page_state_offset(unsigned long offset, unsigned long delta); #define __ClearPageDirty(page) __clear_bit(PG_dirty, &(page)->flags) #define TestClearPageDirty(page) test_and_clear_bit(PG_dirty, &(page)->flags) -#define SetPageLRU(page) set_bit(PG_lru, &(page)->flags) #define PageLRU(page) test_bit(PG_lru, &(page)->flags) -#define TestSetPageLRU(page) test_and_set_bit(PG_lru, &(page)->flags) -#define TestClearPageLRU(page) test_and_clear_bit(PG_lru, &(page)->flags) +#define SetPageLRU(page) set_bit(PG_lru, &(page)->flags) +#define ClearPageLRU(page) clear_bit(PG_lru, &(page)->flags) #define PageActive(page) test_bit(PG_active, &(page)->flags) #define SetPageActive(page) set_bit(PG_active, &(page)->flags) diff --git a/mm/swap.c b/mm/swap.c index 3045a0f4c45..985324ee936 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -214,8 +214,8 @@ void fastcall __page_cache_release(struct page *page) struct zone *zone = page_zone(page); spin_lock_irqsave(&zone->lru_lock, flags); - if (!TestClearPageLRU(page)) - BUG(); + BUG_ON(!PageLRU(page)); + ClearPageLRU(page); del_page_from_lru(zone, page); spin_unlock_irqrestore(&zone->lru_lock, flags); } @@ -265,8 +265,8 @@ void release_pages(struct page **pages, int nr, int cold) zone = pagezone; spin_lock_irq(&zone->lru_lock); } - if (!TestClearPageLRU(page)) - BUG(); + BUG_ON(!PageLRU(page)); + ClearPageLRU(page); del_page_from_lru(zone, page); } @@ -345,8 +345,8 @@ void __pagevec_lru_add(struct pagevec *pvec) zone = pagezone; spin_lock_irq(&zone->lru_lock); } - if (TestSetPageLRU(page)) - BUG(); + BUG_ON(PageLRU(page)); + SetPageLRU(page); add_page_to_inactive_list(zone, page); } if (zone) @@ -372,8 +372,8 @@ void __pagevec_lru_add_active(struct pagevec *pvec) zone = pagezone; spin_lock_irq(&zone->lru_lock); } - if (TestSetPageLRU(page)) - BUG(); + BUG_ON(PageLRU(page)); + SetPageLRU(page); if (TestSetPageActive(page)) BUG(); add_page_to_active_list(zone, page); diff --git a/mm/vmscan.c b/mm/vmscan.c index acb7611cd52..40fb37828e8 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1042,9 +1042,10 @@ int isolate_lru_page(struct page *page) if (PageLRU(page)) { struct zone *zone = page_zone(page); spin_lock_irq(&zone->lru_lock); - if (TestClearPageLRU(page)) { + if (PageLRU(page)) { ret = 1; get_page(page); + ClearPageLRU(page); if (PageActive(page)) del_page_from_active_list(zone, page); else @@ -1085,6 +1086,8 @@ static int isolate_lru_pages(int nr_to_scan, struct list_head *src, page = lru_to_page(src); prefetchw_prev_lru_page(page, src, flags); + BUG_ON(!PageLRU(page)); + list_del(&page->lru); if (unlikely(get_page_testone(page))) { /* @@ -1100,8 +1103,7 @@ static int isolate_lru_pages(int nr_to_scan, struct list_head *src, * the page is not being freed elsewhere -- the page release * code relies on it. */ - if (!TestClearPageLRU(page)) - BUG(); + ClearPageLRU(page); list_add(&page->lru, dst); nr_taken++; } @@ -1156,8 +1158,8 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc) */ while (!list_empty(&page_list)) { page = lru_to_page(&page_list); - if (TestSetPageLRU(page)) - BUG(); + BUG_ON(PageLRU(page)); + SetPageLRU(page); list_del(&page->lru); if (PageActive(page)) add_page_to_active_list(zone, page); @@ -1276,8 +1278,8 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc) while (!list_empty(&l_inactive)) { page = lru_to_page(&l_inactive); prefetchw_prev_lru_page(page, &l_inactive, flags); - if (TestSetPageLRU(page)) - BUG(); + BUG_ON(PageLRU(page)); + SetPageLRU(page); if (!TestClearPageActive(page)) BUG(); list_move(&page->lru, &zone->inactive_list); @@ -1305,8 +1307,8 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc) while (!list_empty(&l_active)) { page = lru_to_page(&l_active); prefetchw_prev_lru_page(page, &l_active, flags); - if (TestSetPageLRU(page)) - BUG(); + BUG_ON(PageLRU(page)); + SetPageLRU(page); BUG_ON(!PageActive(page)); list_move(&page->lru, &zone->active_list); pgmoved++; -- cgit v1.2.3-70-g09d2 From 4c84cacfa424264f7ad5287298d3ea4a3e935278 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 22 Mar 2006 00:08:00 -0800 Subject: [PATCH] mm: PageActive no testset PG_active is protected by zone->lru_lock, it does not need TestSet/TestClear operations. Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/page-flags.h | 2 -- mm/swap.c | 4 ++-- mm/vmscan.c | 5 +++-- 3 files changed, 5 insertions(+), 6 deletions(-) (limited to 'mm/swap.c') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 58856c823f8..5d1e7bd8510 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -246,8 +246,6 @@ extern void __mod_page_state_offset(unsigned long offset, unsigned long delta); #define PageActive(page) test_bit(PG_active, &(page)->flags) #define SetPageActive(page) set_bit(PG_active, &(page)->flags) #define ClearPageActive(page) clear_bit(PG_active, &(page)->flags) -#define TestClearPageActive(page) test_and_clear_bit(PG_active, &(page)->flags) -#define TestSetPageActive(page) test_and_set_bit(PG_active, &(page)->flags) #define PageSlab(page) test_bit(PG_slab, &(page)->flags) #define SetPageSlab(page) set_bit(PG_slab, &(page)->flags) diff --git a/mm/swap.c b/mm/swap.c index 985324ee936..cf88226cf96 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -374,8 +374,8 @@ void __pagevec_lru_add_active(struct pagevec *pvec) } BUG_ON(PageLRU(page)); SetPageLRU(page); - if (TestSetPageActive(page)) - BUG(); + BUG_ON(PageActive(page)); + SetPageActive(page); add_page_to_active_list(zone, page); } if (zone) diff --git a/mm/vmscan.c b/mm/vmscan.c index 40fb37828e8..8e477b1a483 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1280,8 +1280,9 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc) prefetchw_prev_lru_page(page, &l_inactive, flags); BUG_ON(PageLRU(page)); SetPageLRU(page); - if (!TestClearPageActive(page)) - BUG(); + BUG_ON(!PageActive(page)); + ClearPageActive(page); + list_move(&page->lru, &zone->inactive_list); pgmoved++; if (!pagevec_add(&pvec, page)) { -- cgit v1.2.3-70-g09d2 From 674539115cc88473f623581e1d53c0e2ecef2179 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 22 Mar 2006 00:08:00 -0800 Subject: [PATCH] mm: less atomic ops In the page release paths, we can be sure that nobody will mess with our page->flags because the refcount has dropped to 0. So no need for atomic operations here. Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 2 +- include/linux/page-flags.h | 2 ++ mm/swap.c | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'mm/swap.c') diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 8ac854f7f19..3b6723dfaff 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -32,7 +32,7 @@ del_page_from_lru(struct zone *zone, struct page *page) { list_del(&page->lru); if (PageActive(page)) { - ClearPageActive(page); + __ClearPageActive(page); zone->nr_active--; } else { zone->nr_inactive--; diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 5d1e7bd8510..da71d63df46 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -242,10 +242,12 @@ extern void __mod_page_state_offset(unsigned long offset, unsigned long delta); #define PageLRU(page) test_bit(PG_lru, &(page)->flags) #define SetPageLRU(page) set_bit(PG_lru, &(page)->flags) #define ClearPageLRU(page) clear_bit(PG_lru, &(page)->flags) +#define __ClearPageLRU(page) __clear_bit(PG_lru, &(page)->flags) #define PageActive(page) test_bit(PG_active, &(page)->flags) #define SetPageActive(page) set_bit(PG_active, &(page)->flags) #define ClearPageActive(page) clear_bit(PG_active, &(page)->flags) +#define __ClearPageActive(page) __clear_bit(PG_active, &(page)->flags) #define PageSlab(page) test_bit(PG_slab, &(page)->flags) #define SetPageSlab(page) set_bit(PG_slab, &(page)->flags) diff --git a/mm/swap.c b/mm/swap.c index cf88226cf96..91b7e2026f6 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -215,7 +215,7 @@ void fastcall __page_cache_release(struct page *page) spin_lock_irqsave(&zone->lru_lock, flags); BUG_ON(!PageLRU(page)); - ClearPageLRU(page); + __ClearPageLRU(page); del_page_from_lru(zone, page); spin_unlock_irqrestore(&zone->lru_lock, flags); } @@ -266,7 +266,7 @@ void release_pages(struct page **pages, int nr, int cold) spin_lock_irq(&zone->lru_lock); } BUG_ON(!PageLRU(page)); - ClearPageLRU(page); + __ClearPageLRU(page); del_page_from_lru(zone, page); } -- cgit v1.2.3-70-g09d2