From 7eaceaccab5f40bbfda044629a6298616aeaed50 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 10 Mar 2011 08:52:07 +0100 Subject: block: remove per-queue plugging Code has been converted over to the new explicit on-stack plugging, and delay users have been converted to use the new API for that. So lets kill off the old plugging along with aops->sync_page(). Signed-off-by: Jens Axboe --- include/linux/pagemap.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include/linux/pagemap.h') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 9c66e994540..e112b8db2f3 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -298,7 +298,6 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma, extern void __lock_page(struct page *page); extern int __lock_page_killable(struct page *page); -extern void __lock_page_nosync(struct page *page); extern int __lock_page_or_retry(struct page *page, struct mm_struct *mm, unsigned int flags); extern void unlock_page(struct page *page); @@ -341,17 +340,6 @@ static inline int lock_page_killable(struct page *page) return 0; } -/* - * lock_page_nosync should only be used if we can't pin the page's inode. - * Doesn't play quite so well with block device plugging. - */ -static inline void lock_page_nosync(struct page *page) -{ - might_sleep(); - if (!trylock_page(page)) - __lock_page_nosync(page); -} - /* * lock_page_or_retry - Lock the page, unless this would block and the * caller indicated that it can handle a retry. -- cgit v1.2.3-70-g09d2 From ef6a3c63112e865d632ff7c478ba7c7160cad0d1 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 22 Mar 2011 16:30:52 -0700 Subject: mm: add replace_page_cache_page() function This function basically does: remove_from_page_cache(old); page_cache_release(old); add_to_page_cache_locked(new); Except it does this atomically, so there's no possibility for the "add" to fail because of a race. If memory cgroups are enabled, then the memory cgroup charge is also moved from the old page to the new. This function is currently used by fuse to move pages into the page cache on read, instead of copying the page contents. [minchan.kim@gmail.com: add freepage() hook to replace_page_cache_page()] Signed-off-by: Miklos Szeredi Acked-by: Rik van Riel Acked-by: KAMEZAWA Hiroyuki Cc: Mel Gorman Signed-off-by: Minchan Kim Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/fuse/dev.c | 10 +++---- include/linux/memcontrol.h | 4 +-- include/linux/pagemap.h | 1 + mm/filemap.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++ mm/memcontrol.c | 4 +-- mm/migrate.c | 2 +- 6 files changed, 80 insertions(+), 11 deletions(-) (limited to 'include/linux/pagemap.h') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 213d3cf4f5e..640fc229df1 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -737,14 +737,12 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) if (WARN_ON(PageMlocked(oldpage))) goto out_fallback_unlock; - remove_from_page_cache(oldpage); - page_cache_release(oldpage); - - err = add_to_page_cache_locked(newpage, mapping, index, GFP_KERNEL); + err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL); if (err) { - printk(KERN_WARNING "fuse_try_move_page: failed to add page"); - goto out_fallback_unlock; + unlock_page(newpage); + return err; } + page_cache_get(newpage); if (!(buf->flags & PIPE_BUF_FLAG_LRU)) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index f512e189be5..a1a1e5384f6 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -96,7 +96,7 @@ extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem); extern int mem_cgroup_prepare_migration(struct page *page, - struct page *newpage, struct mem_cgroup **ptr); + struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask); extern void mem_cgroup_end_migration(struct mem_cgroup *mem, struct page *oldpage, struct page *newpage, bool migration_ok); @@ -249,7 +249,7 @@ static inline struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem) static inline int mem_cgroup_prepare_migration(struct page *page, struct page *newpage, - struct mem_cgroup **ptr) + struct mem_cgroup **ptr, gfp_t gfp_mask) { return 0; } diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 9c66e994540..26946ad483b 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -457,6 +457,7 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); extern void remove_from_page_cache(struct page *page); extern void __remove_from_page_cache(struct page *page); +int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask); /* * Like add_to_page_cache_locked, but used to add newly allocated pages: diff --git a/mm/filemap.c b/mm/filemap.c index 312b6eb7843..c1459f2cdb5 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -386,6 +386,76 @@ int filemap_write_and_wait_range(struct address_space *mapping, } EXPORT_SYMBOL(filemap_write_and_wait_range); +/** + * replace_page_cache_page - replace a pagecache page with a new one + * @old: page to be replaced + * @new: page to replace with + * @gfp_mask: allocation mode + * + * This function replaces a page in the pagecache with a new one. On + * success it acquires the pagecache reference for the new page and + * drops it for the old page. Both the old and new pages must be + * locked. This function does not add the new page to the LRU, the + * caller must do that. + * + * The remove + add is atomic. The only way this function can fail is + * memory allocation failure. + */ +int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) +{ + int error; + struct mem_cgroup *memcg = NULL; + + VM_BUG_ON(!PageLocked(old)); + VM_BUG_ON(!PageLocked(new)); + VM_BUG_ON(new->mapping); + + /* + * This is not page migration, but prepare_migration and + * end_migration does enough work for charge replacement. + * + * In the longer term we probably want a specialized function + * for moving the charge from old to new in a more efficient + * manner. + */ + error = mem_cgroup_prepare_migration(old, new, &memcg, gfp_mask); + if (error) + return error; + + error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); + if (!error) { + struct address_space *mapping = old->mapping; + void (*freepage)(struct page *); + + pgoff_t offset = old->index; + freepage = mapping->a_ops->freepage; + + page_cache_get(new); + new->mapping = mapping; + new->index = offset; + + spin_lock_irq(&mapping->tree_lock); + __remove_from_page_cache(old); + error = radix_tree_insert(&mapping->page_tree, offset, new); + BUG_ON(error); + mapping->nrpages++; + __inc_zone_page_state(new, NR_FILE_PAGES); + if (PageSwapBacked(new)) + __inc_zone_page_state(new, NR_SHMEM); + spin_unlock_irq(&mapping->tree_lock); + radix_tree_preload_end(); + if (freepage) + freepage(old); + page_cache_release(old); + mem_cgroup_end_migration(memcg, old, new, true); + } else { + mem_cgroup_end_migration(memcg, old, new, false); + } + + return error; +} +EXPORT_SYMBOL_GPL(replace_page_cache_page); + /** * add_to_page_cache_locked - add a locked page to the pagecache * @page: page to add diff --git a/mm/memcontrol.c b/mm/memcontrol.c index da53a252b25..6ef5c53dffc 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2883,7 +2883,7 @@ static inline int mem_cgroup_move_swap_account(swp_entry_t entry, * page belongs to. */ int mem_cgroup_prepare_migration(struct page *page, - struct page *newpage, struct mem_cgroup **ptr) + struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask) { struct page_cgroup *pc; struct mem_cgroup *mem = NULL; @@ -2940,7 +2940,7 @@ int mem_cgroup_prepare_migration(struct page *page, return 0; *ptr = mem; - ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false, PAGE_SIZE); + ret = __mem_cgroup_try_charge(NULL, gfp_mask, ptr, false, PAGE_SIZE); css_put(&mem->css);/* drop extra refcnt */ if (ret || *ptr == NULL) { if (PageAnon(page)) { diff --git a/mm/migrate.c b/mm/migrate.c index 352de555626..8aacce3af8c 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -678,7 +678,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private, } /* charge against new page */ - charge = mem_cgroup_prepare_migration(page, newpage, &mem); + charge = mem_cgroup_prepare_migration(page, newpage, &mem, GFP_KERNEL); if (charge == -ENOMEM) { rc = -ENOMEM; goto unlock; -- cgit v1.2.3-70-g09d2 From 97cecb5a254fec22d28ef32235d888bfbfd7c783 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Tue, 22 Mar 2011 16:30:53 -0700 Subject: mm: introduce delete_from_page_cache() Presently we increase the page refcount in add_to_page_cache() but don't decrease it in remove_from_page_cache(). Such asymmetry adds confusion, requiring that callers notice it and a comment explaining why they release a page reference. It's not a good API. A long time ago, Hugh tried it (http://lkml.org/lkml/2004/10/24/140) but gave up because reiser4's drop_page() had to unlock the page between removing it from page cache and doing the page_cache_release(). But now the situation is changed. I think at least things in current mainline don't have any obstacles. The problem is for out-of-mainline filesystems - if they have done such things as reiser4, this patch could be a problem but they will discover this at compile time since we remove remove_from_page_cache(). This patch: This function works as just wrapper remove_from_page_cache(). The difference is that it decreases page references in itself. So caller have to make sure it has a page reference before calling. This patch is ready for removing remove_from_page_cache(). Signed-off-by: Minchan Kim Cc: Christoph Hellwig Acked-by: Hugh Dickins Acked-by: Mel Gorman Reviewed-by: KAMEZAWA Hiroyuki Reviewed-by: Johannes Weiner Reviewed-by: KOSAKI Motohiro Cc: Edward Shishkin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/pagemap.h | 1 + mm/filemap.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'include/linux/pagemap.h') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 26946ad483b..0aa258f4678 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -455,6 +455,7 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); +extern void delete_from_page_cache(struct page *page); extern void remove_from_page_cache(struct page *page); extern void __remove_from_page_cache(struct page *page); int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask); diff --git a/mm/filemap.c b/mm/filemap.c index c1459f2cdb5..e7b59785ceb 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -155,6 +155,22 @@ void remove_from_page_cache(struct page *page) } EXPORT_SYMBOL(remove_from_page_cache); +/** + * delete_from_page_cache - delete page from page cache + * @page: the page which the kernel is trying to remove from page cache + * + * This must be called only on pages that have + * been verified to be in the page cache and locked. + * It will never put the page into the free list, + * the caller has a reference on the page. + */ +void delete_from_page_cache(struct page *page) +{ + remove_from_page_cache(page); + page_cache_release(page); +} +EXPORT_SYMBOL(delete_from_page_cache); + static int sync_page(void *word) { struct address_space *mapping; -- cgit v1.2.3-70-g09d2 From 702cfbf93aaf3a091b0c64c8766c1ade0a820c38 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Tue, 22 Mar 2011 16:32:43 -0700 Subject: mm: goodbye remove_from_page_cache() Now delete_from_page_cache() replaces remove_from_page_cache(). So we remove remove_from_page_cache so fs or something out of mainline will notice it when compile time and can fix it. Signed-off-by: Minchan Kim Cc: Christoph Hellwig Acked-by: Hugh Dickins Acked-by: Mel Gorman Reviewed-by: KAMEZAWA Hiroyuki Reviewed-by: Johannes Weiner Reviewed-by: KOSAKI Motohiro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/pagemap.h | 1 - mm/filemap.c | 25 +++++++++---------------- 2 files changed, 9 insertions(+), 17 deletions(-) (limited to 'include/linux/pagemap.h') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 0aa258f4678..ea36ff29a51 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -456,7 +456,6 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping, int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); extern void delete_from_page_cache(struct page *page); -extern void remove_from_page_cache(struct page *page); extern void __remove_from_page_cache(struct page *page); int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask); diff --git a/mm/filemap.c b/mm/filemap.c index e7b59785ceb..1cfb8fd84b2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -137,7 +137,15 @@ void __remove_from_page_cache(struct page *page) } } -void remove_from_page_cache(struct page *page) +/** + * delete_from_page_cache - delete page from page cache + * @page: the page which the kernel is trying to remove from page cache + * + * This must be called only on pages that have been verified to be in the page + * cache and locked. It will never put the page into the free list, the caller + * has a reference on the page. + */ +void delete_from_page_cache(struct page *page) { struct address_space *mapping = page->mapping; void (*freepage)(struct page *); @@ -152,21 +160,6 @@ void remove_from_page_cache(struct page *page) if (freepage) freepage(page); -} -EXPORT_SYMBOL(remove_from_page_cache); - -/** - * delete_from_page_cache - delete page from page cache - * @page: the page which the kernel is trying to remove from page cache - * - * This must be called only on pages that have - * been verified to be in the page cache and locked. - * It will never put the page into the free list, - * the caller has a reference on the page. - */ -void delete_from_page_cache(struct page *page) -{ - remove_from_page_cache(page); page_cache_release(page); } EXPORT_SYMBOL(delete_from_page_cache); -- cgit v1.2.3-70-g09d2 From e64a782fec684c29a8204c51b3cb554dce588592 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Tue, 22 Mar 2011 16:32:44 -0700 Subject: mm: change __remove_from_page_cache() Now we renamed remove_from_page_cache with delete_from_page_cache. As consistency of __remove_from_swap_cache and remove_from_swap_cache, we change internal page cache handling function name, too. Signed-off-by: Minchan Kim Cc: Christoph Hellwig Acked-by: Hugh Dickins Acked-by: Mel Gorman Reviewed-by: KAMEZAWA Hiroyuki Reviewed-by: Johannes Weiner Reviewed-by: KOSAKI Motohiro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/pagemap.h | 2 +- mm/filemap.c | 8 ++++---- mm/memory-failure.c | 2 +- mm/truncate.c | 2 +- mm/vmscan.c | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) (limited to 'include/linux/pagemap.h') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ea36ff29a51..29ebba54c23 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -456,7 +456,7 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping, int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); extern void delete_from_page_cache(struct page *page); -extern void __remove_from_page_cache(struct page *page); +extern void __delete_from_page_cache(struct page *page); int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask); /* diff --git a/mm/filemap.c b/mm/filemap.c index 1cfb8fd84b2..cb476e70cf1 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -108,11 +108,11 @@ */ /* - * Remove a page from the page cache and free it. Caller has to make + * Delete a page from the page cache and free it. Caller has to make * sure the page is locked and that nobody else uses it - or that usage * is safe. The caller must hold the mapping's tree_lock. */ -void __remove_from_page_cache(struct page *page) +void __delete_from_page_cache(struct page *page) { struct address_space *mapping = page->mapping; @@ -154,7 +154,7 @@ void delete_from_page_cache(struct page *page) freepage = mapping->a_ops->freepage; spin_lock_irq(&mapping->tree_lock); - __remove_from_page_cache(page); + __delete_from_page_cache(page); spin_unlock_irq(&mapping->tree_lock); mem_cgroup_uncharge_cache_page(page); @@ -444,7 +444,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) new->index = offset; spin_lock_irq(&mapping->tree_lock); - __remove_from_page_cache(old); + __delete_from_page_cache(old); error = radix_tree_insert(&mapping->page_tree, offset, new); BUG_ON(error); mapping->nrpages++; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 99ccb447262..e0af336530c 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1130,7 +1130,7 @@ int __memory_failure(unsigned long pfn, int trapno, int flags) /* * Now take care of user space mappings. - * Abort on fail: __remove_from_page_cache() assumes unmapped page. + * Abort on fail: __delete_from_page_cache() assumes unmapped page. */ if (hwpoison_user_mappings(p, pfn, trapno) != SWAP_SUCCESS) { printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn); diff --git a/mm/truncate.c b/mm/truncate.c index 7c617b5f03e..3d2ae1f423d 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -388,7 +388,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page) clear_page_mlock(page); BUG_ON(page_has_private(page)); - __remove_from_page_cache(page); + __delete_from_page_cache(page); spin_unlock_irq(&mapping->tree_lock); mem_cgroup_uncharge_cache_page(page); diff --git a/mm/vmscan.c b/mm/vmscan.c index 3b4a41d7248..665b090b6c7 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -514,7 +514,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page) freepage = mapping->a_ops->freepage; - __remove_from_page_cache(page); + __delete_from_page_cache(page); spin_unlock_irq(&mapping->tree_lock); mem_cgroup_uncharge_cache_page(page); -- cgit v1.2.3-70-g09d2