diff options
53 files changed, 326 insertions, 132 deletions
diff --git a/arch/frv/mm/pgalloc.c b/arch/frv/mm/pgalloc.c index 1a2e5c8d03a..66f616fb486 100644 --- a/arch/frv/mm/pgalloc.c +++ b/arch/frv/mm/pgalloc.c @@ -28,7 +28,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) return pte; } -struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *page; @@ -37,9 +37,11 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) #else page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); #endif - if (page) + if (page) { clear_highpage(page); - flush_dcache_page(page); + pgtable_page_ctor(page); + flush_dcache_page(page); + } return page; } diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index f80f90c4d58..ac3390f8190 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -107,19 +107,20 @@ __init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long add return pte; } -struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *ptepage; #ifdef CONFIG_HIGHPTE - gfp_t flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT; + gfp_t flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT | __GFP_ZERO; #else - gfp_t flags = GFP_KERNEL | __GFP_REPEAT; + gfp_t flags = GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO; #endif ptepage = alloc_pages(flags, 0); - if (ptepage) - clear_highpage(ptepage); + if (!ptepage) + return NULL; + pgtable_page_ctor(ptepage); return ptepage; } @@ -131,11 +132,12 @@ void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -void pte_free(struct mm_struct *mm, struct page *ptepage) +void pte_free(struct mm_struct *mm, pgtable_t ptepage) { #ifdef CONFIG_SMP hash_page_sync(); #endif + pgtable_page_dtor(ptepage); __free_page(ptepage); } diff --git a/arch/ppc/mm/pgtable.c b/arch/ppc/mm/pgtable.c index 409fcaa4994..03a79bff127 100644 --- a/arch/ppc/mm/pgtable.c +++ b/arch/ppc/mm/pgtable.c @@ -95,7 +95,7 @@ __init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long add return pte; } -struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *ptepage; @@ -106,8 +106,10 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) #endif ptepage = alloc_pages(flags, 0); - if (ptepage) + if (ptepage) { clear_highpage(ptepage); + pgtable_page_ctor(ptepage); + } return ptepage; } @@ -119,11 +121,12 @@ void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -void pte_free(struct mm_struct *mm, struct page *ptepage) +void pte_free(struct mm_struct *mm, pgtable_t ptepage) { #ifdef CONFIG_SMP hash_page_sync(); #endif + pgtable_page_dtor(ptepage); __free_page(ptepage); } diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index e60e0ae1340..019f518cd5a 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -78,6 +78,7 @@ unsigned long *page_table_alloc(int noexec) clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); page->index = (addr_t) table; } + pgtable_page_ctor(page); table = (unsigned long *) page_to_phys(page); clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); return table; @@ -87,6 +88,7 @@ void page_table_free(unsigned long *table) { unsigned long *shadow = get_shadow_pte(table); + pgtable_page_dtor(virt_to_page(table)); if (shadow) free_page((unsigned long) shadow); free_page((unsigned long) table); diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index dc98e3844a0..23d3291a3e8 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -489,14 +489,17 @@ srmmu_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE); } -static struct page * +static pgtable_t srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address) { unsigned long pte; + struct page *page; if ((pte = (unsigned long)srmmu_pte_alloc_one_kernel(mm, address)) == 0) return NULL; - return pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT ); + page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT ); + pgtable_page_ctor(page); + return page; } static void srmmu_free_pte_fast(pte_t *pte) @@ -504,10 +507,11 @@ static void srmmu_free_pte_fast(pte_t *pte) srmmu_free_nocache((unsigned long)pte, PTE_SIZE); } -static void srmmu_pte_free(struct page *pte) +static void srmmu_pte_free(pgtable_t pte) { unsigned long p; + pgtable_page_dtor(pte); p = (unsigned long)page_address(pte); /* Cached address (for test) */ if (p == 0) BUG(); diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c index 0729305f2f5..c0442e8c4b1 100644 --- a/arch/sparc/mm/sun4c.c +++ b/arch/sparc/mm/sun4c.c @@ -1947,12 +1947,17 @@ static pte_t *sun4c_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long add return pte; } -static struct page *sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address) +static pgtable_t sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address) { - pte_t *pte = sun4c_pte_alloc_one_kernel(mm, address); + pte_t *pte; + struct page *page; + + pte = sun4c_pte_alloc_one_kernel(mm, address); if (pte == NULL) return NULL; - return virt_to_page(pte); + page = virt_to_page(pte); + pgtable_page_ctor(page); + return page; } static inline void sun4c_free_pte_fast(pte_t *pte) @@ -1962,8 +1967,9 @@ static inline void sun4c_free_pte_fast(pte_t *pte) pgtable_cache_size++; } -static void sun4c_pte_free(struct page *pte) +static void sun4c_pte_free(pgtable_t pte) { + pgtable_page_dtor(pte); sun4c_free_pte_fast(page_address(pte)); } diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index d872fdce1d7..2627ce82e91 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -354,11 +354,13 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) return pte; } -struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte; pte = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + if (pte) + pgtable_page_ctor(pte); return pte; } diff --git a/arch/x86/mm/pgtable_32.c b/arch/x86/mm/pgtable_32.c index 6c1914622a8..73aba712520 100644 --- a/arch/x86/mm/pgtable_32.c +++ b/arch/x86/mm/pgtable_32.c @@ -183,7 +183,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) return (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); } -struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte; @@ -192,6 +192,8 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) #else pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0); #endif + if (pte) + pgtable_page_ctor(pte); return pte; } @@ -365,6 +367,7 @@ void check_pgt_cache(void) void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte) { + pgtable_page_dtor(pte); paravirt_release_pt(page_to_pfn(pte)); tlb_remove_page(tlb, pte); } diff --git a/include/asm-alpha/page.h b/include/asm-alpha/page.h index 05f09f997d8..22ff9762d17 100644 --- a/include/asm-alpha/page.h +++ b/include/asm-alpha/page.h @@ -62,6 +62,8 @@ typedef unsigned long pgprot_t; #endif /* STRICT_MM_TYPECHECKS */ +typedef struct page *pgtable_t; + #ifdef USE_48_BIT_KSEG #define PAGE_OFFSET 0xffff800000000000UL #else diff --git a/include/asm-alpha/pgalloc.h b/include/asm-alpha/pgalloc.h index fdbedacc737..fd090155dcc 100644 --- a/include/asm-alpha/pgalloc.h +++ b/include/asm-alpha/pgalloc.h @@ -11,10 +11,11 @@ */ static inline void -pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte) +pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte) { pmd_set(pmd, (pte_t *)(page_to_pa(pte) + PAGE_OFFSET)); } +#define pmd_pgtable(pmd) pmd_page(pmd) static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) @@ -57,18 +58,23 @@ pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -static inline struct page * -pte_alloc_one(struct mm_struct *mm, unsigned long addr) +static inline pgtable_t +pte_alloc_one(struct mm_struct *mm, unsigned long address) { - pte_t *pte = pte_alloc_one_kernel(mm, addr); - if (pte) - return virt_to_page(pte); - return NULL; + pte_t *pte = pte_alloc_one_kernel(mm, address); + struct page *page; + + if (!pte) + return NULL; + page = virt_to_page(pte); + pgtable_page_ctor(page); + return page; } static inline void -pte_free(struct mm_struct *mm, struct page *page) +pte_free(struct mm_struct *mm, pgtable_t page) { + pgtable_page_dtor(page); __free_page(page); } diff --git a/include/asm-arm/page.h b/include/asm-arm/page.h index 31ff12f4ffb..c86f68ee651 100644 --- a/include/asm-arm/page.h +++ b/include/asm-arm/page.h @@ -171,6 +171,8 @@ typedef unsigned long pgprot_t; #endif /* STRICT_MM_TYPECHECKS */ +typedef struct page *pgtable_t; + #endif /* CONFIG_MMU */ #include <asm/memory.h> diff --git a/include/asm-arm/pgalloc.h b/include/asm-arm/pgalloc.h index fb6c6e3222b..163b0305dd7 100644 --- a/include/asm-arm/pgalloc.h +++ b/include/asm-arm/pgalloc.h @@ -66,7 +66,7 @@ pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) return pte; } -static inline struct page * +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr) { struct page *pte; @@ -75,6 +75,7 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr) if (pte) { void *page = page_address(pte); clean_dcache_area(page, sizeof(pte_t) * PTRS_PER_PTE); + pgtable_page_ctor(pte); } return pte; @@ -91,8 +92,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) } } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } @@ -123,10 +125,11 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) } static inline void -pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) +pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) { __pmd_populate(pmdp, page_to_pfn(ptep) << PAGE_SHIFT | _PAGE_USER_TABLE); } +#define pmd_pgtable(pmd) pmd_page(pmd) #endif /* CONFIG_MMU */ diff --git a/include/asm-avr32/page.h b/include/asm-avr32/page.h index ee23499cec3..5582968feee 100644 --- a/include/asm-avr32/page.h +++ b/include/asm-avr32/page.h @@ -34,6 +34,7 @@ extern void copy_page(void *to, void *from); typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct page *pgtable_t; #define pte_val(x) ((x).pte) #define pgd_val(x) ((x).pgd) diff --git a/include/asm-avr32/pgalloc.h b/include/asm-avr32/pgalloc.h index b77e364b4c4..51fc1f6e4b1 100644 --- a/include/asm-avr32/pgalloc.h +++ b/include/asm-avr32/pgalloc.h @@ -17,10 +17,11 @@ set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))) static __inline__ void pmd_populate(struct mm_struct *mm, pmd_t *pmd, - struct page *pte) + pgtable_t pte) { set_pmd(pmd, __pmd(_PAGE_TABLE + page_to_phys(pte))); } +#define pmd_pgtable(pmd) pmd_page(pmd) /* * Allocate and free page tables @@ -51,7 +52,9 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, struct page *pte; pte = alloc_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); - + if (!pte) + return NULL; + pgtable_page_ctor(pte); return pte; } @@ -60,12 +63,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } -#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0) #define check_pgt_cache() do { } while(0) diff --git a/include/asm-cris/page.h b/include/asm-cris/page.h index 3b0156c4631..c45bb1ef397 100644 --- a/include/asm-cris/page.h +++ b/include/asm-cris/page.h @@ -26,6 +26,7 @@ typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct page *pgtable_t; #endif #define pte_val(x) ((x).pte) diff --git a/include/asm-cris/pgalloc.h b/include/asm-cris/pgalloc.h index 8ddd66f8177..a1ba761d057 100644 --- a/include/asm-cris/pgalloc.h +++ b/include/asm-cris/pgalloc.h @@ -6,6 +6,7 @@ #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, pte) #define pmd_populate(mm, pmd, pte) pmd_set(pmd, page_address(pte)) +#define pmd_pgtable(pmd) pmd_page(pmd) /* * Allocate and free page tables. @@ -27,10 +28,11 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long ad return pte; } -static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte; pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0); + pgtable_page_ctor(pte); return pte; } @@ -39,13 +41,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } - -#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0) #define check_pgt_cache() do { } while (0) diff --git a/include/asm-frv/page.h b/include/asm-frv/page.h index cacc045700d..c2c1e89e747 100644 --- a/include/asm-frv/page.h +++ b/include/asm-frv/page.h @@ -25,6 +25,7 @@ typedef struct { unsigned long ste[64];} pmd_t; typedef struct { pmd_t pue[1]; } pud_t; typedef struct { pud_t pge[1]; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct page *pgtable_t; #define pte_val(x) ((x).pte) #define pmd_val(x) ((x).ste[0]) diff --git a/include/asm-frv/pgalloc.h b/include/asm-frv/pgalloc.h index e89620ef08c..971e6addb00 100644 --- a/include/asm-frv/pgalloc.h +++ b/include/asm-frv/pgalloc.h @@ -25,6 +25,7 @@ do { \ __set_pmd((PMD), page_to_pfn(PAGE) << PAGE_SHIFT | _PAGE_TABLE); \ } while(0) +#define pmd_pgtable(pmd) pmd_page(pmd) /* * Allocate and free page tables. @@ -35,19 +36,24 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *); extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); -extern struct page *pte_alloc_one(struct mm_struct *, unsigned long); +extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) { free_page((unsigned long)pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } -#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb),(pte)); \ +} while (0) /* * allocating and freeing a pmd is trivial: the 1-entry pmd is diff --git a/include/asm-ia64/page.h b/include/asm-ia64/page.h index 8a8aa3fd7cd..4999a6c6377 100644 --- a/include/asm-ia64/page.h +++ b/include/asm-ia64/page.h @@ -185,6 +185,7 @@ get_order (unsigned long size) #endif typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; + typedef struct page *pgtable_t; # define pte_val(x) ((x).pte) # define pmd_val(x) ((x).pmd) @@ -206,6 +207,7 @@ get_order (unsigned long size) typedef unsigned long pmd_t; typedef unsigned long pgd_t; typedef unsigned long pgprot_t; + typedef struct page *pgtable_t; # endif # define pte_val(x) (x) diff --git a/include/asm-ia64/pgalloc.h b/include/asm-ia64/pgalloc.h index 556d988123a..b9ac1a6fc21 100644 --- a/include/asm-ia64/pgalloc.h +++ b/include/asm-ia64/pgalloc.h @@ -70,10 +70,11 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) #define __pmd_free_tlb(tlb, pmd) pmd_free((tlb)->mm, pmd) static inline void -pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, struct page *pte) +pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, pgtable_t pte) { pmd_val(*pmd_entry) = page_to_phys(pte); } +#define pmd_pgtable(pmd) pmd_page(pmd) static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte) @@ -81,11 +82,17 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte) pmd_val(*pmd_entry) = __pa(pte); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, - unsigned long addr) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr) { - void *pg = quicklist_alloc(0, GFP_KERNEL, NULL); - return pg ? virt_to_page(pg) : NULL; + struct page *page; + void *pg; + + pg = quicklist_alloc(0, GFP_KERNEL, NULL); + if (!pg) + return NULL; + page = virt_to_page(pg); + pgtable_page_ctor(page); + return page; } static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, @@ -94,8 +101,9 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, return quicklist_alloc(0, GFP_KERNEL, NULL); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); quicklist_free_page(0, NULL, pte); } diff --git a/include/asm-m32r/page.h b/include/asm-m32r/page.h index 05d43bbbf94..8a677f3fca6 100644 --- a/include/asm-m32r/page.h +++ b/include/asm-m32r/page.h @@ -28,6 +28,7 @@ typedef struct { unsigned long pgd; } pgd_t; #define PTE_MASK PAGE_MASK typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct page *pgtable_t; #define pmd_val(x) ((x).pmd) #define pgd_val(x) ((x).pgd) diff --git a/include/asm-m32r/pgalloc.h b/include/asm-m32r/pgalloc.h index e5921adfad1..f11a2b909cd 100644 --- a/include/asm-m32r/pgalloc.h +++ b/include/asm-m32r/pgalloc.h @@ -9,10 +9,11 @@ set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))) static __inline__ void pmd_populate(struct mm_struct *mm, pmd_t *pmd, - struct page *pte) + pgtable_t pte) { set_pmd(pmd, __pmd(_PAGE_TABLE + page_to_phys(pte))); } +#define pmd_pgtable(pmd) pmd_page(pmd) /* * Allocate and free page tables. @@ -37,12 +38,12 @@ static __inline__ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, return pte; } -static __inline__ struct page *pte_alloc_one(struct mm_struct *mm, +static __inline__ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *pte = alloc_page(GFP_KERNEL|__GFP_ZERO); - + pgtable_page_ctor(pte); return pte; } @@ -51,8 +52,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } diff --git a/include/asm-m68k/motorola_pgalloc.h b/include/asm-m68k/motorola_pgalloc.h index 500ec9b8b18..d08bf6261df 100644 --- a/include/asm-m68k/motorola_pgalloc.h +++ b/include/asm-m68k/motorola_pgalloc.h @@ -7,7 +7,6 @@ extern pmd_t *get_pointer_table(void); extern int free_pointer_table(pmd_t *); - static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte; @@ -28,7 +27,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long) pte); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0); pte_t *pte; @@ -43,19 +42,21 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long add nocache_page(pte); } kunmap(pte); - + pgtable_page_ctor(page); return page; } -static inline void pte_free(struct mm_struct *mm, struct page *page) +static inline void pte_free(struct mm_struct *mm, pgtable_t page) { + pgtable_page_dtor(page); cache_page(kmap(page)); kunmap(page); __free_page(page); } -static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *page) +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page) { + pgtable_page_dtor(page); cache_page(kmap(page)); kunmap(page); __free_page(page); @@ -94,10 +95,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t * pmd_set(pmd, pte); } -static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page) { pmd_set(pmd, page_address(page)); } +#define pmd_pgtable(pmd) pmd_page(pmd) static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) { diff --git a/include/asm-m68k/page.h b/include/asm-m68k/page.h index 3f29e2a03a4..880c2cbff8a 100644 --- a/include/asm-m68k/page.h +++ b/include/asm-m68k/page.h @@ -91,6 +91,7 @@ typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pmd[16]; } pmd_t; typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct page *pgtable_t; #define pte_val(x) ((x).pte) #define pmd_val(x) ((&x)->pmd[0]) diff --git a/include/asm-m68k/sun3_pgalloc.h b/include/asm-m68k/sun3_pgalloc.h index a5a91e72714..d4c83f14381 100644 --- a/include/asm-m68k/sun3_pgalloc.h +++ b/include/asm-m68k/sun3_pgalloc.h @@ -26,12 +26,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long) pte); } -static inline void pte_free(struct mm_struct *mm, struct page *page) +static inline void pte_free(struct mm_struct *mm, pgtable_t page) { + pgtable_page_dtor(page); __free_page(page); } -#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0) static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) @@ -45,8 +50,8 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, return (pte_t *) (page); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, - unsigned long address) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address) { struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); @@ -54,6 +59,7 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, return NULL; clear_highpage(page); + pgtable_page_ctor(page); return page; } @@ -63,10 +69,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t * pmd_val(*pmd) = __pa((unsigned long)pte); } -static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page) { pmd_val(*pmd) = __pa((unsigned long)page_address(page)); } +#define pmd_pgtable(pmd) pmd_page(pmd) /* * allocating and freeing a pmd is trivial: the 1-entry pmd is diff --git a/include/asm-mips/page.h b/include/asm-mips/page.h index 635aa44d229..8735aa0b896 100644 --- a/include/asm-mips/page.h +++ b/include/asm-mips/page.h @@ -90,6 +90,7 @@ typedef struct { unsigned long pte; } pte_t; #define pte_val(x) ((x).pte) #define __pte(x) ((pte_t) { (x) } ) #endif +typedef struct page *pgtable_t; /* * For 3-level pagetables we defines these ourselves, for 2-level the diff --git a/include/asm-mips/pgalloc.h b/include/asm-mips/pgalloc.h index c4efeced839..1275831dda2 100644 --- a/include/asm-mips/pgalloc.h +++ b/include/asm-mips/pgalloc.h @@ -20,10 +20,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, } static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, - struct page *pte) + pgtable_t pte) { set_pmd(pmd, __pmd((unsigned long)page_address(pte))); } +#define pmd_pgtable(pmd) pmd_page(pmd) /* * Initialize a new pmd table with invalid pointers. @@ -79,9 +80,10 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, struct page *pte; pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER); - if (pte) + if (pte) { clear_highpage(pte); - + pgtable_page_ctor(pte); + } return pte; } @@ -90,12 +92,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_pages((unsigned long)pte, PTE_ORDER); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_pages(pte, PTE_ORDER); } -#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0) #ifdef CONFIG_32BIT diff --git a/include/asm-parisc/page.h b/include/asm-parisc/page.h index b08d9151c71..27d50b85954 100644 --- a/include/asm-parisc/page.h +++ b/include/asm-parisc/page.h @@ -91,6 +91,7 @@ typedef unsigned long pgprot_t; #endif /* STRICT_MM_TYPECHECKS */ +typedef struct page *pgtable_t; typedef struct __physmem_range { unsigned long start_pfn; diff --git a/include/asm-parisc/pgalloc.h b/include/asm-parisc/pgalloc.h index aab66f1bea1..3996dfc30a3 100644 --- a/include/asm-parisc/pgalloc.h +++ b/include/asm-parisc/pgalloc.h @@ -115,11 +115,14 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) #define pmd_populate(mm, pmd, pte_page) \ pmd_populate_kernel(mm, pmd, page_address(pte_page)) +#define pmd_pgtable(pmd) pmd_page(pmd) -static inline struct page * +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + if (page) + pgtable_page_ctor(page); return page; } @@ -135,7 +138,11 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -#define pte_free(mm, page) pte_free_kernel(page_address(page)) +static inline void pte_free_kernel(struct mm_struct *mm, struct page *pte) +{ + pgtable_page_dtor(pte); + pte_free_kernel(page_address((pte)); +} #define check_pgt_cache() do { } while (0) diff --git a/include/asm-powerpc/page.h b/include/asm-powerpc/page.h index 61e3725bbd3..df47bbb6ea1 100644 --- a/include/asm-powerpc/page.h +++ b/include/asm-powerpc/page.h @@ -190,6 +190,8 @@ extern int page_is_ram(unsigned long pfn); struct vm_area_struct; +typedef struct page *pgtable_t; + #include <asm-generic/memory_model.h> #endif /* __ASSEMBLY__ */ diff --git a/include/asm-powerpc/pgalloc-32.h b/include/asm-powerpc/pgalloc-32.h index c162a4c37b3..58c07147b3e 100644 --- a/include/asm-powerpc/pgalloc-32.h +++ b/include/asm-powerpc/pgalloc-32.h @@ -22,17 +22,19 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); (pmd_val(*(pmd)) = __pa(pte) | _PMD_PRESENT) #define pmd_populate(mm, pmd, pte) \ (pmd_val(*(pmd)) = (page_to_pfn(pte) << PAGE_SHIFT) | _PMD_PRESENT) +#define pmd_pgtable(pmd) pmd_page(pmd) #else #define pmd_populate_kernel(mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)pte | _PMD_PRESENT) #define pmd_populate(mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)lowmem_page_address(pte) | _PMD_PRESENT) +#define pmd_pgtable(pmd) pmd_page(pmd) #endif extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr); -extern struct page *pte_alloc_one(struct mm_struct *mm, unsigned long addr); +extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr); extern void pte_free_kernel(struct mm_struct *mm, pte_t *pte); -extern void pte_free(struct mm_struct *mm, struct page *pte); +extern void pte_free(struct mm_struct *mm, pgtable_t pte); #define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, (pte)) diff --git a/include/asm-powerpc/pgalloc-64.h b/include/asm-powerpc/pgalloc-64.h index 5afae859393..68980990f62 100644 --- a/include/asm-powerpc/pgalloc-64.h +++ b/include/asm-powerpc/pgalloc-64.h @@ -58,6 +58,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) #define pmd_populate(mm, pmd, pte_page) \ pmd_populate_kernel(mm, pmd, page_address(pte_page)) #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte)) +#define pmd_pgtable(pmd) pmd_page(pmd) #else /* CONFIG_PPC_64K_PAGES */ @@ -72,6 +73,7 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, #define pmd_populate(mm, pmd, pte_page) \ pmd_populate_kernel(mm, pmd, page_address(pte_page)) +#define pmd_pgtable(pmd) pmd_page(pmd) #endif /* CONFIG_PPC_64K_PAGES */ @@ -92,11 +94,18 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, - unsigned long address) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address) { - pte_t *pte = pte_alloc_one_kernel(mm, address); - return pte ? virt_to_page(pte) : NULL; + struct page *page; + pte_t *pte; + + pte = pte_alloc_one_kernel(mm, address); + if (!pte) + return NULL; + page = virt_to_page(pte); + pgtable_page_ctor(page); + return page; } static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) @@ -104,8 +113,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -static inline void pte_free(struct mm_struct *mm, struct page *ptepage) +static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) { + pgtable_page_dtor(ptepage); __free_page(ptepage); } @@ -136,9 +146,12 @@ static inline void pgtable_free(pgtable_free_t pgf) extern void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf); -#define __pte_free_tlb(tlb, ptepage) \ +#define __pte_free_tlb(tlb,ptepage) \ +do { \ + pgtable_page_dtor(ptepage); \ pgtable_free_tlb(tlb, pgtable_free_cache(page_address(ptepage), \ - PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)) + PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \ +} while (0) #define __pmd_free_tlb(tlb, pmd) \ pgtable_free_tlb(tlb, pgtable_free_cache(pmd, \ PMD_CACHE_NUM, PMD_TABLE_SIZE-1)) diff --git a/include/asm-ppc/pgalloc.h b/include/asm-ppc/pgalloc.h index 7c39a95829c..fd4d1d74cfb 100644 --- a/include/asm-ppc/pgalloc.h +++ b/include/asm-ppc/pgalloc.h @@ -23,17 +23,19 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); (pmd_val(*(pmd)) = __pa(pte) | _PMD_PRESENT) #define pmd_populate(mm, pmd, pte) \ (pmd_val(*(pmd)) = (page_to_pfn(pte) << PAGE_SHIFT) | _PMD_PRESENT) +#define pmd_pgtable(pmd) pmd_page(pmd) #else #define pmd_populate_kernel(mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)pte | _PMD_PRESENT) #define pmd_populate(mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)lowmem_page_address(pte) | _PMD_PRESENT) +#define pmd_pgtable(pmd) pmd_page(pmd) #endif extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr); -extern struct page *pte_alloc_one(struct mm_struct *mm, unsigned long addr); +extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr); extern void pte_free_kernel(struct mm_struct *mm, pte_t *pte); -extern void pte_free(struct mm_struct *mm, struct page *pte); +extern void pte_free(struct mm_struct *mm, pgtable_t pte); #define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, (pte)) diff --git a/include/asm-s390/page.h b/include/asm-s390/page.h index a55f9d979df..7f29a981f48 100644 --- a/include/asm-s390/page.h +++ b/include/asm-s390/page.h @@ -109,6 +109,8 @@ typedef struct { unsigned long pgd; } pgd_t; #endif /* __s390x__ */ +typedef struct page *pgtable_t; + #define __pte(x) ((pte_t) { (x) } ) #define __pmd(x) ((pmd_t) { (x) } ) #define __pgd(x) ((pgd_t) { (x) } ) diff --git a/include/asm-s390/pgalloc.h b/include/asm-s390/pgalloc.h index 6f6619ba898..900d44807e1 100644 --- a/include/asm-s390/pgalloc.h +++ b/include/asm-s390/pgalloc.h @@ -132,7 +132,7 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) } static inline void -pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) +pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page) { pte_t *pte = (pte_t *)page_to_phys(page); pmd_t *shadow_pmd = get_shadow_table(pmd); @@ -142,6 +142,7 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) if (shadow_pmd && shadow_pte) pmd_populate_kernel(mm, shadow_pmd, shadow_pte); } +#define pmd_pgtable(pmd) pmd_page(pmd) /* * page table entry allocation/free routines. diff --git a/include/asm-s390/tlb.h b/include/asm-s390/tlb.h index 985de2b8827..3c8177fa9e0 100644 --- a/include/asm-s390/tlb.h +++ b/include/asm-s390/tlb.h @@ -95,7 +95,7 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) * pte_free_tlb frees a pte table and clears the CRSTE for the * page table from the tlb. */ -static inline void pte_free_tlb(struct mmu_gather *tlb, struct page *page) +static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t page) { if (!tlb->fullmm) { tlb->array[tlb->nr_ptes++] = page; diff --git a/include/asm-sh/page.h b/include/asm-sh/page.h index e0fe02950f5..134562dc8c4 100644 --- a/include/asm-sh/page.h +++ b/include/asm-sh/page.h @@ -100,6 +100,8 @@ typedef struct { unsigned long pgd; } pgd_t; #define __pgd(x) ((pgd_t) { (x) } ) #define __pgprot(x) ((pgprot_t) { (x) } ) +typedef struct page *pgtable_t; + #endif /* !__ASSEMBLY__ */ /* diff --git a/include/asm-sh/pgalloc.h b/include/asm-sh/pgalloc.h index 59ca16d77a1..84dd2db7104 100644 --- a/include/asm-sh/pgalloc.h +++ b/include/asm-sh/pgalloc.h @@ -14,10 +14,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, } static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, - struct page *pte) + pgtable_t pte) { set_pmd(pmd, __pmd((unsigned long)page_address(pte))); } +#define pmd_pgtable(pmd) pmd_page(pmd) static inline void pgd_ctor(void *x) { @@ -47,11 +48,18 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, return quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, - unsigned long address) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address) { - void *pg = quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL); - return pg ? virt_to_page(pg) : NULL; + struct page *page; + void *pg; + + pg = quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL); + if (!pg) + return NULL; + page = virt_to_page(pg); + pgtable_page_ctor(page); + return page; } static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) @@ -59,12 +67,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) quicklist_free(QUICK_PT, NULL, pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); quicklist_free_page(QUICK_PT, NULL, pte); } -#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), (pte)); \ +} while (0) /* * allocating and freeing a pmd is trivial: the 1-entry pmd is diff --git a/include/asm-sparc/page.h b/include/asm-sparc/page.h index cbc48c0c4e1..39ccf2da297 100644 --- a/include/asm-sparc/page.h +++ b/include/asm-sparc/page.h @@ -123,6 +123,8 @@ typedef unsigned long iopgprot_t; #endif +typedef struct page *pgtable_t; + extern unsigned long sparc_unmapped_base; BTFIXUPDEF_SETHI(sparc_unmapped_base) diff --git a/include/asm-sparc/pgalloc.h b/include/asm-sparc/pgalloc.h index b5fbdd36447..6292cd00e5a 100644 --- a/include/asm-sparc/pgalloc.h +++ b/include/asm-sparc/pgalloc.h @@ -50,10 +50,11 @@ BTFIXUPDEF_CALL(void, free_pmd_fast, pmd_t *) BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *) #define pmd_populate(MM, PMD, PTE) BTFIXUP_CALL(pmd_populate)(PMD, PTE) +#define pmd_pgtable(pmd) pmd_page(pmd) BTFIXUPDEF_CALL(void, pmd_set, pmd_t *, pte_t *) #define pmd_populate_kernel(MM, PMD, PTE) BTFIXUP_CALL(pmd_set)(PMD, PTE) -BTFIXUPDEF_CALL(struct page *, pte_alloc_one, struct mm_struct *, unsigned long) +BTFIXUPDEF_CALL(pgtable_t , pte_alloc_one, struct mm_struct *, unsigned long) #define pte_alloc_one(mm, address) BTFIXUP_CALL(pte_alloc_one)(mm, address) BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_kernel, struct mm_struct *, unsigned long) #define pte_alloc_one_kernel(mm, addr) BTFIXUP_CALL(pte_alloc_one_kernel)(mm, addr) @@ -61,7 +62,7 @@ BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_kernel, struct mm_struct *, unsigned long BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *) #define pte_free_kernel(mm, pte) BTFIXUP_CALL(free_pte_fast)(pte) -BTFIXUPDEF_CALL(void, pte_free, struct page *) +BTFIXUPDEF_CALL(void, pte_free, pgtable_t ) #define pte_free(mm, pte) BTFIXUP_CALL(pte_free)(pte) #define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte) diff --git a/include/asm-sparc64/page.h b/include/asm-sparc64/page.h index cdf950e017e..e93a482aa24 100644 --- a/include/asm-sparc64/page.h +++ b/include/asm-sparc64/page.h @@ -104,6 +104,8 @@ typedef unsigned long pgprot_t; #endif /* (STRICT_MM_TYPECHECKS) */ +typedef struct page *pgtable_t; + #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ (_AC(0x0000000070000000,UL)) : \ (_AC(0xfffff80000000000,UL) + (1UL << 32UL))) diff --git a/include/asm-sparc64/pgalloc.h b/include/asm-sparc64/pgalloc.h index b48f73c2274..3ee2d406373 100644 --- a/include/asm-sparc64/pgalloc.h +++ b/include/asm-sparc64/pgalloc.h @@ -43,11 +43,18 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, return quicklist_alloc(0, GFP_KERNEL, NULL); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, - unsigned long address) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address) { - void *pg = quicklist_alloc(0, GFP_KERNEL, NULL); - return pg ? virt_to_page(pg) : NULL; + struct page *page; + void *pg; + + pg = quicklist_alloc(0, GFP_KERNEL, NULL); + if (!pg) + return NULL; + page = virt_to_page(pg); + pgtable_page_ctor(page); + return page; } static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) @@ -55,8 +62,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) quicklist_free(0, NULL, pte); } -static inline void pte_free(struct mm_struct *mm, struct page *ptepage) +static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) { + pgtable_page_dtor(ptepage); quicklist_free_page(0, NULL, ptepage); } @@ -64,6 +72,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *ptepage) #define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE) #define pmd_populate(MM,PMD,PTE_PAGE) \ pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE)) +#define pmd_pgtable(pmd) pmd_page(pmd) static inline void check_pgt_cache(void) { diff --git a/include/asm-um/page.h b/include/asm-um/page.h index fe2374d705d..381f96b1c82 100644 --- a/include/asm-um/page.h +++ b/include/asm-um/page.h @@ -79,6 +79,8 @@ typedef unsigned long phys_t; typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct page *pgtable_t; + #define pgd_val(x) ((x).pgd) #define pgprot_val(x) ((x).pgprot) diff --git a/include/asm-um/pgalloc.h b/include/asm-um/pgalloc.h index 4f3e62b0286..9062a6e7224 100644 --- a/include/asm-um/pgalloc.h +++ b/include/asm-um/pgalloc.h @@ -18,6 +18,7 @@ set_pmd(pmd, __pmd(_PAGE_TABLE + \ ((unsigned long long)page_to_pfn(pte) << \ (unsigned long long) PAGE_SHIFT))) +#define pmd_pgtable(pmd) pmd_page(pmd) /* * Allocate and free page tables. @@ -26,19 +27,24 @@ extern pgd_t *pgd_alloc(struct mm_struct *); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); -extern struct page *pte_alloc_one(struct mm_struct *, unsigned long); +extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) { free_page((unsigned long) pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } -#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb),(pte)); \ +} while (0) #ifdef CONFIG_3_LEVEL_PGTABLES diff --git a/include/asm-x86/page_32.h b/include/asm-x86/page_32.h index a6fd10f230d..ba715d9798b 100644 --- a/include/asm-x86/page_32.h +++ b/include/asm-x86/page_32.h @@ -50,6 +50,8 @@ typedef unsigned long phys_addr_t; typedef union { pteval_t pte, pte_low; } pte_t; typedef pte_t boot_pte_t; +typedef struct page *pgtable_t; + #endif /* __ASSEMBLY__ */ #endif /* CONFIG_X86_PAE */ diff --git a/include/asm-x86/page_64.h b/include/asm-x86/page_64.h index dcf0c074607..f7393bc516e 100644 --- a/include/asm-x86/page_64.h +++ b/include/asm-x86/page_64.h @@ -71,6 +71,8 @@ typedef unsigned long pgdval_t; typedef unsigned long pgprotval_t; typedef unsigned long phys_addr_t; +typedef struct page *pgtable_t; + typedef struct { pteval_t pte; } pte_t; #define vmemmap ((struct page *)VMEMMAP_START) diff --git a/include/asm-x86/pgalloc_32.h b/include/asm-x86/pgalloc_32.h index bab12718a91..6bea6e5b5ee 100644 --- a/include/asm-x86/pgalloc_32.h +++ b/include/asm-x86/pgalloc_32.h @@ -31,6 +31,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *p paravirt_alloc_pt(mm, pfn); set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); } +#define pmd_pgtable(pmd) pmd_page(pmd) /* * Allocate and free page tables. @@ -39,15 +40,16 @@ extern pgd_t *pgd_alloc(struct mm_struct *); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); -extern struct page *pte_alloc_one(struct mm_struct *, unsigned long); +extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long); static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) { free_page((unsigned long)pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } diff --git a/include/asm-x86/pgalloc_64.h b/include/asm-x86/pgalloc_64.h index 4f6220db22b..8d6722320dc 100644 --- a/include/asm-x86/pgalloc_64.h +++ b/include/asm-x86/pgalloc_64.h @@ -12,6 +12,8 @@ #define pgd_populate(mm, pgd, pud) \ set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud))) +#define pmd_pgtable(pmd) pmd_page(pmd) + static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte) { set_pmd(pmd, __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT))); @@ -91,12 +93,17 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long ad return (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { - void *p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); + struct page *page; + void *p; + + p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); if (!p) return NULL; - return virt_to_page(p); + page = virt_to_page(p); + pgtable_page_ctor(page); + return page; } /* Should really implement gc for free page table pages. This could be @@ -108,12 +115,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) free_page((unsigned long)pte); } -static inline void pte_free(struct mm_struct *mm, struct page *pte) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { + pgtable_page_dtor(pte); __free_page(pte); } -#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb(tlb,pte) \ +do { \ + pgtable_page_dtor((pte)); \ + tlb_remove_page((tlb), (pte)); \ +} while (0) #define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x)) #define __pud_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x)) diff --git a/include/asm-xtensa/page.h b/include/asm-xtensa/page.h index 1adedbf41d0..80a6ae0dd25 100644 --- a/include/asm-xtensa/page.h +++ b/include/asm-xtensa/page.h @@ -98,6 +98,7 @@ typedef struct { unsigned long pte; } pte_t; /* page table entry */ typedef struct { unsigned long pgd; } pgd_t; /* PGD table entry */ typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct page *pgtable_t; #define pte_val(x) ((x).pte) #define pgd_val(x) ((x).pgd) diff --git a/include/asm-xtensa/pgalloc.h b/include/asm-xtensa/pgalloc.h index 1d51ba5463f..8d1544eb461 100644 --- a/include/asm-xtensa/pgalloc.h +++ b/include/asm-xtensa/pgalloc.h @@ -24,6 +24,7 @@ (pmd_val(*(pmdp)) = ((unsigned long)ptep)) #define pmd_populate(mm, pmdp, page) \ (pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page))) +#define pmd_pgtable(pmd) pmd_page(pmd) static inline pgd_t* pgd_alloc(struct mm_struct *mm) @@ -46,10 +47,14 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, return kmem_cache_alloc(pgtable_cache, GFP_KERNEL|__GFP_REPEAT); } -static inline struct page *pte_alloc_one(struct mm_struct *mm, - unsigned long addr) +static inline pte_token_t pte_alloc_one(struct mm_struct *mm, + unsigned long addr) { - return virt_to_page(pte_alloc_one_kernel(mm, addr)); + struct page *page; + + page = virt_to_page(pte_alloc_one_kernel(mm, addr)); + pgtable_page_ctor(page); + return page; } static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) @@ -57,10 +62,12 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) kmem_cache_free(pgtable_cache, pte); } -static inline void pte_free(struct mm_struct *mm, struct page *page) +static inline void pte_free(struct mm_struct *mm, pgtable_t pte) { - kmem_cache_free(pgtable_cache, page_address(page)); + pgtable_page_dtor(pte); + kmem_cache_free(pgtable_cache, page_address(pte)); } +#define pmd_pgtable(pmd) pmd_page(pmd) #endif /* __KERNEL__ */ #endif /* _XTENSA_PGALLOC_H */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 89d7c691b93..e8abb381420 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -894,6 +894,18 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a #define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;}) #endif /* NR_CPUS < CONFIG_SPLIT_PTLOCK_CPUS */ +static inline void pgtable_page_ctor(struct page *page) +{ + pte_lock_init(page); + inc_zone_page_state(page, NR_PAGETABLE); +} + +static inline void pgtable_page_dtor(struct page *page) +{ + pte_lock_deinit(page); + dec_zone_page_state(page, NR_PAGETABLE); +} + #define pte_offset_map_lock(mm, pmd, address, ptlp) \ ({ \ spinlock_t *__ptl = pte_lockptr(mm, pmd); \ @@ -1136,7 +1148,7 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address, #define FOLL_GET 0x04 /* do get_page on page */ #define FOLL_ANON 0x08 /* give ZERO_PAGE if no pgtable */ -typedef int (*pte_fn_t)(pte_t *pte, struct page *pmd_page, unsigned long addr, +typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, void *data); extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data); diff --git a/mm/memory.c b/mm/memory.c index 153a54b2013..e5628a5fd67 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -134,11 +134,9 @@ void pmd_clear_bad(pmd_t *pmd) */ static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd) { - struct page *page = pmd_page(*pmd); + pgtable_t token = pmd_pgtable(*pmd); pmd_clear(pmd); - pte_lock_deinit(page); - pte_free_tlb(tlb, page); - dec_zone_page_state(page, NR_PAGETABLE); + pte_free_tlb(tlb, token); tlb->mm->nr_ptes--; } @@ -309,21 +307,19 @@ void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma, int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address) { - struct page *new = pte_alloc_one(mm, address); + pgtable_t new = pte_alloc_one(mm, address); if (!new) return -ENOMEM; - pte_lock_init(new); spin_lock(&mm->page_table_lock); - if (pmd_present(*pmd)) { /* Another has populated it */ - pte_lock_deinit(new); - pte_free(mm, new); - } else { + if (!pmd_present(*pmd)) { /* Has another populated it ? */ mm->nr_ptes++; - inc_zone_page_state(new, NR_PAGETABLE); pmd_populate(mm, pmd, new); + new = NULL; } spin_unlock(&mm->page_table_lock); + if (new) + pte_free(mm, new); return 0; } @@ -334,11 +330,13 @@ int __pte_alloc_kernel(pmd_t *pmd, unsigned long address) return -ENOMEM; spin_lock(&init_mm.page_table_lock); - if (pmd_present(*pmd)) /* Another has populated it */ - pte_free_kernel(&init_mm, new); - else + if (!pmd_present(*pmd)) { /* Has another populated it ? */ pmd_populate_kernel(&init_mm, pmd, new); + new = NULL; + } spin_unlock(&init_mm.page_table_lock); + if (new) + pte_free_kernel(&init_mm, new); return 0; } @@ -1390,7 +1388,7 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, { pte_t *pte; int err; - struct page *pmd_page; + pgtable_t token; spinlock_t *uninitialized_var(ptl); pte = (mm == &init_mm) ? @@ -1401,10 +1399,10 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, BUG_ON(pmd_huge(*pmd)); - pmd_page = pmd_page(*pmd); + token = pmd_pgtable(*pmd); do { - err = fn(pte, pmd_page, addr, data); + err = fn(pte, token, addr, data); if (err) break; } while (pte++, addr += PAGE_SIZE, addr != end); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 0536dde139d..950c0be9ca8 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -820,7 +820,7 @@ void __attribute__((weak)) vmalloc_sync_all(void) } -static int f(pte_t *pte, struct page *pmd_page, unsigned long addr, void *data) +static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data) { /* apply_to_page_range() does all the hard work. */ return 0; |