diff options
| author | Zachary Amsden <zach@vmware.com> | 2007-05-02 19:27:19 +0200 |
|---|---|---|
| committer | Andi Kleen <andi@basil.nowhere.org> | 2007-05-02 19:27:19 +0200 |
| commit | 142dd975911fdd82b1b6f6617cd20ac90a8ccf00 (patch) | |
| tree | 1e72e96bce64d9301d3305b4dd0b30675c8676b0 /include/asm-i386 | |
| parent | c2c1accd4b2f9c82fb89d40611c7f581948db255 (diff) | |
[PATCH] i386: pte xchg optimization
In situations where page table updates need only be made locally, and there is
no cross-processor A/D bit races involved, we need not use the heavyweight
xchg instruction to atomically fetch and clear page table entries. Instead,
we can just read and clear them directly.
This introduces a neat optimization for non-SMP kernels; drop the atomic xchg
operations from page table updates.
Thanks to Michel Lespinasse for noting this potential optimization.
Signed-off-by: Zachary Amsden <zach@vmware.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Andi Kleen <ak@suse.de>
Diffstat (limited to 'include/asm-i386')
| -rw-r--r-- | include/asm-i386/pgtable-2level.h | 14 | ||||
| -rw-r--r-- | include/asm-i386/pgtable-3level.h | 14 |
2 files changed, 28 insertions, 0 deletions
diff --git a/include/asm-i386/pgtable-2level.h b/include/asm-i386/pgtable-2level.h index 85d9005c0cd..3daab67cd36 100644 --- a/include/asm-i386/pgtable-2level.h +++ b/include/asm-i386/pgtable-2level.h @@ -41,10 +41,24 @@ static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, pt *xp = __pte(0); } +/* local pte updates need not use xchg for locking */ +static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) +{ + pte_t res; + + res = *ptep; + native_pte_clear(NULL, 0, ptep); + return res; +} + +#ifdef CONFIG_SMP static inline pte_t native_ptep_get_and_clear(pte_t *xp) { return __pte(xchg(&xp->pte_low, 0)); } +#else +#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp) +#endif #define pte_page(x) pfn_to_page(pte_pfn(x)) #define pte_none(x) (!(x).pte_low) diff --git a/include/asm-i386/pgtable-3level.h b/include/asm-i386/pgtable-3level.h index 664bfee5a2f..45b02418150 100644 --- a/include/asm-i386/pgtable-3level.h +++ b/include/asm-i386/pgtable-3level.h @@ -139,6 +139,17 @@ static inline void pud_clear (pud_t * pud) { } #define pmd_offset(pud, address) ((pmd_t *) pud_page(*(pud)) + \ pmd_index(address)) +/* local pte updates need not use xchg for locking */ +static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) +{ + pte_t res; + + res = *ptep; + native_pte_clear(NULL, 0, ptep); + return res; +} + +#ifdef CONFIG_SMP static inline pte_t native_ptep_get_and_clear(pte_t *ptep) { pte_t res; @@ -150,6 +161,9 @@ static inline pte_t native_ptep_get_and_clear(pte_t *ptep) return res; } +#else +#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp) +#endif #define __HAVE_ARCH_PTE_SAME static inline int pte_same(pte_t a, pte_t b) |