diff options
Diffstat (limited to 'arch/arm/include/asm')
| -rw-r--r-- | arch/arm/include/asm/hardware/cache-l2x0.h | 1 | ||||
| -rw-r--r-- | arch/arm/include/asm/hardware/sp810.h | 3 | ||||
| -rw-r--r-- | arch/arm/include/asm/io.h | 33 | ||||
| -rw-r--r-- | arch/arm/include/asm/mach/arch.h | 4 | ||||
| -rw-r--r-- | arch/arm/include/asm/pgalloc.h | 2 | ||||
| -rw-r--r-- | arch/arm/include/asm/tlb.h | 105 | ||||
| -rw-r--r-- | arch/arm/include/asm/tlbflush.h | 7 |
7 files changed, 116 insertions, 39 deletions
diff --git a/arch/arm/include/asm/hardware/cache-l2x0.h b/arch/arm/include/asm/hardware/cache-l2x0.h index 5aeec1e1735..16bd4803158 100644 --- a/arch/arm/include/asm/hardware/cache-l2x0.h +++ b/arch/arm/include/asm/hardware/cache-l2x0.h @@ -36,6 +36,7 @@ #define L2X0_RAW_INTR_STAT 0x21C #define L2X0_INTR_CLEAR 0x220 #define L2X0_CACHE_SYNC 0x730 +#define L2X0_DUMMY_REG 0x740 #define L2X0_INV_LINE_PA 0x770 #define L2X0_INV_WAY 0x77C #define L2X0_CLEAN_LINE_PA 0x7B0 diff --git a/arch/arm/include/asm/hardware/sp810.h b/arch/arm/include/asm/hardware/sp810.h index 721847dc68a..e0d1c0cfa54 100644 --- a/arch/arm/include/asm/hardware/sp810.h +++ b/arch/arm/include/asm/hardware/sp810.h @@ -58,6 +58,9 @@ static inline void sysctl_soft_reset(void __iomem *base) { + /* switch to slow mode */ + writel(0x2, base + SCCTRL); + /* writing any value to SCSYSSTAT reg will reset system */ writel(0, base + SCSYSSTAT); } diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h index 20e0f7c9e03..d66605dea55 100644 --- a/arch/arm/include/asm/io.h +++ b/arch/arm/include/asm/io.h @@ -95,6 +95,15 @@ static inline void __iomem *__typesafe_io(unsigned long addr) return (void __iomem *)addr; } +/* IO barriers */ +#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE +#define __iormb() rmb() +#define __iowmb() wmb() +#else +#define __iormb() do { } while (0) +#define __iowmb() do { } while (0) +#endif + /* * Now, pick up the machine-defined IO definitions */ @@ -125,17 +134,17 @@ static inline void __iomem *__typesafe_io(unsigned long addr) * The {in,out}[bwl] macros are for emulating x86-style PCI/ISA IO space. */ #ifdef __io -#define outb(v,p) __raw_writeb(v,__io(p)) -#define outw(v,p) __raw_writew((__force __u16) \ - cpu_to_le16(v),__io(p)) -#define outl(v,p) __raw_writel((__force __u32) \ - cpu_to_le32(v),__io(p)) +#define outb(v,p) ({ __iowmb(); __raw_writeb(v,__io(p)); }) +#define outw(v,p) ({ __iowmb(); __raw_writew((__force __u16) \ + cpu_to_le16(v),__io(p)); }) +#define outl(v,p) ({ __iowmb(); __raw_writel((__force __u32) \ + cpu_to_le32(v),__io(p)); }) -#define inb(p) ({ __u8 __v = __raw_readb(__io(p)); __v; }) +#define inb(p) ({ __u8 __v = __raw_readb(__io(p)); __iormb(); __v; }) #define inw(p) ({ __u16 __v = le16_to_cpu((__force __le16) \ - __raw_readw(__io(p))); __v; }) + __raw_readw(__io(p))); __iormb(); __v; }) #define inl(p) ({ __u32 __v = le32_to_cpu((__force __le32) \ - __raw_readl(__io(p))); __v; }) + __raw_readl(__io(p))); __iormb(); __v; }) #define outsb(p,d,l) __raw_writesb(__io(p),d,l) #define outsw(p,d,l) __raw_writesw(__io(p),d,l) @@ -192,14 +201,6 @@ extern void _memset_io(volatile void __iomem *, int, size_t); #define writel_relaxed(v,c) ((void)__raw_writel((__force u32) \ cpu_to_le32(v),__mem_pci(c))) -#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE -#define __iormb() rmb() -#define __iowmb() wmb() -#else -#define __iormb() do { } while (0) -#define __iowmb() do { } while (0) -#endif - #define readb(c) ({ u8 __v = readb_relaxed(c); __iormb(); __v; }) #define readw(c) ({ u16 __v = readw_relaxed(c); __iormb(); __v; }) #define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(); __v; }) diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index 3a0893a76a3..bf13b814c1b 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -15,10 +15,6 @@ struct meminfo; struct sys_timer; struct machine_desc { - /* - * Note! The first two elements are used - * by assembler code in head.S, head-common.S - */ unsigned int nr; /* architecture number */ const char *name; /* architecture name */ unsigned long boot_params; /* tagged list */ diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h index 9763be04f77..22de005f159 100644 --- a/arch/arm/include/asm/pgalloc.h +++ b/arch/arm/include/asm/pgalloc.h @@ -10,6 +10,8 @@ #ifndef _ASMARM_PGALLOC_H #define _ASMARM_PGALLOC_H +#include <linux/pagemap.h> + #include <asm/domain.h> #include <asm/pgtable-hwdef.h> #include <asm/processor.h> diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index f41a6f57cd1..82dfe5d0c41 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -18,16 +18,34 @@ #define __ASMARM_TLB_H #include <asm/cacheflush.h> -#include <asm/tlbflush.h> #ifndef CONFIG_MMU #include <linux/pagemap.h> + +#define tlb_flush(tlb) ((void) tlb) + #include <asm-generic/tlb.h> #else /* !CONFIG_MMU */ +#include <linux/swap.h> #include <asm/pgalloc.h> +#include <asm/tlbflush.h> + +/* + * We need to delay page freeing for SMP as other CPUs can access pages + * which have been removed but not yet had their TLB entries invalidated. + * Also, as ARMv7 speculative prefetch can drag new entries into the TLB, + * we need to apply this same delaying tactic to ensure correct operation. + */ +#if defined(CONFIG_SMP) || defined(CONFIG_CPU_32v7) +#define tlb_fast_mode(tlb) 0 +#define FREE_PTE_NR 500 +#else +#define tlb_fast_mode(tlb) 1 +#define FREE_PTE_NR 0 +#endif /* * TLB handling. This allows us to remove pages from the page @@ -36,12 +54,58 @@ struct mmu_gather { struct mm_struct *mm; unsigned int fullmm; + struct vm_area_struct *vma; unsigned long range_start; unsigned long range_end; + unsigned int nr; + struct page *pages[FREE_PTE_NR]; }; DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); +/* + * This is unnecessarily complex. There's three ways the TLB shootdown + * code is used: + * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). + * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. + * tlb->vma will be non-NULL. + * 2. Unmapping all vmas. See exit_mmap(). + * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. + * tlb->vma will be non-NULL. Additionally, page tables will be freed. + * 3. Unmapping argument pages. See shift_arg_pages(). + * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. + * tlb->vma will be NULL. + */ +static inline void tlb_flush(struct mmu_gather *tlb) +{ + if (tlb->fullmm || !tlb->vma) + flush_tlb_mm(tlb->mm); + else if (tlb->range_end > 0) { + flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end); + tlb->range_start = TASK_SIZE; + tlb->range_end = 0; + } +} + +static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) +{ + if (!tlb->fullmm) { + if (addr < tlb->range_start) + tlb->range_start = addr; + if (addr + PAGE_SIZE > tlb->range_end) + tlb->range_end = addr + PAGE_SIZE; + } +} + +static inline void tlb_flush_mmu(struct mmu_gather *tlb) +{ + tlb_flush(tlb); + if (!tlb_fast_mode(tlb)) { + free_pages_and_swap_cache(tlb->pages, tlb->nr); + tlb->nr = 0; + } +} + static inline struct mmu_gather * tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) { @@ -49,6 +113,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) tlb->mm = mm; tlb->fullmm = full_mm_flush; + tlb->vma = NULL; + tlb->nr = 0; return tlb; } @@ -56,8 +122,7 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) static inline void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) { - if (tlb->fullmm) - flush_tlb_mm(tlb->mm); + tlb_flush_mmu(tlb); /* keep the page table cache within bounds */ check_pgt_cache(); @@ -71,12 +136,7 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) static inline void tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr) { - if (!tlb->fullmm) { - if (addr < tlb->range_start) - tlb->range_start = addr; - if (addr + PAGE_SIZE > tlb->range_end) - tlb->range_end = addr + PAGE_SIZE; - } + tlb_add_flush(tlb, addr); } /* @@ -89,6 +149,7 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { if (!tlb->fullmm) { flush_cache_range(vma, vma->vm_start, vma->vm_end); + tlb->vma = vma; tlb->range_start = TASK_SIZE; tlb->range_end = 0; } @@ -97,12 +158,30 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { - if (!tlb->fullmm && tlb->range_end > 0) - flush_tlb_range(vma, tlb->range_start, tlb->range_end); + if (!tlb->fullmm) + tlb_flush(tlb); +} + +static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) +{ + if (tlb_fast_mode(tlb)) { + free_page_and_swap_cache(page); + } else { + tlb->pages[tlb->nr++] = page; + if (tlb->nr >= FREE_PTE_NR) + tlb_flush_mmu(tlb); + } +} + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, + unsigned long addr) +{ + pgtable_page_dtor(pte); + tlb_add_flush(tlb, addr); + tlb_remove_page(tlb, pte); } -#define tlb_remove_page(tlb,page) free_page_and_swap_cache(page) -#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep) +#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) #define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) #define tlb_migrate_finish(mm) do { } while (0) diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index ce7378ea15a..d2005de383b 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -10,12 +10,7 @@ #ifndef _ASMARM_TLBFLUSH_H #define _ASMARM_TLBFLUSH_H - -#ifndef CONFIG_MMU - -#define tlb_flush(tlb) ((void) tlb) - -#else /* CONFIG_MMU */ +#ifdef CONFIG_MMU #include <asm/glue.h> |