diff options
Diffstat (limited to 'arch/arm/include')
| -rw-r--r-- | arch/arm/include/asm/cacheflush.h | 133 | ||||
| -rw-r--r-- | arch/arm/include/asm/cpu-multi32.h | 69 | ||||
| -rw-r--r-- | arch/arm/include/asm/cpu-single.h | 44 | ||||
| -rw-r--r-- | arch/arm/include/asm/fncpy.h | 94 | ||||
| -rw-r--r-- | arch/arm/include/asm/glue-cache.h | 146 | ||||
| -rw-r--r-- | arch/arm/include/asm/glue-df.h | 110 | ||||
| -rw-r--r-- | arch/arm/include/asm/glue-pf.h | 57 | ||||
| -rw-r--r-- | arch/arm/include/asm/glue-proc.h | 264 | ||||
| -rw-r--r-- | arch/arm/include/asm/glue.h | 138 | ||||
| -rw-r--r-- | arch/arm/include/asm/hardware/cache-l2x0.h | 1 | ||||
| -rw-r--r-- | arch/arm/include/asm/hardware/sp810.h | 3 | ||||
| -rw-r--r-- | arch/arm/include/asm/mach/arch.h | 4 | ||||
| -rw-r--r-- | arch/arm/include/asm/pgalloc.h | 2 | ||||
| -rw-r--r-- | arch/arm/include/asm/proc-fns.h | 306 | ||||
| -rw-r--r-- | arch/arm/include/asm/processor.h | 2 | ||||
| -rw-r--r-- | arch/arm/include/asm/smp_scu.h | 7 | ||||
| -rw-r--r-- | arch/arm/include/asm/tlb.h | 105 | ||||
| -rw-r--r-- | arch/arm/include/asm/tlbflush.h | 7 |
18 files changed, 851 insertions, 641 deletions
diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index 3acd8fa25e3..18a56640d97 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -12,7 +12,7 @@ #include <linux/mm.h> -#include <asm/glue.h> +#include <asm/glue-cache.h> #include <asm/shmparam.h> #include <asm/cachetype.h> #include <asm/outercache.h> @@ -20,123 +20,6 @@ #define CACHE_COLOUR(vaddr) ((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT) /* - * Cache Model - * =========== - */ -#undef _CACHE -#undef MULTI_CACHE - -#if defined(CONFIG_CPU_CACHE_V3) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE v3 -# endif -#endif - -#if defined(CONFIG_CPU_CACHE_V4) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE v4 -# endif -#endif - -#if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \ - defined(CONFIG_CPU_ARM925T) || defined(CONFIG_CPU_ARM1020) || \ - defined(CONFIG_CPU_ARM1026) -# define MULTI_CACHE 1 -#endif - -#if defined(CONFIG_CPU_FA526) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE fa -# endif -#endif - -#if defined(CONFIG_CPU_ARM926T) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE arm926 -# endif -#endif - -#if defined(CONFIG_CPU_ARM940T) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE arm940 -# endif -#endif - -#if defined(CONFIG_CPU_ARM946E) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE arm946 -# endif -#endif - -#if defined(CONFIG_CPU_CACHE_V4WB) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE v4wb -# endif -#endif - -#if defined(CONFIG_CPU_XSCALE) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE xscale -# endif -#endif - -#if defined(CONFIG_CPU_XSC3) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE xsc3 -# endif -#endif - -#if defined(CONFIG_CPU_MOHAWK) -# ifdef _CACHE -# define MULTI_CACHE 1 -# else -# define _CACHE mohawk -# endif -#endif - -#if defined(CONFIG_CPU_FEROCEON) -# define MULTI_CACHE 1 -#endif - -#if defined(CONFIG_CPU_V6) -//# ifdef _CACHE -# define MULTI_CACHE 1 -//# else -//# define _CACHE v6 -//# endif -#endif - -#if defined(CONFIG_CPU_V7) -//# ifdef _CACHE -# define MULTI_CACHE 1 -//# else -//# define _CACHE v7 -//# endif -#endif - -#if !defined(_CACHE) && !defined(MULTI_CACHE) -#error Unknown cache maintainence model -#endif - -/* * This flag is used to indicate that the page pointed to by a pte is clean * and does not require cleaning before returning it to the user. */ @@ -249,19 +132,11 @@ extern struct cpu_cache_fns cpu_cache; * visible to the CPU. */ #define dmac_map_area cpu_cache.dma_map_area -#define dmac_unmap_area cpu_cache.dma_unmap_area +#define dmac_unmap_area cpu_cache.dma_unmap_area #define dmac_flush_range cpu_cache.dma_flush_range #else -#define __cpuc_flush_icache_all __glue(_CACHE,_flush_icache_all) -#define __cpuc_flush_kern_all __glue(_CACHE,_flush_kern_cache_all) -#define __cpuc_flush_user_all __glue(_CACHE,_flush_user_cache_all) -#define __cpuc_flush_user_range __glue(_CACHE,_flush_user_cache_range) -#define __cpuc_coherent_kern_range __glue(_CACHE,_coherent_kern_range) -#define __cpuc_coherent_user_range __glue(_CACHE,_coherent_user_range) -#define __cpuc_flush_dcache_area __glue(_CACHE,_flush_kern_dcache_area) - extern void __cpuc_flush_icache_all(void); extern void __cpuc_flush_kern_all(void); extern void __cpuc_flush_user_all(void); @@ -276,10 +151,6 @@ extern void __cpuc_flush_dcache_area(void *, size_t); * is visible to DMA, or data written by DMA to system memory is * visible to the CPU. */ -#define dmac_map_area __glue(_CACHE,_dma_map_area) -#define dmac_unmap_area __glue(_CACHE,_dma_unmap_area) -#define dmac_flush_range __glue(_CACHE,_dma_flush_range) - extern void dmac_map_area(const void *, size_t, int); extern void dmac_unmap_area(const void *, size_t, int); extern void dmac_flush_range(const void *, const void *); diff --git a/arch/arm/include/asm/cpu-multi32.h b/arch/arm/include/asm/cpu-multi32.h deleted file mode 100644 index e2b5b0b2116..00000000000 --- a/arch/arm/include/asm/cpu-multi32.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * arch/arm/include/asm/cpu-multi32.h - * - * Copyright (C) 2000 Russell King - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#include <asm/page.h> - -struct mm_struct; - -/* - * Don't change this structure - ASM code - * relies on it. - */ -extern struct processor { - /* MISC - * get data abort address/flags - */ - void (*_data_abort)(unsigned long pc); - /* - * Retrieve prefetch fault address - */ - unsigned long (*_prefetch_abort)(unsigned long lr); - /* - * Set up any processor specifics - */ - void (*_proc_init)(void); - /* - * Disable any processor specifics - */ - void (*_proc_fin)(void); - /* - * Special stuff for a reset - */ - void (*reset)(unsigned long addr) __attribute__((noreturn)); - /* - * Idle the processor - */ - int (*_do_idle)(void); - /* - * Processor architecture specific - */ - /* - * clean a virtual address range from the - * D-cache without flushing the cache. - */ - void (*dcache_clean_area)(void *addr, int size); - - /* - * Set the page table - */ - void (*switch_mm)(unsigned long pgd_phys, struct mm_struct *mm); - /* - * Set a possibly extended PTE. Non-extended PTEs should - * ignore 'ext'. - */ - void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext); -} processor; - -#define cpu_proc_init() processor._proc_init() -#define cpu_proc_fin() processor._proc_fin() -#define cpu_reset(addr) processor.reset(addr) -#define cpu_do_idle() processor._do_idle() -#define cpu_dcache_clean_area(addr,sz) processor.dcache_clean_area(addr,sz) -#define cpu_set_pte_ext(ptep,pte,ext) processor.set_pte_ext(ptep,pte,ext) -#define cpu_do_switch_mm(pgd,mm) processor.switch_mm(pgd,mm) diff --git a/arch/arm/include/asm/cpu-single.h b/arch/arm/include/asm/cpu-single.h deleted file mode 100644 index f073a6d2a40..00000000000 --- a/arch/arm/include/asm/cpu-single.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * arch/arm/include/asm/cpu-single.h - * - * Copyright (C) 2000 Russell King - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -/* - * Single CPU - */ -#ifdef __STDC__ -#define __catify_fn(name,x) name##x -#else -#define __catify_fn(name,x) name/**/x -#endif -#define __cpu_fn(name,x) __catify_fn(name,x) - -/* - * If we are supporting multiple CPUs, then we must use a table of - * function pointers for this lot. Otherwise, we can optimise the - * table away. - */ -#define cpu_proc_init __cpu_fn(CPU_NAME,_proc_init) -#define cpu_proc_fin __cpu_fn(CPU_NAME,_proc_fin) -#define cpu_reset __cpu_fn(CPU_NAME,_reset) -#define cpu_do_idle __cpu_fn(CPU_NAME,_do_idle) -#define cpu_dcache_clean_area __cpu_fn(CPU_NAME,_dcache_clean_area) -#define cpu_do_switch_mm __cpu_fn(CPU_NAME,_switch_mm) -#define cpu_set_pte_ext __cpu_fn(CPU_NAME,_set_pte_ext) - -#include <asm/page.h> - -struct mm_struct; - -/* declare all the functions as extern */ -extern void cpu_proc_init(void); -extern void cpu_proc_fin(void); -extern int cpu_do_idle(void); -extern void cpu_dcache_clean_area(void *, int); -extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); -extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); -extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); diff --git a/arch/arm/include/asm/fncpy.h b/arch/arm/include/asm/fncpy.h new file mode 100644 index 00000000000..de535474692 --- /dev/null +++ b/arch/arm/include/asm/fncpy.h @@ -0,0 +1,94 @@ +/* + * arch/arm/include/asm/fncpy.h - helper macros for function body copying + * + * Copyright (C) 2011 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * These macros are intended for use when there is a need to copy a low-level + * function body into special memory. + * + * For example, when reconfiguring the SDRAM controller, the code doing the + * reconfiguration may need to run from SRAM. + * + * NOTE: that the copied function body must be entirely self-contained and + * position-independent in order for this to work properly. + * + * NOTE: in order for embedded literals and data to get referenced correctly, + * the alignment of functions must be preserved when copying. To ensure this, + * the source and destination addresses for fncpy() must be aligned to a + * multiple of 8 bytes: you will be get a BUG() if this condition is not met. + * You will typically need a ".align 3" directive in the assembler where the + * function to be copied is defined, and ensure that your allocator for the + * destination buffer returns 8-byte-aligned pointers. + * + * Typical usage example: + * + * extern int f(args); + * extern uint32_t size_of_f; + * int (*copied_f)(args); + * void *sram_buffer; + * + * copied_f = fncpy(sram_buffer, &f, size_of_f); + * + * ... later, call the function: ... + * + * copied_f(args); + * + * The size of the function to be copied can't be determined from C: + * this must be determined by other means, such as adding assmbler directives + * in the file where f is defined. + */ + +#ifndef __ASM_FNCPY_H +#define __ASM_FNCPY_H + +#include <linux/types.h> +#include <linux/string.h> + +#include <asm/bug.h> +#include <asm/cacheflush.h> + +/* + * Minimum alignment requirement for the source and destination addresses + * for function copying. + */ +#define FNCPY_ALIGN 8 + +#define fncpy(dest_buf, funcp, size) ({ \ + uintptr_t __funcp_address; \ + typeof(funcp) __result; \ + \ + asm("" : "=r" (__funcp_address) : "0" (funcp)); \ + \ + /* \ + * Ensure alignment of source and destination addresses, \ + * disregarding the function's Thumb bit: \ + */ \ + BUG_ON((uintptr_t)(dest_buf) & (FNCPY_ALIGN - 1) || \ + (__funcp_address & ~(uintptr_t)1 & (FNCPY_ALIGN - 1))); \ + \ + memcpy(dest_buf, (void const *)(__funcp_address & ~1), size); \ + flush_icache_range((unsigned long)(dest_buf), \ + (unsigned long)(dest_buf) + (size)); \ + \ + asm("" : "=r" (__result) \ + : "0" ((uintptr_t)(dest_buf) | (__funcp_address & 1))); \ + \ + __result; \ +}) + +#endif /* !__ASM_FNCPY_H */ diff --git a/arch/arm/include/asm/glue-cache.h b/arch/arm/include/asm/glue-cache.h new file mode 100644 index 00000000000..0591d35001e --- /dev/null +++ b/arch/arm/include/asm/glue-cache.h @@ -0,0 +1,146 @@ +/* + * arch/arm/include/asm/glue-cache.h + * + * Copyright (C) 1999-2002 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef ASM_GLUE_CACHE_H +#define ASM_GLUE_CACHE_H + +#include <asm/glue.h> + +/* + * Cache Model + * =========== + */ +#undef _CACHE +#undef MULTI_CACHE + +#if defined(CONFIG_CPU_CACHE_V3) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE v3 +# endif +#endif + +#if defined(CONFIG_CPU_CACHE_V4) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE v4 +# endif +#endif + +#if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \ + defined(CONFIG_CPU_ARM925T) || defined(CONFIG_CPU_ARM1020) || \ + defined(CONFIG_CPU_ARM1026) +# define MULTI_CACHE 1 +#endif + +#if defined(CONFIG_CPU_FA526) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE fa +# endif +#endif + +#if defined(CONFIG_CPU_ARM926T) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE arm926 +# endif +#endif + +#if defined(CONFIG_CPU_ARM940T) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE arm940 +# endif +#endif + +#if defined(CONFIG_CPU_ARM946E) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE arm946 +# endif +#endif + +#if defined(CONFIG_CPU_CACHE_V4WB) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE v4wb +# endif +#endif + +#if defined(CONFIG_CPU_XSCALE) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE xscale +# endif +#endif + +#if defined(CONFIG_CPU_XSC3) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE xsc3 +# endif +#endif + +#if defined(CONFIG_CPU_MOHAWK) +# ifdef _CACHE +# define MULTI_CACHE 1 +# else +# define _CACHE mohawk +# endif +#endif + +#if defined(CONFIG_CPU_FEROCEON) +# define MULTI_CACHE 1 +#endif + +#if defined(CONFIG_CPU_V6) +//# ifdef _CACHE +# define MULTI_CACHE 1 +//# else +//# define _CACHE v6 +//# endif +#endif + +#if defined(CONFIG_CPU_V7) +//# ifdef _CACHE +# define MULTI_CACHE 1 +//# else +//# define _CACHE v7 +//# endif +#endif + +#if !defined(_CACHE) && !defined(MULTI_CACHE) +#error Unknown cache maintainence model +#endif + +#ifndef MULTI_CACHE +#define __cpuc_flush_icache_all __glue(_CACHE,_flush_icache_all) +#define __cpuc_flush_kern_all __glue(_CACHE,_flush_kern_cache_all) +#define __cpuc_flush_user_all __glue(_CACHE,_flush_user_cache_all) +#define __cpuc_flush_user_range __glue(_CACHE,_flush_user_cache_range) +#define __cpuc_coherent_kern_range __glue(_CACHE,_coherent_kern_range) +#define __cpuc_coherent_user_range __glue(_CACHE,_coherent_user_range) +#define __cpuc_flush_dcache_area __glue(_CACHE,_flush_kern_dcache_area) + +#define dmac_map_area __glue(_CACHE,_dma_map_area) +#define dmac_unmap_area __glue(_CACHE,_dma_unmap_area) +#define dmac_flush_range __glue(_CACHE,_dma_flush_range) +#endif + +#endif diff --git a/arch/arm/include/asm/glue-df.h b/arch/arm/include/asm/glue-df.h new file mode 100644 index 00000000000..354d571e8bc --- /dev/null +++ b/arch/arm/include/asm/glue-df.h @@ -0,0 +1,110 @@ +/* + * arch/arm/include/asm/glue-df.h + * + * Copyright (C) 1997-1999 Russell King + * Copyright (C) 2000-2002 Deep Blue Solutions Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef ASM_GLUE_DF_H +#define ASM_GLUE_DF_H + +#include <asm/glue.h> + +/* + * Data Abort Model + * ================ + * + * We have the following to choose from: + * arm6 - ARM6 style + * arm7 - ARM7 style + * v4_early - ARMv4 without Thumb early abort handler + * v4t_late - ARMv4 with Thumb late abort handler + * v4t_early - ARMv4 with Thumb early abort handler + * v5tej_early - ARMv5 with Thumb and Java early abort handler + * xscale - ARMv5 with Thumb with Xscale extensions + * v6_early - ARMv6 generic early abort handler + * v7_early - ARMv7 generic early abort handler + */ +#undef CPU_DABORT_HANDLER +#undef MULTI_DABORT + +#if defined(CONFIG_CPU_ARM610) +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER cpu_arm6_data_abort +# endif +#endif + +#if defined(CONFIG_CPU_ARM710) +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER cpu_arm7_data_abort +# endif +#endif + +#ifdef CONFIG_CPU_ABRT_LV4T +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER v4t_late_abort +# endif +#endif + +#ifdef CONFIG_CPU_ABRT_EV4 +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER v4_early_abort +# endif +#endif + +#ifdef CONFIG_CPU_ABRT_EV4T +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER v4t_early_abort +# endif +#endif + +#ifdef CONFIG_CPU_ABRT_EV5TJ +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER v5tj_early_abort +# endif +#endif + +#ifdef CONFIG_CPU_ABRT_EV5T +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER v5t_early_abort +# endif +#endif + +#ifdef CONFIG_CPU_ABRT_EV6 +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER v6_early_abort +# endif +#endif + +#ifdef CONFIG_CPU_ABRT_EV7 +# ifdef CPU_DABORT_HANDLER +# define MULTI_DABORT 1 +# else +# define CPU_DABORT_HANDLER v7_early_abort +# endif +#endif + +#ifndef CPU_DABORT_HANDLER +#error Unknown data abort handler type +#endif + +#endif diff --git a/arch/arm/include/asm/glue-pf.h b/arch/arm/include/asm/glue-pf.h new file mode 100644 index 00000000000..d385f37c13f --- /dev/null +++ b/arch/arm/include/asm/glue-pf.h @@ -0,0 +1,57 @@ +/* + * arch/arm/include/asm/glue-pf.h + * + * Copyright (C) 1997-1999 Russell King + * Copyright (C) 2000-2002 Deep Blue Solutions Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef ASM_GLUE_PF_H +#define ASM_GLUE_PF_H + +#include <asm/glue.h> + +/* + * Prefetch Abort Model + * ================ + * + * We have the following to choose from: + * legacy - no IFSR, no IFAR + * v6 - ARMv6: IFSR, no IFAR + * v7 - ARMv7: IFSR and IFAR + */ + +#undef CPU_PABORT_HANDLER +#undef MULTI_PABORT + +#ifdef CONFIG_CPU_PABRT_LEGACY +# ifdef CPU_PABORT_HANDLER +# define MULTI_PABORT 1 +# else +# define CPU_PABORT_HANDLER legacy_pabort +# endif +#endif + +#ifdef CONFIG_CPU_PABRT_V6 +# ifdef CPU_PABORT_HANDLER +# define MULTI_PABORT 1 +# else +# define CPU_PABORT_HANDLER v6_pabort +# endif +#endif + +#ifdef CONFIG_CPU_PABRT_V7 +# ifdef CPU_PABORT_HANDLER +# define MULTI_PABORT 1 +# else +# define CPU_PABORT_HANDLER v7_pabort +# endif +#endif + +#ifndef CPU_PABORT_HANDLER +#error Unknown prefetch abort handler type +#endif + +#endif diff --git a/arch/arm/include/asm/glue-proc.h b/arch/arm/include/asm/glue-proc.h new file mode 100644 index 00000000000..6469521d092 --- /dev/null +++ b/arch/arm/include/asm/glue-proc.h @@ -0,0 +1,264 @@ +/* + * arch/arm/include/asm/glue-proc.h + * + * Copyright (C) 1997-1999 Russell King + * Copyright (C) 2000 Deep Blue Solutions Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef ASM_GLUE_PROC_H +#define ASM_GLUE_PROC_H + +#include <asm/glue.h> + +/* + * Work out if we need multiple CPU support + */ +#undef MULTI_CPU +#undef CPU_NAME + +/* + * CPU_NAME - the prefix for CPU related functions + */ + +#ifdef CONFIG_CPU_ARM610 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm6 +# endif +#endif + +#ifdef CONFIG_CPU_ARM7TDMI +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm7tdmi +# endif +#endif + +#ifdef CONFIG_CPU_ARM710 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm7 +# endif +#endif + +#ifdef CONFIG_CPU_ARM720T +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm720 +# endif +#endif + +#ifdef CONFIG_CPU_ARM740T +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm740 +# endif +#endif + +#ifdef CONFIG_CPU_ARM9TDMI +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm9tdmi +# endif +#endif + +#ifdef CONFIG_CPU_ARM920T +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm920 +# endif +#endif + +#ifdef CONFIG_CPU_ARM922T +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm922 +# endif +#endif + +#ifdef CONFIG_CPU_FA526 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_fa526 +# endif +#endif + +#ifdef CONFIG_CPU_ARM925T +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm925 +# endif +#endif + +#ifdef CONFIG_CPU_ARM926T +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm926 +# endif +#endif + +#ifdef CONFIG_CPU_ARM940T +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm940 +# endif +#endif + +#ifdef CONFIG_CPU_ARM946E +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm946 +# endif +#endif + +#ifdef CONFIG_CPU_SA110 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_sa110 +# endif +#endif + +#ifdef CONFIG_CPU_SA1100 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_sa1100 +# endif +#endif + +#ifdef CONFIG_CPU_ARM1020 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm1020 +# endif +#endif + +#ifdef CONFIG_CPU_ARM1020E +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm1020e +# endif +#endif + +#ifdef CONFIG_CPU_ARM1022 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm1022 +# endif +#endif + +#ifdef CONFIG_CPU_ARM1026 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_arm1026 +# endif +#endif + +#ifdef CONFIG_CPU_XSCALE +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_xscale +# endif +#endif + +#ifdef CONFIG_CPU_XSC3 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_xsc3 +# endif +#endif + +#ifdef CONFIG_CPU_MOHAWK +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_mohawk +# endif +#endif + +#ifdef CONFIG_CPU_FEROCEON +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_feroceon +# endif +#endif + +#ifdef CONFIG_CPU_V6 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_v6 +# endif +#endif + +#ifdef CONFIG_CPU_V7 +# ifdef CPU_NAME +# undef MULTI_CPU +# define MULTI_CPU +# else +# define CPU_NAME cpu_v7 +# endif +#endif + +#ifndef MULTI_CPU +#define cpu_proc_init __glue(CPU_NAME,_proc_init) +#define cpu_proc_fin __glue(CPU_NAME,_proc_fin) +#define cpu_reset __glue(CPU_NAME,_reset) +#define cpu_do_idle __glue(CPU_NAME,_do_idle) +#define cpu_dcache_clean_area __glue(CPU_NAME,_dcache_clean_area) +#define cpu_do_switch_mm __glue(CPU_NAME,_switch_mm) +#define cpu_set_pte_ext __glue(CPU_NAME,_set_pte_ext) +#define cpu_suspend_size __glue(CPU_NAME,_suspend_size) +#define cpu_do_suspend __glue(CPU_NAME,_do_suspend) +#define cpu_do_resume __glue(CPU_NAME,_do_resume) +#endif + +#endif diff --git a/arch/arm/include/asm/glue.h b/arch/arm/include/asm/glue.h index 234a3fc1c78..0ec35d1698a 100644 --- a/arch/arm/include/asm/glue.h +++ b/arch/arm/include/asm/glue.h @@ -15,7 +15,6 @@ */ #ifdef __KERNEL__ - #ifdef __STDC__ #define ____glue(name,fn) name##fn #else @@ -23,141 +22,4 @@ #endif #define __glue(name,fn) ____glue(name,fn) - - -/* - * Data Abort Model - * ================ - * - * We have the following to choose from: - * arm6 - ARM6 style - * arm7 - ARM7 style - * v4_early - ARMv4 without Thumb early abort handler - * v4t_late - ARMv4 with Thumb late abort handler - * v4t_early - ARMv4 with Thumb early abort handler - * v5tej_early - ARMv5 with Thumb and Java early abort handler - * xscale - ARMv5 with Thumb with Xscale extensions - * v6_early - ARMv6 generic early abort handler - * v7_early - ARMv7 generic early abort handler - */ -#undef CPU_DABORT_HANDLER -#undef MULTI_DABORT - -#if defined(CONFIG_CPU_ARM610) -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER cpu_arm6_data_abort -# endif -#endif - -#if defined(CONFIG_CPU_ARM710) -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER cpu_arm7_data_abort -# endif -#endif - -#ifdef CONFIG_CPU_ABRT_LV4T -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER v4t_late_abort -# endif -#endif - -#ifdef CONFIG_CPU_ABRT_EV4 -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER v4_early_abort -# endif -#endif - -#ifdef CONFIG_CPU_ABRT_EV4T -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER v4t_early_abort -# endif -#endif - -#ifdef CONFIG_CPU_ABRT_EV5TJ -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER v5tj_early_abort -# endif -#endif - -#ifdef CONFIG_CPU_ABRT_EV5T -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER v5t_early_abort -# endif -#endif - -#ifdef CONFIG_CPU_ABRT_EV6 -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER v6_early_abort -# endif -#endif - -#ifdef CONFIG_CPU_ABRT_EV7 -# ifdef CPU_DABORT_HANDLER -# define MULTI_DABORT 1 -# else -# define CPU_DABORT_HANDLER v7_early_abort -# endif -#endif - -#ifndef CPU_DABORT_HANDLER -#error Unknown data abort handler type -#endif - -/* - * Prefetch Abort Model - * ================ - * - * We have the following to choose from: - * legacy - no IFSR, no IFAR - * v6 - ARMv6: IFSR, no IFAR - * v7 - ARMv7: IFSR and IFAR - */ - -#undef CPU_PABORT_HANDLER -#undef MULTI_PABORT - -#ifdef CONFIG_CPU_PABRT_LEGACY -# ifdef CPU_PABORT_HANDLER -# define MULTI_PABORT 1 -# else -# define CPU_PABORT_HANDLER legacy_pabort -# endif -#endif - -#ifdef CONFIG_CPU_PABRT_V6 -# ifdef CPU_PABORT_HANDLER -# define MULTI_PABORT 1 -# else -# define CPU_PABORT_HANDLER v6_pabort -# endif -#endif - -#ifdef CONFIG_CPU_PABRT_V7 -# ifdef CPU_PABORT_HANDLER -# define MULTI_PABORT 1 -# else -# define CPU_PABORT_HANDLER v7_pabort -# endif -#endif - -#ifndef CPU_PABORT_HANDLER -#error Unknown prefetch abort handler type -#endif - #endif diff --git a/arch/arm/include/asm/hardware/cache-l2x0.h b/arch/arm/include/asm/hardware/cache-l2x0.h index 5aeec1e1735..16bd4803158 100644 --- a/arch/arm/include/asm/hardware/cache-l2x0.h +++ b/arch/arm/include/asm/hardware/cache-l2x0.h @@ -36,6 +36,7 @@ #define L2X0_RAW_INTR_STAT 0x21C #define L2X0_INTR_CLEAR 0x220 #define L2X0_CACHE_SYNC 0x730 +#define L2X0_DUMMY_REG 0x740 #define L2X0_INV_LINE_PA 0x770 #define L2X0_INV_WAY 0x77C #define L2X0_CLEAN_LINE_PA 0x7B0 diff --git a/arch/arm/include/asm/hardware/sp810.h b/arch/arm/include/asm/hardware/sp810.h index 721847dc68a..e0d1c0cfa54 100644 --- a/arch/arm/include/asm/hardware/sp810.h +++ b/arch/arm/include/asm/hardware/sp810.h @@ -58,6 +58,9 @@ static inline void sysctl_soft_reset(void __iomem *base) { + /* switch to slow mode */ + writel(0x2, base + SCCTRL); + /* writing any value to SCSYSSTAT reg will reset system */ writel(0, base + SCSYSSTAT); } diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index 3a0893a76a3..bf13b814c1b 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -15,10 +15,6 @@ struct meminfo; struct sys_timer; struct machine_desc { - /* - * Note! The first two elements are used - * by assembler code in head.S, head-common.S - */ unsigned int nr; /* architecture number */ const char *name; /* architecture name */ unsigned long boot_params; /* tagged list */ diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h index 9763be04f77..22de005f159 100644 --- a/arch/arm/include/asm/pgalloc.h +++ b/arch/arm/include/asm/pgalloc.h @@ -10,6 +10,8 @@ #ifndef _ASMARM_PGALLOC_H #define _ASMARM_PGALLOC_H +#include <linux/pagemap.h> + #include <asm/domain.h> #include <asm/pgtable-hwdef.h> #include <asm/processor.h> diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h index 8fdae9bc9ab..8ec535e11fd 100644 --- a/arch/arm/include/asm/proc-fns.h +++ b/arch/arm/include/asm/proc-fns.h @@ -13,250 +13,86 @@ #ifdef __KERNEL__ +#include <asm/glue-proc.h> +#include <asm/page.h> -/* - * Work out if we need multiple CPU support - */ -#undef MULTI_CPU -#undef CPU_NAME +#ifndef __ASSEMBLY__ + +struct mm_struct; /* - * CPU_NAME - the prefix for CPU related functions + * Don't change this structure - ASM code relies on it. */ - -#ifdef CONFIG_CPU_ARM610 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm6 -# endif -#endif - -#ifdef CONFIG_CPU_ARM7TDMI -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm7tdmi -# endif -#endif - -#ifdef CONFIG_CPU_ARM710 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm7 -# endif -#endif - -#ifdef CONFIG_CPU_ARM720T -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm720 -# endif -#endif - -#ifdef CONFIG_CPU_ARM740T -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm740 -# endif -#endif - -#ifdef CONFIG_CPU_ARM9TDMI -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm9tdmi -# endif -#endif - -#ifdef CONFIG_CPU_ARM920T -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm920 -# endif -#endif - -#ifdef CONFIG_CPU_ARM922T -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm922 -# endif -#endif - -#ifdef CONFIG_CPU_FA526 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_fa526 -# endif -#endif - -#ifdef CONFIG_CPU_ARM925T -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm925 -# endif -#endif - -#ifdef CONFIG_CPU_ARM926T -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm926 -# endif -#endif - -#ifdef CONFIG_CPU_ARM940T -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm940 -# endif -#endif - -#ifdef CONFIG_CPU_ARM946E -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm946 -# endif -#endif - -#ifdef CONFIG_CPU_SA110 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_sa110 -# endif -#endif - -#ifdef CONFIG_CPU_SA1100 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_sa1100 -# endif -#endif - -#ifdef CONFIG_CPU_ARM1020 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm1020 -# endif -#endif - -#ifdef CONFIG_CPU_ARM1020E -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm1020e -# endif -#endif - -#ifdef CONFIG_CPU_ARM1022 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm1022 -# endif -#endif - -#ifdef CONFIG_CPU_ARM1026 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_arm1026 -# endif -#endif - -#ifdef CONFIG_CPU_XSCALE -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_xscale -# endif -#endif - -#ifdef CONFIG_CPU_XSC3 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_xsc3 -# endif -#endif - -#ifdef CONFIG_CPU_MOHAWK -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_mohawk -# endif -#endif - -#ifdef CONFIG_CPU_FEROCEON -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_feroceon -# endif -#endif - -#ifdef CONFIG_CPU_V6 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_v6 -# endif -#endif - -#ifdef CONFIG_CPU_V7 -# ifdef CPU_NAME -# undef MULTI_CPU -# define MULTI_CPU -# else -# define CPU_NAME cpu_v7 -# endif -#endif - -#ifndef __ASSEMBLY__ +extern struct processor { + /* MISC + * get data abort address/flags + */ + void (*_data_abort)(unsigned long pc); + /* + * Retrieve prefetch fault address + */ + unsigned long (*_prefetch_abort)(unsigned long lr); + /* + * Set up any processor specifics + */ + void (*_proc_init)(void); + /* + * Disable any processor specifics + */ + void (*_proc_fin)(void); + /* + * Special stuff for a reset + */ + void (*reset)(unsigned long addr) __attribute__((noreturn)); + /* + * Idle the processor + */ + int (*_do_idle)(void); + /* + * Processor architecture specific + */ + /* + * clean a virtual address range from the + * D-cache without flushing the cache. + */ + void (*dcache_clean_area)(void *addr, int size); + + /* + * Set the page table + */ + void (*switch_mm)(unsigned long pgd_phys, struct mm_struct *mm); + /* + * Set a possibly extended PTE. Non-extended PTEs should + * ignore 'ext'. + */ + void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext); + + /* Suspend/resume */ + unsigned int suspend_size; + void (*do_suspend)(void *); + void (*do_resume)(void *); +} processor; #ifndef MULTI_CPU -#include <asm/cpu-single.h> +extern void cpu_proc_init(void); +extern void cpu_proc_fin(void); +extern int cpu_do_idle(void); +extern void cpu_dcache_clean_area(void *, int); +extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); +extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); #else -#include <asm/cpu-multi32.h> +#define cpu_proc_init() processor._proc_init() +#define cpu_proc_fin() processor._proc_fin() +#define cpu_reset(addr) processor.reset(addr) +#define cpu_do_idle() processor._do_idle() +#define cpu_dcache_clean_area(addr,sz) processor.dcache_clean_area(addr,sz) +#define cpu_set_pte_ext(ptep,pte,ext) processor.set_pte_ext(ptep,pte,ext) +#define cpu_do_switch_mm(pgd,mm) processor.switch_mm(pgd,mm) #endif +extern void cpu_resume(void); + #include <asm/memory.h> #ifdef CONFIG_MMU diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index 67357baaeee..7a1f03c10f1 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -95,7 +95,7 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); -#if __LINUX_ARM_ARCH__ == 6 +#if __LINUX_ARM_ARCH__ == 6 || defined(CONFIG_ARM_ERRATA_754327) #define cpu_relax() smp_mb() #else #define cpu_relax() barrier() diff --git a/arch/arm/include/asm/smp_scu.h b/arch/arm/include/asm/smp_scu.h index 2376835015d..4eb6d005ffa 100644 --- a/arch/arm/include/asm/smp_scu.h +++ b/arch/arm/include/asm/smp_scu.h @@ -1,7 +1,14 @@ #ifndef __ASMARM_ARCH_SCU_H #define __ASMARM_ARCH_SCU_H +#define SCU_PM_NORMAL 0 +#define SCU_PM_DORMANT 2 +#define SCU_PM_POWEROFF 3 + +#ifndef __ASSEMBLER__ unsigned int scu_get_core_count(void __iomem *); void scu_enable(void __iomem *); +int scu_power_mode(void __iomem *, unsigned int); +#endif #endif diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index f41a6f57cd1..82dfe5d0c41 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -18,16 +18,34 @@ #define __ASMARM_TLB_H #include <asm/cacheflush.h> -#include <asm/tlbflush.h> #ifndef CONFIG_MMU #include <linux/pagemap.h> + +#define tlb_flush(tlb) ((void) tlb) + #include <asm-generic/tlb.h> #else /* !CONFIG_MMU */ +#include <linux/swap.h> #include <asm/pgalloc.h> +#include <asm/tlbflush.h> + +/* + * We need to delay page freeing for SMP as other CPUs can access pages + * which have been removed but not yet had their TLB entries invalidated. + * Also, as ARMv7 speculative prefetch can drag new entries into the TLB, + * we need to apply this same delaying tactic to ensure correct operation. + */ +#if defined(CONFIG_SMP) || defined(CONFIG_CPU_32v7) +#define tlb_fast_mode(tlb) 0 +#define FREE_PTE_NR 500 +#else +#define tlb_fast_mode(tlb) 1 +#define FREE_PTE_NR 0 +#endif /* * TLB handling. This allows us to remove pages from the page @@ -36,12 +54,58 @@ struct mmu_gather { struct mm_struct *mm; unsigned int fullmm; + struct vm_area_struct *vma; unsigned long range_start; unsigned long range_end; + unsigned int nr; + struct page *pages[FREE_PTE_NR]; }; DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); +/* + * This is unnecessarily complex. There's three ways the TLB shootdown + * code is used: + * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). + * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. + * tlb->vma will be non-NULL. + * 2. Unmapping all vmas. See exit_mmap(). + * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. + * tlb->vma will be non-NULL. Additionally, page tables will be freed. + * 3. Unmapping argument pages. See shift_arg_pages(). + * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. + * tlb->vma will be NULL. + */ +static inline void tlb_flush(struct mmu_gather *tlb) +{ + if (tlb->fullmm || !tlb->vma) + flush_tlb_mm(tlb->mm); + else if (tlb->range_end > 0) { + flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end); + tlb->range_start = TASK_SIZE; + tlb->range_end = 0; + } +} + +static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) +{ + if (!tlb->fullmm) { + if (addr < tlb->range_start) + tlb->range_start = addr; + if (addr + PAGE_SIZE > tlb->range_end) + tlb->range_end = addr + PAGE_SIZE; + } +} + +static inline void tlb_flush_mmu(struct mmu_gather *tlb) +{ + tlb_flush(tlb); + if (!tlb_fast_mode(tlb)) { + free_pages_and_swap_cache(tlb->pages, tlb->nr); + tlb->nr = 0; + } +} + static inline struct mmu_gather * tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) { @@ -49,6 +113,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) tlb->mm = mm; tlb->fullmm = full_mm_flush; + tlb->vma = NULL; + tlb->nr = 0; return tlb; } @@ -56,8 +122,7 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) static inline void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) { - if (tlb->fullmm) - flush_tlb_mm(tlb->mm); + tlb_flush_mmu(tlb); /* keep the page table cache within bounds */ check_pgt_cache(); @@ -71,12 +136,7 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) static inline void tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr) { - if (!tlb->fullmm) { - if (addr < tlb->range_start) - tlb->range_start = addr; - if (addr + PAGE_SIZE > tlb->range_end) - tlb->range_end = addr + PAGE_SIZE; - } + tlb_add_flush(tlb, addr); } /* @@ -89,6 +149,7 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { if (!tlb->fullmm) { flush_cache_range(vma, vma->vm_start, vma->vm_end); + tlb->vma = vma; tlb->range_start = TASK_SIZE; tlb->range_end = 0; } @@ -97,12 +158,30 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { - if (!tlb->fullmm && tlb->range_end > 0) - flush_tlb_range(vma, tlb->range_start, tlb->range_end); + if (!tlb->fullmm) + tlb_flush(tlb); +} + +static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) +{ + if (tlb_fast_mode(tlb)) { + free_page_and_swap_cache(page); + } else { + tlb->pages[tlb->nr++] = page; + if (tlb->nr >= FREE_PTE_NR) + tlb_flush_mmu(tlb); + } +} + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, + unsigned long addr) +{ + pgtable_page_dtor(pte); + tlb_add_flush(tlb, addr); + tlb_remove_page(tlb, pte); } -#define tlb_remove_page(tlb,page) free_page_and_swap_cache(page) -#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep) +#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) #define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) #define tlb_migrate_finish(mm) do { } while (0) diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index ce7378ea15a..d2005de383b 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -10,12 +10,7 @@ #ifndef _ASMARM_TLBFLUSH_H #define _ASMARM_TLBFLUSH_H - -#ifndef CONFIG_MMU - -#define tlb_flush(tlb) ((void) tlb) - -#else /* CONFIG_MMU */ +#ifdef CONFIG_MMU #include <asm/glue.h> |