From 826681043d7184b4d650cab5b007b9a86b628eb5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 17 May 2009 16:20:18 +0100 Subject: [ARM] smp: fix cpumask usage in ARM SMP code The ARM SMP code wasn't properly updated for the cpumask changes, which results in smp_timer_broadcast() broadcasting ticks to non-online CPUs. Signed-off-by: Russell King --- arch/arm/kernel/smp.c | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 7801aac3c04..6014dfd22af 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -326,14 +326,14 @@ void __init smp_prepare_boot_cpu(void) per_cpu(cpu_data, cpu).idle = current; } -static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg) +static void send_ipi_message(const struct cpumask *mask, enum ipi_msg_type msg) { unsigned long flags; unsigned int cpu; local_irq_save(flags); - for_each_cpu_mask(cpu, callmap) { + for_each_cpu(cpu, mask) { struct ipi_data *ipi = &per_cpu(ipi_data, cpu); spin_lock(&ipi->lock); @@ -344,19 +344,19 @@ static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg) /* * Call the platform specific cross-CPU call function. */ - smp_cross_call(callmap); + smp_cross_call(mask); local_irq_restore(flags); } -void arch_send_call_function_ipi(cpumask_t mask) +void arch_send_call_function_ipi_mask(const struct cpumask *mask) { send_ipi_message(mask, IPI_CALL_FUNC); } void arch_send_call_function_single_ipi(int cpu) { - send_ipi_message(cpumask_of_cpu(cpu), IPI_CALL_FUNC_SINGLE); + send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); } void show_ipi_list(struct seq_file *p) @@ -498,17 +498,10 @@ asmlinkage void __exception do_IPI(struct pt_regs *regs) void smp_send_reschedule(int cpu) { - send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE); + send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } -void smp_send_timer(void) -{ - cpumask_t mask = cpu_online_map; - cpu_clear(smp_processor_id(), mask); - send_ipi_message(mask, IPI_TIMER); -} - -void smp_timer_broadcast(cpumask_t mask) +void smp_timer_broadcast(const struct cpumask *mask) { send_ipi_message(mask, IPI_TIMER); } @@ -517,7 +510,7 @@ void smp_send_stop(void) { cpumask_t mask = cpu_online_map; cpu_clear(smp_processor_id(), mask); - send_ipi_message(mask, IPI_CPU_STOP); + send_ipi_message(&mask, IPI_CPU_STOP); } /* @@ -528,20 +521,17 @@ int setup_profiling_timer(unsigned int multiplier) return -EINVAL; } -static int -on_each_cpu_mask(void (*func)(void *), void *info, int wait, cpumask_t mask) +static void +on_each_cpu_mask(void (*func)(void *), void *info, int wait, + const struct cpumask *mask) { - int ret = 0; - preempt_disable(); - ret = smp_call_function_mask(mask, func, info, wait); - if (cpu_isset(smp_processor_id(), mask)) + smp_call_function_many(mask, func, info, wait); + if (cpumask_test_cpu(smp_processor_id(), mask)) func(info); preempt_enable(); - - return ret; } /**********************************************************************/ @@ -602,20 +592,17 @@ void flush_tlb_all(void) void flush_tlb_mm(struct mm_struct *mm) { - cpumask_t mask = mm->cpu_vm_mask; - - on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { - cpumask_t mask = vma->vm_mm->cpu_vm_mask; struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = uaddr; - on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask); } void flush_tlb_kernel_page(unsigned long kaddr) @@ -630,14 +617,13 @@ void flush_tlb_kernel_page(unsigned long kaddr) void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - cpumask_t mask = vma->vm_mm->cpu_vm_mask; struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = start; ta.ta_end = end; - on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) -- cgit v1.2.3-70-g09d2 From bc28248ee25e5c239cbe6afca35a100b08401de5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 17 May 2009 18:58:34 +0100 Subject: [ARM] smp: move core localtimer support out of platform specific files Signed-off-by: Russell King --- arch/arm/Kconfig | 1 + arch/arm/include/asm/localtimer.h | 51 +++++++++++++++++++++++++++ arch/arm/include/asm/smp.h | 40 --------------------- arch/arm/kernel/smp.c | 56 ++++++++++++++++++++++++----- arch/arm/mach-realview/localtimer.c | 70 ++++++++----------------------------- arch/arm/mach-realview/platsmp.c | 14 ++++---- 6 files changed, 121 insertions(+), 111 deletions(-) create mode 100644 arch/arm/include/asm/localtimer.h (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e60ec54df33..ef4f860d504 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -859,6 +859,7 @@ source "kernel/time/Kconfig" config SMP bool "Symmetric Multi-Processing (EXPERIMENTAL)" depends on EXPERIMENTAL && (REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP) + depends on GENERIC_CLOCKEVENTS select USE_GENERIC_SMP_HELPERS help This enables support for systems with more than one CPU. If you have diff --git a/arch/arm/include/asm/localtimer.h b/arch/arm/include/asm/localtimer.h new file mode 100644 index 00000000000..3f8c9ebb646 --- /dev/null +++ b/arch/arm/include/asm/localtimer.h @@ -0,0 +1,51 @@ +/* + * arch/arm/include/asm/localtimer.h + * + * Copyright (C) 2004-2005 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __ASM_ARM_LOCALTIMER_H +#define __ASM_ARM_LOCALTIMER_H + +struct clock_event_device; + +/* + * Setup a per-cpu timer, whether it be a local timer or dummy broadcast + */ +void percpu_timer_setup(void); + +/* + * Called from assembly, this is the local timer IRQ handler + */ +asmlinkage void do_local_timer(struct pt_regs *); + + +#ifdef CONFIG_LOCAL_TIMERS +/* + * Platform provides this to acknowledge a local timer IRQ. + * Returns true if the local timer IRQ is to be processed. + */ +int local_timer_ack(void); + +/* + * Stop a local timer interrupt. + */ +void local_timer_stop(void); + +/* + * Setup a local timer interrupt for a CPU. + */ +void local_timer_setup(struct clock_event_device *); + +#else + +static inline void local_timer_stop(void) +{ +} + +#endif + +#endif diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 5995935338e..608f2d533ff 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -55,11 +55,6 @@ extern void smp_store_cpu_info(unsigned int cpuid); */ extern void smp_cross_call(const struct cpumask *mask); -/* - * Broadcast a clock event to other CPUs. - */ -extern void smp_timer_broadcast(const struct cpumask *mask); - /* * Boot a secondary CPU, and assign it the specified idle task. * This also gives us the initial stack to use for this CPU. @@ -100,44 +95,9 @@ extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); #define arch_send_call_function_ipi_mask arch_send_call_function_ipi_mask -/* - * Local timer interrupt handling function (can be IPI'ed). - */ -extern void local_timer_interrupt(void); - -#ifdef CONFIG_LOCAL_TIMERS - -/* - * Stop a local timer interrupt. - */ -extern void local_timer_stop(void); - -/* - * Platform provides this to acknowledge a local timer IRQ - */ -extern int local_timer_ack(void); - -#else - -static inline void local_timer_stop(void) -{ -} - -#endif - -/* - * Setup a local timer interrupt for a CPU. - */ -extern void local_timer_setup(void); - /* * show local interrupt info */ extern void show_local_irqs(struct seq_file *); -/* - * Called from assembly, this is the local timer IRQ handler - */ -asmlinkage void do_local_timer(struct pt_regs *); - #endif /* ifndef __ASM_ARM_SMP_H */ diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 6014dfd22af..91130e218ae 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -32,6 +34,7 @@ #include #include #include +#include /* * as from 2.5, kernels no longer have an init_tasks structure @@ -274,9 +277,9 @@ asmlinkage void __cpuinit secondary_start_kernel(void) local_fiq_enable(); /* - * Setup local timer for this CPU. + * Setup the percpu timer for this CPU. */ - local_timer_setup(); + percpu_timer_setup(); calibrate_delay(); @@ -383,10 +386,16 @@ void show_local_irqs(struct seq_file *p) seq_putc(p, '\n'); } +/* + * Timer (local or broadcast) support + */ +static DEFINE_PER_CPU(struct clock_event_device, percpu_clockevent); + static void ipi_timer(void) { + struct clock_event_device *evt = &__get_cpu_var(percpu_clockevent); irq_enter(); - local_timer_interrupt(); + evt->event_handler(evt); irq_exit(); } @@ -405,6 +414,42 @@ asmlinkage void __exception do_local_timer(struct pt_regs *regs) } #endif +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST +static void smp_timer_broadcast(const struct cpumask *mask) +{ + send_ipi_message(mask, IPI_TIMER); +} + +static void broadcast_timer_set_mode(enum clock_event_mode mode, + struct clock_event_device *evt) +{ +} + +static void local_timer_setup(struct clock_event_device *evt) +{ + evt->name = "dummy_timer"; + evt->features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_DUMMY; + evt->rating = 400; + evt->mult = 1; + evt->set_mode = broadcast_timer_set_mode; + evt->broadcast = smp_timer_broadcast; + + clockevents_register_device(evt); +} +#endif + +void __cpuinit percpu_timer_setup(void) +{ + unsigned int cpu = smp_processor_id(); + struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu); + + evt->cpumask = cpumask_of(cpu); + + local_timer_setup(evt); +} + static DEFINE_SPINLOCK(stop_lock); /* @@ -501,11 +546,6 @@ void smp_send_reschedule(int cpu) send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } -void smp_timer_broadcast(const struct cpumask *mask) -{ - send_ipi_message(mask, IPI_TIMER); -} - void smp_send_stop(void) { cpumask_t mask = cpu_online_map; diff --git a/arch/arm/mach-realview/localtimer.c b/arch/arm/mach-realview/localtimer.c index 1c01d13460f..cd98e7acd94 100644 --- a/arch/arm/mach-realview/localtimer.c +++ b/arch/arm/mach-realview/localtimer.c @@ -11,10 +11,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -24,18 +22,6 @@ #include #include -static DEFINE_PER_CPU(struct clock_event_device, local_clockevent); - -/* - * Used on SMP for either the local timer or IPI_TIMER - */ -void local_timer_interrupt(void) -{ - struct clock_event_device *clk = &__get_cpu_var(local_clockevent); - - clk->event_handler(clk); -} - #ifdef CONFIG_LOCAL_TIMERS /* set up by the platform code */ @@ -44,7 +30,7 @@ void __iomem *twd_base; static unsigned long mpcore_timer_rate; static void local_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *clk) + struct clock_event_device *evt) { unsigned long ctrl; @@ -140,32 +126,29 @@ static void __cpuinit twd_calibrate_rate(void) /* * Setup the local clock events for a CPU. */ -void __cpuinit local_timer_setup(void) +void __cpuinit local_timer_setup(struct clock_event_device *evt) { - unsigned int cpu = smp_processor_id(); - struct clock_event_device *clk = &per_cpu(local_clockevent, cpu); unsigned long flags; twd_calibrate_rate(); - clk->name = "local_timer"; - clk->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; - clk->rating = 350; - clk->set_mode = local_timer_set_mode; - clk->set_next_event = local_timer_set_next_event; - clk->irq = IRQ_LOCALTIMER; - clk->cpumask = cpumask_of(cpu); - clk->shift = 20; - clk->mult = div_sc(mpcore_timer_rate, NSEC_PER_SEC, clk->shift); - clk->max_delta_ns = clockevent_delta2ns(0xffffffff, clk); - clk->min_delta_ns = clockevent_delta2ns(0xf, clk); + evt->name = "local_timer"; + evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; + evt->rating = 350; + evt->set_mode = local_timer_set_mode; + evt->set_next_event = local_timer_set_next_event; + evt->irq = IRQ_LOCALTIMER; + evt->shift = 20; + evt->mult = div_sc(mpcore_timer_rate, NSEC_PER_SEC, evt->shift); + evt->max_delta_ns = clockevent_delta2ns(0xffffffff, evt); + evt->min_delta_ns = clockevent_delta2ns(0xf, evt); /* Make sure our local interrupt controller has this enabled */ local_irq_save(flags); get_irq_chip(IRQ_LOCALTIMER)->unmask(IRQ_LOCALTIMER); local_irq_restore(flags); - clockevents_register_device(clk); + clockevents_register_device(evt); } /* @@ -176,29 +159,4 @@ void __cpuexit local_timer_stop(void) __raw_writel(0, twd_base + TWD_TIMER_CONTROL); } -#else /* CONFIG_LOCAL_TIMERS */ - -static void dummy_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *clk) -{ -} - -void __cpuinit local_timer_setup(void) -{ - unsigned int cpu = smp_processor_id(); - struct clock_event_device *clk = &per_cpu(local_clockevent, cpu); - - clk->name = "dummy_timer"; - clk->features = CLOCK_EVT_FEAT_ONESHOT | - CLOCK_EVT_FEAT_PERIODIC | - CLOCK_EVT_FEAT_DUMMY; - clk->rating = 400; - clk->mult = 1; - clk->set_mode = dummy_timer_set_mode; - clk->broadcast = smp_timer_broadcast; - clk->cpumask = cpumask_of(cpu); - - clockevents_register_device(clk); -} - -#endif /* !CONFIG_LOCAL_TIMERS */ +#endif /* CONFIG_LOCAL_TIMERS */ diff --git a/arch/arm/mach-realview/platsmp.c b/arch/arm/mach-realview/platsmp.c index 30a9c68591f..b34d3a57ce9 100644 --- a/arch/arm/mach-realview/platsmp.c +++ b/arch/arm/mach-realview/platsmp.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -217,13 +218,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (max_cpus > ncores) max_cpus = ncores; -#if defined(CONFIG_LOCAL_TIMERS) || defined(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) - /* - * Enable the local timer or broadcast device for the boot CPU. - */ - local_timer_setup(); -#endif - /* * Initialise the present map, which describes the set of CPUs * actually populated at the present time. @@ -239,6 +233,12 @@ void __init smp_prepare_cpus(unsigned int max_cpus) * WFI */ if (max_cpus > 1) { + /* + * Enable the local timer or broadcast device for the + * boot CPU, but only if we have more than one CPU. + */ + percpu_timer_setup(); + scu_enable(); poke_milo(); } -- cgit v1.2.3-70-g09d2 From e03cdade0ca945a04e982525e50fef275190b77b Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 28 May 2009 14:16:52 +0100 Subject: [ARM] smp: use new cpumask functions Convert cpu_*_mask bit twiddling to the new set_cpu_*() API. Signed-off-by: Russell King --- arch/arm/include/asm/smp.h | 2 +- arch/arm/kernel/smp.c | 6 +++--- arch/arm/mach-realview/platsmp.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 608f2d533ff..a06e735b262 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -41,7 +41,7 @@ extern void show_ipi_list(struct seq_file *p); asmlinkage void do_IPI(struct pt_regs *regs); /* - * Setup the SMP cpu_possible_map + * Setup the set of possible CPUs (via set_cpu_possible) */ extern void smp_init_cpus(void); diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 91130e218ae..0d8097fa4ca 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -166,7 +166,7 @@ int __cpuexit __cpu_disable(void) * Take this CPU offline. Once we clear this, we can't return, * and we must not schedule until we're ready to give up the cpu. */ - cpu_clear(cpu, cpu_online_map); + set_cpu_online(cpu, false); /* * OK - migrate IRQs away from this CPU @@ -288,7 +288,7 @@ asmlinkage void __cpuinit secondary_start_kernel(void) /* * OK, now it's safe to let the boot CPU continue */ - cpu_set(cpu, cpu_online_map); + set_cpu_online(cpu, true); /* * OK, it's off to the idle thread for us @@ -462,7 +462,7 @@ static void ipi_cpu_stop(unsigned int cpu) dump_stack(); spin_unlock(&stop_lock); - cpu_clear(cpu, cpu_online_map); + set_cpu_online(cpu, false); local_fiq_disable(); local_irq_disable(); diff --git a/arch/arm/mach-realview/platsmp.c b/arch/arm/mach-realview/platsmp.c index c862ce19587..ca742172ea7 100644 --- a/arch/arm/mach-realview/platsmp.c +++ b/arch/arm/mach-realview/platsmp.c @@ -166,7 +166,7 @@ void __init smp_init_cpus(void) unsigned int i, ncores = get_core_count(); for (i = 0; i < ncores; i++) - cpu_set(i, cpu_possible_map); + set_cpu_possible(i, true); } void __init smp_prepare_cpus(unsigned int max_cpus) @@ -204,7 +204,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) * actually populated at the present time. */ for (i = 0; i < max_cpus; i++) - cpu_set(i, cpu_present_map); + set_cpu_present(i, true); /* * Initialise the SCU if there are more than one CPU and let -- cgit v1.2.3-70-g09d2 From faa7bc51c11d5bbe440ac04710fd7a3208782000 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Sat, 30 May 2009 14:00:14 +0100 Subject: Check whether the TLB operations need broadcasting on SMP systems ARMv7 SMP hardware can handle the TLB maintenance operations broadcasting in hardware so that the software can avoid the costly IPIs. This patch adds the necessary checks (the MMFR3 CPUID register) to avoid the broadcasting if already supported by the hardware. (this patch is based on the work done by Tony Thompson @ ARM) Signed-off-by: Catalin Marinas --- arch/arm/include/asm/cputype.h | 25 +++++++++++++++ arch/arm/include/asm/tlbflush.h | 26 ++++++++++++++-- arch/arm/kernel/smp.c | 69 +++++++++++++++++++++++++---------------- arch/arm/mm/proc-v7.S | 4 +-- arch/arm/mm/tlb-v7.S | 17 ++++++---- 5 files changed, 105 insertions(+), 36 deletions(-) (limited to 'arch/arm/kernel/smp.c') diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h index 7b9d27e749b..b3e656c6fb7 100644 --- a/arch/arm/include/asm/cputype.h +++ b/arch/arm/include/asm/cputype.h @@ -8,6 +8,21 @@ #define CPUID_TCM 2 #define CPUID_TLBTYPE 3 +#define CPUID_EXT_PFR0 "c1, 0" +#define CPUID_EXT_PFR1 "c1, 1" +#define CPUID_EXT_DFR0 "c1, 2" +#define CPUID_EXT_AFR0 "c1, 3" +#define CPUID_EXT_MMFR0 "c1, 4" +#define CPUID_EXT_MMFR1 "c1, 5" +#define CPUID_EXT_MMFR2 "c1, 6" +#define CPUID_EXT_MMFR3 "c1, 7" +#define CPUID_EXT_ISAR0 "c2, 0" +#define CPUID_EXT_ISAR1 "c2, 1" +#define CPUID_EXT_ISAR2 "c2, 2" +#define CPUID_EXT_ISAR3 "c2, 3" +#define CPUID_EXT_ISAR4 "c2, 4" +#define CPUID_EXT_ISAR5 "c2, 5" + #ifdef CONFIG_CPU_CP15 #define read_cpuid(reg) \ ({ \ @@ -18,9 +33,19 @@ : "cc"); \ __val; \ }) +#define read_cpuid_ext(ext_reg) \ + ({ \ + unsigned int __val; \ + asm("mrc p15, 0, %0, c0, " ext_reg \ + : "=r" (__val) \ + : \ + : "cc"); \ + __val; \ + }) #else extern unsigned int processor_id; #define read_cpuid(reg) (processor_id) +#define read_cpuid_ext(reg) 0 #endif /* diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index a62218013c7..c964f3fc3bc 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -40,6 +40,12 @@ #define TLB_V6_I_ASID (1 << 18) #define TLB_BTB (1 << 28) + +/* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ +#define TLB_V7_UIS_PAGE (1 << 19) +#define TLB_V7_UIS_FULL (1 << 20) +#define TLB_V7_UIS_ASID (1 << 21) + #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ #define TLB_DCLEAN (1 << 30) #define TLB_WB (1 << 31) @@ -176,9 +182,17 @@ # define v6wbi_always_flags (-1UL) #endif +#ifdef CONFIG_SMP +#define v7wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \ + TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) +#else +#define v7wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \ + TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) +#endif + #ifdef CONFIG_CPU_TLB_V7 -# define v7wbi_possible_flags v6wbi_tlb_flags -# define v7wbi_always_flags v6wbi_tlb_flags +# define v7wbi_possible_flags v7wbi_tlb_flags +# define v7wbi_always_flags v7wbi_tlb_flags # ifdef _TLB # define MULTI_TLB 1 # else @@ -316,6 +330,8 @@ static inline void local_flush_tlb_all(void) asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL)) asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); + if (tlb_flag(TLB_V7_UIS_FULL)) + asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_BTB)) { /* flush the branch target cache */ @@ -351,6 +367,8 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc"); if (tlb_flag(TLB_V6_I_ASID)) asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc"); + if (tlb_flag(TLB_V7_UIS_ASID)) + asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc"); if (tlb_flag(TLB_BTB)) { /* flush the branch target cache */ @@ -389,6 +407,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_V6_I_PAGE)) asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); + if (tlb_flag(TLB_V7_UIS_PAGE)) + asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_BTB)) { /* flush the branch target cache */ @@ -424,6 +444,8 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); if (tlb_flag(TLB_V6_I_PAGE)) asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); + if (tlb_flag(TLB_V7_UIS_PAGE)) + asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc"); if (tlb_flag(TLB_BTB)) { /* flush the branch target cache */ diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 6014dfd22af..ece658e773a 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -32,6 +32,7 @@ #include #include #include +#include /* * as from 2.5, kernels no longer have an init_tasks structure @@ -545,6 +546,12 @@ struct tlb_args { unsigned long ta_end; }; +/* all SMP configurations have the extended CPUID registers */ +static inline int tlb_ops_need_broadcast(void) +{ + return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; +} + static inline void ipi_flush_tlb_all(void *ignored) { local_flush_tlb_all(); @@ -587,51 +594,61 @@ static inline void ipi_flush_tlb_kernel_range(void *arg) void flush_tlb_all(void) { - on_each_cpu(ipi_flush_tlb_all, NULL, 1); + if (tlb_ops_need_broadcast()) + on_each_cpu(ipi_flush_tlb_all, NULL, 1); + else + local_flush_tlb_all(); } void flush_tlb_mm(struct mm_struct *mm) { - on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask); + if (tlb_ops_need_broadcast()) + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask); + else + local_flush_tlb_mm(mm); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { - struct tlb_args ta; - - ta.ta_vma = vma; - ta.ta_start = uaddr; - - on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_vma = vma; + ta.ta_start = uaddr; + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask); + } else + local_flush_tlb_page(vma, uaddr); } void flush_tlb_kernel_page(unsigned long kaddr) { - struct tlb_args ta; - - ta.ta_start = kaddr; - - on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_start = kaddr; + on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1); + } else + local_flush_tlb_kernel_page(kaddr); } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - struct tlb_args ta; - - ta.ta_vma = vma; - ta.ta_start = start; - ta.ta_end = end; - - on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_vma = vma; + ta.ta_start = start; + ta.ta_end = end; + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask); + } else + local_flush_tlb_range(vma, start, end); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - struct tlb_args ta; - - ta.ta_start = start; - ta.ta_end = end; - - on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_start = start; + ta.ta_end = end; + on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); + } else + local_flush_tlb_kernel_range(start, end); } diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 3397f1e64d7..c3f73751683 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -176,8 +176,8 @@ cpu_v7_name: */ __v7_setup: #ifdef CONFIG_SMP - mrc p15, 0, r0, c1, c0, 1 @ Enable SMP/nAMP mode - orr r0, r0, #(0x1 << 6) + mrc p15, 0, r0, c1, c0, 1 @ Enable SMP/nAMP mode and + orr r0, r0, #(1 << 6) | (1 << 0) @ TLB ops broadcasting mcr p15, 0, r0, c1, c0, 1 #endif adr r12, __v7_setup_stack @ the local stack diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S index b637e7380ab..a26a605b73b 100644 --- a/arch/arm/mm/tlb-v7.S +++ b/arch/arm/mm/tlb-v7.S @@ -42,9 +42,11 @@ ENTRY(v7wbi_flush_user_tlb_range) mov r1, r1, lsl #PAGE_SHIFT vma_vm_flags r2, r2 @ get vma->vm_flags 1: - mcr p15, 0, r0, c8, c6, 1 @ TLB invalidate D MVA (was 1) - tst r2, #VM_EXEC @ Executable area ? - mcrne p15, 0, r0, c8, c5, 1 @ TLB invalidate I MVA (was 1) +#ifdef CONFIG_SMP + mcr p15, 0, r0, c8, c3, 1 @ TLB invalidate U MVA (shareable) +#else + mcr p15, 0, r0, c8, c7, 1 @ TLB invalidate U MVA +#endif add r0, r0, #PAGE_SZ cmp r0, r1 blo 1b @@ -69,8 +71,11 @@ ENTRY(v7wbi_flush_kern_tlb_range) mov r0, r0, lsl #PAGE_SHIFT mov r1, r1, lsl #PAGE_SHIFT 1: - mcr p15, 0, r0, c8, c6, 1 @ TLB invalidate D MVA - mcr p15, 0, r0, c8, c5, 1 @ TLB invalidate I MVA +#ifdef CONFIG_SMP + mcr p15, 0, r0, c8, c3, 1 @ TLB invalidate U MVA (shareable) +#else + mcr p15, 0, r0, c8, c7, 1 @ TLB invalidate U MVA +#endif add r0, r0, #PAGE_SZ cmp r0, r1 blo 1b @@ -87,5 +92,5 @@ ENDPROC(v7wbi_flush_kern_tlb_range) ENTRY(v7wbi_tlb_fns) .long v7wbi_flush_user_tlb_range .long v7wbi_flush_kern_tlb_range - .long v6wbi_tlb_flags + .long v7wbi_tlb_flags .size v7wbi_tlb_fns, . - v7wbi_tlb_fns -- cgit v1.2.3-70-g09d2