From 9fbcbd7e1fa9acde67f3516f7aceef2c0d968a7b Mon Sep 17 00:00:00 2001 From: Hillf Danton Date: Thu, 4 Aug 2011 22:38:31 +0800 Subject: MIPS: VPE: Select correct tc If we could find tc on the tc list for @index, the found tc should be returned. Signed-off-by: Hillf Danton To: LKML Cc: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2692/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/vpe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index 2cd50ad0d5c..3efcb065f78 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -192,7 +192,7 @@ static struct tc *get_tc(int index) } spin_unlock(&vpecontrol.tc_list_lock); - return NULL; + return res; } /* allocate a vpe and associate it with this minor (or index) */ -- cgit v1.2.3-70-g09d2 From fe0b030cf016ee9a4b0ae4adb0095c46d0e461cc Mon Sep 17 00:00:00 2001 From: Hillf Danton Date: Thu, 4 Aug 2011 22:46:41 +0800 Subject: MIPS: i8259: Correct comment for i8259 The comment for the slave PIC is changed from 8259A-1 to 8259A-2. Signed-off-by: Hillf Danton To: LKML Cc: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2693/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/i8259.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/i8259.c b/arch/mips/kernel/i8259.c index 5c74eb797f0..9b22db5dcef 100644 --- a/arch/mips/kernel/i8259.c +++ b/arch/mips/kernel/i8259.c @@ -229,7 +229,7 @@ static void i8259A_shutdown(void) */ if (i8259A_auto_eoi >= 0) { outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */ - outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-1 */ + outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */ } } -- cgit v1.2.3-70-g09d2 From 49de830ad7b94d05f6dbfbae97a03f80169a7462 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 23 Jul 2011 12:41:23 +0000 Subject: MIPS: Ftrace: Fix the CONFIG_DYNAMIC_FTRACE=n compile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit arch/mips/kernel/ftrace.c: In function ‘ftrace_get_parent_ra_addr’: arch/mips/kernel/ftrace.c:212: error: implicit declaration of function ‘in_kernel_space’ arch/mips/kernel/ftrace.c: In function ‘prepare_ftrace_return’: arch/mips/kernel/ftrace.c:314: error: ‘MCOUNT_OFFSET_INSNS’ undeclared (first use in this function) arch/mips/kernel/ftrace.c:314: error: (Each undeclared identifier is reported only once arch/mips/kernel/ftrace.c:314: error: for each function it appears in.) Signed-off-by: Thomas Gleixner To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2634/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/ftrace.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c index feb8021a305..6a2d758dd8e 100644 --- a/arch/mips/kernel/ftrace.c +++ b/arch/mips/kernel/ftrace.c @@ -19,6 +19,26 @@ #include +#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) +#define MCOUNT_OFFSET_INSNS 5 +#else +#define MCOUNT_OFFSET_INSNS 4 +#endif + +/* + * Check if the address is in kernel space + * + * Clone core_kernel_text() from kernel/extable.c, but doesn't call + * init_kernel_text() for Ftrace doesn't trace functions in init sections. + */ +static inline int in_kernel_space(unsigned long ip) +{ + if (ip >= (unsigned long)_stext && + ip <= (unsigned long)_etext) + return 1; + return 0; +} + #ifdef CONFIG_DYNAMIC_FTRACE #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ @@ -54,20 +74,6 @@ static inline void ftrace_dyn_arch_init_insns(void) #endif } -/* - * Check if the address is in kernel space - * - * Clone core_kernel_text() from kernel/extable.c, but doesn't call - * init_kernel_text() for Ftrace doesn't trace functions in init sections. - */ -static inline int in_kernel_space(unsigned long ip) -{ - if (ip >= (unsigned long)_stext && - ip <= (unsigned long)_etext) - return 1; - return 0; -} - static int ftrace_modify_code(unsigned long ip, unsigned int new_code) { int faulted; @@ -112,11 +118,6 @@ static int ftrace_modify_code(unsigned long ip, unsigned int new_code) * 1: offset = 4 instructions */ -#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) -#define MCOUNT_OFFSET_INSNS 5 -#else -#define MCOUNT_OFFSET_INSNS 4 -#endif #define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS) int ftrace_make_nop(struct module *mod, -- cgit v1.2.3-70-g09d2 From 4d85f6afa43d366868fdaa19595722309e4843f6 Mon Sep 17 00:00:00 2001 From: Wu Zhangjin Date: Sat, 23 Jul 2011 12:41:24 +0000 Subject: MIPS: Make the die_lock be raw On preempt-rt this lock needs to be raw, so it does not get converted to a sleeping spinlock. Trying to sleep in a panic is not really desireable. Signed-off-by: Wu Zhangjin Signed-off-by: Thomas Gleixner Cc: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2636/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/traps.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index b7517e3abc8..01eff7e609e 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -364,7 +364,7 @@ static int regs_to_trapnr(struct pt_regs *regs) return (regs->cp0_cause >> 2) & 0x1f; } -static DEFINE_SPINLOCK(die_lock); +static DEFINE_RAW_SPINLOCK(die_lock); void __noreturn die(const char *str, struct pt_regs *regs) { @@ -378,7 +378,7 @@ void __noreturn die(const char *str, struct pt_regs *regs) sig = 0; console_verbose(); - spin_lock_irq(&die_lock); + raw_spin_lock_irq(&die_lock); bust_spinlocks(1); #ifdef CONFIG_MIPS_MT_SMTC mips_mt_regdump(dvpret); @@ -387,7 +387,7 @@ void __noreturn die(const char *str, struct pt_regs *regs) printk("%s[#%d]:\n", str, ++die_counter); show_registers(regs); add_taint(TAINT_DIE); - spin_unlock_irq(&die_lock); + raw_spin_unlock_irq(&die_lock); if (in_interrupt()) panic("Fatal exception in interrupt"); -- cgit v1.2.3-70-g09d2 From 5c22cd407574988cc1580108e0df25ee64ea3d59 Mon Sep 17 00:00:00 2001 From: Liming Wang Date: Fri, 26 Aug 2011 07:00:04 +0800 Subject: MIPS: i8259: Mark cascade interrupt non-threaded Cascade interrupts cannot be threaded. Signed-off-by: Liming Wang Signed-off-by: Bruce Ashfield Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1314370804-21266-1-git-send-email-liming.wang@windriver.com Signed-off-by: Thomas Gleixner Cc: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2770/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/i8259.c | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/i8259.c b/arch/mips/kernel/i8259.c index 9b22db5dcef..32b397b646e 100644 --- a/arch/mips/kernel/i8259.c +++ b/arch/mips/kernel/i8259.c @@ -295,6 +295,7 @@ static void init_8259A(int auto_eoi) static struct irqaction irq2 = { .handler = no_action, .name = "cascade", + .flags = IRQF_NO_THREAD, }; static struct resource pic1_io_resource = { -- cgit v1.2.3-70-g09d2 From 1f717929e987fc55bb6be02df994945f8edbd46c Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Wed, 27 Jul 2011 11:44:47 +0100 Subject: MIPS: Handle __put_user() sleeping. do_signal() does __put_user() which can fault, resulting in a might_sleep() warning in down_read(&mm->mmap_sem) and a "scheduling while atomic" warning when mmap_sem is contented. On Swarm this also results in: WARNING: at kernel/smp.c:459 smp_call_function_many+0x148/0x398() Modules linked in: Call Trace: [] dump_stack+0x1c/0x50 [] warn_slowpath_common+0x8c/0xc8 [] warn_slowpath_null+0x2c/0x40 [] smp_call_function_many+0x148/0x398 [] smp_call_function+0x58/0xa8 [] r4k_flush_data_cache_page+0x54/0xd8 [] handle_pte_fault+0xa9c/0xad0 [] handle_mm_fault+0x158/0x200 [] do_page_fault+0x218/0x3b0 [] ret_from_exception+0x0/0x10 [] copy_siginfo_to_user32+0x50/0x298 [] setup_rt_frame_32+0x90/0x250 [] do_notify_resume+0x154/0x358 [] work_notifysig+0xc/0x14 Fixed by enabling interrupts in do_notify_resume before delivering signals. [ralf@linux-mips.org: Reported and original fix by tglx but I wanted to minimize the amount of code being run with interrupts disabled so I moved the local_irq_disable() call right into do_notify_resume. Which is saner than doing it in entry.S.] Reported-by: Thomas Gleixner Signed-off-by: Ralf Baechle --- arch/mips/kernel/signal.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index dbbe0ce48d8..f8524003676 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -8,6 +8,7 @@ * Copyright (C) 1999, 2000 Silicon Graphics, Inc. */ #include +#include #include #include #include @@ -658,6 +659,8 @@ static void do_signal(struct pt_regs *regs) asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags) { + local_irq_enable(); + /* deal with pending signal delivery */ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) do_signal(regs); -- cgit v1.2.3-70-g09d2 From 1eec6cd08b4feb72a73aff468ab72bdd21e1dc61 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 16 Aug 2011 09:54:54 +0800 Subject: MIPS: Compat: Use 32-bit wrapper for compat_sys_futex. We can't trust userspace to pass signed-extend arguments. Not correctly sign-extended arguments to futex-wait result in architecturally undefined operation of 32-bit arithmetic instructions. For example, if 'val' is too big and bit-31 is 1, the caller may enter endless loop at: futex_wait_setup() { ... if (uval != val) { queue_unlock(q, *hb); ret = -EWOULDBLOCK; ... } Signed-off-by: Yong Zhang To: linux-mips@linux-mips.org To: linux-kernel@vger.kernel.org Patchwork: https://patchwork.linux-mips.org/patch/2714/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/linux32.c | 7 +++++++ arch/mips/kernel/scall64-n32.S | 2 +- arch/mips/kernel/scall64-o32.S | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/linux32.c b/arch/mips/kernel/linux32.c index 876a75cc376..922a554cd10 100644 --- a/arch/mips/kernel/linux32.c +++ b/arch/mips/kernel/linux32.c @@ -349,3 +349,10 @@ SYSCALL_DEFINE6(32_fanotify_mark, int, fanotify_fd, unsigned int, flags, return sys_fanotify_mark(fanotify_fd, flags, merge_64(a3, a4), dfd, pathname); } + +SYSCALL_DEFINE6(32_futex, u32 __user *, uaddr, int, op, u32, val, + struct compat_timespec __user *, utime, u32 __user *, uaddr2, + u32, val3) +{ + return compat_sys_futex(uaddr, op, val, utime, uaddr2, val3); +} diff --git a/arch/mips/kernel/scall64-n32.S b/arch/mips/kernel/scall64-n32.S index f9296e894e4..6de1f598346 100644 --- a/arch/mips/kernel/scall64-n32.S +++ b/arch/mips/kernel/scall64-n32.S @@ -315,7 +315,7 @@ EXPORT(sysn32_call_table) PTR sys_fremovexattr PTR sys_tkill PTR sys_ni_syscall - PTR compat_sys_futex + PTR sys_32_futex PTR compat_sys_sched_setaffinity /* 6195 */ PTR compat_sys_sched_getaffinity PTR sys_cacheflush diff --git a/arch/mips/kernel/scall64-o32.S b/arch/mips/kernel/scall64-o32.S index 4d7c9827706..1d813169e45 100644 --- a/arch/mips/kernel/scall64-o32.S +++ b/arch/mips/kernel/scall64-o32.S @@ -441,7 +441,7 @@ sys_call_table: PTR sys_fremovexattr /* 4235 */ PTR sys_tkill PTR sys_sendfile64 - PTR compat_sys_futex + PTR sys_32_futex PTR compat_sys_sched_setaffinity PTR compat_sys_sched_getaffinity /* 4240 */ PTR compat_sys_io_setup -- cgit v1.2.3-70-g09d2 From 8742cd23471635f8b069bf9a6806200a77397ddb Mon Sep 17 00:00:00 2001 From: Nathan Lynch Date: Fri, 30 Sep 2011 13:49:35 -0500 Subject: MIPS: Call oops_enter, oops_exit in die This allows pause_on_oops and mtdoops to work. Signed-off-by: Nathan Lynch To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2810/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/traps.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 01eff7e609e..cbea618af0b 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -371,14 +372,19 @@ void __noreturn die(const char *str, struct pt_regs *regs) static int die_counter; int sig = SIGSEGV; #ifdef CONFIG_MIPS_MT_SMTC - unsigned long dvpret = dvpe(); + unsigned long dvpret; #endif /* CONFIG_MIPS_MT_SMTC */ + oops_enter(); + if (notify_die(DIE_OOPS, str, regs, 0, regs_to_trapnr(regs), SIGSEGV) == NOTIFY_STOP) sig = 0; console_verbose(); raw_spin_lock_irq(&die_lock); +#ifdef CONFIG_MIPS_MT_SMTC + dvpret = dvpe(); +#endif /* CONFIG_MIPS_MT_SMTC */ bust_spinlocks(1); #ifdef CONFIG_MIPS_MT_SMTC mips_mt_regdump(dvpret); @@ -389,6 +395,8 @@ void __noreturn die(const char *str, struct pt_regs *regs) add_taint(TAINT_DIE); raw_spin_unlock_irq(&die_lock); + oops_exit(); + if (in_interrupt()) panic("Fatal exception in interrupt"); -- cgit v1.2.3-70-g09d2 From 5db6acdb27fdf76b013ce5b7d0d43b83ee851b6d Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Thu, 16 Jun 2011 10:32:15 +0100 Subject: MIPS: 32-bit: Fix number of argument to epoll_wait. The number of arguments only matters for syscalls with stack arguments that is using 5 or more argument slots so this is just cosmetic fix. Signed-off-by: Ralf Baechle --- arch/mips/kernel/scall32-o32.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/scall32-o32.S b/arch/mips/kernel/scall32-o32.S index 865bc7a6f5a..47920657968 100644 --- a/arch/mips/kernel/scall32-o32.S +++ b/arch/mips/kernel/scall32-o32.S @@ -496,7 +496,7 @@ einval: li v0, -ENOSYS sys sys_lookup_dcookie 4 sys sys_epoll_create 1 sys sys_epoll_ctl 4 - sys sys_epoll_wait 3 /* 4250 */ + sys sys_epoll_wait 4 /* 4250 */ sys sys_remap_file_pages 5 sys sys_set_tid_address 1 sys sys_restart_syscall 0 -- cgit v1.2.3-70-g09d2 From a1431b61a874cc1e11a3a8d59a08144eb34ae9eb Mon Sep 17 00:00:00 2001 From: David Daney Date: Sat, 24 Sep 2011 02:29:54 +0200 Subject: MIPS: Add probes for more Octeon II CPUs. Detect cn61XX, cn66XX and cn68XX CPUs in cpu_probe_cavium(). Signed-off-by: David Daney To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2777/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/cpu-probe.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index ebc0cd20b35..aa327a75598 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -978,7 +978,10 @@ static inline void cpu_probe_cavium(struct cpuinfo_mips *c, unsigned int cpu) platform: set_elf_platform(cpu, "octeon"); break; + case PRID_IMP_CAVIUM_CN61XX: case PRID_IMP_CAVIUM_CN63XX: + case PRID_IMP_CAVIUM_CN66XX: + case PRID_IMP_CAVIUM_CN68XX: c->cputype = CPU_CAVIUM_OCTEON2; __cpu_name[cpu] = "Cavium Octeon II"; set_elf_platform(cpu, "octeon2"); -- cgit v1.2.3-70-g09d2 From 4409af37b83587097e6d0f675a4ed0bb2ca0ee59 Mon Sep 17 00:00:00 2001 From: David Daney Date: Sat, 24 Sep 2011 02:29:55 +0200 Subject: MIPS: perf: Cleanup formatting in arch/mips/kernel/perf_event.c Get rid of a bunch of useless inline declarations, and join a bunch of improperly split lines. Signed-off-by: David Daney Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Deng-Cheng Zhu To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2793/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/perf_event.c | 26 ++++++-------- arch/mips/kernel/perf_event_mipsxx.c | 68 ++++++++++++++---------------------- 2 files changed, 37 insertions(+), 57 deletions(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/perf_event.c b/arch/mips/kernel/perf_event.c index 0aee944ac38..f3d6e99ab57 100644 --- a/arch/mips/kernel/perf_event.c +++ b/arch/mips/kernel/perf_event.c @@ -118,10 +118,9 @@ struct mips_pmu { static const struct mips_pmu *mipspmu; -static int -mipspmu_event_set_period(struct perf_event *event, - struct hw_perf_event *hwc, - int idx) +static int mipspmu_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); s64 left = local64_read(&hwc->period_left); @@ -162,8 +161,8 @@ mipspmu_event_set_period(struct perf_event *event, } static void mipspmu_event_update(struct perf_event *event, - struct hw_perf_event *hwc, - int idx) + struct hw_perf_event *hwc, + int idx) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); unsigned long flags; @@ -420,8 +419,7 @@ static struct pmu pmu = { .read = mipspmu_read, }; -static inline unsigned int -mipspmu_perf_event_encode(const struct mips_perf_event *pev) +static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) { /* * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for @@ -437,8 +435,7 @@ mipspmu_perf_event_encode(const struct mips_perf_event *pev) #endif } -static const struct mips_perf_event * -mipspmu_map_general_event(int idx) +static const struct mips_perf_event *mipspmu_map_general_event(int idx) { const struct mips_perf_event *pev; @@ -449,8 +446,7 @@ mipspmu_map_general_event(int idx) return pev; } -static const struct mips_perf_event * -mipspmu_map_cache_event(u64 config) +static const struct mips_perf_event *mipspmu_map_cache_event(u64 config) { unsigned int cache_type, cache_op, cache_result; const struct mips_perf_event *pev; @@ -513,9 +509,9 @@ static int validate_group(struct perf_event *event) } /* This is needed by specific irq handlers in perf_event_*.c */ -static void -handle_associated_event(struct cpu_hw_events *cpuc, - int idx, struct perf_sample_data *data, struct pt_regs *regs) +static void handle_associated_event(struct cpu_hw_events *cpuc, + int idx, struct perf_sample_data *data, + struct pt_regs *regs) { struct perf_event *event = cpuc->events[idx]; struct hw_perf_event *hwc = &event->hw; diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index e5ad09a9baf..a5925b5c792 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -49,37 +49,32 @@ static int cpu_has_mipsmt_pertccounters; #endif /* Copied from op_model_mipsxx.c */ -static inline unsigned int vpe_shift(void) +static unsigned int vpe_shift(void) { if (num_possible_cpus() > 1) return 1; return 0; } -#else /* !CONFIG_MIPS_MT_SMP */ -#define vpe_id() 0 - -static inline unsigned int vpe_shift(void) -{ - return 0; -} -#endif /* CONFIG_MIPS_MT_SMP */ -static inline unsigned int -counters_total_to_per_cpu(unsigned int counters) +static unsigned int counters_total_to_per_cpu(unsigned int counters) { return counters >> vpe_shift(); } -static inline unsigned int -counters_per_cpu_to_total(unsigned int counters) +static unsigned int counters_per_cpu_to_total(unsigned int counters) { return counters << vpe_shift(); } +#else /* !CONFIG_MIPS_MT_SMP */ +#define vpe_id() 0 + +#endif /* CONFIG_MIPS_MT_SMP */ + #define __define_perf_accessors(r, n, np) \ \ -static inline unsigned int r_c0_ ## r ## n(void) \ +static unsigned int r_c0_ ## r ## n(void) \ { \ unsigned int cpu = vpe_id(); \ \ @@ -94,7 +89,7 @@ static inline unsigned int r_c0_ ## r ## n(void) \ return 0; \ } \ \ -static inline void w_c0_ ## r ## n(unsigned int value) \ +static void w_c0_ ## r ## n(unsigned int value) \ { \ unsigned int cpu = vpe_id(); \ \ @@ -121,7 +116,7 @@ __define_perf_accessors(perfctrl, 1, 3) __define_perf_accessors(perfctrl, 2, 0) __define_perf_accessors(perfctrl, 3, 1) -static inline int __n_counters(void) +static int __n_counters(void) { if (!(read_c0_config1() & M_CONFIG1_PC)) return 0; @@ -135,7 +130,7 @@ static inline int __n_counters(void) return 4; } -static inline int n_counters(void) +static int n_counters(void) { int counters; @@ -175,8 +170,7 @@ static void reset_counters(void *arg) } } -static inline u64 -mipsxx_pmu_read_counter(unsigned int idx) +static u64 mipsxx_pmu_read_counter(unsigned int idx) { switch (idx) { case 0: @@ -193,8 +187,7 @@ mipsxx_pmu_read_counter(unsigned int idx) } } -static inline void -mipsxx_pmu_write_counter(unsigned int idx, u64 val) +static void mipsxx_pmu_write_counter(unsigned int idx, u64 val) { switch (idx) { case 0: @@ -212,8 +205,7 @@ mipsxx_pmu_write_counter(unsigned int idx, u64 val) } } -static inline unsigned int -mipsxx_pmu_read_control(unsigned int idx) +static unsigned int mipsxx_pmu_read_control(unsigned int idx) { switch (idx) { case 0: @@ -230,8 +222,7 @@ mipsxx_pmu_read_control(unsigned int idx) } } -static inline void -mipsxx_pmu_write_control(unsigned int idx, unsigned int val) +static void mipsxx_pmu_write_control(unsigned int idx, unsigned int val) { switch (idx) { case 0: @@ -511,9 +502,8 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map }; #ifdef CONFIG_MIPS_MT_SMP -static void -check_and_calc_range(struct perf_event *event, - const struct mips_perf_event *pev) +static void check_and_calc_range(struct perf_event *event, + const struct mips_perf_event *pev) { struct hw_perf_event *hwc = &event->hw; @@ -536,9 +526,8 @@ check_and_calc_range(struct perf_event *event, hwc->config_base |= M_TC_EN_ALL; } #else -static void -check_and_calc_range(struct perf_event *event, - const struct mips_perf_event *pev) +static void check_and_calc_range(struct perf_event *event, + const struct mips_perf_event *pev) { } #endif @@ -733,8 +722,7 @@ static int mipsxx_pmu_handle_shared_irq(void) return handled; } -static irqreturn_t -mipsxx_pmu_handle_irq(int irq, void *dev) +static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev) { return mipsxx_pmu_handle_shared_irq(); } @@ -766,9 +754,8 @@ static void mipsxx_pmu_stop(void) #endif } -static int -mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, - struct hw_perf_event *hwc) +static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, + struct hw_perf_event *hwc) { int i; @@ -797,8 +784,7 @@ mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, return -EAGAIN; } -static void -mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) +static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); unsigned long flags; @@ -816,8 +802,7 @@ mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) local_irq_restore(flags); } -static void -mipsxx_pmu_disable_event(int idx) +static void mipsxx_pmu_disable_event(int idx) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); unsigned long flags; @@ -892,8 +877,7 @@ mipsxx_pmu_disable_event(int idx) * then 128 needs to be added to 15 as the input for the event config, * i.e., 143 (0x8F) to be used. */ -static const struct mips_perf_event * -mipsxx_pmu_map_raw_event(u64 config) +static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) { unsigned int raw_id = config & 0xff; unsigned int base_id = raw_id & 0x7f; -- cgit v1.2.3-70-g09d2 From e5dcb58aa51090f462959b9789eb477286bd2279 Mon Sep 17 00:00:00 2001 From: David Daney Date: Sat, 24 Sep 2011 02:29:55 +0200 Subject: MIPS: perf: Reorganize contents of perf support files. The contents of arch/mips/kernel/perf_event.c and arch/mips/kernel/perf_event_mipsxx.c were divided in a seemingly ad hoc manner, with the first including the second. I moved all the hardware counter support code to perf_event_mipsxx.c and removed the gating #ifdefs to the Kconfig and Makefile. Now perf_event.c contains only the callchain support, everything else is in perf_event_mipsxx.c There are no code changes, only moving of functions from one file to the other, or removing empty unneeded functions. Signed-off-by: David Daney Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Dezhong Diao Cc: Gabor Juhos Cc: Deng-Cheng Zhu To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2791/ Signed-off-by: Ralf Baechle --- arch/mips/Kconfig | 2 +- arch/mips/kernel/Makefile | 5 +- arch/mips/kernel/perf_event.c | 515 +--------------------------------- arch/mips/kernel/perf_event_mipsxx.c | 530 ++++++++++++++++++++++++++++++++++- 4 files changed, 532 insertions(+), 520 deletions(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 0674b12841e..90877154f29 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -2101,7 +2101,7 @@ config NODES_SHIFT config HW_PERF_EVENTS bool "Enable hardware performance counter support for perf events" - depends on PERF_EVENTS && !MIPS_MT_SMTC && OPROFILE=n && CPU_MIPS32 + depends on PERF_EVENTS && !MIPS_MT_SMTC && OPROFILE=n && (CPU_MIPS32 || CPU_MIPS64 || CPU_R10000 || CPU_SB1) default y help Enable hardware performance counter support for perf events. If diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile index 83bba332bbf..1a966183e35 100644 --- a/arch/mips/kernel/Makefile +++ b/arch/mips/kernel/Makefile @@ -11,6 +11,8 @@ obj-y += cpu-probe.o branch.o entry.o genex.o irq.o process.o \ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_early_printk.o = -pg +CFLAGS_REMOVE_perf_event.o = -pg +CFLAGS_REMOVE_perf_event_mipsxx.o = -pg endif obj-$(CONFIG_CEVT_BCM1480) += cevt-bcm1480.o @@ -106,7 +108,8 @@ obj-$(CONFIG_HAVE_STD_PC_SERIAL_PORT) += 8250-platform.o obj-$(CONFIG_MIPS_CPUFREQ) += cpufreq/ -obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_mipsxx.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/mips/kernel/perf_event.c b/arch/mips/kernel/perf_event.c index f3d6e99ab57..c1cf9c6c3f7 100644 --- a/arch/mips/kernel/perf_event.c +++ b/arch/mips/kernel/perf_event.c @@ -14,529 +14,16 @@ * published by the Free Software Foundation. */ -#include -#include -#include -#include #include -#include -#include -#include #include -#include /* For perf_irq */ - -/* These are for 32bit counters. For 64bit ones, define them accordingly. */ -#define MAX_PERIOD ((1ULL << 32) - 1) -#define VALID_COUNT 0x7fffffff -#define TOTAL_BITS 32 -#define HIGHEST_BIT 31 - -#define MIPS_MAX_HWEVENTS 4 - -struct cpu_hw_events { - /* Array of events on this cpu. */ - struct perf_event *events[MIPS_MAX_HWEVENTS]; - - /* - * Set the bit (indexed by the counter number) when the counter - * is used for an event. - */ - unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; - - /* - * The borrowed MSB for the performance counter. A MIPS performance - * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit - * counters) as a factor of determining whether a counter overflow - * should be signaled. So here we use a separate MSB for each - * counter to make things easy. - */ - unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; - - /* - * Software copy of the control register for each performance counter. - * MIPS CPUs vary in performance counters. They use this differently, - * and even may not use it. - */ - unsigned int saved_ctrl[MIPS_MAX_HWEVENTS]; -}; -DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { - .saved_ctrl = {0}, -}; - -/* The description of MIPS performance events. */ -struct mips_perf_event { - unsigned int event_id; - /* - * MIPS performance counters are indexed starting from 0. - * CNTR_EVEN indicates the indexes of the counters to be used are - * even numbers. - */ - unsigned int cntr_mask; - #define CNTR_EVEN 0x55555555 - #define CNTR_ODD 0xaaaaaaaa -#ifdef CONFIG_MIPS_MT_SMP - enum { - T = 0, - V = 1, - P = 2, - } range; -#else - #define T - #define V - #define P -#endif -}; - -static struct mips_perf_event raw_event; -static DEFINE_MUTEX(raw_event_mutex); - -#define UNSUPPORTED_PERF_EVENT_ID 0xffffffff -#define C(x) PERF_COUNT_HW_CACHE_##x - -struct mips_pmu { - const char *name; - int irq; - irqreturn_t (*handle_irq)(int irq, void *dev); - int (*handle_shared_irq)(void); - void (*start)(void); - void (*stop)(void); - int (*alloc_counter)(struct cpu_hw_events *cpuc, - struct hw_perf_event *hwc); - u64 (*read_counter)(unsigned int idx); - void (*write_counter)(unsigned int idx, u64 val); - void (*enable_event)(struct hw_perf_event *evt, int idx); - void (*disable_event)(int idx); - const struct mips_perf_event *(*map_raw_event)(u64 config); - const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; - const struct mips_perf_event (*cache_event_map) - [PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX]; - unsigned int num_counters; -}; - -static const struct mips_pmu *mipspmu; - -static int mipspmu_event_set_period(struct perf_event *event, - struct hw_perf_event *hwc, - int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - s64 left = local64_read(&hwc->period_left); - s64 period = hwc->sample_period; - int ret = 0; - u64 uleft; - unsigned long flags; - - if (unlikely(left <= -period)) { - left = period; - local64_set(&hwc->period_left, left); - hwc->last_period = period; - ret = 1; - } - - if (unlikely(left <= 0)) { - left += period; - local64_set(&hwc->period_left, left); - hwc->last_period = period; - ret = 1; - } - - if (left > (s64)MAX_PERIOD) - left = MAX_PERIOD; - - local64_set(&hwc->prev_count, (u64)-left); - - local_irq_save(flags); - uleft = (u64)(-left) & MAX_PERIOD; - uleft > VALID_COUNT ? - set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs); - mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT); - local_irq_restore(flags); - - perf_event_update_userpage(event); - - return ret; -} - -static void mipspmu_event_update(struct perf_event *event, - struct hw_perf_event *hwc, - int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - unsigned long flags; - int shift = 64 - TOTAL_BITS; - s64 prev_raw_count, new_raw_count; - u64 delta; - -again: - prev_raw_count = local64_read(&hwc->prev_count); - local_irq_save(flags); - /* Make the counter value be a "real" one. */ - new_raw_count = mipspmu->read_counter(idx); - if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) { - new_raw_count &= VALID_COUNT; - clear_bit(idx, cpuc->msbs); - } else - new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT); - local_irq_restore(flags); - - if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, - new_raw_count) != prev_raw_count) - goto again; - - delta = (new_raw_count << shift) - (prev_raw_count << shift); - delta >>= shift; - - local64_add(delta, &event->count); - local64_sub(delta, &hwc->period_left); -} - -static void mipspmu_start(struct perf_event *event, int flags) -{ - struct hw_perf_event *hwc = &event->hw; - - if (!mipspmu) - return; - - if (flags & PERF_EF_RELOAD) - WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); - - hwc->state = 0; - - /* Set the period for the event. */ - mipspmu_event_set_period(event, hwc, hwc->idx); - - /* Enable the event. */ - mipspmu->enable_event(hwc, hwc->idx); -} - -static void mipspmu_stop(struct perf_event *event, int flags) -{ - struct hw_perf_event *hwc = &event->hw; - - if (!mipspmu) - return; - - if (!(hwc->state & PERF_HES_STOPPED)) { - /* We are working on a local event. */ - mipspmu->disable_event(hwc->idx); - barrier(); - mipspmu_event_update(event, hwc, hwc->idx); - hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; - } -} - -static int mipspmu_add(struct perf_event *event, int flags) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - struct hw_perf_event *hwc = &event->hw; - int idx; - int err = 0; - - perf_pmu_disable(event->pmu); - - /* To look for a free counter for this event. */ - idx = mipspmu->alloc_counter(cpuc, hwc); - if (idx < 0) { - err = idx; - goto out; - } - - /* - * If there is an event in the counter we are going to use then - * make sure it is disabled. - */ - event->hw.idx = idx; - mipspmu->disable_event(idx); - cpuc->events[idx] = event; - - hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; - if (flags & PERF_EF_START) - mipspmu_start(event, PERF_EF_RELOAD); - - /* Propagate our changes to the userspace mapping. */ - perf_event_update_userpage(event); - -out: - perf_pmu_enable(event->pmu); - return err; -} - -static void mipspmu_del(struct perf_event *event, int flags) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - - WARN_ON(idx < 0 || idx >= mipspmu->num_counters); - - mipspmu_stop(event, PERF_EF_UPDATE); - cpuc->events[idx] = NULL; - clear_bit(idx, cpuc->used_mask); - - perf_event_update_userpage(event); -} - -static void mipspmu_read(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - - /* Don't read disabled counters! */ - if (hwc->idx < 0) - return; - - mipspmu_event_update(event, hwc, hwc->idx); -} - -static void mipspmu_enable(struct pmu *pmu) -{ - if (mipspmu) - mipspmu->start(); -} - -static void mipspmu_disable(struct pmu *pmu) -{ - if (mipspmu) - mipspmu->stop(); -} - -static atomic_t active_events = ATOMIC_INIT(0); -static DEFINE_MUTEX(pmu_reserve_mutex); -static int (*save_perf_irq)(void); - -static int mipspmu_get_irq(void) -{ - int err; - - if (mipspmu->irq >= 0) { - /* Request my own irq handler. */ - err = request_irq(mipspmu->irq, mipspmu->handle_irq, - IRQF_DISABLED | IRQF_NOBALANCING, - "mips_perf_pmu", NULL); - if (err) { - pr_warning("Unable to request IRQ%d for MIPS " - "performance counters!\n", mipspmu->irq); - } - } else if (cp0_perfcount_irq < 0) { - /* - * We are sharing the irq number with the timer interrupt. - */ - save_perf_irq = perf_irq; - perf_irq = mipspmu->handle_shared_irq; - err = 0; - } else { - pr_warning("The platform hasn't properly defined its " - "interrupt controller.\n"); - err = -ENOENT; - } - - return err; -} - -static void mipspmu_free_irq(void) -{ - if (mipspmu->irq >= 0) - free_irq(mipspmu->irq, NULL); - else if (cp0_perfcount_irq < 0) - perf_irq = save_perf_irq; -} - -/* - * mipsxx/rm9000/loongson2 have different performance counters, they have - * specific low-level init routines. - */ -static void reset_counters(void *arg); -static int __hw_perf_event_init(struct perf_event *event); - -static void hw_perf_event_destroy(struct perf_event *event) -{ - if (atomic_dec_and_mutex_lock(&active_events, - &pmu_reserve_mutex)) { - /* - * We must not call the destroy function with interrupts - * disabled. - */ - on_each_cpu(reset_counters, - (void *)(long)mipspmu->num_counters, 1); - mipspmu_free_irq(); - mutex_unlock(&pmu_reserve_mutex); - } -} - -static int mipspmu_event_init(struct perf_event *event) -{ - int err = 0; - - switch (event->attr.type) { - case PERF_TYPE_RAW: - case PERF_TYPE_HARDWARE: - case PERF_TYPE_HW_CACHE: - break; - - default: - return -ENOENT; - } - - if (!mipspmu || event->cpu >= nr_cpumask_bits || - (event->cpu >= 0 && !cpu_online(event->cpu))) - return -ENODEV; - - if (!atomic_inc_not_zero(&active_events)) { - if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { - atomic_dec(&active_events); - return -ENOSPC; - } - - mutex_lock(&pmu_reserve_mutex); - if (atomic_read(&active_events) == 0) - err = mipspmu_get_irq(); - - if (!err) - atomic_inc(&active_events); - mutex_unlock(&pmu_reserve_mutex); - } - - if (err) - return err; - - err = __hw_perf_event_init(event); - if (err) - hw_perf_event_destroy(event); - - return err; -} - -static struct pmu pmu = { - .pmu_enable = mipspmu_enable, - .pmu_disable = mipspmu_disable, - .event_init = mipspmu_event_init, - .add = mipspmu_add, - .del = mipspmu_del, - .start = mipspmu_start, - .stop = mipspmu_stop, - .read = mipspmu_read, -}; - -static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) -{ -/* - * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for - * event_id. - */ -#ifdef CONFIG_MIPS_MT_SMP - return ((unsigned int)pev->range << 24) | - (pev->cntr_mask & 0xffff00) | - (pev->event_id & 0xff); -#else - return (pev->cntr_mask & 0xffff00) | - (pev->event_id & 0xff); -#endif -} - -static const struct mips_perf_event *mipspmu_map_general_event(int idx) -{ - const struct mips_perf_event *pev; - - pev = ((*mipspmu->general_event_map)[idx].event_id == - UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) : - &(*mipspmu->general_event_map)[idx]); - - return pev; -} - -static const struct mips_perf_event *mipspmu_map_cache_event(u64 config) -{ - unsigned int cache_type, cache_op, cache_result; - const struct mips_perf_event *pev; - - cache_type = (config >> 0) & 0xff; - if (cache_type >= PERF_COUNT_HW_CACHE_MAX) - return ERR_PTR(-EINVAL); - - cache_op = (config >> 8) & 0xff; - if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) - return ERR_PTR(-EINVAL); - - cache_result = (config >> 16) & 0xff; - if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) - return ERR_PTR(-EINVAL); - - pev = &((*mipspmu->cache_event_map) - [cache_type] - [cache_op] - [cache_result]); - - if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID) - return ERR_PTR(-EOPNOTSUPP); - - return pev; - -} - -static int validate_event(struct cpu_hw_events *cpuc, - struct perf_event *event) -{ - struct hw_perf_event fake_hwc = event->hw; - - /* Allow mixed event group. So return 1 to pass validation. */ - if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF) - return 1; - - return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0; -} - -static int validate_group(struct perf_event *event) -{ - struct perf_event *sibling, *leader = event->group_leader; - struct cpu_hw_events fake_cpuc; - - memset(&fake_cpuc, 0, sizeof(fake_cpuc)); - - if (!validate_event(&fake_cpuc, leader)) - return -ENOSPC; - - list_for_each_entry(sibling, &leader->sibling_list, group_entry) { - if (!validate_event(&fake_cpuc, sibling)) - return -ENOSPC; - } - - if (!validate_event(&fake_cpuc, event)) - return -ENOSPC; - - return 0; -} - -/* This is needed by specific irq handlers in perf_event_*.c */ -static void handle_associated_event(struct cpu_hw_events *cpuc, - int idx, struct perf_sample_data *data, - struct pt_regs *regs) -{ - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc = &event->hw; - - mipspmu_event_update(event, hwc, idx); - data->period = event->hw.last_period; - if (!mipspmu_event_set_period(event, hwc, idx)) - return; - - if (perf_event_overflow(event, data, regs)) - mipspmu->disable_event(idx); -} - -#include "perf_event_mipsxx.c" /* Callchain handling code. */ /* * Leave userspace callchain empty for now. When we find a way to trace - * the user stack callchains, we add here. + * the user stack callchains, we will add it here. */ -void perf_callchain_user(struct perf_callchain_entry *entry, - struct pt_regs *regs) -{ -} static void save_raw_perf_callchain(struct perf_callchain_entry *entry, unsigned long reg29) diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index a5925b5c792..eb74dce69c1 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -1,5 +1,529 @@ -#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) || \ - defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_SB1) +/* + * Linux performance counter support for MIPS. + * + * Copyright (C) 2010 MIPS Technologies, Inc. + * Author: Deng-Cheng Zhu + * + * This code is based on the implementation for ARM, which is in turn + * based on the sparc64 perf event code and the x86 code. Performance + * counter access is based on the MIPS Oprofile code. And the callchain + * support references the code of MIPS stacktrace.c. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include /* For perf_irq */ + +/* These are for 32bit counters. For 64bit ones, define them accordingly. */ +#define MAX_PERIOD ((1ULL << 32) - 1) +#define VALID_COUNT 0x7fffffff +#define TOTAL_BITS 32 +#define HIGHEST_BIT 31 + +#define MIPS_MAX_HWEVENTS 4 + +struct cpu_hw_events { + /* Array of events on this cpu. */ + struct perf_event *events[MIPS_MAX_HWEVENTS]; + + /* + * Set the bit (indexed by the counter number) when the counter + * is used for an event. + */ + unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; + + /* + * The borrowed MSB for the performance counter. A MIPS performance + * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit + * counters) as a factor of determining whether a counter overflow + * should be signaled. So here we use a separate MSB for each + * counter to make things easy. + */ + unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; + + /* + * Software copy of the control register for each performance counter. + * MIPS CPUs vary in performance counters. They use this differently, + * and even may not use it. + */ + unsigned int saved_ctrl[MIPS_MAX_HWEVENTS]; +}; +DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { + .saved_ctrl = {0}, +}; + +/* The description of MIPS performance events. */ +struct mips_perf_event { + unsigned int event_id; + /* + * MIPS performance counters are indexed starting from 0. + * CNTR_EVEN indicates the indexes of the counters to be used are + * even numbers. + */ + unsigned int cntr_mask; + #define CNTR_EVEN 0x55555555 + #define CNTR_ODD 0xaaaaaaaa +#ifdef CONFIG_MIPS_MT_SMP + enum { + T = 0, + V = 1, + P = 2, + } range; +#else + #define T + #define V + #define P +#endif +}; + +static struct mips_perf_event raw_event; +static DEFINE_MUTEX(raw_event_mutex); + +#define UNSUPPORTED_PERF_EVENT_ID 0xffffffff +#define C(x) PERF_COUNT_HW_CACHE_##x + +struct mips_pmu { + const char *name; + int irq; + irqreturn_t (*handle_irq)(int irq, void *dev); + int (*handle_shared_irq)(void); + void (*start)(void); + void (*stop)(void); + int (*alloc_counter)(struct cpu_hw_events *cpuc, + struct hw_perf_event *hwc); + u64 (*read_counter)(unsigned int idx); + void (*write_counter)(unsigned int idx, u64 val); + void (*enable_event)(struct hw_perf_event *evt, int idx); + void (*disable_event)(int idx); + const struct mips_perf_event *(*map_raw_event)(u64 config); + const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; + const struct mips_perf_event (*cache_event_map) + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; + unsigned int num_counters; +}; + +static const struct mips_pmu *mipspmu; + +static int mipspmu_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + s64 left = local64_read(&hwc->period_left); + s64 period = hwc->sample_period; + int ret = 0; + u64 uleft; + unsigned long flags; + + if (unlikely(left <= -period)) { + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (unlikely(left <= 0)) { + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (left > (s64)MAX_PERIOD) + left = MAX_PERIOD; + + local64_set(&hwc->prev_count, (u64)-left); + + local_irq_save(flags); + uleft = (u64)(-left) & MAX_PERIOD; + uleft > VALID_COUNT ? + set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs); + mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT); + local_irq_restore(flags); + + perf_event_update_userpage(event); + + return ret; +} + +static void mipspmu_event_update(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + unsigned long flags; + int shift = 64 - TOTAL_BITS; + s64 prev_raw_count, new_raw_count; + u64 delta; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + local_irq_save(flags); + /* Make the counter value be a "real" one. */ + new_raw_count = mipspmu->read_counter(idx); + if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) { + new_raw_count &= VALID_COUNT; + clear_bit(idx, cpuc->msbs); + } else + new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT); + local_irq_restore(flags); + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = (new_raw_count << shift) - (prev_raw_count << shift); + delta >>= shift; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); +} + +static void mipspmu_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!mipspmu) + return; + + if (flags & PERF_EF_RELOAD) + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + + hwc->state = 0; + + /* Set the period for the event. */ + mipspmu_event_set_period(event, hwc, hwc->idx); + + /* Enable the event. */ + mipspmu->enable_event(hwc, hwc->idx); +} + +static void mipspmu_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!mipspmu) + return; + + if (!(hwc->state & PERF_HES_STOPPED)) { + /* We are working on a local event. */ + mipspmu->disable_event(hwc->idx); + barrier(); + mipspmu_event_update(event, hwc, hwc->idx); + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; + } +} + +static int mipspmu_add(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx; + int err = 0; + + perf_pmu_disable(event->pmu); + + /* To look for a free counter for this event. */ + idx = mipspmu->alloc_counter(cpuc, hwc); + if (idx < 0) { + err = idx; + goto out; + } + + /* + * If there is an event in the counter we are going to use then + * make sure it is disabled. + */ + event->hw.idx = idx; + mipspmu->disable_event(idx); + cpuc->events[idx] = event; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + if (flags & PERF_EF_START) + mipspmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + +out: + perf_pmu_enable(event->pmu); + return err; +} + +static void mipspmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + WARN_ON(idx < 0 || idx >= mipspmu->num_counters); + + mipspmu_stop(event, PERF_EF_UPDATE); + cpuc->events[idx] = NULL; + clear_bit(idx, cpuc->used_mask); + + perf_event_update_userpage(event); +} + +static void mipspmu_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + /* Don't read disabled counters! */ + if (hwc->idx < 0) + return; + + mipspmu_event_update(event, hwc, hwc->idx); +} + +static void mipspmu_enable(struct pmu *pmu) +{ + if (mipspmu) + mipspmu->start(); +} + +static void mipspmu_disable(struct pmu *pmu) +{ + if (mipspmu) + mipspmu->stop(); +} + +static atomic_t active_events = ATOMIC_INIT(0); +static DEFINE_MUTEX(pmu_reserve_mutex); +static int (*save_perf_irq)(void); + +static int mipspmu_get_irq(void) +{ + int err; + + if (mipspmu->irq >= 0) { + /* Request my own irq handler. */ + err = request_irq(mipspmu->irq, mipspmu->handle_irq, + IRQF_DISABLED | IRQF_NOBALANCING, + "mips_perf_pmu", NULL); + if (err) { + pr_warning("Unable to request IRQ%d for MIPS " + "performance counters!\n", mipspmu->irq); + } + } else if (cp0_perfcount_irq < 0) { + /* + * We are sharing the irq number with the timer interrupt. + */ + save_perf_irq = perf_irq; + perf_irq = mipspmu->handle_shared_irq; + err = 0; + } else { + pr_warning("The platform hasn't properly defined its " + "interrupt controller.\n"); + err = -ENOENT; + } + + return err; +} + +static void mipspmu_free_irq(void) +{ + if (mipspmu->irq >= 0) + free_irq(mipspmu->irq, NULL); + else if (cp0_perfcount_irq < 0) + perf_irq = save_perf_irq; +} + +/* + * mipsxx/rm9000/loongson2 have different performance counters, they have + * specific low-level init routines. + */ +static void reset_counters(void *arg); +static int __hw_perf_event_init(struct perf_event *event); + +static void hw_perf_event_destroy(struct perf_event *event) +{ + if (atomic_dec_and_mutex_lock(&active_events, + &pmu_reserve_mutex)) { + /* + * We must not call the destroy function with interrupts + * disabled. + */ + on_each_cpu(reset_counters, + (void *)(long)mipspmu->num_counters, 1); + mipspmu_free_irq(); + mutex_unlock(&pmu_reserve_mutex); + } +} + +static int mipspmu_event_init(struct perf_event *event) +{ + int err = 0; + + switch (event->attr.type) { + case PERF_TYPE_RAW: + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + break; + + default: + return -ENOENT; + } + + if (!mipspmu || event->cpu >= nr_cpumask_bits || + (event->cpu >= 0 && !cpu_online(event->cpu))) + return -ENODEV; + + if (!atomic_inc_not_zero(&active_events)) { + if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { + atomic_dec(&active_events); + return -ENOSPC; + } + + mutex_lock(&pmu_reserve_mutex); + if (atomic_read(&active_events) == 0) + err = mipspmu_get_irq(); + + if (!err) + atomic_inc(&active_events); + mutex_unlock(&pmu_reserve_mutex); + } + + if (err) + return err; + + err = __hw_perf_event_init(event); + if (err) + hw_perf_event_destroy(event); + + return err; +} + +static struct pmu pmu = { + .pmu_enable = mipspmu_enable, + .pmu_disable = mipspmu_disable, + .event_init = mipspmu_event_init, + .add = mipspmu_add, + .del = mipspmu_del, + .start = mipspmu_start, + .stop = mipspmu_stop, + .read = mipspmu_read, +}; + +static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) +{ +/* + * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for + * event_id. + */ +#ifdef CONFIG_MIPS_MT_SMP + return ((unsigned int)pev->range << 24) | + (pev->cntr_mask & 0xffff00) | + (pev->event_id & 0xff); +#else + return (pev->cntr_mask & 0xffff00) | + (pev->event_id & 0xff); +#endif +} + +static const struct mips_perf_event *mipspmu_map_general_event(int idx) +{ + const struct mips_perf_event *pev; + + pev = ((*mipspmu->general_event_map)[idx].event_id == + UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) : + &(*mipspmu->general_event_map)[idx]); + + return pev; +} + +static const struct mips_perf_event *mipspmu_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result; + const struct mips_perf_event *pev; + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return ERR_PTR(-EINVAL); + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return ERR_PTR(-EINVAL); + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return ERR_PTR(-EINVAL); + + pev = &((*mipspmu->cache_event_map) + [cache_type] + [cache_op] + [cache_result]); + + if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID) + return ERR_PTR(-EOPNOTSUPP); + + return pev; + +} + +static int validate_event(struct cpu_hw_events *cpuc, + struct perf_event *event) +{ + struct hw_perf_event fake_hwc = event->hw; + + /* Allow mixed event group. So return 1 to pass validation. */ + if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF) + return 1; + + return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0; +} + +static int validate_group(struct perf_event *event) +{ + struct perf_event *sibling, *leader = event->group_leader; + struct cpu_hw_events fake_cpuc; + + memset(&fake_cpuc, 0, sizeof(fake_cpuc)); + + if (!validate_event(&fake_cpuc, leader)) + return -ENOSPC; + + list_for_each_entry(sibling, &leader->sibling_list, group_entry) { + if (!validate_event(&fake_cpuc, sibling)) + return -ENOSPC; + } + + if (!validate_event(&fake_cpuc, event)) + return -ENOSPC; + + return 0; +} + +/* This is needed by specific irq handlers in perf_event_*.c */ +static void handle_associated_event(struct cpu_hw_events *cpuc, + int idx, struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc = &event->hw; + + mipspmu_event_update(event, hwc, idx); + data->period = event->hw.last_period; + if (!mipspmu_event_set_period(event, hwc, idx)) + return; + + if (perf_event_overflow(event, data, regs)) + mipspmu->disable_event(idx); +} #define M_CONFIG1_PC (1 << 4) @@ -1062,5 +1586,3 @@ init_hw_perf_events(void) return 0; } early_initcall(init_hw_perf_events); - -#endif /* defined(CONFIG_CPU_MIPS32)... */ -- cgit v1.2.3-70-g09d2 From 82091564cfd7ab8def42777a9c662dbf655c5d25 Mon Sep 17 00:00:00 2001 From: David Daney Date: Sat, 24 Sep 2011 02:29:55 +0200 Subject: MIPS: perf: Add support for 64-bit perf counters. The hard coded constants are moved to struct mips_pmu. All counter register access move to the read_counter and write_counter function pointers, which are set to either 32-bit or 64-bit access methods at initialization time. Many of the function pointers in struct mips_pmu were not needed as there was only a single implementation, these were removed. I couldn't figure out what made struct cpu_hw_events.msbs[] at all useful, so I removed it too. Some functions and other declarations were reordered to reduce the need for forward declarations. Signed-off-by: David Daney Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Deng-Cheng Zhu To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2792/ Signed-off-by: Ralf Baechle --- arch/mips/kernel/perf_event_mipsxx.c | 858 ++++++++++++++++------------------- 1 file changed, 389 insertions(+), 469 deletions(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index eb74dce69c1..0c9549480c4 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -2,6 +2,7 @@ * Linux performance counter support for MIPS. * * Copyright (C) 2010 MIPS Technologies, Inc. + * Copyright (C) 2011 Cavium Networks, Inc. * Author: Deng-Cheng Zhu * * This code is based on the implementation for ARM, which is in turn @@ -26,12 +27,6 @@ #include #include /* For perf_irq */ -/* These are for 32bit counters. For 64bit ones, define them accordingly. */ -#define MAX_PERIOD ((1ULL << 32) - 1) -#define VALID_COUNT 0x7fffffff -#define TOTAL_BITS 32 -#define HIGHEST_BIT 31 - #define MIPS_MAX_HWEVENTS 4 struct cpu_hw_events { @@ -44,15 +39,6 @@ struct cpu_hw_events { */ unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; - /* - * The borrowed MSB for the performance counter. A MIPS performance - * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit - * counters) as a factor of determining whether a counter overflow - * should be signaled. So here we use a separate MSB for each - * counter to make things easy. - */ - unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; - /* * Software copy of the control register for each performance counter. * MIPS CPUs vary in performance counters. They use this differently, @@ -75,6 +61,7 @@ struct mips_perf_event { unsigned int cntr_mask; #define CNTR_EVEN 0x55555555 #define CNTR_ODD 0xaaaaaaaa + #define CNTR_ALL 0xffffffff #ifdef CONFIG_MIPS_MT_SMP enum { T = 0, @@ -95,18 +82,13 @@ static DEFINE_MUTEX(raw_event_mutex); #define C(x) PERF_COUNT_HW_CACHE_##x struct mips_pmu { + u64 max_period; + u64 valid_count; + u64 overflow; const char *name; int irq; - irqreturn_t (*handle_irq)(int irq, void *dev); - int (*handle_shared_irq)(void); - void (*start)(void); - void (*stop)(void); - int (*alloc_counter)(struct cpu_hw_events *cpuc, - struct hw_perf_event *hwc); u64 (*read_counter)(unsigned int idx); void (*write_counter)(unsigned int idx, u64 val); - void (*enable_event)(struct hw_perf_event *evt, int idx); - void (*disable_event)(int idx); const struct mips_perf_event *(*map_raw_event)(u64 config); const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; const struct mips_perf_event (*cache_event_map) @@ -116,44 +98,302 @@ struct mips_pmu { unsigned int num_counters; }; -static const struct mips_pmu *mipspmu; +static struct mips_pmu mipspmu; + +#define M_CONFIG1_PC (1 << 4) + +#define M_PERFCTL_EXL (1 << 0) +#define M_PERFCTL_KERNEL (1 << 1) +#define M_PERFCTL_SUPERVISOR (1 << 2) +#define M_PERFCTL_USER (1 << 3) +#define M_PERFCTL_INTERRUPT_ENABLE (1 << 4) +#define M_PERFCTL_EVENT(event) (((event) & 0x3ff) << 5) +#define M_PERFCTL_VPEID(vpe) ((vpe) << 16) +#define M_PERFCTL_MT_EN(filter) ((filter) << 20) +#define M_TC_EN_ALL M_PERFCTL_MT_EN(0) +#define M_TC_EN_VPE M_PERFCTL_MT_EN(1) +#define M_TC_EN_TC M_PERFCTL_MT_EN(2) +#define M_PERFCTL_TCID(tcid) ((tcid) << 22) +#define M_PERFCTL_WIDE (1 << 30) +#define M_PERFCTL_MORE (1 << 31) + +#define M_PERFCTL_COUNT_EVENT_WHENEVER (M_PERFCTL_EXL | \ + M_PERFCTL_KERNEL | \ + M_PERFCTL_USER | \ + M_PERFCTL_SUPERVISOR | \ + M_PERFCTL_INTERRUPT_ENABLE) + +#ifdef CONFIG_MIPS_MT_SMP +#define M_PERFCTL_CONFIG_MASK 0x3fff801f +#else +#define M_PERFCTL_CONFIG_MASK 0x1f +#endif +#define M_PERFCTL_EVENT_MASK 0xfe0 + + +#ifdef CONFIG_MIPS_MT_SMP +static int cpu_has_mipsmt_pertccounters; + +static DEFINE_RWLOCK(pmuint_rwlock); + +/* + * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because + * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs. + */ +#if defined(CONFIG_HW_PERF_EVENTS) +#define vpe_id() (cpu_has_mipsmt_pertccounters ? \ + 0 : smp_processor_id()) +#else +#define vpe_id() (cpu_has_mipsmt_pertccounters ? \ + 0 : cpu_data[smp_processor_id()].vpe_id) +#endif + +/* Copied from op_model_mipsxx.c */ +static unsigned int vpe_shift(void) +{ + if (num_possible_cpus() > 1) + return 1; + + return 0; +} + +static unsigned int counters_total_to_per_cpu(unsigned int counters) +{ + return counters >> vpe_shift(); +} + +static unsigned int counters_per_cpu_to_total(unsigned int counters) +{ + return counters << vpe_shift(); +} + +#else /* !CONFIG_MIPS_MT_SMP */ +#define vpe_id() 0 + +#endif /* CONFIG_MIPS_MT_SMP */ + +static void resume_local_counters(void); +static void pause_local_counters(void); +static irqreturn_t mipsxx_pmu_handle_irq(int, void *); +static int mipsxx_pmu_handle_shared_irq(void); + +static unsigned int mipsxx_pmu_swizzle_perf_idx(unsigned int idx) +{ + if (vpe_id() == 1) + idx = (idx + 2) & 3; + return idx; +} + +static u64 mipsxx_pmu_read_counter(unsigned int idx) +{ + idx = mipsxx_pmu_swizzle_perf_idx(idx); + + switch (idx) { + case 0: + /* + * The counters are unsigned, we must cast to truncate + * off the high bits. + */ + return (u32)read_c0_perfcntr0(); + case 1: + return (u32)read_c0_perfcntr1(); + case 2: + return (u32)read_c0_perfcntr2(); + case 3: + return (u32)read_c0_perfcntr3(); + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; + } +} + +static u64 mipsxx_pmu_read_counter_64(unsigned int idx) +{ + idx = mipsxx_pmu_swizzle_perf_idx(idx); + + switch (idx) { + case 0: + return read_c0_perfcntr0_64(); + case 1: + return read_c0_perfcntr1_64(); + case 2: + return read_c0_perfcntr2_64(); + case 3: + return read_c0_perfcntr3_64(); + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; + } +} + +static void mipsxx_pmu_write_counter(unsigned int idx, u64 val) +{ + idx = mipsxx_pmu_swizzle_perf_idx(idx); + + switch (idx) { + case 0: + write_c0_perfcntr0(val); + return; + case 1: + write_c0_perfcntr1(val); + return; + case 2: + write_c0_perfcntr2(val); + return; + case 3: + write_c0_perfcntr3(val); + return; + } +} + +static void mipsxx_pmu_write_counter_64(unsigned int idx, u64 val) +{ + idx = mipsxx_pmu_swizzle_perf_idx(idx); + + switch (idx) { + case 0: + write_c0_perfcntr0_64(val); + return; + case 1: + write_c0_perfcntr1_64(val); + return; + case 2: + write_c0_perfcntr2_64(val); + return; + case 3: + write_c0_perfcntr3_64(val); + return; + } +} + +static unsigned int mipsxx_pmu_read_control(unsigned int idx) +{ + idx = mipsxx_pmu_swizzle_perf_idx(idx); + + switch (idx) { + case 0: + return read_c0_perfctrl0(); + case 1: + return read_c0_perfctrl1(); + case 2: + return read_c0_perfctrl2(); + case 3: + return read_c0_perfctrl3(); + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; + } +} + +static void mipsxx_pmu_write_control(unsigned int idx, unsigned int val) +{ + idx = mipsxx_pmu_swizzle_perf_idx(idx); + + switch (idx) { + case 0: + write_c0_perfctrl0(val); + return; + case 1: + write_c0_perfctrl1(val); + return; + case 2: + write_c0_perfctrl2(val); + return; + case 3: + write_c0_perfctrl3(val); + return; + } +} + +static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, + struct hw_perf_event *hwc) +{ + int i; + + /* + * We only need to care the counter mask. The range has been + * checked definitely. + */ + unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; + + for (i = mipspmu.num_counters - 1; i >= 0; i--) { + /* + * Note that some MIPS perf events can be counted by both + * even and odd counters, wheresas many other are only by + * even _or_ odd counters. This introduces an issue that + * when the former kind of event takes the counter the + * latter kind of event wants to use, then the "counter + * allocation" for the latter event will fail. In fact if + * they can be dynamically swapped, they both feel happy. + * But here we leave this issue alone for now. + */ + if (test_bit(i, &cntr_mask) && + !test_and_set_bit(i, cpuc->used_mask)) + return i; + } + + return -EAGAIN; +} + +static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + + WARN_ON(idx < 0 || idx >= mipspmu.num_counters); + + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | + /* Make sure interrupt enabled. */ + M_PERFCTL_INTERRUPT_ENABLE; + /* + * We do not actually let the counter run. Leave it until start(). + */ +} + +static void mipsxx_pmu_disable_event(int idx) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + unsigned long flags; + + WARN_ON(idx < 0 || idx >= mipspmu.num_counters); + + local_irq_save(flags); + cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) & + ~M_PERFCTL_COUNT_EVENT_WHENEVER; + mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]); + local_irq_restore(flags); +} static int mipspmu_event_set_period(struct perf_event *event, struct hw_perf_event *hwc, int idx) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - s64 left = local64_read(&hwc->period_left); - s64 period = hwc->sample_period; + u64 left = local64_read(&hwc->period_left); + u64 period = hwc->sample_period; int ret = 0; - u64 uleft; - unsigned long flags; - if (unlikely(left <= -period)) { + if (unlikely((left + period) & (1ULL << 63))) { + /* left underflowed by more than period. */ left = period; local64_set(&hwc->period_left, left); hwc->last_period = period; ret = 1; - } - - if (unlikely(left <= 0)) { + } else if (unlikely((left + period) <= period)) { + /* left underflowed by less than period. */ left += period; local64_set(&hwc->period_left, left); hwc->last_period = period; ret = 1; } - if (left > (s64)MAX_PERIOD) - left = MAX_PERIOD; + if (left > mipspmu.max_period) { + left = mipspmu.max_period; + local64_set(&hwc->period_left, left); + } - local64_set(&hwc->prev_count, (u64)-left); + local64_set(&hwc->prev_count, mipspmu.overflow - left); - local_irq_save(flags); - uleft = (u64)(-left) & MAX_PERIOD; - uleft > VALID_COUNT ? - set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs); - mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT); - local_irq_restore(flags); + mipspmu.write_counter(idx, mipspmu.overflow - left); perf_event_update_userpage(event); @@ -164,30 +404,18 @@ static void mipspmu_event_update(struct perf_event *event, struct hw_perf_event *hwc, int idx) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - unsigned long flags; - int shift = 64 - TOTAL_BITS; - s64 prev_raw_count, new_raw_count; + u64 prev_raw_count, new_raw_count; u64 delta; again: prev_raw_count = local64_read(&hwc->prev_count); - local_irq_save(flags); - /* Make the counter value be a "real" one. */ - new_raw_count = mipspmu->read_counter(idx); - if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) { - new_raw_count &= VALID_COUNT; - clear_bit(idx, cpuc->msbs); - } else - new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT); - local_irq_restore(flags); + new_raw_count = mipspmu.read_counter(idx); if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) goto again; - delta = (new_raw_count << shift) - (prev_raw_count << shift); - delta >>= shift; + delta = new_raw_count - prev_raw_count; local64_add(delta, &event->count); local64_sub(delta, &hwc->period_left); @@ -197,9 +425,6 @@ static void mipspmu_start(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; - if (!mipspmu) - return; - if (flags & PERF_EF_RELOAD) WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); @@ -209,19 +434,16 @@ static void mipspmu_start(struct perf_event *event, int flags) mipspmu_event_set_period(event, hwc, hwc->idx); /* Enable the event. */ - mipspmu->enable_event(hwc, hwc->idx); + mipsxx_pmu_enable_event(hwc, hwc->idx); } static void mipspmu_stop(struct perf_event *event, int flags) { struct hw_perf_event *hwc = &event->hw; - if (!mipspmu) - return; - if (!(hwc->state & PERF_HES_STOPPED)) { /* We are working on a local event. */ - mipspmu->disable_event(hwc->idx); + mipsxx_pmu_disable_event(hwc->idx); barrier(); mipspmu_event_update(event, hwc, hwc->idx); hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; @@ -238,7 +460,7 @@ static int mipspmu_add(struct perf_event *event, int flags) perf_pmu_disable(event->pmu); /* To look for a free counter for this event. */ - idx = mipspmu->alloc_counter(cpuc, hwc); + idx = mipsxx_pmu_alloc_counter(cpuc, hwc); if (idx < 0) { err = idx; goto out; @@ -249,7 +471,7 @@ static int mipspmu_add(struct perf_event *event, int flags) * make sure it is disabled. */ event->hw.idx = idx; - mipspmu->disable_event(idx); + mipsxx_pmu_disable_event(idx); cpuc->events[idx] = event; hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; @@ -270,7 +492,7 @@ static void mipspmu_del(struct perf_event *event, int flags) struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; - WARN_ON(idx < 0 || idx >= mipspmu->num_counters); + WARN_ON(idx < 0 || idx >= mipspmu.num_counters); mipspmu_stop(event, PERF_EF_UPDATE); cpuc->events[idx] = NULL; @@ -292,14 +514,29 @@ static void mipspmu_read(struct perf_event *event) static void mipspmu_enable(struct pmu *pmu) { - if (mipspmu) - mipspmu->start(); +#ifdef CONFIG_MIPS_MT_SMP + write_unlock(&pmuint_rwlock); +#endif + resume_local_counters(); } +/* + * MIPS performance counters can be per-TC. The control registers can + * not be directly accessed accross CPUs. Hence if we want to do global + * control, we need cross CPU calls. on_each_cpu() can help us, but we + * can not make sure this function is called with interrupts enabled. So + * here we pause local counters and then grab a rwlock and leave the + * counters on other CPUs alone. If any counter interrupt raises while + * we own the write lock, simply pause local counters on that CPU and + * spin in the handler. Also we know we won't be switched to another + * CPU after pausing local counters and before grabbing the lock. + */ static void mipspmu_disable(struct pmu *pmu) { - if (mipspmu) - mipspmu->stop(); + pause_local_counters(); +#ifdef CONFIG_MIPS_MT_SMP + write_lock(&pmuint_rwlock); +#endif } static atomic_t active_events = ATOMIC_INIT(0); @@ -310,21 +547,21 @@ static int mipspmu_get_irq(void) { int err; - if (mipspmu->irq >= 0) { + if (mipspmu.irq >= 0) { /* Request my own irq handler. */ - err = request_irq(mipspmu->irq, mipspmu->handle_irq, - IRQF_DISABLED | IRQF_NOBALANCING, + err = request_irq(mipspmu.irq, mipsxx_pmu_handle_irq, + IRQF_PERCPU | IRQF_NOBALANCING, "mips_perf_pmu", NULL); if (err) { pr_warning("Unable to request IRQ%d for MIPS " - "performance counters!\n", mipspmu->irq); + "performance counters!\n", mipspmu.irq); } } else if (cp0_perfcount_irq < 0) { /* * We are sharing the irq number with the timer interrupt. */ save_perf_irq = perf_irq; - perf_irq = mipspmu->handle_shared_irq; + perf_irq = mipsxx_pmu_handle_shared_irq; err = 0; } else { pr_warning("The platform hasn't properly defined its " @@ -337,8 +574,8 @@ static int mipspmu_get_irq(void) static void mipspmu_free_irq(void) { - if (mipspmu->irq >= 0) - free_irq(mipspmu->irq, NULL); + if (mipspmu.irq >= 0) + free_irq(mipspmu.irq, NULL); else if (cp0_perfcount_irq < 0) perf_irq = save_perf_irq; } @@ -359,7 +596,7 @@ static void hw_perf_event_destroy(struct perf_event *event) * disabled. */ on_each_cpu(reset_counters, - (void *)(long)mipspmu->num_counters, 1); + (void *)(long)mipspmu.num_counters, 1); mipspmu_free_irq(); mutex_unlock(&pmu_reserve_mutex); } @@ -379,8 +616,8 @@ static int mipspmu_event_init(struct perf_event *event) return -ENOENT; } - if (!mipspmu || event->cpu >= nr_cpumask_bits || - (event->cpu >= 0 && !cpu_online(event->cpu))) + if (event->cpu >= nr_cpumask_bits || + (event->cpu >= 0 && !cpu_online(event->cpu))) return -ENODEV; if (!atomic_inc_not_zero(&active_events)) { @@ -439,9 +676,9 @@ static const struct mips_perf_event *mipspmu_map_general_event(int idx) { const struct mips_perf_event *pev; - pev = ((*mipspmu->general_event_map)[idx].event_id == + pev = ((*mipspmu.general_event_map)[idx].event_id == UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) : - &(*mipspmu->general_event_map)[idx]); + &(*mipspmu.general_event_map)[idx]); return pev; } @@ -463,7 +700,7 @@ static const struct mips_perf_event *mipspmu_map_cache_event(u64 config) if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) return ERR_PTR(-EINVAL); - pev = &((*mipspmu->cache_event_map) + pev = &((*mipspmu.cache_event_map) [cache_type] [cache_op] [cache_result]); @@ -484,7 +721,7 @@ static int validate_event(struct cpu_hw_events *cpuc, if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF) return 1; - return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0; + return mipsxx_pmu_alloc_counter(cpuc, &fake_hwc) >= 0; } static int validate_group(struct perf_event *event) @@ -522,123 +759,9 @@ static void handle_associated_event(struct cpu_hw_events *cpuc, return; if (perf_event_overflow(event, data, regs)) - mipspmu->disable_event(idx); + mipsxx_pmu_disable_event(idx); } -#define M_CONFIG1_PC (1 << 4) - -#define M_PERFCTL_EXL (1UL << 0) -#define M_PERFCTL_KERNEL (1UL << 1) -#define M_PERFCTL_SUPERVISOR (1UL << 2) -#define M_PERFCTL_USER (1UL << 3) -#define M_PERFCTL_INTERRUPT_ENABLE (1UL << 4) -#define M_PERFCTL_EVENT(event) (((event) & 0x3ff) << 5) -#define M_PERFCTL_VPEID(vpe) ((vpe) << 16) -#define M_PERFCTL_MT_EN(filter) ((filter) << 20) -#define M_TC_EN_ALL M_PERFCTL_MT_EN(0) -#define M_TC_EN_VPE M_PERFCTL_MT_EN(1) -#define M_TC_EN_TC M_PERFCTL_MT_EN(2) -#define M_PERFCTL_TCID(tcid) ((tcid) << 22) -#define M_PERFCTL_WIDE (1UL << 30) -#define M_PERFCTL_MORE (1UL << 31) - -#define M_PERFCTL_COUNT_EVENT_WHENEVER (M_PERFCTL_EXL | \ - M_PERFCTL_KERNEL | \ - M_PERFCTL_USER | \ - M_PERFCTL_SUPERVISOR | \ - M_PERFCTL_INTERRUPT_ENABLE) - -#ifdef CONFIG_MIPS_MT_SMP -#define M_PERFCTL_CONFIG_MASK 0x3fff801f -#else -#define M_PERFCTL_CONFIG_MASK 0x1f -#endif -#define M_PERFCTL_EVENT_MASK 0xfe0 - -#define M_COUNTER_OVERFLOW (1UL << 31) - -#ifdef CONFIG_MIPS_MT_SMP -static int cpu_has_mipsmt_pertccounters; - -/* - * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because - * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs. - */ -#if defined(CONFIG_HW_PERF_EVENTS) -#define vpe_id() (cpu_has_mipsmt_pertccounters ? \ - 0 : smp_processor_id()) -#else -#define vpe_id() (cpu_has_mipsmt_pertccounters ? \ - 0 : cpu_data[smp_processor_id()].vpe_id) -#endif - -/* Copied from op_model_mipsxx.c */ -static unsigned int vpe_shift(void) -{ - if (num_possible_cpus() > 1) - return 1; - - return 0; -} - -static unsigned int counters_total_to_per_cpu(unsigned int counters) -{ - return counters >> vpe_shift(); -} - -static unsigned int counters_per_cpu_to_total(unsigned int counters) -{ - return counters << vpe_shift(); -} - -#else /* !CONFIG_MIPS_MT_SMP */ -#define vpe_id() 0 - -#endif /* CONFIG_MIPS_MT_SMP */ - -#define __define_perf_accessors(r, n, np) \ - \ -static unsigned int r_c0_ ## r ## n(void) \ -{ \ - unsigned int cpu = vpe_id(); \ - \ - switch (cpu) { \ - case 0: \ - return read_c0_ ## r ## n(); \ - case 1: \ - return read_c0_ ## r ## np(); \ - default: \ - BUG(); \ - } \ - return 0; \ -} \ - \ -static void w_c0_ ## r ## n(unsigned int value) \ -{ \ - unsigned int cpu = vpe_id(); \ - \ - switch (cpu) { \ - case 0: \ - write_c0_ ## r ## n(value); \ - return; \ - case 1: \ - write_c0_ ## r ## np(value); \ - return; \ - default: \ - BUG(); \ - } \ - return; \ -} \ - -__define_perf_accessors(perfcntr, 0, 2) -__define_perf_accessors(perfcntr, 1, 3) -__define_perf_accessors(perfcntr, 2, 0) -__define_perf_accessors(perfcntr, 3, 1) - -__define_perf_accessors(perfctrl, 0, 2) -__define_perf_accessors(perfctrl, 1, 3) -__define_perf_accessors(perfctrl, 2, 0) -__define_perf_accessors(perfctrl, 3, 1) static int __n_counters(void) { @@ -680,94 +803,20 @@ static void reset_counters(void *arg) int counters = (int)(long)arg; switch (counters) { case 4: - w_c0_perfctrl3(0); - w_c0_perfcntr3(0); - case 3: - w_c0_perfctrl2(0); - w_c0_perfcntr2(0); - case 2: - w_c0_perfctrl1(0); - w_c0_perfcntr1(0); - case 1: - w_c0_perfctrl0(0); - w_c0_perfcntr0(0); - } -} - -static u64 mipsxx_pmu_read_counter(unsigned int idx) -{ - switch (idx) { - case 0: - return r_c0_perfcntr0(); - case 1: - return r_c0_perfcntr1(); - case 2: - return r_c0_perfcntr2(); + mipsxx_pmu_write_control(3, 0); + mipspmu.write_counter(3, 0); case 3: - return r_c0_perfcntr3(); - default: - WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); - return 0; - } -} - -static void mipsxx_pmu_write_counter(unsigned int idx, u64 val) -{ - switch (idx) { - case 0: - w_c0_perfcntr0(val); - return; - case 1: - w_c0_perfcntr1(val); - return; + mipsxx_pmu_write_control(2, 0); + mipspmu.write_counter(2, 0); case 2: - w_c0_perfcntr2(val); - return; - case 3: - w_c0_perfcntr3(val); - return; - } -} - -static unsigned int mipsxx_pmu_read_control(unsigned int idx) -{ - switch (idx) { - case 0: - return r_c0_perfctrl0(); + mipsxx_pmu_write_control(1, 0); + mipspmu.write_counter(1, 0); case 1: - return r_c0_perfctrl1(); - case 2: - return r_c0_perfctrl2(); - case 3: - return r_c0_perfctrl3(); - default: - WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); - return 0; + mipsxx_pmu_write_control(0, 0); + mipspmu.write_counter(0, 0); } } -static void mipsxx_pmu_write_control(unsigned int idx, unsigned int val) -{ - switch (idx) { - case 0: - w_c0_perfctrl0(val); - return; - case 1: - w_c0_perfctrl1(val); - return; - case 2: - w_c0_perfctrl2(val); - return; - case 3: - w_c0_perfctrl3(val); - return; - } -} - -#ifdef CONFIG_MIPS_MT_SMP -static DEFINE_RWLOCK(pmuint_rwlock); -#endif - /* 24K/34K/1004K cores can share the same event map. */ static const struct mips_perf_event mipsxxcore_event_map [PERF_COUNT_HW_MAX] = { @@ -1073,7 +1122,7 @@ static int __hw_perf_event_init(struct perf_event *event) } else if (PERF_TYPE_RAW == event->attr.type) { /* We are working on the global raw event. */ mutex_lock(&raw_event_mutex); - pev = mipspmu->map_raw_event(event->attr.config); + pev = mipspmu.map_raw_event(event->attr.config); } else { /* The event type is not (yet) supported. */ return -EOPNOTSUPP; @@ -1118,7 +1167,7 @@ static int __hw_perf_event_init(struct perf_event *event) hwc->config = 0; if (!hwc->sample_period) { - hwc->sample_period = MAX_PERIOD; + hwc->sample_period = mipspmu.max_period; hwc->last_period = hwc->sample_period; local64_set(&hwc->period_left, hwc->sample_period); } @@ -1131,70 +1180,47 @@ static int __hw_perf_event_init(struct perf_event *event) } event->destroy = hw_perf_event_destroy; - return err; } static void pause_local_counters(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - int counters = mipspmu->num_counters; + int ctr = mipspmu.num_counters; unsigned long flags; local_irq_save(flags); - switch (counters) { - case 4: - cpuc->saved_ctrl[3] = r_c0_perfctrl3(); - w_c0_perfctrl3(cpuc->saved_ctrl[3] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - case 3: - cpuc->saved_ctrl[2] = r_c0_perfctrl2(); - w_c0_perfctrl2(cpuc->saved_ctrl[2] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - case 2: - cpuc->saved_ctrl[1] = r_c0_perfctrl1(); - w_c0_perfctrl1(cpuc->saved_ctrl[1] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - case 1: - cpuc->saved_ctrl[0] = r_c0_perfctrl0(); - w_c0_perfctrl0(cpuc->saved_ctrl[0] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - } + do { + ctr--; + cpuc->saved_ctrl[ctr] = mipsxx_pmu_read_control(ctr); + mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] & + ~M_PERFCTL_COUNT_EVENT_WHENEVER); + } while (ctr > 0); local_irq_restore(flags); } static void resume_local_counters(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - int counters = mipspmu->num_counters; - unsigned long flags; + int ctr = mipspmu.num_counters; - local_irq_save(flags); - switch (counters) { - case 4: - w_c0_perfctrl3(cpuc->saved_ctrl[3]); - case 3: - w_c0_perfctrl2(cpuc->saved_ctrl[2]); - case 2: - w_c0_perfctrl1(cpuc->saved_ctrl[1]); - case 1: - w_c0_perfctrl0(cpuc->saved_ctrl[0]); - } - local_irq_restore(flags); + do { + ctr--; + mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]); + } while (ctr > 0); } static int mipsxx_pmu_handle_shared_irq(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); struct perf_sample_data data; - unsigned int counters = mipspmu->num_counters; - unsigned int counter; + unsigned int counters = mipspmu.num_counters; + u64 counter; int handled = IRQ_NONE; struct pt_regs *regs; if (cpu_has_mips_r2 && !(read_c0_cause() & (1 << 26))) return handled; - /* * First we pause the local counters, so that when we are locked * here, the counters are all paused. When it gets locked due to @@ -1215,13 +1241,9 @@ static int mipsxx_pmu_handle_shared_irq(void) #define HANDLE_COUNTER(n) \ case n + 1: \ if (test_bit(n, cpuc->used_mask)) { \ - counter = r_c0_perfcntr ## n(); \ - if (counter & M_COUNTER_OVERFLOW) { \ - w_c0_perfcntr ## n(counter & \ - VALID_COUNT); \ - if (test_and_change_bit(n, cpuc->msbs)) \ - handle_associated_event(cpuc, \ - n, &data, regs); \ + counter = mipspmu.read_counter(n); \ + if (counter & mipspmu.overflow) { \ + handle_associated_event(cpuc, n, &data, regs); \ handled = IRQ_HANDLED; \ } \ } @@ -1251,95 +1273,6 @@ static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev) return mipsxx_pmu_handle_shared_irq(); } -static void mipsxx_pmu_start(void) -{ -#ifdef CONFIG_MIPS_MT_SMP - write_unlock(&pmuint_rwlock); -#endif - resume_local_counters(); -} - -/* - * MIPS performance counters can be per-TC. The control registers can - * not be directly accessed across CPUs. Hence if we want to do global - * control, we need cross CPU calls. on_each_cpu() can help us, but we - * can not make sure this function is called with interrupts enabled. So - * here we pause local counters and then grab a rwlock and leave the - * counters on other CPUs alone. If any counter interrupt raises while - * we own the write lock, simply pause local counters on that CPU and - * spin in the handler. Also we know we won't be switched to another - * CPU after pausing local counters and before grabbing the lock. - */ -static void mipsxx_pmu_stop(void) -{ - pause_local_counters(); -#ifdef CONFIG_MIPS_MT_SMP - write_lock(&pmuint_rwlock); -#endif -} - -static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, - struct hw_perf_event *hwc) -{ - int i; - - /* - * We only need to care the counter mask. The range has been - * checked definitely. - */ - unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; - - for (i = mipspmu->num_counters - 1; i >= 0; i--) { - /* - * Note that some MIPS perf events can be counted by both - * even and odd counters, wheresas many other are only by - * even _or_ odd counters. This introduces an issue that - * when the former kind of event takes the counter the - * latter kind of event wants to use, then the "counter - * allocation" for the latter event will fail. In fact if - * they can be dynamically swapped, they both feel happy. - * But here we leave this issue alone for now. - */ - if (test_bit(i, &cntr_mask) && - !test_and_set_bit(i, cpuc->used_mask)) - return i; - } - - return -EAGAIN; -} - -static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - unsigned long flags; - - WARN_ON(idx < 0 || idx >= mipspmu->num_counters); - - local_irq_save(flags); - cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | - (evt->config_base & M_PERFCTL_CONFIG_MASK) | - /* Make sure interrupt enabled. */ - M_PERFCTL_INTERRUPT_ENABLE; - /* - * We do not actually let the counter run. Leave it until start(). - */ - local_irq_restore(flags); -} - -static void mipsxx_pmu_disable_event(int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - unsigned long flags; - - WARN_ON(idx < 0 || idx >= mipspmu->num_counters); - - local_irq_save(flags); - cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) & - ~M_PERFCTL_COUNT_EVENT_WHENEVER; - mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]); - local_irq_restore(flags); -} - /* 24K */ #define IS_UNSUPPORTED_24K_EVENT(r, b) \ ((b) == 12 || (r) == 151 || (r) == 152 || (b) == 26 || \ @@ -1478,40 +1411,11 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) return &raw_event; } -static struct mips_pmu mipsxxcore_pmu = { - .handle_irq = mipsxx_pmu_handle_irq, - .handle_shared_irq = mipsxx_pmu_handle_shared_irq, - .start = mipsxx_pmu_start, - .stop = mipsxx_pmu_stop, - .alloc_counter = mipsxx_pmu_alloc_counter, - .read_counter = mipsxx_pmu_read_counter, - .write_counter = mipsxx_pmu_write_counter, - .enable_event = mipsxx_pmu_enable_event, - .disable_event = mipsxx_pmu_disable_event, - .map_raw_event = mipsxx_pmu_map_raw_event, - .general_event_map = &mipsxxcore_event_map, - .cache_event_map = &mipsxxcore_cache_map, -}; - -static struct mips_pmu mipsxx74Kcore_pmu = { - .handle_irq = mipsxx_pmu_handle_irq, - .handle_shared_irq = mipsxx_pmu_handle_shared_irq, - .start = mipsxx_pmu_start, - .stop = mipsxx_pmu_stop, - .alloc_counter = mipsxx_pmu_alloc_counter, - .read_counter = mipsxx_pmu_read_counter, - .write_counter = mipsxx_pmu_write_counter, - .enable_event = mipsxx_pmu_enable_event, - .disable_event = mipsxx_pmu_disable_event, - .map_raw_event = mipsxx_pmu_map_raw_event, - .general_event_map = &mipsxx74Kcore_event_map, - .cache_event_map = &mipsxx74Kcore_cache_map, -}; - static int __init init_hw_perf_events(void) { int counters, irq; + int counter_bits; pr_info("Performance counters: "); @@ -1543,32 +1447,28 @@ init_hw_perf_events(void) } #endif - on_each_cpu(reset_counters, (void *)(long)counters, 1); + mipspmu.map_raw_event = mipsxx_pmu_map_raw_event; switch (current_cpu_type()) { case CPU_24K: - mipsxxcore_pmu.name = "mips/24K"; - mipsxxcore_pmu.num_counters = counters; - mipsxxcore_pmu.irq = irq; - mipspmu = &mipsxxcore_pmu; + mipspmu.name = "mips/24K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; break; case CPU_34K: - mipsxxcore_pmu.name = "mips/34K"; - mipsxxcore_pmu.num_counters = counters; - mipsxxcore_pmu.irq = irq; - mipspmu = &mipsxxcore_pmu; + mipspmu.name = "mips/34K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; break; case CPU_74K: - mipsxx74Kcore_pmu.name = "mips/74K"; - mipsxx74Kcore_pmu.num_counters = counters; - mipsxx74Kcore_pmu.irq = irq; - mipspmu = &mipsxx74Kcore_pmu; + mipspmu.name = "mips/74K"; + mipspmu.general_event_map = &mipsxx74Kcore_event_map; + mipspmu.cache_event_map = &mipsxx74Kcore_cache_map; break; case CPU_1004K: - mipsxxcore_pmu.name = "mips/1004K"; - mipsxxcore_pmu.num_counters = counters; - mipsxxcore_pmu.irq = irq; - mipspmu = &mipsxxcore_pmu; + mipspmu.name = "mips/1004K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; break; default: pr_cont("Either hardware does not support performance " @@ -1576,10 +1476,30 @@ init_hw_perf_events(void) return -ENODEV; } - if (mipspmu) - pr_cont("%s PMU enabled, %d counters available to each " - "CPU, irq %d%s\n", mipspmu->name, counters, irq, - irq < 0 ? " (share with timer interrupt)" : ""); + mipspmu.num_counters = counters; + mipspmu.irq = irq; + + if (read_c0_perfctrl0() & M_PERFCTL_WIDE) { + mipspmu.max_period = (1ULL << 63) - 1; + mipspmu.valid_count = (1ULL << 63) - 1; + mipspmu.overflow = 1ULL << 63; + mipspmu.read_counter = mipsxx_pmu_read_counter_64; + mipspmu.write_counter = mipsxx_pmu_write_counter_64; + counter_bits = 64; + } else { + mipspmu.max_period = (1ULL << 31) - 1; + mipspmu.valid_count = (1ULL << 31) - 1; + mipspmu.overflow = 1ULL << 31; + mipspmu.read_counter = mipsxx_pmu_read_counter; + mipspmu.write_counter = mipsxx_pmu_write_counter; + counter_bits = 32; + } + + on_each_cpu(reset_counters, (void *)(long)counters, 1); + + pr_cont("%s PMU enabled, %d %d-bit counters available to each " + "CPU, irq %d%s\n", mipspmu.name, counters, counter_bits, irq, + irq < 0 ? " (share with timer interrupt)" : ""); perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); -- cgit v1.2.3-70-g09d2 From 939991cff173f769efb8c56286d4e59fb9ced191 Mon Sep 17 00:00:00 2001 From: David Daney Date: Sat, 24 Sep 2011 02:29:55 +0200 Subject: MIPS: perf: Add Octeon support for hardware perf. Enable hardware counters for Octeon, and add the corresponding event mappings. Signed-off-by: David Daney Cc: Peter Zijlstra Cc: Paul Mackerras Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Deng-Cheng Zhu To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2790/ Signed-off-by: Ralf Baechle --- arch/mips/Kconfig | 2 +- arch/mips/kernel/perf_event_mipsxx.c | 147 +++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) (limited to 'arch/mips/kernel') diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 90877154f29..a9264843824 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -2101,7 +2101,7 @@ config NODES_SHIFT config HW_PERF_EVENTS bool "Enable hardware performance counter support for perf events" - depends on PERF_EVENTS && !MIPS_MT_SMTC && OPROFILE=n && (CPU_MIPS32 || CPU_MIPS64 || CPU_R10000 || CPU_SB1) + depends on PERF_EVENTS && !MIPS_MT_SMTC && OPROFILE=n && (CPU_MIPS32 || CPU_MIPS64 || CPU_R10000 || CPU_SB1 || CPU_CAVIUM_OCTEON) default y help Enable hardware performance counter support for perf events. If diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index 0c9549480c4..4f2971bcf8e 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -841,6 +841,16 @@ static const struct mips_perf_event mipsxx74Kcore_event_map [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID }, }; +static const struct mips_perf_event octeon_event_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x03, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x2b, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x2e, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x08, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x09, CNTR_ALL }, + [PERF_COUNT_HW_BUS_CYCLES] = { 0x25, CNTR_ALL }, +}; + /* 24K/34K/1004K cores can share the same cache event map. */ static const struct mips_perf_event mipsxxcore_cache_map [PERF_COUNT_HW_CACHE_MAX] @@ -1074,6 +1084,102 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map }, }; + +static const struct mips_perf_event octeon_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x2b, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x30, CNTR_ALL }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x18, CNTR_ALL }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 0x19, CNTR_ALL }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, +[C(DTLB)] = { + /* + * Only general DTLB misses are counted use the same event for + * read and write. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, +}; + #ifdef CONFIG_MIPS_MT_SMP static void check_and_calc_range(struct perf_event *event, const struct mips_perf_event *pev) @@ -1411,6 +1517,39 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) return &raw_event; } +static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config) +{ + unsigned int raw_id = config & 0xff; + unsigned int base_id = raw_id & 0x7f; + + + raw_event.cntr_mask = CNTR_ALL; + raw_event.event_id = base_id; + + if (current_cpu_type() == CPU_CAVIUM_OCTEON2) { + if (base_id > 0x42) + return ERR_PTR(-EOPNOTSUPP); + } else { + if (base_id > 0x3a) + return ERR_PTR(-EOPNOTSUPP); + } + + switch (base_id) { + case 0x00: + case 0x0f: + case 0x1e: + case 0x1f: + case 0x2f: + case 0x34: + case 0x3b ... 0x3f: + return ERR_PTR(-EOPNOTSUPP); + default: + break; + } + + return &raw_event; +} + static int __init init_hw_perf_events(void) { @@ -1470,6 +1609,14 @@ init_hw_perf_events(void) mipspmu.general_event_map = &mipsxxcore_event_map; mipspmu.cache_event_map = &mipsxxcore_cache_map; break; + case CPU_CAVIUM_OCTEON: + case CPU_CAVIUM_OCTEON_PLUS: + case CPU_CAVIUM_OCTEON2: + mipspmu.name = "octeon"; + mipspmu.general_event_map = &octeon_event_map; + mipspmu.cache_event_map = &octeon_cache_map; + mipspmu.map_raw_event = octeon_pmu_map_raw_event; + break; default: pr_cont("Either hardware does not support performance " "counters, or not yet implemented.\n"); -- cgit v1.2.3-70-g09d2