From 0becb088501886f37ade38762c8eaaf4263572cc Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 24 Jul 2009 12:32:53 +0100 Subject: Thumb-2: Add macros for the unified assembler syntax This patch adds various C and assembler macros that help with using the unified assembler syntax for compiling files to either ARM or Thumb-2 modes. Signed-off-by: Catalin Marinas --- arch/arm/include/asm/unified.h | 126 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 arch/arm/include/asm/unified.h (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/unified.h b/arch/arm/include/asm/unified.h new file mode 100644 index 00000000000..073e85b9b96 --- /dev/null +++ b/arch/arm/include/asm/unified.h @@ -0,0 +1,126 @@ +/* + * include/asm-arm/unified.h - Unified Assembler Syntax helper macros + * + * Copyright (C) 2008 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ASM_UNIFIED_H +#define __ASM_UNIFIED_H + +#if defined(__ASSEMBLY__) && defined(CONFIG_ARM_ASM_UNIFIED) + .syntax unified +#endif + +#ifdef CONFIG_THUMB2_KERNEL + +#if __GNUC__ < 4 +#error Thumb-2 kernel requires gcc >= 4 +#endif + +/* The CPSR bit describing the instruction set (Thumb) */ +#define PSR_ISETSTATE PSR_T_BIT + +#define ARM(x...) +#define THUMB(x...) x +#define W(instr) instr.w +#define BSYM(sym) sym + 1 + +#else /* !CONFIG_THUMB2_KERNEL */ + +/* The CPSR bit describing the instruction set (ARM) */ +#define PSR_ISETSTATE 0 + +#define ARM(x...) x +#define THUMB(x...) +#define W(instr) instr +#define BSYM(sym) sym + +#endif /* CONFIG_THUMB2_KERNEL */ + +#ifndef CONFIG_ARM_ASM_UNIFIED + +/* + * If the unified assembly syntax isn't used (in ARM mode), these + * macros expand to an empty string + */ +#ifdef __ASSEMBLY__ + .macro it, cond + .endm + .macro itt, cond + .endm + .macro ite, cond + .endm + .macro ittt, cond + .endm + .macro itte, cond + .endm + .macro itet, cond + .endm + .macro itee, cond + .endm + .macro itttt, cond + .endm + .macro ittte, cond + .endm + .macro ittet, cond + .endm + .macro ittee, cond + .endm + .macro itett, cond + .endm + .macro itete, cond + .endm + .macro iteet, cond + .endm + .macro iteee, cond + .endm +#else /* !__ASSEMBLY__ */ +__asm__( +" .macro it, cond\n" +" .endm\n" +" .macro itt, cond\n" +" .endm\n" +" .macro ite, cond\n" +" .endm\n" +" .macro ittt, cond\n" +" .endm\n" +" .macro itte, cond\n" +" .endm\n" +" .macro itet, cond\n" +" .endm\n" +" .macro itee, cond\n" +" .endm\n" +" .macro itttt, cond\n" +" .endm\n" +" .macro ittte, cond\n" +" .endm\n" +" .macro ittet, cond\n" +" .endm\n" +" .macro ittee, cond\n" +" .endm\n" +" .macro itett, cond\n" +" .endm\n" +" .macro itete, cond\n" +" .endm\n" +" .macro iteet, cond\n" +" .endm\n" +" .macro iteee, cond\n" +" .endm\n"); +#endif /* __ASSEMBLY__ */ + +#endif /* CONFIG_ARM_ASM_UNIFIED */ + +#endif /* !__ASM_UNIFIED_H */ -- cgit v1.2.3-70-g09d2 From b86040a59feb255a8193173caa4d5199464433d5 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 24 Jul 2009 12:32:54 +0100 Subject: Thumb-2: Implementation of the unified start-up and exceptions code This patch implements the ARM/Thumb-2 unified kernel start-up and exception handling code. Signed-off-by: Catalin Marinas --- arch/arm/include/asm/assembler.h | 11 +++ arch/arm/include/asm/futex.h | 1 + arch/arm/kernel/entry-armv.S | 165 +++++++++++++++++++++++---------------- arch/arm/kernel/entry-common.S | 28 ++----- arch/arm/kernel/entry-header.S | 92 ++++++++++++++++++++-- arch/arm/kernel/head-common.S | 13 +-- arch/arm/kernel/head-nommu.S | 11 ++- arch/arm/kernel/head.S | 28 ++++--- arch/arm/kernel/process.c | 2 +- arch/arm/kernel/setup.c | 28 +++++-- arch/arm/kernel/unwind.c | 4 + 11 files changed, 263 insertions(+), 120 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 15f8a092b70..1acd1da36d0 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -127,3 +127,14 @@ #endif #endif .endm + +#ifdef CONFIG_THUMB2_KERNEL + .macro setmode, mode, reg + mov \reg, #\mode + msr cpsr_c, \reg + .endm +#else + .macro setmode, mode, reg + msr cpsr_c, #\mode + .endm +#endif diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 9ee743b95de..bfcc15929a7 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -99,6 +99,7 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: ldrt %0, [%3]\n" " teq %0, %1\n" + " it eq @ explicit IT needed for the 2b label\n" "2: streqt %2, [%3]\n" "3:\n" " .section __ex_table,\"a\"\n" diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 0befd1cabf4..468425f937d 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -34,7 +34,7 @@ @ @ routine called with r0 = irq number, r1 = struct pt_regs * @ - adrne lr, 1b + adrne lr, BSYM(1b) bne asm_do_IRQ #ifdef CONFIG_SMP @@ -46,13 +46,13 @@ */ test_for_ipi r0, r6, r5, lr movne r0, sp - adrne lr, 1b + adrne lr, BSYM(1b) bne do_IPI #ifdef CONFIG_LOCAL_TIMERS test_for_ltirq r0, r6, r5, lr movne r0, sp - adrne lr, 1b + adrne lr, BSYM(1b) bne do_local_timer #endif #endif @@ -70,7 +70,10 @@ */ .macro inv_entry, reason sub sp, sp, #S_FRAME_SIZE - stmib sp, {r1 - lr} + ARM( stmib sp, {r1 - lr} ) + THUMB( stmia sp, {r0 - r12} ) + THUMB( str sp, [sp, #S_SP] ) + THUMB( str lr, [sp, #S_LR] ) mov r1, #\reason .endm @@ -126,17 +129,24 @@ ENDPROC(__und_invalid) .macro svc_entry, stack_hole=0 UNWIND(.fnstart ) UNWIND(.save {r0 - pc} ) - sub sp, sp, #(S_FRAME_SIZE + \stack_hole) + sub sp, sp, #(S_FRAME_SIZE + \stack_hole - 4) +#ifdef CONFIG_THUMB2_KERNEL + SPFIX( str r0, [sp] ) @ temporarily saved + SPFIX( mov r0, sp ) + SPFIX( tst r0, #4 ) @ test original stack alignment + SPFIX( ldr r0, [sp] ) @ restored +#else SPFIX( tst sp, #4 ) - SPFIX( bicne sp, sp, #4 ) - stmib sp, {r1 - r12} +#endif + SPFIX( subeq sp, sp, #4 ) + stmia sp, {r1 - r12} ldmia r0, {r1 - r3} - add r5, sp, #S_SP @ here for interlock avoidance + add r5, sp, #S_SP - 4 @ here for interlock avoidance mov r4, #-1 @ "" "" "" "" - add r0, sp, #(S_FRAME_SIZE + \stack_hole) - SPFIX( addne r0, r0, #4 ) - str r1, [sp] @ save the "real" r0 copied + add r0, sp, #(S_FRAME_SIZE + \stack_hole - 4) + SPFIX( addeq r0, r0, #4 ) + str r1, [sp, #-4]! @ save the "real" r0 copied @ from the exception stack mov r1, lr @@ -196,9 +206,8 @@ __dabt_svc: @ @ restore SPSR and restart the instruction @ - ldr r0, [sp, #S_PSR] - msr spsr_cxsf, r0 - ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr + ldr r2, [sp, #S_PSR] + svc_exit r2 @ return from exception UNWIND(.fnend ) ENDPROC(__dabt_svc) @@ -225,13 +234,12 @@ __irq_svc: tst r0, #_TIF_NEED_RESCHED blne svc_preempt #endif - ldr r0, [sp, #S_PSR] @ irqs are already disabled - msr spsr_cxsf, r0 + ldr r4, [sp, #S_PSR] @ irqs are already disabled #ifdef CONFIG_TRACE_IRQFLAGS - tst r0, #PSR_I_BIT + tst r4, #PSR_I_BIT bleq trace_hardirqs_on #endif - ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr + svc_exit r4 @ return from exception UNWIND(.fnend ) ENDPROC(__irq_svc) @@ -266,7 +274,7 @@ __und_svc: @ r0 - instruction @ ldr r0, [r2, #-4] - adr r9, 1f + adr r9, BSYM(1f) bl call_fpe mov r0, sp @ struct pt_regs *regs @@ -280,9 +288,8 @@ __und_svc: @ @ restore SPSR and restart the instruction @ - ldr lr, [sp, #S_PSR] @ Get SVC cpsr - msr spsr_cxsf, lr - ldmia sp, {r0 - pc}^ @ Restore SVC registers + ldr r2, [sp, #S_PSR] @ Get SVC cpsr + svc_exit r2 @ return from exception UNWIND(.fnend ) ENDPROC(__und_svc) @@ -323,9 +330,8 @@ __pabt_svc: @ @ restore SPSR and restart the instruction @ - ldr r0, [sp, #S_PSR] - msr spsr_cxsf, r0 - ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr + ldr r2, [sp, #S_PSR] + svc_exit r2 @ return from exception UNWIND(.fnend ) ENDPROC(__pabt_svc) @@ -353,7 +359,8 @@ ENDPROC(__pabt_svc) UNWIND(.fnstart ) UNWIND(.cantunwind ) @ don't unwind the user space sub sp, sp, #S_FRAME_SIZE - stmib sp, {r1 - r12} + ARM( stmib sp, {r1 - r12} ) + THUMB( stmia sp, {r0 - r12} ) ldmia r0, {r1 - r3} add r0, sp, #S_PC @ here for interlock avoidance @@ -372,7 +379,8 @@ ENDPROC(__pabt_svc) @ Also, separately save sp_usr and lr_usr @ stmia r0, {r2 - r4} - stmdb r0, {sp, lr}^ + ARM( stmdb r0, {sp, lr}^ ) + THUMB( store_user_sp_lr r0, r1, S_SP - S_PC ) @ @ Enable the alignment trap while in kernel mode @@ -427,7 +435,7 @@ __dabt_usr: @ enable_irq mov r2, sp - adr lr, ret_from_exception + adr lr, BSYM(ret_from_exception) b do_DataAbort UNWIND(.fnend ) ENDPROC(__dabt_usr) @@ -452,7 +460,9 @@ __irq_usr: ldr r0, [tsk, #TI_PREEMPT] str r8, [tsk, #TI_PREEMPT] teq r0, r7 - strne r0, [r0, -r0] + ARM( strne r0, [r0, -r0] ) + THUMB( movne r0, #0 ) + THUMB( strne r0, [r0] ) #endif #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_on @@ -476,9 +486,10 @@ __und_usr: @ @ r0 - instruction @ - adr r9, ret_from_exception - adr lr, __und_usr_unknown + adr r9, BSYM(ret_from_exception) + adr lr, BSYM(__und_usr_unknown) tst r3, #PSR_T_BIT @ Thumb mode? + itet eq @ explicit IT needed for the 1f label subeq r4, r2, #4 @ ARM instr at LR - 4 subne r4, r2, #2 @ Thumb instr at LR - 2 1: ldreqt r0, [r4] @@ -488,7 +499,10 @@ __und_usr: beq call_fpe @ Thumb instruction #if __LINUX_ARM_ARCH__ >= 7 -2: ldrht r5, [r4], #2 +2: + ARM( ldrht r5, [r4], #2 ) + THUMB( ldrht r5, [r4] ) + THUMB( add r4, r4, #2 ) and r0, r5, #0xf800 @ mask bits 111x x... .... .... cmp r0, #0xe800 @ 32bit instruction if xx != 0 blo __und_usr_unknown @@ -577,9 +591,11 @@ call_fpe: moveq pc, lr get_thread_info r10 @ get current thread and r8, r0, #0x00000f00 @ mask out CP number + THUMB( lsr r8, r8, #8 ) mov r7, #1 add r6, r10, #TI_USED_CP - strb r7, [r6, r8, lsr #8] @ set appropriate used_cp[] + ARM( strb r7, [r6, r8, lsr #8] ) @ set appropriate used_cp[] + THUMB( strb r7, [r6, r8] ) @ set appropriate used_cp[] #ifdef CONFIG_IWMMXT @ Test if we need to give access to iWMMXt coprocessors ldr r5, [r10, #TI_FLAGS] @@ -587,36 +603,38 @@ call_fpe: movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1) bcs iwmmxt_task_enable #endif - add pc, pc, r8, lsr #6 - mov r0, r0 - - mov pc, lr @ CP#0 - b do_fpe @ CP#1 (FPE) - b do_fpe @ CP#2 (FPE) - mov pc, lr @ CP#3 + ARM( add pc, pc, r8, lsr #6 ) + THUMB( lsl r8, r8, #2 ) + THUMB( add pc, r8 ) + nop + + W(mov) pc, lr @ CP#0 + W(b) do_fpe @ CP#1 (FPE) + W(b) do_fpe @ CP#2 (FPE) + W(mov) pc, lr @ CP#3 #ifdef CONFIG_CRUNCH b crunch_task_enable @ CP#4 (MaverickCrunch) b crunch_task_enable @ CP#5 (MaverickCrunch) b crunch_task_enable @ CP#6 (MaverickCrunch) #else - mov pc, lr @ CP#4 - mov pc, lr @ CP#5 - mov pc, lr @ CP#6 + W(mov) pc, lr @ CP#4 + W(mov) pc, lr @ CP#5 + W(mov) pc, lr @ CP#6 #endif - mov pc, lr @ CP#7 - mov pc, lr @ CP#8 - mov pc, lr @ CP#9 + W(mov) pc, lr @ CP#7 + W(mov) pc, lr @ CP#8 + W(mov) pc, lr @ CP#9 #ifdef CONFIG_VFP - b do_vfp @ CP#10 (VFP) - b do_vfp @ CP#11 (VFP) + W(b) do_vfp @ CP#10 (VFP) + W(b) do_vfp @ CP#11 (VFP) #else - mov pc, lr @ CP#10 (VFP) - mov pc, lr @ CP#11 (VFP) + W(mov) pc, lr @ CP#10 (VFP) + W(mov) pc, lr @ CP#11 (VFP) #endif - mov pc, lr @ CP#12 - mov pc, lr @ CP#13 - mov pc, lr @ CP#14 (Debug) - mov pc, lr @ CP#15 (Control) + W(mov) pc, lr @ CP#12 + W(mov) pc, lr @ CP#13 + W(mov) pc, lr @ CP#14 (Debug) + W(mov) pc, lr @ CP#15 (Control) #ifdef CONFIG_NEON .align 6 @@ -667,7 +685,7 @@ no_fp: mov pc, lr __und_usr_unknown: enable_irq mov r0, sp - adr lr, ret_from_exception + adr lr, BSYM(ret_from_exception) b do_undefinstr ENDPROC(__und_usr_unknown) @@ -711,7 +729,10 @@ ENTRY(__switch_to) UNWIND(.cantunwind ) add ip, r1, #TI_CPU_SAVE ldr r3, [r2, #TI_TP_VALUE] - stmia ip!, {r4 - sl, fp, sp, lr} @ Store most regs on stack + ARM( stmia ip!, {r4 - sl, fp, sp, lr} ) @ Store most regs on stack + THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack + THUMB( str sp, [ip], #4 ) + THUMB( str lr, [ip], #4 ) #ifdef CONFIG_MMU ldr r6, [r2, #TI_CPU_DOMAIN] #endif @@ -736,8 +757,12 @@ ENTRY(__switch_to) ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH bl atomic_notifier_call_chain + THUMB( mov ip, r4 ) mov r0, r5 - ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously + ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously + THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously + THUMB( ldr sp, [ip], #4 ) + THUMB( ldr pc, [ip] ) UNWIND(.fnend ) ENDPROC(__switch_to) @@ -772,6 +797,7 @@ ENDPROC(__switch_to) * if your compiled code is not going to use the new instructions for other * purpose. */ + THUMB( .arm ) .macro usr_ret, reg #ifdef CONFIG_ARM_THUMB @@ -1020,6 +1046,7 @@ __kuser_helper_version: @ 0xffff0ffc .globl __kuser_helper_end __kuser_helper_end: + THUMB( .thumb ) /* * Vector stubs. @@ -1054,15 +1081,17 @@ vector_\name: @ Prepare for SVC32 mode. IRQs remain disabled. @ mrs r0, cpsr - eor r0, r0, #(\mode ^ SVC_MODE) + eor r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE) msr spsr_cxsf, r0 @ @ the branch table must immediately follow this code @ and lr, lr, #0x0f + THUMB( adr r0, 1f ) + THUMB( ldr lr, [r0, lr, lsl #2] ) mov r0, sp - ldr lr, [pc, lr, lsl #2] + ARM( ldr lr, [pc, lr, lsl #2] ) movs pc, lr @ branch to handler in SVC mode ENDPROC(vector_\name) @@ -1206,14 +1235,16 @@ __stubs_end: .globl __vectors_start __vectors_start: - swi SYS_ERROR0 - b vector_und + stubs_offset - ldr pc, .LCvswi + stubs_offset - b vector_pabt + stubs_offset - b vector_dabt + stubs_offset - b vector_addrexcptn + stubs_offset - b vector_irq + stubs_offset - b vector_fiq + stubs_offset + ARM( swi SYS_ERROR0 ) + THUMB( svc #0 ) + THUMB( nop ) + W(b) vector_und + stubs_offset + W(ldr) pc, .LCvswi + stubs_offset + W(b) vector_pabt + stubs_offset + W(b) vector_dabt + stubs_offset + W(b) vector_addrexcptn + stubs_offset + W(b) vector_irq + stubs_offset + W(b) vector_fiq + stubs_offset .globl __vectors_end __vectors_end: diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 366e5097a41..a0540c9f1f0 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -33,14 +33,7 @@ ret_fast_syscall: /* perform architecture specific actions before user return */ arch_ret_to_user r1, lr - @ fast_restore_user_regs - ldr r1, [sp, #S_OFF + S_PSR] @ get calling cpsr - ldr lr, [sp, #S_OFF + S_PC]! @ get pc - msr spsr_cxsf, r1 @ save in spsr_svc - ldmdb sp, {r1 - lr}^ @ get calling r1 - lr - mov r0, r0 - add sp, sp, #S_FRAME_SIZE - S_PC - movs pc, lr @ return & move spsr_svc into cpsr + restore_user_regs fast = 1, offset = S_OFF UNWIND(.fnend ) /* @@ -73,14 +66,7 @@ no_work_pending: /* perform architecture specific actions before user return */ arch_ret_to_user r1, lr - @ slow_restore_user_regs - ldr r1, [sp, #S_PSR] @ get calling cpsr - ldr lr, [sp, #S_PC]! @ get pc - msr spsr_cxsf, r1 @ save in spsr_svc - ldmdb sp, {r0 - lr}^ @ get calling r0 - lr - mov r0, r0 - add sp, sp, #S_FRAME_SIZE - S_PC - movs pc, lr @ return & move spsr_svc into cpsr + restore_user_regs fast = 0, offset = 0 ENDPROC(ret_to_user) /* @@ -182,8 +168,10 @@ ftrace_stub: ENTRY(vector_swi) sub sp, sp, #S_FRAME_SIZE stmia sp, {r0 - r12} @ Calling r0 - r12 - add r8, sp, #S_PC - stmdb r8, {sp, lr}^ @ Calling sp, lr + ARM( add r8, sp, #S_PC ) + ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr + THUMB( mov r8, sp ) + THUMB( store_user_sp_lr r8, r10, S_SP ) @ calling sp, lr mrs r8, spsr @ called from non-FIQ mode, so ok. str lr, [sp, #S_PC] @ Save calling PC str r8, [sp, #S_PSR] @ Save CPSR @@ -272,7 +260,7 @@ ENTRY(vector_swi) bne __sys_trace cmp scno, #NR_syscalls @ check upper syscall limit - adr lr, ret_fast_syscall @ return address + adr lr, BSYM(ret_fast_syscall) @ return address ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine add r1, sp, #S_OFF @@ -293,7 +281,7 @@ __sys_trace: mov r0, #0 @ trace entry [IP = 0] bl syscall_trace - adr lr, __sys_trace_return @ return address + adr lr, BSYM(__sys_trace_return) @ return address mov scno, r0 @ syscall number (possibly new) add r1, sp, #S_R0 + S_OFF @ pointer to regs cmp scno, #NR_syscalls @ check upper syscall limit diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 87ab4e15799..a4eaf4f920c 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -36,11 +36,6 @@ #endif .endm - .macro get_thread_info, rd - mov \rd, sp, lsr #13 - mov \rd, \rd, lsl #13 - .endm - .macro alignment_trap, rtemp #ifdef CONFIG_ALIGNMENT_TRAP ldr \rtemp, .LCcralign @@ -49,6 +44,93 @@ #endif .endm + @ + @ Store/load the USER SP and LR registers by switching to the SYS + @ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not + @ available. Should only be called from SVC mode + @ + .macro store_user_sp_lr, rd, rtemp, offset = 0 + mrs \rtemp, cpsr + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch to the SYS mode + + str sp, [\rd, #\offset] @ save sp_usr + str lr, [\rd, #\offset + 4] @ save lr_usr + + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch back to the SVC mode + .endm + + .macro load_user_sp_lr, rd, rtemp, offset = 0 + mrs \rtemp, cpsr + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch to the SYS mode + + ldr sp, [\rd, #\offset] @ load sp_usr + ldr lr, [\rd, #\offset + 4] @ load lr_usr + + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch back to the SVC mode + .endm + +#ifndef CONFIG_THUMB2_KERNEL + .macro svc_exit, rpsr + msr spsr_cxsf, \rpsr + ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr + .endm + + .macro restore_user_regs, fast = 0, offset = 0 + ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr + ldr lr, [sp, #\offset + S_PC]! @ get pc + msr spsr_cxsf, r1 @ save in spsr_svc + .if \fast + ldmdb sp, {r1 - lr}^ @ get calling r1 - lr + .else + ldmdb sp, {r0 - lr}^ @ get calling r0 - lr + .endif + add sp, sp, #S_FRAME_SIZE - S_PC + movs pc, lr @ return & move spsr_svc into cpsr + .endm + + .macro get_thread_info, rd + mov \rd, sp, lsr #13 + mov \rd, \rd, lsl #13 + .endm +#else /* CONFIG_THUMB2_KERNEL */ + .macro svc_exit, rpsr + ldr r0, [sp, #S_SP] @ top of the stack + ldr r1, [sp, #S_PC] @ return address + tst r0, #4 @ orig stack 8-byte aligned? + stmdb r0, {r1, \rpsr} @ rfe context + ldmia sp, {r0 - r12} + ldr lr, [sp, #S_LR] + addeq sp, sp, #S_FRAME_SIZE - 8 @ aligned + addne sp, sp, #S_FRAME_SIZE - 4 @ not aligned + rfeia sp! + .endm + + .macro restore_user_regs, fast = 0, offset = 0 + mov r2, sp + load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr + ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr + ldr lr, [sp, #\offset + S_PC] @ get pc + add sp, sp, #\offset + S_SP + msr spsr_cxsf, r1 @ save in spsr_svc + .if \fast + ldmdb sp, {r1 - r12} @ get calling r1 - r12 + .else + ldmdb sp, {r0 - r12} @ get calling r0 - r12 + .endif + add sp, sp, #S_FRAME_SIZE - S_SP + movs pc, lr @ return & move spsr_svc into cpsr + .endm + + .macro get_thread_info, rd + mov \rd, sp + lsr \rd, \rd, #13 + mov \rd, \rd, lsl #13 + .endm +#endif /* !CONFIG_THUMB2_KERNEL */ /* * These are the registers used in the syscall handler, and allow us to diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S index dbfaef07a1f..93ad576b2d7 100644 --- a/arch/arm/kernel/head-common.S +++ b/arch/arm/kernel/head-common.S @@ -52,7 +52,9 @@ __mmap_switched: strcc fp, [r6],#4 bcc 1b - ldmia r3, {r4, r5, r6, r7, sp} + ARM( ldmia r3, {r4, r5, r6, r7, sp}) + THUMB( ldmia r3, {r4, r5, r6, r7} ) + THUMB( ldr sp, [r3, #16] ) str r9, [r4] @ Save processor ID str r1, [r5] @ Save machine type str r2, [r6] @ Save atags pointer @@ -156,7 +158,8 @@ ENDPROC(__error) */ __lookup_processor_type: adr r3, 3f - ldmda r3, {r5 - r7} + ldmia r3, {r5 - r7} + add r3, r3, #8 sub r3, r3, r7 @ get offset between virt&phys add r5, r5, r3 @ convert virt addresses to add r6, r6, r3 @ physical address space @@ -187,9 +190,9 @@ ENDPROC(lookup_processor_type) * more information about the __proc_info and __arch_info structures. */ .align 2 - .long __proc_info_begin +3: .long __proc_info_begin .long __proc_info_end -3: .long . +4: .long . .long __arch_info_begin .long __arch_info_end @@ -205,7 +208,7 @@ ENDPROC(lookup_processor_type) * r5 = mach_info pointer in physical address space */ __lookup_machine_type: - adr r3, 3b + adr r3, 4b ldmia r3, {r4, r5, r6} sub r3, r3, r4 @ get offset between virt&phys add r5, r5, r3 @ convert virt addresses to diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S index cc87e1765ed..b16393d2b71 100644 --- a/arch/arm/kernel/head-nommu.S +++ b/arch/arm/kernel/head-nommu.S @@ -34,7 +34,7 @@ */ .section ".text.head", "ax" ENTRY(stext) - msr cpsr_c, #PSR_F_BIT | PSR_I_BIT | SVC_MODE @ ensure svc mode + setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode @ and irqs disabled #ifndef CONFIG_CPU_CP15 ldr r9, =CONFIG_PROCESSOR_ID @@ -50,8 +50,10 @@ ENTRY(stext) ldr r13, __switch_data @ address to jump to after @ the initialization is done - adr lr, __after_proc_init @ return (PIC) address - add pc, r10, #PROCINFO_INITFUNC + adr lr, BSYM(__after_proc_init) @ return (PIC) address + ARM( add pc, r10, #PROCINFO_INITFUNC ) + THUMB( add r12, r10, #PROCINFO_INITFUNC ) + THUMB( mov pc, r12 ) ENDPROC(stext) /* @@ -82,7 +84,8 @@ __after_proc_init: mcr p15, 0, r0, c1, c0, 0 @ write control reg #endif /* CONFIG_CPU_CP15 */ - mov pc, r13 @ clear the BSS and jump + mov r3, r13 + mov pc, r3 @ clear the BSS and jump @ to start_kernel ENDPROC(__after_proc_init) .ltorg diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 21e17dc94cb..38ccbe1d3b2 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -76,7 +76,7 @@ */ .section ".text.head", "ax" ENTRY(stext) - msr cpsr_c, #PSR_F_BIT | PSR_I_BIT | SVC_MODE @ ensure svc mode + setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode @ and irqs disabled mrc p15, 0, r9, c0, c0 @ get processor id bl __lookup_processor_type @ r5=procinfo r9=cpuid @@ -97,8 +97,10 @@ ENTRY(stext) */ ldr r13, __switch_data @ address to jump to after @ mmu has been enabled - adr lr, __enable_mmu @ return (PIC) address - add pc, r10, #PROCINFO_INITFUNC + adr lr, BSYM(__enable_mmu) @ return (PIC) address + ARM( add pc, r10, #PROCINFO_INITFUNC ) + THUMB( add r12, r10, #PROCINFO_INITFUNC ) + THUMB( mov pc, r12 ) ENDPROC(stext) #if defined(CONFIG_SMP) @@ -110,7 +112,7 @@ ENTRY(secondary_startup) * the processor type - there is no need to check the machine type * as it has already been validated by the primary processor. */ - msr cpsr_c, #PSR_F_BIT | PSR_I_BIT | SVC_MODE + setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 mrc p15, 0, r9, c0, c0 @ get processor id bl __lookup_processor_type movs r10, r5 @ invalid processor? @@ -121,12 +123,15 @@ ENTRY(secondary_startup) * Use the page tables supplied from __cpu_up. */ adr r4, __secondary_data - ldmia r4, {r5, r7, r13} @ address to jump to after + ldmia r4, {r5, r7, r12} @ address to jump to after sub r4, r4, r5 @ mmu has been enabled ldr r4, [r7, r4] @ get secondary_data.pgdir - adr lr, __enable_mmu @ return address - add pc, r10, #PROCINFO_INITFUNC @ initialise processor - @ (return control reg) + adr lr, BSYM(__enable_mmu) @ return address + mov r13, r12 @ __secondary_switched address + ARM( add pc, r10, #PROCINFO_INITFUNC ) @ initialise processor + @ (return control reg) + THUMB( add r12, r10, #PROCINFO_INITFUNC ) + THUMB( mov pc, r12 ) ENDPROC(secondary_startup) /* @@ -193,8 +198,8 @@ __turn_mmu_on: mcr p15, 0, r0, c1, c0, 0 @ write control reg mrc p15, 0, r3, c0, c0, 0 @ read id reg mov r3, r3 - mov r3, r3 - mov pc, r13 + mov r3, r13 + mov pc, r3 ENDPROC(__turn_mmu_on) @@ -235,7 +240,8 @@ __create_page_tables: * will be removed by paging_init(). We use our current program * counter to determine corresponding section base address. */ - mov r6, pc, lsr #20 @ start of kernel section + mov r6, pc + mov r6, r6, lsr #20 @ start of kernel section orr r3, r7, r6, lsl #20 @ flags + kernel base str r3, [r4, r6, lsl #2] @ identity mapping diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 39196dff478..790fbee92ec 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -388,7 +388,7 @@ pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) regs.ARM_r2 = (unsigned long)fn; regs.ARM_r3 = (unsigned long)kernel_thread_exit; regs.ARM_pc = (unsigned long)kernel_thread_helper; - regs.ARM_cpsr = SVC_MODE | PSR_ENDSTATE; + regs.ARM_cpsr = SVC_MODE | PSR_ENDSTATE | PSR_ISETSTATE; return do_fork(flags|CLONE_VM|CLONE_UNTRACED, 0, ®s, 0, NULL, NULL); } diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index bc5e4128f9f..d4d4f77c91b 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -326,26 +327,39 @@ void cpu_init(void) BUG(); } + /* + * Define the placement constraint for the inline asm directive below. + * In Thumb-2, msr with an immediate value is not allowed. + */ +#ifdef CONFIG_THUMB2_KERNEL +#define PLC "r" +#else +#define PLC "I" +#endif + /* * setup stacks for re-entrant exception handlers */ __asm__ ( "msr cpsr_c, %1\n\t" - "add sp, %0, %2\n\t" + "add r14, %0, %2\n\t" + "mov sp, r14\n\t" "msr cpsr_c, %3\n\t" - "add sp, %0, %4\n\t" + "add r14, %0, %4\n\t" + "mov sp, r14\n\t" "msr cpsr_c, %5\n\t" - "add sp, %0, %6\n\t" + "add r14, %0, %6\n\t" + "mov sp, r14\n\t" "msr cpsr_c, %7" : : "r" (stk), - "I" (PSR_F_BIT | PSR_I_BIT | IRQ_MODE), + PLC (PSR_F_BIT | PSR_I_BIT | IRQ_MODE), "I" (offsetof(struct stack, irq[0])), - "I" (PSR_F_BIT | PSR_I_BIT | ABT_MODE), + PLC (PSR_F_BIT | PSR_I_BIT | ABT_MODE), "I" (offsetof(struct stack, abt[0])), - "I" (PSR_F_BIT | PSR_I_BIT | UND_MODE), + PLC (PSR_F_BIT | PSR_I_BIT | UND_MODE), "I" (offsetof(struct stack, und[0])), - "I" (PSR_F_BIT | PSR_I_BIT | SVC_MODE) + PLC (PSR_F_BIT | PSR_I_BIT | SVC_MODE) : "r14"); } diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index dd56e11f339..39baf1128bf 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -62,7 +62,11 @@ struct unwind_ctrl_block { }; enum regs { +#ifdef CONFIG_THUMB2_KERNEL + FP = 7, +#else FP = 11, +#endif SP = 13, LR = 14, PC = 15 -- cgit v1.2.3-70-g09d2 From 8b592783a2e8b7721a99730bd549aab5208f36af Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 24 Jul 2009 12:32:57 +0100 Subject: Thumb-2: Implement the unified arch/arm/lib functions This patch adds the ARM/Thumb-2 unified support for the arch/arm/lib/* files. Signed-off-by: Catalin Marinas --- arch/arm/include/asm/assembler.h | 73 ++++++++++++++++++++++++++++++++++++++ arch/arm/include/asm/uaccess.h | 7 ++-- arch/arm/lib/ashldi3.S | 4 ++- arch/arm/lib/ashrdi3.S | 4 ++- arch/arm/lib/backtrace.S | 8 +++-- arch/arm/lib/clear_user.S | 15 ++++---- arch/arm/lib/copy_from_user.S | 19 +++++----- arch/arm/lib/copy_template.S | 24 +++++++++++-- arch/arm/lib/copy_to_user.S | 19 +++++----- arch/arm/lib/csumpartialcopyuser.S | 48 +++++++------------------ arch/arm/lib/div64.S | 4 ++- arch/arm/lib/findbit.S | 34 +++++++++++++----- arch/arm/lib/getuser.S | 5 +++ arch/arm/lib/io-writesw-armv4.S | 5 ++- arch/arm/lib/lshrdi3.S | 4 ++- arch/arm/lib/memcpy.S | 7 ++-- arch/arm/lib/memmove.S | 28 +++++++-------- arch/arm/lib/putuser.S | 15 ++++++++ arch/arm/lib/strncpy_from_user.S | 2 +- arch/arm/lib/strnlen_user.S | 2 +- 20 files changed, 229 insertions(+), 98 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 1acd1da36d0..2b60c7d0577 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -138,3 +138,76 @@ msr cpsr_c, #\mode .endm #endif + +/* + * STRT/LDRT access macros with ARM and Thumb-2 variants + */ +#ifdef CONFIG_THUMB2_KERNEL + + .macro usraccoff, instr, reg, ptr, inc, off, cond, abort +9999: + .if \inc == 1 + \instr\cond\()bt \reg, [\ptr, #\off] + .elseif \inc == 4 + \instr\cond\()t \reg, [\ptr, #\off] + .else + .error "Unsupported inc macro argument" + .endif + + .section __ex_table,"a" + .align 3 + .long 9999b, \abort + .previous + .endm + + .macro usracc, instr, reg, ptr, inc, cond, rept, abort + @ explicit IT instruction needed because of the label + @ introduced by the USER macro + .ifnc \cond,al + .if \rept == 1 + itt \cond + .elseif \rept == 2 + ittt \cond + .else + .error "Unsupported rept macro argument" + .endif + .endif + + @ Slightly optimised to avoid incrementing the pointer twice + usraccoff \instr, \reg, \ptr, \inc, 0, \cond, \abort + .if \rept == 2 + usraccoff \instr, \reg, \ptr, \inc, 4, \cond, \abort + .endif + + add\cond \ptr, #\rept * \inc + .endm + +#else /* !CONFIG_THUMB2_KERNEL */ + + .macro usracc, instr, reg, ptr, inc, cond, rept, abort + .rept \rept +9999: + .if \inc == 1 + \instr\cond\()bt \reg, [\ptr], #\inc + .elseif \inc == 4 + \instr\cond\()t \reg, [\ptr], #\inc + .else + .error "Unsupported inc macro argument" + .endif + + .section __ex_table,"a" + .align 3 + .long 9999b, \abort + .previous + .endr + .endm + +#endif /* CONFIG_THUMB2_KERNEL */ + + .macro strusr, reg, ptr, inc, cond=al, rept=1, abort=9001f + usracc str, \reg, \ptr, \inc, \cond, \rept, \abort + .endm + + .macro ldrusr, reg, ptr, inc, cond=al, rept=1, abort=9001f + usracc ldr, \reg, \ptr, \inc, \cond, \rept, \abort + .endm diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 0da9bc9b3b1..1d6bd40a432 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -17,6 +17,7 @@ #include #include #include +#include #define VERIFY_READ 0 #define VERIFY_WRITE 1 @@ -365,8 +366,10 @@ do { \ #define __put_user_asm_dword(x,__pu_addr,err) \ __asm__ __volatile__( \ - "1: strt " __reg_oper1 ", [%1], #4\n" \ - "2: strt " __reg_oper0 ", [%1]\n" \ + ARM( "1: strt " __reg_oper1 ", [%1], #4\n" ) \ + ARM( "2: strt " __reg_oper0 ", [%1]\n" ) \ + THUMB( "1: strt " __reg_oper1 ", [%1]\n" ) \ + THUMB( "2: strt " __reg_oper0 ", [%1, #4]\n" ) \ "3:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \ diff --git a/arch/arm/lib/ashldi3.S b/arch/arm/lib/ashldi3.S index 1154d924080..638deb13da1 100644 --- a/arch/arm/lib/ashldi3.S +++ b/arch/arm/lib/ashldi3.S @@ -43,7 +43,9 @@ ENTRY(__aeabi_llsl) rsb ip, r2, #32 movmi ah, ah, lsl r2 movpl ah, al, lsl r3 - orrmi ah, ah, al, lsr ip + ARM( orrmi ah, ah, al, lsr ip ) + THUMB( lsrmi r3, al, ip ) + THUMB( orrmi ah, ah, r3 ) mov al, al, lsl r2 mov pc, lr diff --git a/arch/arm/lib/ashrdi3.S b/arch/arm/lib/ashrdi3.S index 9f8b35572f8..015e8aa5a1d 100644 --- a/arch/arm/lib/ashrdi3.S +++ b/arch/arm/lib/ashrdi3.S @@ -43,7 +43,9 @@ ENTRY(__aeabi_lasr) rsb ip, r2, #32 movmi al, al, lsr r2 movpl al, ah, asr r3 - orrmi al, al, ah, lsl ip + ARM( orrmi al, al, ah, lsl ip ) + THUMB( lslmi r3, ah, ip ) + THUMB( orrmi al, al, r3 ) mov ah, ah, asr r2 mov pc, lr diff --git a/arch/arm/lib/backtrace.S b/arch/arm/lib/backtrace.S index b0951d0e8b2..aaf7220d9e3 100644 --- a/arch/arm/lib/backtrace.S +++ b/arch/arm/lib/backtrace.S @@ -38,7 +38,9 @@ ENDPROC(c_backtrace) beq no_frame @ we have no stack frames tst r1, #0x10 @ 26 or 32-bit mode? - moveq mask, #0xfc000003 @ mask for 26-bit + ARM( moveq mask, #0xfc000003 ) + THUMB( moveq mask, #0xfc000000 ) + THUMB( orreq mask, #0x03 ) movne mask, #0 @ mask for 32-bit 1: stmfd sp!, {pc} @ calculate offset of PC stored @@ -126,7 +128,9 @@ ENDPROC(c_backtrace) mov reg, #10 mov r7, #0 1: mov r3, #1 - tst instr, r3, lsl reg + ARM( tst instr, r3, lsl reg ) + THUMB( lsl r3, reg ) + THUMB( tst instr, r3 ) beq 2f add r7, r7, #1 teq r7, #6 diff --git a/arch/arm/lib/clear_user.S b/arch/arm/lib/clear_user.S index 844f56785eb..1279abd8b88 100644 --- a/arch/arm/lib/clear_user.S +++ b/arch/arm/lib/clear_user.S @@ -27,21 +27,20 @@ WEAK(__clear_user) ands ip, r0, #3 beq 1f cmp ip, #2 -USER( strbt r2, [r0], #1) -USER( strlebt r2, [r0], #1) -USER( strltbt r2, [r0], #1) + strusr r2, r0, 1 + strusr r2, r0, 1, le + strusr r2, r0, 1, lt rsb ip, ip, #4 sub r1, r1, ip @ 7 6 5 4 3 2 1 1: subs r1, r1, #8 @ -1 -2 -3 -4 -5 -6 -7 -USER( strplt r2, [r0], #4) -USER( strplt r2, [r0], #4) + strusr r2, r0, 4, pl, rept=2 bpl 1b adds r1, r1, #4 @ 3 2 1 0 -1 -2 -3 -USER( strplt r2, [r0], #4) + strusr r2, r0, 4, pl 2: tst r1, #2 @ 1x 1x 0x 0x 1x 1x 0x -USER( strnebt r2, [r0], #1) -USER( strnebt r2, [r0], #1) + strusr r2, r0, 1, ne, rept=2 tst r1, #1 @ x1 x0 x1 x0 x1 x0 x1 + it ne @ explicit IT needed for the label USER( strnebt r2, [r0]) mov r0, #0 ldmfd sp!, {r1, pc} diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S index 56799a165cc..e4fe124aced 100644 --- a/arch/arm/lib/copy_from_user.S +++ b/arch/arm/lib/copy_from_user.S @@ -33,11 +33,15 @@ * Number of bytes NOT copied. */ +#ifndef CONFIG_THUMB2_KERNEL +#define LDR1W_SHIFT 0 +#else +#define LDR1W_SHIFT 1 +#endif +#define STR1W_SHIFT 0 + .macro ldr1w ptr reg abort -100: ldrt \reg, [\ptr], #4 - .section __ex_table, "a" - .long 100b, \abort - .previous + ldrusr \reg, \ptr, 4, abort=\abort .endm .macro ldr4w ptr reg1 reg2 reg3 reg4 abort @@ -53,14 +57,11 @@ .endm .macro ldr1b ptr reg cond=al abort -100: ldr\cond\()bt \reg, [\ptr], #1 - .section __ex_table, "a" - .long 100b, \abort - .previous + ldrusr \reg, \ptr, 1, \cond, abort=\abort .endm .macro str1w ptr reg abort - str \reg, [\ptr], #4 + W(str) \reg, [\ptr], #4 .endm .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort diff --git a/arch/arm/lib/copy_template.S b/arch/arm/lib/copy_template.S index 139cce64605..805e3f8fb00 100644 --- a/arch/arm/lib/copy_template.S +++ b/arch/arm/lib/copy_template.S @@ -57,6 +57,13 @@ * * Restore registers with the values previously saved with the * 'preserv' macro. Called upon code termination. + * + * LDR1W_SHIFT + * STR1W_SHIFT + * + * Correction to be applied to the "ip" register when branching into + * the ldr1w or str1w instructions (some of these macros may expand to + * than one 32bit instruction in Thumb-2) */ @@ -99,9 +106,15 @@ 5: ands ip, r2, #28 rsb ip, ip, #32 +#if LDR1W_SHIFT > 0 + lsl ip, ip, #LDR1W_SHIFT +#endif addne pc, pc, ip @ C is always clear here b 7f -6: nop +6: + .rept (1 << LDR1W_SHIFT) + W(nop) + .endr ldr1w r1, r3, abort=20f ldr1w r1, r4, abort=20f ldr1w r1, r5, abort=20f @@ -110,9 +123,16 @@ ldr1w r1, r8, abort=20f ldr1w r1, lr, abort=20f +#if LDR1W_SHIFT < STR1W_SHIFT + lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT +#elif LDR1W_SHIFT > STR1W_SHIFT + lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT +#endif add pc, pc, ip nop - nop + .rept (1 << STR1W_SHIFT) + W(nop) + .endr str1w r0, r3, abort=20f str1w r0, r4, abort=20f str1w r0, r5, abort=20f diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S index 878820f0a32..1a71e158444 100644 --- a/arch/arm/lib/copy_to_user.S +++ b/arch/arm/lib/copy_to_user.S @@ -33,8 +33,15 @@ * Number of bytes NOT copied. */ +#define LDR1W_SHIFT 0 +#ifndef CONFIG_THUMB2_KERNEL +#define STR1W_SHIFT 0 +#else +#define STR1W_SHIFT 1 +#endif + .macro ldr1w ptr reg abort - ldr \reg, [\ptr], #4 + W(ldr) \reg, [\ptr], #4 .endm .macro ldr4w ptr reg1 reg2 reg3 reg4 abort @@ -50,10 +57,7 @@ .endm .macro str1w ptr reg abort -100: strt \reg, [\ptr], #4 - .section __ex_table, "a" - .long 100b, \abort - .previous + strusr \reg, \ptr, 4, abort=\abort .endm .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort @@ -68,10 +72,7 @@ .endm .macro str1b ptr reg cond=al abort -100: str\cond\()bt \reg, [\ptr], #1 - .section __ex_table, "a" - .long 100b, \abort - .previous + strusr \reg, \ptr, 1, \cond, abort=\abort .endm .macro enter reg1 reg2 diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S index 14677fb4b0c..fd0e9dcd9fd 100644 --- a/arch/arm/lib/csumpartialcopyuser.S +++ b/arch/arm/lib/csumpartialcopyuser.S @@ -26,50 +26,28 @@ .endm .macro load1b, reg1 -9999: ldrbt \reg1, [r0], $1 - .section __ex_table, "a" - .align 3 - .long 9999b, 6001f - .previous + ldrusr \reg1, r0, 1 .endm .macro load2b, reg1, reg2 -9999: ldrbt \reg1, [r0], $1 -9998: ldrbt \reg2, [r0], $1 - .section __ex_table, "a" - .long 9999b, 6001f - .long 9998b, 6001f - .previous + ldrusr \reg1, r0, 1 + ldrusr \reg2, r0, 1 .endm .macro load1l, reg1 -9999: ldrt \reg1, [r0], $4 - .section __ex_table, "a" - .align 3 - .long 9999b, 6001f - .previous + ldrusr \reg1, r0, 4 .endm .macro load2l, reg1, reg2 -9999: ldrt \reg1, [r0], $4 -9998: ldrt \reg2, [r0], $4 - .section __ex_table, "a" - .long 9999b, 6001f - .long 9998b, 6001f - .previous + ldrusr \reg1, r0, 4 + ldrusr \reg2, r0, 4 .endm .macro load4l, reg1, reg2, reg3, reg4 -9999: ldrt \reg1, [r0], $4 -9998: ldrt \reg2, [r0], $4 -9997: ldrt \reg3, [r0], $4 -9996: ldrt \reg4, [r0], $4 - .section __ex_table, "a" - .long 9999b, 6001f - .long 9998b, 6001f - .long 9997b, 6001f - .long 9996b, 6001f - .previous + ldrusr \reg1, r0, 4 + ldrusr \reg2, r0, 4 + ldrusr \reg3, r0, 4 + ldrusr \reg4, r0, 4 .endm /* @@ -92,14 +70,14 @@ */ .section .fixup,"ax" .align 4 -6001: mov r4, #-EFAULT +9001: mov r4, #-EFAULT ldr r5, [fp, #4] @ *err_ptr str r4, [r5] ldmia sp, {r1, r2} @ retrieve dst, len add r2, r2, r1 mov r0, #0 @ zero the buffer -6002: teq r2, r1 +9002: teq r2, r1 strneb r0, [r1], #1 - bne 6002b + bne 9002b load_regs .previous diff --git a/arch/arm/lib/div64.S b/arch/arm/lib/div64.S index 1425e789ba8..faa7748142d 100644 --- a/arch/arm/lib/div64.S +++ b/arch/arm/lib/div64.S @@ -177,7 +177,9 @@ ENTRY(__do_div64) mov yh, xh, lsr ip mov yl, xl, lsr ip rsb ip, ip, #32 - orr yl, yl, xh, lsl ip + ARM( orr yl, yl, xh, lsl ip ) + THUMB( lsl xh, xh, ip ) + THUMB( orr yl, yl, xh ) mov xh, xl, lsl ip mov xh, xh, lsr ip mov pc, lr diff --git a/arch/arm/lib/findbit.S b/arch/arm/lib/findbit.S index 8c4defc4f3c..1e4cbd4e7be 100644 --- a/arch/arm/lib/findbit.S +++ b/arch/arm/lib/findbit.S @@ -25,7 +25,10 @@ ENTRY(_find_first_zero_bit_le) teq r1, #0 beq 3f mov r2, #0 -1: ldrb r3, [r0, r2, lsr #3] +1: + ARM( ldrb r3, [r0, r2, lsr #3] ) + THUMB( lsr r3, r2, #3 ) + THUMB( ldrb r3, [r0, r3] ) eors r3, r3, #0xff @ invert bits bne .L_found @ any now set - found zero bit add r2, r2, #8 @ next bit pointer @@ -44,7 +47,9 @@ ENTRY(_find_next_zero_bit_le) beq 3b ands ip, r2, #7 beq 1b @ If new byte, goto old routine - ldrb r3, [r0, r2, lsr #3] + ARM( ldrb r3, [r0, r2, lsr #3] ) + THUMB( lsr r3, r2, #3 ) + THUMB( ldrb r3, [r0, r3] ) eor r3, r3, #0xff @ now looking for a 1 bit movs r3, r3, lsr ip @ shift off unused bits bne .L_found @@ -61,7 +66,10 @@ ENTRY(_find_first_bit_le) teq r1, #0 beq 3f mov r2, #0 -1: ldrb r3, [r0, r2, lsr #3] +1: + ARM( ldrb r3, [r0, r2, lsr #3] ) + THUMB( lsr r3, r2, #3 ) + THUMB( ldrb r3, [r0, r3] ) movs r3, r3 bne .L_found @ any now set - found zero bit add r2, r2, #8 @ next bit pointer @@ -80,7 +88,9 @@ ENTRY(_find_next_bit_le) beq 3b ands ip, r2, #7 beq 1b @ If new byte, goto old routine - ldrb r3, [r0, r2, lsr #3] + ARM( ldrb r3, [r0, r2, lsr #3] ) + THUMB( lsr r3, r2, #3 ) + THUMB( ldrb r3, [r0, r3] ) movs r3, r3, lsr ip @ shift off unused bits bne .L_found orr r2, r2, #7 @ if zero, then no bits here @@ -95,7 +105,9 @@ ENTRY(_find_first_zero_bit_be) beq 3f mov r2, #0 1: eor r3, r2, #0x18 @ big endian byte ordering - ldrb r3, [r0, r3, lsr #3] + ARM( ldrb r3, [r0, r3, lsr #3] ) + THUMB( lsr r3, #3 ) + THUMB( ldrb r3, [r0, r3] ) eors r3, r3, #0xff @ invert bits bne .L_found @ any now set - found zero bit add r2, r2, #8 @ next bit pointer @@ -111,7 +123,9 @@ ENTRY(_find_next_zero_bit_be) ands ip, r2, #7 beq 1b @ If new byte, goto old routine eor r3, r2, #0x18 @ big endian byte ordering - ldrb r3, [r0, r3, lsr #3] + ARM( ldrb r3, [r0, r3, lsr #3] ) + THUMB( lsr r3, #3 ) + THUMB( ldrb r3, [r0, r3] ) eor r3, r3, #0xff @ now looking for a 1 bit movs r3, r3, lsr ip @ shift off unused bits bne .L_found @@ -125,7 +139,9 @@ ENTRY(_find_first_bit_be) beq 3f mov r2, #0 1: eor r3, r2, #0x18 @ big endian byte ordering - ldrb r3, [r0, r3, lsr #3] + ARM( ldrb r3, [r0, r3, lsr #3] ) + THUMB( lsr r3, #3 ) + THUMB( ldrb r3, [r0, r3] ) movs r3, r3 bne .L_found @ any now set - found zero bit add r2, r2, #8 @ next bit pointer @@ -141,7 +157,9 @@ ENTRY(_find_next_bit_be) ands ip, r2, #7 beq 1b @ If new byte, goto old routine eor r3, r2, #0x18 @ big endian byte ordering - ldrb r3, [r0, r3, lsr #3] + ARM( ldrb r3, [r0, r3, lsr #3] ) + THUMB( lsr r3, #3 ) + THUMB( ldrb r3, [r0, r3] ) movs r3, r3, lsr ip @ shift off unused bits bne .L_found orr r2, r2, #7 @ if zero, then no bits here diff --git a/arch/arm/lib/getuser.S b/arch/arm/lib/getuser.S index 6763088b760..a1814d92712 100644 --- a/arch/arm/lib/getuser.S +++ b/arch/arm/lib/getuser.S @@ -36,8 +36,13 @@ ENTRY(__get_user_1) ENDPROC(__get_user_1) ENTRY(__get_user_2) +#ifdef CONFIG_THUMB2_KERNEL +2: ldrbt r2, [r0] +3: ldrbt r3, [r0, #1] +#else 2: ldrbt r2, [r0], #1 3: ldrbt r3, [r0] +#endif #ifndef __ARMEB__ orr r2, r2, r3, lsl #8 #else diff --git a/arch/arm/lib/io-writesw-armv4.S b/arch/arm/lib/io-writesw-armv4.S index d6585612c86..ff4f71b579e 100644 --- a/arch/arm/lib/io-writesw-armv4.S +++ b/arch/arm/lib/io-writesw-armv4.S @@ -75,7 +75,10 @@ ENTRY(__raw_writesw) #endif .Loutsw_noalign: - ldr r3, [r1, -r3]! + ARM( ldr r3, [r1, -r3]! ) + THUMB( rsb r3, r3, #0 ) + THUMB( ldr r3, [r1, r3] ) + THUMB( sub r1, r3 ) subcs r2, r2, #1 bcs 2f subs r2, r2, #2 diff --git a/arch/arm/lib/lshrdi3.S b/arch/arm/lib/lshrdi3.S index 99ea338bf87..f83d449141f 100644 --- a/arch/arm/lib/lshrdi3.S +++ b/arch/arm/lib/lshrdi3.S @@ -43,7 +43,9 @@ ENTRY(__aeabi_llsr) rsb ip, r2, #32 movmi al, al, lsr r2 movpl al, ah, lsr r3 - orrmi al, al, ah, lsl ip + ARM( orrmi al, al, ah, lsl ip ) + THUMB( lslmi r3, ah, ip ) + THUMB( orrmi al, al, r3 ) mov ah, ah, lsr r2 mov pc, lr diff --git a/arch/arm/lib/memcpy.S b/arch/arm/lib/memcpy.S index e0d002641d3..a9b9e2287a0 100644 --- a/arch/arm/lib/memcpy.S +++ b/arch/arm/lib/memcpy.S @@ -13,8 +13,11 @@ #include #include +#define LDR1W_SHIFT 0 +#define STR1W_SHIFT 0 + .macro ldr1w ptr reg abort - ldr \reg, [\ptr], #4 + W(ldr) \reg, [\ptr], #4 .endm .macro ldr4w ptr reg1 reg2 reg3 reg4 abort @@ -30,7 +33,7 @@ .endm .macro str1w ptr reg abort - str \reg, [\ptr], #4 + W(str) \reg, [\ptr], #4 .endm .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort diff --git a/arch/arm/lib/memmove.S b/arch/arm/lib/memmove.S index 12549187088..5025c863713 100644 --- a/arch/arm/lib/memmove.S +++ b/arch/arm/lib/memmove.S @@ -75,24 +75,24 @@ ENTRY(memmove) addne pc, pc, ip @ C is always clear here b 7f 6: nop - ldr r3, [r1, #-4]! - ldr r4, [r1, #-4]! - ldr r5, [r1, #-4]! - ldr r6, [r1, #-4]! - ldr r7, [r1, #-4]! - ldr r8, [r1, #-4]! - ldr lr, [r1, #-4]! + W(ldr) r3, [r1, #-4]! + W(ldr) r4, [r1, #-4]! + W(ldr) r5, [r1, #-4]! + W(ldr) r6, [r1, #-4]! + W(ldr) r7, [r1, #-4]! + W(ldr) r8, [r1, #-4]! + W(ldr) lr, [r1, #-4]! add pc, pc, ip nop nop - str r3, [r0, #-4]! - str r4, [r0, #-4]! - str r5, [r0, #-4]! - str r6, [r0, #-4]! - str r7, [r0, #-4]! - str r8, [r0, #-4]! - str lr, [r0, #-4]! + W(str) r3, [r0, #-4]! + W(str) r4, [r0, #-4]! + W(str) r5, [r0, #-4]! + W(str) r6, [r0, #-4]! + W(str) r7, [r0, #-4]! + W(str) r8, [r0, #-4]! + W(str) lr, [r0, #-4]! CALGN( bcs 2b ) diff --git a/arch/arm/lib/putuser.S b/arch/arm/lib/putuser.S index 864f3c1c4f1..02fedbf07c0 100644 --- a/arch/arm/lib/putuser.S +++ b/arch/arm/lib/putuser.S @@ -37,6 +37,15 @@ ENDPROC(__put_user_1) ENTRY(__put_user_2) mov ip, r2, lsr #8 +#ifdef CONFIG_THUMB2_KERNEL +#ifndef __ARMEB__ +2: strbt r2, [r0] +3: strbt ip, [r0, #1] +#else +2: strbt ip, [r0] +3: strbt r2, [r0, #1] +#endif +#else /* !CONFIG_THUMB2_KERNEL */ #ifndef __ARMEB__ 2: strbt r2, [r0], #1 3: strbt ip, [r0] @@ -44,6 +53,7 @@ ENTRY(__put_user_2) 2: strbt ip, [r0], #1 3: strbt r2, [r0] #endif +#endif /* CONFIG_THUMB2_KERNEL */ mov r0, #0 mov pc, lr ENDPROC(__put_user_2) @@ -55,8 +65,13 @@ ENTRY(__put_user_4) ENDPROC(__put_user_4) ENTRY(__put_user_8) +#ifdef CONFIG_THUMB2_KERNEL +5: strt r2, [r0] +6: strt r3, [r0, #4] +#else 5: strt r2, [r0], #4 6: strt r3, [r0] +#endif mov r0, #0 mov pc, lr ENDPROC(__put_user_8) diff --git a/arch/arm/lib/strncpy_from_user.S b/arch/arm/lib/strncpy_from_user.S index 330373c26dd..1c9814f346c 100644 --- a/arch/arm/lib/strncpy_from_user.S +++ b/arch/arm/lib/strncpy_from_user.S @@ -23,7 +23,7 @@ ENTRY(__strncpy_from_user) mov ip, r1 1: subs r2, r2, #1 -USER( ldrplbt r3, [r1], #1) + ldrusr r3, r1, 1, pl bmi 2f strb r3, [r0], #1 teq r3, #0 diff --git a/arch/arm/lib/strnlen_user.S b/arch/arm/lib/strnlen_user.S index 90bb9d02083..7855b290665 100644 --- a/arch/arm/lib/strnlen_user.S +++ b/arch/arm/lib/strnlen_user.S @@ -23,7 +23,7 @@ ENTRY(__strnlen_user) mov r2, r0 1: -USER( ldrbt r3, [r0], #1) + ldrusr r3, r0, 1 teq r3, #0 beq 2f subs r1, r1, #1 -- cgit v1.2.3-70-g09d2 From adca6dc23bc620ea95392659625200a252b97be3 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 24 Jul 2009 12:32:59 +0100 Subject: Thumb-2: Add support for loadable modules Modules compiled to Thumb-2 have two additional relocations needing to be resolved at load time, R_ARM_THM_CALL and R_ARM_THM_JUMP24, for BL and B.W instructions. The maximum Thumb-2 addressing range is +/-2^24 (+/-16MB) therefore the MODULES_VADDR macro in asm/memory.h is set to (MODULES_END - 8MB) for the Thumb-2 compiled kernel. Signed-off-by: Catalin Marinas --- arch/arm/include/asm/elf.h | 3 +++ arch/arm/include/asm/memory.h | 6 +++++ arch/arm/kernel/module.c | 53 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/elf.h b/arch/arm/include/asm/elf.h index c207504de84..c3b911ee915 100644 --- a/arch/arm/include/asm/elf.h +++ b/arch/arm/include/asm/elf.h @@ -55,6 +55,9 @@ typedef struct user_fp elf_fpregset_t; #define R_ARM_MOVW_ABS_NC 43 #define R_ARM_MOVT_ABS 44 +#define R_ARM_THM_CALL 10 +#define R_ARM_THM_JUMP24 30 + /* * These are used to set parameters in the core dumps. */ diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index 85763db8744..376be1a6286 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -44,7 +44,13 @@ * The module space lives between the addresses given by TASK_SIZE * and PAGE_OFFSET - it must be within 32MB of the kernel text. */ +#ifndef CONFIG_THUMB2_KERNEL #define MODULES_VADDR (PAGE_OFFSET - 16*1024*1024) +#else +/* smaller range for Thumb-2 symbols relocation (2^24)*/ +#define MODULES_VADDR (PAGE_OFFSET - 8*1024*1024) +#endif + #if TASK_SIZE > MODULES_VADDR #error Top of user space clashes with start of module space #endif diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index bac03c81489..f28c5e9c51e 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -102,6 +102,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned long loc; Elf32_Sym *sym; s32 offset; + u32 upper, lower, sign, j1, j2; offset = ELF32_R_SYM(rel->r_info); if (offset < 0 || offset > (symsec->sh_size / sizeof(Elf32_Sym))) { @@ -184,6 +185,58 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, (offset & 0x0fff); break; + case R_ARM_THM_CALL: + case R_ARM_THM_JUMP24: + upper = *(u16 *)loc; + lower = *(u16 *)(loc + 2); + + /* + * 25 bit signed address range (Thumb-2 BL and B.W + * instructions): + * S:I1:I2:imm10:imm11:0 + * where: + * S = upper[10] = offset[24] + * I1 = ~(J1 ^ S) = offset[23] + * I2 = ~(J2 ^ S) = offset[22] + * imm10 = upper[9:0] = offset[21:12] + * imm11 = lower[10:0] = offset[11:1] + * J1 = lower[13] + * J2 = lower[11] + */ + sign = (upper >> 10) & 1; + j1 = (lower >> 13) & 1; + j2 = (lower >> 11) & 1; + offset = (sign << 24) | ((~(j1 ^ sign) & 1) << 23) | + ((~(j2 ^ sign) & 1) << 22) | + ((upper & 0x03ff) << 12) | + ((lower & 0x07ff) << 1); + if (offset & 0x01000000) + offset -= 0x02000000; + offset += sym->st_value - loc; + + /* only Thumb addresses allowed (no interworking) */ + if (!(offset & 1) || + offset <= (s32)0xff000000 || + offset >= (s32)0x01000000) { + printk(KERN_ERR + "%s: relocation out of range, section " + "%d reloc %d sym '%s'\n", module->name, + relindex, i, strtab + sym->st_name); + return -ENOEXEC; + } + + sign = (offset >> 24) & 1; + j1 = sign ^ (~(offset >> 23) & 1); + j2 = sign ^ (~(offset >> 22) & 1); + *(u16 *)loc = (u16)((upper & 0xf800) | (sign << 10) | + ((offset >> 12) & 0x03ff)); + *(u16 *)(loc + 2) = (u16)((lower & 0xd000) | + (j1 << 13) | (j2 << 11) | + ((offset >> 1) & 0x07ff)); + upper = *(u16 *)loc; + lower = *(u16 *)(loc + 2); + break; + default: printk(KERN_ERR "%s: unknown relocation: %u\n", module->name, ELF32_R_TYPE(rel->r_info)); -- cgit v1.2.3-70-g09d2 From 9a45f026bb3ba720765d46f62f5c464d3317a8ab Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 24 Jul 2009 12:34:56 +0100 Subject: nommu: Remove the context.id from asm-offsets.c when !MMU There is no MMU context switching on MMU-less systems. Signed-off-by: Catalin Marinas --- arch/arm/include/asm/mmu_context.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index 263fed05ea3..bcdb9291ef0 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -62,8 +62,10 @@ static inline void check_context(struct mm_struct *mm) static inline void check_context(struct mm_struct *mm) { +#ifdef CONFIG_MMU if (unlikely(mm->context.kvm_seq != init_mm.context.kvm_seq)) __check_kvm_seq(mm); +#endif } #define init_new_context(tsk,mm) 0 -- cgit v1.2.3-70-g09d2 From 68b7f7153fa58df710924fbb79722717d2d16094 Mon Sep 17 00:00:00 2001 From: Paul Brook Date: Fri, 24 Jul 2009 12:34:58 +0100 Subject: nommu: ptrace support The patch below adds ARM ptrace functions to get the process load address. This is required for useful userspace debugging on mmuless systems. These values are obtained by reading magic offsets with PTRACE_PEEKUSR, as on other nommu targets. I picked arbitrary large values for the offsets. Signed-off-by: Paul Brook --- arch/arm/include/asm/ptrace.h | 8 ++++++++ arch/arm/kernel/ptrace.c | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/ptrace.h b/arch/arm/include/asm/ptrace.h index 67b833c9b6b..bbecccda76d 100644 --- a/arch/arm/include/asm/ptrace.h +++ b/arch/arm/include/asm/ptrace.h @@ -82,6 +82,14 @@ #define PSR_ENDSTATE 0 #endif +/* + * These are 'magic' values for PTRACE_PEEKUSR that return info about where a + * process is located in memory. + */ +#define PT_TEXT_ADDR 0x10000 +#define PT_DATA_ADDR 0x10004 +#define PT_TEXT_END_ADDR 0x10008 + #ifndef __ASSEMBLY__ /* diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 89882a1d018..a2ea3854cb3 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -521,7 +521,13 @@ static int ptrace_read_user(struct task_struct *tsk, unsigned long off, return -EIO; tmp = 0; - if (off < sizeof(struct pt_regs)) + if (off == PT_TEXT_ADDR) + tmp = tsk->mm->start_code; + else if (off == PT_DATA_ADDR) + tmp = tsk->mm->start_data; + else if (off == PT_TEXT_END_ADDR) + tmp = tsk->mm->end_code; + else if (off < sizeof(struct pt_regs)) tmp = get_user_reg(tsk, off >> 2); return put_user(tmp, ret); -- cgit v1.2.3-70-g09d2 From 2732f4b6f11689fa08e2db5689fc652d608936b5 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 24 Jul 2009 12:35:01 +0100 Subject: nommu: Remove the memory_start/end variables from ARM page-nommu.h These variables do not seem to be used anywhere in the kernel. Signed-off-by: Catalin Marinas --- arch/arm/include/asm/page-nommu.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/page-nommu.h b/arch/arm/include/asm/page-nommu.h index 3574c0deb37..d1b162a18dc 100644 --- a/arch/arm/include/asm/page-nommu.h +++ b/arch/arm/include/asm/page-nommu.h @@ -43,7 +43,4 @@ typedef unsigned long pgprot_t; #define __pmd(x) (x) #define __pgprot(x) (x) -extern unsigned long memory_start; -extern unsigned long memory_end; - #endif -- cgit v1.2.3-70-g09d2 From 9e1b32caa525cb236e80e9c671e179bcecccc657 Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Wed, 22 Jul 2009 15:44:28 +1000 Subject: mm: Pass virtual address to [__]p{te,ud,md}_free_tlb() mm: Pass virtual address to [__]p{te,ud,md}_free_tlb() Upcoming paches to support the new 64-bit "BookE" powerpc architecture will need to have the virtual address corresponding to PTE page when freeing it, due to the way the HW table walker works. Basically, the TLB can be loaded with "large" pages that cover the whole virtual space (well, sort-of, half of it actually) represented by a PTE page, and which contain an "indirect" bit indicating that this TLB entry RPN points to an array of PTEs from which the TLB can then create direct entries. Thus, in order to invalidate those when PTE pages are deleted, we need the virtual address to pass to tlbilx or tlbivax instructions. The old trick of sticking it somewhere in the PTE page struct page sucks too much, the address is almost readily available in all call sites and almost everybody implemets these as macros, so we may as well add the argument everywhere. I added it to the pmd and pud variants for consistency. Signed-off-by: Benjamin Herrenschmidt Acked-by: David Howells [MN10300 & FRV] Acked-by: Nick Piggin Acked-by: Martin Schwidefsky [s390] Signed-off-by: Linus Torvalds --- arch/alpha/include/asm/tlb.h | 4 ++-- arch/arm/include/asm/tlb.h | 4 ++-- arch/avr32/include/asm/pgalloc.h | 2 +- arch/cris/include/asm/pgalloc.h | 2 +- arch/frv/include/asm/pgalloc.h | 4 ++-- arch/frv/include/asm/pgtable.h | 2 +- arch/ia64/include/asm/pgalloc.h | 6 +++--- arch/ia64/include/asm/tlb.h | 12 ++++++------ arch/m32r/include/asm/pgalloc.h | 4 ++-- arch/m68k/include/asm/motorola_pgalloc.h | 6 ++++-- arch/m68k/include/asm/sun3_pgalloc.h | 4 ++-- arch/microblaze/include/asm/pgalloc.h | 4 ++-- arch/mips/include/asm/pgalloc.h | 6 +++--- arch/mn10300/include/asm/pgalloc.h | 2 +- arch/parisc/include/asm/tlb.h | 4 ++-- arch/powerpc/include/asm/pgalloc-32.h | 2 +- arch/powerpc/include/asm/pgalloc-64.h | 4 ++-- arch/powerpc/include/asm/pgalloc.h | 6 +++--- arch/powerpc/mm/hugetlbpage.c | 4 ++-- arch/s390/include/asm/tlb.h | 9 ++++++--- arch/sh/include/asm/pgalloc.h | 4 ++-- arch/sh/include/asm/tlb.h | 6 +++--- arch/sparc/include/asm/pgalloc_32.h | 8 ++++---- arch/sparc/include/asm/tlb_64.h | 6 +++--- arch/um/include/asm/pgalloc.h | 4 ++-- arch/um/include/asm/tlb.h | 6 +++--- arch/x86/include/asm/pgalloc.h | 25 ++++++++++++++++++++++--- arch/x86/mm/pgtable.c | 6 +++--- arch/xtensa/include/asm/tlb.h | 2 +- include/asm-generic/4level-fixup.h | 4 ++-- include/asm-generic/pgtable-nopmd.h | 2 +- include/asm-generic/pgtable-nopud.h | 2 +- include/asm-generic/tlb.h | 12 ++++++------ mm/memory.c | 11 ++++++----- 34 files changed, 107 insertions(+), 82 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/alpha/include/asm/tlb.h b/arch/alpha/include/asm/tlb.h index c13636575fb..42866759f3f 100644 --- a/arch/alpha/include/asm/tlb.h +++ b/arch/alpha/include/asm/tlb.h @@ -9,7 +9,7 @@ #include -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte) -#define __pmd_free_tlb(tlb, pmd) pmd_free((tlb)->mm, pmd) +#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) +#define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) #endif diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index 321c83e43a1..f41a6f57cd1 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -102,8 +102,8 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) } #define tlb_remove_page(tlb,page) free_page_and_swap_cache(page) -#define pte_free_tlb(tlb, ptep) pte_free((tlb)->mm, ptep) -#define pmd_free_tlb(tlb, pmdp) pmd_free((tlb)->mm, pmdp) +#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep) +#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) #define tlb_migrate_finish(mm) do { } while (0) diff --git a/arch/avr32/include/asm/pgalloc.h b/arch/avr32/include/asm/pgalloc.h index 64082132394..92ecd8446ef 100644 --- a/arch/avr32/include/asm/pgalloc.h +++ b/arch/avr32/include/asm/pgalloc.h @@ -83,7 +83,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) quicklist_free_page(QUICK_PT, NULL, pte); } -#define __pte_free_tlb(tlb,pte) \ +#define __pte_free_tlb(tlb,pte,addr) \ do { \ pgtable_page_dtor(pte); \ tlb_remove_page((tlb), pte); \ diff --git a/arch/cris/include/asm/pgalloc.h b/arch/cris/include/asm/pgalloc.h index a1ba761d057..6da975db112 100644 --- a/arch/cris/include/asm/pgalloc.h +++ b/arch/cris/include/asm/pgalloc.h @@ -47,7 +47,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) __free_page(pte); } -#define __pte_free_tlb(tlb,pte) \ +#define __pte_free_tlb(tlb,pte,address) \ do { \ pgtable_page_dtor(pte); \ tlb_remove_page((tlb), pte); \ diff --git a/arch/frv/include/asm/pgalloc.h b/arch/frv/include/asm/pgalloc.h index 971e6addb00..416d19a632f 100644 --- a/arch/frv/include/asm/pgalloc.h +++ b/arch/frv/include/asm/pgalloc.h @@ -49,7 +49,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) __free_page(pte); } -#define __pte_free_tlb(tlb,pte) \ +#define __pte_free_tlb(tlb,pte,address) \ do { \ pgtable_page_dtor(pte); \ tlb_remove_page((tlb),(pte)); \ @@ -62,7 +62,7 @@ do { \ */ #define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *) 2); }) #define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb,x) do { } while (0) +#define __pmd_free_tlb(tlb,x,a) do { } while (0) #endif /* CONFIG_MMU */ diff --git a/arch/frv/include/asm/pgtable.h b/arch/frv/include/asm/pgtable.h index 33233011b1c..22c60692b55 100644 --- a/arch/frv/include/asm/pgtable.h +++ b/arch/frv/include/asm/pgtable.h @@ -225,7 +225,7 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) */ #define pud_alloc_one(mm, address) NULL #define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x) do { } while (0) +#define __pud_free_tlb(tlb, x, address) do { } while (0) /* * The "pud_xxx()" functions here are trivial for a folded two-level diff --git a/arch/ia64/include/asm/pgalloc.h b/arch/ia64/include/asm/pgalloc.h index b9ac1a6fc21..96a8d927db2 100644 --- a/arch/ia64/include/asm/pgalloc.h +++ b/arch/ia64/include/asm/pgalloc.h @@ -48,7 +48,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) { quicklist_free(0, NULL, pud); } -#define __pud_free_tlb(tlb, pud) pud_free((tlb)->mm, pud) +#define __pud_free_tlb(tlb, pud, address) pud_free((tlb)->mm, pud) #endif /* CONFIG_PGTABLE_4 */ static inline void @@ -67,7 +67,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) quicklist_free(0, NULL, pmd); } -#define __pmd_free_tlb(tlb, pmd) pmd_free((tlb)->mm, pmd) +#define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) static inline void pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, pgtable_t pte) @@ -117,6 +117,6 @@ static inline void check_pgt_cache(void) quicklist_trim(0, NULL, 25, 16); } -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) #endif /* _ASM_IA64_PGALLOC_H */ diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h index 20d8a39680c..85d965cb19a 100644 --- a/arch/ia64/include/asm/tlb.h +++ b/arch/ia64/include/asm/tlb.h @@ -236,22 +236,22 @@ do { \ __tlb_remove_tlb_entry(tlb, ptep, addr); \ } while (0) -#define pte_free_tlb(tlb, ptep) \ +#define pte_free_tlb(tlb, ptep, address) \ do { \ tlb->need_flush = 1; \ - __pte_free_tlb(tlb, ptep); \ + __pte_free_tlb(tlb, ptep, address); \ } while (0) -#define pmd_free_tlb(tlb, ptep) \ +#define pmd_free_tlb(tlb, ptep, address) \ do { \ tlb->need_flush = 1; \ - __pmd_free_tlb(tlb, ptep); \ + __pmd_free_tlb(tlb, ptep, address); \ } while (0) -#define pud_free_tlb(tlb, pudp) \ +#define pud_free_tlb(tlb, pudp, address) \ do { \ tlb->need_flush = 1; \ - __pud_free_tlb(tlb, pudp); \ + __pud_free_tlb(tlb, pudp, address); \ } while (0) #endif /* _ASM_IA64_TLB_H */ diff --git a/arch/m32r/include/asm/pgalloc.h b/arch/m32r/include/asm/pgalloc.h index f11a2b909cd..0fc73619897 100644 --- a/arch/m32r/include/asm/pgalloc.h +++ b/arch/m32r/include/asm/pgalloc.h @@ -58,7 +58,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) __free_page(pte); } -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, (pte)) +#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, (pte)) /* * allocating and freeing a pmd is trivial: the 1-entry pmd is @@ -68,7 +68,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) #define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); }) #define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb, x) do { } while (0) +#define __pmd_free_tlb(tlb, x, addr) do { } while (0) #define pgd_populate(mm, pmd, pte) BUG() #define check_pgt_cache() do { } while (0) diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h index d08bf6261df..15ee4c74a9f 100644 --- a/arch/m68k/include/asm/motorola_pgalloc.h +++ b/arch/m68k/include/asm/motorola_pgalloc.h @@ -54,7 +54,8 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t page) __free_page(page); } -static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page) +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, + unsigned long address) { pgtable_page_dtor(page); cache_page(kmap(page)); @@ -73,7 +74,8 @@ static inline int pmd_free(struct mm_struct *mm, pmd_t *pmd) return free_pointer_table(pmd); } -static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) +static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long address) { return free_pointer_table(pmd); } diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h index d4c83f14381..48d80d5a666 100644 --- a/arch/m68k/include/asm/sun3_pgalloc.h +++ b/arch/m68k/include/asm/sun3_pgalloc.h @@ -32,7 +32,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t page) __free_page(page); } -#define __pte_free_tlb(tlb,pte) \ +#define __pte_free_tlb(tlb,pte,addr) \ do { \ pgtable_page_dtor(pte); \ tlb_remove_page((tlb), pte); \ @@ -80,7 +80,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page * inside the pgd, so has no extra memory associated with it. */ #define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb, x) do { } while (0) +#define __pmd_free_tlb(tlb, x, addr) do { } while (0) static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) { diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h index 59a757e46ba..b0131da1387 100644 --- a/arch/microblaze/include/asm/pgalloc.h +++ b/arch/microblaze/include/asm/pgalloc.h @@ -180,7 +180,7 @@ extern inline void pte_free(struct mm_struct *mm, struct page *ptepage) __free_page(ptepage); } -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, (pte)) +#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, (pte)) #define pmd_populate(mm, pmd, pte) (pmd_val(*(pmd)) = page_address(pte)) @@ -193,7 +193,7 @@ extern inline void pte_free(struct mm_struct *mm, struct page *ptepage) */ #define pmd_alloc_one(mm, address) ({ BUG(); ((pmd_t *)2); }) /*#define pmd_free(mm, x) do { } while (0)*/ -#define __pmd_free_tlb(tlb, x) do { } while (0) +#define __pmd_free_tlb(tlb, x, addr) do { } while (0) #define pgd_populate(mm, pmd, pte) BUG() extern int do_check_pgt_cache(int, int); diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h index 1275831dda2..f705735feef 100644 --- a/arch/mips/include/asm/pgalloc.h +++ b/arch/mips/include/asm/pgalloc.h @@ -98,7 +98,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) __free_pages(pte, PTE_ORDER); } -#define __pte_free_tlb(tlb,pte) \ +#define __pte_free_tlb(tlb,pte,address) \ do { \ pgtable_page_dtor(pte); \ tlb_remove_page((tlb), pte); \ @@ -111,7 +111,7 @@ do { \ * inside the pgd, so has no extra memory associated with it. */ #define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb, x) do { } while (0) +#define __pmd_free_tlb(tlb, x, addr) do { } while (0) #endif @@ -132,7 +132,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) free_pages((unsigned long)pmd, PMD_ORDER); } -#define __pmd_free_tlb(tlb, x) pmd_free((tlb)->mm, x) +#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) #endif diff --git a/arch/mn10300/include/asm/pgalloc.h b/arch/mn10300/include/asm/pgalloc.h index ec057e1bd4c..a19f11327cd 100644 --- a/arch/mn10300/include/asm/pgalloc.h +++ b/arch/mn10300/include/asm/pgalloc.h @@ -51,6 +51,6 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte) } -#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte)) +#define __pte_free_tlb(tlb, pte, addr) tlb_remove_page((tlb), (pte)) #endif /* _ASM_PGALLOC_H */ diff --git a/arch/parisc/include/asm/tlb.h b/arch/parisc/include/asm/tlb.h index 383b1db310e..07924903989 100644 --- a/arch/parisc/include/asm/tlb.h +++ b/arch/parisc/include/asm/tlb.h @@ -21,7 +21,7 @@ do { if (!(tlb)->fullmm) \ #include -#define __pmd_free_tlb(tlb, pmd) pmd_free((tlb)->mm, pmd) -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte) +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) #endif diff --git a/arch/powerpc/include/asm/pgalloc-32.h b/arch/powerpc/include/asm/pgalloc-32.h index 0815eb40aca..c9500d666a1 100644 --- a/arch/powerpc/include/asm/pgalloc-32.h +++ b/arch/powerpc/include/asm/pgalloc-32.h @@ -16,7 +16,7 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); */ /* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */ #define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb,x) do { } while (0) +#define __pmd_free_tlb(tlb,x,a) do { } while (0) /* #define pgd_populate(mm, pmd, pte) BUG() */ #ifndef CONFIG_BOOKE diff --git a/arch/powerpc/include/asm/pgalloc-64.h b/arch/powerpc/include/asm/pgalloc-64.h index afda2bdd860..e6f069c4f71 100644 --- a/arch/powerpc/include/asm/pgalloc-64.h +++ b/arch/powerpc/include/asm/pgalloc-64.h @@ -118,11 +118,11 @@ static inline void pgtable_free(pgtable_free_t pgf) kmem_cache_free(pgtable_cache[cachenum], p); } -#define __pmd_free_tlb(tlb, pmd) \ +#define __pmd_free_tlb(tlb, pmd,addr) \ pgtable_free_tlb(tlb, pgtable_free_cache(pmd, \ PMD_CACHE_NUM, PMD_TABLE_SIZE-1)) #ifndef CONFIG_PPC_64K_PAGES -#define __pud_free_tlb(tlb, pud) \ +#define __pud_free_tlb(tlb, pud, addr) \ pgtable_free_tlb(tlb, pgtable_free_cache(pud, \ PUD_CACHE_NUM, PUD_TABLE_SIZE-1)) #endif /* CONFIG_PPC_64K_PAGES */ diff --git a/arch/powerpc/include/asm/pgalloc.h b/arch/powerpc/include/asm/pgalloc.h index 5d8480265a7..1730e5e298d 100644 --- a/arch/powerpc/include/asm/pgalloc.h +++ b/arch/powerpc/include/asm/pgalloc.h @@ -38,14 +38,14 @@ static inline pgtable_free_t pgtable_free_cache(void *p, int cachenum, extern void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf); #ifdef CONFIG_SMP -#define __pte_free_tlb(tlb,ptepage) \ +#define __pte_free_tlb(tlb,ptepage,address) \ do { \ pgtable_page_dtor(ptepage); \ pgtable_free_tlb(tlb, pgtable_free_cache(page_address(ptepage), \ - PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \ + PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \ } while (0) #else -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, (pte)) +#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, (pte)) #endif diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 9920d6a7cf2..c46ef2ffa3d 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -305,7 +305,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, pmd = pmd_offset(pud, start); pud_clear(pud); - pmd_free_tlb(tlb, pmd); + pmd_free_tlb(tlb, pmd, start); } static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, @@ -348,7 +348,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, pud = pud_offset(pgd, start); pgd_clear(pgd); - pud_free_tlb(tlb, pud); + pud_free_tlb(tlb, pud, start); } /* diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index 3d8a96d39d9..81150b05368 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -96,7 +96,8 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) * pte_free_tlb frees a pte table and clears the CRSTE for the * page table from the tlb. */ -static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte) +static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, + unsigned long address) { if (!tlb->fullmm) { tlb->array[tlb->nr_ptes++] = pte; @@ -113,7 +114,8 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte) * as the pgd. pmd_free_tlb checks the asce_limit against 2GB * to avoid the double free of the pmd in this case. */ -static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) +static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long address) { #ifdef __s390x__ if (tlb->mm->context.asce_limit <= (1UL << 31)) @@ -134,7 +136,8 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) * as the pgd. pud_free_tlb checks the asce_limit against 4TB * to avoid the double free of the pud in this case. */ -static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) +static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, + unsigned long address) { #ifdef __s390x__ if (tlb->mm->context.asce_limit <= (1UL << 42)) diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h index 84dd2db7104..89a482750a5 100644 --- a/arch/sh/include/asm/pgalloc.h +++ b/arch/sh/include/asm/pgalloc.h @@ -73,7 +73,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) quicklist_free_page(QUICK_PT, NULL, pte); } -#define __pte_free_tlb(tlb,pte) \ +#define __pte_free_tlb(tlb,pte,addr) \ do { \ pgtable_page_dtor(pte); \ tlb_remove_page((tlb), (pte)); \ @@ -85,7 +85,7 @@ do { \ */ #define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb,x) do { } while (0) +#define __pmd_free_tlb(tlb,x,addr) do { } while (0) static inline void check_pgt_cache(void) { diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h index 9c16f737074..da8fe7ab872 100644 --- a/arch/sh/include/asm/tlb.h +++ b/arch/sh/include/asm/tlb.h @@ -91,9 +91,9 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) } #define tlb_remove_page(tlb,page) free_page_and_swap_cache(page) -#define pte_free_tlb(tlb, ptep) pte_free((tlb)->mm, ptep) -#define pmd_free_tlb(tlb, pmdp) pmd_free((tlb)->mm, pmdp) -#define pud_free_tlb(tlb, pudp) pud_free((tlb)->mm, pudp) +#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep) +#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) +#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) #define tlb_migrate_finish(mm) do { } while (0) diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h index 681582d2696..ca2b34456c4 100644 --- a/arch/sparc/include/asm/pgalloc_32.h +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -44,8 +44,8 @@ BTFIXUPDEF_CALL(pmd_t *, pmd_alloc_one, struct mm_struct *, unsigned long) BTFIXUPDEF_CALL(void, free_pmd_fast, pmd_t *) #define free_pmd_fast(pmd) BTFIXUP_CALL(free_pmd_fast)(pmd) -#define pmd_free(mm, pmd) free_pmd_fast(pmd) -#define __pmd_free_tlb(tlb, pmd) pmd_free((tlb)->mm, pmd) +#define pmd_free(mm, pmd) free_pmd_fast(pmd) +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *) #define pmd_populate(MM, PMD, PTE) BTFIXUP_CALL(pmd_populate)(PMD, PTE) @@ -62,7 +62,7 @@ BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *) #define pte_free_kernel(mm, pte) BTFIXUP_CALL(free_pte_fast)(pte) BTFIXUPDEF_CALL(void, pte_free, pgtable_t ) -#define pte_free(mm, pte) BTFIXUP_CALL(pte_free)(pte) -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte) +#define pte_free(mm, pte) BTFIXUP_CALL(pte_free)(pte) +#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) #endif /* _SPARC_PGALLOC_H */ diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h index ee38e731bfa..dca406b9b6f 100644 --- a/arch/sparc/include/asm/tlb_64.h +++ b/arch/sparc/include/asm/tlb_64.h @@ -100,9 +100,9 @@ static inline void tlb_remove_page(struct mmu_gather *mp, struct page *page) } #define tlb_remove_tlb_entry(mp,ptep,addr) do { } while (0) -#define pte_free_tlb(mp, ptepage) pte_free((mp)->mm, ptepage) -#define pmd_free_tlb(mp, pmdp) pmd_free((mp)->mm, pmdp) -#define pud_free_tlb(tlb,pudp) __pud_free_tlb(tlb,pudp) +#define pte_free_tlb(mp, ptepage, addr) pte_free((mp)->mm, ptepage) +#define pmd_free_tlb(mp, pmdp, addr) pmd_free((mp)->mm, pmdp) +#define pud_free_tlb(tlb,pudp, addr) __pud_free_tlb(tlb,pudp,addr) #define tlb_migrate_finish(mm) do { } while (0) #define tlb_start_vma(tlb, vma) do { } while (0) diff --git a/arch/um/include/asm/pgalloc.h b/arch/um/include/asm/pgalloc.h index 718984359f8..32c8ce4e151 100644 --- a/arch/um/include/asm/pgalloc.h +++ b/arch/um/include/asm/pgalloc.h @@ -40,7 +40,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) __free_page(pte); } -#define __pte_free_tlb(tlb,pte) \ +#define __pte_free_tlb(tlb,pte, address) \ do { \ pgtable_page_dtor(pte); \ tlb_remove_page((tlb),(pte)); \ @@ -53,7 +53,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) free_page((unsigned long)pmd); } -#define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x)) +#define __pmd_free_tlb(tlb,x, address) tlb_remove_page((tlb),virt_to_page(x)) #endif #define check_pgt_cache() do { } while (0) diff --git a/arch/um/include/asm/tlb.h b/arch/um/include/asm/tlb.h index 5240fa1c5e0..660caedac9e 100644 --- a/arch/um/include/asm/tlb.h +++ b/arch/um/include/asm/tlb.h @@ -116,11 +116,11 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) -#define pte_free_tlb(tlb, ptep) __pte_free_tlb(tlb, ptep) +#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) -#define pud_free_tlb(tlb, pudp) __pud_free_tlb(tlb, pudp) +#define pud_free_tlb(tlb, pudp, addr) __pud_free_tlb(tlb, pudp, addr) -#define pmd_free_tlb(tlb, pmdp) __pmd_free_tlb(tlb, pmdp) +#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) #define tlb_migrate_finish(mm) do {} while (0) diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h index dd14c54ac71..0e8c2a0fd92 100644 --- a/arch/x86/include/asm/pgalloc.h +++ b/arch/x86/include/asm/pgalloc.h @@ -46,7 +46,13 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte) __free_page(pte); } -extern void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte); +extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte); + +static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, + unsigned long address) +{ + ___pte_free_tlb(tlb, pte); +} static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) @@ -78,7 +84,13 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) free_page((unsigned long)pmd); } -extern void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); +extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd); + +static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long adddress) +{ + ___pmd_free_tlb(tlb, pmd); +} #ifdef CONFIG_X86_PAE extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd); @@ -108,7 +120,14 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) free_page((unsigned long)pud); } -extern void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); +extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud); + +static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, + unsigned long address) +{ + ___pud_free_tlb(tlb, pud); +} + #endif /* PAGETABLE_LEVELS > 3 */ #endif /* PAGETABLE_LEVELS > 2 */ diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 8e43bdd4545..af8f9650058 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -25,7 +25,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) return pte; } -void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte) +void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) { pgtable_page_dtor(pte); paravirt_release_pte(page_to_pfn(pte)); @@ -33,14 +33,14 @@ void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte) } #if PAGETABLE_LEVELS > 2 -void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) +void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) { paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT); tlb_remove_page(tlb, virt_to_page(pmd)); } #if PAGETABLE_LEVELS > 3 -void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) +void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) { paravirt_release_pud(__pa(pud) >> PAGE_SHIFT); tlb_remove_page(tlb, virt_to_page(pud)); diff --git a/arch/xtensa/include/asm/tlb.h b/arch/xtensa/include/asm/tlb.h index 31c220faca0..0d766f9c108 100644 --- a/arch/xtensa/include/asm/tlb.h +++ b/arch/xtensa/include/asm/tlb.h @@ -42,6 +42,6 @@ #include -#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) #endif /* _XTENSA_TLB_H */ diff --git a/include/asm-generic/4level-fixup.h b/include/asm-generic/4level-fixup.h index 9d40e879f99..77ff547730a 100644 --- a/include/asm-generic/4level-fixup.h +++ b/include/asm-generic/4level-fixup.h @@ -27,9 +27,9 @@ #define pud_page_vaddr(pud) pgd_page_vaddr(pud) #undef pud_free_tlb -#define pud_free_tlb(tlb, x) do { } while (0) +#define pud_free_tlb(tlb, x, addr) do { } while (0) #define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x) do { } while (0) +#define __pud_free_tlb(tlb, x, addr) do { } while (0) #undef pud_addr_end #define pud_addr_end(addr, end) (end) diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index a7cdc48e8b7..725612b793c 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -59,7 +59,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) { } -#define __pmd_free_tlb(tlb, x) do { } while (0) +#define __pmd_free_tlb(tlb, x, a) do { } while (0) #undef pmd_addr_end #define pmd_addr_end(addr, end) (end) diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index 87cf449a6df..810431d8351 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -52,7 +52,7 @@ static inline pud_t * pud_offset(pgd_t * pgd, unsigned long address) */ #define pud_alloc_one(mm, address) NULL #define pud_free(mm, x) do { } while (0) -#define __pud_free_tlb(tlb, x) do { } while (0) +#define __pud_free_tlb(tlb, x, a) do { } while (0) #undef pud_addr_end #define pud_addr_end(addr, end) (end) diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index f490e43a90b..e43f9766259 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -123,24 +123,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) -#define pte_free_tlb(tlb, ptep) \ +#define pte_free_tlb(tlb, ptep, address) \ do { \ tlb->need_flush = 1; \ - __pte_free_tlb(tlb, ptep); \ + __pte_free_tlb(tlb, ptep, address); \ } while (0) #ifndef __ARCH_HAS_4LEVEL_HACK -#define pud_free_tlb(tlb, pudp) \ +#define pud_free_tlb(tlb, pudp, address) \ do { \ tlb->need_flush = 1; \ - __pud_free_tlb(tlb, pudp); \ + __pud_free_tlb(tlb, pudp, address); \ } while (0) #endif -#define pmd_free_tlb(tlb, pmdp) \ +#define pmd_free_tlb(tlb, pmdp, address) \ do { \ tlb->need_flush = 1; \ - __pmd_free_tlb(tlb, pmdp); \ + __pmd_free_tlb(tlb, pmdp, address); \ } while (0) #define tlb_migrate_finish(mm) do {} while (0) diff --git a/mm/memory.c b/mm/memory.c index 65216194eb8..aede2ce3aba 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -135,11 +135,12 @@ void pmd_clear_bad(pmd_t *pmd) * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */ -static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd) +static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long addr) { pgtable_t token = pmd_pgtable(*pmd); pmd_clear(pmd); - pte_free_tlb(tlb, token); + pte_free_tlb(tlb, token, addr); tlb->mm->nr_ptes--; } @@ -157,7 +158,7 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, next = pmd_addr_end(addr, end); if (pmd_none_or_clear_bad(pmd)) continue; - free_pte_range(tlb, pmd); + free_pte_range(tlb, pmd, addr); } while (pmd++, addr = next, addr != end); start &= PUD_MASK; @@ -173,7 +174,7 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, pmd = pmd_offset(pud, start); pud_clear(pud); - pmd_free_tlb(tlb, pmd); + pmd_free_tlb(tlb, pmd, start); } static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, @@ -206,7 +207,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, pud = pud_offset(pgd, start); pgd_clear(pgd); - pud_free_tlb(tlb, pud); + pud_free_tlb(tlb, pud, start); } /* -- cgit v1.2.3-70-g09d2 From 181f817eaaca4c1f8a9c265d339d2b96de8b245d Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Thu, 13 Aug 2009 20:38:16 +0200 Subject: [ARM] support tracing when using newer compilers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since gcc 4.4 the name and calling convention for function profiling on ARM changed. With this patch both types are supported. See http://sourceware.org/ml/libc-ports/2008-04/msg00009.html for some details. Lightly-Tested-by: Anand Gadiyar Tested-by: Kevin Hilman Signed-off-by: Uwe Kleine-König --- arch/arm/include/asm/ftrace.h | 1 + arch/arm/kernel/armksyms.c | 1 + arch/arm/kernel/entry-common.S | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index 39c8bc1a006..0d4c478e01b 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -7,6 +7,7 @@ #ifndef __ASSEMBLY__ extern void mcount(void); +extern void __gnu_mcount_nc(void); #endif #endif diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c index 531e1860e54..0e627705f74 100644 --- a/arch/arm/kernel/armksyms.c +++ b/arch/arm/kernel/armksyms.c @@ -186,4 +186,5 @@ EXPORT_SYMBOL(_find_next_bit_be); #ifdef CONFIG_FUNCTION_TRACER EXPORT_SYMBOL(mcount); +EXPORT_SYMBOL(__gnu_mcount_nc); #endif diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 366e5097a41..99208728d48 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -132,6 +132,25 @@ ftrace_call: #else +ENTRY(__gnu_mcount_nc) + stmdb sp!, {r0-r3, lr} + ldr r0, =ftrace_trace_function + ldr r2, [r0] + adr r0, ftrace_stub + cmp r0, r2 + bne gnu_trace + ldmia sp!, {r0-r3, ip, lr} + bx ip + +gnu_trace: + ldr r1, [sp, #20] @ lr of instrumented routine + mov r0, lr + sub r0, r0, #MCOUNT_INSN_SIZE + mov lr, pc + mov pc, r2 + ldmia sp!, {r0-r3, ip, lr} + bx ip + ENTRY(mcount) stmdb sp!, {r0-r3, lr} ldr r0, =ftrace_trace_function -- cgit v1.2.3-70-g09d2 From 0d928b0b616d1c5c5fe76019a87cba171ca91633 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Thu, 13 Aug 2009 20:38:17 +0200 Subject: Complete irq tracing support for ARM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this patch enabling and disabling irqs in assembler code and by the hardware wasn't tracked completly. I had to transpose two instructions in arch/arm/lib/bitops.h because restore_irqs doesn't preserve the flags with CONFIG_TRACE_IRQFLAGS=y Signed-off-by: Uwe Kleine-König Cc: Russell King Cc: Peter Zijlstra Cc: Ingo Molnar Signed-off-by: Uwe Kleine-König --- arch/arm/include/asm/assembler.h | 49 ++++++++++++++++++++++++++++++++++++---- arch/arm/kernel/entry-armv.S | 10 ++++---- arch/arm/lib/bitops.h | 2 +- 3 files changed, 49 insertions(+), 12 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 15f8a092b70..44912cd5da1 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -74,23 +74,56 @@ * Enable and disable interrupts */ #if __LINUX_ARM_ARCH__ >= 6 - .macro disable_irq + .macro disable_irq_notrace cpsid i .endm - .macro enable_irq + .macro enable_irq_notrace cpsie i .endm #else - .macro disable_irq + .macro disable_irq_notrace msr cpsr_c, #PSR_I_BIT | SVC_MODE .endm - .macro enable_irq + .macro enable_irq_notrace msr cpsr_c, #SVC_MODE .endm #endif + .macro asm_trace_hardirqs_off +#if defined(CONFIG_TRACE_IRQFLAGS) + stmdb sp!, {r0-r3, ip, lr} + bl trace_hardirqs_off + ldmia sp!, {r0-r3, ip, lr} +#endif + .endm + + .macro asm_trace_hardirqs_on_cond, cond +#if defined(CONFIG_TRACE_IRQFLAGS) + /* + * actually the registers should be pushed and pop'd conditionally, but + * after bl the flags are certainly clobbered + */ + stmdb sp!, {r0-r3, ip, lr} + bl\cond trace_hardirqs_on + ldmia sp!, {r0-r3, ip, lr} +#endif + .endm + + .macro asm_trace_hardirqs_on + asm_trace_hardirqs_on_cond al + .endm + + .macro disable_irq + disable_irq_notrace + asm_trace_hardirqs_off + .endm + + .macro enable_irq + asm_trace_hardirqs_on + enable_irq_notrace + .endm /* * Save the current IRQ state and disable IRQs. Note that this macro * assumes FIQs are enabled, and that the processor is in SVC mode. @@ -104,10 +137,16 @@ * Restore interrupt state previously stored in a register. We don't * guarantee that this will preserve the flags. */ - .macro restore_irqs, oldcpsr + .macro restore_irqs_notrace, oldcpsr msr cpsr_c, \oldcpsr .endm + .macro restore_irqs, oldcpsr + tst \oldcpsr, #PSR_I_BIT + asm_trace_hardirqs_on_cond eq + restore_irqs_notrace \oldcpsr + .endm + #define USER(x...) \ 9999: x; \ .section __ex_table,"a"; \ diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index fc8af43c500..792abd0dfae 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -151,6 +151,8 @@ ENDPROC(__und_invalid) @ r4 - orig_r0 (see pt_regs definition in ptrace.h) @ stmia r5, {r0 - r4} + + asm_trace_hardirqs_off .endm .align 5 @@ -206,9 +208,6 @@ ENDPROC(__dabt_svc) __irq_svc: svc_entry -#ifdef CONFIG_TRACE_IRQFLAGS - bl trace_hardirqs_off -#endif #ifdef CONFIG_PREEMPT get_thread_info tsk ldr r8, [tsk, #TI_PREEMPT] @ get preempt count @@ -383,6 +382,8 @@ ENDPROC(__pabt_svc) @ Clear FP to mark the first stack frame @ zero_fp + + asm_trace_hardirqs_off .endm .macro kuser_cmpxchg_check @@ -437,9 +438,6 @@ __irq_usr: usr_entry kuser_cmpxchg_check -#ifdef CONFIG_TRACE_IRQFLAGS - bl trace_hardirqs_off -#endif get_thread_info tsk #ifdef CONFIG_PREEMPT ldr r8, [tsk, #TI_PREEMPT] @ get preempt count diff --git a/arch/arm/lib/bitops.h b/arch/arm/lib/bitops.h index c7f2627385e..d42252918bf 100644 --- a/arch/arm/lib/bitops.h +++ b/arch/arm/lib/bitops.h @@ -60,8 +60,8 @@ tst r2, r0, lsl r3 \instr r2, r2, r0, lsl r3 \store r2, [r1] - restore_irqs ip moveq r0, #0 + restore_irqs ip mov pc, lr .endm #endif -- cgit v1.2.3-70-g09d2