diff options
Diffstat (limited to 'arch/powerpc/kernel')
26 files changed, 667 insertions, 80 deletions
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index c9a660e4c2d..882889b1592 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -136,6 +136,9 @@ int main(void) DEFINE(PACAEMERGSP, offsetof(struct paca_struct, emergency_sp)); DEFINE(PACALPPACAPTR, offsetof(struct paca_struct, lppaca_ptr)); DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id)); + DEFINE(PACA_STARTPURR, offsetof(struct paca_struct, startpurr)); + DEFINE(PACA_USER_TIME, offsetof(struct paca_struct, user_time)); + DEFINE(PACA_SYSTEM_TIME, offsetof(struct paca_struct, system_time)); DEFINE(LPPACASRR0, offsetof(struct lppaca, saved_srr0)); DEFINE(LPPACASRR1, offsetof(struct lppaca, saved_srr1)); diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index e4e81374cb9..39e348a3ade 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -894,7 +894,7 @@ struct cpu_spec cpu_specs[] = { .platform = "ppc405", }, { /* Xilinx Virtex-II Pro */ - .pvr_mask = 0xffff0000, + .pvr_mask = 0xfffff000, .pvr_value = 0x20010000, .cpu_name = "Virtex-II Pro", .cpu_features = CPU_FTRS_40X, @@ -904,6 +904,16 @@ struct cpu_spec cpu_specs[] = { .dcache_bsize = 32, .platform = "ppc405", }, + { /* Xilinx Virtex-4 FX */ + .pvr_mask = 0xfffff000, + .pvr_value = 0x20011000, + .cpu_name = "Virtex-4 FX", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | + PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, + .icache_bsize = 32, + .dcache_bsize = 32, + }, { /* 405EP */ .pvr_mask = 0xffff0000, .pvr_value = 0x51210000, diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 24be0cf86d7..1060155d84c 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -1,6 +1,4 @@ /* - * arch/ppc64/kernel/entry.S - * * PowerPC version * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP @@ -63,6 +61,7 @@ system_call_common: std r12,_MSR(r1) std r0,GPR0(r1) std r10,GPR1(r1) + ACCOUNT_CPU_USER_ENTRY(r10, r11) std r2,GPR2(r1) std r3,GPR3(r1) std r4,GPR4(r1) @@ -170,8 +169,9 @@ syscall_error_cont: stdcx. r0,0,r1 /* to clear the reservation */ andi. r6,r8,MSR_PR ld r4,_LINK(r1) - beq- 1f /* only restore r13 if */ - ld r13,GPR13(r1) /* returning to usermode */ + beq- 1f + ACCOUNT_CPU_USER_EXIT(r11, r12) + ld r13,GPR13(r1) /* only restore r13 if returning to usermode */ 1: ld r2,GPR2(r1) li r12,MSR_RI andc r11,r10,r12 @@ -322,7 +322,7 @@ _GLOBAL(ret_from_fork) * the fork code also. * * The code which creates the new task context is in 'copy_thread' - * in arch/ppc64/kernel/process.c + * in arch/powerpc/kernel/process.c */ .align 7 _GLOBAL(_switch) @@ -486,6 +486,7 @@ restore: * userspace */ beq 1f + ACCOUNT_CPU_USER_EXIT(r3, r4) REST_GPR(13, r1) 1: ld r3,_CTR(r1) diff --git a/arch/powerpc/kernel/firmware.c b/arch/powerpc/kernel/firmware.c index 65eae752a52..4d37a3cb80f 100644 --- a/arch/powerpc/kernel/firmware.c +++ b/arch/powerpc/kernel/firmware.c @@ -18,28 +18,3 @@ #include <asm/firmware.h> unsigned long ppc64_firmware_features; - -#ifdef CONFIG_PPC_PSERIES -firmware_feature_t firmware_features_table[FIRMWARE_MAX_FEATURES] = { - {FW_FEATURE_PFT, "hcall-pft"}, - {FW_FEATURE_TCE, "hcall-tce"}, - {FW_FEATURE_SPRG0, "hcall-sprg0"}, - {FW_FEATURE_DABR, "hcall-dabr"}, - {FW_FEATURE_COPY, "hcall-copy"}, - {FW_FEATURE_ASR, "hcall-asr"}, - {FW_FEATURE_DEBUG, "hcall-debug"}, - {FW_FEATURE_PERF, "hcall-perf"}, - {FW_FEATURE_DUMP, "hcall-dump"}, - {FW_FEATURE_INTERRUPT, "hcall-interrupt"}, - {FW_FEATURE_MIGRATE, "hcall-migrate"}, - {FW_FEATURE_PERFMON, "hcall-perfmon"}, - {FW_FEATURE_CRQ, "hcall-crq"}, - {FW_FEATURE_VIO, "hcall-vio"}, - {FW_FEATURE_RDMA, "hcall-rdma"}, - {FW_FEATURE_LLAN, "hcall-lLAN"}, - {FW_FEATURE_BULK, "hcall-bulk"}, - {FW_FEATURE_XDABR, "hcall-xdabr"}, - {FW_FEATURE_MULTITCE, "hcall-multi-tce"}, - {FW_FEATURE_SPLPAR, "hcall-splpar"}, -}; -#endif diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index 8b49679fad5..47c7fa148c9 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -1,6 +1,4 @@ /* - * arch/ppc/kernel/head_44x.S - * * Kernel execution entry point code. * * Copyright (c) 1995-1996 Gary Thomas <gdt@linuxppc.org> diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index 9b65029dd2a..35084f3a841 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -1,6 +1,4 @@ /* - * arch/ppc64/kernel/head.S - * * PowerPC version * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) * @@ -279,6 +277,7 @@ exception_marker: std r10,0(r1); /* make stack chain pointer */ \ std r0,GPR0(r1); /* save r0 in stackframe */ \ std r10,GPR1(r1); /* save r1 in stackframe */ \ + ACCOUNT_CPU_USER_ENTRY(r9, r10); \ std r2,GPR2(r1); /* save r2 in stackframe */ \ SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ @@ -846,6 +845,14 @@ fast_exception_return: ld r11,_NIP(r1) andi. r3,r12,MSR_RI /* check if RI is set */ beq- unrecov_fer + +#ifdef CONFIG_VIRT_CPU_ACCOUNTING + andi. r3,r12,MSR_PR + beq 2f + ACCOUNT_CPU_USER_EXIT(r3, r4) +2: +#endif + ld r3,_CCR(r1) ld r4,_LINK(r1) ld r5,_CTR(r1) diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index bc6d1ac5523..28941f5ce67 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -1,6 +1,4 @@ /* - * arch/ppc/kernel/except_8xx.S - * * PowerPC version * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h new file mode 100644 index 00000000000..8536e767616 --- /dev/null +++ b/arch/powerpc/kernel/head_booke.h @@ -0,0 +1,363 @@ +#ifndef __HEAD_BOOKE_H__ +#define __HEAD_BOOKE_H__ + +/* + * Macros used for common Book-e exception handling + */ + +#define SET_IVOR(vector_number, vector_label) \ + li r26,vector_label@l; \ + mtspr SPRN_IVOR##vector_number,r26; \ + sync + +#define NORMAL_EXCEPTION_PROLOG \ + mtspr SPRN_SPRG0,r10; /* save two registers to work with */\ + mtspr SPRN_SPRG1,r11; \ + mtspr SPRN_SPRG4W,r1; \ + mfcr r10; /* save CR in r10 for now */\ + mfspr r11,SPRN_SRR1; /* check whether user or kernel */\ + andi. r11,r11,MSR_PR; \ + beq 1f; \ + mfspr r1,SPRN_SPRG3; /* if from user, start at top of */\ + lwz r1,THREAD_INFO-THREAD(r1); /* this thread's kernel stack */\ + addi r1,r1,THREAD_SIZE; \ +1: subi r1,r1,INT_FRAME_SIZE; /* Allocate an exception frame */\ + mr r11,r1; \ + stw r10,_CCR(r11); /* save various registers */\ + stw r12,GPR12(r11); \ + stw r9,GPR9(r11); \ + mfspr r10,SPRN_SPRG0; \ + stw r10,GPR10(r11); \ + mfspr r12,SPRN_SPRG1; \ + stw r12,GPR11(r11); \ + mflr r10; \ + stw r10,_LINK(r11); \ + mfspr r10,SPRN_SPRG4R; \ + mfspr r12,SPRN_SRR0; \ + stw r10,GPR1(r11); \ + mfspr r9,SPRN_SRR1; \ + stw r10,0(r11); \ + rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\ + stw r0,GPR0(r11); \ + SAVE_4GPRS(3, r11); \ + SAVE_2GPRS(7, r11) + +/* To handle the additional exception priority levels on 40x and Book-E + * processors we allocate a 4k stack per additional priority level. The various + * head_xxx.S files allocate space (exception_stack_top) for each priority's + * stack times the number of CPUs + * + * On 40x critical is the only additional level + * On 44x/e500 we have critical and machine check + * On e200 we have critical and debug (machine check occurs via critical) + * + * Additionally we reserve a SPRG for each priority level so we can free up a + * GPR to use as the base for indirect access to the exception stacks. This + * is necessary since the MMU is always on, for Book-E parts, and the stacks + * are offset from KERNELBASE. + * + */ +#define BOOKE_EXCEPTION_STACK_SIZE (8192) + +/* CRIT_SPRG only used in critical exception handling */ +#define CRIT_SPRG SPRN_SPRG2 +/* MCHECK_SPRG only used in machine check exception handling */ +#define MCHECK_SPRG SPRN_SPRG6W + +#define MCHECK_STACK_TOP (exception_stack_top - 4096) +#define CRIT_STACK_TOP (exception_stack_top) + +/* only on e200 for now */ +#define DEBUG_STACK_TOP (exception_stack_top - 4096) +#define DEBUG_SPRG SPRN_SPRG6W + +#ifdef CONFIG_SMP +#define BOOKE_LOAD_EXC_LEVEL_STACK(level) \ + mfspr r8,SPRN_PIR; \ + mulli r8,r8,BOOKE_EXCEPTION_STACK_SIZE; \ + neg r8,r8; \ + addis r8,r8,level##_STACK_TOP@ha; \ + addi r8,r8,level##_STACK_TOP@l +#else +#define BOOKE_LOAD_EXC_LEVEL_STACK(level) \ + lis r8,level##_STACK_TOP@h; \ + ori r8,r8,level##_STACK_TOP@l +#endif + +/* + * Exception prolog for critical/machine check exceptions. This is a + * little different from the normal exception prolog above since a + * critical/machine check exception can potentially occur at any point + * during normal exception processing. Thus we cannot use the same SPRG + * registers as the normal prolog above. Instead we use a portion of the + * critical/machine check exception stack at low physical addresses. + */ +#define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, exc_level_srr0, exc_level_srr1) \ + mtspr exc_level##_SPRG,r8; \ + BOOKE_LOAD_EXC_LEVEL_STACK(exc_level);/* r8 points to the exc_level stack*/ \ + stw r10,GPR10-INT_FRAME_SIZE(r8); \ + stw r11,GPR11-INT_FRAME_SIZE(r8); \ + mfcr r10; /* save CR in r10 for now */\ + mfspr r11,exc_level_srr1; /* check whether user or kernel */\ + andi. r11,r11,MSR_PR; \ + mr r11,r8; \ + mfspr r8,exc_level##_SPRG; \ + beq 1f; \ + /* COMING FROM USER MODE */ \ + mfspr r11,SPRN_SPRG3; /* if from user, start at top of */\ + lwz r11,THREAD_INFO-THREAD(r11); /* this thread's kernel stack */\ + addi r11,r11,THREAD_SIZE; \ +1: subi r11,r11,INT_FRAME_SIZE; /* Allocate an exception frame */\ + stw r10,_CCR(r11); /* save various registers */\ + stw r12,GPR12(r11); \ + stw r9,GPR9(r11); \ + mflr r10; \ + stw r10,_LINK(r11); \ + mfspr r12,SPRN_DEAR; /* save DEAR and ESR in the frame */\ + stw r12,_DEAR(r11); /* since they may have had stuff */\ + mfspr r9,SPRN_ESR; /* in them at the point where the */\ + stw r9,_ESR(r11); /* exception was taken */\ + mfspr r12,exc_level_srr0; \ + stw r1,GPR1(r11); \ + mfspr r9,exc_level_srr1; \ + stw r1,0(r11); \ + mr r1,r11; \ + rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\ + stw r0,GPR0(r11); \ + SAVE_4GPRS(3, r11); \ + SAVE_2GPRS(7, r11) + +#define CRITICAL_EXCEPTION_PROLOG \ + EXC_LEVEL_EXCEPTION_PROLOG(CRIT, SPRN_CSRR0, SPRN_CSRR1) +#define DEBUG_EXCEPTION_PROLOG \ + EXC_LEVEL_EXCEPTION_PROLOG(DEBUG, SPRN_DSRR0, SPRN_DSRR1) +#define MCHECK_EXCEPTION_PROLOG \ + EXC_LEVEL_EXCEPTION_PROLOG(MCHECK, SPRN_MCSRR0, SPRN_MCSRR1) + +/* + * Exception vectors. + */ +#define START_EXCEPTION(label) \ + .align 5; \ +label: + +#define FINISH_EXCEPTION(func) \ + bl transfer_to_handler_full; \ + .long func; \ + .long ret_from_except_full + +#define EXCEPTION(n, label, hdlr, xfer) \ + START_EXCEPTION(label); \ + NORMAL_EXCEPTION_PROLOG; \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + xfer(n, hdlr) + +#define CRITICAL_EXCEPTION(n, label, hdlr) \ + START_EXCEPTION(label); \ + CRITICAL_EXCEPTION_PROLOG; \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ + NOCOPY, crit_transfer_to_handler, \ + ret_from_crit_exc) + +#define MCHECK_EXCEPTION(n, label, hdlr) \ + START_EXCEPTION(label); \ + MCHECK_EXCEPTION_PROLOG; \ + mfspr r5,SPRN_ESR; \ + stw r5,_ESR(r11); \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ + NOCOPY, mcheck_transfer_to_handler, \ + ret_from_mcheck_exc) + +#define EXC_XFER_TEMPLATE(hdlr, trap, msr, copyee, tfer, ret) \ + li r10,trap; \ + stw r10,_TRAP(r11); \ + lis r10,msr@h; \ + ori r10,r10,msr@l; \ + copyee(r10, r9); \ + bl tfer; \ + .long hdlr; \ + .long ret + +#define COPY_EE(d, s) rlwimi d,s,0,16,16 +#define NOCOPY(d, s) + +#define EXC_XFER_STD(n, hdlr) \ + EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, NOCOPY, transfer_to_handler_full, \ + ret_from_except_full) + +#define EXC_XFER_LITE(n, hdlr) \ + EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \ + ret_from_except) + +#define EXC_XFER_EE(n, hdlr) \ + EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, COPY_EE, transfer_to_handler_full, \ + ret_from_except_full) + +#define EXC_XFER_EE_LITE(n, hdlr) \ + EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, COPY_EE, transfer_to_handler, \ + ret_from_except) + +/* Check for a single step debug exception while in an exception + * handler before state has been saved. This is to catch the case + * where an instruction that we are trying to single step causes + * an exception (eg ITLB/DTLB miss) and thus the first instruction of + * the exception handler generates a single step debug exception. + * + * If we get a debug trap on the first instruction of an exception handler, + * we reset the MSR_DE in the _exception handler's_ MSR (the debug trap is + * a critical exception, so we are using SPRN_CSRR1 to manipulate the MSR). + * The exception handler was handling a non-critical interrupt, so it will + * save (and later restore) the MSR via SPRN_CSRR1, which will still have + * the MSR_DE bit set. + */ +#ifdef CONFIG_E200 +#define DEBUG_EXCEPTION \ + START_EXCEPTION(Debug); \ + DEBUG_EXCEPTION_PROLOG; \ + \ + /* \ + * If there is a single step or branch-taken exception in an \ + * exception entry sequence, it was probably meant to apply to \ + * the code where the exception occurred (since exception entry \ + * doesn't turn off DE automatically). We simulate the effect \ + * of turning off DE on entry to an exception handler by turning \ + * off DE in the CSRR1 value and clearing the debug status. \ + */ \ + mfspr r10,SPRN_DBSR; /* check single-step/branch taken */ \ + andis. r10,r10,DBSR_IC@h; \ + beq+ 2f; \ + \ + lis r10,KERNELBASE@h; /* check if exception in vectors */ \ + ori r10,r10,KERNELBASE@l; \ + cmplw r12,r10; \ + blt+ 2f; /* addr below exception vectors */ \ + \ + lis r10,Debug@h; \ + ori r10,r10,Debug@l; \ + cmplw r12,r10; \ + bgt+ 2f; /* addr above exception vectors */ \ + \ + /* here it looks like we got an inappropriate debug exception. */ \ +1: rlwinm r9,r9,0,~MSR_DE; /* clear DE in the CDRR1 value */ \ + lis r10,DBSR_IC@h; /* clear the IC event */ \ + mtspr SPRN_DBSR,r10; \ + /* restore state and get out */ \ + lwz r10,_CCR(r11); \ + lwz r0,GPR0(r11); \ + lwz r1,GPR1(r11); \ + mtcrf 0x80,r10; \ + mtspr SPRN_DSRR0,r12; \ + mtspr SPRN_DSRR1,r9; \ + lwz r9,GPR9(r11); \ + lwz r12,GPR12(r11); \ + mtspr DEBUG_SPRG,r8; \ + BOOKE_LOAD_EXC_LEVEL_STACK(DEBUG); /* r8 points to the debug stack */ \ + lwz r10,GPR10-INT_FRAME_SIZE(r8); \ + lwz r11,GPR11-INT_FRAME_SIZE(r8); \ + mfspr r8,DEBUG_SPRG; \ + \ + RFDI; \ + b .; \ + \ + /* continue normal handling for a critical exception... */ \ +2: mfspr r4,SPRN_DBSR; \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), NOCOPY, debug_transfer_to_handler, ret_from_debug_exc) +#else +#define DEBUG_EXCEPTION \ + START_EXCEPTION(Debug); \ + CRITICAL_EXCEPTION_PROLOG; \ + \ + /* \ + * If there is a single step or branch-taken exception in an \ + * exception entry sequence, it was probably meant to apply to \ + * the code where the exception occurred (since exception entry \ + * doesn't turn off DE automatically). We simulate the effect \ + * of turning off DE on entry to an exception handler by turning \ + * off DE in the CSRR1 value and clearing the debug status. \ + */ \ + mfspr r10,SPRN_DBSR; /* check single-step/branch taken */ \ + andis. r10,r10,DBSR_IC@h; \ + beq+ 2f; \ + \ + lis r10,KERNELBASE@h; /* check if exception in vectors */ \ + ori r10,r10,KERNELBASE@l; \ + cmplw r12,r10; \ + blt+ 2f; /* addr below exception vectors */ \ + \ + lis r10,Debug@h; \ + ori r10,r10,Debug@l; \ + cmplw r12,r10; \ + bgt+ 2f; /* addr above exception vectors */ \ + \ + /* here it looks like we got an inappropriate debug exception. */ \ +1: rlwinm r9,r9,0,~MSR_DE; /* clear DE in the CSRR1 value */ \ + lis r10,DBSR_IC@h; /* clear the IC event */ \ + mtspr SPRN_DBSR,r10; \ + /* restore state and get out */ \ + lwz r10,_CCR(r11); \ + lwz r0,GPR0(r11); \ + lwz r1,GPR1(r11); \ + mtcrf 0x80,r10; \ + mtspr SPRN_CSRR0,r12; \ + mtspr SPRN_CSRR1,r9; \ + lwz r9,GPR9(r11); \ + lwz r12,GPR12(r11); \ + mtspr CRIT_SPRG,r8; \ + BOOKE_LOAD_EXC_LEVEL_STACK(CRIT); /* r8 points to the debug stack */ \ + lwz r10,GPR10-INT_FRAME_SIZE(r8); \ + lwz r11,GPR11-INT_FRAME_SIZE(r8); \ + mfspr r8,CRIT_SPRG; \ + \ + rfci; \ + b .; \ + \ + /* continue normal handling for a critical exception... */ \ +2: mfspr r4,SPRN_DBSR; \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), NOCOPY, crit_transfer_to_handler, ret_from_crit_exc) +#endif + +#define INSTRUCTION_STORAGE_EXCEPTION \ + START_EXCEPTION(InstructionStorage) \ + NORMAL_EXCEPTION_PROLOG; \ + mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \ + stw r5,_ESR(r11); \ + mr r4,r12; /* Pass SRR0 as arg2 */ \ + li r5,0; /* Pass zero as arg3 */ \ + EXC_XFER_EE_LITE(0x0400, handle_page_fault) + +#define ALIGNMENT_EXCEPTION \ + START_EXCEPTION(Alignment) \ + NORMAL_EXCEPTION_PROLOG; \ + mfspr r4,SPRN_DEAR; /* Grab the DEAR and save it */ \ + stw r4,_DEAR(r11); \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_EE(0x0600, alignment_exception) + +#define PROGRAM_EXCEPTION \ + START_EXCEPTION(Program) \ + NORMAL_EXCEPTION_PROLOG; \ + mfspr r4,SPRN_ESR; /* Grab the ESR and save it */ \ + stw r4,_ESR(r11); \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_STD(0x0700, program_check_exception) + +#define DECREMENTER_EXCEPTION \ + START_EXCEPTION(Decrementer) \ + NORMAL_EXCEPTION_PROLOG; \ + lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \ + mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_LITE(0x0900, timer_interrupt) + +#define FP_UNAVAILABLE_EXCEPTION \ + START_EXCEPTION(FloatingPointUnavailable) \ + NORMAL_EXCEPTION_PROLOG; \ + bne load_up_fpu; /* if from user, just load it up */ \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + EXC_XFER_EE_LITE(0x800, kernel_fp_unavailable_exception) + +#endif /* __HEAD_BOOKE_H__ */ diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index 8d60fa99fc4..dd86bbed762 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -1,6 +1,4 @@ /* - * arch/ppc/kernel/head_fsl_booke.S - * * Kernel execution entry point code. * * Copyright (c) 1995-1996 Gary Thomas <gdt@linuxppc.org> @@ -316,6 +314,7 @@ skpinv: addi r6,r6,1 /* Increment */ */ lis r2,DBCR0_IDM@h mtspr SPRN_DBCR0,r2 + isync /* clear any residual debug events */ li r2,-1 mtspr SPRN_DBSR,r2 @@ -1002,12 +1001,15 @@ _GLOBAL(giveup_fpu) _GLOBAL(abort) li r13,0 mtspr SPRN_DBCR0,r13 /* disable all debug events */ + isync mfmsr r13 ori r13,r13,MSR_DE@l /* Enable Debug Events */ mtmsr r13 + isync mfspr r13,SPRN_DBCR0 lis r13,(DBCR0_IDM|DBCR0_RST_CHIP)@h mtspr SPRN_DBCR0,r13 + isync _GLOBAL(set_context) diff --git a/arch/powerpc/kernel/iomap.c b/arch/powerpc/kernel/iomap.c index 6160c8dbb7c..fd8214caede 100644 --- a/arch/powerpc/kernel/iomap.c +++ b/arch/powerpc/kernel/iomap.c @@ -1,6 +1,4 @@ /* - * arch/ppc64/kernel/iomap.c - * * ppc64 "iomap" interface implementation. * * (C) Copyright 2004 Linus Torvalds diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 946f3219fd2..d9a7fdef59b 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -1,5 +1,4 @@ /* - * arch/ppc64/kernel/iommu.c * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation * * Rewrite, cleanup, new allocation schemes, virtual merging: diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index d1fffce86df..24dc8117b82 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -1,6 +1,4 @@ /* - * arch/ppc/kernel/irq.c - * * Derived from arch/i386/kernel/irq.c * Copyright (C) 1992 Linus Torvalds * Adapted from arch/i386 by Gary Thomas @@ -371,6 +369,7 @@ unsigned int real_irq_to_virt_slowpath(unsigned int real_irq) return NO_IRQ; } +#endif /* CONFIG_PPC64 */ #ifdef CONFIG_IRQSTACKS struct thread_info *softirq_ctx[NR_CPUS]; @@ -394,10 +393,24 @@ void irq_ctx_init(void) } } +static inline void do_softirq_onstack(void) +{ + struct thread_info *curtp, *irqtp; + + curtp = current_thread_info(); + irqtp = softirq_ctx[smp_processor_id()]; + irqtp->task = curtp->task; + call_do_softirq(irqtp); + irqtp->task = NULL; +} + +#else +#define do_softirq_onstack() __do_softirq() +#endif /* CONFIG_IRQSTACKS */ + void do_softirq(void) { unsigned long flags; - struct thread_info *curtp, *irqtp; if (in_interrupt()) return; @@ -405,19 +418,18 @@ void do_softirq(void) local_irq_save(flags); if (local_softirq_pending()) { - curtp = current_thread_info(); - irqtp = softirq_ctx[smp_processor_id()]; - irqtp->task = curtp->task; - call_do_softirq(irqtp); - irqtp->task = NULL; + account_system_vtime(current); + local_bh_disable(); + do_softirq_onstack(); + account_system_vtime(current); + __local_bh_enable(); } local_irq_restore(flags); } EXPORT_SYMBOL(do_softirq); -#endif /* CONFIG_IRQSTACKS */ - +#ifdef CONFIG_PPC64 static int __init setup_noirqdistrib(char *str) { distribute_irqs = 0; diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c index cfab48566db..258039fb301 100644 --- a/arch/powerpc/kernel/kprobes.c +++ b/arch/powerpc/kernel/kprobes.c @@ -1,6 +1,5 @@ /* * Kernel Probes (KProbes) - * arch/ppc64/kernel/kprobes.c * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/arch/powerpc/kernel/of_device.c b/arch/powerpc/kernel/of_device.c index 22d83d4d1af..9feeeef5a87 100644 --- a/arch/powerpc/kernel/of_device.c +++ b/arch/powerpc/kernel/of_device.c @@ -147,15 +147,12 @@ postcore_initcall(of_bus_driver_init); int of_register_driver(struct of_platform_driver *drv) { - int count = 0; - /* initialize common driver fields */ drv->driver.name = drv->name; drv->driver.bus = &of_platform_bus_type; /* register with core */ - count = driver_register(&drv->driver); - return count ? count : 1; + return driver_register(&drv->driver); } void of_unregister_driver(struct of_platform_driver *drv) diff --git a/arch/powerpc/kernel/pci_iommu.c b/arch/powerpc/kernel/pci_iommu.c index bdf15dbbf4f..c336f3e31cf 100644 --- a/arch/powerpc/kernel/pci_iommu.c +++ b/arch/powerpc/kernel/pci_iommu.c @@ -1,5 +1,4 @@ /* - * arch/ppc64/kernel/pci_iommu.c * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation * * Rewrite, cleanup, new allocation schemes: diff --git a/arch/powerpc/kernel/ppc_ksyms.c b/arch/powerpc/kernel/ppc_ksyms.c index 63ecbec0520..dfa5398ab3c 100644 --- a/arch/powerpc/kernel/ppc_ksyms.c +++ b/arch/powerpc/kernel/ppc_ksyms.c @@ -57,7 +57,6 @@ extern void machine_check_exception(struct pt_regs *regs); extern void alignment_exception(struct pt_regs *regs); extern void program_check_exception(struct pt_regs *regs); extern void single_step_exception(struct pt_regs *regs); -extern int pmac_newworld; extern int sys_sigreturn(struct pt_regs *regs); EXPORT_SYMBOL(clear_pages); diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index c225cf154bf..1770a066c21 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1,6 +1,4 @@ /* - * arch/ppc/kernel/process.c - * * Derived from "arch/i386/kernel/process.c" * Copyright (C) 1995 Linus Torvalds * @@ -47,9 +45,9 @@ #include <asm/mmu.h> #include <asm/prom.h> #include <asm/machdep.h> +#include <asm/time.h> #ifdef CONFIG_PPC64 #include <asm/firmware.h> -#include <asm/time.h> #endif extern unsigned long _get_SP(void); @@ -330,6 +328,11 @@ struct task_struct *__switch_to(struct task_struct *prev, #endif local_irq_save(flags); + + account_system_vtime(current); + account_process_vtime(current); + calculate_steal_time(); + last = _switch(old_thread, new_thread); local_irq_restore(flags); diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 6dbd2172677..d63cd562d9d 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -829,10 +829,6 @@ void __init unflatten_device_tree(void) /* Allocate memory for the expanded device tree */ mem = lmb_alloc(size + 4, __alignof__(struct device_node)); - if (!mem) { - DBG("Couldn't allocate memory with lmb_alloc()!\n"); - panic("Couldn't allocate memory with lmb_alloc()!\n"); - } mem = (unsigned long) __va(mem); ((u32 *)mem)[size / 4] = 0xdeadbeef; diff --git a/arch/powerpc/kernel/ptrace-common.h b/arch/powerpc/kernel/ptrace-common.h index 5ccbdbe0d5c..c42a860c8d2 100644 --- a/arch/powerpc/kernel/ptrace-common.h +++ b/arch/powerpc/kernel/ptrace-common.h @@ -1,6 +1,4 @@ /* - * linux/arch/ppc64/kernel/ptrace-common.h - * * Copyright (c) 2002 Stephen Rothwell, IBM Coproration * Extracted from ptrace.c and ptrace32.c * diff --git a/arch/powerpc/kernel/rtas-proc.c b/arch/powerpc/kernel/rtas-proc.c index 7a95b8a2835..1f03fb28cc0 100644 --- a/arch/powerpc/kernel/rtas-proc.c +++ b/arch/powerpc/kernel/rtas-proc.c @@ -1,5 +1,4 @@ /* - * arch/ppc64/kernel/rtas-proc.c * Copyright (C) 2000 Tilmann Bitterberg * (tilmann@bitterberg.de) * diff --git a/arch/powerpc/kernel/rtas_pci.c b/arch/powerpc/kernel/rtas_pci.c index 7442775ef2a..57b539a03fa 100644 --- a/arch/powerpc/kernel/rtas_pci.c +++ b/arch/powerpc/kernel/rtas_pci.c @@ -1,6 +1,4 @@ /* - * arch/ppc64/kernel/rtas_pci.c - * * Copyright (C) 2001 Dave Engebretsen, IBM Corporation * Copyright (C) 2003 Anton Blanchard <anton@au.ibm.com>, IBM * diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index f96c49b03ba..2f3fdad3559 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -497,8 +497,6 @@ void __init setup_system(void) #endif printk("-----------------------------------------------------\n"); - mm_init_ppc64(); - DBG(" <- setup_system()\n"); } diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c index 4324f8a8ba2..47f910380a6 100644 --- a/arch/powerpc/kernel/signal_64.c +++ b/arch/powerpc/kernel/signal_64.c @@ -1,6 +1,4 @@ /* - * linux/arch/ppc64/kernel/signal.c - * * PowerPC version * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) * diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 13595a64f01..805eaedbc30 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -541,7 +541,7 @@ int __devinit start_secondary(void *unused) smp_ops->take_timebase(); if (system_state > SYSTEM_BOOTING) - per_cpu(last_jiffy, cpu) = get_tb(); + snapshot_timebase(); spin_lock(&call_lock); cpu_set(cpu, cpu_online_map); @@ -573,6 +573,8 @@ void __init smp_cpus_done(unsigned int max_cpus) set_cpus_allowed(current, old_mask); + snapshot_timebases(); + dump_numa_cpu_topology(); } diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 86f7e3d154d..4a27218a086 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -51,6 +51,7 @@ #include <linux/percpu.h> #include <linux/rtc.h> #include <linux/jiffies.h> +#include <linux/posix-timers.h> #include <asm/io.h> #include <asm/processor.h> @@ -98,6 +99,7 @@ unsigned long tb_ticks_per_jiffy; unsigned long tb_ticks_per_usec = 100; /* sane default */ EXPORT_SYMBOL(tb_ticks_per_usec); unsigned long tb_ticks_per_sec; +EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime_t conversions */ u64 tb_to_xs; unsigned tb_to_us; @@ -135,6 +137,224 @@ unsigned long tb_last_stamp; */ DEFINE_PER_CPU(unsigned long, last_jiffy); +#ifdef CONFIG_VIRT_CPU_ACCOUNTING +/* + * Factors for converting from cputime_t (timebase ticks) to + * jiffies, milliseconds, seconds, and clock_t (1/USER_HZ seconds). + * These are all stored as 0.64 fixed-point binary fractions. + */ +u64 __cputime_jiffies_factor; +EXPORT_SYMBOL(__cputime_jiffies_factor); +u64 __cputime_msec_factor; +EXPORT_SYMBOL(__cputime_msec_factor); +u64 __cputime_sec_factor; +EXPORT_SYMBOL(__cputime_sec_factor); +u64 __cputime_clockt_factor; +EXPORT_SYMBOL(__cputime_clockt_factor); + +static void calc_cputime_factors(void) +{ + struct div_result res; + + div128_by_32(HZ, 0, tb_ticks_per_sec, &res); + __cputime_jiffies_factor = res.result_low; + div128_by_32(1000, 0, tb_ticks_per_sec, &res); + __cputime_msec_factor = res.result_low; + div128_by_32(1, 0, tb_ticks_per_sec, &res); + __cputime_sec_factor = res.result_low; + div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res); + __cputime_clockt_factor = res.result_low; +} + +/* + * Read the PURR on systems that have it, otherwise the timebase. + */ +static u64 read_purr(void) +{ + if (cpu_has_feature(CPU_FTR_PURR)) + return mfspr(SPRN_PURR); + return mftb(); +} + +/* + * Account time for a transition between system, hard irq + * or soft irq state. + */ +void account_system_vtime(struct task_struct *tsk) +{ + u64 now, delta; + unsigned long flags; + + local_irq_save(flags); + now = read_purr(); + delta = now - get_paca()->startpurr; + get_paca()->startpurr = now; + if (!in_interrupt()) { + delta += get_paca()->system_time; + get_paca()->system_time = 0; + } + account_system_time(tsk, 0, delta); + local_irq_restore(flags); +} + +/* + * Transfer the user and system times accumulated in the paca + * by the exception entry and exit code to the generic process + * user and system time records. + * Must be called with interrupts disabled. + */ +void account_process_vtime(struct task_struct *tsk) +{ + cputime_t utime; + + utime = get_paca()->user_time; + get_paca()->user_time = 0; + account_user_time(tsk, utime); +} + +static void account_process_time(struct pt_regs *regs) +{ + int cpu = smp_processor_id(); + + account_process_vtime(current); + run_local_timers(); + if (rcu_pending(cpu)) + rcu_check_callbacks(cpu, user_mode(regs)); + scheduler_tick(); + run_posix_cpu_timers(current); +} + +#ifdef CONFIG_PPC_SPLPAR +/* + * Stuff for accounting stolen time. + */ +struct cpu_purr_data { + int initialized; /* thread is running */ + u64 tb0; /* timebase at origin time */ + u64 purr0; /* PURR at origin time */ + u64 tb; /* last TB value read */ + u64 purr; /* last PURR value read */ + u64 stolen; /* stolen time so far */ + spinlock_t lock; +}; + +static DEFINE_PER_CPU(struct cpu_purr_data, cpu_purr_data); + +static void snapshot_tb_and_purr(void *data) +{ + struct cpu_purr_data *p = &__get_cpu_var(cpu_purr_data); + + p->tb0 = mftb(); + p->purr0 = mfspr(SPRN_PURR); + p->tb = p->tb0; + p->purr = 0; + wmb(); + p->initialized = 1; +} + +/* + * Called during boot when all cpus have come up. + */ +void snapshot_timebases(void) +{ + int cpu; + + if (!cpu_has_feature(CPU_FTR_PURR)) + return; + for_each_cpu(cpu) + spin_lock_init(&per_cpu(cpu_purr_data, cpu).lock); + on_each_cpu(snapshot_tb_and_purr, NULL, 0, 1); +} + +void calculate_steal_time(void) +{ + u64 tb, purr, t0; + s64 stolen; + struct cpu_purr_data *p0, *pme, *phim; + int cpu; + + if (!cpu_has_feature(CPU_FTR_PURR)) + return; + cpu = smp_processor_id(); + pme = &per_cpu(cpu_purr_data, cpu); + if (!pme->initialized) + return; /* this can happen in early boot */ + p0 = &per_cpu(cpu_purr_data, cpu & ~1); + phim = &per_cpu(cpu_purr_data, cpu ^ 1); + spin_lock(&p0->lock); + tb = mftb(); + purr = mfspr(SPRN_PURR) - pme->purr0; + if (!phim->initialized || !cpu_online(cpu ^ 1)) { + stolen = (tb - pme->tb) - (purr - pme->purr); + } else { + t0 = pme->tb0; + if (phim->tb0 < t0) + t0 = phim->tb0; + stolen = phim->tb - t0 - phim->purr - purr - p0->stolen; + } + if (stolen > 0) { + account_steal_time(current, stolen); + p0->stolen += stolen; + } + pme->tb = tb; + pme->purr = purr; + spin_unlock(&p0->lock); +} + +/* + * Must be called before the cpu is added to the online map when + * a cpu is being brought up at runtime. + */ +static void snapshot_purr(void) +{ + int cpu; + u64 purr; + struct cpu_purr_data *p0, *pme, *phim; + unsigned long flags; + + if (!cpu_has_feature(CPU_FTR_PURR)) + return; + cpu = smp_processor_id(); + pme = &per_cpu(cpu_purr_data, cpu); + p0 = &per_cpu(cpu_purr_data, cpu & ~1); + phim = &per_cpu(cpu_purr_data, cpu ^ 1); + spin_lock_irqsave(&p0->lock, flags); + pme->tb = pme->tb0 = mftb(); + purr = mfspr(SPRN_PURR); + if (!phim->initialized) { + pme->purr = 0; + pme->purr0 = purr; + } else { + /* set p->purr and p->purr0 for no change in p0->stolen */ + pme->purr = phim->tb - phim->tb0 - phim->purr - p0->stolen; + pme->purr0 = purr - pme->purr; + } + pme->initialized = 1; + spin_unlock_irqrestore(&p0->lock, flags); +} + +#endif /* CONFIG_PPC_SPLPAR */ + +#else /* ! CONFIG_VIRT_CPU_ACCOUNTING */ +#define calc_cputime_factors() +#define account_process_time(regs) update_process_times(user_mode(regs)) +#define calculate_steal_time() do { } while (0) +#endif + +#if !(defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR)) +#define snapshot_purr() do { } while (0) +#endif + +/* + * Called when a cpu comes up after the system has finished booting, + * i.e. as a result of a hotplug cpu action. + */ +void snapshot_timebase(void) +{ + __get_cpu_var(last_jiffy) = get_tb(); + snapshot_purr(); +} + void __delay(unsigned long loops) { unsigned long start; @@ -392,6 +612,7 @@ static void iSeries_tb_recal(void) new_tb_ticks_per_jiffy, sign, tick_diff ); tb_ticks_per_jiffy = new_tb_ticks_per_jiffy; tb_ticks_per_sec = new_tb_ticks_per_sec; + calc_cputime_factors(); div128_by_32( XSEC_PER_SEC, 0, tb_ticks_per_sec, &divres ); do_gtod.tb_ticks_per_sec = tb_ticks_per_sec; tb_to_xs = divres.result_low; @@ -440,6 +661,7 @@ void timer_interrupt(struct pt_regs * regs) irq_enter(); profile_tick(CPU_PROFILING, regs); + calculate_steal_time(); #ifdef CONFIG_PPC_ISERIES get_lppaca()->int_dword.fields.decr_int = 0; @@ -461,7 +683,7 @@ void timer_interrupt(struct pt_regs * regs) * is the case. */ if (!cpu_is_offline(cpu)) - update_process_times(user_mode(regs)); + account_process_time(regs); /* * No need to check whether cpu is offline here; boot_cpuid @@ -518,13 +740,27 @@ void wakeup_decrementer(void) void __init smp_space_timers(unsigned int max_cpus) { int i; + unsigned long half = tb_ticks_per_jiffy / 2; unsigned long offset = tb_ticks_per_jiffy / max_cpus; unsigned long previous_tb = per_cpu(last_jiffy, boot_cpuid); /* make sure tb > per_cpu(last_jiffy, cpu) for all cpus always */ previous_tb -= tb_ticks_per_jiffy; + /* + * The stolen time calculation for POWER5 shared-processor LPAR + * systems works better if the two threads' timebase interrupts + * are staggered by half a jiffy with respect to each other. + */ for_each_cpu(i) { - if (i != boot_cpuid) { + if (i == boot_cpuid) + continue; + if (i == (boot_cpuid ^ 1)) + per_cpu(last_jiffy, i) = + per_cpu(last_jiffy, boot_cpuid) - half; + else if (i & 1) + per_cpu(last_jiffy, i) = + per_cpu(last_jiffy, i ^ 1) + half; + else { previous_tb += offset; per_cpu(last_jiffy, i) = previous_tb; } @@ -720,6 +956,7 @@ void __init time_init(void) tb_ticks_per_sec = ppc_tb_freq; tb_ticks_per_usec = ppc_tb_freq / 1000000; tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000); + calc_cputime_factors(); /* * Calculate the length of each tick in ns. It will not be diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index 04f7df39ffb..ec837036842 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -1,6 +1,4 @@ /* - * linux/arch/ppc64/kernel/vdso.c - * * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp. * <benh@kernel.crashing.org> * |