diff options
Diffstat (limited to 'arch/s390/kernel/entry64.S')
| -rw-r--r-- | arch/s390/kernel/entry64.S | 106 |
1 files changed, 73 insertions, 33 deletions
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index 05e26d1fdf4..e15c80efdd0 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -67,12 +67,28 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK | _TIF_NEED_RESCHED | \ brasl %r14,trace_hardirqs_off .endm - .macro LOCKDEP_SYS_EXIT - brasl %r14,lockdep_sys_exit + .macro TRACE_IRQS_CHECK + tm SP_PSW(%r15),0x03 # irqs enabled? + jz 0f + brasl %r14,trace_hardirqs_on + j 1f +0: brasl %r14,trace_hardirqs_off +1: .endm #else #define TRACE_IRQS_ON #define TRACE_IRQS_OFF +#define TRACE_IRQS_CHECK +#endif + +#ifdef CONFIG_LOCKDEP + .macro LOCKDEP_SYS_EXIT + tm SP_PSW+1(%r15),0x01 # returning to user ? + jz 0f + brasl %r14,lockdep_sys_exit +0: + .endm +#else #define LOCKDEP_SYS_EXIT #endif @@ -222,8 +238,6 @@ sysc_saveall: llgh %r7,__LC_SVC_INT_CODE # get svc number from lowcore #ifdef CONFIG_VIRT_CPU_ACCOUNTING sysc_vtime: - tm SP_PSW+1(%r15),0x01 # interrupting from user ? - jz sysc_do_svc UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER sysc_stime: UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER @@ -257,19 +271,34 @@ sysc_noemu: sysc_return: tm SP_PSW+1(%r15),0x01 # returning to user ? - jno sysc_leave + jno sysc_restore tm __TI_flags+7(%r9),_TIF_WORK_SVC jnz sysc_work # there is work to do (signals etc.) +sysc_restore: +#ifdef CONFIG_TRACE_IRQFLAGS + larl %r1,sysc_restore_trace_psw + lpswe 0(%r1) +sysc_restore_trace: + TRACE_IRQS_CHECK LOCKDEP_SYS_EXIT +#endif sysc_leave: RESTORE_ALL __LC_RETURN_PSW,1 +sysc_done: + +#ifdef CONFIG_TRACE_IRQFLAGS + .align 8 + .globl sysc_restore_trace_psw +sysc_restore_trace_psw: + .quad 0, sysc_restore_trace +#endif # # recheck if there is more work to do # sysc_work_loop: tm __TI_flags+7(%r9),_TIF_WORK_SVC - jz sysc_leave # there is no work to do + jz sysc_restore # there is no work to do # # One of the work bits is on. Find out which one. # @@ -284,8 +313,8 @@ sysc_work: jo sysc_restart tm __TI_flags+7(%r9),_TIF_SINGLE_STEP jo sysc_singlestep - LOCKDEP_SYS_EXIT - j sysc_leave + j sysc_restore +sysc_work_done: # # _TIF_NEED_RESCHED is set, call schedule @@ -445,6 +474,7 @@ pgm_check_handler: pgm_no_vtime: #endif lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct + TRACE_IRQS_OFF lgf %r3,__LC_PGM_ILC # load program interruption code lghi %r8,0x7f ngr %r8,%r3 @@ -484,6 +514,7 @@ pgm_per_std: pgm_no_vtime2: #endif lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct + TRACE_IRQS_OFF lg %r1,__TI_task(%r9) tm SP_PSW+1(%r15),0x01 # kernel per event ? jz kernel_per @@ -504,12 +535,9 @@ pgm_svcper: SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA #ifdef CONFIG_VIRT_CPU_ACCOUNTING - tm SP_PSW+1(%r15),0x01 # interrupting from user ? - jz pgm_no_vtime3 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER -pgm_no_vtime3: #endif llgh %r7,__LC_SVC_INT_CODE # get svc number from lowcore lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct @@ -529,7 +557,7 @@ kernel_per: lhi %r0,__LC_PGM_OLD_PSW sth %r0,SP_TRAP(%r15) # set trap indication to pgm check la %r2,SP_PTREGS(%r15) # address of register-save area - larl %r14,sysc_leave # load adr. of system ret, no work + larl %r14,sysc_restore # load adr. of system ret, no work jg do_single_step # branch to do_single_step /* @@ -554,26 +582,38 @@ io_no_vtime: TRACE_IRQS_OFF la %r2,SP_PTREGS(%r15) # address of register-save area brasl %r14,do_IRQ # call standard irq handler - TRACE_IRQS_ON - io_return: tm SP_PSW+1(%r15),0x01 # returning to user ? #ifdef CONFIG_PREEMPT jno io_preempt # no -> check for preemptive scheduling #else - jno io_leave # no-> skip resched & signal + jno io_restore # no-> skip resched & signal #endif tm __TI_flags+7(%r9),_TIF_WORK_INT jnz io_work # there is work to do (signals etc.) +io_restore: +#ifdef CONFIG_TRACE_IRQFLAGS + larl %r1,io_restore_trace_psw + lpswe 0(%r1) +io_restore_trace: + TRACE_IRQS_CHECK LOCKDEP_SYS_EXIT +#endif io_leave: RESTORE_ALL __LC_RETURN_PSW,0 io_done: +#ifdef CONFIG_TRACE_IRQFLAGS + .align 8 + .globl io_restore_trace_psw +io_restore_trace_psw: + .quad 0, io_restore_trace +#endif + #ifdef CONFIG_PREEMPT io_preempt: icm %r0,15,__TI_precount(%r9) - jnz io_leave + jnz io_restore # switch to kernel stack lg %r1,SP_R15(%r15) aghi %r1,-SP_SIZE @@ -582,12 +622,14 @@ io_preempt: lgr %r15,%r1 io_resume_loop: tm __TI_flags+7(%r9),_TIF_NEED_RESCHED - jno io_leave + jno io_restore larl %r1,.Lc_pactive mvc __TI_precount(4,%r9),0(%r1) + TRACE_IRQS_ON stosm __SF_EMPTY(%r15),0x03 # reenable interrupts brasl %r14,schedule # call schedule stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts + TRACE_IRQS_OFF xc __TI_precount(4,%r9),__TI_precount(%r9) j io_resume_loop #endif @@ -613,37 +655,39 @@ io_work_loop: jo io_reschedule tm __TI_flags+7(%r9),(_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK) jnz io_sigpending - LOCKDEP_SYS_EXIT - j io_leave + j io_restore +io_work_done: # # _TIF_MCCK_PENDING is set, call handler # io_mcck_pending: - TRACE_IRQS_OFF brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler - TRACE_IRQS_ON j io_work_loop # # _TIF_NEED_RESCHED is set, call schedule # io_reschedule: + TRACE_IRQS_ON stosm __SF_EMPTY(%r15),0x03 # reenable interrupts brasl %r14,schedule # call scheduler stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts + TRACE_IRQS_OFF tm __TI_flags+7(%r9),_TIF_WORK_INT - jz io_leave # there is no work to do + jz io_restore # there is no work to do j io_work_loop # # _TIF_SIGPENDING or _TIF_RESTORE_SIGMASK is set, call do_signal # io_sigpending: + TRACE_IRQS_ON stosm __SF_EMPTY(%r15),0x03 # reenable interrupts la %r2,SP_PTREGS(%r15) # load pt_regs brasl %r14,do_signal # call do_signal stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts + TRACE_IRQS_OFF j io_work_loop /* @@ -669,7 +713,6 @@ ext_no_vtime: la %r2,SP_PTREGS(%r15) # address of register-save area llgh %r3,__LC_EXT_INT_CODE # get interruption code brasl %r14,do_extint - TRACE_IRQS_ON j io_return __critical_end: @@ -824,15 +867,15 @@ cleanup_table_system_call: cleanup_table_sysc_return: .quad sysc_return, sysc_leave cleanup_table_sysc_leave: - .quad sysc_leave, sysc_work_loop + .quad sysc_leave, sysc_done cleanup_table_sysc_work_loop: - .quad sysc_work_loop, sysc_reschedule + .quad sysc_work_loop, sysc_work_done cleanup_table_io_return: .quad io_return, io_leave cleanup_table_io_leave: .quad io_leave, io_done cleanup_table_io_work_loop: - .quad io_work_loop, io_mcck_pending + .quad io_work_loop, io_work_done cleanup_critical: clc 8(8,%r12),BASED(cleanup_table_system_call) @@ -901,8 +944,6 @@ cleanup_system_call: cleanup_vtime: clc __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+24) jhe cleanup_stime - tm SP_PSW+1(%r15),0x01 # interrupting from user ? - jz cleanup_novtime UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER cleanup_stime: clc __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+32) @@ -910,7 +951,6 @@ cleanup_stime: UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER cleanup_update: mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER -cleanup_novtime: #endif mvc __LC_RETURN_PSW+8(8),BASED(cleanup_table_system_call+8) la %r12,__LC_RETURN_PSW @@ -949,10 +989,10 @@ cleanup_sysc_leave: 2: la %r12,__LC_RETURN_PSW br %r14 cleanup_sysc_leave_insn: + .quad sysc_done - 4 #ifdef CONFIG_VIRT_CPU_ACCOUNTING - .quad sysc_leave + 16 + .quad sysc_done - 8 #endif - .quad sysc_leave + 12 cleanup_io_return: mvc __LC_RETURN_PSW(8),0(%r12) @@ -979,10 +1019,10 @@ cleanup_io_leave: 2: la %r12,__LC_RETURN_PSW br %r14 cleanup_io_leave_insn: + .quad io_done - 4 #ifdef CONFIG_VIRT_CPU_ACCOUNTING - .quad io_leave + 20 + .quad io_done - 8 #endif - .quad io_leave + 16 /* * Integer constants |