[S390] improve irq tracing code in entry[64].S

The system call path in entry[64].S is run with interrupts enabled.
Remove the irq tracing check from the system call exit code. If a
program check interrupted a context enabled for interrupts do a
call to trace_irq_off_caller in the program check handler before
branching to the system call exit code.
Restructure the system call and io interrupt return code to avoid
avoid the lpsw[e] to disable machine checks.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

1 files changed, 92 insertions, 105 deletions

diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index ca02b10a2c3..860cea1d1c8 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -61,28 +61,33 @@ _TIF_SYSCALL = (_TIF_SYSCALL_TRACE>>8 | _TIF_SYSCALL_AUDIT>>8 | \
 
 #ifdef CONFIG_TRACE_IRQFLAGS
 	.macro	TRACE_IRQS_ON
-	 basr	%r2,%r0
-	 brasl	%r14,trace_hardirqs_on_caller
+	basr	%r2,%r0
+	brasl	%r14,trace_hardirqs_on_caller
 	.endm
 
 	.macro	TRACE_IRQS_OFF
-	 basr	%r2,%r0
-	 brasl	%r14,trace_hardirqs_off_caller
+	basr	%r2,%r0
+	brasl	%r14,trace_hardirqs_off_caller
 	.endm
 
-	.macro TRACE_IRQS_CHECK
-	basr	%r2,%r0
+	.macro TRACE_IRQS_CHECK_ON
 	tm	SP_PSW(%r15),0x03	# irqs enabled?
 	jz	0f
-	brasl	%r14,trace_hardirqs_on_caller
-	j	1f
-0:	brasl	%r14,trace_hardirqs_off_caller
-1:
+	TRACE_IRQS_ON
+0:
+	.endm
+
+	.macro TRACE_IRQS_CHECK_OFF
+	tm	SP_PSW(%r15),0x03	# irqs enabled?
+	jz	0f
+	TRACE_IRQS_OFF
+0:
 	.endm
 #else
 #define TRACE_IRQS_ON
 #define TRACE_IRQS_OFF
-#define TRACE_IRQS_CHECK
+#define TRACE_IRQS_CHECK_ON
+#define TRACE_IRQS_CHECK_OFF
 #endif
 
 #ifdef CONFIG_LOCKDEP
@@ -267,29 +272,14 @@ sysc_noemu:
 	stg	%r2,SP_R2(%r15) # store return value (change R2 on stack)
 
 sysc_return:
+	LOCKDEP_SYS_EXIT
+sysc_tif:
 	tm	__TI_flags+7(%r9),_TIF_WORK_SVC
 	jnz	sysc_work	# there is work to do (signals etc.)
 sysc_restore:
-#ifdef CONFIG_TRACE_IRQFLAGS
-	larl	%r1,sysc_restore_trace_psw
-	lpswe	0(%r1)
-sysc_restore_trace:
-	TRACE_IRQS_CHECK
-	LOCKDEP_SYS_EXIT
-#endif
-sysc_leave:
 	RESTORE_ALL __LC_RETURN_PSW,1
 sysc_done:
 
-#ifdef CONFIG_TRACE_IRQFLAGS
-	.section .data,"aw",@progbits
-	.align	8
-	.globl sysc_restore_trace_psw
-sysc_restore_trace_psw:
-	.quad	0, sysc_restore_trace
-	.previous
-#endif
-
 #
 # There is work to do, but first we need to check if we return to userspace.
 #
@@ -300,7 +290,7 @@ sysc_work:
 #
 # One of the work bits is on. Find out which one.
 #
-sysc_work_loop:
+sysc_work_tif:
 	tm	__TI_flags+7(%r9),_TIF_MCCK_PENDING
 	jo	sysc_mcck_pending
 	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
@@ -319,14 +309,14 @@ sysc_work_loop:
 # _TIF_NEED_RESCHED is set, call schedule
 #
 sysc_reschedule:
-	larl	%r14,sysc_work_loop
-	jg	schedule		# return point is sysc_work_loop
+	larl	%r14,sysc_return
+	jg	schedule		# return point is sysc_return
 
 #
 # _TIF_MCCK_PENDING is set, call handler
 #
 sysc_mcck_pending:
-	larl	%r14,sysc_work_loop
+	larl	%r14,sysc_return
 	jg	s390_handle_mcck	# TIF bit will be cleared by handler
 
 #
@@ -340,14 +330,14 @@ sysc_sigpending:
 	jo	sysc_restart
 	tm	__TI_flags+7(%r9),_TIF_SINGLE_STEP
 	jo	sysc_singlestep
-	j	sysc_work_loop
+	j	sysc_return
 
 #
 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
 #
 sysc_notify_resume:
 	la	%r2,SP_PTREGS(%r15)	# load pt_regs
-	larl	%r14,sysc_work_loop
+	larl	%r14,sysc_return
 	jg	do_notify_resume	# call do_notify_resume
 
 #
@@ -367,7 +357,7 @@ sysc_singlestep:
 	ni	__TI_flags+7(%r9),255-_TIF_SINGLE_STEP	# clear TIF_SINGLE_STEP
 	xc	SP_SVCNR(2,%r15),SP_SVCNR(%r15)		# clear svc number
 	la	%r2,SP_PTREGS(%r15)	# address of register-save area
-	larl	%r14,sysc_work_loop	# load adr. of system return
+	larl	%r14,sysc_return	# load adr. of system return
 	jg	do_single_step		# branch to do_sigtrap
 
 #
@@ -433,12 +423,14 @@ kernel_execve:
 	br	%r14
 	# execve succeeded.
 0:	stnsm	__SF_EMPTY(%r15),0xfc	# disable interrupts
+#	TRACE_IRQS_OFF
 	lg	%r15,__LC_KERNEL_STACK	# load ksp
 	aghi	%r15,-SP_SIZE		# make room for registers & psw
 	lg	%r13,__LC_SVC_NEW_PSW+8
 	lg	%r9,__LC_THREAD_INFO
 	mvc	SP_PTREGS(__PT_SIZE,%r15),0(%r12)	# copy pt_regs
 	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
+#	TRACE_IRQS_ON
 	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
 	brasl	%r14,execve_tail
 	j	sysc_return
@@ -474,9 +466,9 @@ pgm_check_handler:
 	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
 	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
 pgm_no_vtime:
+	TRACE_IRQS_CHECK_OFF
 	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
 	mvc	SP_ARGS(8,%r15),__LC_LAST_BREAK
-	TRACE_IRQS_OFF
 	lgf	%r3,__LC_PGM_ILC	# load program interruption code
 	lghi	%r8,0x7f
 	ngr	%r8,%r3
@@ -485,8 +477,10 @@ pgm_do_call:
 	larl	%r1,pgm_check_table
 	lg	%r1,0(%r8,%r1)		# load address of handler routine
 	la	%r2,SP_PTREGS(%r15)	# address of register-save area
-	larl	%r14,sysc_return
-	br	%r1			# branch to interrupt-handler
+	basr	%r14,%r1		# branch to interrupt-handler
+pgm_exit:
+	TRACE_IRQS_CHECK_ON
+	j	sysc_return
 
 #
 # handle per exception
@@ -513,8 +507,8 @@ pgm_per_std:
 	UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
 	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
 pgm_no_vtime2:
+	TRACE_IRQS_CHECK_OFF
 	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
-	TRACE_IRQS_OFF
 	lg	%r1,__TI_task(%r9)
 	tm	SP_PSW+1(%r15),0x01	# kernel per event ?
 	jz	kernel_per
@@ -525,7 +519,7 @@ pgm_no_vtime2:
 	lgf	%r3,__LC_PGM_ILC	# load program interruption code
 	lghi	%r8,0x7f
 	ngr	%r8,%r3			# clear per-event-bit and ilc
-	je	sysc_return
+	je	pgm_exit
 	j	pgm_do_call
 
 #
@@ -539,14 +533,15 @@ pgm_svcper:
 	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
 	llgh	%r7,__LC_SVC_INT_CODE	# get svc number from lowcore
 	lg	%r9,__LC_THREAD_INFO	# load pointer to thread_info struct
+	TRACE_IRQS_OFF
 	lg	%r8,__TI_task(%r9)
 	mvc	__THREAD_per+__PER_atmid(2,%r8),__LC_PER_ATMID
 	mvc	__THREAD_per+__PER_address(8,%r8),__LC_PER_ADDRESS
 	mvc	__THREAD_per+__PER_access_id(1,%r8),__LC_PER_ACCESS_ID
 	oi	__TI_flags+7(%r9),_TIF_SINGLE_STEP # set TIF_SINGLE_STEP
 	TRACE_IRQS_ON
-	lmg	%r2,%r6,SP_R2(%r15)	# load svc arguments
 	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
+	lmg	%r2,%r6,SP_R2(%r15)	# load svc arguments
 	j	sysc_do_svc
 
 #
@@ -555,8 +550,8 @@ pgm_svcper:
 kernel_per:
 	xc	SP_SVCNR(2,%r15),SP_SVCNR(%r15)	# clear svc number
 	la	%r2,SP_PTREGS(%r15)	# address of register-save area
-	larl	%r14,sysc_restore	# load adr. of system ret, no work
-	jg	do_single_step		# branch to do_single_step
+	brasl	%r14,do_single_step
+	j	pgm_exit
 
 /*
  * IO interrupt handler routine
@@ -579,29 +574,15 @@ io_no_vtime:
 	la	%r2,SP_PTREGS(%r15)	# address of register-save area
 	brasl	%r14,do_IRQ		# call standard irq handler
 io_return:
+	LOCKDEP_SYS_EXIT
+	TRACE_IRQS_ON
+io_tif:
 	tm	__TI_flags+7(%r9),_TIF_WORK_INT
 	jnz	io_work 		# there is work to do (signals etc.)
 io_restore:
-#ifdef CONFIG_TRACE_IRQFLAGS
-	larl	%r1,io_restore_trace_psw
-	lpswe	0(%r1)
-io_restore_trace:
-	TRACE_IRQS_CHECK
-	LOCKDEP_SYS_EXIT
-#endif
-io_leave:
 	RESTORE_ALL __LC_RETURN_PSW,0
 io_done:
 
-#ifdef CONFIG_TRACE_IRQFLAGS
-	.section .data,"aw",@progbits
-	.align	8
-	.globl io_restore_trace_psw
-io_restore_trace_psw:
-	.quad	0, io_restore_trace
-	.previous
-#endif
-
 #
 # There is work todo, find out in which context we have been interrupted:
 # 1) if we return to user space we can do all _TIF_WORK_INT work
@@ -627,18 +608,22 @@ io_work:
 	# check for preemptive scheduling
 	icm	%r0,15,__TI_precount(%r9)
 	jnz	io_restore		# preemption is disabled
+	tm	__TI_flags+7(%r12),_TIF_NEED_RESCHED
+	jno	io_restore
 	# switch to kernel stack
 	lg	%r1,SP_R15(%r15)
 	aghi	%r1,-SP_SIZE
 	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
 	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain
 	lgr	%r15,%r1
-io_resume_loop:
-	larl	%r14,io_resume_loop
-	tm	__TI_flags+7(%r12),_TIF_NEED_RESCHED
-	jgo	preempt_schedule_irq
-#endif
+	# TRACE_IRQS_ON already done at io_return, call
+	# TRACE_IRQS_OFF to keep things symmetrical
+	TRACE_IRQS_OFF
+	brasl	%r14,preempt_schedule_irq
+	j	io_return
+#else
 	j	io_restore
+#endif
 
 #
 # Need to do work before returning to userspace, switch to kernel stack
@@ -655,7 +640,7 @@ io_work_user:
 # Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
 #	       and _TIF_MCCK_PENDING
 #
-io_work_loop:
+io_work_tif:
 	tm	__TI_flags+7(%r9),_TIF_MCCK_PENDING
 	jo	io_mcck_pending
 	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
@@ -670,43 +655,45 @@ io_work_loop:
 # _TIF_MCCK_PENDING is set, call handler
 #
 io_mcck_pending:
+	# TRACE_IRQS_ON already done at io_return
 	brasl	%r14,s390_handle_mcck	# TIF bit will be cleared by handler
-	j	io_work_loop
+	TRACE_IRQS_OFF
+	j	io_return
 
 #
 # _TIF_NEED_RESCHED is set, call schedule
 #
 io_reschedule:
-	TRACE_IRQS_ON
+	# TRACE_IRQS_ON already done at io_return
 	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
 	brasl	%r14,schedule		# call scheduler
 	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
 	TRACE_IRQS_OFF
-	j	io_work_loop
+	j	io_return
 
 #
 # _TIF_SIGPENDING or is set, call do_signal
 #
 io_sigpending:
-	TRACE_IRQS_ON
+	# TRACE_IRQS_ON already done at io_return
 	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
 	la	%r2,SP_PTREGS(%r15)	# load pt_regs
 	brasl	%r14,do_signal		# call do_signal
 	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
 	TRACE_IRQS_OFF
-	j	io_work_loop
+	j	io_return
 
 #
 # _TIF_NOTIFY_RESUME or is set, call do_notify_resume
 #
 io_notify_resume:
-	TRACE_IRQS_ON
+	# TRACE_IRQS_ON already done at io_return
 	stosm	__SF_EMPTY(%r15),0x03	# reenable interrupts
 	la	%r2,SP_PTREGS(%r15)	# load pt_regs
 	brasl	%r14,do_notify_resume	# call do_notify_resume
 	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
 	TRACE_IRQS_OFF
-	j	io_work_loop
+	j	io_return
 
 /*
  * External interrupt handler routine
@@ -883,14 +870,14 @@ stack_overflow:
 
 cleanup_table_system_call:
 	.quad	system_call, sysc_do_svc
-cleanup_table_sysc_return:
-	.quad	sysc_return, sysc_leave
-cleanup_table_sysc_leave:
-	.quad	sysc_leave, sysc_done
-cleanup_table_io_return:
-	.quad	io_return, io_leave
-cleanup_table_io_leave:
-	.quad	io_leave, io_done
+cleanup_table_sysc_tif:
+	.quad	sysc_tif, sysc_restore
+cleanup_table_sysc_restore:
+	.quad	sysc_restore, sysc_done
+cleanup_table_io_tif:
+	.quad	io_tif, io_restore
+cleanup_table_io_restore:
+	.quad	io_restore, io_done
 
 cleanup_critical:
 	clc	8(8,%r12),BASED(cleanup_table_system_call)
@@ -898,25 +885,25 @@ cleanup_critical:
 	clc	8(8,%r12),BASED(cleanup_table_system_call+8)
 	jl	cleanup_system_call
 0:
-	clc	8(8,%r12),BASED(cleanup_table_sysc_return)
+	clc	8(8,%r12),BASED(cleanup_table_sysc_tif)
 	jl	0f
-	clc	8(8,%r12),BASED(cleanup_table_sysc_return+8)
-	jl	cleanup_sysc_return
+	clc	8(8,%r12),BASED(cleanup_table_sysc_tif+8)
+	jl	cleanup_sysc_tif
 0:
-	clc	8(8,%r12),BASED(cleanup_table_sysc_leave)
+	clc	8(8,%r12),BASED(cleanup_table_sysc_restore)
 	jl	0f
-	clc	8(8,%r12),BASED(cleanup_table_sysc_leave+8)
-	jl	cleanup_sysc_leave
+	clc	8(8,%r12),BASED(cleanup_table_sysc_restore+8)
+	jl	cleanup_sysc_restore
 0:
-	clc	8(8,%r12),BASED(cleanup_table_io_return)
+	clc	8(8,%r12),BASED(cleanup_table_io_tif)
 	jl	0f
-	clc	8(8,%r12),BASED(cleanup_table_io_return+8)
-	jl	cleanup_io_return
+	clc	8(8,%r12),BASED(cleanup_table_io_tif+8)
+	jl	cleanup_io_tif
 0:
-	clc	8(8,%r12),BASED(cleanup_table_io_leave)
+	clc	8(8,%r12),BASED(cleanup_table_io_restore)
 	jl	0f
-	clc	8(8,%r12),BASED(cleanup_table_io_leave+8)
-	jl	cleanup_io_leave
+	clc	8(8,%r12),BASED(cleanup_table_io_restore+8)
+	jl	cleanup_io_restore
 0:
 	br	%r14
 
@@ -963,16 +950,16 @@ cleanup_system_call_insn:
 	.quad	sysc_stime
 	.quad	sysc_update
 
-cleanup_sysc_return:
+cleanup_sysc_tif:
 	mvc	__LC_RETURN_PSW(8),0(%r12)
-	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_sysc_return)
+	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_sysc_tif)
 	la	%r12,__LC_RETURN_PSW
 	br	%r14
 
-cleanup_sysc_leave:
-	clc	8(8,%r12),BASED(cleanup_sysc_leave_insn)
+cleanup_sysc_restore:
+	clc	8(8,%r12),BASED(cleanup_sysc_restore_insn)
 	je	3f
-	clc	8(8,%r12),BASED(cleanup_sysc_leave_insn+8)
+	clc	8(8,%r12),BASED(cleanup_sysc_restore_insn+8)
 	jhe	0f
 	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
 0:	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)
@@ -985,20 +972,20 @@ cleanup_sysc_leave:
 	lg	%r15,SP_R15(%r15)
 3:	la	%r12,__LC_RETURN_PSW
 	br	%r14
-cleanup_sysc_leave_insn:
+cleanup_sysc_restore_insn:
 	.quad	sysc_done - 4
 	.quad	sysc_done - 16
 
-cleanup_io_return:
+cleanup_io_tif:
 	mvc	__LC_RETURN_PSW(8),0(%r12)
-	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_io_return)
+	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_io_tif)
 	la	%r12,__LC_RETURN_PSW
 	br	%r14
 
-cleanup_io_leave:
-	clc	8(8,%r12),BASED(cleanup_io_leave_insn)
+cleanup_io_restore:
+	clc	8(8,%r12),BASED(cleanup_io_restore_insn)
 	je	3f
-	clc	8(8,%r12),BASED(cleanup_io_leave_insn+8)
+	clc	8(8,%r12),BASED(cleanup_io_restore_insn+8)
 	jhe	0f
 	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
 0:	mvc	__LC_RETURN_PSW(16),SP_PSW(%r15)
@@ -1011,7 +998,7 @@ cleanup_io_leave:
 	lg	%r15,SP_R15(%r15)
 3:	la	%r12,__LC_RETURN_PSW
 	br	%r14
-cleanup_io_leave_insn:
+cleanup_io_restore_insn:
 	.quad	io_done - 4
 	.quad	io_done - 16