[S390] cleanup sysc_work and io_work code

Cleanup the #ifdef mess at io_work in entry[64].S and streamline the
TIF work code of the system call and io exit path.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

2 files changed, 64 insertions, 119 deletions

diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 6af7045280a..ffebfb64b91 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -301,31 +301,29 @@ sysc_restore_trace_psw:
 #endif
 
 #
-# recheck if there is more work to do
-#
-sysc_work_loop:
-	tm	__TI_flags+3(%r9),_TIF_WORK_SVC
-	bz	BASED(sysc_restore)	# there is no work to do
-#
-# One of the work bits is on. Find out which one.
+# There is work to do, but first we need to check if we return to userspace.
 #
 sysc_work:
 	tm	SP_PSW+1(%r15),0x01	# returning to user ?
 	bno	BASED(sysc_restore)
+
+#
+# One of the work bits is on. Find out which one.
+#
+sysc_work_loop:
 	tm	__TI_flags+3(%r9),_TIF_MCCK_PENDING
 	bo	BASED(sysc_mcck_pending)
 	tm	__TI_flags+3(%r9),_TIF_NEED_RESCHED
 	bo	BASED(sysc_reschedule)
 	tm	__TI_flags+3(%r9),_TIF_SIGPENDING
-	bnz	BASED(sysc_sigpending)
+	bo	BASED(sysc_sigpending)
 	tm	__TI_flags+3(%r9),_TIF_NOTIFY_RESUME
-	bnz	BASED(sysc_notify_resume)
+	bo	BASED(sysc_notify_resume)
 	tm	__TI_flags+3(%r9),_TIF_RESTART_SVC
 	bo	BASED(sysc_restart)
 	tm	__TI_flags+3(%r9),_TIF_SINGLE_STEP
 	bo	BASED(sysc_singlestep)
-	b	BASED(sysc_restore)
-sysc_work_done:
+	b	BASED(sysc_return)	# beware of critical section cleanup
 
 #
 # _TIF_NEED_RESCHED is set, call schedule
@@ -386,7 +384,7 @@ sysc_singlestep:
 	mvi	SP_SVCNR+1(%r15),0xff
 	la	%r2,SP_PTREGS(%r15)	# address of register-save area
 	l	%r1,BASED(.Lhandle_per)	# load adr. of per handler
-	la	%r14,BASED(sysc_return)	# load adr. of system return
+	la	%r14,BASED(sysc_work_loop)	# load adr. of system return
 	br	%r1			# branch to do_single_step
 
 #
@@ -636,30 +634,36 @@ io_restore_trace_psw:
 #endif
 
 #
-# switch to kernel stack, then check the TIF bits
+# There is work todo, find out in which context we have been interrupted:
+# 1) if we return to user space we can do all _TIF_WORK_INT work
+# 2) if we return to kernel code and preemptive scheduling is enabled check
+#    the preemption counter and if it is zero call preempt_schedule_irq
+# Before any work can be done, a switch to the kernel stack is required.
 #
 io_work:
 	tm	SP_PSW+1(%r15),0x01	# returning to user ?
-#ifndef CONFIG_PREEMPT
-	bno	BASED(io_restore)	# no-> skip resched & signal
-#else
-	bnz	BASED(io_work_user)	# no -> check for preemptive scheduling
+	bo	BASED(io_work_user)	# yes -> do resched & signal
+#ifdef CONFIG_PREEMPT
 	# check for preemptive scheduling
 	icm	%r0,15,__TI_precount(%r9)
 	bnz	BASED(io_restore)	# preemption disabled
+	# switch to kernel stack
 	l	%r1,SP_R15(%r15)
 	s	%r1,BASED(.Lc_spsize)
 	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
 	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
 	lr	%r15,%r1
 io_resume_loop:
-	tm	__TI_flags+3(%r9),_TIF_NEED_RESCHED
-	bno	BASED(io_restore)
 	l	%r1,BASED(.Lpreempt_schedule_irq)
 	la	%r14,BASED(io_resume_loop)
-	br	%r1			# call schedule
+	tm	__TI_flags+3(%r9),_TIF_NEED_RESCHED
+	bor	%r1			# call preempt_schedule_irq
 #endif
+	b	BASED(io_restore)
 
+#
+# Need to do work before returning to userspace, switch to kernel stack
+#
 io_work_user:
 	l	%r1,__LC_KERNEL_STACK
 	s	%r1,BASED(.Lc_spsize)
@@ -668,7 +672,7 @@ io_work_user:
 	lr	%r15,%r1
 #
 # One of the work bits is on. Find out which one.
-# Checked are: _TIF_SIGPENDING, _TIF_NEED_RESCHED
+# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
 #		and _TIF_MCCK_PENDING
 #
 io_work_loop:
@@ -677,11 +681,10 @@ io_work_loop:
 	tm	__TI_flags+3(%r9),_TIF_NEED_RESCHED
 	bo	BASED(io_reschedule)
 	tm	__TI_flags+3(%r9),_TIF_SIGPENDING
-	bnz	BASED(io_sigpending)
+	bo	BASED(io_sigpending)
 	tm	__TI_flags+3(%r9),_TIF_NOTIFY_RESUME
-	bnz	BASED(io_notify_resume)
-	b	BASED(io_restore)
-io_work_done:
+	bo	BASED(io_notify_resume)
+	b	BASED(io_return)	# beware of critical section cleanup
 
 #
 # _TIF_MCCK_PENDING is set, call handler
@@ -701,8 +704,6 @@ io_reschedule:
 	basr	%r14,%r1		# call scheduler
 	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
 	TRACE_IRQS_OFF
-	tm	__TI_flags+3(%r9),_TIF_WORK_INT
-	bz	BASED(io_restore)	# there is no work to do
 	b	BASED(io_work_loop)
 
 #
@@ -921,14 +922,10 @@ cleanup_table_sysc_return:
 	.long	sysc_return + 0x80000000, sysc_leave + 0x80000000
 cleanup_table_sysc_leave:
 	.long	sysc_leave + 0x80000000, sysc_done + 0x80000000
-cleanup_table_sysc_work_loop:
-	.long	sysc_work_loop + 0x80000000, sysc_work_done + 0x80000000
 cleanup_table_io_return:
 	.long	io_return + 0x80000000, io_leave + 0x80000000
 cleanup_table_io_leave:
 	.long	io_leave + 0x80000000, io_done + 0x80000000
-cleanup_table_io_work_loop:
-	.long	io_work_loop + 0x80000000, io_work_done + 0x80000000
 
 cleanup_critical:
 	clc	4(4,%r12),BASED(cleanup_table_system_call)
@@ -946,11 +943,6 @@ cleanup_critical:
 	clc	4(4,%r12),BASED(cleanup_table_sysc_leave+4)
 	bl	BASED(cleanup_sysc_leave)
 0:
-	clc	4(4,%r12),BASED(cleanup_table_sysc_work_loop)
-	bl	BASED(0f)
-	clc	4(4,%r12),BASED(cleanup_table_sysc_work_loop+4)
-	bl	BASED(cleanup_sysc_return)
-0:
 	clc	4(4,%r12),BASED(cleanup_table_io_return)
 	bl	BASED(0f)
 	clc	4(4,%r12),BASED(cleanup_table_io_return+4)
@@ -961,11 +953,6 @@ cleanup_critical:
 	clc	4(4,%r12),BASED(cleanup_table_io_leave+4)
 	bl	BASED(cleanup_io_leave)
 0:
-	clc	4(4,%r12),BASED(cleanup_table_io_work_loop)
-	bl	BASED(0f)
-	clc	4(4,%r12),BASED(cleanup_table_io_work_loop+4)
-	bl	BASED(cleanup_io_work_loop)
-0:
 	br	%r14
 
 cleanup_system_call:
@@ -1043,12 +1030,6 @@ cleanup_io_return:
 	la	%r12,__LC_RETURN_PSW
 	br	%r14
 
-cleanup_io_work_loop:
-	mvc	__LC_RETURN_PSW(4),0(%r12)
-	mvc	__LC_RETURN_PSW+4(4),BASED(cleanup_table_io_work_loop)
-	la	%r12,__LC_RETURN_PSW
-	br	%r14
-
 cleanup_io_leave:
 	clc	4(4,%r12),BASED(cleanup_io_leave_insn)
 	be	BASED(2f)
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 52106d53271..ca02b10a2c3 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -291,38 +291,36 @@ sysc_restore_trace_psw:
 #endif
 
 #
-# recheck if there is more work to do
-#
-sysc_work_loop:
-	tm	__TI_flags+7(%r9),_TIF_WORK_SVC
-	jz	sysc_restore	  # there is no work to do
-#
-# One of the work bits is on. Find out which one.
+# There is work to do, but first we need to check if we return to userspace.
 #
 sysc_work:
 	tm	SP_PSW+1(%r15),0x01	# returning to user ?
 	jno	sysc_restore
+
+#
+# One of the work bits is on. Find out which one.
+#
+sysc_work_loop:
 	tm	__TI_flags+7(%r9),_TIF_MCCK_PENDING
 	jo	sysc_mcck_pending
 	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
 	jo	sysc_reschedule
 	tm	__TI_flags+7(%r9),_TIF_SIGPENDING
-	jnz	sysc_sigpending
+	jo	sysc_sigpending
 	tm	__TI_flags+7(%r9),_TIF_NOTIFY_RESUME
-	jnz	sysc_notify_resume
+	jo	sysc_notify_resume
 	tm	__TI_flags+7(%r9),_TIF_RESTART_SVC
 	jo	sysc_restart
 	tm	__TI_flags+7(%r9),_TIF_SINGLE_STEP
 	jo	sysc_singlestep
-	j	sysc_restore
-sysc_work_done:
+	j	sysc_return		# beware of critical section cleanup
 
 #
 # _TIF_NEED_RESCHED is set, call schedule
 #
 sysc_reschedule:
 	larl	%r14,sysc_work_loop
-	jg	schedule	# return point is sysc_return
+	jg	schedule		# return point is sysc_work_loop
 
 #
 # _TIF_MCCK_PENDING is set, call handler
@@ -369,7 +367,7 @@ sysc_singlestep:
 	ni	__TI_flags+7(%r9),255-_TIF_SINGLE_STEP	# clear TIF_SINGLE_STEP
 	xc	SP_SVCNR(2,%r15),SP_SVCNR(%r15)		# clear svc number
 	la	%r2,SP_PTREGS(%r15)	# address of register-save area
-	larl	%r14,sysc_return	# load adr. of system return
+	larl	%r14,sysc_work_loop	# load adr. of system return
 	jg	do_single_step		# branch to do_sigtrap
 
 #
@@ -605,37 +603,27 @@ io_restore_trace_psw:
 #endif
 
 #
-# There is work todo, we need to check if we return to userspace, then
-# check, if we are in SIE, if yes leave it
+# There is work todo, find out in which context we have been interrupted:
+# 1) if we return to user space we can do all _TIF_WORK_INT work
+# 2) if we return to kernel code and kvm is enabled check if we need to
+#    modify the psw to leave SIE
+# 3) if we return to kernel code and preemptive scheduling is enabled check
+#    the preemption counter and if it is zero call preempt_schedule_irq
+# Before any work can be done, a switch to the kernel stack is required.
 #
 io_work:
 	tm	SP_PSW+1(%r15),0x01	# returning to user ?
-#ifndef CONFIG_PREEMPT
-#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
-	jnz	io_work_user		# yes -> no need to check for SIE
-	la	%r1, BASED(sie_opcode)	# we return to kernel here
-	lg	%r2, SP_PSW+8(%r15)
-	clc	0(2,%r1), 0(%r2)	# is current instruction = SIE?
-	jne	io_restore		# no-> return to kernel
-	lg	%r1, SP_PSW+8(%r15)	# yes-> add 4 bytes to leave SIE
-	aghi	%r1, 4
-	stg	%r1, SP_PSW+8(%r15)
-	j	io_restore		# return to kernel
-#else
-	jno	io_restore		# no-> skip resched & signal
-#endif
-#else
-	jnz	io_work_user		# yes -> do resched & signal
+	jo	io_work_user		# yes -> do resched & signal
 #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
-	la	%r1, BASED(sie_opcode)
-	lg	%r2, SP_PSW+8(%r15)
-	clc	0(2,%r1), 0(%r2)	# is current instruction = SIE?
+	lg	%r2,SP_PSW+8(%r15)	# check if current instruction is SIE
+	lh	%r1,0(%r2)
+	chi	%r1,-19948		# signed 16 bit compare with 0xb214
 	jne	0f			# no -> leave PSW alone
-	lg	%r1, SP_PSW+8(%r15)	# yes-> add 4 bytes to leave SIE
-	aghi	%r1, 4
-	stg	%r1, SP_PSW+8(%r15)
+	aghi	%r2,4			# yes-> add 4 bytes to leave SIE
+	stg	%r2,SP_PSW+8(%r15)
 0:
 #endif
+#ifdef CONFIG_PREEMPT
 	# check for preemptive scheduling
 	icm	%r0,15,__TI_precount(%r9)
 	jnz	io_restore		# preemption is disabled
@@ -646,21 +634,25 @@ io_work:
 	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain
 	lgr	%r15,%r1
 io_resume_loop:
-	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
-	jno	io_restore
 	larl	%r14,io_resume_loop
-	jg	preempt_schedule_irq
+	tm	__TI_flags+7(%r12),_TIF_NEED_RESCHED
+	jgo	preempt_schedule_irq
 #endif
+	j	io_restore
 
+#
+# Need to do work before returning to userspace, switch to kernel stack
+#
 io_work_user:
 	lg	%r1,__LC_KERNEL_STACK
 	aghi	%r1,-SP_SIZE
 	mvc	SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
 	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) # clear back chain
 	lgr	%r15,%r1
+
 #
 # One of the work bits is on. Find out which one.
-# Checked are: _TIF_SIGPENDING, _TIF_RESTORE_SIGPENDING, _TIF_NEED_RESCHED
+# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
 #	       and _TIF_MCCK_PENDING
 #
 io_work_loop:
@@ -669,16 +661,10 @@ io_work_loop:
 	tm	__TI_flags+7(%r9),_TIF_NEED_RESCHED
 	jo	io_reschedule
 	tm	__TI_flags+7(%r9),_TIF_SIGPENDING
-	jnz	io_sigpending
+	jo	io_sigpending
 	tm	__TI_flags+7(%r9),_TIF_NOTIFY_RESUME
-	jnz	io_notify_resume
-	j	io_restore
-io_work_done:
-
-#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
-sie_opcode:
-	.long 0xb2140000
-#endif
+	jo	io_notify_resume
+	j	io_return		# beware of critical section cleanup
 
 #
 # _TIF_MCCK_PENDING is set, call handler
@@ -696,8 +682,6 @@ io_reschedule:
 	brasl	%r14,schedule		# call scheduler
 	stnsm	__SF_EMPTY(%r15),0xfc	# disable I/O and ext. interrupts
 	TRACE_IRQS_OFF
-	tm	__TI_flags+7(%r9),_TIF_WORK_INT
-	jz	io_restore		# there is no work to do
 	j	io_work_loop
 
 #
@@ -903,14 +887,10 @@ cleanup_table_sysc_return:
 	.quad	sysc_return, sysc_leave
 cleanup_table_sysc_leave:
 	.quad	sysc_leave, sysc_done
-cleanup_table_sysc_work_loop:
-	.quad	sysc_work_loop, sysc_work_done
 cleanup_table_io_return:
 	.quad	io_return, io_leave
 cleanup_table_io_leave:
 	.quad	io_leave, io_done
-cleanup_table_io_work_loop:
-	.quad	io_work_loop, io_work_done
 
 cleanup_critical:
 	clc	8(8,%r12),BASED(cleanup_table_system_call)
@@ -928,11 +908,6 @@ cleanup_critical:
 	clc	8(8,%r12),BASED(cleanup_table_sysc_leave+8)
 	jl	cleanup_sysc_leave
 0:
-	clc	8(8,%r12),BASED(cleanup_table_sysc_work_loop)
-	jl	0f
-	clc	8(8,%r12),BASED(cleanup_table_sysc_work_loop+8)
-	jl	cleanup_sysc_return
-0:
 	clc	8(8,%r12),BASED(cleanup_table_io_return)
 	jl	0f
 	clc	8(8,%r12),BASED(cleanup_table_io_return+8)
@@ -943,11 +918,6 @@ cleanup_critical:
 	clc	8(8,%r12),BASED(cleanup_table_io_leave+8)
 	jl	cleanup_io_leave
 0:
-	clc	8(8,%r12),BASED(cleanup_table_io_work_loop)
-	jl	0f
-	clc	8(8,%r12),BASED(cleanup_table_io_work_loop+8)
-	jl	cleanup_io_work_loop
-0:
 	br	%r14
 
 cleanup_system_call:
@@ -1025,12 +995,6 @@ cleanup_io_return:
 	la	%r12,__LC_RETURN_PSW
 	br	%r14
 
-cleanup_io_work_loop:
-	mvc	__LC_RETURN_PSW(8),0(%r12)
-	mvc	__LC_RETURN_PSW+8(8),BASED(cleanup_table_io_work_loop)
-	la	%r12,__LC_RETURN_PSW
-	br	%r14
-
 cleanup_io_leave:
 	clc	8(8,%r12),BASED(cleanup_io_leave_insn)
 	je	3f