summaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/s390/kernel/entry.S12
-rw-r--r--arch/s390/kernel/entry64.S9
-rw-r--r--arch/x86/kernel/entry_32.S4
-rw-r--r--arch/x86/kernel/entry_64.S23
-rw-r--r--arch/x86/kernel/kprobes_32.c7
-rw-r--r--arch/x86/kernel/kprobes_64.c7
-rw-r--r--arch/x86/lib/thunk_64.S4
7 files changed, 54 insertions, 12 deletions
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index f3bceb16532..139ca153d5c 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -68,9 +68,15 @@ STACK_SIZE = 1 << STACK_SHIFT
l %r1,BASED(.Ltrace_irq_off)
basr %r14,%r1
.endm
+
+ .macro LOCKDEP_SYS_EXIT
+ l %r1,BASED(.Llockdep_sys_exit)
+ basr %r14,%r1
+ .endm
#else
#define TRACE_IRQS_ON
#define TRACE_IRQS_OFF
+#define LOCKDEP_SYS_EXIT
#endif
/*
@@ -260,6 +266,7 @@ sysc_return:
bno BASED(sysc_leave)
tm __TI_flags+3(%r9),_TIF_WORK_SVC
bnz BASED(sysc_work) # there is work to do (signals etc.)
+ LOCKDEP_SYS_EXIT
sysc_leave:
RESTORE_ALL __LC_RETURN_PSW,1
@@ -283,6 +290,7 @@ sysc_work:
bo BASED(sysc_restart)
tm __TI_flags+3(%r9),_TIF_SINGLE_STEP
bo BASED(sysc_singlestep)
+ LOCKDEP_SYS_EXIT
b BASED(sysc_leave)
#
@@ -572,6 +580,7 @@ io_return:
#endif
tm __TI_flags+3(%r9),_TIF_WORK_INT
bnz BASED(io_work) # there is work to do (signals etc.)
+ LOCKDEP_SYS_EXIT
io_leave:
RESTORE_ALL __LC_RETURN_PSW,0
io_done:
@@ -618,6 +627,7 @@ io_work_loop:
bo BASED(io_reschedule)
tm __TI_flags+3(%r9),(_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)
bnz BASED(io_sigpending)
+ LOCKDEP_SYS_EXIT
b BASED(io_leave)
#
@@ -1040,6 +1050,8 @@ cleanup_io_leave_insn:
.Ltrace_irq_on: .long trace_hardirqs_on
.Ltrace_irq_off:
.long trace_hardirqs_off
+.Llockdep_sys_exit:
+ .long lockdep_sys_exit
#endif
.Lcritical_start:
.long __critical_start + 0x80000000
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 9c0d5cc8269..05e26d1fdf4 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -66,9 +66,14 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK | _TIF_NEED_RESCHED | \
.macro TRACE_IRQS_OFF
brasl %r14,trace_hardirqs_off
.endm
+
+ .macro LOCKDEP_SYS_EXIT
+ brasl %r14,lockdep_sys_exit
+ .endm
#else
#define TRACE_IRQS_ON
#define TRACE_IRQS_OFF
+#define LOCKDEP_SYS_EXIT
#endif
.macro STORE_TIMER lc_offset
@@ -255,6 +260,7 @@ sysc_return:
jno sysc_leave
tm __TI_flags+7(%r9),_TIF_WORK_SVC
jnz sysc_work # there is work to do (signals etc.)
+ LOCKDEP_SYS_EXIT
sysc_leave:
RESTORE_ALL __LC_RETURN_PSW,1
@@ -278,6 +284,7 @@ sysc_work:
jo sysc_restart
tm __TI_flags+7(%r9),_TIF_SINGLE_STEP
jo sysc_singlestep
+ LOCKDEP_SYS_EXIT
j sysc_leave
#
@@ -558,6 +565,7 @@ io_return:
#endif
tm __TI_flags+7(%r9),_TIF_WORK_INT
jnz io_work # there is work to do (signals etc.)
+ LOCKDEP_SYS_EXIT
io_leave:
RESTORE_ALL __LC_RETURN_PSW,0
io_done:
@@ -605,6 +613,7 @@ io_work_loop:
jo io_reschedule
tm __TI_flags+7(%r9),(_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)
jnz io_sigpending
+ LOCKDEP_SYS_EXIT
j io_leave
#
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 290b7bc82da..8099fea0a72 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -251,6 +251,7 @@ check_userspace:
jb resume_kernel # not returning to v8086 or userspace
ENTRY(resume_userspace)
+ LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
# setting need_resched or sigpending
# between sampling and the iret
@@ -338,6 +339,7 @@ sysenter_past_esp:
jae syscall_badsys
call *sys_call_table(,%eax,4)
movl %eax,PT_EAX(%esp)
+ LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY)
TRACE_IRQS_OFF
movl TI_flags(%ebp), %ecx
@@ -377,6 +379,7 @@ syscall_call:
call *sys_call_table(,%eax,4)
movl %eax,PT_EAX(%esp) # store the return value
syscall_exit:
+ LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
# setting need_resched or sigpending
# between sampling and the iret
@@ -467,6 +470,7 @@ work_pending:
jz work_notifysig
work_resched:
call schedule
+ LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
# setting need_resched or sigpending
# between sampling and the iret
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1d232e5f565..f1cacd4897f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -244,6 +244,7 @@ ret_from_sys_call:
movl $_TIF_ALLWORK_MASK,%edi
/* edi: flagmask */
sysret_check:
+ LOCKDEP_SYS_EXIT
GET_THREAD_INFO(%rcx)
cli
TRACE_IRQS_OFF
@@ -333,6 +334,7 @@ int_ret_from_sys_call:
movl $_TIF_ALLWORK_MASK,%edi
/* edi: mask to check */
int_with_check:
+ LOCKDEP_SYS_EXIT_IRQ
GET_THREAD_INFO(%rcx)
movl threadinfo_flags(%rcx),%edx
andl %edi,%edx
@@ -544,11 +546,13 @@ exit_intr:
retint_with_reschedule:
movl $_TIF_WORK_MASK,%edi
retint_check:
+ LOCKDEP_SYS_EXIT_IRQ
movl threadinfo_flags(%rcx),%edx
andl %edi,%edx
CFI_REMEMBER_STATE
jnz retint_careful
-retint_swapgs:
+
+retint_swapgs: /* return to user-space */
/*
* The iretq could re-enable interrupts:
*/
@@ -557,7 +561,7 @@ retint_swapgs:
swapgs
jmp restore_args
-retint_restore_args:
+retint_restore_args: /* return to kernel space */
cli
/*
* The iretq could re-enable interrupts:
@@ -866,26 +870,21 @@ error_sti:
movq ORIG_RAX(%rsp),%rsi /* get error code */
movq $-1,ORIG_RAX(%rsp)
call *%rax
- /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
-error_exit:
- movl %ebx,%eax
+ /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
+error_exit:
+ movl %ebx,%eax
RESTORE_REST
cli
TRACE_IRQS_OFF
GET_THREAD_INFO(%rcx)
testl %eax,%eax
jne retint_kernel
+ LOCKDEP_SYS_EXIT_IRQ
movl threadinfo_flags(%rcx),%edx
movl $_TIF_WORK_MASK,%edi
andl %edi,%edx
jnz retint_careful
- /*
- * The iret might restore flags:
- */
- TRACE_IRQS_IRETQ
- swapgs
- RESTORE_ARGS 0,8,0
- jmp iret_label
+ jmp retint_swapgs
CFI_ENDPROC
error_kernelspace:
diff --git a/arch/x86/kernel/kprobes_32.c b/arch/x86/kernel/kprobes_32.c
index c2d03e96ae9..e7d0d3c2ef6 100644
--- a/arch/x86/kernel/kprobes_32.c
+++ b/arch/x86/kernel/kprobes_32.c
@@ -557,6 +557,12 @@ static int __kprobes post_kprobe_handler(struct pt_regs *regs)
resume_execution(cur, regs, kcb);
regs->eflags |= kcb->kprobe_saved_eflags;
+#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
+ if (raw_irqs_disabled_flags(regs->eflags))
+ trace_hardirqs_off();
+ else
+ trace_hardirqs_on();
+#endif
/*Restore back the original saved kprobes variables and continue. */
if (kcb->kprobe_status == KPROBE_REENTER) {
@@ -694,6 +700,7 @@ int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
MIN_STACK_SIZE(addr));
regs->eflags &= ~IF_MASK;
+ trace_hardirqs_off();
regs->eip = (unsigned long)(jp->entry);
return 1;
}
diff --git a/arch/x86/kernel/kprobes_64.c b/arch/x86/kernel/kprobes_64.c
index 1df17a0ec0c..62e28e52d78 100644
--- a/arch/x86/kernel/kprobes_64.c
+++ b/arch/x86/kernel/kprobes_64.c
@@ -544,6 +544,12 @@ int __kprobes post_kprobe_handler(struct pt_regs *regs)
resume_execution(cur, regs, kcb);
regs->eflags |= kcb->kprobe_saved_rflags;
+#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
+ if (raw_irqs_disabled_flags(regs->eflags))
+ trace_hardirqs_off();
+ else
+ trace_hardirqs_on();
+#endif
/* Restore the original saved kprobes variables and continue. */
if (kcb->kprobe_status == KPROBE_REENTER) {
@@ -684,6 +690,7 @@ int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
MIN_STACK_SIZE(addr));
regs->eflags &= ~IF_MASK;
+ trace_hardirqs_off();
regs->rip = (unsigned long)(jp->entry);
return 1;
}
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index 55e586d352d..6ea73f3de56 100644
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -50,6 +50,10 @@
thunk trace_hardirqs_on_thunk,trace_hardirqs_on
thunk trace_hardirqs_off_thunk,trace_hardirqs_off
#endif
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+ thunk lockdep_sys_exit_thunk,lockdep_sys_exit
+#endif
/* SAVE_ARGS below is used only for the .cfi directives it contains. */
CFI_STARTPROC
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-11-27 19:55:07 +0000