lockdep: fixup irq tracing

Ensure we fixup the IRQ state before we hit any locking code.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/kernel/kprobes_32.c b/arch/x86/kernel/kprobes_32.c
index 90f778c04b3f1b847ea6c8653ae2f0c9503a8e39..d87a523070d111d853cf8ca05a0d3dbad4bcf0cd 100644 (file)
--- a/arch/x86/kernel/kprobes_32.c
+++ b/arch/x86/kernel/kprobes_32.c
@@ -564,12 +564,7 @@ static int __kprobes post_kprobe_handler(struct pt_regs *regs)
 
        resume_execution(cur, regs, kcb);
        regs->eflags |= kcb->kprobe_saved_eflags;
-#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
-       if (raw_irqs_disabled_flags(regs->eflags))
-               trace_hardirqs_off();
-       else
-               trace_hardirqs_on();
-#endif
+       trace_hardirqs_fixup_flags(regs->eflags);
 
        /*Restore back the original saved kprobes variables and continue. */
        if (kcb->kprobe_status == KPROBE_REENTER) {

diff --git a/arch/x86/kernel/kprobes_64.c b/arch/x86/kernel/kprobes_64.c
index 681b801c5e26bf7be2a09c24134f590381214258..3db3611933d89971d82a05942fecda6f898a7cb3 100644 (file)
--- a/arch/x86/kernel/kprobes_64.c
+++ b/arch/x86/kernel/kprobes_64.c
@@ -551,12 +551,7 @@ int __kprobes post_kprobe_handler(struct pt_regs *regs)
 
        resume_execution(cur, regs, kcb);
        regs->eflags |= kcb->kprobe_saved_rflags;
-#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
-       if (raw_irqs_disabled_flags(regs->eflags))
-               trace_hardirqs_off();
-       else
-               trace_hardirqs_on();
-#endif
+       trace_hardirqs_fixup_flags(regs->eflags);
 
        /* Restore the original saved kprobes variables and continue. */
        if (kcb->kprobe_status == KPROBE_REENTER) {

diff --git a/arch/x86/kernel/traps_32.c b/arch/x86/kernel/traps_32.c
index cc9acace7e23ba05b810afcf3c9d96bff0516d6c..298d13ed3ab34601cf0d025d827be05c961e7f92 100644 (file)
--- a/arch/x86/kernel/traps_32.c
+++ b/arch/x86/kernel/traps_32.c
@@ -789,6 +789,8 @@ void restart_nmi(void)
 #ifdef CONFIG_KPROBES
 fastcall void __kprobes do_int3(struct pt_regs *regs, long error_code)
 {
+       trace_hardirqs_fixup();
+
        if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
                        == NOTIFY_STOP)
                return;

diff --git a/arch/x86/kernel/traps_64.c b/arch/x86/kernel/traps_64.c
index d0c2bc7ab2ec561dc91e12dee87fde8b563f8770..4a6bd4965f567d39c0fc8c80fb8aa5f095be3a80 100644 (file)
--- a/arch/x86/kernel/traps_64.c
+++ b/arch/x86/kernel/traps_64.c
@@ -807,6 +807,8 @@ asmlinkage __kprobes void default_do_nmi(struct pt_regs *regs)
 /* runs on IST stack. */
 asmlinkage void __kprobes do_int3(struct pt_regs * regs, long error_code)
 {
+       trace_hardirqs_fixup();
+
        if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP) == NOTIFY_STOP) {
                return;
        }

diff --git a/arch/x86/mm/fault_32.c b/arch/x86/mm/fault_32.c
index 503dfc05111b051de74265e2d44ae7de86a80145..2e542e89a3e2fc5d6dd6c7787fd6427e21a475e2 100644 (file)
--- a/arch/x86/mm/fault_32.c
+++ b/arch/x86/mm/fault_32.c
@@ -303,6 +303,11 @@ fastcall void __kprobes do_page_fault(struct pt_regs *regs,
        int write, si_code;
        int fault;
 
+       /*
+        * We can fault from pretty much anywhere, with unknown IRQ state.
+        */
+       trace_hardirqs_fixup();
+
        /* get the address */
         address = read_cr2();
 

diff --git a/arch/x86/mm/fault_64.c b/arch/x86/mm/fault_64.c
index 644b4f7ece10acc498fd9ac61c0d93637f519a82..0e26230669ca21aabb92a60f134100d8f453f5fc 100644 (file)
--- a/arch/x86/mm/fault_64.c
+++ b/arch/x86/mm/fault_64.c
@@ -304,6 +304,11 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
        unsigned long flags;
        siginfo_t info;
 
+       /*
+        * We can fault from pretty much anywhere, with unknown IRQ state.
+        */
+       trace_hardirqs_fixup();
+
        tsk = current;
        mm = tsk->mm;
        prefetchw(&mm->mmap_sem);

diff --git a/include/asm-x86/irqflags_32.h b/include/asm-x86/irqflags_32.h
index d058b04e0083a2e1aa02f59b6ea6c42987188f8a..4c7720089cb5cd68d1d26af4171dcf6f1518226a 100644 (file)
--- a/include/asm-x86/irqflags_32.h
+++ b/include/asm-x86/irqflags_32.h
@@ -129,6 +129,27 @@ static inline int raw_irqs_disabled(void)
 
        return raw_irqs_disabled_flags(flags);
 }
+
+/*
+ * makes the traced hardirq state match with the machine state
+ *
+ * should be a rarely used function, only in places where its
+ * otherwise impossible to know the irq state, like in traps.
+ */
+static inline void trace_hardirqs_fixup_flags(unsigned long flags)
+{
+       if (raw_irqs_disabled_flags(flags))
+               trace_hardirqs_off();
+       else
+               trace_hardirqs_on();
+}
+
+static inline void trace_hardirqs_fixup(void)
+{
+       unsigned long flags = __raw_local_save_flags();
+
+       trace_hardirqs_fixup_flags(flags);
+}
 #endif /* __ASSEMBLY__ */
 
 /*

diff --git a/include/asm-x86/irqflags_64.h b/include/asm-x86/irqflags_64.h
index 5341ea1f815ac1728dd9dab6e06f068ebfa5bdea..bb9163bb29d1d914651ac672e062b5229a31be6d 100644 (file)
--- a/include/asm-x86/irqflags_64.h
+++ b/include/asm-x86/irqflags_64.h
@@ -111,6 +111,26 @@ static inline int raw_irqs_disabled(void)
        return raw_irqs_disabled_flags(flags);
 }
 
+/*
+ * makes the traced hardirq state match with the machine state
+ *
+ * should be a rarely used function, only in places where its
+ * otherwise impossible to know the irq state, like in traps.
+ */
+static inline void trace_hardirqs_fixup_flags(unsigned long flags)
+{
+       if (raw_irqs_disabled_flags(flags))
+               trace_hardirqs_off();
+       else
+               trace_hardirqs_on();
+}
+
+static inline void trace_hardirqs_fixup(void)
+{
+       unsigned long flags = __raw_local_save_flags();
+
+       trace_hardirqs_fixup_flags(flags);
+}
 /*
  * Used in the idle loop; sti takes one instruction cycle
  * to complete: