lockdep: x86_64: connect the sysexit hook

Run the lockdep_sys_exit hook after all other C code on the syscall
return path.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1d232e5f5658e889ef8642fc98301b3f9527fbe1..f1cacd4897f74892eea1e8c4399b1891c6b523ad 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -244,6 +244,7 @@ ret_from_sys_call:
        movl $_TIF_ALLWORK_MASK,%edi
        /* edi: flagmask */
 sysret_check:          
+       LOCKDEP_SYS_EXIT
        GET_THREAD_INFO(%rcx)
        cli
        TRACE_IRQS_OFF
@@ -333,6 +334,7 @@ int_ret_from_sys_call:
        movl $_TIF_ALLWORK_MASK,%edi
        /* edi: mask to check */
 int_with_check:
+       LOCKDEP_SYS_EXIT_IRQ
        GET_THREAD_INFO(%rcx)
        movl threadinfo_flags(%rcx),%edx
        andl %edi,%edx
@@ -544,11 +546,13 @@ exit_intr:
 retint_with_reschedule:
        movl $_TIF_WORK_MASK,%edi
 retint_check:
+       LOCKDEP_SYS_EXIT_IRQ
        movl threadinfo_flags(%rcx),%edx
        andl %edi,%edx
        CFI_REMEMBER_STATE
        jnz  retint_careful
-retint_swapgs:         
+
+retint_swapgs:         /* return to user-space */
        /*
         * The iretq could re-enable interrupts:
         */
@@ -557,7 +561,7 @@ retint_swapgs:
        swapgs 
        jmp restore_args
 
-retint_restore_args:                           
+retint_restore_args:   /* return to kernel space */
        cli
        /*
         * The iretq could re-enable interrupts:
@@ -866,26 +870,21 @@ error_sti:
        movq ORIG_RAX(%rsp),%rsi        /* get error code */ 
        movq $-1,ORIG_RAX(%rsp)
        call *%rax
-       /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */     
-error_exit:            
-       movl %ebx,%eax          
+       /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
+error_exit:
+       movl %ebx,%eax
        RESTORE_REST
        cli
        TRACE_IRQS_OFF
        GET_THREAD_INFO(%rcx)   
        testl %eax,%eax
        jne  retint_kernel
+       LOCKDEP_SYS_EXIT_IRQ
        movl  threadinfo_flags(%rcx),%edx
        movl  $_TIF_WORK_MASK,%edi
        andl  %edi,%edx
        jnz  retint_careful
-       /*
-        * The iret might restore flags:
-        */
-       TRACE_IRQS_IRETQ
-       swapgs 
-       RESTORE_ARGS 0,8,0                                              
-       jmp iret_label
+       jmp retint_swapgs
        CFI_ENDPROC
 
 error_kernelspace:

diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index 55e586d352d3c444d35fe7399726666ba94b9adf..6ea73f3de5677320c835234b70613413e746db35 100644 (file)
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -50,6 +50,10 @@
        thunk trace_hardirqs_on_thunk,trace_hardirqs_on
        thunk trace_hardirqs_off_thunk,trace_hardirqs_off
 #endif
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+       thunk lockdep_sys_exit_thunk,lockdep_sys_exit
+#endif
        
        /* SAVE_ARGS below is used only for the .cfi directives it contains. */
        CFI_STARTPROC

diff --git a/include/asm-x86/irqflags_64.h b/include/asm-x86/irqflags_64.h
index 86e70fe23659bded0499fc85b0bb359aa7e3387c..5341ea1f815ac1728dd9dab6e06f068ebfa5bdea 100644 (file)
--- a/include/asm-x86/irqflags_64.h
+++ b/include/asm-x86/irqflags_64.h
@@ -137,6 +137,20 @@ static inline void halt(void)
 #  define TRACE_IRQS_ON
 #  define TRACE_IRQS_OFF
 # endif
+# ifdef CONFIG_DEBUG_LOCK_ALLOC
+#  define LOCKDEP_SYS_EXIT     call lockdep_sys_exit_thunk
+#  define LOCKDEP_SYS_EXIT_IRQ \
+       TRACE_IRQS_ON; \
+       sti; \
+       SAVE_REST; \
+       LOCKDEP_SYS_EXIT; \
+       RESTORE_REST; \
+       cli; \
+       TRACE_IRQS_OFF;
+# else
+#  define LOCKDEP_SYS_EXIT
+#  define LOCKDEP_SYS_EXIT_IRQ
+# endif
 #endif
 
 #endif